├── .coveragerc ├── .docbadge.svg ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── feature_request.md │ └── new-unit-request.md ├── actions │ ├── setup │ │ └── action.yml │ └── test │ │ └── action.yml └── workflows │ ├── docs.yml │ ├── preview.yml │ ├── publish.yml │ └── test.yml ├── .gitignore ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE.md ├── MANIFEST.in ├── README.md ├── SECURITY.md ├── codecov.yml ├── pdoc3-template ├── FixedSysEx.ttf ├── config.mako ├── credits.mako ├── css.mako ├── head.mako ├── html.mako ├── logo.mako ├── pdf.mako └── text.mako ├── pyproject.toml ├── refinery ├── __init__.py ├── data │ ├── __init__.py │ └── rich.json ├── explore.py ├── lib │ ├── __init__.py │ ├── argformats.py │ ├── argparser.py │ ├── array.py │ ├── cab.py │ ├── chunks.py │ ├── crypto.py │ ├── decompression.py │ ├── decorators.py │ ├── deobfuscation.py │ ├── dex.py │ ├── dotnet │ │ ├── __init__.py │ │ ├── deserialize.py │ │ ├── disassembler │ │ │ ├── __init__.py │ │ │ ├── factory.py │ │ │ ├── model.py │ │ │ └── repository.py │ │ ├── header.py │ │ ├── resources.py │ │ └── types.py │ ├── emulator.py │ ├── environment.py │ ├── executable.py │ ├── frame.py │ ├── id.py │ ├── inline.py │ ├── inno │ │ ├── __init__.py │ │ ├── archive.py │ │ ├── emulator.py │ │ ├── ifps.py │ │ └── symbols.py │ ├── intervals.py │ ├── java.py │ ├── json.py │ ├── lcid.py │ ├── lief.py │ ├── loader.py │ ├── lzx.py │ ├── magic.py │ ├── maru.py │ ├── meta.py │ ├── mime.py │ ├── mscrypto.py │ ├── murmur.py │ ├── patterns │ │ ├── __init__.py │ │ └── tlds.py │ ├── powershell.py │ ├── resources.py │ ├── ripemd128.py │ ├── serpent.py │ ├── speck.py │ ├── structures.py │ ├── suffixtree.py │ ├── thirdparty │ │ ├── __init__.py │ │ ├── acefile.py │ │ ├── batch_interpreter.py │ │ ├── pcode2code.py │ │ ├── pyflate.py │ │ └── xxhash.py │ ├── tools.py │ ├── types.py │ ├── vfs.py │ ├── winclip.py │ └── xml.py ├── shell.py └── units │ ├── __init__.py │ ├── blockwise │ ├── __init__.py │ ├── add.py │ ├── alu.py │ ├── bitrev.py │ ├── bitsnip.py │ ├── byteswap.py │ ├── map.py │ ├── neg.py │ ├── pack.py │ ├── rev.py │ ├── rotl.py │ ├── rotr.py │ ├── shl.py │ ├── shr.py │ ├── sub.py │ ├── terminate.py │ └── xor.py │ ├── compression │ ├── __init__.py │ ├── ap.py │ ├── blz.py │ ├── brotli.py │ ├── bz2.py │ ├── decompress.py │ ├── jcalg.py │ ├── lz.py │ ├── lz4.py │ ├── lzf.py │ ├── lzg.py │ ├── lzip.py │ ├── lzjb.py │ ├── lznt1.py │ ├── lzo.py │ ├── lzw.py │ ├── lzx.py │ ├── mscf.py │ ├── nrv.py │ ├── qlz.py │ ├── szdd.py │ ├── zl.py │ └── zstd.py │ ├── crypto │ ├── __init__.py │ ├── cipher │ │ ├── __init__.py │ │ ├── aes.py │ │ ├── blabla.py │ │ ├── blowfish.py │ │ ├── camellia.py │ │ ├── cast.py │ │ ├── chacha.py │ │ ├── chaskey.py │ │ ├── des.py │ │ ├── des3.py │ │ ├── fernet.py │ │ ├── gost.py │ │ ├── hc128.py │ │ ├── hc256.py │ │ ├── isaac.py │ │ ├── rabbit.py │ │ ├── rc2.py │ │ ├── rc4.py │ │ ├── rc4mod.py │ │ ├── rc5.py │ │ ├── rc6.py │ │ ├── rijndael.py │ │ ├── rncrypt.py │ │ ├── rot.py │ │ ├── rsa.py │ │ ├── rsakey.py │ │ ├── salsa.py │ │ ├── seal.py │ │ ├── secstr.py │ │ ├── serpent.py │ │ ├── sm4.py │ │ ├── sosemanuk.py │ │ ├── speck.py │ │ ├── tea.py │ │ ├── vigenere.py │ │ ├── xtea.py │ │ └── xxtea.py │ ├── hash │ │ ├── __init__.py │ │ ├── checksums.py │ │ ├── cryptographic.py │ │ ├── imphash.py │ │ ├── maru.py │ │ ├── murmur.py │ │ ├── password_hashes.py │ │ └── xxhash.py │ └── keyderive │ │ ├── __init__.py │ │ ├── deskd.py │ │ ├── hkdf.py │ │ ├── hmac.py │ │ ├── kblob.py │ │ ├── mscdk.py │ │ ├── mspdb.py │ │ ├── pbkdf1.py │ │ ├── pbkdf2.py │ │ └── unixcrypt.py │ ├── encoding │ ├── __init__.py │ ├── a85.py │ ├── atbash.py │ ├── b32.py │ ├── b58.py │ ├── b62.py │ ├── b64.py │ ├── b65536.py │ ├── b85.py │ ├── b92.py │ ├── base.py │ ├── cp1252.py │ ├── esc.py │ ├── escps.py │ ├── escvb.py │ ├── hex.py │ ├── htmlesc.py │ ├── morse.py │ ├── netbios.py │ ├── recode.py │ ├── u16.py │ ├── url.py │ ├── uuenc.py │ └── wshenc.py │ ├── formats │ ├── __init__.py │ ├── a3x.py │ ├── archive │ │ ├── __init__.py │ │ ├── innopwd.py │ │ ├── xt.py │ │ ├── xt7z.py │ │ ├── xtace.py │ │ ├── xtasar.py │ │ ├── xtcab.py │ │ ├── xtcpio.py │ │ ├── xtgz.py │ │ ├── xtinno.py │ │ ├── xtiso.py │ │ ├── xtiss.py │ │ ├── xtmacho.py │ │ ├── xtmagtape.py │ │ ├── xtnode.py │ │ ├── xtnsis.py │ │ ├── xtnuitka.py │ │ ├── xtpyi.py │ │ ├── xtsim.py │ │ ├── xtsql.py │ │ ├── xttar.py │ │ ├── xtzip.py │ │ └── xtzpaq.py │ ├── bat.py │ ├── csv.py │ ├── deserialize_php.py │ ├── dexstr.py │ ├── email.py │ ├── evtx.py │ ├── exe │ │ ├── __init__.py │ │ ├── opc.py │ │ ├── vaddr.py │ │ ├── vmemref.py │ │ ├── vsect.py │ │ ├── vsnip.py │ │ └── vstack.py │ ├── hexload.py │ ├── html.py │ ├── httprequest.py │ ├── httpresponse.py │ ├── ifps.py │ ├── ifpsstr.py │ ├── imgdb.py │ ├── imgtp.py │ ├── java │ │ ├── __init__.py │ │ ├── deserialize.py │ │ ├── jvdasm.py │ │ └── jvstr.py │ ├── json.py │ ├── lnk.py │ ├── macho │ │ ├── __init__.py │ │ └── machometa.py │ ├── msgpack.py │ ├── msi.py │ ├── office │ │ ├── __init__.py │ │ ├── docmeta.py │ │ ├── doctxt.py │ │ ├── officecrypt.py │ │ ├── rtfc.py │ │ ├── vbapc.py │ │ ├── vbastr.py │ │ ├── xlmdeobf.py │ │ ├── xlxtr.py │ │ ├── xtdoc.py │ │ ├── xtone.py │ │ ├── xtrtf.py │ │ ├── xtvba.py │ │ └── xtxs.py │ ├── pcap.py │ ├── pcap_http.py │ ├── pdf.py │ ├── pe │ │ ├── __init__.py │ │ ├── dotnet │ │ │ ├── __init__.py │ │ │ ├── dnarrays.py │ │ │ ├── dnblob.py │ │ │ ├── dnds.py │ │ │ ├── dnfields.py │ │ │ ├── dnhdr.py │ │ │ ├── dnmr.py │ │ │ ├── dnopc.py │ │ │ ├── dnrc.py │ │ │ ├── dnsfx.py │ │ │ └── dnstr.py │ │ ├── pedebloat.py │ │ ├── pemeta.py │ │ ├── peoverlay.py │ │ ├── perc.py │ │ ├── pesig.py │ │ └── pestrip.py │ ├── pkcs7.py │ ├── pkcs7sig.py │ ├── pyc.py │ ├── pym.py │ ├── stego.py │ ├── winreg.py │ └── xml.py │ ├── malware │ ├── __init__.py │ ├── kramer.py │ └── n40.py │ ├── meta │ ├── __init__.py │ ├── chop.py │ ├── cm.py │ ├── dedup.py │ ├── eat.py │ ├── ef.py │ ├── emit.py │ ├── group.py │ ├── groupby.py │ ├── iff.py │ ├── iffc.py │ ├── iffp.py │ ├── iffs.py │ ├── iffx.py │ ├── jamv.py │ ├── loop.py │ ├── max.py │ ├── min.py │ ├── mvg.py │ ├── pad.py │ ├── pick.py │ ├── pop.py │ ├── push.py │ ├── put.py │ ├── queue.py │ ├── reduce.py │ ├── rmv.py │ ├── scope.py │ ├── sep.py │ ├── sorted.py │ ├── swap.py │ ├── transpose.py │ ├── urn.py │ └── xfcc.py │ ├── misc │ ├── __init__.py │ ├── autoxor.py │ ├── datefix.py │ ├── drp.py │ ├── nop.py │ ├── run.py │ ├── urlfix.py │ └── xkey.py │ ├── obfuscation │ ├── __init__.py │ ├── js │ │ ├── __init__.py │ │ ├── arithmetic.py │ │ ├── arrays.py │ │ ├── comments.py │ │ ├── concat.py │ │ ├── getattr.py │ │ └── tuples.py │ ├── ps1 │ │ ├── __init__.py │ │ ├── all.py │ │ ├── b64convert.py │ │ ├── brackets.py │ │ ├── cases.py │ │ ├── concat.py │ │ ├── encodings.py │ │ ├── escape.py │ │ ├── format.py │ │ ├── invoke.py │ │ ├── securestring.py │ │ ├── stringreplace.py │ │ ├── typecast.py │ │ └── uncurly.py │ └── vba │ │ ├── __init__.py │ │ ├── all.py │ │ ├── arithmetic.py │ │ ├── brackets.py │ │ ├── char.py │ │ ├── comments.py │ │ ├── concat.py │ │ ├── constants.py │ │ ├── dummies.py │ │ ├── stringreplace.py │ │ ├── stringreverse.py │ │ └── vba.py │ ├── pattern │ ├── __init__.py │ ├── carve.py │ ├── carve_7z.py │ ├── carve_json.py │ ├── carve_lnk.py │ ├── carve_pe.py │ ├── carve_rtf.py │ ├── carve_xml.py │ ├── carve_zip.py │ ├── defang.py │ ├── dnsdomain.py │ ├── mimewords.py │ ├── resplit.py │ ├── resub.py │ ├── rex.py │ ├── struct_parser.py │ ├── subfiles.py │ ├── urlguards.py │ ├── xtp.py │ └── xtw.py │ ├── sinks │ ├── __init__.py │ ├── asm.py │ ├── dnasm.py │ ├── dump.py │ ├── iemap.py │ ├── peek.py │ ├── ppjscript.py │ ├── ppjson.py │ └── ppxml.py │ └── strings │ ├── __init__.py │ ├── bruteforce.py │ ├── cca.py │ ├── ccp.py │ ├── cfmt.py │ ├── clower.py │ ├── cswap.py │ ├── cupper.py │ ├── ngrams.py │ ├── rep.py │ ├── repl.py │ ├── snip.py │ ├── stretch.py │ ├── termfit.py │ └── trim.py ├── run-crawl.py ├── run-flake.py ├── run-ldeps.py ├── run-pdoc3.py ├── run-tests.py ├── samples └── __init__.py ├── setup.cfg ├── setup.py ├── shells ├── README.md └── zsh ├── strip-tutorials.py ├── test ├── __init__.py ├── lib │ ├── __init__.py │ ├── test_argformats.py │ ├── test_chunks.py │ ├── test_dotnet.py │ ├── test_dotnet_disassembler.py │ ├── test_frame.py │ ├── test_ifps.py │ ├── test_inline.py │ ├── test_interval.py │ ├── test_loader.py │ ├── test_meta.py │ ├── test_mime.py │ ├── test_multibin.py │ ├── test_patterns.py │ ├── test_powershell.py │ ├── test_structures.py │ ├── test_suffixtree.py │ ├── test_tools.py │ ├── test_vfs.py │ └── test_winclip.py └── units │ ├── __init__.py │ ├── blockwise │ ├── __init__.py │ ├── test_add.py │ ├── test_alu.py │ ├── test_arithmetic_units.py │ ├── test_bitrev.py │ ├── test_bitsnip.py │ ├── test_byteswap.py │ ├── test_map.py │ ├── test_neg.py │ ├── test_pack.py │ ├── test_rev.py │ ├── test_rotl.py │ ├── test_sub.py │ ├── test_terminate.py │ └── test_xor.py │ ├── compression │ ├── __init__.py │ ├── test_ap.py │ ├── test_blz.py │ ├── test_brotli.py │ ├── test_decompress.py │ ├── test_jcalg.py │ ├── test_lz.py │ ├── test_lz4.py │ ├── test_lzf.py │ ├── test_lzg.py │ ├── test_lzip.py │ ├── test_lzo.py │ ├── test_lzw.py │ ├── test_lzx.py │ ├── test_mscf.py │ ├── test_nrv.py │ ├── test_qlz.py │ └── test_szdd.py │ ├── crypto │ ├── __init__.py │ ├── cipher │ │ ├── __init__.py │ │ ├── test_aes.py │ │ ├── test_blabla.py │ │ ├── test_camellia.py │ │ ├── test_chacha.py │ │ ├── test_chaskey.py │ │ ├── test_custom_cipher_modes.py │ │ ├── test_fernet.py │ │ ├── test_gost.py │ │ ├── test_hc256.py │ │ ├── test_isaac.py │ │ ├── test_latin_ciphers.py │ │ ├── test_rabbit.py │ │ ├── test_rc2.py │ │ ├── test_rc4.py │ │ ├── test_rc4mod.py │ │ ├── test_rc5.py │ │ ├── test_rc6.py │ │ ├── test_rijndael.py │ │ ├── test_rot.py │ │ ├── test_rsa.py │ │ ├── test_rsakey.py │ │ ├── test_salsa.py │ │ ├── test_serpent.py │ │ ├── test_sm4.py │ │ ├── test_sosemanuk.py │ │ ├── test_speck.py │ │ ├── test_tea.py │ │ ├── test_vigenere.py │ │ ├── test_xtea.py │ │ └── test_xxtea.py │ ├── hash │ │ ├── __init__.py │ │ ├── test_checksums.py │ │ ├── test_cryptographic.py │ │ ├── test_imphash.py │ │ ├── test_maru.py │ │ ├── test_murmur.py │ │ ├── test_password_hashes.py │ │ └── test_xxhash.py │ ├── keyderive │ │ ├── __init__.py │ │ ├── test_deskd.py │ │ ├── test_hkdf.py │ │ ├── test_kblob.py │ │ ├── test_mscdk.py │ │ ├── test_mspdb.py │ │ ├── test_pbkdf1.py │ │ ├── test_pbkdf2.py │ │ └── test_unixcrypt.py │ └── test_cipher.py │ ├── encoding │ ├── __init__.py │ ├── test_a85.py │ ├── test_atbash.py │ ├── test_b32.py │ ├── test_b58.py │ ├── test_b62.py │ ├── test_b64.py │ ├── test_b65536.py │ ├── test_b85.py │ ├── test_b92.py │ ├── test_base.py │ ├── test_esc.py │ ├── test_escps.py │ ├── test_escvb.py │ ├── test_morse.py │ ├── test_netbios.py │ ├── test_recode.py │ ├── test_url.py │ ├── test_uuenc.py │ └── test_wshenc.py │ ├── formats │ ├── __init__.py │ ├── archive │ │ ├── __init__.py │ │ ├── test_innopwd.py │ │ ├── test_xt7z.py │ │ ├── test_xtace.py │ │ ├── test_xtcab.py │ │ ├── test_xtcpio.py │ │ ├── test_xtinno.py │ │ ├── test_xtiso.py │ │ ├── test_xtiss.py │ │ ├── test_xtnode.py │ │ ├── test_xtnsis.py │ │ ├── test_xtnuitka.py │ │ ├── test_xtpyi.py │ │ ├── test_xtsim.py │ │ ├── test_xttar.py │ │ ├── test_xtzip.py │ │ └── test_xtzpaq.py │ ├── exe │ │ ├── __init__.py │ │ ├── test_vaddr.py │ │ ├── test_vmemref.py │ │ ├── test_vsect.py │ │ ├── test_vsnip.py │ │ └── test_vstack.py │ ├── java │ │ ├── __init__.py │ │ ├── test_deserialize.py │ │ ├── test_jvdasm.py │ │ └── test_jvstr.py │ ├── macho │ │ ├── __init__.py │ │ └── test_machometa.py │ ├── office │ │ ├── __init__.py │ │ ├── test_doctxt.py │ │ ├── test_officecrypt.py │ │ ├── test_vbapc.py │ │ ├── test_vbastr.py │ │ ├── test_xlmdeobf.py │ │ ├── test_xlxtr.py │ │ ├── test_xtdoc.py │ │ ├── test_xtone.py │ │ ├── test_xtrtf.py │ │ └── test_xtvba.py │ ├── pe │ │ ├── __init__.py │ │ ├── dotnet │ │ │ ├── __init__.py │ │ │ ├── test_dnds.py │ │ │ ├── test_dnfields.py │ │ │ ├── test_dnhdr.py │ │ │ ├── test_dnmr.py │ │ │ ├── test_dnrc.py │ │ │ ├── test_dnsfx.py │ │ │ └── test_dnstr.py │ │ ├── test_pedebloat.py │ │ ├── test_pemeta.py │ │ └── test_perc.py │ ├── test_a3x.py │ ├── test_bat.py │ ├── test_csv.py │ ├── test_deserialize_php.py │ ├── test_dexstr.py │ ├── test_email.py │ ├── test_hexload.py │ ├── test_html.py │ ├── test_httpresponse.py │ ├── test_ifps.py │ ├── test_json.py │ ├── test_lnk.py │ ├── test_msgpack.py │ ├── test_msi.py │ ├── test_pcap.py │ ├── test_pdf.py │ ├── test_pkcs7.py │ ├── test_pym.py │ ├── test_stego.py │ ├── test_winreg.py │ └── test_xml.py │ ├── malware │ ├── __init__.py │ └── test_kramer.py │ ├── meta │ ├── __init__.py │ ├── test_chop.py │ ├── test_cm.py │ ├── test_dedup.py │ ├── test_eat.py │ ├── test_ef.py │ ├── test_emit.py │ ├── test_group.py │ ├── test_iff.py │ ├── test_iffp.py │ ├── test_iffs.py │ ├── test_iffx.py │ ├── test_loop.py │ ├── test_max_min.py │ ├── test_mvg.py │ ├── test_pad.py │ ├── test_pick.py │ ├── test_pop.py │ ├── test_push.py │ ├── test_put.py │ ├── test_queue.py │ ├── test_reduce.py │ ├── test_rmv.py │ ├── test_scope.py │ ├── test_sep.py │ ├── test_sorted.py │ ├── test_swap.py │ ├── test_transpose.py │ ├── test_urn.py │ └── test_xfcc.py │ ├── misc │ ├── __init__.py │ ├── rest_run.py │ ├── test_autoxor.py │ ├── test_datefix.py │ ├── test_drp.py │ ├── test_nop.py │ ├── test_urlfix.py │ └── test_xkey.py │ ├── obfuscation │ ├── __init__.py │ ├── ps1 │ │ ├── __init__.py │ │ ├── test_all.py │ │ ├── test_brackets.py │ │ ├── test_cases.py │ │ ├── test_concat.py │ │ ├── test_format.py │ │ ├── test_stringreplace.py │ │ └── test_typecast.py │ └── vba │ │ ├── __init__.py │ │ ├── test_all.py │ │ ├── test_arithmetic.py │ │ ├── test_comments.py │ │ ├── test_constants.py │ │ ├── test_dummies.py │ │ └── test_stringreplace.py │ ├── pattern │ ├── __init__.py │ ├── test_carve.py │ ├── test_carve_7z.py │ ├── test_carve_json.py │ ├── test_carve_lnk.py │ ├── test_carve_pe.py │ ├── test_carve_xml.py │ ├── test_carve_zip.py │ ├── test_defang.py │ ├── test_dnsdomain.py │ ├── test_mimewords.py │ ├── test_resplit.py │ ├── test_resub.py │ ├── test_rex.py │ ├── test_struct.py │ ├── test_urlguards.py │ └── test_xtp.py │ ├── sinks │ ├── __init__.py │ ├── test_asm.py │ ├── test_dnasm.py │ ├── test_dump.py │ ├── test_iemap.py │ ├── test_peek.py │ ├── test_ppjson.py │ └── test_ppxml.py │ ├── strings │ ├── __init__.py │ ├── test_bruteforce.py │ ├── test_cfmt.py │ ├── test_clower.py │ ├── test_concat.py │ ├── test_cswap.py │ ├── test_cupper.py │ ├── test_ngrams.py │ ├── test_rep.py │ ├── test_repl.py │ ├── test_snip.py │ ├── test_stretch.py │ └── test_trim.py │ ├── test_grabbag.py │ └── test_misc.py ├── tutorials ├── README.md ├── boilerplate.py └── notebooks │ ├── tbr-files.v0x01.netwalker.dropper.ipynb │ ├── tbr-files.v0x02.amadey.loader.ipynb │ ├── tbr-files.v0x03.seduploader.ipynb │ ├── tbr-files.v0x04.run.length.encoding.ipynb │ ├── tbr-files.v0x05.flare.on.9.ipynb │ ├── tbr-files.v0x06.qakbot.decoder.ipynb │ ├── tbr-files.v0x07.dc.rat.ipynb │ ├── tbr-files.v0x08.flare.on.10.ipynb │ └── tbr-files.v0x09.exploit.document.ipynb ├── update.ps1 └── update.sh /.coveragerc: -------------------------------------------------------------------------------- 1 | [report] 2 | exclude_also = 3 | if TYPE_CHECKING: 4 | except ImportError 5 | except KeyboardInterrupt 6 | except ArgparseError 7 | except Exception 8 | raise NotImplementedError 9 | yappi -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Report a bug in the binary refinery. 4 | title: '' 5 | labels: bug 6 | assignees: huettenhain 7 | 8 | --- 9 | 10 | ### Description 11 | A clear and concise description of what the bug is. 12 | 13 | ### To Reproduce 14 | Steps to reproduce the bug. 15 | 16 | ### Environment 17 | - Operating System 18 | - Python Version 19 | - Refinery Version 20 | 21 | ### Additional Context 22 | Add any other context about the problem here. 23 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest a new feature that isn't a unit 4 | title: '' 5 | labels: feature 6 | assignees: huettenhain 7 | 8 | --- 9 | 10 | 11 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/new-unit-request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: New unit request 3 | about: Suggest a new unit 4 | title: '' 5 | labels: new-unit 6 | assignees: huettenhain 7 | 8 | --- 9 | 10 | ### Specification 11 | Give a concise description of the algorithm or data transformation that the unit should implement. 12 | 13 | ### Test Cases 14 | Provide test data. Real-world examples (i.e. from in-the-wild malware samples) are preferred. 15 | -------------------------------------------------------------------------------- /.github/actions/setup/action.yml: -------------------------------------------------------------------------------- 1 | name: "Python Setup" 2 | description: "Python Setup and Dependencies" 3 | inputs: 4 | py: 5 | required: true 6 | description: "Target Python Version" 7 | os: 8 | required: true 9 | description: "Target Operating System" 10 | runs: 11 | using: "composite" 12 | steps: 13 | - name: Set up Python ${{ inputs.py }} 14 | uses: actions/setup-python@v5 15 | with: 16 | python-version: ${{ inputs.py }} 17 | - name: Install Dependencies (Linux) 18 | if: ${{ contains(inputs.os, 'ubuntu') }} 19 | shell: ${{ contains(inputs.os, 'windows') && 'cmd' || 'bash' }} 20 | run: sudo apt-get install xclip xvfb 21 | - name: Install Dependencies (MacOS) 22 | if: ${{ contains(inputs.os, 'macos') }} 23 | shell: bash 24 | run: brew install libmagic 25 | - name: Install Dependencies 26 | shell: ${{ contains(inputs.os, 'windows') && 'cmd' || 'bash' }} 27 | run: | 28 | python -m pip install --upgrade pip 29 | python -m pip install pycodestyle pyflakes 30 | pip install .[all] 31 | -------------------------------------------------------------------------------- /.github/actions/test/action.yml: -------------------------------------------------------------------------------- 1 | name: "Python Test" 2 | description: "Python Test Template" 3 | inputs: 4 | py: 5 | required: true 6 | description: "Target Python Version" 7 | os: 8 | required: true 9 | description: "Target Operating System" 10 | runs: 11 | using: "composite" 12 | steps: 13 | - name: Unit Tests 14 | if: ${{ !contains(inputs.os, 'ubuntu') }} 15 | shell: ${{ contains(inputs.os, 'windows') && 'cmd' || 'bash' }} 16 | run: python -m unittest discover -p test_*.py 17 | - name: Unit Tests 18 | if: ${{ contains(inputs.os, 'ubuntu') }} 19 | shell: bash 20 | run: XDG_SESSION_TYPE=x11 xvfb-run python -m unittest discover -p test_*.py 21 | -------------------------------------------------------------------------------- /.github/workflows/docs.yml: -------------------------------------------------------------------------------- 1 | name: documentation 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | 7 | jobs: 8 | docs: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Checkout Main Repository 12 | uses: actions/checkout@v4 13 | with: 14 | fetch-depth: '2' 15 | - name: Set up Python 16 | uses: actions/setup-python@v5 17 | with: 18 | python-version: '3.8' 19 | - name: Install Dependencies 20 | run: | 21 | python -m pip install --upgrade pip 22 | pip install .[all] 23 | pip install 'pdoc3<0.11.0' 24 | - name: Generate documentation 25 | run: | 26 | pdoc3 --html --force --template-dir pdoc3-template refinery 27 | - name: Upload documentation 28 | env: 29 | auth: ${{ secrets.GH_PAGES_TOKEN }} 30 | working-directory: ./html/refinery 31 | shell: bash 32 | run: | 33 | git init 34 | git config user.email "huettenhain@users.noreply.github.com" 35 | git config user.name jesko 36 | git remote add origin https://huettenhain:$auth@github.com/binref/binref.github.io 37 | git add --all 38 | git commit -m refinery/${{github.sha}} 39 | git push origin master --force 40 | -------------------------------------------------------------------------------- /.github/workflows/preview.yml: -------------------------------------------------------------------------------- 1 | name: preview 2 | 3 | on: 4 | workflow_call: 5 | inputs: 6 | python: 7 | description: 'The Python version to test against.' 8 | default: '3.13' 9 | required: true 10 | type: string 11 | 12 | jobs: 13 | preview: 14 | name: check forward compatibility 15 | continue-on-error: true 16 | runs-on: 'ubuntu-latest' 17 | env: 18 | MALSHARE_API: ${{ secrets.MALSHARE_API }} 19 | steps: 20 | - name: Checkout Main Repository 21 | uses: actions/checkout@v4 22 | - name: Install Dependencies 23 | uses: ./.github/actions/setup/ 24 | with: 25 | py: ${{ inputs.python }} 26 | os: 'ubuntu-latest' 27 | - name: Run Tests 28 | uses: ./.github/actions/test/ 29 | with: 30 | py: ${{ inputs.python }} 31 | os: 'ubuntu-latest' 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | /refinery/data/units.pkl 3 | /venv* 4 | /temp* 5 | /html 6 | /build 7 | /share 8 | /lib 9 | /dist 10 | /pyvenv.cfg 11 | /lib64 12 | *.egg-info 13 | *.eggs 14 | *.pyc -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include requirements.txt 2 | include README.md 3 | include LICENSE.md 4 | include refinery/__init__.pkl 5 | include refinery/data/rich.json -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | Binary Refinery is intended for **static** malware analysis, 4 | but it is nevertheless recommended to never analyze malware outside a sufficiently secured, preferably virtual, environment. 5 | That said refinery units should be robust against any input and: 6 | - Units should never perform uncontrolled execution of any part of the input. 7 | - Units should never write part of the input anywhere to disk, not even temporarily, except when this is their explicit given task. 8 | 9 | Should you identify any security vulnerabilities or violations of these principles, please file a 10 | [bug report](https://github.com/binref/refinery/issues/new?assignees=huettenhain&labels=bug&template=bug_report.md). 11 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | ignore: 2 | - "refinery/lib/thirdparty" 3 | coverage: 4 | status: 5 | project: 6 | default: 7 | target: 0% 8 | threshold: 100% 9 | patch: 10 | default: 11 | target: 0% 12 | threshold: 100% -------------------------------------------------------------------------------- /pdoc3-template/FixedSysEx.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/binref/refinery/44238b79ae1666d01d6d9cdf8eeb9818154cd843/pdoc3-template/FixedSysEx.ttf -------------------------------------------------------------------------------- /pdoc3-template/credits.mako: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/binref/refinery/44238b79ae1666d01d6d9cdf8eeb9818154cd843/pdoc3-template/credits.mako -------------------------------------------------------------------------------- /pdoc3-template/head.mako: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/binref/refinery/44238b79ae1666d01d6d9cdf8eeb9818154cd843/pdoc3-template/head.mako -------------------------------------------------------------------------------- /pdoc3-template/logo.mako: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/binref/refinery/44238b79ae1666d01d6d9cdf8eeb9818154cd843/pdoc3-template/logo.mako -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = [ 3 | "colorama", 4 | "defusedxml", 5 | "lief", 6 | "msgpack>=1.0.0", 7 | "pycryptodomex", 8 | "setuptools", 9 | "toml", 10 | "wheel", 11 | ] 12 | build-backend = "setuptools.build_meta" -------------------------------------------------------------------------------- /refinery/data/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module contains data resources. 3 | """ 4 | -------------------------------------------------------------------------------- /refinery/lib/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Library functions used by various refinery units. 5 | """ 6 | -------------------------------------------------------------------------------- /refinery/lib/dotnet/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | A library to parse .NET headers and metadata. 5 | """ 6 | 7 | 8 | def integer_from_ldc(ins: bytes): 9 | """ 10 | This function parses an integer value from the bytes representing an LDC instruction. 11 | """ 12 | if len(ins) == 1: 13 | return ins[0] - 0x16 14 | if ins[0] == 0x1F: 15 | return ins[1] 16 | return int.from_bytes(ins[1:], 'little') 17 | -------------------------------------------------------------------------------- /refinery/lib/inno/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Library functions for processing of Inno Setup files. 5 | """ 6 | -------------------------------------------------------------------------------- /refinery/lib/magic.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | A cross-platform interface to libmagic. 5 | """ 6 | try: 7 | from winmagic import magic 8 | except ModuleNotFoundError: 9 | import os 10 | if os.name == 'nt': 11 | # Attempting to import magic on Windows without winmagic being 12 | # installed may result in an uncontrolled crash. 13 | magic = None 14 | else: 15 | try: 16 | import magic 17 | except ImportError: 18 | magic = None 19 | 20 | 21 | def magicparse(data, *args, **kwargs): 22 | if magic: 23 | data = bytes(data) if not isinstance(data, bytes) else data 24 | return magic.Magic(*args, **kwargs).from_buffer(data) 25 | -------------------------------------------------------------------------------- /refinery/lib/resources.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | A wrapper module to read local data resources. 5 | """ 6 | from importlib import resources 7 | 8 | import sys 9 | 10 | 11 | def datapath(name: str): 12 | if sys.version_info >= (3, 9): 13 | from refinery import data 14 | return resources.files(data).joinpath(name) 15 | with resources.path('refinery', 'data') as data: 16 | return data / name 17 | -------------------------------------------------------------------------------- /refinery/lib/thirdparty/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module contains third-party libraries that usually have different licensing than 3 | Binary Refinery itself. 4 | """ 5 | -------------------------------------------------------------------------------- /refinery/shell.py: -------------------------------------------------------------------------------- 1 | """ 2 | # Shell-Like Unit Interface 3 | 4 | Any unit from the `refinery` module can also be imported from this module. When imported from here, 5 | the units are initialized differently: They can be given string arguments as they would receive on 6 | the command line. For example: 7 | 8 | >>> from refinery.shell import * 9 | >>> emit('ABC', 'DEF') [ pop('t') | xor('var:t') | pack('-R') ] | str 10 | '575' 11 | 12 | This especially gives easier access to the powerful `refinery.lib.meta` variables and the entire 13 | multibin format expressions, see `refinery.lib.argformats`. 14 | """ 15 | from functools import wraps 16 | from refinery import __unit_loader__, Unit 17 | 18 | with __unit_loader__: 19 | __all__ = sorted(__unit_loader__.units, key=lambda x: x.lower()) 20 | 21 | 22 | class __pdoc2__: 23 | def __class_getitem__(*_): 24 | return '' 25 | 26 | 27 | def __getattr__(name): 28 | with __unit_loader__: 29 | unit: Unit = __unit_loader__.resolve(name) 30 | 31 | if unit is None: 32 | raise AttributeError(name) 33 | 34 | class _unit(unit): 35 | def __new__(cls, *args, **kwargs): 36 | return unit.assemble(*args, **kwargs) 37 | 38 | return wraps(unit, updated=[])(_unit) 39 | 40 | 41 | def __dir__(): 42 | return __all__ 43 | -------------------------------------------------------------------------------- /refinery/units/blockwise/add.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units.blockwise import BinaryOperationWithAutoBlockAdjustment 4 | 5 | 6 | class add(BinaryOperationWithAutoBlockAdjustment): 7 | """ 8 | Add the given argument to each block. 9 | """ 10 | @staticmethod 11 | def operate(a, b): return a + b 12 | @staticmethod 13 | def inplace(a, b): a += b 14 | -------------------------------------------------------------------------------- /refinery/units/blockwise/neg.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units.blockwise import UnaryOperation 4 | 5 | 6 | class neg(UnaryOperation): 7 | """ 8 | Each block of the input data is negated bitwise. This is sometimes 9 | also called the bitwise complement or inverse. 10 | """ 11 | def operate(self, a): return ~a 12 | def inplace(self, a): a ^= self.fmask 13 | -------------------------------------------------------------------------------- /refinery/units/blockwise/rotl.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units.blockwise import BinaryOperation 4 | 5 | 6 | class rotl(BinaryOperation): 7 | """ 8 | Rotate the bits of each block left. 9 | """ 10 | def operate(self, value, shift): 11 | shift %= self.fbits 12 | return (value << shift) | (value >> (self.fbits - shift)) 13 | 14 | def inplace(self, value, shift): 15 | shift %= self.fbits 16 | lower = value >> (self.fbits - shift) 17 | value <<= shift 18 | value |= lower 19 | -------------------------------------------------------------------------------- /refinery/units/blockwise/rotr.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units.blockwise import BinaryOperation 4 | 5 | 6 | class rotr(BinaryOperation): 7 | """ 8 | Rotate the bits of each block right. 9 | """ 10 | def operate(self, value, shift): 11 | shift %= self.fbits 12 | return (value >> shift) | (value << (self.fbits - shift)) 13 | 14 | def inplace(self, value, shift): 15 | shift %= self.fbits 16 | lower = value >> shift 17 | value <<= self.fbits - shift 18 | value |= lower 19 | -------------------------------------------------------------------------------- /refinery/units/blockwise/shl.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units.blockwise import BinaryOperation 4 | 5 | 6 | class shl(BinaryOperation): 7 | """ 8 | Shift the bits of each block left, filling with zero bits. 9 | """ 10 | @staticmethod 11 | def operate(a, b): return a << b 12 | @staticmethod 13 | def inplace(a, b): a <<= b 14 | -------------------------------------------------------------------------------- /refinery/units/blockwise/shr.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units.blockwise import BinaryOperation 4 | 5 | 6 | class shr(BinaryOperation): 7 | """ 8 | Shift the bits of each block right, filling with zero bits. 9 | """ 10 | @staticmethod 11 | def operate(a, b): return a >> b 12 | @staticmethod 13 | def inplace(a, b): a >>= b 14 | -------------------------------------------------------------------------------- /refinery/units/blockwise/sub.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units.blockwise import BinaryOperationWithAutoBlockAdjustment 4 | 5 | 6 | class sub(BinaryOperationWithAutoBlockAdjustment): 7 | """ 8 | Subtract the given argument from each block. 9 | """ 10 | @staticmethod 11 | def operate(a, b): return a - b 12 | @staticmethod 13 | def inplace(a, b): a -= b 14 | -------------------------------------------------------------------------------- /refinery/units/blockwise/xor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units.blockwise import BinaryOperationWithAutoBlockAdjustment, FastBlockError 4 | 5 | 6 | class xor(BinaryOperationWithAutoBlockAdjustment): 7 | """ 8 | Form the exclusive or of the input data with the given argument. 9 | """ 10 | @staticmethod 11 | def operate(a, b): return a ^ b 12 | @staticmethod 13 | def inplace(a, b): a ^= b 14 | 15 | def _fastblock(self, data): 16 | try: 17 | return super()._fastblock(data) 18 | except FastBlockError as E: 19 | try: 20 | from Cryptodome.Util.strxor import strxor 21 | except ModuleNotFoundError: 22 | raise E 23 | else: 24 | from itertools import islice 25 | size = len(data) 26 | arg0 = self._normalize_argument(*self._argument_parse_hook(self.args.argument[0])) 27 | take = len(data) // self.blocksize + 1 28 | argb = self.unchunk(islice(arg0, take)) 29 | del argb[size:] 30 | return strxor(data, argb) 31 | -------------------------------------------------------------------------------- /refinery/units/compression/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | A collection of compression algorithms. The unit `refinery.decompress` 5 | implements a brute force heuristic decompressor that attempts all known 6 | algorithms against the input data. 7 | """ 8 | -------------------------------------------------------------------------------- /refinery/units/compression/brotli.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units import Unit 4 | 5 | 6 | class brotli(Unit): 7 | """ 8 | Brotli compression and decompression. 9 | """ 10 | 11 | @Unit.Requires('brotlipy', 'all') 12 | def _brotli(): 13 | import brotli 14 | return brotli 15 | 16 | def process(self, data): 17 | return self._brotli.decompress(bytes(data)) 18 | 19 | def reverse(self, data): 20 | return self._brotli.compress(bytes(data)) 21 | -------------------------------------------------------------------------------- /refinery/units/compression/bz2.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | import bz2 as bz2_ 4 | 5 | from refinery.units import Arg, Unit 6 | from refinery.lib.argformats import number 7 | 8 | 9 | class bz2(Unit): 10 | """ 11 | BZip2 compression and decompression. 12 | """ 13 | def __init__(self, level: Arg('-l', type=number[1:9], help='compression level preset between 1 and 9') = 9): 14 | super().__init__(level=level) 15 | 16 | def process(self, data): 17 | return bz2_.decompress(data) 18 | 19 | def reverse(self, data): 20 | return bz2_.compress(data, self.args.level) 21 | 22 | @classmethod 23 | def handles(self, data: bytearray): 24 | return data[:3] == B'BZh' 25 | -------------------------------------------------------------------------------- /refinery/units/compression/lzx.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from __future__ import annotations 4 | 5 | from refinery.units import Arg, Unit, RefineryPartialResult 6 | from refinery.lib.lzx import LzxDecoder 7 | 8 | 9 | class lzx(Unit): 10 | 11 | def __init__( 12 | self, 13 | window: Arg.Number('window', metavar='window', 14 | help='Optionally specify the window size; the default is {default}.') = 15, 15 | wim: Arg.Switch('-w', help='Use the WIM flavor of LZX.') = False, 16 | ): 17 | super().__init__(window=window, wim=wim) 18 | 19 | def process(self, data): 20 | lzx = LzxDecoder(self.args.wim) 21 | lzx.set_params_and_alloc(self.args.window) 22 | 23 | try: 24 | return lzx.decompress(memoryview(data)) 25 | except Exception as E: 26 | if out := lzx.get_output_data(): 27 | raise RefineryPartialResult(str(E), out) from E 28 | raise 29 | -------------------------------------------------------------------------------- /refinery/units/compression/zstd.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units import Unit, RefineryPartialResult 4 | 5 | 6 | class zstd(Unit): 7 | """ 8 | ZStandard (ZSTD) compression and decompression. 9 | """ 10 | @Unit.Requires('pyzstd', 'all') 11 | def _pyzstd(): 12 | import pyzstd 13 | return pyzstd 14 | 15 | def process(self, data): 16 | zd = self._pyzstd.ZstdDecompressor() 17 | out = zd.decompress(data) 18 | if zd.needs_input: 19 | raise RefineryPartialResult('Incomplete ZSTD stream.', out) 20 | return out 21 | 22 | def reverse(self, data): 23 | zc = self._pyzstd.ZstdCompressor() 24 | return zc.compress(data) + zc.flush() 25 | 26 | @classmethod 27 | def handles(self, data: bytearray) -> bool: 28 | return data[:4] == B'\x28\xB5\x2F\xFD' 29 | -------------------------------------------------------------------------------- /refinery/units/crypto/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Cryptographic routines, cipher and key derivation units. 5 | """ 6 | -------------------------------------------------------------------------------- /refinery/units/crypto/cipher/aes.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from Cryptodome.Cipher import AES 4 | 5 | from refinery.units.crypto.cipher import StandardBlockCipherUnit 6 | from refinery.lib.crypto import PyCryptoFactoryWrapper 7 | 8 | 9 | class aes(StandardBlockCipherUnit, cipher=PyCryptoFactoryWrapper(AES)): 10 | """ 11 | AES encryption and decryption. 12 | """ 13 | pass 14 | -------------------------------------------------------------------------------- /refinery/units/crypto/cipher/blowfish.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from Cryptodome.Cipher import Blowfish 4 | 5 | from refinery.units.crypto.cipher import StandardBlockCipherUnit 6 | from refinery.lib.crypto import PyCryptoFactoryWrapper 7 | 8 | 9 | class blowfish(StandardBlockCipherUnit, cipher=PyCryptoFactoryWrapper(Blowfish)): 10 | """ 11 | Blowfish encryption and decryption. 12 | """ 13 | pass 14 | -------------------------------------------------------------------------------- /refinery/units/crypto/cipher/cast.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from Cryptodome.Cipher import CAST 4 | 5 | from refinery.units.crypto.cipher import StandardBlockCipherUnit 6 | from refinery.lib.crypto import PyCryptoFactoryWrapper 7 | 8 | 9 | class cast(StandardBlockCipherUnit, cipher=PyCryptoFactoryWrapper(CAST)): 10 | """ 11 | CAST encryption and decryption. 12 | """ 13 | pass 14 | -------------------------------------------------------------------------------- /refinery/units/crypto/cipher/des.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from Cryptodome.Cipher import DES 4 | 5 | from refinery.units.crypto.cipher import StandardBlockCipherUnit 6 | from refinery.lib.crypto import PyCryptoFactoryWrapper 7 | 8 | 9 | class des(StandardBlockCipherUnit, cipher=PyCryptoFactoryWrapper(DES)): 10 | """ 11 | DES encryption and decryption. 12 | """ 13 | pass 14 | -------------------------------------------------------------------------------- /refinery/units/crypto/cipher/des3.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from Cryptodome.Cipher import DES3 4 | 5 | from refinery.units.crypto.cipher import StandardBlockCipherUnit 6 | from refinery.lib.crypto import PyCryptoFactoryWrapper 7 | 8 | 9 | class des3(StandardBlockCipherUnit, cipher=PyCryptoFactoryWrapper(DES3)): 10 | """ 11 | 3-DES encryption and decryption. 12 | """ 13 | pass 14 | -------------------------------------------------------------------------------- /refinery/units/crypto/cipher/rc4.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from Cryptodome.Cipher import ARC4 4 | 5 | from refinery.units.crypto.cipher import StandardCipherUnit, Arg 6 | from refinery.lib.crypto import PyCryptoFactoryWrapper 7 | 8 | ARC4.key_size = range(1, 257) 9 | 10 | 11 | class rc4(StandardCipherUnit, cipher=PyCryptoFactoryWrapper(ARC4)): 12 | """ 13 | RC4 encryption and decryption. 14 | """ 15 | def __init__( 16 | self, key, 17 | discard: Arg.Number('-d', help='Discard the first {varname} bytes of the keystream, {default} by default.') = 0, 18 | ): 19 | super().__init__(key, discard=discard) 20 | 21 | def _new_cipher(self, **optionals): 22 | return super()._new_cipher(drop=self.args.discard, **optionals) 23 | -------------------------------------------------------------------------------- /refinery/units/crypto/cipher/rot.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units import Arg, Unit 4 | 5 | _UCASE = range(ord('A'), ord('Z') + 1) 6 | _LCASE = range(ord('a'), ord('z') + 1) 7 | 8 | 9 | class rot(Unit): 10 | """ 11 | Rotate the characters of the alphabet by the given amount. The default 12 | amount is 13, providing the common (and weak) string obfuscation method. 13 | """ 14 | 15 | def __init__(self, amount: Arg.Number(help='Number of letters to rotate by; Default is 13.') = 13): 16 | super().__init__(amount=amount) 17 | 18 | def process(self, data: bytearray): 19 | rot = self.args.amount % 26 20 | for index, byte in enumerate(data): 21 | for alphabet in _LCASE, _UCASE: 22 | if byte in alphabet: 23 | zero = alphabet[0] 24 | data[index] = zero + (byte - zero + rot) % 26 25 | break 26 | return data 27 | -------------------------------------------------------------------------------- /refinery/units/crypto/hash/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Implements various hashing algorithms. 5 | """ 6 | from refinery.units import Arg, Unit, abc 7 | 8 | 9 | class HashUnit(Unit, abstract=True): 10 | 11 | @abc.abstractmethod 12 | def _algorithm(self, data: bytes) -> bytes: 13 | raise NotImplementedError 14 | 15 | def __init__( 16 | self, 17 | reps: Arg.Number('-r', help='Optionally specify a number of times to apply the hash to its own output.') = 1, 18 | text: Arg.Switch('-t', help='Output a hexadecimal representation of the hash.') = False, 19 | **kwargs 20 | ): 21 | super().__init__(text=text, reps=reps, **kwargs) 22 | 23 | def process(self, data: bytes) -> bytes: 24 | reps = self.args.reps 25 | digest = data 26 | for _ in range(reps): 27 | digest = self._algorithm(digest) 28 | if self.args.text: 29 | digest = digest.hex().encode(self.codec) 30 | return digest 31 | -------------------------------------------------------------------------------- /refinery/units/crypto/hash/checksums.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Implements hash algorithms of short length, commonly used as checksums. 5 | """ 6 | import zlib 7 | import struct 8 | 9 | from refinery.units.crypto.hash import HashUnit 10 | 11 | 12 | class crc32(HashUnit): 13 | """ 14 | Returns the CRC32 Hash of the input data. 15 | """ 16 | def _algorithm(self, data: bytes) -> bytes: 17 | return struct.pack('>I', zlib.crc32(data)) 18 | 19 | 20 | class adler32(HashUnit): 21 | """ 22 | Returns the Adler32 Hash of the input data. 23 | """ 24 | def _algorithm(self, data: bytes) -> bytes: 25 | return struct.pack('>I', zlib.adler32(data)) 26 | -------------------------------------------------------------------------------- /refinery/units/crypto/hash/imphash.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units.crypto.hash import HashUnit 4 | from refinery.lib import lief 5 | 6 | 7 | class imphash(HashUnit): 8 | """ 9 | Implements the import hash for PE files. 10 | """ 11 | 12 | def _algorithm(self, data): 13 | pe = lief.load_pe(data) 14 | th = lief.PE.get_imphash(pe, lief.PE.IMPHASH_MODE.PEFILE) 15 | return bytes.fromhex(th) 16 | -------------------------------------------------------------------------------- /refinery/units/crypto/hash/maru.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.lib.maru import maru32digest 4 | from refinery.units.crypto.hash import HashUnit, Arg 5 | 6 | 7 | class maru(HashUnit): 8 | """ 9 | Returns the 64bit maru hash of the input data. 10 | """ 11 | def __init__( 12 | self, 13 | seed: Arg.Number(help='optional seed value') = 0, 14 | reps=1, 15 | text=False, 16 | ): 17 | super().__init__(seed=seed, text=text, reps=reps) 18 | 19 | def _algorithm(self, data: bytes) -> bytes: 20 | return maru32digest(data, self.args.seed) 21 | -------------------------------------------------------------------------------- /refinery/units/crypto/hash/password_hashes.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Implements password hashing algorithms. 5 | """ 6 | from refinery.units.crypto.hash import HashUnit 7 | 8 | 9 | class ntlm(HashUnit): 10 | """ 11 | Returns the Windows NTLM hash of the input. 12 | """ 13 | def _algorithm(self, data: bytes) -> bytes: 14 | from Cryptodome.Hash import MD4 15 | return MD4.new(data.decode(self.codec).encode('utf-16le')).digest() 16 | -------------------------------------------------------------------------------- /refinery/units/crypto/hash/xxhash.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units.crypto.hash import HashUnit 4 | from refinery.lib.thirdparty.xxhash import xxhash 5 | 6 | 7 | class xxh(HashUnit): 8 | """ 9 | Implements the xxHash hashing algorithm. 10 | """ 11 | def __init__( 12 | self, 13 | seed: HashUnit.Arg.Number(metavar='seed', help='specify the seed value; the default is {default}') = 0, 14 | text=False 15 | ): 16 | super().__init__(text, seed=seed) 17 | 18 | def _algorithm(self, data): 19 | return xxhash(data, self.args.seed).digest() 20 | -------------------------------------------------------------------------------- /refinery/units/crypto/keyderive/hkdf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units.crypto.keyderive import KeyDerivation 4 | 5 | 6 | class hkdf(KeyDerivation): 7 | """HKDF Key derivation""" 8 | 9 | def __init__(self, size, salt, hash='SHA512'): 10 | super().__init__(size=size, salt=salt, hash=hash) 11 | 12 | def process(self, data): 13 | from Cryptodome.Protocol.KDF import HKDF 14 | return HKDF(data, self.args.size, self.args.salt, self.hash) 15 | -------------------------------------------------------------------------------- /refinery/units/crypto/keyderive/hmac.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units.crypto.keyderive import KeyDerivation 4 | 5 | 6 | class hmac(KeyDerivation): 7 | """ 8 | HMAC based key derivation 9 | """ 10 | 11 | def __init__(self, salt, hash='SHA1', size=None): 12 | super().__init__(salt=salt, size=size, hash=hash) 13 | 14 | def process(self, data): 15 | from Cryptodome.Hash import HMAC 16 | return HMAC.new(data, self.args.salt, digestmod=self.hash).digest() 17 | -------------------------------------------------------------------------------- /refinery/units/crypto/keyderive/kblob.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.lib.mscrypto import CRYPTOKEY 4 | from refinery.units import Unit 5 | 6 | 7 | class kblob(Unit): 8 | """ 9 | Extracts a key from a Microsoft Crypto API BLOB structure. 10 | """ 11 | 12 | def process(self, data): 13 | blob = CRYPTOKEY(data) 14 | try: 15 | return self.labelled( 16 | bytes(blob.key), 17 | type=blob.header.type.name, 18 | algorithm=blob.header.algorithm.name 19 | ) 20 | except AttributeError as A: 21 | raise ValueError(F'unable to derive key from {blob.header.type!s}') from A 22 | -------------------------------------------------------------------------------- /refinery/units/crypto/keyderive/mspdb.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units.crypto.keyderive import KeyDerivation 4 | 5 | 6 | class mspdb(KeyDerivation): 7 | """ 8 | An implementation of the PasswordDeriveBytes routine available from the .NET 9 | standard library. According to documentation, it is an extension of PBKDF1. 10 | """ 11 | def __init__(self, size, salt, iter=100, hash='SHA1'): 12 | self.superinit(super(), **vars()) 13 | 14 | def process(self, data): 15 | if self.codec != 'UTF8': 16 | data = data.decode(self.codec).encode('UTF8') 17 | data += self.args.salt 18 | for _ in range(self.args.iter - 1): 19 | data = self.hash.new(data).digest() 20 | counter, seedhash = 1, data 21 | data = self.hash.new(data).digest() 22 | while len(data) < self.args.size: 23 | data += self.hash.new(B'%d%s' % (counter, seedhash)).digest() 24 | counter += 1 25 | return data[:self.args.size] 26 | -------------------------------------------------------------------------------- /refinery/units/crypto/keyderive/pbkdf1.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units.crypto.keyderive import Arg, KeyDerivation, multidecode 4 | 5 | 6 | class pbkdf1(KeyDerivation): 7 | """PBKDF1 Key derivation""" 8 | 9 | @Arg('salt', help='Salt for the derivation; default are 8 null bytes.') 10 | def __init__(self, size, salt=bytes(8), iter=1000, hash='SHA1'): 11 | self.superinit(super(), **vars()) 12 | 13 | def process(self, data): 14 | from Cryptodome.Protocol.KDF import PBKDF1 15 | return multidecode(data, lambda pwd: ( 16 | PBKDF1(pwd, self.args.salt, dkLen=self.args.size, count=self.args.iter, hashAlgo=self.hash) 17 | )) 18 | -------------------------------------------------------------------------------- /refinery/units/crypto/keyderive/pbkdf2.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from functools import partial 4 | from refinery.lib.types import ByteStr 5 | from refinery.units.crypto.keyderive import KeyDerivation, multidecode 6 | 7 | 8 | class pbkdf2(KeyDerivation): 9 | """ 10 | PBKDF2 Key derivation. This is implemented as Rfc2898DeriveBytes in .NET 11 | binaries. 12 | """ 13 | 14 | def __init__(self, size, salt, iter=1000, hash='SHA1'): 15 | self.superinit(super(), **vars()) 16 | 17 | def process(self, data: ByteStr): 18 | from Cryptodome.Protocol.KDF import PBKDF2 19 | return multidecode(data, partial( 20 | PBKDF2, 21 | salt=self.args.salt, 22 | dkLen=self.args.size, 23 | hmac_hash_module=self.hash, 24 | count=self.args.iter 25 | )) 26 | -------------------------------------------------------------------------------- /refinery/units/encoding/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Encoding and decoding of various formats. 5 | """ 6 | -------------------------------------------------------------------------------- /refinery/units/encoding/a85.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | import base64 4 | import re 5 | 6 | from refinery.units import Unit 7 | 8 | 9 | class a85(Unit): 10 | """ 11 | Ascii85 encoding and decoding, the predecessor variant of Base85 with a different alphabet. 12 | """ 13 | def reverse(self, data): 14 | return base64.a85encode(data) 15 | 16 | def process(self, data): 17 | if re.search(BR'\s', data) is not None: 18 | data = re.sub(BR'\s+', B'', data) 19 | return base64.a85decode(data) 20 | 21 | @classmethod 22 | def handles(self, data: bytearray): 23 | from refinery.lib.patterns import formats 24 | return formats.spaced_a85.value.fullmatch(data) 25 | -------------------------------------------------------------------------------- /refinery/units/encoding/atbash.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units import Unit 4 | 5 | 6 | class atbash(Unit): 7 | """ 8 | https://en.wikipedia.org/wiki/Atbash 9 | Atbash encoding and decoding. Fairly useless in the 21st century, except 10 | for picking out crypto nerds. 11 | """ 12 | 13 | def process(self, data: bytearray): 14 | uc = range(B'A'[0], B'Z'[0] + 1) 15 | lc = range(B'a'[0], B'z'[0] + 1) 16 | for k, letter in enumerate(data): 17 | if letter in uc: 18 | data[k] = uc[~uc.index(letter)] 19 | continue 20 | if letter in lc: 21 | data[k] = lc[~lc.index(letter)] 22 | continue 23 | return data 24 | 25 | reverse = process 26 | -------------------------------------------------------------------------------- /refinery/units/encoding/b32.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | import base64 4 | 5 | from refinery.units import Unit 6 | 7 | 8 | class b32(Unit): 9 | """ 10 | Base32 encoding and decoding. 11 | """ 12 | def reverse(self, data): 13 | return base64.b32encode(data) 14 | 15 | def process(self, data: bytearray): 16 | before_padding = 0 17 | for before_padding in range(len(data), 0, -1): 18 | if data[before_padding - 1:before_padding] != B'=': 19 | break 20 | padding_size = -before_padding % 8 21 | missing = before_padding + padding_size - len(data) 22 | if missing > 0: 23 | self.log_info(F'detected incorrect padding: added {missing} padding characters') 24 | data.extend(B'=' * missing) 25 | if missing < 0: 26 | self.log_info(F'detected incorrect padding: removed {-missing} padding characters') 27 | data[padding_size + before_padding:] = [] 28 | return base64.b32decode(data, casefold=True) 29 | 30 | @classmethod 31 | def handles(cls, data): 32 | from refinery.lib.patterns import formats 33 | if not formats.b32.value.fullmatch(data): 34 | return False 35 | return not formats.hex.value.fullmatch(data) 36 | -------------------------------------------------------------------------------- /refinery/units/encoding/b58.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units.encoding.base import base 4 | 5 | 6 | class b58(base): 7 | """ 8 | Base58 encoding and decoding. It is famously used as an encoding in Bitcoin addresses 9 | because the alphabet omits digits and letters that look similar. 10 | """ 11 | def __init__(self): 12 | super().__init__(b'123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz') 13 | 14 | @classmethod 15 | def handles(cls, data): 16 | from refinery.lib.patterns import formats 17 | return ( 18 | formats.b58.value.fullmatch(data) 19 | and not formats.hex.value.fullmatch(data) 20 | and not formats.b32.value.fullmatch(data) 21 | ) 22 | -------------------------------------------------------------------------------- /refinery/units/encoding/b62.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units.encoding.base import base 4 | 5 | 6 | class b62(base): 7 | """ 8 | Base62 encoding and decoding. 9 | """ 10 | def __init__(self): 11 | super().__init__(b'0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz') 12 | 13 | @classmethod 14 | def handles(cls, data): 15 | from refinery.lib.patterns import formats 16 | return ( 17 | formats.b62.value.fullmatch(data) 18 | and not formats.hex.value.fullmatch(data) 19 | and not formats.b32.value.fullmatch(data) 20 | ) 21 | -------------------------------------------------------------------------------- /refinery/units/encoding/b85.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | import base64 4 | import re 5 | 6 | from refinery.units import Unit 7 | 8 | 9 | class b85(Unit): 10 | """ 11 | Base85 encoding and decoding. 12 | """ 13 | def reverse(self, data): 14 | return base64.b85encode(data) 15 | 16 | def process(self, data): 17 | if re.search(BR'\s', data) is not None: 18 | data = re.sub(BR'\s+', B'', data) 19 | return base64.b85decode(data) 20 | 21 | @classmethod 22 | def handles(self, data: bytearray): 23 | from refinery.lib.patterns import formats 24 | return formats.spaced_b85.value.fullmatch(data) 25 | -------------------------------------------------------------------------------- /refinery/units/encoding/cp1252.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units import Unit 4 | 5 | 6 | class cp1252(Unit): 7 | """ 8 | Encodes and decodes Windows CP 1252 (aka Latin1) encoded string data. 9 | """ 10 | 11 | def process(self, data): 12 | return data.decode(self.codec).encode('cp1252') 13 | 14 | def reverse(self, data): 15 | return data.decode('cp1252').encode(self.codec) 16 | -------------------------------------------------------------------------------- /refinery/units/encoding/escvb.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units import Unit 4 | 5 | 6 | class escvb(Unit): 7 | """ 8 | Escapes and unescapes Visual Basic strings. 9 | """ 10 | def process(self, data): 11 | if data[:1] == B'"' and data[-1:] == B'"': 12 | data = data[1:-1] 13 | return data.replace(B'""', B'"') 14 | 15 | def reverse(self, data): 16 | return B'"%s"' % data.replace(B'"', B'""') 17 | -------------------------------------------------------------------------------- /refinery/units/encoding/hex.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units import Unit 4 | 5 | 6 | class hex(Unit): 7 | """ 8 | Hex-decodes and encodes binary data. Non-hex characters are removed from 9 | the input. For decoding, any odd trailing hex digits are stripped as two 10 | hex digits are required to represent a byte. 11 | """ 12 | 13 | def reverse(self, data): 14 | import base64 15 | return base64.b16encode(data) 16 | 17 | def process(self, data): 18 | import re 19 | import base64 20 | data = re.sub(B'[^A-Fa-f0-9]+', B'', data) 21 | if len(data) % 2: 22 | data = data[:-1] 23 | return base64.b16decode(data, casefold=True) 24 | 25 | @classmethod 26 | def handles(self, data: bytearray): 27 | from refinery.lib.patterns import formats 28 | if formats.spaced_hex.fullmatch(data): 29 | return True 30 | -------------------------------------------------------------------------------- /refinery/units/encoding/htmlesc.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | import html as html_entities 4 | 5 | from refinery.units import Unit 6 | from refinery.lib.decorators import unicoded 7 | 8 | 9 | class htmlesc(Unit): 10 | """ 11 | Encodes and decodes HTML entities. 12 | """ 13 | 14 | @unicoded 15 | def process(self, data: str) -> str: 16 | return html_entities.unescape(data) 17 | 18 | @unicoded 19 | def reverse(self, data: str) -> str: 20 | return html_entities.escape(data) 21 | -------------------------------------------------------------------------------- /refinery/units/encoding/u16.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units import Unit 4 | 5 | 6 | class u16(Unit): 7 | """ 8 | Encodes and decodes UTF-16 encoded string data. 9 | """ 10 | 11 | def reverse(self, data): 12 | return data.decode(self.codec).encode('utf-16LE') 13 | 14 | def process(self, data): 15 | return data.decode('utf-16').encode(self.codec) 16 | 17 | @classmethod 18 | def handles(self, data: bytearray): 19 | view = memoryview(data) 20 | if len(view) % 2 != 0: 21 | return False 22 | if not any(view[1:0x100:2]): 23 | return True 24 | if not any(view[0:0x100:2]): 25 | return any(view[:4]) 26 | -------------------------------------------------------------------------------- /refinery/units/formats/archive/xtace.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units.formats.archive import ArchiveUnit 4 | from refinery.lib.thirdparty import acefile 5 | from refinery.lib.structures import MemoryFile 6 | 7 | 8 | class xtace(ArchiveUnit, docs='{0}{s}{PathExtractorUnit}'): 9 | """ 10 | Extract files from an ACE archive. 11 | """ 12 | def unpack(self, data): 13 | ace = acefile.open(MemoryFile(data, read_as_bytes=True)) 14 | for member in ace.getmembers(): 15 | member: acefile.AceMember 16 | comment = {} if not member.comment else {'comment': member.comment} 17 | yield self._pack( 18 | member.filename, 19 | member.datetime, 20 | lambda a=ace, m=member: a.read(m, pwd=self.args.pwd), 21 | **comment 22 | ) 23 | 24 | @classmethod 25 | def handles(cls, data: bytearray) -> bool: 26 | return b'**ACE**' in data[:0x100] 27 | -------------------------------------------------------------------------------- /refinery/units/formats/dexstr.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units import Unit 4 | from refinery.lib.dex import DexFile 5 | 6 | 7 | class dexstr(Unit): 8 | """ 9 | Extract strings from DEX (Dalvik Executable) files. 10 | """ 11 | def process(self, data): 12 | dex = DexFile(data) 13 | for string in dex.read_strings(): 14 | yield string.encode(self.codec) 15 | -------------------------------------------------------------------------------- /refinery/units/formats/evtx.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf - 8 -* - 3 | from refinery.units import Unit 4 | from refinery.lib.vfs import VirtualFileSystem 5 | 6 | 7 | class evtx(Unit): 8 | """ 9 | Extracts data from Windows Event Log files (EVTX). Each extracted log entry is returned as a single 10 | output chunk in XML format. 11 | """ 12 | 13 | def __init__(self, raw: Unit.Arg.Switch('-r', help='Extract raw event data rather than XML.') = False): 14 | super().__init__(raw=raw) 15 | 16 | @Unit.Requires('python-evtx', 'formats') 17 | def _evtx(): 18 | from Evtx.Evtx import Evtx 19 | return Evtx 20 | 21 | def process(self, data): 22 | with VirtualFileSystem() as vfs: 23 | raw = self.args.raw 24 | with self._evtx(vfs.new(data)) as log: 25 | for record in log.records(): 26 | yield record.data() if raw else record.xml().encode(self.codec) 27 | -------------------------------------------------------------------------------- /refinery/units/formats/exe/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | A package with units for generic executables. Usually, PE, ELF, and MachO formats are covered. 5 | """ 6 | -------------------------------------------------------------------------------- /refinery/units/formats/httpresponse.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from http.client import HTTPResponse, IncompleteRead 4 | 5 | from refinery.units import Unit, RefineryPartialResult 6 | from refinery.lib.structures import MemoryFile 7 | 8 | 9 | class SockWrapper(MemoryFile): 10 | def sendall(self, ___): pass 11 | def makefile(self, *_): return self 12 | 13 | 14 | class httpresponse(Unit): 15 | """ 16 | Parses HTTP response text, as you would obtain from a packet dump. This can be 17 | useful if chunked or compressed transfer encoding was used. 18 | """ 19 | def process(self, data): 20 | with SockWrapper(data) as mock: 21 | mock.seek(0) 22 | parser = HTTPResponse(mock) 23 | parser.begin() 24 | try: 25 | return parser.read() 26 | except IncompleteRead as incomplete: 27 | msg = F'incomplete read: {len(incomplete.partial)} bytes processed, {incomplete.expected} more expected' 28 | raise RefineryPartialResult(msg, incomplete.partial) from incomplete 29 | -------------------------------------------------------------------------------- /refinery/units/formats/ifps.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | The code is based on the logic implemented in IFPSTools: 5 | https://github.com/Wack0/IFPSTools 6 | """ 7 | from __future__ import annotations 8 | 9 | from refinery.units.formats import Unit 10 | from refinery.lib.inno.ifps import IFPSFile 11 | 12 | 13 | class IFPSBase(Unit, abstract=True): 14 | def __init__( 15 | self, 16 | codec: Unit.Arg.String( 17 | help='Optionally specify the string encoding. The default is "{default}".') = 'cp1252' 18 | ): 19 | super().__init__(codec=codec) 20 | 21 | 22 | class ifps(IFPSBase): 23 | """ 24 | Disassembles compiled Pascal script files that start with the magic sequence "IFPS". These 25 | scripts can be found, for example, when unpacking InnoSetup installers using innounp. 26 | """ 27 | def process(self, data): 28 | return IFPSFile(data, self.args.codec).disassembly().encode(self.codec) 29 | 30 | @classmethod 31 | def handles(self, data: bytearray) -> bool: 32 | return data.startswith(IFPSFile.Magic) 33 | -------------------------------------------------------------------------------- /refinery/units/formats/ifpsstr.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.lib.inno.ifps import IFPSFile 4 | from refinery.units.formats.ifps import IFPSBase 5 | 6 | 7 | class ifpsstr(IFPSBase): 8 | """ 9 | Extracts strings from compiled Pascal script files that start with the magic sequence "IFPS". 10 | These scripts can be found, for example, when unpacking InnoSetup installers using innounp. 11 | """ 12 | def process(self, data): 13 | ifps = IFPSFile(data, self.args.codec) 14 | for string in ifps.strings: 15 | yield string.encode(self.codec) 16 | 17 | @classmethod 18 | def handles(self, data: bytearray) -> bool: 19 | return data.startswith(IFPSFile.Magic) 20 | -------------------------------------------------------------------------------- /refinery/units/formats/java/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Units that process Java related binary formats such as class files and serialized Java objects. 5 | """ 6 | -------------------------------------------------------------------------------- /refinery/units/formats/java/jvstr.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units import Unit 4 | from refinery.lib.java import JvClassFile 5 | 6 | 7 | class jvstr(Unit): 8 | """ 9 | Extract string constants from Java class files. 10 | """ 11 | def process(self, data): 12 | jc = JvClassFile(data) 13 | for string in jc.strings: 14 | yield string.encode(self.codec) 15 | 16 | @classmethod 17 | def handles(self, data): 18 | return data[:4] == B'\xCA\xFE\xBA\xBE' 19 | -------------------------------------------------------------------------------- /refinery/units/formats/macho/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/binref/refinery/44238b79ae1666d01d6d9cdf8eeb9818154cd843/refinery/units/formats/macho/__init__.py -------------------------------------------------------------------------------- /refinery/units/formats/msgpack.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | import itertools 4 | import json 5 | import msgpack as mp 6 | 7 | from refinery.units import RefineryPartialResult, Unit 8 | from refinery.lib.structures import MemoryFile 9 | 10 | 11 | class msgpack(Unit): 12 | """ 13 | Converts a message-pack (msgpack) buffer to JSON and vice-versa. 14 | """ 15 | def reverse(self, data): 16 | return mp.dumps(json.loads(data)) 17 | 18 | def process(self, data): 19 | unpacker: mp.fallback.Unpacker = mp.Unpacker(MemoryFile(data, read_as_bytes=True)) 20 | for k in itertools.count(): 21 | try: 22 | last = unpacker.tell() 23 | item = unpacker.unpack() 24 | except Exception as E: 25 | if isinstance(E, mp.OutOfData) and k == 1: 26 | break 27 | raise RefineryPartialResult(str(E), memoryview(data)[last:]) from E 28 | else: 29 | yield json.dumps(item).encode(self.codec) 30 | -------------------------------------------------------------------------------- /refinery/units/formats/office/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | These units process data formats related to Microsoft Office. 5 | """ 6 | -------------------------------------------------------------------------------- /refinery/units/formats/office/docmeta.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.lib import xml 4 | from refinery.units.formats import PathExtractorUnit, UnpackResult 5 | from refinery.units.formats.office.xtdoc import xtdoc 6 | 7 | 8 | class docmeta(PathExtractorUnit): 9 | """ 10 | Extract metadata from Word Documents such as custom document properties. 11 | """ 12 | @PathExtractorUnit.Requires('olefile', 'formats', 'office') 13 | def _olefile(): 14 | import olefile 15 | return olefile 16 | 17 | def unpack(self, data: bytearray): 18 | properties = data | xtdoc('docProps/custom.xml') | str 19 | if not properties: 20 | return 21 | properties = xml.parse(properties) 22 | while properties.tag.lower() != 'properties': 23 | properties = properties.children[0] 24 | for node in properties: 25 | assert node.tag.lower() == 'property' 26 | assert len(node.children) == 1 27 | content = node.children[0].content 28 | assert content is not None 29 | yield UnpackResult(node.attributes['name'], content.encode(self.codec)) 30 | -------------------------------------------------------------------------------- /refinery/units/formats/office/rtfc.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units import Unit 4 | 5 | 6 | class rtfc(Unit): 7 | """ 8 | Implements the RTF compression format. This compression algorithm is used, for example, to 9 | compress RTF data in Outlook messages. 10 | """ 11 | @Unit.Requires('compressed_rtf', 'formats', 'office', 'default', 'extended') 12 | def _rtfc(): 13 | import compressed_rtf 14 | return compressed_rtf 15 | 16 | def process(self, data): 17 | return self._rtfc.decompress(data) 18 | 19 | def reverse(self, data): 20 | return self._rtfc.compress(data) 21 | -------------------------------------------------------------------------------- /refinery/units/formats/pe/dotnet/dnblob.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units import Unit 4 | from refinery.lib.dotnet.header import DotNetHeader 5 | 6 | 7 | class dnblob(Unit): 8 | """ 9 | Extracts all blobs defined in the `#Blob` stream of .NET executables. 10 | """ 11 | def process(self, data): 12 | header = DotNetHeader(data, parse_resources=False) 13 | for blob in header.meta.Streams.Blob.values(): 14 | yield blob 15 | 16 | @classmethod 17 | def handles(cls, data): 18 | from refinery.lib.id import is_likely_pe_dotnet 19 | return is_likely_pe_dotnet(data) 20 | -------------------------------------------------------------------------------- /refinery/units/formats/pe/dotnet/dnhdr.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units.formats.pe.dotnet import Arg, JSONEncoderUnit 4 | from refinery.lib.dotnet.header import DotNetHeader 5 | 6 | 7 | class dnhdr(JSONEncoderUnit): 8 | """ 9 | Expects data that has been formatted with the `BinaryFormatter` class. The 10 | output is a representation of the deserialized data in JSON format. 11 | """ 12 | def __init__( 13 | self, 14 | resources: Arg.Switch('-r', '--resources', help='Also parse .NET resources.') = False, 15 | encode=None, digest=None 16 | ): 17 | super().__init__(encode=encode, digest=digest, resources=resources) 18 | 19 | def process(self, data): 20 | dn = DotNetHeader(data, parse_resources=self.args.resources) 21 | dn = { 22 | 'Head': dn.head, 23 | 'Meta': dn.meta 24 | } 25 | 26 | if self.args.resources: 27 | dn['RSRC'] = dn.resources 28 | 29 | return self.to_json(dn) 30 | 31 | @classmethod 32 | def handles(cls, data): 33 | from refinery.lib.id import is_likely_pe_dotnet 34 | return is_likely_pe_dotnet(data) 35 | -------------------------------------------------------------------------------- /refinery/units/formats/pe/dotnet/dnrc.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units.formats import PathExtractorUnit, UnpackResult 4 | from refinery.lib.dotnet.header import DotNetHeader 5 | 6 | 7 | class dnrc(PathExtractorUnit): 8 | """ 9 | Extracts all .NET resources whose name matches any of the given patterns 10 | and outputs them. Use the `refinery.units.formats.pe.dotnet.dnmr` unit to 11 | extract subfiles from managed .NET resources. 12 | """ 13 | def unpack(self, data): 14 | header = DotNetHeader(data) 15 | 16 | if not header.resources: 17 | if self.args.list: 18 | return 19 | raise ValueError('This file contains no resources.') 20 | 21 | for resource in header.resources: 22 | yield UnpackResult(resource.Name, resource.Data) 23 | 24 | @classmethod 25 | def handles(cls, data): 26 | from refinery.lib.id import is_likely_pe_dotnet 27 | return is_likely_pe_dotnet(data) 28 | -------------------------------------------------------------------------------- /refinery/units/formats/pe/peoverlay.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units.formats.pe import OverlayUnit 4 | 5 | 6 | class peoverlay(OverlayUnit): 7 | """ 8 | Returns the overlay of a PE file, i.e. anything that may have been appended to the file. 9 | This does not include digital signatures. Use `refinery.pestrip` to obtain only the body 10 | of the PE file after removing the overlay. 11 | """ 12 | def process(self, data: bytearray) -> bytearray: 13 | size = self._get_size(data) 14 | try: 15 | data[:size] = [] 16 | except Exception: 17 | return data[size:] 18 | else: 19 | return data 20 | -------------------------------------------------------------------------------- /refinery/units/formats/pe/pestrip.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units.formats.pe import OverlayUnit 4 | 5 | 6 | class pestrip(OverlayUnit): 7 | """ 8 | Removes the overlay of a PE file and returns the main executable. Use `refinery.peoverlay` to 9 | extract the overlay. 10 | """ 11 | 12 | def process(self, data: bytearray) -> bytearray: 13 | size = self._get_size(data) 14 | try: 15 | data[size:] = [] 16 | except Exception: 17 | data = data[:size] 18 | else: 19 | return data 20 | -------------------------------------------------------------------------------- /refinery/units/formats/pkcs7sig.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units import Unit, Arg 4 | from refinery.units.formats.pe.pemeta import pemeta 5 | from refinery.units.sinks.ppjson import ppjson 6 | 7 | 8 | class pkcs7sig(Unit): 9 | """ 10 | Converts PKCS7 encoded signatures into a human-readable JSON representation. This can be used 11 | to parse authenticode signatures appended to files that are not PE files to get the same output 12 | that is produced by the pemeta unit. 13 | """ 14 | def __init__(self, tabular: Arg('-t', help='Print information in a table rather than as JSON') = False): 15 | super().__init__(tabular=tabular) 16 | 17 | def process(self, data: bytes): 18 | json = pemeta.parse_signature(data) 19 | yield from ppjson(tabular=self.args.tabular)._pretty_output(json, indent=4, ensure_ascii=False) 20 | -------------------------------------------------------------------------------- /refinery/units/formats/pyc.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from __future__ import annotations 4 | 5 | from datetime import datetime 6 | 7 | from refinery.units.formats.archive import ArchiveUnit 8 | from refinery.units.formats.archive.xtpyi import decompile_buffer, extract_code_from_buffer 9 | from refinery.lib.meta import metavars 10 | 11 | 12 | class pyc(ArchiveUnit): 13 | """ 14 | Decompiles Python bytecode (PYC) files back to source code. A known limitation is that it does 15 | not work on recent Python versions, but anything below 3.9 should work. 16 | """ 17 | 18 | def unpack(self, data): 19 | input_path = metavars(data).get(self.args.path.decode(self.codec)) 20 | for k, code in enumerate(extract_code_from_buffer(bytes(data), input_path)): 21 | path = code.container.co_filename or F'__unknown_name_{k:02d}.py' 22 | date = datetime.fromtimestamp(code.timestamp) 23 | data = decompile_buffer(code) 24 | yield self._pack(path, date, data) 25 | -------------------------------------------------------------------------------- /refinery/units/malware/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Contains units that provide malware-specific decoding operations, such as homebrew ciphers that 3 | are only found in specific malware families. 4 | """ 5 | -------------------------------------------------------------------------------- /refinery/units/malware/n40.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from base64 import b16decode 4 | from binascii import Error 5 | 6 | from refinery import Unit, Arg 7 | from refinery.units.blockwise.xor import xor 8 | 9 | 10 | class n40(Unit): 11 | """ 12 | Decrypts hex-encoded strings in various latin-american banker families, including N40. 13 | """ 14 | def __init__(self, key: Arg(help='Decryption key.')): 15 | ... 16 | 17 | def process(self, data): 18 | try: 19 | data = b16decode(data, casefold=True) 20 | except Error: 21 | self.log_info('Input was not hex-encoded; ignoring this step.') 22 | mask = data[1:] | xor(self.args.key) | bytearray 23 | return bytearray(0xFF + b - a if b <= a else b - a for a, b in zip(data, mask)) 24 | -------------------------------------------------------------------------------- /refinery/units/meta/chop.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units import Arg, Unit 4 | from refinery.lib.tools import splitchunks 5 | 6 | 7 | class chop(Unit): 8 | """ 9 | Reinterprets the input as a sequence of equally sized chunks and outputs this sequence. 10 | """ 11 | 12 | def __init__( 13 | self, 14 | size: Arg.Number('size', help='Chop data into chunks of this size'), 15 | step: Arg.Number('step', help=( 16 | 'Optionally specify a step size (which is equal to the size by default) which indicates the number of bytes by ' 17 | 'which the cursor will be increased after extracting a chunk.')) = None, 18 | truncate: Arg.Switch('-t', help=( 19 | 'Truncate possible excess bytes at the end of the input, by default they are appended as a single chunk.')) = False, 20 | ): 21 | return super().__init__(size=size, step=step, truncate=truncate) 22 | 23 | def process(self, data): 24 | view = memoryview(data) 25 | size = self.args.size 26 | step = self.args.step 27 | if size < 1: 28 | raise ValueError('The chunk size has to be a positive integer value.') 29 | yield from splitchunks(view, size, step, self.args.truncate) 30 | -------------------------------------------------------------------------------- /refinery/units/meta/emit.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | A simple tool to output binary data. Multiple arguments are output in framed 5 | format, see `refinery.lib.frame`. 6 | """ 7 | import os 8 | 9 | from refinery.units import Arg, Unit 10 | 11 | 12 | class emit(Unit): 13 | 14 | def __init__(self, *data: Arg(help=( 15 | 'Data to be emitted. If no argument is specified, data is retrieved from ' 16 | 'the clipboard. Multiple arguments are output in framed format.' 17 | ))): 18 | super().__init__(data=data) 19 | 20 | @Unit.Requires('pyperclip') 21 | def _pyperclip(): 22 | import pyperclip 23 | return pyperclip 24 | 25 | def process(self, data): 26 | if self.args.data: 27 | yield from self.args.data 28 | return 29 | if os.name == 'nt': 30 | from refinery.lib.winclip import get_any_data 31 | mode, data = get_any_data() 32 | if mode is not None: 33 | self.log_info(F'retrieved clipboard data in {mode.name} format') 34 | yield data 35 | else: 36 | data = self._pyperclip.paste() 37 | if not data: 38 | return 39 | yield data.encode(self.codec, 'replace') 40 | -------------------------------------------------------------------------------- /refinery/units/meta/group.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units import Arg, Unit, Chunk 4 | 5 | from itertools import islice 6 | 7 | 8 | class group(Unit): 9 | """ 10 | Group incoming chunks into frames of the given size. 11 | """ 12 | def __init__(self, size: Arg.Number(help='Size of each group; must be at least 2.', bound=(2, None))): 13 | super().__init__(size=size) 14 | 15 | def process(self, data: Chunk): 16 | if not data.temp: 17 | return 18 | yield data 19 | yield from islice(data.temp, 0, self.args.size - 1) 20 | 21 | def filter(self, chunks): 22 | it = iter(chunks) 23 | while True: 24 | try: 25 | head: Chunk = next(it) 26 | except StopIteration: 27 | return 28 | head.temp = it 29 | yield head 30 | -------------------------------------------------------------------------------- /refinery/units/meta/groupby.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from collections import defaultdict 4 | from typing import Generator, Iterable 5 | 6 | from refinery.units import Arg, Unit, Chunk 7 | from refinery.lib.meta import check_variable_name 8 | 9 | 10 | class groupby(Unit): 11 | """ 12 | Group incoming chunks by the contents of a meta variable. Note that the unit 13 | blocks and cannot stream any output until the input frame is consumed: It has 14 | to read every input chunk to make sure that all groupings are complete. 15 | """ 16 | def __init__(self, name: Arg(type=str, help='name of the meta variable')): 17 | super().__init__(name=check_variable_name(name)) 18 | 19 | def process(self, data): 20 | yield from data.temp 21 | 22 | def filter(self, chunks: Iterable[Chunk]) -> Generator[Chunk, None, None]: 23 | name = self.args.name 24 | members = defaultdict(list) 25 | for chunk in chunks: 26 | try: 27 | value = chunk.meta[name] 28 | except KeyError: 29 | value = None 30 | members[value].append(chunk) 31 | for chunklist in members.values(): 32 | dummy = chunklist[0] 33 | dummy.temp = chunklist 34 | yield dummy 35 | -------------------------------------------------------------------------------- /refinery/units/meta/iffc.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.lib.types import INF 4 | from refinery.units.meta import Arg, ConditionalUnit 5 | 6 | 7 | class iffc(ConditionalUnit, docs='{0}{p}{1}'): 8 | """ 9 | Filter incoming chunks depending on whether their size is within any of the given bounds. 10 | """ 11 | def __init__( 12 | self, 13 | *bounds: Arg.Bounds(help='Specifies an (inclusive) range to check for.', intok=True), 14 | retain=False, 15 | ): 16 | if not bounds: 17 | raise ValueError('cannot filter for size without specifying any bounds') 18 | super().__init__( 19 | bounds=bounds, 20 | retain=retain, 21 | ) 22 | 23 | def match(self, chunk): 24 | length: int = len(chunk) 25 | for bounds in self.args.bounds: 26 | if isinstance(bounds, int): 27 | if length == bounds: 28 | return True 29 | if isinstance(bounds, slice): 30 | a = bounds.start or 0 31 | b = bounds.stop or INF 32 | t = bounds.step or 1 33 | if a <= length <= b and not (length - a) % t: 34 | return True 35 | return False 36 | -------------------------------------------------------------------------------- /refinery/units/meta/iffs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units.meta import Arg, ConditionalUnit 4 | 5 | 6 | class iffs(ConditionalUnit, docs='{0}{p}{1}'): 7 | """ 8 | Filter incoming chunks depending on whether they contain a given binary substring. 9 | """ 10 | def __init__( 11 | self, 12 | needle: Arg(help='the string to search for'), 13 | retain=False, 14 | ): 15 | super().__init__( 16 | needle=needle, 17 | retain=retain, 18 | ) 19 | 20 | def match(self, chunk): 21 | return self.args.needle in chunk 22 | -------------------------------------------------------------------------------- /refinery/units/meta/iffx.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units.pattern import SingleRegexUnit 4 | from refinery.units.meta import ConditionalUnit 5 | 6 | 7 | class iffx(SingleRegexUnit, ConditionalUnit, docs='{0}{p}{1}'): 8 | """ 9 | Filter incoming chunks by discarding those that do not match the given 10 | regular expression. 11 | """ 12 | def match(self, chunk): 13 | return bool(self.matcher(chunk)) 14 | -------------------------------------------------------------------------------- /refinery/units/meta/rmv.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units import Unit, Chunk, Arg 4 | from refinery.lib.meta import metavars 5 | 6 | 7 | class rmv(Unit): 8 | """ 9 | Short for "ReMove Variable": Removes meta variables that were created in the current frame. If no 10 | variable names are given, the unit removes all of them. Note that this can recover variables from 11 | outer frames that were previously shadowed. 12 | """ 13 | def __init__(self, *names: Arg(type=str, metavar='name', help='Name of a variable to be removed.')): 14 | super().__init__(names=names) 15 | 16 | def process(self, data: Chunk): 17 | meta = metavars(data) 18 | keys = self.args.names or list(meta.variable_names()) 19 | for key in keys: 20 | meta.discard(key) 21 | return data 22 | -------------------------------------------------------------------------------- /refinery/units/misc/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Units whose purpose is narrow or very special and does not fit well into any 5 | other category. 6 | """ 7 | -------------------------------------------------------------------------------- /refinery/units/misc/nop.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units import Unit 4 | from refinery.lib.argparser import ArgumentParserWithKeywordHooks 5 | from refinery.lib.tools import documentation 6 | 7 | 8 | class NopArgParser(ArgumentParserWithKeywordHooks): 9 | def parse_args(self, args, namespace=None): 10 | parsed, _ = self.parse_known_args(args, namespace=namespace) 11 | return parsed 12 | 13 | 14 | class nop(Unit): 15 | """ 16 | The unit generates the exact output that was received as input. All unknown arguments passed 17 | to nop are completely ignored, which is different from the behavior of other units. As such, 18 | nop can be used to comment out other units in longer refinery pipelines by simply prefixing a 19 | command with nop. 20 | """ 21 | @classmethod 22 | def argparser(cls, **keywords): 23 | argp = NopArgParser( 24 | keywords, prog=cls.name, description=documentation(cls), add_help=False) 25 | argp.set_defaults(nesting=0) 26 | return cls._interface(argp) 27 | -------------------------------------------------------------------------------- /refinery/units/obfuscation/js/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Deobfuscation of JavaScript documents. 5 | """ 6 | -------------------------------------------------------------------------------- /refinery/units/obfuscation/js/arrays.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | import re 4 | 5 | from refinery.lib.patterns import formats 6 | from refinery.units.obfuscation import Deobfuscator, StringLiterals 7 | 8 | 9 | class deob_js_arrays(Deobfuscator): 10 | """ 11 | JavaScript deobfuscator to turn `["Z", "t", "s", "e"][0]` into `"Z"`. 12 | """ 13 | 14 | def deobfuscate(self, data): 15 | strlit = StringLiterals(formats.string, data) 16 | 17 | @strlit.outside 18 | def litpick(match: re.Match[str]): 19 | try: 20 | array = match[1] 21 | index = int(match[2]) 22 | lpick = array.split(',')[index].strip() 23 | self.log_debug(lambda: F'{lpick} = {match[0]}') 24 | except (TypeError, IndexError): 25 | lpick = match[0] 26 | return lpick 27 | 28 | p = R'\s{{0,5}}'.join([ 29 | '\\[', '((?:{i}|{s})', '(?:,', '(?:{i}|{s})', ')*)', '\\]', '\\[', '({i})', '\\]' 30 | ]).format(i=formats.integer, s=formats.string) 31 | return re.sub(p, litpick, data) 32 | -------------------------------------------------------------------------------- /refinery/units/obfuscation/js/comments.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | import re 4 | 5 | from refinery.lib.patterns import formats 6 | from refinery.units.obfuscation import Deobfuscator, StringLiterals 7 | 8 | 9 | class deob_js_comments(Deobfuscator): 10 | """ 11 | JavaScript deobfuscator that removes comments from the script. 12 | """ 13 | def deobfuscate(self, data): 14 | strings = StringLiterals(formats.string, data) 15 | @strings.outside 16 | def remove(_): return '' 17 | 18 | data = re.sub(R'/\*.*?\*/', remove, data, flags=re.DOTALL) 19 | data = re.sub(R'(?m)//.*$', remove, data) 20 | return data 21 | -------------------------------------------------------------------------------- /refinery/units/obfuscation/js/getattr.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | import re 4 | 5 | from refinery.lib.patterns import formats 6 | from refinery.units.obfuscation import Deobfuscator, StringLiterals 7 | 8 | 9 | class deob_js_getattr(Deobfuscator): 10 | """ 11 | JavaScript deobfuscator to turn `WScript["CreateObject"]` into `WScript.CreateObject`. 12 | """ 13 | 14 | def deobfuscate(self, data): 15 | strlit = StringLiterals(formats.string, data) 16 | 17 | @strlit.outside 18 | def dottify(match: re.Match[str]): 19 | name = match[2][1:-1] 20 | if name.isidentifier(): 21 | return F'{match[1]}.{name}' 22 | return match[0] 23 | 24 | return re.sub(FR'(\w+)\[({formats.string})\]', dottify, data) 25 | -------------------------------------------------------------------------------- /refinery/units/obfuscation/js/tuples.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | import re 4 | 5 | from refinery.lib.patterns import formats 6 | from refinery.units.obfuscation import Deobfuscator 7 | 8 | 9 | class deob_js_tuples(Deobfuscator): 10 | """ 11 | JavaScript deobfuscator to turn `("Z", "t", "s", "e")` into `"e"`. 12 | """ 13 | 14 | def deobfuscate(self, data): 15 | 16 | def litpick(match): 17 | try: 18 | array = match[1] 19 | lpick = array.split(',')[-1].strip() 20 | self.log_debug(lambda: F'{lpick} = {match[0]}') 21 | except (TypeError, IndexError): 22 | lpick = match[0] 23 | return lpick 24 | 25 | p = R'\s{{0,5}}'.join([ 26 | '\\(', '((?:{i}|{s})', '(?:,', '(?:{i}|{s})', ')*)', '\\)' 27 | ]).format(i=formats.integer, s=formats.string) 28 | return re.sub(p, litpick, data) 29 | -------------------------------------------------------------------------------- /refinery/units/obfuscation/ps1/b64convert.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from __future__ import annotations 4 | 5 | import re 6 | import base64 7 | 8 | from refinery.units.obfuscation import Deobfuscator 9 | from refinery.units.obfuscation.ps1 import string_unquote, Ps1StringLiterals 10 | from refinery.lib.patterns import formats 11 | 12 | 13 | class deob_ps1_b64convert(Deobfuscator): 14 | 15 | _SENTINEL = re.compile('\\s*'.join( 16 | (re.escape('[System.Convert]::FromBase64String'), '\\(', '({s})', '\\)') 17 | ).format(s=formats.ps1str), flags=re.IGNORECASE) 18 | 19 | def deobfuscate(self, data): 20 | strlit = Ps1StringLiterals(data) 21 | 22 | def replacer(match: re.Match[str]): 23 | if strlit.get_container(match.start()): 24 | return match[0] 25 | try: 26 | string, = string_unquote(match[1]) 27 | except ValueError: 28 | return match[0] 29 | try: 30 | bytes = base64.b64decode(string) 31 | except Exception: 32 | return match[0] 33 | return '@({})'.format(','.join(F'0x{b:02X}' for b in bytes)) 34 | 35 | return self._SENTINEL.sub(replacer, data) 36 | -------------------------------------------------------------------------------- /refinery/units/obfuscation/ps1/cases.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | import re 4 | 5 | from refinery.units.obfuscation import Deobfuscator, outside 6 | from refinery.lib.patterns import formats 7 | 8 | 9 | class deob_ps1_cases(Deobfuscator): 10 | _NAMES = [ 11 | '-BXor', 12 | '-Exec Bypass', 13 | '-NoLogo', 14 | '-NonInter', 15 | '-Replace', 16 | '-Windows Hidden', 17 | '.Invoke', 18 | 'Assembly', 19 | 'Byte', 20 | 'Char', 21 | 'ChildItem', 22 | 'CreateThread', 23 | 'Get-Variable', 24 | 'GetType', 25 | 'IntPtr', 26 | 'Invoke-Expression', 27 | 'Invoke', 28 | 'Length', 29 | 'Net.WebClient', 30 | 'PowerShell', 31 | 'PSVersionTable', 32 | 'Set-Item', 33 | 'Set-Variable', 34 | 'Start-Sleep', 35 | 'ToString', 36 | 'Type', 37 | 'Value', 38 | 'Void', 39 | ] 40 | 41 | @outside(formats.ps1str) 42 | def deobfuscate(self, data): 43 | for name in self._NAMES: 44 | data = re.sub(RF'\b{re.escape(name)}\b', name, data, flags=re.IGNORECASE) 45 | return data 46 | -------------------------------------------------------------------------------- /refinery/units/obfuscation/ps1/escape.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | import re 4 | 5 | from refinery.units.obfuscation import Deobfuscator 6 | from refinery.units.obfuscation.ps1 import Ps1StringLiterals 7 | 8 | 9 | class deob_ps1_escape(Deobfuscator): 10 | 11 | def deobfuscate(self, data): 12 | strlit = Ps1StringLiterals(data) 13 | @strlit.outside 14 | def repl(m): return m[1] 15 | return re.sub(R'''`([^0abfnrtv`#'"\$])''', repl, data) 16 | -------------------------------------------------------------------------------- /refinery/units/obfuscation/ps1/uncurly.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | import re 4 | 5 | from refinery.units.obfuscation import Deobfuscator 6 | from refinery.units.obfuscation.ps1 import Ps1StringLiterals 7 | 8 | 9 | class deob_ps1_uncurly(Deobfuscator): 10 | """ 11 | PowerShell deobfuscation that removes superfluous curly braces around variable 12 | names that do not require it, i.e. `${variable}` is transformed to just `$variable`. 13 | """ 14 | 15 | _SENTINEL = re.compile(R'\$\{(\w+)\}') 16 | 17 | def deobfuscate(self, data): 18 | strlit = Ps1StringLiterals(data) 19 | @strlit.outside 20 | def strip(m): return F'${m[1]}' 21 | return self._SENTINEL.sub(strip, data) 22 | -------------------------------------------------------------------------------- /refinery/units/obfuscation/vba/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | A package containing deobfuscators for Visual Basic for Applications (VBA). 5 | """ 6 | 7 | 8 | def string_unquote(string: str) -> str: 9 | if string[0] != '"' or string[~0] != '"': 10 | raise ValueError(string) 11 | return string[1:-1].replace('""', '"') 12 | 13 | 14 | def string_quote(string: str) -> str: 15 | return '"{}"'.format(string.replace('"', '""')) 16 | -------------------------------------------------------------------------------- /refinery/units/obfuscation/vba/brackets.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | import re 4 | 5 | from refinery.units.obfuscation import Deobfuscator, StringLiterals 6 | from refinery.lib.patterns import formats 7 | 8 | 9 | class deob_vba_brackets(Deobfuscator): 10 | _SENTINEL = re.compile( 11 | RF'''(? 1: 27 | return match[0] 28 | return '"{}"'.format(c) 29 | 30 | return re.sub(R'(?i)\bchrw?\s*\(\s*(\d+)\s*\)', evaluate_char_function, data) 31 | -------------------------------------------------------------------------------- /refinery/units/obfuscation/vba/comments.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | import re 4 | 5 | from refinery.units.obfuscation import Deobfuscator 6 | 7 | 8 | class deob_vba_comments(Deobfuscator): 9 | def deobfuscate(self, data): 10 | return re.sub(R"(?im)^\s{0,20}(?:'|rem\b|dim\b).*(?:\Z|$\n\r?)", '', data) 11 | -------------------------------------------------------------------------------- /refinery/units/obfuscation/vba/stringreplace.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from __future__ import annotations 4 | import re 5 | 6 | from refinery.lib.patterns import formats 7 | from refinery.units.obfuscation import Deobfuscator, StringLiterals 8 | from refinery.units.obfuscation.vba import string_quote, string_unquote 9 | 10 | 11 | class deob_vba_stringreplace(Deobfuscator): 12 | 13 | _SENTINEL = re.compile(( 14 | R'(?i)\bReplace\s*\(' # the replace call 15 | R'\s*({s}),' # haystack (with brackets) 16 | R'\s*({s}),' # needle (with brackets) 17 | R'\s*({s})\s*\)' # insert (with brackets) 18 | ).format(s=formats.vbastr), flags=re.IGNORECASE) 19 | 20 | def deobfuscate(self, data): 21 | strlit = StringLiterals(formats.vbastr, data) 22 | 23 | @strlit.outside 24 | def replacement(match: re.Match[str]): 25 | return string_quote( 26 | string_unquote(match[1]).replace( 27 | string_unquote(match[2]), 28 | string_unquote(match[3]) 29 | ) 30 | ) 31 | 32 | return self._SENTINEL.sub(replacement, data) 33 | -------------------------------------------------------------------------------- /refinery/units/obfuscation/vba/stringreverse.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from __future__ import annotations 4 | import re 5 | 6 | from refinery.lib.patterns import formats 7 | from refinery.units.obfuscation import Deobfuscator, StringLiterals 8 | from refinery.units.obfuscation.vba import string_quote, string_unquote 9 | 10 | 11 | class deob_vba_stringreverse(Deobfuscator): 12 | 13 | _SENTINEL = re.compile(( 14 | R'(?i)\bStrReverse\s*\(' # the reverse call 15 | R'\s*({s})\s*\)' # string 16 | ).format(s=formats.vbastr), flags=re.IGNORECASE) 17 | 18 | def deobfuscate(self, data): 19 | strlit = StringLiterals(formats.vbastr, data) 20 | 21 | @strlit.outside 22 | def replacement(match: re.Match[str]): 23 | return string_quote(''.join(reversed(string_unquote(match[1])))) 24 | 25 | return self._SENTINEL.sub(replacement, data) 26 | -------------------------------------------------------------------------------- /refinery/units/obfuscation/vba/vba.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | import re 4 | 5 | from refinery.units import Unit 6 | 7 | 8 | class deob_vba_chr_literals(Unit): 9 | def process(self, data): 10 | def _chr(m): 11 | code = int(m[1], 0) 12 | if code == 34: 13 | return B'""""' 14 | return B'"%s"' % chr(code).encode('unicode_escape') 15 | data = re.sub(BR'Chr\((\d+x?\d+)\)', _chr, data, flags=re.IGNORECASE) 16 | data = re.sub(BR'"\s*\&\s*"', B'', data) 17 | return data 18 | -------------------------------------------------------------------------------- /refinery/units/pattern/carve_rtf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | import re 4 | 5 | from refinery.units import Unit 6 | 7 | 8 | class carve_rtf(Unit): 9 | """ 10 | Extracts anything from the input data that looks like an RTF document. 11 | """ 12 | 13 | def process(self, data: bytearray): 14 | pos = 0 15 | mem = memoryview(data) 16 | sig = re.escape(b'{\\rtf') 17 | 18 | while True: 19 | match = re.search(sig, mem[pos:], flags=re.IGNORECASE) 20 | if match is None: 21 | break 22 | pos = pos + match.start() 23 | end = pos + 1 24 | depth = 1 25 | while depth and end < len(mem): 26 | if mem[end] == 0x7B: # { 27 | depth += 1 28 | if mem[end] == 0x7D: # } 29 | depth -= 1 30 | end += 1 31 | if depth > 0: 32 | break 33 | yield self.labelled(mem[pos:end], offset=pos) 34 | pos = end 35 | -------------------------------------------------------------------------------- /refinery/units/pattern/mimewords.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | import re 4 | import codecs 5 | 6 | from email.header import decode_header 7 | 8 | from refinery.units import Unit 9 | from refinery.lib.decorators import unicoded 10 | 11 | 12 | class mimewords(Unit): 13 | """ 14 | Implements the decoding of MIME encoded-word syntax from RFC-2047. 15 | """ 16 | 17 | @unicoded 18 | def process(self, data: str) -> str: 19 | def replacer(match): 20 | self.log_info('encoded mime word:', match[0]) 21 | decoded, = decode_header(match[0]) 22 | raw, codec = decoded 23 | return codecs.decode(raw, codec, errors='surrogateescape') 24 | return re.sub(R"=(?:\?[^\?]*){3}\?=", replacer, data) 25 | -------------------------------------------------------------------------------- /refinery/units/pattern/resplit.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units.pattern import SingleRegexUnit 4 | 5 | 6 | class resplit(SingleRegexUnit): 7 | """ 8 | Splits the data at the given regular expression and returns the sequence of 9 | chunks between the separators. By default, the input is split along line breaks. 10 | """ 11 | 12 | def __init__( 13 | self, regex=RB'\r?\n', multiline=False, ignorecase=False, count=0 14 | ): 15 | super().__init__(regex=regex, multiline=multiline, ignorecase=ignorecase, count=count) 16 | 17 | def process(self, data): 18 | view = memoryview(data) 19 | cursor = 0 20 | count = self.args.count 21 | for k, match in enumerate(self.regex.finditer(view), 2): 22 | yield view[cursor:match.start()] 23 | cursor = match.end() 24 | yield from match.groups() 25 | if k > count > 0: 26 | break 27 | yield view[cursor:] 28 | -------------------------------------------------------------------------------- /refinery/units/pattern/xtw.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from __future__ import annotations 4 | 5 | import re 6 | 7 | from refinery.units.pattern import PatternExtractor 8 | from refinery.units import RefineryCriticalException 9 | from refinery.lib.patterns import wallets 10 | 11 | 12 | class xtw(PatternExtractor): 13 | """ 14 | Extract Wallets: Extracts anything that looks like a cryptocurrency wallet address. 15 | This works similar to the `refinery.xtp` unit. 16 | """ 17 | 18 | def __init__(self, stripspace=False, duplicates=False, longest=False, take=None): 19 | self.superinit(super(), **vars(), ascii=True, utf16=True) 20 | 21 | def process(self, data): 22 | pattern = '|'.join(FR'(?P<{p.name}>\b{p.value}\b)' for p in wallets) 23 | pattern = FR'\b{pattern}\b'.encode('latin1') 24 | 25 | def check(match: re.Match[bytes]): 26 | for name, value in match.groupdict().items(): 27 | if value is not None: 28 | break 29 | else: 30 | raise RefineryCriticalException('Received empty match.') 31 | return self.labelled(value, kind=name) 32 | 33 | yield from self.matches_filtered(memoryview(data), pattern, check) 34 | -------------------------------------------------------------------------------- /refinery/units/strings/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Simple operations on strings, such as concatenation, replacement, slicing, 5 | trimming, etcetera. 6 | """ 7 | -------------------------------------------------------------------------------- /refinery/units/strings/cca.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units import Arg, Unit 4 | 5 | 6 | class cca(Unit): 7 | """ 8 | Short for ConCatAppend: This unit concatenates the input data with its argument by 9 | appending the latter to the former. See also `refinery.ccp` for the unit that prepends 10 | instead. 11 | """ 12 | 13 | def __init__(self, data: Arg(help='Binary string to be appended to the input.')): 14 | super().__init__(data=data) 15 | 16 | def process(self, data: bytearray): 17 | data.extend(self.args.data) 18 | return data 19 | -------------------------------------------------------------------------------- /refinery/units/strings/ccp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units import Arg, Unit 4 | 5 | 6 | class ccp(Unit): 7 | """ 8 | Short for ConCatPrepend: This unit concatenates the input data with its argument by 9 | prepending the latter to the former. See also `refinery.cca` for the unit that appends 10 | instead. 11 | """ 12 | 13 | def __init__(self, data: Arg(help='Binary string to be prepended to the input.')): 14 | super().__init__(data=data) 15 | 16 | def process(self, data: bytearray): 17 | data[:0] = self.args.data 18 | return data 19 | -------------------------------------------------------------------------------- /refinery/units/strings/clower.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units import Unit 4 | 5 | 6 | class clower(Unit): 7 | """ 8 | Stands for "Convert to LOWER case"; The unit simply converts all latin alphabet chacters in the 9 | input to lowercase. 10 | """ 11 | def process(self, data): 12 | return data.lower() 13 | -------------------------------------------------------------------------------- /refinery/units/strings/cswap.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units import Unit 4 | 5 | 6 | class cswap(Unit): 7 | """ 8 | Swap the case of the input string; all lowercase letters are turned into their uppercase 9 | variant and vice-versa. 10 | """ 11 | def process(self, data: bytearray): 12 | lcase = bytes(range(B'a'[0], B'z'[0] + 1)) 13 | ucase = bytes(range(B'A'[0], B'Z'[0] + 1)) 14 | delta = lcase[0] - ucase[0] 15 | for k, letter in enumerate(data): 16 | if letter in ucase: 17 | data[k] += delta 18 | elif letter in lcase: 19 | data[k] -= delta 20 | return data 21 | -------------------------------------------------------------------------------- /refinery/units/strings/cupper.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units import Unit 4 | 5 | 6 | class cupper(Unit): 7 | """ 8 | Stands for "Convert to UPPER case"; The unit simply converts all latin alphabet chacters in the 9 | input to uppercase. 10 | """ 11 | def process(self, data): 12 | return data.upper() 13 | -------------------------------------------------------------------------------- /refinery/units/strings/ngrams.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units import Arg, Unit 4 | from refinery.lib.tools import integers_of_slice 5 | 6 | 7 | class ngrams(Unit): 8 | """ 9 | Extract all n-grams from the input. The algorithm is naive, i.e. it simply iterates all n-grams 10 | and deduplicates using a set data structure. The number n is taken from an arbitrary range given 11 | as a Python slice expression. 12 | """ 13 | def __init__( 14 | self, size: Arg.Bounds( 15 | help='Specifies the sizes of each n-gram, i.e. the number n. Defaults to {default}.') = slice(2, None), 16 | ): 17 | super().__init__(size=size) 18 | 19 | def process(self, data: bytearray): 20 | for n in integers_of_slice(self.args.size): 21 | self.log_info(F'emitting {n}-grams') 22 | if n > len(data): 23 | break 24 | deduplicator = set() 25 | view = memoryview(data) 26 | for index in range(len(data) - n + 1): 27 | block = bytes(view[index:index + n]) 28 | if block in deduplicator: 29 | continue 30 | deduplicator.add(block) 31 | yield self.labelled(block, offset=index) 32 | -------------------------------------------------------------------------------- /refinery/units/strings/repl.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units import Arg, Unit 4 | 5 | 6 | class repl(Unit): 7 | """ 8 | Performs a simple binary string replacement on the input data. 9 | """ 10 | 11 | def __init__( 12 | self, 13 | search : Arg(help='This is the search term.'), 14 | replace: Arg(help='The substitution string. Leave this empty to remove all occurrences of the search term.') = B'', 15 | count : Arg.Number('-n', help='Only replace the given number of occurrences') = -1 16 | ): 17 | super().__init__(search=search, replace=replace, count=count) 18 | 19 | def process(self, data: bytes): 20 | return data.replace( 21 | self.args.search, 22 | self.args.replace, 23 | self.args.count 24 | ) 25 | -------------------------------------------------------------------------------- /refinery/units/strings/termfit.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.units import Arg, Unit 4 | from refinery.lib.tools import terminalfit 5 | from refinery.lib.decorators import unicoded 6 | 7 | 8 | class termfit(Unit): 9 | """ 10 | Reformat incoming text data to fit a certain width. 11 | """ 12 | 13 | def __init__( 14 | self, 15 | width: Arg('width', help='Optionally specify the width, by default the current terminal width is used.') = 0, 16 | delta: Arg.Number('-d', help='Subtract this number from the calculated width (0 by default).') = 0, 17 | tight: Arg.Switch('-t', help='Separate paragraphs by a single line break instead of two.') = False, 18 | ): 19 | super().__init__(width=width, delta=delta, tight=tight) 20 | 21 | @unicoded 22 | def process(self, data: str) -> str: 23 | parsep = '\n' if self.args.tight else '\n\n' 24 | return terminalfit(data, self.args.delta, self.args.width, parsep) 25 | -------------------------------------------------------------------------------- /run-flake.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | import os.path 4 | import sys 5 | 6 | from inspect import stack 7 | from glob import glob 8 | from flake8.api import legacy as flake8 9 | 10 | if __name__ != '__main__': 11 | raise ImportError('This script should not be imported.') 12 | 13 | here = os.path.dirname(os.path.abspath(stack()[0][1])) 14 | 15 | rules = flake8.get_style_guide(ignore=[ 16 | 'E128', # A continuation line is under-indented for a visual indentation. 17 | 'E203', # Colons should not have any space before them. 18 | 'E701', # Multiple statements on one line (colon) 19 | 'E704', # Multiple statements on one line (def) 20 | 'W503', # Line break occurred before a binary operator 21 | ], max_line_length=140) 22 | 23 | report = rules.check_files(glob( 24 | os.path.join(here, 'refinery', '**', '*.py'), recursive=True)) 25 | errors = len(report.get_statistics('E')) 26 | 27 | if not errors: 28 | print('Success! No FLAKE8 violations were found.') 29 | 30 | sys.exit(errors) 31 | -------------------------------------------------------------------------------- /run-ldeps.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Generates the list of dependency categories. 5 | """ 6 | from __future__ import annotations 7 | 8 | from refinery import __unit_loader__ 9 | from refinery import * 10 | 11 | all_optional: set[str] = set() 12 | all_required: set[str] = set() 13 | extras: dict[str, set[str]] = {'all': all_optional} 14 | 15 | with __unit_loader__: 16 | for executable in __unit_loader__.cache.values(): 17 | if executable.optional_dependencies: 18 | for key, deps in executable.optional_dependencies.items(): 19 | bucket = extras.setdefault(key, set()) 20 | bucket.update(deps) 21 | all_optional.update(deps) 22 | if executable.required_dependencies: 23 | all_required.update(executable.required_dependencies) 24 | 25 | for category, deps in extras.items(): 26 | print(category) 27 | deps = list(deps) 28 | deps.sort() 29 | for dep in deps: 30 | print('\t', dep) 31 | -------------------------------------------------------------------------------- /run-tests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Runs all tests from the command line. 5 | """ 6 | import argparse 7 | import unittest 8 | import os 9 | import sys 10 | from inspect import stack 11 | 12 | from refinery.lib.environment import environment, LogLevel 13 | 14 | here = os.path.dirname(os.path.abspath(stack()[0][1])) 15 | 16 | argp = argparse.ArgumentParser() 17 | argp.add_argument('pattern', type=lambda s: str(s).strip('*'), nargs='?', default='*', 18 | help='run all tests whose file name contains the given pattern.') 19 | args = argp.parse_args() 20 | 21 | os.chdir('test') 22 | os.environ[environment.verbosity.key] = LogLevel.DETACHED.name 23 | 24 | suite = unittest.TestLoader().discover('test', F'test_*{args.pattern}*') 25 | tests = unittest.TextTestRunner(verbosity=2) 26 | result = tests.run(suite) 27 | sys.exit(0 if result.wasSuccessful() else 1) 28 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | license_files = LICENSE.md -------------------------------------------------------------------------------- /shells/README.md: -------------------------------------------------------------------------------- 1 | # Shell Support Scripts 2 | 3 | This folder contains additional scripts that help setting up refinery for various shells. 4 | Currently, it only contains a script for [zsh](zsh) which fixes [certain issues with the 5 | framing syntax](https://github.com/binref/refinery/discussions/18). -------------------------------------------------------------------------------- /shells/zsh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/zsh 2 | 3 | function alias-noglob { 4 | while read -r entrypoint; do 5 | alias $entrypoint="noglob $entrypoint" 6 | done 7 | } 8 | 9 | python <','').replace('',''))''' 17 | ) 18 | test = data | self.load(b'E15Vb0ro8C-RQVm_HonJQeYM7QqH_QL6GXe3BpqaJJw=') | bytes 19 | self.assertEqual(test, goal) 20 | -------------------------------------------------------------------------------- /test/units/crypto/cipher/test_latin_ciphers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from ... import TestUnitBase 4 | 5 | 6 | class TestLatinCiphers(TestUnitBase): 7 | 8 | def assertEqualOutputs(self, u1, u2, size): 9 | data = self.generate_random_buffer(size) 10 | args = dict( 11 | key=self.generate_random_buffer(32), 12 | nonce=self.generate_random_buffer(8) 13 | ) 14 | unit1 = self.ldu(u1, **args) 15 | unit2 = self.ldu(u2, **args) 16 | self.assertEqual(unit1(data), unit2(data)) 17 | 18 | def test_salsa(self): 19 | for size in (3, 5, 12, 56, 2013): 20 | self.assertEqualOutputs('salsa', 'salsa20', size) 21 | 22 | def test_chacha(self): 23 | for size in (3, 5, 12, 56, 2013): 24 | self.assertEqualOutputs('chacha', 'chacha20', size) 25 | -------------------------------------------------------------------------------- /test/units/crypto/cipher/test_rc2.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from ... import TestUnitBase 4 | 5 | 6 | class TestRC2(TestUnitBase): 7 | 8 | def test_dotnet_eks_derivation(self): 9 | data = b'U4dbuch3yV0yPCA7W+zIag' 10 | rc2 = self.load(b'System', derive_eks=True, mode='cbc') 11 | b64 = self.ldu('b64') 12 | self.assertEqual(data | b64 | rc2 | str, 'InvokeMethod') 13 | 14 | def test_bare_pycryptodome_vectors(self): 15 | vectors = { 16 | 10: b'\xb8\x96\xb7\x2a\x5b\x4a\x03\x0c', 17 | 20: b'\x22\x0b\xfd\x7a\xf4\x32\x8c\xb0', 18 | 30: b'\xf8\x1d\xbe\xfe\xff\x9e\x70\xf4', 19 | 40: b'\x52\xe3\x32\xd1\xb5\xd6\xb0\x77', 20 | 50: b'\xb5\xc0\x92\xd6\xef\x75\x34\x85', 21 | 60: b'\xdc\xc3\xa7\xdd\xa6\x5f\x5b\x7a', 22 | 70: b'\x32\x55\x67\x68\x7a\x39\x8a\xfe', 23 | 80: b'\xac\xb0\xd2\xe4\x63\xbb\xec\xb2', 24 | 90: b'\x6a\xf7\x13\x91\xf0\xe1\x44\xbc', 25 | } 26 | for k, vec in vectors.items(): 27 | rc2 = self.load(B'A' * k, mode='ecb', reverse=True, raw=True) 28 | self.assertEqual(bytes(range(8)) | rc2 | bytes, vec) 29 | -------------------------------------------------------------------------------- /test/units/crypto/cipher/test_rc4.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from ... import TestUnitBase 4 | 5 | 6 | class TestRC4(TestUnitBase): 7 | 8 | def test_discard(self): 9 | goal = bytes.fromhex('DF81C217EF2D066F41891527293C7AAD') 10 | data = b'1RQCmvqeSxBYnXXD' 11 | self.assertEqual(data | self.load(b'C9J2oU8orRsjZ73J', discard=505) | bytes, goal) 12 | -------------------------------------------------------------------------------- /test/units/crypto/cipher/test_rc4mod.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from ... import TestUnitBase 4 | 5 | 6 | class TestModifiedRC4(TestUnitBase): 7 | 8 | def test_real_world_01(self): 9 | unit = self.load(( 10 | BR'E0qaj^)K9M76nnztYKRyDUBJHox?I75eS>mFoLo3WbHpxmYY9!yYJ?Qgy_' 11 | BR'T 0x0000095e', dasm) 14 | self.assertIn(B'lookupswitch ___default => 0x000002b2', dasm) 15 | self.assertIn(B'tableswitch ___default => 0x00000049', dasm) 16 | self.assertIn(B'00000049: 2b', dasm) 17 | self.assertIn(B'invokestatic org.jnativehook.keyboard.NativeKeyEvent::getKeyText', dasm) 18 | -------------------------------------------------------------------------------- /test/units/formats/macho/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/binref/refinery/44238b79ae1666d01d6d9cdf8eeb9818154cd843/test/units/formats/macho/__init__.py -------------------------------------------------------------------------------- /test/units/formats/office/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/binref/refinery/44238b79ae1666d01d6d9cdf8eeb9818154cd843/test/units/formats/office/__init__.py -------------------------------------------------------------------------------- /test/units/formats/office/test_officecrypt.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from ... import TestUnitBase 4 | from .test_doctxt import PARAGRAPHS 5 | 6 | 7 | class TestOfficeCrypt(TestUnitBase): 8 | 9 | def test_simple_samples(self): 10 | crypt = self.load('space-cowboy') 11 | doctxt = self.ldu('doctxt') 12 | data = self.download_sample('e12a6f21e62a300ee86a26d8a1f876113bebf1b52709421d4894f832dd54bcf1') 13 | output = str(data | crypt | doctxt) 14 | for p in PARAGRAPHS: 15 | self.assertIn(p, output) 16 | -------------------------------------------------------------------------------- /test/units/formats/office/test_vbapc.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | import inspect 4 | from ... import TestUnitBase 5 | 6 | 7 | class TestVBAPC(TestUnitBase): 8 | 9 | def test_stomped_document_01(self): 10 | data = self.download_sample('6d8a0f5949adf37330348cc9a231958ad8fb3ea3a3d905abe5e72dbfd75a3d1d') 11 | unit = self.load() 12 | code = str(data | unit) 13 | goal = inspect.cleandoc( 14 | """ 15 | Function justify_text_to_left(dt As String) As String 16 | On Error Resume Next 17 | Dim ks As String 18 | ks = page_border_width 19 | Dim dl As Long 20 | dl = ((Len(dt) / 2) - 1) 21 | kl = Len(ks) 22 | Dim s As String 23 | s = "" 24 | For i = 0 To dl 25 | Dim c1 As Integer 26 | Dim c2 As Integer 27 | c1 = Val("&H" & Mid(dt, ((i * 2) + 1), 2)) 28 | c2 = Asc(Mid(ks, (i Mod kl) + 1, 1)) 29 | s = s & Chr(c1 Xor c2) 30 | Next 31 | justify_text_to_left = s 32 | End Function 33 | """ 34 | ) 35 | self.assertIn(goal, code) 36 | -------------------------------------------------------------------------------- /test/units/formats/office/test_vbastr.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from ... import TestUnitBase 4 | 5 | 6 | class TestVBAStr(TestUnitBase): 7 | 8 | def test_blogpost_example(self): 9 | # https://nvdp01.github.io/analysis/2022/06/29/extracting-vba-userform-field-values.html 10 | data = self.download_sample('303bc0f4742c61166d05f7a14a25b3c118fa3ba04298b8370071b4ed19f1a987') 11 | test = data | self.load('tabyAeEx3574.tip') | str 12 | self.assertEqual(test, 'nnacShtlaeHW\\putratS\\sm') 13 | 14 | def test_stomped_document_01(self): 15 | data = self.download_sample('6d8a0f5949adf37330348cc9a231958ad8fb3ea3a3d905abe5e72dbfd75a3d1d') 16 | unit = self.load() 17 | strings = list(data | unit) 18 | self.assertEqual(len(strings), 5) 19 | self.assertIn(bytes.fromhex('48 EF BF BD 2C 5C 59'), strings) 20 | self.assertIn(bytes.fromhex('48 EF BF BD 2C 56 37'), strings) 21 | self.assertIn(b'Tahomae', strings) 22 | strings = [ 23 | bytes(s | self.ldu('hex') | self.ldu('xor', 'An2Lcw6Gseh')) 24 | for s in strings if len(s) > 100] 25 | self.assertEqual(len(strings), 2) 26 | self.assertTrue(any(B'function a0_0x4511()' in string for string in strings)) 27 | -------------------------------------------------------------------------------- /test/units/formats/office/test_xlmdeobf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from ... import TestUnitBase 4 | 5 | 6 | class TestXLMMacroDeobfuscator(TestUnitBase): 7 | def test_maldoc(self): 8 | data = self.download_sample( 9 | 'dc44bbfc845fc078cf38b9a3543a32ae1742be8c6320b81cf6cd5a8cee3c696a' 10 | ) 11 | unit = self.load() 12 | code = str(data | unit) 13 | self.assertIn(r'C:\ProgramData\Ropedjo1.ocx', code) 14 | 15 | def test_maldoc_extract_only(self): 16 | data = self.download_sample( 17 | 'dc44bbfc845fc078cf38b9a3543a32ae1742be8c6320b81cf6cd5a8cee3c696a' 18 | ) 19 | unit = self.load(extract_only=True) 20 | code = str(data | unit) 21 | self.assertNotIn(r'C:\ProgramData\Ropedjo1.ocx', code) 22 | self.assertIn(r'"h"&"t"&"tp"&":"&"/"&"/"&', code) 23 | -------------------------------------------------------------------------------- /test/units/formats/office/test_xtdoc.py: -------------------------------------------------------------------------------- 1 | 2 | #!/usr/bin/env python3 3 | # -*- coding: utf-8 -*- 4 | # flake8: noqa 5 | from ... import TestUnitBase 6 | from refinery.lib.loader import load as L 7 | 8 | 9 | class TestDocExtractor(TestUnitBase): 10 | 11 | def test_maldoc(self): 12 | data = self.download_sample('969ff75448ea54feccc0d5f652e00172af8e1848352e9a5877d705fc97fa0238') 13 | pipeline = self.load_pipeline( 14 | 'xtdoc WordDoc | push [| drp | pop junk | repl var:junk | carve -ds b64 | u16 | deob-ps1 | repl var:junk http | xtp url ]') 15 | c2s = pipeline(data) 16 | self.assertIn(B'http://depannage-vehicule-maroc'B'.com/wp-admin/c/', c2s) 17 | -------------------------------------------------------------------------------- /test/units/formats/office/test_xtone.py: -------------------------------------------------------------------------------- 1 | 2 | #!/usr/bin/env python3 3 | # -*- coding: utf-8 -*- 4 | # flake8: noqa 5 | from ... import TestUnitBase 6 | 7 | 8 | class TestOneNoteExtractor(TestUnitBase): 9 | 10 | def test_maldoc(self): 11 | data = self.download_sample('15212428deeeabcd5b11a1b8383c654476a3ea1b19b804e4aca606fac285387f') 12 | out = data | self.load() | {'path': {...}} 13 | self.assertSetEqual(set(out), { 14 | '268ad1c2-aafb-4d3b-91f8-74977fee1cc7.png', 15 | 'b5e524e3-a396-4746-93a3-c5616f0932cc.hta', 16 | '851ac4a2-6395-4508-9aee-289a87e1c342.png', 17 | }) 18 | for key, value in out.items(): 19 | self.assertEqual(len(value), 1) 20 | value, = value 21 | out[key] = value 22 | self.assertContains(data, value) 23 | doc = out['b5e524e3-a396-4746-93a3-c5616f0932cc.hta'] 24 | self.assertContains(doc, b'https:'b'//transfer'b'.sh/get/5dLEvB/sky.bat') 25 | -------------------------------------------------------------------------------- /test/units/formats/office/test_xtrtf.py: -------------------------------------------------------------------------------- 1 | 2 | #!/usr/bin/env python3 3 | # -*- coding: utf-8 -*- 4 | from ... import TestUnitBase 5 | 6 | 7 | class TestRTFExtractor(TestUnitBase): 8 | 9 | def test_maldoc(self): 10 | data = self.download_sample('40f97cf37c136209a65d5582963a72352509eb802da7f1f5b4478a0d9e0817e8') 11 | unit = self.load() 12 | bins = list(unit.process(data)) 13 | self.assertLessEqual( 14 | {b'aaaaaaaaaa.txt', b'SutLzbCFI.txt', b'fbZjJrTooKyVebB.sct'}, 15 | {bin.meta['path'] for bin in bins} 16 | ) 17 | for bin in bins: 18 | if bin.meta['path'] == 'fbZjJrTooKyVebB.sct': 19 | self.assertIn(B'{76456238-5834-4f65-4437-4c3555674232}', bin) 20 | break 21 | -------------------------------------------------------------------------------- /test/units/formats/office/test_xtvba.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from ... import TestUnitBase 4 | 5 | 6 | class TestVBAExtractor(TestUnitBase): 7 | 8 | def test_maldoc(self): 9 | data = self.download_sample('4bdc8e660ff4fb05e5b6c0a2dd70c537817f46ac3270d779fdddc8e459829c08') 10 | unit = self.load() 11 | code = list(data | unit) 12 | self.assertIn(B'http://109.94.209'B'.91/12340.txt', code[0]) 13 | 14 | def test_do_not_extract_plaintext(self): 15 | data = b"some plaintext data" 16 | unit = self.load() 17 | self.assertEqual(bytes(data | unit), b'') 18 | -------------------------------------------------------------------------------- /test/units/formats/pe/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/binref/refinery/44238b79ae1666d01d6d9cdf8eeb9818154cd843/test/units/formats/pe/__init__.py -------------------------------------------------------------------------------- /test/units/formats/pe/dotnet/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/binref/refinery/44238b79ae1666d01d6d9cdf8eeb9818154cd843/test/units/formats/pe/dotnet/__init__.py -------------------------------------------------------------------------------- /test/units/formats/pe/dotnet/test_dnfields.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | import json 4 | 5 | from .... import TestUnitBase 6 | 7 | 8 | class TestDotNetFieldExtractor(TestUnitBase): 9 | 10 | def test_real_world(self): 11 | data = self.download_sample('82831deadbb41d00df1f45c1b1e7cb89901531ab784a55171f11c891f92fffaf') 12 | unit = self.load('mykey', '*6DDE*') 13 | key, payload = unit.process(data) 14 | test = payload | self.ldu('xor', key) | self.ldu('pemeta') | json.loads 15 | self.assertIn('DotNet', test) 16 | self.assertEqual(test['DotNet']['ModuleName'], 'atwork.exe') 17 | -------------------------------------------------------------------------------- /test/units/formats/pe/dotnet/test_dnmr.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from .... import TestUnitBase 4 | 5 | 6 | class TestDotNetManagedResourceDeserializer(TestUnitBase): 7 | 8 | def test_real_world_01(self): 9 | rsrc = self.ldu('dnrc', '70218dfd-5f9f-d4.Resources.resources') 10 | data = rsrc(self.download_sample('82831deadbb41d00df1f45c1b1e7cb89901531ab784a55171f11c891f92fffaf')) 11 | unit = self.load('f0787dcf-8df6-f70') 12 | self.assertTrue(unit(data).startswith(bytes.fromhex('89 50 4E 47 0D 0A 1A 0A'))) 13 | 14 | def test_real_world_02(self): 15 | rsrc = self.ldu('dnrc', '70218dfd-5f9f-d4.Resources.resources') 16 | data = rsrc(self.download_sample('82831deadbb41d00df1f45c1b1e7cb89901531ab784a55171f11c891f92fffaf')) 17 | unit = self.load('b091b52a-98c2-06') 18 | self.assertEqual(unit(data), bytes((29, 0, 0, 0))) 19 | -------------------------------------------------------------------------------- /test/units/formats/pe/dotnet/test_dnrc.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from .... import TestUnitBase 4 | 5 | 6 | class TestDotNetResourceExtractor(TestUnitBase): 7 | 8 | def test_real_world_01(self): 9 | unit = self.load('70218dfd-5f9f-d4*') 10 | data = self.download_sample('82831deadbb41d00df1f45c1b1e7cb89901531ab784a55171f11c891f92fffaf') 11 | data = unit(data) 12 | self.assertTrue(data.startswith(b'\xCE\xCA\xEF\xBE')) 13 | self.assertIn(b'PublicKeyToken=b03f5f7f11d50a3a', data) 14 | -------------------------------------------------------------------------------- /test/units/formats/pe/dotnet/test_dnsfx.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from .... import TestUnitBase 4 | 5 | 6 | class TestDotNetSingleFileApplicationExtractor(TestUnitBase): 7 | 8 | def test_compression(self): 9 | unit = self.load() 10 | data = self.download_sample('34dbeb0b19ae70596a1abd00f57002fbef59f390dec759498298e6e93f252db2') 11 | data = unit(data) 12 | self.assertTrue(data.startswith(b'\x7B\x0D\x0A\x20\x20')) 13 | self.assertIn(b'Compression.pdb', data) 14 | 15 | def test_no_compression(self): 16 | unit = self.load() 17 | data = self.download_sample('69f32fe40a58e5051c7616b186ababe7466a9e973713c80689c50aea943674eb') 18 | data = unit(data) 19 | self.assertTrue(data.startswith(b'\x7B\x0D\x0A\x20\x20')) 20 | self.assertIn(b'Compression.pdb', data) 21 | 22 | -------------------------------------------------------------------------------- /test/units/formats/test_deserialize_php.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | import json 4 | 5 | from .. import TestUnitBase 6 | 7 | 8 | class TestPHPDeserializer(TestUnitBase): 9 | 10 | def test_reversible_property(self): 11 | data = {"42": True, "A to Z": {"0": 1, "1": 2, "2": 3}} 12 | ds = self.load() 13 | self.assertEqual(json.dumps(data) | -ds | ds | json.loads, data) 14 | 15 | def test_wikipedia(self): 16 | out = B'O:8:"stdClass":2:{s:4:"John";d:3.14;s:4:"Jane";d:2.718;}' | self.load() | json.loads 17 | self.assertEqual(out, { 18 | "John": 3.14, 19 | "Jane": 2.718 20 | }) 21 | -------------------------------------------------------------------------------- /test/units/formats/test_dexstr.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from .. import TestUnitBase 4 | 5 | 6 | class TestDexStrings(TestUnitBase): 7 | 8 | def test_real_world_01(self): 9 | data = self.download_sample('0266de70dec416d1046d1b52e794a9138cf2520a57d5c9badf6411b19a9a03f0') 10 | unit = self.load() 11 | strings = data | unit | {str} 12 | self.assertContains(strings, 'http:''//37.''57.96''.145:8099/') 13 | self.assertContains(strings, 'http:''//37.''57.96''.145:8099/permanentlyRemove') 14 | self.assertContains(strings, 'http:''//37.''57.96''.145:8099/registerUser') 15 | self.assertContains(strings, 'http:''//37.''57.96''.145:8099/sync') 16 | -------------------------------------------------------------------------------- /test/units/formats/test_httpresponse.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from .. import TestUnitBase 4 | 5 | 6 | class TestHTTPParser(TestUnitBase): 7 | 8 | def test_basic(self): 9 | unit = self.load() 10 | data = B'\r\n'.join([ 11 | B'HTTP/1.1 200 OK', 12 | B'Date: Thu, 09 Apr 2020 21:20:32 GMT', 13 | B'Content-Type: text/plain', 14 | B'Transfer-Encoding: chunked', 15 | B'Connection: keep-alive', 16 | B'Last-Modified: Mon, 01 Apr 2053 12:55:23 GMT', 17 | B'Vary: Accept-Encoding', 18 | B'CF-Cache-Status: REVALIDATED', 19 | B'Server: pipel1nez4lyfe', 20 | B'', 21 | B'0010', 22 | B'BINARY REFINERY!', 23 | B'0010', 24 | B'BINARY REFINERY!', 25 | B'0', 26 | ]) 27 | self.assertEqual(unit(data), B'BINARY REFINERY!BINARY REFINERY!') 28 | -------------------------------------------------------------------------------- /test/units/formats/test_msgpack.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | import json 4 | import inspect 5 | 6 | from .. import TestUnitBase 7 | 8 | 9 | class TestMessagePack(TestUnitBase): 10 | 11 | def test_reversible_property(self): 12 | @inspect.getdoc 13 | class data: 14 | """ 15 | { 16 | "jsonrpc": "2.0", 17 | "method": "refine", 18 | "id": 1254, 19 | "params": [ 20 | "Binary", 21 | 78, 22 | 0, 23 | false, 24 | true, 25 | 2, 26 | false, 27 | "FOOBAR" 28 | ] 29 | } 30 | """ 31 | msgpack = self.load() 32 | data = data.encode(msgpack.codec) 33 | self.assertEqual( 34 | json.loads(data), 35 | json.loads(str(data | -msgpack | msgpack)) 36 | ) 37 | -------------------------------------------------------------------------------- /test/units/formats/test_msi.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | import json 4 | import hashlib 5 | 6 | from .. import TestUnitBase 7 | 8 | 9 | class TestMSI(TestUnitBase): 10 | 11 | def test_real_world_01(self): 12 | data = self.download_sample('5316330427de98c60661233369909132f6b838b5ff111f7f29efe3e90636a34a') 13 | msi = data | self.load() | {'path': ...} 14 | meta = json.loads(msi['MsiTables.json']) 15 | self.assertEqual(meta['Property'][19]['Value'], "Típica") 16 | self.assertContains(msi['Action/basicsmarch.js'], b'FDWPLOVWRX(PIENNNAXXI+ QRQHCMZIEY,NAXXIPIENN+ BJHZRJLZFX)') 17 | self.assertEqual(hashlib.sha256(msi['Binary/aicustact.dll']).hexdigest(), 18 | 'f2f3ae8ca06f5cf320ca1d234a623bf55cf2b84c1d6dea3d85d5392e29aaf437') 19 | 20 | def test_cab_extraction(self): 21 | data = self.download_sample('5c698edeba5260b1eb170c375015273324b86bae82722d85d2f013b22ae52d0c') 22 | msi = data | self.load() | {'path': ...} 23 | cab = '_A43DCF057E5B03E2396812E1C2F1D349' 24 | for t in '123de': 25 | self.assertIn(F'{cab}/{t}', msi) 26 | self.assertEqual(msi[F'{cab}/1'], B'M') 27 | self.assertEqual(msi[F'{cab}/2'], B'Z') 28 | -------------------------------------------------------------------------------- /test/units/formats/test_pdf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from .. import TestUnitBase 4 | 5 | 6 | class TestPDF(TestUnitBase): 7 | 8 | def test_pdf_maldoc_with_embedded_flash(self): 9 | data = self.download_sample('76b19c1e705328cab4d98e546095eb5eb601d23d8102e6e0bfb0a8a6ab157366') 10 | 11 | unit = self.load(list=True) 12 | self.assertEqual({bytes(c) for c in data | unit}, { 13 | B'Metadata', 14 | B'OpenAction/JS', 15 | B'Pages/Kids/0/Annots/0/NM', 16 | B'Pages/Kids/0/Annots/0/RichMediaContent/Assets/Names/fqMBgzoMVutzNwk.swf', 17 | }) 18 | 19 | unit = self.load('Pages/*') 20 | name, swf = data | unit 21 | self.assertEqual(name, B'fqMBgzoMVutzNwk.swf') 22 | self.assertEqual(swf[:3], B'CWS') 23 | 24 | unit = self.load('fqMBgzoMVutzNwk') | self.ldu('sha256', text=True) 25 | self.assertEqual(str(data | unit), 'c2b666a3ef4c191b77b78c037656e50477b8ba3d35fd61ae843a3a1f4d41c5c1') 26 | -------------------------------------------------------------------------------- /test/units/malware/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/binref/refinery/44238b79ae1666d01d6d9cdf8eeb9818154cd843/test/units/malware/__init__.py -------------------------------------------------------------------------------- /test/units/malware/test_kramer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from .. import TestUnitBase 4 | 5 | 6 | class TestKramer(TestUnitBase): 7 | 8 | def test_real_world_01(self): 9 | data = self.download_sample('077333ebaf7b557f419147787950d876decbeb9437cc292c93d3cc3eceb028c8') 10 | test = data | self.load() | str 11 | self.assertIn('n = await client.get_messages', test) 12 | -------------------------------------------------------------------------------- /test/units/meta/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | import os 4 | 5 | from refinery.lib.frame import FrameUnpacker 6 | 7 | from .. import TestUnitBase 8 | 9 | 10 | class TestMetaBase(TestUnitBase): 11 | 12 | def load(self, *args, **kwargs): 13 | unit = super().load(*args, '[', **kwargs) 14 | 15 | def wrapper(*inputs): 16 | with open(os.devnull, 'rb') as stream: 17 | unpacker = FrameUnpacker(stream | self.ldu('emit', '[', data=list(inputs)) | unit) 18 | results = [] 19 | while unpacker.nextframe(): 20 | results.extend((bytes(item) for item in unpacker)) 21 | return results 22 | return wrapper 23 | -------------------------------------------------------------------------------- /test/units/meta/test_group.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from .. import TestUnitBase 4 | from refinery.lib.loader import load_pipeline as L 5 | 6 | 7 | # The magic word is bananapalooza 8 | class TestGroupingUnit(TestUnitBase): 9 | 10 | def test_01(self): 11 | pipeline = L('emit A B C D E F G H I J K [| group 3 []| sep - ]') 12 | self.assertEqual(pipeline(), B'ABC-DEF-GHI-JK') 13 | 14 | def test_02(self): 15 | pipeline = L('emit :98:52:91:6 | rex :(.)(.) {2} {1} [| group 2 [| pop k | sub var:k ]]') 16 | self.assertEqual(pipeline(), B'\x01\x03\x08') 17 | 18 | def test_03(self): 19 | pipeline = L('emit A B C D [| put q c::1 | group 2 [| put q test | pick 0 ]| cca var:q ]') 20 | self.assertEqual(pipeline(), B'AACC') 21 | -------------------------------------------------------------------------------- /test/units/meta/test_iffp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # flake8: noqa 4 | from refinery.lib.loader import load_pipeline 5 | from .. import TestUnitBase 6 | 7 | 8 | class TestIfPattern(TestUnitBase): 9 | 10 | def test_simple_01(self): 11 | ps = BR'"C:\\work\\is\\fun\\"'.hex() 12 | result = load_pipeline(RF'emit H:{ps} | carve -d string [| iffp path ]') 13 | result = result() 14 | self.assertEqual(result, B'C:\\work\\is\\fun\\') 15 | -------------------------------------------------------------------------------- /test/units/meta/test_iffs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # flake8: noqa 4 | from refinery.lib.loader import load_detached as L 5 | from .. import TestUnitBase 6 | 7 | 8 | class TestIfStr(TestUnitBase): 9 | 10 | def test_simple_01(self): 11 | pl = L('emit raffle waffle rattle battle cattle settle') [ self.load('att') ] 12 | self.assertEqual(pl(), B'rattlebattlecattle') 13 | -------------------------------------------------------------------------------- /test/units/meta/test_iffx.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.lib.loader import load_detached as L 4 | from .. import TestUnitBase 5 | 6 | 7 | class TestIfRex(TestUnitBase): 8 | 9 | def test_filter_identifier_letters(self): 10 | pl = L('emit range::256') | L('chop 1')[self.load('\\w')] 11 | self.assertEqual(pl(), B'0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz') 12 | 13 | def test_negate(self): 14 | pl = self.load_pipeline('emit ab bc cb [| iffx -R .b ]') 15 | self.assertEqual(pl(), b'bc') 16 | -------------------------------------------------------------------------------- /test/units/meta/test_max_min.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from . import TestMetaBase 4 | from refinery.lib.loader import load_pipeline as L 5 | 6 | 7 | class TestMinMax(TestMetaBase): 8 | 9 | def test_max_01(self): 10 | pl = L('emit the Binary Refinery refines the finest binaries [| max size ]') 11 | self.assertEqual(pl(), B'Refinery') 12 | 13 | def test_min_01(self): 14 | pl = L('emit the Binary Refinery refines the finest binaries [| min size ]') 15 | self.assertEqual(pl(), B'the') 16 | 17 | def test_max_works_with_pop(self): 18 | pl = L('emit Y:FOO:DN:GOOP [| push | resplit : | max size | pop ll | ccp var:ll ]') 19 | self.assertEqual(pl(), b'GOOPY:FOO:DN:GOOP') 20 | 21 | def test_min_works_with_pop(self): 22 | pl = L('emit YY:FOO:A:GOOP [| push | resplit : | min size | pop ll | ccp var:ll ]') 23 | self.assertEqual(pl(), b'AYY:FOO:A:GOOP') 24 | -------------------------------------------------------------------------------- /test/units/meta/test_mvg.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from . import TestMetaBase 4 | from refinery.lib.loader import load_pipeline as L 5 | 6 | 7 | class TestMetaVarGlobal(TestMetaBase): 8 | 9 | def test_mvg_01(self): 10 | pl = L('emit FOO [| nop [| put x BAR | mvg ]| cca var:x ]') 11 | self.assertEqual(pl(), B'FOOBAR') 12 | 13 | def test_mvg_02(self): 14 | pl = L('emit FOO [| nop [[| put x BAR | mvg ]| nop ]| cfmt {}{x} ]') 15 | self.assertEqual(pl(), B'FOO{x}') 16 | 17 | def test_mvg_03(self): 18 | pl = L('emit FOO [| nop [[| put x BAR | mvg -t ]| nop ]| cfmt {}{x} ]') 19 | self.assertEqual(pl(), B'FOOBAR') 20 | 21 | def test_cannot_propagate_variables_with_ambiguous_values(self): 22 | pl = L('emit FOO [| rex . [| put x | mvg ]| cfmt {}{x} ]') 23 | self.assertEqual(pl(), B'FOO{x}') 24 | 25 | def test_can_propagate_variables_with_unambiguous_values(self): 26 | pl = L('emit FOO [| rex . [| put x BAR | mvg ]| cfmt {}{x} ]') 27 | self.assertEqual(pl(), B'FOOBAR') 28 | 29 | def test_scope_cannot_be_increased(self): 30 | pl = L('emit s: [| put x alpha [| nop [| put x beta | mvg ]| nop ]| cfmt {x} ]') 31 | self.assertEqual(pl(), B'alpha') 32 | -------------------------------------------------------------------------------- /test/units/meta/test_pop.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from .. import TestUnitBase 4 | from refinery.lib.loader import load_pipeline as L 5 | 6 | 7 | class TestPop(TestUnitBase): 8 | 9 | def test_error_when_variables_cannot_be_assigned(self): 10 | pl = self.ldu('push') [ self.ldu('rex', 'XX(.)', '{1}') | self.load('oops') ] # noqa 11 | with self.assertRaises(Exception): 12 | b'TEST' | pl | None 13 | 14 | def test_regression_pop_does_not_key_error(self): 15 | pl = L('emit FOO | rex . [| push [| put k XO | pop ]| cfmt {k} ]') 16 | self.assertEqual(pl(), B'XOXOXO') 17 | -------------------------------------------------------------------------------- /test/units/meta/test_put.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.lib.loader import load_pipeline as L 4 | from .. import TestUnitBase 5 | 6 | 7 | class TestMetaPut(TestUnitBase): 8 | 9 | def test_simple_variable_01(self): 10 | pl = L('emit "FOO BAR" [| put ff rep[5]:copy::1 | nop | ccp var:ff ]') 11 | self.assertEqual(pl(), B'FFFFFFOO BAR') 12 | 13 | def test_pop_variable(self): 14 | pl = L('emit AB CD EF [| put k x::1 | sub eat:k ]') 15 | self.assertEqual(pl(), B'\x01\x01\x01') 16 | 17 | def test_regression_put_removes_variable(self): 18 | pl = L('emit BAR [| rex . [| put x | cfmt {x}{offset} ]]') 19 | self.assertEqual(pl(), B'B0A1R2') 20 | -------------------------------------------------------------------------------- /test/units/meta/test_queue.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.lib.loader import load_pipeline as L 4 | from .. import TestUnitBase 5 | 6 | 7 | class TestQueue(TestUnitBase): 8 | 9 | def test_variables_are_available(self): 10 | data = B'{"foo":"X","bar":"Y"}' 11 | pipeline = L('xtjson [| rex . [| qb var:path ]]') 12 | out = pipeline(data) 13 | self.assertEqual(out, B'XfooYbar') 14 | 15 | def test_inserted_chunks_are_visible(self): 16 | pipeline = L('emit AA | push [| qb BB | pop a b | cfmt {a}{b} ]') 17 | self.assertEqual(pipeline(), b'AABB') 18 | -------------------------------------------------------------------------------- /test/units/meta/test_rmv.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from . import TestMetaBase 4 | from refinery.lib.loader import load_pipeline as L 5 | 6 | 7 | class TestRemoveMetaVar(TestMetaBase): 8 | 9 | def test_rmv_01(self): 10 | pl = self.load_pipeline('emit FOO [| put x BAR [| put x BOO | rmv x | cfmt {}{x} ]| cfmt {}{x} ]') 11 | self.assertEqual(pl(), B'FOO{x}BAR') 12 | -------------------------------------------------------------------------------- /test/units/meta/test_scope.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from . import TestMetaBase 4 | from refinery.lib.loader import load_detached as L 5 | 6 | 7 | class TestScope(TestMetaBase): 8 | 9 | def test_only_local_scope(self): 10 | pipeline = L('rep') [ L('scope 1') | L('rep') [ L('scope 0') | L('cca .') ]] # noqa 11 | self.assertEqual(pipeline(B'FOO'), B'FOOFOO.FOO') 12 | 13 | def test_layer1_rescope(self): 14 | pipeline = L('rep') [ L('scope 0') | L('cca ,') | L('scope 1') | L('cca .') ] # noqa 15 | self.assertEqual(pipeline(B'FOO'), B'FOO,FOO.') 16 | 17 | def test_layer2_rescope(self): 18 | pipeline = L('rep 6') [ L('scope 4:') | L('chop 1') [ L('scope 1:') | L('cca A') | L('scope 0') | L('ccp -') ]] # noqa 19 | self.assertEqual(pipeline(B'NA'), B'NANANANA-NAA-NAA') 20 | 21 | def test_scope_in_frame_regression(self): 22 | pipeline = self.load_pipeline('emit FOO BAR BAZ [| scope 0 [| rex O [| scope 1 | cca F ]| ccp B ]| sep : ]') 23 | self.assertEqual(pipeline(), B'BOOF:BAR:BAZ') 24 | -------------------------------------------------------------------------------- /test/units/meta/test_sep.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from . import TestMetaBase 4 | 5 | 6 | class TestSep(TestMetaBase): 7 | 8 | def test_delayed_arguments_copy(self): 9 | unit = self.load('c:-1:') 10 | self.assertEqual( 11 | unit( 12 | B'Foo', 13 | B'Bar', 14 | B'Baz', 15 | ), [ 16 | B'Foo', B'o', 17 | B'Bar', B'r', 18 | B'Baz', 19 | ] 20 | ) 21 | 22 | def test_delayed_arguments_cut(self): 23 | unit = self.load('x:-1:') 24 | self.assertEqual( 25 | unit( 26 | B'Foo', 27 | B'Bar', 28 | B'Baz', 29 | ), [ 30 | B'Fo', B'o', 31 | B'Ba', B'r', 32 | B'Ba', 33 | ] 34 | ) 35 | -------------------------------------------------------------------------------- /test/units/meta/test_sorted.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | import io 4 | 5 | from . import TestMetaBase 6 | from refinery.lib.loader import load_detached as L 7 | 8 | 9 | class TestSorting(TestMetaBase): 10 | 11 | def test_with_scoping(self): 12 | with io.BytesIO() as result: 13 | L('emit F2 F3 S4 S6 S5 F1 S2 S3 S1')[ 14 | L('scope 2:5 6:') | L('sorted') | L('sep -')] | result 15 | self.assertEqual(result.getvalue(), B'F2-F3-S6-S5-S4-F1-S3-S2-S1') 16 | 17 | def test_trailing_chunks(self): 18 | pipeline = L('rep') [ L('scope 0') | L('chop 1') [ L('sorted -a') ]] # noqa 19 | self.assertEqual(pipeline(B'8492756031'), B'01234567898492756031') 20 | 21 | def test_sorting_by_size(self): 22 | results = list(L('emit A AA B ABA C ABABABB CD EFEW A BVB')[L('sorted -a size')]) 23 | self.assertEqual(results, [ 24 | B'A', B'A', B'B', B'C', B'AA', B'CD', B'ABA', B'BVB', B'EFEW', B'ABABABB']) 25 | -------------------------------------------------------------------------------- /test/units/meta/test_swap.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from refinery.lib.loader import load_detached as L 4 | from .. import TestUnitBase 5 | 6 | 7 | class TestSwap(TestUnitBase): 8 | 9 | def test_swap_variable_01(self): 10 | pl = L('emit BAZ [') | L('put BAR S:FOO') | L('put FOO S:BAR') | L('swap FOO BAR') | L('cfmt {FOO}{BAR} ]') 11 | self.assertEqual(pl(), B'FOOBAR') 12 | 13 | def test_swap_variable_02(self): 14 | pl = L('emit BAZ [') | L('put BAR FOO') | L('swap BAR FOO') | L('cfmt {FOO} ]') 15 | self.assertEqual(pl(), B'FOO') 16 | 17 | def test_swap_with_data(self): 18 | pl = L('emit BAR [') | L('put BAR FOO') | L('swap BAR') | L('cfmt {}{BAR}{FOO} ]') 19 | self.assertEqual(pl(), B'FOOBAR{FOO}') 20 | 21 | def test_swap_skips_invisible_chunks(self): 22 | pl = L('emit range:65:91') [ L('push') | L('rex MNO') | L('swap mno') | L('pop') | L('ccp var:mno') ] # noqa 23 | self.assertEqual(pl(), B'MNOABCDEFGHIJKLMNOPQRSTUVWXYZ') 24 | -------------------------------------------------------------------------------- /test/units/meta/test_xfcc.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from .. import TestUnitBase 4 | 5 | from refinery.lib.loader import load_pipeline as L 6 | 7 | 8 | class TestCrossFrameChunkCount(TestUnitBase): 9 | 10 | def test_01(self): 11 | pipeline = L('emit ABDF AEC ABE [| rex . [| xfcc ]]') 12 | results = {bytes(chunk): chunk['count'] for chunk in pipeline} 13 | self.assertEqual(results, { 14 | B'A': 3, 15 | B'B': 2, 16 | B'C': 1, 17 | B'D': 1, 18 | B'E': 2, 19 | B'F': 1, 20 | }) 21 | -------------------------------------------------------------------------------- /test/units/misc/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/binref/refinery/44238b79ae1666d01d6d9cdf8eeb9818154cd843/test/units/misc/__init__.py -------------------------------------------------------------------------------- /test/units/misc/test_nop.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from .. import TestUnitBase 4 | 5 | 6 | class TestNop(TestUnitBase): 7 | 8 | def test_doing_nothing(self): 9 | unit = self.load() 10 | data = self.generate_random_buffer(200) 11 | self.assertEqual(data, unit(data)) 12 | -------------------------------------------------------------------------------- /test/units/misc/test_urlfix.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from .. import TestUnitBase 4 | 5 | 6 | class TestURLFixer(TestUnitBase): 7 | 8 | def test_01(self): 9 | self.assertEqual( 10 | b'HTtP://exAmpLE.COM/path/to/script.php?arg=1' | self.load() | str, 11 | r'http://example.com/path/to/script.php') 12 | 13 | def test_02(self): 14 | self.assertEqual( 15 | b'GOPHER://exAmpLE.COM/path/to/script.php?arg=1#fragmento' | self.load() | str, 16 | r'gopher://example.com/path/to/script.php') 17 | -------------------------------------------------------------------------------- /test/units/obfuscation/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/binref/refinery/44238b79ae1666d01d6d9cdf8eeb9818154cd843/test/units/obfuscation/__init__.py -------------------------------------------------------------------------------- /test/units/obfuscation/ps1/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/binref/refinery/44238b79ae1666d01d6d9cdf8eeb9818154cd843/test/units/obfuscation/ps1/__init__.py -------------------------------------------------------------------------------- /test/units/obfuscation/ps1/test_brackets.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from ... import TestUnitBase 4 | 5 | 6 | class TestBracketRemover(TestUnitBase): 7 | 8 | def setUp(self): 9 | super().setUp() 10 | self.unit = self.load() 11 | 12 | def test_string_literal_01(self): 13 | self.assertEqual(self.unit(B'("{0}{2}{1}")'), b'"{0}{2}{1}"') 14 | 15 | def test_string_literal_02(self): 16 | self.assertEqual(self.unit(B'( (( \n( "Test")))'), b'( "Test"') 17 | 18 | def test_string_literal_03(self): 19 | self.assertEqual(self.unit(B'(((\n( "Tes""t")\n)) )'), b'"Tes""t"') 20 | 21 | def test_numeric_literal_01(self): 22 | self.assertEqual(self.unit(B'(0x12)'), b'0x12') 23 | 24 | def test_numeric_literal_02(self): 25 | self.assertEqual(self.unit(B'( (( \n( 0x12) ))'), b'( 0x12') 26 | 27 | def test_numeric_literal_03(self): 28 | self.assertEqual(self.unit(B'((31337) )'), b'31337') 29 | -------------------------------------------------------------------------------- /test/units/obfuscation/ps1/test_cases.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from ... import TestUnitBase 4 | 5 | 6 | class TestNameCases(TestUnitBase): 7 | 8 | def test_set_variable(self): 9 | self.assertEqual( 10 | self.load().process(b'SET-vaRIabLE'), 11 | b'Set-Variable' 12 | ) 13 | -------------------------------------------------------------------------------- /test/units/obfuscation/ps1/test_typecast.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from ... import TestUnitBase 4 | 5 | 6 | class TestPs1Typecast(TestUnitBase): 7 | 8 | def setUp(self): 9 | super().setUp() 10 | self.unit = self.load() 11 | 12 | def test_useless_string_cast(self): 13 | self.assertEqual( 14 | self.unit.deobfuscate('''.replAce(("M0I"),[strIng]"'")'''), 15 | '''.replAce(("M0I"),"'")''' 16 | ) 17 | -------------------------------------------------------------------------------- /test/units/obfuscation/vba/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/binref/refinery/44238b79ae1666d01d6d9cdf8eeb9818154cd843/test/units/obfuscation/vba/__init__.py -------------------------------------------------------------------------------- /test/units/obfuscation/vba/test_all.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from ... import TestUnitBase 4 | 5 | 6 | class TestVBADeobfuscator(TestUnitBase): 7 | 8 | def test_real_world_01(self): 9 | data = BR'''Execute chr(311-(&HF1))&chr(1112-(&H3E3))&chr(422-(&H138))&chr(1064-(&H3C5))''' 10 | result = self.load().process(data) 11 | self.assertEqual(result, b'Execute "Func"') 12 | 13 | def test_real_world_02(self): 14 | data = self.download_sample('07e25cb7d427ac047f53b3badceacf6fc5fb395612ded5d3566a09800499cd7d') 15 | unit = self.load() 16 | self.assertIn( 17 | r'POwerShell.exe -noProfilE -ExEcutionPolicy Bypass -Command C:\ProgramData\UPFCRQOFGHVNBVUABXGFIW\UPFCRQOFGHVNBVUABXGFIW.bat', 18 | data | unit | str 19 | ) 20 | -------------------------------------------------------------------------------- /test/units/obfuscation/vba/test_arithmetic.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from ... import TestUnitBase 4 | 5 | 6 | class TestArithmeticEvaluator(TestUnitBase): 7 | 8 | def setUp(self): 9 | super().setUp() 10 | self.unit = self.load() 11 | 12 | def test_xor_operator(self): 13 | self.assertEqual(self.unit.deobfuscate('CLng((0 Xor 0))'), 'CLng((0 Xor 0))') 14 | -------------------------------------------------------------------------------- /test/units/obfuscation/vba/test_comments.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from ... import TestUnitBase 4 | 5 | 6 | class TestCommentRemover(TestUnitBase): 7 | 8 | def setUp(self): 9 | super().setUp() 10 | self.unit = self.load() 11 | 12 | def test_simple(self): 13 | self.assertEqual(self.unit.deobfuscate(r''' 14 | ' Test 15 | b = a 16 | ' Test''').strip(), r'b = a') 17 | -------------------------------------------------------------------------------- /test/units/obfuscation/vba/test_constants.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from ... import TestUnitBase 4 | 5 | 6 | class TestConstantReplacer(TestUnitBase): 7 | 8 | def setUp(self): 9 | super().setUp() 10 | self.unit = self.load() 11 | 12 | def test_regex_matchgroup_regression(self): 13 | self.assertEqual(self.unit.deobfuscate(r''' 14 | const a = "\3" 15 | b = a 16 | ''').strip(), r'b = "\3"') 17 | -------------------------------------------------------------------------------- /test/units/obfuscation/vba/test_dummies.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from ... import TestUnitBase 4 | 5 | 6 | class TestDummyVariableRemover(TestUnitBase): 7 | 8 | def setUp(self): 9 | super().setUp() 10 | self.unit = self.load() 11 | 12 | def test_overeager_removal_regression_01(self): 13 | data = 'a.Close\nb = z.function(x)\n' 14 | self.assertEqual(self.unit.deobfuscate(data).strip(), data.strip()) 15 | -------------------------------------------------------------------------------- /test/units/obfuscation/vba/test_stringreplace.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from ... import TestUnitBase 4 | 5 | 6 | class TestStringReplace(TestUnitBase): 7 | 8 | def setUp(self): 9 | super().setUp() 10 | self.unit = self.load() 11 | 12 | def test_trivial(self): 13 | self.assertEqual( 14 | self.unit.deobfuscate( 15 | '''impairingsgutta = Replace("ADs@j|P3FODBs@j|P3F.Sts@j|P3Fs@j|P3Fs@j|P3Fream", "s@j|P3F", "")'''), 16 | 'impairingsgutta = "ADODB.Stream"' 17 | ) 18 | -------------------------------------------------------------------------------- /test/units/pattern/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/binref/refinery/44238b79ae1666d01d6d9cdf8eeb9818154cd843/test/units/pattern/__init__.py -------------------------------------------------------------------------------- /test/units/pattern/test_carve_7z.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | import base64 4 | from .. import TestUnitBase 5 | 6 | 7 | class TestCarve7Zip(TestUnitBase): 8 | 9 | def test_01(self): 10 | unit = self.load() 11 | data = base64.b64decode( 12 | 'N3q8ryccAAT9xtacpQAAAAAAAAAiAAAAAAAAAEerl+XBRlkrcJwwoqgijCyu' 13 | 'Eh0SqLfjamv2F2vNJGFGyHDfpAAAgTMHrg/QDrA8nzkQnJ+m1TPasi6xAvSH' 14 | 'zZaZrISLD+EsvFULZ44Kf7Ewy47PApbKruXCaOSUsjzeqpG8VBcx66h2cV/l' 15 | 'nGDfUjtVsyGBHmmmTaSI/atXtuwiN5mGrqyFZTC/V2VEohWua1Yk1K+jXy+3' 16 | '2hBwnK2clyr3rN5LAbv5g2wXBiABCYCFAAcLAQABIwMBAQVdABAAAAyAlgoB' 17 | 'ouB4BAAA' 18 | ) 19 | for prefix in ( 20 | unit.HEADER_SIGNATURE + self.generate_random_buffer(20), 21 | unit.HEADER_SIGNATURE * 3, 22 | self.generate_random_buffer(64) + unit.HEADER_SIGNATURE + b'AAAA' 23 | ): 24 | for pf in (2, 4, 90, 200, 2048): 25 | blob = prefix + data + self.generate_random_buffer(pf) 26 | self.assertEqual(blob | unit | bytearray, data) 27 | -------------------------------------------------------------------------------- /test/units/pattern/test_carve_xml.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from .. import TestUnitBase 4 | 5 | 6 | class TestCarveXML(TestUnitBase): 7 | 8 | def test_wikipedia_unicode_example(self): 9 | xstr = '<俄语 լեզու="ռուսերեն">данные' 10 | unit = self.load() 11 | norm = xstr.encode(unit.codec) 12 | for encoding in ['UTF8', 'UTF-16LE']: 13 | xbin = xstr.encode(encoding) 14 | data = self.generate_random_buffer(200) + xbin + self.generate_random_buffer(100) 15 | self.assertEqual(unit(data), norm) 16 | -------------------------------------------------------------------------------- /test/units/pattern/test_dnsdomain.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from .. import TestUnitBase 4 | 5 | 6 | class TestDNSDomainCarver(TestUnitBase): 7 | 8 | def test_url_defang(self): 9 | unit = self.load() 10 | data = bytes.fromhex( 11 | '4A0100000100000000000009616D6F6E676F6C696103636F6D0000010001708FC05D73110A' 12 | '0059000000590000000A00273BD1A5AC1F6B0DDB8408004500004B7BAB0000791104D30808' 13 | '0808C0A8F06B0035C18D00375F39A64A8180000100010000000009616D6F6E676F6C696103' 14 | '636F6D0000010001C00C000100010000003B0004B97642FE708FC05D64610A004200000042' 15 | '00000000005E0001010A00273BD1A50800450000340164400080064BD7C0A8F06BB97642FE' 16 | ) 17 | self.assertIn(B'amongolia', unit(data)) 18 | -------------------------------------------------------------------------------- /test/units/pattern/test_resplit.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from .. import TestUnitBase 4 | from refinery.lib.loader import load_detached as L 5 | 6 | 7 | class TestRegexSplitter(TestUnitBase): 8 | 9 | def test_change_separator(self): 10 | pl = L('emit eeny,meeny,miny,moe') | L('resplit (,) [') | L('scope 1::2') | L('cfmt - ]') 11 | self.assertEqual(pl(), B'eeny-meeny-miny-moe') 12 | 13 | def test_count_restriction(self): 14 | pl = L('emit eeny,meeny,miny,moe') | L('resplit -c1 ,') 15 | self.assertEqual(pl(), B'eeny\nmeeny,miny,moe') 16 | 17 | def test_multibin_argument(self): 18 | pl = L('emit foobar') [ L('put split oo') | L('resplit eat:split') ] # noqa 19 | self.assertEqual(list(pl), [b'f', b'bar']) 20 | -------------------------------------------------------------------------------- /test/units/sinks/test_dnasm.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | from .. import TestUnitBase 5 | 6 | 7 | class TestDnASM(TestUnitBase): 8 | 9 | def test_asm(self): 10 | dnasm = self.load('-IAH', '-c2') 11 | data = bytes.fromhex('666665') 12 | result = dnasm(data).decode('utf-8') 13 | self.assertEqual(['not', 'not'], result.splitlines()) 14 | -------------------------------------------------------------------------------- /test/units/sinks/test_iemap.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from .. import TestUnitBase 4 | from . import errbuf 5 | 6 | 7 | class TestIEMap(TestUnitBase): 8 | 9 | def test_java_class_file(self): 10 | iemap = self.load() 11 | from refinery.lib.environment import environment 12 | environment.term_size.value = 80 13 | with errbuf() as stderr: 14 | iemap(self.download_sample('31055a528f9a139104a0ce8f4da6b4b89a37a800715292ae7f8f190b2a7b6582')) 15 | output = stderr.getvalue() 16 | 17 | self.assertEqual( 18 | output, 19 | '[' 20 | '\x1b[37m' 21 | '\x1b[92m#####' 22 | '\x1b[94m##########' 23 | '\x1b[92m#####################' 24 | '\x1b[96m######################' 25 | '\x1b[94m#####' 26 | '\x1b[96m#####' 27 | '\x1b[40m\x1b[37m\x1b[0m' 28 | '] [---.--%]' 29 | '\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08' 30 | '] [ 70.82%]\n' 31 | ) 32 | -------------------------------------------------------------------------------- /test/units/strings/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/binref/refinery/44238b79ae1666d01d6d9cdf8eeb9818154cd843/test/units/strings/__init__.py -------------------------------------------------------------------------------- /test/units/strings/test_bruteforce.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from .. import TestUnitBase 4 | 5 | 6 | class TestBruteForce(TestUnitBase): 7 | 8 | def test_bit_string_brute_force(self): 9 | pl = self.load('t', 4, pattern='[01]') [ self.ldu('swap', 't') ] # noqa 10 | self.assertSetEqual({int(x, 2) for x in pl}, set(range(0b10000))) 11 | 12 | def test_brute_force_format(self): 13 | pl = self.load('t', 1, pattern='[XZ]', format='{1}{0}BAR') [ self.ldu('swap', 't') ] # noqa 14 | self.assertEqual(pl(B'FOO'), B'FOOXBARFOOZBAR') 15 | -------------------------------------------------------------------------------- /test/units/strings/test_clower.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from .. import TestUnitBase 4 | 5 | 6 | class TestCLower(TestUnitBase): 7 | 8 | def test_simple_01(self): 9 | unit = self.load() 10 | data = B'That is not dead which can eternal lie, And with strange aeons even death may die.' 11 | wish = B'that is not dead which can eternal lie, and with strange aeons even death may die.' 12 | self.assertEqual(bytes(data | unit), wish) 13 | -------------------------------------------------------------------------------- /test/units/strings/test_concat.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from .. import TestUnitBase 4 | 5 | 6 | class TestConcatentation(TestUnitBase): 7 | 8 | def test_prepend(self): 9 | self.assertEqual(self.ldu('ccp', 's:Hello').process(B' World'), B'Hello World') 10 | 11 | def test_append(self): 12 | self.assertEqual(self.ldu('cca', 's:World').process(B'Hello '), B'Hello World') 13 | 14 | def test_formatter_simple(self): 15 | self.assertEqual(self.ldu('cfmt', 'Hello {} World')(B'cruel'), B'Hello cruel World') 16 | -------------------------------------------------------------------------------- /test/units/strings/test_cswap.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from .. import TestUnitBase 4 | 5 | 6 | class TestCSwap(TestUnitBase): 7 | 8 | def test_simple_01(self): 9 | unit = self.load() 10 | data = B'That is not dead which can eternal lie, And with strange aeons even death may die.' 11 | wish = B'tHAT IS NOT DEAD WHICH CAN ETERNAL LIE, aND WITH STRANGE AEONS EVEN DEATH MAY DIE.' 12 | self.assertEqual(bytes(data | unit), wish) 13 | -------------------------------------------------------------------------------- /test/units/strings/test_cupper.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from .. import TestUnitBase 4 | 5 | 6 | class TestCUpper(TestUnitBase): 7 | 8 | def test_simple_01(self): 9 | unit = self.load() 10 | data = B'That is not dead which can eternal lie, And with strange aeons even death may die.' 11 | wish = B'THAT IS NOT DEAD WHICH CAN ETERNAL LIE, AND WITH STRANGE AEONS EVEN DEATH MAY DIE.' 12 | self.assertEqual(bytes(data | unit), wish) 13 | -------------------------------------------------------------------------------- /test/units/strings/test_ngrams.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from .. import TestUnitBase 4 | 5 | 6 | class TestNGrams(TestUnitBase): 7 | 8 | def test_simple_01(self): 9 | pl = self.load_pipeline('emit ABC | ngrams 1 []') 10 | self.assertEqual(pl(), B'ABC') 11 | 12 | def test_simple_02(self): 13 | pl = self.load_pipeline('emit ABC | ngrams 2 []') 14 | self.assertEqual(pl(), B'ABBC') 15 | -------------------------------------------------------------------------------- /test/units/strings/test_rep.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from .. import TestUnitBase 4 | 5 | 6 | class TestRep(TestUnitBase): 7 | 8 | def test_batman(self): 9 | unit = self.load('16', '[]') 10 | self.assertEqual(B'Na' | unit | bytes, B'NaNaNaNaNaNaNaNaNaNaNaNaNaNaNaNa') 11 | 12 | def test_sequence(self): 13 | unit = self.load('range:4:7', 't') 14 | self.assertListEqual(list(chunk['t'] for chunk in B"" | unit), list(range(4, 7))) 15 | -------------------------------------------------------------------------------- /test/units/strings/test_repl.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from .. import TestUnitBase 4 | 5 | 6 | class TestRepl(TestUnitBase): 7 | 8 | def test_limit_with_annihilation(self): 9 | unit = self.load('-n', 4, 'e') 10 | self.assertEqual(unit(12 * B'e'), 8 * B'e') 11 | 12 | def test_limit_with_replacement(self): 13 | unit = self.load('-n', 4, 'e', 'o') 14 | self.assertEqual(unit( 15 | B'The keen explorer entered the cave'), 16 | B'Tho koon oxplorer entered the cave') 17 | 18 | def test_without_limit(self): 19 | unit = self.load('e', 'o') 20 | self.assertEqual(unit( 21 | B'The keen explorer entered the cave'), 22 | B'Tho koon oxploror ontorod tho cavo') 23 | 24 | def test_with_longer_strings(self): 25 | unit = self.load('foo', 'oof') 26 | self.assertEqual(unit( 27 | B'my food tastes like foo.'), 28 | B'my oofd tastes like oof.') 29 | -------------------------------------------------------------------------------- /test/units/strings/test_stretch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from .. import TestUnitBase 4 | 5 | 6 | class TestStretch(TestUnitBase): 7 | 8 | def test_simple_stretch(self): 9 | data = B'iIIiIIIIiImain' 10 | wish = B'iiIIIIiiIIIIIIIIiiIImmaaiinn' 11 | unit = self.load() 12 | self.assertEqual(unit(data), wish) 13 | 14 | def test_multi_stretch(self): 15 | data = B'BINARY REFINERY!' 16 | wish = B'BBINNNARRY REEFIIINEERYYY!' 17 | unit = self.load(2, 1, 3, 1) 18 | self.assertEqual(unit(data), wish) 19 | 20 | def test_invalid_input(self): 21 | with self.assertRaises(ValueError): 22 | self.load(1, 2, 0, 3) 23 | with self.assertRaises(ValueError): 24 | self.load(1, -2, 0) 25 | 26 | def test_invert(self): 27 | data = B'born to refine binaries' 28 | stretch = self.load(2, 4, 1, 3) 29 | clinch = self.load(2, 4, 1, 3, reverse=True) 30 | self.assertEqual(data, clinch(stretch(data))) 31 | -------------------------------------------------------------------------------- /update.ps1: -------------------------------------------------------------------------------- 1 | #!powershell 2 | $ErrorActionPreference = "Stop" 3 | $venv = "venv"; 4 | 5 | if ($args.Count -ge 1) { 6 | $venv = $args[0]; 7 | } 8 | 9 | if (-not (Test-Path $venv)) { 10 | . py -3 -m venv $venv 11 | } 12 | 13 | . "$venv\Scripts\Activate.ps1" 14 | 15 | python -m pip install --upgrade pip 16 | try { 17 | rm ./refinery/data/units.pkl 18 | } catch {} 19 | git pull --rebase --autostash 20 | pip uninstall -y binary-refinery 21 | pip install --use-pep517 -U -e .[all] -------------------------------------------------------------------------------- /update.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | venv="$1" 4 | 5 | if [ -z "$venv" ]; then 6 | venv="venv" 7 | fi 8 | 9 | if [ ! -d "$venv" ]; then 10 | python3 -m venv "$venv" 11 | fi 12 | 13 | source "$venv/bin/activate" 14 | 15 | python -m pip install --upgrade pip 16 | 17 | rm ./refinery/data/units.pkl 18 | 19 | git pull --rebase --autostash 20 | pip uninstall -y binary-refinery 21 | pip install --use-pep517 -U -e .[all] --------------------------------------------------------------------------------