├── .project ├── .pydevproject ├── .settings └── org.eclipse.core.resources.prefs ├── README.md ├── build.properties ├── build.xml ├── doc ├── New.txt ├── Ragpicker-hpfriends.png ├── Ragpicker.png ├── RagpickerManager.md ├── ToDo.txt ├── sample1.html ├── sample1.json ├── sample2.html └── sample2.json ├── scripts ├── beendeRagpicker ├── initialisiereCrawler ├── startVxCage └── updateScanner ├── src ├── config │ ├── crawler.conf │ ├── preProcessing.conf │ ├── processing.conf │ └── reporting.conf ├── core │ ├── __init__.py │ ├── abstracts.py │ ├── commonutils.py │ ├── config.py │ ├── constants.py │ ├── database.py │ ├── databaseMysql.py │ ├── objfile.py │ ├── statistics.py │ ├── vxCageHandler.py │ └── worker.py ├── crawler │ ├── __init__.py │ ├── cleanmx.py │ ├── cleanmx.yapsy-plugin │ ├── joxeankoret.py │ ├── joxeankoret.yapsy-plugin │ ├── malShare.py │ ├── malShare.yapsy-plugin │ ├── malc0de.py │ ├── malc0de.yapsy-plugin │ ├── malwarebl.py │ ├── malwarebl.yapsy-plugin │ ├── malwaredl.py │ ├── malwaredl.yapsy-plugin │ ├── secuboxlabs.py │ ├── secuboxlabs.yapsy-plugin │ ├── spyeyetracker.py │ ├── spyeyetracker.yapsy-plugin │ ├── urlquery.py │ ├── urlquery.yapsy-plugin │ ├── vxvault.py │ ├── vxvault.yapsy-plugin │ ├── zeustracker.py │ └── zeustracker.yapsy-plugin ├── data │ ├── clamav_dummy.hdb │ ├── html │ │ ├── base-report.html │ │ ├── htmlSections │ │ │ ├── antivirus.html │ │ │ ├── family.html │ │ │ ├── file_information.html │ │ │ ├── network_analysis.html │ │ │ ├── office.html │ │ │ ├── other.html │ │ │ ├── own_location.html │ │ │ ├── pdf.html │ │ │ ├── pe.html │ │ │ ├── rtf.html │ │ │ ├── source_information.html │ │ │ ├── subfile.html │ │ │ └── yara.html │ │ └── report.html │ ├── index_result.yar │ ├── index_sample.yar │ ├── peiddb.txt │ └── yara_signatures │ │ ├── result_apihttp.yar │ │ ├── result_blackListLanguages.yar │ │ ├── sample_blackListFunctions.yar │ │ ├── sample_blackliststrings.yar │ │ ├── sample_capabilities.yar │ │ ├── sample_embedded.yar │ │ ├── sample_kins.yar │ │ ├── sample_maldoc.yara │ │ ├── sample_njRAT.yar │ │ └── sample_vmdetect.yar ├── dumpdir │ ├── files │ │ └── empty.txt │ └── reports │ │ └── empty.txt ├── manager.py ├── preProcessing │ ├── 01_unpack_archive.py │ ├── 01_unpack_archive.yapsy-plugin │ ├── 02_unpacker_clamav.py │ ├── 02_unpacker_clamav.yapsy-plugin │ ├── 03_extract_rsrc.py │ ├── 03_extract_rsrc.yapsy-plugin │ ├── 04_extract_office.py │ ├── 04_extract_office.yapsy-plugin │ ├── 05_pe_carve.py │ ├── 05_pe_carve.yapsy-plugin │ └── __init__.py ├── processing │ ├── __init__.py │ ├── all_bluecoatMalwareAnalysisAppliance.py │ ├── all_bluecoatMalwareAnalysisAppliance.yapsy-plugin │ ├── all_cuckooSandbox.py │ ├── all_cuckooSandbox.yapsy-plugin │ ├── all_info.py │ ├── all_info.yapsy-plugin │ ├── all_subFile.py │ ├── all_subFile.yapsy-plugin │ ├── all_virustotal.py │ ├── all_virustotal.yapsy-plugin │ ├── all_yara.py │ ├── all_yara.yapsy-plugin │ ├── antivirus_avast.py │ ├── antivirus_avast.yapsy-plugin │ ├── antivirus_avg.py │ ├── antivirus_avg.yapsy-plugin │ ├── antivirus_avira.py │ ├── antivirus_avira.yapsy-plugin │ ├── antivirus_bitDefender.py │ ├── antivirus_bitDefender.yapsy-plugin │ ├── antivirus_clamav.py │ ├── antivirus_clamav.yapsy-plugin │ ├── antivirus_comodo.py │ ├── antivirus_comodo.yapsy-plugin │ ├── antivirus_eset.py │ ├── antivirus_eset.yapsy-plugin │ ├── antivirus_fprot.py │ ├── antivirus_fprot.yapsy-plugin │ ├── antivirus_fsecure.py │ ├── antivirus_fsecure.yapsy-plugin │ ├── antivirus_teamcymru.py │ ├── antivirus_teamcymru.yapsy-plugin │ ├── net_getOwnLocation.py │ ├── net_getOwnLocation.yapsy-plugin │ ├── net_inetSourceAnalysis.py │ ├── net_inetSourceAnalysis.yapsy-plugin │ ├── ole_officescan.py │ ├── ole_officescan.yapsy-plugin │ ├── pdf_pdfid.py │ ├── pdf_pdfid.yapsy-plugin │ ├── pe_pefile.py │ ├── pe_pefile.yapsy-plugin │ ├── pe_peid.py │ ├── pe_peid.yapsy-plugin │ ├── pe_verifySigs.py │ ├── pe_verifySigs.yapsy-plugin │ ├── rtf_rtfscan.py │ └── rtf_rtfscan.yapsy-plugin ├── ragpicker.py ├── reporting │ ├── __init__.py │ ├── codeDB.py │ ├── codeDB.yapsy-plugin │ ├── codeDBold.py │ ├── filedump.py │ ├── filedump.yapsy-plugin │ ├── hpfriends.py │ ├── hpfriends.yapsy-plugin │ ├── jsondump.py │ ├── jsondump.yapsy-plugin │ ├── mongodb.py │ ├── mongodb.yapsy-plugin │ ├── mysql.py │ ├── mysql.yapsy-plugin │ ├── reporthtml.py │ ├── reporthtml.yapsy-plugin │ ├── reportxml.py │ ├── reportxml.yapsy-plugin │ ├── vxcage.py │ └── vxcage.yapsy-plugin ├── statistics.py └── utils │ ├── OfficeMalScanner │ ├── Cadt.dll │ ├── DisView.exe │ ├── LZNT1Decompress.dll │ ├── MalHost-Setup.exe │ ├── OfficeMalScanner.exe │ ├── RTFScan.exe │ └── Unzipper.dll │ ├── __init__.py │ ├── codeDBobjects.py │ ├── dict2xml.py │ ├── exiftool.py │ ├── hpfeeds.py │ ├── logo.py │ ├── magic.py │ ├── multiPartForm.py │ ├── oletools │ ├── LICENSE.txt │ ├── __init__.py │ ├── ezhexviewer.py │ ├── olebrowse.py │ ├── oleid.py │ ├── olemeta.py │ ├── oletimes.py │ ├── olevba.py │ ├── pyxswf.py │ ├── rtfobj.py │ └── thirdparty │ │ ├── OleFileIO_PL │ │ ├── LICENSE.txt │ │ ├── OleFileIO_PL.py │ │ ├── README.txt │ │ └── __init__.py │ │ ├── __init__.py │ │ ├── easygui │ │ ├── LICENSE.txt │ │ ├── __init__.py │ │ └── easygui.py │ │ ├── olefile │ │ ├── CONTRIBUTORS.txt │ │ ├── LICENSE.txt │ │ ├── README.html │ │ ├── README.rst │ │ ├── __init__.py │ │ ├── doc │ │ │ ├── API.html │ │ │ ├── API.md │ │ │ ├── Contribute.html │ │ │ ├── Contribute.md │ │ │ ├── Home.html │ │ │ ├── Home.md │ │ │ ├── Install.html │ │ │ ├── Install.md │ │ │ ├── License.html │ │ │ ├── License.md │ │ │ ├── OLE_Overview.html │ │ │ ├── OLE_Overview.md │ │ │ └── OLE_VBA_sample.png │ │ ├── olefile.html │ │ ├── olefile.py │ │ ├── olefile2.html │ │ └── olefile2.py │ │ ├── prettytable │ │ ├── CHANGELOG │ │ ├── COPYING │ │ ├── README │ │ ├── __init__.py │ │ └── prettytable.py │ │ ├── xglob │ │ ├── LICENSE.txt │ │ ├── __init__.py │ │ └── xglob.py │ │ └── xxxswf │ │ ├── LICENSE.txt │ │ ├── __init__.py │ │ └── xxxswf.py │ ├── ordlookup │ ├── __init__.py │ ├── oleaut32.py │ └── ws2_32.py │ ├── pdf-parser.py │ ├── pdfid.py │ ├── pefile.py │ ├── peutils.py │ ├── pyasn1 │ ├── __init__.py │ ├── codec │ │ ├── __init__.py │ │ ├── ber │ │ │ ├── __init__.py │ │ │ ├── decoder.py │ │ │ ├── encoder.py │ │ │ └── eoo.py │ │ ├── cer │ │ │ ├── __init__.py │ │ │ ├── decoder.py │ │ │ └── encoder.py │ │ └── der │ │ │ ├── __init__.py │ │ │ ├── decoder.py │ │ │ └── encoder.py │ ├── compat │ │ ├── __init__.py │ │ └── octets.py │ ├── debug.py │ ├── dn.py │ ├── error.py │ ├── oids.py │ ├── pkcs7.py │ ├── spc.py │ ├── time_test.py │ ├── type │ │ ├── __init__.py │ │ ├── base.py │ │ ├── char.py │ │ ├── constraint.py │ │ ├── error.py │ │ ├── namedtype.py │ │ ├── namedval.py │ │ ├── tag.py │ │ ├── tagmap.py │ │ ├── univ.py │ │ └── useful.py │ ├── x509.py │ └── x509_time.py │ ├── rarfile.py │ ├── socks.py │ └── verifySigs │ ├── __init__.py │ ├── auth_data.py │ ├── fingerprint.py │ ├── pecoff_blob.py │ └── sigcheck.exe └── versions ├── build.number ├── ragpicker_v0.06.0_remnux.zip └── ragpicker_version.json /.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | Ragpicker 4 | 5 | 6 | 7 | 8 | 9 | org.python.pydev.PyDevBuilder 10 | 11 | 12 | 13 | 14 | 15 | org.python.pydev.pythonNature 16 | 17 | 18 | -------------------------------------------------------------------------------- /.pydevproject: -------------------------------------------------------------------------------- 1 | 2 | 3 | Default 4 | python 2.7 5 | 6 | /${PROJECT_DIR_NAME}/src 7 | 8 | 9 | -------------------------------------------------------------------------------- /.settings/org.eclipse.core.resources.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | encoding//src/utils/codeDBobjects.py=utf-8 3 | -------------------------------------------------------------------------------- /build.properties: -------------------------------------------------------------------------------- 1 | #Sat, 07 Dec 2013 23:20:23 +0100 2 | ragpicker.build.major.number=0 3 | ragpicker.build.minor.number=06 4 | doc=doc 5 | src=src 6 | install_dir=/opt/ragpicker 7 | -------------------------------------------------------------------------------- /doc/New.txt: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /doc/Ragpicker-hpfriends.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/robbyFux/Ragpicker/77d68ff3a13851ed4c256635f0673a4c2a84ef51/doc/Ragpicker-hpfriends.png -------------------------------------------------------------------------------- /doc/Ragpicker.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/robbyFux/Ragpicker/77d68ff3a13851ed4c256635f0673a4c2a84ef51/doc/Ragpicker.png -------------------------------------------------------------------------------- /doc/ToDo.txt: -------------------------------------------------------------------------------- 1 | Crawler 2 | Geodo Tracker: 3 | http://gtrack.h3x.eu/corpus/120 4 | http://gtrack.h3x.eu/corpus/122 5 | 6 | http://malwareurls.joxeankoret.com/grayarea.txt 7 | http://malwareurls.joxeankoret.com/normal.txt 8 | 9 | _________________________________________________________________________________________________________________ 10 | 11 | 12 | rtf-shellcode-carv.py 13 | 14 | _________________________________________________________________________________________________________________ 15 | 16 | 17 | # If not enough free disk space is available, then we print an 18 | # error message and wait another round (this check is ignored 19 | # when the freespace configuration variable is set to zero). 20 | import os 21 | import sys 22 | 23 | if __name__ == '__main__': 24 | FREE_SPACE = 1024 #MB 25 | 26 | dir_stats = os.statvfs("/opt/") 27 | # Calculate the free disk space in megabytes. 28 | space_available = dir_stats.f_bavail * dir_stats.f_frsize 29 | space_available /= 1024 * 1024 30 | 31 | if space_available < FREE_SPACE: 32 | sys.stderr.write("Not enough free disk space! (Only %d MB!)" % space_available) 33 | sys.exit() -------------------------------------------------------------------------------- /scripts/beendeRagpicker: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | killall -r .*ragpicker.* 4 | -------------------------------------------------------------------------------- /scripts/initialisiereCrawler: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | /opt/bin/updateScanner 4 | 5 | echo " " 6 | echo " " 7 | echo "starte VxCage" 8 | /opt/bin/startVxCage 9 | -------------------------------------------------------------------------------- /scripts/startVxCage: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd /opt/vxcage 4 | ./api.py -p 8888 5 | -------------------------------------------------------------------------------- /scripts/updateScanner: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo 4 | echo "--- Update F-Prot ---" 5 | sudo /opt/f-prot/fpupdate 6 | 7 | echo 8 | echo "--- Update ClamAV ---" 9 | sudo /usr/bin/freshclam 10 | 11 | echo 12 | echo "--- Update BitDefender ---" 13 | sudo /opt/BitDefender-scanner/bin/bdscan --update 14 | 15 | echo 16 | echo "--- Update AVG ---" 17 | sudo /usr/bin/avgupdate 18 | 19 | echo 20 | echo "--- Update Avira ---" 21 | sudo /usr/sbin/avguard stop 22 | sudo /usr/sbin/avupdate-guard --product=Guard 23 | sudo /usr/sbin/avguard start 24 | 25 | echo 26 | echo "--- Update F-Secure Linux Security 10.10.142 ---" 27 | sudo /opt/f-secure/fsav/bin/dbupdate 28 | 29 | echo 30 | echo "--- Update Avast (avast4workstation) ---" 31 | sudo sysctl -w kernel.shmmax=200000000 32 | sudo /opt/avast/bin/avast-update 33 | 34 | echo 35 | echo "--- Update Eset File Security v4.0.10 ---" 36 | sudo /opt/eset/esets/sbin/esets_update 37 | 38 | echo -------------------------------------------------------------------------------- /src/config/crawler.conf: -------------------------------------------------------------------------------- 1 | [clientConfig] 2 | #Route Malware requests through the TOR SOCKS proxy 3 | #https://www.torproject.org/docs/debian 4 | tor_enabled = no 5 | tor_proxyadress = localhost 6 | tor_proxyport = 9050 7 | 8 | #browserHeaders 9 | browser_accept_language = de-de, de, en;q=0.5, fr;q=0.2 10 | #browser_accept_language = en-US,en;q=0.8 11 | 12 | #Internet Explorer 8.0 on Windows XP 13 | browser_user_agent = Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 1.1.4322) 14 | 15 | #Internet Explorer 8.0 on Windows 7 16 | #browser_user_agent = Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; Media Center PC 6.0; InfoPath.2; MS-RTC LM 8) 17 | 18 | #Firefox 12 on Windows XP 19 | #browser_user_agent = Mozilla/5.0 (Windows NT 5.1; rv:12.0) Gecko/20120403211507 Firefox/12.0 20 | 21 | [cleanmx] 22 | enabled = yes 23 | 24 | [malShare] 25 | enabled = yes 26 | # Add your MalShare Project API key here. 27 | # MalShare offer free public API keys. 28 | # Malshare are currently limiting requests from each key to 1000 sample pulls per day. 29 | # http://www.malshare.com/register.php 30 | limit = 1000 31 | apikey = 32 | 33 | [malc0de] 34 | enabled = yes 35 | 36 | [malwarebl] 37 | enabled = yes 38 | 39 | [malwaredl] 40 | enabled = yes 41 | 42 | [secuboxlabs] 43 | enabled = yes 44 | 45 | [vxvault] 46 | enabled = yes 47 | 48 | [zeustracker] 49 | enabled = yes 50 | 51 | [spyeyetracker] 52 | enabled = yes 53 | 54 | [joxeankoret] 55 | enabled = yes 56 | 57 | [urlquery] 58 | enabled = no 59 | 60 | #URLs not being processing 61 | [urlBlackList] 62 | url = google-earth.xtremedownload.com, xy.test.com, 123.test.com 63 | 64 | [test] 65 | enabled = no 66 | url = http://127.0.0.1/malware/11.pdf,http://127.0.0.1/malware/1.pdf,http://127.0.0.1/malware/2.pdf,http://127.0.0.1/malware/3.pdf,http://127.0.0.1/malware/4.pdf,http://127.0.0.1/malware/5.pdf,http://127.0.0.1/malware/6.pdf,http://127.0.0.1/malware/7.pdf,http://127.0.0.1/malware/8.pdf,http://127.0.0.1/malware/9.pdf,http://127.0.0.1/malware/10.pdf,http://127.0.0.1/malware/1.exe,http://127.0.0.1/malware/2.exe,http://127.0.0.1/malware/3.exe,http://127.0.0.1/malware/4.exe,http://127.0.0.1/malware/5.exe,http://127.0.0.1/malware/6.exe,http://127.0.0.1/malware/7.exe,http://127.0.0.1/malware/8.exe,http://127.0.0.1/malware/9.exe,http://127.0.0.1/malware/10.exe,http://127.0.0.1/malware/sigtest1.exe,http://127.0.0.1/malware/sigtest2.exe,http://127.0.0.1/malware/sigtest3.exe,http://127.0.0.1/malware/sigtest4.exe,http://127.0.0.1/malware/sigtest5.exe,http://127.0.0.1/malware/sigtest6.exe,http://127.0.0.1/malware/sigtest7.exe,http://127.0.0.1/malware/sigtest8.exe,http://127.0.0.1/malware/sigtest9.exe,http://127.0.0.1/malware/sigtest10.exe,http://127.0.0.1/malware/sigtest11.sys,http://127.0.0.1/malware/10.exe,http://127.0.0.1/malware/11.exe,http://127.0.0.1/malware/12.exe,http://127.0.0.1/malware/13.exe -------------------------------------------------------------------------------- /src/config/preProcessing.conf: -------------------------------------------------------------------------------- 1 | [01_unpack_archive] 2 | enabled = yes 3 | dataTypes = Zip, RAR 4 | 5 | [02_unpacker_clamav] 6 | enabled = yes 7 | dataTypes = PE32, PE32+, MS-DOS 8 | clamscan_path = /usr/bin/clamscan 9 | 10 | [03_extract_rsrc] 11 | enabled = yes 12 | dataTypes = PE32, PE32+, MS-DOS 13 | extractTypes = Zip, RAR 14 | 15 | [04_extract_office] 16 | #uses RTFScan and OfficeMalScanner from OfficeMalScanner on Wine (http://www.reconstructer.org/code.html) 17 | #requires a Wine installation! 18 | enabled = yes 19 | dataTypes = Rich, Composite 20 | #path to wine 21 | wine = /usr/bin/wine 22 | #brute - enables the "brute force mode" to find encrypted stuff (slow!) 23 | brute = true 24 | 25 | [05_pe_carve] 26 | enabled = yes 27 | dataTypes = PE32, PE32+, MS-DOS -------------------------------------------------------------------------------- /src/config/processing.conf: -------------------------------------------------------------------------------- 1 | [all_info] 2 | #Don't disable "info"-Module!!! 3 | enabled = yes 4 | 5 | [all_bluecoatMalwareAnalysisAppliance] 6 | #https://www.bluecoat.com/products/malware-analysis-appliance 7 | enabled = no 8 | 9 | #MAA server Host 10 | host = 127.0.0.1 11 | #MAA server Port 12 | port = 443 13 | #tp_IVM.TIMEOUT 14 | timeout = 120 15 | #The MAA needs an API key 16 | apikey = 17 | #The user name under which the files are to be executed 18 | user = 19 | # 20 | https = yes 21 | 22 | [all_cuckooSandbox] 23 | #Adds the Sample to the list of pending tasks to be processed and analyzed. 24 | #Response = Task-ID 25 | #Needs Cuckoo's lightweight REST API server! 26 | #http://docs.cuckoosandbox.org/en/latest/usage/api.html 27 | enabled = no 28 | 29 | #REST API server Host 30 | host = 127.0.0.1 31 | #REST API server Port 32 | port = 8090 33 | 34 | [all_subFile] 35 | #Find subfile in any binary stream 36 | enabled = yes 37 | 38 | [all_virustotal] 39 | enabled = no 40 | # Add your VirusTotal API key here. The default API key, kindly provided 41 | # by the VirusTotal team, should enable you with a sufficient throughput 42 | # and while being shared with all our users, it shouldn't affect your use. 43 | apikey = f05482f8d7018cfb03d3f2974c673690662daa641bec29a251d9992fe9220e31 44 | 45 | [all_yara] 46 | enabled = no 47 | rulepath = ./data/index_sample.yar 48 | 49 | [net_inetSourceAnalysis] 50 | enabled = yes 51 | urlvoid = yes 52 | fortiguard = yes 53 | urlquery = yes 54 | ipvoid = yes 55 | #No analysis of these Domains (comma-separated list) 56 | whitelist = api.malshare.com,malwr.com,127.0.0.1 57 | 58 | [net_getOwnLocation] 59 | #returns the own internet location when you use tor, proxy or v-server 60 | enabled = no 61 | 62 | [antivirus_teamcymru] 63 | enabled = no 64 | 65 | [antivirus_avast] 66 | enabled = no 67 | avast_path = /usr/bin/avast 68 | 69 | [antivirus_avg] 70 | enabled = no 71 | avg_path = /usr/bin/avgscan 72 | 73 | [antivirus_avira] 74 | enabled = no 75 | avira_path = /usr/bin/avscan 76 | avira_heur_level = 3 77 | quarantine-dir = /tmp/ 78 | 79 | [antivirus_bitDefender] 80 | enabled = no 81 | bdscan_path = /usr/bin/bdscan 82 | 83 | [antivirus_clamav] 84 | enabled = no 85 | clamscan_path = /usr/bin/clamscan 86 | 87 | [antivirus_eset] 88 | enabled = no 89 | eset_path = /opt/eset/esets/sbin/esets_scan 90 | 91 | [antivirus_comodo] 92 | enabled = no 93 | comodo_path = /opt/COMODO/cmdscan 94 | 95 | [antivirus_fprot] 96 | enabled = no 97 | fprot_path = /usr/local/bin/fpscan 98 | 99 | [antivirus_fsecure] 100 | enabled = no 101 | fsecure_path = /opt/f-secure/fsav/bin/fsav 102 | 103 | [pdf_pdfid] 104 | enabled = yes 105 | dataTypes = PDF 106 | 107 | [pe_verifySigs] 108 | #Compute hashes and validate signatures. 109 | enabled = yes 110 | dataTypes = PE32, PE32+, MS-DOS 111 | #second chance with sysinternals sigcheck on wine 112 | sysinternals_enabled = true 113 | #path to wine 114 | wine = /usr/bin/wine 115 | 116 | [pe_pefile] 117 | enabled = yes 118 | dataTypes = PE32, PE32+, MS-DOS 119 | 120 | [pe_peid] 121 | enabled = yes 122 | dataTypes = PE32, PE32+, MS-DOS 123 | 124 | [ole_officescan] 125 | #uses OfficeMalScanner on Wine (http://www.reconstructer.org/code.html) 126 | #requires a Wine installation! 127 | enabled = yes 128 | dataTypes = Composite 129 | #path to wine 130 | wine = /usr/bin/wine 131 | #brute - enables the "brute force mode" to find encrypted stuff (slow!) 132 | brute = true 133 | 134 | [rtf_rtfscan] 135 | #uses RTFScan from OfficeMalScanner on Wine (http://www.reconstructer.org/code.html) 136 | #requires a Wine installation! 137 | enabled = yes 138 | dataTypes = Rich 139 | #path to wine 140 | wine = /usr/bin/wine 141 | -------------------------------------------------------------------------------- /src/config/reporting.conf: -------------------------------------------------------------------------------- 1 | [reporthtml] 2 | enabled = yes 3 | 4 | #Make sure this path is correct. 5 | dumpdir = ./dumpdir/reports/ 6 | 7 | [reportxml] 8 | enabled = no 9 | 10 | #Make sure this path is correct. 11 | dumpdir = ./dumpdir/reports/ 12 | 13 | [jsondump] 14 | enabled = no 15 | 16 | #Make sure this path is correct. 17 | dumpdir = ./dumpdir/reports/ 18 | 19 | [filedump] 20 | enabled = yes 21 | suffix = .vir 22 | 23 | #Make sure this path is correct. 24 | dumpdir = ./dumpdir/files/ 25 | 26 | [mongodb] 27 | enabled = no 28 | host = 127.0.0.1 29 | port = 27017 30 | 31 | [mysql] 32 | enabled = no 33 | host = 127.0.0.1 34 | port = 3306 35 | database = ragpicker 36 | user = ragpicker 37 | password = ragpicker 38 | 39 | [vxcage] 40 | enabled = no 41 | 42 | #REST API server Host 43 | host = 127.0.0.1 44 | #REST API server Port 45 | port = 8080 46 | 47 | [hpfriends] 48 | enabled = no 49 | channel_reports = ragpicker.reports 50 | channel_files = ragpicker.files 51 | host = hpfriends.honeycloud.net 52 | port = 20000 53 | ident = 54 | secret = 55 | 56 | [codeDB] 57 | enabled = no 58 | 59 | #REST API-Server Host 60 | host = localhost 61 | #REST API-Server Port 62 | port = 8888 63 | #API-Server user 64 | user = test 65 | #password 66 | password = 1234qwer -------------------------------------------------------------------------------- /src/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/robbyFux/Ragpicker/77d68ff3a13851ed4c256635f0673a4c2a84ef51/src/core/__init__.py -------------------------------------------------------------------------------- /src/core/config.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | import ConfigParser 5 | 6 | 7 | class Config: 8 | """Configuration file parser.""" 9 | 10 | def __init__(self, cfg): 11 | """@param cfg: configuration file path.""" 12 | config = ConfigParser.ConfigParser() 13 | config.read(cfg) 14 | 15 | for section in config.sections(): 16 | setattr(self, section, Dictionary()) 17 | for name, raw_value in config.items(section): 18 | try: 19 | value = config.getboolean(section, name) 20 | except ValueError: 21 | try: 22 | value = config.getint(section, name) 23 | except ValueError: 24 | value = config.get(section, name) 25 | 26 | setattr(getattr(self, section), name, value) 27 | 28 | def get(self, section): 29 | """Get option. 30 | @param section: section to fetch. 31 | @raise Exception: if section not found. 32 | @return: option value. 33 | """ 34 | try: 35 | return getattr(self, section) 36 | except AttributeError as e: 37 | raise Exception("Option %s is not found in configuration, error: %s" % (section, e)) 38 | 39 | def getOption(self, section, option): 40 | """Get option. 41 | @param section: section to fetch, option: value of a specified option in section 42 | @raise Exception: if section not found. 43 | @return: option value. 44 | """ 45 | try: 46 | return getattr(getattr(self, section), option) 47 | except AttributeError as e: 48 | raise Exception("Section %s is not found in configuration, error: %s" % (section, e)) 49 | 50 | class Dictionary(dict): 51 | """Ragpicker custom dict.""" 52 | 53 | def __getattr__(self, key): 54 | return self.get(key, None) 55 | 56 | __setattr__ = dict.__setitem__ 57 | __delattr__ = dict.__delitem__ 58 | -------------------------------------------------------------------------------- /src/core/databaseMysql.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | import os 5 | import logging 6 | from core.config import Config 7 | from core.constants import RAGPICKER_ROOT 8 | 9 | try: 10 | import MySQLdb 11 | except ImportError: 12 | raise Exception("MySQLdb is required for working with MySQL: http://mysql-python.sourceforge.net/") 13 | 14 | def singleton(class_): 15 | instances = {} 16 | def getinstance(*args, **kwargs): 17 | if class_ not in instances: 18 | instances[class_] = class_(*args, **kwargs) 19 | return instances[class_] 20 | return getinstance 21 | 22 | log = logging.getLogger("DatabaseMySQL") 23 | 24 | @singleton 25 | class DatabaseMySQL(): 26 | 27 | def __init__(self): 28 | self.__cfgReporting = Config(os.path.join(RAGPICKER_ROOT, 'config', 'reporting.conf')) 29 | self.__mysqlEnabled = self.__cfgReporting.getOption("mysql", "enabled") 30 | 31 | if self.__mysqlEnabled: 32 | #Anbindung an Datenbank MySQL herstellen 33 | try: 34 | mysqlHost = self.__cfgReporting.getOption("mysql", "host") 35 | mysqlPort = self.__cfgReporting.getOption("mysql", "port") 36 | mysqlDatabase = self.__cfgReporting.getOption("mysql", "database") 37 | mysqlUser = self.__cfgReporting.getOption("mysql", "user") 38 | mysqlPassword = self.__cfgReporting.getOption("mysql", "password") 39 | self.__mysqlConnection = MySQLdb.Connect(host=mysqlHost, port=mysqlPort, db=mysqlDatabase, user=mysqlUser, passwd=mysqlPassword) 40 | except (Exception) as e: 41 | raise Exception("Cannot connect to MySQL (ragpicker): %s" % e) 42 | 43 | def __del__(self): 44 | if self.__mysqlEnabled: 45 | self.__mysqlConnection.close() 46 | 47 | # ------------------------------------------------------------------------------ 48 | # Ragpicker Database (MySQL) 49 | # ------------------------------------------------------------------------------ 50 | 51 | def isRagpickerDBEnabledMySQL(self): 52 | return self.__mysqlEnabled 53 | -------------------------------------------------------------------------------- /src/crawler/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/robbyFux/Ragpicker/77d68ff3a13851ed4c256635f0673a4c2a84ef51/src/crawler/__init__.py -------------------------------------------------------------------------------- /src/crawler/cleanmx.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | import logging 5 | import re 6 | 7 | from core.abstracts import Crawler 8 | 9 | 10 | try: 11 | from yapsy.IPlugin import IPlugin 12 | except ImportError: 13 | raise ImportError, 'Yapsy (Yet Another Plugin System) is required to run this program : http://yapsy.sourceforge.net' 14 | 15 | log = logging.getLogger("CleanmxCrawler") 16 | 17 | class Cleanmx(IPlugin, Crawler): 18 | 19 | def run(self): 20 | self.mapURL = {} 21 | log.debug("Fetching from Cleanmx RSS") 22 | 23 | # parser 24 | soup = self.parse('http://support.clean-mx.de/clean-mx/xmlviruses.php?') 25 | 26 | for row in soup('url'): 27 | try: 28 | self.storeURL(re.sub('[\[CDATA\]]', '', row.string)) 29 | except Exception as e: 30 | log.error('Error in get from soup: (%s)', e) 31 | 32 | log.info("Found %s urls" % len(self.mapURL)) 33 | 34 | return self.mapURL 35 | -------------------------------------------------------------------------------- /src/crawler/cleanmx.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = cleanmx 3 | Module = cleanmx 4 | 5 | [Documentation] 6 | Description = Fetching Malware-URLs from Cleanmx RSS (http://support.clean-mx.de) 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/crawler/joxeankoret.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2014 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | import itertools 5 | import logging 6 | import urllib2 7 | 8 | from core.abstracts import Crawler 9 | 10 | try: 11 | from yapsy.IPlugin import IPlugin 12 | except ImportError: 13 | raise ImportError, 'Yapsy (Yet Another Plugin System) is required to run this program : http://yapsy.sourceforge.net' 14 | 15 | log = logging.getLogger("JoxeanKoretCrawler") 16 | URL_DAILY = "http://malwareurls.joxeankoret.com/normal.txt" 17 | 18 | class JoxeanKoret(IPlugin, Crawler): 19 | 20 | def run(self): 21 | self.mapURL = {} 22 | log.debug("Fetching from JoxeanKoret daily list") 23 | 24 | for url in self.pull_daily_list(): 25 | self.storeURL(url) 26 | 27 | log.info("Found %s urls" % len(self.mapURL)) 28 | 29 | return self.mapURL 30 | 31 | def pull_daily_list(self): 32 | try: 33 | request = urllib2.Request(URL_DAILY) 34 | data = urllib2.urlopen(request, timeout=60) 35 | for line in data.readlines(): 36 | yield line 37 | except Exception as e: 38 | log.error("Problem connecting to pull the daily list.") 39 | -------------------------------------------------------------------------------- /src/crawler/joxeankoret.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = joxeankoret 3 | Module = joxeankoret 4 | 5 | [Documentation] 6 | Description = Fetching Malware-URLs from joxeankoret.com (http://malwareurls.joxeankoret.com) 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/crawler/malShare.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2014 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | import itertools 5 | import logging 6 | import urllib2 7 | 8 | from core.abstracts import Crawler 9 | 10 | 11 | try: 12 | from yapsy.IPlugin import IPlugin 13 | except ImportError: 14 | raise ImportError, 'Yapsy (Yet Another Plugin System) is required to run this program : http://yapsy.sourceforge.net' 15 | 16 | log = logging.getLogger("MalShareCrawler") 17 | URL_DAILY = "http://www.malshare.com/daily/malshare.current.txt" 18 | URL_GET_FILE = "http://api.malshare.com/sampleshare.php?action=getfile&api_key=%s&hash=%s" 19 | 20 | class MalShare(IPlugin, Crawler): 21 | 22 | def run(self): 23 | apikeys = self.options.get("apikey", None) 24 | # Cycle through these values. 25 | apikeys = itertools.cycle(apikeys.split(',')) 26 | limit = int(self.options.get("limit", 1000)) 27 | self.mapURL = {} 28 | log.debug("Fetching from MalShare daily list") 29 | 30 | if not apikeys: 31 | raise Exception("MalShare API key not configured, skip") 32 | 33 | for md5_hash in self.pull_daily_list(): 34 | limit -= 1 35 | if limit >= 0 and md5_hash: 36 | url = URL_GET_FILE % (apikeys.next(), md5_hash) 37 | self.storeURL(url) 38 | 39 | log.info("Found %s urls" % len(self.mapURL)) 40 | 41 | return self.mapURL 42 | 43 | def pull_daily_list(self): 44 | try: 45 | request = urllib2.Request(URL_DAILY) 46 | data = urllib2.urlopen(request, timeout=60) 47 | for line in data.readlines(): 48 | yield line 49 | except Exception as e: 50 | log.error("Problem connecting to pull the daily list.") 51 | -------------------------------------------------------------------------------- /src/crawler/malShare.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = malShare 3 | Module = malShare 4 | 5 | [Documentation] 6 | Description = Fetching Malware-URLs from MalShare daily list (http://www.malshare.com) 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/crawler/malc0de.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | import logging 5 | import re 6 | 7 | from core.abstracts import Crawler 8 | 9 | 10 | try: 11 | from yapsy.IPlugin import IPlugin 12 | except ImportError: 13 | raise ImportError, 'Yapsy (Yet Another Plugin System) is required to run this program : http://yapsy.sourceforge.net' 14 | 15 | log = logging.getLogger("Malc0deCrawler") 16 | 17 | class Malc0de(IPlugin, Crawler): 18 | 19 | def run(self): 20 | self.mapURL = {} 21 | mlc = [] 22 | log.debug("Fetching from Malc0de RSS") 23 | 24 | # parser 25 | soup = self.parse('http://malc0de.com/rss') 26 | 27 | for row in soup('description'): 28 | mlc.append(row) 29 | del mlc[0] 30 | 31 | for row in mlc: 32 | site = re.sub('&', '&', str(row).split()[1]).replace(',', '') 33 | self.storeURL(site) 34 | 35 | log.info("Found %s urls" % len(self.mapURL)) 36 | 37 | return self.mapURL 38 | -------------------------------------------------------------------------------- /src/crawler/malc0de.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = malc0de 3 | Module = malc0de 4 | 5 | [Documentation] 6 | Description = Fetching Malware-URLs from Malc0de RSS (http://malc0de.com) 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/crawler/malwarebl.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | import logging 5 | 6 | from core.abstracts import Crawler 7 | 8 | 9 | try: 10 | from yapsy.IPlugin import IPlugin 11 | except ImportError: 12 | raise ImportError, 'Yapsy (Yet Another Plugin System) is required to run this program : http://yapsy.sourceforge.net' 13 | 14 | log = logging.getLogger("MalwareblCrawler") 15 | 16 | class Malwarebl(IPlugin, Crawler): 17 | 18 | def run(self): 19 | self.mapURL = {} 20 | log.debug("Fetching from Malware Black List") 21 | 22 | # parser 23 | soup = self.parse('http://www.malwareblacklist.com/mbl.xml') 24 | 25 | for row in soup('description'): 26 | site = str(row).split()[1].replace(',', '') 27 | self.storeURL(site) 28 | 29 | log.info("Found %s urls" % len(self.mapURL)) 30 | 31 | return self.mapURL 32 | -------------------------------------------------------------------------------- /src/crawler/malwarebl.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = malwarebl 3 | Module = malwarebl 4 | 5 | [Documentation] 6 | Description = Fetching Malware-URLs from Malware Black List (http://www.malwareblacklist.com) 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/crawler/malwaredl.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | import logging 5 | import re 6 | 7 | from core.abstracts import Crawler 8 | 9 | 10 | try: 11 | from yapsy.IPlugin import IPlugin 12 | except ImportError: 13 | raise ImportError, 'Yapsy (Yet Another Plugin System) is required to run this program : http://yapsy.sourceforge.net' 14 | 15 | log = logging.getLogger("MalwaredlCrawler") 16 | 17 | class Malwaredl(IPlugin, Crawler): 18 | 19 | def run(self): 20 | self.mapURL = {} 21 | mdl = [] 22 | log.debug("Fetching from Malware Domain List") 23 | 24 | # parser 25 | soup = self.parse('http://www.malwaredomainlist.com/hostslist/mdl.xml') 26 | 27 | for row in soup('description'): 28 | mdl.append(row) 29 | del mdl[0] 30 | 31 | for row in mdl: 32 | site = re.sub('&', '&', str(row).split()[1]).replace(',', '') 33 | if site == '-': 34 | self.storeURL(re.sub('&', '&', str(row).split()[4]).replace(',', '')) 35 | else: 36 | self.storeURL(site) 37 | 38 | log.info("Found %s urls" % len(self.mapURL)) 39 | 40 | return self.mapURL 41 | -------------------------------------------------------------------------------- /src/crawler/malwaredl.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = malwaredl 3 | Module = malwaredl 4 | 5 | [Documentation] 6 | Description = Fetching Malware-URLs from Malware Domain List (http://www.malwaredomainlist.com) 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/crawler/secuboxlabs.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | import logging 5 | import re 6 | 7 | from core.abstracts import Crawler 8 | 9 | 10 | try: 11 | from yapsy.IPlugin import IPlugin 12 | except ImportError: 13 | raise ImportError, 'Yapsy (Yet Another Plugin System) is required to run this program : http://yapsy.sourceforge.net' 14 | 15 | log = logging.getLogger("SecuboxlabsCrawler") 16 | 17 | class Secuboxlabs(IPlugin, Crawler): 18 | 19 | def run(self): 20 | self.mapURL = {} 21 | log.debug("Fetching from secuboxlabs.fr") 22 | 23 | # parser 24 | soup = self.parse('http://secuboxlabs.fr/152caf4d68ad169e7ffc370afa4f2939.rss') 25 | 26 | for row in soup('description'): 27 | for x in re.findall("malware://[-A-Za-z0-9+&@#/%?=~_|!:,.;]*[-A-Za-z0-9+&@#/%=~_|]", str(row)): 28 | site = x.replace('malware://', 'http://') 29 | self.storeURL(site) 30 | 31 | log.info("Found %s urls" % len(self.mapURL)) 32 | 33 | return self.mapURL 34 | -------------------------------------------------------------------------------- /src/crawler/secuboxlabs.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = secuboxlabs 3 | Module = secuboxlabs 4 | 5 | [Documentation] 6 | Description = Fetching Malware-URLs from SecuBox Labs (FRANCE) RSS (http://secuboxlabs.fr) 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/crawler/spyeyetracker.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | import logging 5 | 6 | from core.abstracts import Crawler 7 | 8 | 9 | try: 10 | from yapsy.IPlugin import IPlugin 11 | except ImportError: 12 | raise ImportError, 'Yapsy (Yet Another Plugin System) is required to run this program : http://yapsy.sourceforge.net' 13 | 14 | log = logging.getLogger("SpyEyetrackerCrawler") 15 | 16 | class SpyEyetracker(IPlugin, Crawler): 17 | 18 | def run(self): 19 | self.mapURL = {} 20 | log.debug("Fetching from SpyEyetracker RSS") 21 | 22 | # parser 23 | soup = self.parse('https://spyeyetracker.abuse.ch/monitor.php?rssfeed=binaryurls') 24 | 25 | for row in soup('description'): 26 | site = str(row).split()[2].replace(',', '') 27 | if site != "This": 28 | self.storeURL(site) 29 | 30 | log.info("Found %s urls" % len(self.mapURL)) 31 | 32 | return self.mapURL 33 | -------------------------------------------------------------------------------- /src/crawler/spyeyetracker.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = spyeyetracker 3 | Module = spyeyetracker 4 | 5 | [Documentation] 6 | Description = Fetching Malware-URLs from SpyEyetracker RSS (https://spyeyetracker.abuse.ch) 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/crawler/urlquery.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | import logging 5 | import re 6 | 7 | from core.abstracts import Crawler 8 | 9 | 10 | try: 11 | from yapsy.IPlugin import IPlugin 12 | except ImportError: 13 | raise ImportError, 'Yapsy (Yet Another Plugin System) is required to run this program : http://yapsy.sourceforge.net' 14 | 15 | log = logging.getLogger("UrlQueryCrawler") 16 | 17 | class UrlQuery(IPlugin, Crawler): 18 | 19 | def run(self): 20 | self.mapURL = {} 21 | log.debug("Fetching from UrlQuery") 22 | 23 | # parser 24 | soup = self.parse('http://urlquery.net') 25 | 26 | for t in soup("table", { "class" : "test" }): 27 | for a in t("a"): 28 | self.storeURL('http://' + re.sub('&', '&', a.text)) 29 | 30 | log.info("Found %s urls" % len(self.mapURL)) 31 | 32 | return self.mapURL -------------------------------------------------------------------------------- /src/crawler/urlquery.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = urlquery 3 | Module = urlquery 4 | 5 | [Documentation] 6 | Description = Fetching Malware-URLs from urlquery.net (http://urlquery.net) 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/crawler/vxvault.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | import logging 5 | 6 | from core.abstracts import Crawler 7 | 8 | 9 | try: 10 | from yapsy.IPlugin import IPlugin 11 | except ImportError: 12 | raise ImportError, 'Yapsy (Yet Another Plugin System) is required to run this program : http://yapsy.sourceforge.net' 13 | 14 | log = logging.getLogger("VxvaultCrawler") 15 | 16 | class Vxvault(IPlugin, Crawler): 17 | 18 | def run(self): 19 | self.mapURL = {} 20 | log.debug("Fetching from VXVault List") 21 | 22 | # parser 23 | soup = self.parse('http://vxvault.siri-urz.net/URL_List.php') 24 | 25 | vxv = [] 26 | 27 | for row in soup('pre'): 28 | vxv = row.string.split('\r\n') 29 | del vxv[:4] 30 | del vxv[-1] 31 | log.info("Found %s urls" % len(vxv)) 32 | for row in vxv: 33 | self.storeURL(row) 34 | 35 | return self.mapURL 36 | -------------------------------------------------------------------------------- /src/crawler/vxvault.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = vxvault 3 | Module = vxvault 4 | 5 | [Documentation] 6 | Description = Fetching Malware-URLs from VXVault (http://vxvault.siri-urz.net) 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/crawler/zeustracker.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | import logging 5 | 6 | from core.abstracts import Crawler 7 | 8 | 9 | try: 10 | from yapsy.IPlugin import IPlugin 11 | except ImportError: 12 | raise ImportError, 'Yapsy (Yet Another Plugin System) is required to run this program : http://yapsy.sourceforge.net' 13 | 14 | log = logging.getLogger("ZeustrackerCrawler") 15 | 16 | class Zeustracker(IPlugin, Crawler): 17 | 18 | def run(self): 19 | self.mapURL = {} 20 | log.debug("Fetching from Zeustracker RSS") 21 | 22 | # parser 23 | soup = self.parse('https://zeustracker.abuse.ch/monitor.php?urlfeed=binaries') 24 | 25 | for row in soup('description'): 26 | site = str(row).split()[1].replace(',', '') 27 | if site != "ZeuS": 28 | self.storeURL(site) 29 | 30 | log.info("Found %s urls" % len(self.mapURL)) 31 | 32 | return self.mapURL 33 | -------------------------------------------------------------------------------- /src/crawler/zeustracker.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = zeustracker 3 | Module = zeustracker 4 | 5 | [Documentation] 6 | Description = Fetching Malware-URLs from Zeustracker RSS (https://zeustracker.abuse.ch) 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/data/clamav_dummy.hdb: -------------------------------------------------------------------------------- 1 | 0f8816902a5bd1c10a87e06100e68d68:128804:Hello_World.exe 2 | -------------------------------------------------------------------------------- /src/data/html/htmlSections/family.html: -------------------------------------------------------------------------------- 1 |
^
2 | 3 |
4 |
Family Information
5 | 6 | 7 | 8 | 9 | 10 | 11 | {% if results.Info.file.family.unpackedObjectSHA256 %} 12 | 13 | 14 | 15 | 16 | {% endif %} 17 | 18 | {% if results.Info.file.family.siblingObjectsSHA256 %} 19 | {% for siblingObject in results.Info.file.family.siblingObjectsSHA256 %} 20 | 21 | 22 | 23 | 24 | {% endfor %} 25 | {% endif %} 26 | 27 |
Unpacked-Object SHA256:{{results.Info.file.family.unpackedObjectSHA256}}
Sibling-Object SHA256:{{siblingObject}}
-------------------------------------------------------------------------------- /src/data/html/htmlSections/own_location.html: -------------------------------------------------------------------------------- 1 | {% if results.OwnLocation %} 2 |
^
3 | 4 |
5 |
Own Location
6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 |
IP-Address:{{results.OwnLocation.ipaddress}}
Country:{{results.OwnLocation.country}}
22 | {% endif %} -------------------------------------------------------------------------------- /src/data/html/htmlSections/source_information.html: -------------------------------------------------------------------------------- 1 |
^
2 | 3 |
4 |
Source Information
5 | {% if results.Info %} 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 |
URL:{{results.Info.url.url}}
Hostname:{{results.Info.url.hostname}}
Protokoll:{{results.Info.url.protocol}}
Port:{{results.Info.url.port}}
MD5-URL:{{results.Info.url.md5}}
34 | {% endif %} -------------------------------------------------------------------------------- /src/data/html/htmlSections/subfile.html: -------------------------------------------------------------------------------- 1 | {% if results.SubFile %} 2 |
3 | Subfile 4 |
5 | 6 | 7 | 8 | 9 | 10 | 11 | {% if results.SubFile.Errors %} 12 | {% for errors in results.SubFile.Errors %} 13 | 14 | 15 | 16 | 17 | {% endfor%} 18 | {% endif %} 19 | {% if results.SubFile.Files %} 20 | {% for file in results.SubFile.Files %} 21 | 22 | 23 | 24 | 25 | {% endfor%} 26 | {% endif %} 27 | 28 |
Error:{{errors}}
{{ file.FileType }}:{{ file.Desc }}
29 |
30 |
31 | {% endif %} -------------------------------------------------------------------------------- /src/data/html/htmlSections/yara.html: -------------------------------------------------------------------------------- 1 | {% if results.Yara|length > 0 %} 2 |
^
3 | 4 |
5 |
Yara
6 | {% for hit in results.Yara %} 7 |
8 | 9 | {{hit.rule}} + 10 | 11 | 12 | 20 |
21 | {% endfor %} 22 | {% endif %} -------------------------------------------------------------------------------- /src/data/html/report.html: -------------------------------------------------------------------------------- 1 | {% extends "base-report.html" %} 2 | {% block content %} 3 | {% include "htmlSections/file_information.html" %} 4 | {% if results.Info.file.family|length > 0 %} 5 | {% include "htmlSections/family.html" %} 6 | {% endif %} 7 | {% include "htmlSections/source_information.html" %} 8 | {% include "htmlSections/own_location.html" %} 9 | {% include "htmlSections/antivirus.html" %} 10 | {% include "htmlSections/yara.html" %} 11 |
^
12 |
13 | 14 |
15 |
Static Analysis
16 | {% include "htmlSections/office.html" %} 17 | {% include "htmlSections/rtf.html" %} 18 | {% include "htmlSections/pdf.html" %} 19 | {% include "htmlSections/subfile.html" %} 20 | {% include "htmlSections/pe.html" %} 21 | {% include "htmlSections/network_analysis.html" %} 22 | {% include "htmlSections/other.html" %} 23 | {% endblock %} -------------------------------------------------------------------------------- /src/data/index_result.yar: -------------------------------------------------------------------------------- 1 | include "yara_signatures/result_blackListLanguages.yar" 2 | include "yara_signatures/result_apihttp.yar" -------------------------------------------------------------------------------- /src/data/index_sample.yar: -------------------------------------------------------------------------------- 1 | include "yara_signatures/sample_capabilities.yar" 2 | include "yara_signatures/sample_embedded.yar" 3 | include "yara_signatures/sample_kins.yar" 4 | include "yara_signatures/sample_njRAT.yar" 5 | include "yara_signatures/sample_vmdetect.yar" 6 | include "yara_signatures/sample_blackliststrings.yar" 7 | include "yara_signatures/sample_blackListFunctions.yar" 8 | include "yara_signatures/sample_maldoc.yara" -------------------------------------------------------------------------------- /src/data/peiddb.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/robbyFux/Ragpicker/77d68ff3a13851ed4c256635f0673a4c2a84ef51/src/data/peiddb.txt -------------------------------------------------------------------------------- /src/data/yara_signatures/result_apihttp.yar: -------------------------------------------------------------------------------- 1 | rule HTTPApiFunctions { 2 | meta: 3 | description = "Uses the HTTP protocol functions" 4 | strings: 5 | $api1 = "HttpSendRequest" nocase 6 | $api2 = "HttpSendRequestEx" nocase 7 | $api3 = "HttpQueryInfo" nocase 8 | $api4 = "HttpAddRequestHeaders" nocase 9 | $api5 = "HttpEndRequest" nocase 10 | $api6 = "HttpOpenRequest" nocase 11 | condition: 12 | any of them 13 | } -------------------------------------------------------------------------------- /src/data/yara_signatures/result_blackListLanguages.yar: -------------------------------------------------------------------------------- 1 | rule BlackListLanguages { 2 | meta: 3 | description = "Unconventionial binary language" 4 | strings: 5 | $lang1 = "LANG_ARABIC" 6 | $lang2 = "LANG_BULGARIAN" 7 | $lang3 = "LANG_CHINESE" 8 | $lang4 = "LANG_ROMANIAN" 9 | $lang5 = "LANG_RUSSIAN" 10 | $lang6 = "LANG_CROATO-SERBIAN" 11 | $lang7 = "LANG_SLOVAK" 12 | $lang8 = "LANG_ALBANIAN" 13 | $lang9 = "LANG_TURKISH" 14 | $lang10 = "LANG_HEBREW" 15 | $lang11 = "LANG_KOREAN" 16 | $lang12 = "SUBLANG_ENGLISH_JAMAICA" 17 | $lang13 = "SUBLANG_ENGLISH_CARIBBEAN" 18 | $lang14 = "SUBLANG_ENGLISH_BELIZE" 19 | $lang15 = "SUBLANG_ENGLISH_TRINIDAD" 20 | $lang16 = "SUBLANG_ENGLISH_ZIMBABWE" 21 | $lang17 = "SUBLANG_ENGLISH_PHILIPPINES" 22 | $lang18 = "LANG_UZBEK" 23 | $lang19 = "LANG_VIETNAMESE" 24 | $lang20 = "LANG_UKRAINIAN" 25 | $lang21 = "LANG_TELUGU" 26 | $lang22 = "LANG_SYRIAC" 27 | $lang23 = "LANG_SERBIAN" 28 | $lang24 = "LANG_LATVIAN" 29 | condition: 30 | any of them 31 | } -------------------------------------------------------------------------------- /src/data/yara_signatures/sample_embedded.yar: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2010-2014 Cuckoo Foundation. 2 | // This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org 3 | // See the file 'docs/LICENSE' for copying permission. 4 | 5 | rule embedded_macho 6 | { 7 | meta: 8 | author = "nex" 9 | description = "Contains an embedded Mach-O file" 10 | 11 | strings: 12 | $magic1 = { ca fe ba be } 13 | $magic2 = { ce fa ed fe } 14 | $magic3 = { fe ed fa ce } 15 | condition: 16 | any of ($magic*) and not ($magic1 at 0) and not ($magic2 at 0) and not ($magic3 at 0) 17 | } 18 | 19 | rule embedded_pe 20 | { 21 | meta: 22 | author = "nex" 23 | description = "Contains an embedded PE32 file" 24 | 25 | strings: 26 | $a = "PE32" 27 | $b = "This program" 28 | $mz = { 4d 5a } 29 | condition: 30 | ($a or $b) and not ($mz at 0) 31 | } 32 | 33 | rule embedded_win_api 34 | { 35 | meta: 36 | author = "nex" 37 | description = "A non-Windows executable contains win32 API functions names" 38 | 39 | strings: 40 | $mz = { 4d 5a } 41 | $api1 = "CreateFileA" 42 | $api2 = "GetProcAddress" 43 | $api3 = "LoadLibraryA" 44 | $api4 = "WinExec" 45 | $api5 = "GetSystemDirectoryA" 46 | $api6 = "WriteFile" 47 | $api7 = "ShellExecute" 48 | $api8 = "GetWindowsDirectory" 49 | $api9 = "URLDownloadToFile" 50 | $api10 = "IsBadReadPtr" 51 | $api11 = "IsBadWritePtr" 52 | $api12 = "SetFilePointer" 53 | $api13 = "GetTempPath" 54 | $api14 = "GetWindowsDirectory" 55 | condition: 56 | not ($mz at 0) and any of ($api*) 57 | } 58 | -------------------------------------------------------------------------------- /src/data/yara_signatures/sample_kins.yar: -------------------------------------------------------------------------------- 1 | 2 | rule KINS_dropper { 3 | meta: 4 | author = "AlienVault Labs aortega@alienvault.com" 5 | description = "Match protocol, process injects and windows exploit present in KINS dropper" 6 | strings: 7 | // Network protocol 8 | $n1 = "tid=%d&ta=%s-%x" fullword 9 | $n2 = "fid=%d" fullword 10 | $n3 = "%[^.].%[^(](%[^)])" fullword 11 | // Injects 12 | $i0 = "%s [%s %d] 77 %s" 13 | $i01 = "Global\\%s%x" 14 | $i1 = "Inject::InjectProcessByName()" 15 | $i2 = "Inject::CopyImageToProcess()" 16 | $i3 = "Inject::InjectProcess()" 17 | $i4 = "Inject::InjectImageToProcess()" 18 | $i5 = "Drop::InjectStartThread()" 19 | // UAC bypass 20 | $uac1 = "ExploitMS10_092" 21 | $uac2 = "\\globalroot\\systemroot\\system32\\tasks\\" ascii wide 22 | $uac3 = "HighestAvailable" ascii wide 23 | condition: 24 | 2 of ($n*) and 2 of ($i*) and 2 of ($uac*) 25 | } 26 | 27 | rule KINS_DLL_zeus { 28 | meta: 29 | author = "AlienVault Labs aortega@alienvault.com" 30 | description = "Match default bot in KINS leaked dropper, Zeus" 31 | strings: 32 | // Network protocol 33 | $n1 = "%BOTID%" fullword 34 | $n2 = "%opensocks%" fullword 35 | $n3 = "%openvnc%" fullword 36 | $n4 = /Global\\(s|v)_ev/ fullword 37 | // Crypted strings 38 | $s1 = "\x72\x6E\x6D\x2C\x36\x7D\x76\x77" 39 | $s2 = "\x18\x04\x0F\x12\x16\x0A\x1E\x08\x5B\x11\x0F\x13" 40 | $s3 = "\x39\x1F\x01\x07\x15\x19\x1A\x33\x19\x0D\x1F" 41 | $s4 = "\x62\x6F\x71\x78\x63\x61\x7F\x69\x2D\x67\x79\x65" 42 | $s5 = "\x6F\x69\x7F\x6B\x61\x53\x6A\x7C\x73\x6F\x71" 43 | condition: 44 | all of ($n*) and 1 of ($s*) 45 | } 46 | 47 | -------------------------------------------------------------------------------- /src/data/yara_signatures/sample_njRAT.yar: -------------------------------------------------------------------------------- 1 | rule win_exe_njRAT 2 | { 3 | meta: 4 | author = "info@fidelissecurity.com" 5 | descripion = "njRAT - Remote Access Trojan" 6 | comment = "Variants have also been observed obfuscated with .NET Reactor" 7 | filetype = "pe" 8 | date = "2013-07-15" 9 | version = "1.0" 10 | hash1 = "92ee1fb5df21d8cfafa2b02b6a25bd3b" 11 | hash2 = "3576d40ce18bb0349f9dfa42b8911c3a" 12 | hash3 = "24cc5b811a7f9591e7f2cb9a818be104" 13 | hash4 = "3ad5fded9d7fdf1c2f6102f4874b2d52" 14 | hash5 = "a98b4c99f64315aac9dd992593830f35" 15 | hash6 ="5fcb5282da1a2a0f053051c8da1686ef" 16 | hash7 = "a669c0da6309a930af16381b18ba2f9d" 17 | hash8 = "79dce17498e1997264346b162b09bde8" 18 | hash9 = "fc96a7e27b1d3dab715b2732d5c86f80" 19 | ref1 = "http://bit.ly/19tlf4s" 20 | ref2 = "http://www.fidelissecurity.com/threatadvisory" 21 | ref3 = "http://www.threatgeek.com/2013/06/fidelis-threat-advisory-1009-njratuncovered.html" 22 | ref4 = "http://threatgeek.typepad.com/files/fta-1009---njrat-uncovered.pdf" 23 | 24 | strings: 25 | $magic = "MZ" 26 | $string_setA_1 = "FromBase64String" 27 | $string_setA_2 = "Base64String" 28 | $string_setA_3 = "Connected" wide ascii 29 | $string_setA_4 = "Receive" 30 | $string_setA_5 = "DeleteSubKey" wide ascii 31 | $string_setA_6 = "get_MachineName" 32 | $string_setA_7 = "get_UserName" 33 | $string_setA_8 = "get_LastWriteTime" 34 | $string_setA_9 = "GetVolumeInformation" 35 | 36 | $string_setB_1 = "OSFullName" wide ascii 37 | $string_setB_2 = "Send" wide ascii 38 | $string_setB_3 = "Connected" wide ascii 39 | $string_setB_4 = "DownloadData" wide ascii 40 | $string_setB_5 = "netsh firewall" wide 41 | $string_setB_6 = "cmd.exe /k ping 0 & del" wide 42 | 43 | condition: 44 | ($magic at 0) and ( all of ($string_setA*) or all of ($string_setB*) ) 45 | } 46 | 47 | rule network_traffic_njRAT 48 | { 49 | meta: 50 | author = "info@fidelissecurity.com" 51 | descripion = "njRAT - Remote Access Trojan" 52 | comment = "Rule to alert on network traffic indicators" 53 | filetype = "PCAP - Network Traffic" 54 | date = "2013-07-15" 55 | version = "1.0" 56 | hash1 = "92ee1fb5df21d8cfafa2b02b6a25bd3b" 57 | hash2 ="3576d40ce18bb0349f9dfa42b8911c3a" 58 | hash3 ="24cc5b811a7f9591e7f2cb9a818be104" 59 | hash4 = "3ad5fded9d7fdf1c2f6102f4874b2d52" 60 | hash5 = "a98b4c99f64315aac9dd992593830f35" 61 | hash6 = "5fcb5282da1a2a0f053051c8da1686ef" 62 | hash7 = "a669c0da6309a930af16381b18ba2f9d" 63 | hash8 = "79dce17498e1997264346b162b09bde8" 64 | hash9 = "fc96a7e27b1d3dab715b2732d5c86f80" 65 | ref1 = "http://bit.ly/19tlf4s" 66 | ref2 = "http://www.fidelissecurity.com/threatadvisory" 67 | ref3 = "http://www.threatgeek.com/2013/06/fidelis-threat-advisory-1009-njrat-uncovered.html" 68 | ref4 = "http://threatgeek.typepad.com/files/fta-1009---njrat-uncovered.pdf" 69 | 70 | strings: 71 | $string1 = "FM|'|'|" // File Manager 72 | $string2 = "nd|'|'|" // File Manager 73 | $string3 = "rn|'|'|" // Run File 74 | $string4 = "sc~|'|'|" // Remote Desktop 75 | $string5 = "scPK|'|'|" // Remote Desktop 76 | $string6 = "CAM|'|'|" // Remote Cam 77 | $string7 = "USB Video Device[endof]" // Remote Cam 78 | $string8 = "rs|'|'|" // Reverse Shell 79 | $string9 = "proc|'|'|" // Process Manager 80 | $string10 = "k|'|'|" // Process Manager 81 | $string11 = "RG|'|'|~|'|'|" // Registry Manipulation 82 | $string12 = "kl|'|'|" // Keylogger file 83 | $string13 = "ret|'|'|" // Get Browser Passwords 84 | $string14 = "pl|'|'|" // Get Browser Passwords 85 | $string15 = "lv|'|'|" // General 86 | $string16 = "prof|'|'|~|'|'|" // Server rename 87 | $string17 = "un|'|'|~[endof]" // Uninstall 88 | $idle_string = "P[endof]" // Idle Connection 89 | 90 | condition: 91 | any of ($string*) or #idle_string > 4 92 | 93 | } -------------------------------------------------------------------------------- /src/data/yara_signatures/sample_vmdetect.yar: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2010-2014 Cuckoo Foundation. 2 | // This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org 3 | // See the file 'docs/LICENSE' for copying permission. 4 | 5 | rule vmdetect 6 | { 7 | meta: 8 | author = "nex" 9 | description = "Possibly employs anti-virtualization techniques" 10 | 11 | strings: 12 | // Binary tricks 13 | $vmware = {56 4D 58 68} 14 | $virtualpc = {0F 3F 07 0B} 15 | $ssexy = {66 0F 70 ?? ?? 66 0F DB ?? ?? ?? ?? ?? 66 0F DB ?? ?? ?? ?? ?? 66 0F EF} 16 | $vmcheckdll = {45 C7 00 01} 17 | $redpill = {0F 01 0D 00 00 00 00 C3} 18 | 19 | // Random strings 20 | $vmware1 = "VMXh" 21 | $vmware2 = "Ven_VMware_" nocase 22 | $vmware3 = "Prod_VMware_Virtual_" nocase 23 | $vmware4 = "hgfs.sys" nocase 24 | $vmware5 = "mhgfs.sys" nocase 25 | $vmware6 = "prleth.sys" nocase 26 | $vmware7 = "prlfs.sys" nocase 27 | $vmware8 = "prlmouse.sys" nocase 28 | $vmware9 = "prlvideo.sys" nocase 29 | $vmware10 = "prl_pv32.sys" nocase 30 | $vmware11 = "vpc-s3.sys" nocase 31 | $vmware12 = "vmsrvc.sys" nocase 32 | $vmware13 = "vmx86.sys" nocase 33 | $vmware14 = "vmnet.sys" nocase 34 | $vmware15 = "vmicheartbeat" nocase 35 | $vmware16 = "vmicvss" nocase 36 | $vmware17 = "vmicshutdown" nocase 37 | $vmware18 = "vmicexchange" nocase 38 | $vmware19 = "vmdebug" nocase 39 | $vmware20 = "vmmouse" nocase 40 | $vmware21 = "vmtools" nocase 41 | $vmware22 = "VMMEMCTL" nocase 42 | $vmware23 = "vmx86" nocase 43 | $vmware24 = "vmware" nocase 44 | $virtualpc1 = "vpcbus" nocase 45 | $virtualpc2 = "vpc-s3" nocase 46 | $virtualpc3 = "vpcuhub" nocase 47 | $virtualpc4 = "msvmmouf" nocase 48 | $xen1 = "xenevtchn" nocase 49 | $xen2 = "xennet" nocase 50 | $xen3 = "xennet6" nocase 51 | $xen4 = "xensvc" nocase 52 | $xen5 = "xenvdb" nocase 53 | $xen6 = "XenVMM" nocase 54 | $virtualbox1 = "VBoxHook.dll" nocase 55 | $virtualbox2 = "VBoxService" nocase 56 | $virtualbox3 = "VBoxTray" nocase 57 | $virtualbox4 = "VBoxMouse" nocase 58 | $virtualbox5 = "VBoxGuest" nocase 59 | $virtualbox6 = "VBoxSF" nocase 60 | $virtualbox7 = "VBoxGuestAdditions" nocase 61 | $virtualbox8 = "VBOX HARDDISK" nocase 62 | 63 | // MAC addresses 64 | $vmware_mac_1a = "00-05-69" 65 | $vmware_mac_1b = "00:05:69" 66 | $vmware_mac_1c = "000569" 67 | $vmware_mac_2a = "00-50-56" 68 | $vmware_mac_2b = "00:50:56" 69 | $vmware_mac_2c = "005056" 70 | $vmware_mac_3a = "00-0C-29" nocase 71 | $vmware_mac_3b = "00:0C:29" nocase 72 | $vmware_mac_3c = "000C29" nocase 73 | $vmware_mac_4a = "00-1C-14" nocase 74 | $vmware_mac_4b = "00:1C:14" nocase 75 | $vmware_mac_4c = "001C14" nocase 76 | $virtualbox_mac_1a = "08-00-27" 77 | $virtualbox_mac_1b = "08:00:27" 78 | $virtualbox_mac_1c = "080027" 79 | 80 | $vm0 = "VIRTUAL HD" nocase 81 | $vm1 = "VMWARE VIRTUAL IDE HARD DRIVE" nocase 82 | $vm2 = "QEMU HARDDISK" nocase 83 | $vm3 = "VBOX HARDDRIVE" nocase 84 | $vm4 = "The Wireshark Network Analyzer" 85 | $vm7 = "Process Monitor - Sysinternals: www.sysinternals.com" 86 | $vm8 = "File Monitor - Sysinternals: www.sysinternals.com" 87 | $vm9 = "Registry Monitor - Sysinternals: www.sysinternals.com" 88 | 89 | condition: 90 | any of them 91 | } 92 | -------------------------------------------------------------------------------- /src/dumpdir/files/empty.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/robbyFux/Ragpicker/77d68ff3a13851ed4c256635f0673a4c2a84ef51/src/dumpdir/files/empty.txt -------------------------------------------------------------------------------- /src/dumpdir/reports/empty.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/robbyFux/Ragpicker/77d68ff3a13851ed4c256635f0673a4c2a84ef51/src/dumpdir/reports/empty.txt -------------------------------------------------------------------------------- /src/preProcessing/01_unpack_archive.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | from os import walk 5 | from os.path import join 6 | import logging 7 | import shutil 8 | import tempfile 9 | 10 | from core.abstracts import Processing 11 | import core.commonutils as commonutils 12 | 13 | 14 | try: 15 | from yapsy.IPlugin import IPlugin 16 | except ImportError: 17 | raise ImportError, 'Yapsy (Yet Another Plugin System) is required to run this program : http://yapsy.sourceforge.net' 18 | 19 | log = logging.getLogger("UnpackArchive") 20 | 21 | class UnpackArchive(IPlugin, Processing): 22 | 23 | def run(self, objfile): 24 | tmpdir = tempfile.mkdtemp() 25 | tmp_files = [] 26 | 27 | try: 28 | commonutils.unpackArchive(objfile.file.temp_file, tmpdir) 29 | 30 | for root, dirs, files in walk(tmpdir): 31 | for file in files: 32 | full_path = join(root, file) 33 | if not commonutils.isArchive(full_path): 34 | tmp_files.append(full_path) 35 | 36 | if len(tmp_files) == 1 and commonutils.isPermittedType(tmp_files[0]): 37 | # len == 1 > easy, replace archive file in objfile 38 | objfile.set_file_from_path(tmp_files[0]) 39 | else: 40 | for file in tmp_files: 41 | if commonutils.isPermittedType(file): 42 | objfile.set_unpacked_file(file) 43 | finally: 44 | # Temp-Folder loeschen 45 | shutil.rmtree(tmpdir) 46 | 47 | return objfile 48 | -------------------------------------------------------------------------------- /src/preProcessing/01_unpack_archive.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = 01_unpack_archive 3 | Module = 01_unpack_archive 4 | 5 | [Documentation] 6 | Description = Recurse unpack archive 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/preProcessing/02_unpacker_clamav.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | import logging 5 | import os 6 | import re 7 | import shutil 8 | import subprocess 9 | 10 | from core.abstracts import Processing 11 | from core.constants import RAGPICKER_ROOT 12 | 13 | 14 | try: 15 | from yapsy.IPlugin import IPlugin 16 | except ImportError: 17 | raise ImportError, 'Yapsy (Yet Another Plugin System) is required to run this program : http://yapsy.sourceforge.net' 18 | 19 | log = logging.getLogger("ClamAvUnpacker") 20 | 21 | class ClamAvUnpacker(IPlugin, Processing): 22 | """ Supported packer: Aspack, UPX, FSG, Petite, PeSpin, NsPack, wwpack32, Mew, Upack, Yoda Cryptor 23 | """ 24 | def run(self, objfile): 25 | self.avPath = self.options.get("clamscan_path", None) 26 | tmpPath = self.getTmpPath('clamscan_') 27 | 28 | try: 29 | self._unpack(objfile, tmpPath) 30 | except (Exception) as e: 31 | log.error("The module \"ClamAvUnpacker\" returned the following error: %s" % e) 32 | finally: 33 | # Delete tmpPath 34 | shutil.rmtree(tmpPath, ignore_errors=True) 35 | 36 | return objfile 37 | 38 | def _unpack(self, objfile, temp_path): 39 | if not self.avPath: 40 | raise Exception("ClamAv path is not set in the configuration!") 41 | 42 | if os.path.isfile(self.avPath): 43 | try: 44 | dummyDB = os.path.join(RAGPICKER_ROOT, 'data', 'clamav_dummy.hdb') 45 | output = subprocess.Popen([self.avPath, "--debug", "--leave-temps", "--database=" + dummyDB, \ 46 | "--tempdir=" + temp_path, objfile.file.temp_file], \ 47 | stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate() 48 | 49 | # FSG: Unpacked and rebuilt executable saved in 50 | # UPX/FSG: Decompressed data saved in 51 | # MEW: Unpacked and rebuilt executable saved in 52 | if " saved in " in output[0]: 53 | log.info("File unpacked !!!!") 54 | unpackedFile = self._parseUnpacked(output[0]) 55 | objfile.set_unpacked_file(unpackedFile) 56 | except (Exception) as e: 57 | raise Exception("ClamAV _unpack file %s Failed" % e) 58 | 59 | def _parseUnpacked(self, data): 60 | rpd = re.compile('debug:.*saved\sin\s(.+)', re.IGNORECASE) 61 | rpdFind = re.findall(rpd, data) 62 | 63 | for file in rpdFind: 64 | log.debug(file) 65 | return file 66 | -------------------------------------------------------------------------------- /src/preProcessing/02_unpacker_clamav.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = 02_unpacker_clamav 3 | Module = 02_unpacker_clamav 4 | 5 | [Documentation] 6 | Description = Unpacking with ClamAv AntiVirus (http://www.clamav.net/lang/en/) 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/preProcessing/03_extract_rsrc.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | import logging 5 | 6 | from core.commonutils import getFileTypeFromBuffer 7 | from core.abstracts import Processing 8 | from utils.pefile import PE 9 | from utils.pefile import PEFormatError 10 | 11 | try: 12 | from yapsy.IPlugin import IPlugin 13 | except ImportError: 14 | raise ImportError, 'Yapsy (Yet Another Plugin System) is required to run this program : http://yapsy.sourceforge.net' 15 | 16 | log = logging.getLogger("ExtractRSRC") 17 | 18 | class ExtractRSRC(IPlugin, Processing): 19 | 20 | def run(self, objfile): 21 | self.extractTypes = self.options.get("extracttypes") 22 | 23 | try: 24 | pe = PE(data=objfile.file.file_data) 25 | resources = self.getResources(pe) 26 | 27 | if objfile.unpacked_file: 28 | pe = PE(data=objfile.unpacked_file.file_data) 29 | resources = resources + self.getResources(pe) 30 | 31 | for resource in resources: 32 | objfile.add_included_file(resource) 33 | except PEFormatError, e: 34 | log.warn("Error - No Portable Executable: %s" % e) 35 | except (Exception) as e: 36 | log.error("The module \"ExtractRSRC\" returned the following error: %s" % e) 37 | finally: 38 | pass 39 | 40 | return objfile 41 | 42 | def getResources(self, pe): 43 | ret = [] 44 | if hasattr(pe, 'DIRECTORY_ENTRY_RESOURCE'): 45 | for resource_type in pe.DIRECTORY_ENTRY_RESOURCE.entries: 46 | if hasattr(resource_type, 'directory'): 47 | for resource_id in resource_type.directory.entries: 48 | if hasattr(resource_id, 'directory'): 49 | for resource_lang in resource_id.directory.entries: 50 | data = pe.get_data(resource_lang.data.struct.OffsetToData, resource_lang.data.struct.Size) 51 | filetype = getFileTypeFromBuffer(data) 52 | 53 | log.debug("Found: " + filetype) 54 | 55 | if filetype in self.extractTypes: 56 | #TODO: RAR und ZIP entpacken!!! 57 | ret.append(data) 58 | log.info("Append File: " + filetype) 59 | return ret 60 | -------------------------------------------------------------------------------- /src/preProcessing/03_extract_rsrc.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = 03_extract_rsrc 3 | Module = 03_extract_rsrc 4 | 5 | [Documentation] 6 | Description = Extract .rsrc-Section 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/preProcessing/04_extract_office.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | import logging 5 | import os 6 | import re 7 | import shutil 8 | import subprocess 9 | 10 | from core.abstracts import Processing 11 | from core.constants import RAGPICKER_ROOT 12 | 13 | 14 | try: 15 | from yapsy.IPlugin import IPlugin 16 | except ImportError: 17 | raise ImportError, 'Yapsy (Yet Another Plugin System) is required to run this program : http://yapsy.sourceforge.net' 18 | 19 | log = logging.getLogger("ExtractOffice") 20 | 21 | class ExtractOffice(IPlugin, Processing): 22 | 23 | def run(self, objfile): 24 | wine = self.options.get("wine", "/usr/bin/wine") 25 | brute = self.options.get("brute", True) 26 | tmpPath = self.getTmpPath("extractOffice_") 27 | 28 | try: 29 | officeFile = os.path.join(tmpPath, "temp.file") 30 | # Copy TempFile > new TempDir 31 | shutil.copyfile(objfile.file.temp_file, officeFile) 32 | scanner = None 33 | option = None 34 | 35 | if objfile.file.get_type() == "Composite": 36 | scanner = os.path.join(RAGPICKER_ROOT, 'utils', 'OfficeMalScanner', 'OfficeMalScanner.exe') 37 | option = "' scan" 38 | if brute: 39 | option = option + " brute" 40 | elif objfile.file.get_type() == "Rich": 41 | scanner = os.path.join(RAGPICKER_ROOT, 'utils', 'OfficeMalScanner', 'RTFScan.exe') 42 | option = "' scan" 43 | 44 | cmd = wine + " " + scanner + " '" + officeFile + option 45 | log.debug(cmd) 46 | self.officeScanExe(cmd, tmpPath, objfile) 47 | 48 | except (Exception) as e: 49 | log.error("The module \"ExtractOffice\" returned the following error: %s" % e) 50 | finally: 51 | # Delete tmpPath 52 | shutil.rmtree(tmpPath, ignore_errors=True) 53 | 54 | return objfile 55 | 56 | def officeScanExe(self, cmd, tmpPath, objfile): 57 | process = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=tmpPath, shell=True) 58 | (stdout, stderr) = process.communicate() 59 | 60 | stdout = stdout.decode('utf-8', 'ignore') 61 | 62 | if stdout: 63 | if "unencrypted MZ/PE signature found" in stdout: 64 | log.info("MZ/PE File unpacked !!!!") 65 | unpackedFile = self._parseUnpacked(stdout) 66 | log.info("OLEScan = " + unpackedFile) 67 | objfile.set_unpacked_file(os.path.join(tmpPath, str(unpackedFile).strip())) 68 | else: 69 | raise Exception(stderr) 70 | 71 | def _parseUnpacked(self, data): 72 | rpd = re.compile('Dumping Memory to disk as filename:\s(.+)', re.IGNORECASE) 73 | rpdFind = re.findall(rpd, data) 74 | 75 | for file in rpdFind: 76 | log.debug(file) 77 | return file 78 | -------------------------------------------------------------------------------- /src/preProcessing/04_extract_office.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = 04_extract_office 3 | Module = 04_extract_office 4 | 5 | [Documentation] 6 | Description = Extract PE-Files from Office-Documents 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/preProcessing/05_pe_carve.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | import re 5 | import logging 6 | import utils.pefile as pefile 7 | from core.commonutils import getFileTypeFromBuffer 8 | from core.abstracts import Processing 9 | 10 | 11 | 12 | try: 13 | from yapsy.IPlugin import IPlugin 14 | except ImportError: 15 | raise ImportError, 'Yapsy (Yet Another Plugin System) is required to run this program : http://yapsy.sourceforge.net' 16 | 17 | log = logging.getLogger("PeCarve") 18 | 19 | class PeCarve(IPlugin, Processing): 20 | 21 | def run(self, objfile): 22 | 23 | try: 24 | objfile.file.temp_file 25 | carvedFiles = self.carve(objfile.file.temp_file) 26 | 27 | if objfile.unpacked_file: 28 | carvedFiles = carvedFiles + self.carve(objfile.unpacked_file.temp_file) 29 | 30 | for carvedFile in carvedFiles: 31 | objfile.add_included_file(carvedFile) 32 | # except (Exception) as e: 33 | # log.error("The module \"PeCarve\" returned the following error: %s" % e) 34 | finally: 35 | pass 36 | 37 | return objfile 38 | 39 | def carve(self, filePath): 40 | cavedFiles = [] 41 | # read the file into a buffer 42 | try: 43 | file = open(filePath, 'rb') 44 | buffer = file.read() 45 | except (Exception) as e: 46 | raise Exception("PeCarve could not access file: %s" % filePath) 47 | 48 | # carve out embeddded executables 49 | # For each address that contains MZ 50 | for offset in [tmp.start() for tmp in re.finditer('\x4d\x5a', buffer)]: 51 | file.seek(offset) 52 | 53 | # MZ on offset 0x0 54 | if offset == 0: 55 | continue 56 | 57 | try: 58 | pe = pefile.PE(data=file.read()) 59 | except: 60 | # Failed to parse EXE 61 | continue 62 | 63 | log.info('PE found at offset: ' + hex(offset)) 64 | cavedFiles.append(pe.trim()) 65 | 66 | file.seek(0) 67 | pe.close() 68 | 69 | return cavedFiles -------------------------------------------------------------------------------- /src/preProcessing/05_pe_carve.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = 05_pe_carve 3 | Module = 05_pe_carve 4 | 5 | [Documentation] 6 | Description = Carve out embeddded executables 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/preProcessing/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/robbyFux/Ragpicker/77d68ff3a13851ed4c256635f0673a4c2a84ef51/src/preProcessing/__init__.py -------------------------------------------------------------------------------- /src/processing/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /src/processing/all_bluecoatMalwareAnalysisAppliance.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = all_bluecoatMalwareAnalysisAppliance 3 | Module = all_bluecoatMalwareAnalysisAppliance 4 | 5 | [Documentation] 6 | Description = Adds the Sample to Blue Coat Malware Analysis Appliance (https://www.bluecoat.com) 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 9 | Website = https://www.bluecoat.com/products/malware-analysis-appliance -------------------------------------------------------------------------------- /src/processing/all_cuckooSandbox.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | import json 5 | import logging 6 | import urllib2 7 | 8 | from core.abstracts import Processing 9 | from utils.multiPartForm import MultiPartForm 10 | 11 | 12 | try: 13 | from yapsy.IPlugin import IPlugin 14 | except ImportError: 15 | raise ImportError, 'Yapsy (Yet Another Plugin System) is required to run this program : http://yapsy.sourceforge.net' 16 | 17 | log = logging.getLogger("ProcessingCuckooSandbox") 18 | 19 | CUCKOO_TASK_CREATE_URL = "http://%s:%s/tasks/create/file " 20 | 21 | class CuckooSandbox(IPlugin, Processing): 22 | 23 | def run(self, objfile): 24 | self.key = "CuckooSandbox" 25 | self.score = -1 26 | self.host = self.options.get("host") 27 | self.port = self.options.get("port") 28 | reponsejson = {} 29 | 30 | if not self.host or not self.port: 31 | raise Exception("Cuckoo REST API server not configurated") 32 | 33 | file_extension = '.' + objfile.file.file_extension() 34 | fileName = objfile.file.get_fileMd5() + file_extension 35 | rawFile = open(objfile.file.temp_file, 'rb') 36 | 37 | log.debug(CUCKOO_TASK_CREATE_URL % (self.host, self.port) + " file=" + fileName) 38 | 39 | try: 40 | form = MultiPartForm() 41 | form.add_file('file', fileName, fileHandle=rawFile) 42 | 43 | request = urllib2.Request(CUCKOO_TASK_CREATE_URL % (self.host, self.port)) 44 | request.add_header('User-agent', 'Ragpicker') 45 | body = str(form) 46 | request.add_header('Content-type', form.get_content_type()) 47 | request.add_header('Content-length', len(body)) 48 | request.add_data(body) 49 | 50 | data = urllib2.urlopen(request, timeout=60).read() 51 | reponsejson = json.loads(data) 52 | log.info("Submitted to cuckoo, task ID %s", reponsejson["task_id"]) 53 | except urllib2.URLError as e: 54 | raise Exception("Unable to establish connection to Cuckoo REST API server: %s" % e) 55 | except urllib2.HTTPError as e: 56 | raise Exception("Unable to perform HTTP request to Cuckoo REST API server (http code=%s)" % e) 57 | except ValueError as e: 58 | raise Exception("Unable to convert response to JSON: %s" % e) 59 | 60 | return reponsejson 61 | -------------------------------------------------------------------------------- /src/processing/all_cuckooSandbox.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = all_cuckooSandbox 3 | Module = all_cuckooSandbox 4 | 5 | [Documentation] 6 | Description = Adds the Sample to the list of cuckoo-sandbox tasks to be processed and analyzed 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 9 | Website = -------------------------------------------------------------------------------- /src/processing/all_info.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = all_info 3 | Module = all_info 4 | 5 | [Documentation] 6 | Description = Sample Base Infos (Don't disable "info"-Module!!!) 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 9 | Website = -------------------------------------------------------------------------------- /src/processing/all_subFile.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | import StringIO 5 | import logging 6 | import re 7 | import sys 8 | 9 | from core.abstracts import Processing 10 | 11 | 12 | try: 13 | from hachoir_subfile.search import SearchSubfile 14 | from hachoir_core.stream import FileInputStream 15 | from hachoir_core.cmd_line import unicodeFilename 16 | except ImportError: 17 | raise ImportError, 'hachoir-subfile is required to run this program : http://bitbucket.org/haypo/hachoir/wiki/hachoir-subfile' 18 | 19 | try: 20 | from yapsy.IPlugin import IPlugin 21 | except ImportError: 22 | raise ImportError, 'Yapsy (Yet Another Plugin System) is required to run this program : http://yapsy.sourceforge.net' 23 | 24 | log = logging.getLogger("ProcessingSubFile") 25 | 26 | class subFile(IPlugin, Processing): 27 | 28 | def run(self, objfile): 29 | self.key = "SubFile" 30 | self.score = -1 31 | # Dictionary containing all the results of this processing. 32 | results = {} 33 | 34 | try: 35 | results = self.subfile(objfile.file.temp_file) 36 | except (Exception) as e: 37 | log.error("The module \"SubFile\" returned the following error: %s" % e) 38 | 39 | return results 40 | 41 | def subfile(self, filePath): 42 | # hachoir-subfile is a tool based on hachoir-parser to find subfiles in any binary stream. 43 | # Website: http://bitbucket.org/haypo/hachoir/wiki/hachoir-subfile 44 | # bypass sys.stdout, sys.stderr 45 | oldStdOut = sys.stdout 46 | oldStdErr = sys.stderr 47 | outputStdErr = StringIO.StringIO() 48 | outputStdOut = StringIO.StringIO() 49 | sys.stdout = outputStdOut 50 | sys.stderr = outputStdErr 51 | 52 | stream = FileInputStream(unicodeFilename(filePath), real_filename=filePath) 53 | 54 | # Search for subfiles 55 | subfile = SearchSubfile(stream, 0, None) 56 | subfile.loadParsers(categories=None, parser_ids=None) 57 | subfile.main() 58 | 59 | # sys.stdout, sys.stderr reset 60 | sys.stdout = oldStdOut 61 | sys.stderr = oldStdErr 62 | 63 | # parse stdout, stderr from SearchSubfile 64 | return self.parse(outputStdOut.getvalue(), outputStdErr.getvalue()) 65 | 66 | def parse(self, stdout, stderr): 67 | results = {} 68 | if stderr: 69 | if not len(stderr.split('\n')) <= 1: 70 | ret = [] 71 | line = stderr.split('\n') 72 | for l in line: 73 | if len(l) > 0: ret.append(l) 74 | results["Errors"] = ret 75 | if stdout: 76 | if not len(stdout.split('\n')) <= 2: 77 | ret = [] 78 | line = stdout.split('\n') 79 | for l in line: 80 | if not re.findall('File at 0 size=', l): 81 | subfile = l.replace("[+] ", "").split(': ') 82 | if len(subfile) > 1: 83 | sFile = {} 84 | sFile["FileType"] = subfile[1] 85 | sFile["Desc"] = subfile[0] 86 | ret.append(sFile) 87 | results["Files"] = ret 88 | 89 | return results 90 | -------------------------------------------------------------------------------- /src/processing/all_subFile.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = all_subFile 3 | Module = all_subFile 4 | 5 | [Documentation] 6 | Description = Find subfile in any binary stream 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 9 | Website = -------------------------------------------------------------------------------- /src/processing/all_virustotal.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = all_virustotal 3 | Module = all_virustotal 4 | 5 | [Documentation] 6 | Description = Gets detection ratio from VirusTotal.com (via VT API) 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 9 | Website = -------------------------------------------------------------------------------- /src/processing/all_yara.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | import logging 5 | 6 | from core.abstracts import Processing 7 | import core.commonutils as commonutils 8 | 9 | 10 | try: 11 | from yapsy.IPlugin import IPlugin 12 | except ImportError: 13 | raise ImportError, 'Yapsy (Yet Another Plugin System) is required to run this program : http://yapsy.sourceforge.net' 14 | 15 | log = logging.getLogger("ProcessingYara") 16 | 17 | class Yara(IPlugin, Processing): 18 | 19 | def run(self, objfile): 20 | self.key = "Yara" 21 | self.score = 0 22 | rulepath = self.options.get("rulepath") 23 | results = [] 24 | resultOrginalFile = commonutils.processYara(rulepath, filepath=objfile.file.temp_file) 25 | 26 | # Run YARA for unpacked file 27 | if objfile.unpacked_file: 28 | resultUnpackedFile = commonutils.processYara(rulepath, filepath=objfile.unpacked_file.temp_file, 29 | prefix="UnpackedFile") 30 | 31 | # Update Meta-description -> set marker UnpackedFile 32 | for res in resultUnpackedFile: 33 | desc = res.get("meta").get("description") 34 | desc = "UnpackedFile - %s" % desc 35 | res["meta"]["description"] = desc 36 | results.append(res) 37 | 38 | results = results + resultOrginalFile 39 | 40 | return results 41 | -------------------------------------------------------------------------------- /src/processing/all_yara.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = all_yara 3 | Module = all_yara 4 | 5 | [Documentation] 6 | Description = Identify and classify malware with yara (http://plusvic.github.io/yara/) 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/processing/antivirus_avast.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | import logging 5 | import os 6 | import re 7 | import subprocess 8 | 9 | from core.abstracts import Processing 10 | 11 | 12 | try: 13 | from yapsy.IPlugin import IPlugin 14 | except ImportError: 15 | raise ImportError, 'Yapsy (Yet Another Plugin System) is required to run this program : http://yapsy.sourceforge.net' 16 | 17 | log = logging.getLogger("Processing avast Antivirus") 18 | 19 | class Avast(IPlugin, Processing): 20 | 21 | def run(self, objfile): 22 | self.key = "AntivirusScanAvast" 23 | self.score = 0 24 | self.avPath = self.options.get("avast_path", None) 25 | result = {} 26 | 27 | try: 28 | result["avast"] = self.avast(objfile.file.temp_file) 29 | except (Exception) as e: 30 | log.error("The module \"avast\" returned the following error: %s" % e) 31 | 32 | return result 33 | 34 | def avast(self, file_path): 35 | result = None 36 | 37 | if not self.avPath: 38 | raise Exception("avast Antivir path is not set in the configuration!") 39 | 40 | if os.path.isfile(self.avPath): 41 | try: 42 | output = subprocess.Popen([self.avPath, '-a', file_path], stdout=subprocess.PIPE).communicate()[0] 43 | 44 | if not "[OK]" in output: 45 | self.score = 10 46 | rpd = re.compile('\[infected by:\s(.+)]', re.IGNORECASE) 47 | rpdFind = re.findall(rpd, output) 48 | 49 | r = '' 50 | for r in rpdFind: 51 | result = r 52 | else: 53 | result = 'OK' 54 | except (Exception) as e: 55 | raise Exception("avast Antivir scan file '%s' Failed" % e) 56 | 57 | log.info(result) 58 | 59 | return result 60 | -------------------------------------------------------------------------------- /src/processing/antivirus_avast.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = antivirus_avast 3 | Module = antivirus_avast 4 | 5 | [Documentation] 6 | Description = avast! Free Antivirus für Linux v1.3.0 (http://www.avast.com) 7 | Author = Mark Lawrenz 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/processing/antivirus_avg.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | import logging 5 | import os 6 | import subprocess 7 | 8 | from core.abstracts import Processing 9 | 10 | 11 | try: 12 | from yapsy.IPlugin import IPlugin 13 | except ImportError: 14 | raise ImportError, 'Yapsy (Yet Another Plugin System) is required to run this program : http://yapsy.sourceforge.net' 15 | 16 | log = logging.getLogger("ProcessingAvgAv") 17 | 18 | class Avg(IPlugin, Processing): 19 | 20 | def run(self, objfile): 21 | self.key = "AntivirusScanAvg" 22 | self.score = 0 23 | self.avPath = self.options.get("avg_path", None) 24 | result = {} 25 | 26 | try: 27 | result["Avg"] = self.avg(objfile.file.temp_file) 28 | except (Exception) as e: 29 | log.error("The module \"Avg\" returned the following error: %s" % e) 30 | 31 | return result 32 | 33 | def avg(self, file_path): 34 | result = None 35 | 36 | if not self.avPath: 37 | raise Exception("AvgAV path is not set in the configuration!") 38 | 39 | if os.path.isfile(self.avPath): 40 | try: 41 | output = subprocess.Popen([self.avPath, '-w', '-m', '-a', file_path], stdout=subprocess.PIPE).communicate()[0] 42 | 43 | if not "Infections found : 0(0)" in output: 44 | self.score = 10 45 | result = output.split('\n')[6].split(' ')[1] 46 | else: 47 | result = 'OK' 48 | except (Exception) as e: 49 | raise Exception("Avg-Antivir scan file '%s' Failed" % e) 50 | 51 | return result 52 | -------------------------------------------------------------------------------- /src/processing/antivirus_avg.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = antivirus_avg 3 | Module = antivirus_avg 4 | 5 | [Documentation] 6 | Description = Avg AntiVirus Scan (http://free.avg.com) 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/processing/antivirus_avira.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | import logging 5 | import os 6 | import re 7 | import subprocess 8 | 9 | from core.abstracts import Processing 10 | 11 | 12 | try: 13 | from yapsy.IPlugin import IPlugin 14 | except ImportError: 15 | raise ImportError, 'Yapsy (Yet Another Plugin System) is required to run this program : http://yapsy.sourceforge.net' 16 | 17 | log = logging.getLogger("ProcessingAviraAv") 18 | 19 | class Avira(IPlugin, Processing): 20 | 21 | def run(self, objfile): 22 | self.key = "AntivirusScanAvira" 23 | self.score = 0 24 | self.avPath = self.options.get("avira_path", None) 25 | self.aviraHeurLevel = self.options.get("avira_heur_level", None) 26 | result = {} 27 | 28 | try: 29 | result["Avira"] = self.avira(objfile.file.temp_file) 30 | except (Exception) as e: 31 | log.error("The module \"Avira\" returned the following error: %s" % e) 32 | 33 | return result 34 | 35 | def avira(self, file_path): 36 | # Dictionary containing all the results of this processing. 37 | results = {} 38 | resultFund = None 39 | resultUrl = None 40 | 41 | if not self.avPath: 42 | raise Exception("AviraAV path is not set in the configuration!") 43 | 44 | if self.aviraHeurLevel and (self.aviraHeurLevel < 0 and self.aviraHeurLevel > 3): 45 | self.aviraHeurLevel = 3 46 | 47 | quarantineDir = self.options.get("quarantine-dir", "/tmp/") 48 | 49 | if os.path.isfile(self.avPath): 50 | try: 51 | output = subprocess.Popen([self.avPath, '--batch', '--scan-mode=all', \ 52 | '--heur-macro=yes', '--heur-level=' + str(self.aviraHeurLevel), \ 53 | '--quarantine-dir=' + quarantineDir, 54 | file_path], stdout=subprocess.PIPE).communicate()[0] 55 | 56 | rpd = re.compile('\sFUND:\s(.+)', re.IGNORECASE) 57 | rpdFind = re.findall(rpd, output) 58 | rpdSorted = sorted(rpdFind) 59 | 60 | r = '' 61 | for r in rpdSorted: 62 | resultFund = r 63 | 64 | if r == '': 65 | resultFund = 'OK' 66 | else: 67 | # Fund AV-Hit 68 | self.score = 10 69 | rpd = re.compile('\sFUND-URL:\s(.+)', re.IGNORECASE) 70 | rpdFind = re.findall(rpd, output) 71 | rpdSorted = sorted(rpdFind) 72 | 73 | r = '' 74 | for r in rpdSorted: 75 | resultUrl = r 76 | 77 | results["scan"] = resultFund 78 | 79 | if resultUrl: 80 | results["url"] = resultUrl 81 | 82 | except (Exception) as e: 83 | raise Exception("Avira-Antivir scan file '%s' Failed" % e) 84 | 85 | return results 86 | -------------------------------------------------------------------------------- /src/processing/antivirus_avira.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = antivirus_avira 3 | Module = antivirus_avira 4 | 5 | [Documentation] 6 | Description = Avira AntiVirus Scan (http://www.avira.com/de/avira-free-antivirus) 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/processing/antivirus_bitDefender.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | import logging 5 | import os 6 | import re 7 | import subprocess 8 | 9 | from core.abstracts import Processing 10 | 11 | 12 | try: 13 | from yapsy.IPlugin import IPlugin 14 | except ImportError: 15 | raise ImportError, 'Yapsy (Yet Another Plugin System) is required to run this program : http://yapsy.sourceforge.net' 16 | 17 | log = logging.getLogger("ProcessingBitDefenderAv") 18 | 19 | class BitDefender(IPlugin, Processing): 20 | 21 | def run(self, objfile): 22 | self.key = "AntivirusScanBitDefender" 23 | self.score = 0 24 | self.avPath = self.options.get("bdscan_path", None) 25 | result = {} 26 | 27 | try: 28 | result["BitDefender"] = self.bitDefender(objfile.file.temp_file) 29 | except (Exception) as e: 30 | log.error("The module \"BitDefender\" returned the following error: %s" % e) 31 | 32 | return result 33 | 34 | def bitDefender(self, file_path): 35 | result = None 36 | 37 | if not self.avPath: 38 | raise Exception("BitDefender Antivir path is not set in the configuration!") 39 | 40 | if os.path.isfile(self.avPath): 41 | try: 42 | output = subprocess.Popen([self.avPath, file_path], stdout=subprocess.PIPE).communicate()[0] 43 | 44 | if not "Infected files: 0" in output: 45 | self.score = 10 46 | 47 | rpd = re.compile('\sinfected:\s(.+)', re.IGNORECASE) 48 | rpdFind = re.findall(rpd, output) 49 | 50 | r = '' 51 | for r in rpdFind: 52 | result = r 53 | else: 54 | result = 'OK' 55 | except (Exception) as e: 56 | raise Exception("BitDefender-Antivir scan file '%s' Failed" % e) 57 | 58 | log.info(result) 59 | return result 60 | -------------------------------------------------------------------------------- /src/processing/antivirus_bitDefender.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = antivirus_bitDefender 3 | Module = antivirus_bitDefender 4 | 5 | [Documentation] 6 | Description = BitDefender AntiVirus Scan (http://www.bitdefender.co.uk/) 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/processing/antivirus_clamav.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | import logging 5 | import os 6 | import subprocess 7 | 8 | from core.abstracts import Processing 9 | 10 | 11 | try: 12 | from yapsy.IPlugin import IPlugin 13 | except ImportError: 14 | raise ImportError, 'Yapsy (Yet Another Plugin System) is required to run this program : http://yapsy.sourceforge.net' 15 | 16 | log = logging.getLogger("ProcessingClamAv") 17 | 18 | class ClamAv(IPlugin, Processing): 19 | 20 | def run(self, objfile): 21 | self.key = "AntivirusScanClamAv" 22 | self.score = 0 23 | self.avPath = self.options.get("clamscan_path", None) 24 | result = {} 25 | 26 | try: 27 | result["ClamAv"] = self.clamav(objfile.file.temp_file) 28 | except (Exception) as e: 29 | log.error("The module \"ClamAv\" returned the following error: %s" % e) 30 | 31 | return result 32 | 33 | def clamav(self, file_path): 34 | result = None 35 | 36 | if not self.avPath: 37 | raise Exception("ClamAv path is not set in the configuration!") 38 | 39 | if os.path.isfile(self.avPath): 40 | try: 41 | output = subprocess.Popen([self.avPath, "--no-summary", file_path], \ 42 | stdout=subprocess.PIPE).communicate()[0] 43 | result = output.split('\n')[0].split(':')[1] 44 | 45 | if not "OK" in result: 46 | self.score = 10 47 | except (Exception) as e: 48 | raise Exception("ClamAV scan file %s Failed" % e) 49 | return result 50 | -------------------------------------------------------------------------------- /src/processing/antivirus_clamav.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = antivirus_clamav 3 | Module = antivirus_clamav 4 | 5 | [Documentation] 6 | Description = ClamAv AntiVirus Scan (http://www.clamav.net/lang/en/) 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/processing/antivirus_comodo.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | import logging 5 | import os 6 | import re 7 | import subprocess 8 | 9 | from core.abstracts import Processing 10 | 11 | 12 | try: 13 | from yapsy.IPlugin import IPlugin 14 | except ImportError: 15 | raise ImportError, 'Yapsy (Yet Another Plugin System) is required to run this program : http://yapsy.sourceforge.net' 16 | 17 | log = logging.getLogger("Processing COMODOAv") 18 | 19 | class Comodo(IPlugin, Processing): 20 | 21 | def run(self, objfile): 22 | self.key = "AntivirusScanCOMODO" 23 | self.score = 0 24 | self.avPath = self.options.get("comodo_path", None) 25 | result = {} 26 | 27 | try: 28 | result["COMODO"] = self.comodo(objfile.file.temp_file) 29 | except (Exception) as e: 30 | log.error("The module \"COMODO\" returned the following error: %s" % e) 31 | 32 | return result 33 | 34 | def comodo(self, file_path): 35 | result = None 36 | 37 | if not self.avPath: 38 | raise Exception("COMODO Antivir path is not set in the configuration!") 39 | 40 | if os.path.isfile(self.avPath): 41 | try: 42 | output = subprocess.Popen([self.avPath, '-vs', file_path], stdout=subprocess.PIPE).communicate()[0] 43 | 44 | if not "Number of Found Viruses: 0" in output: 45 | self.score = 10 46 | 47 | rpd = re.compile('\sFound Virus, Malware Name is\s(.+)', re.IGNORECASE) 48 | rpdFind = re.findall(rpd, output) 49 | 50 | r = '' 51 | for r in rpdFind: 52 | result = r 53 | else: 54 | result = 'OK' 55 | except (Exception) as e: 56 | raise Exception("COMODO Antivir scan file '%s' Failed" % e) 57 | 58 | log.info(result) 59 | return result 60 | -------------------------------------------------------------------------------- /src/processing/antivirus_comodo.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = antivirus_comodo 3 | Module = antivirus_comodo 4 | 5 | [Documentation] 6 | Description = COMODO Antivirus Scan (http://www.comodo.com/) 7 | Author = Mark Lawrenz 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/processing/antivirus_eset.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | import logging 5 | import os 6 | import re 7 | import subprocess 8 | 9 | from core.abstracts import Processing 10 | 11 | 12 | try: 13 | from yapsy.IPlugin import IPlugin 14 | except ImportError: 15 | raise ImportError, 'Yapsy (Yet Another Plugin System) is required to run this program : http://yapsy.sourceforge.net' 16 | 17 | log = logging.getLogger("Processing ESET Antivir") 18 | 19 | class Eset(IPlugin, Processing): 20 | 21 | def run(self, objfile): 22 | self.key = "AntivirusScanESET" 23 | self.score = 0 24 | self.avPath = self.options.get("eset_path", None) 25 | result = {} 26 | 27 | try: 28 | result["ESET"] = self.eset(objfile.file.temp_file) 29 | except (Exception) as e: 30 | log.error("The module \"ESET\" returned the following error: %s" % e) 31 | 32 | return result 33 | 34 | def eset(self, file_path): 35 | result = None 36 | 37 | if not self.avPath: 38 | raise Exception("ESET Antivir path is not set in the configuration!") 39 | 40 | if os.path.isfile(self.avPath): 41 | try: 42 | output = subprocess.Popen([self.avPath, '--clean-mode=none', file_path], stdout=subprocess.PIPE).communicate()[0] 43 | 44 | if (("threat=" in output) and (not 'threat=""' in output)): 45 | self.score = 10 46 | rpd = re.compile('\sthreat=\"(.+)\",\saction=', re.IGNORECASE) 47 | rpdFind = re.findall(rpd, output) 48 | 49 | r = '' 50 | for r in rpdFind: 51 | result = r 52 | else: 53 | result = 'OK' 54 | 55 | except (Exception) as e: 56 | raise Exception("ESET Antivir scan file '%s' Failed" % e) 57 | 58 | log.info(result) 59 | 60 | return result 61 | -------------------------------------------------------------------------------- /src/processing/antivirus_eset.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = antivirus_eset 3 | Module = antivirus_eset 4 | 5 | [Documentation] 6 | Description = ESET File Security für Linux v4.0.10 (http://www.eset.com) 7 | Author = Mark Lawrenz 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/processing/antivirus_fprot.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | import logging 5 | import os 6 | import re 7 | import subprocess 8 | 9 | from core.abstracts import Processing 10 | 11 | 12 | try: 13 | from yapsy.IPlugin import IPlugin 14 | except ImportError: 15 | raise ImportError, 'Yapsy (Yet Another Plugin System) is required to run this program : http://yapsy.sourceforge.net' 16 | 17 | log = logging.getLogger("ProcessingFProtAv") 18 | 19 | class FProt(IPlugin, Processing): 20 | 21 | def run(self, objfile): 22 | self.key = "AntivirusScanFProt" 23 | self.score = 0 24 | self.avPath = self.options.get("fprot_path", None) 25 | result = {} 26 | 27 | try: 28 | result["FProt"] = self.fprot(objfile.file.temp_file) 29 | except (Exception) as e: 30 | log.error("The module \"F-Prot\" returned the following error: %s" % e) 31 | 32 | return result 33 | 34 | def fprot(self, file_path): 35 | result = None 36 | 37 | if not self.avPath: 38 | raise Exception("F-Prot Antivir path is not set in the configuration!") 39 | 40 | if os.path.isfile(self.avPath): 41 | try: 42 | output = subprocess.Popen([self.avPath, '-r', '-f', '--scanlevel=4', '--heurlevel=4', '--adware', '--applications', 43 | file_path], stdout=subprocess.PIPE).communicate()[0] 44 | 45 | if not "Infected files: 0" in output: 46 | self.score = 10 47 | 48 | rpd = re.compile('\[Found\s.*\]\s<(.+)>\s\s', re.IGNORECASE) 49 | rpdFind = re.findall(rpd, output) 50 | 51 | r = '' 52 | for r in rpdFind: 53 | result = r 54 | else: 55 | result = 'OK' 56 | except (Exception) as e: 57 | raise Exception("F-Prot-Antivir scan file '%s' Failed" % e) 58 | 59 | log.info(result) 60 | return result 61 | -------------------------------------------------------------------------------- /src/processing/antivirus_fprot.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = antivirus_fprot 3 | Module = antivirus_fprot 4 | 5 | [Documentation] 6 | Description = F-Prot AntiVirus Scan (http://www.f-prot.com/) 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/processing/antivirus_fsecure.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | import logging 5 | import os 6 | import re 7 | import subprocess 8 | 9 | from core.abstracts import Processing 10 | 11 | 12 | try: 13 | from yapsy.IPlugin import IPlugin 14 | except ImportError: 15 | raise ImportError, 'Yapsy (Yet Another Plugin System) is required to run this program : http://yapsy.sourceforge.net' 16 | 17 | log = logging.getLogger("Processing F-Secure AV") 18 | 19 | class FSecure(IPlugin, Processing): 20 | 21 | def run(self, objfile): 22 | self.key = "AntivirusScanF-Secure" 23 | self.score = 0 24 | self.avPath = self.options.get("fsecure_path", None) 25 | result = {} 26 | 27 | try: 28 | result["F-Secure"] = self.fsecure(objfile.file.temp_file) 29 | except (Exception) as e: 30 | log.error("The module \"F-Secure\" returned the following error: %s" % e) 31 | 32 | return result 33 | 34 | def fsecure(self, file_path): 35 | result = None 36 | 37 | if not self.avPath: 38 | raise Exception("F-Secure Antivir path is not set in the configuration!") 39 | 40 | if os.path.isfile(self.avPath): 41 | try: 42 | output = subprocess.Popen([self.avPath, '--virus-action1=none', file_path], stdout=subprocess.PIPE).communicate()[0] 43 | 44 | 45 | if "Infected:" in output: 46 | self.score = 10 47 | 48 | rpd = re.compile('\sInfected:\s(.+)', re.IGNORECASE) 49 | rpdFind = re.findall(rpd, output) 50 | 51 | r = '' 52 | for r in rpdFind: 53 | result = r 54 | 55 | elif "Riskware:" in output: 56 | self.score = 10 57 | 58 | rpd = re.compile('\sRiskware:\s(.+)', re.IGNORECASE) 59 | rpdFind = re.findall(rpd, output) 60 | 61 | r = '' 62 | for r in rpdFind: 63 | result = r 64 | else: 65 | result = 'OK' 66 | except (Exception) as e: 67 | raise Exception("F-Secure Antivir scan file '%s' Failed" % e) 68 | 69 | log.info(result) 70 | return result 71 | -------------------------------------------------------------------------------- /src/processing/antivirus_fsecure.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = antivirus_fsecure 3 | Module = antivirus_fsecure 4 | 5 | [Documentation] 6 | Description = F-Secure Linux Security v10.10.142 (http://www.f-secure.com) 7 | Author = Mark Lawrenz 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/processing/antivirus_teamcymru.py: -------------------------------------------------------------------------------- 1 | import os 2 | import logging 3 | import datetime 4 | import simplejson 5 | 6 | from core.abstracts import Processing 7 | 8 | try: 9 | from yapsy.IPlugin import IPlugin 10 | except ImportError: 11 | raise ImportError, 'Yapsy (Yet Another Plugin System) is required to run this program : http://yapsy.sourceforge.net' 12 | 13 | log = logging.getLogger("Processing Teamcymru") 14 | 15 | CMD = 'dig +short x.malware.hash.cymru.com TXT' 16 | 17 | class Teamcymru(IPlugin, Processing): 18 | 19 | def run(self, objfile): 20 | self.key = "Teamcymru" 21 | self.score = -1 22 | 23 | try: 24 | value = self.submit(objfile.file.get_fileMd5()) 25 | if value: 26 | malwarepercent = value.get("malwarepercent") 27 | 28 | self.score = 10 29 | 30 | if malwarepercent > 10 and malwarepercent < 30: 31 | self.score = 25 32 | elif malwarepercent >= 30: 33 | self.score = 50 34 | 35 | return value 36 | except (Exception) as e: 37 | log.error("The module \"Teamcymru\" returned the following error: %s" % e) 38 | 39 | return 40 | 41 | def submit(self, md5): 42 | cmd = CMD.replace('x', md5) 43 | log.debug('Submit '+ cmd) 44 | result=os.popen(cmd) 45 | 46 | result=result.readlines() 47 | 48 | if len(result)> 0: 49 | result=result[0] 50 | result=result.replace('\n','') 51 | result=result.replace('"','') 52 | if result != '127.0.0.2': 53 | timestamp=int(result.split(' ')[0]) 54 | lastseen = datetime.datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S') 55 | malwarepercent=result.split(' ')[1] 56 | data = {"lastseen":lastseen, 57 | "malwarepercent":malwarepercent} #anti-virus package detection rate 58 | return data 59 | -------------------------------------------------------------------------------- /src/processing/antivirus_teamcymru.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = antivirus_teamcymru 3 | Module = antivirus_teamcymru 4 | 5 | [Documentation] 6 | Description = Malware Hash Registry (MHR) lookup (https://www.team-cymru.org) 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/processing/net_getOwnLocation.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | import logging 5 | import re 6 | import socket 7 | 8 | from core.abstracts import Processing 9 | import core.commonutils as commonutils 10 | 11 | 12 | try: 13 | from BeautifulSoup import BeautifulSoup as bs 14 | except ImportError: 15 | raise ImportError, 'Beautiful Soup parser: http://www.crummy.com/software/BeautifulSoup/' 16 | 17 | try: 18 | from yapsy.IPlugin import IPlugin 19 | except ImportError: 20 | raise ImportError, 'Yapsy (Yet Another Plugin System) is required to run this program : http://yapsy.sourceforge.net' 21 | 22 | log = logging.getLogger("ProcessingGetOwnLocations") 23 | 24 | class getOwnLocation(IPlugin, Processing): 25 | 26 | def run(self, objfile): 27 | self.key = "OwnLocation" 28 | self.score = -1 29 | 30 | result = self.wieistmeineip() 31 | return result 32 | 33 | def wieistmeineip(self): 34 | result = {} 35 | # Save original socket 36 | originalSocket = socket.socket 37 | # Set TOR Socks proxy 38 | commonutils.setTorProxy() 39 | 40 | try: 41 | # Load 42 | soup = self.parse("http://www.wieistmeineip.de") 43 | location = soup.findAll("div", { "class" : "location" })[0] 44 | location = bs(location.text, convertEntities=bs.HTML_ENTITIES) 45 | 46 | ip = soup.findAll('div', id='ipv4')[0] 47 | raw_ip = bs(ip.text, convertEntities=bs.HTML_ENTITIES) 48 | pattern = re.compile('[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}') 49 | ip = re.search(pattern, raw_ip.text) 50 | 51 | result["ipaddress"] = ip.group(0) 52 | result["country"] = str(location) 53 | finally: 54 | # Removing SOCKS Tor Proxy 55 | socket.socket = originalSocket 56 | 57 | return result 58 | 59 | -------------------------------------------------------------------------------- /src/processing/net_getOwnLocation.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = net_getOwnLocation 3 | Module = net_getOwnLocation 4 | 5 | [Documentation] 6 | Description = Returns the own internet location. 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/processing/net_inetSourceAnalysis.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = net_inetSourceAnalysis 3 | Module = net_inetSourceAnalysis 4 | 5 | [Documentation] 6 | Description = Check IP and Host for reputation. 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/processing/ole_officescan.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = ole_officescan 3 | Module = ole_officescan 4 | 5 | [Documentation] 6 | Description = OLE - Analyze the suspicious Office documents 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/processing/pdf_pdfid.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | import logging 5 | 6 | from core.abstracts import Processing 7 | from utils.pdfid import PDFiD 8 | 9 | 10 | try: 11 | from yapsy.IPlugin import IPlugin 12 | except ImportError: 13 | raise ImportError, 'Yapsy (Yet Another Plugin System) is required to run this program : http://yapsy.sourceforge.net' 14 | 15 | log = logging.getLogger("ProcessingPDFID") 16 | 17 | RISKY_PDF_SECTIONS = ['/OpenAction', '/AA', '/JS', '/JavaScript', '/Launch', '/URI', '/Action', '/GoToR', '/RichMedia', '/ObjStm'] 18 | 19 | class PDFID(IPlugin, Processing): 20 | 21 | def run(self, objfile): 22 | """Gets PDF identify and informations. 23 | """ 24 | self.key = "PDFID" 25 | self.score = -1 26 | returnValue = None 27 | 28 | try: 29 | pdfIdXML = PDFiD(file=objfile.file.temp_file, allNames=False, extraData=True, disarm=False, force=True) 30 | returnValue = self.xml2dic(pdfIdXML) 31 | log.debug("PDFID returns: %s" % returnValue) 32 | except Exception, e: 33 | log.warn("Error - PDFID returns: %s" % e) 34 | 35 | return returnValue 36 | 37 | def xml2dic(self, xmlDoc): 38 | scoreHit = 0 39 | # Get Top Layer Data 40 | errorOccured = xmlDoc.documentElement.getAttribute('ErrorOccured') 41 | errorMessage = xmlDoc.documentElement.getAttribute('ErrorMessage') 42 | header = xmlDoc.documentElement.getAttribute('Header') 43 | isPdf = xmlDoc.documentElement.getAttribute('IsPDF') 44 | 45 | # extra data 46 | countEof = xmlDoc.documentElement.getAttribute('CountEOF') 47 | countChatAfterLastEof = xmlDoc.documentElement.getAttribute('CountCharsAfterLastEOF') 48 | totalEntropy = xmlDoc.documentElement.getAttribute('TotalEntropy') 49 | streamEntropy = xmlDoc.documentElement.getAttribute('StreamEntropy') 50 | nonStreamEntropy = xmlDoc.documentElement.getAttribute('NonStreamEntropy') 51 | 52 | keywords = [] 53 | 54 | # grab all keywords 55 | for node in xmlDoc.documentElement.getElementsByTagName('Keywords')[0].childNodes: 56 | name = node.getAttribute('Name') 57 | count = int(node.getAttribute('Count')) 58 | if int(node.getAttribute('HexcodeCount')) > 0: 59 | hexCount = int(node.getAttribute('HexcodeCount')) 60 | else: 61 | hexCount = 0 62 | 63 | if name in RISKY_PDF_SECTIONS and count > 0: 64 | name = "[RISKY] %s" % name 65 | scoreHit += 1 66 | 67 | if name == "/Page" and count < 2: 68 | scoreHit += 1 69 | 70 | keyword = { 'count':count, 'hexcodecount':hexCount, 'name':name } 71 | keywords.append(keyword) 72 | 73 | if totalEntropy > 5.9: 74 | scoreHit += 1 75 | 76 | data = {'header':header, 77 | 'isPdf':isPdf, 78 | 'errorOccured':errorOccured, 79 | 'errorMessage':errorMessage, 80 | 'totalEntropy':totalEntropy, 81 | 'streamEntropy':streamEntropy, 82 | 'countEof':countEof, 83 | 'countChatAfterLastEof':countChatAfterLastEof, 84 | 'nonStreamEntropy':nonStreamEntropy, 85 | 'keywords':keywords } 86 | 87 | # Calculate Scoring 88 | if scoreHit == 2: 89 | self.score = 8 90 | elif scoreHit > 2: 91 | self.score = 10 92 | 93 | return data 94 | -------------------------------------------------------------------------------- /src/processing/pdf_pdfid.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = pdf_pdfid 3 | Module = pdf_pdfid 4 | 5 | [Documentation] 6 | Description = PDF - Analyze the suspicious PDF documents 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/processing/pe_pefile.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = pe_peid 3 | Module = pe_peid 4 | 5 | [Documentation] 6 | Description = PE - Detects most common packers, cryptors and compilers for PE files 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/processing/pe_peid.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | import logging 5 | import os 6 | 7 | from core.abstracts import Processing 8 | from core.constants import RAGPICKER_ROOT 9 | from utils.pefile import PE 10 | from utils.pefile import PEFormatError 11 | from utils.peutils import SignatureDatabase 12 | 13 | 14 | try: 15 | from yapsy.IPlugin import IPlugin 16 | except ImportError: 17 | raise ImportError, 'Yapsy (Yet Another Plugin System) is required to run this program : http://yapsy.sourceforge.net' 18 | 19 | log = logging.getLogger("ProcessingPEID") 20 | 21 | class PEID(IPlugin, Processing): 22 | 23 | def run(self, objfile): 24 | """Gets PEID signatures. 25 | @return: matched signatures or None. 26 | """ 27 | self.key = "PEID" 28 | self.score = -1 29 | 30 | try: 31 | pe = PE(data=objfile.file.file_data) 32 | signatures = SignatureDatabase(os.path.join(RAGPICKER_ROOT, 'data', 'peiddb.txt')) 33 | match = signatures.match(pe, ep_only=True) 34 | if match: 35 | log.info("PEID match: %s" % match) 36 | self.score = 10 37 | return match 38 | except PEFormatError, e: 39 | log.warn("Error - No Portable Executable: %s" % e) 40 | 41 | return None 42 | -------------------------------------------------------------------------------- /src/processing/pe_peid.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = pe_pefile 3 | Module = pe_pefile 4 | 5 | [Documentation] 6 | Description = PE - pefile is a module to read and work with PE (Portable Executable) files 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/processing/pe_verifySigs.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = pe_verifySigs 3 | Module = pe_verifySigs 4 | 5 | [Documentation] 6 | Description = PE - Compute hashes, validate digital signature and list details 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/processing/rtf_rtfscan.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = rtf_rtfscan 3 | Module = rtf_rtfscan 4 | 5 | [Documentation] 6 | Description = RTF - Analyze the suspicious RTF documents 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/reporting/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/robbyFux/Ragpicker/77d68ff3a13851ed4c256635f0673a4c2a84ef51/src/reporting/__init__.py -------------------------------------------------------------------------------- /src/reporting/codeDB.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = codeDB 3 | Module = codeDB 4 | 5 | [Documentation] 6 | Description = codeDB report-plugin for Viviane's Code Database 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/reporting/filedump.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | import logging 5 | import os 6 | import tempfile 7 | 8 | from core.abstracts import Report 9 | 10 | 11 | try: 12 | from yapsy.IPlugin import IPlugin 13 | except ImportError: 14 | raise ImportError, 'Yapsy (Yet Another Plugin System) is required to run this program : http://yapsy.sourceforge.net' 15 | 16 | log = logging.getLogger(__name__) 17 | 18 | class FileDump(IPlugin, Report): 19 | """Save downloaded file on the file system""" 20 | 21 | def run(self, results, objfile): 22 | dumpdir = self.options.get("dumpdir", None) 23 | suffix = self.options.get("suffix", None) 24 | 25 | if not dumpdir: 26 | raise Exception("dumpdir not configured, skip") 27 | 28 | try: 29 | if not os.path.exists(dumpdir): 30 | os.makedirs(dumpdir) 31 | d = tempfile.mkdtemp(dir=dumpdir) 32 | except Exception as e: 33 | raise Exception('Could not open %s for writing (%s)', dumpdir, e) 34 | else: 35 | os.rmdir(d) 36 | 37 | dest = dumpdir + objfile.file.get_type() 38 | 39 | if not os.path.exists(dest): 40 | os.makedirs(dest) 41 | 42 | # Save file 43 | fpath = dest + '/' + objfile.file.get_fileMd5() + self.getFileExtension(suffix, objfile.file) 44 | self.saveFile(objfile.file.file_data, fpath) 45 | 46 | # Save unpacked file 47 | if objfile.unpacked_file: 48 | fpath = "%s/%s_unpacked_(%s)%s" % (dest, objfile.unpacked_file.get_fileMd5(), objfile.file.get_fileMd5(), \ 49 | self.getFileExtension(suffix, objfile.unpacked_file)) 50 | self.saveFile(objfile.unpacked_file.file_data, fpath) 51 | 52 | # Save included files 53 | if len(objfile.included_files) > 0: 54 | log.info("Save included files") 55 | for incl_file in objfile.included_files: 56 | log.info("File: " + incl_file.get_fileMd5()) 57 | fpath = "%s/%s_included_(%s)%s" % (dest, incl_file.get_fileMd5(), objfile.file.get_fileMd5(), \ 58 | self.getFileExtension(suffix, incl_file)) 59 | self.saveFile(incl_file.file_data, fpath) 60 | 61 | return None 62 | 63 | def getFileExtension(self, suffix, file): 64 | if file.file_extension(): 65 | file_extension = '.' + file.file_extension() + suffix 66 | else: 67 | file_extension = suffix 68 | return file_extension 69 | 70 | def saveFile(self, file_data, fpath): 71 | if not os.path.exists(fpath): 72 | file = open(fpath, 'wb') 73 | file.write(file_data) 74 | file.close 75 | log.info("Saved file %s" % fpath) 76 | -------------------------------------------------------------------------------- /src/reporting/filedump.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = filedump 3 | Module = filedump 4 | 5 | [Documentation] 6 | Description = Save sample file on the file system 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/reporting/hpfriends.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | import datetime 5 | import json 6 | import jsonpickle 7 | import logging 8 | 9 | from core.abstracts import Report 10 | from core.commonutils import DatetimeHandler 11 | 12 | 13 | try: 14 | from yapsy.IPlugin import IPlugin 15 | except ImportError: 16 | raise ImportError, 'Yapsy (Yet Another Plugin System) is required to run this program : http://yapsy.sourceforge.net' 17 | 18 | try: 19 | import utils.hpfeeds as hpfeeds 20 | except ImportError: 21 | raise ImportError, 'Unable to import HPFeeds library: https://github.com/rep/hpfeeds' 22 | 23 | log = logging.getLogger("ReportingHPFriends") 24 | 25 | class HPFriends(IPlugin, Report): 26 | 27 | def run(self, results, objfile): 28 | host = self.options.get("host", "hpfriends.honeycloud.net") 29 | port = self.options.get("port", 20000) 30 | ident = self.options.get("ident") 31 | secret = self.options.get("secret") 32 | channel_reports = self.options.get("channel_reports") 33 | channel_files = self.options.get("channel_files") 34 | 35 | if not ident or not secret: 36 | raise Exception("HPFriends Identifier and Secret not configurated") 37 | 38 | try: 39 | # Connect to HPFriends 40 | hpc = hpfeeds.HPC(host, port, ident, secret, timeout=60) 41 | 42 | if channel_reports: 43 | # publish JSON-Report on the HPFriends channel 44 | log.info("publish JSON-Report on the HPFriends channel %s" % channel_reports) 45 | 46 | jsonpickle.set_encoder_options('simplejson', indent=4) 47 | jsonpickle.handlers.registry.register(datetime.datetime, DatetimeHandler) 48 | hpc.publish(channel_reports, jsonpickle.encode(results)) 49 | 50 | if channel_files: 51 | # publish RAW-File as BASE64 on the HPFriends channel 52 | log.info("publish BASE64 on the HPFriends channel %s" % channel_files) 53 | hpc.publish(channel_files, json.dumps(objfile.file.get_fileB64encode(), sort_keys=False, indent=4)) 54 | except hpfeeds.FeedException as e: 55 | raise Exception("publish on the channel failed: %s" % e) 56 | finally: 57 | hpc.close() 58 | -------------------------------------------------------------------------------- /src/reporting/hpfriends.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = hpfriends 3 | Module = hpfriends 4 | 5 | [Documentation] 6 | Description = Publishes the results on an HPFeeds channel 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/reporting/jsondump.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | import codecs 5 | import datetime 6 | import jsonpickle 7 | import logging 8 | import os 9 | import tempfile 10 | import uuid 11 | 12 | from core.abstracts import Report 13 | from core.commonutils import DatetimeHandler 14 | from core.commonutils import UUIDHandler 15 | from core.commonutils import convertDirtyDict2ASCII 16 | 17 | 18 | try: 19 | from yapsy.IPlugin import IPlugin 20 | except ImportError: 21 | raise ImportError, 'Yapsy (Yet Another Plugin System) is required to run this program : http://yapsy.sourceforge.net' 22 | 23 | log = logging.getLogger(__name__) 24 | 25 | class JsonDump(IPlugin, Report): 26 | """Saves analysis results in JSON format.""" 27 | 28 | def run(self, results, objfile): 29 | """Writes report. 30 | @param results: results dict. 31 | @param objfile: file object 32 | @raise Exception: if fails to write report. 33 | """ 34 | dumpdir = self.options.get("dumpdir", None) 35 | 36 | if not dumpdir: 37 | raise Exception("dumpdir not configured, skip") 38 | 39 | try: 40 | if not os.path.exists(dumpdir): 41 | os.makedirs(dumpdir) 42 | d = tempfile.mkdtemp(dir=dumpdir) 43 | except Exception as e: 44 | raise Exception('Could not open %s for writing (%s)', dumpdir, e) 45 | else: 46 | os.rmdir(d) 47 | 48 | url_md5 = results["Info"]["url"]["md5"] 49 | file_md5 = results["Info"]["file"]["md5"] 50 | jfile = url_md5 + "_" + file_md5 + ".json" 51 | 52 | try: 53 | jsonpickle.set_encoder_options('simplejson', indent=4) 54 | jsonpickle.handlers.registry.register(datetime.datetime, DatetimeHandler) 55 | jsonpickle.handlers.registry.register(uuid.UUID, UUIDHandler) 56 | jsonReport = jsonpickle.encode(results) 57 | except (UnicodeError, TypeError): 58 | jsonReport = jsonpickle.encode(convertDirtyDict2ASCII(results)) 59 | 60 | try: 61 | if not os.path.exists(dumpdir + jfile): 62 | report = codecs.open(os.path.join(dumpdir, jfile), "w", "utf-8") 63 | report.write(jsonReport) 64 | report.close() 65 | except (TypeError, IOError) as e: 66 | raise Exception("Failed to generate JSON report: %s" % e) 67 | -------------------------------------------------------------------------------- /src/reporting/jsondump.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = jsondump 3 | Module = jsondump 4 | 5 | [Documentation] 6 | Description = Saves analysis results in JSON format 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/reporting/mongodb.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | import logging 5 | 6 | from yapsy.IPlugin import IPlugin 7 | from core.database import Database 8 | from core.abstracts import Report 9 | 10 | log = logging.getLogger("MongoDB") 11 | 12 | class MongoDB(IPlugin, Report): 13 | """Stores report in MongoDB.""" 14 | 15 | def run(self, results, objfile): 16 | """Writes report. 17 | @param results: analysis results dictionary. 18 | @param objfile: file object 19 | """ 20 | database = Database() 21 | 22 | # Count query using URL hash and file hash 23 | count = database.countRagpickerDB(results["Info"]["file"]["md5"], results["Info"]["url"]["md5"]) 24 | 25 | # If report available for the file and url -> not insert 26 | if count == 0: 27 | # Create a copy of the dictionary. This is done in order to not modify 28 | # the original dictionary and possibly compromise the following 29 | # reporting modules. 30 | report = dict(results) 31 | # Store the report 32 | database.insertRagpickerDB(report) 33 | 34 | count = database.countFamilyDB(objfile.family.parentObjectSHA256) 35 | if count == 0: 36 | if objfile.family.unpackedObjectSHA256 != "" or len(objfile.family.siblingObjectsSHA256) > 0: 37 | log.info(objfile.family) 38 | report = dict(objfile.family.__dict__) 39 | database.insertFamily(report) 40 | 41 | def deleteAll(self): 42 | """Deletes all reports. 43 | """ 44 | # Alle Ragpicker-Daten aus der MongoDB loeschen 45 | count = Database().deleteRagpickerDB() 46 | 47 | print "*** MongoDB (Ragpicker)***" 48 | print "deleted documents:" + str(count) 49 | print "" 50 | 51 | count = Database().deleteFamilyDB() 52 | 53 | print "*** MongoDB (Family)***" 54 | print "deleted documents:" + str(count) 55 | print "" 56 | 57 | count = Database().deleteSandboxTaskQueueDB() 58 | 59 | print "*** MongoDB (SandboxTaskQueue)***" 60 | print "deleted documents:" + str(count) 61 | print "" 62 | -------------------------------------------------------------------------------- /src/reporting/mongodb.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = mongodb 3 | Module = mongodb 4 | 5 | [Documentation] 6 | Description = Reporting-Modul for MongoDB 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/reporting/mysql.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | from yapsy.IPlugin import IPlugin 5 | 6 | from core.abstracts import Report 7 | 8 | class MySQL(IPlugin, Report): 9 | """Stores data from long-run analysis in MySQL.""" 10 | 11 | def run(self, results, objfile): 12 | # Import muss hier stehen, sonst kommt es bei Konfiguration ohne Mysql zum Fehler 13 | from core.databaseMysql import DatabaseMySQL 14 | """Writes report. 15 | @param results: analysis results dictionary. 16 | @param objfile: file object 17 | """ 18 | database = DatabaseMySQL() 19 | 20 | print "mysql.py Methode Run" 21 | """ 22 | # Count query using URL hash and file hash 23 | count = database.countRagpickerDB(results["Info"]["file"]["md5"], results["Info"]["url"]["md5"]) 24 | 25 | # If report available for the file and url -> not insert 26 | if count == 0: 27 | # Create a copy of the dictionary. This is done in order to not modify 28 | # the original dictionary and possibly compromise the following 29 | # reporting modules. 30 | report = dict(results) 31 | # Store the report 32 | database.insertRagpickerDB(report) 33 | """ 34 | def deleteAll(self): 35 | """Deletes all reports. 36 | """ 37 | print "mysql.py Methode DeleteAll" 38 | """ 39 | # Alle Ragpicker-Daten aus der MongoDB loeschen 40 | count = Database().deleteRagpickerDB() 41 | 42 | print "*** MongoDB (Ragpicker)***" 43 | print "deleted documents:" + str(count) 44 | print "" 45 | """ -------------------------------------------------------------------------------- /src/reporting/mysql.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = mysql 3 | Module = mysql 4 | 5 | [Documentation] 6 | Description = Reporting-Modul for MySQL 7 | Author = Mark Lawrenz 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/reporting/reporthtml.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | 4 | import codecs 5 | import logging 6 | import os 7 | import tempfile 8 | 9 | from core.abstracts import Report 10 | from core.commonutils import convertDirtyDict2ASCII 11 | from core.constants import RAGPICKER_ROOT 12 | 13 | 14 | try: 15 | from yapsy.IPlugin import IPlugin 16 | except ImportError: 17 | raise ImportError, 'Yapsy (Yet Another Plugin System) is required to run this program: http://yapsy.sourceforge.net' 18 | 19 | try: 20 | from jinja2.loaders import FileSystemLoader 21 | from jinja2.environment import Environment 22 | except ImportError: 23 | raise ImportError("Jinja2 Python library is required to generate HTML reports: http://jinja.pocoo.org") 24 | 25 | log = logging.getLogger(__name__) 26 | 27 | class ReportHTML(IPlugin, Report): 28 | """Save report in HTML format.""" 29 | 30 | def run(self, results, objfile): 31 | dumpdir = self.options.get("dumpdir", None) 32 | 33 | if not dumpdir: 34 | raise Exception("dumpdir not configured, skip") 35 | 36 | try: 37 | if not os.path.exists(dumpdir): 38 | os.makedirs(dumpdir) 39 | d = tempfile.mkdtemp(dir=dumpdir) 40 | except Exception as e: 41 | raise Exception('Could not open %s for writing (%s)', dumpdir, e) 42 | else: 43 | os.rmdir(d) 44 | 45 | url_md5 = results["Info"]["url"]["md5"] 46 | file_md5 = results["Info"]["file"]["md5"] 47 | jfile = url_md5 + "_" + file_md5 + ".html" 48 | 49 | if not os.path.exists(dumpdir + jfile): 50 | try: 51 | env = Environment(autoescape=True) 52 | env.loader = FileSystemLoader(os.path.join(RAGPICKER_ROOT, "data", "html")) 53 | template = env.get_template("report.html") 54 | reporthtml = template.render({"results" : results}) 55 | except UnicodeDecodeError: 56 | reporthtml = template.render({"results" : convertDirtyDict2ASCII(results)}) 57 | except Exception as e: 58 | raise Exception("Failed to generate HTML report: %s" % e) 59 | 60 | try: 61 | reportfile = codecs.open(os.path.join(dumpdir, jfile), "w", "utf-8") 62 | reportfile.write(reporthtml) 63 | reportfile.close() 64 | except (TypeError, IOError) as e: 65 | raise Exception("Failed to write HTML report: %s" % e) 66 | -------------------------------------------------------------------------------- /src/reporting/reporthtml.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = reporthtml 3 | Module = reporthtml 4 | 5 | [Documentation] 6 | Description = HTML Reporting-Modul 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/reporting/reportxml.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2015 Ragpicker Developers. 2 | # This file is part of Ragpicker Malware Crawler - http://code.google.com/p/malware-crawler/ 3 | # coding: utf-8 4 | 5 | import codecs 6 | import logging 7 | import os 8 | import tempfile 9 | 10 | from core.abstracts import Report 11 | from core.commonutils import convertDirtyDict2ASCII 12 | import utils.dict2xml as dict2xml 13 | 14 | 15 | try: 16 | from yapsy.IPlugin import IPlugin 17 | except ImportError: 18 | raise ImportError, 'Yapsy (Yet Another Plugin System) is required to run this program: http://yapsy.sourceforge.net' 19 | 20 | log = logging.getLogger(__name__) 21 | 22 | class ReportXML(IPlugin, Report): 23 | """Save report in XML format.""" 24 | 25 | def run(self, results, objfile): 26 | dumpdir = self.options.get("dumpdir", None) 27 | 28 | if not dumpdir: 29 | raise Exception("dumpdir not configured, skip") 30 | 31 | try: 32 | if not os.path.exists(dumpdir): 33 | os.makedirs(dumpdir) 34 | d = tempfile.mkdtemp(dir=dumpdir) 35 | except Exception as e: 36 | raise Exception('Could not open %s for writing (%s)', dumpdir, e) 37 | else: 38 | os.rmdir(d) 39 | 40 | url_md5 = results["Info"]["url"]["md5"] 41 | file_md5 = results["Info"]["file"]["md5"] 42 | jfile = url_md5 + "_" + file_md5 + ".xml" 43 | 44 | if not os.path.exists(dumpdir + jfile): 45 | try: 46 | reportxml = dict2xml.dicttoxml(results) 47 | except UnicodeDecodeError: 48 | reportxml = dict2xml.dicttoxml(convertDirtyDict2ASCII(results)) 49 | except Exception as e: 50 | raise Exception("Failed to generate XML report: %s" % e) 51 | 52 | try: 53 | 54 | reportfile = codecs.open(os.path.join(dumpdir, jfile), "w", "utf-8") 55 | reportfile.write(reportxml) 56 | reportfile.close() 57 | except (TypeError, IOError) as e: 58 | raise Exception("Failed to write XML report: %s" % e) 59 | 60 | -------------------------------------------------------------------------------- /src/reporting/reportxml.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = reportxml 3 | Module = reportxml 4 | 5 | [Documentation] 6 | Description = Store analysis results as xml document 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/reporting/vxcage.yapsy-plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = vxcage 3 | Module = vxcage 4 | 5 | [Documentation] 6 | Description = VxCage is a Python application for managing a malware samples repository 7 | Author = Robby Zeitfuchs 8 | Version = 1.0 -------------------------------------------------------------------------------- /src/statistics.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # _ _ 3 | # _ __ __ _ __ _ _ __ (_) ___ | | __ ___ _ __ 4 | # | '__| / _` | / _` || '_ \ | | / __|| |/ / / _ \| '__| 5 | # | | | (_| || (_| || |_) || || (__ | < | __/| | 6 | # |_| \__,_| \__, || .__/ |_| \___||_|\_\ \___||_| 7 | # |___/ |_| 8 | # 9 | # Plugin based malware crawler. 10 | # Use this tool if you are testing antivirus products, collecting malware 11 | # for another analyzer/zoo. 12 | # Many thanks to the cuckoo-sandbox team for the Architectural design ideas. 13 | # Includes code from cuckoo-sandbox (c) 2013 http://www.cuckoosandbox.org 14 | # and mwcrawler, (c) 2012 Ricardo Dias 15 | # 16 | # http://code.google.com/p/malware-crawler/ 17 | # 18 | # Robby Zeitfuchs - robby@zeitfuchs.org - 2013-2015 19 | # Mark Lawrenz - Mark.Lawrenz@web.de 20 | # 21 | # any subjection, tips, improvement are welcome 22 | # 23 | # Licence: GNU GPL v.3.0 24 | # 25 | # This program is free software; you can redistribute it and/or modify 26 | # it under the terms of the GNU General Public License as published by 27 | # the Free Software Foundation; either version 3 of the License, or 28 | # (at your option) any later version. 29 | # 30 | # This program is distributed in the hope that it will be useful, 31 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 32 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 33 | # GNU General Public License for more details. 34 | 35 | import argparse 36 | import logging 37 | 38 | from utils.logo import logo 39 | from core.statistics import Statistics 40 | 41 | log = logging.getLogger("Main") 42 | 43 | if __name__ == '__main__': 44 | 45 | parser = argparse.ArgumentParser(description='Ragpicker Statistics') 46 | subparsers = parser.add_subparsers(title='subcommands', description='valid subcommands', help='additional help') 47 | 48 | parser_long = subparsers.add_parser('long', help="Show statistics (long version)") 49 | parser_long.set_defaults(which='long') 50 | parser_short = subparsers.add_parser('short', help="Show statistics (short version)") 51 | parser_short.set_defaults(which='short') 52 | parser_av = subparsers.add_parser('av', help="Show statistics (AV version)") 53 | parser_av.set_defaults(which='av') 54 | 55 | args = vars(parser.parse_args()) 56 | 57 | logo() 58 | 59 | if args['which'] == 'long': 60 | Statistics().runStatisticsLong() 61 | elif args['which'] == 'short': 62 | Statistics().runStatisticsShort() 63 | elif args['which'] == 'av': 64 | Statistics().runStatisticsAV() -------------------------------------------------------------------------------- /src/utils/OfficeMalScanner/Cadt.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/robbyFux/Ragpicker/77d68ff3a13851ed4c256635f0673a4c2a84ef51/src/utils/OfficeMalScanner/Cadt.dll -------------------------------------------------------------------------------- /src/utils/OfficeMalScanner/DisView.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/robbyFux/Ragpicker/77d68ff3a13851ed4c256635f0673a4c2a84ef51/src/utils/OfficeMalScanner/DisView.exe -------------------------------------------------------------------------------- /src/utils/OfficeMalScanner/LZNT1Decompress.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/robbyFux/Ragpicker/77d68ff3a13851ed4c256635f0673a4c2a84ef51/src/utils/OfficeMalScanner/LZNT1Decompress.dll -------------------------------------------------------------------------------- /src/utils/OfficeMalScanner/MalHost-Setup.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/robbyFux/Ragpicker/77d68ff3a13851ed4c256635f0673a4c2a84ef51/src/utils/OfficeMalScanner/MalHost-Setup.exe -------------------------------------------------------------------------------- /src/utils/OfficeMalScanner/OfficeMalScanner.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/robbyFux/Ragpicker/77d68ff3a13851ed4c256635f0673a4c2a84ef51/src/utils/OfficeMalScanner/OfficeMalScanner.exe -------------------------------------------------------------------------------- /src/utils/OfficeMalScanner/RTFScan.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/robbyFux/Ragpicker/77d68ff3a13851ed4c256635f0673a4c2a84ef51/src/utils/OfficeMalScanner/RTFScan.exe -------------------------------------------------------------------------------- /src/utils/OfficeMalScanner/Unzipper.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/robbyFux/Ragpicker/77d68ff3a13851ed4c256635f0673a4c2a84ef51/src/utils/OfficeMalScanner/Unzipper.dll -------------------------------------------------------------------------------- /src/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/robbyFux/Ragpicker/77d68ff3a13851ed4c256635f0673a4c2a84ef51/src/utils/__init__.py -------------------------------------------------------------------------------- /src/utils/oletools/LICENSE.txt: -------------------------------------------------------------------------------- 1 | LICENSE for the python-oletools package: 2 | 3 | This license applies to the python-oletools package, apart from the thirdparty 4 | folder which contains third-party files published with their own license. 5 | 6 | The python-oletools package is copyright (c) 2012-2015 Philippe Lagadec (http://www.decalage.info) 7 | 8 | All rights reserved. 9 | 10 | Redistribution and use in source and binary forms, with or without modification, 11 | are permitted provided that the following conditions are met: 12 | 13 | * Redistributions of source code must retain the above copyright notice, this 14 | list of conditions and the following disclaimer. 15 | * Redistributions in binary form must reproduce the above copyright notice, 16 | this list of conditions and the following disclaimer in the documentation 17 | and/or other materials provided with the distribution. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 20 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 21 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 22 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 23 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 25 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 26 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 27 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | 31 | ---------- 32 | 33 | olevba contains modified source code from the officeparser project, published 34 | under the following MIT License (MIT): 35 | 36 | officeparser is copyright (c) 2014 John William Davison 37 | 38 | Permission is hereby granted, free of charge, to any person obtaining a copy 39 | of this software and associated documentation files (the "Software"), to deal 40 | in the Software without restriction, including without limitation the rights 41 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 42 | copies of the Software, and to permit persons to whom the Software is 43 | furnished to do so, subject to the following conditions: 44 | 45 | The above copyright notice and this permission notice shall be included in all 46 | copies or substantial portions of the Software. 47 | 48 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 49 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 50 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 51 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 52 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 53 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 54 | SOFTWARE. 55 | -------------------------------------------------------------------------------- /src/utils/oletools/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/robbyFux/Ragpicker/77d68ff3a13851ed4c256635f0673a4c2a84ef51/src/utils/oletools/__init__.py -------------------------------------------------------------------------------- /src/utils/oletools/olemeta.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | olemeta.py 4 | 5 | olemeta is a script to parse OLE files such as MS Office documents (e.g. Word, 6 | Excel), to extract all standard properties present in the OLE file. 7 | 8 | Usage: olemeta.py 9 | 10 | olemeta project website: http://www.decalage.info/python/olemeta 11 | 12 | olemeta is part of the python-oletools package: 13 | http://www.decalage.info/python/oletools 14 | """ 15 | 16 | #=== LICENSE ================================================================= 17 | 18 | # olemeta is copyright (c) 2013-2014, Philippe Lagadec (http://www.decalage.info) 19 | # All rights reserved. 20 | # 21 | # Redistribution and use in source and binary forms, with or without modification, 22 | # are permitted provided that the following conditions are met: 23 | # 24 | # * Redistributions of source code must retain the above copyright notice, this 25 | # list of conditions and the following disclaimer. 26 | # * Redistributions in binary form must reproduce the above copyright notice, 27 | # this list of conditions and the following disclaimer in the documentation 28 | # and/or other materials provided with the distribution. 29 | # 30 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 31 | # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 32 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 33 | # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 34 | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 35 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 36 | # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 37 | # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 38 | # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 39 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 40 | 41 | #------------------------------------------------------------------------------ 42 | # CHANGELOG: 43 | # 2013-07-24 v0.01 PL: - first version 44 | # 2014-11-29 v0.02 PL: - use olefile instead of OleFileIO_PL 45 | # - improved usage display 46 | 47 | __version__ = '0.02' 48 | 49 | #------------------------------------------------------------------------------ 50 | # TODO: 51 | # + optparse 52 | # + nicer output: table with fixed columns, datetime, etc 53 | # + CSV output 54 | # + option to only show available properties (by default) 55 | 56 | #=== IMPORTS ================================================================= 57 | 58 | import sys 59 | import thirdparty.olefile as olefile 60 | 61 | 62 | #=== MAIN ================================================================= 63 | 64 | try: 65 | ole = olefile.OleFileIO(sys.argv[1]) 66 | except IndexError: 67 | sys.exit(__doc__) 68 | 69 | # parse and display metadata: 70 | meta = ole.get_metadata() 71 | meta.dump() 72 | 73 | ole.close() 74 | -------------------------------------------------------------------------------- /src/utils/oletools/thirdparty/OleFileIO_PL/LICENSE.txt: -------------------------------------------------------------------------------- 1 | LICENSE for the OleFileIO_PL module: 2 | 3 | 4 | OleFileIO_PL is an improved version of the OleFileIO module from the 5 | Python Imaging Library (PIL). 6 | 7 | OleFileIO_PL changes are Copyright (c) 2005-2013 by Philippe Lagadec 8 | 9 | The Python Imaging Library (PIL) is 10 | Copyright (c) 1997-2005 by Secret Labs AB 11 | Copyright (c) 1995-2005 by Fredrik Lundh 12 | 13 | By obtaining, using, and/or copying this software and/or its associated 14 | documentation, you agree that you have read, understood, and will comply with 15 | the following terms and conditions: 16 | 17 | Permission to use, copy, modify, and distribute this software and its 18 | associated documentation for any purpose and without fee is hereby granted, 19 | provided that the above copyright notice appears in all copies, and that both 20 | that copyright notice and this permission notice appear in supporting 21 | documentation, and that the name of Secret Labs AB or the author(s) not be used 22 | in advertising or publicity pertaining to distribution of the software 23 | without specific, written prior permission. 24 | 25 | SECRET LABS AB AND THE AUTHORS DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS 26 | SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. 27 | IN NO EVENT SHALL SECRET LABS AB OR THE AUTHORS BE LIABLE FOR ANY SPECIAL, 28 | INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 29 | LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR 30 | OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 31 | PERFORMANCE OF THIS SOFTWARE. 32 | -------------------------------------------------------------------------------- /src/utils/oletools/thirdparty/OleFileIO_PL/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/robbyFux/Ragpicker/77d68ff3a13851ed4c256635f0673a4c2a84ef51/src/utils/oletools/thirdparty/OleFileIO_PL/__init__.py -------------------------------------------------------------------------------- /src/utils/oletools/thirdparty/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/robbyFux/Ragpicker/77d68ff3a13851ed4c256635f0673a4c2a84ef51/src/utils/oletools/thirdparty/__init__.py -------------------------------------------------------------------------------- /src/utils/oletools/thirdparty/easygui/LICENSE.txt: -------------------------------------------------------------------------------- 1 | LICENSE INFORMATION 2 | 3 | EasyGui version 0.96 4 | 5 | Copyright (c) 2010, Stephen Raymond Ferg 6 | 7 | All rights reserved. 8 | 9 | Redistribution and use in source and binary forms, with or without modification, 10 | are permitted provided that the following conditions are met: 11 | 12 | 1. Redistributions of source code must retain the above copyright notice, 13 | this list of conditions and the following disclaimer. 14 | 15 | 2. Redistributions in binary form must reproduce the above copyright notice, 16 | this list of conditions and the following disclaimer in the documentation and/or 17 | other materials provided with the distribution. 18 | 19 | 3. The name of the author may not be used to endorse or promote products derived 20 | from this software without specific prior written permission. 21 | 22 | THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" 23 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 24 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 | ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 26 | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 27 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 | HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 30 | STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 31 | IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 32 | EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | -------------------------------------------------------------------------------- /src/utils/oletools/thirdparty/easygui/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/robbyFux/Ragpicker/77d68ff3a13851ed4c256635f0673a4c2a84ef51/src/utils/oletools/thirdparty/easygui/__init__.py -------------------------------------------------------------------------------- /src/utils/oletools/thirdparty/olefile/CONTRIBUTORS.txt: -------------------------------------------------------------------------------- 1 | CONTRIBUTORS for the olefile project 2 | ==================================== 3 | 4 | This is a non-exhaustive list of all the people who helped me improve the 5 | olefile project (formerly OleFileIO_PL), in approximative chronological order. 6 | Please contact me if I forgot to mention your name. 7 | 8 | A big thank you to all of them: 9 | 10 | - Niko Ehrenfeuchter: added support for Jython 11 | - Niko Ehrenfeuchter, Martijn Berger and Dave Jones: helped fix 4K sector support 12 | - Martin Panter: conversion to Python 3.x/2.6+ 13 | - mete0r_kr: added support for file-like objects 14 | - chuckleberryfinn: fixed bug in getproperties 15 | - Martijn, Ben G.: bug report for 64 bits platforms 16 | - Philippe Lagadec: main author and maintainer since 2005 17 | - and of course Fredrik Lundh: original author of OleFileIO from 1995 to 2005 18 | -------------------------------------------------------------------------------- /src/utils/oletools/thirdparty/olefile/LICENSE.txt: -------------------------------------------------------------------------------- 1 | LICENSE for the olefile package: 2 | 3 | olefile (formerly OleFileIO_PL) is copyright (c) 2005-2014 Philippe Lagadec 4 | (http://www.decalage.info) 5 | 6 | All rights reserved. 7 | 8 | Redistribution and use in source and binary forms, with or without modification, 9 | are permitted provided that the following conditions are met: 10 | 11 | * Redistributions of source code must retain the above copyright notice, this 12 | list of conditions and the following disclaimer. 13 | * Redistributions in binary form must reproduce the above copyright notice, 14 | this list of conditions and the following disclaimer in the documentation 15 | and/or other materials provided with the distribution. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 18 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 20 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 21 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 23 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 24 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 25 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | 28 | 29 | ---------- 30 | 31 | olefile is based on source code from the OleFileIO module of the Python 32 | Imaging Library (PIL) published by Fredrik Lundh under the following license: 33 | 34 | The Python Imaging Library (PIL) is 35 | - Copyright (c) 1997-2005 by Secret Labs AB 36 | - Copyright (c) 1995-2005 by Fredrik Lundh 37 | 38 | By obtaining, using, and/or copying this software and/or its associated 39 | documentation, you agree that you have read, understood, and will comply with 40 | the following terms and conditions: 41 | 42 | Permission to use, copy, modify, and distribute this software and its 43 | associated documentation for any purpose and without fee is hereby granted, 44 | provided that the above copyright notice appears in all copies, and that both 45 | that copyright notice and this permission notice appear in supporting 46 | documentation, and that the name of Secret Labs AB or the author not be used 47 | in advertising or publicity pertaining to distribution of the software without 48 | specific, written prior permission. 49 | 50 | SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS 51 | SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN 52 | NO EVENT SHALL SECRET LABS AB OR THE AUTHOR BE LIABLE FOR ANY SPECIAL, 53 | INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 54 | LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR 55 | OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 56 | PERFORMANCE OF THIS SOFTWARE. 57 | -------------------------------------------------------------------------------- /src/utils/oletools/thirdparty/olefile/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python 2 | # -*- coding: latin-1 -*- 3 | """ 4 | olefile (formerly OleFileIO_PL) 5 | 6 | Module to read/write Microsoft OLE2 files (also called Structured Storage or 7 | Microsoft Compound Document File Format), such as Microsoft Office 97-2003 8 | documents, Image Composer and FlashPix files, Outlook messages, ... 9 | This version is compatible with Python 2.6+ and 3.x 10 | 11 | Project website: http://www.decalage.info/olefile 12 | 13 | olefile is copyright (c) 2005-2014 Philippe Lagadec (http://www.decalage.info) 14 | 15 | olefile is based on the OleFileIO module from the PIL library v1.1.6 16 | See: http://www.pythonware.com/products/pil/index.htm 17 | 18 | The Python Imaging Library (PIL) is 19 | Copyright (c) 1997-2005 by Secret Labs AB 20 | Copyright (c) 1995-2005 by Fredrik Lundh 21 | 22 | See source code and LICENSE.txt for information on usage and redistribution. 23 | """ 24 | 25 | try: 26 | # first try to import olefile for Python 2.6+/3.x 27 | from .olefile import * 28 | # import metadata not covered by *: 29 | from .olefile import __version__, __author__, __date__ 30 | 31 | except: 32 | # if it fails, fallback to the old version olefile2 for Python 2.x: 33 | from .olefile2 import * 34 | # import metadata not covered by *: 35 | from .olefile2 import __doc__, __version__, __author__, __date__ 36 | -------------------------------------------------------------------------------- /src/utils/oletools/thirdparty/olefile/doc/Contribute.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 |

How to Suggest Improvements, Report Issues or Contribute

11 |

This is a personal open-source project, developed on my spare time. Any contribution, suggestion, feedback or bug report is welcome.

12 |

To suggest improvements, report a bug or any issue, please use the issue reporting page, providing all the information and files to reproduce the problem.

13 |

If possible please join the debugging output of olefile. For this, launch the following command :

14 | 
    olefile.py -d -c file >debug.txt
15 |

You may also contact the author directly to provide feedback.

16 |

The code is available in a Mercurial repository on Bitbucket. You may use it to submit enhancements using forks and pull requests.

17 |
18 |

olefile documentation

19 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /src/utils/oletools/thirdparty/olefile/doc/Contribute.md: -------------------------------------------------------------------------------- 1 | How to Suggest Improvements, Report Issues or Contribute 2 | ======================================================== 3 | 4 | This is a personal open-source project, developed on my spare time. Any contribution, suggestion, feedback or bug report is welcome. 5 | 6 | To **suggest improvements, report a bug or any issue**, please use the [issue reporting page](https://bitbucket.org/decalage/olefileio_pl/issues?status=new&status=open), providing all the information and files to reproduce the problem. 7 | 8 | If possible please join the debugging output of olefile. For this, launch the following command : 9 | 10 | :::text 11 | olefile.py -d -c file >debug.txt 12 | 13 | 14 | You may also [contact the author](http://decalage.info/contact) directly to **provide feedback**. 15 | 16 | The code is available in [a Mercurial repository on Bitbucket](https://bitbucket.org/decalage/olefileio_pl). You may use it to **submit enhancements** using forks and pull requests. 17 | 18 | -------------------------------------------------------------------------- 19 | 20 | olefile documentation 21 | --------------------- 22 | 23 | - [[Home]] 24 | - [[License]] 25 | - [[Install]] 26 | - [[Contribute]], Suggest Improvements or Report Issues 27 | - [[OLE_Overview]] 28 | - [[API]] and Usage 29 | -------------------------------------------------------------------------------- /src/utils/oletools/thirdparty/olefile/doc/Install.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 |

How to Download and Install olefile

11 |

Pre-requisites

12 |

olefile requires Python 2.6, 2.7 or 3.x.

13 |

For Python 2.5 and older, olefile falls back to an older version (based on OleFileIO_PL 0.26) which might not contain all the enhancements implemented in olefile.

14 |

Download and Install

15 |

To use olefile with other Python applications or your own scripts, the simplest solution is to run "pip install olefile" or "easy_install olefile" to download and install the package in one go.

16 |

Otherwise you may download/extract the zip archive in a temporary directory and run "python setup.py install".

17 |

On Windows you may simply double-click on install.bat.

18 |
19 |

olefile documentation

20 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /src/utils/oletools/thirdparty/olefile/doc/Install.md: -------------------------------------------------------------------------------- 1 | How to Download and Install olefile 2 | =================================== 3 | 4 | Pre-requisites 5 | -------------- 6 | 7 | olefile requires Python 2.6, 2.7 or 3.x. 8 | 9 | For Python 2.5 and older, olefile falls back to an older version (based on OleFileIO_PL 0.26) which might not contain all the enhancements implemented in olefile. 10 | 11 | 12 | Download and Install 13 | -------------------- 14 | 15 | To use olefile with other Python applications or your own scripts, the simplest solution is to run "**pip install olefile**" or "**easy_install olefile**" to download and install the package in one go. 16 | 17 | Otherwise you may download/extract the [zip archive](https://bitbucket.org/decalage/olefileio_pl/downloads) in a temporary directory and run "**python setup.py install**". 18 | 19 | On Windows you may simply double-click on **install.bat**. 20 | 21 | -------------------------------------------------------------------------- 22 | 23 | olefile documentation 24 | --------------------- 25 | 26 | - [[Home]] 27 | - [[License]] 28 | - [[Install]] 29 | - [[Contribute]], Suggest Improvements or Report Issues 30 | - [[OLE_Overview]] 31 | - [[API]] and Usage 32 | -------------------------------------------------------------------------------- /src/utils/oletools/thirdparty/olefile/doc/License.md: -------------------------------------------------------------------------------- 1 | License for olefile 2 | =================== 3 | 4 | olefile (formerly OleFileIO_PL) is copyright (c) 2005-2014 Philippe Lagadec ([http://www.decalage.info](http://www.decalage.info)) 5 | 6 | All rights reserved. 7 | 8 | Redistribution and use in source and binary forms, with or without modification, 9 | are permitted provided that the following conditions are met: 10 | 11 | * Redistributions of source code must retain the above copyright notice, this 12 | list of conditions and the following disclaimer. 13 | * Redistributions in binary form must reproduce the above copyright notice, 14 | this list of conditions and the following disclaimer in the documentation 15 | and/or other materials provided with the distribution. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 18 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 20 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 21 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 23 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 24 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 25 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | 28 | 29 | ---------- 30 | 31 | olefile is based on source code from the OleFileIO module of the Python Imaging Library (PIL) published by Fredrik Lundh under the following license: 32 | 33 | The Python Imaging Library (PIL) is 34 | 35 | - Copyright (c) 1997-2005 by Secret Labs AB 36 | - Copyright (c) 1995-2005 by Fredrik Lundh 37 | 38 | By obtaining, using, and/or copying this software and/or its associated documentation, you agree that you have read, understood, and will comply with the following terms and conditions: 39 | 40 | Permission to use, copy, modify, and distribute this software and its associated documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appears in all copies, and that both that copyright notice and this permission notice appear in supporting documentation, and that the name of Secret Labs AB or the author not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. 41 | 42 | SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 43 | 44 | -------------------------------------------------------------------------- 45 | 46 | olefile documentation 47 | --------------------- 48 | 49 | - [[Home]] 50 | - [[License]] 51 | - [[Install]] 52 | - [[Contribute]], Suggest Improvements or Report Issues 53 | - [[OLE_Overview]] 54 | - [[API]] and Usage 55 | -------------------------------------------------------------------------------- /src/utils/oletools/thirdparty/olefile/doc/OLE_Overview.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 |

About the structure of OLE files

11 |

This page is part of the documentation for olefile. It provides a brief overview of the structure of Microsoft OLE2 files (also called Structured Storage, Compound File Binary Format or Compound Document File Format), such as Microsoft Office 97-2003 documents, Image Composer and FlashPix files, Outlook messages, StickyNotes, several Microscopy file formats, McAfee antivirus quarantine files, etc.

12 |

An OLE file can be seen as a mini file system or a Zip archive: It contains streams of data that look like files embedded within the OLE file. Each stream has a name. For example, the main stream of a MS Word document containing its text is named "WordDocument".

13 |

An OLE file can also contain storages. A storage is a folder that contains streams or other storages. For example, a MS Word document with VBA macros has a storage called "Macros".

14 |

Special streams can contain properties. A property is a specific value that can be used to store information such as the metadata of a document (title, author, creation date, etc). Property stream names usually start with the character '05'.

15 |

For example, a typical MS Word document may look like this:

16 |
17 |

18 |
19 |

Go to the API page to see how to use all olefile features to parse OLE files.

20 |
21 |

olefile documentation

22 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /src/utils/oletools/thirdparty/olefile/doc/OLE_Overview.md: -------------------------------------------------------------------------------- 1 | About the structure of OLE files 2 | ================================ 3 | 4 | This page is part of the documentation for [olefile](https://bitbucket.org/decalage/olefileio_pl/wiki). It provides a brief overview of the structure of [Microsoft OLE2 files (also called Structured Storage, Compound File Binary Format or Compound Document File Format)](http://en.wikipedia.org/wiki/Compound_File_Binary_Format), such as Microsoft Office 97-2003 documents, Image Composer and FlashPix files, Outlook messages, StickyNotes, several Microscopy file formats, McAfee antivirus quarantine files, etc. 5 | 6 | An OLE file can be seen as a mini file system or a Zip archive: It contains **streams** of data that look like files embedded within the OLE file. Each stream has a name. For example, the main stream of a MS Word document containing its text is named "WordDocument". 7 | 8 | An OLE file can also contain **storages**. A storage is a folder that contains streams or other storages. For example, a MS Word document with VBA macros has a storage called "Macros". 9 | 10 | Special streams can contain **properties**. A property is a specific value that can be used to store information such as the metadata of a document (title, author, creation date, etc). Property stream names usually start with the character '\x05'. 11 | 12 | For example, a typical MS Word document may look like this: 13 | 14 | ![](OLE_VBA_sample.png) 15 | 16 | Go to the [[API]] page to see how to use all olefile features to parse OLE files. 17 | 18 | 19 | -------------------------------------------------------------------------- 20 | 21 | olefile documentation 22 | --------------------- 23 | 24 | - [[Home]] 25 | - [[License]] 26 | - [[Install]] 27 | - [[Contribute]], Suggest Improvements or Report Issues 28 | - [[OLE_Overview]] 29 | - [[API]] and Usage 30 | -------------------------------------------------------------------------------- /src/utils/oletools/thirdparty/olefile/doc/OLE_VBA_sample.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/robbyFux/Ragpicker/77d68ff3a13851ed4c256635f0673a4c2a84ef51/src/utils/oletools/thirdparty/olefile/doc/OLE_VBA_sample.png -------------------------------------------------------------------------------- /src/utils/oletools/thirdparty/prettytable/COPYING: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2009-2013 Luke Maurits 2 | # All rights reserved. 3 | # With contributions from: 4 | # * Chris Clark 5 | # * Christoph Robbert 6 | # * Klein Stephane 7 | # * "maartendb" 8 | # 9 | # Redistribution and use in source and binary forms, with or without 10 | # modification, are permitted provided that the following conditions are met: 11 | # 12 | # * Redistributions of source code must retain the above copyright notice, 13 | # this list of conditions and the following disclaimer. 14 | # * Redistributions in binary form must reproduce the above copyright notice, 15 | # this list of conditions and the following disclaimer in the documentation 16 | # and/or other materials provided with the distribution. 17 | # * The name of the author may not be used to endorse or promote products 18 | # derived from this software without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 | # POSSIBILITY OF SUCH DAMAGE. 31 | -------------------------------------------------------------------------------- /src/utils/oletools/thirdparty/prettytable/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/robbyFux/Ragpicker/77d68ff3a13851ed4c256635f0673a4c2a84ef51/src/utils/oletools/thirdparty/prettytable/__init__.py -------------------------------------------------------------------------------- /src/utils/oletools/thirdparty/xglob/LICENSE.txt: -------------------------------------------------------------------------------- 1 | LICENSE: 2 | 3 | xglob is copyright (c) 2013-2015, Philippe Lagadec (http://www.decalage.info) 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without modification, 7 | are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | * Redistributions in binary form must reproduce the above copyright notice, 12 | this list of conditions and the following disclaimer in the documentation 13 | and/or other materials provided with the distribution. 14 | 15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 16 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 17 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 19 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 22 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 23 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 24 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | -------------------------------------------------------------------------------- /src/utils/oletools/thirdparty/xglob/__init__.py: -------------------------------------------------------------------------------- 1 | from .xglob import * -------------------------------------------------------------------------------- /src/utils/oletools/thirdparty/xxxswf/LICENSE.txt: -------------------------------------------------------------------------------- 1 | xxxswf.py is published by Alexander Hanel on 2 | http://hooked-on-mnemonics.blogspot.nl/2011/12/xxxswfpy.html 3 | without explicit license. -------------------------------------------------------------------------------- /src/utils/oletools/thirdparty/xxxswf/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/robbyFux/Ragpicker/77d68ff3a13851ed4c256635f0673a4c2a84ef51/src/utils/oletools/thirdparty/xxxswf/__init__.py -------------------------------------------------------------------------------- /src/utils/ordlookup/__init__.py: -------------------------------------------------------------------------------- 1 | import ws2_32 2 | import oleaut32 3 | 4 | ''' 5 | A small module for keeping a database of ordinal to symbol 6 | mappings for DLLs which frequently get linked without symbolic 7 | infoz. 8 | ''' 9 | 10 | ords = { 11 | 'ws2_32.dll':ws2_32.ord_names, 12 | 'wsock32.dll':ws2_32.ord_names, 13 | 'oleaut32.dll':oleaut32.ord_names, 14 | } 15 | 16 | def ordLookup(libname, ord, make_name=False): 17 | ''' 18 | Lookup a name for the given ordinal if it's in our 19 | database. 20 | ''' 21 | names = ords.get(libname.lower()) 22 | if names == None: 23 | if make_name is True: 24 | return 'ord%d' % ord 25 | return None 26 | name = names.get(ord) 27 | if name == None: 28 | return 'ord%d' % ord 29 | return name 30 | 31 | 32 | -------------------------------------------------------------------------------- /src/utils/ordlookup/ws2_32.py: -------------------------------------------------------------------------------- 1 | 2 | ord_names = { 3 | 1:'accept', 4 | 2:'bind', 5 | 3:'closesocket', 6 | 4:'connect', 7 | 5:'getpeername', 8 | 6:'getsockname', 9 | 7:'getsockopt', 10 | 8:'htonl', 11 | 9:'htons', 12 | 10:'ioctlsocket', 13 | 11:'inet_addr', 14 | 12:'inet_ntoa', 15 | 13:'listen', 16 | 14:'ntohl', 17 | 15:'ntohs', 18 | 16:'recv', 19 | 17:'recvfrom', 20 | 18:'select', 21 | 19:'send', 22 | 20:'sendto', 23 | 21:'setsockopt', 24 | 22:'shutdown', 25 | 23:'socket', 26 | 24:'GetAddrInfoW', 27 | 25:'GetNameInfoW', 28 | 26:'WSApSetPostRoutine', 29 | 27:'FreeAddrInfoW', 30 | 28:'WPUCompleteOverlappedRequest', 31 | 29:'WSAAccept', 32 | 30:'WSAAddressToStringA', 33 | 31:'WSAAddressToStringW', 34 | 32:'WSACloseEvent', 35 | 33:'WSAConnect', 36 | 34:'WSACreateEvent', 37 | 35:'WSADuplicateSocketA', 38 | 36:'WSADuplicateSocketW', 39 | 37:'WSAEnumNameSpaceProvidersA', 40 | 38:'WSAEnumNameSpaceProvidersW', 41 | 39:'WSAEnumNetworkEvents', 42 | 40:'WSAEnumProtocolsA', 43 | 41:'WSAEnumProtocolsW', 44 | 42:'WSAEventSelect', 45 | 43:'WSAGetOverlappedResult', 46 | 44:'WSAGetQOSByName', 47 | 45:'WSAGetServiceClassInfoA', 48 | 46:'WSAGetServiceClassInfoW', 49 | 47:'WSAGetServiceClassNameByClassIdA', 50 | 48:'WSAGetServiceClassNameByClassIdW', 51 | 49:'WSAHtonl', 52 | 50:'WSAHtons', 53 | 51:'gethostbyaddr', 54 | 52:'gethostbyname', 55 | 53:'getprotobyname', 56 | 54:'getprotobynumber', 57 | 55:'getservbyname', 58 | 56:'getservbyport', 59 | 57:'gethostname', 60 | 58:'WSAInstallServiceClassA', 61 | 59:'WSAInstallServiceClassW', 62 | 60:'WSAIoctl', 63 | 61:'WSAJoinLeaf', 64 | 62:'WSALookupServiceBeginA', 65 | 63:'WSALookupServiceBeginW', 66 | 64:'WSALookupServiceEnd', 67 | 65:'WSALookupServiceNextA', 68 | 66:'WSALookupServiceNextW', 69 | 67:'WSANSPIoctl', 70 | 68:'WSANtohl', 71 | 69:'WSANtohs', 72 | 70:'WSAProviderConfigChange', 73 | 71:'WSARecv', 74 | 72:'WSARecvDisconnect', 75 | 73:'WSARecvFrom', 76 | 74:'WSARemoveServiceClass', 77 | 75:'WSAResetEvent', 78 | 76:'WSASend', 79 | 77:'WSASendDisconnect', 80 | 78:'WSASendTo', 81 | 79:'WSASetEvent', 82 | 80:'WSASetServiceA', 83 | 81:'WSASetServiceW', 84 | 82:'WSASocketA', 85 | 83:'WSASocketW', 86 | 84:'WSAStringToAddressA', 87 | 85:'WSAStringToAddressW', 88 | 86:'WSAWaitForMultipleEvents', 89 | 87:'WSCDeinstallProvider', 90 | 88:'WSCEnableNSProvider', 91 | 89:'WSCEnumProtocols', 92 | 90:'WSCGetProviderPath', 93 | 91:'WSCInstallNameSpace', 94 | 92:'WSCInstallProvider', 95 | 93:'WSCUnInstallNameSpace', 96 | 94:'WSCUpdateProvider', 97 | 95:'WSCWriteNameSpaceOrder', 98 | 96:'WSCWriteProviderOrder', 99 | 97:'freeaddrinfo', 100 | 98:'getaddrinfo', 101 | 99:'getnameinfo', 102 | 101:'WSAAsyncSelect', 103 | 102:'WSAAsyncGetHostByAddr', 104 | 103:'WSAAsyncGetHostByName', 105 | 104:'WSAAsyncGetProtoByNumber', 106 | 105:'WSAAsyncGetProtoByName', 107 | 106:'WSAAsyncGetServByPort', 108 | 107:'WSAAsyncGetServByName', 109 | 108:'WSACancelAsyncRequest', 110 | 109:'WSASetBlockingHook', 111 | 110:'WSAUnhookBlockingHook', 112 | 111:'WSAGetLastError', 113 | 112:'WSASetLastError', 114 | 113:'WSACancelBlockingCall', 115 | 114:'WSAIsBlocking', 116 | 115:'WSAStartup', 117 | 116:'WSACleanup', 118 | 151:'__WSAFDIsSet', 119 | 500:'WEP', 120 | } 121 | -------------------------------------------------------------------------------- /src/utils/pyasn1/__init__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | # http://www.python.org/dev/peps/pep-0396/ 4 | __version__ = '0.1.4' 5 | 6 | if sys.version_info[:2] < (2, 4): 7 | raise RuntimeError('PyASN1 requires Python 2.4 or later') 8 | 9 | -------------------------------------------------------------------------------- /src/utils/pyasn1/codec/__init__.py: -------------------------------------------------------------------------------- 1 | # This file is necessary to make this directory a package. 2 | -------------------------------------------------------------------------------- /src/utils/pyasn1/codec/ber/__init__.py: -------------------------------------------------------------------------------- 1 | # This file is necessary to make this directory a package. 2 | -------------------------------------------------------------------------------- /src/utils/pyasn1/codec/ber/eoo.py: -------------------------------------------------------------------------------- 1 | from utils.pyasn1.type import base, tag 2 | 3 | class EndOfOctets(base.AbstractSimpleAsn1Item): 4 | defaultValue = 0 5 | tagSet = tag.initTagSet( 6 | tag.Tag(tag.tagClassUniversal, tag.tagFormatSimple, 0x00) 7 | ) 8 | endOfOctets = EndOfOctets() 9 | -------------------------------------------------------------------------------- /src/utils/pyasn1/codec/cer/__init__.py: -------------------------------------------------------------------------------- 1 | # This file is necessary to make this directory a package. 2 | -------------------------------------------------------------------------------- /src/utils/pyasn1/codec/cer/decoder.py: -------------------------------------------------------------------------------- 1 | # CER decoder 2 | from utils.pyasn1.type import univ 3 | from utils.pyasn1.codec.ber import decoder 4 | from utils.pyasn1.compat.octets import oct2int 5 | from utils.pyasn1 import error 6 | 7 | class BooleanDecoder(decoder.AbstractSimpleDecoder): 8 | protoComponent = univ.Boolean(0) 9 | def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, length, 10 | state, decodeFun, substrateFun): 11 | head, tail = substrate[:length], substrate[length:] 12 | if not head: 13 | raise error.PyAsn1Error('Empty substrate') 14 | byte = oct2int(head[0]) 15 | # CER/DER specifies encoding of TRUE as 0xFF and FALSE as 0x0, while 16 | # BER allows any non-zero value as TRUE; cf. sections 8.2.2. and 11.1 17 | # in http://www.itu.int/ITU-T/studygroups/com17/languages/X.690-0207.pdf 18 | if byte == 0xff: 19 | value = 1 20 | elif byte == 0x00: 21 | value = 0 22 | else: 23 | raise error.PyAsn1Error('Boolean CER violation: %s' % byte) 24 | return self._createComponent(asn1Spec, tagSet, value), tail 25 | 26 | tagMap = decoder.tagMap.copy() 27 | tagMap.update({ 28 | univ.Boolean.tagSet: BooleanDecoder() 29 | }) 30 | 31 | typeMap = decoder.typeMap 32 | 33 | class Decoder(decoder.Decoder): pass 34 | 35 | decode = Decoder(tagMap, decoder.typeMap) 36 | -------------------------------------------------------------------------------- /src/utils/pyasn1/codec/cer/encoder.py: -------------------------------------------------------------------------------- 1 | # CER encoder 2 | from utils.pyasn1.type import univ 3 | from utils.pyasn1.codec.ber import encoder 4 | from utils.pyasn1.compat.octets import int2oct, null 5 | 6 | class BooleanEncoder(encoder.IntegerEncoder): 7 | def encodeValue(self, encodeFun, client, defMode, maxChunkSize): 8 | if client == 0: 9 | substrate = int2oct(0) 10 | else: 11 | substrate = int2oct(255) 12 | return substrate, 0 13 | 14 | class BitStringEncoder(encoder.BitStringEncoder): 15 | def encodeValue(self, encodeFun, client, defMode, maxChunkSize): 16 | return encoder.BitStringEncoder.encodeValue( 17 | self, encodeFun, client, defMode, 1000 18 | ) 19 | 20 | class OctetStringEncoder(encoder.OctetStringEncoder): 21 | def encodeValue(self, encodeFun, client, defMode, maxChunkSize): 22 | return encoder.OctetStringEncoder.encodeValue( 23 | self, encodeFun, client, defMode, 1000 24 | ) 25 | 26 | # specialized RealEncoder here 27 | # specialized GeneralStringEncoder here 28 | # specialized GeneralizedTimeEncoder here 29 | # specialized UTCTimeEncoder here 30 | 31 | class SetOfEncoder(encoder.SequenceOfEncoder): 32 | def encodeValue(self, encodeFun, client, defMode, maxChunkSize): 33 | if isinstance(client, univ.SequenceAndSetBase): 34 | client.setDefaultComponents() 35 | client.verifySizeSpec() 36 | substrate = null; idx = len(client) 37 | # This is certainly a hack but how else do I distinguish SetOf 38 | # from Set if they have the same tags&constraints? 39 | if isinstance(client, univ.SequenceAndSetBase): 40 | # Set 41 | comps = [] 42 | while idx > 0: 43 | idx = idx - 1 44 | if client[idx] is None: # Optional component 45 | continue 46 | if client.getDefaultComponentByPosition(idx) == client[idx]: 47 | continue 48 | comps.append(client[idx]) 49 | comps.sort(key=lambda x: isinstance(x, univ.Choice) and \ 50 | x.getMinTagSet() or x.getTagSet()) 51 | for c in comps: 52 | substrate += encodeFun(c, defMode, maxChunkSize) 53 | else: 54 | # SetOf 55 | compSubs = [] 56 | while idx > 0: 57 | idx = idx - 1 58 | compSubs.append( 59 | encodeFun(client[idx], defMode, maxChunkSize) 60 | ) 61 | compSubs.sort() # perhaps padding's not needed 62 | substrate = null 63 | for compSub in compSubs: 64 | substrate += compSub 65 | return substrate, 1 66 | 67 | tagMap = encoder.tagMap.copy() 68 | tagMap.update({ 69 | univ.Boolean.tagSet: BooleanEncoder(), 70 | univ.BitString.tagSet: BitStringEncoder(), 71 | univ.OctetString.tagSet: OctetStringEncoder(), 72 | univ.SetOf().tagSet: SetOfEncoder() # conflcts with Set 73 | }) 74 | 75 | typeMap = encoder.typeMap.copy() 76 | typeMap.update({ 77 | univ.Set.typeId: SetOfEncoder(), 78 | univ.SetOf.typeId: SetOfEncoder() 79 | }) 80 | 81 | class Encoder(encoder.Encoder): 82 | def __call__(self, client, defMode=0, maxChunkSize=0): 83 | return encoder.Encoder.__call__(self, client, defMode, maxChunkSize) 84 | 85 | encode = Encoder(tagMap, typeMap) 86 | 87 | # EncoderFactory queries class instance and builds a map of tags -> encoders 88 | -------------------------------------------------------------------------------- /src/utils/pyasn1/codec/der/__init__.py: -------------------------------------------------------------------------------- 1 | # This file is necessary to make this directory a package. 2 | -------------------------------------------------------------------------------- /src/utils/pyasn1/codec/der/decoder.py: -------------------------------------------------------------------------------- 1 | # DER decoder 2 | from utils.pyasn1.type import univ 3 | from utils.pyasn1.codec.cer import decoder 4 | 5 | tagMap = decoder.tagMap 6 | typeMap = decoder.typeMap 7 | Decoder = decoder.Decoder 8 | 9 | decode = Decoder(tagMap, typeMap) 10 | -------------------------------------------------------------------------------- /src/utils/pyasn1/codec/der/encoder.py: -------------------------------------------------------------------------------- 1 | # DER encoder 2 | from utils.pyasn1.type import univ 3 | from utils.pyasn1.codec.cer import encoder 4 | 5 | class SetOfEncoder(encoder.SetOfEncoder): 6 | def _cmpSetComponents(self, c1, c2): 7 | tagSet1 = isinstance(c1, univ.Choice) and \ 8 | c1.getEffectiveTagSet() or c1.getTagSet() 9 | tagSet2 = isinstance(c2, univ.Choice) and \ 10 | c2.getEffectiveTagSet() or c2.getTagSet() 11 | return cmp(tagSet1, tagSet2) 12 | 13 | tagMap = encoder.tagMap.copy() 14 | tagMap.update({ 15 | # Overload CER encodrs with BER ones (a bit hackerish XXX) 16 | univ.BitString.tagSet: encoder.encoder.BitStringEncoder(), 17 | univ.OctetString.tagSet: encoder.encoder.OctetStringEncoder(), 18 | # Set & SetOf have same tags 19 | univ.SetOf().tagSet: SetOfEncoder() 20 | }) 21 | 22 | typeMap = encoder.typeMap 23 | 24 | class Encoder(encoder.Encoder): 25 | def __call__(self, client, defMode=1, maxChunkSize=0): 26 | return encoder.Encoder.__call__(self, client, defMode, maxChunkSize) 27 | 28 | encode = Encoder(tagMap, typeMap) 29 | -------------------------------------------------------------------------------- /src/utils/pyasn1/compat/__init__.py: -------------------------------------------------------------------------------- 1 | # This file is necessary to make this directory a package. 2 | -------------------------------------------------------------------------------- /src/utils/pyasn1/compat/octets.py: -------------------------------------------------------------------------------- 1 | from sys import version_info 2 | 3 | if version_info[0] <= 2: 4 | int2oct = chr 5 | ints2octs = lambda s: ''.join([ int2oct(x) for x in s ]) 6 | null = '' 7 | oct2int = ord 8 | octs2ints = lambda s: [ oct2int(x) for x in s ] 9 | str2octs = lambda x: x 10 | octs2str = lambda x: x 11 | isOctetsType = lambda s: isinstance(s, str) 12 | else: 13 | ints2octs = bytes 14 | int2oct = lambda x: ints2octs((x,)) 15 | null = ints2octs() 16 | oct2int = lambda x: x 17 | octs2ints = lambda s: [ x for x in s ] 18 | str2octs = lambda x: x.encode() 19 | octs2str = lambda x: x.decode() 20 | isOctetsType = lambda s: isinstance(s, bytes) 21 | -------------------------------------------------------------------------------- /src/utils/pyasn1/debug.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from utils.pyasn1.compat.octets import octs2ints 3 | from utils.pyasn1 import error 4 | 5 | flagNone = 0x0000 6 | flagEncoder = 0x0001 7 | flagDecoder = 0x0002 8 | flagAll = 0xffff 9 | 10 | flagMap = { 11 | 'encoder': flagEncoder, 12 | 'decoder': flagDecoder, 13 | 'all': flagAll 14 | } 15 | 16 | class Debug: 17 | defaultPrinter = sys.stderr.write 18 | def __init__(self, *flags): 19 | self._flags = flagNone 20 | self._printer = self.defaultPrinter 21 | for f in flags: 22 | if f not in flagMap: 23 | raise error.PyAsn1Error('bad debug flag %s' % (f,)) 24 | self._flags = self._flags | flagMap[f] 25 | self('debug category %s enabled' % f) 26 | 27 | def __str__(self): 28 | return 'logger %s, flags %x' % (self._printer, self._flags) 29 | 30 | def __call__(self, msg): 31 | self._printer('DBG: %s\n' % msg) 32 | 33 | def __and__(self, flag): 34 | return self._flags & flag 35 | 36 | def __rand__(self, flag): 37 | return flag & self._flags 38 | 39 | logger = 0 40 | 41 | def setLogger(l): 42 | global logger 43 | logger = l 44 | 45 | def hexdump(octets): 46 | return ' '.join( 47 | [ '%s%.2X' % (n%16 == 0 and ('\n%.5d: ' % n) or '', x) 48 | for n,x in zip(range(len(octets)), octs2ints(octets)) ] 49 | ) 50 | 51 | class Scope: 52 | def __init__(self): 53 | self._list = [] 54 | 55 | def __str__(self): return '.'.join(self._list) 56 | 57 | def push(self, token): 58 | self._list.append(token) 59 | 60 | def pop(self): 61 | return self._list.pop() 62 | 63 | scope = Scope() 64 | -------------------------------------------------------------------------------- /src/utils/pyasn1/dn.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2011 Google Inc. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | # Author: caronni@google.com (Germano Caronni) 18 | 19 | """dn converts ASN.1 Distinguished Names to strings. 20 | 21 | TODO(user): TraverseRdn should build a better string representation, 22 | see comments below. RFC2253 provides the right way to do this. Instead of 23 | returning a dict, return a string. 24 | May also want an inverse function, parsing a string into an RDN sequence. 25 | """ 26 | 27 | #try: 28 | from utils.pyasn1.codec.ber import decoder 29 | #except ImportError: 30 | # raise ImportError, 'pyasn1 is required to run this program : http://pyasn1.sourceforge.net/ or sudo apt-get install python-pyasn1' 31 | 32 | class DistinguishedName(object): 33 | """Container for relevant OIDs and static conversion methods.""" 34 | 35 | # I want the formatting to make sense and be readable, really. 36 | # pylint: disable-msg=C6007 37 | OIDs = { 38 | (2, 5, 4, 3) : 'CN', # common name 39 | (2, 5, 4, 6) : 'C', # country 40 | (2, 5, 4, 7) : 'L', # locality 41 | (2, 5, 4, 8) : 'ST', # stateOrProvince 42 | (2, 5, 4, 10) : 'O', # organization 43 | (2, 5, 4, 11) : 'OU', # organizationalUnit 44 | (0, 9, 2342, 19200300, 100, 1, 25) : 'DC', # domainComponent 45 | (1, 2, 840, 113549, 1, 9, 1) : 'EMAIL', # emailaddress 46 | } 47 | # pylint: enable-msg=C6007 48 | 49 | @staticmethod 50 | def OidToName(oid): 51 | return DistinguishedName.OIDs.get(oid, str(oid)) 52 | 53 | @staticmethod 54 | def TraverseRdn(rdn): 55 | """Traverses RDN structure and returns string encoding of the DN. 56 | 57 | Args: 58 | rdn: ASN.1 SET (or SEQUENCE) containing RDNs (relative distinguished 59 | names), as identified by type / value pairs. A typical input would 60 | be of type X.509 RelativeDistinguishedName. 61 | 62 | Returns: 63 | A dict representing the Distinguished Name. 64 | """ 65 | val = dict() 66 | for n in rdn: 67 | # Note that this does not work for e.g. DC which is present 68 | # multiple times. 69 | # For a real DN parser, make sure to follow the spec in regards 70 | # to multiple occurence of a field in subsequent RDNs, maintaining 71 | # original ordering etc. 72 | # TODO(user): What about elements other than [0]?? 73 | name = DistinguishedName.OidToName(n[0]['type']) 74 | value = decoder.decode(n[0]['value']) 75 | if name in val: 76 | val[name] = str(value[0]) + ', ' + val.get(name, '') 77 | else: 78 | val[name] = str(value[0]) 79 | return val 80 | -------------------------------------------------------------------------------- /src/utils/pyasn1/error.py: -------------------------------------------------------------------------------- 1 | class PyAsn1Error(Exception): pass 2 | class ValueConstraintError(PyAsn1Error): pass 3 | class SubstrateUnderrunError(PyAsn1Error): pass 4 | -------------------------------------------------------------------------------- /src/utils/pyasn1/oids.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2011 Google Inc. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | # Author: caronni@google.com (Germano Caronni) 18 | 19 | """ASN.1 OIDs mappings to parser classes or strings, where there is no class.""" 20 | 21 | import hashlib 22 | 23 | import pkcs7 24 | import spc 25 | 26 | 27 | # I want the formatting to make sense and be readable, really. 28 | # pylint: disable-msg=C6006,C6007 29 | OID_TO_CLASS = { 30 | (1, 2, 840, 113549, 1, 7, 1) : 'PKCS#7 Data', 31 | (1, 2, 840, 113549, 1, 7, 2) : pkcs7.SignedData, 32 | (1, 2, 840, 113549, 2, 5) : hashlib.md5, 33 | (1, 3, 14, 3, 2, 26) : hashlib.sha1, 34 | (1, 3, 6, 1, 4, 1, 311, 2, 1, 4) : spc.SpcIndirectDataContent, 35 | (1, 2, 840, 113549, 1, 9, 3) : pkcs7.ContentType, 36 | (1, 2, 840, 113549, 1, 9, 4) : pkcs7.DigestInfo, 37 | (1, 3, 6, 1, 4, 1, 311, 2, 1, 12) : spc.SpcSpOpusInfo, 38 | (1, 2, 840, 113549, 1, 9, 6) : pkcs7.CountersignInfo, # 'RSA_counterSign' 39 | (1, 2, 840, 113549, 1, 9, 5) : pkcs7.SigningTime, 40 | } 41 | 42 | OID_TO_PUBKEY = { 43 | (1, 2, 840, 113549, 1, 1, 1) : 'rsa', 44 | (1, 2, 840, 113549, 1, 1, 5) : 'rsa-sha1', 45 | (1, 2, 840, 10040, 4, 1) : 'dsa', 46 | } 47 | -------------------------------------------------------------------------------- /src/utils/pyasn1/spc.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2011 Google Inc. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | # Author: caronni@google.com (Germano Caronni) 18 | 19 | """Authenticode-specific ASN.1 data structures.""" 20 | 21 | try: 22 | from utils.pyasn1.type import char 23 | from utils.pyasn1.type import namedtype 24 | from utils.pyasn1.type import tag 25 | from utils.pyasn1.type import univ 26 | except ImportError: 27 | raise ImportError, 'pyasn1 is required to run this program : http://pyasn1.sourceforge.net/ or sudo apt-get install python-pyasn1' 28 | 29 | from pkcs7 import DigestInfo 30 | import x509 31 | 32 | 33 | class SpcAttributeTypeAndOptionalValue(univ.Sequence): 34 | componentType = namedtype.NamedTypes( 35 | namedtype.NamedType('type', x509.AttributeType()), 36 | namedtype.OptionalNamedType('value', x509.AttributeValue())) 37 | 38 | 39 | class SpcIndirectDataContent(univ.Sequence): 40 | componentType = namedtype.NamedTypes( 41 | namedtype.NamedType('data', SpcAttributeTypeAndOptionalValue()), 42 | namedtype.NamedType('messageDigest', DigestInfo())) 43 | 44 | 45 | class SpcUuid(univ.OctetString): 46 | pass 47 | 48 | 49 | class SpcSerializedObject(univ.Sequence): 50 | componentType = namedtype.NamedTypes( 51 | namedtype.NamedType('classId', SpcUuid()), 52 | namedtype.NamedType('serializedData', univ.OctetString())) 53 | 54 | 55 | class SpcString(univ.Choice): 56 | componentType = namedtype.NamedTypes( 57 | namedtype.NamedType('unicode', char.BMPString().subtype( 58 | implicitTag=tag.Tag(tag.tagClassContext, 59 | tag.tagFormatConstructed, 0))), 60 | namedtype.NamedType('ascii', char.IA5String().subtype( 61 | implicitTag=tag.Tag(tag.tagClassContext, 62 | tag.tagFormatConstructed, 1)))) 63 | 64 | 65 | class SpcLink(univ.Choice): 66 | """According to Authenticode specification.""" 67 | componentType = namedtype.NamedTypes( 68 | namedtype.NamedType('url', char.IA5String().subtype( 69 | implicitTag=tag.Tag(tag.tagClassContext, 70 | tag.tagFormatConstructed, 0))), 71 | namedtype.NamedType('moniker', SpcSerializedObject().subtype( 72 | implicitTag=tag.Tag(tag.tagClassContext, 73 | tag.tagFormatConstructed, 1))), 74 | namedtype.NamedType('file', SpcString().subtype( 75 | explicitTag=tag.Tag(tag.tagClassContext, 76 | tag.tagFormatConstructed, 2)))) 77 | 78 | 79 | class SpcSpOpusInfo(univ.Sequence): 80 | componentType = namedtype.NamedTypes( 81 | namedtype.OptionalNamedType('programName', SpcString().subtype( 82 | explicitTag=tag.Tag(tag.tagClassContext, 83 | tag.tagFormatConstructed, 0))), 84 | namedtype.OptionalNamedType('moreInfo', SpcLink().subtype( 85 | explicitTag=tag.Tag(tag.tagClassContext, 86 | tag.tagFormatConstructed, 1)))) 87 | -------------------------------------------------------------------------------- /src/utils/pyasn1/time_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2012 Google Inc. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | # Author: caronni@google.com (Germano Caronni) 18 | 19 | """Tests for time conversion utility.""" 20 | 21 | from utils.pyasn1.type import useful 22 | import time 23 | 24 | import unittest as test 25 | from x509_time import Time 26 | 27 | 28 | class TimeTest(test.TestCase): 29 | 30 | def testConvert(self): 31 | utctime = useful.UTCTime('120614235959Z') 32 | t = Time() 33 | t.setComponentByName('utcTime', utctime) 34 | t_str = time.asctime(time.gmtime(t.ToPythonEpochTime())) 35 | self.assertEquals(t_str, 'Thu Jun 14 23:59:59 2012') 36 | 37 | 38 | def main(): 39 | test.main() 40 | 41 | if __name__ == '__main__': 42 | main() 43 | 44 | -------------------------------------------------------------------------------- /src/utils/pyasn1/type/__init__.py: -------------------------------------------------------------------------------- 1 | # This file is necessary to make this directory a package. 2 | -------------------------------------------------------------------------------- /src/utils/pyasn1/type/char.py: -------------------------------------------------------------------------------- 1 | # ASN.1 "character string" types 2 | from utils.pyasn1.type import univ, tag 3 | 4 | class UTF8String(univ.OctetString): 5 | tagSet = univ.OctetString.tagSet.tagImplicitly( 6 | tag.Tag(tag.tagClassUniversal, tag.tagFormatSimple, 12) 7 | ) 8 | encoding = "utf-8" 9 | 10 | class NumericString(univ.OctetString): 11 | tagSet = univ.OctetString.tagSet.tagImplicitly( 12 | tag.Tag(tag.tagClassUniversal, tag.tagFormatSimple, 18) 13 | ) 14 | 15 | class PrintableString(univ.OctetString): 16 | tagSet = univ.OctetString.tagSet.tagImplicitly( 17 | tag.Tag(tag.tagClassUniversal, tag.tagFormatSimple, 19) 18 | ) 19 | 20 | class TeletexString(univ.OctetString): 21 | tagSet = univ.OctetString.tagSet.tagImplicitly( 22 | tag.Tag(tag.tagClassUniversal, tag.tagFormatSimple, 20) 23 | ) 24 | 25 | 26 | class VideotexString(univ.OctetString): 27 | tagSet = univ.OctetString.tagSet.tagImplicitly( 28 | tag.Tag(tag.tagClassUniversal, tag.tagFormatSimple, 21) 29 | ) 30 | 31 | class IA5String(univ.OctetString): 32 | tagSet = univ.OctetString.tagSet.tagImplicitly( 33 | tag.Tag(tag.tagClassUniversal, tag.tagFormatSimple, 22) 34 | ) 35 | 36 | class GraphicString(univ.OctetString): 37 | tagSet = univ.OctetString.tagSet.tagImplicitly( 38 | tag.Tag(tag.tagClassUniversal, tag.tagFormatSimple, 25) 39 | ) 40 | 41 | class VisibleString(univ.OctetString): 42 | tagSet = univ.OctetString.tagSet.tagImplicitly( 43 | tag.Tag(tag.tagClassUniversal, tag.tagFormatSimple, 26) 44 | ) 45 | 46 | class GeneralString(univ.OctetString): 47 | tagSet = univ.OctetString.tagSet.tagImplicitly( 48 | tag.Tag(tag.tagClassUniversal, tag.tagFormatSimple, 27) 49 | ) 50 | 51 | class UniversalString(univ.OctetString): 52 | tagSet = univ.OctetString.tagSet.tagImplicitly( 53 | tag.Tag(tag.tagClassUniversal, tag.tagFormatSimple, 28) 54 | ) 55 | encoding = "utf-32-be" 56 | 57 | class BMPString(univ.OctetString): 58 | tagSet = univ.OctetString.tagSet.tagImplicitly( 59 | tag.Tag(tag.tagClassUniversal, tag.tagFormatSimple, 30) 60 | ) 61 | encoding = "utf-16-be" 62 | -------------------------------------------------------------------------------- /src/utils/pyasn1/type/error.py: -------------------------------------------------------------------------------- 1 | from utils.pyasn1.error import PyAsn1Error 2 | 3 | class ValueConstraintError(PyAsn1Error): pass 4 | -------------------------------------------------------------------------------- /src/utils/pyasn1/type/namedval.py: -------------------------------------------------------------------------------- 1 | # ASN.1 named integers 2 | from utils.pyasn1 import error 3 | 4 | __all__ = [ 'NamedValues' ] 5 | 6 | class NamedValues: 7 | def __init__(self, *namedValues): 8 | self.nameToValIdx = {}; self.valToNameIdx = {} 9 | self.namedValues = () 10 | automaticVal = 1 11 | for namedValue in namedValues: 12 | if isinstance(namedValue, tuple): 13 | name, val = namedValue 14 | else: 15 | name = namedValue 16 | val = automaticVal 17 | if name in self.nameToValIdx: 18 | raise error.PyAsn1Error('Duplicate name %s' % (name,)) 19 | self.nameToValIdx[name] = val 20 | if val in self.valToNameIdx: 21 | raise error.PyAsn1Error('Duplicate value %s=%s' % (name, val)) 22 | self.valToNameIdx[val] = name 23 | self.namedValues = self.namedValues + ((name, val),) 24 | automaticVal = automaticVal + 1 25 | def __str__(self): return str(self.namedValues) 26 | 27 | def getName(self, value): 28 | if value in self.valToNameIdx: 29 | return self.valToNameIdx[value] 30 | 31 | def getValue(self, name): 32 | if name in self.nameToValIdx: 33 | return self.nameToValIdx[name] 34 | 35 | def __getitem__(self, i): return self.namedValues[i] 36 | def __len__(self): return len(self.namedValues) 37 | 38 | def __add__(self, namedValues): 39 | return self.__class__(*self.namedValues + namedValues) 40 | def __radd__(self, namedValues): 41 | return self.__class__(*namedValues + tuple(self)) 42 | 43 | def clone(self, *namedValues): 44 | return self.__class__(*tuple(self) + namedValues) 45 | 46 | # XXX clone/subtype? 47 | -------------------------------------------------------------------------------- /src/utils/pyasn1/type/tagmap.py: -------------------------------------------------------------------------------- 1 | from utils.pyasn1 import error 2 | 3 | class TagMap: 4 | def __init__(self, posMap={}, negMap={}, defType=None): 5 | self.__posMap = posMap.copy() 6 | self.__negMap = negMap.copy() 7 | self.__defType = defType 8 | 9 | def __contains__(self, tagSet): 10 | return tagSet in self.__posMap or \ 11 | self.__defType is not None and tagSet not in self.__negMap 12 | 13 | def __getitem__(self, tagSet): 14 | if tagSet in self.__posMap: 15 | return self.__posMap[tagSet] 16 | elif tagSet in self.__negMap: 17 | raise error.PyAsn1Error('Key in negative map') 18 | elif self.__defType is not None: 19 | return self.__defType 20 | else: 21 | raise KeyError() 22 | 23 | def __repr__(self): 24 | s = '%r/%r' % (self.__posMap, self.__negMap) 25 | if self.__defType is not None: 26 | s = s + '/%r' % (self.__defType,) 27 | return s 28 | 29 | def clone(self, parentType, tagMap, uniq=False): 30 | if self.__defType is not None and tagMap.getDef() is not None: 31 | raise error.PyAsn1Error('Duplicate default value at %s' % (self,)) 32 | if tagMap.getDef() is not None: 33 | defType = tagMap.getDef() 34 | else: 35 | defType = self.__defType 36 | 37 | posMap = self.__posMap.copy() 38 | for k in tagMap.getPosMap(): 39 | if uniq and k in posMap: 40 | raise error.PyAsn1Error('Duplicate positive key %s' % (k,)) 41 | posMap[k] = parentType 42 | 43 | negMap = self.__negMap.copy() 44 | negMap.update(tagMap.getNegMap()) 45 | 46 | return self.__class__( 47 | posMap, negMap, defType, 48 | ) 49 | 50 | def getPosMap(self): return self.__posMap.copy() 51 | def getNegMap(self): return self.__negMap.copy() 52 | def getDef(self): return self.__defType 53 | -------------------------------------------------------------------------------- /src/utils/pyasn1/type/useful.py: -------------------------------------------------------------------------------- 1 | # ASN.1 "useful" types 2 | from utils.pyasn1.type import char, tag 3 | 4 | class GeneralizedTime(char.VisibleString): 5 | tagSet = char.VisibleString.tagSet.tagImplicitly( 6 | tag.Tag(tag.tagClassUniversal, tag.tagFormatSimple, 24) 7 | ) 8 | 9 | class UTCTime(char.VisibleString): 10 | tagSet = char.VisibleString.tagSet.tagImplicitly( 11 | tag.Tag(tag.tagClassUniversal, tag.tagFormatSimple, 23) 12 | ) 13 | -------------------------------------------------------------------------------- /src/utils/pyasn1/x509_time.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2012 Google Inc. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | # Author: caronni@google.com (Germano Caronni) 18 | 19 | """X.509 Time class and utility functions. 20 | 21 | Limited interpretation of ASN.1 time formats, 22 | as specified in RFC2459, section 4.1.2.5 23 | """ 24 | 25 | import calendar 26 | import time 27 | 28 | 29 | try: 30 | from utils.pyasn1 import error 31 | from utils.pyasn1.type import namedtype 32 | from utils.pyasn1.type import univ 33 | from utils.pyasn1.type import useful 34 | except ImportError: 35 | raise ImportError, 'pyasn1 is required to run this program : http://pyasn1.sourceforge.net/ or sudo apt-get install python-pyasn1' 36 | 37 | class Time(univ.Choice): 38 | componentType = namedtype.NamedTypes( 39 | namedtype.NamedType('utcTime', useful.UTCTime()), 40 | namedtype.NamedType('generalTime', useful.GeneralizedTime())) 41 | 42 | def ToPythonEpochTime(self): 43 | """Takes a ASN.1 Time choice, and returns seconds since epoch in UTC.""" 44 | utc_time = self.getComponentByName('utcTime') 45 | general_time = self.getComponentByName('generalTime') 46 | if utc_time and general_time: 47 | raise error.PyAsn1Error('Both elements of a choice are present.') 48 | if general_time: 49 | format_str = '%Y%m%d%H%M%SZ' 50 | time_str = str(general_time) 51 | else: 52 | format_str = '%y%m%d%H%M%SZ' 53 | time_str = str(utc_time) 54 | time_tpl = time.strptime(time_str, format_str) 55 | return calendar.timegm(time_tpl) 56 | -------------------------------------------------------------------------------- /src/utils/verifySigs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/robbyFux/Ragpicker/77d68ff3a13851ed4c256635f0673a4c2a84ef51/src/utils/verifySigs/__init__.py -------------------------------------------------------------------------------- /src/utils/verifySigs/pecoff_blob.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2011 Google Inc. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | # Author: caronni@google.com (Germano Caronni) 18 | 19 | """Deal with Microsoft-specific Authenticode data.""" 20 | 21 | # Comments and constant names as extracted from pecoff_v8 specs. 22 | # Variable names are also used as defined in the PECOFF specification. 23 | # pylint: disable-msg=C6409 24 | 25 | # Version 1, legacy version of Win_Certificate structure. It is supported 26 | # only for purposes of verifying legacy Authenticode signatures. 27 | WIN_CERT_REVISION_1_0 = 0x100 28 | 29 | # Version 2 is the current version of the Win_Certificate structure. 30 | WIN_CERT_REVISION_2_0 = 0x200 31 | 32 | # Only type PKCS is supported by the pecoff specification. 33 | WIN_CERT_TYPE_X509 = 1 34 | WIN_CERT_TYPE_PKCS_SIGNED_DATA = 2 35 | WIN_CERT_TYPE_RESERVED_1 = 3 36 | WIN_CERT_TYPE_TS_STACK_SIGNED = 4 37 | 38 | 39 | class PecoffBlob(object): 40 | """Encapsulating class for Microsoft-specific Authenticode data. 41 | 42 | As defined in the PECOFF (v8) and Authenticode specifications. 43 | This is data as it is extracted from the signature_data field by 44 | the fingerprinter. 45 | """ 46 | 47 | def __init__(self, signed_data_tuple): 48 | self._wRevision = signed_data_tuple[0] 49 | self._wCertificateType = signed_data_tuple[1] 50 | self._bCertificate = signed_data_tuple[2] 51 | 52 | if self._wRevision != WIN_CERT_REVISION_2_0: 53 | raise RuntimeError("Unknown revision %#x." % self._wRevision) 54 | if self._wCertificateType != WIN_CERT_TYPE_PKCS_SIGNED_DATA: 55 | raise RuntimeError("Unknown cert type %#x." % self._wCertificateType) 56 | 57 | def getCertificateBlob(self): 58 | return self._bCertificate 59 | -------------------------------------------------------------------------------- /src/utils/verifySigs/sigcheck.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/robbyFux/Ragpicker/77d68ff3a13851ed4c256635f0673a4c2a84ef51/src/utils/verifySigs/sigcheck.exe -------------------------------------------------------------------------------- /versions/build.number: -------------------------------------------------------------------------------- 1 | #Build Number for ANT. Do not edit! 2 | #Fri Mar 21 11:12:00 CET 2014 3 | build.number=0 4 | -------------------------------------------------------------------------------- /versions/ragpicker_v0.06.0_remnux.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/robbyFux/Ragpicker/77d68ff3a13851ed4c256635f0673a4c2a84ef51/versions/ragpicker_v0.06.0_remnux.zip -------------------------------------------------------------------------------- /versions/ragpicker_version.json: -------------------------------------------------------------------------------- 1 | { 2 | "ragpicker.build.major.number":"0", 3 | "ragpicker.build.minor.number":"06", 4 | "ragpicker.build.number":"00", 5 | "ragpicker.build.date":"19.05.2014 00:00" 6 | } --------------------------------------------------------------------------------