├── project ├── build.properties ├── assembly.sbt └── plugins.sbt ├── .travis.yml ├── src ├── main │ ├── scala │ │ └── org │ │ │ └── holmesprocessing │ │ │ └── totem │ │ │ ├── services │ │ │ ├── shodan │ │ │ │ ├── acl.conf │ │ │ │ ├── watchdog.scala │ │ │ │ ├── service.conf.example │ │ │ │ ├── Dockerfile │ │ │ │ ├── LICENSE │ │ │ │ ├── shodanfile.py │ │ │ │ ├── ShodanREST.scala │ │ │ │ ├── README.md │ │ │ │ └── service.py │ │ │ ├── asnmeta │ │ │ │ ├── acl.conf │ │ │ │ ├── watchdog.scala │ │ │ │ ├── service.conf.example │ │ │ │ ├── README.md │ │ │ │ ├── Dockerfile │ │ │ │ ├── LICENSE │ │ │ │ ├── ANSMetaREST.scala │ │ │ │ ├── asnmeta.py │ │ │ │ └── gatherasn.py │ │ │ ├── dnsmeta │ │ │ │ ├── acl.conf │ │ │ │ ├── watchdog.scala │ │ │ │ ├── service.conf.example │ │ │ │ ├── Dockerfile │ │ │ │ ├── LICENSE │ │ │ │ ├── DNSMetaREST.scala │ │ │ │ ├── README.md │ │ │ │ └── dnsmeta.py │ │ │ ├── objdump │ │ │ │ ├── acl.conf │ │ │ │ ├── watchdog.scala │ │ │ │ ├── service.conf.example │ │ │ │ ├── LICENSE │ │ │ │ ├── Dockerfile │ │ │ │ ├── README.md │ │ │ │ └── ObjdumpREST.scala │ │ │ ├── peid │ │ │ │ ├── acl.conf │ │ │ │ ├── watchguard.scala │ │ │ │ ├── rulepack │ │ │ │ │ ├── UserDB.yar │ │ │ │ │ ├── userdb_panda.yar │ │ │ │ │ ├── userdb_exeinfope.yar │ │ │ │ │ ├── userdb_jclausing.yar │ │ │ │ │ └── peid-userdb-rules-with-pe-module.yar │ │ │ │ ├── service.conf.example │ │ │ │ ├── rules.yar │ │ │ │ ├── README.md │ │ │ │ ├── LICENSE │ │ │ │ ├── Dockerfile │ │ │ │ ├── getrules.py │ │ │ │ ├── PEiDREST.scala │ │ │ │ └── peid_worker.py │ │ │ ├── peinfo │ │ │ │ ├── acl.conf │ │ │ │ ├── watchdog.scala │ │ │ │ ├── service.conf.example │ │ │ │ ├── Dockerfile │ │ │ │ ├── LICENSE │ │ │ │ └── PEInfoREST.scala │ │ │ ├── pemeta │ │ │ │ ├── acl.conf │ │ │ │ ├── watchdog.scala │ │ │ │ ├── service.conf.example │ │ │ │ ├── LICENSE │ │ │ │ ├── Dockerfile │ │ │ │ └── PEMetaREST.scala │ │ │ ├── yara │ │ │ │ ├── acl.conf │ │ │ │ ├── watchguard.scala │ │ │ │ ├── service.conf.example │ │ │ │ ├── README.md │ │ │ │ ├── LICENSE │ │ │ │ ├── rules.yar │ │ │ │ ├── Dockerfile │ │ │ │ ├── getrules.py │ │ │ │ ├── YaraREST.scala │ │ │ │ └── yara_worker.py │ │ │ ├── cfg │ │ │ │ ├── watchdog.scala │ │ │ │ ├── service.conf.example │ │ │ │ ├── README.md │ │ │ │ ├── LICENSE │ │ │ │ ├── Dockerfile │ │ │ │ └── cfgREST.scala │ │ │ ├── gogadget │ │ │ │ ├── acl.conf │ │ │ │ ├── watchdog.scala │ │ │ │ ├── service.conf.example │ │ │ │ ├── LICENSE │ │ │ │ ├── README.md │ │ │ │ ├── Dockerfile │ │ │ │ └── GoGadgetREST.scala │ │ │ ├── passivetotal │ │ │ │ ├── acl.conf │ │ │ │ ├── watchdog.scala │ │ │ │ ├── .gitignore │ │ │ │ ├── service.conf.example │ │ │ │ ├── LICENSE │ │ │ │ ├── src │ │ │ │ │ └── passivetotal-service │ │ │ │ │ │ └── passivetotal │ │ │ │ │ │ └── client │ │ │ │ │ │ ├── helpers.go │ │ │ │ │ │ └── passivetotal-client.go │ │ │ │ ├── Dockerfile │ │ │ │ ├── README.md │ │ │ │ └── PassiveTotalREST.scala │ │ │ ├── richheader │ │ │ │ ├── acl.conf │ │ │ │ ├── watchdog.scala │ │ │ │ ├── service.conf.example │ │ │ │ ├── Dockerfile │ │ │ │ ├── LICENSE │ │ │ │ ├── README.md │ │ │ │ ├── RichHeaderREST.scala │ │ │ │ ├── rich.py │ │ │ │ ├── standalone.py │ │ │ │ └── richlibrary.py │ │ │ ├── virustotal │ │ │ │ ├── acl.conf │ │ │ │ ├── watchguard.scala │ │ │ │ ├── service.conf.example │ │ │ │ ├── LICENSE │ │ │ │ ├── Dockerfile │ │ │ │ ├── README.md │ │ │ │ └── VirustotalREST.scala │ │ │ ├── zipmeta │ │ │ │ ├── holmeslibrary │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── .gitignore │ │ │ │ │ └── files.py │ │ │ │ ├── service.conf.example │ │ │ │ ├── LICENSE │ │ │ │ ├── Dockerfile │ │ │ │ ├── ZipMetaREST.scala │ │ │ │ ├── README.md │ │ │ │ └── zipmeta.py │ │ │ ├── cfgangr │ │ │ │ ├── watchdog.scala │ │ │ │ ├── service.conf.example │ │ │ │ ├── README.md │ │ │ │ ├── Dockerfile │ │ │ │ ├── LICENSE │ │ │ │ ├── cfgangrREST.scala │ │ │ │ ├── convertbinary.py │ │ │ │ └── cfgangr.py │ │ │ └── pdfparse │ │ │ │ ├── watchdog.scala │ │ │ │ ├── service.conf.example │ │ │ │ ├── LICENSE │ │ │ │ ├── README.md │ │ │ │ ├── Dockerfile │ │ │ │ ├── pdfparseREST.scala │ │ │ │ └── pdfparse.go │ │ │ ├── types │ │ │ ├── RabbitActorTypes.scala │ │ │ ├── RMQM.scala │ │ │ ├── CommandTypes.scala │ │ │ └── WorkTypes.scala │ │ │ ├── util │ │ │ ├── DownloadMethods.scala │ │ │ └── MetricService.scala │ │ │ └── actors │ │ │ └── WorkGroup.scala │ └── resources │ │ └── application.conf └── test │ └── scala │ └── org │ └── holmesprocessing │ └── totem │ └── actors │ ├── WorkGroupTest.scala │ └── WorkActorTest.scala ├── assembly.sbt ├── config ├── upload_configs.sh ├── compose_download_conf.sh ├── totem.conf.example └── docker-compose.yml.example ├── .gitignore └── CONTRIBUTING.md /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=1.0.1 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: scala 2 | scala: 3 | - 2.11.2 4 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/shodan/acl.conf: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/shodan/watchdog.scala: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /project/assembly.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.5") 2 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/asnmeta/acl.conf: -------------------------------------------------------------------------------- 1 | placeholder 2 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/dnsmeta/acl.conf: -------------------------------------------------------------------------------- 1 | placeholder 2 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/objdump/acl.conf: -------------------------------------------------------------------------------- 1 | placeholder 2 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/peid/acl.conf: -------------------------------------------------------------------------------- 1 | placeholder 2 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/peinfo/acl.conf: -------------------------------------------------------------------------------- 1 | placeholder 2 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/pemeta/acl.conf: -------------------------------------------------------------------------------- 1 | placeholder 2 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/yara/acl.conf: -------------------------------------------------------------------------------- 1 | placeholder 2 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/cfg/watchdog.scala: -------------------------------------------------------------------------------- 1 | // placeholder 2 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/gogadget/acl.conf: -------------------------------------------------------------------------------- 1 | placeholder 2 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/passivetotal/acl.conf: -------------------------------------------------------------------------------- 1 | placeholder 2 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/richheader/acl.conf: -------------------------------------------------------------------------------- 1 | placeholder 2 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/virustotal/acl.conf: -------------------------------------------------------------------------------- 1 | placeholder 2 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/zipmeta/holmeslibrary/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/asnmeta/watchdog.scala: -------------------------------------------------------------------------------- 1 | //placeholder 2 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/cfgangr/watchdog.scala: -------------------------------------------------------------------------------- 1 | // placeholder 2 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/dnsmeta/watchdog.scala: -------------------------------------------------------------------------------- 1 | //placeholder 2 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/gogadget/watchdog.scala: -------------------------------------------------------------------------------- 1 | //placeholder 2 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/objdump/watchdog.scala: -------------------------------------------------------------------------------- 1 | //placeholder 2 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/pdfparse/watchdog.scala: -------------------------------------------------------------------------------- 1 | //placeholder 2 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/peid/watchguard.scala: -------------------------------------------------------------------------------- 1 | //placeholder 2 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/peinfo/watchdog.scala: -------------------------------------------------------------------------------- 1 | //placeholder 2 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/pemeta/watchdog.scala: -------------------------------------------------------------------------------- 1 | //placeholder 2 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/yara/watchguard.scala: -------------------------------------------------------------------------------- 1 | //placeholder 2 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/passivetotal/watchdog.scala: -------------------------------------------------------------------------------- 1 | //placeholder 2 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/richheader/watchdog.scala: -------------------------------------------------------------------------------- 1 | //placeholder 2 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/virustotal/watchguard.scala: -------------------------------------------------------------------------------- 1 | //placeholder 2 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/passivetotal/.gitignore: -------------------------------------------------------------------------------- 1 | src/github.com 2 | pkg/ 3 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/peinfo/service.conf.example: -------------------------------------------------------------------------------- 1 | { 2 | "settings": { 3 | "port" : 8080 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/zipmeta/service.conf.example: -------------------------------------------------------------------------------- 1 | { 2 | "settings": { 3 | "httpbinding": 8080 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/pemeta/service.conf.example: -------------------------------------------------------------------------------- 1 | { 2 | "settings":{ 3 | "HTTPBinding": ":8080" 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/richheader/service.conf.example: -------------------------------------------------------------------------------- 1 | { 2 | "settings": { 3 | "httpbinding": 8080 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/cfgangr/service.conf.example: -------------------------------------------------------------------------------- 1 | { 2 | "settings": { 3 | "port": 8080, 4 | "max_size_binary": 900 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.typesafe.sbteclipse" % "sbteclipse-plugin" % "5.2.2") 2 | 3 | //No SBT 1.0 or 1.0.1 compatible version 4 | //addSbtPlugin("com.github.mpeltonen" % "sbt-idea" % "1.6.0") 5 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/cfg/service.conf.example: -------------------------------------------------------------------------------- 1 | { 2 | "settings": { 3 | "HTTPBinding": ":8080" 4 | }, 5 | 6 | "cfg": { 7 | "MaxNumberOfArcs": 100000 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/peid/rulepack/UserDB.yar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HolmesProcessing/Holmes-Totem/HEAD/src/main/scala/org/holmesprocessing/totem/services/peid/rulepack/UserDB.yar -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/objdump/service.conf.example: -------------------------------------------------------------------------------- 1 | { 2 | "settings": { 3 | "HTTPBinding": ":8080" 4 | }, 5 | 6 | "objdump": { 7 | "MaxNumberOfOpcodes" : 10000 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/pdfparse/service.conf.example: -------------------------------------------------------------------------------- 1 | { 2 | "settings": { 3 | "HTTPBinding": ":8080" 4 | }, 5 | 6 | "pdfparse": { 7 | "MaxNumberofObjects": 1000 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/shodan/service.conf.example: -------------------------------------------------------------------------------- 1 | { 2 | "settings": { 3 | "httpbinding": 8080 4 | }, 5 | 6 | "shodan": { 7 | "apikey": SHODAN_API_KEY 8 | } 9 | } 10 | 11 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/peid/rulepack/userdb_panda.yar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HolmesProcessing/Holmes-Totem/HEAD/src/main/scala/org/holmesprocessing/totem/services/peid/rulepack/userdb_panda.yar -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/peid/rulepack/userdb_exeinfope.yar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HolmesProcessing/Holmes-Totem/HEAD/src/main/scala/org/holmesprocessing/totem/services/peid/rulepack/userdb_exeinfope.yar -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/peid/rulepack/userdb_jclausing.yar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HolmesProcessing/Holmes-Totem/HEAD/src/main/scala/org/holmesprocessing/totem/services/peid/rulepack/userdb_jclausing.yar -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/gogadget/service.conf.example: -------------------------------------------------------------------------------- 1 | { 2 | "settings": { 3 | "HTTPBinding": ":8080" 4 | }, 5 | 6 | "gogadget": { 7 | "MaxNumberOfGadgets": 10000, 8 | "SearchDepth": 10 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/peid/rulepack/peid-userdb-rules-with-pe-module.yar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HolmesProcessing/Holmes-Totem/HEAD/src/main/scala/org/holmesprocessing/totem/services/peid/rulepack/peid-userdb-rules-with-pe-module.yar -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/dnsmeta/service.conf.example: -------------------------------------------------------------------------------- 1 | { 2 | "settings": { 3 | "httpbinding": 8080 4 | }, 5 | "dnsmeta": { 6 | "dns_server": "8.8.8.8", 7 | "rdtypes": ["A","AAAA","NS","MX","SOA","CNAME","TXT","PTR"] 8 | } 9 | } 10 | 11 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/virustotal/service.conf.example: -------------------------------------------------------------------------------- 1 | { 2 | "settings": { 3 | "HTTPBinding": ":8080" 4 | }, 5 | 6 | "virustotal": { 7 | "ApiKey": "PutYourKeyHere", 8 | "UploadUnknownSamples": false, 9 | "RequestTimeout": 60 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/passivetotal/service.conf.example: -------------------------------------------------------------------------------- 1 | { 2 | "settings": { 3 | "port": ":8080" 4 | }, 5 | 6 | "passivetotal": { 7 | "APIUser": YOUR-PASSIVETOTAL-USERNAME, 8 | "APIKey": YOUR-PASSIVETOTAL-APIKEY, 9 | "RequestTimeout": 60 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/peid/service.conf.example: -------------------------------------------------------------------------------- 1 | { 2 | "settings": { 3 | "httpbinding": 8080 4 | }, 5 | "yara_rules": { 6 | "get_remote": false, 7 | "download_url": "", 8 | "local_path": "rules.yar" 9 | } 10 | 11 | } 12 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/yara/service.conf.example: -------------------------------------------------------------------------------- 1 | { 2 | "settings": { 3 | "httpbinding": 8080 4 | }, 5 | "yara_rules": { 6 | "get_remote": false, 7 | "download_url": "", 8 | "local_path": "rules.yar" 9 | } 10 | 11 | } 12 | -------------------------------------------------------------------------------- /assembly.sbt: -------------------------------------------------------------------------------- 1 | assemblyMergeStrategy in assembly := { 2 | case PathList("META-INF", "MANIFEST.MF") => MergeStrategy.discard // not needed to keep these, discard 3 | case "reference.conf" => MergeStrategy.concat // concat all akka reference.conf files 4 | case _ => MergeStrategy.first // important to keep LICENSE files in place 5 | } 6 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/asnmeta/service.conf.example: -------------------------------------------------------------------------------- 1 | { 2 | "settings": { 3 | "httpbinding": 8080 4 | }, 5 | "asnmeta": { 6 | "dns_server": "8.8.8.8", 7 | "asn_ipv4_query": "origin.asn.cymru.com", 8 | "asn_ipv6_query": "origin6.asn.cymru.com", 9 | "asn_peer_query": "peer.asn.cymru.com", 10 | "asn_name_query": "name.asn.cymru.com" 11 | } 12 | } 13 | 14 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/peid/rules.yar: -------------------------------------------------------------------------------- 1 | /* 2 | This rules includes files from rulepack. Please uncomment the rules you 3 | would like to use. 4 | All credit goes to the original authors. 5 | */ 6 | 7 | include "./rulepack/epcompilersigs.yar" 8 | include "./rulepack/eppackersigs.yar" 9 | //include "./rulepack/userdb_exeinfope.yar" 10 | //include "./rulepack/userdb_jclausing.yar" 11 | //include "./rulepack/userdb_panda.yar" 12 | //include "./rulepack/UserDB.yar" 13 | include "./rulepack/peid-userdb-rules-with-pe-module.yar" 14 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/asnmeta/README.md: -------------------------------------------------------------------------------- 1 | # ASNMeta service for Holmes-Totem 2 | 3 | ## Description 4 | 5 | A simple service for gathering ANS information for an IP v4/6 address. 6 | 7 | ## Output 8 | ```json 9 | "bgp_prefix": "", 10 | "asn_peers": [ 11 | "", 12 | "", 13 | ], 14 | "registry": "", 15 | "asn_number": "", 16 | "data_allocated": "", 17 | "cc": "", 18 | ``` 19 | 20 | ## Usage 21 | 22 | Build and start the docker container using the included Dockerfile. 23 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/richheader/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:alpine 2 | 3 | # add tornado 4 | RUN pip3 install tornado 5 | 6 | # create folder 7 | RUN mkdir -p /service 8 | WORKDIR /service 9 | 10 | ### 11 | # richheader specific options 12 | ### 13 | 14 | # add the files to the container 15 | COPY LICENSE /service 16 | COPY README.md /service 17 | COPY richlibrary.py /service 18 | COPY rich.py /service 19 | # add the configuration file (possibly from a storage uri) 20 | ARG conf=service.conf 21 | ADD $conf /service/service.conf 22 | 23 | CMD ["python3", "rich.py"] 24 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/cfgangr/README.md: -------------------------------------------------------------------------------- 1 | # CFG Angr service for Holmes-Totem 2 | 3 | ## Description 4 | 5 | A simple service that utilizes the open-source library 'angr' to retrieve the control flow graph (CFG) from a binary. This service dumps the CFG into JSON format. 6 | 7 | ## Usage 8 | 9 | Build and start the docker container using the included Dockerfile. Since this container needs to have access to the sample file, you need to run this container with: 10 | 11 | `-v /tmp:/tmp:ro` 12 | 13 | This allows the container to access /tmp on the local file system in read-only mode. 14 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/peinfo/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:alpine 2 | 3 | # add tornado 4 | RUN pip3 install tornado 5 | 6 | # create folder 7 | RUN mkdir -p /service 8 | WORKDIR /service 9 | 10 | ### 11 | # peinfo v2 specific options 12 | ## 13 | 14 | # add dependencies for peinfo v2 15 | RUN pip3 install --upgrade pefile 16 | 17 | # add the files to the container 18 | COPY LICENSE /service 19 | COPY README.md /service 20 | COPY peinfov2.py /service 21 | 22 | # add the configuration file (possibly from a storage uri) 23 | ARG conf=service.conf 24 | ADD $conf /service/service.conf 25 | 26 | CMD ["python3", "peinfov2.py"] 27 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/yara/README.md: -------------------------------------------------------------------------------- 1 | # Yara service for Holmes-Totem 2 | 3 | ## Description 4 | 5 | Performs a yara scan from a master list that is either local or remote. The service also allows for tasking with a custom rule set. 6 | 7 | ## Usage 8 | 9 | Copy `service.conf.example` to `service.conf` and fill in your own values. 10 | 11 | Build and start the docker container using the included Dockerfile. Since this container needs to have access to the sample file, you need to run this container with: 12 | 13 | `-v /tmp:/tmp:ro` 14 | 15 | This allows the container to access /tmp on the local file system in read-only mode. 16 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/dnsmeta/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:alpine 2 | 3 | # add tornado 4 | RUN pip3 install tornado 5 | 6 | # create folder 7 | RUN mkdir -p /service 8 | WORKDIR /service 9 | 10 | ### 11 | # dnsmeta specific options 12 | ### 13 | 14 | # add dependencies for asnmeta 15 | RUN pip3 install dnspython 16 | 17 | # add the files to the container 18 | COPY LICENSE /service 19 | COPY README.md /service 20 | COPY gatherdns.py /service 21 | COPY dnsmeta.py /service 22 | # add the configuration file (possibly from a storage uri) 23 | ARG conf=service.conf 24 | ADD $conf /service/service.conf 25 | 26 | CMD ["python3", "dnsmeta.py"] 27 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/peinfo/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2017 Holmes Group LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/pemeta/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2017 Holmes Group LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/shodan/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:alpine 2 | 3 | # add tornado 4 | RUN pip3 install tornado 5 | 6 | # create folder 7 | RUN mkdir -p /service 8 | WORKDIR /service 9 | 10 | ### 11 | # shodan specific options 12 | ### 13 | 14 | # add dependencies for shodan 15 | RUN pip3 install shodan 16 | 17 | # add the files to the container 18 | COPY LICENSE /service 19 | COPY README.md /service 20 | COPY service.py /service 21 | COPY shodanfile.py /service 22 | 23 | # add the configuration file (possibly from a storage uri) 24 | ARG conf=service.conf 25 | ADD $conf /service/service.conf 26 | 27 | CMD ["python3", "service.py"] 28 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/shodan/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2016 Holmes Group LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/yara/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2016 Holmes Group LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/asnmeta/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:alpine 2 | 3 | # add tornado 4 | RUN pip3 install tornado 5 | 6 | # create folder 7 | RUN mkdir -p /service 8 | WORKDIR /service 9 | 10 | # add dependencies for asnmeta 11 | RUN pip3 install dnspython 12 | 13 | ### 14 | # ansmeta specific options 15 | ### 16 | 17 | # add the files to the container 18 | COPY LICENSE /service 19 | COPY README.md /service 20 | COPY gatherasn.py /service 21 | COPY asnmeta.py /service 22 | 23 | # add the configuration file (possibly from a storage uri) 24 | ARG conf=service.conf 25 | ADD $conf /service/service.conf 26 | 27 | CMD ["python3", "asnmeta.py"] 28 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/asnmeta/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2017 Holmes Group LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/dnsmeta/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2017 Holmes Group LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/gogadget/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2016 Holmes Group LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/pdfparse/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2016 Holmes Group LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/richheader/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2016 Holmes Group LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/virustotal/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2016 Holmes Group LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/zipmeta/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2016 Holmes Group LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/zipmeta/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:alpine 2 | 3 | # add tornado 4 | RUN pip3 install tornado 5 | 6 | # create folder 7 | RUN mkdir -p /service 8 | WORKDIR /service 9 | 10 | ### 11 | # zipmeta specific options 12 | ### 13 | 14 | # add the files to the container 15 | COPY LICENSE /service 16 | COPY README.md /service 17 | COPY ZipParser.py /service 18 | COPY extra_field_parse.py /service 19 | COPY zipmeta.py /service 20 | COPY holmeslibrary /service/holmeslibrary 21 | # add the configuration file (possibly from a storage uri) 22 | ARG conf=service.conf 23 | ADD $conf /service/service.conf 24 | 25 | CMD ["python3", "/service/zipmeta.py"] 26 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/shodan/shodanfile.py: -------------------------------------------------------------------------------- 1 | import shodan 2 | 3 | from tornado.web import HTTPError 4 | 5 | 6 | def runShodan(api, input): 7 | 8 | # Wrap the request in a try/ except block to catch errors 9 | try: 10 | # Lookup the host 11 | return { 12 | "host": api.host(input), 13 | } 14 | 15 | except shodan.APIError as e: 16 | print('Error: {}'.format(e)) 17 | if e.value == 'No information available for that IP.': 18 | raise HTTPError(404, "API Error: {}".format(e), reason="API Error") 19 | else: 20 | raise HTTPError(401, "API Error: {}".format(e), reason="API Error") 21 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/types/RabbitActorTypes.scala: -------------------------------------------------------------------------------- 1 | package org.holmesprocessing.totem.types 2 | 3 | case class QueueSettings(queueName: String, routingKey: List[String], durable: Boolean, exclusive: Boolean, autodelete: Boolean) 4 | case class ExchangeSettings(exchangeName: String, exchangeType: String, durable: Boolean) 5 | case class HostSettings(host: String, port: Int, user: String, password: String, vhost: String) 6 | 7 | case class Send(message: RMQSendMessage) 8 | case class Ack(deliveryTag: Long) 9 | case class NAck(deliveryTag: Long) 10 | case class RMQSendMessage(body: Array[Byte], routingKey: String) 11 | 12 | case class RabbitMessage(deliveryTag: Long, body: Array[Byte]) 13 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/passivetotal/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2016 Holmes Group LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | 15 | # Implementation for TOTEM made by Maximilian Schott 16 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/passivetotal/src/passivetotal-service/passivetotal/client/helpers.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "github.com/antonholmquist/jason" 5 | ) 6 | 7 | // Internal helper function to get a string from a jason.Object 8 | func jasonGetString(o *jason.Object, key string) string { 9 | if o == nil || key == "" { 10 | return "" 11 | } 12 | v, _ := o.GetString(key) 13 | return v 14 | } 15 | 16 | // Internal helper function to get a string slice from a jason.Object 17 | func jasonGetStringArray(o *jason.Object, key string) []string { 18 | if o == nil || key == "" { 19 | return []string{} 20 | } 21 | v, _ := o.GetStringArray(key) 22 | return v 23 | } 24 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/pdfparse/README.md: -------------------------------------------------------------------------------- 1 | # Pdfparse service for Holmes-Totem 2 | 3 | ## Description 4 | 5 | A simple service that parses the pdf file. This service explores the structure of the pdf file and dumps the object content into JSON format. 6 | 7 | ### Output 8 | ```json 9 | results = { 10 | "Comments": , 11 | "XREF": , 12 | "Trailer": , 13 | "StartXref": , 14 | "IndirectObject": 15 | } 16 | ``` 17 | 18 | ## Usage 19 | 20 | Build and start the docker container using the included Dockerfile. Since this container needs to have access to the sample file, you need to run this container with: 21 | 22 | `-v /tmp:/tmp:ro` 23 | 24 | This allows the container to access /tmp on the local file system in read-only mode. 25 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/peid/README.md: -------------------------------------------------------------------------------- 1 | # PEID service for Holmes-Totem 2 | 3 | ## Description 4 | 5 | This service identifies compilers and packers based on sets of yara rule packs. We have set the default to what we have found performs best, without collisions, in our analysis. However, we have includes additional packs that can be selected by modifying `rules.yar`. 6 | 7 | ## Usage 8 | 9 | Copy `service.conf.example` to `service.conf` and fill in your own values. 10 | 11 | Build and start the docker container using the included Dockerfile. Since this container needs to have access to the sample file, you need to run this container with: 12 | 13 | `-v /tmp:/tmp:ro` 14 | 15 | This allows the container to access /tmp on the local file system in read-only mode. 16 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/cfgangr/Dockerfile: -------------------------------------------------------------------------------- 1 | # Cannot build angr from python:2.7-alpine 2 | FROM python:2.7 3 | 4 | # add tornado 5 | RUN pip2 install tornado 6 | 7 | # create folder 8 | RUN mkdir -p /service 9 | WORKDIR /service 10 | 11 | ### 12 | # cfgangr specific options 13 | ### 14 | 15 | # add dependencies for cfgangr 16 | RUN pip2 install angr networkx==1.11 simuvex 17 | RUN pip2 install -I --no-use-wheel capstone 18 | 19 | 20 | # add the files to the container 21 | COPY LICENSE /service 22 | COPY README.md /service 23 | COPY cfgangr.py /service 24 | COPY convertbinary.py /service 25 | 26 | 27 | # add the configuration file (possibly from a storage uri) 28 | ARG conf=service.conf 29 | ADD $conf /service/service.conf 30 | CMD ["python2", "cfgangr.py"] 31 | 32 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/peid/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2016 Holmes Group LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | 15 | Please Note: 16 | The rulepack files are provided for convenience and are property 17 | of their respective originators. 18 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/objdump/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2016 Holmes Group LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | 15 | # Implementation for TOTEM made by Maximilian Schott 16 | # Objdump is available under the GPL (https://www.gnu.org/licenses/gpl.html) 17 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/cfg/README.md: -------------------------------------------------------------------------------- 1 | # CFG service for Holmes-Totem 2 | 3 | ## Description 4 | 5 | A simple service that utilizes the open-source library 'nucleus' to retrieve the control flow graph (CFG) from a binary. This service dumps the CFG into JSON format. 6 | 7 | ### Output 8 | ```json 9 | results = { 10 | "truncated": , 11 | "arcs": { 12 | [ 13 | "tail": , 14 | "head": , 15 | "label": 16 | ] 17 | ... 18 | } 19 | } 20 | ``` 21 | 22 | ## Usage 23 | 24 | Build and start the docker container using the included Dockerfile. Since this container needs to have access to the sample file, you need to run this container with: 25 | 26 | `-v /tmp:/tmp:ro` 27 | 28 | This allows the container to access /tmp on the local file system in read-only mode. 29 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/virustotal/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:alpine 2 | 3 | # create folder 4 | RUN mkdir -p /service 5 | WORKDIR /service 6 | 7 | # get go dependencies 8 | RUN apk add --no-cache \ 9 | git \ 10 | && go get github.com/julienschmidt/httprouter \ 11 | && rm -rf /var/cache/apk/* 12 | 13 | ### 14 | # virustotal specific options 15 | ### 16 | 17 | # clean up 18 | RUN apk del --purge \ 19 | git \ 20 | && rm -rf /var/cache/apk/* yara-3.5.0 21 | 22 | # add the files to the container 23 | COPY LICENSE /service 24 | COPY README.md /service 25 | COPY vtsample.go /service 26 | 27 | # build vtsample 28 | RUN go build vtsample.go 29 | 30 | # add the configuration file (possibly from a storage uri) 31 | ARG conf=service.conf 32 | ADD $conf /service/service.conf 33 | CMD ["./vtsample", "-config=service.conf"] 34 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/gogadget/README.md: -------------------------------------------------------------------------------- 1 | # GoGadget service for Holmes-Totem 2 | 3 | ## Description 4 | 5 | A simple service to extract the ROP Gadgets of a binary file. 6 | 7 | ### Output 8 | ```json 9 | results = { 10 | "total_unique_gadgets": "", 11 | "total_gadgets_recorded": "", 12 | "truncated": "", 13 | "search_depth": "", 14 | "gadgets": { 15 | "offset": "", 16 | "instructions": [ 17 | "(opcodes)", 18 | "(opcodes)", 19 | ], 20 | }, 21 | } 22 | ``` 23 | 24 | ## Usage 25 | 26 | Build and start the docker container using the included Dockerfile. Since this container needs to have access to the sample file, you need to run this container with: 27 | 28 | `-v /tmp:/tmp:ro` 29 | 30 | This allows the container to access /tmp on the local file system in read-only mode. 31 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/objdump/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:alpine 2 | 3 | # create folder 4 | RUN mkdir -p /service 5 | WORKDIR /service 6 | 7 | # get go dependencies 8 | RUN apk add --no-cache \ 9 | git \ 10 | && go get github.com/julienschmidt/httprouter \ 11 | && rm -rf /var/cache/apk/* 12 | 13 | ### 14 | # objdump specific options 15 | ### 16 | 17 | # Get binutils : objdump is part of binutils 18 | RUN apk add --no-cache \ 19 | binutils \ 20 | && rm -rf /var/cache/apk/* 21 | 22 | # add the files to the container 23 | COPY LICENSE /service 24 | COPY README.md /service 25 | COPY objdump.go /service 26 | 27 | # build Objdump 28 | RUN go build objdump.go 29 | 30 | # add the configuration file (possibly from a storage uri) 31 | ARG conf=service.conf 32 | ADD $conf /service/service.conf 33 | 34 | CMD ["./objdump", "--config=service.conf"] 35 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/cfg/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2017 Holmes Group LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | 15 | ------------------------------------------------------------------------ 16 | 17 | Nucleus is an open-source tool developed by the Systems and Network 18 | Security Group at VU Amsterdam. For more information, please visit: 19 | https://www.vusec.net/projects/function-detection/ 20 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/cfgangr/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2017 Holmes Group LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | 15 | ------------------------------------------------------------------------ 16 | 17 | Nucleus is an open-source tool developed by the Systems and Network 18 | Security Group at VU Amsterdam. For more information, please visit: 19 | https://www.vusec.net/projects/function-detection/ 20 | -------------------------------------------------------------------------------- /config/upload_configs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | if [[ $# -eq 0 ]] ; then 3 | echo "Please specify the IP Address and the Port of Holmes-Storage, e.g.:" 4 | echo "./upload_configs.sh 127.0.0.1:8017" 5 | exit 0 6 | fi 7 | 8 | export CONFSTORAGE=http://${1}/config/services/ 9 | for i in ../src/main/scala/org/holmesprocessing/totem/services/* 10 | do 11 | service=$(basename $i) 12 | echo $service 13 | if [ -e $i/service.conf ] 14 | then 15 | echo "using service.conf" 16 | read -p "continue uploading? [yN] " 17 | if [[ $REPLY =~ ^[Yy]$ ]]; then 18 | curl -F config=@$i/service.conf ${CONFSTORAGE}${service}/service.conf 19 | fi 20 | echo "" 21 | elif [ -e $i/service.conf.example ] 22 | then 23 | echo "using service.conf.example" 24 | read -p "continue uploading? [yN]" 25 | if [[ $REPLY =~ ^[Yy]$ ]]; then 26 | curl -F config=@$i/service.conf.example ${CONFSTORAGE}${service}/service.conf 27 | fi 28 | echo "" 29 | else 30 | echo "NO CONFIGURATION!" 31 | fi 32 | done 33 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | *.log 3 | 4 | # sbt specific 5 | .cache 6 | .history 7 | .lib/ 8 | dist/* 9 | target/ 10 | lib_managed/ 11 | src_managed/ 12 | project/boot/ 13 | project/plugins/project/ 14 | 15 | # Scala-IDE specific 16 | .scala_dependencies 17 | .worksheet 18 | .sc 19 | .idea/ 20 | *.iml 21 | 22 | # ENSIME specific 23 | .ensime_cache/ 24 | .ensime 25 | 26 | ### 27 | # Holmes Processing Specific 28 | ### 29 | 30 | # Service files 31 | *.pyc 32 | .yar 33 | 34 | # blacklist config files 35 | src/main/scala/org/novetta/totem/services/**/service.conf 36 | src/main/scala/org/novetta/totem/services/**/service.conf.json 37 | src/main/scala/org/holmesprocessing/totem/services/**/service.conf 38 | src/main/scala/org/holmesprocessing/totem/services/**/service.conf.json 39 | config/totem.conf 40 | config/docker-compose.yml 41 | 42 | # builds 43 | src/main/scala/org/holmesprocessing/totem/services/virustotal/virustotal 44 | src/main/scala/org/holmesprocessing/totem/services/gogadget/gogadget 45 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/yara/rules.yar: -------------------------------------------------------------------------------- 1 | /* 2 | This rule is just an example! Place your own rules here! 3 | All credit goes to the original author. 4 | */ 5 | rule Windows_Malware : Zeus_1134 6 | { 7 | meta: 8 | author = "Xylitol xylitol@malwareint.com" 9 | date = "2014-03-03" 10 | description = "Match first two bytes, protocol and string present in Zeus 1.1.3.4" 11 | reference = "http://www.xylibox.com/2014/03/zeus-1134.html" 12 | strings: 13 | $mz = {4D 5A} 14 | $protocol1 = "X_ID: " 15 | $protocol2 = "X_OS: " 16 | $protocol3 = "X_BV: " 17 | $stringR1 = "InitializeSecurityDescriptor" 18 | $stringR2 = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1)" 19 | condition: 20 | ($mz at 0 and all of ($protocol*) and ($stringR1 or $stringR2)) 21 | } 22 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/util/DownloadMethods.scala: -------------------------------------------------------------------------------- 1 | package org.holmesprocessing.totem.util 2 | 3 | import java.security.MessageDigest 4 | import com.typesafe.scalalogging.{Logger, LazyLogging} 5 | import org.holmesprocessing.totem.types.{WorkState, WorkResult, TaskedWork} 6 | import org.slf4j.LoggerFactory 7 | 8 | case class DownloadSettings(connection_pooling: Boolean, connect_timeout: Int, download_directory: String, thread_multiplier: Int, request_timeout: Int, validate_ssl_cert: Boolean) 9 | 10 | object DownloadMethods extends Instrumented with LazyLogging { 11 | val log = Logger(LoggerFactory.getLogger("name")) 12 | 13 | def MD5(s: Array[Byte]): String = { 14 | MessageDigest.getInstance("MD5").digest(s).map("%02x".format(_)).mkString 15 | } 16 | def SHA1(s: Array[Byte]): String = { 17 | MessageDigest.getInstance("SHA-1").digest(s).map("%02x".format(_)).mkString 18 | } 19 | def SHA256(s: Array[Byte]): String = { 20 | MessageDigest.getInstance("SHA-256").digest(s).map("%02x".format(_)).mkString 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/gogadget/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:alpine 2 | 3 | # create folder 4 | RUN mkdir -p /service 5 | WORKDIR /service 6 | 7 | # get go dependencies 8 | RUN apk add --no-cache \ 9 | git \ 10 | && go get github.com/julienschmidt/httprouter \ 11 | && rm -rf /var/cache/apk/* 12 | 13 | ### 14 | # gogadget specific options 15 | ### 16 | 17 | # Get Go ROPGadget dependencies 18 | RUN apk add --no-cache \ 19 | bash \ 20 | build-base \ 21 | python \ 22 | py-pip \ 23 | && rm -rf /var/cache/apk/* 24 | 25 | RUN pip install ropgadget 26 | 27 | # clean up 28 | RUN apk del --purge \ 29 | build-base \ 30 | git \ 31 | && rm -rf /var/cache/apk/* yara-3.5.0 32 | 33 | # add the files to the container 34 | COPY LICENSE /service 35 | COPY README.md /service 36 | COPY gogadget.go /service 37 | 38 | # build gogadget 39 | RUN go build gogadget.go 40 | 41 | # add the configuration file (possibly from a storage uri) 42 | ARG conf=service.conf 43 | ADD $conf /service/service.conf 44 | 45 | CMD ["./gogadget", "--config=service.conf"] 46 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/pdfparse/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:alpine 2 | 3 | # create folder 4 | RUN mkdir -p /service 5 | WORKDIR /service 6 | 7 | # get go dependencies 8 | RUN apk add --no-cache \ 9 | git \ 10 | && go get github.com/julienschmidt/httprouter \ 11 | && rm -rf /var/cache/apk/* 12 | 13 | ### 14 | # pdfparse specific options 15 | ### 16 | 17 | # Get Go and pdfparse dependencies 18 | RUN apk add --no-cache \ 19 | bash \ 20 | build-base \ 21 | python \ 22 | py-pip \ 23 | && rm -rf /var/cache/apk/* 24 | 25 | # clean up 26 | RUN apk del --purge \ 27 | build-base \ 28 | git \ 29 | && rm -rf /var/cache/apk/* yara-3.5.0 30 | 31 | RUN pip install pdfparse --upgrade 32 | 33 | # add the files to the container 34 | COPY LICENSE /service 35 | COPY README.md /service 36 | COPY pdfparse.go /service 37 | # build pdfparse 38 | RUN go build pdfparse.go 39 | 40 | # add the configuration file (possibly from a storage uri) 41 | ARG conf=service.conf 42 | ADD $conf /service/service.conf 43 | 44 | CMD ["./pdfparse", "--config=service.conf"] 45 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/virustotal/README.md: -------------------------------------------------------------------------------- 1 | # VirusTotal service for Holmes-Totem 2 | 3 | ## Description 4 | 5 | A very simple Holmes-Totem service which will collect everything Virustotal knows about a sample. It can also upload unknown samples to Virustotal. 6 | 7 | ## Usage 8 | 9 | Copy `service.conf.example` to `service.conf` and fill in your own values. 10 | 11 | If you decide to activate `UploadUnknownSamples` make sure to up the time a Holmes-Totem request may run in your `WorkActor.scala`: 12 | ```scala 13 | val config = new AsyncHttpClientConfig.Builder() 14 | .setRequestTimeout( 500 ) //up this to _at_least_ 5.1min 15 | .setExecutorService(execServ) 16 | .setAllowPoolingConnections(true) 17 | .setConnectTimeout( 500 ) 18 | .setIOThreadMultiplier(4).build() 19 | ``` 20 | 21 | Build and start the docker container using the included Dockerfile. Since this container needs to have access to the sample file, you need to run this container with: 22 | 23 | `-v /tmp:/tmp:ro` 24 | 25 | This allows the container to access /tmp on the local file system in read-only mode. 26 | -------------------------------------------------------------------------------- /src/test/scala/org/holmesprocessing/totem/actors/WorkGroupTest.scala: -------------------------------------------------------------------------------- 1 | package org.holmesprocessing.totem.actors 2 | 3 | import org.scalatest.BeforeAndAfterAll 4 | import org.scalatest.MustMatchers 5 | import org.scalatest.WordSpecLike 6 | import akka.actor.ActorSystem 7 | import akka.testkit.TestActorRef 8 | import akka.testkit.TestKit 9 | import akka.testkit.EventFilter 10 | import com.typesafe.config.ConfigFactory 11 | 12 | //class ExampleSpec extends FlatSpec with Matchers 13 | class WorkGroupTest extends TestKit( 14 | ActorSystem("TotemTestActorSystem", ConfigFactory.parseString("""akka.loggers = ["akka.testkit.TestEventListener"]""")) 15 | ) with WordSpecLike with MustMatchers with BeforeAndAfterAll { 16 | 17 | "A WorkGroupActor" must { 18 | val WorkGroupRef = TestActorRef(new WorkGroup()) 19 | "log a message when it gets a message it does not understand" in { 20 | 21 | EventFilter.error(pattern = "WorkGroup: received a message I cannot match against *", occurrences = 1) intercept { 22 | WorkGroupRef ! "A string! This shouldn't be parsable." 23 | } 24 | 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/zipmeta/holmeslibrary/.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | 55 | # Sphinx documentation 56 | docs/_build/ 57 | 58 | # PyBuilder 59 | target/ 60 | 61 | #Ipython Notebook 62 | .ipynb_checkpoints 63 | -------------------------------------------------------------------------------- /config/compose_download_conf.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | if [[ $# -eq 0 ]] ; then 3 | echo "Please specify the IP Address and the Port of Holmes-Storage, e.g.:" 4 | echo "./compose_download_conf.sh 127.0.0.1:8017" 5 | exit 0 6 | fi 7 | 8 | export CONFSTORAGE=http://${1}/config/services/ 9 | 10 | export CONFSTORAGE_ASNMETA=${CONFSTORAGE}asnmeta/ 11 | export CONFSTORAGE_DNSMETA=${CONFSTORAGE}dnsmeta/ 12 | export CONFSTORAGE_PASSIVETOTAL=${CONFSTORAGE}passivetotal/ 13 | export CONFSTORAGE_GOGADGET=${CONFSTORAGE}gogadget/ 14 | export CONFSTORAGE_OBJDUMP=${CONFSTORAGE}objdump/ 15 | export CONFSTORAGE_PEID=${CONFSTORAGE}peid/ 16 | export CONFSTORAGE_PEINFO=${CONFSTORAGE}peinfo/ 17 | export CONFSTORAGE_RICHHEADER=${CONFSTORAGE}richheader/ 18 | export CONFSTORAGE_SHODAN=${CONFSTORAGE}shodan/ 19 | export CONFSTORAGE_VIRUSTOTAL=${CONFSTORAGE}virustotal/ 20 | export CONFSTORAGE_YARA=${CONFSTORAGE}yara/ 21 | export CONFSTORAGE_ZIPMETA=${CONFSTORAGE}zipmeta/ 22 | export CONFSTORAGE_PDFPARSE=${CONFSTORAGE}pdfparse/ 23 | export CONFSTORAGE_CFG=${CONFSTORAGE}cfg/ 24 | export CONFSTORAGE_CFGANGR=${CONFSTORAGE}cfgangr/ 25 | export CONFSTORAGE_PEMETA=${CONFSTORAGE}pemeta/ 26 | docker-compose -f ./docker-compose.yml up -d --build 27 | -------------------------------------------------------------------------------- /src/main/resources/application.conf: -------------------------------------------------------------------------------- 1 | akka { 2 | 3 | # Loggers to register at boot time (akka.event.Logging$DefaultLogger logs 4 | # to STDOUT) 5 | loggers = ["akka.event.Logging$DefaultLogger"] 6 | 7 | # Log level used by the configured loggers (see "loggers") as soon 8 | # as they have been started; before that, see "stdout-loglevel" 9 | # Options: OFF, ERROR, WARNING, INFO, DEBUG 10 | loglevel = "INFO" 11 | 12 | # Log level for the very basic logger activated during ActorSystem startup. 13 | # This logger prints the log messages to stdout (System.out). 14 | # Options: OFF, ERROR, WARNING, INFO, DEBUG 15 | stdout-loglevel = "INFO" 16 | 17 | # Filter of log events that is used by the LoggingAdapter before 18 | # publishing log events to the eventStream. 19 | logging-filter = "akka.event.DefaultLoggingFilter" 20 | 21 | actor { 22 | provider = "akka.actor.LocalActorRefProvider" 23 | 24 | default-dispatcher { 25 | # Throughput for default Dispatcher, set to 1 for as fair as possible 26 | throughput = 10 27 | } 28 | my-pinned-dispatcher { 29 | executor = "thread-pool-executor" 30 | type = PinnedDispatcher 31 | } 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/richheader/README.md: -------------------------------------------------------------------------------- 1 | # Rich Header service for Holmes-Totem 2 | 3 | ## NOTICE 4 | 5 | For more details about the Rich Header, most updated version of the extractor, and ML code, please see the [RichHeader-Service_Collection Repository](https://github.com/HolmesProcessing/RichHeader-Service_Collection) 6 | 7 | ## Description 8 | 9 | The Rich Header service extracts the Rich Header from PE32 files. The resulting output is presented in JSON. 10 | 11 | ## Output 12 | ```json 13 | { 14 | "compids": [ 15 | { 16 | "mcv": "", 17 | "pid": "", 18 | "cnt": "", 19 | } 20 | ], 21 | "compids_dup": "", 22 | "csum_calc": "", 23 | "csum_file": "", 24 | "csum_valid": "", 25 | "error": "", 26 | "offset": "", 27 | } 28 | ``` 29 | 30 | ## Usage 31 | 32 | Build and start the docker container using the included Dockerfile. Since this container needs to have access to the sample file, you need to run this container with: 33 | 34 | `-v /tmp:/tmp:ro` 35 | 36 | This allows the container to access /tmp on the local file system in read-only mode. 37 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/yara/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:alpine 2 | 3 | # add tornado 4 | RUN pip3 install tornado 5 | 6 | # create folder 7 | RUN mkdir -p /service 8 | WORKDIR /service 9 | 10 | ### 11 | # yara specific options 12 | ### 13 | 14 | # get dependencies 15 | RUN apk update && apk add ca-certificates wget && update-ca-certificates 16 | RUN apk add --no-cache \ 17 | autoconf \ 18 | automake \ 19 | build-base \ 20 | file \ 21 | file-dev \ 22 | libtool \ 23 | libmagic \ 24 | openssl-dev \ 25 | && wget https://github.com/VirusTotal/yara/archive/v3.5.0.tar.gz \ 26 | && tar xf v3.5.0.tar.gz \ 27 | && rm -rf /var/cache/apk/* yara-3.5.0.tar.gz 28 | 29 | # build yara 30 | WORKDIR /service/yara-3.5.0 31 | RUN ./bootstrap.sh \ 32 | && ./configure \ 33 | --with-crypto \ 34 | --enable-magic \ 35 | && make \ 36 | && make install 37 | WORKDIR /service/ 38 | 39 | # build yara-python 40 | RUN pip3 install yara-python requests 41 | 42 | # clean up 43 | RUN apk del --purge \ 44 | autoconf \ 45 | automake \ 46 | build-base \ 47 | && rm -rf /var/cache/apk/* yara-3.5.0 48 | 49 | # add the files to the container 50 | COPY LICENSE /service 51 | COPY README.md /service 52 | COPY getrules.py /service 53 | COPY yara_worker.py /service 54 | 55 | # gather the rules and add them 56 | COPY rules.yar /service 57 | RUN python3 getrules.py 58 | 59 | # add the configuration file (possibly from a storage uri) 60 | ARG conf=service.conf 61 | ADD $conf /service/service.conf 62 | CMD ["python3", "yara_worker.py"] 63 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/pemeta/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:alpine 2 | 3 | # Create folder 4 | RUN mkdir -p /service 5 | WORKDIR /service 6 | 7 | # Get Golang dependencies 8 | RUN apk add --no-cache \ 9 | git \ 10 | && go get github.com/julienschmidt/httprouter \ 11 | && rm -rf /var/cache/apk/* 12 | 13 | # Get PEV dependencies 14 | RUN apk add --no-cache \ 15 | bash \ 16 | build-base \ 17 | openssl-dev \ 18 | make \ 19 | && git clone https://github.com/merces/libpe.git \ 20 | && cd libpe \ 21 | && git checkout ccd907e86f931a5d66b5f6ce592b953e9f056596 \ 22 | && rm -rf /var/cache/apk/* 23 | 24 | # Clean Up 25 | RUN apk del --purge \ 26 | #build-base \ 27 | git 28 | 29 | # Set environment variables. 30 | ENV CFLAGS="-U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2" 31 | 32 | # Making quick changes. REMOVING PRAGMA VALUES FOR THE SAKE OF GETTING OPTIONAL HEADERS. 33 | RUN sed -i 's/.*pack(push, 1).*//g' /service/libpe/include/libpe/hdr_optional.h 34 | RUN sed -i 's/.*pack(pop).*//g' /service/libpe/include/libpe/hdr_optional.h 35 | 36 | # Installing Libpe (- PEV's library) 37 | RUN cd /service/libpe \ 38 | && make \ 39 | && make install 40 | # add Service files to the container. 41 | 42 | COPY LICENSE /service 43 | COPY README.md /service 44 | COPY pemeta.go /service 45 | RUN go build pemeta.go 46 | 47 | # add the configuration file (possibly from a storage uri) 48 | ARG conf=service.conf 49 | ADD $conf /service/service.conf 50 | 51 | 52 | CMD ["./pemeta", "--config=service.conf"] 53 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/peid/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:alpine 2 | 3 | # add tornado 4 | RUN pip3 install tornado 5 | 6 | # create folder 7 | RUN mkdir -p /service 8 | WORKDIR /service 9 | 10 | ### 11 | # peid specific options 12 | ### 13 | 14 | # get dependencies 15 | RUN apk update && apk add ca-certificates wget && update-ca-certificates 16 | RUN apk add --no-cache \ 17 | autoconf \ 18 | automake \ 19 | build-base \ 20 | file \ 21 | file-dev \ 22 | libtool \ 23 | libmagic \ 24 | openssl-dev \ 25 | && wget https://github.com/VirusTotal/yara/archive/v3.5.0.tar.gz \ 26 | && tar xf v3.5.0.tar.gz \ 27 | && rm -rf /var/cache/apk/* yara-3.5.0.tar.gz 28 | 29 | # build yara 30 | WORKDIR /service/yara-3.5.0 31 | RUN ./bootstrap.sh \ 32 | && ./configure \ 33 | --with-crypto \ 34 | --enable-magic \ 35 | && make \ 36 | && make install 37 | WORKDIR /service/ 38 | 39 | # build yara-python 40 | RUN pip3 install yara-python requests 41 | 42 | # clean up 43 | RUN apk del --purge \ 44 | autoconf \ 45 | automake \ 46 | build-base \ 47 | && rm -rf /var/cache/apk/* yara-3.5.0 48 | 49 | # add the files to the container 50 | COPY LICENSE /service 51 | COPY README.md /service 52 | COPY getrules.py /service 53 | COPY peid_worker.py /service 54 | COPY rulepack /service/rulepack 55 | 56 | # gather the rules and add them 57 | COPY rules.yar /service 58 | RUN python3 getrules.py 59 | 60 | # add the configuration file (possibly from a storage uri) 61 | ARG conf=service.conf 62 | ADD $conf /service/service.conf 63 | CMD ["python3", "peid_worker.py"] 64 | 65 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/passivetotal/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:alpine 2 | 3 | # create folder 4 | RUN mkdir -p /service 5 | WORKDIR /service 6 | 7 | # get go dependencies 8 | RUN apk add --no-cache \ 9 | git \ 10 | && go get github.com/julienschmidt/httprouter \ 11 | && rm -rf /var/cache/apk/* 12 | 13 | ### 14 | # passivetotal specific options 15 | ### 16 | 17 | # get additional dependencies 18 | RUN go get "github.com/go-ini/ini" \ 19 | && go get "github.com/antonholmquist/jason" 20 | 21 | # create directory to hold sources for compilation 22 | RUN mkdir -p src/passivetotal-service 23 | 24 | # add files to the container 25 | # sources files to to GOPATH instead of /service for compilation 26 | COPY LICENSE /service 27 | COPY README.md /service 28 | COPY src/passivetotal-service $GOPATH/src/passivetotal-service 29 | ARG conf=service.conf 30 | ADD $conf /service/service.conf 31 | 32 | # download TLD list from iana.org 33 | RUN wget -O TLDList.txt "http://data.iana.org/TLD/tlds-alpha-by-domain.txt" 34 | 35 | # build service's packages 36 | # build service 37 | # copy service binary to /service 38 | RUN cd $GOPATH/src/passivetotal-service \ 39 | && go build \ 40 | && cp passivetotal-service /service/passivetotal-service 41 | 42 | # clean up git 43 | # clean up behind the service build 44 | # clean up golang as we don't need it anymore 45 | RUN apk del --purge \ 46 | git \ 47 | && rm -rf /var/cache/apk/* \ 48 | && rm -rf $GOPATH \ 49 | && rm -rf /usr/local/go 50 | 51 | CMD ["./passivetotal-service", "-config=service.conf"] 52 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/passivetotal/README.md: -------------------------------------------------------------------------------- 1 | # Passivetotal service for Holmes-Totem 2 | 3 | ## Description 4 | 5 | A simple service to check PassiveTotal for additional enrichment data. 6 | If you do not have an API key, visit http://www.passivetotal.org to get one. 7 | 8 | ## Usage 9 | 10 | Build and start the docker container using the included Dockerfile. 11 | 12 | Upon building the Dockerfile downloads a list of TLDs from iana.org. 13 | To update this list of TLDs, the image needs to be built again. 14 | 15 | The service accepts domain names, ip addresses and emails as request objects. 16 | These have to be supplied as a parameter after the request URL. 17 | (If the analysisURL parameter is set to /passivetotal, then a request for the 18 | domain www.passivetotal.org would look like this: /passivetotal/www.passivetotal.org) 19 | 20 | The service performs some checks to determine the type of the input object. 21 | If a passed domain name contains an invalid TLD, it is invalid and rejected. 22 | If a passed email address contains an invalid domain, it is rejected. 23 | If a passed IP is in a reserved range, it is rejected. (ietf rfcs 6890, 4291) 24 | 25 | Only if a request object is determined valid, it is sent to selected passivetotal 26 | api endpoints. The maximum of simultaneous requests is 9. 27 | If an error is encountered in any of the api queries, the request fails and returns 28 | an appropriate error code. Check the local logs for detailed information. 29 | If the query succeeds, a json struct containing all 9 api end points is returned. 30 | Those endpoints that were not queried are set to null. 31 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/yara/getrules.py: -------------------------------------------------------------------------------- 1 | import configparser 2 | import sys 3 | import requests 4 | import yara 5 | 6 | ERROR_CODE = 1 7 | 8 | def DownloadFile(url, rulefile): 9 | r = requests.get(url, stream=True) 10 | with open(rulefile, 'wb') as f: 11 | for chunk in r.iter_content(chunk_size=(100*1024)): 12 | if chunk: 13 | f.write(chunk) 14 | 15 | 16 | def main(): 17 | """ Main logic for program """ 18 | cfg = configparser.ConfigParser() 19 | cfg.read('service.conf') 20 | rule_location = 'rules.yar' 21 | get_remote = False 22 | 23 | # Parse configuration options 24 | if cfg.has_section('yara_rules'): 25 | rule_location = cfg['yara_rules'].get('local_path', fallback='rules.yar') 26 | get_remote = cfg['yara_rules'].getboolean('get_remote', fallback=False) 27 | if get_remote: 28 | DownloadFile(cfg['yara_rules'].get('download_url'), rule_location) 29 | else: 30 | print("Configuration Error: Cannot find Rules section. Using default values.") 31 | 32 | # attempt to compile rules 33 | try: 34 | rules = yara.compile(rule_location) 35 | rules.save(rule_location) 36 | except YaraSyntaxError: 37 | print("Syntax error in the YARA rules.") 38 | sys.exit(ERROR_CODE) 39 | except YaraError: 40 | print("Unknown YARA error.") 41 | sys.exit(ERROR_CODE) 42 | except Exception as e: 43 | print(e) 44 | sys.exit(ERROR_CODE) 45 | 46 | return 0 47 | 48 | 49 | if __name__ == "__main__": 50 | try: 51 | main() 52 | except KeyboardInterrupt: 53 | sys.exit(ERROR_CODE) 54 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/types/RMQM.scala: -------------------------------------------------------------------------------- 1 | package org.holmesprocessing.totem.types 2 | 3 | import org.json4s._ 4 | import org.json4s.jackson.JsonMethods._ 5 | 6 | 7 | case class ZooWork(download: Boolean, primaryURI: String, secondaryURI: String, filename: String, tasks: Map[String, List[String]], tags: List[String], attempts: Int) { 8 | 9 | def +(that: WorkFailure): ZooWork = { 10 | val newtasks = this.tasks + (that.WorkType -> that.Arguments) 11 | new ZooWork( 12 | download = this.download, 13 | primaryURI = this.primaryURI, 14 | secondaryURI = this.secondaryURI, 15 | filename = this.filename, 16 | tasks = newtasks, 17 | tags = this.tags, 18 | attempts = this.attempts 19 | ) 20 | } 21 | } 22 | 23 | object Parsers { 24 | type Parser[T] = (Array[Byte] => T) 25 | implicit val formats = DefaultFormats 26 | 27 | /** 28 | * Helper function to manage download callbacks. This attempts a download, moves to the fallback URI if the first one fails 29 | * and reports the overall result to the originating actor. 30 | * 31 | * @return Unit. Returns are via callback messages. 32 | * @param sender: a String: The ActorRef that we will reply to. 33 | * @param id: a Long: The ID of the message in question. 34 | * @param filename: a String: The name of the file we are downloading. 35 | * @param svc1: a Req: The first URI to try. 36 | * @param svc2: a Req: The fallback URI. 37 | * @param attempts: an Int: Number of times this download has been attempted. 38 | */ 39 | def parseJ[T: Manifest](data: Array[Byte]): T = { 40 | val result: T = parse(new String(data)).extract[T] 41 | result 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/peid/getrules.py: -------------------------------------------------------------------------------- 1 | import configparser 2 | import sys 3 | import requests 4 | import yara 5 | 6 | ERROR_CODE = 1 7 | 8 | def DownloadFile(url, rulefile): 9 | r = requests.get(url, stream=True) 10 | with open(rulefile, 'wb') as f: 11 | for chunk in r.iter_content(chunk_size=(100*1024)): 12 | if chunk: 13 | f.write(chunk) 14 | 15 | 16 | def main(): 17 | """ Main logic for program """ 18 | cfg = configparser.ConfigParser() 19 | cfg.read('service.conf') 20 | rule_location = 'rules.yar' 21 | get_remote = False 22 | 23 | # Parse configuration options 24 | if cfg.has_section('yara_rules'): 25 | rule_location = cfg['yara_rules'].get('local_path', fallback='rules.yar') 26 | get_remote = cfg['yara_rules'].getboolean('get_remote', fallback=False) 27 | if get_remote: 28 | DownloadFile(cfg['yara_rules'].get('download_url'), rule_location) 29 | 30 | else: 31 | print("Configuration Error: Cannot find yara_rules section in" 32 | "service.conf. Using default values.") 33 | 34 | # attempt to compile rules 35 | try: 36 | rules = yara.compile(rule_location) 37 | rules.save(rule_location) 38 | except yara.YaraSyntaxError: 39 | print("Syntax error in the YARA rules.") 40 | sys.exit(ERROR_CODE) 41 | except yara.YaraError: 42 | print("Unknown YARA error.") 43 | sys.exit(ERROR_CODE) 44 | except Exception as e: 45 | print(e) 46 | sys.exit(ERROR_CODE) 47 | 48 | return 0 49 | 50 | 51 | if __name__ == "__main__": 52 | try: 53 | main() 54 | except KeyboardInterrupt: 55 | sys.exit(ERROR_CODE) 56 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/util/MetricService.scala: -------------------------------------------------------------------------------- 1 | package org.holmesprocessing.totem.util 2 | 3 | import java.util.concurrent.TimeUnit 4 | 5 | import akka.actor.Actor 6 | import com.codahale.metrics.{ConsoleReporter, JmxReporter} 7 | import com.codahale.metrics.json.MetricsModule 8 | import com.fasterxml.jackson.databind.ObjectMapper 9 | import nl.grons.metrics.scala.InstrumentedBuilder 10 | 11 | object MetricService extends InstrumentedBuilder { 12 | 13 | //Create global metrics registry and JMX reporter 14 | val metricRegistry = new com.codahale.metrics.MetricRegistry() 15 | val reporter = JmxReporter.forRegistry(metricRegistry).build() //keeping you so that we have a JMX backup 16 | reporter.start() 17 | sys.addShutdownHook(reporter.stop()) 18 | 19 | //Register some metrics 20 | metrics.gauge("totalMemory")(Runtime.getRuntime.totalMemory) 21 | metrics.gauge("usedMemory")(Runtime.getRuntime.totalMemory - Runtime.getRuntime.freeMemory) 22 | 23 | //Use "metrics-json" to serialize the registry 24 | private val mapper = new ObjectMapper().registerModule(new MetricsModule(TimeUnit.SECONDS, TimeUnit.MILLISECONDS, false)) 25 | 26 | def metricRegistryJsonString = mapper.writeValueAsString(metricRegistry) 27 | 28 | } 29 | 30 | trait Instrumented extends InstrumentedBuilder { 31 | 32 | val metricRegistry = MetricService.metricRegistry 33 | 34 | } 35 | 36 | //TODO: Build this out and remove metrics-scala since its ActorMetrics API is screwed up 37 | trait MonitoredActor extends Actor with Instrumented { 38 | 39 | val timer = metrics.timer("receive") 40 | 41 | final def receive = 42 | timer.timePF { 43 | monitoredReceive //TODO: use orElse to update an unhandled message metric 44 | } 45 | 46 | def monitoredReceive: Actor.Receive 47 | } 48 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/objdump/README.md: -------------------------------------------------------------------------------- 1 | # Objdump service for Holmes-Totem 2 | 3 | ## Description 4 | 5 | A simple service to get the Objdump output of a binary file. 6 | 7 | ## Output 8 | ```json 9 | { 10 | "fileformat": "", 11 | "number_of_opcodes": "", 12 | "truncated": "", 13 | "sections": { 14 | "name:": { 15 | "name": "", 16 | "flags": [""], 17 | "truncated": "", 18 | "blocks": [ 19 | { 20 | "name": "", 21 | "offset": "", 22 | "start_index": "", 23 | "truncated": "", 24 | "opcodes": [""] 25 | } 26 | ] 27 | } 28 | } 29 | } 30 | ``` 31 | Each file may have one or many sections each with a unique name. 32 | Each section may have one or many blocks. 33 | All sections are listed, but may be truncated, this depends on the service.conf opcodes limit setting. 34 | Sections that do not have the CODE flag set, will not be listed as truncated but will have no blocks. 35 | If during parsing, the max number of opcodes is reached, the current block is marked as truncated, the section is left and all further sections are marked as truncated and have 0 blocks. 36 | 37 | ## Usage 38 | 39 | Copy the `service.conf.example` to `service.conf` and adjust the values to your needs. 40 | Build and start the docker container using the included Dockerfile. 41 | Since this container needs to have access to the sample file, you need to run this container with: 42 | 43 | `-v /tmp:/tmp:ro` 44 | 45 | This allows the container to access /tmp on the local file system in read-only mode. 46 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/cfg/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:alpine 2 | 3 | # create folder 4 | RUN mkdir -p /service 5 | WORKDIR /service 6 | 7 | # get go dependencies 8 | RUN apk add --no-cache \ 9 | git \ 10 | && go get github.com/julienschmidt/httprouter \ 11 | && rm -rf /var/cache/apk/* 12 | 13 | ### 14 | # cfg specific options 15 | ### 16 | 17 | # get go cfg dependencies 18 | RUN apk add --no-cache \ 19 | bash \ 20 | build-base \ 21 | binutils \ 22 | binutils-dev \ 23 | g++ \ 24 | git \ 25 | make \ 26 | && git clone https://github.com/aquynh/capstone \ 27 | && git clone https://bitbucket.org/vusec/nucleus.git \ 28 | && rm -rf /var/cache/apk/* 29 | 30 | # clean up && create folder for .dot files 31 | RUN apk del --purge \ 32 | build-base \ 33 | git \ 34 | && rm -rf /var/cache/apk/* yara-3.5.0 \ 35 | && mkdir -p /data/tmp \ 36 | && chmod -R 777 /data 37 | 38 | # add the files to the container 39 | COPY LICENSE /service 40 | COPY README.md /service 41 | COPY cfg.go /service 42 | 43 | # build capstone 44 | RUN cd /service/capstone \ 45 | && ./make.sh \ 46 | && su; ./make.sh install 47 | 48 | # workaround for bfd.h error (see https://github.com/yoshinorim/quickstack/issues/5) 49 | RUN sed -i '1i#define PACKAGE_VERSION 1' /usr/include/bfd.h \ 50 | && sed -i '1i#define PACKAGE 1' /usr/include/bfd.h 51 | 52 | # build nucleus 53 | RUN cd /service/nucleus \ 54 | && make 55 | 56 | # revert changes of workaround if necessary 57 | # RUN sed -i '1,2d' /usr/include/bfd.h 58 | 59 | # build cfg 60 | RUN go build cfg.go 61 | 62 | # add the configuration file (possibly from a storage uri) 63 | ARG conf=service.conf 64 | ADD $conf /service/service.conf 65 | 66 | CMD ["./cfg", "--config=service.conf"] 67 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/cfg/cfgREST.scala: -------------------------------------------------------------------------------- 1 | package org.holmesprocessing.totem.services.cfg 2 | 3 | import dispatch.Defaults._ 4 | import dispatch.{url, _} 5 | import org.json4s.JsonAST.{JString, JValue} 6 | import org.holmesprocessing.totem.types.{TaskedWork, WorkFailure, WorkResult, WorkSuccess} 7 | import collection.mutable 8 | 9 | 10 | case class cfgWork(key: Long, filename: String, TimeoutMillis: Int, WorkType: String, Worker: String, Arguments: List[String]) extends TaskedWork { 11 | def doWork()(implicit myHttp: dispatch.Http): Future[WorkResult] = { 12 | 13 | val uri = cfgREST.constructURL(Worker, filename, Arguments) 14 | val requestResult = myHttp(url(uri) OK as.String) 15 | .either 16 | .map({ 17 | case Right(content) => 18 | cfgSuccess(true, JString(content), Arguments) 19 | 20 | case Left(StatusCode(404)) => 21 | cfgFailure(false, JString("Not found (File already deleted?)"), Arguments) 22 | 23 | case Left(StatusCode(500)) => 24 | cfgFailure(false, JString("Objdump service failed, check local logs"), Arguments) //would be ideal to print response body here 25 | 26 | case Left(StatusCode(code)) => 27 | cfgFailure(false, JString("Some other code: " + code.toString), Arguments) 28 | 29 | case Left(something) => 30 | cfgFailure(false, JString("wildcard failure: " + something.toString), Arguments) 31 | }) 32 | requestResult 33 | } 34 | } 35 | 36 | 37 | case class cfgSuccess(status: Boolean, data: JValue, Arguments: List[String], routingKey: String = "cfg.result.static.totem", WorkType: String = "cfg") extends WorkSuccess 38 | case class cfgFailure(status: Boolean, data: JValue, Arguments: List[String], routingKey: String = "", WorkType: String = "cfg") extends WorkFailure 39 | 40 | 41 | object cfgREST { 42 | def constructURL(root: String, filename: String, arguments: List[String]): String = { 43 | arguments.foldLeft(new mutable.StringBuilder(root+filename))({ 44 | (acc, e) => acc.append(e)}).toString() 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/peinfo/PEInfoREST.scala: -------------------------------------------------------------------------------- 1 | package org.holmesprocessing.totem.services.peinfo 2 | 3 | import dispatch.Defaults._ 4 | import dispatch.{url, _} 5 | import org.json4s.JsonAST.{JString, JValue} 6 | import org.holmesprocessing.totem.types.{TaskedWork, WorkFailure, WorkResult, WorkSuccess} 7 | import collection.mutable 8 | 9 | 10 | case class PEInfoWork(key: Long, filename: String, TimeoutMillis: Int, WorkType: String, Worker: String, Arguments: List[String]) extends TaskedWork { 11 | def doWork()(implicit myHttp: dispatch.Http): Future[WorkResult] = { 12 | 13 | val uri = PEInfoREST.constructURL(Worker, filename, Arguments) 14 | val requestResult = myHttp(url(uri) OK as.String) 15 | .either 16 | .map({ 17 | case Right(content) => 18 | PEInfoSuccess(true, JString(content), Arguments) 19 | 20 | case Left(StatusCode(404)) => 21 | PEInfoFailure(false, JString("Not found (File already deleted?)"), Arguments) 22 | 23 | case Left(StatusCode(500)) => 24 | PEInfoFailure(false, JString("PEInfo service failed, check local logs"), Arguments) //would be ideal to print response body here 25 | 26 | case Left(StatusCode(code)) => 27 | PEInfoFailure(false, JString("Some other code: " + code.toString), Arguments) 28 | 29 | case Left(something) => 30 | PEInfoFailure(false, JString("wildcard failure: " + something.toString), Arguments) 31 | }) 32 | requestResult 33 | } 34 | } 35 | 36 | 37 | case class PEInfoSuccess(status: Boolean, data: JValue, Arguments: List[String], routingKey: String = "peinfo.result.static.totem", WorkType: String = "PEINFO") extends WorkSuccess 38 | case class PEInfoFailure(status: Boolean, data: JValue, Arguments: List[String], routingKey: String = "", WorkType: String = "PEINFO") extends WorkFailure 39 | 40 | 41 | object PEInfoREST { 42 | def constructURL(root: String, filename: String, arguments: List[String]): String = { 43 | arguments.foldLeft(new mutable.StringBuilder(root+filename))({ 44 | (acc, e) => acc.append(e)}).toString() 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/pemeta/PEMetaREST.scala: -------------------------------------------------------------------------------- 1 | package org.holmesprocessing.totem.services.pemeta 2 | 3 | import dispatch.Defaults._ 4 | import dispatch.{url, _} 5 | import org.json4s.JsonAST.{JString, JValue} 6 | import org.holmesprocessing.totem.types.{TaskedWork, WorkFailure, WorkResult, WorkSuccess} 7 | import collection.mutable 8 | 9 | 10 | case class PEMetaWork(key: Long, filename: String, TimeoutMillis: Int, WorkType: String, Worker: String, Arguments: List[String]) extends TaskedWork { 11 | def doWork()(implicit myHttp: dispatch.Http): Future[WorkResult] = { 12 | 13 | val uri = PEMetaREST.constructURL(Worker, filename, Arguments) 14 | val requestResult = myHttp(url(uri) OK as.String) 15 | .either 16 | .map({ 17 | case Right(content) => 18 | PEMetaSuccess(true, JString(content), Arguments) 19 | 20 | case Left(StatusCode(404)) => 21 | PEMetaFailure(false, JString("Not found (File already deleted?)"), Arguments) 22 | 23 | case Left(StatusCode(500)) => 24 | PEMetaFailure(false, JString("PEInfoservice failed, check local logs"), Arguments) //would be ideal to print response body here 25 | 26 | case Left(StatusCode(code)) => 27 | PEMetaFailure(false, JString("Some other code: " + code.toString), Arguments) 28 | 29 | case Left(something) => 30 | PEMetaFailure(false, JString("wildcard failure: " + something.toString), Arguments) 31 | }) 32 | requestResult 33 | } 34 | } 35 | 36 | 37 | case class PEMetaSuccess(status: Boolean, data: JValue, Arguments: List[String], routingKey: String = "pemeta.result.static.totem", WorkType: String = "PEMETA") extends WorkSuccess 38 | case class PEMetaFailure(status: Boolean, data: JValue, Arguments: List[String], routingKey: String = "", WorkType: String = "PEMETA") extends WorkFailure 39 | 40 | 41 | object PEMetaREST { 42 | def constructURL(root: String, filename: String, arguments: List[String]): String = { 43 | arguments.foldLeft(new mutable.StringBuilder(root+filename))({ 44 | (acc, e) => acc.append(e)}).toString() 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/cfgangr/cfgangrREST.scala: -------------------------------------------------------------------------------- 1 | package org.holmesprocessing.totem.services.cfgangr 2 | 3 | import dispatch.Defaults._ 4 | import dispatch.{url, _} 5 | import org.json4s.JsonAST.{JString, JValue} 6 | import org.holmesprocessing.totem.types.{TaskedWork, WorkFailure, WorkResult, WorkSuccess} 7 | import collection.mutable 8 | 9 | 10 | case class cfgAngrWork(key: Long, filename: String, TimeoutMillis: Int, WorkType: String, Worker: String, Arguments: List[String]) extends TaskedWork { 11 | def doWork()(implicit myHttp: dispatch.Http): Future[WorkResult] = { 12 | 13 | val uri = cfgangrREST.constructURL(Worker, filename, Arguments) 14 | val requestResult = myHttp(url(uri) OK as.String) 15 | .either 16 | .map({ 17 | case Right(content) => 18 | cfgAngrSuccess(true, JString(content), Arguments) 19 | 20 | case Left(StatusCode(404)) => 21 | cfgAngrFailure(false, JString("Not found (File already deleted?)"), Arguments) 22 | 23 | case Left(StatusCode(500)) => 24 | cfgAngrFailure(false, JString("Objdump service failed, check local logs"), Arguments) //would be ideal to print response body here 25 | 26 | case Left(StatusCode(code)) => 27 | cfgAngrFailure(false, JString("Some other code: " + code.toString), Arguments) 28 | 29 | case Left(something) => 30 | cfgAngrFailure(false, JString("wildcard failure: " + something.toString), Arguments) 31 | }) 32 | requestResult 33 | } 34 | } 35 | 36 | 37 | case class cfgAngrSuccess(status: Boolean, data: JValue, Arguments: List[String], routingKey: String = "cfgangr.result.static.totem", WorkType: String = "cfgangr") extends WorkSuccess 38 | case class cfgAngrFailure(status: Boolean, data: JValue, Arguments: List[String], routingKey: String = "", WorkType: String = "cfgangr") extends WorkFailure 39 | 40 | 41 | object cfgangrREST { 42 | def constructURL(root: String, filename: String, arguments: List[String]): String = { 43 | arguments.foldLeft(new mutable.StringBuilder(root+filename))({ 44 | (acc, e) => acc.append(e)}).toString() 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/dnsmeta/DNSMetaREST.scala: -------------------------------------------------------------------------------- 1 | package org.holmesprocessing.totem.services.dnsmeta 2 | 3 | import dispatch.Defaults._ 4 | import dispatch.{url, _} 5 | import org.json4s.JsonAST.{JString, JValue} 6 | import org.holmesprocessing.totem.types.{TaskedWork, WorkFailure, WorkResult, WorkSuccess} 7 | import collection.mutable 8 | 9 | 10 | case class DNSMetaWork(key: Long, filename: String, TimeoutMillis: Int, WorkType: String, Worker: String, Arguments: List[String]) extends TaskedWork { 11 | def doWork()(implicit myHttp: dispatch.Http): Future[WorkResult] = { 12 | 13 | val uri = DNSMetaREST.constructURL(Worker, filename, Arguments) 14 | val requestResult = myHttp(url(uri) OK as.String) 15 | .either 16 | .map({ 17 | case Right(content) => 18 | DNSMetaSuccess(true, JString(content), Arguments) 19 | 20 | case Left(StatusCode(404)) => 21 | DNSMetaFailure(false, JString("Not found (malformed address?)"), Arguments) 22 | 23 | case Left(StatusCode(500)) => 24 | DNSMetaFailure(false, JString("DNSMeta service failed, check local logs"), Arguments) //would be ideal to print response body here 25 | 26 | case Left(StatusCode(code)) => 27 | DNSMetaFailure(false, JString("Some other code: " + code.toString), Arguments) 28 | 29 | case Left(something) => 30 | DNSMetaFailure(false, JString("wildcard failure: " + something.toString), Arguments) 31 | }) 32 | requestResult 33 | } 34 | } 35 | 36 | 37 | case class DNSMetaSuccess(status: Boolean, data: JValue, Arguments: List[String], routingKey: String = "dnsmeta.result.static.totem", WorkType: String = "DNSMETA") extends WorkSuccess 38 | case class DNSMetaFailure(status: Boolean, data: JValue, Arguments: List[String], routingKey: String = "", WorkType: String = "DNSMETA") extends WorkFailure 39 | 40 | 41 | object DNSMetaREST { 42 | def constructURL(root: String, filename: String, arguments: List[String]): String = { 43 | arguments.foldLeft(new mutable.StringBuilder(root+filename))({ 44 | (acc, e) => acc.append(e)}).toString() 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/objdump/ObjdumpREST.scala: -------------------------------------------------------------------------------- 1 | package org.holmesprocessing.totem.services.objdump 2 | 3 | import dispatch.Defaults._ 4 | import dispatch.{url, _} 5 | import org.json4s.JsonAST.{JString, JValue} 6 | import org.holmesprocessing.totem.types.{TaskedWork, WorkFailure, WorkResult, WorkSuccess} 7 | import collection.mutable 8 | 9 | 10 | case class ObjdumpWork(key: Long, filename: String, TimeoutMillis: Int, WorkType: String, Worker: String, Arguments: List[String]) extends TaskedWork { 11 | def doWork()(implicit myHttp: dispatch.Http): Future[WorkResult] = { 12 | 13 | val uri = ObjdumpREST.constructURL(Worker, filename, Arguments) 14 | val requestResult = myHttp(url(uri) OK as.String) 15 | .either 16 | .map({ 17 | case Right(content) => 18 | ObjdumpSuccess(true, JString(content), Arguments) 19 | 20 | case Left(StatusCode(404)) => 21 | ObjdumpFailure(false, JString("Not found (File already deleted?)"), Arguments) 22 | 23 | case Left(StatusCode(500)) => 24 | ObjdumpFailure(false, JString("Objdump service failed, check local logs"), Arguments) //would be ideal to print response body here 25 | 26 | case Left(StatusCode(code)) => 27 | ObjdumpFailure(false, JString("Some other code: " + code.toString), Arguments) 28 | 29 | case Left(something) => 30 | ObjdumpFailure(false, JString("wildcard failure: " + something.toString), Arguments) 31 | }) 32 | requestResult 33 | } 34 | } 35 | 36 | 37 | case class ObjdumpSuccess(status: Boolean, data: JValue, Arguments: List[String], routingKey: String = "objdump.result.static.totem", WorkType: String = "OBJDUMP") extends WorkSuccess 38 | case class ObjdumpFailure(status: Boolean, data: JValue, Arguments: List[String], routingKey: String = "", WorkType: String = "OBJDUMP") extends WorkFailure 39 | 40 | 41 | object ObjdumpREST { 42 | def constructURL(root: String, filename: String, arguments: List[String]): String = { 43 | arguments.foldLeft(new mutable.StringBuilder(root+filename))({ 44 | (acc, e) => acc.append(e)}).toString() 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/gogadget/GoGadgetREST.scala: -------------------------------------------------------------------------------- 1 | package org.holmesprocessing.totem.services.gogadget 2 | 3 | import dispatch.Defaults._ 4 | import dispatch.{url, _} 5 | import org.json4s.JsonAST.{JString, JValue} 6 | import org.holmesprocessing.totem.types.{TaskedWork, WorkFailure, WorkResult, WorkSuccess} 7 | import collection.mutable 8 | 9 | 10 | case class GoGadgetWork(key: Long, filename: String, TimeoutMillis: Int, WorkType: String, Worker: String, Arguments: List[String]) extends TaskedWork { 11 | def doWork()(implicit myHttp: dispatch.Http): Future[WorkResult] = { 12 | 13 | val uri = GoGadgetREST.constructURL(Worker, filename, Arguments) 14 | val requestResult = myHttp(url(uri) OK as.String) 15 | .either 16 | .map({ 17 | case Right(content) => 18 | GoGadgetSuccess(true, JString(content), Arguments) 19 | 20 | case Left(StatusCode(404)) => 21 | GoGadgetFailure(false, JString("Not found (File already deleted?)"), Arguments) 22 | 23 | case Left(StatusCode(500)) => 24 | GoGadgetFailure(false, JString("Objdump service failed, check local logs"), Arguments) //would be ideal to print response body here 25 | 26 | case Left(StatusCode(code)) => 27 | GoGadgetFailure(false, JString("Some other code: " + code.toString), Arguments) 28 | 29 | case Left(something) => 30 | GoGadgetFailure(false, JString("wildcard failure: " + something.toString), Arguments) 31 | }) 32 | requestResult 33 | } 34 | } 35 | 36 | 37 | case class GoGadgetSuccess(status: Boolean, data: JValue, Arguments: List[String], routingKey: String = "gogadget.result.static.totem", WorkType: String = "GOGADGET") extends WorkSuccess 38 | case class GoGadgetFailure(status: Boolean, data: JValue, Arguments: List[String], routingKey: String = "", WorkType: String = "GOGADGET") extends WorkFailure 39 | 40 | 41 | object GoGadgetREST { 42 | def constructURL(root: String, filename: String, arguments: List[String]): String = { 43 | arguments.foldLeft(new mutable.StringBuilder(root+filename))({ 44 | (acc, e) => acc.append(e)}).toString() 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/pdfparse/pdfparseREST.scala: -------------------------------------------------------------------------------- 1 | package org.holmesprocessing.totem.services.pdfparse 2 | 3 | import dispatch.Defaults._ 4 | import dispatch.{url, _} 5 | import org.json4s.JsonAST.{JString, JValue} 6 | import org.holmesprocessing.totem.types.{TaskedWork, WorkFailure, WorkResult, WorkSuccess} 7 | import collection.mutable 8 | 9 | 10 | case class pdfparseWork(key: Long, filename: String, TimeoutMillis: Int, WorkType: String, Worker: String, Arguments: List[String]) extends TaskedWork { 11 | def doWork()(implicit myHttp: dispatch.Http): Future[WorkResult] = { 12 | 13 | val uri = pdfparseREST.constructURL(Worker, filename, Arguments) 14 | val requestResult = myHttp(url(uri) OK as.String) 15 | .either 16 | .map({ 17 | case Right(content) => 18 | pdfparseSuccess(true, JString(content), Arguments) 19 | 20 | case Left(StatusCode(404)) => 21 | pdfparseFailure(false, JString("Not found (File already deleted?)"), Arguments) 22 | 23 | case Left(StatusCode(500)) => 24 | pdfparseFailure(false, JString("Objdump service failed, check local logs"), Arguments) //would be ideal to print response body here 25 | 26 | case Left(StatusCode(code)) => 27 | pdfparseFailure(false, JString("Some other code: " + code.toString), Arguments) 28 | 29 | case Left(something) => 30 | pdfparseFailure(false, JString("wildcard failure: " + something.toString), Arguments) 31 | }) 32 | requestResult 33 | } 34 | } 35 | 36 | 37 | case class pdfparseSuccess(status: Boolean, data: JValue, Arguments: List[String], routingKey: String = "pdfparse.result.static.totem", WorkType: String = "PDFPARSE") extends WorkSuccess 38 | case class pdfparseFailure(status: Boolean, data: JValue, Arguments: List[String], routingKey: String = "", WorkType: String = "PDFPARSE") extends WorkFailure 39 | 40 | 41 | object pdfparseREST { 42 | def constructURL(root: String, filename: String, arguments: List[String]): String = { 43 | arguments.foldLeft(new mutable.StringBuilder(root+filename))({ 44 | (acc, e) => acc.append(e)}).toString() 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/virustotal/VirustotalREST.scala: -------------------------------------------------------------------------------- 1 | package org.holmesprocessing.totem.services.virustotal 2 | 3 | import dispatch.Defaults._ 4 | import dispatch.{url, _} 5 | import org.json4s.JsonAST.{JString, JValue} 6 | import org.holmesprocessing.totem.types.{TaskedWork, WorkFailure, WorkResult, WorkSuccess} 7 | import collection.mutable 8 | 9 | 10 | case class VirustotalWork(key: Long, filename: String, TimeoutMillis: Int, WorkType: String, Worker: String, Arguments: List[String]) extends TaskedWork { 11 | def doWork()(implicit myHttp: dispatch.Http): Future[WorkResult] = { 12 | 13 | val uri = VirustotalREST.constructURL(Worker, filename, Arguments) 14 | val requestResult = myHttp(url(uri) OK as.String) 15 | .either 16 | .map({ 17 | case Right(content) => 18 | VirustotalSuccess(true, JString(content), Arguments) 19 | 20 | case Left(StatusCode(404)) => 21 | VirustotalFailure(false, JString("Not found (File already deleted?)"), Arguments) 22 | 23 | case Left(StatusCode(500)) => 24 | VirustotalFailure(false, JString("VT service failed, check local logs"), Arguments) //would be ideal to print response body here 25 | 26 | case Left(StatusCode(code)) => 27 | VirustotalFailure(false, JString("Some other code: " + code.toString), Arguments) 28 | 29 | case Left(something) => 30 | VirustotalFailure(false, JString("wildcard failure: " + something.toString), Arguments) 31 | }) 32 | requestResult 33 | } 34 | } 35 | 36 | 37 | case class VirustotalSuccess(status: Boolean, data: JValue, Arguments: List[String], routingKey: String = "virustotal.result.static.totem", WorkType: String = "VIRUSTOTAL") extends WorkSuccess 38 | case class VirustotalFailure(status: Boolean, data: JValue, Arguments: List[String], routingKey: String = "", WorkType: String = "VIRUSTOTAL") extends WorkFailure 39 | 40 | 41 | object VirustotalREST { 42 | def constructURL(root: String, filename: String, arguments: List[String]): String = { 43 | arguments.foldLeft(new mutable.StringBuilder(root+filename))({ 44 | (acc, e) => acc.append(e)}).toString() 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/richheader/RichHeaderREST.scala: -------------------------------------------------------------------------------- 1 | package org.holmesprocessing.totem.services.richheader 2 | 3 | import dispatch.Defaults._ 4 | import dispatch.{url, _} 5 | import org.json4s.JsonAST.{JString, JValue} 6 | import org.holmesprocessing.totem.types.{TaskedWork, WorkFailure, WorkResult, WorkSuccess} 7 | import collection.mutable 8 | 9 | 10 | case class RichHeaderWork(key: Long, filename: String, TimeoutMillis: Int, WorkType: String, Worker: String, Arguments: List[String]) extends TaskedWork { 11 | def doWork()(implicit myHttp: dispatch.Http): Future[WorkResult] = { 12 | 13 | val uri = RichHeaderREST.constructURL(Worker, filename, Arguments) 14 | val requestResult = myHttp(url(uri) OK as.String) 15 | .either 16 | .map({ 17 | case Right(content) => 18 | RichHeaderSuccess(true, JString(content), Arguments) 19 | 20 | case Left(StatusCode(404)) => 21 | RichHeaderFailure(false, JString("Not found (File already deleted?)"), Arguments) 22 | 23 | case Left(StatusCode(500)) => 24 | RichHeaderFailure(false, JString("RichHeader service failed, check local logs"), Arguments) //would be ideal to print response body here 25 | 26 | case Left(StatusCode(code)) => 27 | RichHeaderFailure(false, JString("Some other code: " + code.toString), Arguments) 28 | 29 | case Left(something) => 30 | RichHeaderFailure(false, JString("wildcard failure: " + something.toString), Arguments) 31 | }) 32 | requestResult 33 | } 34 | } 35 | 36 | 37 | case class RichHeaderSuccess(status: Boolean, data: JValue, Arguments: List[String], routingKey: String = "richheader.result.static.totem", WorkType: String = "RICHHEADER") extends WorkSuccess 38 | case class RichHeaderFailure(status: Boolean, data: JValue, Arguments: List[String], routingKey: String = "", WorkType: String = "RICHHEADER") extends WorkFailure 39 | 40 | 41 | object RichHeaderREST { 42 | def constructURL(root: String, filename: String, arguments: List[String]): String = { 43 | arguments.foldLeft(new mutable.StringBuilder(root+filename))({ 44 | (acc, e) => acc.append(e)}).toString() 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/shodan/ShodanREST.scala: -------------------------------------------------------------------------------- 1 | package org.holmesprocessing.totem.services.shodan 2 | 3 | import dispatch.Defaults._ 4 | import dispatch.{url, _} 5 | import org.json4s.JsonAST.{JString, JValue} 6 | import org.holmesprocessing.totem.types.{TaskedWork, WorkFailure, WorkResult, WorkSuccess} 7 | import collection.mutable 8 | 9 | 10 | case class ShodanWork(key: Long, filename: String, TimeoutMillis: Int, WorkType: String, Worker: String, Arguments: List[String]) extends TaskedWork { 11 | def doWork()(implicit myHttp: dispatch.Http): Future[WorkResult] = { 12 | 13 | val uri = ShodanREST.constructURL(Worker, filename, Arguments) 14 | val requestResult = myHttp(url(uri) OK as.String) 15 | .either 16 | .map({ 17 | case Right(content) => 18 | ShodanSuccess(true, JString(content), Arguments) 19 | 20 | case Left(StatusCode(401)) => 21 | ShodanFailure(false, JString("Invalid IP"), Arguments) 22 | 23 | case Left(StatusCode(404)) => 24 | ShodanFailure(false, JString("API has no information available"), Arguments) 25 | 26 | case Left(StatusCode(500)) => 27 | ShodanFailure(false, JString("Shodan service failed, check local logs"), Arguments) //would be ideal to print response body here 28 | 29 | case Left(StatusCode(code)) => 30 | ShodanFailure(false, JString("Some other code: " + code.toString), Arguments) 31 | 32 | case Left(something) => 33 | ShodanFailure(false, JString("wildcard failure: " + something.toString), Arguments) 34 | }) 35 | requestResult 36 | } 37 | } 38 | 39 | 40 | case class ShodanSuccess(status: Boolean, data: JValue, Arguments: List[String], routingKey: String = "shodan.result.static.totem", WorkType: String = "SHODAN") extends WorkSuccess 41 | case class ShodanFailure(status: Boolean, data: JValue, Arguments: List[String], routingKey: String = "", WorkType: String = "SHODAN") extends WorkFailure 42 | 43 | 44 | object ShodanREST { 45 | def constructURL(root: String, filename: String, arguments: List[String]): String = { 46 | arguments.foldLeft(new mutable.StringBuilder(root+filename))({ 47 | (acc, e) => acc.append(e)}).toString() 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/zipmeta/ZipMetaREST.scala: -------------------------------------------------------------------------------- 1 | package org.holmesprocessing.totem.services.zipmeta 2 | 3 | import dispatch.Defaults._ 4 | import dispatch.{url, _} 5 | import org.json4s.JsonAST.{JString, JValue} 6 | import org.holmesprocessing.totem.types.{TaskedWork, WorkFailure, WorkResult, WorkSuccess} 7 | import collection.mutable 8 | 9 | 10 | case class ZipMetaWork(key: Long, filename: String, TimeoutMillis: Int, WorkType: String, Worker: String, Arguments: List[String]) extends TaskedWork { 11 | def doWork()(implicit myHttp: dispatch.Http): Future[WorkResult] = { 12 | 13 | val uri = ZipMetaREST.constructURL(Worker, filename, Arguments) 14 | val requestResult = myHttp(url(uri) OK as.String) 15 | .either 16 | .map({ 17 | case Right(content) => 18 | ZipMetaSuccess(true, JString(content), Arguments) 19 | 20 | case Left(StatusCode(400)) => 21 | ZipMetaFailure(false, JString("Bad request"), Arguments) 22 | 23 | case Left(StatusCode(404)) => 24 | ZipMetaFailure(false, JString("Not found (File already deleted?)"), Arguments) 25 | 26 | case Left(StatusCode(500)) => 27 | ZipMetaFailure(false, JString("ZipMeta service failed, check local logs"), Arguments) //would be ideal to print response body here 28 | 29 | case Left(StatusCode(code)) => 30 | ZipMetaFailure(false, JString("Some other code: " + code.toString), Arguments) 31 | 32 | case Left(something) => 33 | ZipMetaFailure(false, JString("wildcard failure: " + something.toString), Arguments) 34 | }) 35 | requestResult 36 | } 37 | } 38 | 39 | 40 | case class ZipMetaSuccess(status: Boolean, data: JValue, Arguments: List[String], routingKey: String = "zipmeta.result.static.totem", WorkType: String = "ZIPMETA") extends WorkSuccess 41 | case class ZipMetaFailure(status: Boolean, data: JValue, Arguments: List[String], routingKey: String = "", WorkType: String = "ZIPMETA") extends WorkFailure 42 | 43 | 44 | object ZipMetaREST { 45 | def constructURL(root: String, filename: String, arguments: List[String]): String = { 46 | arguments.foldLeft(new mutable.StringBuilder(root+filename))({ 47 | (acc, e) => acc.append(e)}).toString() 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/asnmeta/ANSMetaREST.scala: -------------------------------------------------------------------------------- 1 | package org.holmesprocessing.totem.services.asnmeta 2 | 3 | import dispatch.Defaults._ 4 | import dispatch.{url, _} 5 | import org.json4s.JsonAST.{JString, JValue} 6 | import org.holmesprocessing.totem.types.{TaskedWork, WorkFailure, WorkResult, WorkSuccess} 7 | import collection.mutable 8 | 9 | //when these work objects are created, we need to provide both the original filename and the uuid one, and when the doWork is called, we use the original, or the uuid as appropriate (if a file) 10 | case class ASNMetaWork(key: Long, filename: String, TimeoutMillis: Int, WorkType: String, Worker: String, Arguments: List[String]) extends TaskedWork { 11 | def doWork()(implicit myHttp: dispatch.Http): Future[WorkResult] = { 12 | 13 | val uri = ASNMetaREST.constructURL(Worker, filename, Arguments) 14 | val requestResult = myHttp(url(uri) OK as.String) 15 | .either 16 | .map({ 17 | case Right(content) => 18 | ASNMetaSuccess(true, JString(content), Arguments) 19 | 20 | case Left(StatusCode(400)) => 21 | ASNMetaFailure(false, JString("Address type is not global"), Arguments) 22 | 23 | case Left(StatusCode(404)) => 24 | ASNMetaFailure(false, JString("Not found (malformed address?)"), Arguments) 25 | 26 | case Left(StatusCode(500)) => 27 | ASNMetaFailure(false, JString("ASNMeta service failed, check local logs"), Arguments) //would be ideal to print response body here 28 | 29 | case Left(StatusCode(code)) => 30 | ASNMetaFailure(false, JString("Some other code: " + code.toString), Arguments) 31 | 32 | case Left(something) => 33 | ASNMetaFailure(false, JString("wildcard failure: " + something.toString), Arguments) 34 | }) 35 | requestResult 36 | } 37 | } 38 | 39 | 40 | case class ASNMetaSuccess(status: Boolean, data: JValue, Arguments: List[String], routingKey: String = "asnmeta.result.static.totem", WorkType: String = "ASNMETA") extends WorkSuccess 41 | case class ASNMetaFailure(status: Boolean, data: JValue, Arguments: List[String], routingKey: String = "", WorkType: String = "ASNMETA") extends WorkFailure 42 | 43 | 44 | object ASNMetaREST { 45 | def constructURL(root: String, filename: String, arguments: List[String]): String = { 46 | arguments.foldLeft(new mutable.StringBuilder(root+filename))({ 47 | (acc, e) => acc.append(e)}).toString() 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/test/scala/org/holmesprocessing/totem/actors/WorkActorTest.scala: -------------------------------------------------------------------------------- 1 | package org.holmesprocessing.totem.actors 2 | /* 3 | import java.util.UUID 4 | 5 | import akka.actor.{Actor, Props, ActorSystem} 6 | import akka.testkit.{EventFilter, TestActorRef, TestKit} 7 | import com.typesafe.config.ConfigFactory 8 | import org.joda.time.DateTime 9 | import org.holmesprocessing.totem.services.MetadataWork 10 | import org.holmesprocessing.totem.types.{Conflict, TaskedWork, ZooWork} 11 | import org.scalatest.{BeforeAndAfterAll, MustMatchers, WordSpecLike} 12 | 13 | class WorkActorTest extends TestKit( 14 | ActorSystem("TotemTestActorSystem", ConfigFactory.parseString("""akka.loggers = ["akka.testkit.TestEventListener"]""")) 15 | ) with WordSpecLike with MustMatchers with BeforeAndAfterAll { 16 | 17 | //val zooworkSecond = ZooWork("http://127.0.0.1:9900/000a887477d86792d38bac9bbe786ed5", "http://127.0.0.1:9900/000a887477d86792d38bac9bbe786ed5", 18 | // "000a887477d86792d38bac9bbe786ed5", Map[String, List[String]]("FILE_METADATA" -> List[String](), "YARA" -> List[String](), "PEINFO" -> List[String]()), 0) 19 | val goodTasks = List[TaskedWork](MetadataWork(1, uuid_name, 60, "FILE_METADATA", "http://127.0.0.1:7701/metadata/", List[String](""))) 20 | val uuid_name = UUID.randomUUID().toString 21 | //val goodActor = goodActorRef.underlyingActor 22 | 23 | "A Good WorkActor" must { 24 | 25 | 26 | "log a message when it successfully downloads a file on creation" in { 27 | EventFilter.info(pattern = "Successfully downloaded *", occurrences = 1) intercept { 28 | val goodActorRef = TestActorRef[WorkActor](Props(new WorkActor(1, uuid_name, "000a887477d86792d38bac9bbe786ed5", 29 | "http://127.0.0.1:9900/000a887477d86792d38bac9bbe786ed5", "http://127.0.0.1:9900/000a887477d86792d38bac9bbe786ed5", goodTasks, 0)) 30 | ) 31 | } 32 | } 33 | "log a message when it gets a message it does not understand" in { 34 | EventFilter.info(pattern = "WorkActor has received a message *", occurrences = 1) intercept { 35 | goodActorRef ! "A string! This shouldn't be parsable." 36 | } 37 | } 38 | } 39 | "Any WorkActor" must { 40 | "Be able to compare timestamps" in { 41 | val goodActor = goodActorRef.underlyingActor 42 | assert(goodActor.timeDelta(Some(goodActor.created), DateTime.now()).getMillis > 0) 43 | } 44 | "Respond properly when Timestamps work poorly" in { 45 | val goodActor = goodActorRef.underlyingActor 46 | assert(goodActor.timeDelta(None, DateTime.now()).getMillis == 0) 47 | } 48 | } 49 | } 50 | */ 51 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/passivetotal/PassiveTotalREST.scala: -------------------------------------------------------------------------------- 1 | package org.holmesprocessing.totem.services.passivetotal 2 | 3 | import dispatch.Defaults._ 4 | import dispatch.{url, _} 5 | import org.json4s.JsonAST.{JString, JValue} 6 | import org.holmesprocessing.totem.types.{TaskedWork, WorkFailure, WorkResult, WorkSuccess} 7 | import collection.mutable 8 | 9 | 10 | case class PassiveTotalWork(key: Long, filename: String, TimeoutMillis: Int, WorkType: String, Worker: String, Arguments: List[String]) extends TaskedWork { 11 | def doWork()(implicit myHttp: dispatch.Http): Future[WorkResult] = { 12 | 13 | val uri = PassiveTotalREST.constructURL(Worker, filename, Arguments) 14 | val requestResult = myHttp(url(uri) OK as.String) 15 | .either 16 | .map({ 17 | case Right(content) => 18 | PassiveTotalSuccess(true, JString(content), Arguments) 19 | 20 | case Left(StatusCode(400)) => 21 | PassiveTotalFailure(false, JString("Malformed Request"), Arguments) 22 | 23 | case Left(StatusCode(401)) => 24 | PassiveTotalFailure(false, JString("Invalid Username or API-Key for PassiveTotal"), Arguments) 25 | 26 | case Left(StatusCode(402)) => 27 | PassiveTotalFailure(false, JString("PassiveTotal Quota Reached"), Arguments) 28 | 29 | case Left(StatusCode(404)) => 30 | PassiveTotalFailure(false, JString("Not found (malformed address?)"), Arguments) 31 | 32 | case Left(StatusCode(422)) => 33 | PassiveTotalFailure(false, JString("Unknown Object Type"), Arguments) 34 | 35 | case Left(StatusCode(500)) => 36 | PassiveTotalFailure(false, JString("PassiveTotal service failed, check local logs"), Arguments) 37 | 38 | case Left(StatusCode(502)) => 39 | PassiveTotalFailure(false, JString("PassiveTotal API Unreachable, check local logs"), Arguments) 40 | 41 | case Left(something) => 42 | PassiveTotalFailure(false, JString("wildcard failure: " + something.toString), Arguments) 43 | }) 44 | requestResult 45 | } 46 | } 47 | 48 | 49 | case class PassiveTotalSuccess(status: Boolean, data: JValue, Arguments: List[String], routingKey: String = "passivetotal.result.static.totem", WorkType: String = "PASSIVETOTAL") extends WorkSuccess 50 | case class PassiveTotalFailure(status: Boolean, data: JValue, Arguments: List[String], routingKey: String = "", WorkType: String = "PASSIVETOTAL") extends WorkFailure 51 | 52 | 53 | object PassiveTotalREST { 54 | def constructURL(root: String, filename: String, arguments: List[String]): String = { 55 | arguments.foldLeft(new mutable.StringBuilder(root+filename))({ 56 | (acc, e) => acc.append(e)}).toString() 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/shodan/README.md: -------------------------------------------------------------------------------- 1 | #Shodan service for Holmes-Totem 2 | 3 | ###Description 4 | A simple service for gathering Shodan information about an ip address. 5 | 6 | ###Output 7 | ```json 8 | { 9 | "host": { 10 | "region_code": "", 11 | "ip": "", 12 | "area_code": "", 13 | "country_name": "", "hostnames": [""], 14 | "postal_code": "", 15 | "dma_code": "", 16 | "country_code": "", 17 | "data": [ 18 | { 19 | "product": "", 20 | "os": "", 21 | "timestamp": "", 22 | "isp": "", 23 | "asn": "", 24 | "banner": "", 25 | "hostnames": [""], 26 | "devicetype": "", 27 | "location": { 28 | "city": "", 29 | "region_code": "", 30 | "area_code": "", 31 | "longitude": "", 32 | "country_code3": "", 33 | "country_name": "", 34 | "postal_code": "", 35 | "dma_code": "", 36 | "country_code": "", 37 | "latitude": "" 38 | }, 39 | "ip": "", 40 | "domains": [""], 41 | "org": "", 42 | "port": "", 43 | "opts": {""} 44 | }, 45 | { 46 | "os": "", 47 | "timestamp": "", 48 | "isp": "", 49 | "asn": "", 50 | "banner": "", 51 | "hostnames": [""], 52 | "location": { 53 | "city": "", 54 | "region_code": "", 55 | "area_code": "", 56 | "longitude": "", 57 | "country_code3": "", 58 | "country_name": "", 59 | "postal_code": "", 60 | "dma_code": "", 61 | "country_code": "", 62 | "latitude": "" 63 | }, 64 | "ip": "", 65 | "domains": [""], 66 | "org": "", 67 | "port": "", 68 | "opts": {""} 69 | } 70 | ], 71 | "city": "", 72 | "longitude": "", 73 | "country_code3": "", 74 | "latitude": "", 75 | "os": "", 76 | "ports": ["", ""] 77 | }, 78 | } 79 | ``` 80 | 81 | ###Usage 82 | Copy service.conf.example to service.conf and fill in your own values. 83 | When assigning the SHODAN_API_KEY to the apikey in service.config 84 | do not use quotation marks around the SHODAN_API_KEY. 85 | Build and start the docker container using the included Dockerfile. 86 | If the service has frequent timeouts you have to adjust totem.conf: 87 | + totem.download_settings.connection_timeout 88 | + totem.download_settings.request_timeout 89 | + totem.tasking_settings.default_service_timeout 90 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/passivetotal/src/passivetotal-service/passivetotal/client/passivetotal-client.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "github.com/antonholmquist/jason" 5 | "net/http" 6 | "time" 7 | ) 8 | 9 | // This object contains the users credentials as well as the query value and a 10 | // http.Client reference. Please use client.New(user, apikey, object, timeout) 11 | // for proper instantiation. 12 | type Client struct { 13 | username string 14 | apikey string 15 | obj string 16 | client *http.Client 17 | } 18 | 19 | // Creates a new client.Client and returns its reference. 20 | func New(username, apikey string, timeout int) *Client { 21 | self := &Client{} 22 | self.username = username 23 | self.apikey = apikey 24 | self.client = &http.Client{Timeout: time.Duration(timeout) * time.Second} 25 | return self 26 | } 27 | 28 | // The typical result of all public query functions on the client.Client object. 29 | // Not all fields may be set. Fields are set in the following order: 30 | // - HttpError 31 | // - StatusCode, Status 32 | // - JsonError 33 | // - Error, ErrorMessage, DeveloperMessage || Json 34 | type ApiResult struct { 35 | QueryDescription string 36 | HttpError error 37 | StatusCode int 38 | Status string 39 | JsonError error 40 | Json interface{} 41 | Error bool 42 | ErrorMessage string 43 | DeveloperMessage string 44 | } 45 | 46 | // Send a request using the supplied authentication tokens. 47 | // Returns an ApiResult that needsto be checked for errors and messages. 48 | func (self *Client) SendApiRequest(url, description string) *ApiResult { 49 | apiResult := &ApiResult{} 50 | apiResult.QueryDescription = description 51 | apiResult.Error = false 52 | 53 | r, _ := http.NewRequest("GET", url, nil) 54 | r.SetBasicAuth(self.username, self.apikey) 55 | 56 | httpResponse, httpError := self.client.Do(r) 57 | if httpError != nil { 58 | apiResult.HttpError = httpError 59 | return apiResult 60 | } 61 | 62 | apiResult.StatusCode = httpResponse.StatusCode 63 | apiResult.Status = httpResponse.Status 64 | 65 | jsonResponse, jsonError := jason.NewObjectFromReader(httpResponse.Body) 66 | if jsonError != nil { 67 | apiResult.JsonError = jsonError 68 | return apiResult 69 | } 70 | apiResult.Json = jsonResponse 71 | httpResponse.Body.Close() 72 | 73 | obj, notFoundError := jsonResponse.GetObject("error") 74 | if notFoundError == nil { 75 | apiResult.Error = true 76 | apiResult.ErrorMessage = jasonGetString(obj, "message") 77 | apiResult.DeveloperMessage = jasonGetString(obj, "developer_message") 78 | } 79 | 80 | return apiResult 81 | } 82 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/zipmeta/README.md: -------------------------------------------------------------------------------- 1 | # zipmeta service for Holmes-Totem 2 | 3 | ## Description 4 | 5 | This service extracts metadata from a zipfile. 6 | 7 | 8 | ## Output 9 | 10 | ```json 11 | { 12 | "__MACOSX/._Screen Shot 2017-09-08 at 10.56.43 AM.png": { 13 | "InternalAttributes": "None", 14 | "Name": "UnknownHeader", 15 | "ZipFileName": "__MACOSX/._Screen Shot 2017-09-08 at 10.56.43 AM.png", 16 | "ZipUncompressedSize": "120", 17 | "FileStartDisk": "0", 18 | "ZipCompressedSize": "50", 19 | "ZipComments": "None", 20 | "BlockTag": "b'5558'", 21 | "VersionMadeBy": "None", 22 | "ZipRequiredVersion": "2.0", 23 | "RelativeOffset": "191325", 24 | "CSize": "12", 25 | "Data": "Data", 26 | "ZipCRC": "b'5c300f34'", 27 | "ExternalAttributes": "2175025152", 28 | "ZipCompression": "Deflated", 29 | "ZipModifyDate": "September 08, 2017 10:56:46.000000", 30 | "ZipBitFlag": "Data Descriptor" 31 | }, 32 | "__MACOSX/": { 33 | "InternalAttributes": "None", 34 | "Name": "UnknownHeader", 35 | "ZipFileName": "__MACOSX/", 36 | "ZipUncompressedSize": "0", 37 | "FileStartDisk": "0", 38 | "ZipCompressedSize": "0", 39 | "ZipComments": "None", 40 | "BlockTag": "b'5558'", 41 | "VersionMadeBy": "None", 42 | "ZipRequiredVersion": "1.0", 43 | "RelativeOffset": "191270", 44 | "CSize": "12", 45 | "Data": "Data", 46 | "ZipCRC": "b'00000000'", 47 | "ExternalAttributes": "1107116032", 48 | "ZipCompression": "No Compression/Stored", 49 | "ZipModifyDate": "September 08, 2017 11:24:58.000000", 50 | "ZipBitFlag": "None" 51 | }, 52 | "Screen Shot 2017-09-08 at 10.56.43 AM.png": { 53 | "InternalAttributes": "None", 54 | "Name": "UnknownHeader", 55 | "ZipFileName": "Screen Shot 2017-09-08 at 10.56.43 AM.png", 56 | "ZipUncompressedSize": "191221", 57 | "FileStartDisk": "0", 58 | "ZipCompressedSize": "191167", 59 | "ZipComments": "None", 60 | "BlockTag": "b'5558'", 61 | "VersionMadeBy": "None", 62 | "ZipRequiredVersion": "2.0", 63 | "RelativeOffset": "0", 64 | "CSize": "12", 65 | "Data": "Data", 66 | "ZipCRC": "b'3123e044'", 67 | "ExternalAttributes": "2175025152", 68 | "ZipCompression": "Deflated", 69 | "ZipModifyDate": "September 08, 2017 10:56:46.000000", 70 | "ZipBitFlag": "Data Descriptor" 71 | }, 72 | "filecount": 3 73 | } 74 | ``` 75 | 76 | ## Usage 77 | 78 | Build and start the docker container using the included Dockerfile. Since this container needs to have access to the sample file, you need to run this container with: 79 | 80 | `-v /tmp:/tmp:ro` 81 | 82 | This allows the container to access /tmp on the local file system in read-only mode. 83 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/shodan/service.py: -------------------------------------------------------------------------------- 1 | import tornado 2 | import tornado.web 3 | import tornado.httpserver 4 | import tornado.ioloop 5 | 6 | import sys 7 | import os 8 | from os import path 9 | import shodan 10 | 11 | from shodanfile import runShodan 12 | sys.path.append(os.path.dirname(os.path.realpath(__file__))) 13 | 14 | import json 15 | 16 | # Reading configuration file 17 | def ServiceConfig(filename): 18 | configPath = filename 19 | try: 20 | config = json.loads(open(configPath).read()) 21 | return config 22 | except FileNotFoundError: 23 | raise tornado.web.HTTPError(500) 24 | 25 | # Get service meta information and configuration 26 | Config = ServiceConfig("./service.conf") 27 | api = shodan.Shodan(Config["shodan"]["apikey"]) 28 | 29 | Metadata = { 30 | "Name" : "Shodan", 31 | "Version" : "1.0", 32 | "Description" : "./README.md", 33 | "Copyright" : "Copyright 2016 Holmes Group LLC", 34 | "License" : "./LICENSE" 35 | } 36 | 37 | 38 | class Service(tornado.web.RequestHandler): 39 | def get(self): 40 | input = self.get_argument("obj", strip=False) 41 | data = runShodan(api, input) 42 | self.write(data) 43 | 44 | 45 | class Info(tornado.web.RequestHandler): 46 | # Emits a string which describes the purpose of the analytics 47 | def get(self): 48 | info = """ 49 |

{name:s} - {version:s}

50 |
51 |

{description:s}

52 |
53 |

{license:s} 54 | """.format( 55 | name = str(Metadata["Name"]).replace("\n", "
"), 56 | version = str(Metadata["Version"]).replace("\n", "
"), 57 | description = str(Metadata["Description"]).replace("\n", "
"), 58 | license = str(Metadata["License"]).replace("\n", "
") 59 | ) 60 | self.write(info) 61 | 62 | 63 | class Application(tornado.web.Application): 64 | def __init__(self): 65 | 66 | for key in ["Description", "License"]: 67 | fpath = Metadata[key] 68 | if os.path.isfile(fpath): 69 | with open(fpath) as file: 70 | Metadata[key] = file.read() 71 | 72 | handlers = [ 73 | (r'/', Info), 74 | (r'/analyze/', Service), 75 | ] 76 | settings = dict( 77 | template_path=path.join(path.dirname(__file__), 'templates'), 78 | static_path=path.join(path.dirname(__file__), 'static'), 79 | ) 80 | tornado.web.Application.__init__(self, handlers, **settings) 81 | self.engine = None 82 | 83 | 84 | def main(): 85 | server = tornado.httpserver.HTTPServer(Application()) 86 | server.listen(Config["settings"]["httpbinding"]) 87 | try: 88 | tornado.ioloop.IOLoop.current().start() 89 | except KeyboardInterrupt: 90 | tornado.ioloop.IOLoop.current().stop() 91 | 92 | 93 | if __name__ == '__main__': 94 | main() 95 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/actors/WorkGroup.scala: -------------------------------------------------------------------------------- 1 | package org.holmesprocessing.totem.actors 2 | 3 | import akka.actor._ 4 | import com.rabbitmq.client._ 5 | import com.typesafe.config.Config 6 | import org.holmesprocessing.totem.types._ 7 | import org.holmesprocessing.totem.util.{DownloadSettings, MonitoredActor} 8 | /** 9 | * @constructor This is the companion object to the class. Simplifies Props() nonsense. 10 | */ 11 | object WorkGroup { 12 | def props(): Props = { 13 | Props(new WorkGroup() ) 14 | }} 15 | 16 | /** 17 | * This actor represents the manager for all WorkActors. This is currently a manager actor, whose purpose is to make scaling, 18 | * deathwatch, and load balancing easier across an arbitrarily complex actor system. It is also the interface between WorkActors 19 | * and their source Consumer. In short, this is a "Best Practice" actor. 20 | * 21 | * Something like: 22 | * {{{ 23 | * val myWorker: ActorRef = context.actorOf(WorkActor.props(), "group") 24 | * }}} 25 | * is the preferred way to create this actor. 26 | * 27 | * The following is a listing of the message types that this Actor explicitly handles, and a brief discussion of their purpose. 28 | * {{{ 29 | * case Create(channel: Channel, key: Long, primaryURI: String, secondaryURI: String, value: WorkState) => { 30 | * Create a new WorkActor, should one not exist already, based on the Key provided, and add it to the watch list. 31 | * } 32 | * case t: Ack => { 33 | * Pass along an Ack message to the parent consumer. 34 | * } 35 | * case t: NAck => { 36 | * Pass along a NAck message to the parent consumer. 37 | * } 38 | * case Terminated(t: ActorRef) => { 39 | * Notification for when a child actor terminates. 40 | * } 41 | * 42 | * }}} 43 | * @constructor Create a new WorkGroup which manages and watches the WorkActors. While not strictly needed, this is helpful for failure recovery and remoting should it be needed. 44 | * 45 | */ 46 | 47 | class WorkGroup extends Actor with ActorLogging with MonitoredActor { 48 | 49 | def monitoredReceive = { 50 | case Create(key: Long, download: Boolean, primaryURI: String, secondaryURI: String, tags: List[String], value: WorkState, downloadconfig: DownloadSettings) => 51 | log.debug("Got a set of work {}", value) 52 | val child = context.child(key.toString).getOrElse({ 53 | log.info("WorkGroup: instantiating a new actor for message {}", key) 54 | context.watch( 55 | context.actorOf( 56 | WorkActor.props(key, value.filename, value.hashfilename, download, primaryURI, secondaryURI, value.workToDo, tags, value.attempts, downloadconfig), key.toString 57 | ) 58 | ) 59 | }) 60 | case t: Ack => 61 | log.info("WorkGroup: Ack from {}, moving up the chain", sender()) 62 | context.parent.tell(t, sender()) 63 | case t: NAck => 64 | log.warning("WorkGroup: NAck from {}, moving up the chain", sender()) 65 | context.parent.tell(t, sender()) 66 | case Terminated(t: ActorRef) => 67 | log.info("WorkGroup: child {} terminated", t) 68 | case msg => 69 | log.error("WorkGroup: received a message I cannot match against -> {}", msg) 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/cfgangr/convertbinary.py: -------------------------------------------------------------------------------- 1 | import angr 2 | import simuvex 3 | import cle 4 | import os 5 | import json 6 | # imports for graph handling 7 | import networkx as nx 8 | from networkx.readwrite import json_graph 9 | 10 | 11 | def generateCFG(binary, max_size, analysisType = 'Fast'): 12 | response = {} 13 | try: 14 | # Create the angr project for the binary 15 | project = angr.Project(binary, auto_load_libs=False) 16 | binary_size = os.stat(binary).st_size 17 | max_binary_size = int(max_size) * 1024 # Here we will set a limit of binary files in bytes 18 | if binary_size > max_binary_size: 19 | response['error'] = 'CFG generation failed because the binary size is too big' 20 | return json.dumps(response) 21 | # Create the Control Flow Graph in Fast or Accurate mode. 22 | if (analysisType == 'Fast'): 23 | cfg = project.analyses.CFG() 24 | elif (analysisType == 'Accurate'):# 25 | cfg = project.analyses.CFGAccurate() 26 | else: 27 | return 28 | 29 | except simuvex.SimSolverModeError: 30 | response['error'] = 'CFG generation failed because of SimSolverModeError' 31 | return json.dumps(response) 32 | except cle.errors.CLECompatibilityError: 33 | response['error'] = 'CFG generation failed because of unsupported format' 34 | return json.dumps(response) 35 | except AttributeError: 36 | response['error'] = 'See https://github.com/angr/angr/issues/288' 37 | return json.dumps(response) 38 | 39 | 40 | # Iterate through all nodes, and for each of them, add information for address, mnemonic and operands. 41 | for node in cfg.nodes_iter(): 42 | instructions = project.factory.block(addr=node.addr).capstone.insns 43 | a = [] 44 | for ins in instructions: 45 | b = {} 46 | b['addr'] = ins.address 47 | b['mnemonic'] = ins.mnemonic 48 | b['operand'] = ins.op_str 49 | a.append(b) 50 | node.label = a 51 | 52 | networkxGraph = cfg.graph 53 | graph = _truncateGraph(networkxGraph) 54 | 55 | # Convert networkx graph to JSON 56 | graph_json = json_graph.node_link_data(graph) 57 | 58 | return graph_json 59 | 60 | 61 | def _truncateGraph(cfg): 62 | # This method will receive a Networkx graph as input. It generates a new networkx graph from it, by keeping only the 63 | # information necessary. It returns the new graph. 64 | sequence = 0 65 | lookup = {} 66 | graph = nx.DiGraph() 67 | # Update the nodes 68 | for n in cfg.nodes(): 69 | if n not in lookup: 70 | new_node = str(sequence) + ' ' + repr(n.addr) + ' ' + str(n.name) 71 | sequence += 1 72 | lookup[n] = new_node 73 | graph.add_node(new_node, label = n.label, syscall = n.syscall) 74 | 75 | # Update the edges 76 | for src, dst, data in cfg.edges(data=True): 77 | new_src = lookup[src] 78 | new_dst = lookup[dst] 79 | data.pop('ins_addr', None) 80 | data.pop('stmt_idx', None) 81 | graph.add_edge(new_src, new_dst, data=data) 82 | 83 | return graph 84 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/dnsmeta/README.md: -------------------------------------------------------------------------------- 1 | # DNSMeta service for Holmes-Totem 2 | 3 | ## Description 4 | 5 | A simple service for gathering DNS information about a domain. This service is capable of extracting the A, AAAA, CNAME, MX, NS, PTR, SOA, and TXT fields. If the authoritative NS can be identified and located, the TTL values will be set to the max TTL values. If it cannot, the TTL values will reflect the current TTL values for that server. 6 | 7 | ## Output 8 | The output will return a json result that is as true as possible to the queried DNS server. In theory this should match the the corresponding RFC for the type. For a list of types and RFC documentation please see [record types](https://en.wikipedia.org/wiki/List_of_DNS_record_types) 9 | 10 | As an example: 11 | ```json 12 | { 13 | "SOA": { 14 | "rdata": [ 15 | { 16 | "expire": 1800, 17 | "retry": 900, 18 | "rname": "dns-admin.google.com.", 19 | "minimum": 60, 20 | "refresh": 900, 21 | "serial": 151073646, 22 | "mname": "ns2.google.com." 23 | } 24 | ], 25 | "type": "SOA", 26 | "name": "google.de.", 27 | "ttl": 60, 28 | "class": "IN" 29 | }, 30 | "NS": { 31 | "rdata": [ 32 | { 33 | "target": "ns1.google.com." 34 | }, 35 | { 36 | "target": "ns4.google.com." 37 | }, 38 | { 39 | "target": "ns2.google.com." 40 | }, 41 | { 42 | "target": "ns3.google.com." 43 | } 44 | ], 45 | "type": "NS", 46 | "name": "google.de.", 47 | "ttl": 345600, 48 | "class": "IN" 49 | }, 50 | "A": { 51 | "rdata": [ 52 | { 53 | "address": "172.217.19.99" 54 | } 55 | ], 56 | "type": "A", 57 | "name": "google.de.", 58 | "ttl": 300, 59 | "class": "IN" 60 | }, 61 | "auth": [ 62 | "ns1.google.com.", 63 | "216.239.32.10" 64 | ], 65 | "TXT": { 66 | "rdata": [ 67 | { 68 | "strings": "\"v=spf1 -all\"" 69 | } 70 | ], 71 | "type": "TXT", 72 | "name": "google.de.", 73 | "ttl": 300, 74 | "class": "IN" 75 | }, 76 | "AAAA": { 77 | "rdata": [ 78 | { 79 | "address": "2a00:1450:4016:802::2003" 80 | } 81 | ], 82 | "type": "AAAA", 83 | "name": "google.de.", 84 | "ttl": 300, 85 | "class": "IN" 86 | }, 87 | "MX": { 88 | "rdata": [ 89 | { 90 | "preference": 10, 91 | "exchange": "aspmx.l.google.com." 92 | }, 93 | { 94 | "preference": 40, 95 | "exchange": "alt3.aspmx.l.google.com." 96 | }, 97 | { 98 | "preference": 50, 99 | "exchange": "alt4.aspmx.l.google.com." 100 | }, 101 | { 102 | "preference": 20, 103 | "exchange": "alt1.aspmx.l.google.com." 104 | }, 105 | { 106 | "preference": 30, 107 | "exchange": "alt2.aspmx.l.google.com." 108 | } 109 | ], 110 | "type": "MX", 111 | "name": "google.de.", 112 | "ttl": 600, 113 | "class": "IN" 114 | } 115 | } 116 | ``` 117 | 118 | ## Usage 119 | 120 | Build and start the docker container using the included Dockerfile. 121 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/cfgangr/cfgangr.py: -------------------------------------------------------------------------------- 1 | # imports for tornado 2 | import tornado 3 | from tornado import web, httpserver, ioloop 4 | 5 | # imports for logging 6 | import traceback 7 | import os 8 | from os import path 9 | 10 | # imports for cfgangr 11 | import convertbinary 12 | 13 | #imports for reading configuration file 14 | import json 15 | 16 | # reading configuration file 17 | def ServiceConfig(filename): 18 | configPath = filename 19 | try: 20 | config = json.loads(open(configPath).read()) 21 | return config 22 | except FileNotFoundError: 23 | raise tornado.web.HTTPError(500) 24 | 25 | Config = ServiceConfig("./service.conf") 26 | 27 | Metadata = { 28 | "Name" : "CFGAngr", 29 | "Version" : "1.0", 30 | "Description" : "./README.md", 31 | "License" : "./LICENSE" 32 | } 33 | 34 | 35 | def CFGAngrRun(binary, size): 36 | # Returns the CFG of a binary in JSON format 37 | data = convertbinary.generateCFG(binary, size) 38 | return data 39 | 40 | 41 | class CFGAngrProcess(tornado.web.RequestHandler): 42 | def get(self): 43 | try: 44 | filename = self.get_argument("obj", strip=False) 45 | fullPath = os.path.join('/tmp/', filename) 46 | data = CFGAngrRun(fullPath, Config["settings"]["max_size_binary"]) 47 | self.write(data) 48 | except tornado.web.MissingArgumentError: 49 | raise tornado.web.HTTPError(400) 50 | except Exception as e: 51 | self.write({"error": traceback.format_exc(e)}) 52 | 53 | 54 | class Info(tornado.web.RequestHandler): 55 | # Emits a string which describes the purpose of the analytics 56 | def get(self): 57 | info = """ 58 |

{name:s} - {version:s}

59 |
60 |

{description:s}

61 |
62 |

{license:s} 63 | """.format( 64 | name = str(Metadata["Name"]).replace("\n", "
"), 65 | version = str(Metadata["Version"]).replace("\n", "
"), 66 | description = str(Metadata["Description"]).replace("\n", "
"), 67 | license = str(Metadata["License"]).replace("\n", "
"), 68 | ) 69 | self.write(info) 70 | 71 | 72 | class CFGAngrApp(tornado.web.Application): 73 | def __init__(self): 74 | for key in ["Description", "License"]: 75 | fpath = Metadata[key] 76 | if os.path.isfile(fpath): 77 | with open(fpath) as file: 78 | Metadata[key] = file.read() 79 | 80 | handlers = [ 81 | (r'/', Info), 82 | (r'/analyze/', CFGAngrProcess), 83 | ] 84 | settings = dict( 85 | template_path=path.join(path.dirname(__file__), 'templates'), 86 | static_path=path.join(path.dirname(__file__), 'static'), 87 | ) 88 | tornado.web.Application.__init__(self, handlers, **settings) 89 | self.engine = None 90 | 91 | 92 | def main(): 93 | server = tornado.httpserver.HTTPServer(CFGAngrApp()) 94 | server.listen(Config["settings"]["port"]) 95 | try: 96 | tornado.ioloop.IOLoop.current().start() 97 | except KeyboardInterrupt: 98 | tornado.ioloop.IOLoop.current().stop() 99 | 100 | if __name__ == '__main__': 101 | main() 102 | -------------------------------------------------------------------------------- /config/totem.conf.example: -------------------------------------------------------------------------------- 1 | totem { 2 | version = "0.5.0" 3 | 4 | download_settings { 5 | connection_pooling = true 6 | connection_timeout = 1000 7 | download_directory = "/tmp/" 8 | thread_multiplier = 4 9 | request_timeout = 1000 10 | validate_ssl_cert = true 11 | } 12 | 13 | tasking_settings { 14 | default_service_timeout = 180 15 | prefetch = 3 16 | retry_attempts = 3 17 | } 18 | 19 | rabbit_settings { 20 | requeueKey = "requeue.static.totem" 21 | host { 22 | server = "127.0.0.1" 23 | port = 5672 24 | username = "guest" 25 | password = "guest" 26 | vhost = "/" 27 | } 28 | exchange { 29 | name = "totem" 30 | type = "topic" 31 | durable = true 32 | } 33 | workqueue { 34 | name = "totem_input" 35 | routing_key = "work.static.totem" 36 | durable = true 37 | exclusive = false 38 | autodelete = false 39 | } 40 | resultsqueue { 41 | name = "totem_output" 42 | routing_key = "*.result.static.totem" 43 | durable = true 44 | exclusive = false 45 | autodelete = false 46 | } 47 | misbehavequeue { 48 | name = "totem_misbehave" 49 | routing_key = "misbehave.static.totem" 50 | durable = true 51 | exclusive = false 52 | autodelete = false 53 | } 54 | } 55 | 56 | services { 57 | asnmeta { 58 | uri = ["http://127.0.0.1:9700/analyze/?obj="] 59 | resultRoutingKey = "asnmeta.result.static.totem" 60 | } 61 | dnsmeta { 62 | uri = ["http://127.0.0.1:9710/analyze/?obj="] 63 | resultRoutingKey = "dnsmeta.result.static.totem" 64 | } 65 | passivetotal { 66 | uri = ["http://127.0.0.1:9720/analyze/?obj="] 67 | resultRoutingKey = "passivetotal.result.static.totem" 68 | } 69 | shodan { 70 | uri = ["http://127.0.0.1:9730/analyze/?obj="] 71 | resultRoutingKey = "shodan.result.static.totem" 72 | } 73 | gogadget { 74 | uri = ["http://127.0.0.1:7700/analyze/?obj="] 75 | resultRoutingKey = "gogadget.result.static.totem" 76 | } 77 | objdump { 78 | uri = ["http://127.0.0.1:7710/analyze/?obj="] 79 | resultRoutingKey = "objdump.result.static.totem" 80 | } 81 | peid { 82 | uri = ["http://127.0.0.1:7720/analyze/?obj="] 83 | resultRoutingKey = "peid.result.static.totem" 84 | } 85 | peinfo { 86 | uri = ["http://127.0.0.1:7730/analyze/?obj="] 87 | resultRoutingKey = "peinfo.result.static.totem" 88 | } 89 | richheader { 90 | uri = ["http://127.0.0.1:7740/analyze/?obj="] 91 | resultRoutingKey = "richheader.result.static.totem" 92 | } 93 | virustotal { 94 | uri = ["http://127.0.0.1:7750/analyze/?obj="] 95 | resultRoutingKey = "virustotal.result.static.totem" 96 | } 97 | yara { 98 | uri = ["http://127.0.0.1:7760/analyze/?obj="] 99 | resultRoutingKey = "yara.result.static.totem" 100 | } 101 | zipmeta { 102 | uri = ["http://127.0.0.1:7770/analyze/?obj="] 103 | resultRoutingKey = "zipmeta.result.static.totem" 104 | } 105 | pdfparse { 106 | uri = ["http://127.0.0.1:7780/analyze/?obj="] 107 | resultRoutingKey = "pdfparse.result.static.totem" 108 | } 109 | cfg { 110 | uri = ["http://127.0.0.1:7790/analyze/?obj="] 111 | resultRoutingKey = "cfg.result.static.totem" 112 | } 113 | cfgangr { 114 | uri = ["http://127.0.0.1:7800/analyze/?obj="] 115 | resultRoutingKey = "cfgangr.result.static.totem" 116 | } 117 | pemeta { 118 | uri = ["http://127.0.0.1:7810/analyze/?obj="] 119 | resultRoutingKey = "pemeta.result.static.totem" 120 | } 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/zipmeta/holmeslibrary/files.py: -------------------------------------------------------------------------------- 1 | import mmap 2 | # import tempfile 3 | # import shutil 4 | # import os 5 | 6 | class LargeFileReader (object): 7 | """ 8 | FOr mapping file to virtual memory. 9 | File-like (read-only) object trimmed for low memory footprint. 10 | Reading and finding does not advance the offset. 11 | Usage: 12 | # open 13 | file = LargeFileReader("/filepath") 14 | # find 15 | start = file.find("needle") 16 | # access data, still offset 0 17 | file[32987000:2323493493] 18 | # create a subfile at offset start 19 | subfile = file.subfile(start) 20 | # find a needle somewhere after the offset, relative to the offset 21 | position = subfile.find("second needle") 22 | # adjust offset in the subfile to after the previous find 23 | subfile.seek_relative(position+1) 24 | """ 25 | __slots__ = ["file","datamap","size","offset"] 26 | def __init__ (self, filename): 27 | self.file = open(filename, "rb") 28 | self.datamap = mmap.mmap(self.file.fileno(), 0, access=mmap.ACCESS_READ) 29 | self.offset = 0 30 | self.size = self.datamap.size() 31 | 32 | def close (self): 33 | self.datamap.close() 34 | del(self.datamap) 35 | self.file.close() 36 | del(self.file) 37 | del(self.offset) 38 | del(self.size) 39 | del(self) 40 | 41 | # provide base functionality 42 | def read (self, start, stop): 43 | # def read(self): 44 | if start is None or start < 0: 45 | start = 0 46 | if stop is None or stop > self.size: 47 | stop = self.size 48 | if start >= self.size: 49 | start = self.size - 1 50 | if stop > self.size: 51 | stop = self.size 52 | self.datamap.seek(0) 53 | return self.datamap[(self.offset+start):(self.offset+stop)] 54 | 55 | def seek (self, position): 56 | self.offset = min(self.size, max(0, position)) 57 | self.datamap.seek(self.offset) 58 | def seek_relative (self, offset): 59 | self.seek(self.offset + offset) 60 | 61 | def tell (self): 62 | return self.offset 63 | def tell_map (self): 64 | return self.datamap.tell() 65 | 66 | def find (self, needle): 67 | self.datamap.seek(0) 68 | result = self.datamap.find(needle, self.offset) 69 | if result != -1: 70 | result -= self.offset 71 | return result 72 | 73 | def startswith (self, needle): 74 | return self[0:len(needle)].decode('UTF-8') == needle 75 | 76 | # extended slicing 77 | def __getitem__ (self, key): 78 | if isinstance(key, slice): 79 | return self.read(key.start, key.stop) 80 | else: 81 | return self.read(key.start, key.start+1) 82 | 83 | def subfile (self, start): 84 | class LargeFileSubReader (LargeFileReader): 85 | __slots__ = ["file","datamap","size","offset"] 86 | # lightweight subtype of LargeFileReader offering adjusted offset 87 | def __init__ (self, file, datamap, start, size): 88 | self.file = file 89 | self.datamap = datamap 90 | self.size = size 91 | self.offset = start 92 | def close (self): 93 | pass # remove close ability 94 | def subfile (self, start): 95 | pass # remove subfile ability 96 | return LargeFileSubReader(self.file, self.datamap, self.offset+start, self.size) 97 | 98 | # provide standard functions 99 | def __len__ (self): 100 | return self.size 101 | 102 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/dnsmeta/dnsmeta.py: -------------------------------------------------------------------------------- 1 | # imports for tornado 2 | import tornado 3 | from tornado import web, httpserver, ioloop 4 | 5 | # imports for logging 6 | import traceback 7 | import os 8 | from os import path 9 | 10 | # imports for DNSMeta 11 | import gatherdns 12 | from time import localtime, strftime 13 | 14 | #imports for reading configuration file 15 | import json 16 | 17 | # reading configuration file 18 | def ServiceConfig(filename): 19 | configPath = filename 20 | try: 21 | config = json.loads(open(configPath).read()) 22 | return config 23 | except FileNotFoundError: 24 | raise tornado.web.HTTPError(500) 25 | 26 | Config = ServiceConfig("./service.conf") 27 | 28 | Metadata = { 29 | "Name" : "DNSMeta", 30 | "Version" : "1.0", 31 | "Description" : "./README.md", 32 | "License" : "./LICENSE" 33 | } 34 | 35 | 36 | def DNSMetaRun(domain): 37 | data = {} 38 | 39 | dnsinfo = gatherdns.GatherDNS(domain, Config["dnsmeta"]["dns_server"]) 40 | data['auth'] = dnsinfo.find_authoritative_nameserver(domain) 41 | 42 | # query for specified types and add to dictionary 43 | dnsinfo.query_domain(Config["dnsmeta"]["rdtypes"]) 44 | for rdtype in Config["dnsmeta"]["rdtypes"]: 45 | function = getattr(dnsinfo, 'get_{}_record'.format(rdtype)) 46 | result = function() 47 | if result is not None: 48 | data[rdtype] = result 49 | 50 | return data 51 | 52 | 53 | class DNSMetaProcess(tornado.web.RequestHandler): 54 | def get(self): 55 | try: 56 | domain = self.get_argument('obj', strip=False) 57 | data = DNSMetaRun(domain) 58 | self.write(data) 59 | except tornado.web.MissingArgumentError: 60 | raise tornado.web.HTTPError(400) 61 | except gatherdns.DomainError: 62 | raise tornado.web.HTTPError(404) 63 | except Exception as e: 64 | self.write({"error": traceback.format_exc(e)}) 65 | 66 | 67 | class Info(tornado.web.RequestHandler): 68 | # Emits a string which describes the purpose of the analytics 69 | def get(self): 70 | info = """ 71 |

{name:s} - {version:s}

72 |
73 |

{description:s}

74 |
75 |

{license:s} 76 | """.format( 77 | name = str(Metadata["Name"]).replace("\n", "
"), 78 | version = str(Metadata["Version"]).replace("\n", "
"), 79 | description = str(Metadata["Description"]).replace("\n", "
"), 80 | license = str(Metadata["License"]).replace("\n", "
"), 81 | ) 82 | self.write(info) 83 | 84 | 85 | class DNSApp(tornado.web.Application): 86 | def __init__(self): 87 | for key in ["Description", "License"]: 88 | fpath = Metadata[key] 89 | if os.path.isfile(fpath): 90 | with open(fpath) as file: 91 | Metadata[key] = file.read() 92 | 93 | handlers = [ 94 | (r'/', Info), 95 | (r'/analyze/', DNSMetaProcess), 96 | ] 97 | settings = dict( 98 | template_path=path.join(path.dirname(__file__), 'templates'), 99 | static_path=path.join(path.dirname(__file__), 'static'), 100 | ) 101 | tornado.web.Application.__init__(self, handlers, **settings) 102 | self.engine = None 103 | 104 | 105 | def main(): 106 | server = tornado.httpserver.HTTPServer(DNSApp()) 107 | server.listen(Config["settings"]["httpbinding"]) 108 | try: 109 | tornado.ioloop.IOLoop.current().start() 110 | except KeyboardInterrupt: 111 | tornado.ioloop.IOLoop.current().stop() 112 | 113 | if __name__ == '__main__': 114 | main() 115 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/yara/YaraREST.scala: -------------------------------------------------------------------------------- 1 | package org.holmesprocessing.totem.services.yara 2 | 3 | import dispatch.Defaults._ 4 | import dispatch.{url, _} 5 | import org.json4s.JsonAST.{JString, JValue} 6 | import org.holmesprocessing.totem.types.{TaskedWork, WorkFailure, WorkResult, WorkSuccess} 7 | import collection.mutable 8 | 9 | /** 10 | * YaraWork case class. Performs the actual enrichment and data parsing. Returns a series of 11 | * @param key: Long => The message key associated with this work. 12 | * @param filename: String => The filename that refers to this work's target instance on disk. 13 | * @param TimeoutMillis: Int => The timeout for this job, assuming it is not already set by the implicit HTTP client 14 | * @param WorkType: String => The type of work that needs to be performed. 15 | * @param Worker: String => The Path that will operate on this work. This is generally an external URI, but can be a local resource 16 | * or actor as well. 17 | * @param Arguments: List[String] => A list of secondary arguments that are associated with this particular work element. 18 | * 19 | * @constructor Create a new YaraWork. 20 | * 21 | */ 22 | 23 | case class YaraWork(key: Long, filename: String, TimeoutMillis: Int, WorkType: String, Worker: String, Arguments: List[String]) extends TaskedWork { 24 | def doWork()(implicit myHttp: dispatch.Http): Future[WorkResult] = { 25 | 26 | // Parameters will be send via Post so we dont need the builder here 27 | //val uri = YaraREST.constructURL(Worker, filename, Arguments) 28 | var req = url(Worker+filename) 29 | if(!Arguments.isEmpty){ 30 | // since the Arguments are passed as list at the moment 31 | // (this will be changed later) we assume that the 32 | // first Argument is the base64 encoded AND compiled 33 | // yara rule we want to pass to the service. 34 | val params = Map("custom_rule" -> Arguments.head) 35 | req = req <<(params) 36 | } 37 | 38 | val requestResult = myHttp(req OK as.String) 39 | .either 40 | .map({ 41 | case Right(content) => 42 | YaraSuccess(true, JString(content), Arguments) //should we parse content? yes. we should convert to a map structure? 43 | case Left(StatusCode(404)) => 44 | YaraFailure(false, JString("Not found"), Arguments) //should our key here be a boolean for success, or failure at the coordinator level? 45 | case Left(StatusCode(code)) => 46 | YaraFailure(false, JString("Some other code: " + code.toString), Arguments) 47 | case Left(something) => 48 | YaraFailure(false, JString("wildcard failure: " + something.toString), Arguments) 49 | }) 50 | requestResult 51 | } 52 | } 53 | 54 | /** 55 | * YaraResult case class. Holds the tasking and worker path appropriate for this TaskedWork 56 | * @param status: Boolean => If this work completed successfully. We are not discriminating between a failure on the service, or a failure in the coordinator 57 | * @param data: String => The results of the worker. All workers MUST return some sane JSON element. Handling of this JSON element 58 | * is to be done at DB ingest time. 59 | * @constructor Create a new YaraResult. 60 | * 61 | */ 62 | case class YaraSuccess(status: Boolean, data: JValue, Arguments: List[String], routingKey: String = "yara.result.static.totem", WorkType: String = "YARA") extends WorkSuccess //want to add a time of completion? might also need to change ID to the original taskedwork 63 | case class YaraFailure(status: Boolean, data: JValue, Arguments: List[String], routingKey: String = "", WorkType: String = "YARA") extends WorkFailure //want to add a time of completion? might also need to change ID to the original taskedwork 64 | 65 | object YaraREST { 66 | def constructURL(root: String, filename: String, arguments: List[String]): String = { 67 | arguments.foldLeft(new mutable.StringBuilder(root+filename))({ 68 | (acc, e) => acc.append(e)}).toString() 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/peid/PEiDREST.scala: -------------------------------------------------------------------------------- 1 | package org.holmesprocessing.totem.services.peid 2 | 3 | import dispatch.Defaults._ 4 | import dispatch.{url, _} 5 | import org.json4s.JsonAST.{JString, JValue} 6 | import org.holmesprocessing.totem.types.{TaskedWork, WorkFailure, WorkResult, WorkSuccess} 7 | import collection.mutable 8 | 9 | /** 10 | * YaraWork case class. Performs the actual enrichment and data parsing. Returns a series of 11 | * @param key: Long => The message key associated with this work. 12 | * @param filename: String => The filename that refers to this work's target instance on disk. 13 | * @param TimeoutMillis: Int => The timeout for this job, assuming it is not already set by the implicit HTTP client 14 | * @param WorkType: String => The type of work that needs to be performed. 15 | * @param Worker: String => The Path that will operate on this work. This is generally an external URI, but can be a local resource 16 | * or actor as well. 17 | * @param Arguments: List[String] => A list of secondary arguments that are associated with this particular work element. 18 | * 19 | * @constructor Create a new YaraWork. 20 | * 21 | */ 22 | 23 | case class PEiDWork(key: Long, filename: String, TimeoutMillis: Int, WorkType: String, Worker: String, Arguments: List[String]) extends TaskedWork { 24 | def doWork()(implicit myHttp: dispatch.Http): Future[WorkResult] = { 25 | 26 | // Parameters will be send via Post so we dont need the builder here 27 | //val uri = YaraREST.constructURL(Worker, filename, Arguments) 28 | var req = url(Worker+filename) 29 | if(!Arguments.isEmpty){ 30 | // since the Arguments are passed as list at the moment 31 | // (this will be changed later) we assume that the 32 | // first Argument is the base64 encoded AND compiled 33 | // yara rule we want to pass to the service. 34 | val params = Map("custom_rule" -> Arguments.head) 35 | req = req <<(params) 36 | } 37 | 38 | val requestResult = myHttp(req OK as.String) 39 | .either 40 | .map({ 41 | case Right(content) => 42 | PEiDSuccess(true, JString(content), Arguments) //should we parse content? yes. we should convert to a map structure? 43 | case Left(StatusCode(404)) => 44 | PEiDFailure(false, JString("Not found"), Arguments) //should our key here be a boolean for success, or failure at the coordinator level? 45 | case Left(StatusCode(code)) => 46 | PEiDFailure(false, JString("Some other code: " + code.toString), Arguments) 47 | case Left(something) => 48 | PEiDFailure(false, JString("wildcard failure: " + something.toString), Arguments) 49 | }) 50 | requestResult 51 | } 52 | } 53 | 54 | /** 55 | * YaraResult case class. Holds the tasking and worker path appropriate for this TaskedWork 56 | * @param status: Boolean => If this work completed successfully. We are not discriminating between a failure on the service, or a failure in the coordinator 57 | * @param data: String => The results of the worker. All workers MUST return some sane JSON element. Handling of this JSON element 58 | * is to be done at DB ingest time. 59 | * @constructor Create a new YaraResult. 60 | * 61 | */ 62 | 63 | //need to take the WorkType from our configuration file 64 | case class PEiDSuccess(status: Boolean, data: JValue, Arguments: List[String], routingKey: String = "peid.result.static.totem", WorkType: String = "PEID") extends WorkSuccess //want to add a time of completion? might also need to change ID to the original taskedwork 65 | case class PEiDFailure(status: Boolean, data: JValue, Arguments: List[String], routingKey: String = "", WorkType: String = "PEID") extends WorkFailure //want to add a time of completion? might also need to change ID to the original taskedwork 66 | 67 | object PEiDREST { 68 | def constructURL(root: String, filename: String, arguments: List[String]): String = { 69 | arguments.foldLeft(new mutable.StringBuilder(root+filename))({ 70 | (acc, e) => acc.append(e)}).toString() 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/richheader/rich.py: -------------------------------------------------------------------------------- 1 | # imports for tornado 2 | import tornado 3 | from tornado import web, httpserver, ioloop 4 | 5 | # imports for logging 6 | import traceback 7 | import os 8 | from os import path 9 | 10 | # imports for rich 11 | import richlibrary 12 | 13 | #imports for reading configuration file 14 | import json 15 | 16 | # Reading configuration file 17 | def ServiceConfig(filename): 18 | configPath = filename 19 | try: 20 | config = json.loads(open(configPath).read()) 21 | return config 22 | except FileNotFoundError: 23 | raise tornado.web.HTTPError(500) 24 | 25 | # Get service meta information and configuration 26 | Config = ServiceConfig("./service.conf") 27 | 28 | Metadata = { 29 | "Name" : "Rich Header", 30 | "Version" : "1.0", 31 | "Description" : "./README.md", 32 | "Copyright" : "Copyright 2017 Holmes Group LLC", 33 | "License" : "./LICENSE" 34 | } 35 | 36 | 37 | def RichHeaderRun(objpath): 38 | parser = richlibrary.RichLibrary(objpath) 39 | return parser.parse() 40 | 41 | 42 | class Service(tornado.web.RequestHandler): 43 | def get(self): 44 | try: 45 | filename = self.get_argument("obj", strip=False) 46 | fullPath = os.path.join('/tmp/', filename) 47 | data = RichHeaderRun(fullPath) 48 | self.write(data) 49 | except tornado.web.MissingArgumentError: 50 | raise tornado.web.HTTPError(400) 51 | except richlibrary.MZSignatureError: 52 | self.write({'error': richlibrary.err2str(-2)}) 53 | except richlibrary.PESignatureError: 54 | self.write({'error': richlibrary.err2str(-3)}) 55 | except richlibrary.RichSignatureError: 56 | self.write({'error': richlibrary.err2str(-4)}) 57 | except richlibrary.DanSSignatureError: 58 | self.write({'error': richlibrary.err2str(-5)}) 59 | except richlibrary.PaddingError: 60 | self.write({'error': richlibrary.err2str(-6)}) 61 | except richlibrary.RichLengthError: 62 | self.write({'error': richlibrary.err2str(-7)}) 63 | except Exception as e: 64 | self.write({"error": traceback.format_exc(e)}) 65 | 66 | 67 | class Info(tornado.web.RequestHandler): 68 | # Emits a string which describes the purpose of the analytics 69 | def get(self): 70 | info = """ 71 |

{name:s} - {version:s}

72 |
73 |

{description:s}

74 |
75 |

{license:s} 76 | """.format( 77 | name = str(Metadata["Name"]).replace("\n", "
"), 78 | version = str(Metadata["Version"]).replace("\n", "
"), 79 | description = str(Metadata["Description"]).replace("\n", "
"), 80 | license = str(Metadata["License"]).replace("\n", "
") 81 | ) 82 | self.write(info) 83 | 84 | 85 | class Application(tornado.web.Application): 86 | def __init__(self): 87 | 88 | for key in ["Description", "License"]: 89 | fpath = Metadata[key] 90 | if os.path.isfile(fpath): 91 | with open(fpath) as file: 92 | Metadata[key] = file.read() 93 | 94 | handlers = [ 95 | (r'/', Info), 96 | (r'/analyze/', Service), 97 | ] 98 | settings = dict( 99 | template_path=path.join(path.dirname(__file__), 'templates'), 100 | static_path=path.join(path.dirname(__file__), 'static'), 101 | ) 102 | tornado.web.Application.__init__(self, handlers, **settings) 103 | self.engine = None 104 | 105 | 106 | def main(): 107 | server = tornado.httpserver.HTTPServer(Application()) 108 | server.listen(Config["settings"]["httpbinding"]) 109 | try: 110 | tornado.ioloop.IOLoop.current().start() 111 | except KeyboardInterrupt: 112 | tornado.ioloop.IOLoop.current().stop() 113 | 114 | 115 | if __name__ == '__main__': 116 | main() 117 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/asnmeta/asnmeta.py: -------------------------------------------------------------------------------- 1 | # imports for tornado 2 | import tornado 3 | from tornado import web, httpserver, ioloop 4 | 5 | # imports for logging 6 | import traceback 7 | import os 8 | from os import path 9 | 10 | #import for parsing configuration file 11 | import json 12 | 13 | # imports for ASNMeta 14 | import gatherasn 15 | from time import localtime, strftime 16 | 17 | # reading configuration file 18 | def ServiceConfig(filename): 19 | configPath = filename 20 | try: 21 | config = json.loads(open(configPath).read()) 22 | return config 23 | except FileNotFoundError: 24 | raise tornado.web.HTTPError(500) 25 | 26 | Config = ServiceConfig("./service.conf") 27 | Metadata = { 28 | "Name" : "ASNMeta", 29 | "Version" : "1.0", 30 | "Description" : "./README.md", 31 | "License" : "./LICENSE" 32 | } 33 | 34 | 35 | def ASNMetaRun(ipaddress): 36 | asninfo = gatherasn.GatherASN(ipaddress, 37 | Config["asnmeta"]["dns_server"], 38 | Config["asnmeta"]["asn_ipv4_query"], 39 | Config["asnmeta"]["asn_ipv6_query"], 40 | Config["asnmeta"]["asn_peer_query"], 41 | Config["asnmeta"]["asn_name_query"]) 42 | 43 | asninfo.query_asn_origin() 44 | 45 | if asninfo.get_ip_version() == 4: 46 | asninfo.query_asn_peer() 47 | 48 | if asninfo.get_asn_number(): 49 | asninfo.query_asn_name('AS{}'.format(asninfo.get_asn_number())) 50 | 51 | return asninfo.get_all_known_data() 52 | 53 | 54 | class ASNMetaProcess(tornado.web.RequestHandler): 55 | def get(self): 56 | try: 57 | ipaddress = self.get_argument('obj', strip=False) 58 | data = ASNMetaRun(ipaddress) 59 | self.write(data) 60 | except tornado.web.MissingArgumentError: 61 | raise tornado.web.HTTPError(400) 62 | except gatherasn.IPTypeError: 63 | raise tornado.web.HTTPError(400) 64 | except gatherasn.IPFormatError: 65 | raise tornado.web.HTTPError(404) 66 | except Exception as e: 67 | self.write({"error": traceback.format_exc(e)}) 68 | 69 | 70 | class Info(tornado.web.RequestHandler): 71 | # Emits a string which describes the purpose of the analytics 72 | def get(self): 73 | info = """ 74 |

{name:s} - {version:s}

75 |
76 |

{description:s}

77 |
78 |

{license:s} 79 | """.format( 80 | name = str(Metadata["Name"]).replace("\n", "
"), 81 | version = str(Metadata["Version"]).replace("\n", "
"), 82 | description = str(Metadata["Description"]).replace("\n", "
"), 83 | license = str(Metadata["License"]).replace("\n", "
") 84 | ) 85 | self.write(info) 86 | 87 | 88 | class ASNApp(tornado.web.Application): 89 | def __init__(self): 90 | for key in ["Description", "License"]: 91 | fpath = Metadata[key] 92 | if os.path.isfile(fpath): 93 | with open(fpath) as file: 94 | Metadata[key] = file.read() 95 | 96 | handlers = [ 97 | (r'/', Info), 98 | (r'/analyze/', ASNMetaProcess), 99 | ] 100 | settings = dict( 101 | template_path=path.join(path.dirname(__file__), 'templates'), 102 | static_path=path.join(path.dirname(__file__), 'static'), 103 | ) 104 | tornado.web.Application.__init__(self, handlers, **settings) 105 | self.engine = None 106 | 107 | 108 | def main(): 109 | server = tornado.httpserver.HTTPServer(ASNApp()) 110 | server.listen(Config["settings"]["httpbinding"]) 111 | try: 112 | tornado.ioloop.IOLoop.current().start() 113 | except KeyboardInterrupt: 114 | tornado.ioloop.IOLoop.current().stop() 115 | 116 | if __name__ == '__main__': 117 | main() 118 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/types/CommandTypes.scala: -------------------------------------------------------------------------------- 1 | package org.holmesprocessing.totem.types 2 | 3 | import org.holmesprocessing.totem.util.DownloadSettings 4 | 5 | /** 6 | * Create() case class. Used between the ConsumerActor and the WorkGroup. 7 | * @param config: DownloadSettings => The configuration options for downloading an object 8 | * @param key: Long => The message key associated with this work. 9 | * @param primaryURI: String => The primary URL for downloading the target resource 10 | * @param secondaryURI: String => The secondary URL for downloading the target resource 11 | * @param value: WorkState => The state of the Job, and component work at time of creation 12 | * 13 | * @constructor Generate a Create message. This is used to initiate the creation of a WorkActor 14 | * 15 | */ 16 | case class Create(key: Long, download: Boolean, primaryURI: String, secondaryURI: String, tags: List[String], value: WorkState, config: DownloadSettings) 17 | 18 | /** 19 | * Result case class. Used between the WorkActor and the ProducerActor 20 | * @param filename: String => The filename representing the target of this Job 21 | * @param result: WorkResult => The WorkResult representing the end state of the Job 22 | * 23 | * @constructor Generate a Result message. This is used to transmit results to the Producer and from there, to the queueing backbone 24 | * 25 | */ 26 | case class Result(filename: String, result: WorkResult) 27 | 28 | /** 29 | * ResultPackage case class. A package of results for transmission. The various filehashes are used for secondary aggregation against 30 | * the target processing file, that information is lost due to the UUID usage in the temporary filestore. 31 | * @param filename: String => The filename representing the target of this Job 32 | * @param results: Iterable[WorkResult] => An Iterable representing the WorkResults to be transmitted 33 | * @param tags: List[String] => A list of tags for the results 34 | * @param MD5: String => MD5 hash of the target file. 35 | * @param SHA1: String => SHA1 hash of the target file. 36 | * @param SHA256: String => SHA256 hash of the target file. 37 | * 38 | * @constructor Generate a ResultPackage message. This is for multiple WorkResult transfers. 39 | * 40 | */ 41 | case class ResultPackage(filename: String, results: Iterable[WorkResult], tags: List[String], MD5: String, SHA1: String, SHA256: String) 42 | 43 | object WorkState { 44 | def create(filename: String, hashfilename: String, workToDo: List[TaskedWork], results: List[WorkResult] = List[WorkResult](), attempts: Int): WorkState = { 45 | WorkState(filename, hashfilename, workToDo, 0, 0, results, attempts) 46 | } 47 | } 48 | 49 | /** 50 | * WorkState case class. A representation of the current state of a given Job. This can be used to merge Jobs, or transfer overall Job state. 51 | * @param filename: String => The filename representing the target of this Job. 52 | * @param hashfilename: String => The hashed filename representing the target of the Job. 53 | * @param workToDo: List[TaskedWork] => A list of all TaskedWork elements, which are the component Work elements 54 | * @param created: Int => The time this work was created. 55 | * @param lastEdited: Int => The last time this work had an altered state. 56 | * @param results: List[WorkResult] => A list of the WorkResults that have been generated. 57 | * @param attempts: Int => The number of times this Job has been attempted across all executors. 58 | 59 | * @constructor Generate a WorkState message. 60 | * 61 | */ 62 | case class WorkState( 63 | filename: String, 64 | hashfilename: String, 65 | workToDo: List[TaskedWork], 66 | created: Int = 0, 67 | lastEdited: Int = 0, 68 | results: List[WorkResult] = List[WorkResult](), 69 | attempts: Int = 0 70 | ) { 71 | def isComplete: Boolean = { 72 | workToDo.size == results.size 73 | } 74 | def +(that: WorkResult): WorkState = { 75 | new WorkState( 76 | filename = this.filename, 77 | hashfilename = this.hashfilename, 78 | workToDo = this.workToDo, 79 | created = this.created, 80 | lastEdited = 1, 81 | results = this.results :+ that, 82 | attempts = this.attempts 83 | ) 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/richheader/standalone.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys, struct 4 | 5 | SIZE_DOS_HEADER = 0x40 6 | 7 | u32 = lambda x: struct.unpack("> (32 - (n & 0x1f))) 10 | 11 | def parse(fname): 12 | 13 | dat = open(fname, 'rb').read() 14 | 15 | ## Do basic sanity checks on the PE 16 | if dat[0:][:2] != b'MZ': 17 | return {'err': -2} 18 | 19 | e_lfanew = u32(dat[0x3c:][:4]) 20 | if dat[e_lfanew:][:2] != b'PE': 21 | return {'err': -3} 22 | 23 | ## IMPORTANT: Do not assume the data to start at 0x80, this is not always 24 | ## the case (modified DOS stub). Instead, start searching backwards for 25 | ## 'Rich', stop at beginning of DOS header. 26 | rich = 0 27 | for rich in range(e_lfanew, SIZE_DOS_HEADER, -1): 28 | if dat[rich:][:4] == b'Rich': 29 | break 30 | 31 | if rich == SIZE_DOS_HEADER: 32 | return {'err': -4} 33 | 34 | ## We found a valid 'Rich' signature in the header from here on 35 | csum = u32(dat[rich + 4:][:4]) 36 | 37 | ## xor backwards with csum until either 'DanS' or end of the DOS header, 38 | ## invert the result to get original order 39 | upack = [ u32(dat[i:][:4]) ^ csum for i in range(rich - 4, SIZE_DOS_HEADER, -4) ][::-1] 40 | if u32(b'DanS') not in upack: 41 | return {'err:': -5} 42 | 43 | upack = upack[upack.index(u32(b'DanS')):] 44 | dans = e_lfanew - len(upack) * 4 - (e_lfanew - rich) 45 | 46 | ## DanS is _always_ followed by three zero dwords 47 | if not all([upack[i] == 0 for i in range(1, 4)]): 48 | return {'err': -6} 49 | 50 | upack = upack[4:] 51 | 52 | if len(upack) & 1: 53 | return {'err': -7} 54 | 55 | cmpids = [] 56 | 57 | ## Bonus feature: Calculate and check the check sum csum 58 | chk = dans 59 | for i in range(dans): 60 | ## Mask out the e_lfanew field as it's not initialized yet 61 | if i in range(0x3c, 0x40): 62 | continue 63 | chk += rol32(dat[i], i) 64 | 65 | for i in range(0, len(upack), 2): 66 | cmpids.append({ 67 | 'mcv': (upack[i + 0] >> 0) & 0xffff, 68 | 'pid': (upack[i + 0] >> 16) & 0xffff, 69 | 'cnt': (upack[i + 1] >> 0) 70 | }) 71 | ## Exclude the "generic file" descriptor from the check sum 72 | #if cmpids[-1]['mcv'] & 0xffff != 0: 73 | chk += rol32(upack[i + 0], upack[i + 1]) 74 | 75 | ## Truncate calculated checksum to 32 bit 76 | chk &= 0xffffffff 77 | 78 | return {'err': 0, 'cmpids': cmpids, 'csum_calc': chk, 'csum_file': csum, 79 | 'offset': dans} 80 | 81 | def err2str(code): 82 | if code == -2: 83 | return "MZ signature not found" 84 | elif code == -3: 85 | return "PE signature not found" 86 | elif code == -4: 87 | return "Rich signature not found. This file probably has no Rich header." 88 | elif code == -5: 89 | return "DanS signature not found. Rich header corrupt." 90 | elif code == -6: 91 | return "Wrong header padding behind DanS signature. Rich header corrupt." 92 | elif code == -7: 93 | return "Rich data length not a multiple of 8. Rich header corrupt." 94 | else: 95 | return "--- NO ERROR DESCRIPTION ---" 96 | 97 | def pprint_cmpids(cmpids): 98 | print("-" * (20 + 16 + 16)) 99 | print("{:>20s}{:>16s}{:>16s}".format("Compiler Version", "Product ID", 100 | "Count")) 101 | print("-" * (20 + 16 + 16)) 102 | 103 | for e in cmpids: 104 | print("{:>20s}{:>16s}{:>16s}".format( 105 | "{:5d}".format(e['mcv']), 106 | "0x{:04x}".format(e['pid']), 107 | "0x{:08x}".format(e['cnt']))) 108 | print("-" * (20 + 16 + 16)) 109 | 110 | def pprint_header(data): 111 | pprint_cmpids(data['cmpids']) 112 | if rich['csum_calc'] == rich['csum_file']: 113 | print("\x1b[32mChecksums match! (0x{:08x})".format(rich['csum_calc'])) 114 | else: 115 | print("\x1b[33mChecksum corrupt! (calc 0x{:08x}, file " 116 | "0x{:08x})".format(rich['csum_calc'], rich['csum_file'])) 117 | print("\x1b[39m" + "-" * (20 + 16 + 16)) 118 | 119 | 120 | if __name__ == "__main__": 121 | if len(sys.argv) != 2: 122 | print("Usage: {} ".format(sys.argv[0])) 123 | sys.exit(-1) 124 | rich = parse(sys.argv[1]) 125 | if rich['err'] < 0: 126 | print("\x1b[33m[-] " + err2str(rich['err']) + "\x1b[39m") 127 | sys.exit(rich['err']) 128 | 129 | pprint_header(rich) 130 | -------------------------------------------------------------------------------- /config/docker-compose.yml.example: -------------------------------------------------------------------------------- 1 | version: '2' 2 | services: 3 | asnmeta: 4 | build: 5 | context: ../src/main/scala/org/holmesprocessing/totem/services/asnmeta 6 | args: 7 | conf: ${CONFSTORAGE_ASNMETA}service.conf 8 | ports: 9 | - "9700:8080" 10 | restart: unless-stopped 11 | 12 | dnsmeta: 13 | build: 14 | context: ../src/main/scala/org/holmesprocessing/totem/services/dnsmeta 15 | args: 16 | conf: ${CONFSTORAGE_DNSMETA}service.conf 17 | ports: 18 | - "9710:8080" 19 | restart: unless-stopped 20 | 21 | passivetotal: 22 | build: 23 | context: ../src/main/scala/org/holmesprocessing/totem/services/passivetotal 24 | args: 25 | conf: ${CONFSTORAGE_PASSIVETOTAL}service.conf 26 | ports: 27 | - "9720:8080" 28 | restart: unless-stopped 29 | 30 | shodan: 31 | build: 32 | context: ../src/main/scala/org/holmesprocessing/totem/services/shodan 33 | args: 34 | conf: ${CONFSTORAGE_SHODAN}service.conf 35 | ports: 36 | - "9730:8080" 37 | restart: unless-stopped 38 | 39 | gogadget: 40 | build: 41 | context: ../src/main/scala/org/holmesprocessing/totem/services/gogadget 42 | args: 43 | conf: ${CONFSTORAGE_GOGADGET}service.conf 44 | ports: 45 | - "7700:8080" 46 | restart: unless-stopped 47 | volumes: 48 | - /tmp:/tmp:ro 49 | 50 | objdump: 51 | build: 52 | context: ../src/main/scala/org/holmesprocessing/totem/services/objdump 53 | args: 54 | conf: ${CONFSTORAGE_OBJDUMP}service.conf 55 | ports: 56 | - "7710:8080" 57 | restart: unless-stopped 58 | volumes: 59 | - /tmp:/tmp:ro 60 | 61 | peid: 62 | build: 63 | context: ../src/main/scala/org/holmesprocessing/totem/services/peid 64 | args: 65 | conf: ${CONFSTORAGE_PEID}service.conf 66 | ports: 67 | - "7720:8080" 68 | restart: unless-stopped 69 | volumes: 70 | - /tmp:/tmp:ro 71 | 72 | peinfo: 73 | build: 74 | context: ../src/main/scala/org/holmesprocessing/totem/services/peinfo 75 | args: 76 | conf: ${CONFSTORAGE_PEINFO}service.conf 77 | ports: 78 | - "7730:8080" 79 | restart: unless-stopped 80 | volumes: 81 | - /tmp:/tmp:ro 82 | 83 | richheader: 84 | build: 85 | context: ../src/main/scala/org/holmesprocessing/totem/services/richheader 86 | args: 87 | conf: ${CONFSTORAGE_RICHHEADER}service.conf 88 | ports: 89 | - "7740:8080" 90 | restart: unless-stopped 91 | volumes: 92 | - /tmp:/tmp:ro 93 | 94 | virustotal: 95 | build: 96 | context: ../src/main/scala/org/holmesprocessing/totem/services/virustotal 97 | args: 98 | conf: ${CONFSTORAGE_VIRUSTOTAL}service.conf 99 | ports: 100 | - "7750:8080" 101 | restart: unless-stopped 102 | volumes: 103 | - /tmp:/tmp:ro 104 | 105 | yara: 106 | build: 107 | context: ../src/main/scala/org/holmesprocessing/totem/services/yara 108 | args: 109 | conf: ${CONFSTORAGE_YARA}service.conf 110 | ports: 111 | - "7760:8080" 112 | restart: unless-stopped 113 | volumes: 114 | - /tmp:/tmp:ro 115 | 116 | zipmeta: 117 | build: 118 | context: ../src/main/scala/org/holmesprocessing/totem/services/zipmeta 119 | args: 120 | conf: ${CONFSTORAGE_ZIPMETA}service.conf 121 | ports: 122 | - "7770:8080" 123 | restart: unless-stopped 124 | volumes: 125 | - /tmp:/tmp:ro 126 | 127 | pdfparse: 128 | build: 129 | context: ../src/main/scala/org/holmesprocessing/totem/services/pdfparse 130 | args: 131 | conf: ${CONFSTORAGE_PDFPARSE}service.conf 132 | ports: 133 | - "7780:8080" 134 | restart: unless-stopped 135 | volumes: 136 | - /tmp:/tmp:ro 137 | 138 | # cfg: 139 | # build: 140 | # context: ../src/main/scala/org/holmesprocessing/totem/services/cfg 141 | # args: 142 | # conf: ${CONFSTORAGE_CFG}service.conf 143 | # ports: 144 | # - "7790:8080" 145 | # restart: unless-stopped 146 | # volumes: 147 | # - /tmp:/tmp:ro 148 | 149 | cfgangr: 150 | build: 151 | context: ../src/main/scala/org/holmesprocessing/totem/services/cfgangr 152 | args: 153 | conf: ${CONFSTORAGE_CFGANGR}service.conf 154 | ports: 155 | - "7800:8080" 156 | restart: unless-stopped 157 | volumes: 158 | - /tmp:/tmp:ro 159 | 160 | pemeta: 161 | build: 162 | context: ../src/main/scala/org/holmesprocessing/totem/services/pemeta 163 | args: 164 | conf: ${CONFSTORAGE_PEMETA}service.conf 165 | ports: 166 | - "7810:8080" 167 | restart: unless-stopped 168 | volumes: 169 | - /tmp:/tmp:ro 170 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/peid/peid_worker.py: -------------------------------------------------------------------------------- 1 | # imports for tornado 2 | import tornado 3 | from tornado import web, httpserver, ioloop 4 | 5 | # imports for logging 6 | import traceback 7 | import os 8 | from os import path 9 | 10 | # imports for yara to work 11 | from io import BytesIO 12 | import base64 13 | import yara 14 | 15 | # imports for reading configuration file 16 | import json 17 | 18 | # Reading configuration file 19 | def ServiceConfig(filename): 20 | configPath = filename 21 | try: 22 | config = json.loads(open(configPath).read()) 23 | return config 24 | except FileNotFoundError: 25 | raise tornado.web.HTTPError(500) 26 | 27 | Config = ServiceConfig("./service.conf") 28 | 29 | Metadata = { 30 | "Name" : "PEiD", 31 | "Version" : "1.0", 32 | "Description" : "./README.md", 33 | "Copyright" : "Copyright 2016 Holmes Group LLC", 34 | "License" : "./LICENSE" 35 | } 36 | 37 | class YaraHandler(tornado.web.RequestHandler): 38 | @property 39 | def YaraEngine(self): 40 | return yara.load(Config["yara_rules"]["local_path"]) 41 | 42 | class PEiDProcess(YaraHandler): 43 | def process(self, filename, rules=None): 44 | try: 45 | if rules: 46 | ruleBuff = BytesIO() 47 | ruleBuff.write(rules) 48 | ruleBuff.seek(0) 49 | rules = yara.load(file=ruleBuff) 50 | results = rules.match(filename[0], externals={'filename': filename[1]}) 51 | else: 52 | results = self.YaraEngine.match(filename[0], externals={'filename': filename[1]}) 53 | results2 = list(map(lambda x: {"rule": x.rule}, results)) 54 | return results2 55 | except yara.Error: 56 | # Rules are uncompiled -> compile them 57 | rules = yara.compile(source=rules.decode('latin-1')) 58 | results = rules.match(filename[0], externals={'filename': filename[1]}) 59 | results2 = list(map(lambda x: {"rule": x.rule}, results)) 60 | return results2 61 | except Exception as e: 62 | return e 63 | 64 | def get(self): 65 | try: 66 | filename = self.get_argument("obj", strip=False) 67 | fullPath = (os.path.join('/tmp/', filename), filename) 68 | data = self.process(fullPath) 69 | self.write({"peid": data}) 70 | except tornado.web.MissingArgumentError: 71 | raise tornado.web.HTTPError(400) 72 | except Exception as e: 73 | self.write({"error": traceback.format_exc(e)}) 74 | 75 | def post(self): 76 | try: 77 | filename = self.get_argument("obj", strip=False) 78 | fullPath = (os.path.join('/tmp/', filename), filename) 79 | rules = base64.b64decode(self.get_body_argument('custom_rule')) 80 | data = self.process(fullPath, rules) 81 | self.write({"peid": data}) 82 | except tornado.web.MissingArgumentError: 83 | raise tornado.web.HTTPError(400) 84 | except Exception as e: 85 | self.write({"error": traceback.format_exc(e)}) 86 | 87 | 88 | class Info(tornado.web.RequestHandler): 89 | # Emits a string which describes the purpose of the analytics 90 | def get(self): 91 | info = """ 92 |

{name:s} - {version:s}

93 |
94 |

{description:s}

95 |
96 |

{license:s} 97 | """.format( 98 | name = str(Metadata["Name"]).replace("\n", "
"), 99 | version = str(Metadata["Version"]).replace("\n", "
"), 100 | description = str(Metadata["Description"]).replace("\n", "
"), 101 | license = str(Metadata["License"]).replace("\n", "
") 102 | ) 103 | self.write(info) 104 | 105 | 106 | class PEiDApp(tornado.web.Application): 107 | def __init__(self): 108 | for key in ["Description", "License"]: 109 | fpath = Metadata[key] 110 | if os.path.isfile(fpath): 111 | with open(fpath) as file: 112 | Metadata[key] = file.read() 113 | 114 | handlers = [ 115 | (r'/', Info), 116 | (r'/analyze/', PEiDProcess), 117 | ] 118 | settings = dict( 119 | template_path=path.join(path.dirname(__file__), 'templates'), 120 | static_path=path.join(path.dirname(__file__), 'static') 121 | ) 122 | tornado.web.Application.__init__(self, handlers, **settings) 123 | self.engine = None 124 | 125 | 126 | def main(): 127 | server = tornado.httpserver.HTTPServer(PEiDApp()) 128 | server.listen(Config["settings"]["httpbinding"]) 129 | try: 130 | tornado.ioloop.IOLoop.current().start() 131 | except KeyboardInterrupt: 132 | tornado.ioloop.IOLoop.current().stop() 133 | 134 | 135 | if __name__ == '__main__': 136 | main() 137 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/yara/yara_worker.py: -------------------------------------------------------------------------------- 1 | # imports for tornado 2 | import tornado 3 | from tornado import web, httpserver, ioloop 4 | 5 | # imports for logging 6 | import traceback 7 | import os 8 | from os import path 9 | 10 | # imports for yara to work 11 | from io import BytesIO 12 | import base64 13 | import yara 14 | 15 | #imports for reading configuration file 16 | import json 17 | 18 | # Reading configuration file 19 | def ServiceConfig(filename): 20 | configPath = filename 21 | try: 22 | config = json.loads(open(configPath).read()) 23 | return config 24 | except FileNotFoundError: 25 | raise tornado.web.HTTPError(500) 26 | 27 | # Get service meta information and configuration 28 | Config = ServiceConfig("./service.conf") 29 | 30 | Metadata = { 31 | "Name" : "Yara", 32 | "Version" : "1.0", 33 | "Description" : "./README.md", 34 | "Copyright" : "Copyright 2016 Holmes Group LLC", 35 | "License" : "./LICENSE" 36 | } 37 | 38 | class YaraHandler(tornado.web.RequestHandler): 39 | @property 40 | def YaraEngine(self): 41 | return yara.load(Config["yara_rules"]["local_path"]) 42 | 43 | class YaraProcess(YaraHandler): 44 | def process(self, filename, rules=None): 45 | try: 46 | if rules: 47 | ruleBuff = BytesIO() 48 | ruleBuff.write(rules) 49 | ruleBuff.seek(0) 50 | rules = yara.load(file=ruleBuff) 51 | results = rules.match(filename[0], externals={'filename': filename[1]}) 52 | else: 53 | results = self.YaraEngine.match(filename[0], externals={'filename': filename[1]}) 54 | results2 = list(map(lambda x: {"rule": x.rule}, results)) 55 | return results2 56 | except yara.Error: 57 | # Rules are uncompiled -> compile them 58 | rules = yara.compile(source=rules.decode('latin-1')) 59 | results = rules.match(filename[0], externals={'filename': filename[1]}) 60 | results2 = list(map(lambda x: {"rule": x.rule}, results)) 61 | return results2 62 | except Exception as e: 63 | return e 64 | 65 | def get(self): 66 | try: 67 | filename = self.get_argument("obj", strip=False) 68 | fullPath = (os.path.join('/tmp/', filename), filename) 69 | data = self.process(fullPath) 70 | self.write({"yara": data}) 71 | except tornado.web.MissingArgumentError: 72 | raise tornado.web.HTTPError(400) 73 | except Exception as e: 74 | self.write({"error": traceback.format_exc(e)}) 75 | 76 | def post(self): 77 | try: 78 | filename = self.get_argument("obj", strip=False) 79 | fullPath = (os.path.join('/tmp/', filename), filename) 80 | rules = base64.b64decode(self.get_body_argument('custom_rule')) 81 | data = self.process(fullPath, rules) 82 | self.write({"yara": data}) 83 | except tornado.web.MissingArgumentError: 84 | raise tornado.web.HTTPError(400) 85 | except Exception as e: 86 | self.write({"error": traceback.format_exc(e)}) 87 | 88 | 89 | class Info(tornado.web.RequestHandler): 90 | # Emits a string which describes the purpose of the analytics 91 | def get(self): 92 | info = """ 93 |

{name:s} - {version:s}

94 |
95 |

{description:s}

96 |
97 |

{license:s} 98 | """.format( 99 | name = str(Metadata["Name"]).replace("\n", "
"), 100 | version = str(Metadata["Version"]).replace("\n", "
"), 101 | description = str(Metadata["Description"]).replace("\n", "
"), 102 | license = str(Metadata["License"]).replace("\n", "
") 103 | ) 104 | self.write(info) 105 | 106 | 107 | class YaraApp(tornado.web.Application): 108 | def __init__(self): 109 | 110 | for key in ["Description", "License"]: 111 | fpath = Metadata[key] 112 | if os.path.isfile(fpath): 113 | with open(fpath) as file: 114 | Metadata[key] = file.read() 115 | 116 | handlers = [ 117 | (r'/', Info), 118 | (r'/analyze/', YaraProcess), 119 | ] 120 | settings = dict( 121 | template_path=path.join(path.dirname(__file__), 'templates'), 122 | static_path=path.join(path.dirname(__file__), 'static') 123 | ) 124 | tornado.web.Application.__init__(self, handlers, **settings) 125 | self.engine = None 126 | 127 | 128 | def main(): 129 | server = tornado.httpserver.HTTPServer(YaraApp()) 130 | server.listen(Config["settings"]["httpbinding"]) 131 | try: 132 | tornado.ioloop.IOLoop.current().start() 133 | except KeyboardInterrupt: 134 | tornado.ioloop.IOLoop.current().stop() 135 | 136 | 137 | 138 | if __name__ == '__main__': 139 | main() 140 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/richheader/richlibrary.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys, struct 4 | 5 | class MZSignatureError(Exception): 6 | pass 7 | 8 | class PESignatureError(Exception): 9 | pass 10 | 11 | class RichSignatureError(Exception): 12 | pass 13 | 14 | class DanSSignatureError(Exception): 15 | pass 16 | 17 | class PaddingError(Exception): 18 | pass 19 | 20 | class RichLengthError(Exception): 21 | pass 22 | 23 | class FileReadError(Exception): 24 | pass 25 | 26 | def err2str(code): 27 | return{ 28 | -2: "MZ signature not found", 29 | -3: "PE signature not found", 30 | -4: "Rich signature not found. This file probably has no Rich header.", 31 | -5: "DanS signature not found. Rich header corrupt.", 32 | -6: "Wrong header padding behind DanS signature. Rich header corrupt.", 33 | -7: "Rich data length not a multiple of 8. Rich header corrupt.", 34 | }[code] 35 | 36 | class RichLibrary: 37 | 38 | def __u32(self, x): 39 | return struct.unpack("> (32 - (n & 0x1f))) 46 | 47 | def generate_csum(self, raw_dat, compids, off): 48 | csum = off 49 | 50 | for i in range(off): 51 | ## Mask out the e_lfanew field as it's not initialized yet 52 | if i in range(0x3c, 0x40): 53 | continue 54 | csum += self.__rol32(raw_dat[i], i) 55 | 56 | for c in compids: 57 | csum += self.__rol32(c['pid'] << 16 | c['mcv'], c['cnt']) 58 | 59 | ## Truncate calculated checksum to 32 bit 60 | return csum & 0xffffffff 61 | 62 | def parse(self): 63 | dat = open(self.fname, 'rb').read() 64 | 65 | ## Do basic sanity checks on the PE 66 | if dat[0:][:2] != b'MZ': 67 | raise MZSignatureError() 68 | 69 | e_lfanew = self.__u32(dat[0x3c:][:4]) 70 | if dat[e_lfanew:][:2] != b'PE': 71 | raise PESignatureError() 72 | 73 | ## IMPORTANT: Do not assume the data to start at 0x80, this is not always 74 | ## the case (modified DOS stub). Instead, start searching backwards for 75 | ## 'Rich', stop at beginning of DOS header. 76 | rich = 0 77 | for rich in range(e_lfanew, self.SIZE_DOS_HEADER, -1): 78 | if dat[rich:][:4] == b'Rich': 79 | break 80 | 81 | if rich == self.SIZE_DOS_HEADER: 82 | raise RichSignatureError() 83 | 84 | ## We found a valid 'Rich' signature in the header from here on 85 | csum = self.__u32(dat[rich + 4:][:4]) 86 | 87 | ## xor backwards with csum until either 'DanS' or end of the DOS header, 88 | ## invert the result to get original order 89 | upack = [ self.__u32(dat[i:][:4]) ^ csum for i in range(rich - 4, self.SIZE_DOS_HEADER, -4) ][::-1] 90 | if self.__u32(b'DanS') not in upack: 91 | raise DanSSignatureError() 92 | 93 | upack = upack[upack.index(self.__u32(b'DanS')):] 94 | dans = e_lfanew - len(upack) * 4 - (e_lfanew - rich) 95 | 96 | ## DanS is _always_ followed by three zero dwords 97 | if not all([upack[i] == 0 for i in range(1, 4)]): 98 | raise PaddingError() 99 | 100 | upack = upack[4:] 101 | 102 | if len(upack) & 1: 103 | raise RichLengthError() 104 | 105 | cmpids = [] 106 | for i in range(0, len(upack), 2): 107 | cmpids.append({ 108 | 'mcv': (upack[i + 0] >> 0) & 0xffff, 109 | 'pid': (upack[i + 0] >> 16) & 0xffff, 110 | 'cnt': (upack[i + 1] >> 0) 111 | }) 112 | 113 | ## Bonus feature: Calculate and check the check sum csum 114 | chk = self.generate_csum(dat, cmpids, dans) 115 | 116 | return {'error': 0, 'cmpids': cmpids, 'csum_calc': chk, 'csum_file': csum, 117 | 'offset': dans} 118 | 119 | def __pprint_cmpids(self, cmpids): 120 | print("-" * (20 + 16 + 16)) 121 | print("{:>20s}{:>16s}{:>16s}".format("Compiler Version", "Product ID", 122 | "Count")) 123 | print("-" * (20 + 16 + 16)) 124 | 125 | for e in cmpids: 126 | print("{:>20s}{:>16s}{:>16s}".format( 127 | "{:5d}".format(e['mcv']), 128 | "0x{:04x}".format(e['pid']), 129 | "0x{:08x}".format(e['cnt']))) 130 | print("-" * (20 + 16 + 16)) 131 | 132 | def pprint_header(self, data): 133 | self.__pprint_cmpids(data['cmpids']) 134 | if rich['csum_calc'] == rich['csum_file']: 135 | print("\x1b[32mChecksums match! (0x{:08x})".format(rich['csum_calc'])) 136 | else: 137 | print("\x1b[33mChecksum corrupt! (calc 0x{:08x}, file " 138 | "0x{:08x})".format(rich['csum_calc'], rich['csum_file'])) 139 | print("\x1b[39m" + "-" * (20 + 16 + 16)) 140 | 141 | def __init__(self, path): 142 | self.data = {} 143 | self.SIZE_DOS_HEADER = 0x40 144 | 145 | self.fname = path 146 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Holmes-Totem: A Holmes Processing Investigation Planner for Large-scale File Analysis 2 | 3 | Contributions are always welcome and appreciated! If you would like to contribute please read the [official document](http://holmes-processing.readthedocs.io/en/latest/) and use the information in this CONTRIBUTING.md file as a guide. If you have questions or are unsure about something you would like to implement, please open a new issue. We would be happy to discuss the idea with you. 4 | 5 | ## Services 6 | New Services are simple to create and are always much appreciated. When implementing a Service you will need to provide: 7 | 8 | 1. A RESTful interface for TOTEM to interact with 9 | 2. Totem code to teach Totem how to handle the Service 10 | 11 | Additionally, please keep in mind that Totem strives to perform rapid parallel analysis. As such, these services should perform concise tasks that are relatively quick to perform. Static analysis and gathering data from 3rd parties are excellent candidates for new Services. When needing to perform long running tasks, such as dynamic analysis, please consider making a [Totem-Dynamic Service](https://github.com/HolmesProcessing/Holmes-Totem-Dynamic) instead. 12 | 13 | ### Core Components 14 | #### RESTful Endpoints 15 | The following endpoints are the standard and expected endpoints for totem and totem-dynamic: 16 | 17 | | Endpoint | Operation | System | 18 | | --- | --- | --- | 19 | | `/` | provide information about the service | Totem and Totem-Dynamic | 20 | | `/analyze/?obj=` | perform tasking and return results | Totem | 21 | | `/feed/?obj=` | submit tasking to the service | Totem-Dynamic | 22 | | `/check/?taskid=` | check to see if the tasking is complete | Totem-Dynamic | 23 | | `/results/?taskid=` | receive service results | Totem-Dynamic | 24 | | `/status/` | retrieve status | Totem-Dynamic | 25 | 26 | #### Docker 27 | We uses Docker and Docker-Compose to manage services. This provides a few nice benefits: keeps most issues from replicating, allowing for easier restart, easier status information, etc. However, to manage the overhead we request that the following DockerFile templates are used. This is because it speeds up the container build time and reduces the on-disk size. 28 | 29 | For Go: 30 | ```dockerfile 31 | FROM golang:alpine 32 | 33 | # create folder 34 | RUN mkdir -p /service 35 | WORKDIR /service 36 | 37 | # get go dependencies 38 | RUN apk add --no-cache \ 39 | git \ 40 | && go get github.com/julienschmidt/httprouter \ 41 | && rm -rf /var/cache/apk/* 42 | 43 | ### 44 | # [Service] specific options 45 | ### 46 | ... 47 | ``` 48 | 49 | For Python: 50 | ```dockerfile 51 | FROM python:alpine 52 | 53 | # add tornado 54 | RUN pip3 install tornado 55 | 56 | # create folder 57 | RUN mkdir -p /service 58 | WORKDIR /service 59 | 60 | # add holmeslibrary 61 | RUN apk add --no-cache \ 62 | wget \ 63 | && wget https://github.com/HolmesProcessing/Holmes-Totem-Service-Library/archive/v0.1.tar.gz \ 64 | && tar xf v0.1.tar.gz \ 65 | && mv Holmes-Totem-Service* holmeslibrary \ 66 | && rm -rf /var/cache/apk/* v0.1.tar.gz 67 | 68 | ### 69 | # [Service] specific options 70 | ### 71 | ... 72 | ``` 73 | 74 | ### Configuration 75 | 76 | #### Configuration File 77 | The Service configuration file should be written in JSON and named `service.conf.example`. 78 | 79 | The Totem should be configured and updated. Details on how to do this can be found in the official [documentation](http://holmes-processing.readthedocs.io/en/latest/). 80 | 81 | #### Port Selection 82 | Internal ports should always be `8080` when using Docker. 83 | 84 | External ports should be listed alphabetically starting with the following range: 85 | 86 | | Range | Service Type | 87 | | --- | --- | 88 | | 97xx | No File | 89 | | 72xx | File Based | 90 | 91 | ### Code Standards 92 | Services can be written in any language. However, we recommend using Go (with [httprouter](https://godoc.org/github.com/julienschmidt/httprouter)) or Python (with [Tornado](http://www.tornadoweb.org/en/stable/)) for the entire Service or at least the interface. The standard Docker Files will provide both packages. 93 | 94 | #### Standard Libraries 95 | The [Holmes Processing standard library](https://github.com/HolmesProcessing/Holmes-Totem-Service-Library) should be used when appropriate. It provides helpful functions for go and python. 96 | 97 | #### Language Style 98 | The code base of a Service should conform to the recommended style for the programming language. 99 | 100 | | Language | Style Documentation | Checking Tool | 101 | | --- | --- | --- | 102 | | Go | [Effective Go](https://golang.org/doc/effective_go.html) | `go fmt` | 103 | | Python | [PEP 9](http://pep8.org/) | [pycodestyle](https://github.com/PyCQA/pycodestyle) | 104 | 105 | ### Output 106 | The Service should return two outputs: HTTP codes and the results of the Service. 107 | 108 | #### HTTP Error Codes 109 | *work in progress* 110 | 111 | #### Results 112 | Results should be returned as sane JSON. All care should be given to return the data in a native format. For example, a Service for VirusTotal should return the original response by VirusTotal as it is already sane JSON. However, in cases when modification is needed, please provide an example in the README.md file. 113 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/asnmeta/gatherasn.py: -------------------------------------------------------------------------------- 1 | import dns 2 | import dns.name 3 | import dns.query 4 | import dns.resolver 5 | import ipaddress 6 | 7 | class IPFormatError(Exception): 8 | pass 9 | 10 | 11 | class IPTypeError(Exception): 12 | pass 13 | 14 | 15 | class GatherASN: 16 | def _reverse_address(self): 17 | if self.ip.version == 4: 18 | reverseip = str(self.ip).split('.')[::-1] 19 | else: 20 | reverseip = self.ip.exploded[::-1].replace(':', '') 21 | return '.'.join(reverseip) 22 | 23 | 24 | def _parse_results(self, data): 25 | return [out.strip() for out in data.rrset[0].strings[0].decode().split('|')] 26 | 27 | 28 | def _perform_query(self, domain, rdtype): 29 | """ 30 | Performs a DNS (UDP) query with flags and configurations parameters. 31 | 32 | Args: 33 | domain (str): sdomain i.e. google.com 34 | nnserver (str): nameserver to query 35 | rtype (str or int): rtype to query http://en.wikipedia.org/wiki/List_of_DNS_record_types 36 | timeout (int): time to wait before timeout 37 | 38 | """ 39 | domain = dns.name.from_text(domain) 40 | 41 | try: 42 | result = self.resolver.query(domain, rdtype=rdtype) 43 | except dns.resolver.NoAnswer: 44 | print("%s : The response did not contain a answer for %s." % (rdtype, domain)) 45 | except dns.resolver.NXDOMAIN: 46 | print("%s : The query name does not exist for %s." % (rdtype, domain)) 47 | except dns.resolver.Timeout: 48 | print("%s : The query could not be found in the specified lifetime for %s." % (rdtype, domain)) 49 | except dns.resolver.NoNameservers: 50 | print("%s : No non-broken nameservers are available to answer the question using nameserver %s" % (rdtype, domain)) 51 | else: 52 | print("%s : Queried %s successfully!" % (rdtype, domain)) 53 | return result 54 | return None 55 | 56 | 57 | def query_asn_name(self, asn): 58 | parsed_ip = "{0}.{1}".format(asn, self.servername) 59 | 60 | query_result = self._perform_query(parsed_ip, 'TXT') 61 | 62 | if query_result is not None: 63 | temp = self._parse_results(query_result) 64 | #self.data['asn_number'] = temp[0] 65 | #self.data['cc'] = temp[1] 66 | #self.data['registry'] = temp[2] 67 | self.data['data_allocated'] = temp[3] 68 | self.data['asn_name'] = temp[4] 69 | print(temp) 70 | 71 | 72 | def query_asn_origin(self): 73 | if self.ip.version == 4: 74 | parsed_ip = "{0}.{1}".format(self.reversed_ip, self.serverv4) 75 | elif self.ip.version == 6: 76 | parsed_ip = "{0}.{1}".format(self.reversed_ip, self.serverv6) 77 | 78 | query_result = self._perform_query(parsed_ip, 'TXT') 79 | 80 | if query_result is not None: 81 | temp = self._parse_results(query_result) 82 | self.data['asn_number'] = temp[0] 83 | self.data['bgp_prefix'] = temp[1] 84 | self.data['cc'] = temp[2] 85 | self.data['registry'] = temp[3] 86 | self.data['data_allocated'] = temp[4] 87 | print(temp) 88 | 89 | 90 | def query_asn_peer(self): 91 | parsed_ip = "{0}.{1}".format(self.reversed_ip, self.serverpeer) 92 | 93 | query_result = self._perform_query(parsed_ip, 'TXT') 94 | 95 | if query_result is not None: 96 | temp = self._parse_results(query_result) 97 | self.data['asn_peers'] = temp[0].split(' ') 98 | print(temp) 99 | 100 | 101 | def get_asn_name(self): 102 | return self.data.get('asn_name', None) 103 | 104 | 105 | def get_asn_number(self): 106 | return self.data.get('asn_number', None) 107 | 108 | 109 | def get_asn_peers(self): 110 | return self.data.get('asn_peers', None) 111 | 112 | 113 | def get_bgp_prefix(self): 114 | return self.data.get('bgp_prefix', None) 115 | 116 | 117 | def get_cc(self): 118 | return self.data.get('cc', None) 119 | 120 | 121 | def get_registry(self): 122 | return self.data.get('registry', None) 123 | 124 | 125 | def get_date_allocated(self): 126 | return self.data.get('data_allocated', None) 127 | 128 | 129 | def get_all_known_data(self): 130 | return self.data 131 | 132 | 133 | def get_ip(self): 134 | return str(self.ip) 135 | 136 | 137 | def get_ip_version(self): 138 | return self.ip.version 139 | 140 | 141 | def __init__(self, ip, nsserver, serverv4, serverv6, serverpeer, servername, timeout=10): 142 | self.resolver = dns.resolver.Resolver() 143 | self.resolver.nameservers = [nsserver] 144 | self.resolver.timeout = timeout 145 | self.resolver.lifetime = 50 146 | self.serverv4 = serverv4 147 | self.serverv6 = serverv6 148 | self.serverpeer = serverpeer 149 | self.servername = servername 150 | 151 | self.data = {} 152 | try: 153 | self.ip = ipaddress.ip_address(ip) 154 | 155 | # test to make sure the address is publicly accessible 156 | ### 157 | # TODO: upgrade to is_global when python 3.5+ is more stable 158 | ### 159 | if self.ip.is_private: 160 | raise IPTypeError() 161 | self.reversed_ip = self._reverse_address() 162 | except ValueError: 163 | raise IPFormatError() 164 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/zipmeta/zipmeta.py: -------------------------------------------------------------------------------- 1 | # imports for tornado 2 | import tornado 3 | from tornado import web, httpserver, ioloop 4 | 5 | # imports for logging 6 | import traceback 7 | import os 8 | from os import path 9 | 10 | # imports for zipmeta 11 | import ZipParser 12 | ZipParser = ZipParser.ZipParser 13 | 14 | # imports for largefile reader 15 | from holmeslibrary.files import LargeFileReader 16 | 17 | import json 18 | 19 | # Reading configuration file 20 | def ServiceConfig(filename): 21 | configPath = filename 22 | try: 23 | config = json.loads(open(configPath).read()) 24 | return config 25 | except FileNotFoundError: 26 | raise tornado.web.HTTPError(500) 27 | 28 | # Get service meta information and configuration 29 | Config = ServiceConfig("./service.conf") 30 | 31 | Metadata = { 32 | "Name" : "ZipMeta", 33 | "Version" : "1.0", 34 | "Description" : "./README.md", 35 | "Copyright" : "Copyright 2016 Holmes Group LLC", 36 | "License" : "./LICENSE" 37 | } 38 | 39 | 40 | # class ZipError (ServiceRequestError): 41 | # pass 42 | 43 | class ZipMetaProcess(tornado.web.RequestHandler): 44 | def get(self): 45 | # resultset = ServiceResultSet() 46 | resultset = {} 47 | try: 48 | filename = self.get_argument("obj", strip=False) 49 | # read file 50 | fullPath = os.path.join('/tmp/', filename) 51 | data = LargeFileReader(fullPath) # create virtual memory map 52 | # exclude non-zip 53 | if len(data) < 4: 54 | raise ZipError(400, "Not enough filedata.") 55 | 56 | if data[:4].decode('UTF-8') not in [ZipParser.zipLDMagic, ZipParser.zipCDMagic]: 57 | raise ZipError(400, "Not a zip file.") 58 | 59 | # parse 60 | parser = ZipParser(data) 61 | 62 | parsedZip = parser.parseZipFile() 63 | if not parsedZip: 64 | raise ZipError(400, "Could not parse file as a zip file") 65 | 66 | # clean up 67 | data.close() 68 | 69 | # fetch result 70 | resultset["filecount"] = len(parsedZip) 71 | for centralDirectory in parsedZip: 72 | zipfilename = centralDirectory["ZipFileName"] 73 | zipentry = {} 74 | 75 | # for name, value in centralDirectory.iteritems(): 76 | for name, value in centralDirectory.items(): 77 | if name == 'ZipExtraField': 78 | continue 79 | 80 | if type(value) is list or type(value) is tuple: 81 | for element in value: 82 | zipentry[name]=str(element) 83 | 84 | else: 85 | zipentry[name] = str(value) 86 | 87 | if centralDirectory["ZipExtraField"]: 88 | for dictionary in centralDirectory["ZipExtraField"]: 89 | zipextra = {} 90 | if dictionary["Name"] == "UnknownHeader": 91 | for name, value in dictionary.items(): 92 | if name == "Data": 93 | value = "Data" 94 | zipextra[name] = str(value) 95 | else: 96 | for name, value in dictionary.items(): 97 | zipextra[name]=str(value) 98 | else: 99 | zipentry["ZipExtraField"] = "None" 100 | 101 | resultset[zipfilename]= {**zipentry, **zipextra} 102 | 103 | self.write(resultset) 104 | except tornado.web.MissingArgumentError: 105 | raise tornado.web.HTTPError(400) 106 | except ZipError as ze: 107 | self.set_status(ze.status, str(ze.error)) 108 | self.write("") 109 | except Exception as e: 110 | self.set_status(500, str(e)) 111 | self.write({"error": traceback.format_exc(e)}) 112 | 113 | 114 | class Info(tornado.web.RequestHandler): 115 | # Emits a string which describes the purpose of the analytics 116 | def get(self): 117 | info = """ 118 |

{name:s} - {version:s}

119 |
120 |

{description:s}

121 |
122 |

{license:s} 123 | """.format( 124 | name = str(Metadata["Name"]).replace("\n", "
"), 125 | version = str(Metadata["Version"]).replace("\n", "
"), 126 | description = str(Metadata["Description"]).replace("\n", "
"), 127 | license = str(Metadata["License"]).replace("\n", "
") 128 | ) 129 | self.write(info) 130 | 131 | 132 | class ZipMetaApp(tornado.web.Application): 133 | def __init__(self): 134 | 135 | for key in ["Description", "License"]: 136 | fpath = Metadata[key] 137 | if os.path.isfile(fpath): 138 | with open(fpath) as file: 139 | Metadata[key] = file.read() 140 | 141 | handlers = [ 142 | (r'/', Info), 143 | (r'/analyze/', ZipMetaProcess), 144 | ] 145 | settings = dict( 146 | template_path=path.join(path.dirname(__file__), 'templates'), 147 | static_path=path.join(path.dirname(__file__), 'static'), 148 | ) 149 | tornado.web.Application.__init__(self, handlers, **settings) 150 | self.engine = None 151 | 152 | 153 | def main(): 154 | server = tornado.httpserver.HTTPServer(ZipMetaApp()) 155 | server.listen(Config["settings"]["httpbinding"]) 156 | try: 157 | tornado.ioloop.IOLoop.current().start() 158 | except KeyboardInterrupt: 159 | tornado.ioloop.IOLoop.current().stop() 160 | 161 | 162 | if __name__ == '__main__': 163 | main() 164 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/services/pdfparse/pdfparse.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "encoding/json" 6 | "flag" 7 | "fmt" 8 | "github.com/julienschmidt/httprouter" 9 | "io/ioutil" 10 | "log" 11 | "net/http" 12 | "os" 13 | "os/exec" 14 | "path/filepath" 15 | "strconv" 16 | "strings" 17 | ) 18 | 19 | var ( 20 | config *Config 21 | info *log.Logger 22 | pdfparse string 23 | metadata Metadata = Metadata{ 24 | Name: "pdfparse", 25 | Version: "0.1", 26 | Description: "./README.md", 27 | Copyright: "Copyright 2017 Holmes Group LLC", 28 | License: "./LICENSE", 29 | } 30 | ) 31 | 32 | // Result structs 33 | type Result struct { 34 | Truncated bool `json:"Truncated"` 35 | Comments int `json:"Comments"` 36 | XREF int `json:"XREF"` 37 | Trailer int `json:"Trailer"` 38 | StartXref int `json:"StartXref"` 39 | IndirectObject int `json:"IndirectObjects"` 40 | Objects []*Object `json:"Objects"` 41 | } 42 | 43 | type Object struct { 44 | Category string `json:"Category"` 45 | Values []int `json:"Values"` 46 | } 47 | 48 | 49 | // Config structs 50 | type Setting struct { 51 | HTTPBinding string `json:"HTTPBinding"` 52 | } 53 | 54 | type PDFPARSE struct { 55 | MaxNumberOfObjects int `json:"MaxNumberOfObjects"` 56 | } 57 | 58 | type Config struct { 59 | Settings Setting `json:"settings"` 60 | Pdfparse PDFPARSE `json:"pdfparse"` 61 | } 62 | 63 | type Metadata struct { 64 | Name string 65 | Version string 66 | Description string 67 | Copyright string 68 | License string 69 | } 70 | 71 | func main() { 72 | 73 | var ( 74 | err error 75 | configPath string 76 | ) 77 | info = log.New(os.Stdout, "", log.Ltime|log.Lshortfile) 78 | 79 | flag.StringVar(&configPath, "config", "", "Path to the configuration file") 80 | flag.Parse() 81 | 82 | config, err = load_config(configPath) 83 | if err != nil { 84 | log.Fatalln("Couldn't decode config file without errors!", err.Error()) 85 | } 86 | 87 | router := httprouter.New() 88 | router.GET("/analyze/", handler_analyze) 89 | router.GET("/", handler_info) 90 | info.Printf("Binding to %s\n", config.Settings.HTTPBinding) 91 | log.Fatal(http.ListenAndServe(config.Settings.HTTPBinding, router)) 92 | } 93 | 94 | func handler_info(f_response http.ResponseWriter, r *http.Request, ps httprouter.Params) { 95 | fmt.Fprintf(f_response, `

%s - %s

96 |
97 |

%s

98 |
99 |

%s

100 | `, 101 | metadata.Name, 102 | metadata.Version, 103 | metadata.Description, 104 | metadata.License) 105 | } 106 | 107 | func load_config(configPath string) (*Config, error) { 108 | config := &Config{} 109 | 110 | // if no path is supplied look in the current dir 111 | if configPath == "" { 112 | configPath, _ = filepath.Abs(filepath.Dir(os.Args[0])) 113 | configPath += "/service.conf" 114 | } 115 | 116 | cfile, _ := os.Open(configPath) 117 | if err := json.NewDecoder(cfile).Decode(&config); err != nil { 118 | return config, err 119 | } 120 | 121 | if metadata.Description != "" { 122 | if data, err := ioutil.ReadFile(string(metadata.Description)); err == nil { 123 | metadata.Description = strings.Replace(string(data), "\n", "
", -1) 124 | } 125 | } 126 | 127 | if metadata.License != "" { 128 | if data, err := ioutil.ReadFile(string(metadata.License)); err == nil { 129 | metadata.License = strings.Replace(string(data), "\n", "
", -1) 130 | } 131 | } 132 | 133 | return config, nil 134 | } 135 | 136 | func handler_analyze(f_response http.ResponseWriter, request *http.Request, params httprouter.Params) { 137 | obj := request.URL.Query().Get("obj") 138 | if obj == "" { 139 | http.Error(f_response, "Missing argument 'obj'", 400) 140 | return 141 | } 142 | sample_path := "/tmp/" + obj 143 | if _, err := os.Stat(sample_path); os.IsNotExist(err) { 144 | http.NotFound(f_response, request) 145 | //info.Printf("Error accessing sample (file: %s):", sample_path) 146 | //info.Println(err) 147 | return 148 | } 149 | process := exec.Command("pdfparse", "--stats", sample_path) 150 | 151 | stdout, err := process.StdoutPipe() 152 | if err != nil { 153 | fmt.Println(err) 154 | return 155 | } 156 | 157 | stdin, err := process.StdinPipe() 158 | if err != nil { 159 | fmt.Println(err) 160 | return 161 | } 162 | stdin.Close() 163 | 164 | if err := process.Start(); err != nil { 165 | fmt.Println(err) 166 | return 167 | } 168 | 169 | line := bufio.NewScanner(stdout) 170 | 171 | // taking first five lines 172 | var final [5]int 173 | for i := 0; i < 5; i++ { 174 | line.Scan() 175 | lineSplit := strings.Split(line.Text(), ": ") 176 | final[i], err = strconv.Atoi(lineSplit[1]) 177 | } 178 | 179 | result := &Result{ 180 | Truncated: false, 181 | Comments: final[0], 182 | XREF: final[1], 183 | Trailer: final[2], 184 | StartXref: final[3], 185 | IndirectObject: final[4], 186 | Objects: make([]*Object, config.Pdfparse.MaxNumberOfObjects), 187 | } 188 | 189 | counter := 0 190 | 191 | for line.Scan() { 192 | var values = []int{} 193 | lineSplit := strings.Split(line.Text(), ": ") 194 | name := lineSplit[0] 195 | value := strings.Split(lineSplit[1], ", ") 196 | 197 | // convert this array of strings to array of integers 198 | for _, i := range value { 199 | j, err := strconv.Atoi(i) 200 | if err != nil { 201 | panic(err) 202 | } 203 | values = append(values, j) 204 | } 205 | 206 | result.Objects[counter] = &Object{ 207 | Category: name[1 : len(name)-2], 208 | Values: values, 209 | } 210 | counter++ 211 | 212 | if counter == config.Pdfparse.MaxNumberOfObjects { 213 | result.Truncated = true 214 | break 215 | } 216 | } 217 | 218 | if result.Truncated { 219 | // if we reached the max amount of objects and breaked we need to throw the rest away 220 | for line.Text() != "" { 221 | line.Scan() 222 | } 223 | } else { 224 | // if not, we need to throw away the unused array slices 225 | result.Objects = result.Objects[:counter] 226 | } 227 | 228 | f_response.Header().Set("Content-Type", "text/json; charset=utf-8") 229 | json2http := json.NewEncoder(f_response) 230 | 231 | if err := json2http.Encode(result); err != nil { 232 | http.Error(f_response, "Generating JSON failed", 500) 233 | return 234 | } 235 | } 236 | -------------------------------------------------------------------------------- /src/main/scala/org/holmesprocessing/totem/types/WorkTypes.scala: -------------------------------------------------------------------------------- 1 | package org.holmesprocessing.totem.types 2 | 3 | import java.util.concurrent.ExecutorService 4 | 5 | import com.typesafe.config.Config 6 | import com.typesafe.scalalogging.Logger 7 | import org.json4s.JsonAST.{JString, JValue} 8 | import org.holmesprocessing.totem.services.yara.{YaraSuccess, YaraWork} 9 | import org.slf4j.LoggerFactory 10 | 11 | import scala.collection.JavaConversions._ 12 | import scala.concurrent.Future 13 | import scala.util.Random 14 | 15 | 16 | /** 17 | * This trait is inherited by all work tasks to be done by Totem. TaskedWork is the superclass which all typechecking and 18 | * matching is done against. 19 | * 20 | * Traits do not have constructors; this trait has the following values which all children must set 21 | * {{{ 22 | * val key: Long => The message key associated with this work 23 | * val filename: String => The filename that refers to this work's instance on disk 24 | * val WorkType: String => The type of work that needs to be performed 25 | * val Worker: String => The ActorPath that will operate on this work 26 | * }}} 27 | * 28 | * Currently, we require explicitly set actor paths for processing. This is slightly burdensome, but as these actor paths 29 | * can be specified in the overall system, this is not an excessive reqirement. 30 | * 31 | * To add a new analytic or enricher, you must 32 | * - Create an appropriate case class below, inheriting from TaskedWork 33 | * - Create an appropriate Result class below, inheriting from WorkResult 34 | * - Add that actor case class to the WorkEncoding.enumerateWork matching function 35 | * - Add the WorkResult classes to the WorkEncoding.workRoutingKey matching functions 36 | * - Specify an ActorPath 37 | * - Create an appropriate Actor (in the described path) to handle the request 38 | * - Deploy the actor in question 39 | * 40 | * @constructor None. This is a trait. 41 | * 42 | */ 43 | //object types { 44 | // type partial = String => Future[WorkResult] 45 | //} 46 | 47 | trait TaskedWork { 48 | val key: Long 49 | val filename: String //this serves as both a local filepath and URL type, 50 | val TimeoutMillis: Int 51 | val WorkType: String 52 | val Worker: String 53 | val Arguments: List[String] 54 | def doWork()(implicit myHttp: dispatch.Http): Future[WorkResult] 55 | } 56 | /** 57 | * HashWork case class. Holds the tasking and worker path appropriate for this TaskedWork 58 | * @param key: Long => The message key associated with this work 59 | * @param filename: String => The filename that refers to this work's instance on disk 60 | * @param WorkType: String => The type of work that needs to be performed 61 | * @param Worker: String => The ActorPath that will operate on this work 62 | * 63 | * @constructor Create a new HashWork. 64 | * 65 | */ 66 | 67 | case class UnsupportedWork(key: Long, filename: String, TimeoutMillis: Int, WorkType: String, Worker: String, Arguments: List[String]) extends TaskedWork { 68 | import scala.concurrent.ExecutionContext.Implicits.global //this makes me uncomfortable, but this is an edge case to begin with. 69 | def doWork()(implicit myHttp: dispatch.Http): Future[WorkResult] = { 70 | Future{UnsupportedFailure(false, JString(""), Arguments, "", WorkType)} 71 | } 72 | } 73 | case class UnsupportedFailure(status: Boolean, data: JValue, Arguments: List[String], routingKey: String, WorkType: String) extends WorkFailure 74 | 75 | /** 76 | * This trait is inherited by all work results generated by Totem. WorkResult is the superclass which all typechecking and 77 | * matching is done against. 78 | * 79 | * Traits do not have constructors; this trait has the following values which all children must set 80 | * {{{ 81 | * val key: Long => The message key associated with this work 82 | * val data: String => The results of the worker. All workers MUST return some sane JSON element. Handling of this JSON element 83 | * is to be done at DB ingest time. 84 | * }}} 85 | * 86 | * @constructor None. This is a trait. 87 | * 88 | */ 89 | trait WorkResult { 90 | val status: Boolean 91 | val data: JValue 92 | val routingKey: String 93 | val WorkType: String 94 | val Arguments: List[String] 95 | } 96 | 97 | abstract class WorkSuccess extends WorkResult 98 | abstract class WorkFailure extends WorkResult 99 | 100 | trait WorkEncoding { 101 | def GeneratePartial(work: String): String 102 | def enumerateWork(key: Long, orig_filename: String, uuid_filename: String, workToDo: Map[String, List[String]]): List[TaskedWork] 103 | def workRoutingKey(work: WorkResult): String 104 | } 105 | 106 | abstract class GenericTotemEncoding extends WorkEncoding { 107 | val services: Map[String, List[String]] 108 | val log: Logger 109 | def GeneratePartial(work: String): String 110 | def enumerateWork(key: Long, filename: String, workToDo: Map[String, List[String]]): List[TaskedWork] 111 | def workRoutingKey(work: WorkResult): String 112 | } 113 | 114 | // TODO: look at moving some of this to driver.scala 115 | case class TaskingSettings(default_service_timeout: Int, prefetch: Int, retry_attempts: Int) 116 | abstract class ConfigTotemEncoding(conf: Config) extends WorkEncoding { 117 | val keys = conf.getObject("totem.services").keySet() 118 | val en = conf.getObject("totem.services").toConfig 119 | val services = keys.map(key => 120 | (key, Random.shuffle(en.getStringList(s"$key.uri").toList)) 121 | ).toMap[String, List[String]] 122 | val log = Logger(LoggerFactory.getLogger("name")) 123 | 124 | def GeneratePartial(work: String): String 125 | def enumerateWork(key: Long, orig_filename: String, uuid_filename: String, workToDo: Map[String, List[String]]): List[TaskedWork] 126 | def workRoutingKey(work: WorkResult): String 127 | } 128 | 129 | trait Resolution { 130 | val status: Boolean 131 | } 132 | 133 | case class ConsumerResolution(status: Boolean) extends Resolution //if we ackked 134 | case class ResultResolution(status: Boolean) extends Resolution //if we transmitted the results 135 | case class RemainderResolution(status: Boolean) extends Resolution //if we transmitted the remainder 136 | case class LocalResolution(status: Boolean) extends Resolution //if all local actions are done 137 | case class NackResolution(status: Boolean) extends Resolution //if we nackked 138 | 139 | case class Conflict(consumer: Boolean, result: Boolean, remainder: Boolean, local: Boolean, nack: Boolean) { 140 | def +(that: Resolution): Conflict = that match { 141 | case ConsumerResolution(status: Boolean) => Conflict(status, this.result, this.remainder, this.local, this.nack) 142 | case ResultResolution(status: Boolean) => Conflict(this.consumer, status, this.remainder, this.local, this.nack) 143 | case RemainderResolution(status: Boolean) => Conflict(this.consumer, this.result, status, this.local, this.nack) 144 | case LocalResolution(status: Boolean) => Conflict(this.consumer, this.result, this.remainder, status, this.nack) 145 | case NackResolution(status: Boolean) => Conflict(this.consumer, this.result, this.remainder, this.local, status) 146 | } 147 | } 148 | --------------------------------------------------------------------------------