├── .github └── issue_template.md ├── .gitignore ├── AUTHORS ├── Dockerfile ├── LICENSE ├── README.md ├── core ├── app.py ├── conf │ ├── auth │ │ └── auth.conf │ ├── feeds │ │ ├── ET_IP.conf │ │ ├── abuseFeodo_IP.conf │ │ ├── abuseSSL_IP.conf │ │ ├── abuseZeus_DOMAIN.conf │ │ ├── abuseZeus_IP.conf │ │ ├── bambenek_c2cDOMAIN.conf │ │ ├── bambenek_c2cIP.conf │ │ ├── bambenek_dgaDOMAIN.conf │ │ ├── binarydefense_IP.conf │ │ ├── ciarmylist_IP.conf │ │ ├── dnsbh_DOMAIN.conf │ │ ├── dshield_high_DOMAIN.conf │ │ ├── dshield_low_DOMAIN.conf │ │ ├── dshield_medium_DOMAIN.conf │ │ ├── greensnow_IP.conf │ │ ├── hpHosts_emdDOMAIN.conf │ │ ├── hpHosts_expDOMAIN.conf │ │ ├── hpHosts_fsaDOMAIN.conf │ │ ├── hpHosts_grmDOMAIN.conf │ │ ├── hpHosts_hjkDOMAIN.conf │ │ ├── hpHosts_pshDOMAIN.conf │ │ ├── hpHosts_pupDOMAIN.conf │ │ ├── hpHosts_wrzDOMAIN.conf │ │ ├── malc0de_IP.conf │ │ ├── openphish_URL.conf │ │ ├── phishtank_URL.conf │ │ ├── shunlist_IP.conf │ │ ├── snort_IP.conf │ │ └── torexit_IP.conf │ └── hippo │ │ └── hippo.conf ├── examples │ ├── distinct.sh │ ├── freshness.sh │ ├── hipposched.sh │ ├── hipposcore.py │ ├── hipposcore.sh │ ├── jobs.sh │ ├── lastQuery.sh │ ├── lastStatus.sh │ ├── monitorSources.sh │ ├── more.sh │ ├── new.sh │ ├── schedReport.sh │ ├── shadowbook.sh │ ├── sizeBySource.sh │ ├── sizeByType.sh │ ├── sources.sh │ └── type.sh ├── services │ ├── __init__.py │ ├── countType.py │ ├── distinct.py │ ├── freshness.py │ ├── hipposched.py │ ├── hipposcore.py │ ├── jobs.py │ ├── lastQuery.py │ ├── lastStatus.py │ ├── modules │ │ ├── __init__.py │ │ ├── common │ │ │ ├── ES.py │ │ │ ├── __init__.py │ │ │ └── getConf.py │ │ ├── countType │ │ │ ├── BagOfIntel.py │ │ │ └── __init__.py │ │ ├── distinct │ │ │ ├── Field.py │ │ │ └── __init__.py │ │ ├── hipposcore │ │ │ ├── ExistingSource.py │ │ │ └── __init__.py │ │ ├── jobs │ │ │ ├── BagOfJobs.py │ │ │ └── __init__.py │ │ ├── more │ │ │ ├── ObjToEnrich.py │ │ │ └── __init__.py │ │ ├── new │ │ │ ├── BagOfNew.py │ │ │ └── __init__.py │ │ ├── shadowbook │ │ │ ├── __init__.py │ │ │ ├── bulkOp.py │ │ │ ├── createSessions.py │ │ │ ├── downloader.py │ │ │ ├── enricher.py │ │ │ ├── extendTools.py │ │ │ ├── objects │ │ │ │ ├── Index.py │ │ │ │ ├── IndexIntel.py │ │ │ │ ├── IndexJob.py │ │ │ │ ├── IndexNew.py │ │ │ │ ├── IndexSource.py │ │ │ │ ├── Intel.py │ │ │ │ ├── Job.py │ │ │ │ ├── NewIntel.py │ │ │ │ ├── ObjToIndex.py │ │ │ │ ├── Source.py │ │ │ │ └── __init__.py │ │ │ ├── parser.py │ │ │ ├── processFeed.py │ │ │ ├── processFeed.py.backup │ │ │ ├── processMsearch.py │ │ │ ├── searchIntel.py │ │ │ └── shadowBook.py.backup │ │ ├── sizeBySources │ │ │ ├── TypeES.py │ │ │ └── __init__.py │ │ └── sources │ │ │ ├── BagOfSources.py │ │ │ └── __init__.py │ ├── monitorSources.py │ ├── more.py │ ├── new.py │ ├── schedReport.py │ ├── shadowbook.py │ ├── sizeBySources.py │ ├── sizeByType.py │ ├── sources.py │ └── typeIntel.py ├── static │ ├── .gitignore │ ├── app │ │ ├── app.js │ │ ├── hipposcore │ │ │ ├── hipposcore.html │ │ │ └── module.js │ │ ├── home │ │ │ ├── home.html │ │ │ └── module.js │ │ ├── jobs │ │ │ ├── jobs.html │ │ │ └── module.js │ │ ├── main │ │ │ ├── main.html │ │ │ └── module.js │ │ ├── monitor │ │ │ ├── module.js │ │ │ └── monitor.html │ │ ├── more │ │ │ ├── module.js │ │ │ └── more.html │ │ ├── shadowbook │ │ │ ├── module.js │ │ │ └── shadowbook.html │ │ ├── sizeBySource │ │ │ ├── donutSizeBySource.html │ │ │ └── module.js │ │ ├── sizeByType │ │ │ ├── donutSizeByType.html │ │ │ └── module.js │ │ ├── sources │ │ │ ├── module.js │ │ │ └── sources.html │ │ └── type │ │ │ ├── module.js │ │ │ └── type.html │ ├── bower.json │ ├── fonts │ │ ├── glyphicons-halflings-regular.eot │ │ ├── glyphicons-halflings-regular.svg │ │ ├── glyphicons-halflings-regular.ttf │ │ ├── glyphicons-halflings-regular.woff │ │ └── glyphicons-halflings-regular.woff2 │ └── images │ │ ├── android-chrome-192x192.png │ │ ├── apple-touch-icon.png │ │ ├── browserconfig.xml │ │ ├── favicon-16x16.png │ │ ├── favicon-32x32.png │ │ ├── favicon.ico │ │ ├── hippocampe-horizontal-bright.png │ │ ├── hippocampe-logo.png │ │ ├── hippocampe-small.png │ │ ├── manifest.json │ │ ├── mstile-150x150.png │ │ └── safari-pinned-tab.svg └── templates │ ├── error.html │ └── index.html ├── docker-compose.yml ├── docker-entrypoint.sh ├── docs ├── api_guide.md ├── hipposcore.md ├── install_guide.md ├── pics │ ├── activate_deactivate.jpg │ ├── graph1.png │ ├── graph2.png │ ├── hipposcore.png │ ├── hipposcore_main_part.png │ ├── hipposcore_percentage.png │ ├── hipposcore_range.png │ ├── hipposcore_sign.png │ ├── n3.png │ ├── p.png │ └── variation_table.png └── tutorial.md └── images └── Hippocampe-logo.png /.github/issue_template.md: -------------------------------------------------------------------------------- 1 | # EDIT THIS TITLE BEFORE POSTING. Use this template for bug reports. If you'd like to request a feature, please be as descriptive as possible and delete the template except the first section (Request Type) 2 | 3 | ### Request Type 4 | (select Bug or Feature Request and **remove this part**) 5 | Bug / Feature Request 6 | 7 | ### Work Environment 8 | 9 | | Question | Answer 10 | |---------------------------|-------------------- 11 | | OS version (server) | Debian, Ubuntu, CentOS, RedHat, ... 12 | | OS version (client) | XP, Seven, 10, Ubuntu, ... 13 | | Hippocampe version / git hash | 1.x, hash of the commit 14 | | Package Type | Docker, Binary, From source 15 | | Browser type & version | If applicable 16 | 17 | 18 | ### Problem Description 19 | Describe the problem/bug as clearly as possible. 20 | 21 | ### Steps to Reproduce 22 | 1. step 1 23 | 1. step 2 24 | 1. step 3... 25 | 26 | ### Possible Solutions 27 | (keep this section if you have suggestions on how to solve the problem. **Otherwise delete it**) 28 | 29 | ### Complementary information 30 | (add anything that can help identifying the problem such as **logs**, **screenshots**, **configuration dumps** etc.) 31 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.log 2 | *.pyc 3 | *.log.* 4 | *~ 5 | *.swp 6 | *.swo 7 | **/logs/ 8 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Authors 2 | ------- 3 | 4 | * Danni Co 5 | 6 | Contributors 7 | ------------ 8 | 9 | * CERT Banque de France (CERT-BDF) 10 | 11 | Copyright (C) 2015-2017 Danni Co 12 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ################################################### 2 | # Lightweight Hippocampe container 3 | ################################################### 4 | 5 | # Build the base from J8 Alpine 6 | FROM openjdk:8-jre-alpine 7 | 8 | RUN apk add --update --no-cache python \ 9 | python-dev \ 10 | py-pip \ 11 | git \ 12 | curl \ 13 | nodejs \ 14 | nodejs-npm 15 | 16 | RUN npm install -g bower 17 | RUN pip install --upgrade pip && \ 18 | pip install apscheduler \ 19 | Configparser \ 20 | elasticsearch \ 21 | flask \ 22 | python-dateutil \ 23 | requests \ 24 | urllib3==1.23 25 | 26 | COPY ./core /opt/Hippocampe/core 27 | COPY docker-entrypoint.sh / 28 | 29 | RUN adduser hippo -D 30 | RUN chown -R hippo:hippo /opt/Hippocampe /docker-entrypoint.sh 31 | 32 | USER hippo 33 | 34 | RUN cd /opt/Hippocampe/core/static && bower install 35 | 36 | ENTRYPOINT /docker-entrypoint.sh 37 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | 3 |

4 | 5 | **Hippocampe** is a threat feed aggregator. It gives your organisation a threat feed 'memory' and lets you query it easily through a REST API or from a Web UI. If you have a [Cortex](https://github.com/CERT-BDF/Cortex) server, there's already an analyzer to query Hippocampe. And if you use [TheHive](https://github.com/CERT-BDF/TheHive) as a security incident response platform, you can customize the JSON output produced by the analyzer to your taste or use the report template that we kindly provide. 6 | 7 | Hippocampe aggregates feeds from the Internet in an Elasticsearch cluster. It has a REST API which allows to search into its 'memory'. It is based on a Python script which fetchs URLs corresponding to feeds, parses and indexes them. 8 | 9 | # Hipposcore 10 | Hippocampe allows analysts to configure a confidence level for each feed that can be changed over time and when queried, it will provide a score called Hipposcore that will aid the analyst decide whether the analyzed observables are innocuous or rather malicious. 11 | 12 | # License 13 | Hippocampe is an open source and free software released under the [AGPL](https://github.com/CERT-BDF/Cortex/blob/master/LICENSE) (Affero General Public License). We, TheHive Project, are committed to ensure that Hippocampe will remain a free and open source project on the long-run. 14 | 15 | # Roadmap 16 | * Extracting observable or IOCs from an email or a report 17 | * Adding data manually 18 | * Distinguish fields generate by Hippocampe from those generated by feeds 19 | * Show related data (eg, when searching for a URL, show the domain as related if hippocampe knows it) 20 | * Index MISP attributes 21 | 22 | # Updates 23 | Information, news and updates are regularly posted on [TheHive Project Twitter account](https://twitter.com/thehive_project) and on [the blog](https://blog.thehive-project.org/). 24 | 25 | # Contributing 26 | We welcome your contributions. Please feel free to fork the code, play with it, make some patches and send us pull requests. 27 | 28 | # Support 29 | Please [open an issue on GitHub](https://github.com/CERT-BDF/Hippocampe/issues/new) if you'd like to report a bug or request a feature. 30 | 31 | Alternatively, if you need to contact the project team, send an email to . 32 | 33 | # Community Discussions 34 | We have set up a Google forum at . To request access, you need a Google account. You may create one [using a Gmail address](https://accounts.google.com/SignUp?hl=en) or [without one](https://accounts.google.com/SignUpWithoutGmail?hl=en). 35 | 36 | # Website 37 | 38 | -------------------------------------------------------------------------------- /core/conf/auth/auth.conf: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | 4 | [top_level_url] 5 | username: user 6 | password : passwd 7 | -------------------------------------------------------------------------------- /core/conf/feeds/ET_IP.conf: -------------------------------------------------------------------------------- 1 | 2 | [source] 3 | url=http://rules.emergingthreats.net/blockrules/compromised-ips.txt 4 | delimiter : , 5 | beginWithBlank : False 6 | fields : ["ip"] 7 | extraFields : 8 | description : This ruleset is compiled from a number of sources. It's contents are hosts that are known to be compromised by bots, phishing sites, etc, or known to be spewing hostile traffic. These are not your everyday infected and sending a bit of spam hosts, these are significantly infected and hostile hosts. 9 | startAt : 0 10 | score : -100 11 | coreIntelligence : ip 12 | validityDate: 13 | useByDate: 14 | 15 | [elasticsearch] 16 | typeIntel = etFree_compromisedIP 17 | 18 | [intel] 19 | ip = { 20 | "mapping" : { 21 | "type": "ip", 22 | "index": "not_analyzed" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /core/conf/feeds/abuseFeodo_IP.conf: -------------------------------------------------------------------------------- 1 | 2 | [source] 3 | url=https://feodotracker.abuse.ch/blocklist/?download=ipblocklist 4 | delimiter : , 5 | beginWithBlank : False 6 | fields : ["ip"] 7 | extraFields : 8 | description : The Feodo Tracker Feodo IP Blocklist contains IP addresses (IPv4) used as C&C communication channel by the Feodo Trojan. This lists contains two types of IP address: Feodo C&C servers used by version A, version C and version D of the Feodo Trojan (these IP addresses are usually compromised servers running an nginx daemon on port 8080 TCP or 7779 TCP that is acting as proxy, forwarding all traffic to a tier 2 proxy node) and Feodo C&C servers used by version B which are usually used for the exclusive purpose of hosting a Feodo C&C server. 9 | startAt : 7 10 | score : -100 11 | coreIntelligence: ip 12 | validityDate: 13 | useByDate: 14 | 15 | [elasticsearch] 16 | typeIntel= abuseFree_feodotrackerIP 17 | 18 | [intel] 19 | ip = { 20 | "mapping" : { 21 | "type": "ip", 22 | "index": "not_analyzed" 23 | } 24 | } 25 | 26 | -------------------------------------------------------------------------------- /core/conf/feeds/abuseSSL_IP.conf: -------------------------------------------------------------------------------- 1 | 2 | [source] 3 | url=https://sslbl.abuse.ch/blacklist/sslipblacklist.csv 4 | delimiter : , 5 | beginWithBlank : False 6 | fields : ["ip", "port", "type"] 7 | extraFields : 8 | description : The SSL IP Blacklist (CSV) contains all hosts (IP addresses) that SSLBL has seen in the past 30 days being associated with a malicious SSL certificate. The CSV contains IP address and port number of malicious SSL hosts. 9 | startAt : 9 10 | score : -100 11 | coreIntelligence : ip 12 | validityDate: 13 | useByDate: 14 | 15 | [elasticsearch] 16 | typeIntel= abuseFree_sslblIP 17 | 18 | [intel] 19 | ip = { 20 | "mapping" : { 21 | "type": "ip", 22 | "index": "not_analyzed" 23 | } 24 | } 25 | 26 | port = { 27 | "mapping" : { 28 | "type": "string", 29 | "index": "not_analyzed" 30 | } 31 | } 32 | 33 | type = { 34 | "mapping" : { 35 | "type": "string", 36 | "index": "not_analyzed" 37 | } 38 | } 39 | 40 | 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /core/conf/feeds/abuseZeus_DOMAIN.conf: -------------------------------------------------------------------------------- 1 | 2 | [source] 3 | url=https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist 4 | delimiter : , 5 | beginWithBlank : False 6 | fields : ["domain"] 7 | extraFields : 8 | description : This blocklist contains the same data as the ZeuS domain blocklist (BadDomains) but with the slight difference that it doesn't exclude hijacked websites (level 2). This means that this blocklist contains all domain names associated with ZeuS C&Cs which are currently being tracked by ZeuS Tracker. Hence this blocklist will likely cause some false positives. 9 | startAt : 5 10 | score : -100 11 | coreIntelligence : domain 12 | validityDate: 13 | useByDate: 14 | 15 | [elasticsearch] 16 | typeIntel= abuseFree_zeustrackerDOMAIN 17 | 18 | [intel] 19 | domain = { 20 | "mapping" : { 21 | "type": "string", 22 | "index": "not_analyzed" 23 | } 24 | } 25 | 26 | 27 | -------------------------------------------------------------------------------- /core/conf/feeds/abuseZeus_IP.conf: -------------------------------------------------------------------------------- 1 | 2 | [source] 3 | url=https://zeustracker.abuse.ch/blocklist.php?download=ipblocklist 4 | delimiter : , 5 | beginWithBlank : False 6 | fields : ["ip"] 7 | extraFields : 8 | description : This blocklist contains the same data as the ZeuS IP blocklist (BadIPs) but with the slight difference that it doesn't exclude hijacked websites (level 2) and free web hosting providers (level 3). This means that this blocklist contains all IPv4 addresses associated with ZeuS C&Cswhich are currently being tracked by ZeuS Tracker. Hence this blocklist will likely cause some false positives. 9 | startAt : 5 10 | score : -100 11 | coreIntelligence : ip 12 | validityDate: 13 | useByDate: 14 | 15 | [elasticsearch] 16 | typeIntel= abuseFree_zeustrackerIP 17 | 18 | [intel] 19 | ip = { 20 | "mapping" : { 21 | "type": "ip", 22 | "index": "not_analyzed" 23 | } 24 | } 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /core/conf/feeds/bambenek_c2cDOMAIN.conf: -------------------------------------------------------------------------------- 1 | 2 | [source] 3 | url=http://osint.bambenekconsulting.com/feeds/c2-dommasterlist.txt 4 | delimiter : , 5 | beginWithBlank : False 6 | fields : ["domain", "used_by", "submission_datetime", "reference_url"] 7 | extraFields : 8 | description : Master Feed of known, active and non-sinkholed C&Cs domain names 9 | startAt : 1 10 | score : -100 11 | coreIntelligence : domain 12 | validityDate: 13 | useByDate: 14 | 15 | [elasticsearch] 16 | typeIntel= bambeneck_c2cDOMAIN 17 | 18 | [intel] 19 | domain = { 20 | "mapping" : { 21 | "type": "string", 22 | "index": "not_analyzed" 23 | } 24 | } 25 | 26 | used_by = { 27 | "mapping" : { 28 | "type": "string", 29 | "index": "not_analyzed" 30 | } 31 | } 32 | 33 | submission_datetime = { 34 | "mapping" : { 35 | "type": "date", 36 | "format" : "yyyy-MM-dd HH:mm", 37 | "index": "no" 38 | } 39 | } 40 | 41 | reference_url = { 42 | "mapping" : { 43 | "type": "string", 44 | "index": "no" 45 | } 46 | } 47 | 48 | -------------------------------------------------------------------------------- /core/conf/feeds/bambenek_c2cIP.conf: -------------------------------------------------------------------------------- 1 | 2 | [source] 3 | url=http://osint.bambenekconsulting.com/feeds/c2-ipmasterlist.txt 4 | delimiter : , 5 | beginWithBlank : False 6 | fields : ["ip", "used_by", "submission_datetime", "reference_url"] 7 | extraFields : 8 | description : Master Feed of known, active and non-sinkholed C&Cs IP addresses 9 | startAt : 1 10 | score : -100 11 | coreIntelligence : ip 12 | validityDate: 13 | useByDate: 14 | 15 | [elasticsearch] 16 | typeIntel= bambeneck_c2cIP 17 | 18 | [intel] 19 | ip = { 20 | "mapping" : { 21 | "type": "ip", 22 | "index": "not_analyzed" 23 | } 24 | } 25 | 26 | used_by = { 27 | "mapping" : { 28 | "type": "string", 29 | "index": "not_analyzed" 30 | } 31 | } 32 | 33 | submission_datetime = { 34 | "mapping" : { 35 | "type": "date", 36 | "format" : "yyyy-MM-dd HH:mm", 37 | "index": "no" 38 | } 39 | } 40 | 41 | reference_url = { 42 | "mapping" : { 43 | "type": "string", 44 | "index": "no" 45 | } 46 | } 47 | 48 | -------------------------------------------------------------------------------- /core/conf/feeds/bambenek_dgaDOMAIN.conf: -------------------------------------------------------------------------------- 1 | 2 | [source] 3 | url=http://osint.bambenekconsulting.com/feeds/dga-feed.txt 4 | delimiter : , 5 | beginWithBlank : False 6 | fields : ["domain", "used_by", "submission_datetime", "reference_url"] 7 | extraFields : 8 | description : Domain feed of known DGA domains from -2 to +3 days 9 | startAt : 1 10 | score : -100 11 | coreIntelligence : domain 12 | validityDate: 13 | useByDate: 14 | 15 | [elasticsearch] 16 | typeIntel= bambeneck_c2cDOMAIN 17 | 18 | [intel] 19 | domain = { 20 | "mapping" : { 21 | "type": "string", 22 | "index": "not_analyzed" 23 | } 24 | } 25 | 26 | used_by = { 27 | "mapping" : { 28 | "type": "string", 29 | "index": "not_analyzed" 30 | } 31 | } 32 | 33 | submission_datetime = { 34 | "mapping" : { 35 | "type": "date", 36 | "format" : "yyyy-MM-dd HH:mm", 37 | "index": "no" 38 | } 39 | } 40 | 41 | reference_url = { 42 | "mapping" : { 43 | "type": "string", 44 | "index": "no" 45 | } 46 | } 47 | 48 | -------------------------------------------------------------------------------- /core/conf/feeds/binarydefense_IP.conf: -------------------------------------------------------------------------------- 1 | 2 | [source] 3 | url=http://www.binarydefense.com/banlist.txt 4 | delimiter : , 5 | beginWithBlank : False 6 | fields : ["ip"] 7 | extraFields : 8 | description : IP from Binary Defense Systems Artillery Threat Intelligence Feed and Banlist Feed 9 | startAt : 13 10 | score : -100 11 | coreIntelligence: ip 12 | validityDate: 13 | useByDate: 14 | 15 | [elasticsearch] 16 | typeIntel= binarydefenseFree_banlistIP 17 | 18 | [intel] 19 | ip = { 20 | "mapping" : { 21 | "type": "ip", 22 | "index": "not_analyzed" 23 | } 24 | } 25 | 26 | 27 | -------------------------------------------------------------------------------- /core/conf/feeds/ciarmylist_IP.conf: -------------------------------------------------------------------------------- 1 | 2 | [source] 3 | url=http://cinsscore.com/list/ci-badguys.txt 4 | delimiter : , 5 | beginWithBlank : False 6 | fields : ["ip"] 7 | extraFields : 8 | description : The CI Army list is a subset of the CINS Active Threat Intelligence ruleset, and consists of IP addresses that meet two basic criteria: 1) The IP's recent Rogue Packet score factor is very poor, and 2) The InfoSec community has not yet identified the IP as malicious. We think this second factor is important: We don't want to waste peoples' time listing thousands of IPs that have already been placed on other reputation lists; our list is meant to supplement and enhance the InfoSec community's existing efforts by providing IPs that haven't been identified yet. 9 | startAt : 0 10 | score : -100 11 | coreIntelligence: ip 12 | validityDate: 13 | useByDate: 14 | 15 | [elasticsearch] 16 | typeIntel= ciArmyListIP 17 | 18 | [intel] 19 | ip = { 20 | "mapping" : { 21 | "type": "ip", 22 | "index": "not_analyzed" 23 | } 24 | } 25 | 26 | -------------------------------------------------------------------------------- /core/conf/feeds/dnsbh_DOMAIN.conf: -------------------------------------------------------------------------------- 1 | 2 | [source] 3 | url : http://mirror1.malwaredomains.com/files/domains.txt 4 | delimiter : \t 5 | beginWithBlank : True 6 | fields : ["nextvalidation", "domain", "type", "original_reference-why_it_was_listed"] 7 | extraFields : extra 8 | description : DNS-BH - Malware Domain Blacklist. A list of domains that are known to be used to propagate malware are listed in Bind and Windows zone files. The domains are loaded onto an internal DNS server. When a computer requests a URL or file from one of these domains, a fake reply is sent, thus preventing many malware installs from occuring. 9 | startAt : 4 10 | score : -100 11 | coreIntelligence : domain 12 | validityDate: 13 | useByDate: 14 | 15 | [elasticsearch] 16 | typeIntel = malwaredomainsFree_dnsbhDOMAIN 17 | 18 | [intel] 19 | nextvalidation = { 20 | "mapping" : { 21 | "type": "string", 22 | "index": "not_analyzed" 23 | } 24 | } 25 | 26 | 27 | domain = { 28 | "mapping" : { 29 | "type": "string", 30 | "index": "not_analyzed" 31 | } 32 | } 33 | 34 | 35 | type = { 36 | "mapping" : { 37 | "type": "string", 38 | "index": "not_analyzed" 39 | } 40 | } 41 | 42 | 43 | original_reference-why_it_was_listed = { 44 | "mapping" : { 45 | "type": "string", 46 | "index": "not_analyzed" 47 | } 48 | } 49 | 50 | 51 | extra = { 52 | "mapping" : { 53 | "type": "string", 54 | "index": "not_analyzed" 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /core/conf/feeds/dshield_high_DOMAIN.conf: -------------------------------------------------------------------------------- 1 | 2 | [source] 3 | url : http://www.dshield.org/feeds/suspiciousdomains_High.txt 4 | delimiter : , 5 | beginWithBlank : False 6 | fields : ["domain"] 7 | extraFields : 8 | description : DShield suspicious domains, this list consists of High Level Sensitivity website URLs 9 | startAt : 15 10 | score : -100 11 | coreIntelligence : domain 12 | validityDate: 13 | useByDate: 14 | 15 | [elasticsearch] 16 | typeIntel = dshieldFree_highDOMAIN 17 | 18 | [intel] 19 | domain = { 20 | "mapping" : { 21 | "type": "string", 22 | "index": "not_analyzed" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /core/conf/feeds/dshield_low_DOMAIN.conf: -------------------------------------------------------------------------------- 1 | 2 | [source] 3 | url=https://dshield.org/feeds/suspiciousdomains_Low.txt 4 | delimiter : , 5 | beginWithBlank : False 6 | fields : ["domain"] 7 | extraFields : 8 | description : low level sensitivity websites URLs from dshield 9 | startAt : 16 10 | score : -100 11 | coreIntelligence : domain 12 | validityDate: 13 | useByDate: 14 | 15 | [elasticsearch] 16 | typeIntel= dshieldFree_lowDOMAIN 17 | 18 | [intel] 19 | domain = { 20 | "mapping" : { 21 | "type": "string", 22 | "index": "not_analyzed" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /core/conf/feeds/dshield_medium_DOMAIN.conf: -------------------------------------------------------------------------------- 1 | 2 | [source] 3 | url=https://dshield.org/feeds/suspiciousdomains_Medium.txt 4 | delimiter : , 5 | beginWithBlank : False 6 | fields : ["domain"] 7 | extraFields : 8 | description : medium level sensitivity websites URLs from dshield 9 | startAt : 16 10 | score : -100 11 | coreIntelligence : domain 12 | validityDate: 13 | useByDate: 14 | 15 | [elasticsearch] 16 | typeIntel= dshieldFree_mediumDOMAIN 17 | 18 | [intel] 19 | domain = { 20 | "mapping" : { 21 | "type": "string", 22 | "index": "not_analyzed" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /core/conf/feeds/greensnow_IP.conf: -------------------------------------------------------------------------------- 1 | 2 | [source] 3 | url=http://blocklist.greensnow.co/greensnow.txt 4 | delimiter : , 5 | beginWithBlank : False 6 | fields : ["ip"] 7 | extraFields : 8 | description : GreenSnow is a team consisting of the best specialists in computer security, we harvest a large number of IPs from different computers located around the world. GreenSnow is comparable with SpamHaus.org for attacks of any kind except for spam. Our list is updated automatically and you can withdraw at any time your IP address if it has been listed. 9 | startAt : 0 10 | score : -100 11 | coreIntelligence: ip 12 | validityDate: 13 | useByDate: 14 | 15 | [elasticsearch] 16 | typeIntel= greensnowIP 17 | 18 | [intel] 19 | ip = { 20 | "mapping" : { 21 | "type": "ip", 22 | "index": "not_analyzed" 23 | } 24 | } 25 | 26 | -------------------------------------------------------------------------------- /core/conf/feeds/hpHosts_emdDOMAIN.conf: -------------------------------------------------------------------------------- 1 | 2 | [source] 3 | url=https://hosts-file.net/emd.txt 4 | delimiter : \t 5 | beginWithBlank : False 6 | fields : ["localhost", "domain"] 7 | extraFields : 8 | description : This file contains malware sites listed in the hpHosts database. This should ONLY be downloaded by those wanting to block malware sites and nothing else, and requires manual merging. 9 | startAt : 7 10 | score : -100 11 | coreIntelligence : domain 12 | validityDate: 13 | useByDate: 14 | 15 | [elasticsearch] 16 | typeIntel = host_file_emdDOMAIN 17 | 18 | [intel] 19 | localhost = { 20 | "mapping" : { 21 | "type": "string", 22 | "index": "no" 23 | } 24 | } 25 | domain = { 26 | "mapping" : { 27 | "type": "string", 28 | "index": "not_analyzed" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /core/conf/feeds/hpHosts_expDOMAIN.conf: -------------------------------------------------------------------------------- 1 | 2 | [source] 3 | url=https://hosts-file.net/exp.txt 4 | delimiter : \t 5 | beginWithBlank : False 6 | fields : ["localhost", "domain"] 7 | extraFields : 8 | description : This file contains exploit sites listed in the hpHosts database. This should ONLY be downloaded by those wanting to block exploit sites and nothing else, and requires manual merging. 9 | startAt : 7 10 | score : -100 11 | coreIntelligence : domain 12 | validityDate: 13 | useByDate: 14 | 15 | [elasticsearch] 16 | typeIntel = host_file_expDOMAIN 17 | 18 | [intel] 19 | localhost = { 20 | "mapping" : { 21 | "type": "string", 22 | "index": "no" 23 | } 24 | } 25 | domain = { 26 | "mapping" : { 27 | "type": "string", 28 | "index": "not_analyzed" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /core/conf/feeds/hpHosts_fsaDOMAIN.conf: -------------------------------------------------------------------------------- 1 | 2 | [source] 3 | url=https://hosts-file.net/fsa.txt 4 | delimiter : \t 5 | beginWithBlank : False 6 | fields : ["localhost", "domain"] 7 | extraFields : 8 | description : This file contains fraud sites listed in the hpHosts database. This should ONLY be downloaded by those wanting to block fraud sites and nothing else, and requires manual merging. 9 | startAt : 7 10 | score : -100 11 | coreIntelligence : domain 12 | validityDate: 13 | useByDate: 14 | 15 | [elasticsearch] 16 | typeIntel = host_file_fsaDOMAIN 17 | 18 | [intel] 19 | localhost = { 20 | "mapping" : { 21 | "type": "string", 22 | "index": "no" 23 | } 24 | } 25 | domain = { 26 | "mapping" : { 27 | "type": "string", 28 | "index": "not_analyzed" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /core/conf/feeds/hpHosts_grmDOMAIN.conf: -------------------------------------------------------------------------------- 1 | 2 | [source] 3 | url=https://hosts-file.net/grm.txt 4 | delimiter : \t 5 | beginWithBlank : False 6 | fields : ["localhost", "domain"] 7 | extraFields : 8 | description : This file contains sites involved in spam (that do not otherwise meet any other classification criteria) listed in the hpHosts database This should ONLY be downloaded by those wanting to block spam sites and nothing else, and requires manual merging. 9 | startAt : 7 10 | score : -100 11 | coreIntelligence : domain 12 | validityDate: 13 | useByDate: 14 | 15 | [elasticsearch] 16 | typeIntel = host_file_grmDOMAIN 17 | 18 | [intel] 19 | localhost = { 20 | "mapping" : { 21 | "type": "string", 22 | "index": "no" 23 | } 24 | } 25 | domain = { 26 | "mapping" : { 27 | "type": "string", 28 | "index": "not_analyzed" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /core/conf/feeds/hpHosts_hjkDOMAIN.conf: -------------------------------------------------------------------------------- 1 | 2 | [source] 3 | url=https://hosts-file.net/hjk.txt 4 | delimiter : \t 5 | beginWithBlank : False 6 | fields : ["localhost", "domain"] 7 | extraFields : 8 | description : This file contains hijack sites listed in the hpHosts database This should ONLY be downloaded by those wanting to block hijack sites and nothing else, and requires manual merging. 9 | startAt : 7 10 | score : -100 11 | coreIntelligence : domain 12 | validityDate: 13 | useByDate: 14 | 15 | [elasticsearch] 16 | typeIntel = host_file_hjkDOMAIN 17 | 18 | [intel] 19 | localhost = { 20 | "mapping" : { 21 | "type": "string", 22 | "index": "no" 23 | } 24 | } 25 | domain = { 26 | "mapping" : { 27 | "type": "string", 28 | "index": "not_analyzed" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /core/conf/feeds/hpHosts_pshDOMAIN.conf: -------------------------------------------------------------------------------- 1 | 2 | [source] 3 | url=https://hosts-file.net/psh.txt 4 | delimiter : \t 5 | beginWithBlank : False 6 | fields : ["localhost", "domain"] 7 | extraFields : 8 | description : This file contains phishing sites listed in the hpHosts database This should ONLY be downloaded by those wanting to block phishing sites and nothing else, and requires manual merging. 9 | startAt : 7 10 | score : -100 11 | coreIntelligence : domain 12 | validityDate: 13 | useByDate: 14 | 15 | [elasticsearch] 16 | typeIntel = host_file_pshDOMAIN 17 | 18 | [intel] 19 | localhost = { 20 | "mapping" : { 21 | "type": "string", 22 | "index": "no" 23 | } 24 | } 25 | domain = { 26 | "mapping" : { 27 | "type": "string", 28 | "index": "not_analyzed" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /core/conf/feeds/hpHosts_pupDOMAIN.conf: -------------------------------------------------------------------------------- 1 | 2 | [source] 3 | url=https://hosts-file.net/pup.txt 4 | delimiter : \t 5 | beginWithBlank : False 6 | fields : ["localhost", "domain"] 7 | extraFields : 8 | description : This file contains PUP sites listed in the hpHosts database This should ONLY be downloaded by those wanting to block PUP (Potentially Unwanted Programs) sites and nothing else, and requires manual merging. 9 | startAt : 7 10 | score : -100 11 | coreIntelligence : domain 12 | validityDate: 13 | useByDate: 14 | 15 | [elasticsearch] 16 | typeIntel = host_file_pupDOMAIN 17 | 18 | [intel] 19 | localhost = { 20 | "mapping" : { 21 | "type": "string", 22 | "index": "no" 23 | } 24 | } 25 | domain = { 26 | "mapping" : { 27 | "type": "string", 28 | "index": "not_analyzed" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /core/conf/feeds/hpHosts_wrzDOMAIN.conf: -------------------------------------------------------------------------------- 1 | 2 | [source] 3 | url=https://hosts-file.net/wrz.txt 4 | delimiter : \t 5 | beginWithBlank : False 6 | fields : ["localhost", "domain"] 7 | extraFields : 8 | description : This file contains warez/piracy sites listed in the hpHosts database This should ONLY be downloaded by those wanting to block warez/piracy sites and nothing else, and requires manual merging. 9 | startAt : 7 10 | score : -100 11 | coreIntelligence : domain 12 | validityDate: 13 | useByDate: 14 | 15 | [elasticsearch] 16 | typeIntel = host_file_wrzDOMAIN 17 | 18 | [intel] 19 | localhost = { 20 | "mapping" : { 21 | "type": "string", 22 | "index": "no" 23 | } 24 | } 25 | domain = { 26 | "mapping" : { 27 | "type": "string", 28 | "index": "not_analyzed" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /core/conf/feeds/malc0de_IP.conf: -------------------------------------------------------------------------------- 1 | 2 | [source] 3 | url=http://malc0de.com/bl/IP_Blacklist.txt 4 | delimiter : , 5 | beginWithBlank : False 6 | fields : ["ip"] 7 | extraFields : 8 | description : IPs that are hosting malicious executables 9 | startAt : 0 10 | score : -100 11 | coreIntelligence : ip 12 | validityDate: 13 | useByDate: 14 | 15 | [elasticsearch] 16 | typeIntel= malc0deFree_BlacklistIP 17 | 18 | [intel] 19 | ip = { 20 | "mapping" : { 21 | "type": "ip", 22 | "index": "not_analyzed" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /core/conf/feeds/openphish_URL.conf: -------------------------------------------------------------------------------- 1 | 2 | [source] 3 | url = https://openphish.com/feed.txt 4 | delimiter = \t 5 | beginWithBlank = False 6 | fields = ["url"] 7 | extraFields = 8 | description = OpenPhish is a repository of active phishing sites that offers free phishing intelligence feeds to its partners 9 | startAt = 0 10 | score : -100 11 | coreIntelligence : url 12 | validityDate: 13 | useByDate: 14 | 15 | [elasticsearch] 16 | typeIntel= openphishFree_feedURL 17 | 18 | [intel] 19 | url = { 20 | "mapping" : { 21 | "type": "string", 22 | "index": "not_analyzed" 23 | } 24 | } 25 | 26 | -------------------------------------------------------------------------------- /core/conf/feeds/phishtank_URL.conf: -------------------------------------------------------------------------------- 1 | 2 | [source] 3 | url=http://data.phishtank.com/data/online-valid.csv 4 | delimiter : , 5 | beginWithBlank : False 6 | fields : ["phish_id", "url", "phish_detail_url", "submission_time", "verified", "verification_time", "online", "target"] 7 | extraFields : 8 | description : PhishTank is a collaborative clearing house for data and information about phishing on the Internet 9 | startAt : 1 10 | score : -100 11 | coreIntelligence : url 12 | validityDate: 13 | useByDate: 14 | 15 | [elasticsearch] 16 | typeIntel= phishtankFree_onlinevalidURL 17 | 18 | [intel] 19 | phish_id = { 20 | "mapping" : { 21 | "type": "string", 22 | "index": "not_analyzed" 23 | } 24 | } 25 | 26 | url = { 27 | "mapping" : { 28 | "type": "string", 29 | "index": "not_analyzed" 30 | } 31 | } 32 | 33 | phish_detail_url = { 34 | "mapping" : { 35 | "type": "string", 36 | "index": "no" 37 | } 38 | } 39 | 40 | submission_time = { 41 | "mapping" : { 42 | "type": "date", 43 | "format" : "date_time_no_millis", 44 | "index": "no" 45 | } 46 | } 47 | 48 | verified = { 49 | "mapping" : { 50 | "type": "string", 51 | "index": "no" 52 | } 53 | } 54 | 55 | verification_time = { 56 | "mapping" : { 57 | "type": "date", 58 | "format" : "date_time_no_millis", 59 | "index": "no" 60 | } 61 | } 62 | 63 | online = { 64 | "mapping" : { 65 | "type": "string", 66 | "index": "no" 67 | } 68 | } 69 | 70 | target = { 71 | "mapping" : { 72 | "type": "string", 73 | "index": "no" 74 | } 75 | } 76 | 77 | -------------------------------------------------------------------------------- /core/conf/feeds/shunlist_IP.conf: -------------------------------------------------------------------------------- 1 | 2 | [source] 3 | url=http://autoshun.org/files/shunlist.csv 4 | delimiter : , 5 | beginWithBlank : False 6 | fields : ["ip", "dateOfRecord", "type"] 7 | extraFields : 8 | description : AutoShun is a Snort plugin that allows you to send your Snort IDS logs to a centralized server that will correlate attacks from your sensor logs with other snort sensors, honeypots, and mail filters from around the world. The input from your logs will be used to identify hostile address that are bots, worms, spam engines which we use to build a shun list. 9 | startAt : 1 10 | score : -100 11 | coreIntelligence : ip 12 | validityDate: 13 | useByDate: 14 | 15 | [elasticsearch] 16 | typeIntel= autoshunFree_shunlistIP 17 | 18 | [intel] 19 | ip = { 20 | "mapping" : { 21 | "type": "ip", 22 | "index": "not_analyzed" 23 | } 24 | } 25 | 26 | dateOfRecord = { 27 | "mapping" : { 28 | "type": "string", 29 | "index": "not_analyzed" 30 | } 31 | } 32 | 33 | type = { 34 | "mapping" : { 35 | "type": "string", 36 | "index": "not_analyzed" 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /core/conf/feeds/snort_IP.conf: -------------------------------------------------------------------------------- 1 | 2 | [source] 3 | url=http://labs.snort.org/feeds/ip-filter.blf 4 | delimiter : , 5 | beginWithBlank : False 6 | fields : ["ip"] 7 | extraFields : 8 | description : IP blacklist from labs.snort.org which is an undertaking by the Sourcefire VRT. 9 | startAt : 0 10 | score : -100 11 | coreIntelligence : ip 12 | validityDate: 13 | useByDate: 14 | 15 | [elasticsearch] 16 | typeIntel= snortFree_filterIP 17 | 18 | [intel] 19 | ip = { 20 | "mapping" : { 21 | "type": "ip", 22 | "index": "not_analyzed" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /core/conf/feeds/torexit_IP.conf: -------------------------------------------------------------------------------- 1 | 2 | [source] 3 | url=https://torstatus.blutmagie.de/ip_list_exit.php/Tor_ip_list_EXIT.csv 4 | delimiter : , 5 | beginWithBlank : False 6 | fields : ["ip"] 7 | extraFields : 8 | description : Tor Exit Nodes. 9 | startAt : 0 10 | score : -50 11 | coreIntelligence : ip 12 | validityDate: 13 | useByDate: 14 | 15 | [elasticsearch] 16 | typeIntel= tor_exit_nodes 17 | 18 | [intel] 19 | ip = { 20 | "mapping" : { 21 | "type": "ip", 22 | "index": "not_analyzed" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /core/conf/hippo/hippo.conf: -------------------------------------------------------------------------------- 1 | 2 | [api] 3 | debug : False 4 | host : 0.0.0.0 5 | port : 5000 6 | threaded : True 7 | 8 | [elasticsearch] 9 | ip : 127.0.0.1 10 | port : 9200 11 | #indexNameES MUST BE LOWERCASE 12 | indexNameES : hippocampe 13 | typeNameESSource : source 14 | typeNameESNew : new 15 | typeNameESJobs: jobs 16 | 17 | [shadowbook] 18 | nbThreadPerCPU : 2 19 | 20 | [freshness] 21 | #in days 22 | threshold : 1 23 | 24 | [schedReport] 25 | #in hours 26 | threshold: 12 27 | -------------------------------------------------------------------------------- /core/examples/distinct.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | 4 | curl -i -H "Content-Type: application/json" -X POST -d ' 5 | { 6 | "field" : ["url"] 7 | }' http://localhost:5000/hippocampe/api/v1.0/distinct 8 | 9 | 10 | -------------------------------------------------------------------------------- /core/examples/freshness.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | 4 | curl -GET http://localhost:5000/hippocampe/api/v1.0/freshness 5 | -------------------------------------------------------------------------------- /core/examples/hipposched.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | 4 | curl -i -H "Content-Type: application/json" -X POST -d ' 5 | { 6 | "time" : "* */12 * * *" 7 | }' http://localhost:5000/hippocampe/api/v1.0/hipposched 8 | -------------------------------------------------------------------------------- /core/examples/hipposcore.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | import json 6 | import urllib2 7 | from pprint import pprint 8 | 9 | def main(): 10 | toRequest = { 11 | "niny.tk" : {"type": "domain"}, 12 | } 13 | 14 | req = urllib2.Request('http://localhost:5000/hippocampe/api/v1.0/hipposcore') 15 | req.add_header('Content-Type', 'application/json') 16 | 17 | response = urllib2.urlopen(req, json.dumps(toRequest)) 18 | data = json.load(response) 19 | pprint(data) 20 | if __name__ == '__main__': 21 | main() 22 | -------------------------------------------------------------------------------- /core/examples/hipposcore.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | 4 | curl -i -H "Content-Type: application/json" -X POST -d ' 5 | { 6 | "perugemstones.com" : {"type" : "domain"}, 7 | "hnliyin.com" : {"type" : "domain"}, 8 | "199.9.24.1" : {"type" : "ip"}, 9 | "198.23.78.98" : {"type" : "ip"}, 10 | "223.184.173.74" : {"type" : "ip"} 11 | }' http://localhost:5000/hippocampe/api/v1.0/hipposcore 12 | 13 | 14 | -------------------------------------------------------------------------------- /core/examples/jobs.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | 4 | curl -GET http://localhost:5000/hippocampe/api/v1.0/jobs 5 | -------------------------------------------------------------------------------- /core/examples/lastQuery.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | 4 | curl -GET http://localhost:5000/hippocampe/api/v1.0/lastQuery 5 | -------------------------------------------------------------------------------- /core/examples/lastStatus.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | 4 | curl -GET http://localhost:5000/hippocampe/api/v1.0/lastStatus 5 | -------------------------------------------------------------------------------- /core/examples/monitorSources.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | 4 | curl -GET http://localhost:5000/hippocampe/api/v1.0/monitorSources 5 | -------------------------------------------------------------------------------- /core/examples/more.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | 4 | curl -i -H "Content-Type: application/json" -X POST -d ' 5 | { 6 | "http://www.mkpoytuyr.com/f/" : {"type" : "url"}, 7 | "199.9.24.1" : {"type" : "ip"}, 8 | "198.23.78.98" : {"type" : "ip"}, 9 | "223.184.173.74" : {"type" : "ip"} 10 | }' http://localhost:5000/hippocampe/api/v1.0/more 11 | 12 | 13 | -------------------------------------------------------------------------------- /core/examples/new.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | 4 | curl -GET http://localhost:5000/hippocampe/api/v1.0/new 5 | -------------------------------------------------------------------------------- /core/examples/schedReport.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | 4 | curl -GET http://localhost:5000/hippocampe/api/v1.0/schedReport 5 | -------------------------------------------------------------------------------- /core/examples/shadowbook.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | 4 | curl -XGET localhost:5000/hippocampe/api/v1.0/shadowbook 5 | -------------------------------------------------------------------------------- /core/examples/sizeBySource.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | 4 | curl -GET http://localhost:5000/hippocampe/api/v1.0/sizeBySources 5 | -------------------------------------------------------------------------------- /core/examples/sizeByType.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | 4 | curl -GET http://localhost:5000/hippocampe/api/v1.0/sizeByType 5 | -------------------------------------------------------------------------------- /core/examples/sources.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | 4 | curl -GET http://localhost:5000/hippocampe/api/v1.0/sources 5 | -------------------------------------------------------------------------------- /core/examples/type.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | 4 | curl -XGET localhost:5000/hippocampe/api/v1.0/type 5 | -------------------------------------------------------------------------------- /core/services/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheHive-Project/Hippocampe/03df065bd304b1515db6925f6cd01a754d42825e/core/services/__init__.py -------------------------------------------------------------------------------- /core/services/countType.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | from modules.countType.BagOfIntel import BagOfIntel 6 | import typeIntel 7 | import logging 8 | logger = logging.getLogger(__name__) 9 | from pprint import pprint 10 | def main(): 11 | #logger.info('countType.main launched') 12 | #try: 13 | report = dict() 14 | #retrieving all intelligences's types available 15 | typeReport = typeIntel.main() 16 | for typeIntelligence in typeReport['type']: 17 | bagOfIntel = BagOfIntel(typeIntelligence) 18 | size = bagOfIntel.getSize() 19 | print size 20 | report[typeIntelligence] = dict() 21 | report[typeIntelligence]['size'] = size 22 | logger.info('countType.main end') 23 | pprint(report) 24 | return report 25 | # except Exception as e: 26 | # logger.error('countType.main failed, no idea where it came from...', exc_info = True) 27 | # response = dict() 28 | # response['error'] = str(e) 29 | # return response 30 | 31 | if __name__ == '__main__': 32 | main() 33 | -------------------------------------------------------------------------------- /core/services/distinct.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | from modules.distinct.Field import Field 6 | import logging 7 | logger = logging.getLogger(__name__) 8 | def main(jsonRequest): 9 | logger.info('distinct.main launched') 10 | response = dict() 11 | #jsonRequest looks like: 12 | #{ 13 | # 'field': ['ip', 'domain', 'url'] 14 | #} 15 | try: 16 | for field in jsonRequest['field']: 17 | logger.info('All %s requested', field) 18 | fieldToRequest = Field(field) 19 | fieldToRequest.getDistinct() 20 | response[field] = fieldToRequest.distinctList 21 | logger.info('distinct.main end') 22 | return response 23 | except Exception as e: 24 | logger.error('distinct.main failed, no idea where it came from...', exc_info = True) 25 | response['error'] = str(e) 26 | return response 27 | if __name__ == '__main__': 28 | main() 29 | -------------------------------------------------------------------------------- /core/services/freshness.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | import sources 6 | import dateutil.parser 7 | import time 8 | from modules.common.getConf import getHippoConf 9 | from configparser import NoOptionError, NoSectionError 10 | import logging 11 | logger = logging.getLogger(__name__) 12 | 13 | def main(): 14 | logger.info('freshness.main launched') 15 | 16 | cfg = getHippoConf() 17 | #threshold defined in conf file 18 | #by default equals to 7 19 | #if a source is not querryed for 7 days or more, an alert is sent 20 | 21 | #JSON dict reporting data freshness for all sources 22 | report = dict() 23 | try: 24 | threshold = cfg.getint('freshness', 'threshold') 25 | 26 | #retrieve all source 27 | dictSource = sources.main() 28 | #dictSource looks like: 29 | #{ 30 | # "https://feodotracker.abuse.ch/blocklist/?download=ipblocklist": { 31 | # "coreIntelligence": "ip", 32 | # "description": "The Feodo Tracker Feodo IP Blocklist contains IP addresses (IPv4) used as C&C communication channel by the Feodo Trojan. This lists contains two types of IP address: Feodo C&C servers used by version A, version C and version D of the Feodo Trojan (these IP addresses are usually compromised servers running an nginx daemon on port 8080 TCP or 7779 TCP that is acting as proxy, forwarding all traffic to a tier 2 proxy node) and Feodo C&C servers used by version B which are usually used for the exclusive purpose of hosting a Feodo C&C server.", 33 | # "firstQuery": "20160222T233556+0100", 34 | # "lastQuery": "20160224T134833+0100", 35 | # "score": -100, 36 | # "source": "https://feodotracker.abuse.ch/blocklist/?download=ipblocklist", 37 | # "type": "source" 38 | # } 39 | #} 40 | now = time.strftime("%Y%m%dT%H%M%S%z") 41 | 42 | #converting to dateutil 43 | now = dateutil.parser.parse(now) 44 | for url, dictData in dictSource.items(): 45 | report[url] = dict() 46 | lastQuery =dictData['lastQuery'] 47 | lastQuery = dateutil.parser.parse(lastQuery) 48 | 49 | #delta is the time difference in days between now and lastQuery 50 | delta = (now - lastQuery).days 51 | if (delta > threshold): 52 | report[url]['freshness'] = 'NOK' 53 | else: 54 | report[url]['freshness'] = 'OK' 55 | logger.info('freshness.main end') 56 | return report 57 | except NoSectionError as e: 58 | logger.error('parsing hippo.conf failed', exc_info = True) 59 | report['error'] = str(e) 60 | return report 61 | except NoOptionError as e: 62 | logger.error('parsing hippo.conf failed', exc_info = True) 63 | report['error'] = str(e) 64 | return report 65 | except Exception as e: 66 | logger.error('frehsness.main failed, no idea where it came from...', exc_info = True) 67 | report['error'] = str(e) 68 | return report 69 | if __name__ == '__main__': 70 | main() 71 | 72 | -------------------------------------------------------------------------------- /core/services/hipposched.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | from apscheduler.schedulers.background import BackgroundScheduler 6 | import shadowbook 7 | import logging 8 | logger = logging.getLogger(__name__) 9 | def main(request): 10 | logger.info('hipposched.main launched') 11 | try: 12 | jobId = 'shadowbook' 13 | time = request['time'] 14 | logger.info('schedule submitted as: ' + str(time)) 15 | #processing time to isolate each field 16 | tabTime = time.split(' ') 17 | #tabTime 18 | # [0] min 19 | # [1] hour 20 | # [2] day of month 21 | # [3] month 22 | # [4] day of week 23 | sched = BackgroundScheduler() 24 | #always erase the previous schedule 25 | #because of replace_existing = True 26 | #logger.info('creating job') 27 | sched.add_job(shadowbook.hipposchedVersion, 28 | 'cron', 29 | minute = tabTime[0], 30 | hour = tabTime[1], 31 | day = tabTime[2], 32 | month = tabTime[3], 33 | day_of_week = tabTime[4], 34 | replace_existing = True, 35 | id = jobId) 36 | sched.start() 37 | logger.info('job succesfully schedulled as: ' + str(time)) 38 | report = dict() 39 | report['schedule'] = time 40 | logger.info('hipposched.main end') 41 | return report 42 | except Exception as e: 43 | logger.error('hipposched.main failed, no idea where it came from', exc_info = True) 44 | report = dict() 45 | report['error'] = str(e) 46 | return report 47 | 48 | 49 | -------------------------------------------------------------------------------- /core/services/hipposcore.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | from time import strftime 6 | from modules.hipposcore.ExistingSource import ExistingSource 7 | import dateutil.parser 8 | from math import exp 9 | from werkzeug.contrib.cache import SimpleCache 10 | import logging 11 | logger = logging.getLogger(__name__) 12 | def calcHipposcore(dictResult): 13 | logger.info('hipposcore.calcHipposcore launched') 14 | try: 15 | T = 182.625 16 | n1 = float() 17 | n2 = float() 18 | n3 = float() 19 | #the final result 20 | hippodict = dict() 21 | #json object for hipposcore 22 | #{ 23 | # 'ioc': { 24 | # 'hipposcore': -89 25 | # } 26 | #} 27 | scoredict = dict() 28 | now = strftime("%Y%m%dT%H%M%S%z") 29 | P = float() 30 | #a cache is used to speed up the source's score retrieve 31 | cache = SimpleCache() 32 | 33 | for ioc, listMatches in dictResult.items(): 34 | #if there is no match for the ioc 35 | #listmaches will be empt, then hipposcore will be 0 36 | if not listMatches : 37 | hippodict[ioc] = dict() 38 | hippodict[ioc]['hipposcore'] = 0 39 | #match case 40 | else: 41 | P = 0.0 42 | for match in listMatches: 43 | #n1 retrieving the source according to its id 44 | idSource = match['idSource'] 45 | n1 = cache.get(idSource) 46 | if n1 is None: 47 | source = ExistingSource(idSource) 48 | source.forgeDocMatch() 49 | if source.search(): 50 | source.processMatchResponse() 51 | n1 = source.getScore() 52 | #the source's scor is between -100 and +100 53 | #for convenience, 54 | #in the formula it is required to be within -1 and +1 55 | n1 = n1 / 100.0 56 | cache.set(idSource, n1, timeout=3 * 60) 57 | #n2 58 | #rank is specific to alexa feed 59 | if 'rank' in match: 60 | n2 = match['rank'] 61 | else: 62 | n2 = 1.0 63 | #n3 64 | #last time hippocampe saw the ioc 65 | lastAppearance = match['lastAppearance'] 66 | #lastAppearance is a string, to calculate the time 67 | #difference, in other word the age of the ioc, 68 | #between lastAppearance and now, it is 69 | #needed to convert those strings to date 70 | lastAppearanceDate = dateutil.parser.parse(lastAppearance) 71 | nowDate = dateutil.parser.parse(now) 72 | #ioc's age in days 73 | age = (nowDate - lastAppearanceDate).days 74 | n3 = exp(-age / T) 75 | #P is the sum of n3 * n2 * n1 for every matches 76 | P = P + (n3 * n2 * n1) 77 | score = (1 - exp(-2 * abs(P))) * (abs(P) / P) * 100 78 | score = "%.2f" % score 79 | scoredict['hipposcore'] = score 80 | #scoredict['hipposcore'] = (1 - exp(-2 * abs(P))) * (abs(P) / P) * 100 81 | hippodict[ioc] = scoredict 82 | logger.info('hipposcore.main end') 83 | return hippodict 84 | except Exception as e: 85 | logger.error('hipposcore.calcHipposcore failed, no idea where it came from...', exc_info = True) 86 | report = dict() 87 | report['error'] = str(e) 88 | if __name__ == '__main__': 89 | calcHipposcore() 90 | -------------------------------------------------------------------------------- /core/services/jobs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | from modules.jobs.BagOfJobs import BagOfJobs 6 | import logging 7 | logger = logging.getLogger(__name__) 8 | 9 | def main(): 10 | logger.info('jobs.main launched') 11 | try: 12 | bag = BagOfJobs() 13 | result = bag.getMatchList() 14 | logger.info('jobs.main end') 15 | return result 16 | except Exception as e: 17 | logger.error('jobs.main failed, no idea where it came from...', exc_info = True) 18 | report = dict() 19 | report['error'] = str(e) 20 | return report 21 | -------------------------------------------------------------------------------- /core/services/lastQuery.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | from modules.common.ES import getES 6 | import sources 7 | import logging 8 | logger = logging.getLogger(__name__) 9 | def main(): 10 | logger.info('lastQuery.main launched') 11 | try: 12 | report = dict() 13 | #retrieving sources report 14 | sourcesReport = sources.main() 15 | #isolating lastQuery date for every type under hippocampe index 16 | for url, dictAttributes in sourcesReport.items(): 17 | report[url] = dict() 18 | report[url]['lastQuery'] = dictAttributes['lastQuery'] 19 | logger.info('lastQuery.main end') 20 | return report 21 | except Exception as e: 22 | logger.error('distinct.main failed, no idea where it came from...', exc_info = True) 23 | response = dict() 24 | response['error'] = str(e) 25 | return response 26 | 27 | if __name__ == '__main__': 28 | main() 29 | -------------------------------------------------------------------------------- /core/services/lastStatus.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | from modules.jobs.BagOfJobs import BagOfJobs 6 | import logging 7 | logger = logging.getLogger(__name__) 8 | 9 | def main(): 10 | logger.info('lastStatus.main launched') 11 | try: 12 | result = dict() 13 | bag = BagOfJobs() 14 | globalReport = bag.getLastJob() 15 | for confFileName, report in globalReport['report'].items(): 16 | result[report['link']] = dict() 17 | if report['error']: 18 | #there's one or more errors 19 | result[report['link']]['lastStatus'] = 'NOK' 20 | elif report['nbIndex'] == 0 and report['nbUpdate'] == 0: 21 | #the feed might have not been downloaded 22 | result[report['link']]['lastStatus'] = 'NOK' 23 | else: 24 | result[report['link']]['lastStatus'] = 'OK' 25 | logger.info('lastStatus.main end') 26 | return result 27 | except Exception as e: 28 | logger.error('lastStatus.main failed, no idea where it came from...', exc_info = True) 29 | report = dict() 30 | report['error'] = str(e) 31 | return report 32 | -------------------------------------------------------------------------------- /core/services/modules/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheHive-Project/Hippocampe/03df065bd304b1515db6925f6cd01a754d42825e/core/services/modules/__init__.py -------------------------------------------------------------------------------- /core/services/modules/common/ES.py: -------------------------------------------------------------------------------- 1 | #/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | 6 | from elasticsearch import Elasticsearch 7 | from elasticsearch.client import IndicesClient 8 | from configparser import ConfigParser 9 | import os 10 | app_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../../' 11 | from getConf import getHippoConf 12 | import logging 13 | logger = logging.getLogger(__name__) 14 | 15 | def getES(): 16 | confPath = app_dir + '/conf/hippo/hippo.conf' 17 | cfg = ConfigParser() 18 | cfg.read(confPath) 19 | host = cfg.get('elasticsearch', 'ip') 20 | port = cfg.getint('elasticsearch', 'port') 21 | ES = Elasticsearch([{'host': host, 'port' : port}], timeout = 60) 22 | return ES 23 | 24 | def checkES(): 25 | logger.info('ES.checkES launched') 26 | try: 27 | ES = getES() 28 | return ES.ping() 29 | except Exception as e: 30 | logger.error('ES.checkES failed: %s', e, exc_info=True) 31 | 32 | def checkData(checkList): 33 | #checkList is the list of types to check 34 | 35 | #check if the hippocampe's index exists in ES 36 | #and check if ES type exists according to checkList 37 | logger.info('ES.checkData launched') 38 | logger.info(checkList) 39 | ES = getES() 40 | index = IndicesClient(ES) 41 | 42 | cfg = getHippoConf() 43 | 44 | indexName = cfg.get('elasticsearch', 'indexNameES') 45 | #references contains the name of types used in Hippocampe 46 | references = dict() 47 | references['sourceType'] = cfg.get('elasticsearch', 'typeNameESSource') 48 | references['newType'] = cfg.get('elasticsearch', 'typeNameESNew') 49 | references['jobsType'] = cfg.get('elasticsearch', 'typeNameESJobs') 50 | 51 | #listType = list() 52 | #listType.append(sourceType) 53 | #listType.append(newType) 54 | #listType.append(jobsType) 55 | 56 | #check index 57 | if index.exists(index = indexName): 58 | #check types 59 | for check in checkList: 60 | if index.exists_type(index = indexName, doc_type = references[check]): 61 | logger.info('index %s and type %s exist', indexName, references[check]) 62 | else: 63 | logger.info('index %s exists but type %s does not', indexName, references[check] ) 64 | return False 65 | return True 66 | else: 67 | logger.info('index %s does not exist', indexName) 68 | return False 69 | -------------------------------------------------------------------------------- /core/services/modules/common/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | -------------------------------------------------------------------------------- /core/services/modules/common/getConf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | from configparser import ConfigParser 5 | import os 6 | 7 | def getConf(confPath): 8 | cfg = ConfigParser() 9 | cfg.read(confPath) 10 | return cfg 11 | 12 | def getHippoConf(): 13 | current_dir = os.path.dirname(os.path.abspath(__file__)) 14 | app_dir = current_dir + '/../../../' 15 | 16 | confPath = app_dir + 'conf/hippo/hippo.conf' 17 | return getConf(confPath) 18 | 19 | if __name__ == '__main__': 20 | getHippoConf() 21 | -------------------------------------------------------------------------------- /core/services/modules/countType/BagOfIntel.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | import os 6 | import sys 7 | app_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../../' 8 | pathModule = app_dir + 'services/modules/common' 9 | sys.path.insert(0, pathModule) 10 | from ES import getES 11 | from getConf import getHippoConf 12 | 13 | import logging 14 | logger = logging.getLogger(__name__) 15 | class BagOfIntel: 16 | def __init__(self, typeIntel): 17 | 18 | cfg = getHippoConf() 19 | self.typeIntel = typeIntel 20 | self.es = getES() 21 | self.docSearch = dict() 22 | self.indexNameES = cfg.get('elasticsearch', 'indexNameES') 23 | #contains every distinct value from a field 24 | self.size = int() 25 | 26 | def forgeDocSearch(self): 27 | #size=0 means "no limit to the number of terms" 28 | self.docSearch = { 29 | "aggs": { 30 | "distinct": { 31 | "cardinality": { 32 | "field": self.typeIntel 33 | } 34 | } 35 | } 36 | } 37 | 38 | def getSize(self): 39 | self.forgeDocSearch() 40 | res = self.es.search(index=self.indexNameES, body = self.docSearch) 41 | #res looks like: 42 | #{u'_shards': {u'failed': 0, u'successful': 5, u'total': 5}, 43 | # u'aggregations': {u'distinct': {u'value': 3055}}, 44 | # u'hits': {u'hits': [{u'_id': u'AVOpOpfHUw7jx-ihWUHM', 45 | # u'_index': u'hippocampe', 46 | # u'_score': 1.0, 47 | # u'_source': {u'firstAppearance': u'20160324T162502+0100', 48 | # u'idSource': u'AVOpOpBxUw7jx-ihWUGb', 49 | # u'ip': u'144.76.162.245', 50 | # u'lastAppearance': u'20160324T162502+0100', 51 | # u'source': u'https://zeustracker.abuse.ch/blocklist.php?download=ipblocklist'}, 52 | # u'_type': u'abuseFree_zeustrackerIP'}, 53 | self.size = res['aggregations']['distinct']['value'] 54 | return self.size 55 | -------------------------------------------------------------------------------- /core/services/modules/countType/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheHive-Project/Hippocampe/03df065bd304b1515db6925f6cd01a754d42825e/core/services/modules/countType/__init__.py -------------------------------------------------------------------------------- /core/services/modules/distinct/Field.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | import os 6 | import sys 7 | app_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../' 8 | pathModule = app_dir + 'services/modules/common' 9 | sys.path.insert(0, pathModule) 10 | from ES import getES 11 | from getConf import getHippoConf 12 | 13 | import logging 14 | logger = logging.getLogger(__name__) 15 | class Field: 16 | def __init__(self, field): 17 | cfg = getHippoConf() 18 | self.field = field 19 | self.es = getES() 20 | self.docSearch = dict() 21 | self.matchResponse = dict() 22 | self.indexName = cfg.get('elasticsearch', 'indexNameES') 23 | #contains every distinct value from a field 24 | self.distinctList = list() 25 | #number of distinct values 26 | self.size = int() 27 | 28 | def forgeDocSearch(self): 29 | self.docSearch = { 30 | "size": 0, 31 | "aggs": { 32 | "distinct": { 33 | "terms": { 34 | "field": self.field, 35 | "size": self.size 36 | } 37 | } 38 | } 39 | } 40 | 41 | 42 | 43 | def forgeDocSearchSize(self): 44 | self.docSearch = { 45 | "query": { 46 | "bool": { 47 | "must": [ 48 | {"exists": {"field": self.field}} 49 | ] 50 | } 51 | }, 52 | "aggs": { 53 | "distinct": { 54 | "cardinality": { 55 | "field": self.field, 56 | } 57 | } 58 | } 59 | } 60 | 61 | 62 | def search(self): 63 | #request_timeout value to avoid ConnectionTimeout exception 64 | self.matchResponse = self.es.search( 65 | index = self.indexName, 66 | body = self.docSearch, 67 | request_timeout=60) 68 | 69 | def processMatchResponse(self): 70 | ''' 71 | raw matchResponse looks like: 72 | 73 | {u'_shards': {u'failed': 0, u'successful': 5, u'total': 5}, 74 | u'aggregations': {u'distinct': {u'buckets': [{u'doc_count': 1, 75 | u'key': u'192.168.1.67'}], 76 | u'doc_count_error_upper_bound': 0, 77 | u'sum_other_doc_count': 0}}, 78 | u'hits': {u'hits': [], u'max_score': 0.0, u'total': 4}, 79 | u'timed_out': False, 80 | u'took': 9} 81 | 82 | ''' 83 | 84 | for element in self.matchResponse['aggregations']['distinct']['buckets']: 85 | self.distinctList.append(element['key']) 86 | 87 | def getSize(self): 88 | self.forgeDocSearchSize() 89 | self.search() 90 | ''' 91 | matchResponse looks like: 92 | 93 | {u'_shards': {u'failed': 0, u'successful': 5, u'total': 5}, 94 | u'aggregations': {u'distinct': {u'value': 446}}, 95 | u'hits': {u'hits': [{u'_id': u'AVmJp90mcUY7Hd0SG5cP', 96 | u'_index': u'hippocampe', 97 | u'_score': 1.0, 98 | u'_source': {u'domain': u'03bbec4.netsolhost.com', 99 | u'firstAppearance': u'20170110T193316+0100', 100 | u'idSource': u'AVmJp9c2cUY7Hd0SG5VN', 101 | u'lastAppearance': u'20170110T193316+0100', 102 | u'source': u''}, 103 | ''' 104 | self.size = self.matchResponse['aggregations']['distinct']['value'] 105 | 106 | 107 | 108 | def getDistinct(self): 109 | #since Elasticsearch does not allow the "size=0" parameters 110 | #we must get the number of all distinct doc according a specific field 111 | self.getSize() 112 | #then we execute the search with the exact size 113 | self.forgeDocSearch() 114 | self.search() 115 | self.processMatchResponse() 116 | -------------------------------------------------------------------------------- /core/services/modules/distinct/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheHive-Project/Hippocampe/03df065bd304b1515db6925f6cd01a754d42825e/core/services/modules/distinct/__init__.py -------------------------------------------------------------------------------- /core/services/modules/hipposcore/ExistingSource.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | from elasticsearch import Elasticsearch 6 | import os 7 | import sys 8 | install_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../' 9 | pathModule = install_dir + 'modules/ES' 10 | sys.path.insert(0, pathModule) 11 | from ES import getES 12 | 13 | class ExistingSource: 14 | """ 15 | ExistingSource represents a source indexed in elasticsearch 16 | It is constructed with the id of the source document in elasticsearch 17 | """ 18 | def __init__(self, idSource): 19 | self.idSource = idSource 20 | self.docMatch = str() 21 | self.matchResponse = str() 22 | self.scoreSource = float() 23 | self.es = getES() 24 | 25 | def forgeDocMatch(self): 26 | """ 27 | Forging the document to search the source, according to its 28 | id in elasticsearch 29 | """ 30 | self.docMatch = { 31 | 'query' : { 32 | 'bool' : { 33 | 'must' : [ 34 | {'match' : {'_id' : self.idSource}} 35 | ] 36 | } 37 | } 38 | } 39 | 40 | def search(self): 41 | """ 42 | Searching the document 43 | """ 44 | self.matchResponse = self.es.search(body = self.docMatch) 45 | if self.matchResponse['hits']['total'] > 0 : 46 | return True 47 | elif self.matchResponse['hits']['total'] == 0: 48 | return False 49 | 50 | def processMatchResponse(self): 51 | """ 52 | Retrieving the source's score 53 | """ 54 | self.scoreSource = self.matchResponse['hits']['hits'][0]['_source']['score'] 55 | 56 | def getScore(self): 57 | return self.scoreSource 58 | -------------------------------------------------------------------------------- /core/services/modules/hipposcore/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | -------------------------------------------------------------------------------- /core/services/modules/jobs/BagOfJobs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | import os 6 | import sys 7 | app_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../../' 8 | pathModule = app_dir + 'services/modules/common' 9 | sys.path.insert(0, pathModule) 10 | from ES import getES 11 | from getConf import getHippoConf 12 | 13 | class BagOfJobs: 14 | """ 15 | class that will retrieve all jobs 16 | it seems that a match all query and a scan scroll does not work 17 | together, that's why a match all query is used 18 | however only 10 matches are returned by elasticsearch. 19 | To go beyond that number, the number of document in source index 20 | is determined and indicated in the match all query 21 | """ 22 | def __init__(self): 23 | cfg = getHippoConf() 24 | self.docSearch = str() 25 | self.matchResponse = str() 26 | self.matchDict = dict() 27 | self.es = getES() 28 | self.indexNameES = cfg.get('elasticsearch', 'indexNameES') 29 | self.typeNameES = cfg.get('elasticsearch', 'typeNameESJobs') 30 | self.nbDoc = int() 31 | 32 | 33 | def forgeDocSearch(self): 34 | self.docSearch = { 35 | 'query' : { 36 | 'match_all' : {} 37 | } 38 | } 39 | 40 | def forgeDocSearchAll(self): 41 | self.docSearch = { 42 | 'size': self.nbDoc, 43 | 'query' : { 44 | 'match_all' : {} 45 | } 46 | } 47 | 48 | def forgeDocSearchLastJob(self): 49 | self.docSearch = { 50 | 'query' : { 51 | 'bool' : { 52 | 'must': [ 53 | { 54 | 'match': { 55 | 'status': 'done' 56 | } 57 | } 58 | ] 59 | } 60 | }, 61 | 'size' : 1, 62 | 'sort' : [ 63 | { 64 | 'endTime': { 65 | 'order': 'desc' 66 | } 67 | } 68 | ] 69 | } 70 | 71 | 72 | def search(self): 73 | self.matchResponse = self.es.search( 74 | body = self.docSearch, index= self.indexNameES, 75 | doc_type = self.typeNameES) 76 | if self.matchResponse['hits']['total'] > 0: 77 | return True 78 | elif self.matchResponse['hits']['total'] == 0: 79 | return False 80 | 81 | def processMatchResponse(self): 82 | #the match all query returns all documents, however there is 83 | #no key, only value 84 | #so the field source is setted as key and the all document 85 | #as value 86 | for element in self.matchResponse['hits']['hits']: 87 | self.matchDict[element['_id']] = element['_source'] 88 | 89 | def getMatchList(self): 90 | #because elasticsearch only returns 10 results 91 | #w have to search in two phase, first determine the nb of docs 92 | #and then retrieve all docs 93 | #matchAll query 94 | self.forgeDocSearch() 95 | #retrieve nb doc 96 | res = self.es.count(index=self.indexNameES, doc_type=self.typeNameES, body = self.docSearch) 97 | #res looks like: 98 | #{u'_shards': {u'failed': 0, u'successful': 5, u'total': 5}, u'count': 45} 99 | self.nbDoc = res['count'] 100 | self.forgeDocSearchAll() 101 | self.search() 102 | self.processMatchResponse() 103 | return self.matchDict 104 | 105 | def getLastJob(self): 106 | self.forgeDocSearchLastJob() 107 | self.search() 108 | report = self.matchResponse['hits']['hits'][0]['_source'] 109 | return report 110 | -------------------------------------------------------------------------------- /core/services/modules/jobs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheHive-Project/Hippocampe/03df065bd304b1515db6925f6cd01a754d42825e/core/services/modules/jobs/__init__.py -------------------------------------------------------------------------------- /core/services/modules/more/ObjToEnrich.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | import os, sys 5 | app_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../' 6 | modulePath = app_dir + 'services/modules/common' 7 | sys.path.insert(0, modulePath) 8 | from ES import getES 9 | from getConf import getHippoConf 10 | 11 | class ObjToEnrich: 12 | def __init__(self, typeIntel, ioc): 13 | cfg = getHippoConf() 14 | self.typeIntel = typeIntel 15 | self.value = ioc 16 | self.docMatch = str() 17 | self.matchResponse = str() 18 | self.matchList = list() 19 | self.es = getES() 20 | 21 | #data stored in index hippocampe, so search is only in this index 22 | self.indexNameES = cfg.get('elasticsearch', 'indexNameES') 23 | 24 | def forgeDocMatch(self): 25 | self.docMatch = { 26 | 'query' : { 27 | 'bool' : { 28 | 'must' : [ 29 | {'match' : {self.typeIntel : self.value}} 30 | ] 31 | } 32 | } 33 | } 34 | 35 | def forgeDocMatchInIpNetwork(self): 36 | self.docMatch = { 37 | "query" : { 38 | "bool" : { 39 | "must": [ 40 | {"range" : {"ip.from" : {"from": "0.0.0.0", "to" : self.value}}}, 41 | {"range" : {"ip.to" : {"from" : self.value, "to" : "255.255.255.255"}}} 42 | ] 43 | } 44 | } 45 | } 46 | 47 | 48 | def search(self): 49 | self.matchResponse = self.es.search( 50 | index = self.indexNameES, 51 | body = self.docMatch) 52 | if self.matchResponse['hits']['total'] > 0: 53 | return True 54 | elif self.matchResponse['hits']['total'] == 0: 55 | return False 56 | 57 | def processMatchResponse(self): 58 | for element in self.matchResponse['hits']['hits']: 59 | self.matchList.append(element['_source']) 60 | 61 | def getDetailsIP(self): 62 | self.forgeDocMatchInIpNetwork() 63 | if self.search(): 64 | self.processMatchResponse() 65 | return self.matchList 66 | else: 67 | return [] 68 | 69 | def getDetails(self): 70 | self.forgeDocMatch() 71 | if self.search(): 72 | self.processMatchResponse() 73 | return self.matchList 74 | else: 75 | return [] 76 | -------------------------------------------------------------------------------- /core/services/modules/more/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | -------------------------------------------------------------------------------- /core/services/modules/new/BagOfNew.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | import os 6 | import sys 7 | app_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../../' 8 | pathModule = app_dir + 'services/modules/common/ES' 9 | sys.path.insert(0, pathModule) 10 | from ES import getES 11 | from getConf import getHippoConf 12 | 13 | pathModule = app_dir + 'services' 14 | sys.path.insert(0, pathModule) 15 | import typeIntel 16 | 17 | class BagOfNew: 18 | def __init__(self): 19 | cfg = getHippoConf() 20 | self.docSearch = str() 21 | self.matchResponse = str() 22 | self.matchDict = dict() 23 | self.es = getES() 24 | self.indexNameES = cfg.get('elasticsearch', 'indexNameES') 25 | self.typeNameES = cfg.get('elasticsearch', 'typeNameESNew') 26 | self.nbDoc = int() 27 | 28 | 29 | def forgeDocSearch(self): 30 | self.docSearch = { 31 | 'query' : { 32 | 'match_all' : {} 33 | } 34 | } 35 | 36 | def forgeDocSearchAll(self): 37 | self.docSearch = { 38 | 'size': self.nbDoc, 39 | 'query' : { 40 | 'match_all' : {} 41 | } 42 | } 43 | 44 | def search(self): 45 | self.matchResponse = self.es.search( 46 | body = self.docSearch, index= self.indexNameES, 47 | doc_type = self.typeNameES) 48 | if self.matchResponse['hits']['total'] > 0: 49 | return True 50 | elif self.matchResponse['hits']['total'] == 0: 51 | return False 52 | 53 | def processMatchResponse(self): 54 | #{u'_shards': {u'failed': 0, u'successful': 5, u'total': 5}, 55 | # u'hits': {u'hits': [{u'_id': u'AVMLH_9WhQvX6w62oQJL', 56 | # u'_index': u'hippocampe', 57 | # u'_score': 1.0, 58 | # u'_source': {u'intelligence': u'113.53.234.218', 59 | # u'originalId': u'AVMLH_9LhQvX6w62oQJK', 60 | # u'typeIoc': u'ip'}, 61 | # u'_type': u'new'}, 62 | # {u'_id': u'AVMLH_ynhQvX6w62oQIj', 63 | # u'_index': u'hippocampe', 64 | # u'_score': 1.0, 65 | # u'_source': {u'intelligence': u'google.com', 66 | # u'originalId': u'AVMLH_yZhQvX6w62oQIi', 67 | # u'typeIoc': u'domain'}, 68 | 69 | for element in self.matchResponse['hits']['hits']: 70 | self.matchDict[element['_id']] = element['_source'] 71 | 72 | def getMatchDict(self): 73 | #because elasticsearch only returns 10 results 74 | #w have to search in two phase, first determine the nb of docs 75 | #and then retrieve all docs 76 | #matchAll query 77 | self.forgeDocSearch() 78 | #retrieve nb doc 79 | res = self.es.count(index=self.indexNameES, doc_type=self.typeNameES, body = self.docSearch) 80 | #res looks like: 81 | #{u'_shards': {u'failed': 0, u'successful': 5, u'total': 5}, u'count': 45} 82 | self.nbDoc = res['count'] 83 | self.forgeDocSearchAll() 84 | self.search() 85 | self.processMatchResponse() 86 | return self.matchDict 87 | -------------------------------------------------------------------------------- /core/services/modules/new/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheHive-Project/Hippocampe/03df065bd304b1515db6925f6cd01a754d42825e/core/services/modules/new/__init__.py -------------------------------------------------------------------------------- /core/services/modules/shadowbook/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheHive-Project/Hippocampe/03df065bd304b1515db6925f6cd01a754d42825e/core/services/modules/shadowbook/__init__.py -------------------------------------------------------------------------------- /core/services/modules/shadowbook/bulkOp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | from elasticsearch import helpers 6 | import logging 7 | logger = logging.getLogger(__name__) 8 | from time import strftime 9 | import os, sys 10 | app_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../../../' 11 | getESpath = app_dir + 'services/common' 12 | sys.path.insert(0, getESpath) 13 | from ES import getES 14 | from getConf import getConf 15 | from getConf import getHippoConf 16 | from objects.IndexNew import IndexNew 17 | from objects.IndexIntel import IndexIntel 18 | 19 | def update(typeNameES, listId): 20 | logger.info('bulkOp.update launched') 21 | hippoCfg = getHippoConf() 22 | es = getES() 23 | now = strftime("%Y%m%dT%H%M%S%z") 24 | indexNameES = hippoCfg.get('elasticsearch', 'indexNameES') 25 | # k is a generator expression that produces 26 | # dict to update every doc wich id is in listId 27 | k = ({'_op_type': 'update', '_index':indexNameES, '_type':typeNameES, 'doc':{'lastQuery': now}, '_id': id} 28 | for id in listId) 29 | 30 | res = helpers.bulk(es, k) 31 | logger.info('bulkOp.update res: %s', res) 32 | #res looks like 33 | #(2650, []) 34 | logger.info('bulkOp.update end') 35 | return res[0] 36 | 37 | def index(cfgPath, listData): 38 | logger.info('bulkOp.index launched') 39 | hippoCfg = getHippoConf() 40 | indexNameES = hippoCfg.get('elasticsearch', 'indexNameES') 41 | 42 | cfg = getConf(cfgPath) 43 | typeNameES = cfg.get('elasticsearch', 'typeIntel') 44 | 45 | #creating the index, only if does not exist 46 | index = IndexIntel(cfgPath) 47 | index.createIndexIntel() 48 | 49 | es = getES() 50 | k = ({'_op_type': 'index', '_index':indexNameES, '_type':typeNameES, '_source': data} 51 | for data in listData) 52 | res = helpers.bulk(es,k, raise_on_error=False) 53 | #res = helpers.bulk(es,k, raise_on_exception=False) 54 | #res = helpers.bulk(es,k) 55 | logger.info('bulkOp.index res: %s', res) 56 | logger.info('bulkOp.index end') 57 | return res 58 | 59 | def indexNew(coreIntelligence, listData): 60 | logger.info('bulkOp.indexNew launched') 61 | 62 | hippoCfg = getHippoConf() 63 | indexNameES = hippoCfg.get('elasticsearch', 'indexNameES') 64 | typeNameES = hippoCfg.get('elasticsearch', 'typeNameESNew') 65 | 66 | indexNew = IndexNew() 67 | indexNew.createIndexNew() 68 | 69 | es = getES() 70 | k = ({'_op_type': 'index', '_index':indexNameES, '_type':typeNameES, '_source': {'type': coreIntelligence, 'toSearch': data[coreIntelligence]}} 71 | for data in listData) 72 | #k.next() gives: 73 | #{'_op_type': 'index', '_index':'hippocampe', '_type':'new', '_source': {'typeIntel': 'ip', 'intelligence': '1.1.1.1'} 74 | res = helpers.bulk(es,k) 75 | logger.info('bulkOp.index res: %s', res) 76 | logger.info('bulkOp.indexNew end') 77 | return res[0] 78 | if __name__ == '__main__': 79 | main() 80 | -------------------------------------------------------------------------------- /core/services/modules/shadowbook/createSessions.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | import os, sys 6 | app_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../../' 7 | #path to getConf function 8 | getConfPath = app_dir + 'services/modules/common' 9 | sys.path.insert(0, getConfPath) 10 | from getConf import getConf 11 | import requests 12 | from copy import deepcopy 13 | import logging 14 | logger = logging.getLogger(__name__) 15 | 16 | def createSessions(): 17 | logger.info('createSessions.createSession launched') 18 | authConfPath = app_dir + 'conf/auth/auth.conf' 19 | cfg = getConf(authConfPath) 20 | listSessions = list() 21 | element = dict() 22 | 23 | ''' 24 | a session is represented by a JSON object named element: 25 | { 26 | 'sessionName': '', 27 | 'session': objectSession 28 | } 29 | this JSON object is then put into a list: listSessions 30 | 31 | [ 32 | { 33 | 'session': , 34 | 'sessionName': 'default' 35 | }, 36 | { 37 | 'session': , 38 | 'sessionName': u'https://top_level_url.foo' 39 | } 40 | ] 41 | ''' 42 | 43 | #create default session element 44 | element['sessionName'] = 'default' 45 | element['session'] = requests.Session() 46 | #adding the default session element to the listSessions 47 | #deepcopy is used because we need a complete copy and not a reference 48 | listSessions.append(deepcopy(element)) 49 | 50 | #creating session for each top_level_url to be authenticated 51 | #each section is the top_level_url 52 | for section in cfg.sections(): 53 | 54 | # retrieving username and password from conf file 55 | username = cfg.get(section, 'username') 56 | password = cfg.get(section, 'password') 57 | 58 | #naming the session with the top_level_url 59 | element['sessionName'] = section 60 | element['session'] = requests.Session() 61 | #authenticating the session 62 | element['session'].auth=(username, password) 63 | 64 | logger.info('Authenticated session created for: %s', section) 65 | 66 | #adding the new session element to the list 67 | listSessions.append(deepcopy(element)) 68 | logger.info('createSessions.createSessions end') 69 | return listSessions 70 | 71 | if __name__ == '__main__': 72 | createSessions() 73 | -------------------------------------------------------------------------------- /core/services/modules/shadowbook/downloader.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | import requests 5 | from copy import deepcopy 6 | 7 | def simpleDownload(url, listSessions): 8 | #listSessions is a list with authenticated sessions for feed which need to be authenticate 9 | 10 | #Before retrieving feed's data, we have to determine which session to use 11 | #by using listSessions[1:] we do not choose the first element which is the default session 12 | #in case that the top_level_url contains 'default' 13 | for session in listSessions[1:]: 14 | if session['sessionName'] in url: 15 | selectedSession = deepcopy(session) 16 | else: 17 | #the default session is always the first element in the list 18 | selectedSession = listSessions[0] 19 | 20 | raw_page = selectedSession['session'].get(url, stream = True) 21 | #requests does not raise exception if status is not 200 by default 22 | #that's why we use raise_for_status() 23 | raw_page.raise_for_status() 24 | #returning raw content and not text to avoid encoding issues 25 | return raw_page.content 26 | 27 | class EmptyFeedException(Exception): 28 | def __init__(self,url): 29 | Exception.__init__(self, 'Feed {0} is empty'.format(url)) 30 | self.url = url 31 | 32 | 33 | if __name__ == '__main__': 34 | main() 35 | -------------------------------------------------------------------------------- /core/services/modules/shadowbook/enricher.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | from time import strftime 5 | 6 | def metadata(idSource, source, data): 7 | #data is a list of dict 8 | #each element is a dict representing a feed's parsed line 9 | for element in data: 10 | element.pop('blank', None) 11 | element['lastAppearance'] = strftime("%Y%m%dT%H%M%S%z") 12 | element['firstAppearance'] = strftime("%Y%m%dT%H%M%S%z") 13 | element['idSource'] = idSource 14 | element['source'] = source 15 | 16 | return data 17 | if __name__ == '__main__': 18 | main() 19 | -------------------------------------------------------------------------------- /core/services/modules/shadowbook/extendTools.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | from itertools import imap, groupby 5 | from operator import itemgetter 6 | 7 | def unique_justseen(iterable, key=None): 8 | "List unique elements, preserving order. Remember only the element just seen." 9 | # unique_justseen('AAAABBBCCDAABBB') --> A B C D A B 10 | # unique_justseen('ABBCcAD', str.lower) --> A B C A D 11 | return imap(next, imap(itemgetter(1), groupby(iterable, key))) 12 | -------------------------------------------------------------------------------- /core/services/modules/shadowbook/objects/Index.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | """ 6 | This module contains the class Index 7 | ==================================== 8 | """ 9 | from elasticsearch import client 10 | import os, sys 11 | install_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../../../' 12 | pathModule = install_dir + 'services/modules/common' 13 | sys.path.insert(0, pathModule) 14 | from ES import getES 15 | import logging 16 | logger = logging.getLogger(__name__) 17 | class Index(object): 18 | """ 19 | Index class represents an elasticsearch's index. 20 | 21 | This class is kind of an abstract class, because it is the common base 22 | of IndexSource and IndexIOC classes. 23 | """ 24 | 25 | def __init__(self): 26 | """ 27 | Index class' constructor. 28 | """ 29 | 30 | self.indexNameES = str() 31 | self.typeNameES = str() 32 | self.docMapping = dict() 33 | self.es = getES() 34 | #self.logger = logging.getLogger('app.shadowbook') 35 | 36 | def create(self): 37 | #create indexES instance 38 | indexES = client.IndicesClient(self.es) 39 | if(self.es.indices.exists(index = self.indexNameES)): 40 | #logger.info('index %s already exists', self.indexNameES) 41 | #index already exists but it does not mean that the type exists 42 | if(self.es.indices.exists_type(index = self.indexNameES, doc_type = [self.typeNameES])): 43 | #logger.info('type %s already exists', self.typeNameES) 44 | #type already exists nothing to do 45 | pass 46 | else: 47 | #type does not exists, creating it with the mapping to apply 48 | #logger.info('type %s does no exist, creating it', self.typeNameES) 49 | indexES.put_mapping(doc_type = self.typeNameES, body = self.docMapping) 50 | else: 51 | #index does not exists, neither type (type can't exist without index) 52 | #creating both 53 | #logger.info('index %s and type %s do not exist, creating them', self.indexNameES, self.typeNameES) 54 | indexES.create(index = self.indexNameES) 55 | #indicate mapping which applies only on index/type 56 | indexES.put_mapping(doc_type = self.typeNameES, body = self.docMapping) 57 | -------------------------------------------------------------------------------- /core/services/modules/shadowbook/objects/IndexIntel.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | """ 6 | This module contains the class IndexIOC. 7 | ======================================== 8 | """ 9 | from Index import Index 10 | import os, sys 11 | install_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../../../' 12 | getConfPath = install_dir + 'services/modules/common' 13 | sys.path.insert(0, getConfPath) 14 | import getConf 15 | from itertools import izip_longest 16 | import ast 17 | class IndexIntel(Index): 18 | """ 19 | IndexIOC class represents an elasticsearch's index. And more precisely 20 | an index storing IOC. 21 | 22 | The class inherits from Index class. 23 | """ 24 | 25 | def __init__(self, cfgPath): 26 | """ 27 | IndexIOC class' constructor. 28 | 29 | :param config: shadowBook process's configuration 30 | :type config: Config object instance 31 | """ 32 | 33 | super(IndexIntel, self).__init__() 34 | self.conf = getConf.getConf(cfgPath) 35 | hippoConf = getConf.getHippoConf() 36 | self.indexNameES = hippoConf.get('elasticsearch', 'indexNameES') 37 | self.typeNameES = self.conf.get('elasticsearch', 'typeIntel') 38 | 39 | #the mapping has to be indicated at the creation of the index 40 | #the confMapping, is the mapping indicated in conf file for each field 41 | #it has to be parsed bellow 42 | self.confMapping = dict() 43 | 44 | def buildConfMapping(self): 45 | def listToDict(inputList): 46 | """ 47 | Little function that converts a list into a dict. 48 | 49 | This function is needed because ConfigParser.items() returns all options 50 | from a given section as a list. A dict is more user-freindly to use. 51 | 52 | :param inputList: list to be converted into dict 53 | :type inputList: list 54 | :return: the list converted as a dict 55 | :rtype: dict 56 | """ 57 | dictOutput = dict() 58 | for couple in inputList: 59 | tmpDict = dict(izip_longest(*[iter(couple)] * 2, fillvalue='')) 60 | for nameIoc, value in tmpDict.items(): 61 | dictOutput[nameIoc] = value 62 | return dictOutput 63 | 64 | #items retrieve all options from 'intel' section, but as a list. 65 | #the list is converted to a dict 66 | listMapping = self.conf.items('intel') 67 | self.confMapping = listToDict(listMapping) 68 | #at the end, self.confMapping looks like this: 69 | #{u'domain': u'{\n"mapping" : {\n"type": "string",\n"index": "not_analyzed"\n}\n}', 70 | # u'extra': u'{\n"mapping" : {\n"type": "string",\n"index": "not_analyzed"\n}\n}', 71 | # u'nextvalidation': u'{\n"mapping" : {\n"type": "string",\n"index": "not_analyzed"\n}\n}', 72 | # u'original_reference-why_it_was_listed': u'{\n"mapping" : {\n"type": "string",\n"index": "not_analyzed"\n}\n}', 73 | # u'type': u'{\n"mapping" : {\n"type": "string",\n"index": "not_analyzed"\n}\n}'} 74 | #However, please note that the key AND the value is unicode 75 | #so u'{\n"mapping" : {\n"type": "string",\n"index": "not_analyzed"\n}\n}'} is a unicode 76 | #formated as a string 77 | 78 | def forgeDocMapping(self): 79 | """ 80 | Forges the elasticsearch's mappings for IndexIOC. 81 | """ 82 | #mapping for the meta-data 83 | self.docMapping = { 84 | self.typeNameES : { 85 | "properties" : { 86 | "firstAppearance": { 87 | "type": "date", 88 | "format": "basic_date_time_no_millis" 89 | }, 90 | "idSource": { 91 | "type": "keyword" 92 | }, 93 | "lastAppearance": { 94 | "type": "date", 95 | "format": "basic_date_time_no_millis" 96 | }, 97 | "source": { 98 | "type": "keyword", 99 | "index": "not_analyzed" 100 | } 101 | } 102 | } 103 | } 104 | for intel, conf in self.confMapping.items(): 105 | #adding ioc's mapping 106 | #conf here is a unicode formated as a dict, has to actually "convert" it to dict 107 | dicoTmp = ast.literal_eval(conf) 108 | self.docMapping[self.typeNameES]['properties'][intel] = dicoTmp['mapping'] 109 | #at the end, the docMapping looks like this: 110 | #{u'malwaredomainsFree_dnsbhDOMAIN': {'properties': {u'domain': {'index': 'not_analyzed', 111 | # 'type': 'string'}, 112 | # u'extra': {'index': 'not_analyzed', 113 | # 'type': 'string'}, 114 | # 'firstAppearance': {'format': 'basic_date_time_no_millis', 115 | # 'type': 'date'}, 116 | # 'idSource': {'type': 'string'}, 117 | # 'lastAppearance': {'format': 'basic_date_time_no_millis', 118 | # 'type': 'date'}, 119 | # u'nextvalidation': {'index': 'not_analyzed', 120 | # 'type': 'string'}, 121 | # u'original_reference-why_it_was_listed': {'index': 'not_analyzed', 122 | # 'type': 'string'}, 123 | # 'source': {'type': 'string'}, 124 | # u'type': {'index': 'not_analyzed', 125 | # 'type': 'string'}}}} 126 | # 127 | 128 | def createIndexIntel(self): 129 | self.buildConfMapping() 130 | self.forgeDocMapping() 131 | self.create() 132 | -------------------------------------------------------------------------------- /core/services/modules/shadowbook/objects/IndexJob.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | from Index import Index 6 | import os, sys 7 | app_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../../../' 8 | getConfPath = app_dir + 'services/modules/common' 9 | sys.path.insert(0, getConfPath) 10 | from getConf import getHippoConf 11 | from itertools import izip_longest 12 | import ast 13 | import logging 14 | 15 | logger = logging.getLogger(__name__) 16 | class IndexJob(Index): 17 | """ 18 | IndexIOC class represents an elasticsearch's index. And more precisely 19 | an index storing IOC. 20 | 21 | The class inherits from Index class. 22 | """ 23 | 24 | # def __init__(self, cfgPath): 25 | def __init__(self): 26 | 27 | super(IndexJob, self).__init__() 28 | cfg = getHippoConf() 29 | self.indexNameES = cfg.get('elasticsearch', 'indexNameES') 30 | self.typeNameES = cfg.get('elasticsearch', 'typeNameESJobs') 31 | 32 | #the mapping has to be indicated at the creation of the index 33 | #the confMapping, is the mapping indicated in conf file for each field 34 | #it has to be parsed bellow 35 | self.confMapping = dict() 36 | 37 | def forgeDocMapping(self): 38 | """ 39 | Forges the elasticsearch's mappings for IndexIOC. 40 | """ 41 | #mapping for the meta-data 42 | self.docMapping = { 43 | self.typeNameES : { 44 | # "dynamic": "strict", 45 | "properties" : { 46 | # "report": { 47 | # "type": "date", 48 | # "index": "not_analyzed" 49 | # }, 50 | "status": { 51 | "type": "keyword", 52 | "index": "not_analyzed" 53 | }, 54 | "startTime": { 55 | "type": "date", 56 | "format": "basic_date_time_no_millis" 57 | }, 58 | "endTime": { 59 | "type": "date", 60 | "format": "basic_date_time_no_millis" 61 | }, 62 | "duration": { 63 | "type": "float" 64 | } 65 | } 66 | } 67 | } 68 | 69 | def createIndexJob(self): 70 | self.forgeDocMapping() 71 | self.create() 72 | -------------------------------------------------------------------------------- /core/services/modules/shadowbook/objects/IndexNew.py: -------------------------------------------------------------------------------- 1 | #@see The GNU Public License (GPL) 2 | #This file is part of Hippocampe. 3 | # 4 | #Hippocampe is free software; you can redistribute it and/or modify 5 | #it under the terms of the GNU General Public License as published by 6 | #the Free Software Foundation; either version 3 of the License, or 7 | #(at your option) any later version. 8 | # 9 | #Hippocampe is distributed in the hope that it will be useful, but 10 | #WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY 11 | #or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 | #for more details. 13 | # 14 | #You should have received a copy of the GNU General Public License along 15 | #with Hippocampe; if not, write to the Free Software Foundation, Inc., 16 | #59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 17 | #or see . 18 | # 19 | 20 | """ 21 | This module contains the class IndexIOC. 22 | ======================================== 23 | """ 24 | from Index import Index 25 | import os, sys 26 | app_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../../../' 27 | getConfPath = app_dir + 'services/modules/common' 28 | sys.path.insert(0, getConfPath) 29 | from getConf import getHippoConf 30 | 31 | class IndexNew(Index): 32 | """ 33 | The class inherits from Index class. 34 | """ 35 | 36 | def __init__(self): 37 | 38 | super(IndexNew, self).__init__() 39 | cfg = getHippoConf() 40 | self.indexNameES = cfg.get('elasticsearch', 'indexNameES') 41 | self.typeNameES = 'new' 42 | 43 | def forgeDocMapping(self): 44 | """ 45 | Forges the elasticsearch's mappings for IndexIOC. 46 | """ 47 | #mapping for the meta-data 48 | self.docMapping = { 49 | #"mappings" : { 50 | self.typeNameES : { 51 | "properties" : { 52 | "toSearch": { 53 | "type": "keyword", 54 | "index": "not_analyzed" 55 | }, 56 | "type": { 57 | "type": "keyword", 58 | "index": "not_analyzed" 59 | } 60 | } 61 | } 62 | } 63 | 64 | def createIndexNew(self): 65 | self.forgeDocMapping() 66 | self.create() 67 | -------------------------------------------------------------------------------- /core/services/modules/shadowbook/objects/IndexSource.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | """ 6 | This module contains the class IndexSource. 7 | =========================================== 8 | """ 9 | from Index import Index 10 | import os, sys 11 | install_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../../../' 12 | getConfPath = install_dir + 'services/modules/common' 13 | sys.path.insert(0, getConfPath) 14 | from getConf import getHippoConf 15 | 16 | class IndexSource(Index): 17 | """ 18 | IndexIOC class represents an elasticsearch's index. And more precisely 19 | an index storing source. 20 | 21 | The class inherits from Index class. 22 | """ 23 | 24 | def __init__(self, cfgPath): 25 | """ 26 | IndexSource class' constructor. 27 | 28 | :param source: shadowBook process's configuration 29 | :type config: Config object instance 30 | """ 31 | conf = getHippoConf() 32 | 33 | super(IndexSource, self).__init__() 34 | self.indexNameES = conf.get('elasticsearch', 'indexNameES') 35 | self.typeNameES = conf.get('elasticsearch', 'typeNameESSource') 36 | 37 | 38 | def forgeDocMapping(self): 39 | """ 40 | Forges the elasticsearch's mappings for IndexSource. 41 | """ 42 | self.docMapping = { 43 | self.typeNameES: { 44 | "properties": { 45 | "lastStatus": { 46 | "type": "keyword", 47 | "index": "not_analyzed" 48 | }, 49 | "link": { 50 | "type": "keyword", 51 | "index": "not_analyzed" 52 | }, 53 | "type": { 54 | "type": "keyword", 55 | "index": "not_analyzed" 56 | }, 57 | "firstQuery": { 58 | "type": "date", 59 | "format": "basic_date_time_no_millis" 60 | }, 61 | "lastQuery": { 62 | "type": "date", 63 | "format": "basic_date_time_no_millis" 64 | }, 65 | "description": { 66 | "fielddata": True, 67 | "type": "text" 68 | }, 69 | "score": { 70 | "type": "integer" 71 | }, 72 | "coreIntelligence": { 73 | "type": "keyword" 74 | } 75 | } 76 | } 77 | } 78 | 79 | def createIndexSource(self): 80 | self.forgeDocMapping() 81 | self.create() 82 | -------------------------------------------------------------------------------- /core/services/modules/shadowbook/objects/Intel.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | from ObjToIndex import ObjToIndex 5 | import os, sys 6 | install_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../../../' 7 | getConfPath = install_dir + 'services/modules/common' 8 | sys.path.insert(0, getConfPath) 9 | import getConf 10 | from IndexIntel import IndexIntel 11 | from time import strftime 12 | from NewIntel import NewIntel 13 | class Intel(ObjToIndex): 14 | def __init__(self, intelligence, cfgPath, idSource): 15 | 16 | #intelligence is a dict: 17 | #{'blank': '', 18 | # 'domain': 'uploadersonline.com', 19 | # u'extra': ['20160229'], 20 | # 'nextvalidation': '', 21 | # 'original_reference-why_it_was_listed': 'zeustracker.abuse.ch', 22 | # 'type': 'zeus'} 23 | super(Intel, self).__init__() 24 | self.cfgPath = cfgPath 25 | conf = getConf.getConf(cfgPath) 26 | self.indexNameES = conf.get('elasticsearch', 'indexIOC') 27 | self.typeNameES = conf.get('elasticsearch', 'typeIntel') 28 | self.idSource = idSource 29 | self.firstAppearance = str() 30 | self.lastAppearance = str() 31 | self.coreIntelligence = conf.get('source', 'coreIntelligence') 32 | self.dictData = intelligence 33 | self.source= conf.get('source', 'url') 34 | 35 | def indexIntelInES(self): 36 | #create the index 37 | index = IndexIntel(self.cfgPath) 38 | index.buildConfMapping() 39 | index.forgeDocMapping() 40 | index.create() 41 | #search the coreIntelligence in the index/type ONLY 42 | nbMatch = self.littleSearch() 43 | if nbMatch > 0: 44 | #match, so update last appearance date 45 | self.updateIntel() 46 | elif nbMatch == 0: 47 | #no match, so index 48 | self.forgeDocIndex() 49 | self.indexInES() 50 | #at this point, we are sure that there's no match in the index/type 51 | #have to check in the whole index hippocampe 52 | nbMatch = self.bigSearch() 53 | if nbMatch == 0: 54 | newIntel = NewIntel(self.coreIntelligence, self.dictData[self.coreIntelligence]) 55 | #indexing the new intel in the index/type hippocampe/new 56 | newIntel.indexNewIntel() 57 | 58 | def bigSearch(self): 59 | self.es.indices.refresh(index = self.indexNameES) 60 | #searching in the index hippocampe 61 | self.resSearch = self.es.search(index = self.indexNameES, body = self.docSearch) 62 | return self.resSearch['hits']['total'] 63 | 64 | 65 | def littleSearch(self): 66 | self.forgeDocSearch() 67 | return self.search() 68 | 69 | def forgeDocSearch(self): 70 | self.docSearch = { 71 | "query": { 72 | "bool": { 73 | "must": [ 74 | { 75 | "match": { 76 | self.coreIntelligence : self.dictData[self.coreIntelligence] 77 | } 78 | } 79 | ] 80 | } 81 | } 82 | } 83 | 84 | def forgeDocUpdate(self): 85 | self.docUpdate = { 86 | "script": { 87 | "lang": "painless", 88 | "inline": "ctx._source.lastAppearance = params.lastAppearance", 89 | "params": { 90 | "lastAppearance": self.lastAppearance 91 | } 92 | } 93 | } 94 | 95 | def forgeDocIndex(self): 96 | #the dictData is already well-formed for index queries 97 | #however, sometimes, it can have a blank field, which is useless 98 | #and will be deleted 99 | self.dictData.pop('blank', None) 100 | self.docIndex = self.dictData 101 | #adding the meta data 102 | #new intelligence, so lastAppearance and first appearance are setted to today 103 | self.lastAppearance = strftime("%Y%m%dT%H%M%S%z") 104 | self.firstAppearance = strftime("%Y%m%dT%H%M%S%z") 105 | self.docIndex ['lastAppearance'] = self.lastAppearance 106 | self.docIndex['firstAppearance'] = self.firstAppearance 107 | self.docIndex['idSource'] = self.idSource 108 | self.docIndex['source'] = self.source 109 | 110 | def updateIntel(self): 111 | self.lastAppearance = strftime("%Y%m%dT%H%M%S%z") 112 | self.forgeDocUpdate() 113 | self.update() 114 | -------------------------------------------------------------------------------- /core/services/modules/shadowbook/objects/Job.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | """ 6 | This module contains the class Source. 7 | ====================================== 8 | """ 9 | from ObjToIndex import ObjToIndex 10 | import os, sys 11 | install_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../../../' 12 | getConfPath = install_dir + 'services/modules/common' 13 | sys.path.insert(0, getConfPath) 14 | from getConf import getHippoConf 15 | from IndexSource import IndexSource 16 | from time import strftime 17 | import logging 18 | logger = logging.getLogger(__name__) 19 | import dateutil.parser 20 | 21 | class Job(ObjToIndex): 22 | 23 | def __init__(self): 24 | super(Job, self).__init__() 25 | cfg = getHippoConf() 26 | self.indexNameES = cfg.get('elasticsearch', 'indexNameES') 27 | self.typeNameES = cfg.get('elasticsearch', 'typeNameESJobs') 28 | self.status = str() 29 | self.startTime = str() 30 | self.endTime = str() 31 | self.duration = str() 32 | self.report = dict() 33 | 34 | def forgeDocSearchOngoing(self): 35 | """ 36 | Forges the search query to search source in elasticsearch. 37 | """ 38 | self.docSearch = { 39 | "query": { 40 | "bool": { 41 | "must": [ 42 | { 43 | "match": { 44 | "status": "ongoing" 45 | } 46 | } 47 | ] 48 | } 49 | } 50 | } 51 | 52 | 53 | def forgeDocIndexOngoing(self): 54 | self.docIndex = { 55 | "status": "ongoing", 56 | "startTime": strftime("%Y%m%dT%H%M%S%z"), 57 | 58 | } 59 | 60 | def forgeDocUpdateJob(self): 61 | self.docUpdate = { 62 | "script": { 63 | "lang": "painless", 64 | "inline": "ctx._source.status = params.status; ctx._source.endTime = params.endTime; ctx._source.duration = params.duration; ctx._source.report = params.report", 65 | "params": { 66 | "status": self.status, 67 | "endTime": self.endTime, 68 | "duration": self.duration, 69 | "report": self.report, 70 | } 71 | } 72 | } 73 | 74 | def searchOngoingJob(self): 75 | self.forgeDocSearchOngoing() 76 | nbOngoingJob = self.search() 77 | return nbOngoingJob 78 | 79 | def indexOngoingJob(self): 80 | self.forgeDocIndexOngoing() 81 | self.indexInES() 82 | return self.idInES 83 | 84 | def getStartTime(self): 85 | self.searchOngoingJob() 86 | #self.resSearch looks like: 87 | #{u'hits': {u'hits': [{u'_score': 11.907724, u'_type': u'jobs', u'_id': u'AVQkDgNs5CYfHJgxlSLE', u'_source': {u'status': u'ongoing', u'startTime': u'20160417T134938+0200'}, u'_index': u'hippocampe'}], u'total': 1, u'max_score': 11.907724}, u'_shards': {u'successful': 5, u'failed': 0, u'total': 5}, u'took': 1, u'timed_out': False} 88 | startTime = self.resSearch['hits']['hits'][0]['_source']['startTime'] 89 | return startTime 90 | 91 | def calcDuration(self): 92 | self.startTime = self.getStartTime() 93 | duration = dateutil.parser.parse(self.endTime) - dateutil.parser.parse(self.startTime) 94 | #duration in seconds 95 | duration = duration.total_seconds() 96 | #duration in minutes 97 | duration = duration / 60.0 98 | #limiting to 2 digits 99 | self.duration = "%.2f" % duration 100 | 101 | def updateStatus(self, report, status): 102 | self.endTime = strftime("%Y%m%dT%H%M%S%z") 103 | self.status = status 104 | self.report = report 105 | 106 | self.calcDuration() 107 | self.forgeDocUpdateJob() 108 | self.update() 109 | -------------------------------------------------------------------------------- /core/services/modules/shadowbook/objects/NewIntel.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | """ 6 | This module contains Ioc class. 7 | =============================== 8 | """ 9 | from ObjToIndex import ObjToIndex 10 | from IndexNew import IndexNew 11 | import os, sys 12 | app_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../../../' 13 | getConfPath = app_dir + 'services/modules/common' 14 | sys.path.insert(0, getConfPath) 15 | from getConf import getHippoConf 16 | 17 | class NewIntel(ObjToIndex): 18 | """ 19 | 20 | It inherits from ObjToIndex class. 21 | """ 22 | 23 | def __init__(self, typeIntel, intelligence): 24 | """ 25 | Ioc class' constructor. 26 | 27 | :param config: Config object that contains informations about the ioc. 28 | :type config: Config object instance. 29 | :param source: Ioc's Source object. 30 | :type source: Source object instance. 31 | """ 32 | 33 | super(NewIntel, self).__init__() 34 | cfg = getHippoConf() 35 | self.indexNameES = cfg.get('elasticsearch', 'indexNameES') 36 | self.typeNameES = 'new' 37 | self.typeIntel = typeIntel 38 | self.intelligence = intelligence 39 | 40 | def forgeDocIndex(self): 41 | self.docIndex['typeIntel'] = self.typeIntel 42 | self.docIndex['intelligence'] = self.intelligence 43 | 44 | def indexNewIntel(self): 45 | index = IndexNew() 46 | index.createIndexNew() 47 | self.forgeDocIndex() 48 | self.indexInES() 49 | -------------------------------------------------------------------------------- /core/services/modules/shadowbook/objects/ObjToIndex.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | """ 6 | This module contains ObjToIndex class. 7 | ====================================== 8 | """ 9 | 10 | from elasticsearch import Elasticsearch 11 | import os, sys 12 | install_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../../../' 13 | getESpath = install_dir + 'services/common' 14 | sys.path.insert(0, getESpath) 15 | from ES import getES 16 | import traceback 17 | 18 | class ObjToIndex(object): 19 | """ 20 | ObjToIndex class represents any object that can be indexed in elasticsearch. 21 | 22 | This class is kind of an abstract class, because is is the common base 23 | of Source and Ioc classes. 24 | """ 25 | 26 | def __init__(self): 27 | """ 28 | ObjToIndex class' constructor. 29 | """ 30 | 31 | self.es = getES() 32 | self.idInES = str() 33 | self.indexNameES = str() 34 | self.typeNameES = str() 35 | self.docIndex = dict() 36 | self.docSearch = dict() 37 | self.docUpdate = dict() 38 | self.resSearch = dict() 39 | 40 | def indexInES(self): 41 | """ 42 | Indexes the object in elasticsearch from the docIndex query. 43 | 44 | :raises: RequestError, it can happened when the mappings is misindicated. 45 | Especially for date type. 46 | 47 | """ 48 | try: 49 | qIndex = self.es.index(index = self.indexNameES, doc_type = self.typeNameES, body = self.docIndex) 50 | self.idInES = qIndex['_id'] 51 | except Exception as e: 52 | exception_type = e.__class__.__name__ 53 | output = 'index failed: ' + exception_type 54 | output = output + '\n' + traceback.format_exc() 55 | #self.logger.error(output) 56 | #self.logger.info(self.docIndex) 57 | 58 | def search(self): 59 | """ 60 | Searchs the object in elasticsearch from the docSearch query. 61 | If match, the match's id is setted to idES. 62 | 63 | :return: self.resSearch['hits']['total'] which is the number of matches. 64 | :rtype: int 65 | """ 66 | self.es.indices.refresh(index = self.indexNameES) 67 | self.resSearch = self.es.search(index = self.indexNameES, doc_type = self.typeNameES, body = self.docSearch) 68 | if (self.resSearch['hits']['total'] > 0): 69 | self.idES = self.resSearch['hits']['hits'][0]['_id'] 70 | return self.resSearch['hits']['total'] 71 | 72 | def update(self): 73 | """ 74 | Updates the object in elasticsearch from the docUpdate query. 75 | """ 76 | qUpdate = self.es.update(index = self.indexNameES, doc_type = self.typeNameES, id = self.idES, body = self.docUpdate) 77 | 78 | -------------------------------------------------------------------------------- /core/services/modules/shadowbook/objects/Source.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | """ 6 | This module contains the class Source. 7 | ====================================== 8 | """ 9 | from ObjToIndex import ObjToIndex 10 | import os, sys 11 | install_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../../../' 12 | getConfPath = install_dir + 'services/modules/common' 13 | sys.path.insert(0, getConfPath) 14 | import getConf 15 | from IndexSource import IndexSource 16 | from time import strftime 17 | import dateutil.parser 18 | import logging 19 | logger = logging.getLogger(__name__) 20 | 21 | class Source(ObjToIndex): 22 | """ 23 | Source class represents a feed and its informations. 24 | 25 | It inherits from ObjToIndex class. 26 | """ 27 | 28 | def __init__(self, cfgPath): 29 | """ 30 | Source class' constructor. 31 | 32 | :param config: Config object that contains informations about the feed. 33 | :type config: Config object instance. 34 | """ 35 | self.cfgPath = cfgPath 36 | #conf from Hippocampe/core/conf/feeds 37 | conf = getConf.getConf(cfgPath) 38 | 39 | #conf from Hippocampe/core/conf/hippo 40 | cfg = getConf.getHippoConf() 41 | 42 | super(Source, self).__init__() 43 | self.source = conf.get('source', 'url') 44 | self.firstQuery = str() 45 | self.lastQuery = str() 46 | self.description = conf.get('source', 'description') 47 | 48 | self.indexNameES = cfg.get('elasticsearch', 'indexNameES') 49 | self.typeNameES = cfg.get('elasticsearch', 'typeNameESSource') 50 | 51 | self.score = conf.getint('source', 'score') 52 | self.coreIntelligence = conf.get('source', 'coreIntelligence') 53 | self.typeNameESIntel = conf.get('elasticsearch', 'typeIntel') 54 | self.validityDate = conf.get('source', 'validityDate') 55 | self.useByDate = conf.get('source', 'useBydate') 56 | 57 | def forgeDocSearch(self): 58 | """ 59 | Forges the search query to search source in elasticsearch. 60 | """ 61 | self.docSearch = { 62 | "query": { 63 | "bool": { 64 | "must": [ 65 | { 66 | "match": { 67 | "link": self.source 68 | } 69 | } 70 | #{ 71 | # "match": { 72 | # "type": self.typeNameES 73 | # } 74 | #} 75 | ] 76 | } 77 | } 78 | } 79 | 80 | 81 | def forgeDocIndex(self): 82 | """ 83 | Forges the index query to index source in elasticsearch. 84 | """ 85 | self.docIndex = { 86 | "link": self.source, 87 | "type": self.typeNameESIntel, 88 | "firstQuery": self.firstQuery, 89 | "lastQuery": self.lastQuery, 90 | "description": self.description, 91 | "score": self.score, 92 | "coreIntelligence": self.coreIntelligence 93 | } 94 | 95 | def forgeDocUpdateLastQuery(self): 96 | """ 97 | Forges the update query to update source in elasticsearch. 98 | """ 99 | self.docUpdate = { 100 | "script": { 101 | "lang": "painless", 102 | "inline": "ctx._source.lastQuery = params.lastQuery", 103 | "params": { 104 | "lastQuery": self.lastQuery 105 | } 106 | } 107 | } 108 | 109 | def forgeDocUpdateScore(self): 110 | self.docUpdate = { 111 | "script": { 112 | "lang": "painless", 113 | "inline": "ctx._source.score = params.score", 114 | "params": { 115 | "score": self.score 116 | } 117 | } 118 | } 119 | 120 | def getScoreInES(self): 121 | """ 122 | Returns the source's score as it is indexed in ES and 123 | not as in the conf file 124 | """ 125 | return self.resSearch['hits']['hits'][0]['_source']['score'] 126 | 127 | def indexSourceInES(self): 128 | indexSource = IndexSource(self.cfgPath) 129 | #creating the index only if does not already exist 130 | #the method will check by itself 131 | indexSource.createIndexSource() 132 | 133 | #first, searching if the source exists before indexing it 134 | self.forgeDocSearch() 135 | nbMatch = self.search() 136 | if nbMatch == 0: 137 | logger.info('%s querried for the first time', self.source) 138 | #no match, setting firstQuery and lastQuery to current time 139 | self.firstQuery = strftime("%Y%m%dT%H%M%S%z") 140 | self.lastQuery = strftime("%Y%m%dT%H%M%S%z") 141 | self.forgeDocIndex() 142 | self.indexInES() 143 | if nbMatch > 0: 144 | #match, updating lastQuery 145 | logger.info('Updating %s lastQuery', self.source) 146 | self.lastQuery = strftime('%Y%m%dT%H%M%S%z') 147 | self.forgeDocUpdateLastQuery() 148 | self.update() 149 | #the source exists in ES 150 | #checking if the source's score in ES and in conf file 151 | #are the same, if not, updating in ES 152 | if self.score != self.getScoreInES(): 153 | self.forgeDocUpdateScore() 154 | self.update() 155 | 156 | def isActive(self): 157 | logger.info('Source.isActive starts') 158 | #A 159 | if self.validityDate != '' and self.useByDate == '': 160 | logger.info('A scenario') 161 | return True 162 | #B 163 | elif self.validityDate == '' and self.useByDate != '': 164 | logger.info('B scenario') 165 | return False 166 | #E 167 | elif self.validityDate == '' and self.useByDate == '': 168 | logger.info('E scenario') 169 | return True 170 | 171 | #converting in datetime 172 | today = strftime('%Y%m%d') 173 | today = dateutil.parser.parse(today) 174 | useByDate = dateutil.parser.parse(self.useByDate) 175 | validityDate = dateutil.parser.parse(self.validityDate) 176 | #C 177 | if validityDate < useByDate: 178 | if today < validityDate and today < useByDate: 179 | logger.info('C scenario (C1), source is inactive') 180 | return False 181 | elif validityDate < today and today < useByDate: 182 | logger.info('C scenario (C2), source is active') 183 | return True 184 | elif today > validityDate and today > useByDate: 185 | logger.info('C scenario (C3), source is inactive') 186 | return False 187 | #D 188 | elif useByDate < validityDate: 189 | if today < useByDate and today < validityDate: 190 | logger.info('D scenario (D1), source is active') 191 | return True 192 | elif useByDate < today and today < validityDate: 193 | logger.info('D scenario (D2), source is inactive') 194 | return False 195 | elif today > useByDate and today > validityDate: 196 | logger.info('D scenario (D3), source is active') 197 | return True 198 | -------------------------------------------------------------------------------- /core/services/modules/shadowbook/objects/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheHive-Project/Hippocampe/03df065bd304b1515db6925f6cd01a754d42825e/core/services/modules/shadowbook/objects/__init__.py -------------------------------------------------------------------------------- /core/services/modules/shadowbook/parser.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | import itertools 5 | import csv 6 | import os, sys 7 | install_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../../' 8 | #path to getConf function 9 | getConfPath = install_dir + 'services/modules/common' 10 | sys.path.insert(0, getConfPath) 11 | import getConf 12 | import ast 13 | 14 | def csvParser(feedPage, cfgPath): 15 | 16 | #retrieving parsing parameters 17 | conf = getConf.getConf(cfgPath) 18 | 19 | #start parsing from startAt 20 | startAt = conf.getint('source', 'startAt') 21 | 22 | #the line does not begin with '#' nor '/' 23 | parsedPage = itertools.ifilter(lambda x: len(x) > 1 and x[0] != '#' and x[0] != '/', feedPage.splitlines()[startAt:]) 24 | 25 | #csv's delimiter, has to be str and not unicode 26 | delimiter = str(conf.get('source', 'delimiter')) 27 | #fields in the csv 28 | fields = conf.get('source', 'fields') 29 | #fields is string formated as a list, the instruction below convert it as a real list 30 | fields = [item.encode('ascii') for item in ast.literal_eval(fields)] 31 | #sometimes the first field is empty 32 | beginWithBlank = conf.getboolean('source', 'beginWithBlank') 33 | if beginWithBlank: 34 | #the feed parsed's first row is blank, to avoid discrepancy between fields and data 35 | #the first row is labelled as 'blank' 36 | fields.insert(0, 'blank') 37 | extraFields = conf.get('source', 'extraFields') 38 | 39 | #Can not use the variable delimiter as delimiter when it equals to '\t' 40 | #has to use dialect 'excel-tab' 41 | #Apparently, this is due to ConfigParser 42 | if delimiter != '\\t': 43 | parsedPage = csv.DictReader(parsedPage, fields, extraFields, delimiter = delimiter) 44 | 45 | if delimiter == '\\t': 46 | parsedPage = csv.DictReader(parsedPage, fields, extraFields, dialect = 'excel-tab') 47 | 48 | return parsedPage 49 | 50 | if __name__ == '__main__': 51 | main() 52 | -------------------------------------------------------------------------------- /core/services/modules/shadowbook/processFeed.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | import os 5 | from objects.Source import Source 6 | from downloader import simpleDownload, EmptyFeedException 7 | import searchIntel 8 | import processMsearch 9 | import bulkOp 10 | import enricher 11 | import extendTools 12 | 13 | import operator 14 | import requests 15 | from parser import csvParser 16 | from configparser import NoOptionError, NoSectionError 17 | from objects.Intel import Intel 18 | import logging 19 | logger = logging.getLogger(__name__) 20 | 21 | from pprint import pprint 22 | def main(listSessions, cfgPath): 23 | confName = os.path.basename(os.path.normpath(cfgPath)) 24 | 25 | logger.info('processFeed.main launched for %s', confName) 26 | #the function returns a report concerning the feed process 27 | #the instructions below are used to build the report 28 | report = dict() 29 | report[confName] = dict() 30 | report[confName]['error'] = list() 31 | nbUpdate = 0 32 | nbIndex = 0 33 | nbNew = 0 34 | nbFailed = 0 35 | sourceActivated = bool() 36 | 37 | try: 38 | source = Source(cfgPath) 39 | #adding the source's url to the report 40 | report[confName]['link'] = source.source 41 | 42 | #either True or False 43 | sourceActivated = source.isActive() 44 | 45 | if sourceActivated: 46 | #index the source's data into ES if it does not already exist 47 | #if it does, update the lastQuery date and make sure that the source's score in ES 48 | #equals the source's score in conf file 49 | #conf file's score is the reference and will always be pushed to ES 50 | source.indexSourceInES() 51 | 52 | try: 53 | #downloading feed with authenticated sessions 54 | #streaming mode 55 | feedPage = simpleDownload(source.source, listSessions) 56 | if len(feedPage) == 0: 57 | raise EmptyFeedException(source.source) 58 | try: 59 | #the cfgPath is given in parameters because it contains the delimiter 60 | #the function will return a csvDictReader which we convert to dict afterward 61 | parsedPage = csvParser(feedPage, cfgPath) 62 | parsedPage = list(extendTools.unique_justseen(list(parsedPage), key = operator.itemgetter(source.coreIntelligence))) 63 | #parsedPage will look like: 64 | #[{'domain': 'stie.pbsoedirman.com', 'original_reference-why_it_was_listed': 'spamhaus.org', u'extra': ['20160324'], 'nextvalidation': '', 'blank': '', 'type': 'malware'}, {'domain': 'thecommercialalliance.com', 'original_reference-why_it_was_listed': 'spamhaus.org', u'extra': ['20160324'], 'nextvalidation': '', 'blank': '', 'type': 'botnet'}] 65 | 66 | #searching all intels at once with msearch 67 | resMsearch = searchIntel.littleMsearch(source.coreIntelligence, source.typeNameESIntel, parsedPage) 68 | #sorting intels which will be updated and indexed 69 | sortedDict = processMsearch.littleSort(resMsearch, parsedPage) 70 | 71 | if len(sortedDict['toUpdate']) > 0: 72 | #updating 73 | #sortedDict['toUpdate'] is a list, each element is a doc's id 74 | nbUpdate = bulkOp.update(source.typeNameESIntel, sortedDict['toUpdate']) 75 | 76 | if len(sortedDict['toIndex']) > 0: 77 | #indexing 78 | #intels which have never been seen before are indexed under type new 79 | #in addition to the type 80 | #sortedDict['toIndex'] is a list, each element is a dict representing a feed's parsed line 81 | resMsearch = searchIntel.bigMsearch(source.coreIntelligence, sortedDict['toIndex']) 82 | toIndexInNew = processMsearch.bigSort(resMsearch, sortedDict['toIndex']) 83 | #toIndexInNew is a list, each element is a dict representing a feed's parsed line which intel has never been seen before 84 | #indexing only the coreIntelligence 85 | nbNew = bulkOp.indexNew(source.coreIntelligence, toIndexInNew) 86 | toIndex = enricher.metadata(source.idInES, source.source, sortedDict['toIndex']) 87 | res = bulkOp.index(cfgPath, toIndex) 88 | #res looks like 89 | #(1337, [{u'create': {u'status': 400, u'_type': u'type', u'_id': u'AVSFZmDLGOrJYzzqtBgw', u'error': u'MapperParsingException[failed to parse [ip]]; nested: ElasticsearchIllegalArgumentException[failed to parse ip [nxdomain], not a valid ip address]; ', u'_index': u'hippocampe'}}, {u'create': {u'status': 400, u'_type': u'type', u'_id': u'AVSFZmDLGOrJYzzqtBg0', u'error': u'MapperParsingException[failed to parse [ip]]; nested: ElasticsearchIllegalArgumentException[failed to parse ip [nxdomain], not a valid ip address]; ', u'_index': u'hippocampeinte'}}]) 90 | logger.info(type(res)) 91 | nbIndex = res[0] 92 | if len(res) > 1: 93 | nbFailed = len(res[1]) 94 | for bulkReport in res[1]: 95 | report[confName]['error'].append(bulkReport['create']['error']) 96 | 97 | 98 | 99 | except NoSectionError as e: 100 | logger.error('fail in parsing %s for the parser', confName, exc_info = True) 101 | report[confName]['error'].append(str(e)) 102 | except NoOptionError as e: 103 | logger.error('fail in parsing %s file for the parser', confName, exc_info = True) 104 | report[confName]['error'].append(str(e)) 105 | except EmptyFeedException as e: 106 | logger.error(e, exc_info = True) 107 | report[confName]['error'].append(str(e)) 108 | except requests.exceptions.RequestException as e: 109 | logger.error('downloading %s failed', source.source, exc_info = True) 110 | #filling in the report 111 | report[confName]['error'].append(str(e)) 112 | except NoSectionError as e: 113 | logger.error('fail in parsing %s for the source', confName, exc_info = True) 114 | report[confName]['error'].append(str(e)) 115 | except NoOptionError as e: 116 | logger.error('fail in parsing %s for the source', confName, exc_info = True) 117 | report[confName]['error'].append(str(e)) 118 | except Exception as e: 119 | logger.error('processFeed.main failed for %s, no idea where it came from', confName, exc_info = True) 120 | report[confName]['error'].append(str(e)) 121 | 122 | report[confName]['nbIndex'] = nbIndex 123 | report[confName]['nbUpdate'] = nbUpdate 124 | report[confName]['activated'] = sourceActivated 125 | report[confName]['nbNew'] = nbNew 126 | report[confName]['nbFailed'] = nbFailed 127 | 128 | #the report looks like 129 | #{'dnsbh_DOMAIN.conf': {'error': [], 'nbFail': 0, 'nbSucess': 13921}} 130 | logger.info('processFeed.main for %s end', confName) 131 | return report 132 | 133 | if __name__ == '__main__': 134 | main() 135 | -------------------------------------------------------------------------------- /core/services/modules/shadowbook/processFeed.py.backup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | import os 5 | from objects.Source import Source 6 | from downloader import simpleDownload 7 | import requests 8 | from parser import csvParser 9 | from configparser import NoOptionError, NoSectionError 10 | from objects.Intel import Intel 11 | import logging 12 | logger = logging.getLogger(__name__) 13 | 14 | def main(listSessions, cfgPath): 15 | confName = os.path.basename(os.path.normpath(cfgPath)) 16 | 17 | logger.info('processFeed.main launched for %s', confName) 18 | #the function returns a report concerning the feed process 19 | #the instructions below are used to build the report 20 | report = dict() 21 | report[confName] = dict() 22 | report[confName]['error'] = list() 23 | 24 | try: 25 | source = Source(cfgPath) 26 | #index the source's data into ES if it does not already exist 27 | #if it does, update the lastQuery date and make sure that the source's score in ES 28 | #equals the source's score in conf file 29 | #conf file's score is the reference and will always be pushed to ES 30 | source.indexSourceInES() 31 | 32 | try: 33 | #downloading feed with authenticated sessions 34 | #streaming mode 35 | feedPage = simpleDownload(source.source, listSessions) 36 | nbFail = 0 37 | nbSuccess = 0 38 | try: 39 | #the cfgPath is given in parameters because it contains the delimiter 40 | #the function will return a csvDictReader 41 | parsedPage = csvParser(feedPage, cfgPath) 42 | parsedPage = list(parsedPage) 43 | print parsedPage[0] 44 | #indexing information from the parsedLine in elasticsearch 45 | #if it does not exist, creates it 46 | #and creates it in the index/type: hippocampe/new 47 | #if it does, update the lastAppearance date 48 | for intelligence in parsedPage: 49 | #intelligence looks like 50 | #{'domain': 'www.galliagroup.com', 'original_reference-why_it_was_listed': 'app.webinspector.com', u'extra': ['20160304', '20160113', '20140817', '20140322', '20131006', '', '20140624'], 'nextvalidation': '20160401', 'blank': '', 'type': 'highrisk'} 51 | try: 52 | intel = Intel(intelligence, cfgPath, source.idInES) 53 | intel.indexIntelInES() 54 | nbSuccess += 1 55 | except Exception as e: 56 | logger.error('Indexing %s failed', intelligence, exc_info = True) 57 | report[confName]['error'].append(str(e)) 58 | nbFail += 1 59 | except NoSectionError as e: 60 | logger.error('fail in parsing %s for the parser', confName, exc_info = True) 61 | report[confName]['error'].append(str(e)) 62 | except NoOptionError as e: 63 | logger.error('fail in parsing %s file for the parser', confName, exc_info = True) 64 | report[confName]['error'].append(str(e)) 65 | except requests.exceptions.RequestException as e: 66 | logger.error('downloading %s failed', source.source, exc_info = True) 67 | #filling the report 68 | report[confName]['error'].append(str(e)) 69 | except NoSectionError as e: 70 | logger.error('fail in parsing %s for the source', confName, exc_info = True) 71 | report[confName]['error'].append(str(e)) 72 | except NoOptionError as e: 73 | logger.error('fail in parsing %s for the source', confName, exc_info = True) 74 | report[confName]['error'].append(str(e)) 75 | except Exception as e: 76 | logger.error('processFeed.main failed for %s, no idea where it came from', confName, exc_info = True) 77 | 78 | report[confName]['nbSucess'] = nbSuccess 79 | report[confName]['nbFail'] = nbFail 80 | 81 | #the report looks like 82 | #{'dnsbh_DOMAIN.conf': {'error': [], 'nbFail': 0, 'nbSucess': 13921}} 83 | logger.info('processFeed.main for %s end', confName) 84 | return report 85 | 86 | if __name__ == '__main__': 87 | main() 88 | -------------------------------------------------------------------------------- /core/services/modules/shadowbook/processMsearch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | import logging 6 | logger = logging.getLogger(__name__) 7 | 8 | def littleSort(resMsearch, data): 9 | logger.info('processMsearch.littleSort launched') 10 | #data is a list of dict 11 | #each dict is a parsed line from the feed 12 | # resMsearch will look like 13 | #{u'responses': [{u'_shards': {u'failed': 0, u'successful': 5, u'total': 5}, 14 | # u'hits': {u'hits': [{u'_id': u'AVOuC41q6EIAXcyxAFz0', 15 | # u'_index': u'hippocampe', 16 | # u'_score': 7.470799, 17 | # u'_source': {u'firstAppearance': u'20160325T145146+0100', 18 | # u'idSource': u'AVOuCsBt6EIAXcyxAEn3', 19 | # u'lastAppearance': u'20160325T145146+0100', 20 | # u'source': u'https://openphish.com/feed.txt', 21 | # u'url': u'https://www.myfridaygoodies.ch/sandbox/1/'}, 22 | # u'_type': u'openphishFree_feedURL'}], 23 | # u'max_score': 7.470799, 24 | # u'total': 1}, 25 | # u'timed_out': False, 26 | # u'took': 124}, 27 | 28 | # {u'_shards': {u'failed': 0, u'successful': 5, u'total': 5}, 29 | # u'hits': {u'hits': [], u'max_score': None, u'total': 0}, 30 | # u'timed_out': False, 31 | # u'took': 107}, 32 | 33 | # {u'_shards': {u'failed': 0, u'successful': 5, u'total': 5}, 34 | # u'hits': {u'hits': [{u'_id': u'AVOuCxyD6EIAXcyxAFA0', 35 | # u'_index': u'hippocampe', 36 | # u'_score': 7.4480977, 37 | # u'_source': {u'firstAppearance': u'20160325T145117+0100', 38 | # u'idSource': u'AVOuCsBt6EIAXcyxAEn3', 39 | # u'lastAppearance': u'20160325T145117+0100', 40 | # u'source': u'https://openphish.com/feed.txt', 41 | # u'url': u'http://www.rutzcellars.com/dd-dd/art/'}, 42 | # u'_type': u'openphishFree_feedURL'}], 43 | # u'max_score': 7.4480977, 44 | # u'total': 1}, 45 | # u'timed_out': False, 46 | # u'took': 117}]} 47 | i = 0 48 | toIndex = list() 49 | toUpdate = list() 50 | while i < len(data): 51 | if len(resMsearch['responses'][i]['hits']['hits']) == 0: 52 | #we are in this case: 53 | #u'hits': {u'hits': [] 54 | #it means that the intel at line i of the feed is unknown in ES 55 | #and we have to index it 56 | toIndex.append(data[i]) 57 | if len(resMsearch['responses'][i]['hits']['hits']) > 0: 58 | #we are in this case: 59 | # u'hits': {u'hits': [{u'_id': u'AVOuCxyD6EIAXcyxAFA0', 60 | # u'_index': u'hippocampe', 61 | # u'_score': 7.4480977, 62 | # u'_source': {u'firstAppearance': u'20160325T145117+0100', 63 | # u'idSource': u'AVOuCsBt6EIAXcyxAEn3', 64 | # u'lastAppearance': u'20160325T145117+0100', 65 | # u'source': u'https://openphish.com/feed.txt', 66 | # u'url': u'http://www.rutzcellars.com/dd-dd/art/'}, 67 | # u'_type': u'openphishFree_feedURL'}], 68 | 69 | #retrieving ids to update the doc in ES 70 | for match in resMsearch['responses'][i]['hits']['hits']: 71 | toUpdate.append(match['_id']) 72 | i += 1 73 | 74 | res = dict() 75 | res['toIndex'] = toIndex 76 | res['toUpdate'] = toUpdate 77 | logger.info('processMsearch.littleSort end') 78 | return res 79 | 80 | def bigSort(resMsearch, data): 81 | logger.info('processMsearch.bigSort launched') 82 | i = 0 83 | toIndexInNew = list() 84 | while i < len(data): 85 | if len(resMsearch['responses'][i]['hits']['hits']) == 0: 86 | toIndexInNew.append(data[i]) 87 | i += 1 88 | logger.info('processMsearch.bigSort end') 89 | return toIndexInNew 90 | if __name__ == '__main__': 91 | main() 92 | -------------------------------------------------------------------------------- /core/services/modules/shadowbook/searchIntel.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | import os, sys 5 | app_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../../' 6 | getESpath = app_dir + 'services/common' 7 | sys.path.insert(0, getESpath) 8 | from ES import getES 9 | from getConf import getHippoConf 10 | 11 | import logging 12 | logger = logging.getLogger(__name__) 13 | 14 | def littleMsearch(coreIntelligence, typeNameES, listParsedData): 15 | logger.info('searchIntel.littleMsearch launched') 16 | cfg = getHippoConf() 17 | indexNameES = cfg.get('elasticsearch', 'indexNameES') 18 | es = getES() 19 | 20 | #forging littleMsearch request 21 | #request to be sent to ES for littleMsearch 22 | req = list() 23 | #request header 24 | req_head = {'index': indexNameES, 'type': typeNameES} 25 | 26 | coreIntelligence = coreIntelligence 27 | #in the previous example, coreIntelligence is 'domain' 28 | for element in listParsedData: 29 | req_body = { 30 | 'query' : { 31 | 'bool' : { 32 | 'must' : [ 33 | { 34 | 'match' : { 35 | coreIntelligence : element[coreIntelligence] 36 | } 37 | } 38 | ] 39 | } 40 | } 41 | } 42 | req.extend([req_head, req_body]) 43 | #req will look like 44 | #[{ 45 | # 'index': 'hippocampe', 46 | # 'type': u 'malwaredomainsFree_dnsbhDOMAIN' 47 | #}, { 48 | # 'query': { 49 | # 'bool': { 50 | # 'must': [{ 51 | # 'match': { 52 | # u 'domain': 'skandastech.com' 53 | # } 54 | # }] 55 | # } 56 | # } 57 | #}, { 58 | # 'index': 'hippocampe', 59 | # 'type': u 'malwaredomainsFree_dnsbhDOMAIN' 60 | #}, { 61 | # 'query': { 62 | # 'bool': { 63 | # 'must': [{ 64 | # 'match': { 65 | # u 'domain': 'stie.pbsoedirman.com' 66 | # } 67 | # }] 68 | # } 69 | # } 70 | #}] 71 | 72 | res = es.msearch(body = req) 73 | # res will look like 74 | #{u'responses': [{u'_shards': {u'failed': 0, u'successful': 5, u'total': 5}, 75 | # u'hits': {u'hits': [{u'_id': u'AVOuC41q6EIAXcyxAFz0', 76 | # u'_index': u'hippocampe', 77 | # u'_score': 7.470799, 78 | # u'_source': {u'firstAppearance': u'20160325T145146+0100', 79 | # u'idSource': u'AVOuCsBt6EIAXcyxAEn3', 80 | # u'lastAppearance': u'20160325T145146+0100', 81 | # u'source': u'https://openphish.com/feed.txt', 82 | # u'url': u'https://www.myfridaygoodies.ch/sandbox/1/'}, 83 | # u'_type': u'openphishFree_feedURL'}], 84 | # u'max_score': 7.470799, 85 | # u'total': 1}, 86 | # u'timed_out': False, 87 | # u'took': 124}, 88 | 89 | # {u'_shards': {u'failed': 0, u'successful': 5, u'total': 5}, 90 | # u'hits': {u'hits': [], u'max_score': None, u'total': 0}, 91 | # u'timed_out': False, 92 | # u'took': 107}, 93 | 94 | # {u'_shards': {u'failed': 0, u'successful': 5, u'total': 5}, 95 | # u'hits': {u'hits': [{u'_id': u'AVOuCxyD6EIAXcyxAFA0', 96 | # u'_index': u'hippocampe', 97 | # u'_score': 7.4480977, 98 | # u'_source': {u'firstAppearance': u'20160325T145117+0100', 99 | # u'idSource': u'AVOuCsBt6EIAXcyxAEn3', 100 | # u'lastAppearance': u'20160325T145117+0100', 101 | # u'source': u'https://openphish.com/feed.txt', 102 | # u'url': u'http://www.rutzcellars.com/dd-dd/art/'}, 103 | # u'_type': u'openphishFree_feedURL'}], 104 | # u'max_score': 7.4480977, 105 | # u'total': 1}, 106 | # u'timed_out': False, 107 | # u'took': 117}]} 108 | logger.info('searchIntel.littleMsearch end') 109 | return res 110 | 111 | def bigMsearch(coreIntelligence, listParsedData): 112 | logger.info('searchIntel.bigMsearch launched') 113 | es = getES() 114 | 115 | cfg = getHippoConf() 116 | indexNameES = cfg.get('elasticsearch', 'indexNameES') 117 | 118 | req = list() 119 | req_head = {'index': indexNameES} 120 | 121 | coreIntelligence = coreIntelligence 122 | for element in listParsedData: 123 | req_body = { 124 | 'query' : { 125 | 'bool' : { 126 | 'must' : [ 127 | { 128 | 'match' : { 129 | coreIntelligence : element[coreIntelligence] 130 | } 131 | } 132 | ] 133 | } 134 | } 135 | } 136 | req.extend([req_head, req_body]) 137 | 138 | res = es.msearch(body = req) 139 | logger.info('searchIntel.bigMsearch end') 140 | return res 141 | if __name__ == '__main__': 142 | main() 143 | -------------------------------------------------------------------------------- /core/services/modules/sizeBySources/TypeES.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | """ 6 | This module contains ObjToIndex class. 7 | ====================================== 8 | """ 9 | 10 | import os, sys 11 | install_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../../' 12 | getESpath = install_dir + 'services/modules/common' 13 | sys.path.insert(0, getESpath) 14 | from ES import getES 15 | from getConf import getHippoConf 16 | 17 | class TypeES(object): 18 | """ 19 | ObjToIndex class represents any object that can be indexed in elasticsearch. 20 | 21 | This class is kind of an abstract class, because is is the common base 22 | of Source and Ioc classes. 23 | """ 24 | 25 | def __init__(self, typeNameES): 26 | """ 27 | ObjToIndex class' constructor. 28 | """ 29 | cfg = getHippoConf() 30 | self.es = getES() 31 | self.indexNameES = cfg.get('elasticsearch', 'indexNameES') 32 | self.typeNameES = typeNameES 33 | self.docSearch = dict() 34 | self.size = int() 35 | 36 | def forgeDocSearch(self): 37 | self.docSearch = { 38 | 'query' : { 39 | 'match_all' : {} 40 | } 41 | } 42 | 43 | def getSize(self): 44 | self.forgeDocSearch() 45 | res = self.es.count(index=self.indexNameES, doc_type=self.typeNameES, body = self.docSearch) 46 | #res looks like: 47 | #{u'_shards': {u'failed': 0, u'successful': 5, u'total': 5}, u'count': 45} 48 | self.size = res['count'] 49 | return self.size 50 | 51 | -------------------------------------------------------------------------------- /core/services/modules/sizeBySources/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheHive-Project/Hippocampe/03df065bd304b1515db6925f6cd01a754d42825e/core/services/modules/sizeBySources/__init__.py -------------------------------------------------------------------------------- /core/services/modules/sources/BagOfSources.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | import os 6 | import sys 7 | app_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../../' 8 | pathModule = app_dir + 'services/modules/common' 9 | sys.path.insert(0, pathModule) 10 | from ES import getES 11 | from getConf import getHippoConf 12 | 13 | class BagOfSources: 14 | """ 15 | class that will retrieve all sources 16 | it seems that a match all query and a scan scroll does not work 17 | together, that's why a match all query is used 18 | however only 10 matches are returned by elasticsearch. 19 | To go beyond that number, the number of document in source index 20 | is determined and indicated in the match all query 21 | """ 22 | def __init__(self): 23 | cfg = getHippoConf() 24 | self.docSearch = str() 25 | self.matchResponse = str() 26 | self.matchDict = dict() 27 | self.es = getES() 28 | self.indexNameES = cfg.get('elasticsearch', 'indexNameES') 29 | self.typeNameES = 'source' 30 | self.nbDoc = int() 31 | 32 | 33 | def forgeDocSearch(self): 34 | self.docSearch = { 35 | 'query' : { 36 | 'match_all' : {} 37 | } 38 | } 39 | 40 | def forgeDocSearchAll(self): 41 | self.docSearch = { 42 | 'size': self.nbDoc, 43 | 'query' : { 44 | 'match_all' : {} 45 | } 46 | } 47 | 48 | def search(self): 49 | self.matchResponse = self.es.search( 50 | body = self.docSearch, index= self.indexNameES, 51 | doc_type = self.typeNameES) 52 | if self.matchResponse['hits']['total'] > 0: 53 | return True 54 | elif self.matchResponse['hits']['total'] == 0: 55 | return False 56 | 57 | def processMatchResponse(self): 58 | #the match all query returns all documents, however there is 59 | #no key, only value 60 | #so the field source is setted as key and the all document 61 | #as value 62 | for element in self.matchResponse['hits']['hits']: 63 | self.matchDict[element['_source']['link']] = element['_source'] 64 | 65 | def getMatchList(self): 66 | #because elasticsearch only returns 10 results 67 | #w have to search in two phase, first determine the nb of docs 68 | #and then retrieve all docs 69 | #matchAll query 70 | self.forgeDocSearch() 71 | #retrieve nb doc 72 | res = self.es.count(index=self.indexNameES, doc_type=self.typeNameES, body = self.docSearch) 73 | #res looks like: 74 | #{u'_shards': {u'failed': 0, u'successful': 5, u'total': 5}, u'count': 45} 75 | self.nbDoc = res['count'] 76 | self.forgeDocSearchAll() 77 | self.search() 78 | self.processMatchResponse() 79 | return self.matchDict 80 | -------------------------------------------------------------------------------- /core/services/modules/sources/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheHive-Project/Hippocampe/03df065bd304b1515db6925f6cd01a754d42825e/core/services/modules/sources/__init__.py -------------------------------------------------------------------------------- /core/services/monitorSources.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | import lastQuery, schedReport, sizeBySources, freshness, lastStatus 6 | import logging 7 | logger = logging.getLogger(__name__) 8 | from copy import deepcopy 9 | 10 | def main(): 11 | logger.info('monitorSources.main launched') 12 | try: 13 | lastQueryReport = lastQuery.main() 14 | sched_Report = schedReport.main() 15 | sizeBySourceReport = sizeBySources.main() 16 | freshnessReport = freshness.main() 17 | lastStatusReport = lastStatus.main() 18 | 19 | report = dict_merge(lastQueryReport, sched_Report) 20 | report = dict_merge(report, sizeBySourceReport) 21 | report = dict_merge(report, freshnessReport) 22 | report = dict_merge(report, lastStatusReport) 23 | 24 | logger.info('monitorSources.main end') 25 | return report 26 | except Exception as e: 27 | logger.error('monitorSources.main failed, no idea where it came from...', exc_info = True) 28 | response = dict() 29 | response['error'] = str(e) 30 | return response 31 | 32 | def dict_merge(a, b): 33 | '''recursively merges dict's. not just simple a['key'] = b['key'], if 34 | both a and bhave a key who's value is a dict then dict_merge is called 35 | on both values and the result stored in the returned dictionary.''' 36 | if not isinstance(b, dict): 37 | return b 38 | result = deepcopy(a) 39 | for k, v in b.iteritems(): 40 | if k in result and isinstance(result[k], dict): 41 | result[k] = dict_merge(result[k], v) 42 | else: 43 | result[k] = deepcopy(v) 44 | return result 45 | 46 | if __name__ == '__main__': 47 | main() 48 | -------------------------------------------------------------------------------- /core/services/more.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | from modules.more.ObjToEnrich import ObjToEnrich 6 | from multiprocessing.dummy import Pool as ThreadPool 7 | from multiprocessing import cpu_count, Queue 8 | from hipposcore import calcHipposcore 9 | nbProcessPerCPU = 1 10 | import logging 11 | logger = logging.getLogger(__name__) 12 | 13 | def tellMeMore(chunkedList): 14 | chunkedDict = dict(chunkedList) 15 | result = dict() 16 | for ioc, attributes in chunkedDict.items(): 17 | objIoc = ObjToEnrich(attributes['type'], ioc) 18 | try: 19 | result[ioc] = objIoc.getDetails() 20 | except Exception as e: 21 | logger.error('tellMeMore failed, no idea where it came from', exc_info = True) 22 | result[ioc] = dict() 23 | result[ioc]['error'] = str(e) 24 | return result 25 | 26 | def main(jsonRequest): 27 | logger.info('more.main launched') 28 | logger.info(jsonRequest) 29 | #dictionnary that contains the response 30 | response = dict() 31 | 32 | #the client-request (JSON dict) has to be converted into a list 33 | #Indeed, in order to "multi-threaded" the process 34 | #pool.map will iterate on the argument 35 | #however, iterate on a dict gives only the key (which is unicode type) 36 | #whereas dict type is needed to tellMeMore 37 | #a list of dict is therefore the solution 38 | items = list(jsonRequest.items()) 39 | 40 | #the client-request list (items variable) will be splited into n small 41 | #requests which included chunksize elements 42 | chunksize = 10 43 | 44 | #cutting the items list into several small ones with chunksize length 45 | chunks = [items[i:i + chunksize] for i in range(0, len(items), chunksize)] 46 | 47 | pool_size = cpu_count() * nbProcessPerCPU 48 | pool = ThreadPool(processes=pool_size) 49 | queue = Queue() 50 | 51 | #updating directly response dict causes errors 52 | #that's why result from tellMeMore function is queued 53 | #logger.info('beginning tellMeMore with several threads') 54 | queue.put(pool.map(tellMeMore, chunks)) 55 | 56 | #gathering all result from the queue into one dict 57 | for element in queue.get(): 58 | response.update(element) 59 | 60 | pool.close() 61 | pool.join() 62 | #logger.info('tellMeMore.main pool closed and joined') 63 | 64 | #yet, the response does not include hipposcore 65 | #let's add the hipposcore to the response 66 | 67 | hipposcoreDict = calcHipposcore(response) 68 | 69 | #two dicts, response with all data 70 | #{u'103.16.26.228': [{u'date': u'20151001T112643+0200', 71 | # u'firstAppearance': u'20151001T112643+0200', 72 | # u'idSource': u'AVAiuZ7ADAiil8mrVC0p', 73 | # u'ip': u'103.16.26.228', 74 | # u'lastAppearance': u'20151001T112643+0200', 75 | # u'source': u'abuseFree_feodotrackerIP'}], 76 | # u'103.16.26.36': [{u'date': u'20151001T112643+0200', 77 | # u'firstAppearance': u'20151001T112643+0200', 78 | # u'idSource': u'AVAiuZ7ADAiil8mrVC0p', 79 | # u'ip': u'103.16.26.36', 80 | # u'lastAppearance': u'20151001T112643+0200', 81 | # u'source': u'abuseFree_feodotrackerIP'}], 82 | # u'199.9.24.1': [], 83 | # u'datingzzzj.ru': [], 84 | # u'trkl.su': []} 85 | 86 | #hipposcoreDict with only hipposcore 87 | # {u'103.16.26.228': {'hipposcore': -86.46647167633873}, 88 | # u'103.16.26.36': {'hipposcore': -86.46647167633873}, 89 | # u'199.9.24.1': 0, 90 | # u'datingzzzj.ru': 0, 91 | # u'trkl.su': 0} 92 | 93 | for keyRes, listMatch in response.items(): 94 | for keyScore, scoreDict in hipposcoreDict.items(): 95 | if keyRes == keyScore: 96 | for match in listMatch: 97 | match['hipposcore'] = scoreDict 98 | logger.info('more.main end') 99 | return response 100 | 101 | if __name__ == '__main__': 102 | main() 103 | -------------------------------------------------------------------------------- /core/services/new.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | from modules.new.BagOfNew import BagOfNew 6 | import logging 7 | logger = logging.getLogger(__name__) 8 | def main(): 9 | logger.info('new.main launched') 10 | try: 11 | bag = BagOfNew() 12 | result = bag.getMatchDict() 13 | logger.info('new.main end') 14 | return result 15 | except Exception as e: 16 | logger.error('new.main failed, no idea where it came from', exc_info = True) 17 | report = dict() 18 | report['error'] = str(e) 19 | -------------------------------------------------------------------------------- /core/services/schedReport.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | import sources 6 | import dateutil.parser 7 | import time 8 | from modules.common.getConf import getHippoConf 9 | import logging 10 | from configparser import NoOptionError, NoSectionError 11 | logger = logging.getLogger(__name__) 12 | 13 | def main(): 14 | logger.info('schedReport.main launched') 15 | 16 | cfg = getHippoConf() 17 | #threshold defined in conf file 18 | #by default equals to 7 19 | #if a source is not querryed for 7 days or more, an alert is sent 20 | 21 | #JSON dict reporting data freshness for all sources 22 | report = dict() 23 | 24 | try: 25 | threshold = cfg.getint('schedReport', 'threshold') 26 | 27 | #retrieve all source 28 | dictSource = sources.main() 29 | #dictSource looks like: 30 | #{ 31 | # "https://feodotracker.abuse.ch/blocklist/?download=ipblocklist": { 32 | # "coreIntelligence": "ip", 33 | # "description": "The Feodo Tracker Feodo IP Blocklist contains IP addresses (IPv4) used as C&C communication channel by the Feodo Trojan. This lists contains two types of IP address: Feodo C&C servers used by version A, version C and version D of the Feodo Trojan (these IP addresses are usually compromised servers running an nginx daemon on port 8080 TCP or 7779 TCP that is acting as proxy, forwarding all traffic to a tier 2 proxy node) and Feodo C&C servers used by version B which are usually used for the exclusive purpose of hosting a Feodo C&C server.", 34 | # "firstQuery": "20160222T233556+0100", 35 | # "lastQuery": "20160224T134833+0100", 36 | # "score": -100, 37 | # "source": "https://feodotracker.abuse.ch/blocklist/?download=ipblocklist", 38 | # "type": "source" 39 | # } 40 | #} 41 | if 'error' in dictSource: 42 | logger.error('sources.main for schedReport.main failed') 43 | return dictSource 44 | 45 | now = time.strftime("%Y%m%dT%H%M%S%z") 46 | 47 | #converting to dateutil 48 | now = dateutil.parser.parse(now) 49 | for url, dictData in dictSource.items(): 50 | report[url] =dict() 51 | lastQuery =dictData['lastQuery'] 52 | lastQuery = dateutil.parser.parse(lastQuery) 53 | 54 | #delta is the time difference in days between now and lastQuery 55 | delta = (now - lastQuery).days 56 | #converting in hours 57 | delta = delta * 24 58 | if (delta > threshold): 59 | report[url]['schedReport'] = 'NOK' 60 | else: 61 | report[url]['schedReport'] = 'OK' 62 | logger.info('schedReport.main end') 63 | return report 64 | except NoSectionError as e: 65 | logger.error('parsing hippo.conf failed', exc_info = True) 66 | report['error'] = str(e) 67 | except NoOptionError as e: 68 | logger.error('parsing hippo.conf failed', exc_info = True) 69 | report['error'] = str(e) 70 | except Exception as e: 71 | logger.error('schedReport.main failed, no idea where it came from...', exc_info = True) 72 | report['error'] = str(e) 73 | 74 | if __name__ == '__main__': 75 | main() 76 | 77 | -------------------------------------------------------------------------------- /core/services/shadowbook.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | """ 6 | This module contains the shadowBook process. 7 | """ 8 | from modules.shadowbook import createSessions 9 | from modules.common import getConf 10 | from multiprocessing import cpu_count 11 | from multiprocessing.dummy import Pool as ThreadPool 12 | from multiprocessing import Queue 13 | import os 14 | from os import listdir 15 | from os.path import join 16 | from functools import partial 17 | from modules.shadowbook import processFeed 18 | from traceback import print_exc 19 | import logging 20 | logger = logging.getLogger(__name__) 21 | 22 | from modules.shadowbook.objects.IndexJob import IndexJob 23 | from modules.shadowbook.objects.Job import Job 24 | from time import sleep 25 | def startJob(): 26 | logger.info('shadowbook.startJob launched') 27 | try: 28 | #creating sessions for feeds which need authentification 29 | listSessions = createSessions.createSessions() 30 | 31 | #retrieving the config from hippo.conf file 32 | cfg = getConf.getHippoConf() 33 | nbThreadPerCPU = cfg.getint('shadowbook', 'nbThreadPerCPU') 34 | 35 | #processFeed needs 2 parameters: 36 | # * the conf path 37 | # * the listSessions 38 | #processIoc is launched in a multithreaded way 39 | #However pool.map does not handle multiple arguments 40 | #we use functools.partial to bypass this limitation 41 | pool_size = cpu_count() * nbThreadPerCPU 42 | pool = ThreadPool(processes=pool_size) 43 | queue = Queue() 44 | 45 | #json which reports the result from all feeds 46 | reportAllFeeds = dict() 47 | #arguments must be in a list 48 | argInputs = list() 49 | feedsPath = os.path.dirname(os.path.abspath(__file__)) + '/../conf/feeds' 50 | 51 | #building the list which each element is a feedConfPath 52 | for cfgFile in listdir(feedsPath): 53 | cfgPath = join(feedsPath, cfgFile) 54 | argInputs.append(cfgPath) 55 | 56 | for link in argInputs: 57 | reportAllFeeds.update(processFeed.main(listSessions, link)) 58 | 59 | #func = partial(processFeed.main, listSessions) 60 | #queue.put(pool.map(func, argInputs)) 61 | #pool.close() 62 | #pool.join() 63 | #for reportOneFeed in queue.get(): 64 | # reportAllFeeds.update(reportOneFeed) 65 | logger.info('shadowbook.startJob end') 66 | return reportAllFeeds 67 | except Exception as e: 68 | logger.info('shadowbook.startJob failed, no idea where it came from...', exc_info = True) 69 | report = dict() 70 | report['error shadowbook'] = str(e) 71 | return report 72 | 73 | def initJob(): 74 | logger.info('shadowbook.initJob launched') 75 | try: 76 | indexJob = IndexJob() 77 | #create index/type hippocampe/shadowbookJobs only if it does not exist 78 | indexJob.createIndexJob() 79 | job = Job() 80 | nbOngoingJob = job.searchOngoingJob() 81 | logger.info('number ongoing job: %s', nbOngoingJob) 82 | 83 | report = dict() 84 | if nbOngoingJob == 1: 85 | #only 1 shadowbook job at a time 86 | report['error'] = 'Ongoing job already running' 87 | logger.error(report['error']) 88 | elif nbOngoingJob == 0: 89 | #create a document in ES which represents a JOB 90 | #index with a ongoing status & startTime as 91 | idJob = job.indexOngoingJob() 92 | report = dict() 93 | report['job'] = dict() 94 | report['job'][idJob] = 'ongoing' 95 | logger.info(report) 96 | logger.info('shadowbook.initJob end') 97 | return report 98 | except Exception as e: 99 | logger.error(str(e), exc_info = True) 100 | 101 | def updateJob(report, status): 102 | logger.info('shadowbook.updateJob launched') 103 | logger.info('updating with status: %s', status) 104 | #update job status, add the report and calculates the duration 105 | job = Job() 106 | job.updateStatus(report, status) 107 | logger.info('shadowbook.updateJob end') 108 | 109 | def manageJob(): 110 | logger.info('shadowbook.manageJob launched') 111 | #index feeds and update the job's status 112 | report = startJob() 113 | if 'error shadowbook' in report: 114 | status = 'failed' 115 | else: 116 | status = 'done' 117 | updateJob(report, status) 118 | logger.info('shadowbook.manageJob end') 119 | 120 | def hipposchedVersion(): 121 | logger.info('shadowbook.hipposchedVersion launched') 122 | #special version for the scheduler 123 | jobDone = False 124 | while jobDone == False: 125 | report = initJob() 126 | if not 'error' in report: 127 | manageJob() 128 | jobDone = True 129 | else: 130 | #one shadowbook job is already running 131 | #wait for 1 minute and try to launch another one if possible 132 | sleep(60) 133 | logger.info('shadowbook.hipposchedVersion end') 134 | 135 | if __name__ == '__main__': 136 | main() 137 | -------------------------------------------------------------------------------- /core/services/sizeBySources.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | from modules.sizeBySources.TypeES import TypeES 6 | import sources 7 | import logging 8 | logger = logging.getLogger(__name__) 9 | def main(): 10 | logger.info('sizeBySources.main launched') 11 | try: 12 | sourcesReport = sources.main() 13 | report = dict() 14 | for url, dictAttributes in sourcesReport.items(): 15 | typeES = TypeES(dictAttributes['type']) 16 | report[url] = dict() 17 | report[url]['size'] = typeES.getSize() 18 | logger.info('sizeBySources.main end') 19 | return report 20 | except Exception as e: 21 | logger.error('distinct.main failed, no idea where it came from...', exc_info = True) 22 | response = dict() 23 | response['error'] = str(e) 24 | return response 25 | 26 | if __name__ == '__main__': 27 | main() 28 | -------------------------------------------------------------------------------- /core/services/sizeByType.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | from modules.countType.BagOfIntel import BagOfIntel 6 | import typeIntel 7 | import logging 8 | logger = logging.getLogger(__name__) 9 | def main(): 10 | logger.info('sizeByType.main launched') 11 | try: 12 | report = dict() 13 | #retrieving all intelligences's types available 14 | typeReport = typeIntel.main() 15 | for typeIntelligence in typeReport['type']: 16 | bagOfIntel = BagOfIntel(typeIntelligence) 17 | size = bagOfIntel.getSize() 18 | report[typeIntelligence] = dict() 19 | report[typeIntelligence]['size'] = size 20 | logger.info('sizeByType.main end') 21 | return report 22 | except Exception as e: 23 | logger.error('sizeByType.main failed, no idea where it came from...', exc_info = True) 24 | response = dict() 25 | response['error'] = str(e) 26 | return response 27 | 28 | if __name__ == '__main__': 29 | main() 30 | -------------------------------------------------------------------------------- /core/services/sources.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | from modules.sources.BagOfSources import BagOfSources 6 | import logging 7 | logger = logging.getLogger(__name__) 8 | 9 | def main(): 10 | logger.info('sources.main launched') 11 | try: 12 | bag = BagOfSources() 13 | result = bag.getMatchList() 14 | logger.info('sources.main end') 15 | return result 16 | except Exception as e: 17 | logger.error('sources.main failed, no idea where it came from...', exc_info = True) 18 | report = dict() 19 | report['error'] = str(e) 20 | return report 21 | -------------------------------------------------------------------------------- /core/services/typeIntel.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf8 -*- 3 | 4 | 5 | from modules.distinct.Field import Field 6 | import logging 7 | logger = logging.getLogger(__name__) 8 | def main(): 9 | logger.info('typeIntel.main launched') 10 | try: 11 | response = dict() 12 | #type service is actually a distinct with the field "coreIntelligence" 13 | #that's why Field object is used 14 | allType = Field('coreIntelligence') 15 | allType.getDistinct() 16 | response['type'] = allType.distinctList 17 | logger.info('typeIntel.main end') 18 | return response 19 | except Exception as e: 20 | logger.error('typeIntel.main failed, no idea where it came from...', exc_info = True) 21 | report = dict() 22 | report['error'] = str(e) 23 | return report 24 | if __name__ == '__main__': 25 | main() 26 | -------------------------------------------------------------------------------- /core/static/.gitignore: -------------------------------------------------------------------------------- 1 | bower_components 2 | -------------------------------------------------------------------------------- /core/static/app/app.js: -------------------------------------------------------------------------------- 1 | (function() { 2 | // Create an application 3 | var app = angular.module('hippocampe', 4 | ['ui.router', 5 | 'chart.js', 6 | 'ui.bootstrap', 7 | 'main' 8 | ]); 9 | 10 | app.config(function($stateProvider, $urlRouterProvider) { 11 | $urlRouterProvider.otherwise('/home'); 12 | 13 | // $stateProvider 14 | // .state('sources', { 15 | // url: "/sources", 16 | // templateUrl: "app/sources/sources.html", 17 | // controller: "app/sources/module.js" 18 | // }) 19 | }); 20 | 21 | app.filter('toArray', function() { 22 | return function(input) { 23 | var out = []; 24 | // check if input is array 25 | for (key in input) { 26 | var value = input[key]; 27 | value["id"] = key; 28 | out.push(value); 29 | } 30 | return out; 31 | }; 32 | }); 33 | 34 | })(); 35 | -------------------------------------------------------------------------------- /core/static/app/hipposcore/hipposcore.html: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /core/static/app/hipposcore/module.js: -------------------------------------------------------------------------------- 1 | (function() { 2 | 'use strict'; 3 | 4 | var hipposcore = angular.module('hipposcore', ['ui.router', 'ui-notification']) 5 | .config(function(NotificationProvider) { 6 | NotificationProvider.setOptions({ 7 | delay: 10000, 8 | startTop: 55, 9 | startRight: 10, 10 | verticalSpacing: 20, 11 | horizontalSpacing: 20, 12 | positionX: 'right', 13 | positionY: 'top' 14 | }); 15 | }); 16 | 17 | hipposcore.controller('HipposcoreCtrl', function($scope, TypeSrv, HipposcoreSrv, Notification) { 18 | $scope.textValue; 19 | $scope.observable; 20 | $scope.listTypes = []; 21 | $scope.selectedType; 22 | $scope.score; 23 | 24 | 25 | $scope.closeAlert = function(index) { 26 | $scope.alerts.splice(index, 1); 27 | }; 28 | 29 | $scope.hipposcore = function() { 30 | if ($scope.textValue) { 31 | $scope.observable = this.textValue; 32 | $scope.textValue = ''; 33 | } 34 | 35 | HipposcoreSrv.list($scope.selectedType, $scope.observable).then( 36 | function(response) { 37 | $scope.score = response.data[$scope.observable]['hipposcore']; 38 | 39 | if ($scope.score === 0) { 40 | //var msg = $scope.selectedType.concat(': ').concat($scope.observable).concat(' Unknown'); 41 | var msg = $scope.observable.concat(' Unknown'); 42 | Notification.warning({message: msg}); 43 | } 44 | else if ($scope.score > 0) { 45 | var msg = $scope.observable.concat(' : ').concat($scope.score); 46 | Notification.info({message: msg}); 47 | } 48 | else if ($scope.score < 0) { 49 | var msg = $scope.observable.concat(' : ').concat($scope.score); 50 | Notification.error({message: msg}); 51 | } 52 | }, 53 | function(rejection) { 54 | console.log(rejection.data); 55 | } 56 | ); 57 | }; 58 | 59 | //dropdown toggle stuff 60 | function init() { 61 | //retrieve type for dropdown toggle 62 | TypeSrv.list().then( 63 | function(response) { 64 | $scope.listTypes = response.data['type']; 65 | }, 66 | function(rejection) { 67 | console.log(rejection.data); 68 | } 69 | ); 70 | }; 71 | 72 | // change dropdown toggle label according to selected type 73 | $scope.selectTypeHipposcore = function(selectedType) { 74 | $scope.selectedType = selectedType; 75 | }; 76 | 77 | init(); 78 | }); 79 | 80 | hipposcore.service('TypeSrv', function($http) { 81 | return { 82 | list: function() { 83 | return $http.get('hippocampe/api/v1.0/type'); 84 | } 85 | }; 86 | }); 87 | 88 | hipposcore.service('HipposcoreSrv', function($http) { 89 | return { 90 | list: function(selectedType, observable) { 91 | var requestBody = {}; 92 | var subBody = {}; 93 | subBody['type'] = selectedType; 94 | 95 | requestBody[observable] = subBody; 96 | 97 | return $http.post('hippocampe/api/v1.0/hipposcore', requestBody); 98 | } 99 | }; 100 | }); 101 | 102 | })(); -------------------------------------------------------------------------------- /core/static/app/home/home.html: -------------------------------------------------------------------------------- 1 |
2 |
3 | Hippocampe 4 |
5 |
6 | 7 |
8 |
9 |
10 |
11 | Size By Type 12 |
13 | 14 |
15 |
16 |
17 |
18 |
19 | 20 |
21 |
22 |
23 | Size By Source 24 |
25 | 26 |
27 |
28 |
29 |
30 |
31 |
32 | 33 |
34 |
35 |
36 |
37 | Monitor Sources 38 |
39 | 40 |
41 |
42 |
43 |
44 |
45 |
46 | -------------------------------------------------------------------------------- /core/static/app/home/module.js: -------------------------------------------------------------------------------- 1 | (function() { 2 | 'use strict'; 3 | 4 | var home = angular.module('home', 5 | ['ui.router', 6 | 'chart.js', 7 | 'sizeByType', 8 | 'sizeBySource', 9 | 'monitor' 10 | ]); 11 | 12 | home.config(function($stateProvider, $urlRouterProvider) { 13 | $stateProvider 14 | .state('main.home', { 15 | url: "/home", 16 | views: { 17 | "": { templateUrl: "app/home/home.html"}, 18 | "monitor@main.home": { 19 | templateUrl: "app/monitor/monitor.html", 20 | controller: "MonitorCtrl"}, 21 | "sizeByType@main.home": { 22 | templateUrl: "app/sizeByType/donutSizeByType.html", 23 | controller : "SizeByTypeCtrl"}, 24 | "sizeBySource@main.home": { 25 | templateUrl: "app/sizeBySource/donutSizeBySource.html", 26 | controller: "SizeBySourceCtrl"} 27 | } 28 | }) 29 | }); 30 | })(); 31 | -------------------------------------------------------------------------------- /core/static/app/jobs/jobs.html: -------------------------------------------------------------------------------- 1 |
2 |
3 | Jobs Supervision 4 |
5 | 6 |
7 | 8 |
9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 |
Job id{{columnNameM}}
{{reportMetadataLvlOne.id}}{{reportMetadataLvlOne.duration}}{{reportMetadataLvlOne.endTime}}{{reportMetadataLvlOne.startTime}}{{reportMetadataLvlOne.status}}
27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 |
Conf Filename{{columnNameR}}
{{confFileName}}{{valueInReportConf}}
45 | 46 | 47 |
48 | 51 | 52 |
53 |
54 | -------------------------------------------------------------------------------- /core/static/app/jobs/module.js: -------------------------------------------------------------------------------- 1 | (function() { 2 | 'use strict'; 3 | 4 | var jobs = angular.module('jobs', ['ui.router', 'chart.js', 'ui.bootstrap']); 5 | 6 | jobs.controller('JobsCtrl', function($scope, $log, JobsSrv) { 7 | $scope.reportJobsRAW = {}; 8 | $scope.metadataRoot = []; 9 | $scope.metadataColumnNames = []; 10 | $scope.reportColumnNames = []; 11 | $scope.reportRoot = []; 12 | 13 | 14 | function getColumnNames() { 15 | //Process for metadata 16 | //get a key 17 | var someJobId = Object.keys($scope.metadataRoot)[0]; 18 | var someConfFilename = Object.keys($scope.reportJobsRAW[someJobId]['report'])[0]; 19 | 20 | for (var columnName in $scope.reportJobsRAW[someJobId]['report'][someConfFilename]) { 21 | if($scope.reportJobsRAW[someJobId]['report'][someConfFilename].hasOwnProperty(columnName)) { 22 | $scope.reportColumnNames.push(columnName); 23 | } 24 | } 25 | 26 | for (var columnName in $scope.metadataRoot[someJobId]) { 27 | if($scope.metadataRoot[someJobId].hasOwnProperty(columnName)) { 28 | $scope.metadataColumnNames.push(columnName); 29 | } 30 | } 31 | }; 32 | 33 | function isolateMetadata() { 34 | /* copy reportJobsRAW into metadataRoot but without 35 | key value couple 'report' 36 | */ 37 | $scope.metadataRoot = JSON.parse(JSON.stringify($scope.reportJobsRAW)); 38 | for (var idJobs in $scope.metadataRoot) { 39 | delete $scope.metadataRoot[idJobs]['report']; 40 | } 41 | }; 42 | 43 | function isolateReport() { 44 | //$scope.dictReport = JSON.parse(JSON.stringify($scope.reportJobsRAW)); 45 | for (var idJobs in $scope.reportJobsRAW) { 46 | $scope.reportRoot[idJobs] = $scope.reportJobsRAW[idJobs]['report']; 47 | } 48 | }; 49 | 50 | function processJobsSources() { 51 | //get jobs's report 52 | JobsSrv.list().then( 53 | function(response) { 54 | $scope.reportJobsRAW = response.data; 55 | isolateMetadata(); 56 | getColumnNames(); 57 | isolateReport(); 58 | }, 59 | function(rejection) { 60 | console.log(rejection.data); 61 | } 62 | ); 63 | }; 64 | 65 | 66 | processJobsSources(); 67 | }); 68 | 69 | jobs.service('JobsSrv', function($http) { 70 | return { 71 | list: function() { 72 | return $http.get('hippocampe/api/v1.0/jobs'); 73 | } 74 | }; 75 | }); 76 | 77 | })(); 78 | -------------------------------------------------------------------------------- /core/static/app/main/main.html: -------------------------------------------------------------------------------- 1 |
2 | 33 |
34 | 35 |
36 | -------------------------------------------------------------------------------- /core/static/app/main/module.js: -------------------------------------------------------------------------------- 1 | (function() { 2 | // Create an application 3 | var main = angular.module('main', 4 | ['ui.router', 5 | 'chart.js', 6 | 'ui.bootstrap', 7 | 'home', 8 | 'sizeByType', 9 | 'sizeBySource', 10 | 'monitor', 11 | 'jobs', 12 | 'more', 13 | 'sources', 14 | 'type', 15 | 'hipposcore', 16 | 'shadowbook' 17 | ]); 18 | 19 | main.config(function($stateProvider, $urlRouterProvider) { 20 | 21 | $stateProvider 22 | .state('main', { 23 | views: { 24 | "": {templateUrl: 'app/main/main.html'}, 25 | "hipposcore@main": { 26 | templateUrl: 'app/hipposcore/hipposcore.html', 27 | controller: 'HipposcoreCtrl' 28 | }, 29 | "shadowbook@main": { 30 | templateUrl: 'app/shadowbook/shadowbook.html', 31 | controller: 'ShadowbookCtrl' 32 | } 33 | } 34 | }) 35 | .state('main.more', { 36 | url: '/more', 37 | templateUrl: 'app/more/more.html', 38 | controller: 'MoreCtrl' 39 | }) 40 | .state('main.jobs', { 41 | url: '/jobs', 42 | templateUrl: 'app/jobs/jobs.html', 43 | controller: 'JobsCtrl' 44 | }) 45 | .state('main.sources', { 46 | url: '/sources', 47 | templateUrl: 'app/sources/sources.html', 48 | controller: 'SourcesCtrl' 49 | }) 50 | // .state('main.hipposcore', { 51 | // url: '/hipposcore', 52 | // templateUrl: 'app/hipposcore/hipposcore.html', 53 | // controller: 'HipposcoreCtrl' 54 | // }) 55 | // .state('main.shadowbook', { 56 | // url: '/shadowbook', 57 | // templateUrl: 'app/shadowbook/shadowbook.html', 58 | // controller: 'ShadowbookCtrl' 59 | // }) 60 | }); 61 | 62 | })(); -------------------------------------------------------------------------------- /core/static/app/monitor/module.js: -------------------------------------------------------------------------------- 1 | (function() { 2 | 'use strict'; 3 | 4 | var monitor = angular.module('monitor', ['ui.router', "chart.js"]); 5 | 6 | monitor.controller('MonitorCtrl', function($scope, MonitorSrv) { 7 | $scope.reportMonitor = {}; 8 | $scope.reportSize = {}; 9 | $scope.columnNames = []; 10 | 11 | function getColumnNames() { 12 | //get a key (which is an url) 13 | var someKey = Object.keys($scope.reportMonitor)[0]; 14 | 15 | for (var columnName in $scope.reportMonitor[someKey]) { 16 | if($scope.reportMonitor[someKey].hasOwnProperty(columnName)) { 17 | $scope.columnNames.push(columnName); 18 | } 19 | } 20 | }; 21 | 22 | function processMonitorSources() { 23 | //get monitorSource's report 24 | MonitorSrv.list().then( 25 | function(response) { 26 | $scope.reportMonitor = response.data; 27 | getColumnNames(); 28 | }, 29 | function(rejection) { 30 | console.log(rejection.data); 31 | } 32 | ); 33 | }; 34 | 35 | 36 | processMonitorSources(); 37 | }); 38 | 39 | monitor.service('MonitorSrv', function($http) { 40 | return { 41 | list: function() { 42 | return $http.get('hippocampe/api/v1.0/monitorSources'); 43 | } 44 | }; 45 | }); 46 | 47 | })(); 48 | -------------------------------------------------------------------------------- /core/static/app/monitor/monitor.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 |
URL{{columnName}}
{{url}}{{value}}
15 | -------------------------------------------------------------------------------- /core/static/app/more/module.js: -------------------------------------------------------------------------------- 1 | (function() { 2 | 'use strict'; 3 | 4 | var more = angular.module('more', ['ui.router', 'ui.bootstrap']); 5 | 6 | // more.config(function($stateProvider, $urlRouterProvider) { 7 | // $stateProvider 8 | // .state('main.more', { 9 | // url: '/more', 10 | // views: { 11 | // "": { 12 | // templateUrl: "app/more/more.html", 13 | // controller: "MoreCtrl" 14 | // }, 15 | // "type@main.more": { 16 | // templateUrl: "app/type/type.html", 17 | // controller: "TypeCtrl" 18 | // } 19 | // } 20 | // }) 21 | // }); 22 | 23 | more.controller('MoreCtrl', function($scope, MoreSrv, TypeSrv) { 24 | $scope.list = []; 25 | $scope.textareaValue = ''; 26 | $scope.observables = ''; 27 | $scope.listObservables = []; 28 | $scope.listTypes = []; 29 | $scope.selectedType; 30 | $scope.moreResult = {}; 31 | 32 | $scope.submit = function() { 33 | 34 | if ($scope.textareaValue) { 35 | $scope.observables=this.textareaValue; 36 | $scope.textareaValue = ''; 37 | } 38 | 39 | $scope.listObservables = $scope.observables.split('\n'); 40 | 41 | MoreSrv.list($scope.selectedType, $scope.listObservables).then( 42 | function(response) { 43 | //alert(JSON.stringify(response.data)); 44 | $scope.moreResult = JSON.parse(JSON.stringify(response.data)); 45 | }, 46 | function(rejection) { 47 | console.log(rejection.data); 48 | } 49 | ); 50 | }; 51 | 52 | //dropdown toggle stuff 53 | function init() { 54 | //retrieve type for dropdown toggle 55 | TypeSrv.list().then( 56 | function(response) { 57 | $scope.listTypes = response.data['type']; 58 | }, 59 | function(rejection) { 60 | console.log(rejection.data); 61 | } 62 | ); 63 | }; 64 | 65 | // change dropdown toggle label according to selected type 66 | $scope.selectTypeMore = function(selectedType) { 67 | $scope.selectedType = selectedType; 68 | }; 69 | 70 | init(); 71 | }); 72 | 73 | more.service('MoreSrv', function($http) { 74 | return { 75 | list: function(selectedType, listObservables) { 76 | var requestBody = {}; 77 | var subBody = {}; 78 | subBody['type'] = selectedType; 79 | var observable; 80 | 81 | listObservables.forEach(function(observable) { 82 | requestBody[observable] = subBody; 83 | }); 84 | 85 | //alert(JSON.stringify(requestBody)); 86 | return $http.post('hippocampe/api/v1.0/more', requestBody); 87 | } 88 | }; 89 | }); 90 | 91 | more.service('TypeSrv', function($http) { 92 | return { 93 | list: function() { 94 | return $http.get('hippocampe/api/v1.0/type'); 95 | } 96 | }; 97 | }); 98 | 99 | })(); 100 | -------------------------------------------------------------------------------- /core/static/app/more/more.html: -------------------------------------------------------------------------------- 1 |
2 | 3 |
4 | 5 |
6 |
7 | 11 | 16 |
17 |
18 |
19 | 20 |
21 | 22 |
23 | 24 |
25 |
26 | 27 |
28 |
29 | 30 |
31 |
32 | 33 |
34 |
35 |
36 | 37 |
38 | 39 |
40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 |
{{key}}{{value}}{{value['hipposcore']}}
48 | 49 |
50 | 51 |
52 | 53 |
54 |
55 |
56 | 57 |
58 | -------------------------------------------------------------------------------- /core/static/app/shadowbook/module.js: -------------------------------------------------------------------------------- 1 | (function() { 2 | 'use strict'; 3 | 4 | var shadowbook = angular.module('shadowbook', ['ui.router', 'ui.bootstrap']) 5 | .config(function(NotificationProvider) { 6 | NotificationProvider.setOptions({ 7 | delay: 10000, 8 | startTop: 55, 9 | startRight: 10, 10 | verticalSpacing: 20, 11 | horizontalSpacing: 20, 12 | positionX: 'right', 13 | positionY: 'top' 14 | }); 15 | }); 16 | 17 | shadowbook.controller('ShadowbookCtrl', function($scope, ShadowbookSrv, Notification) { 18 | $scope.alerts; 19 | var response; 20 | 21 | 22 | $scope.shadowbook = function() { 23 | //clean out the alerts list, one alert at a time 24 | $scope.alerts = []; 25 | 26 | $scope.closeAlert = function(index) { 27 | $scope.alerts.splice(index, 1); 28 | }; 29 | 30 | ShadowbookSrv.list().then( 31 | function(response) { 32 | //job launched with success 33 | response = JSON.parse(JSON.stringify(response.data)); 34 | if (response.hasOwnProperty('job')) { 35 | var idJobList = Object.keys(response['job']); 36 | //there's only one element in idJobList => idJobList[0] 37 | var msg = 'Job '.concat(idJobList[0]).concat(' sucessfully launched'); 38 | Notification.success({message: msg}); 39 | } 40 | }, 41 | function(rejection) { 42 | //one job already running 43 | var msg = rejection.data['error']; 44 | Notification.error({message: msg}); 45 | } 46 | ); 47 | }; 48 | 49 | }); 50 | 51 | shadowbook.service('ShadowbookSrv', function($http) { 52 | return { 53 | list: function() { 54 | return $http.get('hippocampe/api/v1.0/shadowbook'); 55 | } 56 | }; 57 | }); 58 | 59 | 60 | })(); -------------------------------------------------------------------------------- /core/static/app/shadowbook/shadowbook.html: -------------------------------------------------------------------------------- 1 |
2 | 3 | 4 | 5 |
6 | -------------------------------------------------------------------------------- /core/static/app/sizeBySource/donutSizeBySource.html: -------------------------------------------------------------------------------- 1 | 3 | 4 | -------------------------------------------------------------------------------- /core/static/app/sizeBySource/module.js: -------------------------------------------------------------------------------- 1 | (function() { 2 | 'use strict'; 3 | 4 | var sizeBySource = angular.module('sizeBySource', ['ui.router', "chart.js"]); 5 | 6 | sizeBySource.controller('SizeBySourceCtrl', function($scope, SizeBySourceSrv) { 7 | $scope.labelsDonut = []; 8 | $scope.dataDonut = []; 9 | 10 | function getDonutData() { 11 | for (var url in $scope.reportSize){ 12 | $scope.labelsDonut.push(url); 13 | $scope.dataDonut.push($scope.reportSize[url]["size"]); 14 | } 15 | }; 16 | 17 | 18 | function processSizeBySource() { 19 | SizeBySourceSrv.list().then( 20 | function(response) { 21 | $scope.reportSize = response.data; 22 | getDonutData(); 23 | }, 24 | function(rejection) { 25 | console.log(rejection.data); 26 | } 27 | ); 28 | }; 29 | 30 | processSizeBySource(); 31 | }); 32 | 33 | 34 | sizeBySource.service('SizeBySourceSrv', function($http) { 35 | return { 36 | list: function() { 37 | return $http.get('hippocampe/api/v1.0/sizeBySources'); 38 | } 39 | }; 40 | }); 41 | 42 | })(); 43 | -------------------------------------------------------------------------------- /core/static/app/sizeByType/donutSizeByType.html: -------------------------------------------------------------------------------- 1 | 3 | 4 | -------------------------------------------------------------------------------- /core/static/app/sizeByType/module.js: -------------------------------------------------------------------------------- 1 | (function() { 2 | 'use strict'; 3 | 4 | var sizeByType = angular.module('sizeByType', ['ui.router', "chart.js"]); 5 | 6 | sizeByType.controller('SizeByTypeCtrl', function($scope, SizeByTypeSrv) { 7 | $scope.labelsDonut = []; 8 | $scope.dataDonut = []; 9 | 10 | function getDonutData() { 11 | for (var intelType in $scope.reportSize){ 12 | $scope.labelsDonut.push(intelType); 13 | $scope.dataDonut.push($scope.reportSize[intelType]["size"]); 14 | } 15 | }; 16 | 17 | 18 | function processSizeByType() { 19 | SizeByTypeSrv.list().then( 20 | function(response) { 21 | $scope.reportSize = response.data; 22 | getDonutData(); 23 | }, 24 | function(rejection) { 25 | console.log(rejection.data); 26 | } 27 | ); 28 | }; 29 | 30 | processSizeByType(); 31 | }); 32 | 33 | 34 | sizeByType.service('SizeByTypeSrv', function($http) { 35 | return { 36 | list: function() { 37 | return $http.get('hippocampe/api/v1.0/sizeByType'); 38 | } 39 | }; 40 | }); 41 | 42 | })(); 43 | -------------------------------------------------------------------------------- /core/static/app/sources/module.js: -------------------------------------------------------------------------------- 1 | (function() { 2 | 'use strict'; 3 | 4 | var sources = angular.module('sources', ['ui.router', "chart.js"]); 5 | 6 | sources.controller('SourcesCtrl', function($scope, SourcesSrv) { 7 | $scope.reportSources = {}; 8 | $scope.columnNames = []; 9 | 10 | function getColumnNames() { 11 | //get a key (which is an url) 12 | var someKey = Object.keys($scope.reportSources)[0]; 13 | 14 | for (var columnName in $scope.reportSources[someKey]) { 15 | if($scope.reportSources[someKey].hasOwnProperty(columnName)) { 16 | $scope.columnNames.push(columnName); 17 | } 18 | } 19 | }; 20 | 21 | function cleanReportSources(){ 22 | //reportSources has duplicates info (URL and link) 23 | //deleting link 24 | for (var url in $scope.reportSources){ 25 | delete $scope.reportSources[url]['link']; 26 | } 27 | }; 28 | 29 | function processSources() { 30 | //get sourcesSource's report 31 | SourcesSrv.list().then( 32 | function(response) { 33 | $scope.reportSources = response.data; 34 | cleanReportSources(); 35 | getColumnNames(); 36 | }, 37 | function(rejection) { 38 | console.log(rejection.data); 39 | } 40 | ); 41 | }; 42 | 43 | processSources(); 44 | }); 45 | 46 | sources.service('SourcesSrv', function($http) { 47 | return { 48 | list: function() { 49 | return $http.get('hippocampe/api/v1.0/sources'); 50 | } 51 | }; 52 | }); 53 | 54 | })(); 55 | -------------------------------------------------------------------------------- /core/static/app/sources/sources.html: -------------------------------------------------------------------------------- 1 |
2 |
3 | Available Sources 4 |
5 | 6 |
7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 |
URL{{columnName}}
{{url}}{{value}}
21 |
22 |
23 | -------------------------------------------------------------------------------- /core/static/app/type/module.js: -------------------------------------------------------------------------------- 1 | (function() { 2 | 'use strict'; 3 | 4 | var type = angular.module('type', ['ui.router', "chart.js"]); 5 | 6 | type.controller('TypeCtrl', function($scope, TypeSrv) { 7 | 8 | $scope.listTypes = []; 9 | $scope.selectedType; 10 | 11 | //dropdown toggle stuff 12 | function init() { 13 | //retrieve type for dropdown toggle 14 | TypeSrv.list().then( 15 | function(response) { 16 | $scope.listTypes = response.data['type']; 17 | }, 18 | function(rejection) { 19 | console.log(rejection.data); 20 | } 21 | ); 22 | }; 23 | 24 | // change dropdown toggle label according to selected type 25 | $scope.selectTypeMore = function(selectedType) { 26 | $scope.selectedType = selectedType; 27 | }; 28 | 29 | init(); 30 | }); 31 | 32 | type.service('TypeSrv', function($http) { 33 | return { 34 | list: function() { 35 | return $http.get('hippocampe/api/v1.0/type'); 36 | } 37 | }; 38 | }); 39 | })(); 40 | -------------------------------------------------------------------------------- /core/static/app/type/type.html: -------------------------------------------------------------------------------- 1 |
2 | 6 | 11 |
12 | -------------------------------------------------------------------------------- /core/static/bower.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Hippocampe", 3 | "version": "1.0.0", 4 | "dependencies": { 5 | "angular": "^1.5.0", 6 | "angular-ui-router": "~0.2.15", 7 | "angular-chart.js": "~0.8.5", 8 | "angular-bootstrap": "~1.3.3", 9 | "angular-ui-notification": "0.3.5", 10 | "bootstrap": "^3.3.7" 11 | }, 12 | "resolutions": { 13 | "angular": "^1.5.0" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /core/static/fonts/glyphicons-halflings-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheHive-Project/Hippocampe/03df065bd304b1515db6925f6cd01a754d42825e/core/static/fonts/glyphicons-halflings-regular.eot -------------------------------------------------------------------------------- /core/static/fonts/glyphicons-halflings-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheHive-Project/Hippocampe/03df065bd304b1515db6925f6cd01a754d42825e/core/static/fonts/glyphicons-halflings-regular.ttf -------------------------------------------------------------------------------- /core/static/fonts/glyphicons-halflings-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheHive-Project/Hippocampe/03df065bd304b1515db6925f6cd01a754d42825e/core/static/fonts/glyphicons-halflings-regular.woff -------------------------------------------------------------------------------- /core/static/fonts/glyphicons-halflings-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheHive-Project/Hippocampe/03df065bd304b1515db6925f6cd01a754d42825e/core/static/fonts/glyphicons-halflings-regular.woff2 -------------------------------------------------------------------------------- /core/static/images/android-chrome-192x192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheHive-Project/Hippocampe/03df065bd304b1515db6925f6cd01a754d42825e/core/static/images/android-chrome-192x192.png -------------------------------------------------------------------------------- /core/static/images/apple-touch-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheHive-Project/Hippocampe/03df065bd304b1515db6925f6cd01a754d42825e/core/static/images/apple-touch-icon.png -------------------------------------------------------------------------------- /core/static/images/browserconfig.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | #da532c 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /core/static/images/favicon-16x16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheHive-Project/Hippocampe/03df065bd304b1515db6925f6cd01a754d42825e/core/static/images/favicon-16x16.png -------------------------------------------------------------------------------- /core/static/images/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheHive-Project/Hippocampe/03df065bd304b1515db6925f6cd01a754d42825e/core/static/images/favicon-32x32.png -------------------------------------------------------------------------------- /core/static/images/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheHive-Project/Hippocampe/03df065bd304b1515db6925f6cd01a754d42825e/core/static/images/favicon.ico -------------------------------------------------------------------------------- /core/static/images/hippocampe-horizontal-bright.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheHive-Project/Hippocampe/03df065bd304b1515db6925f6cd01a754d42825e/core/static/images/hippocampe-horizontal-bright.png -------------------------------------------------------------------------------- /core/static/images/hippocampe-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheHive-Project/Hippocampe/03df065bd304b1515db6925f6cd01a754d42825e/core/static/images/hippocampe-logo.png -------------------------------------------------------------------------------- /core/static/images/hippocampe-small.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheHive-Project/Hippocampe/03df065bd304b1515db6925f6cd01a754d42825e/core/static/images/hippocampe-small.png -------------------------------------------------------------------------------- /core/static/images/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hippocampe", 3 | "icons": [ 4 | { 5 | "src": "\/android-chrome-192x192.png", 6 | "sizes": "192x192", 7 | "type": "image\/png" 8 | } 9 | ], 10 | "theme_color": "#ffffff", 11 | "display": "standalone" 12 | } 13 | -------------------------------------------------------------------------------- /core/static/images/mstile-150x150.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheHive-Project/Hippocampe/03df065bd304b1515db6925f6cd01a754d42825e/core/static/images/mstile-150x150.png -------------------------------------------------------------------------------- /core/static/images/safari-pinned-tab.svg: -------------------------------------------------------------------------------- 1 | 2 | 4 | 7 | 8 | Created by potrace 1.11, written by Peter Selinger 2001-2013 9 | 10 | 12 | 28 | 37 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /core/templates/error.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Hippocampe 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 33 | 34 | 35 | 36 | {% raw %} 37 |
{{alert.msg}}
38 | {% endraw %} 39 | 51 | 52 | 53 | 54 | -------------------------------------------------------------------------------- /core/templates/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Hippocampe 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 |
20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.2' 2 | 3 | services: 4 | elasticsearch: 5 | image: docker.elastic.co/elasticsearch/elasticsearch:5.3.0 6 | container_name: hipposearch 7 | ports: 8 | - "127.0.0.1:9200:9200" 9 | environment: 10 | http.host: "0.0.0.0" 11 | transport.host: "127.0.0.1" 12 | xpack.security.enabled: "false" 13 | xpack.monitoring.enabled: "false" 14 | xpack.graph.enabled: "false" 15 | xpack.watcher.enabled: "false" 16 | bootstrap.memory_lock: "true" 17 | ulimits: 18 | memlock: 19 | soft: -1 20 | hard: -1 21 | nofile: 22 | soft: 65536 23 | hard: 65536 24 | networks: 25 | - hipponet 26 | 27 | hippocampe: 28 | build: . 29 | container_name: hippocampe 30 | ports: 31 | - "5000:5000" 32 | networks: 33 | - hipponet 34 | depends_on: 35 | - elasticsearch 36 | 37 | networks: 38 | hipponet: 39 | driver: bridge 40 | -------------------------------------------------------------------------------- /docker-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd /opt/Hippocampe/core 4 | python app.py 5 | -------------------------------------------------------------------------------- /docs/install_guide.md: -------------------------------------------------------------------------------- 1 | # Install Guide 2 | 3 | ## Intro 4 | You will find bellow the installation instructions. 5 | Don't want to install it ? You can use the docker version instead, check [this](#docker). 6 | Check also the [tutorial](tutorial.md) for more details. 7 | 8 | ## Requirements 9 | 10 | Hippocampe needs some external tools to work, you will find below the list of requirements: 11 | 12 | + JAVA (either [Oracle](http://www.webupd8.org/2014/03/oracle-java-8-stable-released-install.html) or [openJDK](http://openjdk.java.net/install/index.html), however we have noticed better performance with oracle) 13 | + [Elasticsearch **5.1**](https://www.elastic.co/guide/en/elasticsearch/reference/current/deb.html) 14 | + Some python libraries: 15 | + elasticsearch 16 | + Configparser 17 | + flask 18 | + python-dateutil 19 | + apscheduler 20 | + requests 21 | 22 | ``` 23 | pip install elasticsearch Configparser netaddr flask python-dateutil apscheduler requests 24 | ``` 25 | 26 | ## Configuration 27 | 28 | The default Elasticsearch 5.1's configuration is enough to make Hippocampe works. 29 | 30 | ## Installation 31 | * Clone or download the project 32 | * Install the web dependencies with bower (https://bower.io/) 33 | ``` 34 | cd Hippocampe/core/static 35 | bower install 36 | ``` 37 | * Start elasticsearch 38 | ``` 39 | service elasticsearch start 40 | ``` 41 | * run app.py script 42 | ``` 43 | python Hippocampe/core/app.py 44 | ``` 45 | By default, Hippocampe is listening on port 5000. 46 | 47 | ## Docker 48 | If you just want to give it a try, you may want to use Hippocampe inside of Docker: 49 | 50 | ``` 51 | cd Hippocampe/ 52 | docker build -t hippocampe . 53 | docker run -p 5000:5000 hippocampe 54 | ``` 55 | 56 | Now Hippocampe is available on port 5000 and runs inside a docker. 57 | 58 | If you want to spin-up both Elasticsearch and Hippocampe, you can use docker-compose: 59 | 60 | **Note:** If you use this method, you need to edit core/conf/hippo/hippo.conf and change the elasticsearch address to the container name 61 | ``` 62 | [elasticsearch] 63 | ip : hipposearch 64 | port : 9200 65 | ``` 66 | ``` 67 | cd Hippocampe/ 68 | docker-compose up 69 | ``` 70 | 71 | 72 | ## Hippocampe as a service 73 | 74 | To turn Hippocampe into a service, the uWSGI tool and a NGINX server will be used. 75 | 76 | NGINX will host all the web content while uWSGI will execute the python code. 77 | 78 | In this example, Hippocampe is located at ```/opt/Hippocampe``` and configuration files for both nginx and uWSGI are located at ```/var/www/demoapp```. 79 | 80 | ### Install NGINX 81 | 82 | ``` 83 | sudo apt-get install nginx 84 | ``` 85 | 86 | ### Install uWSGI 87 | 88 | ``` 89 | sudo apt-get install build-essential python python-dev 90 | sudo pip install uwsgi 91 | ``` 92 | 93 | ### Configuring nginx 94 | 95 | * Delete the default nginx's site 96 | 97 | ``` 98 | sudo rm /etc/nginx/sites-enabled/default 99 | ``` 100 | 101 | * Create the nginx's configuration file at ```/var/www/demoapp/hippo_nginx.conf``` 102 | 103 | ``` 104 | sudo mkdir /var/www/demoapp 105 | ``` 106 | 107 | ``` 108 | server { 109 | listen 80; 110 | server_name localhost; 111 | charset utf-8; 112 | client_max_body_size 75M; 113 | 114 | location / { try_files $uri @hippocampe; } 115 | location @hippocampe { 116 | include uwsgi_params; 117 | uwsgi_pass unix:/var/www/demoapp/demoapp_uwsgi.sock; 118 | } 119 | } 120 | ``` 121 | 122 | * Link the file with nginx 123 | 124 | ``` 125 | sudo ln -s /var/www/demoapp/hippo_nginx.conf /etc/nginx/conf.d/ 126 | sudo /etc/init.d/nginx restart 127 | ``` 128 | 129 | ### Configuring uWSGI 130 | 131 | * Create the configuration file at ```/var/www/demoapp/demoapp_uwsgi.ini``` 132 | 133 | ``` 134 | [uwsgi] 135 | #application's base folder 136 | chdir = /opt/Hippocampe/core 137 | 138 | #python module to import 139 | app = app 140 | module = %(app) 141 | 142 | processes = 8 143 | thread = 16 144 | enable-threads = true 145 | pythonpath = /usr/local/lib/python2.7/dist-packages 146 | pythonpath = /usr/lib/python2.7 147 | 148 | #socket file's location 149 | socket = /var/www/demoapp/%n.sock 150 | 151 | #permissions for the socket file 152 | chmod-socket = 666 153 | 154 | #the variable that holds a flask application inside the module imported at line #6 155 | callable = app 156 | 157 | #location of log files 158 | logto = /var/www/demoapp/log/uwsgi/%n.log 159 | uid = www-data 160 | gid = www-data 161 | ``` 162 | 163 | * Create the folder for uWSGI's logs 164 | 165 | ``` 166 | sudo mkdir -p /var/www/demoapp/log/uwsgi 167 | ``` 168 | 169 | * Give the adequat rights 170 | 171 | ``` 172 | sudo chown -R www-data:www-data /var/www/demoapp/ 173 | sudo chown -R www-data:www-data /opt/Hippocampe 174 | ``` 175 | 176 | ### Turn all stuff into a service with ```systemctl``` 177 | 178 | * Create the file ```/etc/systemd/system/uwsgi.service``` 179 | 180 | ``` 181 | [Unit] 182 | Description=uWSGI for hippocampe demo 183 | After=syslog.target 184 | 185 | [Service] 186 | ExecStart=/usr/local/bin/uwsgi --master --emperor /etc/uwsgi/vassals --die-on-term --uid www-data --gid www-data --logto /var/www/demoapp/log/uwsgi/emperor.log 187 | Restart=always 188 | KillSignal=SIGQUIT 189 | Type=notify 190 | StandardError=syslog 191 | NotifyAccess=all 192 | 193 | [Install] 194 | WantedBy=multi-user.target 195 | ``` 196 | 197 | * Create the directory ```/etc/uwsgi/vassals``` 198 | 199 | ``` 200 | sudo mkdir -p /etc/uwsgi/vassals 201 | ``` 202 | 203 | * Link the uWSGI config file to it 204 | 205 | ``` 206 | sudo ln -s /var/www/demoapp/demoapp_uwsgi.ini /etc/uwsgi/vassals 207 | ``` 208 | 209 | * Start the service 210 | 211 | ``` 212 | sudo systemctl daemon-reload 213 | sudo systemctl start uwsgi 214 | ``` 215 | 216 | ### Test it 217 | 218 | Go to ```http://localhost/hippocampe``` and it should work. 219 | 220 | Moreover the API is now expose to port 80. 221 | 222 | ### Logs path 223 | 224 | * Hippocampe's logs 225 | * ```Hippocampe/core/logs/hippocampe.log``` 226 | * nginx's logs 227 | * ```/var/log/nginx/access.log``` 228 | * ```/var/log/nginx/error.log``` 229 | * uWSGI's logs 230 | * ```/var/www/demoapp/log/uwsgi/demoapp_uwsgi.log``` 231 | * ```/var/www/demoapp/log/uwsgi/emperor.log``` 232 | -------------------------------------------------------------------------------- /docs/pics/activate_deactivate.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheHive-Project/Hippocampe/03df065bd304b1515db6925f6cd01a754d42825e/docs/pics/activate_deactivate.jpg -------------------------------------------------------------------------------- /docs/pics/graph1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheHive-Project/Hippocampe/03df065bd304b1515db6925f6cd01a754d42825e/docs/pics/graph1.png -------------------------------------------------------------------------------- /docs/pics/graph2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheHive-Project/Hippocampe/03df065bd304b1515db6925f6cd01a754d42825e/docs/pics/graph2.png -------------------------------------------------------------------------------- /docs/pics/hipposcore.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheHive-Project/Hippocampe/03df065bd304b1515db6925f6cd01a754d42825e/docs/pics/hipposcore.png -------------------------------------------------------------------------------- /docs/pics/hipposcore_main_part.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheHive-Project/Hippocampe/03df065bd304b1515db6925f6cd01a754d42825e/docs/pics/hipposcore_main_part.png -------------------------------------------------------------------------------- /docs/pics/hipposcore_percentage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheHive-Project/Hippocampe/03df065bd304b1515db6925f6cd01a754d42825e/docs/pics/hipposcore_percentage.png -------------------------------------------------------------------------------- /docs/pics/hipposcore_range.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheHive-Project/Hippocampe/03df065bd304b1515db6925f6cd01a754d42825e/docs/pics/hipposcore_range.png -------------------------------------------------------------------------------- /docs/pics/hipposcore_sign.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheHive-Project/Hippocampe/03df065bd304b1515db6925f6cd01a754d42825e/docs/pics/hipposcore_sign.png -------------------------------------------------------------------------------- /docs/pics/n3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheHive-Project/Hippocampe/03df065bd304b1515db6925f6cd01a754d42825e/docs/pics/n3.png -------------------------------------------------------------------------------- /docs/pics/p.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheHive-Project/Hippocampe/03df065bd304b1515db6925f6cd01a754d42825e/docs/pics/p.png -------------------------------------------------------------------------------- /docs/pics/variation_table.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheHive-Project/Hippocampe/03df065bd304b1515db6925f6cd01a754d42825e/docs/pics/variation_table.png -------------------------------------------------------------------------------- /images/Hippocampe-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheHive-Project/Hippocampe/03df065bd304b1515db6925f6cd01a754d42825e/images/Hippocampe-logo.png --------------------------------------------------------------------------------