| URL | 5 |{{columnName}} | 6 |
|---|---|
| {{url}} | 11 |{{value}} | 12 |
2 |
6 |
11 |
12 |
--------------------------------------------------------------------------------
/core/conf/feeds/dshield_high_DOMAIN.conf:
--------------------------------------------------------------------------------
1 |
2 | [source]
3 | url : http://www.dshield.org/feeds/suspiciousdomains_High.txt
4 | delimiter : ,
5 | beginWithBlank : False
6 | fields : ["domain"]
7 | extraFields :
8 | description : DShield suspicious domains, this list consists of High Level Sensitivity website URLs
9 | startAt : 15
10 | score : -100
11 | coreIntelligence : domain
12 | validityDate:
13 | useByDate:
14 |
15 | [elasticsearch]
16 | typeIntel = dshieldFree_highDOMAIN
17 |
18 | [intel]
19 | domain = {
20 | "mapping" : {
21 | "type": "string",
22 | "index": "not_analyzed"
23 | }
24 | }
25 |
--------------------------------------------------------------------------------
/core/conf/feeds/hpHosts_fsaDOMAIN.conf:
--------------------------------------------------------------------------------
1 |
2 | [source]
3 | url=https://hosts-file.net/fsa.txt
4 | delimiter : \t
5 | beginWithBlank : False
6 | fields : ["localhost", "domain"]
7 | extraFields :
8 | description : This file contains fraud sites listed in the hpHosts database. This should ONLY be downloaded by those wanting to block fraud sites and nothing else, and requires manual merging.
9 | startAt : 7
10 | score : -100
11 | coreIntelligence : domain
12 | validityDate:
13 | useByDate:
14 |
15 | [elasticsearch]
16 | typeIntel = host_file_fsaDOMAIN
17 |
18 | [intel]
19 | localhost = {
20 | "mapping" : {
21 | "type": "string",
22 | "index": "no"
23 | }
24 | }
25 | domain = {
26 | "mapping" : {
27 | "type": "string",
28 | "index": "not_analyzed"
29 | }
30 | }
31 |
--------------------------------------------------------------------------------
/core/conf/feeds/hpHosts_hjkDOMAIN.conf:
--------------------------------------------------------------------------------
1 |
2 | [source]
3 | url=https://hosts-file.net/hjk.txt
4 | delimiter : \t
5 | beginWithBlank : False
6 | fields : ["localhost", "domain"]
7 | extraFields :
8 | description : This file contains hijack sites listed in the hpHosts database This should ONLY be downloaded by those wanting to block hijack sites and nothing else, and requires manual merging.
9 | startAt : 7
10 | score : -100
11 | coreIntelligence : domain
12 | validityDate:
13 | useByDate:
14 |
15 | [elasticsearch]
16 | typeIntel = host_file_hjkDOMAIN
17 |
18 | [intel]
19 | localhost = {
20 | "mapping" : {
21 | "type": "string",
22 | "index": "no"
23 | }
24 | }
25 | domain = {
26 | "mapping" : {
27 | "type": "string",
28 | "index": "not_analyzed"
29 | }
30 | }
31 |
--------------------------------------------------------------------------------
/core/conf/feeds/hpHosts_emdDOMAIN.conf:
--------------------------------------------------------------------------------
1 |
2 | [source]
3 | url=https://hosts-file.net/emd.txt
4 | delimiter : \t
5 | beginWithBlank : False
6 | fields : ["localhost", "domain"]
7 | extraFields :
8 | description : This file contains malware sites listed in the hpHosts database. This should ONLY be downloaded by those wanting to block malware sites and nothing else, and requires manual merging.
9 | startAt : 7
10 | score : -100
11 | coreIntelligence : domain
12 | validityDate:
13 | useByDate:
14 |
15 | [elasticsearch]
16 | typeIntel = host_file_emdDOMAIN
17 |
18 | [intel]
19 | localhost = {
20 | "mapping" : {
21 | "type": "string",
22 | "index": "no"
23 | }
24 | }
25 | domain = {
26 | "mapping" : {
27 | "type": "string",
28 | "index": "not_analyzed"
29 | }
30 | }
31 |
--------------------------------------------------------------------------------
/core/conf/feeds/hpHosts_expDOMAIN.conf:
--------------------------------------------------------------------------------
1 |
2 | [source]
3 | url=https://hosts-file.net/exp.txt
4 | delimiter : \t
5 | beginWithBlank : False
6 | fields : ["localhost", "domain"]
7 | extraFields :
8 | description : This file contains exploit sites listed in the hpHosts database. This should ONLY be downloaded by those wanting to block exploit sites and nothing else, and requires manual merging.
9 | startAt : 7
10 | score : -100
11 | coreIntelligence : domain
12 | validityDate:
13 | useByDate:
14 |
15 | [elasticsearch]
16 | typeIntel = host_file_expDOMAIN
17 |
18 | [intel]
19 | localhost = {
20 | "mapping" : {
21 | "type": "string",
22 | "index": "no"
23 | }
24 | }
25 | domain = {
26 | "mapping" : {
27 | "type": "string",
28 | "index": "not_analyzed"
29 | }
30 | }
31 |
--------------------------------------------------------------------------------
/core/conf/feeds/hpHosts_pshDOMAIN.conf:
--------------------------------------------------------------------------------
1 |
2 | [source]
3 | url=https://hosts-file.net/psh.txt
4 | delimiter : \t
5 | beginWithBlank : False
6 | fields : ["localhost", "domain"]
7 | extraFields :
8 | description : This file contains phishing sites listed in the hpHosts database This should ONLY be downloaded by those wanting to block phishing sites and nothing else, and requires manual merging.
9 | startAt : 7
10 | score : -100
11 | coreIntelligence : domain
12 | validityDate:
13 | useByDate:
14 |
15 | [elasticsearch]
16 | typeIntel = host_file_pshDOMAIN
17 |
18 | [intel]
19 | localhost = {
20 | "mapping" : {
21 | "type": "string",
22 | "index": "no"
23 | }
24 | }
25 | domain = {
26 | "mapping" : {
27 | "type": "string",
28 | "index": "not_analyzed"
29 | }
30 | }
31 |
--------------------------------------------------------------------------------
/core/conf/feeds/ET_IP.conf:
--------------------------------------------------------------------------------
1 |
2 | [source]
3 | url=http://rules.emergingthreats.net/blockrules/compromised-ips.txt
4 | delimiter : ,
5 | beginWithBlank : False
6 | fields : ["ip"]
7 | extraFields :
8 | description : This ruleset is compiled from a number of sources. It's contents are hosts that are known to be compromised by bots, phishing sites, etc, or known to be spewing hostile traffic. These are not your everyday infected and sending a bit of spam hosts, these are significantly infected and hostile hosts.
9 | startAt : 0
10 | score : -100
11 | coreIntelligence : ip
12 | validityDate:
13 | useByDate:
14 |
15 | [elasticsearch]
16 | typeIntel = etFree_compromisedIP
17 |
18 | [intel]
19 | ip = {
20 | "mapping" : {
21 | "type": "ip",
22 | "index": "not_analyzed"
23 | }
24 | }
25 |
--------------------------------------------------------------------------------
/core/conf/feeds/hpHosts_wrzDOMAIN.conf:
--------------------------------------------------------------------------------
1 |
2 | [source]
3 | url=https://hosts-file.net/wrz.txt
4 | delimiter : \t
5 | beginWithBlank : False
6 | fields : ["localhost", "domain"]
7 | extraFields :
8 | description : This file contains warez/piracy sites listed in the hpHosts database This should ONLY be downloaded by those wanting to block warez/piracy sites and nothing else, and requires manual merging.
9 | startAt : 7
10 | score : -100
11 | coreIntelligence : domain
12 | validityDate:
13 | useByDate:
14 |
15 | [elasticsearch]
16 | typeIntel = host_file_wrzDOMAIN
17 |
18 | [intel]
19 | localhost = {
20 | "mapping" : {
21 | "type": "string",
22 | "index": "no"
23 | }
24 | }
25 | domain = {
26 | "mapping" : {
27 | "type": "string",
28 | "index": "not_analyzed"
29 | }
30 | }
31 |
--------------------------------------------------------------------------------
/core/static/app/sources/sources.html:
--------------------------------------------------------------------------------
1 |
2 |
23 |
--------------------------------------------------------------------------------
/core/conf/feeds/hpHosts_pupDOMAIN.conf:
--------------------------------------------------------------------------------
1 |
2 | [source]
3 | url=https://hosts-file.net/pup.txt
4 | delimiter : \t
5 | beginWithBlank : False
6 | fields : ["localhost", "domain"]
7 | extraFields :
8 | description : This file contains PUP sites listed in the hpHosts database This should ONLY be downloaded by those wanting to block PUP (Potentially Unwanted Programs) sites and nothing else, and requires manual merging.
9 | startAt : 7
10 | score : -100
11 | coreIntelligence : domain
12 | validityDate:
13 | useByDate:
14 |
15 | [elasticsearch]
16 | typeIntel = host_file_pupDOMAIN
17 |
18 | [intel]
19 | localhost = {
20 | "mapping" : {
21 | "type": "string",
22 | "index": "no"
23 | }
24 | }
25 | domain = {
26 | "mapping" : {
27 | "type": "string",
28 | "index": "not_analyzed"
29 | }
30 | }
31 |
--------------------------------------------------------------------------------
/core/conf/feeds/greensnow_IP.conf:
--------------------------------------------------------------------------------
1 |
2 | [source]
3 | url=http://blocklist.greensnow.co/greensnow.txt
4 | delimiter : ,
5 | beginWithBlank : False
6 | fields : ["ip"]
7 | extraFields :
8 | description : GreenSnow is a team consisting of the best specialists in computer security, we harvest a large number of IPs from different computers located around the world. GreenSnow is comparable with SpamHaus.org for attacks of any kind except for spam. Our list is updated automatically and you can withdraw at any time your IP address if it has been listed.
9 | startAt : 0
10 | score : -100
11 | coreIntelligence: ip
12 | validityDate:
13 | useByDate:
14 |
15 | [elasticsearch]
16 | typeIntel= greensnowIP
17 |
18 | [intel]
19 | ip = {
20 | "mapping" : {
21 | "type": "ip",
22 | "index": "not_analyzed"
23 | }
24 | }
25 |
26 |
--------------------------------------------------------------------------------
/core/services/typeIntel.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf8 -*-
3 |
4 |
5 | from modules.distinct.Field import Field
6 | import logging
7 | logger = logging.getLogger(__name__)
8 | def main():
9 | logger.info('typeIntel.main launched')
10 | try:
11 | response = dict()
12 | #type service is actually a distinct with the field "coreIntelligence"
13 | #that's why Field object is used
14 | allType = Field('coreIntelligence')
15 | allType.getDistinct()
16 | response['type'] = allType.distinctList
17 | logger.info('typeIntel.main end')
18 | return response
19 | except Exception as e:
20 | logger.error('typeIntel.main failed, no idea where it came from...', exc_info = True)
21 | report = dict()
22 | report['error'] = str(e)
23 | return report
24 | if __name__ == '__main__':
25 | main()
26 |
--------------------------------------------------------------------------------
/core/static/app/app.js:
--------------------------------------------------------------------------------
1 | (function() {
2 | // Create an application
3 | var app = angular.module('hippocampe',
4 | ['ui.router',
5 | 'chart.js',
6 | 'ui.bootstrap',
7 | 'main'
8 | ]);
9 |
10 | app.config(function($stateProvider, $urlRouterProvider) {
11 | $urlRouterProvider.otherwise('/home');
12 |
13 | // $stateProvider
14 | // .state('sources', {
15 | // url: "/sources",
16 | // templateUrl: "app/sources/sources.html",
17 | // controller: "app/sources/module.js"
18 | // })
19 | });
20 |
21 | app.filter('toArray', function() {
22 | return function(input) {
23 | var out = [];
24 | // check if input is array
25 | for (key in input) {
26 | var value = input[key];
27 | value["id"] = key;
28 | out.push(value);
29 | }
30 | return out;
31 | };
32 | });
33 |
34 | })();
35 |
--------------------------------------------------------------------------------
/core/conf/feeds/hpHosts_grmDOMAIN.conf:
--------------------------------------------------------------------------------
1 |
2 | [source]
3 | url=https://hosts-file.net/grm.txt
4 | delimiter : \t
5 | beginWithBlank : False
6 | fields : ["localhost", "domain"]
7 | extraFields :
8 | description : This file contains sites involved in spam (that do not otherwise meet any other classification criteria) listed in the hpHosts database This should ONLY be downloaded by those wanting to block spam sites and nothing else, and requires manual merging.
9 | startAt : 7
10 | score : -100
11 | coreIntelligence : domain
12 | validityDate:
13 | useByDate:
14 |
15 | [elasticsearch]
16 | typeIntel = host_file_grmDOMAIN
17 |
18 | [intel]
19 | localhost = {
20 | "mapping" : {
21 | "type": "string",
22 | "index": "no"
23 | }
24 | }
25 | domain = {
26 | "mapping" : {
27 | "type": "string",
28 | "index": "not_analyzed"
29 | }
30 | }
31 |
--------------------------------------------------------------------------------
/core/services/sizeBySources.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf8 -*-
3 |
4 |
5 | from modules.sizeBySources.TypeES import TypeES
6 | import sources
7 | import logging
8 | logger = logging.getLogger(__name__)
9 | def main():
10 | logger.info('sizeBySources.main launched')
11 | try:
12 | sourcesReport = sources.main()
13 | report = dict()
14 | for url, dictAttributes in sourcesReport.items():
15 | typeES = TypeES(dictAttributes['type'])
16 | report[url] = dict()
17 | report[url]['size'] = typeES.getSize()
18 | logger.info('sizeBySources.main end')
19 | return report
20 | except Exception as e:
21 | logger.error('distinct.main failed, no idea where it came from...', exc_info = True)
22 | response = dict()
23 | response['error'] = str(e)
24 | return response
25 |
26 | if __name__ == '__main__':
27 | main()
28 |
--------------------------------------------------------------------------------
/core/static/app/home/module.js:
--------------------------------------------------------------------------------
1 | (function() {
2 | 'use strict';
3 |
4 | var home = angular.module('home',
5 | ['ui.router',
6 | 'chart.js',
7 | 'sizeByType',
8 | 'sizeBySource',
9 | 'monitor'
10 | ]);
11 |
12 | home.config(function($stateProvider, $urlRouterProvider) {
13 | $stateProvider
14 | .state('main.home', {
15 | url: "/home",
16 | views: {
17 | "": { templateUrl: "app/home/home.html"},
18 | "monitor@main.home": {
19 | templateUrl: "app/monitor/monitor.html",
20 | controller: "MonitorCtrl"},
21 | "sizeByType@main.home": {
22 | templateUrl: "app/sizeByType/donutSizeByType.html",
23 | controller : "SizeByTypeCtrl"},
24 | "sizeBySource@main.home": {
25 | templateUrl: "app/sizeBySource/donutSizeBySource.html",
26 | controller: "SizeBySourceCtrl"}
27 | }
28 | })
29 | });
30 | })();
31 |
--------------------------------------------------------------------------------
/core/services/distinct.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf8 -*-
3 |
4 |
5 | from modules.distinct.Field import Field
6 | import logging
7 | logger = logging.getLogger(__name__)
8 | def main(jsonRequest):
9 | logger.info('distinct.main launched')
10 | response = dict()
11 | #jsonRequest looks like:
12 | #{
13 | # 'field': ['ip', 'domain', 'url']
14 | #}
15 | try:
16 | for field in jsonRequest['field']:
17 | logger.info('All %s requested', field)
18 | fieldToRequest = Field(field)
19 | fieldToRequest.getDistinct()
20 | response[field] = fieldToRequest.distinctList
21 | logger.info('distinct.main end')
22 | return response
23 | except Exception as e:
24 | logger.error('distinct.main failed, no idea where it came from...', exc_info = True)
25 | response['error'] = str(e)
26 | return response
27 | if __name__ == '__main__':
28 | main()
29 |
--------------------------------------------------------------------------------
/core/conf/feeds/abuseZeus_DOMAIN.conf:
--------------------------------------------------------------------------------
1 |
2 | [source]
3 | url=https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist
4 | delimiter : ,
5 | beginWithBlank : False
6 | fields : ["domain"]
7 | extraFields :
8 | description : This blocklist contains the same data as the ZeuS domain blocklist (BadDomains) but with the slight difference that it doesn't exclude hijacked websites (level 2). This means that this blocklist contains all domain names associated with ZeuS C&Cs which are currently being tracked by ZeuS Tracker. Hence this blocklist will likely cause some false positives.
9 | startAt : 5
10 | score : -100
11 | coreIntelligence : domain
12 | validityDate:
13 | useByDate:
14 |
15 | [elasticsearch]
16 | typeIntel= abuseFree_zeustrackerDOMAIN
17 |
18 | [intel]
19 | domain = {
20 | "mapping" : {
21 | "type": "string",
22 | "index": "not_analyzed"
23 | }
24 | }
25 |
26 |
27 |
--------------------------------------------------------------------------------
/core/services/lastQuery.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf8 -*-
3 |
4 |
5 | from modules.common.ES import getES
6 | import sources
7 | import logging
8 | logger = logging.getLogger(__name__)
9 | def main():
10 | logger.info('lastQuery.main launched')
11 | try:
12 | report = dict()
13 | #retrieving sources report
14 | sourcesReport = sources.main()
15 | #isolating lastQuery date for every type under hippocampe index
16 | for url, dictAttributes in sourcesReport.items():
17 | report[url] = dict()
18 | report[url]['lastQuery'] = dictAttributes['lastQuery']
19 | logger.info('lastQuery.main end')
20 | return report
21 | except Exception as e:
22 | logger.error('distinct.main failed, no idea where it came from...', exc_info = True)
23 | response = dict()
24 | response['error'] = str(e)
25 | return response
26 |
27 | if __name__ == '__main__':
28 | main()
29 |
--------------------------------------------------------------------------------
/core/conf/feeds/abuseZeus_IP.conf:
--------------------------------------------------------------------------------
1 |
2 | [source]
3 | url=https://zeustracker.abuse.ch/blocklist.php?download=ipblocklist
4 | delimiter : ,
5 | beginWithBlank : False
6 | fields : ["ip"]
7 | extraFields :
8 | description : This blocklist contains the same data as the ZeuS IP blocklist (BadIPs) but with the slight difference that it doesn't exclude hijacked websites (level 2) and free web hosting providers (level 3). This means that this blocklist contains all IPv4 addresses associated with ZeuS C&Cswhich are currently being tracked by ZeuS Tracker. Hence this blocklist will likely cause some false positives.
9 | startAt : 5
10 | score : -100
11 | coreIntelligence : ip
12 | validityDate:
13 | useByDate:
14 |
15 | [elasticsearch]
16 | typeIntel= abuseFree_zeustrackerIP
17 |
18 | [intel]
19 | ip = {
20 | "mapping" : {
21 | "type": "ip",
22 | "index": "not_analyzed"
23 | }
24 | }
25 |
26 |
27 |
28 |
--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
1 | ###################################################
2 | # Lightweight Hippocampe container
3 | ###################################################
4 |
5 | # Build the base from J8 Alpine
6 | FROM openjdk:8-jre-alpine
7 |
8 | RUN apk add --update --no-cache python \
9 | python-dev \
10 | py-pip \
11 | git \
12 | curl \
13 | nodejs \
14 | nodejs-npm
15 |
16 | RUN npm install -g bower
17 | RUN pip install --upgrade pip && \
18 | pip install apscheduler \
19 | Configparser \
20 | elasticsearch \
21 | flask \
22 | python-dateutil \
23 | requests \
24 | urllib3==1.23
25 |
26 | COPY ./core /opt/Hippocampe/core
27 | COPY docker-entrypoint.sh /
28 |
29 | RUN adduser hippo -D
30 | RUN chown -R hippo:hippo /opt/Hippocampe /docker-entrypoint.sh
31 |
32 | USER hippo
33 |
34 | RUN cd /opt/Hippocampe/core/static && bower install
35 |
36 | ENTRYPOINT /docker-entrypoint.sh
37 |
--------------------------------------------------------------------------------
/core/services/sizeByType.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf8 -*-
3 |
4 |
5 | from modules.countType.BagOfIntel import BagOfIntel
6 | import typeIntel
7 | import logging
8 | logger = logging.getLogger(__name__)
9 | def main():
10 | logger.info('sizeByType.main launched')
11 | try:
12 | report = dict()
13 | #retrieving all intelligences's types available
14 | typeReport = typeIntel.main()
15 | for typeIntelligence in typeReport['type']:
16 | bagOfIntel = BagOfIntel(typeIntelligence)
17 | size = bagOfIntel.getSize()
18 | report[typeIntelligence] = dict()
19 | report[typeIntelligence]['size'] = size
20 | logger.info('sizeByType.main end')
21 | return report
22 | except Exception as e:
23 | logger.error('sizeByType.main failed, no idea where it came from...', exc_info = True)
24 | response = dict()
25 | response['error'] = str(e)
26 | return response
27 |
28 | if __name__ == '__main__':
29 | main()
30 |
--------------------------------------------------------------------------------
/core/conf/feeds/abuseSSL_IP.conf:
--------------------------------------------------------------------------------
1 |
2 | [source]
3 | url=https://sslbl.abuse.ch/blacklist/sslipblacklist.csv
4 | delimiter : ,
5 | beginWithBlank : False
6 | fields : ["ip", "port", "type"]
7 | extraFields :
8 | description : The SSL IP Blacklist (CSV) contains all hosts (IP addresses) that SSLBL has seen in the past 30 days being associated with a malicious SSL certificate. The CSV contains IP address and port number of malicious SSL hosts.
9 | startAt : 9
10 | score : -100
11 | coreIntelligence : ip
12 | validityDate:
13 | useByDate:
14 |
15 | [elasticsearch]
16 | typeIntel= abuseFree_sslblIP
17 |
18 | [intel]
19 | ip = {
20 | "mapping" : {
21 | "type": "ip",
22 | "index": "not_analyzed"
23 | }
24 | }
25 |
26 | port = {
27 | "mapping" : {
28 | "type": "string",
29 | "index": "not_analyzed"
30 | }
31 | }
32 |
33 | type = {
34 | "mapping" : {
35 | "type": "string",
36 | "index": "not_analyzed"
37 | }
38 | }
39 |
40 |
41 |
42 |
43 |
44 |
--------------------------------------------------------------------------------
/docker-compose.yml:
--------------------------------------------------------------------------------
1 | version: '3.2'
2 |
3 | services:
4 | elasticsearch:
5 | image: docker.elastic.co/elasticsearch/elasticsearch:5.3.0
6 | container_name: hipposearch
7 | ports:
8 | - "127.0.0.1:9200:9200"
9 | environment:
10 | http.host: "0.0.0.0"
11 | transport.host: "127.0.0.1"
12 | xpack.security.enabled: "false"
13 | xpack.monitoring.enabled: "false"
14 | xpack.graph.enabled: "false"
15 | xpack.watcher.enabled: "false"
16 | bootstrap.memory_lock: "true"
17 | ulimits:
18 | memlock:
19 | soft: -1
20 | hard: -1
21 | nofile:
22 | soft: 65536
23 | hard: 65536
24 | networks:
25 | - hipponet
26 |
27 | hippocampe:
28 | build: .
29 | container_name: hippocampe
30 | ports:
31 | - "5000:5000"
32 | networks:
33 | - hipponet
34 | depends_on:
35 | - elasticsearch
36 |
37 | networks:
38 | hipponet:
39 | driver: bridge
40 |
--------------------------------------------------------------------------------
/core/static/app/type/module.js:
--------------------------------------------------------------------------------
1 | (function() {
2 | 'use strict';
3 |
4 | var type = angular.module('type', ['ui.router', "chart.js"]);
5 |
6 | type.controller('TypeCtrl', function($scope, TypeSrv) {
7 |
8 | $scope.listTypes = [];
9 | $scope.selectedType;
10 |
11 | //dropdown toggle stuff
12 | function init() {
13 | //retrieve type for dropdown toggle
14 | TypeSrv.list().then(
15 | function(response) {
16 | $scope.listTypes = response.data['type'];
17 | },
18 | function(rejection) {
19 | console.log(rejection.data);
20 | }
21 | );
22 | };
23 |
24 | // change dropdown toggle label according to selected type
25 | $scope.selectTypeMore = function(selectedType) {
26 | $scope.selectedType = selectedType;
27 | };
28 |
29 | init();
30 | });
31 |
32 | type.service('TypeSrv', function($http) {
33 | return {
34 | list: function() {
35 | return $http.get('hippocampe/api/v1.0/type');
36 | }
37 | };
38 | });
39 | })();
40 |
--------------------------------------------------------------------------------
/core/conf/feeds/bambenek_c2cIP.conf:
--------------------------------------------------------------------------------
1 |
2 | [source]
3 | url=http://osint.bambenekconsulting.com/feeds/c2-ipmasterlist.txt
4 | delimiter : ,
5 | beginWithBlank : False
6 | fields : ["ip", "used_by", "submission_datetime", "reference_url"]
7 | extraFields :
8 | description : Master Feed of known, active and non-sinkholed C&Cs IP addresses
9 | startAt : 1
10 | score : -100
11 | coreIntelligence : ip
12 | validityDate:
13 | useByDate:
14 |
15 | [elasticsearch]
16 | typeIntel= bambeneck_c2cIP
17 |
18 | [intel]
19 | ip = {
20 | "mapping" : {
21 | "type": "ip",
22 | "index": "not_analyzed"
23 | }
24 | }
25 |
26 | used_by = {
27 | "mapping" : {
28 | "type": "string",
29 | "index": "not_analyzed"
30 | }
31 | }
32 |
33 | submission_datetime = {
34 | "mapping" : {
35 | "type": "date",
36 | "format" : "yyyy-MM-dd HH:mm",
37 | "index": "no"
38 | }
39 | }
40 |
41 | reference_url = {
42 | "mapping" : {
43 | "type": "string",
44 | "index": "no"
45 | }
46 | }
47 |
48 |
--------------------------------------------------------------------------------
/core/conf/feeds/bambenek_dgaDOMAIN.conf:
--------------------------------------------------------------------------------
1 |
2 | [source]
3 | url=http://osint.bambenekconsulting.com/feeds/dga-feed.txt
4 | delimiter : ,
5 | beginWithBlank : False
6 | fields : ["domain", "used_by", "submission_datetime", "reference_url"]
7 | extraFields :
8 | description : Domain feed of known DGA domains from -2 to +3 days
9 | startAt : 1
10 | score : -100
11 | coreIntelligence : domain
12 | validityDate:
13 | useByDate:
14 |
15 | [elasticsearch]
16 | typeIntel= bambeneck_c2cDOMAIN
17 |
18 | [intel]
19 | domain = {
20 | "mapping" : {
21 | "type": "string",
22 | "index": "not_analyzed"
23 | }
24 | }
25 |
26 | used_by = {
27 | "mapping" : {
28 | "type": "string",
29 | "index": "not_analyzed"
30 | }
31 | }
32 |
33 | submission_datetime = {
34 | "mapping" : {
35 | "type": "date",
36 | "format" : "yyyy-MM-dd HH:mm",
37 | "index": "no"
38 | }
39 | }
40 |
41 | reference_url = {
42 | "mapping" : {
43 | "type": "string",
44 | "index": "no"
45 | }
46 | }
47 |
48 |
--------------------------------------------------------------------------------
/core/services/countType.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf8 -*-
3 |
4 |
5 | from modules.countType.BagOfIntel import BagOfIntel
6 | import typeIntel
7 | import logging
8 | logger = logging.getLogger(__name__)
9 | from pprint import pprint
10 | def main():
11 | #logger.info('countType.main launched')
12 | #try:
13 | report = dict()
14 | #retrieving all intelligences's types available
15 | typeReport = typeIntel.main()
16 | for typeIntelligence in typeReport['type']:
17 | bagOfIntel = BagOfIntel(typeIntelligence)
18 | size = bagOfIntel.getSize()
19 | print size
20 | report[typeIntelligence] = dict()
21 | report[typeIntelligence]['size'] = size
22 | logger.info('countType.main end')
23 | pprint(report)
24 | return report
25 | # except Exception as e:
26 | # logger.error('countType.main failed, no idea where it came from...', exc_info = True)
27 | # response = dict()
28 | # response['error'] = str(e)
29 | # return response
30 |
31 | if __name__ == '__main__':
32 | main()
33 |
--------------------------------------------------------------------------------
/core/conf/feeds/bambenek_c2cDOMAIN.conf:
--------------------------------------------------------------------------------
1 |
2 | [source]
3 | url=http://osint.bambenekconsulting.com/feeds/c2-dommasterlist.txt
4 | delimiter : ,
5 | beginWithBlank : False
6 | fields : ["domain", "used_by", "submission_datetime", "reference_url"]
7 | extraFields :
8 | description : Master Feed of known, active and non-sinkholed C&Cs domain names
9 | startAt : 1
10 | score : -100
11 | coreIntelligence : domain
12 | validityDate:
13 | useByDate:
14 |
15 | [elasticsearch]
16 | typeIntel= bambeneck_c2cDOMAIN
17 |
18 | [intel]
19 | domain = {
20 | "mapping" : {
21 | "type": "string",
22 | "index": "not_analyzed"
23 | }
24 | }
25 |
26 | used_by = {
27 | "mapping" : {
28 | "type": "string",
29 | "index": "not_analyzed"
30 | }
31 | }
32 |
33 | submission_datetime = {
34 | "mapping" : {
35 | "type": "date",
36 | "format" : "yyyy-MM-dd HH:mm",
37 | "index": "no"
38 | }
39 | }
40 |
41 | reference_url = {
42 | "mapping" : {
43 | "type": "string",
44 | "index": "no"
45 | }
46 | }
47 |
48 |
--------------------------------------------------------------------------------
/core/conf/feeds/ciarmylist_IP.conf:
--------------------------------------------------------------------------------
1 |
2 | [source]
3 | url=http://cinsscore.com/list/ci-badguys.txt
4 | delimiter : ,
5 | beginWithBlank : False
6 | fields : ["ip"]
7 | extraFields :
8 | description : The CI Army list is a subset of the CINS Active Threat Intelligence ruleset, and consists of IP addresses that meet two basic criteria: 1) The IP's recent Rogue Packet score factor is very poor, and 2) The InfoSec community has not yet identified the IP as malicious. We think this second factor is important: We don't want to waste peoples' time listing thousands of IPs that have already been placed on other reputation lists; our list is meant to supplement and enhance the InfoSec community's existing efforts by providing IPs that haven't been identified yet.
9 | startAt : 0
10 | score : -100
11 | coreIntelligence: ip
12 | validityDate:
13 | useByDate:
14 |
15 | [elasticsearch]
16 | typeIntel= ciArmyListIP
17 |
18 | [intel]
19 | ip = {
20 | "mapping" : {
21 | "type": "ip",
22 | "index": "not_analyzed"
23 | }
24 | }
25 |
26 |
--------------------------------------------------------------------------------
/core/conf/feeds/abuseFeodo_IP.conf:
--------------------------------------------------------------------------------
1 |
2 | [source]
3 | url=https://feodotracker.abuse.ch/blocklist/?download=ipblocklist
4 | delimiter : ,
5 | beginWithBlank : False
6 | fields : ["ip"]
7 | extraFields :
8 | description : The Feodo Tracker Feodo IP Blocklist contains IP addresses (IPv4) used as C&C communication channel by the Feodo Trojan. This lists contains two types of IP address: Feodo C&C servers used by version A, version C and version D of the Feodo Trojan (these IP addresses are usually compromised servers running an nginx daemon on port 8080 TCP or 7779 TCP that is acting as proxy, forwarding all traffic to a tier 2 proxy node) and Feodo C&C servers used by version B which are usually used for the exclusive purpose of hosting a Feodo C&C server.
9 | startAt : 7
10 | score : -100
11 | coreIntelligence: ip
12 | validityDate:
13 | useByDate:
14 |
15 | [elasticsearch]
16 | typeIntel= abuseFree_feodotrackerIP
17 |
18 | [intel]
19 | ip = {
20 | "mapping" : {
21 | "type": "ip",
22 | "index": "not_analyzed"
23 | }
24 | }
25 |
26 |
--------------------------------------------------------------------------------
/core/services/lastStatus.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf8 -*-
3 |
4 |
5 | from modules.jobs.BagOfJobs import BagOfJobs
6 | import logging
7 | logger = logging.getLogger(__name__)
8 |
9 | def main():
10 | logger.info('lastStatus.main launched')
11 | try:
12 | result = dict()
13 | bag = BagOfJobs()
14 | globalReport = bag.getLastJob()
15 | for confFileName, report in globalReport['report'].items():
16 | result[report['link']] = dict()
17 | if report['error']:
18 | #there's one or more errors
19 | result[report['link']]['lastStatus'] = 'NOK'
20 | elif report['nbIndex'] == 0 and report['nbUpdate'] == 0:
21 | #the feed might have not been downloaded
22 | result[report['link']]['lastStatus'] = 'NOK'
23 | else:
24 | result[report['link']]['lastStatus'] = 'OK'
25 | logger.info('lastStatus.main end')
26 | return result
27 | except Exception as e:
28 | logger.error('lastStatus.main failed, no idea where it came from...', exc_info = True)
29 | report = dict()
30 | report['error'] = str(e)
31 | return report
32 |
--------------------------------------------------------------------------------
/core/conf/feeds/shunlist_IP.conf:
--------------------------------------------------------------------------------
1 |
2 | [source]
3 | url=http://autoshun.org/files/shunlist.csv
4 | delimiter : ,
5 | beginWithBlank : False
6 | fields : ["ip", "dateOfRecord", "type"]
7 | extraFields :
8 | description : AutoShun is a Snort plugin that allows you to send your Snort IDS logs to a centralized server that will correlate attacks from your sensor logs with other snort sensors, honeypots, and mail filters from around the world. The input from your logs will be used to identify hostile address that are bots, worms, spam engines which we use to build a shun list.
9 | startAt : 1
10 | score : -100
11 | coreIntelligence : ip
12 | validityDate:
13 | useByDate:
14 |
15 | [elasticsearch]
16 | typeIntel= autoshunFree_shunlistIP
17 |
18 | [intel]
19 | ip = {
20 | "mapping" : {
21 | "type": "ip",
22 | "index": "not_analyzed"
23 | }
24 | }
25 |
26 | dateOfRecord = {
27 | "mapping" : {
28 | "type": "string",
29 | "index": "not_analyzed"
30 | }
31 | }
32 |
33 | type = {
34 | "mapping" : {
35 | "type": "string",
36 | "index": "not_analyzed"
37 | }
38 | }
39 |
--------------------------------------------------------------------------------
/core/static/app/sizeByType/module.js:
--------------------------------------------------------------------------------
1 | (function() {
2 | 'use strict';
3 |
4 | var sizeByType = angular.module('sizeByType', ['ui.router', "chart.js"]);
5 |
6 | sizeByType.controller('SizeByTypeCtrl', function($scope, SizeByTypeSrv) {
7 | $scope.labelsDonut = [];
8 | $scope.dataDonut = [];
9 |
10 | function getDonutData() {
11 | for (var intelType in $scope.reportSize){
12 | $scope.labelsDonut.push(intelType);
13 | $scope.dataDonut.push($scope.reportSize[intelType]["size"]);
14 | }
15 | };
16 |
17 |
18 | function processSizeByType() {
19 | SizeByTypeSrv.list().then(
20 | function(response) {
21 | $scope.reportSize = response.data;
22 | getDonutData();
23 | },
24 | function(rejection) {
25 | console.log(rejection.data);
26 | }
27 | );
28 | };
29 |
30 | processSizeByType();
31 | });
32 |
33 |
34 | sizeByType.service('SizeByTypeSrv', function($http) {
35 | return {
36 | list: function() {
37 | return $http.get('hippocampe/api/v1.0/sizeByType');
38 | }
39 | };
40 | });
41 |
42 | })();
43 |
--------------------------------------------------------------------------------
/core/static/app/home/home.html:
--------------------------------------------------------------------------------
1 |
3 | Available Sources
4 |
5 |
6 |
7 |
8 |
9 |
21 |
22 | | URL | 11 |{{columnName}} | 12 |
|---|---|
| {{url}} | 17 |{{value}} | 18 |
2 |
6 |
7 |
3 | 
4 |
5 |
4 |
8 |
32 |
33 |
9 |
19 |
20 |
10 |
18 |
11 | Size By Type
12 |
13 |
14 |
15 |
16 |
17 |
21 |
31 |
22 |
30 |
23 | Size By Source
24 |
25 |
26 |
27 |
28 |
29 |
34 |
46 |
--------------------------------------------------------------------------------
/core/static/app/sizeBySource/module.js:
--------------------------------------------------------------------------------
1 | (function() {
2 | 'use strict';
3 |
4 | var sizeBySource = angular.module('sizeBySource', ['ui.router', "chart.js"]);
5 |
6 | sizeBySource.controller('SizeBySourceCtrl', function($scope, SizeBySourceSrv) {
7 | $scope.labelsDonut = [];
8 | $scope.dataDonut = [];
9 |
10 | function getDonutData() {
11 | for (var url in $scope.reportSize){
12 | $scope.labelsDonut.push(url);
13 | $scope.dataDonut.push($scope.reportSize[url]["size"]);
14 | }
15 | };
16 |
17 |
18 | function processSizeBySource() {
19 | SizeBySourceSrv.list().then(
20 | function(response) {
21 | $scope.reportSize = response.data;
22 | getDonutData();
23 | },
24 | function(rejection) {
25 | console.log(rejection.data);
26 | }
27 | );
28 | };
29 |
30 | processSizeBySource();
31 | });
32 |
33 |
34 | sizeBySource.service('SizeBySourceSrv', function($http) {
35 | return {
36 | list: function() {
37 | return $http.get('hippocampe/api/v1.0/sizeBySources');
38 | }
39 | };
40 | });
41 |
42 | })();
43 |
--------------------------------------------------------------------------------
/core/static/app/hipposcore/hipposcore.html:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/.github/issue_template.md:
--------------------------------------------------------------------------------
1 | # EDIT THIS TITLE BEFORE POSTING. Use this template for bug reports. If you'd like to request a feature, please be as descriptive as possible and delete the template except the first section (Request Type)
2 |
3 | ### Request Type
4 | (select Bug or Feature Request and **remove this part**)
5 | Bug / Feature Request
6 |
7 | ### Work Environment
8 |
9 | | Question | Answer
10 | |---------------------------|--------------------
11 | | OS version (server) | Debian, Ubuntu, CentOS, RedHat, ...
12 | | OS version (client) | XP, Seven, 10, Ubuntu, ...
13 | | Hippocampe version / git hash | 1.x, hash of the commit
14 | | Package Type | Docker, Binary, From source
15 | | Browser type & version | If applicable
16 |
17 |
18 | ### Problem Description
19 | Describe the problem/bug as clearly as possible.
20 |
21 | ### Steps to Reproduce
22 | 1. step 1
23 | 1. step 2
24 | 1. step 3...
25 |
26 | ### Possible Solutions
27 | (keep this section if you have suggestions on how to solve the problem. **Otherwise delete it**)
28 |
29 | ### Complementary information
30 | (add anything that can help identifying the problem such as **logs**, **screenshots**, **configuration dumps** etc.)
31 |
--------------------------------------------------------------------------------
/core/services/modules/shadowbook/downloader.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf8 -*-
3 |
4 | import requests
5 | from copy import deepcopy
6 |
7 | def simpleDownload(url, listSessions):
8 | #listSessions is a list with authenticated sessions for feed which need to be authenticate
9 |
10 | #Before retrieving feed's data, we have to determine which session to use
11 | #by using listSessions[1:] we do not choose the first element which is the default session
12 | #in case that the top_level_url contains 'default'
13 | for session in listSessions[1:]:
14 | if session['sessionName'] in url:
15 | selectedSession = deepcopy(session)
16 | else:
17 | #the default session is always the first element in the list
18 | selectedSession = listSessions[0]
19 |
20 | raw_page = selectedSession['session'].get(url, stream = True)
21 | #requests does not raise exception if status is not 200 by default
22 | #that's why we use raise_for_status()
23 | raw_page.raise_for_status()
24 | #returning raw content and not text to avoid encoding issues
25 | return raw_page.content
26 |
27 | class EmptyFeedException(Exception):
28 | def __init__(self,url):
29 | Exception.__init__(self, 'Feed {0} is empty'.format(url))
30 | self.url = url
31 |
32 |
33 | if __name__ == '__main__':
34 | main()
35 |
--------------------------------------------------------------------------------
/core/conf/feeds/dnsbh_DOMAIN.conf:
--------------------------------------------------------------------------------
1 |
2 | [source]
3 | url : http://mirror1.malwaredomains.com/files/domains.txt
4 | delimiter : \t
5 | beginWithBlank : True
6 | fields : ["nextvalidation", "domain", "type", "original_reference-why_it_was_listed"]
7 | extraFields : extra
8 | description : DNS-BH - Malware Domain Blacklist. A list of domains that are known to be used to propagate malware are listed in Bind and Windows zone files. The domains are loaded onto an internal DNS server. When a computer requests a URL or file from one of these domains, a fake reply is sent, thus preventing many malware installs from occuring.
9 | startAt : 4
10 | score : -100
11 | coreIntelligence : domain
12 | validityDate:
13 | useByDate:
14 |
15 | [elasticsearch]
16 | typeIntel = malwaredomainsFree_dnsbhDOMAIN
17 |
18 | [intel]
19 | nextvalidation = {
20 | "mapping" : {
21 | "type": "string",
22 | "index": "not_analyzed"
23 | }
24 | }
25 |
26 |
27 | domain = {
28 | "mapping" : {
29 | "type": "string",
30 | "index": "not_analyzed"
31 | }
32 | }
33 |
34 |
35 | type = {
36 | "mapping" : {
37 | "type": "string",
38 | "index": "not_analyzed"
39 | }
40 | }
41 |
42 |
43 | original_reference-why_it_was_listed = {
44 | "mapping" : {
45 | "type": "string",
46 | "index": "not_analyzed"
47 | }
48 | }
49 |
50 |
51 | extra = {
52 | "mapping" : {
53 | "type": "string",
54 | "index": "not_analyzed"
55 | }
56 | }
57 |
--------------------------------------------------------------------------------
/core/services/hipposched.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf8 -*-
3 |
4 |
5 | from apscheduler.schedulers.background import BackgroundScheduler
6 | import shadowbook
7 | import logging
8 | logger = logging.getLogger(__name__)
9 | def main(request):
10 | logger.info('hipposched.main launched')
11 | try:
12 | jobId = 'shadowbook'
13 | time = request['time']
14 | logger.info('schedule submitted as: ' + str(time))
15 | #processing time to isolate each field
16 | tabTime = time.split(' ')
17 | #tabTime
18 | # [0] min
19 | # [1] hour
20 | # [2] day of month
21 | # [3] month
22 | # [4] day of week
23 | sched = BackgroundScheduler()
24 | #always erase the previous schedule
25 | #because of replace_existing = True
26 | #logger.info('creating job')
27 | sched.add_job(shadowbook.hipposchedVersion,
28 | 'cron',
29 | minute = tabTime[0],
30 | hour = tabTime[1],
31 | day = tabTime[2],
32 | month = tabTime[3],
33 | day_of_week = tabTime[4],
34 | replace_existing = True,
35 | id = jobId)
36 | sched.start()
37 | logger.info('job succesfully schedulled as: ' + str(time))
38 | report = dict()
39 | report['schedule'] = time
40 | logger.info('hipposched.main end')
41 | return report
42 | except Exception as e:
43 | logger.error('hipposched.main failed, no idea where it came from', exc_info = True)
44 | report = dict()
45 | report['error'] = str(e)
46 | return report
47 |
48 |
49 |
--------------------------------------------------------------------------------
/core/static/app/monitor/module.js:
--------------------------------------------------------------------------------
1 | (function() {
2 | 'use strict';
3 |
4 | var monitor = angular.module('monitor', ['ui.router', "chart.js"]);
5 |
6 | monitor.controller('MonitorCtrl', function($scope, MonitorSrv) {
7 | $scope.reportMonitor = {};
8 | $scope.reportSize = {};
9 | $scope.columnNames = [];
10 |
11 | function getColumnNames() {
12 | //get a key (which is an url)
13 | var someKey = Object.keys($scope.reportMonitor)[0];
14 |
15 | for (var columnName in $scope.reportMonitor[someKey]) {
16 | if($scope.reportMonitor[someKey].hasOwnProperty(columnName)) {
17 | $scope.columnNames.push(columnName);
18 | }
19 | }
20 | };
21 |
22 | function processMonitorSources() {
23 | //get monitorSource's report
24 | MonitorSrv.list().then(
25 | function(response) {
26 | $scope.reportMonitor = response.data;
27 | getColumnNames();
28 | },
29 | function(rejection) {
30 | console.log(rejection.data);
31 | }
32 | );
33 | };
34 |
35 |
36 | processMonitorSources();
37 | });
38 |
39 | monitor.service('MonitorSrv', function($http) {
40 | return {
41 | list: function() {
42 | return $http.get('hippocampe/api/v1.0/monitorSources');
43 | }
44 | };
45 | });
46 |
47 | })();
48 |
--------------------------------------------------------------------------------
/core/services/modules/sizeBySources/TypeES.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf8 -*-
3 |
4 |
5 | """
6 | This module contains ObjToIndex class.
7 | ======================================
8 | """
9 |
10 | import os, sys
11 | install_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../../'
12 | getESpath = install_dir + 'services/modules/common'
13 | sys.path.insert(0, getESpath)
14 | from ES import getES
15 | from getConf import getHippoConf
16 |
17 | class TypeES(object):
18 | """
19 | ObjToIndex class represents any object that can be indexed in elasticsearch.
20 |
21 | This class is kind of an abstract class, because is is the common base
22 | of Source and Ioc classes.
23 | """
24 |
25 | def __init__(self, typeNameES):
26 | """
27 | ObjToIndex class' constructor.
28 | """
29 | cfg = getHippoConf()
30 | self.es = getES()
31 | self.indexNameES = cfg.get('elasticsearch', 'indexNameES')
32 | self.typeNameES = typeNameES
33 | self.docSearch = dict()
34 | self.size = int()
35 |
36 | def forgeDocSearch(self):
37 | self.docSearch = {
38 | 'query' : {
39 | 'match_all' : {}
40 | }
41 | }
42 |
43 | def getSize(self):
44 | self.forgeDocSearch()
45 | res = self.es.count(index=self.indexNameES, doc_type=self.typeNameES, body = self.docSearch)
46 | #res looks like:
47 | #{u'_shards': {u'failed': 0, u'successful': 5, u'total': 5}, u'count': 45}
48 | self.size = res['count']
49 | return self.size
50 |
51 |
--------------------------------------------------------------------------------
/core/static/app/main/main.html:
--------------------------------------------------------------------------------
1 |
35 |
45 |
36 |
44 |
37 | Monitor Sources
38 |
39 |
40 |
41 |
42 |
43 |
2 |
33 |
34 |
35 |
36 |
--------------------------------------------------------------------------------
/core/conf/feeds/phishtank_URL.conf:
--------------------------------------------------------------------------------
1 |
2 | [source]
3 | url=http://data.phishtank.com/data/online-valid.csv
4 | delimiter : ,
5 | beginWithBlank : False
6 | fields : ["phish_id", "url", "phish_detail_url", "submission_time", "verified", "verification_time", "online", "target"]
7 | extraFields :
8 | description : PhishTank is a collaborative clearing house for data and information about phishing on the Internet
9 | startAt : 1
10 | score : -100
11 | coreIntelligence : url
12 | validityDate:
13 | useByDate:
14 |
15 | [elasticsearch]
16 | typeIntel= phishtankFree_onlinevalidURL
17 |
18 | [intel]
19 | phish_id = {
20 | "mapping" : {
21 | "type": "string",
22 | "index": "not_analyzed"
23 | }
24 | }
25 |
26 | url = {
27 | "mapping" : {
28 | "type": "string",
29 | "index": "not_analyzed"
30 | }
31 | }
32 |
33 | phish_detail_url = {
34 | "mapping" : {
35 | "type": "string",
36 | "index": "no"
37 | }
38 | }
39 |
40 | submission_time = {
41 | "mapping" : {
42 | "type": "date",
43 | "format" : "date_time_no_millis",
44 | "index": "no"
45 | }
46 | }
47 |
48 | verified = {
49 | "mapping" : {
50 | "type": "string",
51 | "index": "no"
52 | }
53 | }
54 |
55 | verification_time = {
56 | "mapping" : {
57 | "type": "date",
58 | "format" : "date_time_no_millis",
59 | "index": "no"
60 | }
61 | }
62 |
63 | online = {
64 | "mapping" : {
65 | "type": "string",
66 | "index": "no"
67 | }
68 | }
69 |
70 | target = {
71 | "mapping" : {
72 | "type": "string",
73 | "index": "no"
74 | }
75 | }
76 |
77 |
--------------------------------------------------------------------------------
/core/services/modules/hipposcore/ExistingSource.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf8 -*-
3 |
4 |
5 | from elasticsearch import Elasticsearch
6 | import os
7 | import sys
8 | install_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../'
9 | pathModule = install_dir + 'modules/ES'
10 | sys.path.insert(0, pathModule)
11 | from ES import getES
12 |
13 | class ExistingSource:
14 | """
15 | ExistingSource represents a source indexed in elasticsearch
16 | It is constructed with the id of the source document in elasticsearch
17 | """
18 | def __init__(self, idSource):
19 | self.idSource = idSource
20 | self.docMatch = str()
21 | self.matchResponse = str()
22 | self.scoreSource = float()
23 | self.es = getES()
24 |
25 | def forgeDocMatch(self):
26 | """
27 | Forging the document to search the source, according to its
28 | id in elasticsearch
29 | """
30 | self.docMatch = {
31 | 'query' : {
32 | 'bool' : {
33 | 'must' : [
34 | {'match' : {'_id' : self.idSource}}
35 | ]
36 | }
37 | }
38 | }
39 |
40 | def search(self):
41 | """
42 | Searching the document
43 | """
44 | self.matchResponse = self.es.search(body = self.docMatch)
45 | if self.matchResponse['hits']['total'] > 0 :
46 | return True
47 | elif self.matchResponse['hits']['total'] == 0:
48 | return False
49 |
50 | def processMatchResponse(self):
51 | """
52 | Retrieving the source's score
53 | """
54 | self.scoreSource = self.matchResponse['hits']['hits'][0]['_source']['score']
55 |
56 | def getScore(self):
57 | return self.scoreSource
58 |
--------------------------------------------------------------------------------
/core/static/app/sources/module.js:
--------------------------------------------------------------------------------
1 | (function() {
2 | 'use strict';
3 |
4 | var sources = angular.module('sources', ['ui.router', "chart.js"]);
5 |
6 | sources.controller('SourcesCtrl', function($scope, SourcesSrv) {
7 | $scope.reportSources = {};
8 | $scope.columnNames = [];
9 |
10 | function getColumnNames() {
11 | //get a key (which is an url)
12 | var someKey = Object.keys($scope.reportSources)[0];
13 |
14 | for (var columnName in $scope.reportSources[someKey]) {
15 | if($scope.reportSources[someKey].hasOwnProperty(columnName)) {
16 | $scope.columnNames.push(columnName);
17 | }
18 | }
19 | };
20 |
21 | function cleanReportSources(){
22 | //reportSources has duplicates info (URL and link)
23 | //deleting link
24 | for (var url in $scope.reportSources){
25 | delete $scope.reportSources[url]['link'];
26 | }
27 | };
28 |
29 | function processSources() {
30 | //get sourcesSource's report
31 | SourcesSrv.list().then(
32 | function(response) {
33 | $scope.reportSources = response.data;
34 | cleanReportSources();
35 | getColumnNames();
36 | },
37 | function(rejection) {
38 | console.log(rejection.data);
39 | }
40 | );
41 | };
42 |
43 | processSources();
44 | });
45 |
46 | sources.service('SourcesSrv', function($http) {
47 | return {
48 | list: function() {
49 | return $http.get('hippocampe/api/v1.0/sources');
50 | }
51 | };
52 | });
53 |
54 | })();
55 |
--------------------------------------------------------------------------------
/core/services/modules/shadowbook/objects/NewIntel.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf8 -*-
3 |
4 |
5 | """
6 | This module contains Ioc class.
7 | ===============================
8 | """
9 | from ObjToIndex import ObjToIndex
10 | from IndexNew import IndexNew
11 | import os, sys
12 | app_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../../../'
13 | getConfPath = app_dir + 'services/modules/common'
14 | sys.path.insert(0, getConfPath)
15 | from getConf import getHippoConf
16 |
17 | class NewIntel(ObjToIndex):
18 | """
19 |
20 | It inherits from ObjToIndex class.
21 | """
22 |
23 | def __init__(self, typeIntel, intelligence):
24 | """
25 | Ioc class' constructor.
26 |
27 | :param config: Config object that contains informations about the ioc.
28 | :type config: Config object instance.
29 | :param source: Ioc's Source object.
30 | :type source: Source object instance.
31 | """
32 |
33 | super(NewIntel, self).__init__()
34 | cfg = getHippoConf()
35 | self.indexNameES = cfg.get('elasticsearch', 'indexNameES')
36 | self.typeNameES = 'new'
37 | self.typeIntel = typeIntel
38 | self.intelligence = intelligence
39 |
40 | def forgeDocIndex(self):
41 | self.docIndex['typeIntel'] = self.typeIntel
42 | self.docIndex['intelligence'] = self.intelligence
43 |
44 | def indexNewIntel(self):
45 | index = IndexNew()
46 | index.createIndexNew()
47 | self.forgeDocIndex()
48 | self.indexInES()
49 |
--------------------------------------------------------------------------------
/core/services/monitorSources.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf8 -*-
3 |
4 |
5 | import lastQuery, schedReport, sizeBySources, freshness, lastStatus
6 | import logging
7 | logger = logging.getLogger(__name__)
8 | from copy import deepcopy
9 |
10 | def main():
11 | logger.info('monitorSources.main launched')
12 | try:
13 | lastQueryReport = lastQuery.main()
14 | sched_Report = schedReport.main()
15 | sizeBySourceReport = sizeBySources.main()
16 | freshnessReport = freshness.main()
17 | lastStatusReport = lastStatus.main()
18 |
19 | report = dict_merge(lastQueryReport, sched_Report)
20 | report = dict_merge(report, sizeBySourceReport)
21 | report = dict_merge(report, freshnessReport)
22 | report = dict_merge(report, lastStatusReport)
23 |
24 | logger.info('monitorSources.main end')
25 | return report
26 | except Exception as e:
27 | logger.error('monitorSources.main failed, no idea where it came from...', exc_info = True)
28 | response = dict()
29 | response['error'] = str(e)
30 | return response
31 |
32 | def dict_merge(a, b):
33 | '''recursively merges dict's. not just simple a['key'] = b['key'], if
34 | both a and bhave a key who's value is a dict then dict_merge is called
35 | on both values and the result stored in the returned dictionary.'''
36 | if not isinstance(b, dict):
37 | return b
38 | result = deepcopy(a)
39 | for k, v in b.iteritems():
40 | if k in result and isinstance(result[k], dict):
41 | result[k] = dict_merge(result[k], v)
42 | else:
43 | result[k] = deepcopy(v)
44 | return result
45 |
46 | if __name__ == '__main__':
47 | main()
48 |
--------------------------------------------------------------------------------
/core/static/app/shadowbook/module.js:
--------------------------------------------------------------------------------
1 | (function() {
2 | 'use strict';
3 |
4 | var shadowbook = angular.module('shadowbook', ['ui.router', 'ui.bootstrap'])
5 | .config(function(NotificationProvider) {
6 | NotificationProvider.setOptions({
7 | delay: 10000,
8 | startTop: 55,
9 | startRight: 10,
10 | verticalSpacing: 20,
11 | horizontalSpacing: 20,
12 | positionX: 'right',
13 | positionY: 'top'
14 | });
15 | });
16 |
17 | shadowbook.controller('ShadowbookCtrl', function($scope, ShadowbookSrv, Notification) {
18 | $scope.alerts;
19 | var response;
20 |
21 |
22 | $scope.shadowbook = function() {
23 | //clean out the alerts list, one alert at a time
24 | $scope.alerts = [];
25 |
26 | $scope.closeAlert = function(index) {
27 | $scope.alerts.splice(index, 1);
28 | };
29 |
30 | ShadowbookSrv.list().then(
31 | function(response) {
32 | //job launched with success
33 | response = JSON.parse(JSON.stringify(response.data));
34 | if (response.hasOwnProperty('job')) {
35 | var idJobList = Object.keys(response['job']);
36 | //there's only one element in idJobList => idJobList[0]
37 | var msg = 'Job '.concat(idJobList[0]).concat(' sucessfully launched');
38 | Notification.success({message: msg});
39 | }
40 | },
41 | function(rejection) {
42 | //one job already running
43 | var msg = rejection.data['error'];
44 | Notification.error({message: msg});
45 | }
46 | );
47 | };
48 |
49 | });
50 |
51 | shadowbook.service('ShadowbookSrv', function($http) {
52 | return {
53 | list: function() {
54 | return $http.get('hippocampe/api/v1.0/shadowbook');
55 | }
56 | };
57 | });
58 |
59 |
60 | })();
--------------------------------------------------------------------------------
/core/services/modules/shadowbook/parser.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf8 -*-
3 |
4 | import itertools
5 | import csv
6 | import os, sys
7 | install_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../../'
8 | #path to getConf function
9 | getConfPath = install_dir + 'services/modules/common'
10 | sys.path.insert(0, getConfPath)
11 | import getConf
12 | import ast
13 |
14 | def csvParser(feedPage, cfgPath):
15 |
16 | #retrieving parsing parameters
17 | conf = getConf.getConf(cfgPath)
18 |
19 | #start parsing from startAt
20 | startAt = conf.getint('source', 'startAt')
21 |
22 | #the line does not begin with '#' nor '/'
23 | parsedPage = itertools.ifilter(lambda x: len(x) > 1 and x[0] != '#' and x[0] != '/', feedPage.splitlines()[startAt:])
24 |
25 | #csv's delimiter, has to be str and not unicode
26 | delimiter = str(conf.get('source', 'delimiter'))
27 | #fields in the csv
28 | fields = conf.get('source', 'fields')
29 | #fields is string formated as a list, the instruction below convert it as a real list
30 | fields = [item.encode('ascii') for item in ast.literal_eval(fields)]
31 | #sometimes the first field is empty
32 | beginWithBlank = conf.getboolean('source', 'beginWithBlank')
33 | if beginWithBlank:
34 | #the feed parsed's first row is blank, to avoid discrepancy between fields and data
35 | #the first row is labelled as 'blank'
36 | fields.insert(0, 'blank')
37 | extraFields = conf.get('source', 'extraFields')
38 |
39 | #Can not use the variable delimiter as delimiter when it equals to '\t'
40 | #has to use dialect 'excel-tab'
41 | #Apparently, this is due to ConfigParser
42 | if delimiter != '\\t':
43 | parsedPage = csv.DictReader(parsedPage, fields, extraFields, delimiter = delimiter)
44 |
45 | if delimiter == '\\t':
46 | parsedPage = csv.DictReader(parsedPage, fields, extraFields, dialect = 'excel-tab')
47 |
48 | return parsedPage
49 |
50 | if __name__ == '__main__':
51 | main()
52 |
--------------------------------------------------------------------------------
/core/services/modules/shadowbook/objects/IndexJob.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf8 -*-
3 |
4 |
5 | from Index import Index
6 | import os, sys
7 | app_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../../../'
8 | getConfPath = app_dir + 'services/modules/common'
9 | sys.path.insert(0, getConfPath)
10 | from getConf import getHippoConf
11 | from itertools import izip_longest
12 | import ast
13 | import logging
14 |
15 | logger = logging.getLogger(__name__)
16 | class IndexJob(Index):
17 | """
18 | IndexIOC class represents an elasticsearch's index. And more precisely
19 | an index storing IOC.
20 |
21 | The class inherits from Index class.
22 | """
23 |
24 | # def __init__(self, cfgPath):
25 | def __init__(self):
26 |
27 | super(IndexJob, self).__init__()
28 | cfg = getHippoConf()
29 | self.indexNameES = cfg.get('elasticsearch', 'indexNameES')
30 | self.typeNameES = cfg.get('elasticsearch', 'typeNameESJobs')
31 |
32 | #the mapping has to be indicated at the creation of the index
33 | #the confMapping, is the mapping indicated in conf file for each field
34 | #it has to be parsed bellow
35 | self.confMapping = dict()
36 |
37 | def forgeDocMapping(self):
38 | """
39 | Forges the elasticsearch's mappings for IndexIOC.
40 | """
41 | #mapping for the meta-data
42 | self.docMapping = {
43 | self.typeNameES : {
44 | # "dynamic": "strict",
45 | "properties" : {
46 | # "report": {
47 | # "type": "date",
48 | # "index": "not_analyzed"
49 | # },
50 | "status": {
51 | "type": "keyword",
52 | "index": "not_analyzed"
53 | },
54 | "startTime": {
55 | "type": "date",
56 | "format": "basic_date_time_no_millis"
57 | },
58 | "endTime": {
59 | "type": "date",
60 | "format": "basic_date_time_no_millis"
61 | },
62 | "duration": {
63 | "type": "float"
64 | }
65 | }
66 | }
67 | }
68 |
69 | def createIndexJob(self):
70 | self.forgeDocMapping()
71 | self.create()
72 |
--------------------------------------------------------------------------------
/core/static/app/main/module.js:
--------------------------------------------------------------------------------
1 | (function() {
2 | // Create an application
3 | var main = angular.module('main',
4 | ['ui.router',
5 | 'chart.js',
6 | 'ui.bootstrap',
7 | 'home',
8 | 'sizeByType',
9 | 'sizeBySource',
10 | 'monitor',
11 | 'jobs',
12 | 'more',
13 | 'sources',
14 | 'type',
15 | 'hipposcore',
16 | 'shadowbook'
17 | ]);
18 |
19 | main.config(function($stateProvider, $urlRouterProvider) {
20 |
21 | $stateProvider
22 | .state('main', {
23 | views: {
24 | "": {templateUrl: 'app/main/main.html'},
25 | "hipposcore@main": {
26 | templateUrl: 'app/hipposcore/hipposcore.html',
27 | controller: 'HipposcoreCtrl'
28 | },
29 | "shadowbook@main": {
30 | templateUrl: 'app/shadowbook/shadowbook.html',
31 | controller: 'ShadowbookCtrl'
32 | }
33 | }
34 | })
35 | .state('main.more', {
36 | url: '/more',
37 | templateUrl: 'app/more/more.html',
38 | controller: 'MoreCtrl'
39 | })
40 | .state('main.jobs', {
41 | url: '/jobs',
42 | templateUrl: 'app/jobs/jobs.html',
43 | controller: 'JobsCtrl'
44 | })
45 | .state('main.sources', {
46 | url: '/sources',
47 | templateUrl: 'app/sources/sources.html',
48 | controller: 'SourcesCtrl'
49 | })
50 | // .state('main.hipposcore', {
51 | // url: '/hipposcore',
52 | // templateUrl: 'app/hipposcore/hipposcore.html',
53 | // controller: 'HipposcoreCtrl'
54 | // })
55 | // .state('main.shadowbook', {
56 | // url: '/shadowbook',
57 | // templateUrl: 'app/shadowbook/shadowbook.html',
58 | // controller: 'ShadowbookCtrl'
59 | // })
60 | });
61 |
62 | })();
--------------------------------------------------------------------------------
/core/services/modules/countType/BagOfIntel.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf8 -*-
3 |
4 |
5 | import os
6 | import sys
7 | app_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../../'
8 | pathModule = app_dir + 'services/modules/common'
9 | sys.path.insert(0, pathModule)
10 | from ES import getES
11 | from getConf import getHippoConf
12 |
13 | import logging
14 | logger = logging.getLogger(__name__)
15 | class BagOfIntel:
16 | def __init__(self, typeIntel):
17 |
18 | cfg = getHippoConf()
19 | self.typeIntel = typeIntel
20 | self.es = getES()
21 | self.docSearch = dict()
22 | self.indexNameES = cfg.get('elasticsearch', 'indexNameES')
23 | #contains every distinct value from a field
24 | self.size = int()
25 |
26 | def forgeDocSearch(self):
27 | #size=0 means "no limit to the number of terms"
28 | self.docSearch = {
29 | "aggs": {
30 | "distinct": {
31 | "cardinality": {
32 | "field": self.typeIntel
33 | }
34 | }
35 | }
36 | }
37 |
38 | def getSize(self):
39 | self.forgeDocSearch()
40 | res = self.es.search(index=self.indexNameES, body = self.docSearch)
41 | #res looks like:
42 | #{u'_shards': {u'failed': 0, u'successful': 5, u'total': 5},
43 | # u'aggregations': {u'distinct': {u'value': 3055}},
44 | # u'hits': {u'hits': [{u'_id': u'AVOpOpfHUw7jx-ihWUHM',
45 | # u'_index': u'hippocampe',
46 | # u'_score': 1.0,
47 | # u'_source': {u'firstAppearance': u'20160324T162502+0100',
48 | # u'idSource': u'AVOpOpBxUw7jx-ihWUGb',
49 | # u'ip': u'144.76.162.245',
50 | # u'lastAppearance': u'20160324T162502+0100',
51 | # u'source': u'https://zeustracker.abuse.ch/blocklist.php?download=ipblocklist'},
52 | # u'_type': u'abuseFree_zeustrackerIP'},
53 | self.size = res['aggregations']['distinct']['value']
54 | return self.size
55 |
--------------------------------------------------------------------------------
/core/services/modules/more/ObjToEnrich.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf8 -*-
3 |
4 | import os, sys
5 | app_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../'
6 | modulePath = app_dir + 'services/modules/common'
7 | sys.path.insert(0, modulePath)
8 | from ES import getES
9 | from getConf import getHippoConf
10 |
11 | class ObjToEnrich:
12 | def __init__(self, typeIntel, ioc):
13 | cfg = getHippoConf()
14 | self.typeIntel = typeIntel
15 | self.value = ioc
16 | self.docMatch = str()
17 | self.matchResponse = str()
18 | self.matchList = list()
19 | self.es = getES()
20 |
21 | #data stored in index hippocampe, so search is only in this index
22 | self.indexNameES = cfg.get('elasticsearch', 'indexNameES')
23 |
24 | def forgeDocMatch(self):
25 | self.docMatch = {
26 | 'query' : {
27 | 'bool' : {
28 | 'must' : [
29 | {'match' : {self.typeIntel : self.value}}
30 | ]
31 | }
32 | }
33 | }
34 |
35 | def forgeDocMatchInIpNetwork(self):
36 | self.docMatch = {
37 | "query" : {
38 | "bool" : {
39 | "must": [
40 | {"range" : {"ip.from" : {"from": "0.0.0.0", "to" : self.value}}},
41 | {"range" : {"ip.to" : {"from" : self.value, "to" : "255.255.255.255"}}}
42 | ]
43 | }
44 | }
45 | }
46 |
47 |
48 | def search(self):
49 | self.matchResponse = self.es.search(
50 | index = self.indexNameES,
51 | body = self.docMatch)
52 | if self.matchResponse['hits']['total'] > 0:
53 | return True
54 | elif self.matchResponse['hits']['total'] == 0:
55 | return False
56 |
57 | def processMatchResponse(self):
58 | for element in self.matchResponse['hits']['hits']:
59 | self.matchList.append(element['_source'])
60 |
61 | def getDetailsIP(self):
62 | self.forgeDocMatchInIpNetwork()
63 | if self.search():
64 | self.processMatchResponse()
65 | return self.matchList
66 | else:
67 | return []
68 |
69 | def getDetails(self):
70 | self.forgeDocMatch()
71 | if self.search():
72 | self.processMatchResponse()
73 | return self.matchList
74 | else:
75 | return []
76 |
--------------------------------------------------------------------------------
/core/templates/error.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | {{alert.msg}}
38 | {% endraw %}
39 |
51 |
52 |
53 |
54 |
--------------------------------------------------------------------------------
/core/services/modules/shadowbook/objects/Index.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf8 -*-
3 |
4 |
5 | """
6 | This module contains the class Index
7 | ====================================
8 | """
9 | from elasticsearch import client
10 | import os, sys
11 | install_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../../../'
12 | pathModule = install_dir + 'services/modules/common'
13 | sys.path.insert(0, pathModule)
14 | from ES import getES
15 | import logging
16 | logger = logging.getLogger(__name__)
17 | class Index(object):
18 | """
19 | Index class represents an elasticsearch's index.
20 |
21 | This class is kind of an abstract class, because it is the common base
22 | of IndexSource and IndexIOC classes.
23 | """
24 |
25 | def __init__(self):
26 | """
27 | Index class' constructor.
28 | """
29 |
30 | self.indexNameES = str()
31 | self.typeNameES = str()
32 | self.docMapping = dict()
33 | self.es = getES()
34 | #self.logger = logging.getLogger('app.shadowbook')
35 |
36 | def create(self):
37 | #create indexES instance
38 | indexES = client.IndicesClient(self.es)
39 | if(self.es.indices.exists(index = self.indexNameES)):
40 | #logger.info('index %s already exists', self.indexNameES)
41 | #index already exists but it does not mean that the type exists
42 | if(self.es.indices.exists_type(index = self.indexNameES, doc_type = [self.typeNameES])):
43 | #logger.info('type %s already exists', self.typeNameES)
44 | #type already exists nothing to do
45 | pass
46 | else:
47 | #type does not exists, creating it with the mapping to apply
48 | #logger.info('type %s does no exist, creating it', self.typeNameES)
49 | indexES.put_mapping(doc_type = self.typeNameES, body = self.docMapping)
50 | else:
51 | #index does not exists, neither type (type can't exist without index)
52 | #creating both
53 | #logger.info('index %s and type %s do not exist, creating them', self.indexNameES, self.typeNameES)
54 | indexES.create(index = self.indexNameES)
55 | #indicate mapping which applies only on index/type
56 | indexES.put_mapping(doc_type = self.typeNameES, body = self.docMapping)
57 |
--------------------------------------------------------------------------------
/core/services/modules/common/ES.py:
--------------------------------------------------------------------------------
1 | #/usr/bin/env python
2 | # -*- coding: utf8 -*-
3 |
4 |
5 |
6 | from elasticsearch import Elasticsearch
7 | from elasticsearch.client import IndicesClient
8 | from configparser import ConfigParser
9 | import os
10 | app_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../../'
11 | from getConf import getHippoConf
12 | import logging
13 | logger = logging.getLogger(__name__)
14 |
15 | def getES():
16 | confPath = app_dir + '/conf/hippo/hippo.conf'
17 | cfg = ConfigParser()
18 | cfg.read(confPath)
19 | host = cfg.get('elasticsearch', 'ip')
20 | port = cfg.getint('elasticsearch', 'port')
21 | ES = Elasticsearch([{'host': host, 'port' : port}], timeout = 60)
22 | return ES
23 |
24 | def checkES():
25 | logger.info('ES.checkES launched')
26 | try:
27 | ES = getES()
28 | return ES.ping()
29 | except Exception as e:
30 | logger.error('ES.checkES failed: %s', e, exc_info=True)
31 |
32 | def checkData(checkList):
33 | #checkList is the list of types to check
34 |
35 | #check if the hippocampe's index exists in ES
36 | #and check if ES type exists according to checkList
37 | logger.info('ES.checkData launched')
38 | logger.info(checkList)
39 | ES = getES()
40 | index = IndicesClient(ES)
41 |
42 | cfg = getHippoConf()
43 |
44 | indexName = cfg.get('elasticsearch', 'indexNameES')
45 | #references contains the name of types used in Hippocampe
46 | references = dict()
47 | references['sourceType'] = cfg.get('elasticsearch', 'typeNameESSource')
48 | references['newType'] = cfg.get('elasticsearch', 'typeNameESNew')
49 | references['jobsType'] = cfg.get('elasticsearch', 'typeNameESJobs')
50 |
51 | #listType = list()
52 | #listType.append(sourceType)
53 | #listType.append(newType)
54 | #listType.append(jobsType)
55 |
56 | #check index
57 | if index.exists(index = indexName):
58 | #check types
59 | for check in checkList:
60 | if index.exists_type(index = indexName, doc_type = references[check]):
61 | logger.info('index %s and type %s exist', indexName, references[check])
62 | else:
63 | logger.info('index %s exists but type %s does not', indexName, references[check] )
64 | return False
65 | return True
66 | else:
67 | logger.info('index %s does not exist', indexName)
68 | return False
69 |
--------------------------------------------------------------------------------
/core/static/app/jobs/module.js:
--------------------------------------------------------------------------------
1 | (function() {
2 | 'use strict';
3 |
4 | var jobs = angular.module('jobs', ['ui.router', 'chart.js', 'ui.bootstrap']);
5 |
6 | jobs.controller('JobsCtrl', function($scope, $log, JobsSrv) {
7 | $scope.reportJobsRAW = {};
8 | $scope.metadataRoot = [];
9 | $scope.metadataColumnNames = [];
10 | $scope.reportColumnNames = [];
11 | $scope.reportRoot = [];
12 |
13 |
14 | function getColumnNames() {
15 | //Process for metadata
16 | //get a key
17 | var someJobId = Object.keys($scope.metadataRoot)[0];
18 | var someConfFilename = Object.keys($scope.reportJobsRAW[someJobId]['report'])[0];
19 |
20 | for (var columnName in $scope.reportJobsRAW[someJobId]['report'][someConfFilename]) {
21 | if($scope.reportJobsRAW[someJobId]['report'][someConfFilename].hasOwnProperty(columnName)) {
22 | $scope.reportColumnNames.push(columnName);
23 | }
24 | }
25 |
26 | for (var columnName in $scope.metadataRoot[someJobId]) {
27 | if($scope.metadataRoot[someJobId].hasOwnProperty(columnName)) {
28 | $scope.metadataColumnNames.push(columnName);
29 | }
30 | }
31 | };
32 |
33 | function isolateMetadata() {
34 | /* copy reportJobsRAW into metadataRoot but without
35 | key value couple 'report'
36 | */
37 | $scope.metadataRoot = JSON.parse(JSON.stringify($scope.reportJobsRAW));
38 | for (var idJobs in $scope.metadataRoot) {
39 | delete $scope.metadataRoot[idJobs]['report'];
40 | }
41 | };
42 |
43 | function isolateReport() {
44 | //$scope.dictReport = JSON.parse(JSON.stringify($scope.reportJobsRAW));
45 | for (var idJobs in $scope.reportJobsRAW) {
46 | $scope.reportRoot[idJobs] = $scope.reportJobsRAW[idJobs]['report'];
47 | }
48 | };
49 |
50 | function processJobsSources() {
51 | //get jobs's report
52 | JobsSrv.list().then(
53 | function(response) {
54 | $scope.reportJobsRAW = response.data;
55 | isolateMetadata();
56 | getColumnNames();
57 | isolateReport();
58 | },
59 | function(rejection) {
60 | console.log(rejection.data);
61 | }
62 | );
63 | };
64 |
65 |
66 | processJobsSources();
67 | });
68 |
69 | jobs.service('JobsSrv', function($http) {
70 | return {
71 | list: function() {
72 | return $http.get('hippocampe/api/v1.0/jobs');
73 | }
74 | };
75 | });
76 |
77 | })();
78 |
--------------------------------------------------------------------------------
/core/services/modules/shadowbook/objects/IndexNew.py:
--------------------------------------------------------------------------------
1 | #@see The GNU Public License (GPL)
2 | #This file is part of Hippocampe.
3 | #
4 | #Hippocampe is free software; you can redistribute it and/or modify
5 | #it under the terms of the GNU General Public License as published by
6 | #the Free Software Foundation; either version 3 of the License, or
7 | #(at your option) any later version.
8 | #
9 | #Hippocampe is distributed in the hope that it will be useful, but
10 | #WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
11 | #or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 | #for more details.
13 | #
14 | #You should have received a copy of the GNU General Public License along
15 | #with Hippocampe; if not, write to the Free Software Foundation, Inc.,
16 | #59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 | #or see
2 | 
3 |
2 |
54 |
--------------------------------------------------------------------------------
/core/services/schedReport.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf8 -*-
3 |
4 |
5 | import sources
6 | import dateutil.parser
7 | import time
8 | from modules.common.getConf import getHippoConf
9 | import logging
10 | from configparser import NoOptionError, NoSectionError
11 | logger = logging.getLogger(__name__)
12 |
13 | def main():
14 | logger.info('schedReport.main launched')
15 |
16 | cfg = getHippoConf()
17 | #threshold defined in conf file
18 | #by default equals to 7
19 | #if a source is not querryed for 7 days or more, an alert is sent
20 |
21 | #JSON dict reporting data freshness for all sources
22 | report = dict()
23 |
24 | try:
25 | threshold = cfg.getint('schedReport', 'threshold')
26 |
27 | #retrieve all source
28 | dictSource = sources.main()
29 | #dictSource looks like:
30 | #{
31 | # "https://feodotracker.abuse.ch/blocklist/?download=ipblocklist": {
32 | # "coreIntelligence": "ip",
33 | # "description": "The Feodo Tracker Feodo IP Blocklist contains IP addresses (IPv4) used as C&C communication channel by the Feodo Trojan. This lists contains two types of IP address: Feodo C&C servers used by version A, version C and version D of the Feodo Trojan (these IP addresses are usually compromised servers running an nginx daemon on port 8080 TCP or 7779 TCP that is acting as proxy, forwarding all traffic to a tier 2 proxy node) and Feodo C&C servers used by version B which are usually used for the exclusive purpose of hosting a Feodo C&C server.",
34 | # "firstQuery": "20160222T233556+0100",
35 | # "lastQuery": "20160224T134833+0100",
36 | # "score": -100,
37 | # "source": "https://feodotracker.abuse.ch/blocklist/?download=ipblocklist",
38 | # "type": "source"
39 | # }
40 | #}
41 | if 'error' in dictSource:
42 | logger.error('sources.main for schedReport.main failed')
43 | return dictSource
44 |
45 | now = time.strftime("%Y%m%dT%H%M%S%z")
46 |
47 | #converting to dateutil
48 | now = dateutil.parser.parse(now)
49 | for url, dictData in dictSource.items():
50 | report[url] =dict()
51 | lastQuery =dictData['lastQuery']
52 | lastQuery = dateutil.parser.parse(lastQuery)
53 |
54 | #delta is the time difference in days between now and lastQuery
55 | delta = (now - lastQuery).days
56 | #converting in hours
57 | delta = delta * 24
58 | if (delta > threshold):
59 | report[url]['schedReport'] = 'NOK'
60 | else:
61 | report[url]['schedReport'] = 'OK'
62 | logger.info('schedReport.main end')
63 | return report
64 | except NoSectionError as e:
65 | logger.error('parsing hippo.conf failed', exc_info = True)
66 | report['error'] = str(e)
67 | except NoOptionError as e:
68 | logger.error('parsing hippo.conf failed', exc_info = True)
69 | report['error'] = str(e)
70 | except Exception as e:
71 | logger.error('schedReport.main failed, no idea where it came from...', exc_info = True)
72 | report['error'] = str(e)
73 |
74 | if __name__ == '__main__':
75 | main()
76 |
77 |
--------------------------------------------------------------------------------
/core/static/app/more/more.html:
--------------------------------------------------------------------------------
1 |
58 |
--------------------------------------------------------------------------------
/core/static/app/more/module.js:
--------------------------------------------------------------------------------
1 | (function() {
2 | 'use strict';
3 |
4 | var more = angular.module('more', ['ui.router', 'ui.bootstrap']);
5 |
6 | // more.config(function($stateProvider, $urlRouterProvider) {
7 | // $stateProvider
8 | // .state('main.more', {
9 | // url: '/more',
10 | // views: {
11 | // "": {
12 | // templateUrl: "app/more/more.html",
13 | // controller: "MoreCtrl"
14 | // },
15 | // "type@main.more": {
16 | // templateUrl: "app/type/type.html",
17 | // controller: "TypeCtrl"
18 | // }
19 | // }
20 | // })
21 | // });
22 |
23 | more.controller('MoreCtrl', function($scope, MoreSrv, TypeSrv) {
24 | $scope.list = [];
25 | $scope.textareaValue = '';
26 | $scope.observables = '';
27 | $scope.listObservables = [];
28 | $scope.listTypes = [];
29 | $scope.selectedType;
30 | $scope.moreResult = {};
31 |
32 | $scope.submit = function() {
33 |
34 | if ($scope.textareaValue) {
35 | $scope.observables=this.textareaValue;
36 | $scope.textareaValue = '';
37 | }
38 |
39 | $scope.listObservables = $scope.observables.split('\n');
40 |
41 | MoreSrv.list($scope.selectedType, $scope.listObservables).then(
42 | function(response) {
43 | //alert(JSON.stringify(response.data));
44 | $scope.moreResult = JSON.parse(JSON.stringify(response.data));
45 | },
46 | function(rejection) {
47 | console.log(rejection.data);
48 | }
49 | );
50 | };
51 |
52 | //dropdown toggle stuff
53 | function init() {
54 | //retrieve type for dropdown toggle
55 | TypeSrv.list().then(
56 | function(response) {
57 | $scope.listTypes = response.data['type'];
58 | },
59 | function(rejection) {
60 | console.log(rejection.data);
61 | }
62 | );
63 | };
64 |
65 | // change dropdown toggle label according to selected type
66 | $scope.selectTypeMore = function(selectedType) {
67 | $scope.selectedType = selectedType;
68 | };
69 |
70 | init();
71 | });
72 |
73 | more.service('MoreSrv', function($http) {
74 | return {
75 | list: function(selectedType, listObservables) {
76 | var requestBody = {};
77 | var subBody = {};
78 | subBody['type'] = selectedType;
79 | var observable;
80 |
81 | listObservables.forEach(function(observable) {
82 | requestBody[observable] = subBody;
83 | });
84 |
85 | //alert(JSON.stringify(requestBody));
86 | return $http.post('hippocampe/api/v1.0/more', requestBody);
87 | }
88 | };
89 | });
90 |
91 | more.service('TypeSrv', function($http) {
92 | return {
93 | list: function() {
94 | return $http.get('hippocampe/api/v1.0/type');
95 | }
96 | };
97 | });
98 |
99 | })();
100 |
--------------------------------------------------------------------------------
/core/services/hipposcore.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf8 -*-
3 |
4 |
5 | from time import strftime
6 | from modules.hipposcore.ExistingSource import ExistingSource
7 | import dateutil.parser
8 | from math import exp
9 | from werkzeug.contrib.cache import SimpleCache
10 | import logging
11 | logger = logging.getLogger(__name__)
12 | def calcHipposcore(dictResult):
13 | logger.info('hipposcore.calcHipposcore launched')
14 | try:
15 | T = 182.625
16 | n1 = float()
17 | n2 = float()
18 | n3 = float()
19 | #the final result
20 | hippodict = dict()
21 | #json object for hipposcore
22 | #{
23 | # 'ioc': {
24 | # 'hipposcore': -89
25 | # }
26 | #}
27 | scoredict = dict()
28 | now = strftime("%Y%m%dT%H%M%S%z")
29 | P = float()
30 | #a cache is used to speed up the source's score retrieve
31 | cache = SimpleCache()
32 |
33 | for ioc, listMatches in dictResult.items():
34 | #if there is no match for the ioc
35 | #listmaches will be empt, then hipposcore will be 0
36 | if not listMatches :
37 | hippodict[ioc] = dict()
38 | hippodict[ioc]['hipposcore'] = 0
39 | #match case
40 | else:
41 | P = 0.0
42 | for match in listMatches:
43 | #n1 retrieving the source according to its id
44 | idSource = match['idSource']
45 | n1 = cache.get(idSource)
46 | if n1 is None:
47 | source = ExistingSource(idSource)
48 | source.forgeDocMatch()
49 | if source.search():
50 | source.processMatchResponse()
51 | n1 = source.getScore()
52 | #the source's scor is between -100 and +100
53 | #for convenience,
54 | #in the formula it is required to be within -1 and +1
55 | n1 = n1 / 100.0
56 | cache.set(idSource, n1, timeout=3 * 60)
57 | #n2
58 | #rank is specific to alexa feed
59 | if 'rank' in match:
60 | n2 = match['rank']
61 | else:
62 | n2 = 1.0
63 | #n3
64 | #last time hippocampe saw the ioc
65 | lastAppearance = match['lastAppearance']
66 | #lastAppearance is a string, to calculate the time
67 | #difference, in other word the age of the ioc,
68 | #between lastAppearance and now, it is
69 | #needed to convert those strings to date
70 | lastAppearanceDate = dateutil.parser.parse(lastAppearance)
71 | nowDate = dateutil.parser.parse(now)
72 | #ioc's age in days
73 | age = (nowDate - lastAppearanceDate).days
74 | n3 = exp(-age / T)
75 | #P is the sum of n3 * n2 * n1 for every matches
76 | P = P + (n3 * n2 * n1)
77 | score = (1 - exp(-2 * abs(P))) * (abs(P) / P) * 100
78 | score = "%.2f" % score
79 | scoredict['hipposcore'] = score
80 | #scoredict['hipposcore'] = (1 - exp(-2 * abs(P))) * (abs(P) / P) * 100
81 | hippodict[ioc] = scoredict
82 | logger.info('hipposcore.main end')
83 | return hippodict
84 | except Exception as e:
85 | logger.error('hipposcore.calcHipposcore failed, no idea where it came from...', exc_info = True)
86 | report = dict()
87 | report['error'] = str(e)
88 | if __name__ == '__main__':
89 | calcHipposcore()
90 |
--------------------------------------------------------------------------------
/core/static/images/safari-pinned-tab.svg:
--------------------------------------------------------------------------------
1 |
2 |
4 |
47 |
--------------------------------------------------------------------------------
/core/static/app/hipposcore/module.js:
--------------------------------------------------------------------------------
1 | (function() {
2 | 'use strict';
3 |
4 | var hipposcore = angular.module('hipposcore', ['ui.router', 'ui-notification'])
5 | .config(function(NotificationProvider) {
6 | NotificationProvider.setOptions({
7 | delay: 10000,
8 | startTop: 55,
9 | startRight: 10,
10 | verticalSpacing: 20,
11 | horizontalSpacing: 20,
12 | positionX: 'right',
13 | positionY: 'top'
14 | });
15 | });
16 |
17 | hipposcore.controller('HipposcoreCtrl', function($scope, TypeSrv, HipposcoreSrv, Notification) {
18 | $scope.textValue;
19 | $scope.observable;
20 | $scope.listTypes = [];
21 | $scope.selectedType;
22 | $scope.score;
23 |
24 |
25 | $scope.closeAlert = function(index) {
26 | $scope.alerts.splice(index, 1);
27 | };
28 |
29 | $scope.hipposcore = function() {
30 | if ($scope.textValue) {
31 | $scope.observable = this.textValue;
32 | $scope.textValue = '';
33 | }
34 |
35 | HipposcoreSrv.list($scope.selectedType, $scope.observable).then(
36 | function(response) {
37 | $scope.score = response.data[$scope.observable]['hipposcore'];
38 |
39 | if ($scope.score === 0) {
40 | //var msg = $scope.selectedType.concat(': ').concat($scope.observable).concat(' Unknown');
41 | var msg = $scope.observable.concat(' Unknown');
42 | Notification.warning({message: msg});
43 | }
44 | else if ($scope.score > 0) {
45 | var msg = $scope.observable.concat(' : ').concat($scope.score);
46 | Notification.info({message: msg});
47 | }
48 | else if ($scope.score < 0) {
49 | var msg = $scope.observable.concat(' : ').concat($scope.score);
50 | Notification.error({message: msg});
51 | }
52 | },
53 | function(rejection) {
54 | console.log(rejection.data);
55 | }
56 | );
57 | };
58 |
59 | //dropdown toggle stuff
60 | function init() {
61 | //retrieve type for dropdown toggle
62 | TypeSrv.list().then(
63 | function(response) {
64 | $scope.listTypes = response.data['type'];
65 | },
66 | function(rejection) {
67 | console.log(rejection.data);
68 | }
69 | );
70 | };
71 |
72 | // change dropdown toggle label according to selected type
73 | $scope.selectTypeHipposcore = function(selectedType) {
74 | $scope.selectedType = selectedType;
75 | };
76 |
77 | init();
78 | });
79 |
80 | hipposcore.service('TypeSrv', function($http) {
81 | return {
82 | list: function() {
83 | return $http.get('hippocampe/api/v1.0/type');
84 | }
85 | };
86 | });
87 |
88 | hipposcore.service('HipposcoreSrv', function($http) {
89 | return {
90 | list: function(selectedType, observable) {
91 | var requestBody = {};
92 | var subBody = {};
93 | subBody['type'] = selectedType;
94 |
95 | requestBody[observable] = subBody;
96 |
97 | return $http.post('hippocampe/api/v1.0/hipposcore', requestBody);
98 | }
99 | };
100 | });
101 |
102 | })();
--------------------------------------------------------------------------------
/core/services/modules/new/BagOfNew.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf8 -*-
3 |
4 |
5 | import os
6 | import sys
7 | app_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../../'
8 | pathModule = app_dir + 'services/modules/common/ES'
9 | sys.path.insert(0, pathModule)
10 | from ES import getES
11 | from getConf import getHippoConf
12 |
13 | pathModule = app_dir + 'services'
14 | sys.path.insert(0, pathModule)
15 | import typeIntel
16 |
17 | class BagOfNew:
18 | def __init__(self):
19 | cfg = getHippoConf()
20 | self.docSearch = str()
21 | self.matchResponse = str()
22 | self.matchDict = dict()
23 | self.es = getES()
24 | self.indexNameES = cfg.get('elasticsearch', 'indexNameES')
25 | self.typeNameES = cfg.get('elasticsearch', 'typeNameESNew')
26 | self.nbDoc = int()
27 |
28 |
29 | def forgeDocSearch(self):
30 | self.docSearch = {
31 | 'query' : {
32 | 'match_all' : {}
33 | }
34 | }
35 |
36 | def forgeDocSearchAll(self):
37 | self.docSearch = {
38 | 'size': self.nbDoc,
39 | 'query' : {
40 | 'match_all' : {}
41 | }
42 | }
43 |
44 | def search(self):
45 | self.matchResponse = self.es.search(
46 | body = self.docSearch, index= self.indexNameES,
47 | doc_type = self.typeNameES)
48 | if self.matchResponse['hits']['total'] > 0:
49 | return True
50 | elif self.matchResponse['hits']['total'] == 0:
51 | return False
52 |
53 | def processMatchResponse(self):
54 | #{u'_shards': {u'failed': 0, u'successful': 5, u'total': 5},
55 | # u'hits': {u'hits': [{u'_id': u'AVMLH_9WhQvX6w62oQJL',
56 | # u'_index': u'hippocampe',
57 | # u'_score': 1.0,
58 | # u'_source': {u'intelligence': u'113.53.234.218',
59 | # u'originalId': u'AVMLH_9LhQvX6w62oQJK',
60 | # u'typeIoc': u'ip'},
61 | # u'_type': u'new'},
62 | # {u'_id': u'AVMLH_ynhQvX6w62oQIj',
63 | # u'_index': u'hippocampe',
64 | # u'_score': 1.0,
65 | # u'_source': {u'intelligence': u'google.com',
66 | # u'originalId': u'AVMLH_yZhQvX6w62oQIi',
67 | # u'typeIoc': u'domain'},
68 |
69 | for element in self.matchResponse['hits']['hits']:
70 | self.matchDict[element['_id']] = element['_source']
71 |
72 | def getMatchDict(self):
73 | #because elasticsearch only returns 10 results
74 | #w have to search in two phase, first determine the nb of docs
75 | #and then retrieve all docs
76 | #matchAll query
77 | self.forgeDocSearch()
78 | #retrieve nb doc
79 | res = self.es.count(index=self.indexNameES, doc_type=self.typeNameES, body = self.docSearch)
80 | #res looks like:
81 | #{u'_shards': {u'failed': 0, u'successful': 5, u'total': 5}, u'count': 45}
82 | self.nbDoc = res['count']
83 | self.forgeDocSearchAll()
84 | self.search()
85 | self.processMatchResponse()
86 | return self.matchDict
87 |
--------------------------------------------------------------------------------
/core/services/modules/jobs/BagOfJobs.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf8 -*-
3 |
4 |
5 | import os
6 | import sys
7 | app_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../../'
8 | pathModule = app_dir + 'services/modules/common'
9 | sys.path.insert(0, pathModule)
10 | from ES import getES
11 | from getConf import getHippoConf
12 |
13 | class BagOfJobs:
14 | """
15 | class that will retrieve all jobs
16 | it seems that a match all query and a scan scroll does not work
17 | together, that's why a match all query is used
18 | however only 10 matches are returned by elasticsearch.
19 | To go beyond that number, the number of document in source index
20 | is determined and indicated in the match all query
21 | """
22 | def __init__(self):
23 | cfg = getHippoConf()
24 | self.docSearch = str()
25 | self.matchResponse = str()
26 | self.matchDict = dict()
27 | self.es = getES()
28 | self.indexNameES = cfg.get('elasticsearch', 'indexNameES')
29 | self.typeNameES = cfg.get('elasticsearch', 'typeNameESJobs')
30 | self.nbDoc = int()
31 |
32 |
33 | def forgeDocSearch(self):
34 | self.docSearch = {
35 | 'query' : {
36 | 'match_all' : {}
37 | }
38 | }
39 |
40 | def forgeDocSearchAll(self):
41 | self.docSearch = {
42 | 'size': self.nbDoc,
43 | 'query' : {
44 | 'match_all' : {}
45 | }
46 | }
47 |
48 | def forgeDocSearchLastJob(self):
49 | self.docSearch = {
50 | 'query' : {
51 | 'bool' : {
52 | 'must': [
53 | {
54 | 'match': {
55 | 'status': 'done'
56 | }
57 | }
58 | ]
59 | }
60 | },
61 | 'size' : 1,
62 | 'sort' : [
63 | {
64 | 'endTime': {
65 | 'order': 'desc'
66 | }
67 | }
68 | ]
69 | }
70 |
71 |
72 | def search(self):
73 | self.matchResponse = self.es.search(
74 | body = self.docSearch, index= self.indexNameES,
75 | doc_type = self.typeNameES)
76 | if self.matchResponse['hits']['total'] > 0:
77 | return True
78 | elif self.matchResponse['hits']['total'] == 0:
79 | return False
80 |
81 | def processMatchResponse(self):
82 | #the match all query returns all documents, however there is
83 | #no key, only value
84 | #so the field source is setted as key and the all document
85 | #as value
86 | for element in self.matchResponse['hits']['hits']:
87 | self.matchDict[element['_id']] = element['_source']
88 |
89 | def getMatchList(self):
90 | #because elasticsearch only returns 10 results
91 | #w have to search in two phase, first determine the nb of docs
92 | #and then retrieve all docs
93 | #matchAll query
94 | self.forgeDocSearch()
95 | #retrieve nb doc
96 | res = self.es.count(index=self.indexNameES, doc_type=self.typeNameES, body = self.docSearch)
97 | #res looks like:
98 | #{u'_shards': {u'failed': 0, u'successful': 5, u'total': 5}, u'count': 45}
99 | self.nbDoc = res['count']
100 | self.forgeDocSearchAll()
101 | self.search()
102 | self.processMatchResponse()
103 | return self.matchDict
104 |
105 | def getLastJob(self):
106 | self.forgeDocSearchLastJob()
107 | self.search()
108 | report = self.matchResponse['hits']['hits'][0]['_source']
109 | return report
110 |
--------------------------------------------------------------------------------
/core/services/modules/shadowbook/objects/Job.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf8 -*-
3 |
4 |
5 | """
6 | This module contains the class Source.
7 | ======================================
8 | """
9 | from ObjToIndex import ObjToIndex
10 | import os, sys
11 | install_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../../../'
12 | getConfPath = install_dir + 'services/modules/common'
13 | sys.path.insert(0, getConfPath)
14 | from getConf import getHippoConf
15 | from IndexSource import IndexSource
16 | from time import strftime
17 | import logging
18 | logger = logging.getLogger(__name__)
19 | import dateutil.parser
20 |
21 | class Job(ObjToIndex):
22 |
23 | def __init__(self):
24 | super(Job, self).__init__()
25 | cfg = getHippoConf()
26 | self.indexNameES = cfg.get('elasticsearch', 'indexNameES')
27 | self.typeNameES = cfg.get('elasticsearch', 'typeNameESJobs')
28 | self.status = str()
29 | self.startTime = str()
30 | self.endTime = str()
31 | self.duration = str()
32 | self.report = dict()
33 |
34 | def forgeDocSearchOngoing(self):
35 | """
36 | Forges the search query to search source in elasticsearch.
37 | """
38 | self.docSearch = {
39 | "query": {
40 | "bool": {
41 | "must": [
42 | {
43 | "match": {
44 | "status": "ongoing"
45 | }
46 | }
47 | ]
48 | }
49 | }
50 | }
51 |
52 |
53 | def forgeDocIndexOngoing(self):
54 | self.docIndex = {
55 | "status": "ongoing",
56 | "startTime": strftime("%Y%m%dT%H%M%S%z"),
57 |
58 | }
59 |
60 | def forgeDocUpdateJob(self):
61 | self.docUpdate = {
62 | "script": {
63 | "lang": "painless",
64 | "inline": "ctx._source.status = params.status; ctx._source.endTime = params.endTime; ctx._source.duration = params.duration; ctx._source.report = params.report",
65 | "params": {
66 | "status": self.status,
67 | "endTime": self.endTime,
68 | "duration": self.duration,
69 | "report": self.report,
70 | }
71 | }
72 | }
73 |
74 | def searchOngoingJob(self):
75 | self.forgeDocSearchOngoing()
76 | nbOngoingJob = self.search()
77 | return nbOngoingJob
78 |
79 | def indexOngoingJob(self):
80 | self.forgeDocIndexOngoing()
81 | self.indexInES()
82 | return self.idInES
83 |
84 | def getStartTime(self):
85 | self.searchOngoingJob()
86 | #self.resSearch looks like:
87 | #{u'hits': {u'hits': [{u'_score': 11.907724, u'_type': u'jobs', u'_id': u'AVQkDgNs5CYfHJgxlSLE', u'_source': {u'status': u'ongoing', u'startTime': u'20160417T134938+0200'}, u'_index': u'hippocampe'}], u'total': 1, u'max_score': 11.907724}, u'_shards': {u'successful': 5, u'failed': 0, u'total': 5}, u'took': 1, u'timed_out': False}
88 | startTime = self.resSearch['hits']['hits'][0]['_source']['startTime']
89 | return startTime
90 |
91 | def calcDuration(self):
92 | self.startTime = self.getStartTime()
93 | duration = dateutil.parser.parse(self.endTime) - dateutil.parser.parse(self.startTime)
94 | #duration in seconds
95 | duration = duration.total_seconds()
96 | #duration in minutes
97 | duration = duration / 60.0
98 | #limiting to 2 digits
99 | self.duration = "%.2f" % duration
100 |
101 | def updateStatus(self, report, status):
102 | self.endTime = strftime("%Y%m%dT%H%M%S%z")
103 | self.status = status
104 | self.report = report
105 |
106 | self.calcDuration()
107 | self.forgeDocUpdateJob()
108 | self.update()
109 |
--------------------------------------------------------------------------------
/core/services/modules/distinct/Field.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf8 -*-
3 |
4 |
5 | import os
6 | import sys
7 | app_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../'
8 | pathModule = app_dir + 'services/modules/common'
9 | sys.path.insert(0, pathModule)
10 | from ES import getES
11 | from getConf import getHippoConf
12 |
13 | import logging
14 | logger = logging.getLogger(__name__)
15 | class Field:
16 | def __init__(self, field):
17 | cfg = getHippoConf()
18 | self.field = field
19 | self.es = getES()
20 | self.docSearch = dict()
21 | self.matchResponse = dict()
22 | self.indexName = cfg.get('elasticsearch', 'indexNameES')
23 | #contains every distinct value from a field
24 | self.distinctList = list()
25 | #number of distinct values
26 | self.size = int()
27 |
28 | def forgeDocSearch(self):
29 | self.docSearch = {
30 | "size": 0,
31 | "aggs": {
32 | "distinct": {
33 | "terms": {
34 | "field": self.field,
35 | "size": self.size
36 | }
37 | }
38 | }
39 | }
40 |
41 |
42 |
43 | def forgeDocSearchSize(self):
44 | self.docSearch = {
45 | "query": {
46 | "bool": {
47 | "must": [
48 | {"exists": {"field": self.field}}
49 | ]
50 | }
51 | },
52 | "aggs": {
53 | "distinct": {
54 | "cardinality": {
55 | "field": self.field,
56 | }
57 | }
58 | }
59 | }
60 |
61 |
62 | def search(self):
63 | #request_timeout value to avoid ConnectionTimeout exception
64 | self.matchResponse = self.es.search(
65 | index = self.indexName,
66 | body = self.docSearch,
67 | request_timeout=60)
68 |
69 | def processMatchResponse(self):
70 | '''
71 | raw matchResponse looks like:
72 |
73 | {u'_shards': {u'failed': 0, u'successful': 5, u'total': 5},
74 | u'aggregations': {u'distinct': {u'buckets': [{u'doc_count': 1,
75 | u'key': u'192.168.1.67'}],
76 | u'doc_count_error_upper_bound': 0,
77 | u'sum_other_doc_count': 0}},
78 | u'hits': {u'hits': [], u'max_score': 0.0, u'total': 4},
79 | u'timed_out': False,
80 | u'took': 9}
81 |
82 | '''
83 |
84 | for element in self.matchResponse['aggregations']['distinct']['buckets']:
85 | self.distinctList.append(element['key'])
86 |
87 | def getSize(self):
88 | self.forgeDocSearchSize()
89 | self.search()
90 | '''
91 | matchResponse looks like:
92 |
93 | {u'_shards': {u'failed': 0, u'successful': 5, u'total': 5},
94 | u'aggregations': {u'distinct': {u'value': 446}},
95 | u'hits': {u'hits': [{u'_id': u'AVmJp90mcUY7Hd0SG5cP',
96 | u'_index': u'hippocampe',
97 | u'_score': 1.0,
98 | u'_source': {u'domain': u'03bbec4.netsolhost.com',
99 | u'firstAppearance': u'20170110T193316+0100',
100 | u'idSource': u'AVmJp9c2cUY7Hd0SG5VN',
101 | u'lastAppearance': u'20170110T193316+0100',
102 | u'source': u''},
103 | '''
104 | self.size = self.matchResponse['aggregations']['distinct']['value']
105 |
106 |
107 |
108 | def getDistinct(self):
109 | #since Elasticsearch does not allow the "size=0" parameters
110 | #we must get the number of all distinct doc according a specific field
111 | self.getSize()
112 | #then we execute the search with the exact size
113 | self.forgeDocSearch()
114 | self.search()
115 | self.processMatchResponse()
116 |
--------------------------------------------------------------------------------
/core/services/modules/shadowbook/processFeed.py.backup:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf8 -*-
3 |
4 | import os
5 | from objects.Source import Source
6 | from downloader import simpleDownload
7 | import requests
8 | from parser import csvParser
9 | from configparser import NoOptionError, NoSectionError
10 | from objects.Intel import Intel
11 | import logging
12 | logger = logging.getLogger(__name__)
13 |
14 | def main(listSessions, cfgPath):
15 | confName = os.path.basename(os.path.normpath(cfgPath))
16 |
17 | logger.info('processFeed.main launched for %s', confName)
18 | #the function returns a report concerning the feed process
19 | #the instructions below are used to build the report
20 | report = dict()
21 | report[confName] = dict()
22 | report[confName]['error'] = list()
23 |
24 | try:
25 | source = Source(cfgPath)
26 | #index the source's data into ES if it does not already exist
27 | #if it does, update the lastQuery date and make sure that the source's score in ES
28 | #equals the source's score in conf file
29 | #conf file's score is the reference and will always be pushed to ES
30 | source.indexSourceInES()
31 |
32 | try:
33 | #downloading feed with authenticated sessions
34 | #streaming mode
35 | feedPage = simpleDownload(source.source, listSessions)
36 | nbFail = 0
37 | nbSuccess = 0
38 | try:
39 | #the cfgPath is given in parameters because it contains the delimiter
40 | #the function will return a csvDictReader
41 | parsedPage = csvParser(feedPage, cfgPath)
42 | parsedPage = list(parsedPage)
43 | print parsedPage[0]
44 | #indexing information from the parsedLine in elasticsearch
45 | #if it does not exist, creates it
46 | #and creates it in the index/type: hippocampe/new
47 | #if it does, update the lastAppearance date
48 | for intelligence in parsedPage:
49 | #intelligence looks like
50 | #{'domain': 'www.galliagroup.com', 'original_reference-why_it_was_listed': 'app.webinspector.com', u'extra': ['20160304', '20160113', '20140817', '20140322', '20131006', '', '20140624'], 'nextvalidation': '20160401', 'blank': '', 'type': 'highrisk'}
51 | try:
52 | intel = Intel(intelligence, cfgPath, source.idInES)
53 | intel.indexIntelInES()
54 | nbSuccess += 1
55 | except Exception as e:
56 | logger.error('Indexing %s failed', intelligence, exc_info = True)
57 | report[confName]['error'].append(str(e))
58 | nbFail += 1
59 | except NoSectionError as e:
60 | logger.error('fail in parsing %s for the parser', confName, exc_info = True)
61 | report[confName]['error'].append(str(e))
62 | except NoOptionError as e:
63 | logger.error('fail in parsing %s file for the parser', confName, exc_info = True)
64 | report[confName]['error'].append(str(e))
65 | except requests.exceptions.RequestException as e:
66 | logger.error('downloading %s failed', source.source, exc_info = True)
67 | #filling the report
68 | report[confName]['error'].append(str(e))
69 | except NoSectionError as e:
70 | logger.error('fail in parsing %s for the source', confName, exc_info = True)
71 | report[confName]['error'].append(str(e))
72 | except NoOptionError as e:
73 | logger.error('fail in parsing %s for the source', confName, exc_info = True)
74 | report[confName]['error'].append(str(e))
75 | except Exception as e:
76 | logger.error('processFeed.main failed for %s, no idea where it came from', confName, exc_info = True)
77 |
78 | report[confName]['nbSucess'] = nbSuccess
79 | report[confName]['nbFail'] = nbFail
80 |
81 | #the report looks like
82 | #{'dnsbh_DOMAIN.conf': {'error': [], 'nbFail': 0, 'nbSucess': 13921}}
83 | logger.info('processFeed.main for %s end', confName)
84 | return report
85 |
86 | if __name__ == '__main__':
87 | main()
88 |
--------------------------------------------------------------------------------
/core/services/modules/shadowbook/objects/Intel.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf8 -*-
3 |
4 | from ObjToIndex import ObjToIndex
5 | import os, sys
6 | install_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../../../'
7 | getConfPath = install_dir + 'services/modules/common'
8 | sys.path.insert(0, getConfPath)
9 | import getConf
10 | from IndexIntel import IndexIntel
11 | from time import strftime
12 | from NewIntel import NewIntel
13 | class Intel(ObjToIndex):
14 | def __init__(self, intelligence, cfgPath, idSource):
15 |
16 | #intelligence is a dict:
17 | #{'blank': '',
18 | # 'domain': 'uploadersonline.com',
19 | # u'extra': ['20160229'],
20 | # 'nextvalidation': '',
21 | # 'original_reference-why_it_was_listed': 'zeustracker.abuse.ch',
22 | # 'type': 'zeus'}
23 | super(Intel, self).__init__()
24 | self.cfgPath = cfgPath
25 | conf = getConf.getConf(cfgPath)
26 | self.indexNameES = conf.get('elasticsearch', 'indexIOC')
27 | self.typeNameES = conf.get('elasticsearch', 'typeIntel')
28 | self.idSource = idSource
29 | self.firstAppearance = str()
30 | self.lastAppearance = str()
31 | self.coreIntelligence = conf.get('source', 'coreIntelligence')
32 | self.dictData = intelligence
33 | self.source= conf.get('source', 'url')
34 |
35 | def indexIntelInES(self):
36 | #create the index
37 | index = IndexIntel(self.cfgPath)
38 | index.buildConfMapping()
39 | index.forgeDocMapping()
40 | index.create()
41 | #search the coreIntelligence in the index/type ONLY
42 | nbMatch = self.littleSearch()
43 | if nbMatch > 0:
44 | #match, so update last appearance date
45 | self.updateIntel()
46 | elif nbMatch == 0:
47 | #no match, so index
48 | self.forgeDocIndex()
49 | self.indexInES()
50 | #at this point, we are sure that there's no match in the index/type
51 | #have to check in the whole index hippocampe
52 | nbMatch = self.bigSearch()
53 | if nbMatch == 0:
54 | newIntel = NewIntel(self.coreIntelligence, self.dictData[self.coreIntelligence])
55 | #indexing the new intel in the index/type hippocampe/new
56 | newIntel.indexNewIntel()
57 |
58 | def bigSearch(self):
59 | self.es.indices.refresh(index = self.indexNameES)
60 | #searching in the index hippocampe
61 | self.resSearch = self.es.search(index = self.indexNameES, body = self.docSearch)
62 | return self.resSearch['hits']['total']
63 |
64 |
65 | def littleSearch(self):
66 | self.forgeDocSearch()
67 | return self.search()
68 |
69 | def forgeDocSearch(self):
70 | self.docSearch = {
71 | "query": {
72 | "bool": {
73 | "must": [
74 | {
75 | "match": {
76 | self.coreIntelligence : self.dictData[self.coreIntelligence]
77 | }
78 | }
79 | ]
80 | }
81 | }
82 | }
83 |
84 | def forgeDocUpdate(self):
85 | self.docUpdate = {
86 | "script": {
87 | "lang": "painless",
88 | "inline": "ctx._source.lastAppearance = params.lastAppearance",
89 | "params": {
90 | "lastAppearance": self.lastAppearance
91 | }
92 | }
93 | }
94 |
95 | def forgeDocIndex(self):
96 | #the dictData is already well-formed for index queries
97 | #however, sometimes, it can have a blank field, which is useless
98 | #and will be deleted
99 | self.dictData.pop('blank', None)
100 | self.docIndex = self.dictData
101 | #adding the meta data
102 | #new intelligence, so lastAppearance and first appearance are setted to today
103 | self.lastAppearance = strftime("%Y%m%dT%H%M%S%z")
104 | self.firstAppearance = strftime("%Y%m%dT%H%M%S%z")
105 | self.docIndex ['lastAppearance'] = self.lastAppearance
106 | self.docIndex['firstAppearance'] = self.firstAppearance
107 | self.docIndex['idSource'] = self.idSource
108 | self.docIndex['source'] = self.source
109 |
110 | def updateIntel(self):
111 | self.lastAppearance = strftime("%Y%m%dT%H%M%S%z")
112 | self.forgeDocUpdate()
113 | self.update()
114 |
--------------------------------------------------------------------------------
/core/services/more.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf8 -*-
3 |
4 |
5 | from modules.more.ObjToEnrich import ObjToEnrich
6 | from multiprocessing.dummy import Pool as ThreadPool
7 | from multiprocessing import cpu_count, Queue
8 | from hipposcore import calcHipposcore
9 | nbProcessPerCPU = 1
10 | import logging
11 | logger = logging.getLogger(__name__)
12 |
13 | def tellMeMore(chunkedList):
14 | chunkedDict = dict(chunkedList)
15 | result = dict()
16 | for ioc, attributes in chunkedDict.items():
17 | objIoc = ObjToEnrich(attributes['type'], ioc)
18 | try:
19 | result[ioc] = objIoc.getDetails()
20 | except Exception as e:
21 | logger.error('tellMeMore failed, no idea where it came from', exc_info = True)
22 | result[ioc] = dict()
23 | result[ioc]['error'] = str(e)
24 | return result
25 |
26 | def main(jsonRequest):
27 | logger.info('more.main launched')
28 | logger.info(jsonRequest)
29 | #dictionnary that contains the response
30 | response = dict()
31 |
32 | #the client-request (JSON dict) has to be converted into a list
33 | #Indeed, in order to "multi-threaded" the process
34 | #pool.map will iterate on the argument
35 | #however, iterate on a dict gives only the key (which is unicode type)
36 | #whereas dict type is needed to tellMeMore
37 | #a list of dict is therefore the solution
38 | items = list(jsonRequest.items())
39 |
40 | #the client-request list (items variable) will be splited into n small
41 | #requests which included chunksize elements
42 | chunksize = 10
43 |
44 | #cutting the items list into several small ones with chunksize length
45 | chunks = [items[i:i + chunksize] for i in range(0, len(items), chunksize)]
46 |
47 | pool_size = cpu_count() * nbProcessPerCPU
48 | pool = ThreadPool(processes=pool_size)
49 | queue = Queue()
50 |
51 | #updating directly response dict causes errors
52 | #that's why result from tellMeMore function is queued
53 | #logger.info('beginning tellMeMore with several threads')
54 | queue.put(pool.map(tellMeMore, chunks))
55 |
56 | #gathering all result from the queue into one dict
57 | for element in queue.get():
58 | response.update(element)
59 |
60 | pool.close()
61 | pool.join()
62 | #logger.info('tellMeMore.main pool closed and joined')
63 |
64 | #yet, the response does not include hipposcore
65 | #let's add the hipposcore to the response
66 |
67 | hipposcoreDict = calcHipposcore(response)
68 |
69 | #two dicts, response with all data
70 | #{u'103.16.26.228': [{u'date': u'20151001T112643+0200',
71 | # u'firstAppearance': u'20151001T112643+0200',
72 | # u'idSource': u'AVAiuZ7ADAiil8mrVC0p',
73 | # u'ip': u'103.16.26.228',
74 | # u'lastAppearance': u'20151001T112643+0200',
75 | # u'source': u'abuseFree_feodotrackerIP'}],
76 | # u'103.16.26.36': [{u'date': u'20151001T112643+0200',
77 | # u'firstAppearance': u'20151001T112643+0200',
78 | # u'idSource': u'AVAiuZ7ADAiil8mrVC0p',
79 | # u'ip': u'103.16.26.36',
80 | # u'lastAppearance': u'20151001T112643+0200',
81 | # u'source': u'abuseFree_feodotrackerIP'}],
82 | # u'199.9.24.1': [],
83 | # u'datingzzzj.ru': [],
84 | # u'trkl.su': []}
85 |
86 | #hipposcoreDict with only hipposcore
87 | # {u'103.16.26.228': {'hipposcore': -86.46647167633873},
88 | # u'103.16.26.36': {'hipposcore': -86.46647167633873},
89 | # u'199.9.24.1': 0,
90 | # u'datingzzzj.ru': 0,
91 | # u'trkl.su': 0}
92 |
93 | for keyRes, listMatch in response.items():
94 | for keyScore, scoreDict in hipposcoreDict.items():
95 | if keyRes == keyScore:
96 | for match in listMatch:
97 | match['hipposcore'] = scoreDict
98 | logger.info('more.main end')
99 | return response
100 |
101 | if __name__ == '__main__':
102 | main()
103 |
--------------------------------------------------------------------------------
/core/services/modules/shadowbook/processMsearch.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf8 -*-
3 |
4 |
5 | import logging
6 | logger = logging.getLogger(__name__)
7 |
8 | def littleSort(resMsearch, data):
9 | logger.info('processMsearch.littleSort launched')
10 | #data is a list of dict
11 | #each dict is a parsed line from the feed
12 | # resMsearch will look like
13 | #{u'responses': [{u'_shards': {u'failed': 0, u'successful': 5, u'total': 5},
14 | # u'hits': {u'hits': [{u'_id': u'AVOuC41q6EIAXcyxAFz0',
15 | # u'_index': u'hippocampe',
16 | # u'_score': 7.470799,
17 | # u'_source': {u'firstAppearance': u'20160325T145146+0100',
18 | # u'idSource': u'AVOuCsBt6EIAXcyxAEn3',
19 | # u'lastAppearance': u'20160325T145146+0100',
20 | # u'source': u'https://openphish.com/feed.txt',
21 | # u'url': u'https://www.myfridaygoodies.ch/sandbox/1/'},
22 | # u'_type': u'openphishFree_feedURL'}],
23 | # u'max_score': 7.470799,
24 | # u'total': 1},
25 | # u'timed_out': False,
26 | # u'took': 124},
27 |
28 | # {u'_shards': {u'failed': 0, u'successful': 5, u'total': 5},
29 | # u'hits': {u'hits': [], u'max_score': None, u'total': 0},
30 | # u'timed_out': False,
31 | # u'took': 107},
32 |
33 | # {u'_shards': {u'failed': 0, u'successful': 5, u'total': 5},
34 | # u'hits': {u'hits': [{u'_id': u'AVOuCxyD6EIAXcyxAFA0',
35 | # u'_index': u'hippocampe',
36 | # u'_score': 7.4480977,
37 | # u'_source': {u'firstAppearance': u'20160325T145117+0100',
38 | # u'idSource': u'AVOuCsBt6EIAXcyxAEn3',
39 | # u'lastAppearance': u'20160325T145117+0100',
40 | # u'source': u'https://openphish.com/feed.txt',
41 | # u'url': u'http://www.rutzcellars.com/dd-dd/art/'},
42 | # u'_type': u'openphishFree_feedURL'}],
43 | # u'max_score': 7.4480977,
44 | # u'total': 1},
45 | # u'timed_out': False,
46 | # u'took': 117}]}
47 | i = 0
48 | toIndex = list()
49 | toUpdate = list()
50 | while i < len(data):
51 | if len(resMsearch['responses'][i]['hits']['hits']) == 0:
52 | #we are in this case:
53 | #u'hits': {u'hits': []
54 | #it means that the intel at line i of the feed is unknown in ES
55 | #and we have to index it
56 | toIndex.append(data[i])
57 | if len(resMsearch['responses'][i]['hits']['hits']) > 0:
58 | #we are in this case:
59 | # u'hits': {u'hits': [{u'_id': u'AVOuCxyD6EIAXcyxAFA0',
60 | # u'_index': u'hippocampe',
61 | # u'_score': 7.4480977,
62 | # u'_source': {u'firstAppearance': u'20160325T145117+0100',
63 | # u'idSource': u'AVOuCsBt6EIAXcyxAEn3',
64 | # u'lastAppearance': u'20160325T145117+0100',
65 | # u'source': u'https://openphish.com/feed.txt',
66 | # u'url': u'http://www.rutzcellars.com/dd-dd/art/'},
67 | # u'_type': u'openphishFree_feedURL'}],
68 |
69 | #retrieving ids to update the doc in ES
70 | for match in resMsearch['responses'][i]['hits']['hits']:
71 | toUpdate.append(match['_id'])
72 | i += 1
73 |
74 | res = dict()
75 | res['toIndex'] = toIndex
76 | res['toUpdate'] = toUpdate
77 | logger.info('processMsearch.littleSort end')
78 | return res
79 |
80 | def bigSort(resMsearch, data):
81 | logger.info('processMsearch.bigSort launched')
82 | i = 0
83 | toIndexInNew = list()
84 | while i < len(data):
85 | if len(resMsearch['responses'][i]['hits']['hits']) == 0:
86 | toIndexInNew.append(data[i])
87 | i += 1
88 | logger.info('processMsearch.bigSort end')
89 | return toIndexInNew
90 | if __name__ == '__main__':
91 | main()
92 |
--------------------------------------------------------------------------------
/core/services/shadowbook.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf8 -*-
3 |
4 |
5 | """
6 | This module contains the shadowBook process.
7 | """
8 | from modules.shadowbook import createSessions
9 | from modules.common import getConf
10 | from multiprocessing import cpu_count
11 | from multiprocessing.dummy import Pool as ThreadPool
12 | from multiprocessing import Queue
13 | import os
14 | from os import listdir
15 | from os.path import join
16 | from functools import partial
17 | from modules.shadowbook import processFeed
18 | from traceback import print_exc
19 | import logging
20 | logger = logging.getLogger(__name__)
21 |
22 | from modules.shadowbook.objects.IndexJob import IndexJob
23 | from modules.shadowbook.objects.Job import Job
24 | from time import sleep
25 | def startJob():
26 | logger.info('shadowbook.startJob launched')
27 | try:
28 | #creating sessions for feeds which need authentification
29 | listSessions = createSessions.createSessions()
30 |
31 | #retrieving the config from hippo.conf file
32 | cfg = getConf.getHippoConf()
33 | nbThreadPerCPU = cfg.getint('shadowbook', 'nbThreadPerCPU')
34 |
35 | #processFeed needs 2 parameters:
36 | # * the conf path
37 | # * the listSessions
38 | #processIoc is launched in a multithreaded way
39 | #However pool.map does not handle multiple arguments
40 | #we use functools.partial to bypass this limitation
41 | pool_size = cpu_count() * nbThreadPerCPU
42 | pool = ThreadPool(processes=pool_size)
43 | queue = Queue()
44 |
45 | #json which reports the result from all feeds
46 | reportAllFeeds = dict()
47 | #arguments must be in a list
48 | argInputs = list()
49 | feedsPath = os.path.dirname(os.path.abspath(__file__)) + '/../conf/feeds'
50 |
51 | #building the list which each element is a feedConfPath
52 | for cfgFile in listdir(feedsPath):
53 | cfgPath = join(feedsPath, cfgFile)
54 | argInputs.append(cfgPath)
55 |
56 | for link in argInputs:
57 | reportAllFeeds.update(processFeed.main(listSessions, link))
58 |
59 | #func = partial(processFeed.main, listSessions)
60 | #queue.put(pool.map(func, argInputs))
61 | #pool.close()
62 | #pool.join()
63 | #for reportOneFeed in queue.get():
64 | # reportAllFeeds.update(reportOneFeed)
65 | logger.info('shadowbook.startJob end')
66 | return reportAllFeeds
67 | except Exception as e:
68 | logger.info('shadowbook.startJob failed, no idea where it came from...', exc_info = True)
69 | report = dict()
70 | report['error shadowbook'] = str(e)
71 | return report
72 |
73 | def initJob():
74 | logger.info('shadowbook.initJob launched')
75 | try:
76 | indexJob = IndexJob()
77 | #create index/type hippocampe/shadowbookJobs only if it does not exist
78 | indexJob.createIndexJob()
79 | job = Job()
80 | nbOngoingJob = job.searchOngoingJob()
81 | logger.info('number ongoing job: %s', nbOngoingJob)
82 |
83 | report = dict()
84 | if nbOngoingJob == 1:
85 | #only 1 shadowbook job at a time
86 | report['error'] = 'Ongoing job already running'
87 | logger.error(report['error'])
88 | elif nbOngoingJob == 0:
89 | #create a document in ES which represents a JOB
90 | #index with a ongoing status & startTime as
3 | Jobs Supervision
4 |
5 |
6 |
7 |
8 |
52 |
53 |
9 |
10 |
11 |
12 |
13 |
14 |
27 |
28 |
29 |
30 |
31 |
32 |
45 |
46 |
47 |
48 |
51 | | Job id | 16 |{{columnNameM}} | 17 |{{reportMetadataLvlOne.id}} | 21 |{{reportMetadataLvlOne.duration}} | 22 |{{reportMetadataLvlOne.endTime}} | 23 |{{reportMetadataLvlOne.startTime}} | 24 |{{reportMetadataLvlOne.status}} | 25 | 26 |
|---|
| Conf Filename | 34 |{{columnNameR}} | 35 |
|---|---|
| {{confFileName}} | 40 |{{valueInReportConf}} | 41 |