├── test ├── __init__.py ├── zadvanced │ ├── __init__.py │ ├── test_hunter.py │ └── test_router.py ├── zsqlite │ ├── __init__.py │ └── test_store_sqlite_indicators_nonpersistent.py ├── zelasticsearch │ ├── __init__.py │ ├── test_store_elasticsearch_tokens.py │ └── test_store_elasticsearch_tokens_edit.py ├── test_router.py ├── test_hunter.py ├── test_gatherer_geo.py ├── test_store.py ├── test_peers.py ├── test_gatherer_peer.py ├── test_gatherer_asn.py └── test_httpd.py ├── README.txt ├── cif ├── httpd │ ├── static │ │ └── favicon.ico │ ├── views │ │ ├── __init__.py │ │ ├── u │ │ │ ├── __init__.py │ │ │ ├── submit.py │ │ │ ├── indicators.py │ │ │ └── tokens.py │ │ ├── feed │ │ │ ├── md5.py │ │ │ ├── sha1.py │ │ │ ├── ssdeep.py │ │ │ ├── url.py │ │ │ ├── sha256.py │ │ │ ├── sha512.py │ │ │ ├── email.py │ │ │ ├── ipv6.py │ │ │ ├── fqdn.py │ │ │ └── ipv4.py │ │ ├── confidence.py │ │ ├── help.py │ │ ├── ping.py │ │ ├── health.py │ │ └── tokens.py │ ├── templates │ │ ├── layout.html │ │ ├── flash.html │ │ ├── login.html │ │ ├── tokens │ │ │ ├── form.html │ │ │ ├── show.html │ │ │ ├── index.html │ │ │ └── edit.html │ │ ├── base.html │ │ ├── application.html │ │ ├── submit.html │ │ ├── indicators.html │ │ └── nav.html │ └── common.py ├── __init__.py ├── auth │ ├── plugin.py │ └── cif_store │ │ └── __init__.py ├── exceptions.py ├── store │ ├── plugin.py │ ├── sqlite │ │ ├── message.py │ │ ├── ip.py │ │ └── __init__.py │ ├── indicator_plugin.py │ ├── dummy.py │ ├── zelasticsearch │ │ ├── constants.py │ │ ├── schema.py │ │ ├── locks.py │ │ ├── helpers.py │ │ └── __init__.py │ └── token_plugin.py ├── utils │ ├── __init__.py │ └── asn_client.py ├── hunter │ ├── ipv4_resolve_prefix_whitelist.py │ ├── fqdn_subdomain.py │ ├── fqdn_wl.py │ ├── fqdn.py │ ├── fqdn_ns.py │ ├── url.py │ ├── fqdn_cname.py │ ├── fqdn_mx.py │ ├── farsight.py │ ├── spamhaus_fqdn.py │ └── spamhaus_ip.py ├── gatherer │ ├── ja3.py │ ├── peers.py │ ├── asn.py │ └── __init__.py └── constants.py ├── packaging ├── debian │ ├── compat │ ├── install │ ├── rules │ ├── cif-smrt.conf │ ├── cif-httpd.conf │ ├── cif-router.conf │ ├── cif-hunters.conf │ ├── changelog │ ├── cif-storage.conf │ ├── control │ └── cif-services.init ├── docker │ ├── Makefile │ ├── supervisord.conf │ ├── Dockerfile │ └── Dockerfile.base ├── pyinstaller │ ├── cif.spec │ ├── cif-router.spec │ ├── cif-httpd.spec │ └── csirtg-smrt.spec ├── rpm │ └── cif.spec ├── macports │ ├── sysutils │ │ └── bearded-avenger │ │ │ └── Portfile │ └── README.md └── homebrew │ └── Library │ └── Formula │ └── bearded-avenger.rb ├── .gitattributes ├── dev_requirements.txt ├── docs ├── httpd.rst └── index.rst ├── helpers ├── es.sh ├── test_ubuntu_blank.sh ├── test_rhel.sh ├── test_ubuntu16.sh ├── test_centos7.sh ├── test_centos7_es.sh ├── test_rhel_es.sh ├── buildbasebox.sh ├── test_ubuntu16_es.sh ├── test_ubuntu16_reboot.sh └── test_ubuntu16_es_upsert.sh ├── setup.cfg ├── rules ├── default │ ├── vxvault.yml │ ├── sblam.yml │ ├── apwg.yml │ ├── openphish.yml │ ├── darklist_de.yml │ ├── urlhaus_abuse_ch.yml │ ├── mirc.yml │ ├── emergingthreats.yml │ ├── torproject_org.yml │ ├── sans_edu.yml │ ├── danger_rules_sk.yml │ ├── cisco_umbrella.yml │ ├── tranco.yml │ ├── phishtank.yml │ ├── bambenek.yml │ ├── majestic.yml │ ├── feodotracker.yml │ ├── spamhaus.yml │ ├── sslbl_abuse_ch.yml │ ├── stopforumspam.yml │ ├── normshield.yml │ ├── csirtg.yml │ └── dataplane.yml └── examples │ ├── alienvault.yml │ ├── faf-bambenek.yml │ └── blocklist_de.yml ├── hacking └── develop.conf ├── MANIFEST.in ├── requirements.txt ├── .github ├── issue_template.md └── workflows │ └── pr_test.yml ├── Vagrantfile_blank ├── .coveragerc ├── .gitignore ├── Vagrantfile_buildbox ├── README.md ├── Vagrantfile ├── contributing.md ├── reindex_tokens.py └── setup.py /test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /README.txt: -------------------------------------------------------------------------------- 1 | see README.md 2 | -------------------------------------------------------------------------------- /cif/httpd/static/favicon.ico: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /cif/httpd/views/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packaging/debian/compat: -------------------------------------------------------------------------------- 1 | 9 -------------------------------------------------------------------------------- /test/zadvanced/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/zsqlite/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /cif/httpd/views/u/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/zelasticsearch/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | 'cif' export-subst 2 | cif/_version.py export-subst 3 | -------------------------------------------------------------------------------- /dev_requirements.txt: -------------------------------------------------------------------------------- 1 | coverage>=4.2 2 | pytest-cov>=2.6 3 | pytest>=4.2 4 | -r requirements.txt 5 | -------------------------------------------------------------------------------- /packaging/debian/install: -------------------------------------------------------------------------------- 1 | cif /usr/bin 2 | cif-router /usr/bin 3 | cif-httpd /usr/bin 4 | cif-smrt /usr/bin -------------------------------------------------------------------------------- /cif/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from ._version import get_versions 3 | VERSION = get_versions()['version'] 4 | del get_versions -------------------------------------------------------------------------------- /cif/httpd/templates/layout.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% include "nav.html" %} 4 | 5 | {% include "flash.html" %} 6 | -------------------------------------------------------------------------------- /docs/httpd.rst: -------------------------------------------------------------------------------- 1 | HTTP API 2 | ======== 3 | 4 | .. automodule:: httpd 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /helpers/es.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | VERSION=5.6.16 6 | 7 | docker run -p 9200:9200 -p 9300:9300 elasticsearch:$VERSION 8 | -------------------------------------------------------------------------------- /packaging/debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | export DH_VERBOSE=1 3 | 4 | override_dh_auto_build override_dh_auto_install: 5 | @ 6 | 7 | %: 8 | dh $@ -------------------------------------------------------------------------------- /helpers/test_ubuntu_blank.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export VAGRANT_VAGRANTFILE=Vagrantfile_blank 4 | export CIF_BOOTSTRAP_TEST=1 5 | 6 | time vagrant up 7 | -------------------------------------------------------------------------------- /test/test_router.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from cif.router import Router 3 | 4 | 5 | def test_router_basics(): 6 | with Router(test=True) as r: 7 | pass 8 | -------------------------------------------------------------------------------- /packaging/debian/cif-smrt.conf: -------------------------------------------------------------------------------- 1 | [supervisord] 2 | pidfile = /var/run/cif-smrt.pid 3 | 4 | [program:cif-smrt] 5 | command = cif-smrt 6 | autostart = true 7 | stderr_logfile = /var/log/cif/cif-smrt.log -------------------------------------------------------------------------------- /packaging/debian/cif-httpd.conf: -------------------------------------------------------------------------------- 1 | [supervisord] 2 | pidfile = /var/run/cif-httpd.pid 3 | 4 | [program:cif-httpd] 5 | command = cif-httpd 6 | autostart = true 7 | stderr_logfile = /var/log/cif/cif-httpd.log 8 | -------------------------------------------------------------------------------- /packaging/debian/cif-router.conf: -------------------------------------------------------------------------------- 1 | [supervisord] 2 | pidfile = /var/run/cif-router.pid 3 | 4 | [program:cif-router] 5 | command = cif-router 6 | autostart = true 7 | stderr_logfile = /var/log/cif/cif-router.log -------------------------------------------------------------------------------- /packaging/debian/cif-hunters.conf: -------------------------------------------------------------------------------- 1 | [supervisord] 2 | pidfile = /var/run/cif-hunters.pid 3 | 4 | [program:cif-hunters] 5 | command = cif-hunters 6 | autostart = true 7 | stderr_logfile = /var/log/cif/cif-hunters.log -------------------------------------------------------------------------------- /packaging/debian/changelog: -------------------------------------------------------------------------------- 1 | bearded-avenger (%VERSION%-%RELEASE%~%DIST%) %DIST%; urgency=low 2 | 3 | * %VERSION% release 4 | 5 | -- CSIRT Gadgets Foundation. %DATE% 6 | -------------------------------------------------------------------------------- /packaging/debian/cif-storage.conf: -------------------------------------------------------------------------------- 1 | [supervisord] 2 | pidfile = /var/run/cif-storage.pid 3 | 4 | [program:cif-storage] 5 | command = cif-storage --store sqlite -d 6 | autostart = true 7 | stderr_logfile = /var/log/cif/cif-storage.log -------------------------------------------------------------------------------- /test/test_hunter.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from cif.hunter import Hunter 4 | from zmq import Context 5 | 6 | 7 | def test_hunter(): 8 | with Hunter(Context.instance()) as h: 9 | assert isinstance(h, Hunter) 10 | -------------------------------------------------------------------------------- /cif/auth/plugin.py: -------------------------------------------------------------------------------- 1 | import abc 2 | 3 | 4 | class Auth(object): 5 | __metaclass__ = abc.ABCMeta 6 | 7 | name = 'base' 8 | 9 | @abc.abstractmethod 10 | def __init__(self, **kwargs): 11 | raise NotImplementedError 12 | -------------------------------------------------------------------------------- /cif/httpd/templates/flash.html: -------------------------------------------------------------------------------- 1 | {% import "bootstrap/utils.html" as utils %} 2 | 3 | {% with messages = get_flashed_messages(with_categories=true) %} 4 | {% if messages %} 5 | {{ utils.flashed_messages(messages) }} 6 | {% endif %} 7 | {% endwith %} -------------------------------------------------------------------------------- /helpers/test_rhel.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export CIF_BOOTSTRAP_TEST=1 4 | export CIF_ANSIBLE_SDIST=/vagrant 5 | export CIF_HUNTER_THREADS=2 6 | export CIF_HUNTER_ADVANCED=1 7 | export CIF_GATHERER_GEO_FQDN=1 8 | export CIF_VAGRANT_DISTRO=redhat 9 | 10 | time vagrant up 11 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal = 1 3 | 4 | [tool:pytest] 5 | norecursedirs = build 6 | 7 | [versioneer] 8 | VCS = git 9 | style = pep440 10 | versionfile_source = cif/_version.py 11 | versionfile_build = cif/_version.py 12 | tag_prefix = 13 | parentdir_prefix = cif- 14 | -------------------------------------------------------------------------------- /helpers/test_ubuntu16.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export VAGRANT_VAGRANTFILE=Vagrantfile 4 | export CIF_BOOTSTRAP_TEST=1 5 | export CIF_ANSIBLE_SDIST=/vagrant 6 | export CIF_HUNTER_THREADS=2 7 | export CIF_HUNTER_ADVANCED=1 8 | #export CIF_GATHERER_GEO_FQDN=1 9 | 10 | time vagrant up 11 | -------------------------------------------------------------------------------- /rules/default/vxvault.yml: -------------------------------------------------------------------------------- 1 | defaults: 2 | provider: vxvault.net 3 | confidence: 9 4 | tlp: green 5 | altid_tlp: clear 6 | tags: malware 7 | values: 8 | - indicator 9 | 10 | feeds: 11 | urls: 12 | remote: http://vxvault.net/URL_List.php 13 | pattern: '^(http:\/\/\S+)$' -------------------------------------------------------------------------------- /helpers/test_centos7.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export CIF_BOOTSTRAP_TEST=1 4 | export CIF_ANSIBLE_SDIST=/vagrant 5 | export CIF_HUNTER_THREADS=2 6 | export CIF_HUNTER_ADVANCED=1 7 | #export CIF_GATHERER_GEO_FQDN=1 8 | export CIF_VAGRANT_DISTRO=centos 9 | 10 | vagrant box update 11 | time vagrant up 12 | -------------------------------------------------------------------------------- /rules/default/sblam.yml: -------------------------------------------------------------------------------- 1 | defaults: 2 | provider: sblam.com 3 | confidence: 7 4 | tlp: green 5 | altid_tlp: clear 6 | tags: 7 | - spam 8 | - spammers 9 | feeds: 10 | proxy: 11 | remote: https://sblam.com/blacklist.txt 12 | pattern: '^(\S+)$' 13 | values: 14 | - indicator -------------------------------------------------------------------------------- /rules/default/apwg.yml: -------------------------------------------------------------------------------- 1 | # requires APWG_TOKEN environment var to be set 2 | # more info from our friends at apwg.org 3 | fetcher: apwg 4 | parser: indicator 5 | 6 | defaults: 7 | confidence: 9 8 | provider: apwg.org 9 | tlp: amber 10 | 11 | feeds: 12 | urls: 13 | defaults: 14 | confidence: 9 -------------------------------------------------------------------------------- /cif/exceptions.py: -------------------------------------------------------------------------------- 1 | from cifsdk.exceptions import CIFException 2 | 3 | 4 | class StoreSubmissionFailed(CIFException): 5 | pass 6 | 7 | 8 | class InvalidSearch(CIFException): 9 | pass 10 | 11 | 12 | class StoreLockError(CIFException): 13 | pass 14 | 15 | 16 | class CIFBusy(CIFException): 17 | pass 18 | -------------------------------------------------------------------------------- /helpers/test_centos7_es.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export CIF_BOOTSTRAP_TEST=1 4 | export CIF_ANSIBLE_SDIST=/vagrant 5 | export CIF_HUNTER_THREADS=2 6 | export CIF_HUNTER_ADVANCED=1 7 | #export CIF_GATHERER_GEO_FQDN=1 8 | export CIF_VAGRANT_DISTRO=centos 9 | export CIF_ANSIBLE_ES=localhost:9200 10 | 11 | time vagrant up 12 | -------------------------------------------------------------------------------- /helpers/test_rhel_es.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export CIF_BOOTSTRAP_TEST=1 4 | export CIF_ANSIBLE_SDIST=/vagrant 5 | export CIF_HUNTER_THREADS=2 6 | export CIF_HUNTER_ADVANCED=1 7 | export CIF_GATHERER_GEO_FQDN=1 8 | export CIF_VAGRANT_DISTRO=redhat 9 | export CIF_ANSIBLE_ES=localhost:9200 10 | 11 | time vagrant up 12 | -------------------------------------------------------------------------------- /rules/default/openphish.yml: -------------------------------------------------------------------------------- 1 | defaults: 2 | tags: phishing 3 | protocol: tcp 4 | provider: openphish.com 5 | tlp: green 6 | reference_tlp: clear 7 | confidence: 9 8 | values: indicator 9 | 10 | feeds: 11 | urls: 12 | itype: url 13 | pattern: ^(.+)$ 14 | remote: https://openphish.com/feed.txt 15 | -------------------------------------------------------------------------------- /cif/store/plugin.py: -------------------------------------------------------------------------------- 1 | import abc 2 | 3 | 4 | class Store(object): 5 | __metaclass__ = abc.ABCMeta 6 | 7 | name = 'base' 8 | 9 | @abc.abstractmethod 10 | def __init__(self): 11 | raise NotImplementedError 12 | 13 | @abc.abstractmethod 14 | def ping(self): 15 | return True 16 | 17 | -------------------------------------------------------------------------------- /helpers/buildbasebox.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export VAGRANT_VAGRANTFILE=Vagrantfile_buildbox 4 | 5 | if [ -e cifv3.box ]; then 6 | rm cifv3.box 7 | fi 8 | 9 | vagrant box remove cifv3 10 | time vagrant up 11 | vagrant package --output cifv3.box 12 | vagrant box add cifv3 cifv3.box 13 | vagrant destroy -f 14 | rm -rf cifv3.box 15 | -------------------------------------------------------------------------------- /rules/default/darklist_de.yml: -------------------------------------------------------------------------------- 1 | defaults: 2 | provider: darklist.de 3 | confidence: 7 4 | tlp: green 5 | altid_tlp: clear 6 | tags: 7 | - blacklist 8 | altid: http://www.darklist.de/raw.php 9 | 10 | 11 | feeds: 12 | compromised-ips: 13 | remote: http://www.darklist.de/raw.php 14 | pattern: '^(\S+)$' 15 | values: 16 | - indicator -------------------------------------------------------------------------------- /helpers/test_ubuntu16_es.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export VAGRANT_VAGRANTFILE=Vagrantfile 4 | export CIF_BOOTSTRAP_TEST=1 5 | export CIF_ANSIBLE_SDIST=/vagrant 6 | export CIF_HUNTER_THREADS=2 7 | export CIF_HUNTER_ADVANCED=1 8 | export CIF_ANSIBLE_ES=localhost:9200 9 | export CIF_ELASTICSEARCH_TEST=1 10 | #export CIF_GATHERER_GEO_FQDN=1 11 | 12 | time vagrant up 13 | -------------------------------------------------------------------------------- /helpers/test_ubuntu16_reboot.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export VAGRANT_VAGRANTFILE=Vagrantfile 4 | export CIF_BOOTSTRAP_TEST=1 5 | export CIF_ANSIBLE_SDIST=/vagrant 6 | export CIF_HUNTER_THREADS=2 7 | export CIF_HUNTER_ADVANCED=1 8 | #export CIF_GATHERER_GEO_FQDN=1 9 | 10 | time vagrant up 11 | 12 | vagrant reload 13 | vagrant ssh -c 'cd /vagrant/deploymentkit && bash /vagrant/deploymentkit/test.sh' -------------------------------------------------------------------------------- /packaging/docker/Makefile: -------------------------------------------------------------------------------- 1 | # https://github.com/phusion/baseimage-docker/blob/master/Makefile 2 | NAME = cif 3 | VERSION = latest # make this a version number! 4 | 5 | .PHONY: all build run 6 | 7 | all: build 8 | 9 | build: 10 | @docker build -t $(NAME):$(VERSION) . 11 | 12 | build-clean: 13 | @docker build -t $(NAME):$(VERSION) --no-cache . 14 | 15 | run: 16 | @docker run $(NAME):$(VERSION) 17 | -------------------------------------------------------------------------------- /helpers/test_ubuntu16_es_upsert.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export VAGRANT_VAGRANTFILE=Vagrantfile 4 | export CIF_BOOTSTRAP_TEST=1 5 | export CIF_ANSIBLE_SDIST=/vagrant 6 | export CIF_HUNTER_THREADS=2 7 | export CIF_HUNTER_ADVANCED=1 8 | export CIF_ANSIBLE_ES=localhost:9200 9 | export CIF_ELASTICSEARCH_TEST=1 10 | export CIF_STORE_ES_UPSERT_MODE=1 11 | #export CIF_GATHERER_GEO_FQDN=1 12 | 13 | time vagrant up 14 | -------------------------------------------------------------------------------- /cif/httpd/views/feed/md5.py: -------------------------------------------------------------------------------- 1 | class Md5(object): 2 | 3 | def __init__(self): 4 | pass 5 | 6 | def process(self, data, whitelist): 7 | wl = set() 8 | for x in whitelist: 9 | wl.add(x['indicator']) 10 | 11 | rv = [] 12 | for x in data: 13 | if x['indicator'] not in wl: 14 | rv.append(x) 15 | 16 | return rv 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /cif/httpd/views/feed/sha1.py: -------------------------------------------------------------------------------- 1 | class Sha1(object): 2 | 3 | def __init__(self): 4 | pass 5 | 6 | def process(self, data, whitelist): 7 | wl = set() 8 | for x in whitelist: 9 | wl.add(x['indicator']) 10 | 11 | rv = [] 12 | for x in data: 13 | if x['indicator'] not in wl: 14 | rv.append(x) 15 | 16 | return rv 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /cif/httpd/views/feed/ssdeep.py: -------------------------------------------------------------------------------- 1 | class Ssdeep(object): 2 | 3 | def __init__(self): 4 | pass 5 | 6 | def process(self, data, allowlist): 7 | allowed = set() 8 | for x in allowlist: 9 | allowed.add(x['indicator']) 10 | 11 | rv = [] 12 | for x in data: 13 | if x['indicator'] not in allowed: 14 | rv.append(x) 15 | 16 | return rv 17 | -------------------------------------------------------------------------------- /cif/httpd/views/feed/url.py: -------------------------------------------------------------------------------- 1 | class Url(object): 2 | 3 | def __init__(self): 4 | pass 5 | 6 | def process(self, data, whitelist): 7 | wl = set() 8 | for x in whitelist: 9 | wl.add(x['indicator']) 10 | 11 | rv = [] 12 | for x in data: 13 | if x['indicator'] not in wl: 14 | rv.append(x) 15 | 16 | return rv 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /cif/httpd/views/feed/sha256.py: -------------------------------------------------------------------------------- 1 | class Sha256(object): 2 | 3 | def __init__(self): 4 | pass 5 | 6 | def process(self, data, whitelist): 7 | wl = set() 8 | for x in whitelist: 9 | wl.add(x['indicator']) 10 | 11 | rv = [] 12 | for x in data: 13 | if x['indicator'] not in wl: 14 | rv.append(x) 15 | 16 | return rv 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /cif/httpd/views/feed/sha512.py: -------------------------------------------------------------------------------- 1 | class Sha512(object): 2 | 3 | def __init__(self): 4 | pass 5 | 6 | def process(self, data, whitelist): 7 | wl = set() 8 | for x in whitelist: 9 | wl.add(x['indicator']) 10 | 11 | rv = [] 12 | for x in data: 13 | if x['indicator'] not in wl: 14 | rv.append(x) 15 | 16 | return rv 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /hacking/develop.conf: -------------------------------------------------------------------------------- 1 | [supervisord] 2 | nodaemon = true 3 | loglevel = DEBUG 4 | 5 | [program:cif-router] 6 | command = cif-router -d 7 | autostart = true 8 | stderr_logfile = log/cif-router.log 9 | 10 | [program:cif-httpd] 11 | command = cif-httpd -d 12 | autostart = true 13 | stderr_logfile = log/cif-httpd.log 14 | 15 | #[program:csirtg-smrt] 16 | #command = csirtg-smrt -d 17 | #autostart = true 18 | #stderr_logfile = log/csirtg-smrt.log 19 | -------------------------------------------------------------------------------- /rules/default/urlhaus_abuse_ch.yml: -------------------------------------------------------------------------------- 1 | parser: csv 2 | defaults: 3 | provider: urlhaus.abuse.ch 4 | tlp: green 5 | altid_tlp: clear 6 | confidence: 9 7 | tags: 8 | - malware 9 | application: https 10 | protocol: tcp 11 | values: 12 | - null 13 | - reporttime 14 | - indicator 15 | - null 16 | - null 17 | - description 18 | - null 19 | feeds: 20 | Malware: 21 | remote: https://urlhaus.abuse.ch/downloads/csv/ -------------------------------------------------------------------------------- /rules/default/mirc.yml: -------------------------------------------------------------------------------- 1 | defaults: 2 | tags: whitelist 3 | provider: mirc.com 4 | protocol: tcp 5 | tlp: green 6 | altid_tlp: clear 7 | confidence: 8 8 | application: irc 9 | remote: http://www.mirc.com/servers.ini 10 | altid: http://www.mirc.com/servers.ini 11 | lasttime: 'month' 12 | 13 | feeds: 14 | domains: 15 | pattern: SERVER:([a-zA-Z0-9-.]+\.[a-z]{2,3}):(\d+[-[\d+,]+):?GROUP 16 | values: 17 | - indicator 18 | - portlist -------------------------------------------------------------------------------- /rules/default/emergingthreats.yml: -------------------------------------------------------------------------------- 1 | defaults: 2 | provider: emergingthreats.net 3 | confidence: 8 4 | tlp: green 5 | altid_tlp: clear 6 | tags: 7 | - malware 8 | description: 'compromised host' 9 | altid: http://rules.emergingthreats.net/blockrules/compromised-ips.txt 10 | 11 | 12 | feeds: 13 | compromised-ips: 14 | remote: http://rules.emergingthreats.net/blockrules/compromised-ips.txt 15 | pattern: '^(\S+)$' 16 | values: 17 | - indicator -------------------------------------------------------------------------------- /packaging/debian/control: -------------------------------------------------------------------------------- 1 | Source: bearded-avenger 2 | Section: admin 3 | Priority: optional 4 | Standards-Version: 3.9.3 5 | Maintainer: Wes Young 6 | Build-Depends: cdbs, debhelper (>= 5.0.0) 7 | Homepage: https://github.com/csirtgadgets/bearded-avenger 8 | 9 | Package: bearded-avenger 10 | Architecture: all 11 | Depends: ${misc:Depends} 12 | Description: The smartest way to consume threat intelligence 13 | The smartest way to consume threat intelligence 14 | -------------------------------------------------------------------------------- /rules/default/torproject_org.yml: -------------------------------------------------------------------------------- 1 | parser: pattern 2 | defaults: 3 | provider: torproject.org 4 | tlp: green 5 | altid_tlp: clear 6 | confidence: 8.5 7 | feeds: 8 | tor_exit_nodes: 9 | remote: https://check.torproject.org/exit-addresses 10 | pattern: '^ExitAddress\s(\S+)\s(\d{4}-\d{2}-\d{2})\s\S+$' 11 | values: 12 | - indicator 13 | - lasttime 14 | defaults: 15 | tags: 16 | - tor 17 | protocol: tcp 18 | description: 'Tor Exit Node' 19 | -------------------------------------------------------------------------------- /rules/default/sans_edu.yml: -------------------------------------------------------------------------------- 1 | skip: '^Site$' 2 | defaults: 3 | tlp: green 4 | reference_tlp: clear 5 | provider: 'isc.sans.edu' 6 | pattern: '^(.+)$' 7 | values: indicator 8 | tags: suspicious 9 | 10 | feeds: 11 | block: 12 | remote: https://isc.sans.edu/feeds/block.txt 13 | defaults: 14 | confidence: 8 15 | pattern: ^(\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b)\t\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b\t(\d+) 16 | values: 17 | - indicator 18 | - mask 19 | tags: scanner 20 | -------------------------------------------------------------------------------- /cif/httpd/views/confidence.py: -------------------------------------------------------------------------------- 1 | from flask import jsonify 2 | from flask.views import MethodView 3 | 4 | CMAP = { 5 | 10: 'Certain', 6 | 9: 'Highly Confident', 7 | 8: 'Very Confident', 8 | 7: 'Confident', 9 | 6: 'Slightly better than a coin flip', 10 | 5: 'Coin Flip', 11 | 4: 'Slightly worse than a coin flip', 12 | 3: 'Not Confident', 13 | 2: 'Unknown', 14 | 1: 'Unknown', 15 | 0: 'Unknown', 16 | } 17 | 18 | 19 | class ConfidenceAPI(MethodView): 20 | 21 | def get(self): 22 | return jsonify(CMAP) 23 | -------------------------------------------------------------------------------- /rules/default/danger_rules_sk.yml: -------------------------------------------------------------------------------- 1 | parser: pattern 2 | defaults: 3 | provider: 'danger.rulez.sk' 4 | confidence: 9 5 | tlp: green 6 | altid_tlp: clear 7 | 8 | feeds: 9 | ssh: 10 | remote: http://danger.rulez.sk/projects/bruteforceblocker/blist.php 11 | pattern: '^(\S+)[\s|\t]+#\s(\S+\s\S+)' 12 | values: 13 | - indicator 14 | - lasttime 15 | defaults: 16 | application: ssh 17 | protocol: tcp 18 | portlist: 22 19 | tags: 20 | - scanner 21 | - bruteforce 22 | description: scanner 23 | 24 | -------------------------------------------------------------------------------- /cif/httpd/templates/login.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% include "flash.html" %} 3 | 4 | {% block content %} 5 |

Login

6 | {% if error %}

Error: {{ error }}{% endif %} 7 |

8 |

9 |

10 |

11 | 12 |
13 |

14 | 15 | 16 |
17 |

18 | {% endblock %} -------------------------------------------------------------------------------- /cif/store/sqlite/message.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Column, Integer, ForeignKey, UnicodeText 2 | from sqlalchemy.ext.declarative import declarative_base 3 | from sqlalchemy.orm import relationship 4 | from .indicator import Indicator 5 | 6 | Base = declarative_base() 7 | 8 | 9 | class Message(Base): 10 | __tablename__ = 'messages' 11 | 12 | id = Column(Integer, primary_key=True) 13 | message = Column(UnicodeText) 14 | 15 | indicator_id = Column(Integer, ForeignKey('indicators.id', ondelete='CASCADE')) 16 | indicator = relationship( 17 | Indicator, 18 | ) 19 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. bearded-avenger documentation master file, created by 2 | sphinx-quickstart on Tue Oct 27 10:20:35 2015. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to bearded-avenger's documentation! 7 | =========================================== 8 | 9 | Contents: 10 | 11 | .. toctree:: 12 | :maxdepth: 2 13 | 14 | httpd.rst 15 | 16 | Getting Started 17 | =============== 18 | 19 | .. code-block:: bash 20 | 21 | $ supervisord 22 | 23 | 24 | Indices and tables 25 | ================== 26 | 27 | * :ref:`genindex` 28 | * :ref:`modindex` 29 | * :ref:`search` 30 | 31 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | prune packaging 2 | include .coveragerc 3 | include README.md COPYING LICENSE 4 | include requirements.txt dev_requirements.txt 5 | include Makefile 6 | include VERSION 7 | include MANIFEST.in 8 | include versioneer.py 9 | include 'cif' 10 | include cif/_version.py 11 | include Vagrantfile 12 | include Vagrantfile.centos 13 | include Vagrantfile_es 14 | include Vagrantfile_prod 15 | include Vagrantfile_prod.centos 16 | recursive-include test * 17 | recursive-include deployment * 18 | recursive-include hacking * 19 | recursive-include docs * 20 | recursive-include packaging * 21 | recursive-include rules * 22 | global-exclude *.retry 23 | global-exclude *.pyc 24 | global-exclude __pycache__ 25 | -------------------------------------------------------------------------------- /packaging/docker/supervisord.conf: -------------------------------------------------------------------------------- 1 | [supervisord] 2 | nodaemon = true 3 | loglevel = debug 4 | user = cif 5 | logfile=/var/log/cif/supervisord.log 6 | pidfile=/tmp/supervisord.pid 7 | redirect_stderr = true 8 | 9 | [program:cif-router] 10 | command=cif-router -d 11 | autostart=true 12 | redirect_stderr=true 13 | redirect_stdout = true 14 | logfile=/var/log/cif/cif-router.log 15 | 16 | [program:cif-storage] 17 | command=cif-storage -d 18 | autostart=true 19 | redirect_stderr=true 20 | redirect_stdout = true 21 | logfile=/var/log/cif/cif-storage.log 22 | 23 | [program:cif-http] 24 | command=cif-httpd -d 25 | autostart=true 26 | redirect_stderr=true 27 | redirect_stdout = true 28 | logfile=/var/log/cif/cif-http.log 29 | -------------------------------------------------------------------------------- /rules/default/cisco_umbrella.yml: -------------------------------------------------------------------------------- 1 | defaults: 2 | description: 'eval("cisco umbrella #{rank}".format(**obs))' 3 | tags: whitelist 4 | protocol: tcp 5 | altid: 'http://s3-us-west-1.amazonaws.com/umbrella-static/index.html' 6 | provider: umbrella.cisco.com 7 | tlp: green 8 | altid_tlp: clear 9 | lasttime: 'month' 10 | values: 11 | - rank 12 | - indicator 13 | confidence: | 14 | eval(max(0, min( 15 | 12.5 - 2.5 * math.ceil( 16 | math.log10( 17 | int(obs['rank']) 18 | ) 19 | ), 20 | 10))) 21 | 22 | feeds: 23 | top-1000: 24 | remote: http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip 25 | pattern: '^(\d+),(\S{4,})$' 26 | limit: 1000 -------------------------------------------------------------------------------- /rules/default/tranco.yml: -------------------------------------------------------------------------------- 1 | parser: csv 2 | defaults: 3 | values: 4 | - rank 5 | - indicator 6 | description: 'eval("tranco list #{rank}".format(**obs))' 7 | tags: whitelist 8 | application: 9 | - http 10 | - https 11 | protocol: tcp 12 | altid: 'eval("https://tranco-list.eu/api/ranks/domain/{indicator}".format(**obs))' 13 | provider: tranco-list.eu 14 | tlp: green 15 | altid_tlp: clear 16 | lasttime: 'month' 17 | confidence: | 18 | eval(max(0, min( 19 | 12.5 - 2.5 * math.ceil( 20 | math.log10( 21 | int(obs['rank']) 22 | ) 23 | ), 24 | 10))) 25 | 26 | feeds: 27 | top-1000: 28 | remote: https://tranco-list.eu/top-1m.csv.zip 29 | limit: 1000 -------------------------------------------------------------------------------- /test/zadvanced/test_hunter.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from cif.hunter import Hunter 4 | from tornado.ioloop import IOLoop 5 | import threading 6 | import tempfile 7 | import os 8 | 9 | loop = IOLoop() 10 | ADDR = 'ipc://{}'.format(tempfile.NamedTemporaryFile().name) 11 | 12 | 13 | @pytest.mark.skipif(not os.environ.get('CIF_ADVANCED_TESTS'), reason='requires CIF_ADVANCED_TEST to be true') 14 | def test_zadvanced_hunter_start(): 15 | with Hunter(loop=loop, remote=ADDR) as h: 16 | h = Hunter(loop=loop, remote=ADDR) 17 | 18 | t = threading.Thread(target=h.start) 19 | 20 | t.start() 21 | 22 | assert t.is_alive() 23 | 24 | loop.stop() 25 | 26 | t.join() 27 | 28 | assert not t.is_alive() 29 | -------------------------------------------------------------------------------- /rules/default/phishtank.yml: -------------------------------------------------------------------------------- 1 | # https://www.phishtank.com/developer_info.php 2 | # remote: http://data.phishtank.com/data//online-valid.json.gz 3 | parser: json 4 | token: 5 | remote: http://data.phishtank.com/data/{token}/online-valid.json.gz 6 | defaults: 7 | provider: phishtank.com 8 | tlp: green 9 | altid_tlp: clear 10 | application: 11 | - http 12 | - https 13 | confidence: 9 14 | tags: phishing 15 | protocol: tcp 16 | 17 | feeds: 18 | urls: 19 | itype: url 20 | map: 21 | - submission_time 22 | - url 23 | - target 24 | - phish_detail_url 25 | - details 26 | values: 27 | - lasttime 28 | - indicator 29 | - description 30 | - altid 31 | - additional_data 32 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | setuptools 2 | cython>=0.2 3 | pyzmq>=23.2.1 4 | csirtg_indicator>=1.0.1,<2.0 5 | cifsdk>=3.0.0rc4,<4.0 6 | Flask-Limiter>=0.9.4,<=2.7.0 7 | limits>=1.1.1,<=2.7.1 8 | maxminddb>=2.2.0 9 | geoip2>=2.8.0,<2.9 10 | dnspython>=1.15.0,<=2.2.1 11 | Flask>=1.0 12 | flask-cors>=3.0,<4.0 13 | PyYAML>=4.2b1 14 | SQLAlchemy>=1.4.41 15 | elasticsearch>=5.3,<5.5 16 | elasticsearch-dsl>=5.3,<5.5 17 | html5lib==1.0b8 # bug in csirtg-smrt upstream 18 | msgpack-python>=0.4.8,<0.5.0 19 | apwgsdk==0.0.0a6 20 | csirtg_smrt>=1.0,<2.0 21 | csirtg_dnsdb==0.0.0a4 22 | tornado>=5.1.0 23 | faker==0.7.10 24 | Flask-Bootstrap==3.3.6.0 25 | gevent>=21.12.0 26 | gunicorn==20.1.0 27 | urllib3>=1.26.5 28 | requests>=2.27.1 29 | ujson<=5.5.0 30 | 31 | nltk>=3.6.6 # not directly required, pinned by Snyk to avoid a vulnerability -------------------------------------------------------------------------------- /packaging/pyinstaller/cif.spec: -------------------------------------------------------------------------------- 1 | # -*- mode: python -*- 2 | 3 | name = 'cif' 4 | 5 | block_cipher = None 6 | 7 | a = Analysis(['cifsdk/client/__init__.py'], 8 | binaries=None, 9 | datas=None, 10 | hiddenimports=[], 11 | hookspath=None, 12 | runtime_hooks=None, 13 | excludes=None, 14 | win_no_prefer_redirects=None, 15 | win_private_assemblies=None, 16 | cipher=block_cipher) 17 | pyz = PYZ(a.pure, a.zipped_data, 18 | cipher=block_cipher) 19 | exe = EXE(pyz, 20 | a.scripts, 21 | a.binaries, 22 | a.zipfiles, 23 | a.datas, 24 | name=name, 25 | debug=False, 26 | strip=None, 27 | upx=True, 28 | console=True ) -------------------------------------------------------------------------------- /rules/default/bambenek.yml: -------------------------------------------------------------------------------- 1 | # This is for the open source feeds from Bambenek Consulting 2 | parser: csv 3 | 4 | defaults: 5 | provider: osint.bambenekconsulting.com 6 | tlp: clear 7 | altid_tlp: clear 8 | confidence: 8 9 | tags: botnet 10 | values: 11 | - indicator 12 | - description 13 | - lasttime 14 | - altid 15 | 16 | feeds: 17 | c2_ipmasterlist_high: 18 | remote: http://osint.bambenekconsulting.com/feeds/c2-ipmasterlist-high.txt 19 | 20 | c2_domain_masterlist_high: 21 | remote: http://osint.bambenekconsulting.com/feeds/c2-dommasterlist-high.txt 22 | 23 | dga_domains_high: 24 | remote: http://osint.bambenekconsulting.com/feeds/dga-feed-high.csv.gz 25 | cache: dga-feed-high.csv 26 | defaults: 27 | tags: 28 | - dga 29 | - botnet -------------------------------------------------------------------------------- /packaging/pyinstaller/cif-router.spec: -------------------------------------------------------------------------------- 1 | # -*- mode: python -*- 2 | 3 | name = 'cif-router' 4 | 5 | block_cipher = None 6 | 7 | a = Analysis(['cif/router.py'], 8 | binaries=None, 9 | datas=None, 10 | hiddenimports=[], 11 | hookspath=None, 12 | runtime_hooks=None, 13 | excludes=None, 14 | win_no_prefer_redirects=None, 15 | win_private_assemblies=None, 16 | cipher=block_cipher) 17 | pyz = PYZ(a.pure, a.zipped_data, 18 | cipher=block_cipher) 19 | exe = EXE(pyz, 20 | a.scripts, 21 | a.binaries, 22 | a.zipfiles, 23 | a.datas, 24 | name=name, 25 | debug=False, 26 | strip=None, 27 | upx=True, 28 | console=True ) -------------------------------------------------------------------------------- /packaging/pyinstaller/cif-httpd.spec: -------------------------------------------------------------------------------- 1 | # -*- mode: python -*- 2 | 3 | name = 'cif-httpd' 4 | 5 | block_cipher = None 6 | 7 | a = Analysis(['cif/httpd/__init__.py'], 8 | binaries=None, 9 | datas=None, 10 | hiddenimports=[], 11 | hookspath=None, 12 | runtime_hooks=None, 13 | excludes=None, 14 | win_no_prefer_redirects=None, 15 | win_private_assemblies=None, 16 | cipher=block_cipher) 17 | pyz = PYZ(a.pure, a.zipped_data, 18 | cipher=block_cipher) 19 | exe = EXE(pyz, 20 | a.scripts, 21 | a.binaries, 22 | a.zipfiles, 23 | a.datas, 24 | name=name, 25 | debug=False, 26 | strip=None, 27 | upx=True, 28 | console=True ) -------------------------------------------------------------------------------- /rules/default/majestic.yml: -------------------------------------------------------------------------------- 1 | parser: csv 2 | defaults: 3 | values: 4 | - rank 5 | - null 6 | - indicator 7 | description: 'eval("majestic million #{rank}".format(**obs))' 8 | tags: whitelist 9 | application: 10 | - http 11 | - https 12 | protocol: tcp 13 | altid: 'eval("https://majestic.com/reports/majestic-million?domain={indicator}".format(**obs))' 14 | provider: majestic.com 15 | tlp: green 16 | altid_tlp: clear 17 | lasttime: 'month' 18 | confidence: | 19 | eval(max(0, min( 20 | 12.5 - 2.5 * math.ceil( 21 | math.log10( 22 | int(obs['rank']) 23 | ) 24 | ), 25 | 10))) 26 | 27 | feeds: 28 | top-1000: 29 | remote: https://downloads.majesticseo.com/majestic_million.csv 30 | skip_first: true 31 | limit: 1000 -------------------------------------------------------------------------------- /packaging/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:2.7.10 2 | MAINTAINER Wes Young (wes@csirtgadgets.org) 3 | 4 | ENV NEWUSER cif 5 | RUN useradd -m $NEWUSER 6 | 7 | RUN pip install pyzmq --install-option="--zmq=bundled" 8 | RUN pip install git+https://github.com/csirtgadgets/py-whiteface-sdk.git 9 | RUN pip install git+https://github.com/csirtgadgets/bearded-avenger.git 10 | 11 | VOLUME /var/lib 12 | 13 | RUN for path in \ 14 | /var/lib/cif/cache \ 15 | /var/lib/cif/rules \ 16 | /var/log/cif \ 17 | ; do \ 18 | mkdir -p $path; \ 19 | chown cif:cif "$path"; \ 20 | done 21 | 22 | VOLUME /var/log/cif 23 | VOLUME /var/lib/cif/rules 24 | VOLUME /var/lib/cif/cache 25 | 26 | COPY supervisord.conf /etc/supervisord.conf 27 | 28 | EXPOSE 5000 29 | 30 | CMD ["supervisord", "-c", "/etc/supervisord.conf"] 31 | -------------------------------------------------------------------------------- /packaging/pyinstaller/csirtg-smrt.spec: -------------------------------------------------------------------------------- 1 | # -*- mode: python -*- 2 | 3 | name = 'csirtg-smrt' 4 | 5 | block_cipher = None 6 | 7 | a = Analysis(['csirtg_smrt/__init__.py'], 8 | binaries=None, 9 | datas=None, 10 | hiddenimports=[], 11 | hookspath=None, 12 | runtime_hooks=None, 13 | excludes=None, 14 | win_no_prefer_redirects=None, 15 | win_private_assemblies=None, 16 | cipher=block_cipher) 17 | pyz = PYZ(a.pure, a.zipped_data, 18 | cipher=block_cipher) 19 | exe = EXE(pyz, 20 | a.scripts, 21 | a.binaries, 22 | a.zipfiles, 23 | a.datas, 24 | name=name, 25 | debug=False, 26 | strip=None, 27 | upx=True, 28 | console=True ) -------------------------------------------------------------------------------- /packaging/docker/Dockerfile.base: -------------------------------------------------------------------------------- 1 | FROM python:2.7.10 2 | MAINTAINER Wes Young (wes@csirtgadgets.org) 3 | 4 | ENV NEWUSER cif 5 | RUN useradd -m $NEWUSER 6 | 7 | RUN pip install pyzmq --install-option="--zmq=bundled" 8 | RUN pip install git+https://github.com/csirtgadgets/py-whiteface-sdk.git 9 | RUN pip install git+https://github.com/csirtgadgets/bearded-avenger.git 10 | 11 | VOLUME /var/lib 12 | 13 | RUN for path in \ 14 | /var/lib/cif/cache \ 15 | /var/lib/cif/rules \ 16 | /var/log/cif \ 17 | ; do \ 18 | mkdir -p $path; \ 19 | chown cif:cif "$path"; \ 20 | done 21 | 22 | VOLUME /var/log/cif 23 | VOLUME /var/lib/cif/rules 24 | VOLUME /var/lib/cif/cache 25 | 26 | COPY supervisord.conf /etc/supervisord.conf 27 | 28 | EXPOSE 5000 29 | 30 | CMD ["supervisord", "-c", "/etc/supervisord.conf"] 31 | -------------------------------------------------------------------------------- /cif/httpd/views/feed/email.py: -------------------------------------------------------------------------------- 1 | PERM_WHITELIST = [] 2 | 3 | 4 | class Email(object): 5 | 6 | def __init__(self): 7 | self.wl = set() 8 | for w in PERM_WHITELIST: 9 | self.wl.add(w) 10 | 11 | def match_whitelist(self, wl, d): 12 | bits = d.split('.') 13 | 14 | for i, b in enumerate(bits): 15 | if '.'.join(bits) in wl: 16 | return True 17 | bits.pop(0) 18 | 19 | # https://github.com/jsommers/pytricia 20 | def process(self, data, whitelist): 21 | 22 | wl = self.wl 23 | 24 | for w in whitelist: 25 | wl.add(w['indicator']) 26 | 27 | rv = [] 28 | for x in data: 29 | if not self.match_whitelist(wl, x['indicator']): 30 | rv.append(x) 31 | 32 | return rv 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /cif/httpd/views/feed/ipv6.py: -------------------------------------------------------------------------------- 1 | import pytricia 2 | import logging 3 | 4 | PERM_WHITELIST = [ 5 | ## TODO -- more 6 | # http://www.iana.org/assignments/ipv6-multicast-addresses/ipv6-multicast-addresses.xhtml 7 | # v6 8 | 'FF01:0:0:0:0:0:0:1', 9 | 'FF01:0:0:0:0:0:0:2', 10 | ] 11 | 12 | 13 | class Ipv6(object): 14 | 15 | def __init__(self): 16 | self.logger = logging.getLogger(__name__) 17 | pass 18 | 19 | def process(self, data, whitelist=[]): 20 | wl = pytricia.PyTricia(128) 21 | 22 | [wl.insert(x, True) for x in PERM_WHITELIST] 23 | 24 | [wl.insert(str(y['indicator']), True) for y in whitelist] 25 | 26 | rv = [] 27 | for y in data: 28 | if str(y['indicator']) not in wl: 29 | rv.append(y) 30 | 31 | return rv 32 | 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /test/zadvanced/test_router.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import threading 3 | from cif.router import Router 4 | from cif.constants import ROUTER_ADDR 5 | from tornado.ioloop import IOLoop 6 | import tempfile 7 | 8 | loop = IOLoop() 9 | 10 | ROUTER_ADDR = 'ipc://{}'.format(tempfile.NamedTemporaryFile().name) 11 | 12 | 13 | def _router_start(): 14 | r = Router(listen=ROUTER_ADDR) 15 | global thread 16 | thread = threading.Thread(target=r.run, args=[loop]) 17 | thread.start() 18 | return True 19 | 20 | 21 | def _router_stop(): 22 | global thread 23 | loop.stop() 24 | thread.join() 25 | 26 | 27 | @pytest.fixture 28 | def router(): 29 | yield _router_start() 30 | _router_stop() 31 | 32 | 33 | @pytest.fixture 34 | def client(): 35 | from cif.client.zeromq import ZMQ as Client 36 | 37 | yield Client(ROUTER_ADDR, '1234') 38 | -------------------------------------------------------------------------------- /rules/default/feodotracker.yml: -------------------------------------------------------------------------------- 1 | defaults: 2 | provider: feodotracker.abuse.ch 3 | tlp: green 4 | altid_tlp: clear 5 | altid: https://feodotracker.abuse.ch/host/ 6 | description: feodo 7 | 8 | feeds: 9 | c2: 10 | confidence: 8 11 | remote: https://feodotracker.abuse.ch/downloads/ipblocklist.csv 12 | pattern: ^(\S+\s\S+),(\S+),(\S+),(\S+)$ 13 | values: 14 | - firsttime 15 | - indicator 16 | - null 17 | - null 18 | defaults: 19 | tags: 20 | - feodo 21 | - botnet 22 | - c2 23 | 24 | hashes: 25 | confidence: 8 26 | remote: https://feodotracker.abuse.ch/downloads/malware_hashes.csv 27 | pattern: ^(\S+\s\S+),(\S+),(\S+)$ 28 | values: 29 | - firsttime 30 | - indicator 31 | - null 32 | defaults: 33 | tags: 34 | - feodo 35 | - botnet -------------------------------------------------------------------------------- /rules/default/spamhaus.yml: -------------------------------------------------------------------------------- 1 | defaults: 2 | provider: spamhaus.org 3 | confidence: 9 4 | tlp: green 5 | reference_tlp: clear 6 | tags: 7 | - hijacked 8 | reference: http://www.spamhaus.org/sbl/sbl.lasso?query= 9 | pattern: '(.+)\s;\s(.+)' 10 | values: 11 | - indicator 12 | - reference 13 | lasttime: 'month' 14 | 15 | feeds: 16 | drop: 17 | remote: http://www.spamhaus.org/drop/drop.txt 18 | 19 | edrop: 20 | remote: http://www.spamhaus.org/drop/edrop.txt 21 | 22 | dropv6: 23 | remote: https://www.spamhaus.org/drop/dropv6.txt 24 | 25 | # https://github.com/csirtgadgets/csirtg-smrt-py/issues/230 26 | asndrop: 27 | parser: pattern 28 | remote: https://www.spamhaus.org/drop/asndrop.txt 29 | pattern: '^(\S+) ; ([\S+]{2}) \| ([^\n]+)$' 30 | values: 31 | - indicator 32 | - cc 33 | - asn_desc 34 | -------------------------------------------------------------------------------- /packaging/rpm/cif.spec: -------------------------------------------------------------------------------- 1 | %define name bearded-avenger 2 | %define _version $VERSION 3 | 4 | Name: %{name} 5 | Version: %{_version} 6 | Release: 1%{?dist} 7 | Url: https://github.com/csirtgadgets/bearded-avenger 8 | Summary: The smartest way to consume threat intelligence. 9 | License: GPLv3 10 | Group: Development/Libraries 11 | Source: https://github.com/csirtgadgets/bearded-avenger/archive/%{version}.tar.gz 12 | BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot 13 | 14 | BuildArch: noarch 15 | 16 | %description 17 | 18 | The smartest way to consume threat intelligence. 19 | 20 | %prep 21 | %setup -q 22 | 23 | %build 24 | echo "nothing to build" 25 | 26 | %install 27 | cp -a cif* /usr/bin/ 28 | 29 | %clean 30 | rm -rf %{buildroot} 31 | 32 | %files 33 | %defattr(-,root,root) 34 | 35 | %changelog 36 | 37 | * 2015-09-28 - 3.0.0-1 38 | - Release of 3.0.0a1 -------------------------------------------------------------------------------- /rules/default/sslbl_abuse_ch.yml: -------------------------------------------------------------------------------- 1 | parser: csv 2 | defaults: 3 | provider: sslbl.abuse.ch 4 | tlp: green 5 | altid_tlp: clear 6 | confidence: 10 7 | tags: botnet 8 | application: https 9 | protocol: tcp 10 | 11 | feeds: 12 | sslipblacklist: 13 | remote: https://sslbl.abuse.ch/blacklist/sslipblacklist.csv 14 | defaults: 15 | values: 16 | - indicator 17 | - portlist 18 | - description 19 | 20 | dyre_sslipblacklist: 21 | remote: https://sslbl.abuse.ch/blacklist/dyre_sslipblacklist.csv 22 | defaults: 23 | values: 24 | - indicator 25 | - portlist 26 | - description 27 | 28 | ssl_fingerprints: 29 | remote: https://sslbl.abuse.ch/blacklist/sslblacklist.csv 30 | defaults: 31 | tags: 32 | - ssl 33 | - blacklist 34 | values: 35 | - lasttime 36 | - indicator 37 | - description 38 | -------------------------------------------------------------------------------- /.github/issue_template.md: -------------------------------------------------------------------------------- 1 | # Are you getting value from the project? Have you donated to the project? 2 | 3 | https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=YZPQXDLNYZZ3W 4 | 5 | # Did you check the FAQ https://github.com/csirtgadgets/bearded-avenger-deploymentkit/wiki/FAQ ? 6 | 7 | # Are you running with hunters enabled? If so, does turning them off resolve the problem? 8 | 9 | # Expected behavior and actual behavior. 10 | 11 | # Steps to reporduce the problem 12 | 13 | # Relevant logs as a result of the actual behavior 14 | # https://github.com/csirtgadgets/bearded-avenger-deploymentkit/wiki/FAQ#searching-logs 15 | 16 | # Did you attempt to fix the problem and submit a pull request? 17 | 18 | # Specifications like the version of the project, operating system, or hardware. 19 | 20 | # Does adding additional memory to the box resolve the problem? 21 | 22 | # How large is your /var/lib/cif.sqlite database? 23 | -------------------------------------------------------------------------------- /test/test_gatherer_geo.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from cif.gatherer.geo import Geo 3 | from csirtg_indicator import Indicator 4 | from pprint import pprint 5 | 6 | data = { 7 | 'cc': 'US', 8 | 'city': 'Chesterfield' 9 | } 10 | 11 | 12 | def test_gatherer_geo_v4(): 13 | a = Geo() 14 | 15 | def _resolve(i): 16 | i.cc = data['cc'] 17 | i.city = data['city'] 18 | 19 | a._resolve = _resolve 20 | 21 | i = Indicator(indicator='216.90.108.0') 22 | 23 | a.process(i) 24 | 25 | assert i.cc == data['cc'] 26 | assert i.city == data['city'] 27 | 28 | 29 | def test_gatherer_geo_v6(): 30 | a = Geo() 31 | 32 | def _resolve(i): 33 | i.cc = data['cc'] 34 | i.city = data['city'] 35 | 36 | a._resolve = _resolve 37 | 38 | i = Indicator(indicator='2607:ff10::c0:1:1:10d') 39 | 40 | a.process(i) 41 | 42 | assert i.cc == data['cc'] 43 | assert i.city == data['city'] 44 | -------------------------------------------------------------------------------- /rules/default/stopforumspam.yml: -------------------------------------------------------------------------------- 1 | defaults: 2 | confidence: 7 3 | tlp: green 4 | altid_tlp: clear 5 | provider: stopforumspam.com 6 | 7 | feeds: 8 | ip_list: 9 | remote: 'http://stopforumspam.com/downloads/listed_ip_1_all.zip' 10 | pattern: '^\"(\S+)\",\"(\S+)\",\"(\S+\s\S+)\"$' 11 | values: 12 | - indicator 13 | - null 14 | - lasttime 15 | defaults: 16 | tags: 17 | - spam 18 | 19 | domain-list: 20 | remote: http://stopforumspam.com/downloads/toxic_domains_whole_filtered_10000.txt 21 | pattern: '^(\S+)$' 22 | values: 23 | - indicator 24 | defaults: 25 | tags: 26 | - spam 27 | 28 | mail_list: 29 | remote: 'http://stopforumspam.com/downloads/listed_email_1_all.zip' 30 | pattern: '^\"(\S+)\",\"(\S+)\",\"(\S+\s\S+)\"$' 31 | values: 32 | - indicator 33 | - null 34 | - lasttime 35 | defaults: 36 | tags: 37 | - spam -------------------------------------------------------------------------------- /test/test_store.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import tempfile 4 | from argparse import Namespace 5 | import pytest 6 | from cif.store import Store 7 | from cifsdk.utils import setup_logging 8 | import arrow 9 | 10 | args = Namespace(debug=True, verbose=None) 11 | setup_logging(args) 12 | 13 | logger = logging.getLogger(__name__) 14 | 15 | 16 | @pytest.fixture 17 | def store(): 18 | dbfile = tempfile.mktemp() 19 | with Store(store_type='sqlite', dbfile=dbfile) as s: 20 | s._load_plugin(dbfile=dbfile) 21 | yield s 22 | 23 | if os.path.isfile(dbfile): 24 | os.unlink(dbfile) 25 | 26 | 27 | @pytest.fixture 28 | def indicator(): 29 | return { 30 | 'indicator': 'example.com', 31 | 'tags': 'botnet', 32 | 'provider': 'csirtgadgets.org', 33 | 'group': 'everyone', 34 | 'lasttime': arrow.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%fZ'), 35 | 'itype': 'fqdn', 36 | } 37 | -------------------------------------------------------------------------------- /test/test_peers.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from cif.gatherer.peers import Peer 3 | from csirtg_indicator import Indicator 4 | from pprint import pprint 5 | import warnings 6 | import os 7 | 8 | DISABLE_TESTS = True 9 | if os.environ.get('CIF_GATHERER_PEERS_TEST'): 10 | if os.environ['CIF_GATHERER_PEERS_TEST'] == '1': 11 | DISABLE_TESTS = False 12 | 13 | os.environ['CIF_GATHERERS_PEERS_ENABLED'] = '1' 14 | 15 | data = [ 16 | '701 1239 3549 3561 7132 | 216.90.108.0/24 | US | arin | 1998-09-25', 17 | ] 18 | 19 | @pytest.mark.skipif(DISABLE_TESTS, reason='need to set CIF_GATHERER_PEERS_TEST=1 to run') 20 | def test_gatherer_peers(): 21 | p = Peer() 22 | 23 | def _resolve(i): 24 | return data 25 | 26 | p._resolve_ns = _resolve 27 | x = p.process(Indicator(indicator='216.90.108.0')) 28 | 29 | if x.peers: 30 | for pp in x.peers: 31 | assert 65535 > int(pp['asn']) > 1 32 | else: 33 | warnings.warn('TC Not Responding...', UserWarning) 34 | 35 | -------------------------------------------------------------------------------- /test/test_gatherer_peer.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from cif.gatherer.peers import Peer 3 | from csirtg_indicator import Indicator 4 | import warnings 5 | from pprint import pprint 6 | import os 7 | 8 | DISABLE_TESTS = True 9 | if os.environ.get('CIF_GATHERER_PEERS_TEST'): 10 | if os.environ['CIF_GATHERER_PEERS_TEST'] == '1': 11 | DISABLE_TESTS = False 12 | 13 | os.environ['CIF_GATHERERS_PEERS_ENABLED'] = '1' 14 | 15 | data = [ 16 | '23028 | 216.90.108.0/24 | US | arin | 1998-09-25', 17 | '701 1239 3549 3561 7132 | 216.90.108.0/24 | US | arin | 1998-09-25', 18 | ] 19 | 20 | @pytest.mark.skipif(DISABLE_TESTS, reason='need to set CIF_GATHERER_PEERS_TEST=1 to run') 21 | def test_gatherer_peer(): 22 | a = Peer() 23 | 24 | def _resolve(i): 25 | return data 26 | 27 | a._resolve_ns = _resolve 28 | x = a.process(Indicator(indicator='216.90.108.0')) 29 | 30 | if x.peers: 31 | assert 65535 > int(x.peers[0]['asn']) > 1 32 | else: 33 | warnings.warn('TC Not Responding...', UserWarning) -------------------------------------------------------------------------------- /cif/auth/cif_store/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | 4 | from cif.auth.plugin import Auth 5 | from cif.store import Store 6 | from cif.utils import strtobool 7 | 8 | STORE_DEFAULT = os.environ.get('CIF_STORE_STORE', 'sqlite') 9 | STORE_NODES = os.getenv('CIF_STORE_NODES') 10 | 11 | TRACE = strtobool(os.environ.get('CIF_AUTH_CIFSTORE_TRACE', True)) 12 | 13 | logger = logging.getLogger(__name__) 14 | logger.setLevel(logging.ERROR) 15 | 16 | if TRACE: 17 | logger.setLevel(logging.DEBUG) 18 | 19 | class CifStore(Auth): 20 | 21 | name = 'cif_store' 22 | 23 | def __init__(self, **kwargs): 24 | self.token_cache = kwargs.get('token_cache', {}) 25 | self.store = Store(store_type=STORE_DEFAULT, nodes=STORE_NODES) 26 | self.store._load_plugin(store_type=STORE_DEFAULT, 27 | nodes=STORE_NODES, token_cache=self.token_cache) 28 | 29 | def handle_token_search(self, token, **kwargs): 30 | return self.store.store.tokens.auth_search({'token': token}) 31 | 32 | Plugin = CifStore 33 | -------------------------------------------------------------------------------- /Vagrantfile_blank: -------------------------------------------------------------------------------- 1 | #e -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | # Vagrantfile API/syntax version. Don't touch unless you know what you're doing! 5 | VAGRANTFILE_API_VERSION = "2" 6 | VAGRANTFILE_LOCAL = 'Vagrantfile.local' 7 | 8 | unless File.directory?('deploymentkit') 9 | puts "Please unzip the latest release of the deploymentkit before continuing..." 10 | puts "" 11 | puts "https://github.com/csirtgadgets/bearded-avenger-deploymentkit/wiki" 12 | puts "" 13 | exit 14 | end 15 | 16 | $script = < 17 | {%- endblock styles %} 18 | {%- endblock head %} 19 | 20 | 21 | {% block body -%} 22 | 23 | 27 | 28 |
29 |
30 |
31 | {% block content -%} 32 | {%- endblock content %} 33 |
34 |
35 |
36 | 37 | 38 | {% block scripts %} 39 | {%- endblock scripts %} 40 | 41 | {%- endblock body %} 42 | 43 | {%- endblock html %} 44 | 45 | 46 | {% endblock doc -%} 47 | 48 | -------------------------------------------------------------------------------- /packaging/debian/cif-services.init: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # /etc/init.d/cif-services -- startup script 4 | # 5 | ### BEGIN INIT INFO 6 | # Provides: cif-services 7 | # Required-Start: $network $named 8 | # Required-Stop: $network $named 9 | # Default-Start: 2 3 4 5 10 | # Default-Stop: 0 1 6 11 | # Short-Description: Starts cif-services 12 | # Description: Starts cif-services using start-stop-daemon 13 | ### END INIT INFO 14 | 15 | PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin 16 | PATH=/home/vagrant/.virtualenvs/cif/bin:$PATH 17 | NAME=cif-services 18 | DESC="cif-services" 19 | 20 | CIF_SERVICES="cif-router cif-storage cif-httpd cif-hunters cif-smrt" 21 | 22 | if [ `id -u` -ne 0 ]; then 23 | echo "You need root privileges to run this script" 24 | exit 1 25 | fi 26 | 27 | . /lib/lsb/init-functions 28 | 29 | if [ -r /etc/default/rcS ]; then 30 | . /etc/default/rcS 31 | fi 32 | 33 | # Include cif global defaults if avail 34 | if [ -f /etc/default/cif ] ; then 35 | . /etc/default/cif 36 | fi 37 | 38 | case "$1" in 39 | start) 40 | for i in $CIF_SERVICES; do 41 | service $i start 42 | done 43 | ;; 44 | stop) 45 | for i in $CIF_SERVICES; do 46 | service $i stop 47 | done 48 | ;; 49 | status) 50 | for i in $CIF_SERVICES; do 51 | service $i status 52 | done 53 | ;; 54 | restart|force-reload) 55 | for i in $CIF_SERVICES; do 56 | service $i restart 57 | done 58 | ;; 59 | *) 60 | log_success_msg "Usage: $0 {start|stop|restart|force-reload|status}" 61 | exit 1 62 | ;; 63 | esac 64 | 65 | exit 0 -------------------------------------------------------------------------------- /cif/utils/__init__.py: -------------------------------------------------------------------------------- 1 | import dns.resolver 2 | from dns.resolver import NoAnswer, NXDOMAIN, NoNameservers, Timeout 3 | from dns.name import EmptyLabel 4 | from cif.constants import HUNTER_RESOLVER_TIMEOUT 5 | import logging 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | 10 | def resolve_ns(data, t='A', timeout=HUNTER_RESOLVER_TIMEOUT): 11 | resolver = dns.resolver.Resolver() 12 | resolver.timeout = timeout 13 | resolver.lifetime = timeout 14 | resolver.search = [] 15 | try: 16 | answers = resolver.resolve(data, t) 17 | resp = [] 18 | for rdata in answers: 19 | resp.append(rdata) 20 | except (NoAnswer, NXDOMAIN, EmptyLabel, NoNameservers, Timeout) as e: 21 | if isinstance(e, Timeout): 22 | logger.info('{} - {} -- this may be normal'.format(data, e)) 23 | return [] 24 | 25 | if not isinstance(e, NoAnswer) and not isinstance(e, NXDOMAIN): 26 | logger.info('{} - {}'.format(data, e)) 27 | return [] 28 | 29 | return resp 30 | 31 | def strtobool(val): 32 | """ 33 | reimplementation of distutils.util.strtobool which is being deprecated 34 | :param val: value to check. True values are y, yes, t, true, on, and 1; false values are n, 35 | no, f, false, off, and 0. Raises ValueError if val is anything else. 36 | 37 | :return: bool True or False 38 | """ 39 | val = str(val).lower() 40 | if val in ('y', 'yes', 't', 'true', 'on', '1'): 41 | return True 42 | elif val in ('n', 'no', 'f', 'false', 'off', '0'): 43 | return False 44 | else: 45 | raise ValueError('invalid truth value {!r}'.format(val)) 46 | -------------------------------------------------------------------------------- /cif/hunter/ipv4_resolve_prefix_whitelist.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from csirtg_indicator import Indicator, InvalidIndicator 3 | import arrow 4 | import ipaddress 5 | 6 | class Ipv4ResolvePrefixWhitelist(object): 7 | 8 | def __init__(self): 9 | self.logger = logging.getLogger(__name__) 10 | self.is_advanced = False 11 | self.mtypes_supported = { 'indicators_create' } 12 | self.itypes_supported = { 'ipv4' } 13 | 14 | def _prereqs_met(self, i, **kwargs): 15 | if kwargs.get('mtype') not in self.mtypes_supported: 16 | return False 17 | 18 | if i.itype not in self.itypes_supported: 19 | return False 20 | 21 | if 'whitelist' not in i.tags: 22 | return False 23 | 24 | # only run this hunter if it's a single address (no CIDRs) 25 | if ipaddress.IPv4Network(i.indicator).prefixlen != 32: 26 | return False 27 | 28 | return True 29 | 30 | def process(self, i, router, **kwargs): 31 | if not self._prereqs_met(i, **kwargs): 32 | return 33 | 34 | prefix = i.indicator.split('.') 35 | prefix = prefix[:3] 36 | prefix.append('0/24') 37 | prefix = '.'.join(prefix) 38 | 39 | try: 40 | ii = Indicator(**i.__dict__()) 41 | except InvalidIndicator as e: 42 | self.logger.error(e) 43 | return 44 | 45 | ii.lasttime = ii.reporttime = arrow.utcnow() 46 | 47 | ii.indicator = prefix 48 | ii.tags = ['whitelist', 'hunter'] 49 | ii.confidence = (ii.confidence - 2) if ii.confidence >= 2 else 0 50 | router.indicators_create(ii) 51 | 52 | 53 | Plugin = Ipv4ResolvePrefixWhitelist -------------------------------------------------------------------------------- /cif/store/zelasticsearch/schema.py: -------------------------------------------------------------------------------- 1 | from elasticsearch_dsl import DocType, Text, Date, Integer, Float, Ip, Text, Keyword, \ 2 | GeoPoint, analyzer, tokenizer 3 | from elasticsearch_dsl.field import Field 4 | 5 | 6 | class IpRange(Field): 7 | # canonical in elasticsearch_dsl >6.x, doesn't support CIDRs until ES 6.1 8 | # if elasticsearch_dsl/ES updated past those versions, this class should be removed 9 | name = 'ip_range' 10 | 11 | ssdeep_tokenizer = tokenizer('ssdeep_tokenizer', type='ngram', min_gram=7, max_gram=7) 12 | ssdeep_analyzer = analyzer('ssdeep_analyzer', tokenizer=ssdeep_tokenizer) 13 | 14 | class Indicator(DocType): 15 | indicator = Keyword() 16 | indicator_ipv4 = Ip() 17 | indicator_ipv4_mask = Integer() 18 | indicator_iprange = IpRange() # works for both IPv4 and v6 19 | indicator_ipv6 = Keyword() 20 | indicator_ipv6_mask = Integer() 21 | indicator_ssdeep_chunksize = Integer() 22 | indicator_ssdeep_chunk = Text(analyzer=ssdeep_analyzer) 23 | indicator_ssdeep_double_chunk = Text(analyzer=ssdeep_analyzer) 24 | group = Keyword() 25 | itype = Keyword() 26 | tlp = Keyword() 27 | provider = Keyword() 28 | portlist = Text() 29 | asn = Float() 30 | asn_desc = Text() 31 | cc = Text(fields={'raw': Keyword()}) 32 | protocol = Text(fields={'raw': Keyword()}) 33 | reporttime = Date() 34 | lasttime = Date() 35 | firsttime = Date() 36 | confidence = Float() 37 | timezone = Text() 38 | city = Text(fields={'raw': Keyword()}) 39 | description = Keyword() 40 | tags = Keyword(multi=True, fields={'raw': Keyword()}) 41 | rdata = Keyword() 42 | count = Integer() 43 | message = Text(multi=True) 44 | location = GeoPoint() 45 | -------------------------------------------------------------------------------- /.github/workflows/pr_test.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a single version of Python 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: Python application 5 | 6 | on: 7 | pull_request: 8 | branches: [ master ] 9 | 10 | jobs: 11 | test: 12 | 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - name: Set up Python 3.9 17 | uses: actions/setup-python@v1 18 | with: 19 | python-version: 3.9 20 | 21 | - uses: actions/checkout@v2 22 | with: 23 | fetch-depth: 0 24 | 25 | - name: Install dependencies 26 | run: | 27 | sudo add-apt-repository -y ppa:maxmind/ppa 28 | sudo apt-get update 29 | sudo apt-get install geoipupdate libsnappy-dev docker 30 | 31 | python -m pip install --upgrade pip 32 | pip install --upgrade "setuptools<46" 33 | git clone https://github.com/csirtgadgets/csirtg-indicator-py-v1.git /tmp/csirtg_indicator 34 | pip install -r /tmp/csirtg_indicator/requirements.txt 35 | pip install /tmp/csirtg_indicator 36 | git clone https://github.com/csirtgadgets/csirtg-smrt-v1.git /tmp/csirtg_smrt 37 | pip install -r /tmp/csirtg_smrt/requirements.txt 38 | pip install /tmp/csirtg_smrt 39 | git clone https://github.com/csirtgadgets/cifsdk-py-v3.git /tmp/cifsdk-py-3 40 | pip install -r /tmp/cifsdk-py-3/requirements.txt 41 | pip install /tmp/cifsdk-py-3 42 | pip install -r dev_requirements.txt 43 | python setup.py develop 44 | make test 45 | python setup.py sdist 46 | -------------------------------------------------------------------------------- /packaging/homebrew/Library/Formula/bearded-avenger.rb: -------------------------------------------------------------------------------- 1 | # Documentation: https://github.com/Homebrew/homebrew/blob/master/share/doc/homebrew/Formula-Cookbook.md 2 | # /usr/local/Library/Contributions/example-formula.rb 3 | # PLEASE REMOVE ALL GENERATED COMMENTS BEFORE SUBMITTING YOUR PULL REQUEST! 4 | 5 | class BeardedAvenger < Formula 6 | homepage "https://csirtgadgets.org/collective-intelligence-framework" 7 | url "https://github.com/csirtgadgets/bearded-avenger/archive/master.zip" 8 | sha256 "" 9 | 10 | head "https://github.com/csirtgadgets/bearded-avenger.git", :branch => "master" 11 | depends_on "libyaml" 12 | 13 | # depends_on "cmake" => :build 14 | resource "pyzmq" do 15 | url "https://pypi.python.org/packages/source/p/pyzmq/pyzmq-14.7.0.tar.gz" 16 | sha256 "77994f80360488e7153e64e5959dc5471531d1648e3a4bff14a714d074a38cc2" 17 | end 18 | 19 | def install 20 | # ENV.deparallelize # if your formula fails when building in parallel 21 | cd "source/python" do 22 | system "python", *Language::Python.setup_install_args(prefix) 23 | end 24 | end 25 | 26 | test do 27 | # `test do` will create, run in and delete a temporary directory. 28 | # 29 | # This test will fail and we won't accept that! It's enough to just replace 30 | # "false" with the main program this formula installs, but it'd be nice if you 31 | # were more thorough. Run the test with `brew test bearded-avenger`. Options passed 32 | # to `brew install` such as `--HEAD` also need to be provided to `brew test`. 33 | # 34 | # The installed folder is not in the path, so use the entire path to any 35 | # executables being tested: `system "#{bin}/program", "do", "something"`. 36 | system "pytest" 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /cif/hunter/fqdn_subdomain.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from csirtg_indicator import Indicator 3 | from csirtg_indicator import resolve_itype 4 | from csirtg_indicator.exceptions import InvalidIndicator 5 | import arrow 6 | 7 | 8 | class FqdnSubdomain(object): 9 | 10 | def __init__(self): 11 | self.logger = logging.getLogger(__name__) 12 | self.is_advanced = False 13 | self.mtypes_supported = { 'indicators_create' } 14 | self.itypes_supported = { 'fqdn' } 15 | 16 | def _prereqs_met(self, i, **kwargs): 17 | if kwargs.get('mtype') not in self.mtypes_supported: 18 | return False 19 | 20 | if i.itype not in self.itypes_supported: 21 | return False 22 | 23 | if 'search' in i.tags: 24 | return False 25 | 26 | if not i.is_subdomain(): 27 | return False 28 | 29 | return True 30 | 31 | def process(self, i, router, **kwargs): 32 | if not self._prereqs_met(i, **kwargs): 33 | return 34 | 35 | fqdn = Indicator(**i.__dict__()) 36 | fqdn.indicator = i.is_subdomain() 37 | fqdn.lasttime = fqdn.reporttime = arrow.utcnow() 38 | 39 | try: 40 | resolve_itype(fqdn.indicator) 41 | except InvalidIndicator as e: 42 | self.logger.error(fqdn) 43 | self.logger.error(e) 44 | else: 45 | fqdn.confidence = (fqdn.confidence - 3) if fqdn.confidence >= 3 else 0 46 | fqdn.rdata = '{} subdomain'.format(i.indicator) 47 | if 'hunter' not in fqdn.tags: 48 | fqdn.tags.append('hunter') 49 | router.indicators_create(fqdn) 50 | self.logger.debug("FQDN Subdomain Hunter: {}".format(fqdn)) 51 | 52 | 53 | Plugin = FqdnSubdomain -------------------------------------------------------------------------------- /cif/hunter/fqdn_wl.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from csirtg_indicator import Indicator 3 | from csirtg_indicator import resolve_itype 4 | from csirtg_indicator.exceptions import InvalidIndicator 5 | import arrow 6 | 7 | class FqdnWl(object): 8 | 9 | def __init__(self): 10 | self.logger = logging.getLogger(__name__) 11 | self.is_advanced = False 12 | self.mtypes_supported = { 'indicators_create' } 13 | self.itypes_supported = { 'fqdn' } 14 | 15 | def _prereqs_met(self, i, **kwargs): 16 | if kwargs.get('mtype') not in self.mtypes_supported: 17 | return False 18 | 19 | if i.itype not in self.itypes_supported: 20 | return False 21 | 22 | if 'whitelist' not in i.tags: 23 | return False 24 | 25 | return True 26 | 27 | def process(self, i, router, **kwargs): 28 | if not self._prereqs_met(i, **kwargs): 29 | return 30 | 31 | urls = [] 32 | for p in ['http://', 'https://']: 33 | urls.append('{}{}'.format(p, i.indicator)) 34 | if not i.indicator.startswith('www.'): 35 | urls.append('{}www.{}'.format(p, i.indicator)) 36 | 37 | for u in urls: 38 | url = Indicator(**i.__dict__()) 39 | url.indicator = u 40 | 41 | try: 42 | resolve_itype(url.indicator) 43 | except InvalidIndicator as e: 44 | self.logger.error(url) 45 | self.logger.error(e) 46 | else: 47 | url.tags = ['whitelist', 'hunter'] 48 | url.itype = 'url' 49 | url.rdata = i.indicator 50 | url.lasttime = url.reporttime = arrow.utcnow() 51 | router.indicators_create(url) 52 | 53 | 54 | Plugin = FqdnWl -------------------------------------------------------------------------------- /cif/gatherer/ja3.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import ujson as json 3 | import logging 4 | import os 5 | 6 | #Set CIF_GATHERERS_JA3_ENABLED=0 to disable this gatherer 7 | ENABLE_JA3 = os.environ.get('CIF_GATHERERS_JA3_ENABLED') 8 | 9 | 10 | class Ja3(object): 11 | 12 | def __init__(self, *args, **kwargs): 13 | self.logger = logging.getLogger(__name__) 14 | if ENABLE_JA3 == '0': 15 | self.enabled = False 16 | else: 17 | self.enabled = True 18 | 19 | def _resolve(self, data): 20 | try: 21 | request = requests.get('https://ja3er.com/search/{}'.format(data), timeout=1) 22 | except Exception as e: 23 | self.logger.debug(e) 24 | return 25 | return json.loads(request.text) 26 | 27 | def process(self, indicator): 28 | if not self.enabled: 29 | self.logger.debug('self.enabled is set to {}'.format(self.enabled)) 30 | return indicator 31 | 32 | #if indicator isn't a md5 hash with ja3 tag, or it's already coming from the ja3er.com provider (bc ja3 smrt rule adds desc), exit 33 | if (not indicator.itype == 'md5') or (indicator.provider == 'ja3er.com') or ('ja3' not in indicator.tags): 34 | return indicator 35 | 36 | #if indicator has a description do not attempt to enrich further 37 | if indicator.description: 38 | return indicator 39 | 40 | try: 41 | i = str(indicator.indicator) 42 | 43 | ua = self._resolve(i) 44 | if not ua or len(ua) == 0: 45 | return indicator 46 | except Exception as e: 47 | self.logger.error('Gatherer [ja3]: Error on indicator {} - {}'.format(indicator, e)) 48 | 49 | for each in ua: 50 | self.logger.debug(each) 51 | indicator.lasttime = each.get('Last_seen') 52 | indicator.description = each.get('User-Agent') 53 | 54 | return indicator 55 | 56 | 57 | Plugin = Ja3 58 | -------------------------------------------------------------------------------- /cif/hunter/fqdn.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from cif.utils import resolve_ns 3 | from csirtg_indicator import Indicator 4 | from dns.resolver import Timeout 5 | from csirtg_indicator import resolve_itype 6 | from csirtg_indicator.exceptions import InvalidIndicator 7 | import arrow 8 | 9 | 10 | class Fqdn(object): 11 | 12 | def __init__(self): 13 | self.logger = logging.getLogger(__name__) 14 | self.is_advanced = True 15 | self.mtypes_supported = { 'indicators_create' } 16 | self.itypes_supported = { 'fqdn' } 17 | 18 | def _prereqs_met(self, i, **kwargs): 19 | if kwargs.get('mtype') not in self.mtypes_supported: 20 | return False 21 | 22 | if i.itype not in self.itypes_supported: 23 | return False 24 | 25 | if 'search' in i.tags: 26 | return False 27 | 28 | return True 29 | 30 | def process(self, i, router, **kwargs): 31 | if not self._prereqs_met(i, **kwargs): 32 | return 33 | 34 | try: 35 | r = resolve_ns(i.indicator) 36 | except Timeout: 37 | self.logger.info('timeout trying to resolve: {}'.format(i.indicator)) 38 | return 39 | 40 | for rr in r: 41 | rr = str(rr) 42 | if rr in ["", 'localhost', '0.0.0.0']: 43 | continue 44 | 45 | ip = Indicator(**i.__dict__()) 46 | ip.lasttime = ip.reporttime = arrow.utcnow() 47 | 48 | ip.indicator = rr 49 | try: 50 | resolve_itype(ip.indicator) 51 | except InvalidIndicator as e: 52 | self.logger.error(ip) 53 | self.logger.error(e) 54 | else: 55 | ip.itype = 'ipv4' 56 | ip.rdata = i.indicator 57 | ip.tags = ['pdns', 'hunter'] 58 | ip.confidence = 10 59 | router.indicators_create(ip) 60 | self.logger.debug("FQDN Hunter: {}".format(ip)) 61 | 62 | 63 | Plugin = Fqdn -------------------------------------------------------------------------------- /cif/constants.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | from cifsdk.constants import RUNTIME_PATH 3 | import sys 4 | 5 | PYVERSION = 2 6 | if sys.version_info > (3,): 7 | PYVERSION = 3 8 | 9 | from ._version import get_versions 10 | VERSION = get_versions()['version'] 11 | del get_versions 12 | 13 | TOKEN_LENGTH = 40 14 | 15 | ROUTER_ADDR = "ipc://{}".format(os.path.join(RUNTIME_PATH, 'router.ipc')) 16 | ROUTER_ADDR = os.environ.get('CIF_ROUTER_ADDR', ROUTER_ADDR) 17 | 18 | ROUTER_LOCAL_ADDR = "ipc://{}".format(os.path.join(RUNTIME_PATH, 'router_local.ipc')) 19 | ROUTER_LOCAL_ADDR = os.environ.get('CIF_ROUTER_ADDR', ROUTER_LOCAL_ADDR) 20 | 21 | STORE_ADDR = 'ipc://{}'.format(os.path.join(RUNTIME_PATH, 'store.ipc')) 22 | STORE_ADDR = os.environ.get('CIF_STORE_ADDR', STORE_ADDR) 23 | 24 | CTRL_ADDR = 'ipc://{}'.format(os.path.join(RUNTIME_PATH, 'ctrl.ipc')) 25 | CTRL_ADDR = os.environ.get('CIF_CTRL_ADDR', CTRL_ADDR) 26 | 27 | HUNTER_ADDR = 'ipc://{}'.format(os.path.join(RUNTIME_PATH, 'hunter.ipc')) 28 | HUNTER_ADDR = os.environ.get('CIF_HUNTER_ADDR', HUNTER_ADDR) 29 | 30 | HUNTER_SINK_ADDR = 'ipc://{}'.format(os.path.join(RUNTIME_PATH, 'hunter_sink.ipc')) 31 | HUNTER_SINK_ADDR = os.environ.get('CIF_HUNTER_SINK_ADDR', HUNTER_SINK_ADDR) 32 | 33 | GATHERER_ADDR = 'ipc://{}'.format(os.path.join(RUNTIME_PATH, 'gatherer.ipc')) 34 | GATHERER_ADDR = os.environ.get('CIF_GATHERER_ADDR', GATHERER_ADDR) 35 | 36 | GATHERER_SINK_ADDR = 'ipc://{}'.format(os.path.join(RUNTIME_PATH, 'gatherer_sink.ipc')) 37 | GATHERER_SINK_ADDR = os.environ.get('CIF_GATHERER_SINK_ADDR', GATHERER_SINK_ADDR) 38 | 39 | TOKEN_CACHE_DELAY = 45 40 | 41 | HUNTER_RESOLVER_TIMEOUT = os.environ.get('CIF_HUNTER_RESOLVER_TIMEOUT', 5) 42 | 43 | FEEDS_LIMIT = os.environ.get('CIF_FEEDS_LIMIT', 50000) 44 | FEEDS_WHITELIST_LIMIT = os.environ.get('CIF_FEEDS_WHITELIST_LIMIT', 25000) 45 | 46 | HTTPD_FEED_WHITELIST_CONFIDENCE = os.environ.get('CIF_HTTPD_FEED_WHITELIST_CONFIDENCE', 5) 47 | 48 | AUTH_ADDR = 'ipc://{}'.format(os.path.join(RUNTIME_PATH, 'auth.ipc')) 49 | AUTH_ENABLED = os.environ.get('CIF_AUTH_REQUIRED', True) 50 | -------------------------------------------------------------------------------- /cif/httpd/templates/tokens/edit.html: -------------------------------------------------------------------------------- 1 | {% extends "layout.html" %} 2 | 3 | {% block content %} 4 | 5 |
6 | 7 |

8 |

9 |

Username

10 | 11 |
12 |

13 | 14 |

15 |

16 |

Groups

17 | 18 |
19 |

20 | 21 |

22 |

23 |

Read Enabled?

24 | {% set checked = 'checked=""' %} 25 | {% if not token['read'] %} 26 | {% set checked = '' %} 27 | {% endif %} 28 | 29 |
30 |

31 | 32 |

33 |

34 |

Write Enabled?

35 | {% set checked = 'checked=""' %} 36 | {% if not token['write'] %} 37 | {% set checked = '' %} 38 | {% endif %} 39 | 40 |
41 |

42 | 43 |

44 |

45 |

Admin?

46 | {% set checked = 'checked=""' %} 47 | {% if not token['admin'] %} 48 | {% set checked = '' %} 49 | {% endif %} 50 | 51 |
52 |

53 | 54 | 55 | 56 |
57 | 58 | {% endblock %} -------------------------------------------------------------------------------- /cif/store/zelasticsearch/locks.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | import elasticsearch 3 | from elasticsearch_dsl.connections import connections 4 | from time import sleep 5 | from cif.exceptions import StoreLockError 6 | import os 7 | import socket 8 | 9 | LOCK_RETRIES = 45 10 | LOCK_RETRIES = os.getenv('CIF_ES_LOCK_RETRIES', LOCK_RETRIES) 11 | TEST_MODE = os.getenv('CIF_ELASTICCSEARCH_TEST', 0) 12 | LOCKS_FORCE = os.getenv('CIF_ES_LOCK_FORCE', 0) 13 | 14 | 15 | class LockManager(object): 16 | def __init__(self, handle, logger): 17 | self.lock = False 18 | self.handle = handle 19 | self.logger = logger 20 | self.name = socket.gethostname() 21 | self.nodes = self._nodes() 22 | 23 | def _nodes(self): 24 | info = elasticsearch.client.NodesClient(self.handle).info() 25 | return int(info['_nodes']['total']) 26 | 27 | def lock_aquire(self): 28 | if TEST_MODE: 29 | return 30 | 31 | if self.nodes == 1 and not LOCKS_FORCE: 32 | return 33 | 34 | es = self.handle 35 | n = int(LOCK_RETRIES) 36 | while not self.lock and n > 0: 37 | try: 38 | es.create( 39 | index='fs', doc_type='lock', id='global', 40 | body={'timestamp': datetime.utcnow(), 'node': self.name}) 41 | self.lock = True 42 | except elasticsearch.exceptions.TransportError as e: 43 | l = es.get(index='fs', doc_type='lock', id='global') 44 | self.logger.debug(l) 45 | self.logger.info('waiting on global lock from %s %s' % (l['_source']['node'], l['_source']['timestamp'])) 46 | n -= 1 47 | sleep(2) 48 | 49 | if n == 0: 50 | raise StoreLockError('failed to acquire lock') 51 | 52 | def lock_release(self): 53 | if TEST_MODE: 54 | return 55 | 56 | if self.nodes == 1 and not LOCKS_FORCE: 57 | return 58 | 59 | self.handle.delete(index='fs', doc_type='lock', id='global') 60 | self.lock = False 61 | -------------------------------------------------------------------------------- /cif/gatherer/peers.py: -------------------------------------------------------------------------------- 1 | 2 | from cif.utils import resolve_ns 3 | import logging 4 | import os 5 | import re 6 | 7 | ENABLE_PEERS = os.environ.get('CIF_GATHERERS_PEERS_ENABLED') 8 | 9 | 10 | class Peer(object): 11 | 12 | def __init__(self, *args, **kwargs): 13 | self.logger = logging.getLogger(__name__) 14 | self.enabled = kwargs.get('enabled', os.environ.get('CIF_GATHERERS_PEERS_ENABLED')) 15 | 16 | def _resolve(self, data): 17 | return resolve_ns('{}.{}'.format(data, 'peer.asn.cymru.com', timeout=15), t='TXT') 18 | 19 | def process(self, indicator): 20 | if not self.enabled: 21 | return indicator 22 | 23 | if not indicator.itype == 'ipv4': 24 | return indicator 25 | 26 | if indicator.is_private(): 27 | return indicator 28 | 29 | i = str(indicator.indicator) 30 | match = re.search(r'^(\S+)\/\d+$', i) 31 | if match: 32 | i = match.group(1) 33 | 34 | # cache it to the /24 35 | i = list(reversed(i.split('.'))) 36 | i = '0.{}.{}.{}'.format(i[1], i[2], i[3]) 37 | 38 | answers = self._resolve(i) 39 | if len(answers) > 0: 40 | if not indicator.peers: 41 | indicator.peers = [] 42 | # Separate fields and order by netmask length 43 | # 23028 | 216.90.108.0/24 | US | arin | 1998-09-25 44 | # 701 1239 3549 3561 7132 | 216.90.108.0/24 | US | arin | 1998-09-25 45 | for p in answers: 46 | self.logger.debug(p) 47 | bits = str(p).replace('"', '').strip().split(' | ') 48 | asn = bits[0] 49 | prefix = bits[1] 50 | cc = bits[2] 51 | rir = bits[3] 52 | asns = asn.split(' ') 53 | for a in asns: 54 | indicator.peers.append({ 55 | 'asn': a, 56 | 'prefix': prefix, 57 | 'cc': cc, 58 | 'rir': rir 59 | }) 60 | 61 | return indicator 62 | 63 | Plugin = Peer -------------------------------------------------------------------------------- /rules/default/csirtg.yml: -------------------------------------------------------------------------------- 1 | # cif-smrt configuration file to pull feeds from csirtg.io 2 | # For more information see https://csirtg.io 3 | # 4 | # If no token is given, the feed by default is a "limited feed" 5 | # provided by https://csirtg.io. The limits of the "limited feed" 6 | # are: 7 | # 8 | # 1. Only results from the last hour are returned 9 | # 2. A maximum of 25 results are returned per feed 10 | # 11 | # To remove the limits, sign up for an API key at https://csirtg.io 12 | 13 | parser: csv 14 | token: 'CSIRTG_TOKEN' # ENV['CSIRTG_TOKEN'] 15 | limit: 250 16 | defaults: 17 | provider: csirtg.io 18 | altid_tlp: clear 19 | altid: https://csirtg.io/search?q={indicator} 20 | tlp: clear 21 | confidence: 9 22 | values: 23 | - null 24 | - indicator 25 | - itype 26 | - portlist 27 | - null 28 | - null 29 | - protocol 30 | - application 31 | - null 32 | - null 33 | - lasttime 34 | - description 35 | - null 36 | - null 37 | 38 | feeds: 39 | # A feed of IP addresses block by a firewall (e.g. port scanners) 40 | darknet: 41 | remote: https://csirtg.io/api/users/csirtgadgets/feeds/darknet.csv 42 | defaults: 43 | tags: 44 | - scanner 45 | 46 | # A feed of URLs seen in the message body of UCE email. Do not alert or block 47 | # on these urls without additional post-processing. 48 | uce-urls: 49 | remote: https://csirtg.io/api/users/csirtgadgets/feeds/uce-urls.csv 50 | defaults: 51 | tags: 52 | - uce 53 | - uce-url 54 | 55 | # A feed of email addresses seen in UCE email. Do not alert or block on these 56 | # email addresses without additional post-processing. 57 | uce-email-address: 58 | remote: https://csirtg.io/api/users/csirtgadgets/feeds/uce-email-addresses.csv 59 | defaults: 60 | tags: 61 | - uce 62 | - uce-email-address 63 | 64 | # A feed of IP addresses seen delivering UCE email. This could be a machine that 65 | # is compromised or a user account has been compromised and used to send UCE. 66 | uce-ip: 67 | remote: https://csirtg.io/api/users/csirtgadgets/feeds/uce-ip.csv 68 | defaults: 69 | tags: 70 | - uce 71 | - uce-ip 72 | -------------------------------------------------------------------------------- /cif/hunter/fqdn_ns.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from csirtg_indicator import resolve_itype 3 | from csirtg_indicator.exceptions import InvalidIndicator 4 | from cif.utils import resolve_ns 5 | from cif.hunter import HUNTER_MIN_CONFIDENCE 6 | from csirtg_indicator import Indicator 7 | from dns.resolver import Timeout 8 | import arrow 9 | 10 | 11 | class FqdnNs(object): 12 | 13 | def __init__(self): 14 | self.logger = logging.getLogger(__name__) 15 | self.is_advanced = True 16 | self.mtypes_supported = { 'indicators_create' } 17 | self.itypes_supported = { 'fqdn' } 18 | 19 | def _prereqs_met(self, i, **kwargs): 20 | if kwargs.get('mtype') not in self.mtypes_supported: 21 | return False 22 | 23 | if i.itype not in self.itypes_supported: 24 | return False 25 | 26 | if 'search' in i.tags: 27 | return False 28 | 29 | return True 30 | 31 | def process(self, i, router, **kwargs): 32 | if not self._prereqs_met(i, **kwargs): 33 | return 34 | 35 | try: 36 | r = resolve_ns(i.indicator, t='NS') 37 | except Timeout: 38 | self.logger.info('timeout trying to resolve: {}'.format(i.indicator)) 39 | return 40 | 41 | for rr in r: 42 | rr = str(rr).rstrip('.') 43 | if rr in ["", 'localhost', '0.0.0.0']: 44 | continue 45 | 46 | i_ns = Indicator(**i.__dict__()) 47 | i_ns.indicator = rr 48 | 49 | try: 50 | i_ns_itype = resolve_itype(i_ns.indicator) 51 | except InvalidIndicator as e: 52 | self.logger.error(i_ns) 53 | self.logger.error(e) 54 | else: 55 | i_ns.lasttime = i_ns.reporttime = arrow.utcnow() 56 | i_ns.itype = i_ns_itype 57 | i_ns.rdata = "{} nameserver".format(i.indicator) 58 | if 'hunter' not in i_ns.tags: 59 | i_ns.tags.append('hunter') 60 | # prevent hunters from running on insertion of this ns 61 | i_ns.confidence = max(0, min(i_ns.confidence, HUNTER_MIN_CONFIDENCE - 1)) 62 | router.indicators_create(i_ns) 63 | self.logger.debug("FQDN NS Hunter: {}".format(i_ns)) 64 | 65 | Plugin = FqdnNs -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Getting Started 2 | **Do NOT try to install from the master repo.** 3 | 4 | For installation instructions and various unix distribution guides, use the **DeploymentKit**. 5 | 6 | https://github.com/csirtgadgets/bearded-avenger-deploymentkit/wiki 7 | 8 | # Integrations 9 | Several integrations, plugins, and extensions have been written to bridge CIFv3 with other tools. Here are some examples to consider building an ecosystem: 10 | 11 | * [Python client](https://github.com/csirtgadgets/cifsdk-py-v3) 12 | * [Dockerized Python client](https://hub.docker.com/r/renisac/cifv3-cli) 13 | * [PowerShell client](https://github.com/renisac/CIF3-pwsh) 14 | * [MineMeld plugin](https://github.com/renisac/CIFv3-Minemeld) 15 | * [Elastic threat intel integration](https://github.com/elastic/integrations/tree/main/packages/ti_cif3) 16 | * [Threatbus backbone](https://github.com/tenzir/threatbus/tree/main/plugins/apps/threatbus_cif3) 17 | * [Intelmq output bot](https://github.com/certtools/intelmq/pull/2244) 18 | 19 | # Getting Help 20 | * [the Wiki](https://github.com/csirtgadgets/bearded-avenger-deploymentkit/wiki) 21 | * [Known Issues](https://github.com/csirtgadgets/bearded-avenger/issues?labels=bug&state=open) 22 | * [FAQ](https://github.com/csirtgadgets/bearded-avenger-deploymentkit/wiki/FAQ) 23 | 24 | # Getting Involved 25 | There are many ways to get involved with the project. If you have a new and exciting feature, or even a simple bugfix, simply [fork the repo](https://help.github.com/articles/fork-a-repo), create some simple test cases, [generate a pull-request](https://help.github.com/articles/using-pull-requests) and give yourself credit! 26 | 27 | If you've never worked on a GitHub project, [this is a good piece](https://guides.github.com/activities/contributing-to-open-source) for getting started. 28 | 29 | * [How To Contribute](contributing.md) 30 | * [Mailing List](https://groups.google.com/forum/#!forum/ci-framework) 31 | * [Project Page](http://csirtgadgets.org/collective-intelligence-framework/) 32 | 33 | # Getting Started with Development 34 | 35 | https://github.com/csirtgadgets/bearded-avenger/wiki 36 | 37 | # COPYRIGHT AND LICENSE 38 | 39 | Copyright (C) 2017 [the CSIRT Gadgets Foundation](http://csirtgadgets.org) 40 | 41 | Free use of this software is granted under the terms of the [Mozilla Public License (MPLv2)](https://www.mozilla.org/en-US/MPL/2.0/). 42 | -------------------------------------------------------------------------------- /cif/hunter/url.py: -------------------------------------------------------------------------------- 1 | from cifsdk.constants import PYVERSION 2 | import logging 3 | from csirtg_indicator import Indicator, resolve_itype 4 | from csirtg_indicator.exceptions import InvalidIndicator 5 | import arrow 6 | 7 | if PYVERSION > 2: 8 | from urllib.parse import urlparse 9 | else: 10 | from urlparse import urlparse 11 | 12 | 13 | class Url(object): 14 | 15 | def __init__(self): 16 | self.logger = logging.getLogger(__name__) 17 | self.is_advanced = False 18 | self.mtypes_supported = { 'indicators_create' } 19 | self.itypes_supported = { 'url' } 20 | 21 | def _prereqs_met(self, i, **kwargs): 22 | if kwargs.get('mtype') not in self.mtypes_supported: 23 | return False 24 | 25 | if i.itype not in self.itypes_supported: 26 | return False 27 | 28 | if 'search' in i.tags: 29 | return False 30 | 31 | # prevent recursion with fqdn_wl hunter 32 | if ('whitelist') in i.tags and (i.rdata is not None or i.rdata != ''): 33 | return False 34 | 35 | return True 36 | 37 | def process(self, i, router, **kwargs): 38 | if not self._prereqs_met(i, **kwargs): 39 | return 40 | 41 | u = urlparse(i.indicator) 42 | if not u.hostname: 43 | return 44 | 45 | try: 46 | itype = resolve_itype(u.hostname) 47 | except InvalidIndicator as e: 48 | self.logger.error(u.hostname) 49 | self.logger.error(e) 50 | else: 51 | new_indicator = Indicator(**i.__dict__()) 52 | new_indicator.lasttime = new_indicator.reporttime = arrow.utcnow() 53 | new_indicator.indicator = u.hostname 54 | new_indicator.itype = itype 55 | if 'hunter' not in new_indicator.tags: 56 | new_indicator.tags.append('hunter') 57 | 58 | if new_indicator.itype in [ 'ipv4', 'ipv6' ]: 59 | new_indicator.confidence = (new_indicator.confidence - 2) if new_indicator.confidence >= 2 else 0 60 | else: 61 | new_indicator.confidence = (int(new_indicator.confidence) / 2) 62 | new_indicator.rdata = i.indicator 63 | 64 | self.logger.debug('[Hunter: Url] sending to router {}'.format(new_indicator)) 65 | router.indicators_create(new_indicator) 66 | 67 | 68 | Plugin = Url -------------------------------------------------------------------------------- /cif/hunter/fqdn_cname.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from cif.utils import resolve_ns 3 | from cif.hunter import HUNTER_MIN_CONFIDENCE 4 | from csirtg_indicator import Indicator 5 | from dns.resolver import Timeout 6 | from csirtg_indicator import resolve_itype 7 | from csirtg_indicator.exceptions import InvalidIndicator 8 | import arrow 9 | 10 | class FqdnCname(object): 11 | 12 | def __init__(self): 13 | self.logger = logging.getLogger(__name__) 14 | self.is_advanced = True 15 | self.mtypes_supported = { 'indicators_create' } 16 | self.itypes_supported = { 'fqdn' } 17 | 18 | def _prereqs_met(self, i, **kwargs): 19 | if kwargs.get('mtype') not in self.mtypes_supported: 20 | return False 21 | 22 | if i.itype not in self.itypes_supported: 23 | return False 24 | 25 | if 'search' in i.tags: 26 | return False 27 | 28 | return True 29 | 30 | def process(self, i, router, **kwargs): 31 | if not self._prereqs_met(i, **kwargs): 32 | return 33 | 34 | try: 35 | r = resolve_ns(i.indicator, t='CNAME') 36 | except Timeout: 37 | self.logger.info('timeout trying to resolve: {}'.format(i.indicator)) 38 | return 39 | 40 | for rr in r: 41 | # http://serverfault.com/questions/44618/is-a-wildcard-cname-dns-record-valid 42 | rr = str(rr).rstrip('.').lstrip('*.') 43 | if rr in ['', 'localhost', '0.0.0.0']: 44 | continue 45 | 46 | cname = Indicator(**i.__dict__()) 47 | cname.indicator = rr 48 | cname.lasttime = cname.reporttime = arrow.utcnow() 49 | 50 | try: 51 | resolve_itype(cname.indicator) 52 | except InvalidIndicator as e: 53 | self.logger.error(cname) 54 | self.logger.error(e) 55 | return 56 | 57 | cname.itype = 'fqdn' 58 | cname.rdata = '{} cname'.format(i.indicator) 59 | if 'hunter' not in cname.tags: 60 | cname.tags.append('hunter') 61 | # prevent hunters from running on insertion of this cname 62 | cname.confidence = max(0, min(cname.confidence, HUNTER_MIN_CONFIDENCE - 1)) 63 | router.indicators_create(cname) 64 | self.logger.debug("FQDN CNAME Hunter: {}".format(cname)) 65 | 66 | 67 | Plugin = FqdnCname -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | #e -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | # Vagrantfile API/syntax version. Don't touch unless you know what you're doing! 5 | VAGRANTFILE_API_VERSION = "2" 6 | VAGRANTFILE_LOCAL = 'Vagrantfile.local' 7 | 8 | sdist=ENV['CIF_ANSIBLE_SDIST'] 9 | es=ENV['CIF_ANSIBLE_ES'] 10 | hunter_threads=ENV['CIF_HUNTER_THREADS'] 11 | hunter_advanced=ENV['CIF_HUNTER_ADVANCED'] 12 | geo_fqdn=ENV['CIF_GATHERER_GEO_FQDN'] 13 | csirtg_token=ENV['CSIRTG_TOKEN'] 14 | distro=ENV.fetch('CIF_VAGRANT_DISTRO', 'ubuntu') 15 | redhat=0 16 | rhel_user=ENV['RHEL_USER'] 17 | rhel_pass=ENV['RHEL_PASSWORD'] 18 | es_tests=ENV.fetch('CIF_ELASTICSEARCH_TEST', '0') 19 | es_upsert=ENV.fetch('CIF_STORE_ES_UPSERT_MODE', '0') 20 | 21 | redhat=1 if distro == 'redhat' 22 | 23 | unless File.directory?('deploymentkit') 24 | puts "Please unzip the latest release of the deploymentkit before continuing..." 25 | puts "" 26 | puts "https://github.com/csirtgadgets/bearded-avenger-deploymentkit/wiki" 27 | puts "" 28 | exit 29 | end 30 | 31 | $script = < 84 | {% endblock %} 85 | -------------------------------------------------------------------------------- /cif/store/zelasticsearch/helpers.py: -------------------------------------------------------------------------------- 1 | from csirtg_indicator.utils import resolve_itype 2 | import re 3 | import binascii 4 | import socket 5 | import uuid 6 | from hashlib import sha256 7 | import ipaddress 8 | 9 | 10 | def expand_indicator(data): 11 | itype = resolve_itype(data['indicator']) 12 | if itype not in ['ipv4', 'ipv6', 'ssdeep']: 13 | return 14 | 15 | if itype == 'ipv4': 16 | match = re.search(r'^(\S+)\/(\d+)$', data['indicator']) 17 | if match: 18 | data['indicator_ipv4'] = match.group(1) 19 | data['indicator_ipv4_mask'] = match.group(2) 20 | start, end, _ = cidr_to_range(data['indicator']) 21 | data['indicator_iprange'] = { 'gte': start, 'lte': end } 22 | else: 23 | data['indicator_ipv4'] = data['indicator'] 24 | 25 | return 26 | 27 | if itype == 'ipv6': 28 | match = re.search(r'^(\S+)\/(\d+)$', data['indicator']) 29 | if match: 30 | 31 | data['indicator_ipv6'] = binascii.b2a_hex(socket.inet_pton(socket.AF_INET6, match.group(1))).decode( 32 | 'utf-8') 33 | data['indicator_ipv6_mask'] = match.group(2) 34 | start, end, _ = cidr_to_range(data['indicator']) 35 | data['indicator_iprange'] = { 'gte': start, 'lte': end } 36 | else: 37 | data['indicator_ipv6'] = binascii.b2a_hex(socket.inet_pton(socket.AF_INET6, data['indicator'])).decode( 38 | 'utf-8') 39 | 40 | return 41 | 42 | if itype == 'ssdeep': 43 | chunksize, chunk, double_chunk = data['indicator'].split(':') 44 | chunksize = int(chunksize) 45 | data['indicator_ssdeep_chunk'] = chunk 46 | data['indicator_ssdeep_chunksize'] = chunksize 47 | data['indicator_ssdeep_double_chunk'] = double_chunk 48 | 49 | 50 | def cidr_to_range(cidr): 51 | try: 52 | ip = ipaddress.IPv4Network(cidr) 53 | except Exception as e: 54 | ip = ipaddress.IPv6Network(cidr) 55 | 56 | start = str(ip.network_address) 57 | end = str(ip.broadcast_address) 58 | mask = ip.prefixlen 59 | 60 | return start, end, mask 61 | 62 | def _id_random(i): 63 | id = str(uuid.uuid4()) 64 | id = sha256(id.encode('utf-8')).hexdigest() 65 | return id 66 | 67 | 68 | def _id_deterministic(i): 69 | tags = ','.join(sorted(i['tags'])) 70 | groups = ','.join(sorted(i['group'])) 71 | 72 | id = ','.join([groups, i['provider'], i['indicator'], tags]) 73 | ts = i.get('reporttime') 74 | ts = i.get('lasttime') 75 | if ts: 76 | id = '{},{}'.format(id, ts) 77 | 78 | return id 79 | 80 | 81 | def i_to_id(i): 82 | #id = _id_random(i) 83 | id = _id_deterministic(i) 84 | return sha256(id.encode('utf-8')).hexdigest() 85 | -------------------------------------------------------------------------------- /cif/hunter/fqdn_mx.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from csirtg_indicator import resolve_itype 3 | from csirtg_indicator.exceptions import InvalidIndicator 4 | from cif.utils import resolve_ns 5 | from cif.hunter import HUNTER_MIN_CONFIDENCE 6 | from csirtg_indicator import Indicator 7 | from dns.resolver import Timeout 8 | import re 9 | import arrow 10 | 11 | 12 | class FqdnMx(object): 13 | 14 | def __init__(self): 15 | self.logger = logging.getLogger(__name__) 16 | self.is_advanced = True 17 | self.mtypes_supported = { 'indicators_create' } 18 | self.itypes_supported = { 'fqdn' } 19 | 20 | def _prereqs_met(self, i, **kwargs): 21 | if kwargs.get('mtype') not in self.mtypes_supported: 22 | return False 23 | 24 | if i.itype not in self.itypes_supported: 25 | return False 26 | 27 | if 'search' in i.tags: 28 | return False 29 | 30 | return True 31 | 32 | def process(self, i, router, **kwargs): 33 | if not self._prereqs_met(i, **kwargs): 34 | return 35 | 36 | try: 37 | r = resolve_ns(i.indicator, t='MX') 38 | except Timeout: 39 | self.logger.info('timeout trying to resolve MX for: {}'.format(i.indicator)) 40 | return 41 | 42 | try: 43 | for rr in r: 44 | rr = re.sub(r'^\d+ ', '', str(rr)) 45 | rr = str(rr).rstrip('.') 46 | 47 | if rr in ["", 'localhost', '0.0.0.0']: 48 | continue 49 | elif re.match(r'^\d+$', rr) or re.match(r'^.{0,3}$', rr): 50 | # exclude spurious entries like those too short to be real 51 | continue 52 | 53 | mx = Indicator(**i.__dict__()) 54 | mx.indicator = rr.rstrip('.') 55 | mx.lasttime = mx.reporttime = arrow.utcnow() 56 | 57 | try: 58 | resolve_itype(mx.indicator) 59 | except InvalidIndicator as e: 60 | self.logger.info(mx) 61 | self.logger.info(e) 62 | else: 63 | mx.itype = 'fqdn' 64 | if 'hunter' not in mx.tags: 65 | mx.tags.append('hunter') 66 | mx.rdata = '{} mx'.format(i.indicator) 67 | # prevent hunters from running on insertion of this mx 68 | mx.confidence = max(0, min(mx.confidence, HUNTER_MIN_CONFIDENCE - 1)) 69 | router.indicators_create(mx) 70 | self.logger.debug("FQDN MX Hunter: {}".format(mx)) 71 | 72 | except Exception as e: 73 | self.logger.error('[Hunter: FqdnMx] {}: giving up on rr {} from indicator {}'.format(e, rr, i)) 74 | 75 | Plugin = FqdnMx -------------------------------------------------------------------------------- /cif/httpd/common.py: -------------------------------------------------------------------------------- 1 | from flask import request, jsonify 2 | import re 3 | import zlib 4 | import gzip 5 | 6 | VALID_FILTERS = { 7 | 'indicator', 'itype', 'confidence', 'provider', 'limit', 'application', 'nolog', 'tags', 'days', 8 | 'hours', 'groups', 'reporttime', 'cc', 'asn', 'asn_desc', 'rdata', 'firsttime', 'lasttime', 'region', 'id', 9 | 'portlist', 'protocol', 'tlp', 'sort', 10 | } 11 | TOKEN_FILTERS = ['username', 'token'] 12 | 13 | 14 | def pull_token(): 15 | t = None 16 | if request.headers.get("Authorization"): 17 | t = re.match(r"^Token token=(\S+)$", request.headers.get("Authorization")) 18 | if t: 19 | t = t.group(1) 20 | return t 21 | 22 | 23 | def request_v2(): 24 | if request.headers.get('Accept'): 25 | if 'vnd.cif.v2+json' in request.headers['Accept']: 26 | return True 27 | 28 | 29 | def jsonify_unauth(msg='unauthorized'): 30 | response = jsonify({ 31 | "message": msg, 32 | "data": [] 33 | }) 34 | response.status_code = 401 35 | return response 36 | 37 | 38 | def jsonify_unknown(msg='failed', code=503): 39 | response = jsonify({ 40 | "message": msg, 41 | "data": [] 42 | }) 43 | response.status_code = code 44 | return response 45 | 46 | 47 | def jsonify_busy(msg='system is busy, try again later', code=503): 48 | response = jsonify({ 49 | 'message': msg, 50 | 'data': [], 51 | }) 52 | response.status_code = code 53 | return response 54 | 55 | 56 | def jsonify_success(data=[], code=200): 57 | response = jsonify({ 58 | 'message': 'success', 59 | 'data': data 60 | }) 61 | response.status_code = code 62 | return response 63 | 64 | 65 | def response_compress(): 66 | if request.args.get('nocompress'): 67 | return False 68 | 69 | if request.args.get('gzip'): 70 | return 'gzip' 71 | 72 | if request.headers.get('Accept-Encoding'): 73 | if request.headers['Accept-Encoding'] == 'gzip': 74 | return 'gzip' 75 | 76 | if request.headers['Accept-Encoding'] == 'deflate': 77 | return 'deflate' 78 | 79 | 80 | def compress(data, ctype='deflate'): 81 | if ctype == 'deflate': 82 | return zlib.compress(data) 83 | 84 | return gzip.compress(data) 85 | 86 | 87 | def aggregate(data, field='indicator', sort='confidence', sort_secondary='reporttime', dedup_only=False): 88 | x = set() 89 | rv = [] 90 | if dedup_only: 91 | for d in data: 92 | if d[field] not in x: 93 | x.add(d[field]) 94 | rv.append(d) 95 | return rv 96 | 97 | for d in sorted(data, key=lambda x: x[sort], reverse=True): 98 | if d[field] not in x: 99 | x.add(d[field]) 100 | rv.append(d) 101 | 102 | rv = sorted(rv, key=lambda x: x[sort_secondary], reverse=True) 103 | return rv 104 | -------------------------------------------------------------------------------- /cif/httpd/views/u/indicators.py: -------------------------------------------------------------------------------- 1 | from flask.views import MethodView 2 | from flask import flash 3 | from flask import request, render_template, session 4 | from cifsdk.client.zeromq import ZMQ as Client 5 | from cif.constants import ROUTER_ADDR 6 | import logging 7 | import arrow 8 | 9 | remote = ROUTER_ADDR 10 | 11 | logger = logging.getLogger('cif-httpd') 12 | 13 | 14 | class IndicatorsUI(MethodView): 15 | def get(self): 16 | session['filters'] = {} 17 | now = arrow.utcnow() 18 | 19 | if request.args.get('q'): 20 | session['filters']['q'] = request.args.get('q') 21 | if request.args.get('confidence'): 22 | session['filters']['confidence'] = request.args.get('confidence') 23 | if request.args.get('provider'): 24 | session['filters']['provider'] = request.args.get('provider') 25 | if request.args.get('group'): 26 | session['filters']['group'] = request.args.get('group') 27 | if request.args.get('tags'): 28 | session['filters']['tags'] = request.args.get('tags') 29 | if request.args.get('starttime') or request.args.get('endtime'): 30 | if request.args.get('starttime'): 31 | starttime = request.args.get('starttime') + 'T00:00:00Z' 32 | else: 33 | starttime = '1900-01-01T00:00:00Z' 34 | if request.args.get('endtime'): 35 | endtime = request.args.get('endtime') + 'T23:59:59Z' 36 | else: 37 | endtime = '{0}Z'.format(now.format('YYYY-MM-DDT23:59:59')) 38 | 39 | session['filters']['reporttime'] = '%s,%s' % (starttime, endtime) 40 | 41 | response = render_template('indicators.html') 42 | 43 | return response 44 | 45 | def post(self): 46 | pass 47 | 48 | 49 | def DataTables(): 50 | filters = {} 51 | 52 | if session['filters'].get('q'): 53 | q = session['filters'].get('q') 54 | if q in ['ipv4', 'ipv6', 'fqdn', 'url', 'email']: 55 | filters['itype'] = q 56 | else: 57 | filters['indicator'] = q 58 | 59 | if session['filters'].get('confidence'): 60 | filters['confidence'] = session['filters'].get('confidence') 61 | if session['filters'].get('provider'): 62 | filters['provider'] = session['filters'].get('provider') 63 | if session['filters'].get('group'): 64 | filters['groups'] = session['filters'].get('group') 65 | if session['filters'].get('tags'): 66 | filters['tags'] = session['filters'].get('tags') 67 | if session['filters'].get('reporttime'): 68 | filters['reporttime'] = session['filters'].get('reporttime') 69 | 70 | filters['find_relatives'] = True 71 | 72 | if not session['filters']: 73 | return [] 74 | 75 | try: 76 | r = Client(remote, session['token']).indicators_search(filters) 77 | except Exception as e: 78 | logger.error(e) 79 | flash(e, 'error') 80 | response = [] 81 | else: 82 | response = r 83 | finally: 84 | session['filters'] = {} 85 | 86 | return response 87 | -------------------------------------------------------------------------------- /cif/store/zelasticsearch/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | 4 | from cifsdk.exceptions import CIFException 5 | from cif.store.plugin import Store 6 | from cif.store.zelasticsearch.token import TokenManager 7 | from cif.store.zelasticsearch.indicator import IndicatorManager 8 | from cif.utils import strtobool 9 | from elasticsearch_dsl.connections import connections 10 | from elasticsearch.exceptions import ConnectionError 11 | import traceback 12 | from time import sleep 13 | 14 | ES_NODES = os.getenv('CIF_ES_NODES', '127.0.0.1:9200') 15 | TRACE = strtobool(os.environ.get('CIF_STORE_ES_TRACE', False)) 16 | TRACE_HTTP = strtobool(os.environ.get('CIF_STORE_ES_HTTP_TRACE', False)) 17 | 18 | logger = logging.getLogger(__name__) 19 | logger.setLevel(logging.ERROR) 20 | 21 | logging.getLogger('urllib3').setLevel(logging.ERROR) 22 | logging.getLogger('elasticsearch').setLevel(logging.ERROR) 23 | 24 | if TRACE: 25 | logger.setLevel(logging.DEBUG) 26 | 27 | if TRACE_HTTP: 28 | logging.getLogger('urllib3').setLevel(logging.INFO) 29 | logging.getLogger('elasticsearch').setLevel(logging.DEBUG) 30 | 31 | 32 | class _ElasticSearch(Store): 33 | # http://stackoverflow.com/questions/533631/what-is-a-mixin-and-why-are-they-useful 34 | 35 | name = 'elasticsearch' 36 | 37 | def __init__(self, nodes=ES_NODES, **kwargs): 38 | 39 | if type(nodes) == str: 40 | nodes = nodes.split(',') 41 | 42 | if not nodes: 43 | nodes = ES_NODES 44 | 45 | self.indicators_prefix = kwargs.get('indicators_prefix', 'indicators') 46 | self.tokens_prefix = kwargs.get('tokens_prefix', 'tokens') 47 | self.token_cache = kwargs.get('token_cache', {}) 48 | 49 | logger.info('setting es nodes {}'.format(nodes)) 50 | 51 | connections.create_connection(hosts=nodes) 52 | 53 | self._alive = False 54 | 55 | while not self._alive: 56 | if not self._health_check(): 57 | logger.warn('ES cluster not accessible') 58 | logger.info('retrying connection in 30s') 59 | sleep(30) 60 | 61 | self._alive = True 62 | 63 | logger.info('ES connection successful') 64 | logger.info('CIF_STORE_ES_HTTP_TRACE set to {}'.format(TRACE_HTTP)) 65 | self.tokens = TokenManager(token_cache=self.token_cache) 66 | self.indicators = IndicatorManager() 67 | 68 | def _health_check(self): 69 | try: 70 | x = connections.get_connection().cluster.health() 71 | except ConnectionError as e: 72 | logger.warn('elasticsearch connection error') 73 | logger.error(e) 74 | return 75 | 76 | except Exception as e: 77 | logger.error(traceback.print_exc()) 78 | return 79 | 80 | logger.info('ES cluster is: %s' % x['status']) 81 | return x 82 | 83 | def ping(self): 84 | s = self._health_check() 85 | 86 | if s is None or s['status'] == 'red': 87 | raise CIFException('ES Cluster Issue') 88 | 89 | return True 90 | 91 | 92 | Plugin = _ElasticSearch 93 | -------------------------------------------------------------------------------- /cif/hunter/farsight.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from csirtg_dnsdb.client import Client 3 | from csirtg_dnsdb.exceptions import QuotaLimit 4 | import os 5 | from csirtg_indicator import Indicator, InvalidIndicator 6 | import arrow 7 | import re 8 | 9 | TOKEN = os.environ.get('FARSIGHT_TOKEN') 10 | PROVIDER = os.environ.get('FARSIGHT_PROVIDER', 'dnsdb.info') 11 | MAX_QUERY_RESULTS = os.environ.get('FARSIGHT_QUERY_MAX', 10000) 12 | 13 | logger = logging.getLogger(__name__) 14 | 15 | 16 | class Farsight(object): 17 | 18 | def __init__(self, *args, **kwargs): 19 | self.logger = logging.getLogger(__name__) 20 | self.client = Client() 21 | self.token = kwargs.get('token', TOKEN) 22 | self.is_advanced = True 23 | self.mtypes_supported = { 'indicators_search' } 24 | self.itypes_supported = { 'ipv4' } 25 | 26 | def _prereqs_met(self, i, **kwargs): 27 | if kwargs.get('mtype') not in self.mtypes_supported: 28 | return False 29 | 30 | if not self.token: 31 | return False 32 | 33 | if i.itype not in self.itypes_supported: 34 | return False 35 | 36 | if 'search' not in i.tags: 37 | return False 38 | 39 | if i.confidence and i.confidence < 9: 40 | return False 41 | 42 | if re.search(r'^(\S+)\/(\d+)$', i.indicator): 43 | return False 44 | 45 | return True 46 | 47 | def process(self, i, router, **kwargs): 48 | if not self._prereqs_met(i, **kwargs): 49 | return 50 | 51 | max = MAX_QUERY_RESULTS 52 | 53 | try: 54 | for r in self.client.search(i.indicator): 55 | first = arrow.get(r.get('time_first') or r.get('zone_time_first')) 56 | first = first.datetime 57 | last = arrow.get(r.get('time_last') or r.get('zone_time_last')) 58 | last = last.datetime 59 | 60 | reporttime = arrow.utcnow().datetime 61 | 62 | r['rrname'] = r['rrname'].rstrip('.') 63 | 64 | try: 65 | ii = Indicator( 66 | indicator=r['rdata'], 67 | rdata=r['rrname'].rstrip('.'), 68 | count=r['count'], 69 | tags=['pdns', 'hunter'], 70 | confidence=10, 71 | firsttime=first, 72 | lasttime=last, 73 | reporttime=reporttime, 74 | provider=PROVIDER, 75 | tlp='amber', 76 | group='everyone' 77 | ) 78 | except InvalidIndicator as e: 79 | self.logger.error(e) 80 | return 81 | 82 | router.indicators_create(ii) 83 | max -= 1 84 | if max == 0: 85 | break 86 | 87 | except QuotaLimit: 88 | logger.warn('farsight quota limit reached... skipping') 89 | except Exception as e: 90 | logger.exception(e) 91 | return 92 | 93 | 94 | Plugin = Farsight -------------------------------------------------------------------------------- /cif/utils/asn_client.py: -------------------------------------------------------------------------------- 1 | import ujson as json 2 | import logging 3 | import sys 4 | import zmq 5 | 6 | logger = logging.getLogger(__name__) 7 | 8 | 9 | class Timeout(Exception): 10 | pass 11 | 12 | 13 | def chunk(it, slice=50): 14 | """Generate sublists from an iterator 15 | >>> list(chunk(iter(range(10)),11)) 16 | [[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]] 17 | >>> list(chunk(iter(range(10)),9)) 18 | [[0, 1, 2, 3, 4, 5, 6, 7, 8], [9]] 19 | >>> list(chunk(iter(range(10)),5)) 20 | [[0, 1, 2, 3, 4], [5, 6, 7, 8, 9]] 21 | >>> list(chunk(iter(range(10)),3)) 22 | [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]] 23 | >>> list(chunk(iter(range(10)),1)) 24 | [[0], [1], [2], [3], [4], [5], [6], [7], [8], [9]] 25 | """ 26 | 27 | assert(slice > 0) 28 | a=[] 29 | 30 | for x in it: 31 | if len(a) >= slice : 32 | yield a 33 | a=[] 34 | a.append(x) 35 | 36 | if a: 37 | yield a 38 | 39 | 40 | class ASNClient: 41 | def __init__(self, endpoint='tcp://localhost:5555'): 42 | context = zmq.Context() 43 | logger.debug("Connecting to asn lookup server") 44 | socket = context.socket(zmq.DEALER) 45 | socket.set(zmq.LINGER, 200) 46 | socket.connect(endpoint) 47 | self.socket = socket 48 | 49 | 50 | self.get_fields() 51 | 52 | def get_fields(self): 53 | self.socket.send_string("fields") 54 | if not self.socket.poll(timeout=3000): 55 | raise Timeout() 56 | 57 | self.fields = json.loads(self.socket.recv_string()) 58 | logger.debug("fields=%s", self.fields) 59 | 60 | def lookup_many(self, ips): 61 | outstanding = 0 62 | 63 | for batch in chunk(ips, 100): 64 | msg = ' '.join(batch) 65 | self.socket.send_string(msg) 66 | outstanding += 1 67 | if outstanding < 10: 68 | continue 69 | if not self.socket.poll(timeout=3000): 70 | raise Timeout() 71 | response = self.socket.recv_string() 72 | outstanding -=1 73 | records = json.loads(response) 74 | for rec in records: 75 | yield dict(zip(self.fields, rec)) 76 | 77 | for _ in range(outstanding): 78 | if not self.socket.poll(timeout=3000): 79 | raise Timeout() 80 | response = self.socket.recv_string() 81 | outstanding -=1 82 | records = json.loads(response) 83 | for rec in records: 84 | yield dict(zip(self.fields, rec)) 85 | 86 | def lookup(self, ip): 87 | return next(self.lookup_many([ip])) 88 | 89 | 90 | def main(): 91 | logging.basicConfig(level=logging.INFO) 92 | 93 | endpoint = 'tcp://localhost:5555' 94 | if len(sys.argv) > 1: 95 | endpoint = sys.argv[1] 96 | c = ASNClient(endpoint) 97 | ips = (line.rstrip() for line in sys.stdin) 98 | for rec in c.lookup_many(ips): 99 | print("\t".join(str(rec[f]) for f in c.fields)) 100 | 101 | if __name__ == "__main__": 102 | main() -------------------------------------------------------------------------------- /test/zelasticsearch/test_store_elasticsearch_tokens.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from csirtg_indicator import Indicator 3 | from cif.store import Store 4 | from elasticsearch_dsl.connections import connections 5 | import os 6 | import arrow 7 | 8 | DISABLE_TESTS = True 9 | if os.environ.get('CIF_ELASTICSEARCH_TEST'): 10 | if os.environ['CIF_ELASTICSEARCH_TEST'] == '1': 11 | DISABLE_TESTS = False 12 | 13 | 14 | @pytest.fixture 15 | def store(): 16 | try: 17 | connections.get_connection().indices.delete(index='indicators-*') 18 | connections.get_connection().indices.delete(index='tokens') 19 | except Exception as e: 20 | pass 21 | 22 | with Store(store_type='elasticsearch', nodes='127.0.0.1:9200') as s: 23 | s._load_plugin(nodes='127.0.0.1:9200') 24 | yield s 25 | 26 | try: 27 | assert connections.get_connection().indices.delete(index='indicators-*') 28 | assert connections.get_connection().indices.delete(index='tokens') 29 | except Exception: 30 | pass 31 | 32 | 33 | @pytest.fixture 34 | def token(store): 35 | t = store.store.tokens.create({ 36 | 'username': u'test_admin', 37 | 'groups': [u'everyone'], 38 | 'read': u'1', 39 | 'write': u'1', 40 | 'admin': u'1' 41 | }) 42 | 43 | assert t 44 | yield t 45 | 46 | @pytest.fixture 47 | def indicator(): 48 | return Indicator( 49 | indicator='example.com', 50 | tags='botnet', 51 | provider='csirtg.io', 52 | group='everyone', 53 | lasttime=arrow.utcnow().datetime, 54 | reporttime=arrow.utcnow().datetime 55 | ) 56 | 57 | 58 | @pytest.mark.skipif(DISABLE_TESTS, reason='need to set CIF_ELASTICSEARCH_TEST=1 to run') 59 | def test_store_elasticsearch_tokens(store, token): 60 | # the below funcs should cache the checked token 61 | assert store.store.tokens.check(token, 'read') 62 | assert store.store.tokens.read(token) 63 | assert store.store.tokens.write(token) 64 | assert store.store.tokens.admin(token) 65 | assert store.store.tokens._cache_check(token['token']) is not False 66 | 67 | 68 | @pytest.mark.skipif(DISABLE_TESTS, reason='need to set CIF_ELASTICSEARCH_TEST=1 to run') 69 | def test_store_elasticsearch_tokens_advanced(store, token): 70 | x = store.handle_tokens_search(token, {'token': token['token']}) 71 | assert len(list(x)) > 0 72 | 73 | x = store.handle_tokens_search(token, {'admin': True}) 74 | assert len(list(x)) > 0 75 | 76 | x = store.handle_tokens_search(token, {'write': True}) 77 | assert len(list(x)) > 0 78 | 79 | t = store.store.tokens.create({ 80 | 'username': u'test_admin2', 81 | 'groups': [u'everyone'] 82 | }) 83 | 84 | x = store.handle_tokens_search(token, {'admin': True}) 85 | assert len(list(x)) == 1 86 | 87 | x = store.handle_tokens_search(token, {}) 88 | assert len(list(x)) == 2 89 | 90 | # test last_activity_at 91 | now_str = arrow.utcnow().datetime.strftime('%Y-%m-%dT%H:%M:%S.%fZ') 92 | # the below func should update last_activity_at and cache that 93 | assert store.store.tokens.auth_search({'token': t['token']}) 94 | 95 | assert store.store.tokens.last_activity_at(t) > now_str 96 | -------------------------------------------------------------------------------- /contributing.md: -------------------------------------------------------------------------------- 1 | How We Work Together 2 | == 3 | The CSIRT Gadgets Community uses the [C4 process](https://github.com/csirtgadgets/c4) for it's core projects. If you contribute a nice patch and you have some GitHub reputation we'll invite you to join the Maintainer's team. We strive to adopt most [if not all] of the wonderful outcomes the [ZMQ community](http://zguide.zeromq.org/page:all#toc130) has pionneered. If you're not familar with it, please read up on it's architecture and history, it's a great story! 4 | 5 | GitHub also has a [great piece](https://guides.github.com/activities/contributing-to-open-source) on the nuances associated with contributing to opensource projects. If you've never worked within a github project before, this is a great "where do i start?". 6 | 7 | Our Process 8 | === 9 | 10 | * Log an issue that explains the problem you are solving. 11 | * Provide a test case unless absolutely impossible. 12 | * Get familar with [GitHub](https://help.github.com/articles/set-up-git) and [GitFlow](http://datasift.github.io/gitflow/IntroducingGitFlow.html) 13 | * We always need an issue, test case and [pull-request](https://help.github.com/articles/using-pull-requests). 14 | * Make your change as a [pull request](https://github.com/blog/1943-how-to-write-the-perfect-pull-request) (see below). 15 | * Discuss on the mailing list as needed. 16 | * Close the issue when your pull request is merged and the test case passes. 17 | 18 | Separate Your Changes 19 | === 20 | Separate different independent logical changes into separate commits (and thus separate patch submissions) when at all possible. This allows each change to be considered on it's own merits. Also, it is easier to review a batch of independent [smaller] changes rather than one large patch. 21 | 22 | Write Good Commit Messages 23 | === 24 | Commit messages become the public record of your changes, as such it's important that they be well-written. The basic format of git commit messages is: 25 | 26 | * A single summary line. This should be short — no more than 70 characters or so, since it can be used as the e-mail subject when submitting your patch and also for generating patch file names by 'git format-patch'. If your change only touches a single file or subsystem you may wish to prefix the summary with the file or subsystem name. 27 | * A blank line. 28 | * A detailed description of your change. Where possible, write in the present tense, e.g. "Add assertions to funct_foo()". If your changes have not resulted from previous discussion on the mailing list you may also wish to include brief rationale on your change. Your description should be formatted as plain text with each line not exceeding 80 characters. 29 | 30 | Give Yourself Credit 31 | === 32 | Instead of a traditional 'AUTHORS' file, contributors are automatically tracked [here](https://github.com/csirtgadgets/bearded-avenger/graphs/contributors). 33 | 34 | Copyrights and Licenses 35 | === 36 | Make sure your contributions do not include code from projects with incompatible licenses. Our projects mostly use the LGPLv3 with a static linking exception. If your code isn't compatible with this, it will sooner or later be spotted and removed. The best way to avoid any license issues is to write your own code. 37 | 38 | Test Cases 39 | === 40 | For stable releases, patches (if they change the behavior of the code) must have issues, and test cases. -------------------------------------------------------------------------------- /cif/store/sqlite/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | 4 | from sqlalchemy import create_engine, event 5 | from sqlalchemy.ext.declarative import declarative_base 6 | from sqlalchemy.orm import sessionmaker, scoped_session 7 | from sqlalchemy.engine import Engine 8 | import sqlite3 9 | 10 | from cifsdk.constants import RUNTIME_PATH 11 | from cif.store.plugin import Store 12 | from cif.utils import strtobool 13 | from cifsdk.constants import PYVERSION 14 | 15 | Base = declarative_base() 16 | from .token import TokenManager 17 | from .indicator import IndicatorManager 18 | 19 | DATA_PATH = os.getenv('CIF_DATA_PATH') 20 | DB_FILE = os.path.join(RUNTIME_PATH, 'cif.sqlite') 21 | 22 | if DATA_PATH: 23 | DB_FILE = os.path.join(DATA_PATH, 'cif.db') 24 | 25 | logger = logging.getLogger(__name__) 26 | TRACE = strtobool(os.environ.get('CIF_STORE_SQLITE_TRACE', False)) 27 | 28 | # http://stackoverflow.com/q/9671490/7205341 29 | SYNC = os.environ.get('CIF_STORE_SQLITE_SYNC', 'NORMAL') 30 | 31 | AUTOFLUSH = os.getenv('CIF_STORE_SQLITE_AUTOFLUSH', '1') 32 | if AUTOFLUSH == '0': 33 | AUTOFLUSH = False 34 | else: 35 | AUTOFLUSH = True 36 | 37 | 38 | # https://www.sqlite.org/pragma.html#pragma_cache_size 39 | CACHE_SIZE = os.environ.get('CIF_STORE_SQLITE_CACHE_SIZE', 512000000) # 256MB 40 | 41 | logger = logging.getLogger(__name__) 42 | logger.setLevel(logging.DEBUG) 43 | logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) 44 | 45 | if not TRACE: 46 | logger.setLevel(logging.ERROR) 47 | logging.getLogger('sqlalchemy.engine').setLevel(logging.ERROR) 48 | 49 | if PYVERSION > 2: 50 | basestring = (str, bytes) 51 | 52 | 53 | @event.listens_for(Engine, "connect") 54 | def set_sqlite_pragma(dbapi_connection, connection_record): 55 | cursor = dbapi_connection.cursor() 56 | cursor.execute("PRAGMA foreign_keys=ON") 57 | cursor.execute("PRAGMA journal_mode = MEMORY") 58 | cursor.execute("PRAGMA synchronous = {}".format(SYNC)) 59 | cursor.execute("PRAGMA temp_store = MEMORY") 60 | cursor.execute("PRAGMA cache_size = {}".format(CACHE_SIZE)) 61 | cursor.close() 62 | 63 | 64 | class SQLite(Store): 65 | # http://www.pythoncentral.io/sqlalchemy-orm-examples/ 66 | name = 'sqlite' 67 | 68 | def __init__(self, dbfile=DB_FILE, autocommit=False, autoflush=AUTOFLUSH, dictrows=True, **kwargs): 69 | self.logger = logging.getLogger(__name__) 70 | 71 | self.dbfile = dbfile 72 | self.autocommit = autocommit 73 | self.autoflush = autoflush 74 | self.dictrows = dictrows 75 | self.path = "sqlite:///{0}".format(self.dbfile) 76 | 77 | echo = False 78 | if TRACE: 79 | echo = False 80 | 81 | # http://docs.sqlalchemy.org/en/latest/orm/contextual.html 82 | self.engine = create_engine(self.path, echo=echo) 83 | self.handle = sessionmaker(bind=self.engine, autocommit=autocommit, autoflush=autoflush) 84 | self.handle = scoped_session(self.handle) 85 | 86 | Base.metadata.create_all(self.engine) 87 | 88 | self.logger.debug('database path: {}'.format(self.path)) 89 | 90 | self.token_cache = kwargs.get('token_cache', {}) 91 | 92 | self.tokens = TokenManager(self.handle, self.engine, token_cache=self.token_cache) 93 | self.indicators = IndicatorManager(self.handle, self.engine) 94 | 95 | def ping(self): 96 | return True 97 | 98 | Plugin = SQLite 99 | -------------------------------------------------------------------------------- /test/zsqlite/test_store_sqlite_indicators_nonpersistent.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import tempfile 4 | from argparse import Namespace 5 | import pytest 6 | from cif.store import Store 7 | from cifsdk.utils import setup_logging 8 | import arrow 9 | from csirtg_indicator.exceptions import InvalidIndicator 10 | import copy 11 | from pprint import pprint 12 | 13 | args = Namespace(debug=True, verbose=None) 14 | setup_logging(args) 15 | 16 | logger = logging.getLogger(__name__) 17 | 18 | @pytest.fixture 19 | def store(): 20 | dbfile = tempfile.mktemp() 21 | with Store(store_type='sqlite', dbfile=dbfile) as s: 22 | s._load_plugin(dbfile=dbfile) 23 | s.token_create_admin() 24 | yield s 25 | 26 | s = None 27 | if os.path.isfile(dbfile): 28 | os.unlink(dbfile) 29 | 30 | @pytest.fixture 31 | def token(store): 32 | t = store.store.tokens.create({ 33 | 'username': u'test_sqlite_admin', 34 | 'groups': [u'everyone'], 35 | 'read': u'1', 36 | 'write': u'1', 37 | 'admin': u'1' 38 | }) 39 | 40 | assert t 41 | yield t 42 | 43 | 44 | @pytest.fixture 45 | def indicator(): 46 | now = arrow.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%fZ') 47 | return { 48 | 'indicator': 'example.com', 49 | 'tags': 'botnet', 50 | 'provider': 'csirtgadgets.org', 51 | 'group': 'everyone', 52 | 'lasttime': now, 53 | 'itype': 'fqdn', 54 | 'confidence': 6 55 | } 56 | 57 | 58 | def test_store_indicators_search_reporttime(store, token, indicator): 59 | now = arrow.utcnow() 60 | now_str = now.datetime.strftime('%Y-%m-%dT%H:%M:%SZ') 61 | days_ago_arrow = now.shift(days=-5) 62 | days_ago_str = days_ago_arrow.datetime.strftime('%Y-%m-%dT%H:%M:%SZ') 63 | weeks_ago_str = now.shift(weeks=-3).datetime.strftime('%Y-%m-%dT%H:%M:%SZ') 64 | 65 | indicator['reporttime'] = indicator['lasttime'] = weeks_ago_str 66 | 67 | indicator_url = copy.deepcopy(indicator) 68 | indicator_url['indicator'] = 'https://example.com' 69 | indicator_url['reporttime'] = indicator_url['lasttime'] = days_ago_str 70 | 71 | indicator_alt_provider = copy.deepcopy(indicator) 72 | indicator_alt_provider['provider'] = 'csirtg.io' 73 | indicator_alt_provider['reporttime'] = indicator_alt_provider['lasttime'] = now_str 74 | 75 | x = store.handle_indicators_create(token, indicator, flush=True) 76 | assert x > 0 77 | 78 | y = store.handle_indicators_create(token, indicator_url, flush=True) 79 | assert y > 0 80 | 81 | z = store.handle_indicators_create(token, indicator_alt_provider, flush=True) 82 | assert z > 0 83 | 84 | x = store.handle_indicators_search(token, { 85 | 'days': '6' 86 | }) 87 | 88 | pprint(x) 89 | 90 | assert len(x) == 2 91 | 92 | for indicator in x: 93 | assert arrow.get(indicator['reporttime']) >= arrow.get(days_ago_str) 94 | 95 | start_str = weeks_ago_str 96 | end_str = days_ago_str 97 | startend = '{},{}'.format(start_str, end_str) 98 | 99 | y = store.handle_indicators_search(token, { 100 | 'reporttime': startend 101 | }) 102 | 103 | pprint(y) 104 | 105 | assert len(y) == 2 106 | 107 | for indicator in y: 108 | assert arrow.get(indicator['reporttime']) <= arrow.get(days_ago_str) 109 | -------------------------------------------------------------------------------- /cif/gatherer/asn.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import re 4 | 5 | from cif.utils import resolve_ns 6 | from cif.utils.asn_client import ASNClient 7 | 8 | ASN_FAST = os.environ.get('CIF_GATHERER_ASN_FAST') 9 | ENABLE_PEERS = os.environ.get('CIF_GATHERERS_PEERS_ENABLED') 10 | 11 | 12 | class Asn(object): 13 | 14 | def __init__(self, *args, **kwargs): 15 | self.logger = logging.getLogger(__name__) 16 | self.asn_fast = kwargs.get('fast', ASN_FAST) 17 | if self.asn_fast: 18 | self.asn_fast = ASNClient(self.asn_fast) 19 | 20 | self.enabled = kwargs.get('enabled', os.environ.get('CIF_GATHERERS_PEERS_ENABLED')) 21 | 22 | def _resolve(self, data): 23 | return resolve_ns('{}.{}'.format(data, 'origin.asn.cymru.com'), t='TXT') 24 | 25 | def _resolve_fast(self, data): 26 | return self.asn_fast.lookup(data) 27 | 28 | def process(self, indicator): 29 | if not self.enabled: 30 | return 31 | 32 | if indicator.is_private(): 33 | return 34 | 35 | # TODO ipv6 36 | if indicator.itype != 'ipv4': 37 | return 38 | 39 | i = str(indicator.indicator) 40 | match = re.search(r'^(\S+)\/\d+$', i) 41 | if match: 42 | i = match.group(1) 43 | 44 | if self.asn_fast: 45 | bits = self._resolve_fast(indicator.indicator) 46 | for k in bits: 47 | if bits[k] == 'NA': 48 | bits[k] = False 49 | 50 | if bits['asn']: 51 | bits['asn'] = str(bits['asn']) 52 | 53 | indicator.asn = bits['asn'] 54 | indicator.prefix = bits['prefix'] 55 | indicator.asn_desc = bits['owner'] 56 | indicator.cc = bits['cc'] 57 | 58 | return indicator 59 | 60 | # cache it to the /24 61 | # 115.87.213.115 62 | # 0.213.87.115 63 | i = list(reversed(i.split('.'))) 64 | i = '0.{}.{}.{}'.format(i[1], i[2], i[3]) 65 | 66 | answers = self._resolve(i) 67 | 68 | if len(answers) > 0: 69 | # Separate fields and order by netmask length 70 | # 23028 | 216.90.108.0/24 | US | arin | 1998-09-25 71 | # 701 1239 3549 3561 7132 | 216.90.108.0/24 | US | arin | 1998-09-25 72 | 73 | # i.asn_desc ???? 74 | self.logger.debug(answers[0]) 75 | bits = str(answers[0]).replace('"', '').strip().split(' | ') 76 | asns = bits[0].split(' ') 77 | 78 | indicator.asn = asns[0] 79 | indicator.prefix = bits[1] 80 | indicator.cc = bits[2] 81 | indicator.rir = bits[3] 82 | answers = resolve_ns('as{}.{}'.format(asns[0], 'asn.cymru.com'), t='TXT', timeout=15) 83 | 84 | try: 85 | tmp = str(answers[0]) 86 | except UnicodeDecodeError as e: 87 | # requires fix latin-1 fix _escapeify to dnspython > 1.14 88 | self.logger.debug(e) 89 | return indicator 90 | except IndexError: 91 | from pprint import pprint 92 | pprint(answers) 93 | return indicator 94 | 95 | bits = tmp.replace('"', '').strip().split(' | ') 96 | if len(bits) > 4: 97 | indicator.asn_desc = bits[4] 98 | 99 | # send back to router 100 | return indicator 101 | 102 | Plugin = Asn -------------------------------------------------------------------------------- /reindex_tokens.py: -------------------------------------------------------------------------------- 1 | from elasticsearch_dsl import DocType, String, Date, Integer, Boolean, Float, Ip, GeoPoint, Keyword, Index 2 | from elasticsearch_dsl.connections import connections 3 | import time 4 | import os 5 | ### If you need to restore the tokens index to the previous schema run the following from a python interactive shell: 6 | ### from reindex_tokens import * 7 | ### restore_tokens() 8 | 9 | INDEX_NAME = 'tokens' 10 | BACKUP_INDEX_NAME = 'tokens_backup' 11 | ES_NODES = os.getenv('CIF_ES_NODES', '127.0.0.1:9200') 12 | connections.create_connection(hosts=ES_NODES) 13 | 14 | 15 | class TokenBackup(DocType): 16 | username = Keyword() 17 | token = Keyword() 18 | expires = Date() 19 | read = Boolean() 20 | write = Boolean() 21 | revoked = Boolean() 22 | acl = Keyword() 23 | groups = Keyword() 24 | admin = Boolean() 25 | last_activity_at = Date() 26 | 27 | class Meta: 28 | index = BACKUP_INDEX_NAME 29 | 30 | 31 | class Token(DocType): 32 | username = Keyword() 33 | token = Keyword() 34 | expires = Date() 35 | read = Boolean() 36 | write = Boolean() 37 | revoked = Boolean() 38 | acl = Keyword() 39 | groups = Keyword() 40 | admin = Boolean() 41 | last_activity_at = Date() 42 | 43 | class Meta: 44 | index = INDEX_NAME 45 | 46 | 47 | def reindex_tokens(): 48 | TokenBackup.init() 49 | connections.create_connection(hosts=ES_NODES) 50 | backup_results = connections.get_connection().reindex(body={"source": {"index": INDEX_NAME}, "dest": {"index": BACKUP_INDEX_NAME}}, request_timeout=3600) 51 | if backup_results.get('created') + backup_results.get('updated') == backup_results.get('total'): 52 | Index(INDEX_NAME).delete() 53 | else: 54 | return ('Tokens did not backup properly') 55 | time.sleep(1) 56 | Token.init() 57 | reindex_results = connections.get_connection().reindex(body={"source": {"index": BACKUP_INDEX_NAME}, "dest": {"index": INDEX_NAME}}, request_timeout=3600) 58 | if reindex_results.get('created') + reindex_results.get('updated') == reindex_results.get('total'): 59 | return ('Tokens reindexed successfully!') 60 | else: 61 | return ('Tokens did not reindex from backup properly') 62 | 63 | 64 | def restore_tokens(): 65 | connections.create_connection(hosts=ES_NODES) 66 | Index(INDEX_NAME).delete() 67 | 68 | class Token(DocType): 69 | username = String() 70 | token = String() 71 | expires = Date() 72 | read = Boolean() 73 | write = Boolean() 74 | revoked = Boolean() 75 | acl = String() 76 | groups = String() 77 | admin = Boolean() 78 | last_activity_at = Date() 79 | 80 | class Meta: 81 | index = INDEX_NAME 82 | 83 | Token.init() 84 | reindex_results = connections.get_connection().reindex(body={"source": {"index": BACKUP_INDEX_NAME}, "dest": {"index": INDEX_NAME}}, request_timeout=3600) 85 | if reindex_results.get('created') + reindex_results.get('updated') == reindex_results.get('total'): 86 | return ('Tokens restored to previous schema successfully!') 87 | else: 88 | return ('Tokens did not restore from backup properly') 89 | 90 | 91 | def main(): 92 | results = reindex_tokens() 93 | if results == 'Tokens reindexed successfully!': 94 | print("Tokens reindexed successfully!") 95 | else: 96 | print("Tokens did not reindex properly") 97 | 98 | 99 | if __name__ == '__main__': 100 | main() 101 | -------------------------------------------------------------------------------- /cif/httpd/templates/nav.html: -------------------------------------------------------------------------------- 1 | {% block navbar %} 2 | 70 | {% endblock %} 71 | -------------------------------------------------------------------------------- /cif/gatherer/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import ujson as json 4 | import logging 5 | import traceback 6 | import zmq 7 | import multiprocessing 8 | from cifsdk.msg import Msg 9 | import os 10 | import cif.gatherer 11 | from cif.constants import GATHERER_ADDR, GATHERER_SINK_ADDR 12 | from cif.utils import strtobool 13 | from csirtg_indicator import Indicator, InvalidIndicator 14 | import time 15 | 16 | SNDTIMEO = 30000 17 | LINGER = 0 18 | 19 | logger = logging.getLogger(__name__) 20 | logger.setLevel(logging.ERROR) 21 | 22 | TRACE = strtobool(os.environ.get('CIF_GATHERER_TRACE', False)) 23 | if TRACE: 24 | logger.setLevel(logging.DEBUG) 25 | 26 | 27 | class Gatherer(multiprocessing.Process): 28 | def __enter__(self): 29 | return self 30 | 31 | def __exit__(self, type, value, traceback): 32 | return self 33 | 34 | def __init__(self, pull=GATHERER_ADDR, push=GATHERER_SINK_ADDR): 35 | multiprocessing.Process.__init__(self) 36 | self.pull = pull 37 | self.push = push 38 | self.exit = multiprocessing.Event() 39 | 40 | def _init_plugins(self): 41 | import pkgutil 42 | self.gatherers = [] 43 | logger.debug('loading plugins...') 44 | for loader, modname, is_pkg in pkgutil.iter_modules(cif.gatherer.__path__, 'cif.gatherer.'): 45 | p = loader.find_module(modname).load_module(modname) 46 | self.gatherers.append(p.Plugin()) 47 | logger.debug('plugin loaded: {}'.format(modname)) 48 | 49 | def terminate(self): 50 | self.exit.set() 51 | 52 | def start(self): 53 | self._init_plugins() 54 | 55 | context = zmq.Context() 56 | pull_s = context.socket(zmq.PULL) 57 | push_s = context.socket(zmq.PUSH) 58 | 59 | push_s.SNDTIMEO = SNDTIMEO 60 | 61 | logger.debug('connecting to sockets...') 62 | pull_s.connect(self.pull) 63 | push_s.connect(self.push) 64 | logger.debug('starting Gatherer') 65 | 66 | poller = zmq.Poller() 67 | poller.register(pull_s) 68 | 69 | while not self.exit.is_set(): 70 | try: 71 | s = dict(poller.poll(1000)) 72 | except Exception as e: 73 | logger.error(e) 74 | break 75 | 76 | if pull_s in s: 77 | id, token, mtype, data = Msg().recv(pull_s) 78 | 79 | try: 80 | data = json.loads(data) 81 | except Exception as e: 82 | logger.error('malformed data send to gatherer: {}'.format(e)) 83 | break 84 | 85 | if isinstance(data, dict): 86 | data = [data] 87 | 88 | rv = [] 89 | start = time.time() 90 | for d in data: 91 | try: 92 | i = Indicator(**d) 93 | 94 | except InvalidIndicator as e: 95 | logger.error('resolving failed for indicator: {}'.format(d)) 96 | logger.error(e) 97 | traceback.print_exc() 98 | # skip failed indicator 99 | continue 100 | 101 | for g in self.gatherers: 102 | try: 103 | g.process(i) 104 | except Exception as e: 105 | logger.error('gatherer failed on indicator {}: {}'.format(i, g)) 106 | logger.error(e) 107 | traceback.print_exc() 108 | 109 | rv.append(i.__dict__()) 110 | 111 | data = json.dumps(rv) 112 | logger.debug('sending back to router: %f' % (time.time() - start)) 113 | Msg(id=id, mtype=mtype, token=token, data=data).send(push_s) 114 | 115 | logger.info('shutting down gatherer..') 116 | -------------------------------------------------------------------------------- /rules/default/dataplane.yml: -------------------------------------------------------------------------------- 1 | parser: pipe 2 | defaults: 3 | provider: dataplane.org 4 | tlp: green 5 | altid_tlp: clear 6 | confidence: 7.5 7 | values: 8 | - null 9 | - null 10 | - indicator 11 | - lasttime 12 | - null 13 | 14 | feeds: 15 | dnsrd: 16 | remote: https://dataplane.org/dnsrd.txt 17 | defaults: 18 | application: dns 19 | portlist: 53 20 | protocol: udp 21 | tags: 22 | - scanner 23 | - dns 24 | - spoofable 25 | description: 'identified as sending recursive DNS queries to a remote host' 26 | 27 | dnsrdany: 28 | remote: https://dataplane.org/dnsrdany.txt 29 | defaults: 30 | application: dns 31 | portlist: 53 32 | protocol: udp 33 | tags: 34 | - scanner 35 | - dns 36 | - spoofable 37 | description: 'identified as sending recursive DNS IN ANY queries to a remote host' 38 | 39 | dnsversion: 40 | remote: https://dataplane.org/dnsversion.txt 41 | defaults: 42 | application: dns 43 | portlist: 53 44 | protocol: udp 45 | tags: 46 | - scanner 47 | - dns 48 | - spoofable 49 | description: 'identified as sending DNS CH TXT VERSION.BIND queries to a remote host' 50 | 51 | proto41: 52 | remote: https://dataplane.org/proto41.txt 53 | defaults: 54 | description: 'identified as an open IPv4 protocol 41 relay (i.e., IPv6 over IPv4)' 55 | tags: 56 | - protocol41 57 | - proxy 58 | values: 59 | - null 60 | - null 61 | - indicator 62 | - firsttime 63 | - lasttime 64 | - null 65 | 66 | # not enough info to be confident they're doing bad things 67 | sshclient: 68 | remote: https://dataplane.org/sshclient.txt 69 | defaults: 70 | application: ssh 71 | portlist: 22 72 | protocol: tcp 73 | confidence: 7 74 | tags: scanner 75 | description: 'has been seen initiating an SSH connection' 76 | 77 | # pinging the protocol, bad stuff.. 78 | ssh: 79 | remote: https://dataplane.org/sshpwauth.txt 80 | defaults: 81 | application: ssh 82 | portlist: 22 83 | protocol: tcp 84 | confidence: 9 85 | tags: 86 | - scanner 87 | - bruteforce 88 | description: 'seen attempting to remotely login using SSH password authentication' 89 | 90 | sipquery: 91 | remote: https://dataplane.org/sipquery.txt 92 | defaults: 93 | application: sip 94 | protocol: udp 95 | portlist: 5060 96 | tags: 97 | - scanner 98 | - bruteforce 99 | - spoofable 100 | description: 'seen initiating a SIP OPTIONS query to a remote host' 101 | 102 | sipinvitation: 103 | remote: https://dataplane.org/sipinvitation.txt 104 | defaults: 105 | application: sip 106 | protocol: udp 107 | portlist: 5060 108 | tags: 109 | - scanner 110 | - bruteforce 111 | - spoofable 112 | description: 'seen initiating a SIP INVITE operation to a remote host' 113 | 114 | sipregistration: 115 | remote: https://dataplane.org/sipregistration.txt 116 | application: sip 117 | protocol: udp 118 | portlist: 5060 119 | description: 'seen initiating a SIP REGISTER operation to a remote host' 120 | 121 | smtpdata: 122 | remote: https://dataplane.org/smtpdata.txt 123 | defaults: 124 | application: smtp 125 | portlist: 25 126 | protocol: tcp 127 | tags: 128 | - scanner 129 | - smtp 130 | description: 'identified as SMTP clients sending DATA commands to smtp sensor' 131 | 132 | smtpgreet: 133 | remote: https://dataplane.org/smtpgreet.txt 134 | defaults: 135 | application: smtp 136 | protocol: tcp 137 | portlist: 25 138 | confidence: 7 139 | tags: 140 | - scanner 141 | - smtp 142 | description: 'identified as SMTP clients sending unsolicited EHLO/HELO commands to smtp sensor' -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | from setuptools import setup, find_packages 3 | import versioneer 4 | import sys 5 | 6 | ENABLE_INSTALL = os.getenv('CIF_ENABLE_INSTALL') 7 | 8 | # vagrant doesn't appreciate hard-linking 9 | if os.environ.get('USER') == 'vagrant' or os.path.isdir('/vagrant'): 10 | del os.link 11 | 12 | MINIMUM_COVERAGE = 33 13 | if os.environ.get('CIF_ELASTICSEARCH_TEST') == '1': 14 | MINIMUM_COVERAGE = 40 15 | 16 | # https://www.pydanny.com/python-dot-py-tricks.html 17 | if sys.argv[-1] == 'test': 18 | test_requirements = [ 19 | 'pytest', 20 | 'coverage', 21 | 'pytest_cov', 22 | ] 23 | try: 24 | modules = map(__import__, test_requirements) 25 | except ImportError as e: 26 | err_msg = e.message.replace("No module named ", "") 27 | msg = "%s is not installed. Install your test requirements." % err_msg 28 | raise ImportError(msg) 29 | r = os.system('pytest test -v --cov=cif --cov-fail-under={}'.format(MINIMUM_COVERAGE)) 30 | if r == 0: 31 | sys.exit() 32 | else: 33 | raise RuntimeError('tests failed') 34 | 35 | if sys.argv[-1] == 'install': 36 | if not ENABLE_INSTALL: 37 | print('') 38 | print('CIFv3 Should NOT be installed using traditional install methods') 39 | print('Please see the DeploymentKit Wiki and use the EasyButton') 40 | print('the EasyButton uses Ansible to customize the underlying OS and all the moving parts..') 41 | print('') 42 | print('https://github.com/csirtgadgets/bearded-avenger-deploymentkit/wiki') 43 | print('') 44 | raise SystemError 45 | 46 | token_files = [ 47 | 'cif/httpd/templates/tokens/edit.html', 48 | 'cif/httpd/templates/tokens/form.html', 49 | 'cif/httpd/templates/tokens/index.html', 50 | 'cif/httpd/templates/tokens/show.html' 51 | ] 52 | 53 | template_files = [ 54 | 'cif/httpd/templates/application.html', 55 | 'cif/httpd/templates/base.html', 56 | 'cif/httpd/templates/flash.html', 57 | 'cif/httpd/templates/indicators.html', 58 | 'cif/httpd/templates/layout.html', 59 | 'cif/httpd/templates/login.html', 60 | 'cif/httpd/templates/nav.html', 61 | 'cif/httpd/templates/submit.html' 62 | ] 63 | 64 | static_files = [ 65 | 'cif/httpd/static/favicon.ico' 66 | ] 67 | 68 | setup( 69 | name="cif", 70 | version=versioneer.get_version(), 71 | cmdclass=versioneer.get_cmdclass(), 72 | description="CIFv3", 73 | long_description="", 74 | url="https://github.com/csirtgadgets/bearded-avenger", 75 | license='LGPL3', 76 | classifiers=[ 77 | "Topic :: System :: Networking", 78 | "Environment :: Other Environment", 79 | "Intended Audience :: Developers", 80 | "License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)", 81 | "Programming Language :: Python", 82 | ], 83 | keywords=['security'], 84 | author="Wes Young", 85 | author_email="wes@csirtgadgets.org", 86 | packages=find_packages(), 87 | data_files=[('cif/httpd/static/', static_files), 88 | ('cif/httpd/templates/', template_files), 89 | ('cif/httpd/templates/tokens/', token_files)], 90 | install_requires=[ 91 | 'html5lib', 92 | 'Flask-Limiter', 93 | 'limits', 94 | 'maxminddb', 95 | 'geoip2', 96 | 'dnspython', 97 | 'Flask', 98 | 'PyYAML', 99 | 'SQLAlchemy', 100 | 'elasticsearch', 101 | 'elasticsearch_dsl', 102 | 'ujson', 103 | 'pyzmq>=16.0', 104 | 'csirtg_indicator>=1.0.0,<2.0', 105 | 'cifsdk>=3.0.0rc2,<4.0', 106 | 'csirtg_smrt', 107 | 'csirtg_dnsdb' 108 | ], 109 | scripts=[], 110 | entry_points={ 111 | 'console_scripts': [ 112 | 'cif-router=cif.router:main', 113 | 'cif-hunter=cif.hunter:main', 114 | 'cif-gatherer=cif.gatherer:main', 115 | 'cif-httpd=cif.httpd:main', 116 | 'cif-store=cif.store:main', 117 | 'cif-es-archive=cif.utils.es_archiver:main' 118 | ] 119 | }, 120 | ) 121 | -------------------------------------------------------------------------------- /cif/hunter/spamhaus_fqdn.py: -------------------------------------------------------------------------------- 1 | import os 2 | import logging 3 | from csirtg_indicator import Indicator 4 | from cif.utils import resolve_ns 5 | import arrow 6 | 7 | CONFIDENCE = 9 8 | PROVIDER = 'spamhaus.org' 9 | SPAMHAUS_DQS_KEY = os.environ.get('SPAMHAUS_DQS_KEY', None) 10 | 11 | BASE_QUERY_URL = 'dbl.spamhaus.org' 12 | if SPAMHAUS_DQS_KEY and len(SPAMHAUS_DQS_KEY) == 26: 13 | BASE_QUERY_URL = '{}.dbl.dq.spamhaus.net'.format(SPAMHAUS_DQS_KEY) 14 | 15 | CODES = { 16 | '127.0.1.2': { 17 | 'tags': 'suspicious', 18 | 'description': 'spammed domain', 19 | }, 20 | '127.0.1.3': { 21 | 'tags': 'suspicious', 22 | 'description': 'spammed redirector / url shortener', 23 | }, 24 | '127.0.1.4': { 25 | 'tags': 'phishing', 26 | 'description': 'phishing domain', 27 | }, 28 | '127.0.1.5': { 29 | 'tags': 'malware', 30 | 'description': 'malware domain', 31 | }, 32 | '127.0.1.6': { 33 | 'tags': 'botnet', 34 | 'description': 'Botnet C&C domain', 35 | }, 36 | '127.0.1.102': { 37 | 'tags': 'suspicious', 38 | 'description': 'abused legit spam', 39 | }, 40 | '127.0.1.103': { 41 | 'tags': 'suspicious', 42 | 'description': 'abused legit spammed redirector', 43 | }, 44 | '127.0.1.104': { 45 | 'tags': 'phishing', 46 | 'description': 'abused legit phish', 47 | }, 48 | '127.0.1.105': { 49 | 'tags': 'malware', 50 | 'description': 'abused legit malware', 51 | }, 52 | '127.0.1.106': { 53 | 'tags': 'botnet', 54 | 'description': 'abused legit botnet', 55 | }, 56 | '127.0.1.255': { 57 | 'description': 'BANNED', 58 | }, 59 | } 60 | 61 | 62 | class SpamhausFqdn(object): 63 | 64 | def __init__(self): 65 | self.logger = logging.getLogger(__name__) 66 | self.is_advanced = True 67 | self.mtypes_supported = { 'indicators_create' } 68 | self.itypes_supported = { 'fqdn' } 69 | 70 | def _prereqs_met(self, i, **kwargs): 71 | if kwargs.get('mtype') not in self.mtypes_supported: 72 | return False 73 | 74 | if i.itype not in self.itypes_supported: 75 | return False 76 | 77 | if kwargs.get('nolog'): 78 | return False 79 | 80 | if i.provider == 'spamhaus.org': 81 | return False 82 | 83 | return True 84 | 85 | def _resolve(self, data): 86 | data = '{}.{}'.format(data, BASE_QUERY_URL) 87 | data = resolve_ns(data) 88 | if data and data[0]: 89 | return data[0] 90 | 91 | def process(self, i, router, **kwargs): 92 | if not self._prereqs_met(i, **kwargs): 93 | return 94 | 95 | try: 96 | r = self._resolve(i.indicator) 97 | try: 98 | r = CODES.get(str(r), None) 99 | except Exception as e: 100 | # https://www.spamhaus.org/faq/section/DNSBL%20Usage 101 | self.logger.error(e) 102 | self.logger.info('check spamhaus return codes') 103 | r = None 104 | 105 | if r: 106 | confidence = CONFIDENCE 107 | if ' legit ' in r['description']: 108 | confidence = 6 109 | 110 | f = Indicator(**i.__dict__()) 111 | 112 | f.tags = [r['tags']] 113 | if 'hunter' not in f.tags: 114 | f.tags.append('hunter') 115 | f.description = r['description'] 116 | f.confidence = confidence 117 | f.provider = PROVIDER 118 | f.reference_tlp = 'clear' 119 | f.reference = 'http://www.spamhaus.org/query/dbl?domain={}'.format(f.indicator) 120 | f.lasttime = f.reporttime = arrow.utcnow() 121 | x = router.indicators_create(f) 122 | self.logger.debug("Spamhaus FQDN: {}".format(x)) 123 | except KeyError as e: 124 | self.logger.error(e) 125 | except Exception as e: 126 | self.logger.error("[Hunter: SpamhausFqdn] {}: giving up on indicator {}".format(e, i)) 127 | 128 | 129 | Plugin = SpamhausFqdn -------------------------------------------------------------------------------- /cif/hunter/spamhaus_ip.py: -------------------------------------------------------------------------------- 1 | import os 2 | import logging 3 | from csirtg_indicator import Indicator 4 | from csirtg_indicator.utils import is_ipv4_net 5 | from cif.utils import resolve_ns 6 | import arrow 7 | from ipaddress import ip_address 8 | 9 | CONFIDENCE = 9 10 | PROVIDER = 'spamhaus.org' 11 | SPAMHAUS_DQS_KEY = os.environ.get('SPAMHAUS_DQS_KEY', None) 12 | 13 | BASE_QUERY_URL = 'zen.spamhaus.org' 14 | if SPAMHAUS_DQS_KEY and len(SPAMHAUS_DQS_KEY) == 26: 15 | BASE_QUERY_URL = '{}.zen.dq.spamhaus.net'.format(SPAMHAUS_DQS_KEY) 16 | 17 | CODES = { 18 | '127.0.0.2': { 19 | 'tags': 'spam', 20 | 'description': 'Direct UBE sources, spam operations & spam services', 21 | }, 22 | '127.0.0.3': { 23 | 'tags': 'spam', 24 | 'description': 'Direct snowshoe spam sources detected via automation', 25 | }, 26 | '127.0.0.4': { 27 | 'tags': 'exploit', 28 | 'description': 'CBL + customised NJABL. 3rd party exploits (proxies, trojans, etc.)', 29 | }, 30 | '127.0.0.5': { 31 | 'tags': 'exploit', 32 | 'description': 'CBL + customised NJABL. 3rd party exploits (proxies, trojans, etc.)', 33 | }, 34 | '127.0.0.6': { 35 | 'tags': 'exploit', 36 | 'description': 'CBL + customised NJABL. 3rd party exploits (proxies, trojans, etc.)', 37 | }, 38 | '127.0.0.7': { 39 | 'tags': 'exploit', 40 | 'description': 'CBL + customised NJABL. 3rd party exploits (proxies, trojans, etc.)', 41 | }, 42 | '127.0.0.9': { 43 | 'tags': 'hijacked', 44 | 'description': 'Spamhaus DROP/EDROP Data', 45 | }, 46 | } 47 | 48 | 49 | class SpamhausIp(object): 50 | 51 | def __init__(self): 52 | self.logger = logging.getLogger(__name__) 53 | self.is_advanced = True 54 | self.mtypes_supported = { 'indicators_create' } 55 | self.itypes_supported = { 'ipv4', 'ipv6' } 56 | 57 | def _prereqs_met(self, i, **kwargs): 58 | if kwargs.get('mtype') not in self.mtypes_supported: 59 | return False 60 | 61 | if kwargs.get('nolog'): 62 | return False 63 | 64 | if i.itype not in self.itypes_supported or '/' in i.indicator: 65 | # don't support CIDRs even in ip itypes 66 | return False 67 | 68 | if i.provider == 'spamhaus.org' and not is_ipv4_net(i.indicator): 69 | return False 70 | 71 | return True 72 | 73 | def _preprocess_by_ipversion(self, indicator, itype): 74 | ip_str = str(indicator) 75 | # https://www.spamhaus.org/organization/statement/012/spamhaus-ipv6-blocklists-strategy-statement 76 | if itype == 'ipv6': 77 | data = ip_address(ip_str).exploded.replace(':', '') 78 | data = reversed(data) 79 | else: 80 | data = reversed(ip_str.split('.')) 81 | 82 | return data 83 | 84 | def _resolve(self, data): 85 | data = '{}.{}'.format('.'.join(data), BASE_QUERY_URL) 86 | data = resolve_ns(data) 87 | if data and data[0]: 88 | return data[0] 89 | 90 | def process(self, i, router, **kwargs): 91 | if not self._prereqs_met(i, **kwargs): 92 | return 93 | 94 | try: 95 | zen_lookup_str = self._preprocess_by_ipversion(i.indicator, i.itype) 96 | r = self._resolve(zen_lookup_str) 97 | 98 | try: 99 | r = CODES.get(str(r), None) 100 | except Exception as e: 101 | # https://www.spamhaus.org/faq/section/DNSBL%20Usage 102 | self.logger.error(e) 103 | self.logger.info('check spamhaus return codes') 104 | r = None 105 | 106 | if r: 107 | f = Indicator(**i.__dict__()) 108 | 109 | f.tags = [r['tags']] 110 | if 'hunter' not in f.tags: 111 | f.tags.append('hunter') 112 | f.description = r['description'] 113 | f.confidence = CONFIDENCE 114 | f.provider = PROVIDER 115 | f.reference_tlp = 'clear' 116 | f.reference = 'http://www.spamhaus.org/query/bl?ip={}'.format(f.indicator) 117 | f.lasttime = f.reporttime = arrow.utcnow() 118 | x = router.indicators_create(f) 119 | self.logger.debug("Spamhaus IP: {}".format(x)) 120 | 121 | except Exception as e: 122 | self.logger.error("[Hunter: SpamhausIp] {}: giving up on indicator {}".format(e, i)) 123 | import traceback 124 | traceback.print_exc() 125 | 126 | 127 | Plugin = SpamhausIp -------------------------------------------------------------------------------- /cif/store/token_plugin.py: -------------------------------------------------------------------------------- 1 | import arrow 2 | from cif.constants import TOKEN_CACHE_DELAY, TOKEN_LENGTH 3 | from cif.utils import strtobool 4 | from cifsdk.exceptions import AuthError 5 | import os 6 | import binascii 7 | import abc 8 | import logging 9 | from datetime import datetime 10 | 11 | TRACE = strtobool(os.environ.get('CIF_TOKEN_TRACE', False)) 12 | 13 | logger = logging.getLogger(__name__) 14 | logger.setLevel(logging.ERROR) 15 | 16 | if TRACE: 17 | logger.setLevel(logging.DEBUG) 18 | 19 | class TokenManagerPlugin(object): 20 | __metaclass__ = abc.ABCMeta 21 | 22 | def __init__(self, *args, **kwargs): 23 | self._cache = kwargs.get('token_cache', {}) 24 | self._cache_check_next = arrow.utcnow().int_timestamp + TOKEN_CACHE_DELAY 25 | 26 | @abc.abstractmethod 27 | def create(self, data): 28 | raise NotImplementedError 29 | 30 | @abc.abstractmethod 31 | def delete(self, data): 32 | raise NotImplementedError 33 | 34 | @abc.abstractmethod 35 | def search(self, data): 36 | raise NotImplementedError 37 | 38 | @abc.abstractmethod 39 | def auth_search(self, token): 40 | raise NotImplementedError 41 | 42 | @abc.abstractmethod 43 | def edit(self, data, bulk=False, token=None): 44 | raise NotImplementedError 45 | 46 | def _update_token_cache_field(self, token_str, field, new_value): 47 | # since self._cache is a mp.Manager.dict, directly updating a nested dict inside 48 | # won't propagate. need to re-assign modified nested dict 49 | # https://docs.python.org/3.8/library/multiprocessing.html#proxy-objects 50 | token_dict = self._cache[token_str] 51 | token_dict[field] = new_value 52 | self._cache[token_str] = token_dict 53 | 54 | def _update_last_activity_at(self, token_str, timestamp): 55 | # internal method and should only be called by auth_search 56 | # takes in a token str and timestamp as datetime obj 57 | timestamp = timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ') 58 | # update the cache where it will be flushed later 59 | self._update_token_cache_field(token_str, 'last_activity_at', timestamp) 60 | 61 | def _generate(self): 62 | return binascii.b2a_hex(os.urandom(TOKEN_LENGTH)).decode('utf-8') 63 | 64 | def _flush_cache(self, force=False): 65 | if force or arrow.utcnow().int_timestamp > self._cache_check_next: 66 | logger.debug('flushing token cache...') 67 | # write cache back to store, where _cache is a dict of dicts with token_strs as keys 68 | self.edit(self._cache, bulk=True) 69 | self._cache.clear() 70 | self._cache_check_next = arrow.utcnow().int_timestamp + TOKEN_CACHE_DELAY 71 | 72 | logger.debug('token cache flushed..') 73 | 74 | def _cache_check(self, token_str, k=None): 75 | self._flush_cache() 76 | if token_str not in self._cache: 77 | return False 78 | 79 | return self._cache[token_str] 80 | 81 | def admin_exists(self): 82 | t = list(self.search({'admin': True})) 83 | if len(t) > 0: 84 | return t[0]['token'] 85 | 86 | def hunter_exists(self): 87 | t = list(self.search({'username': 'hunter'})) 88 | if len(t) > 0: 89 | return t[0] 90 | 91 | def check(self, token, k, v=True): 92 | self._flush_cache() 93 | token_str = token['token'] 94 | if token_str in self._cache and self._cache[token_str].get(k): 95 | return self._cache[token_str] 96 | 97 | rv = list(self.search({'token': token_str, k: v})) 98 | if len(rv) == 0: 99 | raise AuthError('unauthorized') 100 | 101 | self._cache[token_str] = rv[0] 102 | return rv[0] 103 | 104 | def admin(self, token): 105 | return self.check(token, 'admin') 106 | 107 | def read(self, token): 108 | return self.check(token, 'read') 109 | 110 | def write(self, token): 111 | return self.check(token, 'write') 112 | 113 | def last_activity_at(self, token): 114 | token_str = token['token'] 115 | if self._cache.get(token_str, {}).get('last_activity_at'): 116 | rv = self._cache[token_str]['last_activity_at'] 117 | else: 118 | rv = list(self.search({'token': token_str})) 119 | 120 | if not rv: 121 | return None 122 | 123 | rv = rv[0].get('last_activity_at', None) 124 | 125 | if isinstance(rv, datetime): 126 | # return RFC3339 formatted str instead of datetime 127 | rv = rv.strftime('%Y-%m-%dT%H:%M:%S.%fZ') 128 | 129 | return rv 130 | -------------------------------------------------------------------------------- /test/test_httpd.py: -------------------------------------------------------------------------------- 1 | import ujson as json 2 | import os 3 | import tempfile 4 | 5 | import pytest 6 | from cif import httpd 7 | from cif.store import Store 8 | 9 | from cifsdk.constants import PYVERSION 10 | 11 | ROUTER_ADDR = 'ipc://{}'.format(tempfile.NamedTemporaryFile().name) 12 | 13 | @pytest.fixture 14 | def client(request): 15 | httpd.app.config['TESTING'] = True 16 | httpd.app.config['CIF_ROUTER_ADDR'] = ROUTER_ADDR 17 | httpd.app.config['dummy'] = True 18 | return httpd.app.test_client() 19 | 20 | 21 | @pytest.fixture 22 | def store(): 23 | dbfile = tempfile.mktemp() 24 | with Store(store_type='sqlite', dbfile=dbfile) as s: 25 | yield s 26 | 27 | os.unlink(dbfile) 28 | 29 | 30 | def test_httpd_help(client): 31 | rv = client.get('/') 32 | assert rv.status_code == 200 33 | 34 | rv = client.get('/help') 35 | assert rv.status_code == 200 36 | 37 | 38 | def test_httpd_ping(client): 39 | rv = client.get('/ping') 40 | assert rv.status_code == 401 41 | 42 | rv = client.get('/ping', headers={'Authorization': 'Token token=1234'}) 43 | assert rv.status_code == 200 44 | 45 | 46 | def test_httpd_confidence(client): 47 | rv = client.get('/help/confidence') 48 | assert rv.status_code == 200 49 | 50 | rv = client.get('/help/confidence', headers={'Authorization': 'Token token=1234'}) 51 | assert rv.status_code == 200 52 | 53 | 54 | def test_httpd_search(client): 55 | rv = client.get('/search?q=example.com', headers={'Authorization': 'Token token=1234'}) 56 | assert rv.status_code == 200 57 | 58 | data = rv.data 59 | if PYVERSION > 2: 60 | data = data.decode('utf-8') 61 | 62 | rv = json.loads(data) 63 | assert rv['data'][0]['indicator'] == 'example.com' 64 | 65 | 66 | def test_httpd_indicators(client): 67 | rv = client.get('/indicators?q=example.com', headers={'Authorization': 'Token token=1234'}) 68 | assert rv.status_code == 200 69 | 70 | data = rv.data 71 | if PYVERSION > 2: 72 | data = data.decode('utf-8') 73 | 74 | rv = json.loads(data) 75 | assert rv['data'][0]['indicator'] == 'example.com' 76 | 77 | 78 | def test_httpd_feed(client): 79 | import arrow 80 | httpd.app.config['feed'] = {} 81 | httpd.app.config['feed']['data'] = [ 82 | { 83 | 'indicator': '128.205.1.1', 84 | 'confidence': '8', 85 | 'tags': ['malware'], 86 | 'reporttime': str(arrow.utcnow()), 87 | }, 88 | { 89 | 'indicator': '128.205.2.1', 90 | 'confidence': '8', 91 | 'tags': ['malware'], 92 | 'reporttime': str(arrow.utcnow()), 93 | }, 94 | ] 95 | httpd.app.config['feed']['wl'] = [ 96 | { 97 | 'indicator': '128.205.0.0/16', 98 | 'confidence': '8', 99 | 'tags': ['whitelist'], 100 | 'reporttime': str(arrow.utcnow()), 101 | }, 102 | ] 103 | 104 | rv = client.get('/feed?itype=ipv4&confidence=7', headers={'Authorization': 'Token token=1234'}) 105 | 106 | assert rv.status_code == 200 107 | 108 | r = json.loads(rv.data.decode('utf-8')) 109 | assert len(r['data']) == 0 110 | 111 | def test_httpd_feed_fqdn(client): 112 | import arrow 113 | httpd.app.config['feed'] = {} 114 | httpd.app.config['feed']['data'] = [ 115 | { 116 | 'indicator': 'page-test.weebly.com', 117 | 'confidence': '8', 118 | 'tags': ['malware'], 119 | 'reporttime': str(arrow.utcnow()), 120 | }, 121 | { 122 | 'indicator': 'example.com', 123 | 'confidence': '8', 124 | 'tags': ['malware'], 125 | 'reporttime': str(arrow.utcnow()), 126 | }, 127 | { 128 | 'indicator': 'test.google.com', 129 | 'confidence': '8', 130 | 'tags': ['malware'], 131 | 'reporttime': str(arrow.utcnow()), 132 | }, 133 | { 134 | 'indicator': 'test.test.ex.com', 135 | 'confidence': '8', 136 | 'tags': ['malware'], 137 | 'reporttime': str(arrow.utcnow()), 138 | }, 139 | ] 140 | httpd.app.config['feed']['wl'] = [ 141 | { 142 | 'indicator': 'ex.com', 143 | 'confidence': '8', 144 | 'tags': ['whitelist'], 145 | 'reporttime': str(arrow.utcnow()), 146 | }, 147 | ] 148 | 149 | rv = client.get('/feed?itype=fqdn&confidence=7', headers={'Authorization': 'Token token=1234'}) 150 | 151 | assert rv.status_code == 200 152 | 153 | r = json.loads(rv.data.decode('utf-8')) 154 | assert len(r['data']) == 1 155 | 156 | 157 | def test_httpd_tokens(client): 158 | rv = client.get('/tokens', headers={'Authorization': 'Token token=1234'}) 159 | assert rv.status_code == 200 -------------------------------------------------------------------------------- /test/zelasticsearch/test_store_elasticsearch_tokens_edit.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from csirtg_indicator import Indicator 3 | from cif.store import Store 4 | from elasticsearch_dsl.connections import connections 5 | import os 6 | import arrow 7 | from time import sleep 8 | from pprint import pprint 9 | 10 | DISABLE_TESTS = True 11 | if os.environ.get('CIF_ELASTICSEARCH_TEST', '0') == '1': 12 | DISABLE_TESTS = False 13 | 14 | 15 | @pytest.fixture 16 | def store(): 17 | 18 | with Store(store_type='elasticsearch', nodes='127.0.0.1:9200') as s: 19 | s._load_plugin(nodes='127.0.0.1:9200') 20 | try: 21 | connections.get_connection().indices.delete(index='indicators-*') 22 | connections.get_connection().indices.delete(index='tokens') 23 | except Exception as e: 24 | pass 25 | yield s 26 | 27 | assert connections.get_connection().indices.delete(index='indicators-*') 28 | assert connections.get_connection().indices.delete(index='tokens') 29 | 30 | 31 | @pytest.fixture 32 | def token(store): 33 | t = store.store.tokens.create({ 34 | 'username': u'test_admin', 35 | 'groups': [u'everyone'], 36 | 'read': u'1', 37 | 'write': u'1', 38 | 'admin': u'1' 39 | }) 40 | 41 | assert t 42 | yield t 43 | 44 | @pytest.fixture 45 | def user_token(store): 46 | t = store.store.tokens.create({ 47 | 'username': u'test_user', 48 | 'groups': [u'everyone'], 49 | 'read': u'1', 50 | 'write': False, 51 | 'admin': False 52 | }) 53 | 54 | assert t 55 | yield t 56 | 57 | @pytest.fixture 58 | def indicator(): 59 | return Indicator( 60 | indicator='example.com', 61 | tags='botnet', 62 | provider='csirtg.io', 63 | group='everyone', 64 | lasttime=arrow.utcnow().datetime, 65 | reporttime=arrow.utcnow().datetime 66 | ) 67 | 68 | @pytest.mark.skipif(DISABLE_TESTS, reason='need to set CIF_ELASTICSEARCH_TEST=1 to run') 69 | def test_store_elasticsearch_tokens_edit_groups(store, token): 70 | 71 | x = store.handle_tokens_search(token, {'token': token['token']}) 72 | x = list(x) 73 | 74 | pprint(x) 75 | 76 | assert x[0]['groups'] == ['everyone'] 77 | 78 | u = { 79 | 'token': token['token'], 80 | 'groups': ['staff', 'everyone'] 81 | } 82 | 83 | x = store.handle_tokens_edit(token, u) 84 | 85 | assert x 86 | 87 | x = store.handle_tokens_search(token, {'token': token['token']}) 88 | x = list(x) 89 | 90 | pprint(x) 91 | 92 | assert x[0]['read'] 93 | assert x[0]['write'] 94 | assert x[0]['admin'] 95 | assert x[0]['groups'] == ['staff', 'everyone'] 96 | 97 | @pytest.mark.skipif(DISABLE_TESTS, reason='need to set CIF_ELASTICSEARCH_TEST=1 to run') 98 | def test_store_elasticsearch_tokens_edit_rw_perms(store, token): 99 | 100 | x = store.handle_tokens_search(token, {'token': token['token']}) 101 | x = list(x) 102 | 103 | pprint(x) 104 | 105 | assert x[0]['read'] 106 | assert x[0]['write'] 107 | 108 | u = { 109 | 'token': token['token'], 110 | 'write': False 111 | } 112 | 113 | x = store.handle_tokens_edit(token, u) 114 | 115 | assert x 116 | 117 | x = store.handle_tokens_search(token, {'token': token['token']}) 118 | x = list(x) 119 | 120 | pprint(x) 121 | 122 | assert x[0]['read'] 123 | assert x[0]['admin'] 124 | assert x[0]['groups'] == ['everyone'] 125 | assert x[0]['write'] == False 126 | 127 | @pytest.mark.skipif(DISABLE_TESTS, reason='need to set CIF_ELASTICSEARCH_TEST=1 to run') 128 | def test_store_elasticsearch_tokens_edit_time_fields_with_cache(store, token, user_token): 129 | 130 | starttime = arrow.utcnow().datetime.strftime('%Y-%m-%dT%H:%M:%S.%fZ') 131 | # do an auth search to cache user token data 132 | x = store.store.tokens.auth_search(user_token) 133 | x = list(x) 134 | 135 | pprint(x) 136 | 137 | assert x[0]['groups'] == ['everyone'] 138 | 139 | new_groups = ['staff', 'everyone'] 140 | 141 | u = { 142 | 'token': user_token['token'], 143 | 'groups': new_groups 144 | } 145 | 146 | # a token edit will update the token cache then force immediate cache flush to ES 147 | x = store.handle_tokens_edit(token, u) 148 | 149 | assert x 150 | 151 | # provide enough time for cache to write-behind 152 | sleep(2) 153 | 154 | x = store.store.tokens.auth_search(user_token) 155 | x = list(x) 156 | 157 | pprint(x) 158 | 159 | assert x[0]['read'] 160 | assert not x[0]['write'] 161 | assert not x[0]['admin'] 162 | assert x[0]['groups'] == new_groups 163 | assert x[0]['last_activity_at'] 164 | assert x[0]['last_activity_at'] > starttime 165 | assert x[0]['last_edited_by'] 166 | assert x[0]['last_edited_by'] == 'test_admin' 167 | assert x[0]['last_edited_at'] 168 | assert x[0]['last_edited_at'] > starttime 169 | assert x[0]['created_at'] 170 | assert x[0]['created_at'] < starttime 171 | -------------------------------------------------------------------------------- /cif/httpd/views/u/tokens.py: -------------------------------------------------------------------------------- 1 | from flask.views import MethodView 2 | from flask import redirect 3 | from flask import request, render_template, session, url_for, flash 4 | from cifsdk.client.zeromq import ZMQ as Client 5 | from cif.constants import ROUTER_ADDR 6 | import logging 7 | import os 8 | import ujson as json 9 | 10 | remote = ROUTER_ADDR 11 | HTTPD_TOKEN = os.getenv('CIF_HTTPD_TOKEN') 12 | 13 | logger = logging.getLogger('cif-httpd') 14 | 15 | 16 | class TokensUI(MethodView): 17 | def get(self, token_id): 18 | filters = {} 19 | 20 | if not session['admin']: 21 | filters['username'] = session['username'] 22 | else: 23 | if request.args.get('q'): 24 | filters['username'] = request.args['q'] 25 | 26 | if token_id and token_id != 'new': 27 | filters['token'] = token_id 28 | 29 | try: 30 | r = Client(remote, HTTPD_TOKEN).tokens_search(filters) 31 | 32 | except Exception as e: 33 | logger.error(e) 34 | response = render_template('tokens/index.html') 35 | 36 | else: 37 | if token_id and token_id != 'new': 38 | response = render_template('tokens/show.html', records=r) 39 | else: 40 | response = render_template('tokens/index.html', records=r) 41 | 42 | return response 43 | 44 | def post(self, token_id): 45 | if not session['admin']: 46 | return redirect('/u/login', code=401) 47 | 48 | filters = {} 49 | if token_id: 50 | filters['token'] = token_id 51 | 52 | # TODO- need to search for default token values first, update those 53 | write = False 54 | if request.form.get('write') == 'on': 55 | write = True 56 | 57 | admin = False 58 | if request.form.get('admin') == 'on': 59 | admin = True 60 | 61 | t = { 62 | 'token': token_id, 63 | 'username': request.form['username'], 64 | 'admin': admin, 65 | 'write': write, 66 | 'groups': request.form['groups'], 67 | } 68 | 69 | t = json.dumps(t) 70 | 71 | logger.debug(t) 72 | 73 | try: 74 | Client(remote, session['token']).tokens_edit(t) 75 | except Exception as e: 76 | logger.error(e) 77 | return render_template('tokens.html', error='search failed') 78 | 79 | filters = {} 80 | filters['username'] = request.args.get('username') 81 | 82 | try: 83 | r = Client(remote, session['token']).tokens_search(filters) 84 | 85 | except Exception as e: 86 | logger.error(e) 87 | flash(e, 'error') 88 | response = render_template('tokens.html') 89 | 90 | else: 91 | flash('success!') 92 | return redirect(url_for('/u/tokens')) 93 | 94 | return response 95 | 96 | def put(self, token_id): 97 | if not session['admin']: 98 | return redirect('/u/login', code=401) 99 | 100 | if request.form.get('username') == '': 101 | flash('missing username', 'error') 102 | return redirect('/u/tokens') 103 | 104 | write = False 105 | if request.form.get('write') == 'on': 106 | write = True 107 | 108 | admin = False 109 | if request.form.get('admin') == 'on': 110 | admin = True 111 | 112 | read = False 113 | if request.form.get('read') == 'on': 114 | read = True 115 | 116 | t = { 117 | 'username': request.form['username'], 118 | 'groups': request.form['groups'].split(','), 119 | 'admin': admin, 120 | 'write': write, 121 | 'read': read 122 | } 123 | 124 | t = json.dumps(t) 125 | 126 | try: 127 | Client(remote, session['token']).tokens_create(t) 128 | except Exception as e: 129 | logger.error(e) 130 | flash(e, 'error') 131 | return redirect(url_for('/u/tokens')) 132 | 133 | filters = {} 134 | filters['username'] = request.args.get('username') 135 | 136 | try: 137 | r = Client(remote, session['token']).tokens_search(filters) 138 | 139 | except Exception as e: 140 | logger.error(e) 141 | flash(e, 'error') 142 | return redirect(url_for('/u/tokens')) 143 | 144 | flash('success', 'success') 145 | return redirect(url_for('/u/tokens')) 146 | 147 | def delete(self, token_id): 148 | if not session['admin']: 149 | return redirect('/u/login', code=401) 150 | 151 | filters = {} 152 | if token_id: 153 | filters['token'] = token_id 154 | filters['username'] = None 155 | 156 | filters = json.dumps(filters) 157 | try: 158 | r = Client(remote, HTTPD_TOKEN).tokens_delete(filters) 159 | 160 | except Exception as e: 161 | logger.error(e) 162 | flash(e, 'error') 163 | else: 164 | flash('success', 'success') 165 | 166 | return redirect(url_for('/u/tokens')) 167 | --------------------------------------------------------------------------------