├── test ├── __init__.py ├── utils │ ├── test.json │ ├── test.csv │ ├── test.xml │ ├── test.tsv │ ├── test_smrt_utils.py │ ├── test.rss │ └── test_smrt_column_detection.py ├── test_nltk.py ├── phishtank │ ├── feed.json.gz │ ├── phishtank.yml │ └── test_phishtank.py ├── alexa │ ├── alexa_top-1m.csv.zip │ ├── alexa.yml │ └── test_alexa.py ├── malwaredomains │ ├── domains.zip │ ├── bulk_registrars.zip │ ├── url_shorteners.zip │ ├── malwaredomains.yml │ └── test_malwaredomains.py ├── smrt │ ├── data │ │ ├── csv_quoted.txt │ │ ├── feed_regex_2015-01-01.csv │ │ ├── feed_defanged.txt │ │ └── feed.txt │ ├── rules │ │ ├── csv_quoted.yml │ │ ├── csirtg_defang.yml │ │ ├── csirtg.yml │ │ └── archiver.yml │ ├── test_defang.py │ ├── test_csv_quoted.py │ ├── test_remote_regex.py │ ├── remote_regex.yml │ ├── test_smrt.py │ └── test_archiver.py ├── vxvault │ ├── vxvault.yml │ ├── feed.txt │ └── test_vxvault.py ├── openphish │ ├── openphish.yml │ ├── test_openphish.py │ └── feed.txt ├── stix │ ├── test.yml │ └── test_stix.py ├── zemail │ ├── zemail.yml │ ├── test_zemail.py │ └── single_plain_01.eml ├── pastebin │ ├── feed.txt │ ├── pastebin.yml │ └── test_pastebin.py ├── spamcop │ ├── spamcop.yml │ └── test_spamcop.py ├── cef │ ├── test_cef.py │ └── cef.log ├── client │ ├── test_cif.py │ ├── test_syslog.py │ ├── test_splunk.py │ ├── test_zyre.py │ ├── test_elasticsearch.py │ └── test_csirtg_live.py ├── ransomware_abuse_ch │ ├── ransomware_abuse_ch.yml │ └── test_ransomware_abuse_ch.py ├── spamhaus │ ├── spamhaus.yml │ ├── test_spamhaus.py │ └── edrop.txt ├── packetmail │ ├── packetmail.yml │ ├── test_packetmail.py │ └── feed.txt ├── bro │ ├── test_bro.py │ └── bro.log ├── csirtg │ ├── feed2_csv.txt │ ├── csirtg.yml │ ├── test_csirtg.py │ └── feed.txt ├── sansedu │ ├── test_sans.py │ ├── sans_edu.yml │ ├── low.txt │ └── block.txt ├── malc0de │ ├── malc0de.yml │ └── test_malc0de.py ├── test_timestamps.py ├── ufw │ ├── test_ufw.py │ ├── ufw_ubuntu16.log │ └── ufw.log ├── alienvault │ ├── feed.txt │ ├── alienvault.yml │ └── test_alienvault.py └── bambenek │ ├── bambenek.yml │ ├── test_bambenek.py │ ├── fqdn_feed.txt │ └── ipv4_feed.txt ├── packaging ├── debian │ ├── compat │ ├── install │ ├── changelog │ ├── rules │ ├── control │ └── copyright └── pyinstaller │ └── csirtg-smrt.spec ├── csirtg_smrt ├── client │ ├── __init__.py │ ├── zcifzmq.py │ ├── dummy.py │ ├── zcif.py │ ├── plugin.py │ ├── zsplunk.py │ ├── zzyre.py │ ├── zcsirtg.py │ ├── zsyslog.py │ ├── zzmq.py │ ├── zelasticsearch.py │ ├── zcifv2.py │ └── ztaxii11.py ├── decoders │ ├── __init__.py │ ├── zgzip.py │ └── zzip.py ├── __init__.py ├── parser │ ├── zcsv.py │ ├── ztsv.py │ ├── pipe.py │ ├── semicolon.py │ ├── zsyslog.py │ ├── zindicator.py │ ├── zcifv2.py │ ├── zjson.py │ ├── zrss.py │ ├── zcifv3.py │ ├── pattern.py │ ├── delim.py │ ├── zemail.py │ ├── zsmtpd.py │ ├── __init__.py │ └── bro.py ├── exceptions.py ├── constants.py ├── utils │ ├── znltk.py │ ├── ztail.py │ ├── zcolumns.py │ ├── zcontent.py │ ├── zarrow.py │ └── __init__.py └── rule.py ├── .gitattributes ├── tools ├── magic1.dll └── run_with_env.cmd ├── zyre_requirements.txt ├── extras_requirements.txt ├── dev_requirements.txt ├── .github └── issue_template.md ├── setup.cfg ├── examples ├── cifv2.yml ├── cifv3.yml └── csirtg.yml ├── requirements.txt ├── MANIFEST.in ├── .coveragerc ├── Vagrantfile ├── .gitignore ├── .travis.yml ├── setup.py ├── appveyor.yml ├── README.md └── Makefile /test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packaging/debian/compat: -------------------------------------------------------------------------------- 1 | 9 -------------------------------------------------------------------------------- /csirtg_smrt/client/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /csirtg_smrt/decoders/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /csirtg_smrt/__init__.py: -------------------------------------------------------------------------------- 1 | from .smrt import * 2 | -------------------------------------------------------------------------------- /packaging/debian/install: -------------------------------------------------------------------------------- 1 | csirtg-smrt /usr/bin -------------------------------------------------------------------------------- /test/utils/test.json: -------------------------------------------------------------------------------- 1 | '[{"test": "test"}]' 2 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | csirtg_smrt/_version.py export-subst 2 | -------------------------------------------------------------------------------- /test/utils/test.csv: -------------------------------------------------------------------------------- 1 | asdf,asdf,asdf,asdf 2 | asdf,asdf,asdf,asdf 3 | -------------------------------------------------------------------------------- /test/utils/test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /test/test_nltk.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csirtgadgets/csirtg-smrt-v1/HEAD/test/test_nltk.py -------------------------------------------------------------------------------- /tools/magic1.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csirtgadgets/csirtg-smrt-v1/HEAD/tools/magic1.dll -------------------------------------------------------------------------------- /test/phishtank/feed.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csirtgadgets/csirtg-smrt-v1/HEAD/test/phishtank/feed.json.gz -------------------------------------------------------------------------------- /test/utils/test.tsv: -------------------------------------------------------------------------------- 1 | # asdf 2 | #asf 3 | # asdasdf 4 | asdf asdf asdf asdf 5 | asdfasdf asdfasdf asdfasdf asdfasdf 6 | -------------------------------------------------------------------------------- /test/alexa/alexa_top-1m.csv.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csirtgadgets/csirtg-smrt-v1/HEAD/test/alexa/alexa_top-1m.csv.zip -------------------------------------------------------------------------------- /test/malwaredomains/domains.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csirtgadgets/csirtg-smrt-v1/HEAD/test/malwaredomains/domains.zip -------------------------------------------------------------------------------- /test/malwaredomains/bulk_registrars.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csirtgadgets/csirtg-smrt-v1/HEAD/test/malwaredomains/bulk_registrars.zip -------------------------------------------------------------------------------- /test/malwaredomains/url_shorteners.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csirtgadgets/csirtg-smrt-v1/HEAD/test/malwaredomains/url_shorteners.zip -------------------------------------------------------------------------------- /zyre_requirements.txt: -------------------------------------------------------------------------------- 1 | https://github.com/zeromq/pyzmq/archive/f37052960e1dc510ea2c1d2e69a32d727afa9d69.tar.gz#egg=pyzmq>16.0.1 2 | pyzyre>=0.0.0a4,<1.0 -------------------------------------------------------------------------------- /extras_requirements.txt: -------------------------------------------------------------------------------- 1 | elasticsearch<3.0.0,>=2.0.0 2 | elasticsearch_dsl<3.0.0,>=2.0.0 3 | stix>=1.0,<2.0 4 | libtaxii>=1.1.119 5 | maec 6 | -r requirements.txt 7 | -------------------------------------------------------------------------------- /test/smrt/data/csv_quoted.txt: -------------------------------------------------------------------------------- 1 | "2018-05-30 05:39:37","192.168.1.1",66,"example.com","1.2.3, aaabbbcccddd","",,, 2 | "2018-05-30 03:33:51","192.168.1.2",66,"example.com","1,2,3","",,, -------------------------------------------------------------------------------- /csirtg_smrt/decoders/zgzip.py: -------------------------------------------------------------------------------- 1 | import gzip 2 | 3 | 4 | def get_lines(file, split="\n"): 5 | 6 | with gzip.open(file, 'rb') as f: 7 | for l in f: 8 | yield l 9 | -------------------------------------------------------------------------------- /packaging/debian/changelog: -------------------------------------------------------------------------------- 1 | csirtg-smrt (%VERSION%-%RELEASE%~%DIST%) %DIST%; urgency=low 2 | 3 | * %VERSION% release 4 | 5 | -- CSIRT Gadgets Foundation. %DATE% 6 | -------------------------------------------------------------------------------- /dev_requirements.txt: -------------------------------------------------------------------------------- 1 | coverage>=4.2 2 | pytest-cov>=2.6 3 | pytest>=4.2 4 | https://github.com/pyinstaller/pyinstaller/archive/b78bfe530cdc2904f65ce098bdf2de08c9037abb.tar.gz 5 | -r extras_requirements.txt 6 | -------------------------------------------------------------------------------- /.github/issue_template.md: -------------------------------------------------------------------------------- 1 | # Expected behavior and actual behavior. 2 | 3 | # Steps to reporduce the problem 4 | 5 | # Relevant logs as a result of the actual behavior 6 | 7 | # Specifications like the version of the project, operating system, or hardware. 8 | -------------------------------------------------------------------------------- /packaging/debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | export DH_VERBOSE=1 3 | 4 | override_dh_auto_build override_dh_auto_install: 5 | @ 6 | 7 | override_dh_shlibdeps: 8 | dh_shlibdeps --dpkg-shlibdeps-params=--ignore-missing-info 9 | 10 | %: 11 | dh $@ -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [tool:pytest] 2 | norecursedirs = build 3 | 4 | [versioneer] 5 | VCS = git 6 | style = pep440 7 | versionfile_source = csirtg_smrt/_version.py 8 | versionfile_build = csirtg_smrt/_version.py 9 | tag_prefix = 10 | parentdir_prefix = csirtg-smrt- 11 | -------------------------------------------------------------------------------- /test/vxvault/vxvault.yml: -------------------------------------------------------------------------------- 1 | defaults: 2 | provider: vxvault.net 3 | confidence: 9 4 | tlp: green 5 | altid_tlp: white 6 | tags: malware 7 | values: 8 | - indicator 9 | 10 | feeds: 11 | urls: 12 | remote: http://vxvault.net/URL_List.php 13 | pattern: '^(http:\/\/\S+)$' -------------------------------------------------------------------------------- /csirtg_smrt/parser/zcsv.py: -------------------------------------------------------------------------------- 1 | from csirtg_smrt.parser.delim import Delim 2 | import re 3 | 4 | 5 | class Csv(Delim): 6 | 7 | def __init__(self, *args, **kwargs): 8 | super(Csv, self).__init__(*args, **kwargs) 9 | 10 | self.pattern = re.compile(",") 11 | 12 | Plugin = Csv 13 | -------------------------------------------------------------------------------- /csirtg_smrt/parser/ztsv.py: -------------------------------------------------------------------------------- 1 | from csirtg_smrt.parser.delim import Delim 2 | import re 3 | 4 | 5 | class Tsv(Delim): 6 | 7 | def __init__(self, *args, **kwargs): 8 | super(Tsv, self).__init__(*args, **kwargs) 9 | 10 | self.pattern = re.compile("\t+") 11 | 12 | Plugin = Tsv 13 | -------------------------------------------------------------------------------- /examples/cifv2.yml: -------------------------------------------------------------------------------- 1 | token: '1234....' 2 | parser: cifv2 3 | defaults: 4 | provider: example.com 5 | 6 | feeds: 7 | port-scanners: 8 | filters: 9 | otype: ipv4 10 | tags: scanner 11 | group: everyone 12 | limit: 10 13 | remote: 'https://feeds.example.com/observables' 14 | -------------------------------------------------------------------------------- /csirtg_smrt/parser/pipe.py: -------------------------------------------------------------------------------- 1 | from csirtg_smrt.parser.delim import Delim 2 | import re 3 | 4 | 5 | class Pipe(Delim): 6 | 7 | def __init__(self, *args, **kwargs): 8 | super(Pipe, self).__init__(*args, **kwargs) 9 | 10 | self.pattern = re.compile('\||\s+\|\s+') 11 | 12 | 13 | Plugin = Pipe 14 | -------------------------------------------------------------------------------- /test/openphish/openphish.yml: -------------------------------------------------------------------------------- 1 | defaults: 2 | tags: phishing 3 | protocol: tcp 4 | provider: openphish.com 5 | tlp: green 6 | reference_tlp: white 7 | confidence: 9 8 | values: indicator 9 | feeds: 10 | urls: 11 | itype: url 12 | pattern: ^(.+)$ 13 | remote: http://openphish.com/feed.txt 14 | -------------------------------------------------------------------------------- /test/stix/test.yml: -------------------------------------------------------------------------------- 1 | parser: stix 2 | remote: 'test/stix/feed.xml' 3 | 4 | defaults: 5 | provider: test.com 6 | confidence: 2 7 | tlp: green 8 | altid_tlp: white 9 | provider: test.com 10 | tags: botnet 11 | 12 | feeds: 13 | fqdn: 14 | itype: fqdn 15 | defaults: 16 | asn_desc: '12' 17 | -------------------------------------------------------------------------------- /test/zemail/zemail.yml: -------------------------------------------------------------------------------- 1 | parser: email 2 | defaults: 3 | tlp: green 4 | reference_tlp: white 5 | provider: 'csirtg.io' 6 | tags: 7 | - uce 8 | description: 'uce' 9 | group: 'everyone' 10 | confidence: 9 11 | 12 | feeds: 13 | abuse: 14 | remote: stdin 15 | headers: 16 | date: lasttime 17 | -------------------------------------------------------------------------------- /test/pastebin/feed.txt: -------------------------------------------------------------------------------- 1 | PSN ComboLIST 5K+ 2 | 3 | 15pr17ny@gmail.com:freedom1 4 | 1thegin@gmail.com:18903417 5 | 2003@yahoo.com:vale123 6 | 3seis0@gmail.com:pikolo 7 | 85toro@gmail.com:toro1985 8 | a.artz@yahoo.com:trublu21 9 | a.c.mauriac@wanadoo.fr:9w3cpxt 10 | a.cefai123@outlook.com:Thomas 11 | a.clayton10@gmail.com:clayton5 -------------------------------------------------------------------------------- /test/pastebin/pastebin.yml: -------------------------------------------------------------------------------- 1 | defaults: 2 | provider: pastebin.com 3 | confidence: 7 4 | tlp: green 5 | altid_tlp: white 6 | tags: 7 | - compromise 8 | - credential 9 | 10 | feeds: 11 | creds: 12 | remote: stdin 13 | pattern: '^(.+):(.+)$' 14 | values: 15 | - indicator 16 | - additional_data -------------------------------------------------------------------------------- /csirtg_smrt/parser/semicolon.py: -------------------------------------------------------------------------------- 1 | from csirtg_smrt.parser.delim import Delim 2 | import re 3 | 4 | 5 | class Semicolon(Delim): 6 | 7 | def __init__(self, *args, **kwargs): 8 | super(Semicolon, self).__init__(*args, **kwargs) 9 | 10 | self.pattern = re.compile('[\s+]?;[\s+]?') 11 | 12 | 13 | Plugin = Semicolon 14 | -------------------------------------------------------------------------------- /test/smrt/rules/csv_quoted.yml: -------------------------------------------------------------------------------- 1 | parser: csv 2 | defaults: 3 | provider: example 4 | tags: 5 | - suspicious 6 | 7 | feeds: 8 | test: 9 | remote: 'test/smrt/data/csv_quoted.txt' 10 | defaults: 11 | values: 12 | - lasttime 13 | - indicator 14 | - portlist 15 | - additional_data 16 | - description 17 | -------------------------------------------------------------------------------- /test/spamcop/spamcop.yml: -------------------------------------------------------------------------------- 1 | parser: email 2 | defaults: 3 | tlp: green 4 | reference_tlp: white 5 | provider: 'spamcop.net' 6 | tags: 7 | - spam 8 | - abuse 9 | description: 'abuse report' 10 | group: 'everyone' 11 | confidence: 9 12 | 13 | feeds: 14 | abuse: 15 | remote: stdin 16 | headers: 17 | x-spamcop-sourceip: indicator 18 | date: lasttime 19 | -------------------------------------------------------------------------------- /test/cef/test_cef.py: -------------------------------------------------------------------------------- 1 | from csirtg_smrt.parser.cef import parse_line 2 | 3 | 4 | def test_cef(): 5 | file = 'test/cef/cef.log' 6 | 7 | events = [] 8 | with open(file) as f: 9 | for l in f.read().split("\n"): 10 | i = parse_line(l) 11 | events.append(i) 12 | 13 | assert len(events) > 0 14 | assert events[0]['indicator'] == '113.195.145.52' 15 | -------------------------------------------------------------------------------- /csirtg_smrt/client/zcifzmq.py: -------------------------------------------------------------------------------- 1 | from cifsdk.client.zeromq import ZMQ as ZMQClient 2 | from csirtg_smrt.constants import ROUTER_ADDR 3 | 4 | 5 | class CIF(ZMQClient): 6 | 7 | def __init__(self, remote=ROUTER_ADDR, token=None, **kwargs): 8 | if not remote: 9 | remote = ROUTER_ADDR 10 | 11 | super(CIF, self).__init__(remote, token, **kwargs) 12 | 13 | Plugin = CIF 14 | 15 | -------------------------------------------------------------------------------- /test/client/test_cif.py: -------------------------------------------------------------------------------- 1 | import py.test 2 | 3 | from csirtg_smrt import Smrt 4 | from pprint import pprint 5 | 6 | 7 | def test_smrt(): 8 | with Smrt(remote='localhost:514', client='syslog') as s: 9 | assert type(s) is Smrt 10 | 11 | rule, feed = next(s.load_feeds('test/smrt/rules/csirtg.yml', 'port-scanners')) 12 | x = list(s.process(rule, feed)) 13 | assert len(x) > 0 14 | -------------------------------------------------------------------------------- /csirtg_smrt/client/dummy.py: -------------------------------------------------------------------------------- 1 | from csirtg_smrt.client.plugin import Client 2 | 3 | 4 | class Dummy(Client): 5 | 6 | def __init__(self, remote, token, **kwargs): 7 | super(Dummy, self).__init__(remote, token) 8 | 9 | def indicators_create(self, data): 10 | if isinstance(data, dict): 11 | data = self._kv_to_indicator(data) 12 | 13 | return data 14 | 15 | Plugin = Dummy 16 | -------------------------------------------------------------------------------- /test/smrt/test_defang.py: -------------------------------------------------------------------------------- 1 | import py.test 2 | 3 | from csirtg_smrt import Smrt 4 | from pprint import pprint 5 | 6 | 7 | def test_smrt_defang(): 8 | with Smrt(None, None, client='dummy') as s: 9 | assert type(s) is Smrt 10 | 11 | x = [] 12 | for r, f in s.load_feeds('test/smrt/rules/csirtg_defang.yml'): 13 | x = list(s.process(r, f)) 14 | assert len(x) > 0 15 | 16 | -------------------------------------------------------------------------------- /test/client/test_syslog.py: -------------------------------------------------------------------------------- 1 | import py.test 2 | 3 | from csirtg_smrt import Smrt 4 | from pprint import pprint 5 | 6 | 7 | def test_syslog(): 8 | with Smrt(remote='localhost:514', client='syslog') as s: 9 | assert type(s) is Smrt 10 | 11 | rule, feed = next(s.load_feeds('test/smrt/rules/csirtg.yml', feed='port-scanners')) 12 | x = list(s.process(rule, feed)) 13 | assert len(x) > 0 14 | -------------------------------------------------------------------------------- /test/ransomware_abuse_ch/ransomware_abuse_ch.yml: -------------------------------------------------------------------------------- 1 | parser: csv 2 | defaults: 3 | provider: ransomware.abuse.ch 4 | tlp: green 5 | altid_tlp: white 6 | confidence: 9 7 | tags: botnet 8 | application: https 9 | protocol: tcp 10 | values: 11 | - reporttime 12 | - null 13 | - description 14 | - null 15 | - indicator 16 | feeds: 17 | ransomware: 18 | remote: http://ransomwaretracker.abuse.ch/feeds/csv -------------------------------------------------------------------------------- /test/utils/test_smrt_utils.py: -------------------------------------------------------------------------------- 1 | #from csirtg_smrt.utils.zcontent import get_type 2 | import os.path 3 | 4 | T = { 5 | 'test.csv': 'csv', 6 | 'test.rss': 'rss', 7 | 'test.json': 'json', 8 | 'test.tsv': 'tsv', 9 | 'test.xml': 'xml', 10 | } 11 | 12 | 13 | def test_smrt_utils(): 14 | for t in T: 15 | p = ['test', 'utils', t] 16 | p = os.path.join(*p) 17 | #assert T[t] == get_type(p) 18 | -------------------------------------------------------------------------------- /csirtg_smrt/exceptions.py: -------------------------------------------------------------------------------- 1 | 2 | class CsirtgException(Exception): 3 | def __init__(self, msg): 4 | self.msg = "{}".format(msg) 5 | 6 | def __str__(self): 7 | return self.msg 8 | 9 | 10 | class AuthError(CsirtgException): 11 | pass 12 | 13 | 14 | class TimeoutError(CsirtgException): 15 | pass 16 | 17 | 18 | class RuleUnsupported(CsirtgException): 19 | pass 20 | 21 | 22 | class SubmissionFailure(CsirtgException): 23 | pass 24 | -------------------------------------------------------------------------------- /packaging/debian/control: -------------------------------------------------------------------------------- 1 | Source: csirtg-smrt 2 | Priority: optional 3 | Standards-Version: 3.9.3 4 | Maintainer: Wes Young 5 | Build-Depends: cdbs, debhelper (>= 5.0.0) 6 | Homepage: https://github.com/csirtgadgets/csirtg-smrt-py 7 | Section: contrib/csirtg 8 | 9 | Package: csirtg-smrt 10 | Architecture: amd64 11 | Depends: ${misc:Depends} 12 | Description: commandline utility for parsing threat intelligence 13 | The fastest way to consume threat intelligence 14 | -------------------------------------------------------------------------------- /test/client/test_splunk.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import os 3 | from csirtg_smrt import Smrt 4 | from pprint import pprint 5 | 6 | REMOTE = os.environ.get('CSIRTG_SMRT_SPLUNK_NODES', 'localhost:9200') 7 | 8 | @pytest.mark.skip(reason="no way of currently testing this") 9 | def test_smrt_splunk(): 10 | with Smrt(remote=REMOTE, client='splunk') as s: 11 | assert type(s) is Smrt 12 | 13 | x = s.process('test/smrt/rules/csirtg.yml', feed='port-scanners') 14 | assert len(x) > 0 15 | -------------------------------------------------------------------------------- /test/spamhaus/spamhaus.yml: -------------------------------------------------------------------------------- 1 | defaults: 2 | provider: spamhaus.org 3 | confidence: 9 4 | tlp: green 5 | reference_tlp: white 6 | tags: 7 | - suspicious 8 | - hijacked 9 | reference: http://www.spamhaus.org/sbl/sbl.lasso?query= 10 | pattern: '(.+)\s;\s(.+)' 11 | values: 12 | - indicator 13 | - reference 14 | 15 | feeds: 16 | drop: 17 | remote: http://www.spamhaus.org/drop/drop.txt 18 | edrop: 19 | remote: http://www.spamhaus.org/drop/edrop.txt 20 | -------------------------------------------------------------------------------- /csirtg_smrt/decoders/zzip.py: -------------------------------------------------------------------------------- 1 | from zipfile import ZipFile 2 | from csirtg_smrt.constants import PYVERSION 3 | 4 | 5 | def get_lines(file, split="\n"): 6 | with ZipFile(file) as f: 7 | for m in f.infolist(): 8 | if PYVERSION == 2: 9 | for l in f.read(m.filename).split(split): 10 | yield l 11 | else: 12 | with f.open(m.filename) as zip: 13 | for l in zip.readlines(): 14 | yield l 15 | -------------------------------------------------------------------------------- /csirtg_smrt/parser/zsyslog.py: -------------------------------------------------------------------------------- 1 | from csirtg_smrt.parser.pattern import Pattern 2 | import re 3 | 4 | RE_SYSLOG = '\s|\t' 5 | 6 | 7 | class _Syslog(Pattern): 8 | # todo - syslog receiver 9 | # https://gist.github.com/pklaus/c4c37152e261a9e9331f 10 | # https://gist.github.com/marcelom/4218010 11 | 12 | def __init__(self, *args, **kwargs): 13 | super(_Syslog, self).__init__(*args, **kwargs) 14 | 15 | self.pattern = RE_SYSLOG 16 | self.split = '=' 17 | 18 | 19 | Plugin = _Syslog 20 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | csirtg_indicator>=1.0.6,<2 2 | PyYAML>=5.1.0 3 | csirtgsdk>=1.1,<2.0 4 | csirtg_mail>=0.0.0a9,<2 5 | ipaddress>=1.0.16 6 | feedparser>=5.2.1 7 | nltk>=3.7 8 | requests>=2.27.1 9 | pendulum>=2.0.5 10 | arrow>=0.15.2 11 | python-magic>=0.4.6 12 | pyaml>=15.8.2 13 | chardet>=5.0.0 14 | html5lib>=1.1 15 | SQLAlchemy>=1.0.14 16 | tornado>=5.1.0 17 | apwgsdk>=0.0.0a4,<1.0 18 | docker>=6.0.0 19 | lxml==4.9.1 20 | tzlocal>=4.2.0 21 | 22 | urllib3>=1.26.5 # not directly required, pinned by Snyk to avoid a vulnerability 23 | -------------------------------------------------------------------------------- /test/client/test_zyre.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from csirtg_smrt import Smrt 3 | from pprint import pprint 4 | 5 | ZYRE_TEST = False 6 | 7 | try: 8 | import pyzyre 9 | ZYRE_TEST = True 10 | except ImportError: 11 | pass 12 | 13 | 14 | @pytest.mark.skipif(ZYRE_TEST is False, reason='pyzyre not installed') 15 | def test_zyre(): 16 | with Smrt(remote=None, client='zyre') as s: 17 | assert type(s) is Smrt 18 | 19 | x = s.process('test/smrt/rules/csirtg.yml', feed='port-scanners', limit=2) 20 | assert len(x) > 0 21 | -------------------------------------------------------------------------------- /test/smrt/test_csv_quoted.py: -------------------------------------------------------------------------------- 1 | import py.test 2 | 3 | from csirtg_smrt import Smrt 4 | from pprint import pprint 5 | 6 | 7 | def test_smrt_csv_quoted(): 8 | with Smrt(None, None, client='dummy') as s: 9 | assert type(s) is Smrt 10 | 11 | x = [] 12 | for r, f in s.load_feeds('test/smrt/rules/csv_quoted.yml', feed='test'): 13 | x = list(s.process(r, f)) 14 | assert len(x) > 0 15 | 16 | assert x[0].description == '1.2.3, aaabbbcccddd' 17 | assert x[1].description == '1,2,3' 18 | -------------------------------------------------------------------------------- /test/packetmail/packetmail.yml: -------------------------------------------------------------------------------- 1 | parser: semicolon 2 | 3 | defaults: 4 | protocol: tcp 5 | provider: packetmail.net 6 | tlp: green 7 | altid_tlp: white 8 | confidence: 8 9 | description: honeypot traffic 10 | tags: 11 | - scanner 12 | - honeynet 13 | - suspicious 14 | 15 | values: 16 | - indicator 17 | - lasttime 18 | - null 19 | - null 20 | 21 | feeds: 22 | iprep: 23 | remote: https://www.packetmail.net/iprep.txt 24 | description: 'TCP SYN to 206.82.85.196/30 to a non-listening service or daemon' 25 | -------------------------------------------------------------------------------- /examples/cifv3.yml: -------------------------------------------------------------------------------- 1 | parser: cifv3 2 | token: xxxxxxxx 3 | remote: 'https://remote/indicators' 4 | defaults: 5 | provider: provider 6 | group: group 7 | 8 | feeds: 9 | ipv4: 10 | itype: ipv4 11 | filters: 12 | period: hour 13 | itype: ipv4 14 | tags: malware 15 | map: 16 | - tlp 17 | - lasttime 18 | - indicator 19 | - count 20 | - tags 21 | - confidence 22 | values: 23 | - tlp 24 | - lasttime 25 | - indicator 26 | - count 27 | - tags 28 | - confidence 29 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include 'csirtg_smrt' 2 | include versioneer.py 3 | include README README.md COPYING LICENSE 4 | include MANIFEST.in 5 | include requirements.txt extras_requirements.txt 6 | exclude dev_requirements.txt 7 | include test 8 | recursive-include examples * 9 | recursive-include test *.yml 10 | recursive-include test *.txt 11 | recursive-include test *.gz 12 | recursive-include test *.zip 13 | 14 | recursive-exclude * __pycache__ 15 | recursive-exclude * *.pyc 16 | recursive-exclude * *.pyo 17 | recursive-exclude * *.orig 18 | recursive-exclude packaging * 19 | -------------------------------------------------------------------------------- /test/vxvault/feed.txt: -------------------------------------------------------------------------------- 1 | VX Vault last 100 Links 2 | Wed, 23 Mar 2016 23:10:10 +0000 3 | 4 | http://store.suhaskhamkar.in/plmaz 5 | http://antalyanalburiye.com/image/payment/client.exe 6 | http://tirekoypazari.com/lso30sd 7 | http://naipeclandestino.com.br/image/data/office.exe 8 | http://tribudellusato.altervista.org/image/templates/office.exe 9 | http://jeansowghtqq.com/93.exe 10 | http://jeansowghtqq.com/87.exe 11 | http://jeansowghtqq.com/85.exe 12 | http://jeansowghtqq.com/80.exe 13 | http://jeansowghtqq.com/69.exe 14 | http://jeansowghtqq.com/25.exe 15 | http://jeansowghtqq.com/23.exe -------------------------------------------------------------------------------- /test/bro/test_bro.py: -------------------------------------------------------------------------------- 1 | from csirtg_smrt.parser.bro import BroTailer 2 | from pprint import pprint 3 | 4 | 5 | def test_bro(): 6 | file = 'test/bro/bro.log' 7 | 8 | b = BroTailer(file) 9 | 10 | events = [] 11 | with open(file) as f: 12 | for l in f.read().split("\n"): 13 | if l.startswith('#'): 14 | continue 15 | 16 | i = b.parse_line(l) 17 | 18 | if not i: 19 | continue 20 | 21 | events.append(i) 22 | 23 | assert len(events) > 0 24 | assert events[0]['indicator'] == '138.117.125.206' 25 | -------------------------------------------------------------------------------- /test/openphish/test_openphish.py: -------------------------------------------------------------------------------- 1 | import py.test 2 | 3 | from csirtg_smrt import Smrt 4 | from csirtg_smrt.rule import Rule 5 | from csirtg_smrt.constants import REMOTE_ADDR 6 | from pprint import pprint 7 | import json 8 | 9 | rule = 'test/openphish/openphish.yml' 10 | rule = Rule(path=rule) 11 | rule.fetcher = 'file' 12 | s = Smrt(REMOTE_ADDR, 1234, client='dummy') 13 | 14 | 15 | def test_openphish(): 16 | rule.feeds['urls']['remote'] = 'test/openphish/feed.txt' 17 | x = s.process(rule, feed="urls") 18 | x = list(x) 19 | 20 | assert len(x) > 0 21 | assert len(x[0].indicator) > 4 22 | -------------------------------------------------------------------------------- /test/phishtank/phishtank.yml: -------------------------------------------------------------------------------- 1 | parser: json 2 | remote: http://data.phishtank.com/data/online-valid.json.gz 3 | defaults: 4 | provider: phishtank.com 5 | tlp: green 6 | altid_tlp: white 7 | application: 8 | - http 9 | - https 10 | confidence: 9 11 | tags: phishing 12 | protocol: tcp 13 | 14 | feeds: 15 | urls: 16 | itype: url 17 | map: 18 | - submission_time 19 | - url 20 | - target 21 | - phish_detail_url 22 | - details 23 | values: 24 | - lasttime 25 | - indicator 26 | - description 27 | - altid 28 | - additional_data 29 | -------------------------------------------------------------------------------- /csirtg_smrt/client/zcif.py: -------------------------------------------------------------------------------- 1 | from cifsdk.client.http import HTTP as HTTPClient 2 | 3 | import os 4 | 5 | REMOTE = os.getenv('CIF_REMOTE', 'http://localhost:5000') 6 | TOKEN = os.getenv('CIF_TOKEN') 7 | 8 | if REMOTE == '': 9 | REMOTE = 'http://localhost:5000' 10 | 11 | if TOKEN == '': 12 | TOKEN = None 13 | 14 | 15 | class CIF(HTTPClient): 16 | 17 | def __init__(self, remote=REMOTE, token=TOKEN, **kwargs): 18 | if not remote: 19 | remote = REMOTE 20 | 21 | if not token: 22 | token = TOKEN 23 | 24 | super(CIF, self).__init__(remote, token, **kwargs) 25 | 26 | Plugin = CIF 27 | -------------------------------------------------------------------------------- /test/utils/test.rss: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Malc0de Database Feed 6 | http://malc0de.com/database/ 7 | Updated Feed of Malicious Executables 8 | en-us 9 | Copyright (C) 2010 malc0de.com 10 | 11 | down12.xiazaidc.com 12 | http://malc0de.com/database/index.php?search=down12.xiazaidc.com 13 | URL: , IP Address: 121.41.10.159, Country: CN, ASN: 37963, MD5: 2a65f85d09f36402fbd91484a9a4adac 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /csirtg_smrt/client/plugin.py: -------------------------------------------------------------------------------- 1 | from csirtg_indicator import Indicator 2 | import abc 3 | 4 | 5 | class Client(object): 6 | 7 | def __init__(self, remote=None, token=None, username=None): 8 | if remote: 9 | self.remote = remote 10 | 11 | if token: 12 | self.token = token 13 | 14 | if username: 15 | self.username = username 16 | 17 | def _kv_to_indicator(self, kv): 18 | return Indicator(**kv) 19 | 20 | def ping(self, write=False): 21 | return True 22 | 23 | @abc.abstractmethod 24 | def indicators_create(self, data, **kwargs): 25 | raise NotImplementedError 26 | -------------------------------------------------------------------------------- /csirtg_smrt/client/zsplunk.py: -------------------------------------------------------------------------------- 1 | from csirtg_smrt.client.plugin import Client 2 | 3 | # https://github.com/splunk/splunk-sdk-python/blob/master/examples/submit.py 4 | class _Splunk(Client): 5 | 6 | __name__ = 'splunk' 7 | 8 | def __init__(self, remote='localhost:514', *args, **kwargs): 9 | super(_Splunk, self).__init__(remote) 10 | 11 | raise NotImplemented 12 | 13 | def ping(self, write=False): 14 | raise NotImplemented 15 | 16 | def indicators_create(self, data, **kwargs): 17 | # https://github.com/splunk/splunk-sdk-python/blob/master/examples/submit.py 18 | raise NotImplemented 19 | 20 | Plugin = _Splunk 21 | -------------------------------------------------------------------------------- /test/zemail/test_zemail.py: -------------------------------------------------------------------------------- 1 | import py.test 2 | 3 | from csirtg_smrt import Smrt 4 | from csirtg_smrt.rule import Rule 5 | from csirtg_smrt.constants import REMOTE_ADDR 6 | from csirtg_smrt.constants import PYVERSION 7 | 8 | rule = 'test/zemail/zemail.yml' 9 | rule = Rule(path=rule) 10 | rule.fetcher = 'stdin' 11 | s = Smrt(REMOTE_ADDR, 1234, client='dummy') 12 | 13 | 14 | def test_zemail(): 15 | feed = 'abuse' 16 | with open('test/zemail/single_plain_01.eml') as f: 17 | data = f.read() 18 | 19 | x = list(s.process(rule, feed=feed, data=data)) 20 | 21 | assert len(x) > 0 22 | 23 | assert x[0].indicator == 'http://www.socialservices.cn/detail.php?id=9' 24 | -------------------------------------------------------------------------------- /csirtg_smrt/parser/zindicator.py: -------------------------------------------------------------------------------- 1 | from csirtg_smrt.parser import Parser 2 | from pprint import pprint 3 | 4 | 5 | class _Indicator(Parser): 6 | def __init__(self, *args, **kwargs): 7 | super(_Indicator, self).__init__(*args, **kwargs) 8 | 9 | def process(self): 10 | defaults = self._defaults() 11 | 12 | for l in self.fetcher.process(): 13 | for e in l: 14 | i = {} 15 | e = e.__dict__() 16 | for k, v in e.items(): 17 | i[k] = v 18 | 19 | for k, v in defaults.items(): 20 | i[k] = v 21 | 22 | yield i 23 | 24 | 25 | Plugin = _Indicator 26 | -------------------------------------------------------------------------------- /test/alexa/alexa.yml: -------------------------------------------------------------------------------- 1 | parser: csv 2 | defaults: 3 | values: 4 | - rank 5 | - indicator 6 | description: 'eval("alexa #{rank}".format(**obs))' 7 | tags: whitelist 8 | application: 9 | - http 10 | - https 11 | protocol: tcp 12 | altid: 'eval("http://www.alexa.com/siteinfo/{indicator}".format(**obs))' 13 | provider: alexa.com 14 | tlp: green 15 | altid_tlp: white 16 | confidence: | 17 | eval(max(0, min( 18 | 125 - 25 * math.ceil( 19 | math.log10( 20 | int(obs['rank']) 21 | ) 22 | ), 23 | 95))) 24 | 25 | feeds: 26 | topN: 27 | remote: http://s3.amazonaws.com/alexa-static/top-1m.csv.zip 28 | limit: 10100 29 | -------------------------------------------------------------------------------- /test/csirtg/feed2_csv.txt: -------------------------------------------------------------------------------- 1 | "id","indicator","itype","portlist","firsttime","lasttime","protocol","application","feed_id","created_at","updated_at","description","portlist_src" 2 | "337406","80.82.78.27","ipv4","3389","","","6","","46","2016-03-23 20:35:01 UTC","2016-03-23 20:35:01 UTC","sourced from firewall logs (incomming WAN, TCP, Syn, blocked)","42775" 3 | #"337401","216.243.31.2","ipv4","443","","2016-03-23 20:17:26 UTC","6","","46,2016-03-23 20:22:27 UTC,2016-03-23 20:22:27 UTC","sourced from firewall logs (incomming, TCP, Syn, blocked)", 4 | #"337400","91.236.75.4","ipv4","8080","","2016-03-23 20:14:53 UTC","6","","46,2016-03-23 20:17:26 UTC,2016-03-23 20:17:26 UTC","sourced from firewall logs (incomming, TCP, Syn, blocked)", -------------------------------------------------------------------------------- /test/pastebin/test_pastebin.py: -------------------------------------------------------------------------------- 1 | import py.test 2 | 3 | from csirtg_smrt import Smrt 4 | from csirtg_smrt.rule import Rule 5 | from csirtg_smrt.constants import REMOTE_ADDR 6 | from pprint import pprint 7 | 8 | rule = 'test/pastebin/pastebin.yml' 9 | rule = Rule(path=rule) 10 | rule.fetcher = 'file' 11 | 12 | s = Smrt(REMOTE_ADDR, 1234, client='dummy') 13 | 14 | 15 | def test_pastebin_creds(): 16 | rule.feeds['creds']['remote'] = 'test/pastebin/feed.txt' 17 | x = s.process(rule, feed="creds") 18 | x = list(x) 19 | 20 | assert len(x) > 0 21 | 22 | indicators = set() 23 | 24 | for xx in x: 25 | indicators.add(xx.indicator) 26 | 27 | assert 'a.clayton10@gmail.com' in indicators 28 | -------------------------------------------------------------------------------- /test/smrt/test_remote_regex.py: -------------------------------------------------------------------------------- 1 | import py.test 2 | 3 | from csirtg_smrt import Smrt 4 | from pprint import pprint 5 | 6 | 7 | def test_smrt_remote_regex(): 8 | with Smrt(None, None, client='dummy') as s: 9 | assert type(s) is Smrt 10 | 11 | x = [] 12 | for r, f in s.load_feeds('test/smrt/remote_regex.yml', feed='port-scanners'): 13 | x = list(s.process(r, f)) 14 | assert len(x) > 0 15 | 16 | x = [] 17 | for r, f in s.load_feeds('test/smrt/remote_regex.yml', feed='port-scanners-fail'): 18 | try: 19 | x = list(s.process(r, f)) 20 | except RuntimeError as e: 21 | pass 22 | 23 | assert len(x) == 0 24 | 25 | -------------------------------------------------------------------------------- /test/sansedu/test_sans.py: -------------------------------------------------------------------------------- 1 | import py.test 2 | 3 | from csirtg_smrt import Smrt 4 | from csirtg_smrt.rule import Rule 5 | from csirtg_smrt.constants import REMOTE_ADDR 6 | from pprint import pprint 7 | 8 | rule = 'test/sansedu/sans_edu.yml' 9 | rule = Rule(path=rule) 10 | rule.fetcher = 'file' 11 | s = Smrt(REMOTE_ADDR, 1234, client='dummy') 12 | 13 | 14 | def test_sans_low(): 15 | feed = '02_domains_low' 16 | x = s.process(rule, feed=feed) 17 | x = list(x) 18 | 19 | assert len(x) > 0 20 | 21 | assert len(x[0].indicator) > 4 22 | 23 | 24 | def test_sans_block(): 25 | feed = 'block' 26 | x = s.process(rule, feed=feed) 27 | x = list(x) 28 | 29 | assert len(x) > 0 30 | assert len(x[0].indicator) > 4 31 | -------------------------------------------------------------------------------- /test/spamhaus/test_spamhaus.py: -------------------------------------------------------------------------------- 1 | import py.test 2 | 3 | from csirtg_smrt import Smrt 4 | from csirtg_smrt.rule import Rule 5 | from csirtg_smrt.constants import REMOTE_ADDR 6 | from pprint import pprint 7 | 8 | rule = 'test/spamhaus/spamhaus.yml' 9 | rule = Rule(path=rule) 10 | rule.fetcher = 'file' 11 | s = Smrt(REMOTE_ADDR, 1234, client='dummy') 12 | 13 | 14 | def test_spamhaus_drop(): 15 | rule.feeds['drop']['remote'] = 'test/spamhaus/drop.txt' 16 | x = s.process(rule, feed="drop") 17 | x = list(x) 18 | assert len(list(x)) > 0 19 | 20 | 21 | def test_spamhaus_edrop(): 22 | rule.feeds['edrop']['remote'] = 'test/spamhaus/edrop.txt' 23 | x = s.process(rule, feed="edrop") 24 | x = list(x) 25 | assert len(x) > 0 26 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | # .coveragerc to control coverage.py 2 | [run] 3 | branch = True 4 | 5 | [report] 6 | # Regexes for lines to exclude from consideration 7 | exclude_lines = 8 | # Have to re-enable the standard pragma 9 | pragma: no cover 10 | 11 | # Don't complain about missing debug-only code: 12 | def __repr__ 13 | if self\.debug 14 | 15 | # Don't complain if tests don't hit defensive assertion code: 16 | raise AssertionError 17 | raise NotImplementedError 18 | 19 | # Don't complain if non-runnable code isn't run: 20 | if 0: 21 | if __name__ == .__main__.: 22 | 23 | ignore_errors = True 24 | 25 | omit = 26 | #csirtg/smrt/parser/pipe.py 27 | #cif/_version.py 28 | 29 | [html] 30 | directory = coverage_html_report 31 | -------------------------------------------------------------------------------- /test/phishtank/test_phishtank.py: -------------------------------------------------------------------------------- 1 | import py.test 2 | 3 | from csirtg_smrt import Smrt 4 | from csirtg_smrt.rule import Rule 5 | from csirtg_smrt.constants import REMOTE_ADDR 6 | from pprint import pprint 7 | 8 | rule = 'test/phishtank/phishtank.yml' 9 | rule = Rule(path=rule) 10 | rule.fetcher = 'file' 11 | s = Smrt(REMOTE_ADDR, 1234, client='dummy') 12 | 13 | 14 | def test_phishtank_urls(): 15 | rule.remote = 'test/phishtank/feed.json.gz' 16 | x = s.process(rule, feed="urls") 17 | x = list(x) 18 | 19 | assert len(x) > 0 20 | assert len(x[0].indicator) > 4 21 | 22 | indicators = set() 23 | for i in x: 24 | indicators.add(i.indicator) 25 | 26 | assert 'http://charlesleonardconstruction.com/irs/confim/index.html' in indicators 27 | -------------------------------------------------------------------------------- /test/smrt/rules/csirtg_defang.yml: -------------------------------------------------------------------------------- 1 | --- 2 | parser: csv 3 | replace: 4 | indicator: 5 | '[.]': '.' 6 | 7 | defaults: 8 | provider: csirtg.io 9 | altid_tlp: white 10 | altid: https://csirtg.io/search?q={indicator} 11 | tlp: white 12 | confidence: 9 13 | indicator: eval(indicator.replace('[.]', '.')) 14 | values: 15 | - null 16 | - indicator 17 | - itype 18 | - portlist 19 | - null 20 | - null 21 | - protocol 22 | - application 23 | - null 24 | - firsttime 25 | - lasttime 26 | - description 27 | - null 28 | 29 | feeds: 30 | # A feed of IP addresses block by a firewall (e.g. port scanners) 31 | port-scanners: 32 | remote: 'test/smrt/data/feed_defanged.txt' 33 | defaults: 34 | tags: 35 | - scanner -------------------------------------------------------------------------------- /test/vxvault/test_vxvault.py: -------------------------------------------------------------------------------- 1 | import py.test 2 | 3 | from csirtg_smrt import Smrt 4 | from csirtg_smrt.rule import Rule 5 | from csirtg_smrt.constants import REMOTE_ADDR 6 | from pprint import pprint 7 | 8 | rule = 'test/vxvault/vxvault.yml' 9 | rule = Rule(path=rule) 10 | rule.fetcher = 'file' 11 | 12 | s = Smrt(REMOTE_ADDR, 1234, client='dummy') 13 | 14 | 15 | def test_vxvault_urls(): 16 | rule.feeds['urls']['remote'] = 'test/vxvault/feed.txt' 17 | x = s.process(rule, feed="urls") 18 | x = list(x) 19 | 20 | assert len(x) > 0 21 | 22 | urls = set() 23 | tags = set() 24 | 25 | for xx in x: 26 | urls.add(xx.indicator) 27 | tags.add(xx.tags[0]) 28 | 29 | assert 'http://jeansowghtqq.com/85.exe' in urls 30 | assert 'malware' in tags -------------------------------------------------------------------------------- /test/malc0de/malc0de.yml: -------------------------------------------------------------------------------- 1 | parser: rss 2 | remote: http://malc0de.com/rss 3 | defaults: 4 | confidence: 9 5 | tlp: green 6 | altid_tlp: white 7 | provider: malc0de.com 8 | tags: malware 9 | 10 | feeds: 11 | urls: 12 | itype: url 13 | pattern: 14 | description: 15 | pattern: '^URL: (.+), IP Address: \S+?, Country: \S+, ASN: \S+, MD5: \S+' 16 | values: 17 | - indicator 18 | link: 19 | pattern: '(\S+)' 20 | values: 21 | - altid 22 | malware: 23 | itype: md5 24 | pattern: 25 | description: 26 | pattern: '^URL: .+, IP Address: \S+?, Country: \S+, ASN: \S+, MD5: (\S+)' 27 | values: 28 | - indicator 29 | link: 30 | pattern: '(\S+)' 31 | values: 32 | - altid -------------------------------------------------------------------------------- /test/test_timestamps.py: -------------------------------------------------------------------------------- 1 | import py.test 2 | from csirtg_smrt.utils.zarrow import parse_timestamp 3 | import arrow 4 | 5 | 6 | def test_timestamps(): 7 | ts = { 8 | '2015-01-01': arrow.get('2015-01-01 00:00:00Z'), 9 | '2015-01-01T23:59:59Z': arrow.get('2015-01-01 23:59:59Z'), 10 | 1367900664: arrow.get('2013-05-07T04:24:24+00:00'), 11 | '20160401': arrow.get('2016-04-01T00:00:00+00:00'), 12 | '2015-01-05T00:00:00.00Z': arrow.get('2015-01-05 00:00:00Z'), 13 | '2014-01-01T23:59+04:00': arrow.get('2014-01-01T23:59:00+04:00'), 14 | '20130601235959': arrow.get('2013-06-01T23:59:59+00:00'), 15 | } 16 | 17 | for t in ts: 18 | x = parse_timestamp(t) 19 | print(t, x) 20 | assert x == ts[t] -------------------------------------------------------------------------------- /test/ufw/test_ufw.py: -------------------------------------------------------------------------------- 1 | from csirtg_smrt.parser.ufw import parse_line 2 | 3 | 4 | def test_ufw(): 5 | file = 'test/ufw/ufw.log' 6 | 7 | events = [] 8 | with open(file) as f: 9 | for l in f.read().split("\n"): 10 | i = parse_line(l) 11 | 12 | events.append(i) 13 | 14 | assert len(events) > 0 15 | assert events[0]['indicator'] == '114.33.197.193' 16 | 17 | 18 | def test_ufw_ubuntu16(): 19 | file = 'test/ufw/ufw_ubuntu16.log' 20 | 21 | events = [] 22 | with open(file) as f: 23 | for l in f.read().split("\n"): 24 | i = parse_line(l) 25 | 26 | events.append(i) 27 | 28 | assert len(events) > 0 29 | assert events[0]['indicator'] == '10.0.2.2' 30 | assert events[1]['indicator'] == '61.7.190.140' 31 | -------------------------------------------------------------------------------- /test/sansedu/sans_edu.yml: -------------------------------------------------------------------------------- 1 | skip: '^Site$' 2 | defaults: 3 | tlp: green 4 | reference_tlp: white 5 | provider: 'isc.sans.edu' 6 | pattern: '^(.+)$' 7 | values: indicator 8 | tags: suspicious 9 | 10 | feeds: 11 | 02_domains_low: 12 | confidence: 7 13 | remote: 'test/sansedu/low.txt' 14 | 15 | 01_domains_medium: 16 | confidence: 8 17 | remote: http://isc.sans.edu/feeds/suspiciousdomains_Medium.txt 18 | 19 | 00_domains_high: 20 | confidence: 9 21 | remote: http://isc.sans.edu/feeds/suspiciousdomains_High.txt 22 | 23 | block: 24 | remote: 'test/sansedu/block.txt' 25 | confidence: 8 26 | pattern: ^(\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b)\t\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b\t(\d+) 27 | values: 28 | - indicator 29 | - mask 30 | tags: scanner 31 | -------------------------------------------------------------------------------- /test/spamcop/test_spamcop.py: -------------------------------------------------------------------------------- 1 | import py.test 2 | 3 | from csirtg_smrt import Smrt 4 | from csirtg_smrt.rule import Rule 5 | from csirtg_smrt.constants import REMOTE_ADDR 6 | from csirtg_smrt.constants import PYVERSION 7 | 8 | rule = 'test/spamcop/spamcop.yml' 9 | rule = Rule(path=rule) 10 | rule.fetcher = 'stdin' 11 | s = Smrt(REMOTE_ADDR, 1234, client='dummy') 12 | 13 | 14 | def test_spamcop(): 15 | feed = 'abuse' 16 | rule.feeds[feed]['remote'] = 'stdin' 17 | with open('test/spamcop/email1.txt') as f: 18 | data = f.read() 19 | 20 | x = list(s.process(rule, feed=feed, data=data)) 21 | 22 | assert len(x) > 0 23 | 24 | assert len(x[0].indicator) > 4 25 | 26 | assert x[0].indicator == '204.93.2.6' 27 | 28 | assert x[0].message.startswith('znjvbtogehh4qhj') 29 | -------------------------------------------------------------------------------- /test/packetmail/test_packetmail.py: -------------------------------------------------------------------------------- 1 | import py.test 2 | 3 | from csirtg_smrt import Smrt 4 | from csirtg_smrt.rule import Rule 5 | from csirtg_smrt.constants import REMOTE_ADDR 6 | from pprint import pprint 7 | 8 | rule = 'test/packetmail/packetmail.yml' 9 | rule = Rule(path=rule) 10 | rule.fetcher = 'file' 11 | 12 | s = Smrt(REMOTE_ADDR, 1234, client='dummy') 13 | 14 | 15 | def test_packetmail_iprep(): 16 | rule.feeds['iprep']['remote'] = 'test/packetmail/feed.txt' 17 | x = s.process(rule, feed="iprep") 18 | x = list(x) 19 | assert len(x) > 0 20 | 21 | ips = set() 22 | tags = set() 23 | 24 | for xx in x: 25 | ips.add(xx.indicator) 26 | for t in xx.tags: 27 | tags.add(t) 28 | 29 | assert '179.40.212.141' in ips 30 | assert '104.131.128.9' in ips 31 | 32 | assert 'honeynet' in tags 33 | -------------------------------------------------------------------------------- /test/smrt/remote_regex.yml: -------------------------------------------------------------------------------- 1 | --- 2 | parser: csv 3 | 4 | defaults: 5 | provider: csirtg.io 6 | altid_tlp: white 7 | altid: https://csirtg.io/search?q= 8 | tlp: white 9 | confidence: 9 10 | values: 11 | - null 12 | - indicator 13 | - itype 14 | - portlist 15 | - null 16 | - null 17 | - protocol 18 | - application 19 | - null 20 | - firsttime 21 | - lasttime 22 | - description 23 | - null 24 | 25 | feeds: 26 | # A feed of IP addresses block by a firewall (e.g. port scanners) 27 | port-scanners: 28 | remote: 'test/smrt/data/' 29 | remote_pattern: '^feed_regex_\d+\-\d+-\d+\.csv$' 30 | defaults: 31 | tags: 32 | - scanner 33 | 34 | port-scanners-fail: 35 | remote: 'test/smrt/data/' 36 | remote_pattern: '^feed_regex_\d+\-\d+-\d+\.csv2$' 37 | defaults: 38 | tags: 39 | - scanner -------------------------------------------------------------------------------- /test/alienvault/feed.txt: -------------------------------------------------------------------------------- 1 | 114.143.191.19#3#2#Scanning Host#IN#Pune#18.5333003998,73.8666992188#11 2 | 68.15.122.196#3#2#Scanning Host#US#Harrah#35.5018005371,-97.1324005127#11 3 | 60.173.12.98#3#2#Scanning Host#CN#Shanghai#31.0456008911,121.39969635#11 4 | 93.158.211.210#4#4#Malicious Host;Scanning Host#NL##52.3666992188,4.90000009537#3;11 5 | 223.252.32.73#3#2#Scanning Host#AU#Brisbane#-27.4710006714,153.024307251#11 6 | 180.97.215.63#3#2#Scanning Host#CN#Nanjing#32.0616989136,118.777801514#11 7 | 150.185.222.55#3#2#Scanning Host#VE#Maracaibo#10.6316995621,-71.6406021118#11 8 | 47.50.164.186#3#2#Scanning Host#US#Saint Louis#38.6783981323,-90.3769989014#11 9 | 63.143.42.247#6#4#Spamming#US#Dallas#32.7790985107,-96.8028030396#12 10 | 23.92.83.73#6#2#Spamming#US##38.0,-97.0#12 11 | 93.127.228.36#6#3#Spamming#DE#Frankfurt Am Main#50.1166992188,8.68330001831#12 12 | 63.143.42.246#9#3#Spamming#US#Dallas#32.7790985107,-96.8028030396#12 -------------------------------------------------------------------------------- /test/ransomware_abuse_ch/test_ransomware_abuse_ch.py: -------------------------------------------------------------------------------- 1 | import py.test 2 | 3 | from csirtg_smrt import Smrt 4 | from csirtg_smrt.rule import Rule 5 | from csirtg_smrt.constants import REMOTE_ADDR 6 | from pprint import pprint 7 | 8 | rule = 'test/ransomware_abuse_ch/ransomware_abuse_ch.yml' 9 | rule = Rule(path=rule) 10 | rule.fetcher = 'file' 11 | 12 | s = Smrt(REMOTE_ADDR, 1234, client='dummy') 13 | 14 | 15 | def test_abuse_ch_ransomware(): 16 | rule.feeds['ransomware']['remote'] = 'test/ransomware_abuse_ch/feed.txt' 17 | x = s.process(rule, feed="ransomware") 18 | x = list(x) 19 | assert len(x) > 0 20 | 21 | indicators = set() 22 | tags = set() 23 | 24 | from pprint import pprint 25 | pprint(x) 26 | 27 | for xx in x: 28 | indicators.add(xx.indicator) 29 | tags.add(xx.tags[0]) 30 | 31 | assert 'http://grandaareyoucc.asia/85.exe' in indicators 32 | assert 'botnet' in tags 33 | -------------------------------------------------------------------------------- /test/bambenek/bambenek.yml: -------------------------------------------------------------------------------- 1 | defaults: 2 | provider: osint.bambenekconsulting.com 3 | tlp: white 4 | altid_tlp: white 5 | confidence: 9 6 | tags: botnet 7 | values: 8 | - indicator 9 | - description 10 | - lasttime 11 | - altid 12 | 13 | feeds: 14 | c2-dommasterlist: 15 | remote: http://osint.bambenekconsulting.com/feeds/c2-dommasterlist.txt 16 | pattern: ^(\S+)\,Domain used by ([^,]+)\,([^,]+)\,(\S+)$ 17 | c2-ipmasterlist: 18 | remote: http://osint.bambenekconsulting.com/feeds/c2-ipmasterlist.txt 19 | pattern: ^(\S+)\,IP used by ([^,]+)\,([^,]+)\,(\S+)$ 20 | 21 | # Warning: the dga-feed is disabled by default as it is a very large 22 | # feed at ~800K records and 95MB in size. You must have a 23 | # large CIF instance to process this data set. 24 | # dga-feed: 25 | # remote: http://osint.bambenekconsulting.com/feeds/dga-feed.txt 26 | # pattern: ^(\S+)\,Domain used by ([^,]+)\,([^,]+)\,(\S+)$ -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | #e -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | # Vagrantfile API/syntax version. Don't touch unless you know what you're doing! 5 | VAGRANTFILE_API_VERSION = "2" 6 | VAGRANTFILE_LOCAL = 'Vagrantfile.local' 7 | 8 | $script = <