├── tip
    ├── __init__.py
    ├── __main__.py
    ├── elasticsearch
    │   ├── index_settings.json
    │   └── index_mapping.json
    ├── emergingthreats.py
    ├── abuseipdb.py
    ├── eset.py
    ├── test_ioc.py
    ├── spamhaus.py
    ├── botvrij.py
    ├── abuse_bazaar.py
    ├── elastic_tip_cli.py
    ├── elastic_tip.py
    └── ioc.py
├── .gitignore
├── requirements.txt
├── schema.json
├── .github
    ├── ISSUE_TEMPLATE
    │   ├── -fr---title-.md
    │   └── -bug---short-title-.md
    └── workflows
    │   └── main.yml
├── examples
    ├── ioc_matching_ips.toml
    └── ioc_matching_files.toml
├── README.md
└── LICENSE.md


/tip/__init__.py:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | .idea/
2 | venv/
3 | tip_venv/
4 | *pyc*
5 | schema_proposal.md
6 | 


--------------------------------------------------------------------------------
/tip/__main__.py:
--------------------------------------------------------------------------------
1 | from elastic_tip_cli import CLI
2 | tip_cli = CLI()
3 | tip_cli.cli()


--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | pip~=9.0.1
2 | wheel~=0.30.0
3 | setuptools~=39.0.1
4 | requests
5 | elasticsearch>=7.0.0,<8.0.0
6 | gitpython
7 | tqdm


--------------------------------------------------------------------------------
/tip/elasticsearch/index_settings.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "index": {
 3 |     "number_of_shards": 1,
 4 |     "number_of_replicas": 1,
 5 |     "refresh_interval": "1m",
 6 |     "max_result_window": 10000,
 7 |     "routing": {
 8 |       "allocation": {
 9 |         "include": {
10 |           "_tier_preference": "data_content,data_hot"
11 |         }
12 |       }
13 |     }
14 |   }
15 | }


--------------------------------------------------------------------------------
/schema.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "etip": {
 3 |     "ioc": {
 4 |       "reference": [
 5 |         "https://example.com/"
 6 |       ],
 7 |       "value": "",
 8 |       "type": "",
 9 |       "provider": {
10 |         "name": "",
11 |         "creator": "",
12 |         "reference": ""
13 |       },
14 |       "original": "",
15 |     }
16 |   },
17 |   "threat": {},
18 |   "rule": {},
19 |   "vulnerability": {}
20 | }


--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/-fr---title-.md:
--------------------------------------------------------------------------------
 1 | ---
 2 | name: "[FR] <title>"
 3 | about: Suggest an idea for this project
 4 | title: ''
 5 | labels: enhancement
 6 | assignees: ''
 7 | 
 8 | ---
 9 | 
10 | **Is your feature request related to a problem? Please describe.**
11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
12 | 
13 | **Describe the solution you'd like**
14 | A clear and concise description of what you want to happen.
15 | 
16 | **Additional context**
17 | Add any other context or screenshots about the feature request here.
18 | 


--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/-bug---short-title-.md:
--------------------------------------------------------------------------------
 1 | ---
 2 | name: "[BUG] <short title>"
 3 | about: Create a report to help us improve
 4 | title: ''
 5 | labels: bug
 6 | assignees: ''
 7 | 
 8 | ---
 9 | 
10 | **Describe the bug**
11 | A clear and concise description of what the bug is.
12 | 
13 | **To Reproduce**
14 | Steps to reproduce the behavior:
15 | 1. Go to '...'
16 | 2. Click on '....'
17 | 3. Scroll down to '....'
18 | 4. See error
19 | 
20 | **Expected behavior**
21 | A clear and concise description of what you expected to happen.
22 | 
23 | **Screenshots**
24 | If applicable, add screenshots to help explain your problem.
25 | 
26 | **Additional context**
27 | Add any other context about the problem here.
28 | 


--------------------------------------------------------------------------------
/examples/ioc_matching_ips.toml:
--------------------------------------------------------------------------------
 1 | [metadata]
 2 | creation_date = "2021/02/28"
 3 | maturity = "production"
 4 | updated_date = "2021/02/28"
 5 | 
 6 | [rule]
 7 | author = ["Stijn Holzhauer"]
 8 | description = '''
 9 | A host is communicating with a destination.ip which is known to be malicious and is registered as Indicator Of Compromise.
10 | '''
11 | index = ["packetbeat-*"]
12 | language = "kuery"
13 | license = "Elastic License"
14 | name = "IOC match on IP address"
15 | risk_score = 100
16 | rule_id = "312971de-17de-405e-8a6a-978d5e9841e2"
17 | severity = "high"
18 | tags = ["ioc", "network"]
19 | type = "threat_matching"
20 | 
21 | query = '''
22 | destination.ip:*
23 | '''
24 | 
25 | threat_index = ["elastic-tip"]
26 | threat_query = "*:*"
27 | 
28 | [[threat_mapping]]
29 | [[threat_mapping.entries]]
30 | field="destination.ip"
31 | type="mapping"
32 | value="threat.ip"
33 | 


--------------------------------------------------------------------------------
/examples/ioc_matching_files.toml:
--------------------------------------------------------------------------------
 1 | [metadata]
 2 | creation_date = "2021/02/28"
 3 | maturity = "production"
 4 | updated_date = "2021/02/28"
 5 | 
 6 | [rule]
 7 | author = ["Stijn Holzhauer"]
 8 | description = '''
 9 | A file was detected which has a hash corresponding to a known Indicator Of Compromise
10 | '''
11 | index = ["auditbeat-*"]
12 | language = "kuery"
13 | license = "Elastic License"
14 | name = "IOC match on file hash"
15 | risk_score = 99
16 | rule_id = "a2db1d22-de51-4cbe-bb40-d7abd2450512"
17 | severity = "high"
18 | tags = ["ioc", "file"]
19 | type = "threat_matching"
20 | query = '''
21 | event.module:file_integrity
22 | '''
23 | threat_index = ["elastic-tip"]
24 | threat_query = "*:*"
25 | 
26 | [[threat_mapping]]
27 | [[threat_mapping.entries]]
28 | field="file.hash.md5"
29 | type="mapping"
30 | value="file.hash.md5"
31 | [[threat_mapping]]
32 | [[threat_mapping.entries]]
33 | field="file.hash.sha1"
34 | type="mapping"
35 | value="file.hash.sha1"
36 | [[threat_mapping]]
37 | [[threat_mapping.entries]]
38 | field="file.hash.sha256"
39 | type="mapping"
40 | value="file.hash.sha256"
41 | [[threat_mapping]]
42 | [[threat_mapping.entries]]
43 | field="file.hash.sha512"
44 | type="mapping"
45 | value="file.hash.sha512"
46 | 


--------------------------------------------------------------------------------
/.github/workflows/main.yml:
--------------------------------------------------------------------------------
 1 | # This is a basic workflow to help you get started with Actions
 2 | 
 3 | name: CI
 4 | 
 5 | # Controls when the action will run. 
 6 | on:
 7 |   # Triggers the workflow on push or pull request events but only for the main branch
 8 |   push:
 9 |     branches: [ main ]
10 | 
11 |   # Allows you to run this workflow manually from the Actions tab
12 |   workflow_dispatch:
13 | 
14 | # A workflow run is made up of one or more jobs that can run sequentially or in parallel
15 | jobs:
16 |   # This workflow contains a single job called "build"
17 |   build:
18 |     # The type of runner that the job will run on
19 |     runs-on: ubuntu-latest
20 | 
21 |     # Steps represent a sequence of tasks that will be executed as part of the job
22 |     steps:
23 |       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
24 |       - uses: actions/checkout@v2
25 | 
26 |       - name: Install dependencies
27 |         run: |
28 |           python -m pip install --upgrade pip
29 |           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
30 |       # Runs a set of commands using the runners shell
31 |       - name: Unit testing
32 |         run: |
33 |           python tip/test_ioc.py
34 | 


--------------------------------------------------------------------------------
/tip/emergingthreats.py:
--------------------------------------------------------------------------------
 1 | from ioc import Intel
 2 | import requests
 3 | from time import time
 4 | 
 5 | 
 6 | class ETFireWallBlockIps:
 7 | 
 8 |     def __init__(self):
 9 |         self.intel = []
10 |         self._retrieved = None
11 |         self._feed_url = "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt"
12 | 
13 |     def run(self):
14 |         self._download()
15 |         self._parse()
16 | 
17 |     def _download(self):
18 |         self._retrieved = time()
19 |         response = requests.get(self._feed_url)
20 |         if response.status_code is 200:
21 |             self._raw_threat_intel = response.text
22 | 
23 |     def _parse(self):
24 |         for line in self._raw_threat_intel.split("\n"):
25 |             if line[:1] is "#" or len(line) < 2:
26 |                 pass
27 |             else:
28 |                 # Add as source ip
29 |                 try:
30 |                     if "/" in line:
31 |                         type = "ip_range"
32 |                     else:
33 |                         type = "ip_address"
34 | 
35 |                     intel = Intel(
36 |                         original=line,
37 |                         event_type="indicator",
38 |                         event_reference=self._feed_url,
39 |                         event_provider="EmergingThreats",
40 |                         event_dataset="fwrules/emerging-Block-IPs",
41 |                         threat_first_seen=None,
42 |                         threat_last_seen=None,
43 |                         threat_type=type
44 |                     )
45 |                     intel.add_ip(ip=line)
46 |                 except Exception:
47 |                     pass
48 |                 else:
49 |                     intel.add_docid()
50 |                     self.intel.append(intel)


--------------------------------------------------------------------------------
/tip/abuseipdb.py:
--------------------------------------------------------------------------------
 1 | from ioc import Intel
 2 | import requests
 3 | import json
 4 | from time import time
 5 | from os import environ
 6 | 
 7 | 
 8 | class AbuseIPDB:
 9 | 
10 |     def __init__(self):
11 |         self.intel = []
12 |         self._retrieved = None
13 |         self._feed_url = "https://api.abuseipdb.com/api/v2/blacklist"
14 |         self.confidenceminimum = '90'
15 |         self.key = None
16 |         self._raw_threat_intel = {
17 |             "data": []
18 |         }
19 | 
20 |     def run(self):
21 |         if not self.key:
22 |             try:
23 |                 self.key = environ["ABUSE_IP_KEY"]
24 |             except KeyError:
25 |                 self.key = input("AbuseIP DB API Key: ")
26 |         self._download()
27 |         self._parse()
28 | 
29 |     def _download(self):
30 |         self._retrieved = time()
31 | 
32 |         querystring = {
33 |             "confidenceMinimum": self.confidenceminimum
34 |         }
35 |         headers = {
36 |             'Accept': 'application/json',
37 |             'Key': self.key
38 |         }
39 |         response = requests.get(
40 |             url=self._feed_url,
41 |             headers=headers,
42 |             params=querystring
43 |         )
44 |         if response.status_code is 200:
45 |             self._raw_threat_intel = json.loads(response.text)
46 |         elif response.status_code is 429:
47 |             print("Rate limit exceeded for abuseipdb")
48 | 
49 |     def _parse(self):
50 |         for obj in self._raw_threat_intel["data"]:
51 |             # Add as source ip
52 |             try:
53 |                 intel = Intel(
54 |                     original=json.dumps(obj),
55 |                     event_type="indicator",
56 |                     event_reference=self._feed_url,
57 |                     event_provider="AbuseIPdb",
58 |                     event_dataset="blacklist",
59 |                     threat_first_seen=None,
60 |                     threat_last_seen=obj["lastReportedAt"],
61 |                     threat_type="ip_address"
62 |                 )
63 |                 intel.add_ip(ip=obj["ipAddress"])
64 |             except Exception:
65 |                 pass
66 |             else:
67 |                 intel.add_docid()
68 |                 self.intel.append(intel)
69 | 


--------------------------------------------------------------------------------
/tip/eset.py:
--------------------------------------------------------------------------------
 1 | from os import walk, path, mkdir
 2 | from shutil import rmtree
 3 | from ioc import Intel
 4 | from time import time
 5 | from git import Git
 6 | 
 7 | 
 8 | class EsetMalwareIOC:
 9 | 
10 |     def __init__(self):
11 |         self.intel = []
12 |         self._retrieved = None
13 |         self._feed_url = "https://github.com/eset/malware-ioc.git"
14 | 
15 |     def run(self):
16 |         self._download()
17 |         try:
18 |             self._parse()
19 |         except Exception as err:
20 |             print("Failed to parse ESET IOCS: {}".format(err))
21 |         finally:
22 |             self._cleanup()
23 |         print(len(self.intel))
24 | 
25 |     def _download(self):
26 |         self._retrieved = time()
27 |         # Make sure the dir exists
28 |         if not path.exists("tip/githubclones"):
29 |             mkdir("tip/githubclones")
30 |         # Clone the git repo
31 |         Git("tip/githubclones/eset").clone(self._feed_url)
32 | 
33 |     def _parse(self):
34 |         for root, dirs, files in walk("tip/githubclones/eset/malware-ioc"):
35 |             for file in files:
36 |                 if ".git" in root:
37 |                     continue
38 |                 elif "README" in file:
39 |                     continue
40 |                 elif "samples" in file:
41 |                     lines = ""
42 |                     with open("{}/{}".format(root, file), "r") as iocfile:
43 |                         lines = iocfile.read().split("\n")
44 | 
45 |                     for line in lines:
46 |                         try:
47 |                             intel = Intel(
48 |                                 original=line,
49 |                                 event_type="indicator",
50 |                                 event_reference=self._feed_url,
51 |                                 event_provider="Eset",
52 |                                 event_dataset="malware-ioc",
53 |                                 threat_first_seen=None,
54 |                                 threat_last_seen=None,
55 |                                 threat_type="file_hash"
56 |                             )
57 |                             if file == "samples.sha1":
58 |                                 intel.add_file(sha1=line)
59 |                             elif file == "samples.sha256":
60 |                                 intel.add_file(sha256=line)
61 |                             elif file == "samples.md5":
62 |                                 intel.add_file(md5=line)
63 |                         except Exception as err:
64 |                             print(err)
65 |                         else:
66 |                             intel.add_docid()
67 |                             self.intel.append(intel)
68 | 
69 |     def _cleanup(self):
70 |         rmtree("tip/githubclones/eset/malware-ioc")


--------------------------------------------------------------------------------
/tip/test_ioc.py:
--------------------------------------------------------------------------------
 1 | import unittest
 2 | from ioc import Intel
 3 | 
 4 | 
 5 | class MyTestCase(unittest.TestCase):
 6 |     def test_something(self):
 7 |         self.assertEqual(True, True)
 8 | 
 9 |     def test_add_ip(self):
10 |         intel = Intel()
11 |         intel.add_ip(ip="1.1.1.1")
12 |         self.assertEqual(intel.intel["threat"]["indicator"]["ip"], "1.1.1.1")
13 |         intel = Intel()
14 |         intel.add_ip(ip="1.1.1.1", port=443)
15 |         self.assertEqual(intel.intel["threat"]["indicator"]["ip"], "1.1.1.1")
16 |         self.assertEqual(intel.intel["threat"]["indicator"]["port"], 443)
17 | 
18 |     def test_add_ip(self):
19 |         intel = Intel()
20 |         intel.add_ip(ip="1.1.1.1")
21 |         self.assertEqual(intel.intel["threat"]["indicator"]["ip"], "1.1.1.1")
22 |         intel = Intel()
23 |         intel.add_ip(ip="1.1.1.1", port=443)
24 |         self.assertEqual(intel.intel["threat"]["indicator"]["ip"], "1.1.1.1")
25 |         self.assertEqual(intel.intel["threat"]["indicator"]["port"], 443)
26 | 
27 |     def test_add_malware(self):
28 |         intel = Intel()
29 |         intel.add_malware(name="Rake")
30 |         self.assertEqual(intel.intel["threat"]["malware"]["name"], "Rake")
31 |         intel = Intel()
32 |         intel.add_malware("Rake")
33 |         self.assertEqual(intel.intel["threat"]["malware"]["name"], "Rake")
34 |         intel = Intel()
35 |         intel.add_malware(name="Rake", family="Rake", malware_type="C&C")
36 |         self.assertEqual(intel.intel["threat"]["malware"]["name"], "Rake")
37 |         self.assertEqual(intel.intel["threat"]["malware"]["family"], "Rake")
38 |         self.assertEqual(intel.intel["threat"]["malware"]["type"], "C&C")
39 | 
40 |     def test_add_file(self):
41 |         intel = Intel()
42 |         intel.add_file(name="example.exe")
43 |         self.assertEqual(intel.intel["file"]["name"], "example.exe")
44 |         intel = Intel()
45 |         intel.add_file(name="example.exe", sha1="04ea0d99e724bae38f63b34955a669a13da65485",
46 |                        sha256="4d6feee47b15e24f526f8d9053b04a6ff5cefef4f9df71b8dffede2de31fcc57")
47 |         self.assertEqual(intel.intel["file"]["name"], "example.exe")
48 |         self.assertEqual(intel.intel["file"]["hash"]["sha1"], "04ea0d99e724bae38f63b34955a669a13da65485")
49 |         self.assertEqual(intel.intel["file"]["hash"]["sha256"], "4d6feee47b15e24f526f8d9053b04a6ff5cefef4f9df71b8dffede2de31fcc57")
50 |         intel = Intel()
51 |         intel.add_file(name="example.exe", sha1="04ea0d99e724bae38f63b34955a669a13da65485",
52 |                        sha256="4d6feee47b15e24f526f8d9053b04a6ff5cefef4f9df71b8dffede2de31fcc57",
53 |                        drive_letter="C")
54 |         self.assertEqual(intel.intel["file"]["name"], "example.exe")
55 |         self.assertEqual(intel.intel["file"]["drive_letter"], "C")
56 |         self.assertEqual(intel.intel["file"]["hash"]["sha1"], "04ea0d99e724bae38f63b34955a669a13da65485")
57 |         self.assertEqual(intel.intel["file"]["hash"]["sha256"],
58 |                          "4d6feee47b15e24f526f8d9053b04a6ff5cefef4f9df71b8dffede2de31fcc57")
59 | 
60 |     def test_add_url(self):
61 |         intel = Intel()
62 |         intel.add_url(original="https://test.domain.com:9500/")
63 |         self.assertEqual(intel.intel["url"]["original"], "https://test.domain.com:9500/")
64 |         self.assertEqual(intel.intel["url"]["scheme"], "https")
65 |         intel.add_url(full="https://test.domain.com:9500/")
66 |         self.assertEqual(intel.intel["url"]["original"], "https://test.domain.com:9500/")
67 |         self.assertEqual(intel.intel["url"]["full"], "https://test.domain.com:9500/")
68 |         self.assertEqual(intel.intel["url"]["scheme"], "https")
69 | 
70 |     def test_add_tls(self):
71 |         intel = Intel()
72 |         intel.add_tls(s_sha1="8964f9caf2c4e688a395f4666db072b165f9c28e")
73 |         self.assertEqual(intel.intel["tls"]["server"]["hash"]["sha1"], "8964f9caf2c4e688a395f4666db072b165f9c28e")
74 | 
75 | 
76 | if __name__ == '__main__':
77 |     unittest.main()
78 | 


--------------------------------------------------------------------------------
/tip/spamhaus.py:
--------------------------------------------------------------------------------
  1 | from ioc import Intel
  2 | import requests
  3 | from time import time
  4 | 
  5 | 
  6 | class SpamhausDrop:
  7 | 
  8 |     def __init__(self):
  9 |         self._raw_threat_intel = None
 10 |         self.intel = []
 11 |         self._retrieved = None
 12 |         self._feed_url = "https://www.spamhaus.org/drop/drop.txt"
 13 | 
 14 |     def run(self):
 15 |         self._download()
 16 |         self._parse()
 17 | 
 18 |     def _download(self):
 19 |         self._retrieved = time()
 20 |         response = requests.get(self._feed_url)
 21 |         if response.status_code is 200:
 22 |             self._raw_threat_intel = response.text
 23 | 
 24 |     def _parse(self):
 25 |         for line in self._raw_threat_intel.split("\n"):
 26 |             if line[:1] is ";":
 27 |                 pass
 28 |             else:
 29 |                 split_line = line.split(';')
 30 |                 # Add as source ip
 31 |                 try:
 32 |                     intel = Intel(
 33 |                         original=line,
 34 |                         event_type="indicator",
 35 |                         event_reference=self._feed_url,
 36 |                         event_provider="Spamhaus",
 37 |                         event_dataset="Spamhaus.drop",
 38 |                         threat_first_seen=None,
 39 |                         threat_last_seen=None,
 40 |                         threat_type="domain",
 41 |                         threat_description=split_line[1]
 42 |                     )
 43 |                     intel.add_ip(ip=split_line[0])
 44 |                 except IndexError:
 45 |                     pass
 46 |                 else:
 47 |                     intel.add_docid()
 48 |                     self.intel.append(intel)
 49 | 
 50 | 
 51 | class SpamhausExtendedDrop:
 52 | 
 53 |     def __init__(self):
 54 |         self._raw_threat_intel = None
 55 |         self.intel = []
 56 |         self._retrieved = None
 57 |         self._feed_url = "https://www.spamhaus.org/drop/edrop.txt"
 58 | 
 59 |     def run(self):
 60 |         self._download()
 61 |         self._parse()
 62 | 
 63 |     def _download(self):
 64 |         self._retrieved = time()
 65 |         response = requests.get(self._feed_url)
 66 |         if response.status_code is 200:
 67 |             self._raw_threat_intel = response.text
 68 | 
 69 |     def _parse(self):
 70 |         for line in self._raw_threat_intel.split("\n"):
 71 |             if line[:1] is ";":
 72 |                 pass
 73 |             else:
 74 |                 split_line = line.split(';')
 75 |                 # Add as source ip
 76 |                 try:
 77 |                     intel = Intel(
 78 |                         original=line,
 79 |                         event_type="indicator",
 80 |                         event_reference=self._feed_url,
 81 |                         event_provider="Spamhaus",
 82 |                         event_dataset="Spamhaus.extendeddrop",
 83 |                         threat_first_seen=None,
 84 |                         threat_last_seen=None,
 85 |                         threat_type="domain",
 86 |                         threat_description=split_line[1]
 87 |                     )
 88 |                     intel.add_ip(ip=split_line[0])
 89 |                 except IndexError:
 90 |                     pass
 91 |                 else:
 92 |                     intel.add_docid()
 93 |                     self.intel.append(intel)
 94 | 
 95 | 
 96 | class SpamhausDropIpv6:
 97 | 
 98 |     def __init__(self):
 99 |         self._raw_threat_intel = None
100 |         self.intel = []
101 |         self._retrieved = None
102 |         self._feed_url = "https://www.spamhaus.org/drop/dropv6.txt"
103 | 
104 |     def run(self):
105 |         self._download()
106 |         self._parse()
107 | 
108 |     def _download(self):
109 |         self._retrieved = time()
110 |         response = requests.get(self._feed_url)
111 |         if response.status_code is 200:
112 |             self._raw_threat_intel = response.text
113 | 
114 |     def _parse(self):
115 |         for line in self._raw_threat_intel.split("\n"):
116 |             if line[:1] is ";":
117 |                 pass
118 |             else:
119 |                 split_line = line.split(';')
120 |                 # Add as source ip
121 |                 try:
122 |                     intel = Intel(
123 |                         original=line,
124 |                         event_type="indicator",
125 |                         event_reference=self._feed_url,
126 |                         event_provider="Spamhaus",
127 |                         event_dataset="Spamhaus.ipv6drop",
128 |                         threat_first_seen=None,
129 |                         threat_last_seen=None,
130 |                         threat_type="domain",
131 |                         threat_description=split_line[1]
132 |                     )
133 |                     intel.add_ip(ip=split_line[0])
134 |                     intel.intel["threat"]["type"] = "IPV6"
135 |                 except IndexError:
136 |                     pass
137 |                 else:
138 |                     intel.add_docid()
139 |                     self.intel.append(intel)
140 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # Elastic Threat Intelligence Platform (elastic-tip)
  2 | Elastic TIP is a python tool which automates the process of aggregating Threat Intelligence and ingesting
  3 | the intelligence into a common format into Elasticsearch with the main goal of being used by the Security
  4 | solution.
  5 | 
  6 | The intelligence ingested by elastic-tip is meant to be used by the [threat matching](https://github.com/elastic/kibana/pull/78955)
  7 | detection rule type.
  8 | 
  9 | ## Recommended usage
 10 | I build this tool to work together with the "threat matching" detection capability of the Elastic stack.
 11 | For this the recommended usage is to periodically run the tool with the modules you want and setup a threat matching
 12 | rule to match against the `elatic-tip` index.
 13 | 
 14 | ## Requirements
 15 | This tool has been tested on Python3.6, Python3.8
 16 | 
 17 | Make sure to install the dependencies by creating a virtual environment and pip installing them.
 18 | ```bash
 19 | python3 -m venv venv
 20 | source venv/bin/activate
 21 | pip install -r requirements.txt
 22 | ```
 23 | 
 24 | ## CLI
 25 | ```bash
 26 | python3 tip/elastic_tip_cli.py help
 27 | 
 28 | Elastic Threat Intelligence Platform
 29 |                             ----------------------
 30 |                                  community project
 31 | ==================================================
 32 | 
 33 | python tip/elastic_tip_cli.py [command] [options]
 34 | 
 35 | Commands:
 36 |     help           Print this help output
 37 |     run            Run the platform and ingest IOC's into ElasticSearch
 38 |     init           Initilize for the first time and load the full IOC lists into ElasticSearch
 39 |     verify         Verify the ElasticSearch index and connection
 40 | 
 41 | ==================================================
 42 | Author   Stijn Holzhauer
 43 | Website  https://github.com/SHolzhauer/elastic-tip
 44 | ```
 45 | 
 46 | #### Run
 47 | The run command can be used to run elastic-tip to gather and ingest threat intelligence into Elasticsearch index
 48 | 
 49 | ```bash
 50 | python tip/elastic_tip_cli.py run -e localhost -m all --tls
 51 | ```
 52 | 
 53 | **Help**
 54 | ```bash
 55 | Elastic Threat Intelligence Platform
 56 |                             ----------------------
 57 |                                  community project
 58 | ==================================================
 59 | 
 60 | python tip/elastic_tip_cli.py run [options]
 61 | 
 62 |     The run command is used to run the Elastic Threat Intelligence Platform and load
 63 |     the Threat Intelligence, in the form of Indicators Of Compromise (IOC) into
 64 |     your ElasticSearch cluster to be used by the build in Detection-Engine
 65 | 
 66 | Options
 67 |     -h, --help                Print help output
 68 |     -e, --es-hosts <value>    Comma seperated list of Elasticsearch hosts to use
 69 |     -u, --user <value>        Username to use for Authentication to ES
 70 |     -p, --passwd <value>      Password to use for Authentication to ES
 71 |     --modules-list            List module names and the reference link
 72 |     -m, --modules <values>    Modules to enable (* for all):
 73 |                                   URLhaus
 74 |                                   MalwareBazaar
 75 |                                   FeodoTracker
 76 |                                   SSLBlacklist
 77 |                                   EmergingThreats-Blocklist
 78 |                                   ESET-MalwareIOC
 79 |     -T, --tls                 Use TLS/SSL when connecting to Elasticsearch
 80 |     -c, --ca-cert <value>     Use the cert specified by path
 81 |     --no-verify               Don't verify the TLS/SSL certificate
 82 | 
 83 | ==================================================
 84 | Author   Stijn Holzhauer
 85 | Website  https://github.com/SHolzhauer/elastic-tip
 86 | ```
 87 | 
 88 | ## Feeds
 89 | Elastic-TIP supports multiple threat intelligence feeds, it currently supports:
 90 | 
 91 | _for the exact feed url's look at the `event.reference` field_
 92 | 
 93 | | Module name | name | url | note |
 94 | |-------------|------|-----|------|
 95 | | URLhaus | Abuse.ch URLhaus | https://urlhaus.abuse.ch/ | |
 96 | | MalwareBazaar | Abuse.ch MalwareBazaar | https://bazaar.abuse.ch/ | |
 97 | | FeodoTracker | Abuse.ch FeodoTracker | https://feodotracker.abuse.ch/ | |
 98 | | SSLBlacklist | Abuse.ch SSLBlacklist | https://sslbl.abuse.ch/ | |
 99 | | EmergingThreats-Blocklist | Emerging Threats | https://rules.emergingthreats.net/ | This is just the firewall blocklist |
100 | | ESET-MalwareIOC | ESET malwareIOC repo | https://github.com/eset/malware-ioc | |
101 | | AbuseIPdb | AbuseIPdb | https://www.abuseipdb.com/ | Only 10.000 results, API key is required. |
102 | | Spamhaus-Drop | Spamhaus droplist | https://www.spamhaus.org/drop/ | |
103 | | Spamhaus-ExtendedDrop | Spamhaus extended droplist | https://www.spamhaus.org/drop/ | |
104 | | Spamhaus-IPv6Drop | Spamhaus IPv6 droplist | https://www.spamhaus.org/drop/ | |
105 | | Botvrij-filenames | Botvrij files | https://botvrij.eu/data/ioclist.filename.raw | |
106 | | Botvrij-domains | Botvrij Domain names | https://botvrij.eu/data/ioclist.domain.raw | |
107 | | Botvrij-destinations | Botvrij destinations | https://botvrij.eu/data/ioclist.ip-dst.raw | |
108 | | Spamhaus-urls | Botvrij url list | https://botvrij.eu/data/ioclist.url.raw | |
109 | 


--------------------------------------------------------------------------------
/tip/botvrij.py:
--------------------------------------------------------------------------------
  1 | from ioc import Intel
  2 | import requests
  3 | import json
  4 | from time import time
  5 | from os import environ
  6 | 
  7 | 
  8 | class BotvrijFileNames:
  9 | 
 10 |     def __init__(self):
 11 |         self.intel = []
 12 |         self._retrieved = None
 13 |         self._feed_url = "https://botvrij.eu/data/ioclist.filename.raw"
 14 |         self.key = None
 15 |         self._raw_threat_intel = ""
 16 | 
 17 |     def run(self):
 18 |         self._download()
 19 |         self._parse()
 20 | 
 21 |     def _download(self):
 22 |         self._retrieved = time()
 23 | 
 24 |         response = requests.get(
 25 |             url=self._feed_url
 26 |         )
 27 |         if response.status_code is 200:
 28 |             self._raw_threat_intel = response.text
 29 | 
 30 |     def _parse(self):
 31 |         for line in self._raw_threat_intel.split("\n"):
 32 |             # Add as source ip
 33 |             try:
 34 |                 intel = Intel(
 35 |                     original=line,
 36 |                     event_type="indicator",
 37 |                     event_reference=self._feed_url,
 38 |                     event_provider="botvrij",
 39 |                     event_dataset="botvrij.filenames",
 40 |                     threat_first_seen=None,
 41 |                     threat_last_seen=None,
 42 |                     threat_type="file"
 43 |                 )
 44 |                 intel.add_file(name=line)
 45 |             except Exception:
 46 |                 pass
 47 |             else:
 48 |                 intel.add_docid()
 49 |                 self.intel.append(intel)
 50 | 
 51 | 
 52 | class BotvrijDomains:
 53 | 
 54 |     def __init__(self):
 55 |         self.intel = []
 56 |         self._retrieved = None
 57 |         self._feed_url = "https://botvrij.eu/data/ioclist.domain.raw"
 58 |         self.key = None
 59 |         self._raw_threat_intel = ""
 60 | 
 61 |     def run(self):
 62 |         self._download()
 63 |         self._parse()
 64 | 
 65 |     def _download(self):
 66 |         self._retrieved = time()
 67 | 
 68 |         response = requests.get(
 69 |             url=self._feed_url
 70 |         )
 71 |         if response.status_code is 200:
 72 |             self._raw_threat_intel = response.text
 73 | 
 74 |     def _parse(self):
 75 |         for line in self._raw_threat_intel.split("\n"):
 76 |             # Add as source ip
 77 |             try:
 78 |                 intel = Intel(
 79 |                     original=line,
 80 |                     event_type="indicator",
 81 |                     event_reference=self._feed_url,
 82 |                     event_provider="botvrij",
 83 |                     event_dataset="botvrij.domains",
 84 |                     threat_first_seen=None,
 85 |                     threat_last_seen=None,
 86 |                     threat_type="url"
 87 |                 )
 88 |                 intel.add_url(domain=line, top_level_domain=line.split(".")[1])
 89 |             except Exception:
 90 |                 pass
 91 |             else:
 92 |                 intel.add_docid()
 93 |                 self.intel.append(intel)
 94 | 
 95 | 
 96 | class BotvrijDstIP:
 97 | 
 98 |     def __init__(self):
 99 |         self.intel = []
100 |         self._retrieved = None
101 |         self._feed_url = "https://botvrij.eu/data/ioclist.ip-dst.raw"
102 |         self.key = None
103 |         self._raw_threat_intel = ""
104 | 
105 |     def run(self):
106 |         self._download()
107 |         self._parse()
108 | 
109 |     def _download(self):
110 |         self._retrieved = time()
111 | 
112 |         response = requests.get(
113 |             url=self._feed_url
114 |         )
115 |         if response.status_code is 200:
116 |             self._raw_threat_intel = response.text
117 | 
118 |     def _parse(self):
119 |         for line in self._raw_threat_intel.split("\n"):
120 |             # Add as source ip
121 |             try:
122 |                 intel = Intel(
123 |                     original=line,
124 |                     event_type="indicator",
125 |                     event_reference=self._feed_url,
126 |                     event_provider="botvrij",
127 |                     event_dataset="botvrij.ip-dst",
128 |                     threat_first_seen=None,
129 |                     threat_last_seen=None,
130 |                     threat_type="IPV4"
131 |                 )
132 |                 intel.add_ip(ip=line)
133 |             except Exception:
134 |                 pass
135 |             else:
136 |                 intel.add_docid()
137 |                 self.intel.append(intel)
138 | 
139 | 
140 | class BotvrijUrl:
141 | 
142 |     def __init__(self):
143 |         self.intel = []
144 |         self._retrieved = None
145 |         self._feed_url = "https://botvrij.eu/data/ioclist.url.raw"
146 |         self.key = None
147 |         self._raw_threat_intel = ""
148 | 
149 |     def run(self):
150 |         self._download()
151 |         self._parse()
152 | 
153 |     def _download(self):
154 |         self._retrieved = time()
155 | 
156 |         response = requests.get(
157 |             url=self._feed_url
158 |         )
159 |         if response.status_code is 200:
160 |             self._raw_threat_intel = response.text
161 | 
162 |     def _parse(self):
163 |         for line in self._raw_threat_intel.split("\n"):
164 |             # Add as source ip
165 |             try:
166 |                 intel = Intel(
167 |                     original=line,
168 |                     event_type="indicator",
169 |                     event_reference=self._feed_url,
170 |                     event_provider="botvrij",
171 |                     event_dataset="botvrij.url",
172 |                     threat_first_seen=None,
173 |                     threat_last_seen=None,
174 |                     threat_type="url"
175 |                 )
176 |                 intel.add_url(original=line)
177 |             except Exception:
178 |                 pass
179 |             else:
180 |                 intel.add_docid()
181 |                 self.intel.append(intel)
182 | 


--------------------------------------------------------------------------------
/tip/abuse_bazaar.py:
--------------------------------------------------------------------------------
  1 | from ioc import Intel
  2 | import requests
  3 | import hashlib
  4 | from time import time
  5 | 
  6 | 
  7 | class URLhaus:
  8 | 
  9 |     def __init__(self):
 10 |         self._raw_threat_intel = None
 11 |         self.intel = []
 12 |         self._retrieved = None
 13 |         self._feed_url = "https://urlhaus.abuse.ch/downloads/csv_recent/"
 14 | 
 15 |     def run(self):
 16 |         self._download()
 17 |         self._parse()
 18 | 
 19 |     def _download(self):
 20 |         self._retrieved = time()
 21 |         response = requests.get(self._feed_url)
 22 |         if response.status_code is 200:
 23 |             self._raw_threat_intel = response.text
 24 | 
 25 |     def _parse(self):
 26 |         for line in self._raw_threat_intel.split("\n"):
 27 |             if line[:1] is "#":
 28 |                 pass
 29 |             else:
 30 |                 split_line = line.split('","')
 31 |                 try:
 32 |                     intel = Intel(
 33 |                         original=line,
 34 |                         event_type="indicator",
 35 |                         event_reference=self._feed_url,
 36 |                         event_provider="Abuse.ch",
 37 |                         event_dataset="URLhaus",
 38 |                         threat_first_seen=split_line[1],
 39 |                         threat_last_seen=None,
 40 |                         threat_type="domain",
 41 |                         threat_description=split_line[4]
 42 |                     )
 43 |                     intel.add_url(original=split_line[2])
 44 |                 except IndexError:
 45 |                     pass
 46 |                 else:
 47 |                     intel.add_docid()
 48 |                     self.intel.append(intel)
 49 | 
 50 | 
 51 | class MalwareBazaar:
 52 | 
 53 |     def __init__(self):
 54 |         self._raw_threat_intel = None
 55 |         self.intel = []
 56 |         self._retrieved = None
 57 |         self._feed_url = "https://bazaar.abuse.ch/export/csv/recent/"
 58 | 
 59 |     def run(self):
 60 |         self._download()
 61 |         self._parse()
 62 | 
 63 |     def _download(self):
 64 |         self._retrieved = time()
 65 |         response = requests.get(self._feed_url)
 66 |         if response.status_code is 200:
 67 |             self._raw_threat_intel = response.text
 68 | 
 69 |     def _parse(self):
 70 |         for line in self._raw_threat_intel.split("\n"):
 71 |             if line[:1] is "#":
 72 |                 pass
 73 |             else:
 74 |                 try:
 75 |                     split_line = line.split('", "')
 76 |                     intel = Intel(
 77 |                         original=line,
 78 |                         event_type="indicator",
 79 |                         event_reference=self._feed_url,
 80 |                         event_provider="Abuse.ch",
 81 |                         event_dataset="MalwareBazaar",
 82 |                         threat_first_seen=split_line[0],
 83 |                         threat_last_seen=None,
 84 |                         threat_type="file_hash"
 85 |                     )
 86 |                     intel.add_file(name=split_line[5], extension=split_line[6], mime_type=split_line[7],
 87 |                                    sha1=split_line[3], sha256=split_line[1], md5=split_line[2])
 88 |                     intel.add_malware(split_line[8])
 89 |                 except Exception as err:
 90 |                     print(err)
 91 |                 else:
 92 |                     intel.add_docid()
 93 |                     self.intel.append(intel)
 94 | 
 95 | 
 96 | class FeodoTracker:
 97 | 
 98 |     def __init__(self):
 99 |         self._raw_threat_intel = None
100 |         self.intel = []
101 |         self._retrieved = None
102 |         self._feed_url = "https://feodotracker.abuse.ch/downloads/ipblocklist.csv"
103 | 
104 |     def run(self):
105 |         self._download()
106 |         self._parse()
107 | 
108 |     def _download(self):
109 |         self._retrieved = time()
110 |         response = requests.get(self._feed_url)
111 |         if response.status_code is 200:
112 |             self._raw_threat_intel = response.text
113 | 
114 |     def _parse(self):
115 |         for line in self._raw_threat_intel.split("\n"):
116 |             if line[:1] is "#":
117 |                 pass
118 |             else:
119 |                 split_line = line.split(",")
120 |                 # add as destination ip
121 |                 try:
122 |                     intel = Intel(
123 |                         original=line,
124 |                         event_type="indicator",
125 |                         event_reference=self._feed_url,
126 |                         event_provider="Abuse.ch",
127 |                         event_dataset="FeodoTracker",
128 |                         threat_first_seen=split_line[0],
129 |                         threat_last_seen=split_line[3],
130 |                         threat_type="ip_address",
131 |                         threat_description=split_line[4]
132 |                     )
133 |                     intel.add_ip(ip=split_line[1], port=split_line[2])
134 |                     intel.add_malware(name=split_line[4])
135 |                 except IndexError as err:
136 |                     pass
137 |                 else:
138 |                     intel.add_docid()
139 |                     self.intel.append(intel)
140 | 
141 | 
142 | class SSLBlacklist:
143 | 
144 |     def __init__(self):
145 |         self._raw_threat_intel = None
146 |         self.intel = []
147 |         self._retrieved = None
148 |         self._feed_url = "https://sslbl.abuse.ch/blacklist/sslblacklist.csv"
149 | 
150 |     def run(self):
151 |         self._download()
152 |         self._parse()
153 | 
154 |     def _download(self):
155 |         self._retrieved = time()
156 |         response = requests.get(self._feed_url)
157 |         if response.status_code is 200:
158 |             self._raw_threat_intel = response.text
159 | 
160 |     def _parse(self):
161 |         for line in self._raw_threat_intel.split("\n"):
162 |             if line[:1] is "#":
163 |                 pass
164 |             else:
165 |                 split_line = line.split(",")
166 |                 try:
167 |                     intel = Intel(
168 |                         original=line,
169 |                         event_type="indicator",
170 |                         event_reference=self._feed_url,
171 |                         event_provider="Abuse.ch",
172 |                         event_dataset="SSLBlackList",
173 |                         threat_first_seen=split_line[0],
174 |                         threat_last_seen=None,
175 |                         threat_type="ssl_hash",
176 |                         threat_description=split_line[2]
177 |                     )
178 |                     intel.add_tls(s_sha1=split_line[1])
179 |                     if "C&C" in intel.intel["threat"]["indicator"]["description"]:
180 |                         intel.add_mitre("TA0011")
181 |                     elif "" in intel.intel["threat"]["indicator"]["description"]:
182 |                         intel.add_mitre("TA0042", "T1588.001")
183 |                 except IndexError as err:
184 |                     pass
185 |                 else:
186 |                     intel.add_docid()
187 |                     self.intel.append(intel)


--------------------------------------------------------------------------------
/tip/elastic_tip_cli.py:
--------------------------------------------------------------------------------
  1 | from sys import argv
  2 | import getopt
  3 | from elastic_tip import ElasticTip
  4 | 
  5 | 
  6 | class CLI:
  7 | 
  8 |     def __init__(self):
  9 |         self._arguments = []
 10 |         self._cli_head = """
 11 | Elastic Threat Intelligence Platform
 12 |                             ----------------------
 13 |                                  community project
 14 | ==================================================
 15 | """
 16 |         self._cli_footer = """
 17 | ==================================================
 18 | Author   Stijn Holzhauer
 19 | Website  https://github.com/SHolzhauer/elastic-tip"""
 20 |         self._tip = None
 21 |         self._mod = None
 22 | 
 23 |     def cli(self):
 24 |         if argv[1] == "help":
 25 |             self._help()
 26 |         elif argv[1] == "run":
 27 |             self._run_cli()
 28 |         elif argv[1] == "init":
 29 |             pass
 30 |         elif argv[1] == "verify":
 31 |             self._verify_cli()
 32 |         else:
 33 |             self._help()
 34 | 
 35 |     def _run_cli(self):
 36 |         if len(argv) < 3:
 37 |             self._run_help()
 38 |             exit()
 39 | 
 40 |         try:
 41 |             opts, args = getopt.getopt(argv[2:], "hm:e:Tu:p:P:i:c:",
 42 |                                        ["help", "modules=", "modules-list", "es-hosts=", "es-port=", "tls", "user=", "passwd=", "index=", "ca-cert=", "no-verify"])
 43 |         except getopt.GetoptError as err:
 44 |             print(err)
 45 |             exit(1)
 46 |         else:
 47 |             self._tip = ElasticTip()
 48 | 
 49 |         for opt, arg in opts:
 50 |             if opt in ["-h", "--help"]:
 51 |                 self._run_help()
 52 |                 exit()
 53 |             elif opt in ["--modules-list"]:
 54 |                 print(self._cli_head)
 55 |                 print("IOC Modules:")
 56 |                 for mod in self._tip.modules:
 57 |                     spaces = " "
 58 |                     for i in range(0, (30 - len(mod))):
 59 |                         spaces += " "
 60 |                     print("  {}{}{}".format(mod, spaces, self._tip.modules[mod]["ref"]))
 61 |                     if self._tip.modules[mod]["note"]:
 62 |                         print("          {}".format(self._tip.modules[mod]["note"]))
 63 |                 exit()
 64 |                 print(self._cli_footer)
 65 |             elif opt in ["-m", "--modules"]:
 66 |                 if arg in "all":
 67 |                     for mod in self._tip.modules:
 68 |                         self._tip.modules[mod]["enabled"] = True
 69 |                 else:
 70 |                     for mod in arg.split(","):
 71 |                         try:
 72 |                             # Enable the module
 73 |                             self._tip.modules["{}".format(mod)]["enabled"] = True
 74 |                         except KeyError:
 75 |                             print("Module {} does not exist".format(mod))
 76 |             elif opt in ["-e", "--es-hosts"]:
 77 |                 hosts = arg.split(",")
 78 |                 for host in hosts:
 79 |                     if "://" in host:
 80 |                         parsedhost = host.split("://")[1]
 81 |                     else:
 82 |                         parsedhost = host
 83 |                     self._tip.eshosts.append(parsedhost)
 84 |             elif opt in ["-P", "--es-port"]:
 85 |                 self._tip.esport = int(float(arg))
 86 |             elif opt in ["-u", "--user"]:
 87 |                 self._tip.esuser = arg
 88 |             elif opt in ["-p", "--passwd"]:
 89 |                 self._tip.espass = arg
 90 |             elif opt in ["-i", "--index"]:
 91 |                 self._tip.index = arg
 92 |             elif opt in ["-T", "--tls"]:
 93 |                 self._tip.tls["use"] = False
 94 |             elif opt in ["-c", "--ca-cert"]:
 95 |                 self._tip.tls["cacert"] = arg
 96 |             elif opt in ["--no-verify"]:
 97 |                 self._tip.tls["verify"] = False
 98 |             elif opt in ["--no-setup"]:
 99 |                 self._tip.setup_index = False
100 | 
101 |         self._tip.run()
102 | 
103 |     def _init_cli(self):
104 |         pass
105 | 
106 |     def _verify_cli(self):
107 |         if len(argv) < 3:
108 |             self._verify_help()
109 |             exit()
110 | 
111 |         try:
112 |             opts, args = getopt.getopt(argv[2:], "he:Tu:P:p:i:c:",
113 |                                        ["help", "es-hosts=", "es-port=" "tls", "user=", "passwd=", "index=", "ca-cert=", "no-verify"])
114 |         except getopt.GetoptError as err:
115 |             print(err)
116 |             exit(1)
117 |         else:
118 |             self._tip = ElasticTip()
119 |             for opt, arg in opts:
120 |                 if opt in ["-h", "--help"]:
121 |                     self._verify_help()
122 |                     exit()
123 |                 elif opt in ["-e", "--es-hosts"]:
124 |                     hosts = arg.split(",")
125 |                     for host in hosts:
126 |                         if "://" in host:
127 |                             parsedhost = host.split("://")[1]
128 |                         else:
129 |                             parsedhost = host
130 |                         self._tip.eshosts.append(parsedhost)
131 |                 elif opt in ["-P", "--es-port"]:
132 |                     self._tip.esport = int(float(arg))
133 |                 elif opt in ["-u", "--user"]:
134 |                     self._tip.esuser = arg
135 |                 elif opt in ["-p", "--passwd"]:
136 |                     self._tip.espass = arg
137 |                 elif opt in ["-i", "--index"]:
138 |                     self._tip.index = arg
139 |                 elif opt in ["-T", "--tls"]:
140 |                     self._tip.tls["use"] = False
141 |                 elif opt in ["-c", "--ca-cert"]:
142 |                     self._tip.tls["cacert"] = arg
143 |                 elif opt in ["--no-verify"]:
144 |                     self._tip.tls["verify"] = False
145 | 
146 |             self._tip.verify_tip()
147 | 
148 |     def _help(self):
149 |         print(self._cli_head)
150 |         print("python tip/elastic_tip_cli.py [command] [options]")
151 |         print("")
152 |         print("Commands:")
153 |         print("    help           Print this help output")
154 |         print("    run            Run the platform and ingest IOC's into ElasticSearch")
155 |         print("    init           Initilize for the first time and load the full IOC lists into ElasticSearch")
156 |         print("    verify         Verify the ElasticSearch index and connection")
157 |         print(self._cli_footer)
158 | 
159 |     def _run_help(self):
160 |         print(self._cli_head)
161 |         print("python tip/elastic_tip_cli.py run [options]")
162 |         print("")
163 |         print("    The run command is used to run the Elastic Threat Intelligence Platform and load")
164 |         print("    the Threat Intelligence, in the form of Indicators Of Compromise (IOC) into")
165 |         print("    your ElasticSearch cluster to be used by the build in Detection-Engine")
166 |         print("")
167 |         print("Options")
168 |         print("    -h, --help                Print help output")
169 |         print("    -e, --es-hosts <value>    Comma seperated list of Elasticsearch hosts to use")
170 |         print("                              E.G:"
171 |               "                                  localhost,127.0.0.2"
172 |               "                                  my-es.com:9300")
173 |         print("    -P, --es-port <value>     Port to use when connecting to Elasticsearch hosts")
174 |         print("    -i, --index <value>       The index to ingest data into")
175 |         print("    -u, --user <value>        Username to use for Authentication to ES")
176 |         print("    -p, --passwd <value>      Password to use for Authentication to ES")
177 |         print("    --modules-list            List module names and the reference link")
178 |         print("    -m, --modules <values>    Modules to enable (all to run all modules):")
179 |         tip = ElasticTip()
180 |         for mod in tip.modules:
181 |             print("                                  {}".format(mod))
182 |         print("    -T, --tls                 Do not use TLS/SSL when connecting to Elasticsearch")
183 |         print("    -c, --ca-cert <value>     Use the cert specified by path")
184 |         print("    --no-verify               Don't verify the TLS/SSL certificate")
185 |         print("    --no-setup                Do not add the index mapping and settings to the given index. Only applicable if the index doesn't exist yet.")
186 |         print(self._cli_footer)
187 | 
188 |     def _verify_help(self):
189 |         print(self._cli_head)
190 |         print("python tip/elastic_tip_cli.py verify [options]")
191 |         print("")
192 |         print("Options")
193 |         print("    -h, --help                Print help output")
194 |         print("    -e, --es-hosts <value>    Comma seperated list of Elasticsearch hosts to use")
195 |         print("    -i, --index <value>       The index to ingest data into")
196 |         print("    -u, --user <value>        Username to use for Authentication to ES")
197 |         print("    -p, --passwd <value>      Password to use for Authentication to ES")
198 |         print("    -T, --tls                 Do not use TLS/SSL when connecting to Elasticsearch")
199 |         print("    -c, --ca-cert <value>     Use the cert specified by path")
200 |         print("    --no-verify               Don't verify the TLS/SSL certificate")
201 |         print(self._cli_footer)
202 | 
203 | 
204 | tip_cli = CLI()
205 | tip_cli.cli()


--------------------------------------------------------------------------------
/tip/elastic_tip.py:
--------------------------------------------------------------------------------
  1 | import hashlib
  2 | import json
  3 | import re
  4 | from datetime import datetime
  5 | from elasticsearch import Elasticsearch
  6 | from elasticsearch.helpers import streaming_bulk
  7 | import tqdm
  8 | 
  9 | from abuse_bazaar import URLhaus, MalwareBazaar, FeodoTracker, SSLBlacklist
 10 | from emergingthreats import ETFireWallBlockIps
 11 | from eset import EsetMalwareIOC
 12 | from abuseipdb import AbuseIPDB
 13 | from spamhaus import SpamhausDrop, SpamhausExtendedDrop, SpamhausDropIpv6
 14 | from botvrij import BotvrijFileNames, BotvrijDomains, BotvrijDstIP, BotvrijUrl
 15 | 
 16 | 
 17 | class ElasticTip:
 18 | 
 19 |     def __init__(self):
 20 |         self.index = "elastic-tip"
 21 |         self.eshosts = []
 22 |         self.esport = 9200
 23 |         self.esuser = None
 24 |         self.espass = None
 25 |         self.setup_index = True
 26 |         self.tls = {
 27 |             "use": True,
 28 |             "cacert": None,
 29 |             "verify": True
 30 |         }
 31 |         self._es = None
 32 |         self._total_count = 0
 33 |         self._ids = []
 34 |         self.modules = {
 35 |             "URLhaus": {
 36 |                 "enabled": False,
 37 |                 "class": URLhaus(),
 38 |                 "ref": "https://urlhaus.abuse.ch/",
 39 |                 "note": None
 40 |             },
 41 |             "MalwareBazaar": {
 42 |                 "enabled": False,
 43 |                 "class": MalwareBazaar(),
 44 |                 "ref": "https://bazaar.abuse.ch/",
 45 |                 "note": None
 46 |             },
 47 |             "FeodoTracker": {
 48 |                 "enabled": False,
 49 |                 "class": FeodoTracker(),
 50 |                 "ref": "https://feodotracker.abuse.ch/",
 51 |                 "note": None
 52 |             },
 53 |             "SSLBlacklist": {
 54 |                 "enabled": False,
 55 |                 "class": SSLBlacklist(),
 56 |                 "ref": "https://sslbl.abuse.ch/",
 57 |                 "note": None
 58 |             },
 59 |             "EmergingThreats-Blocklist": {
 60 |                 "enabled": False,
 61 |                 "class": ETFireWallBlockIps(),
 62 |                 "ref": "https://rules.emergingthreats.net/",
 63 |                 "note": None
 64 |             },
 65 |             "ESET-MalwareIOC": {
 66 |                 "enabled": False,
 67 |                 "class": EsetMalwareIOC(),
 68 |                 "ref": "https://github.com/eset/malware-ioc",
 69 |                 "note": None
 70 |             },
 71 |             "AbuseIPdb": {
 72 |                 "enabled": False,
 73 |                 "class": AbuseIPDB(),
 74 |                 "ref": "https://www.abuseipdb.com/",
 75 |                 "note": "AbuseIPdb requires an API key to work, this can be set through the 'ABUSE_IP_KEY' environment variable or will be requested upon runtime"
 76 |             },
 77 |             "Spamhaus-Drop": {
 78 |                 "enabled": False,
 79 |                 "class": SpamhausDrop(),
 80 |                 "ref": "https://www.spamhaus.org/drop/",
 81 |                 "note": None
 82 |             },
 83 |             "Spamhaus-ExtendedDrop": {
 84 |                 "enabled": False,
 85 |                 "class": SpamhausExtendedDrop(),
 86 |                 "ref": "https://www.spamhaus.org/drop/",
 87 |                 "note": None
 88 |             },
 89 |             "Spamhaus-IPv6Drop": {
 90 |                 "enabled": False,
 91 |                 "class": SpamhausDropIpv6(),
 92 |                 "ref": "https://www.spamhaus.org/drop/",
 93 |                 "note": None
 94 |             },
 95 |             "Botvrij-filenames": {
 96 |                 "enabled": False,
 97 |                 "class": BotvrijFileNames(),
 98 |                 "ref": "https://botvrij.eu/data/ioclist.filename.raw",
 99 |                 "note": None
100 |             },
101 |             "Botvrij-domains": {
102 |                 "enabled": False,
103 |                 "class": BotvrijDomains(),
104 |                 "ref": "https://botvrij.eu/data/ioclist.domain.raw",
105 |                 "note": None
106 |             },
107 |             "Botvrij-destinations": {
108 |                 "enabled": False,
109 |                 "class": BotvrijDstIP(),
110 |                 "ref": "https://botvrij.eu/data/ioclist.ip-dst.raw",
111 |                 "note": None
112 |             },
113 |             "Botvrij-urls": {
114 |                 "enabled": False,
115 |                 "class": BotvrijUrl(),
116 |                 "ref": "https://botvrij.eu/data/ioclist.url.raw",
117 |                 "note": None
118 |             }
119 |         }
120 |         self._total_ingested = 0
121 | 
122 |     def run(self):
123 |         self._build_es_conn()
124 |         self.verify_tip()
125 |         print("Running TIP")
126 |         for module in self.modules:
127 |             if self.modules[module]["enabled"]:
128 |                 mod = self.modules[module]["class"]
129 |                 mod.run()
130 |                 try:
131 |                     self._ingest(mod.iocs, module)
132 |                 except AttributeError:
133 |                     if len(mod.intel) > 0:
134 |                         self._ingest(mod.intel, module, True)
135 |         self._es.indices.refresh(index=self.index)
136 |         print("=======================================")
137 |         print("Ingested a total of {} IOC's".format(self._total_ingested))
138 |         print("=======================================")
139 | 
140 |     def init_tip(self):
141 |         """Initilize the TIP"""
142 |         print("Initilizing TIP")
143 |         for module in self.modules:
144 |             if self.modules[module]["enabled"]:
145 |                 mod = self.modules[module]["class"]
146 |                 mod.run()
147 | 
148 |     def verify_tip(self):
149 |         """Verify the config of the TIP"""
150 |         self._build_es_conn()
151 |         print("Verifying TIP")
152 |         # Get elasticsearch index settings from files
153 |         index_settings = None
154 |         index_mapping = None
155 |         with open("tip/elasticsearch/index_settings.json", "r") as file:
156 |             index_settings = json.loads(file.read())
157 |         with open("tip/elasticsearch/index_mapping.json", "r") as file:
158 |             index_mapping = json.loads(file.read())
159 |         # Verify the index exists
160 |         if self._es.indices.exists(index=self.index):
161 |             print("Index {} exists".format(self.index))
162 |         else:
163 |             print("Index {} does not exists, creating...".format(self.index))
164 |             if self.setup_index:
165 |                 try:
166 |                     self._es.indices.create(
167 |                         index=self.index,
168 |                         body={
169 |                             "settings": index_settings,
170 |                             "mappings": index_mapping
171 |                         }
172 |                     )
173 |                 except Exception as err:
174 |                     print(err)
175 |                     exit()
176 |             else:
177 |                 pass
178 | 
179 |     def _build_es_conn(self):
180 |         if not self._es:
181 |             eshosts = []
182 |             for hoststring in self.eshosts:
183 | 
184 |                 # Determine host and port
185 |                 host, port = self._parse_hosts(hoststring)
186 | 
187 |                 host_block = {
188 |                     'host': host,
189 |                     'port': port
190 |                 }
191 |                 if not self.tls["use"]:
192 |                     host_block["use_ssl"] = False
193 |                 else:
194 |                     host_block["use_ssl"] = True
195 | 
196 |                 if self.tls["cacert"]:
197 |                     host_block["ca_certs"] = self.tls["cacert"]
198 | 
199 |                 if not self.tls["verify"]:
200 |                     host_block["verify_certs"] = False
201 |                     host_block["ssl_show_warn"] = False
202 |                 eshosts.append(host_block)
203 |             self.eshosts = eshosts
204 |             if self.esuser:
205 |                 self._es = Elasticsearch(hosts=self.eshosts, http_auth=(self.esuser, self.espass))
206 |             else:
207 |                 self._es = Elasticsearch(hosts=self.eshosts)
208 |         print("Connection: {}".format(self._es))
209 | 
210 |     def _parse_hosts(self, hoststring):
211 |         """Parse a host string to determine host and port"""
212 |         host = port = None
213 |         if ":" in hoststring:
214 |             arr = hoststring.split(":")
215 |             if len(arr) > 2:
216 |                 raise IndexError("es hosts is malformed")
217 |             host = arr[0]
218 |             port = int(float(arr[1]))
219 |         else:
220 |             host = hoststring
221 |             port = self.esport
222 | 
223 |         return host, port
224 | 
225 |     def _ingest(self, iocs, mod="", intel=False):
226 |         """Ingest IOC's into Elasticsearch"""
227 |         tens_of_thousands = "(^[1-9]*0{4,}$|^[0-9]{2,}0{3,}$)"
228 | 
229 |         print("Ingesting {} iocs from {}".format(len(iocs), mod, self.eshosts))
230 |         self._total_count += len(iocs)
231 | 
232 |         progress = tqdm.tqdm(unit="docs", total=len(iocs))
233 |         successes = 0
234 |         try:
235 |             for ok, action in streaming_bulk(
236 |                     client=self._es,
237 |                     index=self.index,
238 |                     actions=self._generate_es_actions(iocs),
239 |             ):
240 |                 if ok:
241 |                     progress.update(1)
242 |                     successes += ok
243 |                 else:
244 |                     print(ok)
245 |                     print(action)
246 |         except Exception as err:
247 |             print(err)
248 |         print("Indexed %d/%d documents" % (successes, len(iocs)))
249 |         print("Duplicates are not counted!")
250 | 
251 |     def _generate_es_actions(self, documents):
252 |         ids = []
253 |         for ioc in documents:
254 |             if not ioc.id in ids:
255 |                 ids.append(ioc.id)
256 |                 self._total_ingested += 1
257 |                 doc = ioc.intel
258 |                 doc["@timestamp"] = datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
259 |                 doc["_index"] = self.index
260 |                 doc["_id"] = ioc.id
261 |                 doc["_op_type"] = "index"
262 |                 yield doc
263 | 


--------------------------------------------------------------------------------
/tip/ioc.py:
--------------------------------------------------------------------------------
  1 | import json
  2 | import re
  3 | import hashlib
  4 | 
  5 | 
  6 | class IOC:
  7 | 
  8 |     def __init__(self, ref=[], value="", type="", pname="", pcreator=None, pref=None, original=None):
  9 |         self.id = None
 10 |         self.ioc = {
 11 |           "reference": ref,
 12 |           "value": value,
 13 |           "type": type,
 14 |           "provider": {
 15 |             "name": pname
 16 |           }
 17 |         }
 18 |         if pcreator:
 19 |             self.ioc["provider"]["creator"] = pcreator
 20 |         if original:
 21 |             self.ioc["original"] = original
 22 |         if pref:
 23 |             self.ioc["provider"]["reference"] = pref
 24 |         self.threat = None
 25 |         self.vulnerability = None
 26 |         self.rule = None
 27 |         self._validate()
 28 |         self.add_docid()
 29 | 
 30 |     def _validate(self):
 31 |         """Validate the ioc holds to the schema"""
 32 | 
 33 |         # reference should be an empty array or an array of URL's
 34 |         urlmatch = 'https?:\/\/'
 35 |         if len(self.ioc["reference"]) > 0:
 36 |             for x in self.ioc["reference"]:
 37 |                 if re.search(urlmatch, x):
 38 |                     continue
 39 |                 else:
 40 |                     raise SchemaException("The IOC reference field is not a URL: {}".format(x))
 41 | 
 42 |         # Validate the type is one of the accepted values
 43 |         type_accepted = ["hash", "domain", "ip", "string", "unknown"]
 44 |         if self.ioc["type"] not in type_accepted:
 45 |             raise SchemaException("The IOC type field is not one of {}".format(type_accepted))
 46 | 
 47 |     def add_docid(self):
 48 |         self.id = hashlib.sha1(json.dumps(self.ioc).encode('utf-8')).hexdigest()
 49 | 
 50 | 
 51 | class Intel:
 52 | 
 53 |     def __init__(self,
 54 |                  original=None,
 55 |                  event_type=None,
 56 |                  event_reference=None,
 57 |                  event_provider=None,
 58 |                  event_dataset=None,
 59 |                  threat_first_seen=None,
 60 |                  threat_last_seen=None,
 61 |                  threat_last_update=None,
 62 |                  threat_type=None,
 63 |                  threat_description=None):
 64 |         """
 65 | 
 66 |         :param original: original intel in its original format
 67 |         :param event_type: Type of event (indicator)
 68 |         :param event_reference: url which provides context
 69 |         :param event_module: event.module field
 70 |         :param event_dataset: event.dataset field
 71 |         :param threat_first_seen: date at which the threat was first seen or added
 72 |         :param threat_last_seen: date at which the threat was last seen to be active
 73 |         :param threat_last_update: date at which the intell has last been updated
 74 |         :param threat_type: threat.type field
 75 |         :param threat_description: description field to provide context on the intel
 76 |         """
 77 |         self.id = None
 78 |         self.intel = {
 79 |             "event": {
 80 |                 "kind": "enrichment",
 81 |                 "category": "threat",
 82 |                 "type": event_type,
 83 |                 "reference": event_reference,
 84 |                 "provider": event_provider,
 85 |                 "dataset": event_dataset,
 86 |                 "severity": 0,
 87 |                 "risk_score": 0,
 88 |                 "original": original
 89 |             },
 90 |             "ecs": {
 91 |               "version": "1.8.0"
 92 |             },
 93 |             "threat": {
 94 |                 "indicator": {
 95 |                     "first_seen": threat_first_seen,
 96 |                     "last_seen": threat_last_seen,
 97 |                     "sightings": 0,
 98 |                     "type": [],
 99 |                     "description": threat_description,
100 |                 },
101 |                 "tactic": {},
102 |                 "technique": {},
103 |                 "feed": {
104 |                     "name": event_provider,
105 |                     "reference": event_reference
106 |                 }
107 |             }
108 |         }
109 |         self._add_type(threat_type)
110 | 
111 |     def add_mitre(self, tactic=None, technique=None):
112 |         """
113 | 
114 |         :param tactic: Tactic ID e.g TA0002
115 |         :param technique: Technique ID e.g T1059
116 |         :return:
117 |         """
118 | 
119 |         if tactic or technique:
120 |             self.intel["threat"]["framework"] = "MITRE ATT&CK"
121 | 
122 |         if tactic:
123 |             self.intel["threat"]["tactic"]["id"] = tactic
124 | 
125 |         if technique:
126 |             self.intel["threat"]["technique"]["id"] = technique
127 | 
128 |     def add_docid(self):
129 |         self.id = hashlib.sha1(json.dumps(self.intel).encode('utf-8')).hexdigest()
130 |         self.intel["event"]["hash"] = self.id
131 | 
132 |     def _add_type(self, indicator_type=None):
133 |         if indicator_type:
134 |             self.intel["threat"]["indicator"]["type"].append(indicator_type)
135 | 
136 |     def _build_traffic(self, object,
137 |                        domain=None,
138 |                        ip=None,
139 |                        mac=None,
140 |                        port=None,
141 |                        registered_domain=None,
142 |                        subdomain=None,
143 |                        top_level_domain=None):
144 |         """Shared function to build source & destination blocks
145 |         :param object: The name of the field to build (source/destination)
146 |         :param domain:
147 |         :param ip:
148 |         :param mac:
149 |         :param port:
150 |         :param registered_domain:
151 |         :param subdomain:
152 |         :param top_level_domain:
153 |         :return:"""
154 |         raise OutDatedException()
155 |         if object not in ["source", "destination"]:
156 |             raise KeyError("wrong object specified")
157 |         try:
158 |             obj = self.intel[object]
159 |         except KeyError:
160 |             obj = {}
161 |         finally:
162 |             if domain:
163 |                 obj["domain"] = domain
164 |             if ip:
165 |                 obj["ip"] = ip
166 |             if mac:
167 |                 obj["mac"] = mac
168 |             if port:
169 |                 obj["port"] = port
170 |             if registered_domain:
171 |                 obj["registered_domain"] = registered_domain
172 |             if subdomain:
173 |                 obj["subdomain"] = subdomain
174 |             if top_level_domain:
175 |                 obj["top_level_domain"] = top_level_domain
176 | 
177 |             self.intel[object] = obj
178 |             self.intel["threat"]["type"] = "IPV4"
179 | 
180 |     def add_ip(self, domain=None, ip=None, mac=None, port=None, registered_domain=None, subdomain=None, top_level_domain=None):
181 |         """
182 |         Add network information as indicator
183 |         :param domain: domain name
184 |         :param ip: IPv4 or IPv6 address
185 |         :param mac:
186 |         :param port: Port number
187 |         :param registered_domain:
188 |         :param subdomain:
189 |         :param top_level_domain:
190 |         :return:
191 |         """
192 |         if domain:
193 |             self.intel["threat"]["indicator"]["domain"] = domain
194 |         if ip:
195 |             self.intel["threat"]["indicator"]["ip"] = ip
196 |         if port:
197 |             self.intel["threat"]["indicator"]["port"] = port
198 | 
199 |     def add_malware(self, name=None, family=None, malware_type=None):
200 |         try:
201 |             obj = self.intel["threat"]["malware"]
202 |         except KeyError:
203 |             obj = {}
204 |         finally:
205 |             if name:
206 |                 obj["name"] = name
207 |             if family:
208 |                 obj["family"] = family
209 |             if malware_type:
210 |                 obj["type"] = malware_type
211 | 
212 |             # Add object to intel
213 |             self.intel["threat"]["malware"] = obj
214 | 
215 |     def add_file(self, directory=None, drive_letter=None, extension=None, gid=None, group=None, mime_type=None,
216 |                  mode=None, name=None, owner=None, path=None, size=None, uid=None, md5=None, sha1=None, sha256=None,
217 |                  sha512=None):
218 |         try:
219 |             obj = self.intel["file"]
220 |         except KeyError:
221 |             obj = {}
222 |         finally:
223 |             if directory:
224 |                 obj["directory"] = directory
225 |             if drive_letter:
226 |                 obj["drive_letter"] = drive_letter
227 |             if extension:
228 |                 obj["extension"] = extension
229 |             if gid:
230 |                 obj["gid"] = gid
231 |             if group:
232 |                 obj["group"] = group
233 |             if mime_type:
234 |                 obj["mime_type"] = mime_type
235 |             if mode:
236 |                 obj["mode"] = mode
237 |             if name:
238 |                 obj["name"] = name
239 |             if owner:
240 |                 obj["owner"] = owner
241 |             if path:
242 |                 obj["path"] = path
243 |             if size:
244 |                 obj["size"] = size
245 |             if uid:
246 |                 obj["uid"] = uid
247 |             if md5:
248 |                 try:
249 |                     x = obj["hash"]
250 |                 except KeyError:
251 |                     obj["hash"] = {}
252 |                 finally:
253 |                     obj["hash"]["md5"] = md5
254 |             if sha1:
255 |                 try:
256 |                     x = obj["hash"]
257 |                 except KeyError:
258 |                     obj["hash"] = {}
259 |                 finally:
260 |                     obj["hash"]["sha1"] = sha1
261 |             if sha256:
262 |                 try:
263 |                     x = obj["hash"]
264 |                 except KeyError:
265 |                     obj["hash"] = {}
266 |                 finally:
267 |                     obj["hash"]["sha256"] = sha256
268 |             if sha512:
269 |                 try:
270 |                     x = obj["hash"]
271 |                 except KeyError:
272 |                     obj["hash"] = {}
273 |                 finally:
274 |                     obj["hash"]["sha512"] = sha512
275 | 
276 |             self.intel["file"] = obj
277 | 
278 |     def add_process(self):
279 |         pass
280 | 
281 |     def add_x509(self, alt_names=None, iss_common=None, iss_country=None, iss_distinguished=None, iss_local=None,
282 |                  iss_org=None, iss_org_unit=None, iss_state=None, not_after=None, not_before=None, pup_key_algo=None,
283 |                  pup_key_curve=None, pup_key_ex=None, pup_key_size=None, serial=None, signature_algo=None,
284 |                  subject_common=None, subject_country=None, subject_distinguished=None, subject_local=None,
285 |                  subject_org=None, subject_org_unit=None, subject_state=None, version=None):
286 |         try:
287 |             obj = self.intel["x509"]
288 |         except KeyError:
289 |             obj = {}
290 | 
291 |         # make sure issuer object exists
292 |         if iss_common or iss_country or iss_distinguished or iss_local or iss_org or iss_org_unit or iss_state:
293 |             try:
294 |                 iss = obj["issuer"]
295 |             except KeyError:
296 |                 obj["issuer"] = {}
297 |         # make sure subject object exists
298 |         if subject_common or subject_country or subject_distinguished or subject_local or subject_org or subject_org_unit or subject_state:
299 |             try:
300 |                 sub = obj["subject"]
301 |             except KeyError:
302 |                 obj["subject"] = {}
303 | 
304 |         if alt_names:
305 |             obj["alternative_names"] = alt_names
306 |         if iss_common:
307 |             obj["issuer"]["common_name"] = iss_common
308 |         if iss_country:
309 |             obj["issuer"]["country"] = iss_country
310 |         if iss_distinguished:
311 |             obj["issuer"]["distinguished_name"] = iss_distinguished
312 |         if iss_local:
313 |             obj["issuer"]["locality"] = iss_local
314 |         if iss_org:
315 |             obj["issuer"]["organization"] = iss_org
316 |         if iss_org_unit:
317 |             obj["issuer"]["organizational_unit"] = iss_org_unit
318 |         if iss_state:
319 |             obj["issuer"]["state_or_province"] = iss_state
320 |         if not_after:
321 |             obj["not_after"] = not_after
322 |         if not_before:
323 |             obj["not_before"] = not_before
324 |         if pup_key_algo:
325 |             obj["public_key_algorithm"] = pup_key_algo
326 |         if pup_key_curve:
327 |             obj["public_key_curve"] = pup_key_curve
328 |         if pup_key_ex:
329 |             obj["public_key_exponent"] = pup_key_ex
330 |         if pup_key_size:
331 |             obj["public_key_size"] = pup_key_size
332 |         if serial:
333 |             obj["serial_number"] = serial
334 |         if signature_algo:
335 |             obj["signature_algorithm"] = signature_algo
336 |         if subject_common:
337 |             obj["subject"]["common_name"] = subject_common
338 |         if subject_country:
339 |             obj["subject"]["country"] = subject_country
340 |         if subject_distinguished:
341 |             obj["subject"]["distinguished_name"] = subject_distinguished
342 |         if subject_local:
343 |             obj["subject"]["locality"] = subject_local
344 |         if subject_org:
345 |             obj["subject"]["organization"] = subject_org
346 |         if subject_org_unit:
347 |             obj["subject"]["organizational_unit"] = subject_org_unit
348 |         if subject_state:
349 |             obj["subject"]["state_or_province"] = subject_state
350 |         if version:
351 |             obj["version_number"] = version
352 | 
353 |         self.intel["x509"] = obj
354 | 
355 |     def add_pe(self):
356 |         pass
357 | 
358 |     def add_url(self, domain=None, extension=None, fragment=None, full=None, original=None, password=None, path=None,
359 |                 port=None, query=None, registered_domain=None, scheme=None, subdomain=None, top_level_domain=None,
360 |                 username=None):
361 |         try:
362 |             obj = self.intel["url"]
363 |         except KeyError:
364 |             obj = {}
365 |         finally:
366 |             # Do some parsing of stuff to fill in non-existing fields
367 |             if not scheme and original and "://" in original:
368 |                 scheme = original.split("://")[0]
369 |             elif not scheme and full and "://" in full:
370 |                 scheme = full.split("://")[0]
371 |             if not original and full:
372 |                 original = full
373 | 
374 |             if domain:
375 |                 obj["domain"] = domain
376 |             if extension:
377 |                 obj["extension"] = extension
378 |             if fragment:
379 |                 obj["fragment"] = fragment
380 |             if full:
381 |                 obj["full"] = full
382 |             if original:
383 |                 obj["original"] = original
384 |             if password:
385 |                 obj["password"] = password
386 |             if path:
387 |                 obj["path"] = path
388 |             if port:
389 |                 obj["port"] = port
390 |             if query:
391 |                 obj["query"] = query
392 |             if registered_domain:
393 |                 obj["registered_domain"] = registered_domain
394 |             if scheme:
395 |                 obj["scheme"] = scheme
396 |             if subdomain:
397 |                 obj["subdomain"] = subdomain
398 |             if top_level_domain:
399 |                 obj["top_level_domain"] = top_level_domain
400 |             if username:
401 |                 obj["username"] = username
402 | 
403 |         self.intel["url"] = obj
404 | 
405 |     def add_tls(self, cipher=None, c_cert=None, c_chain=None, c_md5=None, c_sha1=None, c_sha256=None, c_issuer=None,
406 |                 c_ja3=None, c_not_after=None, c_not_before=None, c_server_name=None, c_subject=None, curve=None,
407 |                 s_cert=None, s_chain=None, s_md5=None, s_sha1=None, s_sha256=None, s_issuer=None,
408 |                 s_ja3=None, s_not_after=None, s_not_before=None, s_server_name=None, s_subject=None):
409 |         try:
410 |             obj = self.intel["tls"]
411 |         except KeyError:
412 |             obj = {}
413 | 
414 |         # Make sure the required nested fields are available
415 |         if c_cert or c_chain or c_md5 or c_sha1 or c_sha256 or c_issuer or c_ja3 or c_not_after or c_not_before or c_server_name or c_subject:
416 |             try:
417 |                 iss = obj["client"]
418 |             except KeyError:
419 |                 obj["client"] = {}
420 |             if c_md5 or c_sha1 or c_sha256:
421 |                 try:
422 |                     hash = obj["client"]["hash"]
423 |                 except KeyError:
424 |                     obj["client"]["hash"] = {}
425 |         if s_cert or s_chain or s_md5 or s_sha1 or s_sha256 or s_issuer or s_ja3 or s_not_after or s_not_before or s_server_name or s_subject:
426 |             try:
427 |                 iss = obj["server"]
428 |             except KeyError:
429 |                 obj["server"] = {}
430 |             if s_md5 or s_sha1 or s_sha256:
431 |                 try:
432 |                     hash = obj["server"]["hash"]
433 |                 except KeyError:
434 |                     obj["server"]["hash"] = {}
435 | 
436 |         if cipher:
437 |             obj[""] = cipher
438 |         if c_cert:
439 |             obj["client"]["certificate"] = c_cert
440 |         if c_chain:
441 |             obj["client"]["certificate_chain"] = c_chain
442 |         if c_md5:
443 |             obj["client"]["hash"]["md5"] = c_md5
444 |         if c_sha1:
445 |             obj["client"]["hash"]["sha1"] = c_sha1
446 |         if c_sha256:
447 |             obj["client"]["hash"]["sha256"] = c_sha256
448 |         if c_issuer:
449 |             obj["client"]["issuer"] = c_issuer
450 |         if c_ja3:
451 |             obj["client"]["ja3"] = c_ja3
452 |         if c_not_after:
453 |             obj["client"]["not_after"] = c_not_after
454 |         if c_not_before:
455 |             obj["client"]["not_before"] = c_not_before
456 |         if c_server_name:
457 |             obj["client"]["server_name"] = c_server_name
458 |         if c_subject:
459 |             obj["client"]["subject"] = c_subject
460 |         if curve:
461 |             obj[""] = curve
462 |         if s_cert:
463 |             obj["server"]["certificate"] = s_cert
464 |         if s_chain:
465 |             obj["server"]["certificate_chain"] = s_chain
466 |         if s_md5:
467 |             obj["server"]["hash"]["md5"] = s_md5
468 |         if s_sha1:
469 |             obj["server"]["hash"]["sha1"] = s_sha1
470 |         if s_sha256:
471 |             obj["server"]["hash"]["sha256"] = s_sha256
472 |         if s_issuer:
473 |             obj["server"]["issuer"] = s_issuer
474 |         if s_ja3:
475 |             obj["server"]["ja3s"] = s_ja3
476 |         if s_not_after:
477 |             obj["server"]["not_after"] = s_not_after
478 |         if s_not_before:
479 |             obj["server"]["not_before"] = s_not_before
480 |         if s_server_name:
481 |             obj["server"]["server_name"] = s_server_name
482 |         if s_subject:
483 |             obj["server"]["subject"] = s_subject
484 | 
485 |         self.intel["tls"] = obj
486 | 
487 | 
488 | class SchemaException(Exception):
489 |     pass
490 | 
491 | 
492 | class OutDatedException(Exception):
493 |     pass
494 | 


--------------------------------------------------------------------------------
/tip/elasticsearch/index_mapping.json:
--------------------------------------------------------------------------------
   1 | {
   2 |   "properties": {
   3 |     "@timestamp": {
   4 |       "type": "date"
   5 |     },
   6 |     "reference": {
   7 |       "type": "keyword",
   8 |       "ignore_above": 1024
   9 |     },
  10 |     "value": {
  11 |       "type": "keyword",
  12 |       "ignore_above": 1024
  13 |     },
  14 |     "type": {
  15 |       "type": "keyword",
  16 |       "ignore_above": 1024
  17 |     },
  18 |     "provider": {
  19 |       "properties": {
  20 |         "name": {
  21 |           "type": "keyword",
  22 |           "ignore_above": 1024
  23 |         },
  24 |         "creator": {
  25 |           "type": "keyword",
  26 |           "ignore_above": 1024
  27 |         },
  28 |         "reference": {
  29 |           "type": "keyword",
  30 |           "ignore_above": 1024
  31 |         }
  32 |       }
  33 |     },
  34 |     "original": {
  35 |       "type": "text"
  36 |     },
  37 |     "threat": {
  38 |       "properties": {
  39 |         "indicator": {
  40 |           "properties": {
  41 |             "ip": {
  42 |               "type": "ip"
  43 |             },
  44 |             "domain": {
  45 |               "ignore_above": 1024,
  46 |               "type": "keyword"
  47 |             },
  48 |             "port": {
  49 |               "type": "long"
  50 |             },
  51 |             "file": {
  52 |       "properties": {
  53 |         "accessed": {
  54 |           "type": "date"
  55 |         },
  56 |         "attributes": {
  57 |           "ignore_above": 1024,
  58 |           "type": "keyword"
  59 |         },
  60 |         "code_signature": {
  61 |           "properties": {
  62 |             "exists": {
  63 |               "type": "boolean"
  64 |             },
  65 |             "status": {
  66 |               "ignore_above": 1024,
  67 |               "type": "keyword"
  68 |             },
  69 |             "subject_name": {
  70 |               "ignore_above": 1024,
  71 |               "type": "keyword"
  72 |             },
  73 |             "trusted": {
  74 |               "type": "boolean"
  75 |             },
  76 |             "valid": {
  77 |               "type": "boolean"
  78 |             }
  79 |           }
  80 |         },
  81 |         "created": {
  82 |           "type": "date"
  83 |         },
  84 |         "ctime": {
  85 |           "type": "date"
  86 |         },
  87 |         "device": {
  88 |           "ignore_above": 1024,
  89 |           "type": "keyword"
  90 |         },
  91 |         "directory": {
  92 |           "ignore_above": 1024,
  93 |           "type": "keyword"
  94 |         },
  95 |         "drive_letter": {
  96 |           "ignore_above": 1,
  97 |           "type": "keyword"
  98 |         },
  99 |         "extension": {
 100 |           "ignore_above": 1024,
 101 |           "type": "keyword"
 102 |         },
 103 |         "gid": {
 104 |           "ignore_above": 1024,
 105 |           "type": "keyword"
 106 |         },
 107 |         "group": {
 108 |           "ignore_above": 1024,
 109 |           "type": "keyword"
 110 |         },
 111 |         "hash": {
 112 |           "properties": {
 113 |             "md5": {
 114 |               "ignore_above": 1024,
 115 |               "type": "keyword"
 116 |             },
 117 |             "sha1": {
 118 |               "ignore_above": 1024,
 119 |               "type": "keyword"
 120 |             },
 121 |             "sha256": {
 122 |               "ignore_above": 1024,
 123 |               "type": "keyword"
 124 |             },
 125 |             "sha512": {
 126 |               "ignore_above": 1024,
 127 |               "type": "keyword"
 128 |             }
 129 |           }
 130 |         },
 131 |         "inode": {
 132 |           "ignore_above": 1024,
 133 |           "type": "keyword"
 134 |         },
 135 |         "mime_type": {
 136 |           "ignore_above": 1024,
 137 |           "type": "keyword"
 138 |         },
 139 |         "mode": {
 140 |           "ignore_above": 1024,
 141 |           "type": "keyword"
 142 |         },
 143 |         "mtime": {
 144 |           "type": "date"
 145 |         },
 146 |         "name": {
 147 |           "ignore_above": 1024,
 148 |           "type": "keyword"
 149 |         },
 150 |         "owner": {
 151 |           "ignore_above": 1024,
 152 |           "type": "keyword"
 153 |         },
 154 |         "path": {
 155 |           "fields": {
 156 |             "text": {
 157 |               "norms": false,
 158 |               "type": "text"
 159 |             }
 160 |           },
 161 |           "ignore_above": 1024,
 162 |           "type": "keyword"
 163 |         },
 164 |         "pe": {
 165 |           "properties": {
 166 |             "company": {
 167 |               "ignore_above": 1024,
 168 |               "type": "keyword"
 169 |             },
 170 |             "description": {
 171 |               "ignore_above": 1024,
 172 |               "type": "keyword"
 173 |             },
 174 |             "file_version": {
 175 |               "ignore_above": 1024,
 176 |               "type": "keyword"
 177 |             },
 178 |             "original_file_name": {
 179 |               "ignore_above": 1024,
 180 |               "type": "keyword"
 181 |             },
 182 |             "product": {
 183 |               "ignore_above": 1024,
 184 |               "type": "keyword"
 185 |             }
 186 |           }
 187 |         },
 188 |         "size": {
 189 |           "type": "long"
 190 |         },
 191 |         "target_path": {
 192 |           "fields": {
 193 |             "text": {
 194 |               "norms": false,
 195 |               "type": "text"
 196 |             }
 197 |           },
 198 |           "ignore_above": 1024,
 199 |           "type": "keyword"
 200 |         },
 201 |         "type": {
 202 |           "ignore_above": 1024,
 203 |           "type": "keyword"
 204 |         },
 205 |         "uid": {
 206 |           "ignore_above": 1024,
 207 |           "type": "keyword"
 208 |         }
 209 |       }
 210 |     },
 211 |             "url": {
 212 |       "properties": {
 213 |         "domain": {
 214 |           "ignore_above": 1024,
 215 |           "type": "keyword"
 216 |         },
 217 |         "extension": {
 218 |           "ignore_above": 1024,
 219 |           "type": "keyword"
 220 |         },
 221 |         "fragment": {
 222 |           "ignore_above": 1024,
 223 |           "type": "keyword"
 224 |         },
 225 |         "full": {
 226 |           "fields": {
 227 |             "text": {
 228 |               "norms": false,
 229 |               "type": "text"
 230 |             }
 231 |           },
 232 |           "ignore_above": 1024,
 233 |           "type": "keyword"
 234 |         },
 235 |         "original": {
 236 |           "fields": {
 237 |             "text": {
 238 |               "norms": false,
 239 |               "type": "text"
 240 |             }
 241 |           },
 242 |           "ignore_above": 1024,
 243 |           "type": "keyword"
 244 |         },
 245 |         "password": {
 246 |           "ignore_above": 1024,
 247 |           "type": "keyword"
 248 |         },
 249 |         "path": {
 250 |           "ignore_above": 1024,
 251 |           "type": "keyword"
 252 |         },
 253 |         "port": {
 254 |           "type": "long"
 255 |         },
 256 |         "query": {
 257 |           "ignore_above": 1024,
 258 |           "type": "keyword"
 259 |         },
 260 |         "registered_domain": {
 261 |           "ignore_above": 1024,
 262 |           "type": "keyword"
 263 |         },
 264 |         "scheme": {
 265 |           "ignore_above": 1024,
 266 |           "type": "keyword"
 267 |         },
 268 |         "top_level_domain": {
 269 |           "ignore_above": 1024,
 270 |           "type": "keyword"
 271 |         },
 272 |         "username": {
 273 |           "ignore_above": 1024,
 274 |           "type": "keyword"
 275 |         }
 276 |       }
 277 |     }
 278 |           }
 279 |         },
 280 |         "type": {
 281 |           "type": "keyword",
 282 |           "ignore_above": 1024
 283 |         }
 284 |       }
 285 |     },
 286 |     "event": {
 287 |       "properties": {
 288 |         "kind": {
 289 |           "type": "keyword",
 290 |           "ignore_above": 1024
 291 |         },
 292 |         "category": {
 293 |           "type": "keyword",
 294 |           "ignore_above": 1024
 295 |         },
 296 |         "type": {
 297 |           "type": "keyword",
 298 |           "ignore_above": 1024
 299 |         },
 300 |         "reference": {
 301 |           "type": "keyword",
 302 |           "ignore_above": 1024
 303 |         },
 304 |         "module": {
 305 |           "type": "keyword",
 306 |           "ignore_above": 1024
 307 |         },
 308 |         "dataset": {
 309 |           "type": "keyword",
 310 |           "ignore_above": 1024
 311 |         },
 312 |         "original": {
 313 |           "type": "text"
 314 |         },
 315 |         "hash": {
 316 |           "type": "keyword",
 317 |           "ignore_above": 1024
 318 |         }
 319 |       }
 320 |     },
 321 |     "file": {
 322 |       "properties": {
 323 |         "accessed": {
 324 |           "type": "date"
 325 |         },
 326 |         "attributes": {
 327 |           "ignore_above": 1024,
 328 |           "type": "keyword"
 329 |         },
 330 |         "code_signature": {
 331 |           "properties": {
 332 |             "exists": {
 333 |               "type": "boolean"
 334 |             },
 335 |             "status": {
 336 |               "ignore_above": 1024,
 337 |               "type": "keyword"
 338 |             },
 339 |             "subject_name": {
 340 |               "ignore_above": 1024,
 341 |               "type": "keyword"
 342 |             },
 343 |             "trusted": {
 344 |               "type": "boolean"
 345 |             },
 346 |             "valid": {
 347 |               "type": "boolean"
 348 |             }
 349 |           }
 350 |         },
 351 |         "created": {
 352 |           "type": "date"
 353 |         },
 354 |         "ctime": {
 355 |           "type": "date"
 356 |         },
 357 |         "device": {
 358 |           "ignore_above": 1024,
 359 |           "type": "keyword"
 360 |         },
 361 |         "directory": {
 362 |           "ignore_above": 1024,
 363 |           "type": "keyword"
 364 |         },
 365 |         "drive_letter": {
 366 |           "ignore_above": 1,
 367 |           "type": "keyword"
 368 |         },
 369 |         "extension": {
 370 |           "ignore_above": 1024,
 371 |           "type": "keyword"
 372 |         },
 373 |         "gid": {
 374 |           "ignore_above": 1024,
 375 |           "type": "keyword"
 376 |         },
 377 |         "group": {
 378 |           "ignore_above": 1024,
 379 |           "type": "keyword"
 380 |         },
 381 |         "hash": {
 382 |           "properties": {
 383 |             "md5": {
 384 |               "ignore_above": 1024,
 385 |               "type": "keyword"
 386 |             },
 387 |             "sha1": {
 388 |               "ignore_above": 1024,
 389 |               "type": "keyword"
 390 |             },
 391 |             "sha256": {
 392 |               "ignore_above": 1024,
 393 |               "type": "keyword"
 394 |             },
 395 |             "sha512": {
 396 |               "ignore_above": 1024,
 397 |               "type": "keyword"
 398 |             }
 399 |           }
 400 |         },
 401 |         "inode": {
 402 |           "ignore_above": 1024,
 403 |           "type": "keyword"
 404 |         },
 405 |         "mime_type": {
 406 |           "ignore_above": 1024,
 407 |           "type": "keyword"
 408 |         },
 409 |         "mode": {
 410 |           "ignore_above": 1024,
 411 |           "type": "keyword"
 412 |         },
 413 |         "mtime": {
 414 |           "type": "date"
 415 |         },
 416 |         "name": {
 417 |           "ignore_above": 1024,
 418 |           "type": "keyword"
 419 |         },
 420 |         "owner": {
 421 |           "ignore_above": 1024,
 422 |           "type": "keyword"
 423 |         },
 424 |         "path": {
 425 |           "fields": {
 426 |             "text": {
 427 |               "norms": false,
 428 |               "type": "text"
 429 |             }
 430 |           },
 431 |           "ignore_above": 1024,
 432 |           "type": "keyword"
 433 |         },
 434 |         "pe": {
 435 |           "properties": {
 436 |             "company": {
 437 |               "ignore_above": 1024,
 438 |               "type": "keyword"
 439 |             },
 440 |             "description": {
 441 |               "ignore_above": 1024,
 442 |               "type": "keyword"
 443 |             },
 444 |             "file_version": {
 445 |               "ignore_above": 1024,
 446 |               "type": "keyword"
 447 |             },
 448 |             "original_file_name": {
 449 |               "ignore_above": 1024,
 450 |               "type": "keyword"
 451 |             },
 452 |             "product": {
 453 |               "ignore_above": 1024,
 454 |               "type": "keyword"
 455 |             }
 456 |           }
 457 |         },
 458 |         "size": {
 459 |           "type": "long"
 460 |         },
 461 |         "target_path": {
 462 |           "fields": {
 463 |             "text": {
 464 |               "norms": false,
 465 |               "type": "text"
 466 |             }
 467 |           },
 468 |           "ignore_above": 1024,
 469 |           "type": "keyword"
 470 |         },
 471 |         "type": {
 472 |           "ignore_above": 1024,
 473 |           "type": "keyword"
 474 |         },
 475 |         "uid": {
 476 |           "ignore_above": 1024,
 477 |           "type": "keyword"
 478 |         }
 479 |       }
 480 |     },
 481 |     "process": {
 482 |       "properties": {
 483 |         "args": {
 484 |           "ignore_above": 1024,
 485 |           "type": "keyword"
 486 |         },
 487 |         "args_count": {
 488 |           "type": "long"
 489 |         },
 490 |         "code_signature": {
 491 |           "properties": {
 492 |             "exists": {
 493 |               "type": "boolean"
 494 |             },
 495 |             "status": {
 496 |               "ignore_above": 1024,
 497 |               "type": "keyword"
 498 |             },
 499 |             "subject_name": {
 500 |               "ignore_above": 1024,
 501 |               "type": "keyword"
 502 |             },
 503 |             "trusted": {
 504 |               "type": "boolean"
 505 |             },
 506 |             "valid": {
 507 |               "type": "boolean"
 508 |             }
 509 |           }
 510 |         },
 511 |         "command_line": {
 512 |           "fields": {
 513 |             "text": {
 514 |               "norms": false,
 515 |               "type": "text"
 516 |             }
 517 |           },
 518 |           "ignore_above": 1024,
 519 |           "type": "keyword"
 520 |         },
 521 |         "entity_id": {
 522 |           "ignore_above": 1024,
 523 |           "type": "keyword"
 524 |         },
 525 |         "executable": {
 526 |           "fields": {
 527 |             "text": {
 528 |               "norms": false,
 529 |               "type": "text"
 530 |             }
 531 |           },
 532 |           "ignore_above": 1024,
 533 |           "type": "keyword"
 534 |         },
 535 |         "exit_code": {
 536 |           "type": "long"
 537 |         },
 538 |         "hash": {
 539 |           "properties": {
 540 |             "md5": {
 541 |               "ignore_above": 1024,
 542 |               "type": "keyword"
 543 |             },
 544 |             "sha1": {
 545 |               "ignore_above": 1024,
 546 |               "type": "keyword"
 547 |             },
 548 |             "sha256": {
 549 |               "ignore_above": 1024,
 550 |               "type": "keyword"
 551 |             },
 552 |             "sha512": {
 553 |               "ignore_above": 1024,
 554 |               "type": "keyword"
 555 |             }
 556 |           }
 557 |         },
 558 |         "name": {
 559 |           "fields": {
 560 |             "text": {
 561 |               "norms": false,
 562 |               "type": "text"
 563 |             }
 564 |           },
 565 |           "ignore_above": 1024,
 566 |           "type": "keyword"
 567 |         },
 568 |         "parent": {
 569 |           "properties": {
 570 |             "args": {
 571 |               "ignore_above": 1024,
 572 |               "type": "keyword"
 573 |             },
 574 |             "args_count": {
 575 |               "type": "long"
 576 |             },
 577 |             "code_signature": {
 578 |               "properties": {
 579 |                 "exists": {
 580 |                   "type": "boolean"
 581 |                 },
 582 |                 "status": {
 583 |                   "ignore_above": 1024,
 584 |                   "type": "keyword"
 585 |                 },
 586 |                 "subject_name": {
 587 |                   "ignore_above": 1024,
 588 |                   "type": "keyword"
 589 |                 },
 590 |                 "trusted": {
 591 |                   "type": "boolean"
 592 |                 },
 593 |                 "valid": {
 594 |                   "type": "boolean"
 595 |                 }
 596 |               }
 597 |             },
 598 |             "command_line": {
 599 |               "fields": {
 600 |                 "text": {
 601 |                   "norms": false,
 602 |                   "type": "text"
 603 |                 }
 604 |               },
 605 |               "ignore_above": 1024,
 606 |               "type": "keyword"
 607 |             },
 608 |             "entity_id": {
 609 |               "ignore_above": 1024,
 610 |               "type": "keyword"
 611 |             },
 612 |             "executable": {
 613 |               "fields": {
 614 |                 "text": {
 615 |                   "norms": false,
 616 |                   "type": "text"
 617 |                 }
 618 |               },
 619 |               "ignore_above": 1024,
 620 |               "type": "keyword"
 621 |             },
 622 |             "exit_code": {
 623 |               "type": "long"
 624 |             },
 625 |             "hash": {
 626 |               "properties": {
 627 |                 "md5": {
 628 |                   "ignore_above": 1024,
 629 |                   "type": "keyword"
 630 |                 },
 631 |                 "sha1": {
 632 |                   "ignore_above": 1024,
 633 |                   "type": "keyword"
 634 |                 },
 635 |                 "sha256": {
 636 |                   "ignore_above": 1024,
 637 |                   "type": "keyword"
 638 |                 },
 639 |                 "sha512": {
 640 |                   "ignore_above": 1024,
 641 |                   "type": "keyword"
 642 |                 }
 643 |               }
 644 |             },
 645 |             "name": {
 646 |               "fields": {
 647 |                 "text": {
 648 |                   "norms": false,
 649 |                   "type": "text"
 650 |                 }
 651 |               },
 652 |               "ignore_above": 1024,
 653 |               "type": "keyword"
 654 |             },
 655 |             "pgid": {
 656 |               "type": "long"
 657 |             },
 658 |             "pid": {
 659 |               "type": "long"
 660 |             },
 661 |             "ppid": {
 662 |               "type": "long"
 663 |             },
 664 |             "start": {
 665 |               "type": "date"
 666 |             },
 667 |             "thread": {
 668 |               "properties": {
 669 |                 "id": {
 670 |                   "type": "long"
 671 |                 },
 672 |                 "name": {
 673 |                   "ignore_above": 1024,
 674 |                   "type": "keyword"
 675 |                 }
 676 |               }
 677 |             },
 678 |             "title": {
 679 |               "fields": {
 680 |                 "text": {
 681 |                   "norms": false,
 682 |                   "type": "text"
 683 |                 }
 684 |               },
 685 |               "ignore_above": 1024,
 686 |               "type": "keyword"
 687 |             },
 688 |             "uptime": {
 689 |               "type": "long"
 690 |             },
 691 |             "working_directory": {
 692 |               "fields": {
 693 |                 "text": {
 694 |                   "norms": false,
 695 |                   "type": "text"
 696 |                 }
 697 |               },
 698 |               "ignore_above": 1024,
 699 |               "type": "keyword"
 700 |             }
 701 |           }
 702 |         },
 703 |         "pe": {
 704 |           "properties": {
 705 |             "company": {
 706 |               "ignore_above": 1024,
 707 |               "type": "keyword"
 708 |             },
 709 |             "description": {
 710 |               "ignore_above": 1024,
 711 |               "type": "keyword"
 712 |             },
 713 |             "file_version": {
 714 |               "ignore_above": 1024,
 715 |               "type": "keyword"
 716 |             },
 717 |             "original_file_name": {
 718 |               "ignore_above": 1024,
 719 |               "type": "keyword"
 720 |             },
 721 |             "product": {
 722 |               "ignore_above": 1024,
 723 |               "type": "keyword"
 724 |             }
 725 |           }
 726 |         },
 727 |         "pgid": {
 728 |           "type": "long"
 729 |         },
 730 |         "pid": {
 731 |           "type": "long"
 732 |         },
 733 |         "ppid": {
 734 |           "type": "long"
 735 |         },
 736 |         "program": {
 737 |           "ignore_above": 1024,
 738 |           "type": "keyword"
 739 |         },
 740 |         "start": {
 741 |           "type": "date"
 742 |         },
 743 |         "thread": {
 744 |           "properties": {
 745 |             "id": {
 746 |               "type": "long"
 747 |             },
 748 |             "name": {
 749 |               "ignore_above": 1024,
 750 |               "type": "keyword"
 751 |             }
 752 |           }
 753 |         },
 754 |         "title": {
 755 |           "fields": {
 756 |             "text": {
 757 |               "norms": false,
 758 |               "type": "text"
 759 |             }
 760 |           },
 761 |           "ignore_above": 1024,
 762 |           "type": "keyword"
 763 |         },
 764 |         "uptime": {
 765 |           "type": "long"
 766 |         },
 767 |         "working_directory": {
 768 |           "fields": {
 769 |             "text": {
 770 |               "norms": false,
 771 |               "type": "text"
 772 |             }
 773 |           },
 774 |           "ignore_above": 1024,
 775 |           "type": "keyword"
 776 |         }
 777 |       }
 778 |     },
 779 |     "host": {
 780 |       "properties": {
 781 |         "architecture": {
 782 |           "ignore_above": 1024,
 783 |           "type": "keyword"
 784 |         },
 785 |         "containerized": {
 786 |           "type": "boolean"
 787 |         },
 788 |         "domain": {
 789 |           "ignore_above": 1024,
 790 |           "type": "keyword"
 791 |         },
 792 |         "geo": {
 793 |           "properties": {
 794 |             "city_name": {
 795 |               "ignore_above": 1024,
 796 |               "type": "keyword"
 797 |             },
 798 |             "continent_name": {
 799 |               "ignore_above": 1024,
 800 |               "type": "keyword"
 801 |             },
 802 |             "country_iso_code": {
 803 |               "ignore_above": 1024,
 804 |               "type": "keyword"
 805 |             },
 806 |             "country_name": {
 807 |               "ignore_above": 1024,
 808 |               "type": "keyword"
 809 |             },
 810 |             "location": {
 811 |               "type": "geo_point"
 812 |             },
 813 |             "name": {
 814 |               "ignore_above": 1024,
 815 |               "type": "keyword"
 816 |             },
 817 |             "region_iso_code": {
 818 |               "ignore_above": 1024,
 819 |               "type": "keyword"
 820 |             },
 821 |             "region_name": {
 822 |               "ignore_above": 1024,
 823 |               "type": "keyword"
 824 |             }
 825 |           }
 826 |         },
 827 |         "hostname": {
 828 |           "ignore_above": 1024,
 829 |           "type": "keyword"
 830 |         },
 831 |         "id": {
 832 |           "ignore_above": 1024,
 833 |           "type": "keyword"
 834 |         },
 835 |         "ip": {
 836 |           "type": "ip"
 837 |         },
 838 |         "mac": {
 839 |           "ignore_above": 1024,
 840 |           "type": "keyword"
 841 |         },
 842 |         "name": {
 843 |           "ignore_above": 1024,
 844 |           "type": "keyword"
 845 |         },
 846 |         "os": {
 847 |           "properties": {
 848 |             "build": {
 849 |               "ignore_above": 1024,
 850 |               "type": "keyword"
 851 |             },
 852 |             "codename": {
 853 |               "ignore_above": 1024,
 854 |               "type": "keyword"
 855 |             },
 856 |             "family": {
 857 |               "ignore_above": 1024,
 858 |               "type": "keyword"
 859 |             },
 860 |             "full": {
 861 |               "fields": {
 862 |                 "text": {
 863 |                   "norms": false,
 864 |                   "type": "text"
 865 |                 }
 866 |               },
 867 |               "ignore_above": 1024,
 868 |               "type": "keyword"
 869 |             },
 870 |             "kernel": {
 871 |               "ignore_above": 1024,
 872 |               "type": "keyword"
 873 |             },
 874 |             "name": {
 875 |               "fields": {
 876 |                 "text": {
 877 |                   "norms": false,
 878 |                   "type": "text"
 879 |                 }
 880 |               },
 881 |               "ignore_above": 1024,
 882 |               "type": "keyword"
 883 |             },
 884 |             "platform": {
 885 |               "ignore_above": 1024,
 886 |               "type": "keyword"
 887 |             },
 888 |             "version": {
 889 |               "ignore_above": 1024,
 890 |               "type": "keyword"
 891 |             }
 892 |           }
 893 |         },
 894 |         "type": {
 895 |           "ignore_above": 1024,
 896 |           "type": "keyword"
 897 |         },
 898 |         "uptime": {
 899 |           "type": "long"
 900 |         },
 901 |         "user": {
 902 |           "properties": {
 903 |             "domain": {
 904 |               "ignore_above": 1024,
 905 |               "type": "keyword"
 906 |             },
 907 |             "email": {
 908 |               "ignore_above": 1024,
 909 |               "type": "keyword"
 910 |             },
 911 |             "full_name": {
 912 |               "fields": {
 913 |                 "text": {
 914 |                   "norms": false,
 915 |                   "type": "text"
 916 |                 }
 917 |               },
 918 |               "ignore_above": 1024,
 919 |               "type": "keyword"
 920 |             },
 921 |             "group": {
 922 |               "properties": {
 923 |                 "domain": {
 924 |                   "ignore_above": 1024,
 925 |                   "type": "keyword"
 926 |                 },
 927 |                 "id": {
 928 |                   "ignore_above": 1024,
 929 |                   "type": "keyword"
 930 |                 },
 931 |                 "name": {
 932 |                   "ignore_above": 1024,
 933 |                   "type": "keyword"
 934 |                 }
 935 |               }
 936 |             },
 937 |             "hash": {
 938 |               "ignore_above": 1024,
 939 |               "type": "keyword"
 940 |             },
 941 |             "id": {
 942 |               "ignore_above": 1024,
 943 |               "type": "keyword"
 944 |             },
 945 |             "name": {
 946 |               "fields": {
 947 |                 "text": {
 948 |                   "norms": false,
 949 |                   "type": "text"
 950 |                 }
 951 |               },
 952 |               "ignore_above": 1024,
 953 |               "type": "keyword"
 954 |             }
 955 |           }
 956 |         }
 957 |       }
 958 |     },
 959 |     "x509": {
 960 |       "properties": {
 961 |         "basic_constraints": {
 962 |           "properties": {
 963 |             "certificate_authority": {
 964 |               "type": "boolean"
 965 |             },
 966 |             "path_length": {
 967 |               "type": "long"
 968 |             }
 969 |           }
 970 |         },
 971 |         "certificate": {
 972 |           "properties": {
 973 |             "common_name": {
 974 |               "ignore_above": 1024,
 975 |               "type": "keyword"
 976 |             },
 977 |             "curve": {
 978 |               "ignore_above": 1024,
 979 |               "type": "keyword"
 980 |             },
 981 |             "exponent": {
 982 |               "ignore_above": 1024,
 983 |               "type": "keyword"
 984 |             },
 985 |             "issuer": {
 986 |               "properties": {
 987 |                 "common_name": {
 988 |                   "ignore_above": 1024,
 989 |                   "type": "keyword"
 990 |                 },
 991 |                 "country": {
 992 |                   "ignore_above": 1024,
 993 |                   "type": "keyword"
 994 |                 },
 995 |                 "locality": {
 996 |                   "ignore_above": 1024,
 997 |                   "type": "keyword"
 998 |                 },
 999 |                 "organization": {
1000 |                   "ignore_above": 1024,
1001 |                   "type": "keyword"
1002 |                 },
1003 |                 "organizational_unit": {
1004 |                   "ignore_above": 1024,
1005 |                   "type": "keyword"
1006 |                 },
1007 |                 "state": {
1008 |                   "ignore_above": 1024,
1009 |                   "type": "keyword"
1010 |                 }
1011 |               }
1012 |             },
1013 |             "key": {
1014 |               "properties": {
1015 |                 "algorithm": {
1016 |                   "ignore_above": 1024,
1017 |                   "type": "keyword"
1018 |                 },
1019 |                 "length": {
1020 |                   "type": "long"
1021 |                 },
1022 |                 "type": {
1023 |                   "ignore_above": 1024,
1024 |                   "type": "keyword"
1025 |                 }
1026 |               }
1027 |             },
1028 |             "serial": {
1029 |               "ignore_above": 1024,
1030 |               "type": "keyword"
1031 |             },
1032 |             "signature_algorithm": {
1033 |               "ignore_above": 1024,
1034 |               "type": "keyword"
1035 |             },
1036 |             "subject": {
1037 |               "properties": {
1038 |                 "common_name": {
1039 |                   "ignore_above": 1024,
1040 |                   "type": "keyword"
1041 |                 },
1042 |                 "country": {
1043 |                   "ignore_above": 1024,
1044 |                   "type": "keyword"
1045 |                 },
1046 |                 "locality": {
1047 |                   "ignore_above": 1024,
1048 |                   "type": "keyword"
1049 |                 },
1050 |                 "organization": {
1051 |                   "ignore_above": 1024,
1052 |                   "type": "keyword"
1053 |                 },
1054 |                 "organizational_unit": {
1055 |                   "ignore_above": 1024,
1056 |                   "type": "keyword"
1057 |                 },
1058 |                 "state": {
1059 |                   "ignore_above": 1024,
1060 |                   "type": "keyword"
1061 |                 }
1062 |               }
1063 |             },
1064 |             "valid": {
1065 |               "properties": {
1066 |                 "from": {
1067 |                   "type": "date"
1068 |                 },
1069 |                 "until": {
1070 |                   "type": "date"
1071 |                 }
1072 |               }
1073 |             },
1074 |             "version": {
1075 |               "type": "long"
1076 |             }
1077 |           }
1078 |         },
1079 |         "id": {
1080 |           "ignore_above": 1024,
1081 |           "type": "keyword"
1082 |         },
1083 |         "log_cert": {
1084 |           "type": "boolean"
1085 |         },
1086 |         "san": {
1087 |           "properties": {
1088 |             "dns": {
1089 |               "ignore_above": 1024,
1090 |               "type": "keyword"
1091 |             },
1092 |             "email": {
1093 |               "ignore_above": 1024,
1094 |               "type": "keyword"
1095 |             },
1096 |             "ip": {
1097 |               "type": "ip"
1098 |             },
1099 |             "other_fields": {
1100 |               "type": "boolean"
1101 |             },
1102 |             "uri": {
1103 |               "ignore_above": 1024,
1104 |               "type": "keyword"
1105 |             }
1106 |           }
1107 |         }
1108 |       }
1109 |     },
1110 |     "pe": {
1111 |       "properties": {
1112 |         "company": {
1113 |           "ignore_above": 1024,
1114 |           "type": "keyword"
1115 |         },
1116 |         "description": {
1117 |           "ignore_above": 1024,
1118 |           "type": "keyword"
1119 |         },
1120 |         "file_version": {
1121 |           "ignore_above": 1024,
1122 |           "type": "keyword"
1123 |         },
1124 |         "original_file_name": {
1125 |           "ignore_above": 1024,
1126 |           "type": "keyword"
1127 |         },
1128 |         "product": {
1129 |           "ignore_above": 1024,
1130 |           "type": "keyword"
1131 |         }
1132 |       }
1133 |     },
1134 |     "url": {
1135 |       "properties": {
1136 |         "domain": {
1137 |           "ignore_above": 1024,
1138 |           "type": "keyword"
1139 |         },
1140 |         "extension": {
1141 |           "ignore_above": 1024,
1142 |           "type": "keyword"
1143 |         },
1144 |         "fragment": {
1145 |           "ignore_above": 1024,
1146 |           "type": "keyword"
1147 |         },
1148 |         "full": {
1149 |           "fields": {
1150 |             "text": {
1151 |               "norms": false,
1152 |               "type": "text"
1153 |             }
1154 |           },
1155 |           "ignore_above": 1024,
1156 |           "type": "keyword"
1157 |         },
1158 |         "original": {
1159 |           "fields": {
1160 |             "text": {
1161 |               "norms": false,
1162 |               "type": "text"
1163 |             }
1164 |           },
1165 |           "ignore_above": 1024,
1166 |           "type": "keyword"
1167 |         },
1168 |         "password": {
1169 |           "ignore_above": 1024,
1170 |           "type": "keyword"
1171 |         },
1172 |         "path": {
1173 |           "ignore_above": 1024,
1174 |           "type": "keyword"
1175 |         },
1176 |         "port": {
1177 |           "type": "long"
1178 |         },
1179 |         "query": {
1180 |           "ignore_above": 1024,
1181 |           "type": "keyword"
1182 |         },
1183 |         "registered_domain": {
1184 |           "ignore_above": 1024,
1185 |           "type": "keyword"
1186 |         },
1187 |         "scheme": {
1188 |           "ignore_above": 1024,
1189 |           "type": "keyword"
1190 |         },
1191 |         "top_level_domain": {
1192 |           "ignore_above": 1024,
1193 |           "type": "keyword"
1194 |         },
1195 |         "username": {
1196 |           "ignore_above": 1024,
1197 |           "type": "keyword"
1198 |         }
1199 |       }
1200 |     }
1201 |   }
1202 | }


--------------------------------------------------------------------------------
/LICENSE.md:
--------------------------------------------------------------------------------
  1 | GNU GENERAL PUBLIC LICENSE
  2 |    Version 3, 29 June 2007
  3 | 
  4 | Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
  5 | Everyone is permitted to copy and distribute verbatim copies
  6 | of this license document, but changing it is not allowed.
  7 | 
  8 |         Preamble
  9 | 
 10 | The GNU General Public License is a free, copyleft license for
 11 | software and other kinds of works.
 12 | 
 13 | The licenses for most software and other practical works are designed
 14 | to take away your freedom to share and change the works.  By contrast,
 15 | the GNU General Public License is intended to guarantee your freedom to
 16 | share and change all versions of a program--to make sure it remains free
 17 | software for all its users.  We, the Free Software Foundation, use the
 18 | GNU General Public License for most of our software; it applies also to
 19 | any other work released this way by its authors.  You can apply it to
 20 | your programs, too.
 21 | 
 22 | When we speak of free software, we are referring to freedom, not
 23 | price.  Our General Public Licenses are designed to make sure that you
 24 | have the freedom to distribute copies of free software (and charge for
 25 | them if you wish), that you receive source code or can get it if you
 26 | want it, that you can change the software or use pieces of it in new
 27 | free programs, and that you know you can do these things.
 28 | 
 29 | To protect your rights, we need to prevent others from denying you
 30 | these rights or asking you to surrender the rights.  Therefore, you have
 31 | certain responsibilities if you distribute copies of the software, or if
 32 | you modify it: responsibilities to respect the freedom of others.
 33 | 
 34 | For example, if you distribute copies of such a program, whether
 35 | gratis or for a fee, you must pass on to the recipients the same
 36 | freedoms that you received.  You must make sure that they, too, receive
 37 | or can get the source code.  And you must show them these terms so they
 38 | know their rights.
 39 | 
 40 | Developers that use the GNU GPL protect your rights with two steps:
 41 | (1) assert copyright on the software, and (2) offer you this License
 42 | giving you legal permission to copy, distribute and/or modify it.
 43 | 
 44 | For the developers' and authors' protection, the GPL clearly explains
 45 | that there is no warranty for this free software.  For both users' and
 46 | authors' sake, the GPL requires that modified versions be marked as
 47 | changed, so that their problems will not be attributed erroneously to
 48 | authors of previous versions.
 49 | 
 50 | Some devices are designed to deny users access to install or run
 51 | modified versions of the software inside them, although the manufacturer
 52 | can do so.  This is fundamentally incompatible with the aim of
 53 | protecting users' freedom to change the software.  The systematic
 54 | pattern of such abuse occurs in the area of products for individuals to
 55 | use, which is precisely where it is most unacceptable.  Therefore, we
 56 | have designed this version of the GPL to prohibit the practice for those
 57 | products.  If such problems arise substantially in other domains, we
 58 | stand ready to extend this provision to those domains in future versions
 59 | of the GPL, as needed to protect the freedom of users.
 60 | 
 61 | Finally, every program is threatened constantly by software patents.
 62 | States should not allow patents to restrict development and use of
 63 | software on general-purpose computers, but in those that do, we wish to
 64 | avoid the special danger that patents applied to a free program could
 65 | make it effectively proprietary.  To prevent this, the GPL assures that
 66 | patents cannot be used to render the program non-free.
 67 | 
 68 | The precise terms and conditions for copying, distribution and
 69 | modification follow.
 70 | 
 71 |    TERMS AND CONDITIONS
 72 | 
 73 | 0. Definitions.
 74 | 
 75 | "This License" refers to version 3 of the GNU General Public License.
 76 | 
 77 | "Copyright" also means copyright-like laws that apply to other kinds of
 78 | works, such as semiconductor masks.
 79 | 
 80 | "The Program" refers to any copyrightable work licensed under this
 81 | License.  Each licensee is addressed as "you".  "Licensees" and
 82 | "recipients" may be individuals or organizations.
 83 | 
 84 | To "modify" a work means to copy from or adapt all or part of the work
 85 | in a fashion requiring copyright permission, other than the making of an
 86 | exact copy.  The resulting work is called a "modified version" of the
 87 | earlier work or a work "based on" the earlier work.
 88 | 
 89 | A "covered work" means either the unmodified Program or a work based
 90 | on the Program.
 91 | 
 92 | To "propagate" a work means to do anything with it that, without
 93 | permission, would make you directly or secondarily liable for
 94 | infringement under applicable copyright law, except executing it on a
 95 | computer or modifying a private copy.  Propagation includes copying,
 96 | distribution (with or without modification), making available to the
 97 | public, and in some countries other activities as well.
 98 | 
 99 | To "convey" a work means any kind of propagation that enables other
100 | parties to make or receive copies.  Mere interaction with a user through
101 | a computer network, with no transfer of a copy, is not conveying.
102 | 
103 | An interactive user interface displays "Appropriate Legal Notices"
104 | to the extent that it includes a convenient and prominently visible
105 | feature that (1) displays an appropriate copyright notice, and (2)
106 | tells the user that there is no warranty for the work (except to the
107 | extent that warranties are provided), that licensees may convey the
108 | work under this License, and how to view a copy of this License.  If
109 | the interface presents a list of user commands or options, such as a
110 | menu, a prominent item in the list meets this criterion.
111 | 
112 | 1. Source Code.
113 | 
114 | The "source code" for a work means the preferred form of the work
115 | for making modifications to it.  "Object code" means any non-source
116 | form of a work.
117 | 
118 | A "Standard Interface" means an interface that either is an official
119 | standard defined by a recognized standards body, or, in the case of
120 | interfaces specified for a particular programming language, one that
121 | is widely used among developers working in that language.
122 | 
123 | The "System Libraries" of an executable work include anything, other
124 | than the work as a whole, that (a) is included in the normal form of
125 | packaging a Major Component, but which is not part of that Major
126 | Component, and (b) serves only to enable use of the work with that
127 | Major Component, or to implement a Standard Interface for which an
128 | implementation is available to the public in source code form.  A
129 | "Major Component", in this context, means a major essential component
130 | (kernel, window system, and so on) of the specific operating system
131 | (if any) on which the executable work runs, or a compiler used to
132 | produce the work, or an object code interpreter used to run it.
133 | 
134 | The "Corresponding Source" for a work in object code form means all
135 | the source code needed to generate, install, and (for an executable
136 | work) run the object code and to modify the work, including scripts to
137 | control those activities.  However, it does not include the work's
138 | System Libraries, or general-purpose tools or generally available free
139 | programs which are used unmodified in performing those activities but
140 | which are not part of the work.  For example, Corresponding Source
141 | includes interface definition files associated with source files for
142 | the work, and the source code for shared libraries and dynamically
143 | linked subprograms that the work is specifically designed to require,
144 | such as by intimate data communication or control flow between those
145 | subprograms and other parts of the work.
146 | 
147 | The Corresponding Source need not include anything that users
148 | can regenerate automatically from other parts of the Corresponding
149 | Source.
150 | 
151 | The Corresponding Source for a work in source code form is that
152 | same work.
153 | 
154 | 2. Basic Permissions.
155 | 
156 | All rights granted under this License are granted for the term of
157 | copyright on the Program, and are irrevocable provided the stated
158 | conditions are met.  This License explicitly affirms your unlimited
159 | permission to run the unmodified Program.  The output from running a
160 | covered work is covered by this License only if the output, given its
161 | content, constitutes a covered work.  This License acknowledges your
162 | rights of fair use or other equivalent, as provided by copyright law.
163 | 
164 | You may make, run and propagate covered works that you do not
165 | convey, without conditions so long as your license otherwise remains
166 | in force.  You may convey covered works to others for the sole purpose
167 | of having them make modifications exclusively for you, or provide you
168 | with facilities for running those works, provided that you comply with
169 | the terms of this License in conveying all material for which you do
170 | not control copyright.  Those thus making or running the covered works
171 | for you must do so exclusively on your behalf, under your direction
172 | and control, on terms that prohibit them from making any copies of
173 | your copyrighted material outside their relationship with you.
174 | 
175 | Conveying under any other circumstances is permitted solely under
176 | the conditions stated below.  Sublicensing is not allowed; section 10
177 | makes it unnecessary.
178 | 
179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law.
180 | 
181 | No covered work shall be deemed part of an effective technological
182 | measure under any applicable law fulfilling obligations under article
183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or
184 | similar laws prohibiting or restricting circumvention of such
185 | measures.
186 | 
187 | When you convey a covered work, you waive any legal power to forbid
188 | circumvention of technological measures to the extent such circumvention
189 | is effected by exercising rights under this License with respect to
190 | the covered work, and you disclaim any intention to limit operation or
191 | modification of the work as a means of enforcing, against the work's
192 | users, your or third parties' legal rights to forbid circumvention of
193 | technological measures.
194 | 
195 | 4. Conveying Verbatim Copies.
196 | 
197 | You may convey verbatim copies of the Program's source code as you
198 | receive it, in any medium, provided that you conspicuously and
199 | appropriately publish on each copy an appropriate copyright notice;
200 | keep intact all notices stating that this License and any
201 | non-permissive terms added in accord with section 7 apply to the code;
202 | keep intact all notices of the absence of any warranty; and give all
203 | recipients a copy of this License along with the Program.
204 | 
205 | You may charge any price or no price for each copy that you convey,
206 | and you may offer support or warranty protection for a fee.
207 | 
208 | 5. Conveying Modified Source Versions.
209 | 
210 | You may convey a work based on the Program, or the modifications to
211 | produce it from the Program, in the form of source code under the
212 | terms of section 4, provided that you also meet all of these conditions:
213 | 
214 | a) The work must carry prominent notices stating that you modified
215 | it, and giving a relevant date.
216 | 
217 | b) The work must carry prominent notices stating that it is
218 | released under this License and any conditions added under section
219 | 7.  This requirement modifies the requirement in section 4 to
220 | "keep intact all notices".
221 | 
222 | c) You must license the entire work, as a whole, under this
223 | License to anyone who comes into possession of a copy.  This
224 | License will therefore apply, along with any applicable section 7
225 | additional terms, to the whole of the work, and all its parts,
226 | regardless of how they are packaged.  This License gives no
227 | permission to license the work in any other way, but it does not
228 | invalidate such permission if you have separately received it.
229 | 
230 | d) If the work has interactive user interfaces, each must display
231 | Appropriate Legal Notices; however, if the Program has interactive
232 | interfaces that do not display Appropriate Legal Notices, your
233 | work need not make them do so.
234 | 
235 | A compilation of a covered work with other separate and independent
236 | works, which are not by their nature extensions of the covered work,
237 | and which are not combined with it such as to form a larger program,
238 | in or on a volume of a storage or distribution medium, is called an
239 | "aggregate" if the compilation and its resulting copyright are not
240 | used to limit the access or legal rights of the compilation's users
241 | beyond what the individual works permit.  Inclusion of a covered work
242 | in an aggregate does not cause this License to apply to the other
243 | parts of the aggregate.
244 | 
245 | 6. Conveying Non-Source Forms.
246 | 
247 | You may convey a covered work in object code form under the terms
248 | of sections 4 and 5, provided that you also convey the
249 | machine-readable Corresponding Source under the terms of this License,
250 | in one of these ways:
251 | 
252 | a) Convey the object code in, or embodied in, a physical product
253 | (including a physical distribution medium), accompanied by the
254 | Corresponding Source fixed on a durable physical medium
255 | customarily used for software interchange.
256 | 
257 | b) Convey the object code in, or embodied in, a physical product
258 | (including a physical distribution medium), accompanied by a
259 | written offer, valid for at least three years and valid for as
260 | long as you offer spare parts or customer support for that product
261 | model, to give anyone who possesses the object code either (1) a
262 | copy of the Corresponding Source for all the software in the
263 | product that is covered by this License, on a durable physical
264 | medium customarily used for software interchange, for a price no
265 | more than your reasonable cost of physically performing this
266 | conveying of source, or (2) access to copy the
267 | Corresponding Source from a network server at no charge.
268 | 
269 | c) Convey individual copies of the object code with a copy of the
270 | written offer to provide the Corresponding Source.  This
271 | alternative is allowed only occasionally and noncommercially, and
272 | only if you received the object code with such an offer, in accord
273 | with subsection 6b.
274 | 
275 | d) Convey the object code by offering access from a designated
276 | place (gratis or for a charge), and offer equivalent access to the
277 | Corresponding Source in the same way through the same place at no
278 | further charge.  You need not require recipients to copy the
279 | Corresponding Source along with the object code.  If the place to
280 | copy the object code is a network server, the Corresponding Source
281 | may be on a different server (operated by you or a third party)
282 | that supports equivalent copying facilities, provided you maintain
283 | clear directions next to the object code saying where to find the
284 | Corresponding Source.  Regardless of what server hosts the
285 | Corresponding Source, you remain obligated to ensure that it is
286 | available for as long as needed to satisfy these requirements.
287 | 
288 | e) Convey the object code using peer-to-peer transmission, provided
289 | you inform other peers where the object code and Corresponding
290 | Source of the work are being offered to the general public at no
291 | charge under subsection 6d.
292 | 
293 | A separable portion of the object code, whose source code is excluded
294 | from the Corresponding Source as a System Library, need not be
295 | included in conveying the object code work.
296 | 
297 | A "User Product" is either (1) a "consumer product", which means any
298 | tangible personal property which is normally used for personal, family,
299 | or household purposes, or (2) anything designed or sold for incorporation
300 | into a dwelling.  In determining whether a product is a consumer product,
301 | doubtful cases shall be resolved in favor of coverage.  For a particular
302 | product received by a particular user, "normally used" refers to a
303 | typical or common use of that class of product, regardless of the status
304 | of the particular user or of the way in which the particular user
305 | actually uses, or expects or is expected to use, the product.  A product
306 | is a consumer product regardless of whether the product has substantial
307 | commercial, industrial or non-consumer uses, unless such uses represent
308 | the only significant mode of use of the product.
309 | 
310 | "Installation Information" for a User Product means any methods,
311 | procedures, authorization keys, or other information required to install
312 | and execute modified versions of a covered work in that User Product from
313 | a modified version of its Corresponding Source.  The information must
314 | suffice to ensure that the continued functioning of the modified object
315 | code is in no case prevented or interfered with solely because
316 | modification has been made.
317 | 
318 | If you convey an object code work under this section in, or with, or
319 | specifically for use in, a User Product, and the conveying occurs as
320 | part of a transaction in which the right of possession and use of the
321 | User Product is transferred to the recipient in perpetuity or for a
322 | fixed term (regardless of how the transaction is characterized), the
323 | Corresponding Source conveyed under this section must be accompanied
324 | by the Installation Information.  But this requirement does not apply
325 | if neither you nor any third party retains the ability to install
326 | modified object code on the User Product (for example, the work has
327 | been installed in ROM).
328 | 
329 | The requirement to provide Installation Information does not include a
330 | requirement to continue to provide support service, warranty, or updates
331 | for a work that has been modified or installed by the recipient, or for
332 | the User Product in which it has been modified or installed.  Access to a
333 | network may be denied when the modification itself materially and
334 | adversely affects the operation of the network or violates the rules and
335 | protocols for communication across the network.
336 | 
337 | Corresponding Source conveyed, and Installation Information provided,
338 | in accord with this section must be in a format that is publicly
339 | documented (and with an implementation available to the public in
340 | source code form), and must require no special password or key for
341 | unpacking, reading or copying.
342 | 
343 | 7. Additional Terms.
344 | 
345 | "Additional permissions" are terms that supplement the terms of this
346 | License by making exceptions from one or more of its conditions.
347 | Additional permissions that are applicable to the entire Program shall
348 | be treated as though they were included in this License, to the extent
349 | that they are valid under applicable law.  If additional permissions
350 | apply only to part of the Program, that part may be used separately
351 | under those permissions, but the entire Program remains governed by
352 | this License without regard to the additional permissions.
353 | 
354 | When you convey a copy of a covered work, you may at your option
355 | remove any additional permissions from that copy, or from any part of
356 | it.  (Additional permissions may be written to require their own
357 | removal in certain cases when you modify the work.)  You may place
358 | additional permissions on material, added by you to a covered work,
359 | for which you have or can give appropriate copyright permission.
360 | 
361 | Notwithstanding any other provision of this License, for material you
362 | add to a covered work, you may (if authorized by the copyright holders of
363 | that material) supplement the terms of this License with terms:
364 | 
365 | a) Disclaiming warranty or limiting liability differently from the
366 | terms of sections 15 and 16 of this License; or
367 | 
368 | b) Requiring preservation of specified reasonable legal notices or
369 | author attributions in that material or in the Appropriate Legal
370 | Notices displayed by works containing it; or
371 | 
372 | c) Prohibiting misrepresentation of the origin of that material, or
373 | requiring that modified versions of such material be marked in
374 | reasonable ways as different from the original version; or
375 | 
376 | d) Limiting the use for publicity purposes of names of licensors or
377 | authors of the material; or
378 | 
379 | e) Declining to grant rights under trademark law for use of some
380 | trade names, trademarks, or service marks; or
381 | 
382 | f) Requiring indemnification of licensors and authors of that
383 | material by anyone who conveys the material (or modified versions of
384 | it) with contractual assumptions of liability to the recipient, for
385 | any liability that these contractual assumptions directly impose on
386 | those licensors and authors.
387 | 
388 | All other non-permissive additional terms are considered "further
389 | restrictions" within the meaning of section 10.  If the Program as you
390 | received it, or any part of it, contains a notice stating that it is
391 | governed by this License along with a term that is a further
392 | restriction, you may remove that term.  If a license document contains
393 | a further restriction but permits relicensing or conveying under this
394 | License, you may add to a covered work material governed by the terms
395 | of that license document, provided that the further restriction does
396 | not survive such relicensing or conveying.
397 | 
398 | If you add terms to a covered work in accord with this section, you
399 | must place, in the relevant source files, a statement of the
400 | additional terms that apply to those files, or a notice indicating
401 | where to find the applicable terms.
402 | 
403 | Additional terms, permissive or non-permissive, may be stated in the
404 | form of a separately written license, or stated as exceptions;
405 | the above requirements apply either way.
406 | 
407 | 8. Termination.
408 | 
409 | You may not propagate or modify a covered work except as expressly
410 | provided under this License.  Any attempt otherwise to propagate or
411 | modify it is void, and will automatically terminate your rights under
412 | this License (including any patent licenses granted under the third
413 | paragraph of section 11).
414 | 
415 | However, if you cease all violation of this License, then your
416 | license from a particular copyright holder is reinstated (a)
417 | provisionally, unless and until the copyright holder explicitly and
418 | finally terminates your license, and (b) permanently, if the copyright
419 | holder fails to notify you of the violation by some reasonable means
420 | prior to 60 days after the cessation.
421 | 
422 | Moreover, your license from a particular copyright holder is
423 | reinstated permanently if the copyright holder notifies you of the
424 | violation by some reasonable means, this is the first time you have
425 | received notice of violation of this License (for any work) from that
426 | copyright holder, and you cure the violation prior to 30 days after
427 | your receipt of the notice.
428 | 
429 | Termination of your rights under this section does not terminate the
430 | licenses of parties who have received copies or rights from you under
431 | this License.  If your rights have been terminated and not permanently
432 | reinstated, you do not qualify to receive new licenses for the same
433 | material under section 10.
434 | 
435 | 9. Acceptance Not Required for Having Copies.
436 | 
437 | You are not required to accept this License in order to receive or
438 | run a copy of the Program.  Ancillary propagation of a covered work
439 | occurring solely as a consequence of using peer-to-peer transmission
440 | to receive a copy likewise does not require acceptance.  However,
441 | nothing other than this License grants you permission to propagate or
442 | modify any covered work.  These actions infringe copyright if you do
443 | not accept this License.  Therefore, by modifying or propagating a
444 | covered work, you indicate your acceptance of this License to do so.
445 | 
446 | 10. Automatic Licensing of Downstream Recipients.
447 | 
448 | Each time you convey a covered work, the recipient automatically
449 | receives a license from the original licensors, to run, modify and
450 | propagate that work, subject to this License.  You are not responsible
451 | for enforcing compliance by third parties with this License.
452 | 
453 | An "entity transaction" is a transaction transferring control of an
454 | organization, or substantially all assets of one, or subdividing an
455 | organization, or merging organizations.  If propagation of a covered
456 | work results from an entity transaction, each party to that
457 | transaction who receives a copy of the work also receives whatever
458 | licenses to the work the party's predecessor in interest had or could
459 | give under the previous paragraph, plus a right to possession of the
460 | Corresponding Source of the work from the predecessor in interest, if
461 | the predecessor has it or can get it with reasonable efforts.
462 | 
463 | You may not impose any further restrictions on the exercise of the
464 | rights granted or affirmed under this License.  For example, you may
465 | not impose a license fee, royalty, or other charge for exercise of
466 | rights granted under this License, and you may not initiate litigation
467 | (including a cross-claim or counterclaim in a lawsuit) alleging that
468 | any patent claim is infringed by making, using, selling, offering for
469 | sale, or importing the Program or any portion of it.
470 | 
471 | 11. Patents.
472 | 
473 | A "contributor" is a copyright holder who authorizes use under this
474 | License of the Program or a work on which the Program is based.  The
475 | work thus licensed is called the contributor's "contributor version".
476 | 
477 | A contributor's "essential patent claims" are all patent claims
478 | owned or controlled by the contributor, whether already acquired or
479 | hereafter acquired, that would be infringed by some manner, permitted
480 | by this License, of making, using, or selling its contributor version,
481 | but do not include claims that would be infringed only as a
482 | consequence of further modification of the contributor version.  For
483 | purposes of this definition, "control" includes the right to grant
484 | patent sublicenses in a manner consistent with the requirements of
485 | this License.
486 | 
487 | Each contributor grants you a non-exclusive, worldwide, royalty-free
488 | patent license under the contributor's essential patent claims, to
489 | make, use, sell, offer for sale, import and otherwise run, modify and
490 | propagate the contents of its contributor version.
491 | 
492 | In the following three paragraphs, a "patent license" is any express
493 | agreement or commitment, however denominated, not to enforce a patent
494 | (such as an express permission to practice a patent or covenant not to
495 | sue for patent infringement).  To "grant" such a patent license to a
496 | party means to make such an agreement or commitment not to enforce a
497 | patent against the party.
498 | 
499 | If you convey a covered work, knowingly relying on a patent license,
500 | and the Corresponding Source of the work is not available for anyone
501 | to copy, free of charge and under the terms of this License, through a
502 | publicly available network server or other readily accessible means,
503 | then you must either (1) cause the Corresponding Source to be so
504 | available, or (2) arrange to deprive yourself of the benefit of the
505 | patent license for this particular work, or (3) arrange, in a manner
506 | consistent with the requirements of this License, to extend the patent
507 | license to downstream recipients.  "Knowingly relying" means you have
508 | actual knowledge that, but for the patent license, your conveying the
509 | covered work in a country, or your recipient's use of the covered work
510 | in a country, would infringe one or more identifiable patents in that
511 | country that you have reason to believe are valid.
512 | 
513 | If, pursuant to or in connection with a single transaction or
514 | arrangement, you convey, or propagate by procuring conveyance of, a
515 | covered work, and grant a patent license to some of the parties
516 | receiving the covered work authorizing them to use, propagate, modify
517 | or convey a specific copy of the covered work, then the patent license
518 | you grant is automatically extended to all recipients of the covered
519 | work and works based on it.
520 | 
521 | A patent license is "discriminatory" if it does not include within
522 | the scope of its coverage, prohibits the exercise of, or is
523 | conditioned on the non-exercise of one or more of the rights that are
524 | specifically granted under this License.  You may not convey a covered
525 | work if you are a party to an arrangement with a third party that is
526 | in the business of distributing software, under which you make payment
527 | to the third party based on the extent of your activity of conveying
528 | the work, and under which the third party grants, to any of the
529 | parties who would receive the covered work from you, a discriminatory
530 | patent license (a) in connection with copies of the covered work
531 | conveyed by you (or copies made from those copies), or (b) primarily
532 | for and in connection with specific products or compilations that
533 | contain the covered work, unless you entered into that arrangement,
534 | or that patent license was granted, prior to 28 March 2007.
535 | 
536 | Nothing in this License shall be construed as excluding or limiting
537 | any implied license or other defenses to infringement that may
538 | otherwise be available to you under applicable patent law.
539 | 
540 | 12. No Surrender of Others' Freedom.
541 | 
542 | If conditions are imposed on you (whether by court order, agreement or
543 | otherwise) that contradict the conditions of this License, they do not
544 | excuse you from the conditions of this License.  If you cannot convey a
545 | covered work so as to satisfy simultaneously your obligations under this
546 | License and any other pertinent obligations, then as a consequence you may
547 | not convey it at all.  For example, if you agree to terms that obligate you
548 | to collect a royalty for further conveying from those to whom you convey
549 | the Program, the only way you could satisfy both those terms and this
550 | License would be to refrain entirely from conveying the Program.
551 | 
552 | 13. Use with the GNU Affero General Public License.
553 | 
554 | Notwithstanding any other provision of this License, you have
555 | permission to link or combine any covered work with a work licensed
556 | under version 3 of the GNU Affero General Public License into a single
557 | combined work, and to convey the resulting work.  The terms of this
558 | License will continue to apply to the part which is the covered work,
559 | but the special requirements of the GNU Affero General Public License,
560 | section 13, concerning interaction through a network will apply to the
561 | combination as such.
562 | 
563 | 14. Revised Versions of this License.
564 | 
565 | The Free Software Foundation may publish revised and/or new versions of
566 | the GNU General Public License from time to time.  Such new versions will
567 | be similar in spirit to the present version, but may differ in detail to
568 | address new problems or concerns.
569 | 
570 | Each version is given a distinguishing version number.  If the
571 | Program specifies that a certain numbered version of the GNU General
572 | Public License "or any later version" applies to it, you have the
573 | option of following the terms and conditions either of that numbered
574 | version or of any later version published by the Free Software
575 | Foundation.  If the Program does not specify a version number of the
576 | GNU General Public License, you may choose any version ever published
577 | by the Free Software Foundation.
578 | 
579 | If the Program specifies that a proxy can decide which future
580 | versions of the GNU General Public License can be used, that proxy's
581 | public statement of acceptance of a version permanently authorizes you
582 | to choose that version for the Program.
583 | 
584 | Later license versions may give you additional or different
585 | permissions.  However, no additional obligations are imposed on any
586 | author or copyright holder as a result of your choosing to follow a
587 | later version.
588 | 
589 | 15. Disclaimer of Warranty.
590 | 
591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
592 | APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
596 | PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
597 | IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
599 | 
600 | 16. Limitation of Liability.
601 | 
602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
610 | SUCH DAMAGES.
611 | 
612 | 17. Interpretation of Sections 15 and 16.
613 | 
614 | If the disclaimer of warranty and limitation of liability provided
615 | above cannot be given local legal effect according to their terms,
616 | reviewing courts shall apply local law that most closely approximates
617 | an absolute waiver of all civil liability in connection with the
618 | Program, unless a warranty or assumption of liability accompanies a
619 | copy of the Program in return for a fee.
620 | 
621 |  END OF TERMS AND CONDITIONS
622 | 
623 | How to Apply These Terms to Your New Programs
624 | 
625 | If you develop a new program, and you want it to be of the greatest
626 | possible use to the public, the best way to achieve this is to make it
627 | free software which everyone can redistribute and change under these terms.
628 | 
629 | To do so, attach the following notices to the program.  It is safest
630 | to attach them to the start of each source file to most effectively
631 | state the exclusion of warranty; and each file should have at least
632 | the "copyright" line and a pointer to where the full notice is found.
633 | 
634 | <one line to give the program's name and a brief idea of what it does.>
635 | Copyright (C) <year>  <name of author>
636 | 
637 | This program is free software: you can redistribute it and/or modify
638 | it under the terms of the GNU General Public License as published by
639 | the Free Software Foundation, either version 3 of the License, or
640 | (at your option) any later version.
641 | 
642 | This program is distributed in the hope that it will be useful,
643 | but WITHOUT ANY WARRANTY; without even the implied warranty of
644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
645 | GNU General Public License for more details.
646 | 
647 | You should have received a copy of the GNU General Public License
648 | along with this program.  If not, see <https://www.gnu.org/licenses/>.
649 | 
650 | Also add information on how to contact you by electronic and paper mail.
651 | 
652 | If the program does terminal interaction, make it output a short
653 | notice like this when it starts in an interactive mode:
654 | 
655 | <program>  Copyright (C) <year>  <name of author>
656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
657 | This is free software, and you are welcome to redistribute it
658 | under certain conditions; type `show c' for details.
659 | 
660 | The hypothetical commands `show w' and `show c' should show the appropriate
661 | parts of the General Public License.  Of course, your program's commands
662 | might be different; for a GUI interface, you would use an "about box".
663 | 
664 | You should also get your employer (if you work as a programmer) or school,
665 | if any, to sign a "copyright disclaimer" for the program, if necessary.
666 | For more information on this, and how to apply and follow the GNU GPL, see
667 | <https://www.gnu.org/licenses/>.
668 | 
669 | The GNU General Public License does not permit incorporating your program
670 | into proprietary programs.  If your program is a subroutine library, you
671 | may consider it more useful to permit linking proprietary applications with
672 | the library.  If this is what you want to do, use the GNU Lesser General
673 | Public License instead of this License.  But first, please read
674 | <https://www.gnu.org/licenses/why-not-lgpl.html>.
675 | 


--------------------------------------------------------------------------------