├── .gitignore ├── .readthedocs.yml ├── CHANGELOG.rst ├── LICENSE.txt ├── README.md ├── __init__.py ├── apiosintDS ├── __init__.py ├── __pycache__ │ ├── __init__.cpython-310.pyc │ └── apiosintDS.cpython-310.pyc ├── apiosintDS.py ├── modules │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-310.pyc │ │ ├── dosearch.cpython-310.pyc │ │ ├── listutils.cpython-310.pyc │ │ ├── scriptinfo.cpython-310.pyc │ │ └── stixreport.cpython-310.pyc │ ├── dosearch.py │ ├── listutils.py │ ├── scriptinfo.py │ └── stixreport.py ├── schema │ └── schema.json └── utilities │ ├── __init__.py │ ├── __pycache__ │ ├── __init__.cpython-310.pyc │ ├── logutils.cpython-310.pyc │ └── prettycli.cpython-310.pyc │ ├── logutils.py │ └── prettycli.py ├── docs ├── Makefile ├── _static │ ├── css │ │ └── custom.css │ └── img │ │ ├── apiosintDS.png │ │ ├── mispmoduleconfiguration.png │ │ ├── moduleenrich.png │ │ └── modulehover.png ├── make.bat ├── requirements.txt └── source │ ├── _toc.yml │ ├── changes.rst │ ├── cliexamples.rst │ ├── conf.py │ ├── index.rst │ ├── install.rst │ ├── license.rst │ ├── userguideapi.rst │ ├── userguidecli.rst │ └── userguidemisp.rst ├── requirements.txt ├── setup.cfg └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | /docs/build/ 2 | /apiosintDS.egg-info/ 3 | /build/ 4 | /dist/ 5 | /test/ 6 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | # Required 2 | version: 2 3 | # Set the OS, Python version and other tools you might need 4 | build: 5 | os: ubuntu-22.04 6 | tools: 7 | python: "3.10" 8 | 9 | # Build documentation in the "docs/" directory with Sphinx 10 | 11 | sphinx: 12 | configuration: docs/source/conf.py 13 | 14 | formats: all 15 | # Optional but recommended, declare the Python requirements required 16 | # to build your documentation 17 | # See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html 18 | 19 | python: 20 | install: 21 | - requirements: docs/requirements.txt 22 | -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | Changelog 2 | ========= 3 | 4 | 2.0.3 (2024-05-08) 5 | ------------------ 6 | 7 | * Removed ``validators`` import. Regex are used instead. 8 | * Minor bug fixes 9 | * Improved MISP-modules version for better `compatibility `_ with the core project. 10 | 11 | 2.0.1 (2023-07-07) 12 | ------------------ 13 | 14 | * Bug fix to stix reports cache management 15 | 16 | 17 | 2.0 (2023-07-07) 18 | ---------------- 19 | 20 | * Many minor bug fixes 21 | * Implemented python ``getLogger`` as suggested in issue `#2 `_ 22 | * Added ``--stix`` option. Dowload and parse additional information from online STIX report. 23 | * Added ``--pretty`` option. Show results in terminal with a little bit of formatting applied. 24 | * Added ``--nocolor`` option. Suppers colors in –pretty output. For accessibility purpose. 25 | * Added ``--cachetimeout`` option. Define the cache timeout in hours. 26 | * Added ``--localdirectory`` option. Absolute path to the ‘Threat-Intel’ directory related to a local project clone. Searches are performed against local data. 27 | * Added ``--logfile`` option. Define the log file path. 28 | * Added ``--loglevel`` option. Define the log level. 29 | * Added ``--logconsole`` option. Suppress log messages to the console’s `STDOUT`. 30 | * Added ``--version`` option. Show the library version. 31 | * Improved ``apiosintDS.request`` method according new available options. 32 | * `New MISP Module plugin version `_ 33 | * `Documentation updated `_ 34 | 35 | 1.8.2 (2019-10-25) 36 | ------------------ 37 | 38 | * Bug fix for cache management of latesthashes.txt list 39 | 40 | 1.8 (2019-10-22) 41 | ---------------- 42 | 43 | * Added MD5/SHA1/SHA256 strings as entity to search 44 | * Added lookup to hash files for hash entities 45 | * Added support su hash lookup for related urls detected 46 | * Minor bug fixes 47 | * New schema json for response 48 | 49 | 1.7 (2019-10-20) 50 | ---------------- 51 | 52 | * Added support to be used as standard python library 53 | * Added docs 54 | * Minor bug fixes 55 | 56 | 1.6 (2019-10-13) 57 | ----------------- 58 | 59 | * Not a real new release. Just added support to pip. 60 | 61 | 1.6 (2019-10-13) 62 | ----------------- 63 | 64 | * First release for python library version usable as CLI tool. 65 | * Added Cache support 66 | * Multiple IoCs submission via text file 67 | * Output management 68 | * New schema response 69 | 70 | 1.0 (2019-10-07) 71 | ----------------- 72 | 73 | * Released version 1.0 published on `DigitalSide Threat-Intel `_ repository. 74 | 75 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Davide Baglieri 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # apiosintDS 2 | Latest stable release is **v2.0.3** 3 | 4 | **apiosintDS** is a [python client library](https://github.com/davidonzo/apiosintDS) for public *API* lookup service over *OSINT* IoCs stored at [DigitalSide Threat-Intel](https://osint.digitalside.it) repository. It can be defined a **Service as a Library** tool designed to act both as a standard Python library to be included in your own Python application and as command line tool. Query can be performed against souspicious IPs, domains, urls and file hashes. Data stored has a 7 days retention. 5 | 6 | ![apiosintDS v2.0.3](https://raw.githubusercontent.com/davidonzo/apiosintDS/master/docs/_static/img/apiosintDS.png) 7 | 8 | [DigitalSide Threat-Intel (also on GitHub.com)](https://github.com/davidonzo/Threat-Intel) shares a set of **Open Source Cyber Threat Intellegence** information, monstly based on malware analysis and compromised URLs, IPs and domains. The purpose of the project is to develop and test new wayes to hunt, analyze, collect and share relevants sets of IoCs to be used by SOC/CSIRT/CERT with minimun effort. 9 | 10 | **apiosintDS** is also available as [enrichment MISP Modules](https://misp.github.io/misp-modules/expansion/#apiosintds). Documentation is available on [apiosintDS Read The Docs](https://apiosintds.readthedocs.io/en/latest/userguidemisp.html). 11 | 12 | This library has been specially designed for people and organizations don't want to import the whole [DigitalSide Threat-Intel](https://osint.digitalside.it) dataset and prefer to use it as an on demand service. 13 | 14 | ## Documentation 15 | Complete documentation availables at [apiosintDS.ReadTheDocs.org](https://apiosintds.readthedocs.io/en/latest/) 16 | 17 | ## Install 18 | ### The easy way via pip 19 | ``` 20 | ~# pip3 install apiosintDS 21 | ``` 22 | 23 | ### From sources 24 | ``` 25 | ~$ cd /your/path/src/ 26 | ~$ git clone https://github.com/davidonzo/apiosintDS.git 27 | ~$ python3 -m pip install . 28 | ``` 29 | 30 | ## Usage 31 | ``` 32 | usage: apiosintDS [-h] [-e [IPv4|domain|url|hash]] [-f /path/to/file.txt] [-st] [-o /path/to/output.json] [-p] [-nc] [-v] [-c] [-cd /path/to/cachedir] [-ct [0-9]] [-cc] 33 | [-ld /path/to/git/clone/Threat-Intel/] [-ll [DEBUG|INFO|WARNING|ERROR|CRITICAL]] [-l /path/to/logfile.log] [-lc] [-i] [-s] [-vv] 34 | 35 | apiosintDS v.2.0.3. On demand query API for OSINT.digitalside.it project. You can query for souspicious domains, urls and IPv4. 36 | 37 | options: 38 | -h, --help show this help message and exit 39 | -e [IPv4|domain|url|hash], --entity [IPv4|domain|url|hash] 40 | Single item to search. Supported entities are IPv4/FQDN/URLs and file hashes in md5, sha1 or sha256. It can't be used in combination with the --file option. 41 | -f /path/to/file.txt, --file /path/to/file.txt 42 | Path to file containing entities to search. Supported entities are IPv4/FQDN/URLs. It can't be used in combination with the --entity option. 43 | -st, --stix Dowload and parse additional information from STIX report (if available). Default is False. 44 | -o /path/to/output.json, --output /path/to/output.json 45 | Path to output file (/path/to/output.json). If not specified the output will be redirect to the STDOUT. 46 | -p, --pretty Show results in terminal with a little bit of formatting applied. Default is False. 47 | -nc, --nocolor Suppers colors in --pretty output. For accessibility purpose. 48 | -v, --verbose Include unmatched results in report. Default is False. 49 | -c, --cache Enable cache mode. Downloaded lists will be stored a won't be downloaded until the cache timeout period is reached. Default is False. 50 | -cd /path/to/cachedir, --cachedirectory /path/to/cachedir 51 | The cache directory where the script check for cached lists files and where them will be stored on cache creation or update. Must be specified the same every script run unless 52 | your are using the system temp directory. Default is '/tmp' 53 | -ct [0-9], --cachetimeout [0-9] 54 | Define the cache timeout in hours. 0 is allowed but means no timeout. Default value is 4 hours. This option needs to be used in combination with --cache option configured to 55 | True. 56 | -cc, --clearcache Force the script to download updated lists and reports even if the cache timeout has not yet been reached. Default is False. Must be used in combination with --cache. 57 | -ld /path/to/git/clone/Threat-Intel/, --localdirectory /path/to/git/clone/Threat-Intel/ 58 | Absolute path to the 'Threat-Intel' directory related to local github repository clone. Searches are performed against local data. Before using this option, clone the GitHub 59 | project repository. When this option is in use, all cache related options are ignored. Default is False. 60 | -ll [DEBUG|INFO|WARNING|ERROR|CRITICAL], --loglevel [DEBUG|INFO|WARNING|ERROR|CRITICAL] 61 | Define the log level. Default value is DEBUG. 62 | -l /path/to/logfile.log, --logfile /path/to/logfile.log 63 | Define the log file path. Default value is None. No file log will be created by default. 64 | -lc, --logconsole Suppress log messages in the console STDOUT. Default value is False. 65 | -i, --info Print information about the library. 66 | -s, --schema Display the response json schema. 67 | -vv, --version Show the library version. 68 | 69 | ``` 70 | 71 | ### Basic example 72 | ``` 73 | $ apiosintDS -e 7cb796c875cccc9233d82854a4e2fdf0 74 | { 75 | "hash": { 76 | "items": [ 77 | { 78 | "item": "7cb796c875cccc9233d82854a4e2fdf0", 79 | "response": true, 80 | "response_text": "Item found in latesthashes.json list", 81 | "hashes": { 82 | "md5": "7cb796c875cccc9233d82854a4e2fdf0", 83 | "sha1": "158514acfa87d0b99e2af07a28004480bbf66e83", 84 | "sha256": "49e64d72d5ed4fb7967da4b6851d94cdceffe4ba0316587767a13901fe580239" 85 | }, 86 | "online_reports": { 87 | "MISP_EVENT": "https://osint.digitalside.it/Threat-Intel/digitalside-misp-feed/d6146389-4294-4a41-b4ca-6e74c74b7f8b.json", 88 | "MISP_CSV": "https://osint.digitalside.it/Threat-Intel/csv/d6146389-4294-4a41-b4ca-6e74c74b7f8b.csv", 89 | "OSINTDS_REPORT": "https://osint.digitalside.it/report/7cb796c875cccc9233d82854a4e2fdf0.html", 90 | "STIX": "https://osint.digitalside.it/Threat-Intel/stix2/7cb796c875cccc9233d82854a4e2fdf0.json" 91 | }, 92 | "related_urls": [ 93 | "http://185.246.220.60/plugmanzx.exe" 94 | ] 95 | } 96 | ], 97 | "statistics": { 98 | "itemsFound": 1, 99 | "itemsSubmitted": 1 100 | }, 101 | "list": { 102 | "file": "latesthashes.json", 103 | "date": "2023-07-07 08:03:29+02:00", 104 | "url": "https://raw.githubusercontent.com/davidonzo/Threat-Intel/master/lists/latesthashes.json" 105 | } 106 | }, 107 | "generalstatistics": { 108 | "url": 0, 109 | "ip": 0, 110 | "domain": 0, 111 | "hash": 1, 112 | "invalid": 0, 113 | "duplicates": 0, 114 | "itemsFound": 1, 115 | "itemsSubmitted": 1, 116 | "urlfound": 0, 117 | "ipfound": 0, 118 | "domainfound": 0, 119 | "hashfound": 1 120 | }, 121 | "apiosintDSversion": "apiosintDS v.2.0.3" 122 | } 123 | ``` 124 | 125 | ### Example usage: one item using `--pretty` 126 | ``` 127 | $ apiosintDS -e h[REMOVED]p://193.35.18.147/bins/k.arm -st -p -nc 128 | _ _ _ ____ ____ 129 | __ _ _ __ (_) ___ ___(_)_ __ | |_| _ \/ ___| 130 | / _` | '_ \| |/ _ \/ __| | '_ \| __| | | \___ \ 131 | | (_| | |_) | | (_) \__ \ | | | | |_| |_| |___) | 132 | \__,_| .__/|_|\___/|___/_|_| |_|\__|____/|____/ v.2.0.3 133 | |_|OSINT.DigitalSide.IT Threat-Intel Repository 134 | 135 | Submission summary 136 | ------------------------------------------------------- 137 | | Items parsed: 1 | Items submitted: 1 | Items found: 1 | 138 | ------------------------------------------------------- 139 | | Invalid(s): 0 | URL(s): 1 | URL(s): 1 | 140 | | Duplicate(s): 0 | Hash(es): 0 | Hash(es): 0 | 141 | | Not found: 0 | Domain(s): 0 | Domain(s): 0 | 142 | | | IP(s): 0 | IP(s): 0 | 143 | ------------------------------------------------------- 144 | ---------------------------------------------------------------------------- 145 | | hXXp://193.35.18.147/bins/k.arm | 146 | ---------------------------------------------------------------------------- 147 | | TLP:white | First Seen 2023-07-06 07:36:02 | Last Seen 2023-07-06 07:36:02 | 148 | ---------------------------------------------------------------------------- 149 | | Filename: k.arm | 150 | ---------------------------------------------------------------------------- 151 | | MD5: bc152acad73829358847e5f5bbf3edc0 | 152 | | SHA1: f2e26e44709ba5a9766c3c00226bdb663ede5957 | 153 | | SHA256: c8b0e1c5fa98bb407fe5bd3f2760b0ec2e5e33db0cee10a0085cac4505ef16cc | 154 | ---------------------------------------------------------------------------- 155 | | Size: 244647 | Type: application/x-executable | Observed: 1 | VT: 34/61 | 156 | ---------------------------------------------------------------------------- 157 | | Observation time frame: N/A | 158 | ---------------------------------------------------------------------------- 159 | | STIX network indicators: URLs => 1 | Domains => 0 | IPs: 1 | 160 | ---------------------------------------------------------------------------- 161 | Online Reports (availability depends on data retention) 162 | -> MISP EVENT: https://osint.digitalside.it/Threat-Intel/digitalside-misp-feed/f5e313d2-3d64-4d0f-af77-37a925bcd08f.json 163 | -> MISP CSV: https://osint.digitalside.it/Threat-Intel/csv/f5e313d2-3d64-4d0f-af77-37a925bcd08f.csv 164 | -> DS Report: https://osint.digitalside.it/report/bc152acad73829358847e5f5bbf3edc0.html 165 | -> STIX: https://osint.digitalside.it/Threat-Intel/stix2/bc152acad73829358847e5f5bbf3edc0.json 166 | ############################################################################# 167 | ``` 168 | 169 | ### Multiple items using `--file` with `--pretty` output 170 | 171 | Example file ioc.txt 172 | ``` 173 | ~$ cat ioc.txt 174 | 7cb796c875cccc9233d82854a4e2fdf0 175 | monke.re 176 | 177 | ``` 178 | 179 | Response 180 | ``` 181 | ~$ apiosintDS -f ioc.txt -p -nc -st 182 | 183 | _ _ _ ____ ____ 184 | __ _ _ __ (_) ___ ___(_)_ __ | |_| _ \/ ___| 185 | / _` | '_ \| |/ _ \/ __| | '_ \| __| | | \___ \ 186 | | (_| | |_) | | (_) \__ \ | | | | |_| |_| |___) | 187 | \__,_| .__/|_|\___/|___/_|_| |_|\__|____/|____/ v.2.0.3 188 | |_|OSINT.DigitalSide.IT Threat-Intel Repository 189 | 190 | Submission summary 191 | ------------------------------------------------------- 192 | | Items parsed: 2 | Items submitted: 2 | Items found: 2 | 193 | ------------------------------------------------------- 194 | | Invalid(s): 0 | URL(s): 0 | URL(s): 0 | 195 | | Duplicate(s): 0 | Hash(es): 1 | Hash(es): 1 | 196 | | Not found: 0 | Domain(s): 1 | Domain(s): 1 | 197 | | | IP(s): 0 | IP(s): 0 | 198 | ------------------------------------------------------- 199 | ---------------------------------------------------------------------------- 200 | | 7cb796c875cccc9233d82854a4e2fdf0 | 201 | ---------------------------------------------------------------------------- 202 | | TLP:white | First Seen 2023-07-04 09:33:03 | Last Seen 2023-07-04 09:33:03 | 203 | ---------------------------------------------------------------------------- 204 | | Filename: plugmanzx.exe | 205 | ---------------------------------------------------------------------------- 206 | | MD5: 7cb796c875cccc9233d82854a4e2fdf0 | 207 | | SHA1: 158514acfa87d0b99e2af07a28004480bbf66e83 | 208 | | SHA256: 49e64d72d5ed4fb7967da4b6851d94cdceffe4ba0316587767a13901fe580239 | 209 | ---------------------------------------------------------------------------- 210 | | Size: 924672 | Type: application/x-dosexec | Observed: 1 | VT: 32/71 | 211 | ---------------------------------------------------------------------------- 212 | | Observation time frame: N/A | 213 | ---------------------------------------------------------------------------- 214 | | STIX network indicators: URLs => 1 | Domains => 0 | IPs: 1 | 215 | ---------------------------------------------------------------------------- 216 | Online Reports (availability depends on data retention) 217 | -> MISP EVENT: https://osint.digitalside.it/Threat-Intel/digitalside-misp-feed/d6146389-4294-4a41-b4ca-6e74c74b7f8b.json 218 | -> MISP CSV: https://osint.digitalside.it/Threat-Intel/csv/d6146389-4294-4a41-b4ca-6e74c74b7f8b.csv 219 | -> DS Report: https://osint.digitalside.it/report/7cb796c875cccc9233d82854a4e2fdf0.html 220 | -> STIX: https://osint.digitalside.it/Threat-Intel/stix2/7cb796c875cccc9233d82854a4e2fdf0.json 221 | ############################################################################# 222 | 223 | --------------------------------------------------------------------------- 224 | | monke[.]re - Related URL(s) 2 | 225 | --------------------------------------------------------------------------- 226 | ---------------------------------------------------------------------------- 227 | | hXXp://monke.re/arm7 | 228 | ---------------------------------------------------------------------------- 229 | | TLP:white | First Seen 2023-07-06 23:51:01 | Last Seen 2023-07-06 23:51:01 | 230 | ---------------------------------------------------------------------------- 231 | | Filename: arm7 | 232 | ---------------------------------------------------------------------------- 233 | | MD5: 318323c9da34bf25833f7da32eab23d6 | 234 | | SHA1: e2bb927b08ebcbaad8f304d02309af776312c9bf | 235 | | SHA256: bb1f9e108daa389e62b79067d1cdbef548f9934c9cc85a92565da7063cf36f89 | 236 | ---------------------------------------------------------------------------- 237 | | Size: 57148 | Type: application/x-executable | Observed: 1 | VT: 14/61 | 238 | ---------------------------------------------------------------------------- 239 | | Observation time frame: N/A | 240 | ---------------------------------------------------------------------------- 241 | | STIX network indicators: URLs => 1 | Domains => 1 | IPs: 0 | 242 | ---------------------------------------------------------------------------- 243 | Online Reports (availability depends on data retention) 244 | -> MISP EVENT: https://osint.digitalside.it/Threat-Intel/digitalside-misp-feed/f83d06e6-aa2f-452e-a19d-59d40e874355.json 245 | -> MISP CSV: https://osint.digitalside.it/Threat-Intel/csv/f83d06e6-aa2f-452e-a19d-59d40e874355.csv 246 | -> DS Report: https://osint.digitalside.it/report/318323c9da34bf25833f7da32eab23d6.html 247 | -> STIX: https://osint.digitalside.it/Threat-Intel/stix2/318323c9da34bf25833f7da32eab23d6.json 248 | ---------------------------------------------------------------------------- 249 | | hXXp://monke.re/mips | 250 | ---------------------------------------------------------------------------- 251 | | TLP:white | First Seen 2023-07-07 00:31:02 | Last Seen 2023-07-07 00:31:02 | 252 | ---------------------------------------------------------------------------- 253 | | Filename: mips | 254 | ---------------------------------------------------------------------------- 255 | | MD5: 579081f528d9279a87b298b9838c377b | 256 | | SHA1: 45048073aad5997881dffe41e32f9b17beb1c2e1 | 257 | | SHA256: 8186a1d140631e6391978c08c35e01efb58963f65a86fddf7dec44eec7681c6b | 258 | ---------------------------------------------------------------------------- 259 | | Size: 48272 | Type: application/x-executable | Observed: 1 | VT: 12/61 | 260 | ---------------------------------------------------------------------------- 261 | | Observation time frame: N/A | 262 | ---------------------------------------------------------------------------- 263 | | STIX network indicators: URLs => 1 | Domains => 1 | IPs: 0 | 264 | ---------------------------------------------------------------------------- 265 | Online Reports (availability depends on data retention) 266 | -> MISP EVENT: https://osint.digitalside.it/Threat-Intel/digitalside-misp-feed/d01c2ad1-0e2c-4b26-9725-f8a86025bd75.json 267 | -> MISP CSV: https://osint.digitalside.it/Threat-Intel/csv/d01c2ad1-0e2c-4b26-9725-f8a86025bd75.csv 268 | -> DS Report: https://osint.digitalside.it/report/579081f528d9279a87b298b9838c377b.html 269 | -> STIX: https://osint.digitalside.it/Threat-Intel/stix2/579081f528d9279a87b298b9838c377b.json 270 | ################################################################################################################################## 271 | ``` 272 | 273 | [Json schema](https://github.com/davidonzo/apiosintDS/blob/master/apiosintDS/schema/schema.json) 274 | 275 | ## Python 3 requiremet 276 | The script runs using python intepreter at version 3.x. No support will be given to python 2.x. 277 | 278 | -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davidonzo/apiosintDS/513c5dc7f1da09be349062206dac918c17bd4512/__init__.py -------------------------------------------------------------------------------- /apiosintDS/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davidonzo/apiosintDS/513c5dc7f1da09be349062206dac918c17bd4512/apiosintDS/__init__.py -------------------------------------------------------------------------------- /apiosintDS/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davidonzo/apiosintDS/513c5dc7f1da09be349062206dac918c17bd4512/apiosintDS/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /apiosintDS/__pycache__/apiosintDS.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davidonzo/apiosintDS/513c5dc7f1da09be349062206dac918c17bd4512/apiosintDS/__pycache__/apiosintDS.cpython-310.pyc -------------------------------------------------------------------------------- /apiosintDS/apiosintDS.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import logging 3 | import pytz 4 | import tempfile 5 | import argparse 6 | import os 7 | import requests 8 | import re 9 | import json 10 | italyTZ = pytz.timezone("Europe/Rome") 11 | from apiosintDS.modules import listutils, dosearch 12 | from apiosintDS.modules.scriptinfo import scriptinfo 13 | from apiosintDS.utilities import logutils, prettycli 14 | import platform 15 | 16 | 17 | def checkpyversion(logger): 18 | if (sys.version_info < (3, 0)):#NO MORE PYTHON 2!!! https://pythonclock.org/ 19 | errormessage = """ 20 | ########################### ERROR ########################### 21 | ============================================================= 22 | Invalid python version detected: """+str(sys.version_info[0])+"."+str(sys.version_info[1])+""" 23 | ============================================================= 24 | It seems your are still using python 2 even if you should 25 | now it will be retire next 2020. 26 | For more info please read https://pythonclock.org/ 27 | ============================================================= 28 | Try again typing: python3 """+sys.argv[0]+""" 29 | ########################### ERROR ########################### 30 | """ 31 | logger.error(errormessage) 32 | exit(0) 33 | 34 | def nolinuxwarning(logger): 35 | if platform.system() not in ['Linux']: 36 | logger.warning("Script not tested on "+platform.system()+" systems. Use at your own risks.") 37 | 38 | def checkfile(file): 39 | if os.path.isfile(file) == False: 40 | msg = "File not found: %r." % file 41 | raise argparse.ArgumentTypeError(msg) 42 | else: 43 | lines = [line.rstrip('\n') for line in open(file)] 44 | if len(lines) == 0: 45 | msg2 = "File is empty or unreadable: %r." % file 46 | raise argparse.ArgumentTypeError(msg2) 47 | return lines 48 | 49 | def checklocalpath(path): 50 | if os.path.isdir(path) == False: 51 | msg = "Directory not found: %r." % path 52 | raise argparse.ArgumentTypeError(msg) 53 | return path 54 | 55 | def writablefile(file): 56 | if os.path.isfile(file) == True: 57 | msg = "File %r already exists. Please, delete it first." % file 58 | raise argparse.ArgumentTypeError(msg) 59 | else: 60 | try: 61 | f = open(file, "w+") 62 | f.close() 63 | except: 64 | msg2 = "Unable to open/create the file: %r." % file 65 | raise argparse.ArgumentTypeError(msg2) 66 | return file 67 | 68 | def writablelog(file): 69 | if os.path.isfile(file) == True: 70 | try: 71 | f = open(file, "a") 72 | f.close() 73 | except: 74 | msg = "File %r already exists. Please, delete it first." % file 75 | raise argparse.ArgumentTypeError(msg) 76 | else: 77 | try: 78 | f = open(file, "w+") 79 | f.close() 80 | except: 81 | msg2 = "Unable to open/create the file: %r." % file 82 | raise argparse.ArgumentTypeError(msg2) 83 | return file 84 | 85 | def writablecache(tmpdir): 86 | if os.path.isfile(tmpdir): 87 | msg = "%r seems to be a file, not a directory." % tmpdir 88 | raise argparse.ArgumentTypeError(msg) 89 | elif os.path.exists(tmpdir) == False: 90 | msg = "%r directory not found." % tmpdir 91 | raise argparse.ArgumentTypeError(msg) 92 | elif os.access(tmpdir, os.W_OK) == False: 93 | msg = "%r directory not writable." % tmpdir 94 | raise argparse.ArgumentTypeError(msg) 95 | return tmpdir 96 | 97 | def positiveint(val): 98 | if (int(val) < 0) or (isinstance(int(val), int) == False): 99 | msg = "Cache timeout must be a positive integer, %r received instead" % val 100 | raise argparse.ArgumentTypeError(msg) 101 | return val 102 | 103 | def filebspath(directory, file): 104 | _BSR = os.path.abspath(os.path.dirname(__file__)) 105 | return os.path.join(_BSR, directory, file) 106 | 107 | def info(): 108 | htext = scriptinfo["scriptname"]+" v."+scriptinfo["majorversion"]+"."+scriptinfo["minorversion"]+"." 109 | htext += "\nOn demand query API for OSINT.digitalside.it project.\n" 110 | htext += "You can query for souspicious domains, urls and IPv4.\n\n" 111 | htext += "For more information read the README.md file and the JSON schema hosted on GitHub.com:\n" 112 | htext += " - "+scriptinfo["git"]+"/README.md\n" 113 | htext += " - "+scriptinfo["git"]+"/schema.json\n" 114 | htext += "\n" 115 | htext += "This file is part of the OSINT.digitalside.it project.\n" 116 | htext += "For more information about the project please visit the following links:\n" 117 | htext += " - "+scriptinfo["DSProjectHP"]+"\n" 118 | htext += " - "+scriptinfo["DSGitHubHP"]+"\n" 119 | htext += "\n" 120 | htext += "This software is released under the "+scriptinfo["license"]+" license\n" 121 | htext += " - "+scriptinfo["licenseurl"]+"\n" 122 | htext += "\n" 123 | htext += "Coded with love by\n "+scriptinfo["author"]+" <"+scriptinfo["mail"]+">\n" 124 | htext += " PGP "+scriptinfo["pgp"]+"\n" 125 | htext += " Fingerprint "+scriptinfo["fingerprint"] 126 | htext += "\n" 127 | return htext 128 | 129 | def version(): 130 | return scriptinfo["scriptname"]+" v."+scriptinfo["majorversion"]+"."+scriptinfo["minorversion"] 131 | 132 | def schema(logger): 133 | try: 134 | schema = open(filebspath('schema', 'schema.json'), "r") 135 | content = schema.read() 136 | schema.close() 137 | return content 138 | except IOError as e: 139 | logger.error(e) 140 | logger.error("Unable to load schema file.") 141 | exit(1) 142 | 143 | def request(entities=list, stix=False, cache=False, cachedirectory=None, clearcache=False, cachetimeout=False, verbose=False, loglevel="DEBUG", logconsole=True, logfile=False, localdirectory=False, *args, **kwargs): 144 | logger = logutils.logutils(loglevel, logfile, logconsole) 145 | if isinstance(entities, list): 146 | if localdirectory: 147 | try: 148 | checklocalpath(localdirectory) 149 | cache=False 150 | cachedirectory=None 151 | clearcache=False 152 | except Exception as localdirectoryerror: 153 | logger.error(localdirectoryerror) 154 | exit(1) 155 | if cachetimeout == False: 156 | cachetimeout = 4 157 | if clearcache and ((not cache) or (cache == False)): 158 | logger.error("Unable to clear cache with cache disabled. Please set the cache to 'True'") 159 | exit(1) 160 | if cachedirectory and ((not cache) or (cache == False)): 161 | logger.error("Unable to use a cache directory with the cache option disabled. Please set the cache to 'True'") 162 | exit(1) 163 | if cache and not cachedirectory: 164 | logger.error("When using apiosintDS as python library, you always have to specify the temporary files directory to be used.") 165 | exit(1) 166 | if (isinstance(cachetimeout, int) == False): 167 | logger.error("Cache timeout must be a positive integer, '"+str(cachetimeout)+"' received instead") 168 | exit(1) 169 | if int(cachetimeout) < 0: 170 | logger.error("Cache timeout must be a positive integer, '"+str(cachetimeout)+"' received instead") 171 | exit(1) 172 | if cache: 173 | try: 174 | writablecache(cachedirectory) 175 | except Exception as clearcacheerror: 176 | logger.error(clearcacheerror) 177 | exit(1) 178 | 179 | lutils = listutils.listutils(None, entities, cache, cachedirectory, clearcache, cachetimeout, localdirectory, logger) 180 | makelist = lutils.prepareLists() 181 | if isinstance(makelist, dict): 182 | serarch = dosearch.dosearch(makelist, verbose, stix, cache, cachedirectory, clearcache, cachetimeout, localdirectory, logger) 183 | results = serarch.prepareResults() 184 | if isinstance(results, dict): 185 | return results 186 | else: 187 | logger.error("create_request must return a dict.") 188 | else: 189 | logger.error("create_request must return a dict.") 190 | else: 191 | logger.error("entities must be an instance of list.") 192 | exit(1) 193 | 194 | def main(): 195 | parserdescription = scriptinfo["scriptname"]+" v."+scriptinfo["majorversion"]+"."+scriptinfo["minorversion"]+"." 196 | parserdescription +=" On demand query API for OSINT.digitalside.it project." 197 | parserdescription +=" You can query for souspicious domains, urls and IPv4." 198 | parser = argparse.ArgumentParser(description=parserdescription) 199 | parser.add_argument("-e","--entity", type=str, action="store", metavar="[IPv4|domain|url|hash]", dest="ITEM", help="Single item to search. Supported entities are IPv4/FQDN/URLs and file hashes in md5, sha1 or sha256. It can't be used in combination with the --file option.", default=None) 200 | parser.add_argument("-f","--file", type=checkfile, action="store", metavar="/path/to/file.txt", dest="FILE", help="Path to file containing entities to search. Supported entities are IPv4/FQDN/URLs. It can't be used in combination with the --entity option.", default=None) 201 | parser.add_argument("-st", "--stix", action="store_true", dest="STIX", help="Dowload and parse additional information from STIX report (if available). Default is False.", default=False) 202 | parser.add_argument("-o", "--output", type=writablefile, action="store", metavar="/path/to/output.json", dest="OUTPUT", help="Path to output file (/path/to/output.json). If not specified the output will be redirect to the STDOUT.", default=False) 203 | parser.add_argument("-p", "--pretty", action="store_true", dest="PRETTY", help="Show results in terminal with a little bit of formatting applied. Default is False.", default=False) 204 | parser.add_argument("-nc", "--nocolor", action="store_true", dest="NOCOLOR", help="Suppers colors in --pretty output. For accessibility purpose.", default=False) 205 | parser.add_argument("-v", "--verbose", action="store_true", dest="VERBOSE", help="Include unmatched results in report. Default is False.", default=False) 206 | parser.add_argument("-c","--cache", action="store_true", dest="CACHE", help="Enable cache mode. Downloaded lists will be stored a won't be downloaded until the cache timeout period is reached. Default is False.", default=False) 207 | parser.add_argument("-cd","--cachedirectory", type=writablecache, action="store", metavar="/path/to/cachedir", dest="CACHEDIRECTORY", help="The cache directory where the script check for cached lists files and where them will be stored on cache creation or update. Must be specified the same every script run unless your are using the system temp directory. Default is '"+tempfile.gettempdir()+"'", default=tempfile.gettempdir()) 208 | parser.add_argument("-ct", "--cachetimeout", type=positiveint, action="store", metavar="[0-9]", dest="CACHETIMEOUT", help="Define the cache timeout in hours. 0 is allowed but means no timeout. Default value is 4 hours. This option needs to be used in combination with --cache option configured to True.", default=4) 209 | parser.add_argument("-cc","--clearcache", action="store_true", dest="CLEARCACHE", help="Force the script to download updated lists and reports even if the cache timeout has not yet been reached. Default is False. Must be used in combination with --cache.", default=False) 210 | parser.add_argument("-ld","--localdirectory", type=checklocalpath, metavar="/path/to/git/clone/Threat-Intel/", dest="LOCALDIRECTORY", help="Absolute path to the 'Threat-Intel' directory related to local github repository clone. Searches are performed against local data. Before using this option, clone the GitHub project repository. When this option is in use, all cache related options are ignored. Default is False.", default=False) 211 | parser.add_argument("-ll","--loglevel", type=str, metavar="[DEBUG|INFO|WARNING|ERROR|CRITICAL]", dest="LOGLEVEL", help="Define the log level. Default value is DEBUG.", default="DEBUG") 212 | parser.add_argument("-l","--logfile", type=writablelog, metavar="/path/to/logfile.log", dest="LOGFILE", help="Define the log file path. Default value is None. No log file is created by default.", default=None) 213 | parser.add_argument("-lc","--logconsole", action="store_true", dest="LOGCONSOLE", help="Suppress log messages to the console's STDOUT. Default value is False.", default=False) 214 | parser.add_argument("-i","--info", action="store_true", dest="INFO", help="Print information about the library.") 215 | parser.add_argument("-s","--schema", action="store_true", dest="SCHEMA", help="Display the response json schema.") 216 | parser.add_argument("-vv","--version", action="store_true", dest="VERSION", help="Show the library version.") 217 | 218 | try: 219 | args = parser.parse_args() 220 | OSDLogger = logutils.logutils(args.LOGLEVEL, args.LOGFILE, args.LOGCONSOLE) 221 | 222 | checkpyversion(OSDLogger) 223 | nolinuxwarning(OSDLogger) 224 | 225 | if (args.VERSION): 226 | sys.stdout.write(version()) 227 | exit(1) 228 | if (args.INFO): 229 | sys.stdout.write(info()) 230 | exit(1) 231 | if (args.SCHEMA): 232 | try: 233 | schema = open(filebspath('schema', 'schema.json'), "r") 234 | for schemaline in schema.readlines(): 235 | sys.stdout.write(schemaline) 236 | schema.close() 237 | exit(0) 238 | except IOError as e: 239 | OSDLogger.error(e) 240 | OSDLogger.error("Unable to load schema file.") 241 | exit(1) 242 | if (args.ITEM == None) and (args.FILE == None): 243 | parser.error("No targets selected! Please, specify one option between --entity and --file.\nTry option -h or --help.") 244 | OSDLogger.error("No targets selected! Please, specify one option between --entity and --file.\nTry option -h or --help.") 245 | exit(1) 246 | elif (args.ITEM != None) and (args.FILE != None): 247 | parser.error("Too much targets selected! Sorry, you can't specify both options --entity and --file.\nTry option -h or --help.") 248 | OSDLogger.error("Too much targets selected! Sorry, you can't specify both options --entity and --file.\nTry option -h or --help.") 249 | exit(1) 250 | elif args.CLEARCACHE and not args.CACHE: 251 | args.CLEARCACHE = False 252 | OSDLogger.warning("Expected -c or --cache option declared. Ignoring all cache settings.\nTry option -h or --help.") 253 | if args.LOCALDIRECTORY: 254 | args.CACHE = False 255 | OSDLogger.info("Detected local copy of the Threat-Intel repository. Cache options will be ignored.") 256 | if args.PRETTY and args.OUTPUT: 257 | args.PRETTY = False 258 | OSDLogger.warning("Detected --output argument in combination with --pretty option. The --pretty option will be ignored.") 259 | lutils = listutils.listutils(args.ITEM, args.FILE, args.CACHE, args.CACHEDIRECTORY, args.CLEARCACHE, int(args.CACHETIMEOUT), args.LOCALDIRECTORY, OSDLogger) 260 | makelist = lutils.prepareLists() 261 | if isinstance(makelist, dict): 262 | serarch = dosearch.dosearch(makelist, args.VERBOSE, args.STIX, args.CACHE, args.CACHEDIRECTORY, args.CLEARCACHE, int(args.CACHETIMEOUT), args.LOCALDIRECTORY, OSDLogger) 263 | results = serarch.prepareResults() 264 | if isinstance(results, dict): 265 | output = json.dumps(results, indent=4, separators=(",", ": ")) 266 | if args.OUTPUT == False: 267 | if args.PRETTY: 268 | pretty = prettycli.prettycli(output, args.NOCOLOR, OSDLogger) 269 | output = pretty.output 270 | sys.stdout.write(output) 271 | else: 272 | fileoutput = open(args.OUTPUT, "w+") 273 | fileoutput.write(output) 274 | fileoutput.close() 275 | OSDLogger.info("Output saved in file: "+args.OUTPUT) 276 | else: 277 | OSDLogger.error("'results' is not an dict. Quit!") 278 | else: 279 | OSDLogger.error("'makelist' is not an dict. Quit!") 280 | except argparse.ArgumentError as e: 281 | OSDLogger.error(e) 282 | parser.error("Unexpected Error.\nTry option -h or --help.") 283 | exit(2) 284 | 285 | if __name__ == '__main__': 286 | try: 287 | main() 288 | except Exception as detectedError: 289 | print(detectedError) 290 | logger.critical(detectedError) 291 | -------------------------------------------------------------------------------- /apiosintDS/modules/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davidonzo/apiosintDS/513c5dc7f1da09be349062206dac918c17bd4512/apiosintDS/modules/__init__.py -------------------------------------------------------------------------------- /apiosintDS/modules/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davidonzo/apiosintDS/513c5dc7f1da09be349062206dac918c17bd4512/apiosintDS/modules/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /apiosintDS/modules/__pycache__/dosearch.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davidonzo/apiosintDS/513c5dc7f1da09be349062206dac918c17bd4512/apiosintDS/modules/__pycache__/dosearch.cpython-310.pyc -------------------------------------------------------------------------------- /apiosintDS/modules/__pycache__/listutils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davidonzo/apiosintDS/513c5dc7f1da09be349062206dac918c17bd4512/apiosintDS/modules/__pycache__/listutils.cpython-310.pyc -------------------------------------------------------------------------------- /apiosintDS/modules/__pycache__/scriptinfo.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davidonzo/apiosintDS/513c5dc7f1da09be349062206dac918c17bd4512/apiosintDS/modules/__pycache__/scriptinfo.cpython-310.pyc -------------------------------------------------------------------------------- /apiosintDS/modules/__pycache__/stixreport.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davidonzo/apiosintDS/513c5dc7f1da09be349062206dac918c17bd4512/apiosintDS/modules/__pycache__/stixreport.cpython-310.pyc -------------------------------------------------------------------------------- /apiosintDS/modules/dosearch.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | import json 4 | from datetime import datetime 5 | import pytz 6 | from urllib.parse import urlparse 7 | from apiosintDS.modules.stixreport import stixreport 8 | italyTZ = pytz.timezone("Europe/Rome") 9 | 10 | 11 | class dosearch(): 12 | def __init__(self, lists, verbose, stix, cache, cachedirectory, clearcache, cachetimeout, localdirectory, logger): 13 | self.logger = logger 14 | self.lists = lists 15 | self.verbose = verbose 16 | self.stix = stix 17 | self.cache = cache 18 | self.cachedirectory = cachedirectory 19 | self.clearcache = clearcache 20 | self.cachetimeout = cachetimeout 21 | if localdirectory != False: 22 | self.localdirectory = localdirectory if localdirectory[(len(localdirectory)-1):] == os.sep else localdirectory+os.sep 23 | else: 24 | self.localdirectory = localdirectory 25 | self.osintDS = "https://osint.digitalside.it" 26 | 27 | ### !!! ### o_O 28 | self.generalStats = self.lists["input"]["GeneralStats"] 29 | self.entitype = ['url', 'ip', 'domain', 'hash'] #Supported Entities type 30 | self.output = { 31 | "url": { 32 | "items": [], 33 | "statistics": { 34 | "itemsFound": 0, 35 | "itemsSubmitted": 0 36 | }, 37 | "list": { 38 | "file": "undefined", 39 | "date": "undefined", 40 | "url": "undefined" 41 | } 42 | }, 43 | "ip": { 44 | "items": [], 45 | "statistics": { 46 | "itemsFound": 0, 47 | "itemsSubmitted": 0 48 | }, 49 | "list": { 50 | "file": "undefined", 51 | "date": "undefined", 52 | "url": "undefined" 53 | } 54 | }, 55 | "domain": { 56 | "items": [], 57 | "statistics": { 58 | "itemsFound": 0, 59 | "itemsSubmitted": 0 60 | }, 61 | "list": { 62 | "file": "undefined", 63 | "date": "undefined", 64 | "url": "undefined" 65 | } 66 | }, 67 | "hash": { 68 | "items": [], 69 | "statistics": { 70 | "itemsFound": 0, 71 | "itemsSubmitted": 0 72 | }, 73 | "list": { 74 | "file": "undefined", 75 | "date": "undefined", 76 | "url": "undefined" 77 | } 78 | }, 79 | "generalstatistics": self.lists["input"]["GeneralStats"], 80 | "apiosintDSversion": self.lists["input"]["apiosintDSversion"], 81 | } 82 | 83 | self.getResults = self.ExecSearch() 84 | 85 | def goodResult(self, item, entype, relatedByHash=False, hashes=False): 86 | ret = {} 87 | ret["item"] = item 88 | ret["response"] = True 89 | ret["response_text"] = "Item found in "+self.lists["lookup"][entype]["file"]+" list" 90 | if entype in ['ip', 'domain']: 91 | ret["related_urls"] = self.parseRelatedUrl(item) 92 | if(len(ret["related_urls"])>0): 93 | for relatedurls in ret["related_urls"]: 94 | onlineReports = self.searchMISPSTIXCSV(relatedurls["url"], "url") 95 | relatedurls["online_reports"] = onlineReports 96 | elif entype == 'url': 97 | searchUrlHash = self.searchUrlHash(item) 98 | ret["hashes"] = searchUrlHash 99 | ret["online_reports"] = self.searchMISPSTIXCSV(item, entype) 100 | getItem = urlparse(item) 101 | ret["related_urls"] = self.parseRelatedUrl(getItem.hostname, item) 102 | if(len(ret["related_urls"])>0): 103 | for relatedurls in ret["related_urls"]: 104 | if relatedurls["hashes"]["md5"] != ret["hashes"]["md5"]: 105 | onlineReports = self.searchMISPSTIXCSV(relatedurls["url"], "url") 106 | relatedurls["online_reports"] = onlineReports 107 | elif entype == 'hash': 108 | ret["hashes"] = hashes 109 | ret["online_reports"] = self.searchMISPSTIXCSV(item, entype) 110 | ret["related_urls"] = relatedByHash 111 | else: 112 | pass 113 | return ret 114 | 115 | def badResult(self, item, entype): 116 | ret = {} 117 | ret["item"] = item 118 | ret["response"] = False 119 | ret["response_text"] = "Item not found" 120 | return ret 121 | 122 | def parseRelatedUrl(self, item, ItemURL=False): 123 | relatedURLs = [] 124 | resUrls = self.lists["lookup"]["url"]["items"] 125 | for lineurl in resUrls: 126 | ParseURL = urlparse(lineurl) 127 | if (ParseURL.hostname == item) and (lineurl != ItemURL): 128 | relhashes = self.searchUrlHash(lineurl) 129 | reldict = {'url': lineurl, 'hashes': relhashes} 130 | relatedURLs.append(reldict) 131 | return relatedURLs 132 | 133 | def searchUrlHash(self, item): 134 | #ret = False 135 | for obj in self.lists["lookup"]["hash"]["items"]: 136 | if item in self.lists["lookup"]["hash"]["items"][obj]["url"]: 137 | return {"md5": self.lists["lookup"]["hash"]["items"][obj]["md5"], "sha1": self.lists["lookup"]["hash"]["items"][obj]["sha1"], "sha256": self.lists["lookup"]["hash"]["items"][obj]["sha256"]} 138 | 139 | return ret 140 | 141 | def searchMISPSTIXCSV(self, item, entype): 142 | ret = False 143 | for key, obj in self.lists["lookup"]["hash"]["items"].items(): 144 | if entype == "hash": 145 | hashtype = self.hashtype(item) 146 | elif entype == "url": 147 | hashtype = "url" 148 | 149 | if item in self.lists["lookup"]["hash"]["items"][key][hashtype]: 150 | if self.localdirectory: 151 | ret = {} 152 | if os.path.isfile(self.localdirectory+"digitalside-misp-feed/"+key+".json"): 153 | ret["MISP_EVENT"] = self.localdirectory+"digitalside-misp-feed/"+key+".json" 154 | if os.path.isfile(self.localdirectory+"csv/"+key+".csv"): 155 | ret["MISP_CSV"] = self.localdirectory+"csv/"+key+".csv" 156 | ret["OSINTDS_REPORT"] = self.osintDS+"/report/"+self.lists["lookup"]["hash"]["items"][key]["md5"]+".html" 157 | if os.path.isfile(self.localdirectory+"stix2/"+self.lists["lookup"]["hash"]["items"][key]["md5"]+".json"): 158 | ret["STIX"] = self.localdirectory+"stix2/"+self.lists["lookup"]["hash"]["items"][key]["md5"]+".json" 159 | else: 160 | ret = { 161 | "MISP_EVENT": self.osintDS+"/Threat-Intel/digitalside-misp-feed/"+key+".json", 162 | "MISP_CSV": self.osintDS+"/Threat-Intel/csv/"+key+".csv", 163 | "OSINTDS_REPORT": self.osintDS+"/report/"+self.lists["lookup"]["hash"]["items"][key]["md5"]+".html", 164 | "STIX": self.osintDS+"/Threat-Intel/stix2/"+self.lists["lookup"]["hash"]["items"][key]["md5"]+".json" 165 | } 166 | 167 | if self.stix: 168 | ret["STIXDETAILS"] = stixreport(self.lists["lookup"]["hash"]["items"][key]["md5"], self.cache, self.cachedirectory, self.clearcache, self.cachetimeout, self.localdirectory, self.logger).parseReport() 169 | 170 | 171 | 172 | 173 | return ret 174 | 175 | def hashtype(self, item): 176 | ret = False 177 | hashtype_dict = {32: "md5", 40: "sha1", 64: "sha256"} 178 | hashlen = len(item) 179 | ret = hashtype_dict[hashlen] 180 | 181 | return ret 182 | 183 | def searchHash(self, item): 184 | hashtype = self.hashtype(item) 185 | hashes = False 186 | for obj in self.lists["lookup"]["hash"]["items"]: 187 | if item in self.lists["lookup"]["hash"]["items"][obj][hashtype]: 188 | hashes = {"md5": self.lists["lookup"]["hash"]["items"][obj]["md5"], "sha1": self.lists["lookup"]["hash"]["items"][obj]["sha1"], "sha256": self.lists["lookup"]["hash"]["items"][obj]["sha256"]} 189 | self.output["hash"]["items"].append(self.goodResult(item, "hash", self.lists["lookup"]["hash"]["items"][obj]["url"], hashes)) 190 | self.output["hash"]["statistics"]["itemsFound"] +=1 191 | self.generalStats["itemsFound"] +=1 192 | self.generalStats["hashfound"] +=1 193 | break 194 | 195 | if self.verbose and hashes == False: 196 | self.output["hash"]["items"].append(self.badResult(item, "hash")) 197 | 198 | return 199 | 200 | def ExecSearch(self): 201 | for entype in self.entitype: 202 | if len(self.lists["input"]["entities"][entype]) > 0: 203 | self.output[entype]["list"]["date"] = self.lists["lookup"][entype]["date"] 204 | self.output[entype]["list"]["url"] = self.lists["lookup"][entype]["url"] 205 | self.output[entype]["list"]["file"] = self.lists["lookup"][entype]["file"] 206 | 207 | for item in self.lists["input"]["entities"][entype]: 208 | if entype == "hash": 209 | self.searchHash(item) 210 | else: 211 | if item in self.lists["lookup"][entype]["items"]: 212 | self.output[entype]["items"].append(self.goodResult(item, entype)) 213 | self.output[entype]["statistics"]["itemsFound"] +=1 214 | self.generalStats["itemsFound"] +=1 215 | self.generalStats[entype+"found"] +=1 216 | 217 | else: 218 | if self.verbose: 219 | self.output[entype]["items"].append(self.badResult(item, entype)) 220 | self.output[entype]["statistics"]["itemsSubmitted"] = len(self.lists["input"]["entities"][entype]) 221 | self.generalStats["itemsSubmitted"] = self.generalStats["itemsSubmitted"]+self.output[entype]["statistics"]["itemsSubmitted"] 222 | else: 223 | del self.output[entype] 224 | return self.output 225 | 226 | def prepareResults(self): 227 | return self.output 228 | -------------------------------------------------------------------------------- /apiosintDS/modules/listutils.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | import requests 4 | import json 5 | from datetime import datetime 6 | from apiosintDS.modules.scriptinfo import scriptinfo 7 | import re 8 | import pytz 9 | 10 | italyTZ = pytz.timezone("Europe/Rome") 11 | 12 | class listutils(): 13 | def __init__(self, item, listfile, cache, cachedir, clearcache, cachetimeout, localdirectory, logger): 14 | self.logger = logger 15 | self.item = item 16 | self.listfile = listfile 17 | self.urls = {"master_url": "https://raw.githubusercontent.com/davidonzo/Threat-Intel/master/lists/latest", 18 | "slave_url": "https://osint.digitalside.it/Threat-Intel/lists/latest"} 19 | self.cache = cache 20 | if isinstance(cachedir, str): 21 | self.cachedir = cachedir+"/" 22 | else: 23 | self.cachedir = False 24 | self.clearcache = clearcache 25 | self.cachetimeout = cachetimeout 26 | if localdirectory != False: 27 | self.localdirectory = localdirectory if localdirectory[:len(localdirectory)] == os.sep else localdirectory+os.sep 28 | else: 29 | self.localdirectory = localdirectory 30 | self.version = scriptinfo["scriptname"]+" v."+scriptinfo["majorversion"]+"."+scriptinfo["minorversion"] 31 | 32 | ### !!! ### o_O 33 | self.template = {"url": [], "ip": [], "domain": [], "hash": []} 34 | self.invaliditems = 0 35 | self.duplicateditems = 0 36 | self.items = self.getItems() 37 | self.entities = dict(self.getEntities()) 38 | self.checkdate = italyTZ.localize(datetime.strptime(datetime.today().strftime('%Y-%m-%d %H:%M:%S'), '%Y-%m-%d %H:%M:%S')) 39 | self.cached = dict(self.getCache()) 40 | 41 | self.entitiesStats = {"url": len(self.template["url"]), 42 | "ip": len(self.template["ip"]), 43 | "domain": len(self.template["domain"]), 44 | "hash": len(self.template["hash"]), 45 | "invalid": self.invaliditems, 46 | "duplicates": self.duplicateditems, 47 | "itemsFound": 0, 48 | "itemsSubmitted": 0, 49 | "urlfound": 0, 50 | "ipfound": 0, 51 | "domainfound": 0, 52 | "hashfound": 0, 53 | } 54 | 55 | def getItems(self): 56 | if self.item is not None: 57 | self.items = [] 58 | self.items.append(self.item) 59 | elif self.listfile is not None: 60 | self.items = list(dict.fromkeys(self.listfile)) 61 | duplicate = len(self.listfile) - len(self.items) 62 | if duplicate > 0: 63 | self.duplicateditems = duplicate 64 | self.logger.warning(f'Removed {duplicate} duplicate(s) from IoCs lists to check') 65 | return self.items 66 | 67 | def validatefqdn(self, domain): 68 | regex = re.compile(r'(?=^.{4,253}$)(^((?!-)[a-zA-Z0-9-]{1,63}(? https://github.com/django/django/blob/stable/1.3.x/django/core/validators.py#L45 74 | regex = re.compile( 75 | r'^(?:http|ftp)s?://' # http:// or https:// 76 | r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' #domain... 77 | r'localhost|' #localhost... 78 | r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip 79 | r'(?::\d+)?' # optional port 80 | r'(?:/?|[/?]\S+)$', re.IGNORECASE 81 | ) 82 | 83 | return regex.match(url) 84 | 85 | def validateipv4(self, ipv4): 86 | #from django src => https://github.com/django/django/blob/stable/1.3.x/django/core/validators.py#L161 87 | regex = re.compile(r'^(25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}$') 88 | 89 | return regex.match(ipv4) 90 | 91 | def validatehash(self, hstring): 92 | ret = False 93 | digits = len(hstring) 94 | if digits in [32, 40, 64]: 95 | try: 96 | regex = re.compile(r'^[A-F0-9a-f]{'+str(digits)+'}$', re.IGNORECASE) 97 | ret = regex.match(hstring.lower()) 98 | except: 99 | self.logger.info("Invalid hash detected: failed validation for ["+hstring+"]") 100 | else: 101 | self.logger.info("Invalid hash detected: failed lenght check for ["+hstring+" len("+str(digits)+")]") 102 | return ret 103 | 104 | def getEntities(self): 105 | counter = 0 106 | for line in self.items: 107 | line = line.strip() 108 | if self.validateurl(line): 109 | counter += 1 110 | self.template["url"].append(line) 111 | elif self.validateipv4(line): 112 | counter +=1 113 | self.template["ip"].append(line.lower()) 114 | elif self.validatefqdn(line): 115 | counter +=1 116 | self.template["domain"].append(line.lower()) 117 | elif self.validatehash(line): 118 | counter +=1 119 | self.template["hash"].append(line.lower()) 120 | elif len(line) == 0: 121 | self.logger.warning("Empty line ignored!") 122 | else: 123 | self.invaliditems +=1 124 | self.logger.warning("{} is not a valid IPv4/domain/url/hash. REMOVED!.".format(line)) 125 | if counter == 0: 126 | self.logger.error("No valid elements detected, sorry! Supported entities are IPv4/Domain/URL/Hash ['MD5', 'SHA1', 'SHA256'].") 127 | exit(1) 128 | ret = {} 129 | ret["entities"] = self.template 130 | ret["centities"] = {"url": len(self.template["url"]), "ip": len(self.template["ip"]), "domain": len(self.template["domain"]), "hash": len(self.template["hash"])} 131 | return ret 132 | 133 | def getListDate(self, context, hashes=False): 134 | if hashes: 135 | context = json.loads(context) 136 | listdate = context["generatedAt"] 137 | else: 138 | listdate = context[9][16:35] 139 | return italyTZ.localize(datetime.strptime(listdate, '%Y-%m-%d %H:%M:%S')) 140 | 141 | def getListItems(self, context, hashes=False): 142 | if hashes: 143 | context = json.loads(context) 144 | listitems = context["lookup"] 145 | else: 146 | listitems = context[11:] 147 | return listitems 148 | 149 | def getCache(self): #Mmhhh, not a good method at all, but works. Added to TODO list for early refactor. 2019-10-12 150 | cached = {} 151 | for entity in self.entities["centities"]: 152 | getHash = (True if entity == "hash" else False) 153 | cached[entity] = {} 154 | cached[entity]["file"] = "latest"+entity+"es.json" if entity == "hash" else "latest"+entity+"s.txt" 155 | if (self.entities["centities"][entity] == 0) and (entity in ["ip", "domain"]): 156 | pass 157 | else: 158 | if self.cache: 159 | cachedfile = self.cachedir+cached[entity]["file"] 160 | if os.path.exists(cachedfile): 161 | if self.clearcache: 162 | dwlist = self.downloadLists(entity, getHash) 163 | cached[entity]["date"] = self.getListDate(dwlist["text"], getHash) 164 | cached[entity]["items"] = self.getListItems(dwlist["text"], getHash) 165 | cached[entity]["url"] = dwlist["url"] 166 | else: 167 | if entity == "hash": 168 | cacheHandler = open(cachedfile, 'r') 169 | content = cacheHandler.read() 170 | cacheHandler.close() 171 | dwlist = content 172 | else: 173 | dwlist = [line.rstrip('\n') for line in open(cachedfile)] 174 | 175 | listdate = self.getListDate(dwlist, getHash) 176 | listitems = self.getListItems(dwlist, getHash) 177 | 178 | if self.cachetimeout > 0: 179 | diffdate = ((self.checkdate-listdate).total_seconds())/3600 180 | if ((diffdate) < 4) and (len(listitems) > 0): 181 | cached[entity]["date"] = listdate 182 | cached[entity]["items"] = listitems 183 | cached[entity]["url"] = "Loaded from cache '"+cachedfile+"'" 184 | else: 185 | dwlist = self.downloadLists(entity, getHash) 186 | cached[entity]["date"] = self.getListDate(dwlist["text"], getHash) 187 | cached[entity]["items"] = self.getListItems(dwlist["text"], getHash) 188 | cached[entity]["url"] = dwlist["url"] 189 | else: 190 | dwlist = self.downloadLists(entity, getHash) 191 | cached[entity]["date"] = self.getListDate(dwlist["text"], getHash) 192 | cached[entity]["items"] = self.getListItems(dwlist["text"], getHash) 193 | cached[entity]["url"] = dwlist["url"] 194 | else: 195 | dwlist = self.downloadLists(entity, getHash) 196 | cached[entity]["date"] = self.getListDate(dwlist["text"], getHash) 197 | cached[entity]["items"] = self.getListItems(dwlist["text"], getHash) 198 | cached[entity]["url"] = dwlist["url"] 199 | else: 200 | dwlist = self.downloadLists(entity, getHash) 201 | cached[entity]["date"] = self.getListDate(dwlist["text"], getHash) 202 | cached[entity]["items"] = self.getListItems(dwlist["text"], getHash) 203 | cached[entity]["url"] = dwlist["url"] 204 | cached[entity]["date"] = str(cached[entity]["date"]) 205 | return cached 206 | 207 | def saveCache(self, entity, content): 208 | try: 209 | file_ext = 'txt' 210 | if (entity == 'hashe'): 211 | file_ext = 'json' 212 | 213 | cachefile = open(self.cachedir+"latest"+entity+"s."+file_ext, "w") 214 | cachefile.write(content) 215 | cachefile.close() 216 | except IOError as e: 217 | self.logger.error(e) 218 | self.logger.error("Unable save list! Make sure you have write permission on file "+self.cachedir+"latest"+entity+"s."+file_ext+".") 219 | self.logger.error("Retry without -c, --cache option.") 220 | exit(1) 221 | 222 | def openLocalLists(self, listfile): 223 | ret = False 224 | thelistfile = self.localdirectory+"lists/"+listfile 225 | if os.path.isfile(thelistfile): 226 | try: 227 | ret = open(thelistfile).read() 228 | 229 | except ValueError as e: 230 | self.logger.error(e) 231 | exit(1) 232 | else: 233 | self.logger.error("File not found: "+listfile) 234 | exit(1) 235 | return ret 236 | 237 | def downloadLists(self, entity, hashes=False): 238 | # micro patch for hashes vs hash 239 | entity = entity+"e" if entity == "hash" else entity 240 | ret = {} 241 | 242 | # New file format for hash entities 243 | 244 | file_ext = 'txt' 245 | if (entity == 'hashe'): 246 | file_ext = 'json' 247 | 248 | if self.localdirectory: 249 | text = self.openLocalLists('latest'+entity+'s.'+file_ext) 250 | ret["url"] = self.localdirectory+'latest'+entity+'s.'+file_ext 251 | else: 252 | ret["url"] = self.urls['master_url']+entity+'s.'+file_ext 253 | r = requests.get(ret["url"]) 254 | if r.status_code != 200: 255 | ret["url"] = self.urls['slave_url']+entity+'s.'+file_ext 256 | self.logger.warning("Error downloading lastes{}.{} from GitHub repository.".format(entity, file_ext)) 257 | self.logger.warning("Returned HTTP status code is {}:".format(r.status_code)) 258 | self.logger.warning("Try downloading file from osint.digitalside.it") 259 | r = requests.get(ret["url"]) 260 | if r.status_code != 200: 261 | self.logger.warning("Error downloading lastes{}s.{} both from GitHub repository and OSINT.digitalside.it".format(entity, file_ext)) 262 | self.logger.warning("Returned HTTP status code is {}:".format(r.status_code)) 263 | self.logger.error(self.status_error(entity)) 264 | exit(1) 265 | return 1 266 | text = r.text 267 | if len(text) == 0: 268 | self.logger.error("The downloaded list seems to be empty!\n") 269 | self.logger.error(self.status_error(entity)) 270 | exit(1) 271 | if hashes: 272 | return_text = text 273 | else: 274 | return_text = text.split('\n') 275 | if len(return_text[11:]) == 0: #just because I'm a very paranoid man, but same time a lazy one ^_^''' 276 | self.logger.error("The downloaded list seems to be empty!\n") 277 | if self.localdirectory == False: 278 | self.logger.error(self.status_error(entity)) 279 | exit(1) 280 | else: 281 | ret["text"] = return_text 282 | if self.cache: 283 | self.saveCache(entity, text) 284 | return ret 285 | 286 | def status_error(self, entity): 287 | file_ext = 'txt' 288 | if (entity == 'hashe'): 289 | file_ext = 'json' 290 | 291 | error="Check the following urls using your prefered browser:\n" 292 | error+="- https://raw.githubusercontent.com/davidonzo/Threat-Intel/master/lists/latest"+entity+"."+file_ext+"\n" 293 | error+="- https://osint.digitalside.it/Threat-Intel/lists/latest"+entity+"."+file_ext+"\n" 294 | error+="\n" 295 | error+="Are you able to view the desired IoC list? If not, please, report this opening an issue on Threat-Intel GitHub repository:\n" 296 | error+="- https://github.com/davidonzo/Threat-Intel/issues\n" 297 | error+="\n" 298 | error+="Aren't you familiar with GitHub? No worries. You can send a PGP signed and encrypted email to info@digitalside.it\n" 299 | error+="PGP key ID: 30B31BDA\n" 300 | error+="PGP fingerprint: 0B4C F801 E8FF E9A3 A602 D2C7 9C36 93B2 30B3 1BDA\n" 301 | error+="\n" 302 | error+="Aren't you familiar with PGP? Be worried... maybe you should not use this script ;-)\n" 303 | return error 304 | 305 | def prepareLists(self): 306 | ret = {} 307 | ret["input"] = self.entities 308 | ret["input"]["GeneralStats"] = self.entitiesStats 309 | ret["input"]["apiosintDSversion"] = self.version 310 | ret["input"]["scriptinfo"] = scriptinfo 311 | ret["lookup"] = self.cached 312 | return ret 313 | -------------------------------------------------------------------------------- /apiosintDS/modules/scriptinfo.py: -------------------------------------------------------------------------------- 1 | scriptinfo = {"scriptname": "apiosintDS", 2 | "subscriptname": "OSINT.DigitalSide.IT Threat-Intel Repository", 3 | "majorversion": "2", 4 | "minorversion": "0.3", 5 | "license": "MIT", 6 | "licenseurl": "https://raw.githubusercontent.com/davidonzo/apiosintDS/master/LICENSE.txt", 7 | "author": "Davide Baglieri", 8 | "mail": "info[at]digitalside.it", 9 | "pgp": "30B31BDA", 10 | "fingerprint": "0B4C F801 E8FF E9A3 A602 D2C7 9C36 93B2 30B3 1BDA", 11 | "git": "https://github.com/davidonzo/apiosintDS", 12 | "DSProjectHP": "https://osint.digitalside.it", 13 | "DSGitHubHP": "https://github.com/davidonzo/Threat-Intel"} 14 | -------------------------------------------------------------------------------- /apiosintDS/modules/stixreport.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | import requests 4 | import json 5 | from datetime import datetime 6 | import time 7 | import re 8 | from stix2 import parse 9 | import pytz 10 | 11 | italyTZ = pytz.timezone("Europe/Rome") 12 | 13 | class stixreport(): 14 | def __init__(self, item, cache, cachedir, clearcache, cachetimeout, localdirectory, logger): 15 | 16 | self.logger = logger 17 | self.item = item 18 | self.report = item+".json" 19 | self.urls = {"master_url": "https://raw.githubusercontent.com/davidonzo/Threat-Intel/master/stix2/", 20 | "slave_url": "https://osint.digitalside.it/Threat-Intel/stix2/"} 21 | self.cache = cache 22 | if isinstance(cachedir, str): 23 | self.cachedir = cachedir+"/" 24 | else: 25 | self.cachedir = False 26 | self.clearcache = clearcache 27 | self.cachetimeout = cachetimeout 28 | if localdirectory != False: 29 | self.localdirectory = localdirectory if localdirectory[:len(localdirectory)] == os.sep else localdirectory+os.sep 30 | else: 31 | self.localdirectory = localdirectory 32 | 33 | self.checkdate = italyTZ.localize(datetime.strptime(datetime.today().strftime('%Y-%m-%d %H:%M:%S'), '%Y-%m-%d %H:%M:%S')) 34 | self.getStix = self.getCache() 35 | 36 | 37 | def getListDate(self, reportContent): 38 | ret = False 39 | for obj in reportContent["objects"]: 40 | if obj["type"] == "report": 41 | ret = obj["published"] 42 | #datetime.strptime(str(fromdate), '%Y-%m-%d %H:%M:%S+00:00') 43 | return italyTZ.localize(datetime.strptime(ret, '%Y-%m-%dT%H:%M:%SZ')) 44 | 45 | def getCache(self): 46 | dwreport = False 47 | if self.cache: 48 | cachedfile = self.cachedir+self.report 49 | if os.path.exists(cachedfile): 50 | if self.clearcache: 51 | dwreport = json.loads(self.downloadReport()) 52 | else: 53 | cacheHandler = open(cachedfile, 'r') 54 | content = cacheHandler.read() 55 | cacheHandler.close() 56 | dwreport = json.loads(content) 57 | 58 | if self.cachetimeout > 0: 59 | reportDate = self.getListDate(dwreport) 60 | diffdate = ((self.checkdate-reportDate).total_seconds())/3600 61 | 62 | if diffdate < self.cachetimeout: 63 | self.logger.info("Report "+self.report+" loaded from cache") 64 | else: 65 | dwreport = self.downloadReport() 66 | else: 67 | dwreport = self.downloadReport() 68 | else: 69 | dwreport = self.downloadReport() 70 | else: 71 | dwreport = self.downloadReport() 72 | return dwreport 73 | 74 | def saveCache(self, entity, content): 75 | try: 76 | cachefile = open(self.cachedir+self.report, "w") 77 | cachefile.write(content) 78 | cachefile.close() 79 | except IOError as e: 80 | self.logger.error(e) 81 | self.logger.error("Unable save list! Make sure you have write permission on file "+self.cachedir+self.report) 82 | self.logger.error("Retry without -c, --cache option.") 83 | exit(1) 84 | 85 | def openLocalReport(self, reportfile): 86 | ret = False 87 | thereportfile = self.localdirectory+"stix2/"+reportfile 88 | if os.path.isfile(thereportfile): 89 | try: 90 | ret = open(thereportfile).read() 91 | except ValueError as e: 92 | self.logger.error(e) 93 | exit(1) 94 | return ret 95 | 96 | def downloadReport(self): 97 | ret = False 98 | 99 | if self.localdirectory: 100 | stixreport = self.openLocalReport(self.report) 101 | else: 102 | reportURL = self.urls['master_url']+self.report 103 | r = requests.get(reportURL) 104 | if r.status_code != 200: 105 | reportURL = self.urls['slave_url']+self.report 106 | self.logger.warning("Error downloading {} from GitHub repository.".format(self.report)) 107 | self.logger.warning("Returned HTTP status code is {}:".format(r.status_code)) 108 | self.logger.warning("Try downloading file from osint.digitalside.it") 109 | r = requests.get(ret["url"]) 110 | if r.status_code != 200: 111 | self.logger.warning("Error downloading {} both from GitHub repository and OSINT.digitalside.it".format(self.report)) 112 | self.logger.warning("Returned HTTP status code is {}:".format(r.status_code)) 113 | self.logger.error(self.status_error(self.report)) 114 | return ret 115 | return ret 116 | 117 | stixreport = r.text 118 | if len(stixreport) == 0: 119 | self.logger.error("The downloaded list seems to be empty!\n") 120 | if self.localdirectory == False: 121 | self.logger.error(self.status_error(self.report)) 122 | return ret 123 | 124 | if self.cache: 125 | self.saveCache(self, stixreport) 126 | return stixreport 127 | 128 | def status_error(self): 129 | error="Check the following urls using your prefered browser:\n" 130 | error+="- https://raw.githubusercontent.com/davidonzo/Threat-Intel/master/stix2/"+self.report+"\n" 131 | error+="- https://osint.digitalside.it/Threat-Intel/stix2/"+self.report+"\n" 132 | error+="\n" 133 | error+="Are you able to view the desired IoC list? If not, please, report this opening an issue on Threat-Intel GitHub repository:\n" 134 | error+="- https://github.com/davidonzo/Threat-Intel/issues\n" 135 | error+="\n" 136 | error+="Aren't you familiar with GitHub? No worries. You can send a PGP signed and encrypted email to info@digitalside.it\n" 137 | error+="PGP key ID: 30B31BDA\n" 138 | error+="PGP fingerprint: 0B4C F801 E8FF E9A3 A602 D2C7 9C36 93B2 30B3 1BDA\n" 139 | error+="\n" 140 | error+="Aren't you familiar with PGP? Be worried... maybe you should not use this script ;-)\n" 141 | return error 142 | 143 | def releaseReport(self): 144 | if self.getStix == False: 145 | return False 146 | return parse(self.getStix) 147 | 148 | def parseReport(self): 149 | ret = False 150 | 151 | report = self.releaseReport() 152 | 153 | if report != False: 154 | ret = {} 155 | ret["observed_time_frame"] = False 156 | ret["indicators_count"] = {"hashes": 0, "urls": 0, "domains": 0, "ipv4": 0} 157 | 158 | for obj in report.objects: 159 | if obj.type == "marking-definition": 160 | ret["tlp"] = obj.definition.tlp 161 | if obj.type == "malware": 162 | ret["first_observed"] = obj.first_seen.strftime('%Y-%m-%d %H:%M:%S') 163 | ret["last_observed"] = obj.last_seen.strftime('%Y-%m-%d %H:%M:%S') 164 | if obj.first_seen != obj.last_seen: 165 | ret["observed_time_frame"] = self.calcTimeFrame(obj.first_seen, obj.last_seen) 166 | ret["virus_total"] = self.getVTReport(obj) 167 | if obj.type == "observed-data": 168 | ret["number_observed"] = obj.number_observed 169 | if obj.type == "file": 170 | ret["filename"] = obj.name 171 | ret["filesize"] = obj.size 172 | ret["mime_type"] = obj.mime_type 173 | if obj.type == "indicator": 174 | searchPattern = str(obj.pattern)[:10] 175 | if searchPattern == "[file:hash": 176 | ret["indicators_count"]["hashes"] +=1 177 | elif searchPattern == "[url:value": 178 | ret["indicators_count"]["urls"] +=1 179 | elif searchPattern == "[domain-na": 180 | ret["indicators_count"]["domains"] +=1 181 | elif searchPattern == "[ipv4-addr": 182 | ret["indicators_count"]["ipv4"] +=1 183 | return ret 184 | 185 | def getVTReport(self, obj): 186 | ret = False 187 | if obj.external_references: 188 | for url in obj.external_references: 189 | if str(url.description)[:11] == "Virus Total": 190 | ret = {"vt_detection_ratio": url.description[28:], "vt_report": url.url} 191 | return ret 192 | 193 | def calcTimeFrame(self, fromdate, todate): 194 | ret = False 195 | timedeltaobj = (datetime.strptime(str(todate), '%Y-%m-%d %H:%M:%S+00:00') - datetime.strptime(str(fromdate), '%Y-%m-%d %H:%M:%S+00:00')) 196 | 197 | if timedeltaobj.total_seconds() > 0: 198 | ret = str(timedeltaobj) 199 | return ret 200 | -------------------------------------------------------------------------------- /apiosintDS/schema/schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/schema#", 3 | "title": "Validator for apiosintDS response", 4 | "id": "https://github.com/davidonzo/apiosintDS/tree/master/apiosintDS/schema/schema.json", 5 | "type": "object", 6 | "properties": { 7 | "url": { 8 | "description": "Response for submitted urls", 9 | "type": "object", 10 | "minOccur": 0, 11 | "properties": { 12 | "items": { 13 | "description": "Array listing results", 14 | "minOccur": 0, 15 | "items": [ 16 | { 17 | "type": "object", 18 | "properties": { 19 | "item": { 20 | "type": "string", 21 | "description": "The searched entity", 22 | "minOccur": 1, 23 | "maxOccur": 1 24 | }, 25 | "response": { 26 | "type": "boolean", 27 | "description": "'True' if a match has been found. 'False' if no match has been found", 28 | "minOccur": 1, 29 | "maxOccur": 1 30 | }, 31 | "response_text": { 32 | "type": "string", 33 | "description": "Just a human readble representation of the 'response' item", 34 | "minOccur": 1, 35 | "maxOccur": 1 36 | }, 37 | "hashes": { 38 | "description": "Hashes related to the found URL", 39 | "minOccur": 0, 40 | "maxOccur": 1, 41 | "type": "object", 42 | "properties": { 43 | "md5": { 44 | "type": "string", 45 | "description": "The MD5 hash related to the submitted URL", 46 | "minOccur": 0, 47 | "maxOccur": 1 48 | }, 49 | "sha1": { 50 | "type": "string", 51 | "description": "The SHA1 hash related to the submitted URL", 52 | "minOccur": 0, 53 | "maxOccur": 1 54 | }, 55 | "sha256": { 56 | "type": "string", 57 | "description": "The SHA256 hash related to the submitted URL", 58 | "minOccur": 0, 59 | "maxOccur": 1 60 | } 61 | } 62 | }, 63 | "online_reports": { 64 | "type": "object", 65 | "description": "Links or local path to the related reports", 66 | "minOccur": 1, 67 | "maxOccur": 1, 68 | "properties": { 69 | "MISP_EVENT": { 70 | "type": "string", 71 | "description": "URL or path to the related MISP event json file", 72 | "minOccur": 1, 73 | "maxOccur": 1 74 | }, 75 | "MISP_CSV": { 76 | "type": "string", 77 | "description": "URL or path to the related MISP event CSV file", 78 | "minOccur": 1, 79 | "maxOccur": 1 80 | }, 81 | "OSINTDS_REPORT": { 82 | "type": "string", 83 | "description": "URL to the related HTML report published at osint.digitalside.it", 84 | "minOccur": 1, 85 | "maxOccur": 1 86 | }, 87 | "STIX": { 88 | "type": "string", 89 | "description": "URL or path to the related STIX2 json file", 90 | "minOccur": 1, 91 | "maxOccur": 1 92 | }, 93 | "STIXDETAILS": { 94 | "type": "object", 95 | "description": "Details extracted by the STIX2 file. Available when option --st, --stix is True", 96 | "minOccur": 0, 97 | "maxOccur": 1, 98 | "properties": { 99 | "observed_time_frame": { 100 | "type": ["boolean", "string"], 101 | "description": "Timedelta between first and last file submission. if 'number_observed' is 1 the value is False", 102 | "minOccur": 1, 103 | "maxOccur": 1, 104 | }, 105 | "indicators_count": { 106 | "type": "object", 107 | "description": "Indicators count per type, as extracted by the related STIX 2 report", 108 | "minOccur": 1, 109 | "maxOccur": 1, 110 | "properties": { 111 | "hashes": { 112 | "type": "integer", 113 | "description": "Hashes count", 114 | "minOccur": 1, 115 | "maxOccur": 1, 116 | }, 117 | "urls": { 118 | "type": "integer", 119 | "description": "Urls count", 120 | "minOccur": 1, 121 | "maxOccur": 1, 122 | }, 123 | "domains": { 124 | "type": "integer", 125 | "description": "Domains count", 126 | "minOccur": 1, 127 | "maxOccur": 1, 128 | }, 129 | "ipv4": { 130 | "type": "integer", 131 | "description": "IPs count", 132 | "minOccur": 1, 133 | "maxOccur": 1, 134 | } 135 | } 136 | }, 137 | "tlp": { 138 | "type": "string", 139 | "description": "Traffic Light Protocol", 140 | "minOccur": 1, 141 | "maxOccur": 1, 142 | }, 143 | "first_observed": { 144 | "type": "string", 145 | "description": "First file submission date", 146 | "minOccur": 1, 147 | "maxOccur": 1, 148 | }, 149 | "last_observed": { 150 | "type": "string", 151 | "description": "Last file submission date", 152 | "minOccur": 1, 153 | "maxOccur": 1, 154 | }, 155 | "virus_total": { 156 | "type": ["object", "boolean"], 157 | "description": "Virus Total report. False in case no VT report is available", 158 | "minOccur": 1, 159 | "maxOccur": 1, 160 | "properties": { 161 | "vt_detection_ratio": { 162 | "type": "string", 163 | "description": "VT detection ratio", 164 | "minOccur": 0, 165 | "maxOccur": 1, 166 | }, 167 | "vt_report": { 168 | "type": "string", 169 | "description": "Link to the Virus Total report", 170 | "minOccur": 1, 171 | "maxOccur": 1, 172 | } 173 | } 174 | }, 175 | "filename": { 176 | "type": "string", 177 | "description": "Detected filename. Available only the latest filename", 178 | "minOccur": 1, 179 | "maxOccur": 1, 180 | }, 181 | "filesize": { 182 | "type": "integer", 183 | "description": "File size in bytes", 184 | "minOccur": 1, 185 | "maxOccur": 1, 186 | }, 187 | "mime_type": { 188 | "type": "string", 189 | "description": "File type", 190 | "minOccur": 1, 191 | "maxOccur": 1, 192 | }, 193 | "number_observed": { 194 | "type": "integer", 195 | "description": "How many time the file has been downloaded by the system", 196 | "minOccur": 1, 197 | "maxOccur": 1, 198 | } 199 | } 200 | } 201 | } 202 | }, 203 | "related_urls": { 204 | "description": "The related URLs associated to the submitted item. If no related IoC found, the list will be empty", 205 | "minOccur": 0, 206 | "maxOccur": 1, 207 | "type": "array", 208 | "items": [ 209 | { 210 | "type": "object", 211 | "description": "Object containing related URLs and hashes detected", 212 | "minOccur": 0, 213 | "properties": { 214 | "url": { 215 | "type": "string", 216 | "description": "Related URL detected", 217 | "minOccur": 0, 218 | "maxOccur": 1, 219 | }, 220 | "hashes": { 221 | "description": "Hashes related to the found URL", 222 | "minOccur": 0, 223 | "maxOccur": 1, 224 | "type": "object", 225 | "properties": { 226 | "md5": { 227 | "type": "string", 228 | "description": "The MD5 hash related to the submitted URL", 229 | "minOccur": 0, 230 | "maxOccur": 1 231 | }, 232 | "sha1": { 233 | "type": "string", 234 | "description": "The SHA1 hash related to the submitted URL", 235 | "minOccur": 0, 236 | "maxOccur": 1 237 | }, 238 | "sha256": { 239 | "type": "string", 240 | "description": "The SHA256 hash related to the submitted URL", 241 | "minOccur": 0, 242 | "maxOccur": 1 243 | } 244 | } 245 | } 246 | } 247 | } 248 | ] 249 | } 250 | }, 251 | "required": [ 252 | "item", 253 | "response", 254 | "response_text" 255 | ] 256 | } 257 | ] 258 | }, 259 | "statistics": { 260 | "type": "object", 261 | "description": "Basic statistics about submitted and found urls", 262 | "minOccur": 1, 263 | "maxOccur": 1, 264 | "properties": { 265 | "itemsFound": { 266 | "type": "integer", 267 | "description": "Numbers of found items", 268 | "minOccur": 1, 269 | "maxOccur": 1 270 | }, 271 | "itemsSubmitted": { 272 | "type": "integer", 273 | "description": "Numbers of submitted items", 274 | "minOccur": 1, 275 | "maxOccur": 1 276 | } 277 | }, 278 | "required": [ 279 | "itemsFound", 280 | "itemsSubmitted" 281 | ] 282 | }, 283 | "list": { 284 | "type": "object", 285 | "properties": { 286 | "file": { 287 | "type": "string", 288 | "description": "The list name used for lookup", 289 | "minOccur": 1, 290 | "maxOccur": 1 291 | }, 292 | "date": { 293 | "type": "string", 294 | "description": "The generated date of the list used for lookup", 295 | "minOccur": 1, 296 | "maxOccur": 1 297 | }, 298 | "url": { 299 | "type": "string", 300 | "description": "The download link will be reported or a cache notice if the list has been loaded from previous generated cache", 301 | "minOccur": 1, 302 | "maxOccur": 1 303 | } 304 | }, 305 | "required": [ 306 | "file", 307 | "date", 308 | "url" 309 | ] 310 | } 311 | }, 312 | "required": [ 313 | "statistics", 314 | "list" 315 | ] 316 | }, 317 | "hash": { 318 | "description": "Response for submitted hash", 319 | "type": "object", 320 | "minOccur": 0, 321 | "properties": { 322 | "items": { 323 | "description": "Array listing results", 324 | "minOccur": 0, 325 | "items": [ 326 | { 327 | "type": "object", 328 | "properties": { 329 | "item": { 330 | "type": "string", 331 | "description": "The searched entity", 332 | "minOccur": 1, 333 | "maxOccur": 1 334 | }, 335 | "response": { 336 | "type": "boolean", 337 | "description": "'True' if a match has been found. 'False' if no match has been found", 338 | "minOccur": 1, 339 | "maxOccur": 1 340 | }, 341 | "response_text": { 342 | "type": "string", 343 | "description": "Just a human readble representation of the 'response' item", 344 | "minOccur": 1, 345 | "maxOccur": 1 346 | }, 347 | "hashes": { 348 | "description": "Hashes related to the found item", 349 | "minOccur": 0, 350 | "maxOccur": 1, 351 | "type": "object", 352 | "properties": { 353 | "md5": { 354 | "type": "string", 355 | "description": "The MD5 hash related to the submitted hash", 356 | "minOccur": 0, 357 | "maxOccur": 1 358 | }, 359 | "sha1": { 360 | "type": "string", 361 | "description": "The SHA1 hash related to the submitted hash", 362 | "minOccur": 0, 363 | "maxOccur": 1 364 | }, 365 | "sha256": { 366 | "type": "string", 367 | "description": "The SHA256 hash related to the submitted hash", 368 | "minOccur": 0, 369 | "maxOccur": 1 370 | } 371 | } 372 | }, 373 | "online_reports": { 374 | "type": "object", 375 | "description": "Links or local path to the related reports", 376 | "minOccur": 1, 377 | "maxOccur": 1, 378 | "properties": { 379 | "MISP_EVENT": { 380 | "type": "string", 381 | "description": "URL or path to the related MISP event json file", 382 | "minOccur": 1, 383 | "maxOccur": 1 384 | }, 385 | "MISP_CSV": { 386 | "type": "string", 387 | "description": "URL or path to the related MISP event CSV file", 388 | "minOccur": 1, 389 | "maxOccur": 1 390 | }, 391 | "OSINTDS_REPORT": { 392 | "type": "string", 393 | "description": "URL to the related HTML report published at osint.digitalside.it", 394 | "minOccur": 1, 395 | "maxOccur": 1 396 | }, 397 | "STIX": { 398 | "type": "string", 399 | "description": "URL or path to the related STIX2 json file", 400 | "minOccur": 1, 401 | "maxOccur": 1 402 | }, 403 | "STIXDETAILS": { 404 | "type": "object", 405 | "description": "Details extracted by the STIX2 file. Available when option --st, --stix is True", 406 | "minOccur": 0, 407 | "maxOccur": 1, 408 | "properties": { 409 | "observed_time_frame": { 410 | "type": ["boolean", "string"], 411 | "description": "Timedelta between first and last file submission. if 'number_observed' is 1 the value is False", 412 | "minOccur": 1, 413 | "maxOccur": 1, 414 | }, 415 | "indicators_count": { 416 | "type": "object", 417 | "description": "Indicators count per type, as extracted by the related STIX 2 report", 418 | "minOccur": 1, 419 | "maxOccur": 1, 420 | "properties": { 421 | "hashes": { 422 | "type": "integer", 423 | "description": "Hashes count", 424 | "minOccur": 1, 425 | "maxOccur": 1, 426 | }, 427 | "urls": { 428 | "type": "integer", 429 | "description": "Urls count", 430 | "minOccur": 1, 431 | "maxOccur": 1, 432 | }, 433 | "domains": { 434 | "type": "integer", 435 | "description": "Domains count", 436 | "minOccur": 1, 437 | "maxOccur": 1, 438 | }, 439 | "ipv4": { 440 | "type": "integer", 441 | "description": "IPs count", 442 | "minOccur": 1, 443 | "maxOccur": 1, 444 | } 445 | } 446 | }, 447 | "tlp": { 448 | "type": "string", 449 | "description": "Traffic Light Protocol", 450 | "minOccur": 1, 451 | "maxOccur": 1, 452 | }, 453 | "first_observed": { 454 | "type": "string", 455 | "description": "First file submission date", 456 | "minOccur": 1, 457 | "maxOccur": 1, 458 | }, 459 | "last_observed": { 460 | "type": "string", 461 | "description": "Last file submission date", 462 | "minOccur": 1, 463 | "maxOccur": 1, 464 | }, 465 | "virus_total": { 466 | "type": ["object", "boolean"], 467 | "description": "Virus Total report. False in case no VT report is available", 468 | "minOccur": 1, 469 | "maxOccur": 1, 470 | "properties": { 471 | "vt_detection_ratio": { 472 | "type": "string", 473 | "description": "VT detection ratio", 474 | "minOccur": 0, 475 | "maxOccur": 1, 476 | }, 477 | "vt_report": { 478 | "type": "string", 479 | "description": "Link to the Virus Total report", 480 | "minOccur": 1, 481 | "maxOccur": 1, 482 | } 483 | } 484 | }, 485 | "filename": { 486 | "type": "string", 487 | "description": "Detected filename. Available only the latest filename", 488 | "minOccur": 1, 489 | "maxOccur": 1, 490 | }, 491 | "filesize": { 492 | "type": "integer", 493 | "description": "File size in bytes", 494 | "minOccur": 1, 495 | "maxOccur": 1, 496 | }, 497 | "mime_type": { 498 | "type": "string", 499 | "description": "File type", 500 | "minOccur": 1, 501 | "maxOccur": 1, 502 | }, 503 | "number_observed": { 504 | "type": "integer", 505 | "description": "How many time the file has been downloaded by the system", 506 | "minOccur": 1, 507 | "maxOccur": 1, 508 | } 509 | } 510 | } 511 | } 512 | }, 513 | "related_urls": { 514 | "description": "The related URLs associated to the submitted item. If no related IoC found, the list will be empty", 515 | "minOccur": 0, 516 | "maxOccur": 1, 517 | "type": "array", 518 | "items": [ 519 | { 520 | "type": "string", 521 | "description": "URLs spreading the hash file submitted", 522 | "minOccur": 0 523 | } 524 | ] 525 | } 526 | }, 527 | "required": [ 528 | "item", 529 | "response", 530 | "response_text" 531 | ] 532 | } 533 | ] 534 | }, 535 | "statistics": { 536 | "type": "object", 537 | "description": "Basic statistics about submitted and found urls", 538 | "minOccur": 1, 539 | "maxOccur": 1, 540 | "properties": { 541 | "itemsFound": { 542 | "type": "integer", 543 | "description": "Numbers of found items", 544 | "minOccur": 1, 545 | "maxOccur": 1 546 | }, 547 | "itemsSubmitted": { 548 | "type": "integer", 549 | "description": "Numbers of submitted items", 550 | "minOccur": 1, 551 | "maxOccur": 1 552 | } 553 | }, 554 | "required": [ 555 | "itemsFound", 556 | "itemsSubmitted" 557 | ] 558 | }, 559 | "list": { 560 | "type": "object", 561 | "properties": { 562 | "file": { 563 | "type": "string", 564 | "description": "The list name used for lookup", 565 | "minOccur": 1, 566 | "maxOccur": 1 567 | }, 568 | "date": { 569 | "type": "string", 570 | "description": "The generated date of the list used for lookup", 571 | "minOccur": 1, 572 | "maxOccur": 1 573 | }, 574 | "url": { 575 | "type": "string", 576 | "description": "The download link will be reported or a cache notice if the list has been loaded from previous generated cache", 577 | "minOccur": 1, 578 | "maxOccur": 1 579 | } 580 | }, 581 | "required": [ 582 | "file", 583 | "date", 584 | "url" 585 | ] 586 | } 587 | }, 588 | "required": [ 589 | "statistics", 590 | "list" 591 | ] 592 | }, 593 | "ip": { 594 | "description": "Response for submitted ips", 595 | "type": "object", 596 | "minOccur": 0, 597 | "maxOccur": 1, 598 | "properties": { 599 | "items": { 600 | "description": "Array listing results", 601 | "minOccur": 1, 602 | "maxOccur": 1, 603 | "items": [ 604 | { 605 | "type": "object", 606 | "properties": { 607 | "item": { 608 | "type": "string", 609 | "description": "The entity to search", 610 | "minOccur": 1, 611 | "maxOccur": 1 612 | }, 613 | "response": { 614 | "type": "boolean", 615 | "description": "'True' if a match has been found. 'False' if no match has been found", 616 | "minOccur": 1, 617 | "maxOccur": 1 618 | }, 619 | "response_text": { 620 | "type": "string", 621 | "description": "Just a human readble representation of the 'response' item", 622 | "minOccur": 1, 623 | "maxOccur": 1 624 | }, 625 | "related_urls": { 626 | "description": "The related URLs associated to the submitted item.", 627 | "minOccur": 0, 628 | "maxOccur": 1, 629 | "type": "array", 630 | "items": [ 631 | { 632 | "type": "object", 633 | "description": "Object containing related URLs and hashes detected", 634 | "minOccur": 0, 635 | "properties": { 636 | "url": { 637 | "type": "string", 638 | "description": "Related URL detected", 639 | "minOccur": 0, 640 | "maxOccur": 1, 641 | }, 642 | "hashes": { 643 | "description": "Hashes related to the found URL", 644 | "minOccur": 0, 645 | "maxOccur": 1, 646 | "type": "object", 647 | "properties": { 648 | "md5": { 649 | "type": "string", 650 | "description": "The MD5 hash related to the submitted URL", 651 | "minOccur": 0, 652 | "maxOccur": 1 653 | }, 654 | "sha1": { 655 | "type": "string", 656 | "description": "The SHA1 hash related to the submitted URL", 657 | "minOccur": 0, 658 | "maxOccur": 1 659 | }, 660 | "sha256": { 661 | "type": "string", 662 | "description": "The SHA256 hash related to the submitted URL", 663 | "minOccur": 0, 664 | "maxOccur": 1 665 | } 666 | } 667 | }, 668 | "online_reports": { 669 | "type": "object", 670 | "description": "Links or local path to the related reports", 671 | "minOccur": 1, 672 | "maxOccur": 1, 673 | "properties": { 674 | "MISP_EVENT": { 675 | "type": "string", 676 | "description": "URL or path to the related MISP event json file", 677 | "minOccur": 1, 678 | "maxOccur": 1 679 | }, 680 | "MISP_CSV": { 681 | "type": "string", 682 | "description": "URL or path to the related MISP event CSV file", 683 | "minOccur": 1, 684 | "maxOccur": 1 685 | }, 686 | "OSINTDS_REPORT": { 687 | "type": "string", 688 | "description": "URL to the related HTML report published at osint.digitalside.it", 689 | "minOccur": 1, 690 | "maxOccur": 1 691 | }, 692 | "STIX": { 693 | "type": "string", 694 | "description": "URL or path to the related STIX2 json file", 695 | "minOccur": 1, 696 | "maxOccur": 1 697 | }, 698 | "STIXDETAILS": { 699 | "type": "object", 700 | "description": "Details extracted by the STIX2 file. Available when option --st, --stix is True", 701 | "minOccur": 0, 702 | "maxOccur": 1, 703 | "properties": { 704 | "observed_time_frame": { 705 | "type": ["boolean", "string"], 706 | "description": "Timedelta between first and last file submission. if 'number_observed' is 1 the value is False", 707 | "minOccur": 1, 708 | "maxOccur": 1, 709 | }, 710 | "indicators_count": { 711 | "type": "object", 712 | "description": "Indicators count per type, as extracted by the related STIX 2 report", 713 | "minOccur": 1, 714 | "maxOccur": 1, 715 | "properties": { 716 | "hashes": { 717 | "type": "integer", 718 | "description": "Hashes count", 719 | "minOccur": 1, 720 | "maxOccur": 1, 721 | }, 722 | "urls": { 723 | "type": "integer", 724 | "description": "Urls count", 725 | "minOccur": 1, 726 | "maxOccur": 1, 727 | }, 728 | "domains": { 729 | "type": "integer", 730 | "description": "Domains count", 731 | "minOccur": 1, 732 | "maxOccur": 1, 733 | }, 734 | "ipv4": { 735 | "type": "integer", 736 | "description": "IPs count", 737 | "minOccur": 1, 738 | "maxOccur": 1, 739 | } 740 | } 741 | }, 742 | "tlp": { 743 | "type": "string", 744 | "description": "Traffic Light Protocol", 745 | "minOccur": 1, 746 | "maxOccur": 1, 747 | }, 748 | "first_observed": { 749 | "type": "string", 750 | "description": "First file submission date", 751 | "minOccur": 1, 752 | "maxOccur": 1, 753 | }, 754 | "last_observed": { 755 | "type": "string", 756 | "description": "Last file submission date", 757 | "minOccur": 1, 758 | "maxOccur": 1, 759 | }, 760 | "virus_total": { 761 | "type": ["object", "boolean"], 762 | "description": "Virus Total report. False in case no VT report is available", 763 | "minOccur": 1, 764 | "maxOccur": 1, 765 | "properties": { 766 | "vt_detection_ratio": { 767 | "type": "string", 768 | "description": "VT detection ratio", 769 | "minOccur": 0, 770 | "maxOccur": 1, 771 | }, 772 | "vt_report": { 773 | "type": "string", 774 | "description": "Link to the Virus Total report", 775 | "minOccur": 1, 776 | "maxOccur": 1, 777 | } 778 | } 779 | }, 780 | "filename": { 781 | "type": "string", 782 | "description": "Detected filename. Available only the latest filename", 783 | "minOccur": 1, 784 | "maxOccur": 1, 785 | }, 786 | "filesize": { 787 | "type": "integer", 788 | "description": "File size in bytes", 789 | "minOccur": 1, 790 | "maxOccur": 1, 791 | }, 792 | "mime_type": { 793 | "type": "string", 794 | "description": "File type", 795 | "minOccur": 1, 796 | "maxOccur": 1, 797 | }, 798 | "number_observed": { 799 | "type": "integer", 800 | "description": "How many time the file has been downloaded by the system", 801 | "minOccur": 1, 802 | "maxOccur": 1, 803 | }, 804 | } 805 | } 806 | } 807 | } 808 | } 809 | } 810 | ] 811 | } 812 | }, 813 | "required": [ 814 | "item", 815 | "response", 816 | "response_text" 817 | ] 818 | } 819 | ] 820 | }, 821 | "statistics": { 822 | "type": "object", 823 | "description": "Basic statistics about submitted and found ips", 824 | "minOccur": 1, 825 | "maxOccur": 1, 826 | "properties": { 827 | "itemsFound": { 828 | "type": "integer", 829 | "description": "Numbers of found items", 830 | "minOccur": 1, 831 | "maxOccur": 1 832 | }, 833 | "itemsSubmitted": { 834 | "type": "integer", 835 | "description": "Numbers of submitted items", 836 | "minOccur": 1, 837 | "maxOccur": 1 838 | } 839 | }, 840 | "required": [ 841 | "itemsFound", 842 | "itemsSubmitted" 843 | ] 844 | }, 845 | "list": { 846 | "type": "object", 847 | "properties": { 848 | "file": { 849 | "type": "string", 850 | "description": "The list name used for lookup", 851 | "minOccur": 1, 852 | "maxOccur": 1 853 | }, 854 | "date": { 855 | "type": "string", 856 | "description": "The generated date of the list used for lookup", 857 | "minOccur": 1, 858 | "maxOccur": 1 859 | }, 860 | "url": { 861 | "type": "string", 862 | "description": "The download link will be reported or a cache notice if the list has been loaded from previous generated cache", 863 | "minOccur": 1, 864 | "maxOccur": 1 865 | } 866 | }, 867 | "required": [ 868 | "file", 869 | "date", 870 | "url" 871 | ] 872 | } 873 | }, 874 | "required": [ 875 | "statistics", 876 | "list" 877 | ] 878 | }, 879 | "domain": { 880 | "description": "Response for submitted domains", 881 | "type": "object", 882 | "minOccur": 0, 883 | "properties": { 884 | "items": { 885 | "description": "Array listing results", 886 | "minOccur": 0, 887 | "items": [ 888 | { 889 | "type": "object", 890 | "properties": { 891 | "item": { 892 | "type": "string", 893 | "description": "The entity to search", 894 | "minOccur": 1, 895 | "maxOccur": 1 896 | }, 897 | "response": { 898 | "type": "boolean", 899 | "description": "'True' if a match has been found. 'False' if no match has been found", 900 | "minOccur": 1, 901 | "maxOccur": 1 902 | }, 903 | "response_text": { 904 | "type": "string", 905 | "description": "Just a human readble representation of the 'response' item", 906 | "minOccur": 1, 907 | "maxOccur": 1 908 | }, 909 | "related_urls": { 910 | "description": "The related URLs associated to the submitted item.", 911 | "minOccur": 0, 912 | "maxOccur": 1, 913 | "type": "array", 914 | "items": [ 915 | { 916 | "type": "object", 917 | "description": "Object containing related URLs and hashes detected", 918 | "minOccur": 0, 919 | "properties": { 920 | "url": { 921 | "type": "string", 922 | "description": "Related URL detected", 923 | "minOccur": 0, 924 | "maxOccur": 1, 925 | }, 926 | "hashes": { 927 | "description": "Hashes related to the found URL", 928 | "minOccur": 0, 929 | "maxOccur": 1, 930 | "type": "object", 931 | "properties": { 932 | "md5": { 933 | "type": "string", 934 | "description": "The MD5 hash related to the submitted URL", 935 | "minOccur": 0, 936 | "maxOccur": 1 937 | }, 938 | "sha1": { 939 | "type": "string", 940 | "description": "The SHA1 hash related to the submitted URL", 941 | "minOccur": 0, 942 | "maxOccur": 1 943 | }, 944 | "sha256": { 945 | "type": "string", 946 | "description": "The SHA256 hash related to the submitted URL", 947 | "minOccur": 0, 948 | "maxOccur": 1 949 | } 950 | } 951 | }, 952 | "online_reports": { 953 | "type": "object", 954 | "description": "Links or local path to the related reports", 955 | "minOccur": 1, 956 | "maxOccur": 1, 957 | "properties": { 958 | "MISP_EVENT": { 959 | "type": "string", 960 | "description": "URL or path to the related MISP event json file", 961 | "minOccur": 1, 962 | "maxOccur": 1 963 | }, 964 | "MISP_CSV": { 965 | "type": "string", 966 | "description": "URL or path to the related MISP event CSV file", 967 | "minOccur": 1, 968 | "maxOccur": 1 969 | }, 970 | "OSINTDS_REPORT": { 971 | "type": "string", 972 | "description": "URL to the related HTML report published at osint.digitalside.it", 973 | "minOccur": 1, 974 | "maxOccur": 1 975 | }, 976 | "STIX": { 977 | "type": "string", 978 | "description": "URL or path to the related STIX2 json file", 979 | "minOccur": 1, 980 | "maxOccur": 1 981 | }, 982 | "STIXDETAILS": { 983 | "type": "object", 984 | "description": "Details extracted by the STIX2 file. Available when option --st, --stix is True", 985 | "minOccur": 0, 986 | "maxOccur": 1, 987 | "properties": { 988 | "observed_time_frame": { 989 | "type": ["boolean", "string"], 990 | "description": "Timedelta between first and last file submission. if 'number_observed' is 1 the value is False", 991 | "minOccur": 1, 992 | "maxOccur": 1, 993 | }, 994 | "indicators_count": { 995 | "type": "object", 996 | "description": "Indicators count per type, as extracted by the related STIX 2 report", 997 | "minOccur": 1, 998 | "maxOccur": 1, 999 | "properties": { 1000 | "hashes": { 1001 | "type": "integer", 1002 | "description": "Hashes count", 1003 | "minOccur": 1, 1004 | "maxOccur": 1, 1005 | }, 1006 | "urls": { 1007 | "type": "integer", 1008 | "description": "Urls count", 1009 | "minOccur": 1, 1010 | "maxOccur": 1, 1011 | }, 1012 | "domains": { 1013 | "type": "integer", 1014 | "description": "Domains count", 1015 | "minOccur": 1, 1016 | "maxOccur": 1, 1017 | }, 1018 | "ipv4": { 1019 | "type": "integer", 1020 | "description": "IPs count", 1021 | "minOccur": 1, 1022 | "maxOccur": 1, 1023 | } 1024 | } 1025 | }, 1026 | "tlp": { 1027 | "type": "string", 1028 | "description": "Traffic Light Protocol", 1029 | "minOccur": 1, 1030 | "maxOccur": 1, 1031 | }, 1032 | "first_observed": { 1033 | "type": "string", 1034 | "description": "First file submission date", 1035 | "minOccur": 1, 1036 | "maxOccur": 1, 1037 | }, 1038 | "last_observed": { 1039 | "type": "string", 1040 | "description": "Last file submission date", 1041 | "minOccur": 1, 1042 | "maxOccur": 1, 1043 | }, 1044 | "virus_total": { 1045 | "type": ["object", "boolean"], 1046 | "description": "Virus Total report. False in case no VT report is available", 1047 | "minOccur": 1, 1048 | "maxOccur": 1, 1049 | "properties": { 1050 | "vt_detection_ratio": { 1051 | "type": "string", 1052 | "description": "VT detection ratio", 1053 | "minOccur": 0, 1054 | "maxOccur": 1, 1055 | }, 1056 | "vt_report": { 1057 | "type": "string", 1058 | "description": "Link to the Virus Total report", 1059 | "minOccur": 1, 1060 | "maxOccur": 1, 1061 | } 1062 | } 1063 | }, 1064 | "filename": { 1065 | "type": "string", 1066 | "description": "Detected filename. Available only the latest filename", 1067 | "minOccur": 1, 1068 | "maxOccur": 1, 1069 | }, 1070 | "filesize": { 1071 | "type": "integer", 1072 | "description": "File size in bytes", 1073 | "minOccur": 1, 1074 | "maxOccur": 1, 1075 | }, 1076 | "mime_type": { 1077 | "type": "string", 1078 | "description": "File type", 1079 | "minOccur": 1, 1080 | "maxOccur": 1, 1081 | }, 1082 | "number_observed": { 1083 | "type": "integer", 1084 | "description": "How many time the file has been downloaded by the system", 1085 | "minOccur": 1, 1086 | "maxOccur": 1, 1087 | } 1088 | 1089 | 1090 | } 1091 | } 1092 | } 1093 | } 1094 | } 1095 | } 1096 | ] 1097 | } 1098 | }, 1099 | "required": [ 1100 | "item", 1101 | "response", 1102 | "response_text" 1103 | ] 1104 | } 1105 | ] 1106 | }, 1107 | "statistics": { 1108 | "type": "object", 1109 | "description": "Basic statistics about submitted and found domains", 1110 | "minOccur": 1, 1111 | "maxOccur": 1, 1112 | "properties": { 1113 | "itemsFound": { 1114 | "type": "integer", 1115 | "description": "Numbers of found items", 1116 | "minOccur": 1, 1117 | "maxOccur": 1 1118 | }, 1119 | "itemsSubmitted": { 1120 | "type": "integer", 1121 | "description": "Numbers of submitted items", 1122 | "minOccur": 1, 1123 | "maxOccur": 1 1124 | } 1125 | }, 1126 | "required": [ 1127 | "itemsFound", 1128 | "itemsSubmitted" 1129 | ] 1130 | }, 1131 | "list": { 1132 | "type": "object", 1133 | "properties": { 1134 | "file": { 1135 | "type": "string", 1136 | "description": "The list name used for lookup", 1137 | "minOccur": 1, 1138 | "maxOccur": 1 1139 | }, 1140 | "date": { 1141 | "type": "string", 1142 | "description": "The generated date of the list used for lookup", 1143 | "minOccur": 1, 1144 | "maxOccur": 1 1145 | }, 1146 | "url": { 1147 | "type": "string", 1148 | "description": "The download link will be reported or a cache notice if the list has been loaded from previous generated cache", 1149 | "minOccur": 1, 1150 | "maxOccur": 1 1151 | } 1152 | }, 1153 | "required": [ 1154 | "file", 1155 | "date", 1156 | "url" 1157 | ] 1158 | } 1159 | }, 1160 | "required": [ 1161 | "statistics", 1162 | "list" 1163 | ] 1164 | }, 1165 | "generalstatistics":{ 1166 | "type": "object", 1167 | "description": "Global statistics about the sumbission", 1168 | "properties": { 1169 | "url": { 1170 | "type": "integer", 1171 | "description": "Number of submitted urls", 1172 | "minOccur": 1, 1173 | "maxOccur": 1 1174 | }, 1175 | "ip": { 1176 | "type": "integer", 1177 | "description": "Number of submitted ips", 1178 | "minOccur": 1, 1179 | "maxOccur": 1 1180 | }, 1181 | "domain": { 1182 | "type": "integer", 1183 | "description": "Number of submitted domains", 1184 | "minOccur": 1, 1185 | "maxOccur": 1 1186 | }, 1187 | "hash": { 1188 | "type": "integer", 1189 | "description": "Number of submitted hashes", 1190 | "minOccur": 1, 1191 | "maxOccur": 1 1192 | }, 1193 | "invalid": { 1194 | "type": "integer", 1195 | "description": "Number of invalid items", 1196 | "minOccur": 1, 1197 | "maxOccur": 1 1198 | }, 1199 | "duplicates": { 1200 | "type": "integer", 1201 | "description": "Number of duplicates", 1202 | "minOccur": 1, 1203 | "maxOccur": 1 1204 | }, 1205 | "itemsFound": { 1206 | "type": "integer", 1207 | "description": "Number of items found", 1208 | "minOccur": 1, 1209 | "maxOccur": 1 1210 | }, 1211 | "itemsSubmitted": { 1212 | "type": "integer", 1213 | "description": "Number of items submitted", 1214 | "minOccur": 1, 1215 | "maxOccur": 1 1216 | }, 1217 | "urlfound": { 1218 | "type": "integer", 1219 | "description": "Number of urls found", 1220 | "minOccur": 1, 1221 | "maxOccur": 1 1222 | }, 1223 | "ipfound": { 1224 | "type": "integer", 1225 | "description": "Number of ips found", 1226 | "minOccur": 1, 1227 | "maxOccur": 1 1228 | }, 1229 | "domainfound": { 1230 | "type": "integer", 1231 | "description": "Number of domains found", 1232 | "minOccur": 1, 1233 | "maxOccur": 1 1234 | }, 1235 | "hashfound": { 1236 | "type": "integer", 1237 | "description": "Number of hashes found", 1238 | "minOccur": 1, 1239 | "maxOccur": 1 1240 | }, 1241 | }, 1242 | "apiosintDSversion":{ 1243 | "type": "string", 1244 | "description": "apiosintDS version", 1245 | "minOccur": 1, 1246 | "maxOccur": 1 1247 | } 1248 | } 1249 | } 1250 | } 1251 | -------------------------------------------------------------------------------- /apiosintDS/utilities/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davidonzo/apiosintDS/513c5dc7f1da09be349062206dac918c17bd4512/apiosintDS/utilities/__init__.py -------------------------------------------------------------------------------- /apiosintDS/utilities/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davidonzo/apiosintDS/513c5dc7f1da09be349062206dac918c17bd4512/apiosintDS/utilities/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /apiosintDS/utilities/__pycache__/logutils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davidonzo/apiosintDS/513c5dc7f1da09be349062206dac918c17bd4512/apiosintDS/utilities/__pycache__/logutils.cpython-310.pyc -------------------------------------------------------------------------------- /apiosintDS/utilities/__pycache__/prettycli.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davidonzo/apiosintDS/513c5dc7f1da09be349062206dac918c17bd4512/apiosintDS/utilities/__pycache__/prettycli.cpython-310.pyc -------------------------------------------------------------------------------- /apiosintDS/utilities/logutils.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import sys 3 | 4 | log = logging.getLogger(__name__) 5 | 6 | class logutils(): 7 | def __init__(self, level, logstream, logconsole): 8 | self.level = level.upper() 9 | self.logstream = logstream 10 | self.logconsole = True if logconsole == False else True 11 | self.dslog = self.dsloginit() 12 | 13 | def dsloginit(self): 14 | myformatter = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" 15 | levels = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] 16 | 17 | logger = logging.getLogger(__name__) 18 | 19 | formatter = logging.Formatter(myformatter) 20 | 21 | if self.logstream: 22 | try: 23 | logfile = logging.FileHandler(self.logstream) 24 | except ImportError as ierror: 25 | log.error(ierror) 26 | log.error("Unable to create log file. Please check if the given path exists and the user has read/write permission.") 27 | exit(0) 28 | else: 29 | logfile = logging.StreamHandler() 30 | 31 | if self.level not in levels: 32 | logfile.setLevel(logging.DEBUG) 33 | logging.warning("Invalid log level "+self.level+". Log level configured to DEBUG") 34 | elif self.level == 'INFO': 35 | logfile.setLevel(logging.INFO) 36 | elif self.level == 'WARNING': 37 | logfile.setLevel(logging.WARNING) 38 | elif self.level == 'ERROR': 39 | logfile.setLevel(logging.ERROR) 40 | elif self.level == 'CRITICAL': 41 | logfile.setLevel(logging.CRITICAL) 42 | else: 43 | logfile.setLevel(logging.DEBUG) 44 | 45 | logfile.setFormatter(formatter) 46 | logger.propagate = self.logconsole 47 | logger.addHandler(logfile) 48 | 49 | return logger 50 | 51 | def info(self, message): 52 | return self.dslog.info(message) 53 | 54 | def warning(self, message): 55 | return self.dslog.warning(message) 56 | 57 | def error(self, message): 58 | return self.dslog.error(message) 59 | 60 | def critical(self, message): 61 | return self.dslog.critical(message) 62 | 63 | def debug(self, message): 64 | return self.dslog.debug(message) 65 | -------------------------------------------------------------------------------- /apiosintDS/utilities/prettycli.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import json 3 | from apiosintDS.modules.scriptinfo import scriptinfo 4 | 5 | class prettycli(): 6 | """ 7 | What a terrible code... -_- please help! 8 | """ 9 | def __init__(self, results, nocolor, logger): 10 | self.results = json.loads(results) 11 | self.logger = logger 12 | self.nocolor = nocolor 13 | 14 | self.HEADER = '\033[95m' if self.nocolor == False else '' 15 | self.OKBLUE = '\033[94m' if self.nocolor == False else '' 16 | self.OKCYAN = '\033[96m' if self.nocolor == False else '' 17 | self.GREEN = '\033[92m' if self.nocolor == False else '' 18 | self.DARKGREEN = '\033[32m' if self.nocolor == False else '' 19 | self.WARNING = '\033[93m' if self.nocolor == False else '' 20 | self.FAIL = '\033[91m' if self.nocolor == False else '' 21 | self.ORANGE = '\033[93m' if self.nocolor == False else '' 22 | self.DARKYELL = '\033[33m' if self.nocolor == False else '' 23 | self.RESET = '\033[0m' if self.nocolor == False else '' 24 | self.GREY = '\033[37m' if self.nocolor == False else '' 25 | self.BOLD = '\033[1m' if self.nocolor == False else '' 26 | self.UNDERLINE = '\033[4m' if self.nocolor == False else '' 27 | self.output = self.output() 28 | 29 | 30 | def partStix(self, stix): 31 | ret = {} 32 | 33 | timeframe = 0 34 | if stix["observed_time_frame"]: 35 | countobtf = len(stix["observed_time_frame"]) 36 | ret["otf"] = stix["observed_time_frame"] 37 | else: 38 | ret["otf"] = "N/A" 39 | counttlp = len(stix["tlp"]) 40 | countfilename = len(stix["filename"]) 41 | countfilesize = len(str(stix["filesize"])) 42 | countmime_type = len(stix["mime_type"]) 43 | countobser = len(str(stix["number_observed"])) 44 | 45 | if countmime_type > 24: 46 | stix["mime_type"] = stix["mime_type"][:21]+"..." 47 | 48 | virustotal = "N/A" 49 | countvt = 3 50 | 51 | if stix["virus_total"] != False: 52 | countvt = len(stix["virus_total"]["vt_detection_ratio"]) 53 | virustotal = stix["virus_total"]["vt_detection_ratio"] 54 | 55 | fl = " | TLP:"+self.BOLD+stix["tlp"]+self.RESET 56 | fl += " | First Seen "+self.BOLD+stix["first_observed"]+self.RESET 57 | fl += " | Last Seen "+self.BOLD+stix["last_observed"]+self.RESET 58 | ret["fl"] = fl 59 | ret["flc"] = counttlp+73 60 | 61 | ret["fn"] = stix["filename"] 62 | ret["fnc"] = countfilename 63 | 64 | ret["dt"] = " | Size: "+self.BOLD+str(stix["filesize"])+self.RESET 65 | ret["dt"] += " | Type: "+self.BOLD+stix["mime_type"]+self.RESET 66 | ret["dt"] += " | Observed: "+self.BOLD+str(stix["number_observed"])+self.RESET 67 | ret["dt"] += " | VT: "+self.BOLD+virustotal+self.RESET 68 | 69 | if "indicators_count" in stix: 70 | ret["ic"] = " | STIX network indicators: URLs => "+self.BOLD+str(stix["indicators_count"]["urls"])+self.RESET 71 | ret["ic"] += " | Domains => "+self.BOLD+str(stix["indicators_count"]["domains"])+self.RESET 72 | ret["ic"] += " | IPs: "+self.BOLD+str(stix["indicators_count"]["ipv4"])+self.RESET 73 | 74 | return ret 75 | 76 | 77 | def parseolReport(self, reports, lenrow, related=False): 78 | ret = "" 79 | check = False 80 | 81 | countmips = 0 82 | countcsv = 0 83 | countods = 0 84 | countstix = 0 85 | if "MISP_EVENT" in reports: 86 | countmips = len(reports["MISP_EVENT"]) 87 | check = True 88 | if "MISP_CSV" in reports: 89 | countcsv = len(reports["MISP_CSV"]) 90 | check = True 91 | if "OSINTDS_REPORT" in reports: 92 | countods = len(reports["OSINTDS_REPORT"]) 93 | check = True 94 | if "STIX" in reports: 95 | check = True 96 | countstix = len(reports["STIX"]) 97 | 98 | if check: 99 | ret += " "+self.BOLD+self.DARKGREEN+"Online Reports"+self.RESET+self.GREY+" (availability depends on data retention)\n"+self.RESET 100 | 101 | if countmips > 0: 102 | ret +=" -> MISP EVENT: "+reports["MISP_EVENT"]+"\n" 103 | if countcsv > 0: 104 | ret +=" -> MISP CSV: "+reports["MISP_CSV"]+"\n" 105 | if countods > 0: 106 | ret +=" -> DS Report: "+reports["OSINTDS_REPORT"]+"\n" 107 | if countstix > 0: 108 | ret +=" -> STIX: "+reports["STIX"] 109 | return ret 110 | 111 | def parseitem(self, item, related=False, host=False): 112 | ret = "" 113 | if related: 114 | ioc = item["url"] 115 | else: 116 | ioc = item["item"] 117 | 118 | if ioc.startswith("http"): 119 | ioc = "hXXp"+ioc[4:] 120 | else: 121 | ioc = ioc.replace(".", "[.]") 122 | coutioc = len(ioc) 123 | 124 | padding = 12 125 | fpadding = 4 126 | linepadding = 0 127 | stixpredict = 0 128 | fncount = 0 129 | 130 | fl = False 131 | if "online_reports" in item: 132 | if "STIXDETAILS" in item["online_reports"]: 133 | fl = self.partStix(item["online_reports"]["STIXDETAILS"]) 134 | stixpredict = fl["flc"] 135 | fncount = fl["fnc"] 136 | 137 | maxrow = (max([coutioc, 77, stixpredict, fncount])) 138 | if maxrow == coutioc: 139 | padding = 8 140 | linepadding = 4 141 | fpadding = 0 142 | 143 | if related: 144 | ret += self.GREY+" -".ljust(maxrow+linepadding, '-')+self.RESET+"\n" 145 | ret += " | "+self.BOLD+self.GREY+ioc.ljust(maxrow-fpadding)+self.RESET+" | \n" 146 | else: 147 | if item["response"]: 148 | ret += self.GREY+" -".ljust(maxrow+linepadding, '-')+self.RESET+"\n" 149 | relan = len(item["related_urls"]) 150 | if self.nocolor: 151 | countrelan = len(str(relan))+coutioc+2 152 | else: 153 | countrelan = len(str(relan))+coutioc-11 154 | reltxt = " - Related URL(s) "+self.BOLD+self.FAIL+str(relan)+self.RESET 155 | if host: 156 | if maxrow > 77: 157 | if self.nocolor: 158 | countrelan = len(str(relan))+18 159 | else: 160 | countrelan = len(str(relan)) 161 | ret += " | "+self.BOLD+self.FAIL+ioc+self.GREY+str(reltxt).ljust(maxrow-countrelan)+self.RESET+" | \n" 162 | else: 163 | ret += " | "+self.BOLD+self.FAIL+ioc.ljust(maxrow-fpadding)+self.RESET+" | \n" 164 | else: 165 | ioc = ioc+" (not found) " 166 | if maxrow > 77: 167 | linepadding = linepadding+13 168 | ret += self.GREY+" -".ljust(maxrow+linepadding, '-')+self.RESET+"\n" 169 | ret += " | "+self.BOLD+self.GREY+ioc.ljust(maxrow-fpadding)+self.RESET+" | \n" 170 | 171 | if fl: 172 | ret += self.GREY+" -".ljust(maxrow+linepadding, '-')+self.RESET+"\n" 173 | linedates = linepadding 174 | if maxrow > 77: 175 | if self.nocolor == False: 176 | linedates = linedates+23 177 | else: 178 | linedates = linedates-1 179 | ret += fl["fl"].ljust(maxrow+linedates)+" | " 180 | ret += "\n" 181 | 182 | if "hashes" in item: 183 | ret += self.GREY+" -".ljust(maxrow+linepadding, '-')+self.RESET+"\n" 184 | if fl: 185 | if len(fl["fn"]) >= maxrow: 186 | fl["fn"] = fl["fn"][:maxrow-padding-5]+"..." 187 | ret += " | Filename: "+self.BOLD+self.FAIL+fl["fn"].ljust(maxrow-padding-2)+self.RESET+" | \n" 188 | ret += self.GREY+" -".ljust(maxrow+linepadding, '-')+self.RESET+"\n" 189 | ret += " | MD5: "+item["hashes"]["md5"].ljust(maxrow-padding)+" | \n" 190 | ret += " | SHA1: "+item["hashes"]["sha1"].ljust(maxrow-padding)+" | \n" 191 | ret += " | SHA256: "+item["hashes"]["sha256"].ljust(maxrow-padding)+" | \n" 192 | ret += self.GREY+" -".ljust(maxrow+linepadding, '-')+self.RESET+"\n" 193 | else: 194 | ret += self.GREY+" -".ljust(maxrow+linepadding, '-')+self.RESET+"\n" 195 | 196 | if fl: 197 | pipepadding = linepadding+8 198 | if maxrow > 77: 199 | if self.nocolor == False: 200 | pipepadding = pipepadding+23 201 | else: 202 | pipepadding = pipepadding-9 203 | ret += fl["dt"].ljust(maxrow+pipepadding)+" | \n" 204 | ret += self.GREY+" -".ljust(maxrow+linepadding, '-')+self.RESET+"\n" 205 | if "otf" in fl: 206 | pipepaddingotf = 28 207 | if maxrow > 78: 208 | if self.nocolor == False: 209 | pipepaddingotf = pipepaddingotf 210 | else: 211 | pipepaddingotf = pipepaddingotf 212 | ret += " | Observation time frame: "+fl["otf"].ljust(maxrow-pipepaddingotf)+" | \n" 213 | ret += self.GREY+" -".ljust(maxrow+linepadding, '-')+self.RESET+"\n" 214 | 215 | if "ic" in fl: 216 | pipepaddingotf = 23 217 | if self.nocolor: 218 | pipepaddingotf = -1 219 | 220 | if maxrow > 78: 221 | if self.nocolor == False: 222 | pipepaddingotf = pipepaddingotf 223 | else: 224 | pipepaddingotf = pipepaddingotf 225 | ret += fl["ic"].ljust(maxrow+pipepaddingotf)+" | \n" 226 | ret += self.GREY+" -".ljust(maxrow+linepadding, '-')+self.RESET+"\n" 227 | 228 | if "online_reports" in item: 229 | ret += self.parseolReport(item["online_reports"], maxrow, related) 230 | ret += "\n" 231 | return ret 232 | 233 | def showsummary(self): 234 | stats = self.results["generalstatistics"] 235 | total = stats["itemsSubmitted"]+stats["duplicates"]+stats["invalid"] 236 | itemsnotfoud = stats["itemsSubmitted"]-stats["itemsFound"] 237 | emptystring = "" 238 | 239 | 240 | countletterparsed = len(str(total)) 241 | countnotfound = len(str(itemsnotfoud)) 242 | countinvalid = len(str(stats["duplicates"])) 243 | countduplicates = len(str(stats["duplicates"])) 244 | countsubmitted = len(str(stats["itemsSubmitted"])) 245 | countsuburl = len(str(stats["url"])) 246 | countsubdomain = len(str(stats["domain"])) 247 | countip = len(str(stats["ip"])) 248 | countsubhash = len(str(stats["hash"])) 249 | countfound = len(str(stats["itemsFound"])) 250 | countfoundurl = len(str(stats["urlfound"])) 251 | countfounddomain = len(str(stats["domainfound"])) 252 | countfoundip = len(str(stats["ipfound"])) 253 | countfoundhash = len(str(stats["hashfound"])) 254 | 255 | maxfirstColumn = max([countletterparsed, countinvalid, countduplicates, countnotfound]) 256 | maxsecondColums = max([countsubmitted, countsuburl, countsubdomain, countip, countsubhash]) 257 | maxterColums = max([countfound, countfoundurl, countfounddomain, countfoundip, countfoundhash]) 258 | 259 | 260 | Iparsed = "\n | Items parsed: "+str(total).ljust(maxfirstColumn)+self.RESET+" | " 261 | Iinvalid = "\n | Invalid(s): "+str(stats["invalid"]).ljust(maxfirstColumn)+self.RESET+" | " 262 | Iduplicates = "\n | Duplicate(s): "+str(stats["duplicates"]).ljust(maxfirstColumn)+self.RESET+" | " 263 | Inotfound = "\n | Not found: "+str(itemsnotfoud).ljust(maxfirstColumn)+self.RESET+" | " 264 | emptyline = "\n | "+emptystring.ljust(maxfirstColumn)+" | " 265 | 266 | Isumbitted = "Items submitted: "+self.ORANGE+self.BOLD+str(stats["itemsSubmitted"]).ljust(maxsecondColums)+self.RESET+" | " #17 267 | Iurl = "URL(s): "+self.ORANGE+self.BOLD+str(stats["url"]).ljust(maxsecondColums)+self.RESET+" | " 268 | Idomain = "Domain(s): "+self.ORANGE+self.BOLD+str(stats["domain"]).ljust(maxsecondColums)+self.RESET+" | " 269 | Iip = "IP(s): "+self.ORANGE+self.BOLD+str(stats["ip"]).ljust(maxsecondColums)+self.RESET+" | " 270 | Ihash = "Hash(es): "+self.ORANGE+self.BOLD+str(stats["hash"]).ljust(maxsecondColums)+self.RESET+" | " 271 | 272 | Ifound = "Items found: "+self.FAIL+self.BOLD+str(stats["itemsFound"]).ljust(maxterColums)+self.RESET+" | " #17 273 | Ifoundurl = "URL(s): "+self.FAIL+self.BOLD+str(stats["urlfound"]).ljust(maxterColums)+self.RESET+" | " 274 | Ifounddomain = "Domain(s): "+self.FAIL+self.BOLD+str(stats["domainfound"]).ljust(maxterColums)+self.RESET+" | " 275 | Ifoundip = "IP(s): "+self.FAIL+self.BOLD+str(stats["ipfound"]).ljust(maxterColums)+self.RESET+" | " 276 | Ifoundhash = "Hash(es): "+self.FAIL+self.BOLD+str(stats["hashfound"]).ljust(maxterColums)+self.RESET+" | " 277 | 278 | maxrow = max([len(Iparsed+Isumbitted+Ifound), 279 | len(Iinvalid+Iurl+Ifoundurl), 280 | len(Iduplicates+Idomain+Ifoundurl), 281 | len(Inotfound+Iip+Ifoundip), 282 | len(emptyline+Ihash+Ifoundhash)]) 283 | 284 | paddinglines = 33 285 | if self.nocolor: 286 | paddinglines = 3 287 | 288 | ret = self.BOLD+self.DARKGREEN+" Submission summary\n"+self.RESET 289 | ret += self.GREY+" -".ljust(maxrow-paddinglines, '-')+self.RESET 290 | ret += Iparsed 291 | ret += Isumbitted 292 | ret += Ifound 293 | ret += "\n"+self.GREY+" -".ljust(maxrow-paddinglines, '-')+self.RESET 294 | ret += Iinvalid 295 | ret += Iurl 296 | ret += Ifoundurl 297 | ret += Iduplicates 298 | ret += Ihash 299 | ret += Ifoundhash 300 | ret += Inotfound 301 | ret += Idomain 302 | ret += Ifounddomain 303 | ret += emptyline 304 | ret += Iip 305 | ret += Ifoundip 306 | ret += "\n"+self.GREY+" -".ljust(maxrow-paddinglines, '-')+self.RESET 307 | ret += "\n" 308 | 309 | return ret 310 | 311 | def logo(self): 312 | version = self.BOLD+self.GREEN+"v."+scriptinfo["majorversion"]+"."+scriptinfo["minorversion"]+self.RESET+self.GREEN 313 | subtitle = self.ORANGE+scriptinfo["subscriptname"]+self.RESET+self.GREEN 314 | return """%s 315 | _ _ _ ____ ____ 316 | __ _ _ __ (_) ___ ___(_)_ __ | |_| _ \/ ___| 317 | / _` | '_ \| |/ _ \/ __| | '_ \| __| | | \___ \ 318 | | (_| | |_) | | (_) \__ \ | | | | |_| |_| |___) | 319 | \__,_| .__/|_|\___/|___/_|_| |_|\__|____/|____/ %s 320 | |_|%s%s 321 | \n""" % (self.GREEN, version, subtitle, self.RESET) 322 | 323 | 324 | 325 | 326 | def output(self): 327 | logo = self.logo() 328 | summary = self.showsummary() 329 | 330 | tmpparseitem = "" 331 | if "url" in self.results: 332 | for result in self.results["url"]["items"]: 333 | tmpparseitem += self.parseitem(result) 334 | tmpparseitem += self.GREY+"#############################################################################\n\n"+self.RESET 335 | if "hash" in self.results: 336 | for result in self.results["hash"]["items"]: 337 | tmpparseitem += self.parseitem(result) 338 | tmpparseitem += self.GREY+"#############################################################################\n\n"+self.RESET 339 | if "ip" in self.results: 340 | for result in self.results["ip"]["items"]: 341 | tmpparseitem += self.parseitem(result, False, True) 342 | if "related_urls" in result: 343 | for related in result["related_urls"]: 344 | tmpparseitem += self.parseitem(related, True, False) 345 | tmpparseitem += self.GREY+"#############################################################################\n\n"+self.RESET 346 | if "domain" in self.results: 347 | for result in self.results["domain"]["items"]: 348 | tmpparseitem += self.parseitem(result, False, True) 349 | if "related_urls" in result: 350 | for related in result["related_urls"]: 351 | tmpparseitem += self.parseitem(related, True, False) 352 | tmpparseitem += self.GREY+"#".ljust(130, '#')+"\n\n"+self.RESET 353 | 354 | ret = logo+summary+tmpparseitem 355 | return ret 356 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/_static/css/custom.css: -------------------------------------------------------------------------------- 1 | img { 2 | border: 1px solid #cfcfcf; 3 | margin-bottom: 1em; 4 | } 5 | -------------------------------------------------------------------------------- /docs/_static/img/apiosintDS.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davidonzo/apiosintDS/513c5dc7f1da09be349062206dac918c17bd4512/docs/_static/img/apiosintDS.png -------------------------------------------------------------------------------- /docs/_static/img/mispmoduleconfiguration.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davidonzo/apiosintDS/513c5dc7f1da09be349062206dac918c17bd4512/docs/_static/img/mispmoduleconfiguration.png -------------------------------------------------------------------------------- /docs/_static/img/moduleenrich.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davidonzo/apiosintDS/513c5dc7f1da09be349062206dac918c17bd4512/docs/_static/img/moduleenrich.png -------------------------------------------------------------------------------- /docs/_static/img/modulehover.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davidonzo/apiosintDS/513c5dc7f1da09be349062206dac918c17bd4512/docs/_static/img/modulehover.png -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx-toolbox==3.5.0 2 | recommonmark==0.7.1 3 | sphinx_external_toc==1.0.1 4 | sphinx-rtd-theme==2.0.0 5 | -------------------------------------------------------------------------------- /docs/source/_toc.yml: -------------------------------------------------------------------------------- 1 | root: index 2 | options: 3 | maxdepth: 1 4 | entries: 5 | - url: https://apiosintds.readthedocs.io/en/latest/index.html 6 | title: apiosintDS v2.0 7 | - file: install 8 | title: Installation guide 9 | - file: userguidecli 10 | title: Usage via command line (CLI) 11 | - file: cliexamples 12 | title: Command Line examples 13 | - file: userguideapi 14 | title: Using as Python library 15 | - file: userguidemisp 16 | title: MISP Module 17 | - file: changes 18 | title: Changelog 19 | - file: license 20 | title: License 21 | - url: https://github.com/davidonzo/apiosintDS 22 | title: apiosintDS Project 23 | - url: https://osint.digitalside.it 24 | title: OSINT.DigitalSide.IT 25 | -------------------------------------------------------------------------------- /docs/source/changes.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../../CHANGELOG.rst 2 | 3 | -------------------------------------------------------------------------------- /docs/source/cliexamples.rst: -------------------------------------------------------------------------------- 1 | ============================ 2 | Command Line examples 3 | ============================ 4 | 5 | .. code-block:: bash 6 | 7 | ~$ apiosintDS 8 | usage: apiosintDS [-h] [-e [IPv4|domain|url|hash]] [-f /path/to/file.txt] [-st] 9 | [-o /path/to/output.json] [-p] [-nc] [-v] 10 | [-c] [-cd /path/to/cachedir] [-ct [0-9]] [-cc] 11 | [-ld /path/to/git/clone/Threat-Intel/] [-ll [DEBUG|INFO|WARNING|ERROR|CRITICAL]] 12 | [-l /path/to/logfile.log] [-lc] [-i] [-s] [-vv] 13 | apiosintDS: error: No targets selected! Please, specify one option between --entity and --file. 14 | Try option -h or --help. 15 | 16 | One item using :confval:`--pretty` 17 | ================================== 18 | .. code-block:: bash 19 | 20 | $ apiosintDS -e h[REMOVED]p://193.35.18.147/bins/k.arm -st -p -nc 21 | _ _ _ ____ ____ 22 | __ _ _ __ (_) ___ ___(_)_ __ | |_| _ \/ ___| 23 | / _` | '_ \| |/ _ \/ __| | '_ \| __| | | \___ \ 24 | | (_| | |_) | | (_) \__ \ | | | | |_| |_| |___) | 25 | \__,_| .__/|_|\___/|___/_|_| |_|\__|____/|____/ v.2.0.3 26 | |_|OSINT.DigitalSide.IT Threat-Intel Repository 27 | 28 | Submission summary 29 | ------------------------------------------------------- 30 | | Items parsed: 1 | Items submitted: 1 | Items found: 1 | 31 | ------------------------------------------------------- 32 | | Invalid(s): 0 | URL(s): 1 | URL(s): 1 | 33 | | Duplicate(s): 0 | Hash(es): 0 | Hash(es): 0 | 34 | | Not found: 0 | Domain(s): 0 | Domain(s): 0 | 35 | | | IP(s): 0 | IP(s): 0 | 36 | ------------------------------------------------------- 37 | ---------------------------------------------------------------------------- 38 | | hXXp://193.35.18.147/bins/k.arm | 39 | ---------------------------------------------------------------------------- 40 | | TLP:white | First Seen 2023-07-06 07:36:02 | Last Seen 2023-07-06 07:36:02 | 41 | ---------------------------------------------------------------------------- 42 | | Filename: k.arm | 43 | ---------------------------------------------------------------------------- 44 | | MD5: bc152acad73829358847e5f5bbf3edc0 | 45 | | SHA1: f2e26e44709ba5a9766c3c00226bdb663ede5957 | 46 | | SHA256: c8b0e1c5fa98bb407fe5bd3f2760b0ec2e5e33db0cee10a0085cac4505ef16cc | 47 | ---------------------------------------------------------------------------- 48 | | Size: 244647 | Type: application/x-executable | Observed: 1 | VT: 34/61 | 49 | ---------------------------------------------------------------------------- 50 | | Observation time frame: N/A | 51 | ---------------------------------------------------------------------------- 52 | | STIX network indicators: URLs => 1 | Domains => 0 | IPs: 1 | 53 | ---------------------------------------------------------------------------- 54 | Online Reports (availability depends on data retention) 55 | -> MISP EVENT: https://osint.digitalside.it/Threat-Intel/digitalside-misp-feed/f5e313d2-3d64-4d0f-af77-37a925bcd08f.json 56 | -> MISP CSV: https://osint.digitalside.it/Threat-Intel/csv/f5e313d2-3d64-4d0f-af77-37a925bcd08f.csv 57 | -> DS Report: https://osint.digitalside.it/report/bc152acad73829358847e5f5bbf3edc0.html 58 | -> STIX: https://osint.digitalside.it/Threat-Intel/stix2/bc152acad73829358847e5f5bbf3edc0.json 59 | ############################################################################# 60 | 61 | 62 | Multiple items using :confval:`--file` with :confval:`--pretty` output 63 | ====================================================================== 64 | 65 | Example file ioc.txt. 66 | 67 | .. code-block:: bash 68 | 69 | ~$ cat ioc.txt 70 | 7cb796c875cccc9233d82854a4e2fdf0 71 | monke.re 72 | 73 | Response. 74 | 75 | .. code-block:: bash 76 | 77 | ~$ apiosintDS -f ioc.txt -p -nc -st 78 | 79 | _ _ _ ____ ____ 80 | __ _ _ __ (_) ___ ___(_)_ __ | |_| _ \/ ___| 81 | / _` | '_ \| |/ _ \/ __| | '_ \| __| | | \___ \ 82 | | (_| | |_) | | (_) \__ \ | | | | |_| |_| |___) | 83 | \__,_| .__/|_|\___/|___/_|_| |_|\__|____/|____/ v.2.0.3 84 | |_|OSINT.DigitalSide.IT Threat-Intel Repository 85 | 86 | Submission summary 87 | ------------------------------------------------------- 88 | | Items parsed: 2 | Items submitted: 2 | Items found: 2 | 89 | ------------------------------------------------------- 90 | | Invalid(s): 0 | URL(s): 0 | URL(s): 0 | 91 | | Duplicate(s): 0 | Hash(es): 1 | Hash(es): 1 | 92 | | Not found: 0 | Domain(s): 1 | Domain(s): 1 | 93 | | | IP(s): 0 | IP(s): 0 | 94 | ------------------------------------------------------- 95 | ---------------------------------------------------------------------------- 96 | | 7cb796c875cccc9233d82854a4e2fdf0 | 97 | ---------------------------------------------------------------------------- 98 | | TLP:white | First Seen 2023-07-04 09:33:03 | Last Seen 2023-07-04 09:33:03 | 99 | ---------------------------------------------------------------------------- 100 | | Filename: plugmanzx.exe | 101 | ---------------------------------------------------------------------------- 102 | | MD5: 7cb796c875cccc9233d82854a4e2fdf0 | 103 | | SHA1: 158514acfa87d0b99e2af07a28004480bbf66e83 | 104 | | SHA256: 49e64d72d5ed4fb7967da4b6851d94cdceffe4ba0316587767a13901fe580239 | 105 | ---------------------------------------------------------------------------- 106 | | Size: 924672 | Type: application/x-dosexec | Observed: 1 | VT: 32/71 | 107 | ---------------------------------------------------------------------------- 108 | | Observation time frame: N/A | 109 | ---------------------------------------------------------------------------- 110 | | STIX network indicators: URLs => 1 | Domains => 0 | IPs: 1 | 111 | ---------------------------------------------------------------------------- 112 | Online Reports (availability depends on data retention) 113 | -> MISP EVENT: https://osint.digitalside.it/Threat-Intel/digitalside-misp-feed/d6146389-4294-4a41-b4ca-6e74c74b7f8b.json 114 | -> MISP CSV: https://osint.digitalside.it/Threat-Intel/csv/d6146389-4294-4a41-b4ca-6e74c74b7f8b.csv 115 | -> DS Report: https://osint.digitalside.it/report/7cb796c875cccc9233d82854a4e2fdf0.html 116 | -> STIX: https://osint.digitalside.it/Threat-Intel/stix2/7cb796c875cccc9233d82854a4e2fdf0.json 117 | ############################################################################# 118 | 119 | --------------------------------------------------------------------------- 120 | | monke[.]re - Related URL(s) 2 | 121 | --------------------------------------------------------------------------- 122 | ---------------------------------------------------------------------------- 123 | | hXXp://monke.re/arm7 | 124 | ---------------------------------------------------------------------------- 125 | | TLP:white | First Seen 2023-07-06 23:51:01 | Last Seen 2023-07-06 23:51:01 | 126 | ---------------------------------------------------------------------------- 127 | | Filename: arm7 | 128 | ---------------------------------------------------------------------------- 129 | | MD5: 318323c9da34bf25833f7da32eab23d6 | 130 | | SHA1: e2bb927b08ebcbaad8f304d02309af776312c9bf | 131 | | SHA256: bb1f9e108daa389e62b79067d1cdbef548f9934c9cc85a92565da7063cf36f89 | 132 | ---------------------------------------------------------------------------- 133 | | Size: 57148 | Type: application/x-executable | Observed: 1 | VT: 14/61 | 134 | ---------------------------------------------------------------------------- 135 | | Observation time frame: N/A | 136 | ---------------------------------------------------------------------------- 137 | | STIX network indicators: URLs => 1 | Domains => 1 | IPs: 0 | 138 | ---------------------------------------------------------------------------- 139 | Online Reports (availability depends on data retention) 140 | -> MISP EVENT: https://osint.digitalside.it/Threat-Intel/digitalside-misp-feed/f83d06e6-aa2f-452e-a19d-59d40e874355.json 141 | -> MISP CSV: https://osint.digitalside.it/Threat-Intel/csv/f83d06e6-aa2f-452e-a19d-59d40e874355.csv 142 | -> DS Report: https://osint.digitalside.it/report/318323c9da34bf25833f7da32eab23d6.html 143 | -> STIX: https://osint.digitalside.it/Threat-Intel/stix2/318323c9da34bf25833f7da32eab23d6.json 144 | ---------------------------------------------------------------------------- 145 | | hXXp://monke.re/mips | 146 | ---------------------------------------------------------------------------- 147 | | TLP:white | First Seen 2023-07-07 00:31:02 | Last Seen 2023-07-07 00:31:02 | 148 | ---------------------------------------------------------------------------- 149 | | Filename: mips | 150 | ---------------------------------------------------------------------------- 151 | | MD5: 579081f528d9279a87b298b9838c377b | 152 | | SHA1: 45048073aad5997881dffe41e32f9b17beb1c2e1 | 153 | | SHA256: 8186a1d140631e6391978c08c35e01efb58963f65a86fddf7dec44eec7681c6b | 154 | ---------------------------------------------------------------------------- 155 | | Size: 48272 | Type: application/x-executable | Observed: 1 | VT: 12/61 | 156 | ---------------------------------------------------------------------------- 157 | | Observation time frame: N/A | 158 | ---------------------------------------------------------------------------- 159 | | STIX network indicators: URLs => 1 | Domains => 1 | IPs: 0 | 160 | ---------------------------------------------------------------------------- 161 | Online Reports (availability depends on data retention) 162 | -> MISP EVENT: https://osint.digitalside.it/Threat-Intel/digitalside-misp-feed/d01c2ad1-0e2c-4b26-9725-f8a86025bd75.json 163 | -> MISP CSV: https://osint.digitalside.it/Threat-Intel/csv/d01c2ad1-0e2c-4b26-9725-f8a86025bd75.csv 164 | -> DS Report: https://osint.digitalside.it/report/579081f528d9279a87b298b9838c377b.html 165 | -> STIX: https://osint.digitalside.it/Threat-Intel/stix2/579081f528d9279a87b298b9838c377b.json 166 | ################################################################################################################################## 167 | 168 | Multiple items using :confval:`--file` with ``JSON`` output 169 | =========================================================== 170 | 171 | Example file ioc.txt. 172 | 173 | .. code-block:: bash 174 | 175 | ~$ cat ioc.txt 176 | 7cb796c875cccc9233d82854a4e2fdf0 177 | monke.re 178 | 179 | Response. 180 | 181 | .. code-block:: bash 182 | 183 | ~$ apiosintDS -f ioc.txt -st 184 | 185 | { 186 | "domain": { 187 | "items": [ 188 | { 189 | "item": "monke.re", 190 | "response": true, 191 | "response_text": "Item found in latestdomains.txt list", 192 | "related_urls": [ 193 | { 194 | "url": "h[REMOVED]p://monke.re/arm7", 195 | "hashes": { 196 | "md5": "318323c9da34bf25833f7da32eab23d6", 197 | "sha1": "e2bb927b08ebcbaad8f304d02309af776312c9bf", 198 | "sha256": "bb1f9e108daa389e62b79067d1cdbef548f9934c9cc85a92565da7063cf36f89" 199 | }, 200 | "online_reports": { 201 | "MISP_EVENT": "https://osint.digitalside.it/Threat-Intel/digitalside-misp-feed/f83d06e6-aa2f-452e-a19d-59d40e874355.json", 202 | "MISP_CSV": "https://osint.digitalside.it/Threat-Intel/csv/f83d06e6-aa2f-452e-a19d-59d40e874355.csv", 203 | "OSINTDS_REPORT": "https://osint.digitalside.it/report/318323c9da34bf25833f7da32eab23d6.html", 204 | "STIX": "https://osint.digitalside.it/Threat-Intel/stix2/318323c9da34bf25833f7da32eab23d6.json", 205 | "STIXDETAILS": { 206 | "observed_time_frame": false, 207 | "indicators_count": { 208 | "hashes": 3, 209 | "urls": 1, 210 | "domains": 1, 211 | "ipv4": 0 212 | }, 213 | "tlp": "white", 214 | "first_observed": "2023-07-06 23:51:01", 215 | "last_observed": "2023-07-06 23:51:01", 216 | "virus_total": { 217 | "vt_detection_ratio": "14/61", 218 | "vt_report": "https://www.virustotal.com/gui/file/bb1f9e108daa389e62b79067d1cdbef548f9934c9cc85a92565da7063cf36f89/detection" 219 | }, 220 | "filename": "arm7", 221 | "filesize": 57148, 222 | "mime_type": "application/x-executable", 223 | "number_observed": 1 224 | } 225 | } 226 | }, 227 | { 228 | "url": "h[REMOVED]p://monke.re/mips", 229 | "hashes": { 230 | "md5": "579081f528d9279a87b298b9838c377b", 231 | "sha1": "45048073aad5997881dffe41e32f9b17beb1c2e1", 232 | "sha256": "8186a1d140631e6391978c08c35e01efb58963f65a86fddf7dec44eec7681c6b" 233 | }, 234 | "online_reports": { 235 | "MISP_EVENT": "https://osint.digitalside.it/Threat-Intel/digitalside-misp-feed/d01c2ad1-0e2c-4b26-9725-f8a86025bd75.json", 236 | "MISP_CSV": "https://osint.digitalside.it/Threat-Intel/csv/d01c2ad1-0e2c-4b26-9725-f8a86025bd75.csv", 237 | "OSINTDS_REPORT": "https://osint.digitalside.it/report/579081f528d9279a87b298b9838c377b.html", 238 | "STIX": "https://osint.digitalside.it/Threat-Intel/stix2/579081f528d9279a87b298b9838c377b.json", 239 | "STIXDETAILS": { 240 | "observed_time_frame": false, 241 | "indicators_count": { 242 | "hashes": 3, 243 | "urls": 1, 244 | "domains": 1, 245 | "ipv4": 0 246 | }, 247 | "tlp": "white", 248 | "first_observed": "2023-07-07 00:31:02", 249 | "last_observed": "2023-07-07 00:31:02", 250 | "virus_total": { 251 | "vt_detection_ratio": "12/61", 252 | "vt_report": "https://www.virustotal.com/gui/file/8186a1d140631e6391978c08c35e01efb58963f65a86fddf7dec44eec7681c6b/detection" 253 | }, 254 | "filename": "mips", 255 | "filesize": 48272, 256 | "mime_type": "application/x-executable", 257 | "number_observed": 1 258 | } 259 | } 260 | } 261 | ] 262 | } 263 | ], 264 | "statistics": { 265 | "itemsFound": 1, 266 | "itemsSubmitted": 1 267 | }, 268 | "list": { 269 | "file": "latestdomains.txt", 270 | "date": "2023-07-07 08:03:07+02:00", 271 | "url": "https://raw.githubusercontent.com/davidonzo/Threat-Intel/master/lists/latestdomains.txt" 272 | } 273 | }, 274 | "hash": { 275 | "items": [ 276 | { 277 | "item": "7cb796c875cccc9233d82854a4e2fdf0", 278 | "response": true, 279 | "response_text": "Item found in latesthashes.json list", 280 | "hashes": { 281 | "md5": "7cb796c875cccc9233d82854a4e2fdf0", 282 | "sha1": "158514acfa87d0b99e2af07a28004480bbf66e83", 283 | "sha256": "49e64d72d5ed4fb7967da4b6851d94cdceffe4ba0316587767a13901fe580239" 284 | }, 285 | "online_reports": { 286 | "MISP_EVENT": "https://osint.digitalside.it/Threat-Intel/digitalside-misp-feed/d6146389-4294-4a41-b4ca-6e74c74b7f8b.json", 287 | "MISP_CSV": "https://osint.digitalside.it/Threat-Intel/csv/d6146389-4294-4a41-b4ca-6e74c74b7f8b.csv", 288 | "OSINTDS_REPORT": "https://osint.digitalside.it/report/7cb796c875cccc9233d82854a4e2fdf0.html", 289 | "STIX": "https://osint.digitalside.it/Threat-Intel/stix2/7cb796c875cccc9233d82854a4e2fdf0.json", 290 | "STIXDETAILS": { 291 | "observed_time_frame": false, 292 | "indicators_count": { 293 | "hashes": 3, 294 | "urls": 1, 295 | "domains": 0, 296 | "ipv4": 1 297 | }, 298 | "tlp": "white", 299 | "first_observed": "2023-07-04 09:33:03", 300 | "last_observed": "2023-07-04 09:33:03", 301 | "virus_total": { 302 | "vt_detection_ratio": "32/71", 303 | "vt_report": "https://www.virustotal.com/gui/file/49e64d72d5ed4fb7967da4b6851d94cdceffe4ba0316587767a13901fe580239/detection" 304 | }, 305 | "filename": "plugmanzx.exe", 306 | "filesize": 924672, 307 | "mime_type": "application/x-dosexec", 308 | "number_observed": 1 309 | } 310 | }, 311 | "related_urls": [ 312 | "h[REMOVED]p://185.246.220.60/plugmanzx.exe" 313 | ] 314 | } 315 | ], 316 | "statistics": { 317 | "itemsFound": 1, 318 | "itemsSubmitted": 1 319 | }, 320 | "list": { 321 | "file": "latesthashes.json", 322 | "date": "2023-07-07 08:03:29+02:00", 323 | "url": "https://raw.githubusercontent.com/davidonzo/Threat-Intel/master/lists/latesthashes.json" 324 | } 325 | }, 326 | "generalstatistics": { 327 | "url": 0, 328 | "ip": 0, 329 | "domain": 1, 330 | "hash": 1, 331 | "invalid": 0, 332 | "duplicates": 0, 333 | "itemsFound": 2, 334 | "itemsSubmitted": 2, 335 | "urlfound": 0, 336 | "ipfound": 0, 337 | "domainfound": 1, 338 | "hashfound": 1 339 | }, 340 | "apiosintDSversion": "apiosintDS v.2.0.3" 341 | } 342 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | # import os 14 | # import sys 15 | # sys.path.insert(0, os.path.abspath('.')) 16 | 17 | 18 | # -- Project information ----------------------------------------------------- 19 | 20 | project = 'apiosintDS' 21 | copyright = '2023, Davide Baglieri' 22 | author = 'Davide Baglieri' 23 | 24 | # The full version, including alpha/beta/rc tags 25 | release = '2.0' 26 | 27 | 28 | # -- General configuration --------------------------------------------------- 29 | 30 | # Add any Sphinx extension module names here, as strings. They can be 31 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 32 | # ones. 33 | extensions = ['recommonmark', 'sphinx_toolbox.confval', 'sphinx_external_toc'] 34 | 35 | # Add any paths that contain templates here, relative to this directory. 36 | templates_path = ['_templates'] 37 | 38 | # List of patterns, relative to source directory, that match files and 39 | # directories to ignore when looking for source files. 40 | # This pattern also affects html_static_path and html_extra_path. 41 | exclude_patterns = [] 42 | 43 | 44 | # -- Options for HTML output ------------------------------------------------- 45 | 46 | # The theme to use for HTML and HTML Help pages. See the documentation for 47 | # a list of builtin themes. 48 | # 49 | html_theme = 'sphinx_rtd_theme' 50 | 51 | # Add any paths that contain custom static files (such as style sheets) here, 52 | # relative to this directory. They are copied after the builtin static files, 53 | # so a file named "default.css" will overwrite the builtin "default.css". 54 | html_static_path = ['../_static'] 55 | 56 | html_css_files = [ 57 | 'css/custom.css', 58 | ] 59 | 60 | external_toc_path = "_toc.yml" 61 | external_toc_exclude_missing = False 62 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | =========== 2 | apiosintDS 3 | =========== 4 | 5 | Latest stable release is **v2.0** (:doc:`Changelog `) 6 | 7 | **apiosintDS** is a `python client library `_ for public *API* lookup service over *OSINT* IoCs stored at `DigitalSide Threat-Intel `_ repository. It can be defined a **service as a library** tool designed to act both as a standard Python library to be included in your own Python application and as command line tool. Query can be performed against souspicious IPs, domains, urls and file hashes. Data stored has a 7 days retention. 8 | 9 | .. image:: ../_static/img/apiosintDS.png 10 | :width: 640 11 | :alt: apiosintDS v2.0 12 | 13 | 14 | `DigitalSide Threat-Intel (also on GitHub.com) `_ shares a set of **Open Source Cyber Threat Intellegence** information, monstly based on malware analysis and compromised URLs, IPs and domains. The purpose of the project is to develop and test new wayes to hunt, analyze, collect and share relevants sets of IoCs to be used by SOC/CSIRT/CERT with minimun effort. 15 | 16 | This library has been specially designed for people and organizations don't want to import the whole `DigitalSide Threat-Intel `_ dataset and prefer to use it as an on demand service. 17 | 18 | Welcome to apiosintDS's documentation! 19 | ====================================== 20 | 21 | .. rubric:: Documentation contents 22 | 23 | .. tableofcontents:: 24 | 25 | -------------------------------------------------------------------------------- /docs/source/install.rst: -------------------------------------------------------------------------------- 1 | ================== 2 | Installation guide 3 | ================== 4 | 5 | Install python > 3.5.2 6 | ====================== 7 | 8 | Make sure you installed on your system python > 3.5.2. Try typing ``python3`` on your terminal. 9 | 10 | .. code-block:: bash 11 | 12 | ~$ python3 13 | Python 3.6.8 (default, Oct 7 2019, 12:59:55) 14 | [GCC 8.3.0] on linux 15 | Type "help", "copyright", "credits" or "license" for more information. 16 | >>> 17 | 18 | Install from sources 19 | ==================== 20 | 21 | Make sure you installed ``python3-setuptools`` and ``git`` packages on your system. If not, install missings according your distribution. 22 | 23 | .. code-block:: bash 24 | 25 | ~$ cd /your/path/src/ 26 | ~$ git clone https://github.com/davidonzo/apiosintDS.git 27 | ~$ cd apiosintDS/ 28 | ~$ python3 -m pip install . 29 | 30 | Install via pip3 31 | ================ 32 | 33 | Make sure you installed ``python3-pip`` package on your system. If not, install it according your distribution. 34 | 35 | .. code-block:: bash 36 | 37 | ~$ pip3 install apiosintDS 38 | 39 | Windows support 40 | =============== 41 | 42 | The library has never been tested on Windows platform. Actually only UNIX system are supported. 43 | -------------------------------------------------------------------------------- /docs/source/license.rst: -------------------------------------------------------------------------------- 1 | License 2 | ========= 3 | 4 | .. include:: ../../LICENSE.txt 5 | 6 | -------------------------------------------------------------------------------- /docs/source/userguideapi.rst: -------------------------------------------------------------------------------- 1 | ======================= 2 | Using as Python library 3 | ======================= 4 | 5 | Below a few examples of how to use **apiosintDS** in your code. 6 | 7 | .. code-block:: python 8 | 9 | #!/usr/bin/env python3 10 | from apiosintDS import apiosintDS 11 | 12 | try: 13 | OSINTCHECK = apiosintDS.request( 14 | entities=['192.168.1.54', 15 | '0a2d170abbf5031566377b01431e3b82d3426301', 16 | 'somehost.ext', 17 | 'http://www.example.com/malicious.exe'], 18 | stix=True 19 | cache=True, 20 | cachedirectory="/tmp", 21 | verbose=True) 22 | print(OSINTCHECK) # print dict results 23 | except ValueError as e: 24 | print(e) # some error 25 | 26 | Module contents 27 | =============== 28 | 29 | .. |request| function:: apiosintDS.request(entities=list, stix=False, cache=False, cachedirectory=None, clearcache=False, cachetimeout=False, verbose=False, loglevel="DEBUG", logconsole=True, logfile=False, localdirectory=False, *args, **kwargs) 30 | 31 | Uniq method to query the service. Return a ``dict`` that can be validated against the json schema returned by the ``apiosintDS.schema()`` method. 32 | 33 | Parameters 34 | `````````` 35 | 36 | .. confval:: entities 37 | 38 | List of entities to be submitted. One per row. 39 | 40 | :type: list 41 | :default: ``None`` 42 | :allowed: ``[IPv4|domain|url|hash(['md5', 'sha1', 'sha256'])]`` 43 | 44 | .. confval:: stix 45 | 46 | Dowload and parse additional information from online STIX report. 47 | 48 | :type: boolean 49 | :default: ``False`` 50 | 51 | .. note:: 52 | STIX2 reports may be not available due to data retention policy. 53 | 54 | .. confval:: cache 55 | 56 | Enable cache mode. Downloaded lists will be stored and won't be downloaded untile the cache timeout is reached. 57 | 58 | :type: boolean 59 | :default: ``False`` 60 | 61 | .. confval:: cachedirectory 62 | 63 | The cache directory where the script check for cached lists files and where them will be stored on cache creation or update. 64 | 65 | :type: string 66 | :default: ``System tmp directory`` 67 | :example: ``/path/to/cachedir`` 68 | 69 | .. note:: 70 | Must be specified the same every script run unless your are using 71 | the system temp directory. 72 | 73 | .. confval:: clearcache 74 | 75 | Force the script to download updated lists even if the :confval:`cachetimeout` period has not yet been reached. 76 | 77 | :type: boolean 78 | :default: ``False`` 79 | 80 | .. note:: 81 | Must be used in combination with :confval:`cache` 82 | 83 | .. confval:: cachetimeout 84 | 85 | Define the cache timeout in hours. 86 | 87 | :type: integer 88 | :default: ``4`` 89 | 90 | .. note:: 91 | ``0`` is allowed but means no timeout. Default value is ``4`` hours. 92 | This option needs to be used in combination with :confval:`cache` option configured to True. 93 | 94 | .. confval:: verbose 95 | 96 | Include unmatched results in report. 97 | 98 | :type: boolean 99 | :default: ``False`` 100 | 101 | .. confval:: loglevel 102 | 103 | Define the log level. 104 | 105 | :type: enum 106 | :default: ``DEBUG`` 107 | :allowed: ``[DEBUG|INFO|WARNING|ERROR|CRITICAL]`` 108 | 109 | .. confval:: logconsole 110 | 111 | Suppress log messages to the console's ``STDOUT``. 112 | 113 | :type: boolean 114 | :default: ``True`` 115 | 116 | .. confval:: logfile 117 | 118 | Define the log file path. 119 | 120 | :type: string 121 | :default: ``False`` 122 | :example: ``/path/to/logfile.log`` 123 | 124 | .. note:: 125 | No log file is created by default. ``STDOUT`` is used instead. 126 | 127 | .. confval:: localdirectory 128 | 129 | Absolute path to the 'Threat-Intel' directory related to a local project clone. Searches are performed against local data. 130 | 131 | :type: string 132 | :default: ``False`` 133 | :example: ``/path/to/git/clone/Threat-Intel/`` 134 | 135 | .. note:: 136 | Before using this option, clone the GitHub project in a file system where 137 | the library has read permissions. Don't forget to use `--depth=1` and `--branch=master` 138 | options if you don't want to download all project commits. 139 | 140 | .. code-block:: bash 141 | 142 | $ cd /path/to/git/clone/ 143 | $ git clone --depth=1 --branch=master https://github.com/davidonzo/Threat-Intel.git 144 | 145 | When this option is in use, all cache related options are ignored. To update data 146 | in your local repository destroy the existing data and clone it again. 147 | 148 | .. code-block:: bash 149 | 150 | $ cd /path/to/git/clone/ 151 | $ rm -rf Threat-Intel/ 152 | $ git clone --depth=1 --branch=master https://github.com/davidonzo/Threat-Intel.git 153 | 154 | .. |schema| function:: apiosintDS.schema() 155 | 156 | Return an object containing the ``json`` schema. 157 | 158 | -------------------------------------------------------------------------------- /docs/source/userguidecli.rst: -------------------------------------------------------------------------------- 1 | ============================ 2 | Usage via command line (CLI) 3 | ============================ 4 | 5 | .. code-block:: bash 6 | 7 | ~$ apiosintDS [-h] [-e [IPv4|domain|url|hash]] [-f /path/to/file.txt] [-st] 8 | [-o /path/to/output.json] [-p] [-nc] [-v] [-c] [-cd /path/to/cachedir] 9 | [-ct [0-9]] [-cc] [-ld /path/to/git/clone/Threat-Intel/] 10 | [-l /path/to/logfile.log] [-ll [DEBUG|INFO|WARNING|ERROR|CRITICAL]] 11 | [-lc] [-i] [-s] [-vv] 12 | 13 | Command line options 14 | ```````````````````` 15 | .. confval:: -h, --help 16 | 17 | Show the help and exit. 18 | 19 | :type: boolean 20 | :default: ``False`` 21 | 22 | .. confval:: -e, --entity 23 | 24 | Single item to search. Supported entities are IPv4/FQDN/URLs or file hashes in MD5, SHA1 or SHA256 format. 25 | 26 | :type: string 27 | :default: ``None`` 28 | :allowed: ``[IPv4|domain|url|hash(['md5', 'sha1', 'sha256'])]`` 29 | 30 | .. note:: 31 | It can't be used in combination with the :confval:`--file` option. 32 | 33 | .. confval:: -f, --file 34 | 35 | Path to file containing entities to search. Supported entities are IPv4/FQDN/URLs and file hashes (MD5, SHA1, SHA256). 36 | Insert one item per row. 37 | 38 | :type: string 39 | :default: ``None`` 40 | :example: ``/path/to/file.txt`` 41 | 42 | .. note:: 43 | It can't be used in combination with the :confval:`--entity` option. 44 | 45 | .. confval:: -st, --stix 46 | 47 | Dowload and parse additional information from online STIX report. 48 | 49 | :type: boolean 50 | :default: ``False`` 51 | 52 | .. note:: 53 | STIX2 reports may be not available due to data retention policy. 54 | 55 | .. confval:: -o, --output 56 | 57 | Path to output file. If not specified the output will be redirect to the system ``STDOUT``. 58 | 59 | :type: string 60 | :default: ``STDOUT`` 61 | :example: ``/path/to/output.json`` 62 | 63 | .. note:: 64 | It can't be used in combination with the :confval:`--pretty` option. 65 | 66 | .. confval:: -p, --pretty 67 | 68 | Show results in terminal with a little bit of formatting applied. 69 | 70 | :type: boolean 71 | :default: ``False`` 72 | 73 | .. note:: 74 | Default output format is ``JSON``. Data displayed in pretty view 75 | does not cover all informations included in the JSON response 76 | format. 77 | 78 | .. confval:: -nc, --nocolor 79 | 80 | Suppers colors in --pretty output. For accessibility purpose. 81 | 82 | :type: boolean 83 | :default: ``False`` 84 | 85 | .. confval:: -v, --verbose 86 | 87 | Include unmatched results in report. 88 | 89 | :type: boolean 90 | :default: ``False`` 91 | 92 | .. confval:: -c, --cache 93 | 94 | Enable cache mode. Downloaded lists will be stored and won't be downloaded untile the cache timeout is reached. 95 | 96 | :type: boolean 97 | :default: ``False`` 98 | 99 | .. confval:: -cd, --cachedirectory 100 | 101 | The cache directory where the script check for cached lists files and where them will be stored on cache creation or update. 102 | 103 | :type: string 104 | :default: ``System tmp directory`` 105 | :example: ``/path/to/cachedir`` 106 | 107 | .. note:: 108 | Must be specified the same every script run unless your are using 109 | the system temp directory. 110 | 111 | .. confval:: -ct, --cachetimeout 112 | 113 | Define the cache timeout in hours. 114 | 115 | :type: integer 116 | :default: ``4`` 117 | 118 | .. note:: 119 | ``0`` is allowed but means no timeout. Default value is ``4`` hours. 120 | This option needs to be used in combination with :confval:`--cache` option configured to ``True``. 121 | 122 | .. confval:: -cc, --clearcache 123 | 124 | Force the script to download updated lists even if the :confval:`--cachetimeout` period has not yet been reached. 125 | 126 | :type: boolean 127 | :default: ``False`` 128 | 129 | .. note:: 130 | Must be used in combination with :confval:`--cache` 131 | 132 | .. confval:: -ld, --localdirectory 133 | 134 | Absolute path to the 'Threat-Intel' directory related to a local project clone. Searches are performed against local data. 135 | 136 | :type: string 137 | :default: ``False`` 138 | :example: ``/path/to/git/clone/Threat-Intel/`` 139 | 140 | .. note:: 141 | Before using this option, clone the GitHub project in a file system where 142 | the library has read permissions. Don't forget to use ``--depth=1`` and ``--branch=master`` 143 | options if you don't want to download all project commits. 144 | 145 | .. code-block:: bash 146 | 147 | $ cd /path/to/git/clone/ 148 | $ git clone --depth=1 --branch=master https://github.com/davidonzo/Threat-Intel.git 149 | 150 | When this option is in use, all cache related options are ignored. To update data 151 | in your local repository destroy the existing data and clone it again. 152 | 153 | .. code-block:: bash 154 | 155 | $ cd /path/to/git/clone/ 156 | $ rm -rf Threat-Intel/ 157 | $ git clone --depth=1 --branch=master https://github.com/davidonzo/Threat-Intel.git 158 | 159 | .. confval:: -l, --logfile 160 | 161 | Define the log file path. 162 | 163 | :type: string 164 | :default: ``NONE`` 165 | :example: ``/path/to/logfile.log`` 166 | 167 | .. note:: 168 | No log file is created by default. ``STDOUT`` is used instead. 169 | 170 | .. confval:: -ll, --loglevel 171 | 172 | Define the log level. 173 | 174 | :type: enum 175 | :default: ``DEBUG`` 176 | :allowed: ``[DEBUG|INFO|WARNING|ERROR|CRITICAL]`` 177 | 178 | .. confval:: -lc, --logconsole 179 | 180 | Suppress log messages to the console's ``STDOUT``. 181 | 182 | :type: boolean 183 | :default: ``False`` 184 | 185 | .. confval:: -i, --info 186 | 187 | Print information about the library. 188 | 189 | :type: boolean 190 | :default: ``False`` 191 | 192 | .. confval:: -s, --schema 193 | 194 | Display the response `json schema `_. 195 | 196 | :type: boolean 197 | :default: ``False`` 198 | 199 | .. confval:: -vv, --version 200 | 201 | Show the library version. 202 | 203 | :type: boolean 204 | :default: ``False`` 205 | -------------------------------------------------------------------------------- /docs/source/userguidemisp.rst: -------------------------------------------------------------------------------- 1 | ====================== 2 | apiosintDS MISP Module 3 | ====================== 4 | 5 | **apiosintDS** is included as enrichment module in the official `MISP-Modules repository `_. This guide assume you have your MISP instance up and running with MISP Modules correctly initializated. 6 | 7 | The module has been specially designed for people and organizations don't want to subscribe the `DigitalSide Threat-Intel MISP Feed `_ and prefer to query it as an on demand service. 8 | 9 | .. warning:: 10 | 11 | If `DigitalSide Threat-Intel MISP Feed `_ is enabled and regulary fetched by your MISP instance, 12 | don't use this plugin. All information retrivable by the plugin are just included in your MISP events dataset. 13 | The MISP correlation engine should be used instead. 14 | 15 | Input / Output 16 | `````````````` 17 | 18 | .. confval:: Module type 19 | 20 | MISP module type. 21 | 22 | :module-type: ``['hover', 'expansion']`` 23 | 24 | .. confval:: Input 25 | 26 | The module runs against the following MISP attributes type. 27 | 28 | :input-attributes: ``["domain", "domain|ip", "hostname", "ip-dst", "ip-src", "ip-dst|port", "ip-src|port"]`` 29 | ``["url", "md5", "sha1", "sha256", "filename|md5", "filename|sha1", "filename|sha256"]`` 30 | 31 | .. confval:: Output 32 | 33 | The module returns the following MISP attributes type. 34 | 35 | :output-attributes: ``["domain", "ip-dst", "url", "comment", "md5", "sha1", "sha256", "link", "text"]`` 36 | 37 | 38 | Configuration 39 | ````````````` 40 | 41 | Go to your MISP web interface and login with a user account able to edit plugins configuration. Once logged in go to ``Administration >> Server Settings & Maintenance >> Plugin`` and select the ``Enrichment`` tab. Put in the search input filter ``apiosintds`` in order to show only the needed configuration settings. 42 | 43 | .. image:: ../_static/img/mispmoduleconfiguration.png 44 | :width: 640 45 | :alt: apiosintDS MISP Module 46 | 47 | .. confval:: Plugin.Enrichment_apiosintds_enabled 48 | 49 | MISP internal configuration to enable or disable the module. 50 | 51 | :type: boolean 52 | :default: ``false`` 53 | 54 | .. note:: 55 | To enable the plugint configure the valute to ``true``. 56 | 57 | .. confval:: Plugin.Enrichment_apiosintds_restrict 58 | 59 | Restrict the plugin use to a single organization. 60 | 61 | :type: enum 62 | :default: ``No organization selected`` 63 | :allowed: ``ORG in the given MISP instance`` 64 | 65 | .. confval:: Plugin.Enrichment_apiosintds_STIX2_details 66 | 67 | Dowload and parse additional information from online STIX report. 68 | 69 | :type: enum 70 | :default: ``no`` 71 | :allowed: ``[yes|no]`` 72 | 73 | .. note:: 74 | STIX2 reports may be not available due to data retention policy. 75 | 76 | .. confval:: Plugin.Enrichment_apiosintds_import_related 77 | 78 | Parse and include in the results related items. 79 | 80 | :type: enum 81 | :default: ``no`` 82 | :allowed: ``[yes|no]`` 83 | 84 | .. note:: 85 | Is strongly reccommended to configure it to ``yes`` to obtain best results. 86 | 87 | .. confval:: Plugin.Enrichment_apiosintds_cache 88 | 89 | Enable cache mode. Downloaded lists will be stored and won't be downloaded untile the cache timeout is reached. 90 | 91 | :type: enum 92 | :default: ``no`` 93 | :allowed: ``[yes|no]`` 94 | 95 | .. confval:: Plugin.Enrichment_apiosintds_cache_directory 96 | 97 | The cache directory where the script check for cached list files and where them will be stored on cache cache creation or update. 98 | 99 | :type: string 100 | :default: ``None`` 101 | :example: ``/path/to/cachedir`` 102 | 103 | .. note:: 104 | Read and write permissions are required for the system user running the MISP instance 105 | (depends on your installation configuration, should be one between :confval:`www-data`, :confval:`misp`, :confval:`apache`, others...) 106 | 107 | .. confval:: Plugin.Enrichment_apiosintds_cache_timeout_h 108 | 109 | Define the cache timeout in hours. 110 | 111 | :type: integer 112 | :default: ``4`` 113 | 114 | .. note:: 115 | ``0`` is allowed but means no timeout. Default value is ``4`` hours. 116 | This option needs to be used in combination with :confval:`apiosintds_cache` option configured to True. 117 | 118 | .. confval:: Plugin.Enrichment_apiosintds_local_directory 119 | 120 | Absolute path to the 'Threat-Intel' directory related to a local project clone. Searches are performed against local data. 121 | 122 | :type: string 123 | :default: ``Empty`` 124 | :example: ``/path/to/git/clone/Threat-Intel/`` 125 | 126 | .. note:: 127 | Before using this option, clone the GitHub project in a file system where 128 | the library has read permissions. Don't forget to use `--depth=1` and `--branch=master` 129 | options if you don't want to download all project commits. 130 | 131 | Make sure the system user running the MISP instance has read permissions on the directory. 132 | 133 | .. code-block:: bash 134 | 135 | $ cd /path/to/git/clone/ 136 | $ git clone --depth=1 --branch=master https://github.com/davidonzo/Threat-Intel.git 137 | $ chown -R $MISP_SYSTEM_USER:$MISP_SYSTEM_GROUP Threat-Intel 138 | 139 | When this option is in use, all cache related options are ignored. To update data 140 | in your local repository destroy the existing data and clone it again. 141 | 142 | .. code-block:: bash 143 | 144 | $ cd /path/to/git/clone/ 145 | $ rm -rf Threat-Intel/ 146 | $ git clone --depth=1 --branch=master https://github.com/davidonzo/Threat-Intel.git 147 | $ chown -R $MISP_SYSTEM_USER:$MISP_SYSTEM_GROUP Threat-Intel 148 | 149 | Usage: hover 150 | ```````````` 151 | 152 | Using the module as ``hover`` plugin retrived data will be displayed as follow. 153 | 154 | .. image:: ../_static/img/modulehover.png 155 | :width: 640 156 | :alt: apiosintDS MISP Module Hover 157 | 158 | Usage: enrichment 159 | ````````````````` 160 | 161 | Using the module as ``enrichment`` plugin retrived data will be imported as follow. 162 | 163 | .. image:: ../_static/img/moduleenrich.png 164 | :width: 640 165 | :alt: apiosintDS MISP Module Enrichment 166 | 167 | 168 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pytz 2 | requests 3 | urllib3 4 | stix2 5 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description_file = README.md 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | import logging 4 | from apiosintDS.modules.scriptinfo import scriptinfo 5 | logging.basicConfig(format='%(levelname)s: %(message)s') 6 | log = logging.getLogger(__name__) 7 | if (sys.version_info < (3, 0)):#NO MORE PYTHON 2!!! https://pythonclock.org/ 8 | logging.error(" ########################### ERROR ###########################") 9 | logging.error(" =============================================================") 10 | logging.error(" Invalid python version detected: "+str(sys.version_info[0])+"."+str(sys.version_info[1])) 11 | logging.error(" =============================================================") 12 | logging.error(" It seems your are still using python 2 even if you should") 13 | logging.error(" now it will be retire next 2020.") 14 | logging.error(" For more info please read https://pythonclock.org/") 15 | logging.error(" =============================================================") 16 | logging.error(" Try again typing: python3 /path/to/"+sys.argv[0]) 17 | logging.error(" =============================================================") 18 | logging.error(" ########################### ERROR ###########################") 19 | exit(0) 20 | 21 | from os import path 22 | from setuptools import setup 23 | from codecs import open 24 | 25 | requirements = [line.rstrip('\n') for line in open('requirements.txt')] 26 | with open("README.md", "r") as fh: 27 | mylong_description = fh.read() 28 | scriptinfo = {"scriptname": "apiosintDS", 29 | "majorversion": "2", 30 | "minorversion": "0.3", 31 | "license": "MIT", 32 | "licenseurl": "https://raw.githubusercontent.com/davidonzo/Threat-Intel/master/LICENSE", 33 | "author": "Davide Baglieri", 34 | "mail": "info@digitalside.it", 35 | "pgp": "30B31BDA", 36 | "fingerprint": "0B4C F801 E8FF E9A3 A602 D2C7 9C36 93B2 30B3 1BDA", 37 | "git": "https://github.com/davidonzo/Threat-Intel/blob/master/tools/DigitalSide-API/v1", 38 | "DSProjectHP": "https://osint.digitalside.it", 39 | "DSGitHubHP": "https://github.com/davidonzo/apiosintDS"} 40 | 41 | setup( 42 | name=scriptinfo["scriptname"], 43 | packages=["apiosintDS", "apiosintDS.modules", "apiosintDS.utilities"], 44 | python_requires='>3.5.2', 45 | version=scriptinfo["majorversion"]+"."+scriptinfo["minorversion"], 46 | url=scriptinfo["DSGitHubHP"], 47 | description="On demand query API for OSINT.digitalside.it project. You can query for souspicious domains, urls, IPv4 and file hashes.", 48 | long_description=mylong_description, 49 | long_description_content_type="text/markdown", 50 | license=scriptinfo["license"], 51 | author=scriptinfo["author"], 52 | author_email=scriptinfo["mail"], 53 | keywords=['apiosintDS', 'OSINT', 'Threat-Intel', 'IoC', 'Security'], 54 | classifiers=[ 55 | "Development Status :: 4 - Beta", 56 | "Intended Audience :: Information Technology", 57 | "Topic :: Security", 58 | "License :: OSI Approved :: MIT License", 59 | 'Programming Language :: Python :: 3' 60 | ], 61 | package_data={ 62 | 'apiosintDS': [ 63 | 'schema/schema.json', 64 | 'README.md' 65 | ], 66 | }, 67 | install_requires=requirements, 68 | 69 | entry_points={ 70 | "console_scripts": [ 71 | scriptinfo["scriptname"]+"="+scriptinfo["scriptname"]+"."+scriptinfo["scriptname"]+":main", 72 | ], 73 | }, 74 | 75 | ) 76 | --------------------------------------------------------------------------------