├── examples ├── __init__.py ├── anydesk_stairwell.py ├── keys.py.sample ├── email_reputation.py ├── dt_domain_risk_score.py ├── dt_domain_risk_to_csv.py ├── vt_hash_threat_classification.py ├── email_reputation_to_csv.py └── enrich_misp_event.py ├── pyoti ├── __init__.py ├── emails │ ├── __init__.py │ ├── disposableemails.py │ └── emailrepio.py ├── ips │ ├── __init__.py │ ├── abuseipdb.py │ ├── spamhausintel.py │ └── greynoise.py ├── domains │ ├── __init__.py │ ├── circlpdns.py │ ├── irisinvestigate.py │ └── checkdmarc.py ├── hashes │ ├── __init__.py │ ├── malwarebazaar.py │ ├── malwarehashregistry.py │ └── circlhashlookup.py ├── urls │ ├── __init__.py │ ├── phishtank.py │ ├── linkpreview.py │ ├── googlesafebrowsing.py │ └── proofpointurldecoder.py ├── utils.py ├── multis │ ├── __init__.py │ ├── pulsedive.py │ ├── onyphe.py │ ├── whoisxml.py │ ├── binaryedge.py │ ├── circlpssl.py │ ├── xforce.py │ ├── maltiverseioc.py │ ├── stairwell.py │ ├── urlhaus.py │ ├── otx.py │ ├── ip2location.py │ ├── filescanio.py │ ├── joesandbox.py │ ├── threatfox.py │ ├── virustotal.py │ ├── hybridanalysis.py │ ├── metadefendercloud.py │ ├── misp.py │ ├── ciscoumbrella.py │ ├── dnsblocklist.py │ ├── urlscan.py │ └── triage.py ├── exceptions.py └── classes.py ├── setup.py ├── .gitignore ├── .github └── ISSUE_TEMPLATE │ └── bug_report.md ├── setup.cfg ├── update_keys.sh ├── docs ├── linux │ └── README.md ├── windows │ └── README.md ├── tutorials │ └── phishing_triage_urls.ipynb └── misp │ └── README.md ├── update_keys.ps1 ├── CHANGELOG.md └── README.md /examples/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pyoti/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = '0.4.0' 2 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup() 4 | -------------------------------------------------------------------------------- /pyoti/emails/__init__.py: -------------------------------------------------------------------------------- 1 | from .disposableemails import DisposableEmails 2 | from .emailrepio import EmailRepIO 3 | -------------------------------------------------------------------------------- /pyoti/ips/__init__.py: -------------------------------------------------------------------------------- 1 | from .abuseipdb import AbuseIPDB 2 | from .greynoise import GreyNoise 3 | from .spamhausintel import SpamhausIntel 4 | -------------------------------------------------------------------------------- /pyoti/domains/__init__.py: -------------------------------------------------------------------------------- 1 | from .checkdmarc import CheckDMARC 2 | from .circlpdns import CIRCLPDNS 3 | from .irisinvestigate import IrisInvestigate 4 | -------------------------------------------------------------------------------- /pyoti/hashes/__init__.py: -------------------------------------------------------------------------------- 1 | from .circlhashlookup import CIRCLHashLookup 2 | from .malwarebazaar import MalwareBazaar 3 | from .malwarehashregistry import MalwareHashRegistry 4 | -------------------------------------------------------------------------------- /pyoti/urls/__init__.py: -------------------------------------------------------------------------------- 1 | from .googlesafebrowsing import GoogleSafeBrowsing 2 | from .linkpreview import LinkPreview 3 | from .phishtank import Phishtank 4 | from .proofpointurldecoder import ProofpointURLDecoder 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Secrets 2 | keys.py 3 | 4 | # TODO 5 | todo.txt 6 | 7 | # Byte-compiled 8 | __pycache__/ 9 | *.py[cod] 10 | *.py.class 11 | 12 | # PyCharm stuff 13 | .idea 14 | 15 | # Environments 16 | venv/ 17 | 18 | # Mac Files 19 | .DS_Store 20 | 21 | pyoti.egg-info/ 22 | build/ 23 | dist/ -------------------------------------------------------------------------------- /examples/anydesk_stairwell.py: -------------------------------------------------------------------------------- 1 | from examples.keys import stairwell 2 | from pyoti.multis import Stairwell 3 | 4 | 5 | sw = Stairwell(stairwell) 6 | all_objects = sw.query_objects( 7 | query='rule.name in ["SUSP_AnyDesk_Compromised_Certificate_Jan24_1"]', 8 | page_size=150 9 | ) 10 | 11 | label_lists = {} 12 | 13 | for obj in all_objects: 14 | labels = obj['malEval']['labels'] 15 | if labels: 16 | first_label = labels[0] 17 | if first_label not in label_lists: 18 | label_lists[first_label] = [] 19 | label_lists[first_label].append(obj) 20 | 21 | sorted_label_lists = dict(sorted(label_lists.items(), key=lambda item: len(item[1]), reverse=True)) 22 | -------------------------------------------------------------------------------- /examples/keys.py.sample: -------------------------------------------------------------------------------- 1 | abuseipdb = '' 2 | binaryedge = '' 3 | circlpassive = '{USER}:{SECRET}' 4 | ciscoumbrella = '' 5 | domaintools = '{USER}:{SECRET}' 6 | filescanio = '' 7 | googlesafebrowsing = '' 8 | greynoise = '' 9 | hybridanalysis = '' 10 | ip2location = '' 11 | joesandbox = '' 12 | linkpreview = '' 13 | maltiverse = '' 14 | malwarebazaar = '' 15 | malwarehashregistry = '{USER}:{SECRET}' 16 | misp = '' 17 | onyphe = '' 18 | opswat = '' 19 | otx = '' 20 | phishtank = '' 21 | pulsedive = '' 22 | spamhausintel = '{USER}:{SECRET}' 23 | stairwell = '' 24 | sublime = '' 25 | threatfox = '' 26 | triage = '' 27 | urlscan = '' 28 | virustotal = '' 29 | whoisxml = '' 30 | xforce = '{USER}:{SECRET}' 31 | -------------------------------------------------------------------------------- /examples/email_reputation.py: -------------------------------------------------------------------------------- 1 | from argparse import ArgumentParser 2 | 3 | from pyoti.emails import EmailRepIO 4 | from keys import sublime 5 | 6 | 7 | def run(args): 8 | eml = EmailRepIO(api_key=sublime) 9 | eml.email = args 10 | eml_rep = eml.check_email() 11 | 12 | return eml_rep 13 | 14 | 15 | def main(): 16 | parser = ArgumentParser( 17 | prog="EmailRep.io Email Reputation", 18 | description="Check EmailRep.io's API for email reputation on a given email address.", 19 | ) 20 | parser.add_argument( 21 | "-e", "--email", dest="email", help="email address to check reputation" 22 | ) 23 | args = parser.parse_args() 24 | 25 | print(run(args.email)) 26 | 27 | 28 | if __name__ == "__main__": 29 | main() 30 | -------------------------------------------------------------------------------- /pyoti/utils.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from datetime import datetime, timedelta 4 | from urllib.parse import urlsplit 5 | 6 | 7 | def split_eml_domain(email: str) -> str: 8 | """Splits Domain from an Email Address""" 9 | return email.split("@")[1] 10 | 11 | 12 | def split_url_domain(url: str) -> str: 13 | """Splits first level domain from an URL""" 14 | return urlsplit(url=url).netloc 15 | 16 | 17 | def time_check_since_epoch(epoch: int) -> bool: 18 | seconds = epoch - int(time.time()) 19 | hours = (seconds / 60) / 60 20 | if hours >= 1: 21 | return True 22 | else: 23 | return False 24 | 25 | 26 | def epoch_to_date(epoch: int) -> str: 27 | return datetime.fromtimestamp(epoch).strftime("%Y-%m-%d %H:%M:%S") 28 | 29 | 30 | def time_since_seconds(seconds: int) -> str: 31 | return str(timedelta(seconds=seconds)) 32 | -------------------------------------------------------------------------------- /pyoti/multis/__init__.py: -------------------------------------------------------------------------------- 1 | from .binaryedge import BinaryEdge 2 | from .circlpssl import CIRCLPSSL 3 | from .ciscoumbrella import CiscoUmbrellaInvestigate 4 | from .dnsblocklist import DNSBlockList 5 | from .filescanio import FileScanIO 6 | from .hybridanalysis import HybridAnalysis 7 | from .ip2location import IP2Location, IP2WHOIS 8 | from .joesandbox import JoeSandbox 9 | from .maltiverseioc import MaltiverseIOC 10 | from .metadefendercloud import MetaDefenderCloudV4 11 | from .misp import MISP 12 | from .onyphe import Onyphe 13 | from .otx import OTX 14 | from .pulsedive import Pulsedive 15 | from .stairwell import Stairwell 16 | from .threatfox import ThreatFox 17 | from .triage import Triage 18 | from .urlhaus import URLhaus 19 | from .urlscan import URLscan 20 | from .virustotal import VirusTotalV3 21 | from .whoisxml import WhoisXML 22 | from .xforce import XForceExchange 23 | -------------------------------------------------------------------------------- /pyoti/emails/disposableemails.py: -------------------------------------------------------------------------------- 1 | from disposable_email_domains import blocklist 2 | from typing import Dict 3 | 4 | from pyoti.classes import EmailAddress 5 | 6 | 7 | class DisposableEmails(EmailAddress): 8 | """DisposableEmails Email Address Reputation 9 | 10 | This class checks if an email address is contained within a set of known disposable email domains. 11 | """ 12 | def __init__(self, email: str = None): 13 | EmailAddress.__init__(self, email=email) 14 | 15 | def check_email(self) -> Dict: 16 | """Checks if email domain is a known disposable email service. 17 | 18 | :return: dict of email address and if it is disposable 19 | """ 20 | domain = self.email.split("@")[1] 21 | info = {"email": self.email} 22 | if domain in blocklist: 23 | info["disposable"] = True 24 | else: 25 | info["disposable"] = False 26 | 27 | return info 28 | -------------------------------------------------------------------------------- /examples/dt_domain_risk_score.py: -------------------------------------------------------------------------------- 1 | from argparse import ArgumentParser 2 | 3 | from pyoti.domains import IrisInvestigate 4 | from keys import domaintools 5 | 6 | 7 | def run(args): 8 | iris = IrisInvestigate(api_key=domaintools) 9 | iris.domain = args 10 | domain_rep = iris.check_domain() 11 | 12 | try: 13 | return f"Iris risk score: {domain_rep[0]['domain_risk']['risk_score']}" 14 | except IndexError: 15 | return "Iris risk score: N/A" 16 | 17 | 18 | def main(): 19 | parser = ArgumentParser( 20 | prog="IrisInvestigate Domain Risk Score", 21 | description="Check Domaintools Iris Investigate for domain risk score of a given domain", 22 | ) 23 | parser.add_argument( 24 | "-d", "--domain", dest="domain", help="domain to check reputation" 25 | ) 26 | args = parser.parse_args() 27 | 28 | print(run(args.domain)) 29 | 30 | 31 | if __name__ == "__main__": 32 | main() 33 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: "[BUG]" 5 | labels: '' 6 | assignees: goodlandsecurity 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. (**please be careful and redact any private information!**) 25 | 26 | **Operating System Information (please complete the following information):** 27 | - OS: [e.g. Windows 10, OS X] 28 | - Python Version [e.g. 3.9] 29 | - Using Python Virtual Environment [e.g. Yes] 30 | 31 | 32 | **Additional context** 33 | Add any other context about the problem here. 34 | -------------------------------------------------------------------------------- /pyoti/exceptions.py: -------------------------------------------------------------------------------- 1 | class PyOTIError(Exception): 2 | """Base PyOTI exception.""" 3 | 4 | def __init__(self, message): 5 | super().__init__(message) 6 | self.message = message 7 | 8 | 9 | class CIRCLPDNSError(PyOTIError): 10 | """Exception raised for CIRCLPDNS errors.""" 11 | pass 12 | 13 | 14 | class CIRCLHashLookupError(PyOTIError): 15 | """Exception raised for CIRCLHashLookup errors.""" 16 | pass 17 | 18 | 19 | class LinkPreviewError(PyOTIError): 20 | """Exception raised for LinkPreview errors.""" 21 | pass 22 | 23 | 24 | class MaltiverseIOCError(PyOTIError): 25 | """Exception raised for MaltiverseIOC errors.""" 26 | pass 27 | 28 | 29 | class MalwareHashRegistryError(PyOTIError): 30 | """Exception raised for MalwareHashRegistry errors.""" 31 | pass 32 | 33 | 34 | class SpamhausIntelError(PyOTIError): 35 | """Exception raised for SpamhausIntel errors.""" 36 | pass 37 | 38 | 39 | class URLhausError(PyOTIError): 40 | """Exception raised for URLhaus errors.""" 41 | pass 42 | 43 | 44 | class VirusTotalError(PyOTIError): 45 | """Exception raised for VirusTotal errors.""" 46 | pass -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = pyoti 3 | version = attr: pyoti.__version__ 4 | description = Python API for Threat Intelligence 5 | long_description = file: README.md 6 | long_description_content_type = text/markdown 7 | author = goodlandsecurity 8 | author_email = jj.josing@rhisac.org 9 | license = GNU GPLv3 10 | classifiers = 11 | Development Status :: 5 - Production/Stable 12 | Environment :: Console 13 | License :: OSI Approved :: GNU General Public License v3 (GPLv3) 14 | Programming Language :: Python :: 3.9 15 | Intended Audience :: Information Technology 16 | Topic :: Security 17 | Topic :: Utilities 18 | platform = any 19 | url = https://github.com/RH-ISAC/PyOTI 20 | project_urls = 21 | Bug Tracker = https://github.com/RH-ISAC/PyOTI/issues 22 | Changelog = https://github.com/RH-ISAC/PyOTI/blob/master/CHANGELOG.md 23 | 24 | [options] 25 | zip_safe = false 26 | packages = find: 27 | python_requires = >=3.9 28 | install_requires = 29 | requests 30 | aiodns 31 | disposable-email-domains 32 | 33 | [options.extras_require] 34 | jupyter_notebook = 35 | jupyterlab 36 | notebook 37 | 38 | [options.packages.find] 39 | exclude = 40 | examples* 41 | docs* 42 | -------------------------------------------------------------------------------- /pyoti/hashes/malwarebazaar.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from typing import Dict 3 | 4 | from pyoti import __version__ 5 | from pyoti.classes import FileHash 6 | 7 | 8 | class MalwareBazaar(FileHash): 9 | """MalwareBazaar by abuse.ch 10 | 11 | MalwareBazaar is a project from abuse.ch with the goal of sharing malware samples with the infosec community, AV 12 | vendors and threat intelligence providers. 13 | """ 14 | def __init__(self, api_key: str, api_url: str = "https://mb-api.abuse.ch/api/v1"): 15 | FileHash.__init__(self, api_key=api_key, api_url=api_url) 16 | 17 | def _api_post(self, data) -> requests.models.Response: 18 | """POST request to MalwareBazaar API""" 19 | headers = { 20 | "API-KEY": self.api_key, 21 | "User-Agent": f"PyOTI {__version__}" 22 | } 23 | 24 | response = requests.request("POST", url=self.api_url, data=data, headers=headers) 25 | 26 | return response 27 | 28 | def check_hash(self) -> Dict: 29 | """Checks File Hash reputation""" 30 | data = {"query": "get_info", "hash": self.file_hash} 31 | 32 | response = self._api_post(data=data) 33 | 34 | return response.json() 35 | -------------------------------------------------------------------------------- /update_keys.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # This bash script is meant to be run from the root directory of PyOTI. 4 | # It will check the diff between examples/keys.py.sample and examples/keys.py and append 5 | # any API key variables that are missing from examples/keys.py and set them to ''. 6 | # Please make sure to update examples/keys.py with API secrets after running. 7 | 8 | RED='\033[0;31m' 9 | GREEN='\033[0;32m' 10 | NC='\033[0m' # no color 11 | 12 | # check if input file ends with newline 13 | function is_newline(){ 14 | [[ $(tail -c 1 "$1" | wc -l) -gt 0 ]] 15 | } 16 | 17 | KEYS=$(diff <(awk '{print $1}' examples/keys.py.sample | sort -u) <(awk '{print $1}' examples/keys.py | sort -u) | grep "<" | cut -d " " -f2) 18 | 19 | if [ -z "$KEYS" ] 20 | then 21 | echo -e "${GREEN}[*]${NC} No keys need to be updated!" 22 | else 23 | echo -e "${GREEN}[!]${NC} New keys found! Adding to examples/keys.py..." 24 | echo "" 25 | 26 | if ! is_newline examples/keys.py 27 | then 28 | echo "" >> examples/keys.py 29 | fi 30 | 31 | for key in $KEYS 32 | do 33 | echo $key "= ''" >> examples/keys.py 34 | echo -e "${GREEN}[+]${NC}" $key "added to examples/keys.py!" 35 | done 36 | 37 | echo "" 38 | echo -e "${GREEN}[*]${RED} Add API secrets to examples/keys.py!" 39 | fi 40 | -------------------------------------------------------------------------------- /pyoti/urls/phishtank.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from typing import Dict 3 | 4 | from pyoti import __version__ 5 | from pyoti.classes import URL 6 | 7 | 8 | class Phishtank(URL): 9 | """Phishtank Anti-Phishing 10 | 11 | Phishtank is a collaborative clearing house for data and information about 12 | phishing on the internet. 13 | """ 14 | def __init__(self, api_key: str, api_url: str = "https://checkurl.phishtank.com/checkurl/"): 15 | """ 16 | :param api_key: Phishtank API key 17 | :param api_url: Phishtank API URL 18 | """ 19 | URL.__init__(self, api_key, api_url) 20 | 21 | def _api_post(self) -> requests.models.Response: 22 | """POST request to API""" 23 | data = { 24 | "format": "json", 25 | "url": self.url 26 | } 27 | 28 | headers = { 29 | "app_key": self.api_key, 30 | "User-agent": f"phishtank/PyOTI {__version__}" 31 | } 32 | 33 | response = requests.request("POST", url=self.api_url, data=data, headers=headers) 34 | 35 | return response 36 | 37 | def check_url(self) -> Dict: 38 | """Checks URL reputation 39 | 40 | :return: dict of request response 41 | """ 42 | response = self._api_post() 43 | r = response.json() 44 | 45 | return r.get('results') 46 | -------------------------------------------------------------------------------- /pyoti/urls/linkpreview.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from typing import Dict 3 | from urllib.parse import urlencode 4 | 5 | from pyoti import __version__ 6 | from pyoti.classes import URL 7 | from pyoti.exceptions import LinkPreviewError 8 | 9 | 10 | class LinkPreview(URL): 11 | """LinkPreview Shortened URL Previewer 12 | 13 | LinkPreview API provides basic website information from any given URL. 14 | """ 15 | def __init__(self, api_key: str, api_url: str = "https://api.linkpreview.net"): 16 | URL.__init__(self, api_key, api_url) 17 | 18 | def _api_get(self) -> requests.models.Response: 19 | """GET request to API""" 20 | error_code = [400, 401, 403, 404, 423, 425, 426, 429] 21 | 22 | headers = {"User-Agent": f"PyOTI {__version__}"} 23 | 24 | params = {"key": self.api_key, "q": self.url} 25 | 26 | encoded = urlencode(params) 27 | 28 | response = requests.request("GET", url=self.api_url, headers=headers, params=encoded) 29 | 30 | if response.status_code == 200: 31 | return response 32 | 33 | elif response.status_code in error_code: 34 | raise LinkPreviewError(response.json()["description"]) 35 | 36 | def check_url(self) -> Dict: 37 | """Checks URL reputation 38 | 39 | :return: dict of request response 40 | """ 41 | response = self._api_get() 42 | 43 | return response.json() 44 | -------------------------------------------------------------------------------- /docs/linux/README.md: -------------------------------------------------------------------------------- 1 | ## Installation for Linux 2 | 3 | Virtualenv (recommended): 4 | ```bash 5 | 6 | # clone PyOTI repository and copy sample keys file 7 | git clone https://github.com/RH-ISAC/PyOTI ~/PyOTI 8 | cd ~/PyOTI 9 | cp examples/keys.py.sample examples/keys.py 10 | # install/setup virtual environment 11 | python3 -m pip install virtualenv 12 | python3 -m venv venv 13 | source ~/PyOTI/venv/bin/activate 14 | # make sure to fill in your API secrets! 15 | vim examples/keys.py 16 | # install PyOTI library 17 | python3 -m pip install . 18 | ``` 19 | No virtualenv: 20 | ```bash 21 | # clone PyOTI repository and copy sample keys file 22 | git clone https://github.com/RH-ISAC/PyOTI ~/PyOTI 23 | cd ~/PyOTI 24 | cp examples/keys.py.sample examples/keys.py 25 | # make sure to fill in your API secrets! 26 | vim examples/keys.py 27 | # install PyOTI library 28 | python3 -m pip install . 29 | ``` 30 | ## Updating 31 | 32 | Virtualenv (recommended): 33 | ```bash 34 | # activate virtual environment 35 | source ~/PyOTI/venv/bin/activate 36 | # pull PyOTI repository 37 | cd ~/PyOTI 38 | git pull 39 | bash update_keys.sh 40 | # make sure to fill in your updated API secrets! 41 | vim examples/keys.py 42 | # make sure PyOTI library is updated 43 | python3 -m pip install . 44 | ``` 45 | No virtualenv: 46 | ```bash 47 | # pull PyOTI repository 48 | cd ~/PyOTI 49 | git pull 50 | bash update_keys.sh 51 | # make sure to fill in your updated API secrets! 52 | vim examples/keys.py 53 | # make sure PyOTI library is updated 54 | python3 -m pip install . 55 | ``` -------------------------------------------------------------------------------- /pyoti/hashes/malwarehashregistry.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from typing import Dict 3 | 4 | from pyoti import __version__ 5 | from pyoti.classes import FileHash 6 | from pyoti.exceptions import MalwareHashRegistryError 7 | 8 | 9 | class MalwareHashRegistry(FileHash): 10 | """MalwareHashRegistry Malicious File Hashes 11 | 12 | Team Cymru aggregates results of over 30 AV tools, including their own analysis, 13 | to improve detection rates of malicious files. 14 | """ 15 | def __init__(self, api_key: str, api_url: str = "https://hash.cymru.com/v2"): 16 | """ 17 | :param api_key: MalwareHashRegistry API key 18 | :param api_url: MalwareHashRegistry API URL 19 | """ 20 | FileHash.__init__(self, api_key=api_key, api_url=api_url) 21 | 22 | def _api_get(self) -> requests.models.Response: 23 | """GET request to API for single hash lookup""" 24 | credentials = self.api_key.split(":") 25 | 26 | headers = {"User-Agent": f"PyOTI {__version__}"} 27 | 28 | response = requests.request( 29 | "GET", 30 | url=f"{self.api_url}/{self.file_hash}", 31 | auth=(credentials[0], credentials[1]), 32 | headers=headers 33 | ) 34 | 35 | return response 36 | 37 | def check_hash(self) -> Dict: 38 | """Checks File Hash reputation 39 | 40 | :return: request response dict 41 | """ 42 | response = self._api_get() 43 | r = response.json() 44 | if r.get('error'): 45 | raise MalwareHashRegistryError(r.get('msg')) 46 | else: 47 | return r 48 | -------------------------------------------------------------------------------- /pyoti/emails/emailrepio.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from typing import Dict 3 | 4 | from pyoti import __version__ 5 | from pyoti.classes import EmailAddress 6 | 7 | 8 | class EmailRepIO(EmailAddress): 9 | """EmailRepIO Email Address Reputation 10 | 11 | EmailRep is a system of crawlers, scanners, and enrichment services that 12 | collects data on email addresses, domains, and internet personas. EmailRep uses 13 | hundreds of data points from social media profiles, professional networking 14 | sites, dark web credential leaks, data breaches, phishing kits, phishing emails, 15 | spam lists, open mail relays, domain age and reputation, deliverability, and 16 | more to predict the risk of an email address. 17 | """ 18 | def __init__(self, api_key: str, api_url: str = "https://emailrep.io"): 19 | """ 20 | :param api_key: EmailRepIO API key 21 | :param api_url: EmailRepIO base API URL 22 | """ 23 | EmailAddress.__init__(self, api_key=api_key, api_url=api_url) 24 | 25 | def _api_get(self) -> requests.models.Response: 26 | """GET request to API""" 27 | headers = { 28 | "User-Agent": f"PyOTI {__version__}", 29 | "Key": self.api_key 30 | } 31 | 32 | response = requests.request("GET", url=f"{self.api_url}/{self.email}", headers=headers) 33 | 34 | return response 35 | 36 | def check_email(self) -> Dict: 37 | """Checks Email Address reputation 38 | 39 | :return: request response dict 40 | """ 41 | response = self._api_get() 42 | 43 | r = response.json() 44 | r['remaining_daily_quota'] = response.headers.get('X-Rate-Limit-Daily-Remaining') 45 | r['remaining_monthly_quota'] = response.headers.get('X-Rate-Limit-Monthly-Remaining') 46 | 47 | return r 48 | -------------------------------------------------------------------------------- /pyoti/multis/pulsedive.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from typing import Dict 3 | 4 | from pyoti import __version__ 5 | from pyoti.classes import Domain, IPAddress 6 | 7 | 8 | class Pulsedive(Domain, IPAddress): 9 | """Pulsedive Threat Intelligence Made Easy 10 | 11 | Pulsedive is a free threat intelligence platform. Search, scan, and enrich IPs, URLs, domains and other IOCs from OSINT feeds or submit your own. 12 | """ 13 | def __init__(self, api_key: str, api_url: str = "https://pulsedive.com/api"): 14 | """ 15 | :param api_key: Pulsedive API key 16 | :param api_url: Pulsedive API URL 17 | """ 18 | Domain.__init__(self, api_key=api_key, api_url=api_url) 19 | IPAddress.__init__(self, api_key=api_key, api_url=api_url) 20 | 21 | def _api_get(self, endpoint: str, iocvalue: str) -> requests.models.Response: 22 | """GET request to API 23 | 24 | :param endpoint: Pulsedive API endpoint for query 25 | :param iocvalue: domain or ip 26 | """ 27 | headers = {"User-Agent": f"PyOTI {__version__}"} 28 | params = {"indicator": iocvalue, "key": self.api_key} 29 | 30 | response = requests.request("GET", url=f"{self.api_url}{endpoint}", headers=headers, params=params) 31 | 32 | return response 33 | 34 | def check_domain(self) -> Dict: 35 | """Checks Domain reputation 36 | 37 | :return: dict of request response 38 | """ 39 | response = self._api_get(endpoint="/info.php", iocvalue=self.domain) 40 | 41 | return response.json() 42 | 43 | def check_ip(self) -> Dict: 44 | """Checks IP Address reputation 45 | 46 | :return: dict of request response 47 | """ 48 | response = self._api_get(endpoint="/info.php", iocvalue=self.ip) 49 | 50 | return response.json() 51 | -------------------------------------------------------------------------------- /pyoti/multis/onyphe.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from typing import Dict 3 | 4 | from pyoti import __version__ 5 | from pyoti.classes import Domain, IPAddress 6 | 7 | 8 | class Onyphe(Domain, IPAddress): 9 | """Onyphe Cyber Defense Search Engine 10 | 11 | ONYPHE is a cyber defense search engine for opensource and threat intelligence 12 | data collected by crawling various sources available on the internet or by 13 | listening to internet background noise. 14 | """ 15 | def __init__(self, api_key: str, api_url: str = "https://www.onyphe.io/api/v2"): 16 | """ 17 | :param api_key: Onyphe API key 18 | :param api_url: Onyphe base API URL 19 | """ 20 | Domain.__init__(self, api_key=api_key, api_url=api_url) 21 | IPAddress.__init__(self, api_key=api_key, api_url=api_url) 22 | 23 | def _api_get(self, endpoint: str) -> requests.models.Response: 24 | """Get request to API 25 | 26 | :param endpoint: Onyphe API endpoint 27 | :return: dict of request response 28 | """ 29 | headers = { 30 | "Authorization": f"apikey {self.api_key}", 31 | "Content-Type": "application/json", 32 | "User-Agent": f"PyOTI {__version__}" 33 | } 34 | 35 | response = requests.request("GET", url=endpoint, headers=headers) 36 | 37 | return response 38 | 39 | def check_domain(self) -> Dict: 40 | """Checks Domain reputation 41 | 42 | :return: dict of request response 43 | """ 44 | url = f"{self.api_url}/summary/domain/{self.domain}" 45 | response = self._api_get(url) 46 | 47 | return response.json() 48 | 49 | def check_ip(self) -> Dict: 50 | """Checks IP reputation 51 | 52 | :return: dict of request response 53 | """ 54 | url = f"{self.api_url}/summary/ip/{self.ip}" 55 | response = self._api_get(url) 56 | 57 | return response.json() 58 | -------------------------------------------------------------------------------- /pyoti/ips/abuseipdb.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from typing import Dict 3 | 4 | from pyoti import __version__ 5 | from pyoti.classes import IPAddress 6 | 7 | 8 | class AbuseIPDB(IPAddress): 9 | """AbuseIPDB IP Blacklist 10 | 11 | AbuseIPDB is a project dedicated to helping combat the spread of hackers, spammers, 12 | and abusive activity on the internet by providing a central blacklist to report and 13 | find IP addresses that have been associated with malicious activity. 14 | """ 15 | def __init__( 16 | self, api_key: str, api_url: str = "https://api.abuseipdb.com/api/v2" 17 | ): 18 | """ 19 | :param api_key: AbuseIPDB API key 20 | :param api_url: AbuseIPDB base API URL 21 | """ 22 | IPAddress.__init__(self, api_key, api_url) 23 | 24 | def _api_get(self, max_age: int) -> requests.models.Response: 25 | """ 26 | :param max_age: How far back in time (days) to fetch reports. (defaults to 30 days) 27 | """ 28 | headers = { 29 | "Accept": "application/json", 30 | "Key": self.api_key, 31 | "User-Agent": f"PyOTI {__version__}" 32 | } 33 | 34 | params = {"ipAddress": self.ip, "maxAgeInDays": max_age} 35 | 36 | response = requests.request( 37 | "GET", url=f"{self.api_url}/check", headers=headers, params=params 38 | ) 39 | 40 | return response 41 | 42 | def check_ip(self, max_age: int = 30) -> Dict: 43 | """ 44 | Checks IP reputation 45 | 46 | The check endpoint (api.abuseipdb.com/api/v2/check) accepts a single IP 47 | address (v4 or v6). Optionally you may set the max_age parameter to only 48 | return reports within the last X number of days. 49 | 50 | :param max_age: How far back in time (days) to fetch reports. (defaults to 30 days) 51 | :return: dict 52 | """ 53 | response = self._api_get(max_age=max_age) 54 | 55 | return response.json() 56 | -------------------------------------------------------------------------------- /examples/dt_domain_risk_to_csv.py: -------------------------------------------------------------------------------- 1 | import csv 2 | from argparse import ArgumentParser 3 | from tld.exceptions import TldDomainNotFound 4 | 5 | from pyoti.domains import IrisInvestigate 6 | from pyoti.utils import split_url_domain 7 | from keys import domaintools 8 | 9 | 10 | def run(args): 11 | iris = IrisInvestigate(api_key=domaintools) 12 | 13 | fields = ["Domain", "Risk Score"] 14 | 15 | with open("domain_risk.csv", "w") as csvfile: 16 | csvwriter = csv.writer(csvfile) 17 | 18 | csvwriter.writerow(fields) 19 | 20 | for dmn in args: 21 | if 'http' in dmn: 22 | try: 23 | iris.domain = split_url_domain(dmn) 24 | except TldDomainNotFound: 25 | continue 26 | else: 27 | dmnsplt = dmn.split('.') 28 | if len(dmnsplt) > 2: 29 | iris.domain = '.'.join(dmnsplt[-2:]) 30 | else: 31 | iris.domain = dmn 32 | 33 | try: 34 | domain_rep = iris.check_domain() 35 | risk_score = domain_rep[0]["domain_risk"]["risk_score"] 36 | except IndexError: 37 | risk_score = "N/A" 38 | except KeyError: 39 | risk_score = "N/A" 40 | except AttributeError: 41 | risk_score = "N/A" 42 | 43 | csvwriter.writerow([dmn, risk_score]) 44 | 45 | 46 | def main(): 47 | parser = ArgumentParser( 48 | prog="Domain Risk Score to CSV", 49 | description="Check Domaintools Iris Investigate for risk score outputted to CSV", 50 | ) 51 | parser.add_argument( 52 | "-f", 53 | "--domain_file", 54 | dest="domain_file", 55 | help="txt file of domains (one per line)", 56 | ) 57 | args = parser.parse_args() 58 | 59 | input_file = open(args.domain_file) 60 | 61 | run(input_file) 62 | print("[*] Finished! Check domain_risk.csv for your domain risk scores.") 63 | 64 | 65 | if __name__ == "__main__": 66 | main() 67 | -------------------------------------------------------------------------------- /update_keys.ps1: -------------------------------------------------------------------------------- 1 | <# 2 | .SYNOPSIS 3 | Update PyOTI API Keys 4 | .DESCRIPTION 5 | This PowerShell script is meant to be run from the root directory of PyOTI. 6 | It will check the diff between .\examples\keys.py.sample and .\examples\keys.py and 7 | append any API key variables that are new. 8 | Please make sure to update .\examples\keys.py with API secrets after running. 9 | .OUTPUTS 10 | Appends new API key variables in .\examples\keys.py and sets them to '' (empty). 11 | .NOTES 12 | Version: 1.0 13 | Author: JJ Josing 14 | Creation Date: 02/23/2021 15 | Purpose/Change: Initial script development 16 | 17 | .EXAMPLE 18 | powershell .\update_keys.ps1 19 | #> 20 | 21 | function Check-NewLine{ 22 | $content = [IO.File]::ReadAllText('.\examples\keys.py') 23 | ($content -match '(?<=\r\n)\z') 24 | } 25 | 26 | 27 | $sample_file = '.\examples\keys.py.sample' 28 | $keys_file = '.\examples\keys.py' 29 | 30 | $sample_variables = Get-Content $sample_file | ForEach-Object{$_.split("=")[0]} 31 | $keys_variables = Get-Content $keys_file | ForEach-Object{$_.split("=")[0]} 32 | 33 | $compare = Compare-Object -ReferenceObject $keys_variables -DifferenceObject $sample_variables 34 | $count = $compare | Measure-Object 35 | 36 | if($count.Count -eq '0'){ 37 | Write-Host -ForegroundColor Green "[*] No keys need to be updated!" 38 | } else 39 | { 40 | Write-Host -ForegroundColor Green "[!] New keys found!" 41 | $newline = Check-NewLine 42 | if ($newline -eq $false ){ 43 | Add-Content -Path ".\examples\keys.py" -Value "" 44 | } 45 | $compare | ForEach-Object{ 46 | if ($_.SideIndicator -eq "=>") 47 | { 48 | Write-Host -ForegroundColor Green "[!] Adding to .\examples\keys.py!" 49 | Add-Content -Path ".\examples\keys.py" -Value $_.InputObject -NoNewline 50 | Add-Content -Path ".\examples\keys.py" -Value "= ''" 51 | Write-Host -ForegroundColor Green "[+] $($_.InputObject) added to .\examples\keys.py!" 52 | } 53 | } 54 | Write-Host -ForegroundColor Yellow "[*] Add API secrets to .\examples\keys.py!" 55 | } 56 | -------------------------------------------------------------------------------- /examples/vt_hash_threat_classification.py: -------------------------------------------------------------------------------- 1 | import csv 2 | from argparse import ArgumentParser 3 | 4 | from pyoti.multis import VirusTotalV3 5 | from keys import virustotal 6 | 7 | 8 | def run(args): 9 | vt = VirusTotalV3(api_key=virustotal) 10 | 11 | fields = ["Hash", "Threat Classification", "Crowdsourced Yara Results"] 12 | 13 | with open("hash_classification.csv", "w") as csvfile: 14 | csvwriter = csv.writer(csvfile) 15 | 16 | csvwriter.writerow(fields) 17 | 18 | for hash in args: 19 | vt.file_hash = hash.strip('\n') 20 | hash_resp = vt.check_hash() 21 | 22 | if hash_resp.get('data'): 23 | row = ([hash_resp['data']['attributes']['sha256']]) 24 | if hash_resp['data']['attributes'].get('popular_threat_classification'): 25 | row += ([hash_resp['data']['attributes']['popular_threat_classification'].get('suggested_threat_label')]) 26 | if hash_resp['data']['attributes'].get('crowdsourced_yara_results'): 27 | for yara_result in hash_resp['data']['attributes']['crowdsourced_yara_results']: 28 | row += ([yara_result.get('description')]) 29 | elif hash_resp['data']['attributes'].get('known_distributors'): 30 | row += ([hash_resp['data']['attributes']['known_distributors'].get('distributors')]) 31 | csvwriter.writerow(row) 32 | else: 33 | row = ([hash, 'Not found!']) 34 | csvwriter.writerow(row) 35 | 36 | 37 | def main(): 38 | parser = ArgumentParser( 39 | prog="VT hash reputation to CSV", 40 | description="Check Virustotal for hash threat classification and any matches on yara rules", 41 | ) 42 | parser.add_argument( 43 | "-f", 44 | "--hash_file", 45 | dest="hash_file", 46 | help="txt file of file hashes (one per line)", 47 | ) 48 | args = parser.parse_args() 49 | 50 | input_file = open(args.hash_file) 51 | 52 | run(input_file) 53 | print("[*] Finished! Check hash_classification.csv for your file hash reputations.") 54 | 55 | 56 | if __name__ == "__main__": 57 | main() 58 | -------------------------------------------------------------------------------- /pyoti/multis/whoisxml.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from typing import Dict 3 | 4 | from pyoti import __version__ 5 | from pyoti.classes import Domain, EmailAddress, IPAddress 6 | 7 | 8 | class WhoisXML(Domain, EmailAddress, IPAddress): 9 | """WhoisXML WHOIS Records 10 | 11 | WhoisXML gathers a variety of domain ownership and registration data points from WHOIS database 12 | """ 13 | def __init__(self, api_key: str, api_url: str = "https://www.whoisxmlapi.com/whoisserver/WhoisService"): 14 | """ 15 | :param api_key: WhoisXML API key 16 | :param api_url: WhoisXML API URL 17 | """ 18 | Domain.__init__(self, api_key=api_key, api_url=api_url) 19 | EmailAddress.__init__(self, api_key=api_key, api_url=api_url) 20 | IPAddress.__init__(self, api_key=api_key, api_url=api_url) 21 | 22 | def _api_get(self, lookup: str) -> requests.models.Response: 23 | """Get request to API""" 24 | headers = {"User-Agent": f"PyOTI {__version__}"} 25 | 26 | params = { 27 | "apiKey": self.api_key, 28 | "domainName": lookup, 29 | "outputFormat": "JSON", 30 | "ip": 1, # return IPs for the domain name 31 | "ipWhois": 1, # return the WHOIS record for the hosting IP if the WHOIS record for the tld of the input domain is not supported 32 | "checkProxyData": 1, # fetch proxy/WHOIS guard data in the WhoisRecord → privateWhoisProxy schema element 33 | "ignoreRawTexts": 1 # strip all raw text from the output 34 | } 35 | 36 | response = requests.request("GET", url=self.api_url, headers=headers, params=params) 37 | 38 | return response 39 | 40 | def check_domain(self) -> Dict: 41 | """Checks Domain WHOIS""" 42 | response = self._api_get(lookup=self.domain) 43 | 44 | return response.json() 45 | 46 | def check_email(self) -> Dict: 47 | """Checks Domain WHOIS from Email Address""" 48 | response = self._api_get(lookup=self.email) 49 | 50 | return response.json() 51 | 52 | def check_ip(self) -> Dict: 53 | """Checks IP WHOIS""" 54 | response = self._api_get(lookup=self.ip) 55 | 56 | return response.json() 57 | 58 | -------------------------------------------------------------------------------- /pyoti/multis/binaryedge.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from typing import Dict 3 | 4 | from pyoti import __version__ 5 | from pyoti.classes import Domain, EmailAddress, IPAddress 6 | 7 | 8 | class BinaryEdge(Domain, EmailAddress, IPAddress): 9 | """BinaryEdge continuously collects and correlates data from internet accessible devices, allowing organizations 10 | to see what is their attack surface and what they are exposing to attackers.""" 11 | def __init__(self, api_key: str, api_url: str = "https://api.binaryedge.io/v2"): 12 | """ 13 | :param api_key: BinaryEdge API key 14 | :param api_url: BinaryEdge base API url 15 | """ 16 | Domain.__init__(self, api_key=api_key, api_url=api_url) 17 | EmailAddress.__init__(self, api_key=api_key, api_url=api_url) 18 | IPAddress.__init__(self, api_key=api_key, api_url=api_url) 19 | 20 | def _api_get(self, url: str) -> requests.models.Response: 21 | """Get request to API""" 22 | headers = { 23 | "User-Agent": f"PyOTI {__version__}", 24 | "X-Key": self.api_key 25 | } 26 | response = requests.request("GET", url=url, headers=headers) 27 | 28 | return response 29 | 30 | def check_email_dataleaks(self) -> Dict: 31 | """Check email address in dataleaks""" 32 | url = f"{self.api_url}/query/dataleaks/email/{self.email}" 33 | response = self._api_get(url=url) 34 | 35 | return response.json() 36 | 37 | def check_ip_cve(self) -> Dict: 38 | """Get list of CVEs that might affect IP""" 39 | url = f"{self.api_url}/query/cve/ip/{self.ip}" 40 | response = self._api_get(url=url) 41 | 42 | return response.json() 43 | 44 | def check_ip_host(self) -> Dict: 45 | """Check IP host reputation 46 | 47 | :return: dict of query results 48 | """ 49 | url = f"{self.api_url}/query/ip/{self.ip}" 50 | response = self._api_get(url=url) 51 | 52 | return response.json() 53 | 54 | def get_domain_subdomains(self) -> Dict: 55 | """Get list of subdomains known from a domain""" 56 | url = f"{self.api_url}/query/domains/subdomain/{self.domain}" 57 | response = self._api_get(url=url) 58 | 59 | return response.json() 60 | -------------------------------------------------------------------------------- /pyoti/multis/circlpssl.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from typing import Dict, Optional 3 | 4 | from pyoti import __version__ 5 | from pyoti.classes import FileHash, IPAddress 6 | 7 | 8 | class CIRCLPSSL(FileHash, IPAddress): 9 | """CIRCLPSSL Historical X.509 Certificates 10 | 11 | CIRCL Passive SSL stores historical X.509 certificates seen per IP address. 12 | """ 13 | def __init__(self, api_key: str, api_url: str = "https://www.circl.lu/v2pssl"): 14 | """ 15 | :param api_key: CIRCLPSSL API key 16 | :param api_url: CIRCLPSSL base API URL 17 | """ 18 | FileHash.__init__(self, api_key=api_key, api_url=api_url) 19 | IPAddress.__init__(self, api_key=api_key, api_url=api_url) 20 | 21 | def _api_get(self, url: str) -> requests.models.Response: 22 | """GET request to API""" 23 | credentials = self.api_key.split(":") 24 | 25 | headers = {"User-Agent": f"PyOTI {__version__}"} 26 | 27 | response = requests.request("GET", url=url, auth=(credentials[0], credentials[1]), headers=headers) 28 | 29 | return response 30 | 31 | def check_ip(self, cidr_block: Optional[int] = 32) -> Dict: 32 | """Checks IP reputation 33 | 34 | Checks CIRCL Passive SSL for historical X.509 certificates for a given IP. 35 | 36 | :param cidr_block: can be CIDR blocks between /23 and /32 37 | :return: dict of query results 38 | """ 39 | url = f"{self.api_url}/query/{self.ip}/{cidr_block}" 40 | response = self._api_get(url=url) 41 | 42 | return response.json() 43 | 44 | def check_hash(self) -> Dict: 45 | """Checks SHA1 fingerprint of a certificate 46 | 47 | Checks CIRCL Passive SSL for historical X.509 certificates for a given 48 | certificate fingerprint. 49 | 50 | :return: dict of query results 51 | """ 52 | url = f"{self.api_url}/cquery/{self.file_hash}" 53 | response = self._api_get(url=url) 54 | 55 | return response.json() 56 | 57 | def fetch_cert(self) -> Dict: 58 | """Fetch Certificate 59 | 60 | Fetches/parses a specified certificate from CIRCL Passive SSL for a 61 | given certificate fingerprint. 62 | 63 | :return: dict with certificate info 64 | """ 65 | url = f"{self.api_url}/cfetch/{self.file_hash}" 66 | response = self._api_get(url=url) 67 | 68 | return response.json() 69 | -------------------------------------------------------------------------------- /pyoti/multis/xforce.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import requests 3 | from typing import Dict 4 | 5 | from pyoti import __version__ 6 | from pyoti.classes import Domain, FileHash, IPAddress, URL 7 | 8 | 9 | class XForceExchange(Domain, FileHash, IPAddress, URL): 10 | """XForceExchange 11 | 12 | IBM X-Force Exchange is a cloud-based threat intelligence platform that allows you to consume, share and act on 13 | threat intelligence. It enables you to rapidly research the latest global security threats, aggregate actionable 14 | intelligence, consult with experts and collaborate with peers. IBM X-Force Exchange, supported by human- and 15 | machine-generated intelligence, leverages the scale of IBM X-Force to help users stay ahead of emerging threats. 16 | """ 17 | def __init__(self, api_key: str, api_url: str = "https://api.xforce.ibmcloud.com/api"): 18 | enc_bytes = base64.b64encode(api_key.encode("utf-8")) 19 | api_key = str(enc_bytes, "utf-8") 20 | Domain.__init__(self, api_key=api_key, api_url=api_url) 21 | FileHash.__init__(self, api_key=api_key, api_url=api_url) 22 | IPAddress.__init__(self, api_key=api_key, api_url=api_url) 23 | URL.__init__(self, api_key=api_key, api_url=api_url) 24 | 25 | def _api_get(self, url) -> requests.models.Response: 26 | """GET request to XForce API""" 27 | headers = { 28 | "Accept": "application/json", 29 | "Authorization": f"Basic {self.api_key}", 30 | "User-Agent": f"PyOTI {__version__}" 31 | } 32 | 33 | response = requests.request("GET", url=url, headers=headers) 34 | 35 | return response 36 | 37 | def check_domain(self) -> Dict: 38 | """Checks Domain reputation""" 39 | response = self._api_get(url=f"{self.api_url}/url/{self.domain}") 40 | 41 | return response.json() 42 | 43 | def check_hash(self) -> Dict: 44 | """Checks File Hash reputation""" 45 | response = self._api_get(url=f"{self.api_url}/malware/{self.file_hash}") 46 | 47 | return response.json() 48 | 49 | def check_ip(self) -> Dict: 50 | """Checks IP Address reputation""" 51 | response = self._api_get(url=f"{self.api_url}/ipr/{self.ip}") 52 | 53 | return response.json() 54 | 55 | def check_url(self) -> Dict: 56 | """Checks URL reputation""" 57 | response = self._api_get(url=f"{self.api_url}/url/{self.url}") 58 | 59 | return response.json() 60 | -------------------------------------------------------------------------------- /docs/windows/README.md: -------------------------------------------------------------------------------- 1 | ## Installation for Windows 2 | It is a requirement to have both Git and Python3 installed and in your $PATH. 3 | 4 | You may also need to set execution policy to unrestricted in order to create/activate a Python3 virtual environment. 5 | 6 | Virtualenv (recommended): 7 | ```powershell 8 | # clone PyOTI repository and copy sample keys file 9 | git clone https://github.com/RH-ISAC/PyOTI "$env:USERPROFILE\PyOTI" 10 | Set-Location -Path "$env:USERPROFILE\PyOTI" 11 | Copy-Item "$env:USERPROFILE\PyOTI\examples\keys.py.sample" -Destination "$env:USERPROFILE\PyOTI\examples\keys.py" 12 | # install/setup virtual environment 13 | Set-ExecutionPolicy Unrestricted -Force 14 | py -m pip install virtualenv 15 | py -m venv venv 16 | .\venv\Scripts\Activate.ps1 17 | # make sure to fill in your API secrets! 18 | notepad "$env:USERPROFILE\PyOTI\examples\keys.py" 19 | # install PyOTI library 20 | py -m pip install . 21 | ``` 22 | 23 | **Important Note:** 24 | 25 | If you are using SSL inspection/MITM proxy and having issues running PyOTI, try appending your root certificate to the following file: 26 | ```powershell 27 | $env:USERPROFILE\PyOTI\venv\Lib\site-packages\certifi\cacert.pem 28 | ``` 29 | 30 | No virtualenv: 31 | ```powershell 32 | # clone PyOTI repository and copy sample keys file 33 | git clone https://github.com/RH-ISAC/PyOTI "$env:USERPROFILE\PyOTI" 34 | Set-Location -Path "$env:USERPROFILE\PyOTI" 35 | Copy-Item "$env:USERPROFILE\PyOTI\examples\keys.py.sample" -Destination "$env:USERPROFILE\PyOTI\examples\keys.py" 36 | # make sure to fill in your API secrets! 37 | notepad "$env:USERPROFILE\PyOTI\examples\keys.py" 38 | # install PyOTI library 39 | py -m pip install . 40 | ``` 41 | ## 42 | 43 | ## Updating for Windows 44 | Virtualenv: 45 | ```powershell 46 | # activate virtual environment 47 | Set-ExecutionPolicy Unrestricted -Force 48 | Set-Location -Path "$env:USERPROFILE\PyOTI" 49 | .\venv\Scripts\Activate.ps1 50 | # pull PyOTI repository and update keys 51 | git pull 52 | powershell .\update_keys.ps1 53 | # make sure to fill in your updated API secrets! 54 | notepad "$env:USERPROFILE\PyOTI\examples\keys.py" 55 | # make sure PyOTI library is updated 56 | py -m pip install . 57 | ``` 58 | No virtualenv: 59 | ```powershell 60 | # pull PyOTI repository 61 | Set-ExecutionPolicy Unrestricted -Force 62 | Set-Location -Path "$env:USERPROFILE\PyOTI" 63 | git pull 64 | powershell .\update_keys.ps1 65 | # make sure to fill in your updated API secrets! 66 | notepad "$env:USERPROFILE\PyOTI\examples\keys.py" 67 | # make sure PyOTI library is updated 68 | py -m pip install . 69 | ``` -------------------------------------------------------------------------------- /pyoti/multis/maltiverseioc.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import requests 3 | from typing import Dict 4 | 5 | from pyoti import __version__ 6 | from pyoti.classes import Domain, FileHash, IPAddress, URL 7 | from pyoti.exceptions import MaltiverseIOCError 8 | 9 | 10 | class MaltiverseIOC(Domain, FileHash, IPAddress, URL): 11 | """MaltiverseIOC IOC Search Engine 12 | 13 | Maltiverse is an open IOC search engine providing collective intelligence. 14 | """ 15 | def __init__(self, api_key: str, api_url: str = "https://api.maltiverse.com"): 16 | """ 17 | :param api_key: Maltiverse API key 18 | :param api_url: Maltiverse base API URL 19 | """ 20 | Domain.__init__(self, api_key=api_key, api_url=api_url) 21 | FileHash.__init__(self, api_key=api_key, api_url=api_url) 22 | IPAddress.__init__(self, api_key=api_key, api_url=api_url) 23 | URL.__init__(self, api_key=api_key, api_url=api_url) 24 | 25 | def _api_get(self, endpoint: str) -> requests.models.Response: 26 | """GET request to API 27 | 28 | :param endpoint: API endpoint 29 | """ 30 | headers = { 31 | "Accept": "application/json", 32 | "Authorization": f"Bearer {self.api_key}", 33 | "User-Agent": f"PyOTI {__version__}" 34 | } 35 | 36 | response = requests.request("GET", url=f"{self.api_url}/{endpoint}", headers=headers) 37 | 38 | return response 39 | 40 | def check_domain(self) -> Dict: 41 | """Checks Domain reputation 42 | 43 | :return: dict of query result 44 | """ 45 | response = self._api_get(endpoint=f"hostname/{self.domain}") 46 | 47 | return response.json() 48 | 49 | def check_hash(self) -> Dict: 50 | """Checks File Hash reputation 51 | 52 | :return: dict of query result 53 | """ 54 | if len(self.file_hash) == 32: 55 | response = self._api_get(endpoint=f"search?query=md5:{self.file_hash}") 56 | elif len(self.file_hash) == 64: 57 | response = self._api_get(endpoint=f"sample/{self.file_hash}") 58 | else: 59 | raise MaltiverseIOCError("You can only query Maltiverse for MD5 or SHA256!") 60 | 61 | return response.json() 62 | 63 | def check_ip(self) -> Dict: 64 | """Checks IP reputation 65 | 66 | :return: dict of query result 67 | """ 68 | response = self._api_get(endpoint=f"ip/{self.ip}") 69 | 70 | return response.json() 71 | 72 | def check_url(self) -> Dict: 73 | """Checks URL reputation 74 | 75 | :return: dict of query result 76 | """ 77 | url_hash = hashlib.sha256(self.url.encode("utf-8")).hexdigest() 78 | response = self._api_get(endpoint=f"url/{url_hash}") 79 | 80 | return response.json() 81 | -------------------------------------------------------------------------------- /examples/email_reputation_to_csv.py: -------------------------------------------------------------------------------- 1 | import csv 2 | from argparse import ArgumentParser 3 | 4 | from pyoti.emails import EmailRepIO 5 | from keys import sublime 6 | 7 | 8 | def run(args): 9 | eml = EmailRepIO(api_key=sublime) 10 | 11 | fields = [ 12 | "Email Address", 13 | "Reputation", 14 | "Suspicious", 15 | "Domain Exists", 16 | "Domain Reputation", 17 | "New Domain", 18 | "Disposable", 19 | "Deliverable", 20 | "Spoofable", 21 | "SPF Strict", 22 | "DMARC Enforced" 23 | ] 24 | 25 | with open("email_reputation.csv", "w") as csvfile: 26 | csvwriter = csv.writer(csvfile) 27 | 28 | csvwriter.writerow(fields) 29 | 30 | for email in args: 31 | eml.email = email 32 | eml_rep = eml.check_email() 33 | 34 | eml_address = email 35 | try: 36 | eml_reputation = eml_rep["reputation"] 37 | eml_sus = eml_rep["suspicious"] 38 | eml_dmn_exsts = eml_rep["details"]["domain_exists"] 39 | eml_dmn_rep = eml_rep["details"]["domain_reputation"] 40 | eml_dmn_new = eml_rep["details"]["new_domain"] 41 | eml_disposable = eml_rep["details"]["disposable"] 42 | eml_deliverable = eml_rep["details"]["deliverable"] 43 | eml_spoofable = eml_rep["details"]["spoofable"] 44 | eml_spf_strict = eml_rep["details"]["spf_strict"] 45 | eml_dmarc_enforced = eml_rep["details"]["dmarc_enforced"] 46 | 47 | csvwriter.writerow( 48 | [ 49 | eml_address, 50 | eml_reputation, 51 | eml_sus, 52 | eml_dmn_exsts, 53 | eml_dmn_rep, 54 | eml_dmn_new, 55 | eml_disposable, 56 | eml_deliverable, 57 | eml_spoofable, 58 | eml_spf_strict, 59 | eml_dmarc_enforced 60 | ] 61 | ) 62 | except KeyError: 63 | csvwriter.writerow([eml_address, None, None, None, None, None, None, None, None, None, None]) 64 | 65 | 66 | def main(): 67 | parser = ArgumentParser( 68 | prog="Email Reputation to CSV", 69 | description="Check EmailRep.io API for email address reputation outputted to CSV", 70 | ) 71 | parser.add_argument( 72 | "-f", 73 | "--email_file", 74 | dest="email_file", 75 | help="txt file of email addresses (one per line)", 76 | ) 77 | args = parser.parse_args() 78 | 79 | input_file = open(args.email_file) 80 | 81 | run(input_file) 82 | print( 83 | "[*] Finished! Check email_reputation.csv for your email address reputations." 84 | ) 85 | 86 | 87 | if __name__ == "__main__": 88 | main() 89 | -------------------------------------------------------------------------------- /pyoti/domains/circlpdns.py: -------------------------------------------------------------------------------- 1 | import json 2 | import requests 3 | from typing import Dict, List, Union 4 | 5 | from pyoti import __version__ 6 | from pyoti.classes import Domain 7 | from pyoti.exceptions import CIRCLPDNSError 8 | from pyoti.utils import epoch_to_date 9 | 10 | 11 | class CIRCLPDNS(Domain): 12 | """CIRCLPDNS Historical DNS Records 13 | 14 | CIRCL Passive DNS stores historical DNS records from various resources including malware analysis or partners. 15 | """ 16 | def __init__(self, api_key: str, api_url: str = "https://www.circl.lu/pdns/query"): 17 | """ 18 | :param api_key: CIRCLPDNS API key 19 | :param api_url: CIRCLPDNS base API URL 20 | """ 21 | self.sort_choice = ['count', 'rdata', 'rrname', 'rrtype', 'time_first', 'time_last'] 22 | Domain.__init__(self, api_key=api_key, api_url=api_url) 23 | 24 | def _api_get(self) -> Union[requests.models.Response, Dict]: 25 | """GET request to API""" 26 | credentials = self.api_key.split(":") 27 | 28 | headers = {"User-Agent": f"PyOTI {__version__}"} 29 | 30 | response = requests.request( 31 | "GET", 32 | url=f"{self.api_url}/{self.domain}", 33 | auth=(credentials[0], credentials[1]), 34 | headers=headers) 35 | 36 | if response.status_code != 200: 37 | if response.status_code == 401: 38 | return {"error": {f"{response.status_code}": "Not authenticated: is authentication correct?"}} 39 | elif response.status_code == 403: 40 | return {"error": {f"{response.status_code}": "Not authorized to access resource!"}} 41 | elif response.status_code == 429: 42 | return {"error": {f"{response.status_code}": "Quota exhausted!"}} 43 | elif 500 <= response.status_code < 600: 44 | return {"error": {f"{response.status_code}": "Server error!"}} 45 | else: 46 | return {"error": "Something went wrong!"} 47 | 48 | return response 49 | 50 | def check_domain(self, sort_by: str = "time_last") -> List[Dict]: 51 | """Checks domain reputation 52 | 53 | Checks CIRCL Passive DNS for historial DNS records for a given domain. 54 | 55 | :param sort_by: how returned data should be sorted 56 | :return: list of dicts 57 | """ 58 | if sort_by not in self.sort_choice: 59 | raise CIRCLPDNSError(f"You can only sort by the following: {self.sort_choice}") 60 | response = self._api_get() 61 | 62 | to_return = [] 63 | for line in response.text.split('\n'): 64 | if len(line) == 0: 65 | continue 66 | try: 67 | obj = json.loads(line) 68 | except Exception: 69 | continue 70 | obj['time_first'] = epoch_to_date(obj['time_first']) 71 | obj['time_last'] = epoch_to_date(obj['time_last']) 72 | to_return.append(obj) 73 | to_return = sorted(to_return, key=lambda k: k[sort_by]) 74 | 75 | return to_return 76 | -------------------------------------------------------------------------------- /pyoti/domains/irisinvestigate.py: -------------------------------------------------------------------------------- 1 | import hmac 2 | import hashlib 3 | import requests 4 | from datetime import datetime 5 | from typing import Dict, List 6 | from urllib.parse import urlsplit 7 | 8 | from pyoti import __version__ 9 | from pyoti.classes import Domain 10 | 11 | 12 | class IrisInvestigate(Domain): 13 | """IrisInvestigate Domain Risk Score/Historical DNS Records/SSL Profiles 14 | 15 | Iris is a proprietary threat intelligence/investigation platform by Domaintools 16 | """ 17 | def __init__(self, api_key: str, api_url: str = "https://api.domaintools.com/v1/iris-investigate/"): 18 | """ 19 | :param api_key: Domaintools API key in 'USER:SECRET' format 20 | :param api_url: Domaintools API url 21 | """ 22 | Domain.__init__(self, api_key, api_url) 23 | 24 | def _api_post(self, **kwargs) -> requests.models.Response: 25 | """POST request to Iris Investigate API 26 | 27 | :return: dict of request response 28 | """ 29 | creds = self.api_key.split(":") 30 | timestamp = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ') 31 | param = ''.join([creds[0], timestamp, urlsplit(self.api_url).path]) 32 | signature = hmac.new(creds[1].encode('utf-8'), param.encode('utf-8'), digestmod=hashlib.sha256).hexdigest() 33 | 34 | headers = {"User-Agent": f"PyOTI {__version__}"} 35 | 36 | params = { 37 | 'api_username': creds[0], 38 | 'signature': signature, 39 | 'timestamp': timestamp, 40 | } 41 | 42 | if kwargs.get('domain_list'): 43 | params['domain'] = ','.join(kwargs.get('domain_list')) 44 | 45 | else: 46 | params['domain'] = self.domain 47 | 48 | response = requests.request("POST", url=self.api_url, headers=headers, params=params) 49 | 50 | return response 51 | 52 | def check_domain(self) -> List[Dict]: 53 | """Checks domain reputation 54 | 55 | :return: list of dict containing results from request response 56 | """ 57 | response = self._api_post() 58 | r = response.json().get('response') 59 | 60 | return r.get('results') 61 | 62 | def bulk_check_domain(self, domain_list: List[str]) -> List[Dict]: 63 | """Bulk check domain reputation 64 | 65 | :param domain_list: List of domains to check reputation 66 | :return: list of dicts containing results from request response 67 | """ 68 | if len(domain_list) <= 100: 69 | response = self._api_post(domain_list=domain_list) 70 | r = response.json().get('response') 71 | 72 | return r.get('results') 73 | else: 74 | chunk_size = 100 75 | chunks = [domain_list[i:i + chunk_size] for i in range(0, len(domain_list), chunk_size)] 76 | results = [] 77 | 78 | for chunk in chunks: 79 | response = self._api_post(domain_list=chunk) 80 | r = response.json().get('response') 81 | [results.append(result) for result in r.get('results')] 82 | 83 | return results 84 | -------------------------------------------------------------------------------- /pyoti/hashes/circlhashlookup.py: -------------------------------------------------------------------------------- 1 | import json 2 | import requests 3 | from typing import Dict, List 4 | 5 | from pyoti import __version__ 6 | from pyoti.classes import FileHash 7 | from pyoti.exceptions import CIRCLHashLookupError 8 | 9 | 10 | class CIRCLHashLookup(FileHash): 11 | """CIRCLHashLookup Known Hash Reputation 12 | 13 | CIRCLHashLookup is a public API to lookup hash values against known database of files. 14 | NSRL RDS database is included as well as many others that are also included. 15 | """ 16 | def __init__(self, api_url: str = "https://hashlookup.circl.lu/"): 17 | """ 18 | :param api_url: CIRCLHashLookup base API URL 19 | """ 20 | FileHash.__init__(self, api_url=api_url) 21 | 22 | def _api_get(self, url: str) -> requests.models.Response: 23 | """GET request to API for single hash lookup""" 24 | headers = { 25 | "Accept": "application/json", 26 | "User-Agent": f"PyOTI {__version__}" 27 | } 28 | 29 | response = requests.request("GET", url=url, headers=headers) 30 | 31 | return response 32 | 33 | def _api_post(self, url: str, hash_list: List[str]) -> requests.models.Response: 34 | """POST request to API for bulk hash lookup""" 35 | headers = { 36 | "Content-Type": "application/json", 37 | "User-Agent": f"PyOTI {__version__}" 38 | } 39 | 40 | data = {"hashes": hash_list} 41 | 42 | response = requests.request("POST", url=url, headers=headers, data=json.dumps(data)) 43 | 44 | return response 45 | 46 | def check_hash(self) -> Dict: 47 | """ Checks File Hash reputation 48 | 49 | :return: request response dict 50 | """ 51 | if len(self.file_hash) == 32: 52 | url = f"{self.api_url}/lookup/md5/{self.file_hash}" 53 | elif len(self.file_hash) == 40: 54 | url = f"{self.api_url}/lookup/sha1/{self.file_hash}" 55 | elif len(self.file_hash) == 64: 56 | url = f"{self.api_url}/lookup/sha256/{self.file_hash}" 57 | else: 58 | return {} 59 | response = self._api_get(url=url) 60 | 61 | return response.json() 62 | 63 | def bulk_check_hash(self, algo: str, hash_list: List[str]) -> Dict: 64 | """Bulk Check File Hash Reputation 65 | 66 | :param algo: Hashing algorithm to bulk check (MD5 or SHA1) 67 | :param hash_list: List of hashes (MD5 or SHA1) 68 | :return: request response dict 69 | """ 70 | if algo.lower() == "md5": 71 | url = f"{self.api_url}/bulk/md5" 72 | if [h for h in hash_list if len(h) != 32]: 73 | raise CIRCLHashLookupError("Hash list must be all MD5 hashes!") 74 | elif algo.lower() == "sha1": 75 | url = f"{self.api_url}/bulk/sha1" 76 | if [h for h in hash_list if len(h) != 40]: 77 | raise CIRCLHashLookupError("Hash list must be all SHA1 hashes!") 78 | else: 79 | return {} 80 | response = self._api_post(url=url, hash_list=hash_list) 81 | 82 | return response.json() 83 | -------------------------------------------------------------------------------- /pyoti/domains/checkdmarc.py: -------------------------------------------------------------------------------- 1 | import aiodns 2 | import asyncio 3 | import pycares 4 | import re 5 | import sys 6 | from typing import Dict, List 7 | 8 | from pyoti.classes import Domain 9 | 10 | 11 | class CheckDMARC(Domain): 12 | """CheckDMARC SPF/DMARC Records 13 | 14 | CheckDMARC validates SPF and DMARC DNS records. 15 | """ 16 | def __init__(self, domain: str = None): 17 | Domain.__init__(self, domain=domain) 18 | 19 | def query_txt(self, name: str) -> List[pycares.ares_query_txt_result]: 20 | """Asynchronous DNS query for TXT record""" 21 | try: 22 | if sys.platform == "win32": 23 | asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) 24 | 25 | async def query(host, record): 26 | resolver = aiodns.DNSResolver(nameservers=["208.67.222.222"]) # OpenDNS nameserver 27 | return await resolver.query(host, record) 28 | 29 | result = asyncio.run(query(host=name, record="TXT")) 30 | 31 | return result 32 | except aiodns.error.DNSError: 33 | return [] 34 | 35 | def _get_spf(self) -> Dict: 36 | """Get SPF record for a given Domain""" 37 | result = self.query_txt(name=self.domain) 38 | 39 | spf_json = {} 40 | for r in result: 41 | if isinstance(r.text, str): 42 | if re.search(r"^v=spf1", r.text): 43 | spf_json["txt"] = r.text 44 | spf_json["ttl"] = r.ttl 45 | 46 | return spf_json 47 | 48 | def _get_dmarc(self) -> Dict: 49 | """GET DMARC record for a given Domain""" 50 | result = self.query_txt(name=f"_dmarc.{self.domain}") 51 | 52 | dmarc_json = {} 53 | 54 | try: 55 | if re.search(r"^v=DMARC", result[0].text): 56 | dmarc_json["txt"] = result[0].text 57 | dmarc_json["ttl"] = result[0].ttl 58 | 59 | return dmarc_json 60 | 61 | except IndexError: 62 | return {} 63 | 64 | def _spoofable_check(self, results: Dict) -> Dict: 65 | """Check if domain is spoofable""" 66 | if not results.get("dmarc") or not results.get("spf"): 67 | results['spoofable'] = True 68 | elif not re.search(r"([-~]+(all))$", results.get("spf").get("txt")): 69 | results['spoofable'] = True 70 | elif re.search(r"\s*p=none", results.get("dmarc").get("txt")): 71 | results['spoofable'] = True 72 | elif not re.search(r"\s*p=([^;]*)\s*", results.get("dmarc").get("txt")): 73 | results['spoofable'] = True 74 | 75 | return results 76 | 77 | def check_domain(self) -> Dict: 78 | """Checks Domain reputation 79 | 80 | Checks for any SPF or DMARC records for a given Domain. 81 | 82 | :return: dict 83 | """ 84 | spf = self._get_spf() 85 | dmarc = self._get_dmarc() 86 | 87 | results = {"domain": self.domain, "dmarc": dmarc, "spf": spf, 'spoofable': False} 88 | 89 | spoofable_results = self._spoofable_check(results) 90 | 91 | return spoofable_results 92 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | Changelog 2 | ========= 3 | v0.4.0 (2025-01-14) 4 | ----------------- 5 | 6 | Changes 7 | ~~~~~~~ 8 | - Bumped PyOTI version 9 | - Updated README.md 10 | - Removed old VirusTotalV2 multis class 11 | ~~~~~~~ 12 | 13 | New 14 | ~~~~~~ 15 | - Added BinaryEdge multis class 16 | - Added bulk hash check CIRCLHashLookup method and exception 17 | - Added bulk hash check IrisInvestigate method 18 | - Added bulk quick and bulk context check GreyNoise methods 19 | - Added bulk url check GoogleSafeBrowsing method 20 | - Added upload file VirusTotalV3 method 21 | - Added Triage multis class 22 | - Added Stairwell multis class 23 | - Added URLscan class method to submit urls 24 | - Added check_ip and check_email class methods to WhoisXML 25 | - Added IP2WHOIS and IP2Location multis classes 26 | - Added FileScanIO and MetaDefenderCloudV4 multi classes 27 | - Added JoeSandbox multi class 28 | - Added CiscoUmbrellaInvestigate multi class 29 | ~~~~~~ 30 | 31 | 32 | v0.3.3.2 (2023-02-22) 33 | --------------------- 34 | 35 | Changes 36 | ~~~~~~~ 37 | - Bumped PyOTI version 38 | - Added conditional check in CheckDMARC _get_spf() method 39 | ~~~~~~~ 40 | 41 | 42 | v0.3.3.1 (2022-11-10) 43 | --------------------- 44 | 45 | Changes 46 | ~~~~~~~ 47 | - Bumped PyOTI version 48 | - Removed semicolon from regex search for DMARC policy in CheckDMARC 49 | ~~~~~~~ 50 | 51 | 52 | v0.3.3 (2022-08-01) 53 | ------------------- 54 | 55 | Changes 56 | ~~~~~~~ 57 | - Bumped PyOTI version 58 | - Added handling in DNSBlocklist for surbl when domain appears on multiple lists 59 | - Removed GoogleSafeBrowsing exception, return the error instead of raising an exception 60 | ~~~~~~~ 61 | 62 | v0.3.2.1 (2022-07-22) [bugfix] 63 | ---------------------------- 64 | 65 | Changes 66 | ~~~~~~~ 67 | - Refactored regex used in CheckDMARC ._spoofable_check() [AttributeError: 'NoneType' object has no attribute 'group'] 68 | ~~~~~~~ 69 | 70 | v0.3.2 (2022-07-19) 71 | ------------------- 72 | 73 | Changes 74 | ~~~~~~~ 75 | - Bumped PyOTI version 76 | - Removed SpamhausError exception 77 | - Refactored DNSBlocklist module and added additional blocklist return codes 78 | ~~~~~~~ 79 | 80 | New 81 | ~~~ 82 | - Added ThreatFox integration 83 | - Added MalwareBazaar integration 84 | - Added XforceExchange integration 85 | - Added example script to enrich a MISP event using PyOTI 86 | ~~~ 87 | 88 | v0.3.1 (2022-06-22) 89 | ------------------- 90 | 91 | Changes 92 | ~~~~~~~ 93 | - Added package exclusions in setup.cfg 94 | - Bumped PyOTI version 95 | ~~~~~~~ 96 | 97 | v0.3.0 (2022-06-17) 98 | ------------------- 99 | 100 | Changes 101 | ~~~~~~~ 102 | - Separated each integration into its own file rather than in one module based on IOC types. 103 | - MalwareHashRegistry uses new REST API rather than DNS query. 104 | - Phishtank uses HTTPS API and data is returned in JSON format from API. 105 | - HybridAnalysis can now search for domains and ip addresses. 106 | - Removed get_hash_type() utility in favor of using len() on hash to determine hash types. 107 | - Removed all external library dependencies except requests, aiodns, and disposable-email-domains. 108 | - Switched from setup.py in favor of using setup.cfg. 109 | - Updated Linux/Windows install docs 110 | ~~~~~~~ 111 | 112 | New 113 | ~~~ 114 | - Added CIRCLHashLookup integration. 115 | - Added GreyNoise integration for community, context, quick check, and RIOT APIs. 116 | ~~~ -------------------------------------------------------------------------------- /pyoti/multis/stairwell.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from typing import Dict, List, Optional 3 | 4 | from pyoti import __version__ 5 | from pyoti.classes import Domain, FileHash, IPAddress 6 | 7 | 8 | class Stairwell(Domain, FileHash, IPAddress): 9 | """ 10 | Stairwell provides a unique view into your enterprise and operational environments via lightweight forwarders or 11 | cli utilities which leverage automated analysis, robust YARA rule libraries, shared malware feeds, privately run AV 12 | verdicts, static & dynamic analysis, malware unpacking, and variant discovery. 13 | """ 14 | def __init__(self, api_key: str, api_url: str = "https://app.stairwell.com/v1"): 15 | Domain.__init__(self, api_key, api_url) 16 | FileHash.__init__(self, api_key, api_url) 17 | IPAddress.__init__(self, api_key, api_url) 18 | 19 | def _api_get(self, endpoint: str, params: Optional[Dict] = None) -> requests.models.Response: 20 | """GET request to Stairwell API 21 | 22 | :param endpoint: Stairwell API endpoint 23 | :param params: params for request 24 | """ 25 | headers = { 26 | "Accept": "application/json", 27 | "Authorization": self.api_key, 28 | "User-Agent": f"PyOTI {__version__}" 29 | } 30 | rparams = params 31 | 32 | uri = self.api_url + endpoint 33 | response = requests.request("GET", url=uri, headers=headers, params=rparams) 34 | 35 | return response 36 | 37 | def check_domain(self) -> Dict: 38 | params = {'filter': f'net.hostname == "{self.domain}"'} 39 | 40 | response = self._api_get(endpoint='/objects/metadata', params=params) 41 | 42 | return response.json() 43 | 44 | def check_hash(self) -> Dict: 45 | response = self._api_get(endpoint=f'/objects/{self.file_hash}/metadata') 46 | 47 | return response.json() 48 | 49 | def check_ip(self) -> Dict: 50 | params = {'filter': f'net.ip == "{self.ip}"'} 51 | response = self._api_get(endpoint='/objects/metadata', params=params) 52 | 53 | return response.json() 54 | 55 | def query_objects(self, query: str, page_size: int = None) -> List[Dict]: 56 | """ 57 | Fetches a list of object metadata. Objects returned match the filter specified in the request. 58 | 59 | :param query: CEL string filter which objects must match. https://help.stairwell.com/en/knowledge/how-do-i-write-a-cel-query 60 | :param page_size: The maximum number of objects to return. The service may return fewer than this value. If unspecified, at most 50 objects will be returned. The maximum value is 1000; values above 1000 will be coerced to 1000. 61 | """ 62 | # TODO: async requests would likely speed this up significantly 63 | all_objects = [] 64 | next_page_token = None 65 | 66 | while True: 67 | params = { 68 | 'filter': query, 69 | 'pageSize': page_size, 70 | 'pageToken': next_page_token 71 | } 72 | response = self._api_get(endpoint='/objects/metadata', params=params) 73 | data = response.json() 74 | object_metadatas = data.get('objectMetadatas', []) 75 | all_objects.extend(object_metadatas) 76 | 77 | next_page_token = data.get('nextPageToken') 78 | if not next_page_token: 79 | break 80 | 81 | return all_objects 82 | -------------------------------------------------------------------------------- /pyoti/multis/urlhaus.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from typing import Dict, Optional 3 | 4 | from pyoti import __version__ 5 | from pyoti.classes import Domain, FileHash, IPAddress, URL 6 | from pyoti.exceptions import PyOTIError, URLhausError 7 | 8 | 9 | class URLhaus(Domain, FileHash, IPAddress, URL): 10 | """URLhaus Malware URL Exchange 11 | 12 | URLhaus is a project from abuse.ch with the goal of collecting, tracking, 13 | and sharing malicious URLs that are being used for malware distribution. 14 | """ 15 | def __init__(self, api_url: str = "https://urlhaus-api.abuse.ch/v1", url_id: Optional[str] = None): 16 | """ 17 | :param api_url: URLhaus API URL 18 | :param url_id: search by URLhaus urlid 19 | """ 20 | self._url_id = url_id 21 | Domain.__init__(self, api_url=api_url) 22 | FileHash.__init__(self, api_url=api_url) 23 | IPAddress.__init__(self, api_url=api_url) 24 | URL.__init__(self, api_url=api_url) 25 | 26 | @property 27 | def url_id(self): 28 | return self._url_id 29 | 30 | @url_id.setter 31 | def url_id(self, value): 32 | self._url_id = value 33 | 34 | def _api_post(self, url: str, data: Dict) -> requests.models.Response: 35 | """POST request to API""" 36 | headers = {"User-Agent": f"PyOTI {__version__}"} 37 | 38 | response = requests.request("POST", url=url, data=data, headers=headers) 39 | 40 | return response 41 | 42 | def _check_host(self, ioc) -> Dict: 43 | """POST request to /host/ endpoint 44 | 45 | :param ioc: domain, ip address, hostname, filehash, url 46 | :return: dict of request response 47 | """ 48 | data = {"host": ioc} 49 | url = f"{self.api_url}/host/" 50 | response = self._api_post(url, data) 51 | 52 | return response.json() 53 | 54 | def check_domain(self) -> Dict: 55 | """Checks Domain reputation 56 | 57 | :return: dict of request response 58 | """ 59 | return self._check_host(self.domain) 60 | 61 | def check_hash(self) -> Dict: 62 | """Checks File Hash reputation 63 | 64 | :return: dict of request response 65 | """ 66 | url = f"{self.api_url}/payload/" 67 | if len(self.file_hash) == 32: 68 | data = {"md5_hash": self.file_hash} 69 | response = self._api_post(url, data) 70 | elif len(self.file_hash) == 64: 71 | data = {"sha256_hash": self.file_hash} 72 | response = self._api_post(url, data) 73 | else: 74 | raise URLhausError( 75 | "/payload/ endpoint requires a valid MD5 or SHA-256 hash!" 76 | ) 77 | 78 | return response.json() 79 | 80 | def check_ip(self) -> Dict: 81 | """Checks IP reputation 82 | 83 | :return: dict of request response 84 | """ 85 | return self._check_host(self.ip) 86 | 87 | def check_url(self) -> Dict: 88 | """Checks URL reputation 89 | 90 | :return: dict of request response 91 | """ 92 | data = {"url": self.url} 93 | url = f"{self.api_url}/url/" 94 | response = self._api_post(url, data) 95 | 96 | return response.json() 97 | 98 | def check_urlid(self) -> Dict: 99 | """Checks ID of a URL tracked by URLhaus 100 | 101 | :return: dict of request response 102 | """ 103 | data = {"urlid": self.url_id} 104 | url = f"{self.api_url}/urlid/" 105 | response = self._api_post(url, data) 106 | 107 | return response.json() 108 | -------------------------------------------------------------------------------- /pyoti/multis/otx.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from typing import Dict, List 3 | 4 | from pyoti import __version__ 5 | from pyoti.classes import Domain, FileHash, IPAddress, URL 6 | 7 | 8 | OTX_IOC_SECTIONS = { 9 | "domain": ["general", "geo", "malware", "url_list", "passive_dns"], 10 | "file_hash": ["general", "analysis"], 11 | "ip": ["general", "reputation", "geo", "malware", "url_list", "passive_dns"], 12 | "url": ["general", "url_list"] 13 | } 14 | 15 | 16 | class OTX(Domain, FileHash, IPAddress, URL): 17 | """OTX Open Threat Exchange 18 | 19 | AlienVault OTX is a threat data platform that allows security researchers 20 | and threat data producers to share research and investigate new threats. 21 | """ 22 | def __init__(self, api_key: str, api_url: str = "https://otx.alienvault.com/api/v1"): 23 | """ 24 | :param api_key: OTX API key 25 | :param api_url: OTX base API URL 26 | """ 27 | Domain.__init__(self, api_key=api_key, api_url=api_url) 28 | FileHash.__init__(self, api_key=api_key, api_url=api_url) 29 | IPAddress.__init__(self, api_key=api_key, api_url=api_url) 30 | URL.__init__(self, api_key=api_key, api_url=api_url) 31 | 32 | def _api_get(self, endpoint: str, ioctype: str, iocvalue: str, otx_sections: List) -> Dict: 33 | """GET request to API 34 | 35 | :param endpoint: OTX API endpoint 36 | :param ioctype: type of IOC to check 37 | :param iocvalue: IOC value to check 38 | :param otx_sections: List of different OTX sections to perform lookup 39 | """ 40 | headers = { 41 | "Content-Type": "application/json", 42 | "User-Agent": f"PyOTI {__version__}", 43 | "X-OTX-API-KEY": self.api_key 44 | } 45 | 46 | indicator_dict = {} 47 | 48 | for section in otx_sections: 49 | 50 | response = requests.request( 51 | "GET", 52 | url=f"{self.api_url}/{endpoint}/{ioctype}/{iocvalue}/{section}", 53 | headers=headers 54 | ) 55 | 56 | indicator_dict[section] = response.json() 57 | 58 | return indicator_dict 59 | 60 | def check_domain(self) -> Dict: 61 | """Checks Domain reputation 62 | 63 | :return: dict of query result 64 | """ 65 | response = self._api_get( 66 | endpoint="indicators", 67 | ioctype="domain", 68 | iocvalue=self.domain, 69 | otx_sections=OTX_IOC_SECTIONS.get("domain") 70 | ) 71 | 72 | return response 73 | 74 | def check_hash(self) -> Dict: 75 | """Checks File Hash reputation 76 | 77 | :return: dict of query results 78 | """ 79 | response = self._api_get( 80 | endpoint="indicators", 81 | ioctype="file", 82 | iocvalue=self.file_hash, 83 | otx_sections=OTX_IOC_SECTIONS.get("file_hash") 84 | ) 85 | 86 | return response 87 | 88 | def check_ip(self) -> Dict: 89 | """Checks IP reputation 90 | 91 | :return: dict of query results 92 | """ 93 | response = self._api_get( 94 | endpoint="indicators", 95 | ioctype="IPv4", 96 | iocvalue=self.ip, 97 | otx_sections=OTX_IOC_SECTIONS.get("ip") 98 | ) 99 | 100 | return response 101 | 102 | def check_url(self) -> Dict: 103 | """Checks URL reputation 104 | 105 | :return: dict of query results 106 | """ 107 | response = self._api_get( 108 | endpoint="indicators", 109 | ioctype="url", 110 | iocvalue=self.url, 111 | otx_sections=OTX_IOC_SECTIONS.get("url") 112 | ) 113 | 114 | return response 115 | -------------------------------------------------------------------------------- /pyoti/classes.py: -------------------------------------------------------------------------------- 1 | class API: 2 | """Base API for PyOTI""" 3 | 4 | def __init__(self, api_key: str = None, api_url: str = None): 5 | """ 6 | :param api_key: API key for the endpoint to connect to 7 | :param api_url: URL of the API endpoint to connect to 8 | """ 9 | self._api_key = api_key 10 | self._api_url = api_url 11 | 12 | @property 13 | def api_key(self): 14 | return self._api_key 15 | 16 | @api_key.setter 17 | def api_key(self, value): 18 | self._api_key = value 19 | 20 | @property 21 | def api_url(self): 22 | return self._api_url 23 | 24 | @api_url.setter 25 | def api_url(self, value): 26 | self._api_url = value 27 | 28 | 29 | class Domain(API): 30 | """Domain API for PyOTI""" 31 | 32 | def __init__(self, api_key: str = None, api_url: str = None, domain: str = None): 33 | """ 34 | :param api_key: API key for the endpoint to connect to 35 | :param api_url: URL of the API endpoint to connect to 36 | :param domain: domain to check/scan 37 | """ 38 | self._domain = domain 39 | API.__init__(self, api_key, api_url) 40 | 41 | @property 42 | def domain(self): 43 | return self._domain 44 | 45 | @domain.setter 46 | def domain(self, value): 47 | self._domain = value 48 | 49 | 50 | class FileHash(API): 51 | """FileHash API for PyOTI""" 52 | 53 | def __init__(self, api_key: str = None, api_url: str = None, file_hash: str = None): 54 | """ 55 | :param api_key: API key for the endpoint to connect to 56 | :param api_url: URL of the API endpoint to connect to 57 | :param file_hash: file hash to check/scan 58 | """ 59 | self._file_hash = file_hash 60 | API.__init__(self, api_key, api_url) 61 | 62 | @property 63 | def file_hash(self): 64 | return self._file_hash 65 | 66 | @file_hash.setter 67 | def file_hash(self, value): 68 | self._file_hash = value 69 | 70 | 71 | class IPAddress(API): 72 | """IPAddress API for PyOTI""" 73 | 74 | def __init__(self, api_key: str = None, api_url: str = None, ip: str = None): 75 | """ 76 | :param api_key: API key for the endpoint to connect to 77 | :param api_url: URL of the API endpoint to connect to 78 | :param ip: IP address to check/scan 79 | """ 80 | self._ip = ip 81 | API.__init__(self, api_key, api_url) 82 | 83 | @property 84 | def ip(self): 85 | return self._ip 86 | 87 | @ip.setter 88 | def ip(self, value): 89 | self._ip = value 90 | 91 | 92 | class URL(API): 93 | """URL API for PyOTI""" 94 | 95 | def __init__(self, api_key: str = None, api_url: str = None, url: str = None): 96 | """ 97 | :param api_key: API key for the endpoint to connect to 98 | :param api_url: URL of the API endpoint to connect to 99 | :param url: URL to scan/check 100 | """ 101 | self._url = url 102 | API.__init__(self, api_key, api_url) 103 | 104 | @property 105 | def url(self): 106 | return self._url 107 | 108 | @url.setter 109 | def url(self, value): 110 | self._url = value 111 | 112 | 113 | class EmailAddress(API): 114 | """EmailAddress API for PyOTI""" 115 | 116 | def __init__(self, api_key: str = None, api_url: str = None, email: str = None): 117 | """ 118 | :param api_key: API key for the endpoint to connect to 119 | :param api_url: URL of the API endpoint to connect to 120 | :param email: URL to scan/check 121 | """ 122 | self._email = email 123 | API.__init__(self, api_key, api_url) 124 | 125 | @property 126 | def email(self): 127 | return self._email 128 | 129 | @email.setter 130 | def email(self, value): 131 | self._email = value 132 | -------------------------------------------------------------------------------- /pyoti/multis/ip2location.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from typing import Dict, List, Union 3 | 4 | from pyoti import __version__ 5 | from pyoti.classes import Domain, IPAddress 6 | 7 | 8 | class IP2WHOIS(Domain): 9 | """IP2WHOIS domain WHOIS 10 | 11 | IP2WHOIS Domain WHOIS API helps users to obtain domain information and WHOIS record by using a domain name. The 12 | WHOIS API returns a comprehensive WHOIS data such as creation date, updated date, expiration date, domain age, the 13 | contact information of the registrant, mailing address, phone number, email address, nameservers the domain is 14 | using and much more. 15 | """ 16 | def __init__(self, api_key: str, api_url: str = "https://api.ip2whois.com/v2"): 17 | """ 18 | :param api_key: IP2Location.io API key 19 | :param api_url: IP2WHOIS API URL 20 | """ 21 | Domain.__init__(self, api_key=api_key, api_url=api_url) 22 | 23 | def _api_get(self, domain: str) -> requests.models.Response: 24 | """GET request to API""" 25 | headers = {"User-Agent": f"PyOTI {__version__}"} 26 | 27 | params = { 28 | 'domain': domain, 29 | 'format': 'json', 30 | 'key': self.api_key 31 | } 32 | 33 | response = requests.request("GET", url=self.api_url, headers=headers, params=params) 34 | 35 | return response 36 | 37 | def check_domain(self) -> Union[Dict, str]: 38 | """Checks Domain WHOIS""" 39 | response = self._api_get(self.domain) 40 | 41 | if response.status_code == 200: 42 | return response.json() 43 | elif response.status_code == 400: 44 | return response.json()['error']['error_message'] 45 | 46 | 47 | class IP2Location(IPAddress): 48 | """IP2Location IP Geolocation 49 | 50 | IP2Location.io provides RESTful API allowing users to check IP address location in real time. The REST API supports 51 | both IPv4 and IPv6 address lookup. 52 | """ 53 | def __init__(self, api_key: str, api_url: str = "https://api.ip2location.io/"): 54 | """ 55 | :param api_key: IP2Location.io API key 56 | :param api_url: IP2Location API URL 57 | """ 58 | IPAddress.__init__(self, api_key=api_key, api_url=api_url) 59 | 60 | def _api_get(self, ip: str) -> requests.models.Response: 61 | """GET request to API""" 62 | headers = {"User-Agent": f"PyOTI {__version__}"} 63 | 64 | params = { 65 | 'ip': ip, 66 | 'format': 'json', 67 | 'key': self.api_key 68 | } 69 | 70 | response = requests.request("GET", url=self.api_url, headers=headers, params=params) 71 | 72 | return response 73 | 74 | def _api_post(self, ips: List) -> requests.models.Response: 75 | """POST request to API""" 76 | headers = {"User-Agent": f"PyOTI {__version__}"} 77 | 78 | params = { 79 | 'format': 'json', 80 | 'key': self.api_key 81 | } 82 | 83 | data = ips 84 | 85 | response = requests.request("POST", url=self.api_url, headers=headers, params=params, data=data) 86 | 87 | return response 88 | 89 | def check_ip(self) -> Union[Dict, str]: 90 | """Checks IP Geolocation""" 91 | response = self._api_get(self.ip) 92 | 93 | if response.status_code == 200: 94 | return response.json() 95 | elif response.status_code == 400: 96 | return response.json()['error']['error_message'] 97 | 98 | def bulk_check_ip(self, ips: List) -> Union[Dict, str]: 99 | """Bulk check (up to 1000) IP addresses geolocation 100 | 101 | [!] This API endpoint requires a Starter, Plus or Security Plan to work. [!] 102 | """ 103 | response = self._api_post(ips=ips) 104 | 105 | if response.status_code == 200: 106 | return response.json() 107 | else: 108 | return response.json()['error']['error_message'] 109 | -------------------------------------------------------------------------------- /pyoti/ips/spamhausintel.py: -------------------------------------------------------------------------------- 1 | import json 2 | import requests 3 | from typing import Dict, Optional 4 | 5 | from pyoti import __version__ 6 | from pyoti.classes import IPAddress 7 | from pyoti.exceptions import SpamhausIntelError 8 | from pyoti.utils import time_check_since_epoch 9 | 10 | 11 | class SpamhausIntel(IPAddress): 12 | """SpamhauzIntel IP Address Metadata 13 | 14 | SpamhausIntel is an API with metadata relating to compromised IP Addresses. 15 | """ 16 | def __init__( 17 | self, api_key: str, api_url: str = "https://api.spamhaus.org/api" 18 | ): 19 | """ 20 | :param api_key: SpamhausIntel API key 21 | :param api_url: SPamhausIntel base API URL 22 | """ 23 | self._token: str = None 24 | self._expires: int = None 25 | IPAddress.__init__(self, api_key, api_url) 26 | 27 | def _api_login(self): 28 | """Authenticate to Spamhaus API to get bearer token""" 29 | data = { 30 | "username": self.api_key.split(":")[0], 31 | "password": self.api_key.split(":")[1], 32 | "realm": "intel", 33 | } 34 | 35 | headers = {"User-Agent": f"PyOTI {__version__}"} 36 | 37 | response = requests.request("POST", url=f"{self.api_url}/v1/login", data=json.dumps(data), headers=headers) 38 | 39 | if response.status_code == 200: 40 | self._token = response.json()["token"] 41 | self._expires = response.json()["expires"] 42 | elif response.status_code == 401: 43 | raise SpamhausIntelError("Authentication Failed!") 44 | 45 | def _api_get(self, limit: Optional[int], since: Optional[int], until: Optional[int], type: str, ip: str, mask: str) -> requests.models.Response: 46 | """GET request to Spamhaus API 47 | 48 | :param limit: Constrain the number of rows returned by the query 49 | :param since: Results with a timestamp greater than or equal to 'since' (default 12 months if not passed) 50 | :param until: Results with a timestamp less than or equal to 'until' (default current timestamp if not passed) 51 | :param type: 'live' or 'history' return listings that are either active or inactive 52 | :param ip: IP address to look for 53 | :param mask: Optional netmask to use. (defaults to 32) 54 | """ 55 | if not self._token: 56 | self._api_login() 57 | if not time_check_since_epoch(self._expires): 58 | self._api_login() 59 | 60 | headers = { 61 | "Authorization": f"Bearer {self._token}", 62 | "User-Agent": f"PyOTI {__version__}" 63 | } 64 | 65 | params = {"limit": limit, "since": since, "until": until} 66 | 67 | response = requests.request( 68 | "GET", 69 | url=f"{self.api_url}/intel/v1/byobject/cidr/XBL/listed/{type}/{ip}/{mask}", 70 | headers=headers, 71 | params=params, 72 | ) 73 | 74 | return response 75 | 76 | def check_ip( 77 | self, 78 | limit: Optional[int] = None, 79 | since: Optional[int] = None, 80 | until: Optional[int] = None, 81 | type: str = "live", 82 | mask: str = "32" 83 | ) -> Dict: 84 | """Checks IP reputation 85 | 86 | :param limit: Constrain the number of rows returned by the query 87 | :param since: Results with a timestamp greater than or equal to 'since' (default 12 months if not passed) 88 | :param until: Results with a timestamp less than or equal to 'until' (default current timestamp if not passed) 89 | :param type: 'live' or 'history' return listings that are either active or inactive 90 | :param mask: Optional netmask to use. (defaults to 32) 91 | :return: dict of request response 92 | """ 93 | response = self._api_get( 94 | limit=limit, since=since, until=until, type=type, ip=self.ip, mask=mask 95 | ) 96 | 97 | if response.status_code == 200 or response.status_code == 404: 98 | return response.json() 99 | else: 100 | raise SpamhausIntelError(response.text) 101 | -------------------------------------------------------------------------------- /pyoti/multis/filescanio.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from typing import Dict, List, Optional 3 | 4 | from pyoti import __version__ 5 | from pyoti.classes import Domain, FileHash, IPAddress, URL 6 | 7 | 8 | class FileScanIO(Domain, FileHash, IPAddress, URL): 9 | """ 10 | FileScan.IO is a Next-Gen Sandbox and free malware analysis service. Operating at 10x speed compared to traditional 11 | sandboxes with 90% less resource usage, its unique adaptive threat analysis technology also enables zero-day 12 | malware detection and more Indicator of Compromise (IOCs) extraction. 13 | """ 14 | 15 | def __init__(self, api_key: str, api_url: str = "https://www.filescan.io"): 16 | """ 17 | :param api_key: FileScanIO API key 18 | :param api_url: FileScanIO base API url 19 | """ 20 | Domain.__init__(self, api_key=api_key, api_url=api_url) 21 | IPAddress.__init__(self, api_key=api_key, api_url=api_url) 22 | FileHash.__init__(self, api_key=api_key, api_url=api_url) 23 | URL.__init__(self, api_key=api_key, api_url=api_url) 24 | 25 | def _api_get(self, url: str, params: Optional[Dict]) -> requests.models.Response: 26 | """GET request to API""" 27 | headers = { 28 | "User-Agent": f"PyOTI {__version__}" 29 | } 30 | response = requests.request("GET", url=url, headers=headers, params=params) 31 | 32 | return response 33 | 34 | def _api_post(self, url: str, data: List): 35 | """POST request to API""" 36 | headers = { 37 | "User-Agent": f"PyOTI {__version__}", 38 | "Content-Type": "application/json" 39 | } 40 | response = requests.request("POST", url=url, headers=headers, json=data) 41 | 42 | return response 43 | 44 | def check_domain(self) -> Dict: 45 | """Get the reputation for one given domain""" 46 | params = {"ioc_value": self.domain} 47 | response = self._api_get(url=f"{self.api_url}/api/reputation/domain", params=params) 48 | 49 | return response.json() 50 | 51 | def check_hash(self) -> Dict: 52 | """Get the reputation for one given hash""" 53 | if len(self.file_hash) == 64: 54 | params = {'sha256': self.file_hash} 55 | response = self._api_get(url=f"{self.api_url}/api/reputation/hash", params=params) 56 | 57 | return response.json() 58 | else: 59 | return {"error": "You can only search for SHA256 hashes!"} 60 | 61 | def check_ip(self) -> Dict: 62 | """Get the reputation for one given IP address""" 63 | params = {"ioc_value": self.ip} 64 | response = self._api_get(url=f"{self.api_url}/api/reputation/ip", params=params) 65 | 66 | return response.json() 67 | 68 | def check_url(self) -> Dict: 69 | """Get the reputation for one given URL""" 70 | params = {"ioc_value": self.url} 71 | response = self._api_get(url=f"{self.api_url}/api/reputation/url", params=params) 72 | 73 | return response.json() 74 | 75 | def bulk_check_domains(self, domains: List) -> Dict: 76 | """Get the reputation for multiple domains""" 77 | response = self._api_post(url=f"{self.api_url}/api/reputation/domain", data=domains) 78 | 79 | return response.json() 80 | 81 | def bulk_check_hashes(self, hashes: List) -> Dict: 82 | """Get the reputation for multiple hashes""" 83 | for h in hashes: 84 | if len(h) != 64: 85 | return {"error": "You can only search for SHA256 hashes!"} 86 | 87 | response = self._api_post(url=f"{self.api_url}/api/reputation/hash", data=hashes) 88 | 89 | return response.json() 90 | 91 | def bulk_check_ips(self, ips: List) -> Dict: 92 | """Get the reputation for multiple IP addresses""" 93 | response = self._api_post(url=f"{self.api_url}/api/reputation/ip", data=ips) 94 | 95 | return response.json() 96 | 97 | def bulk_check_urls(self, urls: List) -> Dict: 98 | """Get the reputation for multiple URLs""" 99 | response = self._api_post(url=f"{self.api_url}/api/reputation/url", data=urls) 100 | 101 | return response.json() 102 | -------------------------------------------------------------------------------- /pyoti/urls/googlesafebrowsing.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from typing import Dict, List 3 | 4 | from pyoti import __version__ 5 | from pyoti.classes import URL 6 | 7 | 8 | class GoogleSafeBrowsing(URL): 9 | """GoogleSafeBrowsing URL Blacklist 10 | 11 | Google Safe Browsing is a blacklist service provided by Google that 12 | provides lists of URLs for web resources that contain malware or phishing 13 | content. 14 | """ 15 | def __init__( 16 | self, 17 | api_key: str, 18 | api_url: str = "https://safebrowsing.googleapis.com/v4/threatMatches:find", 19 | ): 20 | URL.__init__(self, api_key, api_url) 21 | 22 | def _api_post(self, endpoint: str, platforms: List[str], **kwargs) -> requests.models.Response: 23 | """POST request to API 24 | 25 | :param endpoint: API URL 26 | :param platforms: Default: ANY_PLATFORM. For all available options please see: 27 | https://developers.google.com/safe-browsing/v4/reference/rest/v4/PlatformType 28 | :return: dict of request response 29 | """ 30 | if kwargs.get('url_list'): 31 | threat_entries = [] 32 | for url in kwargs.get('url_list'): 33 | threat_entries.append({"url": url}) 34 | else: 35 | threat_entries = [{"url": self.url}] 36 | 37 | data = { 38 | "client": {"clientId": "PyOTI", "clientVersion": f"{__version__}"}, 39 | "threatInfo": { 40 | "threatTypes": [ 41 | "MALWARE", 42 | "SOCIAL_ENGINEERING", 43 | "THREAT_TYPE_UNSPECIFIED", 44 | "POTENTIALLY_HARMFUL_APPLICATION", 45 | "UNWANTED_SOFTWARE", 46 | ], 47 | "platformTypes": platforms, 48 | "threatEntryTypes": ["URL"], 49 | "threatEntries": threat_entries, 50 | }, 51 | } 52 | 53 | headers = { 54 | "Accept-Encoding": "gzip", 55 | "Content-type": "application/json", 56 | "User-Agent": f"PyOTI {__version__}" 57 | } 58 | 59 | response = requests.request( 60 | "POST", 61 | url=endpoint, 62 | headers=headers, 63 | json=data, 64 | params={"key": self.api_key} 65 | ) 66 | 67 | return response 68 | 69 | def check_url(self, platforms: List[str] = ["ANY_PLATFORM"]) -> Dict: 70 | """Checks URL reputation 71 | 72 | :param platforms: Default: ANY_PLATFORM. For all available options please see: 73 | https://developers.google.com/safe-browsing/v4/reference/rest/v4/PlatformType 74 | :return: dict of request response 75 | """ 76 | error_code = [400, 403, 429, 500, 503, 504] 77 | 78 | response = self._api_post(self.api_url, platforms) 79 | 80 | if response.status_code == 200: 81 | if response.json() == {}: 82 | r = {'matches': []} 83 | return r 84 | else: 85 | return response.json() 86 | 87 | elif response.status_code in error_code: 88 | r = {'error': response.json()["error"]["message"]} 89 | return r 90 | 91 | def bulk_check_url(self, url_list: List[str], platforms: List[str] = ["ANY_PLATFORM"]) -> Dict: 92 | """Bulk check URL reputation 93 | 94 | :param url_list: List or URLs to check reputation 95 | :param platforms: Default: ANY_PLATFORM. For all available options please see: 96 | https://developers.google.com/safe-browsing/v4/reference/rest/v4/PlatformType 97 | :return: dict of request response 98 | """ 99 | error_code = [400, 403, 429, 500, 503, 504] 100 | 101 | response = self._api_post(self.api_url, platforms, url_list=url_list) 102 | 103 | if response.status_code == 200: 104 | if response.json() == {}: 105 | r = {'matches': []} 106 | return r 107 | else: 108 | return response.json() 109 | 110 | elif response.status_code in error_code: 111 | r = {'error': response.json()["error"]["message"]} 112 | return r -------------------------------------------------------------------------------- /pyoti/multis/joesandbox.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from typing import Dict, Optional 3 | 4 | from pyoti import __version__ 5 | from pyoti.classes import Domain, FileHash, IPAddress, URL 6 | 7 | 8 | class JoeSandbox(Domain, FileHash, IPAddress, URL): 9 | """ 10 | Joe Sandbox is a platform that allows you to analyze malware and phishing in depth on various platforms and 11 | environments. It uses advanced technologies such as hybrid analysis, emulation, machine learning and AI to detect 12 | and report threats. 13 | """ 14 | 15 | def __init__(self, api_key: str, api_url: str = "https://www.joesandbox.com/api"): 16 | """ 17 | :param api_key: 18 | :param api_url: 19 | """ 20 | Domain.__init__(self, api_key=api_key, api_url=api_url) 21 | FileHash.__init__(self, api_key=api_key, api_url=api_url) 22 | IPAddress.__init__(self, api_key=api_key, api_url=api_url) 23 | URL.__init__(self, api_key=api_key, api_url=api_url) 24 | 25 | def _api_post(self, url: str, params: Optional[Dict] = None) -> requests.models.Response: 26 | """POST request to API""" 27 | headers = { 28 | "User-Agent": f"PyOTI {__version__}" 29 | } 30 | if params: 31 | params['apikey'] = self.api_key 32 | else: 33 | params = {'apikey': self.api_key} 34 | 35 | response = requests.request("POST", url=url, headers=headers, data=params) 36 | 37 | return response 38 | 39 | def server_online(self) -> bool: 40 | """Check if the Joe Sandbox analysis back end is online or in maintenance mode.""" 41 | response = self._api_post(url=f"{self.api_url}/v2/server/online") 42 | 43 | return response.json()['data']['online'] 44 | 45 | def server_info(self) -> Dict: 46 | """Query information about the server.""" 47 | response = self._api_post(url=f"{self.api_url}/v2/server/info") 48 | 49 | return response.json() 50 | 51 | def account_info(self) -> Dict: 52 | """Query information about your account.""" 53 | response = self._api_post(url=f"{self.api_url}/v2/account/info") 54 | 55 | return response.json() 56 | 57 | def check_domain(self, ioc: bool = False) -> Dict: 58 | """ 59 | Check a domain against all sandbox analyses. 60 | :param ioc: Search domain against all URLs submitted for analysis (False) or search domains contacted during anlysis (True) 61 | """ 62 | if ioc: 63 | params = {'ioc-domain': self.domain} 64 | else: 65 | params = {'url': self.domain} 66 | 67 | response = self._api_post(url=f"{self.api_url}/v2/analysis/search", params=params) 68 | 69 | return response.json() 70 | 71 | def check_hash(self) -> Dict: 72 | """Check a file hash against all sandbox analyses""" 73 | params = {} 74 | if len(self.file_hash) == 32: 75 | params['md5'] = self.file_hash 76 | elif len(self.file_hash) == 40: 77 | params['sha1'] = self.file_hash 78 | elif len(self.file_hash) == 64: 79 | params['sha256'] = self.file_hash 80 | else: 81 | return {'error': 'You must provide either an MD5, SHA1, or SHA256 hash to query!'} 82 | 83 | response = self._api_post(url=f"{self.api_url}/v2/analysis/search", params=params) 84 | 85 | return response.json() 86 | 87 | def check_ip(self) -> Dict: 88 | """Check an IP address against all sandbox analyses""" 89 | params = {'ioc-public-ip': self.ip} 90 | 91 | response = self._api_post(url=f"{self.api_url}/v2/analysis/search", params=params) 92 | 93 | return response.json() 94 | 95 | def check_url(self, ioc: bool = False) -> Dict: 96 | """ 97 | Check a URL against all sandbox analyses 98 | :param ioc: Search URLs submitted for analysis (False) or search URLs contacted during analysis (True) 99 | """ 100 | if ioc: 101 | params = {'ioc-url': self.url} 102 | else: 103 | params = {'url': self.url} 104 | 105 | response = self._api_post(url=f"{self.api_url}/v2/analysis/search", params=params) 106 | 107 | return response.json() 108 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PyOTI - Python Open Threat Intelligence 2 | *** 3 | 4 | PyOTI is an API framework to easily query threat intel APIs to get fast, accurate and consistent enrichment data to provide added context to your indicators of compromise. Built as a modular framework to make it easy to use any of the available APIs without needing to be an experienced coder. If a service or tool you use isn’t already in PyOTI it is simple to add a new enrichment module or you may open an issue for a feature request and we can work to get it added into the project. 5 | 6 | 7 | 8 | | Indicator Types | APIs | 9 | |-----------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| 10 | | Domains | CheckDMARC, CIRCLPDNS, IrisInvestigate | 11 | | Email Addresses | DisposableEmails, EmailRepIO | 12 | | Hashes | CIRCLHashLookup, MalwareBazaar, MalwareHashRegistry | 13 | | IP Addresses | AbuseIPDB, GreyNoise, SpamhausIntel | 14 | | URLs | GoogleSafeBrowsing, LinkPreview, Phishtank, ProofpointURLDecoder | 15 | | Multis | BinaryEdge, CIRCLPSSL, CiscoUmbrellaInvestigate, DNSBlockList, FileScanIO, HybridAnalysis, IP2Location/IP2WHOIS, JoeSandbox, MaltiverseIOC, MetaDefenderCloudV4, MISP, Onyphe, OTX, Pulsedive, Stairwell, ThreatFox, Triage, URLhaus, URLscan, VirusTotalV3, WhoisXML, XForceExchange | 16 | 17 | *** 18 | ## Installing via pip 19 | It is advised to use a virtual environment. 20 | ```python 21 | python3 -m pip install pyoti 22 | ``` 23 | 24 | If you want to also use the Jupyter Notebook please install additional dependencies. 25 | ```python 26 | python3 -m pip install pyoti[jupyter_notebook] 27 | ``` 28 | *** 29 | ## Installing/Updating from source 30 | Windows instructions can be found in the docs directory [here](https://github.com/RH-ISAC/PyOTI/blob/main/docs/windows/README.md). 31 | 32 | Linux instructions can be found in the docs directory [here](https://github.com/RH-ISAC/PyOTI/blob/main/docs/linux/README.md). 33 | *** 34 | ## Tutorial 35 | For a quick tutorial on the ease and benefit of using PyOTI you can view the Phishing URL Triage Jupyter Notebook [here](https://github.com/RH-ISAC/PyOTI/blob/main/docs/tutorials/phishing_triage_urls.ipynb). 36 | *** 37 | ## License 38 | Copyright © 2021-2025, RH-ISAC 39 | 40 | This work is free software. You may redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or 41 | at your option, any later version. 42 | 43 | This work is distributed in the hope that it will be useful, but is made available WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. 44 | 45 | Please review the GNU General Public License at https://www.gnu.org/licenses/ for additional information. 46 | -------------------------------------------------------------------------------- /pyoti/urls/proofpointurldecoder.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import re 3 | import string 4 | from html import unescape 5 | from urllib.parse import unquote 6 | 7 | from pyoti.classes import URL 8 | 9 | 10 | class ProofpointURLDecoder(URL): 11 | """Decode URLs rewritten by Proofpoint URL Defense. Supports v1, v2, and v3 URLs. 12 | 13 | Adopted from the script originally authored by Eric Van Cleve: 14 | https://help.proofpoint.com/@api/deki/files/177/URLDefenseDecode.py?revision=3 15 | """ 16 | def __init__(self): 17 | self.ud_pattern = re.compile(r'https://urldefense(?:\.proofpoint)?\.com/(v[0-9])/') 18 | self.v1_pattern = re.compile(r'u=(?P.+?)&k|amp;k=') 19 | self.v2_pattern = re.compile(r'u=(?P.+?)&[dc]=') 20 | self.v3_pattern = re.compile(r'v3/__(?P.+?)__;(?P.*?)!') 21 | self.v3_token_pattern = re.compile(r"\*(\*.)?") 22 | self.v3_run_mapping = {} 23 | run_values = string.ascii_uppercase + string.ascii_lowercase + string.digits + '-' + '_' 24 | run_length = 2 25 | for value in run_values: 26 | self.v3_run_mapping[value] = run_length 27 | run_length += 1 28 | self.maketrans = str.maketrans 29 | URL.__init__(self) 30 | 31 | def _decode(self, rewritten_url): 32 | match = self.ud_pattern.search(rewritten_url) 33 | if match: 34 | if match.group(1) == 'v1': 35 | return self._decode_v1(rewritten_url) 36 | elif match.group(1) == 'v2': 37 | return self._decode_v2(rewritten_url) 38 | elif match.group(1) == 'v3': 39 | return self._decode_v3(rewritten_url) 40 | else: 41 | raise ValueError('Unrecognized version in: ', rewritten_url) 42 | else: 43 | raise ValueError('Does not appear to be a URL Defense URL') 44 | 45 | def _decode_v1(self, rewritten_url): 46 | match = self.v1_pattern.search(rewritten_url) 47 | if match: 48 | url_encoded_url = match.group('url') 49 | html_encoded_url = unquote(url_encoded_url) 50 | url = unescape(html_encoded_url) 51 | return url 52 | else: 53 | raise ValueError('Error parsing URL') 54 | 55 | def _decode_v2(self, rewritten_url): 56 | match = self.v2_pattern.search(rewritten_url) 57 | if match: 58 | special_encoded_url = match.group('url') 59 | trans = self.maketrans('-_', '%/') 60 | url_encoded_url = special_encoded_url.translate(trans) 61 | html_encoded_url = unquote(url_encoded_url) 62 | url = unescape(html_encoded_url) 63 | return url 64 | else: 65 | raise ValueError('Error parsing URL') 66 | 67 | def _decode_v3(self, rewritten_url): 68 | def replace_token(token): 69 | if token == '*': 70 | character = self.dec_bytes[self.current_marker] 71 | self.current_marker += 1 72 | return character 73 | if token.startswith('**'): 74 | run_length = self.v3_run_mapping[token[-1]] 75 | run = self.dec_bytes[self.current_marker:self.current_marker + run_length] 76 | self.current_marker += run_length 77 | return run 78 | 79 | def substitute_tokens(text, start_pos=0): 80 | match = self.v3_token_pattern.search(text, start_pos) 81 | if match: 82 | start = text[start_pos:match.start()] 83 | built_string = start 84 | token = text[match.start():match.end()] 85 | built_string += replace_token(token) 86 | built_string += substitute_tokens(text, match.end()) 87 | return built_string 88 | else: 89 | return text[start_pos:len(text)] 90 | 91 | match = self.v3_pattern.search(rewritten_url) 92 | if match: 93 | url = match.group('url') 94 | encoded_url = unquote(url) 95 | enc_bytes = match.group('enc_bytes') 96 | enc_bytes += '==' 97 | self.dec_bytes = (base64.urlsafe_b64decode(enc_bytes)).decode('utf-8') 98 | self.current_marker = 0 99 | return substitute_tokens(encoded_url) 100 | else: 101 | raise ValueError('Error parsing URL') 102 | 103 | def check_url(self): 104 | return self._decode(self.url) 105 | -------------------------------------------------------------------------------- /pyoti/multis/threatfox.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from typing import Dict, List, Union 3 | 4 | from pyoti import __version__ 5 | from pyoti.classes import Domain, FileHash, IPAddress, URL 6 | from pyoti.exceptions import PyOTIError 7 | 8 | 9 | class ThreatFox(Domain, FileHash, IPAddress, URL): 10 | """ThreatFox by abuse.ch 11 | 12 | ThreatFox is a free platform from abuse.ch with the goal of sharing indicators of compromise (IOCs) associated with 13 | malware with the infosec community, AV vendors and threat intelligence providers 14 | """ 15 | def __init__(self, api_key: str, api_url: str = "https://threatfox-api.abuse.ch/api/v1"): 16 | Domain.__init__(self, api_key=api_key, api_url=api_url) 17 | FileHash.__init__(self, api_key=api_key, api_url=api_url) 18 | IPAddress.__init__(self, api_key=api_key, api_url=api_url) 19 | URL.__init__(self, api_key=api_key, api_url=api_url) 20 | 21 | def _api_post(self, data) -> requests.models.Response: 22 | """POST request to ThreatFox API""" 23 | headers = { 24 | "API-KEY": self.api_key, 25 | "Content-Type": "application/json", 26 | "User-Agent": f"PyOTI {__version__}" 27 | } 28 | 29 | response = requests.request("POST", url=self.api_url, headers=headers, json=data) 30 | 31 | return response 32 | 33 | def _sort_results(self, result: List[Dict]) -> List[Dict]: 34 | """Sorts ThreatFox results 35 | 36 | :param result: list of results from API request 37 | :return: list of results from API request sorted in descending order of first_seen 38 | """ 39 | sorted_result = sorted(result, key=lambda k: k['first_seen'], reverse=True) 40 | 41 | return sorted_result 42 | 43 | def check_domain(self) -> Union[Dict, List[Dict]]: 44 | """Checks Domain reputation""" 45 | data = {"query": "search_ioc", "search_term": self.domain} 46 | 47 | response = self._api_post(data=data) 48 | 49 | if response.json().get("query_status") == "no_result": 50 | return response.json() 51 | else: 52 | to_return = self._sort_results(result=response.json().get("data")) 53 | 54 | return to_return 55 | 56 | def check_hash(self) -> Union[Dict, List[Dict]]: 57 | """Checks File Hash reputation""" 58 | if len(self.file_hash) == 32 or len(self.file_hash) == 64: 59 | data = {"query": "search_ioc", "search_term": self.file_hash} 60 | else: 61 | raise PyOTIError("You must supply MD5 or SHA256 hash to query ThreatFox.") 62 | 63 | response = self._api_post(data=data) 64 | 65 | if response.json().get("query_status") == "no_result": 66 | return response.json() 67 | else: 68 | to_return = self._sort_results(result=response.json().get("data")) 69 | 70 | return to_return 71 | 72 | def check_hash_associated_iocs(self) -> Union[Dict, List[Dict]]: 73 | """Checks for IOCs associated with a certain File Hash""" 74 | if len(self.file_hash) == 32 or len(self.file_hash) == 64: 75 | data = {"query": "search_hash", "hash": self.file_hash} 76 | else: 77 | raise PyOTIError("You must supply MD5 or SHA256 hash to query ThreatFox.") 78 | 79 | response = self._api_post(data=data) 80 | 81 | if response.json().get("query_status") == "no_result": 82 | return response.json() 83 | else: 84 | to_return = self._sort_results(result=response.json().get("data")) 85 | 86 | return to_return 87 | 88 | def check_ip(self) -> Union[Dict, List[Dict]]: 89 | """Checks IP Address reputation""" 90 | data = {"query": "search_ioc", "search_term": self.ip} 91 | 92 | response = self._api_post(data=data) 93 | 94 | if response.json().get("query_status") == "no_result": 95 | return response.json() 96 | else: 97 | to_return = self._sort_results(result=response.json().get("data")) 98 | 99 | return to_return 100 | 101 | def check_url(self) -> Union[Dict, List[Dict]]: 102 | """Checks URL reputation""" 103 | data = {"query": "search_ioc", "search_term": self.url} 104 | 105 | response = self._api_post(data=data) 106 | 107 | if response.json().get("query_status") == "no_result": 108 | return response.json() 109 | else: 110 | to_return = self._sort_results(result=response.json().get("data")) 111 | 112 | return to_return 113 | -------------------------------------------------------------------------------- /pyoti/multis/virustotal.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import os 3 | import requests 4 | import time 5 | from typing import Dict 6 | 7 | from pyoti import __version__ 8 | from pyoti.classes import Domain, FileHash, IPAddress, URL 9 | 10 | 11 | class VirusTotalV3(Domain, FileHash, IPAddress, URL): 12 | """VirusTotal IOC Analyzer 13 | 14 | VirusTotal analyzes files and URLs enabling detection of malicious content 15 | using antivirus engines and website scanners. (VT API v3) 16 | """ 17 | def __init__( 18 | self, api_key, api_url="https://www.virustotal.com/api/v3" 19 | ): 20 | """ 21 | :param api_key: VirusTotal API key 22 | :param api_url: VirusTotal v3 base API URL 23 | """ 24 | Domain.__init__(self, api_key=api_key, api_url=api_url) 25 | FileHash.__init__(self, api_key=api_key, api_url=api_url) 26 | IPAddress.__init__(self, api_key=api_key, api_url=api_url) 27 | URL.__init__(self, api_key=api_key, api_url=api_url) 28 | 29 | def _api_get(self, url: str) -> requests.models.Response: 30 | """GET request to API 31 | 32 | :param url: VirusTotal API endpoint URL 33 | """ 34 | headers = { 35 | "x-apikey": self.api_key, 36 | "User-Agent": f"PyOTI {__version__}" 37 | } 38 | 39 | response = requests.request("GET", url=url, headers=headers) 40 | 41 | return response 42 | 43 | def _api_post(self, url: str, file_path: str, zip_pw: str = None) -> requests.models.Response: 44 | """POST request to API 45 | :param url: VirusTotal API endpoint URL 46 | :param file_path: Path for file to submit 47 | :param zip_pw: Password if ZIP file submission 48 | """ 49 | headers = { 50 | "x-apikey": self.api_key, 51 | "User-Agent": f"PyOTI {__version__}", 52 | "Accept": "application/json", 53 | } 54 | 55 | files = { 56 | "file": ( 57 | os.path.basename(file_path), 58 | open(os.path.abspath(file_path), "rb") 59 | ) 60 | } 61 | 62 | if zip_pw is not None: 63 | payload = {"password": zip_pw} 64 | response = requests.request("POST", url=url, files=files, headers=headers, data=payload) 65 | else: 66 | response = requests.request("POST", url=url, files=files, headers=headers) 67 | 68 | return response 69 | 70 | def check_domain(self) -> Dict: 71 | """Retrieve information about an Internet domain 72 | 73 | :return: dict of request response 74 | """ 75 | url = f"{self.api_url}/domains/{self.domain}" 76 | response = self._api_get(url=url) 77 | 78 | return response.json() 79 | 80 | def check_hash(self) -> Dict: 81 | """Retrieve information about a file 82 | 83 | :return: dict of request response 84 | """ 85 | url = f"{self.api_url}/files/{self.file_hash}" 86 | response = self._api_get(url=url) 87 | 88 | return response.json() 89 | 90 | def check_ip(self) -> Dict: 91 | """Retrieve information about an IP address 92 | 93 | :return: dict of request response 94 | """ 95 | url = f"{self.api_url}/ip_addresses/{self.ip}" 96 | response = self._api_get(url=url) 97 | 98 | return response.json() 99 | 100 | def check_url(self) -> Dict: 101 | """Retrieve information about a URL 102 | 103 | :return: dict of request response 104 | """ 105 | url_id = base64.urlsafe_b64encode(self.url.encode()).decode().strip("=") 106 | url = f"{self.api_url}/urls/{url_id}" 107 | response = self._api_get(url=url) 108 | 109 | return response.json() 110 | 111 | def upload_file(self, file_path: str, zip_pw: str = None) -> Dict: 112 | """Upload and analyse a file 113 | 114 | :param file_path: Path for file to submit 115 | :param zip_pw: Password if ZIP file submission 116 | """ 117 | # TODO: file size checks 118 | # - this endpoint allows <= 32mb 119 | # - /files/upload_url allows 32mb >= FILE <= 650mb 120 | # add a timeout to the while loop so we don't sit here infinitely 121 | url = f"{self.api_url}/files" 122 | response = self._api_post(url=url, file_path=file_path, zip_pw=zip_pw) 123 | analysis_id = response.json()["data"]["id"] 124 | analysis_url = f"{self.api_url}/analyses/{analysis_id}" 125 | print("[+] File queued for analysis!") 126 | while self._api_get(url=analysis_url).json()["data"]["attributes"]["status"] == "queued": 127 | time.sleep(5) 128 | analysis_resp = self._api_get(url=f"{self.api_url}/analyses/{analysis_id}") 129 | link = f"https://virustotal.com/gui/file/{analysis_resp.json()['meta']['file_info']['sha256']}" 130 | print(f"[!] File analysis completed! VT Sample Link: {link}") 131 | 132 | return analysis_resp.json() 133 | -------------------------------------------------------------------------------- /pyoti/multis/hybridanalysis.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from typing import Dict, List, Optional, Union 3 | 4 | from pyoti import __version__ 5 | from pyoti.classes import Domain, FileHash, IPAddress, URL 6 | 7 | 8 | class HybridAnalysis(Domain, FileHash, IPAddress, URL): 9 | """HybridAnalysis Malware Analysis 10 | 11 | HybridAnalysis is a free malware analysis service for the community that detects and analyzes unknown threats using 12 | a unique Hybrid Analysis technology. 13 | """ 14 | def __init__( 15 | self, 16 | api_key: str, 17 | api_url: str = "https://www.hybrid-analysis.com/api/v2", 18 | job_id: Optional[str] = None, 19 | ): 20 | """ 21 | :param api_key: HybridAnalysis API key 22 | :param api_url: HybridAnalysis base API URL 23 | :param job_id: HybridAnalysis ID for report 24 | """ 25 | self._job_id = job_id 26 | Domain.__init__(self, api_url=api_url, api_key=api_key) 27 | FileHash.__init__(self, api_url=api_url, api_key=api_key) 28 | IPAddress.__init__(self, api_url=api_url, api_key=api_key) 29 | URL.__init__(self, api_url=api_url, api_key=api_key) 30 | 31 | @property 32 | def job_id(self): 33 | return self._job_id 34 | 35 | @job_id.setter 36 | def job_id(self, value): 37 | self._job_id = value 38 | 39 | def _api_post(self, url: str, data: Dict) -> requests.models.Response: 40 | """POST request to API 41 | 42 | :return: dict of request response 43 | """ 44 | headers = { 45 | "Accept": "application/json", 46 | "Accept-Encoding": "gzip", 47 | "Content-Type": "application/x-www-form-urlencoded", 48 | "User-Agent": f"PyOTI {__version__}", 49 | "api-key": self.api_key, 50 | } 51 | 52 | response = requests.request("POST", url=url, headers=headers, data=data) 53 | 54 | return response 55 | 56 | def _api_get(self, url: str) -> requests.models.Response: 57 | """GET request to API 58 | 59 | :return: list of dicts in request response 60 | """ 61 | headers = { 62 | "Accept": "application/json", 63 | "Accept-Encoding": "gzip", 64 | "User-Agent": f"PyOTI {__version__}", 65 | "api-key": self.api_key, 66 | } 67 | 68 | response = requests.request("GET", url=url, headers=headers) 69 | 70 | return response 71 | 72 | def _sort_results(self, result: List[Dict]) -> List[Dict]: 73 | """Sorts HybridAnalysis Results 74 | 75 | :param result: list of results from API request 76 | :return: list of results from API request sorted by last analysis time 77 | """ 78 | if result: 79 | sorted_result = sorted(result, key=lambda k: k['analysis_start_time'], reverse=True) 80 | self.job_id = sorted_result[0].get("job_id") if sorted_result else None 81 | 82 | return sorted_result 83 | 84 | def check_domain(self) -> Union[List[Dict], None]: 85 | """Checks Domain reputation 86 | 87 | :return: list of dicts in request result 88 | """ 89 | data = {"domain": self.domain} 90 | url = f"{self.api_url}/search/terms" 91 | response = self._api_post(url=url, data=data) 92 | sorted_result = self._sort_results(result=response.json().get("result")) 93 | 94 | return sorted_result 95 | 96 | def check_ip(self) -> Union[List[Dict], None]: 97 | """Checks IP Address reputation 98 | 99 | :return: list of dicts in request result 100 | """ 101 | data = {"host": self.ip} 102 | url = f"{self.api_url}/search/terms" 103 | response = self._api_post(url=url, data=data) 104 | sorted_result = self._sort_results(result=response.json().get("result")) 105 | 106 | return sorted_result 107 | 108 | def check_hash(self) -> Union[List[Dict], None]: 109 | """Checks File Hash reputation 110 | 111 | :return: list of dicts in request result 112 | """ 113 | data = {"hash": self.file_hash} 114 | url = f"{self.api_url}/search/hash" 115 | response = self._api_post(url=url, data=data) 116 | sorted_result = self._sort_results(result=response.json()) 117 | 118 | return sorted_result 119 | 120 | def check_url(self) -> Union[List[Dict], None]: 121 | """Checks URL reputation 122 | 123 | :return: list of dicts in request result 124 | """ 125 | data = {"url": self.url} 126 | url = f"{self.api_url}/search/terms" 127 | response = self._api_post(url=url, data=data) 128 | sorted_result = self._sort_results(result=response.json().get("result")) 129 | 130 | return sorted_result 131 | 132 | def check_report(self, sandbox_report: Optional[str] = "summary") -> Dict: 133 | """Checks for summary of a submission 134 | 135 | :param sandbox_report: default summary (see https://www.hybrid-analysis.com/docs/api/v2/) 136 | :return: dict of request response 137 | """ 138 | url = f"{self.api_url}/report/{self.job_id}/{sandbox_report}" 139 | response = self._api_get(url=url) 140 | 141 | return response.json() 142 | -------------------------------------------------------------------------------- /pyoti/multis/metadefendercloud.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from typing import Dict, List 3 | from urllib.parse import quote 4 | 5 | from pyoti import __version__ 6 | from pyoti.classes import Domain, IPAddress, FileHash, URL 7 | 8 | 9 | class MetaDefenderCloudV4(Domain, IPAddress, FileHash, URL): 10 | """ 11 | MetaDefender Cloud is a cloud-based platform that offers multiple technologies to protect against file-based 12 | attacks, such as Deep Content Disarm and Reconstruction, Multiscanning, Sandbox and Website Scanning. It has a high 13 | malware detection rate, a large file reputation database and a CVE scanner. 14 | """ 15 | 16 | def __init__(self, api_key: str, api_url: str = "https://api.metadefender.com/v4"): 17 | """ 18 | :param api_key: MetaDefender Cloud API key 19 | :param api_url: MetaDefender Cloud base API url 20 | """ 21 | Domain.__init__(self, api_key=api_key, api_url=api_url) 22 | IPAddress.__init__(self, api_key=api_key, api_url=api_url) 23 | FileHash.__init__(self, api_key=api_key, api_url=api_url) 24 | URL.__init__(self, api_key=api_key, api_url=api_url) 25 | 26 | def _api_get(self, url: str) -> requests.models.Response: 27 | """GET request to API""" 28 | headers = { 29 | "User-Agent": f"PyOTI {__version__}", 30 | "apikey": self.api_key 31 | } 32 | response = requests.request("GET", url=url, headers=headers) 33 | 34 | return response 35 | 36 | def _api_post(self, url: str, data: Dict, scan_details: bool) -> requests.models.Response: 37 | """POST request to API""" 38 | headers = { 39 | "User-Agent": f"PyOTI {__version__}", 40 | "apikey": self.api_key, 41 | "Content-Type": "application/json" 42 | } 43 | if scan_details: 44 | headers["includescandetails"] = "1" 45 | response = requests.request("POST", url=url, headers=headers, json=data) 46 | 47 | return response 48 | 49 | def get_api_info(self) -> Dict: 50 | """Retrieve information about your apikey""" 51 | response = self._api_get(url=f"{self.api_url}/apikey") 52 | 53 | return response.json() 54 | 55 | def check_domain(self) -> Dict: 56 | """ 57 | Retrieve information about a given fully qualified domain name (FQDN) from a CIF server including but not 58 | limited to: provider of the FQDN, a security assessment about the FQDN, and time of detection. 59 | """ 60 | response = self._api_get(url=f"{self.api_url}/domain/{self.domain}") 61 | 62 | return response.json() 63 | 64 | def check_hash(self) -> Dict: 65 | """Retrieve scan reports by looking up a hash using MD5, SHA1 or SHA256""" 66 | if len(self.file_hash) == 32 or len(self.file_hash) == 40 or len(self.file_hash) == 64: 67 | response = self._api_get(url=f"{self.api_url}/hash/{self.file_hash}") 68 | 69 | return response.json() 70 | else: 71 | return {"error": "You must provide an MD5, SHA1, or SHA256 file hash"} 72 | 73 | def check_ip(self) -> Dict: 74 | """Retrieve information about given IP (IPv4 + IPv6) from a CIF server""" 75 | response = self._api_get(url=f"{self.api_url}/ip/{self.ip}") 76 | 77 | return response.json() 78 | 79 | def check_url(self) -> Dict: 80 | """Retrieve information about given observable (URL) from a CIF server.""" 81 | response = self._api_get(url=f"{self.api_url}/url/{quote(self.api_url, safe='')}") 82 | 83 | return response.json() 84 | 85 | def bulk_check_domains(self, domains: List[str]) -> Dict: 86 | """ 87 | Bulk retrieve information about a list of fully qualified domain names (FQDNs) from a CIF server including but 88 | not limited to: provider of the FQDNs, a security assessment about the FQDNs, and time of detection. 89 | """ 90 | data = {"fqdn": domains} 91 | response = self._api_post(url=f"{self.api_url}/domain", data=data, scan_details=False) 92 | 93 | return response.json() 94 | 95 | def bulk_check_hashes(self, hashes: List[str], include_scan_details: bool = False) -> Dict: 96 | """Look up the scan results based on MD5, SHA1, or SHA256 for multiple data hashes""" 97 | if len(hashes) <= 1000: 98 | data = {"hash": hashes} 99 | response = self._api_post(url=f"{self.api_url}/hash", data=data, scan_details=include_scan_details) 100 | 101 | return response.json() 102 | else: 103 | return {"error": "You can only bulk search up to 1000 hashes at a single time!"} 104 | 105 | def bulk_check_ips(self, ips: List[str]) -> Dict: 106 | """Retrieve information about a list of IP's (Pv4/IPv6)""" 107 | data = {"address": ips} 108 | response = self._api_post(url=f"{self.api_url}/ip", data=data, scan_details=False) 109 | 110 | return response.json() 111 | 112 | def bulk_check_urls(self, urls: List[str]) -> Dict: 113 | """Retrieve information about a list of given observables (URLs) from a CIF server.""" 114 | data = {"url": urls} 115 | response = self._api_post(url=f"{self.api_url}/url", data=data, scan_details=False) 116 | 117 | return response.json() 118 | -------------------------------------------------------------------------------- /pyoti/multis/misp.py: -------------------------------------------------------------------------------- 1 | import json 2 | import requests 3 | from typing import Dict 4 | 5 | from pyoti import __version__ 6 | from pyoti.classes import Domain, EmailAddress, FileHash, IPAddress, URL 7 | 8 | 9 | class MISP(Domain, EmailAddress, FileHash, IPAddress, URL): 10 | """MISP Threat Intel Platform 11 | 12 | The MISP threat sharing platform is a free and open source software helping 13 | information sharing of threat intelligence including cyber security 14 | indicators. 15 | """ 16 | def __init__(self, api_key: str, api_url: str): 17 | """ 18 | :param api_key: MISP API key 19 | :param api_url: MISP base API URL 20 | """ 21 | Domain.__init__(self, api_key=api_key, api_url=api_url) 22 | EmailAddress.__init__(self, api_key=api_key, api_url=api_url) 23 | FileHash.__init__(self, api_key=api_key, api_url=api_url) 24 | IPAddress.__init__(self, api_key=api_key, api_url=api_url) 25 | URL.__init__(self, api_key=api_key, api_url=api_url) 26 | 27 | def _api_post(self, controller: str, query: Dict, verify_ssl: bool) -> requests.models.Response: 28 | """POST request to API""" 29 | headers = { 30 | "Accept": "application/json", 31 | "Authorization": self.api_key, 32 | "Content-Type": "application/json", 33 | "User-Agent": f"PyOTI {__version__}" 34 | } 35 | 36 | response = requests.request( 37 | "POST", 38 | url=f"{self.api_url}/{controller}/restSearch", 39 | data=json.dumps(query), 40 | headers=headers, 41 | verify=verify_ssl 42 | ) 43 | 44 | return response 45 | 46 | def _search_query(self, iocvalue: str, limit: int, warninglist: bool) -> Dict: 47 | """Sets parameters for search query 48 | 49 | :param iocvalue: IOC value 50 | :param limit: number of results 51 | :param warninglist: enforce MISP warninglist 52 | :return: dict 53 | """ 54 | query = {"value": iocvalue, "limit": limit, "enforceWarninglist": warninglist} 55 | 56 | return query 57 | 58 | def check_domain(self, controller: str = "events", limit: int = 50, verify_ssl: bool = True, warninglist: bool = True) -> Dict: 59 | """Checks Domain reputation 60 | 61 | :param controller: the MISP controller to query 62 | :param limit: number of results 63 | :param verify_ssl: verify cert 64 | :param warninglist: enforce MISP warninglist 65 | :return: dict of lists of MISP events 66 | """ 67 | query = self._search_query(iocvalue=self.domain, limit=limit, warninglist=warninglist) 68 | 69 | response = self._api_post(controller=controller, query=query, verify_ssl=verify_ssl) 70 | 71 | return response.json() 72 | 73 | def check_email(self, controller: str = "events", limit: int = 50, verify_ssl: bool = True, warninglist: bool = True) -> Dict: 74 | """Checks Email Address reputation 75 | 76 | :param controller: the MISP controller to query 77 | :param limit: number of results 78 | :param verify_ssl: verify cert 79 | :param warninglist: enforce MISP warninglist 80 | :return: dict of lists of MISP events 81 | """ 82 | query = self._search_query(iocvalue=self.email, limit=limit, warninglist=warninglist) 83 | 84 | response = self._api_post(controller=controller, query=query, verify_ssl=verify_ssl) 85 | 86 | return response.json() 87 | 88 | def check_hash(self, controller: str = "events", limit: int = 50, verify_ssl: bool = True, warninglist: bool = True) -> Dict: 89 | """Checks File Hash reputation 90 | 91 | :param controller: the MISP controller to query 92 | :param limit: number of results 93 | :param verify_ssl: verify cert 94 | :param warninglist: enforce MISP warninglist 95 | :return: dict of lists of MISP events 96 | """ 97 | query = self._search_query(iocvalue=self.file_hash, limit=limit, warninglist=warninglist) 98 | 99 | response = self._api_post(controller=controller, query=query, verify_ssl=verify_ssl) 100 | 101 | return response.json() 102 | 103 | def check_ip(self, controller: str = "events", limit: int = 50, verify_ssl: bool = True, warninglist: bool = True) -> Dict: 104 | """Checks IP reputation 105 | 106 | :param controller: the MISP controller to query 107 | :param limit: number of results 108 | :param verify_ssl: verify cert 109 | :param warninglist: enforce MISP warninglist 110 | :return: dict of lists of MISP events 111 | """ 112 | query = self._search_query(iocvalue=self.ip, limit=limit, warninglist=warninglist) 113 | 114 | response = self._api_post(controller=controller, query=query, verify_ssl=verify_ssl) 115 | 116 | return response.json() 117 | 118 | def check_url(self, controller: str = "events", limit: int = 50, verify_ssl: bool = True, warninglist: bool = True) -> Dict: 119 | """Checks URL reputation 120 | 121 | :param controller: the MISP controller to query 122 | :param limit: number of results 123 | :param verify_ssl: verify cert 124 | :param warninglist: enforce MISP warninglist 125 | :return: dict of lists of MISP events 126 | """ 127 | query = self._search_query(iocvalue=self.url, limit=limit, warninglist=warninglist) 128 | 129 | response = self._api_post(controller=controller, query=query, verify_ssl=verify_ssl) 130 | 131 | return response.json() 132 | -------------------------------------------------------------------------------- /pyoti/multis/ciscoumbrella.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from requests.auth import HTTPBasicAuth 3 | from typing import Dict, List, Optional, Union 4 | 5 | from pyoti import __version__ 6 | from pyoti.classes import Domain, IPAddress 7 | 8 | 9 | class CiscoUmbrellaInvestigate(Domain, IPAddress): 10 | """ 11 | CiscoUmbrellaInvestigate 12 | 13 | Cisco Umbrella Investigate provides detection, scoring, and prediction of emerging threats. You can predict the 14 | likelihood that a domain, an IP address, or entire ASN may contribute to the origin of an attack or pose a security 15 | threat before an attack or threat occurs. Umbrella Investigate is based on domain information gathered by the 16 | Umbrella Global Network. 17 | """ 18 | def __init__( 19 | self, 20 | api_key: str, 21 | api_url: str = "https://api.umbrella.com/investigate/v2", 22 | api_token: Optional[str] = None 23 | ): 24 | """ 25 | :param api_key: CiscoUmbrellaInvestigate API key 26 | :param api_url: CiscoUmbrellaInvestigate base API URL 27 | """ 28 | self._api_token = api_token 29 | Domain.__init__(self, api_url=api_url, api_key=api_key) 30 | IPAddress.__init__(self, api_url=api_url, api_key=api_key) 31 | 32 | @property 33 | def api_token(self): 34 | return self._api_token 35 | 36 | @api_token.setter 37 | def api_token(self, value): 38 | self._api_token = value 39 | 40 | def _api_post(self, url: str, data: Union[Dict, str], auth: Optional[HTTPBasicAuth] = None) -> requests.models.Response: 41 | """POST request to API""" 42 | headers = { 43 | 'Accept': 'application/json', 44 | 'Content-Type': 'application/json', 45 | 'User-Agent': f"PyOTI {__version__}" 46 | } 47 | 48 | if not auth and self.api_token: 49 | headers['Authorization'] = f"Bearer {self.api_token}" 50 | 51 | response = requests.request("POST", url=url, data=data, headers=headers, auth=auth) 52 | 53 | return response 54 | 55 | def _api_get(self, url: str) -> requests.models.Response: 56 | """GET request to API""" 57 | headers = { 58 | 'Accept': 'application/json', 59 | 'Content-Type': 'application/json', 60 | 'User-Agent': f"PyOTI {__version__}" 61 | } 62 | if self.api_token: 63 | headers['Authorization'] = f"Bearer {self._api_token}" 64 | 65 | response = requests.request("GET", url=url, headers=headers) 66 | 67 | return response 68 | 69 | def _get_token(self) -> None: 70 | """Get OAuth API token""" 71 | client_id = self.api_key.split(":")[0] 72 | client_secret = self.api_key.split(":")[1] 73 | 74 | data = {'grant_type': 'client_credentials'} 75 | auth = HTTPBasicAuth(client_id, client_secret) 76 | self.api_token = self._api_post( 77 | url="https://api.umbrella.com/auth/v2/token", 78 | data=data, 79 | auth=auth 80 | ).json().get('access_token') 81 | 82 | def check_domain_status_and_categorization(self, show_labels: bool = True) -> Union[Dict, None]: 83 | """ 84 | Check domain status and categorization 85 | 86 | Look up the status and security and content category IDs for a domain. 87 | 88 | The domain status is a numerical value determined by the Cisco Security Labs team. 89 | Valid status values are: '-1' (malicious), '1' (safe), or '0' (undetermined status). 90 | 91 | :param show_labels: display the security and content category labels in the response 92 | """ 93 | if not self.api_token: 94 | self._get_token() 95 | 96 | if show_labels: 97 | url = f"{self.api_url}/domains/categorization/{self.domain}?showLabels" 98 | else: 99 | url = f"{self.api_url}/domains/categorization/{self.domain}" 100 | response = self._api_get(url=url) 101 | 102 | return response.json() 103 | 104 | def bulk_check_domain_status_and_categorization( 105 | self, 106 | domains: List[str], 107 | show_labels: bool = True 108 | ) -> Union[List[Dict], None]: 109 | """ 110 | Bulk check domain status and categorization 111 | 112 | Provide a list of domains and look up the status, and security and content category IDs for each domain. 113 | 114 | In a single request, the payload must not exceed 100KB and contain no more than 1000 domains. 115 | 116 | :param domains: list of domains to check status and categorization 117 | :param show_labels: display the security and content category labels in the response 118 | """ 119 | if not self.api_token: 120 | self._get_token() 121 | 122 | payload = f'''{domains}''' 123 | 124 | if show_labels: 125 | url = f"{self.api_url}/domains/categorization?showLabels" 126 | else: 127 | url = f"{self.api_url}/domains/categorization" 128 | response = self._api_post(url=url, data=payload) 129 | 130 | return response.json() 131 | 132 | def check_domain_security_score(self) -> Union[Dict, None]: 133 | """ 134 | Check domain security score information 135 | 136 | List multiple scores or security features for a domain. You can use the scores or security features to determine 137 | relevant data points and build insights on the reputation or security risk posed by the site. No one security 138 | information feature is conclusive. Instead, consider these features as part of your security research. 139 | """ 140 | if not self.api_token: 141 | self._get_token() 142 | 143 | url = f"{self.api_url}/security/name/{self.domain}" 144 | 145 | response = self._api_get(url=url) 146 | 147 | return response.json() 148 | 149 | def check_domain_risk_score(self) -> Union[Dict, None]: 150 | """ 151 | Check domain risk score 152 | 153 | The Investigate Risk Score is based on an analysis of the lexical characteristics of the domain name and 154 | patterns in queries and requests to the domain. The risk score is scaled from 0 to 100 where 100 is the highest 155 | risk and 0 represents no risk at all. Periodically, Investigate updates this score based on additional inputs. 156 | A domain blocked by Umbrella receives a score of 100. 157 | """ 158 | if not self.api_token: 159 | self._get_token() 160 | 161 | url = f"{self.api_url}/domains/risk-score/{self.domain}" 162 | 163 | response = self._api_get(url=url) 164 | 165 | return response.json() 166 | 167 | def check_ip_resource_records(self) -> Union[Dict, None]: 168 | """ 169 | Check IP resource records 170 | 171 | Get the Resource Record (RR) data for DNS responses, and categorization data, where the answer (or rdata) is 172 | the domain(s). 173 | """ 174 | if not self.api_token: 175 | self._get_token() 176 | 177 | url = f"{self.api_url}/pdns/ip/{self.ip}" 178 | 179 | response = self._api_get(url=url) 180 | 181 | return response.json() -------------------------------------------------------------------------------- /pyoti/ips/greynoise.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from typing import Dict, List 3 | 4 | from pyoti import __version__ 5 | from pyoti.classes import IPAddress 6 | 7 | 8 | class GreyNoise(IPAddress): 9 | """GreyNoise 10 | 11 | GreyNoise produces two datasets of IP information that can be used for threat enrichment. GreyNoise’s internet-wide 12 | sensor network passively collects packets from hundreds of thousands of IPs seen scanning the internet every day. 13 | """ 14 | def __init__(self, api_key: str, api_url: str = "https://api.greynoise.io"): 15 | """ 16 | :param api_key: GreyNoise API key 17 | :param api_url: GreyNoise base API URL 18 | """ 19 | IPAddress.__init__(self, api_key, api_url) 20 | self.codes = { 21 | "0x00": "IP hasn't been observed scanning the internet.", 22 | "0x01": "IP has been observed by GreyNoise sensor network.", 23 | "0x02": "IP has been observed scanning GreyNoise sensor network, but hasn't completed a full connection, " 24 | "meaning this can be spoofed.", 25 | "0x03": "IP is adjacent to another host that has been directly observed by GreyNoise sensor network.", 26 | "0x04": "Reserved.", 27 | "0x05": "IP is commonly spoofed in internet-scan activity.", 28 | "0x06": "IP has been observed as noise, but this host belongs to a cloud provider where IPs can be cycled.", 29 | "0x07": "IP is invalid.", 30 | "0x08": "IP was classified as noise, but has not been observed engaging in internet-wide scans or attacks " 31 | "in over 90 days.", 32 | "0x09": "IP was found in RIOT.", 33 | "0x10": "IP has been observed by GreyNoise sensor network and was found in RIOT." 34 | } 35 | 36 | def _api_get(self, url: str) -> requests.models.Response: 37 | """GET request to API""" 38 | headers = { 39 | "Accept": "application/json", 40 | "key": self.api_key, 41 | "User-Agent": f"PyOTI/ {__version__}" 42 | } 43 | 44 | response = requests.request("GET", url=url, headers=headers) 45 | 46 | return response 47 | 48 | def _api_post(self, url: str, ip_list: List[str]) -> requests.models.Response: 49 | """POST request to API""" 50 | payload = {"ips": ip_list} 51 | 52 | headers = { 53 | "Accept": "application/json", 54 | "Content-Type": "application/json", 55 | "key": self.api_key, 56 | "User-Agent": f"PyOTI {__version__}" 57 | } 58 | 59 | response = requests.request("POST", url=url, json=payload, headers=headers) 60 | 61 | return response 62 | 63 | def check_ip_community(self) -> Dict: 64 | """Check IP reputation community 65 | 66 | The Community API provides community users with a free tool to query IPs in the GreyNoise dataset and retrieve 67 | a subset of the full IP context data returned by the IP Lookup API. 68 | """ 69 | url = f"{self.api_url}/v3/community/{self.ip}" 70 | response = self._api_get(url=url) 71 | 72 | return response.json() 73 | 74 | def check_ip_quick(self) -> Dict: 75 | """ Check IP reputation quick 76 | 77 | Requires premium API key. 78 | 79 | Check whether a given IP address is “Internet background noise”, or has been observed scanning or attacking 80 | devices across the Internet. 81 | """ 82 | url = f"{self.api_url}/v2/noise/quick/{self.ip}" 83 | response = self._api_get(url=url) 84 | 85 | r = response.json() 86 | r_code = r.get("code") 87 | r['code_message'] = self.codes.get(r_code) 88 | 89 | return r 90 | 91 | def bulk_check_ips_quick(self, ip_list: List[str]) -> List[Dict]: 92 | """Bulk check IP reputation quick 93 | 94 | Requires premium API key. 95 | 96 | Check whether a set of IP addresses are "Internet background noise", or have been observed scanning or 97 | attacking devices across the Internet. 98 | 99 | :param ip_list: List of IPs to check reputation 100 | """ 101 | url = f"{self.api_url}/v2/noise/multi/quick" 102 | if len(ip_list) <= 1000: 103 | response = self._api_post(url=url, ip_list=ip_list) 104 | r = response.json() 105 | for result in r: 106 | result['code_message'] = self.codes.get(result['code']) 107 | return r 108 | else: 109 | chunk_size = 1000 110 | chunks = [ip_list[i:i + chunk_size] for i in range(0, len(ip_list), chunk_size)] 111 | results = [] 112 | 113 | for chunk in chunks: 114 | response = self._api_post(url=url, ip_list=chunk) 115 | r = response.json() 116 | for result in r: 117 | result['code_message'] = self.codes.get(result['code']) 118 | [results.append(result) for result in r] 119 | 120 | return results 121 | 122 | def check_ip_context(self) -> Dict: 123 | """ Check IP reputation context 124 | 125 | Requires premium API key. 126 | 127 | Get more information about a given IP address. Returns time ranges, IP metadata (network owner, ASN, reverse 128 | DNS pointer, country), associated actors, activity tags, and raw port scan and web request information. 129 | """ 130 | url = f"{self.api_url}/v2/noise/context/{self.ip}" 131 | response = self._api_get(url=url) 132 | 133 | return response.json() 134 | 135 | def bulk_check_ip_context(self, ip_list: List[str]) -> List[Dict]: 136 | """ Bulk check IP reputation context 137 | 138 | Requires premium API key. 139 | 140 | Get more information about a set of IP addresses. Returns time ranges, IP metadata (network owner, ASN, 141 | reverse DNS pointer, country), associated actors, activity tags, and raw port scan and web request information. 142 | 143 | :param ip_list: List of IPs to check reputation 144 | """ 145 | url = f"{self.api_url}/v2/noise/multi/context" 146 | 147 | if len(ip_list) <= 1000: 148 | response = self._api_post(url=url, ip_list=ip_list) 149 | 150 | return response.json().get('data') 151 | else: 152 | chunk_size = 1000 153 | chunks = [ip_list[i:i + chunk_size] for i in range(0, len(ip_list), chunk_size)] 154 | results = [] 155 | 156 | for chunk in chunks: 157 | response = self.api_post(url=url, ip_list=chunk) 158 | r = response.json().get('data') 159 | [results.append(result) for result in r] 160 | 161 | return results 162 | 163 | def check_ip_riot(self) -> Dict: 164 | """ Check IP reputation RIOT 165 | 166 | Requires premium API key. 167 | 168 | RIOT identifies IPs from known benign services and organizations that commonly cause false positives in network 169 | security and threat intelligence products. The collection of IPs in RIOT is continually curated and verified to 170 | provide accurate results. 171 | """ 172 | url = f"{self.api_url}/v2/riot/{self.ip}" 173 | response = self._api_get(url=url) 174 | 175 | return response.json() 176 | -------------------------------------------------------------------------------- /pyoti/multis/dnsblocklist.py: -------------------------------------------------------------------------------- 1 | import aiodns 2 | import asyncio 3 | import pycares 4 | import sys 5 | from typing import Dict, List, Union 6 | 7 | from pyoti.classes import Domain, IPAddress 8 | 9 | 10 | class DNSBlockList(Domain, IPAddress): 11 | """DNSBlockList Domain/IP Block List 12 | 13 | DNSBlockList queries a list of DNS block lists for Domains or IP Addresses, 14 | and returns the answer address and the block list it hit on. 15 | """ 16 | RBL = { # IP-Based Zones 17 | "b.barracudacentral.org", 18 | "bl.spamcop.net", 19 | "zen.spamhaus.org", 20 | } 21 | 22 | DBL = { # Domain-Based Zones 23 | "dbl.spamhaus.org", 24 | "multi.uribl.com", 25 | "multi.surbl.org", 26 | } 27 | 28 | RBL_CODES = { 29 | "barracudacentral": { 30 | "127.0.0.2": "brbl" 31 | }, 32 | "spamcop": { 33 | "127.0.0.2": "scbl" 34 | }, 35 | "spamhaus": { 36 | "127.0.0.2": "sbl", 37 | "127.0.0.3": "css", 38 | "127.0.0.4": "xbl", 39 | "127.0.0.9": "drop", 40 | "127.0.0.10": "pbl", 41 | "127.0.0.11": "pbl", 42 | "127.255.255.252": "Typing error in DNSBL name!", 43 | "127.255.255.245": "Query via public/open resolver!", 44 | "127.255.255.255": "Excessive number of queries!" 45 | } 46 | } 47 | 48 | DBL_CODES = { 49 | "spamhaus": { 50 | "127.0.1.2": "spam", 51 | "127.0.1.4": "phish", 52 | "127.0.1.5": "malware", 53 | "127.0.1.6": "botnet-c2", 54 | "127.0.1.102": "abused-legit-spam", 55 | "127.0.1.103": "abused-spammed-redirector", 56 | "127.0.1.104": "abused-legit-phish", 57 | "127.0.1.105": "abused-legit-malware", 58 | "127.0.1.106": "abused-legit-botnet-c2", 59 | "127.0.1.255": "IP queries prohibited!", 60 | "127.255.255.252": "Typing error in DNSBL name!", 61 | "127.255.255.254": "Anonymous query through public resolver!", 62 | "127.255.255.255": "Excessive number of queries!" 63 | }, 64 | "surbl": { 65 | "127.0.0.1": "Access is blocked!", 66 | "127.0.0.8": "phish", 67 | "127.0.0.16": "malware", 68 | "127.0.0.24": ["phish", "malware"], 69 | "127.0.0.64": "spam", 70 | "127.0.0.72": ["phish", "spam"], 71 | "127.0.0.80": ["malware", "spam"], 72 | "127.0.0.88": ["phish", "malware", "spam"], 73 | "127.0.0.128": "abused-legit", 74 | "127.0.0.136": ["phish", "abused-legit"], 75 | "127.0.0.144": ["malware", "abused-legit"], 76 | "127.0.0.152": ["phish", "malware", "abused-legit"], 77 | "127.0.0.192": ["spam", "abused-legit"], 78 | "127.0.0.200": ["phish", "spam", "abused-legit"], 79 | "127.0.0.208": ["malware", "spam", "abused-legit"], 80 | "127.0.0.216": ["phish", "malware", "spam", "abused-legit"] 81 | }, 82 | "uribl": { 83 | "127.0.0.1": "Query is blocked! Possibly due to high volume.", 84 | "127.0.0.2": "black", 85 | "127.0.0.4": "grey", 86 | "127.0.0.8": "red", 87 | "127.0.0.14": "multi" 88 | } 89 | } 90 | 91 | def __init__(self, domain: str = None, ip: str = None): 92 | Domain.__init__(self, domain=domain) 93 | IPAddress.__init__(self, ip=ip) 94 | 95 | def check_domain(self) -> List[Dict]: 96 | """Checks Domain reputation 97 | 98 | Checks DNS lookup query for a given domain and maps return codes to 99 | appropriate data source. 100 | 101 | :return: list of dict with query response address and blocklist the domain was found on 102 | """ 103 | result_list = [] 104 | for dbl in self.DBL: 105 | answer = self._a_query(blocklist=dbl, type="domain") 106 | if answer: 107 | results = {} 108 | bl = dbl.split(".")[1] 109 | zone = self.DBL_CODES[bl].get(answer[0].host, "unknown") 110 | results["address"] = answer[0].host 111 | if answer[0].host in [ 112 | "127.0.0.1", 113 | "127.0.1.255", 114 | "127.255.255.252", 115 | "127.255.255.254", 116 | "127.255.255.255" 117 | ]: 118 | results["error"] = f"{bl}:{zone}" 119 | else: 120 | if isinstance(zone, str): 121 | results["blocklist"] = f"{bl}-{zone}" 122 | elif isinstance(zone, list): 123 | results["blocklist"] = [f"{bl}-{z}" for z in zone] 124 | result_list.append(results) 125 | return result_list 126 | 127 | def check_ip(self) -> List[Dict]: 128 | """Checks IP reputation 129 | 130 | Checks reverse DNS lookup query for a given IP and maps return codes to 131 | appropriate data source. 132 | 133 | :return: list of dict with query response address and blocklist the IP was found on 134 | """ 135 | result_list = [] 136 | for rbl in self.RBL: 137 | answer = self._a_query(blocklist=rbl, type="ip") 138 | if answer: 139 | results = {} 140 | bl = rbl.split(".")[1] 141 | zone = self.RBL_CODES[bl].get(answer[0].host, "unknown") 142 | results["address"] = answer[0].host 143 | if answer[0].host in [ 144 | "127.255.255.252", 145 | "127.255.255.254", 146 | "127.255.255.255" 147 | ]: 148 | results["error"] = f"{bl}:{zone}" 149 | else: 150 | results["blocklist"] = f"{bl}-{zone}" 151 | result_list.append(results) 152 | return result_list 153 | 154 | def _reverse_ip(self, ipaddr: str) -> str: 155 | """Prepares IPv4 address for reverse lookup 156 | 157 | :param ipaddr: IP Address 158 | :return: reversed IP address for DNS query 159 | """ 160 | return ".".join(reversed(ipaddr.split("."))) 161 | 162 | def _a_query(self, blocklist: str, type: str) -> Union[List[pycares.ares_query_a_result], None]: 163 | """DNS A record query 164 | 165 | :param blocklist: DNS blocklist URL 166 | :param type: ip or domain 167 | :return: list of ares_query_a_result 168 | """ 169 | try: 170 | if sys.platform == "win32": 171 | asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) 172 | 173 | async def query_a(name): 174 | resolver = aiodns.DNSResolver(nameservers=["208.67.222.222"]) # OpenDNS nameserver 175 | return await resolver.query(name, "A") 176 | 177 | if type == "ip": 178 | host = f"{self._reverse_ip(ipaddr=self.ip)}.{blocklist}" 179 | elif type == "domain": 180 | host = f"{self.domain}.{blocklist}" 181 | 182 | result = asyncio.run(query_a(name=host)) 183 | 184 | return result 185 | 186 | except aiodns.error.DNSError: 187 | return 188 | -------------------------------------------------------------------------------- /pyoti/multis/urlscan.py: -------------------------------------------------------------------------------- 1 | import json 2 | import requests 3 | import time 4 | from typing import Dict, Optional 5 | from uuid import UUID 6 | 7 | from pyoti import __version__ 8 | from pyoti.classes import Domain, FileHash, IPAddress, URL 9 | from pyoti.exceptions import PyOTIError 10 | 11 | 12 | class URLscan(Domain, FileHash, IPAddress, URL): 13 | """URLscan a sandbox for the web 14 | 15 | URLscan is a free service to scan and analyse websites. 16 | """ 17 | def __init__(self, api_key: str, api_url: str = "https://urlscan.io/api/v1", id=None): 18 | self._id = id 19 | Domain.__init__(self, api_key=api_key, api_url=api_url) 20 | FileHash.__init__(self, api_key=api_key, api_url=api_url) 21 | IPAddress.__init__(self, api_key=api_key, api_url=api_url) 22 | URL.__init__(self, api_key=api_key, api_url=api_url) 23 | 24 | self.countries = [ 25 | "de", 26 | "us", 27 | "jp", 28 | "fr", 29 | "gb", 30 | "nl", 31 | "ca", 32 | "it", 33 | "es", 34 | "se", 35 | "fi", 36 | "dk", 37 | "no", 38 | "is", 39 | "au", 40 | "nz", 41 | "pl", 42 | "sg", 43 | "ge", 44 | "pt", 45 | "at", 46 | "ch" 47 | ] 48 | 49 | @property 50 | def id(self): 51 | return self._id 52 | 53 | @id.setter 54 | def id(self, value): 55 | self._id = value 56 | 57 | def _api_get(self, endpoint: str, params: Optional[Dict]) -> requests.models.Response: 58 | """GET request to urlscan API 59 | 60 | :param endpoint: urlscan API endpoint 61 | :param params: params for request 62 | """ 63 | headers = { 64 | "API-Key": self.api_key, 65 | "User-Agent": f"PyOTI {__version__}" 66 | } 67 | rparams = params 68 | 69 | uri = self.api_url + endpoint 70 | response = requests.request("GET", url=uri, headers=headers, params=rparams) 71 | 72 | return response 73 | 74 | def _api_post(self, endpoint: str, data: Optional[Dict]) -> requests.models.Response: 75 | """POST request to urlscan API""" 76 | headers = { 77 | "API-Key": self.api_key, 78 | "Content-Type": "application/json", 79 | "User-Agent": f"PyOTI {__version__}" 80 | } 81 | response = requests.request("POST", url=endpoint, headers=headers, data=json.dumps(data)) 82 | 83 | return response 84 | 85 | def _escape_url(self, url: str) -> str: 86 | """Escape URL for elastic syntax 87 | 88 | :param url: url to escape 89 | :return: escaped url for elastic syntax 90 | """ 91 | url = url.replace(":", "\:") 92 | url = url.replace("/", "\/") 93 | 94 | return url 95 | 96 | def search_domain(self, contacted: bool = False, limit: int = 100) -> Dict: 97 | """ 98 | :param contacted: default False (domain was contacted but isn't the page/primary domain) 99 | :param limit: default 100 (number of results to return, max: 10000) 100 | :return: dict of request response 101 | """ 102 | if contacted: 103 | params = {"q": f"domain:{self.domain} AND NOT page.domain:{self.domain}", "size": limit} 104 | else: 105 | params = {"q": f"domain:{self.domain}", "size": limit} 106 | 107 | response = self._api_get(endpoint="/search/", params=params) 108 | 109 | return response.json() 110 | 111 | def search_hash(self, limit: int = 100) -> Dict: 112 | """ 113 | :param limit: default 100 (number of results to return, max: 10000) 114 | :return: dict of request response 115 | """ 116 | params = {"q": f"hash:{self.file_hash}", "size": limit} 117 | 118 | response = self._api_get(endpoint="/search/", params=params) 119 | 120 | return response.json() 121 | 122 | def search_ip(self, limit: int = 100) -> Dict: 123 | """ 124 | :param limit: default 100 (number of results to return, max: 10000) 125 | :return: dict of request response 126 | """ 127 | params = {"q": f"page.ip:{self.ip}", "size": limit} 128 | 129 | response = self._api_get(endpoint="/search/", params=params) 130 | 131 | return response.json() 132 | 133 | def search_url(self, limit: int = 100) -> Dict: 134 | """ 135 | :param limit: default 100 (number of results to return, max: 10000) 136 | :return: dict of request response 137 | """ 138 | params = {"q": f"task.url:{self._escape_url(self.url)}", "size": limit} 139 | 140 | response = self._api_get(endpoint="/search/", params=params) 141 | 142 | return response.json() 143 | 144 | def check_domain(self, uuid: UUID = None) -> Dict: 145 | """ 146 | :param uuid: urlscan result UUID 147 | :return: dict of request response 148 | """ 149 | if uuid: 150 | response = self._api_get(endpoint=f"/result/{uuid}", params=None) 151 | else: 152 | raise PyOTIError("Missing result UUID. Use search_domain method to get result UUID.") 153 | 154 | return response.json() 155 | 156 | def check_hash(self, uuid: UUID = None) -> Dict: 157 | """ 158 | :param uuid: urlscan result UUID 159 | :return: dict of request response 160 | """ 161 | if uuid: 162 | response = self._api_get(endpoint=f"/result/{uuid}", params=None) 163 | else: 164 | raise PyOTIError("Missing result UUID. Use search_hash method to get result UUID.") 165 | 166 | return response.json() 167 | 168 | def check_ip(self, uuid: UUID = None) -> Dict: 169 | """ 170 | :param uuid: urlscan result UUID 171 | :return: dict of request response 172 | """ 173 | if uuid: 174 | response = self._api_get(endpoint=f"/result/{uuid}", params=None) 175 | else: 176 | raise PyOTIError("Missing result UUID. Use search_ip method to get result UUID.") 177 | 178 | return response.json() 179 | 180 | def check_url(self, uuid: UUID = None) -> Dict: 181 | """ 182 | :param uuid: urlscan result UUID 183 | :return: dict of request response 184 | """ 185 | if uuid: 186 | response = self._api_get(endpoint=f"/result/{uuid}", params=None) 187 | else: 188 | raise PyOTIError("Missing result UUID. Use search_url method to get result UUID.") 189 | 190 | return response.json() 191 | 192 | def submit_url( 193 | self, 194 | user_agent: Optional[str], 195 | referer: Optional[str], 196 | visibility: Optional[str], 197 | country: Optional[str] 198 | ) -> Dict: 199 | """Submit a URL to be scanned and set some options for the scan 200 | 201 | :param user_agent: Override User-Agent for this scan 202 | :param referer: Override HTTP referer for this scan 203 | :param visibility: One of [public, unlisted, private]. Defaults to your URLscan account configured default visibility 204 | :param country: Specify which country the scan should be performed from (2-letter ISO-3166-1 alpha-2 country) 205 | """ 206 | data = {"url": self.url} 207 | if user_agent: 208 | data["customagent"] = user_agent 209 | if referer: 210 | data["referer"] = referer 211 | if visibility: 212 | data["visibility"] = visibility 213 | if country: 214 | if country.lower() in self.countries: 215 | data["country"] = country 216 | else: 217 | raise Exception( 218 | f"[!] {country} is not a valid entry. Please choose from the following list: {self.countries}" 219 | ) 220 | 221 | response = self._api_post(endpoint=f"{self.api_url}/scan", data=data) 222 | 223 | return response.json() 224 | 225 | def get_submission_results(self, uuid: UUID) -> Dict: 226 | start_time = time.time() 227 | timeout = 180 # seconds 228 | 229 | while True: 230 | elapsed_time = time.time() - start_time 231 | 232 | if elapsed_time > timeout: 233 | # upper timeout reached 234 | raise TimeoutError(f"Timeout reached while waiting on submission results for UUID: {uuid}") 235 | 236 | response = self._api_get(endpoint=f"/result/{uuid}/", params=None) 237 | if response.status_code == 404: 238 | time.sleep(5) 239 | elif response.status_code == 200: 240 | return response.json() 241 | -------------------------------------------------------------------------------- /pyoti/multis/triage.py: -------------------------------------------------------------------------------- 1 | import binascii 2 | import json 3 | import os 4 | import requests 5 | from io import BytesIO 6 | from pathlib import Path 7 | from typing import BinaryIO, Dict, Optional, Tuple, Union 8 | 9 | from pyoti import __version__ 10 | from pyoti.classes import Domain, IPAddress, FileHash, URL 11 | 12 | 13 | def encode_multipart_formdata(fields): 14 | boundary = binascii.hexlify(os.urandom(16)).decode('ascii') 15 | 16 | body = BytesIO() 17 | for field, value in fields.items(): # (name, file) 18 | if isinstance(value, tuple): 19 | filename, file = value 20 | body.write( 21 | '--{boundary}\r\nContent-Disposition: form-data; ' 22 | 'filename="{filename}"; name=\"{field}\"\r\n\r\n' 23 | .format(boundary=boundary, field=field, filename=filename) 24 | .encode('utf-8') 25 | ) 26 | b = file.read() 27 | if isinstance(b, str): # If the file was opened in text mode 28 | b = b.encode('ascii') 29 | body.write(b) 30 | body.write(b'\r\n') 31 | else: 32 | body.write( 33 | '--{boundary}\r\nContent-Disposition: form-data;' 34 | 'name="{field}"\r\n\r\n{value}\r\n' 35 | .format(boundary=boundary, field=field, value=value) 36 | .encode('utf-8') 37 | ) 38 | body.write('--{0}--\r\n'.format(boundary).encode('utf-8')) 39 | body.seek(0) 40 | 41 | return body, "multipart/form-data; boundary=" + boundary 42 | 43 | 44 | class Triage(Domain, IPAddress, FileHash, URL): 45 | """ 46 | Triage is Hatching's revolutionary sandboxing solution. It leverages a unique architecture, developed with scaling 47 | and performance in mind from the start. Triage features Windows, Linux, Android, and macOS analysis capabilities 48 | and can scale up to 500,000 analyses per day. 49 | """ 50 | 51 | def __init__(self, api_key: str, api_url: str = "https://tria.ge/api/v0"): 52 | """ 53 | :param api_key: Triage API key 54 | :param api_url: Triage base API url 55 | """ 56 | Domain.__init__(self, api_key=api_key, api_url=api_url) 57 | IPAddress.__init__(self, api_key=api_key, api_url=api_url) 58 | FileHash.__init__(self, api_key=api_key, api_url=api_url) 59 | URL.__init__(self, api_key=api_key, api_url=api_url) 60 | 61 | def _api_get(self, url: str, params: Optional[Dict]) -> requests.models.Response: 62 | """GET request to API""" 63 | headers = { 64 | "User-Agent": f"PyOTI {__version__}", 65 | "Authorization": f"Bearer {self.api_key}" 66 | } 67 | response = requests.request("GET", url=url, headers=headers, params=params) 68 | 69 | return response 70 | 71 | def _api_post( 72 | self, 73 | endpoint: str, 74 | submission_type: str, 75 | data: Optional[Dict] = None, 76 | file: Optional[Tuple[str, BinaryIO]] = None, 77 | json_data: Optional[Dict] = None 78 | ) -> requests.models.Response: 79 | """POST request to API""" 80 | headers = { 81 | "Authorization": f"Bearer {self.api_key}", 82 | "User-Agent": f"PyOTI {__version__}" 83 | } 84 | body = None 85 | if submission_type == "url": 86 | headers["Content-Type"] = "application/json" 87 | elif submission_type == "file": 88 | body, content_type = encode_multipart_formdata( 89 | { 90 | "_json": json.dumps(data), 91 | "file": file 92 | } 93 | ) 94 | headers["Content-Type"] = content_type 95 | 96 | response = requests.request( 97 | "POST", 98 | url=f"{self.api_url}{endpoint}", 99 | headers=headers, 100 | data=body, 101 | json=json_data 102 | ) 103 | 104 | return response 105 | 106 | def check_domain(self) -> Dict: 107 | """Check if domain was extracted from C2 data""" 108 | params = {"query": f"domain:{self.domain}"} 109 | response = self._api_get(url=f"{self.api_url}/search", params=params) 110 | 111 | return response.json() 112 | 113 | def check_hash(self) -> Dict: 114 | """Check if file hash has been seen by Triage""" 115 | params = {} 116 | if len(self.file_hash) == 32: 117 | params["query"] = f"md5:{self.file_hash}" 118 | elif len(self.file_hash) == 40: 119 | params["query"] = f"sha1:{self.file_hash}" 120 | elif len(self.file_hash) == 64: 121 | params["query"] = f"sha256:{self.file_hash}" 122 | else: 123 | return {"error": "You can only search by MD5, SHA1, or SHA256!"} 124 | response = self._api_get(url=f"{self.api_url}/search", params=params) 125 | 126 | return response.json() 127 | 128 | def check_ip(self) -> Dict: 129 | """Check if IP address was extracted from C2 data""" 130 | params = {"query": f"ip:{self.ip}"} 131 | response = self._api_get(url=f"{self.api_url}/search", params=params) 132 | 133 | return response.json() 134 | 135 | def check_url(self) -> Dict: 136 | """Check if URL was extracted from C2 data""" 137 | params = {"query": f"url:{self.url}"} 138 | response = self._api_get(url=f"{self.api_url}/search", params=params) 139 | 140 | return response.json() 141 | 142 | def get_sample_summary(self, sample_id: str) -> Dict: 143 | """Get the short summary of a sample and its analysis tasks 144 | 145 | :param sample_id: The sample ID to get summary of 146 | """ 147 | response = self._api_get(url=f"{self.api_url}/samples/{sample_id}/summary", params=None) 148 | 149 | if response.status_code == 200: 150 | return response.json() 151 | elif response.status_code == 404: 152 | return {} 153 | 154 | def get_sample_overview(self, sample_id: str) -> Dict: 155 | """Get the overview of a sample and its analysis tasks. This contains a one-pager with all the high-level 156 | information related to the sample including malware configuration, signatures, scoring, etc. 157 | 158 | :param sample_id: The sample ID to get summary overview of 159 | """ 160 | response = self._api_get(url=f"{self.api_url}/samples/{sample_id}/overview.json", params=None) 161 | 162 | if response.status_code == 200: 163 | return response.json() 164 | elif response.status_code == 404: 165 | return {} 166 | 167 | def get_sample(self, sample_id: str) -> Dict: 168 | """Queries the sample with the specified ID 169 | 170 | :param sample_id: The sample ID to query 171 | """ 172 | response = self._api_get(url=f"{self.api_url}/samples/{sample_id}", params=None) 173 | 174 | if response.status_code == 200: 175 | return response.json() 176 | elif response.status_code == 404: 177 | return {} 178 | 179 | def submit_file(self, file_path: Union[str, Path], pw: Optional[str] = None, timeout: Optional[int] = 60, 180 | network: Optional[str] = "internet"): 181 | """Submit a sample file to be analyzed by Triage 182 | 183 | :param file_path: Path to the submission file 184 | :param pw: Password if file is a ZIP file 185 | :param timeout: The timeout of analysis (in seconds) 186 | :param network: The type of network routing to use ("internet"|"drop"|"tor") 187 | """ 188 | data = { 189 | "kind": "file", 190 | "defaults": { 191 | "timeout": timeout, 192 | "network": network 193 | } 194 | } 195 | if pw: 196 | data["password"] = pw 197 | 198 | if not isinstance(file_path, Path): 199 | file_path = Path(file_path) 200 | 201 | file = (file_path.name, open(file_path, "rb")) 202 | 203 | response = self._api_post(endpoint="/samples", submission_type="file", data=data, file=file) 204 | 205 | return response.json() 206 | 207 | def submit_url(self, timeout: Optional[int] = 60, network: Optional[str] = "internet") -> Dict: 208 | data = { 209 | "kind": "url", 210 | "url": self.url, 211 | "defaults": { 212 | "timeout": timeout, 213 | "network": network 214 | } 215 | } 216 | 217 | response = self._api_post(endpoint="/samples", submission_type="url", json_data=data) 218 | 219 | return response.json() 220 | 221 | def fetch_file(self, timeout: Optional[int] = 60, network: Optional[str] = "internet") -> Dict: 222 | data = { 223 | "kind": "fetch", 224 | "url": self.url, 225 | "defaults": { 226 | "timeout": timeout, 227 | "network": network 228 | } 229 | } 230 | 231 | response = self._api_post(endpoint="/samples", submission_type="url", json_data=data) 232 | 233 | return response.json() 234 | -------------------------------------------------------------------------------- /docs/tutorials/phishing_triage_urls.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "source": [ 6 | "# PyOTI Phishing Triage\n", 7 | "***\n", 8 | "## Installation\n", 9 | "\n", 10 | "```bash\n", 11 | "python3 -m pip install virtualenv\n", 12 | "git clone https://github.com/RH-ISAC/PyOTI\n", 13 | "cd PyOTI\n", 14 | "python3 -m venv venv\n", 15 | "source venv/bin/activate\n", 16 | "python3 -m pip install -r requirements.txt\n", 17 | "python3 -m pip install .\n", 18 | "```\n", 19 | ">If you experience issues installing pycares, please uninstall c-ares from ```/usr/local``` or run ```brew uninstall --ignore-dependencies c-ares```. Pycares depends on the bundled version. (https://github.com/ccxt/ccxt/issues/4798)\n", 20 | "\n", 21 | "## API Keys\n", 22 | "Set your API key variables below:" 23 | ], 24 | "metadata": { 25 | "collapsed": false 26 | } 27 | }, 28 | { 29 | "cell_type": "code", 30 | "execution_count": null, 31 | "outputs": [], 32 | "source": [ 33 | "domaintools = '{USER}:{SECRET}'\n", 34 | "googlesafebrowsing = ''\n", 35 | "hybridanalysis = ''\n", 36 | "phishtank = ''\n", 37 | "urlscan = ''\n", 38 | "virustotal = ''" 39 | ], 40 | "metadata": { 41 | "collapsed": false, 42 | "pycharm": { 43 | "name": "#%%\n" 44 | } 45 | } 46 | }, 47 | { 48 | "cell_type": "markdown", 49 | "source": [ 50 | "***\n", 51 | "## URL Analysis\n", 52 | "Set the suspicious URL variable below:\n", 53 | "\n", 54 | ">If the URL contains base64-encoded username/address please replace with ``` redacted@redacted.com ``` or ``` cmVkYWN0ZWRAcmVkYWN0ZWQuY29tCg== ```\n", 55 | "\n", 56 | "**Suspicious URL**" 57 | ], 58 | "metadata": { 59 | "collapsed": false 60 | } 61 | }, 62 | { 63 | "cell_type": "code", 64 | "execution_count": null, 65 | "outputs": [], 66 | "source": [ 67 | "# we import this to safely display the suspicious URL to avoid accidental clicks\n", 68 | "from defang import defang\n", 69 | "\n", 70 | "phish_url = ''" 71 | ], 72 | "metadata": { 73 | "collapsed": false, 74 | "pycharm": { 75 | "name": "#%%\n" 76 | } 77 | } 78 | }, 79 | { 80 | "cell_type": "markdown", 81 | "source": [ 82 | "**Iris Investigate**" 83 | ], 84 | "metadata": { 85 | "collapsed": false 86 | } 87 | }, 88 | { 89 | "cell_type": "code", 90 | "execution_count": null, 91 | "outputs": [], 92 | "source": [ 93 | "from pyoti.domains import IrisInvestigate\n", 94 | "from pyoti.utils import split_url_domain\n", 95 | "\n", 96 | "iris = IrisInvestigate(api_key=domaintools)\n", 97 | "phish_domain = split_url_domain(phish_url)\n", 98 | "iris.domain = phish_domain\n", 99 | "domain = iris.check_domain()\n", 100 | "\n", 101 | "print(f\"[+] Domain risk score for {phish_domain}: {domain[0]['domain_risk']['risk_score']}\")" 102 | ], 103 | "metadata": { 104 | "collapsed": false, 105 | "pycharm": { 106 | "name": "#%%\n" 107 | } 108 | } 109 | }, 110 | { 111 | "cell_type": "markdown", 112 | "source": [ 113 | "\n", 114 | "**Google Safe Browsing**" 115 | ], 116 | "metadata": { 117 | "collapsed": false 118 | } 119 | }, 120 | { 121 | "cell_type": "code", 122 | "execution_count": null, 123 | "outputs": [], 124 | "source": [ 125 | "from pyoti.urls import GoogleSafeBrowsing\n", 126 | "\n", 127 | "gsb = GoogleSafeBrowsing(api_key=googlesafebrowsing)\n", 128 | "gsb.url = phish_url\n", 129 | "url = gsb.check_url()\n", 130 | "\n", 131 | "if url['matches']:\n", 132 | " print(f\"[+] Threat Type: {url['matches'][0]['threatType']}\")\n", 133 | " print(f\"[+] Platform Type: {url['matches'][0]['platformType']}\")\n", 134 | "else:\n", 135 | " print(f\"[*] No results for {defang(phish_url)}!\")" 136 | ], 137 | "metadata": { 138 | "collapsed": false, 139 | "pycharm": { 140 | "name": "#%%\n" 141 | } 142 | } 143 | }, 144 | { 145 | "cell_type": "markdown", 146 | "source": [ 147 | "**Hybrid Analysis**" 148 | ], 149 | "metadata": { 150 | "collapsed": false 151 | } 152 | }, 153 | { 154 | "cell_type": "code", 155 | "execution_count": null, 156 | "outputs": [], 157 | "source": [ 158 | "from pyoti.multis import HybridAnalysis\n", 159 | "\n", 160 | "ha = HybridAnalysis(api_key=hybridanalysis)\n", 161 | "ha.url = phish_url\n", 162 | "url = ha.check_url()\n", 163 | "\n", 164 | "if url:\n", 165 | " print(f\"[+] Hybrid Analysis verdict: {url[0]['verdict']}\")\n", 166 | " print(f\"[+] Date of analysis: {url[0]['analysis_start_time']}\")\n", 167 | " print(f\"[*] Link to analysis: https://www.hybrid-analysis.com/sample/{url[0]['sha256']}\")\n", 168 | "else:\n", 169 | " print(f\"[*] No results for {defang(phish_url)}!\")" 170 | ], 171 | "metadata": { 172 | "collapsed": false, 173 | "pycharm": { 174 | "name": "#%%\n" 175 | } 176 | } 177 | }, 178 | { 179 | "cell_type": "markdown", 180 | "source": [ 181 | "**Phishtank**" 182 | ], 183 | "metadata": { 184 | "collapsed": false 185 | } 186 | }, 187 | { 188 | "cell_type": "code", 189 | "execution_count": null, 190 | "outputs": [], 191 | "source": [ 192 | "from pyoti.urls import Phishtank\n", 193 | "\n", 194 | "pt = Phishtank(api_key=phishtank)\n", 195 | "pt.url = phish_url\n", 196 | "url = pt.check_url()\n", 197 | "\n", 198 | "if url['in_database'] == 'true':\n", 199 | " print(f\"[+] Valid: {url['valid']}\")\n", 200 | " print(f\"[+] Verified: {url['verified']}\")\n", 201 | " print(f\"[+] Date Verified: {url['verified_at']}\")\n", 202 | " print(f\"[*] Link to analysis: {url['phish_detail_page']}\")\n", 203 | "else:\n", 204 | " print(f\"[*] No results for {defang(phish_url)}!\")" 205 | ], 206 | "metadata": { 207 | "collapsed": false, 208 | "pycharm": { 209 | "name": "#%%\n" 210 | } 211 | } 212 | }, 213 | { 214 | "cell_type": "markdown", 215 | "source": [ 216 | "**URLhaus**" 217 | ], 218 | "metadata": { 219 | "collapsed": false 220 | } 221 | }, 222 | { 223 | "cell_type": "code", 224 | "execution_count": null, 225 | "outputs": [], 226 | "source": [ 227 | "import pandas\n", 228 | "\n", 229 | "from pyoti.multis import URLhaus\n", 230 | "\n", 231 | "urlhaus = URLhaus()\n", 232 | "urlhaus.url = phish_url\n", 233 | "url = urlhaus.check_url()\n", 234 | "\n", 235 | "if url['query_status'] == 'ok':\n", 236 | " print(f\"[+] URL threat: {url['threat']}\")\n", 237 | " print(f\"[+] URL status: {url['url_status']}\")\n", 238 | " print(f\"[+] Date added: {url['date_added']}\")\n", 239 | " print(f\"[+] Tags: {[i for i in url['tags']]}\")\n", 240 | " print(\"[*] Payload delivery:\")\n", 241 | " data = url['payloads']\n", 242 | " df = pandas.DataFrame.from_dict(data)\n", 243 | " df_payloads = df[['firstseen', 'file_type', 'response_sha256', 'signature']]\n", 244 | " print(df_payloads.to_string())\n", 245 | " print(f\"[*] Link to analysis: {url['urlhaus_reference']}\")\n", 246 | "else:\n", 247 | " print(f\"[*] No results for {defang(phish_url)}!\")" 248 | ], 249 | "metadata": { 250 | "collapsed": false, 251 | "pycharm": { 252 | "name": "#%%\n" 253 | } 254 | } 255 | }, 256 | { 257 | "cell_type": "markdown", 258 | "source": [ 259 | "**URLscan**\n", 260 | "\n", 261 | "Search URLscan to see if URL has been submitted already" 262 | ], 263 | "metadata": { 264 | "collapsed": false 265 | } 266 | }, 267 | { 268 | "cell_type": "code", 269 | "execution_count": null, 270 | "outputs": [], 271 | "source": [ 272 | "from pyoti.multis import URLscan\n", 273 | "\n", 274 | "urls = URLscan(api_key=urlscan)\n", 275 | "urls.url = phish_url\n", 276 | "tasks = urls.search_url()\n", 277 | "\n", 278 | "count = 1\n", 279 | "for i in tasks['results']:\n", 280 | " print(f\"Result #{count}:\")\n", 281 | " print(f\"\\tIndexed at: {i['indexedAt']}\")\n", 282 | " print(f\"\\tTask UUID: {i['task']['uuid']}\")\n", 283 | " count += 1" 284 | ], 285 | "metadata": { 286 | "collapsed": false, 287 | "pycharm": { 288 | "name": "#%%\n" 289 | } 290 | } 291 | }, 292 | { 293 | "cell_type": "markdown", 294 | "source": [ 295 | "Copy the Task UUID and set the task_uuid variable to check URLscan for results" 296 | ], 297 | "metadata": { 298 | "collapsed": false 299 | } 300 | }, 301 | { 302 | "cell_type": "code", 303 | "execution_count": null, 304 | "outputs": [], 305 | "source": [ 306 | "task_uuid = 'bba8eac4-209a-40e2-b270-b3d8cc2d3e64'\n", 307 | "\n", 308 | "task_url = urls.check_url(uuid=task_uuid)\n", 309 | "print(f\"[+] URLscan Verdict score: {task_url['verdicts']['urlscan']['score']}\")\n", 310 | "if task_url['verdicts']['urlscan']['categories']:\n", 311 | " print(f\"[+] Categories: {task_url['verdicts']['urlscan']['categories']}\")\n", 312 | "if task_url['verdicts']['urlscan']['brands']:\n", 313 | " print(f\"[+] Brands: {task_url['verdicts']['urlscan']['brands']}\")\n", 314 | "if task_url['verdicts']['urlscan']['tags']:\n", 315 | " print(f\"[+] Tags: {task_url['verdicts']['urlscan']['tags']}\")\n", 316 | "print(f\"[+] Malicious: {task_url['verdicts']['urlscan']['malicious']}\")\n", 317 | "print(f\"[+] ASN: {task_url['page']['asn']}\")\n", 318 | "print(f\"[+] ASN Name: {task_url['page']['asnname']}\")\n", 319 | "print(f\"[+] Country: {task_url['page']['country']}\")\n", 320 | "print(f\"[+] Server: {task_url['page']['server']}\")\n", 321 | "print(f\"[+] IP: {task_url['page']['ip']}\")\n", 322 | "print(f\"[*] Link to analysis: {task_url['task']['reportURL']}\")" 323 | ], 324 | "metadata": { 325 | "collapsed": false, 326 | "pycharm": { 327 | "name": "#%%\n" 328 | } 329 | } 330 | }, 331 | { 332 | "cell_type": "markdown", 333 | "source": [ 334 | "**VirusTotal**" 335 | ], 336 | "metadata": { 337 | "collapsed": false 338 | } 339 | }, 340 | { 341 | "cell_type": "code", 342 | "execution_count": null, 343 | "outputs": [], 344 | "source": [ 345 | "from pyoti.multis import VirusTotalV2\n", 346 | "\n", 347 | "vt = VirusTotalV2(api_key=virustotal)\n", 348 | "vt.url = phish_url\n", 349 | "url = vt.check_url()\n", 350 | "\n", 351 | "if url['response_code'] == 1:\n", 352 | " print(f\"[+] Scan date: {url['scan_date']}\")\n", 353 | " print(f\"[+] Positives: {url['positives']}\")\n", 354 | " print(f\"[+] Total: {url['total']}\")\n", 355 | " print(f\"[*] Link to analysis: {url['permalink']}\")\n", 356 | "else:\n", 357 | " print(f\"[*] Verbose Message: {url['verbose_msg']}\")" 358 | ], 359 | "metadata": { 360 | "collapsed": false, 361 | "pycharm": { 362 | "name": "#%%\n" 363 | } 364 | } 365 | } 366 | ], 367 | "metadata": { 368 | "kernelspec": { 369 | "display_name": "Python 3", 370 | "language": "python", 371 | "name": "python3" 372 | }, 373 | "language_info": { 374 | "codemirror_mode": { 375 | "name": "ipython", 376 | "version": 2 377 | }, 378 | "file_extension": ".py", 379 | "mimetype": "text/x-python", 380 | "name": "python", 381 | "nbconvert_exporter": "python", 382 | "pygments_lexer": "ipython2", 383 | "version": "2.7.6" 384 | } 385 | }, 386 | "nbformat": 4, 387 | "nbformat_minor": 0 388 | } -------------------------------------------------------------------------------- /docs/misp/README.md: -------------------------------------------------------------------------------- 1 | ## PyOTI Taxonomy Library 2 | 3 | --- 4 | ### PyOTI automated enrichment schemes and definitions for point in time classification of indicators. 5 | pyoti namespace available in JSON format at this [location](https://github.com/MISP/misp-taxonomies/blob/main/pyoti/machinetag.json). The JSON format can be freely reused in your application or automatically enabled in [MISP](https://www.github.com/MISP/MISP) taxonomy. 6 | 7 | --- 8 | A machine tag is composed of a namespace, a predicate and a value. Machine tags are often called triple tag due to their format. 9 | 10 | - namespace:predicate=value 11 | 12 | --- 13 | ### checkdmarc 14 | #### pyoti:checkdmarc="spoofable" 15 | * #### Spoofable 16 | > The email address can be spoofed (e.g. no strict SPF policy/DMARC is not enforced). 17 | 18 | --- 19 | ### disposable-email 20 | #### pyoti:disposable-email 21 | > The email domain is from a disposable email service. 22 | 23 | --- 24 | ### emailrepio 25 | #### pyoti:emailrepio="spoofable" 26 | * #### Spoofable 27 | > The email address can be spoofed (e.g. no strict SPF policy/DMARC is not enforced). 28 | 29 | 30 | #### pyoti:emailrepio="suspicious" 31 | * #### Suspicious 32 | > The email address should be treated as suspicious or risky. 33 | 34 | #### pyoti:emailrepio="blacklisted" 35 | * #### Blacklisted 36 | > The email address is believed to be malicious or spammy. 37 | 38 | #### pyoti:emailrepio="malicious-activity" 39 | * #### Malicious Activity 40 | > The email address has exhibited malicious behavior (e.g. phishing/fraud). 41 | 42 | #### pyoti:emailrepio="malicious-activity-recent" 43 | * #### Malicious Activity Recent 44 | > The email address has exhibited malicious behavior in the last 90 days (e.g. in the case of temporal account takeovers). 45 | 46 | #### pyoti:emailrepio="credentials-leaked" 47 | * #### Credentials Leaked 48 | > The email address has had credentials leaked at some point in time (e.g. a data breach, pastebin, dark web, etc). 49 | 50 | #### pyoti:emailrepio="credentials-leaked-recent" 51 | * #### Credentials Leaked Recent 52 | > The email address has had credentials leaked in the last 90 days. 53 | 54 | #### pyoti:emailrepio="reputation-high" 55 | * #### Reputation High 56 | > The email address has a high reputation. 57 | 58 | #### pyoti:emailrepio="reputation-medium" 59 | * #### Reputation Medium 60 | > The email address has a medium reputation. 61 | 62 | #### pyoti:emailrepio="reputation-low" 63 | * #### Reputation Low 64 | > The email address has a low reputation. 65 | 66 | #### pyoti:emailrepio="suspicious-tld" 67 | * #### Suspicious TLD 68 | > The email address top-level domain is suspicious. 69 | 70 | #### pyoti:emailrepio="spam" 71 | * #### Spam 72 | > The email address has exhibited spammy behavior (e.g. spam traps, login form abuse, etc). 73 | 74 | --- 75 | ### iris-investigate 76 | #### pyoti:iris-investigate="high" 77 | * #### High 78 | > The domain risk score is high (76-100). 79 | 80 | #### pyoti:iris-investigate="medium-high" 81 | * #### Medium High 82 | > The domain risk score is medium-high (51-75). 83 | 84 | #### pyoti:iris-investigate="medium" 85 | * #### Medium 86 | > The domain risk score is medium (26-50). 87 | 88 | #### pyoti:iris-investigate="low" 89 | * #### Low 90 | > The domain risk score is low (0-25). 91 | 92 | --- 93 | ### virustotal 94 | #### pyoti:virustotal="known-distributor" 95 | * #### Known Distributor 96 | > The known-distributor entry indicates a file is from a known distributor. 97 | 98 | #### pyoti:virustotal="valid-signature" 99 | * #### Valid Signature 100 | > The valid-signature entry indicates a file is signed with a valid signature. 101 | 102 | #### pyoti:virustotal="invalid-signature" 103 | * #### Invalid Signature 104 | > The invalid-signature entry indicates a file is signed with an invalid signature. 105 | 106 | --- 107 | ### circl-hashlookup 108 | #### pyoti:circl-hashlookup="high-trust" 109 | * #### High Trust 110 | > The trust level is high (76-100). 111 | 112 | #### pyoti:circl-hashlookup="medium-high-trust" 113 | * #### Medium High Trust 114 | > The trust level is medium-high (51-75). 115 | 116 | #### pyoti:circl-hashlookup="medium-trust" 117 | * #### Medium Trust 118 | > The trust level is medium (26-50). 119 | 120 | #### pyoti:circl-hashlookup="low-trust" 121 | * #### Low Trust 122 | > The trust level is low (0-25). 123 | 124 | --- 125 | ### reputation-block-list 126 | #### pyoti:reputation-block-list="barracudacentral-brbl" 127 | * #### Barracuda Reputation Block List 128 | > Barracuda Reputation Block List (BRBL) is a free DNSBL of IP addresses known to send spam. Barracuda Networks fights spam and created the BRBL to help stop the spread of spam. 129 | 130 | #### pyoti:reputation-block-list="spamcop-scbl" 131 | * #### SpamCop Blocking List 132 | > The SpamCop Blocking List (SCBL) lists IP addresses which have transmitted reported email to SpamCop users. SpamCop, service providers and individual users then use the SCBL to block and filter unwanted email. 133 | 134 | #### pyoti:reputation-block-list="spamhaus-sbl" 135 | * #### Spamhaus Block List 136 | > The Spamhaus Block List (SBL) Advisory is a database of IP addresses from which Spamhaus does not recommend the acceptance of electronic mail. 137 | 138 | #### pyoti:reputation-block-list="spamhaus-xbl" 139 | * #### Spamhaus Exploits Block List 140 | > The Spamhaus Exploits Block List (XBL) is a realtime database of IP addresses of hijacked PCs infected by illegal 3rd party exploits, including open proxies (HTTP, socks, AnalogX, wingate, etc), worms/viruses with built-in spam engines, and other types of trojan-horse exploits. 141 | 142 | #### pyoti:reputation-block-list="spamhaus-pbl" 143 | * #### Spamhaus Policy Block List 144 | > The Spamhaus PBL is a DNSBL database of end-user IP address ranges which should not be delivering unauthenticated SMTP email to any Internet mail server except those provided for specifically by an ISP for that customer’s use. 145 | 146 | #### pyoti:reputation-block-list="spamhaus-css" 147 | * #### Spamhaus CSS 148 | > The Spamhaus CSS list is an automatically produced dataset of IP addresses that are involved in sending low-reputation email. CSS mostly targets static spam emitters that are not covered in the PBL or XBL, such as snowshoe spam operations, but may also include other senders that display a risk to our users, such as compromised hosts. 149 | 150 | #### pyoti:reputation-block-list="spamhaus-drop" 151 | * #### Spamhaus Don’t Route Or Peer 152 | > Spamhaus Don’t Route Or Peer (DROP) is an advisory 'drop all traffic' list. DROP is a tiny subset of the SBL which is designed for use by firewalls or routing equipment. 153 | 154 | #### pyoti:reputation-block-list="spamhaus-spam" 155 | * #### Spamhaus Domain Block List Spam Domain 156 | > Spamhaus Domain Block List (DBL) is a list of domain names with poor reputations used for spam. 157 | 158 | #### pyoti:reputation-block-list="spamhaus-phish" 159 | * #### Spamhaus Domain Block List Phish Domain 160 | > Spamhaus Domain Block List (DBL) is a list of domain names with poor reputations used for phishing. 161 | 162 | #### pyoti:reputation-block-list="spamhaus-malware" 163 | * #### Spamhaus Domain Block List Malware Domain 164 | > Spamhaus Domain Block List (DBL) is a list of domain names with poor reputations used to serve malware. 165 | 166 | #### pyoti:reputation-block-list="spamhaus-botnet-c2" 167 | * #### Spamhaus Domain Block List Botnet C2 Domain 168 | > Spamhaus Domain Block List (DBL) is a list of domain names with poor reputations used for botnet command and control. 169 | 170 | #### pyoti:reputation-block-list="spamhaus-abused-legit-spam" 171 | * #### Spamhaus Domain Block List Abused Legit Spam Domain 172 | > Spamhaus Domain Block List (DBL) is a list of abused legitimate domain names with poor reputations used for spam. 173 | 174 | #### pyoti:reputation-block-list="spamhaus-abused-spammed-redirector" 175 | * #### Spamhaus Domain Block List Abused Spammed Redirector Domain 176 | > Spamhaus Domain Block List (DBL) is a list of abused legitimate spammed domain names with poor reputations used as redirector domains. 177 | 178 | #### pyoti:reputation-block-list="spamhaus-abused-legit-phish" 179 | * #### Spamhaus Domain Block List Abused Legit Phish Domain 180 | > Spamhaus Domain Block List (DBL) is a list of abused legitimate domain names with poor reputations used for phishing. 181 | 182 | #### pyoti:reputation-block-list="spamhaus-abused-legit-malware" 183 | * #### Spamhaus Domain Block List Abused Legit Malware Domain 184 | > Spamhaus Domain Block List (DBL) is a list of abused legitimate domain names with poor reputations used to serve malware. 185 | 186 | #### pyoti:reputation-block-list="spamhaus-abused-legit-botnet-c2" 187 | * #### Spamhaus Domain Block List Abused Legit Botnet C2 Domain 188 | > Spamhaus Domain Block List (DBL) is a list of abused legitimate domain names with poor reputations used for botnet command and control. 189 | 190 | #### pyoti:reputation-block-list="surbl-phish" 191 | * #### SURBL Phishing Sites 192 | > Phishing data from multiple sources is included in this list. Data includes PhishTank, OITC, PhishLabs, Malware Domains and several other sources, including proprietary research by SURBL. 193 | 194 | #### pyoti:reputation-block-list="surbl-malware" 195 | * #### SURBL Malware Sites 196 | > This list contains data from multiple sources that cover sites hosting malware. This includes OITC, abuse.ch, The DNS blackhole malicious site data from malwaredomains.com and others. Malware data also includes significant proprietary research by SURBL. 197 | 198 | #### pyoti:reputation-block-list="surbl-spam" 199 | * #### SURBL Spam Sites 200 | > This list contains mainly general spam sites. It combines data from the formerly separate JP, WS, SC and AB lists. It also includes data from Internet security, anti-abuse, ISP, ESP and other communities, such as Telenor. Most of the data in this list comes from internal, proprietary research by SURBL. 201 | 202 | #### pyoti:reputation-block-list="surbl-abused-legit" 203 | * #### SURBL Abused Legit Sites 204 | > This list contains data from multiple sources that cover cracked sites, including SURBL internal ones. Criminals steal credentials or abuse vulnerabilities to break into websites and add malicious content. Often cracked pages will redirect to spam sites or to other cracked sites. Cracked sites usually still contain the original legitimate content and may still be mentioned in legitimate emails, besides the malicious pages referenced in spam. 205 | 206 | #### pyoti:reputation-block-list="uribl-black" 207 | * #### URIBL Black 208 | > URIBL Black list contains domain names belonging to and used by spammers, including but not restricted to those that appear in URIs found in Unsolicited Bulk and/or Commercial Email (UBE/UCE). This list has a goal of zero False Positives. 209 | 210 | #### pyoti:reputation-block-list="uribl-grey" 211 | * #### URIBL Grey 212 | > URIBL Grey list contains domains found in UBE/UCE, and possibly honour opt-out requests. It may include ESPs which allow customers to import their recipient lists and may have no control over the subscription methods. This list can and probably will cause False Positives depending on your definition of UBE/UCE. 213 | 214 | #### pyoti:reputation-block-list="uribl-red" 215 | * #### URIBL Red 216 | > URIBL Red list contains domains that actively show up in mail flow, are not listed on URIBL black, and are either: being monitored, very young (domain age via whois), or use whois privacy features to protect their identity. This list is automated in nature, so please use at your own risk. 217 | 218 | #### pyoti:reputation-block-list="uribl-multi" 219 | * #### URIBL Multi 220 | > URIBL Multi list contains all of the public URIBL lists. 221 | 222 | --- 223 | ### abuseipdb 224 | #### pyoti:abuseipdb="high" 225 | * #### High 226 | > The IP abuse confidence score is high (76-100). 227 | 228 | #### pyoti:abuseipdb="medium-high" 229 | * #### Medium High 230 | > The IP abuse confidence score is medium-high (51-75). 231 | 232 | #### pyoti:abuseipdb="medium" 233 | * #### Medium 234 | > The IP abuse confidence score is medium (26-50). 235 | 236 | #### pyoti:abuseipdb="low" 237 | * #### Low 238 | > The IP abuse confidence score is low (0-25). 239 | 240 | --- 241 | ### greynoise-riot 242 | #### pyoti:greynoise-riot="trust-level-1" 243 | * #### Trust Level 1 244 | > These IPs are trustworthy because the companies or services assigned are generally responsible for the interactions with this IP. Adding these ranges to an allow-list may make sense. 245 | 246 | #### pyoti:greynoise-riot="trust-level-2" 247 | * #### Trust Level 2 248 | > These IPs are somewhat trustworthy because they are necessary for regular and common business internet use. Companies that own these IPs typically do not claim responsibility or have accountability for interactions with these IPs. Malicious actions may be associated with these IPs but adding this entire range to a block-list does not make sense. 249 | 250 | --- 251 | ### googlesafebrowsing 252 | #### pyoti:googlesafebrowsing="malware" 253 | * #### MALWARE 254 | > Malware threat type. 255 | 256 | #### pyoti:googlesafebrowsing="social-engineering" 257 | * #### SOCIAL_ENGINEERING 258 | > Social engineering threat type. 259 | 260 | #### pyoti:googlesafebrowsing="unwanted-software" 261 | * #### UNWANTED_SOFTWARE 262 | > Unwanted software threat type. 263 | 264 | #### pyoti:googlesafebrowsing="potentially-harmful-application" 265 | * #### POTENTIALLY_HARMFUL_APPLICATION 266 | > Potentially harmful application threat type. 267 | 268 | #### pyoti:googlesafebrowsing="unspecified" 269 | * #### THREAT_TYPE_UNSPECIFIED 270 | > Unknown threat type. -------------------------------------------------------------------------------- /examples/enrich_misp_event.py: -------------------------------------------------------------------------------- 1 | from argparse import ArgumentParser 2 | from pymisp import ExpandedPyMISP, MISPAttribute, MISPObjectAttribute 3 | from typing import Dict, List, Union 4 | 5 | from pyoti.domains import CheckDMARC, IrisInvestigate 6 | from pyoti.emails import DisposableEmails, EmailRepIO 7 | from pyoti.hashes import CIRCLHashLookup 8 | from pyoti.ips import AbuseIPDB, GreyNoise 9 | from pyoti.multis import DNSBlockList, VirusTotalV3 10 | from pyoti.urls import GoogleSafeBrowsing 11 | 12 | from keys import abuseipdb, domaintools, googlesafebrowsing, greynoise, misp, sublime, virustotal 13 | 14 | 15 | def enrich_hashes(file_hash: str) -> Dict: 16 | enrichment = {} 17 | 18 | hl = CIRCLHashLookup() 19 | hl.file_hash = file_hash 20 | enrichment['hashlookup'] = hl.check_hash() 21 | 22 | vt = VirusTotalV3(api_key=virustotal) 23 | vt.file_hash = file_hash 24 | enrichment['virustotal'] = vt.check_hash() 25 | 26 | return enrichment 27 | 28 | 29 | def get_hashlookup_tags(hltrust: int) -> str: 30 | lt = 'pyoti:circl-hashlookup="low-trust"' 31 | mt = 'pyoti:circl-hashlookup="medium-trust"' 32 | mht = 'pyoti:circl-hashlookup="medium-high-trust"' 33 | ht = 'pyoti:circl-hashlookup="high-trust"' 34 | 35 | if hltrust <= 25: 36 | return lt 37 | elif 25 < hltrust <= 50: 38 | return mt 39 | elif 50 < hltrust <= 75: 40 | return mht 41 | elif hltrust > 75: 42 | return ht 43 | 44 | 45 | def enrich_domains(domain: str) -> Dict: 46 | enrichment = {} 47 | 48 | iris = IrisInvestigate(api_key=domaintools) 49 | iris.domain = domain 50 | i_domain = iris.check_domain() 51 | enrichment['iris'] = i_domain 52 | 53 | dbl = DNSBlockList() 54 | dbl.domain = domain 55 | enrichment['dbl'] = dbl.check_domain() 56 | 57 | return enrichment 58 | 59 | 60 | def get_domainrisk_tags(risk_score: int) -> str: 61 | hr = 'pyoti:iris-investigate="high"' 62 | mhr = 'pyoti:iris-investigate="medium-high"' 63 | mr = 'pyoti:iris-investigate="medium"' 64 | lr = 'pyoti:iris-investigate="low"' 65 | 66 | if risk_score <= 25: 67 | return lr 68 | elif 25 < risk_score <= 50: 69 | return mr 70 | elif 50 < risk_score <= 75: 71 | return mhr 72 | elif risk_score > 75: 73 | return hr 74 | 75 | 76 | def enrich_emails(email: str) -> Dict: 77 | enrichment = {} 78 | domain = email.split("@")[1] 79 | 80 | dmarc = CheckDMARC() 81 | dmarc.domain = domain 82 | enrichment['checkdmarc'] = dmarc.check_domain() 83 | 84 | disposable = DisposableEmails() 85 | disposable.email = email 86 | enrichment['disposable'] = disposable.check_email() 87 | 88 | erep = EmailRepIO(api_key=sublime) 89 | erep.email = email 90 | enrichment['emailrep'] = erep.check_email() 91 | 92 | return enrichment 93 | 94 | 95 | def get_emailrep_tags(reputation: str) -> str: 96 | hr = 'pyoti:emailrepio="reputation-high"' 97 | mr = 'pyoti:emailrepio="reputation-medium"' 98 | lr = 'pyoti:emailrepio="reputation-low"' 99 | 100 | if reputation == "high": 101 | return hr 102 | elif reputation == "medium": 103 | return mr 104 | elif reputation == "low": 105 | return lr 106 | 107 | 108 | def enrich_ips(ip: str) -> Dict: 109 | enrichment = {} 110 | 111 | abuse = AbuseIPDB(api_key=abuseipdb) 112 | abuse.ip = ip 113 | enrichment['abuseipdb'] = abuse.check_ip() 114 | 115 | gn = GreyNoise(api_key=greynoise) 116 | gn.ip = ip 117 | enrichment['greynoise'] = gn.check_ip_riot() 118 | 119 | rbl = DNSBlockList() 120 | rbl.ip = ip 121 | enrichment['rbl'] = rbl.check_ip() 122 | 123 | return enrichment 124 | 125 | 126 | def get_abuseipdb_tags(abuse_score: int) -> str: 127 | ha = 'pyoti:abuseipdb="high"' 128 | mha = 'pyoti:abuseipdb="medium-high"' 129 | ma = 'pyoti:abuseipdb="medium"' 130 | la = 'pyoti:abuseipdb="low"' 131 | 132 | if abuse_score <= 25: 133 | return la 134 | elif 25 < abuse_score <= 50: 135 | return ma 136 | elif 50 < abuse_score <= 75: 137 | return mha 138 | elif abuse_score > 75: 139 | return ha 140 | 141 | 142 | def enrich_urls(url: str) -> Dict: 143 | enrichment = {} 144 | 145 | gsb = GoogleSafeBrowsing(api_key=googlesafebrowsing) 146 | gsb.url = url 147 | g_url = gsb.check_url() 148 | enrichment['google'] = g_url 149 | 150 | return enrichment 151 | 152 | 153 | def get_gsb_tags(threat_type: List[str]) -> List[str]: 154 | mal = 'pyoti:googlesafebrowsing="malware"' 155 | se = 'pyoti:googlesafebrowsing="social-engineering"' 156 | us = 'pyoti:googlesafebrowsing="unwanted-software"' 157 | pha = 'pyoti:googlesafebrowsing="potentially-harmful-application"' 158 | un = 'pyoti:googlesafebrowsing="unspecified"' 159 | 160 | for threat in threat_type: 161 | if threat == "MALWARE": 162 | yield mal 163 | elif threat == "SOCIAL_ENGINEERING": 164 | yield se 165 | elif threat == "UNWANTED_SOFTWARE": 166 | yield us 167 | elif threat == "POTENTIALLY_HARMFUL_APPLICATION": 168 | yield pha 169 | elif threat == "THREAT_TYPE_UNSPECIFIED": 170 | yield un 171 | 172 | 173 | def run_enrichment(attributes: Union[List[MISPAttribute], List[MISPObjectAttribute]]): 174 | for attr in attributes: 175 | # do PyOTI hash enrichment 176 | if attr.type == "md5" or attr.type == "sha1" or attr.type == "sha256": 177 | if attr.value in processed_iocs: 178 | # apply tags from attribute that has already been checked and enriched 179 | [attr.add_tag(tag) for tag in processed_iocs[attr.value]] 180 | continue 181 | processed_iocs[attr.value] = [] 182 | h_enrichment = enrich_hashes(attr.value) 183 | 184 | # get hashlookup trust level and apply pyoti taxonomy tag 185 | hltrust = h_enrichment['hashlookup'].get('hashlookup:trust') 186 | if hltrust: 187 | hl_tag = get_hashlookup_tags(hltrust) 188 | processed_iocs[attr.value].append(hl_tag) 189 | attr.add_tag(hl_tag) 190 | 191 | if h_enrichment['virustotal'].get('error'): 192 | # looking for file not found error and continuing to next attribute 193 | continue 194 | else: 195 | # get virstotal known software distributor and apply pyoti taxonomy tag 196 | vt_known = h_enrichment['virustotal'].get('data').get('attributes').get('known_distributors') 197 | if vt_known: 198 | vt_known_tag = 'pyoti:virustotal="known-distributor"' 199 | processed_iocs[attr.value].append(vt_known_tag) 200 | attr.add_tag(vt_known_tag) 201 | 202 | # get virstotal file signature info and apply pyoti taxonomy tag 203 | vt_sig = h_enrichment['virustotal'].get('data').get('attributes').get('signature_info') 204 | if vt_sig: 205 | vt_sig_tag = 'pyoti:virustotal="valid-signature"' 206 | processed_iocs[attr.value].append(vt_sig_tag) 207 | attr.add_tag(vt_sig_tag) 208 | 209 | # get virustotal threat classification info and apply pyoti taxonomy tag 210 | vt_tc = h_enrichment['virustotal'].get('data').get('attributes').get('popular_threat_classification') 211 | if vt_tc: 212 | vt_threat_label = vt_tc.get('suggested_threat_label') 213 | processed_iocs[attr.value].append(vt_threat_label) 214 | attr.add_tag(vt_threat_label) 215 | 216 | elif attr.type == "domain" or attr.type == "hostname": 217 | # do PyOTI domain enrichment 218 | if attr.value in processed_iocs: 219 | # apply tags from attribute that has already been checked and enriched 220 | [attr.add_tag(tag) for tag in processed_iocs[attr.value]] 221 | continue 222 | processed_iocs[attr.value] = [] 223 | d_enrichment = enrich_domains(attr.value) 224 | 225 | # get iris-investigate domain risk score and apply pyoti taxonomy tag 226 | risk_score = d_enrichment['iris'][0].get('domain_risk').get('risk_score') 227 | iris_tag = get_domainrisk_tags(risk_score) 228 | processed_iocs[attr.value].append(iris_tag) 229 | attr.add_tag(iris_tag) 230 | 231 | # check if domain is on dns block lists 232 | dbl_tags = [x.get('blocklist') for x in d_enrichment['dbl']] 233 | if dbl_tags: 234 | rep_bl = 'pyoti:reputation-block-list=' 235 | [processed_iocs[attr.value].append(f'{rep_bl}"{tag}"') for tag in dbl_tags if tag is not None] 236 | [attr.add_tag(f'{rep_bl}"{tag}"') for tag in dbl_tags if tag is not None] 237 | 238 | elif attr.type == "email-src": 239 | # do PyOTI email enrichment 240 | if attr.value in processed_iocs: 241 | # apply tags from attribute that has already been checked and enriched 242 | [attr.add_tag(tag) for tag in processed_iocs[attr.value]] 243 | continue 244 | processed_iocs[attr.value] = [] 245 | e_enrichment = enrich_emails(attr.value) 246 | 247 | # check if email address domain is spoofable and apply pyoti taxonomy tag 248 | d_spoofable = e_enrichment['checkdmarc'].get('spoofable') 249 | if d_spoofable: 250 | dmarc_tag = 'pyoti:checkdmarc="spoofable"' 251 | processed_iocs[attr.value].append(dmarc_tag) 252 | attr.add_tag(dmarc_tag) 253 | 254 | # check if email address is disposable 255 | disposable = e_enrichment['disposable'].get('disposable') 256 | if disposable: 257 | dis_tag = 'pyoti:disposable-email' 258 | processed_iocs[attr.value].append(dis_tag) 259 | attr.add_tag(dis_tag) 260 | 261 | # get emailrep.io email address reputation 262 | e_reputation = e_enrichment['emailrep'].get('reputation') 263 | if e_reputation != "none": 264 | rep_tag = get_emailrep_tags(e_reputation) 265 | processed_iocs[attr.value].append(rep_tag) 266 | attr.add_tag(rep_tag) 267 | 268 | # check emailrep.io if email address is suspicious 269 | e_sus = e_enrichment['emailrep'].get('suspicious') 270 | if e_sus: 271 | sus_tag = 'pyoti:emailrepio="suspicious"' 272 | processed_iocs[attr.value].append(sus_tag) 273 | attr.add_tag(sus_tag) 274 | 275 | # check emailrep.io for recent malicious activity 276 | e_mal = e_enrichment['emailrep'].get('details').get('malicious_activity_recent') 277 | if e_mal: 278 | mal_tag = 'pyoti:emailrepio="malicious-activity-recent"' 279 | processed_iocs[attr.value].append(mal_tag) 280 | attr.add_tag(mal_tag) 281 | 282 | # check emailrep.io for recent credential leak 283 | e_creds = e_enrichment['emailrep'].get('details').get('credentials_leaked_recent') 284 | if e_creds: 285 | creds_tag = 'pyoti:emailrepio="credentials-leaked-recent"' 286 | processed_iocs[attr.value].append(creds_tag) 287 | attr.add_tag(creds_tag) 288 | 289 | # check emailrep.io if email address is blacklisted 290 | e_bl = e_enrichment['emailrep'].get('details').get('blacklisted') 291 | if e_bl: 292 | bl_tag = 'pyoti:emailrepio="blacklisted"' 293 | processed_iocs[attr.value].append(bl_tag) 294 | attr.add_tag(bl_tag) 295 | 296 | # check emailrep.io if email address is spammy 297 | e_spam = e_enrichment['emailrep'].get('details').get('spam') 298 | if e_spam: 299 | spam_tag = 'pyoti:emailrepio="spam"' 300 | processed_iocs[attr.value].append(spam_tag) 301 | attr.add_tag(spam_tag) 302 | 303 | # check emailrep.io if email address has suspicious tld 304 | e_tld = e_enrichment['emailrep'].get('details').get('suspicious_tld') 305 | if e_tld: 306 | tld_tag = 'pyoti:emailrepio="suspicious-tld"' 307 | processed_iocs[attr.value].append(tld_tag) 308 | attr.add_tag(tld_tag) 309 | 310 | elif attr.type == "ip-src" or attr.type == "ip-dst": 311 | # do PyOTI ip enrichment 312 | if attr.value in processed_iocs: 313 | # apply tags from attribute that has already been checked and enriched 314 | [attr.add_tag(tag) for tag in processed_iocs[attr.value]] 315 | continue 316 | processed_iocs[attr.value] = [] 317 | i_enrichment = enrich_ips(attr.value) 318 | 319 | # check abuseipdb for abuse score 320 | abuse_confidence = i_enrichment['abuseipdb'].get('data').get('abuseConfidenceScore') 321 | abuse_tag = get_abuseipdb_tags(abuse_confidence) 322 | processed_iocs[attr.value].append(abuse_tag) 323 | attr.add_tag(abuse_tag) 324 | 325 | # check greynoise riot for ip trust level 326 | trust_level = i_enrichment['greynoise'].get('trust_level') 327 | tl_1 = 'pyoti:greynoise-riot="trust-level-1"' 328 | tl_2 = 'pyoti:greynoise-riot="trust-level-2"' 329 | if trust_level == '1': 330 | processed_iocs[attr.value].append(tl_1) 331 | attr.add_tag(tl_1) 332 | elif trust_level == '2': 333 | processed_iocs[attr.value].append(tl_2) 334 | attr.add_tag(tl_2) 335 | 336 | # check if ip address is on reputation block lists 337 | rbl_tags = [x.get('blocklist') for x in i_enrichment['rbl']] 338 | if rbl_tags: 339 | rep_bl = 'pyoti:reputation-block-list=' 340 | [processed_iocs[attr.value].append(f'{rep_bl}"{tag}"') for tag in rbl_tags if tag is not None] 341 | [attr.add_tag(f'{rep_bl}"{tag}"') for tag in rbl_tags if tag is not None] 342 | 343 | elif attr.type == "url": 344 | # do PyOTI url enrichment 345 | if attr.value in processed_iocs: 346 | # apply tags from attribute that has already been checked and enriched 347 | [attr.add_tag(tag) for tag in processed_iocs[attr.value]] 348 | continue 349 | processed_iocs[attr.value] = [] 350 | u_enrichment = enrich_urls(attr.value) 351 | 352 | # check google safe browsing for url threat 353 | g_threat = [x['threatType'] for x in u_enrichment['google'].get('matches')] 354 | g_tags = get_gsb_tags(g_threat) 355 | if g_tags: 356 | [processed_iocs[attr.value].append(tag) for tag in g_tags] 357 | [attr.add_tag(tag) for tag in g_tags] 358 | 359 | 360 | def main(): 361 | parser = ArgumentParser( 362 | prog="Automated MISP Event Enrichment", 363 | description="This script will use PyOTI modules to run automated enrichment on all attributes attached to a " 364 | "MISP Event and/or attributes attached to MISP Object(s) within a MISP Event and add appropriate " 365 | "PyOTI MISP Taxonomy tags. " 366 | ) 367 | parser.add_argument( 368 | "-u", 369 | "--url", 370 | dest="url", 371 | required=True, 372 | help="MISP URL", 373 | type=str 374 | ) 375 | parser.add_argument( 376 | "-s", 377 | "--ssl", 378 | dest="ssl", 379 | required=False, 380 | help="Verify SSL certificate", 381 | type=bool, 382 | nargs="?", 383 | default=True 384 | ) 385 | parser.add_argument( 386 | "-e", 387 | "--event-id", 388 | dest="event_id", 389 | required=True, 390 | help="MISP Event ID", 391 | type=int 392 | ) 393 | parser.add_argument( 394 | "-p", 395 | "--publish", 396 | dest="publish", 397 | required=False, 398 | help="Publish MISP Event", 399 | type=bool, 400 | nargs="?", 401 | default=False 402 | ) 403 | args = parser.parse_args() 404 | 405 | m = ExpandedPyMISP(url=args.url, key=misp, ssl=args.ssl) 406 | 407 | event = m.get_event(args.event_id, pythonify=True) 408 | 409 | attrs = event.attributes 410 | 411 | objects = [o.attributes for o in event.objects] 412 | 413 | global processed_iocs 414 | # use this dict to track processed indicators to ensure we don't query APIs multiple times for the same indicator 415 | processed_iocs = {} 416 | 417 | if attrs: 418 | print(f"[*] Found {len(attrs)} attributes in MISP Event: {event.id}. Running enrichment...") 419 | run_enrichment(attributes=event.attributes) 420 | 421 | if objects: 422 | print(f"[*] Found {len(objects)} objects attached to MISP Event: {event.id}.") 423 | for object_attr in objects: 424 | print(f"[*] Found {len(object_attr)} attributes attached to MISP Object. Running enrichment...") 425 | run_enrichment(attributes=object_attr) 426 | 427 | m.update_event(event) 428 | print(f"[!] Enrichment complete! Updated MISP Event ID: {event.id}!") 429 | 430 | if args.publish: 431 | m.publish(args.event_id) 432 | print(f"[!] Published MISP Event ID: {event.id}!") 433 | 434 | 435 | if __name__ == "__main__": 436 | main() 437 | --------------------------------------------------------------------------------