├── tests ├── __init__.py └── test_shodan.py ├── shodan ├── cli │ ├── __init__.py │ ├── converter │ │ ├── base.py │ │ ├── __init__.py │ │ ├── geojson.py │ │ ├── images.py │ │ ├── csvc.py │ │ ├── kml.py │ │ └── excel.py │ ├── settings.py │ ├── organization.py │ ├── data.py │ ├── helpers.py │ ├── host.py │ ├── worldmap.py │ ├── scan.py │ └── alert.py ├── __init__.py ├── exception.py ├── threatnet.py ├── helpers.py ├── stream.py ├── client.py └── __main__.py ├── AUTHORS ├── requirements.txt ├── MANIFEST.in ├── .gitignore ├── tox.ini ├── docs ├── api.rst ├── examples │ ├── basic-search.rst │ ├── cert-stream.rst │ ├── query-summary.rst │ └── gifcreator.rst ├── index.rst ├── tutorial.rst ├── Makefile ├── make.bat └── conf.py ├── LICENSE ├── setup.py ├── README.rst └── CHANGELOG.md /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /shodan/cli/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Primary authors: 2 | 3 | * John Matherly 4 | -------------------------------------------------------------------------------- /shodan/__init__.py: -------------------------------------------------------------------------------- 1 | from shodan.client import Shodan 2 | from shodan.exception import APIError 3 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | click 2 | click-plugins 3 | colorama 4 | requests>=2.2.1 5 | XlsxWriter 6 | ipaddress;python_version<='2.7' 7 | tldextract -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include AUTHORS 2 | include LICENSE 3 | include requirements.txt 4 | include CHANGELOG.md 5 | graft docs 6 | recursive-include shodan *.py 7 | -------------------------------------------------------------------------------- /shodan/cli/converter/base.py: -------------------------------------------------------------------------------- 1 | 2 | class Converter: 3 | 4 | def __init__(self, fout): 5 | self.fout = fout 6 | 7 | def process(self, fout): 8 | pass 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build/* 2 | *.tar.gz 3 | *.json.gz 4 | *.kml 5 | *.egg 6 | *.pyc 7 | shodan.egg-info/* 8 | tmp/* 9 | MANIFEST 10 | .vscode/ 11 | PKG-INFO 12 | venv/* 13 | .idea/* -------------------------------------------------------------------------------- /shodan/cli/converter/__init__.py: -------------------------------------------------------------------------------- 1 | from .csvc import CsvConverter 2 | from .excel import ExcelConverter 3 | from .geojson import GeoJsonConverter 4 | from .images import ImagesConverter 5 | from .kml import KmlConverter 6 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = 3 | E501 W293 4 | 5 | exclude = 6 | build, 7 | docs, 8 | shodan.egg-info, 9 | tmp, 10 | 11 | per-file-ignores = 12 | shodan/__init__.py:F401, 13 | shodan/cli/converter/__init__.py:F401, 14 | shodan/cli/worldmap.py:W291, -------------------------------------------------------------------------------- /shodan/exception.py: -------------------------------------------------------------------------------- 1 | class APIError(Exception): 2 | """This exception gets raised whenever a non-200 status code was returned by the Shodan API.""" 3 | def __init__(self, value): 4 | self.value = value 5 | 6 | def __str__(self): 7 | return self.value 8 | 9 | 10 | class APITimeout(APIError): 11 | pass 12 | -------------------------------------------------------------------------------- /docs/api.rst: -------------------------------------------------------------------------------- 1 | .. _api: 2 | 3 | shodan 4 | ====== 5 | 6 | .. module:: shodan 7 | 8 | .. autoclass:: Shodan 9 | :inherited-members: 10 | 11 | .. autoclass:: shodan::Shodan.Exploits 12 | :inherited-members: 13 | 14 | .. autoclass:: shodan::Shodan.Stream 15 | :inherited-members: 16 | 17 | Exceptions 18 | ~~~~~~~~~~ 19 | 20 | .. autoexception:: shodan.APIError -------------------------------------------------------------------------------- /shodan/cli/settings.py: -------------------------------------------------------------------------------- 1 | 2 | from os import path 3 | 4 | if path.exists(path.expanduser("~/.shodan")): 5 | SHODAN_CONFIG_DIR = '~/.shodan/' 6 | else: 7 | SHODAN_CONFIG_DIR = "~/.config/shodan/" 8 | 9 | COLORIZE_FIELDS = { 10 | 'ip_str': 'green', 11 | 'port': 'yellow', 12 | 'data': 'white', 13 | 'hostnames': 'magenta', 14 | 'org': 'cyan', 15 | 'vulns': 'red', 16 | } 17 | -------------------------------------------------------------------------------- /docs/examples/basic-search.rst: -------------------------------------------------------------------------------- 1 | Basic Shodan Search 2 | ------------------- 3 | 4 | .. code-block:: python 5 | 6 | #!/usr/bin/env python 7 | # 8 | # shodan_ips.py 9 | # Search SHODAN and print a list of IPs matching the query 10 | # 11 | # Author: achillean 12 | 13 | import shodan 14 | import sys 15 | 16 | # Configuration 17 | API_KEY = "YOUR_API_KEY" 18 | 19 | # Input validation 20 | if len(sys.argv) == 1: 21 | print 'Usage: %s ' % sys.argv[0] 22 | sys.exit(1) 23 | 24 | try: 25 | # Setup the api 26 | api = shodan.Shodan(API_KEY) 27 | 28 | # Perform the search 29 | query = ' '.join(sys.argv[1:]) 30 | result = api.search(query) 31 | 32 | # Loop through the matches and print each IP 33 | for service in result['matches']: 34 | print service['ip_str'] 35 | except Exception as e: 36 | print 'Error: %s' % e 37 | sys.exit(1) -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. shodan-python documentation master file, created by 2 | sphinx-quickstart on Thu Jan 23 00:56:29 2014. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | shodan - The official Python library for the Shodan search engine 7 | ================================================================= 8 | 9 | This is the official Python wrapper around both the Shodan REST API as well as the experimental 10 | Streaming API. And as a bonus it also lets you search for exploits using the Shodan Exploits REST API. 11 | If you're not sure where to start simply go through the "Getting Started" section of the documentation and work your 12 | way down through the examples. 13 | 14 | For more information about Shodan and how to use the API please visit our official help center at: 15 | 16 | https://help.shodan.io 17 | 18 | Introduction 19 | ~~~~~~~~~~~~ 20 | .. toctree:: 21 | :maxdepth: 2 22 | 23 | tutorial 24 | 25 | Examples 26 | ~~~~~~~~ 27 | .. toctree:: 28 | :maxdepth: 2 29 | 30 | examples/basic-search 31 | examples/query-summary 32 | examples/cert-stream 33 | examples/gifcreator 34 | 35 | API Reference 36 | ~~~~~~~~~~~~~ 37 | .. toctree:: 38 | :maxdepth: 2 39 | 40 | api 41 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014- John Matherly 2 | 3 | Permission is hereby granted, free of charge, to any person 4 | obtaining a copy of this software and associated documentation 5 | files (the "Software"), to deal in the Software without 6 | restriction, including without limitation the rights to use, 7 | copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the 9 | Software is furnished to do so, subject to the following 10 | conditions: 11 | 12 | The above copyright notice and this permission notice shall be 13 | included in all copies or substantial portions of the Software. 14 | 15 | Except as contained in this notice, the name(s) of the above 16 | copyright holders shall not be used in advertising or otherwise 17 | to promote the sale, use or other dealings in this Software 18 | without prior written authorization. 19 | 20 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 21 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 22 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 23 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 24 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 25 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 26 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 27 | OTHER DEALINGS IN THE SOFTWARE. 28 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from setuptools import setup 4 | 5 | 6 | DEPENDENCIES = open('requirements.txt', 'r').read().split('\n') 7 | README = open('README.rst', 'r').read() 8 | 9 | 10 | setup( 11 | name='shodan', 12 | version='1.31.0', 13 | description='Python library and command-line utility for Shodan (https://developer.shodan.io)', 14 | long_description=README, 15 | long_description_content_type='text/x-rst', 16 | author='John Matherly', 17 | author_email='jmath@shodan.io', 18 | url='https://github.com/achillean/shodan-python', 19 | packages=['shodan', 'shodan.cli', 'shodan.cli.converter'], 20 | entry_points={'console_scripts': ['shodan=shodan.__main__:main']}, 21 | install_requires=DEPENDENCIES, 22 | keywords=['security', 'network'], 23 | classifiers=[ 24 | 'Development Status :: 5 - Production/Stable', 25 | 'Intended Audience :: Developers', 26 | 'License :: OSI Approved :: MIT License', 27 | 'Natural Language :: English', 28 | 'Operating System :: OS Independent', 29 | 'Programming Language :: Python', 30 | 'Programming Language :: Python :: 2', 31 | 'Programming Language :: Python :: 2.6', 32 | 'Programming Language :: Python :: 2.7', 33 | 'Programming Language :: Python :: 3', 34 | 'Programming Language :: Python :: 3.4', 35 | 'Programming Language :: Python :: 3.5', 36 | 'Programming Language :: Python :: 3.6', 37 | 'Topic :: Software Development :: Libraries :: Python Modules', 38 | ], 39 | ) 40 | -------------------------------------------------------------------------------- /shodan/cli/converter/geojson.py: -------------------------------------------------------------------------------- 1 | from json import dumps 2 | from .base import Converter 3 | from ...helpers import get_ip, iterate_files 4 | 5 | 6 | class GeoJsonConverter(Converter): 7 | 8 | def header(self): 9 | self.fout.write("""{ 10 | "type": "FeatureCollection", 11 | "features": [ 12 | """) 13 | 14 | def footer(self): 15 | self.fout.write("""{ }]}""") 16 | 17 | def process(self, files, file_size): 18 | # Write the header 19 | self.header() 20 | 21 | # We only want to generate 1 datapoint for each IP - not per service 22 | unique_hosts = set() 23 | for banner in iterate_files(files): 24 | ip = get_ip(banner) 25 | if not ip: 26 | continue 27 | 28 | if ip not in unique_hosts: 29 | self.write(ip, banner) 30 | unique_hosts.add(ip) 31 | 32 | self.footer() 33 | 34 | def write(self, ip, host): 35 | try: 36 | lat, lon = host['location']['latitude'], host['location']['longitude'] 37 | feature = { 38 | 'type': 'Feature', 39 | 'id': ip, 40 | 'properties': { 41 | 'name': ip, 42 | 'lat': lat, 43 | 'lon': lon, 44 | }, 45 | 'geometry': { 46 | 'type': 'Point', 47 | 'coordinates': [lon, lat], 48 | }, 49 | } 50 | self.fout.write(dumps(feature) + ',') 51 | except Exception: 52 | pass 53 | -------------------------------------------------------------------------------- /docs/examples/cert-stream.rst: -------------------------------------------------------------------------------- 1 | Access SSL certificates in Real-Time 2 | ------------------------------------ 3 | 4 | The new Shodan Streaming API provides real-time access to the information that Shodan is gathering at the moment. 5 | Using the Streaming API, you get the raw access to potentially all the data that ends up in the Shodan search engine. 6 | Note that you can't search with the Streaming API or perform any other operations that you're accustomed to with 7 | the REST API. This is meant for large-scale consumption of real-time data. 8 | 9 | This script only works with people that have a subscription API plan! 10 | And by default the Streaming API only returns 1% of the data that Shodan gathers. 11 | If you wish to have more access please contact us at support@shodan.io for pricing 12 | information. 13 | 14 | .. code-block:: python 15 | 16 | #!/usr/bin/env python 17 | # 18 | # cert-stream.py 19 | # Stream the SSL certificates that Shodan is collecting at the moment 20 | # 21 | # WARNING: This script only works with people that have a subscription API plan! 22 | # And by default the Streaming API only returns 1% of the data that Shodan gathers. 23 | # If you wish to have more access please contact us at sales@shodan.io for pricing 24 | # information. 25 | # 26 | # Author: achillean 27 | import shodan 28 | import sys 29 | 30 | # Configuration 31 | API_KEY = 'YOUR API KEY' 32 | 33 | try: 34 | # Setup the api 35 | api = shodan.Shodan(API_KEY) 36 | 37 | print('Listening for certs...') 38 | for banner in api.stream.ports([443, 8443]): 39 | if 'ssl' in banner: 40 | # Print out all the SSL information that Shodan has collected 41 | print(banner['ssl']) 42 | 43 | except Exception as e: 44 | print('Error: {}'.format(e)) 45 | sys.exit(1) 46 | -------------------------------------------------------------------------------- /shodan/cli/converter/images.py: -------------------------------------------------------------------------------- 1 | 2 | from .base import Converter 3 | from ...helpers import iterate_files, get_ip, get_screenshot 4 | 5 | # Needed for decoding base64-strings in Python3 6 | from codecs import decode 7 | 8 | import os 9 | 10 | 11 | class ImagesConverter(Converter): 12 | 13 | # The Images converter is special in that it creates a directory and there's 14 | # special code in the Shodan CLI that relies on the "dirname" property to let 15 | # the user know where the images have been stored. 16 | dirname = None 17 | 18 | def process(self, files, file_size): 19 | # Get the filename from the already-open file handle and use it as 20 | # the directory name to store the images. 21 | self.dirname = self.fout.name[:-7] + '-images' 22 | 23 | # Remove the original file that was created 24 | self.fout.close() 25 | os.unlink(self.fout.name) 26 | 27 | # Create the directory if it doesn't yet exist 28 | if not os.path.exists(self.dirname): 29 | os.mkdir(self.dirname) 30 | 31 | # Close the existing file as the XlsxWriter library handles that for us 32 | self.fout.close() 33 | 34 | # Loop through all the banners in the data file 35 | for banner in iterate_files(files): 36 | screenshot = get_screenshot(banner) 37 | if screenshot: 38 | filename = '{}/{}-{}'.format(self.dirname, get_ip(banner), banner['port']) 39 | 40 | # If a file with the name already exists then count up until we 41 | # create a new, unique filename 42 | counter = 0 43 | tmpname = filename 44 | while os.path.exists(tmpname + '.jpg'): 45 | tmpname = '{}-{}'.format(filename, counter) 46 | counter += 1 47 | filename = tmpname + '.jpg' 48 | 49 | fout = open(filename, 'wb') 50 | fout.write(decode(screenshot['data'].encode(), 'base64')) 51 | fout.close() 52 | -------------------------------------------------------------------------------- /shodan/threatnet.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import json 3 | 4 | from .exception import APIError 5 | 6 | 7 | class Threatnet: 8 | """Wrapper around the Threatnet REST and Streaming APIs 9 | 10 | :param key: The Shodan API key that can be obtained from your account page (https://account.shodan.io) 11 | :type key: str 12 | :ivar stream: An instance of `shodan.Threatnet.Stream` that provides access to the Streaming API. 13 | """ 14 | 15 | class Stream: 16 | 17 | base_url = 'https://stream.shodan.io' 18 | 19 | def __init__(self, parent, proxies=None): 20 | self.parent = parent 21 | self.proxies = proxies 22 | 23 | def _create_stream(self, name): 24 | try: 25 | req = requests.get(self.base_url + name, params={'key': self.parent.api_key}, 26 | stream=True, proxies=self.proxies) 27 | except Exception: 28 | raise APIError('Unable to contact the Shodan Streaming API') 29 | 30 | if req.status_code != 200: 31 | try: 32 | raise APIError(req.json()['error']) 33 | except Exception: 34 | pass 35 | raise APIError('Invalid API key or you do not have access to the Streaming API') 36 | return req 37 | 38 | def events(self): 39 | stream = self._create_stream('/threatnet/events') 40 | for line in stream.iter_lines(): 41 | if line: 42 | banner = json.loads(line) 43 | yield banner 44 | 45 | def backscatter(self): 46 | stream = self._create_stream('/threatnet/backscatter') 47 | for line in stream.iter_lines(): 48 | if line: 49 | banner = json.loads(line) 50 | yield banner 51 | 52 | def activity(self): 53 | stream = self._create_stream('/threatnet/ssh') 54 | for line in stream.iter_lines(): 55 | if line: 56 | banner = json.loads(line) 57 | yield banner 58 | 59 | def __init__(self, key): 60 | """Initializes the API object. 61 | 62 | :param key: The Shodan API key. 63 | :type key: str 64 | """ 65 | self.api_key = key 66 | self.base_url = 'https://api.shodan.io' 67 | self.stream = self.Stream(self) 68 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | shodan: The official Python library and CLI for Shodan 2 | ====================================================== 3 | 4 | .. image:: https://img.shields.io/pypi/v/shodan.svg 5 | :target: https://pypi.org/project/shodan/ 6 | 7 | .. image:: https://img.shields.io/github/contributors/achillean/shodan-python.svg 8 | :target: https://github.com/achillean/shodan-python/graphs/contributors 9 | 10 | Shodan is a search engine for Internet-connected devices. Google lets you search for websites, 11 | Shodan lets you search for devices. This library provides developers easy access to all of the 12 | data stored in Shodan in order to automate tasks and integrate into existing tools. 13 | 14 | Features 15 | -------- 16 | 17 | - Search Shodan 18 | - `Fast/ bulk IP lookups `_ 19 | - Streaming API support for real-time consumption of Shodan firehose 20 | - `Network alerts (aka private firehose) `_ 21 | - `Manage Email Notifications `_ 22 | - Exploit search API fully implemented 23 | - Bulk data downloads 24 | - Access the Shodan DNS DB to view domain information 25 | - `Command-line interface `_ 26 | 27 | .. image:: https://cli.shodan.io/img/shodan-cli-preview.png 28 | :target: https://asciinema.org/~Shodan 29 | :width: 400px 30 | :align: center 31 | 32 | 33 | Quick Start 34 | ----------- 35 | 36 | .. code-block:: python 37 | 38 | from shodan import Shodan 39 | 40 | api = Shodan('MY API KEY') 41 | 42 | # Lookup an IP 43 | ipinfo = api.host('8.8.8.8') 44 | print(ipinfo) 45 | 46 | # Search for websites that have been "hacked" 47 | for banner in api.search_cursor('http.title:"hacked by"'): 48 | print(banner) 49 | 50 | # Get the total number of industrial control systems services on the Internet 51 | ics_services = api.count('tag:ics') 52 | print('Industrial Control Systems: {}'.format(ics_services['total'])) 53 | 54 | Grab your API key from https://account.shodan.io 55 | 56 | Installation 57 | ------------ 58 | 59 | To install the Shodan library, simply: 60 | 61 | .. code-block:: bash 62 | 63 | $ pip install shodan 64 | 65 | Or if you don't have pip installed (which you should seriously install): 66 | 67 | .. code-block:: bash 68 | 69 | $ easy_install shodan 70 | 71 | 72 | Documentation 73 | ------------- 74 | 75 | Documentation is available at https://shodan.readthedocs.org/ and https://help.shodan.io 76 | -------------------------------------------------------------------------------- /shodan/cli/organization.py: -------------------------------------------------------------------------------- 1 | import click 2 | import shodan 3 | 4 | from shodan.cli.helpers import get_api_key, humanize_api_plan 5 | 6 | 7 | @click.group() 8 | def org(): 9 | """Manage your organization's access to Shodan""" 10 | pass 11 | 12 | 13 | @org.command() 14 | @click.option('--silent', help="Don't send a notification to the user", default=False, is_flag=True) 15 | @click.argument('user', metavar='') 16 | def add(silent, user): 17 | """Add a new member""" 18 | key = get_api_key() 19 | api = shodan.Shodan(key) 20 | 21 | try: 22 | api.org.add_member(user, notify=not silent) 23 | except shodan.APIError as e: 24 | raise click.ClickException(e.value) 25 | 26 | click.secho('Successfully added the new member', fg='green') 27 | 28 | 29 | @org.command() 30 | def info(): 31 | """Show an overview of the organization""" 32 | key = get_api_key() 33 | api = shodan.Shodan(key) 34 | try: 35 | organization = api.org.info() 36 | except shodan.APIError as e: 37 | raise click.ClickException(e.value) 38 | 39 | click.secho(organization['name'], fg='cyan') 40 | click.secho('Access Level: ', nl=False, dim=True) 41 | click.secho(humanize_api_plan(organization['upgrade_type']), fg='magenta') 42 | 43 | if organization['domains']: 44 | click.secho('Authorized Domains: ', nl=False, dim=True) 45 | click.echo(', '.join(organization['domains'])) 46 | 47 | click.echo('') 48 | click.secho('Administrators:', dim=True) 49 | 50 | for admin in organization['admins']: 51 | click.echo(u' > {:30}\t{:30}'.format( 52 | click.style(admin['username'], fg='yellow'), 53 | admin['email']) 54 | ) 55 | 56 | click.echo('') 57 | if organization['members']: 58 | click.secho('Members:', dim=True) 59 | for member in organization['members']: 60 | click.echo(u' > {:30}\t{:30}'.format( 61 | click.style(member['username'], fg='yellow'), 62 | member['email']) 63 | ) 64 | else: 65 | click.secho('No members yet', dim=True) 66 | 67 | 68 | @org.command() 69 | @click.argument('user', metavar='') 70 | def remove(user): 71 | """Remove and downgrade a member""" 72 | key = get_api_key() 73 | api = shodan.Shodan(key) 74 | 75 | try: 76 | api.org.remove_member(user) 77 | except shodan.APIError as e: 78 | raise click.ClickException(e.value) 79 | 80 | click.secho('Successfully removed the member', fg='green') 81 | -------------------------------------------------------------------------------- /shodan/cli/converter/csvc.py: -------------------------------------------------------------------------------- 1 | 2 | from .base import Converter 3 | from ...helpers import iterate_files 4 | 5 | try: 6 | # python 3.x: Import ABC from collections.abc 7 | from collections.abc import MutableMapping 8 | except ImportError: 9 | # Python 2.x: Import ABC from collections 10 | from collections import MutableMapping 11 | 12 | from csv import writer as csv_writer, excel 13 | 14 | 15 | class CsvConverter(Converter): 16 | 17 | fields = [ 18 | 'data', 19 | 'hostnames', 20 | 'ip', 21 | 'ip_str', 22 | 'ipv6', 23 | 'org', 24 | 'isp', 25 | 'location.country_code', 26 | 'location.city', 27 | 'location.country_name', 28 | 'location.latitude', 29 | 'location.longitude', 30 | 'os', 31 | 'asn', 32 | 'port', 33 | 'tags', 34 | 'timestamp', 35 | 'transport', 36 | 'product', 37 | 'version', 38 | 'vulns', 39 | 40 | 'ssl.cipher.version', 41 | 'ssl.cipher.bits', 42 | 'ssl.cipher.name', 43 | 'ssl.alpn', 44 | 'ssl.versions', 45 | 'ssl.cert.serial', 46 | 'ssl.cert.fingerprint.sha1', 47 | 'ssl.cert.fingerprint.sha256', 48 | 49 | 'html', 50 | 'title', 51 | ] 52 | 53 | def process(self, files, file_size): 54 | writer = csv_writer(self.fout, dialect=excel, lineterminator='\n') 55 | 56 | # Write the header 57 | writer.writerow(self.fields) 58 | 59 | for banner in iterate_files(files): 60 | # The "vulns" property can't be nicely flattened as-is so we turn 61 | # it into a list before processing the banner. 62 | if 'vulns' in banner: 63 | banner['vulns'] = list(banner['vulns'].keys()) # Python3 returns dict_keys so we neeed to cover that to a list 64 | 65 | try: 66 | row = [] 67 | for field in self.fields: 68 | value = self.banner_field(banner, field) 69 | row.append(value) 70 | writer.writerow(row) 71 | except Exception: 72 | pass 73 | 74 | def banner_field(self, banner, flat_field): 75 | # The provided field is a collapsed form of the actual field 76 | fields = flat_field.split('.') 77 | 78 | try: 79 | current_obj = banner 80 | for field in fields: 81 | current_obj = current_obj[field] 82 | 83 | # Convert a list into a concatenated string 84 | if isinstance(current_obj, list): 85 | current_obj = ','.join([str(i) for i in current_obj]) 86 | 87 | return current_obj 88 | except Exception: 89 | pass 90 | 91 | return '' 92 | 93 | def flatten(self, d, parent_key='', sep='.'): 94 | items = [] 95 | for k, v in d.items(): 96 | new_key = parent_key + sep + k if parent_key else k 97 | if isinstance(v, MutableMapping): 98 | items.extend(self.flatten(v, new_key, sep=sep).items()) 99 | else: 100 | items.append((new_key, v)) 101 | return dict(items) 102 | -------------------------------------------------------------------------------- /shodan/cli/data.py: -------------------------------------------------------------------------------- 1 | import click 2 | import requests 3 | import shodan 4 | import shodan.helpers as helpers 5 | 6 | from shodan.cli.helpers import get_api_key 7 | 8 | 9 | @click.group() 10 | def data(): 11 | """Bulk data access to Shodan""" 12 | pass 13 | 14 | 15 | @data.command(name='list') 16 | @click.option('--dataset', help='See the available files in the given dataset', default=None, type=str) 17 | def data_list(dataset): 18 | """List available datasets or the files within those datasets.""" 19 | # Setup the API connection 20 | key = get_api_key() 21 | api = shodan.Shodan(key) 22 | 23 | if dataset: 24 | # Show the files within this dataset 25 | files = api.data.list_files(dataset) 26 | 27 | for file in files: 28 | click.echo(click.style(u'{:20s}'.format(file['name']), fg='cyan'), nl=False) 29 | click.echo(click.style('{:10s}'.format(helpers.humanize_bytes(file['size'])), fg='yellow'), nl=False) 30 | 31 | # Show the SHA1 checksum if available 32 | if file.get('sha1'): 33 | click.echo(click.style('{:42s}'.format(file['sha1']), fg='green'), nl=False) 34 | 35 | click.echo('{}'.format(file['url'])) 36 | else: 37 | # If no dataset was provided then show a list of all datasets 38 | datasets = api.data.list_datasets() 39 | 40 | for ds in datasets: 41 | click.echo(click.style('{:15s}'.format(ds['name']), fg='cyan'), nl=False) 42 | click.echo('{}'.format(ds['description'])) 43 | 44 | 45 | @data.command(name='download') 46 | @click.option('--chunksize', help='The size of the chunks that are downloaded into memory before writing them to disk.', default=1024, type=int) 47 | @click.option('--filename', '-O', help='Save the file as the provided filename instead of the default.') 48 | @click.argument('dataset', metavar='') 49 | @click.argument('name', metavar='') 50 | def data_download(chunksize, filename, dataset, name): 51 | # Setup the API connection 52 | key = get_api_key() 53 | api = shodan.Shodan(key) 54 | 55 | # Get the file object that the user requested which will contain the URL and total file size 56 | file = None 57 | try: 58 | files = api.data.list_files(dataset) 59 | for tmp in files: 60 | if tmp['name'] == name: 61 | file = tmp 62 | break 63 | except shodan.APIError as e: 64 | raise click.ClickException(e.value) 65 | 66 | # The file isn't available 67 | if not file: 68 | raise click.ClickException('File not found') 69 | 70 | # Start downloading the file 71 | response = requests.get(file['url'], stream=True) 72 | 73 | # Figure out the size of the file based on the headers 74 | filesize = response.headers.get('content-length', None) 75 | if not filesize: 76 | # Fall back to using the filesize provided by the API 77 | filesize = file['size'] 78 | else: 79 | filesize = int(filesize) 80 | 81 | chunk_size = 1024 82 | limit = filesize / chunk_size 83 | 84 | # Create a default filename based on the dataset and the filename within that dataset 85 | if not filename: 86 | filename = '{}-{}'.format(dataset, name) 87 | 88 | # Open the output file and start writing to it in chunks 89 | with open(filename, 'wb') as fout: 90 | with click.progressbar(response.iter_content(chunk_size=chunk_size), length=limit) as bar: 91 | for chunk in bar: 92 | if chunk: 93 | fout.write(chunk) 94 | 95 | click.echo(click.style('Download completed: {}'.format(filename), 'green')) 96 | -------------------------------------------------------------------------------- /docs/tutorial.rst: -------------------------------------------------------------------------------- 1 | 2 | Getting Started 3 | =============== 4 | 5 | Installation 6 | ------------------ 7 | 8 | To get started with the Python library for Shodan, first make sure that you've 9 | `received your API key `_. Once that's done, 10 | install the library via the cheeseshop using: 11 | 12 | .. code-block:: bash 13 | 14 | $ easy_install shodan 15 | 16 | Or if you already have it installed and want to upgrade to the latest version: 17 | 18 | .. code-block:: bash 19 | 20 | $ easy_install -U shodan 21 | 22 | It's always safe to update your library as backwards-compatibility is preserved. 23 | Usually a new version of the library simply means there are new methods/ features 24 | available. 25 | 26 | 27 | Connect to the API 28 | ------------------ 29 | 30 | The first thing we need to do in our code is to initialize the API object: 31 | 32 | .. code-block:: python 33 | 34 | import shodan 35 | 36 | SHODAN_API_KEY = "insert your API key here" 37 | 38 | api = shodan.Shodan(SHODAN_API_KEY) 39 | 40 | 41 | Searching Shodan 42 | ---------------- 43 | 44 | Now that we have our API object all good to go, we're ready to perform a search: 45 | 46 | .. code-block:: python 47 | 48 | # Wrap the request in a try/ except block to catch errors 49 | try: 50 | # Search Shodan 51 | results = api.search('apache') 52 | 53 | # Show the results 54 | print('Results found: {}'.format(results['total'])) 55 | for result in results['matches']: 56 | print('IP: {}'.format(result['ip_str'])) 57 | print(result['data']) 58 | print('') 59 | except shodan.APIError as e: 60 | print('Error: {}'.format(e)) 61 | 62 | Stepping through the code, we first call the :py:func:`Shodan.search` method on the `api` object which 63 | returns a dictionary of result information. We then print how many results were found in total, 64 | and finally loop through the returned matches and print their IP and banner. Each page of search results 65 | contains up to 100 results. 66 | 67 | There's a lot more information that gets returned by the function. See below for a shortened example 68 | dictionary that :py:func:`Shodan.search` returns: 69 | 70 | .. code-block:: python 71 | 72 | { 73 | 'total': 8669969, 74 | 'matches': [ 75 | { 76 | 'data': 'HTTP/1.0 200 OK\r\nDate: Mon, 08 Nov 2010 05:09:59 GMT\r\nSer...', 77 | 'hostnames': ['pl4t1n.de'], 78 | 'ip': 3579573318, 79 | 'ip_str': '89.110.147.239', 80 | 'os': 'FreeBSD 4.4', 81 | 'port': 80, 82 | 'timestamp': '2014-01-15T05:49:56.283713' 83 | }, 84 | ... 85 | ] 86 | } 87 | 88 | Please visit the `REST API documentation `_ for the complete list of properties that the methods can return. 89 | 90 | It's also good practice to wrap all API requests in a try/ except clause, since any error 91 | will raise an exception. But for simplicity's sake, I will leave that part out from now on. 92 | 93 | Looking up a host 94 | ----------------- 95 | 96 | To see what Shodan has available on a specific IP we can use the :py:func:`Shodan.host` function: 97 | 98 | .. code-block:: python 99 | 100 | # Lookup the host 101 | host = api.host('217.140.75.46') 102 | 103 | # Print general info 104 | print(""" 105 | IP: {} 106 | Organization: {} 107 | Operating System: {} 108 | """.format(host['ip_str'], host.get('org', 'n/a'), host.get('os', 'n/a'))) 109 | 110 | # Print all banners 111 | for item in host['data']: 112 | print(""" 113 | Port: {} 114 | Banner: {} 115 | 116 | """.format(item['port'], item['data'])) 117 | -------------------------------------------------------------------------------- /docs/examples/query-summary.rst: -------------------------------------------------------------------------------- 1 | Collecting Summary Information using Facets 2 | ------------------------------------------- 3 | 4 | A powerful ability of the Shodan API is to get summary information on a variety of properties. For example, 5 | if you wanted to learn which countries have the most Apache servers then you would use facets. If you wanted 6 | to figure out which version of nginx is most popular, you would use facets. Or if you wanted to see what the 7 | uptime distribution is for Microsoft-IIS servers then you would use facets. 8 | 9 | The following script shows how to use the `shodan.Shodan.count()` method to search Shodan without returning 10 | any results as well as asking the API to return faceted information on the organization, domain, port, ASN 11 | and country. 12 | 13 | .. code-block:: python 14 | 15 | #!/usr/bin/env python 16 | # 17 | # query-summary.py 18 | # Search Shodan and print summary information for the query. 19 | # 20 | # Author: achillean 21 | 22 | import shodan 23 | import sys 24 | 25 | # Configuration 26 | API_KEY = 'YOUR API KEY' 27 | 28 | # The list of properties we want summary information on 29 | FACETS = [ 30 | 'org', 31 | 'domain', 32 | 'port', 33 | 'asn', 34 | 35 | # We only care about the top 3 countries, this is how we let Shodan know to return 3 instead of the 36 | # default 5 for a facet. If you want to see more than 5, you could do ('country', 1000) for example 37 | # to see the top 1,000 countries for a search query. 38 | ('country', 3), 39 | ] 40 | 41 | FACET_TITLES = { 42 | 'org': 'Top 5 Organizations', 43 | 'domain': 'Top 5 Domains', 44 | 'port': 'Top 5 Ports', 45 | 'asn': 'Top 5 Autonomous Systems', 46 | 'country': 'Top 3 Countries', 47 | } 48 | 49 | # Input validation 50 | if len(sys.argv) == 1: 51 | print('Usage: %s ' % sys.argv[0]) 52 | sys.exit(1) 53 | 54 | try: 55 | # Setup the api 56 | api = shodan.Shodan(API_KEY) 57 | 58 | # Generate a query string out of the command-line arguments 59 | query = ' '.join(sys.argv[1:]) 60 | 61 | # Use the count() method because it doesn't return results and doesn't require a paid API plan 62 | # And it also runs faster than doing a search(). 63 | result = api.count(query, facets=FACETS) 64 | 65 | print('Shodan Summary Information') 66 | print('Query: %s' % query) 67 | print('Total Results: %s\n' % result['total']) 68 | 69 | # Print the summary info from the facets 70 | for facet in result['facets']: 71 | print(FACET_TITLES[facet]) 72 | 73 | for term in result['facets'][facet]: 74 | print('%s: %s' % (term['value'], term['count'])) 75 | 76 | # Print an empty line between summary info 77 | print('') 78 | 79 | except Exception as e: 80 | print('Error: %s' % e) 81 | sys.exit(1) 82 | 83 | """ 84 | Sample Output 85 | ============= 86 | 87 | ./query-summary.py apache 88 | Shodan Summary Information 89 | Query: apache 90 | Total Results: 34612043 91 | 92 | Top 5 Organizations 93 | Amazon.com: 808061 94 | Ecommerce Corporation: 788704 95 | Verio Web Hosting: 760112 96 | Unified Layer: 627827 97 | GoDaddy.com, LLC: 567004 98 | 99 | Top 5 Domains 100 | secureserver.net: 562047 101 | unifiedlayer.com: 494399 102 | t-ipconnect.de: 385792 103 | netart.pl: 194817 104 | wanadoo.fr: 151925 105 | 106 | Top 5 Ports 107 | 80: 24118703 108 | 443: 8330932 109 | 8080: 1479050 110 | 81: 359025 111 | 8443: 231441 112 | 113 | Top 5 Autonomous Systems 114 | as32392: 580002 115 | as2914: 465786 116 | as26496: 414998 117 | as48030: 332000 118 | as8560: 255774 119 | 120 | Top 3 Countries 121 | US: 13227366 122 | DE: 2900530 123 | JP: 2014506 124 | """ 125 | -------------------------------------------------------------------------------- /docs/examples/gifcreator.rst: -------------------------------------------------------------------------------- 1 | GIF Creator 2 | ----------- 3 | 4 | Shodan keeps a full history of all the information that has been gathered on an IP address. With the API, 5 | you're able to retrieve that history and we're going to use that to create a tool that outputs GIFs made of 6 | the screenshots that the Shodan crawlers gather. 7 | 8 | The below code requires the following Python packages: 9 | 10 | - arrow 11 | - shodan 12 | 13 | The **arrow** package is used to parse the *timestamp* field of the banner into a Python `datetime` object. 14 | 15 | In addition to the above Python packages, you also need to have the **ImageMagick** software installed. If you're 16 | working on Ubuntu or another distro using **apt** you can run the following command: 17 | 18 | .. code-block:: bash 19 | 20 | sudo apt-get install imagemagick 21 | 22 | This will provide us with the **convert** command which is needed to merge several images into an animated GIF. 23 | 24 | There are a few key Shodan methods/ parameters that make the script work: 25 | 26 | 1. :py:func:`shodan.helpers.iterate_files()` to loop through the Shodan data file 27 | 2. **history** flag on the :py:func:`shodan.Shodan.host` method to get all the banners for an IP that Shodan has collected over the years 28 | 29 | 30 | 31 | .. code-block:: python 32 | 33 | #!/usr/bin/env python 34 | # gifcreator.py 35 | # 36 | # Dependencies: 37 | # - arrow 38 | # - shodan 39 | # 40 | # Installation: 41 | # sudo easy_install arrow shodan 42 | # sudo apt-get install imagemagick 43 | # 44 | # Usage: 45 | # 1. Download a json.gz file using the website or the Shodan command-line tool (https://cli.shodan.io). 46 | # For example: 47 | # shodan download screenshots.json.gz has_screenshot:true 48 | # 2. Run the tool on the file: 49 | # python gifcreator.py screenshots.json.gz 50 | 51 | import arrow 52 | import os 53 | import shodan 54 | import shodan.helpers as helpers 55 | import sys 56 | 57 | 58 | # Settings 59 | API_KEY = '' 60 | MIN_SCREENS = 5 # Number of screenshots that Shodan needs to have in order to make a GIF 61 | MAX_SCREENS = 24 62 | 63 | if len(sys.argv) != 2: 64 | print('Usage: {} '.format(sys.argv[0])) 65 | sys.exit(1) 66 | 67 | # GIFs are stored in the local "data" directory 68 | os.mkdir('data') 69 | 70 | # We need to connect to the API to lookup the historical host information 71 | api = shodan.Shodan(API_KEY) 72 | 73 | # Use the shodan.helpers.iterate_files() method to loop over the Shodan data file 74 | for result in helpers.iterate_files(sys.argv[1]): 75 | # Get the historic info 76 | host = api.host(result['ip_str'], history=True) 77 | 78 | # Count how many screenshots this host has 79 | screenshots = [] 80 | for banner in host['data']: 81 | # Extract the image from the banner data 82 | if 'opts' in banner and 'screenshot' in banner['opts']: 83 | # Sort the images by the time they were collected so the GIF will loop 84 | # based on the local time regardless of which day the banner was taken. 85 | timestamp = arrow.get(banner['timestamp']).time() 86 | sort_key = timestamp.hour 87 | screenshots.append(( 88 | sort_key, 89 | banner['opts']['screenshot']['data'] 90 | )) 91 | 92 | # Ignore any further screenshots if we already have MAX_SCREENS number of images 93 | if len(screenshots) >= MAX_SCREENS: 94 | break 95 | 96 | # Extract the screenshots and turn them into a GIF if we've got the necessary 97 | # amount of images. 98 | if len(screenshots) >= MIN_SCREENS: 99 | for (i, screenshot) in enumerate(sorted(screenshots, key=lambda x: x[0], reverse=True)): 100 | open('/tmp/gif-image-{}.jpg'.format(i), 'w').write(screenshot[1].decode('base64')) 101 | 102 | # Create the actual GIF using the ImageMagick "convert" command 103 | os.system('convert -layers OptimizePlus -delay 5x10 /tmp/gif-image-*.jpg -loop 0 +dither -colors 256 -depth 8 data/{}.gif'.format(result['ip_str'])) 104 | 105 | # Clean up the temporary files 106 | os.system('rm -f /tmp/gif-image-*.jpg') 107 | 108 | # Show a progress indicator 109 | print(result['ip_str']) 110 | 111 | 112 | The full code is also available on GitHub: https://gist.github.com/achillean/963eea552233d9550101 113 | -------------------------------------------------------------------------------- /shodan/cli/helpers.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Helper methods used across the CLI commands. 3 | ''' 4 | import click 5 | import datetime 6 | import gzip 7 | import itertools 8 | import os 9 | import sys 10 | from ipaddress import ip_network, ip_address 11 | 12 | from .settings import SHODAN_CONFIG_DIR 13 | 14 | try: 15 | basestring # Python 2 16 | except NameError: 17 | basestring = (str, ) # Python 3 18 | 19 | 20 | def get_api_key(): 21 | '''Returns the API key of the current logged-in user.''' 22 | shodan_dir = os.path.expanduser(SHODAN_CONFIG_DIR) 23 | keyfile = shodan_dir + '/api_key' 24 | 25 | # If the file doesn't yet exist let the user know that they need to 26 | # initialize the shodan cli 27 | if not os.path.exists(keyfile): 28 | raise click.ClickException('Please run "shodan init " before using this command') 29 | 30 | # Make sure it is a read-only file 31 | if not oct(os.stat(keyfile).st_mode).endswith("600"): 32 | os.chmod(keyfile, 0o600) 33 | 34 | with open(keyfile, 'r') as fin: 35 | return fin.read().strip() 36 | 37 | 38 | def escape_data(args): 39 | # Make sure the string is unicode so the terminal can properly display it 40 | # We do it using format() so it works across Python 2 and 3 41 | args = u'{}'.format(args) 42 | return args.replace('\n', '\\n').replace('\r', '\\r').replace('\t', '\\t') 43 | 44 | 45 | def timestr(): 46 | return datetime.datetime.utcnow().strftime('%Y-%m-%d') 47 | 48 | 49 | def open_streaming_file(directory, timestr, compresslevel=9): 50 | return gzip.open('{}/{}.json.gz'.format(directory, timestr), 'a', compresslevel) 51 | 52 | 53 | def get_banner_field(banner, flat_field): 54 | # The provided field is a collapsed form of the actual field 55 | fields = flat_field.split('.') 56 | 57 | try: 58 | current_obj = banner 59 | for field in fields: 60 | current_obj = current_obj[field] 61 | return current_obj 62 | except Exception: 63 | pass 64 | 65 | return None 66 | 67 | 68 | def filter_with_netmask(banner, netmask): 69 | # filtering based on netmask is a more abstract concept than 70 | # a mere check for a specific field and thus needs its own mechanism 71 | # this will enable users to use the net:10.0.0.0/8 syntax they are used to 72 | # to find specific networks from a big shodan download. 73 | network = ip_network(netmask) 74 | ip_field = get_banner_field(banner, 'ip') 75 | if not ip_field: 76 | return False 77 | banner_ip_address = ip_address(ip_field) 78 | return banner_ip_address in network 79 | 80 | 81 | def match_filters(banner, filters): 82 | for args in filters: 83 | flat_field, check = args.split(':', 1) 84 | if flat_field == 'net': 85 | return filter_with_netmask(banner, check) 86 | 87 | value = get_banner_field(banner, flat_field) 88 | 89 | # If the field doesn't exist on the banner then ignore the record 90 | if not value: 91 | return False 92 | 93 | # It must match all filters to be allowed 94 | field_type = type(value) 95 | 96 | # For lists of strings we see whether the desired value is contained in the field 97 | if field_type == list or isinstance(value, basestring): 98 | if check not in value: 99 | return False 100 | elif field_type == int: 101 | if int(check) != value: 102 | return False 103 | elif field_type == float: 104 | if float(check) != value: 105 | return False 106 | else: 107 | # Ignore unknown types 108 | pass 109 | 110 | return True 111 | 112 | 113 | def async_spinner(finished): 114 | spinner = itertools.cycle(['-', '/', '|', '\\']) 115 | while not finished.is_set(): 116 | sys.stdout.write('\b{}'.format(next(spinner))) 117 | sys.stdout.flush() 118 | finished.wait(0.2) 119 | 120 | 121 | def humanize_api_plan(plan): 122 | return { 123 | 'oss': 'Free', 124 | 'dev': 'Membership', 125 | 'basic': 'Freelancer API', 126 | 'plus': 'Small Business API', 127 | 'corp': 'Corporate API', 128 | 'stream-100': 'Enterprise', 129 | }[plan] 130 | -------------------------------------------------------------------------------- /shodan/cli/converter/kml.py: -------------------------------------------------------------------------------- 1 | 2 | from .base import Converter 3 | from ...helpers import iterate_files 4 | 5 | 6 | class KmlConverter(Converter): 7 | 8 | def header(self): 9 | self.fout.write(""" 10 | 11 | """) 12 | 13 | def footer(self): 14 | self.fout.write("""""") 15 | 16 | def process(self, files, file_size): 17 | # Write the header 18 | self.header() 19 | 20 | hosts = {} 21 | for banner in iterate_files(files): 22 | ip = banner.get('ip_str', banner.get('ipv6', None)) 23 | if not ip: 24 | continue 25 | 26 | if ip not in hosts: 27 | hosts[ip] = banner 28 | hosts[ip]['ports'] = [] 29 | 30 | hosts[ip]['ports'].append(banner['port']) 31 | 32 | for ip, host in iter(hosts.items()): 33 | self.write(host) 34 | 35 | self.footer() 36 | 37 | def write(self, host): 38 | try: 39 | ip = host.get('ip_str', host.get('ipv6', None)) 40 | lat, lon = host['location']['latitude'], host['location']['longitude'] 41 | 42 | placemark = '{}]]>'.format(ip) 43 | placemark += '{0}'.format(host['hostnames'][0]) 47 | 48 | placemark += '

Ports

    ' 49 | 50 | for port in host['ports']: 51 | placemark += """ 52 |
  • {} 68 |
    • 69 | """.format(port) 70 | 71 | placemark += '
' 72 | 73 | placemark += """ 74 | 97 |
powered by Shodan
98 | """.format(ip) 99 | 100 | placemark += ']]>' 101 | placemark += '{},{}'.format(lon, lat) 102 | placemark += '' 103 | 104 | self.fout.write(placemark.encode('utf-8')) 105 | except Exception: 106 | pass 107 | -------------------------------------------------------------------------------- /shodan/cli/converter/excel.py: -------------------------------------------------------------------------------- 1 | 2 | from .base import Converter 3 | from ...helpers import iterate_files, get_ip 4 | 5 | from collections import defaultdict 6 | from xlsxwriter import Workbook 7 | 8 | 9 | class ExcelConverter(Converter): 10 | 11 | fields = [ 12 | 'port', 13 | 'timestamp', 14 | 'data', 15 | 'hostnames', 16 | 'org', 17 | 'isp', 18 | 'location.country_name', 19 | 'location.country_code', 20 | 'location.city', 21 | 'os', 22 | 'asn', 23 | 'transport', 24 | 'product', 25 | 'version', 26 | 27 | 'http.server', 28 | 'http.title', 29 | ] 30 | 31 | field_names = { 32 | 'org': 'Organization', 33 | 'isp': 'ISP', 34 | 'location.country_code': 'Country ISO Code', 35 | 'location.country_name': 'Country', 36 | 'location.city': 'City', 37 | 'os': 'OS', 38 | 'asn': 'ASN', 39 | 40 | 'http.server': 'Web Server', 41 | 'http.title': 'Website Title', 42 | } 43 | 44 | def process(self, files, file_size): 45 | # Get the filename from the already-open file handle 46 | filename = self.fout.name 47 | 48 | # Close the existing file as the XlsxWriter library handles that for us 49 | self.fout.close() 50 | 51 | # Create the new workbook 52 | workbook = Workbook(filename) 53 | 54 | # Check if Excel file is larger than 4GB 55 | if file_size > 4e9: 56 | workbook.use_zip64() 57 | 58 | # Define some common styles/ formats 59 | bold = workbook.add_format({ 60 | 'bold': 1, 61 | }) 62 | 63 | # Create the main worksheet where all the raw data is shown 64 | main_sheet = workbook.add_worksheet('Raw Data') 65 | 66 | # Write the header 67 | main_sheet.write(0, 0, 'IP', bold) # The IP field can be either ip_str or ipv6 so we treat it differently 68 | main_sheet.set_column(0, 0, 20) 69 | 70 | row = 0 71 | col = 1 72 | for field in self.fields: 73 | name = self.field_names.get(field, field.capitalize()) 74 | main_sheet.write(row, col, name, bold) 75 | col += 1 76 | row += 1 77 | 78 | total = 0 79 | ports = defaultdict(int) 80 | for banner in iterate_files(files): 81 | try: 82 | # Build the list that contains all the relevant values 83 | data = [] 84 | for field in self.fields: 85 | value = self.banner_field(banner, field) 86 | data.append(value) 87 | 88 | # Write those values to the main workbook 89 | # Starting off w/ the special "IP" property 90 | main_sheet.write_string(row, 0, get_ip(banner)) 91 | col = 1 92 | 93 | for value in data: 94 | main_sheet.write(row, col, value) 95 | col += 1 96 | row += 1 97 | except Exception: 98 | pass 99 | 100 | # Aggregate summary information 101 | total += 1 102 | ports[banner['port']] += 1 103 | 104 | summary_sheet = workbook.add_worksheet('Summary') 105 | summary_sheet.write(0, 0, 'Total', bold) 106 | summary_sheet.write(0, 1, total) 107 | 108 | # Ports Distribution 109 | summary_sheet.write(0, 3, 'Ports Distribution', bold) 110 | row = 1 111 | col = 3 112 | for key, value in sorted(ports.items(), reverse=True, key=lambda kv: (kv[1], kv[0])): 113 | summary_sheet.write(row, col, key) 114 | summary_sheet.write(row, col + 1, value) 115 | row += 1 116 | 117 | workbook.close() 118 | 119 | def banner_field(self, banner, flat_field): 120 | # The provided field is a collapsed form of the actual field 121 | fields = flat_field.split('.') 122 | 123 | try: 124 | current_obj = banner 125 | for field in fields: 126 | current_obj = current_obj[field] 127 | 128 | # Convert a list into a concatenated string 129 | if isinstance(current_obj, list): 130 | current_obj = ','.join([str(i) for i in current_obj]) 131 | 132 | return current_obj 133 | except Exception: 134 | pass 135 | 136 | return '' 137 | -------------------------------------------------------------------------------- /shodan/helpers.py: -------------------------------------------------------------------------------- 1 | import gzip 2 | import requests 3 | import json 4 | 5 | from .exception import APIError 6 | 7 | try: 8 | basestring 9 | except NameError: 10 | basestring = str 11 | 12 | 13 | def create_facet_string(facets): 14 | """Converts a Python list of facets into a comma-separated string that can be understood by 15 | the Shodan API. 16 | """ 17 | facet_str = '' 18 | for facet in facets: 19 | if isinstance(facet, basestring): 20 | facet_str += facet 21 | else: 22 | facet_str += '{}:{}'.format(facet[0], facet[1]) 23 | facet_str += ',' 24 | return facet_str[:-1] 25 | 26 | 27 | def api_request(key, function, params=None, data=None, base_url='https://api.shodan.io', 28 | method='get', retries=1, proxies=None): 29 | """General-purpose function to create web requests to SHODAN. 30 | 31 | Arguments: 32 | function -- name of the function you want to execute 33 | params -- dictionary of parameters for the function 34 | proxies -- a proxies array for the requests library 35 | 36 | Returns 37 | A dictionary containing the function's results. 38 | 39 | """ 40 | # Add the API key parameter automatically 41 | params['key'] = key 42 | 43 | # Send the request 44 | tries = 0 45 | error = False 46 | while tries <= retries: 47 | try: 48 | if method.lower() == 'post': 49 | data = requests.post(base_url + function, json.dumps(data), params=params, 50 | headers={'content-type': 'application/json'}, 51 | proxies=proxies) 52 | elif method.lower() == 'delete': 53 | data = requests.delete(base_url + function, params=params, proxies=proxies) 54 | elif method.lower() == 'put': 55 | data = requests.put(base_url + function, params=params, proxies=proxies) 56 | else: 57 | data = requests.get(base_url + function, params=params, proxies=proxies) 58 | 59 | # Exit out of the loop 60 | break 61 | except Exception: 62 | error = True 63 | tries += 1 64 | 65 | if error and tries >= retries: 66 | raise APIError('Unable to connect to Shodan') 67 | 68 | # Check that the API key wasn't rejected 69 | if data.status_code == 401: 70 | try: 71 | raise APIError(data.json()['error']) 72 | except (ValueError, KeyError): 73 | pass 74 | raise APIError('Invalid API key') 75 | 76 | # Parse the text into JSON 77 | try: 78 | data = data.json() 79 | except Exception: 80 | raise APIError('Unable to parse JSON response') 81 | 82 | # Raise an exception if an error occurred 83 | if type(data) == dict and data.get('error', None): 84 | raise APIError(data['error']) 85 | 86 | # Return the data 87 | return data 88 | 89 | 90 | def iterate_files(files, fast=False): 91 | """Loop over all the records of the provided Shodan output file(s).""" 92 | loads = json.loads 93 | if fast: 94 | # Try to use ujson for parsing JSON if it's available and the user requested faster throughput 95 | # It's significantly faster at encoding/ decoding JSON but it doesn't support as 96 | # many options as the standard library. As such, we're mostly interested in using it for 97 | # decoding since reading/ parsing files will use up the most time. 98 | # pylint: disable=E0401 99 | try: 100 | from ujson import loads 101 | except Exception: 102 | pass 103 | 104 | if isinstance(files, basestring): 105 | files = [files] 106 | 107 | for filename in files: 108 | # Create a file handle depending on the filetype 109 | if filename.endswith('.gz'): 110 | fin = gzip.open(filename, 'r') 111 | else: 112 | fin = open(filename, 'r') 113 | 114 | for line in fin: 115 | # Ensure the line has been decoded into a string to prevent errors w/ Python3 116 | if not isinstance(line, basestring): 117 | line = line.decode('utf-8') 118 | 119 | # Convert the JSON into a native Python object 120 | banner = loads(line) 121 | yield banner 122 | 123 | 124 | def get_screenshot(banner): 125 | if 'screenshot' in banner and banner['screenshot']: 126 | return banner['screenshot'] 127 | elif 'opts' in banner and 'screenshot' in banner['opts']: 128 | return banner['opts']['screenshot'] 129 | 130 | return None 131 | 132 | 133 | def get_ip(banner): 134 | if 'ipv6' in banner: 135 | return banner['ipv6'] 136 | return banner['ip_str'] 137 | 138 | 139 | def open_file(filename, mode='a', compresslevel=9): 140 | return gzip.open(filename, mode, compresslevel) 141 | 142 | 143 | def write_banner(fout, banner): 144 | line = json.dumps(banner) + '\n' 145 | fout.write(line.encode('utf-8')) 146 | 147 | 148 | def humanize_bytes(byte_count, precision=1): 149 | """Return a humanized string representation of a number of bytes. 150 | >>> humanize_bytes(1) 151 | '1 byte' 152 | >>> humanize_bytes(1024) 153 | '1.0 kB' 154 | >>> humanize_bytes(1024*123) 155 | '123.0 kB' 156 | >>> humanize_bytes(1024*12342) 157 | '12.1 MB' 158 | >>> humanize_bytes(1024*12342,2) 159 | '12.05 MB' 160 | >>> humanize_bytes(1024*1234,2) 161 | '1.21 MB' 162 | >>> humanize_bytes(1024*1234*1111,2) 163 | '1.31 GB' 164 | >>> humanize_bytes(1024*1234*1111,1) 165 | '1.3 GB' 166 | """ 167 | if byte_count == 1: 168 | return '1 byte' 169 | if byte_count < 1024: 170 | return '{0:0.{1}f} {2}'.format(byte_count, 0, 'bytes') 171 | 172 | suffixes = ['KB', 'MB', 'GB', 'TB', 'PB'] 173 | multiple = 1024.0 # .0 to force float on python 2 174 | for suffix in suffixes: 175 | byte_count /= multiple 176 | if byte_count < multiple: 177 | return '{0:0.{1}f} {2}'.format(byte_count, precision, suffix) 178 | return '{0:0.{1}f} {2}'.format(byte_count, precision, suffix) 179 | -------------------------------------------------------------------------------- /shodan/cli/host.py: -------------------------------------------------------------------------------- 1 | # Helper methods for printing `host` information to the terminal. 2 | import click 3 | 4 | from shodan.helpers import get_ip 5 | 6 | 7 | def host_print_pretty(host, history=False): 8 | """Show the host information in a user-friendly way and try to include 9 | as much relevant information as possible.""" 10 | # General info 11 | click.echo(click.style(get_ip(host), fg='green')) 12 | if len(host['hostnames']) > 0: 13 | click.echo(u'{:25s}{}'.format('Hostnames:', ';'.join(host['hostnames']))) 14 | 15 | if 'city' in host and host['city']: 16 | click.echo(u'{:25s}{}'.format('City:', host['city'])) 17 | 18 | if 'country_name' in host and host['country_name']: 19 | click.echo(u'{:25s}{}'.format('Country:', host['country_name'])) 20 | 21 | if 'os' in host and host['os']: 22 | click.echo(u'{:25s}{}'.format('Operating System:', host['os'])) 23 | 24 | if 'org' in host and host['org']: 25 | click.echo(u'{:25s}{}'.format('Organization:', host['org'])) 26 | 27 | if 'last_update' in host and host['last_update']: 28 | click.echo('{:25s}{}'.format('Updated:', host['last_update'])) 29 | 30 | click.echo('{:25s}{}'.format('Number of open ports:', len(host['ports']))) 31 | 32 | # Output the vulnerabilities the host has 33 | if 'vulns' in host and len(host['vulns']) > 0: 34 | vulns = [] 35 | for vuln in host['vulns']: 36 | if vuln.startswith('!'): 37 | continue 38 | if vuln.upper() == 'CVE-2014-0160': 39 | vulns.append(click.style('Heartbleed', fg='red')) 40 | else: 41 | vulns.append(click.style(vuln, fg='red')) 42 | 43 | if len(vulns) > 0: 44 | click.echo('{:25s}'.format('Vulnerabilities:'), nl=False) 45 | 46 | for vuln in vulns: 47 | click.echo(vuln + '\t', nl=False) 48 | 49 | click.echo('') 50 | 51 | click.echo('') 52 | 53 | # If the user doesn't have access to SSL/ Telnet results then we need 54 | # to pad the host['data'] property with empty banners so they still see 55 | # the port listed as open. (#63) 56 | if len(host['ports']) != len(host['data']): 57 | # Find the ports the user can't see the data for 58 | ports = host['ports'] 59 | for banner in host['data']: 60 | if banner['port'] in ports: 61 | ports.remove(banner['port']) 62 | 63 | # Add the placeholder banners 64 | for port in ports: 65 | banner = { 66 | 'port': port, 67 | 'transport': 'tcp', # All the filtered services use TCP 68 | 'timestamp': host['data'][-1]['timestamp'], # Use the timestamp of the oldest banner 69 | 'placeholder': True, # Don't store this banner when the file is saved 70 | } 71 | host['data'].append(banner) 72 | 73 | click.echo('Ports:') 74 | for banner in sorted(host['data'], key=lambda k: k['port']): 75 | product = '' 76 | version = '' 77 | if 'product' in banner and banner['product']: 78 | product = banner['product'] 79 | if 'version' in banner and banner['version']: 80 | version = '({})'.format(banner['version']) 81 | 82 | click.echo(click.style('{:>7d}'.format(banner['port']), fg='cyan'), nl=False) 83 | if 'transport' in banner: 84 | click.echo('/', nl=False) 85 | click.echo(click.style('{} '.format(banner['transport']), fg='yellow'), nl=False) 86 | click.echo('{} {}'.format(product, version), nl=False) 87 | 88 | if history: 89 | # Format the timestamp to only show the year-month-day 90 | date = banner['timestamp'][:10] 91 | click.echo(click.style('\t\t({})'.format(date), fg='white', dim=True), nl=False) 92 | click.echo('') 93 | 94 | # Show optional HTTP information 95 | if 'http' in banner: 96 | if 'title' in banner['http'] and banner['http']['title']: 97 | click.echo('\t|-- HTTP title: {}'.format(banner['http']['title'])) 98 | 99 | # Show optional ssl info 100 | if 'ssl' in banner: 101 | if 'cert' in banner['ssl'] and banner['ssl']['cert']: 102 | if 'issuer' in banner['ssl']['cert'] and banner['ssl']['cert']['issuer']: 103 | issuer = ', '.join(['{}={}'.format(key, value) for key, value in banner['ssl']['cert']['issuer'].items()]) 104 | click.echo('\t|-- Cert Issuer: {}'.format(issuer)) 105 | if 'subject' in banner['ssl']['cert'] and banner['ssl']['cert']['subject']: 106 | subject = ', '.join(['{}={}'.format(key, value) for key, value in banner['ssl']['cert']['subject'].items()]) 107 | click.echo('\t|-- Cert Subject: {}'.format(subject)) 108 | if 'versions' in banner['ssl'] and banner['ssl']['versions']: 109 | click.echo('\t|-- SSL Versions: {}'.format(', '.join([item for item in sorted(banner['ssl']['versions']) if not version.startswith('-')]))) 110 | if 'dhparams' in banner['ssl'] and banner['ssl']['dhparams']: 111 | click.echo('\t|-- Diffie-Hellman Parameters:') 112 | click.echo('\t\t{:15s}{}\n\t\t{:15s}{}'.format('Bits:', banner['ssl']['dhparams']['bits'], 'Generator:', banner['ssl']['dhparams']['generator'])) 113 | if 'fingerprint' in banner['ssl']['dhparams']: 114 | click.echo('\t\t{:15s}{}'.format('Fingerprint:', banner['ssl']['dhparams']['fingerprint'])) 115 | 116 | 117 | def host_print_tsv(host, history=False): 118 | """Show the host information in a succinct, grep-friendly manner.""" 119 | for banner in sorted(host['data'], key=lambda k: k['port']): 120 | click.echo(click.style('{:>7d}'.format(banner['port']), fg='cyan'), nl=False) 121 | click.echo('\t', nl=False) 122 | click.echo(click.style('{} '.format(banner['transport']), fg='yellow'), nl=False) 123 | 124 | if history: 125 | # Format the timestamp to only show the year-month-day 126 | date = banner['timestamp'][:10] 127 | click.echo(click.style('\t({})'.format(date), fg='white', dim=True), nl=False) 128 | click.echo('') 129 | 130 | 131 | HOST_PRINT = { 132 | 'pretty': host_print_pretty, 133 | 'tsv': host_print_tsv, 134 | } 135 | -------------------------------------------------------------------------------- /tests/test_shodan.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import shodan 3 | 4 | try: 5 | basestring 6 | except NameError: 7 | basestring = str 8 | 9 | 10 | class ShodanTests(unittest.TestCase): 11 | 12 | api = None 13 | FACETS = [ 14 | 'port', 15 | ('domain', 1) 16 | ] 17 | QUERIES = { 18 | 'simple': 'cisco-ios', 19 | 'minify': 'apache', 20 | 'advanced': 'apache port:443', 21 | 'empty': 'asdasdasdasdasdasdasdasdasdhjihjkjk', 22 | } 23 | 24 | def setUp(self): 25 | with open('SHODAN-API-KEY') as f: 26 | self.api = shodan.Shodan(f.read().strip()) 27 | 28 | def test_search_simple(self): 29 | results = self.api.search(self.QUERIES['simple']) 30 | 31 | # Make sure the properties exist 32 | self.assertIn('matches', results) 33 | self.assertIn('total', results) 34 | 35 | # Make sure no error occurred 36 | self.assertNotIn('error', results) 37 | 38 | # Make sure some values were returned 39 | self.assertTrue(results['matches']) 40 | self.assertTrue(results['total']) 41 | 42 | # A regular search shouldn't have the optional info 43 | self.assertNotIn('opts', results['matches'][0]) 44 | 45 | def test_search_empty(self): 46 | results = self.api.search(self.QUERIES['empty']) 47 | self.assertTrue(len(results['matches']) == 0) 48 | self.assertEqual(results['total'], 0) 49 | 50 | def test_search_facets(self): 51 | results = self.api.search(self.QUERIES['simple'], facets=self.FACETS) 52 | 53 | self.assertTrue(results['facets']['port']) 54 | self.assertEqual(len(results['facets']['domain']), 1) 55 | 56 | def test_count_simple(self): 57 | results = self.api.count(self.QUERIES['simple']) 58 | 59 | # Make sure the properties exist 60 | self.assertIn('matches', results) 61 | self.assertIn('total', results) 62 | 63 | # Make sure no error occurred 64 | self.assertNotIn('error', results) 65 | 66 | # Make sure no values were returned 67 | self.assertFalse(results['matches']) 68 | self.assertTrue(results['total']) 69 | 70 | def test_count_facets(self): 71 | results = self.api.count(self.QUERIES['simple'], facets=self.FACETS) 72 | 73 | self.assertTrue(results['facets']['port']) 74 | self.assertEqual(len(results['facets']['domain']), 1) 75 | 76 | def test_host_details(self): 77 | host = self.api.host('147.228.101.7') 78 | 79 | self.assertEqual('147.228.101.7', host['ip_str']) 80 | self.assertFalse(isinstance(host['ip'], basestring)) 81 | 82 | def test_search_minify(self): 83 | results = self.api.search(self.QUERIES['minify'], minify=False) 84 | self.assertIn('opts', results['matches'][0]) 85 | 86 | def test_exploits_search(self): 87 | results = self.api.exploits.search('apache') 88 | self.assertIn('matches', results) 89 | self.assertIn('total', results) 90 | self.assertTrue(results['matches']) 91 | 92 | def test_exploits_search_paging(self): 93 | results = self.api.exploits.search('apache', page=1) 94 | match1 = results['matches'][0] 95 | results = self.api.exploits.search('apache', page=2) 96 | match2 = results['matches'][0] 97 | 98 | self.assertNotEqual(match1['_id'], match2['_id']) 99 | 100 | def test_exploits_search_facets(self): 101 | results = self.api.exploits.search('apache', facets=['source', ('author', 1)]) 102 | self.assertIn('facets', results) 103 | self.assertTrue(results['facets']['source']) 104 | self.assertTrue(len(results['facets']['author']) == 1) 105 | 106 | def test_exploits_count(self): 107 | results = self.api.exploits.count('apache') 108 | self.assertIn('matches', results) 109 | self.assertIn('total', results) 110 | self.assertTrue(len(results['matches']) == 0) 111 | 112 | def test_exploits_count_facets(self): 113 | results = self.api.exploits.count('apache', facets=['source', ('author', 1)]) 114 | self.assertEqual(len(results['matches']), 0) 115 | self.assertIn('facets', results) 116 | self.assertTrue(results['facets']['source']) 117 | self.assertTrue(len(results['facets']['author']) == 1) 118 | 119 | def test_trends_search(self): 120 | results = self.api.trends.search('apache', facets=[('product', 10)]) 121 | self.assertIn('total', results) 122 | self.assertIn('matches', results) 123 | self.assertIn('facets', results) 124 | self.assertTrue(results['matches']) 125 | self.assertIn('2023-06', [bucket['key'] for bucket in results['facets']['product']]) 126 | 127 | results = self.api.trends.search('apache', facets=[]) 128 | self.assertIn('total', results) 129 | self.assertIn('matches', results) 130 | self.assertNotIn('facets', results) 131 | self.assertTrue(results['matches']) 132 | self.assertIn('2023-06', [match['month'] for match in results['matches']]) 133 | 134 | def test_trends_search_filters(self): 135 | results = self.api.trends.search_filters() 136 | self.assertIn('has_ipv6', results) 137 | self.assertNotIn('http.html', results) 138 | 139 | def test_trends_search_facets(self): 140 | results = self.api.trends.search_facets() 141 | self.assertIn('product', results) 142 | self.assertNotIn('cpe', results) 143 | 144 | # Test error responses 145 | def test_invalid_key(self): 146 | api = shodan.Shodan('garbage') 147 | raised = False 148 | try: 149 | api.search('something') 150 | except shodan.APIError: 151 | raised = True 152 | 153 | self.assertTrue(raised) 154 | 155 | def test_invalid_host_ip(self): 156 | raised = False 157 | try: 158 | self.api.host('test') 159 | except shodan.APIError: 160 | raised = True 161 | 162 | self.assertTrue(raised) 163 | 164 | def test_search_empty_query(self): 165 | raised = False 166 | try: 167 | self.api.search('') 168 | except shodan.APIError: 169 | raised = True 170 | self.assertTrue(raised) 171 | 172 | def test_search_advanced_query(self): 173 | # The free API plan can't use filters 174 | raised = False 175 | try: 176 | self.api.search(self.QUERIES['advanced']) 177 | except shodan.APIError: 178 | raised = True 179 | self.assertTrue(raised) 180 | 181 | 182 | if __name__ == '__main__': 183 | unittest.main() 184 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | CHANGELOG 2 | ========= 3 | 4 | 1.28.0 5 | ------ 6 | * Add the ability to whitelist a specific vulnerability in Shodan Monitor instead of whitelisting the while IP:port 7 | * Show scan ID when scanning without showing results (credit to @seadog007) 8 | * Handle bad gateway errors (credit to @yaron-cider) 9 | 10 | 11 | 1.27.0 12 | ------ 13 | * New command: ``shodan alert export`` to save the current network monitoring configuration 14 | * New command: ``shodan alert import`` to restore a previous network monitoring configuration 15 | * Automatically rate limit API requests to 1 request per second (credit to @malvidin) 16 | 17 | 1.26.1 18 | ------ 19 | * Fix a unicode issue that caused the streams to get truncated and error out due to invalid JSON 20 | 21 | 1.26.0 22 | ------ 23 | * Add the ability to create custom data streams in the Shodan() class as well as the CLI (``shodan stream --custom-filters ``) 24 | 25 | 1.25.0 26 | ------ 27 | * Add new CLI command: shodan alert download 28 | 29 | 1.24.0 30 | ------ 31 | * Add new CLI command: shodan alert stats 32 | 33 | 1.23.0 34 | ------ 35 | * Add new CLI command: shodan alert domain 36 | 37 | 1.22.1 38 | ------ 39 | * Fix bug when converting data file to CSV using Python3 40 | 41 | 1.22.0 42 | ------ 43 | * Add support for new vulnerability streaming endpoints 44 | 45 | 1.21.3 46 | ------ 47 | * Fix geo.json file converter 48 | 49 | 1.21.2 50 | ------ 51 | * Add support for paging through the domain information 52 | 53 | 1.21.1 54 | ------ 55 | * Add ``history`` and ``type`` parameters to ``Shodan.dns.domain_info()`` method and CLI command 56 | 57 | 1.21.0 58 | ------ 59 | * New API methods ``api.search_facets()`` and ``api.search_filters()`` to get a list of available facets and filters. 60 | 61 | 1.20.0 62 | ------ 63 | * New option "-S" for **shodan domain** to save results from the lookup 64 | * New option "-D" for **shodan domain** to lookup open ports for IPs in the results 65 | 66 | 1.19.0 67 | ------ 68 | * New method to edit the list of IPs for an existing network alert 69 | 70 | 1.18.0 71 | ------ 72 | * Add library methods for the new Notifications API 73 | 74 | 1.17.0 75 | ------ 76 | * Fix bug that caused unicode error when printing domain information (#106) 77 | * Add flag to let users get their IPv6 address **shodan myip -6**(#35) 78 | 79 | 1.16.0 80 | ------ 81 | * Ability to specify list of fields to include when converting to CSV/ Excel (#107) 82 | * Filter the Shodan Firehose based on tags in the banner 83 | 84 | 1.15.0 85 | ------ 86 | * New option "--skip" for download command to help users resume a download 87 | 88 | 1.14.0 89 | ------ 90 | * New command **shodan version** (#104). 91 | * Only change api_key file permissions if needed (#103) 92 | 93 | 1.13.0 94 | ------ 95 | * New command **shodan domain** to lookup a domain in Shodan's DNS database 96 | * Override environment configured settings if explicit proxy settings are supplied (@cudeso) 97 | 98 | 1.12.1 99 | ------ 100 | * Fix Excel file conversion that resulted in empty .xlsx files 101 | 102 | 1.12.0 103 | ------ 104 | * Add new methods to ignore/ unignore trigger notifications 105 | 106 | 1.11.1 107 | ------ 108 | * Allow a single network alert to monitor multiple IP ranges (#93) 109 | 110 | 1.11.0 111 | ------ 112 | * New command **shodan scan list** to list recently launched scans 113 | * New command **shodan alert triggers** to list the available notification triggers 114 | * New command **shodan alert enable** to enable a notification trigger 115 | * New command **shodan alert disable** to disable a notification trigger 116 | * New command **shodan alert info** to show details of a specific alert 117 | * Include timestamp, vulns and tags in CSV converter (#85) 118 | * Fixed bug that caused an exception when parsing uncompressed data files in Python3 119 | * Code quality improvements 120 | * Thank you for contributions from @wagner-certat, @cclauss, @opt9, @voldmar and Antoine Neuenschwander 121 | 122 | 1.10.4 123 | ------ 124 | * Fix a bug when showing old banner records that don't have the "transport" property 125 | * Code quality improvements (bare excepts) 126 | 127 | 1.10.3 128 | ------ 129 | * Change bare 'except:' statements to 'except Exception:' or more specific ones 130 | * remove unused imports 131 | * Convert line endings of `shodan/client.py` and `tests/test_shodan.py` to unix 132 | * List file types in **shodan convert** (#80) 133 | 134 | 1.10.2 135 | ------ 136 | * Fix **shodan stats** formatting exception when faceting on **port** 137 | 138 | 1.10.1 139 | ------ 140 | * Support PUT requests in the API request helper method 141 | 142 | 1.10.0 143 | ------ 144 | * New command **shodan org**: manage enterprise access to Shodan for your team 145 | * Improved unicode handling (#78) 146 | * Remove deprecated API wrapper for shodanhq.com/api 147 | 148 | 1.9.1 149 | ----- 150 | * The CHANGELOG is now part of the packages. 151 | * Improved unicode handling in Python2 (#78) 152 | * Add `tsv` output format for **shodan host** (#65) 153 | * Show user-friendly error messages when running **shodan radar** without permission or in a window that's too small (#74) 154 | * Improved exception handling to improve debugging **shodan init** (#77) 155 | 156 | 1.9.0 157 | ----- 158 | * New optional parameter `proxies` for all interfaces to specify a proxy array for the requests library (#72) 159 | 160 | 1.8.1 161 | ----- 162 | * Fixed bug that prevented **shodan scan submit** from finishing (#70) 163 | 164 | 1.8.0 165 | ----- 166 | * Shodan CLI now installs properly on Windows (#66) 167 | * Improved output of "shodan host" (#64, #67) 168 | * Fixed bug that prevented an open port from being shown in "shodan host" (#63) 169 | * No longer show an empty page if "shodan search" didn't return results (#62) 170 | * Updated docs to make them Python3 compatible 171 | 172 | 1.7.7 173 | ----- 174 | * Added "shodan data download" command to help download bulk data files 175 | 176 | 1.7.6 177 | ----- 178 | * Add basic support for the Bulk Data API 179 | 180 | 1.7.5 181 | ----- 182 | * Handle Cloudflare timeouts 183 | 184 | 1.7.4 185 | ----- 186 | * Added "shodan radar" command 187 | 188 | 1.7.3 189 | ----- 190 | * Fixed the bug #47 which was caused by the CLI using a timeout value of "0" which resulted in the "requests" library failing to connect 191 | 192 | 1.7.2 193 | ----- 194 | * stream: automatically decode to unicode, fixes streaming on python3 (#45) 195 | * Include docs in packages (#46) 196 | * stream: handle timeout=None, None (default) can't be compared with integers (#44) 197 | 198 | 1.7.1 199 | ----- 200 | * Python3 fixes for outputting images (#42) 201 | * Add the ability to save results from host lookups via the CLI (#43) 202 | 203 | 1.7.0 204 | ----- 205 | * Added "images" convert output format to let users extract images from Shodan data files (#42) 206 | -------------------------------------------------------------------------------- /shodan/stream.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import json 3 | import ssl 4 | 5 | from .exception import APIError 6 | 7 | 8 | class Stream: 9 | 10 | base_url = 'https://stream.shodan.io' 11 | 12 | def __init__(self, api_key, proxies=None): 13 | self.api_key = api_key 14 | self.proxies = proxies 15 | 16 | def _create_stream(self, name, query=None, timeout=None): 17 | params = { 18 | 'key': self.api_key, 19 | } 20 | stream_url = self.base_url + name 21 | 22 | # The user doesn't want to use a timeout 23 | # If the timeout is specified as 0 then we also don't want to have a timeout 24 | if (timeout and timeout <= 0) or (timeout == 0): 25 | timeout = None 26 | 27 | # If the user requested a timeout then we need to disable heartbeat messages 28 | # which are intended to keep stream connections alive even if there isn't any data 29 | # flowing through. 30 | if timeout: 31 | params['heartbeat'] = False 32 | 33 | if query is not None: 34 | params['query'] = query 35 | 36 | try: 37 | while True: 38 | req = requests.get(stream_url, params=params, stream=True, timeout=timeout, 39 | proxies=self.proxies) 40 | 41 | # Status code 524 is special to Cloudflare 42 | # It means that no data was sent from the streaming servers which caused Cloudflare 43 | # to terminate the connection. 44 | # 45 | # We only want to exit if there was a timeout specified or the HTTP status code is 46 | # not specific to Cloudflare. 47 | if req.status_code != 524 or timeout >= 0: 48 | break 49 | except Exception: 50 | raise APIError('Unable to contact the Shodan Streaming API') 51 | 52 | if req.status_code != 200: 53 | try: 54 | data = json.loads(req.text) 55 | raise APIError(data['error']) 56 | except APIError: 57 | raise 58 | except Exception: 59 | pass 60 | raise APIError('Invalid API key or you do not have access to the Streaming API') 61 | if req.encoding is None: 62 | req.encoding = 'utf-8' 63 | return req 64 | 65 | def _iter_stream(self, stream, raw): 66 | for line in stream.iter_lines(): 67 | # The Streaming API sends out heartbeat messages that are newlines 68 | # We want to ignore those messages since they don't contain any data 69 | if line: 70 | if raw: 71 | yield line 72 | else: 73 | yield json.loads(line) 74 | 75 | def alert(self, aid=None, timeout=None, raw=False): 76 | if aid: 77 | stream = self._create_stream('/shodan/alert/{}'.format(aid), timeout=timeout) 78 | else: 79 | stream = self._create_stream('/shodan/alert', timeout=timeout) 80 | 81 | try: 82 | for line in self._iter_stream(stream, raw): 83 | yield line 84 | except requests.exceptions.ConnectionError: 85 | raise APIError('Stream timed out') 86 | except ssl.SSLError: 87 | raise APIError('Stream timed out') 88 | 89 | def asn(self, asn, raw=False, timeout=None): 90 | """ 91 | A filtered version of the "banners" stream to only return banners that match the ASNs of interest. 92 | 93 | :param asn: A list of ASN to return banner data on. 94 | :type asn: string[] 95 | """ 96 | stream = self._create_stream('/shodan/asn/{}'.format(','.join(asn)), timeout=timeout) 97 | for line in self._iter_stream(stream, raw): 98 | yield line 99 | 100 | def banners(self, raw=False, timeout=None): 101 | """A real-time feed of the data that Shodan is currently collecting. Note that this is only available to 102 | API subscription plans and for those it only returns a fraction of the data. 103 | """ 104 | stream = self._create_stream('/shodan/banners', timeout=timeout) 105 | for line in self._iter_stream(stream, raw): 106 | yield line 107 | 108 | def countries(self, countries, raw=False, timeout=None): 109 | """ 110 | A filtered version of the "banners" stream to only return banners that match the countries of interest. 111 | 112 | :param countries: A list of countries to return banner data on. 113 | :type countries: string[] 114 | """ 115 | stream = self._create_stream('/shodan/countries/{}'.format(','.join(countries)), timeout=timeout) 116 | for line in self._iter_stream(stream, raw): 117 | yield line 118 | 119 | def custom(self, query, raw=False, timeout=None): 120 | """ 121 | A filtered version of the "banners" stream to only return banners that match the query of interest. The query 122 | can vary and mix-match with different arguments (ports, tags, vulns, etc). 123 | 124 | :param query: A space-separated list of key:value filters query to return banner data on. 125 | :type query: string 126 | """ 127 | stream = self._create_stream('/shodan/custom', query=query, timeout=timeout) 128 | for line in self._iter_stream(stream, raw): 129 | yield line 130 | 131 | def ports(self, ports, raw=False, timeout=None): 132 | """ 133 | A filtered version of the "banners" stream to only return banners that match the ports of interest. 134 | 135 | :param ports: A list of ports to return banner data on. 136 | :type ports: int[] 137 | """ 138 | stream = self._create_stream('/shodan/ports/{}'.format(','.join([str(port) for port in ports])), timeout=timeout) 139 | for line in self._iter_stream(stream, raw): 140 | yield line 141 | 142 | def tags(self, tags, raw=False, timeout=None): 143 | """ 144 | A filtered version of the "banners" stream to only return banners that match the tags of interest. 145 | 146 | :param tags: A list of tags to return banner data on. 147 | :type tags: string[] 148 | """ 149 | stream = self._create_stream('/shodan/tags/{}'.format(','.join(tags)), timeout=timeout) 150 | for line in self._iter_stream(stream, raw): 151 | yield line 152 | 153 | def vulns(self, vulns, raw=False, timeout=None): 154 | """ 155 | A filtered version of the "banners" stream to only return banners that match the vulnerabilities of interest. 156 | 157 | :param vulns: A list of vulns to return banner data on. 158 | :type vulns: string[] 159 | """ 160 | stream = self._create_stream('/shodan/vulns/{}'.format(','.join(vulns)), timeout=timeout) 161 | for line in self._iter_stream(stream, raw): 162 | yield line 163 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 36 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 37 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 38 | @echo " text to make text files" 39 | @echo " man to make manual pages" 40 | @echo " texinfo to make Texinfo files" 41 | @echo " info to make Texinfo files and run them through makeinfo" 42 | @echo " gettext to make PO message catalogs" 43 | @echo " changes to make an overview of all changed/added/deprecated items" 44 | @echo " xml to make Docutils-native XML files" 45 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 46 | @echo " linkcheck to check all external links for integrity" 47 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 48 | 49 | clean: 50 | rm -rf $(BUILDDIR)/* 51 | 52 | html: 53 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 54 | @echo 55 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 56 | 57 | dirhtml: 58 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 59 | @echo 60 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 61 | 62 | singlehtml: 63 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 64 | @echo 65 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 66 | 67 | pickle: 68 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 69 | @echo 70 | @echo "Build finished; now you can process the pickle files." 71 | 72 | json: 73 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 74 | @echo 75 | @echo "Build finished; now you can process the JSON files." 76 | 77 | htmlhelp: 78 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 79 | @echo 80 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 81 | ".hhp project file in $(BUILDDIR)/htmlhelp." 82 | 83 | qthelp: 84 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 85 | @echo 86 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 87 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 88 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/shodan-python.qhcp" 89 | @echo "To view the help file:" 90 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/shodan-python.qhc" 91 | 92 | devhelp: 93 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 94 | @echo 95 | @echo "Build finished." 96 | @echo "To view the help file:" 97 | @echo "# mkdir -p $$HOME/.local/share/devhelp/shodan-python" 98 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/shodan-python" 99 | @echo "# devhelp" 100 | 101 | epub: 102 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 103 | @echo 104 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 105 | 106 | latex: 107 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 108 | @echo 109 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 110 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 111 | "(use \`make latexpdf' here to do that automatically)." 112 | 113 | latexpdf: 114 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 115 | @echo "Running LaTeX files through pdflatex..." 116 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 117 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 118 | 119 | latexpdfja: 120 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 121 | @echo "Running LaTeX files through platex and dvipdfmx..." 122 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 123 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 124 | 125 | text: 126 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 127 | @echo 128 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 129 | 130 | man: 131 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 132 | @echo 133 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 134 | 135 | texinfo: 136 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 137 | @echo 138 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 139 | @echo "Run \`make' in that directory to run these through makeinfo" \ 140 | "(use \`make info' here to do that automatically)." 141 | 142 | info: 143 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 144 | @echo "Running Texinfo files through makeinfo..." 145 | make -C $(BUILDDIR)/texinfo info 146 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 147 | 148 | gettext: 149 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 150 | @echo 151 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 152 | 153 | changes: 154 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 155 | @echo 156 | @echo "The overview file is in $(BUILDDIR)/changes." 157 | 158 | linkcheck: 159 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 160 | @echo 161 | @echo "Link check complete; look for any errors in the above output " \ 162 | "or in $(BUILDDIR)/linkcheck/output.txt." 163 | 164 | doctest: 165 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 166 | @echo "Testing of doctests in the sources finished, look at the " \ 167 | "results in $(BUILDDIR)/doctest/output.txt." 168 | 169 | xml: 170 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 171 | @echo 172 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 173 | 174 | pseudoxml: 175 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 176 | @echo 177 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 178 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | set I18NSPHINXOPTS=%SPHINXOPTS% . 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. xml to make Docutils-native XML files 37 | echo. pseudoxml to make pseudoxml-XML files for display purposes 38 | echo. linkcheck to check all external links for integrity 39 | echo. doctest to run all doctests embedded in the documentation if enabled 40 | goto end 41 | ) 42 | 43 | if "%1" == "clean" ( 44 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 45 | del /q /s %BUILDDIR%\* 46 | goto end 47 | ) 48 | 49 | 50 | %SPHINXBUILD% 2> nul 51 | if errorlevel 9009 ( 52 | echo. 53 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 54 | echo.installed, then set the SPHINXBUILD environment variable to point 55 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 56 | echo.may add the Sphinx directory to PATH. 57 | echo. 58 | echo.If you don't have Sphinx installed, grab it from 59 | echo.http://sphinx-doc.org/ 60 | exit /b 1 61 | ) 62 | 63 | if "%1" == "html" ( 64 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 65 | if errorlevel 1 exit /b 1 66 | echo. 67 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 68 | goto end 69 | ) 70 | 71 | if "%1" == "dirhtml" ( 72 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 73 | if errorlevel 1 exit /b 1 74 | echo. 75 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 76 | goto end 77 | ) 78 | 79 | if "%1" == "singlehtml" ( 80 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 81 | if errorlevel 1 exit /b 1 82 | echo. 83 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 84 | goto end 85 | ) 86 | 87 | if "%1" == "pickle" ( 88 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 89 | if errorlevel 1 exit /b 1 90 | echo. 91 | echo.Build finished; now you can process the pickle files. 92 | goto end 93 | ) 94 | 95 | if "%1" == "json" ( 96 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 97 | if errorlevel 1 exit /b 1 98 | echo. 99 | echo.Build finished; now you can process the JSON files. 100 | goto end 101 | ) 102 | 103 | if "%1" == "htmlhelp" ( 104 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 105 | if errorlevel 1 exit /b 1 106 | echo. 107 | echo.Build finished; now you can run HTML Help Workshop with the ^ 108 | .hhp project file in %BUILDDIR%/htmlhelp. 109 | goto end 110 | ) 111 | 112 | if "%1" == "qthelp" ( 113 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 114 | if errorlevel 1 exit /b 1 115 | echo. 116 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 117 | .qhcp project file in %BUILDDIR%/qthelp, like this: 118 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\shodan-python.qhcp 119 | echo.To view the help file: 120 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\shodan-python.ghc 121 | goto end 122 | ) 123 | 124 | if "%1" == "devhelp" ( 125 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished. 129 | goto end 130 | ) 131 | 132 | if "%1" == "epub" ( 133 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 134 | if errorlevel 1 exit /b 1 135 | echo. 136 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 137 | goto end 138 | ) 139 | 140 | if "%1" == "latex" ( 141 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 142 | if errorlevel 1 exit /b 1 143 | echo. 144 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 145 | goto end 146 | ) 147 | 148 | if "%1" == "latexpdf" ( 149 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 150 | cd %BUILDDIR%/latex 151 | make all-pdf 152 | cd %BUILDDIR%/.. 153 | echo. 154 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 155 | goto end 156 | ) 157 | 158 | if "%1" == "latexpdfja" ( 159 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 160 | cd %BUILDDIR%/latex 161 | make all-pdf-ja 162 | cd %BUILDDIR%/.. 163 | echo. 164 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 165 | goto end 166 | ) 167 | 168 | if "%1" == "text" ( 169 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 170 | if errorlevel 1 exit /b 1 171 | echo. 172 | echo.Build finished. The text files are in %BUILDDIR%/text. 173 | goto end 174 | ) 175 | 176 | if "%1" == "man" ( 177 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 178 | if errorlevel 1 exit /b 1 179 | echo. 180 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 181 | goto end 182 | ) 183 | 184 | if "%1" == "texinfo" ( 185 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 186 | if errorlevel 1 exit /b 1 187 | echo. 188 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 189 | goto end 190 | ) 191 | 192 | if "%1" == "gettext" ( 193 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 194 | if errorlevel 1 exit /b 1 195 | echo. 196 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 197 | goto end 198 | ) 199 | 200 | if "%1" == "changes" ( 201 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 202 | if errorlevel 1 exit /b 1 203 | echo. 204 | echo.The overview file is in %BUILDDIR%/changes. 205 | goto end 206 | ) 207 | 208 | if "%1" == "linkcheck" ( 209 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 210 | if errorlevel 1 exit /b 1 211 | echo. 212 | echo.Link check complete; look for any errors in the above output ^ 213 | or in %BUILDDIR%/linkcheck/output.txt. 214 | goto end 215 | ) 216 | 217 | if "%1" == "doctest" ( 218 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 219 | if errorlevel 1 exit /b 1 220 | echo. 221 | echo.Testing of doctests in the sources finished, look at the ^ 222 | results in %BUILDDIR%/doctest/output.txt. 223 | goto end 224 | ) 225 | 226 | if "%1" == "xml" ( 227 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 228 | if errorlevel 1 exit /b 1 229 | echo. 230 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 231 | goto end 232 | ) 233 | 234 | if "%1" == "pseudoxml" ( 235 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 236 | if errorlevel 1 exit /b 1 237 | echo. 238 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 239 | goto end 240 | ) 241 | 242 | :end 243 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # shodan-python documentation build configuration file, created by 4 | # sphinx-quickstart on Thu Jan 23 00:56:29 2014. 5 | # 6 | # This file is execfile()d with the current directory set to its 7 | # containing dir. 8 | # 9 | # Note that not all possible configuration values are present in this 10 | # autogenerated file. 11 | # 12 | # All configuration values have a default; values that are commented out 13 | # serve to show the default. 14 | 15 | import sys 16 | import os 17 | 18 | # If extensions (or modules to document with autodoc) are in another directory, 19 | # add these directories to sys.path here. If the directory is relative to the 20 | # documentation root, use os.path.abspath to make it absolute, like shown here. 21 | #sys.path.insert(0, os.path.abspath('.')) 22 | 23 | # -- General configuration ------------------------------------------------ 24 | 25 | # If your documentation needs a minimal Sphinx version, state it here. 26 | #needs_sphinx = '1.0' 27 | 28 | # Add any Sphinx extension module names here, as strings. They can be 29 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 30 | # ones. 31 | extensions = [ 32 | 'sphinx.ext.autodoc', 33 | 'sphinx.ext.todo', 34 | 'sphinx.ext.viewcode', 35 | ] 36 | 37 | # Add any paths that contain templates here, relative to this directory. 38 | templates_path = ['_templates'] 39 | 40 | # The suffix of source filenames. 41 | source_suffix = '.rst' 42 | 43 | # The encoding of source files. 44 | #source_encoding = 'utf-8-sig' 45 | 46 | # The master toctree document. 47 | master_doc = 'index' 48 | 49 | # General information about the project. 50 | project = u'shodan-python' 51 | copyright = u'2014, achillean' 52 | 53 | # The version info for the project you're documenting, acts as replacement for 54 | # |version| and |release|, also used in various other places throughout the 55 | # built documents. 56 | # 57 | # The short X.Y version. 58 | version = '1.0' 59 | # The full version, including alpha/beta/rc tags. 60 | release = '1.0' 61 | 62 | # The language for content autogenerated by Sphinx. Refer to documentation 63 | # for a list of supported languages. 64 | #language = None 65 | 66 | # There are two options for replacing |today|: either, you set today to some 67 | # non-false value, then it is used: 68 | #today = '' 69 | # Else, today_fmt is used as the format for a strftime call. 70 | #today_fmt = '%B %d, %Y' 71 | 72 | # List of patterns, relative to source directory, that match files and 73 | # directories to ignore when looking for source files. 74 | exclude_patterns = ['_build'] 75 | 76 | # The reST default role (used for this markup: `text`) to use for all 77 | # documents. 78 | #default_role = None 79 | 80 | # If true, '()' will be appended to :func: etc. cross-reference text. 81 | #add_function_parentheses = True 82 | 83 | # If true, the current module name will be prepended to all description 84 | # unit titles (such as .. function::). 85 | #add_module_names = True 86 | 87 | # If true, sectionauthor and moduleauthor directives will be shown in the 88 | # output. They are ignored by default. 89 | #show_authors = False 90 | 91 | # The name of the Pygments (syntax highlighting) style to use. 92 | pygments_style = 'sphinx' 93 | 94 | # A list of ignored prefixes for module index sorting. 95 | #modindex_common_prefix = [] 96 | 97 | # If true, keep warnings as "system message" paragraphs in the built documents. 98 | #keep_warnings = False 99 | 100 | 101 | # -- Options for HTML output ---------------------------------------------- 102 | 103 | # The theme to use for HTML and HTML Help pages. See the documentation for 104 | # a list of builtin themes. 105 | html_theme = 'default' 106 | 107 | # Theme options are theme-specific and customize the look and feel of a theme 108 | # further. For a list of options available for each theme, see the 109 | # documentation. 110 | #html_theme_options = {} 111 | 112 | # Add any paths that contain custom themes here, relative to this directory. 113 | #html_theme_path = [] 114 | 115 | # The name for this set of Sphinx documents. If None, it defaults to 116 | # " v documentation". 117 | #html_title = None 118 | 119 | # A shorter title for the navigation bar. Default is the same as html_title. 120 | #html_short_title = None 121 | 122 | # The name of an image file (relative to this directory) to place at the top 123 | # of the sidebar. 124 | #html_logo = None 125 | 126 | # The name of an image file (within the static path) to use as favicon of the 127 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 128 | # pixels large. 129 | #html_favicon = None 130 | 131 | # Add any paths that contain custom static files (such as style sheets) here, 132 | # relative to this directory. They are copied after the builtin static files, 133 | # so a file named "default.css" will overwrite the builtin "default.css". 134 | html_static_path = ['_static'] 135 | 136 | # Add any extra paths that contain custom files (such as robots.txt or 137 | # .htaccess) here, relative to this directory. These files are copied 138 | # directly to the root of the documentation. 139 | #html_extra_path = [] 140 | 141 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 142 | # using the given strftime format. 143 | #html_last_updated_fmt = '%b %d, %Y' 144 | 145 | # If true, SmartyPants will be used to convert quotes and dashes to 146 | # typographically correct entities. 147 | #html_use_smartypants = True 148 | 149 | # Custom sidebar templates, maps document names to template names. 150 | #html_sidebars = {} 151 | 152 | # Additional templates that should be rendered to pages, maps page names to 153 | # template names. 154 | #html_additional_pages = {} 155 | 156 | # If false, no module index is generated. 157 | #html_domain_indices = True 158 | 159 | # If false, no index is generated. 160 | #html_use_index = True 161 | 162 | # If true, the index is split into individual pages for each letter. 163 | #html_split_index = False 164 | 165 | # If true, links to the reST sources are added to the pages. 166 | #html_show_sourcelink = True 167 | 168 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 169 | #html_show_sphinx = True 170 | 171 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 172 | #html_show_copyright = True 173 | 174 | # If true, an OpenSearch description file will be output, and all pages will 175 | # contain a tag referring to it. The value of this option must be the 176 | # base URL from which the finished HTML is served. 177 | #html_use_opensearch = '' 178 | 179 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 180 | #html_file_suffix = None 181 | 182 | # Output file base name for HTML help builder. 183 | htmlhelp_basename = 'shodan-pythondoc' 184 | 185 | 186 | # -- Options for LaTeX output --------------------------------------------- 187 | 188 | latex_elements = { 189 | # The paper size ('letterpaper' or 'a4paper'). 190 | #'papersize': 'letterpaper', 191 | 192 | # The font size ('10pt', '11pt' or '12pt'). 193 | #'pointsize': '10pt', 194 | 195 | # Additional stuff for the LaTeX preamble. 196 | #'preamble': '', 197 | } 198 | 199 | # Grouping the document tree into LaTeX files. List of tuples 200 | # (source start file, target name, title, 201 | # author, documentclass [howto, manual, or own class]). 202 | latex_documents = [ 203 | ('index', 'shodan-python.tex', u'shodan-python Documentation', 204 | u'achillean', 'manual'), 205 | ] 206 | 207 | # The name of an image file (relative to this directory) to place at the top of 208 | # the title page. 209 | #latex_logo = None 210 | 211 | # For "manual" documents, if this is true, then toplevel headings are parts, 212 | # not chapters. 213 | #latex_use_parts = False 214 | 215 | # If true, show page references after internal links. 216 | #latex_show_pagerefs = False 217 | 218 | # If true, show URL addresses after external links. 219 | #latex_show_urls = False 220 | 221 | # Documents to append as an appendix to all manuals. 222 | #latex_appendices = [] 223 | 224 | # If false, no module index is generated. 225 | #latex_domain_indices = True 226 | 227 | 228 | # -- Options for manual page output --------------------------------------- 229 | 230 | # One entry per manual page. List of tuples 231 | # (source start file, name, description, authors, manual section). 232 | man_pages = [ 233 | ('index', 'shodan-python', u'shodan-python Documentation', 234 | [u'achillean'], 1) 235 | ] 236 | 237 | # If true, show URL addresses after external links. 238 | #man_show_urls = False 239 | 240 | 241 | # -- Options for Texinfo output ------------------------------------------- 242 | 243 | # Grouping the document tree into Texinfo files. List of tuples 244 | # (source start file, target name, title, author, 245 | # dir menu entry, description, category) 246 | texinfo_documents = [ 247 | ('index', 'shodan-python', u'shodan-python Documentation', 248 | u'achillean', 'shodan-python', 'One line description of project.', 249 | 'Miscellaneous'), 250 | ] 251 | 252 | # Documents to append as an appendix to all manuals. 253 | #texinfo_appendices = [] 254 | 255 | # If false, no module index is generated. 256 | #texinfo_domain_indices = True 257 | 258 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 259 | #texinfo_show_urls = 'footnote' 260 | 261 | # If true, do not generate a @detailmenu in the "Top" node's menu. 262 | #texinfo_no_detailmenu = False 263 | -------------------------------------------------------------------------------- /shodan/cli/worldmap.py: -------------------------------------------------------------------------------- 1 | ''' 2 | F-Secure Virus World Map console edition 3 | 4 | See README.md for more details 5 | 6 | Copyright 2012-2013 Jyrki Muukkonen 7 | 8 | Released under the MIT license. 9 | See LICENSE.txt or http://www.opensource.org/licenses/mit-license.php 10 | 11 | ASCII map in map-world-01.txt is copyright: 12 | "Map 1998 Matthew Thomas. Freely usable as long as this line is included" 13 | 14 | ''' 15 | import curses 16 | import locale 17 | import random 18 | import time 19 | 20 | from shodan.exception import APIError 21 | from shodan.helpers import get_ip 22 | 23 | 24 | MAPS = { 25 | 'world': { 26 | # offset (as (y, x) for curses...) 27 | 'corners': (1, 4, 23, 73), 28 | # lat top, lon left, lat bottom, lon right 29 | 'coords': [90.0, -180.0, -90.0, 180.0], 30 | 31 | # PyLint freaks out about the world map backslashes so ignore those warnings 32 | 'data': r''' 33 | . _..::__: ,-"-"._ |7 , _,.__ 34 | _.___ _ _<_>`!(._`.`-. / _._ `_ ,_/ ' '-._.---.-.__ 35 | .{ " " `-==,',._\{ \ / {) / _ ">_,-' ` mt-2_ 36 | \_.:--. `._ )`^-. "' , [_/( __,/-' 37 | '"' \ " _L oD_,--' ) /. (| 38 | | ,' _)_.\\._<> 6 _,' / ' 39 | `. / [_/_'` `"( <'} ) 40 | \\ .-. ) / `-'"..' `:._ _) ' 41 | ` \ ( `( / `:\ > \ ,-^. /' ' 42 | `._, "" | \`' \| ?_) {\ 43 | `=.---. `._._ ,' "` |' ,- '. 44 | | `-._ | / `:`<_|h--._ 45 | ( > . | , `=.__.`-'\ 46 | `. / | |{| ,-.,\ . 47 | | ,' \ / `' ," \ 48 | | / |_' | __ / 49 | | | '-' `-' \. 50 | |/ " / 51 | \. ' 52 | 53 | ,/ ______._.--._ _..---.---------._ 54 | ,-----"-..?----_/ ) _,-'" " ( 55 | Map 1998 Matthew Thomas. Freely usable as long as this line is included 56 | ''' 57 | } 58 | } 59 | 60 | 61 | class AsciiMap(object): 62 | """ 63 | Helper class for handling map drawing and coordinate calculations 64 | """ 65 | def __init__(self, map_name='world', map_conf=None, window=None, encoding=None): 66 | if map_conf is None: 67 | map_conf = MAPS[map_name] 68 | self.map = map_conf['data'] 69 | self.coords = map_conf['coords'] 70 | self.corners = map_conf['corners'] 71 | if window is None: 72 | window = curses.newwin(0, 0) 73 | self.window = window 74 | 75 | self.data = [] 76 | self.data_timestamp = None 77 | 78 | # JSON contents _should_ be UTF8 (so, python internal unicode here...) 79 | if encoding is None: 80 | encoding = locale.getpreferredencoding() 81 | self.encoding = encoding 82 | 83 | # check if we can use transparent background or not 84 | if curses.can_change_color(): 85 | curses.use_default_colors() 86 | background = -1 87 | else: 88 | background = curses.COLOR_BLACK 89 | 90 | tmp_colors = [ 91 | ('red', curses.COLOR_RED, background), 92 | ('blue', curses.COLOR_BLUE, background), 93 | ('pink', curses.COLOR_MAGENTA, background) 94 | ] 95 | 96 | self.colors = {} 97 | if curses.has_colors(): 98 | for i, (name, fgcolor, bgcolor) in enumerate(tmp_colors, 1): 99 | curses.init_pair(i, fgcolor, bgcolor) 100 | self.colors[name] = i 101 | 102 | def latlon_to_coords(self, lat, lon): 103 | """ 104 | Convert lat/lon coordinates to character positions. 105 | Very naive version, assumes that we are drawing the whole world 106 | TODO: filter out stuff that doesn't fit 107 | TODO: make it possible to use "zoomed" maps 108 | """ 109 | width = (self.corners[3] - self.corners[1]) 110 | height = (self.corners[2] - self.corners[0]) 111 | 112 | # change to 0-180, 0-360 113 | abs_lat = -lat + 90 114 | abs_lon = lon + 180 115 | x = (abs_lon / 360.0) * width + self.corners[1] 116 | y = (abs_lat / 180.0) * height + self.corners[0] 117 | return int(x), int(y) 118 | 119 | def set_data(self, data): 120 | """ 121 | Set / convert internal data. 122 | For now it just selects a random set to show. 123 | """ 124 | entries = [] 125 | 126 | # Grab 5 random banners to display 127 | for banner in random.sample(data, min(len(data), 5)): 128 | desc = '{} -> {} / {}'.format(get_ip(banner), banner['port'], banner['location']['country_code']) 129 | if banner['location']['city']: 130 | # Not all cities can be encoded in ASCII so ignore any errors 131 | try: 132 | desc += ' {}'.format(banner['location']['city']) 133 | except Exception: 134 | pass 135 | 136 | if 'tags' in banner and banner['tags']: 137 | desc += ' / {}'.format(','.join(banner['tags'])) 138 | 139 | entry = ( 140 | float(banner['location']['latitude']), 141 | float(banner['location']['longitude']), 142 | '*', 143 | desc, 144 | curses.A_BOLD, 145 | 'red', 146 | ) 147 | entries.append(entry) 148 | self.data = entries 149 | 150 | def draw(self, target): 151 | """ Draw internal data to curses window """ 152 | self.window.clear() 153 | self.window.addstr(0, 0, self.map) 154 | 155 | # FIXME: position to be defined in map config? 156 | row = self.corners[2] - 6 157 | items_to_show = 5 158 | for lat, lon, char, desc, attrs, color in self.data: 159 | # to make this work almost everywhere. see http://docs.python.org/2/library/curses.html 160 | if desc: 161 | desc = desc.encode(self.encoding, 'ignore').decode() 162 | if items_to_show <= 0: 163 | break 164 | char_x, char_y = self.latlon_to_coords(lat, lon) 165 | if self.colors and color: 166 | attrs |= curses.color_pair(self.colors[color]) 167 | self.window.addstr(char_y, char_x, char, attrs) 168 | if desc: 169 | det_show = "{} {}".format(char, desc) 170 | else: 171 | det_show = None 172 | 173 | if det_show is not None: 174 | try: 175 | self.window.addstr(row, 1, det_show, attrs) 176 | row += 1 177 | items_to_show -= 1 178 | except Exception: 179 | # FIXME: check window size before addstr() 180 | break 181 | self.window.overwrite(target) 182 | self.window.leaveok(True) 183 | 184 | 185 | class MapApp(object): 186 | """ Virus World Map ncurses application """ 187 | def __init__(self, api): 188 | self.api = api 189 | self.data = None 190 | self.last_fetch = 0 191 | self.sleep = 10 # tenths of seconds, for curses.halfdelay() 192 | self.polling_interval = 60 193 | 194 | def fetch_data(self, epoch_now, force_refresh=False): 195 | """ (Re)fetch data from JSON stream """ 196 | refresh = False 197 | if force_refresh or self.data is None: 198 | refresh = True 199 | else: 200 | if self.last_fetch + self.polling_interval <= epoch_now: 201 | refresh = True 202 | 203 | if refresh: 204 | try: 205 | # Grab 20 banners from the main stream 206 | banners = [] 207 | for banner in self.api.stream.banners(): 208 | if 'location' in banner and banner['location']['latitude']: 209 | banners.append(banner) 210 | if len(banners) >= 20: 211 | break 212 | self.data = banners 213 | self.last_fetch = epoch_now 214 | except APIError: 215 | raise 216 | return refresh 217 | 218 | def run(self, scr): 219 | """ Initialize and run the application """ 220 | m = AsciiMap() 221 | curses.halfdelay(self.sleep) 222 | while True: 223 | now = int(time.time()) 224 | refresh = self.fetch_data(now) 225 | m.set_data(self.data) 226 | try: 227 | m.draw(scr) 228 | except curses.error: 229 | raise Exception('Terminal window too small') 230 | scr.addstr(0, 1, 'Shodan Radar', curses.A_BOLD) 231 | scr.addstr(0, 40, time.strftime("%c UTC", time.gmtime(now)).rjust(37), curses.A_BOLD) 232 | 233 | # Key Input 234 | # q - Quit 235 | event = scr.getch() 236 | if event == ord('q'): 237 | break 238 | 239 | # redraw window (to fix encoding/rendering bugs and to hide other messages to same tty) 240 | # user pressed 'r' or new data was fetched 241 | if refresh: 242 | m.window.redrawwin() 243 | 244 | 245 | def launch_map(api): 246 | app = MapApp(api) 247 | return curses.wrapper(app.run) 248 | 249 | 250 | def main(argv=None): 251 | """ Main function / entry point """ 252 | from shodan import Shodan 253 | from shodan.cli.helpers import get_api_key 254 | 255 | api = Shodan(get_api_key()) 256 | return launch_map(api) 257 | 258 | 259 | if __name__ == '__main__': 260 | import sys 261 | sys.exit(main()) 262 | -------------------------------------------------------------------------------- /shodan/cli/scan.py: -------------------------------------------------------------------------------- 1 | import click 2 | import collections 3 | import datetime 4 | import shodan 5 | import shodan.helpers as helpers 6 | import socket 7 | import threading 8 | import time 9 | 10 | from shodan.cli.helpers import get_api_key, async_spinner 11 | from shodan.cli.settings import COLORIZE_FIELDS 12 | 13 | 14 | @click.group() 15 | def scan(): 16 | """Scan an IP/ netblock using Shodan.""" 17 | pass 18 | 19 | 20 | @scan.command(name='list') 21 | def scan_list(): 22 | """Show recently launched scans""" 23 | key = get_api_key() 24 | 25 | # Get the list 26 | api = shodan.Shodan(key) 27 | try: 28 | scans = api.scans() 29 | except shodan.APIError as e: 30 | raise click.ClickException(e.value) 31 | 32 | if len(scans) > 0: 33 | click.echo(u'# {} Scans Total - Showing 10 most recent scans:'.format(scans['total'])) 34 | click.echo(u'# {:20} {:<15} {:<10} {:<15s}'.format('Scan ID', 'Status', 'Size', 'Timestamp')) 35 | # click.echo('#' * 65) 36 | for scan in scans['matches'][:10]: 37 | click.echo( 38 | u'{:31} {:<24} {:<10} {:<15s}'.format( 39 | click.style(scan['id'], fg='yellow'), 40 | click.style(scan['status'], fg='cyan'), 41 | scan['size'], 42 | scan['created'] 43 | ) 44 | ) 45 | else: 46 | click.echo("You haven't yet launched any scans.") 47 | 48 | 49 | @scan.command(name='internet') 50 | @click.option('--quiet', help='Disable the printing of information to the screen.', default=False, is_flag=True) 51 | @click.argument('port', type=int) 52 | @click.argument('protocol', type=str) 53 | def scan_internet(quiet, port, protocol): 54 | """Scan the Internet for a specific port and protocol using the Shodan infrastructure.""" 55 | key = get_api_key() 56 | api = shodan.Shodan(key) 57 | 58 | try: 59 | # Submit the request to Shodan 60 | click.echo('Submitting Internet scan to Shodan...', nl=False) 61 | scan = api.scan_internet(port, protocol) 62 | click.echo('Done') 63 | 64 | # If the requested port is part of the regular Shodan crawling, then 65 | # we don't know when the scan is done so lets return immediately and 66 | # let the user decide when to stop waiting for further results. 67 | official_ports = api.ports() 68 | if port in official_ports: 69 | click.echo('The requested port is already indexed by Shodan. A new scan for the port has been launched, please subscribe to the real-time stream for results.') 70 | else: 71 | # Create the output file 72 | filename = '{0}-{1}.json.gz'.format(port, protocol) 73 | counter = 0 74 | with helpers.open_file(filename, 'w') as fout: 75 | click.echo('Saving results to file: {0}'.format(filename)) 76 | 77 | # Start listening for results 78 | done = False 79 | 80 | # Keep listening for results until the scan is done 81 | click.echo('Waiting for data, please stand by...') 82 | while not done: 83 | try: 84 | for banner in api.stream.ports([port], timeout=90): 85 | counter += 1 86 | helpers.write_banner(fout, banner) 87 | 88 | if not quiet: 89 | click.echo('{0:<40} {1:<20} {2}'.format( 90 | click.style(helpers.get_ip(banner), fg=COLORIZE_FIELDS['ip_str']), 91 | click.style(str(banner['port']), fg=COLORIZE_FIELDS['port']), 92 | ';'.join(banner['hostnames'])) 93 | ) 94 | except shodan.APIError: 95 | # We stop waiting for results if the scan has been processed by the crawlers and 96 | # there haven't been new results in a while 97 | if done: 98 | break 99 | 100 | scan = api.scan_status(scan['id']) 101 | if scan['status'] == 'DONE': 102 | done = True 103 | except socket.timeout: 104 | # We stop waiting for results if the scan has been processed by the crawlers and 105 | # there haven't been new results in a while 106 | if done: 107 | break 108 | 109 | scan = api.scan_status(scan['id']) 110 | if scan['status'] == 'DONE': 111 | done = True 112 | except Exception as e: 113 | raise click.ClickException(repr(e)) 114 | click.echo('Scan finished: {0} devices found'.format(counter)) 115 | except shodan.APIError as e: 116 | raise click.ClickException(e.value) 117 | 118 | 119 | @scan.command(name='protocols') 120 | def scan_protocols(): 121 | """List the protocols that you can scan with using Shodan.""" 122 | key = get_api_key() 123 | api = shodan.Shodan(key) 124 | try: 125 | protocols = api.protocols() 126 | 127 | for name, description in iter(protocols.items()): 128 | click.echo(click.style('{0:<30}'.format(name), fg='cyan') + description) 129 | except shodan.APIError as e: 130 | raise click.ClickException(e.value) 131 | 132 | 133 | @scan.command(name='submit') 134 | @click.option('--wait', help='How long to wait for results to come back. If this is set to "0" or below return immediately.', default=20, type=int) 135 | @click.option('--filename', help='Save the results in the given file.', default='', type=str) 136 | @click.option('--force', default=False, is_flag=True) 137 | @click.option('--verbose', default=False, is_flag=True) 138 | @click.argument('netblocks', metavar='', nargs=-1) 139 | def scan_submit(wait, filename, force, verbose, netblocks): 140 | """Scan an IP/ netblock using Shodan.""" 141 | key = get_api_key() 142 | api = shodan.Shodan(key) 143 | alert = None 144 | 145 | # Submit the IPs for scanning 146 | try: 147 | # Submit the scan 148 | scan = api.scan(netblocks, force=force) 149 | 150 | now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M') 151 | 152 | click.echo('') 153 | click.echo('Starting Shodan scan at {} - {} scan credits left'.format(now, scan['credits_left'])) 154 | 155 | if verbose: 156 | click.echo('# Scan ID: {}'.format(scan['id'])) 157 | 158 | # Return immediately 159 | if wait <= 0: 160 | click.echo('Scan ID: {}'.format(scan['id'])) 161 | click.echo('Exiting now, not waiting for results. Use the API or website to retrieve the results of the scan.') 162 | else: 163 | # Setup an alert to wait for responses 164 | alert = api.create_alert('Scan: {}'.format(', '.join(netblocks)), netblocks) 165 | 166 | # Create the output file if necessary 167 | filename = filename.strip() 168 | fout = None 169 | if filename != '': 170 | # Add the appropriate extension if it's not there atm 171 | if not filename.endswith('.json.gz'): 172 | filename += '.json.gz' 173 | fout = helpers.open_file(filename, 'w') 174 | 175 | # Start a spinner 176 | finished_event = threading.Event() 177 | progress_bar_thread = threading.Thread(target=async_spinner, args=(finished_event,)) 178 | progress_bar_thread.start() 179 | 180 | # Now wait a few seconds for items to get returned 181 | hosts = collections.defaultdict(dict) 182 | done = False 183 | scan_start = time.time() 184 | cache = {} 185 | while not done: 186 | try: 187 | for banner in api.stream.alert(aid=alert['id'], timeout=wait): 188 | ip = banner.get('ip', banner.get('ipv6', None)) 189 | if not ip: 190 | continue 191 | 192 | # Don't show duplicate banners 193 | cache_key = '{}:{}'.format(ip, banner['port']) 194 | if cache_key not in cache: 195 | hosts[helpers.get_ip(banner)][banner['port']] = banner 196 | cache[cache_key] = True 197 | 198 | # If we've grabbed data for more than 60 seconds it might just be a busy network and we should move on 199 | if time.time() - scan_start >= 60: 200 | scan = api.scan_status(scan['id']) 201 | 202 | if verbose: 203 | click.echo('# Scan status: {}'.format(scan['status'])) 204 | 205 | if scan['status'] == 'DONE': 206 | done = True 207 | break 208 | 209 | except shodan.APIError: 210 | # If the connection timed out before the timeout, that means the streaming server 211 | # that the user tried to reach is down. In that case, lets wait briefly and try 212 | # to connect again! 213 | if (time.time() - scan_start) < wait: 214 | time.sleep(0.5) 215 | continue 216 | 217 | # Exit if the scan was flagged as done somehow 218 | if done: 219 | break 220 | 221 | scan = api.scan_status(scan['id']) 222 | if scan['status'] == 'DONE': 223 | done = True 224 | 225 | if verbose: 226 | click.echo('# Scan status: {}'.format(scan['status'])) 227 | except socket.timeout: 228 | # If the connection timed out before the timeout, that means the streaming server 229 | # that the user tried to reach is down. In that case, lets wait a second and try 230 | # to connect again! 231 | if (time.time() - scan_start) < wait: 232 | continue 233 | 234 | done = True 235 | except Exception as e: 236 | finished_event.set() 237 | progress_bar_thread.join() 238 | raise click.ClickException(repr(e)) 239 | 240 | finished_event.set() 241 | progress_bar_thread.join() 242 | 243 | def print_field(name, value): 244 | click.echo(' {:25s}{}'.format(name, value)) 245 | 246 | def print_banner(banner): 247 | click.echo(' {:20s}'.format(click.style(str(banner['port']), fg='green') + '/' + banner['transport']), nl=False) 248 | 249 | if 'product' in banner: 250 | click.echo(banner['product'], nl=False) 251 | 252 | if 'version' in banner: 253 | click.echo(' ({})'.format(banner['version']), nl=False) 254 | 255 | click.echo('') 256 | 257 | # Show optional ssl info 258 | if 'ssl' in banner: 259 | if 'versions' in banner['ssl']: 260 | # Only print SSL versions if they were successfully tested 261 | versions = [version for version in sorted(banner['ssl']['versions']) if not version.startswith('-')] 262 | if len(versions) > 0: 263 | click.echo(' |-- SSL Versions: {}'.format(', '.join(versions))) 264 | if 'dhparams' in banner['ssl'] and banner['ssl']['dhparams']: 265 | click.echo(' |-- Diffie-Hellman Parameters:') 266 | click.echo(' {:15s}{}\n {:15s}{}'.format('Bits:', banner['ssl']['dhparams']['bits'], 'Generator:', banner['ssl']['dhparams']['generator'])) 267 | if 'fingerprint' in banner['ssl']['dhparams']: 268 | click.echo(' {:15s}{}'.format('Fingerprint:', banner['ssl']['dhparams']['fingerprint'])) 269 | 270 | if hosts: 271 | # Remove the remaining spinner character 272 | click.echo('\b ') 273 | 274 | for ip in sorted(hosts): 275 | host = next(iter(hosts[ip].items()))[1] 276 | 277 | click.echo(click.style(ip, fg='cyan'), nl=False) 278 | if 'hostnames' in host and host['hostnames']: 279 | click.echo(' ({})'.format(', '.join(host['hostnames'])), nl=False) 280 | click.echo('') 281 | 282 | if 'location' in host and 'country_name' in host['location'] and host['location']['country_name']: 283 | print_field('Country', host['location']['country_name']) 284 | 285 | if 'city' in host['location'] and host['location']['city']: 286 | print_field('City', host['location']['city']) 287 | if 'org' in host and host['org']: 288 | print_field('Organization', host['org']) 289 | if 'os' in host and host['os']: 290 | print_field('Operating System', host['os']) 291 | click.echo('') 292 | 293 | # Output the vulnerabilities the host has 294 | if 'vulns' in host and len(host['vulns']) > 0: 295 | vulns = [] 296 | for vuln in host['vulns']: 297 | if vuln.startswith('!'): 298 | continue 299 | if vuln.upper() == 'CVE-2014-0160': 300 | vulns.append(click.style('Heartbleed', fg='red')) 301 | else: 302 | vulns.append(click.style(vuln, fg='red')) 303 | 304 | if len(vulns) > 0: 305 | click.echo(' {:25s}'.format('Vulnerabilities:'), nl=False) 306 | 307 | for vuln in vulns: 308 | click.echo(vuln + '\t', nl=False) 309 | 310 | click.echo('') 311 | 312 | # Print all the open ports: 313 | click.echo(' Open Ports:') 314 | for port in sorted(hosts[ip]): 315 | print_banner(hosts[ip][port]) 316 | 317 | # Save the banner in a file if necessary 318 | if fout: 319 | helpers.write_banner(fout, hosts[ip][port]) 320 | 321 | click.echo('') 322 | else: 323 | # Prepend a \b to remove the spinner 324 | click.echo('\bNo open ports found or the host has been recently crawled and cant get scanned again so soon.') 325 | except shodan.APIError as e: 326 | raise click.ClickException(e.value) 327 | finally: 328 | # Remove any alert 329 | if alert: 330 | api.delete_alert(alert['id']) 331 | 332 | 333 | @scan.command(name='status') 334 | @click.argument('scan_id', type=str) 335 | def scan_status(scan_id): 336 | """Check the status of an on-demand scan.""" 337 | key = get_api_key() 338 | api = shodan.Shodan(key) 339 | try: 340 | scan = api.scan_status(scan_id) 341 | click.echo(scan['status']) 342 | except shodan.APIError as e: 343 | raise click.ClickException(e.value) 344 | -------------------------------------------------------------------------------- /shodan/cli/alert.py: -------------------------------------------------------------------------------- 1 | import click 2 | import csv 3 | import gzip 4 | import json 5 | import shodan 6 | from tldextract import extract 7 | from ipaddress import ip_address 8 | 9 | from collections import defaultdict 10 | from operator import itemgetter 11 | from shodan import APIError 12 | from shodan.cli.helpers import get_api_key 13 | from shodan.helpers import open_file, write_banner 14 | from time import sleep 15 | 16 | 17 | MAX_QUERY_LENGTH = 1000 18 | 19 | 20 | def aggregate_facet(api, networks, facets): 21 | """Merge the results from multiple facet API queries into a single result object. 22 | This is necessary because a user might be monitoring a lot of IPs/ networks so it doesn't fit 23 | into a single API call. 24 | """ 25 | def _merge_custom_facets(lfacets, results): 26 | for key in results['facets']: 27 | if key not in lfacets: 28 | lfacets[key] = defaultdict(int) 29 | 30 | for item in results['facets'][key]: 31 | lfacets[key][item['value']] += item['count'] 32 | 33 | # We're going to create a custom facets dict where 34 | # the key is the value of a facet. Normally the facets 35 | # object is a list where each item has a "value" and "count" property. 36 | tmp_facets = {} 37 | count = 0 38 | 39 | query = 'net:' 40 | 41 | for net in networks: 42 | query += '{},'.format(net) 43 | 44 | # Start running API queries if the query length is getting long 45 | if len(query) > MAX_QUERY_LENGTH: 46 | results = api.count(query[:-1], facets=facets) 47 | 48 | _merge_custom_facets(tmp_facets, results) 49 | count += results['total'] 50 | query = 'net:' 51 | 52 | # Run any remaining search query 53 | if query[-1] != ':': 54 | results = api.count(query[:-1], facets=facets) 55 | 56 | _merge_custom_facets(tmp_facets, results) 57 | count += results['total'] 58 | 59 | # Convert the internal facets structure back to the one that 60 | # the API returns. 61 | new_facets = {} 62 | for facet in tmp_facets: 63 | sorted_items = sorted(tmp_facets[facet].items(), key=itemgetter(1), reverse=True) 64 | new_facets[facet] = [{'value': key, 'count': value} for key, value in sorted_items] 65 | 66 | # Make sure the facet keys exist even if there weren't any results 67 | for facet, _ in facets: 68 | if facet not in new_facets: 69 | new_facets[facet] = [] 70 | 71 | return { 72 | 'matches': [], 73 | 'facets': new_facets, 74 | 'total': count, 75 | } 76 | 77 | 78 | @click.group() 79 | def alert(): 80 | """Manage the network alerts for your account""" 81 | pass 82 | 83 | 84 | @alert.command(name='clear') 85 | def alert_clear(): 86 | """Remove all alerts""" 87 | key = get_api_key() 88 | 89 | # Get the list 90 | api = shodan.Shodan(key) 91 | try: 92 | alerts = api.alerts() 93 | for alert in alerts: 94 | click.echo(u'Removing {} ({})'.format(alert['name'], alert['id'])) 95 | api.delete_alert(alert['id']) 96 | except shodan.APIError as e: 97 | raise click.ClickException(e.value) 98 | click.echo("Alerts deleted") 99 | 100 | 101 | @alert.command(name='create') 102 | @click.argument('name', metavar='') 103 | @click.argument('netblocks', metavar='', nargs=-1) 104 | def alert_create(name, netblocks): 105 | """Create a network alert to monitor an external network""" 106 | key = get_api_key() 107 | 108 | # Get the list 109 | api = shodan.Shodan(key) 110 | try: 111 | alert = api.create_alert(name, netblocks) 112 | except shodan.APIError as e: 113 | raise click.ClickException(e.value) 114 | 115 | click.secho('Successfully created network alert!', fg='green') 116 | click.secho('Alert ID: {}'.format(alert['id']), fg='cyan') 117 | 118 | 119 | @alert.command(name='domain') 120 | @click.argument('domain', metavar='', type=str) 121 | @click.option('--triggers', help='List of triggers to enable', default='malware,industrial_control_system,internet_scanner,iot,open_database,new_service,ssl_expired,vulnerable') 122 | def alert_domain(domain, triggers): 123 | """Create a network alert based on a domain name""" 124 | key = get_api_key() 125 | 126 | api = shodan.Shodan(key) 127 | try: 128 | # Grab a list of IPs for the domain 129 | domain = domain.lower() 130 | domain_parse = extract(domain) 131 | click.secho('Looking up domain information...', dim=True) 132 | info = api.dns.domain_info(domain, type='A') 133 | 134 | if domain_parse.subdomain: 135 | domain_ips = set([record['value'] for record in info['data'] 136 | if record['subdomain'] == domain_parse.subdomain and 137 | not ip_address(record['value']).is_private]) 138 | else: 139 | domain_ips = set([record['value'] for record in info['data'] 140 | if not ip_address(record['value']).is_private]) 141 | 142 | if not domain_ips: 143 | raise click.ClickException('No external IPs were found to be associated with this domain. ' 144 | 'No alert was created.') 145 | 146 | # Create the actual alert 147 | click.secho('Creating alert...', dim=True) 148 | alert = api.create_alert('__domain: {}'.format(domain), list(domain_ips)) 149 | 150 | # Enable the triggers so it starts getting managed by Shodan Monitor 151 | click.secho('Enabling triggers...', dim=True) 152 | api.enable_alert_trigger(alert['id'], triggers) 153 | except shodan.APIError as e: 154 | raise click.ClickException(e.value) 155 | 156 | click.secho('Successfully created domain alert!', fg='green') 157 | click.secho('Alert ID: {}'.format(alert['id']), fg='cyan') 158 | 159 | 160 | @alert.command(name='download') 161 | @click.argument('filename', metavar='', type=str) 162 | @click.option('--alert-id', help='Specific alert ID to download the data of', default=None) 163 | def alert_download(filename, alert_id): 164 | """Download all information for monitored networks/ IPs.""" 165 | key = get_api_key() 166 | 167 | api = shodan.Shodan(key) 168 | ips = set() 169 | networks = set() 170 | 171 | # Helper method to process batches of IPs 172 | def batch(iterable, size=1): 173 | iter_length = len(iterable) 174 | for ndx in range(0, iter_length, size): 175 | yield iterable[ndx:min(ndx + size, iter_length)] 176 | 177 | try: 178 | # Get the list of alerts for the user 179 | click.echo('Looking up alert information...') 180 | if alert_id: 181 | alerts = [api.alerts(aid=alert_id.strip())] 182 | else: 183 | alerts = api.alerts() 184 | 185 | click.echo('Compiling list of networks/ IPs to download...') 186 | for alert in alerts: 187 | for net in alert['filters']['ip']: 188 | if '/' in net: 189 | networks.add(net) 190 | else: 191 | ips.add(net) 192 | 193 | click.echo('Downloading...') 194 | with open_file(filename) as fout: 195 | # Check if the user is able to use batch IP lookups 196 | batch_size = 1 197 | if len(ips) > 0: 198 | api_info = api.info() 199 | if api_info['plan'] in ['corp', 'stream-100']: 200 | batch_size = 100 201 | 202 | # Convert it to a list so we can index into it 203 | ips = list(ips) 204 | 205 | # Grab all the IP information 206 | for ip in batch(ips, size=batch_size): 207 | try: 208 | click.echo(ip) 209 | results = api.host(ip) 210 | if not isinstance(results, list): 211 | results = [results] 212 | 213 | for host in results: 214 | for banner in host['data']: 215 | write_banner(fout, banner) 216 | except APIError: 217 | pass 218 | sleep(1) # Slow down a bit to make sure we don't hit the rate limit 219 | 220 | # Grab all the network ranges 221 | for net in networks: 222 | try: 223 | counter = 0 224 | click.echo(net) 225 | for banner in api.search_cursor('net:{}'.format(net)): 226 | write_banner(fout, banner) 227 | 228 | # Slow down a bit to make sure we don't hit the rate limit 229 | if counter % 100 == 0: 230 | sleep(1) 231 | counter += 1 232 | except APIError: 233 | pass 234 | except shodan.APIError as e: 235 | raise click.ClickException(e.value) 236 | 237 | click.secho('Successfully downloaded results into: {}'.format(filename), fg='green') 238 | 239 | 240 | @alert.command(name='export') 241 | @click.option('--filename', help='Name of the output file', default='shodan-alerts.json.gz', type=str) 242 | def alert_export(filename): 243 | """Export the configuration of monitored networks/ IPs to be used by ``shodan alert import``.""" 244 | # Setup the API wrapper 245 | key = get_api_key() 246 | api = shodan.Shodan(key) 247 | 248 | try: 249 | # Get the list of alerts for the user 250 | click.echo('Looking up alert information...') 251 | alerts = api.alerts() 252 | 253 | # Create the output file 254 | click.echo('Writing alerts to file: {}'.format(filename)) 255 | with gzip.open(filename, 'wt', encoding='utf-8') as fout: 256 | json.dump(alerts, fout) 257 | except Exception as e: 258 | raise click.ClickException(e.value) 259 | 260 | click.secho('Successfully exported monitored networks', fg='green') 261 | 262 | 263 | @alert.command(name='import') 264 | @click.argument('filename', metavar='') 265 | def alert_import(filename): 266 | """Export the configuration of monitored networks/ IPs to be used by ``shodan alert import``.""" 267 | # Setup the API wrapper 268 | key = get_api_key() 269 | api = shodan.Shodan(key) 270 | 271 | # A mapping of the old notifier IDs to the new ones 272 | notifier_map = {} 273 | 274 | try: 275 | # Loading the alerts 276 | click.echo('Loading alerts from: {}'.format(filename)) 277 | with gzip.open(filename, 'rt', encoding='utf-8') as fin: 278 | alerts = json.load(fin) 279 | 280 | for item in alerts: 281 | # Create the alert 282 | click.echo('Creating: {}'.format(item['name'])) 283 | alert = api.create_alert(item['name'], item['filters']['ip']) 284 | 285 | # Enable any triggers 286 | if item.get('triggers', {}): 287 | triggers = ','.join(item['triggers'].keys()) 288 | 289 | api.enable_alert_trigger(alert['id'], triggers) 290 | 291 | # Add any whitelisted services for this trigger 292 | for trigger, info in item['triggers'].items(): 293 | if info.get('ignore', []): 294 | for whitelist in info['ignore']: 295 | api.ignore_alert_trigger_notification(alert['id'], trigger, whitelist['ip'], whitelist['port']) 296 | 297 | # Enable the notifiers 298 | for prev_notifier in item.get('notifiers', []): 299 | # We don't need to do anything for the default notifier as that 300 | # uses the account's email address automatically. 301 | if prev_notifier['id'] == 'default': 302 | continue 303 | 304 | # Get the new notifier based on the ID of the old one 305 | notifier = notifier_map.get(prev_notifier['id']) 306 | 307 | # Create the notifier if it doesn't yet exist 308 | if notifier is None: 309 | notifier = api.notifier.create(prev_notifier['provider'], prev_notifier['args'], description=prev_notifier['description']) 310 | 311 | # Add it to our map of old notifier IDs to new notifiers 312 | notifier_map[prev_notifier['id']] = notifier 313 | 314 | api.add_alert_notifier(alert['id'], notifier['id']) 315 | except Exception as e: 316 | raise click.ClickException(e.value) 317 | 318 | click.secho('Successfully imported monitored networks', fg='green') 319 | 320 | 321 | @alert.command(name='info') 322 | @click.argument('alert', metavar='') 323 | def alert_info(alert): 324 | """Show information about a specific alert""" 325 | key = get_api_key() 326 | api = shodan.Shodan(key) 327 | 328 | try: 329 | info = api.alerts(aid=alert) 330 | except shodan.APIError as e: 331 | raise click.ClickException(e.value) 332 | 333 | click.secho(info['name'], fg='cyan') 334 | click.secho('Created: ', nl=False, dim=True) 335 | click.secho(info['created'], fg='magenta') 336 | 337 | click.secho('Notifications: ', nl=False, dim=True) 338 | if 'triggers' in info and info['triggers']: 339 | click.secho('enabled', fg='green') 340 | else: 341 | click.echo('disabled') 342 | 343 | click.echo('') 344 | click.secho('Network Range(s):', dim=True) 345 | 346 | for network in info['filters']['ip']: 347 | click.echo(u' > {}'.format(click.style(network, fg='yellow'))) 348 | 349 | click.echo('') 350 | if 'triggers' in info and info['triggers']: 351 | click.secho('Triggers:', dim=True) 352 | for trigger in info['triggers']: 353 | click.echo(u' > {}'.format(click.style(trigger, fg='yellow'))) 354 | click.echo('') 355 | 356 | 357 | @alert.command(name='list') 358 | @click.option('--expired', help='Whether or not to show expired alerts.', default=True, type=bool) 359 | def alert_list(expired): 360 | """List all the active alerts""" 361 | key = get_api_key() 362 | 363 | # Get the list 364 | api = shodan.Shodan(key) 365 | try: 366 | results = api.alerts(include_expired=expired) 367 | except shodan.APIError as e: 368 | raise click.ClickException(e.value) 369 | 370 | if len(results) > 0: 371 | click.echo(u'# {:14} {:<21} {:<15s}'.format('Alert ID', 'Name', 'IP/ Network')) 372 | 373 | for alert in results: 374 | click.echo( 375 | u'{:16} {:<30} {:<35} '.format( 376 | click.style(alert['id'], fg='yellow'), 377 | click.style(alert['name'], fg='cyan'), 378 | click.style(', '.join(alert['filters']['ip']), fg='white') 379 | ), 380 | nl=False 381 | ) 382 | 383 | if 'triggers' in alert and alert['triggers']: 384 | click.secho('Triggers: ', fg='magenta', nl=False) 385 | click.echo(', '.join(alert['triggers'].keys()), nl=False) 386 | 387 | if 'expired' in alert and alert['expired']: 388 | click.secho('expired', fg='red') 389 | else: 390 | click.echo('') 391 | else: 392 | click.echo("You haven't created any alerts yet.") 393 | 394 | 395 | @alert.command(name='stats') 396 | @click.option('--limit', help='The number of results to return.', default=10, type=int) 397 | @click.option('--filename', '-O', help='Save the results in a CSV file of the provided name.', default=None) 398 | @click.argument('facets', metavar='', nargs=-1) 399 | def alert_stats(limit, filename, facets): 400 | """Show summary information about your monitored networks""" 401 | # Setup Shodan 402 | key = get_api_key() 403 | api = shodan.Shodan(key) 404 | 405 | # Make sure the user didn't supply an empty string 406 | if not facets: 407 | raise click.ClickException('No facets provided') 408 | 409 | facets = [(facet, limit) for facet in facets] 410 | 411 | # Get the list of IPs/ networks that the user is monitoring 412 | networks = set() 413 | try: 414 | alerts = api.alerts() 415 | for alert in alerts: 416 | for tmp in alert['filters']['ip']: 417 | networks.add(tmp) 418 | except shodan.APIError as e: 419 | raise click.ClickException(e.value) 420 | 421 | # Grab the facets the user requested 422 | try: 423 | results = aggregate_facet(api, networks, facets) 424 | except shodan.APIError as e: 425 | raise click.ClickException(e.value) 426 | 427 | # TODO: The below code was taken from __main__.py:stats() - we should refactor it so the code can be shared 428 | # Print the stats tables 429 | for facet in results['facets']: 430 | click.echo('Top {} Results for Facet: {}'.format(len(results['facets'][facet]), facet)) 431 | 432 | for item in results['facets'][facet]: 433 | # Force the value to be a string - necessary because some facet values are numbers 434 | value = u'{}'.format(item['value']) 435 | 436 | click.echo(click.style(u'{:28s}'.format(value), fg='cyan'), nl=False) 437 | click.echo(click.style(u'{:12,d}'.format(item['count']), fg='green')) 438 | 439 | click.echo('') 440 | 441 | # Create the output file if requested 442 | fout = None 443 | if filename: 444 | if not filename.endswith('.csv'): 445 | filename += '.csv' 446 | fout = open(filename, 'w') 447 | writer = csv.writer(fout, dialect=csv.excel) 448 | 449 | # Write the header that contains the facets 450 | row = [] 451 | for facet in results['facets']: 452 | row.append(facet) 453 | row.append('') 454 | writer.writerow(row) 455 | 456 | # Every facet has 2 columns (key, value) 457 | counter = 0 458 | has_items = True 459 | while has_items: 460 | # pylint: disable=W0612 461 | row = ['' for i in range(len(results['facets']) * 2)] 462 | 463 | pos = 0 464 | has_items = False 465 | for facet in results['facets']: 466 | values = results['facets'][facet] 467 | 468 | # Add the values for the facet into the current row 469 | if len(values) > counter: 470 | has_items = True 471 | row[pos] = values[counter]['value'] 472 | row[pos + 1] = values[counter]['count'] 473 | 474 | pos += 2 475 | 476 | # Write out the row 477 | if has_items: 478 | writer.writerow(row) 479 | 480 | # Move to the next row of values 481 | counter += 1 482 | 483 | 484 | @alert.command(name='remove') 485 | @click.argument('alert_id', metavar='') 486 | def alert_remove(alert_id): 487 | """Remove the specified alert""" 488 | key = get_api_key() 489 | 490 | # Get the list 491 | api = shodan.Shodan(key) 492 | try: 493 | api.delete_alert(alert_id) 494 | except shodan.APIError as e: 495 | raise click.ClickException(e.value) 496 | click.echo("Alert deleted") 497 | 498 | 499 | @alert.command(name='triggers') 500 | def alert_list_triggers(): 501 | """List the available notification triggers""" 502 | key = get_api_key() 503 | 504 | # Get the list 505 | api = shodan.Shodan(key) 506 | try: 507 | results = api.alert_triggers() 508 | except shodan.APIError as e: 509 | raise click.ClickException(e.value) 510 | 511 | if len(results) > 0: 512 | click.secho('The following triggers can be enabled on alerts:', dim=True) 513 | click.echo('') 514 | 515 | for trigger in sorted(results, key=itemgetter('name')): 516 | click.secho('{:<12} '.format('Name'), dim=True, nl=False) 517 | click.secho(trigger['name'], fg='yellow') 518 | 519 | click.secho('{:<12} '.format('Description'), dim=True, nl=False) 520 | click.secho(trigger['description'], fg='cyan') 521 | 522 | click.secho('{:<12} '.format('Rule'), dim=True, nl=False) 523 | click.echo(trigger['rule']) 524 | 525 | click.echo('') 526 | else: 527 | click.echo("No triggers currently available.") 528 | 529 | 530 | @alert.command(name='enable') 531 | @click.argument('alert_id', metavar='') 532 | @click.argument('trigger', metavar='') 533 | def alert_enable_trigger(alert_id, trigger): 534 | """Enable a trigger for the alert""" 535 | key = get_api_key() 536 | 537 | # Get the list 538 | api = shodan.Shodan(key) 539 | try: 540 | api.enable_alert_trigger(alert_id, trigger) 541 | except shodan.APIError as e: 542 | raise click.ClickException(e.value) 543 | 544 | click.secho('Successfully enabled the trigger: {}'.format(trigger), fg='green') 545 | 546 | 547 | @alert.command(name='disable') 548 | @click.argument('alert_id', metavar='') 549 | @click.argument('trigger', metavar='') 550 | def alert_disable_trigger(alert_id, trigger): 551 | """Disable a trigger for the alert""" 552 | key = get_api_key() 553 | 554 | # Get the list 555 | api = shodan.Shodan(key) 556 | try: 557 | api.disable_alert_trigger(alert_id, trigger) 558 | except shodan.APIError as e: 559 | raise click.ClickException(e.value) 560 | 561 | click.secho('Successfully disabled the trigger: {}'.format(trigger), fg='green') 562 | -------------------------------------------------------------------------------- /shodan/client.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | shodan.client 4 | ~~~~~~~~~~~~~ 5 | 6 | This module implements the Shodan API. 7 | 8 | :copyright: (c) 2014- by John Matherly 9 | """ 10 | import math 11 | import os 12 | import time 13 | 14 | import requests 15 | import json 16 | 17 | from .exception import APIError 18 | from .helpers import create_facet_string 19 | from .stream import Stream 20 | 21 | 22 | # Try to disable the SSL warnings in urllib3 since not everybody can install 23 | # C extensions. If you're able to install C extensions you can try to run: 24 | # 25 | # pip install requests[security] 26 | # 27 | # Which will download libraries that offer more full-featured SSL classes 28 | # pylint: disable=E1101 29 | try: 30 | requests.packages.urllib3.disable_warnings() 31 | except Exception: 32 | pass 33 | 34 | # Define a basestring type if necessary for Python3 compatibility 35 | try: 36 | basestring 37 | except NameError: 38 | basestring = str 39 | 40 | 41 | class Shodan: 42 | """Wrapper around the Shodan REST and Streaming APIs 43 | 44 | :param key: The Shodan API key that can be obtained from your account page (https://account.shodan.io) 45 | :type key: str 46 | :ivar exploits: An instance of `shodan.Shodan.Exploits` that provides access to the Exploits REST API. 47 | :ivar stream: An instance of `shodan.Shodan.Stream` that provides access to the Streaming API. 48 | """ 49 | 50 | class Data: 51 | 52 | def __init__(self, parent): 53 | self.parent = parent 54 | 55 | def list_datasets(self): 56 | """Returns a list of datasets that the user has permission to download. 57 | 58 | :returns: A list of objects where every object describes a dataset 59 | """ 60 | return self.parent._request('/shodan/data', {}) 61 | 62 | def list_files(self, dataset): 63 | """Returns a list of files that belong to the given dataset. 64 | 65 | :returns: A list of objects where each object contains a 'name', 'size', 'timestamp' and 'url' 66 | """ 67 | return self.parent._request('/shodan/data/{}'.format(dataset), {}) 68 | 69 | class Dns: 70 | 71 | def __init__(self, parent): 72 | self.parent = parent 73 | 74 | def domain_info(self, domain, history=False, type=None, page=1): 75 | """Grab the DNS information for a domain. 76 | """ 77 | args = { 78 | 'page': page, 79 | } 80 | if history: 81 | args['history'] = history 82 | if type: 83 | args['type'] = type 84 | return self.parent._request('/dns/domain/{}'.format(domain), args) 85 | 86 | class Notifier: 87 | 88 | def __init__(self, parent): 89 | self.parent = parent 90 | 91 | def create(self, provider, args, description=None): 92 | """Get the settings for the specified notifier that a user has configured. 93 | 94 | :param provider: Provider name 95 | :type provider: str 96 | :param args: Provider arguments 97 | :type args: dict 98 | :param description: Human-friendly description of the notifier 99 | :type description: str 100 | :returns: dict -- fields are 'success' and 'id' of the notifier 101 | """ 102 | args['provider'] = provider 103 | 104 | if description: 105 | args['description'] = description 106 | 107 | return self.parent._request('/notifier', args, method='post') 108 | 109 | def edit(self, nid, args): 110 | """Get the settings for the specified notifier that a user has configured. 111 | 112 | :param nid: Notifier ID 113 | :type nid: str 114 | :param args: Provider arguments 115 | :type args: dict 116 | :returns: dict -- fields are 'success' and 'id' of the notifier 117 | """ 118 | return self.parent._request('/notifier/{}'.format(nid), args, method='put') 119 | 120 | def get(self, nid): 121 | """Get the settings for the specified notifier that a user has configured. 122 | 123 | :param nid: Notifier ID 124 | :type nid: str 125 | :returns: dict -- object describing the notifier settings 126 | """ 127 | return self.parent._request('/notifier/{}'.format(nid), {}) 128 | 129 | def list_notifiers(self): 130 | """Returns a list of notifiers that the user has added. 131 | 132 | :returns: A list of notifierse that are available on the account 133 | """ 134 | return self.parent._request('/notifier', {}) 135 | 136 | def list_providers(self): 137 | """Returns a list of supported notification providers. 138 | 139 | :returns: A list of providers where each object describes a provider 140 | """ 141 | return self.parent._request('/notifier/provider', {}) 142 | 143 | def remove(self, nid): 144 | """Delete the provided notifier. 145 | 146 | :param nid: Notifier ID 147 | :type nid: str 148 | :returns: dict -- 'success' set to True if action succeeded 149 | """ 150 | return self.parent._request('/notifier/{}'.format(nid), {}, method='delete') 151 | 152 | class Tools: 153 | 154 | def __init__(self, parent): 155 | self.parent = parent 156 | 157 | def myip(self): 158 | """Get your current IP address as seen from the Internet. 159 | 160 | :returns: str -- your IP address 161 | """ 162 | return self.parent._request('/tools/myip', {}) 163 | 164 | class Exploits: 165 | 166 | def __init__(self, parent): 167 | self.parent = parent 168 | 169 | def search(self, query, page=1, facets=None): 170 | """Search the entire Shodan Exploits archive using the same query syntax 171 | as the website. 172 | 173 | :param query: The exploit search query; same syntax as website. 174 | :type query: str 175 | :param facets: A list of strings or tuples to get summary information on. 176 | :type facets: str 177 | :param page: The page number to access. 178 | :type page: int 179 | :returns: dict -- a dictionary containing the results of the search. 180 | """ 181 | query_args = { 182 | 'query': query, 183 | 'page': page, 184 | } 185 | if facets: 186 | query_args['facets'] = create_facet_string(facets) 187 | 188 | return self.parent._request('/api/search', query_args, service='exploits') 189 | 190 | def count(self, query, facets=None): 191 | """Search the entire Shodan Exploits archive but only return the total # of results, 192 | not the actual exploits. 193 | 194 | :param query: The exploit search query; same syntax as website. 195 | :type query: str 196 | :param facets: A list of strings or tuples to get summary information on. 197 | :type facets: str 198 | :returns: dict -- a dictionary containing the results of the search. 199 | 200 | """ 201 | query_args = { 202 | 'query': query, 203 | } 204 | if facets: 205 | query_args['facets'] = create_facet_string(facets) 206 | 207 | return self.parent._request('/api/count', query_args, service='exploits') 208 | 209 | class Labs: 210 | 211 | def __init__(self, parent): 212 | self.parent = parent 213 | 214 | def honeyscore(self, ip): 215 | """Calculate the probability of an IP being an ICS honeypot. 216 | 217 | :param ip: IP address of the device 218 | :type ip: str 219 | 220 | :returns: int -- honeyscore ranging from 0.0 to 1.0 221 | """ 222 | return self.parent._request('/labs/honeyscore/{}'.format(ip), {}) 223 | 224 | class Organization: 225 | 226 | def __init__(self, parent): 227 | self.parent = parent 228 | 229 | def add_member(self, user, notify=True): 230 | """Add the user to the organization. 231 | 232 | :param user: username or email address 233 | :type user: str 234 | :param notify: whether or not to send the user an email notification 235 | :type notify: bool 236 | 237 | :returns: True if it succeeded and raises an Exception otherwise 238 | """ 239 | return self.parent._request('/org/member/{}'.format(user), { 240 | 'notify': notify, 241 | }, method='PUT')['success'] 242 | 243 | def info(self): 244 | """Returns general information about the organization the current user is a member of. 245 | """ 246 | return self.parent._request('/org', {}) 247 | 248 | def remove_member(self, user): 249 | """Remove the user from the organization. 250 | 251 | :param user: username or email address 252 | :type user: str 253 | 254 | :returns: True if it succeeded and raises an Exception otherwise 255 | """ 256 | return self.parent._request('/org/member/{}'.format(user), {}, method='DELETE')['success'] 257 | 258 | class Trends: 259 | 260 | def __init__(self, parent): 261 | self.parent = parent 262 | 263 | def search(self, query, facets): 264 | """Search the Shodan historical database. 265 | 266 | :param query: Search query; identical syntax to the website 267 | :type query: str 268 | :param facets: (optional) A list of properties to get summary information on 269 | :type facets: str 270 | 271 | :returns: A dictionary with 3 main items: matches, facets and total. Visit the website for more detailed information. 272 | """ 273 | args = { 274 | 'query': query, 275 | 'facets': create_facet_string(facets), 276 | } 277 | 278 | return self.parent._request('/api/v1/search', args, service='trends') 279 | 280 | def search_facets(self): 281 | """This method returns a list of facets that can be used to get a breakdown of the top values for a property. 282 | 283 | :returns: A list of strings where each is a facet name 284 | """ 285 | return self.parent._request('/api/v1/search/facets', {}, service='trends') 286 | 287 | def search_filters(self): 288 | """This method returns a list of search filters that can be used in the search query. 289 | 290 | :returns: A list of strings where each is a filter name 291 | """ 292 | return self.parent._request('/api/v1/search/filters', {}, service='trends') 293 | 294 | def __init__(self, key, proxies=None): 295 | """Initializes the API object. 296 | 297 | :param key: The Shodan API key. 298 | :type key: str 299 | :param proxies: A proxies array for the requests library, e.g. {'https': 'your proxy'} 300 | :type proxies: dict 301 | """ 302 | self.api_key = key 303 | self.base_url = 'https://api.shodan.io' 304 | self.base_exploits_url = 'https://exploits.shodan.io' 305 | self.base_trends_url = 'https://trends.shodan.io' 306 | self.data = self.Data(self) 307 | self.dns = self.Dns(self) 308 | self.exploits = self.Exploits(self) 309 | self.trends = self.Trends(self) 310 | self.labs = self.Labs(self) 311 | self.notifier = self.Notifier(self) 312 | self.org = self.Organization(self) 313 | self.tools = self.Tools(self) 314 | self.stream = Stream(key, proxies=proxies) 315 | self._session = requests.Session() 316 | self.api_rate_limit = 1 # Requests per second 317 | self._api_query_time = None 318 | 319 | if proxies: 320 | self._session.proxies.update(proxies) 321 | self._session.trust_env = False 322 | 323 | if os.environ.get('SHODAN_API_URL'): 324 | self.base_url = os.environ.get('SHODAN_API_URL') 325 | 326 | def _request(self, function, params, service='shodan', method='get', json_data=None): 327 | """General-purpose function to create web requests to SHODAN. 328 | 329 | Arguments: 330 | function -- name of the function you want to execute 331 | params -- dictionary of parameters for the function 332 | 333 | Returns 334 | A dictionary containing the function's results. 335 | 336 | """ 337 | # Add the API key parameter automatically 338 | params['key'] = self.api_key 339 | 340 | # Determine the base_url based on which service we're interacting with 341 | base_url = { 342 | 'shodan': self.base_url, 343 | 'exploits': self.base_exploits_url, 344 | 'trends': self.base_trends_url, 345 | }.get(service, 'shodan') 346 | 347 | # Wait for API rate limit 348 | if self._api_query_time is not None and self.api_rate_limit > 0: 349 | while (1.0 / self.api_rate_limit) + self._api_query_time >= time.time(): 350 | time.sleep(0.1 / self.api_rate_limit) 351 | 352 | # Send the request 353 | try: 354 | method = method.lower() 355 | if method == 'post': 356 | if json_data: 357 | data = self._session.post(base_url + function, params=params, 358 | data=json.dumps(json_data), 359 | headers={'content-type': 'application/json'}, 360 | ) 361 | else: 362 | data = self._session.post(base_url + function, params) 363 | elif method == 'put': 364 | data = self._session.put(base_url + function, params=params) 365 | elif method == 'delete': 366 | data = self._session.delete(base_url + function, params=params) 367 | else: 368 | data = self._session.get(base_url + function, params=params) 369 | self._api_query_time = time.time() 370 | except Exception: 371 | raise APIError('Unable to connect to Shodan') 372 | 373 | # Check that the API key wasn't rejected 374 | if data.status_code == 401: 375 | try: 376 | # Return the actual error message if the API returned valid JSON 377 | error = data.json()['error'] 378 | except Exception as e: 379 | # If the response looks like HTML then it's probably the 401 page that nginx returns 380 | # for 401 responses by default 381 | if data.text.startswith('<'): 382 | error = 'Invalid API key' 383 | else: 384 | # Otherwise lets raise the error message 385 | error = u'{}'.format(e) 386 | 387 | raise APIError(error) 388 | elif data.status_code == 403: 389 | raise APIError('Access denied (403 Forbidden)') 390 | elif data.status_code == 502: 391 | raise APIError('Bad Gateway (502)') 392 | 393 | # Parse the text into JSON 394 | try: 395 | data = data.json() 396 | except ValueError: 397 | raise APIError('Unable to parse JSON response') 398 | 399 | # Raise an exception if an error occurred 400 | if type(data) == dict and 'error' in data: 401 | raise APIError(data['error']) 402 | 403 | # Return the data 404 | return data 405 | 406 | def count(self, query, facets=None): 407 | """Returns the total number of search results for the query. 408 | 409 | :param query: Search query; identical syntax to the website 410 | :type query: str 411 | :param facets: (optional) A list of properties to get summary information on 412 | :type facets: str 413 | 414 | :returns: A dictionary with 1 main property: total. If facets have been provided then another property called "facets" will be available at the top-level of the dictionary. Visit the website for more detailed information. 415 | """ 416 | query_args = { 417 | 'query': query, 418 | } 419 | if facets: 420 | query_args['facets'] = create_facet_string(facets) 421 | return self._request('/shodan/host/count', query_args) 422 | 423 | def host(self, ips, history=False, minify=False): 424 | """Get all available information on an IP. 425 | 426 | :param ip: IP of the computer 427 | :type ip: str 428 | :param history: (optional) True if you want to grab the historical (non-current) banners for the host, False otherwise. 429 | :type history: bool 430 | :param minify: (optional) True to only return the list of ports and the general host information, no banners, False otherwise. 431 | :type minify: bool 432 | """ 433 | if isinstance(ips, basestring): 434 | ips = [ips] 435 | 436 | params = {} 437 | if history: 438 | params['history'] = history 439 | if minify: 440 | params['minify'] = minify 441 | return self._request('/shodan/host/{}'.format(','.join(ips)), params) 442 | 443 | def info(self): 444 | """Returns information about the current API key, such as a list of add-ons 445 | and other features that are enabled for the current user's API plan. 446 | """ 447 | return self._request('/api-info', {}) 448 | 449 | def ports(self): 450 | """Get a list of ports that Shodan crawls 451 | 452 | :returns: An array containing the ports that Shodan crawls for. 453 | """ 454 | return self._request('/shodan/ports', {}) 455 | 456 | def protocols(self): 457 | """Get a list of protocols that the Shodan on-demand scanning API supports. 458 | 459 | :returns: A dictionary containing the protocol name and description. 460 | """ 461 | return self._request('/shodan/protocols', {}) 462 | 463 | def scan(self, ips, force=False): 464 | """Scan a network using Shodan 465 | 466 | :param ips: A list of IPs or netblocks in CIDR notation or an object structured like: 467 | { 468 | "9.9.9.9": [ 469 | (443, "https"), 470 | (8080, "http") 471 | ], 472 | "1.1.1.0/24": [ 473 | (503, "modbus") 474 | ] 475 | } 476 | :type ips: str or dict 477 | :param force: Whether or not to force Shodan to re-scan the provided IPs. Only available to enterprise users. 478 | :type force: bool 479 | 480 | :returns: A dictionary with a unique ID to check on the scan progress, the number of IPs that will be crawled and how many scan credits are left. 481 | """ 482 | if isinstance(ips, basestring): 483 | ips = [ips] 484 | 485 | if isinstance(ips, dict): 486 | networks = json.dumps(ips) 487 | else: 488 | networks = ','.join(ips) 489 | 490 | params = { 491 | 'ips': networks, 492 | 'force': force, 493 | } 494 | 495 | return self._request('/shodan/scan', params, method='post') 496 | 497 | def scans(self, page=1): 498 | """Get a list of scans submitted 499 | 500 | :param page: Page through the list of scans 100 results at a time 501 | :type page: int 502 | """ 503 | return self._request('/shodan/scans', { 504 | 'page': page, 505 | }) 506 | 507 | def scan_internet(self, port, protocol): 508 | """Scan a network using Shodan 509 | 510 | :param port: The port that should get scanned. 511 | :type port: int 512 | :param port: The name of the protocol as returned by the protocols() method. 513 | :type port: str 514 | 515 | :returns: A dictionary with a unique ID to check on the scan progress. 516 | """ 517 | params = { 518 | 'port': port, 519 | 'protocol': protocol, 520 | } 521 | 522 | return self._request('/shodan/scan/internet', params, method='post') 523 | 524 | def scan_status(self, scan_id): 525 | """Get the status information about a previously submitted scan. 526 | 527 | :param id: The unique ID for the scan that was submitted 528 | :type id: str 529 | 530 | :returns: A dictionary with general information about the scan, including its status in getting processed. 531 | """ 532 | return self._request('/shodan/scan/{}'.format(scan_id), {}) 533 | 534 | def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True, fields=None): 535 | """Search the SHODAN database. 536 | 537 | :param query: Search query; identical syntax to the website 538 | :type query: str 539 | :param page: (optional) Page number of the search results 540 | :type page: int 541 | :param limit: (optional) Number of results to return 542 | :type limit: int 543 | :param offset: (optional) Search offset to begin getting results from 544 | :type offset: int 545 | :param facets: (optional) A list of properties to get summary information on 546 | :type facets: str 547 | :param minify: (optional) Whether to minify the banner and only return the important data 548 | :type minify: bool 549 | :param fields: (optional) List of properties that should get returned. This option is mutually exclusive with the "minify" parameter 550 | :type fields: str 551 | 552 | :returns: A dictionary with 2 main items: matches and total. If facets have been provided then another property called "facets" will be available at the top-level of the dictionary. Visit the website for more detailed information. 553 | """ 554 | args = { 555 | 'query': query, 556 | 'minify': minify, 557 | } 558 | if limit: 559 | args['limit'] = limit 560 | if offset: 561 | args['offset'] = offset 562 | else: 563 | args['page'] = page 564 | 565 | if facets: 566 | args['facets'] = create_facet_string(facets) 567 | 568 | if fields and isinstance(fields, list): 569 | args['fields'] = ','.join(fields) 570 | 571 | return self._request('/shodan/host/search', args) 572 | 573 | def search_cursor(self, query, minify=True, retries=5, fields=None): 574 | """Search the SHODAN database. 575 | 576 | This method returns an iterator that can directly be in a loop. Use it when you want to loop over 577 | all of the results of a search query. But this method doesn't return a "matches" array or the "total" 578 | information. And it also can't be used with facets, it's only use is to iterate over results more 579 | easily. 580 | 581 | :param query: Search query; identical syntax to the website 582 | :type query: str 583 | :param minify: (optional) Whether to minify the banner and only return the important data 584 | :type minify: bool 585 | :param retries: (optional) How often to retry the search in case it times out 586 | :type retries: int 587 | 588 | :returns: A search cursor that can be used as an iterator/ generator. 589 | """ 590 | page = 1 591 | total_pages = 0 592 | tries = 0 593 | 594 | # Grab the initial page and use the total to calculate the expected number of pages 595 | results = self.search(query, minify=minify, page=page, fields=fields) 596 | if results['total']: 597 | total_pages = int(math.ceil(results['total'] / 100)) 598 | 599 | for banner in results['matches']: 600 | try: 601 | yield banner 602 | except GeneratorExit: 603 | return # exit out of the function 604 | page += 1 605 | 606 | # Keep iterating over the results from page 2 onwards 607 | while page <= total_pages: 608 | try: 609 | results = self.search(query, minify=minify, page=page, fields=fields) 610 | for banner in results['matches']: 611 | try: 612 | yield banner 613 | except GeneratorExit: 614 | return # exit out of the function 615 | page += 1 616 | tries = 0 617 | except Exception: 618 | # We've retried several times but it keeps failing, so lets error out 619 | if tries >= retries: 620 | raise APIError('Retry limit reached ({:d})'.format(retries)) 621 | 622 | tries += 1 623 | time.sleep(tries) # wait (1 second * retry number) if the search errored out for some reason 624 | 625 | def search_facets(self): 626 | """Returns a list of search facets that can be used to get aggregate information about a search query. 627 | 628 | :returns: A list of strings where each is a facet name 629 | """ 630 | return self._request('/shodan/host/search/facets', {}) 631 | 632 | def search_filters(self): 633 | """Returns a list of search filters that are available. 634 | 635 | :returns: A list of strings where each is a filter name 636 | """ 637 | return self._request('/shodan/host/search/filters', {}) 638 | 639 | def search_tokens(self, query): 640 | """Returns information about the search query itself (filters used etc.) 641 | 642 | :param query: Search query; identical syntax to the website 643 | :type query: str 644 | 645 | :returns: A dictionary with 4 main properties: filters, errors, attributes and string. 646 | """ 647 | query_args = { 648 | 'query': query, 649 | } 650 | return self._request('/shodan/host/search/tokens', query_args) 651 | 652 | def services(self): 653 | """Get a list of services that Shodan crawls 654 | 655 | :returns: A dictionary containing the ports/ services that Shodan crawls for. The key is the port number and the value is the name of the service. 656 | """ 657 | return self._request('/shodan/services', {}) 658 | 659 | def queries(self, page=1, sort='timestamp', order='desc'): 660 | """List the search queries that have been shared by other users. 661 | 662 | :param page: Page number to iterate over results; each page contains 10 items 663 | :type page: int 664 | :param sort: Sort the list based on a property. Possible values are: votes, timestamp 665 | :type sort: str 666 | :param order: Whether to sort the list in ascending or descending order. Possible values are: asc, desc 667 | :type order: str 668 | 669 | :returns: A list of saved search queries (dictionaries). 670 | """ 671 | args = { 672 | 'page': page, 673 | 'sort': sort, 674 | 'order': order, 675 | } 676 | return self._request('/shodan/query', args) 677 | 678 | def queries_search(self, query, page=1): 679 | """Search the directory of saved search queries in Shodan. 680 | 681 | :param query: The search string to look for in the search query 682 | :type query: str 683 | :param page: Page number to iterate over results; each page contains 10 items 684 | :type page: int 685 | 686 | :returns: A list of saved search queries (dictionaries). 687 | """ 688 | args = { 689 | 'page': page, 690 | 'query': query, 691 | } 692 | return self._request('/shodan/query/search', args) 693 | 694 | def queries_tags(self, size=10): 695 | """Search the directory of saved search queries in Shodan. 696 | 697 | :param size: The number of tags to return 698 | :type size: int 699 | 700 | :returns: A list of tags. 701 | """ 702 | args = { 703 | 'size': size, 704 | } 705 | return self._request('/shodan/query/tags', args) 706 | 707 | def create_alert(self, name, ip, expires=0): 708 | """Create a network alert/ private firehose for the specified IP range(s) 709 | 710 | :param name: Name of the alert 711 | :type name: str 712 | :param ip: Network range(s) to monitor 713 | :type ip: str OR list of str 714 | 715 | :returns: A dict describing the alert 716 | """ 717 | data = { 718 | 'name': name, 719 | 'filters': { 720 | 'ip': ip, 721 | }, 722 | 'expires': expires, 723 | } 724 | 725 | response = self._request('/shodan/alert', params={}, json_data=data, method='post') 726 | 727 | return response 728 | 729 | def edit_alert(self, aid, ip): 730 | """Edit the IPs that should be monitored by the alert. 731 | 732 | :param aid: Alert ID 733 | :type name: str 734 | :param ip: Network range(s) to monitor 735 | :type ip: str OR list of str 736 | 737 | :returns: A dict describing the alert 738 | """ 739 | data = { 740 | 'filters': { 741 | 'ip': ip, 742 | }, 743 | } 744 | 745 | response = self._request('/shodan/alert/{}'.format(aid), params={}, json_data=data, method='post') 746 | 747 | return response 748 | 749 | def alerts(self, aid=None, include_expired=True): 750 | """List all of the active alerts that the user created.""" 751 | if aid: 752 | func = '/shodan/alert/{}/info'.format(aid) 753 | else: 754 | func = '/shodan/alert/info' 755 | 756 | response = self._request(func, params={ 757 | 'include_expired': include_expired, 758 | }) 759 | 760 | return response 761 | 762 | def delete_alert(self, aid): 763 | """Delete the alert with the given ID.""" 764 | func = '/shodan/alert/{}'.format(aid) 765 | 766 | response = self._request(func, params={}, method='delete') 767 | 768 | return response 769 | 770 | def alert_triggers(self): 771 | """Return a list of available triggers that can be enabled for alerts. 772 | 773 | :returns: A list of triggers 774 | """ 775 | return self._request('/shodan/alert/triggers', {}) 776 | 777 | def enable_alert_trigger(self, aid, trigger): 778 | """Enable the given trigger on the alert.""" 779 | return self._request('/shodan/alert/{}/trigger/{}'.format(aid, trigger), {}, method='put') 780 | 781 | def disable_alert_trigger(self, aid, trigger): 782 | """Disable the given trigger on the alert.""" 783 | return self._request('/shodan/alert/{}/trigger/{}'.format(aid, trigger), {}, method='delete') 784 | 785 | def ignore_alert_trigger_notification(self, aid, trigger, ip, port, vulns=None): 786 | """Ignore trigger notifications for the provided IP and port.""" 787 | # The "vulnerable" and "vulnerable_unverified" triggers let you specify specific vulnerabilities 788 | # to ignore. If a user provides a "vulns" list and specifies on of those triggers then we'll use 789 | # a different API endpoint. 790 | if trigger in ('vulnerable', 'vulnerable_unverified') and vulns and isinstance(vulns, list): 791 | return self._request('/shodan/alert/{}/trigger/{}/ignore/{}:{}/{}'.format(aid, trigger, ip, port, ','.join(vulns)), {}, method='put') 792 | 793 | return self._request('/shodan/alert/{}/trigger/{}/ignore/{}:{}'.format(aid, trigger, ip, port), {}, method='put') 794 | 795 | def unignore_alert_trigger_notification(self, aid, trigger, ip, port): 796 | """Re-enable trigger notifications for the provided IP and port""" 797 | return self._request('/shodan/alert/{}/trigger/{}/ignore/{}:{}'.format(aid, trigger, ip, port), {}, method='delete') 798 | 799 | def add_alert_notifier(self, aid, nid): 800 | """Enable the given notifier for an alert that has triggers enabled.""" 801 | return self._request('/shodan/alert/{}/notifier/{}'.format(aid, nid), {}, method='put') 802 | 803 | def remove_alert_notifier(self, aid, nid): 804 | """Remove the given notifier for an alert that has triggers enabled.""" 805 | return self._request('/shodan/alert/{}/notifier/{}'.format(aid, nid), {}, method='delete') 806 | -------------------------------------------------------------------------------- /shodan/__main__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Shodan CLI 3 | 4 | Note: Always run "shodan init " before trying to execute any other command! 5 | 6 | A simple interface to search Shodan, download data and parse compressed JSON files. 7 | The following commands are currently supported: 8 | 9 | alert 10 | convert 11 | count 12 | data 13 | download 14 | honeyscore 15 | host 16 | info 17 | init 18 | myip 19 | parse 20 | radar 21 | scan 22 | search 23 | stats 24 | stream 25 | trends 26 | 27 | """ 28 | 29 | import click 30 | import csv 31 | import os 32 | import os.path 33 | import pkg_resources 34 | import shodan 35 | import shodan.helpers as helpers 36 | import threading 37 | import requests 38 | import time 39 | import json 40 | 41 | # The file converters that are used to go from .json.gz to various other formats 42 | from shodan.cli.converter import CsvConverter, KmlConverter, GeoJsonConverter, ExcelConverter, ImagesConverter 43 | 44 | # Constants 45 | from shodan.cli.settings import SHODAN_CONFIG_DIR, COLORIZE_FIELDS 46 | 47 | # Helper methods 48 | from shodan.cli.helpers import async_spinner, get_api_key, escape_data, timestr, open_streaming_file, get_banner_field, match_filters 49 | from shodan.cli.host import HOST_PRINT 50 | 51 | # Allow 3rd-parties to develop custom commands 52 | from click_plugins import with_plugins 53 | from pkg_resources import iter_entry_points 54 | 55 | # Large subcommands are stored in separate modules 56 | from shodan.cli.alert import alert 57 | from shodan.cli.data import data 58 | from shodan.cli.organization import org 59 | from shodan.cli.scan import scan 60 | 61 | 62 | # Make "-h" work like "--help" 63 | CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) 64 | CONVERTERS = { 65 | 'kml': KmlConverter, 66 | 'csv': CsvConverter, 67 | 'geo.json': GeoJsonConverter, 68 | 'images': ImagesConverter, 69 | 'xlsx': ExcelConverter, 70 | } 71 | 72 | # Define a basestring type if necessary for Python3 compatibility 73 | try: 74 | basestring 75 | except NameError: 76 | basestring = str 77 | 78 | 79 | # Define the main entry point for all of our commands 80 | # and expose a way for 3rd-party plugins to tie into the Shodan CLI. 81 | @with_plugins(iter_entry_points('shodan.cli.plugins')) 82 | @click.group(context_settings=CONTEXT_SETTINGS) 83 | def main(): 84 | pass 85 | 86 | 87 | # Setup the large subcommands 88 | main.add_command(alert) 89 | main.add_command(data) 90 | main.add_command(org) 91 | main.add_command(scan) 92 | 93 | 94 | @main.command() 95 | @click.option('--fields', help='List of properties to output.', default=None) 96 | @click.argument('input', metavar='', type=click.Path(exists=True)) 97 | @click.argument('format', metavar='', type=click.Choice(CONVERTERS.keys())) 98 | def convert(fields, input, format): 99 | """Convert the given input data file into a different format. The following file formats are supported: 100 | 101 | kml, csv, geo.json, images, xlsx 102 | 103 | Example: shodan convert data.json.gz kml 104 | """ 105 | file_size = 0 106 | # Check that the converter allows a custom list of fields 107 | converter_class = CONVERTERS.get(format) 108 | if fields: 109 | if not hasattr(converter_class, 'fields'): 110 | raise click.ClickException('File format doesnt support custom list of fields') 111 | converter_class.fields = [item.strip() for item in fields.split(',')] # Use the custom fields the user specified 112 | 113 | # click.Path ensures that file path exists 114 | file_size = os.path.getsize(input) 115 | 116 | # Get the basename for the input file 117 | basename = input.replace('.json.gz', '').replace('.json', '') 118 | 119 | # Add the new file extension based on the format 120 | filename = '{}.{}'.format(basename, format) 121 | 122 | # Open the output file 123 | fout = open(filename, 'w') 124 | 125 | # Start a spinner 126 | finished_event = threading.Event() 127 | progress_bar_thread = threading.Thread(target=async_spinner, args=(finished_event,)) 128 | progress_bar_thread.start() 129 | 130 | # Initialize the file converter 131 | converter = converter_class(fout) 132 | 133 | converter.process([input], file_size) 134 | 135 | finished_event.set() 136 | progress_bar_thread.join() 137 | 138 | if format == 'images': 139 | click.echo(click.style('\rSuccessfully extracted images to directory: {}'.format(converter.dirname), fg='green')) 140 | else: 141 | click.echo(click.style('\rSuccessfully created new file: {}'.format(filename), fg='green')) 142 | 143 | 144 | @main.command(name='domain') 145 | @click.argument('domain', metavar='') 146 | @click.option('--details', '-D', help='Lookup host information for any IPs in the domain results', default=False, is_flag=True) 147 | @click.option('--save', '-S', help='Save the information in the a file named after the domain (append if file exists).', default=False, is_flag=True) 148 | @click.option('--history', '-H', help='Include historical DNS data in the results', default=False, is_flag=True) 149 | @click.option('--type', '-T', help='Only returns DNS records of the provided type', default=None) 150 | def domain_info(domain, details, save, history, type): 151 | """View all available information for a domain""" 152 | key = get_api_key() 153 | api = shodan.Shodan(key) 154 | 155 | try: 156 | info = api.dns.domain_info(domain, history=history, type=type) 157 | except shodan.APIError as e: 158 | raise click.ClickException(e.value) 159 | 160 | # Grab the host information for any IP records that were returned 161 | hosts = {} 162 | if details: 163 | ips = [record['value'] for record in info['data'] if record['type'] in ['A', 'AAAA']] 164 | ips = set(ips) 165 | 166 | fout = None 167 | if save: 168 | filename = u'{}-hosts.json.gz'.format(domain) 169 | fout = helpers.open_file(filename) 170 | 171 | for ip in ips: 172 | try: 173 | hosts[ip] = api.host(ip) 174 | 175 | # Store the banners if requested 176 | if fout: 177 | for banner in hosts[ip]['data']: 178 | if 'placeholder' not in banner: 179 | helpers.write_banner(fout, banner) 180 | except shodan.APIError: 181 | pass # Ignore any API lookup errors as this isn't critical information 182 | 183 | # Save the DNS data 184 | if save: 185 | filename = u'{}.json.gz'.format(domain) 186 | fout = helpers.open_file(filename) 187 | 188 | for record in info['data']: 189 | helpers.write_banner(fout, record) 190 | 191 | click.secho(info['domain'].upper(), fg='green') 192 | 193 | click.echo('') 194 | for record in info['data']: 195 | click.echo( 196 | u'{:32} {:14} {}'.format( 197 | click.style(record['subdomain'], fg='cyan'), 198 | click.style(record['type'], fg='yellow'), 199 | record['value'] 200 | ), 201 | nl=False, 202 | ) 203 | 204 | if record['value'] in hosts: 205 | host = hosts[record['value']] 206 | click.secho(u' Ports: {}'.format(', '.join([str(port) for port in sorted(host['ports'])])), fg='blue', nl=False) 207 | 208 | click.echo('') 209 | 210 | 211 | @main.command() 212 | @click.argument('key', metavar='') 213 | def init(key): 214 | """Initialize the Shodan command-line""" 215 | # Create the directory if necessary 216 | shodan_dir = os.path.expanduser(SHODAN_CONFIG_DIR) 217 | if not os.path.isdir(shodan_dir): 218 | try: 219 | os.makedirs(shodan_dir) 220 | except OSError: 221 | raise click.ClickException('Unable to create directory to store the Shodan API key ({})'.format(shodan_dir)) 222 | 223 | # Make sure it's a valid API key 224 | key = key.strip() 225 | try: 226 | api = shodan.Shodan(key) 227 | api.info() 228 | except shodan.APIError as e: 229 | raise click.ClickException(e.value) 230 | 231 | # Store the API key in the user's directory 232 | keyfile = shodan_dir + '/api_key' 233 | with open(keyfile, 'w') as fout: 234 | fout.write(key.strip()) 235 | click.echo(click.style('Successfully initialized', fg='green')) 236 | 237 | os.chmod(keyfile, 0o600) 238 | 239 | 240 | @main.command() 241 | @click.argument('query', metavar='', nargs=-1) 242 | def count(query): 243 | """Returns the number of results for a search""" 244 | key = get_api_key() 245 | 246 | # Create the query string out of the provided tuple 247 | query = ' '.join(query).strip() 248 | 249 | # Make sure the user didn't supply an empty string 250 | if query == '': 251 | raise click.ClickException('Empty search query') 252 | 253 | # Perform the search 254 | api = shodan.Shodan(key) 255 | try: 256 | results = api.count(query) 257 | except shodan.APIError as e: 258 | raise click.ClickException(e.value) 259 | 260 | click.echo(results['total']) 261 | 262 | 263 | @main.command() 264 | @click.option('--fields', help='Specify the list of properties to download instead of grabbing the full banner', default=None, type=str) 265 | @click.option('--limit', help='The number of results you want to download. -1 to download all the data possible.', default=1000, type=int) 266 | @click.argument('filename', metavar='') 267 | @click.argument('query', metavar='', nargs=-1) 268 | def download(fields, limit, filename, query): 269 | """Download search results and save them in a compressed JSON file.""" 270 | key = get_api_key() 271 | 272 | # Create the query string out of the provided tuple 273 | query = ' '.join(query).strip() 274 | 275 | # Make sure the user didn't supply an empty string 276 | if query == '': 277 | raise click.ClickException('Empty search query') 278 | 279 | filename = filename.strip() 280 | if filename == '': 281 | raise click.ClickException('Empty filename') 282 | 283 | # Add the appropriate extension if it's not there atm 284 | if not filename.endswith('.json.gz'): 285 | filename += '.json.gz' 286 | 287 | # Strip out any whitespace in the fields and turn them into an array 288 | if fields is not None: 289 | fields = [item.strip() for item in fields.split(',')] 290 | 291 | # Perform the search 292 | api = shodan.Shodan(key) 293 | 294 | try: 295 | total = api.count(query)['total'] 296 | info = api.info() 297 | except Exception: 298 | raise click.ClickException('The Shodan API is unresponsive at the moment, please try again later.') 299 | 300 | # Print some summary information about the download request 301 | click.echo('Search query:\t\t\t{}'.format(query)) 302 | click.echo('Total number of results:\t{}'.format(total)) 303 | click.echo('Query credits left:\t\t{}'.format(info['unlocked_left'])) 304 | click.echo('Output file:\t\t\t{}'.format(filename)) 305 | 306 | if limit > total: 307 | limit = total 308 | 309 | # A limit of -1 means that we should download all the data 310 | if limit <= 0: 311 | limit = total 312 | 313 | with helpers.open_file(filename, 'w') as fout: 314 | count = 0 315 | try: 316 | cursor = api.search_cursor(query, minify=False, fields=fields) 317 | with click.progressbar(cursor, length=limit) as bar: 318 | for banner in bar: 319 | helpers.write_banner(fout, banner) 320 | count += 1 321 | 322 | if count >= limit: 323 | break 324 | except Exception: 325 | pass 326 | 327 | # Let the user know we're done 328 | if count < limit: 329 | click.echo(click.style('Notice: fewer results were saved than requested', 'yellow')) 330 | click.echo(click.style(u'Saved {} results into file {}'.format(count, filename), 'green')) 331 | 332 | 333 | @main.command() 334 | @click.option('--format', help='The output format for the host information. Possible values are: pretty, tsv.', default='pretty', type=click.Choice(['pretty', 'tsv'])) 335 | @click.option('--history', help='Show the complete history of the host.', default=False, is_flag=True) 336 | @click.option('--filename', '-O', help='Save the host information in the given file (append if file exists).', default=None) 337 | @click.option('--save', '-S', help='Save the host information in the a file named after the IP (append if file exists).', default=False, is_flag=True) 338 | @click.argument('ip', metavar='') 339 | def host(format, history, filename, save, ip): 340 | """View all available information for an IP address""" 341 | key = get_api_key() 342 | api = shodan.Shodan(key) 343 | 344 | try: 345 | host = api.host(ip, history=history) 346 | 347 | # Print the host information to the terminal using the user-specified format 348 | HOST_PRINT[format](host, history=history) 349 | 350 | # Store the results 351 | if filename or save: 352 | if save: 353 | filename = '{}.json.gz'.format(ip) 354 | 355 | # Add the appropriate extension if it's not there atm 356 | if not filename.endswith('.json.gz'): 357 | filename += '.json.gz' 358 | 359 | # Create/ append to the file 360 | fout = helpers.open_file(filename) 361 | 362 | for banner in sorted(host['data'], key=lambda k: k['port']): 363 | if 'placeholder' not in banner: 364 | helpers.write_banner(fout, banner) 365 | except shodan.APIError as e: 366 | raise click.ClickException(e.value) 367 | 368 | 369 | @main.command() 370 | def info(): 371 | """Shows general information about your account""" 372 | key = get_api_key() 373 | api = shodan.Shodan(key) 374 | try: 375 | results = api.info() 376 | except shodan.APIError as e: 377 | raise click.ClickException(e.value) 378 | 379 | click.echo("""Query credits available: {0} 380 | Scan credits available: {1} 381 | """.format(results['query_credits'], results['scan_credits'])) 382 | 383 | 384 | @main.command() 385 | @click.option('--color/--no-color', default=True) 386 | @click.option('--fields', help='List of properties to output.', default='ip_str,port,hostnames,data') 387 | @click.option('--filters', '-f', help='Filter the results for specific values using key:value pairs.', multiple=True) 388 | @click.option('--filename', '-O', help='Save the filtered results in the given file (append if file exists).') 389 | @click.option('--separator', help='The separator between the properties of the search results.', default=u'\t') 390 | @click.argument('filenames', metavar='', type=click.Path(exists=True), nargs=-1) 391 | def parse(color, fields, filters, filename, separator, filenames): 392 | """Extract information out of compressed JSON files.""" 393 | # Strip out any whitespace in the fields and turn them into an array 394 | fields = [item.strip() for item in fields.split(',')] 395 | 396 | if len(fields) == 0: 397 | raise click.ClickException('Please define at least one property to show') 398 | 399 | has_filters = len(filters) > 0 400 | 401 | # Setup the output file handle 402 | fout = None 403 | if filename: 404 | # If no filters were provided raise an error since it doesn't make much sense w/out them 405 | if not has_filters: 406 | raise click.ClickException('Output file specified without any filters. Need to use filters with this option.') 407 | 408 | # Add the appropriate extension if it's not there atm 409 | if not filename.endswith('.json.gz'): 410 | filename += '.json.gz' 411 | fout = helpers.open_file(filename) 412 | 413 | for banner in helpers.iterate_files(filenames): 414 | row = u'' 415 | 416 | # Validate the banner against any provided filters 417 | if has_filters and not match_filters(banner, filters): 418 | continue 419 | 420 | # Append the data 421 | if fout: 422 | helpers.write_banner(fout, banner) 423 | 424 | # Loop over all the fields and print the banner as a row 425 | for i, field in enumerate(fields): 426 | tmp = u'' 427 | value = get_banner_field(banner, field) 428 | if value: 429 | field_type = type(value) 430 | 431 | # If the field is an array then merge it together 432 | if field_type == list: 433 | tmp = u';'.join(value) 434 | elif field_type in [int, float]: 435 | tmp = u'{}'.format(value) 436 | else: 437 | tmp = escape_data(value) 438 | 439 | # Colorize certain fields if the user wants it 440 | if color: 441 | tmp = click.style(tmp, fg=COLORIZE_FIELDS.get(field, 'white')) 442 | 443 | # Add the field information to the row 444 | if i > 0: 445 | row += separator 446 | row += tmp 447 | 448 | click.echo(row) 449 | 450 | 451 | @main.command() 452 | @click.option('--ipv6', '-6', is_flag=True, default=False, help='Try to use IPv6 instead of IPv4') 453 | def myip(ipv6): 454 | """Print your external IP address""" 455 | key = get_api_key() 456 | 457 | api = shodan.Shodan(key) 458 | 459 | # Use the IPv6-enabled domain if requested 460 | if ipv6: 461 | api.base_url = 'https://apiv6.shodan.io' 462 | 463 | try: 464 | click.echo(api.tools.myip()) 465 | except shodan.APIError as e: 466 | raise click.ClickException(e.value) 467 | 468 | 469 | @main.command() 470 | @click.option('--color/--no-color', default=True) 471 | @click.option('--fields', help='List of properties to show in the search results.', default='ip_str,port,hostnames,data') 472 | @click.option('--limit', help='The number of search results that should be returned. Maximum: 1000', default=100, type=int) 473 | @click.option('--separator', help='The separator between the properties of the search results.', default='\t') 474 | @click.argument('query', metavar='', nargs=-1) 475 | def search(color, fields, limit, separator, query): 476 | """Search the Shodan database""" 477 | key = get_api_key() 478 | 479 | # Create the query string out of the provided tuple 480 | query = ' '.join(query).strip() 481 | 482 | # Make sure the user didn't supply an empty string 483 | if query == '': 484 | raise click.ClickException('Empty search query') 485 | 486 | # For now we only allow up to 1000 results at a time 487 | if limit > 1000: 488 | raise click.ClickException('Too many results requested, maximum is 1,000') 489 | 490 | # Strip out any whitespace in the fields and turn them into an array 491 | fields = [item.strip() for item in fields.split(',')] 492 | 493 | if len(fields) == 0: 494 | raise click.ClickException('Please define at least one property to show') 495 | 496 | # Perform the search 497 | api = shodan.Shodan(key) 498 | try: 499 | results = api.search(query, limit=limit, minify=False, fields=fields) 500 | except shodan.APIError as e: 501 | raise click.ClickException(e.value) 502 | 503 | # Error out if no results were found 504 | if results['total'] == 0: 505 | raise click.ClickException('No search results found') 506 | 507 | # We buffer the entire output so we can use click's pager functionality 508 | output = u'' 509 | for banner in results['matches']: 510 | row = u'' 511 | 512 | # Loop over all the fields and print the banner as a row 513 | for field in fields: 514 | tmp = u'' 515 | value = get_banner_field(banner, field) 516 | if value: 517 | field_type = type(value) 518 | 519 | # If the field is an array then merge it together 520 | if field_type == list: 521 | tmp = u';'.join(value) 522 | elif field_type in [int, float]: 523 | tmp = u'{}'.format(value) 524 | else: 525 | tmp = escape_data(value) 526 | 527 | # Colorize certain fields if the user wants it 528 | if color: 529 | tmp = click.style(tmp, fg=COLORIZE_FIELDS.get(field, 'white')) 530 | 531 | # Add the field information to the row 532 | row += tmp 533 | row += separator 534 | 535 | # click.echo(out + separator, nl=False) 536 | output += row + u'\n' 537 | # click.echo('') 538 | click.echo_via_pager(output) 539 | 540 | 541 | @main.command() 542 | @click.option('--limit', help='The number of results to return.', default=10, type=int) 543 | @click.option('--facets', help='List of facets to get statistics for.', default='country,org') 544 | @click.option('--filename', '-O', help='Save the results in a CSV file of the provided name.', default=None) 545 | @click.argument('query', metavar='', nargs=-1) 546 | def stats(limit, facets, filename, query): 547 | """Provide summary information about a search query""" 548 | # Setup Shodan 549 | key = get_api_key() 550 | api = shodan.Shodan(key) 551 | 552 | # Create the query string out of the provided tuple 553 | query = ' '.join(query).strip() 554 | 555 | # Make sure the user didn't supply an empty string 556 | if query == '': 557 | raise click.ClickException('Empty search query') 558 | 559 | facets = facets.split(',') 560 | facets = [(facet, limit) for facet in facets] 561 | 562 | # Perform the search 563 | try: 564 | results = api.count(query, facets=facets) 565 | except shodan.APIError as e: 566 | raise click.ClickException(e.value) 567 | 568 | # Print the stats tables 569 | for facet in results['facets']: 570 | click.echo('Top {} Results for Facet: {}'.format(len(results['facets'][facet]), facet)) 571 | 572 | for item in results['facets'][facet]: 573 | # Force the value to be a string - necessary because some facet values are numbers 574 | value = u'{}'.format(item['value']) 575 | 576 | click.echo(click.style(u'{:28s}'.format(value), fg='cyan'), nl=False) 577 | click.echo(click.style(u'{:12,d}'.format(item['count']), fg='green')) 578 | 579 | click.echo('') 580 | 581 | # Create the output file if requested 582 | fout = None 583 | if filename: 584 | if not filename.endswith('.csv'): 585 | filename += '.csv' 586 | fout = open(filename, 'w') 587 | writer = csv.writer(fout, dialect=csv.excel) 588 | 589 | # Write the header 590 | writer.writerow(['Query', query]) 591 | 592 | # Add an empty line to separate rows 593 | writer.writerow([]) 594 | 595 | # Write the header that contains the facets 596 | row = [] 597 | for facet in results['facets']: 598 | row.append(facet) 599 | row.append('') 600 | writer.writerow(row) 601 | 602 | # Every facet has 2 columns (key, value) 603 | counter = 0 604 | has_items = True 605 | while has_items: 606 | # pylint: disable=W0612 607 | row = ['' for i in range(len(results['facets']) * 2)] 608 | 609 | pos = 0 610 | has_items = False 611 | for facet in results['facets']: 612 | values = results['facets'][facet] 613 | 614 | # Add the values for the facet into the current row 615 | if len(values) > counter: 616 | has_items = True 617 | row[pos] = values[counter]['value'] 618 | row[pos + 1] = values[counter]['count'] 619 | 620 | pos += 2 621 | 622 | # Write out the row 623 | if has_items: 624 | writer.writerow(row) 625 | 626 | # Move to the next row of values 627 | counter += 1 628 | 629 | 630 | @main.command() 631 | @click.option('--streamer', help='Specify a custom Shodan stream server to use for grabbing data.', default='https://stream.shodan.io', type=str) 632 | @click.option('--fields', help='List of properties to output.', default='ip_str,port,hostnames,data') 633 | @click.option('--separator', help='The separator between the properties of the search results.', default='\t') 634 | @click.option('--datadir', help='Save the stream data into the specified directory as .json.gz files.', default=None, type=str) 635 | @click.option('--asn', help='A comma-separated list of ASNs to grab data on.', default=None, type=str) 636 | @click.option('--alert', help='The network alert ID or "all" to subscribe to all network alerts on your account.', default=None, type=str) 637 | @click.option('--countries', help='A comma-separated list of countries to grab data on.', default=None, type=str) 638 | @click.option('--custom-filters', help='A space-separated list of filters query to grab data on.', default=None, type=str) 639 | @click.option('--ports', help='A comma-separated list of ports to grab data on.', default=None, type=str) 640 | @click.option('--tags', help='A comma-separated list of tags to grab data on.', default=None, type=str) 641 | @click.option('--vulns', help='A comma-separated list of vulnerabilities to grab data on.', default=None, type=str) 642 | @click.option('--limit', help='The number of results you want to download. -1 to download all the data possible.', default=-1, type=int) 643 | @click.option('--compresslevel', help='The gzip compression level (0-9; 0 = no compression, 9 = most compression', default=9, type=int) 644 | @click.option('--timeout', help='Timeout. Should the shodan stream cease to send data, then timeout after seconds.', default=0, type=int) 645 | @click.option('--color/--no-color', default=True) 646 | @click.option('--quiet', help='Disable the printing of information to the screen.', is_flag=True) 647 | def stream(streamer, fields, separator, datadir, asn, alert, countries, custom_filters, ports, tags, vulns, limit, compresslevel, timeout, color, quiet): 648 | """Stream data in real-time.""" 649 | # Setup the Shodan API 650 | key = get_api_key() 651 | api = shodan.Shodan(key) 652 | 653 | # Temporarily change the baseurl 654 | api.stream.base_url = streamer 655 | 656 | # Strip out any whitespace in the fields and turn them into an array 657 | fields = [item.strip() for item in fields.split(',')] 658 | 659 | if len(fields) == 0: 660 | raise click.ClickException('Please define at least one property to show') 661 | 662 | # The user must choose "ports", "countries", "asn" or nothing - can't select multiple 663 | # filtered streams at once. 664 | stream_type = [] 665 | if ports: 666 | stream_type.append('ports') 667 | if countries: 668 | stream_type.append('countries') 669 | if asn: 670 | stream_type.append('asn') 671 | if alert: 672 | stream_type.append('alert') 673 | if tags: 674 | stream_type.append('tags') 675 | if vulns: 676 | stream_type.append('vulns') 677 | if custom_filters: 678 | stream_type.append('custom_filters') 679 | 680 | if len(stream_type) > 1: 681 | raise click.ClickException('Please use --ports, --countries, --custom, --tags, --vulns OR --asn. You cant subscribe to multiple filtered streams at once.') 682 | 683 | stream_args = None 684 | 685 | # Turn the list of ports into integers 686 | if ports: 687 | try: 688 | stream_args = [int(item.strip()) for item in ports.split(',')] 689 | except ValueError: 690 | raise click.ClickException('Invalid list of ports') 691 | 692 | if alert: 693 | alert = alert.strip() 694 | if alert.lower() != 'all': 695 | stream_args = alert 696 | 697 | if asn: 698 | stream_args = asn.split(',') 699 | 700 | if countries: 701 | stream_args = countries.split(',') 702 | 703 | if tags: 704 | stream_args = tags.split(',') 705 | 706 | if vulns: 707 | stream_args = vulns.split(',') 708 | 709 | if custom_filters: 710 | stream_args = custom_filters 711 | 712 | # Flatten the list of stream types 713 | # Possible values are: 714 | # - all 715 | # - asn 716 | # - countries 717 | # - ports 718 | if len(stream_type) == 1: 719 | stream_type = stream_type[0] 720 | else: 721 | stream_type = 'all' 722 | 723 | # Decide which stream to subscribe to based on whether or not ports were selected 724 | def _create_stream(name, args, timeout): 725 | return { 726 | 'all': api.stream.banners(timeout=timeout), 727 | 'alert': api.stream.alert(args, timeout=timeout), 728 | 'asn': api.stream.asn(args, timeout=timeout), 729 | 'countries': api.stream.countries(args, timeout=timeout), 730 | 'custom_filters': api.stream.custom(args, timeout=timeout), 731 | 'ports': api.stream.ports(args, timeout=timeout), 732 | 'tags': api.stream.tags(args, timeout=timeout), 733 | 'vulns': api.stream.vulns(args, timeout=timeout), 734 | }.get(name, 'all') 735 | 736 | stream = _create_stream(stream_type, stream_args, timeout=timeout) 737 | 738 | counter = 0 739 | quit = False 740 | last_time = timestr() 741 | fout = None 742 | 743 | if datadir: 744 | fout = open_streaming_file(datadir, last_time, compresslevel) 745 | 746 | while not quit: 747 | try: 748 | for banner in stream: 749 | # Limit the number of results to output 750 | if limit > 0: 751 | counter += 1 752 | 753 | if counter > limit: 754 | quit = True 755 | break 756 | 757 | # Write the data to the file 758 | if datadir: 759 | cur_time = timestr() 760 | if cur_time != last_time: 761 | last_time = cur_time 762 | fout.close() 763 | fout = open_streaming_file(datadir, last_time) 764 | helpers.write_banner(fout, banner) 765 | 766 | # Print the banner information to stdout 767 | if not quiet: 768 | row = u'' 769 | 770 | # Loop over all the fields and print the banner as a row 771 | for field in fields: 772 | tmp = u'' 773 | value = get_banner_field(banner, field) 774 | if value: 775 | field_type = type(value) 776 | 777 | # If the field is an array then merge it together 778 | if field_type == list: 779 | tmp = u';'.join(value) 780 | elif field_type in [int, float]: 781 | tmp = u'{}'.format(value) 782 | else: 783 | tmp = escape_data(value) 784 | 785 | # Colorize certain fields if the user wants it 786 | if color: 787 | tmp = click.style(tmp, fg=COLORIZE_FIELDS.get(field, 'white')) 788 | 789 | # Add the field information to the row 790 | row += tmp 791 | row += separator 792 | 793 | click.echo(row) 794 | except requests.exceptions.Timeout: 795 | raise click.ClickException('Connection timed out') 796 | except KeyboardInterrupt: 797 | quit = True 798 | except shodan.APIError as e: 799 | raise click.ClickException(e.value) 800 | except Exception: 801 | # For other errors lets just wait a bit and try to reconnect again 802 | time.sleep(1) 803 | 804 | # Create a new stream object to subscribe to 805 | stream = _create_stream(stream_type, stream_args, timeout=timeout) 806 | 807 | 808 | @main.command() 809 | @click.option('--facets', help='List of facets to get summary information on, if empty then show query total results over time', default='', type=str) 810 | @click.option('--filename', '-O', help='Save the full results in the given file (append if file exists).', default=None) 811 | @click.option('--save', '-S', help='Save the full results in the a file named after the query (append if file exists).', default=False, is_flag=True) 812 | @click.argument('query', metavar='', nargs=-1) 813 | def trends(filename, save, facets, query): 814 | """Search Shodan historical database""" 815 | key = get_api_key() 816 | api = shodan.Shodan(key) 817 | 818 | # Create the query string out of the provided tuple 819 | query = ' '.join(query).strip() 820 | facets = facets.strip() 821 | 822 | # Make sure the user didn't supply an empty query or facets 823 | if query == '': 824 | raise click.ClickException('Empty search query') 825 | 826 | # Convert comma-separated facets string to list 827 | parsed_facets = [] 828 | for facet in facets.split(','): 829 | if not facet: 830 | continue 831 | 832 | parts = facet.strip().split(":") 833 | if len(parts) > 1: 834 | parsed_facets.append((parts[0], parts[1])) 835 | else: 836 | parsed_facets.append((parts[0])) 837 | 838 | # Perform the search 839 | try: 840 | results = api.trends.search(query, facets=parsed_facets) 841 | except shodan.APIError as e: 842 | raise click.ClickException(e.value) 843 | 844 | # Error out if no results were found 845 | if results['total'] == 0: 846 | raise click.ClickException('No search results found') 847 | 848 | result_facets = [] 849 | if results.get("facets"): 850 | result_facets = list(results["facets"].keys()) 851 | 852 | # Save the results first to file if user request 853 | if filename or save: 854 | if not filename: 855 | filename = '{}-trends.json.gz'.format(query.replace(' ', '-')) 856 | elif not filename.endswith('.json.gz'): 857 | filename += '.json.gz' 858 | 859 | # Create/ append to the file 860 | with helpers.open_file(filename) as fout: 861 | for index, match in enumerate(results['matches']): 862 | # Append facet info to make up a line 863 | if result_facets: 864 | match["facets"] = {} 865 | for facet in result_facets: 866 | match["facets"][facet] = results['facets'][facet][index]['values'] 867 | 868 | line = json.dumps(match) + '\n' 869 | fout.write(line.encode('utf-8')) 870 | 871 | click.echo(click.style(u'Saved results into file {}'.format(filename), 'green')) 872 | 873 | # We buffer the entire output so we can use click's pager functionality 874 | output = u'' 875 | 876 | # Output examples: 877 | # - Facet by os 878 | # 2017-06 879 | # os 880 | # Linux 3.x 146,502 881 | # Windows 7 or 8 2,189 882 | # 883 | # - Without facets 884 | # 2017-06 19,799,459 885 | # 2017-07 21,077,099 886 | if result_facets: 887 | for index, match in enumerate(results['matches']): 888 | output += click.style(match['month'] + u'\n', fg='green') 889 | if match['count'] > 0: 890 | for facet in result_facets: 891 | output += click.style(u' {}\n'.format(facet), fg='cyan') 892 | for bucket in results['facets'][facet][index]['values']: 893 | output += u' {:60}{}\n'.format(click.style(bucket['value'], bold=True), click.style(u'{:20,d}'.format(bucket['count']), fg='green')) 894 | else: 895 | output += u'{}\n'.format(click.style('N/A', bold=True)) 896 | else: 897 | # Without facets, show query total results over time 898 | for index, match in enumerate(results['matches']): 899 | output += u'{:20}{}\n'.format(click.style(match['month'], bold=True), click.style(u'{:20,d}'.format(match['count']), fg='green')) 900 | 901 | click.echo_via_pager(output) 902 | 903 | 904 | @main.command() 905 | @click.argument('ip', metavar='') 906 | def honeyscore(ip): 907 | """Check whether the IP is a honeypot or not.""" 908 | key = get_api_key() 909 | api = shodan.Shodan(key) 910 | 911 | try: 912 | score = api.labs.honeyscore(ip) 913 | 914 | if score == 1.0: 915 | click.echo(click.style('Honeypot detected', fg='red')) 916 | elif score > 0.5: 917 | click.echo(click.style('Probably a honeypot', fg='yellow')) 918 | else: 919 | click.echo(click.style('Not a honeypot', fg='green')) 920 | 921 | click.echo('Score: {}'.format(score)) 922 | except Exception: 923 | raise click.ClickException('Unable to calculate honeyscore') 924 | 925 | 926 | @main.command() 927 | def radar(): 928 | """Real-Time Map of some results as Shodan finds them.""" 929 | key = get_api_key() 930 | api = shodan.Shodan(key) 931 | 932 | from shodan.cli.worldmap import launch_map 933 | 934 | try: 935 | launch_map(api) 936 | except shodan.APIError as e: 937 | raise click.ClickException(e.value) 938 | except Exception as e: 939 | raise click.ClickException(u'{}'.format(e)) 940 | 941 | 942 | @main.command() 943 | def version(): 944 | """Print version of this tool.""" 945 | print(pkg_resources.get_distribution("shodan").version) 946 | 947 | 948 | if __name__ == '__main__': 949 | main() 950 | --------------------------------------------------------------------------------