├── .gitignore ├── LICENSE ├── README.md ├── ndcrawl-sample.ini ├── ndcrawl.py ├── ndlib ├── __init__.py ├── execute.py ├── log.py ├── output.py ├── parse.py └── topology.py └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | venv/ 3 | start-env.sh 4 | *.pyc 5 | *.log 6 | output/ 7 | *.csv 8 | ndcrawl.ini 9 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2017 Jonathan Yantis 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CDP/LLDP Network Discovery Crawler for Cisco networks 2 | 3 | This is experimental at the moment. Uses netmiko and some seed devices to scrape 4 | your network and output CSV files of all neighbor topology data, as well as a 5 | device list file. Uses threaded connections at each iteration, moving out from 6 | seed devices to next level of devices until all devices are discovered. 7 | 8 | Uses a BFS algorithm with netmiko, and calculates distances from seed devices. 9 | Currently only works with SSH access to Cisco IOS and NXOS devices, but other 10 | devices could be added easily. Will gather all neighbors and devices found, but 11 | can only scrape cisco devices to discover next level devices at the moment. 12 | 13 | ## Usage Example: Scrape network starting with the core devices 14 | 15 | * Note: The initial devices should have the same device ID in the cdp neighbors 16 | to avoid duplicate device entries and proper distance calculations 17 | 18 | ```./ndcrawl.py -seed core1.domain.com,core2.domain.com --user yantisj -nei_file nd.csv -dev_file devices.csv --debug 1``` 19 | 20 | ## Config File Notes 21 | 22 | Copy the ndcrawl-sample.ini to ndcrawl.ini and edit options. All CLI options can be specified 23 | from the config file for daemonizing the script. CLI options always override the config file. 24 | 25 | ## Neighbor List Output Example 26 | ``` 27 | local_device_id,distance,remote_device_id,platform,local_int,remote_int,ipv4,os 28 | core1.domain.com,0,mdcoobsw1.domain.com,WS-C4948,mgmt0,GigabitEthernet1/1,10.25.9.1,cisco_ios 29 | core1.domain.com,0,servchas1.domain.com,WS-C6504-E,Ethernet7/25,TenGigabitEthernet3/1,10.24.70.51,cisco_ios 30 | core1.domain.com,0,core2.domain.com,N7K-C7010,Ethernet7/26,Ethernet8/26,10.25.156.103,cisco_nxos 31 | core1.domain.com,0,artmdf1.domain.com,N7K-C7010,Ethernet7/27,Ethernet2/26,10.25.80.103,cisco_nxos 32 | ``` 33 | 34 | ## Device List Output Example 35 | ``` 36 | device_id,ipv4,platform,os 37 | core1.domain.com,10.25.9.103,N7K-C7010,cisco_nxos 38 | cnew1.domain.com,10.25.9.10,N9K-C93180YC-EX,cisco_nxos 39 | ``` 40 | -------------------------------------------------------------------------------- /ndcrawl-sample.ini: -------------------------------------------------------------------------------- 1 | # Copy this file to ndcrawl.ini before editing 2 | [main] 3 | log_file = ndcrawl.log 4 | 5 | # Max Threads 6 | thread_count = 50 7 | 8 | # Ignore any CDP neighbors that match this regex 9 | ignore_regex = (oobsw|lab) 10 | 11 | # Max devices to crawl 12 | max_crawl = 10000 13 | 14 | # Seed OS type (otherwise discovered) 15 | seed_os = cisco_nxos 16 | 17 | ## Optional Variables, define to enable 18 | 19 | # Seeds are comma separated with no spaces 20 | seeds = 21 | 22 | # Stored username and password (define to enable) 23 | username = 24 | password = 25 | 26 | # Neighbor File 27 | nei_file = neighbors.csv 28 | 29 | # Device File 30 | dev_file = devices.csv 31 | -------------------------------------------------------------------------------- /ndcrawl.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os.path 3 | import sys 4 | import argparse 5 | import configparser 6 | import logging 7 | import getpass 8 | from ndlib.log import init_logging 9 | from ndlib import topology 10 | from ndlib import parse 11 | 12 | CONFIG_FILE = 'ndcrawl.ini' 13 | 14 | parser = argparse.ArgumentParser(description='Discover Network Topology via CDP/LLDP') 15 | parser.add_argument('-seed', metavar="switch1[,switch2]", help="Seed devices to start crawl") 16 | parser.add_argument('-nei_file', metavar="file", help="Output Neighbors to File", type=str) 17 | parser.add_argument('-dev_file', metavar="file", help="Output Neighbors to File", type=str) 18 | parser.add_argument('-ng_file', metavar="file", help="Output NetGrph Topology File", type=str) 19 | parser.add_argument('--quiet', help='Quiet output, log to file only', action="store_true") 20 | parser.add_argument("--seed_os", metavar='cisco_nxos', help="Netmiko OS type for seed devices", 21 | type=str) 22 | parser.add_argument("--seed_file", metavar='file', help="Seed devices from a file, one per line", 23 | type=str) 24 | parser.add_argument("--user", metavar='username', help="Username to execute as", 25 | type=str) 26 | parser.add_argument("--max_crawl", metavar='int', help="Max devices to crawl (default 10000)", 27 | type=int) 28 | parser.add_argument("--conf", metavar='file', help="Alternate Config File", 29 | type=str) 30 | parser.add_argument("--debug", help="Set debugging level", type=int) 31 | parser.add_argument("-v", help="Verbose Output", action="store_true") 32 | 33 | args = parser.parse_args() 34 | 35 | log_level = logging.WARNING 36 | logging.getLogger('paramiko').setLevel(logging.WARNING) 37 | 38 | if args.v and not args.debug: 39 | args.debug = 1 40 | 41 | if args.debug: 42 | if args.debug: 43 | log_level = logging.INFO 44 | if args.debug > 1: 45 | log_level = logging.DEBUG 46 | if args.debug > 1 and args.debug < 3: 47 | logging.getLogger('netmiko').setLevel(logging.INFO) 48 | logging.getLogger('paramiko').setLevel(logging.INFO) 49 | 50 | logger = logging.getLogger('ndcrawl.py') 51 | 52 | # Local config files to import 53 | config = configparser.ConfigParser() 54 | 55 | if os.path.exists(CONFIG_FILE): 56 | config.read(CONFIG_FILE) 57 | else: 58 | logger.warning('Warning: Loading Sample Config File: Please create ndcrawl.ini from ndcrawl-sample.ini') 59 | config.read('ndcrawl-sample.ini') 60 | 61 | config['main']['log_level'] = str(log_level) 62 | topology.config = config 63 | parse.config = config 64 | 65 | if args.quiet: 66 | config['main']['quiet'] = '1' 67 | else: 68 | config['main']['quiet'] = '' 69 | 70 | init_logging(log_level, config['main']['log_file'], args.quiet) 71 | 72 | if args.max_crawl: 73 | config['main']['max_crawl'] = str(args.max_crawl) 74 | 75 | if args.seed_os: 76 | config['main']['seed_os'] = args.seed_os 77 | 78 | if not args.seed: 79 | if 'seeds' in config['main'] and config['main']['seeds']: 80 | args.seed = config['main']['seeds'] 81 | 82 | if args.seed or args.seed_file: 83 | 84 | if not args.user: 85 | if 'username' in config['main'] and config['main']['username']: 86 | args.user = config['main']['username'] 87 | else: 88 | print('\nError: Must provide --user if not using config file\n') 89 | sys.exit(1) 90 | if 'password' in config['main'] and config['main']['password']: 91 | password = config['main']['password'] 92 | else: 93 | password = getpass.getpass('Password for ' + args.user + ': ') 94 | 95 | # Check for output files from config 96 | if not args.nei_file: 97 | if 'nei_file' in config['main'] and config['main']['nei_file']: 98 | args.nei_file = config['main']['nei_file'] 99 | if not args.dev_file: 100 | if 'dev_file' in config['main'] and config['main']['dev_file']: 101 | args.dev_file = config['main']['dev_file'] 102 | 103 | if args.seed_file: 104 | seeds = list() 105 | f = open(args.seed_file, 'r') 106 | for l in f: 107 | l = l.strip() 108 | if l: 109 | seeds.append(l) 110 | else: 111 | seeds = args.seed.split(',') 112 | 113 | if not args.quiet: 114 | print('Beginning crawl on:', ', '.join(seeds)) 115 | 116 | topology.crawl(seeds, args.user, password, outf=args.nei_file, dout=args.dev_file, ngout=args.ng_file) 117 | else: 118 | print('\nError: Must provide -seed devices if not using config file\n') 119 | parser.print_help() 120 | -------------------------------------------------------------------------------- /ndlib/__init__.py: -------------------------------------------------------------------------------- 1 | 'Neighbor Discovery Crawler Library' 2 | 3 | -------------------------------------------------------------------------------- /ndlib/execute.py: -------------------------------------------------------------------------------- 1 | 'Netmiko Routines' 2 | import logging 3 | import re 4 | from time import sleep 5 | from netmiko import ConnectHandler 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | def get_session(host, platform, username, password): 10 | """ Get an SSH session on device """ 11 | 12 | net_connect = ConnectHandler(device_type=platform, 13 | ip=host, 14 | global_delay_factor=0.2, 15 | username=username, 16 | password=password, 17 | timeout=20) 18 | 19 | net_connect.enable() 20 | 21 | return net_connect 22 | 23 | def send_command_timing(session, cmd, delay_factor=1, host=''): 24 | """ Send command and return results as list """ 25 | 26 | logger.debug('Executing Command on %s: %s', host, cmd) 27 | results = session.send_command_timing(cmd, delay_factor=delay_factor) 28 | return results.split('\n') 29 | 30 | def send_command(session, cmd, host=''): 31 | """ Send command and return results as list """ 32 | 33 | logger.debug('Executing Command on %s: %s', host, cmd) 34 | results = session.send_command(cmd) 35 | return results.split('\n') 36 | -------------------------------------------------------------------------------- /ndlib/log.py: -------------------------------------------------------------------------------- 1 | 'Logging Initialization' 2 | import logging 3 | from logging.handlers import DatagramHandler 4 | 5 | logger = logging.getLogger(__name__) 6 | 7 | def init_logging(log_level, log_file, quiet): 8 | 'Initialize Logging Globally' 9 | 10 | # Specify our log format for handlers 11 | log_format = logging.Formatter('%(asctime)s %(name)s:%(levelname)s: %(message)s') 12 | 13 | # Get the root_logger to attach log handlers to 14 | root_logger = logging.getLogger() 15 | 16 | # Set root logger to debug level always 17 | # Control log levels at log handler level 18 | root_logger.setLevel(logging.DEBUG) 19 | 20 | if not quiet: 21 | # Console Handler (always use log_level) 22 | ch = logging.StreamHandler() 23 | ch.setLevel(log_level) 24 | ch.setFormatter(log_format) 25 | root_logger.addHandler(ch) 26 | 27 | # Logfile Handler 28 | fh = logging.FileHandler(log_file) 29 | 30 | # Always log at INFO or below 31 | if log_level < logging.INFO: 32 | fh.setLevel(log_level) 33 | else: 34 | fh.setLevel(logging.INFO) 35 | 36 | # Attach logfile handler 37 | fh.setFormatter(log_format) 38 | root_logger.addHandler(fh) 39 | 40 | # # Attach Datagram Handler 41 | # dh = DatagramHandler('232.8.8.8', port=1900) 42 | # dh.setFormatter(log_format) 43 | # dh.setLevel(log_level) 44 | # root_logger.addHandler(dh) 45 | -------------------------------------------------------------------------------- /ndlib/output.py: -------------------------------------------------------------------------------- 1 | 'File output routines' 2 | import csv 3 | import logging 4 | 5 | logger = logging.getLogger(__name__) 6 | 7 | def output_files(outf, ngout, dout, neighbors, devices, distances): 8 | """ Output files to CSV if requested """ 9 | 10 | # Output Neighbor CSV File 11 | if outf: 12 | fieldnames = ['local_device_id', 'remote_device_id', 'distance', 'local_int', \ 13 | 'remote_int', 'ipv4', 'os', 'platform', 'description'] 14 | f = open(outf, 'w') 15 | dw = csv.DictWriter(f, fieldnames=fieldnames) 16 | dw.writeheader() 17 | for n in neighbors: 18 | nw = n.copy() 19 | if 'logged_in' in nw: 20 | nw.pop('logged_in') 21 | dw.writerow(nw) 22 | f.close() 23 | 24 | # Output NetGrph CSV File 25 | if ngout: 26 | fieldnames = ['LocalName', 'LocalPort', 'RemoteName', 'RemotePort'] 27 | f = open(ngout, 'w') 28 | dw = csv.DictWriter(f, fieldnames=fieldnames) 29 | dw.writeheader() 30 | for n in neighbors: 31 | 32 | ng = {'LocalName': n['local_device_id'].split('.')[0], 33 | 'LocalPort': n['local_int'], 34 | 'RemoteName': n['remote_device_id'].split('.')[0], 35 | 'RemotePort': n['remote_int'], 36 | } 37 | dw.writerow(ng) 38 | f.close() 39 | 40 | if dout: 41 | fieldnames = ['device_id', 'ipv4', 'platform', 'os', 'distance', 'logged_in'] 42 | f = open(dout, 'w') 43 | dw = csv.DictWriter(f, fieldnames=fieldnames) 44 | dw.writeheader() 45 | for d in sorted(devices): 46 | dist = 100 47 | if devices[d]['remote_device_id'] in distances: 48 | dist = distances[devices[d]['remote_device_id']] 49 | 50 | logged_in = False 51 | if 'logged_in' in devices[d] and devices[d]['logged_in']: 52 | logged_in = True 53 | 54 | dd = {'device_id': devices[d]['remote_device_id'], 'ipv4': devices[d]['ipv4'], \ 55 | 'platform': devices[d]['platform'], 'os': devices[d]['os'], \ 56 | 'distance': dist, 'logged_in': logged_in} 57 | dw.writerow(dd) 58 | -------------------------------------------------------------------------------- /ndlib/parse.py: -------------------------------------------------------------------------------- 1 | 'Parser Routines' 2 | import re 3 | import logging 4 | 5 | logger = logging.getLogger(__name__) 6 | 7 | config = dict() 8 | 9 | def merge_nd(nd_cdp, nd_lldp): 10 | """ Merge CDP and LLDP data into one structure """ 11 | 12 | neis = dict() 13 | nd = list() 14 | 15 | for n in nd_lldp: 16 | neis[(n['local_device_id'], n['remote_device_id'], n['local_int'], n['remote_int'])] = n 17 | 18 | for n in nd_cdp: 19 | 20 | # Always prefer CDP, but grab description from LLDP if available 21 | if (n['local_device_id'], n['remote_device_id'], n['local_int'], n['remote_int']) in n: 22 | if 'description' in neis[(n['local_device_id'], n['remote_device_id'], n['local_int'], n['remote_int'])]: 23 | n['description'] = neis[(n['local_device_id'], n['remote_device_id'], n['local_int'], n['remote_int'])]['description'] 24 | neis[(n['local_device_id'], n['remote_device_id'], n['local_int'], n['remote_int'])] = n 25 | 26 | 27 | for n in neis: 28 | nd.append(neis[n]) 29 | 30 | return nd 31 | 32 | def parse_cdp(cdp, device): 33 | 'Return nd neighbors for IOS/NXOS CDP output' 34 | 35 | current = dict() 36 | dname = device['remote_device_id'] 37 | nd = list() 38 | 39 | for l in cdp: 40 | l = l.rstrip() 41 | devid = re.search(r'^Device\sID\:\s*([A-Za-z0-9\.\-\_]+)', l) 42 | platform = re.search(r'^Platform\:\s([A-Za-z0-9\.\-\_]+)\s*([A-Za-z0-9\.\-\_]*)', l) 43 | ints = re.search(r'^Interface\:\s([A-Za-z0-9\.\-\_\/]+).*\:\s([A-Za-z0-9\.\-\_\/]+)$', l) 44 | ipv4 = re.search(r'^\s+IPv4\sAddress\:\s(\d+\.\d+\.\d+\.\d+)', l) 45 | ip = re.search(r'^\s+IP\saddress\:\s(\d+\.\d+\.\d+\.\d+)', l) 46 | nxos = re.search(r'Cisco Nexus', l) 47 | ios = re.search(r'Cisco IOS', l) 48 | if devid: 49 | if current: 50 | if not re.search(config['main']['ignore_regex'], current['remote_device_id']): 51 | nd.append(current.copy()) 52 | else: 53 | logger.info('Regex Ignore on %s neighbor from %s', \ 54 | current['remote_device_id'], current['local_device_id']) 55 | current = dict() 56 | rname = devid.group(1) 57 | current['local_device_id'] = dname 58 | current['remote_device_id'] = rname 59 | current['platform'] = 'Unknown' 60 | current['local_int'] = 'Unknown' 61 | current['remote_int'] = 'Unknown' 62 | current['ipv4'] = 'Unknown' 63 | current['os'] = 'Unknown' 64 | current['description'] = '' 65 | if ints: 66 | #print(l, ints.group(1), ints.group(2)) 67 | current['local_int'] = ints.group(1) 68 | current['remote_int'] = ints.group(2) 69 | if ipv4: 70 | current['ipv4'] = ipv4.group(1) 71 | if ip: 72 | current['ipv4'] = ip.group(1) 73 | if platform: 74 | if platform.group(1) == 'cisco': 75 | current['platform'] = platform.group(2) 76 | else: 77 | current['platform'] = platform.group(1) 78 | if nxos: 79 | current['os'] = 'cisco_nxos' 80 | if ios: 81 | current['os'] = 'cisco_ios' 82 | 83 | if current: 84 | if not re.search(config['main']['ignore_regex'], current['remote_device_id']): 85 | nd.append(current.copy()) 86 | else: 87 | logger.warning('Regex Ignore on %s neighbor from %s', \ 88 | current['remote_device_id'], current['local_device_id']) 89 | return nd 90 | 91 | 92 | def parse_lldp(lldp_det, lldp_sum, device): 93 | 'Return nd neighbors for IOS/NXOS LLDP output' 94 | 95 | current = dict() 96 | dname = device['remote_device_id'] 97 | nd = list() 98 | dmap = dict() 99 | 100 | # Get local ports from summary, store in dmap 101 | for l in lldp_sum: 102 | l = l[20:] 103 | ln = l.split() 104 | if len(ln) > 3 and re.search(r'\w+\d+\/\d+', ln[0]): 105 | dmap[ln[3]] = ln[0] 106 | elif len(ln) == 3 and re.search(r'\w+\d+\/\d+', ln[0]): 107 | dmap[ln[2]] = ln[0] 108 | 109 | for l in lldp_det: 110 | l = l.rstrip() 111 | devid = re.search(r'^Chassis\sid\:\s*([A-Za-z0-9\.\-\_]+)', l) 112 | sysname = re.search(r'^System\sName\:\s*([A-Za-z0-9\.\-\_]+)', l) 113 | platform = re.search(r'^Platform\:\s([A-Za-z0-9\.\-\_]+)\s*([A-Za-z0-9\.\-\_]*)', l) 114 | l_int = re.search(r'^Local\sPort\sid\:\s([A-Za-z0-9\.\-\_\/]+)$', l) 115 | r_int = re.search(r'^Port\sid\:\s([A-Za-z0-9\.\-\_\/]+)$', l) 116 | ipv4 = re.search(r'^\s+IP\:\s(\d+\.\d+\.\d+\.\d+)', l) 117 | ip = re.search(r'^Management\sAddress\:\s(\d+\.\d+\.\d+\.\d+)', l) 118 | desc = re.search(r'Port\sDescription\:\s(.*)', l) 119 | nxos = re.search(r'Cisco Nexus', l) 120 | ios = re.search(r'Cisco IOS', l) 121 | if devid: 122 | if current: 123 | if not re.search(config['main']['ignore_regex'], current['remote_device_id']): 124 | nd.append(current.copy()) 125 | else: 126 | logger.info('Regex Ignore on %s neighbor from %s', \ 127 | current['remote_device_id'], current['local_device_id']) 128 | current = dict() 129 | rname = devid.group(1) 130 | current['local_device_id'] = dname 131 | current['remote_device_id'] = rname 132 | current['platform'] = 'Unknown' 133 | current['local_int'] = 'Unknown' 134 | current['remote_int'] = 'Unknown' 135 | current['ipv4'] = 'Unknown' 136 | current['os'] = 'Unknown' 137 | current['description'] = '' 138 | 139 | 140 | if sysname: 141 | if not re.search('advertised', sysname.group(1)): 142 | current['remote_device_id'] = sysname.group(1) 143 | if r_int: 144 | current['remote_int'] = r_int.group(1) 145 | 146 | # Try to map interface via summary if Unknown (IOS) 147 | if device['os'] == 'cisco_ios': 148 | if current['remote_int'] in dmap: 149 | logger.debug('Mapping %s local interface %s to chassis id %s', \ 150 | dname, dmap[current['remote_int']], current['remote_int']) 151 | current['local_int'] = dmap[current['remote_int']] 152 | elif current['remote_device_id'] in dmap: 153 | current['local_int'] = dmap[current['remote_device_id']] 154 | else: 155 | logger.info('No LLDP mapping for %s on %s', current['remote_int'], current['local_device_id']) 156 | if l_int: 157 | current['local_int'] = l_int.group(1) 158 | if ipv4: 159 | current['ipv4'] = ipv4.group(1) 160 | if ip: 161 | current['ipv4'] = ip.group(1) 162 | if platform: 163 | if platform.group(1) == 'cisco': 164 | current['platform'] = platform.group(2) 165 | else: 166 | current['platform'] = platform.group(1) 167 | if desc: 168 | current['description'] = desc.group(1).strip() 169 | if nxos: 170 | current['os'] = 'cisco_nxos' 171 | if ios: 172 | current['os'] = 'cisco_ios' 173 | 174 | if current: 175 | if not re.search(config['main']['ignore_regex'], current['remote_device_id']): 176 | nd.append(current.copy()) 177 | else: 178 | logger.warning('Regex Ignore on %s neighbor from %s', \ 179 | current['remote_device_id'], current['local_device_id']) 180 | return nd 181 | -------------------------------------------------------------------------------- /ndlib/topology.py: -------------------------------------------------------------------------------- 1 | 'Topology Routines' 2 | import logging 3 | import csv 4 | import threading 5 | from time import sleep 6 | from queue import Queue 7 | from . import execute 8 | from . import parse 9 | from . import output 10 | #from progressbar import ProgressBar 11 | from tqdm import tqdm 12 | 13 | logger = logging.getLogger(__name__) 14 | 15 | config = dict() 16 | 17 | def crawl(seeds, username, password, outf=None, dout=None, ngout=None): 18 | 'Crawl CDP/LLDP Neighbors to build a topology' 19 | 20 | # Queue for devices to scrape next 21 | q = Queue() 22 | 23 | # Queue for neighbor output from threads 24 | out_q = Queue() 25 | 26 | # Visited list for loop detection 27 | visited = list() 28 | 29 | # All Neighbor Entries 30 | neighbors = list() 31 | 32 | # Device entries for connection details (ipv4, os etc) 33 | devices = dict() 34 | 35 | # Thread tracking 36 | qtrack = dict() 37 | 38 | # Thread previous join attempts 39 | joined = list() 40 | 41 | # Distance tracking 42 | distances = dict() 43 | 44 | # Counter 45 | crawl_count = 0 46 | iteration_count = 0 47 | 48 | # Queue up seed devices 49 | for s in seeds: 50 | q.put(s) 51 | devices[s] = dict() 52 | devices[s]['remote_device_id'] = s 53 | devices[s]['ipv4'] = s 54 | devices[s]['os'] = config['main']['seed_os'] 55 | devices[s]['platform'] = 'Unknown' 56 | devices[s]['logged_in'] = True 57 | distances[s] = 0 58 | 59 | # Outer Queue, starts inner queue and then adds all unvisited neighbors to queue when 60 | # inner queue is empty. Each iteration of outer queue visits all next level neighbors 61 | # at once inside inner queue via threads. 62 | while not q.empty(): 63 | iteration_count += 1 64 | cqsize = q.qsize() 65 | if not config['main']['quiet']: 66 | if int(config['main']['log_level']) >= logging.WARNING and iteration_count > 1: 67 | pbar = tqdm(total=cqsize, unit='dev') 68 | pbar.set_description('Iteration %s' % str(iteration_count)) 69 | 70 | # Launch threads on everything in queue to scrape 71 | while not q.empty(): 72 | current = q.get() 73 | qsize = q.qsize() 74 | 75 | # Progress bar on warning level or above 76 | if not config['main']['quiet']: 77 | if int(config['main']['log_level']) >= logging.WARNING and iteration_count > 1: 78 | p_int = (cqsize - qsize) 79 | pbar.update(1) 80 | print('\r', end='') 81 | 82 | if crawl_count > int(config['main']['max_crawl']): 83 | logger.warning('Max Devices allowed already crawled') 84 | 85 | # Only scrape unvisited devices 86 | elif current not in visited: 87 | crawl_count += 1 88 | 89 | visited.append(current) 90 | while threading.activeCount() > int(config['main']['thread_count']): 91 | qsize = q.qsize() 92 | logger.debug('Waiting for free thread - Q Size: %s', str(qsize)) 93 | sleep(1) 94 | # Throttle connections 95 | sleep(0.1) 96 | logger.info('Processing %s', current) 97 | 98 | # Start thread to scrape devices 99 | nd_thread = threading.Thread(target=gather_nd, \ 100 | kwargs={"device": devices[current], "username": username, \ 101 | "password": password, "out_q": out_q, \ 102 | "qtrack": qtrack}) 103 | nd_thread.start() 104 | 105 | # Join all threads from last iteration and warn if problems joining threads 106 | logger.info('Joining all active threads') 107 | main_thread = threading.currentThread() 108 | wait_timer = 15 109 | for some_thread in threading.enumerate(): 110 | if some_thread != main_thread: 111 | tid = str(some_thread.ident) 112 | if tid in qtrack: 113 | tid = qtrack[tid] 114 | if tid not in joined: 115 | joined.append(tid) 116 | logger.debug('Joining Thread: %s', tid) 117 | some_thread.join(timeout=wait_timer) 118 | wait_timer = 1 119 | else: 120 | logger.info('Thread running long time, ignoring: %s: %s', tid, str(some_thread)) 121 | 122 | # Process output queue of neighbor data and look for new neighbors to visit 123 | logger.info('Processing output queue') 124 | while not out_q.empty(): 125 | nd = out_q.get() 126 | 127 | # Gather distance info 128 | for n in nd: 129 | if n['local_device_id'] not in distances: 130 | distances[n['local_device_id']] = 100 131 | if n['remote_device_id'] in distances: 132 | if distances[n['local_device_id']] > (distances[n['remote_device_id']] + 1): 133 | distances[n['local_device_id']] = distances[n['remote_device_id']] + 1 134 | logger.info('Found new distances on %s: %s', n['local_device_id'], \ 135 | str(distances[n['remote_device_id']] + 1)) 136 | 137 | # Save all neighbor data 138 | for n in nd: 139 | n['distance'] = distances[n['local_device_id']] 140 | neighbors.append(n) 141 | rname = n['remote_device_id'] 142 | 143 | # Save device to devices 144 | if rname not in devices: 145 | devices[rname] = n 146 | # Update unknown devices, restore logged_in variable 147 | elif devices[rname]['platform'] == 'Unknown': 148 | logged_in = False 149 | if 'logged_in' in devices[rname]: 150 | logged_in = devices[rname]['logged_in'] 151 | devices[rname] = n 152 | devices[rname]['logged_in'] = logged_in 153 | 154 | # Save logged_in as False initially, update on another pass 155 | if 'logged_in' not in devices[n['local_device_id']]: 156 | devices[n['local_device_id']]['logged_in'] = False 157 | 158 | # Local device always was logged in to 159 | devices[n['local_device_id']]['logged_in'] = True 160 | logger.info('Processing Out_q entry %s on %s', rname, n['local_device_id']) 161 | 162 | # New Neighbor that has not been scraped, only scrape IOS/NXOS for now 163 | if rname not in visited: 164 | if n['os'] == 'cisco_nxos': 165 | if rname not in q.queue: 166 | q.put(rname) 167 | elif n['os'] == 'cisco_ios': 168 | if rname not in q.queue: 169 | q.put(rname) 170 | else: 171 | visited.append(rname) 172 | else: 173 | logger.debug('Already visited %s', rname) 174 | 175 | # Count Neighbors 176 | ncount = 0 177 | for n in neighbors: 178 | ncount += 1 179 | logger.info('Total neighbors: %s', str(ncount)) 180 | 181 | 182 | output.output_files(outf, ngout, dout, neighbors, devices, distances) 183 | 184 | 185 | def gather_nd(**kwargs): 186 | 'Gather neighbors from device' 187 | 188 | out_q = kwargs['out_q'] 189 | device = kwargs['device'] 190 | dname = device['remote_device_id'] 191 | tid = str(threading.get_ident()) 192 | kwargs["qtrack"][tid] = dname 193 | 194 | logger.info('Gathering Neighbors on %s: %s', dname, tid) 195 | 196 | nd = dict() 197 | 198 | # Try to connect to device id first 199 | try: 200 | nd = scrape_device(device, dname, kwargs['username'], kwargs['password']) 201 | 202 | except Exception as e: 203 | if device['ipv4'] == 'Unknown': 204 | logger.warning('Connection to %s failed and IPv4 is Unknown', dname) 205 | 206 | # Try IPv4 Address 207 | else: 208 | logger.info('Failed to connect to %s, trying %s', dname, device['ipv4']) 209 | try: 210 | nd = scrape_device(device, device['ipv4'], kwargs['username'], kwargs['password']) 211 | except Exception as e: 212 | logger.warning('Failed to scrape %s: %s', dname, str(e)) 213 | if nd: 214 | out_q.put(nd) 215 | logger.info('Completed Scraping %s: %s', dname, tid) 216 | 217 | def scrape_device(device, host, username, password): 218 | """ Scrape a device and return the results as list of neighbors """ 219 | 220 | dname = device['remote_device_id'] 221 | 222 | ses = execute.get_session(host, device['os'], username, password) 223 | 224 | cdp = execute.send_command(ses, 'show cdp neighbor detail', dname) 225 | 226 | lldp = execute.send_command(ses, 'show lldp neighbor detail', dname) 227 | lldp_sum = execute.send_command(ses, 'show lldp neighbor', dname) 228 | 229 | if device['os'] == 'cisco_nxos': 230 | nd_cdp = parse.parse_cdp(cdp, device) 231 | nd_lldp = parse.parse_lldp(lldp, lldp_sum, device) 232 | elif device['os'] == 'cisco_ios': 233 | nd_cdp = parse.parse_cdp(cdp, device) 234 | nd_lldp = parse.parse_lldp(lldp, lldp_sum, device) 235 | else: 236 | logger.warning('Unknown OS Type to Parse on %s: %s', dname, device['os']) 237 | 238 | for n in nd_cdp: 239 | logger.debug('Found Neighbor %s on %s', n, dname) 240 | ses.disconnect() 241 | 242 | nd = parse.merge_nd(nd_cdp, nd_lldp) 243 | 244 | return nd 245 | 246 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pylint 2 | netmiko 3 | tqdm 4 | --------------------------------------------------------------------------------