├── templates ├── disp_dev.template ├── disp_ip_vpn_inst_intf.template ├── disp_ip_int_br.template ├── disp_ip_vpn_inst.template ├── disp_ip_vpn.template ├── disp_arp_int.template ├── disp_cur_int.template ├── disp_ip_int.template ├── disp_vpn_intf.template ├── disp_esn.template ├── disp_elabel.template ├── disp_int.template └── index ├── pdb_test.py ├── vendor_selector.py ├── README.md ├── nb_ipam_resolver.py └── huawei └── huawei.py /templates/disp_dev.template: -------------------------------------------------------------------------------- 1 | Value Model (.*) 2 | 3 | Start 4 | ^${Model}'s Device 5 | -------------------------------------------------------------------------------- /templates/disp_ip_vpn_inst_intf.template: -------------------------------------------------------------------------------- 1 | Value NAME (\S+) 2 | Value List ADDRESS ([^,]*) 3 | 4 | Start 5 | ^.* ID : ${NAME}.* 6 | 7 | -------------------------------------------------------------------------------- /templates/disp_ip_int_br.template: -------------------------------------------------------------------------------- 1 | Value INTERFACE (\S+) 2 | Value ADDRESS (\d+(.\d+){3}/(\d+)) 3 | 4 | Start 5 | ^${INTERFACE}\s+${ADDRESS}\s+up\s+up 6 | -------------------------------------------------------------------------------- /templates/disp_ip_vpn_inst.template: -------------------------------------------------------------------------------- 1 | Value NAME (\S+) 2 | Value RD (\S+) 3 | Value ADDRESS_FAMILY (\S+) 4 | 5 | Start 6 | ^${NAME}\s+${RD}\s+${ADDRESS_FAMILY}.* 7 | -------------------------------------------------------------------------------- /templates/disp_ip_vpn.template: -------------------------------------------------------------------------------- 1 | Value NAME (\S+) 2 | Value RD (\d+:\d+) 3 | 4 | Start 5 | ^ ${NAME}\s+${RD}\s+ -> Record 6 | ^ ${NAME}\s+IP.* -> Record 7 | 8 | 9 | -------------------------------------------------------------------------------- /templates/disp_arp_int.template: -------------------------------------------------------------------------------- 1 | Value ADDRESS (\d+(.\d+){3}) 2 | Value INTERFACE (\S+) 3 | 4 | Start 5 | ^${ADDRESS}\s+(\w+-\w+-\w+)\s+\S+\s+\S+\s+${INTERFACE}.* -> Record 6 | 7 | -------------------------------------------------------------------------------- /templates/disp_cur_int.template: -------------------------------------------------------------------------------- 1 | Value INTERFACE (\S+) 2 | Value DESCRIPTION (.*) 3 | 4 | Start 5 | ^interface -> Continue.Record 6 | ^interface ${INTERFACE} 7 | ^ description ${DESCRIPTION} 8 | -------------------------------------------------------------------------------- /templates/disp_ip_int.template: -------------------------------------------------------------------------------- 1 | Value NAME (\S+) 2 | Value List ADDRESS (\d+(.\d+){3}/(\d+)) 3 | 4 | Start 5 | ^${NAME} current state : UP 6 | ^Internet Address is ${ADDRESS} 7 | ^Broadcast address.* -> Record 8 | -------------------------------------------------------------------------------- /templates/disp_vpn_intf.template: -------------------------------------------------------------------------------- 1 | Value NAME (.*) 2 | Value List INTERFACE (\S+) 3 | 4 | Start 5 | ^ VPN-Instance Name and ID : -> Continue.Record 6 | ^ VPN-Instance Name and ID : ${NAME}, 7 | ^ Interface list : ${INTERFACE} 8 | ^ ${INTERFACE} 9 | -------------------------------------------------------------------------------- /templates/disp_esn.template: -------------------------------------------------------------------------------- 1 | Value ESN_MASTER (ESN of master: \S+) 2 | Value ESN_SLAVE (ESN of slave) 3 | Value MAC_ADDRESS (\w+-\w+-\w+) 4 | Value MTU (.*) 5 | 6 | Start 7 | ^${NAME} current state.* 8 | ^Description:${DESCRIPTION} 9 | ^.* Maximum Transmit Unit is ${MTU} 10 | ^.* Hardware address is ${MAC_ADDRESS} -> Record 11 | -------------------------------------------------------------------------------- /templates/disp_elabel.template: -------------------------------------------------------------------------------- 1 | Value List NAME (\[.*\]) 2 | Value PART_ID (.*) 3 | Value SERIAL (.*) 4 | Value DESCRIPTION (.*) 5 | Value MANUFACTURER (.*) 6 | 7 | Start 8 | ^${NAME} 9 | ^BoardType=${PART_ID} 10 | ^BarCode=${SERIAL} 11 | ^Description=${DESCRIPTION} 12 | ^(.*VendorName=|VendorName=)${MANUFACTURER} -> Record 13 | -------------------------------------------------------------------------------- /templates/disp_int.template: -------------------------------------------------------------------------------- 1 | Value NAME (\S+) 2 | Value DESCRIPTION (.*) 3 | Value MAC_ADDRESS (\w+-\w+-\w+) 4 | Value MTU (\d+) 5 | 6 | Start 7 | ^\S+ current state.* -> Continue.Record 8 | ^${NAME} current state.* 9 | ^Description:${DESCRIPTION} 10 | ^.* Maximum Transmit Unit is ${MTU} 11 | ^.* Hardware address is ${MAC_ADDRESS} 12 | -------------------------------------------------------------------------------- /pdb_test.py: -------------------------------------------------------------------------------- 1 | # for debugging with pdb 2 | if __name__ == "__main__": 3 | device_params = { 4 | "netbox": "http://netbox_domain_name_or_ip/", 5 | "token": "netbox_token", 6 | "device_type": "huawei", 7 | "ip": "", 8 | "username": "user", 9 | "password": "password", 10 | "inventory": True, 11 | "port": 22, 12 | } 13 | from vendor_selector import VendorSelector 14 | o = VendorSelector(**device_params) 15 | o.send_ip() 16 | -------------------------------------------------------------------------------- /vendor_selector.py: -------------------------------------------------------------------------------- 1 | from huawei.huawei import HuaweiIpam 2 | 3 | 4 | platforms = { 5 | "huawei": HuaweiIpam, 6 | } 7 | 8 | platforms_str = "\n".join(list(platforms.keys())) 9 | 10 | 11 | def VendorSelector(*args, **kwargs): 12 | """Selecting vendor class based on device_type. The same as netmiko ConnectHandler""" 13 | if kwargs["device_type"] not in platforms: 14 | raise ValueError( 15 | "Unfortunately it's not supported device_type yet. " 16 | f"\nSupported platforms:\n{platforms_str}" 17 | ) 18 | IpamClass = platforms[kwargs["device_type"]] 19 | return IpamClass(*args, **kwargs) 20 | -------------------------------------------------------------------------------- /templates/index: -------------------------------------------------------------------------------- 1 | Template, Hostname, Vendor, Command 2 | disp_elabel.template, .*, huawei, di[[splay]] el[[abe]] 3 | disp_elabel.template, .*, huawei, Y[[ES]] 4 | disp_arp_int.template, .*, huawei, di[[splay]] arp a[[ll]] 5 | disp_vpn_intf.template, .*, huawei, di[[splay]] ip v[[pn-instance]] in[[terface]] 6 | disp_cur_int.template, .*, huawei, di[[splay]] cur in[[terface]] 7 | disp_dev.template, .*, huawei, di[[splay]] dev[[ice]] 8 | disp_int.template, .*, huawei, di[[splay]] int[[erface]] 9 | disp_esn.template, .*, huawei, di[[splay]] es[[n]] 10 | disp_ip_int_br.template, .*, huawei, di[[splay]] ip int[[erface]] br[[ief]] 11 | disp_ip_int.template, .*, huawei, di[[splay]] ip int[[erface]] 12 | disp_ip_vpn.template, .*, huawei, di[[splay]] ip v[[pn-instance]] 13 | 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | ## This repository is no longer maintained 4 | The `netbox_resolver` tool works with `netbox 2.8.5`, `netmiko==3.1.0`, `pynetbox==5.0.5`, `textfsm==1.1.0` and not tested with later versions 5 | 6 | --- 7 | 8 | ## Netbox_resolver 9 | 10 | Netbox_resolver is an autoIPAM/DCIM tool for [NetBox](https://github.com/netbox-community/netbox) 11 | 12 | You can add new data, check/correct previously added data in the NetBox database and automate this process using the scheduler. 13 | 14 | Netbox_resolver will collect as much information as possible from devices and add it to netbox (if there is no device, it will create it): 15 | - Site 16 | - Device Type 17 | - Role (Device Role) 18 | - Manufacturer 19 | - Platform 20 | - Serial 21 | - VRF (binds RD) 22 | - Prefixes and IP Addresses (binds to the VRF, to the interface) 23 | - Primary IPv4 24 | - Interfaces (for each interface, inserts the fields: Device, Name, Type, MTU, MAC Address, Description, IP Address (+sub addresses) 25 | - Inventory (with the option inventory: True) 26 | 27 | Netbox_resolver will fix a mismatch in the netbox database based on information from devices: 28 | - Manufacturer 29 | - Device Type 30 | - Serial 31 | - Interfaces (removes those that are not present on the device, adds the latest ones) 32 | - VRF (checks RD compliance) 33 | - IP Addresses (will check the parameters and binding to the interface, delete or update, if necessary) 34 | - Inventory (with the option inventory: True, will check the Inventory for device) 35 | 36 | ## Platforms 37 | 38 | **Supported platforms:** 39 | - Huawei VRP 40 | 41 | _ _ _ 42 | 43 | For Huawei VRP will be created and used by default for new devices: 44 | - Device roles: Network device 45 | - Manufacturers: Huawei Technologies 46 | - Platforms: Huawei VRP 47 | 48 | For Huawei VRP, TextFSM parses the output of the following commands: 49 | - display elabel 50 | - display arp all 51 | - display ip vpn-instance interface 52 | - display current-configuration interface 53 | - display device 54 | - display interface 55 | - display esn 56 | - display ip interface brief 57 | - display ip interface 58 | - display ip vpn-instance 59 | 60 | For Huawei VRP, parameters such as Site(site), Device Type(model), and Name(sysname) are defined in the `def _create_all()` service method. Change them if necessary. 61 | 62 | _ _ _ 63 | 64 | ## Usage 65 | 66 | Requires `Python 3.7+` 67 | 68 | `pip install pynetbox` 69 | 70 | `pip install netmiko` 71 | 72 | 73 | **Netbox_resolver has only 2 methods:** 74 | - `def send_ip(): for single device` 75 | - `def send_ip_list(): for many devices` 76 | 77 | Netbox_resolver has the same syntax as netmiko. 78 | 79 | **You can run netbox_resolver for a single device:** 80 | 81 | ```python 82 | device_params = { 83 | "netbox": "http://netbox_domain_name_or_ip/", 84 | "token": "netbox_token", 85 | "device_type": "huawei", 86 | "ip": "10.0.0.1", 87 | "username": "user", 88 | "password": "password", 89 | } 90 | 91 | from vendor_selector import VendorSelector 92 | 93 | o = VendorSelector(**device_params) 94 | 95 | o.send_ip() 96 | ``` 97 | 98 | 99 | **You can run netbox_resolver for n-th number of devices at the same time:** 100 | 101 | 102 | ```python 103 | devices_params = { 104 | "netbox": "http://netbox_domain_name_or_ip/", 105 | "token": "netbox_token", 106 | "device_type": "huawei", 107 | "ip_list": ["10.1,2,3-5.1.1,2,3,4-10", "10.1.0.0/24"], 108 | "threads": True, 109 | "max_workers": 2, 110 | "username": "user", 111 | "password": "password", 112 | } 113 | 114 | from vendor_selector import VendorSelector 115 | 116 | o = VendorSelector(**devices_params) 117 | 118 | o.send_ip_list() 119 | ``` 120 | `threads` and `max_workers` are optional arguments 121 | 122 | For `def send_ip_list()`: 123 | - if you use incorrect netmiko connection arguments and errors `NetmikoTimeoutException`, `NetMikoAuthenticationException` occur, the method switches to the next device in the queue and sends the corresponding log to the output stream after the timeout expires. 124 | - if you use `threads` and `max_workers`, you can press Ctrl+C for `KeyboardInterrupt` 125 | 126 | ## Netmiko, Pynetbox, TextFSM 127 | 128 | Netbox_resolver is a wrapper on netmiko, so you can use all the arguments available in your version of netmiko. 129 | 130 | Netbox_resolver uses the following modules: 131 | - [Netmiko](https://github.com/ktbyers/netmiko) 132 | - [Pynetbox](https://github.com/digitalocean/pynetbox) - Python API client library for NetBox 133 | - [TextFSM](https://github.com/google/textfsm) - text output parsing 134 | - Concurrent.futures, threading - works with threads 135 | - Logging, time - logs working with threads 136 | 137 | ## Optional arguments 138 | 139 | The following arguments are optional and described in `nb_ipam_resolver.py`: 140 | - threads 141 | - max_workers 142 | - arp 143 | - arp_interfaces 144 | - inventory 145 | 146 | ## Scheduler 147 | 148 | You can make the process regular by adding netbox_resolver to the scheduler, and as a result, automate the routine process of keeping IPAM/DCIM up to date. 149 | 150 | ### Protection from sysname changing 151 | 152 | If you work with the scheduler and decide to change device sysname after it was added to the NetBox database, this can create a lil problem... Script will try to create a new device with new sysname. So it is better to add protection - in this case, script will not do anything until you match the device sysname and site, sysname in the NetBox database. In short, if you change sysname on device, you need to change device name and site in NetBox, otherwise this function returns nothing. 153 | 154 | Please, check `huawei.py` and delete or comment block of code between `Protection from sysname changing` and `End of protection from sysname changing` if you don't need it. 155 | 156 | ## pdb 157 | 158 | For script debugging you can add your parameters (for single device) in `pdb_test.py` and start pdb like `python -m pdb pdb_test.py` in your virtual environment. 159 | -------------------------------------------------------------------------------- /nb_ipam_resolver.py: -------------------------------------------------------------------------------- 1 | import pynetbox 2 | import re 3 | import ipaddress 4 | from netmiko import ConnectHandler, NetMikoAuthenticationException 5 | 6 | 7 | class NetboxIpamResolver: 8 | def __init__( 9 | self, 10 | ip_list=None, 11 | netbox=None, 12 | token=None, 13 | private_key=None, 14 | threads=None, 15 | max_workers=None, 16 | arp=None, 17 | arp_interfaces=None, 18 | inventory=None, 19 | *args, 20 | **device_params 21 | ): 22 | """ 23 | For initialization you can use the same arguments as in netmiko, 24 | because it's wrapper. 25 | Netmiko arguments - https://github.com/ktbyers/netmiko/blob/develop/netmiko/base_connection.py 26 | New arguments for this method only: 27 | 28 | :ip_list: ip networks and addresses in list format. For many hosts in 29 | single params dictionary. 30 | For example: ['10.1.1.1', '10.1,2,3-5.10.1,2,3,4,5,10-125', '10.1.10.1,2,3,4,5,10-125', '10.1.0.0/24'] 31 | :type ip_list: str in list 32 | 33 | Pynetbox arguments (https://github.com/digitalocean/pynetbox) 34 | :netbox: netbox domain name or ip address. 35 | For example: 'http://192.168.0.1' 36 | :type netbox: str 37 | 38 | :token: netbox user token from NetBox Administration > Users > Tokens 39 | :type token: str 40 | 41 | :private_key: netbox private_key or private_key_file. 42 | :type private_key: str 43 | 44 | :threads: True or False(None by default). To enable concurrent.futures and threading. It works with 45 | ip_list only 46 | :type threads: bool 47 | 48 | :max_workers: number of threads for submit in concurrent.futures 49 | :type max_workers: int 50 | 51 | :arp: True or False(None by default). To get ip addresses from arp 52 | cache. To use with arp_interfaces 53 | :type arp: bool 54 | 55 | :arp_interfaces: for example, ['Vlanif100', 'Vlanif200']. It looks like parser for 'display arp 56 | interface Vlanif100 and then for Vlanif200. Use this argument to add ip 57 | addresses from arp to netbox db. Description format - "Added from arp 58 | cache of {sysname}, interface {interface}" 59 | :type arp_interface: str in list 60 | 61 | :inventory: True or False(None by default). To get inventory from 62 | "display elabel" command 63 | :type inventory: bool 64 | """ 65 | self.ip_list = ip_list 66 | self.netbox = netbox 67 | self.token = token 68 | self.private_key = private_key 69 | self.device_params = device_params 70 | self.threads = threads 71 | self.max_workers = max_workers 72 | self.arp = arp 73 | self.arp_interfaces = arp_interfaces 74 | self.inventory = inventory 75 | if ip_list and "ip" in device_params.keys(): 76 | raise ValueError( 77 | "Erorr in arguments. Don't use 'ip' and 'ip_list' together" 78 | ) 79 | if ip_list: 80 | self.ip_list_object = Iterator(self.ip_list, self.device_params) 81 | if arp or arp_interfaces: 82 | self._check_arp() 83 | 84 | def _check_arp(self): 85 | error_message = "Error in arguments, add {}, please" 86 | var_list = ["arp like a True or False", "arp_interfaces like a list please"] 87 | var_list1 = [self.arp, self.arp_interfaces] 88 | for position, k in enumerate(var_list1): 89 | if not k: 90 | raise ValueError(error_message.format(var_list[position])) 91 | if self.arp and self.arp_interfaces: 92 | if not isinstance(self.arp_interfaces, list): 93 | raise ValueError("Erorr in arguments, arp_interfaces is list") 94 | 95 | 96 | # create :ip_list: like iterator 97 | class Iterator: 98 | def __init__(self, ip_list, device_params): 99 | self.ip_list = self._check_ip_list_format(ip_list) 100 | self._index = 0 101 | self.device_params = device_params 102 | 103 | # this is not best solution for checking :ip_list: format 104 | def _check_ip_list_format(self, ip_list): 105 | str_ip_list = [] 106 | params = { 107 | "single_ip": ipaddress.ip_address, 108 | "network": ipaddress.ip_network, 109 | "hard_ip": self._convert_hard_ip, 110 | } 111 | # wow! What is a beautiful regex?!;d 112 | regex = ( 113 | r"(?P\d+(\.\d+){3}/\d+)" 114 | r"|(?P\d{1,3}((\.(\d{1,3},){1,}(\d{1,3}-\d{1,3},){1,})(\d{1,3}-\d{1,3})" 115 | "|(\d{1,3}-\d{1,3},){1,}(\d{1,3}-\d{1,3})" 116 | "|(\.(\d{1,3},){1,}(\d{1,3}-\d{1,3}))" 117 | "|\.(\d{1,3},){1,}\d{1,3}" 118 | "|\.(\d{1,3}-\d{1,3})" 119 | "|\.(\d){1,3}){3})" 120 | r"|(?P\d{1,3}(\.\d{1,3}){3})" 121 | ) 122 | if not all( 123 | [isinstance(ip_list, list)] + [isinstance(value, str) for value in ip_list] 124 | ): 125 | raise ValueError("Error in 'ip_list' argument. Incorrect type.") 126 | for value in ip_list: 127 | match = re.search(regex, value) 128 | if match: 129 | for param in params: 130 | if match.lastgroup == param: 131 | try: 132 | ip_object = params[param](match.group(match.lastgroup)) 133 | if param == "network": 134 | str_ip_list.extend( 135 | [str(ip) for ip in ip_object.hosts()] 136 | ) 137 | elif param == "single_ip": 138 | str_ip_list.append(str(ip_object)) 139 | elif param == "hard_ip": 140 | str_ip_list.extend(ip_object) 141 | except ValueError: 142 | print( 143 | "Error in 'ip_list' argument. Incorrect IPv4 address or IPv4 network" 144 | ) 145 | return str_ip_list 146 | else: 147 | raise ValueError( 148 | "Error in 'ip_list' argument. Incorrect IPv4 address or IPv4 network" 149 | ) 150 | 151 | def _check_hard_ip(self, hard_ip): 152 | correct_ip_list = [hard_ip.split(".")[0]] 153 | for value in hard_ip.split(".")[1:]: 154 | string = "" 155 | for octet in value.split(","): 156 | if "-" in octet: 157 | string += ",".join( 158 | [ 159 | str(i) 160 | for i in range( 161 | int(octet.split("-")[0]), int(octet.split("-")[1]) + 1 162 | ) 163 | ] 164 | ) 165 | else: 166 | string += octet + "," 167 | correct_ip_list.append(string.rstrip(",")) 168 | return correct_ip_list 169 | 170 | def _convert_hard_ip(self, hard_ip): 171 | correct_ip_list = self._check_hard_ip(hard_ip) 172 | full_ip_list = [] 173 | for octet2 in correct_ip_list[1].split(","): 174 | for octet3 in correct_ip_list[2].split(","): 175 | for octet4 in correct_ip_list[3].split(","): 176 | full_ip_list.append( 177 | ".".join([correct_ip_list[0]] + [octet2, octet3, octet4]) 178 | ) 179 | return full_ip_list 180 | 181 | def __iter__(self): 182 | return self 183 | 184 | def __next__(self): 185 | if self._index < len(self.ip_list): 186 | self.device_params["ip"] = self.ip_list[self._index] 187 | self._index += 1 188 | return self.device_params 189 | else: 190 | raise StopIteration 191 | -------------------------------------------------------------------------------- /huawei/huawei.py: -------------------------------------------------------------------------------- 1 | import pynetbox 2 | import os 3 | import re 4 | import ipaddress 5 | import sys 6 | from netmiko import ( 7 | ConnectHandler, 8 | NetMikoAuthenticationException, 9 | NetmikoTimeoutException, 10 | ) 11 | from netmiko.huawei.huawei import HuaweiBase 12 | from nb_ipam_resolver import NetboxIpamResolver 13 | from pynetbox.core.query import RequestError 14 | from concurrent.futures import ( 15 | ThreadPoolExecutor, 16 | as_completed, 17 | Future, 18 | TimeoutError, 19 | ) 20 | from threading import RLock 21 | from datetime import datetime 22 | import time 23 | import logging 24 | 25 | logging.getLogger("paramiko").setLevel(logging.WARNING) 26 | 27 | logging.basicConfig( 28 | format="%(threadName)s %(name)s %(levelname)s: %(message)s", level=logging.INFO 29 | ) 30 | 31 | 32 | class HuaweiIpam(NetboxIpamResolver): 33 | def __init__(self, *args, **device_params): 34 | super().__init__(*args, **device_params) 35 | # instantiate the pynetbox API 36 | self.nb = pynetbox.api( 37 | self.netbox, private_key_file=self.private_key, token=self.token 38 | ) 39 | # common parameters which will create by default(for new devices) 40 | # device_roles - Network device 41 | # manufacturers - Huawei Technologies 42 | # platforms - Huawei VRP 43 | self._global_params = { 44 | self.nb.dcim.device_roles: ( 45 | {"name": "Network device"}, 46 | {"name": "Network device", "slug": "network_device", "color": "4caf50"}, 47 | ), 48 | self.nb.dcim.manufacturers: ( 49 | {"name": "Huawei Technologies"}, 50 | {"name": "Huawei Technologies", "slug": "huawei_technologies"}, 51 | ), 52 | self.nb.dcim.platforms: ( 53 | {"name": "Huawei VRP"}, 54 | {"name": "Huawei VRP", "slug": "huawei-vrp"}, 55 | ), 56 | } 57 | # create these default parameters 58 | self._get_create(self._global_params) 59 | 60 | def send_ip(self): 61 | """ 62 | Method which works with single device 63 | For correct work use :ip: argument instead of :ip_list: 64 | """ 65 | if self.ip_list: 66 | raise ValueError( 67 | "Error in arguments. It's only for single device. Please don't use 'ip_list', use 'ip'" 68 | ) 69 | with ConnectHandler(**self.device_params) as ssh_huawei: 70 | self._create_all(ssh_huawei, self.device_params["ip"]) 71 | 72 | def send_ip_list(self): 73 | """ 74 | Method which works with many devices 75 | For correct work use :ip_list: argument instead of :ip: 76 | Actual arguments: 77 | :threads: True or False(None by default). To enable concurrent.futures and threading. It works with 78 | ip_list only 79 | :type threads: bool 80 | 81 | :max_workers: number of threads for submit in concurrent.futures 82 | :type max_workers: int 83 | """ 84 | if "ip" in self.device_params: 85 | raise ValueError( 86 | "Erorr in arguments. It's only for many devices. Please don't use 'ip', use 'ip_list' for this method" 87 | ) 88 | if self.threads: 89 | result = self._task_queue( 90 | self._try_create_all, self.ip_list_object, concurrency=self.max_workers 91 | ) 92 | try: 93 | while not result.done(): 94 | try: 95 | result.result(0.2) 96 | print( 97 | "\rdone: {done}, waited: " 98 | "{delayed}".format(**result.stats), 99 | flush=True, 100 | ) 101 | # check extensions 102 | except TimeoutError: 103 | pass 104 | # sys.stdout.flush() 105 | # use future.cancel() method as attemption to shutdown executor 106 | # by Ctrl+C 107 | except KeyboardInterrupt: 108 | result.cancel() 109 | raise 110 | 111 | else: 112 | for session_params in self.ip_list_object: 113 | self._try_create_all(session_params, session_params["ip"]) 114 | # try: 115 | # self._try_create_all(session_params, session_params['ip']) 116 | # except (NetmikoTimeoutException, NetMikoAuthenticationException) as error: 117 | # logging.warning(error) 118 | # continue 119 | 120 | def _try_create_all(self, session_params, device_ip): 121 | 122 | msg_info_connection = " --- > {} Trying to connect to {}" 123 | 124 | msg_info_drop = " --- > {} Unable to connect to {}" 125 | 126 | logging.info(msg_info_connection.format(datetime.now().time(), device_ip)) 127 | try: 128 | with ConnectHandler(**session_params) as ssh_huawei: 129 | self._create_all(ssh_huawei, device_ip) 130 | except (NetmikoTimeoutException, NetMikoAuthenticationException) as error: 131 | logging.warning(error) 132 | logging.info(msg_info_drop.format(datetime.now().time(), device_ip)) 133 | return 134 | 135 | def _create_all(self, ssh_session_object, device_ip): 136 | self._check_net_textfsm() 137 | # HRP_M & HRP_S for Huawei firewalls in HRP mode 138 | sysname = re.sub(r"HRP_M|HRP_S|<|>", "", ssh_session_object.find_prompt()) 139 | # site for device binding 140 | # site is characters before '_' or '.' in sysname without '_' or '.' 141 | site = re.sub(r"_.*|\..*", "", sysname) 142 | _model = self._check_model(ssh_session_object) 143 | # device type -> model name 144 | model = re.sub(r" ", "_", _model[0]["model"]) 145 | # interfaces list 146 | interfaces = self._check_interfaces(sysname, ssh_session_object) 147 | # ip addresses list 148 | ip_addresses = self._check_ip_addresses(ssh_session_object) 149 | 150 | # Protection from sysname changing 151 | 152 | # If you work with the scheduler and decide to change device sysname 153 | # after it was added to the NetBox database, this can create a lil 154 | # problem... Script will try to create a new device with new sysname. So 155 | # it is better to add protection - in this case, script will not do 156 | # anything until you match the device sysname and site, sysname in the NetBox database 157 | 158 | # In short, if you change sysname on device, you need to change device name and 159 | # site in netbox, otherwisethis function returns nothing 160 | 161 | # Delete or comments this block of code if you don't need it 162 | for item in ip_addresses: 163 | for address in item.values(): 164 | if device_ip in str(address): 165 | mgmt_ip = address[0] 166 | ip = self.nb.ipam.ip_addresses.get(address=mgmt_ip) 167 | if ip and ip.interface: 168 | if str(ip.interface.device) != sysname: 169 | return 170 | # End of protection from sysname changing 171 | 172 | # parameters for checking and creating 173 | points = { 174 | self.nb.dcim.device_types: ( 175 | {"model": model}, 176 | { 177 | "manufacturer": {"name": "Huawei Technologies"}, 178 | "model": model, 179 | "slug": model, 180 | }, 181 | ), 182 | self.nb.dcim.sites: ( 183 | {"slug": f"dc-{site}"}, 184 | {"name": f"dc-{site}", "slug": f"dc-{site}"}, 185 | ), 186 | } 187 | device_vars = { 188 | self.nb.dcim.devices: ( 189 | {"name": sysname}, 190 | { 191 | "name": sysname, 192 | "device_role": {"name": "Network device"}, 193 | "device_type": {"model": model}, 194 | "manufacturer": {"name": "Huawei Technologies"}, 195 | "site": {"slug": f"dc-{site}"}, 196 | "platform": {"name": "Huawei VRP"}, 197 | }, 198 | ), 199 | } 200 | interfaces_vars = { 201 | ("G",): 1000, 202 | ("Virtual", "Vlanif", "LoopBack", "NULL", "Tunnel"): 0, 203 | ("X",): 1150, 204 | ("Ethernet", "M"): 800, #"M" - Meth 205 | ("Eth-Trunk",): 200, 206 | ("A",): 32767, #"A" - Aux 207 | ("C",): 2830, #"C" - Cellular 208 | } 209 | # check device if it is 210 | self._update_device( 211 | device_vars, 212 | points, 213 | model, 214 | f"dc-{site}", 215 | sysname, 216 | interfaces, 217 | interfaces_vars, 218 | ) 219 | # create new device with default parameters 220 | self._create_device(device_vars, points, sysname, interfaces, interfaces_vars) 221 | # create new VRF's and update existing VRF's 222 | self._create_vrfs(ssh_session_object) 223 | # create new prefixes and ip addresses and delete the incorrect ones 224 | self._create_prefixes_ip_addresses( 225 | ip_addresses, sysname, device_ip, ssh_session_object 226 | ) 227 | # create serial (if software supports 'display esn' command) 228 | self._check_serial(ssh_session_object, sysname) 229 | 230 | # it works with :arp: and :arp_interfaces: 231 | if self.arp and self.arp_interfaces: 232 | self._create_addresses_from_arp_cache(ssh_session_object, sysname) 233 | # it works with :inventory: 234 | if self.inventory: 235 | self._create_inventory(ssh_session_object, sysname) 236 | 237 | def _update_serial(self, serial, sysname): 238 | dev = self.nb.dcim.devices.filter(name=sysname)[0] 239 | if dev.serial != serial: 240 | dev.update({"serial": serial}) 241 | 242 | def _check_serial(self, ssh_session_object, sysname): 243 | serial = ssh_session_object.send_command("display esn") 244 | match = re.search(".*ESN of master:(.*)", serial) 245 | if match: 246 | self._update_serial(match.group(1), sysname) 247 | 248 | def _create_inventory(self, ssh_session_object, sysname): 249 | """ 250 | Checking and creating/updating inventory from 'display elabel' command 251 | """ 252 | inventory = self._check_elabel(ssh_session_object) 253 | for part in inventory: 254 | if not part["serial"]: 255 | continue 256 | inv_object = self.nb.dcim.inventory_items.filter( 257 | serial=part["serial"].strip() 258 | ) 259 | manufacturer_strip = part["manufacturer"].strip() 260 | part["manufacturer"] = manufacturer_strip 261 | part["device"] = sysname 262 | if not self.nb.dcim.manufacturers.filter(name=manufacturer_strip): 263 | vendor_slug = re.sub(r" |-|\.", "_", manufacturer_strip) 264 | self.nb.dcim.manufacturers.create( 265 | name=manufacturer_strip, slug=vendor_slug 266 | ) 267 | upd_part = self._check_part(part, sysname) 268 | if not inv_object: 269 | self.nb.dcim.inventory_items.create(upd_part) 270 | else: 271 | if len(inv_object) > 1: 272 | for o in inv_object[1:]: 273 | o.delete() 274 | inv_obj_dict = { 275 | "device": inv_object[0].device, 276 | "manufacturer": inv_object[0].manufacturer, 277 | "part_id": inv_object[0].part_id, 278 | "description": inv_object[0].description, 279 | } 280 | for key, value in inv_obj_dict.items(): 281 | if part[key] != str(value): 282 | inv_object[0].update(upd_part) 283 | break 284 | 285 | def _check_part(self, part, sysname): 286 | upd_part = part.copy() 287 | # because length of description field is 100 characters 288 | if len(part["description"]) > 100: 289 | upd_description = re.sub( 290 | r"Assembling Components,| |\(|\)", "", part["description"] 291 | )[0:99] 292 | upd_part.update({"description": upd_description}) 293 | upd_name = part["name"][0].strip("[]") 294 | upd_part.update( 295 | { 296 | "name": upd_name, 297 | "device": {"name": sysname}, 298 | "manufacturer": {"name": part["manufacturer"]}, 299 | } 300 | ) 301 | if not upd_part["manufacturer"]["name"]: 302 | del upd_part["manufacturer"] 303 | return upd_part 304 | 305 | def _create_addresses_from_arp_cache(self, ssh_session_object, sysname): 306 | """ 307 | Checking and creating ip addresses from arp cache. 308 | Sysname and arp interface will be in ip address description 309 | """ 310 | regex = r"(?P
\d+(.\d+){3})\s+(\w+-\w+-\w+).*" 311 | description = "Added from arp cache of {}, interface {}" 312 | for intf in self.arp_interfaces: 313 | arp_out = ssh_session_object.send_command(f"disp arp interface {intf}") 314 | if "Error" in arp_out: 315 | raise ValueError( 316 | "Not correct interface for display arp interface command" 317 | ) 318 | arp_list = [match.group("address") for match in re.finditer(regex, arp_out)] 319 | mask = ( 320 | "/" 321 | + str( 322 | self.nb.ipam.ip_addresses.filter(device=sysname, interface=intf)[0] 323 | ).split("/")[1] 324 | ) 325 | for addr in arp_list: 326 | address_with_mask = addr + mask 327 | if not self.nb.ipam.ip_addresses.filter(address=address_with_mask): 328 | self.nb.ipam.ip_addresses.create( 329 | { 330 | "address": address_with_mask, 331 | "description": description.format(sysname, intf), 332 | } 333 | ) 334 | 335 | def _create_prefixes_ip_addresses( 336 | self, ip_addresses, sysname, current_mgmt_ip, ssh_session_object 337 | ): 338 | for addresses in ip_addresses: 339 | if addresses["name"] and addresses["address"]: 340 | self._check_cur_interface_addresses( 341 | sysname, addresses["name"], addresses["address"] 342 | ) 343 | cur_addr = addresses["address"][0] 344 | for addr in addresses["address"]: 345 | network = self._check_network(addr) 346 | self._create_prefix(network) 347 | network_object = self.nb.ipam.prefixes.get(prefix=network) 348 | 349 | if not self.nb.ipam.ip_addresses.filter(address=addr): 350 | self.nb.ipam.ip_addresses.create( 351 | { 352 | "address": addr, 353 | "interface": { 354 | "device": {"name": sysname}, 355 | "name": addresses["name"], 356 | }, 357 | } 358 | ) 359 | address_object = self.nb.ipam.ip_addresses.filter(address=addr)[0] 360 | 361 | description = self._check_cur_int( 362 | addresses["name"], ssh_session_object 363 | ) 364 | 365 | if not address_object.interface: 366 | address_object.update( 367 | { 368 | "address": addr, 369 | "interface": { 370 | "device": {"name": sysname}, 371 | "name": addresses["name"], 372 | }, 373 | "description": description, 374 | } 375 | ) 376 | 377 | else: 378 | address_object_dict = { 379 | address_object.interface.device.name: sysname, 380 | address_object.interface.name: addresses["name"], 381 | address_object.description: description, 382 | } 383 | 384 | for key, value in address_object_dict.items(): 385 | if not key == value: 386 | address_object.update( 387 | { 388 | "address": addr, 389 | "interface": { 390 | "device": {"name": sysname}, 391 | "name": addresses["name"], 392 | }, 393 | "description": description, 394 | } 395 | ) 396 | break 397 | 398 | vrf = self._check_vrf_interfaces( 399 | addresses["name"], ssh_session_object 400 | ) 401 | # binding ip interface to vrf 402 | if vrf and not address_object.vrf == vrf: 403 | self._vrf_binding(vrf, network_object, address_object) 404 | # mgmt ip address 405 | # it can use for napalm_ce driver, for example 406 | self._update_mgmt_ipv4(sysname, current_mgmt_ip, cur_addr) 407 | 408 | def _check_cur_interface_addresses(self, sysname, interface_name, addresses_list): 409 | """ 410 | Delete ip address on ip interfaces if it's incorrect(not actual) 411 | """ 412 | cur_addr_list = self.nb.ipam.ip_addresses.filter( 413 | device=sysname, interface=interface_name 414 | ) 415 | if cur_addr_list: 416 | for cur_intf_address in cur_addr_list: 417 | if not str(cur_intf_address) in addresses_list: 418 | cur_addr_obj = self.nb.ipam.ip_addresses.filter( 419 | device=sysname, 420 | interface=interface_name, 421 | address=str(cur_intf_address), 422 | ) 423 | for obj in cur_addr_obj: 424 | obj.delete() 425 | 426 | def _create_prefix(self, network): 427 | if not "/32" in network and not self.nb.ipam.prefixes.filter(prefix=network): 428 | self.nb.ipam.prefixes.create({"prefix": network}) 429 | 430 | def _vrf_binding(self, vrf, network_object, address_object): 431 | network_object.update({"vrf": {"name": vrf}}) 432 | address_object.update({"vrf": {"name": vrf}}) 433 | 434 | def _update_mgmt_ipv4(self, sysname, current_mgmt_ip, current_list_ip): 435 | """ 436 | Binding ip address of the interface as primary ipv4. 437 | Method checks your :ip: or :ip_list: also 438 | 439 | """ 440 | mgmt_ip = self.nb.dcim.devices.get(name=sysname) 441 | if not mgmt_ip.primary_ip4 or current_mgmt_ip not in mgmt_ip.primary_ip4: 442 | if current_mgmt_ip in current_list_ip: 443 | mgmt_ip.update({"primary_ip4": {"address": current_list_ip}}) 444 | 445 | def _get_create(self, points): 446 | """ 447 | Universal method for requesting and creation something 448 | """ 449 | for key, value in points.items(): 450 | if not key.get(**value[0]): 451 | key.create(value[1]) 452 | 453 | def _create_interfaces(self, interfaces, interfaces_vars): 454 | for intf in interfaces: 455 | # intf.update({'device': {'name': sysname}}) 456 | for key, value in interfaces_vars.items(): 457 | for intf_type in key: 458 | if intf["name"].startswith(intf_type): 459 | intf.update({"type": value}) 460 | self.nb.dcim.interfaces.create(intf) 461 | 462 | def _update_interfaces(self, sysname, interfaces, interfaces_vars): 463 | """ 464 | Delete/update/create interfaces 465 | """ 466 | interfaces_list = [] 467 | netbox_interfaces = self.nb.dcim.interfaces.filter(device=sysname) 468 | if netbox_interfaces: 469 | for intf in interfaces: 470 | interfaces_list.append(intf["name"]) 471 | for intf_object in netbox_interfaces: 472 | if not str(intf_object) in interfaces_list: 473 | intf_object.delete() 474 | else: 475 | intf_object_dict = dict(intf_object) 476 | index = interfaces_list.index(str(intf_object)) 477 | for key, value in interfaces[index].items(): 478 | if value != intf_object_dict[key]: 479 | intf_object.update({key: value}) 480 | self._repeat_interfaces_check(sysname, interfaces, interfaces_vars) 481 | else: 482 | self._create_interfaces(interfaces, interfaces_vars) 483 | 484 | def _repeat_interfaces_check(self, sysname, interfaces, interfaces_vars): 485 | netbox_interfaces = self.nb.dcim.interfaces.filter(device=sysname) 486 | if netbox_interfaces: 487 | str_netbox_interfaces = [str(i) for i in netbox_interfaces] 488 | # str_netbox_interfaces = str(netbox_interfaces) 489 | for intf in interfaces: 490 | if not intf["name"] in str_netbox_interfaces: 491 | self._create_interfaces([intf], interfaces_vars) 492 | else: 493 | self._create_interfaces(interfaces, interfaces_vars) 494 | 495 | def _create_device(self, device_vars, points, sysname, interfaces, interfaces_vars): 496 | key, value = list(device_vars.items())[0] 497 | if not key.get(**value[0]): 498 | self._get_create(points) 499 | key.create(value[1]) 500 | self._create_interfaces(interfaces, interfaces_vars) 501 | 502 | def _update_device( 503 | self, device_vars, points, model, site, sysname, interfaces, interfaces_vars 504 | ): 505 | """ 506 | Update device parameters if they don't match default 507 | """ 508 | key, value = list(device_vars.items())[0] 509 | if key.get(**value[0]): 510 | self._get_create(points) 511 | device_object = self.nb.dcim.devices.get(**value[0]) 512 | update_params = { 513 | device_object.device_type.manufacturer.name: ( 514 | "Huawei Technologies", 515 | {"manufacturer": {"name": "Huawei Technologies"}}, 516 | ), 517 | device_object.device_type.model: ( 518 | model, 519 | {"device_type": {"model": model}}, 520 | ), 521 | # device_object.site.slug: (site, {'site': {'slug': site}}), 522 | } 523 | for key, value in update_params.items(): 524 | if not key == value[0]: 525 | device_object.update(value[1]) 526 | if ( 527 | not device_object.platform 528 | or not device_object.platform.name == "Huawei VRP" 529 | ): 530 | device_object.update({"platform": {"name": "Huawei VRP"}}) 531 | 532 | self._update_interfaces(sysname, interfaces, interfaces_vars) 533 | 534 | def _check_model(self, ssh_session_object): 535 | return ssh_session_object.send_command("disp dev", use_textfsm=True) 536 | 537 | def _check_interfaces(self, sysname, ssh_session_object): 538 | params = ["mtu", "mac_address"] 539 | interfaces = ssh_session_object.send_command("disp interface", use_textfsm=True) 540 | for intf in interfaces: 541 | intf.update({"device": {"name": sysname}}) 542 | for param in params: 543 | if not intf[param]: 544 | del intf[param] 545 | return interfaces 546 | 547 | def _check_ip_addresses(self, ssh_session_object): 548 | return ssh_session_object.send_command("disp ip int", use_textfsm=True) 549 | 550 | def _check_elabel(self, ssh_session_object): 551 | """ 552 | Output parsing. 553 | It works in dialogue mode also. 554 | As an option, "Warning: It may take a long time to excute this command. Continue? [Y/N]:" 555 | Netmiko example for dialogue - https://github.com/ktbyers/netmiko/blob/develop/examples/use_cases/case5_prompting/send_command_prompting.py 556 | """ 557 | output = ssh_session_object.send_command_timing( 558 | "display elabel", delay_factor=3, use_textfsm=True 559 | ) 560 | if "Continue" in output: 561 | output = ssh_session_object.send_command_timing( 562 | "Y", delay_factor=3, use_textfsm=True 563 | ) 564 | return output 565 | 566 | def _check_cur_int(self, interface, ssh_session_object): 567 | parse_cur_int = ssh_session_object.send_command( 568 | "disp cur int", use_textfsm=True 569 | ) 570 | for value in parse_cur_int: 571 | if interface == value["interface"]: 572 | return value["description"] 573 | # break 574 | 575 | def _create_vrfs(self, ssh_session_object): 576 | vrf_dict = {} 577 | vrfs = self.nb.ipam.vrfs.all() 578 | for v in vrfs: 579 | vrf_dict[v.name] = v.rd 580 | parse_vrf = ssh_session_object.send_command("di ip v", use_textfsm=True) 581 | if isinstance(parse_vrf, list): 582 | for item in parse_vrf: 583 | if item["rd"]: 584 | if ( 585 | item["name"] in vrf_dict 586 | and not vrf_dict[item["name"]] == item["rd"] 587 | ): 588 | try: 589 | self._update_vrf(item["name"], item["rd"]) 590 | except RequestError: 591 | continue 592 | elif not (item["name"], item["rd"]) in vrf_dict.items(): 593 | try: 594 | self._create_vrf(item) 595 | except RequestError: 596 | continue 597 | else: 598 | if item["name"] in vrf_dict: 599 | continue 600 | else: 601 | try: 602 | self._create_vrf({"name": item["name"]}) 603 | except RequestError: 604 | continue 605 | # return True 606 | 607 | def _create_vrf(self, vrf_dict): 608 | self.nb.ipam.vrfs.create(vrf_dict) 609 | 610 | def _update_vrf(self, vrf_name, rd): 611 | vrf = self.nb.ipam.vrfs.get(name=vrf_name) 612 | vrf.update({"rd": rd}) 613 | 614 | def _check_vrf_interfaces(self, interface, ssh_session_object): 615 | vrf_interfaces = ssh_session_object.send_command("di ip v in", use_textfsm=True) 616 | if vrf_interfaces and isinstance(vrf_interfaces, list): 617 | for intf in vrf_interfaces: 618 | if interface in intf["interface"]: 619 | vrf = intf["name"] 620 | return vrf 621 | # break 622 | 623 | def _check_network(self, ip_address): 624 | ip_interface = ipaddress.ip_interface(ip_address) 625 | return str(ip_interface.network) 626 | 627 | def _check_net_textfsm(self): 628 | """ 629 | Adding a system variable for netmiko textfsm preparsing. 630 | As an example - https://pynet.twb-tech.com/blog/automation/netmiko-textfsm.html 631 | """ 632 | template_path = os.path.join(os.getcwd(), "templates") 633 | if "NET_TEXTFSM" not in os.environ: 634 | os.environ["NET_TEXTFSM"] = template_path 635 | 636 | def _task_queue(self, task, iterator, concurrency=3, on_fail=lambda _: None): 637 | """ 638 | Method controls numbers of working threads and logs it. 639 | Current number of working threads = concurrency(:max_workers:) always. 640 | """ 641 | 642 | def _submit(): 643 | try: 644 | obj = next(iterator) 645 | except StopIteration: 646 | return 647 | if result.cancelled(): 648 | return 649 | 650 | stats["delayed"] += 1 651 | future = executor.submit(task, obj, obj["ip"]) 652 | future.ip = obj["ip"] 653 | future.obj = obj 654 | # check future status and call '_task_done' method with future 655 | future.add_done_callback(_task_done) 656 | 657 | def _task_done(future): 658 | # log completed future 659 | if future.done(): 660 | logging.info( 661 | msg_info_thread_done.format(datetime.now().time(), future.ip) 662 | ) 663 | 664 | with io_lock: 665 | # create new future because previous one is completed 666 | _submit() 667 | stats["delayed"] -= 1 668 | stats["done"] += 1 669 | 670 | if future.exception(): 671 | on_fail(future.exception(), future.obj) 672 | # when all futures are completed, set status for result(future) 673 | # and shutdown executor 674 | if stats["delayed"] == 0: 675 | result.set_result(stats) 676 | 677 | def _cleanup(_): 678 | with io_lock: 679 | executor.shutdown(wait=False) 680 | 681 | msg_info_thread_done = "< --- {} Thread done for {}" 682 | 683 | io_lock = RLock() 684 | executor = ThreadPoolExecutor(max_workers=concurrency) 685 | result = Future() 686 | result.stats = stats = {"done": 0, "delayed": 0} 687 | result.add_done_callback(_cleanup) 688 | # with ThreadPoolExecutor(max_workers = concurrency) as executor: 689 | # future_list = [executor.submit(task, device, device['ip']) for 690 | # device in iterator] 691 | 692 | with io_lock: 693 | for _ in range(concurrency): 694 | _submit() 695 | return result 696 | --------------------------------------------------------------------------------