├── .gitignore ├── README.md ├── LICENSE ├── custom_module.py ├── mysql_import.py ├── iosxr_sshkeys.py ├── irr_sync.py └── netbox_sync.py /.gitignore: -------------------------------------------------------------------------------- 1 | /__pycache__/ -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | This is a support repository for [Writing a custom Ansible module][] 2 | blog post. 3 | 4 | [Writing a custom Ansible module]: https://vincent.bernat.ch/en/blog/2020-custom-ansible-module 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This is free and unencumbered software released into the public domain. 2 | 3 | Anyone is free to copy, modify, publish, use, compile, sell, or 4 | distribute this software, either in source code form or as a compiled 5 | binary, for any purpose, commercial or non-commercial, and by any 6 | means. 7 | 8 | In jurisdictions that recognize copyright laws, the author or authors 9 | of this software dedicate any and all copyright interest in the 10 | software to the public domain. We make this dedication for the benefit 11 | of the public at large and to the detriment of our heirs and 12 | successors. We intend this dedication to be an overt act of 13 | relinquishment in perpetuity of all present and future rights to this 14 | software under copyright law. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | For more information, please refer to 25 | -------------------------------------------------------------------------------- /custom_module.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | DOCUMENTATION = """ 4 | --- 5 | module: custom_module.py 6 | short_description: Pass provided data to remote service 7 | description: 8 | - Mention anything useful for your workmate. 9 | - Also mention anything you want to remember in 6 months. 10 | options: 11 | user: 12 | description: 13 | - user to identify to remote service 14 | password: 15 | description: 16 | - password for authentication to remote service 17 | data: 18 | description: 19 | - data to send to remote service 20 | """ 21 | 22 | import yaml 23 | from ansible.module_utils.basic import AnsibleModule 24 | 25 | 26 | def main(): 27 | module_args = dict( 28 | user=dict(type='str', required=True), 29 | password=dict(type='str', required=True, no_log=True), 30 | data=dict(type='str', required=True), 31 | ) 32 | 33 | module = AnsibleModule( 34 | argument_spec=module_args, 35 | supports_check_mode=True 36 | ) 37 | 38 | result = dict( 39 | changed=False 40 | ) 41 | 42 | got = {} 43 | wanted = {} 44 | 45 | # Populate both `got` and `wanted`. 46 | # [...] 47 | 48 | if got != wanted: 49 | result['changed'] = True 50 | result['diff'] = dict( 51 | before=yaml.safe_dump(got), 52 | after=yaml.safe_dump(wanted) 53 | ) 54 | 55 | if module.check_mode or not result['changed']: 56 | module.exit_json(**result) 57 | 58 | # Apply changes. 59 | # [...] 60 | 61 | module.exit_json(**result) 62 | 63 | 64 | if __name__ == '__main__': 65 | main() 66 | -------------------------------------------------------------------------------- /mysql_import.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | DOCUMENTATION = """ 4 | --- 5 | module: mysql_import.py 6 | short_description: Update MySQL database with SQL instructions 7 | options: 8 | sql: 9 | description: 10 | - SQL statements to execute 11 | user: 12 | description: 13 | - username to connect to MySQL 14 | password: 15 | description: 16 | - password to connect to MySQL 17 | database: 18 | description: 19 | - database to use 20 | tables: 21 | required: true 22 | description: 23 | - list of tables which will be modified 24 | 25 | """ 26 | 27 | import yaml 28 | import pymysql 29 | 30 | from ansible.module_utils.basic import AnsibleModule 31 | 32 | 33 | def main(): 34 | module_args = dict( 35 | sql=dict(type='str', required=True), 36 | user=dict(type='str', required=True), 37 | password=dict(type='str', required=True, no_log=True), 38 | database=dict(type='str', required=True), 39 | tables=dict(type='list', required=True, elements='str'), 40 | ) 41 | 42 | result = dict( 43 | changed=False 44 | ) 45 | 46 | module = AnsibleModule( 47 | argument_spec=module_args, 48 | supports_check_mode=True 49 | ) 50 | 51 | got = {} 52 | wanted = {} 53 | tables = module.params['tables'] 54 | sql = module.params['sql'] 55 | statements = [statement.strip() 56 | for statement in sql.split(";\n") 57 | if statement.strip()] 58 | 59 | connection = pymysql.connect(user=module.params['user'], 60 | password=module.params['password'], 61 | db=module.params['database'], 62 | charset='utf8mb4', 63 | cursorclass=pymysql.cursors.DictCursor) 64 | with connection.cursor() as cursor: 65 | for table in tables: 66 | cursor.execute("SELECT * FROM {}".format(table)) 67 | got[table] = cursor.fetchall() 68 | 69 | with connection.cursor() as cursor: 70 | for statement in statements: 71 | try: 72 | cursor.execute(statement) 73 | except pymysql.OperationalError as err: 74 | code, message = err.args 75 | result['msg'] = "MySQL error for {}: {}".format( 76 | statement, 77 | message) 78 | module.fail_json(**result) 79 | for table in tables: 80 | cursor.execute("SELECT * FROM {}".format(table)) 81 | wanted[table] = cursor.fetchall() 82 | 83 | if got != wanted: 84 | result['changed'] = True 85 | result['diff'] = [dict( 86 | before_header=table, 87 | after_header=table, 88 | before=yaml.safe_dump(got[table]), 89 | after=yaml.safe_dump(wanted[table])) 90 | for table in tables 91 | if got[table] != wanted[table]] 92 | 93 | if module.check_mode or not result['changed']: 94 | module.exit_json(**result) 95 | 96 | connection.commit() 97 | 98 | module.exit_json(**result) 99 | 100 | 101 | if __name__ == '__main__': 102 | main() 103 | -------------------------------------------------------------------------------- /iosxr_sshkeys.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | DOCUMENTATION = """ 4 | --- 5 | module: iosxr_sshkeys.py 6 | short_description: Copy and enable SSH keys to IOSXR device 7 | description: 8 | - This module copy the provided SSH keys and attach them to the users. 9 | - The users are expected to already exist in the running configuration. 10 | - Users not listed in this module will have their SSH keys removed. 11 | options: 12 | keys: 13 | description: 14 | - dictionary mapping users to their SSH keys in OpenSSH format 15 | """ 16 | 17 | import yaml 18 | import textfsm 19 | import io 20 | import subprocess 21 | import base64 22 | import binascii 23 | import tempfile 24 | 25 | from ansible.module_utils.basic import AnsibleModule 26 | from ansible_collections.cisco.iosxr.plugins.module_utils.network.iosxr.iosxr import ( 27 | copy_file, 28 | get_connection, 29 | run_commands, 30 | ) 31 | 32 | 33 | def ssh2cisco(sshkey): 34 | """Convert a public key in OpenSSH format to the format expected by 35 | Cisco.""" 36 | proc = subprocess.run(["ssh-keygen", "-f", "/dev/stdin", "-e", "-mPKCS8"], 37 | input=sshkey.encode('ascii'), 38 | capture_output=True) 39 | if proc.returncode != 0: 40 | raise RuntimeError(f"unable to convert key: {sshkey}") 41 | decoded = base64.b64decode("".join(proc.stdout.decode( 42 | 'ascii').split("\n")[1:-2])) 43 | return binascii.hexlify(decoded).decode('ascii').upper() 44 | 45 | 46 | def main(): 47 | module_args = dict( 48 | keys=dict(type='dict', elements='str', required=True), 49 | ) 50 | 51 | module = AnsibleModule( 52 | argument_spec=module_args, 53 | supports_check_mode=True 54 | ) 55 | 56 | result = dict( 57 | changed=False 58 | ) 59 | 60 | # Get existing keys 61 | command = "show crypto key authentication rsa all" 62 | out = run_commands(module, command) 63 | 64 | # Parse keys 65 | 66 | # Key label: vincent 67 | # Type : RSA public key authentication 68 | # Size : 2048 69 | # Imported : 16:17:08 UTC Tue Aug 11 2020 70 | # Data : 71 | # 30820122 300D0609 2A864886 F70D0101 01050003 82010F00 3082010A 02820101 72 | # 00D81E5B A73D82F3 77B1E4B5 949FB245 60FB9167 7CD03AB7 ADDE7AFE A0B83174 73 | # A33EC0E6 1C887E02 2338367A 8A1DB0CE 0C3FBC51 15723AEB 07F301A4 B1A9961A 74 | # 2D00DBBD 2ABFC831 B0B25932 05B3BC30 B9514EA1 3DC22CBD DDCA6F02 026DBBB6 75 | # EE3CFADA AFA86F52 CAE7620D 17C3582B 4422D24F D68698A5 52ED1E9E 8E41F062 76 | # 7DE81015 F33AD486 C14D0BB1 68C65259 F9FD8A37 8DE52ED0 7B36E005 8C58516B 77 | # 7EA6C29A EEE0833B 42714618 50B3FFAC 15DBE3EF 8DA5D337 68DAECB9 904DE520 78 | # 2D627CEA 67E6434F E974CF6D 952AB2AB F074FBA3 3FB9B9CC A0CD0ADC 6E0CDB2A 79 | # 6A1CFEBA E97AF5A9 1FE41F6C 92E1F522 673E1A5F 69C68E11 4A13C0F3 0FFC782D 80 | # 27020301 0001 81 | 82 | out = out[0].replace(' \n', '\n') 83 | template = r""" 84 | Value Required Label (\w+) 85 | Value Required,List Data ([A-F0-9 ]+) 86 | 87 | Start 88 | ^Key label: ${Label} 89 | ^Data\s+: -> GetData 90 | 91 | GetData 92 | ^ ${Data} 93 | ^$$ -> Record Start 94 | """.lstrip() 95 | re_table = textfsm.TextFSM(io.StringIO(template)) 96 | got = {data[0]: "".join(data[1]).replace(' ', '') 97 | for data in re_table.ParseText(out)} 98 | 99 | # Check what we want 100 | wanted = {k: ssh2cisco(v) 101 | for k, v in module.params['keys'].items()} 102 | 103 | if got != wanted: 104 | result['changed'] = True 105 | result['diff'] = dict( 106 | before=yaml.safe_dump(got), 107 | after=yaml.safe_dump(wanted) 108 | ) 109 | 110 | if module.check_mode or not result['changed']: 111 | module.exit_json(**result) 112 | 113 | # Copy changed or missing SSH keys 114 | conn = get_connection(module) 115 | for user in wanted: 116 | if user not in got or wanted[user] != got[user]: 117 | dst = f"/harddisk:/publickey_{user}.raw" 118 | with tempfile.NamedTemporaryFile() as src: 119 | decoded = base64.b64decode( 120 | module.params['keys'][user].split()[1]) 121 | src.write(decoded) 122 | src.flush() 123 | copy_file(module, src.name, dst) 124 | command = ("admin crypto key import authentication rsa " 125 | f"username {user} {dst}") 126 | conn.send_command(command, prompt="yes/no", answer="yes") 127 | 128 | # Remove unwanted users 129 | for user in got: 130 | if user not in wanted: 131 | command = ("admin crypto key zeroize authentication rsa " 132 | f"username {user}") 133 | conn.send_command(command, prompt="yes/no", answer="yes") 134 | 135 | module.exit_json(**result) 136 | 137 | 138 | if __name__ == '__main__': 139 | main() 140 | -------------------------------------------------------------------------------- /irr_sync.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | DOCUMENTATION = """ 4 | --- 5 | module: irr_sync.py 6 | short_description: Filter objects to be sent to IRR 7 | options: 8 | irr: 9 | description: 10 | - IRR to target 11 | mntner: 12 | description: 13 | - object to use as a maintainer 14 | source: 15 | description: 16 | - objects to be sent 17 | """ 18 | 19 | RETURN = """ 20 | objects: 21 | description: object to be sent for sync through GPG-email 22 | type: str 23 | returned: changed 24 | """ 25 | 26 | import yaml 27 | import subprocess 28 | import re 29 | import functools 30 | 31 | from ansible.module_utils.basic import AnsibleModule 32 | from ansible.errors import AnsibleError 33 | 34 | 35 | class RPSLObject(object): 36 | """IRR object with normalization and comparison. 37 | 38 | The original form is kept is in the `raw' attribute. 39 | """ 40 | 41 | def __init__(self, raw): 42 | normalized = [] 43 | for line in raw.split("\n"): 44 | mo = re.match(r"(\S+:)\s*(.*)", line) 45 | name, value = mo.groups() 46 | normalized.append(f"{name:16}{value}") 47 | self.raw = "\n".join(normalized) 48 | 49 | def __repr__(self): 50 | key = self.raw.split('\n')[0].replace(" ", "") 51 | return f"" 52 | 53 | def __str__(self): 54 | return "\n".join((s.replace(" # Filtered", "") 55 | for s in self.raw.split("\n") 56 | if not s.startswith(( 57 | "created:", 58 | "last-modified:", 59 | "changed:", # ARIN: date automatically added 60 | "auth:", # RIPE: filtered 61 | "method:", # key-cert: autogenerated 62 | "owner:", # key-cert: autogenerated 63 | "fingerpr:" # key-cert: autogenerated 64 | )))) 65 | 66 | def __eq__(self, other): 67 | if not isinstance(other, RPSLObject): 68 | raise NotImplementedError( 69 | "cannot compare RPSLObject wih something else") 70 | return str(self) == str(other) 71 | 72 | 73 | def extract(raw, excluded): 74 | """Extract objects.""" 75 | # First step, remove comments and unwanted lines 76 | objects = "\n".join([obj 77 | for obj in raw.split("\n") 78 | if not obj.startswith(( 79 | "#", 80 | "%", 81 | ))]) 82 | # Second step, split objects 83 | objects = [RPSLObject(obj.strip()) 84 | for obj in re.split(r"\n\n+", objects) 85 | if obj.strip() 86 | and not obj.startswith( 87 | tuple(f"{x}:" for x in excluded))] 88 | # Last step, put objects in a dict 89 | objects = {repr(obj): obj 90 | for obj in objects} 91 | return objects 92 | 93 | 94 | def main(): 95 | module_args = dict( 96 | irr=dict(type='str', required=True), 97 | mntner=dict(type='str', required=True), 98 | source=dict(type='path', required=True), 99 | ) 100 | 101 | result = dict( 102 | changed=False, 103 | ) 104 | 105 | module = AnsibleModule( 106 | argument_spec=module_args, 107 | supports_check_mode=True 108 | ) 109 | 110 | irr = module.params['irr'] 111 | 112 | # Per-IRR variations: 113 | # - whois server 114 | whois = { 115 | 'ARIN': 'rr.arin.net', 116 | 'RIPE': 'whois.ripe.net', 117 | 'APNIC': 'whois.apnic.net' 118 | } 119 | # - whois options 120 | options = { 121 | 'ARIN': ['-r'], 122 | 'RIPE': ['-BrG'], 123 | 'APNIC': ['-BrG'] 124 | } 125 | # - objects excluded from synchronization 126 | excluded = ["domain"] 127 | if irr == "ARIN": 128 | # ARIN does not return these objects 129 | excluded.extend([ 130 | "key-cert", 131 | "mntner", 132 | ]) 133 | 134 | # Grab existing objects 135 | args = ["-h", whois[irr], 136 | "-s", irr, 137 | *options[irr], 138 | "-i", "mnt-by", 139 | module.params['mntner']] 140 | proc = subprocess.run(["whois", *args], capture_output=True) 141 | if proc.returncode != 0: 142 | raise AnsibleError( 143 | f"unable to query whois: {args}") 144 | got = extract(proc.stdout.decode('ascii'), excluded) 145 | 146 | with open(module.params['source']) as f: 147 | source = f.read() 148 | wanted = extract(source, excluded) 149 | 150 | if got != wanted: 151 | result['changed'] = True 152 | if module._diff: 153 | result['diff'] = [ 154 | dict(before_header=k, 155 | after_header=k, 156 | before=str(got.get(k, "")), 157 | after=str(wanted.get(k, ""))) 158 | for k in set((*wanted.keys(), *got.keys())) 159 | if k not in wanted or k not in got or wanted[k] != got[k]] 160 | 161 | # We send all source objects and deleted objects. 162 | deleted_mark = f"{'delete:':16}deleted by CMDB" 163 | deleted = "\n\n".join([f"{got[k].raw}\n{deleted_mark}" 164 | for k in got 165 | if k not in wanted]) 166 | result['objects'] = f"{source}\n\n{deleted}" 167 | 168 | module.exit_json(**result) 169 | 170 | 171 | if __name__ == '__main__': 172 | main() 173 | -------------------------------------------------------------------------------- /netbox_sync.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | DOCUMENTATION = """ 4 | --- 5 | module: netbox_sync.py 6 | short_description: Synchronize NetBox with changes from CMDB 7 | options: 8 | source: 9 | description: 10 | - YAML file to use as source 11 | api: 12 | description: 13 | - API endpoint for NetBox 14 | token: 15 | description: 16 | - Authentication token for NetBox 17 | max_workers: 18 | description: 19 | - Number of workers to retrieve information from NetBox 20 | """ 21 | 22 | import yaml 23 | import copy 24 | import pynetbox 25 | import attr 26 | import re 27 | from packaging import version 28 | 29 | from concurrent.futures import ThreadPoolExecutor, as_completed 30 | from ansible.module_utils.basic import AnsibleModule 31 | from ansible.errors import AnsibleError 32 | 33 | 34 | def slugify(s): 35 | # Adapted from: 36 | # https://github.com/netbox-community/netbox/blob/7a53e24f9721a8506008e8bafc25ddd04fa2f412/netbox/project-static/js/forms.js#L37 37 | s = re.sub(r'[^-.\w\s]', '', s) 38 | s = re.sub(r'^[\s.]+|[\s.]+$', '', s) 39 | s = re.sub(r'[.\s-]+', '-', s) 40 | return s.lower() 41 | 42 | 43 | @attr.s(kw_only=True) 44 | class Synchronizer(object): 45 | 46 | module = attr.ib() # Ansible module 47 | netbox = attr.ib() # NetBox API 48 | source = attr.ib() # source of truth 49 | before = attr.ib() # what's currently in NetBox 50 | after = attr.ib() # what we want in NetBox 51 | 52 | # Attribute that should be present in concrete classes: 53 | # key = "name" # what attribute to lookup for existing objects 54 | # app = "dcim" # NetBox app containing the data 55 | # table = "manufacturers" # NetBox table containing the data 56 | foreign = {} # foreign attributes (attribute name → class) 57 | only_on_create = () # list of attributes to only use when creating 58 | remove_unused = None # remove not managed anymore (max to remove) 59 | 60 | cache = None 61 | 62 | def wanted(self): 63 | """Extract from source of truth the set of wanted elements.""" 64 | raise NotImplementedError() 65 | 66 | def get(self, ep, key): 67 | """Get current record from Netbox.""" 68 | if self.cache is None: 69 | self.cache = {} 70 | for element in ep.filter(tag=["cmdb"]): 71 | self.cache[element[self.key]] = element 72 | try: 73 | return self.cache[key] 74 | except KeyError: 75 | return ep.get(**{self.key: key}) 76 | 77 | def prepare(self): 78 | """Prepare for synchronization by looking what's currently in NetBox 79 | and what should be updated to match the source of truth. 80 | Return True if there is a change. 81 | 82 | """ 83 | changed = False 84 | ep = getattr(getattr(self.netbox, self.app), self.table) 85 | self.before[self.table] = {} 86 | self.after[self.table] = {} 87 | 88 | # Check what should be added 89 | wanted = self.wanted() 90 | 91 | def process(key, details): 92 | current = self.get(ep, key) 93 | if current is not None: 94 | current = {k: v for k, v in dict(current).items() 95 | if k in ('id',) + tuple(details.keys())} 96 | # When an attribute is a choice, use the value 97 | for attrib in current: 98 | if type(current[attrib]) is dict and \ 99 | set(current[attrib].keys()) in ({"id", "label", "value"}, 100 | {"label", "value"}): 101 | current[attrib] = current[attrib]["value"] 102 | if "tags" in current and current["tags"]: 103 | if type(current["tags"][0]) is dict: 104 | current["tags"] = [c["name"] for c in current["tags"]] 105 | current["tags"].sort() 106 | # Before/after takes the current value 107 | self.before[self.table][key] = dict(current) 108 | self.after[self.table][key] = copy.deepcopy(dict(current)) 109 | # Update attributes with the newest one 110 | for attrib in details: 111 | if attrib in self.only_on_create: 112 | continue 113 | if attrib == "tags": 114 | # Tags could be merged here. We choose not to 115 | # because it's difficult to delete our own 116 | # tags. 117 | if "cmdb" not in details["tags"]: 118 | details["tags"].append("cmdb") 119 | details["tags"].sort() 120 | self.after[self.table][key][attrib] = details[attrib] 121 | # Link foreign keys for "before" 122 | for fkey, fclass in self.foreign.items(): 123 | old = self.before[self.table][key][fkey] 124 | if old is None: 125 | continue 126 | if fclass.key in old: 127 | self.before[self.table][key][fkey] = old[fclass.key] 128 | else: 129 | # We do not have fclass.key directly here, 130 | # let's search by ID! 131 | id = old["id"] 132 | for k, v in self.before[fclass.table].items(): 133 | if id == v["id"]: 134 | self.before[self.table][key][fkey] = k 135 | break 136 | else: 137 | raise RuntimeError("unable to find foreign key " 138 | f"{fkey} for {k}") 139 | # Is there a diff? 140 | for attrib in self.after[self.table][key]: 141 | if attrib not in self.before[self.table][key] or \ 142 | self.after[self.table][key][attrib] != \ 143 | self.before[self.table][key][attrib]: 144 | return True 145 | else: 146 | self.after[self.table][key] = details 147 | return True 148 | return False 149 | 150 | with ThreadPoolExecutor( 151 | max_workers=self.module.params['max_workers']) as executor: 152 | futures = (executor.submit(process, key, details) 153 | for key, details in wanted.items()) 154 | for future in as_completed(futures): 155 | changed |= future.result() 156 | 157 | # Check what should be removed 158 | if not self.remove_unused or \ 159 | not self.module.params['cleanup'] or \ 160 | not self.before["tags"]: 161 | return changed 162 | unused = 0 163 | for key, existing in self.cache.items(): 164 | if key not in self.before[self.table]: 165 | changed = True 166 | unused += 1 167 | self.before[self.table][key] = {} 168 | 169 | if unused > self.remove_unused: 170 | raise AnsibleError(f"refuse to remove {unused} " 171 | f"(more than {self.remove_unused}) " 172 | f"objects from {self.table}") 173 | 174 | return changed 175 | 176 | def _normalize_tags(self, tags): 177 | """Normalize tags as a list of string with Netbox <= 2.8 or a list of 178 | dicts with more recent versions. The provided list is expected 179 | to contain strings and dicts but dicts may only be present 180 | because fetched through the API (internally, we should use 181 | strings only). 182 | """ 183 | if version.parse(self.netbox.version) <= version.parse('2.8'): 184 | return tags 185 | return [{"name": t} if type(t) is str else t 186 | for t in tags] 187 | 188 | def synchronize(self): 189 | """After preparation, synchronize the changes in NetBox. Currently, 190 | only do a one-way synchronization.""" 191 | ep = getattr(getattr(self.netbox, self.app), self.table) 192 | for key, details in self.after[self.table].items(): 193 | if key not in self.before[self.table]: 194 | # We need to create the object 195 | for attrib in details: 196 | if attrib in self.foreign: 197 | details[attrib] = self.after[ 198 | self.foreign[attrib].table][details[attrib]]["id"] 199 | # New objects may not have tags 200 | details["tags"] = self._normalize_tags(list(set( 201 | details.get("tags", [])).union({"cmdb"}))) 202 | result = ep.create(**{self.key: key}, **details) 203 | details["id"] = result.id 204 | else: 205 | # Is there something to update? 206 | current = self.before[self.table][key] 207 | diff = False 208 | for attrib in details: 209 | if attrib not in current or \ 210 | details[attrib] != current[attrib]: 211 | diff = True 212 | break 213 | if diff: 214 | current = self.get(ep, key) 215 | for attrib in details: 216 | if attrib == "id": 217 | continue 218 | if attrib == "tags": 219 | details["tags"] = self._normalize_tags( 220 | details["tags"]) 221 | if attrib not in self.foreign: 222 | setattr(current, attrib, details[attrib]) 223 | else: 224 | # We cannot update a foreign key. Only do 225 | # it if they differ (and die horribly). 226 | if details[attrib] is None: 227 | newid = None 228 | else: 229 | newid = self.after[self.foreign[attrib].table][ 230 | details[attrib]]["id"] 231 | if getattr(current, attrib) is None: 232 | oldid = None 233 | else: 234 | oldid = getattr(current, attrib).id 235 | if newid != oldid: 236 | setattr(current, attrib, newid) 237 | current.save() 238 | 239 | def cleanup(self): 240 | """Cleanup unused entries.""" 241 | ep = getattr(getattr(self.netbox, self.app), self.table) 242 | for key in self.before[self.table]: 243 | if key in self.after[self.table]: 244 | continue 245 | current = self.get(ep, key) 246 | current.delete() 247 | 248 | 249 | class SyncTags(Synchronizer): 250 | app = "extras" 251 | table = "tags" 252 | key = "name" 253 | 254 | def wanted(self): 255 | result = {"cmdb": dict(slug="cmdb", 256 | color="8bc34a", 257 | description="synced by network CMDB")} 258 | result.update({tag: dict(slug=tag, 259 | color="9e9e9e", 260 | description="synced by network CMDB") 261 | for details in self.source['ips'] 262 | for tag in details.get("tags", [])}) 263 | return result 264 | 265 | 266 | class SyncTenants(Synchronizer): 267 | app = "tenancy" 268 | table = "tenants" 269 | key = "name" 270 | 271 | def wanted(self): 272 | return {"Network": dict(slug="network", 273 | description="Network team")} 274 | 275 | 276 | class SyncSites(Synchronizer): 277 | 278 | app = "dcim" 279 | table = "sites" 280 | key = "name" 281 | only_on_create = ("status", "slug") 282 | 283 | def wanted(self): 284 | result = set(details["datacenter"] 285 | for details in self.source['devices'].values() 286 | if "datacenter" in details) 287 | return {k: dict(slug=k, 288 | status="planned") 289 | for k in result} 290 | 291 | 292 | class SyncManufacturers(Synchronizer): 293 | 294 | app = "dcim" 295 | table = "manufacturers" 296 | key = "name" 297 | 298 | def wanted(self): 299 | result = set(details["manufacturer"] 300 | for details in self.source['devices'].values() 301 | if "manufacturer" in details) 302 | return {k: {"slug": slugify(k)} 303 | for k in result} 304 | 305 | 306 | class SyncDeviceTypes(Synchronizer): 307 | 308 | app = "dcim" 309 | table = "device_types" 310 | key = "model" 311 | foreign = {"manufacturer": SyncManufacturers} 312 | 313 | def wanted(self): 314 | result = set((details["manufacturer"], details["model"]) 315 | for details in self.source['devices'].values() 316 | if "model" in details) 317 | return {k[1]: dict(manufacturer=k[0], 318 | slug=slugify(k[1])) 319 | for k in result} 320 | 321 | 322 | class SyncDeviceRoles(Synchronizer): 323 | 324 | app = "dcim" 325 | table = "device_roles" 326 | key = "name" 327 | only_on_create = ("slug") 328 | 329 | def wanted(self): 330 | result = set(details["role"] 331 | for details in self.source['devices'].values() 332 | if "role" in details) 333 | return {k: dict(slug=slugify(k), 334 | color="8bc34a") 335 | for k in result} 336 | 337 | 338 | class SyncDevices(Synchronizer): 339 | app = "dcim" 340 | table = "devices" 341 | key = "name" 342 | foreign = {"device_role": SyncDeviceRoles, 343 | "device_type": SyncDeviceTypes, 344 | "site": SyncSites, 345 | "tenant": SyncTenants} 346 | remove_unused = 10 347 | 348 | def wanted(self): 349 | return {name: dict(device_role=details["role"], 350 | device_type=details["model"], 351 | site=details["datacenter"], 352 | tenant="Network") 353 | for name, details in self.source['devices'].items() 354 | if {"datacenter", "model", "role"} <= set(details.keys())} 355 | 356 | 357 | class SyncIPs(Synchronizer): 358 | app = "ipam" 359 | table = "ip-addresses" 360 | key = "address" 361 | foreign = {"tenant": SyncTenants} 362 | remove_unused = 1000 363 | 364 | def get(self, ep, key): 365 | """Grab IP address from Netbox.""" 366 | if self.cache is None: 367 | self.cache = {} 368 | for element in ep.filter(tag=["cmdb"]): 369 | # Current element if it exists is overriden. We do not 370 | # really handle the case where multiple addresses have 371 | # the cmdb tag. 372 | self.cache[element["address"]] = element 373 | 374 | try: 375 | return self.cache[key] 376 | except KeyError: 377 | pass 378 | 379 | # There may be duplicate. We need to grab the "best". 380 | results = ep.filter(**{self.key: key}) 381 | results = [r for r in results] 382 | if len(results) == 0: 383 | return None 384 | scores = [0]*len(results) 385 | for idx, result in enumerate(results): 386 | if "cmdb" in [str(r) for r in result.tags]: 387 | scores[idx] += 10 388 | if getattr(result, "interface", None) is not None: 389 | scores[idx] += 5 390 | if getattr(result, "assigned_object", None) is not None: 391 | scores[idx] += 5 392 | return sorted(zip(scores, results), 393 | reverse=True, key=lambda k: k[0])[0][1] 394 | 395 | def wanted(self): 396 | wanted = {} 397 | for details in self.source['ips']: 398 | if details['ip'] in wanted: 399 | wanted[details['ip']]['description'] = \ 400 | f"{details['device']} (and others)" 401 | else: 402 | wanted[details['ip']] = dict( 403 | tenant="Network", 404 | status="active", 405 | dns_name="", # information is present in DNS 406 | description=f"{details['device']}: {details['interface']}", 407 | tags=details.get('tags', []), 408 | role=None, 409 | vrf=None) 410 | return wanted 411 | 412 | 413 | def main(): 414 | module_args = dict( 415 | source=dict(type='path', required=True), 416 | api=dict(type='str', required=True), 417 | token=dict(type='str', required=True, no_log=True), 418 | cleanup=dict(type='bool', required=False, default=True), 419 | max_workers=dict(type='int', required=False, default=10) 420 | ) 421 | 422 | result = dict( 423 | changed=False 424 | ) 425 | 426 | module = AnsibleModule( 427 | argument_spec=module_args, 428 | supports_check_mode=True 429 | ) 430 | 431 | source = yaml.safe_load(open(module.params['source'])) 432 | for device, details in source['devices'].items(): 433 | if details is None: 434 | source['devices'][device] = {} 435 | netbox = pynetbox.api(module.params['api'], 436 | token=module.params['token']) 437 | 438 | sync_args = dict( 439 | module=module, 440 | netbox=netbox, 441 | source=source, 442 | before={}, 443 | after={} 444 | ) 445 | synchronizers = [synchronizer(**sync_args) for synchronizer in [ 446 | SyncTags, 447 | SyncTenants, 448 | SyncSites, 449 | SyncManufacturers, 450 | SyncDeviceTypes, 451 | SyncDeviceRoles, 452 | SyncDevices, 453 | SyncIPs 454 | ]] 455 | 456 | # Check what needs to be synchronized 457 | try: 458 | for synchronizer in synchronizers: 459 | result['changed'] |= synchronizer.prepare() 460 | except AnsibleError as e: 461 | result['msg'] = e.message 462 | module.fail_json(**result) 463 | if module._diff and result['changed']: 464 | result['diff'] = [ 465 | dict( 466 | before_header=table, 467 | after_header=table, 468 | before=yaml.safe_dump(sync_args["before"][table]), 469 | after=yaml.safe_dump(sync_args["after"][table])) 470 | for table in sync_args["after"] 471 | if sync_args["before"][table] != sync_args["after"][table] 472 | ] 473 | if module.check_mode or not result['changed']: 474 | module.exit_json(**result) 475 | 476 | # Synchronize 477 | for synchronizer in synchronizers: 478 | synchronizer.synchronize() 479 | for synchronizer in synchronizers[::-1]: 480 | synchronizer.cleanup() 481 | module.exit_json(**result) 482 | 483 | 484 | if __name__ == '__main__': 485 | main() 486 | --------------------------------------------------------------------------------