From de8143e89f76b46321c4a2ae54808068e5a924c5 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Fri, 8 Mar 2024 13:30:21 +0100 Subject: [PATCH 01/39] Cleaning up code --- .github/actions/quality.yml | 25 +++ netbox_zabbix_sync.py | 293 ++++++++++++++++++------------------ 2 files changed, 168 insertions(+), 150 deletions(-) create mode 100644 .github/actions/quality.yml diff --git a/.github/actions/quality.yml b/.github/actions/quality.yml new file mode 100644 index 0000000..f96f14d --- /dev/null +++ b/.github/actions/quality.yml @@ -0,0 +1,25 @@ +--- +name: Pylint + +on: [push] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.8", "3.9", "3.10"] + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pylint + pip install -r requirements.txt + - name: Analysing the code with pylint + run: | + pylint $(git ls-files '*.py') diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index aaf86fa..fe15e9b 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -1,10 +1,10 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 """Netbox to Zabbix sync script.""" -from os import environ, path, sys -from packaging import version import logging import argparse +from os import environ, path, sys +from packaging import version from pynetbox import api from pyzabbix import ZabbixAPI, ZabbixAPIException try: @@ -19,9 +19,8 @@ try: hostgroup_format, nb_device_filter ) - except ModuleNotFoundError: - print(f"Configuration file config.py not found in main directory." + print("Configuration file config.py not found in main directory." "Please create the file or rename the config.py.example file to config.py.") sys.exit(0) @@ -46,7 +45,7 @@ logger.setLevel(logging.WARNING) def main(arguments): """Run the sync process.""" # set environment variables - if(arguments.verbose): + if arguments.verbose: logger.setLevel(logging.DEBUG) env_vars = ["ZABBIX_HOST", "NETBOX_HOST", "NETBOX_TOKEN"] if "ZABBIX_TOKEN" in environ: @@ -61,10 +60,13 @@ def main(arguments): raise EnvironmentVarError(e) # Get all virtual environment variables if "ZABBIX_TOKEN" in env_vars: + zabbix_user = None + zabbix_pass = None zabbix_token = environ.get("ZABBIX_TOKEN") - else: + else: zabbix_user = environ.get("ZABBIX_USER") zabbix_pass = environ.get("ZABBIX_PASS") + zabbix_token = None zabbix_host = environ.get("ZABBIX_HOST") netbox_host = environ.get("NETBOX_HOST") netbox_token = environ.get("NETBOX_TOKEN") @@ -78,9 +80,9 @@ def main(arguments): device_cfs = netbox.extras.custom_fields.filter(type="text", content_type_id=23) for cf in device_cfs: allowed_objects.append(cf.name) - for object in hg_objects: - if(object not in allowed_objects): - e = (f"Hostgroup item {object} is not valid. Make sure you" + for hg_object in hg_objects: + if hg_object not in allowed_objects: + e = (f"Hostgroup item {hg_object} is not valid. Make sure you" " use valid items and seperate them with '/'.") logger.error(e) raise HostgroupError(e) @@ -90,7 +92,7 @@ def main(arguments): if "ZABBIX_TOKEN" in env_vars: zabbix.login(api_token=zabbix_token) else: - m=(f"Logging in with Zabbix user and password," + m=("Logging in with Zabbix user and password," " consider using an API token instead.") logger.warning(m) zabbix.login(zabbix_user, zabbix_pass) @@ -113,7 +115,7 @@ def main(arguments): if proxy_name == "host": for proxy in zabbix_proxies: proxy['name'] = proxy.pop('host') - + # Go through all Netbox devices for nb_device in netbox_devices: try: @@ -123,9 +125,9 @@ def main(arguments): device.set_template(templates_config_context, templates_config_context_overrule) # Checks if device is part of cluster. # Requires clustering variable - if(device.isCluster() and clustering): + if device.isCluster() and clustering: # Check if device is master or slave - if(device.promoteMasterDevice()): + if device.promoteMasterDevice(): e = (f"Device {device.name} is " f"part of cluster and primary.") logger.info(e) @@ -137,8 +139,8 @@ def main(arguments): logger.info(e) continue # Checks if device is in cleanup state - if(device.status in zabbix_device_removal): - if(device.zabbix_id): + if device.status in zabbix_device_removal: + if device.zabbix_id: # Delete device from Zabbix # and remove hostID from Netbox. device.cleanup() @@ -149,22 +151,23 @@ def main(arguments): # but is not in Activate state logger.info(f"Skipping host {device.name} since its " f"not in the active state.") - continue - elif(device.status in zabbix_device_disable): + elif device.status in zabbix_device_disable: device.zabbix_state = 1 + else: + device.zabbix_state = 0 # Add hostgroup is variable is True # and Hostgroup is not present in Zabbix - if(create_hostgroups): + if create_hostgroups: for group in zabbix_groups: # If hostgroup is already present in Zabbix - if(group["name"] == device.hostgroup): + if group["name"] == device.hostgroup: break else: # Create new hostgroup hostgroup = device.createZabbixHostgroup() zabbix_groups.append(hostgroup) # Device is already present in Zabbix - if(device.zabbix_id): + if device.zabbix_id: device.ConsistencyCheck(zabbix_groups, zabbix_templates, zabbix_proxies, full_proxy_sync) # Add device to Zabbix @@ -176,41 +179,33 @@ def main(arguments): class SyncError(Exception): - pass - + """ Class SyncError """ class SyncExternalError(SyncError): - pass - + """ Class SyncExternalError """ class SyncInventoryError(SyncError): - pass - + """ Class SyncInventoryError """ class SyncDuplicateError(SyncError): - pass - + """ Class SyncDuplicateError """ class EnvironmentVarError(SyncError): - pass - + """ Class EnvironmentVarError """ class InterfaceConfigError(SyncError): - pass - + """ Class InterfaceConfigError """ class ProxyConfigError(SyncError): - pass - + """ Class ProxyConfigError """ class HostgroupError(SyncError): - pass + """ Class HostgroupError """ class TemplateError(SyncError): - pass + """ Class TemplateError """ class NetworkDevice(): - """ Represents Network device. INPUT: (Netbox device class, ZabbixAPI class, journal flag, NB journal class) @@ -222,6 +217,11 @@ class NetworkDevice(): self.name = nb.name self.status = nb.status.label self.zabbix = zabbix + self.zabbix_id = None + self.group_id = None + self.zbx_template_names = [] + self.zbx_templates = [] + self.hostgroup = None self.tenant = nb.tenant self.config_context = nb.config_context self.zbxproxy = "0" @@ -235,7 +235,7 @@ class NetworkDevice(): Sets basic information like IP address. """ # Return error if device does not have primary IP. - if(self.nb.primary_ip): + if self.nb.primary_ip: self.cidr = self.nb.primary_ip.address self.ip = self.cidr.split("/")[0] else: @@ -244,14 +244,14 @@ class NetworkDevice(): raise SyncInventoryError(e) # Check if device has custom field for ZBX ID - if(device_cf in self.nb.custom_fields): + if device_cf in self.nb.custom_fields: self.zabbix_id = self.nb.custom_fields[device_cf] else: e = f"Custom field {device_cf} not found for {self.name}." logger.warning(e) raise SyncInventoryError(e) - def set_hostgroup(self, format): + def set_hostgroup(self, hg_format): """Set the hostgroup for this device""" # Get all variables from the NB data dev_location = str(self.nb.location) if self.nb.location else None @@ -268,17 +268,17 @@ class NetworkDevice(): "site": site, "site_group": site_group, "tenant": tenant, "tenant_group": tenant_group} # Generate list based off string input format - hg_items = format.split("/") + hg_items = hg_format.split("/") hostgroup = "" # Go through all hostgroup items for item in hg_items: # Check if the variable (such as Tenant) is empty. - if(not hostgroup_vars[item]): + if not hostgroup_vars[item]: continue # Check if the item is a custom field name - if(item not in hostgroup_vars): + if item not in hostgroup_vars: cf_value = self.nb.custom_fields[item] if item in self.nb.custom_fields else None - if(cf_value): + if cf_value: # If there is a cf match, add the value of this cf to the hostgroup hostgroup += cf_value + "/" # Should there not be a match, this means that @@ -287,15 +287,16 @@ class NetworkDevice(): # Add value of predefined variable to hostgroup format hostgroup += hostgroup_vars[item] + "/" # If the final hostgroup variable is empty - if(not hostgroup): + if not hostgroup: e = (f"{self.name} has no reliable hostgroup. This is" "most likely due to the use of custom fields that are empty.") logger.error(e) raise SyncInventoryError(e) # Remove final inserted "/" and set hostgroup to class var self.hostgroup = hostgroup.rstrip("/") - + def set_template(self, prefer_config_context, overrule_custom): + """ Set Template """ self.zbx_template_names = None # Gather templates ONLY from the device specific context if prefer_config_context: @@ -319,27 +320,26 @@ class NetworkDevice(): return True def get_templates_cf(self): + """ Get template from custom field """ # Get Zabbix templates from the device type device_type_cfs = self.nb.device_type.custom_fields # Check if the ZBX Template CF is present - if(template_cf in device_type_cfs): + if template_cf in device_type_cfs: # Set value to template return [device_type_cfs[template_cf]] - else: - # Custom field not found, return error - e = (f"Custom field {template_cf} not " - f"found for {self.nb.device_type.manufacturer.name}" - f" - {self.nb.device_type.display}.") - - raise TemplateError(e) + # Custom field not found, return error + e = (f"Custom field {template_cf} not " + f"found for {self.nb.device_type.manufacturer.name}" + f" - {self.nb.device_type.display}.") + raise TemplateError(e) def get_templates_context(self): - # Get Zabbix templates from the device context - if("zabbix" not in self.config_context): + """ Get Zabbix templates from the device context """ + if "zabbix" not in self.config_context: e = ("Key 'zabbix' not found in config " f"context for template host {self.name}") raise TemplateError(e) - if("templates" not in self.config_context["zabbix"]): + if "templates" not in self.config_context["zabbix"]: e = ("Key 'zabbix' not found in config " f"context for template host {self.name}") raise TemplateError(e) @@ -349,21 +349,18 @@ class NetworkDevice(): """ Checks if device is part of cluster. """ - if(self.nb.virtual_chassis): - return True - else: - return False + return bool(self.nb.virtual_chassis) def getClusterMaster(self): """ Returns chassis master ID. """ - if(not self.isCluster()): + if not self.isCluster(): e = (f"Unable to proces {self.name} for cluster calculation: " f"not part of a cluster.") logger.warning(e) raise SyncInventoryError(e) - elif(not self.nb.virtual_chassis.master): + elif not self.nb.virtual_chassis.master: e = (f"{self.name} is part of a Netbox virtual chassis which does " "not have a master configured. Skipping for this reason.") logger.error(e) @@ -378,16 +375,14 @@ class NetworkDevice(): Returns True if succesfull, returns False if device is secondary. """ masterid = self.getClusterMaster() - if(masterid == self.id): + if masterid == self.id: logger.debug(f"Device {self.name} is primary cluster member. " f"Modifying hostname from {self.name} to " + f"{self.nb.virtual_chassis.name}.") self.name = self.nb.virtual_chassis.name - return True - else: - logger.debug(f"Device {self.name} is non-primary cluster member.") - return False + logger.debug(f"Device {self.name} is non-primary cluster member.") + return False def zbxTemplatePrepper(self, templates): """ @@ -396,8 +391,8 @@ class NetworkDevice(): OUTPUT: True """ # Check if there are templates defined - if(not self.zbx_template_names): - e = (f"No templates found for device {self.name}") + if not self.zbx_template_names: + e = f"No templates found for device {self.name}" logger.info(e) raise SyncInventoryError() # Set variable to empty list @@ -408,7 +403,7 @@ class NetworkDevice(): # Go through all templates found in Zabbix for zbx_template in templates: # If the template names match - if(zbx_template['name'] == nb_template): + if zbx_template['name'] == nb_template: # Set match variable to true, add template details # to class variable and return debug log template_match = True @@ -418,7 +413,7 @@ class NetworkDevice(): f" for host {self.name}.") logger.debug(e) # Return error should the template not be found in Zabbix - if(not template_match): + if not template_match: e = (f"Unable to find template {nb_template} " f"for host {self.name} in Zabbix. Skipping host...") logger.warning(e) @@ -432,9 +427,9 @@ class NetworkDevice(): """ # Go through all groups for group in groups: - if(group['name'] == self.hostgroup): + if group['name'] == self.hostgroup: self.group_id = group['groupid'] - e = (f"Found group {group['name']} for host {self.name}.") + e = f"Found group {group['name']} for host {self.name}." logger.debug(e) return True else: @@ -448,7 +443,7 @@ class NetworkDevice(): Removes device from external resources. Resets custom fields in Netbox. """ - if(self.zabbix_id): + if self.zabbix_id: try: self.zabbix.host.delete(self.zabbix_id) self.nb.custom_fields[device_cf] = None @@ -466,10 +461,7 @@ class NetworkDevice(): Checks if hostname exists in Zabbix. """ host = self.zabbix.host.get(filter={'name': self.name}, output=[]) - if(host): - return True - else: - return False + return bool(host) def setInterfaceDetails(self): """ @@ -481,9 +473,9 @@ class NetworkDevice(): interface = ZabbixInterface(self.nb.config_context, self.ip) # Check if Netbox has device context. # If not fall back to old config. - if(interface.get_context()): + if interface.get_context(): # If device is SNMP type, add aditional information. - if(interface.interface["type"] == 2): + if interface.interface["type"] == 2: interface.set_snmp() else: interface.set_default() @@ -494,21 +486,22 @@ class NetworkDevice(): raise SyncInventoryError(e) def setProxy(self, proxy_list): - # check if Zabbix Proxy has been defined in config context - if("zabbix" in self.nb.config_context): - if("proxy" in self.nb.config_context["zabbix"]): + """ check if Zabbix Proxy has been defined in config context """ + if "zabbix" in self.nb.config_context: + if "proxy" in self.nb.config_context["zabbix"]: proxy = self.nb.config_context["zabbix"]["proxy"] # Try matching proxy for px in proxy_list: - if(px["name"] == proxy): + if px["name"] == proxy: self.zbxproxy = px["proxyid"] logger.debug(f"Found proxy {proxy}" f" for {self.name}.") return True - else: - e = f"{self.name}: Defined proxy {proxy} not found." - logger.warning(e) - return False + return False + e = f"{self.name}: Defined proxy {proxy} not found." + logger.warning(e) + return False + return False def createInZabbix(self, groups, templates, proxies, description="Host added by Netbox sync script."): @@ -516,14 +509,14 @@ class NetworkDevice(): Creates Zabbix host object with parameters from Netbox object. """ # Check if hostname is already present in Zabbix - if(not self._zabbixHostnameExists()): + if not self._zabbixHostnameExists(): # Get group and template ID's for host - if(not self.getZabbixGroup(groups)): + if not self.getZabbixGroup(groups): raise SyncInventoryError() self.zbxTemplatePrepper(templates) templateids = [] for template in self.zbx_templates: - templateids.append({'templateid': template['templateid']}) + templateids.append({'templateid': template['templateid']}) # Set interface, group and template configuration interfaces = self.setInterfaceDetails() groups = [{"groupid": self.group_id}] @@ -575,7 +568,7 @@ class NetworkDevice(): except ZabbixAPIException as e: e = f"Couldn't add hostgroup, Zabbix returned {str(e)}." logger.error(e) - raise SyncExternalError(e) + raise SyncExternalError(e) from e def updateZabbixHost(self, **kwargs): """ @@ -604,54 +597,54 @@ class NetworkDevice(): 'interfaceid'], selectGroups=["groupid"], selectParentTemplates=["templateid"]) - if(len(host) > 1): + if len(host) > 1: e = (f"Got {len(host)} results for Zabbix hosts " f"with ID {self.zabbix_id} - hostname {self.name}.") logger.error(e) raise SyncInventoryError(e) - elif(len(host) == 0): + if len(host) == 0: e = (f"No Zabbix host found for {self.name}. " f"This is likely the result of a deleted Zabbix host " f"without zeroing the ID field in Netbox.") logger.error(e) raise SyncInventoryError(e) - else: - host = host[0] + host = host[0] - if(host["host"] == self.name): + if host["host"] == self.name: logger.debug(f"Device {self.name}: hostname in-sync.") else: logger.warning(f"Device {self.name}: hostname OUT of sync. " f"Received value: {host['host']}") self.updateZabbixHost(host=self.name) - + # Check if the templates are in-sync - if(not self.zbx_template_comparer(host["parentTemplates"])): + if not self.zbx_template_comparer(host["parentTemplates"]): logger.warning(f"Device {self.name}: template(s) OUT of sync.") # Update Zabbix with NB templates and clear any old / lost templates - self.updateZabbixHost(templates_clear=host["parentTemplates"], templates=self.zbx_templates) + self.updateZabbixHost(templates_clear=host["parentTemplates"], + templates=self.zbx_templates) else: logger.debug(f"Device {self.name}: template(s) in-sync.") for group in host["groups"]: - if(group["groupid"] == self.group_id): + if group["groupid"] == self.group_id: logger.debug(f"Device {self.name}: hostgroup in-sync.") break else: logger.warning(f"Device {self.name}: hostgroup OUT of sync.") self.updateZabbixHost(groups={'groupid': self.group_id}) - if(int(host["status"]) == self.zabbix_state): + if int(host["status"]) == self.zabbix_state: logger.debug(f"Device {self.name}: status in-sync.") else: logger.warning(f"Device {self.name}: status OUT of sync.") self.updateZabbixHost(status=str(self.zabbix_state)) # Check if a proxy has been defined - if(self.zbxproxy != "0"): + if self.zbxproxy != "0": # Check if expected proxyID matches with configured proxy - if(("proxy_hostid" in host and host["proxy_hostid"] == self.zbxproxy) - or ("proxyid" in host and host["proxyid"] == self.zbxproxy)): + if (("proxy_hostid" in host and host["proxy_hostid"] == self.zbxproxy) or + ("proxyid" in host and host["proxyid"] == self.zbxproxy)): logger.debug(f"Device {self.name}: proxy in-sync.") else: # Proxy diff, update value @@ -661,9 +654,9 @@ class NetworkDevice(): else: self.updateZabbixHost(proxyid=self.zbxproxy) else: - if(("proxy_hostid" in host and not host["proxy_hostid"] == "0") + if (("proxy_hostid" in host and not host["proxy_hostid"] == "0") or ("proxyid" in host and not host["proxyid"] == "0")): - if(proxy_power): + if proxy_power: # Variable full_proxy_sync has been enabled # delete the proxy link in Zabbix if version.parse(self.zabbix.api_version()) < version.parse("7.0.0"): @@ -679,41 +672,41 @@ class NetworkDevice(): " -p flag was ommited: no " "changes have been made.") # If only 1 interface has been found - if(len(host['interfaces']) == 1): + if len(host['interfaces']) == 1: updates = {} # Go through each key / item and check if it matches Zabbix for key, item in self.setInterfaceDetails()[0].items(): # Check if Netbox value is found in Zabbix - if(key in host["interfaces"][0]): + if key in host["interfaces"][0]: # If SNMP is used, go through nested dict # to compare SNMP parameters - if(type(item) == dict and key == "details"): + if isinstance(item,dict) and key == "details": for k, i in item.items(): - if(k in host["interfaces"][0][key]): + if k in host["interfaces"][0][key]: # Set update if values don't match - if(host["interfaces"][0][key][k] != str(i)): + if host["interfaces"][0][key][k] != str(i): # If dict has not been created, add it - if(key not in updates): + if key not in updates: updates[key] = {} updates[key][k] = str(i) # If SNMP version has been changed # break loop and force full SNMP update - if(k == "version"): + if k == "version": break # Force full SNMP config update # when version has changed. - if(key in updates): - if("version" in updates[key]): + if key in updates: + if "version" in updates[key]: for k, i in item.items(): updates[key][k] = str(i) continue # Set update if values don't match - if(host["interfaces"][0][key] != str(item)): + if host["interfaces"][0][key] != str(item): updates[key] = item - if(updates): + if updates: # If interface updates have been found: push to Zabbix logger.warning(f"Device {self.name}: Interface OUT of sync.") - if("type" in updates): + if "type" in updates: # Changing interface type not supported. Raise exception. e = (f"Device {self.name}: changing interface type to " f"{str(updates['type'])} is not supported.") @@ -730,7 +723,7 @@ class NetworkDevice(): except ZabbixAPIException as e: e = f"Zabbix returned the following error: {str(e)}." logger.error(e) - raise SyncExternalError(e) + raise SyncExternalError(e) from e else: # If no updates are found, Zabbix interface is in-sync e = f"Device {self.name}: interface in-sync." @@ -740,12 +733,14 @@ class NetworkDevice(): f" Host has total of {len(host['interfaces'])} interfaces. " "Manual interfention required.") logger.error(e) - SyncInventoryError(e) + raise SyncInventoryError(e) def create_journal_entry(self, severity, message): - # Send a new Journal entry to Netbox. Usefull for viewing actions - # in Netbox without having to look in Zabbix or the script log output - if(self.journal): + """ + Send a new Journal entry to Netbox. Usefull for viewing actions + in Netbox without having to look in Zabbix or the script log output + """ + if self.journal: # Check if the severity is valid if severity not in ["info", "success", "warning", "danger"]: logger.warning(f"Value {severity} not valid for NB journal entries.") @@ -757,12 +752,14 @@ class NetworkDevice(): } try: self.nb_journals.create(journal) + logger.debug(f"Created journal entry in NB for host {self.name}") return True - logger.debug(f"Crated journal entry in NB for host {self.name}") except pynetbox.RequestError as e: logger.warning("Unable to create journal entry for " f"{self.name}: NB returned {e}") - + return False + return False + def zbx_template_comparer(self, tmpls_from_zabbix): """ Compares the Netbox and Zabbix templates with each other. @@ -777,15 +774,15 @@ class NetworkDevice(): # Go through each Zabbix template for pos, zbx_tmpl in enumerate(tmpls_from_zabbix): # Check if template IDs match - if(nb_tmpl["templateid"] == zbx_tmpl["templateid"]): + if nb_tmpl["templateid"] == zbx_tmpl["templateid"]: # Templates match. Remove this template from the Zabbix templates # and add this NB template to the list of successfull templates tmpls_from_zabbix.pop(pos) succesfull_templates.append(nb_tmpl) - logger.debug(f"Device {self.name}: template {nb_tmpl['name']} is present in Zabbix.") + logger.debug(f"Device {self.name}: template " + f"{nb_tmpl['name']} is present in Zabbix.") break - if(len(succesfull_templates) == len(self.zbx_templates) and - len(tmpls_from_zabbix) == 0): + if len(succesfull_templates) == len(self.zbx_templates) and len(tmpls_from_zabbix) == 0: # All of the Netbox templates have been confirmed as successfull # and the ZBX template list is empty. This means that # all of the templates match. @@ -793,8 +790,6 @@ class NetworkDevice(): return False - - class ZabbixInterface(): """Class that represents a Zabbix interface.""" @@ -805,40 +800,38 @@ class ZabbixInterface(): self.interface = self.skelet def get_context(self): - # check if Netbox custom context has been defined. - if("zabbix" in self.context): + """ check if Netbox custom context has been defined. """ + if "zabbix" in self.context: zabbix = self.context["zabbix"] if("interface_type" in zabbix and "interface_port" in zabbix): self.interface["type"] = zabbix["interface_type"] self.interface["port"] = zabbix["interface_port"] return True - else: - return False - else: return False + return False def set_snmp(self): - # Check if interface is type SNMP - if(self.interface["type"] == 2): + """ Check if interface is type SNMP """ + if self.interface["type"] == 2: # Checks if SNMP settings are defined in Netbox - if("snmp" in self.context["zabbix"]): + if "snmp" in self.context["zabbix"]: snmp = self.context["zabbix"]["snmp"] self.interface["details"] = {} # Checks if bulk config has been defined - if("bulk" in snmp): + if "bulk" in snmp: self.interface["details"]["bulk"] = str(snmp.pop("bulk")) else: # Fallback to bulk enabled if not specified self.interface["details"]["bulk"] = "1" # SNMP Version config is required in Netbox config context - if(snmp.get("version")): + if snmp.get("version"): self.interface["details"]["version"] = str(snmp.pop("version")) else: e = "SNMP version option is not defined." raise InterfaceConfigError(e) # If version 1 or 2 is used, get community string - if(self.interface["details"]["version"] in ['1','2']): - if("community" in snmp): + if self.interface["details"]["version"] in ['1','2']: + if "community" in snmp: # Set SNMP community to confix context value community = snmp["community"] else: @@ -847,12 +840,12 @@ class ZabbixInterface(): self.interface["details"]["community"] = str(community) # If version 3 has been used, get all # SNMPv3 Netbox related configs - elif(self.interface["details"]["version"] == '3'): + elif self.interface["details"]["version"] == '3': items = ["securityname", "securitylevel", "authpassphrase", "privpassphrase", "authprotocol", "privprotocol", "contextname"] for key, item in snmp.items(): - if(key in items): + if key in items: self.interface["details"][key] = str(item) else: e = "Unsupported SNMP version." @@ -865,7 +858,7 @@ class ZabbixInterface(): raise InterfaceConfigError(e) def set_default(self): - # Set default config to SNMPv2,port 161 and community macro. + """ Set default config to SNMPv2, port 161 and community macro. """ self.interface = self.skelet self.interface["type"] = "2" self.interface["port"] = "161" @@ -874,7 +867,7 @@ class ZabbixInterface(): "bulk": "1"} -if(__name__ == "__main__"): +if __name__ == "__main__": parser = argparse.ArgumentParser( description='A script to sync Zabbix with Netbox device data.' ) From 15d40873b08f6ab8f89231b40fa3569171538eaa Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Fri, 8 Mar 2024 13:32:29 +0100 Subject: [PATCH 02/39] rename actions to workflows --- .github/workflows/quality.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .github/workflows/quality.yml diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml new file mode 100644 index 0000000..f96f14d --- /dev/null +++ b/.github/workflows/quality.yml @@ -0,0 +1,25 @@ +--- +name: Pylint + +on: [push] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.8", "3.9", "3.10"] + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pylint + pip install -r requirements.txt + - name: Analysing the code with pylint + run: | + pylint $(git ls-files '*.py') From 89d5f220649ffe7831c9ed998dd969e62727be00 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Fri, 8 Mar 2024 13:36:42 +0100 Subject: [PATCH 03/39] changed versions for workflow --- .github/workflows/quality.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index f96f14d..23be5fb 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -8,11 +8,11 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.11","3.12"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v5 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies From 3c7079117a3f160011ea6e9a73f0bf567dd76b15 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Fri, 8 Mar 2024 13:40:31 +0100 Subject: [PATCH 04/39] changed versions for workflow (again) --- .github/workflows/quality.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 23be5fb..71792f3 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -10,7 +10,7 @@ jobs: matrix: python-version: ["3.11","3.12"] steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: From 0d02e096e9b4e3e8749390b24080d33adfebc8c3 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Fri, 8 Mar 2024 13:49:18 +0100 Subject: [PATCH 05/39] Disable pylint name checking --- netbox_zabbix_sync.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index fe15e9b..1db4959 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -1,4 +1,6 @@ #!/usr/bin/env python3 +# pylint: disable=invalid-name + """Netbox to Zabbix sync script.""" import logging From 23bef6b5497b7b543a84e0bea6066ea64109a23f Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Fri, 8 Mar 2024 13:52:08 +0100 Subject: [PATCH 06/39] disable pylint module name checks --- .github/workflows/quality.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 71792f3..a56c573 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -22,4 +22,4 @@ jobs: pip install -r requirements.txt - name: Analysing the code with pylint run: | - pylint $(git ls-files '*.py') + pylint --module-naming-style=any $(git ls-files '*.py') From 2fcd21a72396eacad33c648a850538fc831b8910 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Fri, 8 Mar 2024 14:00:15 +0100 Subject: [PATCH 07/39] code cleanup --- netbox_zabbix_sync.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index 1db4959..e4378f3 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -362,13 +362,12 @@ class NetworkDevice(): f"not part of a cluster.") logger.warning(e) raise SyncInventoryError(e) - elif not self.nb.virtual_chassis.master: + if not self.nb.virtual_chassis.master: e = (f"{self.name} is part of a Netbox virtual chassis which does " "not have a master configured. Skipping for this reason.") logger.error(e) raise SyncInventoryError(e) - else: - return self.nb.virtual_chassis.master.id + return self.nb.virtual_chassis.master.id def promoteMasterDevice(self): """ @@ -419,7 +418,7 @@ class NetworkDevice(): e = (f"Unable to find template {nb_template} " f"for host {self.name} in Zabbix. Skipping host...") logger.warning(e) - raise SyncInventoryError(e) + raise SyncInventoryError(e) from e def getZabbixGroup(self, groups): """ @@ -434,11 +433,10 @@ class NetworkDevice(): e = f"Found group {group['name']} for host {self.name}." logger.debug(e) return True - else: - e = (f"Unable to find group '{self.hostgroup}' " + e = (f"Unable to find group '{self.hostgroup}' " f"for host {self.name} in Zabbix.") - logger.warning(e) - raise SyncInventoryError(e) + logger.warning(e) + raise SyncInventoryError(e) def cleanup(self): """ @@ -456,7 +454,7 @@ class NetworkDevice(): except ZabbixAPIException as e: e = f"Zabbix returned the following error: {str(e)}." logger.error(e) - raise SyncExternalError(e) + raise SyncExternalError(e) from e def _zabbixHostnameExists(self): """ @@ -485,7 +483,7 @@ class NetworkDevice(): except InterfaceConfigError as e: e = f"{self.name}: {e}" logger.warning(e) - raise SyncInventoryError(e) + raise SyncInventoryError(e) from e def setProxy(self, proxy_list): """ check if Zabbix Proxy has been defined in config context """ From c684ac4a9dff49707c9f2ddd1b9372ed16044656 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Fri, 8 Mar 2024 14:23:47 +0100 Subject: [PATCH 08/39] Futher cleanup --- netbox_zabbix_sync.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index e4378f3..9b43a94 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 -# pylint: disable=invalid-name +# pylint: disable=invalid-name, logging-not-lazy, too-many-locals, logging-fstring-interpolation + """Netbox to Zabbix sync script.""" @@ -46,6 +47,7 @@ logger.setLevel(logging.WARNING) def main(arguments): """Run the sync process.""" + # pylint: disable=too-many-branches, too-many-statements # set environment variables if arguments.verbose: logger.setLevel(logging.DEBUG) @@ -208,6 +210,7 @@ class TemplateError(SyncError): """ Class TemplateError """ class NetworkDevice(): + # pylint: disable=too-many-instance-attributes """ Represents Network device. INPUT: (Netbox device class, ZabbixAPI class, journal flag, NB journal class) @@ -332,8 +335,8 @@ class NetworkDevice(): # Custom field not found, return error e = (f"Custom field {template_cf} not " f"found for {self.nb.device_type.manufacturer.name}" - f" - {self.nb.device_type.display}.") - raise TemplateError(e) + f" - {self.nb.device_type.display}.") + raise TemplateError(e) from e def get_templates_context(self): """ Get Zabbix templates from the device context """ @@ -544,7 +547,7 @@ class NetworkDevice(): except ZabbixAPIException as e: e = f"Couldn't add {self.name}, Zabbix returned {str(e)}." logger.error(e) - raise SyncExternalError(e) + raise SyncExternalError(e) from e # Set Netbox custom field to hostID value. self.nb.custom_fields[device_cf] = int(self.zabbix_id) self.nb.save() @@ -580,11 +583,12 @@ class NetworkDevice(): except ZabbixAPIException as e: e = f"Zabbix returned the following error: {str(e)}." logger.error(e) - raise SyncExternalError(e) + raise SyncExternalError(e) from e logger.info(f"Updated host {self.name} with data {kwargs}.") - self.create_journal_entry("info", f"Updated host in Zabbix with latest NB data.") + self.create_journal_entry("info", "Updated host in Zabbix with latest NB data.") def ConsistencyCheck(self, groups, templates, proxies, proxy_power): + # pylint: disable=too-many-branches, too-many-statements """ Checks if Zabbix object is still valid with Netbox parameters. """ @@ -672,6 +676,7 @@ class NetworkDevice(): " -p flag was ommited: no " "changes have been made.") # If only 1 interface has been found + # pylint: disable=too-many-nested-blocks if len(host['interfaces']) == 1: updates = {} # Go through each key / item and check if it matches Zabbix @@ -812,6 +817,7 @@ class ZabbixInterface(): def set_snmp(self): """ Check if interface is type SNMP """ + # pylint: disable=too-many-branches if self.interface["type"] == 2: # Checks if SNMP settings are defined in Netbox if "snmp" in self.context["zabbix"]: From 0d7c581ee2d0bf3f0b2731110cd7107d249c8956 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Fri, 8 Mar 2024 14:29:12 +0100 Subject: [PATCH 09/39] fixed undefined-variable --- netbox_zabbix_sync.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index 9b43a94..8cf0812 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -185,6 +185,9 @@ def main(arguments): class SyncError(Exception): """ Class SyncError """ +class JournalError(Exception): + """ Class SyncError """ + class SyncExternalError(SyncError): """ Class SyncExternalError """ @@ -759,7 +762,7 @@ class NetworkDevice(): self.nb_journals.create(journal) logger.debug(f"Created journal entry in NB for host {self.name}") return True - except pynetbox.RequestError as e: + except JournalError(e) as e: logger.warning("Unable to create journal entry for " f"{self.name}: NB returned {e}") return False From c538c51b7b451f50356a488f9da473c20d754949 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Fri, 8 Mar 2024 14:54:48 +0100 Subject: [PATCH 10/39] minor README.md update --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6586b31..7674a71 100644 --- a/README.md +++ b/README.md @@ -271,4 +271,4 @@ To configure the interface parameters you'll need to use custom context. Custom I would recommend using macros for sensitive data such as community strings since the data in Netbox is plain-text. -Note: Not all SNMP data is required for a working configuration. [The following parameters are allowed ](https://www.zabbix.com/documentation/current/manual/api/reference/hostinterface/object#details_tag "The following parameters are allowed ")but are not all required, depending on your environment. +> **_NOTE:_** Not all SNMP data is required for a working configuration. [The following parameters are allowed ](https://www.zabbix.com/documentation/current/manual/api/reference/hostinterface/object#details_tag "The following parameters are allowed ")but are not all required, depending on your environment. From 142aae75e0bbf07404fc43e73e8370a017aadb88 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Fri, 8 Mar 2024 14:56:30 +0100 Subject: [PATCH 11/39] removed directory --- .github/actions/quality.yml | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 .github/actions/quality.yml diff --git a/.github/actions/quality.yml b/.github/actions/quality.yml deleted file mode 100644 index f96f14d..0000000 --- a/.github/actions/quality.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -name: Pylint - -on: [push] - -jobs: - build: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.8", "3.9", "3.10"] - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install pylint - pip install -r requirements.txt - - name: Analysing the code with pylint - run: | - pylint $(git ls-files '*.py') From e05c35a3eaa613d5b9dd115731e5b347f02368c5 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Fri, 8 Mar 2024 22:44:25 +0100 Subject: [PATCH 12/39] added container building workflow --- .github/workflows/publish-image.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/workflows/publish-image.yml diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml new file mode 100644 index 0000000..e516e54 --- /dev/null +++ b/.github/workflows/publish-image.yml @@ -0,0 +1,17 @@ +name: Publish Docker image to GHCR + +on: + push + +jobs: + build_and_publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Login to GHCR.io + run: echo "${{ secret.GHCR_PAT }}" | docker login --username ${{ secret.GHCR_USER }} --password-stdin ghcr.io + - name: Build and tag image + run: docker build . -t ghcr.io/${{ secret.GHCR_USER }}/netbox-zabbix-sync:latest + - name: Push image to GHCR.io + run: docker push ghcr.io/${{ secret.GHCR_USER }}/netbox-zabbix-sync:latest + From d46b749af0d95040b933a5dbe9cef5306926ed4c Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Fri, 8 Mar 2024 22:48:57 +0100 Subject: [PATCH 13/39] corrected typos --- .github/workflows/publish-image.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml index e516e54..d7b29a9 100644 --- a/.github/workflows/publish-image.yml +++ b/.github/workflows/publish-image.yml @@ -9,9 +9,9 @@ jobs: steps: - uses: actions/checkout@v4 - name: Login to GHCR.io - run: echo "${{ secret.GHCR_PAT }}" | docker login --username ${{ secret.GHCR_USER }} --password-stdin ghcr.io + run: echo "${{ secrets.GHCR_PAT }}" | docker login --username ${{ secrets.GHCR_USER }} --password-stdin ghcr.io - name: Build and tag image - run: docker build . -t ghcr.io/${{ secret.GHCR_USER }}/netbox-zabbix-sync:latest + run: docker build . -t ghcr.io/${{ secrets.GHCR_USER }}/netbox-zabbix-sync:latest - name: Push image to GHCR.io - run: docker push ghcr.io/${{ secret.GHCR_USER }}/netbox-zabbix-sync:latest + run: docker push ghcr.io/${{ secrets.GHCR_USER }}/netbox-zabbix-sync:latest From 7c988f9ff850768ac5deb8ecbaccae7a94b48d11 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Fri, 8 Mar 2024 22:51:53 +0100 Subject: [PATCH 14/39] changed over to checkout@v3 --- .github/workflows/publish-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml index d7b29a9..1a56a0f 100644 --- a/.github/workflows/publish-image.yml +++ b/.github/workflows/publish-image.yml @@ -7,7 +7,7 @@ jobs: build_and_publish: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v3 - name: Login to GHCR.io run: echo "${{ secrets.GHCR_PAT }}" | docker login --username ${{ secrets.GHCR_USER }} --password-stdin ghcr.io - name: Build and tag image From 33cf3e5358dfb952c4392193b7a6af3e3a570539 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Fri, 8 Mar 2024 22:53:34 +0100 Subject: [PATCH 15/39] changed back to checkout@v4 and commited Dockerfile --- .github/workflows/publish-image.yml | 2 +- Dockerfile | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 Dockerfile diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml index 1a56a0f..d7b29a9 100644 --- a/.github/workflows/publish-image.yml +++ b/.github/workflows/publish-image.yml @@ -7,7 +7,7 @@ jobs: build_and_publish: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Login to GHCR.io run: echo "${{ secrets.GHCR_PAT }}" | docker login --username ${{ secrets.GHCR_USER }} --password-stdin ghcr.io - name: Build and tag image diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..fa8d9c4 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,9 @@ +# syntax=docker/dockerfile:1 +FROM python:3.12-alpine +RUN mkdir -p /opt/netbox-zabbix +COPY . /opt/netbox-zabbix +WORKDIR /opt/netbox-zabbix +RUN if ! [ -f ./config.py ]; then cp ./config.py.example ./config.py; fi +RUN pip install -r ./requirements.txt +ENTRYPOINT ["python"] +CMD ["/opt/netbox-zabbix/netbox_zabbix_sync.py", "-v"] From dcd84e836b846a68ae4e1af99ff0853e22ec6b7b Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Mon, 11 Mar 2024 11:03:37 +0100 Subject: [PATCH 16/39] Chained in quality check --- .github/workflows/publish-image.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml index d7b29a9..670d9b0 100644 --- a/.github/workflows/publish-image.yml +++ b/.github/workflows/publish-image.yml @@ -4,6 +4,8 @@ on: push jobs: + test_quality: + uses: ./.github/workflows/quality.yml build_and_publish: runs-on: ubuntu-latest steps: From 5922d3e8ae42115e0b09e398bb29d6f3e8296ebc Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Mon, 11 Mar 2024 11:06:31 +0100 Subject: [PATCH 17/39] allow call from another workflow --- .github/workflows/quality.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index a56c573..8af4e9c 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -1,7 +1,7 @@ --- name: Pylint -on: [push] +on: ["push","workflow_call"] jobs: build: From bf325c6839de44b7284f336da633a91a857ade78 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Wed, 13 Mar 2024 11:38:07 +0100 Subject: [PATCH 18/39] testing workflow --- .github/workflows/publish-image.yml | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml index 670d9b0..78b8b64 100644 --- a/.github/workflows/publish-image.yml +++ b/.github/workflows/publish-image.yml @@ -1,7 +1,16 @@ -name: Publish Docker image to GHCR +name: Publish Docker image to GHCR on a new version on: - push + push: + branches: + - main + - dockertest +# tags: +# - [0-9]+.* + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} jobs: test_quality: @@ -10,10 +19,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Login to GHCR.io - run: echo "${{ secrets.GHCR_PAT }}" | docker login --username ${{ secrets.GHCR_USER }} --password-stdin ghcr.io + - name: Login to ghcr.io + run: echo "${{ secrets.GHCR_PAT }}" | docker login --username ${{ github.actor }} --password-stdin ${{ env.REGISTRY }} - name: Build and tag image - run: docker build . -t ghcr.io/${{ secrets.GHCR_USER }}/netbox-zabbix-sync:latest - - name: Push image to GHCR.io - run: docker push ghcr.io/${{ secrets.GHCR_USER }}/netbox-zabbix-sync:latest + run: docker build . -t ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}:latest + - name: Push image to ghcr.io + run: docker push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest From 3a39c314bede7d7f47a580865c170f63e55bbd72 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Wed, 13 Mar 2024 11:42:28 +0100 Subject: [PATCH 19/39] removed `on: push` from pylint --- .github/workflows/quality.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 8af4e9c..7b01f6f 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -1,7 +1,8 @@ --- -name: Pylint +name: Pylint Quality control -on: ["push","workflow_call"] +on: + workflow_call jobs: build: From 4b7f3ec0b9da04c92d5d77ad5caf796d417c8252 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Wed, 13 Mar 2024 11:47:45 +0100 Subject: [PATCH 20/39] try a different way of publishing --- .github/workflows/publish-image.yml | 35 ++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml index 78b8b64..e32d212 100644 --- a/.github/workflows/publish-image.yml +++ b/.github/workflows/publish-image.yml @@ -19,10 +19,33 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Login to ghcr.io - run: echo "${{ secrets.GHCR_PAT }}" | docker login --username ${{ github.actor }} --password-stdin ${{ env.REGISTRY }} - - name: Build and tag image - run: docker build . -t ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}:latest - - name: Push image to ghcr.io - run: docker push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest +# - name: Login to ghcr.io +# run: echo "${{ secrets.GHCR_PAT }}" | docker login --username ${{ github.actor }} --password-stdin ${{ env.REGISTRY }} +# - name: Build and tag image +# run: docker build . -t ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}:latest +# - name: Push image to ghcr.io +# run: docker push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest + + - name: Log in to the Container registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GHCR_PAT }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=semver,pattern={{ version }} + + - name: Build and push Docker image + uses: docker/build-push-action@v4 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} From 661ce8828743a76c51a0e5466b521a0e12eec50a Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Wed, 13 Mar 2024 11:50:43 +0100 Subject: [PATCH 21/39] updated versions --- .github/workflows/publish-image.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml index e32d212..6252706 100644 --- a/.github/workflows/publish-image.yml +++ b/.github/workflows/publish-image.yml @@ -27,7 +27,7 @@ jobs: # run: docker push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest - name: Log in to the Container registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} @@ -35,14 +35,14 @@ jobs: - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@v4 + uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} tags: | type=semver,pattern={{ version }} - name: Build and push Docker image - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: . push: true From 18d29c98d3224f094ec949a34215e7dff03b4876 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Wed, 13 Mar 2024 11:58:40 +0100 Subject: [PATCH 22/39] updated tags --- .github/workflows/publish-image.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml index 6252706..86f9fcd 100644 --- a/.github/workflows/publish-image.yml +++ b/.github/workflows/publish-image.yml @@ -40,6 +40,7 @@ jobs: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} tags: | type=semver,pattern={{ version }} + type=ref,event=branch - name: Build and push Docker image uses: docker/build-push-action@v5 From e82631c89dde23e77e063e571553ee18574c3d0c Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Wed, 13 Mar 2024 13:16:22 +0100 Subject: [PATCH 23/39] modified tags --- .github/workflows/publish-image.yml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml index 86f9fcd..e9e6421 100644 --- a/.github/workflows/publish-image.yml +++ b/.github/workflows/publish-image.yml @@ -18,22 +18,15 @@ jobs: build_and_publish: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 -# - name: Login to ghcr.io -# run: echo "${{ secrets.GHCR_PAT }}" | docker login --username ${{ github.actor }} --password-stdin ${{ env.REGISTRY }} -# - name: Build and tag image -# run: docker build . -t ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}:latest -# - name: Push image to ghcr.io -# run: docker push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest - - - name: Log in to the Container registry + - name: Checkout sources + uses: actions/checkout@v4 + - name: Log in to the container registry uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GHCR_PAT }} - - - name: Extract metadata (tags, labels) for Docker + - name: Extract metadata (tags, labels) id: meta uses: docker/metadata-action@v5 with: @@ -41,7 +34,8 @@ jobs: tags: | type=semver,pattern={{ version }} type=ref,event=branch - + type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} + type=sha - name: Build and push Docker image uses: docker/build-push-action@v5 with: From 4aa8b6d2fbc32e1fd6e2de231333dc80f88457f8 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Wed, 13 Mar 2024 13:55:46 +0100 Subject: [PATCH 24/39] updated README.md with Docker instructions --- README.md | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7674a71..6924183 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,33 @@ A script to create, update and delete Zabbix hosts using Netbox device objects. -## Installation +## Installation via Docker + +To pull the latest stable version to your local cache, use the following docker pull command: +``` +docker pull ghcr.io/TheNetworkGuy/netbox-zabbix-sync:latest +``` + +Make sure to specify the needed environment variables for the script to work (see [here](#set-environment-variables)) +on the command line or use an [env file](https://docs.docker.com/reference/cli/docker/container/run/#env). + +``` +docker run -d -t -i -e ZABBIX_HOST='https://zabbix.local' \ +-e ZABBIX_TOKEN='othersecrettoken' \ +-e NETBOX_HOST='https://netbox.local' \ +-e NETBOX_TOKEN='secrettoken' \ +--name netbox-zabbix-sync ghcr.io/TheNetworkGuy/netbox-zabbix-sync:latest +``` + +This should run a one-time sync, you can check the sync with `docker logs`. + +The image uses the default `config.py` for it's configuration, you can use a volume mount in the docker run command +to override with your own config file if needed (see [config file](#config-file)): +``` +docker run -d -t -i -v $(pwd)/config.py:/opt/netbox-zabbix/config.py ... +``` + +## Installation from Source ### Cloning the repository ``` From 3079a88de8674723d03da213ffb5fc9d26308d04 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Wed, 13 Mar 2024 14:00:03 +0100 Subject: [PATCH 25/39] better `docker logs` example. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6924183..67fa514 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ docker run -d -t -i -e ZABBIX_HOST='https://zabbix.local' \ --name netbox-zabbix-sync ghcr.io/TheNetworkGuy/netbox-zabbix-sync:latest ``` -This should run a one-time sync, you can check the sync with `docker logs`. +This should run a one-time sync, you can check the sync with `docker logs netbox-zabbix-sync`. The image uses the default `config.py` for it's configuration, you can use a volume mount in the docker run command to override with your own config file if needed (see [config file](#config-file)): From 71f604a6f6945bd10d556b61dda21eea99c0393d Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Mon, 25 Mar 2024 11:49:41 +0100 Subject: [PATCH 26/39] Added functionality to build full region and site_group paths to be used in hostgroup names. --- README.md | 5 + config.py.example | 43 +- netbox_zabbix_sync.py | 1807 +++++++++++++++++++++-------------------- 3 files changed, 953 insertions(+), 902 deletions(-) diff --git a/README.md b/README.md index 67fa514..120eee9 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,11 @@ You can specify the value like so, sperated by a "/": ``` hostgroup_format = "tenant/site/dev_location/dev_role" ``` +** Group traversal ** +The default behaviour for `region` is to only use the directly assigned region in the rendered hostgroup name. +However, by setting `traverse_region` to `True` in `config.py` the script will render a full region path of all parent regions for the hostgroup name. +`traverse_site_groups` controls the same behaviour for site_groups. + **custom fields** You can also use the value of custom fields under the device object. diff --git a/config.py.example b/config.py.example index 63f3d6f..51e7dc2 100644 --- a/config.py.example +++ b/config.py.example @@ -1,7 +1,8 @@ -# Template logic. +## Template logic. # Set to true to enable the template source information # coming from config context instead of a custom field. templates_config_context = False + # Set to true to give config context templates a # higher priority then custom field templates templates_config_context_overrule = False @@ -11,37 +12,47 @@ templates_config_context_overrule = False template_cf = "zabbix_template" device_cf = "zabbix_hostid" -# Enable clustering of devices with virtual chassis setup +## Enable clustering of devices with virtual chassis setup clustering = False -# Enable hostgroup generation. Requires permissions in Zabbix +## Enable hostgroup generation. Requires permissions in Zabbix create_hostgroups = True -# Create journal entries +## Create journal entries create_journal = False +## Proxy Sync # Set to true to enable removal of proxy's under hosts. Use with caution and make sure that you specified # all the required proxy's in the device config context before enabeling this option. # With this option disabled proxy's will only be added and modified for Zabbix hosts. full_proxy_sync = False -# Netbox to Zabbix device state convertion +## Netbox to Zabbix device state convertion zabbix_device_removal = ["Decommissioning", "Inventory"] zabbix_device_disable = ["Offline", "Planned", "Staged", "Failed"] -# Hostgroup mapping +## Hostgroup mapping # Available choices: dev_location, dev_role, manufacturer, region, site, site_group, tenant, tenant_group # You can also use CF (custom field) names under the device. The CF content will be used for the hostgroup generation. +# +# When using region in the group name, the default behaviour is to use name of the directly assigned region. +# By setting traverse_regions to True the full path of all parent regions will be used in the hostgroup, e.g.: +# +# 'Global/Europe/Netherlands/Amsterdam' instead of just 'Amsterdam'. +# +# traverse_site_groups controls the same behaviour for any assigned site_groups. hostgroup_format = "site/manufacturer/dev_role" +traverse_regions = False +traverse_site_groups = False -# Custom filter for device filtering. Variable must be present but can be left empty with no filtering. -# A couple of examples are as follows: +## Filtering +# Custom device filter, variable must be present but can be left empty with no filtering. +# A couple of examples: +# nb_device_filter = {} #No filter +# nb_device_filter = {"tag": "zabbix"} #Use a tag +# nb_device_filter = {"site": "HQ-AMS"} #Use a site name +# nb_device_filter = {"site": ["HQ-AMS", "HQ-FRA"]} #Device must be in either one of these sites +# nb_device_filter = {"site": "HQ-AMS", "tag": "zabbix", "role__n": ["PDU", "console-server"]} #Device must be in site HQ-AMS, have the tag zabbix and must not be part of the PDU or console-server role -# nb_device_filter = {} #No filter -# nb_device_filter = {"tag": "zabbix"} #Use a tag -# nb_device_filter = {"site": "HQ-AMS"} #Use a site name -# nb_device_filter = {"site": ["HQ-AMS", "HQ-FRA"]} #Device must be in either one of these sites -# nb_device_filter = {"site": "HQ-AMS", "tag": "zabbix", "role__n": ["PDU", "console-server"]} #Device must be in site HQ-AMS, have the tag zabbix and must not be part of the PDU or console-server role - -# Default device filter, only get devices which have a name in Netbox. -nb_device_filter = {"name__n": "null"} \ No newline at end of file +# Default device filter, only get devices which have a name in Netbox: +nb_device_filter = {"name__n": "null"} diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index 8cf0812..9dad31b 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -1,886 +1,921 @@ -#!/usr/bin/env python3 -# pylint: disable=invalid-name, logging-not-lazy, too-many-locals, logging-fstring-interpolation - - -"""Netbox to Zabbix sync script.""" - -import logging -import argparse -from os import environ, path, sys -from packaging import version -from pynetbox import api -from pyzabbix import ZabbixAPI, ZabbixAPIException -try: - from config import ( - templates_config_context, - templates_config_context_overrule, - clustering, create_hostgroups, - create_journal, full_proxy_sync, - template_cf, device_cf, - zabbix_device_removal, - zabbix_device_disable, - hostgroup_format, - nb_device_filter - ) -except ModuleNotFoundError: - print("Configuration file config.py not found in main directory." - "Please create the file or rename the config.py.example file to config.py.") - sys.exit(0) - -# Set logging -log_format = logging.Formatter('%(asctime)s - %(name)s - ' - '%(levelname)s - %(message)s') -lgout = logging.StreamHandler() -lgout.setFormatter(log_format) -lgout.setLevel(logging.DEBUG) - -lgfile = logging.FileHandler(path.join(path.dirname( - path.realpath(__file__)), "sync.log")) -lgfile.setFormatter(log_format) -lgfile.setLevel(logging.DEBUG) - -logger = logging.getLogger("Netbox-Zabbix-sync") -logger.addHandler(lgout) -logger.addHandler(lgfile) -logger.setLevel(logging.WARNING) - - -def main(arguments): - """Run the sync process.""" - # pylint: disable=too-many-branches, too-many-statements - # set environment variables - if arguments.verbose: - logger.setLevel(logging.DEBUG) - env_vars = ["ZABBIX_HOST", "NETBOX_HOST", "NETBOX_TOKEN"] - if "ZABBIX_TOKEN" in environ: - env_vars.append("ZABBIX_TOKEN") - else: - env_vars.append("ZABBIX_USER") - env_vars.append("ZABBIX_PASS") - for var in env_vars: - if var not in environ: - e = f"Environment variable {var} has not been defined." - logger.error(e) - raise EnvironmentVarError(e) - # Get all virtual environment variables - if "ZABBIX_TOKEN" in env_vars: - zabbix_user = None - zabbix_pass = None - zabbix_token = environ.get("ZABBIX_TOKEN") - else: - zabbix_user = environ.get("ZABBIX_USER") - zabbix_pass = environ.get("ZABBIX_PASS") - zabbix_token = None - zabbix_host = environ.get("ZABBIX_HOST") - netbox_host = environ.get("NETBOX_HOST") - netbox_token = environ.get("NETBOX_TOKEN") - # Set Netbox API - netbox = api(netbox_host, token=netbox_token, threading=True) - # Check if the provided Hostgroup layout is valid - hg_objects = hostgroup_format.split("/") - allowed_objects = ["dev_location", "dev_role", "manufacturer", "region", - "site", "site_group", "tenant", "tenant_group"] - # Create API call to get all custom fields which are on the device objects - device_cfs = netbox.extras.custom_fields.filter(type="text", content_type_id=23) - for cf in device_cfs: - allowed_objects.append(cf.name) - for hg_object in hg_objects: - if hg_object not in allowed_objects: - e = (f"Hostgroup item {hg_object} is not valid. Make sure you" - " use valid items and seperate them with '/'.") - logger.error(e) - raise HostgroupError(e) - # Set Zabbix API - try: - zabbix = ZabbixAPI(zabbix_host) - if "ZABBIX_TOKEN" in env_vars: - zabbix.login(api_token=zabbix_token) - else: - m=("Logging in with Zabbix user and password," - " consider using an API token instead.") - logger.warning(m) - zabbix.login(zabbix_user, zabbix_pass) - except ZabbixAPIException as e: - e = f"Zabbix returned the following error: {str(e)}." - logger.error(e) - # Set API parameter mapping based on API version - if version.parse(zabbix.api_version()) < version.parse("7.0.0"): - proxy_name = "host" - else: - proxy_name = "name" - # Get all Zabbix and Netbox data - netbox_devices = netbox.dcim.devices.filter(**nb_device_filter) - netbox_journals = netbox.extras.journal_entries - zabbix_groups = zabbix.hostgroup.get(output=['groupid', 'name']) - zabbix_templates = zabbix.template.get(output=['templateid', 'name']) - zabbix_proxies = zabbix.proxy.get(output=['proxyid', proxy_name]) - - # Sanitize data - if proxy_name == "host": - for proxy in zabbix_proxies: - proxy['name'] = proxy.pop('host') - - # Go through all Netbox devices - for nb_device in netbox_devices: - try: - device = NetworkDevice(nb_device, zabbix, netbox_journals, - create_journal) - device.set_hostgroup(hostgroup_format) - device.set_template(templates_config_context, templates_config_context_overrule) - # Checks if device is part of cluster. - # Requires clustering variable - if device.isCluster() and clustering: - # Check if device is master or slave - if device.promoteMasterDevice(): - e = (f"Device {device.name} is " - f"part of cluster and primary.") - logger.info(e) - else: - # Device is secondary in cluster. - # Don't continue with this device. - e = (f"Device {device.name} is part of cluster " - f"but not primary. Skipping this host...") - logger.info(e) - continue - # Checks if device is in cleanup state - if device.status in zabbix_device_removal: - if device.zabbix_id: - # Delete device from Zabbix - # and remove hostID from Netbox. - device.cleanup() - logger.info(f"Cleaned up host {device.name}.") - - else: - # Device has been added to Netbox - # but is not in Activate state - logger.info(f"Skipping host {device.name} since its " - f"not in the active state.") - elif device.status in zabbix_device_disable: - device.zabbix_state = 1 - else: - device.zabbix_state = 0 - # Add hostgroup is variable is True - # and Hostgroup is not present in Zabbix - if create_hostgroups: - for group in zabbix_groups: - # If hostgroup is already present in Zabbix - if group["name"] == device.hostgroup: - break - else: - # Create new hostgroup - hostgroup = device.createZabbixHostgroup() - zabbix_groups.append(hostgroup) - # Device is already present in Zabbix - if device.zabbix_id: - device.ConsistencyCheck(zabbix_groups, zabbix_templates, - zabbix_proxies, full_proxy_sync) - # Add device to Zabbix - else: - device.createInZabbix(zabbix_groups, zabbix_templates, - zabbix_proxies) - except SyncError: - pass - - -class SyncError(Exception): - """ Class SyncError """ - -class JournalError(Exception): - """ Class SyncError """ - -class SyncExternalError(SyncError): - """ Class SyncExternalError """ - -class SyncInventoryError(SyncError): - """ Class SyncInventoryError """ - -class SyncDuplicateError(SyncError): - """ Class SyncDuplicateError """ - -class EnvironmentVarError(SyncError): - """ Class EnvironmentVarError """ - -class InterfaceConfigError(SyncError): - """ Class InterfaceConfigError """ - -class ProxyConfigError(SyncError): - """ Class ProxyConfigError """ - -class HostgroupError(SyncError): - """ Class HostgroupError """ - -class TemplateError(SyncError): - """ Class TemplateError """ - -class NetworkDevice(): - # pylint: disable=too-many-instance-attributes - """ - Represents Network device. - INPUT: (Netbox device class, ZabbixAPI class, journal flag, NB journal class) - """ - - def __init__(self, nb, zabbix, nb_journal_class, journal=None): - self.nb = nb - self.id = nb.id - self.name = nb.name - self.status = nb.status.label - self.zabbix = zabbix - self.zabbix_id = None - self.group_id = None - self.zbx_template_names = [] - self.zbx_templates = [] - self.hostgroup = None - self.tenant = nb.tenant - self.config_context = nb.config_context - self.zbxproxy = "0" - self.zabbix_state = 0 - self.journal = journal - self.nb_journals = nb_journal_class - self._setBasics() - - def _setBasics(self): - """ - Sets basic information like IP address. - """ - # Return error if device does not have primary IP. - if self.nb.primary_ip: - self.cidr = self.nb.primary_ip.address - self.ip = self.cidr.split("/")[0] - else: - e = f"Device {self.name}: no primary IP." - logger.warning(e) - raise SyncInventoryError(e) - - # Check if device has custom field for ZBX ID - if device_cf in self.nb.custom_fields: - self.zabbix_id = self.nb.custom_fields[device_cf] - else: - e = f"Custom field {device_cf} not found for {self.name}." - logger.warning(e) - raise SyncInventoryError(e) - - def set_hostgroup(self, hg_format): - """Set the hostgroup for this device""" - # Get all variables from the NB data - dev_location = str(self.nb.location) if self.nb.location else None - dev_role = self.nb.device_role.name - manufacturer = self.nb.device_type.manufacturer.name - region = str(self.nb.site.region) if self.nb.site.region else None - site = self.nb.site.name - site_group = str(self.nb.site.group) if self.nb.site.group else None - tenant = str(self.tenant) if self.tenant else None - tenant_group = str(self.tenant.group) if tenant else None - # Set mapper for string -> variable - hostgroup_vars = {"dev_location": dev_location, "dev_role": dev_role, - "manufacturer": manufacturer, "region": region, - "site": site, "site_group": site_group, - "tenant": tenant, "tenant_group": tenant_group} - # Generate list based off string input format - hg_items = hg_format.split("/") - hostgroup = "" - # Go through all hostgroup items - for item in hg_items: - # Check if the variable (such as Tenant) is empty. - if not hostgroup_vars[item]: - continue - # Check if the item is a custom field name - if item not in hostgroup_vars: - cf_value = self.nb.custom_fields[item] if item in self.nb.custom_fields else None - if cf_value: - # If there is a cf match, add the value of this cf to the hostgroup - hostgroup += cf_value + "/" - # Should there not be a match, this means that - # the variable is invalid. Skip regardless. - continue - # Add value of predefined variable to hostgroup format - hostgroup += hostgroup_vars[item] + "/" - # If the final hostgroup variable is empty - if not hostgroup: - e = (f"{self.name} has no reliable hostgroup. This is" - "most likely due to the use of custom fields that are empty.") - logger.error(e) - raise SyncInventoryError(e) - # Remove final inserted "/" and set hostgroup to class var - self.hostgroup = hostgroup.rstrip("/") - - def set_template(self, prefer_config_context, overrule_custom): - """ Set Template """ - self.zbx_template_names = None - # Gather templates ONLY from the device specific context - if prefer_config_context: - try: - self.zbx_template_names = self.get_templates_context() - except TemplateError as e: - logger.warning(e) - return True - # Gather templates from the custom field but overrule - # them should there be any device specific templates - if overrule_custom: - try: - self.zbx_template_names = self.get_templates_context() - except TemplateError: - pass - if not self.zbx_template_names: - self.zbx_template_names = self.get_templates_cf() - return True - # Gather templates ONLY from the custom field - self.zbx_template_names = self.get_templates_cf() - return True - - def get_templates_cf(self): - """ Get template from custom field """ - # Get Zabbix templates from the device type - device_type_cfs = self.nb.device_type.custom_fields - # Check if the ZBX Template CF is present - if template_cf in device_type_cfs: - # Set value to template - return [device_type_cfs[template_cf]] - # Custom field not found, return error - e = (f"Custom field {template_cf} not " - f"found for {self.nb.device_type.manufacturer.name}" - f" - {self.nb.device_type.display}.") - raise TemplateError(e) from e - - def get_templates_context(self): - """ Get Zabbix templates from the device context """ - if "zabbix" not in self.config_context: - e = ("Key 'zabbix' not found in config " - f"context for template host {self.name}") - raise TemplateError(e) - if "templates" not in self.config_context["zabbix"]: - e = ("Key 'zabbix' not found in config " - f"context for template host {self.name}") - raise TemplateError(e) - return self.config_context["zabbix"]["templates"] - - def isCluster(self): - """ - Checks if device is part of cluster. - """ - return bool(self.nb.virtual_chassis) - - def getClusterMaster(self): - """ - Returns chassis master ID. - """ - if not self.isCluster(): - e = (f"Unable to proces {self.name} for cluster calculation: " - f"not part of a cluster.") - logger.warning(e) - raise SyncInventoryError(e) - if not self.nb.virtual_chassis.master: - e = (f"{self.name} is part of a Netbox virtual chassis which does " - "not have a master configured. Skipping for this reason.") - logger.error(e) - raise SyncInventoryError(e) - return self.nb.virtual_chassis.master.id - - def promoteMasterDevice(self): - """ - If device is Primary in cluster, - promote device name to the cluster name. - Returns True if succesfull, returns False if device is secondary. - """ - masterid = self.getClusterMaster() - if masterid == self.id: - logger.debug(f"Device {self.name} is primary cluster member. " - f"Modifying hostname from {self.name} to " + - f"{self.nb.virtual_chassis.name}.") - self.name = self.nb.virtual_chassis.name - return True - logger.debug(f"Device {self.name} is non-primary cluster member.") - return False - - def zbxTemplatePrepper(self, templates): - """ - Returns Zabbix template IDs - INPUT: list of templates from Zabbix - OUTPUT: True - """ - # Check if there are templates defined - if not self.zbx_template_names: - e = f"No templates found for device {self.name}" - logger.info(e) - raise SyncInventoryError() - # Set variable to empty list - self.zbx_templates = [] - # Go through all templates definded in Netbox - for nb_template in self.zbx_template_names: - template_match = False - # Go through all templates found in Zabbix - for zbx_template in templates: - # If the template names match - if zbx_template['name'] == nb_template: - # Set match variable to true, add template details - # to class variable and return debug log - template_match = True - self.zbx_templates.append({"templateid": zbx_template['templateid'], - "name": zbx_template['name']}) - e = (f"Found template {zbx_template['name']}" - f" for host {self.name}.") - logger.debug(e) - # Return error should the template not be found in Zabbix - if not template_match: - e = (f"Unable to find template {nb_template} " - f"for host {self.name} in Zabbix. Skipping host...") - logger.warning(e) - raise SyncInventoryError(e) from e - - def getZabbixGroup(self, groups): - """ - Returns Zabbix group ID - INPUT: list of hostgroups - OUTPUT: True / False - """ - # Go through all groups - for group in groups: - if group['name'] == self.hostgroup: - self.group_id = group['groupid'] - e = f"Found group {group['name']} for host {self.name}." - logger.debug(e) - return True - e = (f"Unable to find group '{self.hostgroup}' " - f"for host {self.name} in Zabbix.") - logger.warning(e) - raise SyncInventoryError(e) - - def cleanup(self): - """ - Removes device from external resources. - Resets custom fields in Netbox. - """ - if self.zabbix_id: - try: - self.zabbix.host.delete(self.zabbix_id) - self.nb.custom_fields[device_cf] = None - self.nb.save() - e = f"Deleted host {self.name} from Zabbix." - logger.info(e) - self.create_journal_entry("warning", "Deleted host from Zabbix") - except ZabbixAPIException as e: - e = f"Zabbix returned the following error: {str(e)}." - logger.error(e) - raise SyncExternalError(e) from e - - def _zabbixHostnameExists(self): - """ - Checks if hostname exists in Zabbix. - """ - host = self.zabbix.host.get(filter={'name': self.name}, output=[]) - return bool(host) - - def setInterfaceDetails(self): - """ - Checks interface parameters from Netbox and - creates a model for the interface to be used in Zabbix. - """ - try: - # Initiate interface class - interface = ZabbixInterface(self.nb.config_context, self.ip) - # Check if Netbox has device context. - # If not fall back to old config. - if interface.get_context(): - # If device is SNMP type, add aditional information. - if interface.interface["type"] == 2: - interface.set_snmp() - else: - interface.set_default() - return [interface.interface] - except InterfaceConfigError as e: - e = f"{self.name}: {e}" - logger.warning(e) - raise SyncInventoryError(e) from e - - def setProxy(self, proxy_list): - """ check if Zabbix Proxy has been defined in config context """ - if "zabbix" in self.nb.config_context: - if "proxy" in self.nb.config_context["zabbix"]: - proxy = self.nb.config_context["zabbix"]["proxy"] - # Try matching proxy - for px in proxy_list: - if px["name"] == proxy: - self.zbxproxy = px["proxyid"] - logger.debug(f"Found proxy {proxy}" - f" for {self.name}.") - return True - return False - e = f"{self.name}: Defined proxy {proxy} not found." - logger.warning(e) - return False - return False - - def createInZabbix(self, groups, templates, proxies, - description="Host added by Netbox sync script."): - """ - Creates Zabbix host object with parameters from Netbox object. - """ - # Check if hostname is already present in Zabbix - if not self._zabbixHostnameExists(): - # Get group and template ID's for host - if not self.getZabbixGroup(groups): - raise SyncInventoryError() - self.zbxTemplatePrepper(templates) - templateids = [] - for template in self.zbx_templates: - templateids.append({'templateid': template['templateid']}) - # Set interface, group and template configuration - interfaces = self.setInterfaceDetails() - groups = [{"groupid": self.group_id}] - # Set Zabbix proxy if defined - self.setProxy(proxies) - # Add host to Zabbix - try: - if version.parse(self.zabbix.api_version()) < version.parse("7.0.0"): - host = self.zabbix.host.create(host=self.name, - status=self.zabbix_state, - interfaces=interfaces, - groups=groups, - templates=templateids, - proxy_hostid=self.zbxproxy, - description=description) - else: - host = self.zabbix.host.create(host=self.name, - status=self.zabbix_state, - interfaces=interfaces, - groups=groups, - templates=templateids, - proxyid=self.zbxproxy, - description=description) - self.zabbix_id = host["hostids"][0] - except ZabbixAPIException as e: - e = f"Couldn't add {self.name}, Zabbix returned {str(e)}." - logger.error(e) - raise SyncExternalError(e) from e - # Set Netbox custom field to hostID value. - self.nb.custom_fields[device_cf] = int(self.zabbix_id) - self.nb.save() - msg = f"Created host {self.name} in Zabbix." - logger.info(msg) - self.create_journal_entry("success", msg) - else: - e = f"Unable to add {self.name} to Zabbix: host already present." - logger.warning(e) - - def createZabbixHostgroup(self): - """ - Creates Zabbix host group based on hostgroup format. - """ - try: - groupid = self.zabbix.hostgroup.create(name=self.hostgroup) - e = f"Added hostgroup '{self.hostgroup}'." - logger.info(e) - data = {'groupid': groupid["groupids"][0], 'name': self.hostgroup} - return data - except ZabbixAPIException as e: - e = f"Couldn't add hostgroup, Zabbix returned {str(e)}." - logger.error(e) - raise SyncExternalError(e) from e - - def updateZabbixHost(self, **kwargs): - """ - Updates Zabbix host with given parameters. - INPUT: Key word arguments for Zabbix host object. - """ - try: - self.zabbix.host.update(hostid=self.zabbix_id, **kwargs) - except ZabbixAPIException as e: - e = f"Zabbix returned the following error: {str(e)}." - logger.error(e) - raise SyncExternalError(e) from e - logger.info(f"Updated host {self.name} with data {kwargs}.") - self.create_journal_entry("info", "Updated host in Zabbix with latest NB data.") - - def ConsistencyCheck(self, groups, templates, proxies, proxy_power): - # pylint: disable=too-many-branches, too-many-statements - """ - Checks if Zabbix object is still valid with Netbox parameters. - """ - self.getZabbixGroup(groups) - self.zbxTemplatePrepper(templates) - self.setProxy(proxies) - host = self.zabbix.host.get(filter={'hostid': self.zabbix_id}, - selectInterfaces=['type', 'ip', - 'port', 'details', - 'interfaceid'], - selectGroups=["groupid"], - selectParentTemplates=["templateid"]) - if len(host) > 1: - e = (f"Got {len(host)} results for Zabbix hosts " - f"with ID {self.zabbix_id} - hostname {self.name}.") - logger.error(e) - raise SyncInventoryError(e) - if len(host) == 0: - e = (f"No Zabbix host found for {self.name}. " - f"This is likely the result of a deleted Zabbix host " - f"without zeroing the ID field in Netbox.") - logger.error(e) - raise SyncInventoryError(e) - host = host[0] - - if host["host"] == self.name: - logger.debug(f"Device {self.name}: hostname in-sync.") - else: - logger.warning(f"Device {self.name}: hostname OUT of sync. " - f"Received value: {host['host']}") - self.updateZabbixHost(host=self.name) - - # Check if the templates are in-sync - if not self.zbx_template_comparer(host["parentTemplates"]): - logger.warning(f"Device {self.name}: template(s) OUT of sync.") - # Update Zabbix with NB templates and clear any old / lost templates - self.updateZabbixHost(templates_clear=host["parentTemplates"], - templates=self.zbx_templates) - else: - logger.debug(f"Device {self.name}: template(s) in-sync.") - - for group in host["groups"]: - if group["groupid"] == self.group_id: - logger.debug(f"Device {self.name}: hostgroup in-sync.") - break - else: - logger.warning(f"Device {self.name}: hostgroup OUT of sync.") - self.updateZabbixHost(groups={'groupid': self.group_id}) - - if int(host["status"]) == self.zabbix_state: - logger.debug(f"Device {self.name}: status in-sync.") - else: - logger.warning(f"Device {self.name}: status OUT of sync.") - self.updateZabbixHost(status=str(self.zabbix_state)) - - # Check if a proxy has been defined - if self.zbxproxy != "0": - # Check if expected proxyID matches with configured proxy - if (("proxy_hostid" in host and host["proxy_hostid"] == self.zbxproxy) or - ("proxyid" in host and host["proxyid"] == self.zbxproxy)): - logger.debug(f"Device {self.name}: proxy in-sync.") - else: - # Proxy diff, update value - logger.warning(f"Device {self.name}: proxy OUT of sync.") - if version.parse(self.zabbix.api_version()) < version.parse("7.0.0"): - self.updateZabbixHost(proxy_hostid=self.zbxproxy) - else: - self.updateZabbixHost(proxyid=self.zbxproxy) - else: - if (("proxy_hostid" in host and not host["proxy_hostid"] == "0") - or ("proxyid" in host and not host["proxyid"] == "0")): - if proxy_power: - # Variable full_proxy_sync has been enabled - # delete the proxy link in Zabbix - if version.parse(self.zabbix.api_version()) < version.parse("7.0.0"): - self.updateZabbixHost(proxy_hostid=self.zbxproxy) - else: - self.updateZabbixHost(proxyid=self.zbxproxy) - else: - # Instead of deleting the proxy config in zabbix and - # forcing potential data loss, - # an error message is displayed. - logger.error(f"Device {self.name} is configured " - f"with proxy in Zabbix but not in Netbox. The" - " -p flag was ommited: no " - "changes have been made.") - # If only 1 interface has been found - # pylint: disable=too-many-nested-blocks - if len(host['interfaces']) == 1: - updates = {} - # Go through each key / item and check if it matches Zabbix - for key, item in self.setInterfaceDetails()[0].items(): - # Check if Netbox value is found in Zabbix - if key in host["interfaces"][0]: - # If SNMP is used, go through nested dict - # to compare SNMP parameters - if isinstance(item,dict) and key == "details": - for k, i in item.items(): - if k in host["interfaces"][0][key]: - # Set update if values don't match - if host["interfaces"][0][key][k] != str(i): - # If dict has not been created, add it - if key not in updates: - updates[key] = {} - updates[key][k] = str(i) - # If SNMP version has been changed - # break loop and force full SNMP update - if k == "version": - break - # Force full SNMP config update - # when version has changed. - if key in updates: - if "version" in updates[key]: - for k, i in item.items(): - updates[key][k] = str(i) - continue - # Set update if values don't match - if host["interfaces"][0][key] != str(item): - updates[key] = item - if updates: - # If interface updates have been found: push to Zabbix - logger.warning(f"Device {self.name}: Interface OUT of sync.") - if "type" in updates: - # Changing interface type not supported. Raise exception. - e = (f"Device {self.name}: changing interface type to " - f"{str(updates['type'])} is not supported.") - logger.error(e) - raise InterfaceConfigError(e) - # Set interfaceID for Zabbix config - updates["interfaceid"] = host["interfaces"][0]['interfaceid'] - try: - # API call to Zabbix - self.zabbix.hostinterface.update(updates) - e = f"Solved {self.name} interface conflict." - logger.info(e) - self.create_journal_entry("info", e) - except ZabbixAPIException as e: - e = f"Zabbix returned the following error: {str(e)}." - logger.error(e) - raise SyncExternalError(e) from e - else: - # If no updates are found, Zabbix interface is in-sync - e = f"Device {self.name}: interface in-sync." - logger.debug(e) - else: - e = (f"Device {self.name} has unsupported interface configuration." - f" Host has total of {len(host['interfaces'])} interfaces. " - "Manual interfention required.") - logger.error(e) - raise SyncInventoryError(e) - - def create_journal_entry(self, severity, message): - """ - Send a new Journal entry to Netbox. Usefull for viewing actions - in Netbox without having to look in Zabbix or the script log output - """ - if self.journal: - # Check if the severity is valid - if severity not in ["info", "success", "warning", "danger"]: - logger.warning(f"Value {severity} not valid for NB journal entries.") - return False - journal = {"assigned_object_type": "dcim.device", - "assigned_object_id": self.id, - "kind": severity, - "comments": message - } - try: - self.nb_journals.create(journal) - logger.debug(f"Created journal entry in NB for host {self.name}") - return True - except JournalError(e) as e: - logger.warning("Unable to create journal entry for " - f"{self.name}: NB returned {e}") - return False - return False - - def zbx_template_comparer(self, tmpls_from_zabbix): - """ - Compares the Netbox and Zabbix templates with each other. - Should there be a mismatch then the function will return false - - INPUT: list of NB and ZBX templates - OUTPUT: Boolean True/False - """ - succesfull_templates = [] - # Go through each Netbox template - for nb_tmpl in self.zbx_templates: - # Go through each Zabbix template - for pos, zbx_tmpl in enumerate(tmpls_from_zabbix): - # Check if template IDs match - if nb_tmpl["templateid"] == zbx_tmpl["templateid"]: - # Templates match. Remove this template from the Zabbix templates - # and add this NB template to the list of successfull templates - tmpls_from_zabbix.pop(pos) - succesfull_templates.append(nb_tmpl) - logger.debug(f"Device {self.name}: template " - f"{nb_tmpl['name']} is present in Zabbix.") - break - if len(succesfull_templates) == len(self.zbx_templates) and len(tmpls_from_zabbix) == 0: - # All of the Netbox templates have been confirmed as successfull - # and the ZBX template list is empty. This means that - # all of the templates match. - return True - return False - - -class ZabbixInterface(): - """Class that represents a Zabbix interface.""" - - def __init__(self, context, ip): - self.context = context - self.ip = ip - self.skelet = {"main": "1", "useip": "1", "dns": "", "ip": self.ip} - self.interface = self.skelet - - def get_context(self): - """ check if Netbox custom context has been defined. """ - if "zabbix" in self.context: - zabbix = self.context["zabbix"] - if("interface_type" in zabbix and "interface_port" in zabbix): - self.interface["type"] = zabbix["interface_type"] - self.interface["port"] = zabbix["interface_port"] - return True - return False - return False - - def set_snmp(self): - """ Check if interface is type SNMP """ - # pylint: disable=too-many-branches - if self.interface["type"] == 2: - # Checks if SNMP settings are defined in Netbox - if "snmp" in self.context["zabbix"]: - snmp = self.context["zabbix"]["snmp"] - self.interface["details"] = {} - # Checks if bulk config has been defined - if "bulk" in snmp: - self.interface["details"]["bulk"] = str(snmp.pop("bulk")) - else: - # Fallback to bulk enabled if not specified - self.interface["details"]["bulk"] = "1" - # SNMP Version config is required in Netbox config context - if snmp.get("version"): - self.interface["details"]["version"] = str(snmp.pop("version")) - else: - e = "SNMP version option is not defined." - raise InterfaceConfigError(e) - # If version 1 or 2 is used, get community string - if self.interface["details"]["version"] in ['1','2']: - if "community" in snmp: - # Set SNMP community to confix context value - community = snmp["community"] - else: - # Set SNMP community to default - community = "{$SNMP_COMMUNITY}" - self.interface["details"]["community"] = str(community) - # If version 3 has been used, get all - # SNMPv3 Netbox related configs - elif self.interface["details"]["version"] == '3': - items = ["securityname", "securitylevel", "authpassphrase", - "privpassphrase", "authprotocol", "privprotocol", - "contextname"] - for key, item in snmp.items(): - if key in items: - self.interface["details"][key] = str(item) - else: - e = "Unsupported SNMP version." - raise InterfaceConfigError(e) - else: - e = "Interface type SNMP but no parameters provided." - raise InterfaceConfigError(e) - else: - e = "Interface type is not SNMP, unable to set SNMP details" - raise InterfaceConfigError(e) - - def set_default(self): - """ Set default config to SNMPv2, port 161 and community macro. """ - self.interface = self.skelet - self.interface["type"] = "2" - self.interface["port"] = "161" - self.interface["details"] = {"version": "2", - "community": "{$SNMP_COMMUNITY}", - "bulk": "1"} - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description='A script to sync Zabbix with Netbox device data.' - ) - parser.add_argument("-v", "--verbose", help="Turn on debugging.", - action="store_true") - args = parser.parse_args() - main(args) +#!/usr/bin/env python3 +# pylint: disable=invalid-name, logging-not-lazy, too-many-locals, logging-fstring-interpolation + + +"""Netbox to Zabbix sync script.""" + +import logging +import argparse +from os import environ, path, sys +from packaging import version +from pynetbox import api +from pyzabbix import ZabbixAPI, ZabbixAPIException +try: + from config import ( + templates_config_context, + templates_config_context_overrule, + clustering, create_hostgroups, + create_journal, full_proxy_sync, + template_cf, device_cf, + zabbix_device_removal, + zabbix_device_disable, + hostgroup_format, + traverse_site_groups, + traverse_regions, + nb_device_filter + ) +except ModuleNotFoundError: + print("Configuration file config.py not found in main directory." + "Please create the file or rename the config.py.example file to config.py.") + sys.exit(0) + +# Set logging +log_format = logging.Formatter('%(asctime)s - %(name)s - ' + '%(levelname)s - %(message)s') +lgout = logging.StreamHandler() +lgout.setFormatter(log_format) +lgout.setLevel(logging.DEBUG) + +lgfile = logging.FileHandler(path.join(path.dirname( + path.realpath(__file__)), "sync.log")) +lgfile.setFormatter(log_format) +lgfile.setLevel(logging.DEBUG) + +logger = logging.getLogger("Netbox-Zabbix-sync") +logger.addHandler(lgout) +logger.addHandler(lgfile) +logger.setLevel(logging.WARNING) + + +def convert_recordset(recordset): + """ Converts netbox RedcordSet to list of dicts. """ + recordlist = [] + for record in recordset: + recordlist.append(record.__dict__) + return recordlist + +def build_path(endpoint, list_of_dicts): + """ + Builds a path list of related parent/child items. + This can be used to generate a joinable list to + be used in hostgroups. + """ + path = [] + itemlist = [i for i in list_of_dicts if i['name'] == endpoint] + item = itemlist[0] if len(itemlist) == 1 else None + path.append(item['name']) + while item['_depth'] > 0: + itemlist = [i for i in list_of_dicts if i['name'] == str(item['parent'])] + item = itemlist[0] if len(itemlist) == 1 else None + path.append(item['name']) + path.reverse() + return(path) + +def main(arguments): + """Run the sync process.""" + # pylint: disable=too-many-branches, too-many-statements + # set environment variables + if arguments.verbose: + logger.setLevel(logging.DEBUG) + env_vars = ["ZABBIX_HOST", "NETBOX_HOST", "NETBOX_TOKEN"] + if "ZABBIX_TOKEN" in environ: + env_vars.append("ZABBIX_TOKEN") + else: + env_vars.append("ZABBIX_USER") + env_vars.append("ZABBIX_PASS") + for var in env_vars: + if var not in environ: + e = f"Environment variable {var} has not been defined." + logger.error(e) + raise EnvironmentVarError(e) + # Get all virtual environment variables + if "ZABBIX_TOKEN" in env_vars: + zabbix_user = None + zabbix_pass = None + zabbix_token = environ.get("ZABBIX_TOKEN") + else: + zabbix_user = environ.get("ZABBIX_USER") + zabbix_pass = environ.get("ZABBIX_PASS") + zabbix_token = None + zabbix_host = environ.get("ZABBIX_HOST") + netbox_host = environ.get("NETBOX_HOST") + netbox_token = environ.get("NETBOX_TOKEN") + # Set Netbox API + netbox = api(netbox_host, token=netbox_token, threading=True) + # Check if the provided Hostgroup layout is valid + hg_objects = hostgroup_format.split("/") + allowed_objects = ["dev_location", "dev_role", "manufacturer", "region", + "site", "site_group", "tenant", "tenant_group"] + # Create API call to get all custom fields which are on the device objects + device_cfs = netbox.extras.custom_fields.filter(type="text", content_type_id=23) + for cf in device_cfs: + allowed_objects.append(cf.name) + for hg_object in hg_objects: + if hg_object not in allowed_objects: + e = (f"Hostgroup item {hg_object} is not valid. Make sure you" + " use valid items and seperate them with '/'.") + logger.error(e) + raise HostgroupError(e) + # Set Zabbix API + try: + zabbix = ZabbixAPI(zabbix_host) + if "ZABBIX_TOKEN" in env_vars: + zabbix.login(api_token=zabbix_token) + else: + m=("Logging in with Zabbix user and password," + " consider using an API token instead.") + logger.warning(m) + zabbix.login(zabbix_user, zabbix_pass) + except ZabbixAPIException as e: + e = f"Zabbix returned the following error: {str(e)}." + logger.error(e) + # Set API parameter mapping based on API version + if version.parse(zabbix.api_version()) < version.parse("7.0.0"): + proxy_name = "host" + else: + proxy_name = "name" + # Get all Zabbix and Netbox data + netbox_devices = netbox.dcim.devices.filter(**nb_device_filter) + netbox_site_groups = convert_recordset((netbox.dcim.site_groups.all())) + netbox_regions = convert_recordset(netbox.dcim.regions.all()) + netbox_journals = netbox.extras.journal_entries + zabbix_groups = zabbix.hostgroup.get(output=['groupid', 'name']) + zabbix_templates = zabbix.template.get(output=['templateid', 'name']) + zabbix_proxies = zabbix.proxy.get(output=['proxyid', proxy_name]) + + # Sanitize data + if proxy_name == "host": + for proxy in zabbix_proxies: + proxy['name'] = proxy.pop('host') + + # Go through all Netbox devices + for nb_device in netbox_devices: + try: + device = NetworkDevice(nb_device, zabbix, netbox_journals, + create_journal) + device.set_hostgroup(hostgroup_format,netbox_site_groups,netbox_regions) + device.set_template(templates_config_context, templates_config_context_overrule) + # Checks if device is part of cluster. + # Requires clustering variable + if device.isCluster() and clustering: + # Check if device is master or slave + if device.promoteMasterDevice(): + e = (f"Device {device.name} is " + f"part of cluster and primary.") + logger.info(e) + else: + # Device is secondary in cluster. + # Don't continue with this device. + e = (f"Device {device.name} is part of cluster " + f"but not primary. Skipping this host...") + logger.info(e) + continue + # Checks if device is in cleanup state + if device.status in zabbix_device_removal: + if device.zabbix_id: + # Delete device from Zabbix + # and remove hostID from Netbox. + device.cleanup() + logger.info(f"Cleaned up host {device.name}.") + + else: + # Device has been added to Netbox + # but is not in Activate state + logger.info(f"Skipping host {device.name} since its " + f"not in the active state.") + elif device.status in zabbix_device_disable: + device.zabbix_state = 1 + else: + device.zabbix_state = 0 + # Add hostgroup is variable is True + # and Hostgroup is not present in Zabbix + if create_hostgroups: + for group in zabbix_groups: + # If hostgroup is already present in Zabbix + if group["name"] == device.hostgroup: + break + else: + # Create new hostgroup + hostgroup = device.createZabbixHostgroup() + zabbix_groups.append(hostgroup) + # Device is already present in Zabbix + if device.zabbix_id: + device.ConsistencyCheck(zabbix_groups, zabbix_templates, + zabbix_proxies, full_proxy_sync) + # Add device to Zabbix + else: + device.createInZabbix(zabbix_groups, zabbix_templates, + zabbix_proxies) + except SyncError: + pass + + +class SyncError(Exception): + """ Class SyncError """ + +class JournalError(Exception): + """ Class SyncError """ + +class SyncExternalError(SyncError): + """ Class SyncExternalError """ + +class SyncInventoryError(SyncError): + """ Class SyncInventoryError """ + +class SyncDuplicateError(SyncError): + """ Class SyncDuplicateError """ + +class EnvironmentVarError(SyncError): + """ Class EnvironmentVarError """ + +class InterfaceConfigError(SyncError): + """ Class InterfaceConfigError """ + +class ProxyConfigError(SyncError): + """ Class ProxyConfigError """ + +class HostgroupError(SyncError): + """ Class HostgroupError """ + +class TemplateError(SyncError): + """ Class TemplateError """ + +class NetworkDevice(): + # pylint: disable=too-many-instance-attributes + """ + Represents Network device. + INPUT: (Netbox device class, ZabbixAPI class, journal flag, NB journal class) + """ + + def __init__(self, nb, zabbix, nb_journal_class, journal=None): + self.nb = nb + self.id = nb.id + self.name = nb.name + self.status = nb.status.label + self.zabbix = zabbix + self.zabbix_id = None + self.group_id = None + self.zbx_template_names = [] + self.zbx_templates = [] + self.hostgroup = None + self.tenant = nb.tenant + self.config_context = nb.config_context + self.zbxproxy = "0" + self.zabbix_state = 0 + self.journal = journal + self.nb_journals = nb_journal_class + self._setBasics() + + def _setBasics(self): + """ + Sets basic information like IP address. + """ + # Return error if device does not have primary IP. + if self.nb.primary_ip: + self.cidr = self.nb.primary_ip.address + self.ip = self.cidr.split("/")[0] + else: + e = f"Device {self.name}: no primary IP." + logger.warning(e) + raise SyncInventoryError(e) + + # Check if device has custom field for ZBX ID + if device_cf in self.nb.custom_fields: + self.zabbix_id = self.nb.custom_fields[device_cf] + else: + e = f"Custom field {device_cf} not found for {self.name}." + logger.warning(e) + raise SyncInventoryError(e) + + def set_hostgroup(self, hg_format, nb_site_groups, nb_regions): + """Set the hostgroup for this device""" + # Get all variables from the NB data + dev_location = str(self.nb.location) if self.nb.location else None + dev_role = self.nb.device_role.name + manufacturer = self.nb.device_type.manufacturer.name + region = str(self.nb.site.region) if self.nb.site.region else None + site = self.nb.site.name + site_group = str(self.nb.site.group) if self.nb.site.group else None + tenant = str(self.tenant) if self.tenant else None + tenant_group = str(self.tenant.group) if tenant else None + # Set mapper for string -> variable + hostgroup_vars = {"dev_location": dev_location, "dev_role": dev_role, + "manufacturer": manufacturer, "region": region, + "site": site, "site_group": site_group, + "tenant": tenant, "tenant_group": tenant_group} + # Generate list based off string input format + hg_items = hg_format.split("/") + hostgroup = "" + # Go through all hostgroup items + for item in hg_items: + # Check if the variable (such as Tenant) is empty. + if not hostgroup_vars[item]: + continue + # Check if the item is a custom field name + if item not in hostgroup_vars: + cf_value = self.nb.custom_fields[item] if item in self.nb.custom_fields else None + if cf_value: + # If there is a cf match, add the value of this cf to the hostgroup + hostgroup += cf_value + "/" + # Should there not be a match, this means that + # the variable is invalid. Skip regardless. + continue + # Add value of predefined variable to hostgroup format + if item == "site_group" and nb_site_groups and traverse_site_groups: + path = build_path(site_group, nb_site_groups) + hostgroup += "/".join(path) + "/" + elif item == "region" and nb_regions and traverse_regions: + path = build_path(region, nb_regions) + hostgroup += "/".join(path) + "/" + else: + hostgroup += hostgroup_vars[item] + "/" + # If the final hostgroup variable is empty + if not hostgroup: + e = (f"{self.name} has no reliable hostgroup. This is" + "most likely due to the use of custom fields that are empty.") + logger.error(e) + raise SyncInventoryError(e) + # Remove final inserted "/" and set hostgroup to class var + self.hostgroup = hostgroup.rstrip("/") + + def set_template(self, prefer_config_context, overrule_custom): + """ Set Template """ + self.zbx_template_names = None + # Gather templates ONLY from the device specific context + if prefer_config_context: + try: + self.zbx_template_names = self.get_templates_context() + except TemplateError as e: + logger.warning(e) + return True + # Gather templates from the custom field but overrule + # them should there be any device specific templates + if overrule_custom: + try: + self.zbx_template_names = self.get_templates_context() + except TemplateError: + pass + if not self.zbx_template_names: + self.zbx_template_names = self.get_templates_cf() + return True + # Gather templates ONLY from the custom field + self.zbx_template_names = self.get_templates_cf() + return True + + def get_templates_cf(self): + """ Get template from custom field """ + # Get Zabbix templates from the device type + device_type_cfs = self.nb.device_type.custom_fields + # Check if the ZBX Template CF is present + if template_cf in device_type_cfs: + # Set value to template + return [device_type_cfs[template_cf]] + # Custom field not found, return error + e = (f"Custom field {template_cf} not " + f"found for {self.nb.device_type.manufacturer.name}" + f" - {self.nb.device_type.display}.") + raise TemplateError(e) from e + + def get_templates_context(self): + """ Get Zabbix templates from the device context """ + if "zabbix" not in self.config_context: + e = ("Key 'zabbix' not found in config " + f"context for template host {self.name}") + raise TemplateError(e) + if "templates" not in self.config_context["zabbix"]: + e = ("Key 'zabbix' not found in config " + f"context for template host {self.name}") + raise TemplateError(e) + return self.config_context["zabbix"]["templates"] + + def isCluster(self): + """ + Checks if device is part of cluster. + """ + return bool(self.nb.virtual_chassis) + + def getClusterMaster(self): + """ + Returns chassis master ID. + """ + if not self.isCluster(): + e = (f"Unable to proces {self.name} for cluster calculation: " + f"not part of a cluster.") + logger.warning(e) + raise SyncInventoryError(e) + if not self.nb.virtual_chassis.master: + e = (f"{self.name} is part of a Netbox virtual chassis which does " + "not have a master configured. Skipping for this reason.") + logger.error(e) + raise SyncInventoryError(e) + return self.nb.virtual_chassis.master.id + + def promoteMasterDevice(self): + """ + If device is Primary in cluster, + promote device name to the cluster name. + Returns True if succesfull, returns False if device is secondary. + """ + masterid = self.getClusterMaster() + if masterid == self.id: + logger.debug(f"Device {self.name} is primary cluster member. " + f"Modifying hostname from {self.name} to " + + f"{self.nb.virtual_chassis.name}.") + self.name = self.nb.virtual_chassis.name + return True + logger.debug(f"Device {self.name} is non-primary cluster member.") + return False + + def zbxTemplatePrepper(self, templates): + """ + Returns Zabbix template IDs + INPUT: list of templates from Zabbix + OUTPUT: True + """ + # Check if there are templates defined + if not self.zbx_template_names: + e = f"No templates found for device {self.name}" + logger.info(e) + raise SyncInventoryError() + # Set variable to empty list + self.zbx_templates = [] + # Go through all templates definded in Netbox + for nb_template in self.zbx_template_names: + template_match = False + # Go through all templates found in Zabbix + for zbx_template in templates: + # If the template names match + if zbx_template['name'] == nb_template: + # Set match variable to true, add template details + # to class variable and return debug log + template_match = True + self.zbx_templates.append({"templateid": zbx_template['templateid'], + "name": zbx_template['name']}) + e = (f"Found template {zbx_template['name']}" + f" for host {self.name}.") + logger.debug(e) + # Return error should the template not be found in Zabbix + if not template_match: + e = (f"Unable to find template {nb_template} " + f"for host {self.name} in Zabbix. Skipping host...") + logger.warning(e) + raise SyncInventoryError(e) from e + + def getZabbixGroup(self, groups): + """ + Returns Zabbix group ID + INPUT: list of hostgroups + OUTPUT: True / False + """ + # Go through all groups + for group in groups: + if group['name'] == self.hostgroup: + self.group_id = group['groupid'] + e = f"Found group {group['name']} for host {self.name}." + logger.debug(e) + return True + e = (f"Unable to find group '{self.hostgroup}' " + f"for host {self.name} in Zabbix.") + logger.warning(e) + raise SyncInventoryError(e) + + def cleanup(self): + """ + Removes device from external resources. + Resets custom fields in Netbox. + """ + if self.zabbix_id: + try: + self.zabbix.host.delete(self.zabbix_id) + self.nb.custom_fields[device_cf] = None + self.nb.save() + e = f"Deleted host {self.name} from Zabbix." + logger.info(e) + self.create_journal_entry("warning", "Deleted host from Zabbix") + except ZabbixAPIException as e: + e = f"Zabbix returned the following error: {str(e)}." + logger.error(e) + raise SyncExternalError(e) from e + + def _zabbixHostnameExists(self): + """ + Checks if hostname exists in Zabbix. + """ + host = self.zabbix.host.get(filter={'name': self.name}, output=[]) + return bool(host) + + def setInterfaceDetails(self): + """ + Checks interface parameters from Netbox and + creates a model for the interface to be used in Zabbix. + """ + try: + # Initiate interface class + interface = ZabbixInterface(self.nb.config_context, self.ip) + # Check if Netbox has device context. + # If not fall back to old config. + if interface.get_context(): + # If device is SNMP type, add aditional information. + if interface.interface["type"] == 2: + interface.set_snmp() + else: + interface.set_default() + return [interface.interface] + except InterfaceConfigError as e: + e = f"{self.name}: {e}" + logger.warning(e) + raise SyncInventoryError(e) from e + + def setProxy(self, proxy_list): + """ check if Zabbix Proxy has been defined in config context """ + if "zabbix" in self.nb.config_context: + if "proxy" in self.nb.config_context["zabbix"]: + proxy = self.nb.config_context["zabbix"]["proxy"] + # Try matching proxy + for px in proxy_list: + if px["name"] == proxy: + self.zbxproxy = px["proxyid"] + logger.debug(f"Found proxy {proxy}" + f" for {self.name}.") + return True + return False + e = f"{self.name}: Defined proxy {proxy} not found." + logger.warning(e) + return False + return False + + def createInZabbix(self, groups, templates, proxies, + description="Host added by Netbox sync script."): + """ + Creates Zabbix host object with parameters from Netbox object. + """ + # Check if hostname is already present in Zabbix + if not self._zabbixHostnameExists(): + # Get group and template ID's for host + if not self.getZabbixGroup(groups): + raise SyncInventoryError() + self.zbxTemplatePrepper(templates) + templateids = [] + for template in self.zbx_templates: + templateids.append({'templateid': template['templateid']}) + # Set interface, group and template configuration + interfaces = self.setInterfaceDetails() + groups = [{"groupid": self.group_id}] + # Set Zabbix proxy if defined + self.setProxy(proxies) + # Add host to Zabbix + try: + if version.parse(self.zabbix.api_version()) < version.parse("7.0.0"): + host = self.zabbix.host.create(host=self.name, + status=self.zabbix_state, + interfaces=interfaces, + groups=groups, + templates=templateids, + proxy_hostid=self.zbxproxy, + description=description) + else: + host = self.zabbix.host.create(host=self.name, + status=self.zabbix_state, + interfaces=interfaces, + groups=groups, + templates=templateids, + proxyid=self.zbxproxy, + description=description) + self.zabbix_id = host["hostids"][0] + except ZabbixAPIException as e: + e = f"Couldn't add {self.name}, Zabbix returned {str(e)}." + logger.error(e) + raise SyncExternalError(e) from e + # Set Netbox custom field to hostID value. + self.nb.custom_fields[device_cf] = int(self.zabbix_id) + self.nb.save() + msg = f"Created host {self.name} in Zabbix." + logger.info(msg) + self.create_journal_entry("success", msg) + else: + e = f"Unable to add {self.name} to Zabbix: host already present." + logger.warning(e) + + def createZabbixHostgroup(self): + """ + Creates Zabbix host group based on hostgroup format. + """ + try: + groupid = self.zabbix.hostgroup.create(name=self.hostgroup) + e = f"Added hostgroup '{self.hostgroup}'." + logger.info(e) + data = {'groupid': groupid["groupids"][0], 'name': self.hostgroup} + return data + except ZabbixAPIException as e: + e = f"Couldn't add hostgroup, Zabbix returned {str(e)}." + logger.error(e) + raise SyncExternalError(e) from e + + def updateZabbixHost(self, **kwargs): + """ + Updates Zabbix host with given parameters. + INPUT: Key word arguments for Zabbix host object. + """ + try: + self.zabbix.host.update(hostid=self.zabbix_id, **kwargs) + except ZabbixAPIException as e: + e = f"Zabbix returned the following error: {str(e)}." + logger.error(e) + raise SyncExternalError(e) from e + logger.info(f"Updated host {self.name} with data {kwargs}.") + self.create_journal_entry("info", "Updated host in Zabbix with latest NB data.") + + def ConsistencyCheck(self, groups, templates, proxies, proxy_power): + # pylint: disable=too-many-branches, too-many-statements + """ + Checks if Zabbix object is still valid with Netbox parameters. + """ + self.getZabbixGroup(groups) + self.zbxTemplatePrepper(templates) + self.setProxy(proxies) + host = self.zabbix.host.get(filter={'hostid': self.zabbix_id}, + selectInterfaces=['type', 'ip', + 'port', 'details', + 'interfaceid'], + selectGroups=["groupid"], + selectParentTemplates=["templateid"]) + if len(host) > 1: + e = (f"Got {len(host)} results for Zabbix hosts " + f"with ID {self.zabbix_id} - hostname {self.name}.") + logger.error(e) + raise SyncInventoryError(e) + if len(host) == 0: + e = (f"No Zabbix host found for {self.name}. " + f"This is likely the result of a deleted Zabbix host " + f"without zeroing the ID field in Netbox.") + logger.error(e) + raise SyncInventoryError(e) + host = host[0] + + if host["host"] == self.name: + logger.debug(f"Device {self.name}: hostname in-sync.") + else: + logger.warning(f"Device {self.name}: hostname OUT of sync. " + f"Received value: {host['host']}") + self.updateZabbixHost(host=self.name) + + # Check if the templates are in-sync + if not self.zbx_template_comparer(host["parentTemplates"]): + logger.warning(f"Device {self.name}: template(s) OUT of sync.") + # Update Zabbix with NB templates and clear any old / lost templates + self.updateZabbixHost(templates_clear=host["parentTemplates"], + templates=self.zbx_templates) + else: + logger.debug(f"Device {self.name}: template(s) in-sync.") + + for group in host["groups"]: + if group["groupid"] == self.group_id: + logger.debug(f"Device {self.name}: hostgroup in-sync.") + break + else: + logger.warning(f"Device {self.name}: hostgroup OUT of sync.") + self.updateZabbixHost(groups={'groupid': self.group_id}) + + if int(host["status"]) == self.zabbix_state: + logger.debug(f"Device {self.name}: status in-sync.") + else: + logger.warning(f"Device {self.name}: status OUT of sync.") + self.updateZabbixHost(status=str(self.zabbix_state)) + + # Check if a proxy has been defined + if self.zbxproxy != "0": + # Check if expected proxyID matches with configured proxy + if (("proxy_hostid" in host and host["proxy_hostid"] == self.zbxproxy) or + ("proxyid" in host and host["proxyid"] == self.zbxproxy)): + logger.debug(f"Device {self.name}: proxy in-sync.") + else: + # Proxy diff, update value + logger.warning(f"Device {self.name}: proxy OUT of sync.") + if version.parse(self.zabbix.api_version()) < version.parse("7.0.0"): + self.updateZabbixHost(proxy_hostid=self.zbxproxy) + else: + self.updateZabbixHost(proxyid=self.zbxproxy) + else: + if (("proxy_hostid" in host and not host["proxy_hostid"] == "0") + or ("proxyid" in host and not host["proxyid"] == "0")): + if proxy_power: + # Variable full_proxy_sync has been enabled + # delete the proxy link in Zabbix + if version.parse(self.zabbix.api_version()) < version.parse("7.0.0"): + self.updateZabbixHost(proxy_hostid=self.zbxproxy) + else: + self.updateZabbixHost(proxyid=self.zbxproxy) + else: + # Instead of deleting the proxy config in zabbix and + # forcing potential data loss, + # an error message is displayed. + logger.error(f"Device {self.name} is configured " + f"with proxy in Zabbix but not in Netbox. The" + " -p flag was ommited: no " + "changes have been made.") + # If only 1 interface has been found + # pylint: disable=too-many-nested-blocks + if len(host['interfaces']) == 1: + updates = {} + # Go through each key / item and check if it matches Zabbix + for key, item in self.setInterfaceDetails()[0].items(): + # Check if Netbox value is found in Zabbix + if key in host["interfaces"][0]: + # If SNMP is used, go through nested dict + # to compare SNMP parameters + if isinstance(item,dict) and key == "details": + for k, i in item.items(): + if k in host["interfaces"][0][key]: + # Set update if values don't match + if host["interfaces"][0][key][k] != str(i): + # If dict has not been created, add it + if key not in updates: + updates[key] = {} + updates[key][k] = str(i) + # If SNMP version has been changed + # break loop and force full SNMP update + if k == "version": + break + # Force full SNMP config update + # when version has changed. + if key in updates: + if "version" in updates[key]: + for k, i in item.items(): + updates[key][k] = str(i) + continue + # Set update if values don't match + if host["interfaces"][0][key] != str(item): + updates[key] = item + if updates: + # If interface updates have been found: push to Zabbix + logger.warning(f"Device {self.name}: Interface OUT of sync.") + if "type" in updates: + # Changing interface type not supported. Raise exception. + e = (f"Device {self.name}: changing interface type to " + f"{str(updates['type'])} is not supported.") + logger.error(e) + raise InterfaceConfigError(e) + # Set interfaceID for Zabbix config + updates["interfaceid"] = host["interfaces"][0]['interfaceid'] + try: + # API call to Zabbix + self.zabbix.hostinterface.update(updates) + e = f"Solved {self.name} interface conflict." + logger.info(e) + self.create_journal_entry("info", e) + except ZabbixAPIException as e: + e = f"Zabbix returned the following error: {str(e)}." + logger.error(e) + raise SyncExternalError(e) from e + else: + # If no updates are found, Zabbix interface is in-sync + e = f"Device {self.name}: interface in-sync." + logger.debug(e) + else: + e = (f"Device {self.name} has unsupported interface configuration." + f" Host has total of {len(host['interfaces'])} interfaces. " + "Manual interfention required.") + logger.error(e) + raise SyncInventoryError(e) + + def create_journal_entry(self, severity, message): + """ + Send a new Journal entry to Netbox. Usefull for viewing actions + in Netbox without having to look in Zabbix or the script log output + """ + if self.journal: + # Check if the severity is valid + if severity not in ["info", "success", "warning", "danger"]: + logger.warning(f"Value {severity} not valid for NB journal entries.") + return False + journal = {"assigned_object_type": "dcim.device", + "assigned_object_id": self.id, + "kind": severity, + "comments": message + } + try: + self.nb_journals.create(journal) + logger.debug(f"Created journal entry in NB for host {self.name}") + return True + except JournalError(e) as e: + logger.warning("Unable to create journal entry for " + f"{self.name}: NB returned {e}") + return False + return False + + def zbx_template_comparer(self, tmpls_from_zabbix): + """ + Compares the Netbox and Zabbix templates with each other. + Should there be a mismatch then the function will return false + + INPUT: list of NB and ZBX templates + OUTPUT: Boolean True/False + """ + succesfull_templates = [] + # Go through each Netbox template + for nb_tmpl in self.zbx_templates: + # Go through each Zabbix template + for pos, zbx_tmpl in enumerate(tmpls_from_zabbix): + # Check if template IDs match + if nb_tmpl["templateid"] == zbx_tmpl["templateid"]: + # Templates match. Remove this template from the Zabbix templates + # and add this NB template to the list of successfull templates + tmpls_from_zabbix.pop(pos) + succesfull_templates.append(nb_tmpl) + logger.debug(f"Device {self.name}: template " + f"{nb_tmpl['name']} is present in Zabbix.") + break + if len(succesfull_templates) == len(self.zbx_templates) and len(tmpls_from_zabbix) == 0: + # All of the Netbox templates have been confirmed as successfull + # and the ZBX template list is empty. This means that + # all of the templates match. + return True + return False + + +class ZabbixInterface(): + """Class that represents a Zabbix interface.""" + + def __init__(self, context, ip): + self.context = context + self.ip = ip + self.skelet = {"main": "1", "useip": "1", "dns": "", "ip": self.ip} + self.interface = self.skelet + + def get_context(self): + """ check if Netbox custom context has been defined. """ + if "zabbix" in self.context: + zabbix = self.context["zabbix"] + if("interface_type" in zabbix and "interface_port" in zabbix): + self.interface["type"] = zabbix["interface_type"] + self.interface["port"] = zabbix["interface_port"] + return True + return False + return False + + def set_snmp(self): + """ Check if interface is type SNMP """ + # pylint: disable=too-many-branches + if self.interface["type"] == 2: + # Checks if SNMP settings are defined in Netbox + if "snmp" in self.context["zabbix"]: + snmp = self.context["zabbix"]["snmp"] + self.interface["details"] = {} + # Checks if bulk config has been defined + if "bulk" in snmp: + self.interface["details"]["bulk"] = str(snmp.pop("bulk")) + else: + # Fallback to bulk enabled if not specified + self.interface["details"]["bulk"] = "1" + # SNMP Version config is required in Netbox config context + if snmp.get("version"): + self.interface["details"]["version"] = str(snmp.pop("version")) + else: + e = "SNMP version option is not defined." + raise InterfaceConfigError(e) + # If version 1 or 2 is used, get community string + if self.interface["details"]["version"] in ['1','2']: + if "community" in snmp: + # Set SNMP community to confix context value + community = snmp["community"] + else: + # Set SNMP community to default + community = "{$SNMP_COMMUNITY}" + self.interface["details"]["community"] = str(community) + # If version 3 has been used, get all + # SNMPv3 Netbox related configs + elif self.interface["details"]["version"] == '3': + items = ["securityname", "securitylevel", "authpassphrase", + "privpassphrase", "authprotocol", "privprotocol", + "contextname"] + for key, item in snmp.items(): + if key in items: + self.interface["details"][key] = str(item) + else: + e = "Unsupported SNMP version." + raise InterfaceConfigError(e) + else: + e = "Interface type SNMP but no parameters provided." + raise InterfaceConfigError(e) + else: + e = "Interface type is not SNMP, unable to set SNMP details" + raise InterfaceConfigError(e) + + def set_default(self): + """ Set default config to SNMPv2, port 161 and community macro. """ + self.interface = self.skelet + self.interface["type"] = "2" + self.interface["port"] = "161" + self.interface["details"] = {"version": "2", + "community": "{$SNMP_COMMUNITY}", + "bulk": "1"} + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description='A script to sync Zabbix with Netbox device data.' + ) + parser.add_argument("-v", "--verbose", help="Turn on debugging.", + action="store_true") + args = parser.parse_args() + main(args) From d6973dc32d3ebf6e979d6bff361ebcb08750b7ce Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Mon, 25 Mar 2024 11:51:12 +0100 Subject: [PATCH 27/39] Corrected MarkDown error --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 120eee9..6e045f8 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ You can specify the value like so, sperated by a "/": ``` hostgroup_format = "tenant/site/dev_location/dev_role" ``` -** Group traversal ** +**Group traversal** The default behaviour for `region` is to only use the directly assigned region in the rendered hostgroup name. However, by setting `traverse_region` to `True` in `config.py` the script will render a full region path of all parent regions for the hostgroup name. `traverse_site_groups` controls the same behaviour for site_groups. From 5defc1a25e2f2034b8cdba0a1dd790e07651f880 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Mon, 25 Mar 2024 11:52:10 +0100 Subject: [PATCH 28/39] Corrected another MarkDown error --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6e045f8..d7f514d 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ The format can be set with the `hostgroup_format` variable. Make sure that the Zabbix user has proper permissions to create hosts. The hostgroups are in a nested format. This means that proper permissions only need to be applied to the site name hostgroup and cascaded to any child hostgroups. -#### layout +#### Layout The default hostgroup layout is "site/manufacturer/device_role". **Variables** @@ -120,11 +120,12 @@ You can specify the value like so, sperated by a "/": hostgroup_format = "tenant/site/dev_location/dev_role" ``` **Group traversal** + The default behaviour for `region` is to only use the directly assigned region in the rendered hostgroup name. However, by setting `traverse_region` to `True` in `config.py` the script will render a full region path of all parent regions for the hostgroup name. `traverse_site_groups` controls the same behaviour for site_groups. -**custom fields** +**Custom fields** You can also use the value of custom fields under the device object. From 537710a4b958b82e096baad2ede8ead54e50b8be Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Mon, 25 Mar 2024 11:56:17 +0100 Subject: [PATCH 29/39] Corrected pylint errors --- netbox_zabbix_sync.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index 9dad31b..3d2e96f 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -60,16 +60,16 @@ def build_path(endpoint, list_of_dicts): This can be used to generate a joinable list to be used in hostgroups. """ - path = [] + item_path = [] itemlist = [i for i in list_of_dicts if i['name'] == endpoint] item = itemlist[0] if len(itemlist) == 1 else None - path.append(item['name']) + item_path.append(item['name']) while item['_depth'] > 0: itemlist = [i for i in list_of_dicts if i['name'] == str(item['parent'])] item = itemlist[0] if len(itemlist) == 1 else None - path.append(item['name']) - path.reverse() - return(path) + item_path.append(item['name']) + item_path.reverse() + return item_path def main(arguments): """Run the sync process.""" From 27a4a5c6ebc195d930224390ece120a4ab1085be Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Mon, 25 Mar 2024 11:57:46 +0100 Subject: [PATCH 30/39] Corrected more pylint errors --- netbox_zabbix_sync.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index 3d2e96f..28b1fe4 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -322,11 +322,11 @@ class NetworkDevice(): continue # Add value of predefined variable to hostgroup format if item == "site_group" and nb_site_groups and traverse_site_groups: - path = build_path(site_group, nb_site_groups) - hostgroup += "/".join(path) + "/" + group_path = build_path(site_group, nb_site_groups) + hostgroup += "/".join(group_path) + "/" elif item == "region" and nb_regions and traverse_regions: - path = build_path(region, nb_regions) - hostgroup += "/".join(path) + "/" + region_path = build_path(region, nb_regions) + hostgroup += "/".join(region_path) + "/" else: hostgroup += hostgroup_vars[item] + "/" # If the final hostgroup variable is empty From 583d845c40e6b8f166749ce3b75102b63e602501 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Wed, 27 Mar 2024 15:22:43 +0100 Subject: [PATCH 31/39] revert because of file formatting issue --- netbox_zabbix_sync.py | 1807 ++++++++++++++++++++--------------------- 1 file changed, 886 insertions(+), 921 deletions(-) diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index 28b1fe4..8cf0812 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -1,921 +1,886 @@ -#!/usr/bin/env python3 -# pylint: disable=invalid-name, logging-not-lazy, too-many-locals, logging-fstring-interpolation - - -"""Netbox to Zabbix sync script.""" - -import logging -import argparse -from os import environ, path, sys -from packaging import version -from pynetbox import api -from pyzabbix import ZabbixAPI, ZabbixAPIException -try: - from config import ( - templates_config_context, - templates_config_context_overrule, - clustering, create_hostgroups, - create_journal, full_proxy_sync, - template_cf, device_cf, - zabbix_device_removal, - zabbix_device_disable, - hostgroup_format, - traverse_site_groups, - traverse_regions, - nb_device_filter - ) -except ModuleNotFoundError: - print("Configuration file config.py not found in main directory." - "Please create the file or rename the config.py.example file to config.py.") - sys.exit(0) - -# Set logging -log_format = logging.Formatter('%(asctime)s - %(name)s - ' - '%(levelname)s - %(message)s') -lgout = logging.StreamHandler() -lgout.setFormatter(log_format) -lgout.setLevel(logging.DEBUG) - -lgfile = logging.FileHandler(path.join(path.dirname( - path.realpath(__file__)), "sync.log")) -lgfile.setFormatter(log_format) -lgfile.setLevel(logging.DEBUG) - -logger = logging.getLogger("Netbox-Zabbix-sync") -logger.addHandler(lgout) -logger.addHandler(lgfile) -logger.setLevel(logging.WARNING) - - -def convert_recordset(recordset): - """ Converts netbox RedcordSet to list of dicts. """ - recordlist = [] - for record in recordset: - recordlist.append(record.__dict__) - return recordlist - -def build_path(endpoint, list_of_dicts): - """ - Builds a path list of related parent/child items. - This can be used to generate a joinable list to - be used in hostgroups. - """ - item_path = [] - itemlist = [i for i in list_of_dicts if i['name'] == endpoint] - item = itemlist[0] if len(itemlist) == 1 else None - item_path.append(item['name']) - while item['_depth'] > 0: - itemlist = [i for i in list_of_dicts if i['name'] == str(item['parent'])] - item = itemlist[0] if len(itemlist) == 1 else None - item_path.append(item['name']) - item_path.reverse() - return item_path - -def main(arguments): - """Run the sync process.""" - # pylint: disable=too-many-branches, too-many-statements - # set environment variables - if arguments.verbose: - logger.setLevel(logging.DEBUG) - env_vars = ["ZABBIX_HOST", "NETBOX_HOST", "NETBOX_TOKEN"] - if "ZABBIX_TOKEN" in environ: - env_vars.append("ZABBIX_TOKEN") - else: - env_vars.append("ZABBIX_USER") - env_vars.append("ZABBIX_PASS") - for var in env_vars: - if var not in environ: - e = f"Environment variable {var} has not been defined." - logger.error(e) - raise EnvironmentVarError(e) - # Get all virtual environment variables - if "ZABBIX_TOKEN" in env_vars: - zabbix_user = None - zabbix_pass = None - zabbix_token = environ.get("ZABBIX_TOKEN") - else: - zabbix_user = environ.get("ZABBIX_USER") - zabbix_pass = environ.get("ZABBIX_PASS") - zabbix_token = None - zabbix_host = environ.get("ZABBIX_HOST") - netbox_host = environ.get("NETBOX_HOST") - netbox_token = environ.get("NETBOX_TOKEN") - # Set Netbox API - netbox = api(netbox_host, token=netbox_token, threading=True) - # Check if the provided Hostgroup layout is valid - hg_objects = hostgroup_format.split("/") - allowed_objects = ["dev_location", "dev_role", "manufacturer", "region", - "site", "site_group", "tenant", "tenant_group"] - # Create API call to get all custom fields which are on the device objects - device_cfs = netbox.extras.custom_fields.filter(type="text", content_type_id=23) - for cf in device_cfs: - allowed_objects.append(cf.name) - for hg_object in hg_objects: - if hg_object not in allowed_objects: - e = (f"Hostgroup item {hg_object} is not valid. Make sure you" - " use valid items and seperate them with '/'.") - logger.error(e) - raise HostgroupError(e) - # Set Zabbix API - try: - zabbix = ZabbixAPI(zabbix_host) - if "ZABBIX_TOKEN" in env_vars: - zabbix.login(api_token=zabbix_token) - else: - m=("Logging in with Zabbix user and password," - " consider using an API token instead.") - logger.warning(m) - zabbix.login(zabbix_user, zabbix_pass) - except ZabbixAPIException as e: - e = f"Zabbix returned the following error: {str(e)}." - logger.error(e) - # Set API parameter mapping based on API version - if version.parse(zabbix.api_version()) < version.parse("7.0.0"): - proxy_name = "host" - else: - proxy_name = "name" - # Get all Zabbix and Netbox data - netbox_devices = netbox.dcim.devices.filter(**nb_device_filter) - netbox_site_groups = convert_recordset((netbox.dcim.site_groups.all())) - netbox_regions = convert_recordset(netbox.dcim.regions.all()) - netbox_journals = netbox.extras.journal_entries - zabbix_groups = zabbix.hostgroup.get(output=['groupid', 'name']) - zabbix_templates = zabbix.template.get(output=['templateid', 'name']) - zabbix_proxies = zabbix.proxy.get(output=['proxyid', proxy_name]) - - # Sanitize data - if proxy_name == "host": - for proxy in zabbix_proxies: - proxy['name'] = proxy.pop('host') - - # Go through all Netbox devices - for nb_device in netbox_devices: - try: - device = NetworkDevice(nb_device, zabbix, netbox_journals, - create_journal) - device.set_hostgroup(hostgroup_format,netbox_site_groups,netbox_regions) - device.set_template(templates_config_context, templates_config_context_overrule) - # Checks if device is part of cluster. - # Requires clustering variable - if device.isCluster() and clustering: - # Check if device is master or slave - if device.promoteMasterDevice(): - e = (f"Device {device.name} is " - f"part of cluster and primary.") - logger.info(e) - else: - # Device is secondary in cluster. - # Don't continue with this device. - e = (f"Device {device.name} is part of cluster " - f"but not primary. Skipping this host...") - logger.info(e) - continue - # Checks if device is in cleanup state - if device.status in zabbix_device_removal: - if device.zabbix_id: - # Delete device from Zabbix - # and remove hostID from Netbox. - device.cleanup() - logger.info(f"Cleaned up host {device.name}.") - - else: - # Device has been added to Netbox - # but is not in Activate state - logger.info(f"Skipping host {device.name} since its " - f"not in the active state.") - elif device.status in zabbix_device_disable: - device.zabbix_state = 1 - else: - device.zabbix_state = 0 - # Add hostgroup is variable is True - # and Hostgroup is not present in Zabbix - if create_hostgroups: - for group in zabbix_groups: - # If hostgroup is already present in Zabbix - if group["name"] == device.hostgroup: - break - else: - # Create new hostgroup - hostgroup = device.createZabbixHostgroup() - zabbix_groups.append(hostgroup) - # Device is already present in Zabbix - if device.zabbix_id: - device.ConsistencyCheck(zabbix_groups, zabbix_templates, - zabbix_proxies, full_proxy_sync) - # Add device to Zabbix - else: - device.createInZabbix(zabbix_groups, zabbix_templates, - zabbix_proxies) - except SyncError: - pass - - -class SyncError(Exception): - """ Class SyncError """ - -class JournalError(Exception): - """ Class SyncError """ - -class SyncExternalError(SyncError): - """ Class SyncExternalError """ - -class SyncInventoryError(SyncError): - """ Class SyncInventoryError """ - -class SyncDuplicateError(SyncError): - """ Class SyncDuplicateError """ - -class EnvironmentVarError(SyncError): - """ Class EnvironmentVarError """ - -class InterfaceConfigError(SyncError): - """ Class InterfaceConfigError """ - -class ProxyConfigError(SyncError): - """ Class ProxyConfigError """ - -class HostgroupError(SyncError): - """ Class HostgroupError """ - -class TemplateError(SyncError): - """ Class TemplateError """ - -class NetworkDevice(): - # pylint: disable=too-many-instance-attributes - """ - Represents Network device. - INPUT: (Netbox device class, ZabbixAPI class, journal flag, NB journal class) - """ - - def __init__(self, nb, zabbix, nb_journal_class, journal=None): - self.nb = nb - self.id = nb.id - self.name = nb.name - self.status = nb.status.label - self.zabbix = zabbix - self.zabbix_id = None - self.group_id = None - self.zbx_template_names = [] - self.zbx_templates = [] - self.hostgroup = None - self.tenant = nb.tenant - self.config_context = nb.config_context - self.zbxproxy = "0" - self.zabbix_state = 0 - self.journal = journal - self.nb_journals = nb_journal_class - self._setBasics() - - def _setBasics(self): - """ - Sets basic information like IP address. - """ - # Return error if device does not have primary IP. - if self.nb.primary_ip: - self.cidr = self.nb.primary_ip.address - self.ip = self.cidr.split("/")[0] - else: - e = f"Device {self.name}: no primary IP." - logger.warning(e) - raise SyncInventoryError(e) - - # Check if device has custom field for ZBX ID - if device_cf in self.nb.custom_fields: - self.zabbix_id = self.nb.custom_fields[device_cf] - else: - e = f"Custom field {device_cf} not found for {self.name}." - logger.warning(e) - raise SyncInventoryError(e) - - def set_hostgroup(self, hg_format, nb_site_groups, nb_regions): - """Set the hostgroup for this device""" - # Get all variables from the NB data - dev_location = str(self.nb.location) if self.nb.location else None - dev_role = self.nb.device_role.name - manufacturer = self.nb.device_type.manufacturer.name - region = str(self.nb.site.region) if self.nb.site.region else None - site = self.nb.site.name - site_group = str(self.nb.site.group) if self.nb.site.group else None - tenant = str(self.tenant) if self.tenant else None - tenant_group = str(self.tenant.group) if tenant else None - # Set mapper for string -> variable - hostgroup_vars = {"dev_location": dev_location, "dev_role": dev_role, - "manufacturer": manufacturer, "region": region, - "site": site, "site_group": site_group, - "tenant": tenant, "tenant_group": tenant_group} - # Generate list based off string input format - hg_items = hg_format.split("/") - hostgroup = "" - # Go through all hostgroup items - for item in hg_items: - # Check if the variable (such as Tenant) is empty. - if not hostgroup_vars[item]: - continue - # Check if the item is a custom field name - if item not in hostgroup_vars: - cf_value = self.nb.custom_fields[item] if item in self.nb.custom_fields else None - if cf_value: - # If there is a cf match, add the value of this cf to the hostgroup - hostgroup += cf_value + "/" - # Should there not be a match, this means that - # the variable is invalid. Skip regardless. - continue - # Add value of predefined variable to hostgroup format - if item == "site_group" and nb_site_groups and traverse_site_groups: - group_path = build_path(site_group, nb_site_groups) - hostgroup += "/".join(group_path) + "/" - elif item == "region" and nb_regions and traverse_regions: - region_path = build_path(region, nb_regions) - hostgroup += "/".join(region_path) + "/" - else: - hostgroup += hostgroup_vars[item] + "/" - # If the final hostgroup variable is empty - if not hostgroup: - e = (f"{self.name} has no reliable hostgroup. This is" - "most likely due to the use of custom fields that are empty.") - logger.error(e) - raise SyncInventoryError(e) - # Remove final inserted "/" and set hostgroup to class var - self.hostgroup = hostgroup.rstrip("/") - - def set_template(self, prefer_config_context, overrule_custom): - """ Set Template """ - self.zbx_template_names = None - # Gather templates ONLY from the device specific context - if prefer_config_context: - try: - self.zbx_template_names = self.get_templates_context() - except TemplateError as e: - logger.warning(e) - return True - # Gather templates from the custom field but overrule - # them should there be any device specific templates - if overrule_custom: - try: - self.zbx_template_names = self.get_templates_context() - except TemplateError: - pass - if not self.zbx_template_names: - self.zbx_template_names = self.get_templates_cf() - return True - # Gather templates ONLY from the custom field - self.zbx_template_names = self.get_templates_cf() - return True - - def get_templates_cf(self): - """ Get template from custom field """ - # Get Zabbix templates from the device type - device_type_cfs = self.nb.device_type.custom_fields - # Check if the ZBX Template CF is present - if template_cf in device_type_cfs: - # Set value to template - return [device_type_cfs[template_cf]] - # Custom field not found, return error - e = (f"Custom field {template_cf} not " - f"found for {self.nb.device_type.manufacturer.name}" - f" - {self.nb.device_type.display}.") - raise TemplateError(e) from e - - def get_templates_context(self): - """ Get Zabbix templates from the device context """ - if "zabbix" not in self.config_context: - e = ("Key 'zabbix' not found in config " - f"context for template host {self.name}") - raise TemplateError(e) - if "templates" not in self.config_context["zabbix"]: - e = ("Key 'zabbix' not found in config " - f"context for template host {self.name}") - raise TemplateError(e) - return self.config_context["zabbix"]["templates"] - - def isCluster(self): - """ - Checks if device is part of cluster. - """ - return bool(self.nb.virtual_chassis) - - def getClusterMaster(self): - """ - Returns chassis master ID. - """ - if not self.isCluster(): - e = (f"Unable to proces {self.name} for cluster calculation: " - f"not part of a cluster.") - logger.warning(e) - raise SyncInventoryError(e) - if not self.nb.virtual_chassis.master: - e = (f"{self.name} is part of a Netbox virtual chassis which does " - "not have a master configured. Skipping for this reason.") - logger.error(e) - raise SyncInventoryError(e) - return self.nb.virtual_chassis.master.id - - def promoteMasterDevice(self): - """ - If device is Primary in cluster, - promote device name to the cluster name. - Returns True if succesfull, returns False if device is secondary. - """ - masterid = self.getClusterMaster() - if masterid == self.id: - logger.debug(f"Device {self.name} is primary cluster member. " - f"Modifying hostname from {self.name} to " + - f"{self.nb.virtual_chassis.name}.") - self.name = self.nb.virtual_chassis.name - return True - logger.debug(f"Device {self.name} is non-primary cluster member.") - return False - - def zbxTemplatePrepper(self, templates): - """ - Returns Zabbix template IDs - INPUT: list of templates from Zabbix - OUTPUT: True - """ - # Check if there are templates defined - if not self.zbx_template_names: - e = f"No templates found for device {self.name}" - logger.info(e) - raise SyncInventoryError() - # Set variable to empty list - self.zbx_templates = [] - # Go through all templates definded in Netbox - for nb_template in self.zbx_template_names: - template_match = False - # Go through all templates found in Zabbix - for zbx_template in templates: - # If the template names match - if zbx_template['name'] == nb_template: - # Set match variable to true, add template details - # to class variable and return debug log - template_match = True - self.zbx_templates.append({"templateid": zbx_template['templateid'], - "name": zbx_template['name']}) - e = (f"Found template {zbx_template['name']}" - f" for host {self.name}.") - logger.debug(e) - # Return error should the template not be found in Zabbix - if not template_match: - e = (f"Unable to find template {nb_template} " - f"for host {self.name} in Zabbix. Skipping host...") - logger.warning(e) - raise SyncInventoryError(e) from e - - def getZabbixGroup(self, groups): - """ - Returns Zabbix group ID - INPUT: list of hostgroups - OUTPUT: True / False - """ - # Go through all groups - for group in groups: - if group['name'] == self.hostgroup: - self.group_id = group['groupid'] - e = f"Found group {group['name']} for host {self.name}." - logger.debug(e) - return True - e = (f"Unable to find group '{self.hostgroup}' " - f"for host {self.name} in Zabbix.") - logger.warning(e) - raise SyncInventoryError(e) - - def cleanup(self): - """ - Removes device from external resources. - Resets custom fields in Netbox. - """ - if self.zabbix_id: - try: - self.zabbix.host.delete(self.zabbix_id) - self.nb.custom_fields[device_cf] = None - self.nb.save() - e = f"Deleted host {self.name} from Zabbix." - logger.info(e) - self.create_journal_entry("warning", "Deleted host from Zabbix") - except ZabbixAPIException as e: - e = f"Zabbix returned the following error: {str(e)}." - logger.error(e) - raise SyncExternalError(e) from e - - def _zabbixHostnameExists(self): - """ - Checks if hostname exists in Zabbix. - """ - host = self.zabbix.host.get(filter={'name': self.name}, output=[]) - return bool(host) - - def setInterfaceDetails(self): - """ - Checks interface parameters from Netbox and - creates a model for the interface to be used in Zabbix. - """ - try: - # Initiate interface class - interface = ZabbixInterface(self.nb.config_context, self.ip) - # Check if Netbox has device context. - # If not fall back to old config. - if interface.get_context(): - # If device is SNMP type, add aditional information. - if interface.interface["type"] == 2: - interface.set_snmp() - else: - interface.set_default() - return [interface.interface] - except InterfaceConfigError as e: - e = f"{self.name}: {e}" - logger.warning(e) - raise SyncInventoryError(e) from e - - def setProxy(self, proxy_list): - """ check if Zabbix Proxy has been defined in config context """ - if "zabbix" in self.nb.config_context: - if "proxy" in self.nb.config_context["zabbix"]: - proxy = self.nb.config_context["zabbix"]["proxy"] - # Try matching proxy - for px in proxy_list: - if px["name"] == proxy: - self.zbxproxy = px["proxyid"] - logger.debug(f"Found proxy {proxy}" - f" for {self.name}.") - return True - return False - e = f"{self.name}: Defined proxy {proxy} not found." - logger.warning(e) - return False - return False - - def createInZabbix(self, groups, templates, proxies, - description="Host added by Netbox sync script."): - """ - Creates Zabbix host object with parameters from Netbox object. - """ - # Check if hostname is already present in Zabbix - if not self._zabbixHostnameExists(): - # Get group and template ID's for host - if not self.getZabbixGroup(groups): - raise SyncInventoryError() - self.zbxTemplatePrepper(templates) - templateids = [] - for template in self.zbx_templates: - templateids.append({'templateid': template['templateid']}) - # Set interface, group and template configuration - interfaces = self.setInterfaceDetails() - groups = [{"groupid": self.group_id}] - # Set Zabbix proxy if defined - self.setProxy(proxies) - # Add host to Zabbix - try: - if version.parse(self.zabbix.api_version()) < version.parse("7.0.0"): - host = self.zabbix.host.create(host=self.name, - status=self.zabbix_state, - interfaces=interfaces, - groups=groups, - templates=templateids, - proxy_hostid=self.zbxproxy, - description=description) - else: - host = self.zabbix.host.create(host=self.name, - status=self.zabbix_state, - interfaces=interfaces, - groups=groups, - templates=templateids, - proxyid=self.zbxproxy, - description=description) - self.zabbix_id = host["hostids"][0] - except ZabbixAPIException as e: - e = f"Couldn't add {self.name}, Zabbix returned {str(e)}." - logger.error(e) - raise SyncExternalError(e) from e - # Set Netbox custom field to hostID value. - self.nb.custom_fields[device_cf] = int(self.zabbix_id) - self.nb.save() - msg = f"Created host {self.name} in Zabbix." - logger.info(msg) - self.create_journal_entry("success", msg) - else: - e = f"Unable to add {self.name} to Zabbix: host already present." - logger.warning(e) - - def createZabbixHostgroup(self): - """ - Creates Zabbix host group based on hostgroup format. - """ - try: - groupid = self.zabbix.hostgroup.create(name=self.hostgroup) - e = f"Added hostgroup '{self.hostgroup}'." - logger.info(e) - data = {'groupid': groupid["groupids"][0], 'name': self.hostgroup} - return data - except ZabbixAPIException as e: - e = f"Couldn't add hostgroup, Zabbix returned {str(e)}." - logger.error(e) - raise SyncExternalError(e) from e - - def updateZabbixHost(self, **kwargs): - """ - Updates Zabbix host with given parameters. - INPUT: Key word arguments for Zabbix host object. - """ - try: - self.zabbix.host.update(hostid=self.zabbix_id, **kwargs) - except ZabbixAPIException as e: - e = f"Zabbix returned the following error: {str(e)}." - logger.error(e) - raise SyncExternalError(e) from e - logger.info(f"Updated host {self.name} with data {kwargs}.") - self.create_journal_entry("info", "Updated host in Zabbix with latest NB data.") - - def ConsistencyCheck(self, groups, templates, proxies, proxy_power): - # pylint: disable=too-many-branches, too-many-statements - """ - Checks if Zabbix object is still valid with Netbox parameters. - """ - self.getZabbixGroup(groups) - self.zbxTemplatePrepper(templates) - self.setProxy(proxies) - host = self.zabbix.host.get(filter={'hostid': self.zabbix_id}, - selectInterfaces=['type', 'ip', - 'port', 'details', - 'interfaceid'], - selectGroups=["groupid"], - selectParentTemplates=["templateid"]) - if len(host) > 1: - e = (f"Got {len(host)} results for Zabbix hosts " - f"with ID {self.zabbix_id} - hostname {self.name}.") - logger.error(e) - raise SyncInventoryError(e) - if len(host) == 0: - e = (f"No Zabbix host found for {self.name}. " - f"This is likely the result of a deleted Zabbix host " - f"without zeroing the ID field in Netbox.") - logger.error(e) - raise SyncInventoryError(e) - host = host[0] - - if host["host"] == self.name: - logger.debug(f"Device {self.name}: hostname in-sync.") - else: - logger.warning(f"Device {self.name}: hostname OUT of sync. " - f"Received value: {host['host']}") - self.updateZabbixHost(host=self.name) - - # Check if the templates are in-sync - if not self.zbx_template_comparer(host["parentTemplates"]): - logger.warning(f"Device {self.name}: template(s) OUT of sync.") - # Update Zabbix with NB templates and clear any old / lost templates - self.updateZabbixHost(templates_clear=host["parentTemplates"], - templates=self.zbx_templates) - else: - logger.debug(f"Device {self.name}: template(s) in-sync.") - - for group in host["groups"]: - if group["groupid"] == self.group_id: - logger.debug(f"Device {self.name}: hostgroup in-sync.") - break - else: - logger.warning(f"Device {self.name}: hostgroup OUT of sync.") - self.updateZabbixHost(groups={'groupid': self.group_id}) - - if int(host["status"]) == self.zabbix_state: - logger.debug(f"Device {self.name}: status in-sync.") - else: - logger.warning(f"Device {self.name}: status OUT of sync.") - self.updateZabbixHost(status=str(self.zabbix_state)) - - # Check if a proxy has been defined - if self.zbxproxy != "0": - # Check if expected proxyID matches with configured proxy - if (("proxy_hostid" in host and host["proxy_hostid"] == self.zbxproxy) or - ("proxyid" in host and host["proxyid"] == self.zbxproxy)): - logger.debug(f"Device {self.name}: proxy in-sync.") - else: - # Proxy diff, update value - logger.warning(f"Device {self.name}: proxy OUT of sync.") - if version.parse(self.zabbix.api_version()) < version.parse("7.0.0"): - self.updateZabbixHost(proxy_hostid=self.zbxproxy) - else: - self.updateZabbixHost(proxyid=self.zbxproxy) - else: - if (("proxy_hostid" in host and not host["proxy_hostid"] == "0") - or ("proxyid" in host and not host["proxyid"] == "0")): - if proxy_power: - # Variable full_proxy_sync has been enabled - # delete the proxy link in Zabbix - if version.parse(self.zabbix.api_version()) < version.parse("7.0.0"): - self.updateZabbixHost(proxy_hostid=self.zbxproxy) - else: - self.updateZabbixHost(proxyid=self.zbxproxy) - else: - # Instead of deleting the proxy config in zabbix and - # forcing potential data loss, - # an error message is displayed. - logger.error(f"Device {self.name} is configured " - f"with proxy in Zabbix but not in Netbox. The" - " -p flag was ommited: no " - "changes have been made.") - # If only 1 interface has been found - # pylint: disable=too-many-nested-blocks - if len(host['interfaces']) == 1: - updates = {} - # Go through each key / item and check if it matches Zabbix - for key, item in self.setInterfaceDetails()[0].items(): - # Check if Netbox value is found in Zabbix - if key in host["interfaces"][0]: - # If SNMP is used, go through nested dict - # to compare SNMP parameters - if isinstance(item,dict) and key == "details": - for k, i in item.items(): - if k in host["interfaces"][0][key]: - # Set update if values don't match - if host["interfaces"][0][key][k] != str(i): - # If dict has not been created, add it - if key not in updates: - updates[key] = {} - updates[key][k] = str(i) - # If SNMP version has been changed - # break loop and force full SNMP update - if k == "version": - break - # Force full SNMP config update - # when version has changed. - if key in updates: - if "version" in updates[key]: - for k, i in item.items(): - updates[key][k] = str(i) - continue - # Set update if values don't match - if host["interfaces"][0][key] != str(item): - updates[key] = item - if updates: - # If interface updates have been found: push to Zabbix - logger.warning(f"Device {self.name}: Interface OUT of sync.") - if "type" in updates: - # Changing interface type not supported. Raise exception. - e = (f"Device {self.name}: changing interface type to " - f"{str(updates['type'])} is not supported.") - logger.error(e) - raise InterfaceConfigError(e) - # Set interfaceID for Zabbix config - updates["interfaceid"] = host["interfaces"][0]['interfaceid'] - try: - # API call to Zabbix - self.zabbix.hostinterface.update(updates) - e = f"Solved {self.name} interface conflict." - logger.info(e) - self.create_journal_entry("info", e) - except ZabbixAPIException as e: - e = f"Zabbix returned the following error: {str(e)}." - logger.error(e) - raise SyncExternalError(e) from e - else: - # If no updates are found, Zabbix interface is in-sync - e = f"Device {self.name}: interface in-sync." - logger.debug(e) - else: - e = (f"Device {self.name} has unsupported interface configuration." - f" Host has total of {len(host['interfaces'])} interfaces. " - "Manual interfention required.") - logger.error(e) - raise SyncInventoryError(e) - - def create_journal_entry(self, severity, message): - """ - Send a new Journal entry to Netbox. Usefull for viewing actions - in Netbox without having to look in Zabbix or the script log output - """ - if self.journal: - # Check if the severity is valid - if severity not in ["info", "success", "warning", "danger"]: - logger.warning(f"Value {severity} not valid for NB journal entries.") - return False - journal = {"assigned_object_type": "dcim.device", - "assigned_object_id": self.id, - "kind": severity, - "comments": message - } - try: - self.nb_journals.create(journal) - logger.debug(f"Created journal entry in NB for host {self.name}") - return True - except JournalError(e) as e: - logger.warning("Unable to create journal entry for " - f"{self.name}: NB returned {e}") - return False - return False - - def zbx_template_comparer(self, tmpls_from_zabbix): - """ - Compares the Netbox and Zabbix templates with each other. - Should there be a mismatch then the function will return false - - INPUT: list of NB and ZBX templates - OUTPUT: Boolean True/False - """ - succesfull_templates = [] - # Go through each Netbox template - for nb_tmpl in self.zbx_templates: - # Go through each Zabbix template - for pos, zbx_tmpl in enumerate(tmpls_from_zabbix): - # Check if template IDs match - if nb_tmpl["templateid"] == zbx_tmpl["templateid"]: - # Templates match. Remove this template from the Zabbix templates - # and add this NB template to the list of successfull templates - tmpls_from_zabbix.pop(pos) - succesfull_templates.append(nb_tmpl) - logger.debug(f"Device {self.name}: template " - f"{nb_tmpl['name']} is present in Zabbix.") - break - if len(succesfull_templates) == len(self.zbx_templates) and len(tmpls_from_zabbix) == 0: - # All of the Netbox templates have been confirmed as successfull - # and the ZBX template list is empty. This means that - # all of the templates match. - return True - return False - - -class ZabbixInterface(): - """Class that represents a Zabbix interface.""" - - def __init__(self, context, ip): - self.context = context - self.ip = ip - self.skelet = {"main": "1", "useip": "1", "dns": "", "ip": self.ip} - self.interface = self.skelet - - def get_context(self): - """ check if Netbox custom context has been defined. """ - if "zabbix" in self.context: - zabbix = self.context["zabbix"] - if("interface_type" in zabbix and "interface_port" in zabbix): - self.interface["type"] = zabbix["interface_type"] - self.interface["port"] = zabbix["interface_port"] - return True - return False - return False - - def set_snmp(self): - """ Check if interface is type SNMP """ - # pylint: disable=too-many-branches - if self.interface["type"] == 2: - # Checks if SNMP settings are defined in Netbox - if "snmp" in self.context["zabbix"]: - snmp = self.context["zabbix"]["snmp"] - self.interface["details"] = {} - # Checks if bulk config has been defined - if "bulk" in snmp: - self.interface["details"]["bulk"] = str(snmp.pop("bulk")) - else: - # Fallback to bulk enabled if not specified - self.interface["details"]["bulk"] = "1" - # SNMP Version config is required in Netbox config context - if snmp.get("version"): - self.interface["details"]["version"] = str(snmp.pop("version")) - else: - e = "SNMP version option is not defined." - raise InterfaceConfigError(e) - # If version 1 or 2 is used, get community string - if self.interface["details"]["version"] in ['1','2']: - if "community" in snmp: - # Set SNMP community to confix context value - community = snmp["community"] - else: - # Set SNMP community to default - community = "{$SNMP_COMMUNITY}" - self.interface["details"]["community"] = str(community) - # If version 3 has been used, get all - # SNMPv3 Netbox related configs - elif self.interface["details"]["version"] == '3': - items = ["securityname", "securitylevel", "authpassphrase", - "privpassphrase", "authprotocol", "privprotocol", - "contextname"] - for key, item in snmp.items(): - if key in items: - self.interface["details"][key] = str(item) - else: - e = "Unsupported SNMP version." - raise InterfaceConfigError(e) - else: - e = "Interface type SNMP but no parameters provided." - raise InterfaceConfigError(e) - else: - e = "Interface type is not SNMP, unable to set SNMP details" - raise InterfaceConfigError(e) - - def set_default(self): - """ Set default config to SNMPv2, port 161 and community macro. """ - self.interface = self.skelet - self.interface["type"] = "2" - self.interface["port"] = "161" - self.interface["details"] = {"version": "2", - "community": "{$SNMP_COMMUNITY}", - "bulk": "1"} - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description='A script to sync Zabbix with Netbox device data.' - ) - parser.add_argument("-v", "--verbose", help="Turn on debugging.", - action="store_true") - args = parser.parse_args() - main(args) +#!/usr/bin/env python3 +# pylint: disable=invalid-name, logging-not-lazy, too-many-locals, logging-fstring-interpolation + + +"""Netbox to Zabbix sync script.""" + +import logging +import argparse +from os import environ, path, sys +from packaging import version +from pynetbox import api +from pyzabbix import ZabbixAPI, ZabbixAPIException +try: + from config import ( + templates_config_context, + templates_config_context_overrule, + clustering, create_hostgroups, + create_journal, full_proxy_sync, + template_cf, device_cf, + zabbix_device_removal, + zabbix_device_disable, + hostgroup_format, + nb_device_filter + ) +except ModuleNotFoundError: + print("Configuration file config.py not found in main directory." + "Please create the file or rename the config.py.example file to config.py.") + sys.exit(0) + +# Set logging +log_format = logging.Formatter('%(asctime)s - %(name)s - ' + '%(levelname)s - %(message)s') +lgout = logging.StreamHandler() +lgout.setFormatter(log_format) +lgout.setLevel(logging.DEBUG) + +lgfile = logging.FileHandler(path.join(path.dirname( + path.realpath(__file__)), "sync.log")) +lgfile.setFormatter(log_format) +lgfile.setLevel(logging.DEBUG) + +logger = logging.getLogger("Netbox-Zabbix-sync") +logger.addHandler(lgout) +logger.addHandler(lgfile) +logger.setLevel(logging.WARNING) + + +def main(arguments): + """Run the sync process.""" + # pylint: disable=too-many-branches, too-many-statements + # set environment variables + if arguments.verbose: + logger.setLevel(logging.DEBUG) + env_vars = ["ZABBIX_HOST", "NETBOX_HOST", "NETBOX_TOKEN"] + if "ZABBIX_TOKEN" in environ: + env_vars.append("ZABBIX_TOKEN") + else: + env_vars.append("ZABBIX_USER") + env_vars.append("ZABBIX_PASS") + for var in env_vars: + if var not in environ: + e = f"Environment variable {var} has not been defined." + logger.error(e) + raise EnvironmentVarError(e) + # Get all virtual environment variables + if "ZABBIX_TOKEN" in env_vars: + zabbix_user = None + zabbix_pass = None + zabbix_token = environ.get("ZABBIX_TOKEN") + else: + zabbix_user = environ.get("ZABBIX_USER") + zabbix_pass = environ.get("ZABBIX_PASS") + zabbix_token = None + zabbix_host = environ.get("ZABBIX_HOST") + netbox_host = environ.get("NETBOX_HOST") + netbox_token = environ.get("NETBOX_TOKEN") + # Set Netbox API + netbox = api(netbox_host, token=netbox_token, threading=True) + # Check if the provided Hostgroup layout is valid + hg_objects = hostgroup_format.split("/") + allowed_objects = ["dev_location", "dev_role", "manufacturer", "region", + "site", "site_group", "tenant", "tenant_group"] + # Create API call to get all custom fields which are on the device objects + device_cfs = netbox.extras.custom_fields.filter(type="text", content_type_id=23) + for cf in device_cfs: + allowed_objects.append(cf.name) + for hg_object in hg_objects: + if hg_object not in allowed_objects: + e = (f"Hostgroup item {hg_object} is not valid. Make sure you" + " use valid items and seperate them with '/'.") + logger.error(e) + raise HostgroupError(e) + # Set Zabbix API + try: + zabbix = ZabbixAPI(zabbix_host) + if "ZABBIX_TOKEN" in env_vars: + zabbix.login(api_token=zabbix_token) + else: + m=("Logging in with Zabbix user and password," + " consider using an API token instead.") + logger.warning(m) + zabbix.login(zabbix_user, zabbix_pass) + except ZabbixAPIException as e: + e = f"Zabbix returned the following error: {str(e)}." + logger.error(e) + # Set API parameter mapping based on API version + if version.parse(zabbix.api_version()) < version.parse("7.0.0"): + proxy_name = "host" + else: + proxy_name = "name" + # Get all Zabbix and Netbox data + netbox_devices = netbox.dcim.devices.filter(**nb_device_filter) + netbox_journals = netbox.extras.journal_entries + zabbix_groups = zabbix.hostgroup.get(output=['groupid', 'name']) + zabbix_templates = zabbix.template.get(output=['templateid', 'name']) + zabbix_proxies = zabbix.proxy.get(output=['proxyid', proxy_name]) + + # Sanitize data + if proxy_name == "host": + for proxy in zabbix_proxies: + proxy['name'] = proxy.pop('host') + + # Go through all Netbox devices + for nb_device in netbox_devices: + try: + device = NetworkDevice(nb_device, zabbix, netbox_journals, + create_journal) + device.set_hostgroup(hostgroup_format) + device.set_template(templates_config_context, templates_config_context_overrule) + # Checks if device is part of cluster. + # Requires clustering variable + if device.isCluster() and clustering: + # Check if device is master or slave + if device.promoteMasterDevice(): + e = (f"Device {device.name} is " + f"part of cluster and primary.") + logger.info(e) + else: + # Device is secondary in cluster. + # Don't continue with this device. + e = (f"Device {device.name} is part of cluster " + f"but not primary. Skipping this host...") + logger.info(e) + continue + # Checks if device is in cleanup state + if device.status in zabbix_device_removal: + if device.zabbix_id: + # Delete device from Zabbix + # and remove hostID from Netbox. + device.cleanup() + logger.info(f"Cleaned up host {device.name}.") + + else: + # Device has been added to Netbox + # but is not in Activate state + logger.info(f"Skipping host {device.name} since its " + f"not in the active state.") + elif device.status in zabbix_device_disable: + device.zabbix_state = 1 + else: + device.zabbix_state = 0 + # Add hostgroup is variable is True + # and Hostgroup is not present in Zabbix + if create_hostgroups: + for group in zabbix_groups: + # If hostgroup is already present in Zabbix + if group["name"] == device.hostgroup: + break + else: + # Create new hostgroup + hostgroup = device.createZabbixHostgroup() + zabbix_groups.append(hostgroup) + # Device is already present in Zabbix + if device.zabbix_id: + device.ConsistencyCheck(zabbix_groups, zabbix_templates, + zabbix_proxies, full_proxy_sync) + # Add device to Zabbix + else: + device.createInZabbix(zabbix_groups, zabbix_templates, + zabbix_proxies) + except SyncError: + pass + + +class SyncError(Exception): + """ Class SyncError """ + +class JournalError(Exception): + """ Class SyncError """ + +class SyncExternalError(SyncError): + """ Class SyncExternalError """ + +class SyncInventoryError(SyncError): + """ Class SyncInventoryError """ + +class SyncDuplicateError(SyncError): + """ Class SyncDuplicateError """ + +class EnvironmentVarError(SyncError): + """ Class EnvironmentVarError """ + +class InterfaceConfigError(SyncError): + """ Class InterfaceConfigError """ + +class ProxyConfigError(SyncError): + """ Class ProxyConfigError """ + +class HostgroupError(SyncError): + """ Class HostgroupError """ + +class TemplateError(SyncError): + """ Class TemplateError """ + +class NetworkDevice(): + # pylint: disable=too-many-instance-attributes + """ + Represents Network device. + INPUT: (Netbox device class, ZabbixAPI class, journal flag, NB journal class) + """ + + def __init__(self, nb, zabbix, nb_journal_class, journal=None): + self.nb = nb + self.id = nb.id + self.name = nb.name + self.status = nb.status.label + self.zabbix = zabbix + self.zabbix_id = None + self.group_id = None + self.zbx_template_names = [] + self.zbx_templates = [] + self.hostgroup = None + self.tenant = nb.tenant + self.config_context = nb.config_context + self.zbxproxy = "0" + self.zabbix_state = 0 + self.journal = journal + self.nb_journals = nb_journal_class + self._setBasics() + + def _setBasics(self): + """ + Sets basic information like IP address. + """ + # Return error if device does not have primary IP. + if self.nb.primary_ip: + self.cidr = self.nb.primary_ip.address + self.ip = self.cidr.split("/")[0] + else: + e = f"Device {self.name}: no primary IP." + logger.warning(e) + raise SyncInventoryError(e) + + # Check if device has custom field for ZBX ID + if device_cf in self.nb.custom_fields: + self.zabbix_id = self.nb.custom_fields[device_cf] + else: + e = f"Custom field {device_cf} not found for {self.name}." + logger.warning(e) + raise SyncInventoryError(e) + + def set_hostgroup(self, hg_format): + """Set the hostgroup for this device""" + # Get all variables from the NB data + dev_location = str(self.nb.location) if self.nb.location else None + dev_role = self.nb.device_role.name + manufacturer = self.nb.device_type.manufacturer.name + region = str(self.nb.site.region) if self.nb.site.region else None + site = self.nb.site.name + site_group = str(self.nb.site.group) if self.nb.site.group else None + tenant = str(self.tenant) if self.tenant else None + tenant_group = str(self.tenant.group) if tenant else None + # Set mapper for string -> variable + hostgroup_vars = {"dev_location": dev_location, "dev_role": dev_role, + "manufacturer": manufacturer, "region": region, + "site": site, "site_group": site_group, + "tenant": tenant, "tenant_group": tenant_group} + # Generate list based off string input format + hg_items = hg_format.split("/") + hostgroup = "" + # Go through all hostgroup items + for item in hg_items: + # Check if the variable (such as Tenant) is empty. + if not hostgroup_vars[item]: + continue + # Check if the item is a custom field name + if item not in hostgroup_vars: + cf_value = self.nb.custom_fields[item] if item in self.nb.custom_fields else None + if cf_value: + # If there is a cf match, add the value of this cf to the hostgroup + hostgroup += cf_value + "/" + # Should there not be a match, this means that + # the variable is invalid. Skip regardless. + continue + # Add value of predefined variable to hostgroup format + hostgroup += hostgroup_vars[item] + "/" + # If the final hostgroup variable is empty + if not hostgroup: + e = (f"{self.name} has no reliable hostgroup. This is" + "most likely due to the use of custom fields that are empty.") + logger.error(e) + raise SyncInventoryError(e) + # Remove final inserted "/" and set hostgroup to class var + self.hostgroup = hostgroup.rstrip("/") + + def set_template(self, prefer_config_context, overrule_custom): + """ Set Template """ + self.zbx_template_names = None + # Gather templates ONLY from the device specific context + if prefer_config_context: + try: + self.zbx_template_names = self.get_templates_context() + except TemplateError as e: + logger.warning(e) + return True + # Gather templates from the custom field but overrule + # them should there be any device specific templates + if overrule_custom: + try: + self.zbx_template_names = self.get_templates_context() + except TemplateError: + pass + if not self.zbx_template_names: + self.zbx_template_names = self.get_templates_cf() + return True + # Gather templates ONLY from the custom field + self.zbx_template_names = self.get_templates_cf() + return True + + def get_templates_cf(self): + """ Get template from custom field """ + # Get Zabbix templates from the device type + device_type_cfs = self.nb.device_type.custom_fields + # Check if the ZBX Template CF is present + if template_cf in device_type_cfs: + # Set value to template + return [device_type_cfs[template_cf]] + # Custom field not found, return error + e = (f"Custom field {template_cf} not " + f"found for {self.nb.device_type.manufacturer.name}" + f" - {self.nb.device_type.display}.") + raise TemplateError(e) from e + + def get_templates_context(self): + """ Get Zabbix templates from the device context """ + if "zabbix" not in self.config_context: + e = ("Key 'zabbix' not found in config " + f"context for template host {self.name}") + raise TemplateError(e) + if "templates" not in self.config_context["zabbix"]: + e = ("Key 'zabbix' not found in config " + f"context for template host {self.name}") + raise TemplateError(e) + return self.config_context["zabbix"]["templates"] + + def isCluster(self): + """ + Checks if device is part of cluster. + """ + return bool(self.nb.virtual_chassis) + + def getClusterMaster(self): + """ + Returns chassis master ID. + """ + if not self.isCluster(): + e = (f"Unable to proces {self.name} for cluster calculation: " + f"not part of a cluster.") + logger.warning(e) + raise SyncInventoryError(e) + if not self.nb.virtual_chassis.master: + e = (f"{self.name} is part of a Netbox virtual chassis which does " + "not have a master configured. Skipping for this reason.") + logger.error(e) + raise SyncInventoryError(e) + return self.nb.virtual_chassis.master.id + + def promoteMasterDevice(self): + """ + If device is Primary in cluster, + promote device name to the cluster name. + Returns True if succesfull, returns False if device is secondary. + """ + masterid = self.getClusterMaster() + if masterid == self.id: + logger.debug(f"Device {self.name} is primary cluster member. " + f"Modifying hostname from {self.name} to " + + f"{self.nb.virtual_chassis.name}.") + self.name = self.nb.virtual_chassis.name + return True + logger.debug(f"Device {self.name} is non-primary cluster member.") + return False + + def zbxTemplatePrepper(self, templates): + """ + Returns Zabbix template IDs + INPUT: list of templates from Zabbix + OUTPUT: True + """ + # Check if there are templates defined + if not self.zbx_template_names: + e = f"No templates found for device {self.name}" + logger.info(e) + raise SyncInventoryError() + # Set variable to empty list + self.zbx_templates = [] + # Go through all templates definded in Netbox + for nb_template in self.zbx_template_names: + template_match = False + # Go through all templates found in Zabbix + for zbx_template in templates: + # If the template names match + if zbx_template['name'] == nb_template: + # Set match variable to true, add template details + # to class variable and return debug log + template_match = True + self.zbx_templates.append({"templateid": zbx_template['templateid'], + "name": zbx_template['name']}) + e = (f"Found template {zbx_template['name']}" + f" for host {self.name}.") + logger.debug(e) + # Return error should the template not be found in Zabbix + if not template_match: + e = (f"Unable to find template {nb_template} " + f"for host {self.name} in Zabbix. Skipping host...") + logger.warning(e) + raise SyncInventoryError(e) from e + + def getZabbixGroup(self, groups): + """ + Returns Zabbix group ID + INPUT: list of hostgroups + OUTPUT: True / False + """ + # Go through all groups + for group in groups: + if group['name'] == self.hostgroup: + self.group_id = group['groupid'] + e = f"Found group {group['name']} for host {self.name}." + logger.debug(e) + return True + e = (f"Unable to find group '{self.hostgroup}' " + f"for host {self.name} in Zabbix.") + logger.warning(e) + raise SyncInventoryError(e) + + def cleanup(self): + """ + Removes device from external resources. + Resets custom fields in Netbox. + """ + if self.zabbix_id: + try: + self.zabbix.host.delete(self.zabbix_id) + self.nb.custom_fields[device_cf] = None + self.nb.save() + e = f"Deleted host {self.name} from Zabbix." + logger.info(e) + self.create_journal_entry("warning", "Deleted host from Zabbix") + except ZabbixAPIException as e: + e = f"Zabbix returned the following error: {str(e)}." + logger.error(e) + raise SyncExternalError(e) from e + + def _zabbixHostnameExists(self): + """ + Checks if hostname exists in Zabbix. + """ + host = self.zabbix.host.get(filter={'name': self.name}, output=[]) + return bool(host) + + def setInterfaceDetails(self): + """ + Checks interface parameters from Netbox and + creates a model for the interface to be used in Zabbix. + """ + try: + # Initiate interface class + interface = ZabbixInterface(self.nb.config_context, self.ip) + # Check if Netbox has device context. + # If not fall back to old config. + if interface.get_context(): + # If device is SNMP type, add aditional information. + if interface.interface["type"] == 2: + interface.set_snmp() + else: + interface.set_default() + return [interface.interface] + except InterfaceConfigError as e: + e = f"{self.name}: {e}" + logger.warning(e) + raise SyncInventoryError(e) from e + + def setProxy(self, proxy_list): + """ check if Zabbix Proxy has been defined in config context """ + if "zabbix" in self.nb.config_context: + if "proxy" in self.nb.config_context["zabbix"]: + proxy = self.nb.config_context["zabbix"]["proxy"] + # Try matching proxy + for px in proxy_list: + if px["name"] == proxy: + self.zbxproxy = px["proxyid"] + logger.debug(f"Found proxy {proxy}" + f" for {self.name}.") + return True + return False + e = f"{self.name}: Defined proxy {proxy} not found." + logger.warning(e) + return False + return False + + def createInZabbix(self, groups, templates, proxies, + description="Host added by Netbox sync script."): + """ + Creates Zabbix host object with parameters from Netbox object. + """ + # Check if hostname is already present in Zabbix + if not self._zabbixHostnameExists(): + # Get group and template ID's for host + if not self.getZabbixGroup(groups): + raise SyncInventoryError() + self.zbxTemplatePrepper(templates) + templateids = [] + for template in self.zbx_templates: + templateids.append({'templateid': template['templateid']}) + # Set interface, group and template configuration + interfaces = self.setInterfaceDetails() + groups = [{"groupid": self.group_id}] + # Set Zabbix proxy if defined + self.setProxy(proxies) + # Add host to Zabbix + try: + if version.parse(self.zabbix.api_version()) < version.parse("7.0.0"): + host = self.zabbix.host.create(host=self.name, + status=self.zabbix_state, + interfaces=interfaces, + groups=groups, + templates=templateids, + proxy_hostid=self.zbxproxy, + description=description) + else: + host = self.zabbix.host.create(host=self.name, + status=self.zabbix_state, + interfaces=interfaces, + groups=groups, + templates=templateids, + proxyid=self.zbxproxy, + description=description) + self.zabbix_id = host["hostids"][0] + except ZabbixAPIException as e: + e = f"Couldn't add {self.name}, Zabbix returned {str(e)}." + logger.error(e) + raise SyncExternalError(e) from e + # Set Netbox custom field to hostID value. + self.nb.custom_fields[device_cf] = int(self.zabbix_id) + self.nb.save() + msg = f"Created host {self.name} in Zabbix." + logger.info(msg) + self.create_journal_entry("success", msg) + else: + e = f"Unable to add {self.name} to Zabbix: host already present." + logger.warning(e) + + def createZabbixHostgroup(self): + """ + Creates Zabbix host group based on hostgroup format. + """ + try: + groupid = self.zabbix.hostgroup.create(name=self.hostgroup) + e = f"Added hostgroup '{self.hostgroup}'." + logger.info(e) + data = {'groupid': groupid["groupids"][0], 'name': self.hostgroup} + return data + except ZabbixAPIException as e: + e = f"Couldn't add hostgroup, Zabbix returned {str(e)}." + logger.error(e) + raise SyncExternalError(e) from e + + def updateZabbixHost(self, **kwargs): + """ + Updates Zabbix host with given parameters. + INPUT: Key word arguments for Zabbix host object. + """ + try: + self.zabbix.host.update(hostid=self.zabbix_id, **kwargs) + except ZabbixAPIException as e: + e = f"Zabbix returned the following error: {str(e)}." + logger.error(e) + raise SyncExternalError(e) from e + logger.info(f"Updated host {self.name} with data {kwargs}.") + self.create_journal_entry("info", "Updated host in Zabbix with latest NB data.") + + def ConsistencyCheck(self, groups, templates, proxies, proxy_power): + # pylint: disable=too-many-branches, too-many-statements + """ + Checks if Zabbix object is still valid with Netbox parameters. + """ + self.getZabbixGroup(groups) + self.zbxTemplatePrepper(templates) + self.setProxy(proxies) + host = self.zabbix.host.get(filter={'hostid': self.zabbix_id}, + selectInterfaces=['type', 'ip', + 'port', 'details', + 'interfaceid'], + selectGroups=["groupid"], + selectParentTemplates=["templateid"]) + if len(host) > 1: + e = (f"Got {len(host)} results for Zabbix hosts " + f"with ID {self.zabbix_id} - hostname {self.name}.") + logger.error(e) + raise SyncInventoryError(e) + if len(host) == 0: + e = (f"No Zabbix host found for {self.name}. " + f"This is likely the result of a deleted Zabbix host " + f"without zeroing the ID field in Netbox.") + logger.error(e) + raise SyncInventoryError(e) + host = host[0] + + if host["host"] == self.name: + logger.debug(f"Device {self.name}: hostname in-sync.") + else: + logger.warning(f"Device {self.name}: hostname OUT of sync. " + f"Received value: {host['host']}") + self.updateZabbixHost(host=self.name) + + # Check if the templates are in-sync + if not self.zbx_template_comparer(host["parentTemplates"]): + logger.warning(f"Device {self.name}: template(s) OUT of sync.") + # Update Zabbix with NB templates and clear any old / lost templates + self.updateZabbixHost(templates_clear=host["parentTemplates"], + templates=self.zbx_templates) + else: + logger.debug(f"Device {self.name}: template(s) in-sync.") + + for group in host["groups"]: + if group["groupid"] == self.group_id: + logger.debug(f"Device {self.name}: hostgroup in-sync.") + break + else: + logger.warning(f"Device {self.name}: hostgroup OUT of sync.") + self.updateZabbixHost(groups={'groupid': self.group_id}) + + if int(host["status"]) == self.zabbix_state: + logger.debug(f"Device {self.name}: status in-sync.") + else: + logger.warning(f"Device {self.name}: status OUT of sync.") + self.updateZabbixHost(status=str(self.zabbix_state)) + + # Check if a proxy has been defined + if self.zbxproxy != "0": + # Check if expected proxyID matches with configured proxy + if (("proxy_hostid" in host and host["proxy_hostid"] == self.zbxproxy) or + ("proxyid" in host and host["proxyid"] == self.zbxproxy)): + logger.debug(f"Device {self.name}: proxy in-sync.") + else: + # Proxy diff, update value + logger.warning(f"Device {self.name}: proxy OUT of sync.") + if version.parse(self.zabbix.api_version()) < version.parse("7.0.0"): + self.updateZabbixHost(proxy_hostid=self.zbxproxy) + else: + self.updateZabbixHost(proxyid=self.zbxproxy) + else: + if (("proxy_hostid" in host and not host["proxy_hostid"] == "0") + or ("proxyid" in host and not host["proxyid"] == "0")): + if proxy_power: + # Variable full_proxy_sync has been enabled + # delete the proxy link in Zabbix + if version.parse(self.zabbix.api_version()) < version.parse("7.0.0"): + self.updateZabbixHost(proxy_hostid=self.zbxproxy) + else: + self.updateZabbixHost(proxyid=self.zbxproxy) + else: + # Instead of deleting the proxy config in zabbix and + # forcing potential data loss, + # an error message is displayed. + logger.error(f"Device {self.name} is configured " + f"with proxy in Zabbix but not in Netbox. The" + " -p flag was ommited: no " + "changes have been made.") + # If only 1 interface has been found + # pylint: disable=too-many-nested-blocks + if len(host['interfaces']) == 1: + updates = {} + # Go through each key / item and check if it matches Zabbix + for key, item in self.setInterfaceDetails()[0].items(): + # Check if Netbox value is found in Zabbix + if key in host["interfaces"][0]: + # If SNMP is used, go through nested dict + # to compare SNMP parameters + if isinstance(item,dict) and key == "details": + for k, i in item.items(): + if k in host["interfaces"][0][key]: + # Set update if values don't match + if host["interfaces"][0][key][k] != str(i): + # If dict has not been created, add it + if key not in updates: + updates[key] = {} + updates[key][k] = str(i) + # If SNMP version has been changed + # break loop and force full SNMP update + if k == "version": + break + # Force full SNMP config update + # when version has changed. + if key in updates: + if "version" in updates[key]: + for k, i in item.items(): + updates[key][k] = str(i) + continue + # Set update if values don't match + if host["interfaces"][0][key] != str(item): + updates[key] = item + if updates: + # If interface updates have been found: push to Zabbix + logger.warning(f"Device {self.name}: Interface OUT of sync.") + if "type" in updates: + # Changing interface type not supported. Raise exception. + e = (f"Device {self.name}: changing interface type to " + f"{str(updates['type'])} is not supported.") + logger.error(e) + raise InterfaceConfigError(e) + # Set interfaceID for Zabbix config + updates["interfaceid"] = host["interfaces"][0]['interfaceid'] + try: + # API call to Zabbix + self.zabbix.hostinterface.update(updates) + e = f"Solved {self.name} interface conflict." + logger.info(e) + self.create_journal_entry("info", e) + except ZabbixAPIException as e: + e = f"Zabbix returned the following error: {str(e)}." + logger.error(e) + raise SyncExternalError(e) from e + else: + # If no updates are found, Zabbix interface is in-sync + e = f"Device {self.name}: interface in-sync." + logger.debug(e) + else: + e = (f"Device {self.name} has unsupported interface configuration." + f" Host has total of {len(host['interfaces'])} interfaces. " + "Manual interfention required.") + logger.error(e) + raise SyncInventoryError(e) + + def create_journal_entry(self, severity, message): + """ + Send a new Journal entry to Netbox. Usefull for viewing actions + in Netbox without having to look in Zabbix or the script log output + """ + if self.journal: + # Check if the severity is valid + if severity not in ["info", "success", "warning", "danger"]: + logger.warning(f"Value {severity} not valid for NB journal entries.") + return False + journal = {"assigned_object_type": "dcim.device", + "assigned_object_id": self.id, + "kind": severity, + "comments": message + } + try: + self.nb_journals.create(journal) + logger.debug(f"Created journal entry in NB for host {self.name}") + return True + except JournalError(e) as e: + logger.warning("Unable to create journal entry for " + f"{self.name}: NB returned {e}") + return False + return False + + def zbx_template_comparer(self, tmpls_from_zabbix): + """ + Compares the Netbox and Zabbix templates with each other. + Should there be a mismatch then the function will return false + + INPUT: list of NB and ZBX templates + OUTPUT: Boolean True/False + """ + succesfull_templates = [] + # Go through each Netbox template + for nb_tmpl in self.zbx_templates: + # Go through each Zabbix template + for pos, zbx_tmpl in enumerate(tmpls_from_zabbix): + # Check if template IDs match + if nb_tmpl["templateid"] == zbx_tmpl["templateid"]: + # Templates match. Remove this template from the Zabbix templates + # and add this NB template to the list of successfull templates + tmpls_from_zabbix.pop(pos) + succesfull_templates.append(nb_tmpl) + logger.debug(f"Device {self.name}: template " + f"{nb_tmpl['name']} is present in Zabbix.") + break + if len(succesfull_templates) == len(self.zbx_templates) and len(tmpls_from_zabbix) == 0: + # All of the Netbox templates have been confirmed as successfull + # and the ZBX template list is empty. This means that + # all of the templates match. + return True + return False + + +class ZabbixInterface(): + """Class that represents a Zabbix interface.""" + + def __init__(self, context, ip): + self.context = context + self.ip = ip + self.skelet = {"main": "1", "useip": "1", "dns": "", "ip": self.ip} + self.interface = self.skelet + + def get_context(self): + """ check if Netbox custom context has been defined. """ + if "zabbix" in self.context: + zabbix = self.context["zabbix"] + if("interface_type" in zabbix and "interface_port" in zabbix): + self.interface["type"] = zabbix["interface_type"] + self.interface["port"] = zabbix["interface_port"] + return True + return False + return False + + def set_snmp(self): + """ Check if interface is type SNMP """ + # pylint: disable=too-many-branches + if self.interface["type"] == 2: + # Checks if SNMP settings are defined in Netbox + if "snmp" in self.context["zabbix"]: + snmp = self.context["zabbix"]["snmp"] + self.interface["details"] = {} + # Checks if bulk config has been defined + if "bulk" in snmp: + self.interface["details"]["bulk"] = str(snmp.pop("bulk")) + else: + # Fallback to bulk enabled if not specified + self.interface["details"]["bulk"] = "1" + # SNMP Version config is required in Netbox config context + if snmp.get("version"): + self.interface["details"]["version"] = str(snmp.pop("version")) + else: + e = "SNMP version option is not defined." + raise InterfaceConfigError(e) + # If version 1 or 2 is used, get community string + if self.interface["details"]["version"] in ['1','2']: + if "community" in snmp: + # Set SNMP community to confix context value + community = snmp["community"] + else: + # Set SNMP community to default + community = "{$SNMP_COMMUNITY}" + self.interface["details"]["community"] = str(community) + # If version 3 has been used, get all + # SNMPv3 Netbox related configs + elif self.interface["details"]["version"] == '3': + items = ["securityname", "securitylevel", "authpassphrase", + "privpassphrase", "authprotocol", "privprotocol", + "contextname"] + for key, item in snmp.items(): + if key in items: + self.interface["details"][key] = str(item) + else: + e = "Unsupported SNMP version." + raise InterfaceConfigError(e) + else: + e = "Interface type SNMP but no parameters provided." + raise InterfaceConfigError(e) + else: + e = "Interface type is not SNMP, unable to set SNMP details" + raise InterfaceConfigError(e) + + def set_default(self): + """ Set default config to SNMPv2, port 161 and community macro. """ + self.interface = self.skelet + self.interface["type"] = "2" + self.interface["port"] = "161" + self.interface["details"] = {"version": "2", + "community": "{$SNMP_COMMUNITY}", + "bulk": "1"} + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description='A script to sync Zabbix with Netbox device data.' + ) + parser.add_argument("-v", "--verbose", help="Turn on debugging.", + action="store_true") + args = parser.parse_args() + main(args) From 5b08d27a5e78cc1a80f301e30f4ed495021b04c6 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Wed, 27 Mar 2024 15:37:50 +0100 Subject: [PATCH 32/39] Added support for syncing Zabbix Inventory, this is also a fix for https://github.com/TheNetworkGuy/netbox-zabbix-sync/issues/44 --- README.md | 18 ++++++++ config.py.example | 29 ++++++++++++ netbox_zabbix_sync.py | 105 +++++++++++++++++++++++++++++++++++++----- 3 files changed, 141 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index d7f514d..621a1a6 100644 --- a/README.md +++ b/README.md @@ -170,6 +170,24 @@ You can modify this behaviour by changing the following list variables in the sc - `zabbix_device_removal` - `zabbix_device_disable` +### Zabbix Inventory +This script allows you to enable the inventory on managed Zabbix hosts and sync NetBox device properties to the specified inventory fields. +To enable, set `inventory_sync` to `True`. +Set `inventory_automatic` to `False` to use manual inventory, or `True` for automatic. +See [Zabix Manual](https://www.zabbix.com/documentation/current/en/manual/config/hosts/inventory#building-inventory) for more information about the modes. + +Use the `inventory_map` variable to map which NetBox properties are used in which Zabbix Inventory fields. +For nested properties, you can use the '/' seperator. +For example, the following map will assign the custom field 'mycustomfield' to the 'alias' Zabbix inventory field: +``` +inventory_sync = True +inventory_automatic = True +inventory_map = { "custom_fields/mycustomfield/name": "alias"} +``` +See `config.py.example` for an extensive example map. +Any Zabix Inventory fields that are not included in the map will not be touched by the script, +so you can safely add manual values or use items to automatically add values to other fields. + ### Template source You can either use a Netbox device type custom field or Netbox config context for the Zabbix template information. diff --git a/config.py.example b/config.py.example index 51e7dc2..cfa7f6a 100644 --- a/config.py.example +++ b/config.py.example @@ -56,3 +56,32 @@ traverse_site_groups = False # Default device filter, only get devices which have a name in Netbox: nb_device_filter = {"name__n": "null"} + +## Inventory +# To allow syncing of NetBox device properties, set inventory_sync to True +inventory_sync = False + +# Set inventory_automatic to False to use manual inventory, True for automatic +# See https://www.zabbix.com/documentation/current/en/manual/config/hosts/inventory#building-inventory +inventory_automatic = True + +# inventory_map is used to map NetBox properties to Zabbix Inventory fields. +# For nested properties, you can use the '/' seperator. +# For example, the following map will assign the custom field 'mycustomfield' to the 'alias' Zabbix inventory field: +# +# inventory_map = { "custom_fields/mycustomfield/name": "alias"} +# +# The following map should provide some nice defaults: +inventory_map = { "asset_tag": "asset_tag", + "virtual_chassis/name": "chassis", + "status/label": "deployment_status", + "location/name": "location", + "latitude": "location_lat", + "longitude": "location_lon", + "comments": "notes", + "name": "name", + "rack/name": "site_rack", + "serial": "serialno_a", + "device_type/model": "type", + "device_type/manufacturer/name": "vendor", + "oob_ip/address": "oob_ip" } diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index 8cf0812..3fefb9c 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -3,7 +3,6 @@ """Netbox to Zabbix sync script.""" - import logging import argparse from os import environ, path, sys @@ -20,6 +19,11 @@ try: zabbix_device_removal, zabbix_device_disable, hostgroup_format, + traverse_site_groups, + traverse_regions, + inventory_sync, + inventory_automatic, + inventory_map, nb_device_filter ) except ModuleNotFoundError: @@ -45,6 +49,30 @@ logger.addHandler(lgfile) logger.setLevel(logging.WARNING) +def convert_recordset(recordset): + """ Converts netbox RedcordSet to list of dicts. """ + recordlist = [] + for record in recordset: + recordlist.append(record.__dict__) + return recordlist + +def build_path(endpoint, list_of_dicts): + """ + Builds a path list of related parent/child items. + This can be used to generate a joinable list to + be used in hostgroups. + """ + path = [] + itemlist = [i for i in list_of_dicts if i['name'] == endpoint] + item = itemlist[0] if len(itemlist) == 1 else None + path.append(item['name']) + while item['_depth'] > 0: + itemlist = [i for i in list_of_dicts if i['name'] == str(item['parent'])] + item = itemlist[0] if len(itemlist) == 1 else None + path.append(item['name']) + path.reverse() + return(path) + def main(arguments): """Run the sync process.""" # pylint: disable=too-many-branches, too-many-statements @@ -110,6 +138,8 @@ def main(arguments): proxy_name = "name" # Get all Zabbix and Netbox data netbox_devices = netbox.dcim.devices.filter(**nb_device_filter) + netbox_site_groups = convert_recordset((netbox.dcim.site_groups.all())) + netbox_regions = convert_recordset(netbox.dcim.regions.all()) netbox_journals = netbox.extras.journal_entries zabbix_groups = zabbix.hostgroup.get(output=['groupid', 'name']) zabbix_templates = zabbix.template.get(output=['templateid', 'name']) @@ -125,8 +155,9 @@ def main(arguments): try: device = NetworkDevice(nb_device, zabbix, netbox_journals, create_journal) - device.set_hostgroup(hostgroup_format) + device.set_hostgroup(hostgroup_format,netbox_site_groups,netbox_regions) device.set_template(templates_config_context, templates_config_context_overrule) + device.set_inventory(nb_device) # Checks if device is part of cluster. # Requires clustering variable if device.isCluster() and clustering: @@ -236,6 +267,8 @@ class NetworkDevice(): self.zabbix_state = 0 self.journal = journal self.nb_journals = nb_journal_class + self.inventory_mode = -1 + self.inventory = {} self._setBasics() def _setBasics(self): @@ -248,7 +281,7 @@ class NetworkDevice(): self.ip = self.cidr.split("/")[0] else: e = f"Device {self.name}: no primary IP." - logger.warning(e) + logger.info(e) raise SyncInventoryError(e) # Check if device has custom field for ZBX ID @@ -259,7 +292,7 @@ class NetworkDevice(): logger.warning(e) raise SyncInventoryError(e) - def set_hostgroup(self, hg_format): + def set_hostgroup(self, hg_format, nb_site_groups, nb_regions): """Set the hostgroup for this device""" # Get all variables from the NB data dev_location = str(self.nb.location) if self.nb.location else None @@ -274,7 +307,7 @@ class NetworkDevice(): hostgroup_vars = {"dev_location": dev_location, "dev_role": dev_role, "manufacturer": manufacturer, "region": region, "site": site, "site_group": site_group, - "tenant": tenant, "tenant_group": tenant_group} + "tenant": tenant, "tenant_group": tenant_group} # Generate list based off string input format hg_items = hg_format.split("/") hostgroup = "" @@ -293,7 +326,14 @@ class NetworkDevice(): # the variable is invalid. Skip regardless. continue # Add value of predefined variable to hostgroup format - hostgroup += hostgroup_vars[item] + "/" + if item == "site_group" and nb_site_groups and traverse_site_groups: + path = build_path(site_group, nb_site_groups) + hostgroup += "/".join(path) + "/" + elif item == "region" and nb_regions and traverse_regions: + path = build_path(region, nb_regions) + hostgroup += "/".join(path) + "/" + else: + hostgroup += hostgroup_vars[item] + "/" # If the final hostgroup variable is empty if not hostgroup: e = (f"{self.name} has no reliable hostgroup. This is" @@ -353,6 +393,32 @@ class NetworkDevice(): raise TemplateError(e) return self.config_context["zabbix"]["templates"] + def set_inventory(self, nbdevice): + """ Set host inventory """ + self.inventory_mode = -1 + self.inventory = {} + if inventory_sync: + self.inventory_mode = 1 if inventory_automatic else 0 + for nb_inv_field, zbx_inv_field in inventory_map.items(): + field_list = nb_inv_field.split("/") + fieldstr = "nbdevice" + for field in field_list: + fieldstr += "['" + field + "']" + try: + nb_value = eval(fieldstr) + except: + nb_value = None + if nb_value and isinstance(nb_value, int | float | str ): + self.inventory[zbx_inv_field] = str(nb_value) + elif not nb_value: + logger.debug('Inventory lookup for "%s" returned an empty value' % nb_inv_field) + self.inventory[zbx_inv_field] = "" + else: + # Value is not a string or numeral, probably not what the user expected. + logger.error('Inventory lookup for "%s" returned an unexpected type,' + ' it will be skipped.' % nb_inv_field) + return True + def isCluster(self): """ Checks if device is part of cluster. @@ -415,7 +481,7 @@ class NetworkDevice(): # to class variable and return debug log template_match = True self.zbx_templates.append({"templateid": zbx_template['templateid'], - "name": zbx_template['name']}) + "name": zbx_template['name']}) e = (f"Found template {zbx_template['name']}" f" for host {self.name}.") logger.debug(e) @@ -537,7 +603,9 @@ class NetworkDevice(): groups=groups, templates=templateids, proxy_hostid=self.zbxproxy, - description=description) + description=description, + inventory_mode=self.inventory_mode, + inventory=self.inventory) else: host = self.zabbix.host.create(host=self.name, status=self.zabbix_state, @@ -545,7 +613,9 @@ class NetworkDevice(): groups=groups, templates=templateids, proxyid=self.zbxproxy, - description=description) + description=description, + inventory_mode=self.inventory_mode, + inventory=self.inventory) self.zabbix_id = host["hostids"][0] except ZabbixAPIException as e: e = f"Couldn't add {self.name}, Zabbix returned {str(e)}." @@ -603,7 +673,8 @@ class NetworkDevice(): 'port', 'details', 'interfaceid'], selectGroups=["groupid"], - selectParentTemplates=["templateid"]) + selectParentTemplates=["templateid"], + selectInventory=list(inventory_map.values())) if len(host) > 1: e = (f"Got {len(host)} results for Zabbix hosts " f"with ID {self.zabbix_id} - hostname {self.name}.") @@ -616,7 +687,6 @@ class NetworkDevice(): logger.error(e) raise SyncInventoryError(e) host = host[0] - if host["host"] == self.name: logger.debug(f"Device {self.name}: hostname in-sync.") else: @@ -678,6 +748,19 @@ class NetworkDevice(): f"with proxy in Zabbix but not in Netbox. The" " -p flag was ommited: no " "changes have been made.") + # Check host inventory + if inventory_sync: + if str(host['inventory_mode']) == str(self.inventory_mode): + logger.debug(f"Device {self.name}: inventory_mode in-sync.") + else: + logger.warning(f"Device {self.name}: inventory_mode OUT of sync.") + self.updateZabbixHost(inventory_mode=str(self.inventory_mode)) + if host['inventory'] == self.inventory: + logger.debug(f"Device {self.name}: inventory in-sync.") + else: + logger.warning(f"Device {self.name}: inventory OUT of sync.") + self.updateZabbixHost(inventory=self.inventory) + # If only 1 interface has been found # pylint: disable=too-many-nested-blocks if len(host['interfaces']) == 1: From fbb9eeb48c7b1f573211bb095f8808ac5a3004f4 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Wed, 27 Mar 2024 16:24:26 +0100 Subject: [PATCH 33/39] Corrected linting errors --- netbox_zabbix_sync.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index 3fefb9c..f1db55e 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -62,16 +62,16 @@ def build_path(endpoint, list_of_dicts): This can be used to generate a joinable list to be used in hostgroups. """ - path = [] + item_path = [] itemlist = [i for i in list_of_dicts if i['name'] == endpoint] item = itemlist[0] if len(itemlist) == 1 else None - path.append(item['name']) + item_path.append(item['name']) while item['_depth'] > 0: itemlist = [i for i in list_of_dicts if i['name'] == str(item['parent'])] item = itemlist[0] if len(itemlist) == 1 else None - path.append(item['name']) - path.reverse() - return(path) + item_path.append(item['name']) + item_path.reverse() + return item_path def main(arguments): """Run the sync process.""" @@ -327,11 +327,11 @@ class NetworkDevice(): continue # Add value of predefined variable to hostgroup format if item == "site_group" and nb_site_groups and traverse_site_groups: - path = build_path(site_group, nb_site_groups) - hostgroup += "/".join(path) + "/" + group_path = build_path(site_group, nb_site_groups) + hostgroup += "/".join(group_path) + "/" elif item == "region" and nb_regions and traverse_regions: - path = build_path(region, nb_regions) - hostgroup += "/".join(path) + "/" + region_path = build_path(region, nb_regions) + hostgroup += "/".join(region_path) + "/" else: hostgroup += hostgroup_vars[item] + "/" # If the final hostgroup variable is empty @@ -388,8 +388,8 @@ class NetworkDevice(): f"context for template host {self.name}") raise TemplateError(e) if "templates" not in self.config_context["zabbix"]: - e = ("Key 'zabbix' not found in config " - f"context for template host {self.name}") + e = ("Key 'templates' not found in config " + f"context 'zabbix' for template host {self.name}") raise TemplateError(e) return self.config_context["zabbix"]["templates"] @@ -411,12 +411,12 @@ class NetworkDevice(): if nb_value and isinstance(nb_value, int | float | str ): self.inventory[zbx_inv_field] = str(nb_value) elif not nb_value: - logger.debug('Inventory lookup for "%s" returned an empty value' % nb_inv_field) + logger.debug(f"Inventory lookup for '{nb_inv_field}' returned an empty value") self.inventory[zbx_inv_field] = "" else: # Value is not a string or numeral, probably not what the user expected. - logger.error('Inventory lookup for "%s" returned an unexpected type,' - ' it will be skipped.' % nb_inv_field) + logger.error(f"Inventory lookup for '{nb_inv_field}' returned an unexpected type: " + f"it will be skipped.") return True def isCluster(self): From ab2a341fa773fa354bc0f5b015b2e9a5d06bd346 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Wed, 27 Mar 2024 16:26:15 +0100 Subject: [PATCH 34/39] Corrected more linting errors --- netbox_zabbix_sync.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index f1db55e..f79180f 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -415,8 +415,8 @@ class NetworkDevice(): self.inventory[zbx_inv_field] = "" else: # Value is not a string or numeral, probably not what the user expected. - logger.error(f"Inventory lookup for '{nb_inv_field}' returned an unexpected type: " - f"it will be skipped.") + logger.error(f"Inventory lookup for '{nb_inv_field}' returned" + f" an unexpected type: it will be skipped.") return True def isCluster(self): From 364d376f559d52438539ed1e6706a233c4d71d96 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Wed, 27 Mar 2024 16:33:06 +0100 Subject: [PATCH 35/39] corrected even more linting errors --- netbox_zabbix_sync.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index f79180f..a976276 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -406,7 +406,7 @@ class NetworkDevice(): fieldstr += "['" + field + "']" try: nb_value = eval(fieldstr) - except: + except Exception: nb_value = None if nb_value and isinstance(nb_value, int | float | str ): self.inventory[zbx_inv_field] = str(nb_value) @@ -415,7 +415,7 @@ class NetworkDevice(): self.inventory[zbx_inv_field] = "" else: # Value is not a string or numeral, probably not what the user expected. - logger.error(f"Inventory lookup for '{nb_inv_field}' returned" + logger.error(f"Inventory lookup for '{nb_inv_field}' returned" f" an unexpected type: it will be skipped.") return True From 091c9746c023e5ff55b83b1ed61c42eb738214d4 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Wed, 27 Mar 2024 20:33:02 +0100 Subject: [PATCH 36/39] Fixed proxy issue, rewrite of inventory logic (eval was ugly) --- netbox_zabbix_sync.py | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index a976276..3c13607 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -398,19 +398,22 @@ class NetworkDevice(): self.inventory_mode = -1 self.inventory = {} if inventory_sync: + # Set inventory mode to automatic or manual self.inventory_mode = 1 if inventory_automatic else 0 + + # Let's build an inventory dict for each property in the inventory_map for nb_inv_field, zbx_inv_field in inventory_map.items(): - field_list = nb_inv_field.split("/") - fieldstr = "nbdevice" - for field in field_list: - fieldstr += "['" + field + "']" - try: - nb_value = eval(fieldstr) - except Exception: - nb_value = None - if nb_value and isinstance(nb_value, int | float | str ): - self.inventory[zbx_inv_field] = str(nb_value) - elif not nb_value: + field_list = nb_inv_field.split("/") # convert str to list based on delimiter + # start at the base of the dict... + value = nbdevice + # ... and step through the dict till we find the needed value + for item in field_list: + value = value[item] + # Check if the result is usable and expected + if value and isinstance(value, int | float | str ): + self.inventory[zbx_inv_field] = str(value) + elif not value: + # empty value should just be an empty string for API compatibility logger.debug(f"Inventory lookup for '{nb_inv_field}' returned an empty value") self.inventory[zbx_inv_field] = "" else: @@ -569,11 +572,10 @@ class NetworkDevice(): logger.debug(f"Found proxy {proxy}" f" for {self.name}.") return True + e = f"{self.name}: Defined proxy {proxy} not found." + logger.warning(e) return False - e = f"{self.name}: Defined proxy {proxy} not found." - logger.warning(e) - return False - return False + return True def createInZabbix(self, groups, templates, proxies, description="Host added by Netbox sync script."): @@ -748,13 +750,17 @@ class NetworkDevice(): f"with proxy in Zabbix but not in Netbox. The" " -p flag was ommited: no " "changes have been made.") + # Check host inventory if inventory_sync: + # check inventory mode first, as we need it set to parse + # actual inventory values if str(host['inventory_mode']) == str(self.inventory_mode): logger.debug(f"Device {self.name}: inventory_mode in-sync.") else: logger.warning(f"Device {self.name}: inventory_mode OUT of sync.") self.updateZabbixHost(inventory_mode=str(self.inventory_mode)) + # Now we can check if inventory is in-sync. if host['inventory'] == self.inventory: logger.debug(f"Device {self.name}: inventory in-sync.") else: From c006e7feb59b9af19c60c1269deafc23386d96cf Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Wed, 27 Mar 2024 20:35:32 +0100 Subject: [PATCH 37/39] Let's make pylint happy :) --- netbox_zabbix_sync.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index 3c13607..c3d4edd 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -400,12 +400,12 @@ class NetworkDevice(): if inventory_sync: # Set inventory mode to automatic or manual self.inventory_mode = 1 if inventory_automatic else 0 - - # Let's build an inventory dict for each property in the inventory_map + + # Let's build an inventory dict for each property in the inventory_map for nb_inv_field, zbx_inv_field in inventory_map.items(): field_list = nb_inv_field.split("/") # convert str to list based on delimiter # start at the base of the dict... - value = nbdevice + value = nbdevice # ... and step through the dict till we find the needed value for item in field_list: value = value[item] From 634f4b77d5d7d6f60a427851546f92ec50e1386f Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Thu, 28 Mar 2024 09:51:08 +0100 Subject: [PATCH 38/39] tweaked exception handling --- netbox_zabbix_sync.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index c3d4edd..c11feb2 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -379,7 +379,7 @@ class NetworkDevice(): e = (f"Custom field {template_cf} not " f"found for {self.nb.device_type.manufacturer.name}" f" - {self.nb.device_type.display}.") - raise TemplateError(e) from e + raise TemplateError(e) def get_templates_context(self): """ Get Zabbix templates from the device context """ @@ -493,7 +493,7 @@ class NetworkDevice(): e = (f"Unable to find template {nb_template} " f"for host {self.name} in Zabbix. Skipping host...") logger.warning(e) - raise SyncInventoryError(e) from e + raise SyncInventoryError(e) def getZabbixGroup(self, groups): """ From 3e638c6f7895bfef4f6bb4af0df7d470e1d41eb5 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Wed, 10 Apr 2024 14:57:08 +0200 Subject: [PATCH 39/39] Update netbox_zabbix_sync.py Minor bug fix for empty Netbox to zabbix inventory field mappings. --- netbox_zabbix_sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index c11feb2..8b36c6a 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -408,7 +408,7 @@ class NetworkDevice(): value = nbdevice # ... and step through the dict till we find the needed value for item in field_list: - value = value[item] + value = value[item] if value else None # Check if the result is usable and expected if value and isinstance(value, int | float | str ): self.inventory[zbx_inv_field] = str(value)