From 525904cf43d7f77a7010c73001549e86f8d658db Mon Sep 17 00:00:00 2001 From: Wouter de Bruijn Date: Wed, 26 Feb 2025 10:07:51 +0100 Subject: [PATCH 01/18] =?UTF-8?q?=F0=9F=9A=A8=20Linted=20and=20formatted?= =?UTF-8?q?=20file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- netbox_zabbix_sync.py | 175 +++++++++++++++++++++++++----------------- 1 file changed, 104 insertions(+), 71 deletions(-) diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index 04a4e07..b355657 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -2,46 +2,55 @@ # pylint: disable=invalid-name, logging-not-lazy, too-many-locals, logging-fstring-interpolation """NetBox to Zabbix sync script.""" -import logging import argparse +import logging import ssl from os import environ, path, sys + from pynetbox import api from pynetbox.core.query import RequestError as NBRequestError from requests.exceptions import ConnectionError as RequestsConnectionError -from zabbix_utils import ZabbixAPI, APIRequestError, ProcessingError +from zabbix_utils import APIRequestError, ProcessingError, ZabbixAPI + from modules.device import PhysicalDevice -from modules.virtual_machine import VirtualMachine -from modules.tools import convert_recordset, proxy_prepper from modules.exceptions import EnvironmentVarError, HostgroupError, SyncError +from modules.tools import convert_recordset, proxy_prepper +from modules.virtual_machine import VirtualMachine + try: from config import ( + clustering, + create_hostgroups, + create_journal, + full_proxy_sync, + hostgroup_format, + nb_device_filter, + nb_vm_filter, + sync_vms, templates_config_context, templates_config_context_overrule, - clustering, create_hostgroups, - create_journal, full_proxy_sync, - zabbix_device_removal, - zabbix_device_disable, - hostgroup_format, vm_hostgroup_format, - nb_device_filter, - sync_vms, - nb_vm_filter + zabbix_device_disable, + zabbix_device_removal, ) except ModuleNotFoundError: - print("Configuration file config.py not found in main directory." - "Please create the file or rename the config.py.example file to config.py.") + print( + "Configuration file config.py not found in main directory." + "Please create the file or rename the config.py.example file to config.py." + ) sys.exit(1) # Set logging -log_format = logging.Formatter('%(asctime)s - %(name)s - ' - '%(levelname)s - %(message)s') +log_format = logging.Formatter( + "%(asctime)s - %(name)s - " "%(levelname)s - %(message)s" +) lgout = logging.StreamHandler() lgout.setFormatter(log_format) lgout.setLevel(logging.DEBUG) -lgfile = logging.FileHandler(path.join(path.dirname( - path.realpath(__file__)), "sync.log")) +lgfile = logging.FileHandler( + path.join(path.dirname(path.realpath(__file__)), "sync.log") +) lgfile.setFormatter(log_format) lgfile.setLevel(logging.DEBUG) @@ -84,15 +93,26 @@ def main(arguments): netbox = api(netbox_host, token=netbox_token, threading=True) # Check if the provided Hostgroup layout is valid hg_objects = hostgroup_format.split("/") - allowed_objects = ["location", "role", "manufacturer", "region", - "site", "site_group", "tenant", "tenant_group"] + allowed_objects = [ + "location", + "role", + "manufacturer", + "region", + "site", + "site_group", + "tenant", + "tenant_group", + ] # Create API call to get all custom fields which are on the device objects try: - device_cfs = list(netbox.extras.custom_fields.filter( - type="text", content_type_id=23)) + device_cfs = list( + netbox.extras.custom_fields.filter(type="text", content_type_id=23) + ) except RequestsConnectionError: - logger.error(f"Unable to connect to NetBox with URL {netbox_host}." - " Please check the URL and status of NetBox.") + logger.error( + f"Unable to connect to NetBox with URL {netbox_host}." + " Please check the URL and status of NetBox." + ) sys.exit(1) except NBRequestError as e: logger.error(f"NetBox error: {e}") @@ -101,8 +121,10 @@ def main(arguments): allowed_objects.append(cf.name) for hg_object in hg_objects: if hg_object not in allowed_objects: - e = (f"Hostgroup item {hg_object} is not valid. Make sure you" - " use valid items and seperate them with '/'.") + e = ( + f"Hostgroup item {hg_object} is not valid. Make sure you" + " use valid items and seperate them with '/'." + ) logger.error(e) raise HostgroupError(e) # Set Zabbix API @@ -114,18 +136,18 @@ def main(arguments): ssl_ctx.load_verify_locations(environ["REQUESTS_CA_BUNDLE"]) if not zabbix_token: - zabbix = ZabbixAPI(zabbix_host, user=zabbix_user, - password=zabbix_pass, ssl_context=ssl_ctx) - else: zabbix = ZabbixAPI( - zabbix_host, token=zabbix_token, ssl_context=ssl_ctx) + zabbix_host, user=zabbix_user, password=zabbix_pass, ssl_context=ssl_ctx + ) + else: + zabbix = ZabbixAPI(zabbix_host, token=zabbix_token, ssl_context=ssl_ctx) zabbix.check_auth() except (APIRequestError, ProcessingError) as e: e = f"Zabbix returned the following error: {str(e)}" logger.error(e) sys.exit(1) # Set API parameter mapping based on API version - if not str(zabbix.version).startswith('7'): + if not str(zabbix.version).startswith("7"): proxy_name = "host" else: proxy_name = "name" @@ -133,23 +155,21 @@ def main(arguments): netbox_devices = list(netbox.dcim.devices.filter(**nb_device_filter)) netbox_vms = [] if sync_vms: - netbox_vms = list( - netbox.virtualization.virtual_machines.filter(**nb_vm_filter)) + netbox_vms = list(netbox.virtualization.virtual_machines.filter(**nb_vm_filter)) netbox_site_groups = convert_recordset((netbox.dcim.site_groups.all())) netbox_regions = convert_recordset(netbox.dcim.regions.all()) netbox_journals = netbox.extras.journal_entries - zabbix_groups = zabbix.hostgroup.get(output=['groupid', 'name']) - zabbix_templates = zabbix.template.get(output=['templateid', 'name']) - zabbix_proxies = zabbix.proxy.get(output=['proxyid', proxy_name]) + zabbix_groups = zabbix.hostgroup.get(output=["groupid", "name"]) + zabbix_templates = zabbix.template.get(output=["templateid", "name"]) + zabbix_proxies = zabbix.proxy.get(output=["proxyid", proxy_name]) # Set empty list for proxy processing Zabbix <= 6 zabbix_proxygroups = [] - if str(zabbix.version).startswith('7'): - zabbix_proxygroups = zabbix.proxygroup.get( - output=["proxy_groupid", "name"]) + if str(zabbix.version).startswith("7"): + zabbix_proxygroups = zabbix.proxygroup.get(output=["proxy_groupid", "name"]) # Sanitize proxy data if proxy_name == "host": for proxy in zabbix_proxies: - proxy['name'] = proxy.pop('host') + proxy["name"] = proxy.pop("host") # Prepare list of all proxy and proxy_groups zabbix_proxy_list = proxy_prepper(zabbix_proxies, zabbix_proxygroups) @@ -159,15 +179,15 @@ def main(arguments): # Go through all NetBox devices for nb_vm in netbox_vms: try: - vm = VirtualMachine(nb_vm, zabbix, netbox_journals, nb_version, - create_journal, logger) + vm = VirtualMachine( + nb_vm, zabbix, netbox_journals, nb_version, create_journal, logger + ) logger.debug(f"Host {vm.name}: Started operations on VM.") vm.set_vm_template() # Check if a valid template has been found for this VM. if not vm.zbx_template_names: continue - vm.set_hostgroup(vm_hostgroup_format, - netbox_site_groups, netbox_regions) + vm.set_hostgroup(vm_hostgroup_format, netbox_site_groups, netbox_regions) # Check if a valid hostgroup has been found for this VM. if not vm.hostgroup: continue @@ -184,17 +204,23 @@ def main(arguments): continue # Device has been added to NetBox # but is not in Activate state - logger.info(f"VM {vm.name}: skipping since this VM is " - f"not in the active state.") + logger.info( + f"VM {vm.name}: skipping since this VM is " + f"not in the active state." + ) continue # Check if the VM is in the disabled state if vm.status in zabbix_device_disable: vm.zabbix_state = 1 # Check if VM is already in Zabbix if vm.zabbix_id: - vm.ConsistencyCheck(zabbix_groups, zabbix_templates, - zabbix_proxy_list, full_proxy_sync, - create_hostgroups) + vm.ConsistencyCheck( + zabbix_groups, + zabbix_templates, + zabbix_proxy_list, + full_proxy_sync, + create_hostgroups, + ) continue # Add hostgroup is config is set if create_hostgroups: @@ -205,24 +231,24 @@ def main(arguments): # Add new hostgroups to zabbix group list zabbix_groups.append(group) # Add VM to Zabbix - vm.createInZabbix(zabbix_groups, zabbix_templates, - zabbix_proxy_list) + vm.createInZabbix(zabbix_groups, zabbix_templates, zabbix_proxy_list) except SyncError: pass for nb_device in netbox_devices: try: # Set device instance set data such as hostgroup and template information. - device = PhysicalDevice(nb_device, zabbix, netbox_journals, nb_version, - create_journal, logger) + device = PhysicalDevice( + nb_device, zabbix, netbox_journals, nb_version, create_journal, logger + ) logger.debug(f"Host {device.name}: started operations on device.") - device.set_template(templates_config_context, - templates_config_context_overrule) + device.set_template( + templates_config_context, templates_config_context_overrule + ) # Check if a valid template has been found for this VM. if not device.zbx_template_names: continue - device.set_hostgroup( - hostgroup_format, netbox_site_groups, netbox_regions) + device.set_hostgroup(hostgroup_format, netbox_site_groups, netbox_regions) # Check if a valid hostgroup has been found for this VM. if not device.hostgroup: continue @@ -234,14 +260,15 @@ def main(arguments): if device.isCluster() and clustering: # Check if device is primary or secondary if device.promoteMasterDevice(): - e = (f"Device {device.name}: is " - f"part of cluster and primary.") + e = f"Device {device.name}: is " f"part of cluster and primary." logger.info(e) else: # Device is secondary in cluster. # Don't continue with this device. - e = (f"Device {device.name}: is part of cluster " - f"but not primary. Skipping this host...") + e = ( + f"Device {device.name}: is part of cluster " + f"but not primary. Skipping this host..." + ) logger.info(e) continue # Checks if device is in cleanup state @@ -254,17 +281,23 @@ def main(arguments): continue # Device has been added to NetBox # but is not in Activate state - logger.info(f"Device {device.name}: skipping since this device is " - f"not in the active state.") + logger.info( + f"Device {device.name}: skipping since this device is " + f"not in the active state." + ) continue # Check if the device is in the disabled state if device.status in zabbix_device_disable: device.zabbix_state = 1 # Check if device is already in Zabbix if device.zabbix_id: - device.ConsistencyCheck(zabbix_groups, zabbix_templates, - zabbix_proxy_list, full_proxy_sync, - create_hostgroups) + device.ConsistencyCheck( + zabbix_groups, + zabbix_templates, + zabbix_proxy_list, + full_proxy_sync, + create_hostgroups, + ) continue # Add hostgroup is config is set if create_hostgroups: @@ -275,17 +308,17 @@ def main(arguments): # Add new hostgroups to zabbix group list zabbix_groups.append(group) # Add device to Zabbix - device.createInZabbix(zabbix_groups, zabbix_templates, - zabbix_proxy_list) + device.createInZabbix(zabbix_groups, zabbix_templates, zabbix_proxy_list) except SyncError: pass if __name__ == "__main__": parser = argparse.ArgumentParser( - description='A script to sync Zabbix with NetBox device data.' + description="A script to sync Zabbix with NetBox device data." + ) + parser.add_argument( + "-v", "--verbose", help="Turn on debugging.", action="store_true" ) - parser.add_argument("-v", "--verbose", help="Turn on debugging.", - action="store_true") args = parser.parse_args() main(args) From 53066d2d51dd177b2c9cc801d3d5dcfcacef6a88 Mon Sep 17 00:00:00 2001 From: Wouter de Bruijn Date: Wed, 26 Feb 2025 10:09:35 +0100 Subject: [PATCH 02/18] =?UTF-8?q?=E2=9C=A8=20Added=20separate=20log=20leve?= =?UTF-8?q?ls?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- netbox_zabbix_sync.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index b355657..5d6606e 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -41,23 +41,18 @@ except ModuleNotFoundError: sys.exit(1) # Set logging -log_format = logging.Formatter( - "%(asctime)s - %(name)s - " "%(levelname)s - %(message)s" -) lgout = logging.StreamHandler() -lgout.setFormatter(log_format) -lgout.setLevel(logging.DEBUG) - lgfile = logging.FileHandler( path.join(path.dirname(path.realpath(__file__)), "sync.log") ) -lgfile.setFormatter(log_format) -lgfile.setLevel(logging.DEBUG) + +logging.basicConfig( + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + level=logging.WARNING, + handlers=[lgout, lgfile], +) logger = logging.getLogger("NetBox-Zabbix-sync") -logger.addHandler(lgout) -logger.addHandler(lgfile) -logger.setLevel(logging.WARNING) def main(arguments): @@ -65,7 +60,11 @@ def main(arguments): # pylint: disable=too-many-branches, too-many-statements # set environment variables if arguments.verbose: + logger.setLevel(logging.INFO) + if arguments.debug: logger.setLevel(logging.DEBUG) + if arguments.debug_all: + logging.getLogger().setLevel(logging.DEBUG) env_vars = ["ZABBIX_HOST", "NETBOX_HOST", "NETBOX_TOKEN"] if "ZABBIX_TOKEN" in environ: env_vars.append("ZABBIX_TOKEN") @@ -320,5 +319,14 @@ if __name__ == "__main__": parser.add_argument( "-v", "--verbose", help="Turn on debugging.", action="store_true" ) + parser.add_argument( + "-vv", "--debug", help="Turn on debugging.", action="store_true" + ) + parser.add_argument( + "-vvv", + "--debug-all", + help="Turn on debugging for all modules.", + action="store_true", + ) args = parser.parse_args() main(args) From a5312365f9ccc3ee5aaf2ddb6421b66cd18eb0f0 Mon Sep 17 00:00:00 2001 From: Wouter de Bruijn Date: Wed, 26 Feb 2025 10:11:47 +0100 Subject: [PATCH 03/18] =?UTF-8?q?=F0=9F=93=84=20Added=20new=20cli=20params?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d533ec6..0bd6bfc 100644 --- a/README.md +++ b/README.md @@ -575,9 +575,11 @@ python3 netbox_zabbix_sync.py ### Flags -| Flag | Option | Description | -| ---- | ------- | ---------------------- | -| -v | verbose | Log with debugging on. | +| Flag | Option | Description | +| ---- | --------- | ------------------------------------- | +| -v | verbose | Log with info on. | +| -vv | debug | Log with debugging on. | +| -vvv | debug-all | Log with debugging on for all modules | ## Config context From 0c798ec96890da56a7d39d40e5314070217238e9 Mon Sep 17 00:00:00 2001 From: Wouter de Bruijn Date: Wed, 26 Feb 2025 11:10:56 +0100 Subject: [PATCH 04/18] =?UTF-8?q?=E2=9C=A8=20Added=20quiet=20param?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- netbox_zabbix_sync.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index 5d6606e..75ad65c 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -65,6 +65,9 @@ def main(arguments): logger.setLevel(logging.DEBUG) if arguments.debug_all: logging.getLogger().setLevel(logging.DEBUG) + if arguments.quiet: + logging.getLogger().setLevel(logging.ERROR) + env_vars = ["ZABBIX_HOST", "NETBOX_HOST", "NETBOX_TOKEN"] if "ZABBIX_TOKEN" in environ: env_vars.append("ZABBIX_TOKEN") @@ -328,5 +331,6 @@ if __name__ == "__main__": help="Turn on debugging for all modules.", action="store_true", ) + parser.add_argument("-q", "--quiet", help="Turn off warnings.", action="store_true") args = parser.parse_args() main(args) From b314b2c8836a4d0a978bf83407c5ce35ab83c97e Mon Sep 17 00:00:00 2001 From: Wouter de Bruijn Date: Wed, 26 Feb 2025 14:00:18 +0100 Subject: [PATCH 05/18] =?UTF-8?q?=F0=9F=9A=A8=20Formatted=20and=20linted?= =?UTF-8?q?=20files?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/device.py | 452 +++++++++++++++++++++++-------------- modules/exceptions.py | 34 ++- modules/hostgroups.py | 82 +++++-- modules/interface.py | 44 ++-- modules/tags.py | 54 +++-- modules/tools.py | 44 ++-- modules/usermacros.py | 63 ++++-- modules/virtual_machine.py | 45 ++-- 8 files changed, 514 insertions(+), 304 deletions(-) diff --git a/modules/device.py b/modules/device.py index 4ec96b5..83e6fdc 100644 --- a/modules/device.py +++ b/modules/device.py @@ -3,48 +3,61 @@ """ Device specific handeling for NetBox to Zabbix """ -from os import sys -from re import search from copy import deepcopy from logging import getLogger +from os import sys +from re import search + from zabbix_utils import APIRequestError -from modules.exceptions import (SyncInventoryError, TemplateError, SyncExternalError, - InterfaceConfigError, JournalError) -from modules.interface import ZabbixInterface -from modules.usermacros import ZabbixUsermacros -from modules.tags import ZabbixTags + +from modules.exceptions import ( + InterfaceConfigError, + JournalError, + SyncExternalError, + SyncInventoryError, + TemplateError, +) from modules.hostgroups import Hostgroup +from modules.interface import ZabbixInterface +from modules.tags import ZabbixTags from modules.tools import field_mapper, remove_duplicates +from modules.usermacros import ZabbixUsermacros try: from config import ( - template_cf, device_cf, - traverse_site_groups, - traverse_regions, - inventory_sync, - inventory_mode, + device_cf, device_inventory_map, - usermacro_sync, + device_tag_map, device_usermacro_map, - tag_sync, + inventory_mode, + inventory_sync, tag_lower, tag_name, + tag_sync, tag_value, - device_tag_map + template_cf, + traverse_regions, + traverse_site_groups, + usermacro_sync, ) except ModuleNotFoundError: - print("Configuration file config.py not found in main directory." - "Please create the file or rename the config.py.example file to config.py.") + print( + "Configuration file config.py not found in main directory." + "Please create the file or rename the config.py.example file to config.py." + ) sys.exit(0) -class PhysicalDevice(): + +class PhysicalDevice: # pylint: disable=too-many-instance-attributes, too-many-arguments, too-many-positional-arguments """ Represents Network device. INPUT: (NetBox device class, ZabbixAPI class, journal flag, NB journal class) """ - def __init__(self, nb, zabbix, nb_journal_class, nb_version, journal=None, logger=None): + def __init__( + self, nb, zabbix, nb_journal_class, nb_version, journal=None, logger=None + ): self.nb = nb self.id = nb.id self.name = nb.name @@ -77,15 +90,15 @@ class PhysicalDevice(): return self.__repr__() def _inventory_map(self): - """ Use device inventory maps """ + """Use device inventory maps""" return device_inventory_map def _usermacro_map(self): - """ Use device inventory maps """ + """Use device inventory maps""" return device_usermacro_map def _tag_map(self): - """ Use device host tag maps """ + """Use device host tag maps""" return device_tag_map def _setBasics(self): @@ -112,30 +125,38 @@ class PhysicalDevice(): # Validate hostname format. odd_character_list = ["ä", "ö", "ü", "Ä", "Ö", "Ü", "ß"] self.use_visible_name = False - if (any(letter in self.name for letter in odd_character_list) or - bool(search('[\u0400-\u04FF]', self.name))): + if any(letter in self.name for letter in odd_character_list) or bool( + search("[\u0400-\u04ff]", self.name) + ): self.name = f"NETBOX_ID{self.id}" self.visible_name = self.nb.name self.use_visible_name = True - self.logger.info(f"Host {self.visible_name} contains special characters. " - f"Using {self.name} as name for the NetBox object " - f"and using {self.visible_name} as visible name in Zabbix.") + self.logger.info( + f"Host {self.visible_name} contains special characters. " + f"Using {self.name} as name for the NetBox object " + f"and using {self.visible_name} as visible name in Zabbix." + ) else: pass def set_hostgroup(self, hg_format, nb_site_groups, nb_regions): """Set the hostgroup for this device""" # Create new Hostgroup instance - hg = Hostgroup("dev", self.nb, self.nb_api_version, logger=self.logger, - nested_sitegroup_flag=traverse_site_groups, - nested_region_flag=traverse_regions, - nb_groups=nb_site_groups, - nb_regions=nb_regions) + hg = Hostgroup( + "dev", + self.nb, + self.nb_api_version, + logger=self.logger, + nested_sitegroup_flag=traverse_site_groups, + nested_region_flag=traverse_regions, + nb_groups=nb_site_groups, + nb_regions=nb_regions, + ) # Generate hostgroup based on hostgroup format self.hostgroup = hg.generate(hg_format) def set_template(self, prefer_config_context, overrule_custom): - """ Set Template """ + """Set Template""" self.zbx_template_names = None # Gather templates ONLY from the device specific context if prefer_config_context: @@ -159,7 +180,7 @@ class PhysicalDevice(): return True def get_templates_cf(self): - """ Get template from custom field """ + """Get template from custom field""" # Get Zabbix templates from the device type device_type_cfs = self.nb.device_type.custom_fields # Check if the ZBX Template CF is present @@ -167,20 +188,26 @@ class PhysicalDevice(): # Set value to template return [device_type_cfs[template_cf]] # Custom field not found, return error - e = (f"Custom field {template_cf} not " + e = ( + f"Custom field {template_cf} not " f"found for {self.nb.device_type.manufacturer.name}" - f" - {self.nb.device_type.display}.") + f" - {self.nb.device_type.display}." + ) raise TemplateError(e) def get_templates_context(self): - """ Get Zabbix templates from the device context """ + """Get Zabbix templates from the device context""" if "zabbix" not in self.config_context: - e = (f"Host {self.name}: Key 'zabbix' not found in config " - "context for template lookup") + e = ( + f"Host {self.name}: Key 'zabbix' not found in config " + "context for template lookup" + ) raise TemplateError(e) if "templates" not in self.config_context["zabbix"]: - e = (f"Host {self.name}: Key 'templates' not found in config " - "context 'zabbix' for template lookup") + e = ( + f"Host {self.name}: Key 'templates' not found in config " + "context 'zabbix' for template lookup" + ) raise TemplateError(e) # Check if format is list or string. if isinstance(self.config_context["zabbix"]["templates"], str): @@ -188,25 +215,31 @@ class PhysicalDevice(): return self.config_context["zabbix"]["templates"] def set_inventory(self, nbdevice): - """ Set host inventory """ + """Set host inventory""" # Set inventory mode. Default is disabled (see class init function). if inventory_mode == "disabled": if inventory_sync: - self.logger.error(f"Host {self.name}: Unable to map NetBox inventory to Zabbix. " - "Inventory sync is enabled in config but inventory mode is disabled.") + self.logger.error( + f"Host {self.name}: Unable to map NetBox inventory to Zabbix. " + "Inventory sync is enabled in config but inventory mode is disabled." + ) return True if inventory_mode == "manual": self.inventory_mode = 0 elif inventory_mode == "automatic": self.inventory_mode = 1 else: - self.logger.error(f"Host {self.name}: Specified value for inventory mode in" - f" config is not valid. Got value {inventory_mode}") + self.logger.error( + f"Host {self.name}: Specified value for inventory mode in" + f" config is not valid. Got value {inventory_mode}" + ) return False self.inventory = {} - if inventory_sync and self.inventory_mode in [0,1]: + if inventory_sync and self.inventory_mode in [0, 1]: self.logger.debug(f"Host {self.name}: Starting inventory mapper") - self.inventory = field_mapper(self.name, self._inventory_map(), nbdevice, self.logger) + self.inventory = field_mapper( + self.name, self._inventory_map(), nbdevice, self.logger + ) return True def isCluster(self): @@ -220,13 +253,17 @@ class PhysicalDevice(): Returns chassis master ID. """ if not self.isCluster(): - e = (f"Unable to proces {self.name} for cluster calculation: " - f"not part of a cluster.") + e = ( + f"Unable to proces {self.name} for cluster calculation: " + f"not part of a cluster." + ) self.logger.warning(e) raise SyncInventoryError(e) if not self.nb.virtual_chassis.master: - e = (f"{self.name} is part of a NetBox virtual chassis which does " - "not have a master configured. Skipping for this reason.") + e = ( + f"{self.name} is part of a NetBox virtual chassis which does " + "not have a master configured. Skipping for this reason." + ) self.logger.error(e) raise SyncInventoryError(e) return self.nb.virtual_chassis.master.id @@ -239,9 +276,11 @@ class PhysicalDevice(): """ masterid = self.getClusterMaster() if masterid == self.id: - self.logger.debug(f"Host {self.name} is primary cluster member. " - f"Modifying hostname from {self.name} to " + - f"{self.nb.virtual_chassis.name}.") + self.logger.debug( + f"Host {self.name} is primary cluster member. " + f"Modifying hostname from {self.name} to " + + f"{self.nb.virtual_chassis.name}." + ) self.name = self.nb.virtual_chassis.name return True self.logger.debug(f"Host {self.name} is non-primary cluster member.") @@ -266,18 +305,24 @@ class PhysicalDevice(): # Go through all templates found in Zabbix for zbx_template in templates: # If the template names match - if zbx_template['name'] == nb_template: + if zbx_template["name"] == nb_template: # Set match variable to true, add template details # to class variable and return debug log template_match = True - self.zbx_templates.append({"templateid": zbx_template['templateid'], - "name": zbx_template['name']}) + self.zbx_templates.append( + { + "templateid": zbx_template["templateid"], + "name": zbx_template["name"], + } + ) e = f"Host {self.name}: found template {zbx_template['name']}" self.logger.debug(e) # Return error should the template not be found in Zabbix if not template_match: - e = (f"Unable to find template {nb_template} " - f"for host {self.name} in Zabbix. Skipping host...") + e = ( + f"Unable to find template {nb_template} " + f"for host {self.name} in Zabbix. Skipping host..." + ) self.logger.warning(e) raise SyncInventoryError(e) @@ -289,8 +334,8 @@ class PhysicalDevice(): """ # Go through all groups for group in groups: - if group['name'] == self.hostgroup: - self.group_id = group['groupid'] + if group["name"] == self.hostgroup: + self.group_id = group["groupid"] e = f"Host {self.name}: matched group {group['name']}" self.logger.debug(e) return True @@ -304,10 +349,13 @@ class PhysicalDevice(): if self.zabbix_id: try: # Check if the Zabbix host exists in Zabbix - zbx_host = bool(self.zabbix.host.get(filter={'hostid': self.zabbix_id}, - output=[])) - e = (f"Host {self.name}: was already deleted from Zabbix." - " Removed link in NetBox.") + zbx_host = bool( + self.zabbix.host.get(filter={"hostid": self.zabbix_id}, output=[]) + ) + e = ( + f"Host {self.name}: was already deleted from Zabbix." + " Removed link in NetBox." + ) if zbx_host: # Delete host should it exists self.zabbix.host.delete(self.zabbix_id) @@ -332,9 +380,9 @@ class PhysicalDevice(): """ # Validate the hostname or visible name field if not self.use_visible_name: - zbx_filter = {'host': self.name} + zbx_filter = {"host": self.name} else: - zbx_filter = {'name': self.visible_name} + zbx_filter = {"name": self.visible_name} host = self.zabbix.host.get(filter=zbx_filter, output=[]) return bool(host) @@ -364,24 +412,33 @@ class PhysicalDevice(): """ Generates Usermacros """ - macros = ZabbixUsermacros(self.nb, self._usermacro_map(), - usermacro_sync, logger=self.logger, - host=self.name) + macros = ZabbixUsermacros( + self.nb, + self._usermacro_map(), + usermacro_sync, + logger=self.logger, + host=self.name, + ) if macros.sync is False: self.usermacros = [] self.usermacros = macros.generate() return True - def set_tags(self): """ Generates Host Tags """ - tags = ZabbixTags(self.nb, self._tag_map(), - tag_sync, tag_lower, tag_name=tag_name, - tag_value=tag_value, logger=self.logger, - host=self.name) + tags = ZabbixTags( + self.nb, + self._tag_map(), + tag_sync, + tag_lower, + tag_name=tag_name, + tag_value=tag_value, + logger=self.logger, + host=self.name, + ) if tags.sync is False: self.tags = [] @@ -398,14 +455,16 @@ class PhysicalDevice(): # check if the key Zabbix is defined in the config context if not "zabbix" in self.nb.config_context: return False - if ("proxy" in self.nb.config_context["zabbix"] and - not self.nb.config_context["zabbix"]["proxy"]): + if ( + "proxy" in self.nb.config_context["zabbix"] + and not self.nb.config_context["zabbix"]["proxy"] + ): return False # Proxy group takes priority over a proxy due # to it being HA and therefore being more reliable # Includes proxy group fix since Zabbix <= 6 should ignore this proxy_types = ["proxy"] - if str(self.zabbix.version).startswith('7'): + if str(self.zabbix.version).startswith("7"): # Only insert groups in front of list for Zabbix7 proxy_types.insert(0, "proxy_group") for proxy_type in proxy_types: @@ -419,15 +478,23 @@ class PhysicalDevice(): continue # If the proxy name matches if proxy["name"] == proxy_name: - self.logger.debug(f"Host {self.name}: using {proxy['type']}" - f" {proxy_name}") + self.logger.debug( + f"Host {self.name}: using {proxy['type']}" f" {proxy_name}" + ) self.zbxproxy = proxy return True - self.logger.warning(f"Host {self.name}: unable to find proxy {proxy_name}") + self.logger.warning( + f"Host {self.name}: unable to find proxy {proxy_name}" + ) return False - def createInZabbix(self, groups, templates, proxies, - description="Host added by NetBox sync script."): + def createInZabbix( + self, + groups, + templates, + proxies, + description="Host added by NetBox sync script.", + ): """ Creates Zabbix host object with parameters from NetBox object. """ @@ -435,37 +502,40 @@ class PhysicalDevice(): if not self._zabbixHostnameExists(): # Set group and template ID's for host if not self.setZabbixGroupID(groups): - e = (f"Unable to find group '{self.hostgroup}' " - f"for host {self.name} in Zabbix.") + e = ( + f"Unable to find group '{self.hostgroup}' " + f"for host {self.name} in Zabbix." + ) self.logger.warning(e) raise SyncInventoryError(e) self.zbxTemplatePrepper(templates) templateids = [] for template in self.zbx_templates: - templateids.append({'templateid': template['templateid']}) + templateids.append({"templateid": template["templateid"]}) # Set interface, group and template configuration interfaces = self.setInterfaceDetails() groups = [{"groupid": self.group_id}] # Set Zabbix proxy if defined self.setProxy(proxies) # Set basic data for host creation - create_data = {"host": self.name, - "name": self.visible_name, - "status": self.zabbix_state, - "interfaces": interfaces, - "groups": groups, - "templates": templateids, - "description": description, - "inventory_mode": self.inventory_mode, - "inventory": self.inventory, - "macros": self.usermacros, - "tags": self.tags - } + create_data = { + "host": self.name, + "name": self.visible_name, + "status": self.zabbix_state, + "interfaces": interfaces, + "groups": groups, + "templates": templateids, + "description": description, + "inventory_mode": self.inventory_mode, + "inventory": self.inventory, + "macros": self.usermacros, + "tags": self.tags, + } # If a Zabbix proxy or Zabbix Proxy group has been defined if self.zbxproxy: # If a lower version than 7 is used, we can assume that # the proxy is a normal proxy and not a proxy group - if not str(self.zabbix.version).startswith('7'): + if not str(self.zabbix.version).startswith("7"): create_data["proxy_hostid"] = self.zbxproxy["id"] else: # Configure either a proxy or proxy group @@ -496,8 +566,8 @@ class PhysicalDevice(): """ final_data = [] # Check if the hostgroup is in a nested format and check each parent - for pos in range(len(self.hostgroup.split('/'))): - zabbix_hg = self.hostgroup.rsplit('/', pos)[0] + for pos in range(len(self.hostgroup.split("/"))): + zabbix_hg = self.hostgroup.rsplit("/", pos)[0] if self.lookupZabbixHostgroup(hostgroups, zabbix_hg): # Hostgroup already exists continue @@ -508,7 +578,9 @@ class PhysicalDevice(): e = f"Hostgroup '{zabbix_hg}': created in Zabbix." self.logger.info(e) # Add group to final data - final_data.append({'groupid': groupid["groupids"][0], 'name': zabbix_hg}) + final_data.append( + {"groupid": groupid["groupids"][0], "name": zabbix_hg} + ) except APIRequestError as e: msg = f"Hostgroup '{zabbix_hg}': unable to create. Zabbix returned {str(e)}." self.logger.error(msg) @@ -535,20 +607,24 @@ class PhysicalDevice(): try: self.zabbix.host.update(hostid=self.zabbix_id, **kwargs) except APIRequestError as e: - e = (f"Host {self.name}: Unable to update. " - f"Zabbix returned the following error: {str(e)}.") + e = ( + f"Host {self.name}: Unable to update. " + f"Zabbix returned the following error: {str(e)}." + ) self.logger.error(e) raise SyncExternalError(e) from None self.logger.info(f"Updated host {self.name} with data {kwargs}.") self.create_journal_entry("info", "Updated host in Zabbix with latest NB data.") - def ConsistencyCheck(self, groups, templates, proxies, proxy_power, create_hostgroups): + def ConsistencyCheck( + self, groups, templates, proxies, proxy_power, create_hostgroups + ): # pylint: disable=too-many-branches, too-many-statements """ Checks if Zabbix object is still valid with NetBox parameters. """ # If group is found or if the hostgroup is nested - if not self.setZabbixGroupID(groups) or len(self.hostgroup.split('/')) > 1: + if not self.setZabbixGroupID(groups) or len(self.hostgroup.split("/")) > 1: if create_hostgroups: # Script is allowed to create a new hostgroup new_groups = self.createZabbixHostgroup(groups) @@ -559,50 +635,59 @@ class PhysicalDevice(): if not self.group_id: # Function returns true / false but also sets GroupID if not self.setZabbixGroupID(groups) and not create_hostgroups: - e = (f"Host {self.name}: different hostgroup is required but " - "unable to create hostgroup without generation permission.") + e = ( + f"Host {self.name}: different hostgroup is required but " + "unable to create hostgroup without generation permission." + ) self.logger.warning(e) raise SyncInventoryError(e) # Prepare templates and proxy config self.zbxTemplatePrepper(templates) self.setProxy(proxies) # Get host object from Zabbix - host = self.zabbix.host.get(filter={'hostid': self.zabbix_id}, - selectInterfaces=['type', 'ip', - 'port', 'details', - 'interfaceid'], - selectGroups=["groupid"], - selectHostGroups=["groupid"], - selectParentTemplates=["templateid"], - selectInventory=list(self._inventory_map().values()), - selectMacros=["macro","value","type","description"], - selectTags=["tag","value"] - ) + host = self.zabbix.host.get( + filter={"hostid": self.zabbix_id}, + selectInterfaces=["type", "ip", "port", "details", "interfaceid"], + selectGroups=["groupid"], + selectHostGroups=["groupid"], + selectParentTemplates=["templateid"], + selectInventory=list(self._inventory_map().values()), + selectMacros=["macro", "value", "type", "description"], + selectTags=["tag", "value"], + ) if len(host) > 1: - e = (f"Got {len(host)} results for Zabbix hosts " - f"with ID {self.zabbix_id} - hostname {self.name}.") + e = ( + f"Got {len(host)} results for Zabbix hosts " + f"with ID {self.zabbix_id} - hostname {self.name}." + ) self.logger.error(e) raise SyncInventoryError(e) if len(host) == 0: - e = (f"Host {self.name}: No Zabbix host found. " - f"This is likely the result of a deleted Zabbix host " - f"without zeroing the ID field in NetBox.") + e = ( + f"Host {self.name}: No Zabbix host found. " + f"This is likely the result of a deleted Zabbix host " + f"without zeroing the ID field in NetBox." + ) self.logger.error(e) raise SyncInventoryError(e) host = host[0] if host["host"] == self.name: self.logger.debug(f"Host {self.name}: hostname in-sync.") else: - self.logger.warning(f"Host {self.name}: hostname OUT of sync. " - f"Received value: {host['host']}") + self.logger.warning( + f"Host {self.name}: hostname OUT of sync. " + f"Received value: {host['host']}" + ) self.updateZabbixHost(host=self.name) # Execute check depending on wether the name is special or not if self.use_visible_name: if host["name"] == self.visible_name: self.logger.debug(f"Host {self.name}: visible name in-sync.") else: - self.logger.warning(f"Host {self.name}: visible name OUT of sync." - f" Received value: {host['name']}") + self.logger.warning( + f"Host {self.name}: visible name OUT of sync." + f" Received value: {host['name']}" + ) self.updateZabbixHost(name=self.visible_name) # Check if the templates are in-sync @@ -611,23 +696,24 @@ class PhysicalDevice(): # Prepare Templates for API parsing templateids = [] for template in self.zbx_templates: - templateids.append({'templateid': template['templateid']}) + templateids.append({"templateid": template["templateid"]}) # Update Zabbix with NB templates and clear any old / lost templates - self.updateZabbixHost(templates_clear=host["parentTemplates"], - templates=templateids) + self.updateZabbixHost( + templates_clear=host["parentTemplates"], templates=templateids + ) else: self.logger.debug(f"Host {self.name}: template(s) in-sync.") # Check if Zabbix version is 6 or higher. Issue #93 group_dictname = "hostgroups" - if str(self.zabbix.version).startswith(('6', '5')): + if str(self.zabbix.version).startswith(("6", "5")): group_dictname = "groups" for group in host[group_dictname]: if group["groupid"] == self.group_id: self.logger.debug(f"Host {self.name}: hostgroup in-sync.") break self.logger.warning(f"Host {self.name}: hostgroup OUT of sync.") - self.updateZabbixHost(groups={'groupid': self.group_id}) + self.updateZabbixHost(groups={"groupid": self.group_id}) if int(host["status"]) == self.zabbix_state: self.logger.debug(f"Host {self.name}: status in-sync.") @@ -637,8 +723,10 @@ class PhysicalDevice(): # Check if a proxy has been defined if self.zbxproxy: # Check if proxy or proxy group is defined - if (self.zbxproxy["idtype"] in host and - host[self.zbxproxy["idtype"]] == self.zbxproxy["id"]): + if ( + self.zbxproxy["idtype"] in host + and host[self.zbxproxy["idtype"]] == self.zbxproxy["id"] + ): self.logger.debug(f"Host {self.name}: proxy in-sync.") # Backwards compatibility for Zabbix <= 6 elif "proxy_hostid" in host and host["proxy_hostid"] == self.zbxproxy["id"]: @@ -647,13 +735,15 @@ class PhysicalDevice(): else: self.logger.warning(f"Host {self.name}: proxy OUT of sync.") # Zabbix <= 6 patch - if not str(self.zabbix.version).startswith('7'): - self.updateZabbixHost(proxy_hostid=self.zbxproxy['id']) + if not str(self.zabbix.version).startswith("7"): + self.updateZabbixHost(proxy_hostid=self.zbxproxy["id"]) # Zabbix 7+ else: # Prepare data structure for updating either proxy or group - update_data = {self.zbxproxy["idtype"]: self.zbxproxy["id"], - "monitored_by": self.zbxproxy['monitored_by']} + update_data = { + self.zbxproxy["idtype"]: self.zbxproxy["id"], + "monitored_by": self.zbxproxy["monitored_by"], + } self.updateZabbixHost(**update_data) else: # No proxy is defined in NetBox @@ -665,8 +755,10 @@ class PhysicalDevice(): proxy_set = True if proxy_power and proxy_set: # Zabbix <= 6 fix - self.logger.warning(f"Host {self.name}: no proxy is configured in NetBox " - "but is configured in Zabbix. Removing proxy config in Zabbix") + self.logger.warning( + f"Host {self.name}: no proxy is configured in NetBox " + "but is configured in Zabbix. Removing proxy config in Zabbix" + ) if "proxy_hostid" in host and bool(host["proxy_hostid"]): self.updateZabbixHost(proxy_hostid=0) # Zabbix 7 proxy @@ -678,21 +770,23 @@ class PhysicalDevice(): # Checks if a proxy has been defined in Zabbix and if proxy_power config has been set if proxy_set and not proxy_power: # Display error message - self.logger.error(f"Host {self.name} is configured " - f"with proxy in Zabbix but not in NetBox. The" - " -p flag was ommited: no " - "changes have been made.") + self.logger.error( + f"Host {self.name} is configured " + f"with proxy in Zabbix but not in NetBox. The" + " -p flag was ommited: no " + "changes have been made." + ) if not proxy_set: self.logger.debug(f"Host {self.name}: proxy in-sync.") # Check host inventory mode - if str(host['inventory_mode']) == str(self.inventory_mode): + if str(host["inventory_mode"]) == str(self.inventory_mode): self.logger.debug(f"Host {self.name}: inventory_mode in-sync.") else: self.logger.warning(f"Host {self.name}: inventory_mode OUT of sync.") self.updateZabbixHost(inventory_mode=str(self.inventory_mode)) - if inventory_sync and self.inventory_mode in [0,1]: + if inventory_sync and self.inventory_mode in [0, 1]: # Check host inventory mapping - if host['inventory'] == self.inventory: + if host["inventory"] == self.inventory: self.logger.debug(f"Host {self.name}: inventory in-sync.") else: self.logger.warning(f"Host {self.name}: inventory OUT of sync.") @@ -704,12 +798,12 @@ class PhysicalDevice(): # Do not re-sync secret usermacros unless sync is set to 'full' if str(usermacro_sync).lower() != "full": for m in deepcopy(self.usermacros): - if m['type'] == str(1): + if m["type"] == str(1): # Remove the value as the api doesn't return it # this will allow us to only update usermacros that don't exist - m.pop('value') + m.pop("value") macros_filtered.append(m) - if host['macros'] == self.usermacros or host['macros'] == macros_filtered: + if host["macros"] == self.usermacros or host["macros"] == macros_filtered: self.logger.debug(f"Host {self.name}: usermacros in-sync.") else: self.logger.warning(f"Host {self.name}: usermacros OUT of sync.") @@ -717,7 +811,7 @@ class PhysicalDevice(): # Check host usermacros if tag_sync: - if remove_duplicates(host['tags'],sortkey='tag') == self.tags: + if remove_duplicates(host["tags"], sortkey="tag") == self.tags: self.logger.debug(f"Host {self.name}: tags in-sync.") else: self.logger.warning(f"Host {self.name}: tags OUT of sync.") @@ -725,7 +819,7 @@ class PhysicalDevice(): # If only 1 interface has been found # pylint: disable=too-many-nested-blocks - if len(host['interfaces']) == 1: + if len(host["interfaces"]) == 1: updates = {} # Go through each key / item and check if it matches Zabbix for key, item in self.setInterfaceDetails()[0].items(): @@ -733,7 +827,7 @@ class PhysicalDevice(): if key in host["interfaces"][0]: # If SNMP is used, go through nested dict # to compare SNMP parameters - if isinstance(item,dict) and key == "details": + if isinstance(item, dict) and key == "details": for k, i in item.items(): if k in host["interfaces"][0][key]: # Set update if values don't match @@ -761,12 +855,14 @@ class PhysicalDevice(): self.logger.warning(f"Host {self.name}: Interface OUT of sync.") if "type" in updates: # Changing interface type not supported. Raise exception. - e = (f"Host {self.name}: changing interface type to " - f"{str(updates['type'])} is not supported.") + e = ( + f"Host {self.name}: changing interface type to " + f"{str(updates['type'])} is not supported." + ) self.logger.error(e) raise InterfaceConfigError(e) # Set interfaceID for Zabbix config - updates["interfaceid"] = host["interfaces"][0]['interfaceid'] + updates["interfaceid"] = host["interfaces"][0]["interfaceid"] try: # API call to Zabbix self.zabbix.hostinterface.update(updates) @@ -782,9 +878,11 @@ class PhysicalDevice(): e = f"Host {self.name}: interface in-sync." self.logger.debug(e) else: - e = (f"Host {self.name} has unsupported interface configuration." - f" Host has total of {len(host['interfaces'])} interfaces. " - "Manual interfention required.") + e = ( + f"Host {self.name} has unsupported interface configuration." + f" Host has total of {len(host['interfaces'])} interfaces. " + "Manual interfention required." + ) self.logger.error(e) raise SyncInventoryError(e) @@ -796,20 +894,25 @@ class PhysicalDevice(): if self.journal: # Check if the severity is valid if severity not in ["info", "success", "warning", "danger"]: - self.logger.warning(f"Value {severity} not valid for NB journal entries.") + self.logger.warning( + f"Value {severity} not valid for NB journal entries." + ) return False - journal = {"assigned_object_type": "dcim.device", - "assigned_object_id": self.id, - "kind": severity, - "comments": message - } + journal = { + "assigned_object_type": "dcim.device", + "assigned_object_id": self.id, + "kind": severity, + "comments": message, + } try: self.nb_journals.create(journal) self.logger.debug(f"Host {self.name}: Created journal entry in NetBox") return True except JournalError(e) as e: - self.logger.warning("Unable to create journal entry for " - f"{self.name}: NB returned {e}") + self.logger.warning( + "Unable to create journal entry for " + f"{self.name}: NB returned {e}" + ) return False return False @@ -832,10 +935,15 @@ class PhysicalDevice(): # and add this NB template to the list of successfull templates tmpls_from_zabbix.pop(pos) succesfull_templates.append(nb_tmpl) - self.logger.debug(f"Host {self.name}: template " - f"{nb_tmpl['name']} is present in Zabbix.") + self.logger.debug( + f"Host {self.name}: template " + f"{nb_tmpl['name']} is present in Zabbix." + ) break - if len(succesfull_templates) == len(self.zbx_templates) and len(tmpls_from_zabbix) == 0: + if ( + len(succesfull_templates) == len(self.zbx_templates) + and len(tmpls_from_zabbix) == 0 + ): # All of the NetBox templates have been confirmed as successfull # and the ZBX template list is empty. This means that # all of the templates match. diff --git a/modules/exceptions.py b/modules/exceptions.py index 27a141c..ddac2b0 100644 --- a/modules/exceptions.py +++ b/modules/exceptions.py @@ -2,35 +2,47 @@ """ All custom exceptions used for Exception generation """ + + class SyncError(Exception): - """ Class SyncError """ + """Class SyncError""" + class JournalError(Exception): - """ Class SyncError """ + """Class SyncError""" + class SyncExternalError(SyncError): - """ Class SyncExternalError """ + """Class SyncExternalError""" + class SyncInventoryError(SyncError): - """ Class SyncInventoryError """ + """Class SyncInventoryError""" + class SyncDuplicateError(SyncError): - """ Class SyncDuplicateError """ + """Class SyncDuplicateError""" + class EnvironmentVarError(SyncError): - """ Class EnvironmentVarError """ + """Class EnvironmentVarError""" + class InterfaceConfigError(SyncError): - """ Class InterfaceConfigError """ + """Class InterfaceConfigError""" + class ProxyConfigError(SyncError): - """ Class ProxyConfigError """ + """Class ProxyConfigError""" + class HostgroupError(SyncError): - """ Class HostgroupError """ + """Class HostgroupError""" + class TemplateError(SyncError): - """ Class TemplateError """ + """Class TemplateError""" + class UsermacroError(SyncError): - """ Class UsermacroError """ + """Class UsermacroError""" diff --git a/modules/hostgroups.py b/modules/hostgroups.py index 6e2db75..c67f5e6 100644 --- a/modules/hostgroups.py +++ b/modules/hostgroups.py @@ -1,14 +1,26 @@ """Module for all hostgroup related code""" + from logging import getLogger + from modules.exceptions import HostgroupError from modules.tools import build_path -class Hostgroup(): + +class Hostgroup: """Hostgroup class for devices and VM's Takes type (vm or dev) and NB object""" - def __init__(self, obj_type, nb_obj, version, logger=None, #pylint: disable=too-many-arguments, too-many-positional-arguments - nested_sitegroup_flag=False, nested_region_flag=False, - nb_regions=None, nb_groups=None): + + def __init__( + self, + obj_type, + nb_obj, + version, + logger=None, # pylint: disable=too-many-arguments, too-many-positional-arguments + nested_sitegroup_flag=False, + nested_region_flag=False, + nb_regions=None, + nb_groups=None, + ): self.logger = logger if logger else getLogger(__name__) if obj_type not in ("vm", "dev"): msg = f"Unable to create hostgroup with type {type}" @@ -19,8 +31,9 @@ class Hostgroup(): self.name = self.nb.name self.nb_version = version # Used for nested data objects - self.set_nesting(nested_sitegroup_flag, nested_region_flag, - nb_groups, nb_regions) + self.set_nesting( + nested_sitegroup_flag, nested_region_flag, nb_groups, nb_regions + ) self._set_format_options() def __str__(self): @@ -49,20 +62,28 @@ class Hostgroup(): format_options["site_group"] = None if self.nb.site: if self.nb.site.region: - format_options["region"] = self.generate_parents("region", - str(self.nb.site.region)) + format_options["region"] = self.generate_parents( + "region", str(self.nb.site.region) + ) if self.nb.site.group: - format_options["site_group"] = self.generate_parents("site_group", - str(self.nb.site.group)) + format_options["site_group"] = self.generate_parents( + "site_group", str(self.nb.site.group) + ) format_options["role"] = role format_options["site"] = self.nb.site.name if self.nb.site else None format_options["tenant"] = str(self.nb.tenant) if self.nb.tenant else None - format_options["tenant_group"] = str(self.nb.tenant.group) if self.nb.tenant else None - format_options["platform"] = self.nb.platform.name if self.nb.platform else None + format_options["tenant_group"] = ( + str(self.nb.tenant.group) if self.nb.tenant else None + ) + format_options["platform"] = ( + self.nb.platform.name if self.nb.platform else None + ) # Variables only applicable for devices if self.type == "dev": format_options["manufacturer"] = self.nb.device_type.manufacturer.name - format_options["location"] = str(self.nb.location) if self.nb.location else None + format_options["location"] = ( + str(self.nb.location) if self.nb.location else None + ) # Variables only applicable for VM's if self.type == "vm": # Check if a cluster is configured. Could also be configured in a site. @@ -72,17 +93,22 @@ class Hostgroup(): self.format_options = format_options - def set_nesting(self, nested_sitegroup_flag, nested_region_flag, - nb_groups, nb_regions): + def set_nesting( + self, nested_sitegroup_flag, nested_region_flag, nb_groups, nb_regions + ): """Set nesting options for this Hostgroup""" - self.nested_objects = {"site_group": {"flag": nested_sitegroup_flag, "data": nb_groups}, - "region": {"flag": nested_region_flag, "data": nb_regions}} + self.nested_objects = { + "site_group": {"flag": nested_sitegroup_flag, "data": nb_groups}, + "region": {"flag": nested_region_flag, "data": nb_regions}, + } def generate(self, hg_format=None): """Generate hostgroup based on a provided format""" # Set format to default in case its not specified if not hg_format: - hg_format = "site/manufacturer/role" if self.type == "dev" else "cluster/role" + hg_format = ( + "site/manufacturer/role" if self.type == "dev" else "cluster/role" + ) # Split all given names hg_output = [] hg_items = hg_format.split("/") @@ -93,8 +119,10 @@ class Hostgroup(): cf_data = self.custom_field_lookup(hg_item) # CF does not exist if not cf_data["result"]: - msg = (f"Unable to generate hostgroup for host {self.name}. " - f"Item type {hg_item} not supported.") + msg = ( + f"Unable to generate hostgroup for host {self.name}. " + f"Item type {hg_item} not supported." + ) self.logger.error(msg) raise HostgroupError(msg) # CF data is populated @@ -109,10 +137,12 @@ class Hostgroup(): # Check if the hostgroup is populated with at least one item. if bool(hg_output): return "/".join(hg_output) - msg = (f"Unable to generate hostgroup for host {self.name}." - " Not enough valid items. This is most likely" - " due to the use of custom fields that are empty" - " or an invalid hostgroup format.") + msg = ( + f"Unable to generate hostgroup for host {self.name}." + " Not enough valid items. This is most likely" + " due to the use of custom fields that are empty" + " or an invalid hostgroup format." + ) self.logger.error(msg) raise HostgroupError(msg) @@ -157,7 +187,9 @@ class Hostgroup(): return child_object # If the nested flag is True, perform parent calculation if self.nested_objects[nest_type]["flag"]: - final_nested_object = build_path(child_object, self.nested_objects[nest_type]["data"]) + final_nested_object = build_path( + child_object, self.nested_objects[nest_type]["data"] + ) return "/".join(final_nested_object) # Nesting is not allowed for this object. Return child_object return child_object diff --git a/modules/interface.py b/modules/interface.py index e4413c6..1bd1e37 100644 --- a/modules/interface.py +++ b/modules/interface.py @@ -4,7 +4,8 @@ All of the Zabbix interface related configuration """ from modules.exceptions import InterfaceConfigError -class ZabbixInterface(): + +class ZabbixInterface: """Class that represents a Zabbix interface.""" def __init__(self, context, ip): @@ -15,21 +16,16 @@ class ZabbixInterface(): def _set_default_port(self): """Sets default TCP / UDP port for different interface types""" - interface_mapping = { - 1: 10050, - 2: 161, - 3: 623, - 4: 12345 - } + interface_mapping = {1: 10050, 2: 161, 3: 623, 4: 12345} # Check if interface type is listed in mapper. - if self.interface['type'] not in interface_mapping: + if self.interface["type"] not in interface_mapping: return False # Set default port to interface - self.interface["port"] = str(interface_mapping[self.interface['type']]) + self.interface["port"] = str(interface_mapping[self.interface["type"]]) return True def get_context(self): - """ check if NetBox custom context has been defined. """ + """check if NetBox custom context has been defined.""" if "zabbix" in self.context: zabbix = self.context["zabbix"] if "interface_type" in zabbix: @@ -43,7 +39,7 @@ class ZabbixInterface(): return False def set_snmp(self): - """ Check if interface is type SNMP """ + """Check if interface is type SNMP""" # pylint: disable=too-many-branches if self.interface["type"] == 2: # Checks if SNMP settings are defined in NetBox @@ -63,7 +59,7 @@ class ZabbixInterface(): e = "SNMP version option is not defined." raise InterfaceConfigError(e) # If version 1 or 2 is used, get community string - if self.interface["details"]["version"] in ['1','2']: + if self.interface["details"]["version"] in ["1", "2"]: if "community" in snmp: # Set SNMP community to confix context value community = snmp["community"] @@ -73,10 +69,16 @@ class ZabbixInterface(): self.interface["details"]["community"] = str(community) # If version 3 has been used, get all # SNMPv3 NetBox related configs - elif self.interface["details"]["version"] == '3': - items = ["securityname", "securitylevel", "authpassphrase", - "privpassphrase", "authprotocol", "privprotocol", - "contextname"] + elif self.interface["details"]["version"] == "3": + items = [ + "securityname", + "securitylevel", + "authpassphrase", + "privpassphrase", + "authprotocol", + "privprotocol", + "contextname", + ] for key, item in snmp.items(): if key in items: self.interface["details"][key] = str(item) @@ -91,13 +93,15 @@ class ZabbixInterface(): raise InterfaceConfigError(e) def set_default_snmp(self): - """ Set default config to SNMPv2, port 161 and community macro. """ + """Set default config to SNMPv2, port 161 and community macro.""" self.interface = self.skelet self.interface["type"] = "2" self.interface["port"] = "161" - self.interface["details"] = {"version": "2", - "community": "{$SNMP_COMMUNITY}", - "bulk": "1"} + self.interface["details"] = { + "version": "2", + "community": "{$SNMP_COMMUNITY}", + "bulk": "1", + } def set_default_agent(self): """Sets interface to Zabbix agent defaults""" diff --git a/modules/tags.py b/modules/tags.py index 4993cd3..9dda995 100644 --- a/modules/tags.py +++ b/modules/tags.py @@ -4,13 +4,24 @@ All of the Zabbix Usermacro related configuration """ from logging import getLogger + from modules.tools import field_mapper, remove_duplicates -class ZabbixTags(): + +class ZabbixTags: """Class that represents a Zabbix interface.""" - def __init__(self, nb, tag_map, tag_sync, tag_lower=True, - tag_name=None, tag_value=None, logger=None, host=None): + def __init__( + self, + nb, + tag_map, + tag_sync, + tag_lower=True, + tag_name=None, + tag_value=None, + logger=None, + host=None, + ): self.nb = nb self.name = host if host else nb.name self.tag_map = tag_map @@ -42,7 +53,7 @@ class ZabbixTags(): """ Validates tag name """ - if tag_name and isinstance(tag_name, str) and len(tag_name)<=256: + if tag_name and isinstance(tag_name, str) and len(tag_name) <= 256: return True return False @@ -50,7 +61,7 @@ class ZabbixTags(): """ Validates tag value """ - if tag_value and isinstance(tag_value, str) and len(tag_value)<=256: + if tag_value and isinstance(tag_value, str) and len(tag_value) <= 256: return True return False @@ -58,23 +69,25 @@ class ZabbixTags(): """ Renders a tag """ - tag={} + tag = {} if self.validate_tag(tag_name): if self.lower: - tag['tag'] = tag_name.lower() + tag["tag"] = tag_name.lower() else: - tag['tag'] = tag_name + tag["tag"] = tag_name else: - self.logger.error(f'Tag {tag_name} is not a valid tag name, skipping.') + self.logger.error(f"Tag {tag_name} is not a valid tag name, skipping.") return False if self.validate_value(tag_value): if self.lower: - tag['value'] = tag_value.lower() + tag["value"] = tag_value.lower() else: - tag['value'] = tag_value + tag["value"] = tag_value else: - self.logger.error(f'Tag {tag_name} has an invalid value: \'{tag_value}\', skipping.') + self.logger.error( + f"Tag {tag_name} has an invalid value: '{tag_value}', skipping." + ) return False return tag @@ -83,7 +96,7 @@ class ZabbixTags(): Generate full set of Usermacros """ # pylint: disable=too-many-branches - tags=[] + tags = [] # Parse the field mapper for tags if self.tag_map: self.logger.debug(f"Host {self.nb.name}: Starting tag mapper") @@ -94,9 +107,12 @@ class ZabbixTags(): tags.append(t) # Parse NetBox config context for tags - if ("zabbix" in self.nb.config_context and "tags" in self.nb.config_context['zabbix'] - and isinstance(self.nb.config_context['zabbix']['tags'], list)): - for tag in self.nb.config_context['zabbix']['tags']: + if ( + "zabbix" in self.nb.config_context + and "tags" in self.nb.config_context["zabbix"] + and isinstance(self.nb.config_context["zabbix"]["tags"], list) + ): + for tag in self.nb.config_context["zabbix"]["tags"]: if isinstance(tag, dict): for tagname, value in tag.items(): t = self.render_tag(tagname, value) @@ -106,12 +122,12 @@ class ZabbixTags(): # Pull in NetBox device tags if tag_name is set if self.tag_name and isinstance(self.tag_name, str): for tag in self.nb.tags: - if self.tag_value.lower() in ['display', 'name', 'slug']: + if self.tag_value.lower() in ["display", "name", "slug"]: value = tag[self.tag_value] else: - value = tag['name'] + value = tag["name"] t = self.render_tag(self.tag_name, value) if t: tags.append(t) - return remove_duplicates(tags, sortkey='tag') + return remove_duplicates(tags, sortkey="tag") diff --git a/modules/tools.py b/modules/tools.py index 8d658a3..791025d 100644 --- a/modules/tools.py +++ b/modules/tools.py @@ -1,12 +1,14 @@ """A collection of tools used by several classes""" + def convert_recordset(recordset): - """ Converts netbox RedcordSet to list of dicts. """ + """Converts netbox RedcordSet to list of dicts.""" recordlist = [] for record in recordset: recordlist.append(record.__dict__) return recordlist + def build_path(endpoint, list_of_dicts): """ Builds a path list of related parent/child items. @@ -14,16 +16,17 @@ def build_path(endpoint, list_of_dicts): be used in hostgroups. """ item_path = [] - itemlist = [i for i in list_of_dicts if i['name'] == endpoint] + itemlist = [i for i in list_of_dicts if i["name"] == endpoint] item = itemlist[0] if len(itemlist) == 1 else None - item_path.append(item['name']) - while item['_depth'] > 0: - itemlist = [i for i in list_of_dicts if i['name'] == str(item['parent'])] + item_path.append(item["name"]) + while item["_depth"] > 0: + itemlist = [i for i in list_of_dicts if i["name"] == str(item["parent"])] item = itemlist[0] if len(itemlist) == 1 else None - item_path.append(item['name']) + item_path.append(item["name"]) item_path.reverse() return item_path + def proxy_prepper(proxy_list, proxy_group_list): """ Function that takes 2 lists and converts them using a @@ -44,15 +47,16 @@ def proxy_prepper(proxy_list, proxy_group_list): output.append(group) return output + def field_mapper(host, mapper, nbdevice, logger): """ Maps NetBox field data to Zabbix properties. Used for Inventory, Usermacros and Tag mappings. """ - data={} + data = {} # Let's build an dict for each property in the map for nb_field, zbx_field in mapper.items(): - field_list = nb_field.split("/") # convert str to list based on delimiter + field_list = nb_field.split("/") # convert str to list based on delimiter # start at the base of the dict... value = nbdevice # ... and step through the dict till we find the needed value @@ -61,22 +65,30 @@ def field_mapper(host, mapper, nbdevice, logger): # Check if the result is usable and expected # We want to apply any int or float 0 values, # even if python thinks those are empty. - if ((value and isinstance(value, int | float | str )) or - (isinstance(value, int | float) and int(value) ==0)): + if (value and isinstance(value, int | float | str)) or ( + isinstance(value, int | float) and int(value) == 0 + ): data[zbx_field] = str(value) elif not value: # empty value should just be an empty string for API compatibility - logger.debug(f"Host {host}: NetBox lookup for " - f"'{nb_field}' returned an empty value") + logger.debug( + f"Host {host}: NetBox lookup for " + f"'{nb_field}' returned an empty value" + ) data[zbx_field] = "" else: # Value is not a string or numeral, probably not what the user expected. - logger.error(f"Host {host}: Lookup for '{nb_field}'" - " returned an unexpected type: it will be skipped.") - logger.debug(f"Host {host}: Field mapping complete. " - f"Mapped {len(list(filter(None, data.values())))} field(s)") + logger.error( + f"Host {host}: Lookup for '{nb_field}'" + " returned an unexpected type: it will be skipped." + ) + logger.debug( + f"Host {host}: Field mapping complete. " + f"Mapped {len(list(filter(None, data.values())))} field(s)" + ) return data + def remove_duplicates(input_list, sortkey=None): """ Removes duplicate entries from a list and sorts the list diff --git a/modules/usermacros.py b/modules/usermacros.py index 71efbde..29580d1 100644 --- a/modules/usermacros.py +++ b/modules/usermacros.py @@ -3,11 +3,13 @@ """ All of the Zabbix Usermacro related configuration """ -from re import match from logging import getLogger +from re import match + from modules.tools import field_mapper -class ZabbixUsermacros(): + +class ZabbixUsermacros: """Class that represents a Zabbix interface.""" def __init__(self, nb, usermacro_map, usermacro_sync, logger=None, host=None): @@ -42,40 +44,46 @@ class ZabbixUsermacros(): """ Validates usermacro name """ - pattern = r'\{\$[A-Z0-9\._]*(\:.*)?\}' + pattern = r"\{\$[A-Z0-9\._]*(\:.*)?\}" return match(pattern, macro_name) def render_macro(self, macro_name, macro_properties): """ Renders a full usermacro from partial input """ - macro={} - macrotypes={'text': 0, 'secret': 1, 'vault': 2} + macro = {} + macrotypes = {"text": 0, "secret": 1, "vault": 2} if self.validate_macro(macro_name): - macro['macro'] = str(macro_name) + macro["macro"] = str(macro_name) if isinstance(macro_properties, dict): - if not 'value' in macro_properties: - self.logger.error(f'Usermacro {macro_name} has no value, skipping.') + if not "value" in macro_properties: + self.logger.error(f"Usermacro {macro_name} has no value, skipping.") return False - macro['value'] = macro_properties['value'] + macro["value"] = macro_properties["value"] - if 'type' in macro_properties and macro_properties['type'].lower() in macrotypes: - macro['type'] = str(macrotypes[macro_properties['type']]) + if ( + "type" in macro_properties + and macro_properties["type"].lower() in macrotypes + ): + macro["type"] = str(macrotypes[macro_properties["type"]]) else: - macro['type'] = str(0) + macro["type"] = str(0) - if ('description' in macro_properties and - isinstance(macro_properties['description'], str)): - macro['description'] = macro_properties['description'] + if "description" in macro_properties and isinstance( + macro_properties["description"], str + ): + macro["description"] = macro_properties["description"] else: - macro['description'] = "" + macro["description"] = "" elif isinstance(macro_properties, str): - macro['value'] = macro_properties - macro['type'] = str(0) - macro['description'] = "" + macro["value"] = macro_properties + macro["type"] = str(0) + macro["description"] = "" else: - self.logger.error(f'Usermacro {macro_name} is not a valid usermacro name, skipping.') + self.logger.error( + f"Usermacro {macro_name} is not a valid usermacro name, skipping." + ) return False return macro @@ -83,18 +91,25 @@ class ZabbixUsermacros(): """ Generate full set of Usermacros """ - macros=[] + macros = [] # Parse the field mapper for usermacros if self.usermacro_map: self.logger.debug(f"Host {self.nb.name}: Starting usermacro mapper") - field_macros = field_mapper(self.nb.name, self.usermacro_map, self.nb, self.logger) + field_macros = field_mapper( + self.nb.name, self.usermacro_map, self.nb, self.logger + ) for macro, value in field_macros.items(): m = self.render_macro(macro, value) if m: macros.append(m) # Parse NetBox config context for usermacros - if "zabbix" in self.nb.config_context and "usermacros" in self.nb.config_context['zabbix']: - for macro, properties in self.nb.config_context['zabbix']['usermacros'].items(): + if ( + "zabbix" in self.nb.config_context + and "usermacros" in self.nb.config_context["zabbix"] + ): + for macro, properties in self.nb.config_context["zabbix"][ + "usermacros" + ].items(): m = self.render_macro(macro, properties) if m: macros.append(m) diff --git a/modules/virtual_machine.py b/modules/virtual_machine.py index 273f9e7..6038811 100644 --- a/modules/virtual_machine.py +++ b/modules/virtual_machine.py @@ -3,55 +3,66 @@ """Module that hosts all functions for virtual machine processing""" from os import sys + from modules.device import PhysicalDevice +from modules.exceptions import InterfaceConfigError, SyncInventoryError, TemplateError from modules.hostgroups import Hostgroup from modules.interface import ZabbixInterface -from modules.exceptions import TemplateError, InterfaceConfigError, SyncInventoryError + try: from config import ( - vm_inventory_map, - vm_usermacro_map, - vm_tag_map, + traverse_regions, traverse_site_groups, - traverse_regions + vm_inventory_map, + vm_tag_map, + vm_usermacro_map, ) except ModuleNotFoundError: - print("Configuration file config.py not found in main directory." - "Please create the file or rename the config.py.example file to config.py.") + print( + "Configuration file config.py not found in main directory." + "Please create the file or rename the config.py.example file to config.py." + ) sys.exit(0) + class VirtualMachine(PhysicalDevice): """Model for virtual machines""" + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.hostgroup = None self.zbx_template_names = None def _inventory_map(self): - """ use VM inventory maps """ + """use VM inventory maps""" return vm_inventory_map def _usermacro_map(self): - """ use VM usermacro maps """ + """use VM usermacro maps""" return vm_usermacro_map def _tag_map(self): - """ use VM tag maps """ + """use VM tag maps""" return vm_tag_map def set_hostgroup(self, hg_format, nb_site_groups, nb_regions): """Set the hostgroup for this device""" # Create new Hostgroup instance - hg = Hostgroup("vm", self.nb, self.nb_api_version, logger=self.logger, - nested_sitegroup_flag=traverse_site_groups, - nested_region_flag=traverse_regions, - nb_groups=nb_site_groups, - nb_regions=nb_regions) + hg = Hostgroup( + "vm", + self.nb, + self.nb_api_version, + logger=self.logger, + nested_sitegroup_flag=traverse_site_groups, + nested_region_flag=traverse_regions, + nb_groups=nb_site_groups, + nb_regions=nb_regions, + ) # Generate hostgroup based on hostgroup format self.hostgroup = hg.generate(hg_format) def set_vm_template(self): - """ Set Template for VMs. Overwrites default class + """Set Template for VMs. Overwrites default class to skip a lookup of custom fields.""" # Gather templates ONLY from the device specific context try: @@ -60,7 +71,7 @@ class VirtualMachine(PhysicalDevice): self.logger.warning(e) return True - def setInterfaceDetails(self): # pylint: disable=invalid-name + def setInterfaceDetails(self): # pylint: disable=invalid-name """ Overwrites device function to select an agent interface type by default Agent type interfaces are more likely to be used with VMs then SNMP From 886c5b24b999310e127dce80e963b3baccad47af Mon Sep 17 00:00:00 2001 From: Wouter de Bruijn Date: Wed, 26 Feb 2025 14:45:20 +0100 Subject: [PATCH 06/18] =?UTF-8?q?=F0=9F=94=8A=20Improved=20log=20levels?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/device.py | 12 ++++++++---- modules/logging.py | 36 ++++++++++++++++++++++++++++++++++++ modules/tags.py | 4 ++-- netbox_zabbix_sync.py | 25 ++++++++----------------- 4 files changed, 54 insertions(+), 23 deletions(-) create mode 100644 modules/logging.py diff --git a/modules/device.py b/modules/device.py index 83e6fdc..d90e505 100644 --- a/modules/device.py +++ b/modules/device.py @@ -7,6 +7,7 @@ from copy import deepcopy from logging import getLogger from os import sys from re import search +from venv import logger from zabbix_utils import APIRequestError @@ -19,6 +20,7 @@ from modules.exceptions import ( ) from modules.hostgroups import Hostgroup from modules.interface import ZabbixInterface +from modules.logging import get_logger from modules.tags import ZabbixTags from modules.tools import field_mapper, remove_duplicates from modules.usermacros import ZabbixUsermacros @@ -111,7 +113,7 @@ class PhysicalDevice: self.ip = self.cidr.split("/")[0] else: e = f"Host {self.name}: no primary IP." - self.logger.info(e) + self.logger.warning(e) raise SyncInventoryError(e) # Check if device has custom field for ZBX ID @@ -193,6 +195,7 @@ class PhysicalDevice: f"found for {self.nb.device_type.manufacturer.name}" f" - {self.nb.device_type.display}." ) + self.logger.warning(e) raise TemplateError(e) def get_templates_context(self): @@ -548,7 +551,7 @@ class PhysicalDevice: except APIRequestError as e: e = f"Host {self.name}: Couldn't create. Zabbix returned {str(e)}." self.logger.error(e) - raise SyncExternalError(e) from None + raise SyncExternalError(e) from e # Set NetBox custom field to hostID value. self.nb.custom_fields[device_cf] = int(self.zabbix_id) self.nb.save() @@ -556,8 +559,9 @@ class PhysicalDevice: self.logger.info(msg) self.create_journal_entry("success", msg) else: - e = f"Host {self.name}: Unable to add to Zabbix. Host already present." - self.logger.warning(e) + self.logger.error( + f"Host {self.name}: Unable to add to Zabbix. Host already present." + ) def createZabbixHostgroup(self, hostgroups): """ diff --git a/modules/logging.py b/modules/logging.py new file mode 100644 index 0000000..851ea4c --- /dev/null +++ b/modules/logging.py @@ -0,0 +1,36 @@ +import logging +from os import path + +logger = logging.getLogger("NetBox-Zabbix-sync") + + +def get_logger(): + """ + Return the logger for Netbox Zabbix Sync + """ + return logger + + +def setup_logger(): + """ + Prepare a logger with stream and file handlers + """ + # Set logging + lgout = logging.StreamHandler() + lgfile = logging.FileHandler( + path.join(path.dirname(path.realpath(__file__)), "sync.log") + ) + + logging.basicConfig( + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + level=logging.WARNING, + handlers=[lgout, lgfile], + ) + + +def set_log_levels(root_level, own_level): + """ + Configure log levels for root and Netbox-Zabbix-sync logger + """ + logging.getLogger().setLevel(root_level) + logger.setLevel(own_level) diff --git a/modules/tags.py b/modules/tags.py index 9dda995..441ebe2 100644 --- a/modules/tags.py +++ b/modules/tags.py @@ -76,7 +76,7 @@ class ZabbixTags: else: tag["tag"] = tag_name else: - self.logger.error(f"Tag {tag_name} is not a valid tag name, skipping.") + self.logger.warning(f"Tag {tag_name} is not a valid tag name, skipping.") return False if self.validate_value(tag_value): @@ -85,7 +85,7 @@ class ZabbixTags: else: tag["value"] = tag_value else: - self.logger.error( + self.logger.warning( f"Tag {tag_name} has an invalid value: '{tag_value}', skipping." ) return False diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index 75ad65c..2f30ccd 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -14,6 +14,7 @@ from zabbix_utils import APIRequestError, ProcessingError, ZabbixAPI from modules.device import PhysicalDevice from modules.exceptions import EnvironmentVarError, HostgroupError, SyncError +from modules.logging import get_logger, set_log_levels, setup_logger from modules.tools import convert_recordset, proxy_prepper from modules.virtual_machine import VirtualMachine @@ -40,19 +41,9 @@ except ModuleNotFoundError: ) sys.exit(1) -# Set logging -lgout = logging.StreamHandler() -lgfile = logging.FileHandler( - path.join(path.dirname(path.realpath(__file__)), "sync.log") -) -logging.basicConfig( - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", - level=logging.WARNING, - handlers=[lgout, lgfile], -) - -logger = logging.getLogger("NetBox-Zabbix-sync") +setup_logger() +logger = get_logger() def main(arguments): @@ -60,13 +51,13 @@ def main(arguments): # pylint: disable=too-many-branches, too-many-statements # set environment variables if arguments.verbose: - logger.setLevel(logging.INFO) + set_log_levels(logging.WARNING, logging.INFO) if arguments.debug: - logger.setLevel(logging.DEBUG) + set_log_levels(logging.WARNING, logging.DEBUG) if arguments.debug_all: - logging.getLogger().setLevel(logging.DEBUG) + set_log_levels(logging.DEBUG, logging.DEBUG) if arguments.quiet: - logging.getLogger().setLevel(logging.ERROR) + set_log_levels(logging.ERROR, logging.ERROR) env_vars = ["ZABBIX_HOST", "NETBOX_HOST", "NETBOX_TOKEN"] if "ZABBIX_TOKEN" in environ: @@ -202,7 +193,7 @@ def main(arguments): # Delete device from Zabbix # and remove hostID from NetBox. vm.cleanup() - logger.info(f"VM {vm.name}: cleanup complete") + logger.debug(f"VM {vm.name}: cleanup complete") continue # Device has been added to NetBox # but is not in Activate state From 9ab5e09dd51a146edbeea2f4bf4f707e941fbaa2 Mon Sep 17 00:00:00 2001 From: Wouter de Bruijn Date: Wed, 26 Feb 2025 14:54:08 +0100 Subject: [PATCH 07/18] =?UTF-8?q?=F0=9F=92=A1=20Added=20docstring=20for=20?= =?UTF-8?q?module?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/logging.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/logging.py b/modules/logging.py index 851ea4c..c36c2c1 100644 --- a/modules/logging.py +++ b/modules/logging.py @@ -1,3 +1,7 @@ +""" +Logging module for Netbox-Zabbix-sync +""" + import logging from os import path From 7781bc673206439317e60fa517478645261d5f58 Mon Sep 17 00:00:00 2001 From: Wouter de Bruijn Date: Wed, 26 Feb 2025 14:54:20 +0100 Subject: [PATCH 08/18] =?UTF-8?q?=F0=9F=9A=A8=20"Fixed"=20linter=20warning?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/device.py | 11 ++++------- modules/hostgroups.py | 3 ++- modules/virtual_machine.py | 1 - netbox_zabbix_sync.py | 2 +- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/modules/device.py b/modules/device.py index d90e505..b8d038d 100644 --- a/modules/device.py +++ b/modules/device.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python3 -# pylint: disable=invalid-name, logging-not-lazy, too-many-locals, logging-fstring-interpolation, too-many-lines, too-many-public-methods +# pylint: disable=invalid-name, logging-not-lazy, too-many-locals, logging-fstring-interpolation, too-many-lines, too-many-public-methods, duplicate-code """ Device specific handeling for NetBox to Zabbix """ @@ -7,7 +6,6 @@ from copy import deepcopy from logging import getLogger from os import sys from re import search -from venv import logger from zabbix_utils import APIRequestError @@ -20,7 +18,6 @@ from modules.exceptions import ( ) from modules.hostgroups import Hostgroup from modules.interface import ZabbixInterface -from modules.logging import get_logger from modules.tags import ZabbixTags from modules.tools import field_mapper, remove_duplicates from modules.usermacros import ZabbixUsermacros @@ -549,9 +546,9 @@ class PhysicalDevice: host = self.zabbix.host.create(**create_data) self.zabbix_id = host["hostids"][0] except APIRequestError as e: - e = f"Host {self.name}: Couldn't create. Zabbix returned {str(e)}." - self.logger.error(e) - raise SyncExternalError(e) from e + msg = f"Host {self.name}: Couldn't create. Zabbix returned {str(e)}." + self.logger.error(msg) + raise SyncExternalError(msg) from e # Set NetBox custom field to hostID value. self.nb.custom_fields[device_cf] = int(self.zabbix_id) self.nb.save() diff --git a/modules/hostgroups.py b/modules/hostgroups.py index c67f5e6..d1350bd 100644 --- a/modules/hostgroups.py +++ b/modules/hostgroups.py @@ -10,12 +10,13 @@ class Hostgroup: """Hostgroup class for devices and VM's Takes type (vm or dev) and NB object""" + # pylint: disable=too-many-arguments, disable=too-many-positional-arguments def __init__( self, obj_type, nb_obj, version, - logger=None, # pylint: disable=too-many-arguments, too-many-positional-arguments + logger=None, nested_sitegroup_flag=False, nested_region_flag=False, nb_regions=None, diff --git a/modules/virtual_machine.py b/modules/virtual_machine.py index 6038811..34e3394 100644 --- a/modules/virtual_machine.py +++ b/modules/virtual_machine.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # pylint: disable=duplicate-code """Module that hosts all functions for virtual machine processing""" diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index 2f30ccd..79d8a20 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -5,7 +5,7 @@ import argparse import logging import ssl -from os import environ, path, sys +from os import environ, sys from pynetbox import api from pynetbox.core.query import RequestError as NBRequestError From 50918e43fa20217cdde4a17e3bc058cee8700376 Mon Sep 17 00:00:00 2001 From: Wouter de Bruijn Date: Fri, 28 Feb 2025 15:25:18 +0100 Subject: [PATCH 09/18] =?UTF-8?q?=F0=9F=94=A7=20Changed=20user=20for=20doc?= =?UTF-8?q?ker=20container?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index fa8d9c4..70ec03d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,10 @@ # syntax=docker/dockerfile:1 FROM python:3.12-alpine + +USER 1000:1000 + RUN mkdir -p /opt/netbox-zabbix -COPY . /opt/netbox-zabbix +COPY --chown=1000:1000 . /opt/netbox-zabbix WORKDIR /opt/netbox-zabbix RUN if ! [ -f ./config.py ]; then cp ./config.py.example ./config.py; fi RUN pip install -r ./requirements.txt From 4449e040cebab080c29cea3193d18e104108176a Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Wed, 9 Apr 2025 15:49:38 +0200 Subject: [PATCH 10/18] :bug: added check for empty usermacro value. --- modules/usermacros.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/modules/usermacros.py b/modules/usermacros.py index 29580d1..c1d783b 100644 --- a/modules/usermacros.py +++ b/modules/usermacros.py @@ -57,7 +57,7 @@ class ZabbixUsermacros: macro["macro"] = str(macro_name) if isinstance(macro_properties, dict): if not "value" in macro_properties: - self.logger.error(f"Usermacro {macro_name} has no value, skipping.") + self.logger.warning(f"Usermacro {macro_name} has no value, skipping.") return False macro["value"] = macro_properties["value"] @@ -76,10 +76,14 @@ class ZabbixUsermacros: else: macro["description"] = "" - elif isinstance(macro_properties, str): + elif isinstance(macro_properties, str) and macro_properties: macro["value"] = macro_properties macro["type"] = str(0) macro["description"] = "" + + else: + self.logger.warning(f"Usermacro {macro_name} has no value, skipping.") + return False else: self.logger.error( f"Usermacro {macro_name} is not a valid usermacro name, skipping." From 50b7ede81be9cafa479bc4426e0e9af188bbc855 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Wed, 9 Apr 2025 16:03:45 +0200 Subject: [PATCH 11/18] :wrench: quick dockerfile fix --- Dockerfile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 70ec03d..3188195 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,14 @@ # syntax=docker/dockerfile:1 FROM python:3.12-alpine - -USER 1000:1000 +LABEL org.opencontainers.image.source=https://github.com/TheNetworkGuy/netbox-zabbix-sync +LABEL org.opencontainers.image.title="NetBox-Zabbix-Sync" +LABEL org.opencontainers.image.description="Python script to synchronise NetBox devices to Zabbix." +LABEL org.opencontainers.image.documentation=https://github.com/TheNetworkGuy/netbox-zabbix-sync/ +LABEL org.opencontainers.image.licenses=MIT +LABEL org.opencontainers.image.authors="Twan Kamans" RUN mkdir -p /opt/netbox-zabbix +USER 1000:1000 COPY --chown=1000:1000 . /opt/netbox-zabbix WORKDIR /opt/netbox-zabbix RUN if ! [ -f ./config.py ]; then cp ./config.py.example ./config.py; fi From 73d34851fbc8ea2c1926ab3f4722138a9c17f120 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Thu, 10 Apr 2025 15:34:50 +0200 Subject: [PATCH 12/18] Update Dockerfile --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 598d219..3217397 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,7 @@ # syntax=docker/dockerfile:1 FROM python:3.12-alpine - -RUN mkdir -p /opt/netbox-zabbix USER 1000:1000 +RUN mkdir -p /opt/netbox-zabbix COPY --chown=1000:1000 . /opt/netbox-zabbix WORKDIR /opt/netbox-zabbix RUN if ! [ -f ./config.py ]; then cp ./config.py.example ./config.py; fi From b56a4332b9148cce969c90af8520d9539aa280b2 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Thu, 10 Apr 2025 15:35:44 +0200 Subject: [PATCH 13/18] Update Dockerfile --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3217397..c4693c1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,5 @@ # syntax=docker/dockerfile:1 FROM python:3.12-alpine -USER 1000:1000 RUN mkdir -p /opt/netbox-zabbix COPY --chown=1000:1000 . /opt/netbox-zabbix WORKDIR /opt/netbox-zabbix From 20a3c67fd4f1fa24774a5c8dbb0b493844380916 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Thu, 10 Apr 2025 15:37:57 +0200 Subject: [PATCH 14/18] Update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c4693c1..fa8d9c4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1 FROM python:3.12-alpine RUN mkdir -p /opt/netbox-zabbix -COPY --chown=1000:1000 . /opt/netbox-zabbix +COPY . /opt/netbox-zabbix WORKDIR /opt/netbox-zabbix RUN if ! [ -f ./config.py ]; then cp ./config.py.example ./config.py; fi RUN pip install -r ./requirements.txt From 13fe406b635de3ba290d3ffdc1c960bf4652750a Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Thu, 10 Apr 2025 16:00:56 +0200 Subject: [PATCH 15/18] Update Dockerfile --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index fa8d9c4..77628da 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,7 @@ RUN mkdir -p /opt/netbox-zabbix COPY . /opt/netbox-zabbix WORKDIR /opt/netbox-zabbix RUN if ! [ -f ./config.py ]; then cp ./config.py.example ./config.py; fi +USER 1000:1000 RUN pip install -r ./requirements.txt ENTRYPOINT ["python"] CMD ["/opt/netbox-zabbix/netbox_zabbix_sync.py", "-v"] From 6abdac2eb44af4c5f284dcd31a1ef7308594bcb5 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Thu, 10 Apr 2025 16:01:53 +0200 Subject: [PATCH 16/18] Update Dockerfile --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 77628da..ffd868f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,7 @@ RUN mkdir -p /opt/netbox-zabbix COPY . /opt/netbox-zabbix WORKDIR /opt/netbox-zabbix RUN if ! [ -f ./config.py ]; then cp ./config.py.example ./config.py; fi +RUN chown -R 1000:1000 /opt/netbox-zabbix USER 1000:1000 RUN pip install -r ./requirements.txt ENTRYPOINT ["python"] From 37b3bfc7fb12970f38a806985326dd56e7979664 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Thu, 10 Apr 2025 16:05:34 +0200 Subject: [PATCH 17/18] Update Dockerfile --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index ffd868f..d4f7eee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,11 @@ # syntax=docker/dockerfile:1 FROM python:3.12-alpine -RUN mkdir -p /opt/netbox-zabbix -COPY . /opt/netbox-zabbix +RUN mkdir -p /opt/netbox-zabbix && chown -R 1000:1000 /opt/netbox-zabbix + +USER 1000:1000 +COPY --chown=1000:1000 . /opt/netbox-zabbix WORKDIR /opt/netbox-zabbix RUN if ! [ -f ./config.py ]; then cp ./config.py.example ./config.py; fi -RUN chown -R 1000:1000 /opt/netbox-zabbix -USER 1000:1000 RUN pip install -r ./requirements.txt ENTRYPOINT ["python"] CMD ["/opt/netbox-zabbix/netbox_zabbix_sync.py", "-v"] From 2ea2edb6a6e0ee002a2d3e07ed474098c832e921 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Thu, 10 Apr 2025 16:13:37 +0200 Subject: [PATCH 18/18] Update Dockerfile --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index d4f7eee..198dbe5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,8 @@ USER 1000:1000 COPY --chown=1000:1000 . /opt/netbox-zabbix WORKDIR /opt/netbox-zabbix RUN if ! [ -f ./config.py ]; then cp ./config.py.example ./config.py; fi +USER root RUN pip install -r ./requirements.txt +USER 1000:1000 ENTRYPOINT ["python"] CMD ["/opt/netbox-zabbix/netbox_zabbix_sync.py", "-v"]