From 2e867d1129368287b6205467b8a3f052832c9af9 Mon Sep 17 00:00:00 2001 From: TheNetworkGuy Date: Fri, 25 Oct 2024 15:53:33 +0200 Subject: [PATCH 01/27] Added .venv to gitignore for developing. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ce91e5a..c3069c9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *.log +.venv config.py # Byte-compiled / optimized / DLL files __pycache__/ From 053028b283c7d60069754b0215fe08bc0f7d5876 Mon Sep 17 00:00:00 2001 From: TheNetworkGuy Date: Fri, 25 Oct 2024 16:02:08 +0200 Subject: [PATCH 02/27] Splitted hostgroup generation logic into its seperate module. Changed hostgroup "dev_role" to "role" for VM role prepration. Started work on basic VM class. --- config.py.example | 4 +- modules/device.py | 60 ++-------------- modules/hostgroups.py | 146 ++++++++++++++++++++++++++++++++++++++ modules/virtualMachine.py | 23 ++++++ netbox_zabbix_sync.py | 2 +- 5 files changed, 179 insertions(+), 56 deletions(-) create mode 100644 modules/hostgroups.py create mode 100644 modules/virtualMachine.py diff --git a/config.py.example b/config.py.example index b856989..0dbe7a9 100644 --- a/config.py.example +++ b/config.py.example @@ -3,7 +3,7 @@ # coming from config context instead of a custom field. templates_config_context = False -# Set to true to give config context templates a +# Set to true to give config context templates a # higher priority then custom field templates templates_config_context_overrule = False @@ -41,7 +41,7 @@ zabbix_device_disable = ["Offline", "Planned", "Staged", "Failed"] # 'Global/Europe/Netherlands/Amsterdam' instead of just 'Amsterdam'. # # traverse_site_groups controls the same behaviour for any assigned site_groups. -hostgroup_format = "site/manufacturer/dev_role" +hostgroup_format = "site/manufacturer/role" traverse_regions = False traverse_site_groups = False diff --git a/modules/device.py b/modules/device.py index df8918d..07319ff 100644 --- a/modules/device.py +++ b/modules/device.py @@ -9,7 +9,7 @@ from zabbix_utils import APIRequestError from modules.exceptions import (SyncInventoryError, TemplateError, SyncExternalError, InterfaceConfigError, JournalError) from modules.interface import ZabbixInterface -from modules.tools import build_path +from modules.hostgroups import Hostgroup try: from config import ( template_cf, device_cf, @@ -91,58 +91,12 @@ class NetworkDevice(): def set_hostgroup(self, hg_format, nb_site_groups, nb_regions): """Set the hostgroup for this device""" - # Get all variables from the NB data - dev_location = str(self.nb.location) if self.nb.location else None - # Check the Netbox version. Use backwards compatibility for versions 2 and 3. - if self.nb_api_version.startswith(("2", "3")): - dev_role = self.nb.device_role.name - else: - dev_role = self.nb.role.name - manufacturer = self.nb.device_type.manufacturer.name - region = str(self.nb.site.region) if self.nb.site.region else None - site = self.nb.site.name - site_group = str(self.nb.site.group) if self.nb.site.group else None - tenant = str(self.tenant) if self.tenant else None - tenant_group = str(self.tenant.group) if tenant else None - # Set mapper for string -> variable - hostgroup_vars = {"dev_location": dev_location, "dev_role": dev_role, - "manufacturer": manufacturer, "region": region, - "site": site, "site_group": site_group, - "tenant": tenant, "tenant_group": tenant_group} - # Generate list based off string input format - hg_items = hg_format.split("/") - hostgroup = "" - # Go through all hostgroup items - for item in hg_items: - # Check if the variable (such as Tenant) is empty. - if not hostgroup_vars[item]: - continue - # Check if the item is a custom field name - if item not in hostgroup_vars: - cf_value = self.nb.custom_fields[item] if item in self.nb.custom_fields else None - if cf_value: - # If there is a cf match, add the value of this cf to the hostgroup - hostgroup += cf_value + "/" - # Should there not be a match, this means that - # the variable is invalid. Skip regardless. - continue - # Add value of predefined variable to hostgroup format - if item == "site_group" and nb_site_groups and traverse_site_groups: - group_path = build_path(site_group, nb_site_groups) - hostgroup += "/".join(group_path) + "/" - elif item == "region" and nb_regions and traverse_regions: - region_path = build_path(region, nb_regions) - hostgroup += "/".join(region_path) + "/" - else: - hostgroup += hostgroup_vars[item] + "/" - # If the final hostgroup variable is empty - if not hostgroup: - e = (f"{self.name} has no reliable hostgroup. This is" - "most likely due to the use of custom fields that are empty.") - self.logger.error(e) - raise SyncInventoryError(e) - # Remove final inserted "/" and set hostgroup to class var - self.hostgroup = hostgroup.rstrip("/") + # Create new Hostgroup instance + hg = Hostgroup("dev", self.nb, self.nb_api_version, + traverse_site_groups, traverse_regions, + nb_site_groups, nb_regions) + # Generate hostgroup based on hostgroup format + self.hostgroup = hg.generate(hg_format) def set_template(self, prefer_config_context, overrule_custom): """ Set Template """ diff --git a/modules/hostgroups.py b/modules/hostgroups.py new file mode 100644 index 0000000..5c4f151 --- /dev/null +++ b/modules/hostgroups.py @@ -0,0 +1,146 @@ +"""Module for all hostgroup related code""" +from modules.exceptions import HostgroupError +from modules.tools import build_path + +class Hostgroup(): + """Hostgroup class for devices and VM's + Takes type (vm or dev) and NB object""" + def __init__(self, obj_type, nb_obj, version, + nested_sitegroup_flag, nested_region_flag, + nb_groups, nb_regions): + if obj_type not in ("vm", "dev"): + raise HostgroupError(f"Unable to create hostgroup with type {type}") + self.type = str(obj_type) + self.nb = nb_obj + self.name = self.nb.name + self.nb_version = version + # Used for nested data objects + self.nested_objects = {"site_group": {"flag": nested_sitegroup_flag, "data": nb_groups}, + "region": {"flag": nested_region_flag, "data": nb_regions}} + self._set_format_options() + + def __str__(self): + return f"Hostgroup for {self.type} {self.name}" + + def __repr__(self): + return self.__str__() + + def _set_format_options(self): + """ + Set all available variables + for hostgroup generation + """ + format_options = {} + # Set variables for both type of devices + if self.type in ("vm", "dev"): + # Role fix for Netbox <=3 + role = None + if self.nb_version.startswith(("2", "3")) and self.type == "dev": + role = self.nb.device_role.name + else: + role = self.nb.role.name + # Add default formatting options + # Check if a site is configured. A site is optional for VMs + format_options["region"] = None + format_options["location"] = None + format_options["site_group"] = None + if self.nb.site: + if self.nb.site.region: + format_options["region"] = self.generate_parents("region", + str(self.nb.site.region)) + if self.nb.location: + format_options["location"] = str(self.nb.location) + if self.nb.site.group: + format_options["site_group"] = self.generate_parents("site_group", + str(self.nb.site.group)) + format_options["role"] = role + format_options["site"] = self.nb.site.name if self.nb.site else None + format_options["tenant"] = str(self.nb.tenant) if self.nb.tenant else None + format_options["tenant_group"] = str(self.tenant.group) if self.nb.tenant else None + format_options["platform"] = self.nb.platform.name if self.nb.platform else None + # Variables only applicable for devices + if self.type == "dev": + format_options["manufacturer"] = self.nb.device_type.manufacturer.name + # Variables only applicable for VM's + if self.type == "vm": + format_options["cluster"] = str(self.nb.cluster.name) if self.nb.cluster else None + self.format_options = format_options + + def generate(self, hg_format=None): + """Generate hostgroup based on a provided format""" + # Set format to default in case its not specified + if not hg_format: + hg_format = "site/manufacturer/role" if self.type == "dev" else "cluster/role" + # Split all given names + hg_output = list() + hg_items = hg_format.split("/") + for hg_item in hg_items: + # Check if requested data is available as option for this host + if hg_item not in self.format_options: + # Check if a custom field exists with this name + cf_data = self.custom_field_lookup(hg_item) + # CF does not exist + if not cf_data["result"]: + raise HostgroupError(f"Unable to generate hostgroup for host {self.name}. " + f"Item type {hg_item} not supported.") + # CF data is populated + if cf_data["cf"]: + hg_output.append(cf_data["cf"]) + continue + # Check if there is a value associated to the variable. + # For instance, if a device has no location, do not use it with hostgroup calculation + hostgroup_value = self.format_options[hg_item] + if hostgroup_value: + hg_output.append(hostgroup_value) + # Check if the hostgroup is populated with at least one item. + if bool(hg_output): + return "/".join(hg_output) + raise HostgroupError(f"Unable to generate hostgroup for host {self.name}." + "Not enough valid items. This is most likely" + "due to the use of custom fields that are empty.") + + def list_formatoptions(self): + """ + Function to easily troubleshoot which values + are generated for a specific device or VM. + """ + print(f"The following options are available for host {self.name}") + for option_type, value in self.format_options.items(): + if value is not None: + print(f"{option_type} - {value}") + print("The following options are not available") + for option_type, value in self.format_options.items(): + if value is None: + print(f"{option_type}") + + def custom_field_lookup(self, hg_category): + """ + Checks if a valid custom field is present in Netbox. + INPUT: Custom field name + OUTPUT: dictionary with 'result' and 'cf' keys. + """ + # Check if the custom field exists + if hg_category not in self.nb.custom_fields: + return {"result": False, "cf": None} + # Checks if the custom field has been populated + if not bool(self.nb.custom_fields[hg_category]): + return {"result": True, "cf": None} + # Custom field exists and is populated + return {"result": True, "cf": self.nb.custom_fields[hg_category]} + + def generate_parents(self, nest_type, child_object): + """ + Generates parent objects to implement nested regions / nested site groups + INPUT: nest_type to set which type of nesting is going to be processed + child_object: the name of the child object (for instance the last NB region) + OUTPUT: STRING - Either the single child name or child and parents. + """ + # Check if this type of nesting is supported. + if not nest_type in self.nested_objects: + return child_object + # If the nested flag is True, perform parent calculation + if self.nested_objects[nest_type]["flag"]: + final_nested_object = build_path(child_object, self.nested_objects[nest_type]["data"]) + return "/".join(final_nested_object) + # Nesting is not allowed for this object. Return child_object + return child_object diff --git a/modules/virtualMachine.py b/modules/virtualMachine.py new file mode 100644 index 0000000..82ca3d3 --- /dev/null +++ b/modules/virtualMachine.py @@ -0,0 +1,23 @@ +"""Module that hosts all functions for virtual machine processing""" + +from modules.exceptions import * + +class VirtualMachine(): + """Model for virtual machines""" + def __init__(self, nb, name): + self.nb = nb + self.name = name + self.hostgroup = None + + def __repr__(self): + return self.name + + def __str__(self): + return self.__repr__() + + def _data_prep(self): + self.platform = self.nb.platform.name + self.cluster = self.nb.cluster.name + + def set_hostgroup(self): + self.hostgroup = "Virtual machines" diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index ea95bce..52f9652 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -76,7 +76,7 @@ def main(arguments): netbox = api(netbox_host, token=netbox_token, threading=True) # Check if the provided Hostgroup layout is valid hg_objects = hostgroup_format.split("/") - allowed_objects = ["dev_location", "dev_role", "manufacturer", "region", + allowed_objects = ["dev_location", "role", "manufacturer", "region", "site", "site_group", "tenant", "tenant_group"] # Create API call to get all custom fields which are on the device objects device_cfs = netbox.extras.custom_fields.filter(type="text", content_type_id=23) From e827953d8dd2ae695e7ba9b817f015c4fc7e1c25 Mon Sep 17 00:00:00 2001 From: TheNetworkGuy Date: Fri, 25 Oct 2024 16:50:56 +0200 Subject: [PATCH 03/27] Fixed extra space in config.py.example --- config.py.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py.example b/config.py.example index 0dbe7a9..264bb5b 100644 --- a/config.py.example +++ b/config.py.example @@ -40,7 +40,7 @@ zabbix_device_disable = ["Offline", "Planned", "Staged", "Failed"] # # 'Global/Europe/Netherlands/Amsterdam' instead of just 'Amsterdam'. # -# traverse_site_groups controls the same behaviour for any assigned site_groups. +# traverse_site_groups controls the same behaviour for any assigned site_groups. hostgroup_format = "site/manufacturer/role" traverse_regions = False traverse_site_groups = False From 9f29d2b27b00f1eb3ea61c705cf1fd6f209f7bb3 Mon Sep 17 00:00:00 2001 From: TheNetworkGuy Date: Fri, 25 Oct 2024 18:46:20 +0200 Subject: [PATCH 04/27] Added basic VM support --- config.py.example | 7 +++ modules/device.py | 108 ++++++++++++++++++++------------------ modules/hostgroups.py | 4 +- modules/virtualMachine.py | 54 ++++++++++++------- netbox_zabbix_sync.py | 62 +++++++++++++++++++--- 5 files changed, 157 insertions(+), 78 deletions(-) diff --git a/config.py.example b/config.py.example index 264bb5b..ca171cf 100644 --- a/config.py.example +++ b/config.py.example @@ -21,6 +21,13 @@ create_hostgroups = True ## Create journal entries create_journal = False +## Virtual machine sync +# Set sync_vms to True in order to use this new feature +# Use the hostgroup vm_hostgroup_format mapper for specific +# hostgroup atributes of VM's such as cluster_type and cluster +sync_vms = False +vm_hostgroup_format = "cluster_type/cluster/role" + ## Proxy Sync # Set to true to enable removal of proxy's under hosts. Use with caution and make sure that you specified # all the required proxy's in the device config context before enabeling this option. diff --git a/modules/device.py b/modules/device.py index 07319ff..e26c7fe 100644 --- a/modules/device.py +++ b/modules/device.py @@ -24,7 +24,7 @@ except ModuleNotFoundError: "Please create the file or rename the config.py.example file to config.py.") sys.exit(0) -class NetworkDevice(): +class PhysicalDevice(): # pylint: disable=too-many-instance-attributes, too-many-arguments """ Represents Network device. @@ -55,6 +55,12 @@ class NetworkDevice(): self.logger = logger if logger else getLogger(__name__) self._setBasics() + def __repr__(self): + return self.name + + def __str__(self): + return self.__repr__() + def _setBasics(self): """ Sets basic information like IP address. @@ -64,7 +70,7 @@ class NetworkDevice(): self.cidr = self.nb.primary_ip.address self.ip = self.cidr.split("/")[0] else: - e = f"Device {self.name}: no primary IP." + e = f"Host {self.name}: no primary IP." self.logger.info(e) raise SyncInventoryError(e) @@ -72,7 +78,7 @@ class NetworkDevice(): if device_cf in self.nb.custom_fields: self.zabbix_id = self.nb.custom_fields[device_cf] else: - e = f"Device {self.name}: Custom field {device_cf} not present" + e = f"Host {self.name}: Custom field {device_cf} not present" self.logger.warning(e) raise SyncInventoryError(e) @@ -83,7 +89,7 @@ class NetworkDevice(): self.name = f"NETBOX_ID{self.id}" self.visible_name = self.nb.name self.use_visible_name = True - self.logger.info(f"Device {self.visible_name} contains special characters. " + self.logger.info(f"Host {self.visible_name} contains special characters. " f"Using {self.name} as name for the Netbox object " f"and using {self.visible_name} as visible name in Zabbix.") else: @@ -153,7 +159,7 @@ class NetworkDevice(): # Set inventory mode. Default is disabled (see class init function). if inventory_mode == "disabled": if inventory_sync: - self.logger.error(f"Device {self.name}: Unable to map Netbox inventory to Zabbix. " + self.logger.error(f"Host {self.name}: Unable to map Netbox inventory to Zabbix. " "Inventory sync is enabled in config but inventory mode is disabled.") return True if inventory_mode == "manual": @@ -161,12 +167,12 @@ class NetworkDevice(): elif inventory_mode == "automatic": self.inventory_mode = 1 else: - self.logger.error(f"Device {self.name}: Specified value for inventory mode in" + self.logger.error(f"Host {self.name}: Specified value for inventory mode in" f" config is not valid. Got value {inventory_mode}") return False self.inventory = {} if inventory_sync and self.inventory_mode in [0,1]: - self.logger.debug(f"Device {self.name}: Starting inventory mapper") + self.logger.debug(f"Host {self.name}: Starting inventory mapper") # Let's build an inventory dict for each property in the inventory_map for nb_inv_field, zbx_inv_field in inventory_map.items(): field_list = nb_inv_field.split("/") # convert str to list based on delimiter @@ -183,14 +189,14 @@ class NetworkDevice(): self.inventory[zbx_inv_field] = str(value) elif not value: # empty value should just be an empty string for API compatibility - self.logger.debug(f"Device {self.name}: Netbox inventory lookup for " + self.logger.debug(f"Host {self.name}: Netbox inventory lookup for " f"'{nb_inv_field}' returned an empty value") self.inventory[zbx_inv_field] = "" else: # Value is not a string or numeral, probably not what the user expected. - self.logger.error(f"Device {self.name}: Inventory lookup for '{nb_inv_field}'" + self.logger.error(f"Host {self.name}: Inventory lookup for '{nb_inv_field}'" " returned an unexpected type: it will be skipped.") - self.logger.debug(f"Device {self.name}: Inventory mapping complete. " + self.logger.debug(f"Host {self.name}: Inventory mapping complete. " f"Mapped {len(list(filter(None, self.inventory.values())))} field(s)") return True @@ -224,12 +230,12 @@ class NetworkDevice(): """ masterid = self.getClusterMaster() if masterid == self.id: - self.logger.debug(f"Device {self.name} is primary cluster member. " + self.logger.debug(f"Host {self.name} is primary cluster member. " f"Modifying hostname from {self.name} to " + f"{self.nb.virtual_chassis.name}.") self.name = self.nb.virtual_chassis.name return True - self.logger.debug(f"Device {self.name} is non-primary cluster member.") + self.logger.debug(f"Host {self.name} is non-primary cluster member.") return False def zbxTemplatePrepper(self, templates): @@ -240,7 +246,7 @@ class NetworkDevice(): """ # Check if there are templates defined if not self.zbx_template_names: - e = f"Device {self.name}: No templates found" + e = f"Host {self.name}: No templates found" self.logger.info(e) raise SyncInventoryError() # Set variable to empty list @@ -257,7 +263,7 @@ class NetworkDevice(): template_match = True self.zbx_templates.append({"templateid": zbx_template['templateid'], "name": zbx_template['name']}) - e = f"Device {self.name}: found template {zbx_template['name']}" + e = f"Host {self.name}: found template {zbx_template['name']}" self.logger.debug(e) # Return error should the template not be found in Zabbix if not template_match: @@ -276,7 +282,7 @@ class NetworkDevice(): for group in groups: if group['name'] == self.hostgroup: self.group_id = group['groupid'] - e = f"Device {self.name}: matched group {group['name']}" + e = f"Host {self.name}: matched group {group['name']}" self.logger.debug(e) return True return False @@ -291,7 +297,7 @@ class NetworkDevice(): self.zabbix.host.delete(self.zabbix_id) self.nb.custom_fields[device_cf] = None self.nb.save() - e = f"Device {self.name}: Deleted host from Zabbix." + e = f"Host {self.name}: Deleted host from Zabbix." self.logger.info(e) self.create_journal_entry("warning", "Deleted host from Zabbix") except APIRequestError as e: @@ -364,11 +370,11 @@ class NetworkDevice(): continue # If the proxy name matches if proxy["name"] == proxy_name: - self.logger.debug(f"Device {self.name}: using {proxy['type']}" + self.logger.debug(f"Host {self.name}: using {proxy['type']}" f" {proxy_name}") self.zbxproxy = proxy return True - self.logger.warning(f"Device {self.name}: unable to find proxy {proxy_name}") + self.logger.warning(f"Host {self.name}: unable to find proxy {proxy_name}") return False def createInZabbix(self, groups, templates, proxies, @@ -419,17 +425,17 @@ class NetworkDevice(): host = self.zabbix.host.create(**create_data) self.zabbix_id = host["hostids"][0] except APIRequestError as e: - e = f"Device {self.name}: Couldn't create. Zabbix returned {str(e)}." + e = f"Host {self.name}: Couldn't create. Zabbix returned {str(e)}." self.logger.error(e) raise SyncExternalError(e) from None # Set Netbox custom field to hostID value. self.nb.custom_fields[device_cf] = int(self.zabbix_id) self.nb.save() - msg = f"Device {self.name}: Created host in Zabbix." + msg = f"Host {self.name}: Created host in Zabbix." self.logger.info(msg) self.create_journal_entry("success", msg) else: - e = f"Device {self.name}: Unable to add to Zabbix. Host already present." + e = f"Host {self.name}: Unable to add to Zabbix. Host already present." self.logger.warning(e) def createZabbixHostgroup(self, hostgroups): @@ -478,7 +484,7 @@ class NetworkDevice(): try: self.zabbix.host.update(hostid=self.zabbix_id, **kwargs) except APIRequestError as e: - e = (f"Device {self.name}: Unable to update. " + e = (f"Host {self.name}: Unable to update. " f"Zabbix returned the following error: {str(e)}.") self.logger.error(e) raise SyncExternalError(e) from None @@ -502,7 +508,7 @@ class NetworkDevice(): if not self.group_id: # Function returns true / false but also sets GroupID if not self.setZabbixGroupID(groups) and not create_hostgroups: - e = (f"Device {self.name}: different hostgroup is required but " + e = (f"Host {self.name}: different hostgroup is required but " "unable to create hostgroup without generation permission.") self.logger.warning(e) raise SyncInventoryError(e) @@ -523,30 +529,30 @@ class NetworkDevice(): self.logger.error(e) raise SyncInventoryError(e) if len(host) == 0: - e = (f"Device {self.name}: No Zabbix host found. " + e = (f"Host {self.name}: No Zabbix host found. " f"This is likely the result of a deleted Zabbix host " f"without zeroing the ID field in Netbox.") self.logger.error(e) raise SyncInventoryError(e) host = host[0] if host["host"] == self.name: - self.logger.debug(f"Device {self.name}: hostname in-sync.") + self.logger.debug(f"Host {self.name}: hostname in-sync.") else: - self.logger.warning(f"Device {self.name}: hostname OUT of sync. " + self.logger.warning(f"Host {self.name}: hostname OUT of sync. " f"Received value: {host['host']}") self.updateZabbixHost(host=self.name) # Execute check depending on wether the name is special or not if self.use_visible_name: if host["name"] == self.visible_name: - self.logger.debug(f"Device {self.name}: visible name in-sync.") + self.logger.debug(f"Host {self.name}: visible name in-sync.") else: - self.logger.warning(f"Device {self.name}: visible name OUT of sync." + self.logger.warning(f"Host {self.name}: visible name OUT of sync." f" Received value: {host['name']}") self.updateZabbixHost(name=self.visible_name) # Check if the templates are in-sync if not self.zbx_template_comparer(host["parentTemplates"]): - self.logger.warning(f"Device {self.name}: template(s) OUT of sync.") + self.logger.warning(f"Host {self.name}: template(s) OUT of sync.") # Prepare Templates for API parsing templateids = [] for template in self.zbx_templates: @@ -555,33 +561,33 @@ class NetworkDevice(): self.updateZabbixHost(templates_clear=host["parentTemplates"], templates=templateids) else: - self.logger.debug(f"Device {self.name}: template(s) in-sync.") + self.logger.debug(f"Host {self.name}: template(s) in-sync.") for group in host["groups"]: if group["groupid"] == self.group_id: - self.logger.debug(f"Device {self.name}: hostgroup in-sync.") + self.logger.debug(f"Host {self.name}: hostgroup in-sync.") break else: - self.logger.warning(f"Device {self.name}: hostgroup OUT of sync.") + self.logger.warning(f"Host {self.name}: hostgroup OUT of sync.") self.updateZabbixHost(groups={'groupid': self.group_id}) if int(host["status"]) == self.zabbix_state: - self.logger.debug(f"Device {self.name}: status in-sync.") + self.logger.debug(f"Host {self.name}: status in-sync.") else: - self.logger.warning(f"Device {self.name}: status OUT of sync.") + self.logger.warning(f"Host {self.name}: status OUT of sync.") self.updateZabbixHost(status=str(self.zabbix_state)) # Check if a proxy has been defined if self.zbxproxy: # Check if proxy or proxy group is defined if (self.zbxproxy["idtype"] in host and host[self.zbxproxy["idtype"]] == self.zbxproxy["id"]): - self.logger.debug(f"Device {self.name}: proxy in-sync.") + self.logger.debug(f"Host {self.name}: proxy in-sync.") # Backwards compatibility for Zabbix <= 6 elif "proxy_hostid" in host and host["proxy_hostid"] == self.zbxproxy["id"]: - self.logger.debug(f"Device {self.name}: proxy in-sync.") + self.logger.debug(f"Host {self.name}: proxy in-sync.") # Proxy does not match, update Zabbix else: - self.logger.warning(f"Device {self.name}: proxy OUT of sync.") + self.logger.warning(f"Host {self.name}: proxy OUT of sync.") # Zabbix <= 6 patch if not str(self.zabbix.version).startswith('7'): self.updateZabbixHost(proxy_hostid=self.zbxproxy['id']) @@ -601,7 +607,7 @@ class NetworkDevice(): proxy_set = True if proxy_power and proxy_set: # Zabbix <= 6 fix - self.logger.warning(f"Device {self.name}: no proxy is configured in Netbox " + self.logger.warning(f"Host {self.name}: no proxy is configured in Netbox " "but is configured in Zabbix. Removing proxy config in Zabbix") if "proxy_hostid" in host and bool(host["proxy_hostid"]): self.updateZabbixHost(proxy_hostid=0) @@ -614,24 +620,24 @@ class NetworkDevice(): # Checks if a proxy has been defined in Zabbix and if proxy_power config has been set if proxy_set and not proxy_power: # Display error message - self.logger.error(f"Device {self.name} is configured " + self.logger.error(f"Host {self.name} is configured " f"with proxy in Zabbix but not in Netbox. The" " -p flag was ommited: no " "changes have been made.") if not proxy_set: - self.logger.debug(f"Device {self.name}: proxy in-sync.") + self.logger.debug(f"Host {self.name}: proxy in-sync.") # Check host inventory mode if str(host['inventory_mode']) == str(self.inventory_mode): - self.logger.debug(f"Device {self.name}: inventory_mode in-sync.") + self.logger.debug(f"Host {self.name}: inventory_mode in-sync.") else: - self.logger.warning(f"Device {self.name}: inventory_mode OUT of sync.") + self.logger.warning(f"Host {self.name}: inventory_mode OUT of sync.") self.updateZabbixHost(inventory_mode=str(self.inventory_mode)) if inventory_sync and self.inventory_mode in [0,1]: # Check host inventory mapping if host['inventory'] == self.inventory: - self.logger.debug(f"Device {self.name}: inventory in-sync.") + self.logger.debug(f"Host {self.name}: inventory in-sync.") else: - self.logger.warning(f"Device {self.name}: inventory OUT of sync.") + self.logger.warning(f"Host {self.name}: inventory OUT of sync.") self.updateZabbixHost(inventory=self.inventory) # If only 1 interface has been found @@ -669,10 +675,10 @@ class NetworkDevice(): updates[key] = item if updates: # If interface updates have been found: push to Zabbix - self.logger.warning(f"Device {self.name}: Interface OUT of sync.") + self.logger.warning(f"Host {self.name}: Interface OUT of sync.") if "type" in updates: # Changing interface type not supported. Raise exception. - e = (f"Device {self.name}: changing interface type to " + e = (f"Host {self.name}: changing interface type to " f"{str(updates['type'])} is not supported.") self.logger.error(e) raise InterfaceConfigError(e) @@ -681,7 +687,7 @@ class NetworkDevice(): try: # API call to Zabbix self.zabbix.hostinterface.update(updates) - e = f"Device {self.name}: solved interface conflict." + e = f"Host {self.name}: solved interface conflict." self.logger.info(e) self.create_journal_entry("info", e) except APIRequestError as e: @@ -690,10 +696,10 @@ class NetworkDevice(): raise SyncExternalError(e) from e else: # If no updates are found, Zabbix interface is in-sync - e = f"Device {self.name}: interface in-sync." + e = f"Host {self.name}: interface in-sync." self.logger.debug(e) else: - e = (f"Device {self.name} has unsupported interface configuration." + e = (f"Host {self.name} has unsupported interface configuration." f" Host has total of {len(host['interfaces'])} interfaces. " "Manual interfention required.") self.logger.error(e) @@ -716,7 +722,7 @@ class NetworkDevice(): } try: self.nb_journals.create(journal) - self.logger.debug(f"Device {self.name}: Created journal entry in Netbox") + self.logger.debug(f"Host {self.name}: Created journal entry in Netbox") return True except JournalError(e) as e: self.logger.warning("Unable to create journal entry for " @@ -743,7 +749,7 @@ class NetworkDevice(): # and add this NB template to the list of successfull templates tmpls_from_zabbix.pop(pos) succesfull_templates.append(nb_tmpl) - self.logger.debug(f"Device {self.name}: template " + self.logger.debug(f"Host {self.name}: template " f"{nb_tmpl['name']} is present in Zabbix.") break if len(succesfull_templates) == len(self.zbx_templates) and len(tmpls_from_zabbix) == 0: diff --git a/modules/hostgroups.py b/modules/hostgroups.py index 5c4f151..c008dbe 100644 --- a/modules/hostgroups.py +++ b/modules/hostgroups.py @@ -63,7 +63,9 @@ class Hostgroup(): format_options["manufacturer"] = self.nb.device_type.manufacturer.name # Variables only applicable for VM's if self.type == "vm": - format_options["cluster"] = str(self.nb.cluster.name) if self.nb.cluster else None + format_options["cluster"] = self.nb.cluster.name + format_options["cluster_type"] = self.nb.cluster.type.name + self.format_options = format_options def generate(self, hg_format=None): diff --git a/modules/virtualMachine.py b/modules/virtualMachine.py index 82ca3d3..d1ab41b 100644 --- a/modules/virtualMachine.py +++ b/modules/virtualMachine.py @@ -1,23 +1,41 @@ """Module that hosts all functions for virtual machine processing""" -from modules.exceptions import * +from modules.device import PhysicalDevice +from modules.hostgroups import Hostgroup +from modules.exceptions import TemplateError +try: + from config import ( + traverse_site_groups, + traverse_regions, + template_cf + ) +except ModuleNotFoundError: + print("Configuration file config.py not found in main directory." + "Please create the file or rename the config.py.example file to config.py.") + sys.exit(0) -class VirtualMachine(): +class VirtualMachine(PhysicalDevice): """Model for virtual machines""" - def __init__(self, nb, name): - self.nb = nb - self.name = name - self.hostgroup = None + def set_hostgroup(self, hg_format, nb_site_groups, nb_regions): + """Set the hostgroup for this device""" + # Create new Hostgroup instance + hg = Hostgroup("vm", self.nb, self.nb_api_version, + traverse_site_groups, traverse_regions, + nb_site_groups, nb_regions) + # Generate hostgroup based on hostgroup format + self.hostgroup = hg.generate(hg_format) - def __repr__(self): - return self.name - - def __str__(self): - return self.__repr__() - - def _data_prep(self): - self.platform = self.nb.platform.name - self.cluster = self.nb.cluster.name - - def set_hostgroup(self): - self.hostgroup = "Virtual machines" + def set_vm_template(self): + """ Set Template for VMs. Overwrites default class + to skip a lookup of custom fields.""" + self.zbx_template_names = None + # Gather templates ONLY from the device specific context + try: + self.zbx_template_names = self.get_templates_context() + except TemplateError as e: + self.logger.warning(e) + return True + + def set_template(self, **kwargs): + """Simple wrapper fur underlying functions""" + self.set_vm_template() \ No newline at end of file diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index 52f9652..e3f023a 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -7,7 +7,8 @@ import argparse from os import environ, path, sys from pynetbox import api from zabbix_utils import ZabbixAPI, APIRequestError, ProcessingError -from modules.device import NetworkDevice +from modules.device import PhysicalDevice +from modules.virtualMachine import VirtualMachine from modules.tools import convert_recordset, proxy_prepper from modules.exceptions import EnvironmentVarError, HostgroupError, SyncError try: @@ -19,7 +20,9 @@ try: zabbix_device_removal, zabbix_device_disable, hostgroup_format, - nb_device_filter + vm_hostgroup_format, + nb_device_filter, + sync_vms ) except ModuleNotFoundError: print("Configuration file config.py not found in main directory." @@ -106,6 +109,7 @@ def main(arguments): proxy_name = "name" # Get all Zabbix and Netbox data netbox_devices = netbox.dcim.devices.filter(**nb_device_filter) + netbox_vms = netbox.virtualization.virtual_machines.all() if sync_vms else list() netbox_site_groups = convert_recordset((netbox.dcim.site_groups.all())) netbox_regions = convert_recordset(netbox.dcim.regions.all()) netbox_journals = netbox.extras.journal_entries @@ -127,11 +131,53 @@ def main(arguments): nb_version = netbox.version # Go through all Netbox devices - for nb_device in netbox_devices: - try: + try: + for nb_vm in netbox_vms: + vm = VirtualMachine(nb_vm, zabbix, netbox_journals, nb_version, + create_journal, logger) + vm.set_hostgroup(vm_hostgroup_format,netbox_site_groups,netbox_regions) + vm.set_vm_template() + vm.set_inventory(nb_vm) + print(f"Templates: {vm.zbx_template_names}") + + # Checks if device is in cleanup state + if vm.status in zabbix_device_removal: + if vm.zabbix_id: + # Delete device from Zabbix + # and remove hostID from Netbox. + vm.cleanup() + logger.info(f"VM {vm.name}: cleanup complete") + continue + # Device has been added to Netbox + # but is not in Activate state + logger.info(f"VM {vm.name}: skipping since this VM is " + f"not in the active state.") + continue + # Check if the VM is in the disabled state + if vm.status in zabbix_device_disable: + vm.zabbix_state = 1 + # Check if VM is already in Zabbix + if vm.zabbix_id: + vm.ConsistencyCheck(zabbix_groups, zabbix_templates, + zabbix_proxy_list, full_proxy_sync, + create_hostgroups) + continue + # Add hostgroup is config is set + if create_hostgroups: + # Create new hostgroup. Potentially multiple groups if nested + hostgroups = vm.createZabbixHostgroup(zabbix_groups) + # go through all newly created hostgroups + for group in hostgroups: + # Add new hostgroups to zabbix group list + zabbix_groups.append(group) + # Add VM to Zabbix + vm.createInZabbix(zabbix_groups, zabbix_templates, + zabbix_proxy_list) + + for nb_device in netbox_devices: # Set device instance set data such as hostgroup and template information. - device = NetworkDevice(nb_device, zabbix, netbox_journals, nb_version, - create_journal, logger) + device = PhysicalDevice(nb_device, zabbix, netbox_journals, nb_version, + create_journal, logger) device.set_hostgroup(hostgroup_format,netbox_site_groups,netbox_regions) device.set_template(templates_config_context, templates_config_context_overrule) device.set_inventory(nb_device) @@ -183,8 +229,8 @@ def main(arguments): # Add device to Zabbix device.createInZabbix(zabbix_groups, zabbix_templates, zabbix_proxy_list) - except SyncError: - pass + except SyncError: + pass if __name__ == "__main__": From 9c07d7dbc4c059098860c700bc40b318e233a7cd Mon Sep 17 00:00:00 2001 From: TheNetworkGuy Date: Fri, 25 Oct 2024 18:54:17 +0200 Subject: [PATCH 05/27] Updated Readme with VM info --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index dfb540d..54dc3f3 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,11 @@ You can make the `zabbix_hostid` field hidden or read-only to prevent human inte This is optional and there is a use case for leaving it read-write in the UI to manually change the ID. For example to re-run a sync. +## Virtual Machine (MV) Syncing +In order to use VM syncing, make sure that the zabbix_id custom field is also present on Virtual machine objects in Netbox. +Furthermore, use the new config.py.example file and set the "sync_vms" variable to True. +You can set the "vm_hostgroup_format" variable to new VM attributes which are not present on devices such as cluster types. + ## Config file ### Hostgroup From 886ef2a1724f437156ffd8eccfb080ff46f7fd44 Mon Sep 17 00:00:00 2001 From: TheNetworkGuy Date: Fri, 25 Oct 2024 18:56:58 +0200 Subject: [PATCH 06/27] Replaced old device role field with new one --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 54dc3f3..1830e83 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,7 @@ You can change this behaviour with the hostgroup_format variable. The following | name | description | | ------------ | ------------ | |dev_location|The device location name| -|dev_role|The device role name| +|role|The device role name| |manufacturer|Manufacturer name| |region|The region name of the device| |site|Site name| @@ -124,7 +124,7 @@ You can change this behaviour with the hostgroup_format variable. The following You can specify the value like so, sperated by a "/": ``` -hostgroup_format = "tenant/site/dev_location/dev_role" +hostgroup_format = "tenant/site/dev_location/role" ``` **Group traversal** From bff34a8e38d92c448fbd06ba9af80952e6723065 Mon Sep 17 00:00:00 2001 From: TheNetworkGuy Date: Wed, 30 Oct 2024 10:06:29 +0100 Subject: [PATCH 07/27] Renamed VM module for python naming convention. Fixed 2 VM hostgroup bugs. Fixed some whitespace in the VM module. --- modules/hostgroups.py | 12 ++++++------ .../{virtualMachine.py => virtual_machine.py} | 17 +++++++++++------ netbox_zabbix_sync.py | 2 +- 3 files changed, 18 insertions(+), 13 deletions(-) rename modules/{virtualMachine.py => virtual_machine.py} (76%) diff --git a/modules/hostgroups.py b/modules/hostgroups.py index c008dbe..4c64f40 100644 --- a/modules/hostgroups.py +++ b/modules/hostgroups.py @@ -42,14 +42,11 @@ class Hostgroup(): # Add default formatting options # Check if a site is configured. A site is optional for VMs format_options["region"] = None - format_options["location"] = None format_options["site_group"] = None if self.nb.site: if self.nb.site.region: format_options["region"] = self.generate_parents("region", str(self.nb.site.region)) - if self.nb.location: - format_options["location"] = str(self.nb.location) if self.nb.site.group: format_options["site_group"] = self.generate_parents("site_group", str(self.nb.site.group)) @@ -61,11 +58,14 @@ class Hostgroup(): # Variables only applicable for devices if self.type == "dev": format_options["manufacturer"] = self.nb.device_type.manufacturer.name + format_options["location"] = str(self.nb.location) if self.nb.location else None # Variables only applicable for VM's if self.type == "vm": - format_options["cluster"] = self.nb.cluster.name - format_options["cluster_type"] = self.nb.cluster.type.name - + # Check if a cluster is configured. Could also be configured in a site. + if self.nb.cluster: + format_options["cluster"] = self.nb.cluster.name + format_options["cluster_type"] = self.nb.cluster.type.name + self.format_options = format_options def generate(self, hg_format=None): diff --git a/modules/virtualMachine.py b/modules/virtual_machine.py similarity index 76% rename from modules/virtualMachine.py rename to modules/virtual_machine.py index d1ab41b..ba58311 100644 --- a/modules/virtualMachine.py +++ b/modules/virtual_machine.py @@ -1,13 +1,13 @@ """Module that hosts all functions for virtual machine processing""" +from os import sys from modules.device import PhysicalDevice from modules.hostgroups import Hostgroup from modules.exceptions import TemplateError try: from config import ( traverse_site_groups, - traverse_regions, - template_cf + traverse_regions ) except ModuleNotFoundError: print("Configuration file config.py not found in main directory." @@ -35,7 +35,12 @@ class VirtualMachine(PhysicalDevice): except TemplateError as e: self.logger.warning(e) return True - - def set_template(self, **kwargs): - """Simple wrapper fur underlying functions""" - self.set_vm_template() \ No newline at end of file + + def set_template(self, prefer_config_context=None, overrule_custom=None): + """ + This wrapper takes the original function and + overwrites it with the set_vm_template function. + This function (set_template) is used by several + other functions such as the consistency check in the parent class. + """ + self.set_vm_template() diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index e3f023a..306f3f4 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -8,7 +8,7 @@ from os import environ, path, sys from pynetbox import api from zabbix_utils import ZabbixAPI, APIRequestError, ProcessingError from modules.device import PhysicalDevice -from modules.virtualMachine import VirtualMachine +from modules.virtual_machine import VirtualMachine from modules.tools import convert_recordset, proxy_prepper from modules.exceptions import EnvironmentVarError, HostgroupError, SyncError try: From 66f24e68911e9d5cb618eeb1d39930c502c8f67e Mon Sep 17 00:00:00 2001 From: TheNetworkGuy Date: Wed, 30 Oct 2024 12:23:15 +0100 Subject: [PATCH 08/27] Fixed several bugs, set default interface for VM to agent, fixed several linter errors. --- modules/__init__.py | 0 modules/device.py | 34 ++++++++++++++++---------------- modules/hostgroups.py | 15 ++++++++------ modules/interface.py | 7 ++++++- modules/virtual_machine.py | 40 ++++++++++++++++++++++++++++---------- netbox_zabbix_sync.py | 3 +-- 6 files changed, 63 insertions(+), 36 deletions(-) create mode 100644 modules/__init__.py diff --git a/modules/__init__.py b/modules/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/modules/device.py b/modules/device.py index e26c7fe..cbc03ec 100644 --- a/modules/device.py +++ b/modules/device.py @@ -25,7 +25,7 @@ except ModuleNotFoundError: sys.exit(0) class PhysicalDevice(): - # pylint: disable=too-many-instance-attributes, too-many-arguments + # pylint: disable=too-many-instance-attributes, too-many-arguments, too-many-positional-arguments """ Represents Network device. INPUT: (Netbox device class, ZabbixAPI class, journal flag, NB journal class) @@ -98,9 +98,9 @@ class PhysicalDevice(): def set_hostgroup(self, hg_format, nb_site_groups, nb_regions): """Set the hostgroup for this device""" # Create new Hostgroup instance - hg = Hostgroup("dev", self.nb, self.nb_api_version, - traverse_site_groups, traverse_regions, - nb_site_groups, nb_regions) + hg = Hostgroup("dev", self.nb, self.nb_api_version) + # Set Hostgroup nesting options + hg.set_nesting(traverse_site_groups, traverse_regions, nb_site_groups, nb_regions) # Generate hostgroup based on hostgroup format self.hostgroup = hg.generate(hg_format) @@ -301,9 +301,9 @@ class PhysicalDevice(): self.logger.info(e) self.create_journal_entry("warning", "Deleted host from Zabbix") except APIRequestError as e: - e = f"Zabbix returned the following error: {str(e)}." - self.logger.error(e) - raise SyncExternalError(e) from e + message = f"Zabbix returned the following error: {str(e)}." + self.logger.error(message) + raise SyncExternalError(message) from e def _zabbixHostnameExists(self): """ @@ -332,12 +332,12 @@ class PhysicalDevice(): if interface.interface["type"] == 2: interface.set_snmp() else: - interface.set_default() + interface.set_default_snmp() return [interface.interface] except InterfaceConfigError as e: - e = f"{self.name}: {e}" - self.logger.warning(e) - raise SyncInventoryError(e) from e + message = f"{self.name}: {e}" + self.logger.warning(message) + raise SyncInventoryError(message) from e def setProxy(self, proxy_list): """ @@ -459,9 +459,9 @@ class PhysicalDevice(): # Add group to final data final_data.append({'groupid': groupid["groupids"][0], 'name': zabbix_hg}) except APIRequestError as e: - e = f"Hostgroup '{zabbix_hg}': unable to create. Zabbix returned {str(e)}." - self.logger.error(e) - raise SyncExternalError(e) from e + msg = f"Hostgroup '{zabbix_hg}': unable to create. Zabbix returned {str(e)}." + self.logger.error(msg) + raise SyncExternalError(msg) from e return final_data def lookupZabbixHostgroup(self, group_list, lookup_group): @@ -691,9 +691,9 @@ class PhysicalDevice(): self.logger.info(e) self.create_journal_entry("info", e) except APIRequestError as e: - e = f"Zabbix returned the following error: {str(e)}." - self.logger.error(e) - raise SyncExternalError(e) from e + msg = f"Zabbix returned the following error: {str(e)}." + self.logger.error(msg) + raise SyncExternalError(msg) from e else: # If no updates are found, Zabbix interface is in-sync e = f"Host {self.name}: interface in-sync." diff --git a/modules/hostgroups.py b/modules/hostgroups.py index 4c64f40..d2487b5 100644 --- a/modules/hostgroups.py +++ b/modules/hostgroups.py @@ -5,9 +5,7 @@ from modules.tools import build_path class Hostgroup(): """Hostgroup class for devices and VM's Takes type (vm or dev) and NB object""" - def __init__(self, obj_type, nb_obj, version, - nested_sitegroup_flag, nested_region_flag, - nb_groups, nb_regions): + def __init__(self, obj_type, nb_obj, version): if obj_type not in ("vm", "dev"): raise HostgroupError(f"Unable to create hostgroup with type {type}") self.type = str(obj_type) @@ -15,8 +13,7 @@ class Hostgroup(): self.name = self.nb.name self.nb_version = version # Used for nested data objects - self.nested_objects = {"site_group": {"flag": nested_sitegroup_flag, "data": nb_groups}, - "region": {"flag": nested_region_flag, "data": nb_regions}} + self.nested_objects = {} self._set_format_options() def __str__(self): @@ -68,13 +65,19 @@ class Hostgroup(): self.format_options = format_options + def set_nesting(self, nested_sitegroup_flag, nested_region_flag, + nb_groups, nb_regions): + """Set nesting options for this Hostgroup""" + self.nested_objects = {"site_group": {"flag": nested_sitegroup_flag, "data": nb_groups}, + "region": {"flag": nested_region_flag, "data": nb_regions}} + def generate(self, hg_format=None): """Generate hostgroup based on a provided format""" # Set format to default in case its not specified if not hg_format: hg_format = "site/manufacturer/role" if self.type == "dev" else "cluster/role" # Split all given names - hg_output = list() + hg_output = [] hg_items = hg_format.split("/") for hg_item in hg_items: # Check if requested data is available as option for this host diff --git a/modules/interface.py b/modules/interface.py index 54586db..384c35e 100644 --- a/modules/interface.py +++ b/modules/interface.py @@ -90,7 +90,7 @@ class ZabbixInterface(): e = "Interface type is not SNMP, unable to set SNMP details" raise InterfaceConfigError(e) - def set_default(self): + def set_default_snmp(self): """ Set default config to SNMPv2, port 161 and community macro. """ self.interface = self.skelet self.interface["type"] = "2" @@ -98,3 +98,8 @@ class ZabbixInterface(): self.interface["details"] = {"version": "2", "community": "{$SNMP_COMMUNITY}", "bulk": "1"} + + def set_default_agent(self): + """Sets interface to Zabbix agent defaults""" + self.interface["type"] = "1" + self.interface["port"] = "10050" diff --git a/modules/virtual_machine.py b/modules/virtual_machine.py index ba58311..87ba1f3 100644 --- a/modules/virtual_machine.py +++ b/modules/virtual_machine.py @@ -1,9 +1,12 @@ +#!/usr/bin/env python3 +# pylint: disable=duplicate-code """Module that hosts all functions for virtual machine processing""" from os import sys from modules.device import PhysicalDevice from modules.hostgroups import Hostgroup -from modules.exceptions import TemplateError +from modules.interface import ZabbixInterface +from modules.exceptions import TemplateError, InterfaceConfigError, SyncInventoryError try: from config import ( traverse_site_groups, @@ -16,12 +19,16 @@ except ModuleNotFoundError: class VirtualMachine(PhysicalDevice): """Model for virtual machines""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.hostgroup = None + self.zbx_template_names = None + def set_hostgroup(self, hg_format, nb_site_groups, nb_regions): """Set the hostgroup for this device""" # Create new Hostgroup instance - hg = Hostgroup("vm", self.nb, self.nb_api_version, - traverse_site_groups, traverse_regions, - nb_site_groups, nb_regions) + hg = Hostgroup("vm", self.nb, self.nb_api_version) + hg.set_nesting(traverse_site_groups, traverse_regions, nb_site_groups, nb_regions) # Generate hostgroup based on hostgroup format self.hostgroup = hg.generate(hg_format) @@ -36,11 +43,24 @@ class VirtualMachine(PhysicalDevice): self.logger.warning(e) return True - def set_template(self, prefer_config_context=None, overrule_custom=None): + def setInterfaceDetails(self): # pylint: disable=invalid-name """ - This wrapper takes the original function and - overwrites it with the set_vm_template function. - This function (set_template) is used by several - other functions such as the consistency check in the parent class. + Overwrites device function to select an agent interface type by default + Agent type interfaces are more likely to be used with VMs then SNMP """ - self.set_vm_template() + try: + # Initiate interface class + interface = ZabbixInterface(self.nb.config_context, self.ip) + # Check if Netbox has device context. + # If not fall back to old config. + if interface.get_context(): + # If device is SNMP type, add aditional information. + if interface.interface["type"] == 2: + interface.set_snmp() + else: + interface.set_default_agent() + return [interface.interface] + except InterfaceConfigError as e: + message = f"{self.name}: {e}" + self.logger.warning(message) + raise SyncInventoryError(message) from e diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index 306f3f4..3be1793 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -109,7 +109,7 @@ def main(arguments): proxy_name = "name" # Get all Zabbix and Netbox data netbox_devices = netbox.dcim.devices.filter(**nb_device_filter) - netbox_vms = netbox.virtualization.virtual_machines.all() if sync_vms else list() + netbox_vms = netbox.virtualization.virtual_machines.all() if sync_vms else [] netbox_site_groups = convert_recordset((netbox.dcim.site_groups.all())) netbox_regions = convert_recordset(netbox.dcim.regions.all()) netbox_journals = netbox.extras.journal_entries @@ -138,7 +138,6 @@ def main(arguments): vm.set_hostgroup(vm_hostgroup_format,netbox_site_groups,netbox_regions) vm.set_vm_template() vm.set_inventory(nb_vm) - print(f"Templates: {vm.zbx_template_names}") # Checks if device is in cleanup state if vm.status in zabbix_device_removal: From 7bf72de0f945213ec73615f13d44d084a854cd4f Mon Sep 17 00:00:00 2001 From: TheNetworkGuy Date: Wed, 30 Oct 2024 13:50:20 +0100 Subject: [PATCH 09/27] Fixed bug where a single host exception would stop the sync. --- netbox_zabbix_sync.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index 3be1793..8b73437 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -131,8 +131,8 @@ def main(arguments): nb_version = netbox.version # Go through all Netbox devices - try: - for nb_vm in netbox_vms: + for nb_vm in netbox_vms: + try: vm = VirtualMachine(nb_vm, zabbix, netbox_journals, nb_version, create_journal, logger) vm.set_hostgroup(vm_hostgroup_format,netbox_site_groups,netbox_regions) @@ -172,8 +172,11 @@ def main(arguments): # Add VM to Zabbix vm.createInZabbix(zabbix_groups, zabbix_templates, zabbix_proxy_list) + except SyncError: + pass - for nb_device in netbox_devices: + for nb_device in netbox_devices: + try: # Set device instance set data such as hostgroup and template information. device = PhysicalDevice(nb_device, zabbix, netbox_journals, nb_version, create_journal, logger) @@ -186,13 +189,13 @@ def main(arguments): # Check if device is primary or secondary if device.promoteMasterDevice(): e = (f"Device {device.name}: is " - f"part of cluster and primary.") + f"part of cluster and primary.") logger.info(e) else: # Device is secondary in cluster. # Don't continue with this device. e = (f"Device {device.name}: is part of cluster " - f"but not primary. Skipping this host...") + f"but not primary. Skipping this host...") logger.info(e) continue # Checks if device is in cleanup state @@ -228,8 +231,8 @@ def main(arguments): # Add device to Zabbix device.createInZabbix(zabbix_groups, zabbix_templates, zabbix_proxy_list) - except SyncError: - pass + except SyncError: + pass if __name__ == "__main__": From d598a9739a91af76c37cf0b48bb136ec4ed61503 Mon Sep 17 00:00:00 2001 From: TheNetworkGuy Date: Wed, 30 Oct 2024 13:51:30 +0100 Subject: [PATCH 10/27] Fixed whitespace --- netbox_zabbix_sync.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index 8b73437..9431525 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -189,13 +189,13 @@ def main(arguments): # Check if device is primary or secondary if device.promoteMasterDevice(): e = (f"Device {device.name}: is " - f"part of cluster and primary.") + f"part of cluster and primary.") logger.info(e) else: # Device is secondary in cluster. # Don't continue with this device. e = (f"Device {device.name}: is part of cluster " - f"but not primary. Skipping this host...") + f"but not primary. Skipping this host...") logger.info(e) continue # Checks if device is in cleanup state From c1504987f1bac5d86077cc335322d60dd3a3f16f Mon Sep 17 00:00:00 2001 From: TheNetworkGuy Date: Wed, 30 Oct 2024 18:21:42 +0100 Subject: [PATCH 11/27] Fixed bug for Tenant hostgroup generation --- modules/hostgroups.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/hostgroups.py b/modules/hostgroups.py index d2487b5..c1d66ce 100644 --- a/modules/hostgroups.py +++ b/modules/hostgroups.py @@ -50,7 +50,7 @@ class Hostgroup(): format_options["role"] = role format_options["site"] = self.nb.site.name if self.nb.site else None format_options["tenant"] = str(self.nb.tenant) if self.nb.tenant else None - format_options["tenant_group"] = str(self.tenant.group) if self.nb.tenant else None + format_options["tenant_group"] = str(self.nb.tenant.group) if self.nb.tenant else None format_options["platform"] = self.nb.platform.name if self.nb.platform else None # Variables only applicable for devices if self.type == "dev": From 509382328711aa7848dd3877902c301897a62a32 Mon Sep 17 00:00:00 2001 From: TheNetworkGuy Date: Wed, 30 Oct 2024 20:52:28 +0100 Subject: [PATCH 12/27] Added some logging and fixed role assignment for VM's --- modules/hostgroups.py | 27 ++++++++++++++++++--------- modules/virtual_machine.py | 6 ++++-- netbox_zabbix_sync.py | 2 ++ 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/modules/hostgroups.py b/modules/hostgroups.py index c1d66ce..a2c7954 100644 --- a/modules/hostgroups.py +++ b/modules/hostgroups.py @@ -1,13 +1,17 @@ """Module for all hostgroup related code""" +from logging import getLogger from modules.exceptions import HostgroupError from modules.tools import build_path class Hostgroup(): """Hostgroup class for devices and VM's Takes type (vm or dev) and NB object""" - def __init__(self, obj_type, nb_obj, version): + def __init__(self, obj_type, nb_obj, version, logger=None): + self.logger = logger if logger else getLogger(__name__) if obj_type not in ("vm", "dev"): - raise HostgroupError(f"Unable to create hostgroup with type {type}") + msg = f"Unable to create hostgroup with type {type}" + self.logger.error() + raise HostgroupError(msg) self.type = str(obj_type) self.nb = nb_obj self.name = self.nb.name @@ -33,9 +37,9 @@ class Hostgroup(): # Role fix for Netbox <=3 role = None if self.nb_version.startswith(("2", "3")) and self.type == "dev": - role = self.nb.device_role.name + role = self.nb.device_role.name if self.nb.device_role else None else: - role = self.nb.role.name + role = self.nb.role.name if self.nb.role else None # Add default formatting options # Check if a site is configured. A site is optional for VMs format_options["region"] = None @@ -86,8 +90,10 @@ class Hostgroup(): cf_data = self.custom_field_lookup(hg_item) # CF does not exist if not cf_data["result"]: - raise HostgroupError(f"Unable to generate hostgroup for host {self.name}. " - f"Item type {hg_item} not supported.") + msg = (f"Unable to generate hostgroup for host {self.name}. " + f"Item type {hg_item} not supported.") + self.logger.error(msg) + raise HostgroupError(msg) # CF data is populated if cf_data["cf"]: hg_output.append(cf_data["cf"]) @@ -100,9 +106,12 @@ class Hostgroup(): # Check if the hostgroup is populated with at least one item. if bool(hg_output): return "/".join(hg_output) - raise HostgroupError(f"Unable to generate hostgroup for host {self.name}." - "Not enough valid items. This is most likely" - "due to the use of custom fields that are empty.") + msg = (f"Unable to generate hostgroup for host {self.name}." + " Not enough valid items. This is most likely" + " due to the use of custom fields that are empty" + " or an invalid hostgroup format.") + self.logger.error(msg) + raise HostgroupError(msg) def list_formatoptions(self): """ diff --git a/modules/virtual_machine.py b/modules/virtual_machine.py index 87ba1f3..a77a813 100644 --- a/modules/virtual_machine.py +++ b/modules/virtual_machine.py @@ -3,6 +3,7 @@ """Module that hosts all functions for virtual machine processing""" from os import sys +from logging import getLogger from modules.device import PhysicalDevice from modules.hostgroups import Hostgroup from modules.interface import ZabbixInterface @@ -23,11 +24,13 @@ class VirtualMachine(PhysicalDevice): super().__init__(*args, **kwargs) self.hostgroup = None self.zbx_template_names = None + if "logger" not in kwargs: + self.logger = getLogger(__name__) def set_hostgroup(self, hg_format, nb_site_groups, nb_regions): """Set the hostgroup for this device""" # Create new Hostgroup instance - hg = Hostgroup("vm", self.nb, self.nb_api_version) + hg = Hostgroup("vm", self.nb, self.nb_api_version, logger=self.logger) hg.set_nesting(traverse_site_groups, traverse_regions, nb_site_groups, nb_regions) # Generate hostgroup based on hostgroup format self.hostgroup = hg.generate(hg_format) @@ -35,7 +38,6 @@ class VirtualMachine(PhysicalDevice): def set_vm_template(self): """ Set Template for VMs. Overwrites default class to skip a lookup of custom fields.""" - self.zbx_template_names = None # Gather templates ONLY from the device specific context try: self.zbx_template_names = self.get_templates_context() diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index 9431525..b6c2b80 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -135,6 +135,7 @@ def main(arguments): try: vm = VirtualMachine(nb_vm, zabbix, netbox_journals, nb_version, create_journal, logger) + logger.debug(f"Host {vm.name}: started operations on VM.") vm.set_hostgroup(vm_hostgroup_format,netbox_site_groups,netbox_regions) vm.set_vm_template() vm.set_inventory(nb_vm) @@ -180,6 +181,7 @@ def main(arguments): # Set device instance set data such as hostgroup and template information. device = PhysicalDevice(nb_device, zabbix, netbox_journals, nb_version, create_journal, logger) + logger.debug(f"Host {device.name}: started operations on device.") device.set_hostgroup(hostgroup_format,netbox_site_groups,netbox_regions) device.set_template(templates_config_context, templates_config_context_overrule) device.set_inventory(nb_device) From f1da1cfb50ed394dff990702eea9809364d6e026 Mon Sep 17 00:00:00 2001 From: TheNetworkGuy Date: Wed, 30 Oct 2024 20:57:20 +0100 Subject: [PATCH 13/27] Fixed logging formatting for submodules --- modules/virtual_machine.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/virtual_machine.py b/modules/virtual_machine.py index a77a813..8bbcd63 100644 --- a/modules/virtual_machine.py +++ b/modules/virtual_machine.py @@ -24,8 +24,6 @@ class VirtualMachine(PhysicalDevice): super().__init__(*args, **kwargs) self.hostgroup = None self.zbx_template_names = None - if "logger" not in kwargs: - self.logger = getLogger(__name__) def set_hostgroup(self, hg_format, nb_site_groups, nb_regions): """Set the hostgroup for this device""" From 20096a215b49dc8ea6e09856f19cf1f2ca08b856 Mon Sep 17 00:00:00 2001 From: TheNetworkGuy Date: Wed, 30 Oct 2024 21:10:22 +0100 Subject: [PATCH 14/27] Added HG and template checks for devices and VM's. Temp disabled inventory mapping for VMs --- netbox_zabbix_sync.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index b6c2b80..4a58dfa 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -137,8 +137,15 @@ def main(arguments): create_journal, logger) logger.debug(f"Host {vm.name}: started operations on VM.") vm.set_hostgroup(vm_hostgroup_format,netbox_site_groups,netbox_regions) + # Check if a valid hostgroup has been found for this VM. + if not vm.hostgroup: + continue vm.set_vm_template() - vm.set_inventory(nb_vm) + # Check if a valid template has been found for this VM. + if not vm.zbx_template_names: + continue + # Temporary disable inventory sync for VM's + # vm.set_inventory(nb_vm) # Checks if device is in cleanup state if vm.status in zabbix_device_removal: @@ -183,7 +190,13 @@ def main(arguments): create_journal, logger) logger.debug(f"Host {device.name}: started operations on device.") device.set_hostgroup(hostgroup_format,netbox_site_groups,netbox_regions) + # Check if a valid hostgroup has been found for this VM. + if not device.hostgroup: + continue device.set_template(templates_config_context, templates_config_context_overrule) + # Check if a valid template has been found for this VM. + if not device.zbx_template_names: + continue device.set_inventory(nb_device) # Checks if device is part of cluster. # Requires clustering variable From 06f97b132a1be0a409072adb274005bfc254f7ad Mon Sep 17 00:00:00 2001 From: TheNetworkGuy Date: Wed, 30 Oct 2024 21:25:58 +0100 Subject: [PATCH 15/27] Changed some logging messages and removed import logging statement from troubleshooting --- modules/device.py | 8 ++++---- modules/virtual_machine.py | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/modules/device.py b/modules/device.py index cbc03ec..ee54891 100644 --- a/modules/device.py +++ b/modules/device.py @@ -145,12 +145,12 @@ class PhysicalDevice(): def get_templates_context(self): """ Get Zabbix templates from the device context """ if "zabbix" not in self.config_context: - e = ("Key 'zabbix' not found in config " - f"context for template host {self.name}") + e = (f"Host {self.name}: Key 'zabbix' not found in config " + "context for template") raise TemplateError(e) if "templates" not in self.config_context["zabbix"]: - e = ("Key 'templates' not found in config " - f"context 'zabbix' for template host {self.name}") + e = (f"Host {self.name}: Key 'templates' not found in config " + "context 'zabbix' for template host") raise TemplateError(e) return self.config_context["zabbix"]["templates"] diff --git a/modules/virtual_machine.py b/modules/virtual_machine.py index 8bbcd63..788a903 100644 --- a/modules/virtual_machine.py +++ b/modules/virtual_machine.py @@ -3,7 +3,6 @@ """Module that hosts all functions for virtual machine processing""" from os import sys -from logging import getLogger from modules.device import PhysicalDevice from modules.hostgroups import Hostgroup from modules.interface import ZabbixInterface From 9417908994cb79a3f5edaabdde000cc4596a4cbd Mon Sep 17 00:00:00 2001 From: TheNetworkGuy Date: Thu, 31 Oct 2024 15:51:33 +0100 Subject: [PATCH 16/27] Swapped hostgroup and template calculation since template lookup is a read-only operation and hostgroup lookup could be a read-write operation. --- netbox_zabbix_sync.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index 4a58dfa..658713d 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -136,14 +136,14 @@ def main(arguments): vm = VirtualMachine(nb_vm, zabbix, netbox_journals, nb_version, create_journal, logger) logger.debug(f"Host {vm.name}: started operations on VM.") - vm.set_hostgroup(vm_hostgroup_format,netbox_site_groups,netbox_regions) - # Check if a valid hostgroup has been found for this VM. - if not vm.hostgroup: - continue vm.set_vm_template() # Check if a valid template has been found for this VM. if not vm.zbx_template_names: continue + vm.set_hostgroup(vm_hostgroup_format,netbox_site_groups,netbox_regions) + # Check if a valid hostgroup has been found for this VM. + if not vm.hostgroup: + continue # Temporary disable inventory sync for VM's # vm.set_inventory(nb_vm) @@ -189,14 +189,14 @@ def main(arguments): device = PhysicalDevice(nb_device, zabbix, netbox_journals, nb_version, create_journal, logger) logger.debug(f"Host {device.name}: started operations on device.") - device.set_hostgroup(hostgroup_format,netbox_site_groups,netbox_regions) - # Check if a valid hostgroup has been found for this VM. - if not device.hostgroup: - continue device.set_template(templates_config_context, templates_config_context_overrule) # Check if a valid template has been found for this VM. if not device.zbx_template_names: continue + device.set_hostgroup(hostgroup_format,netbox_site_groups,netbox_regions) + # Check if a valid hostgroup has been found for this VM. + if not device.hostgroup: + continue device.set_inventory(nb_device) # Checks if device is part of cluster. # Requires clustering variable From ffc2aa19472596856fe6f8a9397fe498b4a76bc5 Mon Sep 17 00:00:00 2001 From: TheNetworkGuy Date: Thu, 31 Oct 2024 20:03:09 +0100 Subject: [PATCH 17/27] Fixed location hostgroup bug --- netbox_zabbix_sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index 658713d..bcf4558 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -79,7 +79,7 @@ def main(arguments): netbox = api(netbox_host, token=netbox_token, threading=True) # Check if the provided Hostgroup layout is valid hg_objects = hostgroup_format.split("/") - allowed_objects = ["dev_location", "role", "manufacturer", "region", + allowed_objects = ["location", "role", "manufacturer", "region", "site", "site_group", "tenant", "tenant_group"] # Create API call to get all custom fields which are on the device objects device_cfs = netbox.extras.custom_fields.filter(type="text", content_type_id=23) From 56c19d97de8979c54ab1b5f20c6351aeb3dbaeee Mon Sep 17 00:00:00 2001 From: TheNetworkGuy Date: Wed, 6 Nov 2024 15:57:11 +0100 Subject: [PATCH 18/27] Added basic error message when Netbox details are invalid. Fixed logging message for when template context keys are not present. --- modules/device.py | 4 ++-- netbox_zabbix_sync.py | 12 +++++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/modules/device.py b/modules/device.py index ee54891..76a41a7 100644 --- a/modules/device.py +++ b/modules/device.py @@ -146,11 +146,11 @@ class PhysicalDevice(): """ Get Zabbix templates from the device context """ if "zabbix" not in self.config_context: e = (f"Host {self.name}: Key 'zabbix' not found in config " - "context for template") + "context for template lookup") raise TemplateError(e) if "templates" not in self.config_context["zabbix"]: e = (f"Host {self.name}: Key 'templates' not found in config " - "context 'zabbix' for template host") + "context 'zabbix' for template lookup") raise TemplateError(e) return self.config_context["zabbix"]["templates"] diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index bcf4558..cfe95fb 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -6,6 +6,8 @@ import logging import argparse from os import environ, path, sys from pynetbox import api +from pynetbox.core.query import RequestError as NBRequestError +from requests.exceptions import ConnectionError as RequestsConnectionError from zabbix_utils import ZabbixAPI, APIRequestError, ProcessingError from modules.device import PhysicalDevice from modules.virtual_machine import VirtualMachine @@ -82,7 +84,15 @@ def main(arguments): allowed_objects = ["location", "role", "manufacturer", "region", "site", "site_group", "tenant", "tenant_group"] # Create API call to get all custom fields which are on the device objects - device_cfs = netbox.extras.custom_fields.filter(type="text", content_type_id=23) + try: + device_cfs = list(netbox.extras.custom_fields.filter(type="text", content_type_id=23)) + except RequestsConnectionError: + logger.error(f"Unable to connect to Netbox with URL {netbox_host}." + " Please check the URL and status of Netbox.") + sys.exit(1) + except NBRequestError as e: + logger.error(f"Netbox error: {e}") + sys.exit(1) for cf in device_cfs: allowed_objects.append(cf.name) for hg_object in hg_objects: From 30545ec0f33e9014818fa19c35bcf2137dcaeacf Mon Sep 17 00:00:00 2001 From: TheNetworkGuy Date: Wed, 13 Nov 2024 19:39:24 +0100 Subject: [PATCH 19/27] Added hostname filtering based on Cyrillic characters --- modules/device.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/device.py b/modules/device.py index 76a41a7..d96bce5 100644 --- a/modules/device.py +++ b/modules/device.py @@ -4,6 +4,7 @@ Device specific handeling for Netbox to Zabbix """ from os import sys +from re import search from logging import getLogger from zabbix_utils import APIRequestError from modules.exceptions import (SyncInventoryError, TemplateError, SyncExternalError, @@ -85,7 +86,8 @@ class PhysicalDevice(): # Validate hostname format. odd_character_list = ["ä", "ö", "ü", "Ä", "Ö", "Ü", "ß"] self.use_visible_name = False - if any(letter in self.name for letter in odd_character_list): + if (any(letter in self.name for letter in odd_character_list) or + bool(search('[\u0400-\u04FF]', self.name))): self.name = f"NETBOX_ID{self.id}" self.visible_name = self.nb.name self.use_visible_name = True From 09a6906a63c90ccc7f99d674a00d7832080042f9 Mon Sep 17 00:00:00 2001 From: TheNetworkGuy Date: Wed, 13 Nov 2024 19:56:09 +0100 Subject: [PATCH 20/27] Added VM filtering --- config.py.example | 2 ++ netbox_zabbix_sync.py | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/config.py.example b/config.py.example index ca171cf..14ecfb1 100644 --- a/config.py.example +++ b/config.py.example @@ -63,6 +63,8 @@ traverse_site_groups = False # Default device filter, only get devices which have a name in Netbox: nb_device_filter = {"name__n": "null"} +# Default filter for VMs +vm_device_filter = {"name__n": "null"} ## Inventory # See https://www.zabbix.com/documentation/current/en/manual/config/hosts/inventory#building-inventory diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index cfe95fb..123aafa 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -24,7 +24,8 @@ try: hostgroup_format, vm_hostgroup_format, nb_device_filter, - sync_vms + sync_vms, + vm_device_filter ) except ModuleNotFoundError: print("Configuration file config.py not found in main directory." @@ -119,7 +120,7 @@ def main(arguments): proxy_name = "name" # Get all Zabbix and Netbox data netbox_devices = netbox.dcim.devices.filter(**nb_device_filter) - netbox_vms = netbox.virtualization.virtual_machines.all() if sync_vms else [] + netbox_vms = netbox.virtualization.virtual_machines.filter(**vm_device_filter) if sync_vms else [] netbox_site_groups = convert_recordset((netbox.dcim.site_groups.all())) netbox_regions = convert_recordset(netbox.dcim.regions.all()) netbox_journals = netbox.extras.journal_entries From e0827ac428a1247bf60814f759299971f0729ed4 Mon Sep 17 00:00:00 2001 From: TheNetworkGuy Date: Wed, 13 Nov 2024 20:00:50 +0100 Subject: [PATCH 21/27] Fixed long line linter error --- netbox_zabbix_sync.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index 123aafa..b5e382b 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -119,8 +119,10 @@ def main(arguments): else: proxy_name = "name" # Get all Zabbix and Netbox data - netbox_devices = netbox.dcim.devices.filter(**nb_device_filter) - netbox_vms = netbox.virtualization.virtual_machines.filter(**vm_device_filter) if sync_vms else [] + netbox_devices = list(netbox.dcim.devices.filter(**nb_device_filter)) + netbox_vms = [] + if sync_vms: + netbox_vms = list(netbox.virtualization.virtual_machines.filter(**vm_device_filter)) netbox_site_groups = convert_recordset((netbox.dcim.site_groups.all())) netbox_regions = convert_recordset(netbox.dcim.regions.all()) netbox_journals = netbox.extras.journal_entries From 204937b784f157acaef478de8cba0b8983e76858 Mon Sep 17 00:00:00 2001 From: TheNetworkGuy Date: Wed, 13 Nov 2024 20:32:47 +0100 Subject: [PATCH 22/27] Fixed variable name typo --- config.py.example | 2 +- netbox_zabbix_sync.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config.py.example b/config.py.example index 14ecfb1..da2f5cc 100644 --- a/config.py.example +++ b/config.py.example @@ -64,7 +64,7 @@ traverse_site_groups = False # Default device filter, only get devices which have a name in Netbox: nb_device_filter = {"name__n": "null"} # Default filter for VMs -vm_device_filter = {"name__n": "null"} +nb_vm_filter = {"name__n": "null"} ## Inventory # See https://www.zabbix.com/documentation/current/en/manual/config/hosts/inventory#building-inventory diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index b5e382b..7e65ba1 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -25,7 +25,7 @@ try: vm_hostgroup_format, nb_device_filter, sync_vms, - vm_device_filter + nb_vm_filter ) except ModuleNotFoundError: print("Configuration file config.py not found in main directory." @@ -122,7 +122,7 @@ def main(arguments): netbox_devices = list(netbox.dcim.devices.filter(**nb_device_filter)) netbox_vms = [] if sync_vms: - netbox_vms = list(netbox.virtualization.virtual_machines.filter(**vm_device_filter)) + netbox_vms = list(netbox.virtualization.virtual_machines.filter(**nb_vm_filter)) netbox_site_groups = convert_recordset((netbox.dcim.site_groups.all())) netbox_regions = convert_recordset(netbox.dcim.regions.all()) netbox_journals = netbox.extras.journal_entries From 5d4ff9c5edf6662b85c1b3701cb119a01cf11ddc Mon Sep 17 00:00:00 2001 From: TheNetworkGuy Date: Fri, 15 Nov 2024 14:03:42 +0100 Subject: [PATCH 23/27] Fixed #79 --- modules/device.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/modules/device.py b/modules/device.py index d96bce5..d1a19a0 100644 --- a/modules/device.py +++ b/modules/device.py @@ -296,10 +296,16 @@ class PhysicalDevice(): """ if self.zabbix_id: try: - self.zabbix.host.delete(self.zabbix_id) - self.nb.custom_fields[device_cf] = None - self.nb.save() - e = f"Host {self.name}: Deleted host from Zabbix." + # Check if the Zabbix host exists in Zabbix + zbx_host = bool(self.zabbix.host.get(filter={'hostid': self.zabbix_id}, + output=[])) + e = (f"Host {self.name}: was already deleted from Zabbix." + " Removed link in Netbox.") + if zbx_host: + # Delete host should it exists + self.zabbix.host.delete(self.zabbix_id) + e = f"Host {self.name}: Deleted host from Zabbix." + self.zeroize_cf() self.logger.info(e) self.create_journal_entry("warning", "Deleted host from Zabbix") except APIRequestError as e: @@ -307,6 +313,12 @@ class PhysicalDevice(): self.logger.error(message) raise SyncExternalError(message) from e + def zeroize_cf(self): + """Sets the hostID custom field in Netbox to zero, + effectively destroying the link""" + self.nb.custom_fields[device_cf] = None + self.nb.save() + def _zabbixHostnameExists(self): """ Checks if hostname exists in Zabbix. From 0155c29fcc11e9fffaf4d9fe1caf5e86b1ab82d9 Mon Sep 17 00:00:00 2001 From: TheNetworkGuy Date: Fri, 15 Nov 2024 14:08:04 +0100 Subject: [PATCH 24/27] Fixed Pylint too-many-public-methods error --- modules/device.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/device.py b/modules/device.py index d1a19a0..5da8a16 100644 --- a/modules/device.py +++ b/modules/device.py @@ -305,7 +305,7 @@ class PhysicalDevice(): # Delete host should it exists self.zabbix.host.delete(self.zabbix_id) e = f"Host {self.name}: Deleted host from Zabbix." - self.zeroize_cf() + self._zeroize_cf() self.logger.info(e) self.create_journal_entry("warning", "Deleted host from Zabbix") except APIRequestError as e: @@ -313,7 +313,7 @@ class PhysicalDevice(): self.logger.error(message) raise SyncExternalError(message) from e - def zeroize_cf(self): + def _zeroize_cf(self): """Sets the hostID custom field in Netbox to zero, effectively destroying the link""" self.nb.custom_fields[device_cf] = None From 0996059c5f9abe64e987de1ad06f1a9df8fc7325 Mon Sep 17 00:00:00 2001 From: TheNetworkGuy Date: Mon, 18 Nov 2024 12:58:57 +0100 Subject: [PATCH 25/27] Added better documentation for VMs nd fixed typo --- README.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1830e83..1a87395 100644 --- a/README.md +++ b/README.md @@ -87,10 +87,14 @@ You can make the `zabbix_hostid` field hidden or read-only to prevent human inte This is optional and there is a use case for leaving it read-write in the UI to manually change the ID. For example to re-run a sync. -## Virtual Machine (MV) Syncing +## Virtual Machine (VM) Syncing In order to use VM syncing, make sure that the zabbix_id custom field is also present on Virtual machine objects in Netbox. -Furthermore, use the new config.py.example file and set the "sync_vms" variable to True. -You can set the "vm_hostgroup_format" variable to new VM attributes which are not present on devices such as cluster types. + +Use the `config.py` file and set the `sync_vms` variable to `True`. + +You can set the `vm_hostgroup_format` variable to new VM attributes which are not present on devices such as cluster types. + +To enable filtering for VM's, check the `nb_vm_filter` variable out. It works the same as with the device filter. However note that not all filtering capabilities and properties of devices are applicable to VM's and vice-versa. Check the Netbox API documentation to see which filtering options are available for each object type. ## Config file From 3f4d173ac011da214d3da39ef943e54a524cfd35 Mon Sep 17 00:00:00 2001 From: TheNetworkGuy Date: Mon, 18 Nov 2024 12:59:50 +0100 Subject: [PATCH 26/27] Markdown fixed on custom field in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1a87395..d0ba4ee 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ You can make the `zabbix_hostid` field hidden or read-only to prevent human inte This is optional and there is a use case for leaving it read-write in the UI to manually change the ID. For example to re-run a sync. ## Virtual Machine (VM) Syncing -In order to use VM syncing, make sure that the zabbix_id custom field is also present on Virtual machine objects in Netbox. +In order to use VM syncing, make sure that the `zabbix_id` custom field is also present on Virtual machine objects in Netbox. Use the `config.py` file and set the `sync_vms` variable to `True`. From 2177234d7f4f7c88c5c518814901b34687091c13 Mon Sep 17 00:00:00 2001 From: TheNetworkGuy Date: Mon, 18 Nov 2024 13:27:24 +0100 Subject: [PATCH 27/27] Fixed some documentation --- README.md | 50 ++++++++++++++++++++++++++++++++--------------- config.py.example | 3 ++- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index d0ba4ee..b834d95 100644 --- a/README.md +++ b/README.md @@ -92,41 +92,52 @@ In order to use VM syncing, make sure that the `zabbix_id` custom field is also Use the `config.py` file and set the `sync_vms` variable to `True`. -You can set the `vm_hostgroup_format` variable to new VM attributes which are not present on devices such as cluster types. +You can set the `vm_hostgroup_format` variable to a customizable value for VM hostgroups. The default is `cluster_type/cluster/role`. -To enable filtering for VM's, check the `nb_vm_filter` variable out. It works the same as with the device filter. However note that not all filtering capabilities and properties of devices are applicable to VM's and vice-versa. Check the Netbox API documentation to see which filtering options are available for each object type. +To enable filtering for VM's, check the `nb_vm_filter` variable out. It works the same as with the device filter (see documentation under "Hostgroup layout"). Note that not all filtering capabilities and properties of devices are applicable to VM's and vice-versa. Check the Netbox API documentation to see which filtering options are available for each object type. ## Config file ### Hostgroup -Setting the `create_hostgroups` variable to `False` requires manual hostgroup creation for devices in a new category. +Setting the `create_hostgroups` variable to `False` requires manual hostgroup creation for devices in a new category. I would recommend setting this variable to `True` since leaving it on `False` results in a lot of manual work. -The format can be set with the `hostgroup_format` variable. +The format can be set with the `hostgroup_format` variable for devices and `vm_hostgroup_format` for devices. -Any nested parent hostgroups will also be created automatically. +Any nested parent hostgroups will also be created automatically. For instance the region `Berlin` with parent region `Germany` will create the hostgroup `Germany/Berlin`. Make sure that the Zabbix user has proper permissions to create hosts. The hostgroups are in a nested format. This means that proper permissions only need to be applied to the site name hostgroup and cascaded to any child hostgroups. #### Layout The default hostgroup layout is "site/manufacturer/device_role". - -**Variables** - You can change this behaviour with the hostgroup_format variable. The following values can be used: + +**Both devices and virtual machines** | name | description | | ------------ | ------------ | -|dev_location|The device location name| -|role|The device role name| -|manufacturer|Manufacturer name| -|region|The region name of the device| +|role|Role name of a device or VM| +|region|The region name| |site|Site name| |site_group|Site group name| |tenant|Tenant name| |tenant_group|Tenant group name| +|platform|Software platform of a device or VM| +|custom fields|See the section "Layout -> Custom Fields" to use custom fields as hostgroup variable| + +**Only for devices** +| name | description | +| ------------ | ------------ | +|location|The device location name| +|manufacturer|Device manufacturer name| + +**Only for VMs** +| name | description | +| ------------ | ------------ | +|cluster|VM cluster name| +|cluster_type|VM cluster type| -You can specify the value like so, sperated by a "/": +You can specify the value sperated by a "/" like so: ``` hostgroup_format = "tenant/site/dev_location/role" ``` @@ -138,11 +149,18 @@ However, by setting `traverse_region` to `True` in `config.py` the script will r **Custom fields** -You can also use the value of custom fields under the device object. +You can use the value of custom fields for hostgroup generation. This allows more freedom and even allows a full static mapping instead of a dynamic rendered hostgroup name. -This allows more freedom and even allows a full static mapping instead of a dynamic rendered hostgroup name. +For instance a custom field with the name `mycustomfieldname` and type string has the following values for 2 devices: ``` -hostgroup_format = "site/mycustomfieldname" +Device A has the value Train for custom field mycustomfieldname. +Device B has the value Bus for custom field mycustomfieldname. +Both devices are located in the site Paris. +``` +With the hostgroup format `site/mycustomfieldname` the following hostgroups will be generated: +``` +Device A: Paris/Train +Device B: Paris/Bus ``` **Empty variables or hostgroups** diff --git a/config.py.example b/config.py.example index da2f5cc..7228aad 100644 --- a/config.py.example +++ b/config.py.example @@ -26,6 +26,7 @@ create_journal = False # Use the hostgroup vm_hostgroup_format mapper for specific # hostgroup atributes of VM's such as cluster_type and cluster sync_vms = False +# Check the README documentation for values to use in the VM hostgroup format. vm_hostgroup_format = "cluster_type/cluster/role" ## Proxy Sync @@ -39,7 +40,7 @@ zabbix_device_removal = ["Decommissioning", "Inventory"] zabbix_device_disable = ["Offline", "Planned", "Staged", "Failed"] ## Hostgroup mapping -# Available choices: dev_location, dev_role, manufacturer, region, site, site_group, tenant, tenant_group +# See the README documentation for available options # You can also use CF (custom field) names under the device. The CF content will be used for the hostgroup generation. # # When using region in the group name, the default behaviour is to use name of the directly assigned region.