From e2ddb068e9842d2411d20ed7f8725b97511faf29 Mon Sep 17 00:00:00 2001 From: TheNetworkGuy Date: Wed, 21 Feb 2024 13:07:42 +0100 Subject: [PATCH 1/2] Added new hostgroup variables --- netbox_zabbix_sync.py | 66 ++++++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index 0a06c4d..17e9cfa 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -54,7 +54,8 @@ def main(arguments): # Check if the provided Hostgroup layout is valid if(arguments.layout): hg_objects = arguments.layout.split("/") - allowed_objects = ["site", "manufacturer", "tenant", "dev_role"] + allowed_objects = ["dev_location", "dev_role", "manufacturer", "region", + "site", "site_group", "tenant", "tenant_group"] # Create API call to get all custom fields which are on the device objects device_cfs = netbox.extras.custom_fields.filter(type="text", content_type_id=23) for cf in device_cfs: @@ -188,7 +189,6 @@ class NetworkDevice(): self.zabbix = zabbix self.tenant = nb.tenant self.config_context = nb.config_context - self.hostgroup = "" self.zbxproxy = "0" self.zabbix_state = 0 self.journal = journal @@ -219,44 +219,46 @@ class NetworkDevice(): def set_hostgroup(self, format): """Set the hostgroup for this device""" # Get all variables from the NB data - site = self.nb.site.name + dev_location = str(self.nb.location) if self.nb.location else None + dev_role = self.nb.device_role.name manufacturer = self.nb.device_type.manufacturer.name - role = self.nb.device_role.name - tenant = self.tenant.name if self.tenant else None - - hostgroup_vars = {"site": site, "manufacturer": manufacturer, - "dev_role": role, "tenant": tenant} - items = format.split("/") + region = str(self.nb.site.region) if self.nb.site.region else None + site = self.nb.site.name + site_group = str(self.nb.site.group) if self.nb.site.group else None + tenant = str(self.tenant) if self.tenant else None + tenant_group = str(self.tenant.group) if tenant else None + # Set mapper for string -> variable + hostgroup_vars = {"dev_location": dev_location, "dev_role": dev_role, + "manufacturer": manufacturer, "region": region, + "site": site, "site_group": site_group, + "tenant": tenant, "tenant_group": tenant_group} + # Generate list based off string input format + hg_items = format.split("/") + hostgroup = "" # Go through all hostgroup items - for item in items: - # Check if this item is not the first in the hostgroup format - if(self.hostgroup): - self.hostgroup += "/" - # Check if the item is not a standard item, A.K.A. custom field name - if(item not in hostgroup_vars): - # check if the item is in the custom fields - if(item in self.nb.custom_fields): - cf_value = self.nb.custom_fields[item] - # check if the CF is empty. - if(not cf_value): - # Remove the previously inserted / - self.hostgroup = self.hostgroup[:-1] - continue - else: - self.hostgroup += cf_value - continue - else: - continue - # Check if the variable (such as Tenant) is empty + for item in hg_items: + # Check if the variable (such as Tenant) is empty. if(not hostgroup_vars[item]): continue - # Add the item to the hostgroup format - self.hostgroup += hostgroup_vars[item] - if(not self.hostgroup): + # Check if the item is a custom field name + if(item not in hostgroup_vars): + cf_value = self.nb.custom_fields[item] if item in self.nb.custom_fields else None + if(cf_value): + # If there is a cf match, add the value of this cf to the hostgroup + hostgroup += cf_value + "/" + # Should there not be a match, this means that + # the variable is invalid. Skip regardless. + continue + # Add value of predefined variable to hostgroup format + hostgroup += hostgroup_vars[item] + "/" + # If the final hostgroup variable is empty + if(not hostgroup): e = (f"{self.name} has no reliable hostgroup. This is" "most likely due to the use of custom fields that are empty.") logger.error(e) raise SyncInventoryError(e) + # Remove final inserted "/" and set hostgroup to class var + self.hostgroup = hostgroup.rstrip("/") def set_template(self, prefer_config_context, overrule_custom): self.zbx_template_names = None From 3f28986c094991baf7654ede92f3e4a4b7fa504e Mon Sep 17 00:00:00 2001 From: TheNetworkGuy Date: Thu, 22 Feb 2024 14:10:59 +0100 Subject: [PATCH 2/2] Implemented #48 --- README.md | 171 ++++++++++++++++++++++-------------------- config.py.example | 19 +++++ netbox_zabbix_sync.py | 75 ++++++++---------- 3 files changed, 141 insertions(+), 124 deletions(-) diff --git a/README.md b/README.md index c4c9e66..c815fcd 100644 --- a/README.md +++ b/README.md @@ -5,18 +5,19 @@ A script to create, update and delete Zabbix hosts using Netbox device objects. ## Installation -### Packages -Make sure that you have a python environment with the following packages installed. -``` -pynetbox -pyzabbix -``` ### Cloning the repository ``` git clone https://github.com/TheNetworkGuy/netbox-zabbix-sync.git ``` +### Packages +Make sure that you have a python environment with the following packages installed. You can also use the requirements.txt file for installation with pip. +``` +pynetbox +pyzabbix +``` + ### Config file First time user? Copy the config.py.example file to config.py. This file is used for modifying filters and setting variables such as custom field names. ``` @@ -52,6 +53,83 @@ You can make the hostID field hidden or read-only to prevent human intervention. This is optional and there is a use case for leaving it read-write in the UI to manually change the ID. For example to re-run a sync. +## Config file + +### Hostgroup +Setting the create_hostgroups variable to False requires manual hostgroup creation for devices in a new category. + +The format can be set with the hostgroup_format variable. + +Make sure that the Zabbix user has proper permissions to create hosts. +The hostgroups are in a nested format. This means that proper permissions only need to be applied to the site name hostgroup and cascaded to any child hostgroups. + +#### layout +The default hostgroup layout is "site/manufacturer/device_role". + +**Variables** + +You can change this behaviour with the hostgroup_format variable. The following values can be used: +| name | description | +| ------------ | ------------ | +|dev_location|The device location name| +|dev_role|The device role name| +|manufacturer|Manufacturer name| +|region|The region name of the device| +|site|Site name| +|site_group|Site group name| +|tenant|Tenant name| +|tenant_group|Tenant group name| + + +You can specify the value like so, sperated by a "/": +``` +hostgroup_format = "tenant/site/dev_location/dev_role" +``` +**custom fields** + +You can also use the value of custom fields under the device object. + +This allows more freedom and even allows a full static mapping instead of a dynamic rendered hostgroup name. +``` +hostgroup_format = "site/mycustomfieldname" +``` +**Empty variables or hostgroups** + +Should the content of a variable be empty, then the hostgroup position is skipped. + +For example, consider the following scenario with 2 devices, both the same device type and site. One of them is linked to a tenant, the other one does not have a relationship with a tenant. +- Device_role: PDU +- Site: HQ-AMS +``` +hostgroup_format = "site/tenant/device_role" +``` +When running the script like above, the following hostgroup (HG) will be generated for both hosts: + - Device A with no relationship with a tenant: HQ-AMS/PDU + - Device B with a relationship to tenant "Fork Industries": HQ-AMS/Fork Industries/PDU + +The same logic applies to custom fields being used in the HG format: +``` +hostgroup_format = "site/mycustomfieldname" +``` +For device A with the value "ABC123" in the custom field "mycustomfieldname" -> HQ-AMS/ABC123 +For a device which does not have a value in the custom field "mycustomfieldname" -> HQ-AMS + +Should there be a scenario where a custom field does not have a value under a device, and the HG format only uses this single variable, then this will result in an error: +``` +hostgroup_format = "mycustomfieldname" + +Netbox-Zabbix-sync - ERROR - ESXI1 has no reliable hostgroup. This is most likely due to the use of custom fields that are empty. +``` +### Device status +By setting a status on a Netbox device you determine how the host is added (or updated) in Zabbix. There are, by default, 3 options: +* Delete the host from Zabbix (triggered by Netbox status "Decommissioning" and "Inventory") +* Create the host in Zabbix but with a disabled status (Trigger by "Offline", "Planned", "Staged" and "Failed") +* Create the host in Zabbix with an enabled status (For now only enabled with the "Active" status) + +You can modify this behaviour by changing the following list variables in the script: + - zabbix_device_removal + - zabbix_device_disable + ### Template source You can either use a Netbox device type custom field or Netbox config context for the Zabbix template information. @@ -106,83 +184,9 @@ python3 netbox_zabbix_sync.py ### Flags | Flag | Option | Description | | ------------ | ------------ | ------------ | -| -c | cluster | For clustered devices: only add the primary node of a cluster and use the cluster name as hostname. | -| -H | hostgroup | Create non-existing hostgroups in Zabbix. Usefull for a first run to add all required hostgroups. | -| -l | layout | Set the hostgroup layout. Default is site/manufacturer/dev_role. Posible options (seperated with '/'): site, manufacturer, dev_role, tenant | | -v | verbose | Log with debugging on. | -| -j | journal | Create journal entries in Netbox when a host gets added, modified or deleted in Zabbix | -| -p | proxy-power | Force a full proxy sync (includes deleting the proxy in Zabbix if not present in config context in Netbox) | -#### Hostgroup -In case of omitting the -H flag, manual hostgroup creation is required for devices in a new category. - -The format can be set with the -l flag. If not provided the default format will be: -{Site name}/{Manufacturer name}/{Device role name} - -Make sure that the Zabbix user has proper permissions to create hosts. -The hostgroups are in a nested format. This means that proper permissions only need to be applied to the site name hostgroup and cascaded to any child hostgroups. - -#### layout -The default hostgroup layout is "site/manufacturer/device_role". - -**Variables** - -You can change this behaviour with the --layout flag. The following variables can be used: -| name | description | -| ------------ | ------------ | -|tenant|Tenant name| -|site|Site name| -|manufacturer|Manufacturer name| -|device_role|The device role name| - -You can specify the variables like so, sperated by a "/": -``` -python3 netbox_zabbix_sync.py -l tenant/site/device_role -``` -**custom fields** - -You can also use the value of custom fields under the device object. - -This allows more freedom and even allows a ful static mapping instead of a dynamic rendered hostgroup name. -``` -python3 netbox_zabbix_sync.py -l site/mycustomfieldname -``` -**Empty variables or hostgroups** - -Should the content of a variable be empty, then the hostgroup position is skipped. - -For example, consider the following scenario with 2 devices, both the same device type and site. One of them is linked to a tenant, the other one does not have a relationship with a tenant. -- Device_role: PDU -- Site: HQ-AMS -``` -python3 netbox_zabbix_sync.py -l site/tenant/device_role -``` -When running the script like above, the following hostgroup (HG) will be generated for both hosts: - - Device A with no relationship with a tenant: HQ-AMS/PDU - - Device B with a relationship to tenant "Fork Industries": HQ-AMS/Fork Industries/PDU - -The same logic applies to custom fields being used in the HG format: -``` -python3 netbox_zabbix_sync.py -l site/mycustomfieldname -``` -For device A with the value "ABC123" in the custom field "mycustomfieldname" -> HQ-AMS/ABC123 -For a device which does not have a value in the custom field "mycustomfieldname" -> HQ-AMS - -Should there be a scenario where a custom field does not have a value under a device, and the HG format only uses this signle variable, then this will result in an error: -``` -python3 netbox_zabbix_sync.py -l mycustomfieldname - -Netbox-Zabbix-sync - ERROR - ESXI1 has no reliable hostgroup. This is most likely due to the use of custom fields that are empty. -``` -### Device status -By setting a status on a Netbox device you determine how the host is added (or updated) in Zabbix. There are, by default, 3 options: -* Delete the host from Zabbix (triggered by Netbox status "Decommissioning" and "Inventory") -* Create the host in Zabbix but with a disabled status (Trigger by "Offline", "Planned", "Staged" and "Failed") -* Create the host in Zabbix with an enabled status (For now only enabled with the "Active" status) - -You can modify this behaviour by changing the following list variables in the script: - - zabbix_device_removal - - zabbix_device_disable +## Config context ### Zabbix proxy You can set the proxy for a device using the 'proxy' key in config context. @@ -193,7 +197,7 @@ You can set the proxy for a device using the 'proxy' key in config context. } } ``` -Because of the posible amount of destruction when setting up Netbox but forgetting the proxy command, the sync works a bit different. By default everything is synced except in a situation where the Zabbix host has a proxy configured but nothing is configured in Netbox. To force deletion and a full sync, use the -p flag. +Because of the posible amount of destruction when setting up Netbox but forgetting the proxy command, the sync works a bit different. By default everything is synced except in a situation where the Zabbix host has a proxy configured but nothing is configured in Netbox. To force deletion and a full sync, set the full_proxy_sync variable in the config file. ### Set interface parameters within Netbox When adding a new device, you can set the interface type with custom context. By default, the following configuration is applied when no config context is provided: @@ -256,4 +260,7 @@ To configure the interface parameters you'll need to use custom context. Custom } } ``` + +I would recommend using macros for sensitive data such as community strings since the data in Netbox is plain-text. + Note: Not all SNMP data is required for a working configuration. [The following parameters are allowed ](https://www.zabbix.com/documentation/current/manual/api/reference/hostinterface/object#details_tag "The following parameters are allowed ")but are not all required, depending on your environment. \ No newline at end of file diff --git a/config.py.example b/config.py.example index ccf8a3a..63f3d6f 100644 --- a/config.py.example +++ b/config.py.example @@ -11,10 +11,29 @@ templates_config_context_overrule = False template_cf = "zabbix_template" device_cf = "zabbix_hostid" +# Enable clustering of devices with virtual chassis setup +clustering = False + +# Enable hostgroup generation. Requires permissions in Zabbix +create_hostgroups = True + +# Create journal entries +create_journal = False + +# Set to true to enable removal of proxy's under hosts. Use with caution and make sure that you specified +# all the required proxy's in the device config context before enabeling this option. +# With this option disabled proxy's will only be added and modified for Zabbix hosts. +full_proxy_sync = False + # Netbox to Zabbix device state convertion zabbix_device_removal = ["Decommissioning", "Inventory"] zabbix_device_disable = ["Offline", "Planned", "Staged", "Failed"] +# Hostgroup mapping +# Available choices: dev_location, dev_role, manufacturer, region, site, site_group, tenant, tenant_group +# You can also use CF (custom field) names under the device. The CF content will be used for the hostgroup generation. +hostgroup_format = "site/manufacturer/dev_role" + # Custom filter for device filtering. Variable must be present but can be left empty with no filtering. # A couple of examples are as follows: diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index 17e9cfa..4230dbf 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -7,7 +7,18 @@ import argparse from pynetbox import api from pyzabbix import ZabbixAPI, ZabbixAPIException try: - from config import * + from config import ( + templates_config_context, + templates_config_context_overrule, + clustering, create_hostgroups, + create_journal, full_proxy_sync, + template_cf, device_cf, + zabbix_device_removal, + zabbix_device_disable, + hostgroup_format, + nb_device_filter + ) + except ModuleNotFoundError: print(f"Configuration file config.py not found in main directory." "Please create the file or rename the config.py.example file to config.py.") @@ -52,20 +63,19 @@ def main(arguments): # Set Netbox API netbox = api(netbox_host, token=netbox_token, threading=True) # Check if the provided Hostgroup layout is valid - if(arguments.layout): - hg_objects = arguments.layout.split("/") - allowed_objects = ["dev_location", "dev_role", "manufacturer", "region", - "site", "site_group", "tenant", "tenant_group"] - # Create API call to get all custom fields which are on the device objects - device_cfs = netbox.extras.custom_fields.filter(type="text", content_type_id=23) - for cf in device_cfs: - allowed_objects.append(cf.name) - for object in hg_objects: - if(object not in allowed_objects): - e = (f"Hostgroup item {object} is not valid. Make sure you" - " use valid items and seperate them with '/'.") - logger.error(e) - raise HostgroupError(e) + hg_objects = hostgroup_format.split("/") + allowed_objects = ["dev_location", "dev_role", "manufacturer", "region", + "site", "site_group", "tenant", "tenant_group"] + # Create API call to get all custom fields which are on the device objects + device_cfs = netbox.extras.custom_fields.filter(type="text", content_type_id=23) + for cf in device_cfs: + allowed_objects.append(cf.name) + for object in hg_objects: + if(object not in allowed_objects): + e = (f"Hostgroup item {object} is not valid. Make sure you" + " use valid items and seperate them with '/'.") + logger.error(e) + raise HostgroupError(e) # Set Zabbix API try: zabbix = ZabbixAPI(zabbix_host) @@ -83,12 +93,12 @@ def main(arguments): for nb_device in netbox_devices: try: device = NetworkDevice(nb_device, zabbix, netbox_journals, - arguments.journal) - device.set_hostgroup(arguments.layout) + create_journal) + device.set_hostgroup(hostgroup_format) device.set_template(templates_config_context, templates_config_context_overrule) # Checks if device is part of cluster. - # Requires the cluster argument. - if(device.isCluster() and arguments.cluster): + # Requires clustering variable + if(device.isCluster() and clustering): # Check if device is master or slave if(device.promoteMasterDevice()): e = (f"Device {device.name} is " @@ -117,9 +127,9 @@ def main(arguments): continue elif(device.status in zabbix_device_disable): device.zabbix_state = 1 - # Add hostgroup is flag is true + # Add hostgroup is variable is True # and Hostgroup is not present in Zabbix - if(arguments.hostgroups): + if(create_hostgroups): for group in zabbix_groups: # If hostgroup is already present in Zabbix if(group["name"] == device.hostgroup): @@ -131,7 +141,7 @@ def main(arguments): # Device is already present in Zabbix if(device.zabbix_id): device.ConsistencyCheck(zabbix_groups, zabbix_templates, - zabbix_proxys, arguments.proxy_power) + zabbix_proxys, full_proxy_sync) # Add device to Zabbix else: device.createInZabbix(zabbix_groups, zabbix_templates, @@ -612,7 +622,7 @@ class NetworkDevice(): else: if(not host["proxy_hostid"] == "0"): if(proxy_power): - # If the -p flag has been issued, + # Variable full_proxy_sync has been enabled # delete the proxy link in Zabbix self.updateZabbixHost(proxy_hostid=self.zbxproxy) else: @@ -820,29 +830,10 @@ class ZabbixInterface(): if(__name__ == "__main__"): - # Arguments parsing parser = argparse.ArgumentParser( description='A script to sync Zabbix with Netbox device data.' ) parser.add_argument("-v", "--verbose", help="Turn on debugging.", action="store_true") - parser.add_argument("-c", "--cluster", action="store_true", - help=("Only add the primary node of a cluster " - "to Zabbix. Usefull when a shared virtual IP is " - "used for the control plane.")) - parser.add_argument("-H", "--hostgroups", - help="Create Zabbix hostgroups if not present", - action="store_true") - parser.add_argument("-l", "--layout", type=str, - help="Defines the hostgroup layout", - default='site/manufacturer/dev_role') - parser.add_argument("-p", "--proxy_power", action="store_true", - help=("USE WITH CAUTION. If there is a proxy " - "configured in Zabbix but not in Netbox, sync " - "the device and remove the host - proxy " - "link in Zabbix.")) - parser.add_argument("-j", "--journal", action="store_true", - help="Create journal entries in Netbox at write actions") args = parser.parse_args() - main(args)