diff --git a/.gitignore b/.gitignore index c515fe3..0cf4953 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.log .venv +.env config.py Pipfile Pipfile.lock diff --git a/README.md b/README.md index e6cabfa..4c762ed 100644 --- a/README.md +++ b/README.md @@ -289,6 +289,27 @@ hostgroup_format = "mycustomfieldname" NetBox-Zabbix-sync - ERROR - ESXI1 has no reliable hostgroup. This is most likely due to the use of custom fields that are empty. ``` +### Extended site properties + +By default, NetBox will only return the following properties under the 'site' key for a device: + +- site id +- (api) url +- display name +- name +- slug +- description + +However, NetBox-Zabbix-Sync allows you to extend these site properties with the full site information +so you can use this data in inventory fields, tags and usermacros. + +To enable this functionality, enable the following setting in your configuration file: + +`extended_site_properties = True` + +Keep in mind that enabling this option will increase the number of API calls to your NetBox instance, +this might impact performance on large syncs. + ### Device status By setting a status on a NetBox device you determine how the host is added (or @@ -393,9 +414,9 @@ Tags can be synced from the following sources: Syncing tags will override any tags that were set manually on the host, making NetBox the single source-of-truth for managing tags. -To enable syncing, turn on tag_sync in the config file. +To enable syncing, turn on `tag_sync` in the config file. By default, this script will modify tag names and tag values to lowercase. -You can change this behaviour by setting tag_lower to False. +You can change this behavior by setting `tag_lower` to `False`. ```python tag_sync = True @@ -408,7 +429,8 @@ As NetBox doesn't follow the tag/value pattern for tags, we will need a tag name set to register the netbox tags. By default the tag name is "NetBox", but you can change this to whatever you want. -The value for the tag can be set to 'name', 'display', or 'slug', which refers to the property of the NetBox tag object that will be used as the value in Zabbix. +The value for the tag can be set to 'name', 'display', or 'slug', which refers to the +property of the NetBox tag object that will be used as the value in Zabbix. ```python tag_name = 'NetBox' @@ -491,7 +513,7 @@ Through this method, it is possible to define the following types of usermacros: 2. Secret 3. Vault -The default macro type is text if no `type` and `value` have been set. +The default macro type is text, if no `type` and `value` have been set. It is also possible to create usermacros with [context](https://www.zabbix.com/documentation/7.0/en/manual/config/macros/user_macros_context). @@ -611,7 +633,8 @@ python3 netbox_zabbix_sync.py ### Zabbix proxy -You can set the proxy for a device using the 'proxy' key in config context. +#### Config Context +You can set the proxy for a device using the `proxy` key in config context. ```json { @@ -652,6 +675,34 @@ In the example above the host will use the group on Zabbix 7. On Zabbix 6 and below the host will use the proxy. Zabbix 7 will use the proxy value when omitting the proxy_group value. +#### Custom Field + +Alternatively, you can use a custom field for assigning a device or VM to +a Zabbix proxy or proxy group. The custom fields can be assigned to both +Devices and VMs. + +You can also assign these custom fields to a site to allow all devices/VMs +in that site to be configured with the same proxy or proxy group. +In order for this to work, `extended_site_properties` needs to be enabled in +the configuration as well. + +To use the custom fields for proxy configuration, configure one or both +of the following settings in the configuration file with the actual names of your +custom fields: + +```python +proxy_cf = "zabbix_proxy" +proxy_group_cf = "zabbix_proxy_group" +``` + +As with config context proxy configuration, proxy group will take precedence over +standalone proxy when configured. +Proxy settings configured on the device or VM will in their turn take precedence +over any site configuration. + +If the custom fields have no value but the proxy or proxy group is configured in config context, +that setting will be used. + ### Set interface parameters within NetBox When adding a new device, you can set the interface type with custom context. By diff --git a/config.py.example b/config.py.example index e4082e6..acc1629 100644 --- a/config.py.example +++ b/config.py.example @@ -53,6 +53,12 @@ hostgroup_format = "site/manufacturer/role" traverse_regions = False traverse_site_groups = False +## Extended site properties +# By default, NetBox will only return basic site info for any device or VM. +# By setting `extended_site_properties` to True, the script will query NetBox for additional site info. +# Be aware that this will increase the number of API queries to NetBox. +extended_site_properties = False + ## Filtering # Custom device filter, variable must be present but can be left empty with no filtering. # A couple of examples: @@ -96,6 +102,8 @@ device_inventory_map = { "asset_tag": "asset_tag", "device_type/model": "type", "device_type/manufacturer/name": "vendor", "oob_ip/address": "oob_ip" } +# Replace latitude and longitude with site/latitude and and site/longitude to use +# site geo data. Enable extended_site_properties for this to work! # We also support inventory mapping on Virtual Machines. vm_inventory_map = { "status/label": "deployment_status", @@ -112,19 +120,19 @@ usermacro_sync = False # device usermacro_map to map NetBox fields to usermacros. device_usermacro_map = {"serial": "{$HW_SERIAL}", "role/name": "{$DEV_ROLE}", - "url": "{$NB_URL}", + "display_url": "{$NB_URL}", "id": "{$NB_ID}"} # virtual machine usermacro_map to map NetBox fields to usermacros. vm_usermacro_map = {"memory": "{$TOTAL_MEMORY}", "role/name": "{$DEV_ROLE}", - "url": "{$NB_URL}", + "display_url": "{$NB_URL}", "id": "{$NB_ID}"} # To sync host tags to Zabbix, set to True. tag_sync = False -# Setting tag_lower to True will lower capital letters ain tag names and values +# Setting tag_lower to True will lower capital letters in tag names and values # This is more inline with the Zabbix way of working with tags. # # You can however set this to False to ensure capital letters are synced to Zabbix tags. @@ -132,8 +140,6 @@ tag_lower = True # We can sync NetBox device/VM tags to Zabbix, but as NetBox tags don't follow the key/value # pattern, we need to specify a tag name to register the NetBox tags in Zabbix. -# -# # # If tag_name is set to False, we won't sync NetBox device/VM tags to Zabbix. tag_name = 'NetBox' diff --git a/modules/config.py b/modules/config.py index a105892..3d69d6c 100644 --- a/modules/config.py +++ b/modules/config.py @@ -1,6 +1,7 @@ """ Module for parsing configuration from the top level config.py file """ + from pathlib import Path from importlib import util from os import environ, path @@ -16,6 +17,8 @@ DEFAULT_CONFIG = { "templates_config_context_overrule": False, "template_cf": "zabbix_template", "device_cf": "zabbix_hostid", + "proxy_cf": False, + "proxy_group_cf": False, "clustering": False, "create_hostgroups": True, "create_journal": False, @@ -31,6 +34,7 @@ DEFAULT_CONFIG = { "nb_vm_filter": {"name__n": "null"}, "inventory_mode": "disabled", "inventory_sync": False, + "extended_site_properties": False, "device_inventory_map": { "asset_tag": "asset_tag", "virtual_chassis/name": "chassis", @@ -44,40 +48,40 @@ DEFAULT_CONFIG = { "serial": "serialno_a", "device_type/model": "type", "device_type/manufacturer/name": "vendor", - "oob_ip/address": "oob_ip" + "oob_ip/address": "oob_ip", }, "vm_inventory_map": { "status/label": "deployment_status", "comments": "notes", - "name": "name" + "name": "name", }, "usermacro_sync": False, "device_usermacro_map": { "serial": "{$HW_SERIAL}", "role/name": "{$DEV_ROLE}", "url": "{$NB_URL}", - "id": "{$NB_ID}" + "id": "{$NB_ID}", }, "vm_usermacro_map": { "memory": "{$TOTAL_MEMORY}", "role/name": "{$DEV_ROLE}", "url": "{$NB_URL}", - "id": "{$NB_ID}" + "id": "{$NB_ID}", }, "tag_sync": False, "tag_lower": True, - "tag_name": 'NetBox', + "tag_name": "NetBox", "tag_value": "name", "device_tag_map": { "site/name": "site", "rack/name": "rack", - "platform/name": "target" + "platform/name": "target", }, "vm_tag_map": { "site/name": "site", "cluster/name": "cluster", - "platform/name": "target" - } + "platform/name": "target", + }, } diff --git a/modules/device.py b/modules/device.py index cf0e2a2..cc32390 100644 --- a/modules/device.py +++ b/modules/device.py @@ -5,12 +5,14 @@ Device specific handeling for NetBox to Zabbix from copy import deepcopy from logging import getLogger -from re import search from operator import itemgetter +from re import search +from typing import Any -from zabbix_utils import APIRequestError from pynetbox import RequestError as NetboxRequestError +from zabbix_utils import APIRequestError +from modules.config import load_config from modules.exceptions import ( InterfaceConfigError, SyncExternalError, @@ -20,9 +22,13 @@ from modules.exceptions import ( from modules.hostgroups import Hostgroup from modules.interface import ZabbixInterface from modules.tags import ZabbixTags -from modules.tools import field_mapper, remove_duplicates, sanatize_log_output +from modules.tools import ( + cf_to_string, + field_mapper, + remove_duplicates, + sanatize_log_output, +) from modules.usermacros import ZabbixUsermacros -from modules.config import load_config config = load_config() @@ -454,33 +460,54 @@ class PhysicalDevice: self.tags = tags.generate() return True - def setProxy(self, proxy_list): + def _setProxy(self, proxy_list: list[dict[str, Any]]) -> bool: """ Sets proxy or proxy group if this value has been defined in config context + or custom fields. input: List of all proxies and proxy groups in standardized format """ - # check if the key Zabbix is defined in the config context - if "zabbix" not in self.nb.config_context: - return False - if ( - "proxy" in self.nb.config_context["zabbix"] - and not self.nb.config_context["zabbix"]["proxy"] - ): - return False # Proxy group takes priority over a proxy due # to it being HA and therefore being more reliable # Includes proxy group fix since Zabbix <= 6 should ignore this proxy_types = ["proxy"] - if str(self.zabbix.version).startswith("7"): + proxy_name = None + + if self.zabbix.version >= 7.0: # Only insert groups in front of list for Zabbix7 proxy_types.insert(0, "proxy_group") + + # loop through supported proxy-types for proxy_type in proxy_types: - # Check if the key exists in NetBox CC - if proxy_type in self.nb.config_context["zabbix"]: + # Check if we should use custom fields for proxy config + field_config = "proxy_cf" if proxy_type == "proxy" else "proxy_group_cf" + if config[field_config]: + if ( + config[field_config] in self.nb.custom_fields + and self.nb.custom_fields[config[field_config]] + ): + proxy_name = cf_to_string( + self.nb.custom_fields[config[field_config]] + ) + elif ( + config[field_config] in self.nb.site.custom_fields + and self.nb.site.custom_fields[config[field_config]] + ): + proxy_name = cf_to_string( + self.nb.site.custom_fields[config[field_config]] + ) + + # Otherwise check if the proxy is configured in NetBox CC + if ( + not proxy_name + and "zabbix" in self.nb.config_context + and proxy_type in self.nb.config_context["zabbix"] + ): proxy_name = self.nb.config_context["zabbix"][proxy_type] - # go through all proxies + + # If a proxy name was found, loop through all proxies to find a match + if proxy_name: for proxy in proxy_list: # If the proxy does not match the type, ignore and continue if not proxy["type"] == proxy_type: @@ -492,6 +519,7 @@ class PhysicalDevice: ) self.zbxproxy = proxy return True + self.logger.warning( "Host %s: unable to find proxy %s", self.name, proxy_name ) @@ -524,7 +552,7 @@ class PhysicalDevice: # Set interface, group and template configuration interfaces = self.setInterfaceDetails() # Set Zabbix proxy if defined - self.setProxy(proxies) + self._setProxy(proxies) # Set basic data for host creation create_data = { "host": self.name, @@ -656,7 +684,7 @@ class PhysicalDevice: # Prepare templates and proxy config self.zbxTemplatePrepper(templates) - self.setProxy(proxies) + self._setProxy(proxies) # Get host object from Zabbix host = self.zabbix.host.get( filter={"hostid": self.zabbix_id}, @@ -792,7 +820,7 @@ class PhysicalDevice: # Display error message self.logger.warning( "Host %s: Is configured with proxy in Zabbix but not in NetBox." - "The -p flag was ommited: no changes have been made.", + "full_proxy_sync is not set: no changes have been made.", self.name, ) if not proxy_set: diff --git a/modules/hostgroups.py b/modules/hostgroups.py index 5916c0a..49890e6 100644 --- a/modules/hostgroups.py +++ b/modules/hostgroups.py @@ -3,7 +3,7 @@ from logging import getLogger from modules.exceptions import HostgroupError -from modules.tools import build_path +from modules.tools import build_path, cf_to_string class Hostgroup: @@ -118,8 +118,8 @@ class Hostgroup: # Check if requested data is available as option for this host if hg_item not in self.format_options: if hg_item.startswith(("'", '"')) and hg_item.endswith(("'", '"')): - hg_item = hg_item.strip("\'") - hg_item = hg_item.strip('\"') + hg_item = hg_item.strip("'") + hg_item = hg_item.strip('"') hg_output.append(hg_item) else: # Check if a custom field exists with this name @@ -134,7 +134,7 @@ class Hostgroup: raise HostgroupError(msg) # CF data is populated if cf_data["cf"]: - hg_output.append(cf_data["cf"]) + hg_output.append(cf_to_string(cf_data["cf"])) continue # Check if there is a value associated to the variable. # For instance, if a device has no location, do not use it with hostgroup calculation diff --git a/modules/tools.py b/modules/tools.py index dacab20..db6a1a6 100644 --- a/modules/tools.py +++ b/modules/tools.py @@ -50,6 +50,20 @@ def proxy_prepper(proxy_list, proxy_group_list): return output +def cf_to_string(cf, key="name", logger=None): + """ + Converts a dict custom fields to string + """ + if isinstance(cf, dict): + if key in cf: + return cf[key] + logger.error( + "Conversion of custom field failed, '%s' not found in cf dict.", key + ) + return None + return cf + + def field_mapper(host, mapper, nbdevice, logger): """ Maps NetBox field data to Zabbix properties. @@ -160,7 +174,7 @@ def verify_hg_format( if ( hg_object not in allowed_objects[hg_type] and hg_object not in allowed_objects["cfs"][hg_type] - and not hg_object.startswith(('"',"'")) + and not hg_object.startswith(('"', "'")) ): e = ( f"Hostgroup item {hg_object} is not valid. Make sure you" diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index 79fa27e..8791e47 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -12,6 +12,7 @@ from pynetbox import api from pynetbox.core.query import RequestError as NBRequestError from requests.exceptions import ConnectionError as RequestsConnectionError from zabbix_utils import APIRequestError, ProcessingError, ZabbixAPI + from modules.config import load_config from modules.device import PhysicalDevice from modules.exceptions import EnvironmentVarError, SyncError @@ -82,7 +83,9 @@ def main(arguments): device_cfs = [] vm_cfs = [] device_cfs = list( - netbox.extras.custom_fields.filter(type="text", content_types="dcim.device") + netbox.extras.custom_fields.filter( + type=["text", "object", "select"], content_types="dcim.device" + ) ) verify_hg_format( config["hostgroup_format"], device_cfs=device_cfs, hg_type="dev", logger=logger @@ -90,7 +93,8 @@ def main(arguments): if config["sync_vms"]: vm_cfs = list( netbox.extras.custom_fields.filter( - type="text", content_types="virtualization.virtualmachine" + type=["text", "object", "select"], + content_types="virtualization.virtualmachine", ) ) verify_hg_format( @@ -166,6 +170,9 @@ def main(arguments): # Check if a valid hostgroup has been found for this VM. if not vm.hostgroups: continue + if config["extended_site_properties"] and nb_vm.site: + logger.debug("VM %s: extending site information.", vm.name) + vm.site = convert_recordset(netbox.dcim.sites.filter(id=nb_vm.site.id)) vm.set_inventory(nb_vm) vm.set_usermacros() vm.set_tags() @@ -238,6 +245,11 @@ def main(arguments): device.name, ) continue + if config["extended_site_properties"] and nb_device.site: + logger.debug("Device %s: extending site information.", device.name) + device.site = convert_recordset( + netbox.dcim.sites.filter(id=nb_device.site.id) + ) device.set_inventory(nb_device) device.set_usermacros() device.set_tags()