Merge pull request #148 from TheNetworkGuy/develop

Merge latest development code base to main

.gitignore

@@ -1,5 +1,6 @@
 *.log
 .venv
+.env
 config.py
 Pipfile
 Pipfile.lock

README.md

@@ -289,6 +289,27 @@ hostgroup_format = "mycustomfieldname"
 NetBox-Zabbix-sync - ERROR - ESXI1 has no reliable hostgroup. This is most likely due to the use of custom fields that are empty.
 ```
 
+### Extended site properties
+
+By default, NetBox will only return the following properties under the 'site' key for a device:
+
+- site id
+- (api) url
+- display name
+- name
+- slug
+- description
+
+However, NetBox-Zabbix-Sync allows you to extend these site properties with the full site information
+so you can use this data in inventory fields, tags and usermacros.
+
+To enable this functionality, enable the following setting in your configuration file:
+
+`extended_site_properties = True`
+
+Keep in mind that enabling this option will increase the number of API calls to your NetBox instance, 
+this might impact performance on large syncs.
+
 ### Device status
 
 By setting a status on a NetBox device you determine how the host is added (or
@@ -393,9 +414,9 @@ Tags can be synced from the following sources:
 Syncing tags will override any tags that were set manually on the host,
 making NetBox the single source-of-truth for managing tags.
 
-To enable syncing, turn on tag_sync in the config file.
+To enable syncing, turn on `tag_sync` in the config file.
 By default, this script will modify tag names and tag values to lowercase.
-You can change this behaviour by setting tag_lower to False.
+You can change this behavior by setting `tag_lower` to `False`.
 
 ```python
 tag_sync = True
@@ -408,7 +429,8 @@ As NetBox doesn't follow the tag/value pattern for tags, we will need a tag
 name set to register the netbox tags.
 
 By default the tag name is "NetBox", but you can change this to whatever you want.
-The value for the tag can be set to 'name', 'display', or 'slug', which refers to the property of the NetBox tag object that will be used as the value in Zabbix.
+The value for the tag can be set to 'name', 'display', or 'slug', which refers to the
+property of the NetBox tag object that will be used as the value in Zabbix.
 
 ```python
 tag_name = 'NetBox'
@@ -491,7 +513,7 @@ Through this method, it is possible to define the following types of usermacros:
 2. Secret
 3. Vault
 
-The default macro type is text if no `type` and `value` have been set.
+The default macro type is text, if no `type` and `value` have been set.
 It is also possible to create usermacros with
 [context](https://www.zabbix.com/documentation/7.0/en/manual/config/macros/user_macros_context).
 
@@ -611,7 +633,8 @@ python3 netbox_zabbix_sync.py
 
 ### Zabbix proxy
 
-You can set the proxy for a device using the 'proxy' key in config context.
+#### Config Context
+You can set the proxy for a device using the `proxy` key in config context.
 
 ```json
 {
@@ -652,6 +675,34 @@ In the example above the host will use the group on Zabbix 7. On Zabbix 6 and
 below the host will use the proxy. Zabbix 7 will use the proxy value when
 omitting the proxy_group value.
 
+#### Custom Field
+
+Alternatively, you can use a custom field for assigning a device or VM to 
+a Zabbix proxy or proxy group. The custom fields can be assigned to both 
+Devices and VMs. 
+
+You can also assign these custom fields to a site to allow all devices/VMs
+in that site to be configured with the same proxy or proxy group.
+In order for this to work, `extended_site_properties` needs to be enabled in
+the configuration as well.
+
+To use the custom fields for proxy configuration, configure one or both 
+of the following settings in the configuration file with the actual names of your 
+custom fields:
+
+```python
+proxy_cf = "zabbix_proxy"
+proxy_group_cf = "zabbix_proxy_group"
+```
+
+As with config context proxy configuration, proxy group will take precedence over 
+standalone proxy when configured. 
+Proxy settings configured on the device or VM will in their turn take precedence 
+over any site configuration.
+
+If the custom fields have no value but the proxy or proxy group is configured in config context,
+that setting will be used.
+
 ### Set interface parameters within NetBox
 
 When adding a new device, you can set the interface type with custom context. By

config.py.example

@@ -53,6 +53,12 @@ hostgroup_format = "site/manufacturer/role"
 traverse_regions = False
 traverse_site_groups = False
 
+## Extended site properties
+# By default, NetBox will only return basic site info for any device or VM.
+# By setting `extended_site_properties` to True, the script will query NetBox for additional site info.
+# Be aware that this will increase the number of API queries to NetBox.
+extended_site_properties = False
+
 ## Filtering
 # Custom device filter, variable must be present but can be left empty with no filtering.
 # A couple of examples:
@@ -96,6 +102,8 @@ device_inventory_map = { "asset_tag": "asset_tag",
                          "device_type/model": "type",
                          "device_type/manufacturer/name": "vendor",
                          "oob_ip/address": "oob_ip" }
+# Replace latitude and longitude with site/latitude and and site/longitude to use
+# site geo data. Enable extended_site_properties for this to work!
 
 # We also support inventory mapping on Virtual Machines.
 vm_inventory_map = { "status/label": "deployment_status",
@@ -112,19 +120,19 @@ usermacro_sync = False
 # device usermacro_map to map NetBox fields to usermacros.
 device_usermacro_map = {"serial": "{$HW_SERIAL}",
                         "role/name": "{$DEV_ROLE}",
-                        "url": "{$NB_URL}",
+                        "display_url": "{$NB_URL}",
                         "id": "{$NB_ID}"}
 
 # virtual machine usermacro_map to map NetBox fields to usermacros.
 vm_usermacro_map = {"memory": "{$TOTAL_MEMORY}",
                     "role/name": "{$DEV_ROLE}",
-                    "url": "{$NB_URL}",
+                    "display_url": "{$NB_URL}",
                     "id": "{$NB_ID}"}
 
 # To sync host tags to Zabbix, set to True.
 tag_sync = False
 
-# Setting tag_lower to True will lower capital letters ain tag names and values
+# Setting tag_lower to True will lower capital letters in tag names and values
 # This is more inline with the Zabbix way of working with tags.
 #
 # You can however set this to False to ensure capital letters are synced to Zabbix tags.
@@ -133,8 +141,6 @@ tag_lower = True
 # We can sync NetBox device/VM tags to Zabbix, but as NetBox tags don't follow the key/value
 # pattern, we need to specify a tag name to register the NetBox tags in Zabbix.
 # 
-#
-# 
 # If tag_name is set to False, we won't sync NetBox device/VM tags to Zabbix.
 tag_name = 'NetBox'
 

modules/config.py

@@ -1,6 +1,7 @@
 """
 Module for parsing configuration from the top level config.py file
 """
+
 from pathlib import Path
 from importlib import util
 from os import environ, path
@@ -16,6 +17,8 @@ DEFAULT_CONFIG = {
     "templates_config_context_overrule": False,
     "template_cf": "zabbix_template",
     "device_cf": "zabbix_hostid",
+    "proxy_cf": False,
+    "proxy_group_cf": False,
     "clustering": False,
     "create_hostgroups": True,
     "create_journal": False,
@@ -31,6 +34,7 @@ DEFAULT_CONFIG = {
     "nb_vm_filter": {"name__n": "null"},
     "inventory_mode": "disabled",
     "inventory_sync": False,
+    "extended_site_properties": False,
     "device_inventory_map": {
         "asset_tag": "asset_tag",
         "virtual_chassis/name": "chassis",
@@ -44,40 +48,40 @@ DEFAULT_CONFIG = {
         "serial": "serialno_a",
         "device_type/model": "type",
         "device_type/manufacturer/name": "vendor",
-        "oob_ip/address": "oob_ip"
+        "oob_ip/address": "oob_ip",
     },
     "vm_inventory_map": {
         "status/label": "deployment_status",
         "comments": "notes",
-        "name": "name"
+        "name": "name",
     },
     "usermacro_sync": False,
     "device_usermacro_map": {
         "serial": "{$HW_SERIAL}",
         "role/name": "{$DEV_ROLE}",
         "url": "{$NB_URL}",
-        "id": "{$NB_ID}"
+        "id": "{$NB_ID}",
     },
     "vm_usermacro_map": {
         "memory": "{$TOTAL_MEMORY}",
         "role/name": "{$DEV_ROLE}",
         "url": "{$NB_URL}",
-        "id": "{$NB_ID}"
+        "id": "{$NB_ID}",
     },
     "tag_sync": False,
     "tag_lower": True,
-    "tag_name": 'NetBox',
+    "tag_name": "NetBox",
     "tag_value": "name",
     "device_tag_map": {
         "site/name": "site",
         "rack/name": "rack",
-        "platform/name": "target"
+        "platform/name": "target",
     },
     "vm_tag_map": {
         "site/name": "site",
         "cluster/name": "cluster",
-        "platform/name": "target"
-    }
+        "platform/name": "target",
+    },
 }
 
 

modules/device.py

@@ -5,12 +5,14 @@ Device specific handeling for NetBox to Zabbix
 
 from copy import deepcopy
 from logging import getLogger
-from re import search
 from operator import itemgetter
+from re import search
+from typing import Any
 
-from zabbix_utils import APIRequestError
 from pynetbox import RequestError as NetboxRequestError
+from zabbix_utils import APIRequestError
 
+from modules.config import load_config
 from modules.exceptions import (
     InterfaceConfigError,
     SyncExternalError,
@@ -20,9 +22,13 @@ from modules.exceptions import (
 from modules.hostgroups import Hostgroup
 from modules.interface import ZabbixInterface
 from modules.tags import ZabbixTags
-from modules.tools import field_mapper, remove_duplicates, sanatize_log_output
+from modules.tools import (
+    cf_to_string,
+    field_mapper,
+    remove_duplicates,
+    sanatize_log_output,
+)
 from modules.usermacros import ZabbixUsermacros
-from modules.config import load_config
 
 config = load_config()
 
@@ -454,33 +460,54 @@ class PhysicalDevice:
         self.tags = tags.generate()
         return True
 
-    def setProxy(self, proxy_list):
+    def _setProxy(self, proxy_list: list[dict[str, Any]]) -> bool:
         """
         Sets proxy or proxy group if this
         value has been defined in config context
+        or custom fields.
 
         input: List of all proxies and proxy groups in standardized format
         """
-        # check if the key Zabbix is defined in the config context
-        if "zabbix" not in self.nb.config_context:
-            return False
-        if (
-            "proxy" in self.nb.config_context["zabbix"]
-            and not self.nb.config_context["zabbix"]["proxy"]
-        ):
-            return False
         # Proxy group takes priority over a proxy due
         # to it being HA and therefore being more reliable
         # Includes proxy group fix since Zabbix <= 6 should ignore this
         proxy_types = ["proxy"]
-        if str(self.zabbix.version).startswith("7"):
+        proxy_name = None
+
+        if self.zabbix.version >= 7.0:
             # Only insert groups in front of list for Zabbix7
             proxy_types.insert(0, "proxy_group")
+
+        # loop through supported proxy-types
         for proxy_type in proxy_types:
-            # Check if the key exists in NetBox CC
-            if proxy_type in self.nb.config_context["zabbix"]:
+            # Check if we should use custom fields for proxy config
+            field_config = "proxy_cf" if proxy_type == "proxy" else "proxy_group_cf"
+            if config[field_config]:
+                if (
+                    config[field_config] in self.nb.custom_fields
+                    and self.nb.custom_fields[config[field_config]]
+                ):
+                    proxy_name = cf_to_string(
+                        self.nb.custom_fields[config[field_config]]
+                    )
+                elif (
+                    config[field_config] in self.nb.site.custom_fields
+                    and self.nb.site.custom_fields[config[field_config]]
+                ):
+                    proxy_name = cf_to_string(
+                        self.nb.site.custom_fields[config[field_config]]
+                    )
+
+            # Otherwise check if the proxy is configured in NetBox CC
+            if (
+                not proxy_name
+                and "zabbix" in self.nb.config_context
+                and proxy_type in self.nb.config_context["zabbix"]
+            ):
                 proxy_name = self.nb.config_context["zabbix"][proxy_type]
-                # go through all proxies
+
+            # If a proxy name was found, loop through all proxies to find a match
+            if proxy_name:
                 for proxy in proxy_list:
                     # If the proxy does not match the type, ignore and continue
                     if not proxy["type"] == proxy_type:
@@ -492,6 +519,7 @@ class PhysicalDevice:
                         )
                         self.zbxproxy = proxy
                         return True
+
                 self.logger.warning(
                     "Host %s: unable to find proxy %s", self.name, proxy_name
                 )
@@ -524,7 +552,7 @@ class PhysicalDevice:
             # Set interface, group and template configuration
             interfaces = self.setInterfaceDetails()
             # Set Zabbix proxy if defined
-            self.setProxy(proxies)
+            self._setProxy(proxies)
             # Set basic data for host creation
             create_data = {
                 "host": self.name,
@@ -656,7 +684,7 @@ class PhysicalDevice:
 
         # Prepare templates and proxy config
         self.zbxTemplatePrepper(templates)
-        self.setProxy(proxies)
+        self._setProxy(proxies)
         # Get host object from Zabbix
         host = self.zabbix.host.get(
             filter={"hostid": self.zabbix_id},
@@ -792,7 +820,7 @@ class PhysicalDevice:
                 # Display error message
                 self.logger.warning(
                     "Host %s: Is configured with proxy in Zabbix but not in NetBox."
-                    "The -p flag was ommited: no changes have been made.",
+                    "full_proxy_sync is not set: no changes have been made.",
                     self.name,
                 )
             if not proxy_set:

modules/hostgroups.py

@@ -3,7 +3,7 @@
 from logging import getLogger
 
 from modules.exceptions import HostgroupError
-from modules.tools import build_path
+from modules.tools import build_path, cf_to_string
 
 
 class Hostgroup:
@@ -118,8 +118,8 @@ class Hostgroup:
             # Check if requested data is available as option for this host
             if hg_item not in self.format_options:
                 if hg_item.startswith(("'", '"')) and hg_item.endswith(("'", '"')):
-                    hg_item = hg_item.strip("\'")
-                    hg_item = hg_item.strip('\"')
+                    hg_item = hg_item.strip("'")
+                    hg_item = hg_item.strip('"')
                     hg_output.append(hg_item)
                 else:
                     # Check if a custom field exists with this name
@@ -134,7 +134,7 @@ class Hostgroup:
                         raise HostgroupError(msg)
                     # CF data is populated
                     if cf_data["cf"]:
-                        hg_output.append(cf_data["cf"])
+                        hg_output.append(cf_to_string(cf_data["cf"]))
                 continue
             # Check if there is a value associated to the variable.
             # For instance, if a device has no location, do not use it with hostgroup calculation

modules/tools.py

@@ -50,6 +50,20 @@ def proxy_prepper(proxy_list, proxy_group_list):
     return output
 
 
+def cf_to_string(cf, key="name", logger=None):
+    """
+    Converts a dict custom fields to string
+    """
+    if isinstance(cf, dict):
+        if key in cf:
+            return cf[key]
+        logger.error(
+            "Conversion of custom field failed, '%s' not found in cf dict.", key
+        )
+        return None
+    return cf
+
+
 def field_mapper(host, mapper, nbdevice, logger):
     """
     Maps NetBox field data to Zabbix properties.
@@ -160,7 +174,7 @@ def verify_hg_format(
         if (
             hg_object not in allowed_objects[hg_type]
             and hg_object not in allowed_objects["cfs"][hg_type]
-            and not hg_object.startswith(('"',"'"))
+            and not hg_object.startswith(('"', "'"))
         ):
             e = (
                 f"Hostgroup item {hg_object} is not valid. Make sure you"

netbox_zabbix_sync.py

@@ -12,6 +12,7 @@ from pynetbox import api
 from pynetbox.core.query import RequestError as NBRequestError
 from requests.exceptions import ConnectionError as RequestsConnectionError
 from zabbix_utils import APIRequestError, ProcessingError, ZabbixAPI
+
 from modules.config import load_config
 from modules.device import PhysicalDevice
 from modules.exceptions import EnvironmentVarError, SyncError
@@ -82,7 +83,9 @@ def main(arguments):
     device_cfs = []
     vm_cfs = []
     device_cfs = list(
-        netbox.extras.custom_fields.filter(type="text", content_types="dcim.device")
+        netbox.extras.custom_fields.filter(
+            type=["text", "object", "select"], content_types="dcim.device"
+        )
     )
     verify_hg_format(
         config["hostgroup_format"], device_cfs=device_cfs, hg_type="dev", logger=logger
@@ -90,7 +93,8 @@ def main(arguments):
     if config["sync_vms"]:
         vm_cfs = list(
             netbox.extras.custom_fields.filter(
-                type="text", content_types="virtualization.virtualmachine"
+                type=["text", "object", "select"],
+                content_types="virtualization.virtualmachine",
             )
         )
         verify_hg_format(
@@ -166,6 +170,9 @@ def main(arguments):
             # Check if a valid hostgroup has been found for this VM.
             if not vm.hostgroups:
                 continue
+            if config["extended_site_properties"] and nb_vm.site:
+                logger.debug("VM %s: extending site information.", vm.name)
+                vm.site = convert_recordset(netbox.dcim.sites.filter(id=nb_vm.site.id))
             vm.set_inventory(nb_vm)
             vm.set_usermacros()
             vm.set_tags()
@@ -238,6 +245,11 @@ def main(arguments):
                     device.name,
                 )
                 continue
+            if config["extended_site_properties"] and nb_device.site:
+                logger.debug("Device %s: extending site information.", device.name)
+                device.site = convert_recordset(
+                    netbox.dcim.sites.filter(id=nb_device.site.id)
+                )
             device.set_inventory(nb_device)
             device.set_usermacros()
             device.set_tags()