Merge pull request #124 from retigra/additional-hostgroup-support

✨ Additional hostgroup support

Dockerfile

@@ -2,9 +2,16 @@
 FROM python:3.12-alpine
 RUN mkdir -p /opt/netbox-zabbix && chown -R 1000:1000 /opt/netbox-zabbix
 
-USER 1000:1000
+RUN mkdir -p /opt/netbox-zabbix
-COPY --chown=1000:1000 . /opt/netbox-zabbix
+RUN addgroup -g 1000 -S netbox-zabbix && adduser -u 1000 -S netbox-zabbix -G netbox-zabbix
+RUN chown -R 1000:1000 /opt/netbox-zabbix
+
 WORKDIR /opt/netbox-zabbix
+
+COPY --chown=1000:1000 . /opt/netbox-zabbix
+
+USER 1000:1000
+
 RUN if ! [ -f ./config.py ]; then cp ./config.py.example ./config.py; fi
 USER root
 RUN pip install -r ./requirements.txt

README.md

@@ -181,13 +181,14 @@ used:
 | ------------ | ------------------------ |
 | location     | The device location name |
 | manufacturer | Device manufacturer name |
+| rack         | Rack                     |
 
 **Only for VMs**
 
 | name         | description     |
 | ------------ | --------------- |
 | cluster      | VM cluster name |
-| cluster_type | VM cluster type |
+| device       | parent device   |
 
 You can specify the value separated by a "/" like so:
 
@@ -195,6 +196,13 @@ You can specify the value separated by a "/" like so:
 hostgroup_format = "tenant/site/location/role"
 ```
 
+You can also provice a list of groups like so:
+
+```python
+hostgroup_format = ["region/site_group/site", "role", "tenant_group/tenant"]
+``` 
+
+
 **Group traversal**
 
 The default behaviour for `region` is to only use the directly assigned region

modules/device.py

@@ -6,6 +6,7 @@ Device specific handeling for NetBox to Zabbix
 from copy import deepcopy
 from logging import getLogger
 from re import search
+from operator import itemgetter
 
 from zabbix_utils import APIRequestError
 from pynetbox import RequestError as NetboxRequestError
@@ -42,11 +43,11 @@ class PhysicalDevice:
         self.status = nb.status.label
         self.zabbix = zabbix
         self.zabbix_id = None
-        self.group_id = None
+        self.group_ids = []
         self.nb_api_version = nb_version
         self.zbx_template_names = []
         self.zbx_templates = []
-        self.hostgroup = None
+        self.hostgroups = []
         self.tenant = nb.tenant
         self.config_context = nb.config_context
         self.zbxproxy = None
@@ -130,7 +131,10 @@ class PhysicalDevice:
             nb_regions=nb_regions,
         )
         # Generate hostgroup based on hostgroup format
-        self.hostgroup = hg.generate(hg_format)
+        if isinstance(hg_format, list):
+            self.hostgroups = [hg.generate(f) for f in hg_format]
+        else:
+            self.hostgroups.append(hg.generate(hg_format))
 
     def set_template(self, prefer_config_context, overrule_custom):
         """Set Template"""
@@ -312,12 +316,17 @@ class PhysicalDevice:
         OUTPUT: True / False
         """
         # Go through all groups
-        for group in groups:
+        for hg in self.hostgroups:
-            if group["name"] == self.hostgroup:
+            for group in groups:
-                self.group_id = group["groupid"]
+                if group["name"] == hg:
-                e = f"Host {self.name}: matched group {group['name']}"
+                    self.group_ids.append({"groupid": group["groupid"]})
-                self.logger.debug(e)
+                    e = (
-                return True
+                        f"Host {self.name}: matched group "
+                        f"\"{group['name']}\" (ID:{group['groupid']})"
+                    )
+                    self.logger.debug(e)
+        if len(self.group_ids) == len(self.hostgroups):
+            return True
         return False
 
     def cleanup(self):
@@ -494,7 +503,7 @@ class PhysicalDevice:
                 templateids.append({"templateid": template["templateid"]})
             # Set interface, group and template configuration
             interfaces = self.setInterfaceDetails()
-            groups = [{"groupid": self.group_id}]
+            groups = self.group_ids
             # Set Zabbix proxy if defined
             self.setProxy(proxies)
             # Set basic data for host creation
@@ -547,25 +556,26 @@ class PhysicalDevice:
         """
         final_data = []
         # Check if the hostgroup is in a nested format and check each parent
-        for pos in range(len(self.hostgroup.split("/"))):
+        for hostgroup in self.hostgroups:
-            zabbix_hg = self.hostgroup.rsplit("/", pos)[0]
+            for pos in range(len(hostgroup.split("/"))):
-            if self.lookupZabbixHostgroup(hostgroups, zabbix_hg):
+                zabbix_hg = hostgroup.rsplit("/", pos)[0]
-                # Hostgroup already exists
+                if self.lookupZabbixHostgroup(hostgroups, zabbix_hg):
-                continue
+                    # Hostgroup already exists
-            # Create new group
+                    continue
-            try:
+                # Create new group
-                # API call to Zabbix
+                try:
-                groupid = self.zabbix.hostgroup.create(name=zabbix_hg)
+                    # API call to Zabbix
-                e = f"Hostgroup '{zabbix_hg}': created in Zabbix."
+                    groupid = self.zabbix.hostgroup.create(name=zabbix_hg)
-                self.logger.info(e)
+                    e = f"Hostgroup '{zabbix_hg}': created in Zabbix."
-                # Add group to final data
+                    self.logger.info(e)
-                final_data.append(
+                    # Add group to final data
-                    {"groupid": groupid["groupids"][0], "name": zabbix_hg}
+                    final_data.append(
-                )
+                        {"groupid": groupid["groupids"][0], "name": zabbix_hg}
-            except APIRequestError as e:
+                    )
-                msg = f"Hostgroup '{zabbix_hg}': unable to create. Zabbix returned {str(e)}."
+                except APIRequestError as e:
-                self.logger.error(msg)
+                    msg = f"Hostgroup '{zabbix_hg}': unable to create. Zabbix returned {str(e)}."
-                raise SyncExternalError(msg) from e
+                    self.logger.error(msg)
+                    raise SyncExternalError(msg) from e
         return final_data
 
     def lookupZabbixHostgroup(self, group_list, lookup_group):
@@ -605,7 +615,7 @@ class PhysicalDevice:
         Checks if Zabbix object is still valid with NetBox parameters.
         """
         # If group is found or if the hostgroup is nested
-        if not self.setZabbixGroupID(groups) or len(self.hostgroup.split("/")) > 1:
+        if not self.setZabbixGroupID(groups): # or len(self.hostgroups.split("/")) > 1:
             if create_hostgroups:
                 # Script is allowed to create a new hostgroup
                 new_groups = self.createZabbixHostgroup(groups)
@@ -613,7 +623,7 @@ class PhysicalDevice:
                     # Add all new groups to the list of groups
                     groups.append(group)
             # check if the initial group was not already found (and this is a nested folder check)
-            if not self.group_id:
+            if not self.group_ids:
                 # Function returns true / false but also sets GroupID
                 if not self.setZabbixGroupID(groups) and not create_hostgroups:
                     e = (
@@ -622,6 +632,9 @@ class PhysicalDevice:
                     )
                     self.logger.warning(e)
                     raise SyncInventoryError(e)
+        #if self.group_ids:
+        #   self.group_ids.append(self.pri_group_id)
+
         # Prepare templates and proxy config
         self.zbxTemplatePrepper(templates)
         self.setProxy(proxies)
@@ -660,6 +673,7 @@ class PhysicalDevice:
                 f"Received value: {host['host']}"
             )
             self.updateZabbixHost(host=self.name)
+
         # Execute check depending on wether the name is special or not
         if self.use_visible_name:
             if host["name"] == self.visible_name:
@@ -689,18 +703,20 @@ class PhysicalDevice:
         group_dictname = "hostgroups"
         if str(self.zabbix.version).startswith(("6", "5")):
             group_dictname = "groups"
-        for group in host[group_dictname]:
+        # Check if hostgroups match
-            if group["groupid"] == self.group_id:
+        if (sorted(host[group_dictname], key=itemgetter('groupid')) ==
-                self.logger.debug(f"Host {self.name}: hostgroup in-sync.")
+            sorted(self.group_ids, key=itemgetter('groupid'))):
-                break
+            self.logger.debug(f"Host {self.name}: hostgroups in-sync.")
-            self.logger.warning(f"Host {self.name}: hostgroup OUT of sync.")
+        else:
-            self.updateZabbixHost(groups={"groupid": self.group_id})
+            self.logger.warning(f"Host {self.name}: hostgroups OUT of sync.")
+            self.updateZabbixHost(groups=self.group_ids)
 
         if int(host["status"]) == self.zabbix_state:
             self.logger.debug(f"Host {self.name}: status in-sync.")
         else:
             self.logger.warning(f"Host {self.name}: status OUT of sync.")
             self.updateZabbixHost(status=str(self.zabbix_state))
+
         # Check if a proxy has been defined
         if self.zbxproxy:
             # Check if proxy or proxy group is defined
@@ -870,7 +886,7 @@ class PhysicalDevice:
             e = (
                 f"Host {self.name} has unsupported interface configuration."
                 f" Host has total of {len(host['interfaces'])} interfaces. "
-                "Manual interfention required."
+                "Manual intervention required."
             )
             self.logger.error(e)
             raise SyncInventoryError(e)

modules/hostgroups.py

@@ -91,7 +91,6 @@ class Hostgroup:
             if self.nb.cluster:
                 format_options["cluster"] = self.nb.cluster.name
                 format_options["cluster_type"] = self.nb.cluster.type.name
-
         self.format_options = format_options
 
     def set_nesting(

modules/tools.py

@@ -1,5 +1,5 @@
 """A collection of tools used by several classes"""
-
+from modules.exceptions import HostgroupError
 
 def convert_recordset(recordset):
     """Converts netbox RedcordSet to list of dicts."""
@@ -100,6 +100,61 @@ def remove_duplicates(input_list, sortkey=None):
         output_list.sort(key=lambda x: x[sortkey])
     return output_list
 
+
+def verify_hg_format(hg_format, device_cfs=None, vm_cfs=None, hg_type="dev", logger=None):
+    """
+    Verifies hostgroup field format
+    """
+    if not device_cfs:
+        device_cfs = []
+    if not vm_cfs:
+        vm_cfs = []
+    allowed_objects = {"dev": ["location",
+                              "rack",
+                              "role",
+                              "manufacturer",
+                              "region",
+                              "site",
+                              "site_group",
+                              "tenant",
+                              "tenant_group",
+                              "platform",
+                              "cluster"]
+                      ,"vm": ["location",
+                              "role",
+                              "manufacturer",
+                              "region",
+                              "site",
+                              "site_group",
+                              "tenant",
+                              "tenant_group",
+                              "cluster",
+                              "device",
+                              "platform"]
+                       ,"cfs": {"dev": [], "vm": []}
+                      }
+    for cf in device_cfs:
+        allowed_objects['cfs']['dev'].append(cf.name)
+    for cf in vm_cfs:
+        allowed_objects['cfs']['vm'].append(cf.name)
+    hg_objects = []
+    if isinstance(hg_format,list):
+        for f in hg_format:
+            hg_objects = hg_objects + f.split("/")
+    else:
+        hg_objects = hg_format.split("/")
+    hg_objects = sorted(set(hg_objects))
+    for hg_object in hg_objects:
+        if (hg_object not in allowed_objects[hg_type] and
+            hg_object not in allowed_objects['cfs'][hg_type]):
+            e = (
+                f"Hostgroup item {hg_object} is not valid. Make sure you"
+                " use valid items and separate them with '/'."
+            )
+            logger.error(e)
+            raise HostgroupError(e)
+
+
 def sanatize_log_output(data):
     """
     Used for the update function to Zabbix which

modules/virtual_machine.py

@@ -43,7 +43,10 @@ class VirtualMachine(PhysicalDevice):
             nb_regions=nb_regions,
         )
         # Generate hostgroup based on hostgroup format
-        self.hostgroup = hg.generate(hg_format)
+        if isinstance(hg_format, list):
+            self.hostgroups = [hg.generate(f) for f in hg_format]
+        else:
+            self.hostgroups.append(hg.generate(hg_format))
 
     def set_vm_template(self):
         """Set Template for VMs. Overwrites default class

netbox_zabbix_sync.py

@@ -13,9 +13,9 @@ from requests.exceptions import ConnectionError as RequestsConnectionError
 from zabbix_utils import APIRequestError, ProcessingError, ZabbixAPI
 from modules.config import load_config
 from modules.device import PhysicalDevice
-from modules.exceptions import EnvironmentVarError, HostgroupError, SyncError
+from modules.exceptions import EnvironmentVarError, SyncError
 from modules.logging import get_logger, set_log_levels, setup_logger
-from modules.tools import convert_recordset, proxy_prepper
+from modules.tools import convert_recordset, proxy_prepper, verify_hg_format
 from modules.virtual_machine import VirtualMachine
 
 config = load_config()
@@ -63,23 +63,11 @@ def main(arguments):
     netbox_token = environ.get("NETBOX_TOKEN")
     # Set NetBox API
     netbox = api(netbox_host, token=netbox_token, threading=True)
-    # Check if the provided Hostgroup layout is valid
-    hg_objects = config["hostgroup_format"].split("/")
-    allowed_objects = [
-        "location",
-        "role",
-        "manufacturer",
-        "region",
-        "site",
-        "site_group",
-        "tenant",
-        "tenant_group",
-    ]
     # Create API call to get all custom fields which are on the device objects
     try:
-        device_cfs = list(
+        # Get NetBox version
-            netbox.extras.custom_fields.filter(type="text", content_type_id=23)
+        nb_version = netbox.version
-        )
+        logger.debug(f"NetBox version is {nb_version}.")
     except RequestsConnectionError:
         logger.error(
             f"Unable to connect to NetBox with URL {netbox_host}."
@@ -89,16 +77,20 @@ def main(arguments):
     except NBRequestError as e:
         logger.error(f"NetBox error: {e}")
         sys.exit(1)
-    for cf in device_cfs:
+    # Check if the provided Hostgroup layout is valid
-        allowed_objects.append(cf.name)
+    device_cfs = []
-    for hg_object in hg_objects:
+    vm_cfs = []
-        if hg_object not in allowed_objects:
+    device_cfs = list(
-            e = (
+        netbox.extras.custom_fields.filter(type="text", content_types="dcim.device")
-                f"Hostgroup item {hg_object} is not valid. Make sure you"
+    )
-                " use valid items and seperate them with '/'."
+    verify_hg_format(config["hostgroup_format"],
-            )
+                     device_cfs=device_cfs, hg_type="dev", logger=logger)
-            logger.error(e)
+    if config["sync_vms"]:
-            raise HostgroupError(e)
+        vm_cfs = list(
+            netbox.extras.custom_fields.filter(type="text",
+                                               content_types="virtualization.virtualmachine")
+        )
+        verify_hg_format(config["vm_hostgroup_format"], vm_cfs=vm_cfs, hg_type="vm", logger=logger)
     # Set Zabbix API
     try:
         ssl_ctx = ssl.create_default_context()
@@ -146,9 +138,6 @@ def main(arguments):
     # Prepare list of all proxy and proxy_groups
     zabbix_proxy_list = proxy_prepper(zabbix_proxies, zabbix_proxygroups)
 
-    # Get NetBox API version
-    nb_version = netbox.version
-
     # Go through all NetBox devices
     for nb_vm in netbox_vms:
         try:
@@ -162,7 +151,7 @@ def main(arguments):
             vm.set_hostgroup(config["vm_hostgroup_format"],
                              netbox_site_groups, netbox_regions)
             # Check if a valid hostgroup has been found for this VM.
-            if not vm.hostgroup:
+            if not vm.hostgroups:
                 continue
             vm.set_inventory(nb_vm)
             vm.set_usermacros()
@@ -185,6 +174,14 @@ def main(arguments):
             # Check if the VM is in the disabled state
             if vm.status in config["zabbix_device_disable"]:
                 vm.zabbix_state = 1
+            # Add hostgroup if config is set
+            if config["create_hostgroups"]:
+                # Create new hostgroup. Potentially multiple groups if nested
+                hostgroups = vm.createZabbixHostgroup(zabbix_groups)
+                # go through all newly created hostgroups
+                for group in hostgroups:
+                    # Add new hostgroups to zabbix group list
+                    zabbix_groups.append(group)
             # Check if VM is already in Zabbix
             if vm.zabbix_id:
                 vm.ConsistencyCheck(
@@ -195,14 +192,6 @@ def main(arguments):
                     config["create_hostgroups"],
                 )
                 continue
-            # Add hostgroup is config is set
-            if config["create_hostgroups"]:
-                # Create new hostgroup. Potentially multiple groups if nested
-                hostgroups = vm.createZabbixHostgroup(zabbix_groups)
-                # go through all newly created hostgroups
-                for group in hostgroups:
-                    # Add new hostgroups to zabbix group list
-                    zabbix_groups.append(group)
             # Add VM to Zabbix
             vm.createInZabbix(zabbix_groups, zabbix_templates, zabbix_proxy_list)
         except SyncError:
@@ -222,7 +211,7 @@ def main(arguments):
             device.set_hostgroup(
                 config["hostgroup_format"], netbox_site_groups, netbox_regions)
             # Check if a valid hostgroup has been found for this VM.
-            if not device.hostgroup:
+            if not device.hostgroups:
                 continue
             device.set_inventory(nb_device)
             device.set_usermacros()
@@ -261,6 +250,14 @@ def main(arguments):
             # Check if the device is in the disabled state
             if device.status in config["zabbix_device_disable"]:
                 device.zabbix_state = 1
+            # Add hostgroup is config is set
+            if config["create_hostgroups"]:
+                # Create new hostgroup. Potentially multiple groups if nested
+                hostgroups = device.createZabbixHostgroup(zabbix_groups)
+                # go through all newly created hostgroups
+                for group in hostgroups:
+                    # Add new hostgroups to zabbix group list
+                    zabbix_groups.append(group)
             # Check if device is already in Zabbix
             if device.zabbix_id:
                 device.ConsistencyCheck(
@@ -271,14 +268,6 @@ def main(arguments):
                     config["create_hostgroups"],
                 )
                 continue
-            # Add hostgroup is config is set
-            if config["create_hostgroups"]:
-                # Create new hostgroup. Potentially multiple groups if nested
-                hostgroups = device.createZabbixHostgroup(zabbix_groups)
-                # go through all newly created hostgroups
-                for group in hostgroups:
-                    # Add new hostgroups to zabbix group list
-                    zabbix_groups.append(group)
             # Add device to Zabbix
             device.createInZabbix(zabbix_groups, zabbix_templates, zabbix_proxy_list)
         except SyncError: