Removed YAML config logic, added python config logic with default fallback. Added ENV variable support for config parameters.

config.yaml

@@ -1,27 +0,0 @@
-# Required: Custom Field name for Zabbix templates
-template_cf: "zabbix_templates"
-
-# Required: Custom Field name for Zabbix device
-device_cf: "zabbix_hostid"
-
-# Optional: Traverse site groups and assign Zabbix hostgroups based on site groups
-traverse_site_groups: false
-
-# Optional: Traverse regions and assign Zabbix hostgroups based on region hierarchy
-traverse_regions: false
-
-# Optional: Enable inventory syncing for host metadata
-inventory_sync: true
-
-# Optional: Choose which inventory fields to sync ("enabled", "manual", "disabled")
-inventory_mode: "manual"
-
-# Optional: Mapping of NetBox device fields to Zabbix inventory fields
-# See: https://www.zabbix.com/documentation/current/en/manual/api/reference/host/object#host_inventory
-inventory_map:
-  serial: "serial"
-  asset_tag: "asset_tag"
-  description: "comment"
-  location: "location"
-  contact: "contact"
-  site: "site"

modules/config.py

@@ -2,7 +2,9 @@
 Module for parsing configuration from the top level config.yaml file
 """
 from pathlib import Path
-import yaml
+from importlib import util
+from os import environ
+from logging import getLogger
 
 DEFAULT_CONFIG = {
     "templates_config_context": False,
@@ -18,20 +20,43 @@ DEFAULT_CONFIG = {
 }
 
 
-def load_config(config_path="config.yaml"):
+def load_config():
-    """Loads config from YAML file and combines it with default config"""
+    """Returns combined config from all sources"""
-    # Get data from default config.
+    # Overwrite default config with config.py
-    config = DEFAULT_CONFIG.copy()
+    conf = load_config_file(config_default=DEFAULT_CONFIG)
-    # Set config path
+    # Overwrite default config and config.py with environment variables
-    config_file = Path(config_path)
+    for key in conf:
-    # Check if file exists
+        value_setting = load_env_variable(key)
-    if config_file.exists():
+        if value_setting is not None:
-        try:
+            conf[key] = value_setting
-            with open(config_file, "r", encoding="utf-8") as f:
+    return conf
-                user_config = yaml.safe_load(f) or {}
+
-                config.update(user_config)
+
-        except OSError:
+def load_env_variable(config_environvar):
-            # Probably some I/O error with user permissions etc.
+    """Returns config from environment variable"""
-            # Ignore for now and return default config
+    if config_environvar in environ:
-            pass
+        return environ[config_environvar]
-    return config
+    return None
+
+
+def load_config_file(config_default, config_file="config.py"):
+    """Returns config from config.py file"""
+    # Check if config.py exists and load it
+    # If it does not exist, return the default config
+    config_path = Path(config_file)
+    if config_path.exists():
+        dconf = config_default.copy()
+        # Dynamically import the config module
+        spec = util.spec_from_file_location("config", config_path)
+        config_module = util.module_from_spec(spec)
+        spec.loader.exec_module(config_module)
+        # Update DEFAULT_CONFIG with variables from the config module
+        for key in dconf:
+            if hasattr(config_module, key):
+                dconf[key] = getattr(config_module, key)
+        return dconf
+    else:
+        getLogger(__name__).warning(
+            "Config file %s not found. Using default config "
+            "and environment variables.", config_file)
+    return None

modules/device.py

@@ -3,7 +3,6 @@
 """
 Device specific handeling for NetBox to Zabbix
 """
-from os import sys
 from re import search
 from logging import getLogger
 from zabbix_utils import APIRequestError
@@ -11,19 +10,10 @@ from modules.exceptions import (SyncInventoryError, TemplateError, SyncExternalE
                                 InterfaceConfigError, JournalError)
 from modules.interface import ZabbixInterface
 from modules.hostgroups import Hostgroup
-try:
+from modules.config import load_config
-    from config import (
+
-        template_cf, device_cf,
+config = load_config()
-        traverse_site_groups,
+
-        traverse_regions,
-        inventory_sync,
-        inventory_mode,
-        inventory_map
-    )
-except ModuleNotFoundError:
-    print("Configuration file config.py not found in main directory."
-           "Please create the file or rename the config.py.example file to config.py.")
-    sys.exit(0)
 
 class PhysicalDevice():
     # pylint: disable=too-many-instance-attributes, too-many-arguments, too-many-positional-arguments
@@ -76,10 +66,10 @@ class PhysicalDevice():
             raise SyncInventoryError(e)
 
         # Check if device has custom field for ZBX ID
-        if device_cf in self.nb.custom_fields:
+        if config["device_cf"] in self.nb.custom_fields:
-            self.zabbix_id = self.nb.custom_fields[device_cf]
+            self.zabbix_id = self.nb.custom_fields[config["device_cf"]]
         else:
-            e = f"Host {self.name}: Custom field {device_cf} not present"
+            e = f"Host {self.name}: Custom field {config["device_cf"]} not present"
             self.logger.warning(e)
             raise SyncInventoryError(e)
 
@@ -101,8 +91,8 @@ class PhysicalDevice():
         """Set the hostgroup for this device"""
         # Create new Hostgroup instance
         hg = Hostgroup("dev", self.nb, self.nb_api_version, logger=self.logger,
-                       nested_sitegroup_flag=traverse_site_groups,
+                       nested_sitegroup_flag=config["traverse_site_groups"],
-                       nested_region_flag=traverse_regions,
+                       nested_region_flag=config["traverse_regions"],
                        nb_groups=nb_site_groups,
                        nb_regions=nb_regions)
         # Generate hostgroup based on hostgroup format
@@ -137,11 +127,11 @@ class PhysicalDevice():
         # Get Zabbix templates from the device type
         device_type_cfs = self.nb.device_type.custom_fields
         # Check if the ZBX Template CF is present
-        if template_cf in device_type_cfs:
+        if config["template_cf"] in device_type_cfs:
             # Set value to template
-            return [device_type_cfs[template_cf]]
+            return [device_type_cfs[config["template_cf"]]]
         # Custom field not found, return error
-        e = (f"Custom field {template_cf} not "
+        e = (f"Custom field {config["template_cf"]} not "
              f"found for {self.nb.device_type.manufacturer.name}"
              f" - {self.nb.device_type.display}.")
         raise TemplateError(e)
@@ -164,24 +154,24 @@ class PhysicalDevice():
     def set_inventory(self, nbdevice):
         """ Set host inventory """
         # Set inventory mode. Default is disabled (see class init function).
-        if inventory_mode == "disabled":
+        if config["inventory_mode"] == "disabled":
-            if inventory_sync:
+            if config["inventory_sync"]:
                 self.logger.error(f"Host {self.name}: Unable to map NetBox inventory to Zabbix. "
                                   "Inventory sync is enabled in config but inventory mode is disabled.")
             return True
-        if inventory_mode == "manual":
+        if config["inventory_mode"] == "manual":
             self.inventory_mode = 0
-        elif inventory_mode == "automatic":
+        elif config["inventory_mode"] == "automatic":
             self.inventory_mode = 1
         else:
             self.logger.error(f"Host {self.name}: Specified value for inventory mode in"
-                              f" config is not valid. Got value {inventory_mode}")
+                              f" config is not valid. Got value {config["inventory_mode"]}")
             return False
         self.inventory = {}
-        if inventory_sync and self.inventory_mode in [0,1]:
+        if config["inventory_sync"] and self.inventory_mode in [0, 1]:
             self.logger.debug(f"Host {self.name}: Starting inventory mapper")
             # Let's build an inventory dict for each property in the inventory_map
-            for nb_inv_field, zbx_inv_field in inventory_map.items():
+            for nb_inv_field, zbx_inv_field in config["inventory_map"].items():
                 field_list = nb_inv_field.split("/")  # convert str to list based on delimiter
                 # start at the base of the dict...
                 value = nbdevice
@@ -321,7 +311,7 @@ class PhysicalDevice():
     def _zeroize_cf(self):
         """Sets the hostID custom field in NetBox to zero,
         effectively destroying the link"""
-        self.nb.custom_fields[device_cf] = None
+        self.nb.custom_fields[config["device_cf"]] = None
         self.nb.save()
 
     def _zabbixHostnameExists(self):
@@ -366,7 +356,7 @@ class PhysicalDevice():
         input: List of all proxies and proxy groups in standardized format
         """
         # check if the key Zabbix is defined in the config context
-        if not "zabbix" in self.nb.config_context:
+        if "zabbix" not in self.nb.config_context:
             return False
         if ("proxy" in self.nb.config_context["zabbix"] and
                not self.nb.config_context["zabbix"]["proxy"]):
@@ -448,7 +438,7 @@ class PhysicalDevice():
                 self.logger.error(e)
                 raise SyncExternalError(e) from None
             # Set NetBox custom field to hostID value.
-            self.nb.custom_fields[device_cf] = int(self.zabbix_id)
+            self.nb.custom_fields[config["device_cf"]] = int(self.zabbix_id)
             self.nb.save()
             msg = f"Host {self.name}: Created host in Zabbix."
             self.logger.info(msg)
@@ -542,7 +532,7 @@ class PhysicalDevice():
                                     selectGroups=["groupid"],
                                     selectHostGroups=["groupid"],
                                     selectParentTemplates=["templateid"],
-                                    selectInventory=list(inventory_map.values()))
+                                    selectInventory=list(config["inventory_map"].values()))
         if len(host) > 1:
             e = (f"Got {len(host)} results for Zabbix hosts "
                  f"with ID {self.zabbix_id} - hostname {self.name}.")
@@ -656,7 +646,7 @@ class PhysicalDevice():
         else:
             self.logger.warning(f"Host {self.name}: inventory_mode OUT of sync.")
             self.updateZabbixHost(inventory_mode=str(self.inventory_mode))
-        if inventory_sync and self.inventory_mode in [0,1]:
+        if config["inventory_sync"] and self.inventory_mode in [0,1]:
             # Check host inventory mapping
             if host['inventory'] == self.inventory:
                 self.logger.debug(f"Host {self.name}: inventory in-sync.")

modules/virtual_machine.py

@@ -1,21 +1,15 @@
 #!/usr/bin/env python3
 # pylint: disable=duplicate-code
 """Module that hosts all functions for virtual machine processing"""
-
-from os import sys
 from modules.device import PhysicalDevice
 from modules.hostgroups import Hostgroup
 from modules.interface import ZabbixInterface
-from modules.exceptions import TemplateError, InterfaceConfigError, SyncInventoryError
+from modules.exceptions import (TemplateError, InterfaceConfigError,
-try:
+                                SyncInventoryError)
-    from config import (
+from modules.config import load_config
-        traverse_site_groups,
+# Load config
-        traverse_regions
+config = load_config()
-    )
+
-except ModuleNotFoundError:
-    print("Configuration file config.py not found in main directory."
-           "Please create the file or rename the config.py.example file to config.py.")
-    sys.exit(0)
 
 class VirtualMachine(PhysicalDevice):
     """Model for virtual machines"""
@@ -28,8 +22,8 @@ class VirtualMachine(PhysicalDevice):
         """Set the hostgroup for this device"""
         # Create new Hostgroup instance
         hg = Hostgroup("vm", self.nb, self.nb_api_version, logger=self.logger,
-                       nested_sitegroup_flag=traverse_site_groups,
+                       nested_sitegroup_flag=config["traverse_site_groups"],
-                       nested_region_flag=traverse_regions,
+                       nested_region_flag=config["traverse_regions"],
                        nb_groups=nb_site_groups,
                        nb_regions=nb_regions)
         # Generate hostgroup based on hostgroup format

netbox_zabbix_sync.py

@@ -10,28 +10,13 @@ from pynetbox import api
 from pynetbox.core.query import RequestError as NBRequestError
 from requests.exceptions import ConnectionError as RequestsConnectionError
 from zabbix_utils import ZabbixAPI, APIRequestError, ProcessingError
+from modules.config import load_config
 from modules.device import PhysicalDevice
 from modules.virtual_machine import VirtualMachine
 from modules.tools import convert_recordset, proxy_prepper
 from modules.exceptions import EnvironmentVarError, HostgroupError, SyncError
-try:
+
-    from config import (
+config = load_config()
-        templates_config_context,
-        templates_config_context_overrule,
-        clustering, create_hostgroups,
-        create_journal, full_proxy_sync,
-        zabbix_device_removal,
-        zabbix_device_disable,
-        hostgroup_format,
-        vm_hostgroup_format,
-        nb_device_filter,
-        sync_vms,
-        nb_vm_filter
-    )
-except ModuleNotFoundError:
-    print("Configuration file config.py not found in main directory."
-          "Please create the file or rename the config.py.example file to config.py.")
-    sys.exit(1)
 
 # Set logging
 log_format = logging.Formatter('%(asctime)s - %(name)s - '
@@ -83,7 +68,7 @@ def main(arguments):
     # Set NetBox API
     netbox = api(netbox_host, token=netbox_token, threading=True)
     # Check if the provided Hostgroup layout is valid
-    hg_objects = hostgroup_format.split("/")
+    hg_objects = config["hostgroup_format"].split("/")
     allowed_objects = ["location", "role", "manufacturer", "region",
                        "site", "site_group", "tenant", "tenant_group"]
     # Create API call to get all custom fields which are on the device objects
@@ -130,11 +115,11 @@ def main(arguments):
     else:
         proxy_name = "name"
     # Get all Zabbix and NetBox data
-    netbox_devices = list(netbox.dcim.devices.filter(**nb_device_filter))
+    netbox_devices = list(netbox.dcim.devices.filter(**config["nb_device_filter"]))
     netbox_vms = []
-    if sync_vms:
+    if config["sync_vms"]:
         netbox_vms = list(
-            netbox.virtualization.virtual_machines.filter(**nb_vm_filter))
+            netbox.virtualization.virtual_machines.filter(**config["nb_vm_filter"]))
     netbox_site_groups = convert_recordset((netbox.dcim.site_groups.all()))
     netbox_regions = convert_recordset(netbox.dcim.regions.all())
     netbox_journals = netbox.extras.journal_entries
@@ -160,19 +145,19 @@ def main(arguments):
     for nb_vm in netbox_vms:
         try:
             vm = VirtualMachine(nb_vm, zabbix, netbox_journals, nb_version,
-                                create_journal, logger)
+                                config["create_journal"], logger)
             logger.debug(f"Host {vm.name}: started operations on VM.")
             vm.set_vm_template()
             # Check if a valid template has been found for this VM.
             if not vm.zbx_template_names:
                 continue
-            vm.set_hostgroup(vm_hostgroup_format,
+            vm.set_hostgroup(config["vm_hostgroup_format"],
                              netbox_site_groups, netbox_regions)
             # Check if a valid hostgroup has been found for this VM.
             if not vm.hostgroup:
                 continue
             # Checks if device is in cleanup state
-            if vm.status in zabbix_device_removal:
+            if vm.status in config["zabbix_device_removal"]:
                 if vm.zabbix_id:
                     # Delete device from Zabbix
                     # and remove hostID from NetBox.
@@ -185,16 +170,16 @@ def main(arguments):
                             f"not in the active state.")
                 continue
             # Check if the VM is in the disabled state
-            if vm.status in zabbix_device_disable:
+            if vm.status in config["zabbix_device_disable"]:
                 vm.zabbix_state = 1
             # Check if VM is already in Zabbix
             if vm.zabbix_id:
                 vm.ConsistencyCheck(zabbix_groups, zabbix_templates,
-                                    zabbix_proxy_list, full_proxy_sync,
+                                    zabbix_proxy_list, config["full_proxy_sync"],
-                                    create_hostgroups)
+                                    config["create_hostgroups"])
                 continue
             # Add hostgroup is config is set
-            if create_hostgroups:
+            if config["create_hostgroups"]:
                 # Create new hostgroup. Potentially multiple groups if nested
                 hostgroups = vm.createZabbixHostgroup(zabbix_groups)
                 # go through all newly created hostgroups
@@ -211,22 +196,22 @@ def main(arguments):
         try:
             # Set device instance set data such as hostgroup and template information.
             device = PhysicalDevice(nb_device, zabbix, netbox_journals, nb_version,
-                                    create_journal, logger)
+                                    config["create_journal"], logger)
             logger.debug(f"Host {device.name}: started operations on device.")
-            device.set_template(templates_config_context,
+            device.set_template(config["templates_config_context"],
-                                templates_config_context_overrule)
+                                config["templates_config_context_overrule"])
             # Check if a valid template has been found for this VM.
             if not device.zbx_template_names:
                 continue
             device.set_hostgroup(
-                hostgroup_format, netbox_site_groups, netbox_regions)
+                config["hostgroup_format"], netbox_site_groups, netbox_regions)
             # Check if a valid hostgroup has been found for this VM.
             if not device.hostgroup:
                 continue
             device.set_inventory(nb_device)
             # Checks if device is part of cluster.
             # Requires clustering variable
-            if device.isCluster() and clustering:
+            if device.isCluster() and config["clustering"]:
                 # Check if device is primary or secondary
                 if device.promoteMasterDevice():
                     e = (f"Device {device.name}: is "
@@ -240,7 +225,7 @@ def main(arguments):
                     logger.info(e)
                     continue
             # Checks if device is in cleanup state
-            if device.status in zabbix_device_removal:
+            if device.status in config["zabbix_device_removal"]:
                 if device.zabbix_id:
                     # Delete device from Zabbix
                     # and remove hostID from NetBox.
@@ -253,16 +238,16 @@ def main(arguments):
                             f"not in the active state.")
                 continue
             # Check if the device is in the disabled state
-            if device.status in zabbix_device_disable:
+            if device.status in config["zabbix_device_disable"]:
                 device.zabbix_state = 1
             # Check if device is already in Zabbix
             if device.zabbix_id:
                 device.ConsistencyCheck(zabbix_groups, zabbix_templates,
-                                        zabbix_proxy_list, full_proxy_sync,
+                                        zabbix_proxy_list, config["full_proxy_sync"],
-                                        create_hostgroups)
+                                        config["create_hostgroups"])
                 continue
             # Add hostgroup is config is set
-            if create_hostgroups:
+            if config["create_hostgroups"]:
                 # Create new hostgroup. Potentially multiple groups if nested
                 hostgroups = device.createZabbixHostgroup(zabbix_groups)
                 # go through all newly created hostgroups

requirements.txt

@@ -1,3 +1,2 @@
 pynetbox
 zabbix-utils==2.0.1
-pyyaml