NeoAnd/netbox-zabbix-sync
1
0
Fork 0
mirror of https://github.com/TheNetworkGuy/netbox-zabbix-sync.git synced 2025-12-18 19:32:22 -06:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into additional-hostgroup-support

Browse Source
This commit is contained in:
Raymond Kuiper
2025-06-16 10:06:47 +02:00
committed by GitHub
parent a7a79ea81e 5e390396ba
commit a3259c4fe3
22 changed files with 1551 additions and 212 deletions
Download Patch File Download Diff File Expand all files Collapse all files

121
modules/config.py Normal file
View File

@@ -0,0 +1,121 @@
"""
Module for parsing configuration from the top level config.py file
"""
from pathlib import Path
from importlib import util
from os import environ
from logging import getLogger
logger = getLogger(__name__)
# PLEASE NOTE: This is a sample config file. Please do NOT make any edits in this file!
# You should create your own config.py and it will overwrite the default config.
DEFAULT_CONFIG = {
"templates_config_context": False,
"templates_config_context_overrule": False,
"template_cf": "zabbix_template",
"device_cf": "zabbix_hostid",
"clustering": False,
"create_hostgroups": True,
"create_journal": False,
"sync_vms": False,
"vm_hostgroup_format": "cluster_type/cluster/role",
"full_proxy_sync": False,
"zabbix_device_removal": ["Decommissioning", "Inventory"],
"zabbix_device_disable": ["Offline", "Planned", "Staged", "Failed"],
"hostgroup_format": "site/manufacturer/role",
"traverse_regions": False,
"traverse_site_groups": False,
"nb_device_filter": {"name__n": "null"},
"nb_vm_filter": {"name__n": "null"},
"inventory_mode": "disabled",
"inventory_sync": False,
"device_inventory_map": {
"asset_tag": "asset_tag",
"virtual_chassis/name": "chassis",
"status/label": "deployment_status",
"location/name": "location",
"latitude": "location_lat",
"longitude": "location_lon",
"comments": "notes",
"name": "name",
"rack/name": "site_rack",
"serial": "serialno_a",
"device_type/model": "type",
"device_type/manufacturer/name": "vendor",
"oob_ip/address": "oob_ip"
},
"vm_inventory_map": {
"status/label": "deployment_status",
"comments": "notes",
"name": "name"
},
"usermacro_sync": False,
"device_usermacro_map": {
"serial": "{$HW_SERIAL}",
"role/name": "{$DEV_ROLE}",
"url": "{$NB_URL}",
"id": "{$NB_ID}"
},
"vm_usermacro_map": {
"memory": "{$TOTAL_MEMORY}",
"role/name": "{$DEV_ROLE}",
"url": "{$NB_URL}",
"id": "{$NB_ID}"
},
"tag_sync": False,
"tag_lower": True,
"tag_name": 'NetBox',
"tag_value": "name",
"device_tag_map": {
"site/name": "site",
"rack/name": "rack",
"platform/name": "target"
},
"vm_tag_map": {
"site/name": "site",
"cluster/name": "cluster",
"platform/name": "target"
}
}
def load_config():
"""Returns combined config from all sources"""
# Overwrite default config with config.py
conf = load_config_file(config_default=DEFAULT_CONFIG)
# Overwrite default config and config.py with environment variables
for key in conf:
value_setting = load_env_variable(key)
if value_setting is not None:
conf[key] = value_setting
return conf
def load_env_variable(config_environvar):
"""Returns config from environment variable"""
prefix = "NBZX_"
config_environvar = prefix + config_environvar.upper()
if config_environvar in environ:
return environ[config_environvar]
return None
def load_config_file(config_default, config_file="config.py"):
"""Returns config from config.py file"""
# Check if config.py exists and load it
# If it does not exist, return the default config
config_path = Path(config_file)
if config_path.exists():
dconf = config_default.copy()
# Dynamically import the config module
spec = util.spec_from_file_location("config", config_path)
config_module = util.module_from_spec(spec)
spec.loader.exec_module(config_module)
# Update DEFAULT_CONFIG with variables from the config module
for key in dconf:
if hasattr(config_module, key):
dconf[key] = getattr(config_module, key)
return dconf
return config_default

144
modules/device.py
View File

@@ -2,17 +2,17 @@
"""
Device specific handeling for NetBox to Zabbix
"""
from copy import deepcopy
from logging import getLogger
from os import sys
from re import search
from operator import itemgetter
from zabbix_utils import APIRequestError
from pynetbox import RequestError as NetboxRequestError
from modules.exceptions import (
InterfaceConfigError,
JournalError,
SyncExternalError,
SyncInventoryError,
TemplateError,
@@ -20,33 +20,11 @@ from modules.exceptions import (
from modules.hostgroups import Hostgroup
from modules.interface import ZabbixInterface
from modules.tags import ZabbixTags
from modules.tools import field_mapper, remove_duplicates
from modules.tools import field_mapper, remove_duplicates, sanatize_log_output
from modules.usermacros import ZabbixUsermacros
from modules.config import load_config
try:
from config import (
device_cf,
device_inventory_map,
device_tag_map,
device_usermacro_map,
inventory_mode,
inventory_sync,
tag_lower,
tag_name,
tag_sync,
tag_value,
template_cf,
traverse_regions,
traverse_site_groups,
usermacro_sync,
)
except ModuleNotFoundError:
print(
"Configuration file config.py not found in main directory."
"Please create the file or rename the config.py.example file to config.py."
)
sys.exit(0)
config = load_config()
class PhysicalDevice:
# pylint: disable=too-many-instance-attributes, too-many-arguments, too-many-positional-arguments
@@ -78,7 +56,7 @@ class PhysicalDevice:
self.nb_journals = nb_journal_class
self.inventory_mode = -1
self.inventory = {}
self.usermacros = {}
self.usermacros = []
self.tags = {}
self.logger = logger if logger else getLogger(__name__)
self._setBasics()
@@ -91,15 +69,15 @@ class PhysicalDevice:
def _inventory_map(self):
"""Use device inventory maps"""
return device_inventory_map
return config["device_inventory_map"]
def _usermacro_map(self):
"""Use device inventory maps"""
return device_usermacro_map
return config["device_usermacro_map"]
def _tag_map(self):
"""Use device host tag maps"""
return device_tag_map
return config["device_tag_map"]
def _setBasics(self):
"""
@@ -115,10 +93,10 @@ class PhysicalDevice:
raise SyncInventoryError(e)
# Check if device has custom field for ZBX ID
if device_cf in self.nb.custom_fields:
self.zabbix_id = self.nb.custom_fields[device_cf]
if config["device_cf"] in self.nb.custom_fields:
self.zabbix_id = self.nb.custom_fields[config["device_cf"]]
else:
e = f"Host {self.name}: Custom field {device_cf} not present"
e = f'Host {self.name}: Custom field {config["device_cf"]} not present'
self.logger.warning(e)
raise SyncInventoryError(e)
@@ -147,8 +125,8 @@ class PhysicalDevice:
self.nb,
self.nb_api_version,
logger=self.logger,
nested_sitegroup_flag=traverse_site_groups,
nested_region_flag=traverse_regions,
nested_sitegroup_flag=config['traverse_site_groups'],
nested_region_flag=config['traverse_regions'],
nb_groups=nb_site_groups,
nb_regions=nb_regions,
)
@@ -187,18 +165,20 @@ class PhysicalDevice:
# Get Zabbix templates from the device type
device_type_cfs = self.nb.device_type.custom_fields
# Check if the ZBX Template CF is present
if template_cf in device_type_cfs:
if config["template_cf"] in device_type_cfs:
# Set value to template
return [device_type_cfs[template_cf]]
return [device_type_cfs[config["template_cf"]]]
# Custom field not found, return error
e = (
f"Custom field {template_cf} not "
f"Custom field {config['template_cf']} not "
f"found for {self.nb.device_type.manufacturer.name}"
f" - {self.nb.device_type.display}."
)
self.logger.warning(e)
raise TemplateError(e)
def get_templates_context(self):
"""Get Zabbix templates from the device context"""
if "zabbix" not in self.config_context:
@@ -221,25 +201,24 @@ class PhysicalDevice:
def set_inventory(self, nbdevice):
"""Set host inventory"""
# Set inventory mode. Default is disabled (see class init function).
if inventory_mode == "disabled":
if inventory_sync:
self.logger.error(
f"Host {self.name}: Unable to map NetBox inventory to Zabbix. "
"Inventory sync is enabled in config but inventory mode is disabled."
)
if config["inventory_mode"] == "disabled":
if config["inventory_sync"]:
self.logger.error(f"Host {self.name}: Unable to map NetBox inventory to Zabbix. "
"Inventory sync is enabled in "
"config but inventory mode is disabled.")
return True
if inventory_mode == "manual":
if config["inventory_mode"] == "manual":
self.inventory_mode = 0
elif inventory_mode == "automatic":
elif config["inventory_mode"] == "automatic":
self.inventory_mode = 1
else:
self.logger.error(
f"Host {self.name}: Specified value for inventory mode in"
f" config is not valid. Got value {inventory_mode}"
f" config is not valid. Got value {config['inventory_mode']}"
)
return False
self.inventory = {}
if inventory_sync and self.inventory_mode in [0, 1]:
if config["inventory_sync"] and self.inventory_mode in [0, 1]:
self.logger.debug(f"Host {self.name}: Starting inventory mapper")
self.inventory = field_mapper(
self.name, self._inventory_map(), nbdevice, self.logger
@@ -377,7 +356,7 @@ class PhysicalDevice:
def _zeroize_cf(self):
"""Sets the hostID custom field in NetBox to zero,
effectively destroying the link"""
self.nb.custom_fields[device_cf] = None
self.nb.custom_fields[config["device_cf"]] = None
self.nb.save()
def _zabbixHostnameExists(self):
@@ -421,12 +400,13 @@ class PhysicalDevice:
macros = ZabbixUsermacros(
self.nb,
self._usermacro_map(),
usermacro_sync,
config['usermacro_sync'],
logger=self.logger,
host=self.name,
)
if macros.sync is False:
self.usermacros = []
return True
self.usermacros = macros.generate()
return True
@@ -438,10 +418,10 @@ class PhysicalDevice:
tags = ZabbixTags(
self.nb,
self._tag_map(),
tag_sync,
tag_lower,
tag_name=tag_name,
tag_value=tag_value,
config['tag_sync'],
config['tag_lower'],
tag_name=config['tag_name'],
tag_value=config['tag_value'],
logger=self.logger,
host=self.name,
)
@@ -459,7 +439,7 @@ class PhysicalDevice:
input: List of all proxies and proxy groups in standardized format
"""
# check if the key Zabbix is defined in the config context
if not "zabbix" in self.nb.config_context:
if "zabbix" not in self.nb.config_context:
return False
if (
"proxy" in self.nb.config_context["zabbix"]
@@ -557,7 +537,7 @@ class PhysicalDevice:
self.logger.error(msg)
raise SyncExternalError(msg) from e
# Set NetBox custom field to hostID value.
self.nb.custom_fields[device_cf] = int(self.zabbix_id)
self.nb.custom_fields[config["device_cf"]] = int(self.zabbix_id)
self.nb.save()
msg = f"Host {self.name}: Created host in Zabbix."
self.logger.info(msg)
@@ -622,7 +602,7 @@ class PhysicalDevice:
)
self.logger.error(e)
raise SyncExternalError(e) from None
self.logger.info(f"Updated host {self.name} with data {kwargs}.")
self.logger.info(f"Host {self.name}: updated with data {sanatize_log_output(kwargs)}.")
self.create_journal_entry("info", "Updated host in Zabbix with latest NB data.")
def ConsistencyCheck(
@@ -738,10 +718,8 @@ class PhysicalDevice:
# Check if a proxy has been defined
if self.zbxproxy:
# Check if proxy or proxy group is defined
if (
self.zbxproxy["idtype"] in host
and host[self.zbxproxy["idtype"]] == self.zbxproxy["id"]
):
if (self.zbxproxy["idtype"] in host and
host[self.zbxproxy["idtype"]] == self.zbxproxy["id"]):
self.logger.debug(f"Host {self.name}: proxy in-sync.")
# Backwards compatibility for Zabbix <= 6
elif "proxy_hostid" in host and host["proxy_hostid"] == self.zbxproxy["id"]:
@@ -799,7 +777,7 @@ class PhysicalDevice:
else:
self.logger.warning(f"Host {self.name}: inventory_mode OUT of sync.")
self.updateZabbixHost(inventory_mode=str(self.inventory_mode))
if inventory_sync and self.inventory_mode in [0, 1]:
if config["inventory_sync"] and self.inventory_mode in [0, 1]:
# Check host inventory mapping
if host["inventory"] == self.inventory:
self.logger.debug(f"Host {self.name}: inventory in-sync.")
@@ -808,24 +786,33 @@ class PhysicalDevice:
self.updateZabbixHost(inventory=self.inventory)
# Check host usermacros
if usermacro_sync:
macros_filtered = []
# Do not re-sync secret usermacros unless sync is set to 'full'
if str(usermacro_sync).lower() != "full":
for m in deepcopy(self.usermacros):
if m["type"] == str(1):
# Remove the value as the api doesn't return it
# this will allow us to only update usermacros that don't exist
m.pop("value")
macros_filtered.append(m)
if host["macros"] == self.usermacros or host["macros"] == macros_filtered:
if config['usermacro_sync']:
# Make a full copy synce we dont want to lose the original value
# of secret type macros from Netbox
netbox_macros = deepcopy(self.usermacros)
# Set the sync bit
full_sync_bit = bool(str(config['usermacro_sync']).lower() == "full")
for macro in netbox_macros:
# If the Macro is a secret and full sync is NOT activated
if macro["type"] == str(1) and not full_sync_bit:
# Remove the value as the Zabbix api does not return the value key
# This is required when you want to do a diff between both lists
macro.pop("value")
# Sort all lists
def filter_with_macros(macro):
return macro["macro"]
host["macros"].sort(key=filter_with_macros)
netbox_macros.sort(key=filter_with_macros)
# Check if both lists are the same
if host["macros"] == netbox_macros:
self.logger.debug(f"Host {self.name}: usermacros in-sync.")
else:
self.logger.warning(f"Host {self.name}: usermacros OUT of sync.")
# Update Zabbix with NetBox usermacros
self.updateZabbixHost(macros=self.usermacros)
# Check host usermacros
if tag_sync:
# Check host tags
if config['tag_sync']:
if remove_duplicates(host["tags"], sortkey="tag") == self.tags:
self.logger.debug(f"Host {self.name}: tags in-sync.")
else:
@@ -881,7 +868,8 @@ class PhysicalDevice:
try:
# API call to Zabbix
self.zabbix.hostinterface.update(updates)
e = f"Host {self.name}: solved interface conflict."
e = (f"Host {self.name}: updated interface "
f"with data {sanatize_log_output(updates)}.")
self.logger.info(e)
self.create_journal_entry("info", e)
except APIRequestError as e:
@@ -923,7 +911,7 @@ class PhysicalDevice:
self.nb_journals.create(journal)
self.logger.debug(f"Host {self.name}: Created journal entry in NetBox")
return True
except JournalError(e) as e:
except NetboxRequestError as e:
self.logger.warning(
"Unable to create journal entry for "
f"{self.name}: NB returned {e}"

7
modules/logging.py
View File

@@ -21,9 +21,10 @@ def setup_logger():
"""
# Set logging
lgout = logging.StreamHandler()
lgfile = logging.FileHandler(
path.join(path.dirname(path.realpath(__file__)), "sync.log")
)
# Logfile in the project root
project_root = path.dirname(path.dirname(path.realpath(__file__)))
logfile_path = path.join(project_root, "sync.log")
lgfile = logging.FileHandler(logfile_path)
logging.basicConfig(
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",

39
modules/tools.py
View File

@@ -100,6 +100,7 @@ def remove_duplicates(input_list, sortkey=None):
output_list.sort(key=lambda x: x[sortkey])
return output_list
def verify_hg_format(hg_format, device_cfs=[], vm_cfs=[], hg_type="dev", logger=None):
"""
Verifies hostgroup field format
@@ -148,4 +149,40 @@ def verify_hg_format(hg_format, device_cfs=[], vm_cfs=[], hg_type="dev", logger=
)
logger.error(e)
raise HostgroupError(e)
def sanatize_log_output(data):
"""
Used for the update function to Zabbix which
shows the data that its using to update the host.
Removes and sensitive data from the input.
"""
if not isinstance(data, dict):
return data
sanitized_data = data.copy()
# Check if there are any sensitive macros defined in the data
if "macros" in data:
for macro in sanitized_data["macros"]:
# Check if macro is secret type
if not macro["type"] == str(1):
continue
macro["value"] = "********"
# Check for interface data
if "interfaceid" in data:
# Interface ID is a value which is most likely not helpful
# in logging output or for troubleshooting.
del sanitized_data["interfaceid"]
# InterfaceID also hints that this is a interface update.
# A check is required if there are no macro's used for SNMP security parameters.
if not "details" in data:
return sanitized_data
for key, detail in sanitized_data["details"].items():
# If the detail is a secret, we don't want to log it.
if key in ("authpassphrase", "privpassphrase", "securityname", "community"):
# Check if a macro is used.
# If so then logging the output is not a security issue.
if detail.startswith("{$") and detail.endswith("}"):
continue
# A macro is not used, so we sanitize the value.
sanitized_data["details"][key] = "********"
return sanitized_data

14
modules/usermacros.py
View File

@@ -10,7 +10,7 @@ from modules.tools import field_mapper
class ZabbixUsermacros:
"""Class that represents a Zabbix interface."""
"""Class that represents Zabbix usermacros."""
def __init__(self, nb, usermacro_map, usermacro_sync, logger=None, host=None):
self.nb = nb
@@ -57,7 +57,8 @@ class ZabbixUsermacros:
macro["macro"] = str(macro_name)
if isinstance(macro_properties, dict):
if not "value" in macro_properties:
self.logger.error(f"Usermacro {macro_name} has no value, skipping.")
self.logger.warning(f"Host {self.name}: Usermacro {macro_name} has "
"no value in Netbox, skipping.")
return False
macro["value"] = macro_properties["value"]
@@ -76,13 +77,18 @@ class ZabbixUsermacros:
else:
macro["description"] = ""
elif isinstance(macro_properties, str):
elif isinstance(macro_properties, str) and macro_properties:
macro["value"] = macro_properties
macro["type"] = str(0)
macro["description"] = ""
else:
self.logger.warning(f"Host {self.name}: Usermacro {macro_name} "
"has no value, skipping.")
return False
else:
self.logger.error(
f"Usermacro {macro_name} is not a valid usermacro name, skipping."
f"Host {self.name}: Usermacro {macro_name} is not a valid usermacro name, skipping."
)
return False
return macro

31
modules/virtual_machine.py
View File

@@ -1,27 +1,12 @@
# pylint: disable=duplicate-code
"""Module that hosts all functions for virtual machine processing"""
from os import sys
from modules.device import PhysicalDevice
from modules.exceptions import InterfaceConfigError, SyncInventoryError, TemplateError
from modules.hostgroups import Hostgroup
from modules.interface import ZabbixInterface
try:
from config import (
traverse_regions,
traverse_site_groups,
vm_inventory_map,
vm_tag_map,
vm_usermacro_map,
)
except ModuleNotFoundError:
print(
"Configuration file config.py not found in main directory."
"Please create the file or rename the config.py.example file to config.py."
)
sys.exit(0)
from modules.config import load_config
# Load config
config = load_config()
class VirtualMachine(PhysicalDevice):
@@ -34,15 +19,15 @@ class VirtualMachine(PhysicalDevice):
def _inventory_map(self):
"""use VM inventory maps"""
return vm_inventory_map
return config["vm_inventory_map"]
def _usermacro_map(self):
"""use VM usermacro maps"""
return vm_usermacro_map
return config["vm_usermacro_map"]
def _tag_map(self):
"""use VM tag maps"""
return vm_tag_map
return config["vm_tag_map"]
def set_hostgroup(self, hg_format, nb_site_groups, nb_regions):
"""Set the hostgroup for this device"""
@@ -52,8 +37,8 @@ class VirtualMachine(PhysicalDevice):
self.nb,
self.nb_api_version,
logger=self.logger,
nested_sitegroup_flag=traverse_site_groups,
nested_region_flag=traverse_regions,
nested_sitegroup_flag=config["traverse_site_groups"],
nested_region_flag=config["traverse_regions"],
nb_groups=nb_site_groups,
nb_regions=nb_regions,
)