From c76e36ad3849df5c4085acf612e23b3bd2dd1d79 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Thu, 19 Dec 2024 16:26:18 +0100 Subject: [PATCH 01/22] Split inventory from the device module and started working on vm inventory support --- Pipfile | 13 +++ Pipfile.lock | 188 +++++++++++++++++++++++++++++++++++++ modules/device.py | 99 +++++++++---------- modules/inventory.py | 81 ++++++++++++++++ modules/tools.py | 1 + modules/virtual_machine.py | 7 ++ netbox_zabbix_sync.py | 2 + 7 files changed, 344 insertions(+), 47 deletions(-) create mode 100644 Pipfile create mode 100644 Pipfile.lock create mode 100644 modules/inventory.py diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..bd0a2ba --- /dev/null +++ b/Pipfile @@ -0,0 +1,13 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +pynetbox = "*" +zabbix-utils = "*" + +[dev-packages] + +[requires] +python_version = "3.11" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..4be3d95 --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,188 @@ +{ + "_meta": { + "hash": { + "sha256": "6c35ac0ebf3610e4591484dfd9246af60fc4679b2d0d39193818d62961b2703c" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.11" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "certifi": { + "hashes": [ + "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8", + "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9" + ], + "markers": "python_version >= '3.6'", + "version": "==2024.8.30" + }, + "charset-normalizer": { + "hashes": [ + "sha256:0099d79bdfcf5c1f0c2c72f91516702ebf8b0b8ddd8905f97a8aecf49712c621", + "sha256:0713f3adb9d03d49d365b70b84775d0a0d18e4ab08d12bc46baa6132ba78aaf6", + "sha256:07afec21bbbbf8a5cc3651aa96b980afe2526e7f048fdfb7f1014d84acc8b6d8", + "sha256:0b309d1747110feb25d7ed6b01afdec269c647d382c857ef4663bbe6ad95a912", + "sha256:0d99dd8ff461990f12d6e42c7347fd9ab2532fb70e9621ba520f9e8637161d7c", + "sha256:0de7b687289d3c1b3e8660d0741874abe7888100efe14bd0f9fd7141bcbda92b", + "sha256:1110e22af8ca26b90bd6364fe4c763329b0ebf1ee213ba32b68c73de5752323d", + "sha256:130272c698667a982a5d0e626851ceff662565379baf0ff2cc58067b81d4f11d", + "sha256:136815f06a3ae311fae551c3df1f998a1ebd01ddd424aa5603a4336997629e95", + "sha256:14215b71a762336254351b00ec720a8e85cada43b987da5a042e4ce3e82bd68e", + "sha256:1db4e7fefefd0f548d73e2e2e041f9df5c59e178b4c72fbac4cc6f535cfb1565", + "sha256:1ffd9493de4c922f2a38c2bf62b831dcec90ac673ed1ca182fe11b4d8e9f2a64", + "sha256:2006769bd1640bdf4d5641c69a3d63b71b81445473cac5ded39740a226fa88ab", + "sha256:20587d20f557fe189b7947d8e7ec5afa110ccf72a3128d61a2a387c3313f46be", + "sha256:223217c3d4f82c3ac5e29032b3f1c2eb0fb591b72161f86d93f5719079dae93e", + "sha256:27623ba66c183eca01bf9ff833875b459cad267aeeb044477fedac35e19ba907", + "sha256:285e96d9d53422efc0d7a17c60e59f37fbf3dfa942073f666db4ac71e8d726d0", + "sha256:2de62e8801ddfff069cd5c504ce3bc9672b23266597d4e4f50eda28846c322f2", + "sha256:2f6c34da58ea9c1a9515621f4d9ac379871a8f21168ba1b5e09d74250de5ad62", + "sha256:309a7de0a0ff3040acaebb35ec45d18db4b28232f21998851cfa709eeff49d62", + "sha256:35c404d74c2926d0287fbd63ed5d27eb911eb9e4a3bb2c6d294f3cfd4a9e0c23", + "sha256:3710a9751938947e6327ea9f3ea6332a09bf0ba0c09cae9cb1f250bd1f1549bc", + "sha256:3d59d125ffbd6d552765510e3f31ed75ebac2c7470c7274195b9161a32350284", + "sha256:40d3ff7fc90b98c637bda91c89d51264a3dcf210cade3a2c6f838c7268d7a4ca", + "sha256:425c5f215d0eecee9a56cdb703203dda90423247421bf0d67125add85d0c4455", + "sha256:43193c5cda5d612f247172016c4bb71251c784d7a4d9314677186a838ad34858", + "sha256:44aeb140295a2f0659e113b31cfe92c9061622cadbc9e2a2f7b8ef6b1e29ef4b", + "sha256:47334db71978b23ebcf3c0f9f5ee98b8d65992b65c9c4f2d34c2eaf5bcaf0594", + "sha256:4796efc4faf6b53a18e3d46343535caed491776a22af773f366534056c4e1fbc", + "sha256:4a51b48f42d9358460b78725283f04bddaf44a9358197b889657deba38f329db", + "sha256:4b67fdab07fdd3c10bb21edab3cbfe8cf5696f453afce75d815d9d7223fbe88b", + "sha256:4ec9dd88a5b71abfc74e9df5ebe7921c35cbb3b641181a531ca65cdb5e8e4dea", + "sha256:4f9fc98dad6c2eaa32fc3af1417d95b5e3d08aff968df0cd320066def971f9a6", + "sha256:54b6a92d009cbe2fb11054ba694bc9e284dad30a26757b1e372a1fdddaf21920", + "sha256:55f56e2ebd4e3bc50442fbc0888c9d8c94e4e06a933804e2af3e89e2f9c1c749", + "sha256:5726cf76c982532c1863fb64d8c6dd0e4c90b6ece9feb06c9f202417a31f7dd7", + "sha256:5d447056e2ca60382d460a604b6302d8db69476fd2015c81e7c35417cfabe4cd", + "sha256:5ed2e36c3e9b4f21dd9422f6893dec0abf2cca553af509b10cd630f878d3eb99", + "sha256:5ff2ed8194587faf56555927b3aa10e6fb69d931e33953943bc4f837dfee2242", + "sha256:62f60aebecfc7f4b82e3f639a7d1433a20ec32824db2199a11ad4f5e146ef5ee", + "sha256:63bc5c4ae26e4bc6be6469943b8253c0fd4e4186c43ad46e713ea61a0ba49129", + "sha256:6b40e8d38afe634559e398cc32b1472f376a4099c75fe6299ae607e404c033b2", + "sha256:6b493a043635eb376e50eedf7818f2f322eabbaa974e948bd8bdd29eb7ef2a51", + "sha256:6dba5d19c4dfab08e58d5b36304b3f92f3bd5d42c1a3fa37b5ba5cdf6dfcbcee", + "sha256:6fd30dc99682dc2c603c2b315bded2799019cea829f8bf57dc6b61efde6611c8", + "sha256:707b82d19e65c9bd28b81dde95249b07bf9f5b90ebe1ef17d9b57473f8a64b7b", + "sha256:7706f5850360ac01d80c89bcef1640683cc12ed87f42579dab6c5d3ed6888613", + "sha256:7782afc9b6b42200f7362858f9e73b1f8316afb276d316336c0ec3bd73312742", + "sha256:79983512b108e4a164b9c8d34de3992f76d48cadc9554c9e60b43f308988aabe", + "sha256:7f683ddc7eedd742e2889d2bfb96d69573fde1d92fcb811979cdb7165bb9c7d3", + "sha256:82357d85de703176b5587dbe6ade8ff67f9f69a41c0733cf2425378b49954de5", + "sha256:84450ba661fb96e9fd67629b93d2941c871ca86fc38d835d19d4225ff946a631", + "sha256:86f4e8cca779080f66ff4f191a685ced73d2f72d50216f7112185dc02b90b9b7", + "sha256:8cda06946eac330cbe6598f77bb54e690b4ca93f593dee1568ad22b04f347c15", + "sha256:8ce7fd6767a1cc5a92a639b391891bf1c268b03ec7e021c7d6d902285259685c", + "sha256:8ff4e7cdfdb1ab5698e675ca622e72d58a6fa2a8aa58195de0c0061288e6e3ea", + "sha256:9289fd5dddcf57bab41d044f1756550f9e7cf0c8e373b8cdf0ce8773dc4bd417", + "sha256:92a7e36b000bf022ef3dbb9c46bfe2d52c047d5e3f3343f43204263c5addc250", + "sha256:92db3c28b5b2a273346bebb24857fda45601aef6ae1c011c0a997106581e8a88", + "sha256:95c3c157765b031331dd4db3c775e58deaee050a3042fcad72cbc4189d7c8dca", + "sha256:980b4f289d1d90ca5efcf07958d3eb38ed9c0b7676bf2831a54d4f66f9c27dfa", + "sha256:9ae4ef0b3f6b41bad6366fb0ea4fc1d7ed051528e113a60fa2a65a9abb5b1d99", + "sha256:9c98230f5042f4945f957d006edccc2af1e03ed5e37ce7c373f00a5a4daa6149", + "sha256:9fa2566ca27d67c86569e8c85297aaf413ffab85a8960500f12ea34ff98e4c41", + "sha256:a14969b8691f7998e74663b77b4c36c0337cb1df552da83d5c9004a93afdb574", + "sha256:a8aacce6e2e1edcb6ac625fb0f8c3a9570ccc7bfba1f63419b3769ccf6a00ed0", + "sha256:a8e538f46104c815be19c975572d74afb53f29650ea2025bbfaef359d2de2f7f", + "sha256:aa41e526a5d4a9dfcfbab0716c7e8a1b215abd3f3df5a45cf18a12721d31cb5d", + "sha256:aa693779a8b50cd97570e5a0f343538a8dbd3e496fa5dcb87e29406ad0299654", + "sha256:ab22fbd9765e6954bc0bcff24c25ff71dcbfdb185fcdaca49e81bac68fe724d3", + "sha256:ab2e5bef076f5a235c3774b4f4028a680432cded7cad37bba0fd90d64b187d19", + "sha256:ab973df98fc99ab39080bfb0eb3a925181454d7c3ac8a1e695fddfae696d9e90", + "sha256:af73657b7a68211996527dbfeffbb0864e043d270580c5aef06dc4b659a4b578", + "sha256:b197e7094f232959f8f20541ead1d9862ac5ebea1d58e9849c1bf979255dfac9", + "sha256:b295729485b06c1a0683af02a9e42d2caa9db04a373dc38a6a58cdd1e8abddf1", + "sha256:b8831399554b92b72af5932cdbbd4ddc55c55f631bb13ff8fe4e6536a06c5c51", + "sha256:b8dcd239c743aa2f9c22ce674a145e0a25cb1566c495928440a181ca1ccf6719", + "sha256:bcb4f8ea87d03bc51ad04add8ceaf9b0f085ac045ab4d74e73bbc2dc033f0236", + "sha256:bd7af3717683bea4c87acd8c0d3d5b44d56120b26fd3f8a692bdd2d5260c620a", + "sha256:bf4475b82be41b07cc5e5ff94810e6a01f276e37c2d55571e3fe175e467a1a1c", + "sha256:c3e446d253bd88f6377260d07c895816ebf33ffffd56c1c792b13bff9c3e1ade", + "sha256:c57516e58fd17d03ebe67e181a4e4e2ccab1168f8c2976c6a334d4f819fe5944", + "sha256:c94057af19bc953643a33581844649a7fdab902624d2eb739738a30e2b3e60fc", + "sha256:cab5d0b79d987c67f3b9e9c53f54a61360422a5a0bc075f43cab5621d530c3b6", + "sha256:ce031db0408e487fd2775d745ce30a7cd2923667cf3b69d48d219f1d8f5ddeb6", + "sha256:cee4373f4d3ad28f1ab6290684d8e2ebdb9e7a1b74fdc39e4c211995f77bec27", + "sha256:d5b054862739d276e09928de37c79ddeec42a6e1bfc55863be96a36ba22926f6", + "sha256:dbe03226baf438ac4fda9e2d0715022fd579cb641c4cf639fa40d53b2fe6f3e2", + "sha256:dc15e99b2d8a656f8e666854404f1ba54765871104e50c8e9813af8a7db07f12", + "sha256:dcaf7c1524c0542ee2fc82cc8ec337f7a9f7edee2532421ab200d2b920fc97cf", + "sha256:dd4eda173a9fcccb5f2e2bd2a9f423d180194b1bf17cf59e3269899235b2a114", + "sha256:dd9a8bd8900e65504a305bf8ae6fa9fbc66de94178c420791d0293702fce2df7", + "sha256:de7376c29d95d6719048c194a9cf1a1b0393fbe8488a22008610b0361d834ecf", + "sha256:e7fdd52961feb4c96507aa649550ec2a0d527c086d284749b2f582f2d40a2e0d", + "sha256:e91f541a85298cf35433bf66f3fab2a4a2cff05c127eeca4af174f6d497f0d4b", + "sha256:e9e3c4c9e1ed40ea53acf11e2a386383c3304212c965773704e4603d589343ed", + "sha256:ee803480535c44e7f5ad00788526da7d85525cfefaf8acf8ab9a310000be4b03", + "sha256:f09cb5a7bbe1ecae6e87901a2eb23e0256bb524a79ccc53eb0b7629fbe7677c4", + "sha256:f19c1585933c82098c2a520f8ec1227f20e339e33aca8fa6f956f6691b784e67", + "sha256:f1a2f519ae173b5b6a2c9d5fa3116ce16e48b3462c8b96dfdded11055e3d6365", + "sha256:f28f891ccd15c514a0981f3b9db9aa23d62fe1a99997512b0491d2ed323d229a", + "sha256:f3e73a4255342d4eb26ef6df01e3962e73aa29baa3124a8e824c5d3364a65748", + "sha256:f606a1881d2663630ea5b8ce2efe2111740df4b687bd78b34a8131baa007f79b", + "sha256:fe9f97feb71aa9896b81973a7bbada8c49501dc73e58a10fcef6663af95e5079", + "sha256:ffc519621dce0c767e96b9c53f09c5d215578e10b02c285809f76509a3931482" + ], + "markers": "python_full_version >= '3.7.0'", + "version": "==3.4.0" + }, + "idna": { + "hashes": [ + "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", + "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3" + ], + "markers": "python_version >= '3.6'", + "version": "==3.10" + }, + "packaging": { + "hashes": [ + "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759", + "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f" + ], + "markers": "python_version >= '3.8'", + "version": "==24.2" + }, + "pynetbox": { + "hashes": [ + "sha256:3f82b5964ca77a608aef6cc2fc48a3961f7667fbbdbb60646655373e3dae00c3", + "sha256:f42ce4df6ce97765df91bb4cc0c0e315683d15135265270d78f595114dd20e2b" + ], + "index": "pypi", + "version": "==7.4.1" + }, + "requests": { + "hashes": [ + "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760", + "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6" + ], + "markers": "python_version >= '3.8'", + "version": "==2.32.3" + }, + "urllib3": { + "hashes": [ + "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac", + "sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9" + ], + "markers": "python_version >= '3.8'", + "version": "==2.2.3" + }, + "zabbix-utils": { + "hashes": [ + "sha256:1eb918096dcf1980a975ff72e4449b5d72c605f79842595dedd0f4ceba3b1225", + "sha256:3c4a98a24c101d89fd938ebe0ad6c9aaa391ac901f2afb75ae682eea88fb77af" + ], + "index": "pypi", + "version": "==2.0.2" + } + }, + "develop": {} +} diff --git a/modules/device.py b/modules/device.py index 07554d0..97206ce 100644 --- a/modules/device.py +++ b/modules/device.py @@ -11,14 +11,15 @@ from modules.exceptions import (SyncInventoryError, TemplateError, SyncExternalE InterfaceConfigError, JournalError) from modules.interface import ZabbixInterface from modules.hostgroups import Hostgroup +from modules.inventory import Inventory + try: from config import ( template_cf, device_cf, traverse_site_groups, traverse_regions, inventory_sync, - inventory_mode, - inventory_map + device_inventory_map ) except ModuleNotFoundError: print("Configuration file config.py not found in main directory." @@ -162,51 +163,55 @@ class PhysicalDevice(): return self.config_context["zabbix"]["templates"] def set_inventory(self, nbdevice): - """ Set host inventory """ - # Set inventory mode. Default is disabled (see class init function). - if inventory_mode == "disabled": - if inventory_sync: - self.logger.error(f"Host {self.name}: Unable to map NetBox inventory to Zabbix. " - "Inventory sync is enabled in config but inventory mode is disabled.") - return True - if inventory_mode == "manual": - self.inventory_mode = 0 - elif inventory_mode == "automatic": - self.inventory_mode = 1 - else: - self.logger.error(f"Host {self.name}: Specified value for inventory mode in" - f" config is not valid. Got value {inventory_mode}") - return False - self.inventory = {} - if inventory_sync and self.inventory_mode in [0,1]: - self.logger.debug(f"Host {self.name}: Starting inventory mapper") - # Let's build an inventory dict for each property in the inventory_map - for nb_inv_field, zbx_inv_field in inventory_map.items(): - field_list = nb_inv_field.split("/") # convert str to list based on delimiter - # start at the base of the dict... - value = nbdevice - # ... and step through the dict till we find the needed value - for item in field_list: - value = value[item] if value else None - # Check if the result is usable and expected - # We want to apply any int or float 0 values, - # even if python thinks those are empty. - if ((value and isinstance(value, int | float | str )) or - (isinstance(value, int | float) and int(value) ==0)): - self.inventory[zbx_inv_field] = str(value) - elif not value: - # empty value should just be an empty string for API compatibility - self.logger.debug(f"Host {self.name}: NetBox inventory lookup for " - f"'{nb_inv_field}' returned an empty value") - self.inventory[zbx_inv_field] = "" - else: - # Value is not a string or numeral, probably not what the user expected. - self.logger.error(f"Host {self.name}: Inventory lookup for '{nb_inv_field}'" - " returned an unexpected type: it will be skipped.") - self.logger.debug(f"Host {self.name}: Inventory mapping complete. " - f"Mapped {len(list(filter(None, self.inventory.values())))} field(s)") - return True + """ Set inventory """ + Inventory.set_inventory(self, nbdevice) +# def set_inventory(self, nbdevice): +# """ Set host inventory """ +# # Set inventory mode. Default is disabled (see class init function). +# if inventory_mode == "disabled": +# if inventory_sync: +# self.logger.error(f"Host {self.name}: Unable to map NetBox inventory to Zabbix. " +# "Inventory sync is enabled in config but inventory mode is disabled.") +# return True +# if inventory_mode == "manual": +# self.inventory_mode = 0 +# elif inventory_mode == "automatic": +# self.inventory_mode = 1 +# else: +# self.logger.error(f"Host {self.name}: Specified value for inventory mode in" +# f" config is not valid. Got value {inventory_mode}") +# return False +# self.inventory = {} +# if inventory_sync and self.inventory_mode in [0,1]: +# self.logger.debug(f"Host {self.name}: Starting inventory mapper") +# # Let's build an inventory dict for each property in the inventory_map +# for nb_inv_field, zbx_inv_field in inventory_map.items(): +# field_list = nb_inv_field.split("/") # convert str to list based on delimiter +# # start at the base of the dict... +# value = nbdevice +# # ... and step through the dict till we find the needed value +# for item in field_list: +# value = value[item] if value else None +# # Check if the result is usable and expected +# # We want to apply any int or float 0 values, +# # even if python thinks those are empty. +# if ((value and isinstance(value, int | float | str )) or +# (isinstance(value, int | float) and int(value) ==0)): +# self.inventory[zbx_inv_field] = str(value) +# elif not value: +# # empty value should just be an empty string for API compatibility +# self.logger.debug(f"Host {self.name}: NetBox inventory lookup for " +# f"'{nb_inv_field}' returned an empty value") +# self.inventory[zbx_inv_field] = "" +# else: +# # Value is not a string or numeral, probably not what the user expected. +# self.logger.error(f"Host {self.name}: Inventory lookup for '{nb_inv_field}'" +# " returned an unexpected type: it will be skipped.") +# self.logger.debug(f"Host {self.name}: Inventory mapping complete. " +# f"Mapped {len(list(filter(None, self.inventory.values())))} field(s)") +# return True +# def isCluster(self): """ Checks if device is part of cluster. @@ -541,7 +546,7 @@ class PhysicalDevice(): 'interfaceid'], selectGroups=["groupid"], selectParentTemplates=["templateid"], - selectInventory=list(inventory_map.values())) + selectInventory=list(device_inventory_map.values())) if len(host) > 1: e = (f"Got {len(host)} results for Zabbix hosts " f"with ID {self.zabbix_id} - hostname {self.name}.") diff --git a/modules/inventory.py b/modules/inventory.py new file mode 100644 index 0000000..7c7cd78 --- /dev/null +++ b/modules/inventory.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +# pylint: disable=invalid-name, logging-not-lazy, too-many-locals, logging-fstring-interpolation, too-many-lines +""" +Device specific handeling for NetBox to Zabbix +""" +from pprint import pprint +from logging import getLogger +from zabbix_utils import APIRequestError +from modules.exceptions import (SyncInventoryError, TemplateError, SyncExternalError, + InterfaceConfigError, JournalError) +try: + from config import ( + inventory_sync, + inventory_mode, + device_inventory_map, + vm_inventory_map + ) +except ModuleNotFoundError: + print("Configuration file config.py not found in main directory." + "Please create the file or rename the config.py.example file to config.py.") + sys.exit(0) + +class Inventory(): + # pylint: disable=too-many-instance-attributes, too-many-arguments, too-many-positional-arguments + """ + Represents Network device. + INPUT: (NetBox device class, ZabbixAPI class, journal flag, NB journal class) + """ + +# def __init__(self, nb, logger=None): +# self.nb = nb + + def set_inventory(self, nbobject): + if hasattr(nbobject, 'device_type'): + inventory_map = device_inventory_map + else: + inventory_map = vm_inventory_map + """ Set host inventory """ + # Set inventory mode. Default is disabled (see class init function). + if inventory_mode == "disabled": + if inventory_sync: + self.logger.error(f"Host {self.name}: Unable to map NetBox inventory to Zabbix. " + "Inventory sync is enabled in config but inventory mode is disabled.") + return True + if inventory_mode == "manual": + self.inventory_mode = 0 + elif inventory_mode == "automatic": + self.inventory_mode = 1 + else: + self.logger.error(f"Host {self.name}: Specified value for inventory mode in" + f" config is not valid. Got value {inventory_mode}") + return False + self.inventory = {} + if inventory_sync and self.inventory_mode in [0,1]: + self.logger.debug(f"Host {self.name}: Starting inventory mapper") + # Let's build an inventory dict for each property in the inventory_map + for nb_inv_field, zbx_inv_field in inventory_map.items(): + field_list = nb_inv_field.split("/") # convert str to list based on delimiter + # start at the base of the dict... + value = nbobject + # ... and step through the dict till we find the needed value + for item in field_list: + value = value[item] if value else None + # Check if the result is usable and expected + # We want to apply any int or float 0 values, + # even if python thinks those are empty. + if ((value and isinstance(value, int | float | str )) or + (isinstance(value, int | float) and int(value) ==0)): + self.inventory[zbx_inv_field] = str(value) + elif not value: + # empty value should just be an empty string for API compatibility + self.logger.debug(f"Host {self.name}: NetBox inventory lookup for " + f"'{nb_inv_field}' returned an empty value") + self.inventory[zbx_inv_field] = "" + else: + # Value is not a string or numeral, probably not what the user expected. + self.logger.error(f"Host {self.name}: Inventory lookup for '{nb_inv_field}'" + " returned an unexpected type: it will be skipped.") + self.logger.debug(f"Host {self.name}: Inventory mapping complete. " + f"Mapped {len(list(filter(None, self.inventory.values())))} field(s)") +# return True diff --git a/modules/tools.py b/modules/tools.py index f722524..5e09265 100644 --- a/modules/tools.py +++ b/modules/tools.py @@ -42,3 +42,4 @@ def proxy_prepper(proxy_list, proxy_group_list): group["monitored_by"] = 2 output.append(group) return output + diff --git a/modules/virtual_machine.py b/modules/virtual_machine.py index 331a463..27069e6 100644 --- a/modules/virtual_machine.py +++ b/modules/virtual_machine.py @@ -6,9 +6,12 @@ from os import sys from modules.device import PhysicalDevice from modules.hostgroups import Hostgroup from modules.interface import ZabbixInterface +from modules.inventory import Inventory from modules.exceptions import TemplateError, InterfaceConfigError, SyncInventoryError try: from config import ( + inventory_sync, + vm_inventory_map, traverse_site_groups, traverse_regions ) @@ -35,6 +38,10 @@ class VirtualMachine(PhysicalDevice): # Generate hostgroup based on hostgroup format self.hostgroup = hg.generate(hg_format) + def set_inventory(self, nbvm): + """ Set inventory """ + Inventory.set_inventory(self, nbvm) + def set_vm_template(self): """ Set Template for VMs. Overwrites default class to skip a lookup of custom fields.""" diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index 935b55e..12c6960 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -5,6 +5,7 @@ import logging import argparse import ssl +from pprint import pprint from os import environ, path, sys from pynetbox import api from pynetbox.core.query import RequestError as NBRequestError @@ -171,6 +172,7 @@ def main(arguments): # Check if a valid hostgroup has been found for this VM. if not vm.hostgroup: continue + vm.set_inventory(nb_vm) # Checks if device is in cleanup state if vm.status in zabbix_device_removal: if vm.zabbix_id: From 8272e34c129373050317abb3096e3d33f7a38e38 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Wed, 12 Feb 2025 11:20:45 +0100 Subject: [PATCH 02/22] removed pipenv artefacts --- Pipfile | 13 ---- Pipfile.lock | 188 --------------------------------------------------- 2 files changed, 201 deletions(-) delete mode 100644 Pipfile delete mode 100644 Pipfile.lock diff --git a/Pipfile b/Pipfile deleted file mode 100644 index bd0a2ba..0000000 --- a/Pipfile +++ /dev/null @@ -1,13 +0,0 @@ -[[source]] -url = "https://pypi.org/simple" -verify_ssl = true -name = "pypi" - -[packages] -pynetbox = "*" -zabbix-utils = "*" - -[dev-packages] - -[requires] -python_version = "3.11" diff --git a/Pipfile.lock b/Pipfile.lock deleted file mode 100644 index 4be3d95..0000000 --- a/Pipfile.lock +++ /dev/null @@ -1,188 +0,0 @@ -{ - "_meta": { - "hash": { - "sha256": "6c35ac0ebf3610e4591484dfd9246af60fc4679b2d0d39193818d62961b2703c" - }, - "pipfile-spec": 6, - "requires": { - "python_version": "3.11" - }, - "sources": [ - { - "name": "pypi", - "url": "https://pypi.org/simple", - "verify_ssl": true - } - ] - }, - "default": { - "certifi": { - "hashes": [ - "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8", - "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9" - ], - "markers": "python_version >= '3.6'", - "version": "==2024.8.30" - }, - "charset-normalizer": { - "hashes": [ - "sha256:0099d79bdfcf5c1f0c2c72f91516702ebf8b0b8ddd8905f97a8aecf49712c621", - "sha256:0713f3adb9d03d49d365b70b84775d0a0d18e4ab08d12bc46baa6132ba78aaf6", - "sha256:07afec21bbbbf8a5cc3651aa96b980afe2526e7f048fdfb7f1014d84acc8b6d8", - "sha256:0b309d1747110feb25d7ed6b01afdec269c647d382c857ef4663bbe6ad95a912", - "sha256:0d99dd8ff461990f12d6e42c7347fd9ab2532fb70e9621ba520f9e8637161d7c", - "sha256:0de7b687289d3c1b3e8660d0741874abe7888100efe14bd0f9fd7141bcbda92b", - "sha256:1110e22af8ca26b90bd6364fe4c763329b0ebf1ee213ba32b68c73de5752323d", - "sha256:130272c698667a982a5d0e626851ceff662565379baf0ff2cc58067b81d4f11d", - "sha256:136815f06a3ae311fae551c3df1f998a1ebd01ddd424aa5603a4336997629e95", - "sha256:14215b71a762336254351b00ec720a8e85cada43b987da5a042e4ce3e82bd68e", - "sha256:1db4e7fefefd0f548d73e2e2e041f9df5c59e178b4c72fbac4cc6f535cfb1565", - "sha256:1ffd9493de4c922f2a38c2bf62b831dcec90ac673ed1ca182fe11b4d8e9f2a64", - "sha256:2006769bd1640bdf4d5641c69a3d63b71b81445473cac5ded39740a226fa88ab", - "sha256:20587d20f557fe189b7947d8e7ec5afa110ccf72a3128d61a2a387c3313f46be", - "sha256:223217c3d4f82c3ac5e29032b3f1c2eb0fb591b72161f86d93f5719079dae93e", - "sha256:27623ba66c183eca01bf9ff833875b459cad267aeeb044477fedac35e19ba907", - "sha256:285e96d9d53422efc0d7a17c60e59f37fbf3dfa942073f666db4ac71e8d726d0", - "sha256:2de62e8801ddfff069cd5c504ce3bc9672b23266597d4e4f50eda28846c322f2", - "sha256:2f6c34da58ea9c1a9515621f4d9ac379871a8f21168ba1b5e09d74250de5ad62", - "sha256:309a7de0a0ff3040acaebb35ec45d18db4b28232f21998851cfa709eeff49d62", - "sha256:35c404d74c2926d0287fbd63ed5d27eb911eb9e4a3bb2c6d294f3cfd4a9e0c23", - "sha256:3710a9751938947e6327ea9f3ea6332a09bf0ba0c09cae9cb1f250bd1f1549bc", - "sha256:3d59d125ffbd6d552765510e3f31ed75ebac2c7470c7274195b9161a32350284", - "sha256:40d3ff7fc90b98c637bda91c89d51264a3dcf210cade3a2c6f838c7268d7a4ca", - "sha256:425c5f215d0eecee9a56cdb703203dda90423247421bf0d67125add85d0c4455", - "sha256:43193c5cda5d612f247172016c4bb71251c784d7a4d9314677186a838ad34858", - "sha256:44aeb140295a2f0659e113b31cfe92c9061622cadbc9e2a2f7b8ef6b1e29ef4b", - "sha256:47334db71978b23ebcf3c0f9f5ee98b8d65992b65c9c4f2d34c2eaf5bcaf0594", - "sha256:4796efc4faf6b53a18e3d46343535caed491776a22af773f366534056c4e1fbc", - "sha256:4a51b48f42d9358460b78725283f04bddaf44a9358197b889657deba38f329db", - "sha256:4b67fdab07fdd3c10bb21edab3cbfe8cf5696f453afce75d815d9d7223fbe88b", - "sha256:4ec9dd88a5b71abfc74e9df5ebe7921c35cbb3b641181a531ca65cdb5e8e4dea", - "sha256:4f9fc98dad6c2eaa32fc3af1417d95b5e3d08aff968df0cd320066def971f9a6", - "sha256:54b6a92d009cbe2fb11054ba694bc9e284dad30a26757b1e372a1fdddaf21920", - "sha256:55f56e2ebd4e3bc50442fbc0888c9d8c94e4e06a933804e2af3e89e2f9c1c749", - "sha256:5726cf76c982532c1863fb64d8c6dd0e4c90b6ece9feb06c9f202417a31f7dd7", - "sha256:5d447056e2ca60382d460a604b6302d8db69476fd2015c81e7c35417cfabe4cd", - "sha256:5ed2e36c3e9b4f21dd9422f6893dec0abf2cca553af509b10cd630f878d3eb99", - "sha256:5ff2ed8194587faf56555927b3aa10e6fb69d931e33953943bc4f837dfee2242", - "sha256:62f60aebecfc7f4b82e3f639a7d1433a20ec32824db2199a11ad4f5e146ef5ee", - "sha256:63bc5c4ae26e4bc6be6469943b8253c0fd4e4186c43ad46e713ea61a0ba49129", - "sha256:6b40e8d38afe634559e398cc32b1472f376a4099c75fe6299ae607e404c033b2", - "sha256:6b493a043635eb376e50eedf7818f2f322eabbaa974e948bd8bdd29eb7ef2a51", - "sha256:6dba5d19c4dfab08e58d5b36304b3f92f3bd5d42c1a3fa37b5ba5cdf6dfcbcee", - "sha256:6fd30dc99682dc2c603c2b315bded2799019cea829f8bf57dc6b61efde6611c8", - "sha256:707b82d19e65c9bd28b81dde95249b07bf9f5b90ebe1ef17d9b57473f8a64b7b", - "sha256:7706f5850360ac01d80c89bcef1640683cc12ed87f42579dab6c5d3ed6888613", - "sha256:7782afc9b6b42200f7362858f9e73b1f8316afb276d316336c0ec3bd73312742", - "sha256:79983512b108e4a164b9c8d34de3992f76d48cadc9554c9e60b43f308988aabe", - "sha256:7f683ddc7eedd742e2889d2bfb96d69573fde1d92fcb811979cdb7165bb9c7d3", - "sha256:82357d85de703176b5587dbe6ade8ff67f9f69a41c0733cf2425378b49954de5", - "sha256:84450ba661fb96e9fd67629b93d2941c871ca86fc38d835d19d4225ff946a631", - "sha256:86f4e8cca779080f66ff4f191a685ced73d2f72d50216f7112185dc02b90b9b7", - "sha256:8cda06946eac330cbe6598f77bb54e690b4ca93f593dee1568ad22b04f347c15", - "sha256:8ce7fd6767a1cc5a92a639b391891bf1c268b03ec7e021c7d6d902285259685c", - "sha256:8ff4e7cdfdb1ab5698e675ca622e72d58a6fa2a8aa58195de0c0061288e6e3ea", - "sha256:9289fd5dddcf57bab41d044f1756550f9e7cf0c8e373b8cdf0ce8773dc4bd417", - "sha256:92a7e36b000bf022ef3dbb9c46bfe2d52c047d5e3f3343f43204263c5addc250", - "sha256:92db3c28b5b2a273346bebb24857fda45601aef6ae1c011c0a997106581e8a88", - "sha256:95c3c157765b031331dd4db3c775e58deaee050a3042fcad72cbc4189d7c8dca", - "sha256:980b4f289d1d90ca5efcf07958d3eb38ed9c0b7676bf2831a54d4f66f9c27dfa", - "sha256:9ae4ef0b3f6b41bad6366fb0ea4fc1d7ed051528e113a60fa2a65a9abb5b1d99", - "sha256:9c98230f5042f4945f957d006edccc2af1e03ed5e37ce7c373f00a5a4daa6149", - "sha256:9fa2566ca27d67c86569e8c85297aaf413ffab85a8960500f12ea34ff98e4c41", - "sha256:a14969b8691f7998e74663b77b4c36c0337cb1df552da83d5c9004a93afdb574", - "sha256:a8aacce6e2e1edcb6ac625fb0f8c3a9570ccc7bfba1f63419b3769ccf6a00ed0", - "sha256:a8e538f46104c815be19c975572d74afb53f29650ea2025bbfaef359d2de2f7f", - "sha256:aa41e526a5d4a9dfcfbab0716c7e8a1b215abd3f3df5a45cf18a12721d31cb5d", - "sha256:aa693779a8b50cd97570e5a0f343538a8dbd3e496fa5dcb87e29406ad0299654", - "sha256:ab22fbd9765e6954bc0bcff24c25ff71dcbfdb185fcdaca49e81bac68fe724d3", - "sha256:ab2e5bef076f5a235c3774b4f4028a680432cded7cad37bba0fd90d64b187d19", - "sha256:ab973df98fc99ab39080bfb0eb3a925181454d7c3ac8a1e695fddfae696d9e90", - "sha256:af73657b7a68211996527dbfeffbb0864e043d270580c5aef06dc4b659a4b578", - "sha256:b197e7094f232959f8f20541ead1d9862ac5ebea1d58e9849c1bf979255dfac9", - "sha256:b295729485b06c1a0683af02a9e42d2caa9db04a373dc38a6a58cdd1e8abddf1", - "sha256:b8831399554b92b72af5932cdbbd4ddc55c55f631bb13ff8fe4e6536a06c5c51", - "sha256:b8dcd239c743aa2f9c22ce674a145e0a25cb1566c495928440a181ca1ccf6719", - "sha256:bcb4f8ea87d03bc51ad04add8ceaf9b0f085ac045ab4d74e73bbc2dc033f0236", - "sha256:bd7af3717683bea4c87acd8c0d3d5b44d56120b26fd3f8a692bdd2d5260c620a", - "sha256:bf4475b82be41b07cc5e5ff94810e6a01f276e37c2d55571e3fe175e467a1a1c", - "sha256:c3e446d253bd88f6377260d07c895816ebf33ffffd56c1c792b13bff9c3e1ade", - "sha256:c57516e58fd17d03ebe67e181a4e4e2ccab1168f8c2976c6a334d4f819fe5944", - "sha256:c94057af19bc953643a33581844649a7fdab902624d2eb739738a30e2b3e60fc", - "sha256:cab5d0b79d987c67f3b9e9c53f54a61360422a5a0bc075f43cab5621d530c3b6", - "sha256:ce031db0408e487fd2775d745ce30a7cd2923667cf3b69d48d219f1d8f5ddeb6", - "sha256:cee4373f4d3ad28f1ab6290684d8e2ebdb9e7a1b74fdc39e4c211995f77bec27", - "sha256:d5b054862739d276e09928de37c79ddeec42a6e1bfc55863be96a36ba22926f6", - "sha256:dbe03226baf438ac4fda9e2d0715022fd579cb641c4cf639fa40d53b2fe6f3e2", - "sha256:dc15e99b2d8a656f8e666854404f1ba54765871104e50c8e9813af8a7db07f12", - "sha256:dcaf7c1524c0542ee2fc82cc8ec337f7a9f7edee2532421ab200d2b920fc97cf", - "sha256:dd4eda173a9fcccb5f2e2bd2a9f423d180194b1bf17cf59e3269899235b2a114", - "sha256:dd9a8bd8900e65504a305bf8ae6fa9fbc66de94178c420791d0293702fce2df7", - "sha256:de7376c29d95d6719048c194a9cf1a1b0393fbe8488a22008610b0361d834ecf", - "sha256:e7fdd52961feb4c96507aa649550ec2a0d527c086d284749b2f582f2d40a2e0d", - "sha256:e91f541a85298cf35433bf66f3fab2a4a2cff05c127eeca4af174f6d497f0d4b", - "sha256:e9e3c4c9e1ed40ea53acf11e2a386383c3304212c965773704e4603d589343ed", - "sha256:ee803480535c44e7f5ad00788526da7d85525cfefaf8acf8ab9a310000be4b03", - "sha256:f09cb5a7bbe1ecae6e87901a2eb23e0256bb524a79ccc53eb0b7629fbe7677c4", - "sha256:f19c1585933c82098c2a520f8ec1227f20e339e33aca8fa6f956f6691b784e67", - "sha256:f1a2f519ae173b5b6a2c9d5fa3116ce16e48b3462c8b96dfdded11055e3d6365", - "sha256:f28f891ccd15c514a0981f3b9db9aa23d62fe1a99997512b0491d2ed323d229a", - "sha256:f3e73a4255342d4eb26ef6df01e3962e73aa29baa3124a8e824c5d3364a65748", - "sha256:f606a1881d2663630ea5b8ce2efe2111740df4b687bd78b34a8131baa007f79b", - "sha256:fe9f97feb71aa9896b81973a7bbada8c49501dc73e58a10fcef6663af95e5079", - "sha256:ffc519621dce0c767e96b9c53f09c5d215578e10b02c285809f76509a3931482" - ], - "markers": "python_full_version >= '3.7.0'", - "version": "==3.4.0" - }, - "idna": { - "hashes": [ - "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", - "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3" - ], - "markers": "python_version >= '3.6'", - "version": "==3.10" - }, - "packaging": { - "hashes": [ - "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759", - "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f" - ], - "markers": "python_version >= '3.8'", - "version": "==24.2" - }, - "pynetbox": { - "hashes": [ - "sha256:3f82b5964ca77a608aef6cc2fc48a3961f7667fbbdbb60646655373e3dae00c3", - "sha256:f42ce4df6ce97765df91bb4cc0c0e315683d15135265270d78f595114dd20e2b" - ], - "index": "pypi", - "version": "==7.4.1" - }, - "requests": { - "hashes": [ - "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760", - "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6" - ], - "markers": "python_version >= '3.8'", - "version": "==2.32.3" - }, - "urllib3": { - "hashes": [ - "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac", - "sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9" - ], - "markers": "python_version >= '3.8'", - "version": "==2.2.3" - }, - "zabbix-utils": { - "hashes": [ - "sha256:1eb918096dcf1980a975ff72e4449b5d72c605f79842595dedd0f4ceba3b1225", - "sha256:3c4a98a24c101d89fd938ebe0ad6c9aaa391ac901f2afb75ae682eea88fb77af" - ], - "index": "pypi", - "version": "==2.0.2" - } - }, - "develop": {} -} From 4c91c660a8bb55b4b9a52b548ad7fc0a2770f599 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Wed, 12 Feb 2025 11:22:27 +0100 Subject: [PATCH 03/22] removed newline --- modules/tools.py | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/tools.py b/modules/tools.py index 5e09265..f722524 100644 --- a/modules/tools.py +++ b/modules/tools.py @@ -42,4 +42,3 @@ def proxy_prepper(proxy_list, proxy_group_list): group["monitored_by"] = 2 output.append(group) return output - From ba2f77a640d8396c7ebc0535baa186c130d318c7 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Wed, 12 Feb 2025 11:25:27 +0100 Subject: [PATCH 04/22] Added Pipfile ignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index c3069c9..2a3448b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ *.log .venv config.py +Pipfile +Pipfile.lock # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] From c7d3dab27ca762fb2e59b92157ad3d54097b5e93 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Wed, 12 Feb 2025 12:30:28 +0100 Subject: [PATCH 05/22] reverted module split, switched to class inheretance instead. Updated config example. --- config.py.example | 39 ++++++++------ modules/device.py | 106 +++++++++++++++++++------------------ modules/inventory.py | 81 ---------------------------- modules/virtual_machine.py | 12 +++-- 4 files changed, 87 insertions(+), 151 deletions(-) delete mode 100644 modules/inventory.py diff --git a/config.py.example b/config.py.example index 1d83223..7f8861e 100644 --- a/config.py.example +++ b/config.py.example @@ -80,19 +80,28 @@ inventory_sync = False # For nested properties, you can use the '/' seperator. # For example, the following map will assign the custom field 'mycustomfield' to the 'alias' Zabbix inventory field: # -# inventory_map = { "custom_fields/mycustomfield/name": "alias"} +# device_inventory_map = { "custom_fields/mycustomfield/name": "alias"} # -# The following map should provide some nice defaults: -inventory_map = { "asset_tag": "asset_tag", - "virtual_chassis/name": "chassis", - "status/label": "deployment_status", - "location/name": "location", - "latitude": "location_lat", - "longitude": "location_lon", - "comments": "notes", - "name": "name", - "rack/name": "site_rack", - "serial": "serialno_a", - "device_type/model": "type", - "device_type/manufacturer/name": "vendor", - "oob_ip/address": "oob_ip" } +# The following maps should provide some nice defaults: +device_inventory_map = { "asset_tag": "asset_tag", + "virtual_chassis/name": "chassis", + "status/label": "deployment_status", + "location/name": "location", + "latitude": "location_lat", + "longitude": "location_lon", + "comments": "notes", + "name": "name", + "rack/name": "site_rack", + "serial": "serialno_a", + "device_type/model": "type", + "device_type/manufacturer/name": "vendor", + "oob_ip/address": "oob_ip" } + +# We also support inventory mapping on Virtual Machines. +vm_inventory_map = { "asset_tag": "asset_tag", + "status/label": "deployment_status", + "location/name": "location", + "latitude": "location_lat", + "longitude": "location_lon", + "comments": "notes", + "name": "name" } diff --git a/modules/device.py b/modules/device.py index 97206ce..ae1488c 100644 --- a/modules/device.py +++ b/modules/device.py @@ -11,7 +11,6 @@ from modules.exceptions import (SyncInventoryError, TemplateError, SyncExternalE InterfaceConfigError, JournalError) from modules.interface import ZabbixInterface from modules.hostgroups import Hostgroup -from modules.inventory import Inventory try: from config import ( @@ -19,6 +18,7 @@ try: traverse_site_groups, traverse_regions, inventory_sync, + inventory_mode, device_inventory_map ) except ModuleNotFoundError: @@ -63,6 +63,10 @@ class PhysicalDevice(): def __str__(self): return self.__repr__() + def _inventory_map(self): + """ Use device inventory maps """ + return device_inventory_map + def _setBasics(self): """ Sets basic information like IP address. @@ -162,56 +166,56 @@ class PhysicalDevice(): return [self.config_context["zabbix"]["templates"]] return self.config_context["zabbix"]["templates"] - def set_inventory(self, nbdevice): - """ Set inventory """ - Inventory.set_inventory(self, nbdevice) - # def set_inventory(self, nbdevice): -# """ Set host inventory """ -# # Set inventory mode. Default is disabled (see class init function). -# if inventory_mode == "disabled": -# if inventory_sync: -# self.logger.error(f"Host {self.name}: Unable to map NetBox inventory to Zabbix. " -# "Inventory sync is enabled in config but inventory mode is disabled.") -# return True -# if inventory_mode == "manual": -# self.inventory_mode = 0 -# elif inventory_mode == "automatic": -# self.inventory_mode = 1 -# else: -# self.logger.error(f"Host {self.name}: Specified value for inventory mode in" -# f" config is not valid. Got value {inventory_mode}") -# return False -# self.inventory = {} -# if inventory_sync and self.inventory_mode in [0,1]: -# self.logger.debug(f"Host {self.name}: Starting inventory mapper") -# # Let's build an inventory dict for each property in the inventory_map -# for nb_inv_field, zbx_inv_field in inventory_map.items(): -# field_list = nb_inv_field.split("/") # convert str to list based on delimiter -# # start at the base of the dict... -# value = nbdevice -# # ... and step through the dict till we find the needed value -# for item in field_list: -# value = value[item] if value else None -# # Check if the result is usable and expected -# # We want to apply any int or float 0 values, -# # even if python thinks those are empty. -# if ((value and isinstance(value, int | float | str )) or -# (isinstance(value, int | float) and int(value) ==0)): -# self.inventory[zbx_inv_field] = str(value) -# elif not value: -# # empty value should just be an empty string for API compatibility -# self.logger.debug(f"Host {self.name}: NetBox inventory lookup for " -# f"'{nb_inv_field}' returned an empty value") -# self.inventory[zbx_inv_field] = "" -# else: -# # Value is not a string or numeral, probably not what the user expected. -# self.logger.error(f"Host {self.name}: Inventory lookup for '{nb_inv_field}'" -# " returned an unexpected type: it will be skipped.") -# self.logger.debug(f"Host {self.name}: Inventory mapping complete. " -# f"Mapped {len(list(filter(None, self.inventory.values())))} field(s)") -# return True -# +# """ Set inventory """ +# Inventory.set_inventory(self, nbdevice) + + def set_inventory(self, nbdevice): + """ Set host inventory """ + # Set inventory mode. Default is disabled (see class init function). + if inventory_mode == "disabled": + if inventory_sync: + self.logger.error(f"Host {self.name}: Unable to map NetBox inventory to Zabbix. " + "Inventory sync is enabled in config but inventory mode is disabled.") + return True + if inventory_mode == "manual": + self.inventory_mode = 0 + elif inventory_mode == "automatic": + self.inventory_mode = 1 + else: + self.logger.error(f"Host {self.name}: Specified value for inventory mode in" + f" config is not valid. Got value {inventory_mode}") + return False + self.inventory = {} + if inventory_sync and self.inventory_mode in [0,1]: + self.logger.debug(f"Host {self.name}: Starting inventory mapper") + # Let's build an inventory dict for each property in the inventory_map + for nb_inv_field, zbx_inv_field in self._inventory_map().items(): + field_list = nb_inv_field.split("/") # convert str to list based on delimiter + # start at the base of the dict... + value = nbdevice + # ... and step through the dict till we find the needed value + for item in field_list: + value = value[item] if value else None + # Check if the result is usable and expected + # We want to apply any int or float 0 values, + # even if python thinks those are empty. + if ((value and isinstance(value, int | float | str )) or + (isinstance(value, int | float) and int(value) ==0)): + self.inventory[zbx_inv_field] = str(value) + elif not value: + # empty value should just be an empty string for API compatibility + self.logger.debug(f"Host {self.name}: NetBox inventory lookup for " + f"'{nb_inv_field}' returned an empty value") + self.inventory[zbx_inv_field] = "" + else: + # Value is not a string or numeral, probably not what the user expected. + self.logger.error(f"Host {self.name}: Inventory lookup for '{nb_inv_field}'" + " returned an unexpected type: it will be skipped.") + self.logger.debug(f"Host {self.name}: Inventory mapping complete. " + f"Mapped {len(list(filter(None, self.inventory.values())))} field(s)") + return True + def isCluster(self): """ Checks if device is part of cluster. @@ -546,7 +550,7 @@ class PhysicalDevice(): 'interfaceid'], selectGroups=["groupid"], selectParentTemplates=["templateid"], - selectInventory=list(device_inventory_map.values())) + selectInventory=list(self._inventory_map().values())) if len(host) > 1: e = (f"Got {len(host)} results for Zabbix hosts " f"with ID {self.zabbix_id} - hostname {self.name}.") diff --git a/modules/inventory.py b/modules/inventory.py deleted file mode 100644 index 7c7cd78..0000000 --- a/modules/inventory.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python3 -# pylint: disable=invalid-name, logging-not-lazy, too-many-locals, logging-fstring-interpolation, too-many-lines -""" -Device specific handeling for NetBox to Zabbix -""" -from pprint import pprint -from logging import getLogger -from zabbix_utils import APIRequestError -from modules.exceptions import (SyncInventoryError, TemplateError, SyncExternalError, - InterfaceConfigError, JournalError) -try: - from config import ( - inventory_sync, - inventory_mode, - device_inventory_map, - vm_inventory_map - ) -except ModuleNotFoundError: - print("Configuration file config.py not found in main directory." - "Please create the file or rename the config.py.example file to config.py.") - sys.exit(0) - -class Inventory(): - # pylint: disable=too-many-instance-attributes, too-many-arguments, too-many-positional-arguments - """ - Represents Network device. - INPUT: (NetBox device class, ZabbixAPI class, journal flag, NB journal class) - """ - -# def __init__(self, nb, logger=None): -# self.nb = nb - - def set_inventory(self, nbobject): - if hasattr(nbobject, 'device_type'): - inventory_map = device_inventory_map - else: - inventory_map = vm_inventory_map - """ Set host inventory """ - # Set inventory mode. Default is disabled (see class init function). - if inventory_mode == "disabled": - if inventory_sync: - self.logger.error(f"Host {self.name}: Unable to map NetBox inventory to Zabbix. " - "Inventory sync is enabled in config but inventory mode is disabled.") - return True - if inventory_mode == "manual": - self.inventory_mode = 0 - elif inventory_mode == "automatic": - self.inventory_mode = 1 - else: - self.logger.error(f"Host {self.name}: Specified value for inventory mode in" - f" config is not valid. Got value {inventory_mode}") - return False - self.inventory = {} - if inventory_sync and self.inventory_mode in [0,1]: - self.logger.debug(f"Host {self.name}: Starting inventory mapper") - # Let's build an inventory dict for each property in the inventory_map - for nb_inv_field, zbx_inv_field in inventory_map.items(): - field_list = nb_inv_field.split("/") # convert str to list based on delimiter - # start at the base of the dict... - value = nbobject - # ... and step through the dict till we find the needed value - for item in field_list: - value = value[item] if value else None - # Check if the result is usable and expected - # We want to apply any int or float 0 values, - # even if python thinks those are empty. - if ((value and isinstance(value, int | float | str )) or - (isinstance(value, int | float) and int(value) ==0)): - self.inventory[zbx_inv_field] = str(value) - elif not value: - # empty value should just be an empty string for API compatibility - self.logger.debug(f"Host {self.name}: NetBox inventory lookup for " - f"'{nb_inv_field}' returned an empty value") - self.inventory[zbx_inv_field] = "" - else: - # Value is not a string or numeral, probably not what the user expected. - self.logger.error(f"Host {self.name}: Inventory lookup for '{nb_inv_field}'" - " returned an unexpected type: it will be skipped.") - self.logger.debug(f"Host {self.name}: Inventory mapping complete. " - f"Mapped {len(list(filter(None, self.inventory.values())))} field(s)") -# return True diff --git a/modules/virtual_machine.py b/modules/virtual_machine.py index 27069e6..353a245 100644 --- a/modules/virtual_machine.py +++ b/modules/virtual_machine.py @@ -6,11 +6,11 @@ from os import sys from modules.device import PhysicalDevice from modules.hostgroups import Hostgroup from modules.interface import ZabbixInterface -from modules.inventory import Inventory from modules.exceptions import TemplateError, InterfaceConfigError, SyncInventoryError try: from config import ( inventory_sync, + inventory_mode, vm_inventory_map, traverse_site_groups, traverse_regions @@ -27,6 +27,10 @@ class VirtualMachine(PhysicalDevice): self.hostgroup = None self.zbx_template_names = None + def _inventory_map(self): + """ use VM inventory maps """ + return vm_inventory_map + def set_hostgroup(self, hg_format, nb_site_groups, nb_regions): """Set the hostgroup for this device""" # Create new Hostgroup instance @@ -38,9 +42,9 @@ class VirtualMachine(PhysicalDevice): # Generate hostgroup based on hostgroup format self.hostgroup = hg.generate(hg_format) - def set_inventory(self, nbvm): - """ Set inventory """ - Inventory.set_inventory(self, nbvm) +# def set_inventory(self, nbvm): +# """ Set inventory """ +# Inventory.set_inventory(self, nbvm) def set_vm_template(self): """ Set Template for VMs. Overwrites default class From 1157ed9e640c826a689ad3bfe13f03cf921ea8cf Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Wed, 12 Feb 2025 12:32:42 +0100 Subject: [PATCH 06/22] cleanup --- modules/device.py | 4 ---- modules/virtual_machine.py | 4 ---- 2 files changed, 8 deletions(-) diff --git a/modules/device.py b/modules/device.py index ae1488c..349db0a 100644 --- a/modules/device.py +++ b/modules/device.py @@ -166,10 +166,6 @@ class PhysicalDevice(): return [self.config_context["zabbix"]["templates"]] return self.config_context["zabbix"]["templates"] -# def set_inventory(self, nbdevice): -# """ Set inventory """ -# Inventory.set_inventory(self, nbdevice) - def set_inventory(self, nbdevice): """ Set host inventory """ # Set inventory mode. Default is disabled (see class init function). diff --git a/modules/virtual_machine.py b/modules/virtual_machine.py index 353a245..b8fa1a1 100644 --- a/modules/virtual_machine.py +++ b/modules/virtual_machine.py @@ -42,10 +42,6 @@ class VirtualMachine(PhysicalDevice): # Generate hostgroup based on hostgroup format self.hostgroup = hg.generate(hg_format) -# def set_inventory(self, nbvm): -# """ Set inventory """ -# Inventory.set_inventory(self, nbvm) - def set_vm_template(self): """ Set Template for VMs. Overwrites default class to skip a lookup of custom fields.""" From 5f78a2c7890b49cb720ffb4ff1de1bffc32a2168 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Wed, 12 Feb 2025 12:35:21 +0100 Subject: [PATCH 07/22] removed unsupported field from vm_inventory_map --- config.py.example | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/config.py.example b/config.py.example index 7f8861e..dcc307f 100644 --- a/config.py.example +++ b/config.py.example @@ -98,8 +98,7 @@ device_inventory_map = { "asset_tag": "asset_tag", "oob_ip/address": "oob_ip" } # We also support inventory mapping on Virtual Machines. -vm_inventory_map = { "asset_tag": "asset_tag", - "status/label": "deployment_status", +vm_inventory_map = { "status/label": "deployment_status", "location/name": "location", "latitude": "location_lat", "longitude": "location_lon", From b8bb3fb3f091c4da0d9b33507fff5594b1f5abc1 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Wed, 12 Feb 2025 12:36:27 +0100 Subject: [PATCH 08/22] removed unsupported fields from vm_inventory_map --- config.py.example | 3 --- 1 file changed, 3 deletions(-) diff --git a/config.py.example b/config.py.example index dcc307f..0a653d6 100644 --- a/config.py.example +++ b/config.py.example @@ -99,8 +99,5 @@ device_inventory_map = { "asset_tag": "asset_tag", # We also support inventory mapping on Virtual Machines. vm_inventory_map = { "status/label": "deployment_status", - "location/name": "location", - "latitude": "location_lat", - "longitude": "location_lon", "comments": "notes", "name": "name" } From c67180138eb60ee7a8fefd433bd836d3c2ff7032 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Wed, 12 Feb 2025 12:39:36 +0100 Subject: [PATCH 09/22] cleanup --- netbox_zabbix_sync.py | 1 - 1 file changed, 1 deletion(-) diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index 12c6960..3eaea3f 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -5,7 +5,6 @@ import logging import argparse import ssl -from pprint import pprint from os import environ, path, sys from pynetbox import api from pynetbox.core.query import RequestError as NBRequestError From cebefd681e1c47829d849637ba68c6a7be0d05fb Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Wed, 12 Feb 2025 17:43:57 +0100 Subject: [PATCH 10/22] started work on macro support --- modules/device.py | 35 ++++++++++++++++++++++++++++---- modules/exceptions.py | 3 +++ modules/usermacros.py | 47 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 4 deletions(-) create mode 100644 modules/usermacros.py diff --git a/modules/device.py b/modules/device.py index 666fc15..dc12a28 100644 --- a/modules/device.py +++ b/modules/device.py @@ -8,9 +8,11 @@ from re import search from logging import getLogger from zabbix_utils import APIRequestError from modules.exceptions import (SyncInventoryError, TemplateError, SyncExternalError, - InterfaceConfigError, JournalError) + InterfaceConfigError, JournalError, UsermacroError) from modules.interface import ZabbixInterface +from modules.usermacros import ZabbixUsermacros from modules.hostgroups import Hostgroup +from pprint import pprint try: from config import ( @@ -19,7 +21,8 @@ try: traverse_regions, inventory_sync, inventory_mode, - device_inventory_map + device_inventory_map, + device_usermacro_map ) except ModuleNotFoundError: print("Configuration file config.py not found in main directory." @@ -54,6 +57,7 @@ class PhysicalDevice(): self.nb_journals = nb_journal_class self.inventory_mode = -1 self.inventory = {} + self.usermacros = {} self.logger = logger if logger else getLogger(__name__) self._setBasics() @@ -67,6 +71,10 @@ class PhysicalDevice(): """ Use device inventory maps """ return device_inventory_map + def _usermacro_map(self): + """ Use device inventory maps """ + return device_usermacro_map + def _setBasics(self): """ Sets basic information like IP address. @@ -363,6 +371,19 @@ class PhysicalDevice(): self.logger.warning(message) raise SyncInventoryError(message) from e + def setUsermacros(self): + try: + # Initiate interface class + macros = ZabbixUsermacros(self.nb.config_context, self._usermacro_map()) + if macros.sync == False: + return {} + else: + return [{'macro': '{$USERMACRO}', 'value': '123', 'type': 0, 'description': 'just a test'}] + except UsermacroError as e: + message = f"{self.name}: {e}" + self.logger.warning(message) + raise UsermacroError(message) from e + def setProxy(self, proxy_list): """ Sets proxy or proxy group if this @@ -423,6 +444,8 @@ class PhysicalDevice(): groups = [{"groupid": self.group_id}] # Set Zabbix proxy if defined self.setProxy(proxies) + # Set usermacros + self.usermacros = self.setUsermacros() # Set basic data for host creation create_data = {"host": self.name, "name": self.visible_name, @@ -432,7 +455,8 @@ class PhysicalDevice(): "templates": templateids, "description": description, "inventory_mode": self.inventory_mode, - "inventory": self.inventory + "inventory": self.inventory, + "macros": self.usermacros } # If a Zabbix proxy or Zabbix Proxy group has been defined if self.zbxproxy: @@ -547,7 +571,10 @@ class PhysicalDevice(): selectGroups=["groupid"], selectHostGroups=["groupid"], selectParentTemplates=["templateid"], - selectInventory=list(self._inventory_map().values())) + selectInventory=list(self._inventory_map().values()), + selectMacros=["macro","value","type","description"] + ) + pprint(host) if len(host) > 1: e = (f"Got {len(host)} results for Zabbix hosts " f"with ID {self.zabbix_id} - hostname {self.name}.") diff --git a/modules/exceptions.py b/modules/exceptions.py index 856433a..27a141c 100644 --- a/modules/exceptions.py +++ b/modules/exceptions.py @@ -31,3 +31,6 @@ class HostgroupError(SyncError): class TemplateError(SyncError): """ Class TemplateError """ + +class UsermacroError(SyncError): + """ Class UsermacroError """ diff --git a/modules/usermacros.py b/modules/usermacros.py new file mode 100644 index 0000000..9f53760 --- /dev/null +++ b/modules/usermacros.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +""" +All of the Zabbix Usermacro related configuration +""" +from logging import getLogger +from zabbix_utils import APIRequestError +from modules.exceptions import UsermacroError + + +from pprint import pprint + +try: + from config import ( + usermacro_sync, + ) +except ModuleNotFoundError: + print("Configuration file config.py not found in main directory." + "Please create the file or rename the config.py.example file to config.py.") + sys.exit(0) + +class ZabbixUsermacros(): + """Class that represents a Zabbix interface.""" + + def __init__(self, context, usermacro_map, logger=None): + self.context = context + self.usermacro_map = usermacro_map + self.logger = logger if logger else getLogger(__name__) + self.usermacros = {} + self.sync = False + self.force_sync = False + self._setConfig() + + def __repr__(self): + return self.name + + def __str__(self): + return self.__repr__() + + def _setConfig(self): + if str(usermacro_sync) == "full": + self.sync = True + self.force_sync = True + elif usermacro_sync: + self.sync = True + return True + + From 6d4e250b236efe0a70594f6fd604d0a29cbf2d4a Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Fri, 14 Feb 2025 08:28:10 +0100 Subject: [PATCH 11/22] :sparkles: Working usermacros based on config context --- modules/device.py | 40 ++++++++++++++++++++++----------- modules/usermacros.py | 51 +++++++++++++++++++++++++++++++++++++++---- 2 files changed, 74 insertions(+), 17 deletions(-) diff --git a/modules/device.py b/modules/device.py index dc12a28..1a9e452 100644 --- a/modules/device.py +++ b/modules/device.py @@ -5,6 +5,7 @@ Device specific handeling for NetBox to Zabbix """ from os import sys from re import search +from copy import deepcopy from logging import getLogger from zabbix_utils import APIRequestError from modules.exceptions import (SyncInventoryError, TemplateError, SyncExternalError, @@ -22,6 +23,7 @@ try: inventory_sync, inventory_mode, device_inventory_map, + usermacro_sync, device_usermacro_map ) except ModuleNotFoundError: @@ -372,18 +374,13 @@ class PhysicalDevice(): raise SyncInventoryError(message) from e def setUsermacros(self): - try: - # Initiate interface class - macros = ZabbixUsermacros(self.nb.config_context, self._usermacro_map()) - if macros.sync == False: - return {} - else: - return [{'macro': '{$USERMACRO}', 'value': '123', 'type': 0, 'description': 'just a test'}] - except UsermacroError as e: - message = f"{self.name}: {e}" - self.logger.warning(message) - raise UsermacroError(message) from e - + # Initiate Usermacros class + macros = ZabbixUsermacros(self.nb.config_context, self._usermacro_map()) + if macros.sync == False: + return [] + else: + return macros.generate() + def setProxy(self, proxy_list): """ Sets proxy or proxy group if this @@ -574,7 +571,6 @@ class PhysicalDevice(): selectInventory=list(self._inventory_map().values()), selectMacros=["macro","value","type","description"] ) - pprint(host) if len(host) > 1: e = (f"Got {len(host)} results for Zabbix hosts " f"with ID {self.zabbix_id} - hostname {self.name}.") @@ -696,6 +692,24 @@ class PhysicalDevice(): self.logger.warning(f"Host {self.name}: inventory OUT of sync.") self.updateZabbixHost(inventory=self.inventory) + # Check host usermacros + if usermacro_sync: + macros_filtered = [] + self.usermacros = self.setUsermacros() + # Do not re-sync secret usermacros unless sync is set to 'full' + if not str(usermacro_sync).lower() == "full": + for m in deepcopy(self.usermacros): + if m['type'] == str(1): + # Remove the value as the api doesn't return it + # this will allow us to only update usermacros that don't exist + m.pop('value') + macros_filtered.append(m) + if host['macros'] == self.usermacros or host['macros'] == macros_filtered: + self.logger.debug(f"Host {self.name}: usermacros in-sync.") + else: + self.logger.warning(f"Host {self.name}: usermacros OUT of sync.") + self.updateZabbixHost(macros=self.usermacros) + # If only 1 interface has been found # pylint: disable=too-many-nested-blocks if len(host['interfaces']) == 1: diff --git a/modules/usermacros.py b/modules/usermacros.py index 9f53760..19d85a6 100644 --- a/modules/usermacros.py +++ b/modules/usermacros.py @@ -2,11 +2,11 @@ """ All of the Zabbix Usermacro related configuration """ +from re import match from logging import getLogger from zabbix_utils import APIRequestError from modules.exceptions import UsermacroError - from pprint import pprint try: @@ -37,11 +37,54 @@ class ZabbixUsermacros(): return self.__repr__() def _setConfig(self): - if str(usermacro_sync) == "full": + if str(usermacro_sync).lower() == "full": self.sync = True self.force_sync = True elif usermacro_sync: self.sync = True return True - - + + def validate_macro(self, macro_name): + pattern = '\{\$[A-Z0-9\._]*(\:.*)?\}' + return match(pattern, macro_name) + + def render_macro(self, macro_name, macro_properties): + macro={} + macrotypes={'text': 0, 'secret': 1, 'vault': 2} + if self.validate_macro(macro_name): + macro['macro'] = str(macro_name) + if isinstance(macro_properties, dict): + if not 'value' in macro_properties: + self.logger.error(f'Usermacro {macro_name} has no value, skipping.') + return False + else: + macro['value'] = macro_properties['value'] + + if 'type' in macro_properties and macro_properties['type'].lower() in macrotypes: + macro['type'] = str(macrotypes[macro_properties['type']]) + else: + macro['type'] = str(0) + + if 'description' in macro_properties and isinstance(macro_properties['description'], str): + macro['description'] = macro_properties['description'] + else: + macro['description'] = "" + + elif isinstance(macro_properties, str): + macro['value'] = macro_properties + macro['type'] = str(0) + macro['description'] = "" + else: + self.logger.error(f'Usermacro {macro_name} is not a valid usermacro name, skipping.') + return False + return macro + + def generate(self): + macros=[] + if "zabbix" in self.context and "usermacros" in self.context['zabbix']: + for macro, properties in self.context['zabbix']['usermacros'].items(): + m = self.render_macro(macro, properties) + pprint(m) + if m: + macros.append(m) + return macros From 1b831a2d399d834a71796160bf9d6c475238d38a Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Fri, 14 Feb 2025 09:46:55 +0100 Subject: [PATCH 12/22] Moved Inventory mapping logic to tools module --- modules/device.py | 52 ++++++++++++++++++++++++----------------------- modules/tools.py | 36 ++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 25 deletions(-) diff --git a/modules/device.py b/modules/device.py index 1a9e452..380bb56 100644 --- a/modules/device.py +++ b/modules/device.py @@ -13,6 +13,7 @@ from modules.exceptions import (SyncInventoryError, TemplateError, SyncExternalE from modules.interface import ZabbixInterface from modules.usermacros import ZabbixUsermacros from modules.hostgroups import Hostgroup +from modules.tools import field_mapper from pprint import pprint try: @@ -195,31 +196,32 @@ class PhysicalDevice(): self.inventory = {} if inventory_sync and self.inventory_mode in [0,1]: self.logger.debug(f"Host {self.name}: Starting inventory mapper") - # Let's build an inventory dict for each property in the inventory_map - for nb_inv_field, zbx_inv_field in self._inventory_map().items(): - field_list = nb_inv_field.split("/") # convert str to list based on delimiter - # start at the base of the dict... - value = nbdevice - # ... and step through the dict till we find the needed value - for item in field_list: - value = value[item] if value else None - # Check if the result is usable and expected - # We want to apply any int or float 0 values, - # even if python thinks those are empty. - if ((value and isinstance(value, int | float | str )) or - (isinstance(value, int | float) and int(value) ==0)): - self.inventory[zbx_inv_field] = str(value) - elif not value: - # empty value should just be an empty string for API compatibility - self.logger.debug(f"Host {self.name}: NetBox inventory lookup for " - f"'{nb_inv_field}' returned an empty value") - self.inventory[zbx_inv_field] = "" - else: - # Value is not a string or numeral, probably not what the user expected. - self.logger.error(f"Host {self.name}: Inventory lookup for '{nb_inv_field}'" - " returned an unexpected type: it will be skipped.") - self.logger.debug(f"Host {self.name}: Inventory mapping complete. " - f"Mapped {len(list(filter(None, self.inventory.values())))} field(s)") + self.inventory = field_mapper(self.name, self._inventory_map(), nbdevice, self.logger) +# # Let's build an inventory dict for each property in the inventory_map +# for nb_inv_field, zbx_inv_field in self._inventory_map().items(): +# field_list = nb_inv_field.split("/") # convert str to list based on delimiter +# # start at the base of the dict... +# value = nbdevice +# # ... and step through the dict till we find the needed value +# for item in field_list: +# value = value[item] if value else None +# # Check if the result is usable and expected +# # We want to apply any int or float 0 values, +# # even if python thinks those are empty. +# if ((value and isinstance(value, int | float | str )) or +# (isinstance(value, int | float) and int(value) ==0)): +# self.inventory[zbx_inv_field] = str(value) +# elif not value: +# # empty value should just be an empty string for API compatibility +# self.logger.debug(f"Host {self.name}: NetBox inventory lookup for " +# f"'{nb_inv_field}' returned an empty value") +# self.inventory[zbx_inv_field] = "" +# else: +# # Value is not a string or numeral, probably not what the user expected. +# self.logger.error(f"Host {self.name}: Inventory lookup for '{nb_inv_field}'" +# " returned an unexpected type: it will be skipped.") +# self.logger.debug(f"Host {self.name}: Inventory mapping complete. " +# f"Mapped {len(list(filter(None, self.inventory.values())))} field(s)") return True def isCluster(self): diff --git a/modules/tools.py b/modules/tools.py index f722524..1f197b6 100644 --- a/modules/tools.py +++ b/modules/tools.py @@ -1,3 +1,5 @@ +from logging import getLogger + """A collection of tools used by several classes""" def convert_recordset(recordset): """ Converts netbox RedcordSet to list of dicts. """ @@ -42,3 +44,37 @@ def proxy_prepper(proxy_list, proxy_group_list): group["monitored_by"] = 2 output.append(group) return output + + +def field_mapper(host, mapper, nbdevice, logger): + """ + Maps NetBox field data to Zabbix properties. + Used for Inventory, Usermacros and Tag mappings. + """ + data={} + # Let's build an dict for each property in the map + for nb_field, zbx_field in mapper.items(): + field_list = nb_field.split("/") # convert str to list based on delimiter + # start at the base of the dict... + value = nbdevice + # ... and step through the dict till we find the needed value + for item in field_list: + value = value[item] if value else None + # Check if the result is usable and expected + # We want to apply any int or float 0 values, + # even if python thinks those are empty. + if ((value and isinstance(value, int | float | str )) or + (isinstance(value, int | float) and int(value) ==0)): + data[zbx_field] = str(value) + elif not value: + # empty value should just be an empty string for API compatibility + logger.debug(f"Host {host}: NetBox lookup for " + f"'{nb_field}' returned an empty value") + data[zbx_field] = "" + else: + # Value is not a string or numeral, probably not what the user expected. + logger.error(f"Host {host}: Lookup for '{nb_field}'" + " returned an unexpected type: it will be skipped.") + logger.debug(f"Host {host}: Field mapping complete." + f"Mapped {len(list(filter(None, data.values())))} field(s)") + return data From eea7df660a70006d7761ff13715dcb1b8bb339e1 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Fri, 14 Feb 2025 15:18:26 +0100 Subject: [PATCH 13/22] Full usermacro support --- modules/device.py | 72 ++++++++++++------------------------ modules/tools.py | 8 ++-- modules/usermacros.py | 75 ++++++++++++++++++++++---------------- modules/virtual_machine.py | 2 - netbox_zabbix_sync.py | 4 +- 5 files changed, 73 insertions(+), 88 deletions(-) diff --git a/modules/device.py b/modules/device.py index 380bb56..76f07cf 100644 --- a/modules/device.py +++ b/modules/device.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# pylint: disable=invalid-name, logging-not-lazy, too-many-locals, logging-fstring-interpolation, too-many-lines +# pylint: disable=invalid-name, logging-not-lazy, too-many-locals, logging-fstring-interpolation, too-many-lines, too-many-public-methods """ Device specific handeling for NetBox to Zabbix """ @@ -9,12 +9,11 @@ from copy import deepcopy from logging import getLogger from zabbix_utils import APIRequestError from modules.exceptions import (SyncInventoryError, TemplateError, SyncExternalError, - InterfaceConfigError, JournalError, UsermacroError) + InterfaceConfigError, JournalError) from modules.interface import ZabbixInterface from modules.usermacros import ZabbixUsermacros from modules.hostgroups import Hostgroup from modules.tools import field_mapper -from pprint import pprint try: from config import ( @@ -73,7 +72,7 @@ class PhysicalDevice(): def _inventory_map(self): """ Use device inventory maps """ return device_inventory_map - + def _usermacro_map(self): """ Use device inventory maps """ return device_usermacro_map @@ -197,31 +196,6 @@ class PhysicalDevice(): if inventory_sync and self.inventory_mode in [0,1]: self.logger.debug(f"Host {self.name}: Starting inventory mapper") self.inventory = field_mapper(self.name, self._inventory_map(), nbdevice, self.logger) -# # Let's build an inventory dict for each property in the inventory_map -# for nb_inv_field, zbx_inv_field in self._inventory_map().items(): -# field_list = nb_inv_field.split("/") # convert str to list based on delimiter -# # start at the base of the dict... -# value = nbdevice -# # ... and step through the dict till we find the needed value -# for item in field_list: -# value = value[item] if value else None -# # Check if the result is usable and expected -# # We want to apply any int or float 0 values, -# # even if python thinks those are empty. -# if ((value and isinstance(value, int | float | str )) or -# (isinstance(value, int | float) and int(value) ==0)): -# self.inventory[zbx_inv_field] = str(value) -# elif not value: -# # empty value should just be an empty string for API compatibility -# self.logger.debug(f"Host {self.name}: NetBox inventory lookup for " -# f"'{nb_inv_field}' returned an empty value") -# self.inventory[zbx_inv_field] = "" -# else: -# # Value is not a string or numeral, probably not what the user expected. -# self.logger.error(f"Host {self.name}: Inventory lookup for '{nb_inv_field}'" -# " returned an unexpected type: it will be skipped.") -# self.logger.debug(f"Host {self.name}: Inventory mapping complete. " -# f"Mapped {len(list(filter(None, self.inventory.values())))} field(s)") return True def isCluster(self): @@ -375,14 +349,19 @@ class PhysicalDevice(): self.logger.warning(message) raise SyncInventoryError(message) from e - def setUsermacros(self): - # Initiate Usermacros class - macros = ZabbixUsermacros(self.nb.config_context, self._usermacro_map()) - if macros.sync == False: - return [] - else: - return macros.generate() - + def set_usermacros(self): + """ + Generates Usermacros + """ + macros = ZabbixUsermacros(self.nb, self._usermacro_map(), + usermacro_sync, logger=self.logger, + host=self.name) + if macros.sync is False: + self.usermacros = [] + + self.usermacros = macros.generate() + return True + def setProxy(self, proxy_list): """ Sets proxy or proxy group if this @@ -443,8 +422,6 @@ class PhysicalDevice(): groups = [{"groupid": self.group_id}] # Set Zabbix proxy if defined self.setProxy(proxies) - # Set usermacros - self.usermacros = self.setUsermacros() # Set basic data for host creation create_data = {"host": self.name, "name": self.visible_name, @@ -571,7 +548,7 @@ class PhysicalDevice(): selectHostGroups=["groupid"], selectParentTemplates=["templateid"], selectInventory=list(self._inventory_map().values()), - selectMacros=["macro","value","type","description"] + selectMacros=["macro","value","type","description"] ) if len(host) > 1: e = (f"Got {len(host)} results for Zabbix hosts " @@ -621,9 +598,9 @@ class PhysicalDevice(): if group["groupid"] == self.group_id: self.logger.debug(f"Host {self.name}: hostgroup in-sync.") break - else: - self.logger.warning(f"Host {self.name}: hostgroup OUT of sync.") - self.updateZabbixHost(groups={'groupid': self.group_id}) + else: + self.logger.warning(f"Host {self.name}: hostgroup OUT of sync.") + self.updateZabbixHost(groups={'groupid': self.group_id}) if int(host["status"]) == self.zabbix_state: self.logger.debug(f"Host {self.name}: status in-sync.") @@ -697,14 +674,13 @@ class PhysicalDevice(): # Check host usermacros if usermacro_sync: macros_filtered = [] - self.usermacros = self.setUsermacros() # Do not re-sync secret usermacros unless sync is set to 'full' - if not str(usermacro_sync).lower() == "full": + if str(usermacro_sync).lower() != "full": for m in deepcopy(self.usermacros): if m['type'] == str(1): - # Remove the value as the api doesn't return it - # this will allow us to only update usermacros that don't exist - m.pop('value') + # Remove the value as the api doesn't return it + # this will allow us to only update usermacros that don't exist + m.pop('value') macros_filtered.append(m) if host['macros'] == self.usermacros or host['macros'] == macros_filtered: self.logger.debug(f"Host {self.name}: usermacros in-sync.") diff --git a/modules/tools.py b/modules/tools.py index 1f197b6..f32e802 100644 --- a/modules/tools.py +++ b/modules/tools.py @@ -1,6 +1,5 @@ -from logging import getLogger - """A collection of tools used by several classes""" + def convert_recordset(recordset): """ Converts netbox RedcordSet to list of dicts. """ recordlist = [] @@ -45,7 +44,6 @@ def proxy_prepper(proxy_list, proxy_group_list): output.append(group) return output - def field_mapper(host, mapper, nbdevice, logger): """ Maps NetBox field data to Zabbix properties. @@ -75,6 +73,6 @@ def field_mapper(host, mapper, nbdevice, logger): # Value is not a string or numeral, probably not what the user expected. logger.error(f"Host {host}: Lookup for '{nb_field}'" " returned an unexpected type: it will be skipped.") - logger.debug(f"Host {host}: Field mapping complete." - f"Mapped {len(list(filter(None, data.values())))} field(s)") + logger.debug(f"Host {host}: Field mapping complete. " + f"Mapped {len(list(filter(None, data.values())))} field(s)") return data diff --git a/modules/usermacros.py b/modules/usermacros.py index 19d85a6..71efbde 100644 --- a/modules/usermacros.py +++ b/modules/usermacros.py @@ -1,71 +1,71 @@ #!/usr/bin/env python3 +# pylint: disable=too-many-instance-attributes, too-many-arguments, too-many-positional-arguments, logging-fstring-interpolation """ All of the Zabbix Usermacro related configuration """ from re import match from logging import getLogger -from zabbix_utils import APIRequestError -from modules.exceptions import UsermacroError - -from pprint import pprint - -try: - from config import ( - usermacro_sync, - ) -except ModuleNotFoundError: - print("Configuration file config.py not found in main directory." - "Please create the file or rename the config.py.example file to config.py.") - sys.exit(0) +from modules.tools import field_mapper class ZabbixUsermacros(): """Class that represents a Zabbix interface.""" - def __init__(self, context, usermacro_map, logger=None): - self.context = context + def __init__(self, nb, usermacro_map, usermacro_sync, logger=None, host=None): + self.nb = nb + self.name = host if host else nb.name self.usermacro_map = usermacro_map self.logger = logger if logger else getLogger(__name__) self.usermacros = {} + self.usermacro_sync = usermacro_sync self.sync = False self.force_sync = False - self._setConfig() + self._set_config() def __repr__(self): return self.name - + def __str__(self): return self.__repr__() - def _setConfig(self): - if str(usermacro_sync).lower() == "full": + def _set_config(self): + """ + Setup class + """ + if str(self.usermacro_sync).lower() == "full": self.sync = True self.force_sync = True - elif usermacro_sync: + elif self.usermacro_sync: self.sync = True return True def validate_macro(self, macro_name): - pattern = '\{\$[A-Z0-9\._]*(\:.*)?\}' + """ + Validates usermacro name + """ + pattern = r'\{\$[A-Z0-9\._]*(\:.*)?\}' return match(pattern, macro_name) def render_macro(self, macro_name, macro_properties): + """ + Renders a full usermacro from partial input + """ macro={} macrotypes={'text': 0, 'secret': 1, 'vault': 2} if self.validate_macro(macro_name): macro['macro'] = str(macro_name) if isinstance(macro_properties, dict): if not 'value' in macro_properties: - self.logger.error(f'Usermacro {macro_name} has no value, skipping.') - return False - else: - macro['value'] = macro_properties['value'] + self.logger.error(f'Usermacro {macro_name} has no value, skipping.') + return False + macro['value'] = macro_properties['value'] if 'type' in macro_properties and macro_properties['type'].lower() in macrotypes: macro['type'] = str(macrotypes[macro_properties['type']]) else: macro['type'] = str(0) - if 'description' in macro_properties and isinstance(macro_properties['description'], str): + if ('description' in macro_properties and + isinstance(macro_properties['description'], str)): macro['description'] = macro_properties['description'] else: macro['description'] = "" @@ -78,13 +78,24 @@ class ZabbixUsermacros(): self.logger.error(f'Usermacro {macro_name} is not a valid usermacro name, skipping.') return False return macro - + def generate(self): + """ + Generate full set of Usermacros + """ macros=[] - if "zabbix" in self.context and "usermacros" in self.context['zabbix']: - for macro, properties in self.context['zabbix']['usermacros'].items(): - m = self.render_macro(macro, properties) - pprint(m) + # Parse the field mapper for usermacros + if self.usermacro_map: + self.logger.debug(f"Host {self.nb.name}: Starting usermacro mapper") + field_macros = field_mapper(self.nb.name, self.usermacro_map, self.nb, self.logger) + for macro, value in field_macros.items(): + m = self.render_macro(macro, value) if m: - macros.append(m) + macros.append(m) + # Parse NetBox config context for usermacros + if "zabbix" in self.nb.config_context and "usermacros" in self.nb.config_context['zabbix']: + for macro, properties in self.nb.config_context['zabbix']['usermacros'].items(): + m = self.render_macro(macro, properties) + if m: + macros.append(m) return macros diff --git a/modules/virtual_machine.py b/modules/virtual_machine.py index b8fa1a1..d95bfc1 100644 --- a/modules/virtual_machine.py +++ b/modules/virtual_machine.py @@ -9,8 +9,6 @@ from modules.interface import ZabbixInterface from modules.exceptions import TemplateError, InterfaceConfigError, SyncInventoryError try: from config import ( - inventory_sync, - inventory_mode, vm_inventory_map, traverse_site_groups, traverse_regions diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index 3eaea3f..5498edc 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -161,7 +161,7 @@ def main(arguments): try: vm = VirtualMachine(nb_vm, zabbix, netbox_journals, nb_version, create_journal, logger) - logger.debug(f"Host {vm.name}: started operations on VM.") + logger.debug(f"Host {vm.name}: Started operations on VM.") vm.set_vm_template() # Check if a valid template has been found for this VM. if not vm.zbx_template_names: @@ -172,6 +172,7 @@ def main(arguments): if not vm.hostgroup: continue vm.set_inventory(nb_vm) + vm.set_usermacros() # Checks if device is in cleanup state if vm.status in zabbix_device_removal: if vm.zabbix_id: @@ -225,6 +226,7 @@ def main(arguments): if not device.hostgroup: continue device.set_inventory(nb_device) + device.set_usermacros() # Checks if device is part of cluster. # Requires clustering variable if device.isCluster() and clustering: From 72558d3825260677c5dee526c8dc69eee014b770 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Fri, 14 Feb 2025 16:35:40 +0100 Subject: [PATCH 14/22] Updated docs for VM inventory --- README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 959a2fb..bdd850d 100644 --- a/README.md +++ b/README.md @@ -292,15 +292,18 @@ You can set the inventory mode to "disabled", "manual" or "automatic" with the [Zabbix Manual](https://www.zabbix.com/documentation/current/en/manual/config/hosts/inventory#building-inventory) for more information about the modes. -Use the `inventory_map` variable to map which NetBox properties are used in +Use the `device_inventory_map` variable to map which NetBox properties are used in which Zabbix Inventory fields. For nested properties, you can use the '/' seperator. For example, the following map will assign the custom field 'mycustomfield' to the 'alias' Zabbix inventory field: +For Virtual Machines, use `vm_inventory_map`. + ``` inventory_sync = True inventory_mode = "manual" -inventory_map = { "custom_fields/mycustomfield/name": "alias"} +device_inventory_map = {"custom_fields/mycustomfield/name": "alias"} +vm_inventory_map = {"custom_fields/mycustomfield/name": "alias"} ``` See `config.py.example` for an extensive example map. Any Zabix Inventory fields From 3d4e7803ccf35a7e91ed61c86564f9c730f02e5c Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Mon, 17 Feb 2025 12:48:26 +0100 Subject: [PATCH 15/22] Implemented vm_usermacro_map --- modules/virtual_machine.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/virtual_machine.py b/modules/virtual_machine.py index d95bfc1..5afdb18 100644 --- a/modules/virtual_machine.py +++ b/modules/virtual_machine.py @@ -10,6 +10,7 @@ from modules.exceptions import TemplateError, InterfaceConfigError, SyncInventor try: from config import ( vm_inventory_map, + vm_usermacro_map, traverse_site_groups, traverse_regions ) @@ -29,6 +30,10 @@ class VirtualMachine(PhysicalDevice): """ use VM inventory maps """ return vm_inventory_map + def _usermacro_map(self): + """ use VM inventory maps """ + return vm_usermacro_map + def set_hostgroup(self, hg_format, nb_site_groups, nb_regions): """Set the hostgroup for this device""" # Create new Hostgroup instance From f9453cc23cc2b03679155a70fe09abd2c72303cb Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Mon, 17 Feb 2025 12:54:11 +0100 Subject: [PATCH 16/22] Updated documentation for usermacro support --- README.md | 138 +++++++++++++++++++++++++++++++++++++++++----- config.py.example | 20 +++++++ 2 files changed, 143 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index bdd850d..612588f 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ A script to create, update and delete Zabbix hosts using NetBox device objects. To pull the latest stable version to your local cache, use the following docker pull command: -``` +```bash docker pull ghcr.io/thenetworkguy/netbox-zabbix-sync:main ``` @@ -15,7 +15,7 @@ Make sure to specify the needed environment variables for the script to work (see [here](#set-environment-variables)) on the command line or use an [env file](https://docs.docker.com/reference/cli/docker/container/run/#env). -``` +```bash docker run -d -t -i -e ZABBIX_HOST='https://zabbix.local' \ -e ZABBIX_TOKEN='othersecrettoken' \ -e NETBOX_HOST='https://netbox.local' \ @@ -30,7 +30,7 @@ The image uses the default `config.py` for it's configuration, you can use a volume mount in the docker run command to override with your own config file if needed (see [config file](#config-file)): -``` +```bash docker run -d -t -i -v $(pwd)/config.py:/opt/netbox-zabbix/config.py ... ``` @@ -38,7 +38,7 @@ docker run -d -t -i -v $(pwd)/config.py:/opt/netbox-zabbix/config.py ... ### Cloning the repository -``` +```bash git clone https://github.com/TheNetworkGuy/netbox-zabbix-sync.git ``` @@ -66,7 +66,7 @@ cp config.py.example config.py Set the following environment variables: -``` +```bash ZABBIX_HOST="https://zabbix.local" ZABBIX_USER="username" ZABBIX_PASS="Password" @@ -77,7 +77,7 @@ NETBOX_TOKEN="secrettoken" Or, you can use a Zabbix API token to login instead of using a username and password. In that case `ZABBIX_USER` and `ZABBIX_PASS` will be ignored. -``` +```bash ZABBIX_TOKEN=othersecrettoken ``` @@ -183,9 +183,9 @@ used: | cluster | VM cluster name | | cluster_type | VM cluster type | -You can specify the value sperated by a "/" like so: +You can specify the value seperated by a "/" like so: -``` +```python hostgroup_format = "tenant/site/dev_location/role" ``` @@ -232,7 +232,7 @@ have a relationship with a tenant. - Device_role: PDU - Site: HQ-AMS -``` +```python hostgroup_format = "site/tenant/device_role" ``` @@ -245,7 +245,7 @@ generated for both hosts: The same logic applies to custom fields being used in the HG format: -``` +```python hostgroup_format = "site/mycustomfieldname" ``` @@ -299,7 +299,7 @@ seperator. For example, the following map will assign the custom field For Virtual Machines, use `vm_inventory_map`. -``` +```python inventory_sync = True inventory_mode = "manual" device_inventory_map = {"custom_fields/mycustomfield/name": "alias"} @@ -324,14 +324,14 @@ sticking to the custom field. You can change the behaviour in the config file. By default this setting is false but you can set it to true to use config context: -``` +```python templates_config_context = True ``` After that make sure that for each host there is at least one template defined in the config context in this format: -``` +```json { "zabbix": { "templates": [ @@ -349,10 +349,114 @@ added benefit of overwriting the template should a device in NetBox have a device specific context defined. In this case the device specific context template(s) will take priority over the device type custom field template. -``` +```python templates_config_context_overrule = True ``` +### Usermacros + +You can choose to use NetBox as a source for Host usermacros by +enabling the following option in the configuration file: + +``` +usermacro_sync = True +``` + +Please be advised that enabling this option will _clear_ any usermacros +manually set on the managed hosts and override them with the usermacros +from NetBox. + +There are two NetBox sources that can be used to populate usermacros: + +1. NetBox config context +2. NetBox fields + +#### Config context + +By defining a dictionary `usermacros` within the `zabbix` key in +config context, you can dynamically assign usermacro values based on +anything that you can target based on +[config contexts](https://netboxlabs.com/docs/netbox/en/stable/features/context-data/) +within NetBox. + +Through this method, it is possible to define the following types of usermacros: + +1. Text +2. Secret +3. Vault + +The default macro type is text if no `type` and `value` have been set. +It is also possible to create usermacros with +[context](https://www.zabbix.com/documentation/7.0/en/manual/config/macros/user_macros_context). +Examples: + +```json +{ + "zabbix": { + "usermacros": { + "{$USER_MACRO}": "test value", + "{$CONTEXT_MACRO:\"test\"}": "test value", + "{$CONTEXT_REGEX_MACRO:regex:\".*\"}": "test value", + "{$SECRET_MACRO}": { + "type": "secret", + "value": "PaSsPhRaSe" + }, + "{$VAULT_MACRO}": { + "type": "vault", + "value": "secret/vmware:password" + }, + "{$USER_MACRO2}": { + "type": "text", + "value": "another test value" + } + } + } +} + +``` + +Please be aware that secret usermacros are only synced _once_ by default. +This is the default behaviour because Zabbix API won't return the value of +secrets so the script cannot compare the values with the ones set in NetBox. + +If you update a secret usermacro value, just remove the value from the host +in Zabbix and the new value will be synced during the next run. + +Alternatively, you can set the following option in the config file: + +```python +usermacro_sync = "full" +``` + +This will force a full usermacro sync on every run on hosts that have secret usermacros set. +That way, you will know for sure the secret values are always up to date. + +Keep in mind that NetBox (and the log output of this script) will show your secrets +in plain text. If true secrecy is required, consider switching to +[vault](https://www.zabbix.com/documentation/current/en/manual/config/macros/secret_macros#vault-secret) +usermacros. + +#### Netbox Fields + +To use NetBox fields as a source for usermacros, you will need to set up usermacro maps +for devices and/or virtual machines in the configuration file. +This method only supports `text` type usermacros. + +For example: +```python +usermacro_sync = True +device_usermacro_map = {"serial": "{$HW_SERIAL}", + "role/name": "{$DEV_ROLE}", + "url": "{$NB_URL}", + "id": "{$NB_ID}"} +vm_usermacro_map = {"memory": "{$TOTAL_MEMORY}", + "role/name": "{$DEV_ROLE}", + "url": "{$NB_URL}", + "id": "{$NB_ID}"} +``` + + + ## Permissions ### NetBox @@ -521,9 +625,13 @@ environment. For example, you could: } ``` -I would recommend using macros for sensitive data such as community strings +I would recommend using usermacros for sensitive data such as community strings since the data in NetBox is plain-text. > **_NOTE:_** Not all SNMP data is required for a working configuration. > [The following parameters are allowed](https://www.zabbix.com/documentation/current/manual/api/reference/hostinterface/object#details_tag "The following parameters are allowed")but > are not all required, depending on your environment. + + + + diff --git a/config.py.example b/config.py.example index 0a653d6..68f6fea 100644 --- a/config.py.example +++ b/config.py.example @@ -101,3 +101,23 @@ device_inventory_map = { "asset_tag": "asset_tag", vm_inventory_map = { "status/label": "deployment_status", "comments": "notes", "name": "name" } + +# To allow syncing of usermacros from NetBox, set to True. +# this will enable both field mapping and config context usermacros. +# +# If set to "full", it will force the update of secret usermacros every run. +# Please see the README.md for more information. +usermacro_sync = False + +# device usermacro_map to map NetBox fields to usermacros. +device_usermacro_map = {"serial": "{$HW_SERIAL}", + "role/name": "{$DEV_ROLE}", + "url": "{$NB_URL}", + "id": "{$NB_ID}"} + +# virtual machine usermacro_map to map NetBox fields to usermacros. +vm_usermacro_map = {"memory": "{$TOTAL_MEMORY}", + "role/name": "{$DEV_ROLE}", + "url": "{$NB_URL}", + "id": "{$NB_ID}"} + From fd70045c6d0d7b51383a6244748c97fcce0188b4 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Mon, 17 Feb 2025 12:57:57 +0100 Subject: [PATCH 17/22] Minor doc updates --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 612588f..eb0d0c3 100644 --- a/README.md +++ b/README.md @@ -358,7 +358,7 @@ templates_config_context_overrule = True You can choose to use NetBox as a source for Host usermacros by enabling the following option in the configuration file: -``` +```python usermacro_sync = True ``` @@ -388,6 +388,7 @@ Through this method, it is possible to define the following types of usermacros: The default macro type is text if no `type` and `value` have been set. It is also possible to create usermacros with [context](https://www.zabbix.com/documentation/7.0/en/manual/config/macros/user_macros_context). + Examples: ```json @@ -443,6 +444,7 @@ for devices and/or virtual machines in the configuration file. This method only supports `text` type usermacros. For example: + ```python usermacro_sync = True device_usermacro_map = {"serial": "{$HW_SERIAL}", From d65fa5b699385183731f43798a09f628a7de46c9 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Wed, 19 Feb 2025 15:56:01 +0100 Subject: [PATCH 18/22] Added tag support --- config.py.example | 30 ++++++++++ modules/device.py | 49 +++++++++++++--- modules/tags.py | 117 +++++++++++++++++++++++++++++++++++++ modules/tools.py | 11 ++++ modules/virtual_machine.py | 7 ++- netbox_zabbix_sync.py | 2 + 6 files changed, 208 insertions(+), 8 deletions(-) create mode 100644 modules/tags.py diff --git a/config.py.example b/config.py.example index 68f6fea..e4082e6 100644 --- a/config.py.example +++ b/config.py.example @@ -121,3 +121,33 @@ vm_usermacro_map = {"memory": "{$TOTAL_MEMORY}", "url": "{$NB_URL}", "id": "{$NB_ID}"} +# To sync host tags to Zabbix, set to True. +tag_sync = False + +# Setting tag_lower to True will lower capital letters ain tag names and values +# This is more inline with the Zabbix way of working with tags. +# +# You can however set this to False to ensure capital letters are synced to Zabbix tags. +tag_lower = True + +# We can sync NetBox device/VM tags to Zabbix, but as NetBox tags don't follow the key/value +# pattern, we need to specify a tag name to register the NetBox tags in Zabbix. +# +# +# +# If tag_name is set to False, we won't sync NetBox device/VM tags to Zabbix. +tag_name = 'NetBox' + +# We can choose to use 'name', 'slug' or 'display' NetBox tag properties as a value in Zabbix. +# 'name'is used by default. +tag_value = "name" + +# device tag_map to map NetBox fields to host tags. +device_tag_map = {"site/name": "site", + "rack/name": "rack", + "platform/name": "target"} + +# Virtual machine tag_map to map NetBox fields to host tags. +vm_tag_map = {"site/name": "site", + "cluster/name": "cluster", + "platform/name": "target"} diff --git a/modules/device.py b/modules/device.py index 76f07cf..4ec96b5 100644 --- a/modules/device.py +++ b/modules/device.py @@ -12,8 +12,9 @@ from modules.exceptions import (SyncInventoryError, TemplateError, SyncExternalE InterfaceConfigError, JournalError) from modules.interface import ZabbixInterface from modules.usermacros import ZabbixUsermacros +from modules.tags import ZabbixTags from modules.hostgroups import Hostgroup -from modules.tools import field_mapper +from modules.tools import field_mapper, remove_duplicates try: from config import ( @@ -24,7 +25,12 @@ try: inventory_mode, device_inventory_map, usermacro_sync, - device_usermacro_map + device_usermacro_map, + tag_sync, + tag_lower, + tag_name, + tag_value, + device_tag_map ) except ModuleNotFoundError: print("Configuration file config.py not found in main directory." @@ -60,6 +66,7 @@ class PhysicalDevice(): self.inventory_mode = -1 self.inventory = {} self.usermacros = {} + self.tags = {} self.logger = logger if logger else getLogger(__name__) self._setBasics() @@ -77,6 +84,10 @@ class PhysicalDevice(): """ Use device inventory maps """ return device_usermacro_map + def _tag_map(self): + """ Use device host tag maps """ + return device_tag_map + def _setBasics(self): """ Sets basic information like IP address. @@ -362,6 +373,21 @@ class PhysicalDevice(): self.usermacros = macros.generate() return True + + def set_tags(self): + """ + Generates Host Tags + """ + tags = ZabbixTags(self.nb, self._tag_map(), + tag_sync, tag_lower, tag_name=tag_name, + tag_value=tag_value, logger=self.logger, + host=self.name) + if tags.sync is False: + self.tags = [] + + self.tags = tags.generate() + return True + def setProxy(self, proxy_list): """ Sets proxy or proxy group if this @@ -432,7 +458,8 @@ class PhysicalDevice(): "description": description, "inventory_mode": self.inventory_mode, "inventory": self.inventory, - "macros": self.usermacros + "macros": self.usermacros, + "tags": self.tags } # If a Zabbix proxy or Zabbix Proxy group has been defined if self.zbxproxy: @@ -548,7 +575,8 @@ class PhysicalDevice(): selectHostGroups=["groupid"], selectParentTemplates=["templateid"], selectInventory=list(self._inventory_map().values()), - selectMacros=["macro","value","type","description"] + selectMacros=["macro","value","type","description"], + selectTags=["tag","value"] ) if len(host) > 1: e = (f"Got {len(host)} results for Zabbix hosts " @@ -598,9 +626,8 @@ class PhysicalDevice(): if group["groupid"] == self.group_id: self.logger.debug(f"Host {self.name}: hostgroup in-sync.") break - else: - self.logger.warning(f"Host {self.name}: hostgroup OUT of sync.") - self.updateZabbixHost(groups={'groupid': self.group_id}) + self.logger.warning(f"Host {self.name}: hostgroup OUT of sync.") + self.updateZabbixHost(groups={'groupid': self.group_id}) if int(host["status"]) == self.zabbix_state: self.logger.debug(f"Host {self.name}: status in-sync.") @@ -688,6 +715,14 @@ class PhysicalDevice(): self.logger.warning(f"Host {self.name}: usermacros OUT of sync.") self.updateZabbixHost(macros=self.usermacros) + # Check host usermacros + if tag_sync: + if remove_duplicates(host['tags'],sortkey='tag') == self.tags: + self.logger.debug(f"Host {self.name}: tags in-sync.") + else: + self.logger.warning(f"Host {self.name}: tags OUT of sync.") + self.updateZabbixHost(tags=self.tags) + # If only 1 interface has been found # pylint: disable=too-many-nested-blocks if len(host['interfaces']) == 1: diff --git a/modules/tags.py b/modules/tags.py new file mode 100644 index 0000000..4993cd3 --- /dev/null +++ b/modules/tags.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +# pylint: disable=too-many-instance-attributes, too-many-arguments, too-many-positional-arguments, logging-fstring-interpolation +""" +All of the Zabbix Usermacro related configuration +""" +from logging import getLogger +from modules.tools import field_mapper, remove_duplicates + +class ZabbixTags(): + """Class that represents a Zabbix interface.""" + + def __init__(self, nb, tag_map, tag_sync, tag_lower=True, + tag_name=None, tag_value=None, logger=None, host=None): + self.nb = nb + self.name = host if host else nb.name + self.tag_map = tag_map + self.logger = logger if logger else getLogger(__name__) + self.tags = {} + self.lower = tag_lower + self.tag_name = tag_name + self.tag_value = tag_value + self.tag_sync = tag_sync + self.sync = False + self._set_config() + + def __repr__(self): + return self.name + + def __str__(self): + return self.__repr__() + + def _set_config(self): + """ + Setup class + """ + if self.tag_sync: + self.sync = True + + return True + + def validate_tag(self, tag_name): + """ + Validates tag name + """ + if tag_name and isinstance(tag_name, str) and len(tag_name)<=256: + return True + return False + + def validate_value(self, tag_value): + """ + Validates tag value + """ + if tag_value and isinstance(tag_value, str) and len(tag_value)<=256: + return True + return False + + def render_tag(self, tag_name, tag_value): + """ + Renders a tag + """ + tag={} + if self.validate_tag(tag_name): + if self.lower: + tag['tag'] = tag_name.lower() + else: + tag['tag'] = tag_name + else: + self.logger.error(f'Tag {tag_name} is not a valid tag name, skipping.') + return False + + if self.validate_value(tag_value): + if self.lower: + tag['value'] = tag_value.lower() + else: + tag['value'] = tag_value + else: + self.logger.error(f'Tag {tag_name} has an invalid value: \'{tag_value}\', skipping.') + return False + return tag + + def generate(self): + """ + Generate full set of Usermacros + """ + # pylint: disable=too-many-branches + tags=[] + # Parse the field mapper for tags + if self.tag_map: + self.logger.debug(f"Host {self.nb.name}: Starting tag mapper") + field_tags = field_mapper(self.nb.name, self.tag_map, self.nb, self.logger) + for tag, value in field_tags.items(): + t = self.render_tag(tag, value) + if t: + tags.append(t) + + # Parse NetBox config context for tags + if ("zabbix" in self.nb.config_context and "tags" in self.nb.config_context['zabbix'] + and isinstance(self.nb.config_context['zabbix']['tags'], list)): + for tag in self.nb.config_context['zabbix']['tags']: + if isinstance(tag, dict): + for tagname, value in tag.items(): + t = self.render_tag(tagname, value) + if t: + tags.append(t) + + # Pull in NetBox device tags if tag_name is set + if self.tag_name and isinstance(self.tag_name, str): + for tag in self.nb.tags: + if self.tag_value.lower() in ['display', 'name', 'slug']: + value = tag[self.tag_value] + else: + value = tag['name'] + t = self.render_tag(self.tag_name, value) + if t: + tags.append(t) + + return remove_duplicates(tags, sortkey='tag') diff --git a/modules/tools.py b/modules/tools.py index f32e802..8d658a3 100644 --- a/modules/tools.py +++ b/modules/tools.py @@ -76,3 +76,14 @@ def field_mapper(host, mapper, nbdevice, logger): logger.debug(f"Host {host}: Field mapping complete. " f"Mapped {len(list(filter(None, data.values())))} field(s)") return data + +def remove_duplicates(input_list, sortkey=None): + """ + Removes duplicate entries from a list and sorts the list + """ + output_list = [] + if isinstance(input_list, list): + output_list = [dict(t) for t in {tuple(d.items()) for d in input_list}] + if sortkey and isinstance(sortkey, str): + output_list.sort(key=lambda x: x[sortkey]) + return output_list diff --git a/modules/virtual_machine.py b/modules/virtual_machine.py index 5afdb18..273f9e7 100644 --- a/modules/virtual_machine.py +++ b/modules/virtual_machine.py @@ -11,6 +11,7 @@ try: from config import ( vm_inventory_map, vm_usermacro_map, + vm_tag_map, traverse_site_groups, traverse_regions ) @@ -31,9 +32,13 @@ class VirtualMachine(PhysicalDevice): return vm_inventory_map def _usermacro_map(self): - """ use VM inventory maps """ + """ use VM usermacro maps """ return vm_usermacro_map + def _tag_map(self): + """ use VM tag maps """ + return vm_tag_map + def set_hostgroup(self, hg_format, nb_site_groups, nb_regions): """Set the hostgroup for this device""" # Create new Hostgroup instance diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index 5498edc..04a4e07 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -173,6 +173,7 @@ def main(arguments): continue vm.set_inventory(nb_vm) vm.set_usermacros() + vm.set_tags() # Checks if device is in cleanup state if vm.status in zabbix_device_removal: if vm.zabbix_id: @@ -227,6 +228,7 @@ def main(arguments): continue device.set_inventory(nb_device) device.set_usermacros() + device.set_tags() # Checks if device is part of cluster. # Requires clustering variable if device.isCluster() and clustering: From 523393308d6f1e4dd87ea395d73e055d9a5ad055 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Wed, 19 Feb 2025 16:25:11 +0100 Subject: [PATCH 19/22] Updated docs --- README.md | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/README.md b/README.md index eb0d0c3..d533ec6 100644 --- a/README.md +++ b/README.md @@ -353,6 +353,86 @@ template(s) will take priority over the device type custom field template. templates_config_context_overrule = True ``` +### Tags + +This script can sync host tags to your Zabbix hosts for use in filtering, +SLA calculations and event correlation. + +Tags can be synced from the following sources: + +1. NetBox device/vm tags +2. NetBox config ontext +3. NetBox fields + +Syncing tags will override any tags that were set manually on the host, +making NetBox the single source-of-truth for managing tags. + +To enable syncing, turn on tag_sync in the config file. +By default, this script will modify tag names and tag values to lowercase. +You can change this behaviour by setting tag_lower to False. + +```python +tag_sync = True +tag_lower = True +``` + +#### Device tags + +As NetBox doesn't follow the tag/value pattern for tags, we will need a tag +name set to register the netwbox tags. + +By default the tag name is "NetBox", but you can change this to whatever you want. +The value for the tag can be choosen from 'name', 'display' or 'slug'. + +```python +tag_name = 'NetBox' +tag_value = 'name' +``` + +#### Config context + +You can supply custom tags via config context by adding the following: + +```json +{ + "zabbix": { + "tags": [ + { + "MyTagName": "MyTagValue" + }, + { + "environment": "production" + } + ], + } +} +``` + +This will allow you to assign tags based on the config context rules. + +#### NetBox Field + +NetBox field can also be used as input for tags, just like inventory and usermacros. +To enable syncing from fields, make sure to configure a `device_tag_map` and/or a `vm_tag_map`. + +```python +device_tag_map = {"site/name": "site", + "rack/name": "rack", + "platform/name": "target"} + +vm_tag_map = {"site/name": "site", + "cluster/name": "cluster", + "platform/name": "target"} +``` + +To turn off field syncing, set the maps to empty dictionaries: + +```python +device_tag_map = {} +vm_tag_map = {} +``` + + ### Usermacros You can choose to use NetBox as a source for Host usermacros by From 593c8707afeb65d85929271e81c54a24fb638f2d Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Thu, 20 Feb 2025 11:01:04 +0100 Subject: [PATCH 20/22] New publish-image workflow Should remove the dependency on PAT --- .github/workflows/publish-image.yml | 83 +++++++++++++++-------------- 1 file changed, 44 insertions(+), 39 deletions(-) diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml index e9e6421..bf87bf4 100644 --- a/.github/workflows/publish-image.yml +++ b/.github/workflows/publish-image.yml @@ -1,46 +1,51 @@ -name: Publish Docker image to GHCR on a new version +name: Build and Push Docker Image + +permissions: + contents: read + packages: write on: - push: - branches: - - main - - dockertest -# tags: -# - [0-9]+.* - -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} + release: + types: [published] + pull_request: + types: [opened, synchronize] jobs: - test_quality: - uses: ./.github/workflows/quality.yml - build_and_publish: + build: runs-on: ubuntu-latest steps: - - name: Checkout sources - uses: actions/checkout@v4 - - name: Log in to the container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GHCR_PAT }} - - name: Extract metadata (tags, labels) - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tags: | - type=semver,pattern={{ version }} - type=ref,event=branch - type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} - type=sha - - name: Build and push Docker image - uses: docker/build-push-action@v5 - with: - context: . - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 + + - name: Login to GitHub Container Registry + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 + with: + images: ghcr.io/${{ github.repository }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + + - name: Build and push Docker image + uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 + with: + context: . + file: ./Dockerfile + push: true + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + annotations: | + index:org.opencontainers.image.description=Python script to synchronise NetBox devices to Zabbix. From 733df33b7102012772e0c6b4fc30239653af6afa Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Thu, 20 Feb 2025 11:02:43 +0100 Subject: [PATCH 21/22] added step to run linting tests --- .github/workflows/publish-image.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml index bf87bf4..615b784 100644 --- a/.github/workflows/publish-image.yml +++ b/.github/workflows/publish-image.yml @@ -11,6 +11,8 @@ on: types: [opened, synchronize] jobs: + test_quality: + uses: ./.github/workflows/quality.yml build: runs-on: ubuntu-latest steps: From 825d788cfe1c30a35f2c7201e95b0c6f26808490 Mon Sep 17 00:00:00 2001 From: Raymond Kuiper Date: Thu, 20 Feb 2025 11:42:25 +0100 Subject: [PATCH 22/22] Update Dockerfile --- Dockerfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Dockerfile b/Dockerfile index fa8d9c4..0551bd1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,12 @@ # syntax=docker/dockerfile:1 FROM python:3.12-alpine +LABEL org.opencontainers.image.source=https://github.com/TheNetworkGuy/netbox-zabbix-sync +LABEL org.opencontainers.image.title="NetBox-Zabbix-Sync" +LABEL org.opencontainers.image.description="Python script to synchronise NetBox devices to Zabbix." +LABEL org.opencontainers.image.documentation=https://github.com/TheNetworkGuy/netbox-zabbix-sync/ +LABEL org.opencontainers.image.licenses=MIT +LABEL org.opencontainers.image.authors="Twan Kamans" + RUN mkdir -p /opt/netbox-zabbix COPY . /opt/netbox-zabbix WORKDIR /opt/netbox-zabbix