From 9f29d2b27b00f1eb3ea61c705cf1fd6f209f7bb3 Mon Sep 17 00:00:00 2001 From: TheNetworkGuy Date: Fri, 25 Oct 2024 18:46:20 +0200 Subject: [PATCH] Added basic VM support --- config.py.example | 7 +++ modules/device.py | 108 ++++++++++++++++++++------------------ modules/hostgroups.py | 4 +- modules/virtualMachine.py | 54 ++++++++++++------- netbox_zabbix_sync.py | 62 +++++++++++++++++++--- 5 files changed, 157 insertions(+), 78 deletions(-) diff --git a/config.py.example b/config.py.example index 264bb5b..ca171cf 100644 --- a/config.py.example +++ b/config.py.example @@ -21,6 +21,13 @@ create_hostgroups = True ## Create journal entries create_journal = False +## Virtual machine sync +# Set sync_vms to True in order to use this new feature +# Use the hostgroup vm_hostgroup_format mapper for specific +# hostgroup atributes of VM's such as cluster_type and cluster +sync_vms = False +vm_hostgroup_format = "cluster_type/cluster/role" + ## Proxy Sync # Set to true to enable removal of proxy's under hosts. Use with caution and make sure that you specified # all the required proxy's in the device config context before enabeling this option. diff --git a/modules/device.py b/modules/device.py index 07319ff..e26c7fe 100644 --- a/modules/device.py +++ b/modules/device.py @@ -24,7 +24,7 @@ except ModuleNotFoundError: "Please create the file or rename the config.py.example file to config.py.") sys.exit(0) -class NetworkDevice(): +class PhysicalDevice(): # pylint: disable=too-many-instance-attributes, too-many-arguments """ Represents Network device. @@ -55,6 +55,12 @@ class NetworkDevice(): self.logger = logger if logger else getLogger(__name__) self._setBasics() + def __repr__(self): + return self.name + + def __str__(self): + return self.__repr__() + def _setBasics(self): """ Sets basic information like IP address. @@ -64,7 +70,7 @@ class NetworkDevice(): self.cidr = self.nb.primary_ip.address self.ip = self.cidr.split("/")[0] else: - e = f"Device {self.name}: no primary IP." + e = f"Host {self.name}: no primary IP." self.logger.info(e) raise SyncInventoryError(e) @@ -72,7 +78,7 @@ class NetworkDevice(): if device_cf in self.nb.custom_fields: self.zabbix_id = self.nb.custom_fields[device_cf] else: - e = f"Device {self.name}: Custom field {device_cf} not present" + e = f"Host {self.name}: Custom field {device_cf} not present" self.logger.warning(e) raise SyncInventoryError(e) @@ -83,7 +89,7 @@ class NetworkDevice(): self.name = f"NETBOX_ID{self.id}" self.visible_name = self.nb.name self.use_visible_name = True - self.logger.info(f"Device {self.visible_name} contains special characters. " + self.logger.info(f"Host {self.visible_name} contains special characters. " f"Using {self.name} as name for the Netbox object " f"and using {self.visible_name} as visible name in Zabbix.") else: @@ -153,7 +159,7 @@ class NetworkDevice(): # Set inventory mode. Default is disabled (see class init function). if inventory_mode == "disabled": if inventory_sync: - self.logger.error(f"Device {self.name}: Unable to map Netbox inventory to Zabbix. " + self.logger.error(f"Host {self.name}: Unable to map Netbox inventory to Zabbix. " "Inventory sync is enabled in config but inventory mode is disabled.") return True if inventory_mode == "manual": @@ -161,12 +167,12 @@ class NetworkDevice(): elif inventory_mode == "automatic": self.inventory_mode = 1 else: - self.logger.error(f"Device {self.name}: Specified value for inventory mode in" + self.logger.error(f"Host {self.name}: Specified value for inventory mode in" f" config is not valid. Got value {inventory_mode}") return False self.inventory = {} if inventory_sync and self.inventory_mode in [0,1]: - self.logger.debug(f"Device {self.name}: Starting inventory mapper") + self.logger.debug(f"Host {self.name}: Starting inventory mapper") # Let's build an inventory dict for each property in the inventory_map for nb_inv_field, zbx_inv_field in inventory_map.items(): field_list = nb_inv_field.split("/") # convert str to list based on delimiter @@ -183,14 +189,14 @@ class NetworkDevice(): self.inventory[zbx_inv_field] = str(value) elif not value: # empty value should just be an empty string for API compatibility - self.logger.debug(f"Device {self.name}: Netbox inventory lookup for " + self.logger.debug(f"Host {self.name}: Netbox inventory lookup for " f"'{nb_inv_field}' returned an empty value") self.inventory[zbx_inv_field] = "" else: # Value is not a string or numeral, probably not what the user expected. - self.logger.error(f"Device {self.name}: Inventory lookup for '{nb_inv_field}'" + self.logger.error(f"Host {self.name}: Inventory lookup for '{nb_inv_field}'" " returned an unexpected type: it will be skipped.") - self.logger.debug(f"Device {self.name}: Inventory mapping complete. " + self.logger.debug(f"Host {self.name}: Inventory mapping complete. " f"Mapped {len(list(filter(None, self.inventory.values())))} field(s)") return True @@ -224,12 +230,12 @@ class NetworkDevice(): """ masterid = self.getClusterMaster() if masterid == self.id: - self.logger.debug(f"Device {self.name} is primary cluster member. " + self.logger.debug(f"Host {self.name} is primary cluster member. " f"Modifying hostname from {self.name} to " + f"{self.nb.virtual_chassis.name}.") self.name = self.nb.virtual_chassis.name return True - self.logger.debug(f"Device {self.name} is non-primary cluster member.") + self.logger.debug(f"Host {self.name} is non-primary cluster member.") return False def zbxTemplatePrepper(self, templates): @@ -240,7 +246,7 @@ class NetworkDevice(): """ # Check if there are templates defined if not self.zbx_template_names: - e = f"Device {self.name}: No templates found" + e = f"Host {self.name}: No templates found" self.logger.info(e) raise SyncInventoryError() # Set variable to empty list @@ -257,7 +263,7 @@ class NetworkDevice(): template_match = True self.zbx_templates.append({"templateid": zbx_template['templateid'], "name": zbx_template['name']}) - e = f"Device {self.name}: found template {zbx_template['name']}" + e = f"Host {self.name}: found template {zbx_template['name']}" self.logger.debug(e) # Return error should the template not be found in Zabbix if not template_match: @@ -276,7 +282,7 @@ class NetworkDevice(): for group in groups: if group['name'] == self.hostgroup: self.group_id = group['groupid'] - e = f"Device {self.name}: matched group {group['name']}" + e = f"Host {self.name}: matched group {group['name']}" self.logger.debug(e) return True return False @@ -291,7 +297,7 @@ class NetworkDevice(): self.zabbix.host.delete(self.zabbix_id) self.nb.custom_fields[device_cf] = None self.nb.save() - e = f"Device {self.name}: Deleted host from Zabbix." + e = f"Host {self.name}: Deleted host from Zabbix." self.logger.info(e) self.create_journal_entry("warning", "Deleted host from Zabbix") except APIRequestError as e: @@ -364,11 +370,11 @@ class NetworkDevice(): continue # If the proxy name matches if proxy["name"] == proxy_name: - self.logger.debug(f"Device {self.name}: using {proxy['type']}" + self.logger.debug(f"Host {self.name}: using {proxy['type']}" f" {proxy_name}") self.zbxproxy = proxy return True - self.logger.warning(f"Device {self.name}: unable to find proxy {proxy_name}") + self.logger.warning(f"Host {self.name}: unable to find proxy {proxy_name}") return False def createInZabbix(self, groups, templates, proxies, @@ -419,17 +425,17 @@ class NetworkDevice(): host = self.zabbix.host.create(**create_data) self.zabbix_id = host["hostids"][0] except APIRequestError as e: - e = f"Device {self.name}: Couldn't create. Zabbix returned {str(e)}." + e = f"Host {self.name}: Couldn't create. Zabbix returned {str(e)}." self.logger.error(e) raise SyncExternalError(e) from None # Set Netbox custom field to hostID value. self.nb.custom_fields[device_cf] = int(self.zabbix_id) self.nb.save() - msg = f"Device {self.name}: Created host in Zabbix." + msg = f"Host {self.name}: Created host in Zabbix." self.logger.info(msg) self.create_journal_entry("success", msg) else: - e = f"Device {self.name}: Unable to add to Zabbix. Host already present." + e = f"Host {self.name}: Unable to add to Zabbix. Host already present." self.logger.warning(e) def createZabbixHostgroup(self, hostgroups): @@ -478,7 +484,7 @@ class NetworkDevice(): try: self.zabbix.host.update(hostid=self.zabbix_id, **kwargs) except APIRequestError as e: - e = (f"Device {self.name}: Unable to update. " + e = (f"Host {self.name}: Unable to update. " f"Zabbix returned the following error: {str(e)}.") self.logger.error(e) raise SyncExternalError(e) from None @@ -502,7 +508,7 @@ class NetworkDevice(): if not self.group_id: # Function returns true / false but also sets GroupID if not self.setZabbixGroupID(groups) and not create_hostgroups: - e = (f"Device {self.name}: different hostgroup is required but " + e = (f"Host {self.name}: different hostgroup is required but " "unable to create hostgroup without generation permission.") self.logger.warning(e) raise SyncInventoryError(e) @@ -523,30 +529,30 @@ class NetworkDevice(): self.logger.error(e) raise SyncInventoryError(e) if len(host) == 0: - e = (f"Device {self.name}: No Zabbix host found. " + e = (f"Host {self.name}: No Zabbix host found. " f"This is likely the result of a deleted Zabbix host " f"without zeroing the ID field in Netbox.") self.logger.error(e) raise SyncInventoryError(e) host = host[0] if host["host"] == self.name: - self.logger.debug(f"Device {self.name}: hostname in-sync.") + self.logger.debug(f"Host {self.name}: hostname in-sync.") else: - self.logger.warning(f"Device {self.name}: hostname OUT of sync. " + self.logger.warning(f"Host {self.name}: hostname OUT of sync. " f"Received value: {host['host']}") self.updateZabbixHost(host=self.name) # Execute check depending on wether the name is special or not if self.use_visible_name: if host["name"] == self.visible_name: - self.logger.debug(f"Device {self.name}: visible name in-sync.") + self.logger.debug(f"Host {self.name}: visible name in-sync.") else: - self.logger.warning(f"Device {self.name}: visible name OUT of sync." + self.logger.warning(f"Host {self.name}: visible name OUT of sync." f" Received value: {host['name']}") self.updateZabbixHost(name=self.visible_name) # Check if the templates are in-sync if not self.zbx_template_comparer(host["parentTemplates"]): - self.logger.warning(f"Device {self.name}: template(s) OUT of sync.") + self.logger.warning(f"Host {self.name}: template(s) OUT of sync.") # Prepare Templates for API parsing templateids = [] for template in self.zbx_templates: @@ -555,33 +561,33 @@ class NetworkDevice(): self.updateZabbixHost(templates_clear=host["parentTemplates"], templates=templateids) else: - self.logger.debug(f"Device {self.name}: template(s) in-sync.") + self.logger.debug(f"Host {self.name}: template(s) in-sync.") for group in host["groups"]: if group["groupid"] == self.group_id: - self.logger.debug(f"Device {self.name}: hostgroup in-sync.") + self.logger.debug(f"Host {self.name}: hostgroup in-sync.") break else: - self.logger.warning(f"Device {self.name}: hostgroup OUT of sync.") + self.logger.warning(f"Host {self.name}: hostgroup OUT of sync.") self.updateZabbixHost(groups={'groupid': self.group_id}) if int(host["status"]) == self.zabbix_state: - self.logger.debug(f"Device {self.name}: status in-sync.") + self.logger.debug(f"Host {self.name}: status in-sync.") else: - self.logger.warning(f"Device {self.name}: status OUT of sync.") + self.logger.warning(f"Host {self.name}: status OUT of sync.") self.updateZabbixHost(status=str(self.zabbix_state)) # Check if a proxy has been defined if self.zbxproxy: # Check if proxy or proxy group is defined if (self.zbxproxy["idtype"] in host and host[self.zbxproxy["idtype"]] == self.zbxproxy["id"]): - self.logger.debug(f"Device {self.name}: proxy in-sync.") + self.logger.debug(f"Host {self.name}: proxy in-sync.") # Backwards compatibility for Zabbix <= 6 elif "proxy_hostid" in host and host["proxy_hostid"] == self.zbxproxy["id"]: - self.logger.debug(f"Device {self.name}: proxy in-sync.") + self.logger.debug(f"Host {self.name}: proxy in-sync.") # Proxy does not match, update Zabbix else: - self.logger.warning(f"Device {self.name}: proxy OUT of sync.") + self.logger.warning(f"Host {self.name}: proxy OUT of sync.") # Zabbix <= 6 patch if not str(self.zabbix.version).startswith('7'): self.updateZabbixHost(proxy_hostid=self.zbxproxy['id']) @@ -601,7 +607,7 @@ class NetworkDevice(): proxy_set = True if proxy_power and proxy_set: # Zabbix <= 6 fix - self.logger.warning(f"Device {self.name}: no proxy is configured in Netbox " + self.logger.warning(f"Host {self.name}: no proxy is configured in Netbox " "but is configured in Zabbix. Removing proxy config in Zabbix") if "proxy_hostid" in host and bool(host["proxy_hostid"]): self.updateZabbixHost(proxy_hostid=0) @@ -614,24 +620,24 @@ class NetworkDevice(): # Checks if a proxy has been defined in Zabbix and if proxy_power config has been set if proxy_set and not proxy_power: # Display error message - self.logger.error(f"Device {self.name} is configured " + self.logger.error(f"Host {self.name} is configured " f"with proxy in Zabbix but not in Netbox. The" " -p flag was ommited: no " "changes have been made.") if not proxy_set: - self.logger.debug(f"Device {self.name}: proxy in-sync.") + self.logger.debug(f"Host {self.name}: proxy in-sync.") # Check host inventory mode if str(host['inventory_mode']) == str(self.inventory_mode): - self.logger.debug(f"Device {self.name}: inventory_mode in-sync.") + self.logger.debug(f"Host {self.name}: inventory_mode in-sync.") else: - self.logger.warning(f"Device {self.name}: inventory_mode OUT of sync.") + self.logger.warning(f"Host {self.name}: inventory_mode OUT of sync.") self.updateZabbixHost(inventory_mode=str(self.inventory_mode)) if inventory_sync and self.inventory_mode in [0,1]: # Check host inventory mapping if host['inventory'] == self.inventory: - self.logger.debug(f"Device {self.name}: inventory in-sync.") + self.logger.debug(f"Host {self.name}: inventory in-sync.") else: - self.logger.warning(f"Device {self.name}: inventory OUT of sync.") + self.logger.warning(f"Host {self.name}: inventory OUT of sync.") self.updateZabbixHost(inventory=self.inventory) # If only 1 interface has been found @@ -669,10 +675,10 @@ class NetworkDevice(): updates[key] = item if updates: # If interface updates have been found: push to Zabbix - self.logger.warning(f"Device {self.name}: Interface OUT of sync.") + self.logger.warning(f"Host {self.name}: Interface OUT of sync.") if "type" in updates: # Changing interface type not supported. Raise exception. - e = (f"Device {self.name}: changing interface type to " + e = (f"Host {self.name}: changing interface type to " f"{str(updates['type'])} is not supported.") self.logger.error(e) raise InterfaceConfigError(e) @@ -681,7 +687,7 @@ class NetworkDevice(): try: # API call to Zabbix self.zabbix.hostinterface.update(updates) - e = f"Device {self.name}: solved interface conflict." + e = f"Host {self.name}: solved interface conflict." self.logger.info(e) self.create_journal_entry("info", e) except APIRequestError as e: @@ -690,10 +696,10 @@ class NetworkDevice(): raise SyncExternalError(e) from e else: # If no updates are found, Zabbix interface is in-sync - e = f"Device {self.name}: interface in-sync." + e = f"Host {self.name}: interface in-sync." self.logger.debug(e) else: - e = (f"Device {self.name} has unsupported interface configuration." + e = (f"Host {self.name} has unsupported interface configuration." f" Host has total of {len(host['interfaces'])} interfaces. " "Manual interfention required.") self.logger.error(e) @@ -716,7 +722,7 @@ class NetworkDevice(): } try: self.nb_journals.create(journal) - self.logger.debug(f"Device {self.name}: Created journal entry in Netbox") + self.logger.debug(f"Host {self.name}: Created journal entry in Netbox") return True except JournalError(e) as e: self.logger.warning("Unable to create journal entry for " @@ -743,7 +749,7 @@ class NetworkDevice(): # and add this NB template to the list of successfull templates tmpls_from_zabbix.pop(pos) succesfull_templates.append(nb_tmpl) - self.logger.debug(f"Device {self.name}: template " + self.logger.debug(f"Host {self.name}: template " f"{nb_tmpl['name']} is present in Zabbix.") break if len(succesfull_templates) == len(self.zbx_templates) and len(tmpls_from_zabbix) == 0: diff --git a/modules/hostgroups.py b/modules/hostgroups.py index 5c4f151..c008dbe 100644 --- a/modules/hostgroups.py +++ b/modules/hostgroups.py @@ -63,7 +63,9 @@ class Hostgroup(): format_options["manufacturer"] = self.nb.device_type.manufacturer.name # Variables only applicable for VM's if self.type == "vm": - format_options["cluster"] = str(self.nb.cluster.name) if self.nb.cluster else None + format_options["cluster"] = self.nb.cluster.name + format_options["cluster_type"] = self.nb.cluster.type.name + self.format_options = format_options def generate(self, hg_format=None): diff --git a/modules/virtualMachine.py b/modules/virtualMachine.py index 82ca3d3..d1ab41b 100644 --- a/modules/virtualMachine.py +++ b/modules/virtualMachine.py @@ -1,23 +1,41 @@ """Module that hosts all functions for virtual machine processing""" -from modules.exceptions import * +from modules.device import PhysicalDevice +from modules.hostgroups import Hostgroup +from modules.exceptions import TemplateError +try: + from config import ( + traverse_site_groups, + traverse_regions, + template_cf + ) +except ModuleNotFoundError: + print("Configuration file config.py not found in main directory." + "Please create the file or rename the config.py.example file to config.py.") + sys.exit(0) -class VirtualMachine(): +class VirtualMachine(PhysicalDevice): """Model for virtual machines""" - def __init__(self, nb, name): - self.nb = nb - self.name = name - self.hostgroup = None + def set_hostgroup(self, hg_format, nb_site_groups, nb_regions): + """Set the hostgroup for this device""" + # Create new Hostgroup instance + hg = Hostgroup("vm", self.nb, self.nb_api_version, + traverse_site_groups, traverse_regions, + nb_site_groups, nb_regions) + # Generate hostgroup based on hostgroup format + self.hostgroup = hg.generate(hg_format) - def __repr__(self): - return self.name - - def __str__(self): - return self.__repr__() - - def _data_prep(self): - self.platform = self.nb.platform.name - self.cluster = self.nb.cluster.name - - def set_hostgroup(self): - self.hostgroup = "Virtual machines" + def set_vm_template(self): + """ Set Template for VMs. Overwrites default class + to skip a lookup of custom fields.""" + self.zbx_template_names = None + # Gather templates ONLY from the device specific context + try: + self.zbx_template_names = self.get_templates_context() + except TemplateError as e: + self.logger.warning(e) + return True + + def set_template(self, **kwargs): + """Simple wrapper fur underlying functions""" + self.set_vm_template() \ No newline at end of file diff --git a/netbox_zabbix_sync.py b/netbox_zabbix_sync.py index 52f9652..e3f023a 100755 --- a/netbox_zabbix_sync.py +++ b/netbox_zabbix_sync.py @@ -7,7 +7,8 @@ import argparse from os import environ, path, sys from pynetbox import api from zabbix_utils import ZabbixAPI, APIRequestError, ProcessingError -from modules.device import NetworkDevice +from modules.device import PhysicalDevice +from modules.virtualMachine import VirtualMachine from modules.tools import convert_recordset, proxy_prepper from modules.exceptions import EnvironmentVarError, HostgroupError, SyncError try: @@ -19,7 +20,9 @@ try: zabbix_device_removal, zabbix_device_disable, hostgroup_format, - nb_device_filter + vm_hostgroup_format, + nb_device_filter, + sync_vms ) except ModuleNotFoundError: print("Configuration file config.py not found in main directory." @@ -106,6 +109,7 @@ def main(arguments): proxy_name = "name" # Get all Zabbix and Netbox data netbox_devices = netbox.dcim.devices.filter(**nb_device_filter) + netbox_vms = netbox.virtualization.virtual_machines.all() if sync_vms else list() netbox_site_groups = convert_recordset((netbox.dcim.site_groups.all())) netbox_regions = convert_recordset(netbox.dcim.regions.all()) netbox_journals = netbox.extras.journal_entries @@ -127,11 +131,53 @@ def main(arguments): nb_version = netbox.version # Go through all Netbox devices - for nb_device in netbox_devices: - try: + try: + for nb_vm in netbox_vms: + vm = VirtualMachine(nb_vm, zabbix, netbox_journals, nb_version, + create_journal, logger) + vm.set_hostgroup(vm_hostgroup_format,netbox_site_groups,netbox_regions) + vm.set_vm_template() + vm.set_inventory(nb_vm) + print(f"Templates: {vm.zbx_template_names}") + + # Checks if device is in cleanup state + if vm.status in zabbix_device_removal: + if vm.zabbix_id: + # Delete device from Zabbix + # and remove hostID from Netbox. + vm.cleanup() + logger.info(f"VM {vm.name}: cleanup complete") + continue + # Device has been added to Netbox + # but is not in Activate state + logger.info(f"VM {vm.name}: skipping since this VM is " + f"not in the active state.") + continue + # Check if the VM is in the disabled state + if vm.status in zabbix_device_disable: + vm.zabbix_state = 1 + # Check if VM is already in Zabbix + if vm.zabbix_id: + vm.ConsistencyCheck(zabbix_groups, zabbix_templates, + zabbix_proxy_list, full_proxy_sync, + create_hostgroups) + continue + # Add hostgroup is config is set + if create_hostgroups: + # Create new hostgroup. Potentially multiple groups if nested + hostgroups = vm.createZabbixHostgroup(zabbix_groups) + # go through all newly created hostgroups + for group in hostgroups: + # Add new hostgroups to zabbix group list + zabbix_groups.append(group) + # Add VM to Zabbix + vm.createInZabbix(zabbix_groups, zabbix_templates, + zabbix_proxy_list) + + for nb_device in netbox_devices: # Set device instance set data such as hostgroup and template information. - device = NetworkDevice(nb_device, zabbix, netbox_journals, nb_version, - create_journal, logger) + device = PhysicalDevice(nb_device, zabbix, netbox_journals, nb_version, + create_journal, logger) device.set_hostgroup(hostgroup_format,netbox_site_groups,netbox_regions) device.set_template(templates_config_context, templates_config_context_overrule) device.set_inventory(nb_device) @@ -183,8 +229,8 @@ def main(arguments): # Add device to Zabbix device.createInZabbix(zabbix_groups, zabbix_templates, zabbix_proxy_list) - except SyncError: - pass + except SyncError: + pass if __name__ == "__main__":