Merge pull request #80 from TheNetworkGuy/virtual_machines

Virtual machines

.gitignore

@@ -1,4 +1,5 @@
 *.log
+.venv
 config.py
 # Byte-compiled / optimized / DLL files
 __pycache__/

README.md

@@ -87,39 +87,59 @@ You can make the `zabbix_hostid` field hidden or read-only to prevent human inte
 
 This is optional and there is a use case for leaving it read-write in the UI to manually change the ID. For example to re-run a sync.
 
+## Virtual Machine (VM) Syncing
+In order to use VM syncing, make sure that the `zabbix_id` custom field is also present on Virtual machine objects in Netbox.
+
+Use the `config.py` file and set the `sync_vms` variable to `True`.
+
+You can set the `vm_hostgroup_format` variable to a customizable value for VM hostgroups. The default is `cluster_type/cluster/role`.
+
+To enable filtering for VM's, check the `nb_vm_filter` variable out. It works the same as with the device filter (see documentation under "Hostgroup layout"). Note that not all filtering capabilities and properties of devices are applicable to VM's and vice-versa. Check the Netbox API documentation to see which filtering options are available for each object type.
+
 ## Config file
 
 ### Hostgroup
-Setting the `create_hostgroups` variable to `False` requires manual hostgroup creation for devices in a new category.
+Setting the `create_hostgroups` variable to `False` requires manual hostgroup creation for devices in a new category. I would recommend setting this variable to `True` since leaving it on `False` results in a lot of manual work.
 
-The format can be set with the `hostgroup_format` variable.
+The format can be set with the `hostgroup_format` variable for devices and `vm_hostgroup_format` for devices.
 
-Any nested parent hostgroups will also be created automatically.
+Any nested parent hostgroups will also be created automatically. For instance the region `Berlin` with parent region `Germany` will create the hostgroup `Germany/Berlin`.
 
 Make sure that the Zabbix user has proper permissions to create hosts.
 The hostgroups are in a nested format. This means that proper permissions only need to be applied to the site name hostgroup and cascaded to any child hostgroups.
 
 #### Layout
 The default hostgroup layout is "site/manufacturer/device_role".
-
-**Variables**
-
 You can change this behaviour with the hostgroup_format variable. The following values can be used:
+
+**Both devices and virtual machines**
 |  name | description  |
 | ------------ | ------------ |
-|dev_location|The device location name|
+|role|Role name of a device or VM|
-|dev_role|The device role name|
+|region|The region name|
-|manufacturer|Manufacturer name|
-|region|The region name of the device|
 |site|Site name|
 |site_group|Site group name|
 |tenant|Tenant name|
 |tenant_group|Tenant group name|
+|platform|Software platform of a device or VM|
+|custom fields|See the section "Layout -> Custom Fields" to use custom fields as hostgroup variable|
+
+**Only for devices**
+|  name | description  |
+| ------------ | ------------ |
+|location|The device location name|
+|manufacturer|Device manufacturer name|
+
+**Only for VMs**
+|  name | description  |
+| ------------ | ------------ |
+|cluster|VM cluster name|
+|cluster_type|VM cluster type|
 
 
-You can specify the value like so, sperated by a "/":
+You can specify the value sperated by a "/" like so:
 ```
-hostgroup_format = "tenant/site/dev_location/dev_role"
+hostgroup_format = "tenant/site/dev_location/role"
 ```
 **Group traversal**
 
@@ -129,11 +149,18 @@ However, by setting `traverse_region` to `True` in `config.py` the script will r
 
 **Custom fields**
 
-You can also use the value of custom fields under the device object.
+You can use the value of custom fields for hostgroup generation. This allows more freedom and even allows a full static mapping instead of a dynamic rendered hostgroup name.
 
-This allows more freedom and even allows a full static mapping instead of a dynamic rendered hostgroup name.
+For instance a custom field with the name `mycustomfieldname` and type string has the following values for 2 devices:
 ```
-hostgroup_format = "site/mycustomfieldname"
+Device A has the value Train for custom field mycustomfieldname.
+Device B has the value Bus for custom field mycustomfieldname.
+Both devices are located in the site Paris.
+```
+With the hostgroup format `site/mycustomfieldname` the following hostgroups will be generated:
+```
+Device A: Paris/Train
+Device B: Paris/Bus
 ```
 **Empty variables or hostgroups**
 

config.py.example

@@ -3,7 +3,7 @@
 # coming from config context instead of a custom field.
 templates_config_context = False
 
-# Set to true to give config context templates a 
+# Set to true to give config context templates a
 # higher priority then custom field templates
 templates_config_context_overrule = False
 
@@ -21,6 +21,14 @@ create_hostgroups = True
 ## Create journal entries
 create_journal = False
 
+## Virtual machine sync
+# Set sync_vms to True in order to use this new feature
+# Use the hostgroup vm_hostgroup_format mapper for specific
+# hostgroup atributes of VM's such as cluster_type and cluster
+sync_vms = False
+# Check the README documentation for values to use in the VM hostgroup format.
+vm_hostgroup_format = "cluster_type/cluster/role"
+
 ## Proxy Sync
 # Set to true to enable removal of proxy's under hosts. Use with caution and make sure that you specified
 # all the required proxy's in the device config context before enabeling this option.
@@ -32,7 +40,7 @@ zabbix_device_removal = ["Decommissioning", "Inventory"]
 zabbix_device_disable = ["Offline", "Planned", "Staged", "Failed"]
 
 ## Hostgroup mapping
-# Available choices: dev_location, dev_role, manufacturer, region, site, site_group, tenant, tenant_group
+# See the README documentation for available options
 # You can also use CF (custom field) names under the device. The CF content will be used for the hostgroup generation.
 #
 # When using region in the group name, the default behaviour is to use name of the directly assigned region.
@@ -40,8 +48,8 @@ zabbix_device_disable = ["Offline", "Planned", "Staged", "Failed"]
 #
 # 'Global/Europe/Netherlands/Amsterdam' instead of just 'Amsterdam'.
 #
-# traverse_site_groups controls the same behaviour for any assigned site_groups.   
+# traverse_site_groups controls the same behaviour for any assigned site_groups.
-hostgroup_format = "site/manufacturer/dev_role"
+hostgroup_format = "site/manufacturer/role"
 traverse_regions = False
 traverse_site_groups = False
 
@@ -56,6 +64,8 @@ traverse_site_groups = False
 
 # Default device filter, only get devices which have a name in Netbox:
 nb_device_filter = {"name__n": "null"}
+# Default filter for VMs
+nb_vm_filter = {"name__n": "null"}
 
 ## Inventory
 # See https://www.zabbix.com/documentation/current/en/manual/config/hosts/inventory#building-inventory

modules/device.py

@@ -4,12 +4,13 @@
 Device specific handeling for Netbox to Zabbix
 """
 from os import sys
+from re import search
 from logging import getLogger
 from zabbix_utils import APIRequestError
 from modules.exceptions import (SyncInventoryError, TemplateError, SyncExternalError,
                                 InterfaceConfigError, JournalError)
 from modules.interface import ZabbixInterface
-from modules.tools import build_path
+from modules.hostgroups import Hostgroup
 try:
     from config import (
         template_cf, device_cf,
@@ -24,8 +25,8 @@ except ModuleNotFoundError:
            "Please create the file or rename the config.py.example file to config.py.")
     sys.exit(0)
 
-class NetworkDevice():
+class PhysicalDevice():
-    # pylint: disable=too-many-instance-attributes, too-many-arguments
+    # pylint: disable=too-many-instance-attributes, too-many-arguments, too-many-positional-arguments
     """
     Represents Network device.
     INPUT: (Netbox device class, ZabbixAPI class, journal flag, NB journal class)
@@ -55,6 +56,12 @@ class NetworkDevice():
         self.logger = logger if logger else getLogger(__name__)
         self._setBasics()
 
+    def __repr__(self):
+        return self.name
+
+    def __str__(self):
+        return self.__repr__()
+
     def _setBasics(self):
         """
         Sets basic information like IP address.
@@ -64,7 +71,7 @@ class NetworkDevice():
             self.cidr = self.nb.primary_ip.address
             self.ip = self.cidr.split("/")[0]
         else:
-            e = f"Device {self.name}: no primary IP."
+            e = f"Host {self.name}: no primary IP."
             self.logger.info(e)
             raise SyncInventoryError(e)
 
@@ -72,18 +79,19 @@ class NetworkDevice():
         if device_cf in self.nb.custom_fields:
             self.zabbix_id = self.nb.custom_fields[device_cf]
         else:
-            e = f"Device {self.name}: Custom field {device_cf} not present"
+            e = f"Host {self.name}: Custom field {device_cf} not present"
             self.logger.warning(e)
             raise SyncInventoryError(e)
 
         # Validate hostname format.
         odd_character_list = ["ä", "ö", "ü", "Ä", "Ö", "Ü", "ß"]
         self.use_visible_name = False
-        if any(letter in self.name for letter in odd_character_list):
+        if (any(letter in self.name for letter in odd_character_list) or
+            bool(search('[\u0400-\u04FF]', self.name))):
             self.name = f"NETBOX_ID{self.id}"
             self.visible_name = self.nb.name
             self.use_visible_name = True
-            self.logger.info(f"Device {self.visible_name} contains special characters. "
+            self.logger.info(f"Host {self.visible_name} contains special characters. "
                              f"Using {self.name} as name for the Netbox object "
                              f"and using {self.visible_name} as visible name in Zabbix.")
         else:
@@ -91,58 +99,12 @@ class NetworkDevice():
 
     def set_hostgroup(self, hg_format, nb_site_groups, nb_regions):
         """Set the hostgroup for this device"""
-        # Get all variables from the NB data
+        # Create new Hostgroup instance
-        dev_location = str(self.nb.location) if self.nb.location else None
+        hg = Hostgroup("dev", self.nb, self.nb_api_version)
-        # Check the Netbox version. Use backwards compatibility for versions 2 and 3.
+        # Set Hostgroup nesting options
-        if self.nb_api_version.startswith(("2", "3")):
+        hg.set_nesting(traverse_site_groups, traverse_regions, nb_site_groups, nb_regions)
-            dev_role = self.nb.device_role.name
+        # Generate hostgroup based on hostgroup format
-        else:
+        self.hostgroup = hg.generate(hg_format)
-            dev_role = self.nb.role.name
-        manufacturer = self.nb.device_type.manufacturer.name
-        region = str(self.nb.site.region) if self.nb.site.region else None
-        site = self.nb.site.name
-        site_group = str(self.nb.site.group) if self.nb.site.group else None
-        tenant = str(self.tenant) if self.tenant else None
-        tenant_group = str(self.tenant.group) if tenant else None
-        # Set mapper for string -> variable
-        hostgroup_vars = {"dev_location": dev_location, "dev_role": dev_role,
-                          "manufacturer": manufacturer, "region": region,
-                          "site": site, "site_group": site_group,
-                          "tenant": tenant, "tenant_group": tenant_group}
-        # Generate list based off string input format
-        hg_items = hg_format.split("/")
-        hostgroup = ""
-        # Go through all hostgroup items
-        for item in hg_items:
-            # Check if the variable (such as Tenant) is empty.
-            if not hostgroup_vars[item]:
-                continue
-            # Check if the item is a custom field name
-            if item not in hostgroup_vars:
-                cf_value = self.nb.custom_fields[item] if item in self.nb.custom_fields else None
-                if cf_value:
-                    # If there is a cf match, add the value of this cf to the hostgroup
-                    hostgroup += cf_value + "/"
-                # Should there not be a match, this means that
-                # the variable is invalid. Skip regardless.
-                continue
-            # Add value of predefined variable to hostgroup format
-            if item == "site_group" and nb_site_groups and traverse_site_groups:
-                group_path = build_path(site_group, nb_site_groups)
-                hostgroup += "/".join(group_path) + "/"
-            elif item == "region" and nb_regions and traverse_regions:
-                region_path = build_path(region, nb_regions)
-                hostgroup += "/".join(region_path) + "/"
-            else:
-                hostgroup += hostgroup_vars[item] + "/"
-        # If the final hostgroup variable is empty
-        if not hostgroup:
-            e = (f"{self.name} has no reliable hostgroup. This is"
-                 "most likely due to the use of custom fields that are empty.")
-            self.logger.error(e)
-            raise SyncInventoryError(e)
-        # Remove final inserted "/" and set hostgroup to class var
-        self.hostgroup = hostgroup.rstrip("/")
 
     def set_template(self, prefer_config_context, overrule_custom):
         """ Set Template """
@@ -185,12 +147,12 @@ class NetworkDevice():
     def get_templates_context(self):
         """ Get Zabbix templates from the device context """
         if "zabbix" not in self.config_context:
-            e = ("Key 'zabbix' not found in config "
+            e = (f"Host {self.name}: Key 'zabbix' not found in config "
-                    f"context for template host {self.name}")
+                 "context for template lookup")
             raise TemplateError(e)
         if "templates" not in self.config_context["zabbix"]:
-            e = ("Key 'templates' not found in config "
+            e = (f"Host {self.name}: Key 'templates' not found in config "
-                    f"context 'zabbix' for template host {self.name}")
+                 "context 'zabbix' for template lookup")
             raise TemplateError(e)
         return self.config_context["zabbix"]["templates"]
 
@@ -199,7 +161,7 @@ class NetworkDevice():
         # Set inventory mode. Default is disabled (see class init function).
         if inventory_mode == "disabled":
             if inventory_sync:
-                self.logger.error(f"Device {self.name}: Unable to map Netbox inventory to Zabbix. "
+                self.logger.error(f"Host {self.name}: Unable to map Netbox inventory to Zabbix. "
                               "Inventory sync is enabled in config but inventory mode is disabled.")
             return True
         if inventory_mode == "manual":
@@ -207,12 +169,12 @@ class NetworkDevice():
         elif inventory_mode == "automatic":
             self.inventory_mode = 1
         else:
-            self.logger.error(f"Device {self.name}: Specified value for inventory mode in"
+            self.logger.error(f"Host {self.name}: Specified value for inventory mode in"
                               f" config is not valid. Got value {inventory_mode}")
             return False
         self.inventory = {}
         if inventory_sync and self.inventory_mode in [0,1]:
-            self.logger.debug(f"Device {self.name}: Starting inventory mapper")
+            self.logger.debug(f"Host {self.name}: Starting inventory mapper")
             # Let's build an inventory dict for each property in the inventory_map
             for nb_inv_field, zbx_inv_field in inventory_map.items():
                 field_list = nb_inv_field.split("/") # convert str to list based on delimiter
@@ -229,14 +191,14 @@ class NetworkDevice():
                     self.inventory[zbx_inv_field] = str(value)
                 elif not value:
                     # empty value should just be an empty string for API compatibility
-                    self.logger.debug(f"Device {self.name}: Netbox inventory lookup for "
+                    self.logger.debug(f"Host {self.name}: Netbox inventory lookup for "
                                       f"'{nb_inv_field}' returned an empty value")
                     self.inventory[zbx_inv_field] = ""
                 else:
                     # Value is not a string or numeral, probably not what the user expected.
-                    self.logger.error(f"Device {self.name}: Inventory lookup for '{nb_inv_field}'"
+                    self.logger.error(f"Host {self.name}: Inventory lookup for '{nb_inv_field}'"
                                       " returned an unexpected type: it will be skipped.")
-            self.logger.debug(f"Device {self.name}: Inventory mapping complete. "
+            self.logger.debug(f"Host {self.name}: Inventory mapping complete. "
                             f"Mapped {len(list(filter(None, self.inventory.values())))} field(s)")
         return True
 
@@ -270,12 +232,12 @@ class NetworkDevice():
         """
         masterid = self.getClusterMaster()
         if masterid == self.id:
-            self.logger.debug(f"Device {self.name} is primary cluster member. "
+            self.logger.debug(f"Host {self.name} is primary cluster member. "
                               f"Modifying hostname from {self.name} to " +
                               f"{self.nb.virtual_chassis.name}.")
             self.name = self.nb.virtual_chassis.name
             return True
-        self.logger.debug(f"Device {self.name} is non-primary cluster member.")
+        self.logger.debug(f"Host {self.name} is non-primary cluster member.")
         return False
 
     def zbxTemplatePrepper(self, templates):
@@ -286,7 +248,7 @@ class NetworkDevice():
         """
         # Check if there are templates defined
         if not self.zbx_template_names:
-            e = f"Device {self.name}: No templates found"
+            e = f"Host {self.name}: No templates found"
             self.logger.info(e)
             raise SyncInventoryError()
         # Set variable to empty list
@@ -303,7 +265,7 @@ class NetworkDevice():
                     template_match = True
                     self.zbx_templates.append({"templateid": zbx_template['templateid'],
                                                "name": zbx_template['name']})
-                    e = f"Device {self.name}: found template {zbx_template['name']}"
+                    e = f"Host {self.name}: found template {zbx_template['name']}"
                     self.logger.debug(e)
             # Return error should the template not be found in Zabbix
             if not template_match:
@@ -322,7 +284,7 @@ class NetworkDevice():
         for group in groups:
             if group['name'] == self.hostgroup:
                 self.group_id = group['groupid']
-                e = f"Device {self.name}: matched group {group['name']}"
+                e = f"Host {self.name}: matched group {group['name']}"
                 self.logger.debug(e)
                 return True
         return False
@@ -334,16 +296,28 @@ class NetworkDevice():
         """
         if self.zabbix_id:
             try:
-                self.zabbix.host.delete(self.zabbix_id)
+                # Check if the Zabbix host exists in Zabbix
-                self.nb.custom_fields[device_cf] = None
+                zbx_host = bool(self.zabbix.host.get(filter={'hostid': self.zabbix_id},
-                self.nb.save()
+                                                     output=[]))
-                e = f"Device {self.name}: Deleted host from Zabbix."
+                e = (f"Host {self.name}: was already deleted from Zabbix."
+                        " Removed link in Netbox.")
+                if zbx_host:
+                    # Delete host should it exists
+                    self.zabbix.host.delete(self.zabbix_id)
+                    e = f"Host {self.name}: Deleted host from Zabbix."
+                self._zeroize_cf()
                 self.logger.info(e)
                 self.create_journal_entry("warning", "Deleted host from Zabbix")
             except APIRequestError as e:
-                e = f"Zabbix returned the following error: {str(e)}."
+                message = f"Zabbix returned the following error: {str(e)}."
-                self.logger.error(e)
+                self.logger.error(message)
-                raise SyncExternalError(e) from e
+                raise SyncExternalError(message) from e
+
+    def _zeroize_cf(self):
+        """Sets the hostID custom field in Netbox to zero,
+        effectively destroying the link"""
+        self.nb.custom_fields[device_cf] = None
+        self.nb.save()
 
     def _zabbixHostnameExists(self):
         """
@@ -372,12 +346,12 @@ class NetworkDevice():
                 if interface.interface["type"] == 2:
                     interface.set_snmp()
             else:
-                interface.set_default()
+                interface.set_default_snmp()
             return [interface.interface]
         except InterfaceConfigError as e:
-            e = f"{self.name}: {e}"
+            message = f"{self.name}: {e}"
-            self.logger.warning(e)
+            self.logger.warning(message)
-            raise SyncInventoryError(e) from e
+            raise SyncInventoryError(message) from e
 
     def setProxy(self, proxy_list):
         """
@@ -410,11 +384,11 @@ class NetworkDevice():
                         continue
                     # If the proxy name matches
                     if proxy["name"] == proxy_name:
-                        self.logger.debug(f"Device {self.name}: using {proxy['type']}"
+                        self.logger.debug(f"Host {self.name}: using {proxy['type']}"
                                           f" {proxy_name}")
                         self.zbxproxy = proxy
                         return True
-                self.logger.warning(f"Device {self.name}: unable to find proxy {proxy_name}")
+                self.logger.warning(f"Host {self.name}: unable to find proxy {proxy_name}")
         return False
 
     def createInZabbix(self, groups, templates, proxies,
@@ -465,17 +439,17 @@ class NetworkDevice():
                 host = self.zabbix.host.create(**create_data)
                 self.zabbix_id = host["hostids"][0]
             except APIRequestError as e:
-                e = f"Device {self.name}: Couldn't create. Zabbix returned {str(e)}."
+                e = f"Host {self.name}: Couldn't create. Zabbix returned {str(e)}."
                 self.logger.error(e)
                 raise SyncExternalError(e) from None
             # Set Netbox custom field to hostID value.
             self.nb.custom_fields[device_cf] = int(self.zabbix_id)
             self.nb.save()
-            msg = f"Device {self.name}: Created host in Zabbix."
+            msg = f"Host {self.name}: Created host in Zabbix."
             self.logger.info(msg)
             self.create_journal_entry("success", msg)
         else:
-            e = f"Device {self.name}: Unable to add to Zabbix. Host already present."
+            e = f"Host {self.name}: Unable to add to Zabbix. Host already present."
             self.logger.warning(e)
 
     def createZabbixHostgroup(self, hostgroups):
@@ -499,9 +473,9 @@ class NetworkDevice():
                 # Add group to final data
                 final_data.append({'groupid': groupid["groupids"][0], 'name': zabbix_hg})
             except APIRequestError as e:
-                e = f"Hostgroup '{zabbix_hg}': unable to create. Zabbix returned {str(e)}."
+                msg = f"Hostgroup '{zabbix_hg}': unable to create. Zabbix returned {str(e)}."
-                self.logger.error(e)
+                self.logger.error(msg)
-                raise SyncExternalError(e) from e
+                raise SyncExternalError(msg) from e
         return final_data
 
     def lookupZabbixHostgroup(self, group_list, lookup_group):
@@ -524,7 +498,7 @@ class NetworkDevice():
         try:
             self.zabbix.host.update(hostid=self.zabbix_id, **kwargs)
         except APIRequestError as e:
-            e = (f"Device {self.name}: Unable to update. "
+            e = (f"Host {self.name}: Unable to update. "
                  f"Zabbix returned the following error: {str(e)}.")
             self.logger.error(e)
             raise SyncExternalError(e) from None
@@ -548,7 +522,7 @@ class NetworkDevice():
             if not self.group_id:
                 # Function returns true / false but also sets GroupID
                 if not self.setZabbixGroupID(groups) and not create_hostgroups:
-                    e = (f"Device {self.name}: different hostgroup is required but "
+                    e = (f"Host {self.name}: different hostgroup is required but "
                         "unable to create hostgroup without generation permission.")
                     self.logger.warning(e)
                     raise SyncInventoryError(e)
@@ -569,30 +543,30 @@ class NetworkDevice():
             self.logger.error(e)
             raise SyncInventoryError(e)
         if len(host) == 0:
-            e = (f"Device {self.name}: No Zabbix host found. "
+            e = (f"Host {self.name}: No Zabbix host found. "
                  f"This is likely the result of a deleted Zabbix host "
                  f"without zeroing the ID field in Netbox.")
             self.logger.error(e)
             raise SyncInventoryError(e)
         host = host[0]
         if host["host"] == self.name:
-            self.logger.debug(f"Device {self.name}: hostname in-sync.")
+            self.logger.debug(f"Host {self.name}: hostname in-sync.")
         else:
-            self.logger.warning(f"Device {self.name}: hostname OUT of sync. "
+            self.logger.warning(f"Host {self.name}: hostname OUT of sync. "
                                 f"Received value: {host['host']}")
             self.updateZabbixHost(host=self.name)
         # Execute check depending on wether the name is special or not
         if self.use_visible_name:
             if host["name"] == self.visible_name:
-                self.logger.debug(f"Device {self.name}: visible name in-sync.")
+                self.logger.debug(f"Host {self.name}: visible name in-sync.")
             else:
-                self.logger.warning(f"Device {self.name}: visible name OUT of sync."
+                self.logger.warning(f"Host {self.name}: visible name OUT of sync."
                                     f" Received value: {host['name']}")
                 self.updateZabbixHost(name=self.visible_name)
 
         # Check if the templates are in-sync
         if not self.zbx_template_comparer(host["parentTemplates"]):
-            self.logger.warning(f"Device {self.name}: template(s) OUT of sync.")
+            self.logger.warning(f"Host {self.name}: template(s) OUT of sync.")
             # Prepare Templates for API parsing
             templateids = []
             for template in self.zbx_templates:
@@ -601,33 +575,33 @@ class NetworkDevice():
             self.updateZabbixHost(templates_clear=host["parentTemplates"],
                                       templates=templateids)
         else:
-            self.logger.debug(f"Device {self.name}: template(s) in-sync.")
+            self.logger.debug(f"Host {self.name}: template(s) in-sync.")
 
         for group in host["groups"]:
             if group["groupid"] == self.group_id:
-                self.logger.debug(f"Device {self.name}: hostgroup in-sync.")
+                self.logger.debug(f"Host {self.name}: hostgroup in-sync.")
                 break
         else:
-            self.logger.warning(f"Device {self.name}: hostgroup OUT of sync.")
+            self.logger.warning(f"Host {self.name}: hostgroup OUT of sync.")
             self.updateZabbixHost(groups={'groupid': self.group_id})
 
         if int(host["status"]) == self.zabbix_state:
-            self.logger.debug(f"Device {self.name}: status in-sync.")
+            self.logger.debug(f"Host {self.name}: status in-sync.")
         else:
-            self.logger.warning(f"Device {self.name}: status OUT of sync.")
+            self.logger.warning(f"Host {self.name}: status OUT of sync.")
             self.updateZabbixHost(status=str(self.zabbix_state))
         # Check if a proxy has been defined
         if self.zbxproxy:
             # Check if proxy or proxy group is defined
             if (self.zbxproxy["idtype"] in host and
                 host[self.zbxproxy["idtype"]] == self.zbxproxy["id"]):
-                self.logger.debug(f"Device {self.name}: proxy in-sync.")
+                self.logger.debug(f"Host {self.name}: proxy in-sync.")
             # Backwards compatibility for Zabbix <= 6
             elif "proxy_hostid" in host and host["proxy_hostid"] == self.zbxproxy["id"]:
-                self.logger.debug(f"Device {self.name}: proxy in-sync.")
+                self.logger.debug(f"Host {self.name}: proxy in-sync.")
             # Proxy does not match, update Zabbix
             else:
-                self.logger.warning(f"Device {self.name}: proxy OUT of sync.")
+                self.logger.warning(f"Host {self.name}: proxy OUT of sync.")
                 # Zabbix <= 6 patch
                 if not str(self.zabbix.version).startswith('7'):
                     self.updateZabbixHost(proxy_hostid=self.zbxproxy['id'])
@@ -647,7 +621,7 @@ class NetworkDevice():
                         proxy_set = True
             if proxy_power and proxy_set:
                 # Zabbix <= 6 fix
-                self.logger.warning(f"Device {self.name}: no proxy is configured in Netbox "
+                self.logger.warning(f"Host {self.name}: no proxy is configured in Netbox "
                                     "but is configured in Zabbix. Removing proxy config in Zabbix")
                 if "proxy_hostid" in host and bool(host["proxy_hostid"]):
                     self.updateZabbixHost(proxy_hostid=0)
@@ -660,24 +634,24 @@ class NetworkDevice():
             # Checks if a proxy has been defined in Zabbix and if proxy_power config has been set
             if proxy_set and not proxy_power:
                 # Display error message
-                self.logger.error(f"Device {self.name} is configured "
+                self.logger.error(f"Host {self.name} is configured "
                                     f"with proxy in Zabbix but not in Netbox. The"
                                     " -p flag was ommited: no "
                                     "changes have been made.")
             if not proxy_set:
-                self.logger.debug(f"Device {self.name}: proxy in-sync.")
+                self.logger.debug(f"Host {self.name}: proxy in-sync.")
         # Check host inventory mode
         if str(host['inventory_mode']) == str(self.inventory_mode):
-            self.logger.debug(f"Device {self.name}: inventory_mode in-sync.")
+            self.logger.debug(f"Host {self.name}: inventory_mode in-sync.")
         else:
-            self.logger.warning(f"Device {self.name}: inventory_mode OUT of sync.")
+            self.logger.warning(f"Host {self.name}: inventory_mode OUT of sync.")
             self.updateZabbixHost(inventory_mode=str(self.inventory_mode))
         if inventory_sync and self.inventory_mode in [0,1]:
             # Check host inventory mapping
             if host['inventory'] == self.inventory:
-                self.logger.debug(f"Device {self.name}: inventory in-sync.")
+                self.logger.debug(f"Host {self.name}: inventory in-sync.")
             else:
-                self.logger.warning(f"Device {self.name}: inventory OUT of sync.")
+                self.logger.warning(f"Host {self.name}: inventory OUT of sync.")
                 self.updateZabbixHost(inventory=self.inventory)
 
         # If only 1 interface has been found
@@ -715,10 +689,10 @@ class NetworkDevice():
                         updates[key] = item
             if updates:
                 # If interface updates have been found: push to Zabbix
-                self.logger.warning(f"Device {self.name}: Interface OUT of sync.")
+                self.logger.warning(f"Host {self.name}: Interface OUT of sync.")
                 if "type" in updates:
                     # Changing interface type not supported. Raise exception.
-                    e = (f"Device {self.name}: changing interface type to "
+                    e = (f"Host {self.name}: changing interface type to "
                          f"{str(updates['type'])} is not supported.")
                     self.logger.error(e)
                     raise InterfaceConfigError(e)
@@ -727,19 +701,19 @@ class NetworkDevice():
                 try:
                     # API call to Zabbix
                     self.zabbix.hostinterface.update(updates)
-                    e = f"Device {self.name}: solved interface conflict."
+                    e = f"Host {self.name}: solved interface conflict."
                     self.logger.info(e)
                     self.create_journal_entry("info", e)
                 except APIRequestError as e:
-                    e = f"Zabbix returned the following error: {str(e)}."
+                    msg = f"Zabbix returned the following error: {str(e)}."
-                    self.logger.error(e)
+                    self.logger.error(msg)
-                    raise SyncExternalError(e) from e
+                    raise SyncExternalError(msg) from e
             else:
                 # If no updates are found, Zabbix interface is in-sync
-                e = f"Device {self.name}: interface in-sync."
+                e = f"Host {self.name}: interface in-sync."
                 self.logger.debug(e)
         else:
-            e = (f"Device {self.name} has unsupported interface configuration."
+            e = (f"Host {self.name} has unsupported interface configuration."
                  f" Host has total of {len(host['interfaces'])} interfaces. "
                  "Manual interfention required.")
             self.logger.error(e)
@@ -762,7 +736,7 @@ class NetworkDevice():
                        }
             try:
                 self.nb_journals.create(journal)
-                self.logger.debug(f"Device {self.name}: Created journal entry in Netbox")
+                self.logger.debug(f"Host {self.name}: Created journal entry in Netbox")
                 return True
             except JournalError(e) as e:
                 self.logger.warning("Unable to create journal entry for "
@@ -789,7 +763,7 @@ class NetworkDevice():
                     # and add this NB template to the list of successfull templates
                     tmpls_from_zabbix.pop(pos)
                     succesfull_templates.append(nb_tmpl)
-                    self.logger.debug(f"Device {self.name}: template "
+                    self.logger.debug(f"Host {self.name}: template "
                                       f"{nb_tmpl['name']} is present in Zabbix.")
                     break
         if len(succesfull_templates) == len(self.zbx_templates) and len(tmpls_from_zabbix) == 0:

modules/hostgroups.py

@@ -0,0 +1,160 @@
+"""Module for all hostgroup related code"""
+from logging import getLogger
+from modules.exceptions import HostgroupError
+from modules.tools import build_path
+
+class Hostgroup():
+    """Hostgroup class for devices and VM's
+    Takes type (vm or dev) and NB object"""
+    def __init__(self, obj_type, nb_obj, version, logger=None):
+        self.logger = logger if logger else getLogger(__name__)
+        if obj_type not in ("vm", "dev"):
+            msg = f"Unable to create hostgroup with type {type}"
+            self.logger.error()
+            raise HostgroupError(msg)
+        self.type = str(obj_type)
+        self.nb = nb_obj
+        self.name = self.nb.name
+        self.nb_version = version
+        # Used for nested data objects
+        self.nested_objects = {}
+        self._set_format_options()
+
+    def __str__(self):
+        return f"Hostgroup for {self.type} {self.name}"
+
+    def __repr__(self):
+        return self.__str__()
+
+    def _set_format_options(self):
+        """
+        Set all available variables
+        for hostgroup generation
+        """
+        format_options = {}
+        # Set variables for both type of devices
+        if self.type in ("vm", "dev"):
+            # Role fix for Netbox <=3
+            role = None
+            if self.nb_version.startswith(("2", "3")) and self.type == "dev":
+                role = self.nb.device_role.name if self.nb.device_role else None
+            else:
+                role = self.nb.role.name if self.nb.role else None
+            # Add default formatting options
+            # Check if a site is configured. A site is optional for VMs
+            format_options["region"] = None
+            format_options["site_group"] = None
+            if self.nb.site:
+                if self.nb.site.region:
+                    format_options["region"] = self.generate_parents("region",
+                                                                     str(self.nb.site.region))
+                if self.nb.site.group:
+                    format_options["site_group"] = self.generate_parents("site_group",
+                                                                         str(self.nb.site.group))
+            format_options["role"] = role
+            format_options["site"] = self.nb.site.name if self.nb.site else None
+            format_options["tenant"] = str(self.nb.tenant) if self.nb.tenant else None
+            format_options["tenant_group"] = str(self.nb.tenant.group) if self.nb.tenant else None
+            format_options["platform"] = self.nb.platform.name if self.nb.platform else None
+        # Variables only applicable for devices
+        if self.type == "dev":
+            format_options["manufacturer"] = self.nb.device_type.manufacturer.name
+            format_options["location"] = str(self.nb.location) if self.nb.location else None
+        # Variables only applicable for VM's
+        if self.type == "vm":
+            # Check if a cluster is configured. Could also be configured in a site.
+            if self.nb.cluster:
+                format_options["cluster"] = self.nb.cluster.name
+                format_options["cluster_type"] = self.nb.cluster.type.name
+
+        self.format_options = format_options
+
+    def set_nesting(self, nested_sitegroup_flag, nested_region_flag,
+                    nb_groups, nb_regions):
+        """Set nesting options for this Hostgroup"""
+        self.nested_objects = {"site_group": {"flag": nested_sitegroup_flag, "data": nb_groups},
+                               "region": {"flag": nested_region_flag, "data": nb_regions}}
+
+    def generate(self, hg_format=None):
+        """Generate hostgroup based on a provided format"""
+        # Set format to default in case its not specified
+        if not hg_format:
+            hg_format = "site/manufacturer/role" if self.type == "dev" else "cluster/role"
+        # Split all given names
+        hg_output = []
+        hg_items = hg_format.split("/")
+        for hg_item in hg_items:
+            # Check if requested data is available as option for this host
+            if hg_item not in self.format_options:
+                # Check if a custom field exists with this name
+                cf_data = self.custom_field_lookup(hg_item)
+                # CF does not exist
+                if not cf_data["result"]:
+                    msg = (f"Unable to generate hostgroup for host {self.name}. "
+                           f"Item type {hg_item} not supported.")
+                    self.logger.error(msg)
+                    raise HostgroupError(msg)
+                # CF data is populated
+                if cf_data["cf"]:
+                    hg_output.append(cf_data["cf"])
+                continue
+            # Check if there is a value associated to the variable.
+            # For instance, if a device has no location, do not use it with hostgroup calculation
+            hostgroup_value = self.format_options[hg_item]
+            if hostgroup_value:
+                hg_output.append(hostgroup_value)
+        # Check if the hostgroup is populated with at least one item.
+        if bool(hg_output):
+            return "/".join(hg_output)
+        msg = (f"Unable to generate hostgroup for host {self.name}."
+               " Not enough valid items. This is most likely"
+               " due to the use of custom fields that are empty"
+               " or an invalid hostgroup format.")
+        self.logger.error(msg)
+        raise HostgroupError(msg)
+
+    def list_formatoptions(self):
+        """
+        Function to easily troubleshoot which values
+        are generated for a specific device or VM.
+        """
+        print(f"The following options are available for host {self.name}")
+        for option_type, value in self.format_options.items():
+            if value is not None:
+                print(f"{option_type} - {value}")
+        print("The following options are not available")
+        for option_type, value in self.format_options.items():
+            if value is None:
+                print(f"{option_type}")
+
+    def custom_field_lookup(self, hg_category):
+        """
+        Checks if a valid custom field is present in Netbox.
+        INPUT: Custom field name
+        OUTPUT: dictionary with 'result' and 'cf' keys.
+        """
+        # Check if the custom field exists
+        if hg_category not in self.nb.custom_fields:
+            return {"result": False, "cf": None}
+        # Checks if the custom field has been populated
+        if not bool(self.nb.custom_fields[hg_category]):
+            return {"result": True, "cf": None}
+        # Custom field exists and is populated
+        return {"result": True, "cf": self.nb.custom_fields[hg_category]}
+
+    def generate_parents(self, nest_type, child_object):
+        """
+        Generates parent objects to implement nested regions / nested site groups
+        INPUT: nest_type to set which type of nesting is going to be processed
+        child_object: the name of the child object (for instance the last NB region)
+        OUTPUT: STRING - Either the single child name or child and parents.
+        """
+        # Check if this type of nesting is supported.
+        if not nest_type in self.nested_objects:
+            return child_object
+        # If the nested flag is True, perform parent calculation
+        if self.nested_objects[nest_type]["flag"]:
+            final_nested_object = build_path(child_object, self.nested_objects[nest_type]["data"])
+            return "/".join(final_nested_object)
+        # Nesting is not allowed for this object. Return child_object
+        return child_object

modules/interface.py

@@ -90,7 +90,7 @@ class ZabbixInterface():
             e = "Interface type is not SNMP, unable to set SNMP details"
             raise InterfaceConfigError(e)
 
-    def set_default(self):
+    def set_default_snmp(self):
         """ Set default config to SNMPv2, port 161 and community macro. """
         self.interface = self.skelet
         self.interface["type"] = "2"
@@ -98,3 +98,8 @@ class ZabbixInterface():
         self.interface["details"] = {"version": "2",
                                      "community": "{$SNMP_COMMUNITY}",
                                      "bulk": "1"}
+
+    def set_default_agent(self):
+        """Sets interface to Zabbix agent defaults"""
+        self.interface["type"] = "1"
+        self.interface["port"] = "10050"

modules/virtual_machine.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+# pylint: disable=duplicate-code
+"""Module that hosts all functions for virtual machine processing"""
+
+from os import sys
+from modules.device import PhysicalDevice
+from modules.hostgroups import Hostgroup
+from modules.interface import ZabbixInterface
+from modules.exceptions import TemplateError, InterfaceConfigError, SyncInventoryError
+try:
+    from config import (
+        traverse_site_groups,
+        traverse_regions
+    )
+except ModuleNotFoundError:
+    print("Configuration file config.py not found in main directory."
+           "Please create the file or rename the config.py.example file to config.py.")
+    sys.exit(0)
+
+class VirtualMachine(PhysicalDevice):
+    """Model for virtual machines"""
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.hostgroup = None
+        self.zbx_template_names = None
+
+    def set_hostgroup(self, hg_format, nb_site_groups, nb_regions):
+        """Set the hostgroup for this device"""
+        # Create new Hostgroup instance
+        hg = Hostgroup("vm", self.nb, self.nb_api_version, logger=self.logger)
+        hg.set_nesting(traverse_site_groups, traverse_regions, nb_site_groups, nb_regions)
+        # Generate hostgroup based on hostgroup format
+        self.hostgroup = hg.generate(hg_format)
+
+    def set_vm_template(self):
+        """ Set Template for VMs. Overwrites default class
+        to skip a lookup of custom fields."""
+        # Gather templates ONLY from the device specific context
+        try:
+            self.zbx_template_names = self.get_templates_context()
+        except TemplateError as e:
+            self.logger.warning(e)
+        return True
+
+    def setInterfaceDetails(self): # pylint: disable=invalid-name
+        """
+        Overwrites device function to select an agent interface type by default
+        Agent type interfaces are more likely to be used with VMs then SNMP
+        """
+        try:
+            # Initiate interface class
+            interface = ZabbixInterface(self.nb.config_context, self.ip)
+            # Check if Netbox has device context.
+            # If not fall back to old config.
+            if interface.get_context():
+                # If device is SNMP type, add aditional information.
+                if interface.interface["type"] == 2:
+                    interface.set_snmp()
+            else:
+                interface.set_default_agent()
+            return [interface.interface]
+        except InterfaceConfigError as e:
+            message = f"{self.name}: {e}"
+            self.logger.warning(message)
+            raise SyncInventoryError(message) from e

netbox_zabbix_sync.py

@@ -6,8 +6,11 @@ import logging
 import argparse
 from os import environ, path, sys
 from pynetbox import api
+from pynetbox.core.query import RequestError as NBRequestError
+from requests.exceptions import ConnectionError as RequestsConnectionError
 from zabbix_utils import ZabbixAPI, APIRequestError, ProcessingError
-from modules.device import NetworkDevice
+from modules.device import PhysicalDevice
+from modules.virtual_machine import VirtualMachine
 from modules.tools import convert_recordset, proxy_prepper
 from modules.exceptions import EnvironmentVarError, HostgroupError, SyncError
 try:
@@ -19,7 +22,10 @@ try:
         zabbix_device_removal,
         zabbix_device_disable,
         hostgroup_format,
-        nb_device_filter
+        vm_hostgroup_format,
+        nb_device_filter,
+        sync_vms,
+        nb_vm_filter
     )
 except ModuleNotFoundError:
     print("Configuration file config.py not found in main directory."
@@ -76,10 +82,18 @@ def main(arguments):
     netbox = api(netbox_host, token=netbox_token, threading=True)
     # Check if the provided Hostgroup layout is valid
     hg_objects = hostgroup_format.split("/")
-    allowed_objects = ["dev_location", "dev_role", "manufacturer", "region",
+    allowed_objects = ["location", "role", "manufacturer", "region",
                         "site", "site_group", "tenant", "tenant_group"]
     # Create API call to get all custom fields which are on the device objects
-    device_cfs = netbox.extras.custom_fields.filter(type="text", content_type_id=23)
+    try:
+        device_cfs = list(netbox.extras.custom_fields.filter(type="text", content_type_id=23))
+    except RequestsConnectionError:
+        logger.error(f"Unable to connect to Netbox with URL {netbox_host}."
+                     " Please check the URL and status of Netbox.")
+        sys.exit(1)
+    except NBRequestError as e:
+        logger.error(f"Netbox error: {e}")
+        sys.exit(1)
     for cf in device_cfs:
         allowed_objects.append(cf.name)
     for hg_object in hg_objects:
@@ -105,7 +119,10 @@ def main(arguments):
     else:
         proxy_name = "name"
     # Get all Zabbix and Netbox data
-    netbox_devices = netbox.dcim.devices.filter(**nb_device_filter)
+    netbox_devices = list(netbox.dcim.devices.filter(**nb_device_filter))
+    netbox_vms = []
+    if sync_vms:
+        netbox_vms = list(netbox.virtualization.virtual_machines.filter(**nb_vm_filter))
     netbox_site_groups = convert_recordset((netbox.dcim.site_groups.all()))
     netbox_regions = convert_recordset(netbox.dcim.regions.all())
     netbox_journals = netbox.extras.journal_entries
@@ -127,13 +144,72 @@ def main(arguments):
     nb_version = netbox.version
 
     # Go through all Netbox devices
+    for nb_vm in netbox_vms:
+        try:
+            vm = VirtualMachine(nb_vm, zabbix, netbox_journals, nb_version,
+                                create_journal, logger)
+            logger.debug(f"Host {vm.name}: started operations on VM.")
+            vm.set_vm_template()
+            # Check if a valid template has been found for this VM.
+            if not vm.zbx_template_names:
+                continue
+            vm.set_hostgroup(vm_hostgroup_format,netbox_site_groups,netbox_regions)
+            # Check if a valid hostgroup has been found for this VM.
+            if not vm.hostgroup:
+                continue
+            # Temporary disable inventory sync for VM's
+            # vm.set_inventory(nb_vm)
+
+            # Checks if device is in cleanup state
+            if vm.status in zabbix_device_removal:
+                if vm.zabbix_id:
+                    # Delete device from Zabbix
+                    # and remove hostID from Netbox.
+                    vm.cleanup()
+                    logger.info(f"VM {vm.name}: cleanup complete")
+                    continue
+                # Device has been added to Netbox
+                # but is not in Activate state
+                logger.info(f"VM {vm.name}: skipping since this VM is "
+                            f"not in the active state.")
+                continue
+            # Check if the VM is in the disabled state
+            if vm.status in zabbix_device_disable:
+                vm.zabbix_state = 1
+            # Check if VM is already in Zabbix
+            if vm.zabbix_id:
+                vm.ConsistencyCheck(zabbix_groups, zabbix_templates,
+                                        zabbix_proxy_list, full_proxy_sync,
+                                        create_hostgroups)
+                continue
+            # Add hostgroup is config is set
+            if create_hostgroups:
+                # Create new hostgroup. Potentially multiple groups if nested
+                hostgroups = vm.createZabbixHostgroup(zabbix_groups)
+                # go through all newly created hostgroups
+                for group in hostgroups:
+                    # Add new hostgroups to zabbix group list
+                    zabbix_groups.append(group)
+            # Add VM to Zabbix
+            vm.createInZabbix(zabbix_groups, zabbix_templates,
+                                    zabbix_proxy_list)
+        except SyncError:
+            pass
+
     for nb_device in netbox_devices:
         try:
             # Set device instance set data such as hostgroup and template information.
-            device = NetworkDevice(nb_device, zabbix, netbox_journals, nb_version,
+            device = PhysicalDevice(nb_device, zabbix, netbox_journals, nb_version,
-                                   create_journal, logger)
+                                    create_journal, logger)
-            device.set_hostgroup(hostgroup_format,netbox_site_groups,netbox_regions)
+            logger.debug(f"Host {device.name}: started operations on device.")
             device.set_template(templates_config_context, templates_config_context_overrule)
+            # Check if a valid template has been found for this VM.
+            if not device.zbx_template_names:
+                continue
+            device.set_hostgroup(hostgroup_format,netbox_site_groups,netbox_regions)
+            # Check if a valid hostgroup has been found for this VM.
+            if not device.hostgroup:
+                continue
             device.set_inventory(nb_device)
             # Checks if device is part of cluster.
             # Requires clustering variable