NeoAnd/netbox
1
0
Fork 0
mirror of https://github.com/netbox-community/netbox.git synced 2025-07-22 03:56:53 -06:00
Code Issues Actions 8 Packages Projects Releases Wiki Activity

Merge pull request #5266 from netbox-community/4559-config-context-rendering

Browse Source 
4559 config context rendering
This commit is contained in:
Jeremy Stretch 2020-10-30 09:09:40 -04:00 committed by GitHub
commit 04d763d814
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 416 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
netbox/dcim/api/views.py
View File

@ -24,7 +24,7 @@ from dcim.models import (
VirtualChassis,
)
from extras.api.serializers import RenderedGraphSerializer
from extras.api.views import CustomFieldModelViewSet
from extras.api.views import ConfigContextQuerySetMixin, CustomFieldModelViewSet
from extras.models import Graph
from ipam.models import Prefix, VLAN
from utilities.api import (
@ -336,7 +336,7 @@ class PlatformViewSet(ModelViewSet):
# Devices
#
class DeviceViewSet(CustomFieldModelViewSet):
class DeviceViewSet(CustomFieldModelViewSet, ConfigContextQuerySetMixin):
queryset = Device.objects.prefetch_related(
'device_type__manufacturer', 'device_role', 'tenant', 'platform', 'site', 'rack', 'parent_bay',
'virtual_chassis__master', 'primary_ip4__nat_outside', 'primary_ip6__nat_outside', 'tags',

3
netbox/dcim/models/devices.py
View File

@ -15,6 +15,7 @@ from taggit.managers import TaggableManager
from dcim.choices import *
from dcim.constants import *
from extras.models import ChangeLoggedModel, ConfigContextModel, CustomFieldModel, TaggedItem
from extras.querysets import ConfigContextModelQuerySet
from extras.utils import extras_features
from utilities.choices import ColorChoices
from utilities.fields import ColorField, NaturalOrderingField
@ -594,7 +595,7 @@ class Device(ChangeLoggedModel, ConfigContextModel, CustomFieldModel):
)
tags = TaggableManager(through=TaggedItem)
objects = RestrictedQuerySet.as_manager()
objects = ConfigContextModelQuerySet.as_manager()
csv_headers = [
'name', 'device_role', 'tenant', 'manufacturer', 'device_type', 'platform', 'serial', 'asset_tag', 'status',

2
netbox/dcim/views.py
View File

@ -1163,7 +1163,7 @@ class DeviceConfigView(ObjectView):
class DeviceConfigContextView(ObjectConfigContextView):
queryset = Device.objects.all()
queryset = Device.objects.annotate_config_context_data()
base_template = 'dcim/device.html'

23
netbox/extras/api/views.py
View File

@ -26,6 +26,29 @@ from utilities.utils import copy_safe_request
from . import serializers
class ConfigContextQuerySetMixin:
"""
Used by views that work with config context models (device and virtual machine).
Provides a get_queryset() method which deals with adding the config context
data annotation or not.
"""
def get_queryset(self):
"""
Build the proper queryset based on the request context
If the `brief` query param equates to True or the `exclude` query param
includes `config_context` as a value, return the base queryset.
Else, return the queryset annotated with config context data
"""
request = self.get_serializer_context()['request']
if request.query_params.get('brief') or 'config_context' in request.query_params.get('exclude', []):
return self.queryset
return self.queryset.annotate_config_context_data()
class ExtrasRootView(APIRootView):
"""
Extras API root view

12
netbox/extras/models/models.py
View File

@ -542,8 +542,16 @@ class ConfigContextModel(models.Model):
# Compile all config data, overwriting lower-weight values with higher-weight values where a collision occurs
data = OrderedDict()
for context in ConfigContext.objects.get_for_object(self):
data = deepmerge(data, context.data)
if not hasattr(self, 'config_context_data'):
# The annotation is not available, so we fall back to manually querying for the config context objects
config_context_data = ConfigContext.objects.get_for_object(self, aggregate_data=True)
else:
# The attribute may exist, but the annotated value could be None if there is no config context data
config_context_data = self.config_context_data or []
for context in config_context_data:
data = deepmerge(data, context)
# If the object has local config context data defined, merge it last
if self.local_context_data:

79
netbox/extras/querysets.py
View File

@ -1,7 +1,8 @@
from collections import OrderedDict
from django.db.models import Q, QuerySet
from django.db.models import OuterRef, Subquery, Q
from utilities.query_functions import EmptyGroupByJSONBAgg, OrderableJSONBAgg
from utilities.querysets import RestrictedQuerySet
@ -23,9 +24,12 @@ class CustomFieldQueryset:
class ConfigContextQuerySet(RestrictedQuerySet):
def get_for_object(self, obj):
def get_for_object(self, obj, aggregate_data=False):
"""
Return all applicable ConfigContexts for a given object. Only active ConfigContexts will be included.
Args:
aggregate_data: If True, use the JSONBAgg aggregate function to return only the list of JSON data objects
"""
# `device_role` for Device; `role` for VirtualMachine
@ -45,7 +49,7 @@ class ConfigContextQuerySet(RestrictedQuerySet):
else:
regions = []
return self.filter(
queryset = self.filter(
Q(regions__in=regions) | Q(regions=None),
Q(sites=obj.site) | Q(sites=None),
Q(roles=role) | Q(roles=None),
@ -57,3 +61,72 @@ class ConfigContextQuerySet(RestrictedQuerySet):
Q(tags__slug__in=obj.tags.slugs()) | Q(tags=None),
is_active=True,
).order_by('weight', 'name')
if aggregate_data:
return queryset.aggregate(
config_context_data=OrderableJSONBAgg('data', ordering=['weight', 'name'])
)['config_context_data']
return queryset
class ConfigContextModelQuerySet(RestrictedQuerySet):
"""
QuerySet manager used by models which support ConfigContext (device and virtual machine).
Includes a method which appends an annotation of aggregated config context JSON data objects. This is
implemented as a subquery which performs all the joins necessary to filter relevant config context objects.
This offers a substantial performance gain over ConfigContextQuerySet.get_for_object() when dealing with
multiple objects.
This allows the annotation to be entirely optional.
"""
def annotate_config_context_data(self):
"""
Attach the subquery annotation to the base queryset
"""
from extras.models import ConfigContext
return self.annotate(
config_context_data=Subquery(
ConfigContext.objects.filter(
self._get_config_context_filters()
).annotate(
_data=EmptyGroupByJSONBAgg('data', ordering=['weight', 'name'])
).values("_data")
)
)
def _get_config_context_filters(self):
# Construct the set of Q objects for the specific object types
base_query = Q(
Q(platforms=OuterRef('platform')) | Q(platforms=None),
Q(tenant_groups=OuterRef('tenant__group')) | Q(tenant_groups=None),
Q(tenants=OuterRef('tenant')) | Q(tenants=None),
Q(tags=OuterRef('tags')) | Q(tags=None),
is_active=True,
)
if self.model._meta.model_name == 'device':
base_query.add((Q(roles=OuterRef('device_role')) | Q(roles=None)), Q.AND)
base_query.add((Q(sites=OuterRef('site')) | Q(sites=None)), Q.AND)
region_field = 'site__region'
elif self.model._meta.model_name == 'virtualmachine':
base_query.add((Q(roles=OuterRef('role')) | Q(roles=None)), Q.AND)
base_query.add((Q(cluster_groups=OuterRef('cluster__group')) | Q(cluster_groups=None)), Q.AND)
base_query.add((Q(clusters=OuterRef('cluster')) | Q(clusters=None)), Q.AND)
base_query.add((Q(sites=OuterRef('cluster__site')) | Q(sites=None)), Q.AND)
region_field = 'cluster__site__region'
base_query.add(
(Q(
regions__tree_id=OuterRef(f'{region_field}__tree_id'),
regions__level__lte=OuterRef(f'{region_field}__level'),
regions__lft__lte=OuterRef(f'{region_field}__lft'),
regions__rght__gte=OuterRef(f'{region_field}__rght'),
) | Q(regions=None)),
Q.AND
)
return base_query

279
netbox/extras/tests/test_models.py
View File

@ -1,9 +1,11 @@
from django.contrib.contenttypes.models import ContentType
from django.test import TestCase
from dcim.models import Site
from dcim.models import Device, DeviceRole, DeviceType, Manufacturer, Platform, Site, Region
from extras.choices import TemplateLanguageChoices
from extras.models import Graph, Tag
from extras.models import ConfigContext, Graph, Tag
from tenancy.models import Tenant, TenantGroup
from virtualization.models import Cluster, ClusterGroup, ClusterType, VirtualMachine
class GraphTest(TestCase):
@ -53,3 +55,276 @@ class TagTest(TestCase):
tag.save()
self.assertEqual(tag.slug, 'testing-unicode-台灣')
class ConfigContextTest(TestCase):
"""
These test cases deal with the weighting, ordering, and deep merge logic of config context data.
It also ensures the various config context querysets are consistent.
"""
def setUp(self):
manufacturer = Manufacturer.objects.create(name='Manufacturer 1', slug='manufacturer-1')
self.devicetype = DeviceType.objects.create(manufacturer=manufacturer, model='Device Type 1', slug='device-type-1')
self.devicerole = DeviceRole.objects.create(name='Device Role 1', slug='device-role-1')
self.region = Region.objects.create(name="Region")
self.site = Site.objects.create(name='Site-1', slug='site-1', region=self.region)
self.platform = Platform.objects.create(name="Platform")
self.tenantgroup = TenantGroup.objects.create(name="Tenant Group")
self.tenant = Tenant.objects.create(name="Tenant", group=self.tenantgroup)
self.tag = Tag.objects.create(name="Tag", slug="tag")
self.device = Device.objects.create(
name='Device 1',
device_type=self.devicetype,
device_role=self.devicerole,
site=self.site
)
def test_higher_weight_wins(self):
context1 = ConfigContext(
name="context 1",
weight=101,
data={
"a": 123,
"b": 456,
"c": 777
}
)
context2 = ConfigContext(
name="context 2",
weight=100,
data={
"a": 123,
"b": 456,
"c": 789
}
)
ConfigContext.objects.bulk_create([context1, context2])
expected_data = {
"a": 123,
"b": 456,
"c": 777
}
self.assertEqual(self.device.get_config_context(), expected_data)
def test_name_ordering_after_weight(self):
context1 = ConfigContext(
name="context 1",
weight=100,
data={
"a": 123,
"b": 456,
"c": 777
}
)
context2 = ConfigContext(
name="context 2",
weight=100,
data={
"a": 123,
"b": 456,
"c": 789
}
)
ConfigContext.objects.bulk_create([context1, context2])
expected_data = {
"a": 123,
"b": 456,
"c": 789
}
self.assertEqual(self.device.get_config_context(), expected_data)
def test_annotation_same_as_get_for_object(self):
"""
This test incorperates features from all of the above tests cases to ensure
the annotate_config_context_data() and get_for_object() queryset methods are the same.
"""
context1 = ConfigContext(
name="context 1",
weight=101,
data={
"a": 123,
"b": 456,
"c": 777
}
)
context2 = ConfigContext(
name="context 2",
weight=100,
data={
"a": 123,
"b": 456,
"c": 789
}
)
context3 = ConfigContext(
name="context 3",
weight=99,
data={
"d": 1
}
)
context4 = ConfigContext(
name="context 4",
weight=99,
data={
"d": 2
}
)
ConfigContext.objects.bulk_create([context1, context2, context3, context4])
annotated_queryset = Device.objects.filter(name=self.device.name).annotate_config_context_data()
self.assertEqual(self.device.get_config_context(), annotated_queryset[0].get_config_context())
def test_annotation_same_as_get_for_object_device_relations(self):
site_context = ConfigContext.objects.create(
name="site",
weight=100,
data={
"site": 1
}
)
site_context.sites.add(self.site)
region_context = ConfigContext.objects.create(
name="region",
weight=100,
data={
"region": 1
}
)
region_context.regions.add(self.region)
platform_context = ConfigContext.objects.create(
name="platform",
weight=100,
data={
"platform": 1
}
)
platform_context.platforms.add(self.platform)
tenant_group_context = ConfigContext.objects.create(
name="tenant group",
weight=100,
data={
"tenant_group": 1
}
)
tenant_group_context.tenant_groups.add(self.tenantgroup)
tenant_context = ConfigContext.objects.create(
name="tenant",
weight=100,
data={
"tenant": 1
}
)
tenant_context.tenants.add(self.tenant)
tag_context = ConfigContext.objects.create(
name="tag",
weight=100,
data={
"tag": 1
}
)
tag_context.tags.add(self.tag)
device = Device.objects.create(
name="Device 2",
site=self.site,
tenant=self.tenant,
platform=self.platform,
device_role=self.devicerole,
device_type=self.devicetype
)
device.tags.add(self.tag)
annotated_queryset = Device.objects.filter(name=device.name).annotate_config_context_data()
self.assertEqual(device.get_config_context(), annotated_queryset[0].get_config_context())
def test_annotation_same_as_get_for_object_virtualmachine_relations(self):
site_context = ConfigContext.objects.create(
name="site",
weight=100,
data={
"site": 1
}
)
site_context.sites.add(self.site)
region_context = ConfigContext.objects.create(
name="region",
weight=100,
data={
"region": 1
}
)
region_context.regions.add(self.region)
platform_context = ConfigContext.objects.create(
name="platform",
weight=100,
data={
"platform": 1
}
)
platform_context.platforms.add(self.platform)
tenant_group_context = ConfigContext.objects.create(
name="tenant group",
weight=100,
data={
"tenant_group": 1
}
)
tenant_group_context.tenant_groups.add(self.tenantgroup)
tenant_context = ConfigContext.objects.create(
name="tenant",
weight=100,
data={
"tenant": 1
}
)
tenant_context.tenants.add(self.tenant)
tag_context = ConfigContext.objects.create(
name="tag",
weight=100,
data={
"tag": 1
}
)
tag_context.tags.add(self.tag)
cluster_group = ClusterGroup.objects.create(name="Cluster Group")
cluster_group_context = ConfigContext.objects.create(
name="cluster group",
weight=100,
data={
"cluster_group": 1
}
)
cluster_group_context.cluster_groups.add(cluster_group)
cluster_type = ClusterType.objects.create(name="Cluster Type 1")
cluster = Cluster.objects.create(name="Cluster", group=cluster_group, type=cluster_type)
cluster_context = ConfigContext.objects.create(
name="cluster",
weight=100,
data={
"cluster": 1
}
)
cluster_context.clusters.add(cluster)
virtual_machine = VirtualMachine.objects.create(
name="VM 1",
cluster=cluster,
tenant=self.tenant,
platform=self.platform,
role=self.devicerole
)
virtual_machine.tags.add(self.tag)
annotated_queryset = VirtualMachine.objects.filter(name=virtual_machine.name).annotate_config_context_data()
self.assertEqual(virtual_machine.get_config_context(), annotated_queryset[0].get_config_context())

20
netbox/utilities/query_functions.py
View File

@ -1,3 +1,5 @@
from django.contrib.postgres.aggregates import JSONBAgg
from django.contrib.postgres.aggregates.mixins import OrderableAggMixin
from django.db.models import F, Func
@ -7,3 +9,21 @@ class CollateAsChar(Func):
"""
function = 'C'
template = '(%(expressions)s) COLLATE "%(function)s"'
class OrderableJSONBAgg(OrderableAggMixin, JSONBAgg):
"""
TODO in Django 3.2 ordering is supported natively on JSONBAgg so this is no longer needed.
"""
template = '%(function)s(%(distinct)s%(expressions)s %(ordering)s)'
class EmptyGroupByJSONBAgg(OrderableJSONBAgg):
"""
JSONBAgg is a builtin aggregation function which means it includes the use of a GROUP BY clause.
When used as an annotation for collecting config context data objects, the GROUP BY is
incorrect. This subclass overrides the Django ORM aggregation control to remove the GROUP BY.
TODO in Django 3.2 ordering is supported natively on JSONBAgg so we only need to inherit from JSONBAgg.
"""
contains_aggregate = False

4
netbox/virtualization/api/views.py
View File

@ -6,7 +6,7 @@ from rest_framework.routers import APIRootView
from dcim.models import Device
from extras.api.serializers import RenderedGraphSerializer
from extras.api.views import CustomFieldModelViewSet
from extras.api.views import ConfigContextQuerySetMixin, CustomFieldModelViewSet
from extras.models import Graph
from utilities.api import ModelViewSet
from utilities.utils import get_subquery
@ -58,7 +58,7 @@ class ClusterViewSet(CustomFieldModelViewSet):
# Virtual machines
#
class VirtualMachineViewSet(CustomFieldModelViewSet):
class VirtualMachineViewSet(CustomFieldModelViewSet, ConfigContextQuerySetMixin):
queryset = VirtualMachine.objects.prefetch_related(
'cluster__site', 'role', 'tenant', 'platform', 'primary_ip4', 'primary_ip6', 'tags'
)

3
netbox/virtualization/models.py
View File

@ -8,6 +8,7 @@ from taggit.managers import TaggableManager
from dcim.choices import InterfaceModeChoices
from dcim.models import BaseInterface, Device
from extras.models import ChangeLoggedModel, ConfigContextModel, CustomFieldModel, ObjectChange, TaggedItem
from extras.querysets import ConfigContextModelQuerySet
from extras.utils import extras_features
from utilities.fields import NaturalOrderingField
from utilities.ordering import naturalize_interface
@ -282,7 +283,7 @@ class VirtualMachine(ChangeLoggedModel, ConfigContextModel, CustomFieldModel):
)
tags = TaggableManager(through=TaggedItem)
objects = RestrictedQuerySet.as_manager()
objects = ConfigContextModelQuerySet.as_manager()
csv_headers = [
'name', 'status', 'role', 'cluster', 'tenant', 'platform', 'vcpus', 'memory', 'disk', 'comments',

2
netbox/virtualization/views.py
View File

@ -261,7 +261,7 @@ class VirtualMachineView(ObjectView):
class VirtualMachineConfigContextView(ObjectConfigContextView):
queryset = VirtualMachine.objects.all()
queryset = VirtualMachine.objects.annotate_config_context_data()
base_template = 'virtualization/virtualmachine.html'