From f5e09c6f48c7acd8b29a17178430d46418c8f11d Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Thu, 9 Feb 2023 16:32:32 -0500 Subject: [PATCH] WIP --- netbox/extras/api/serializers.py | 21 +++++ netbox/extras/api/urls.py | 1 + netbox/extras/api/views.py | 21 +++++ netbox/extras/filtersets.py | 31 +++++++- netbox/extras/forms/bulk_edit.py | 14 ++++ netbox/extras/forms/bulk_import.py | 10 +++ netbox/extras/forms/filtersets.py | 21 +++++ netbox/extras/forms/model_forms.py | 29 +++++++ netbox/extras/graphql/schema.py | 3 + netbox/extras/graphql/types.py | 9 +++ .../extras/migrations/0086_configtemplate.py | 34 +++++++++ netbox/extras/models/__init__.py | 3 +- .../models/{configcontexts.py => configs.py} | 63 ++++++++++++++- netbox/extras/tables/tables.py | 25 ++++++ netbox/extras/urls.py | 8 ++ netbox/extras/views.py | 52 +++++++++++++ netbox/netbox/navigation/menu.py | 1 + netbox/templates/extras/configtemplate.html | 76 +++++++++++++++++++ .../templates/extras/configtemplate_list.html | 10 +++ netbox/utilities/jinja2.py | 27 +++++++ 20 files changed, 455 insertions(+), 4 deletions(-) create mode 100644 netbox/extras/migrations/0086_configtemplate.py rename netbox/extras/models/{configcontexts.py => configs.py} (74%) create mode 100644 netbox/templates/extras/configtemplate.html create mode 100644 netbox/templates/extras/configtemplate_list.html create mode 100644 netbox/utilities/jinja2.py diff --git a/netbox/extras/api/serializers.py b/netbox/extras/api/serializers.py index 6a8248548..aee729945 100644 --- a/netbox/extras/api/serializers.py +++ b/netbox/extras/api/serializers.py @@ -383,6 +383,27 @@ class ConfigContextSerializer(ValidatedModelSerializer): ] +# +# Config templates +# + +class ConfigTemplateSerializer(ValidatedModelSerializer): + url = serializers.HyperlinkedIdentityField(view_name='extras-api:configtemplate-detail') + data_source = NestedDataSourceSerializer( + required=False + ) + data_file = NestedDataFileSerializer( + read_only=True + ) + + class Meta: + model = ConfigTemplate + fields = [ + 'id', 'url', 'display', 'name', 'description', 'environment_params', 'template_code', 'data_source', + 'data_path', 'data_file', 'data_synced', 'created', 'last_updated', + ] + + # # Job Results # diff --git a/netbox/extras/api/urls.py b/netbox/extras/api/urls.py index 91067d40d..f01cdcd00 100644 --- a/netbox/extras/api/urls.py +++ b/netbox/extras/api/urls.py @@ -14,6 +14,7 @@ router.register('tags', views.TagViewSet) router.register('image-attachments', views.ImageAttachmentViewSet) router.register('journal-entries', views.JournalEntryViewSet) router.register('config-contexts', views.ConfigContextViewSet) +router.register('config-templates', views.ConfigTemplateViewSet) router.register('reports', views.ReportViewSet, basename='report') router.register('scripts', views.ScriptViewSet, basename='script') router.register('object-changes', views.ObjectChangeViewSet) diff --git a/netbox/extras/api/views.py b/netbox/extras/api/views.py index 190b32f53..2755378a6 100644 --- a/netbox/extras/api/views.py +++ b/netbox/extras/api/views.py @@ -157,6 +157,27 @@ class ConfigContextViewSet(SyncedDataMixin, NetBoxModelViewSet): filterset_class = filtersets.ConfigContextFilterSet +# +# Config templates +# + +class ConfigTemplateViewSet(SyncedDataMixin, NetBoxModelViewSet): + queryset = ConfigTemplate.objects.prefetch_related('data_source', 'data_file') + serializer_class = serializers.ConfigTemplateSerializer + filterset_class = filtersets.ConfigTemplateFilterSet + + @action(detail=True, methods=['post']) + def render(self, request, pk): + """ + Render a ConfigTemplate using the context data provided (if any). + """ + configtemplate = self.get_object() + output = configtemplate.render(context=request.data) + + # TODO: Create a proper serializer + return Response({"output": output}) + + # # Reports # diff --git a/netbox/extras/filtersets.py b/netbox/extras/filtersets.py index f7f34e17a..38c828916 100644 --- a/netbox/extras/filtersets.py +++ b/netbox/extras/filtersets.py @@ -4,7 +4,7 @@ from django.contrib.contenttypes.models import ContentType from django.db.models import Q from django.utils.translation import gettext as _ -from core.models import DataFile, DataSource +from core.models import DataSource from dcim.models import DeviceRole, DeviceType, Location, Platform, Region, Site, SiteGroup from netbox.filtersets import BaseFilterSet, ChangeLoggedModelFilterSet, NetBoxModelFilterSet from tenancy.models import Tenant, TenantGroup @@ -13,9 +13,9 @@ from virtualization.models import Cluster, ClusterGroup, ClusterType from .choices import * from .models import * - __all__ = ( 'ConfigContextFilterSet', + 'ConfigTemplateFilterSet', 'ContentTypeFilterSet', 'CustomFieldFilterSet', 'CustomLinkFilterSet', @@ -454,6 +454,33 @@ class ConfigContextFilterSet(ChangeLoggedModelFilterSet): ) +class ConfigTemplateFilterSet(BaseFilterSet): + q = django_filters.CharFilter( + method='search', + label=_('Search'), + ) + data_source_id = django_filters.ModelMultipleChoiceFilter( + queryset=DataSource.objects.all(), + label=_('Data source (ID)'), + ) + data_file_id = django_filters.ModelMultipleChoiceFilter( + queryset=DataSource.objects.all(), + label=_('Data file (ID)'), + ) + + class Meta: + model = ConfigTemplate + fields = ['id', 'name', 'description', 'data_synced'] + + def search(self, queryset, name, value): + if not value.strip(): + return queryset + return queryset.filter( + Q(name__icontains=value) | + Q(description__icontains=value) + ) + + # # Filter for Local Config Context Data # diff --git a/netbox/extras/forms/bulk_edit.py b/netbox/extras/forms/bulk_edit.py index 47a529772..bba585591 100644 --- a/netbox/extras/forms/bulk_edit.py +++ b/netbox/extras/forms/bulk_edit.py @@ -9,6 +9,7 @@ from utilities.forms import ( __all__ = ( 'ConfigContextBulkEditForm', + 'ConfigTemplateBulkEditForm', 'CustomFieldBulkEditForm', 'CustomLinkBulkEditForm', 'ExportTemplateBulkEditForm', @@ -201,6 +202,19 @@ class ConfigContextBulkEditForm(BulkEditForm): nullable_fields = ('description',) +class ConfigTemplateBulkEditForm(BulkEditForm): + pk = forms.ModelMultipleChoiceField( + queryset=ConfigTemplate.objects.all(), + widget=forms.MultipleHiddenInput + ) + description = forms.CharField( + max_length=200, + required=False + ) + + nullable_fields = ('description',) + + class JournalEntryBulkEditForm(BulkEditForm): pk = forms.ModelMultipleChoiceField( queryset=JournalEntry.objects.all(), diff --git a/netbox/extras/forms/bulk_import.py b/netbox/extras/forms/bulk_import.py index cf723c4f7..1998170b0 100644 --- a/netbox/extras/forms/bulk_import.py +++ b/netbox/extras/forms/bulk_import.py @@ -10,6 +10,7 @@ from extras.utils import FeatureQuery from utilities.forms import CSVChoiceField, CSVContentTypeField, CSVModelForm, CSVMultipleContentTypeField, SlugField __all__ = ( + 'ConfigTemplateImportForm', 'CustomFieldImportForm', 'CustomLinkImportForm', 'ExportTemplateImportForm', @@ -83,6 +84,15 @@ class ExportTemplateImportForm(CSVModelForm): ) +class ConfigTemplateImportForm(CSVModelForm): + + class Meta: + model = ConfigTemplate + fields = ( + 'name', 'description', 'environment_params', 'template_code', + ) + + class SavedFilterImportForm(CSVModelForm): content_types = CSVMultipleContentTypeField( queryset=ContentType.objects.all(), diff --git a/netbox/extras/forms/filtersets.py b/netbox/extras/forms/filtersets.py index 685eac9ba..c77f334f9 100644 --- a/netbox/extras/forms/filtersets.py +++ b/netbox/extras/forms/filtersets.py @@ -20,6 +20,7 @@ from .mixins import SavedFiltersMixin __all__ = ( 'ConfigContextFilterForm', + 'ConfigTemplateFilterForm', 'CustomFieldFilterForm', 'CustomLinkFilterForm', 'ExportTemplateFilterForm', @@ -364,6 +365,26 @@ class ConfigContextFilterForm(SavedFiltersMixin, FilterForm): ) +class ConfigTemplateFilterForm(SavedFiltersMixin, FilterForm): + fieldsets = ( + (None, ('q', 'filter_id')), + ('Data', ('data_source_id', 'data_file_id')), + ) + data_source_id = DynamicModelMultipleChoiceField( + queryset=DataSource.objects.all(), + required=False, + label=_('Data source') + ) + data_file_id = DynamicModelMultipleChoiceField( + queryset=DataFile.objects.all(), + required=False, + label=_('Data file'), + query_params={ + 'source_id': '$data_source_id' + } + ) + + class LocalConfigContextFilterForm(forms.Form): local_context_data = forms.NullBooleanField( required=False, diff --git a/netbox/extras/forms/model_forms.py b/netbox/extras/forms/model_forms.py index 69c124ee2..bb9430616 100644 --- a/netbox/extras/forms/model_forms.py +++ b/netbox/extras/forms/model_forms.py @@ -18,6 +18,7 @@ from virtualization.models import Cluster, ClusterGroup, ClusterType __all__ = ( 'ConfigContextForm', + 'ConfigTemplateForm', 'CustomFieldForm', 'CustomLinkForm', 'ExportTemplateForm', @@ -269,6 +270,34 @@ class ConfigContextForm(BootstrapMixin, SyncedDataMixin, forms.ModelForm): return self.cleaned_data +class ConfigTemplateForm(BootstrapMixin, SyncedDataMixin, forms.ModelForm): + tags = DynamicModelMultipleChoiceField( + queryset=Tag.objects.all(), + required=False + ) + template_code = forms.CharField( + required=False, + widget=forms.Textarea(attrs={'class': 'font-monospace'}) + ) + + fieldsets = ( + ('Export Template', ('name', 'description', 'environment_params', 'tags')), + ('Content', ('data_source', 'data_file', 'template_code',)), + ) + + class Meta: + model = ConfigTemplate + fields = '__all__' + + def clean(self): + super().clean() + + if not self.cleaned_data.get('template_code') and not self.cleaned_data.get('data_file'): + raise forms.ValidationError("Must specify either local content or a data file") + + return self.cleaned_data + + class ImageAttachmentForm(BootstrapMixin, forms.ModelForm): class Meta: diff --git a/netbox/extras/graphql/schema.py b/netbox/extras/graphql/schema.py index 0c3113879..3e116023f 100644 --- a/netbox/extras/graphql/schema.py +++ b/netbox/extras/graphql/schema.py @@ -8,6 +8,9 @@ class ExtrasQuery(graphene.ObjectType): config_context = ObjectField(ConfigContextType) config_context_list = ObjectListField(ConfigContextType) + config_template = ObjectField(ConfigTemplateType) + config_template_list = ObjectListField(ConfigTemplateType) + custom_field = ObjectField(CustomFieldType) custom_field_list = ObjectListField(CustomFieldType) diff --git a/netbox/extras/graphql/types.py b/netbox/extras/graphql/types.py index b5d4dffce..ba16ccd3e 100644 --- a/netbox/extras/graphql/types.py +++ b/netbox/extras/graphql/types.py @@ -4,6 +4,7 @@ from netbox.graphql.types import BaseObjectType, ObjectType __all__ = ( 'ConfigContextType', + 'ConfigTemplateType', 'CustomFieldType', 'CustomLinkType', 'ExportTemplateType', @@ -24,6 +25,14 @@ class ConfigContextType(ObjectType): filterset_class = filtersets.ConfigContextFilterSet +class ConfigTemplateType(ObjectType): + + class Meta: + model = models.ConfigTemplate + fields = '__all__' + filterset_class = filtersets.ConfigTemplateFilterSet + + class CustomFieldType(ObjectType): class Meta: diff --git a/netbox/extras/migrations/0086_configtemplate.py b/netbox/extras/migrations/0086_configtemplate.py new file mode 100644 index 000000000..4dbc88f9f --- /dev/null +++ b/netbox/extras/migrations/0086_configtemplate.py @@ -0,0 +1,34 @@ +# Generated by Django 4.1.6 on 2023-02-09 19:33 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0001_initial'), + ('extras', '0085_synced_data'), + ] + + operations = [ + migrations.CreateModel( + name='ConfigTemplate', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False)), + ('created', models.DateTimeField(auto_now_add=True, null=True)), + ('last_updated', models.DateTimeField(auto_now=True, null=True)), + ('data_path', models.CharField(blank=True, editable=False, max_length=1000)), + ('data_synced', models.DateTimeField(blank=True, editable=False, null=True)), + ('name', models.CharField(max_length=100)), + ('description', models.CharField(blank=True, max_length=200)), + ('template_code', models.TextField()), + ('environment_params', models.JSONField(blank=True, null=True)), + ('data_file', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='core.datafile')), + ('data_source', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='core.datasource')), + ], + options={ + 'ordering': ('name',), + }, + ), + ] diff --git a/netbox/extras/models/__init__.py b/netbox/extras/models/__init__.py index 9b5c660c4..33936cc4f 100644 --- a/netbox/extras/models/__init__.py +++ b/netbox/extras/models/__init__.py @@ -1,5 +1,5 @@ from .change_logging import ObjectChange -from .configcontexts import ConfigContext, ConfigContextModel +from .configs import * from .customfields import CustomField from .models import * from .search import * @@ -12,6 +12,7 @@ __all__ = ( 'ConfigContext', 'ConfigContextModel', 'ConfigRevision', + 'ConfigTemplate', 'CustomField', 'CustomLink', 'ExportTemplate', diff --git a/netbox/extras/models/configcontexts.py b/netbox/extras/models/configs.py similarity index 74% rename from netbox/extras/models/configcontexts.py rename to netbox/extras/models/configs.py index eed8babcd..8c275d732 100644 --- a/netbox/extras/models/configcontexts.py +++ b/netbox/extras/models/configs.py @@ -3,15 +3,20 @@ from django.core.validators import ValidationError from django.db import models from django.urls import reverse from django.utils import timezone +from django.utils.translation import gettext as _ +from jinja2.sandbox import SandboxedEnvironment from extras.querysets import ConfigContextQuerySet +from netbox.config import get_config from netbox.models import ChangeLoggedModel -from netbox.models.features import SyncedDataMixin +from netbox.models.features import ExportTemplatesMixin, SyncedDataMixin +from utilities.jinja2 import ConfigTemplateLoader from utilities.utils import deepmerge __all__ = ( 'ConfigContext', 'ConfigContextModel', + 'ConfigTemplate', ) @@ -182,3 +187,59 @@ class ConfigContextModel(models.Model): raise ValidationError( {'local_context_data': 'JSON data must be in object form. Example: {"foo": 123}'} ) + + +# +# Config templates +# + +class ConfigTemplate(SyncedDataMixin, ExportTemplatesMixin, ChangeLoggedModel): + name = models.CharField( + max_length=100 + ) + description = models.CharField( + max_length=200, + blank=True + ) + template_code = models.TextField( + help_text=_('Jinja2 template code.') + ) + environment_params = models.JSONField( + blank=True, + null=True + ) + + class Meta: + ordering = ('name',) + + def __str__(self): + return self.name + + def get_absolute_url(self): + return reverse('extras:configtemplate', args=[self.pk]) + + def sync_data(self): + """ + Synchronize template content from the designated DataFile (if any). + """ + self.template_code = self.data_file.data_as_string + self.data_synced = timezone.now() + + def render(self, context=None): + """ + Render the contents of the template. + """ + context = context or {} + template_code = self.template_code + # output = render_jinja2(template_code, context) + + environment = SandboxedEnvironment( + loader=ConfigTemplateLoader(data_source=self.data_source) + ) + environment.filters.update(get_config().JINJA2_FILTERS) + output = environment.from_string(source=template_code).render(**context) + + # Replace CRLF-style line terminators + output = output.replace('\r\n', '\n') + + return output diff --git a/netbox/extras/tables/tables.py b/netbox/extras/tables/tables.py index 6b2f34de4..b4c584188 100644 --- a/netbox/extras/tables/tables.py +++ b/netbox/extras/tables/tables.py @@ -8,6 +8,7 @@ from .template_code import * __all__ = ( 'ConfigContextTable', + 'ConfigTemplateTable', 'CustomFieldTable', 'CustomLinkTable', 'ExportTemplateTable', @@ -223,6 +224,30 @@ class ConfigContextTable(NetBoxTable): default_columns = ('pk', 'name', 'weight', 'is_active', 'is_synced', 'description') +class ConfigTemplateTable(NetBoxTable): + name = tables.Column( + linkify=True + ) + data_source = tables.Column( + linkify=True + ) + data_file = tables.Column( + linkify=True + ) + is_synced = columns.BooleanColumn( + verbose_name='Synced' + ) + + class Meta(NetBoxTable.Meta): + model = ConfigTemplate + fields = ( + 'pk', 'id', 'name', 'description', 'data_source', 'data_file', 'data_synced', 'created', 'last_updated', + ) + default_columns = ( + 'pk', 'name', 'description', 'is_synced', + ) + + class ObjectChangeTable(NetBoxTable): time = tables.DateTimeColumn( linkify=True, diff --git a/netbox/extras/urls.py b/netbox/extras/urls.py index dabb9f977..dfbaa1bc6 100644 --- a/netbox/extras/urls.py +++ b/netbox/extras/urls.py @@ -64,6 +64,14 @@ urlpatterns = [ path('config-contexts/sync/', views.ConfigContextBulkSyncDataView.as_view(), name='configcontext_bulk_sync'), path('config-contexts//', include(get_model_urls('extras', 'configcontext'))), + # Config templates + path('config-templates/', views.ConfigTemplateListView.as_view(), name='configtemplate_list'), + path('config-templates/add/', views.ConfigTemplateEditView.as_view(), name='configtemplate_add'), + path('config-templates/edit/', views.ConfigTemplateBulkEditView.as_view(), name='configtemplate_bulk_edit'), + path('config-templates/delete/', views.ConfigTemplateBulkDeleteView.as_view(), name='configtemplate_bulk_delete'), + path('config-templates/sync/', views.ConfigTemplateBulkSyncDataView.as_view(), name='configtemplate_bulk_sync'), + path('config-templates//', include(get_model_urls('extras', 'configtemplate'))), + # Image attachments path('image-attachments/add/', views.ImageAttachmentEditView.as_view(), name='imageattachment_add'), path('image-attachments//', include(get_model_urls('extras', 'imageattachment'))), diff --git a/netbox/extras/views.py b/netbox/extras/views.py index de06b5739..3edb70cf1 100644 --- a/netbox/extras/views.py +++ b/netbox/extras/views.py @@ -452,6 +452,58 @@ class ObjectConfigContextView(generic.ObjectView): } +# +# Config templates +# + +class ConfigTemplateListView(generic.ObjectListView): + queryset = ConfigTemplate.objects.all() + filterset = filtersets.ConfigTemplateFilterSet + filterset_form = forms.ConfigTemplateFilterForm + table = tables.ConfigTemplateTable + template_name = 'extras/configtemplate_list.html' + actions = ('add', 'import', 'export', 'bulk_edit', 'bulk_delete', 'bulk_sync') + + +@register_model_view(ConfigTemplate) +class ConfigTemplateView(generic.ObjectView): + queryset = ConfigTemplate.objects.all() + + +@register_model_view(ConfigTemplate, 'edit') +class ConfigTemplateEditView(generic.ObjectEditView): + queryset = ConfigTemplate.objects.all() + form = forms.ConfigTemplateForm + + +@register_model_view(ConfigTemplate, 'delete') +class ConfigTemplateDeleteView(generic.ObjectDeleteView): + queryset = ConfigTemplate.objects.all() + + +class ConfigTemplateBulkImportView(generic.BulkImportView): + queryset = ConfigTemplate.objects.all() + model_form = forms.ConfigTemplateImportForm + table = tables.ConfigTemplateTable + + +class ConfigTemplateBulkEditView(generic.BulkEditView): + queryset = ConfigTemplate.objects.all() + filterset = filtersets.ConfigTemplateFilterSet + table = tables.ConfigTemplateTable + form = forms.ConfigTemplateBulkEditForm + + +class ConfigTemplateBulkDeleteView(generic.BulkDeleteView): + queryset = ConfigTemplate.objects.all() + filterset = filtersets.ConfigTemplateFilterSet + table = tables.ConfigTemplateTable + + +class ConfigTemplateBulkSyncDataView(generic.BulkSyncDataView): + queryset = ConfigTemplate.objects.all() + + # # Change logging # diff --git a/netbox/netbox/navigation/menu.py b/netbox/netbox/navigation/menu.py index 6fce7dfe6..03c361002 100644 --- a/netbox/netbox/navigation/menu.py +++ b/netbox/netbox/navigation/menu.py @@ -311,6 +311,7 @@ OTHER_MENU = Menu( items=( get_model_item('extras', 'tag', 'Tags'), get_model_item('extras', 'configcontext', _('Config Contexts'), actions=['add']), + get_model_item('extras', 'configtemplate', _('Config Templates'), actions=['add']), ), ), ), diff --git a/netbox/templates/extras/configtemplate.html b/netbox/templates/extras/configtemplate.html new file mode 100644 index 000000000..57178acb9 --- /dev/null +++ b/netbox/templates/extras/configtemplate.html @@ -0,0 +1,76 @@ +{% extends 'generic/object.html' %} +{% load helpers %} +{% load plugins %} + +{% block content %} +
+
+
+
Config Template
+
+ + + + + + + + + + + + + + + + + + + + + +
Name{{ object.name }}
Description{{ object.description|placeholder }}
Data Source + {% if object.data_source %} + {{ object.data_source }} + {% else %} + {{ ''|placeholder }} + {% endif %} +
Data File + {% if object.data_file %} + {{ object.data_file }} + {% elif object.data_path %} +
+ +
+ {{ object.data_path }} + {% else %} + {{ ''|placeholder }} + {% endif %} +
Data Synced{{ object.data_synced|placeholder }}
+
+
+ {% plugin_left_page object %} +
+
+
+
Environment Parameters
+
+ 
{{ object.environment_params }}
+
+
+ {% plugin_right_page object %} +
+
+
+
+
+
Template
+
+ {% include 'inc/sync_warning.html' %} + 
{{ object.template_code }}
+
+
+ {% plugin_full_width_page object %} +
+
+{% endblock %} diff --git a/netbox/templates/extras/configtemplate_list.html b/netbox/templates/extras/configtemplate_list.html new file mode 100644 index 000000000..5a3e8781e --- /dev/null +++ b/netbox/templates/extras/configtemplate_list.html @@ -0,0 +1,10 @@ +{% extends 'generic/object_list.html' %} + +{% block bulk_buttons %} + {% if perms.extras.sync_configtemplate %} + + {% endif %} + {{ block.super }} +{% endblock %} diff --git a/netbox/utilities/jinja2.py b/netbox/utilities/jinja2.py new file mode 100644 index 000000000..81d11cc85 --- /dev/null +++ b/netbox/utilities/jinja2.py @@ -0,0 +1,27 @@ +import os + +from django.apps import apps +from jinja2 import BaseLoader, TemplateNotFound + +__all__ = ( + 'ConfigTemplateLoader', +) + + +class ConfigTemplateLoader(BaseLoader): + """ + Custom Jinja2 loader to facilitate populating template content from DataFiles. + """ + def __init__(self, data_source): + self.data_source = data_source + + def get_source(self, environment, template): + DataFile = apps.get_model('core', 'DataFile') + + try: + datafile = DataFile.objects.get(source=self.data_source, path=template) + template_source = datafile.data_as_string + except DataFile.DoesNotExist: + raise TemplateNotFound(template) + + return template_source, template, True