From ac87ce733deb366605a5aeefa68f1b8cebddea28 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Wed, 8 Feb 2023 18:24:18 -0500 Subject: [PATCH] Closes #11693: Enable remote data synchronization for export templates --- docs/models/extras/exporttemplate.md | 4 + netbox/extras/api/serializers.py | 9 +- netbox/extras/api/views.py | 4 +- netbox/extras/filtersets.py | 10 +- netbox/extras/forms/filtersets.py | 16 ++- netbox/extras/forms/model_forms.py | 21 ++- .../0086_exporttemplate_synced_data.py | 35 +++++ netbox/extras/models/models.py | 12 +- netbox/extras/tables/tables.py | 13 +- netbox/extras/urls.py | 1 + netbox/extras/views.py | 6 + netbox/templates/extras/configcontext.html | 26 +--- netbox/templates/extras/exporttemplate.html | 134 +++++++++++------- .../templates/extras/exporttemplate_list.html | 10 ++ netbox/templates/inc/sync_warning.html | 13 ++ netbox/utilities/templates/buttons/sync.html | 6 + netbox/utilities/templatetags/buttons.py | 10 ++ netbox/utilities/templatetags/perms.py | 5 + 18 files changed, 246 insertions(+), 89 deletions(-) create mode 100644 netbox/extras/migrations/0086_exporttemplate_synced_data.py create mode 100644 netbox/templates/extras/exporttemplate_list.html create mode 100644 netbox/templates/inc/sync_warning.html create mode 100644 netbox/utilities/templates/buttons/sync.html diff --git a/docs/models/extras/exporttemplate.md b/docs/models/extras/exporttemplate.md index 3215201b3..d2f9292c6 100644 --- a/docs/models/extras/exporttemplate.md +++ b/docs/models/extras/exporttemplate.md @@ -12,6 +12,10 @@ The name of the export template. This will appear in the "export" dropdown list The type of NetBox object to which the export template applies. +### Data File + +Template code may optionally be sourced from a remote [data file](../core/datafile.md), which is synchronized from a remote data source. When designating a data file, there is no need to specify local content for the template: It will be populated automatically from the data file. + ### Template Code Jinja2 template code for rendering the exported data. diff --git a/netbox/extras/api/serializers.py b/netbox/extras/api/serializers.py index 54627fbb3..6a8248548 100644 --- a/netbox/extras/api/serializers.py +++ b/netbox/extras/api/serializers.py @@ -142,12 +142,19 @@ class ExportTemplateSerializer(ValidatedModelSerializer): queryset=ContentType.objects.filter(FeatureQuery('export_templates').get_query()), many=True ) + data_source = NestedDataSourceSerializer( + required=False + ) + data_file = NestedDataFileSerializer( + read_only=True + ) class Meta: model = ExportTemplate fields = [ 'id', 'url', 'display', 'content_types', 'name', 'description', 'template_code', 'mime_type', - 'file_extension', 'as_attachment', 'created', 'last_updated', + 'file_extension', 'as_attachment', 'data_source', 'data_path', 'data_file', 'data_synced', 'created', + 'last_updated', ] diff --git a/netbox/extras/api/views.py b/netbox/extras/api/views.py index 8b97491b1..190b32f53 100644 --- a/netbox/extras/api/views.py +++ b/netbox/extras/api/views.py @@ -92,9 +92,9 @@ class CustomLinkViewSet(NetBoxModelViewSet): # Export templates # -class ExportTemplateViewSet(NetBoxModelViewSet): +class ExportTemplateViewSet(SyncedDataMixin, NetBoxModelViewSet): metadata_class = ContentTypeMetadata - queryset = ExportTemplate.objects.all() + queryset = ExportTemplate.objects.prefetch_related('data_source', 'data_file') serializer_class = serializers.ExportTemplateSerializer filterset_class = filtersets.ExportTemplateFilterSet diff --git a/netbox/extras/filtersets.py b/netbox/extras/filtersets.py index 799e79123..f7f34e17a 100644 --- a/netbox/extras/filtersets.py +++ b/netbox/extras/filtersets.py @@ -127,10 +127,18 @@ class ExportTemplateFilterSet(BaseFilterSet): field_name='content_types__id' ) content_types = ContentTypeFilter() + data_source_id = django_filters.ModelMultipleChoiceFilter( + queryset=DataSource.objects.all(), + label=_('Data source (ID)'), + ) + data_file_id = django_filters.ModelMultipleChoiceFilter( + queryset=DataSource.objects.all(), + label=_('Data file (ID)'), + ) class Meta: model = ExportTemplate - fields = ['id', 'content_types', 'name', 'description'] + fields = ['id', 'content_types', 'name', 'description', 'data_synced'] def search(self, queryset, name, value): if not value.strip(): diff --git a/netbox/extras/forms/filtersets.py b/netbox/extras/forms/filtersets.py index 46b7aa8f6..5c7a10ac8 100644 --- a/netbox/extras/forms/filtersets.py +++ b/netbox/extras/forms/filtersets.py @@ -21,9 +21,9 @@ from .mixins import SavedFiltersMixin __all__ = ( 'ConfigContextFilterForm', 'CustomFieldFilterForm', - 'JobResultFilterForm', 'CustomLinkFilterForm', 'ExportTemplateFilterForm', + 'JobResultFilterForm', 'JournalEntryFilterForm', 'LocalConfigContextFilterForm', 'ObjectChangeFilterForm', @@ -157,8 +157,22 @@ class CustomLinkFilterForm(SavedFiltersMixin, FilterForm): class ExportTemplateFilterForm(SavedFiltersMixin, FilterForm): fieldsets = ( (None, ('q', 'filter_id')), + ('Data', ('data_source_id', 'data_file_id')), ('Attributes', ('content_types', 'mime_type', 'file_extension', 'as_attachment')), ) + data_source_id = DynamicModelMultipleChoiceField( + queryset=DataSource.objects.all(), + required=False, + label=_('Data source') + ) + data_file_id = DynamicModelMultipleChoiceField( + queryset=DataFile.objects.all(), + required=False, + label=_('Data file'), + query_params={ + 'source_id': '$data_source_id' + } + ) content_types = ContentTypeMultipleChoiceField( queryset=ContentType.objects.filter(FeatureQuery('export_templates').get_query()), required=False diff --git a/netbox/extras/forms/model_forms.py b/netbox/extras/forms/model_forms.py index 429c4140a..0ffc5117c 100644 --- a/netbox/extras/forms/model_forms.py +++ b/netbox/extras/forms/model_forms.py @@ -96,19 +96,28 @@ class ExportTemplateForm(BootstrapMixin, forms.ModelForm): queryset=ContentType.objects.all(), limit_choices_to=FeatureQuery('export_templates') ) + template_code = forms.CharField( + required=False, + widget=forms.Textarea(attrs={'class': 'font-monospace'}) + ) fieldsets = ( ('Export Template', ('name', 'content_types', 'description')), - ('Template', ('template_code',)), + ('Content', ('data_source', 'data_file', 'template_code',)), ('Rendering', ('mime_type', 'file_extension', 'as_attachment')), ) class Meta: model = ExportTemplate fields = '__all__' - widgets = { - 'template_code': forms.Textarea(attrs={'class': 'font-monospace'}), - } + + def clean(self): + super().clean() + + if not self.cleaned_data.get('template_code') and not self.cleaned_data.get('data_file'): + raise forms.ValidationError("Must specify either local content or a data file") + + return self.cleaned_data class SavedFilterForm(BootstrapMixin, forms.ModelForm): @@ -261,8 +270,8 @@ class ConfigContextForm(BootstrapMixin, SyncedDataMixin, forms.ModelForm): def clean(self): super().clean() - if not self.cleaned_data.get('data') and not self.cleaned_data.get('data_source'): - raise forms.ValidationError("Must specify either local data or a data source") + if not self.cleaned_data.get('data') and not self.cleaned_data.get('data_file'): + raise forms.ValidationError("Must specify either local data or a data file") return self.cleaned_data diff --git a/netbox/extras/migrations/0086_exporttemplate_synced_data.py b/netbox/extras/migrations/0086_exporttemplate_synced_data.py new file mode 100644 index 000000000..87de6b71c --- /dev/null +++ b/netbox/extras/migrations/0086_exporttemplate_synced_data.py @@ -0,0 +1,35 @@ +# Generated by Django 4.1.6 on 2023-02-08 22:16 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0001_initial'), + ('extras', '0085_configcontext_synced_data'), + ] + + operations = [ + migrations.AddField( + model_name='exporttemplate', + name='data_file', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='core.datafile'), + ), + migrations.AddField( + model_name='exporttemplate', + name='data_path', + field=models.CharField(blank=True, editable=False, max_length=1000), + ), + migrations.AddField( + model_name='exporttemplate', + name='data_source', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='core.datasource'), + ), + migrations.AddField( + model_name='exporttemplate', + name='data_synced', + field=models.DateTimeField(blank=True, editable=False, null=True), + ), + ] diff --git a/netbox/extras/models/models.py b/netbox/extras/models/models.py index df32d6ac4..63a1e199e 100644 --- a/netbox/extras/models/models.py +++ b/netbox/extras/models/models.py @@ -26,7 +26,8 @@ from netbox.config import get_config from netbox.constants import RQ_QUEUE_DEFAULT from netbox.models import ChangeLoggedModel from netbox.models.features import ( - CloningMixin, CustomFieldsMixin, CustomLinksMixin, ExportTemplatesMixin, JobResultsMixin, TagsMixin, WebhooksMixin, + CloningMixin, CustomFieldsMixin, CustomLinksMixin, ExportTemplatesMixin, JobResultsMixin, SyncedDataMixin, + TagsMixin, WebhooksMixin, ) from utilities.querysets import RestrictedQuerySet from utilities.utils import render_jinja2 @@ -281,7 +282,7 @@ class CustomLink(CloningMixin, ExportTemplatesMixin, WebhooksMixin, ChangeLogged } -class ExportTemplate(ExportTemplatesMixin, WebhooksMixin, ChangeLoggedModel): +class ExportTemplate(SyncedDataMixin, ExportTemplatesMixin, WebhooksMixin, ChangeLoggedModel): content_types = models.ManyToManyField( to=ContentType, related_name='export_templates', @@ -335,6 +336,13 @@ class ExportTemplate(ExportTemplatesMixin, WebhooksMixin, ChangeLoggedModel): 'name': f'"{self.name}" is a reserved name. Please choose a different name.' }) + def sync_data(self): + """ + Synchronize template content from the designated DataFile (if any). + """ + self.template_code = self.data_file.data_as_string + self.data_synced = timezone.now() + def render(self, queryset): """ Render the contents of the template. diff --git a/netbox/extras/tables/tables.py b/netbox/extras/tables/tables.py index 51443ad87..6b2f34de4 100644 --- a/netbox/extras/tables/tables.py +++ b/netbox/extras/tables/tables.py @@ -90,15 +90,24 @@ class ExportTemplateTable(NetBoxTable): ) content_types = columns.ContentTypesColumn() as_attachment = columns.BooleanColumn() + data_source = tables.Column( + linkify=True + ) + data_file = tables.Column( + linkify=True + ) + is_synced = columns.BooleanColumn( + verbose_name='Synced' + ) class Meta(NetBoxTable.Meta): model = ExportTemplate fields = ( 'pk', 'id', 'name', 'content_types', 'description', 'mime_type', 'file_extension', 'as_attachment', - 'created', 'last_updated', + 'data_source', 'data_file', 'data_synced', 'created', 'last_updated', ) default_columns = ( - 'pk', 'name', 'content_types', 'description', 'mime_type', 'file_extension', 'as_attachment', + 'pk', 'name', 'content_types', 'description', 'mime_type', 'file_extension', 'as_attachment', 'is_synced', ) diff --git a/netbox/extras/urls.py b/netbox/extras/urls.py index 6fd178284..dabb9f977 100644 --- a/netbox/extras/urls.py +++ b/netbox/extras/urls.py @@ -29,6 +29,7 @@ urlpatterns = [ path('export-templates/import/', views.ExportTemplateBulkImportView.as_view(), name='exporttemplate_import'), path('export-templates/edit/', views.ExportTemplateBulkEditView.as_view(), name='exporttemplate_bulk_edit'), path('export-templates/delete/', views.ExportTemplateBulkDeleteView.as_view(), name='exporttemplate_bulk_delete'), + path('export-templates/sync/', views.ExportTemplateBulkSyncDataView.as_view(), name='exporttemplate_bulk_sync'), path('export-templates//', include(get_model_urls('extras', 'exporttemplate'))), # Saved filters diff --git a/netbox/extras/views.py b/netbox/extras/views.py index c46890c19..de06b5739 100644 --- a/netbox/extras/views.py +++ b/netbox/extras/views.py @@ -121,6 +121,8 @@ class ExportTemplateListView(generic.ObjectListView): filterset = filtersets.ExportTemplateFilterSet filterset_form = forms.ExportTemplateFilterForm table = tables.ExportTemplateTable + template_name = 'extras/exporttemplate_list.html' + actions = ('add', 'import', 'export', 'bulk_edit', 'bulk_delete', 'bulk_sync') @register_model_view(ExportTemplate) @@ -158,6 +160,10 @@ class ExportTemplateBulkDeleteView(generic.BulkDeleteView): table = tables.ExportTemplateTable +class ExportTemplateBulkSyncDataView(generic.BulkSyncDataView): + queryset = ExportTemplate.objects.all() + + # # Saved filters # diff --git a/netbox/templates/extras/configcontext.html b/netbox/templates/extras/configcontext.html index 3714b3f1c..e9513a3a8 100644 --- a/netbox/templates/extras/configcontext.html +++ b/netbox/templates/extras/configcontext.html @@ -50,10 +50,10 @@ {% endif %} - - Data Synced - {{ object.data_synced|placeholder }} - + + Data Synced + {{ object.data_synced|placeholder }} + @@ -86,22 +86,8 @@ {% include 'extras/inc/configcontext_format.html' %}
- {% if object.data_file and object.data_file.last_updated > object.data_synced %} - - {% endif %} - {% include 'extras/inc/configcontext_data.html' with data=object.data format=format %} + {% include 'inc/sync_warning.html' %} + {% include 'extras/inc/configcontext_data.html' with data=object.data format=format %}
diff --git a/netbox/templates/extras/exporttemplate.html b/netbox/templates/extras/exporttemplate.html index d14294355..a80db8fca 100644 --- a/netbox/templates/extras/exporttemplate.html +++ b/netbox/templates/extras/exporttemplate.html @@ -10,66 +10,92 @@ {% endblock %} {% block content %} -
-
-
-
- Export Template -
-
- - - - - - - - - - - - - - - - - - - - - -
Name{{ object.name }}
Description{{ object.description|placeholder }}
MIME Type{{ object.mime_type|placeholder }}
File Extension{{ object.file_extension|placeholder }}
Attachment{% checkmark object.as_attachment %}
-
-
-
-
Assigned Models
-
- - {% for ct in object.content_types.all %} +
+
+
+
Export Template
+
+
- + + - {% endfor %} -
{{ ct }}Name{{ object.name }}
+ + Description + {{ object.description|placeholder }} + + + MIME Type + {{ object.mime_type|placeholder }} + + + File Extension + {{ object.file_extension|placeholder }} + + + Attachment + {% checkmark object.as_attachment %} + + + Data Source + + {% if object.data_source %} + {{ object.data_source }} + {% else %} + {{ ''|placeholder }} + {% endif %} + + + + Data File + + {% if object.data_file %} + {{ object.data_file }} + {% elif object.data_path %} +
+ +
+ {{ object.data_path }} + {% else %} + {{ ''|placeholder }} + {% endif %} + + + + Data Synced + {{ object.data_synced|placeholder }} + + +
-
- {% plugin_left_page object %} -
-
-
-
- Template -
-
- 
{{ object.template_code }}
+
+
Assigned Models
+
+ + {% for ct in object.content_types.all %} + + + + {% endfor %} +
{{ ct }}
+
+ {% plugin_left_page object %} +
+
+
+
Template
+
+ {% include 'inc/sync_warning.html' %} + 
{{ object.template_code }}
+
+
+ {% plugin_right_page object %}
- {% plugin_right_page object %}
-
-
+
- {% plugin_full_width_page object %} + {% plugin_full_width_page object %}
-
+
{% endblock %} diff --git a/netbox/templates/extras/exporttemplate_list.html b/netbox/templates/extras/exporttemplate_list.html new file mode 100644 index 000000000..c79f9259a --- /dev/null +++ b/netbox/templates/extras/exporttemplate_list.html @@ -0,0 +1,10 @@ +{% extends 'generic/object_list.html' %} + +{% block bulk_buttons %} + {% if perms.extras.sync_configcontext %} + + {% endif %} + {{ block.super }} +{% endblock %} diff --git a/netbox/templates/inc/sync_warning.html b/netbox/templates/inc/sync_warning.html new file mode 100644 index 000000000..1ffc77e15 --- /dev/null +++ b/netbox/templates/inc/sync_warning.html @@ -0,0 +1,13 @@ +{% load buttons %} +{% load perms %} + +{% if object.data_file and object.data_file.last_updated > object.data_synced %} + +{% endif %} diff --git a/netbox/utilities/templates/buttons/sync.html b/netbox/utilities/templates/buttons/sync.html new file mode 100644 index 000000000..58f2b95cc --- /dev/null +++ b/netbox/utilities/templates/buttons/sync.html @@ -0,0 +1,6 @@ +
+ {% csrf_token %} + +
diff --git a/netbox/utilities/templatetags/buttons.py b/netbox/utilities/templatetags/buttons.py index bcdb099d8..8a706ebeb 100644 --- a/netbox/utilities/templatetags/buttons.py +++ b/netbox/utilities/templatetags/buttons.py @@ -46,6 +46,16 @@ def delete_button(instance): } +@register.inclusion_tag('buttons/sync.html') +def sync_button(instance): + viewname = get_viewname(instance, 'sync') + url = reverse(viewname, kwargs={'pk': instance.pk}) + + return { + 'url': url, + } + + # # List buttons # diff --git a/netbox/utilities/templatetags/perms.py b/netbox/utilities/templatetags/perms.py index f1bbf7549..809c74ad1 100644 --- a/netbox/utilities/templatetags/perms.py +++ b/netbox/utilities/templatetags/perms.py @@ -28,3 +28,8 @@ def can_change(user, instance): @register.filter() def can_delete(user, instance): return _check_permission(user, instance, 'delete') + + +@register.filter() +def can_sync(user, instance): + return _check_permission(user, instance, 'sync')