WIP

netbox/core/models/data.py

@@ -1,5 +1,6 @@
 import logging
 import os
+import yaml
 from fnmatch import fnmatchcase
 from urllib.parse import urlparse
 
@@ -283,6 +284,13 @@ class DataFile(ChangeLoggingMixin, models.Model):
         except UnicodeDecodeError:
             return None
 
+    def get_data(self):
+        """
+        Attempt to read the file data as JSON/YAML and return a native Python object.
+        """
+        # TODO: Something more robust
+        return yaml.safe_load(self.data_as_string)
+
     def refresh_from_disk(self, source_root):
         """
         Update instance attributes from the file on disk. Returns True if any attribute

netbox/extras/api/serializers.py

@@ -4,6 +4,7 @@ from django.core.exceptions import ObjectDoesNotExist
 from drf_yasg.utils import swagger_serializer_method
 from rest_framework import serializers
 
+from core.api.nested_serializers import NestedDataSourceSerializer, NestedDataFileSerializer
 from dcim.api.nested_serializers import (
     NestedDeviceRoleSerializer, NestedDeviceTypeSerializer, NestedLocationSerializer, NestedPlatformSerializer,
     NestedRegionSerializer, NestedSiteSerializer, NestedSiteGroupSerializer,
@@ -358,13 +359,18 @@ class ConfigContextSerializer(ValidatedModelSerializer):
         required=False,
         many=True
     )
+    data_source = NestedDataSourceSerializer()
+    data_file = NestedDataFileSerializer(
+        read_only=True
+    )
 
     class Meta:
         model = ConfigContext
         fields = [
             'id', 'url', 'display', 'name', 'weight', 'description', 'is_active', 'regions', 'site_groups', 'sites',
             'locations', 'device_types', 'roles', 'platforms', 'cluster_types', 'cluster_groups', 'clusters',
-            'tenant_groups', 'tenants', 'tags', 'data', 'created', 'last_updated',
+            'tenant_groups', 'tenants', 'tags', 'data_source', 'data_path', 'data_file', 'data_synced', 'data',
+            'created', 'last_updated',
         ]
 
 

netbox/extras/constants.py

@@ -8,6 +8,7 @@ EXTRAS_FEATURES = [
     'export_templates',
     'job_results',
     'journaling',
+    'synced_data',
     'tags',
     'webhooks'
 ]

netbox/extras/filtersets.py

@@ -4,6 +4,7 @@ from django.contrib.contenttypes.models import ContentType
 from django.db.models import Q
 from django.utils.translation import gettext as _
 
+from core.models import DataFile, DataSource
 from dcim.models import DeviceRole, DeviceType, Location, Platform, Region, Site, SiteGroup
 from netbox.filtersets import BaseFilterSet, ChangeLoggedModelFilterSet, NetBoxModelFilterSet
 from tenancy.models import Tenant, TenantGroup
@@ -422,10 +423,18 @@ class ConfigContextFilterSet(ChangeLoggedModelFilterSet):
         to_field_name='slug',
         label=_('Tag (slug)'),
     )
+    data_source_id = django_filters.ModelMultipleChoiceFilter(
+        queryset=DataSource.objects.all(),
+        label=_('Data source (ID)'),
+    )
+    data_file_id = django_filters.ModelMultipleChoiceFilter(
+        queryset=DataSource.objects.all(),
+        label=_('Data file (ID)'),
+    )
 
     class Meta:
         model = ConfigContext
-        fields = ['id', 'name', 'is_active']
+        fields = ['id', 'name', 'is_active', 'data_synced']
 
     def search(self, queryset, name, value):
         if not value.strip():

netbox/extras/forms/filtersets.py

@@ -3,6 +3,7 @@ from django.contrib.auth.models import User
 from django.contrib.contenttypes.models import ContentType
 from django.utils.translation import gettext as _
 
+from core.models import DataFile, DataSource
 from dcim.models import DeviceRole, DeviceType, Location, Platform, Region, Site, SiteGroup
 from extras.choices import *
 from extras.models import *
@@ -263,11 +264,25 @@ class TagFilterForm(SavedFiltersMixin, FilterForm):
 class ConfigContextFilterForm(SavedFiltersMixin, FilterForm):
     fieldsets = (
         (None, ('q', 'filter_id', 'tag_id')),
+        ('Data', ('data_source_id', 'data_file_id')),
         ('Location', ('region_id', 'site_group_id', 'site_id', 'location_id')),
         ('Device', ('device_type_id', 'platform_id', 'role_id')),
         ('Cluster', ('cluster_type_id', 'cluster_group_id', 'cluster_id')),
         ('Tenant', ('tenant_group_id', 'tenant_id'))
     )
+    data_source_id = DynamicModelMultipleChoiceField(
+        queryset=DataSource.objects.all(),
+        required=False,
+        label=_('Data source')
+    )
+    data_file_id = DynamicModelMultipleChoiceField(
+        queryset=DataFile.objects.all(),
+        required=False,
+        label=_('Data file'),
+        query_params={
+            'source_id': '$data_source_id'
+        }
+    )
     region_id = DynamicModelMultipleChoiceField(
         queryset=Region.objects.all(),
         required=False,

netbox/extras/forms/mixins.py

@@ -2,13 +2,15 @@ from django.contrib.contenttypes.models import ContentType
 from django import forms
 from django.utils.translation import gettext as _
 
+from core.models import DataFile, DataSource
 from extras.models import *
 from extras.choices import CustomFieldVisibilityChoices
-from utilities.forms.fields import DynamicModelMultipleChoiceField
+from utilities.forms.fields import DynamicModelChoiceField, DynamicModelMultipleChoiceField
 
 __all__ = (
     'CustomFieldsMixin',
     'SavedFiltersMixin',
+    'SyncedDataMixin',
 )
 
 
@@ -72,3 +74,19 @@ class SavedFiltersMixin(forms.Form):
             'usable': True,
         }
     )
+
+
+class SyncedDataMixin(forms.Form):
+    data_source = DynamicModelChoiceField(
+        queryset=DataSource.objects.all(),
+        required=False,
+        label=_('Data source')
+    )
+    data_file = DynamicModelChoiceField(
+        queryset=DataFile.objects.all(),
+        required=False,
+        label=_('File'),
+        query_params={
+            'source_id': '$data_source',
+        }
+    )

netbox/extras/forms/model_forms.py

@@ -5,6 +5,7 @@ from django.utils.translation import gettext as _
 
 from dcim.models import DeviceRole, DeviceType, Location, Platform, Region, Site, SiteGroup
 from extras.choices import *
+from extras.forms.mixins import SyncedDataMixin
 from extras.models import *
 from extras.utils import FeatureQuery
 from netbox.forms import NetBoxModelForm
@@ -183,7 +184,7 @@ class TagForm(BootstrapMixin, forms.ModelForm):
         ]
 
 
-class ConfigContextForm(BootstrapMixin, forms.ModelForm):
+class ConfigContextForm(BootstrapMixin, SyncedDataMixin, forms.ModelForm):
     regions = DynamicModelMultipleChoiceField(
         queryset=Region.objects.all(),
         required=False
@@ -236,10 +237,13 @@ class ConfigContextForm(BootstrapMixin, forms.ModelForm):
         queryset=Tag.objects.all(),
         required=False
     )
-    data = JSONField()
+    data = JSONField(
+        required=False
+    )
 
     fieldsets = (
         ('Config Context', ('name', 'weight', 'description', 'data', 'is_active')),
+        ('Data Source', ('data_source', 'data_file')),
         ('Assignment', (
             'regions', 'site_groups', 'sites', 'locations', 'device_types', 'roles', 'platforms', 'cluster_types',
             'cluster_groups', 'clusters', 'tenant_groups', 'tenants', 'tags',
@@ -251,9 +255,23 @@ class ConfigContextForm(BootstrapMixin, forms.ModelForm):
         fields = (
             'name', 'weight', 'description', 'data', 'is_active', 'regions', 'site_groups', 'sites', 'locations',
             'roles', 'device_types', 'platforms', 'cluster_types', 'cluster_groups', 'clusters', 'tenant_groups',
-            'tenants', 'tags',
+            'tenants', 'tags', 'data_source', 'data_file',
         )
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        # if self.instance and self.instance.data_file:
+        #     self.fields['data'].widget.attrs.update({'disabled': True})
+
+    def clean(self):
+        super().clean()
+
+        if not self.cleaned_data.get('data') and not self.cleaned_data.get('data_source'):
+            raise forms.ValidationError("Must specify either local data or a data source")
+
+        return self.cleaned_data
+
 
 class ImageAttachmentForm(BootstrapMixin, forms.ModelForm):
 

netbox/extras/migrations/0085_configcontext_data_file_configcontext_data_path_and_more.py

@@ -0,0 +1,35 @@
+# Generated by Django 4.1.6 on 2023-02-06 15:34
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0001_initial'),
+        ('extras', '0084_staging'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='configcontext',
+            name='data_file',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='core.datafile'),
+        ),
+        migrations.AddField(
+            model_name='configcontext',
+            name='data_path',
+            field=models.CharField(blank=True, editable=False, max_length=1000),
+        ),
+        migrations.AddField(
+            model_name='configcontext',
+            name='data_source',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='core.datasource'),
+        ),
+        migrations.AddField(
+            model_name='configcontext',
+            name='data_synced',
+            field=models.DateTimeField(blank=True, editable=False, null=True),
+        ),
+    ]

netbox/extras/models/configcontexts.py

@@ -2,10 +2,11 @@ from django.conf import settings
 from django.core.validators import ValidationError
 from django.db import models
 from django.urls import reverse
+from django.utils import timezone
 
 from extras.querysets import ConfigContextQuerySet
 from netbox.models import ChangeLoggedModel
-from netbox.models.features import WebhooksMixin
+from netbox.models.features import SyncedDataMixin, WebhooksMixin
 from utilities.utils import deepmerge
 
 
@@ -19,7 +20,7 @@ __all__ = (
 # Config contexts
 #
 
-class ConfigContext(WebhooksMixin, ChangeLoggedModel):
+class ConfigContext(SyncedDataMixin, WebhooksMixin, ChangeLoggedModel):
     """
     A ConfigContext represents a set of arbitrary data available to any Device or VirtualMachine matching its assigned
     qualifiers (region, site, etc.). For example, the data stored in a ConfigContext assigned to site A and tenant B
@@ -130,6 +131,13 @@ class ConfigContext(WebhooksMixin, ChangeLoggedModel):
                 {'data': 'JSON data must be in object form. Example: {"foo": 123}'}
             )
 
+    def sync_data(self):
+        """
+        Synchronize context data from the designated DataFile (if any).
+        """
+        self.data = self.data_file.get_data()
+        self.data_synced = timezone.now()
+
 
 class ConfigContextModel(models.Model):
     """

netbox/extras/tables/tables.py

@@ -188,6 +188,12 @@ class TaggedItemTable(NetBoxTable):
 
 
 class ConfigContextTable(NetBoxTable):
+    data_source = tables.Column(
+        linkify=True
+    )
+    data_file = tables.Column(
+        linkify=True
+    )
     name = tables.Column(
         linkify=True
     )
@@ -199,8 +205,8 @@ class ConfigContextTable(NetBoxTable):
         model = ConfigContext
         fields = (
             'pk', 'id', 'name', 'weight', 'is_active', 'description', 'regions', 'sites', 'locations', 'roles',
-            'platforms', 'cluster_types', 'cluster_groups', 'clusters', 'tenant_groups', 'tenants', 'created',
+            'platforms', 'cluster_types', 'cluster_groups', 'clusters', 'tenant_groups', 'tenants', 'data_source',
-            'last_updated',
+            'data_file', 'data_synced', 'created', 'last_updated',
         )
         default_columns = ('pk', 'name', 'weight', 'is_active', 'description')
 

netbox/netbox/models/features.py

@@ -2,11 +2,12 @@ from collections import defaultdict
 from functools import cached_property
 
 from django.contrib.contenttypes.fields import GenericRelation
-from django.db.models.signals import class_prepared
-from django.dispatch import receiver
-
 from django.core.validators import ValidationError
 from django.db import models
+from django.db.models.signals import class_prepared
+from django.dispatch import receiver
+from django.utils.translation import gettext as _
+
 from taggit.managers import TaggableManager
 
 from extras.choices import CustomFieldVisibilityChoices, ObjectChangeActionChoices
@@ -25,6 +26,7 @@ __all__ = (
     'ExportTemplatesMixin',
     'JobResultsMixin',
     'JournalingMixin',
+    'SyncedDataMixin',
     'TagsMixin',
     'WebhooksMixin',
 )
@@ -313,12 +315,78 @@ class WebhooksMixin(models.Model):
         abstract = True
 
 
+class SyncedDataMixin(models.Model):
+    """
+    Enables population of local data from a DataFile object, synchronized from a remote DatSource.
+    """
+    data_source = models.ForeignKey(
+        to='core.DataSource',
+        on_delete=models.PROTECT,
+        blank=True,
+        null=True,
+        related_name='+',
+        help_text=_("Remote data source")
+    )
+    data_file = models.ForeignKey(
+        to='core.DataFile',
+        on_delete=models.SET_NULL,
+        blank=True,
+        null=True,
+        related_name='+'
+    )
+    data_path = models.CharField(
+        max_length=1000,
+        blank=True,
+        editable=False,
+        help_text=_("Path to remote file (relative to data source root)")
+    )
+    data_synced = models.DateTimeField(
+        blank=True,
+        null=True,
+        editable=False
+    )
+
+    class Meta:
+        abstract = True
+
+    def clean(self):
+        if self.data_file:
+            self.sync_data()
+            self.data_path = self.data_file.path
+
+        if self.data_source and not self.data_file:
+            raise ValidationError({
+                'data_file': _(f"Must specify a data file when designating a data source.")
+            })
+        if self.data_file and not self.data_source:
+            self.data_source = self.data_file.source
+
+        super().clean()
+
+    def resolve_data_file(self):
+        """
+        Determine the designated DataFile object identified by its parent DataSource and its path. Returns None if
+        either attribute is unset, or if no matching DataFile is found.
+        """
+        from core.models import DataFile
+
+        if self.data_source and self.data_path:
+            try:
+                return DataFile.objects.get(source=self.data_source, path=self.data_path)
+            except DataFile.DoesNotExist:
+                pass
+
+    def sync_data(self):
+        raise NotImplementedError(f"{self.__class__} must implement a sync_data() method.")
+
+
 FEATURES_MAP = (
     ('custom_fields', CustomFieldsMixin),
     ('custom_links', CustomLinksMixin),
     ('export_templates', ExportTemplatesMixin),
     ('job_results', JobResultsMixin),
     ('journaling', JournalingMixin),
+    ('synced_data', SyncedDataMixin),
     ('tags', TagsMixin),
     ('webhooks', WebhooksMixin),
 )
@@ -344,3 +412,9 @@ def _register_features(sender, **kwargs):
             'changelog',
             kwargs={'model': sender}
         )('netbox.views.generic.ObjectChangeLogView')
+    if issubclass(sender, SyncedDataMixin):
+        register_model_view(
+            sender,
+            'sync',
+            kwargs={'model': sender}
+        )('netbox.views.generic.ObjectSyncDataView')

netbox/netbox/views/generic/feature_views.py

@@ -1,6 +1,7 @@
 from django.contrib.contenttypes.models import ContentType
+from django.contrib import messages
 from django.db.models import Q
-from django.shortcuts import get_object_or_404, render
+from django.shortcuts import get_object_or_404, redirect, render
 from django.utils.translation import gettext as _
 from django.views.generic import View
 
@@ -11,6 +12,7 @@ from utilities.views import ViewTab
 __all__ = (
     'ObjectChangeLogView',
     'ObjectJournalView',
+    'ObjectSyncDataView',
 )
 
 
@@ -126,3 +128,25 @@ class ObjectJournalView(View):
             'base_template': self.base_template,
             'tab': self.tab,
         })
+
+
+class ObjectSyncDataView(View):
+
+    def post(self, request, model, **kwargs):
+        """
+        Synchronize data from the DataFile associated with this object.
+        """
+        qs = model.objects.all()
+        if hasattr(model.objects, 'restrict'):
+            qs = qs.restrict(request.user, 'change')
+        obj = get_object_or_404(qs, **kwargs)
+
+        if not obj.data_file:
+            messages.error(request, f"Unable to synchronize data: No data file set.")
+            return redirect(obj.get_absolute_url())
+
+        obj.sync_data()
+        obj.save()
+        messages.success(request, f"Synchronized data for {model._meta._verbose_name} {obj}.")
+
+        return redirect(obj.get_absolute_url())

netbox/templates/extras/configcontext.html

@@ -6,22 +6,16 @@
   <div class="row">
     <div class="col col-md-5">
       <div class="card">
-                <h5 class="card-header">
+        <h5 class="card-header">Config Context</h5>
-                    Config Context
-                </h5>
         <div class="card-body">
           <table class="table table-hover attr-table">
             <tr>
               <th scope="row">Name</th>
-                            <td>
+              <td>{{ object.name }}</td>
-                                {{ object.name }}
-                            </td>
             </tr>
             <tr>
               <th scope="row">Weight</th>
-                            <td>
+              <td>{{ object.weight }}</td>
-                                {{ object.weight }}
-                            </td>
             </tr>
             <tr>
               <th scope="row">Description</th>
@@ -29,25 +23,42 @@
             </tr>
             <tr>
               <th scope="row">Active</th>
+              <td>{% checkmark object.is_active %}</td>
+            </tr>
+            <tr>
+              <th scope="row">Data Source</th>
               <td>
-                                {% if object.is_active %}
+                {% if object.data_source %}
-                                    <span class="text-success">
+                  <a href="{{ object.data_source.get_absolute_url }}">{{ object.data_source }}</a>
-                                        <i class="mdi mdi-check-bold"></i>
-                                    </span>
                 {% else %}
-                                    <span class="text-danger">
+                  {{ ''|placeholder }}
-                                        <i class="mdi mdi-close"></i>
-                                    </span>
                 {% endif %}
               </td>
             </tr>
+            <tr>
+              <th scope="row">Data File</th>
+              <td>
+                {% if object.data_file %}
+                  <a href="{{ object.data_file.get_absolute_url }}">{{ object.data_file }}</a>
+                {% elif object.data_path %}
+                  <div class="float-end text-warning">
+                    <i class="mdi mdi-alert" title="No file exists at specified path."></i>
+                  </div>
+                  {{ object.data_path }}
+                {% else %}
+                  {{ ''|placeholder }}
+                {% endif %}
+              </td>
+            </tr>
+          <tr>
+            <th scope="row">Data Synced</th>
+            <td>{{ object.data_synced|placeholder }}</td>
+          </tr>
           </table>
         </div>
       </div>
       <div class="card">
-                <h5 class="card-header">
+        <h5 class="card-header">Assignment</h5>
-                    Assignment
-                </h5>
         <div class="card-body">
           <table class="table table-hover attr-table">
             {% for title, objects in assigned_objects %}
@@ -75,6 +86,21 @@
           {% include 'extras/inc/configcontext_format.html' %}
         </div>
         <div class="card-body">
+          {% if object.data_file and object.data_file.last_updated > object.data_synced %}
+            <div class="alert alert-warning" role="alert">
+              <i class="mdi mdi-alert"></i> Data is out of sync with upstream file (<a href="{{ object.data_file.get_absolute_url }}">{{ object.data_file }}</a>).
+              {% if perms.extras.change_configcontext %}
+                <div class="float-end">
+                  <form action="{% url 'extras:configcontext_sync' pk=object.pk %}" method="post">
+                    {% csrf_token %}
+                    <button type="submit" class="btn btn-primary btn-sm">
+                      <i class="mdi mdi-sync" aria-hidden="true"></i> Sync
+                    </button>
+                  </form>
+                </div>
+              {% endif %}
+            </div>
+          {% endif %}
             {% include 'extras/inc/configcontext_data.html' with data=object.data format=format %}
         </div>
       </div>