Closes #11558: Add support for remote data sources (#11646)

* WIP

* WIP

* Add git sync

* Fix file hashing

* Add last_synced to DataSource

* Build out UI & API resources

* Add status field to DataSource

* Add UI control to sync data source

* Add API endpoint to sync data sources

* Fix display of DataSource job results

* DataSource password should be write-only

* General cleanup

* Add data file UI view

* Punt on HTTP, FTP support for now

* Add DataSource URL validation

* Add HTTP proxy support to git fetcher

* Add management command to sync data sources

* DataFile REST API endpoints should be read-only

* Refactor fetch methods into backend classes

* Replace auth & git branch fields with general-purpose parameters

* Fix last_synced time

* Render discrete form fields for backend parameters

* Enable dynamic edit form for DataSource

* Register DataBackend classes in application registry

* Add search indexers for DataSource, DataFile

* Add single & bulk delete views for DataFile

* Add model documentation

* Convert DataSource to a primary model

* Introduce pre_sync & post_sync signals

* Clean up migrations

* Rename url to source_url

* Clean up filtersets

* Add API & filterset tests

* Add view tests

* Add initSelect() to HTMX refresh handler

* Render DataSourceForm fieldsets dynamically

* Update compiled static resources

netbox/core/api/nested_serializers.py

@@ -0,0 +1,25 @@
+from rest_framework import serializers
+
+from core.models import *
+from netbox.api.serializers import WritableNestedSerializer
+
+__all__ = [
+    'NestedDataFileSerializer',
+    'NestedDataSourceSerializer',
+]
+
+
+class NestedDataSourceSerializer(WritableNestedSerializer):
+    url = serializers.HyperlinkedIdentityField(view_name='core-api:datasource-detail')
+
+    class Meta:
+        model = DataSource
+        fields = ['id', 'url', 'display', 'name']
+
+
+class NestedDataFileSerializer(WritableNestedSerializer):
+    url = serializers.HyperlinkedIdentityField(view_name='core-api:datafile-detail')
+
+    class Meta:
+        model = DataFile
+        fields = ['id', 'url', 'display', 'path']

netbox/core/api/serializers.py

@@ -0,0 +1,51 @@
+from rest_framework import serializers
+
+from core.choices import *
+from core.models import *
+from netbox.api.fields import ChoiceField
+from netbox.api.serializers import NetBoxModelSerializer
+from .nested_serializers import *
+
+__all__ = (
+    'DataSourceSerializer',
+)
+
+
+class DataSourceSerializer(NetBoxModelSerializer):
+    url = serializers.HyperlinkedIdentityField(
+        view_name='core-api:datasource-detail'
+    )
+    type = ChoiceField(
+        choices=DataSourceTypeChoices
+    )
+    status = ChoiceField(
+        choices=DataSourceStatusChoices,
+        read_only=True
+    )
+
+    # Related object counts
+    file_count = serializers.IntegerField(
+        read_only=True
+    )
+
+    class Meta:
+        model = DataSource
+        fields = [
+            'id', 'url', 'display', 'name', 'type', 'source_url', 'enabled', 'status', 'description', 'comments',
+            'parameters', 'ignore_rules', 'created', 'last_updated', 'file_count',
+        ]
+
+
+class DataFileSerializer(NetBoxModelSerializer):
+    url = serializers.HyperlinkedIdentityField(
+        view_name='core-api:datafile-detail'
+    )
+    source = NestedDataSourceSerializer(
+        read_only=True
+    )
+
+    class Meta:
+        model = DataFile
+        fields = [
+            'id', 'url', 'display', 'source', 'path', 'last_updated', 'size', 'hash',
+        ]

netbox/core/api/urls.py

@@ -0,0 +1,13 @@
+from netbox.api.routers import NetBoxRouter
+from . import views
+
+
+router = NetBoxRouter()
+router.APIRootView = views.CoreRootView
+
+# Data sources
+router.register('data-sources', views.DataSourceViewSet)
+router.register('data-files', views.DataFileViewSet)
+
+app_name = 'core-api'
+urlpatterns = router.urls

netbox/core/api/views.py

@@ -0,0 +1,52 @@
+from django.shortcuts import get_object_or_404
+
+from rest_framework.decorators import action
+from rest_framework.exceptions import PermissionDenied
+from rest_framework.response import Response
+from rest_framework.routers import APIRootView
+
+from core import filtersets
+from core.models import *
+from netbox.api.viewsets import NetBoxModelViewSet, NetBoxReadOnlyModelViewSet
+from utilities.utils import count_related
+from . import serializers
+
+
+class CoreRootView(APIRootView):
+    """
+    Core API root view
+    """
+    def get_view_name(self):
+        return 'Core'
+
+
+#
+# Data sources
+#
+
+class DataSourceViewSet(NetBoxModelViewSet):
+    queryset = DataSource.objects.annotate(
+        file_count=count_related(DataFile, 'source')
+    )
+    serializer_class = serializers.DataSourceSerializer
+    filterset_class = filtersets.DataSourceFilterSet
+
+    @action(detail=True, methods=['post'])
+    def sync(self, request, pk):
+        """
+        Enqueue a job to synchronize the DataSource.
+        """
+        if not request.user.has_perm('extras.sync_datasource'):
+            raise PermissionDenied("Syncing data sources requires the core.sync_datasource permission.")
+
+        datasource = get_object_or_404(DataSource, pk=pk)
+        datasource.enqueue_sync_job(request)
+        serializer = serializers.DataSourceSerializer(datasource, context={'request': request})
+
+        return Response(serializer.data)
+
+
+class DataFileViewSet(NetBoxReadOnlyModelViewSet):
+    queryset = DataFile.objects.defer('data').prefetch_related('source')
+    serializer_class = serializers.DataFileSerializer
+    filterset_class = filtersets.DataFileFilterSet