Build out UI & API resources

netbox/core/admin/__init__.py

@@ -1,18 +0,0 @@
-from django.contrib import admin
-
-from core.models import DataFile, DataSource
-
-
-@admin.register(DataSource)
-class DataSourceAdmin(admin.ModelAdmin):
-    list_display = ('name', 'file_count', 'last_synced')
-
-    @staticmethod
-    def file_count(obj):
-        return obj.datafiles.count()
-
-
-@admin.register(DataFile)
-class DataFileAdmin(admin.ModelAdmin):
-    list_display = ('path', 'size', 'last_updated')
-    readonly_fields = ('source', 'path', 'last_updated', 'size', 'hash')

netbox/core/api/nested_serializers.py

@@ -0,0 +1,25 @@
+from rest_framework import serializers
+
+from core.models import *
+from netbox.api.serializers import WritableNestedSerializer
+
+__all__ = [
+    'NestedDataFileSerializer',
+    'NestedDataSourceSerializer',
+]
+
+
+class NestedDataSourceSerializer(WritableNestedSerializer):
+    url = serializers.HyperlinkedIdentityField(view_name='core-api:datasource-detail')
+
+    class Meta:
+        model = DataSource
+        fields = ['id', 'url', 'display', 'name']
+
+
+class NestedDataFileSerializer(WritableNestedSerializer):
+    url = serializers.HyperlinkedIdentityField(view_name='core-api:datafile-detail')
+
+    class Meta:
+        model = DataFile
+        fields = ['id', 'url', 'display', 'path']

netbox/core/api/serializers.py

@@ -0,0 +1,37 @@
+from rest_framework import serializers
+
+from core.choices import *
+from core.models import *
+from netbox.api.fields import ChoiceField
+from netbox.api.serializers import NetBoxModelSerializer
+from .nested_serializers import *
+
+__all__ = (
+    'DataSourceSerializer',
+)
+
+
+class DataSourceSerializer(NetBoxModelSerializer):
+    url = serializers.HyperlinkedIdentityField(view_name='core-api:datasource-detail')
+    type = ChoiceField(choices=DataSourceTypeChoices, required=False)
+
+    # Related object counts
+    file_count = serializers.IntegerField(read_only=True)
+
+    class Meta:
+        model = DataSource
+        fields = [
+            'id', 'url', 'display', 'name', 'type', 'url', 'enabled', 'description', 'git_branch', 'ignore_rules',
+            'username', 'password', 'created', 'last_updated', 'file_count',
+        ]
+
+
+class DataFileSerializer(NetBoxModelSerializer):
+    url = serializers.HyperlinkedIdentityField(view_name='core-api:datafile-detail')
+    source = NestedDataSourceSerializer(read_only=True)
+
+    class Meta:
+        model = DataFile
+        fields = [
+            'id', 'url', 'display', 'source', 'path', 'last_updated', 'size', 'hash',
+        ]

netbox/core/api/urls.py

@@ -0,0 +1,13 @@
+from netbox.api.routers import NetBoxRouter
+from . import views
+
+
+router = NetBoxRouter()
+router.APIRootView = views.CoreRootView
+
+# Data sources
+router.register('data-sources', views.DataSourceViewSet)
+router.register('data-files', views.DataFileViewSet)
+
+app_name = 'core-api'
+urlpatterns = router.urls

netbox/core/api/views.py

@@ -0,0 +1,33 @@
+from rest_framework.routers import APIRootView
+
+from core import filtersets
+from core.models import *
+from netbox.api.viewsets import NetBoxModelViewSet
+from utilities.utils import count_related
+from . import serializers
+
+
+class CoreRootView(APIRootView):
+    """
+    Core API root view
+    """
+    def get_view_name(self):
+        return 'Core'
+
+
+#
+# Data sources
+#
+
+class DataSourceViewSet(NetBoxModelViewSet):
+    queryset = DataSource.objects.annotate(
+        file_count=count_related(DataFile, 'source')
+    )
+    serializer_class = serializers.DataSourceSerializer
+    filterset_class = filtersets.DataSourceFilterSet
+
+
+class DataFileViewSet(NetBoxModelViewSet):
+    queryset = DataFile.objects.defer('data').prefetch_related('source')
+    serializer_class = serializers.DataFileSerializer
+    filterset_class = filtersets.DataFileFilterSet

netbox/core/filtersets.py

@@ -0,0 +1,50 @@
+from django.db.models import Q
+from django.utils.translation import gettext as _
+
+import django_filters
+
+from .models import *
+
+__all__ = (
+    'DataFileFilterSet',
+    'DataSourceFilterSet',
+)
+
+
+class DataSourceFilterSet(django_filters.FilterSet):
+
+    class Meta:
+        model = DataSource
+        fields = ('id', 'name', 'type', 'git_branch', 'username')
+
+    def search(self, queryset, name, value):
+        if not value.strip():
+            return queryset
+        return queryset.filter(
+            Q(name__icontains=value) |
+            Q(description__icontains=value)
+        )
+
+
+class DataFileFilterSet(django_filters.FilterSet):
+    datasource_id = django_filters.ModelMultipleChoiceFilter(
+        queryset=DataSource.objects.all(),
+        label=_('Data source (ID)'),
+    )
+    datasource = django_filters.ModelMultipleChoiceFilter(
+        field_name='source__name',
+        queryset=DataSource.objects.all(),
+        to_field_name='name',
+        label=_('Data source (name)'),
+    )
+
+    class Meta:
+        model = DataFile
+        fields = ('id', 'path', 'last_updated', 'size', 'hash')
+
+    def search(self, queryset, name, value):
+        if not value.strip():
+            return queryset
+        return queryset.filter(
+            Q(path__icontains=value)
+        )

netbox/core/forms/__init__.py

@@ -0,0 +1,4 @@
+from .bulk_edit import *
+from .bulk_import import *
+from .filtersets import *
+from .model_forms import *

netbox/core/forms/bulk_edit.py

@@ -0,0 +1,57 @@
+from django import forms
+from django.utils.translation import gettext as _
+
+from core.choices import DataSourceTypeChoices
+from core.models import *
+from netbox.forms import NetBoxModelBulkEditForm
+from utilities.forms import (
+    add_blank_choice, BulkEditNullBooleanSelect, StaticSelect,
+)
+
+__all__ = (
+    'DataSourceBulkEditForm',
+)
+
+
+class DataSourceBulkEditForm(NetBoxModelBulkEditForm):
+    type = forms.ChoiceField(
+        choices=add_blank_choice(DataSourceTypeChoices),
+        required=False,
+        initial='',
+        widget=StaticSelect()
+    )
+    enabled = forms.NullBooleanField(
+        required=False,
+        widget=BulkEditNullBooleanSelect(),
+        label=_('Enforce unique space')
+    )
+    description = forms.CharField(
+        max_length=200,
+        required=False
+    )
+    git_branch = forms.CharField(
+        max_length=100,
+        required=False
+    )
+    ignore_rules = forms.CharField(
+        required=False,
+        widget=forms.Textarea()
+    )
+    username = forms.CharField(
+        max_length=100,
+        required=False
+    )
+    password = forms.CharField(
+        max_length=100,
+        required=False,
+        widget=forms.PasswordInput()
+    )
+
+    model = DataSource
+    fieldsets = (
+        (None, ('type', 'enabled', 'description', 'git_branch', 'ignore_rules')),
+        ('Authentication', ('username', 'password')),
+    )
+    nullable_fields = (
+        'description', 'description', 'git_branch', 'ignore_rules', 'username', 'password',
+    )

netbox/core/forms/bulk_import.py

@@ -0,0 +1,15 @@
+from core.models import *
+from netbox.forms import NetBoxModelImportForm
+
+__all__ = (
+    'DataSourceImportForm',
+)
+
+
+class DataSourceImportForm(NetBoxModelImportForm):
+
+    class Meta:
+        model = DataSource
+        fields = (
+            'name', 'type', 'url', 'enabled', 'description', 'git_branch', 'ignore_rules', 'username', 'password',
+        )

netbox/core/forms/filtersets.py

@@ -0,0 +1,56 @@
+from django import forms
+
+from core.choices import *
+from core.models import *
+from netbox.forms import NetBoxModelFilterSetForm
+from utilities.forms import DynamicModelMultipleChoiceField, MultipleChoiceField
+
+__all__ = (
+    'DataFileFilterForm',
+    'DataSourceFilterForm',
+)
+
+
+class DataSourceFilterForm(NetBoxModelFilterSetForm):
+    model = DataSource
+    fieldsets = (
+        (None, ('q', 'filter_id')),
+        ('Data Source', ('type', 'git_branch')),
+        ('Authentication', ('username',)),
+    )
+    type = MultipleChoiceField(
+        choices=DataSourceTypeChoices,
+        required=False
+    )
+    git_branch = forms.CharField(
+        max_length=100,
+        required=False
+    )
+    username = forms.CharField(
+        max_length=100,
+        required=False
+    )
+
+
+class DataFileFilterForm(NetBoxModelFilterSetForm):
+    model = DataFile
+    fieldsets = (
+        (None, ('q', 'filter_id')),
+        ('File', ('datasource_id',)),
+    )
+    datasource_id = DynamicModelMultipleChoiceField(
+        queryset=DataSource.objects.all(),
+        required=False
+    )
+    type = MultipleChoiceField(
+        choices=DataSourceTypeChoices,
+        required=False
+    )
+    git_branch = forms.CharField(
+        max_length=100,
+        required=False
+    )
+    username = forms.CharField(
+        max_length=100,
+        required=False
+    )

netbox/core/forms/model_forms.py

@@ -0,0 +1,23 @@
+from core.models import *
+from netbox.forms import NetBoxModelForm, StaticSelect
+
+__all__ = (
+    'DataSourceForm',
+)
+
+
+class DataSourceForm(NetBoxModelForm):
+    fieldsets = (
+        ('Source', ('name', 'type', 'url', 'enabled', 'description')),
+        ('Git', ('git_branch',)),
+        ('Authentication', ('username', 'password')),
+    )
+
+    class Meta:
+        model = DataSource
+        fields = [
+            'name', 'type', 'url', 'enabled', 'description', 'git_branch', 'ignore_rules', 'username', 'password',
+        ]
+        widgets = {
+            'type': StaticSelect(),
+        }

netbox/core/graphql/schema.py

@@ -0,0 +1,12 @@
+import graphene
+
+from netbox.graphql.fields import ObjectField, ObjectListField
+from .types import *
+
+
+class CoreQuery(graphene.ObjectType):
+    datafile = ObjectField(DataFileType)
+    datafile_list = ObjectListField(DataFileType)
+
+    datasource = ObjectField(DataSourceType)
+    datasource_list = ObjectListField(DataSourceType)

netbox/core/graphql/types.py

@@ -0,0 +1,21 @@
+from core import filtersets, models
+from netbox.graphql.types import NetBoxObjectType
+
+__all__ = (
+    'DataFileType',
+    'DataSourceType',
+)
+
+
+class DataFileType(NetBoxObjectType):
+    class Meta:
+        model = models.DataFile
+        exclude = ('data',)
+        filterset_class = filtersets.DataFileFilterSet
+
+
+class DataSourceType(NetBoxObjectType):
+    class Meta:
+        model = models.DataSource
+        fields = '__all__'
+        filterset_class = filtersets.DataSourceFilterSet

netbox/core/migrations/0003_datasource_created_datasource_last_updated.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.1.5 on 2023-01-26 20:50
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0002_datasource_last_synced'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='datasource',
+            name='created',
+            field=models.DateTimeField(auto_now_add=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='datasource',
+            name='last_updated',
+            field=models.DateTimeField(auto_now=True, null=True),
+        ),
+    ]

netbox/core/models/__init__.py

@@ -0,0 +1 @@
+from .data import *

netbox/core/models/data.py

@@ -2,18 +2,18 @@ import logging
 import os
 import subprocess
 import tempfile
-from functools import cached_property
 from fnmatch import fnmatchcase
 from urllib.parse import quote, urlunparse, urlparse
 
-from django.conf import settings
 from django.db import models
 from django.urls import reverse
 from django.utils import timezone
 from django.utils.translation import gettext as _
 
+from netbox.models import ChangeLoggedModel
 from utilities.files import sha256_hash
-from .choices import *
+from utilities.querysets import RestrictedQuerySet
+from ..choices import *
 
 __all__ = (
     'DataSource',
@@ -23,7 +23,7 @@ __all__ = (
 logger = logging.getLogger('netbox.core.data')
 
 
-class DataSource(models.Model):
+class DataSource(ChangeLoggedModel):
     """
     A remote source from which DataFiles are synchronized.
     """
@@ -36,6 +36,10 @@ class DataSource(models.Model):
         choices=DataSourceTypeChoices,
         default=DataSourceTypeChoices.LOCAL
     )
+    url = models.CharField(
+        max_length=200,
+        verbose_name=_('URL')
+    )
     enabled = models.BooleanField(
         default=True
     )
@@ -43,9 +47,10 @@ class DataSource(models.Model):
         max_length=200,
         blank=True
     )
-    url = models.CharField(
-        max_length=200,
-        verbose_name=_('URL')
+    git_branch = models.CharField(
+        max_length=100,
+        blank=True,
+        help_text=_("Branch to check out for git sources (if not using the default)")
     )
     ignore_rules = models.TextField(
         blank=True,
@@ -59,10 +64,6 @@ class DataSource(models.Model):
         max_length=100,
         blank=True
     )
-    git_branch = models.CharField(
-        max_length=100,
-        blank=True
-    )
     last_synced = models.DateTimeField(
         blank=True,
         null=True,
@@ -75,8 +76,8 @@ class DataSource(models.Model):
     def __str__(self):
         return f'{self.name}'
 
-    # def get_absolute_url(self):
-    #     return reverse('core:datasource', args=[self.pk])
+    def get_absolute_url(self):
+        return reverse('core:datasource', args=[self.pk])
 
     def sync(self):
         """
@@ -231,6 +232,8 @@ class DataFile(models.Model):
     )
     data = models.BinaryField()
 
+    objects = RestrictedQuerySet.as_manager()
+
     class Meta:
         ordering = ('source', 'path')
         constraints = (

netbox/core/tables/__init__.py

@@ -0,0 +1 @@
+from .data import *

netbox/core/tables/data.py

@@ -0,0 +1,43 @@
+import django_tables2 as tables
+
+from core.models import *
+from netbox.tables import NetBoxTable, columns
+
+__all__ = (
+    'DataFileTable',
+    'DataSourceTable',
+)
+
+
+class DataSourceTable(NetBoxTable):
+    name = tables.Column(
+        linkify=True
+    )
+    type = columns.ChoiceFieldColumn()
+    enabled = columns.BooleanColumn()
+    file_count = tables.Column(
+        verbose_name='Files'
+    )
+
+    class Meta(NetBoxTable.Meta):
+        model = DataSource
+        fields = (
+            'pk', 'id', 'name', 'type', 'enabled', 'url', 'description', 'git_branch', 'username', 'created',
+            'last_updated', 'file_count',
+        )
+        default_columns = ('pk', 'name', 'type', 'enabled', 'description', 'file_count')
+
+
+class DataFileTable(NetBoxTable):
+    source = tables.Column(
+        linkify=True
+    )
+    last_updated = columns.DateTimeColumn()
+    actions = None
+
+    class Meta(NetBoxTable.Meta):
+        model = DataFile
+        fields = (
+            'pk', 'id', 'source', 'path', 'last_updated', 'size', 'hash',
+        )
+        default_columns = ('pk', 'source', 'path', 'size', 'last_updated')

netbox/core/urls.py

@@ -0,0 +1,20 @@
+from django.urls import include, path
+
+from utilities.urls import get_model_urls
+from . import views
+
+app_name = 'core'
+urlpatterns = (
+
+    # Data sources
+    path('data-sources/', views.DataSourceListView.as_view(), name='datasource_list'),
+    path('data-sources/add/', views.DataSourceEditView.as_view(), name='datasource_add'),
+    path('data-sources/import/', views.DataSourceBulkImportView.as_view(), name='datasource_import'),
+    path('data-sources/edit/', views.DataSourceBulkEditView.as_view(), name='datasource_bulk_edit'),
+    path('data-sources/delete/', views.DataSourceBulkDeleteView.as_view(), name='datasource_bulk_delete'),
+    path('data-sources/<int:pk>/', include(get_model_urls('core', 'datasource'))),
+
+    # Data files
+    path('data-files/', views.DataFileListView.as_view(), name='datafile_list'),
+
+)

netbox/core/views.py

@@ -0,0 +1,69 @@
+from netbox.views import generic
+from utilities.utils import count_related
+from utilities.views import register_model_view
+from . import filtersets, forms, tables
+from .models import *
+
+
+#
+# Data sources
+#
+
+class DataSourceListView(generic.ObjectListView):
+    queryset = DataSource.objects.annotate(
+        file_count=count_related(DataFile, 'source')
+    )
+    filterset = filtersets.DataSourceFilterSet
+    filterset_form = forms.DataSourceFilterForm
+    table = tables.DataSourceTable
+
+
+@register_model_view(DataSource)
+class DataSourceView(generic.ObjectView):
+    queryset = DataSource.objects.all()
+
+
+@register_model_view(DataSource, 'edit')
+class DataSourceEditView(generic.ObjectEditView):
+    queryset = DataSource.objects.all()
+    form = forms.DataSourceForm
+
+
+@register_model_view(DataSource, 'delete')
+class DataSourceDeleteView(generic.ObjectDeleteView):
+    queryset = DataSource.objects.all()
+
+
+class DataSourceBulkImportView(generic.BulkImportView):
+    queryset = DataSource.objects.all()
+    model_form = forms.DataSourceImportForm
+    table = tables.DataSourceTable
+
+
+class DataSourceBulkEditView(generic.BulkEditView):
+    queryset = DataSource.objects.annotate(
+        count_files=count_related(DataFile, 'source')
+    )
+    filterset = filtersets.DataSourceFilterSet
+    table = tables.DataSourceTable
+    form = forms.DataSourceBulkEditForm
+
+
+class DataSourceBulkDeleteView(generic.BulkDeleteView):
+    queryset = DataSource.objects.annotate(
+        count_files=count_related(DataFile, 'source')
+    )
+    filterset = filtersets.DataSourceFilterSet
+    table = tables.DataSourceTable
+
+
+#
+# Data files
+#
+
+class DataFileListView(generic.ObjectListView):
+    queryset = DataFile.objects.defer('data')
+    filterset = filtersets.DataFileFilterSet
+    filterset_form = forms.DataFileFilterForm
+    table = tables.DataFileTable
+    actions = ('edit',)

netbox/netbox/api/views.py

@@ -27,6 +27,7 @@ class APIRootView(APIView):
 
         return Response({
             'circuits': reverse('circuits-api:api-root', request=request, format=format),
+            'core': reverse('core-api:api-root', request=request, format=format),
             'dcim': reverse('dcim-api:api-root', request=request, format=format),
             'extras': reverse('extras-api:api-root', request=request, format=format),
             'ipam': reverse('ipam-api:api-root', request=request, format=format),

netbox/netbox/graphql/schema.py

@@ -1,6 +1,7 @@
 import graphene
 
 from circuits.graphql.schema import CircuitsQuery
+from core.graphql.schema import CoreQuery
 from dcim.graphql.schema import DCIMQuery
 from extras.graphql.schema import ExtrasQuery
 from ipam.graphql.schema import IPAMQuery
@@ -14,6 +15,7 @@ from wireless.graphql.schema import WirelessQuery
 class Query(
     UsersQuery,
     CircuitsQuery,
+    CoreQuery,
     DCIMQuery,
     ExtrasQuery,
     IPAMQuery,

netbox/netbox/navigation/menu.py

@@ -287,6 +287,7 @@ OTHER_MENU = Menu(
         MenuGroup(
             label=_('Integrations'),
             items=(
+                get_model_item('core', 'datasource', _('Data Sources')),
                 get_model_item('extras', 'webhook', _('Webhooks')),
                 MenuItem(
                     link='extras:report_list',

netbox/netbox/urls.py

@@ -42,6 +42,7 @@ _patterns = [
 
     # Apps
     path('circuits/', include('circuits.urls')),
+    path('core/', include('core.urls')),
     path('dcim/', include('dcim.urls')),
     path('extras/', include('extras.urls')),
     path('ipam/', include('ipam.urls')),
@@ -53,6 +54,7 @@ _patterns = [
     # API
     path('api/', APIRootView.as_view(), name='api-root'),
     path('api/circuits/', include('circuits.api.urls')),
+    path('api/core/', include('core.api.urls')),
     path('api/dcim/', include('dcim.api.urls')),
     path('api/extras/', include('extras.api.urls')),
     path('api/ipam/', include('ipam.api.urls')),

netbox/templates/core/datasource.html

@@ -0,0 +1,93 @@
+{% extends 'generic/object.html' %}
+{% load static %}
+{% load helpers %}
+{% load plugins %}
+{% load render_table from django_tables2 %}
+
+{% block content %}
+  <div class="row mb-3">
+    <div class="col col-md-6">
+      <div class="card">
+        <h5 class="card-header">Data Source</h5>
+        <div class="card-body">
+          <table class="table table-hover attr-table">
+            <tr>
+              <th scope="row">Name</th>
+              <td>{{ object.name }}</td>
+            </tr>
+            <tr>
+              <th scope="row">Type</th>
+              <td>{{ object.get_type_display }}</td>
+            </tr>
+            <tr>
+              <th scope="row">Enabled</th>
+              <td>{% checkmark object.enabled %}</td>
+            </tr>
+            <tr>
+              <th scope="row">Description</th>
+              <td>{{ object.description|placeholder }}</td>
+            </tr>
+            <tr>
+              <th scope="row">URL</th>
+              <td>
+                <a href="{{ object.url }}">{{ object.url }}</a>
+              </td>
+            </tr>
+            <tr>
+              <th scope="row">Last synced</th>
+              <td>{{ object.last_synced|placeholder }}</td>
+            </tr>
+            <tr>
+              <th scope="row">Git branch</th>
+              <td>{{ object.git_branch|placeholder }}</td>
+            </tr>
+            <tr>
+              <th scope="row">Ignore rules</th>
+              <td>
+                {% if object.ignore_rules %}
+                  <pre>{{ object.ignore_rules }}</pre>
+                {% else %}
+                  {{ ''|placeholder }}
+                {% endif %}</td>
+            </tr>
+            <tr>
+              <th scope="row">File count</th>
+              <td>{{ object.datafiles.count }}</td>
+            </tr>
+          </table>
+        </div>
+      </div>
+      {% plugin_left_page object %}
+    </div>
+    <div class="col col-md-6">
+      <div class="card">
+        <h5 class="card-header">Authentication</h5>
+        <div class="card-body">
+          <table class="table table-hover attr-table">
+            <tr>
+              <th scope="row">Username</th>
+              <td>{{ object.username|placeholder }}</td>
+            </tr>
+            <tr>
+              <th scope="row">Password</th>
+              <td>{% if object.password %}********{% else %}{{ ''|placeholder }}{% endif %}</td>
+            </tr>
+          </table>
+        </div>
+      {% include 'inc/panels/custom_fields.html' %}
+      {% plugin_right_page object %}
+    </div>
+  </div>
+  <div class="row mb-3">
+    <div class="col col-md-12">
+      <div class="card">
+        <h5 class="card-header">Files</h5>
+        <div class="card-body htmx-container table-responsive"
+          hx-get="{% url 'core:datafile_list' %}?source_id={{ object.pk }}"
+          hx-trigger="load"
+        ></div>
+      </div>
+      {% plugin_full_width_page object %}
+    </div>
+  </div>
+{% endblock %}