Initial work on #11890

netbox/core/choices.py

@@ -20,7 +20,6 @@ class DataSourceTypeChoices(ChoiceSet):
 
 
 class DataSourceStatusChoices(ChoiceSet):
-
     NEW = 'new'
     QUEUED = 'queued'
     SYNCING = 'syncing'

netbox/core/forms/model_forms.py

@@ -5,7 +5,7 @@ from django import forms
 from core.models import *
 from netbox.forms import NetBoxModelForm
 from netbox.registry import registry
-from utilities.forms import CommentField, get_field_value
+from utilities.forms import BootstrapMixin, CommentField, get_field_value
 
 __all__ = (
     'DataSourceForm',

netbox/core/migrations/0002_managedfile.py

@@ -0,0 +1,39 @@
+# Generated by Django 4.1.7 on 2023-03-23 17:35
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ManagedFile',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False)),
+                ('data_path', models.CharField(blank=True, editable=False, max_length=1000)),
+                ('data_synced', models.DateTimeField(blank=True, editable=False, null=True)),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('last_updated', models.DateTimeField(blank=True, editable=False, null=True)),
+                ('file_root', models.CharField(max_length=1000)),
+                ('file_path', models.FilePathField(editable=False)),
+                ('data_file', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='core.datafile')),
+                ('data_source', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='core.datasource')),
+            ],
+            options={
+                'ordering': ('file_root', 'file_path'),
+            },
+        ),
+        migrations.AddIndex(
+            model_name='managedfile',
+            index=models.Index(fields=['file_root', 'file_path'], name='core_managedfile_root_path'),
+        ),
+        migrations.AddConstraint(
+            model_name='managedfile',
+            constraint=models.UniqueConstraint(fields=('file_root', 'file_path'), name='core_managedfile_unique_root_path'),
+        ),
+    ]

netbox/core/models/__init__.py

@@ -1 +1,2 @@
 from .data import *
+from .files import *

netbox/core/models/data.py

@@ -14,7 +14,6 @@ from django.utils import timezone
 from django.utils.module_loading import import_string
 from django.utils.translation import gettext as _
 
-from extras.models import JobResult
 from netbox.models import PrimaryModel
 from netbox.registry import registry
 from utilities.files import sha256_hash
@@ -113,6 +112,8 @@ class DataSource(PrimaryModel):
         """
         Enqueue a background job to synchronize the DataSource by calling sync().
         """
+        from extras.models import JobResult
+
         # Set the status to "syncing"
         self.status = DataSourceStatusChoices.QUEUED
         DataSource.objects.filter(pk=self.pk).update(status=self.status)

netbox/core/models/files.py

@@ -0,0 +1,82 @@
+import logging
+import os
+from importlib.machinery import FileFinder
+from pkgutil import ModuleInfo, get_importer
+
+from django.conf import settings
+from django.db import models
+from django.urls import reverse
+from django.utils.translation import gettext as _
+
+from netbox.models.features import SyncedDataMixin
+from utilities.querysets import RestrictedQuerySet
+
+__all__ = (
+    'ManagedFile',
+)
+
+logger = logging.getLogger('netbox.core.files')
+
+ROOT_PATH_CHOICES = (
+    ('scripts', 'Scripts Root'),
+    ('reports', 'Reports Root'),
+)
+
+
+class ManagedFile(SyncedDataMixin, models.Model):
+    """
+    Database representation for a file on disk.
+    """
+    created = models.DateTimeField(
+        auto_now_add=True
+    )
+    last_updated = models.DateTimeField(
+        editable=False,
+        blank=True,
+        null=True
+    )
+    file_root = models.CharField(
+        max_length=1000,
+        choices=ROOT_PATH_CHOICES
+    )
+    file_path = models.FilePathField(
+        editable=False,
+        help_text=_("File path relative to the designated root path")
+    )
+
+    objects = RestrictedQuerySet.as_manager()
+
+    class Meta:
+        ordering = ('file_root', 'file_path')
+        constraints = (
+            models.UniqueConstraint(
+                fields=('file_root', 'file_path'),
+                name='%(app_label)s_%(class)s_unique_root_path'
+            ),
+        )
+        indexes = [
+            models.Index(fields=('file_root', 'file_path'), name='core_managedfile_root_path'),
+        ]
+
+    def __str__(self):
+        return f'{self.get_file_root_display()}: {self.file_path}'
+
+    def get_absolute_url(self):
+        return reverse('core:managedfile', args=[self.pk])
+
+    @property
+    def full_path(self):
+        return os.path.join(self._resolve_root_path(), self.file_path)
+
+    def _resolve_root_path(self):
+        return {
+            'scripts': settings.SCRIPTS_ROOT,
+            'reports': settings.REPORTS_ROOT,
+        }[self.file_root]
+
+    def get_module_info(self):
+        return ModuleInfo(
+            module_finder=get_importer(self.file_root),
+            name=self.file_path.split('.py')[0],
+            ispkg=False
+        )

netbox/core/tables/__init__.py

@@ -1 +1,2 @@
 from .data import *
+from .files import *

netbox/core/tables/files.py

@@ -0,0 +1,25 @@
+import django_tables2 as tables
+
+from core.models import *
+from netbox.tables import NetBoxTable, columns
+
+__all__ = (
+    'ManagedFileTable',
+)
+
+
+class ManagedFileTable(NetBoxTable):
+    file_path = tables.Column(
+        linkify=True
+    )
+    last_updated = columns.DateTimeColumn()
+    actions = columns.ActionsColumn(
+        actions=('delete',)
+    )
+
+    class Meta(NetBoxTable.Meta):
+        model = ManagedFile
+        fields = (
+            'pk', 'id', 'file_root', 'file_path', 'last_updated', 'size', 'hash',
+        )
+        default_columns = ('pk', 'file_root', 'file_path', 'last_updated')

netbox/core/urls.py

@@ -19,4 +19,11 @@ urlpatterns = (
     path('data-files/delete/', views.DataFileBulkDeleteView.as_view(), name='datafile_bulk_delete'),
     path('data-files/<int:pk>/', include(get_model_urls('core', 'datafile'))),
 
+    # Managed files
+    path('files/', views.ManagedFileListView.as_view(), name='managedfile_list'),
+    # path('files/add/', views.ManagedFileEditView.as_view(), name='managedfile_add'),
+    # path('files/edit/', views.ManagedFileBulkEditView.as_view(), name='managedfile_bulk_edit'),
+    # path('files/delete/', views.ManagedFileBulkDeleteView.as_view(), name='managedfile_bulk_delete'),
+    path('files/<int:pk>/', include(get_model_urls('core', 'managedfile'))),
+
 )

netbox/core/views.py

@@ -120,3 +120,43 @@ class DataFileBulkDeleteView(generic.BulkDeleteView):
     queryset = DataFile.objects.defer('data')
     filterset = filtersets.DataFileFilterSet
     table = tables.DataFileTable
+
+
+#
+# Managed files
+#
+
+class ManagedFileListView(generic.ObjectListView):
+    queryset = ManagedFile.objects.all()
+    # filterset = filtersets.ManagedFileFilterSet
+    # filterset_form = forms.ManagedFileFilterForm
+    table = tables.ManagedFileTable
+
+
+@register_model_view(ManagedFile)
+class ManagedFileView(generic.ObjectView):
+    queryset = ManagedFile.objects.all()
+
+
+# @register_model_view(ManagedFile, 'edit')
+# class ManagedFileEditView(generic.ObjectEditView):
+#     queryset = ManagedFile.objects.all()
+#     form = forms.ManagedFileForm
+
+
+@register_model_view(ManagedFile, 'delete')
+class ManagedFileDeleteView(generic.ObjectDeleteView):
+    queryset = ManagedFile.objects.all()
+
+
+# class ManagedFileBulkEditView(generic.BulkEditView):
+#     queryset = ManagedFile.objects.all()
+#     # filterset = filtersets.ManagedFileFilterSet
+#     table = tables.ManagedFileTable
+#     form = forms.ManagedFileBulkEditForm
+
+
+# class ManagedFileBulkDeleteView(generic.BulkDeleteView):
+#     queryset = ManagedFile.objects.all()
+#     # filterset = filtersets.ManagedFileFilterSet
+#     table = tables.ManagedFileTable

netbox/extras/migrations/0091_create_managedfiles.py

@@ -0,0 +1,47 @@
+import pkgutil
+
+from django.conf import settings
+from django.db import migrations
+
+
+def create_files(cls, root_name, path):
+
+    modules = list(pkgutil.iter_modules([path]))
+    filenames = [f'{m.name}.py' for m in modules]
+
+    managed_files = [
+        cls(
+            file_root=root_name,
+            file_path=filename
+        ) for filename in filenames
+    ]
+    cls.objects.bulk_create(managed_files)
+
+
+def replicate_scripts(apps, schema_editor):
+    ManagedFile = apps.get_model('core', 'ManagedFile')
+    create_files(ManagedFile, 'scripts', settings.SCRIPTS_ROOT)
+
+
+def replicate_reports(apps, schema_editor):
+    ManagedFile = apps.get_model('core', 'ManagedFile')
+    create_files(ManagedFile, 'reports', settings.REPORTS_ROOT)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0002_managedfile'),
+        ('extras', '0090_objectchange_index_request_id'),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            code=replicate_scripts,
+            reverse_code=migrations.RunPython.noop
+        ),
+        migrations.RunPython(
+            code=replicate_reports,
+            reverse_code=migrations.RunPython.noop
+        ),
+    ]

netbox/extras/reports.py

@@ -8,6 +8,7 @@ from django.conf import settings
 from django.utils import timezone
 from django_rq import job
 
+from core.models import ManagedFile
 from .choices import JobResultStatusChoices, LogLevelChoices
 from .models import JobResult
 
@@ -53,7 +54,9 @@ def get_reports():
 
     # Iterate through all modules within the reports path. These are the user-created files in which reports are
     # defined.
-    for importer, module_name, _ in pkgutil.iter_modules([settings.REPORTS_ROOT]):
+    # modules = pkgutil.iter_modules([settings.REPORTS_ROOT])
+    modules = [mf.get_module_info() for mf in ManagedFile.objects.filter(file_root='reports')]
+    for importer, module_name, _ in modules:
         module = importer.find_module(module_name).load_module(module_name)
         report_order = getattr(module, "report_order", ())
         ordered_reports = [cls() for cls in report_order if is_report(cls)]

netbox/extras/scripts.py

@@ -15,6 +15,7 @@ from django.core.validators import RegexValidator
 from django.db import transaction
 from django.utils.functional import classproperty
 
+from core.models import ManagedFile
 from extras.api.serializers import ScriptOutputSerializer
 from extras.choices import JobResultStatusChoices, LogLevelChoices
 from extras.models import JobResult
@@ -531,7 +532,8 @@ def get_scripts(use_names=False):
 
     # Get all modules within the scripts path. These are the user-created files in which scripts are
     # defined.
-    modules = list(pkgutil.iter_modules([settings.SCRIPTS_ROOT]))
+    # modules = list(pkgutil.iter_modules([settings.SCRIPTS_ROOT]))
+    modules = [mf.get_module_info() for mf in ManagedFile.objects.filter(file_root='scripts')]
     modules_bases = set([name.split(".")[0] for _, name, _ in modules])
 
     # Deleting from sys.modules needs to done behind a lock to prevent race conditions where a module is

netbox/netbox/navigation/menu.py

@@ -313,6 +313,7 @@ OTHER_MENU = Menu(
                 get_model_item('extras', 'tag', 'Tags'),
                 get_model_item('extras', 'configcontext', _('Config Contexts'), actions=['add']),
                 get_model_item('extras', 'configtemplate', _('Config Templates'), actions=['add']),
+                get_model_item('core', 'managedfile', _('Managed Files'), actions=()),
             ),
         ),
     ),

netbox/templates/core/managedfile.html

@@ -0,0 +1,57 @@
+{% extends 'generic/object.html' %}
+{% load buttons %}
+{% load custom_links %}
+{% load helpers %}
+{% load perms %}
+{% load plugins %}
+
+{% block controls %}
+  <div class="controls">
+    <div class="control-group">
+      {% plugin_buttons object %}
+    </div>
+    {% if request.user|can_delete:object %}
+      {% delete_button object %}
+    {% endif %}
+    <div class="control-group">
+      {% custom_links object %}
+    </div>
+  </div>
+{% endblock controls %}
+
+{% block content %}
+  <div class="row mb-3">
+    <div class="col">
+      <div class="card">
+        <h5 class="card-header">Managed File</h5>
+        <div class="card-body">
+          <table class="table table-hover attr-table">
+            <tr>
+              <th scope="row">Root</th>
+              <td>{{ object.get_file_root_display }}</td>
+            </tr>
+            <tr>
+              <th scope="row">Path</th>
+              <td>
+                <span class="font-monospace" id="datafile_path">{{ object.file_path }}</span>
+                <a class="btn btn-sm btn-primary copy-token" data-clipboard-target="#datafile_path" title="Copy to clipboard">
+                  <i class="mdi mdi-content-copy"></i>
+                </a>
+              </td>
+            </tr>
+            <tr>
+              <th scope="row">Last Updated</th>
+              <td>{{ object.last_updated }}</td>
+            </tr>
+          </table>
+        </div>
+      </div>
+      {% plugin_left_page object %}
+    </div>
+  </div>
+  <div class="row mb-3">
+    <div class="col col-md-12">
+      {% plugin_full_width_page object %}
+    </div>
+  </div>
+{% endblock %}