From ba7fab3a5df4dd4bef9631d77ad6dbbf7e838685 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Thu, 23 Mar 2023 14:36:18 -0400 Subject: [PATCH] Initial work on #11890 --- netbox/core/choices.py | 1 - netbox/core/forms/model_forms.py | 2 +- netbox/core/migrations/0002_managedfile.py | 39 +++++++++ netbox/core/models/__init__.py | 1 + netbox/core/models/data.py | 3 +- netbox/core/models/files.py | 82 +++++++++++++++++++ netbox/core/tables/__init__.py | 1 + netbox/core/tables/files.py | 25 ++++++ netbox/core/urls.py | 7 ++ netbox/core/views.py | 40 +++++++++ .../migrations/0091_create_managedfiles.py | 47 +++++++++++ netbox/extras/reports.py | 5 +- netbox/extras/scripts.py | 4 +- netbox/netbox/navigation/menu.py | 1 + netbox/templates/core/managedfile.html | 57 +++++++++++++ 15 files changed, 310 insertions(+), 5 deletions(-) create mode 100644 netbox/core/migrations/0002_managedfile.py create mode 100644 netbox/core/models/files.py create mode 100644 netbox/core/tables/files.py create mode 100644 netbox/extras/migrations/0091_create_managedfiles.py create mode 100644 netbox/templates/core/managedfile.html diff --git a/netbox/core/choices.py b/netbox/core/choices.py index be33afaba..b392afc1e 100644 --- a/netbox/core/choices.py +++ b/netbox/core/choices.py @@ -20,7 +20,6 @@ class DataSourceTypeChoices(ChoiceSet): class DataSourceStatusChoices(ChoiceSet): - NEW = 'new' QUEUED = 'queued' SYNCING = 'syncing' diff --git a/netbox/core/forms/model_forms.py b/netbox/core/forms/model_forms.py index a3a478be5..5ecb7ef46 100644 --- a/netbox/core/forms/model_forms.py +++ b/netbox/core/forms/model_forms.py @@ -5,7 +5,7 @@ from django import forms from core.models import * from netbox.forms import NetBoxModelForm from netbox.registry import registry -from utilities.forms import CommentField, get_field_value +from utilities.forms import BootstrapMixin, CommentField, get_field_value __all__ = ( 'DataSourceForm', diff --git a/netbox/core/migrations/0002_managedfile.py b/netbox/core/migrations/0002_managedfile.py new file mode 100644 index 000000000..da6b1e3be --- /dev/null +++ b/netbox/core/migrations/0002_managedfile.py @@ -0,0 +1,39 @@ +# Generated by Django 4.1.7 on 2023-03-23 17:35 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='ManagedFile', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False)), + ('data_path', models.CharField(blank=True, editable=False, max_length=1000)), + ('data_synced', models.DateTimeField(blank=True, editable=False, null=True)), + ('created', models.DateTimeField(auto_now_add=True)), + ('last_updated', models.DateTimeField(blank=True, editable=False, null=True)), + ('file_root', models.CharField(max_length=1000)), + ('file_path', models.FilePathField(editable=False)), + ('data_file', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='core.datafile')), + ('data_source', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='core.datasource')), + ], + options={ + 'ordering': ('file_root', 'file_path'), + }, + ), + migrations.AddIndex( + model_name='managedfile', + index=models.Index(fields=['file_root', 'file_path'], name='core_managedfile_root_path'), + ), + migrations.AddConstraint( + model_name='managedfile', + constraint=models.UniqueConstraint(fields=('file_root', 'file_path'), name='core_managedfile_unique_root_path'), + ), + ] diff --git a/netbox/core/models/__init__.py b/netbox/core/models/__init__.py index df22d8bbb..e21b9f8cf 100644 --- a/netbox/core/models/__init__.py +++ b/netbox/core/models/__init__.py @@ -1 +1,2 @@ from .data import * +from .files import * diff --git a/netbox/core/models/data.py b/netbox/core/models/data.py index 8d7164047..0ceb7dc6a 100644 --- a/netbox/core/models/data.py +++ b/netbox/core/models/data.py @@ -14,7 +14,6 @@ from django.utils import timezone from django.utils.module_loading import import_string from django.utils.translation import gettext as _ -from extras.models import JobResult from netbox.models import PrimaryModel from netbox.registry import registry from utilities.files import sha256_hash @@ -113,6 +112,8 @@ class DataSource(PrimaryModel): """ Enqueue a background job to synchronize the DataSource by calling sync(). """ + from extras.models import JobResult + # Set the status to "syncing" self.status = DataSourceStatusChoices.QUEUED DataSource.objects.filter(pk=self.pk).update(status=self.status) diff --git a/netbox/core/models/files.py b/netbox/core/models/files.py new file mode 100644 index 000000000..029df0ef4 --- /dev/null +++ b/netbox/core/models/files.py @@ -0,0 +1,82 @@ +import logging +import os +from importlib.machinery import FileFinder +from pkgutil import ModuleInfo, get_importer + +from django.conf import settings +from django.db import models +from django.urls import reverse +from django.utils.translation import gettext as _ + +from netbox.models.features import SyncedDataMixin +from utilities.querysets import RestrictedQuerySet + +__all__ = ( + 'ManagedFile', +) + +logger = logging.getLogger('netbox.core.files') + +ROOT_PATH_CHOICES = ( + ('scripts', 'Scripts Root'), + ('reports', 'Reports Root'), +) + + +class ManagedFile(SyncedDataMixin, models.Model): + """ + Database representation for a file on disk. + """ + created = models.DateTimeField( + auto_now_add=True + ) + last_updated = models.DateTimeField( + editable=False, + blank=True, + null=True + ) + file_root = models.CharField( + max_length=1000, + choices=ROOT_PATH_CHOICES + ) + file_path = models.FilePathField( + editable=False, + help_text=_("File path relative to the designated root path") + ) + + objects = RestrictedQuerySet.as_manager() + + class Meta: + ordering = ('file_root', 'file_path') + constraints = ( + models.UniqueConstraint( + fields=('file_root', 'file_path'), + name='%(app_label)s_%(class)s_unique_root_path' + ), + ) + indexes = [ + models.Index(fields=('file_root', 'file_path'), name='core_managedfile_root_path'), + ] + + def __str__(self): + return f'{self.get_file_root_display()}: {self.file_path}' + + def get_absolute_url(self): + return reverse('core:managedfile', args=[self.pk]) + + @property + def full_path(self): + return os.path.join(self._resolve_root_path(), self.file_path) + + def _resolve_root_path(self): + return { + 'scripts': settings.SCRIPTS_ROOT, + 'reports': settings.REPORTS_ROOT, + }[self.file_root] + + def get_module_info(self): + return ModuleInfo( + module_finder=get_importer(self.file_root), + name=self.file_path.split('.py')[0], + ispkg=False + ) diff --git a/netbox/core/tables/__init__.py b/netbox/core/tables/__init__.py index df22d8bbb..e21b9f8cf 100644 --- a/netbox/core/tables/__init__.py +++ b/netbox/core/tables/__init__.py @@ -1 +1,2 @@ from .data import * +from .files import * diff --git a/netbox/core/tables/files.py b/netbox/core/tables/files.py new file mode 100644 index 000000000..e1333c000 --- /dev/null +++ b/netbox/core/tables/files.py @@ -0,0 +1,25 @@ +import django_tables2 as tables + +from core.models import * +from netbox.tables import NetBoxTable, columns + +__all__ = ( + 'ManagedFileTable', +) + + +class ManagedFileTable(NetBoxTable): + file_path = tables.Column( + linkify=True + ) + last_updated = columns.DateTimeColumn() + actions = columns.ActionsColumn( + actions=('delete',) + ) + + class Meta(NetBoxTable.Meta): + model = ManagedFile + fields = ( + 'pk', 'id', 'file_root', 'file_path', 'last_updated', 'size', 'hash', + ) + default_columns = ('pk', 'file_root', 'file_path', 'last_updated') diff --git a/netbox/core/urls.py b/netbox/core/urls.py index 128020890..32a286a4e 100644 --- a/netbox/core/urls.py +++ b/netbox/core/urls.py @@ -19,4 +19,11 @@ urlpatterns = ( path('data-files/delete/', views.DataFileBulkDeleteView.as_view(), name='datafile_bulk_delete'), path('data-files//', include(get_model_urls('core', 'datafile'))), + # Managed files + path('files/', views.ManagedFileListView.as_view(), name='managedfile_list'), + # path('files/add/', views.ManagedFileEditView.as_view(), name='managedfile_add'), + # path('files/edit/', views.ManagedFileBulkEditView.as_view(), name='managedfile_bulk_edit'), + # path('files/delete/', views.ManagedFileBulkDeleteView.as_view(), name='managedfile_bulk_delete'), + path('files//', include(get_model_urls('core', 'managedfile'))), + ) diff --git a/netbox/core/views.py b/netbox/core/views.py index 7a603ba1a..35f8f1dc0 100644 --- a/netbox/core/views.py +++ b/netbox/core/views.py @@ -120,3 +120,43 @@ class DataFileBulkDeleteView(generic.BulkDeleteView): queryset = DataFile.objects.defer('data') filterset = filtersets.DataFileFilterSet table = tables.DataFileTable + + +# +# Managed files +# + +class ManagedFileListView(generic.ObjectListView): + queryset = ManagedFile.objects.all() + # filterset = filtersets.ManagedFileFilterSet + # filterset_form = forms.ManagedFileFilterForm + table = tables.ManagedFileTable + + +@register_model_view(ManagedFile) +class ManagedFileView(generic.ObjectView): + queryset = ManagedFile.objects.all() + + +# @register_model_view(ManagedFile, 'edit') +# class ManagedFileEditView(generic.ObjectEditView): +# queryset = ManagedFile.objects.all() +# form = forms.ManagedFileForm + + +@register_model_view(ManagedFile, 'delete') +class ManagedFileDeleteView(generic.ObjectDeleteView): + queryset = ManagedFile.objects.all() + + +# class ManagedFileBulkEditView(generic.BulkEditView): +# queryset = ManagedFile.objects.all() +# # filterset = filtersets.ManagedFileFilterSet +# table = tables.ManagedFileTable +# form = forms.ManagedFileBulkEditForm + + +# class ManagedFileBulkDeleteView(generic.BulkDeleteView): +# queryset = ManagedFile.objects.all() +# # filterset = filtersets.ManagedFileFilterSet +# table = tables.ManagedFileTable diff --git a/netbox/extras/migrations/0091_create_managedfiles.py b/netbox/extras/migrations/0091_create_managedfiles.py new file mode 100644 index 000000000..eefdc06a0 --- /dev/null +++ b/netbox/extras/migrations/0091_create_managedfiles.py @@ -0,0 +1,47 @@ +import pkgutil + +from django.conf import settings +from django.db import migrations + + +def create_files(cls, root_name, path): + + modules = list(pkgutil.iter_modules([path])) + filenames = [f'{m.name}.py' for m in modules] + + managed_files = [ + cls( + file_root=root_name, + file_path=filename + ) for filename in filenames + ] + cls.objects.bulk_create(managed_files) + + +def replicate_scripts(apps, schema_editor): + ManagedFile = apps.get_model('core', 'ManagedFile') + create_files(ManagedFile, 'scripts', settings.SCRIPTS_ROOT) + + +def replicate_reports(apps, schema_editor): + ManagedFile = apps.get_model('core', 'ManagedFile') + create_files(ManagedFile, 'reports', settings.REPORTS_ROOT) + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0002_managedfile'), + ('extras', '0090_objectchange_index_request_id'), + ] + + operations = [ + migrations.RunPython( + code=replicate_scripts, + reverse_code=migrations.RunPython.noop + ), + migrations.RunPython( + code=replicate_reports, + reverse_code=migrations.RunPython.noop + ), + ] diff --git a/netbox/extras/reports.py b/netbox/extras/reports.py index 0a944a0d2..c476e2673 100644 --- a/netbox/extras/reports.py +++ b/netbox/extras/reports.py @@ -8,6 +8,7 @@ from django.conf import settings from django.utils import timezone from django_rq import job +from core.models import ManagedFile from .choices import JobResultStatusChoices, LogLevelChoices from .models import JobResult @@ -53,7 +54,9 @@ def get_reports(): # Iterate through all modules within the reports path. These are the user-created files in which reports are # defined. - for importer, module_name, _ in pkgutil.iter_modules([settings.REPORTS_ROOT]): + # modules = pkgutil.iter_modules([settings.REPORTS_ROOT]) + modules = [mf.get_module_info() for mf in ManagedFile.objects.filter(file_root='reports')] + for importer, module_name, _ in modules: module = importer.find_module(module_name).load_module(module_name) report_order = getattr(module, "report_order", ()) ordered_reports = [cls() for cls in report_order if is_report(cls)] diff --git a/netbox/extras/scripts.py b/netbox/extras/scripts.py index 89dfa3268..fa7a76cb1 100644 --- a/netbox/extras/scripts.py +++ b/netbox/extras/scripts.py @@ -15,6 +15,7 @@ from django.core.validators import RegexValidator from django.db import transaction from django.utils.functional import classproperty +from core.models import ManagedFile from extras.api.serializers import ScriptOutputSerializer from extras.choices import JobResultStatusChoices, LogLevelChoices from extras.models import JobResult @@ -531,7 +532,8 @@ def get_scripts(use_names=False): # Get all modules within the scripts path. These are the user-created files in which scripts are # defined. - modules = list(pkgutil.iter_modules([settings.SCRIPTS_ROOT])) + # modules = list(pkgutil.iter_modules([settings.SCRIPTS_ROOT])) + modules = [mf.get_module_info() for mf in ManagedFile.objects.filter(file_root='scripts')] modules_bases = set([name.split(".")[0] for _, name, _ in modules]) # Deleting from sys.modules needs to done behind a lock to prevent race conditions where a module is diff --git a/netbox/netbox/navigation/menu.py b/netbox/netbox/navigation/menu.py index f5d15b3d8..38037ca70 100644 --- a/netbox/netbox/navigation/menu.py +++ b/netbox/netbox/navigation/menu.py @@ -313,6 +313,7 @@ OTHER_MENU = Menu( get_model_item('extras', 'tag', 'Tags'), get_model_item('extras', 'configcontext', _('Config Contexts'), actions=['add']), get_model_item('extras', 'configtemplate', _('Config Templates'), actions=['add']), + get_model_item('core', 'managedfile', _('Managed Files'), actions=()), ), ), ), diff --git a/netbox/templates/core/managedfile.html b/netbox/templates/core/managedfile.html new file mode 100644 index 000000000..5f9e5cc55 --- /dev/null +++ b/netbox/templates/core/managedfile.html @@ -0,0 +1,57 @@ +{% extends 'generic/object.html' %} +{% load buttons %} +{% load custom_links %} +{% load helpers %} +{% load perms %} +{% load plugins %} + +{% block controls %} +
+
+ {% plugin_buttons object %} +
+ {% if request.user|can_delete:object %} + {% delete_button object %} + {% endif %} +
+ {% custom_links object %} +
+
+{% endblock controls %} + +{% block content %} +
+
+
+
Managed File
+
+ + + + + + + + + + + + + +
Root{{ object.get_file_root_display }}
Path + {{ object.file_path }} + + + +
Last Updated{{ object.last_updated }}
+
+
+ {% plugin_left_page object %} +
+
+
+
+ {% plugin_full_width_page object %} +
+
+{% endblock %}