18423 scripts upload to django-storages

base_requirements.txt

@@ -42,6 +42,10 @@ django-rich
 # https://github.com/rq/django-rq/blob/master/CHANGELOG.md
 django-rq
 
+# Provides a variety of storage backends
+# https://github.com/jschneier/django-storages/blob/master/CHANGELOG.rst
+django-storages
+
 # Abstraction models for rendering and paginating HTML tables
 # https://github.com/jieter/django-tables2/blob/master/CHANGELOG.md
 django-tables2

netbox/core/models/data.py

@@ -351,17 +351,6 @@ class DataFile(models.Model):
 
         return is_modified
 
-    def write_to_disk(self, path, overwrite=False):
-        """
-        Write the object's data to disk at the specified path
-        """
-        # Check whether file already exists
-        if os.path.isfile(path) and not overwrite:
-            raise FileExistsError()
-
-        with open(path, 'wb+') as new_file:
-            new_file.write(self.data)
-
 
 class AutoSyncRecord(models.Model):
     """

netbox/core/models/files.py

@@ -4,6 +4,7 @@ import os
 from django.conf import settings
 from django.core.exceptions import ValidationError
 from django.db import models
+from django.core.files.storage import storages
 from django.urls import reverse
 from django.utils.translation import gettext as _
 
@@ -19,6 +20,8 @@ logger = logging.getLogger('netbox.core.files')
 
 
 class ManagedFile(SyncedDataMixin, models.Model):
+    storage = None
+
     """
     Database representation for a file on disk. This class is typically wrapped by a proxy class (e.g. ScriptModule)
     to provide additional functionality.
@@ -84,7 +87,25 @@ class ManagedFile(SyncedDataMixin, models.Model):
     def sync_data(self):
         if self.data_file:
             self.file_path = os.path.basename(self.data_path)
-            self.data_file.write_to_disk(self.full_path, overwrite=True)
+            self._write_to_disk(self.full_path, overwrite=True)
+
+    def _write_to_disk(self, path, overwrite=False):
+        """
+        Write the object's data to disk at the specified path
+        """
+        # Check whether file already exists
+        storage = self.get_storage()
+        if storage.exists(path) and not overwrite:
+            raise FileExistsError()
+
+        with storage.open(path, 'wb+') as new_file:
+            new_file.write(self.data)
+
+    def get_storage(self):
+        if self.storage is None:
+            self.storage = storages.create_storage(storages.backends["scripts"])
+
+        return self.storage
 
     def clean(self):
         super().clean()
@@ -104,8 +125,9 @@ class ManagedFile(SyncedDataMixin, models.Model):
 
     def delete(self, *args, **kwargs):
         # Delete file from disk
+        storage = self.get_storage()
         try:
-            os.remove(self.full_path)
+            storage.delete(self.full_path)
         except FileNotFoundError:
             pass
 

netbox/extras/forms/scripts.py

@@ -1,11 +1,14 @@
 from django import forms
+from django.core.files.storage import storages
 from django.utils.translation import gettext_lazy as _
 
+from core.forms import ManagedFileForm
 from extras.choices import DurationChoices
 from utilities.forms.widgets import DateTimePicker, NumberWithOptions
 from utilities.datetime import local_now
 
 __all__ = (
+    'ScriptFileForm',
     'ScriptForm',
 )
 
@@ -55,3 +58,20 @@ class ScriptForm(forms.Form):
             self.cleaned_data['_schedule_at'] = local_now()
 
         return self.cleaned_data
+
+
+class ScriptFileForm(ManagedFileForm):
+    """
+    ManagedFileForm with a custom save method to use django-storages.
+    """
+    def save(self, *args, **kwargs):
+        # If a file was uploaded, save it to disk
+        if self.cleaned_data['upload_file']:
+            storage = storages.create_storage(storages.backends["scripts"])
+
+            self.instance.file_path = self.cleaned_data['upload_file'].name
+            data = self.cleaned_data['upload_file']
+            storage.save(self.instance.name, data)
+
+        # need to skip ManagedFileForm save method
+        return super(ManagedFileForm, self).save(*args, **kwargs)

netbox/extras/models/mixins.py

@@ -1,11 +1,39 @@
+import importlib.abc
+import importlib.util
 import os
-from importlib.machinery import SourceFileLoader
+import sys
+from django.core.files.storage import storages
 
 __all__ = (
     'PythonModuleMixin',
 )
 
 
+class CustomStoragesLoader(importlib.abc.Loader):
+    def __init__(self, filename):
+        self.filename = filename
+
+    def create_module(self, spec):
+        return None  # Use default module creation
+
+    def exec_module(self, module):
+        storage = storages.create_storage(storages.backends["scripts"])
+        with storage.open(self.filename, 'rb') as f:
+            code = f.read()
+        exec(code, module.__dict__)
+
+
+def load_module(module_name, filename):
+    spec = importlib.util.spec_from_file_location(module_name, filename)
+    if spec is None:
+        raise ModuleNotFoundError(f"Could not find module: {module_name}")
+    loader = CustomStoragesLoader(filename)
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[module_name] = module
+    loader.exec_module(module)
+    return module
+
+
 class PythonModuleMixin:
 
     def get_jobs(self, name):
@@ -33,6 +61,8 @@ class PythonModuleMixin:
             return name
 
     def get_module(self):
-        loader = SourceFileLoader(self.python_name, self.full_path)
+        # loader = SourceFileLoader(self.python_name, self.full_path)
-        module = loader.load_module()
+        # module = loader.load_module()
+        # module = load_module(self.python_name, self.full_path)
+        module = load_module(self.python_name, self.name)
         return module

netbox/extras/scripts.py

@@ -369,6 +369,7 @@ class BaseScript:
 
     @property
     def source(self):
+        breakpoint()
         return inspect.getsource(self.__class__)
 
     @classmethod
@@ -601,5 +602,6 @@ def is_variable(obj):
 
 def get_module_and_script(module_name, script_name):
     module = ScriptModule.objects.get(file_path=f'{module_name}.py')
+    breakpoint()
     script = module.scripts.get(name=script_name)
     return module, script

netbox/extras/storage.py

@@ -0,0 +1,10 @@
+from django.conf import settings
+from django.core.files.storage import FileSystemStorage
+from django.utils.functional import cached_property
+
+
+class ScriptFileSystemStorage(FileSystemStorage):
+
+    @cached_property
+    def base_location(self):
+        return settings.SCRIPTS_ROOT

netbox/extras/views.py

@@ -12,7 +12,6 @@ from django.utils.translation import gettext as _
 from django.views.generic import View
 
 from core.choices import ManagedFileRootPathChoices
-from core.forms import ManagedFileForm
 from core.models import Job
 from core.tables import JobTable
 from dcim.models import Device, DeviceRole, Platform
@@ -1163,7 +1162,7 @@ class DashboardWidgetDeleteView(LoginRequiredMixin, View):
 @register_model_view(ScriptModule, 'edit')
 class ScriptModuleCreateView(generic.ObjectEditView):
     queryset = ScriptModule.objects.all()
-    form = ManagedFileForm
+    form = forms.ScriptFileForm
 
     def alter_object(self, obj, *args, **kwargs):
         obj.file_root = ManagedFileRootPathChoices.SCRIPTS

netbox/netbox/settings.py

@@ -15,6 +15,7 @@ from netbox.config import PARAMS as CONFIG_PARAMS
 from netbox.constants import RQ_QUEUE_DEFAULT, RQ_QUEUE_HIGH, RQ_QUEUE_LOW
 from netbox.plugins import PluginConfig
 from netbox.registry import registry
+import storages.utils  # type: ignore
 from utilities.release import load_release_data
 from utilities.string import trailing_slash
 
@@ -175,6 +176,7 @@ SESSION_COOKIE_SECURE = getattr(configuration, 'SESSION_COOKIE_SECURE', False)
 SESSION_FILE_PATH = getattr(configuration, 'SESSION_FILE_PATH', None)
 STORAGE_BACKEND = getattr(configuration, 'STORAGE_BACKEND', None)
 STORAGE_CONFIG = getattr(configuration, 'STORAGE_CONFIG', {})
+STORAGES = getattr(configuration, 'STORAGES', None)
 TIME_ZONE = getattr(configuration, 'TIME_ZONE', 'UTC')
 TRANSLATION_ENABLED = getattr(configuration, 'TRANSLATION_ENABLED', True)
 
@@ -223,40 +225,44 @@ DATABASES = {
 # Storage backend
 #
 
+if STORAGE_BACKEND is not None:
+    warnings.warn(
+        "STORAGE_BACKEND is deprecated, use the new STORAGES setting instead."
+    )
+
+if STORAGE_BACKEND is not None and STORAGES is not None:
+    raise ImproperlyConfigured(
+        "STORAGE_BACKEND and STORAGES are both set, remove the deprecated STORAGE_BACKEND setting."
+    )
+
 # Default STORAGES for Django
-STORAGES = {
+if STORAGES is None:
+    STORAGES = {
         "default": {
             "BACKEND": "django.core.files.storage.FileSystemStorage",
         },
         "staticfiles": {
             "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage",
         },
-}
+        "scripts": {
+            "BACKEND": "extras.storage.ScriptFileSystemStorage",
+        },
+    }
 
+# TODO: This code is deprecated and needs to be removed in the future
 if STORAGE_BACKEND is not None:
     STORAGES['default']['BACKEND'] = STORAGE_BACKEND
 
-    # django-storages
+# Monkey-patch django-storages to fetch settings from STORAGE_CONFIG
-    if STORAGE_BACKEND.startswith('storages.'):
+if STORAGE_CONFIG is not None:
-        try:
-            import storages.utils  # type: ignore
-        except ModuleNotFoundError as e:
-            if getattr(e, 'name') == 'storages':
-                raise ImproperlyConfigured(
-                    f"STORAGE_BACKEND is set to {STORAGE_BACKEND} but django-storages is not present. It can be "
-                    f"installed by running 'pip install django-storages'."
-                )
-            raise e
-
-        # Monkey-patch django-storages to fetch settings from STORAGE_CONFIG
     def _setting(name, default=None):
         if name in STORAGE_CONFIG:
             return STORAGE_CONFIG[name]
         return globals().get(name, default)
     storages.utils.setting = _setting
 
-    # django-storage-swift
+# django-storage-swift
-    elif STORAGE_BACKEND == 'swift.storage.SwiftStorage':
+if STORAGE_BACKEND == 'swift.storage.SwiftStorage':
     try:
         import swift.utils  # noqa: F401
     except ModuleNotFoundError as e:
@@ -272,12 +278,6 @@ if STORAGE_BACKEND is not None:
         if param.startswith('SWIFT_'):
             globals()[param] = value
 
-if STORAGE_CONFIG and STORAGE_BACKEND is None:
-    warnings.warn(
-        "STORAGE_CONFIG has been set in configuration.py but STORAGE_BACKEND is not defined. STORAGE_CONFIG will be "
-        "ignored."
-    )
-
 
 #
 # Redis

requirements.txt

@@ -10,6 +10,7 @@ django-prometheus==2.3.1
 django-redis==5.4.0
 django-rich==1.13.0
 django-rq==3.0
+django-storages==1.14.4
 django-taggit==6.1.0
 django-tables2==2.7.5
 django-timezone-field==7.1