Initial work on #12906

.github/workflows/ci.yml

@@ -60,6 +60,7 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install -r requirements.txt
+        pip install boto3 dulwich
         pip install pycodestyle coverage tblib
 
     - name: Build documentation

docs/installation/3-netbox.md

@@ -211,6 +211,22 @@ By default, NetBox will use the local filesystem to store uploaded files. To use
 sudo sh -c "echo 'django-storages' >> /opt/netbox/local_requirements.txt"
 ```
 
+### Remote Data Sources
+
+NetBox supports integration with several remote data sources via configurable backends. Each of these requires the installation of one or more additional libraries.
+
+* Amazon S3: [`boto3`](https://boto3.amazonaws.com/v1/documentation/api/latest/index.html)
+* Git: [`dulwich`](https://www.dulwich.io/)
+
+For example, to enable the Amazon S3 backend, add `boto3` to your local requirements file:
+
+```no-highlight
+sudo sh -c "echo 'boto3' >> /opt/netbox/local_requirements.txt"
+```
+
+!!! info
+    These packages were previously required in NetBox v3.5 but now are optional.
+
 ## Run the Upgrade Script
 
 Once NetBox has been configured, we're ready to proceed with the actual installation. We'll run the packaged upgrade script (`upgrade.sh`) to perform the following actions:

netbox/core/data_backends.py

@@ -6,13 +6,9 @@ from contextlib import contextmanager
 from pathlib import Path
 from urllib.parse import urlparse
 
-import boto3
-from botocore.config import Config as Boto3Config
 from django import forms
 from django.conf import settings
 from django.utils.translation import gettext as _
-from dulwich import porcelain
-from dulwich.config import ConfigDict
 
 from netbox.registry import registry
 from .choices import DataSourceTypeChoices
@@ -43,9 +39,27 @@ class DataBackend:
     parameters = {}
     sensitive_parameters = []
 
+    # Prevent Django's template engine from calling the backend
+    # class when referenced via DataSource.backend_class
+    do_not_call_in_templates = True
+
     def __init__(self, url, **kwargs):
         self.url = url
         self.params = kwargs
+        self.config = self.init_config()
+
+    def init_config(self):
+        """
+        Hook to initialize the instance's configuration.
+        """
+        return
+
+    def handle_missing_dependency(self, e):
+        """
+        Hook for handling exceptions related to the attempted import of missing modules. Returns an exception
+        to be raised.
+        """
+        return e
 
     @property
     def url_scheme(self):
@@ -58,6 +72,7 @@ class DataBackend:
 
 @register_backend(DataSourceTypeChoices.LOCAL)
 class LocalBackend(DataBackend):
+
     @contextmanager
     def fetch(self):
         logger.debug(f"Data source type is local; skipping fetch")
@@ -89,14 +104,40 @@ class GitBackend(DataBackend):
     }
     sensitive_parameters = ['password']
 
+    def init_config(self):
+        try:
+            from dulwich.config import ConfigDict
+        except ModuleNotFoundError as e:
+            raise self.handle_missing_dependency(e)
+
+        # Initialize backend config
+        config = ConfigDict()
+
+        # Apply HTTP proxy (if configured)
+        if settings.HTTP_PROXIES and self.url_scheme in ('http', 'https'):
+            if proxy := settings.HTTP_PROXIES.get(self.url_scheme):
+                config.set("http", "proxy", proxy)
+
+        return config
+
+    def handle_missing_dependency(self, e):
+        return ImportError(_(
+            "Unable to initialize the git data backend: dulwich library is not installed. Run 'pip install dulwich' "
+            "within the NetBox Python environment to install it."
+        ))
+
     @contextmanager
     def fetch(self):
+        try:
+            from dulwich import porcelain
+        except ModuleNotFoundError as e:
+            raise self.handle_missing_dependency(e)
+
         local_path = tempfile.TemporaryDirectory()
 
-        config = ConfigDict()
         clone_args = {
             "branch": self.params.get('branch'),
-            "config": config,
+            "config": self.config,
             "depth": 1,
             "errstream": porcelain.NoneStream(),
             "quiet": True,
@@ -110,10 +151,6 @@ class GitBackend(DataBackend):
                 }
             )
 
-        if settings.HTTP_PROXIES and self.url_scheme in ('http', 'https'):
-            if proxy := settings.HTTP_PROXIES.get(self.url_scheme):
-                config.set("http", "proxy", proxy)
-
         logger.debug(f"Cloning git repo: {self.url}")
         try:
             porcelain.clone(self.url, local_path.name, **clone_args)
@@ -141,15 +178,32 @@ class S3Backend(DataBackend):
 
     REGION_REGEX = r's3\.([a-z0-9-]+)\.amazonaws\.com'
 
-    @contextmanager
+    def init_config(self):
-    def fetch(self):
+        try:
-        local_path = tempfile.TemporaryDirectory()
+            from botocore.config import Config as Boto3Config
+        except ModuleNotFoundError as e:
+            raise self.handle_missing_dependency(e)
 
-        # Build the S3 configuration
+        # Initialize backend config
-        s3_config = Boto3Config(
+        return Boto3Config(
             proxies=settings.HTTP_PROXIES,
         )
 
+    def handle_missing_dependency(self, e):
+        return ImportError(_(
+            "Unable to initialize the Amazon S3 backend: boto3 library is not installed. Run 'pip install boto3' "
+            "within the NetBox Python environment to install it."
+        ))
+
+    @contextmanager
+    def fetch(self):
+        try:
+            import boto3
+        except ModuleNotFoundError as e:
+            raise self.handle_missing_dependency(e)
+
+        local_path = tempfile.TemporaryDirectory()
+
         # Initialize the S3 resource and bucket
         aws_access_key_id = self.params.get('aws_access_key_id')
         aws_secret_access_key = self.params.get('aws_secret_access_key')
@@ -158,7 +212,7 @@ class S3Backend(DataBackend):
             region_name=self._region_name,
             aws_access_key_id=aws_access_key_id,
             aws_secret_access_key=aws_secret_access_key,
-            config=s3_config
+            config=self.config
         )
         bucket = s3.Bucket(self._bucket_name)
 

netbox/core/models/data.py

@@ -104,6 +104,10 @@ class DataSource(JobsMixin, PrimaryModel):
     def url_scheme(self):
         return urlparse(self.source_url).scheme.lower()
 
+    @property
+    def backend_class(self):
+        return registry['data_backends'].get(self.type)
+
     @property
     def is_local(self):
         return self.type == DataSourceTypeChoices.LOCAL
@@ -139,10 +143,8 @@ class DataSource(JobsMixin, PrimaryModel):
         )
 
     def get_backend(self):
-        backend_cls = registry['data_backends'].get(self.type)
         backend_params = self.parameters or {}
-
+        return self.backend_class(self.source_url, **backend_params)
-        return backend_cls(self.source_url, **backend_params)
 
     def sync(self):
         """

netbox/templates/core/datasource.html

@@ -85,24 +85,26 @@
       <div class="card">
         <h5 class="card-header">{% trans "Backend" %}</h5>
         <div class="card-body">
-          <table class="table table-hover attr-table">
+          {% with backend=object.backend_class %}
-            {% for name, field in object.get_backend.parameters.items %}
+            <table class="table table-hover attr-table">
-              <tr>
+              {% for name, field in backend.parameters.items %}
-                <th scope="row">{{ field.label }}</th>
+                <tr>
-                {% if name in object.get_backend.sensitive_parameters and not perms.core.change_datasource %}
+                  <th scope="row">{{ field.label }}</th>
-                  <td>********</td>
+                  {% if name in backend.sensitive_parameters and not perms.core.change_datasource %}
-                {% else %}
+                    <td>********</td>
-                  <td>{{ object.parameters|get_key:name|placeholder }}</td>
+                  {% else %}
-                {% endif %}
+                    <td>{{ object.parameters|get_key:name|placeholder }}</td>
-              </tr>
+                  {% endif %}
-            {% empty %}
+                </tr>
-              <tr>
+              {% empty %}
-                <td colspan="2" class="text-muted">
+                <tr>
-                  {% trans "No parameters defined" %}
+                  <td colspan="2" class="text-muted">
-                </td>
+                    {% trans "No parameters defined" %}
-              </tr>
+                  </td>
-            {% endfor %}
+                </tr>
-          </table>
+              {% endfor %}
+            </table>
+            {% endwith %}
         </div>
       </div>
       {% include 'inc/panels/related_objects.html' %}