diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d9692194..87d5e42ef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,6 +60,7 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements.txt + pip install boto3 dulwich pip install pycodestyle coverage tblib - name: Build documentation diff --git a/docs/installation/3-netbox.md b/docs/installation/3-netbox.md index e0e656ce9..0713d12e3 100644 --- a/docs/installation/3-netbox.md +++ b/docs/installation/3-netbox.md @@ -211,6 +211,22 @@ By default, NetBox will use the local filesystem to store uploaded files. To use sudo sh -c "echo 'django-storages' >> /opt/netbox/local_requirements.txt" ``` +### Remote Data Sources + +NetBox supports integration with several remote data sources via configurable backends. Each of these requires the installation of one or more additional libraries. + +* Amazon S3: [`boto3`](https://boto3.amazonaws.com/v1/documentation/api/latest/index.html) +* Git: [`dulwich`](https://www.dulwich.io/) + +For example, to enable the Amazon S3 backend, add `boto3` to your local requirements file: + +```no-highlight +sudo sh -c "echo 'boto3' >> /opt/netbox/local_requirements.txt" +``` + +!!! info + These packages were previously required in NetBox v3.5 but now are optional. + ## Run the Upgrade Script Once NetBox has been configured, we're ready to proceed with the actual installation. We'll run the packaged upgrade script (`upgrade.sh`) to perform the following actions: diff --git a/netbox/core/data_backends.py b/netbox/core/data_backends.py index 43e6f4e79..73c248d08 100644 --- a/netbox/core/data_backends.py +++ b/netbox/core/data_backends.py @@ -6,13 +6,9 @@ from contextlib import contextmanager from pathlib import Path from urllib.parse import urlparse -import boto3 -from botocore.config import Config as Boto3Config from django import forms from django.conf import settings from django.utils.translation import gettext as _ -from dulwich import porcelain -from dulwich.config import ConfigDict from netbox.registry import registry from .choices import DataSourceTypeChoices @@ -43,9 +39,27 @@ class DataBackend: parameters = {} sensitive_parameters = [] + # Prevent Django's template engine from calling the backend + # class when referenced via DataSource.backend_class + do_not_call_in_templates = True + def __init__(self, url, **kwargs): self.url = url self.params = kwargs + self.config = self.init_config() + + def init_config(self): + """ + Hook to initialize the instance's configuration. + """ + return + + def handle_missing_dependency(self, e): + """ + Hook for handling exceptions related to the attempted import of missing modules. Returns an exception + to be raised. + """ + return e @property def url_scheme(self): @@ -58,6 +72,7 @@ class DataBackend: @register_backend(DataSourceTypeChoices.LOCAL) class LocalBackend(DataBackend): + @contextmanager def fetch(self): logger.debug(f"Data source type is local; skipping fetch") @@ -89,14 +104,40 @@ class GitBackend(DataBackend): } sensitive_parameters = ['password'] + def init_config(self): + try: + from dulwich.config import ConfigDict + except ModuleNotFoundError as e: + raise self.handle_missing_dependency(e) + + # Initialize backend config + config = ConfigDict() + + # Apply HTTP proxy (if configured) + if settings.HTTP_PROXIES and self.url_scheme in ('http', 'https'): + if proxy := settings.HTTP_PROXIES.get(self.url_scheme): + config.set("http", "proxy", proxy) + + return config + + def handle_missing_dependency(self, e): + return ImportError(_( + "Unable to initialize the git data backend: dulwich library is not installed. Run 'pip install dulwich' " + "within the NetBox Python environment to install it." + )) + @contextmanager def fetch(self): + try: + from dulwich import porcelain + except ModuleNotFoundError as e: + raise self.handle_missing_dependency(e) + local_path = tempfile.TemporaryDirectory() - config = ConfigDict() clone_args = { "branch": self.params.get('branch'), - "config": config, + "config": self.config, "depth": 1, "errstream": porcelain.NoneStream(), "quiet": True, @@ -110,10 +151,6 @@ class GitBackend(DataBackend): } ) - if settings.HTTP_PROXIES and self.url_scheme in ('http', 'https'): - if proxy := settings.HTTP_PROXIES.get(self.url_scheme): - config.set("http", "proxy", proxy) - logger.debug(f"Cloning git repo: {self.url}") try: porcelain.clone(self.url, local_path.name, **clone_args) @@ -141,15 +178,32 @@ class S3Backend(DataBackend): REGION_REGEX = r's3\.([a-z0-9-]+)\.amazonaws\.com' - @contextmanager - def fetch(self): - local_path = tempfile.TemporaryDirectory() + def init_config(self): + try: + from botocore.config import Config as Boto3Config + except ModuleNotFoundError as e: + raise self.handle_missing_dependency(e) - # Build the S3 configuration - s3_config = Boto3Config( + # Initialize backend config + return Boto3Config( proxies=settings.HTTP_PROXIES, ) + def handle_missing_dependency(self, e): + return ImportError(_( + "Unable to initialize the Amazon S3 backend: boto3 library is not installed. Run 'pip install boto3' " + "within the NetBox Python environment to install it." + )) + + @contextmanager + def fetch(self): + try: + import boto3 + except ModuleNotFoundError as e: + raise self.handle_missing_dependency(e) + + local_path = tempfile.TemporaryDirectory() + # Initialize the S3 resource and bucket aws_access_key_id = self.params.get('aws_access_key_id') aws_secret_access_key = self.params.get('aws_secret_access_key') @@ -158,7 +212,7 @@ class S3Backend(DataBackend): region_name=self._region_name, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, - config=s3_config + config=self.config ) bucket = s3.Bucket(self._bucket_name) diff --git a/netbox/core/models/data.py b/netbox/core/models/data.py index d1a033798..cdd5bf49b 100644 --- a/netbox/core/models/data.py +++ b/netbox/core/models/data.py @@ -104,6 +104,10 @@ class DataSource(JobsMixin, PrimaryModel): def url_scheme(self): return urlparse(self.source_url).scheme.lower() + @property + def backend_class(self): + return registry['data_backends'].get(self.type) + @property def is_local(self): return self.type == DataSourceTypeChoices.LOCAL @@ -139,10 +143,8 @@ class DataSource(JobsMixin, PrimaryModel): ) def get_backend(self): - backend_cls = registry['data_backends'].get(self.type) backend_params = self.parameters or {} - - return backend_cls(self.source_url, **backend_params) + return self.backend_class(self.source_url, **backend_params) def sync(self): """ diff --git a/netbox/templates/core/datasource.html b/netbox/templates/core/datasource.html index d236e13e5..369c395f8 100644 --- a/netbox/templates/core/datasource.html +++ b/netbox/templates/core/datasource.html @@ -85,24 +85,26 @@
{% trans "Backend" %}
- - {% for name, field in object.get_backend.parameters.items %} - - - {% if name in object.get_backend.sensitive_parameters and not perms.core.change_datasource %} - - {% else %} - - {% endif %} - - {% empty %} - - - - {% endfor %} -
{{ field.label }}********{{ object.parameters|get_key:name|placeholder }}
- {% trans "No parameters defined" %} -
+ {% with backend=object.backend_class %} + + {% for name, field in backend.parameters.items %} + + + {% if name in backend.sensitive_parameters and not perms.core.change_datasource %} + + {% else %} + + {% endif %} + + {% empty %} + + + + {% endfor %} +
{{ field.label }}********{{ object.parameters|get_key:name|placeholder }}
+ {% trans "No parameters defined" %} +
+ {% endwith %}
{% include 'inc/panels/related_objects.html' %}