diff --git a/base_requirements.txt b/base_requirements.txt index 2d8055049..4b75b1313 100644 --- a/base_requirements.txt +++ b/base_requirements.txt @@ -2,10 +2,6 @@ # https://github.com/mozilla/bleach/blob/main/CHANGES bleach -# Python client for Amazon AWS API -# https://github.com/boto/boto3/blob/develop/CHANGELOG.rst -boto3 - # The Python web framework on which NetBox is built # https://docs.djangoproject.com/en/stable/releases/ Django<5.0 @@ -74,10 +70,6 @@ drf-spectacular # https://github.com/tfranzel/drf-spectacular-sidecar drf-spectacular-sidecar -# Git client for file sync -# https://github.com/jelmer/dulwich/releases -dulwich - # RSS feed parser # https://github.com/kurtmckee/feedparser/blob/develop/CHANGELOG.rst feedparser diff --git a/docs/features/synchronized-data.md b/docs/features/synchronized-data.md index ca6dfeda8..244a46f44 100644 --- a/docs/features/synchronized-data.md +++ b/docs/features/synchronized-data.md @@ -12,6 +12,10 @@ To enable remote data synchronization, the NetBox administrator first designates (Local disk paths are considered "remote" in this context as they exist outside NetBox's database. These paths could also be mapped to external network shares.) + +!!! info + Data backends which connect to external sources typically require the installation of one or more supporting Python libraries. The Git backend requires the [`dulwich`](https://www.dulwich.io/) package, and the S3 backend requires the [`boto3`](https://boto3.amazonaws.com/v1/documentation/api/latest/index.html) package. These must be installed within NetBox's environment to enable these backends. + Each type of remote source has its own configuration parameters. For instance, a git source will ask the user to specify a branch and authentication credentials. Once the source has been created, a synchronization job is run to automatically replicate remote files in the local database. The following NetBox models can be associated with replicated data files: diff --git a/docs/installation/3-netbox.md b/docs/installation/3-netbox.md index e0e656ce9..0713d12e3 100644 --- a/docs/installation/3-netbox.md +++ b/docs/installation/3-netbox.md @@ -211,6 +211,22 @@ By default, NetBox will use the local filesystem to store uploaded files. To use sudo sh -c "echo 'django-storages' >> /opt/netbox/local_requirements.txt" ``` +### Remote Data Sources + +NetBox supports integration with several remote data sources via configurable backends. Each of these requires the installation of one or more additional libraries. + +* Amazon S3: [`boto3`](https://boto3.amazonaws.com/v1/documentation/api/latest/index.html) +* Git: [`dulwich`](https://www.dulwich.io/) + +For example, to enable the Amazon S3 backend, add `boto3` to your local requirements file: + +```no-highlight +sudo sh -c "echo 'boto3' >> /opt/netbox/local_requirements.txt" +``` + +!!! info + These packages were previously required in NetBox v3.5 but now are optional. + ## Run the Upgrade Script Once NetBox has been configured, we're ready to proceed with the actual installation. We'll run the packaged upgrade script (`upgrade.sh`) to perform the following actions: diff --git a/netbox/core/data_backends.py b/netbox/core/data_backends.py index 43e6f4e79..d947602a6 100644 --- a/netbox/core/data_backends.py +++ b/netbox/core/data_backends.py @@ -6,13 +6,9 @@ from contextlib import contextmanager from pathlib import Path from urllib.parse import urlparse -import boto3 -from botocore.config import Config as Boto3Config from django import forms from django.conf import settings from django.utils.translation import gettext as _ -from dulwich import porcelain -from dulwich.config import ConfigDict from netbox.registry import registry from .choices import DataSourceTypeChoices @@ -43,9 +39,20 @@ class DataBackend: parameters = {} sensitive_parameters = [] + # Prevent Django's template engine from calling the backend + # class when referenced via DataSource.backend_class + do_not_call_in_templates = True + def __init__(self, url, **kwargs): self.url = url self.params = kwargs + self.config = self.init_config() + + def init_config(self): + """ + Hook to initialize the instance's configuration. + """ + return @property def url_scheme(self): @@ -58,6 +65,7 @@ class DataBackend: @register_backend(DataSourceTypeChoices.LOCAL) class LocalBackend(DataBackend): + @contextmanager def fetch(self): logger.debug(f"Data source type is local; skipping fetch") @@ -89,14 +97,28 @@ class GitBackend(DataBackend): } sensitive_parameters = ['password'] + def init_config(self): + from dulwich.config import ConfigDict + + # Initialize backend config + config = ConfigDict() + + # Apply HTTP proxy (if configured) + if settings.HTTP_PROXIES and self.url_scheme in ('http', 'https'): + if proxy := settings.HTTP_PROXIES.get(self.url_scheme): + config.set("http", "proxy", proxy) + + return config + @contextmanager def fetch(self): + from dulwich import porcelain + local_path = tempfile.TemporaryDirectory() - config = ConfigDict() clone_args = { "branch": self.params.get('branch'), - "config": config, + "config": self.config, "depth": 1, "errstream": porcelain.NoneStream(), "quiet": True, @@ -110,10 +132,6 @@ class GitBackend(DataBackend): } ) - if settings.HTTP_PROXIES and self.url_scheme in ('http', 'https'): - if proxy := settings.HTTP_PROXIES.get(self.url_scheme): - config.set("http", "proxy", proxy) - logger.debug(f"Cloning git repo: {self.url}") try: porcelain.clone(self.url, local_path.name, **clone_args) @@ -141,15 +159,20 @@ class S3Backend(DataBackend): REGION_REGEX = r's3\.([a-z0-9-]+)\.amazonaws\.com' - @contextmanager - def fetch(self): - local_path = tempfile.TemporaryDirectory() + def init_config(self): + from botocore.config import Config as Boto3Config - # Build the S3 configuration - s3_config = Boto3Config( + # Initialize backend config + return Boto3Config( proxies=settings.HTTP_PROXIES, ) + @contextmanager + def fetch(self): + import boto3 + + local_path = tempfile.TemporaryDirectory() + # Initialize the S3 resource and bucket aws_access_key_id = self.params.get('aws_access_key_id') aws_secret_access_key = self.params.get('aws_secret_access_key') @@ -158,7 +181,7 @@ class S3Backend(DataBackend): region_name=self._region_name, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, - config=s3_config + config=self.config ) bucket = s3.Bucket(self._bucket_name) diff --git a/netbox/core/models/data.py b/netbox/core/models/data.py index d1a033798..87ca909c2 100644 --- a/netbox/core/models/data.py +++ b/netbox/core/models/data.py @@ -104,6 +104,10 @@ class DataSource(JobsMixin, PrimaryModel): def url_scheme(self): return urlparse(self.source_url).scheme.lower() + @property + def backend_class(self): + return registry['data_backends'].get(self.type) + @property def is_local(self): return self.type == DataSourceTypeChoices.LOCAL @@ -139,17 +143,15 @@ class DataSource(JobsMixin, PrimaryModel): ) def get_backend(self): - backend_cls = registry['data_backends'].get(self.type) backend_params = self.parameters or {} - - return backend_cls(self.source_url, **backend_params) + return self.backend_class(self.source_url, **backend_params) def sync(self): """ Create/update/delete child DataFiles as necessary to synchronize with the remote source. """ if self.status == DataSourceStatusChoices.SYNCING: - raise SyncError(f"Cannot initiate sync; syncing already in progress.") + raise SyncError("Cannot initiate sync; syncing already in progress.") # Emit the pre_sync signal pre_sync.send(sender=self.__class__, instance=self) @@ -158,7 +160,12 @@ class DataSource(JobsMixin, PrimaryModel): DataSource.objects.filter(pk=self.pk).update(status=self.status) # Replicate source data locally - backend = self.get_backend() + try: + backend = self.get_backend() + except ModuleNotFoundError as e: + raise SyncError( + f"There was an error initializing the backend. A dependency needs to be installed: {e}" + ) with backend.fetch() as local_path: logger.debug(f'Syncing files from source root {local_path}') diff --git a/netbox/templates/core/datasource.html b/netbox/templates/core/datasource.html index d236e13e5..369c395f8 100644 --- a/netbox/templates/core/datasource.html +++ b/netbox/templates/core/datasource.html @@ -85,24 +85,26 @@
{% trans "Backend" %}
- - {% for name, field in object.get_backend.parameters.items %} - - - {% if name in object.get_backend.sensitive_parameters and not perms.core.change_datasource %} - - {% else %} - - {% endif %} - - {% empty %} - - - - {% endfor %} -
{{ field.label }}********{{ object.parameters|get_key:name|placeholder }}
- {% trans "No parameters defined" %} -
+ {% with backend=object.backend_class %} + + {% for name, field in backend.parameters.items %} + + + {% if name in backend.sensitive_parameters and not perms.core.change_datasource %} + + {% else %} + + {% endif %} + + {% empty %} + + + + {% endfor %} +
{{ field.label }}********{{ object.parameters|get_key:name|placeholder }}
+ {% trans "No parameters defined" %} +
+ {% endwith %}
{% include 'inc/panels/related_objects.html' %} diff --git a/requirements.txt b/requirements.txt index eef9e1434..949a1bf87 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ bleach==6.0.0 -boto3==1.28.14 django-cors-headers==4.2.0 django-debug-toolbar==4.1.0 django-filter==23.2 @@ -16,7 +15,6 @@ django-timezone-field==5.1 djangorestframework==3.14.0 drf-spectacular==0.26.4 drf-spectacular-sidecar==2023.7.1 -dulwich==0.21.5 feedparser==6.0.10 graphene-django==3.0.0 gunicorn==21.2.0