diff --git a/base_requirements.txt b/base_requirements.txt index ac1d1115e..4781652cd 100644 --- a/base_requirements.txt +++ b/base_requirements.txt @@ -2,6 +2,10 @@ # https://github.com/mozilla/bleach bleach<6.0 +# Python client for Amazon AWS API +# https://github.com/boto/boto3 +boto3 + # The Python web framework on which NetBox is built # https://github.com/django/django Django<4.2 diff --git a/docs/models/core/datasource.md b/docs/models/core/datasource.md index d16abdd10..0e18a2aae 100644 --- a/docs/models/core/datasource.md +++ b/docs/models/core/datasource.md @@ -14,15 +14,17 @@ The type of data source. Supported options include: * Local directory * git repository +* Amazon S3 bucket ### URL The URL identifying the remote source. Some examples are included below. -| Type | Example URL | -|------|-------------| -| Local | file:///var/my/data/source/ | -| git | https://https://github.com/my-organization/my-repo | +| Type | Example URL | +|-----------|----------------------------------------------------| +| Local | file:///path/to/my/data/ | +| git | https://github.com/my-organization/my-repo | +| Amazon S3 | https://s3.us-east-2.amazonaws.com/my-bucket-name/ | ### Status diff --git a/docs/release-notes/version-3.5.md b/docs/release-notes/version-3.5.md index ac2fab66b..792a244e4 100644 --- a/docs/release-notes/version-3.5.md +++ b/docs/release-notes/version-3.5.md @@ -10,7 +10,7 @@ The static home view has been replaced with a fully customizable dashboard. User #### Remote Data Sources ([#11558](https://github.com/netbox-community/netbox/issues/11558)) -NetBox now has the ability to synchronize arbitrary data from external sources through the new [DataSource](../models/core/datasource.md) and [DataFile](../models/core/datafile.md) models. Synchronized files are stored in the PostgreSQL database, and may be referenced and consumed by other NetBox models, such as export templates and config contexts. Currently, replication from local filesystem paths and from git repositories is supported, and we expect to add support for additional backends in the near future. +NetBox now has the ability to synchronize arbitrary data from external sources through the new [DataSource](../models/core/datasource.md) and [DataFile](../models/core/datafile.md) models. Synchronized files are stored in the PostgreSQL database, and may be referenced and consumed by other NetBox models, such as export templates and config contexts. Currently, replication from local filesystem paths, git repositories, and Amazon S3 buckets is supported, and we expect to introduce additional backends in the near future. #### Configuration Template Rendering ([#11559](https://github.com/netbox-community/netbox/issues/11559)) diff --git a/netbox/core/choices.py b/netbox/core/choices.py index 6927c83fb..be33afaba 100644 --- a/netbox/core/choices.py +++ b/netbox/core/choices.py @@ -10,10 +10,12 @@ from utilities.choices import ChoiceSet class DataSourceTypeChoices(ChoiceSet): LOCAL = 'local' GIT = 'git' + AMAZON_S3 = 'amazon-s3' CHOICES = ( (LOCAL, _('Local'), 'gray'), (GIT, _('Git'), 'blue'), + (AMAZON_S3, _('Amazon S3'), 'blue'), ) diff --git a/netbox/core/data_backends.py b/netbox/core/data_backends.py index 5d0e80584..1160b87ef 100644 --- a/netbox/core/data_backends.py +++ b/netbox/core/data_backends.py @@ -1,9 +1,14 @@ import logging +import os +import re import subprocess import tempfile from contextlib import contextmanager +from pathlib import Path from urllib.parse import quote, urlunparse, urlparse +import boto3 +from botocore.config import Config as Boto3Config from django import forms from django.conf import settings from django.utils.translation import gettext as _ @@ -115,3 +120,70 @@ class GitBackend(DataBackend): yield local_path.name local_path.cleanup() + + +@register_backend(DataSourceTypeChoices.AMAZON_S3) +class S3Backend(DataBackend): + parameters = { + 'aws_access_key_id': forms.CharField( + label=_('AWS access key ID'), + widget=forms.TextInput(attrs={'class': 'form-control'}) + ), + 'aws_secret_access_key': forms.CharField( + label=_('AWS secret access key'), + widget=forms.TextInput(attrs={'class': 'form-control'}) + ), + } + + REGION_REGEX = r's3\.([a-z0-9-]+)\.amazonaws\.com' + + @contextmanager + def fetch(self): + local_path = tempfile.TemporaryDirectory() + + # Build the S3 configuration + s3_config = Boto3Config( + proxies=settings.HTTP_PROXIES, + ) + + # Initialize the S3 resource and bucket + aws_access_key_id = self.params.get('aws_access_key_id') + aws_secret_access_key = self.params.get('aws_secret_access_key') + s3 = boto3.resource( + 's3', + region_name=self._region_name, + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + config=s3_config + ) + bucket = s3.Bucket(self._bucket_name) + + # Download all files within the specified path + for obj in bucket.objects.filter(Prefix=self._remote_path): + local_filename = os.path.join(local_path.name, obj.key) + # Build local path + Path(os.path.dirname(local_filename)).mkdir(parents=True, exist_ok=True) + bucket.download_file(obj.key, local_filename) + + yield local_path.name + + local_path.cleanup() + + @property + def _region_name(self): + domain = urlparse(self.url).netloc + if m := re.match(self.REGION_REGEX, domain): + return m.group(1) + return None + + @property + def _bucket_name(self): + url_path = urlparse(self.url).path.lstrip('/') + return url_path.split('/')[0] + + @property + def _remote_path(self): + url_path = urlparse(self.url).path.lstrip('/') + if '/' in url_path: + return url_path.split('/', 1)[1] + return '' diff --git a/requirements.txt b/requirements.txt index c66477589..d5f495b7a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ bleach==5.0.1 +boto3==1.26.91 Django==4.1.7 django-cors-headers==3.14.0 django-debug-toolbar==3.8.1