Closes #13381: Enable plugins to register custom data backends (#14095)

* Initial work on #13381

* Fix backend type display in table column

* Fix data source type choices during bulk edit

* Misc cleanup

* Move backend utils from core app to netbox

* Move backend type validation from serializer to model
This commit is contained in:
Jeremy Stretch 2023-10-24 11:35:53 -04:00 committed by GitHub
parent 7274e75b26
commit 30ce9edf1c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 250 additions and 113 deletions

View File

@ -0,0 +1,23 @@
# Data Backends
[Data sources](../../models/core/datasource.md) can be defined to reference data which exists on systems of record outside NetBox, such as a git repository or Amazon S3 bucket. Plugins can register their own backend classes to introduce support for additional resource types. This is done by subclassing NetBox's `DataBackend` class.
```python title="data_backends.py"
from netbox.data_backends import DataBackend
class MyDataBackend(DataBackend):
name = 'mybackend'
label = 'My Backend'
...
```
To register one or more data backends with NetBox, define a list named `backends` at the end of this file:
```python title="data_backends.py"
backends = [MyDataBackend]
```
!!! tip
The path to the list of search indexes can be modified by setting `data_backends` in the PluginConfig instance.
::: core.data_backends.DataBackend

View File

@ -109,6 +109,7 @@ NetBox looks for the `config` variable within a plugin's `__init__.py` to load i
| `middleware` | A list of middleware classes to append after NetBox's build-in middleware |
| `queues` | A list of custom background task queues to create |
| `search_extensions` | The dotted path to the list of search index classes (default: `search.indexes`) |
| `data_backends` | The dotted path to the list of data source backend classes (default: `data_backends.backends`) |
| `template_extensions` | The dotted path to the list of template extension classes (default: `template_content.template_extensions`) |
| `menu_items` | The dotted path to the list of menu items provided by the plugin (default: `navigation.menu_items`) |
| `graphql_schema` | The dotted path to the plugin's GraphQL schema class, if any (default: `graphql.schema`) |

View File

@ -136,6 +136,7 @@ nav:
- Forms: 'plugins/development/forms.md'
- Filters & Filter Sets: 'plugins/development/filtersets.md'
- Search: 'plugins/development/search.md'
- Data Backends: 'plugins/development/data-backends.md'
- REST API: 'plugins/development/rest-api.md'
- GraphQL API: 'plugins/development/graphql-api.md'
- Background Tasks: 'plugins/development/background-tasks.md'

View File

@ -4,6 +4,7 @@ from core.choices import *
from core.models import *
from netbox.api.fields import ChoiceField, ContentTypeField
from netbox.api.serializers import BaseModelSerializer, NetBoxModelSerializer
from netbox.utils import get_data_backend_choices
from users.api.nested_serializers import NestedUserSerializer
from .nested_serializers import *
@ -19,7 +20,7 @@ class DataSourceSerializer(NetBoxModelSerializer):
view_name='core-api:datasource-detail'
)
type = ChoiceField(
choices=DataSourceTypeChoices
choices=get_data_backend_choices()
)
status = ChoiceField(
choices=DataSourceStatusChoices,

View File

@ -7,18 +7,6 @@ from utilities.choices import ChoiceSet
# Data sources
#
class DataSourceTypeChoices(ChoiceSet):
LOCAL = 'local'
GIT = 'git'
AMAZON_S3 = 'amazon-s3'
CHOICES = (
(LOCAL, _('Local'), 'gray'),
(GIT, 'Git', 'blue'),
(AMAZON_S3, 'Amazon S3', 'blue'),
)
class DataSourceStatusChoices(ChoiceSet):
NEW = 'new'
QUEUED = 'queued'

View File

@ -10,61 +10,24 @@ from django import forms
from django.conf import settings
from django.utils.translation import gettext as _
from netbox.registry import registry
from .choices import DataSourceTypeChoices
from netbox.data_backends import DataBackend
from netbox.utils import register_data_backend
from .exceptions import SyncError
__all__ = (
'LocalBackend',
'GitBackend',
'LocalBackend',
'S3Backend',
)
logger = logging.getLogger('netbox.data_backends')
def register_backend(name):
"""
Decorator for registering a DataBackend class.
"""
def _wrapper(cls):
registry['data_backends'][name] = cls
return cls
return _wrapper
class DataBackend:
parameters = {}
sensitive_parameters = []
# Prevent Django's template engine from calling the backend
# class when referenced via DataSource.backend_class
do_not_call_in_templates = True
def __init__(self, url, **kwargs):
self.url = url
self.params = kwargs
self.config = self.init_config()
def init_config(self):
"""
Hook to initialize the instance's configuration.
"""
return
@property
def url_scheme(self):
return urlparse(self.url).scheme.lower()
@contextmanager
def fetch(self):
raise NotImplemented()
@register_backend(DataSourceTypeChoices.LOCAL)
@register_data_backend()
class LocalBackend(DataBackend):
name = 'local'
label = _('Local')
is_local = True
@contextmanager
def fetch(self):
@ -74,8 +37,10 @@ class LocalBackend(DataBackend):
yield local_path
@register_backend(DataSourceTypeChoices.GIT)
@register_data_backend()
class GitBackend(DataBackend):
name = 'git'
label = 'Git'
parameters = {
'username': forms.CharField(
required=False,
@ -144,8 +109,10 @@ class GitBackend(DataBackend):
local_path.cleanup()
@register_backend(DataSourceTypeChoices.AMAZON_S3)
@register_data_backend()
class S3Backend(DataBackend):
name = 'amazon-s3'
label = 'Amazon S3'
parameters = {
'aws_access_key_id': forms.CharField(
label=_('AWS access key ID'),

View File

@ -4,6 +4,7 @@ from django.utils.translation import gettext as _
import django_filters
from netbox.filtersets import BaseFilterSet, ChangeLoggedModelFilterSet, NetBoxModelFilterSet
from netbox.utils import get_data_backend_choices
from .choices import *
from .models import *
@ -16,7 +17,7 @@ __all__ = (
class DataSourceFilterSet(NetBoxModelFilterSet):
type = django_filters.MultipleChoiceFilter(
choices=DataSourceTypeChoices,
choices=get_data_backend_choices,
null_value=None
)
status = django_filters.MultipleChoiceFilter(

View File

@ -1,10 +1,9 @@
from django import forms
from django.utils.translation import gettext_lazy as _
from core.choices import DataSourceTypeChoices
from core.models import *
from netbox.forms import NetBoxModelBulkEditForm
from utilities.forms import add_blank_choice
from netbox.utils import get_data_backend_choices
from utilities.forms.fields import CommentField
from utilities.forms.widgets import BulkEditNullBooleanSelect
@ -16,9 +15,8 @@ __all__ = (
class DataSourceBulkEditForm(NetBoxModelBulkEditForm):
type = forms.ChoiceField(
label=_('Type'),
choices=add_blank_choice(DataSourceTypeChoices),
required=False,
initial=''
choices=get_data_backend_choices,
required=False
)
enabled = forms.NullBooleanField(
required=False,

View File

@ -8,6 +8,7 @@ from core.models import *
from extras.forms.mixins import SavedFiltersMixin
from extras.utils import FeatureQuery
from netbox.forms import NetBoxModelFilterSetForm
from netbox.utils import get_data_backend_choices
from utilities.forms import BOOLEAN_WITH_BLANK_CHOICES, FilterForm
from utilities.forms.fields import ContentTypeChoiceField, DynamicModelMultipleChoiceField
from utilities.forms.widgets import APISelectMultiple, DateTimePicker
@ -27,7 +28,7 @@ class DataSourceFilterForm(NetBoxModelFilterSetForm):
)
type = forms.MultipleChoiceField(
label=_('Type'),
choices=DataSourceTypeChoices,
choices=get_data_backend_choices,
required=False
)
status = forms.MultipleChoiceField(

View File

@ -7,6 +7,7 @@ from core.forms.mixins import SyncedDataMixin
from core.models import *
from netbox.forms import NetBoxModelForm
from netbox.registry import registry
from netbox.utils import get_data_backend_choices
from utilities.forms import get_field_value
from utilities.forms.fields import CommentField
from utilities.forms.widgets import HTMXSelect
@ -18,6 +19,10 @@ __all__ = (
class DataSourceForm(NetBoxModelForm):
type = forms.ChoiceField(
choices=get_data_backend_choices,
widget=HTMXSelect()
)
comments = CommentField()
class Meta:
@ -26,7 +31,6 @@ class DataSourceForm(NetBoxModelForm):
'name', 'type', 'source_url', 'enabled', 'description', 'comments', 'ignore_rules', 'tags',
]
widgets = {
'type': HTMXSelect(),
'ignore_rules': forms.Textarea(
attrs={
'rows': 5,
@ -56,12 +60,13 @@ class DataSourceForm(NetBoxModelForm):
# Add backend-specific form fields
self.backend_fields = []
for name, form_field in backend.parameters.items():
field_name = f'backend_{name}'
self.backend_fields.append(field_name)
self.fields[field_name] = copy.copy(form_field)
if self.instance and self.instance.parameters:
self.fields[field_name].initial = self.instance.parameters.get(name)
if backend:
for name, form_field in backend.parameters.items():
field_name = f'backend_{name}'
self.backend_fields.append(field_name)
self.fields[field_name] = copy.copy(form_field)
if self.instance and self.instance.parameters:
self.fields[field_name].initial = self.instance.parameters.get(name)
def save(self, *args, **kwargs):

View File

@ -0,0 +1,18 @@
# Generated by Django 4.2.6 on 2023-10-20 17:47
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0005_job_created_auto_now'),
]
operations = [
migrations.AlterField(
model_name='datasource',
name='type',
field=models.CharField(max_length=50),
),
]

View File

@ -45,9 +45,7 @@ class DataSource(JobsMixin, PrimaryModel):
)
type = models.CharField(
verbose_name=_('type'),
max_length=50,
choices=DataSourceTypeChoices,
default=DataSourceTypeChoices.LOCAL
max_length=50
)
source_url = models.CharField(
max_length=200,
@ -96,8 +94,9 @@ class DataSource(JobsMixin, PrimaryModel):
def docs_url(self):
return f'{settings.STATIC_URL}docs/models/{self._meta.app_label}/{self._meta.model_name}/'
def get_type_color(self):
return DataSourceTypeChoices.colors.get(self.type)
def get_type_display(self):
if backend := registry['data_backends'].get(self.type):
return backend.label
def get_status_color(self):
return DataSourceStatusChoices.colors.get(self.status)
@ -110,10 +109,6 @@ class DataSource(JobsMixin, PrimaryModel):
def backend_class(self):
return registry['data_backends'].get(self.type)
@property
def is_local(self):
return self.type == DataSourceTypeChoices.LOCAL
@property
def ready_for_sync(self):
return self.enabled and self.status not in (
@ -123,8 +118,14 @@ class DataSource(JobsMixin, PrimaryModel):
def clean(self):
# Validate data backend type
if self.type and self.type not in registry['data_backends']:
raise ValidationError({
'type': _("Unknown backend type: {type}".format(type=self.type))
})
# Ensure URL scheme matches selected type
if self.type == DataSourceTypeChoices.LOCAL and self.url_scheme not in ('file', ''):
if self.backend_class.is_local and self.url_scheme not in ('file', ''):
raise ValidationError({
'source_url': f"URLs for local sources must start with file:// (or specify no scheme)"
})

View File

@ -0,0 +1,20 @@
import django_tables2 as tables
from netbox.registry import registry
__all__ = (
'BackendTypeColumn',
)
class BackendTypeColumn(tables.Column):
"""
Display a data backend type.
"""
def render(self, value):
if backend := registry['data_backends'].get(value):
return backend.label
return value
def value(self, value):
return value

View File

@ -3,6 +3,7 @@ import django_tables2 as tables
from core.models import *
from netbox.tables import NetBoxTable, columns
from .columns import BackendTypeColumn
__all__ = (
'DataFileTable',
@ -15,8 +16,8 @@ class DataSourceTable(NetBoxTable):
verbose_name=_('Name'),
linkify=True
)
type = columns.ChoiceFieldColumn(
verbose_name=_('Type'),
type = BackendTypeColumn(
verbose_name=_('Type')
)
status = columns.ChoiceFieldColumn(
verbose_name=_('Status'),
@ -34,8 +35,8 @@ class DataSourceTable(NetBoxTable):
class Meta(NetBoxTable.Meta):
model = DataSource
fields = (
'pk', 'id', 'name', 'type', 'status', 'enabled', 'source_url', 'description', 'comments', 'parameters', 'created',
'last_updated', 'file_count',
'pk', 'id', 'name', 'type', 'status', 'enabled', 'source_url', 'description', 'comments', 'parameters',
'created', 'last_updated', 'file_count',
)
default_columns = ('pk', 'name', 'type', 'status', 'enabled', 'description', 'file_count')

View File

@ -2,7 +2,6 @@ from django.urls import reverse
from django.utils import timezone
from utilities.testing import APITestCase, APIViewTestCases
from ..choices import *
from ..models import *
@ -26,26 +25,26 @@ class DataSourceTest(APIViewTestCases.APIViewTestCase):
@classmethod
def setUpTestData(cls):
data_sources = (
DataSource(name='Data Source 1', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source1/'),
DataSource(name='Data Source 2', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source2/'),
DataSource(name='Data Source 3', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source3/'),
DataSource(name='Data Source 1', type='local', source_url='file:///var/tmp/source1/'),
DataSource(name='Data Source 2', type='local', source_url='file:///var/tmp/source2/'),
DataSource(name='Data Source 3', type='local', source_url='file:///var/tmp/source3/'),
)
DataSource.objects.bulk_create(data_sources)
cls.create_data = [
{
'name': 'Data Source 4',
'type': DataSourceTypeChoices.GIT,
'type': 'git',
'source_url': 'https://example.com/git/source4'
},
{
'name': 'Data Source 5',
'type': DataSourceTypeChoices.GIT,
'type': 'git',
'source_url': 'https://example.com/git/source5'
},
{
'name': 'Data Source 6',
'type': DataSourceTypeChoices.GIT,
'type': 'git',
'source_url': 'https://example.com/git/source6'
},
]
@ -63,7 +62,7 @@ class DataFileTest(
def setUpTestData(cls):
datasource = DataSource.objects.create(
name='Data Source 1',
type=DataSourceTypeChoices.LOCAL,
type='local',
source_url='file:///var/tmp/source1/'
)

View File

@ -18,21 +18,21 @@ class DataSourceTestCase(TestCase, ChangeLoggedFilterSetTests):
data_sources = (
DataSource(
name='Data Source 1',
type=DataSourceTypeChoices.LOCAL,
type='local',
source_url='file:///var/tmp/source1/',
status=DataSourceStatusChoices.NEW,
enabled=True
),
DataSource(
name='Data Source 2',
type=DataSourceTypeChoices.LOCAL,
type='local',
source_url='file:///var/tmp/source2/',
status=DataSourceStatusChoices.SYNCING,
enabled=True
),
DataSource(
name='Data Source 3',
type=DataSourceTypeChoices.GIT,
type='git',
source_url='https://example.com/git/source3',
status=DataSourceStatusChoices.COMPLETED,
enabled=False
@ -45,7 +45,7 @@ class DataSourceTestCase(TestCase, ChangeLoggedFilterSetTests):
self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2)
def test_type(self):
params = {'type': [DataSourceTypeChoices.LOCAL]}
params = {'type': ['local']}
self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2)
def test_enabled(self):
@ -66,9 +66,9 @@ class DataFileTestCase(TestCase, ChangeLoggedFilterSetTests):
@classmethod
def setUpTestData(cls):
data_sources = (
DataSource(name='Data Source 1', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source1/'),
DataSource(name='Data Source 2', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source2/'),
DataSource(name='Data Source 3', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source3/'),
DataSource(name='Data Source 1', type='local', source_url='file:///var/tmp/source1/'),
DataSource(name='Data Source 2', type='local', source_url='file:///var/tmp/source2/'),
DataSource(name='Data Source 3', type='local', source_url='file:///var/tmp/source3/'),
)
DataSource.objects.bulk_create(data_sources)

View File

@ -1,7 +1,6 @@
from django.utils import timezone
from utilities.testing import ViewTestCases, create_tags
from ..choices import *
from ..models import *
@ -11,9 +10,9 @@ class DataSourceTestCase(ViewTestCases.PrimaryObjectViewTestCase):
@classmethod
def setUpTestData(cls):
data_sources = (
DataSource(name='Data Source 1', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source1/'),
DataSource(name='Data Source 2', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source2/'),
DataSource(name='Data Source 3', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source3/'),
DataSource(name='Data Source 1', type='local', source_url='file:///var/tmp/source1/'),
DataSource(name='Data Source 2', type='local', source_url='file:///var/tmp/source2/'),
DataSource(name='Data Source 3', type='local', source_url='file:///var/tmp/source3/'),
)
DataSource.objects.bulk_create(data_sources)
@ -21,7 +20,7 @@ class DataSourceTestCase(ViewTestCases.PrimaryObjectViewTestCase):
cls.form_data = {
'name': 'Data Source X',
'type': DataSourceTypeChoices.GIT,
'type': 'git',
'source_url': 'http:///exmaple/com/foo/bar/',
'description': 'Something',
'comments': 'Foo bar baz',
@ -29,10 +28,10 @@ class DataSourceTestCase(ViewTestCases.PrimaryObjectViewTestCase):
}
cls.csv_data = (
f"name,type,source_url,enabled",
f"Data Source 4,{DataSourceTypeChoices.LOCAL},file:///var/tmp/source4/,true",
f"Data Source 5,{DataSourceTypeChoices.LOCAL},file:///var/tmp/source4/,true",
f"Data Source 6,{DataSourceTypeChoices.GIT},http:///exmaple/com/foo/bar/,false",
"name,type,source_url,enabled",
"Data Source 4,local,file:///var/tmp/source4/,true",
"Data Source 5,local,file:///var/tmp/source4/,true",
"Data Source 6,git,http:///exmaple/com/foo/bar/,false",
)
cls.csv_update_data = (
@ -60,7 +59,7 @@ class DataFileTestCase(
def setUpTestData(cls):
datasource = DataSource.objects.create(
name='Data Source 1',
type=DataSourceTypeChoices.LOCAL,
type='local',
source_url='file:///var/tmp/source1/'
)

View File

@ -0,0 +1,53 @@
from contextlib import contextmanager
from urllib.parse import urlparse
__all__ = (
'DataBackend',
)
class DataBackend:
"""
A data backend represents a specific system of record for data, such as a git repository or Amazon S3 bucket.
Attributes:
name: The identifier under which this backend will be registered in NetBox
label: The human-friendly name for this backend
is_local: A boolean indicating whether this backend accesses local data
parameters: A dictionary mapping configuration form field names to their classes
sensitive_parameters: An iterable of field names for which the values should not be displayed to the user
"""
is_local = False
parameters = {}
sensitive_parameters = []
# Prevent Django's template engine from calling the backend
# class when referenced via DataSource.backend_class
do_not_call_in_templates = True
def __init__(self, url, **kwargs):
self.url = url
self.params = kwargs
self.config = self.init_config()
def init_config(self):
"""
A hook to initialize the instance's configuration. The data returned by this method is assigned to the
instance's `config` attribute upon initialization, which can be referenced by the `fetch()` method.
"""
return
@property
def url_scheme(self):
return urlparse(self.url).scheme.lower()
@contextmanager
def fetch(self):
"""
A context manager which performs the following:
1. Handles all setup and synchronization
2. Yields the local path at which data has been replicated
3. Performs any necessary cleanup
"""
raise NotImplemented()

View File

@ -8,6 +8,7 @@ from packaging import version
from netbox.registry import registry
from netbox.search import register_search
from netbox.utils import register_data_backend
from .navigation import *
from .registration import *
from .templates import *
@ -24,6 +25,7 @@ registry['plugins'].update({
DEFAULT_RESOURCE_PATHS = {
'search_indexes': 'search.indexes',
'data_backends': 'data_backends.backends',
'graphql_schema': 'graphql.schema',
'menu': 'navigation.menu',
'menu_items': 'navigation.menu_items',
@ -70,6 +72,7 @@ class PluginConfig(AppConfig):
# Optional plugin resources
search_indexes = None
data_backends = None
graphql_schema = None
menu = None
menu_items = None
@ -98,6 +101,11 @@ class PluginConfig(AppConfig):
for idx in search_indexes:
register_search(idx)
# Register data backends (if defined)
data_backends = self._load_resource('data_backends') or []
for backend in data_backends:
register_data_backend()(backend)
# Register template content (if defined)
if template_extensions := self._load_resource('template_extensions'):
register_template_extensions(template_extensions)

View File

@ -0,0 +1,18 @@
from contextlib import contextmanager
from netbox.data_backends import DataBackend
class DummyBackend(DataBackend):
name = 'dummy'
label = 'Dummy'
is_local = True
@contextmanager
def fetch(self):
yield '/tmp'
backends = (
DummyBackend,
)

View File

@ -6,6 +6,7 @@ from django.test import Client, TestCase, override_settings
from django.urls import reverse
from netbox.tests.dummy_plugin import config as dummy_config
from netbox.tests.dummy_plugin.data_backends import DummyBackend
from netbox.plugins.navigation import PluginMenu
from netbox.plugins.utils import get_plugin_config
from netbox.graphql.schema import Query
@ -111,6 +112,13 @@ class PluginTest(TestCase):
"""
self.assertIn('netbox.tests.dummy_plugin.middleware.DummyMiddleware', settings.MIDDLEWARE)
def test_data_backends(self):
"""
Check registered data backends.
"""
self.assertIn('dummy', registry['data_backends'])
self.assertIs(registry['data_backends']['dummy'], DummyBackend)
def test_queues(self):
"""
Check that plugin queues are registered with the accurate name.

26
netbox/netbox/utils.py Normal file
View File

@ -0,0 +1,26 @@
from netbox.registry import registry
__all__ = (
'get_data_backend_choices',
'register_data_backend',
)
def get_data_backend_choices():
return [
(None, '---------'),
*[
(name, cls.label) for name, cls in registry['data_backends'].items()
]
]
def register_data_backend():
"""
Decorator for registering a DataBackend class.
"""
def _wrapper(cls):
registry['data_backends'][cls.name] = cls
return cls
return _wrapper

View File

@ -58,7 +58,7 @@
<tr>
<th scope="row">{% trans "URL" %}</th>
<td>
{% if not object.is_local %}
{% if not object.type.is_local %}
<a href="{{ object.source_url }}">{{ object.source_url }}</a>
{% else %}
{{ object.source_url }}