From d8784d4155f5544d7046e17f7310a6b011336964 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 2 Feb 2023 10:06:23 -0500 Subject: [PATCH] Closes #11558: Add support for remote data sources (#11646) * WIP * WIP * Add git sync * Fix file hashing * Add last_synced to DataSource * Build out UI & API resources * Add status field to DataSource * Add UI control to sync data source * Add API endpoint to sync data sources * Fix display of DataSource job results * DataSource password should be write-only * General cleanup * Add data file UI view * Punt on HTTP, FTP support for now * Add DataSource URL validation * Add HTTP proxy support to git fetcher * Add management command to sync data sources * DataFile REST API endpoints should be read-only * Refactor fetch methods into backend classes * Replace auth & git branch fields with general-purpose parameters * Fix last_synced time * Render discrete form fields for backend parameters * Enable dynamic edit form for DataSource * Register DataBackend classes in application registry * Add search indexers for DataSource, DataFile * Add single & bulk delete views for DataFile * Add model documentation * Convert DataSource to a primary model * Introduce pre_sync & post_sync signals * Clean up migrations * Rename url to source_url * Clean up filtersets * Add API & filterset tests * Add view tests * Add initSelect() to HTMX refresh handler * Render DataSourceForm fieldsets dynamically * Update compiled static resources --- docs/models/core/datafile.md | 25 ++ docs/models/core/datasource.md | 47 +++ netbox/core/__init__.py | 0 netbox/core/api/__init__.py | 0 netbox/core/api/nested_serializers.py | 25 ++ netbox/core/api/serializers.py | 51 +++ netbox/core/api/urls.py | 13 + netbox/core/api/views.py | 52 +++ netbox/core/apps.py | 8 + netbox/core/choices.py | 34 ++ netbox/core/data_backends.py | 117 +++++++ netbox/core/exceptions.py | 2 + netbox/core/filtersets.py | 64 ++++ netbox/core/forms/__init__.py | 4 + netbox/core/forms/bulk_edit.py | 50 +++ netbox/core/forms/bulk_import.py | 15 + netbox/core/forms/filtersets.py | 49 +++ netbox/core/forms/model_forms.py | 81 +++++ netbox/core/graphql/__init__.py | 0 netbox/core/graphql/schema.py | 12 + netbox/core/graphql/types.py | 21 ++ netbox/core/jobs.py | 29 ++ netbox/core/management/__init__.py | 0 netbox/core/management/commands/__init__.py | 0 .../management/commands/syncdatasource.py | 41 +++ netbox/core/migrations/0001_initial.py | 62 ++++ netbox/core/migrations/__init__.py | 0 netbox/core/models/__init__.py | 1 + netbox/core/models/data.py | 302 ++++++++++++++++++ netbox/core/search.py | 21 ++ netbox/core/signals.py | 10 + netbox/core/tables/__init__.py | 1 + netbox/core/tables/data.py | 52 +++ netbox/core/tests/__init__.py | 0 netbox/core/tests/test_api.py | 93 ++++++ netbox/core/tests/test_filtersets.py | 120 +++++++ netbox/core/tests/test_views.py | 91 ++++++ netbox/core/urls.py | 22 ++ netbox/core/views.py | 118 +++++++ netbox/extras/management/commands/nbshell.py | 2 +- netbox/extras/models/models.py | 10 +- netbox/netbox/api/views.py | 1 + netbox/netbox/graphql/schema.py | 2 + netbox/netbox/navigation/menu.py | 1 + netbox/netbox/registry.py | 3 +- netbox/netbox/settings.py | 1 + netbox/netbox/urls.py | 2 + netbox/project-static/dist/netbox.js | Bin 380899 -> 380935 bytes netbox/project-static/dist/netbox.js.map | Bin 353676 -> 353697 bytes netbox/project-static/src/htmx.ts | 4 +- netbox/templates/core/datafile.html | 81 +++++ netbox/templates/core/datasource.html | 114 +++++++ netbox/utilities/files.py | 9 + 53 files changed, 1857 insertions(+), 6 deletions(-) create mode 100644 docs/models/core/datafile.md create mode 100644 docs/models/core/datasource.md create mode 100644 netbox/core/__init__.py create mode 100644 netbox/core/api/__init__.py create mode 100644 netbox/core/api/nested_serializers.py create mode 100644 netbox/core/api/serializers.py create mode 100644 netbox/core/api/urls.py create mode 100644 netbox/core/api/views.py create mode 100644 netbox/core/apps.py create mode 100644 netbox/core/choices.py create mode 100644 netbox/core/data_backends.py create mode 100644 netbox/core/exceptions.py create mode 100644 netbox/core/filtersets.py create mode 100644 netbox/core/forms/__init__.py create mode 100644 netbox/core/forms/bulk_edit.py create mode 100644 netbox/core/forms/bulk_import.py create mode 100644 netbox/core/forms/filtersets.py create mode 100644 netbox/core/forms/model_forms.py create mode 100644 netbox/core/graphql/__init__.py create mode 100644 netbox/core/graphql/schema.py create mode 100644 netbox/core/graphql/types.py create mode 100644 netbox/core/jobs.py create mode 100644 netbox/core/management/__init__.py create mode 100644 netbox/core/management/commands/__init__.py create mode 100644 netbox/core/management/commands/syncdatasource.py create mode 100644 netbox/core/migrations/0001_initial.py create mode 100644 netbox/core/migrations/__init__.py create mode 100644 netbox/core/models/__init__.py create mode 100644 netbox/core/models/data.py create mode 100644 netbox/core/search.py create mode 100644 netbox/core/signals.py create mode 100644 netbox/core/tables/__init__.py create mode 100644 netbox/core/tables/data.py create mode 100644 netbox/core/tests/__init__.py create mode 100644 netbox/core/tests/test_api.py create mode 100644 netbox/core/tests/test_filtersets.py create mode 100644 netbox/core/tests/test_views.py create mode 100644 netbox/core/urls.py create mode 100644 netbox/core/views.py create mode 100644 netbox/templates/core/datafile.html create mode 100644 netbox/templates/core/datasource.html create mode 100644 netbox/utilities/files.py diff --git a/docs/models/core/datafile.md b/docs/models/core/datafile.md new file mode 100644 index 000000000..3e2aa2f27 --- /dev/null +++ b/docs/models/core/datafile.md @@ -0,0 +1,25 @@ +# Data Files + +A data file object is the representation in NetBox's database of some file belonging to a remote [data source](./datasource.md). Data files are synchronized automatically, and cannot be modified locally (although they can be deleted). + +## Fields + +### Source + +The [data source](./datasource.md) to which this file belongs. + +### Path + +The path to the file, relative to its source's URL. For example, a file at `/opt/config-data/routing/bgp/peer.yaml` with a source URL of `file:///opt/config-data/` would have its path set to `routing/bgp/peer.yaml`. + +### Last Updated + +The date and time at which the file most recently updated from its source. Note that this attribute is updated only when the file's contents have been modified. Re-synchronizing the data source will not update this timestamp if the upstream file's data has not changed. + +### Size + +The file's size, in bytes. + +### Hash + +A [SHA256 hash](https://en.wikipedia.org/wiki/SHA-2) of the file's data. This can be compared to a hash taken from the original file to determine whether any changes have been made. diff --git a/docs/models/core/datasource.md b/docs/models/core/datasource.md new file mode 100644 index 000000000..d16abdd10 --- /dev/null +++ b/docs/models/core/datasource.md @@ -0,0 +1,47 @@ +# Data Sources + +A data source represents some external repository of data which NetBox can consume, such as a git repository. Files within the data source are synchronized to NetBox by saving them in the database as [data file](./datafile.md) objects. + +## Fields + +### Name + +The data source's human-friendly name. + +### Type + +The type of data source. Supported options include: + +* Local directory +* git repository + +### URL + +The URL identifying the remote source. Some examples are included below. + +| Type | Example URL | +|------|-------------| +| Local | file:///var/my/data/source/ | +| git | https://https://github.com/my-organization/my-repo | + +### Status + +The source's current synchronization status. Note that this cannot be set manually: It is updated automatically when the source is synchronized. + +### Enabled + +If false, synchronization will be disabled. + +### Ignore Rules + +A set of rules (one per line) identifying filenames to ignore during synchronization. Some examples are provided below. See Python's [`fnmatch()` documentation](https://docs.python.org/3/library/fnmatch.html) for a complete reference. + +| Rule | Description | +|----------------|------------------------------------------| +| `README` | Ignore any files named `README` | +| `*.txt` | Ignore any files with a `.txt` extension | +| `data???.json` | Ignore e.g. `data123.json` | + +### Last Synced + +The date and time at which the source was most recently synchronized successfully. diff --git a/netbox/core/__init__.py b/netbox/core/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/netbox/core/api/__init__.py b/netbox/core/api/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/netbox/core/api/nested_serializers.py b/netbox/core/api/nested_serializers.py new file mode 100644 index 000000000..0a8351fec --- /dev/null +++ b/netbox/core/api/nested_serializers.py @@ -0,0 +1,25 @@ +from rest_framework import serializers + +from core.models import * +from netbox.api.serializers import WritableNestedSerializer + +__all__ = [ + 'NestedDataFileSerializer', + 'NestedDataSourceSerializer', +] + + +class NestedDataSourceSerializer(WritableNestedSerializer): + url = serializers.HyperlinkedIdentityField(view_name='core-api:datasource-detail') + + class Meta: + model = DataSource + fields = ['id', 'url', 'display', 'name'] + + +class NestedDataFileSerializer(WritableNestedSerializer): + url = serializers.HyperlinkedIdentityField(view_name='core-api:datafile-detail') + + class Meta: + model = DataFile + fields = ['id', 'url', 'display', 'path'] diff --git a/netbox/core/api/serializers.py b/netbox/core/api/serializers.py new file mode 100644 index 000000000..4c29fd69e --- /dev/null +++ b/netbox/core/api/serializers.py @@ -0,0 +1,51 @@ +from rest_framework import serializers + +from core.choices import * +from core.models import * +from netbox.api.fields import ChoiceField +from netbox.api.serializers import NetBoxModelSerializer +from .nested_serializers import * + +__all__ = ( + 'DataSourceSerializer', +) + + +class DataSourceSerializer(NetBoxModelSerializer): + url = serializers.HyperlinkedIdentityField( + view_name='core-api:datasource-detail' + ) + type = ChoiceField( + choices=DataSourceTypeChoices + ) + status = ChoiceField( + choices=DataSourceStatusChoices, + read_only=True + ) + + # Related object counts + file_count = serializers.IntegerField( + read_only=True + ) + + class Meta: + model = DataSource + fields = [ + 'id', 'url', 'display', 'name', 'type', 'source_url', 'enabled', 'status', 'description', 'comments', + 'parameters', 'ignore_rules', 'created', 'last_updated', 'file_count', + ] + + +class DataFileSerializer(NetBoxModelSerializer): + url = serializers.HyperlinkedIdentityField( + view_name='core-api:datafile-detail' + ) + source = NestedDataSourceSerializer( + read_only=True + ) + + class Meta: + model = DataFile + fields = [ + 'id', 'url', 'display', 'source', 'path', 'last_updated', 'size', 'hash', + ] diff --git a/netbox/core/api/urls.py b/netbox/core/api/urls.py new file mode 100644 index 000000000..364e5db55 --- /dev/null +++ b/netbox/core/api/urls.py @@ -0,0 +1,13 @@ +from netbox.api.routers import NetBoxRouter +from . import views + + +router = NetBoxRouter() +router.APIRootView = views.CoreRootView + +# Data sources +router.register('data-sources', views.DataSourceViewSet) +router.register('data-files', views.DataFileViewSet) + +app_name = 'core-api' +urlpatterns = router.urls diff --git a/netbox/core/api/views.py b/netbox/core/api/views.py new file mode 100644 index 000000000..b2d8c0ed4 --- /dev/null +++ b/netbox/core/api/views.py @@ -0,0 +1,52 @@ +from django.shortcuts import get_object_or_404 + +from rest_framework.decorators import action +from rest_framework.exceptions import PermissionDenied +from rest_framework.response import Response +from rest_framework.routers import APIRootView + +from core import filtersets +from core.models import * +from netbox.api.viewsets import NetBoxModelViewSet, NetBoxReadOnlyModelViewSet +from utilities.utils import count_related +from . import serializers + + +class CoreRootView(APIRootView): + """ + Core API root view + """ + def get_view_name(self): + return 'Core' + + +# +# Data sources +# + +class DataSourceViewSet(NetBoxModelViewSet): + queryset = DataSource.objects.annotate( + file_count=count_related(DataFile, 'source') + ) + serializer_class = serializers.DataSourceSerializer + filterset_class = filtersets.DataSourceFilterSet + + @action(detail=True, methods=['post']) + def sync(self, request, pk): + """ + Enqueue a job to synchronize the DataSource. + """ + if not request.user.has_perm('extras.sync_datasource'): + raise PermissionDenied("Syncing data sources requires the core.sync_datasource permission.") + + datasource = get_object_or_404(DataSource, pk=pk) + datasource.enqueue_sync_job(request) + serializer = serializers.DataSourceSerializer(datasource, context={'request': request}) + + return Response(serializer.data) + + +class DataFileViewSet(NetBoxReadOnlyModelViewSet): + queryset = DataFile.objects.defer('data').prefetch_related('source') + serializer_class = serializers.DataFileSerializer + filterset_class = filtersets.DataFileFilterSet diff --git a/netbox/core/apps.py b/netbox/core/apps.py new file mode 100644 index 000000000..c4886eb41 --- /dev/null +++ b/netbox/core/apps.py @@ -0,0 +1,8 @@ +from django.apps import AppConfig + + +class CoreConfig(AppConfig): + name = "core" + + def ready(self): + from . import data_backends, search diff --git a/netbox/core/choices.py b/netbox/core/choices.py new file mode 100644 index 000000000..6927c83fb --- /dev/null +++ b/netbox/core/choices.py @@ -0,0 +1,34 @@ +from django.utils.translation import gettext as _ + +from utilities.choices import ChoiceSet + + +# +# Data sources +# + +class DataSourceTypeChoices(ChoiceSet): + LOCAL = 'local' + GIT = 'git' + + CHOICES = ( + (LOCAL, _('Local'), 'gray'), + (GIT, _('Git'), 'blue'), + ) + + +class DataSourceStatusChoices(ChoiceSet): + + NEW = 'new' + QUEUED = 'queued' + SYNCING = 'syncing' + COMPLETED = 'completed' + FAILED = 'failed' + + CHOICES = ( + (NEW, _('New'), 'blue'), + (QUEUED, _('Queued'), 'orange'), + (SYNCING, _('Syncing'), 'cyan'), + (COMPLETED, _('Completed'), 'green'), + (FAILED, _('Failed'), 'red'), + ) diff --git a/netbox/core/data_backends.py b/netbox/core/data_backends.py new file mode 100644 index 000000000..5d0e80584 --- /dev/null +++ b/netbox/core/data_backends.py @@ -0,0 +1,117 @@ +import logging +import subprocess +import tempfile +from contextlib import contextmanager +from urllib.parse import quote, urlunparse, urlparse + +from django import forms +from django.conf import settings +from django.utils.translation import gettext as _ + +from netbox.registry import registry +from .choices import DataSourceTypeChoices +from .exceptions import SyncError + +__all__ = ( + 'LocalBackend', + 'GitBackend', +) + +logger = logging.getLogger('netbox.data_backends') + + +def register_backend(name): + """ + Decorator for registering a DataBackend class. + """ + def _wrapper(cls): + registry['data_backends'][name] = cls + return cls + + return _wrapper + + +class DataBackend: + parameters = {} + + def __init__(self, url, **kwargs): + self.url = url + self.params = kwargs + + @property + def url_scheme(self): + return urlparse(self.url).scheme.lower() + + @contextmanager + def fetch(self): + raise NotImplemented() + + +@register_backend(DataSourceTypeChoices.LOCAL) +class LocalBackend(DataBackend): + + @contextmanager + def fetch(self): + logger.debug(f"Data source type is local; skipping fetch") + local_path = urlparse(self.url).path # Strip file:// scheme + + yield local_path + + +@register_backend(DataSourceTypeChoices.GIT) +class GitBackend(DataBackend): + parameters = { + 'username': forms.CharField( + required=False, + label=_('Username'), + widget=forms.TextInput(attrs={'class': 'form-control'}) + ), + 'password': forms.CharField( + required=False, + label=_('Password'), + widget=forms.TextInput(attrs={'class': 'form-control'}) + ), + 'branch': forms.CharField( + required=False, + label=_('Branch'), + widget=forms.TextInput(attrs={'class': 'form-control'}) + ) + } + + @contextmanager + def fetch(self): + local_path = tempfile.TemporaryDirectory() + + # Add authentication credentials to URL (if specified) + username = self.params.get('username') + password = self.params.get('password') + if username and password: + url_components = list(urlparse(self.url)) + # Prepend username & password to netloc + url_components[1] = quote(f'{username}@{password}:') + url_components[1] + url = urlunparse(url_components) + else: + url = self.url + + # Compile git arguments + args = ['git', 'clone', '--depth', '1'] + if branch := self.params.get('branch'): + args.extend(['--branch', branch]) + args.extend([url, local_path.name]) + + # Prep environment variables + env_vars = {} + if settings.HTTP_PROXIES and self.url_scheme in ('http', 'https'): + env_vars['http_proxy'] = settings.HTTP_PROXIES.get(self.url_scheme) + + logger.debug(f"Cloning git repo: {' '.join(args)}") + try: + subprocess.run(args, check=True, capture_output=True, env=env_vars) + except subprocess.CalledProcessError as e: + raise SyncError( + f"Fetching remote data failed: {e.stderr}" + ) + + yield local_path.name + + local_path.cleanup() diff --git a/netbox/core/exceptions.py b/netbox/core/exceptions.py new file mode 100644 index 000000000..8412b0378 --- /dev/null +++ b/netbox/core/exceptions.py @@ -0,0 +1,2 @@ +class SyncError(Exception): + pass diff --git a/netbox/core/filtersets.py b/netbox/core/filtersets.py new file mode 100644 index 000000000..3bff34158 --- /dev/null +++ b/netbox/core/filtersets.py @@ -0,0 +1,64 @@ +from django.db.models import Q +from django.utils.translation import gettext as _ + +import django_filters + +from netbox.filtersets import ChangeLoggedModelFilterSet, NetBoxModelFilterSet +from .choices import * +from .models import * + +__all__ = ( + 'DataFileFilterSet', + 'DataSourceFilterSet', +) + + +class DataSourceFilterSet(NetBoxModelFilterSet): + type = django_filters.MultipleChoiceFilter( + choices=DataSourceTypeChoices, + null_value=None + ) + status = django_filters.MultipleChoiceFilter( + choices=DataSourceStatusChoices, + null_value=None + ) + + class Meta: + model = DataSource + fields = ('id', 'name', 'enabled') + + def search(self, queryset, name, value): + if not value.strip(): + return queryset + return queryset.filter( + Q(name__icontains=value) | + Q(description__icontains=value) | + Q(comments__icontains=value) + ) + + +class DataFileFilterSet(ChangeLoggedModelFilterSet): + q = django_filters.CharFilter( + method='search' + ) + source_id = django_filters.ModelMultipleChoiceFilter( + queryset=DataSource.objects.all(), + label=_('Data source (ID)'), + ) + source = django_filters.ModelMultipleChoiceFilter( + field_name='source__name', + queryset=DataSource.objects.all(), + to_field_name='name', + label=_('Data source (name)'), + ) + + class Meta: + model = DataFile + fields = ('id', 'path', 'last_updated', 'size', 'hash') + + def search(self, queryset, name, value): + if not value.strip(): + return queryset + return queryset.filter( + Q(path__icontains=value) + ) diff --git a/netbox/core/forms/__init__.py b/netbox/core/forms/__init__.py new file mode 100644 index 000000000..1499f98b2 --- /dev/null +++ b/netbox/core/forms/__init__.py @@ -0,0 +1,4 @@ +from .bulk_edit import * +from .bulk_import import * +from .filtersets import * +from .model_forms import * diff --git a/netbox/core/forms/bulk_edit.py b/netbox/core/forms/bulk_edit.py new file mode 100644 index 000000000..c5713b626 --- /dev/null +++ b/netbox/core/forms/bulk_edit.py @@ -0,0 +1,50 @@ +from django import forms +from django.utils.translation import gettext as _ + +from core.choices import DataSourceTypeChoices +from core.models import * +from netbox.forms import NetBoxModelBulkEditForm +from utilities.forms import ( + add_blank_choice, BulkEditNullBooleanSelect, CommentField, SmallTextarea, StaticSelect, +) + +__all__ = ( + 'DataSourceBulkEditForm', +) + + +class DataSourceBulkEditForm(NetBoxModelBulkEditForm): + type = forms.ChoiceField( + choices=add_blank_choice(DataSourceTypeChoices), + required=False, + initial='', + widget=StaticSelect() + ) + enabled = forms.NullBooleanField( + required=False, + widget=BulkEditNullBooleanSelect(), + label=_('Enforce unique space') + ) + description = forms.CharField( + max_length=200, + required=False + ) + comments = CommentField( + widget=SmallTextarea, + label=_('Comments') + ) + parameters = forms.JSONField( + required=False + ) + ignore_rules = forms.CharField( + required=False, + widget=forms.Textarea() + ) + + model = DataSource + fieldsets = ( + (None, ('type', 'enabled', 'description', 'comments', 'parameters', 'ignore_rules')), + ) + nullable_fields = ( + 'description', 'description', 'parameters', 'comments', 'parameters', 'ignore_rules', + ) diff --git a/netbox/core/forms/bulk_import.py b/netbox/core/forms/bulk_import.py new file mode 100644 index 000000000..78a859dcb --- /dev/null +++ b/netbox/core/forms/bulk_import.py @@ -0,0 +1,15 @@ +from core.models import * +from netbox.forms import NetBoxModelImportForm + +__all__ = ( + 'DataSourceImportForm', +) + + +class DataSourceImportForm(NetBoxModelImportForm): + + class Meta: + model = DataSource + fields = ( + 'name', 'type', 'source_url', 'enabled', 'description', 'comments', 'parameters', 'ignore_rules', + ) diff --git a/netbox/core/forms/filtersets.py b/netbox/core/forms/filtersets.py new file mode 100644 index 000000000..433f07067 --- /dev/null +++ b/netbox/core/forms/filtersets.py @@ -0,0 +1,49 @@ +from django import forms +from django.utils.translation import gettext as _ + +from core.choices import * +from core.models import * +from netbox.forms import NetBoxModelFilterSetForm +from utilities.forms import ( + BOOLEAN_WITH_BLANK_CHOICES, DynamicModelMultipleChoiceField, MultipleChoiceField, StaticSelect, +) + +__all__ = ( + 'DataFileFilterForm', + 'DataSourceFilterForm', +) + + +class DataSourceFilterForm(NetBoxModelFilterSetForm): + model = DataSource + fieldsets = ( + (None, ('q', 'filter_id')), + ('Data Source', ('type', 'status')), + ) + type = MultipleChoiceField( + choices=DataSourceTypeChoices, + required=False + ) + status = MultipleChoiceField( + choices=DataSourceStatusChoices, + required=False + ) + enabled = forms.NullBooleanField( + required=False, + widget=StaticSelect( + choices=BOOLEAN_WITH_BLANK_CHOICES + ) + ) + + +class DataFileFilterForm(NetBoxModelFilterSetForm): + model = DataFile + fieldsets = ( + (None, ('q', 'filter_id')), + ('File', ('source_id',)), + ) + source_id = DynamicModelMultipleChoiceField( + queryset=DataSource.objects.all(), + required=False, + label=_('Data source') + ) diff --git a/netbox/core/forms/model_forms.py b/netbox/core/forms/model_forms.py new file mode 100644 index 000000000..786e71c3a --- /dev/null +++ b/netbox/core/forms/model_forms.py @@ -0,0 +1,81 @@ +import copy + +from django import forms + +from core.models import * +from netbox.forms import NetBoxModelForm, StaticSelect +from netbox.registry import registry +from utilities.forms import CommentField + +__all__ = ( + 'DataSourceForm', +) + + +class DataSourceForm(NetBoxModelForm): + comments = CommentField() + + class Meta: + model = DataSource + fields = [ + 'name', 'type', 'source_url', 'enabled', 'description', 'comments', 'ignore_rules', 'tags', + ] + widgets = { + 'type': StaticSelect( + attrs={ + 'hx-get': '.', + 'hx-include': '#form_fields input', + 'hx-target': '#form_fields', + } + ), + 'ignore_rules': forms.Textarea( + attrs={ + 'rows': 5, + 'class': 'font-monospace', + 'placeholder': '.cache\n*.txt' + } + ), + } + + @property + def fieldsets(self): + fieldsets = [ + ('Source', ('name', 'type', 'source_url', 'enabled', 'description', 'tags', 'ignore_rules')), + ] + if self.backend_fields: + fieldsets.append( + ('Backend', self.backend_fields) + ) + + return fieldsets + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + backend_classes = registry['data_backends'] + + if self.is_bound and self.data.get('type') in backend_classes: + type_ = self.data['type'] + elif self.initial and self.initial.get('type') in backend_classes: + type_ = self.initial['type'] + else: + type_ = self.fields['type'].initial + backend = backend_classes.get(type_) + + self.backend_fields = [] + for name, form_field in backend.parameters.items(): + field_name = f'backend_{name}' + self.backend_fields.append(field_name) + self.fields[field_name] = copy.copy(form_field) + if self.instance and self.instance.parameters: + self.fields[field_name].initial = self.instance.parameters.get(name) + + def save(self, *args, **kwargs): + + parameters = {} + for name in self.fields: + if name.startswith('backend_'): + parameters[name[8:]] = self.cleaned_data[name] + self.instance.parameters = parameters + + return super().save(*args, **kwargs) diff --git a/netbox/core/graphql/__init__.py b/netbox/core/graphql/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/netbox/core/graphql/schema.py b/netbox/core/graphql/schema.py new file mode 100644 index 000000000..201965430 --- /dev/null +++ b/netbox/core/graphql/schema.py @@ -0,0 +1,12 @@ +import graphene + +from netbox.graphql.fields import ObjectField, ObjectListField +from .types import * + + +class CoreQuery(graphene.ObjectType): + data_file = ObjectField(DataFileType) + data_file_list = ObjectListField(DataFileType) + + data_source = ObjectField(DataSourceType) + data_source_list = ObjectListField(DataSourceType) diff --git a/netbox/core/graphql/types.py b/netbox/core/graphql/types.py new file mode 100644 index 000000000..402e36345 --- /dev/null +++ b/netbox/core/graphql/types.py @@ -0,0 +1,21 @@ +from core import filtersets, models +from netbox.graphql.types import BaseObjectType, NetBoxObjectType + +__all__ = ( + 'DataFileType', + 'DataSourceType', +) + + +class DataFileType(BaseObjectType): + class Meta: + model = models.DataFile + exclude = ('data',) + filterset_class = filtersets.DataFileFilterSet + + +class DataSourceType(NetBoxObjectType): + class Meta: + model = models.DataSource + fields = '__all__' + filterset_class = filtersets.DataSourceFilterSet diff --git a/netbox/core/jobs.py b/netbox/core/jobs.py new file mode 100644 index 000000000..ee285fa7c --- /dev/null +++ b/netbox/core/jobs.py @@ -0,0 +1,29 @@ +import logging + +from extras.choices import JobResultStatusChoices +from netbox.search.backends import search_backend +from .choices import * +from .exceptions import SyncError +from .models import DataSource + +logger = logging.getLogger(__name__) + + +def sync_datasource(job_result, *args, **kwargs): + """ + Call sync() on a DataSource. + """ + datasource = DataSource.objects.get(name=job_result.name) + + try: + job_result.start() + datasource.sync() + + # Update the search cache for DataFiles belonging to this source + search_backend.cache(datasource.datafiles.iterator()) + + except SyncError as e: + job_result.set_status(JobResultStatusChoices.STATUS_ERRORED) + job_result.save() + DataSource.objects.filter(pk=datasource.pk).update(status=DataSourceStatusChoices.FAILED) + logging.error(e) diff --git a/netbox/core/management/__init__.py b/netbox/core/management/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/netbox/core/management/commands/__init__.py b/netbox/core/management/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/netbox/core/management/commands/syncdatasource.py b/netbox/core/management/commands/syncdatasource.py new file mode 100644 index 000000000..3d73f70ab --- /dev/null +++ b/netbox/core/management/commands/syncdatasource.py @@ -0,0 +1,41 @@ +from django.core.management.base import BaseCommand, CommandError + +from core.models import DataSource + + +class Command(BaseCommand): + help = "Synchronize a data source from its remote upstream" + + def add_arguments(self, parser): + parser.add_argument('name', nargs='*', help="Data source(s) to synchronize") + parser.add_argument( + "--all", action='store_true', dest='sync_all', + help="Synchronize all data sources" + ) + + def handle(self, *args, **options): + + # Find DataSources to sync + if options['sync_all']: + datasources = DataSource.objects.all() + elif options['name']: + datasources = DataSource.objects.filter(name__in=options['name']) + # Check for invalid names + found_names = {ds['name'] for ds in datasources.values('name')} + if invalid_names := set(options['name']) - found_names: + raise CommandError(f"Invalid data source names: {', '.join(invalid_names)}") + else: + raise CommandError(f"Must specify at least one data source, or set --all.") + + if len(options['name']) > 1: + self.stdout.write(f"Syncing {len(datasources)} data sources.") + + for i, datasource in enumerate(datasources, start=1): + self.stdout.write(f"[{i}] Syncing {datasource}... ", ending='') + self.stdout.flush() + datasource.sync() + self.stdout.write(datasource.get_status_display()) + self.stdout.flush() + + if len(options['name']) > 1: + self.stdout.write(f"Finished.") diff --git a/netbox/core/migrations/0001_initial.py b/netbox/core/migrations/0001_initial.py new file mode 100644 index 000000000..803ac3b13 --- /dev/null +++ b/netbox/core/migrations/0001_initial.py @@ -0,0 +1,62 @@ +# Generated by Django 4.1.5 on 2023-02-02 02:37 + +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import taggit.managers +import utilities.json + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('extras', '0084_staging'), + ] + + operations = [ + migrations.CreateModel( + name='DataSource', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False)), + ('created', models.DateTimeField(auto_now_add=True, null=True)), + ('last_updated', models.DateTimeField(auto_now=True, null=True)), + ('custom_field_data', models.JSONField(blank=True, default=dict, encoder=utilities.json.CustomFieldJSONEncoder)), + ('description', models.CharField(blank=True, max_length=200)), + ('comments', models.TextField(blank=True)), + ('name', models.CharField(max_length=100, unique=True)), + ('type', models.CharField(default='local', max_length=50)), + ('source_url', models.CharField(max_length=200)), + ('status', models.CharField(default='new', editable=False, max_length=50)), + ('enabled', models.BooleanField(default=True)), + ('ignore_rules', models.TextField(blank=True)), + ('parameters', models.JSONField(blank=True, null=True)), + ('last_synced', models.DateTimeField(blank=True, editable=False, null=True)), + ('tags', taggit.managers.TaggableManager(through='extras.TaggedItem', to='extras.Tag')), + ], + options={ + 'ordering': ('name',), + }, + ), + migrations.CreateModel( + name='DataFile', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False)), + ('created', models.DateTimeField(auto_now_add=True, null=True)), + ('path', models.CharField(editable=False, max_length=1000)), + ('last_updated', models.DateTimeField(editable=False)), + ('size', models.PositiveIntegerField(editable=False)), + ('hash', models.CharField(editable=False, max_length=64, validators=[django.core.validators.RegexValidator(message='Length must be 64 hexadecimal characters.', regex='^[0-9a-f]{64}$')])), + ('data', models.BinaryField()), + ('source', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='datafiles', to='core.datasource')), + ], + options={ + 'ordering': ('source', 'path'), + }, + ), + migrations.AddConstraint( + model_name='datafile', + constraint=models.UniqueConstraint(fields=('source', 'path'), name='core_datafile_unique_source_path'), + ), + ] diff --git a/netbox/core/migrations/__init__.py b/netbox/core/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/netbox/core/models/__init__.py b/netbox/core/models/__init__.py new file mode 100644 index 000000000..df22d8bbb --- /dev/null +++ b/netbox/core/models/__init__.py @@ -0,0 +1 @@ +from .data import * diff --git a/netbox/core/models/data.py b/netbox/core/models/data.py new file mode 100644 index 000000000..5ad048b0f --- /dev/null +++ b/netbox/core/models/data.py @@ -0,0 +1,302 @@ +import logging +import os +from fnmatch import fnmatchcase +from urllib.parse import urlparse + +from django.conf import settings +from django.contrib.contenttypes.models import ContentType +from django.core.exceptions import ValidationError +from django.core.validators import RegexValidator +from django.db import models +from django.urls import reverse +from django.utils import timezone +from django.utils.module_loading import import_string +from django.utils.translation import gettext as _ + +from extras.models import JobResult +from netbox.models import PrimaryModel +from netbox.models.features import ChangeLoggingMixin +from netbox.registry import registry +from utilities.files import sha256_hash +from utilities.querysets import RestrictedQuerySet +from ..choices import * +from ..exceptions import SyncError +from ..signals import post_sync, pre_sync + +__all__ = ( + 'DataFile', + 'DataSource', +) + +logger = logging.getLogger('netbox.core.data') + + +class DataSource(PrimaryModel): + """ + A remote source, such as a git repository, from which DataFiles are synchronized. + """ + name = models.CharField( + max_length=100, + unique=True + ) + type = models.CharField( + max_length=50, + choices=DataSourceTypeChoices, + default=DataSourceTypeChoices.LOCAL + ) + source_url = models.CharField( + max_length=200, + verbose_name=_('URL') + ) + status = models.CharField( + max_length=50, + choices=DataSourceStatusChoices, + default=DataSourceStatusChoices.NEW, + editable=False + ) + enabled = models.BooleanField( + default=True + ) + ignore_rules = models.TextField( + blank=True, + help_text=_("Patterns (one per line) matching files to ignore when syncing") + ) + parameters = models.JSONField( + blank=True, + null=True + ) + last_synced = models.DateTimeField( + blank=True, + null=True, + editable=False + ) + + class Meta: + ordering = ('name',) + + def __str__(self): + return f'{self.name}' + + def get_absolute_url(self): + return reverse('core:datasource', args=[self.pk]) + + @property + def docs_url(self): + return f'{settings.STATIC_URL}docs/models/{self._meta.app_label}/{self._meta.model_name}/' + + def get_type_color(self): + return DataSourceTypeChoices.colors.get(self.type) + + def get_status_color(self): + return DataSourceStatusChoices.colors.get(self.status) + + @property + def url_scheme(self): + return urlparse(self.source_url).scheme.lower() + + @property + def ready_for_sync(self): + return self.enabled and self.status not in ( + DataSourceStatusChoices.QUEUED, + DataSourceStatusChoices.SYNCING + ) + + def clean(self): + + # Ensure URL scheme matches selected type + if self.type == DataSourceTypeChoices.LOCAL and self.url_scheme not in ('file', ''): + raise ValidationError({ + 'url': f"URLs for local sources must start with file:// (or omit the scheme)" + }) + + def enqueue_sync_job(self, request): + """ + Enqueue a background job to synchronize the DataSource by calling sync(). + """ + # Set the status to "syncing" + self.status = DataSourceStatusChoices.QUEUED + + # Enqueue a sync job + job_result = JobResult.enqueue_job( + import_string('core.jobs.sync_datasource'), + name=self.name, + obj_type=ContentType.objects.get_for_model(DataSource), + user=request.user, + ) + + return job_result + + def get_backend(self): + backend_cls = registry['data_backends'].get(self.type) + backend_params = self.parameters or {} + + return backend_cls(self.source_url, **backend_params) + + def sync(self): + """ + Create/update/delete child DataFiles as necessary to synchronize with the remote source. + """ + if not self.ready_for_sync: + raise SyncError(f"Cannot initiate sync; data source not ready/enabled") + + # Emit the pre_sync signal + pre_sync.send(sender=self.__class__, instance=self) + + self.status = DataSourceStatusChoices.SYNCING + DataSource.objects.filter(pk=self.pk).update(status=self.status) + + # Replicate source data locally + backend = self.get_backend() + with backend.fetch() as local_path: + + logger.debug(f'Syncing files from source root {local_path}') + data_files = self.datafiles.all() + known_paths = {df.path for df in data_files} + logger.debug(f'Starting with {len(known_paths)} known files') + + # Check for any updated/deleted files + updated_files = [] + deleted_file_ids = [] + for datafile in data_files: + + try: + if datafile.refresh_from_disk(source_root=local_path): + updated_files.append(datafile) + except FileNotFoundError: + # File no longer exists + deleted_file_ids.append(datafile.pk) + continue + + # Bulk update modified files + updated_count = DataFile.objects.bulk_update(updated_files, ['hash']) + logger.debug(f"Updated {updated_count} files") + + # Bulk delete deleted files + deleted_count, _ = DataFile.objects.filter(pk__in=deleted_file_ids).delete() + logger.debug(f"Deleted {updated_count} files") + + # Walk the local replication to find new files + new_paths = self._walk(local_path) - known_paths + + # Bulk create new files + new_datafiles = [] + for path in new_paths: + datafile = DataFile(source=self, path=path) + datafile.refresh_from_disk(source_root=local_path) + datafile.full_clean() + new_datafiles.append(datafile) + created_count = len(DataFile.objects.bulk_create(new_datafiles, batch_size=100)) + logger.debug(f"Created {created_count} data files") + + # Update status & last_synced time + self.status = DataSourceStatusChoices.COMPLETED + self.last_synced = timezone.now() + DataSource.objects.filter(pk=self.pk).update(status=self.status, last_synced=self.last_synced) + + # Emit the post_sync signal + post_sync.send(sender=self.__class__, instance=self) + + def _walk(self, root): + """ + Return a set of all non-excluded files within the root path. + """ + logger.debug(f"Walking {root}...") + paths = set() + + for path, dir_names, file_names in os.walk(root): + path = path.split(root)[1].lstrip('/') # Strip root path + if path.startswith('.'): + continue + for file_name in file_names: + if not self._ignore(file_name): + paths.add(os.path.join(path, file_name)) + + logger.debug(f"Found {len(paths)} files") + return paths + + def _ignore(self, filename): + """ + Returns a boolean indicating whether the file should be ignored per the DataSource's configured + ignore rules. + """ + if filename.startswith('.'): + return True + for rule in self.ignore_rules.splitlines(): + if fnmatchcase(filename, rule): + return True + return False + + +class DataFile(ChangeLoggingMixin, models.Model): + """ + The database representation of a remote file fetched from a remote DataSource. DataFile instances should be created, + updated, or deleted only by calling DataSource.sync(). + """ + source = models.ForeignKey( + to='core.DataSource', + on_delete=models.CASCADE, + related_name='datafiles', + editable=False + ) + path = models.CharField( + max_length=1000, + editable=False, + help_text=_("File path relative to the data source's root") + ) + last_updated = models.DateTimeField( + editable=False + ) + size = models.PositiveIntegerField( + editable=False + ) + hash = models.CharField( + max_length=64, + editable=False, + validators=[ + RegexValidator(regex='^[0-9a-f]{64}$', message=_("Length must be 64 hexadecimal characters.")) + ], + help_text=_("SHA256 hash of the file data") + ) + data = models.BinaryField() + + objects = RestrictedQuerySet.as_manager() + + class Meta: + ordering = ('source', 'path') + constraints = ( + models.UniqueConstraint( + fields=('source', 'path'), + name='%(app_label)s_%(class)s_unique_source_path' + ), + ) + + def __str__(self): + return self.path + + def get_absolute_url(self): + return reverse('core:datafile', args=[self.pk]) + + @property + def data_as_string(self): + try: + return self.data.tobytes().decode('utf-8') + except UnicodeDecodeError: + return None + + def refresh_from_disk(self, source_root): + """ + Update instance attributes from the file on disk. Returns True if any attribute + has changed. + """ + file_path = os.path.join(source_root, self.path) + file_hash = sha256_hash(file_path).hexdigest() + + # Update instance file attributes & data + if is_modified := file_hash != self.hash: + self.last_updated = timezone.now() + self.size = os.path.getsize(file_path) + self.hash = file_hash + with open(file_path, 'rb') as f: + self.data = f.read() + + return is_modified diff --git a/netbox/core/search.py b/netbox/core/search.py new file mode 100644 index 000000000..e6d3005e6 --- /dev/null +++ b/netbox/core/search.py @@ -0,0 +1,21 @@ +from netbox.search import SearchIndex, register_search +from . import models + + +@register_search +class DataSourceIndex(SearchIndex): + model = models.DataSource + fields = ( + ('name', 100), + ('source_url', 300), + ('description', 500), + ('comments', 5000), + ) + + +@register_search +class DataFileIndex(SearchIndex): + model = models.DataFile + fields = ( + ('path', 200), + ) diff --git a/netbox/core/signals.py b/netbox/core/signals.py new file mode 100644 index 000000000..65ca293f5 --- /dev/null +++ b/netbox/core/signals.py @@ -0,0 +1,10 @@ +import django.dispatch + +__all__ = ( + 'post_sync', + 'pre_sync', +) + +# DataSource signals +pre_sync = django.dispatch.Signal() +post_sync = django.dispatch.Signal() diff --git a/netbox/core/tables/__init__.py b/netbox/core/tables/__init__.py new file mode 100644 index 000000000..df22d8bbb --- /dev/null +++ b/netbox/core/tables/__init__.py @@ -0,0 +1 @@ +from .data import * diff --git a/netbox/core/tables/data.py b/netbox/core/tables/data.py new file mode 100644 index 000000000..8409e3b82 --- /dev/null +++ b/netbox/core/tables/data.py @@ -0,0 +1,52 @@ +import django_tables2 as tables + +from core.models import * +from netbox.tables import NetBoxTable, columns + +__all__ = ( + 'DataFileTable', + 'DataSourceTable', +) + + +class DataSourceTable(NetBoxTable): + name = tables.Column( + linkify=True + ) + type = columns.ChoiceFieldColumn() + status = columns.ChoiceFieldColumn() + enabled = columns.BooleanColumn() + tags = columns.TagColumn( + url_name='core:datasource_list' + ) + file_count = tables.Column( + verbose_name='Files' + ) + + class Meta(NetBoxTable.Meta): + model = DataSource + fields = ( + 'pk', 'id', 'name', 'type', 'status', 'enabled', 'source_url', 'description', 'comments', 'parameters', 'created', + 'last_updated', 'file_count', + ) + default_columns = ('pk', 'name', 'type', 'status', 'enabled', 'description', 'file_count') + + +class DataFileTable(NetBoxTable): + source = tables.Column( + linkify=True + ) + path = tables.Column( + linkify=True + ) + last_updated = columns.DateTimeColumn() + actions = columns.ActionsColumn( + actions=('delete',) + ) + + class Meta(NetBoxTable.Meta): + model = DataFile + fields = ( + 'pk', 'id', 'source', 'path', 'last_updated', 'size', 'hash', + ) + default_columns = ('pk', 'source', 'path', 'size', 'last_updated') diff --git a/netbox/core/tests/__init__.py b/netbox/core/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/netbox/core/tests/test_api.py b/netbox/core/tests/test_api.py new file mode 100644 index 000000000..dc6d6a5ce --- /dev/null +++ b/netbox/core/tests/test_api.py @@ -0,0 +1,93 @@ +from django.urls import reverse +from django.utils import timezone + +from utilities.testing import APITestCase, APIViewTestCases +from ..choices import * +from ..models import * + + +class AppTest(APITestCase): + + def test_root(self): + url = reverse('core-api:api-root') + response = self.client.get('{}?format=api'.format(url), **self.header) + + self.assertEqual(response.status_code, 200) + + +class DataSourceTest(APIViewTestCases.APIViewTestCase): + model = DataSource + brief_fields = ['display', 'id', 'name', 'url'] + bulk_update_data = { + 'enabled': False, + 'description': 'foo bar baz', + } + + @classmethod + def setUpTestData(cls): + data_sources = ( + DataSource(name='Data Source 1', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source1/'), + DataSource(name='Data Source 2', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source2/'), + DataSource(name='Data Source 3', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source3/'), + ) + DataSource.objects.bulk_create(data_sources) + + cls.create_data = [ + { + 'name': 'Data Source 4', + 'type': DataSourceTypeChoices.GIT, + 'source_url': 'https://example.com/git/source4' + }, + { + 'name': 'Data Source 5', + 'type': DataSourceTypeChoices.GIT, + 'source_url': 'https://example.com/git/source5' + }, + { + 'name': 'Data Source 6', + 'type': DataSourceTypeChoices.GIT, + 'source_url': 'https://example.com/git/source6' + }, + ] + + +class DataFileTest( + APIViewTestCases.GetObjectViewTestCase, + APIViewTestCases.ListObjectsViewTestCase, + APIViewTestCases.GraphQLTestCase +): + model = DataFile + brief_fields = ['display', 'id', 'path', 'url'] + + @classmethod + def setUpTestData(cls): + datasource = DataSource.objects.create( + name='Data Source 1', + type=DataSourceTypeChoices.LOCAL, + source_url='file:///var/tmp/source1/' + ) + + data_files = ( + DataFile( + source=datasource, + path='dir1/file1.txt', + last_updated=timezone.now(), + size=1000, + hash='442da078f0111cbdf42f21903724f6597c692535f55bdfbbea758a1ae99ad9e1' + ), + DataFile( + source=datasource, + path='dir1/file2.txt', + last_updated=timezone.now(), + size=2000, + hash='a78168c7c97115bafd96450ed03ea43acec495094c5caa28f0d02e20e3a76cc2' + ), + DataFile( + source=datasource, + path='dir1/file3.txt', + last_updated=timezone.now(), + size=3000, + hash='12b8827a14c4d5a2f30b6c6e2b7983063988612391c6cbe8ee7493b59054827a' + ), + ) + DataFile.objects.bulk_create(data_files) diff --git a/netbox/core/tests/test_filtersets.py b/netbox/core/tests/test_filtersets.py new file mode 100644 index 000000000..e1e916f70 --- /dev/null +++ b/netbox/core/tests/test_filtersets.py @@ -0,0 +1,120 @@ +from datetime import datetime + +from django.test import TestCase +from django.utils import timezone + +from utilities.testing import ChangeLoggedFilterSetTests +from ..choices import * +from ..filtersets import * +from ..models import * + + +class DataSourceTestCase(TestCase, ChangeLoggedFilterSetTests): + queryset = DataSource.objects.all() + filterset = DataSourceFilterSet + + @classmethod + def setUpTestData(cls): + data_sources = ( + DataSource( + name='Data Source 1', + type=DataSourceTypeChoices.LOCAL, + source_url='file:///var/tmp/source1/', + status=DataSourceStatusChoices.NEW, + enabled=True + ), + DataSource( + name='Data Source 2', + type=DataSourceTypeChoices.LOCAL, + source_url='file:///var/tmp/source2/', + status=DataSourceStatusChoices.SYNCING, + enabled=True + ), + DataSource( + name='Data Source 3', + type=DataSourceTypeChoices.GIT, + source_url='https://example.com/git/source3', + status=DataSourceStatusChoices.COMPLETED, + enabled=False + ), + ) + DataSource.objects.bulk_create(data_sources) + + def test_name(self): + params = {'name': ['Data Source 1', 'Data Source 2']} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + + def test_type(self): + params = {'type': [DataSourceTypeChoices.LOCAL]} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + + def test_enabled(self): + params = {'enabled': 'true'} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + params = {'enabled': 'false'} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) + + def test_status(self): + params = {'status': [DataSourceStatusChoices.NEW, DataSourceStatusChoices.SYNCING]} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + + +class DataFileTestCase(TestCase, ChangeLoggedFilterSetTests): + queryset = DataFile.objects.all() + filterset = DataFileFilterSet + + @classmethod + def setUpTestData(cls): + data_sources = ( + DataSource(name='Data Source 1', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source1/'), + DataSource(name='Data Source 2', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source2/'), + DataSource(name='Data Source 3', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source3/'), + ) + DataSource.objects.bulk_create(data_sources) + + data_files = ( + DataFile( + source=data_sources[0], + path='dir1/file1.txt', + last_updated=datetime(2023, 1, 1, 0, 0, 0, tzinfo=timezone.utc), + size=1000, + hash='442da078f0111cbdf42f21903724f6597c692535f55bdfbbea758a1ae99ad9e1' + ), + DataFile( + source=data_sources[1], + path='dir1/file2.txt', + last_updated=datetime(2023, 1, 2, 0, 0, 0, tzinfo=timezone.utc), + size=2000, + hash='a78168c7c97115bafd96450ed03ea43acec495094c5caa28f0d02e20e3a76cc2' + ), + DataFile( + source=data_sources[2], + path='dir1/file3.txt', + last_updated=datetime(2023, 1, 3, 0, 0, 0, tzinfo=timezone.utc), + size=3000, + hash='12b8827a14c4d5a2f30b6c6e2b7983063988612391c6cbe8ee7493b59054827a' + ), + ) + DataFile.objects.bulk_create(data_files) + + def test_source(self): + sources = DataSource.objects.all() + params = {'source_id': [sources[0].pk, sources[1].pk]} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + params = {'source': [sources[0].name, sources[1].name]} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + + def test_path(self): + params = {'path': ['dir1/file1.txt', 'dir1/file2.txt']} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + + def test_size(self): + params = {'size': [1000, 2000]} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + + def test_hash(self): + params = {'hash': [ + '442da078f0111cbdf42f21903724f6597c692535f55bdfbbea758a1ae99ad9e1', + 'a78168c7c97115bafd96450ed03ea43acec495094c5caa28f0d02e20e3a76cc2', + ]} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) diff --git a/netbox/core/tests/test_views.py b/netbox/core/tests/test_views.py new file mode 100644 index 000000000..fbee031ed --- /dev/null +++ b/netbox/core/tests/test_views.py @@ -0,0 +1,91 @@ +from django.utils import timezone + +from utilities.testing import ViewTestCases, create_tags +from ..choices import * +from ..models import * + + +class DataSourceTestCase(ViewTestCases.PrimaryObjectViewTestCase): + model = DataSource + + @classmethod + def setUpTestData(cls): + data_sources = ( + DataSource(name='Data Source 1', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source1/'), + DataSource(name='Data Source 2', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source2/'), + DataSource(name='Data Source 3', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source3/'), + ) + DataSource.objects.bulk_create(data_sources) + + tags = create_tags('Alpha', 'Bravo', 'Charlie') + + cls.form_data = { + 'name': 'Data Source X', + 'type': DataSourceTypeChoices.GIT, + 'source_url': 'http:///exmaple/com/foo/bar/', + 'description': 'Something', + 'comments': 'Foo bar baz', + 'tags': [t.pk for t in tags], + } + + cls.csv_data = ( + f"name,type,source_url,enabled", + f"Data Source 4,{DataSourceTypeChoices.LOCAL},file:///var/tmp/source4/,true", + f"Data Source 5,{DataSourceTypeChoices.LOCAL},file:///var/tmp/source4/,true", + f"Data Source 6,{DataSourceTypeChoices.GIT},http:///exmaple/com/foo/bar/,false", + ) + + cls.csv_update_data = ( + "id,name,description", + f"{data_sources[0].pk},Data Source 7,New description7", + f"{data_sources[1].pk},Data Source 8,New description8", + f"{data_sources[2].pk},Data Source 9,New description9", + ) + + cls.bulk_edit_data = { + 'enabled': False, + 'description': 'New description', + } + + +class DataFileTestCase( + ViewTestCases.GetObjectViewTestCase, + ViewTestCases.GetObjectChangelogViewTestCase, + ViewTestCases.DeleteObjectViewTestCase, + ViewTestCases.ListObjectsViewTestCase, + ViewTestCases.BulkDeleteObjectsViewTestCase, +): + model = DataFile + + @classmethod + def setUpTestData(cls): + datasource = DataSource.objects.create( + name='Data Source 1', + type=DataSourceTypeChoices.LOCAL, + source_url='file:///var/tmp/source1/' + ) + + data_files = ( + DataFile( + source=datasource, + path='dir1/file1.txt', + last_updated=timezone.now(), + size=1000, + hash='442da078f0111cbdf42f21903724f6597c692535f55bdfbbea758a1ae99ad9e1' + ), + DataFile( + source=datasource, + path='dir1/file2.txt', + last_updated=timezone.now(), + size=2000, + hash='a78168c7c97115bafd96450ed03ea43acec495094c5caa28f0d02e20e3a76cc2' + ), + DataFile( + source=datasource, + path='dir1/file3.txt', + last_updated=timezone.now(), + size=3000, + hash='12b8827a14c4d5a2f30b6c6e2b7983063988612391c6cbe8ee7493b59054827a' + ), + ) + DataFile.objects.bulk_create(data_files) diff --git a/netbox/core/urls.py b/netbox/core/urls.py new file mode 100644 index 000000000..128020890 --- /dev/null +++ b/netbox/core/urls.py @@ -0,0 +1,22 @@ +from django.urls import include, path + +from utilities.urls import get_model_urls +from . import views + +app_name = 'core' +urlpatterns = ( + + # Data sources + path('data-sources/', views.DataSourceListView.as_view(), name='datasource_list'), + path('data-sources/add/', views.DataSourceEditView.as_view(), name='datasource_add'), + path('data-sources/import/', views.DataSourceBulkImportView.as_view(), name='datasource_import'), + path('data-sources/edit/', views.DataSourceBulkEditView.as_view(), name='datasource_bulk_edit'), + path('data-sources/delete/', views.DataSourceBulkDeleteView.as_view(), name='datasource_bulk_delete'), + path('data-sources//', include(get_model_urls('core', 'datasource'))), + + # Data files + path('data-files/', views.DataFileListView.as_view(), name='datafile_list'), + path('data-files/delete/', views.DataFileBulkDeleteView.as_view(), name='datafile_bulk_delete'), + path('data-files//', include(get_model_urls('core', 'datafile'))), + +) diff --git a/netbox/core/views.py b/netbox/core/views.py new file mode 100644 index 000000000..63905228e --- /dev/null +++ b/netbox/core/views.py @@ -0,0 +1,118 @@ +from django.contrib import messages +from django.shortcuts import get_object_or_404, redirect + +from netbox.views import generic +from netbox.views.generic.base import BaseObjectView +from utilities.utils import count_related +from utilities.views import register_model_view +from . import filtersets, forms, tables +from .models import * + + +# +# Data sources +# + +class DataSourceListView(generic.ObjectListView): + queryset = DataSource.objects.annotate( + file_count=count_related(DataFile, 'source') + ) + filterset = filtersets.DataSourceFilterSet + filterset_form = forms.DataSourceFilterForm + table = tables.DataSourceTable + + +@register_model_view(DataSource) +class DataSourceView(generic.ObjectView): + queryset = DataSource.objects.all() + + def get_extra_context(self, request, instance): + related_models = ( + (DataFile.objects.restrict(request.user, 'view').filter(source=instance), 'source_id'), + ) + + return { + 'related_models': related_models, + } + + +@register_model_view(DataSource, 'sync') +class DataSourceSyncView(BaseObjectView): + queryset = DataSource.objects.all() + + def get_required_permission(self): + return 'core.sync_datasource' + + def get(self, request, pk): + # Redirect GET requests to the object view + datasource = get_object_or_404(self.queryset, pk=pk) + return redirect(datasource.get_absolute_url()) + + def post(self, request, pk): + datasource = get_object_or_404(self.queryset, pk=pk) + job_result = datasource.enqueue_sync_job(request) + + messages.success(request, f"Queued job #{job_result.pk} to sync {datasource}") + return redirect(datasource.get_absolute_url()) + + +@register_model_view(DataSource, 'edit') +class DataSourceEditView(generic.ObjectEditView): + queryset = DataSource.objects.all() + form = forms.DataSourceForm + + +@register_model_view(DataSource, 'delete') +class DataSourceDeleteView(generic.ObjectDeleteView): + queryset = DataSource.objects.all() + + +class DataSourceBulkImportView(generic.BulkImportView): + queryset = DataSource.objects.all() + model_form = forms.DataSourceImportForm + table = tables.DataSourceTable + + +class DataSourceBulkEditView(generic.BulkEditView): + queryset = DataSource.objects.annotate( + count_files=count_related(DataFile, 'source') + ) + filterset = filtersets.DataSourceFilterSet + table = tables.DataSourceTable + form = forms.DataSourceBulkEditForm + + +class DataSourceBulkDeleteView(generic.BulkDeleteView): + queryset = DataSource.objects.annotate( + count_files=count_related(DataFile, 'source') + ) + filterset = filtersets.DataSourceFilterSet + table = tables.DataSourceTable + + +# +# Data files +# + +class DataFileListView(generic.ObjectListView): + queryset = DataFile.objects.defer('data') + filterset = filtersets.DataFileFilterSet + filterset_form = forms.DataFileFilterForm + table = tables.DataFileTable + actions = ('bulk_delete',) + + +@register_model_view(DataFile) +class DataFileView(generic.ObjectView): + queryset = DataFile.objects.all() + + +@register_model_view(DataFile, 'delete') +class DataFileDeleteView(generic.ObjectDeleteView): + queryset = DataFile.objects.all() + + +class DataFileBulkDeleteView(generic.BulkDeleteView): + queryset = DataFile.objects.defer('data') + filterset = filtersets.DataFileFilterSet + table = tables.DataFileTable diff --git a/netbox/extras/management/commands/nbshell.py b/netbox/extras/management/commands/nbshell.py index 07f943d15..04a67eb49 100644 --- a/netbox/extras/management/commands/nbshell.py +++ b/netbox/extras/management/commands/nbshell.py @@ -9,7 +9,7 @@ from django.contrib.auth.models import User from django.contrib.contenttypes.models import ContentType from django.core.management.base import BaseCommand -APPS = ('circuits', 'dcim', 'extras', 'ipam', 'tenancy', 'users', 'virtualization', 'wireless') +APPS = ('circuits', 'core', 'dcim', 'extras', 'ipam', 'tenancy', 'users', 'virtualization', 'wireless') BANNER_TEXT = """### NetBox interactive shell ({node}) ### Python {python} | Django {django} | NetBox {netbox} diff --git a/netbox/extras/models/models.py b/netbox/extras/models/models.py index e608f81b1..df32d6ac4 100644 --- a/netbox/extras/models/models.py +++ b/netbox/extras/models/models.py @@ -11,6 +11,7 @@ from django.core.validators import MinValueValidator, ValidationError from django.db import models from django.http import HttpResponse, QueryDict from django.urls import reverse +from django.urls.exceptions import NoReverseMatch from django.utils import timezone from django.utils.formats import date_format from django.utils.translation import gettext as _ @@ -634,7 +635,7 @@ class JobResult(models.Model): def delete(self, *args, **kwargs): super().delete(*args, **kwargs) - rq_queue_name = get_config().QUEUE_MAPPINGS.get(self.obj_type.name, RQ_QUEUE_DEFAULT) + rq_queue_name = get_config().QUEUE_MAPPINGS.get(self.obj_type.model, RQ_QUEUE_DEFAULT) queue = django_rq.get_queue(rq_queue_name) job = queue.fetch_job(str(self.job_id)) @@ -642,7 +643,10 @@ class JobResult(models.Model): job.cancel() def get_absolute_url(self): - return reverse(f'extras:{self.obj_type.name}_result', args=[self.pk]) + try: + return reverse(f'extras:{self.obj_type.model}_result', args=[self.pk]) + except NoReverseMatch: + return None def get_status_color(self): return JobResultStatusChoices.colors.get(self.status) @@ -693,7 +697,7 @@ class JobResult(models.Model): schedule_at: Schedule the job to be executed at the passed date and time interval: Recurrence interval (in minutes) """ - rq_queue_name = get_config().QUEUE_MAPPINGS.get(obj_type.name, RQ_QUEUE_DEFAULT) + rq_queue_name = get_config().QUEUE_MAPPINGS.get(obj_type.model, RQ_QUEUE_DEFAULT) queue = django_rq.get_queue(rq_queue_name) status = JobResultStatusChoices.STATUS_SCHEDULED if schedule_at else JobResultStatusChoices.STATUS_PENDING job_result: JobResult = JobResult.objects.create( diff --git a/netbox/netbox/api/views.py b/netbox/netbox/api/views.py index 6c6083959..023843bca 100644 --- a/netbox/netbox/api/views.py +++ b/netbox/netbox/api/views.py @@ -27,6 +27,7 @@ class APIRootView(APIView): return Response({ 'circuits': reverse('circuits-api:api-root', request=request, format=format), + 'core': reverse('core-api:api-root', request=request, format=format), 'dcim': reverse('dcim-api:api-root', request=request, format=format), 'extras': reverse('extras-api:api-root', request=request, format=format), 'ipam': reverse('ipam-api:api-root', request=request, format=format), diff --git a/netbox/netbox/graphql/schema.py b/netbox/netbox/graphql/schema.py index 82abfb4d5..7224f3c38 100644 --- a/netbox/netbox/graphql/schema.py +++ b/netbox/netbox/graphql/schema.py @@ -1,6 +1,7 @@ import graphene from circuits.graphql.schema import CircuitsQuery +from core.graphql.schema import CoreQuery from dcim.graphql.schema import DCIMQuery from extras.graphql.schema import ExtrasQuery from ipam.graphql.schema import IPAMQuery @@ -14,6 +15,7 @@ from wireless.graphql.schema import WirelessQuery class Query( UsersQuery, CircuitsQuery, + CoreQuery, DCIMQuery, ExtrasQuery, IPAMQuery, diff --git a/netbox/netbox/navigation/menu.py b/netbox/netbox/navigation/menu.py index 83a81690f..6fce7dfe6 100644 --- a/netbox/netbox/navigation/menu.py +++ b/netbox/netbox/navigation/menu.py @@ -287,6 +287,7 @@ OTHER_MENU = Menu( MenuGroup( label=_('Integrations'), items=( + get_model_item('core', 'datasource', _('Data Sources')), get_model_item('extras', 'webhook', _('Webhooks')), MenuItem( link='extras:report_list', diff --git a/netbox/netbox/registry.py b/netbox/netbox/registry.py index 76886e791..670bca683 100644 --- a/netbox/netbox/registry.py +++ b/netbox/netbox/registry.py @@ -25,9 +25,10 @@ class Registry(dict): # Initialize the global registry registry = Registry() +registry['data_backends'] = dict() +registry['denormalized_fields'] = collections.defaultdict(list) registry['model_features'] = { feature: collections.defaultdict(set) for feature in EXTRAS_FEATURES } -registry['denormalized_fields'] = collections.defaultdict(list) registry['search'] = dict() registry['views'] = collections.defaultdict(dict) diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 7f55463df..22849e6ba 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -332,6 +332,7 @@ INSTALLED_APPS = [ 'social_django', 'taggit', 'timezone_field', + 'core', 'circuits', 'dcim', 'ipam', diff --git a/netbox/netbox/urls.py b/netbox/netbox/urls.py index 84e899ed2..22c47f7bb 100644 --- a/netbox/netbox/urls.py +++ b/netbox/netbox/urls.py @@ -42,6 +42,7 @@ _patterns = [ # Apps path('circuits/', include('circuits.urls')), + path('core/', include('core.urls')), path('dcim/', include('dcim.urls')), path('extras/', include('extras.urls')), path('ipam/', include('ipam.urls')), @@ -53,6 +54,7 @@ _patterns = [ # API path('api/', APIRootView.as_view(), name='api-root'), path('api/circuits/', include('circuits.api.urls')), + path('api/core/', include('core.api.urls')), path('api/dcim/', include('dcim.api.urls')), path('api/extras/', include('extras.api.urls')), path('api/ipam/', include('ipam.api.urls')), diff --git a/netbox/project-static/dist/netbox.js b/netbox/project-static/dist/netbox.js index 19cdae0bd318381cbc0701ff151e7f87ad3ae9ee..e5793c1282d63d16d4b6e1eb8cf2ea7e3be55c7a 100644 GIT binary patch delta 16968 zcmZ{L2Y6f6+3<7Dy^5XK*|DAFr6`V+T+4C>NvzyBvMk%OCChta9BnDqUbZ|)fKvLk zrHl(4O3MiIBMA_aG&gJlgoHvUl$JdbAfe1M{z6*{{O`F}R-pg$KMPD_iEu>g9*cxyi8x#o9Ddy)d6sgcWy3b`B% zg@&dw=yF6PblI!9RCcAFF5YbiNK!)U|JLjr%l4;bG;)C??})&OO%XmF(MDonr#l|! zhNV`$I~3+RBfKe6QDKT`Q|?&Y9}dMQCx@Y9INQOEM)*$X>4ct5oW;*21NzZOt=i(} zMypF1}lt( z<(7y#VdoMd&E#acx?Xr>Z>gdlps@(2_cozcVfDTWl{vysOzF*$LAPy)SB@+7t>@0s znTYvAUbR_B>{Bir$%2%y$tNBW?%o&9X_NZfGP}+aw%$|I<&qERt>$)QXN7DTF^KR1 zdH_$7581S?urnEShZ0=O#&<;2o+L=RRBD|uw>{xD1>E$rM!zGo!Av%0tMv$<-BVKO zlXpo~bJ6TBejA?ve&@n6j9lEtyC_rXjlX==y%{634Mw0R>4Y0}}$kWjNjqrQz{zmwHYySo6Zh6jdHP@4! zGoCqrZ-ntD<224QJVD%`|fwnhE_~!(Ng$yf1}_$ux@2g zK7402*PmV5XXCr62t{nt()xbsa4#IFThkW-#YUA@FoPd241u$8jle#X8Yvqer0tf- zRpo`V^nDSnhY#9nTZEy5i&r&9KnrU?_wp&58k8~2q9Aczn?}4~-P0WUy`3+&~s=uh9O76=mL})*p&T+|ER2 zI2m)g%Xy_I8FD5-(3K#kUg4TQto03?W6u8`KdpM?4Gh&>B1pA4J#=S=^+s)ciXPG( z!K}Vdq$6&yBJQA|J*?X54r-<)HYn?VV>K7ct~v%RLrQaPNm}#S zwDn)$$L7`gs|ZMo?rwYF_x za@A{KP{V@oKrKoOFFmlPZ2HSIYGPs77WU)pK^U(O2qh1i7kaV>Ys=u?DU3h3beV+y zWRUNmGTH?46_lV=K~1>t!Aj%-tQ826f6pKTn^C^NKJ*LZ6fS@05d79Xe92N*cG)@` zs4Z-@TFv={Rge4*`GhAQc_Tjkg-ue6W%;OfU_;Q3>6_YN>-^7gP-thh_=jaGAxEKmbBevB^X z$ebeg;HO9H5a3Ps=3VtbbR6VsBue;416o(j zb!L~g>v&fjmhKcjd8)q5CXX1e=DM;Y0yY|Gh?Ivc%%xR*J3D+Fk z3O4SgW6oluyhbnBwCoxl9d8T5E(YlUyPi>y{U5NoLh@`osDlv&XH(UaF9&uFRU+JqbaIJm6sobZn6GAh6#?c5~H zd$vHaBM5vV?0T{Qb5X(Y?8>5Ou%e=TM^N@HC#q}KPI1*5FauN1E<;h_x6f8+cLX!* z_~H`7G*@$~?BQ)XJ`7$AXVA%hl}bW#>8*-fA^*AgP4&{&b-|3{G3sP%WYO^!dZ>CG zAE7G4s^fh?P0p;7wQehvR11$lKONTqpq9_l*9Acf8+Ek}!j;c$L=D1;=PY@>G6Dvw zxpATNqy~*=keM&+IH}Y2%U|75t~(2IG_zTgj*n8YQ|b6#O1YjOnXg3^``GBFZ&px3gvxSVLypIvw8+ z+ktYIR^Y~Ue2n^mjak6A{%J+NHK`ugadGPCTZQ1CG~mXNKQ-mFNXxbeXaDrG*?rSk z?bY#SVf2M6MIXfvxLQkvV=t_l*D3+iOh+|mK0s`dvmisw0U0&nDAz8mesTGnR*L98 zon-#>?Lya!vGWpMiQw@l=k`w9j^+$$+I9Q@#oB<5uctHJ20la`YqwW;5oCGG^4UV! zpIvz)GDhrCE_CpUKhH!cB>eWJ;=)*Vy|$$62H6dKz3|*i<#Q8WSj}bNM_^TbOj?bc zzXg0fHf+LATlP^~@0wIM7g+d;^pP2LpbxY0q;msmM9aR7juQSS2}9r z-mGby01FgL)a*#AtEng8qP4Xe4m`vlt**=RL{eVa73G}5=~tG|vBUYabQl-#@*P6n ztE-g(?;53UlTuU8^P46oM+0i9W32i-scY?8h=zJ!b*_xckVm4ND|^5$18=72w;6a7 zND?UicV69+@0IrqM!BF@Qtc|ASC%}3LDD%~Q<9I;c}4>-5#D0pEyDKKR^o^k93HF> z3OB!2Hs9y1sEEYX6ONkdDGn-N7hYQhzT7`wdkQrPkGw9xZ~Ges@aug;2WI8^H#X&^ z<-Jo;&LO<<##-zE)E_PuxHos^4a=X>QO+bBep5ArLZCMZr`{yRqw*(Hl2 zsgk@B!l*=;dNoR}0h)1V%pXZ8IfIvPHmH?m({Njdt;+@xQC*Us_O{fTi7C$LmD;56 zR>l0K-Zc%$HC^5{iO?DwkgiH8x2HIZ@SC?6BcpKF+hI^oMej^O{n~eK;w@hFSU^+L zZcuA#jVJj5 zIBPN#cZMT&N5Bo<8P}WS4Y8O#P2)@%LHge0;5cl;)72^}5=N~@-J9gblfr+_EEe{i z&cjworh4RY)TILerxjsPHWi~$u2mvu>yHFydQ6)P*jSVs$&x%|oDQW{zI$>HcI3U;mvnfq8$Q$uQ9j7tEo+d20MiI_jcvC$fIphu0dw{{D8a~7xnTz z!uRi~mUIJEjcTPo;f~o8;TRme$qPvhaEm6-F5$VRFxQ{t8k73LAuV-V!Od<=@|gx~ ztqi%x5=wultT9=u?gt6-s^R~}q(;N_3s=3rbbdcnYv__XIZdJx9)4eI>P`;H<`se% z5b;C^G_VRJOWhO(V+^^Ba+h&(Qso6*U!F8VFV{`OK4+%N`HUKUgHUy5V?mR58hHlU z0XIa!>`k0mQB>#Uy>PN=2e8ZT1s7qQYR~1ui)Tvm>*(qcBXwfygm2HR%OgHp~CZRM!uJtdV`Vgq2z4=T2aN+=jHn(1!d$7RF<2) z{5TzFG4ea8)vkx}kY?bTlX7CH-OC553o+p3Tj;!YU~}QZ4~v)quU4>~DHQJgaN$aK zikIUN&Jy62DSIFZ+|MPT7h>v|pEg1%Ve0G>6ubJPtA(#W;)KsXnjt*((OR5H2_;V~ z6;6FLPf&jRSV4H@%rIO{g@DAEGrKl?G@$&BH@*G5kc6Er^)r%jDUj--_hN}7r;2cR<@ zCMA&nssFQK-hWRMX8xlTR@D4s^Ac04qC#EQE$AfUz+rPHtjmkJ_0_Q>a_Uo;bt51RBw&LF#u~lZ%v47uz>JOHDdn*Eu>F9s#3zG6Cy;06AZDOY5N`=nv zhPUpJ&(IU)IhN z6CDli0O#=xwx)(ar!E%SzR!n*#?bc_xos)v*LO%L6~YfFa)4Wa?jypz-&a5$_OIVx zP@$4x>yC0wS=h$l%#^32DakhIo8%)Wf0z%U>SaF+mPV<30m6ZCvq&D77wV64b)e%a zIvP?^Xu%mQvLP{A_@izmJVm_OA{(Su3$KEw(adQ$dydO@ZoWG^R9DTqDuVKqWZn5l-K=nD}NO9cm=Moq<-OAbD&C+62Wv zXP{M3l;r@lE#k=>q(-Qhe4Ps)Bcy63szco*HWSrDOPq;n=Xa;55lV)%eizkVTrbI) zg&-eHYG7RpU zi|W}Z2cZ)~8>F6H4Io=o)Zl#kyW=+P=>|UT)#M&T?A#)a?7ch)|xBzH! zb1_OF4~1qm*;<0;td>5j&q2-%-LvZb8IRn+p3~Wx5(L0-40K zfhbjI5E;mYD%1t1dr5_6DUgZ0twD!Tnq0dY9aE$n6%{EmyauGwL9SebOtXxV1*R?p zDVvWLlW*3b&leayS|cO}-IJ4bVe0Ko8Ogsm)B}sPRigFSn1|1y**MLQbuyZh zDy|{Rk7=uvzU53-euTQ*!&W)h*J;fp&cu0I;Jl3u=hmADxIKv)l|%BhVt%hrZk%(p zdgXmQDsG%yRfU$Iak94xt1+bdWEq(ZbvohgxY@a!N8;4L;)5YBYBNjIHai zwU}CmjfM_W*Yr@-LgLkETVA^iPEf@)lSivj1@QBkYUIjurzNfBRdEBPYc2E-kcqYE z=Qx&@(kzRb07X(x(aQHb0A-0ZC@pRsS~07gHhZkRJ5Bo6p{^X2hwl$?4dl=|6sicN z`9%7sz)wy>&yPXm>PXXiRKB#q1LTE&p)_2GStloZtonW{Xfc(Cyl??iK(1ep97sjJ zU5{!}NYrmYr3lrL!Sg^jHIPfsLw7+@xe;xKPVYu^svwrmYDv{JH;j|>H-TdHlUb31FGHtm7;$Zmy+?AN1$zHv|ll7ALX%T~3;$~RKm6i>?pv!`XZqR%Ra zam`lVo>nXA{T2pPzP>+fihtuj_54ZpjH4Rn|X{KV}SoB%dz2r(gTAbf4Lp`kGoHE7cw_EuT z0I!w90=+X$oqe*ahRGq{>5;OcU!Kva;*#lUMLl4ZE+06zSvsUXNhcMF;2UaCUPNY* zE*0k^SMtbqVC=Mor?<2+BkA5-!SZPWai8hM%ao zA&S!Q-DP0%{LX|@-$!n{2_^GGCJDk+1D7yK zOx@X@=9Mc4?Uf!~<>sM@wNqe0(^ASQqX65}oYLzD8oEtC?pCUmYueLml$tdldi1+~ z+C3}_kze18?#m6iq}$UFG2ViDL6m-X3o1q-^1v-1{{!UJThImAZ6cd?qX;ec>_%6i z1Zlq=ZmN2Y&L`j9i*iWlbtaD#b7&U1@K$t7iC+fZ(ZIz_lv+PIV2o_J4SiD*m&c?V zxbXiwCQR3VDmNmHN%)2N&n~L=xAM&tso_>WWP;Le;{CLYxAJkR-3mD!ek=Twl4V>> zblib1orwb`(t0aeM%eq%yU0mCxDU*?ldRkiYCcKE_oJKhJSNK26EHJLRQH3&HFy9{ zlDr?CrQYEIv=ar$&|#z`-ycA?gXzERAY48!bD7lf(4aju1W$h;JPrU2A=?**oPdk` zEj(E8aPlRn|sM(nY;oA;47|A2O(fC$Si!L#giV@|n3sUa7A4E%Q85md(d zOll8#@CXtX#au%*;Jr)hLoN~?69Its2f*2g5#5963KSC`e-KP6?DX10Xzd)Qi|3VL z$@D5~o#K**QROO?!Nu7IhZ_>2?(ERc z>7H|AjpV)ekP7T%?)zxlk|q}?KZs}xN5DQ?rdI@pe;;{@Y%VSscKJPicdQNUINaM% z04(I~_tE5nQ5VE39(T+ga>`&BN#G1RA6dvVXOJCsSaTMPwUG>*MSs}Y>Ee~=fKzg# z(1w>7E+sbxwd?_vTvv9W)$R{L)*0GCsb-iSD1R_!2TF%iau(ueJsuyqm5-|u_#oN9@dYo@{QLfUvLG#s z?kCT;#%tqowKenX3=*A5KlKo6TZW+kZ$nc9pUp&=i(kgAPSU;^6_8!?a2Sn{&*$OG z(TI5Ad~Ae}hdi+ezXbTbt^ltGSe`AwOEDgClCnj(Ml4^9KWAX8o!6ZJJPMeN7oEH`TE!zhriNbuWSr+MmueqoY3Sf zc0Lt`d|z!OtWNp4VR1Q!>j6c?UWtzYSWC6I5%jb}i#5>twHDt9YI=1Qei-c#Kd8b8 zgP;3hHC~1-PIBWnsGO`?i%sO)wRkp3uf-Rm2Jt^@aV}N$iuKrlqGIEE%prD%t0qi# ze#|Z>!UjA9Ykjc+w;{I4scsNW8(~CQ)TJH{aifrXnec?PG4OojJmi$7c{?gzwF#d> zjLNAd7p=t$$z?UT50*bwgRes+O)h?z-q=ZR_d5AOKfH~IbFny=h!1fp@tQ5TkO3!Q zZ!LC%%P@Z%?niav$TmC^%^P=08V|g^L>cf#9r>975K<>TW58<>xOD%i!%AqD8u2F> z>~nqt{tFbR8}KI7LzXsT2ehUdu@;Jbjd;uAUZ*y5u??)mZw-ai^rZ>cdtDRmh8aCg z_$WApi_Q2#0En3J3Md{jq!0LbxtN@U6??=NtXM!~X&cTX zAsgNYHfmWLK5uTqC&^pJpKl@C+wdyv_6c_XBJsvH%%Zge6ig5iiEEu9Ufz-NFL)^d z>G7gmq4xoljoZmf?Z63rq@V*o07w5*2VR2O$%h^IX=v^5gl`G*WhXYlK4fzjUO~$X zx^UO>pKKL?urUNz4zM}hl|0{tTVZ~2H%NUq`KTL@m-JA_Aspi25P(VQUxnxlkSMrN zZ4oc-!Jn|GTMQ54{Rs7wvLRfr>~~IW=Zu~Fc4r_Qce_**QinzA@HVkd5Z;u$A? z3r5`K!W*G@$Aycb^E($l25#w-ZoC>CzR%pa2E|2<2ZJfHlR7VcsW9vV*kYp;HVKrt ze@ZnG7T5TIFUPCE?Rf5T zT(5BXG_@|V;5YaL!Yxkn_CFK@wnV9bWy_)iqEiaVee z;^#~6#P`#30rASaK|BHCYxm&oF!t*` zcqOEEmhZ({h1bUofK7rbxoIyhCO7QG<&ge*crS=hisbFX=OKq!w+~DhPWi~Td!+V# z_uy@qwvXNi+RY)B?Z*)eQT$a0@EVl~5Eb`@N0r<#pdlCr=bIuz3nW)fq$!QvNpk7{ z{t`^mO9$aJCb9DnC2AV{IXr=eNzq|gZrDdoKZOg)&4*!&Zu0zLybg5|EaG*uM`4F) z3XPEqp2N$?kcg8Dx?n$_-{o=x*6n`S$VT25v2L{i`eW|6f83qvPb4!$)4Kfepg$hZ z^c%_6Be0Ey#ExLy%6ixa+$(#a5^^Kp>Nv;XZ6Z9KQ*zeq5kunfBlv2B$1?KONq+x4 z7`=xc#I@MwBjv@+Dzf4soFIQZif57C58-yuXkR^q*C2~n`Y=9=uoccZK-I#dxQMm* zG#uIeXS|5CK8nv^K-7lEa0K#DKatN)@zcjZYk(8;>`|((jAZgjyh<_Zt5t`{?f|op z+ZCR{!kY}I6b>IlhISH1uRWzK$dW6kBauEUjHzEQK-$f8M?tB5y zr{z^I;ymg%-1ukwTkx}ff($dc@FiRcs1oma2`^<}6o6DpK6n`~1TW>=mjQhpV&D~g zhyj%WS}faW1or_;Q)00DUVIel{`id9?HUe=0=dk`}3Lh(l|UU(T41)%78yKqU8KtnI<^+K_xgT<+B-T-+Z%~ z7Zma26jw${=P*kY{k}}Khy``fO;U51$8k?adcvf0F4G9Jem583ZX|EbWqt!XKQNE! zK(IZ~{R+0hr4Eyh1xz*Mf`7Jvxem-WSyseco-1tvMw#p{Vmfd2~qOrOXM$w)<+D#T%D1Z=#}hpXBfN!VS33 zA88eIuM^$mI<VspoR;*Y zXCzn#??XCu8c1~;ll-u5a3nay+3ma$6>7h1F+ktU0~jj zoT5It5uOm+xsjmcptl4lXdSSf4BD7Mem3FpLS)WsnH3t37qWCxux}0}YM><#G-n2; z;DHEzhG8}cLtmGZJz8eo>=c~>)bM*lH7Rmj%T$4>`9{lJ532aeDrU*bA7#e4Cb-q7 zsm)~OmPwU)3*aHoS20D4lk$yIosoxH_QnYk3g1-$M|wz=j(L1%0+OIISdA7rfg5u0 z5-xpqJ^^VsF71)?gE5DkFEn}h7-VljcBWtK_#INubRu4Q>B0?rc%NM>KN*nd&=H_? z2stPnIz4=&Ww0YKw3Dm%^PQ0Y@ARC@GD4qpi(T&@GUuwC^1#|Z zrjtAj22H`r(C|Cp6{0-W@8CKD8ZP9ZdqG47w@9im1;b~jVAT7`r`5~?l)i2)qXjTv zV@?%@DH4I3J(R#siXg6Cs{5q6H%c9l`uVcEQb>BLMTCeZcG4F!uKDC}1hFrn64b0mZG6JQWn3KSYpKfA42lMy# zX67Rp^Gpp>mSb`7oqqKMIlYZpLT2h2eqOytt>dKsXjo?U2-_c6M0V(z4u!=5H>c#N zp1Bql>D|JVBcqt!!tBFfJS%EJY|SK8%RB-W5Vl`7OEW<^8`4Q;v3(oUf=H^KDIoV7 zm? z0|u2B8u_nztpqnPSHc{jfw>EJIE3AI6_Q`nfWP5xWaj2}c<9TDl$D%b&nza_G%}>C z!=ss?-SVpjc*55qUAO5+9Pl?k@?UwtyF+?vh|MzB?ohMBfu`X87XA=0O`(-nhL|}P zs_YPVHZdZ?Z4UDJhmg~2ZDy|dabp%?TL9%dNLvfD9VmZ)3!}q3926*(m}6nu5su0& za-o&E6gN49%H_+2)xHH}T`MyRDUH3YOaN@|VjE*yw8KGFKFzMjz=I8s0o(2%6E>iD z6M53c{0jE5wK0E&S(NZ1-_E>&SgQy8uq$7K`>H@YGfUjs33mIxI?4B)jE3CN#hfjv zgTunhNKb+@Sb6CYd=KEADrBRT?iEpYcs z{?*6C!A=MJnT;jVmYrrkLUYQaX1 zbbxsXdMgK+CWwbpgG?KPg5vQD81TSS_6w$KDcm)lyp>rF zi+pq|Q#BW-)1>VHQ$*GtU}~``EmYrAApYV2Gmh|phc?KPLyRVGm|oO?=L8Gn$=)N(O!Ba1XxoaYjXIj>6s#KF;jI2DrO9z!s49qYUH`#g$JmM;K6EZy#gou-iqHPct`z z`g!7Mz?FsM9cL;)H?2L+RMU^{76dKcmvCxp#<@VT&)Ngl3cL#a$`)y~5lytCy}NtwYM5V3(2pR~aXKd*oH75{i#sWnKhkd*n4{J?>7ETlOhd zkiyr&xOa;UuQOL;`1abH%t|PJc#}B-q4SZqm`fCp+}ZLT$Ya0Q{T|bbf!%LE!}Op= z@!K=t_Cjg+fGL9ngXs7hsJfi6i`u1#_^%I{-(EvLypUCq!xysaNZU>}ha?WM#bowQ z)(A}5zLV`FkzH&D`FbZi4h}|i7yAN4xOEq?O7QjF7qJOMn7d)Z*B7%V7Yv*DL1ozK zOh)`}mvV@UyJ$#UBz|%UI|o6Q=G&jKF9I8#yp&xzJ7}UgD;X1Za{p!QVzKDw>;?r& ziPm4S0)kym|C;RwM$%u-o@D@bE2n?SE)&1Klg*_;{_7XRsIt4+ z?K4m(x#&K&YJObGES}?mQOFChwP3T+ezp?$x@tfB8W?C8TS!XpXKT=?IB-8Zm%gvq zae%ewjL`$aHX9DHrKIX{Rv~_UfVH5d0l4V> zE6Ch?6f?!h5%zp$u2V|h!=WWcvy(R-VtEvR`x^EiG=VnvQPu;gsnnzFt$C0_TJlGB zQ4XY#v@f#tIgn|Jzsjy=Sf3NFnRdScq)d|KZ?Zqf5hpcw;)8Fp%?Na*;%#;VOwzy2 z=3#jM)Od2;;MiKIatNd&qZQ z_3#&h;IDrYRw%`j->}sP4U0eglYN)6$GhLN+gQfyR1-U+C?GYAq8ai&mof@F5cmT| z@!||P-2<}}Wq^j)XDRlukc+u=uHr+CI>qn;MO`jX>cCQkA0Fw5%a5fv#eH*u$AWE&I?!eN+Z5*6u=T%(6nD-*DYDz8xEMyM+=`tnpzC(Oq93wV-}@D5 zioLNBMJY6|A5qk@1wQ$H^rr_B;Xo6L{4OWo3Xi^|Cp9FVRO|&lDorW=2blftlme`S zNBnnM(Lw=im{7o#ibLGEQ}HC-(7I7smw3MCk%ZFeX-6b=AKOPcOb=*2(Zq43a+H}6rnsSMt{Ptk=VlFlfQ*k1(p zADsbTE;|U=kctxW#X-gGC_=6~1mH!;2Cl??e j|55QoVO^RVH*vitZY0h1rn!KV8!&N6@yTZuW$gb2f*KTk delta 16934 zcmZ{L33yXg+VFGEy`hDcowh8cO9&-_+caenXt?pD$<9E}oov2bBs7krvs6I&|qdX_wY%4+PYq5a$kQDk^MF zb%K6!q0pc^G^Ek{J!*$b;|a&qDSNEU9V!cZ${fOvs1Oz?#2b)FFyk@!J%+DFXklX<{PyOJ@O%8`4>tNE8h-19KcaPyMZ&Q}oG-7Q(&*{x z@c?fNsnr8JxVSWiwuK{nb4;z##|5;1sqn)sOHo)@a_dI;?YLF7ERY@7Zj;7woi;um zP-{55aK)|5=h-7elark`wLQX3z$O9V;ais%C$p2HHa<)TJ0rXuO1(NM?K1DSoLSDu zU?K#Q)lQ-Owi|Oi(i{(T7YJ{Dn&bNM>D{Q%=BQKbp9~Z8@qZS3HHy8eSM`3L|NdeH*$Yn)^Tn>gp zLsJ=ar6Ll#?A4q%yHbyhciRCHuh912H9JSM?G71@oIS~>B5>k#gm*-=kyzO2j>kEZ zRI7J~!rX9#Pe&>$(h+US9gF+Jq4?ya2^vh<2Cgf@4@1i^v~1$cel8i%cSUN|WaOb{o&WO}LlG$~Zu=Va5n@2vNx0>tB&I(y&F=z*X=>d{S zK4jIp!p>yS9ZGO9E8iYbCzBxQQmJ*u-1daq6mZjSjlMIp!Av%0txXD_-Ca`Yl6Of} zb33xT_^o^b_?-*OFmiD#@1bbjA#A>9a$z((&Tf^U&_|_Z-@K=9SyQ$zV3jzPb6R;P zo!ca|-+R^a2AQW#)to9j(wCH1QNb#MAWuX4H^T4L`y1i+&Hd-8d*nI8)m(3OPF?2w z;}PCR37E9<4TAH&a?}fpES(Knt-KD%Mrk%K?7Po78)`ABMoZz-{f&b2z`B(Y`S4xU zTz7V9pOx>SA{4PoOY6I(!##hXZcS?h6dP4q!3=)9R^A0?CC=e24QvGhdDok+$)WsSlg4qlVf zCqdIk&vNA-W(fU1X8@1b(|ec{kZfma6GTFrH2!43)?e_T-(Oltk1c*N~YbcK^K zr@Ne2dXgb$0t8(NavBt_`r}$($64n5&+*f$N8Z3t&AEeAo6|#gWms?2%6sV{2O@lz zRom;fk67#xy*rrI_lb1G4OV0zB4`h*whlxz(-Iq$^}n&23uRXw0~VqLv;mvYG7;p* ztZLFEu@%ydmq&A)aS*TUV!u)gv^#AlHA99bDHCut{gM2Kk^|}+BDM`P4_Ba<8 zE;v#OmT%7yDVy$@#!M^>+roZaJP70UcA?~f=7q`Z!P+yZ4+!HAEL|pH-x=gP zsfacK-hvXeD)lU=sX3R+8> z2+JBu@5vxLNBI7s(!61r1=^}Pm$2&L>yS%$;^EixBSA@rk5qHMAUR!y3WP@=$(u1u z1>7e*|48%vE_qsKH5Zij2_9Vi=nR}YDmU*C${$@Qv>x>pkI4s2y;S%BYCpF8vWDx7B zx#8@}b{+4E!^*?LCr{Rw+2j%9)!bNiL_jAi)X^Z{CA1$~j>hPsM7xQZF>{*Sqrz3k zwt{hc@tAYDL0+R5j9GRKkB+wmVHYb*Ef97+tswh9U~`4!am^fac92WQj|uyZm(DQ2 zP|kSph2vKuWE5N{Ho))iPpn2};n<1Ee4C7)?rN@HM)b^-PBs|znbptBZ(ol{>r8g14#9A-~Z3xvXQe=JEqIXzERpa zD$IMPK+zcl4iR=eQGmG}g5jB!MLU8O73G~l*{PhUu30<9Rck>1Pd&2??GS$VOog^H zm|4dcmsq8xnrq4)-m2rn;J$DMo$OFGNk}feS&=K`KRdt4EN$Ho%xD~=PBuek9dD$E zs@L%msv|5q-Urm=nxRhH+jO$Hj{ucvT?L?)&(b#pLH!zawJPC?XE!31aN=2W-nfi_ zfoiT!C_Sk`bs1#l3p-BgwB2&AJIeKBL5^lNYtr#iDsU^DH zJt>BJ&f;>yA94ngE_b}9H>fVxX*hq#dSOMP?57BL(L8uHQ^}NBCLL)cuq4# zbe~SLd-@i^_Cjof+ba<~9_0qI$7#usrbEYfP^=B;cr%^pHt-?pQ4dJRnD^&IUYm>z zdz4EYy!_8I5lRTZdvSST$UAL+Mv}7YV>j?-;n^3<=eoVHkjucg!GijbS9k&B_1rC> z@k;+vQ#O*U35RO!A$}xD1D(mqqydaBjS_00Q9W`n{g>rvQ7pT<&mdd-m{-`nw@`TO zuiNLh%Mc}_oL?H|7phchRr4VCZ80ApKmIWe2CcvMb63gwrn<&xynNv~(C}_wwyR-Yct> zcJCUcZj(||&hwilCr1NnsbQ@89I0vTT8MypUvaM7AwwRCa-Qq~Z3f;#F>f{SCJ-Xf z_HVzkB|j+d8H{ofucXgaF0U+f27{z-xM-5!LFX9_yhL-efj0`Q(Dm?s}0KXlt7r?LgbsgA}YhT}#=aBbK zMY)vl`s-_P3Q&J|xxl@#GtVSJ%OXO_+iQr6+8Ib`Si*jvQl821b;nT`@F3jtd8QvD< zI)wx8sAh~%BDV`~yt5K@3i#c|+;*>~HW3;5F4sle9C zkb5kl^oPp&leOw@kRY!b{_js}Akr0fzqfRLH+?Po1~2c0lTCYlHoF(RgF31`i-i}?l;k(i)gwmgxi$#jomrXRD`Vdn<;G=f zu_+_pD45>wz~l0?L+`swy5&)3Fr8T%jT+^^u1D6XmJb{&TV>SOgUQPd8#Bt$lg-F? z1Ksrf2mkUxCCclNhgzas=xT%t&$Sx)UK(T>prqt&1X@wW)avD1B?V>V4OEuM6#1VKv(lb={reaXp5J9Q2`nZ8c6aM zN{CoWMyNj}ha}BLzJnq*4A_!%MOpK-7;$Yz-k;LOA-Ecmz97tkz6MA?hEp^NsQPR( z3LiZ0vw6r8OKA-;uawt_^HK%Wj(#AS4G!sNhBDwtKxLXV(@|AI2E2_L)CWH=$MzI8 z(ksaI)McCX#n+ZV>a3M^hFxw$qQ;(5mxG;l`s}exO_w#$A$h%aFsrF)uVh^2GQB4y zNs#Bub2bH2623aYre=rrW^m?8@x#ERS`XJ_{AuvT`BMCqx0DKBe|cVoD@79*Q%-xr z>7#mkIyK+}#0C6Lw;BKfvpf0~O+jA!m8J?N^cv~NETkzJgdI`Nn+2giqcOe07hky` zX6pXB+!BNQ1eKK*qvVbOSPmILuEWR&Q_wR2!bzpdZRS(7o=#CU*yHC1K#2t12|K99 zgqhcIVGr4rhZYGhe_e#)!slP#vNqmX|U z@b3@6S#SNuoYy7eqzN1c;k$3P;^CC!ZuEUyz1S`jRu$z&v*_}e`C(ztw-tC4#(_up z_P4baHhGDLC^wcJ7c|S-%WLNC9?)K6lo2FHbf)0?$3-PZc}`uFGi2vjQ?lP|keDWq zW>;b%E*yf$k^AQj{nnI}7;=ZEb$CFAZyYRhc8yeKUx%6Rk_cm#;}esaAEVMYZ02pW zy-S9RGpE3BTk)@rxL)Sev47o$%m+)pyBUGQboAf$1x|UB-Y930HnGrgr9#*D!&^J$ zGxS8c_U!(nW?8K^r}$Cx&n#sdWf48iX^o_SF_OX;-0BWwHQo23E31sTCNtlg z(t051xYKM{`t=~lz7&lIGW+6xc zliFEmZ=TyBY13ASD(P`meJME%tTXc=ivIBoS$fTU9YuQ!(417|q0(6?DhAzVzPyk0 z<)Ia*k4)yFV%#BFffAwW@jUWG9;)6taMlX+0w-rOno6!C1&e9)J?4svZuqD6*!h+e zO|==})+7;2I>9kicaXLDh(kd#oR13gydK^gm$GpwvMV12q5ZRbREdIO@ocmZAurL* zK^n4bHkwU5b5MX(%|Ua?={cwrr3jjfR;z;^&SmGk9xmm{MBV`z0WDF^?wGbZeh0Ca zq9PO)N9Lk7gu2KB^N^71cR(N<;^U-yKH^ba%w38Uh+Mk>jlqcd3(;w|3s6S(EKn?B zts%ZkoU;fmMaVB|3Qz|^aWYW|L<*Bbh3MUiu%iazmMOaow}mi%lO!`VE1gj`mR76X-TFGtEb zHV<4mc{2XJjl5KjHn*f4d{cckWy)E>emZD|*g6_&kA)-dSR&1->S^P&gUvP6^PO(5 zX)K~vo~K-sa>)LmmGmo7a=yc%jyO0aWYEEmM(n&pd|L^Zw*@{&Z7S?{mFa503S_d) zR-#m)L1ZQ8tB?&&_o52TQlK<>OM?z02f2DRI;QYCDk{8WcnyeUid?YUo0?qv_{Aax+f>=!qnrNGLV0As0S8nuSDx{+CeE}Xf~ZivtgPY8)P&m zRh%lzj%kaOxaCY1euTQ*!xlNs*JY7yQj<7Oi(1e|)!FqX0&Y*DrpY0BTQR@aCs)oo zTjTOR9u-$dc2}V#sE+KdLaXz-<#wNn>mjGAPzflLAFI&gwXITn)RV~__B&*4+-#B6 zMWcgnwrFKS=%XHOcUxC?d%LyMW~v|V=ldPx%WAYRx6z?i+LfG=Ojd)RxV0M1T>xY2 zI<2jywqc{8(`1_-iW*708g0vKk--V7xIXeoHL3tDbEX=(@&@E~uZrs+wzbgSK_=Fs zOK>PHrCAm=0g9w_qJ{5v0Lt8HP+QzOv|?5Zt@c>>0SD<{hio}b9=<=osmP&qC{&S1 z^X~M|fuEd&mY;&iHISzDsC=o)GdbA+{}SoR$zIFkWRFGPZ=u1Yhdh5CQ$Vg=j~pOZ z->pZrC?V=MfD%FtWbhoM0n2moIp_{3DmS9d(CFQWP8EdGSxwM1%?)+r+)bcbV`SGR zbk5FLdWxHHkGY*S@HOp)_sV{SNyXXIRDUUxad#Q$Proyv)YpMNl3X@_TDG#S7QT_% zs&HDSUp$?O-Yjxd*J9!0X|Tz;~9xcxA zkxwzK;sP=S=XY561VFEqLj`>xO#=Y3tA@!T-|LaGqFbKPrQ)2@xv7^oV3F=2ICryj zO1+a#DiR@UsX=+Zv}EaQD$Yf&;E@-*=qXl+#aq!cDA(67>IHgK)~mx!(8mK#tqKVgy} zOf_(Blf>j*Eool4a?oDs(N%69npispHZ?7!pfU=vCCw?lejwt&&zhBL<(iiC8l`3p zh$8LRPy35y3G$m8(Y?6|mvn=gAjX?eFNoH4Hz6=0Vf|wj?Z% zNjGr*|2xJ{*MBnCCyhz?h564cs`j_>eH5wTHa=m3GH&8yQrX6bX=#V{5Wf}vNy#%V zBsy+G7th3Y6KT5{EhFr`=pAqpKDZaGd4R0k4{F~@#`mKe^O7dY)Dtk%NmTbqeq`W2 z^f?*14-Vu$fKtd#h7Kbo`QZS%6+D4k4#ExOGLK0e4-MKwL-71Zqjv{D5Fz^)hCG3X z{N)fj4x3!}N3?!+(xgxLwGn$P?&e+OjX$DY$S%TiOYp2X-I!CZP-@5p9|ON#a|D&K zE|WS*9yo%8MIqNv4Y>2t`jDH12Sos&{eCd_A)dp=w zo^Cli)i9yJPKO*Wn6> z0$?O>y@w_jbh#ja@wj8|kW&W3KmupbxyVSKK7;JA!a#;VXMUB{!BGXtDc4kcEbNQ2Me?50pC$*@4pGl$;S7Mk!8|oOycabk__g zKM2SmyFWlh;@%I?*RzmWyzwi}cQ8`*Wd1kkWn>XQ z{05cGK+R(A_vjKR$>j`wRxWhnZ&-YWg+*dBF-9;pISaqd*0#~bfbQT;WHT1vi}HM1 zyfz+J*JmD{LE1Cvr>o62_bPaS{h%gtwgqxkDV>7sryXN6AY9n9F!|RrV*8^zJ6yT*8w>e4KB3vVuFUDUmFm^#Pevu)zxp)!zXDR+2Y9~S& z{rM8D4)eSF2;7Z7)4$$b2UYK#75Z~@nJK@U*U!qu!kFlU$rz-FrXx4~RZGax| zmC3y-Tq2s)_+1QMl5Pz?y2j+>>tkyi{z`+rvN6mV?X;S9LX|Vy`BWHkf3=aYI_2j~ zVljv70awIciI0%U3Ot)sXtACw(ZW|qizK`1#9s_ za%m0jgXK@v;A>Dx)Wr|et2=7X;$8fpKhD7miCCOV#D};h@v1GjkO7xrZ!LC%12KOa z?ne#c$TmC^&8u@t`VM@*L>cf#1Nnsk(9$42ZNO_0IC%fA!%C=@8u2F>taE+?{tFbR z8}KI7OO`fb2h^q-u@;Jbjd;uAai=zO!wsy&Zw-ai^sNaueoYhZh8aCg_$auCi<|NJ z01#=$E1-C=884kv=b|h)MKb_XTsL{Q8NZp+>x6_H#J5kiz~n}8c`IIrQ7h@N;1URN zBNn`9dA|#i7+G7?CyCGWVOq%^3oc&R>!eTkc)6IIgcWb>bzk?}wfEIMnucL7$s^)rC!Xz(qFO@CsUJfK0*dYNL2z5B`Kj zJz{te??tyjxZ9Wg-+Z6#nVpwCXBelg*QU+ zwhKh7pM3Aa$G}H@!i`shxA&PF*PyVd@nEn+aZ=~SFBbZJ09$O7!X|+d_fM%N{NfrP zaD2Yq$HRR|65dg80r8|Ce}Qn=NA_Hdm%^g2jo?G5PP{yTH)2@rsSv&qBIrlwGkOyI zBhF=`esxT|Cye1qZNN7<84hV9VSfm$n++}(K|R{Z6H)xx|EEuUDu%Z(s9R)H__PA- z`1=$13x(bZ06D{ffISii)t`i#Bd$H$FdpZ(YgH3TdZzR!uhhEi345hI;#YEwWd4Pq zL_Ok}3vmoYPVD;yZiGA(x#?oO48r(_F2-Suk|Os@JgR`^TQ9?_z~^}OGF-3l_%yX1 zvEaA(1j3C@^4NT)Nc`q<{4JH-&vxU#0pGoRC0?!TbN+107CWyr#hrF~mFfaeyCQZc zeEQ(iW_KvL7E&$XYG6~RfVcAzUzP>~lm>m2X@bxQw=u5pXh=12wz(7N1{~sKq%s32(gM{vNVET8VAr33Z2f#hoCY0P)p#;q5T?o4fEz$n+HN z#aczs$8~^7+KX4?h>!eXFD@m&-;2v3ANJ5*kRmV1+lS9VDY0%J7%}Yik!^Q_^*V4j z<}s}uy%(<)SM0|@3=#eA19*)p4S0(C!lO#g1Xu`$A$p>C&;rF(6Ant^0Vg?i0DlE$ z>BWO^n6%h+h|<&nz8s!FCQ@`5N3h99PCtnY$&H6WefE&&4&!xbm|zjFo81LFOcQ97 zocAnVMutS3TrdXv`TQ=I8<1}I!$vmpo``j;tsFd!8*r`cflA1efScnSgSUzBc23FFXOEZ=pE`oCM0hkKUc=-M&wOs;(p4}doN`a!$~8O73v@cRhY!yyN75j2-Rf{R$APs5Qt ze+E~e?Gbzi1D-ZKiX&+B=h8VWe)=fr3~*?kIZCyaflNMuS1G!DwdxSr6JQpS8;>az zBB-Vjn()T3j6{~pBlNg*UGr9B_wxBWc*JA*gTSUik zJ_c1b=LA->iDVX{6L>Bty7Nw86@zT#qGxag%(?3s+zVa;eip2BGpTwOFJI71N$dq& zO_QjNlE{%al0{lOdFEMM2VTIOlVC=hMZ-y~N7&*c7ZA{XZ6cuX9Rx1T&gb!bTJC-U z=TWEO`agqz)b?{^SjhP=;z~f3c=wBVDFdSbq*C(1OL!r;DBryV=xY}PFXKZD_ygsy z<4+LyD0AM#Y6uuNz6rXbNgRF?zrup$-}DX`&JgK+2U=s|&Uf&2Fjidl9#|dlTx#Ek zlY$*5AK*u!+WR+L!m=T_xgwnL9aD1SVtx)|pvtZ- zmsw9Nxy&+heJ;~ZwO=lC3Hdaa$zkccUUGUCvxo>Y8567vuvUma&SYMOG%I;IkI{o) zvM`^y9%S&oe5Ru`43A5+A$zJa;17)`IlnDa1*bl!1lOc|Hbc#tZ#MJ1BAlG!%1G%P zW{INPm-#GWK@Ief)Ewq9+?$aaKk1swG{UUw<^sU|8RbkV*9c3HSK}E{LA!ORPR} zMG2z@6LWtF)6BL`i>kP^lxapqjXt=o2G#2f*yC}tKc3LqT`qO2I9|rIVDO=iDVY}v z24G`QP_XMjd|FAm$=(%k_JsKL3T6w!iHK~cJBWEDb3fTt!LY>nYg|lLRxpFifKNl( ze~qhHcSM%UmE^P8Og_11B{PeBS;1U^2SAGcjF*WwtYRS6Nsv8irUk4AaL_td6&>f( z5X-=0aoHR0gXiEhchzE*b6Fkm9Gn;{nTCyB^eVy%57_7%xKVgP2-h&u!=AQa8N3qd z(rF;!ZAkLNy1};K5NEgZ1`q`Ud90FI4hrV2N~W;cnw-89rjfthKJEJQ*5nlR#SKZA z$F&6|$GkN_L2HNYWYES8^5co57vgbV%dF5Oy^yPuLj2}Xq6TX6z~;0C0ebkop&BoFO3PG%k@;53Tnh^KiYjKw%Ae%JxF(~V;B;m#n^c)s z0ZH;)6;rg>DPKI*!S%9qr;>REkR;z%0Z%4Lm5zCAryEkCGFXjfIgOie@DeV4cHT{I ze;gh;Ef{jhX~MLJ4?zwWq-XlwPRt>-OsC_e_b!~t!~5)7`SE~6ht2?{L&8DnFzn$Q z&4ZnRp`Bd4pC5)>fML&1?g{Gs(tUQle=zNVcNx6NBfZ~uc+~ZNF71J}e@Z8L7z~<% zm7x-Iz*|IlY|O!R1|aw5pnE|i2DeG7Fa^VBr(o3k$*0xK0_3=6Eu%FKdjJ?t6^1Di zf%`m^zyXRNu0{HGN#DUVZ6D4e&g+L~Nz#li5?#x5kjPq4Uf--`O2OO|u4C%J`1G%1 zx}kROI)>V<@7FQ!fXzO&o*9OW!L|*|TNv^JrJIf1D>mpqk86QSfqCgQ;rN` zdJD4;gW0U81-Z45P%ZN?*gn{P*(}WjWo*bHS;UTQOe;06*BhAKVB@Ol7!HALJJ-l8 zMZNWopT5DPb09IMlLbtAb)fG8gy=QUYK9N4O>WEtU!%|jH$Ai?5!$MolCU-6;Vk52 zJ*;jaf2(J71X++zfXkGdH*0L*G}VEadchW-+;{5po&r9?b-8mftqOBffU& zZcX3jP*2CA@bZwJ8UnJ+%{zS6!-1yY3K#zTFHND9SB5w_7OHF)cQ!F1!Xpmy#fOl^ zYinVy`e|hrVOs&;ous{$*$!mCua(i^P6q`_CFYo!4n$73G8N=}3v)4!I)uvNVqvv! z0a@3^j6x1$ZyOT;JG_JdE@tID>_kp1=13 z(y1mk!mRSgMoKz+b4{iL{$nZS)6VlPLF z*Z}1E$$foHh0+7*@Q^*|hL+`9z8kzOJ<=s_DjaL6z&;M-pmxk3Lo9fRL#W>xI;X^6p^(D zm|C*r05en3+wMip*ZrMUIQi88j;HJHkB5pfNG<5VL@V=ADl*Dr|$xnFDMA zdEhZ-7rEszfTQCm14%=1<>Sl|22|Et$Cx@i;33N6%#EN<9zPCvGLpQfm%@5XPKR#*EXGG8k#NAl@zE(xjdUvY6DVBhJa?s67-L8 z2W@H2n&#@$TyuH|6d1@fm2{P)=9|b$_T}Jv}VVuzS@GDFu6d%9Bya0^$@T<&v+><6Z?Nh8Eg|C4{?-3hbW3I%|_v#zW zN+^DOgE<0$@{u>0ixiN`+43&PVz=1+F4KmA)o(q+^q_w6yEEYJLTUJbDT8!^==dAx zxE#NW8Y7?h&kvd3U4~tn0ez!|G#D!fn2rd#o zxrm*EAT9IVFW47=jZR+7uACh)(Il0O3_H2+Qg*RebP2mbfxM#S*DR#wfN6fi_5&~J zFJs?l0DR?c_V>Ba{P6GCUo8PHgz5B2LQj}J%*wo2D|3k3f6xAkKA-sbIyP4U{ymh% z;=Dbqz=E*cdKw!d6>Jj$lJmeIY{13Y*2PH-A3v7K3@`&+Q*wqZ{ za>9+$p4ZuN$Q>2G!Cr!WPHN}G2i{;?5a>z8TkHmyq<@Re!|?X0@f3Tm!sFr^YJ@%S zFBBg<#oooRP!E#o(`*sh^&Yz)maaa-zNEk|7b)8W@zwH=0TUk5_c8lDjzGrcY@2xd zC+z1e4myP|ix$xf*_q_fm+U1#ZT(m5d5Q%t=M>y|$k$u-@E3vLq<<1tD8-ZCvegKg z#2^2`zC(HAogdh3EE9wzxSdfHkQzqO0$HAm8HF9F`vIePVFsM;{#lALK*MXZ6nC*G zEnYlV@gYXTVt9d~E*A)OV5!0n&vV4$Vnt~othr^aV%HofAKs?8cP=nkuw79Hx@>>D zqIouK{jVX#?K40k_P7)m!YGwnv6BUK-Rf8LLvHE^zamYsH#VXuh3d5{e(v;%AfYa|xDZnx$#ebz0;8%5t4HF8uKS_xj zcPgHsdwjP`Q4V`7zd-R{EI@L{#fstFxhcr{p5@aNVe-A8=!Px2u2KxMaM**_D#|f43V7Z93q>LM^!Ex4v=&{bcrIt5%L#wd zBWv24#+Vw?=PvS(n-%{JCT-6xibcpJ9==7` zk6d#I!1IyE4kam;`>JwdvhTUz49nP(;=QYs@Ts0jGr7=EQ8{_rxX?B zmE($qnI>}IaRsLc*d6WwdFQxdC9SW@{|SyFr22&75;EikZMXac%mdCpq4+mk;%c8( y>;vh0{b>b4b7_ch)-pKB+-DSz7dE81aTC{T;@aQ?jD?9CFmX=tgJ%?F?EeD9rU`-o diff --git a/netbox/project-static/dist/netbox.js.map b/netbox/project-static/dist/netbox.js.map index d0563b9fc670f99db1fe7ed2c7fb22555367e360..1ef66b9dbd29ebfc3129c12d135db9c8bbc1916c 100644 GIT binary patch delta 89 zcmeApAUn{}F;K_T(a}?<(8pAlu*3FIncXZbA Vbac*{9+=G9#OSvDN;2zNMgZCk6w?3z diff --git a/netbox/project-static/src/htmx.ts b/netbox/project-static/src/htmx.ts index 70ed4f534..bd80a0c49 100644 --- a/netbox/project-static/src/htmx.ts +++ b/netbox/project-static/src/htmx.ts @@ -1,8 +1,10 @@ import { getElements, isTruthy } from './util'; import { initButtons } from './buttons'; +import { initSelect } from './select'; function initDepedencies(): void { - for (const init of [initButtons]) { + console.log('initDepedencies()'); + for (const init of [initButtons, initSelect]) { init(); } } diff --git a/netbox/templates/core/datafile.html b/netbox/templates/core/datafile.html new file mode 100644 index 000000000..a10f5039d --- /dev/null +++ b/netbox/templates/core/datafile.html @@ -0,0 +1,81 @@ +{% extends 'generic/object.html' %} +{% load buttons %} +{% load custom_links %} +{% load helpers %} +{% load perms %} +{% load plugins %} + +{% block breadcrumbs %} + {{ block.super }} +
  • {{ object.source }}
    • +{% endblock %} + +{% block controls %} +
    +
    + {% plugin_buttons object %} +
    + {% if request.user|can_delete:object %} + {% delete_button object %} + {% endif %} +
    + {% custom_links object %} +
    +
    +{% endblock controls %} + +{% block content %} +
    +
    +
    +
    Data File
    +
    + + + + + + + + + + + + + + + + + + + + + +
    Source{{ object.source }}
    Path + {{ object.path }} + + + +
    Last Updated{{ object.last_updated }}
    Size{{ object.size }} byte{{ object.size|pluralize }}
    SHA256 Hash + {{ object.hash }} + + + +
    +
    +
    +
    +
    Content
    +
    + 
    {{ object.data_as_string }}
    +
    +
    + {% plugin_left_page object %} +
    +
    +
    +
    + {% plugin_full_width_page object %} +
    +
    +{% endblock %} diff --git a/netbox/templates/core/datasource.html b/netbox/templates/core/datasource.html new file mode 100644 index 000000000..168ced700 --- /dev/null +++ b/netbox/templates/core/datasource.html @@ -0,0 +1,114 @@ +{% extends 'generic/object.html' %} +{% load static %} +{% load helpers %} +{% load plugins %} +{% load render_table from django_tables2 %} + +{% block extra_controls %} + {% if perms.core.sync_datasource %} + {% if object.ready_for_sync %} +
    + {% csrf_token %} + +
    + {% else %} + + {% endif %} + {% endif %} +{% endblock %} + +{% block content %} +
    +
    +
    +
    Data Source
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Name{{ object.name }}
    Type{{ object.get_type_display }}
    Enabled{% checkmark object.enabled %}
    Status{% badge object.get_status_display bg_color=object.get_status_color %}
    Last synced{{ object.last_synced|placeholder }}
    Description{{ object.description|placeholder }}
    URL + {{ object.url }} +
    Ignore rules + {% if object.ignore_rules %} + 
    {{ object.ignore_rules }}
    + {% else %} + {{ ''|placeholder }} + {% endif %}
    +
    +
    + {% include 'inc/panels/tags.html' %} + {% include 'inc/panels/comments.html' %} + {% plugin_left_page object %} +
    +
    +
    +
    Backend
    +
    + + {% for name, field in object.get_backend.parameters.items %} + + + + + {% empty %} + + + + {% endfor %} +
    {{ field.label }}{{ object.parameters|get_key:name|placeholder }}
    + No parameters defined +
    +
    +
    + {% include 'inc/panels/related_objects.html' %} + {% include 'inc/panels/custom_fields.html' %} + {% plugin_right_page object %} +
    +
    +
    +
    +
    +
    Files
    +
    +
    + {% plugin_full_width_page object %} +
    +
    +{% endblock %} diff --git a/netbox/utilities/files.py b/netbox/utilities/files.py new file mode 100644 index 000000000..68afe2962 --- /dev/null +++ b/netbox/utilities/files.py @@ -0,0 +1,9 @@ +import hashlib + + +def sha256_hash(filepath): + """ + Return the SHA256 hash of the file at the specified path. + """ + with open(filepath, 'rb') as f: + return hashlib.sha256(f.read())