From 6d5e33b78514561c5c32e47d3cd89129996b7973 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Tue, 24 Jan 2023 13:56:59 -0500 Subject: [PATCH] WIP --- netbox/core/admin/__init__.py | 18 ++++++ netbox/core/migrations/0001_initial.py | 26 +++----- netbox/core/models.py | 66 +++++++++++++------- netbox/extras/management/commands/nbshell.py | 2 +- 4 files changed, 70 insertions(+), 42 deletions(-) create mode 100644 netbox/core/admin/__init__.py diff --git a/netbox/core/admin/__init__.py b/netbox/core/admin/__init__.py new file mode 100644 index 000000000..389f005fc --- /dev/null +++ b/netbox/core/admin/__init__.py @@ -0,0 +1,18 @@ +from django.contrib import admin + +from core.models import DataFile, DataSource + + +@admin.register(DataSource) +class DataSourceAdmin(admin.ModelAdmin): + list_display = ('name', 'file_count') + + @staticmethod + def file_count(obj): + return obj.datafiles.count() + + +@admin.register(DataFile) +class DataFileAdmin(admin.ModelAdmin): + list_display = ('path', 'size') + readonly_fields = ('source', 'path', 'last_updated', 'size', 'checksum') diff --git a/netbox/core/migrations/0001_initial.py b/netbox/core/migrations/0001_initial.py index 1632d464e..5147f7788 100644 --- a/netbox/core/migrations/0001_initial.py +++ b/netbox/core/migrations/0001_initial.py @@ -1,9 +1,7 @@ -# Generated by Django 4.1.5 on 2023-01-22 20:49 +# Generated by Django 4.1.5 on 2023-01-24 18:56 from django.db import migrations, models import django.db.models.deletion -import taggit.managers -import utilities.json class Migration(migrations.Migration): @@ -11,7 +9,6 @@ class Migration(migrations.Migration): initial = True dependencies = [ - ('extras', '0084_staging'), ] operations = [ @@ -19,16 +16,12 @@ class Migration(migrations.Migration): name='DataSource', fields=[ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False)), - ('created', models.DateTimeField(auto_now_add=True, null=True)), - ('last_updated', models.DateTimeField(auto_now=True, null=True)), - ('custom_field_data', models.JSONField(blank=True, default=dict, encoder=utilities.json.CustomFieldJSONEncoder)), - ('comments', models.TextField(blank=True)), ('name', models.CharField(max_length=100, unique=True)), ('type', models.CharField(default='local', max_length=50)), ('enabled', models.BooleanField(default=True)), ('description', models.CharField(blank=True, max_length=200)), - ('url', models.URLField()), - ('tags', taggit.managers.TaggableManager(through='extras.TaggedItem', to='extras.Tag')), + ('url', models.CharField(max_length=200)), + ('ignore_rules', models.TextField(blank=True)), ], options={ 'ordering': ('name',), @@ -38,15 +31,12 @@ class Migration(migrations.Migration): name='DataFile', fields=[ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False)), - ('created', models.DateTimeField(auto_now_add=True, null=True)), - ('custom_field_data', models.JSONField(blank=True, default=dict, encoder=utilities.json.CustomFieldJSONEncoder)), - ('path', models.CharField(max_length=1000, unique=True)), - ('last_updated', models.DateTimeField()), - ('size', models.PositiveIntegerField()), - ('checksum', models.CharField(max_length=64)), + ('path', models.CharField(editable=False, max_length=1000, unique=True)), + ('last_updated', models.DateTimeField(editable=False)), + ('size', models.PositiveIntegerField(editable=False)), + ('checksum', models.CharField(editable=False, max_length=64)), ('data', models.BinaryField()), - ('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='datafiles', to='core.datasource')), - ('tags', taggit.managers.TaggableManager(through='extras.TaggedItem', to='extras.Tag')), + ('source', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='datafiles', to='core.datasource')), ], options={ 'ordering': ('source', 'path'), diff --git a/netbox/core/models.py b/netbox/core/models.py index 3cf2c9eef..7f6ae350b 100644 --- a/netbox/core/models.py +++ b/netbox/core/models.py @@ -1,6 +1,7 @@ import logging import os from functools import cached_property +from fnmatch import fnmatchcase from urllib.parse import urlparse from django.conf import settings @@ -9,7 +10,6 @@ from django.urls import reverse from django.utils import timezone from django.utils.translation import gettext as _ -from netbox.models import NetBoxModel, PrimaryModel from utilities.files import sha256_checksum from .choices import * @@ -21,7 +21,7 @@ __all__ = ( logger = logging.getLogger('netbox.core.data') -class DataSource(PrimaryModel): +class DataSource(models.Model): """ A remote source from which DataFiles are synchronized. """ @@ -41,24 +41,24 @@ class DataSource(PrimaryModel): max_length=200, blank=True ) - url = models.URLField( - verbose_name='URL' + url = models.CharField( + max_length=200, + verbose_name=_('URL') + ) + ignore_rules = models.TextField( + blank=True, + help_text=_("Patterns (one per line) matching files to ignore when syncing") ) class Meta: ordering = ('name',) + def __str__(self): + return self.name + # def get_absolute_url(self): # return reverse('core:datasource', args=[self.pk]) - # @property - # def root_path(self): - # if self.pk is None: - # return None - # if self.type == DataSourceTypeChoices.LOCAL: - # return self.url.lstrip('file://') - # return os.path.join(DATASOURCES_CACHE_PATH, str(self.pk)) - def sync(self): """ Create/update/delete child DataFiles as necessary to synchronize with the remote source. @@ -131,33 +131,50 @@ class DataSource(PrimaryModel): if path.startswith('.'): continue for file_name in file_names: - # TODO: Apply include/exclude rules - if file_name.startswith('.'): - continue - paths.add(os.path.join(path, file_name)) + if not self._ignore(file_name): + paths.add(os.path.join(path, file_name)) logger.debug(f"Found {len(paths)} files") return paths + def _ignore(self, filename): + """ + Returns a boolean indicating whether the file should be ignored per the DataSource's configured + ignore rules. + """ + if filename.startswith('.'): + return True + for rule in self.ignore_rules.splitlines(): + if fnmatchcase(filename, rule): + return True + return False -class DataFile(NetBoxModel): + +class DataFile(models.Model): """ A database object which represents a remote file fetched from a DataSource. """ source = models.ForeignKey( to='core.DataSource', on_delete=models.CASCADE, - related_name='datafiles' + related_name='datafiles', + editable=False ) path = models.CharField( max_length=1000, - unique=True + unique=True, + editable=False + ) + last_updated = models.DateTimeField( + editable=False + ) + size = models.PositiveIntegerField( + editable=False ) - last_updated = models.DateTimeField() - size = models.PositiveIntegerField() # TODO: Create a proper SHA256 field checksum = models.CharField( - max_length=64 + max_length=64, + editable=False ) data = models.BinaryField() @@ -170,6 +187,9 @@ class DataFile(NetBoxModel): ), ) + def __str__(self): + return self.path + # def get_absolute_url(self): # return reverse('core:datafile', args=[self.pk]) @@ -182,7 +202,7 @@ class DataFile(NetBoxModel): # Get attributes from file on disk file_size = os.path.getsize(file_path) - file_checksum = sha256_checksum(file_path) + file_checksum = sha256_checksum(file_path).hexdigest() # Update instance file attributes & data has_changed = file_size != self.size or file_checksum != self.checksum diff --git a/netbox/extras/management/commands/nbshell.py b/netbox/extras/management/commands/nbshell.py index 07f943d15..04a67eb49 100644 --- a/netbox/extras/management/commands/nbshell.py +++ b/netbox/extras/management/commands/nbshell.py @@ -9,7 +9,7 @@ from django.contrib.auth.models import User from django.contrib.contenttypes.models import ContentType from django.core.management.base import BaseCommand -APPS = ('circuits', 'dcim', 'extras', 'ipam', 'tenancy', 'users', 'virtualization', 'wireless') +APPS = ('circuits', 'core', 'dcim', 'extras', 'ipam', 'tenancy', 'users', 'virtualization', 'wireless') BANNER_TEXT = """### NetBox interactive shell ({node}) ### Python {python} | Django {django} | NetBox {netbox}