WIP

netbox/core/admin/__init__.py

@@ -0,0 +1,18 @@
+from django.contrib import admin
+
+from core.models import DataFile, DataSource
+
+
+@admin.register(DataSource)
+class DataSourceAdmin(admin.ModelAdmin):
+    list_display = ('name', 'file_count')
+
+    @staticmethod
+    def file_count(obj):
+        return obj.datafiles.count()
+
+
+@admin.register(DataFile)
+class DataFileAdmin(admin.ModelAdmin):
+    list_display = ('path', 'size')
+    readonly_fields = ('source', 'path', 'last_updated', 'size', 'checksum')

netbox/core/migrations/0001_initial.py

@@ -1,9 +1,7 @@
-# Generated by Django 4.1.5 on 2023-01-22 20:49
+# Generated by Django 4.1.5 on 2023-01-24 18:56
 
 from django.db import migrations, models
 import django.db.models.deletion
-import taggit.managers
-import utilities.json
 
 
 class Migration(migrations.Migration):
@@ -11,7 +9,6 @@ class Migration(migrations.Migration):
     initial = True
 
     dependencies = [
-        ('extras', '0084_staging'),
     ]
 
     operations = [
@@ -19,16 +16,12 @@ class Migration(migrations.Migration):
             name='DataSource',
             fields=[
                 ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False)),
-                ('created', models.DateTimeField(auto_now_add=True, null=True)),
-                ('last_updated', models.DateTimeField(auto_now=True, null=True)),
-                ('custom_field_data', models.JSONField(blank=True, default=dict, encoder=utilities.json.CustomFieldJSONEncoder)),
-                ('comments', models.TextField(blank=True)),
                 ('name', models.CharField(max_length=100, unique=True)),
                 ('type', models.CharField(default='local', max_length=50)),
                 ('enabled', models.BooleanField(default=True)),
                 ('description', models.CharField(blank=True, max_length=200)),
-                ('url', models.URLField()),
+                ('url', models.CharField(max_length=200)),
-                ('tags', taggit.managers.TaggableManager(through='extras.TaggedItem', to='extras.Tag')),
+                ('ignore_rules', models.TextField(blank=True)),
             ],
             options={
                 'ordering': ('name',),
@@ -38,15 +31,12 @@ class Migration(migrations.Migration):
             name='DataFile',
             fields=[
                 ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False)),
-                ('created', models.DateTimeField(auto_now_add=True, null=True)),
+                ('path', models.CharField(editable=False, max_length=1000, unique=True)),
-                ('custom_field_data', models.JSONField(blank=True, default=dict, encoder=utilities.json.CustomFieldJSONEncoder)),
+                ('last_updated', models.DateTimeField(editable=False)),
-                ('path', models.CharField(max_length=1000, unique=True)),
+                ('size', models.PositiveIntegerField(editable=False)),
-                ('last_updated', models.DateTimeField()),
+                ('checksum', models.CharField(editable=False, max_length=64)),
-                ('size', models.PositiveIntegerField()),
-                ('checksum', models.CharField(max_length=64)),
                 ('data', models.BinaryField()),
-                ('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='datafiles', to='core.datasource')),
+                ('source', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='datafiles', to='core.datasource')),
-                ('tags', taggit.managers.TaggableManager(through='extras.TaggedItem', to='extras.Tag')),
             ],
             options={
                 'ordering': ('source', 'path'),

netbox/core/models.py

@@ -1,6 +1,7 @@
 import logging
 import os
 from functools import cached_property
+from fnmatch import fnmatchcase
 from urllib.parse import urlparse
 
 from django.conf import settings
@@ -9,7 +10,6 @@ from django.urls import reverse
 from django.utils import timezone
 from django.utils.translation import gettext as _
 
-from netbox.models import NetBoxModel, PrimaryModel
 from utilities.files import sha256_checksum
 from .choices import *
 
@@ -21,7 +21,7 @@ __all__ = (
 logger = logging.getLogger('netbox.core.data')
 
 
-class DataSource(PrimaryModel):
+class DataSource(models.Model):
     """
     A remote source from which DataFiles are synchronized.
     """
@@ -41,24 +41,24 @@ class DataSource(PrimaryModel):
         max_length=200,
         blank=True
     )
-    url = models.URLField(
+    url = models.CharField(
-        verbose_name='URL'
+        max_length=200,
+        verbose_name=_('URL')
+    )
+    ignore_rules = models.TextField(
+        blank=True,
+        help_text=_("Patterns (one per line) matching files to ignore when syncing")
     )
 
     class Meta:
         ordering = ('name',)
 
+    def __str__(self):
+        return self.name
+
     # def get_absolute_url(self):
     #     return reverse('core:datasource', args=[self.pk])
 
-    # @property
-    # def root_path(self):
-    #     if self.pk is None:
-    #         return None
-    #     if self.type == DataSourceTypeChoices.LOCAL:
-    #         return self.url.lstrip('file://')
-    #     return os.path.join(DATASOURCES_CACHE_PATH, str(self.pk))
-
     def sync(self):
         """
         Create/update/delete child DataFiles as necessary to synchronize with the remote source.
@@ -131,33 +131,50 @@ class DataSource(PrimaryModel):
             if path.startswith('.'):
                 continue
             for file_name in file_names:
-                # TODO: Apply include/exclude rules
+                if not self._ignore(file_name):
-                if file_name.startswith('.'):
+                    paths.add(os.path.join(path, file_name))
-                    continue
-                paths.add(os.path.join(path, file_name))
 
         logger.debug(f"Found {len(paths)} files")
         return paths
 
+    def _ignore(self, filename):
+        """
+        Returns a boolean indicating whether the file should be ignored per the DataSource's configured
+        ignore rules.
+        """
+        if filename.startswith('.'):
+            return True
+        for rule in self.ignore_rules.splitlines():
+            if fnmatchcase(filename, rule):
+                return True
+        return False
 
-class DataFile(NetBoxModel):
+
+class DataFile(models.Model):
     """
     A database object which represents a remote file fetched from a DataSource.
     """
     source = models.ForeignKey(
         to='core.DataSource',
         on_delete=models.CASCADE,
-        related_name='datafiles'
+        related_name='datafiles',
+        editable=False
     )
     path = models.CharField(
         max_length=1000,
-        unique=True
+        unique=True,
+        editable=False
+    )
+    last_updated = models.DateTimeField(
+        editable=False
+    )
+    size = models.PositiveIntegerField(
+        editable=False
     )
-    last_updated = models.DateTimeField()
-    size = models.PositiveIntegerField()
     # TODO: Create a proper SHA256 field
     checksum = models.CharField(
-        max_length=64
+        max_length=64,
+        editable=False
     )
     data = models.BinaryField()
 
@@ -170,6 +187,9 @@ class DataFile(NetBoxModel):
             ),
         )
 
+    def __str__(self):
+        return self.path
+
     # def get_absolute_url(self):
     #     return reverse('core:datafile', args=[self.pk])
 
@@ -182,7 +202,7 @@ class DataFile(NetBoxModel):
 
         # Get attributes from file on disk
         file_size = os.path.getsize(file_path)
-        file_checksum = sha256_checksum(file_path)
+        file_checksum = sha256_checksum(file_path).hexdigest()
 
         # Update instance file attributes & data
         has_changed = file_size != self.size or file_checksum != self.checksum

netbox/extras/management/commands/nbshell.py

@@ -9,7 +9,7 @@ from django.contrib.auth.models import User
 from django.contrib.contenttypes.models import ContentType
 from django.core.management.base import BaseCommand
 
-APPS = ('circuits', 'dcim', 'extras', 'ipam', 'tenancy', 'users', 'virtualization', 'wireless')
+APPS = ('circuits', 'core', 'dcim', 'extras', 'ipam', 'tenancy', 'users', 'virtualization', 'wireless')
 
 BANNER_TEXT = """### NetBox interactive shell ({node})
 ### Python {python} | Django {django} | NetBox {netbox}