Add UI control to sync data source

netbox/core/choices.py

@@ -24,13 +24,15 @@ class DataSourceTypeChoices(ChoiceSet):
 class DataSourceStatusChoices(ChoiceSet):
 
     NEW = 'new'
+    QUEUED = 'queued'
     SYNCING = 'syncing'
     COMPLETED = 'completed'
     FAILED = 'failed'
 
     CHOICES = (
-        (NEW, _('New')),
+        (NEW, _('New'), 'blue'),
-        (SYNCING, _('Syncing')),
+        (QUEUED, _('Queued'), 'orange'),
-        (COMPLETED, _('Completed')),
+        (SYNCING, _('Syncing'), 'cyan'),
-        (FAILED, _('Failed')),
+        (COMPLETED, _('Completed'), 'green'),
+        (FAILED, _('Failed'), 'red'),
     )

netbox/core/jobs.py

@@ -0,0 +1,27 @@
+import logging
+
+from django_rq import job
+
+from extras.choices import JobResultStatusChoices
+from .choices import *
+from .models import DataSource
+
+logger = logging.getLogger(__name__)
+
+
+@job('low')
+def sync_datasource(job_result, *args, **kwargs):
+    """
+    Call sync() on a DataSource.
+    """
+    datasource = DataSource.objects.get(name=job_result.name)
+
+    try:
+        job_result.start()
+        datasource.sync()
+    except Exception:
+        job_result.set_status(JobResultStatusChoices.STATUS_ERRORED)
+        job_result.save()
+        datasource.status = DataSourceStatusChoices.FAILED
+        datasource.save()
+        logging.error(f"Error during syncing of data source {datasource}")

netbox/core/models/data.py

@@ -85,10 +85,16 @@ class DataSource(ChangeLoggedModel):
     def get_absolute_url(self):
         return reverse('core:datasource', args=[self.pk])
 
+    def get_status_color(self):
+        return DataSourceStatusChoices.colors.get(self.status)
+
     def sync(self):
         """
         Create/update/delete child DataFiles as necessary to synchronize with the remote source.
         """
+        self.status = DataSourceStatusChoices.SYNCING
+        self.save()
+
         # Replicate source data locally (if needed)
         temp_dir = tempfile.TemporaryDirectory()
         self.fetch(path=temp_dir.name)
@@ -132,7 +138,8 @@ class DataSource(ChangeLoggedModel):
         created_count = len(DataFile.objects.bulk_create(new_datafiles, batch_size=100))
         logger.debug(f"Created {created_count} data files")
 
-        # Update last_synced time
+        # Update status & last_synced time
+        self.status = DataSourceStatusChoices.COMPLETED
         self.last_synced = timezone.now()
         self.save()
 

netbox/core/tables/data.py

@@ -14,6 +14,7 @@ class DataSourceTable(NetBoxTable):
         linkify=True
     )
     type = columns.ChoiceFieldColumn()
+    status = columns.ChoiceFieldColumn()
     enabled = columns.BooleanColumn()
     file_count = tables.Column(
         verbose_name='Files'
@@ -22,10 +23,10 @@ class DataSourceTable(NetBoxTable):
     class Meta(NetBoxTable.Meta):
         model = DataSource
         fields = (
-            'pk', 'id', 'name', 'type', 'enabled', 'url', 'description', 'git_branch', 'username', 'created',
+            'pk', 'id', 'name', 'type', 'status', 'enabled', 'url', 'description', 'git_branch', 'username', 'created',
             'last_updated', 'file_count',
         )
-        default_columns = ('pk', 'name', 'type', 'enabled', 'description', 'file_count')
+        default_columns = ('pk', 'name', 'type', 'status', 'enabled', 'description', 'file_count')
 
 
 class DataFileTable(NetBoxTable):

netbox/core/views.py

@@ -1,7 +1,14 @@
+from django.contrib import messages
+from django.contrib.contenttypes.models import ContentType
+from django.shortcuts import get_object_or_404, redirect
+
+from extras.models import JobResult
 from netbox.views import generic
+from netbox.views.generic.base import BaseObjectView
 from utilities.utils import count_related
 from utilities.views import register_model_view
-from . import filtersets, forms, tables
+from . import filtersets, forms, jobs, tables
+from .choices import *
 from .models import *
 
 
@@ -23,6 +30,37 @@ class DataSourceView(generic.ObjectView):
     queryset = DataSource.objects.all()
 
 
+@register_model_view(DataSource, 'sync')
+class DataSourceSyncView(BaseObjectView):
+    queryset = DataSource.objects.all()
+
+    def get_required_permission(self):
+        return 'core.sync_datasource'
+
+    def get(self, request, pk):
+        # Redirect GET requests to the object view
+        datasource = get_object_or_404(self.queryset, pk=pk)
+        return redirect(datasource.get_absolute_url())
+
+    def post(self, request, pk):
+        datasource = get_object_or_404(self.queryset, pk=pk)
+
+        # Set the status to "syncing"
+        datasource.status = DataSourceStatusChoices.QUEUED
+        datasource.save()
+
+        # Enqueue a sync job
+        job_result = JobResult.enqueue_job(
+            jobs.sync_datasource,
+            name=datasource.name,
+            obj_type=ContentType.objects.get_for_model(DataSource),
+            user=request.user,
+        )
+
+        messages.success(request, f"Queued job #{job_result.pk} to sync {datasource}")
+        return redirect(datasource.get_absolute_url())
+
+
 @register_model_view(DataSource, 'edit')
 class DataSourceEditView(generic.ObjectEditView):
     queryset = DataSource.objects.all()

netbox/templates/core/datasource.html

@@ -4,6 +4,23 @@
 {% load plugins %}
 {% load render_table from django_tables2 %}
 
+{% block extra_controls %}
+  {% if perms.core.sync_datasource %}
+    {% if object.enabled %}
+      <form action="{% url 'core:datasource_sync' pk=object.pk %}" method="post">
+        {% csrf_token %}
+        <button type="submit" class="btn btn-sm btn-primary">
+          <i class="mdi mdi-sync" aria-hidden="true"></i> Sync
+        </button>
+      </form>
+    {% else %}
+      <button class="btn btn-sm btn-primary" disabled>
+        <i class="mdi mdi-sync" aria-hidden="true"></i> Sync
+      </button>
+    {% endif %}
+  {% endif %}
+{% endblock %}
+
 {% block content %}
   <div class="row mb-3">
     <div class="col col-md-6">
@@ -19,14 +36,18 @@
               <th scope="row">Type</th>
               <td>{{ object.get_type_display }}</td>
             </tr>
-            <tr>
-              <th scope="row">Status</th>
-              <td>{{ object.get_status_display }}</td>
-            </tr>
             <tr>
               <th scope="row">Enabled</th>
               <td>{% checkmark object.enabled %}</td>
             </tr>
+            <tr>
+              <th scope="row">Status</th>
+              <td>{% badge object.get_status_display bg_color=object.get_status_color %}</td>
+            </tr>
+            <tr>
+              <th scope="row">Last synced</th>
+              <td>{{ object.last_synced|placeholder }}</td>
+            </tr>
             <tr>
               <th scope="row">Description</th>
               <td>{{ object.description|placeholder }}</td>
@@ -37,10 +58,6 @@
                 <a href="{{ object.url }}">{{ object.url }}</a>
               </td>
             </tr>
-            <tr>
-              <th scope="row">Last synced</th>
-              <td>{{ object.last_synced|placeholder }}</td>
-            </tr>
             <tr>
               <th scope="row">Git branch</th>
               <td>{{ object.git_branch|placeholder }}</td>