From 15ca5e23268948b5c5b604bf8f6542e6b7b40318 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Fri, 27 Jan 2023 09:57:20 -0500 Subject: [PATCH] Add UI control to sync data source --- netbox/core/choices.py | 10 ++++--- netbox/core/jobs.py | 27 ++++++++++++++++++ netbox/core/models/data.py | 9 +++++- netbox/core/tables/data.py | 5 ++-- netbox/core/views.py | 40 ++++++++++++++++++++++++++- netbox/templates/core/datasource.html | 33 ++++++++++++++++------ 6 files changed, 108 insertions(+), 16 deletions(-) create mode 100644 netbox/core/jobs.py diff --git a/netbox/core/choices.py b/netbox/core/choices.py index c369efe7c..5a2219595 100644 --- a/netbox/core/choices.py +++ b/netbox/core/choices.py @@ -24,13 +24,15 @@ class DataSourceTypeChoices(ChoiceSet): class DataSourceStatusChoices(ChoiceSet): NEW = 'new' + QUEUED = 'queued' SYNCING = 'syncing' COMPLETED = 'completed' FAILED = 'failed' CHOICES = ( - (NEW, _('New')), - (SYNCING, _('Syncing')), - (COMPLETED, _('Completed')), - (FAILED, _('Failed')), + (NEW, _('New'), 'blue'), + (QUEUED, _('Queued'), 'orange'), + (SYNCING, _('Syncing'), 'cyan'), + (COMPLETED, _('Completed'), 'green'), + (FAILED, _('Failed'), 'red'), ) diff --git a/netbox/core/jobs.py b/netbox/core/jobs.py new file mode 100644 index 000000000..6c055c02e --- /dev/null +++ b/netbox/core/jobs.py @@ -0,0 +1,27 @@ +import logging + +from django_rq import job + +from extras.choices import JobResultStatusChoices +from .choices import * +from .models import DataSource + +logger = logging.getLogger(__name__) + + +@job('low') +def sync_datasource(job_result, *args, **kwargs): + """ + Call sync() on a DataSource. + """ + datasource = DataSource.objects.get(name=job_result.name) + + try: + job_result.start() + datasource.sync() + except Exception: + job_result.set_status(JobResultStatusChoices.STATUS_ERRORED) + job_result.save() + datasource.status = DataSourceStatusChoices.FAILED + datasource.save() + logging.error(f"Error during syncing of data source {datasource}") diff --git a/netbox/core/models/data.py b/netbox/core/models/data.py index aa16d4765..e9dd44366 100644 --- a/netbox/core/models/data.py +++ b/netbox/core/models/data.py @@ -85,10 +85,16 @@ class DataSource(ChangeLoggedModel): def get_absolute_url(self): return reverse('core:datasource', args=[self.pk]) + def get_status_color(self): + return DataSourceStatusChoices.colors.get(self.status) + def sync(self): """ Create/update/delete child DataFiles as necessary to synchronize with the remote source. """ + self.status = DataSourceStatusChoices.SYNCING + self.save() + # Replicate source data locally (if needed) temp_dir = tempfile.TemporaryDirectory() self.fetch(path=temp_dir.name) @@ -132,7 +138,8 @@ class DataSource(ChangeLoggedModel): created_count = len(DataFile.objects.bulk_create(new_datafiles, batch_size=100)) logger.debug(f"Created {created_count} data files") - # Update last_synced time + # Update status & last_synced time + self.status = DataSourceStatusChoices.COMPLETED self.last_synced = timezone.now() self.save() diff --git a/netbox/core/tables/data.py b/netbox/core/tables/data.py index 73d8e204a..f31d6bf90 100644 --- a/netbox/core/tables/data.py +++ b/netbox/core/tables/data.py @@ -14,6 +14,7 @@ class DataSourceTable(NetBoxTable): linkify=True ) type = columns.ChoiceFieldColumn() + status = columns.ChoiceFieldColumn() enabled = columns.BooleanColumn() file_count = tables.Column( verbose_name='Files' @@ -22,10 +23,10 @@ class DataSourceTable(NetBoxTable): class Meta(NetBoxTable.Meta): model = DataSource fields = ( - 'pk', 'id', 'name', 'type', 'enabled', 'url', 'description', 'git_branch', 'username', 'created', + 'pk', 'id', 'name', 'type', 'status', 'enabled', 'url', 'description', 'git_branch', 'username', 'created', 'last_updated', 'file_count', ) - default_columns = ('pk', 'name', 'type', 'enabled', 'description', 'file_count') + default_columns = ('pk', 'name', 'type', 'status', 'enabled', 'description', 'file_count') class DataFileTable(NetBoxTable): diff --git a/netbox/core/views.py b/netbox/core/views.py index 98709f7c2..a7e25e4d1 100644 --- a/netbox/core/views.py +++ b/netbox/core/views.py @@ -1,7 +1,14 @@ +from django.contrib import messages +from django.contrib.contenttypes.models import ContentType +from django.shortcuts import get_object_or_404, redirect + +from extras.models import JobResult from netbox.views import generic +from netbox.views.generic.base import BaseObjectView from utilities.utils import count_related from utilities.views import register_model_view -from . import filtersets, forms, tables +from . import filtersets, forms, jobs, tables +from .choices import * from .models import * @@ -23,6 +30,37 @@ class DataSourceView(generic.ObjectView): queryset = DataSource.objects.all() +@register_model_view(DataSource, 'sync') +class DataSourceSyncView(BaseObjectView): + queryset = DataSource.objects.all() + + def get_required_permission(self): + return 'core.sync_datasource' + + def get(self, request, pk): + # Redirect GET requests to the object view + datasource = get_object_or_404(self.queryset, pk=pk) + return redirect(datasource.get_absolute_url()) + + def post(self, request, pk): + datasource = get_object_or_404(self.queryset, pk=pk) + + # Set the status to "syncing" + datasource.status = DataSourceStatusChoices.QUEUED + datasource.save() + + # Enqueue a sync job + job_result = JobResult.enqueue_job( + jobs.sync_datasource, + name=datasource.name, + obj_type=ContentType.objects.get_for_model(DataSource), + user=request.user, + ) + + messages.success(request, f"Queued job #{job_result.pk} to sync {datasource}") + return redirect(datasource.get_absolute_url()) + + @register_model_view(DataSource, 'edit') class DataSourceEditView(generic.ObjectEditView): queryset = DataSource.objects.all() diff --git a/netbox/templates/core/datasource.html b/netbox/templates/core/datasource.html index 797519da6..38a5311cb 100644 --- a/netbox/templates/core/datasource.html +++ b/netbox/templates/core/datasource.html @@ -4,6 +4,23 @@ {% load plugins %} {% load render_table from django_tables2 %} +{% block extra_controls %} + {% if perms.core.sync_datasource %} + {% if object.enabled %} +
+ {% csrf_token %} + +
+ {% else %} + + {% endif %} + {% endif %} +{% endblock %} + {% block content %}
@@ -19,14 +36,18 @@ Type {{ object.get_type_display }} - - Status - {{ object.get_status_display }} - Enabled {% checkmark object.enabled %} + + Status + {% badge object.get_status_display bg_color=object.get_status_color %} + + + Last synced + {{ object.last_synced|placeholder }} + Description {{ object.description|placeholder }} @@ -37,10 +58,6 @@ {{ object.url }} - - Last synced - {{ object.last_synced|placeholder }} - Git branch {{ object.git_branch|placeholder }}