NeoAnd/netbox
1
0
Fork 0
mirror of https://github.com/netbox-community/netbox.git synced 2025-07-14 01:41:22 -06:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

Closes #18287: Enable periodic synchronization for data sources (#18747)

Browse Source 
* Add sync_interval to DataSource

* Enqueue a SyncDataSourceJob when needed after saving a DataSource

* Fix logic for clearing pending jobs on interval change

* Fix lingering background tasks after modifying DataSource
This commit is contained in:
Jeremy Stretch 2025-03-03 09:29:40 -05:00 committed by GitHub
parent cf7e2c8dc9
commit 77b9820577
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 98 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
docs/models/core/datasource.md
View File

@ -44,6 +44,12 @@ A set of rules (one per line) identifying filenames to ignore during synchroniza
| `*.txt` | Ignore any files with a `.txt` extension |
| `data???.json` | Ignore e.g. `data123.json` |
### Sync Interval
!!! info "This field was introduced in NetBox v4.3."
The interval at which the data source should automatically synchronize. If not set, the data source must be synchronized manually.
### Last Synced
The date and time at which the source was most recently synchronized successfully.

4
netbox/core/api/serializers_/data.py
View File

@ -26,8 +26,8 @@ class DataSourceSerializer(NetBoxModelSerializer):
model = DataSource
fields = [
'id', 'url', 'display_url', 'display', 'name', 'type', 'source_url', 'enabled', 'status', 'description',
'parameters', 'ignore_rules', 'comments', 'custom_fields', 'created', 'last_updated', 'last_synced',
'file_count',
'sync_interval', 'parameters', 'ignore_rules', 'comments', 'custom_fields', 'created', 'last_updated',
'last_synced', 'file_count',
]
brief_fields = ('id', 'url', 'display', 'name', 'description')

4
netbox/core/filtersets.py
View File

@ -29,6 +29,10 @@ class DataSourceFilterSet(NetBoxModelFilterSet):
choices=DataSourceStatusChoices,
null_value=None
)
sync_interval = django_filters.MultipleChoiceFilter(
choices=JobIntervalChoices,
null_value=None
)
class Meta:
model = DataSource

10
netbox/core/forms/bulk_edit.py
View File

@ -1,6 +1,7 @@
from django import forms
from django.utils.translation import gettext_lazy as _
from core.choices import JobIntervalChoices
from core.models import *
from netbox.forms import NetBoxModelBulkEditForm
from netbox.utils import get_data_backend_choices
@ -29,6 +30,11 @@ class DataSourceBulkEditForm(NetBoxModelBulkEditForm):
max_length=200,
required=False
)
sync_interval = forms.ChoiceField(
choices=JobIntervalChoices,
required=False,
label=_('Sync interval')
)
comments = CommentField()
parameters = forms.JSONField(
label=_('Parameters'),
@ -42,8 +48,8 @@ class DataSourceBulkEditForm(NetBoxModelBulkEditForm):
model = DataSource
fieldsets = (
FieldSet('type', 'enabled', 'description', 'comments', 'parameters', 'ignore_rules'),
FieldSet('type', 'enabled', 'description', 'sync_interval', 'parameters', 'ignore_rules', 'comments'),
)
nullable_fields = (
'description', 'description', 'parameters', 'comments', 'parameters', 'ignore_rules',
'description', 'description', 'sync_interval', 'parameters', 'parameters', 'ignore_rules' 'comments',
)

3
netbox/core/forms/bulk_import.py
View File

@ -11,5 +11,6 @@ class DataSourceImportForm(NetBoxModelImportForm):
class Meta:
model = DataSource
fields = (
'name', 'type', 'source_url', 'enabled', 'description', 'comments', 'parameters', 'ignore_rules',
'name', 'type', 'source_url', 'enabled', 'description', 'sync_interval', 'parameters', 'ignore_rules',
'comments',
)

7
netbox/core/forms/filtersets.py
View File

@ -27,7 +27,7 @@ class DataSourceFilterForm(NetBoxModelFilterSetForm):
model = DataSource
fieldsets = (
FieldSet('q', 'filter_id'),
FieldSet('type', 'status', name=_('Data Source')),
FieldSet('type', 'status', 'enabled', 'sync_interval', name=_('Data Source')),
)
type = forms.MultipleChoiceField(
label=_('Type'),
@ -46,6 +46,11 @@ class DataSourceFilterForm(NetBoxModelFilterSetForm):
choices=BOOLEAN_WITH_BLANK_CHOICES
)
)
sync_interval = forms.ChoiceField(
label=_('Sync interval'),
choices=JobIntervalChoices,
required=False
)
class DataFileFilterForm(NetBoxModelFilterSetForm):

7
netbox/core/forms/model_forms.py
View File

@ -36,7 +36,7 @@ class DataSourceForm(NetBoxModelForm):
class Meta:
model = DataSource
fields = [
'name', 'type', 'source_url', 'enabled', 'description', 'comments', 'ignore_rules', 'tags',
'name', 'type', 'source_url', 'enabled', 'description', 'sync_interval', 'ignore_rules', 'comments', 'tags',
]
widgets = {
'ignore_rules': forms.Textarea(
@ -51,7 +51,10 @@ class DataSourceForm(NetBoxModelForm):
@property
def fieldsets(self):
fieldsets = [
FieldSet('name', 'type', 'source_url', 'enabled', 'description', 'tags', 'ignore_rules', name=_('Source')),
FieldSet(
'name', 'type', 'source_url', 'description', 'tags', 'ignore_rules', name=_('Source')
),
FieldSet('enabled', 'sync_interval', name=_('Sync')),
]
if self.backend_fields:
fieldsets.append(

18
netbox/core/migrations/0013_datasource_sync_interval.py Normal file
View File

@ -0,0 +1,18 @@
# Generated by Django 5.1.6 on 2025-02-26 19:45
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0012_job_object_type_optional'),
]
operations = [
migrations.AddField(
model_name='datasource',
name='sync_interval',
field=models.PositiveSmallIntegerField(blank=True, null=True),
),
]

6
netbox/core/models/data.py
View File

@ -59,6 +59,12 @@ class DataSource(JobsMixin, PrimaryModel):
verbose_name=_('enabled'),
default=True
)
sync_interval = models.PositiveSmallIntegerField(
verbose_name=_('sync interval'),
choices=JobIntervalChoices,
blank=True,
null=True
)
ignore_rules = models.TextField(
verbose_name=_('ignore rules'),
blank=True,

24
netbox/core/signals.py
View File

@ -8,16 +8,15 @@ from django.dispatch import receiver, Signal
from django.utils.translation import gettext_lazy as _
from django_prometheus.models import model_deletes, model_inserts, model_updates
from core.choices import ObjectChangeActionChoices
from core.choices import JobStatusChoices, ObjectChangeActionChoices
from core.events import *
from core.models import ObjectChange
from extras.events import enqueue_event
from extras.utils import run_validators
from netbox.config import get_config
from netbox.context import current_request, events_queue
from netbox.models.features import ChangeLoggingMixin
from utilities.exceptions import AbortRequest
from .models import ConfigRevision
from .models import ConfigRevision, DataSource, ObjectChange
__all__ = (
'clear_events',
@ -182,6 +181,25 @@ def clear_events_queue(sender, **kwargs):
# DataSource handlers
#
@receiver(post_save, sender=DataSource)
def enqueue_sync_job(instance, created, **kwargs):
"""
When a DataSource is saved, check its sync_interval and enqueue a sync job if appropriate.
"""
from .jobs import SyncDataSourceJob
if instance.enabled and instance.sync_interval:
SyncDataSourceJob.enqueue_once(instance, interval=instance.sync_interval)
elif not created:
# Delete any previously scheduled recurring jobs for this DataSource
for job in SyncDataSourceJob.get_jobs(instance).defer('data').filter(
interval__isnull=False,
status=JobStatusChoices.STATUS_SCHEDULED
):
# Call delete() per instance to ensure the associated background task is deleted as well
job.delete()
@receiver(post_sync)
def auto_sync(instance, **kwargs):
"""

9
netbox/core/tables/data.py
View File

@ -25,6 +25,9 @@ class DataSourceTable(NetBoxTable):
enabled = columns.BooleanColumn(
verbose_name=_('Enabled'),
)
sync_interval = columns.ChoiceFieldColumn(
verbose_name=_('Sync interval'),
)
tags = columns.TagColumn(
url_name='core:datasource_list'
)
@ -35,10 +38,10 @@ class DataSourceTable(NetBoxTable):
class Meta(NetBoxTable.Meta):
model = DataSource
fields = (
'pk', 'id', 'name', 'type', 'status', 'enabled', 'source_url', 'description', 'comments', 'parameters',
'created', 'last_updated', 'file_count',
'pk', 'id', 'name', 'type', 'status', 'enabled', 'source_url', 'description', 'sync_interval', 'comments',
'parameters', 'created', 'last_updated', 'file_count',
)
default_columns = ('pk', 'name', 'type', 'status', 'enabled', 'description', 'file_count')
default_columns = ('pk', 'name', 'type', 'status', 'enabled', 'description', 'sync_interval', 'file_count')
class DataFileTable(NetBoxTable):

13
netbox/core/tests/test_filtersets.py
View File

@ -27,7 +27,8 @@ class DataSourceTestCase(TestCase, ChangeLoggedFilterSetTests):
source_url='file:///var/tmp/source1/',
status=DataSourceStatusChoices.NEW,
enabled=True,
description='foobar1'
description='foobar1',
sync_interval=JobIntervalChoices.INTERVAL_HOURLY
),
DataSource(
name='Data Source 2',
@ -35,14 +36,16 @@ class DataSourceTestCase(TestCase, ChangeLoggedFilterSetTests):
source_url='file:///var/tmp/source2/',
status=DataSourceStatusChoices.SYNCING,
enabled=True,
description='foobar2'
description='foobar2',
sync_interval=JobIntervalChoices.INTERVAL_DAILY
),
DataSource(
name='Data Source 3',
type='git',
source_url='https://example.com/git/source3',
status=DataSourceStatusChoices.COMPLETED,
enabled=False
enabled=False,
sync_interval=JobIntervalChoices.INTERVAL_WEEKLY
),
)
DataSource.objects.bulk_create(data_sources)
@ -73,6 +76,10 @@ class DataSourceTestCase(TestCase, ChangeLoggedFilterSetTests):
params = {'status': [DataSourceStatusChoices.NEW, DataSourceStatusChoices.SYNCING]}
self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2)
def test_sync_interval(self):
params = {'sync_interval': [JobIntervalChoices.INTERVAL_HOURLY, JobIntervalChoices.INTERVAL_DAILY]}
self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2)
class DataFileTestCase(TestCase, ChangeLoggedFilterSetTests):
queryset = DataFile.objects.all()

4
netbox/templates/core/datasource.html
View File

@ -46,6 +46,10 @@
<th scope="row">{% trans "Status" %}</th>
<td>{% badge object.get_status_display bg_color=object.get_status_color %}</td>
</tr>
<tr>
<th scope="row">{% trans "Sync interval" %}</th>
<td>{{ object.get_sync_interval_display|placeholder }}</td>
</tr>
<tr>
<th scope="row">{% trans "Last synced" %}</th>
<td>{{ object.last_synced|placeholder }}</td>