Closes #18287: Enable periodic synchronization for data sources (#18747)

* Add sync_interval to DataSource

* Enqueue a SyncDataSourceJob when needed after saving a DataSource

* Fix logic for clearing pending jobs on interval change

* Fix lingering background tasks after modifying DataSource
This commit is contained in:
Jeremy Stretch
2025-03-03 09:29:40 -05:00
committed by GitHub
parent cf7e2c8dc9
commit 77b9820577
13 changed files with 98 additions and 17 deletions

View File

@@ -27,7 +27,8 @@ class DataSourceTestCase(TestCase, ChangeLoggedFilterSetTests):
source_url='file:///var/tmp/source1/',
status=DataSourceStatusChoices.NEW,
enabled=True,
description='foobar1'
description='foobar1',
sync_interval=JobIntervalChoices.INTERVAL_HOURLY
),
DataSource(
name='Data Source 2',
@@ -35,14 +36,16 @@ class DataSourceTestCase(TestCase, ChangeLoggedFilterSetTests):
source_url='file:///var/tmp/source2/',
status=DataSourceStatusChoices.SYNCING,
enabled=True,
description='foobar2'
description='foobar2',
sync_interval=JobIntervalChoices.INTERVAL_DAILY
),
DataSource(
name='Data Source 3',
type='git',
source_url='https://example.com/git/source3',
status=DataSourceStatusChoices.COMPLETED,
enabled=False
enabled=False,
sync_interval=JobIntervalChoices.INTERVAL_WEEKLY
),
)
DataSource.objects.bulk_create(data_sources)
@@ -73,6 +76,10 @@ class DataSourceTestCase(TestCase, ChangeLoggedFilterSetTests):
params = {'status': [DataSourceStatusChoices.NEW, DataSourceStatusChoices.SYNCING]}
self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2)
def test_sync_interval(self):
params = {'sync_interval': [JobIntervalChoices.INTERVAL_HOURLY, JobIntervalChoices.INTERVAL_DAILY]}
self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2)
class DataFileTestCase(TestCase, ChangeLoggedFilterSetTests):
queryset = DataFile.objects.all()