From 8e6987edbff65864b718d74368c7470f126ee0a0 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 1 Aug 2024 12:58:11 -0400 Subject: [PATCH] #16388: Move change logging signal handlers into core --- netbox/core/models/data.py | 3 +- netbox/core/signals.py | 167 +++++++++++++++++- netbox/extras/jobs.py | 2 +- netbox/extras/signals.py | 182 +------------------- netbox/extras/utils.py | 36 ++++ netbox/netbox/views/generic/bulk_views.py | 4 +- netbox/netbox/views/generic/object_views.py | 2 +- 7 files changed, 210 insertions(+), 186 deletions(-) diff --git a/netbox/core/models/data.py b/netbox/core/models/data.py index a8e90ec3f..97f9fdac7 100644 --- a/netbox/core/models/data.py +++ b/netbox/core/models/data.py @@ -21,7 +21,6 @@ from netbox.registry import registry from utilities.querysets import RestrictedQuerySet from ..choices import * from ..exceptions import SyncError -from ..signals import post_sync, pre_sync __all__ = ( 'AutoSyncRecord', @@ -159,6 +158,8 @@ class DataSource(JobsMixin, PrimaryModel): """ Create/update/delete child DataFiles as necessary to synchronize with the remote source. """ + from core.signals import post_sync, pre_sync + if self.status == DataSourceStatusChoices.SYNCING: raise SyncError(_("Cannot initiate sync; syncing already in progress.")) diff --git a/netbox/core/signals.py b/netbox/core/signals.py index f884a27b4..06432bf4c 100644 --- a/netbox/core/signals.py +++ b/netbox/core/signals.py @@ -1,9 +1,26 @@ -from django.db.models.signals import post_save -from django.dispatch import Signal, receiver +import logging +from django.contrib.contenttypes.models import ContentType +from django.core.exceptions import ValidationError +from django.db.models.fields.reverse_related import ManyToManyRel +from django.db.models.signals import m2m_changed, post_save, pre_delete +from django.dispatch import receiver, Signal +from django.utils.translation import gettext_lazy as _ +from django_prometheus.models import model_deletes, model_inserts, model_updates + +from core.choices import ObjectChangeActionChoices +from core.events import * +from core.models import ObjectChange +from extras.events import enqueue_event +from extras.utils import run_validators +from netbox.config import get_config +from netbox.context import current_request, events_queue +from netbox.models.features import ChangeLoggingMixin +from utilities.exceptions import AbortRequest from .models import ConfigRevision __all__ = ( + 'clear_events', 'job_end', 'job_start', 'post_sync', @@ -18,6 +35,152 @@ job_end = Signal() pre_sync = Signal() post_sync = Signal() +# Event signals +clear_events = Signal() + + +# +# Change logging & event handling +# + +@receiver((post_save, m2m_changed)) +def handle_changed_object(sender, instance, **kwargs): + """ + Fires when an object is created or updated. + """ + m2m_changed = False + + if not hasattr(instance, 'to_objectchange'): + return + + # Get the current request, or bail if not set + request = current_request.get() + if request is None: + return + + # Determine the type of change being made + if kwargs.get('created'): + event_type = OBJECT_CREATED + elif 'created' in kwargs: + event_type = OBJECT_UPDATED + elif kwargs.get('action') in ['post_add', 'post_remove'] and kwargs['pk_set']: + # m2m_changed with objects added or removed + m2m_changed = True + event_type = OBJECT_UPDATED + else: + return + + # Create/update an ObjectChange record for this change + action = { + OBJECT_CREATED: ObjectChangeActionChoices.ACTION_CREATE, + OBJECT_UPDATED: ObjectChangeActionChoices.ACTION_UPDATE, + OBJECT_DELETED: ObjectChangeActionChoices.ACTION_DELETE, + }[event_type] + objectchange = instance.to_objectchange(action) + # If this is a many-to-many field change, check for a previous ObjectChange instance recorded + # for this object by this request and update it + if m2m_changed and ( + prev_change := ObjectChange.objects.filter( + changed_object_type=ContentType.objects.get_for_model(instance), + changed_object_id=instance.pk, + request_id=request.id + ).first() + ): + prev_change.postchange_data = objectchange.postchange_data + prev_change.save() + elif objectchange and objectchange.has_changes: + objectchange.user = request.user + objectchange.request_id = request.id + objectchange.save() + + # Ensure that we're working with fresh M2M assignments + if m2m_changed: + instance.refresh_from_db() + + # Enqueue the object for event processing + queue = events_queue.get() + enqueue_event(queue, instance, request.user, request.id, event_type) + events_queue.set(queue) + + # Increment metric counters + if event_type == OBJECT_CREATED: + model_inserts.labels(instance._meta.model_name).inc() + elif event_type == OBJECT_UPDATED: + model_updates.labels(instance._meta.model_name).inc() + + +@receiver(pre_delete) +def handle_deleted_object(sender, instance, **kwargs): + """ + Fires when an object is deleted. + """ + # Run any deletion protection rules for the object. Note that this must occur prior + # to queueing any events for the object being deleted, in case a validation error is + # raised, causing the deletion to fail. + model_name = f'{sender._meta.app_label}.{sender._meta.model_name}' + validators = get_config().PROTECTION_RULES.get(model_name, []) + try: + run_validators(instance, validators) + except ValidationError as e: + raise AbortRequest( + _("Deletion is prevented by a protection rule: {message}").format(message=e) + ) + + # Get the current request, or bail if not set + request = current_request.get() + if request is None: + return + + # Record an ObjectChange if applicable + if hasattr(instance, 'to_objectchange'): + if hasattr(instance, 'snapshot') and not getattr(instance, '_prechange_snapshot', None): + instance.snapshot() + objectchange = instance.to_objectchange(ObjectChangeActionChoices.ACTION_DELETE) + objectchange.user = request.user + objectchange.request_id = request.id + objectchange.save() + + # Django does not automatically send an m2m_changed signal for the reverse direction of a + # many-to-many relationship (see https://code.djangoproject.com/ticket/17688), so we need to + # trigger one manually. We do this by checking for any reverse M2M relationships on the + # instance being deleted, and explicitly call .remove() on the remote M2M field to delete + # the association. This triggers an m2m_changed signal with the `post_remove` action type + # for the forward direction of the relationship, ensuring that the change is recorded. + for relation in instance._meta.related_objects: + if type(relation) is not ManyToManyRel: + continue + related_model = relation.related_model + related_field_name = relation.remote_field.name + if not issubclass(related_model, ChangeLoggingMixin): + # We only care about triggering the m2m_changed signal for models which support + # change logging + continue + for obj in related_model.objects.filter(**{related_field_name: instance.pk}): + obj.snapshot() # Ensure the change record includes the "before" state + getattr(obj, related_field_name).remove(instance) + + # Enqueue the object for event processing + queue = events_queue.get() + enqueue_event(queue, instance, request.user, request.id, OBJECT_DELETED) + events_queue.set(queue) + + # Increment metric counters + model_deletes.labels(instance._meta.model_name).inc() + + +@receiver(clear_events) +def clear_events_queue(sender, **kwargs): + """ + Delete any queued events (e.g. because of an aborted bulk transaction) + """ + logger = logging.getLogger('events') + logger.info(f"Clearing {len(events_queue.get())} queued events ({sender})") + events_queue.set({}) + + +# +# DataSource handlers +# @receiver(post_sync) def auto_sync(instance, **kwargs): diff --git a/netbox/extras/jobs.py b/netbox/extras/jobs.py index 62f8f6959..2529e9d2b 100644 --- a/netbox/extras/jobs.py +++ b/netbox/extras/jobs.py @@ -5,8 +5,8 @@ from contextlib import nullcontext from django.db import transaction from django.utils.translation import gettext as _ +from core.signals import clear_events from extras.models import Script as ScriptModel -from extras.signals import clear_events from netbox.context_managers import event_tracking from utilities.exceptions import AbortScript, AbortTransaction from utilities.jobs import JobRunner diff --git a/netbox/extras/signals.py b/netbox/extras/signals.py index b9e4726bf..eae9c02a0 100644 --- a/netbox/extras/signals.py +++ b/netbox/extras/signals.py @@ -1,194 +1,18 @@ -import importlib -import logging - from django.contrib.contenttypes.models import ContentType -from django.core.exceptions import ImproperlyConfigured, ValidationError -from django.db.models.fields.reverse_related import ManyToManyRel from django.db.models.signals import m2m_changed, post_save, pre_delete -from django.dispatch import receiver, Signal -from django.utils.translation import gettext_lazy as _ -from django_prometheus.models import model_deletes, model_inserts, model_updates +from django.dispatch import receiver -from core.choices import ObjectChangeActionChoices from core.events import * -from core.models import ObjectChange, ObjectType +from core.models import ObjectType from core.signals import job_end, job_start from extras.events import process_event_rules from extras.models import EventRule, Notification, Subscription from netbox.config import get_config -from netbox.context import current_request, events_queue -from netbox.models.features import ChangeLoggingMixin from netbox.registry import registry from netbox.signals import post_clean from utilities.exceptions import AbortRequest -from .events import enqueue_event from .models import CustomField, TaggedItem -from .validators import CustomValidator - - -def run_validators(instance, validators): - """ - Run the provided iterable of validators for the instance. - """ - request = current_request.get() - for validator in validators: - - # Loading a validator class by dotted path - if type(validator) is str: - module, cls = validator.rsplit('.', 1) - validator = getattr(importlib.import_module(module), cls)() - - # Constructing a new instance on the fly from a ruleset - elif type(validator) is dict: - validator = CustomValidator(validator) - - elif not issubclass(validator.__class__, CustomValidator): - raise ImproperlyConfigured(f"Invalid value for custom validator: {validator}") - - validator(instance, request) - - -# -# Change logging/webhooks -# - -# Define a custom signal that can be sent to clear any queued events -clear_events = Signal() - - -@receiver((post_save, m2m_changed)) -def handle_changed_object(sender, instance, **kwargs): - """ - Fires when an object is created or updated. - """ - m2m_changed = False - - if not hasattr(instance, 'to_objectchange'): - return - - # Get the current request, or bail if not set - request = current_request.get() - if request is None: - return - - # Determine the type of change being made - if kwargs.get('created'): - event_type = OBJECT_CREATED - elif 'created' in kwargs: - event_type = OBJECT_UPDATED - elif kwargs.get('action') in ['post_add', 'post_remove'] and kwargs['pk_set']: - # m2m_changed with objects added or removed - m2m_changed = True - event_type = OBJECT_UPDATED - else: - return - - # Create/update an ObjectChange record for this change - action = { - OBJECT_CREATED: ObjectChangeActionChoices.ACTION_CREATE, - OBJECT_UPDATED: ObjectChangeActionChoices.ACTION_UPDATE, - OBJECT_DELETED: ObjectChangeActionChoices.ACTION_DELETE, - }[event_type] - objectchange = instance.to_objectchange(action) - # If this is a many-to-many field change, check for a previous ObjectChange instance recorded - # for this object by this request and update it - if m2m_changed and ( - prev_change := ObjectChange.objects.filter( - changed_object_type=ContentType.objects.get_for_model(instance), - changed_object_id=instance.pk, - request_id=request.id - ).first() - ): - prev_change.postchange_data = objectchange.postchange_data - prev_change.save() - elif objectchange and objectchange.has_changes: - objectchange.user = request.user - objectchange.request_id = request.id - objectchange.save() - - # Ensure that we're working with fresh M2M assignments - if m2m_changed: - instance.refresh_from_db() - - # Enqueue the object for event processing - queue = events_queue.get() - enqueue_event(queue, instance, request.user, request.id, event_type) - events_queue.set(queue) - - # Increment metric counters - if event_type == OBJECT_CREATED: - model_inserts.labels(instance._meta.model_name).inc() - elif event_type == OBJECT_UPDATED: - model_updates.labels(instance._meta.model_name).inc() - - -@receiver(pre_delete) -def handle_deleted_object(sender, instance, **kwargs): - """ - Fires when an object is deleted. - """ - # Run any deletion protection rules for the object. Note that this must occur prior - # to queueing any events for the object being deleted, in case a validation error is - # raised, causing the deletion to fail. - model_name = f'{sender._meta.app_label}.{sender._meta.model_name}' - validators = get_config().PROTECTION_RULES.get(model_name, []) - try: - run_validators(instance, validators) - except ValidationError as e: - raise AbortRequest( - _("Deletion is prevented by a protection rule: {message}").format(message=e) - ) - - # Get the current request, or bail if not set - request = current_request.get() - if request is None: - return - - # Record an ObjectChange if applicable - if hasattr(instance, 'to_objectchange'): - if hasattr(instance, 'snapshot') and not getattr(instance, '_prechange_snapshot', None): - instance.snapshot() - objectchange = instance.to_objectchange(ObjectChangeActionChoices.ACTION_DELETE) - objectchange.user = request.user - objectchange.request_id = request.id - objectchange.save() - - # Django does not automatically send an m2m_changed signal for the reverse direction of a - # many-to-many relationship (see https://code.djangoproject.com/ticket/17688), so we need to - # trigger one manually. We do this by checking for any reverse M2M relationships on the - # instance being deleted, and explicitly call .remove() on the remote M2M field to delete - # the association. This triggers an m2m_changed signal with the `post_remove` action type - # for the forward direction of the relationship, ensuring that the change is recorded. - for relation in instance._meta.related_objects: - if type(relation) is not ManyToManyRel: - continue - related_model = relation.related_model - related_field_name = relation.remote_field.name - if not issubclass(related_model, ChangeLoggingMixin): - # We only care about triggering the m2m_changed signal for models which support - # change logging - continue - for obj in related_model.objects.filter(**{related_field_name: instance.pk}): - obj.snapshot() # Ensure the change record includes the "before" state - getattr(obj, related_field_name).remove(instance) - - # Enqueue the object for event processing - queue = events_queue.get() - enqueue_event(queue, instance, request.user, request.id, OBJECT_DELETED) - events_queue.set(queue) - - # Increment metric counters - model_deletes.labels(instance._meta.model_name).inc() - - -@receiver(clear_events) -def clear_events_queue(sender, **kwargs): - """ - Delete any queued events (e.g. because of an aborted bulk transaction) - """ - logger = logging.getLogger('events') - logger.info(f"Clearing {len(events_queue.get())} queued events ({sender})") - events_queue.set({}) +from .utils import run_validators # diff --git a/netbox/extras/utils.py b/netbox/extras/utils.py index e67b9b50c..28d2e13f0 100644 --- a/netbox/extras/utils.py +++ b/netbox/extras/utils.py @@ -1,5 +1,19 @@ +import importlib + +from django.core.exceptions import ImproperlyConfigured from taggit.managers import _TaggableManager +from netbox.context import current_request +from .validators import CustomValidator + +__all__ = ( + 'image_upload', + 'is_report', + 'is_script', + 'is_taggable', + 'run_validators', +) + def is_taggable(obj): """ @@ -48,3 +62,25 @@ def is_report(obj): return issubclass(obj, Report) and obj != Report except TypeError: return False + + +def run_validators(instance, validators): + """ + Run the provided iterable of CustomValidators for the instance. + """ + request = current_request.get() + for validator in validators: + + # Loading a validator class by dotted path + if type(validator) is str: + module, cls = validator.rsplit('.', 1) + validator = getattr(importlib.import_module(module), cls)() + + # Constructing a new instance on the fly from a ruleset + elif type(validator) is dict: + validator = CustomValidator(validator) + + elif not issubclass(validator.__class__, CustomValidator): + raise ImproperlyConfigured(f"Invalid value for custom validator: {validator}") + + validator(instance, request) diff --git a/netbox/netbox/views/generic/bulk_views.py b/netbox/netbox/views/generic/bulk_views.py index 71ce411ba..7a2d4c08b 100644 --- a/netbox/netbox/views/generic/bulk_views.py +++ b/netbox/netbox/views/generic/bulk_views.py @@ -8,7 +8,7 @@ from django.core.exceptions import FieldDoesNotExist, ObjectDoesNotExist, Valida from django.db import transaction, IntegrityError from django.db.models import ManyToManyField, ProtectedError, RestrictedError from django.db.models.fields.reverse_related import ManyToManyRel -from django.forms import HiddenInput, ModelMultipleChoiceField, MultipleHiddenInput +from django.forms import ModelMultipleChoiceField, MultipleHiddenInput from django.http import HttpResponse from django.shortcuts import get_object_or_404, redirect, render from django.urls import reverse @@ -17,8 +17,8 @@ from django.utils.translation import gettext as _ from django_tables2.export import TableExport from core.models import ObjectType +from core.signals import clear_events from extras.models import ExportTemplate -from extras.signals import clear_events from utilities.error_handlers import handle_protectederror from utilities.exceptions import AbortRequest, AbortTransaction, PermissionsViolation from utilities.forms import BulkRenameForm, ConfirmationForm, restrict_form_fields diff --git a/netbox/netbox/views/generic/object_views.py b/netbox/netbox/views/generic/object_views.py index cad7facd3..85f90cbc1 100644 --- a/netbox/netbox/views/generic/object_views.py +++ b/netbox/netbox/views/generic/object_views.py @@ -13,7 +13,7 @@ from django.utils.html import escape from django.utils.safestring import mark_safe from django.utils.translation import gettext as _ -from extras.signals import clear_events +from core.signals import clear_events from utilities.error_handlers import handle_protectederror from utilities.exceptions import AbortRequest, PermissionsViolation from utilities.forms import ConfirmationForm, restrict_form_fields