diff --git a/.github/ISSUE_TEMPLATE/bug_report.yaml b/.github/ISSUE_TEMPLATE/bug_report.yaml index 48c14a2da..5e936c5ec 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yaml +++ b/.github/ISSUE_TEMPLATE/bug_report.yaml @@ -14,7 +14,7 @@ body: attributes: label: NetBox version description: What version of NetBox are you currently running? - placeholder: v3.6.5 + placeholder: v3.6.6 validations: required: true - type: dropdown diff --git a/.github/ISSUE_TEMPLATE/feature_request.yaml b/.github/ISSUE_TEMPLATE/feature_request.yaml index 0525659ae..34103e616 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yaml +++ b/.github/ISSUE_TEMPLATE/feature_request.yaml @@ -14,7 +14,7 @@ body: attributes: label: NetBox version description: What version of NetBox are you currently running? - placeholder: v3.6.5 + placeholder: v3.6.6 validations: required: true - type: dropdown diff --git a/docs/configuration/required-parameters.md b/docs/configuration/required-parameters.md index 012d85762..bda365995 100644 --- a/docs/configuration/required-parameters.md +++ b/docs/configuration/required-parameters.md @@ -59,10 +59,7 @@ DATABASE = { ## REDIS -[Redis](https://redis.io/) is an in-memory data store similar to memcached. While Redis has been an optional component of -NetBox since the introduction of webhooks in version 2.4, it is required starting in 2.6 to support NetBox's caching -functionality (as well as other planned features). In 2.7, the connection settings were broken down into two sections for -task queuing and caching, allowing the user to connect to different Redis instances/databases per feature. +[Redis](https://redis.io/) is a lightweight in-memory data store similar to memcached. NetBox employs Redis for background task queuing and other features. Redis is configured using a configuration setting similar to `DATABASE` and these settings are the same for both of the `tasks` and `caching` subsections: @@ -81,7 +78,7 @@ REDIS = { 'tasks': { 'HOST': 'redis.example.com', 'PORT': 1234, - 'USERNAME': 'netbox' + 'USERNAME': 'netbox', 'PASSWORD': 'foobar', 'DATABASE': 0, 'SSL': False, @@ -89,7 +86,7 @@ REDIS = { 'caching': { 'HOST': 'localhost', 'PORT': 6379, - 'USERNAME': '' + 'USERNAME': '', 'PASSWORD': '', 'DATABASE': 1, 'SSL': False, diff --git a/docs/development/application-registry.md b/docs/development/application-registry.md index c845cd5a7..570563431 100644 --- a/docs/development/application-registry.md +++ b/docs/development/application-registry.md @@ -31,7 +31,7 @@ A dictionary of particular features (e.g. custom fields) mapped to the NetBox mo 'dcim': ['site', 'rack', 'devicetype', ...], ... }, - 'webhooks': { + 'event_rules': { 'extras': ['configcontext', 'tag', ...], 'dcim': ['site', 'rack', 'devicetype', ...], }, diff --git a/docs/development/models.md b/docs/development/models.md index d4838570a..f04610ad5 100644 --- a/docs/development/models.md +++ b/docs/development/models.md @@ -10,19 +10,19 @@ The Django [content types](https://docs.djangoproject.com/en/stable/ref/contrib/ Depending on its classification, each NetBox model may support various features which enhance its operation. Each feature is enabled by inheriting from its designated mixin class, and some features also make use of the [application registry](./application-registry.md#model_features). -| Feature | Feature Mixin | Registry Key | Description | -|------------------------------------------------------------|-------------------------|--------------------|--------------------------------------------------------------------------------| -| [Change logging](../features/change-logging.md) | `ChangeLoggingMixin` | - | Changes to these objects are automatically recorded in the change log | -| Cloning | `CloningMixin` | - | Provides the `clone()` method to prepare a copy | -| [Custom fields](../customization/custom-fields.md) | `CustomFieldsMixin` | `custom_fields` | These models support the addition of user-defined fields | -| [Custom links](../customization/custom-links.md) | `CustomLinksMixin` | `custom_links` | These models support the assignment of custom links | -| [Custom validation](../customization/custom-validation.md) | `CustomValidationMixin` | - | Supports the enforcement of custom validation rules | -| [Export templates](../customization/export-templates.md) | `ExportTemplatesMixin` | `export_templates` | Users can create custom export templates for these models | -| [Job results](../features/background-jobs.md) | `JobsMixin` | `jobs` | Users can create custom export templates for these models | -| [Journaling](../features/journaling.md) | `JournalingMixin` | `journaling` | These models support persistent historical commentary | -| [Synchronized data](../integrations/synchronized-data.md) | `SyncedDataMixin` | `synced_data` | Certain model data can be automatically synchronized from a remote data source | -| [Tagging](../models/extras/tag.md) | `TagsMixin` | `tags` | The models can be tagged with user-defined tags | -| [Webhooks](../integrations/webhooks.md) | `WebhooksMixin` | `webhooks` | NetBox is capable of generating outgoing webhooks for these objects | +| Feature | Feature Mixin | Registry Key | Description | +|------------------------------------------------------------|-------------------------|--------------------|-----------------------------------------------------------------------------------------| +| [Change logging](../features/change-logging.md) | `ChangeLoggingMixin` | - | Changes to these objects are automatically recorded in the change log | +| Cloning | `CloningMixin` | - | Provides the `clone()` method to prepare a copy | +| [Custom fields](../customization/custom-fields.md) | `CustomFieldsMixin` | `custom_fields` | These models support the addition of user-defined fields | +| [Custom links](../customization/custom-links.md) | `CustomLinksMixin` | `custom_links` | These models support the assignment of custom links | +| [Custom validation](../customization/custom-validation.md) | `CustomValidationMixin` | - | Supports the enforcement of custom validation rules | +| [Export templates](../customization/export-templates.md) | `ExportTemplatesMixin` | `export_templates` | Users can create custom export templates for these models | +| [Job results](../features/background-jobs.md) | `JobsMixin` | `jobs` | Users can create custom export templates for these models | +| [Journaling](../features/journaling.md) | `JournalingMixin` | `journaling` | These models support persistent historical commentary | +| [Synchronized data](../integrations/synchronized-data.md) | `SyncedDataMixin` | `synced_data` | Certain model data can be automatically synchronized from a remote data source | +| [Tagging](../models/extras/tag.md) | `TagsMixin` | `tags` | The models can be tagged with user-defined tags | +| [Event rules](../features/event-rules.md) | `EventRulesMixin` | `event_rules` | Event rules can send webhooks or run custom scripts automatically in response to events | ## Models Index @@ -111,7 +111,7 @@ Component models represent individual physical or virtual components belonging t ### Component Template Models -These function as templates to effect the replication of device and virtual machine components. Component template models support a limited feature set, including change logging, custom validation, and webhooks. +These function as templates to effect the replication of device and virtual machine components. Component template models support a limited feature set, including change logging, custom validation, and event rules. * [dcim.ConsolePortTemplate](../models/dcim/consoleporttemplate.md) * [dcim.ConsoleServerPortTemplate](../models/dcim/consoleserverporttemplate.md) diff --git a/docs/features/api-integration.md b/docs/features/api-integration.md index 8c0843bfe..94a39d731 100644 --- a/docs/features/api-integration.md +++ b/docs/features/api-integration.md @@ -26,9 +26,9 @@ To learn more about this feature, check out the [GraphQL API documentation](../i ## Webhooks -A webhook is a mechanism for conveying to some external system a change that took place in NetBox. For example, you may want to notify a monitoring system whenever the status of a device is updated in NetBox. This can be done by creating a webhook for the device model in NetBox and identifying the webhook receiver. When NetBox detects a change to a device, an HTTP request containing the details of the change and who made it be sent to the specified receiver. Webhooks are an excellent mechanism for building event-based automation processes. +A webhook is a mechanism for conveying to some external system a change that has taken place in NetBox. For example, you may want to notify a monitoring system whenever the status of a device is updated in NetBox. To do this, first create a [webhook](../models/extras/webhook.md) identifying the remote receiver (URL), HTTP method, and any other necessary parameters. Then, define an [event rule](../models/extras/eventrule.md) which is triggered by device changes to transmit the webhook. -To learn more about this feature, check out the [webhooks documentation](../integrations/webhooks.md). +When NetBox detects a change to a device, an HTTP request containing the details of the change and who made it be sent to the specified receiver. Webhooks are an excellent mechanism for building event-based automation processes. To learn more about this feature, check out the [webhooks documentation](../integrations/webhooks.md). ## Prometheus Metrics diff --git a/docs/features/event-rules.md b/docs/features/event-rules.md new file mode 100644 index 000000000..0e9535223 --- /dev/null +++ b/docs/features/event-rules.md @@ -0,0 +1,31 @@ +# Event Rules + +NetBox includes the ability to execute certain functions in response to internal object changes. These include: + +* [Scripts](../customization/custom-scripts.md) execution +* [Webhooks](../integrations/webhooks.md) execution + +For example, suppose you want to automatically configure a monitoring system to start monitoring a device when its operational status is changed to active, and remove it from monitoring for any other status. You can create a webhook in NetBox for the device model and craft its content and destination URL to effect the desired change on the receiving system. You can then associate an event rule with this webhook and the webhook will be sent automatically by NetBox whenever the configured constraints are met. + +Each event must be associated with at least one NetBox object type and at least one event (e.g. create, update, or delete). + +## Conditional Event Rules + +An event rule may include a set of conditional logic expressed in JSON used to control whether an event triggers for a specific object. For example, you may wish to trigger an event for devices only when the `status` field of an object is "active": + +```json +{ + "and": [ + { + "attr": "status.value", + "value": "active" + } + ] +} +``` + +For more detail, see the reference documentation for NetBox's [conditional logic](../reference/conditions.md). + +## Event Rule Processing + +When a change is detected, any resulting events are placed into a Redis queue for processing. This allows the user's request to complete without needing to wait for the outgoing event(s) to be processed. The events are then extracted from the queue by the `rqworker` process. The current event queue and any failed events can be inspected in the admin UI under System > Background Tasks. diff --git a/docs/index.md b/docs/index.md index 05cd79f23..84334337b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -32,7 +32,7 @@ In addition to its expansive and robust data model, NetBox offers myriad mechani * Custom fields * Custom model validation * Export templates -* Webhooks +* Event rules * Plugins * REST & GraphQL APIs diff --git a/docs/integrations/webhooks.md b/docs/integrations/webhooks.md index 9a1094988..8913fd99c 100644 --- a/docs/integrations/webhooks.md +++ b/docs/integrations/webhooks.md @@ -1,11 +1,9 @@ # Webhooks -NetBox can be configured to transmit outgoing webhooks to remote systems in response to internal object changes. The receiver can act on the data in these webhook messages to perform related tasks. +NetBox can be configured via [Event Rules](../features/event-rules.md) to transmit outgoing webhooks to remote systems in response to internal object changes. The receiver can act on the data in these webhook messages to perform related tasks. For example, suppose you want to automatically configure a monitoring system to start monitoring a device when its operational status is changed to active, and remove it from monitoring for any other status. You can create a webhook in NetBox for the device model and craft its content and destination URL to effect the desired change on the receiving system. Webhooks will be sent automatically by NetBox whenever the configured constraints are met. -Each webhook must be associated with at least one NetBox object type and at least one event (create, update, or delete). Users can specify the receiver URL, HTTP request type (`GET`, `POST`, etc.), content type, and headers. A request body can also be specified; if left blank, this will default to a serialized representation of the affected object. - !!! warning "Security Notice" Webhooks support the inclusion of user-submitted code to generate the URL, custom headers, and payloads, which may pose security risks under certain conditions. Only grant permission to create or modify webhooks to trusted users. @@ -70,26 +68,12 @@ If no body template is specified, the request body will be populated with a JSON } ``` -## Conditional Webhooks - -A webhook may include a set of conditional logic expressed in JSON used to control whether a webhook triggers for a specific object. For example, you may wish to trigger a webhook for devices only when the `status` field of an object is "active": - -```json -{ - "and": [ - { - "attr": "status.value", - "value": "active" - } - ] -} -``` - -For more detail, see the reference documentation for NetBox's [conditional logic](../reference/conditions.md). +!!! note + The setting of conditional webhooks has been moved to [Event Rules](../features/event-rules.md) since NetBox 3.7 ## Webhook Processing -When a change is detected, any resulting webhooks are placed into a Redis queue for processing. This allows the user's request to complete without needing to wait for the outgoing webhook(s) to be processed. The webhooks are then extracted from the queue by the `rqworker` process and HTTP requests are sent to their respective destinations. The current webhook queue and any failed webhooks can be inspected in the admin UI under System > Background Tasks. +Using [Event Rules](../features/event-rules.md), when a change is detected, any resulting webhooks are placed into a Redis queue for processing. This allows the user's request to complete without needing to wait for the outgoing webhook(s) to be processed. The webhooks are then extracted from the queue by the `rqworker` process and HTTP requests are sent to their respective destinations. The current webhook queue and any failed webhooks can be inspected in the admin UI under System > Background Tasks. A request is considered successful if the response has a 2XX status code; otherwise, the request is marked as having failed. Failed requests may be retried manually via the admin UI. diff --git a/docs/models/extras/eventrule.md b/docs/models/extras/eventrule.md new file mode 100644 index 000000000..89645be3c --- /dev/null +++ b/docs/models/extras/eventrule.md @@ -0,0 +1,35 @@ +# EventRule + +An event rule is a mechanism for automatically taking an action (such as running a script or sending a webhook) in response to an event in NetBox. For example, you may want to notify a monitoring system whenever the status of a device is updated in NetBox. This can be done by creating an event for device objects and designating a webhook to be transmitted. When NetBox detects a change to a device, an HTTP request containing the details of the change and who made it be sent to the specified receiver. + +See the [event rules documentation](../features/event-rules.md) for more information. + +## Fields + +### Name + +A unique human-friendly name. + +### Content Types + +The type(s) of object in NetBox that will trigger the rule. + +### Enabled + +If not selected, the event rule will not be processed. + +### Events + +The events which will trigger the rule. At least one event type must be selected. + +| Name | Description | +|------------|--------------------------------------| +| Creations | A new object has been created | +| Updates | An existing object has been modified | +| Deletions | An object has been deleted | +| Job starts | A job for an object starts | +| Job ends | A job for an object terminates | + +### Conditions + +A set of [prescribed conditions](../../reference/conditions.md) against which the triggering object will be evaluated. If the conditions are defined but not met by the object, no action will be taken. An event rule that does not define any conditions will _always_ trigger. diff --git a/docs/plugins/development/models.md b/docs/plugins/development/models.md index 46af135e1..902ee9c82 100644 --- a/docs/plugins/development/models.md +++ b/docs/plugins/development/models.md @@ -123,14 +123,17 @@ For more information about database migrations, see the [Django documentation](h ::: netbox.models.features.CustomValidationMixin +::: netbox.models.features.EventRulesMixin + +!!! note + `EventRulesMixin` was renamed from `WebhooksMixin` in NetBox v3.7. + ::: netbox.models.features.ExportTemplatesMixin ::: netbox.models.features.JournalingMixin ::: netbox.models.features.TagsMixin -::: netbox.models.features.WebhooksMixin - ## Choice Sets For model fields which support the selection of one or more values from a predefined list of choices, NetBox provides the `ChoiceSet` utility class. This can be used in place of a regular choices tuple to provide enhanced functionality, namely dynamic configuration and colorization. (See [Django's documentation](https://docs.djangoproject.com/en/stable/ref/models/fields/#choices) on the `choices` parameter for supported model fields.) diff --git a/docs/release-notes/version-3.6.md b/docs/release-notes/version-3.6.md index b8d316218..10e93be1e 100644 --- a/docs/release-notes/version-3.6.md +++ b/docs/release-notes/version-3.6.md @@ -1,6 +1,29 @@ # NetBox v3.6 -## v3.6.6 (FUTURE) +## v3.6.7 (FUTURE) + +--- + +## v3.6.6 (2023-11-29) + +### Enhancements + +* [#13735](https://github.com/netbox-community/netbox/issues/13735) - Show complete region hierarchy in UI for all relevant objects + +### Bug Fixes + +* [#14056](https://github.com/netbox-community/netbox/issues/14056) - Record a pre-change snapshot when bulk editing objects via CSV +* [#14187](https://github.com/netbox-community/netbox/issues/14187) - Raise a validation error when attempting to create a duplicate script or report +* [#14199](https://github.com/netbox-community/netbox/issues/14199) - Fix jobs list for reports with a custom name +* [#14239](https://github.com/netbox-community/netbox/issues/14239) - Fix CustomFieldChoiceSet search filter +* [#14242](https://github.com/netbox-community/netbox/issues/14242) - Enable export templates for contact assignments +* [#14299](https://github.com/netbox-community/netbox/issues/14299) - Webhook timestamps should be in proper ISO 8601 format +* [#14325](https://github.com/netbox-community/netbox/issues/14325) - Fix numeric ordering of service ports +* [#14339](https://github.com/netbox-community/netbox/issues/14339) - Correctly hash local user password when set via REST API +* [#14343](https://github.com/netbox-community/netbox/issues/14343) - Fix ordering ASN table by ASDOT column +* [#14346](https://github.com/netbox-community/netbox/issues/14346) - Fix running reports via REST API +* [#14349](https://github.com/netbox-community/netbox/issues/14349) - Fix custom validation support for remote data sources +* [#14363](https://github.com/netbox-community/netbox/issues/14363) - Fix bulk editing of interfaces assigned to VM with no cluster --- diff --git a/mkdocs.yml b/mkdocs.yml index f927bf386..8cbfd397b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -87,6 +87,7 @@ nav: - Auth & Permissions: 'features/authentication-permissions.md' - API & Integration: 'features/api-integration.md' - Customization: 'features/customization.md' + - Event Rules: 'features/event-rules.md' - Installation & Upgrade: - Installing NetBox: 'installation/index.md' - 1. PostgreSQL: 'installation/1-postgresql.md' @@ -215,6 +216,7 @@ nav: - CustomField: 'models/extras/customfield.md' - CustomFieldChoiceSet: 'models/extras/customfieldchoiceset.md' - CustomLink: 'models/extras/customlink.md' + - EventRule: 'models/extras/eventrule.md' - ExportTemplate: 'models/extras/exporttemplate.md' - ImageAttachment: 'models/extras/imageattachment.md' - JournalEntry: 'models/extras/journalentry.md' diff --git a/netbox/core/models/contenttypes.py b/netbox/core/models/contenttypes.py index 0731871ec..c98184c3d 100644 --- a/netbox/core/models/contenttypes.py +++ b/netbox/core/models/contenttypes.py @@ -26,7 +26,7 @@ class ContentTypeManager(ContentTypeManager_): Return the ContentTypes only for models which are registered as supporting the specified feature. For example, we can find all ContentTypes for models which support webhooks with - ContentType.objects.with_feature('webhooks') + ContentType.objects.with_feature('event_rules') """ if feature not in registry['model_features']: raise KeyError( diff --git a/netbox/core/models/data.py b/netbox/core/models/data.py index cf40c0bd5..efda879af 100644 --- a/netbox/core/models/data.py +++ b/netbox/core/models/data.py @@ -116,6 +116,7 @@ class DataSource(JobsMixin, PrimaryModel): ) def clean(self): + super().clean() # Validate data backend type if self.type and self.type not in registry['data_backends']: diff --git a/netbox/core/models/files.py b/netbox/core/models/files.py index 138527581..5a321bdc3 100644 --- a/netbox/core/models/files.py +++ b/netbox/core/models/files.py @@ -2,6 +2,7 @@ import logging import os from django.conf import settings +from django.core.exceptions import ValidationError from django.db import models from django.urls import reverse from django.utils.translation import gettext as _ @@ -85,6 +86,14 @@ class ManagedFile(SyncedDataMixin, models.Model): self.file_path = os.path.basename(self.data_path) self.data_file.write_to_disk(self.full_path, overwrite=True) + def clean(self): + super().clean() + + # Ensure that the file root and path make a unique pair + if self._meta.model.objects.filter(file_root=self.file_root, file_path=self.file_path).exclude(pk=self.pk).exists(): + raise ValidationError( + f"A {self._meta.verbose_name.lower()} with this file path already exists ({self.file_root}/{self.file_path}).") + def delete(self, *args, **kwargs): # Delete file from disk try: diff --git a/netbox/core/models/jobs.py b/netbox/core/models/jobs.py index 5b9b41e53..af8191df5 100644 --- a/netbox/core/models/jobs.py +++ b/netbox/core/models/jobs.py @@ -16,7 +16,7 @@ from extras.constants import EVENT_JOB_END, EVENT_JOB_START from netbox.config import get_config from netbox.constants import RQ_QUEUE_DEFAULT from utilities.querysets import RestrictedQuerySet -from utilities.rqworker import get_queue_for_model, get_rq_retry +from utilities.rqworker import get_queue_for_model __all__ = ( 'Job', @@ -168,8 +168,8 @@ class Job(models.Model): self.status = JobStatusChoices.STATUS_RUNNING self.save() - # Handle webhooks - self.trigger_webhooks(event=EVENT_JOB_START) + # Handle events + self.process_event(event=EVENT_JOB_START) def terminate(self, status=JobStatusChoices.STATUS_COMPLETED, error=None): """ @@ -186,8 +186,8 @@ class Job(models.Model): self.completed = timezone.now() self.save() - # Handle webhooks - self.trigger_webhooks(event=EVENT_JOB_END) + # Handle events + self.process_event(event=EVENT_JOB_END) @classmethod def enqueue(cls, func, instance, name='', user=None, schedule_at=None, interval=None, **kwargs): @@ -224,27 +224,18 @@ class Job(models.Model): return job - def trigger_webhooks(self, event): - from extras.models import Webhook + def process_event(self, event): + """ + Process any EventRules relevant to the passed job event (i.e. start or stop). + """ + from extras.models import EventRule + from extras.events import process_event_rules - rq_queue_name = get_config().QUEUE_MAPPINGS.get('webhook', RQ_QUEUE_DEFAULT) - rq_queue = django_rq.get_queue(rq_queue_name, is_async=False) - - # Fetch any webhooks matching this object type and action - webhooks = Webhook.objects.filter( + # Fetch any event rules matching this object type and action + event_rules = EventRule.objects.filter( **{f'type_{event}': True}, content_types=self.object_type, enabled=True ) - for webhook in webhooks: - rq_queue.enqueue( - "extras.webhooks_worker.process_webhook", - webhook=webhook, - model_name=self.object_type.model, - event=event, - data=self.data, - timestamp=str(timezone.now()), - username=self.user.username, - retry=get_rq_retry() - ) + process_event_rules(event_rules, self.object_type.model, event, self.data, self.user.username) diff --git a/netbox/extras/api/nested_serializers.py b/netbox/extras/api/nested_serializers.py index a97c630d2..4bada494f 100644 --- a/netbox/extras/api/nested_serializers.py +++ b/netbox/extras/api/nested_serializers.py @@ -10,15 +10,25 @@ __all__ = [ 'NestedCustomFieldChoiceSetSerializer', 'NestedCustomFieldSerializer', 'NestedCustomLinkSerializer', + 'NestedEventRuleSerializer', 'NestedExportTemplateSerializer', 'NestedImageAttachmentSerializer', 'NestedJournalEntrySerializer', 'NestedSavedFilterSerializer', + 'NestedScriptSerializer', 'NestedTagSerializer', # Defined in netbox.api.serializers 'NestedWebhookSerializer', ] +class NestedEventRuleSerializer(WritableNestedSerializer): + url = serializers.HyperlinkedIdentityField(view_name='extras-api:eventrule-detail') + + class Meta: + model = models.EventRule + fields = ['id', 'url', 'display', 'name'] + + class NestedWebhookSerializer(WritableNestedSerializer): url = serializers.HyperlinkedIdentityField(view_name='extras-api:webhook-detail') @@ -105,3 +115,20 @@ class NestedJournalEntrySerializer(WritableNestedSerializer): class Meta: model = models.JournalEntry fields = ['id', 'url', 'display', 'created'] + + +class NestedScriptSerializer(WritableNestedSerializer): + url = serializers.HyperlinkedIdentityField( + view_name='extras-api:script-detail', + lookup_field='full_name', + lookup_url_kwarg='pk' + ) + name = serializers.CharField(read_only=True) + display = serializers.SerializerMethodField(read_only=True) + + class Meta: + model = models.Script + fields = ['id', 'url', 'display', 'name'] + + def get_display(self, obj): + return f'{obj.name} ({obj.module})' diff --git a/netbox/extras/api/serializers.py b/netbox/extras/api/serializers.py index 55e030a13..ffd0df9ab 100644 --- a/netbox/extras/api/serializers.py +++ b/netbox/extras/api/serializers.py @@ -1,17 +1,17 @@ from django.contrib.auth import get_user_model from django.core.exceptions import ObjectDoesNotExist +from drf_spectacular.types import OpenApiTypes +from drf_spectacular.utils import extend_schema_field from rest_framework import serializers -from core.api.serializers import JobSerializer from core.api.nested_serializers import NestedDataSourceSerializer, NestedDataFileSerializer, NestedJobSerializer +from core.api.serializers import JobSerializer from core.models import ContentType from dcim.api.nested_serializers import ( NestedDeviceRoleSerializer, NestedDeviceTypeSerializer, NestedLocationSerializer, NestedPlatformSerializer, NestedRegionSerializer, NestedSiteSerializer, NestedSiteGroupSerializer, ) from dcim.models import DeviceRole, DeviceType, Location, Platform, Region, Site, SiteGroup -from drf_spectacular.utils import extend_schema_field -from drf_spectacular.types import OpenApiTypes from extras.choices import * from extras.models import * from netbox.api.exceptions import SerializerNotFound @@ -38,6 +38,7 @@ __all__ = ( 'CustomFieldSerializer', 'CustomLinkSerializer', 'DashboardSerializer', + 'EventRuleSerializer', 'ExportTemplateSerializer', 'ImageAttachmentSerializer', 'JournalEntrySerializer', @@ -56,24 +57,59 @@ __all__ = ( ) +# +# Event Rules +# + +class EventRuleSerializer(NetBoxModelSerializer): + url = serializers.HyperlinkedIdentityField(view_name='extras-api:eventrule-detail') + content_types = ContentTypeField( + queryset=ContentType.objects.with_feature('event_rules'), + many=True + ) + action_type = ChoiceField(choices=EventRuleActionChoices) + action_object_type = ContentTypeField( + queryset=ContentType.objects.with_feature('event_rules'), + ) + action_object = serializers.SerializerMethodField(read_only=True) + + class Meta: + model = EventRule + fields = [ + 'id', 'url', 'display', 'content_types', 'name', 'type_create', 'type_update', 'type_delete', + 'type_job_start', 'type_job_end', 'enabled', 'conditions', 'action_type', 'action_object_type', + 'action_object_id', 'action_object', 'description', 'custom_fields', 'tags', 'created', 'last_updated', + ] + + @extend_schema_field(OpenApiTypes.OBJECT) + def get_action_object(self, instance): + context = {'request': self.context['request']} + # We need to manually instantiate the serializer for scripts + if instance.action_type == EventRuleActionChoices.SCRIPT: + module_id, script_name = instance.action_parameters['script_choice'].split(":", maxsplit=1) + script = instance.action_object.scripts[script_name]() + return NestedScriptSerializer(script, context=context).data + else: + serializer = get_serializer_for_model( + model=instance.action_object_type.model_class(), + prefix=NESTED_SERIALIZER_PREFIX + ) + return serializer(instance.action_object, context=context).data + + # # Webhooks # class WebhookSerializer(NetBoxModelSerializer): url = serializers.HyperlinkedIdentityField(view_name='extras-api:webhook-detail') - content_types = ContentTypeField( - queryset=ContentType.objects.with_feature('webhooks'), - many=True - ) class Meta: model = Webhook fields = [ - 'id', 'url', 'display', 'content_types', 'name', 'description', 'type_create', 'type_update', 'type_delete', - 'type_job_start', 'type_job_end', 'payload_url', 'enabled', 'http_method', 'http_content_type', - 'additional_headers', 'body_template', 'secret', 'conditions', 'ssl_verification', 'ca_file_path', - 'custom_fields', 'tags', 'created', 'last_updated', + 'id', 'url', 'display', 'name', 'description', 'payload_url', 'http_method', 'http_content_type', + 'additional_headers', 'body_template', 'secret', 'ssl_verification', 'ca_file_path', 'custom_fields', + 'tags', 'created', 'last_updated', ] diff --git a/netbox/extras/api/urls.py b/netbox/extras/api/urls.py index 5f2b324e6..1616b8554 100644 --- a/netbox/extras/api/urls.py +++ b/netbox/extras/api/urls.py @@ -7,6 +7,7 @@ from . import views router = NetBoxRouter() router.APIRootView = views.ExtrasRootView +router.register('event-rules', views.EventRuleViewSet) router.register('webhooks', views.WebhookViewSet) router.register('custom-fields', views.CustomFieldViewSet) router.register('custom-field-choice-sets', views.CustomFieldChoiceSetViewSet) diff --git a/netbox/extras/api/views.py b/netbox/extras/api/views.py index f518275e0..e0fca8617 100644 --- a/netbox/extras/api/views.py +++ b/netbox/extras/api/views.py @@ -37,6 +37,17 @@ class ExtrasRootView(APIRootView): return 'Extras' +# +# EventRules +# + +class EventRuleViewSet(NetBoxModelViewSet): + metadata_class = ContentTypeMetadata + queryset = EventRule.objects.all() + serializer_class = serializers.EventRuleSerializer + filterset_class = filtersets.EventRuleFilterSet + + # # Webhooks # @@ -283,7 +294,7 @@ class ReportViewSet(ViewSet): # Retrieve and run the Report. This will create a new Job. module, report_cls = self._get_report(pk) - report = report_cls() + report = report_cls input_serializer = serializers.ReportInputSerializer( data=request.data, context={'report': report} diff --git a/netbox/extras/choices.py b/netbox/extras/choices.py index fdb951b7d..14179fb39 100644 --- a/netbox/extras/choices.py +++ b/netbox/extras/choices.py @@ -291,3 +291,18 @@ class DashboardWidgetColorChoices(ChoiceSet): (BLACK, _('Black')), (WHITE, _('White')), ) + + +# +# Event Rules +# + +class EventRuleActionChoices(ChoiceSet): + + WEBHOOK = 'webhook' + SCRIPT = 'script' + + CHOICES = ( + (WEBHOOK, _('Webhook')), + (SCRIPT, _('Script')), + ) diff --git a/netbox/extras/context_managers.py b/netbox/extras/context_managers.py index 32323999e..8de47465e 100644 --- a/netbox/extras/context_managers.py +++ b/netbox/extras/context_managers.py @@ -1,25 +1,25 @@ from contextlib import contextmanager -from netbox.context import current_request, webhooks_queue -from .webhooks import flush_webhooks +from netbox.context import current_request, events_queue +from .events import flush_events @contextmanager -def change_logging(request): +def event_tracking(request): """ - Enable change logging by connecting the appropriate signals to their receivers before code is run, and - disconnecting them afterward. + Queue interesting events in memory while processing a request, then flush that queue for processing by the + events pipline before returning the response. :param request: WSGIRequest object with a unique `id` set """ current_request.set(request) - webhooks_queue.set([]) + events_queue.set([]) yield # Flush queued webhooks to RQ - flush_webhooks(webhooks_queue.get()) + flush_events(events_queue.get()) # Clear context vars current_request.set(None) - webhooks_queue.set([]) + events_queue.set([]) diff --git a/netbox/extras/events.py b/netbox/extras/events.py new file mode 100644 index 000000000..05352b7d1 --- /dev/null +++ b/netbox/extras/events.py @@ -0,0 +1,178 @@ +import logging + +from django.conf import settings +from django.contrib.auth import get_user_model +from django.contrib.contenttypes.models import ContentType +from django.core.exceptions import ObjectDoesNotExist +from django.utils import timezone +from django.utils.module_loading import import_string +from django_rq import get_queue + +from core.models import Job +from netbox.config import get_config +from netbox.constants import RQ_QUEUE_DEFAULT +from netbox.registry import registry +from utilities.api import get_serializer_for_model +from utilities.rqworker import get_rq_retry +from utilities.utils import serialize_object +from .choices import * +from .models import EventRule, ScriptModule + +logger = logging.getLogger('netbox.events_processor') + + +def serialize_for_event(instance): + """ + Return a serialized representation of the given instance suitable for use in a queued event. + """ + serializer_class = get_serializer_for_model(instance.__class__) + serializer_context = { + 'request': None, + } + serializer = serializer_class(instance, context=serializer_context) + + return serializer.data + + +def get_snapshots(instance, action): + snapshots = { + 'prechange': getattr(instance, '_prechange_snapshot', None), + 'postchange': None, + } + if action != ObjectChangeActionChoices.ACTION_DELETE: + # Use model's serialize_object() method if defined; fall back to serialize_object() utility function + if hasattr(instance, 'serialize_object'): + snapshots['postchange'] = instance.serialize_object() + else: + snapshots['postchange'] = serialize_object(instance) + + return snapshots + + +def enqueue_object(queue, instance, user, request_id, action): + """ + Enqueue a serialized representation of a created/updated/deleted object for the processing of + events once the request has completed. + """ + # Determine whether this type of object supports event rules + app_label = instance._meta.app_label + model_name = instance._meta.model_name + if model_name not in registry['model_features']['event_rules'].get(app_label, []): + return + + queue.append({ + 'content_type': ContentType.objects.get_for_model(instance), + 'object_id': instance.pk, + 'event': action, + 'data': serialize_for_event(instance), + 'snapshots': get_snapshots(instance, action), + 'username': user.username, + 'request_id': request_id + }) + + +def process_event_rules(event_rules, model_name, event, data, username, snapshots=None, request_id=None): + try: + user = get_user_model().objects.get(username=username) + except ObjectDoesNotExist: + user = None + + for event_rule in event_rules: + + # Evaluate event rule conditions (if any) + if not event_rule.eval_conditions(data): + return + + # Webhooks + if event_rule.action_type == EventRuleActionChoices.WEBHOOK: + + # Select the appropriate RQ queue + queue_name = get_config().QUEUE_MAPPINGS.get('webhook', RQ_QUEUE_DEFAULT) + rq_queue = get_queue(queue_name) + + # Compile the task parameters + params = { + "event_rule": event_rule, + "model_name": model_name, + "event": event, + "data": data, + "snapshots": snapshots, + "timestamp": timezone.now().isoformat(), + "username": username, + "retry": get_rq_retry() + } + if snapshots: + params["snapshots"] = snapshots + if request_id: + params["request_id"] = request_id + + # Enqueue the task + rq_queue.enqueue( + "extras.webhooks_worker.process_webhook", + **params + ) + + # Scripts + elif event_rule.action_type == EventRuleActionChoices.SCRIPT: + # Resolve the script from action parameters + script_module = event_rule.action_object + _, script_name = event_rule.action_parameters['script_choice'].split(":", maxsplit=1) + script = script_module.scripts[script_name]() + + # Enqueue a Job to record the script's execution + Job.enqueue( + "extras.scripts.run_script", + instance=script_module, + name=script.class_name, + user=user, + data=data + ) + + else: + raise ValueError(f"Unknown action type for an event rule: {event_rule.action_type}") + + +def process_event_queue(events): + """ + Flush a list of object representation to RQ for EventRule processing. + """ + events_cache = { + 'type_create': {}, + 'type_update': {}, + 'type_delete': {}, + } + + for data in events: + action_flag = { + ObjectChangeActionChoices.ACTION_CREATE: 'type_create', + ObjectChangeActionChoices.ACTION_UPDATE: 'type_update', + ObjectChangeActionChoices.ACTION_DELETE: 'type_delete', + }[data['event']] + content_type = data['content_type'] + + # Cache applicable Event Rules + if content_type not in events_cache[action_flag]: + events_cache[action_flag][content_type] = EventRule.objects.filter( + **{action_flag: True}, + content_types=content_type, + enabled=True + ) + event_rules = events_cache[action_flag][content_type] + + process_event_rules( + event_rules, content_type.model, data['event'], data['data'], data['username'], + snapshots=data['snapshots'], request_id=data['request_id'] + ) + + +def flush_events(queue): + """ + Flush a list of object representation to RQ for webhook processing. + """ + if queue: + for name in settings.EVENTS_PIPELINE: + try: + func = import_string(name) + func(queue) + except Exception as e: + logger.error(f"Cannot import events pipeline {name} error: {e}") diff --git a/netbox/extras/filtersets.py b/netbox/extras/filtersets.py index 3cbe8239b..5d36a34c7 100644 --- a/netbox/extras/filtersets.py +++ b/netbox/extras/filtersets.py @@ -22,6 +22,7 @@ __all__ = ( 'CustomFieldChoiceSetFilterSet', 'CustomFieldFilterSet', 'CustomLinkFilterSet', + 'EventRuleFilterSet', 'ExportTemplateFilterSet', 'ImageAttachmentFilterSet', 'JournalEntryFilterSet', @@ -38,19 +39,18 @@ class WebhookFilterSet(NetBoxModelFilterSet): method='search', label=_('Search'), ) - content_type_id = MultiValueNumberFilter( - field_name='content_types__id' - ) - content_types = ContentTypeFilter() http_method = django_filters.MultipleChoiceFilter( choices=WebhookHttpMethodChoices ) + payload_url = MultiValueCharFilter( + lookup_expr='icontains' + ) class Meta: model = Webhook fields = [ - 'id', 'name', 'type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end', 'payload_url', - 'enabled', 'http_method', 'http_content_type', 'secret', 'ssl_verification', 'ca_file_path', + 'id', 'name', 'payload_url', 'http_method', 'http_content_type', 'secret', 'ssl_verification', + 'ca_file_path', ] def search(self, queryset, name, value): @@ -63,6 +63,38 @@ class WebhookFilterSet(NetBoxModelFilterSet): ) +class EventRuleFilterSet(NetBoxModelFilterSet): + q = django_filters.CharFilter( + method='search', + label=_('Search'), + ) + content_type_id = MultiValueNumberFilter( + field_name='content_types__id' + ) + content_types = ContentTypeFilter() + action_type = django_filters.MultipleChoiceFilter( + choices=EventRuleActionChoices + ) + action_object_type = ContentTypeFilter() + action_object_id = MultiValueNumberFilter() + + class Meta: + model = EventRule + fields = [ + 'id', 'name', 'type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end', 'enabled', + 'action_type', 'description', + ] + + def search(self, queryset, name, value): + if not value.strip(): + return queryset + return queryset.filter( + Q(name__icontains=value) | + Q(description__icontains=value) | + Q(comments__icontains=value) + ) + + class CustomFieldFilterSet(BaseFilterSet): q = django_filters.CharFilter( method='search', @@ -122,8 +154,7 @@ class CustomFieldChoiceSetFilterSet(BaseFilterSet): return queryset return queryset.filter( Q(name__icontains=value) | - Q(description__icontains=value) | - Q(extra_choices__contains=value) + Q(description__icontains=value) ) def filter_by_choice(self, queryset, name, value): diff --git a/netbox/extras/forms/bulk_edit.py b/netbox/extras/forms/bulk_edit.py index b44d29c9c..9479fef99 100644 --- a/netbox/extras/forms/bulk_edit.py +++ b/netbox/extras/forms/bulk_edit.py @@ -14,6 +14,7 @@ __all__ = ( 'CustomFieldBulkEditForm', 'CustomFieldChoiceSetBulkEditForm', 'CustomLinkBulkEditForm', + 'EventRuleBulkEditForm', 'ExportTemplateBulkEditForm', 'JournalEntryBulkEditForm', 'SavedFilterBulkEditForm', @@ -182,6 +183,39 @@ class WebhookBulkEditForm(NetBoxModelBulkEditForm): max_length=200, required=False ) + http_method = forms.ChoiceField( + choices=add_blank_choice(WebhookHttpMethodChoices), + required=False, + label=_('HTTP method') + ) + payload_url = forms.CharField( + required=False, + label=_('Payload URL') + ) + ssl_verification = forms.NullBooleanField( + required=False, + widget=BulkEditNullBooleanSelect(), + label=_('SSL verification') + ) + secret = forms.CharField( + label=_('Secret'), + required=False + ) + ca_file_path = forms.CharField( + required=False, + label=_('CA file path') + ) + + nullable_fields = ('secret', 'ca_file_path') + + +class EventRuleBulkEditForm(NetBoxModelBulkEditForm): + model = EventRule + + pk = forms.ModelMultipleChoiceField( + queryset=EventRule.objects.all(), + widget=forms.MultipleHiddenInput + ) enabled = forms.NullBooleanField( label=_('Enabled'), required=False, @@ -212,30 +246,8 @@ class WebhookBulkEditForm(NetBoxModelBulkEditForm): required=False, widget=BulkEditNullBooleanSelect() ) - http_method = forms.ChoiceField( - choices=add_blank_choice(WebhookHttpMethodChoices), - required=False, - label=_('HTTP method') - ) - payload_url = forms.CharField( - required=False, - label=_('Payload URL') - ) - ssl_verification = forms.NullBooleanField( - required=False, - widget=BulkEditNullBooleanSelect(), - label=_('SSL verification') - ) - secret = forms.CharField( - label=_('Secret'), - required=False - ) - ca_file_path = forms.CharField( - required=False, - label=_('CA file path') - ) - nullable_fields = ('description', 'secret', 'conditions', 'ca_file_path') + nullable_fields = ('description', 'conditions',) class TagBulkEditForm(BulkEditForm): diff --git a/netbox/extras/forms/bulk_import.py b/netbox/extras/forms/bulk_import.py index be0a0c63f..e08a6528d 100644 --- a/netbox/extras/forms/bulk_import.py +++ b/netbox/extras/forms/bulk_import.py @@ -1,5 +1,6 @@ from django import forms from django.contrib.postgres.forms import SimpleArrayField +from django.core.exceptions import ObjectDoesNotExist from django.utils.safestring import mark_safe from django.utils.translation import gettext_lazy as _ @@ -17,6 +18,7 @@ __all__ = ( 'CustomFieldChoiceSetImportForm', 'CustomFieldImportForm', 'CustomLinkImportForm', + 'EventRuleImportForm', 'ExportTemplateImportForm', 'JournalEntryImportForm', 'SavedFilterImportForm', @@ -143,21 +145,62 @@ class SavedFilterImportForm(CSVModelForm): class WebhookImportForm(NetBoxModelImportForm): - content_types = CSVMultipleContentTypeField( - label=_('Content types'), - queryset=ContentType.objects.with_feature('webhooks'), - help_text=_("One or more assigned object types") - ) class Meta: model = Webhook fields = ( - 'name', 'enabled', 'content_types', 'type_create', 'type_update', 'type_delete', 'type_job_start', - 'type_job_end', 'payload_url', 'http_method', 'http_content_type', 'additional_headers', 'body_template', + 'name', 'payload_url', 'http_method', 'http_content_type', 'additional_headers', 'body_template', 'secret', 'ssl_verification', 'ca_file_path', 'description', 'tags' ) +class EventRuleImportForm(NetBoxModelImportForm): + content_types = CSVMultipleContentTypeField( + label=_('Content types'), + queryset=ContentType.objects.with_feature('event_rules'), + help_text=_("One or more assigned object types") + ) + action_object = forms.CharField( + label=_('Action object'), + required=True, + help_text=_('Webhook name or script as dotted path module.Class') + ) + + class Meta: + model = EventRule + fields = ( + 'name', 'description', 'enabled', 'conditions', 'content_types', 'type_create', 'type_update', + 'type_delete', 'type_job_start', 'type_job_end', 'action_type', 'action_object', 'comments', 'tags' + ) + + def clean(self): + super().clean() + + action_object = self.cleaned_data.get('action_object') + action_type = self.cleaned_data.get('action_type') + if action_object and action_type: + if action_type == EventRuleActionChoices.WEBHOOK: + try: + webhook = Webhook.objects.get(name=action_object) + except Webhook.ObjectDoesNotExist: + raise forms.ValidationError(f"Webhook {action_object} not found") + self.instance.action_object = webhook + elif action_type == EventRuleActionChoices.SCRIPT: + from extras.scripts import get_module_and_script + module_name, script_name = action_object.split('.', 1) + try: + module, script = get_module_and_script(module_name, script_name) + except ObjectDoesNotExist: + raise forms.ValidationError(f"Script {action_object} not found") + self.instance.action_object = module + self.instance.action_object_type = ContentType.objects.get_for_model(module, for_concrete_model=False) + self.instance.action_parameters = { + 'script_choice': f"{str(module.pk)}:{script_name}", + 'script_name': script.name, + 'script_full_name': script.full_name, + } + + class TagImportForm(CSVModelForm): slug = SlugField() diff --git a/netbox/extras/forms/filtersets.py b/netbox/extras/forms/filtersets.py index b68845c2f..c91e3b8c6 100644 --- a/netbox/extras/forms/filtersets.py +++ b/netbox/extras/forms/filtersets.py @@ -22,6 +22,7 @@ __all__ = ( 'CustomFieldChoiceSetFilterForm', 'CustomFieldFilterForm', 'CustomLinkFilterForm', + 'EventRuleFilterForm', 'ExportTemplateFilterForm', 'ImageAttachmentFilterForm', 'JournalEntryFilterForm', @@ -223,23 +224,45 @@ class SavedFilterFilterForm(SavedFiltersMixin, FilterForm): class WebhookFilterForm(NetBoxModelFilterSetForm): model = Webhook - tag = TagFilterField(model) - fieldsets = ( (None, ('q', 'filter_id', 'tag')), - (_('Attributes'), ('content_type_id', 'http_method', 'enabled')), - (_('Events'), ('type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end')), + (_('Attributes'), ('payload_url', 'http_method', 'http_content_type')), ) - content_type_id = ContentTypeMultipleChoiceField( - queryset=ContentType.objects.with_feature('webhooks'), - required=False, - label=_('Object type') + http_content_type = forms.CharField( + label=_('HTTP content type'), + required=False + ) + payload_url = forms.CharField( + label=_('Payload URL'), + required=False ) http_method = forms.MultipleChoiceField( choices=WebhookHttpMethodChoices, required=False, label=_('HTTP method') ) + tag = TagFilterField(model) + + +class EventRuleFilterForm(NetBoxModelFilterSetForm): + model = EventRule + tag = TagFilterField(model) + + fieldsets = ( + (None, ('q', 'filter_id', 'tag')), + (_('Attributes'), ('content_type_id', 'action_type', 'enabled')), + (_('Events'), ('type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end')), + ) + content_type_id = ContentTypeMultipleChoiceField( + queryset=ContentType.objects.with_feature('event_rules'), + required=False, + label=_('Object type') + ) + action_type = forms.ChoiceField( + choices=add_blank_choice(EventRuleActionChoices), + required=False, + label=_('Action type') + ) enabled = forms.NullBooleanField( label=_('Enabled'), required=False, diff --git a/netbox/extras/forms/model_forms.py b/netbox/extras/forms/model_forms.py index 659c7d9ef..9403165e9 100644 --- a/netbox/extras/forms/model_forms.py +++ b/netbox/extras/forms/model_forms.py @@ -1,6 +1,7 @@ import json from django import forms +from django.contrib.contenttypes.models import ContentType from django.utils.safestring import mark_safe from django.utils.translation import gettext_lazy as _ @@ -11,12 +12,12 @@ from extras.choices import * from extras.models import * from netbox.forms import NetBoxModelForm from tenancy.models import Tenant, TenantGroup -from utilities.forms import BootstrapMixin, add_blank_choice +from utilities.forms import BootstrapMixin, add_blank_choice, get_field_value from utilities.forms.fields import ( CommentField, ContentTypeChoiceField, ContentTypeMultipleChoiceField, DynamicModelChoiceField, DynamicModelMultipleChoiceField, JSONField, SlugField, ) -from utilities.forms.widgets import ChoicesWidget +from utilities.forms.widgets import ChoicesWidget, HTMXSelect from virtualization.models import Cluster, ClusterGroup, ClusterType __all__ = ( @@ -26,6 +27,7 @@ __all__ = ( 'CustomFieldChoiceSetForm', 'CustomFieldForm', 'CustomLinkForm', + 'EventRuleForm', 'ExportTemplateForm', 'ImageAttachmentForm', 'JournalEntryForm', @@ -211,24 +213,59 @@ class BookmarkForm(BootstrapMixin, forms.ModelForm): class WebhookForm(NetBoxModelForm): - content_types = ContentTypeMultipleChoiceField( - label=_('Content types'), - queryset=ContentType.objects.with_feature('webhooks') - ) fieldsets = ( - (_('Webhook'), ('name', 'description', 'content_types', 'enabled', 'tags')), - (_('Events'), ('type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end')), + (_('Webhook'), ('name', 'description', 'tags',)), (_('HTTP Request'), ( 'payload_url', 'http_method', 'http_content_type', 'additional_headers', 'body_template', 'secret', )), - (_('Conditions'), ('conditions',)), (_('SSL'), ('ssl_verification', 'ca_file_path')), ) class Meta: model = Webhook fields = '__all__' + widgets = { + 'additional_headers': forms.Textarea(attrs={'class': 'font-monospace'}), + 'body_template': forms.Textarea(attrs={'class': 'font-monospace'}), + } + + +class EventRuleForm(NetBoxModelForm): + content_types = ContentTypeMultipleChoiceField( + label=_('Content types'), + queryset=ContentType.objects.with_feature('event_rules'), + ) + action_choice = forms.ChoiceField( + label=_('Action choice'), + choices=[] + ) + conditions = JSONField( + required=False, + help_text=_('Enter conditions in JSON format.') + ) + action_data = JSONField( + required=False, + help_text=_('Enter parameters to pass to the action in JSON format.') + ) + + fieldsets = ( + (_('Event Rule'), ('name', 'description', 'content_types', 'enabled', 'tags')), + (_('Events'), ('type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end')), + (_('Conditions'), ('conditions',)), + (_('Action'), ( + 'action_type', 'action_choice', 'action_parameters', 'action_object_type', 'action_object_id', + 'action_data', + )), + ) + + class Meta: + model = EventRule + fields = ( + 'content_types', 'name', 'description', 'type_create', 'type_update', 'type_delete', 'type_job_start', + 'type_job_end', 'enabled', 'conditions', 'action_type', 'action_object_type', 'action_object_id', + 'action_parameters', 'action_data', 'comments', 'tags' + ) labels = { 'type_create': _('Creations'), 'type_update': _('Updates'), @@ -237,11 +274,76 @@ class WebhookForm(NetBoxModelForm): 'type_job_end': _('Job terminations'), } widgets = { - 'additional_headers': forms.Textarea(attrs={'class': 'font-monospace'}), - 'body_template': forms.Textarea(attrs={'class': 'font-monospace'}), 'conditions': forms.Textarea(attrs={'class': 'font-monospace'}), + 'action_type': HTMXSelect(), + 'action_object_type': forms.HiddenInput, + 'action_object_id': forms.HiddenInput, + 'action_parameters': forms.HiddenInput, } + def init_script_choice(self): + choices = [] + for module in ScriptModule.objects.all(): + scripts = [] + for script_name in module.scripts.keys(): + name = f"{str(module.pk)}:{script_name}" + scripts.append((name, script_name)) + + if scripts: + choices.append((str(module), scripts)) + + self.fields['action_choice'].choices = choices + parameters = get_field_value(self, 'action_parameters') + initial = None + if parameters and 'script_choice' in parameters: + initial = parameters['script_choice'] + self.fields['action_choice'].initial = initial + + def init_webhook_choice(self): + initial = None + if self.fields['action_object_type'] and get_field_value(self, 'action_object_id'): + initial = Webhook.objects.get(pk=get_field_value(self, 'action_object_id')) + self.fields['action_choice'] = DynamicModelChoiceField( + label=_('Webhook'), + queryset=Webhook.objects.all(), + required=True, + initial=initial + ) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields['action_object_type'].required = False + self.fields['action_object_id'].required = False + + # Determine the action type + action_type = get_field_value(self, 'action_type') + + if action_type == EventRuleActionChoices.WEBHOOK: + self.init_webhook_choice() + elif action_type == EventRuleActionChoices.SCRIPT: + self.init_script_choice() + + def clean(self): + super().clean() + + action_choice = self.cleaned_data.get('action_choice') + if self.cleaned_data.get('action_type') == EventRuleActionChoices.WEBHOOK: + self.cleaned_data['action_object_type'] = ContentType.objects.get_for_model(action_choice) + self.cleaned_data['action_object_id'] = action_choice.id + elif self.cleaned_data.get('action_type') == EventRuleActionChoices.SCRIPT: + module_id, script_name = action_choice.split(":", maxsplit=1) + script_module = ScriptModule.objects.get(pk=module_id) + self.cleaned_data['action_object_type'] = ContentType.objects.get_for_model(script_module, for_concrete_model=False) + self.cleaned_data['action_object_id'] = script_module.id + script = script_module.scripts[script_name]() + self.cleaned_data['action_parameters'] = { + 'script_choice': action_choice, + 'script_name': script.name, + 'script_full_name': script.full_name, + } + + return self.cleaned_data + class TagForm(BootstrapMixin, forms.ModelForm): slug = SlugField() diff --git a/netbox/extras/graphql/schema.py b/netbox/extras/graphql/schema.py index e13cc0e9f..09e399e37 100644 --- a/netbox/extras/graphql/schema.py +++ b/netbox/extras/graphql/schema.py @@ -72,3 +72,9 @@ class ExtrasQuery(graphene.ObjectType): def resolve_webhook_list(root, info, **kwargs): return gql_query_optimizer(models.Webhook.objects.all(), info) + + event_rule = ObjectField(EventRuleType) + event_rule_list = ObjectListField(EventRuleType) + + def resolve_eventrule_list(root, info, **kwargs): + return gql_query_optimizer(models.EventRule.objects.all(), info) diff --git a/netbox/extras/graphql/types.py b/netbox/extras/graphql/types.py index 068da02f2..4981ddd72 100644 --- a/netbox/extras/graphql/types.py +++ b/netbox/extras/graphql/types.py @@ -8,6 +8,7 @@ __all__ = ( 'CustomFieldChoiceSetType', 'CustomFieldType', 'CustomLinkType', + 'EventRuleType', 'ExportTemplateType', 'ImageAttachmentType', 'JournalEntryType', @@ -110,5 +111,12 @@ class WebhookType(OrganizationalObjectType): class Meta: model = models.Webhook - exclude = ('content_types', ) filterset_class = filtersets.WebhookFilterSet + + +class EventRuleType(OrganizationalObjectType): + + class Meta: + model = models.EventRule + exclude = ('content_types', ) + filterset_class = filtersets.EventRuleFilterSet diff --git a/netbox/extras/management/commands/runscript.py b/netbox/extras/management/commands/runscript.py index 3cf70281c..97ee39f50 100644 --- a/netbox/extras/management/commands/runscript.py +++ b/netbox/extras/management/commands/runscript.py @@ -11,9 +11,9 @@ from django.db import transaction from core.choices import JobStatusChoices from core.models import Job from extras.api.serializers import ScriptOutputSerializer -from extras.context_managers import change_logging +from extras.context_managers import event_tracking from extras.scripts import get_module_and_script -from extras.signals import clear_webhooks +from extras.signals import clear_events from utilities.exceptions import AbortTransaction from utilities.utils import NetBoxFakeRequest @@ -37,7 +37,7 @@ class Command(BaseCommand): def _run_script(): """ Core script execution task. We capture this within a subfunction to allow for conditionally wrapping it with - the change_logging context manager (which is bypassed if commit == False). + the event_tracking context manager (which is bypassed if commit == False). """ try: try: @@ -47,7 +47,7 @@ class Command(BaseCommand): raise AbortTransaction() except AbortTransaction: script.log_info("Database changes have been reverted automatically.") - clear_webhooks.send(request) + clear_events.send(request) job.data = ScriptOutputSerializer(script).data job.terminate() except Exception as e: @@ -57,7 +57,7 @@ class Command(BaseCommand): ) script.log_info("Database changes have been reverted due to error.") logger.error(f"Exception raised during script execution: {e}") - clear_webhooks.send(request) + clear_events.send(request) job.data = ScriptOutputSerializer(script).data job.terminate(status=JobStatusChoices.STATUS_ERRORED, error=str(e)) @@ -136,9 +136,9 @@ class Command(BaseCommand): logger.info(f"Running script (commit={commit})") script.request = request - # Execute the script. If commit is True, wrap it with the change_logging context manager to ensure we process + # Execute the script. If commit is True, wrap it with the event_tracking context manager to ensure we process # change logging, webhooks, etc. - with change_logging(request): + with event_tracking(request): _run_script() else: logger.error('Data is not valid:') diff --git a/netbox/extras/migrations/0101_eventrule.py b/netbox/extras/migrations/0101_eventrule.py new file mode 100644 index 000000000..92ae0e52b --- /dev/null +++ b/netbox/extras/migrations/0101_eventrule.py @@ -0,0 +1,132 @@ +import django.db.models.deletion +import taggit.managers +from django.contrib.contenttypes.models import ContentType +from django.db import migrations, models + +import utilities.json +from extras.choices import * + + +def move_webhooks(apps, schema_editor): + Webhook = apps.get_model("extras", "Webhook") + EventRule = apps.get_model("extras", "EventRule") + + for webhook in Webhook.objects.all(): + event = EventRule() + + event.name = webhook.name + event.type_create = webhook.type_create + event.type_update = webhook.type_update + event.type_delete = webhook.type_delete + event.type_job_start = webhook.type_job_start + event.type_job_end = webhook.type_job_end + event.enabled = webhook.enabled + event.conditions = webhook.conditions + + event.action_type = EventRuleActionChoices.WEBHOOK + event.action_object_type_id = ContentType.objects.get_for_model(webhook).id + event.action_object_id = webhook.id + event.save() + event.content_types.add(*webhook.content_types.all()) + + +class Migration(migrations.Migration): + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ('extras', '0100_customfield_ui_attrs'), + ] + + operations = [ + migrations.CreateModel( + name='EventRule', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False)), + ('created', models.DateTimeField(auto_now_add=True, null=True)), + ('last_updated', models.DateTimeField(auto_now=True, null=True)), + ( + 'custom_field_data', + models.JSONField(blank=True, default=dict, encoder=utilities.json.CustomFieldJSONEncoder), + ), + ('name', models.CharField(max_length=150, unique=True)), + ('description', models.CharField(blank=True, max_length=200)), + ('type_create', models.BooleanField(default=False)), + ('type_update', models.BooleanField(default=False)), + ('type_delete', models.BooleanField(default=False)), + ('type_job_start', models.BooleanField(default=False)), + ('type_job_end', models.BooleanField(default=False)), + ('enabled', models.BooleanField(default=True)), + ('conditions', models.JSONField(blank=True, null=True)), + ('action_type', models.CharField(default='webhook', max_length=30)), + ('action_object_id', models.PositiveBigIntegerField(blank=True, null=True)), + ('action_parameters', models.JSONField(blank=True, null=True)), + ('action_data', models.JSONField(blank=True, null=True)), + ('comments', models.TextField(blank=True)), + ], + options={ + 'verbose_name': 'eventrule', + 'verbose_name_plural': 'eventrules', + 'ordering': ('name',), + }, + ), + migrations.RunPython(move_webhooks), + migrations.RemoveConstraint( + model_name='webhook', + name='extras_webhook_unique_payload_url_types', + ), + migrations.RemoveField( + model_name='webhook', + name='conditions', + ), + migrations.RemoveField( + model_name='webhook', + name='content_types', + ), + migrations.RemoveField( + model_name='webhook', + name='enabled', + ), + migrations.RemoveField( + model_name='webhook', + name='type_create', + ), + migrations.RemoveField( + model_name='webhook', + name='type_delete', + ), + migrations.RemoveField( + model_name='webhook', + name='type_job_end', + ), + migrations.RemoveField( + model_name='webhook', + name='type_job_start', + ), + migrations.RemoveField( + model_name='webhook', + name='type_update', + ), + migrations.AddField( + model_name='eventrule', + name='action_object_type', + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name='eventrule_actions', + to='contenttypes.contenttype', + ), + ), + migrations.AddField( + model_name='eventrule', + name='content_types', + field=models.ManyToManyField(related_name='eventrules', to='contenttypes.contenttype'), + ), + migrations.AddField( + model_name='eventrule', + name='tags', + field=taggit.managers.TaggableManager(through='extras.TaggedItem', to='extras.Tag'), + ), + migrations.AddField( + model_name='webhook', + name='description', + field=models.CharField(blank=True, max_length=200), + ), + ] diff --git a/netbox/extras/migrations/0101_move_configrevision.py b/netbox/extras/migrations/0102_move_configrevision.py similarity index 96% rename from netbox/extras/migrations/0101_move_configrevision.py rename to netbox/extras/migrations/0102_move_configrevision.py index 730e7a096..36eef1205 100644 --- a/netbox/extras/migrations/0101_move_configrevision.py +++ b/netbox/extras/migrations/0102_move_configrevision.py @@ -15,7 +15,7 @@ def update_content_type(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ - ('extras', '0100_customfield_ui_attrs'), + ('extras', '0101_eventrule'), ] operations = [ diff --git a/netbox/extras/migrations/0102_webhook_description.py b/netbox/extras/migrations/0102_webhook_description.py deleted file mode 100644 index cdf1eaa73..000000000 --- a/netbox/extras/migrations/0102_webhook_description.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 4.2.7 on 2023-11-29 15:33 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('extras', '0101_move_configrevision'), - ] - - operations = [ - migrations.AddField( - model_name='webhook', - name='description', - field=models.CharField(blank=True, max_length=200), - ), - ] diff --git a/netbox/extras/models/models.py b/netbox/extras/models/models.py index 367b835b1..f996b50b5 100644 --- a/netbox/extras/models/models.py +++ b/netbox/extras/models/models.py @@ -2,7 +2,7 @@ import json import urllib.parse from django.conf import settings -from django.contrib.contenttypes.fields import GenericForeignKey +from django.contrib.contenttypes.fields import GenericForeignKey, GenericRelation from django.core.validators import ValidationError from django.db import models from django.http import HttpResponse @@ -28,6 +28,7 @@ from utilities.utils import clean_html, dict_to_querydict, render_jinja2 __all__ = ( 'Bookmark', 'CustomLink', + 'EventRule', 'ExportTemplate', 'ImageAttachment', 'JournalEntry', @@ -36,17 +37,17 @@ __all__ = ( ) -class Webhook(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLoggedModel): +class EventRule(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLoggedModel): """ - A Webhook defines a request that will be sent to a remote application when an object is created, updated, and/or - delete in NetBox. The request will contain a representation of the object, which the remote application can act on. - Each Webhook can be limited to firing only on certain actions or certain object types. + An EventRule defines an action to be taken automatically in response to a specific set of events, such as when a + specific type of object is created, modified, or deleted. The action to be taken might entail transmitting a + webhook or executing a custom script. """ content_types = models.ManyToManyField( to='contenttypes.ContentType', - related_name='webhooks', + related_name='eventrules', verbose_name=_('object types'), - help_text=_("The object(s) to which this Webhook applies.") + help_text=_("The object(s) to which this rule applies.") ) name = models.CharField( verbose_name=_('name'), @@ -83,6 +84,109 @@ class Webhook(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLoggedMo default=False, help_text=_("Triggers when a job for a matching object terminates.") ) + enabled = models.BooleanField( + verbose_name=_('enabled'), + default=True + ) + conditions = models.JSONField( + verbose_name=_('conditions'), + blank=True, + null=True, + help_text=_("A set of conditions which determine whether the event will be generated.") + ) + + # Action to take + action_type = models.CharField( + max_length=30, + choices=EventRuleActionChoices, + default=EventRuleActionChoices.WEBHOOK, + verbose_name=_('action type') + ) + action_object_type = models.ForeignKey( + to='contenttypes.ContentType', + related_name='eventrule_actions', + on_delete=models.CASCADE + ) + action_object_id = models.PositiveBigIntegerField( + blank=True, + null=True + ) + action_object = GenericForeignKey( + ct_field='action_object_type', + fk_field='action_object_id' + ) + # internal (not show in UI) - used by scripts to store function name + action_parameters = models.JSONField( + blank=True, + null=True, + ) + action_data = models.JSONField( + verbose_name=_('parameters'), + blank=True, + null=True, + help_text=_("Parameters to pass to the action.") + ) + comments = models.TextField( + verbose_name=_('comments'), + blank=True + ) + + class Meta: + ordering = ('name',) + verbose_name = _('event rule') + verbose_name_plural = _('event rules') + + def __str__(self): + return self.name + + def get_absolute_url(self): + return reverse('extras:eventrule', args=[self.pk]) + + def clean(self): + super().clean() + + # At least one action type must be selected + if not any([ + self.type_create, self.type_update, self.type_delete, self.type_job_start, self.type_job_end + ]): + raise ValidationError( + _("At least one event type must be selected: create, update, delete, job start, and/or job end.") + ) + + # Validate that any conditions are in the correct format + if self.conditions: + try: + ConditionSet(self.conditions) + except ValueError as e: + raise ValidationError({'conditions': e}) + + def eval_conditions(self, data): + """ + Test whether the given data meets the conditions of the event rule (if any). Return True + if met or no conditions are specified. + """ + if not self.conditions: + return True + + return ConditionSet(self.conditions).eval(data) + + +class Webhook(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLoggedModel): + """ + A Webhook defines a request that will be sent to a remote application when an object is created, updated, and/or + delete in NetBox. The request will contain a representation of the object, which the remote application can act on. + Each Webhook can be limited to firing only on certain actions or certain object types. + """ + name = models.CharField( + verbose_name=_('name'), + max_length=150, + unique=True + ) + description = models.CharField( + verbose_name=_('description'), + max_length=200, + blank=True + ) payload_url = models.CharField( max_length=500, verbose_name=_('URL'), @@ -91,10 +195,6 @@ class Webhook(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLoggedMo "processing is supported with the same context as the request body." ) ) - enabled = models.BooleanField( - verbose_name=_('enabled'), - default=True - ) http_method = models.CharField( max_length=30, choices=WebhookHttpMethodChoices, @@ -137,12 +237,6 @@ class Webhook(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLoggedMo "digest of the payload body using the secret as the key. The secret is not transmitted in the request." ) ) - conditions = models.JSONField( - verbose_name=_('conditions'), - blank=True, - null=True, - help_text=_("A set of conditions which determine whether the webhook will be generated.") - ) ssl_verification = models.BooleanField( default=True, verbose_name=_('SSL verification'), @@ -157,15 +251,14 @@ class Webhook(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLoggedMo "The specific CA certificate file to use for SSL verification. Leave blank to use the system defaults." ) ) + events = GenericRelation( + EventRule, + content_type_field='action_object_type', + object_id_field='action_object_id' + ) class Meta: ordering = ('name',) - constraints = ( - models.UniqueConstraint( - fields=('payload_url', 'type_create', 'type_update', 'type_delete'), - name='%(app_label)s_%(class)s_unique_payload_url_types' - ), - ) verbose_name = _('webhook') verbose_name_plural = _('webhooks') @@ -182,20 +275,6 @@ class Webhook(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLoggedMo def clean(self): super().clean() - # At least one action type must be selected - if not any([ - self.type_create, self.type_update, self.type_delete, self.type_job_start, self.type_job_end - ]): - raise ValidationError( - _("At least one event type must be selected: create, update, delete, job_start, and/or job_end.") - ) - - if self.conditions: - try: - ConditionSet(self.conditions) - except ValueError as e: - raise ValidationError({'conditions': e}) - # CA file path requires SSL verification enabled if not self.ssl_verification and self.ca_file_path: raise ValidationError({ diff --git a/netbox/extras/models/reports.py b/netbox/extras/models/reports.py index 223d679bd..f6228ef24 100644 --- a/netbox/extras/models/reports.py +++ b/netbox/extras/models/reports.py @@ -9,7 +9,7 @@ from django.utils.translation import gettext_lazy as _ from core.choices import ManagedFileRootPathChoices from core.models import ManagedFile from extras.utils import is_report -from netbox.models.features import JobsMixin, WebhooksMixin +from netbox.models.features import JobsMixin, EventRulesMixin from utilities.querysets import RestrictedQuerySet from .mixins import PythonModuleMixin @@ -21,7 +21,7 @@ __all__ = ( ) -class Report(WebhooksMixin, models.Model): +class Report(EventRulesMixin, models.Model): """ Dummy model used to generate permissions for reports. Does not exist in the database. """ diff --git a/netbox/extras/models/scripts.py b/netbox/extras/models/scripts.py index 122f56f20..93275acda 100644 --- a/netbox/extras/models/scripts.py +++ b/netbox/extras/models/scripts.py @@ -9,7 +9,7 @@ from django.utils.translation import gettext_lazy as _ from core.choices import ManagedFileRootPathChoices from core.models import ManagedFile from extras.utils import is_script -from netbox.models.features import JobsMixin, WebhooksMixin +from netbox.models.features import JobsMixin, EventRulesMixin from utilities.querysets import RestrictedQuerySet from .mixins import PythonModuleMixin @@ -21,7 +21,7 @@ __all__ = ( logger = logging.getLogger('netbox.data_backends') -class Script(WebhooksMixin, models.Model): +class Script(EventRulesMixin, models.Model): """ Dummy model used to generate permissions for custom scripts. Does not exist in the database. """ diff --git a/netbox/extras/scripts.py b/netbox/extras/scripts.py index df75200e6..495957fd9 100644 --- a/netbox/extras/scripts.py +++ b/netbox/extras/scripts.py @@ -17,13 +17,13 @@ from core.models import Job from extras.api.serializers import ScriptOutputSerializer from extras.choices import LogLevelChoices from extras.models import ScriptModule -from extras.signals import clear_webhooks +from extras.signals import clear_events from ipam.formfields import IPAddressFormField, IPNetworkFormField from ipam.validators import MaxPrefixLengthValidator, MinPrefixLengthValidator, prefix_validator from utilities.exceptions import AbortScript, AbortTransaction from utilities.forms import add_blank_choice from utilities.forms.fields import DynamicModelChoiceField, DynamicModelMultipleChoiceField -from .context_managers import change_logging +from .context_managers import event_tracking from .forms import ScriptForm __all__ = ( @@ -472,10 +472,16 @@ def get_module_and_script(module_name, script_name): return module, script -def run_script(data, request, job, commit=True, **kwargs): +def run_script(data, job, request=None, commit=True, **kwargs): """ A wrapper for calling Script.run(). This performs error handling and provides a hook for committing changes. It exists outside the Script class to ensure it cannot be overridden by a script author. + + Args: + data: A dictionary of data to be passed to the script upon execution + job: The Job associated with this execution + request: The WSGI request associated with this execution (if any) + commit: Passed through to Script.run() """ job.start() @@ -486,9 +492,10 @@ def run_script(data, request, job, commit=True, **kwargs): logger.info(f"Running script (commit={commit})") # Add files to form data - files = request.FILES - for field_name, fileobj in files.items(): - data[field_name] = fileobj + if request: + files = request.FILES + for field_name, fileobj in files.items(): + data[field_name] = fileobj # Add the current request as a property of the script script.request = request @@ -496,7 +503,7 @@ def run_script(data, request, job, commit=True, **kwargs): def _run_script(): """ Core script execution task. We capture this within a subfunction to allow for conditionally wrapping it with - the change_logging context manager (which is bypassed if commit == False). + the event_tracking context manager (which is bypassed if commit == False). """ try: try: @@ -506,7 +513,8 @@ def run_script(data, request, job, commit=True, **kwargs): raise AbortTransaction() except AbortTransaction: script.log_info("Database changes have been reverted automatically.") - clear_webhooks.send(request) + if request: + clear_events.send(request) job.data = ScriptOutputSerializer(script).data job.terminate() except Exception as e: @@ -520,14 +528,15 @@ def run_script(data, request, job, commit=True, **kwargs): script.log_info("Database changes have been reverted due to error.") job.data = ScriptOutputSerializer(script).data job.terminate(status=JobStatusChoices.STATUS_ERRORED, error=str(e)) - clear_webhooks.send(request) + if request: + clear_events.send(request) logger.info(f"Script completed in {job.duration}") - # Execute the script. If commit is True, wrap it with the change_logging context manager to ensure we process - # change logging, webhooks, etc. + # Execute the script. If commit is True, wrap it with the event_tracking context manager to ensure we process + # change logging, event rules, etc. if commit: - with change_logging(request): + with event_tracking(request): _run_script() else: _run_script() diff --git a/netbox/extras/signals.py b/netbox/extras/signals.py index e1d424960..184ee6d9b 100644 --- a/netbox/extras/signals.py +++ b/netbox/extras/signals.py @@ -10,19 +10,19 @@ from django_prometheus.models import model_deletes, model_inserts, model_updates from extras.validators import CustomValidator from netbox.config import get_config -from netbox.context import current_request, webhooks_queue +from netbox.context import current_request, events_queue from netbox.signals import post_clean from utilities.exceptions import AbortRequest from .choices import ObjectChangeActionChoices +from .events import enqueue_object, get_snapshots, serialize_for_event from .models import CustomField, ObjectChange, TaggedItem -from .webhooks import enqueue_object, get_snapshots, serialize_for_webhook # # Change logging/webhooks # -# Define a custom signal that can be sent to clear any queued webhooks -clear_webhooks = Signal() +# Define a custom signal that can be sent to clear any queued events +clear_events = Signal() def is_same_object(instance, webhook_data, request_id): @@ -81,14 +81,14 @@ def handle_changed_object(sender, instance, **kwargs): objectchange.save() # If this is an M2M change, update the previously queued webhook (from post_save) - queue = webhooks_queue.get() + queue = events_queue.get() if m2m_changed and queue and is_same_object(instance, queue[-1], request.id): instance.refresh_from_db() # Ensure that we're working with fresh M2M assignments - queue[-1]['data'] = serialize_for_webhook(instance) + queue[-1]['data'] = serialize_for_event(instance) queue[-1]['snapshots']['postchange'] = get_snapshots(instance, action)['postchange'] else: enqueue_object(queue, instance, request.user, request.id, action) - webhooks_queue.set(queue) + events_queue.set(queue) # Increment metric counters if action == ObjectChangeActionChoices.ACTION_CREATE: @@ -117,22 +117,22 @@ def handle_deleted_object(sender, instance, **kwargs): objectchange.save() # Enqueue webhooks - queue = webhooks_queue.get() + queue = events_queue.get() enqueue_object(queue, instance, request.user, request.id, ObjectChangeActionChoices.ACTION_DELETE) - webhooks_queue.set(queue) + events_queue.set(queue) # Increment metric counters model_deletes.labels(instance._meta.model_name).inc() -@receiver(clear_webhooks) -def clear_webhook_queue(sender, **kwargs): +@receiver(clear_events) +def clear_events_queue(sender, **kwargs): """ - Delete any queued webhooks (e.g. because of an aborted bulk transaction) + Delete any queued events (e.g. because of an aborted bulk transaction) """ - logger = logging.getLogger('webhooks') - logger.info(f"Clearing {len(webhooks_queue.get())} queued webhooks ({sender})") - webhooks_queue.set([]) + logger = logging.getLogger('events') + logger.info(f"Clearing {len(events_queue.get())} queued events ({sender})") + events_queue.set([]) # diff --git a/netbox/extras/tables/tables.py b/netbox/extras/tables/tables.py index d44953632..e02365531 100644 --- a/netbox/extras/tables/tables.py +++ b/netbox/extras/tables/tables.py @@ -15,6 +15,7 @@ __all__ = ( 'CustomFieldChoiceSetTable', 'CustomFieldTable', 'CustomLinkTable', + 'EventRuleTable', 'ExportTemplateTable', 'ImageAttachmentTable', 'JournalEntryTable', @@ -250,6 +251,32 @@ class WebhookTable(NetBoxTable): verbose_name=_('Name'), linkify=True ) + ssl_validation = columns.BooleanColumn( + verbose_name=_('SSL Validation') + ) + tags = columns.TagColumn( + url_name='extras:webhook_list' + ) + + class Meta(NetBoxTable.Meta): + model = Webhook + fields = ( + 'pk', 'id', 'name', 'http_method', 'payload_url', 'http_content_type', 'secret', 'ssl_verification', + 'ca_file_path', 'description', 'tags', 'created', 'last_updated', + ) + default_columns = ( + 'pk', 'name', 'http_method', 'payload_url', 'description', + ) + + +class EventRuleTable(NetBoxTable): + name = tables.Column( + verbose_name=_('Name'), + linkify=True + ) + action_type = tables.Column( + verbose_name=_('Action Type'), + ) content_types = columns.ContentTypesColumn( verbose_name=_('Content Types'), ) @@ -271,23 +298,19 @@ class WebhookTable(NetBoxTable): type_job_end = columns.BooleanColumn( verbose_name=_('Job End') ) - ssl_validation = columns.BooleanColumn( - verbose_name=_('SSL Validation') - ) tags = columns.TagColumn( url_name='extras:webhook_list' ) class Meta(NetBoxTable.Meta): - model = Webhook + model = EventRule fields = ( - 'pk', 'id', 'name', 'content_types', 'enabled', 'type_create', 'type_update', 'type_delete', - 'type_job_start', 'type_job_end', 'http_method', 'payload_url', 'secret', 'ssl_validation', 'ca_file_path', - 'description', 'tags', 'created', 'last_updated', + 'pk', 'id', 'name', 'enabled', 'description', 'action_type', 'content_types', 'type_create', 'type_update', + 'type_delete', 'type_job_start', 'type_job_end', 'tags', 'created', 'last_updated', ) default_columns = ( - 'pk', 'name', 'content_types', 'enabled', 'type_create', 'type_update', 'type_delete', 'type_job_start', - 'type_job_end', 'http_method', 'payload_url', 'description', + 'pk', 'name', 'enabled', 'action_type', 'content_types', 'type_create', 'type_update', 'type_delete', + 'type_job_start', 'type_job_end', ) diff --git a/netbox/extras/tests/test_api.py b/netbox/extras/tests/test_api.py index 025712e36..93be2d2c4 100644 --- a/netbox/extras/tests/test_api.py +++ b/netbox/extras/tests/test_api.py @@ -8,6 +8,7 @@ from rest_framework import status from core.choices import ManagedFileRootPathChoices from dcim.models import Device, DeviceRole, DeviceType, Manufacturer, Rack, Location, RackRole, Site +from extras.choices import * from extras.models import * from extras.reports import Report from extras.scripts import BooleanVar, IntegerVar, Script, StringVar @@ -32,21 +33,15 @@ class WebhookTest(APIViewTestCases.APIViewTestCase): brief_fields = ['display', 'id', 'name', 'url'] create_data = [ { - 'content_types': ['dcim.device', 'dcim.devicetype'], 'name': 'Webhook 4', - 'type_create': True, 'payload_url': 'http://example.com/?4', }, { - 'content_types': ['dcim.device', 'dcim.devicetype'], 'name': 'Webhook 5', - 'type_update': True, 'payload_url': 'http://example.com/?5', }, { - 'content_types': ['dcim.device', 'dcim.devicetype'], 'name': 'Webhook 6', - 'type_delete': True, 'payload_url': 'http://example.com/?6', }, ] @@ -57,29 +52,100 @@ class WebhookTest(APIViewTestCases.APIViewTestCase): @classmethod def setUpTestData(cls): - site_ct = ContentType.objects.get_for_model(Site) - rack_ct = ContentType.objects.get_for_model(Rack) webhooks = ( Webhook( name='Webhook 1', - type_create=True, payload_url='http://example.com/?1', ), Webhook( name='Webhook 2', - type_update=True, payload_url='http://example.com/?1', ), Webhook( name='Webhook 3', - type_delete=True, payload_url='http://example.com/?1', ), ) Webhook.objects.bulk_create(webhooks) - for webhook in webhooks: - webhook.content_types.add(site_ct, rack_ct) + + +class EventRuleTest(APIViewTestCases.APIViewTestCase): + model = EventRule + brief_fields = ['display', 'id', 'name', 'url'] + bulk_update_data = { + 'enabled': False, + 'description': 'New description', + } + update_data = { + 'name': 'Event Rule X', + 'enabled': False, + 'description': 'New description', + } + + @classmethod + def setUpTestData(cls): + webhooks = ( + Webhook( + name='Webhook 1', + payload_url='http://example.com/?1', + ), + Webhook( + name='Webhook 2', + payload_url='http://example.com/?1', + ), + Webhook( + name='Webhook 3', + payload_url='http://example.com/?1', + ), + Webhook( + name='Webhook 4', + payload_url='http://example.com/?1', + ), + Webhook( + name='Webhook 5', + payload_url='http://example.com/?1', + ), + Webhook( + name='Webhook 6', + payload_url='http://example.com/?1', + ), + ) + Webhook.objects.bulk_create(webhooks) + + event_rules = ( + EventRule(name='EventRule 1', type_create=True, action_object=webhooks[0]), + EventRule(name='EventRule 2', type_create=True, action_object=webhooks[1]), + EventRule(name='EventRule 3', type_create=True, action_object=webhooks[2]), + ) + EventRule.objects.bulk_create(event_rules) + + cls.create_data = [ + { + 'name': 'EventRule 4', + 'content_types': ['dcim.device', 'dcim.devicetype'], + 'type_create': True, + 'action_type': EventRuleActionChoices.WEBHOOK, + 'action_object_type': 'extras.webhook', + 'action_object_id': webhooks[3].pk, + }, + { + 'name': 'EventRule 5', + 'content_types': ['dcim.device', 'dcim.devicetype'], + 'type_create': True, + 'action_type': EventRuleActionChoices.WEBHOOK, + 'action_object_type': 'extras.webhook', + 'action_object_id': webhooks[4].pk, + }, + { + 'name': 'EventRule 6', + 'content_types': ['dcim.device', 'dcim.devicetype'], + 'type_create': True, + 'action_type': EventRuleActionChoices.WEBHOOK, + 'action_object_type': 'extras.webhook', + 'action_object_id': webhooks[5].pk, + }, + ] class CustomFieldTest(APIViewTestCases.APIViewTestCase): diff --git a/netbox/extras/tests/test_webhooks.py b/netbox/extras/tests/test_event_rules.py similarity index 72% rename from netbox/extras/tests/test_webhooks.py rename to netbox/extras/tests/test_event_rules.py index ef7637765..ed64ba891 100644 --- a/netbox/extras/tests/test_webhooks.py +++ b/netbox/extras/tests/test_event_rules.py @@ -3,22 +3,22 @@ import uuid from unittest.mock import patch import django_rq +from dcim.choices import SiteStatusChoices +from dcim.models import Site from django.contrib.contenttypes.models import ContentType from django.http import HttpResponse from django.urls import reverse +from extras.choices import EventRuleActionChoices, ObjectChangeActionChoices +from extras.events import enqueue_object, flush_events, serialize_for_event +from extras.models import EventRule, Tag, Webhook +from extras.webhooks import generate_signature +from extras.webhooks_worker import process_webhook from requests import Session from rest_framework import status - -from dcim.choices import SiteStatusChoices -from dcim.models import Site -from extras.choices import ObjectChangeActionChoices -from extras.models import Tag, Webhook -from extras.webhooks import enqueue_object, flush_webhooks, generate_signature, serialize_for_webhook -from extras.webhooks_worker import eval_conditions, process_webhook from utilities.testing import APITestCase -class WebhookTest(APITestCase): +class EventRuleTest(APITestCase): def setUp(self): super().setUp() @@ -35,12 +35,37 @@ class WebhookTest(APITestCase): DUMMY_SECRET = 'LOOKATMEIMASECRETSTRING' webhooks = Webhook.objects.bulk_create(( - Webhook(name='Webhook 1', type_create=True, payload_url=DUMMY_URL, secret=DUMMY_SECRET, additional_headers='X-Foo: Bar'), - Webhook(name='Webhook 2', type_update=True, payload_url=DUMMY_URL, secret=DUMMY_SECRET), - Webhook(name='Webhook 3', type_delete=True, payload_url=DUMMY_URL, secret=DUMMY_SECRET), + Webhook(name='Webhook 1', payload_url=DUMMY_URL, secret=DUMMY_SECRET, additional_headers='X-Foo: Bar'), + Webhook(name='Webhook 2', payload_url=DUMMY_URL, secret=DUMMY_SECRET), + Webhook(name='Webhook 3', payload_url=DUMMY_URL, secret=DUMMY_SECRET), )) - for webhook in webhooks: - webhook.content_types.set([site_ct]) + + ct = ContentType.objects.get(app_label='extras', model='webhook') + event_rules = EventRule.objects.bulk_create(( + EventRule( + name='Webhook Event 1', + type_create=True, + action_type=EventRuleActionChoices.WEBHOOK, + action_object_type=ct, + action_object_id=webhooks[0].id + ), + EventRule( + name='Webhook Event 2', + type_update=True, + action_type=EventRuleActionChoices.WEBHOOK, + action_object_type=ct, + action_object_id=webhooks[0].id + ), + EventRule( + name='Webhook Event 3', + type_delete=True, + action_type=EventRuleActionChoices.WEBHOOK, + action_object_type=ct, + action_object_id=webhooks[0].id + ), + )) + for event_rule in event_rules: + event_rule.content_types.set([site_ct]) Tag.objects.bulk_create(( Tag(name='Foo', slug='foo'), @@ -48,7 +73,42 @@ class WebhookTest(APITestCase): Tag(name='Baz', slug='baz'), )) - def test_enqueue_webhook_create(self): + def test_eventrule_conditions(self): + """ + Test evaluation of EventRule conditions. + """ + event_rule = EventRule( + name='Event Rule 1', + type_create=True, + type_update=True, + conditions={ + 'and': [ + { + 'attr': 'status.value', + 'value': 'active', + } + ] + } + ) + + # Create a Site to evaluate + site = Site.objects.create(name='Site 1', slug='site-1', status=SiteStatusChoices.STATUS_STAGING) + data = serialize_for_event(site) + + # Evaluate the conditions (status='staging') + self.assertFalse(event_rule.eval_conditions(data)) + + # Change the site's status + site.status = SiteStatusChoices.STATUS_ACTIVE + data = serialize_for_event(site) + + # Evaluate the conditions (status='active') + self.assertTrue(event_rule.eval_conditions(data)) + + def test_single_create_process_eventrule(self): + """ + Check that creating an object with an applicable EventRule queues a background task for the rule's action. + """ # Create an object via the REST API data = { 'name': 'Site 1', @@ -65,10 +125,10 @@ class WebhookTest(APITestCase): self.assertEqual(Site.objects.count(), 1) self.assertEqual(Site.objects.first().tags.count(), 2) - # Verify that a job was queued for the object creation webhook + # Verify that a background task was queued for the new object self.assertEqual(self.queue.count, 1) job = self.queue.jobs[0] - self.assertEqual(job.kwargs['webhook'], Webhook.objects.get(type_create=True)) + self.assertEqual(job.kwargs['event_rule'], EventRule.objects.get(type_create=True)) self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_CREATE) self.assertEqual(job.kwargs['model_name'], 'site') self.assertEqual(job.kwargs['data']['id'], response.data['id']) @@ -76,7 +136,11 @@ class WebhookTest(APITestCase): self.assertEqual(job.kwargs['snapshots']['postchange']['name'], 'Site 1') self.assertEqual(job.kwargs['snapshots']['postchange']['tags'], ['Bar', 'Foo']) - def test_enqueue_webhook_bulk_create(self): + def test_bulk_create_process_eventrule(self): + """ + Check that bulk creating multiple objects with an applicable EventRule queues a background task for each + new object. + """ # Create multiple objects via the REST API data = [ { @@ -111,10 +175,10 @@ class WebhookTest(APITestCase): self.assertEqual(Site.objects.count(), 3) self.assertEqual(Site.objects.first().tags.count(), 2) - # Verify that a webhook was queued for each object + # Verify that a background task was queued for each new object self.assertEqual(self.queue.count, 3) for i, job in enumerate(self.queue.jobs): - self.assertEqual(job.kwargs['webhook'], Webhook.objects.get(type_create=True)) + self.assertEqual(job.kwargs['event_rule'], EventRule.objects.get(type_create=True)) self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_CREATE) self.assertEqual(job.kwargs['model_name'], 'site') self.assertEqual(job.kwargs['data']['id'], response.data[i]['id']) @@ -122,7 +186,10 @@ class WebhookTest(APITestCase): self.assertEqual(job.kwargs['snapshots']['postchange']['name'], response.data[i]['name']) self.assertEqual(job.kwargs['snapshots']['postchange']['tags'], ['Bar', 'Foo']) - def test_enqueue_webhook_update(self): + def test_single_update_process_eventrule(self): + """ + Check that updating an object with an applicable EventRule queues a background task for the rule's action. + """ site = Site.objects.create(name='Site 1', slug='site-1') site.tags.set(Tag.objects.filter(name__in=['Foo', 'Bar'])) @@ -139,10 +206,10 @@ class WebhookTest(APITestCase): response = self.client.patch(url, data, format='json', **self.header) self.assertHttpStatus(response, status.HTTP_200_OK) - # Verify that a job was queued for the object update webhook + # Verify that a background task was queued for the updated object self.assertEqual(self.queue.count, 1) job = self.queue.jobs[0] - self.assertEqual(job.kwargs['webhook'], Webhook.objects.get(type_update=True)) + self.assertEqual(job.kwargs['event_rule'], EventRule.objects.get(type_update=True)) self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_UPDATE) self.assertEqual(job.kwargs['model_name'], 'site') self.assertEqual(job.kwargs['data']['id'], site.pk) @@ -152,7 +219,11 @@ class WebhookTest(APITestCase): self.assertEqual(job.kwargs['snapshots']['postchange']['name'], 'Site X') self.assertEqual(job.kwargs['snapshots']['postchange']['tags'], ['Baz']) - def test_enqueue_webhook_bulk_update(self): + def test_bulk_update_process_eventrule(self): + """ + Check that bulk updating multiple objects with an applicable EventRule queues a background task for each + updated object. + """ sites = ( Site(name='Site 1', slug='site-1'), Site(name='Site 2', slug='site-2'), @@ -191,10 +262,10 @@ class WebhookTest(APITestCase): response = self.client.patch(url, data, format='json', **self.header) self.assertHttpStatus(response, status.HTTP_200_OK) - # Verify that a job was queued for the object update webhook + # Verify that a background task was queued for each updated object self.assertEqual(self.queue.count, 3) for i, job in enumerate(self.queue.jobs): - self.assertEqual(job.kwargs['webhook'], Webhook.objects.get(type_update=True)) + self.assertEqual(job.kwargs['event_rule'], EventRule.objects.get(type_update=True)) self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_UPDATE) self.assertEqual(job.kwargs['model_name'], 'site') self.assertEqual(job.kwargs['data']['id'], data[i]['id']) @@ -204,7 +275,10 @@ class WebhookTest(APITestCase): self.assertEqual(job.kwargs['snapshots']['postchange']['name'], response.data[i]['name']) self.assertEqual(job.kwargs['snapshots']['postchange']['tags'], ['Baz']) - def test_enqueue_webhook_delete(self): + def test_single_delete_process_eventrule(self): + """ + Check that deleting an object with an applicable EventRule queues a background task for the rule's action. + """ site = Site.objects.create(name='Site 1', slug='site-1') site.tags.set(Tag.objects.filter(name__in=['Foo', 'Bar'])) @@ -214,17 +288,21 @@ class WebhookTest(APITestCase): response = self.client.delete(url, **self.header) self.assertHttpStatus(response, status.HTTP_204_NO_CONTENT) - # Verify that a job was queued for the object update webhook + # Verify that a task was queued for the deleted object self.assertEqual(self.queue.count, 1) job = self.queue.jobs[0] - self.assertEqual(job.kwargs['webhook'], Webhook.objects.get(type_delete=True)) + self.assertEqual(job.kwargs['event_rule'], EventRule.objects.get(type_delete=True)) self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_DELETE) self.assertEqual(job.kwargs['model_name'], 'site') self.assertEqual(job.kwargs['data']['id'], site.pk) self.assertEqual(job.kwargs['snapshots']['prechange']['name'], 'Site 1') self.assertEqual(job.kwargs['snapshots']['prechange']['tags'], ['Bar', 'Foo']) - def test_enqueue_webhook_bulk_delete(self): + def test_bulk_delete_process_eventrule(self): + """ + Check that bulk deleting multiple objects with an applicable EventRule queues a background task for each + deleted object. + """ sites = ( Site(name='Site 1', slug='site-1'), Site(name='Site 2', slug='site-2'), @@ -243,49 +321,17 @@ class WebhookTest(APITestCase): response = self.client.delete(url, data, format='json', **self.header) self.assertHttpStatus(response, status.HTTP_204_NO_CONTENT) - # Verify that a job was queued for the object update webhook + # Verify that a background task was queued for each deleted object self.assertEqual(self.queue.count, 3) for i, job in enumerate(self.queue.jobs): - self.assertEqual(job.kwargs['webhook'], Webhook.objects.get(type_delete=True)) + self.assertEqual(job.kwargs['event_rule'], EventRule.objects.get(type_delete=True)) self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_DELETE) self.assertEqual(job.kwargs['model_name'], 'site') self.assertEqual(job.kwargs['data']['id'], sites[i].pk) self.assertEqual(job.kwargs['snapshots']['prechange']['name'], sites[i].name) self.assertEqual(job.kwargs['snapshots']['prechange']['tags'], ['Bar', 'Foo']) - def test_webhook_conditions(self): - # Create a conditional Webhook - webhook = Webhook( - name='Conditional Webhook', - type_create=True, - type_update=True, - payload_url='http://localhost:9000/', - conditions={ - 'and': [ - { - 'attr': 'status.value', - 'value': 'active', - } - ] - } - ) - - # Create a Site to evaluate - site = Site.objects.create(name='Site 1', slug='site-1', status=SiteStatusChoices.STATUS_STAGING) - data = serialize_for_webhook(site) - - # Evaluate the conditions (status='staging') - self.assertFalse(eval_conditions(webhook, data)) - - # Change the site's status - site.status = SiteStatusChoices.STATUS_ACTIVE - data = serialize_for_webhook(site) - - # Evaluate the conditions (status='active') - self.assertTrue(eval_conditions(webhook, data)) - def test_webhooks_worker(self): - request_id = uuid.uuid4() def dummy_send(_, request, **kwargs): @@ -293,7 +339,8 @@ class WebhookTest(APITestCase): A dummy implementation of Session.send() to be used for testing. Always returns a 200 HTTP response. """ - webhook = Webhook.objects.get(type_create=True) + event = EventRule.objects.get(type_create=True) + webhook = event.action_object signature = generate_signature(request.body, webhook.secret) # Validate the outgoing request headers @@ -322,7 +369,7 @@ class WebhookTest(APITestCase): request_id=request_id, action=ObjectChangeActionChoices.ACTION_CREATE ) - flush_webhooks(webhooks_queue) + flush_events(webhooks_queue) # Retrieve the job from queue job = self.queue.jobs[0] diff --git a/netbox/extras/tests/test_filtersets.py b/netbox/extras/tests/test_filtersets.py index c5a6706c0..ddc5feb40 100644 --- a/netbox/extras/tests/test_filtersets.py +++ b/netbox/extras/tests/test_filtersets.py @@ -6,6 +6,7 @@ from django.contrib.contenttypes.models import ContentType from django.test import TestCase from circuits.models import Provider +from core.choices import ManagedFileRootPathChoices from dcim.filtersets import SiteFilterSet from dcim.models import DeviceRole, DeviceType, Manufacturer, Platform, Rack, Region, Site, SiteGroup from dcim.models import Location @@ -159,82 +160,174 @@ class WebhookTestCase(TestCase, BaseFilterSetTests): webhooks = ( Webhook( name='Webhook 1', - type_create=True, - type_update=False, - type_delete=False, - type_job_start=False, - type_job_end=False, payload_url='http://example.com/?1', - enabled=True, http_method='GET', ssl_verification=True, ), Webhook( name='Webhook 2', - type_create=False, - type_update=True, - type_delete=False, - type_job_start=False, - type_job_end=False, payload_url='http://example.com/?2', - enabled=True, http_method='POST', ssl_verification=True, ), Webhook( name='Webhook 3', - type_create=False, - type_update=False, - type_delete=True, - type_job_start=False, - type_job_end=False, payload_url='http://example.com/?3', - enabled=False, http_method='PATCH', ssl_verification=False, ), Webhook( name='Webhook 4', - type_create=False, - type_update=False, - type_delete=False, - type_job_start=True, - type_job_end=False, payload_url='http://example.com/?4', - enabled=False, http_method='PATCH', ssl_verification=False, ), Webhook( name='Webhook 5', - type_create=False, - type_update=False, - type_delete=False, - type_job_start=False, - type_job_end=True, payload_url='http://example.com/?5', - enabled=False, http_method='PATCH', ssl_verification=False, ), ) Webhook.objects.bulk_create(webhooks) - webhooks[0].content_types.add(content_types[0]) - webhooks[1].content_types.add(content_types[1]) - webhooks[2].content_types.add(content_types[2]) - webhooks[3].content_types.add(content_types[3]) - webhooks[4].content_types.add(content_types[4]) def test_name(self): params = {'name': ['Webhook 1', 'Webhook 2']} self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + def test_http_method(self): + params = {'http_method': ['GET', 'POST']} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + + def test_ssl_verification(self): + params = {'ssl_verification': True} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + + +class EventRuleTestCase(TestCase, BaseFilterSetTests): + queryset = EventRule.objects.all() + filterset = EventRuleFilterSet + + @classmethod + def setUpTestData(cls): + content_types = ContentType.objects.filter( + model__in=['region', 'site', 'rack', 'location', 'device'] + ) + + webhooks = ( + Webhook( + name='Webhook 1', + payload_url='http://example.com/?1', + ), + Webhook( + name='Webhook 2', + payload_url='http://example.com/?2', + ), + Webhook( + name='Webhook 3', + payload_url='http://example.com/?3', + ), + ) + Webhook.objects.bulk_create(webhooks) + + scripts = ( + ScriptModule( + file_root=ManagedFileRootPathChoices.SCRIPTS, + file_path='/var/tmp/script1.py' + ), + ScriptModule( + file_root=ManagedFileRootPathChoices.SCRIPTS, + file_path='/var/tmp/script2.py' + ), + ) + ScriptModule.objects.bulk_create(scripts) + + event_rules = ( + EventRule( + name='Event Rule 1', + action_object=webhooks[0], + enabled=True, + type_create=True, + type_update=False, + type_delete=False, + type_job_start=False, + type_job_end=False, + action_type=EventRuleActionChoices.WEBHOOK, + ), + EventRule( + name='Event Rule 2', + action_object=webhooks[1], + enabled=True, + type_create=False, + type_update=True, + type_delete=False, + type_job_start=False, + type_job_end=False, + action_type=EventRuleActionChoices.WEBHOOK, + ), + EventRule( + name='Event Rule 3', + action_object=webhooks[2], + enabled=False, + type_create=False, + type_update=False, + type_delete=True, + type_job_start=False, + type_job_end=False, + action_type=EventRuleActionChoices.WEBHOOK, + ), + EventRule( + name='Event Rule 4', + action_object=scripts[0], + enabled=False, + type_create=False, + type_update=False, + type_delete=False, + type_job_start=True, + type_job_end=False, + action_type=EventRuleActionChoices.SCRIPT, + ), + EventRule( + name='Event Rule 5', + action_object=scripts[1], + enabled=False, + type_create=False, + type_update=False, + type_delete=False, + type_job_start=False, + type_job_end=True, + action_type=EventRuleActionChoices.SCRIPT, + ), + ) + EventRule.objects.bulk_create(event_rules) + event_rules[0].content_types.add(content_types[0]) + event_rules[1].content_types.add(content_types[1]) + event_rules[2].content_types.add(content_types[2]) + event_rules[3].content_types.add(content_types[3]) + event_rules[4].content_types.add(content_types[4]) + + def test_name(self): + params = {'name': ['Event Rule 1', 'Event Rule 2']} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + def test_content_types(self): params = {'content_types': 'dcim.region'} self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) params = {'content_type_id': [ContentType.objects.get_for_model(Region).pk]} self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) + def test_action_type(self): + params = {'action_type': [EventRuleActionChoices.WEBHOOK]} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 3) + params = {'action_type': [EventRuleActionChoices.SCRIPT]} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + + def test_enabled(self): + params = {'enabled': True} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + params = {'enabled': False} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 3) + def test_type_create(self): params = {'type_create': True} self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) @@ -255,18 +348,6 @@ class WebhookTestCase(TestCase, BaseFilterSetTests): params = {'type_job_end': True} self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) - def test_enabled(self): - params = {'enabled': True} - self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) - - def test_http_method(self): - params = {'http_method': ['GET', 'POST']} - self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) - - def test_ssl_verification(self): - params = {'ssl_verification': True} - self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) - class CustomLinkTestCase(TestCase, BaseFilterSetTests): queryset = CustomLink.objects.all() diff --git a/netbox/extras/tests/test_views.py b/netbox/extras/tests/test_views.py index aa0a21909..dcb351f75 100644 --- a/netbox/extras/tests/test_views.py +++ b/netbox/extras/tests/test_views.py @@ -1,4 +1,3 @@ -import json import urllib.parse import uuid @@ -11,7 +10,6 @@ from extras.choices import * from extras.models import * from utilities.testing import ViewTestCases, TestCase - User = get_user_model() @@ -336,34 +334,27 @@ class WebhookTestCase(ViewTestCases.PrimaryObjectViewTestCase): @classmethod def setUpTestData(cls): - site_ct = ContentType.objects.get_for_model(Site) webhooks = ( - Webhook(name='Webhook 1', payload_url='http://example.com/?1', type_create=True, http_method='POST'), - Webhook(name='Webhook 2', payload_url='http://example.com/?2', type_create=True, http_method='POST'), - Webhook(name='Webhook 3', payload_url='http://example.com/?3', type_create=True, http_method='POST'), + Webhook(name='Webhook 1', payload_url='http://example.com/?1', http_method='POST'), + Webhook(name='Webhook 2', payload_url='http://example.com/?2', http_method='POST'), + Webhook(name='Webhook 3', payload_url='http://example.com/?3', http_method='POST'), ) for webhook in webhooks: webhook.save() - webhook.content_types.add(site_ct) cls.form_data = { 'name': 'Webhook X', - 'content_types': [site_ct.pk], - 'type_create': False, - 'type_update': True, - 'type_delete': True, 'payload_url': 'http://example.com/?x', 'http_method': 'GET', 'http_content_type': 'application/foo', - 'conditions': None, 'description': 'My webhook', } cls.csv_data = ( - "name,content_types,type_create,payload_url,http_method,http_content_type,description", - "Webhook 4,dcim.site,True,http://example.com/?4,GET,application/json,Foo", - "Webhook 5,dcim.site,True,http://example.com/?5,GET,application/json,Bar", - "Webhook 6,dcim.site,True,http://example.com/?6,GET,application/json,Baz", + "name,payload_url,http_method,http_content_type,description", + "Webhook 4,http://example.com/?4,GET,application/json,Foo", + "Webhook 5,http://example.com/?5,GET,application/json,Bar", + "Webhook 6,http://example.com/?6,GET,application/json,Baz", ) cls.csv_update_data = ( @@ -374,14 +365,65 @@ class WebhookTestCase(ViewTestCases.PrimaryObjectViewTestCase): ) cls.bulk_edit_data = { - 'enabled': False, + 'http_method': 'GET', + } + + +class EventRulesTestCase(ViewTestCases.PrimaryObjectViewTestCase): + model = EventRule + + @classmethod + def setUpTestData(cls): + + webhooks = ( + Webhook(name='Webhook 1', payload_url='http://example.com/?1', http_method='POST'), + Webhook(name='Webhook 2', payload_url='http://example.com/?2', http_method='POST'), + Webhook(name='Webhook 3', payload_url='http://example.com/?3', http_method='POST'), + ) + for webhook in webhooks: + webhook.save() + + site_ct = ContentType.objects.get_for_model(Site) + event_rules = ( + EventRule(name='EventRule 1', type_create=True, action_object=webhooks[0]), + EventRule(name='EventRule 2', type_create=True, action_object=webhooks[1]), + EventRule(name='EventRule 3', type_create=True, action_object=webhooks[2]), + ) + for event in event_rules: + event.save() + event.content_types.add(site_ct) + + webhook_ct = ContentType.objects.get_for_model(Webhook) + cls.form_data = { + 'name': 'Event X', + 'content_types': [site_ct.pk], 'type_create': False, 'type_update': True, 'type_delete': True, - 'http_method': 'GET', + 'conditions': None, + 'action_type': 'webhook', + 'action_object_type': webhook_ct.pk, + 'action_object_id': webhooks[0].pk, + 'action_choice': webhooks[0], 'description': 'New description', } + cls.csv_data = ( + "name,content_types,type_create,action_type,action_object", + "Webhook 4,dcim.site,True,webhook,Webhook 1", + ) + + cls.csv_update_data = ( + "id,name", + f"{event_rules[0].pk},Event 7", + f"{event_rules[1].pk},Event 8", + f"{event_rules[2].pk},Event 9", + ) + + cls.bulk_edit_data = { + 'type_update': True, + } + class TagTestCase(ViewTestCases.OrganizationalObjectViewTestCase): model = Tag diff --git a/netbox/extras/urls.py b/netbox/extras/urls.py index bcab007e7..0a1786f1f 100644 --- a/netbox/extras/urls.py +++ b/netbox/extras/urls.py @@ -61,6 +61,14 @@ urlpatterns = [ path('webhooks/delete/', views.WebhookBulkDeleteView.as_view(), name='webhook_bulk_delete'), path('webhooks//', include(get_model_urls('extras', 'webhook'))), + # Event rules + path('event-rules/', views.EventRuleListView.as_view(), name='eventrule_list'), + path('event-rules/add/', views.EventRuleEditView.as_view(), name='eventrule_add'), + path('event-rules/import/', views.EventRuleBulkImportView.as_view(), name='eventrule_import'), + path('event-rules/edit/', views.EventRuleBulkEditView.as_view(), name='eventrule_bulk_edit'), + path('event-rules/delete/', views.EventRuleBulkDeleteView.as_view(), name='eventrule_bulk_delete'), + path('event-rules//', include(get_model_urls('extras', 'eventrule'))), + # Tags path('tags/', views.TagListView.as_view(), name='tag_list'), path('tags/add/', views.TagEditView.as_view(), name='tag_add'), diff --git a/netbox/extras/views.py b/netbox/extras/views.py index b62165e1a..a3dd7f193 100644 --- a/netbox/extras/views.py +++ b/netbox/extras/views.py @@ -395,6 +395,51 @@ class WebhookBulkDeleteView(generic.BulkDeleteView): table = tables.WebhookTable +# +# Event Rules +# + +class EventRuleListView(generic.ObjectListView): + queryset = EventRule.objects.all() + filterset = filtersets.EventRuleFilterSet + filterset_form = forms.EventRuleFilterForm + table = tables.EventRuleTable + + +@register_model_view(EventRule) +class EventRuleView(generic.ObjectView): + queryset = EventRule.objects.all() + + +@register_model_view(EventRule, 'edit') +class EventRuleEditView(generic.ObjectEditView): + queryset = EventRule.objects.all() + form = forms.EventRuleForm + + +@register_model_view(EventRule, 'delete') +class EventRuleDeleteView(generic.ObjectDeleteView): + queryset = EventRule.objects.all() + + +class EventRuleBulkImportView(generic.BulkImportView): + queryset = EventRule.objects.all() + model_form = forms.EventRuleImportForm + + +class EventRuleBulkEditView(generic.BulkEditView): + queryset = EventRule.objects.all() + filterset = filtersets.EventRuleFilterSet + table = tables.EventRuleTable + form = forms.EventRuleBulkEditForm + + +class EventRuleBulkDeleteView(generic.BulkDeleteView): + queryset = EventRule.objects.all() + filterset = filtersets.EventRuleFilterSet + table = tables.EventRuleTable + + # # Tags # @@ -1093,7 +1138,7 @@ class ReportJobsView(ContentTypePermissionRequiredMixin, View): jobs = Job.objects.filter( object_type=object_type, object_id=module.pk, - name=report.name + name=report.class_name ) jobs_table = JobTable( diff --git a/netbox/extras/webhooks.py b/netbox/extras/webhooks.py index 1fc869ee8..a48a8038b 100644 --- a/netbox/extras/webhooks.py +++ b/netbox/extras/webhooks.py @@ -1,47 +1,6 @@ import hashlib import hmac -from django.contrib.contenttypes.models import ContentType -from django.utils import timezone -from django_rq import get_queue - -from netbox.config import get_config -from netbox.constants import RQ_QUEUE_DEFAULT -from netbox.registry import registry -from utilities.api import get_serializer_for_model -from utilities.rqworker import get_rq_retry -from utilities.utils import serialize_object -from .choices import * -from .models import Webhook - - -def serialize_for_webhook(instance): - """ - Return a serialized representation of the given instance suitable for use in a webhook. - """ - serializer_class = get_serializer_for_model(instance.__class__) - serializer_context = { - 'request': None, - } - serializer = serializer_class(instance, context=serializer_context) - - return serializer.data - - -def get_snapshots(instance, action): - snapshots = { - 'prechange': getattr(instance, '_prechange_snapshot', None), - 'postchange': None, - } - if action != ObjectChangeActionChoices.ACTION_DELETE: - # Use model's serialize_object() method if defined; fall back to serialize_object() utility function - if hasattr(instance, 'serialize_object'): - snapshots['postchange'] = instance.serialize_object() - else: - snapshots['postchange'] = serialize_object(instance) - - return snapshots - def generate_signature(request_body, secret): """ @@ -53,70 +12,3 @@ def generate_signature(request_body, secret): digestmod=hashlib.sha512 ) return hmac_prep.hexdigest() - - -def enqueue_object(queue, instance, user, request_id, action): - """ - Enqueue a serialized representation of a created/updated/deleted object for the processing of - webhooks once the request has completed. - """ - # Determine whether this type of object supports webhooks - app_label = instance._meta.app_label - model_name = instance._meta.model_name - if model_name not in registry['model_features']['webhooks'].get(app_label, []): - return - - queue.append({ - 'content_type': ContentType.objects.get_for_model(instance), - 'object_id': instance.pk, - 'event': action, - 'data': serialize_for_webhook(instance), - 'snapshots': get_snapshots(instance, action), - 'username': user.username, - 'request_id': request_id - }) - - -def flush_webhooks(queue): - """ - Flush a list of object representation to RQ for webhook processing. - """ - rq_queue_name = get_config().QUEUE_MAPPINGS.get('webhook', RQ_QUEUE_DEFAULT) - rq_queue = get_queue(rq_queue_name) - webhooks_cache = { - 'type_create': {}, - 'type_update': {}, - 'type_delete': {}, - } - - for data in queue: - - action_flag = { - ObjectChangeActionChoices.ACTION_CREATE: 'type_create', - ObjectChangeActionChoices.ACTION_UPDATE: 'type_update', - ObjectChangeActionChoices.ACTION_DELETE: 'type_delete', - }[data['event']] - content_type = data['content_type'] - - # Cache applicable Webhooks - if content_type not in webhooks_cache[action_flag]: - webhooks_cache[action_flag][content_type] = Webhook.objects.filter( - **{action_flag: True}, - content_types=content_type, - enabled=True - ) - webhooks = webhooks_cache[action_flag][content_type] - - for webhook in webhooks: - rq_queue.enqueue( - "extras.webhooks_worker.process_webhook", - webhook=webhook, - model_name=content_type.model, - event=data['event'], - data=data['data'], - snapshots=data['snapshots'], - timestamp=str(timezone.now()), - username=data['username'], - request_id=data['request_id'], - retry=get_rq_retry() - ) diff --git a/netbox/extras/webhooks_worker.py b/netbox/extras/webhooks_worker.py index 438231b7e..4d6d8135e 100644 --- a/netbox/extras/webhooks_worker.py +++ b/netbox/extras/webhooks_worker.py @@ -5,36 +5,18 @@ from django.conf import settings from django_rq import job from jinja2.exceptions import TemplateError -from .conditions import ConditionSet from .constants import WEBHOOK_EVENT_TYPES from .webhooks import generate_signature logger = logging.getLogger('netbox.webhooks_worker') -def eval_conditions(webhook, data): - """ - Test whether the given data meets the conditions of the webhook (if any). Return True - if met or no conditions are specified. - """ - if not webhook.conditions: - return True - - logger.debug(f'Evaluating webhook conditions: {webhook.conditions}') - if ConditionSet(webhook.conditions).eval(data): - return True - - return False - - @job('default') -def process_webhook(webhook, model_name, event, data, timestamp, username, request_id=None, snapshots=None): +def process_webhook(event_rule, model_name, event, data, timestamp, username, request_id=None, snapshots=None): """ Make a POST request to the defined Webhook """ - # Evaluate webhook conditions (if any) - if not eval_conditions(webhook, data): - return + webhook = event_rule.action_object # Prepare context data for headers & body templates context = { diff --git a/netbox/ipam/tables/asn.py b/netbox/ipam/tables/asn.py index 6bb15523e..bbe38dc1a 100644 --- a/netbox/ipam/tables/asn.py +++ b/netbox/ipam/tables/asn.py @@ -48,6 +48,7 @@ class ASNTable(TenancyColumnsMixin, NetBoxTable): asn_asdot = tables.Column( accessor=tables.A('asn_asdot'), linkify=True, + order_by=tables.A('asn'), verbose_name=_('ASDOT') ) site_count = columns.LinkedCountColumn( diff --git a/netbox/netbox/context.py b/netbox/netbox/context.py index 5461a4e94..56e41cb63 100644 --- a/netbox/netbox/context.py +++ b/netbox/netbox/context.py @@ -2,9 +2,9 @@ from contextvars import ContextVar __all__ = ( 'current_request', - 'webhooks_queue', + 'events_queue', ) current_request = ContextVar('current_request', default=None) -webhooks_queue = ContextVar('webhooks_queue', default=[]) +events_queue = ContextVar('events_queue', default=[]) diff --git a/netbox/netbox/middleware.py b/netbox/netbox/middleware.py index 18f350fd7..cb7d2c8ba 100644 --- a/netbox/netbox/middleware.py +++ b/netbox/netbox/middleware.py @@ -10,7 +10,7 @@ from django.db import connection, ProgrammingError from django.db.utils import InternalError from django.http import Http404, HttpResponseRedirect -from extras.context_managers import change_logging +from extras.context_managers import event_tracking from netbox.config import clear_config, get_config from netbox.views import handler_500 from utilities.api import is_api_request, rest_api_server_error @@ -42,8 +42,8 @@ class CoreMiddleware: login_url = f'{settings.LOGIN_URL}?next={parse.quote(request.get_full_path_info())}' return HttpResponseRedirect(login_url) - # Enable the change_logging context manager and process the request. - with change_logging(request): + # Enable the event_tracking context manager and process the request. + with event_tracking(request): response = self.get_response(request) # Attach the unique request ID as an HTTP header. diff --git a/netbox/netbox/models/__init__.py b/netbox/netbox/models/__init__.py index 9d7696696..2c262b258 100644 --- a/netbox/netbox/models/__init__.py +++ b/netbox/netbox/models/__init__.py @@ -30,7 +30,7 @@ class NetBoxFeatureSet( ExportTemplatesMixin, JournalingMixin, TagsMixin, - WebhooksMixin + EventRulesMixin ): class Meta: abstract = True @@ -44,7 +44,7 @@ class NetBoxFeatureSet( # Base model classes # -class ChangeLoggedModel(ChangeLoggingMixin, CustomValidationMixin, WebhooksMixin, models.Model): +class ChangeLoggedModel(ChangeLoggingMixin, CustomValidationMixin, EventRulesMixin, models.Model): """ Base model for ancillary models; provides limited functionality for models which don't support NetBox's full feature set. diff --git a/netbox/netbox/models/features.py b/netbox/netbox/models/features.py index f39f35620..ac9893e20 100644 --- a/netbox/netbox/models/features.py +++ b/netbox/netbox/models/features.py @@ -35,7 +35,7 @@ __all__ = ( 'JournalingMixin', 'SyncedDataMixin', 'TagsMixin', - 'WebhooksMixin', + 'EventRulesMixin', ) @@ -400,9 +400,9 @@ class TagsMixin(models.Model): abstract = True -class WebhooksMixin(models.Model): +class EventRulesMixin(models.Model): """ - Enables support for webhooks. + Enables support for event rules, which can be used to transmit webhooks or execute scripts automatically. """ class Meta: abstract = True @@ -555,7 +555,7 @@ FEATURES_MAP = { 'journaling': JournalingMixin, 'synced_data': SyncedDataMixin, 'tags': TagsMixin, - 'webhooks': WebhooksMixin, + 'event_rules': EventRulesMixin, } registry['model_features'].update({ diff --git a/netbox/netbox/navigation/menu.py b/netbox/netbox/navigation/menu.py index 49aee3540..e01e65cc8 100644 --- a/netbox/netbox/navigation/menu.py +++ b/netbox/netbox/navigation/menu.py @@ -343,6 +343,7 @@ OPERATIONS_MENU = Menu( label=_('Integrations'), items=( get_model_item('core', 'datasource', _('Data Sources')), + get_model_item('extras', 'eventrule', _('Event Rules')), get_model_item('extras', 'webhook', _('Webhooks')), ), ), diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index ce8ab5876..1181229f2 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -27,7 +27,7 @@ from netbox.plugins import PluginConfig # Environment setup # -VERSION = '3.6.6-dev' +VERSION = '3.6.7-dev' # Hostname HOSTNAME = platform.node() @@ -115,6 +115,9 @@ DEFAULT_PERMISSIONS = getattr(configuration, 'DEFAULT_PERMISSIONS', { DEVELOPER = getattr(configuration, 'DEVELOPER', False) DOCS_ROOT = getattr(configuration, 'DOCS_ROOT', os.path.join(os.path.dirname(BASE_DIR), 'docs')) EMAIL = getattr(configuration, 'EMAIL', {}) +EVENTS_PIPELINE = getattr(configuration, 'EVENTS_PIPELINE', ( + 'extras.events.process_event_queue', +)) EXEMPT_VIEW_PERMISSIONS = getattr(configuration, 'EXEMPT_VIEW_PERMISSIONS', []) FIELD_CHOICES = getattr(configuration, 'FIELD_CHOICES', {}) FILE_UPLOAD_MAX_MEMORY_SIZE = getattr(configuration, 'FILE_UPLOAD_MAX_MEMORY_SIZE', 2621440) @@ -672,7 +675,7 @@ GRAPHENE = { # -# Django RQ (Webhooks backend) +# Django RQ (events backend) # if TASKS_REDIS_USING_SENTINEL: diff --git a/netbox/netbox/views/generic/bulk_views.py b/netbox/netbox/views/generic/bulk_views.py index fbe3aa2ba..4764642b3 100644 --- a/netbox/netbox/views/generic/bulk_views.py +++ b/netbox/netbox/views/generic/bulk_views.py @@ -17,7 +17,7 @@ from django.utils.safestring import mark_safe from django_tables2.export import TableExport from extras.models import ExportTemplate -from extras.signals import clear_webhooks +from extras.signals import clear_events from utilities.error_handlers import handle_protectederror from utilities.exceptions import AbortRequest, AbortTransaction, PermissionsViolation from utilities.forms import BulkRenameForm, ConfirmationForm, restrict_form_fields @@ -279,7 +279,7 @@ class BulkCreateView(GetReturnURLMixin, BaseMultiObjectView): except (AbortRequest, PermissionsViolation) as e: logger.debug(e.message) form.add_error(None, e.message) - clear_webhooks.send(sender=self) + clear_events.send(sender=self) else: logger.debug("Form validation failed") @@ -394,6 +394,10 @@ class BulkImportView(GetReturnURLMixin, BaseMultiObjectView): form.add_error('data', f"Row {i}: Object with ID {object_id} does not exist") raise ValidationError('') + # Take a snapshot for change logging + if instance.pk and hasattr(instance, 'snapshot'): + instance.snapshot() + # Instantiate the model form for the object model_form_kwargs = { 'data': record, @@ -470,12 +474,12 @@ class BulkImportView(GetReturnURLMixin, BaseMultiObjectView): return redirect(results_url) except (AbortTransaction, ValidationError): - clear_webhooks.send(sender=self) + clear_events.send(sender=self) except (AbortRequest, PermissionsViolation) as e: logger.debug(e.message) form.add_error(None, e.message) - clear_webhooks.send(sender=self) + clear_events.send(sender=self) else: logger.debug("Form validation failed") @@ -628,12 +632,12 @@ class BulkEditView(GetReturnURLMixin, BaseMultiObjectView): except ValidationError as e: messages.error(self.request, ", ".join(e.messages)) - clear_webhooks.send(sender=self) + clear_events.send(sender=self) except (AbortRequest, PermissionsViolation) as e: logger.debug(e.message) form.add_error(None, e.message) - clear_webhooks.send(sender=self) + clear_events.send(sender=self) else: logger.debug("Form validation failed") @@ -729,7 +733,7 @@ class BulkRenameView(GetReturnURLMixin, BaseMultiObjectView): except (AbortRequest, PermissionsViolation) as e: logger.debug(e.message) form.add_error(None, e.message) - clear_webhooks.send(sender=self) + clear_events.send(sender=self) else: form = self.form(initial={'pk': request.POST.getlist('pk')}) @@ -923,12 +927,12 @@ class BulkComponentCreateView(GetReturnURLMixin, BaseMultiObjectView): raise PermissionsViolation except IntegrityError: - clear_webhooks.send(sender=self) + clear_events.send(sender=self) except (AbortRequest, PermissionsViolation) as e: logger.debug(e.message) form.add_error(None, e.message) - clear_webhooks.send(sender=self) + clear_events.send(sender=self) if not form.errors: msg = "Added {} {} to {} {}.".format( diff --git a/netbox/netbox/views/generic/object_views.py b/netbox/netbox/views/generic/object_views.py index 99508c9e3..456c2e14f 100644 --- a/netbox/netbox/views/generic/object_views.py +++ b/netbox/netbox/views/generic/object_views.py @@ -11,7 +11,7 @@ from django.urls import reverse from django.utils.html import escape from django.utils.safestring import mark_safe -from extras.signals import clear_webhooks +from extras.signals import clear_events from utilities.error_handlers import handle_protectederror from utilities.exceptions import AbortRequest, PermissionsViolation from utilities.forms import ConfirmationForm, restrict_form_fields @@ -300,7 +300,7 @@ class ObjectEditView(GetReturnURLMixin, BaseObjectView): except (AbortRequest, PermissionsViolation) as e: logger.debug(e.message) form.add_error(None, e.message) - clear_webhooks.send(sender=self) + clear_events.send(sender=self) else: logger.debug("Form validation failed") @@ -528,7 +528,7 @@ class ComponentCreateView(GetReturnURLMixin, BaseObjectView): except (AbortRequest, PermissionsViolation) as e: logger.debug(e.message) form.add_error(None, e.message) - clear_webhooks.send(sender=self) + clear_events.send(sender=self) return render(request, self.template_name, { 'object': instance, diff --git a/netbox/templates/dcim/device.html b/netbox/templates/dcim/device.html index 5fa6a3314..39e78c81b 100644 --- a/netbox/templates/dcim/device.html +++ b/netbox/templates/dcim/device.html @@ -5,6 +5,7 @@ {% load helpers %} {% load plugins %} {% load i18n %} +{% load mptt %} {% block content %}
@@ -15,16 +16,7 @@ - + @@ -32,16 +24,7 @@ - + diff --git a/netbox/templates/dcim/rack.html b/netbox/templates/dcim/rack.html index 671c7ab2e..857061d00 100644 --- a/netbox/templates/dcim/rack.html +++ b/netbox/templates/dcim/rack.html @@ -4,6 +4,7 @@ {% load static %} {% load plugins %} {% load i18n %} +{% load mptt %} {% block content %}
@@ -15,26 +16,18 @@
{% trans "Region" %} - {% if object.site.region %} - {% for region in object.site.region.get_ancestors %} - {{ region|linkify }} / - {% endfor %} - {{ object.site.region|linkify }} - {% else %} - {{ ''|placeholder }} - {% endif %} - {% nested_tree object.site.region %}
{% trans "Site" %}
{% trans "Location" %} - {% if object.location %} - {% for location in object.location.get_ancestors %} - {{ location|linkify }} / - {% endfor %} - {{ object.location|linkify }} - {% else %} - {{ ''|placeholder }} - {% endif %} - {% nested_tree object.location %}
{% trans "Rack" %}
- + + + + + - + diff --git a/netbox/templates/dcim/rackreservation.html b/netbox/templates/dcim/rackreservation.html index 8edb75f32..3d145145f 100644 --- a/netbox/templates/dcim/rackreservation.html +++ b/netbox/templates/dcim/rackreservation.html @@ -4,6 +4,7 @@ {% load static %} {% load plugins %} {% load i18n %} +{% load mptt %} {% block breadcrumbs %} {{ block.super }} @@ -20,25 +21,24 @@
{% trans "Site" %}{% trans "Region" %} - {% if object.site.region %} - {{ object.site.region|linkify }} / - {% endif %} - {{ object.site|linkify }} + {% nested_tree object.site.region %}
{% trans "Site" %}{{ object.site|linkify }}
{% trans "Location" %} - {% if object.location %} - {% for location in object.location.get_ancestors %} - {{ location|linkify }} / - {% endfor %} - {{ object.location|linkify }} - {% else %} - {{ ''|placeholder }} - {% endif %} - {% nested_tree object.location %}
{% trans "Facility ID" %}
- {% with rack=object.rack %} - - - - - - - - - - - - - {% endwith %} + + + + + + + + + + + + + + + +
{% trans "Site" %} - {% if rack.site.region %} - {{ rack.site.region|linkify }} / - {% endif %} - {{ rack.site|linkify }} -
{% trans "Location" %}{{ rack.location|linkify|placeholder }}
{% trans "Rack" %}{{ rack|linkify }}
{% trans "Region" %} + {% nested_tree object.rack.site.region %} +
{% trans "Site" %}{{ object.rack.site|linkify }}
{% trans "Location" %}{{ object.rack.location|linkify|placeholder }}
{% trans "Rack" %}{{ object.rack|linkify }}
diff --git a/netbox/templates/dcim/site.html b/netbox/templates/dcim/site.html index 7f43a0ab3..16a870182 100644 --- a/netbox/templates/dcim/site.html +++ b/netbox/templates/dcim/site.html @@ -3,6 +3,7 @@ {% load plugins %} {% load tz %} {% load i18n %} +{% load mptt %} {% block breadcrumbs %} {{ block.super }} @@ -29,27 +30,13 @@ {% trans "Region" %} - {% if object.region %} - {% for region in object.region.get_ancestors %} - {{ region|linkify }} / - {% endfor %} - {{ object.region|linkify }} - {% else %} - {{ ''|placeholder }} - {% endif %} + {% nested_tree object.region %} {% trans "Group" %} - {% if object.group %} - {% for group in object.group.get_ancestors %} - {{ group|linkify }} / - {% endfor %} - {{ object.group|linkify }} - {% else %} - {{ ''|placeholder }} - {% endif %} + {% nested_tree object.group %} diff --git a/netbox/templates/extras/eventrule.html b/netbox/templates/extras/eventrule.html new file mode 100644 index 000000000..86c330121 --- /dev/null +++ b/netbox/templates/extras/eventrule.html @@ -0,0 +1,98 @@ +{% extends 'generic/object.html' %} +{% load helpers %} +{% load plugins %} +{% load i18n %} + +{% block content %} +
+
+
+
+ {% trans "Event Rule" %} +
+
+ + + + + + + + + + + + + +
{% trans "Name" %}{{ object.name }}
{% trans "Enabled" %}{% checkmark object.enabled %}
{% trans "Description" %}{{ object.description|placeholder }}
+
+
+
+
+ {% trans "Events" %} +
+
+ + + + + + + + + + + + + + + + + + + + + +
{% trans "Create" %}{% checkmark object.type_create %}
{% trans "Update" %}{% checkmark object.type_update %}
{% trans "Delete" %}{% checkmark object.type_delete %}
{% trans "Job start" %}{% checkmark object.type_job_start %}
{% trans "Job end" %}{% checkmark object.type_job_end %}
+
+
+ {% plugin_left_page object %} +
+
+
+
+ {% trans "Object Types" %} +
+
+ + {% for ct in object.content_types.all %} + + + + {% endfor %} +
{{ ct }}
+
+
+
+
+ {% trans "Conditions" %} +
+
+ {% if object.conditions %} + 
{{ object.conditions|json }}
+ {% else %} +

{% trans "None" %}

+ {% endif %} +
+
+ {% include 'inc/panels/custom_fields.html' %} + {% include 'inc/panels/tags.html' %} + {% plugin_right_page object %} +
+
+
+
+ {% plugin_full_width_page object %} +
+
+{% endblock %} diff --git a/netbox/templates/extras/webhook.html b/netbox/templates/extras/webhook.html index 3338e3f30..0f390d3e4 100644 --- a/netbox/templates/extras/webhook.html +++ b/netbox/templates/extras/webhook.html @@ -20,39 +20,6 @@ {% trans "Description" %} {{ object.description|placeholder }} - - {% trans "Enabled" %} - {% checkmark object.enabled %} - - - - -
-
- {% trans "Events" %} -
-
- - - - - - - - - - - - - - - - - - - - -
{% trans "Create" %}{% checkmark object.type_create %}
{% trans "Update" %}{% checkmark object.type_update %}
{% trans "Delete" %}{% checkmark object.type_delete %}
{% trans "Job start" %}{% checkmark object.type_job_start %}
{% trans "Job end" %}{% checkmark object.type_job_end %}
@@ -101,32 +68,6 @@ {% plugin_left_page object %}
-
-
- {% trans "Assigned Models" %} -
-
- - {% for ct in object.content_types.all %} - - - - {% endfor %} -
{{ ct }}
-
-
-
-
- {% trans "Conditions" %} -
-
- {% if object.conditions %} - 
{{ object.conditions|json }}
- {% else %} -

{% trans "None" %}

- {% endif %} -
-
{% trans "Additional Headers" %} diff --git a/netbox/templates/ipam/prefix.html b/netbox/templates/ipam/prefix.html index 71b240ced..ec6138d69 100644 --- a/netbox/templates/ipam/prefix.html +++ b/netbox/templates/ipam/prefix.html @@ -3,6 +3,7 @@ {% load helpers %} {% load plugins %} {% load i18n %} +{% load mptt %} {% block content %}
@@ -44,18 +45,17 @@ {% endif %} + {% if object.site.region %} + + {% trans "Region" %} + + {% nested_tree object.site.region %} + + + {% endif %} {% trans "Site" %} - - {% if object.site %} - {% if object.site.region %} - {{ object.site.region|linkify }} / - {% endif %} - {{ object.site|linkify }} - {% else %} - {{ ''|placeholder }} - {% endif %} - + {{ object.site|linkify|placeholder }} {% trans "VLAN" %} diff --git a/netbox/templates/ipam/vlan.html b/netbox/templates/ipam/vlan.html index 4ca045d4b..a5ed9d643 100644 --- a/netbox/templates/ipam/vlan.html +++ b/netbox/templates/ipam/vlan.html @@ -3,6 +3,7 @@ {% load render_table from django_tables2 %} {% load plugins %} {% load i18n %} +{% load mptt %} {% block content %}
@@ -13,18 +14,17 @@
+ {% if object.site.region %} + + + + + {% endif %} - + diff --git a/netbox/tenancy/models/contacts.py b/netbox/tenancy/models/contacts.py index e7f319051..28bf92958 100644 --- a/netbox/tenancy/models/contacts.py +++ b/netbox/tenancy/models/contacts.py @@ -6,7 +6,7 @@ from django.utils.translation import gettext_lazy as _ from core.models import ContentType from netbox.models import ChangeLoggedModel, NestedGroupModel, OrganizationalModel, PrimaryModel -from netbox.models.features import CustomFieldsMixin, TagsMixin +from netbox.models.features import CustomFieldsMixin, ExportTemplatesMixin, TagsMixin from tenancy.choices import * __all__ = ( @@ -110,7 +110,7 @@ class Contact(PrimaryModel): return reverse('tenancy:contact', args=[self.pk]) -class ContactAssignment(CustomFieldsMixin, TagsMixin, ChangeLoggedModel): +class ContactAssignment(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLoggedModel): content_type = models.ForeignKey( to='contenttypes.ContentType', on_delete=models.CASCADE diff --git a/netbox/users/api/serializers.py b/netbox/users/api/serializers.py index 75ab877cf..c9775e39a 100644 --- a/netbox/users/api/serializers.py +++ b/netbox/users/api/serializers.py @@ -52,6 +52,16 @@ class UserSerializer(ValidatedModelSerializer): return user + def update(self, instance, validated_data): + """ + Ensure proper updated password hash generation. + """ + password = validated_data.pop('password', None) + if password is not None: + instance.set_password(password) + + return super().update(instance, validated_data) + @extend_schema_field(OpenApiTypes.STR) def get_display(self, obj): if full_name := obj.get_full_name(): diff --git a/netbox/users/tests/test_api.py b/netbox/users/tests/test_api.py index 001142410..090ccc263 100644 --- a/netbox/users/tests/test_api.py +++ b/netbox/users/tests/test_api.py @@ -54,6 +54,38 @@ class UserTest(APIViewTestCases.APIViewTestCase): ) User.objects.bulk_create(users) + def test_that_password_is_changed(self): + """ + Test that password is changed + """ + + obj_perm = ObjectPermission( + name='Test permission', + actions=['change'] + ) + obj_perm.save() + obj_perm.users.add(self.user) + obj_perm.object_types.add(ContentType.objects.get_for_model(self.model)) + + user_credentials = { + 'username': 'user1', + 'password': 'abc123', + } + user = User.objects.create_user(**user_credentials) + + data = { + 'password': 'newpassword' + } + url = reverse('users-api:user-detail', kwargs={'pk': user.id}) + + response = self.client.patch(url, data, format='json', **self.header) + + self.assertEqual(response.status_code, 200) + + updated_user = User.objects.get(id=user.id) + + self.assertTrue(updated_user.check_password(data['password'])) + class GroupTest(APIViewTestCases.APIViewTestCase): model = Group diff --git a/netbox/utilities/forms/fields/fields.py b/netbox/utilities/forms/fields/fields.py index db5e4a30d..d4d4ae19b 100644 --- a/netbox/utilities/forms/fields/fields.py +++ b/netbox/utilities/forms/fields/fields.py @@ -103,7 +103,7 @@ class JSONField(_JSONField): def prepare_value(self, value): if isinstance(value, InvalidJSONInput): return value - if value is None: + if value in ('', None): return '' return json.dumps(value, sort_keys=True, indent=4) diff --git a/netbox/utilities/forms/utils.py b/netbox/utilities/forms/utils.py index 4d737f163..de8e22727 100644 --- a/netbox/utilities/forms/utils.py +++ b/netbox/utilities/forms/utils.py @@ -40,7 +40,7 @@ def parse_numeric_range(string, base=10): except ValueError: raise forms.ValidationError(f'Range "{dash_range}" is invalid.') values.extend(range(begin, end)) - return list(set(values)) + return sorted(set(values)) def parse_alphanumeric_range(string): @@ -128,10 +128,9 @@ def get_field_value(form, field_name): """ field = form.fields[field_name] - if form.is_bound: - if data := form.data.get(field_name): - if field.valid_value(data): - return data + if form.is_bound and (data := form.data.get(field_name)): + if hasattr(field, 'valid_value') and field.valid_value(data): + return data return form.get_initial_for_field(field, field_name) diff --git a/netbox/utilities/templatetags/mptt.py b/netbox/utilities/templatetags/mptt.py new file mode 100644 index 000000000..783c2654f --- /dev/null +++ b/netbox/utilities/templatetags/mptt.py @@ -0,0 +1,20 @@ +from django import template +from django.utils.safestring import mark_safe + +register = template.Library() + + +@register.simple_tag() +def nested_tree(obj): + """ + Renders the entire hierarchy of a recursively-nested object (such as Region or SiteGroup). + """ + if not obj: + return mark_safe('—') + + nodes = obj.get_ancestors(include_self=True) + return mark_safe( + ' / '.join( + f'{node}' for node in nodes + ) + ) diff --git a/netbox/virtualization/forms/bulk_edit.py b/netbox/virtualization/forms/bulk_edit.py index 72990ec76..b76d8a160 100644 --- a/netbox/virtualization/forms/bulk_edit.py +++ b/netbox/virtualization/forms/bulk_edit.py @@ -296,9 +296,10 @@ class VMInterfaceBulkEditForm(NetBoxModelBulkEditForm): # Check interface sites. First interface should set site, further interfaces will either continue the # loop or reset back to no site and break the loop. for interface in interfaces: + vm_site = interface.virtual_machine.site or interface.virtual_machine.cluster.site if site is None: - site = interface.virtual_machine.cluster.site - elif interface.virtual_machine.cluster.site is not site: + site = vm_site + elif vm_site is not site: site = None break diff --git a/requirements.txt b/requirements.txt index 45fb12f80..537c5b77e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,18 +1,18 @@ bleach==6.1.0 Django==4.2.7 -django-cors-headers==4.3.0 +django-cors-headers==4.3.1 django-debug-toolbar==4.2.0 -django-filter==23.3 +django-filter==23.4 django-graphiql-debug-toolbar==0.2.0 django-mptt==0.14.0 django-pglocks==1.0.4 django-prometheus==2.3.1 django-redis==5.4.0 django-rich==1.8.0 -django-rq==2.8.1 +django-rq==2.9.0 django-tables2==2.6.0 django-taggit==4.0.0 -django-timezone-field==6.0.1 +django-timezone-field==6.1.0 djangorestframework==3.14.0 drf-spectacular==0.26.5 drf-spectacular-sidecar==2023.10.1 @@ -21,15 +21,15 @@ graphene-django==3.0.0 gunicorn==21.2.0 Jinja2==3.1.2 Markdown==3.3.7 -mkdocs-material==9.4.8 -mkdocstrings[python-legacy]==0.23.0 +mkdocs-material==9.4.14 +mkdocstrings[python-legacy]==0.24.0 netaddr==0.9.0 Pillow==10.1.0 -psycopg[binary,pool]==3.1.12 +psycopg[binary,pool]==3.1.13 PyYAML==6.0.1 requests==2.31.0 social-auth-app-django==5.4.0 -social-auth-core[openidconnect]==4.5.0 +social-auth-core[openidconnect]==4.5.1 svgwrite==1.4.3 tablib==3.5.0 tzdata==2023.3
{% trans "Region" %} + {% nested_tree object.site.region %} +
{% trans "Site" %} - {% if object.site %} - {% if object.site.region %} - {{ object.site.region|linkify }} / - {% endif %} - {{ object.site|linkify }} - {% else %} - {{ ''|placeholder }} - {% endif %} - {{ object.site|linkify|placeholder }}
{% trans "Group" %}