Implements #81 - webhook event backend (#1640)

* merge branch develop

* bugfix, signals for virtualization's class wasn't correctly defined

* updated webhooks for 2.4 and cleanup

* updated docs to cover changes to supervisor config

* review changes and further cleanup

* updated redis connection settings

* cleanup settings

netbox/extras/__init__.py

@@ -0,0 +1,15 @@
+from django.conf import settings
+from django.core.exceptions import ImproperlyConfigured
+
+
+default_app_config = 'extras.apps.ExtrasConfig'
+
+# check that django-rq is installed and we can connect to redis
+if settings.WEBHOOK_BACKEND_ENABLED:
+    try:
+        import django_rq
+    except ImportError:
+        raise ImproperlyConfigured(
+            "django-rq is not installed! You must install this package per "
+            "the documentation to use the webhook backend."
+        )

netbox/extras/admin.py

@@ -4,7 +4,11 @@ from django import forms
 from django.contrib import admin
 from django.utils.safestring import mark_safe
 
-from .models import CustomField, CustomFieldChoice, Graph, ExportTemplate, TopologyMap, UserAction
+from utilities.forms import LaxURLField
+from .models import (
+    CustomField, CustomFieldChoice, Graph, ExportTemplate, TopologyMap, UserAction,
+    Webhook
+)
 
 
 def order_content_types(field):
@@ -15,6 +19,36 @@ def order_content_types(field):
     field.choices = [(ct.pk, '{} > {}'.format(ct.app_label, ct.name)) for ct in queryset]
 
 
+#
+# Webhooks
+#
+
+class WebhookForm(forms.ModelForm):
+
+    payload_url = LaxURLField()
+
+    class Meta:
+        model = Webhook
+        exclude = []
+
+    def __init__(self, *args, **kwargs):
+        super(WebhookForm, self).__init__(*args, **kwargs)
+
+        order_content_types(self.fields['obj_type'])
+
+
+@admin.register(Webhook)
+class WebhookAdmin(admin.ModelAdmin):
+    list_display = [
+        'name', 'models', 'payload_url', 'http_content_type', 'enabled', 'type_create', 'type_update',
+        'type_delete', 'ssl_verification',
+    ]
+    form = WebhookForm
+
+    def models(self, obj):
+        return ', '.join([ct.name for ct in obj.obj_type.all()])
+
+
 #
 # Custom fields
 #

netbox/extras/api/serializers.py

@@ -10,6 +10,7 @@ from extras.constants import ACTION_CHOICES, GRAPH_TYPE_CHOICES
 from extras.models import ExportTemplate, Graph, ImageAttachment, ReportResult, TopologyMap, UserAction
 from users.api.serializers import NestedUserSerializer
 from utilities.api import ChoiceFieldSerializer, ContentTypeFieldSerializer, ValidatedModelSerializer
+from extras.constants import *
 
 
 #

netbox/extras/apps.py

@@ -0,0 +1,29 @@
+from __future__ import unicode_literals
+
+from django.apps import AppConfig
+from django.core.cache import caches
+from django.db.utils import ProgrammingError
+from django.core.exceptions import ImproperlyConfigured
+from django.conf import settings
+
+
+class ExtrasConfig(AppConfig):
+    name = "extras"
+
+    def ready(self):
+        import extras.signals
+
+        # check that we can connect to redis
+        if settings.WEBHOOK_BACKEND_ENABLED:
+            try:
+                import redis
+                rs = redis.Redis(settings.REDIS_HOST,
+                                 settings.REDIS_PORT,
+                                 settings.REDIS_DB,
+                                 settings.REDIS_PASSWORD or None)
+                rs.ping()
+            except redis.exceptions.ConnectionError:
+                raise ImproperlyConfigured(
+                    "Unable to connect to the redis database. You must provide "
+                    "connection settings to redis per the documentation."
+                )

netbox/extras/constants.py

@@ -97,3 +97,21 @@ LOG_LEVEL_CODES = {
     LOG_WARNING: 'warning',
     LOG_FAILURE: 'failure',
 }
+
+# webhook content types
+WEBHOOK_CT_JSON = 1
+WEBHOOK_CT_X_WWW_FORM_ENCODED = 2
+WEBHOOK_CT_CHOICES = (
+    (WEBHOOK_CT_JSON, 'application/json'),
+    (WEBHOOK_CT_X_WWW_FORM_ENCODED, 'application/x-www-form-urlencoded'),
+)
+
+# Models which support registered webhooks
+WEBHOOK_MODELS = (
+    'provider', 'circuit',                                           # Circuits
+    'site', 'rack', 'rackgroup', 'device', 'interface',              # DCIM
+    'aggregate', 'prefix', 'ipaddress', 'vlan', 'vlangroup', 'vrf',  # IPAM
+    'service',
+    'tenant', 'tenantgroup',                                         # Tenancy
+    'cluster', 'clustergroup', 'virtualmachine',                     # Virtualization
+)

netbox/extras/migrations/0012_webhooks.py

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.10 on 2018-05-23 16:35
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('contenttypes', '0002_remove_content_type_name'),
+        ('extras', '0011_django2'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Webhook',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=150, unique=True)),
+                ('type_create', models.BooleanField(default=False, help_text='A POST will be sent to the URL when the object type(s) is created.')),
+                ('type_update', models.BooleanField(default=False, help_text='A POST will be sent to the URL when the object type(s) is updated.')),
+                ('type_delete', models.BooleanField(default=False, help_text='A POST will be sent to the URL when the object type(s) is deleted.')),
+                ('payload_url', models.CharField(max_length=500, verbose_name='A POST will be sent to this URL based on the webhook criteria.')),
+                ('http_content_type', models.PositiveSmallIntegerField(choices=[(1, 'application/json'), (2, 'application/x-www-form-urlencoded')], default=1)),
+                ('secret', models.CharField(blank=True, help_text="When provided the request will include a 'X-Hook-Signature' header which is a HMAC hex digest of the payload body using the secret as the key. The secret is not transmitted in the request.", max_length=255)),
+                ('enabled', models.BooleanField(default=True)),
+                ('ssl_verification', models.BooleanField(default=True, help_text='By default, use of proper SSL is verified. Disable with caution!')),
+                ('obj_type', models.ManyToManyField(help_text='The object(s) to which this Webhook applies.', related_name='webhooks', to='contenttypes.ContentType', verbose_name='Object(s)')),
+            ],
+        ),
+        migrations.AlterUniqueTogether(
+            name='webhook',
+            unique_together=set([('payload_url', 'type_create', 'type_update', 'type_delete')]),
+        ),
+    ]

netbox/extras/models.py

@@ -21,6 +21,80 @@ from utilities.utils import foreground_color
 from .constants import *
 
 
+#
+# Webhooks
+#
+
+class Webhook(models.Model):
+    """
+    Webhook model that represents all the details for an endoint and how to make a request to
+    that endpoint with the configured payload.
+    """
+
+    obj_type = models.ManyToManyField(
+        ContentType,
+        related_name='webhooks',
+        verbose_name='Object(s)',
+        limit_choices_to={'model__in': WEBHOOK_MODELS},
+        help_text="The object(s) to which this Webhook applies."
+    )
+    name = models.CharField(
+        max_length=150,
+        unique=True
+    )
+    type_create = models.BooleanField(
+        default=False,
+        help_text="A POST will be sent to the URL when the object type(s) is created."
+    )
+    type_update = models.BooleanField(
+        default=False,
+        help_text="A POST will be sent to the URL when the object type(s) is updated."
+    )
+    type_delete = models.BooleanField(
+        default=False,
+        help_text="A POST will be sent to the URL when the object type(s) is deleted."
+    )
+    payload_url = models.CharField(
+        max_length=500,
+        verbose_name="A POST will be sent to this URL based on the webhook criteria."
+    )
+    http_content_type = models.PositiveSmallIntegerField(
+        choices=WEBHOOK_CT_CHOICES,
+        default=WEBHOOK_CT_JSON
+    )
+    secret = models.CharField(
+        max_length=255,
+        blank=True,
+        help_text="When provided the request will include a 'X-Hook-Signature' "
+                  "header which is a HMAC hex digest of the payload body using "
+                  "the secret as the key. The secret is not transmitted in "
+                  "the request."
+    )
+    enabled = models.BooleanField(
+        default=True
+    )
+    ssl_verification = models.BooleanField(
+        default=True,
+        help_text="By default, use of proper SSL is verified. Disable with caution!"
+    )
+
+    class Meta:
+        unique_together = ('payload_url', 'type_create', "type_update", "type_delete",)
+
+    def __str__(self):
+        return self.name
+
+    def clean(self):
+        """
+        Validate model
+        """
+
+        if not self.type_create and not self.type_delete and not self.type_update:
+            raise ValidationError(
+                "You must select at least one type. Either create, update, or delete."
+            )
+
+
 #
 # Custom fields
 #

netbox/extras/signals.py

@@ -0,0 +1,16 @@
+from __future__ import unicode_literals
+
+from django.db.models.signals import post_delete, post_save
+from django.dispatch import receiver
+from django.core.cache import caches
+
+from .models import Webhook
+
+
+@receiver((post_save, post_delete), sender=Webhook)
+def update_webhook_cache(**kwargs):
+    """
+    When a Webhook has been modified, update the webhook cache.
+    """
+    cache = caches['default']
+    cache.set('webhook_cache', Webhook.objects.all())

netbox/extras/webhooks.py

@@ -0,0 +1,142 @@
+import time
+from importlib import import_module
+
+from django.db.models.signals import post_save, post_delete
+from django.conf import settings
+from django.core.cache import caches
+from django.dispatch import Signal
+from django.contrib.contenttypes.models import ContentType
+
+from utilities.utils import dynamic_import
+from .models import Webhook
+
+
+#
+# Webhooks signals regiters and receivers
+#
+
+def get_or_set_webhook_cache():
+    """
+    Retrieve the webhook cache. If it is None set it to the current
+    Webhook queryset
+    """
+    cache = caches['default']
+    webhook_cache = cache.get('webhook_cache', None)
+
+    if webhook_cache is None:
+        webhook_cache = Webhook.objects.all()
+        cache.set('webhook_cache', webhook_cache)
+
+    return webhook_cache
+
+
+def enqueue_webhooks(webhooks, model_class, data, event, signal_received_timestamp):
+    """
+    Serialize data and enqueue webhooks
+    """
+    serializer_context = {
+        'request': None,
+    }
+
+    if isinstance(data, list):
+        serializer_property = data[0].serializer
+        serializer_cls = dynamic_import(serializer_property)
+        serialized_data = serializer_cls(data, context=serializer_context, many=True)
+    else:
+        serializer_property = data.serializer
+        serializer_cls = dynamic_import(serializer_property)
+        serialized_data = serializer_cls(data, context=serializer_context)
+
+    from django_rq import get_queue
+    webhook_queue = get_queue('default')
+
+    for webhook in webhooks:
+        webhook_queue.enqueue("extras.webhooks_worker.process_webhook",
+                              webhook,
+                              serialized_data.data,
+                              model_class,
+                              event,
+                              signal_received_timestamp)
+
+
+def post_save_receiver(sender, instance, created, **kwargs):
+    """
+    Receives post_save signals from registered models. If the webhook
+    backend is enabled, queue any webhooks that apply to the event.
+    """
+    if settings.WEBHOOK_BACKEND_ENABLED:
+        signal_received_timestamp = time.time()
+        webhook_cache = get_or_set_webhook_cache()
+        # look for any webhooks that match this event
+        updated = not created
+        obj_type = ContentType.objects.get_for_model(sender)
+        webhooks = [
+            x
+            for x in webhook_cache
+            if (
+                x.enabled and x.type_create == created or x.type_update == updated and
+                obj_type in x.obj_type.all()
+            )
+        ]
+        event = 'created' if created else 'updated'
+        if webhooks:
+            enqueue_webhooks(webhooks, sender, instance, event, signal_received_timestamp)
+
+
+def post_delete_receiver(sender, instance, **kwargs):
+    """
+    Receives post_delete signals from registered models. If the webhook
+    backend is enabled, queue any webhooks that apply to the event.
+    """
+    if settings.WEBHOOK_BACKEND_ENABLED:
+        signal_received_timestamp = time.time()
+        webhook_cache = get_or_set_webhook_cache()
+        obj_type = ContentType.objects.get_for_model(sender)
+        # look for any webhooks that match this event
+        webhooks = [x for x in webhook_cache if x.enabled and x.type_delete and obj_type in x.obj_type.all()]
+        if webhooks:
+            enqueue_webhooks(webhooks, sender, instance, 'deleted', signal_received_timestamp)
+
+
+def bulk_operation_receiver(sender, **kwargs):
+    """
+    Receives bulk_operation_signal signals from registered models. If the webhook
+    backend is enabled, queue any webhooks that apply to the event.
+    """
+    if settings.WEBHOOK_BACKEND_ENABLED:
+        signal_received_timestamp = time.time()
+        event = kwargs['event']
+        webhook_cache = get_or_set_webhook_cache()
+        obj_type = ContentType.objects.get_for_model(sender)
+        # look for any webhooks that match this event
+        if event == 'created':
+            webhooks = [x for x in webhook_cache if x.enabled and x.type_create and obj_type in x.obj_type.all()]
+        elif event == 'updated':
+            webhooks = [x for x in webhook_cache if x.enabled and x.type_update and obj_type in x.obj_type.all()]
+        elif event == 'deleted':
+            webhooks = [x for x in webhook_cache if x.enabled and x.type_delete and obj_type in x.obj_type.all()]
+        else:
+            webhooks = None
+
+        if webhooks:
+            enqueue_webhooks(webhooks, sender, list(kwargs['instances']), event, signal_received_timestamp)
+
+
+# the bulk operation signal is used to overcome signals not being sent for bulk model changes
+bulk_operation_signal = Signal(providing_args=["instances", "event"])
+bulk_operation_signal.connect(bulk_operation_receiver)
+
+
+def register_signals(senders):
+    """
+    Take a list of senders (Models) and register them to the post_save
+    and post_delete signal receivers.
+    """
+    if settings.WEBHOOK_BACKEND_ENABLED:
+        # only register signals if the backend is enabled
+        # this reduces load by not firing signals if the
+        # webhook backend feature is disabled
+
+        for sender in senders:
+            post_save.connect(post_save_receiver, sender=sender)
+            post_delete.connect(post_delete_receiver, sender=sender)

netbox/extras/webhooks_worker.py

@@ -0,0 +1,52 @@
+import requests
+import hmac
+import hashlib
+from rq.utils import import_attribute
+
+from django_rq import job
+
+from extras.constants import WEBHOOK_CT_JSON, WEBHOOK_CT_X_WWW_FORM_ENCODED
+
+
+@job('default')
+def process_webhook(webhook, data, model_class, event, timestamp):
+    """
+    Make a POST request to the defined Webhook
+    """
+    payload = {
+        'event': event,
+        'timestamp': timestamp,
+        'model': model_class.__name__,
+        'data': data
+    }
+    headers = {
+        'Content-Type': webhook.get_http_content_type_display(),
+    }
+    params = {
+        'method': 'POST',
+        'url': webhook.payload_url,
+        'headers': headers
+    }
+
+    if webhook.http_content_type == WEBHOOK_CT_JSON:
+        params.update({'json': payload})
+    elif webhook.http_content_type == WEBHOOK_CT_X_WWW_FORM_ENCODED:
+        params.update({'data': payload})
+
+    prepared_request = requests.Request(**params).prepare()
+
+    if webhook.secret != '':
+        # sign the request with the secret
+        hmac_prep = hmac.new(bytearray(webhook.secret, 'utf8'), prepared_request.body, digestmod=hashlib.sha512)
+        prepared_request.headers['X-Hook-Signature'] = hmac_prep.hexdigest()
+
+    with requests.Session() as session:
+        session.verify = webhook.ssl_verification
+        response = session.send(prepared_request)
+
+    if response.status_code >= 200 and response.status_code <= 299:
+        return 'Status {} returned, webhook successfully processed.'.format(response.status_code)
+    else:
+        raise requests.exceptions.RequestException(
+            "Status {} returned, webhook FAILED to process.".format(response.status_code)
+        )