From 99038ffc44d9e0c12fb801ea102fdb22581e1041 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Mon, 24 Feb 2020 16:12:46 -0500 Subject: [PATCH 1/7] Enable custom templating for webhook request content --- netbox/extras/admin.py | 20 +++++++++-- netbox/extras/choices.py | 20 ----------- netbox/extras/constants.py | 2 ++ .../migrations/0038_webhook_body_template.py | 23 ++++++++++++ netbox/extras/models.py | 26 ++++++++------ netbox/extras/webhooks.py | 1 - netbox/extras/webhooks_worker.py | 36 +++++++++++++++---- 7 files changed, 87 insertions(+), 41 deletions(-) create mode 100644 netbox/extras/migrations/0038_webhook_body_template.py diff --git a/netbox/extras/admin.py b/netbox/extras/admin.py index 2a39c207e..9a75fb53f 100644 --- a/netbox/extras/admin.py +++ b/netbox/extras/admin.py @@ -26,7 +26,7 @@ class WebhookForm(forms.ModelForm): class Meta: model = Webhook - exclude = [] + exclude = () def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -38,13 +38,27 @@ class WebhookForm(forms.ModelForm): @admin.register(Webhook, site=admin_site) class WebhookAdmin(admin.ModelAdmin): list_display = [ - 'name', 'models', 'payload_url', 'http_content_type', 'enabled', 'type_create', 'type_update', - 'type_delete', 'ssl_verification', + 'name', 'models', 'payload_url', 'http_content_type', 'enabled', 'type_create', 'type_update', 'type_delete', + 'ssl_verification', ] list_filter = [ 'enabled', 'type_create', 'type_update', 'type_delete', 'obj_type', ] form = WebhookForm + fieldsets = ( + (None, { + 'fields': ('name', 'obj_type', 'enabled') + }), + ('Events', { + 'fields': ('type_create', 'type_update', 'type_delete') + }), + ('HTTP Request', { + 'fields': ('payload_url', 'http_content_type', 'additional_headers', 'body_template', 'secret') + }), + ('SSL', { + 'fields': ('ssl_verification', 'ca_file_path') + }) + ) def models(self, obj): return ', '.join([ct.name for ct in obj.obj_type.all()]) diff --git a/netbox/extras/choices.py b/netbox/extras/choices.py index 0ae53f03d..434461e76 100644 --- a/netbox/extras/choices.py +++ b/netbox/extras/choices.py @@ -118,23 +118,3 @@ class TemplateLanguageChoices(ChoiceSet): LANGUAGE_DJANGO: 10, LANGUAGE_JINJA2: 20, } - - -# -# Webhooks -# - -class WebhookContentTypeChoices(ChoiceSet): - - CONTENTTYPE_JSON = 'application/json' - CONTENTTYPE_FORMDATA = 'application/x-www-form-urlencoded' - - CHOICES = ( - (CONTENTTYPE_JSON, 'JSON'), - (CONTENTTYPE_FORMDATA, 'Form data'), - ) - - LEGACY_MAP = { - CONTENTTYPE_JSON: 1, - CONTENTTYPE_FORMDATA: 2, - } diff --git a/netbox/extras/constants.py b/netbox/extras/constants.py index b12bc2f2c..7bb026d34 100644 --- a/netbox/extras/constants.py +++ b/netbox/extras/constants.py @@ -138,6 +138,8 @@ LOG_LEVEL_CODES = { LOG_FAILURE: 'failure', } +HTTP_CONTENT_TYPE_JSON = 'application/json' + # Models which support registered webhooks WEBHOOK_MODELS = Q( Q(app_label='circuits', model__in=[ diff --git a/netbox/extras/migrations/0038_webhook_body_template.py b/netbox/extras/migrations/0038_webhook_body_template.py new file mode 100644 index 000000000..1e23a2303 --- /dev/null +++ b/netbox/extras/migrations/0038_webhook_body_template.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.10 on 2020-02-24 20:12 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('extras', '0037_configcontexts_clusters'), + ] + + operations = [ + migrations.AddField( + model_name='webhook', + name='body_template', + field=models.TextField(blank=True), + ), + migrations.AlterField( + model_name='webhook', + name='http_content_type', + field=models.CharField(default='application/json', max_length=100), + ), + ] diff --git a/netbox/extras/models.py b/netbox/extras/models.py index 5d175d172..896595524 100644 --- a/netbox/extras/models.py +++ b/netbox/extras/models.py @@ -52,7 +52,6 @@ class Webhook(models.Model): delete in NetBox. The request will contain a representation of the object, which the remote application can act on. Each Webhook can be limited to firing only on certain actions or certain object types. """ - obj_type = models.ManyToManyField( to=ContentType, related_name='webhooks', @@ -81,11 +80,15 @@ class Webhook(models.Model): verbose_name='URL', help_text="A POST will be sent to this URL when the webhook is called." ) + enabled = models.BooleanField( + default=True + ) http_content_type = models.CharField( - max_length=50, - choices=WebhookContentTypeChoices, - default=WebhookContentTypeChoices.CONTENTTYPE_JSON, - verbose_name='HTTP content type' + max_length=100, + default=HTTP_CONTENT_TYPE_JSON, + verbose_name='HTTP content type', + help_text='The complete list of official content types is available ' + 'here.' ) additional_headers = JSONField( null=True, @@ -93,6 +96,13 @@ class Webhook(models.Model): help_text="User supplied headers which should be added to the request in addition to the HTTP content type. " "Headers are supplied as key/value pairs in a JSON object." ) + body_template = models.TextField( + blank=True, + help_text='Jinja2 template for a custom request body. If blank, a JSON object or form data representing the ' + 'change will be included. Available context data includes: event, ' + 'timestamp, model, username, request_id, and ' + 'data.' + ) secret = models.CharField( max_length=255, blank=True, @@ -101,9 +111,6 @@ class Webhook(models.Model): "the secret as the key. The secret is not transmitted in " "the request." ) - enabled = models.BooleanField( - default=True - ) ssl_verification = models.BooleanField( default=True, verbose_name='SSL verification', @@ -126,9 +133,6 @@ class Webhook(models.Model): return self.name def clean(self): - """ - Validate model - """ if not self.type_create and not self.type_delete and not self.type_update: raise ValidationError( "You must select at least one type: create, update, and/or delete." diff --git a/netbox/extras/webhooks.py b/netbox/extras/webhooks.py index cfa05d0f6..8b20641d7 100644 --- a/netbox/extras/webhooks.py +++ b/netbox/extras/webhooks.py @@ -1,4 +1,3 @@ -import datetime import hashlib import hmac diff --git a/netbox/extras/webhooks_worker.py b/netbox/extras/webhooks_worker.py index e48d8a2d7..b91561846 100644 --- a/netbox/extras/webhooks_worker.py +++ b/netbox/extras/webhooks_worker.py @@ -1,19 +1,25 @@ import json +import logging import requests from django_rq import job +from jinja2.exceptions import TemplateError from rest_framework.utils.encoders import JSONEncoder -from .choices import ObjectChangeActionChoices, WebhookContentTypeChoices +from utilities.utils import render_jinja2 +from .choices import ObjectChangeActionChoices +from .constants import HTTP_CONTENT_TYPE_JSON from .webhooks import generate_signature +logger = logging.getLogger('netbox.webhooks_worker') + @job('default') def process_webhook(webhook, data, model_name, event, timestamp, username, request_id): """ Make a POST request to the defined Webhook """ - payload = { + context = { 'event': dict(ObjectChangeActionChoices)[event].lower(), 'timestamp': timestamp, 'model': model_name, @@ -21,6 +27,8 @@ def process_webhook(webhook, data, model_name, event, timestamp, username, reque 'request_id': request_id, 'data': data } + + # Build HTTP headers headers = { 'Content-Type': webhook.http_content_type, } @@ -33,10 +41,22 @@ def process_webhook(webhook, data, model_name, event, timestamp, username, reque 'headers': headers } - if webhook.http_content_type == WebhookContentTypeChoices.CONTENTTYPE_JSON: - params.update({'data': json.dumps(payload, cls=JSONEncoder)}) - elif webhook.http_content_type == WebhookContentTypeChoices.CONTENTTYPE_FORMDATA: - params.update({'data': payload}) + logger.info( + "Sending webhook to {}: {} {}".format(params['url'], context['model'], context['event']) + ) + + # Construct the request body. If a template has been defined, use it. Otherwise, dump the context as either JSON + # or form data. + if webhook.body_template: + try: + params['data'] = render_jinja2(webhook.body_template, context) + except TemplateError as e: + logger.error("Error rendering request body: {}".format(e)) + return + elif webhook.http_content_type == HTTP_CONTENT_TYPE_JSON: + params['data'] = json.dumps(context, cls=JSONEncoder) + else: + params['data'] = context prepared_request = requests.Request(**params).prepare() @@ -50,9 +70,13 @@ def process_webhook(webhook, data, model_name, event, timestamp, username, reque session.verify = webhook.ca_file_path response = session.send(prepared_request) + logger.debug(params) + if 200 <= response.status_code <= 299: + logger.info("Request succeeded; response status {}".format(response.status_code)) return 'Status {} returned, webhook successfully processed.'.format(response.status_code) else: + logger.error("Request failed; response status {}: {}".format(response.status_code, response.content)) raise requests.exceptions.RequestException( "Status {} returned with content '{}', webhook FAILED to process.".format( response.status_code, response.content From 1fbd3a2c265935e55a28061328daede616f72c5a Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Mon, 24 Feb 2020 16:50:20 -0500 Subject: [PATCH 2/7] Convert additional_headers to a TextField --- .../migrations/0038_webhook_body_template.py | 23 ---------- .../0038_webhook_template_support.py | 43 +++++++++++++++++++ netbox/extras/models.py | 3 +- 3 files changed, 44 insertions(+), 25 deletions(-) delete mode 100644 netbox/extras/migrations/0038_webhook_body_template.py create mode 100644 netbox/extras/migrations/0038_webhook_template_support.py diff --git a/netbox/extras/migrations/0038_webhook_body_template.py b/netbox/extras/migrations/0038_webhook_body_template.py deleted file mode 100644 index 1e23a2303..000000000 --- a/netbox/extras/migrations/0038_webhook_body_template.py +++ /dev/null @@ -1,23 +0,0 @@ -# Generated by Django 2.2.10 on 2020-02-24 20:12 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('extras', '0037_configcontexts_clusters'), - ] - - operations = [ - migrations.AddField( - model_name='webhook', - name='body_template', - field=models.TextField(blank=True), - ), - migrations.AlterField( - model_name='webhook', - name='http_content_type', - field=models.CharField(default='application/json', max_length=100), - ), - ] diff --git a/netbox/extras/migrations/0038_webhook_template_support.py b/netbox/extras/migrations/0038_webhook_template_support.py new file mode 100644 index 000000000..80a1d2b7d --- /dev/null +++ b/netbox/extras/migrations/0038_webhook_template_support.py @@ -0,0 +1,43 @@ +import json + +from django.db import migrations, models + + +def json_to_text(apps, schema_editor): + """ + Convert a JSON representation of HTTP headers to key-value pairs (one header per line) + """ + Webhook = apps.get_model('extras', 'Webhook') + for webhook in Webhook.objects.exclude(additional_headers=''): + data = json.loads(webhook.additional_headers) + headers = ['{}: {}'.format(k, v) for k, v in data.items()] + Webhook.objects.filter(pk=webhook.pk).update(additional_headers='\n'.join(headers)) + + +class Migration(migrations.Migration): + + dependencies = [ + ('extras', '0037_configcontexts_clusters'), + ] + + operations = [ + migrations.AddField( + model_name='webhook', + name='body_template', + field=models.TextField(blank=True), + ), + migrations.AlterField( + model_name='webhook', + name='additional_headers', + field=models.TextField(blank=True, default=''), + preserve_default=False, + ), + migrations.AlterField( + model_name='webhook', + name='http_content_type', + field=models.CharField(default='application/json', max_length=100), + ), + migrations.RunPython( + code=json_to_text + ), + ] diff --git a/netbox/extras/models.py b/netbox/extras/models.py index 896595524..52aba763d 100644 --- a/netbox/extras/models.py +++ b/netbox/extras/models.py @@ -90,8 +90,7 @@ class Webhook(models.Model): help_text='The complete list of official content types is available ' 'here.' ) - additional_headers = JSONField( - null=True, + additional_headers = models.TextField( blank=True, help_text="User supplied headers which should be added to the request in addition to the HTTP content type. " "Headers are supplied as key/value pairs in a JSON object." From 9a532b1eb274760940bda574b7cc8343b0510dd7 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Mon, 24 Feb 2020 17:47:17 -0500 Subject: [PATCH 3/7] Extend templatization ability to additional_headers field --- netbox/extras/models.py | 34 +++++++++++++----- netbox/extras/tests/test_webhooks.py | 2 +- netbox/extras/webhooks_worker.py | 53 +++++++++++++--------------- 3 files changed, 52 insertions(+), 37 deletions(-) diff --git a/netbox/extras/models.py b/netbox/extras/models.py index 52aba763d..94392189d 100644 --- a/netbox/extras/models.py +++ b/netbox/extras/models.py @@ -1,3 +1,4 @@ +import json from collections import OrderedDict from datetime import date @@ -12,6 +13,7 @@ from django.http import HttpResponse from django.template import Template, Context from django.urls import reverse from django.utils.text import slugify +from rest_framework.utils.encoders import JSONEncoder from taggit.models import TagBase, GenericTaggedItemBase from utilities.fields import ColorField @@ -92,8 +94,9 @@ class Webhook(models.Model): ) additional_headers = models.TextField( blank=True, - help_text="User supplied headers which should be added to the request in addition to the HTTP content type. " - "Headers are supplied as key/value pairs in a JSON object." + help_text="User-supplied HTTP headers to be sent with the request in addition to the HTTP content type. " + "Headers should be defined in the format Name: Value. Jinja2 template processing is " + "support with the same context as the request body (below)." ) body_template = models.TextField( blank=True, @@ -139,14 +142,29 @@ class Webhook(models.Model): if not self.ssl_verification and self.ca_file_path: raise ValidationError({ - 'ca_file_path': 'Do not specify a CA certificate file if SSL verification is dissabled.' + 'ca_file_path': 'Do not specify a CA certificate file if SSL verification is disabled.' }) - # Verify that JSON data is provided as an object - if self.additional_headers and type(self.additional_headers) is not dict: - raise ValidationError({ - 'additional_headers': 'Header JSON data must be in object form. Example: {"X-API-KEY": "abc123"}' - }) + def render_headers(self, context): + """ + Render additional_headers and return a dict of Header: Value pairs. + """ + if not self.additional_headers: + return {} + ret = {} + data = render_jinja2(self.additional_headers, context) + for line in data.splitlines(): + header, value = line.split(':') + ret[header.strip()] = value.strip() + return ret + + def render_body(self, context): + if self.body_template: + return render_jinja2(self.body_template, context) + elif self.http_content_type == HTTP_CONTENT_TYPE_JSON: + return json.dumps(context, cls=JSONEncoder) + else: + return context # diff --git a/netbox/extras/tests/test_webhooks.py b/netbox/extras/tests/test_webhooks.py index 026a82bb8..06b4f7c7e 100644 --- a/netbox/extras/tests/test_webhooks.py +++ b/netbox/extras/tests/test_webhooks.py @@ -34,7 +34,7 @@ class WebhookTest(APITestCase): DUMMY_SECRET = "LOOKATMEIMASECRETSTRING" webhooks = Webhook.objects.bulk_create(( - Webhook(name='Site Create Webhook', type_create=True, payload_url=DUMMY_URL, secret=DUMMY_SECRET, additional_headers={'X-Foo': 'Bar'}), + Webhook(name='Site Create Webhook', type_create=True, payload_url=DUMMY_URL, secret=DUMMY_SECRET, additional_headers='X-Foo: Bar'), Webhook(name='Site Update Webhook', type_update=True, payload_url=DUMMY_URL, secret=DUMMY_SECRET), Webhook(name='Site Delete Webhook', type_delete=True, payload_url=DUMMY_URL, secret=DUMMY_SECRET), )) diff --git a/netbox/extras/webhooks_worker.py b/netbox/extras/webhooks_worker.py index b91561846..5513915ce 100644 --- a/netbox/extras/webhooks_worker.py +++ b/netbox/extras/webhooks_worker.py @@ -1,14 +1,10 @@ -import json import logging import requests from django_rq import job from jinja2.exceptions import TemplateError -from rest_framework.utils.encoders import JSONEncoder -from utilities.utils import render_jinja2 from .choices import ObjectChangeActionChoices -from .constants import HTTP_CONTENT_TYPE_JSON from .webhooks import generate_signature logger = logging.getLogger('netbox.webhooks_worker') @@ -28,55 +24,56 @@ def process_webhook(webhook, data, model_name, event, timestamp, username, reque 'data': data } - # Build HTTP headers + # Build the headers for the HTTP request headers = { 'Content-Type': webhook.http_content_type, } - if webhook.additional_headers: - headers.update(webhook.additional_headers) + try: + headers.update(webhook.render_headers(context)) + except (TemplateError, ValueError) as e: + logger.error("Error parsing HTTP headers for webhook {}: {}".format(webhook, e)) + raise e + # Render the request body + try: + body = webhook.render_body(context) + except TemplateError as e: + logger.error("Error rendering request body for webhook {}: {}".format(webhook, e)) + raise e + + # Prepare the HTTP request params = { 'method': 'POST', 'url': webhook.payload_url, - 'headers': headers + 'headers': headers, + 'data': body, } - logger.info( "Sending webhook to {}: {} {}".format(params['url'], context['model'], context['event']) ) + logger.debug(params) + try: + prepared_request = requests.Request(**params).prepare() + except requests.exceptions.RequestException as e: + logger.error("Error forming HTTP request: {}".format(e)) + raise e - # Construct the request body. If a template has been defined, use it. Otherwise, dump the context as either JSON - # or form data. - if webhook.body_template: - try: - params['data'] = render_jinja2(webhook.body_template, context) - except TemplateError as e: - logger.error("Error rendering request body: {}".format(e)) - return - elif webhook.http_content_type == HTTP_CONTENT_TYPE_JSON: - params['data'] = json.dumps(context, cls=JSONEncoder) - else: - params['data'] = context - - prepared_request = requests.Request(**params).prepare() - + # If a secret key is defined, sign the request with a hash of the key and its content if webhook.secret != '': - # Sign the request with a hash of the secret key and its content. prepared_request.headers['X-Hook-Signature'] = generate_signature(prepared_request.body, webhook.secret) + # Send the request with requests.Session() as session: session.verify = webhook.ssl_verification if webhook.ca_file_path: session.verify = webhook.ca_file_path response = session.send(prepared_request) - logger.debug(params) - if 200 <= response.status_code <= 299: logger.info("Request succeeded; response status {}".format(response.status_code)) return 'Status {} returned, webhook successfully processed.'.format(response.status_code) else: - logger.error("Request failed; response status {}: {}".format(response.status_code, response.content)) + logger.warning("Request failed; response status {}: {}".format(response.status_code, response.content)) raise requests.exceptions.RequestException( "Status {} returned with content '{}', webhook FAILED to process.".format( response.status_code, response.content From 211311be9f37add9b77ad73968ca22be900949f0 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Mon, 24 Feb 2020 20:42:24 -0500 Subject: [PATCH 4/7] Add http_method field to Webhook --- netbox/extras/admin.py | 16 ++++++++++---- netbox/extras/choices.py | 21 +++++++++++++++++++ .../0038_webhook_template_support.py | 5 +++++ netbox/extras/models.py | 6 ++++++ netbox/extras/webhooks_worker.py | 6 ++++-- 5 files changed, 48 insertions(+), 6 deletions(-) diff --git a/netbox/extras/admin.py b/netbox/extras/admin.py index 9a75fb53f..7122f3842 100644 --- a/netbox/extras/admin.py +++ b/netbox/extras/admin.py @@ -47,16 +47,24 @@ class WebhookAdmin(admin.ModelAdmin): form = WebhookForm fieldsets = ( (None, { - 'fields': ('name', 'obj_type', 'enabled') + 'fields': ( + 'name', 'obj_type', 'enabled', + ) }), ('Events', { - 'fields': ('type_create', 'type_update', 'type_delete') + 'fields': ( + 'type_create', 'type_update', 'type_delete', + ) }), ('HTTP Request', { - 'fields': ('payload_url', 'http_content_type', 'additional_headers', 'body_template', 'secret') + 'fields': ( + 'http_method', 'payload_url', 'http_content_type', 'additional_headers', 'body_template', 'secret', + ) }), ('SSL', { - 'fields': ('ssl_verification', 'ca_file_path') + 'fields': ( + 'ssl_verification', 'ca_file_path', + ) }) ) diff --git a/netbox/extras/choices.py b/netbox/extras/choices.py index 434461e76..9811cc0b0 100644 --- a/netbox/extras/choices.py +++ b/netbox/extras/choices.py @@ -118,3 +118,24 @@ class TemplateLanguageChoices(ChoiceSet): LANGUAGE_DJANGO: 10, LANGUAGE_JINJA2: 20, } + + +# +# Webhooks +# + +class WebhookHttpMethodChoices(ChoiceSet): + + METHOD_GET = 'GET' + METHOD_POST = 'POST' + METHOD_PUT = 'PUT' + METHOD_PATCH = 'PATCH' + METHOD_DELETE = 'DELETE' + + CHOICES = ( + (METHOD_GET, 'GET'), + (METHOD_POST, 'POST'), + (METHOD_PUT, 'PUT'), + (METHOD_PATCH, 'PATCH'), + (METHOD_DELETE, 'DELETE'), + ) diff --git a/netbox/extras/migrations/0038_webhook_template_support.py b/netbox/extras/migrations/0038_webhook_template_support.py index 80a1d2b7d..7d563820f 100644 --- a/netbox/extras/migrations/0038_webhook_template_support.py +++ b/netbox/extras/migrations/0038_webhook_template_support.py @@ -21,6 +21,11 @@ class Migration(migrations.Migration): ] operations = [ + migrations.AddField( + model_name='webhook', + name='http_method', + field=models.CharField(default='POST', max_length=30), + ), migrations.AddField( model_name='webhook', name='body_template', diff --git a/netbox/extras/models.py b/netbox/extras/models.py index 94392189d..eaeface6b 100644 --- a/netbox/extras/models.py +++ b/netbox/extras/models.py @@ -85,6 +85,12 @@ class Webhook(models.Model): enabled = models.BooleanField( default=True ) + http_method = models.CharField( + max_length=30, + choices=WebhookHttpMethodChoices, + default=WebhookHttpMethodChoices.METHOD_POST, + verbose_name='HTTP method' + ) http_content_type = models.CharField( max_length=100, default=HTTP_CONTENT_TYPE_JSON, diff --git a/netbox/extras/webhooks_worker.py b/netbox/extras/webhooks_worker.py index 5513915ce..1b1b76dd9 100644 --- a/netbox/extras/webhooks_worker.py +++ b/netbox/extras/webhooks_worker.py @@ -43,13 +43,15 @@ def process_webhook(webhook, data, model_name, event, timestamp, username, reque # Prepare the HTTP request params = { - 'method': 'POST', + 'method': webhook.http_method, 'url': webhook.payload_url, 'headers': headers, 'data': body, } logger.info( - "Sending webhook to {}: {} {}".format(params['url'], context['model'], context['event']) + "Sending {} request to {} ({} {})".format( + params['method'], params['url'], context['model'], context['event'] + ) ) logger.debug(params) try: From 644b4aa42d4177772b9ee800f143f945e15f6504 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Tue, 25 Feb 2020 10:24:27 -0500 Subject: [PATCH 5/7] Revised webhook documentation --- docs/additional-features/webhooks.md | 98 ++++++++++++++++------------ 1 file changed, 55 insertions(+), 43 deletions(-) diff --git a/docs/additional-features/webhooks.md b/docs/additional-features/webhooks.md index 9a02449f8..ebbff8f62 100644 --- a/docs/additional-features/webhooks.md +++ b/docs/additional-features/webhooks.md @@ -1,61 +1,73 @@ # Webhooks -A webhook defines an HTTP request that is sent to an external application when certain types of objects are created, updated, and/or deleted in NetBox. When a webhook is triggered, a POST request is sent to its configured URL. This request will include a full representation of the object being modified for consumption by the receiver. Webhooks are configured via the admin UI under Extras > Webhooks. +A webhook is a mechanism for conveying to some external system a change that took place in NetBox. For example, you may want to notify a monitoring system whenever a device status is changed in NetBox. This can be done by creating a webhook for the device model in NetBox. When NetBox detects a change to a device, an HTTP request containing the details of the change and who made it be sent to the specified receiver. Webhooks are configured in the admin UI under Extras > Webhooks. -An optional secret key can be configured for each webhook. This will append a `X-Hook-Signature` header to the request, consisting of a HMAC (SHA-512) hex digest of the request body using the secret as the key. This digest can be used by the receiver to authenticate the request's content. +## Configuration -## Requests +* **Name** - A unique name for the webhook. The name is not included with outbound messages. +* **Object type(s)** - The type or types of NetBox object that will trigger the webhook. +* **Enabled** - If unchecked, the webhook will be inactive. +* **Events** - A webhook may trigger on any combination of create, update, and delete events. At least one event type must be selected. +* **HTTP method** - The type of HTTP request to send. Options include GET, POST, PUT, PATCH, and DELETE. +* **URL** - The fuly-qualified URL of the request to be sent. This may specify a destination port number if needed. +* **HTTP content type** - The value of the request's `Content-Type` header. (Defaults to `application/json`) +* **Additional headers** - Any additional headers to include with the request (optional). Add one header per line in the format `Name: Value`. Jinja2 templating is supported for this field (see below). +* **Body template** - The content of the request being sent (optional). Jinja2 templating is supported for this field (see below). If blank, NetBox will populate the request body with a raw dump of the webhook context. (If the HTTP cotent type is set to `application/json`, this will be formatted as a JSON object.) +* **Secret** - A secret string used to prove authenticity of the request (optional). This will append a `X-Hook-Signature` header to the request, consisting of a HMAC (SHA-512) hex digest of the request body using the secret as the key. +* **SSL verification** - Uncheck this option to disable validation of the receiver's SSL certificate. (Disable with caution!) +* **CA file path** - The file path to a particular certificate authority (CA) file to use when validating the receiver's SSL certificate (optional). -The webhook POST request is structured as so (assuming `application/json` as the Content-Type): +## Jinja2 Template Support + +[Jinja2 templating](https://jinja.palletsprojects.com/) is supported for the `additional_headers` and `body_template` fields. This enables the user to convey change data in the request headers as well as to craft a customized request body. Request content can be crafted to enable the direct interaction with external systems by ensuring the outgoing message is in a format the receiver expects and understands. + +For example, you might create a NetBox webhook to [trigger a Slack message](https://api.slack.com/messaging/webhooks) any time an IP address is created. You can accomplish this using the following configuration: + +* Object type: IPAM > IP address +* HTTP method: POST +* URL: +* HTTP content type: `application/json` +* Body template: `{"text": "IP address {{ data['address'] }} was created by {{ username }}!"}` + +### Available Context + +The following data is available as context for Jinja2 templates: + +* `event` - The type of event which triggered the webhook: created, updated, or deleted. +* `model` - The NetBox model which triggered the change. +* `timestamp` - The time at which the event occurred (in [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601) format). +* `username` - The name of the user account associated with the change. +* `request_id` - The unique request ID. This may be used to correlate multiple changes associated with a single request. +* `data` - A serialized representation of the object _after_ the change was made. This is typically equivalent to the model's representation in NetBox's REST API. + +### Default Request Body + +If no body template is specified, and the HTTP content type is `application/json`, the request body will be populated with a JSON object containing the context data. For example, a newly created site might appear as follows: ```no-highlight { "event": "created", - "timestamp": "2019-10-12 12:51:29.746944", - "username": "admin", + "timestamp": "2020-02-25 15:10:26.010582+00:00", "model": "site", - "request_id": "43d8e212-94c7-4f67-b544-0dcde4fc0f43", + "username": "jstretch", + "request_id": "fdbca812-3142-4783-b364-2e2bd5c16c6a", "data": { + "id": 19, + "name": "Site 1", + "slug": "site-1", + "status": + "value": "active", + "label": "Active", + "id": 1 + }, + "region": null, ... } } ``` -`data` is the serialized representation of the model instance(s) from the event. The same serializers from the NetBox API are used. So an example of the payload for a Site delete event would be: +## Webhook Processing -```no-highlight -{ - "event": "deleted", - "timestamp": "2019-10-12 12:55:44.030750", - "username": "johnsmith", - "model": "site", - "request_id": "e9bb83b2-ebe4-4346-b13f-07144b1a00b4", - "data": { - "asn": None, - "comments": "", - "contact_email": "", - "contact_name": "", - "contact_phone": "", - "count_circuits": 0, - "count_devices": 0, - "count_prefixes": 0, - "count_racks": 0, - "count_vlans": 0, - "custom_fields": {}, - "facility": "", - "id": 54, - "name": "test", - "physical_address": "", - "region": None, - "shipping_address": "", - "slug": "test", - "tenant": None - } -} -``` +When a change is detected, any resulting webhooks are placed into a Redis queue for processing. This allows the user's request to complete without needing to wait for the outgoing webhook(s) to be processed. The webhooks are then extracted from the queue by the `rqworker` process and HTTP requests are sent to their respective destinations. The current webhook queue and any failed webhooks can be inspected in the admin UI under Django RQ > Queues. -A request is considered successful if the response status code is any one of a list of "good" statuses defined in the [requests library](https://github.com/requests/requests/blob/205755834d34a8a6ecf2b0b5b2e9c3e6a7f4e4b6/requests/models.py#L688), otherwise the request is marked as having failed. The user may manually retry a failed request. - -## Backend Status - -Django-rq includes a status page in the admin site which can be used to view the result of processed webhooks and manually retry any failed webhooks. Access it from http://netbox.local/admin/webhook-backend-status/. +A request is considered successful if the response has a 2XX status code; otherwise, the request is marked as having failed. Failed requests may be retried manually via the admin UI. From c3b64164bad070161eb9d2ee29114db73c962bec Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Tue, 25 Feb 2020 10:43:14 -0500 Subject: [PATCH 6/7] Always use a JSON object to convey change data when no body template is present --- docs/additional-features/webhooks.md | 2 +- netbox/extras/admin.py | 2 +- netbox/extras/models.py | 14 +++++++------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/additional-features/webhooks.md b/docs/additional-features/webhooks.md index ebbff8f62..310e67bf5 100644 --- a/docs/additional-features/webhooks.md +++ b/docs/additional-features/webhooks.md @@ -42,7 +42,7 @@ The following data is available as context for Jinja2 templates: ### Default Request Body -If no body template is specified, and the HTTP content type is `application/json`, the request body will be populated with a JSON object containing the context data. For example, a newly created site might appear as follows: +If no body template is specified, the request body will be populated with a JSON object containing the context data. For example, a newly created site might appear as follows: ```no-highlight { diff --git a/netbox/extras/admin.py b/netbox/extras/admin.py index 7122f3842..f66cc248f 100644 --- a/netbox/extras/admin.py +++ b/netbox/extras/admin.py @@ -58,7 +58,7 @@ class WebhookAdmin(admin.ModelAdmin): }), ('HTTP Request', { 'fields': ( - 'http_method', 'payload_url', 'http_content_type', 'additional_headers', 'body_template', 'secret', + 'payload_url', 'http_method', 'http_content_type', 'additional_headers', 'body_template', 'secret', ) }), ('SSL', { diff --git a/netbox/extras/models.py b/netbox/extras/models.py index eaeface6b..d81fbeab9 100644 --- a/netbox/extras/models.py +++ b/netbox/extras/models.py @@ -106,10 +106,9 @@ class Webhook(models.Model): ) body_template = models.TextField( blank=True, - help_text='Jinja2 template for a custom request body. If blank, a JSON object or form data representing the ' - 'change will be included. Available context data includes: event, ' - 'timestamp, model, username, request_id, and ' - 'data.' + help_text='Jinja2 template for a custom request body. If blank, a JSON object representing the change will be ' + 'included. Available context data includes: event, model, ' + 'timestamp, username, request_id, and data.' ) secret = models.CharField( max_length=255, @@ -165,12 +164,13 @@ class Webhook(models.Model): return ret def render_body(self, context): + """ + Render the body template, if defined. Otherwise, jump the context as a JSON object. + """ if self.body_template: return render_jinja2(self.body_template, context) - elif self.http_content_type == HTTP_CONTENT_TYPE_JSON: - return json.dumps(context, cls=JSONEncoder) else: - return context + return json.dumps(context, cls=JSONEncoder) # From 35786966c618b14f9f525ed5834aa04354223695 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Tue, 25 Feb 2020 10:46:16 -0500 Subject: [PATCH 7/7] Changelog for #4237 --- docs/release-notes/version-2.7.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index 45990bd29..c6b6db832 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -4,6 +4,7 @@ * [#3145](https://github.com/netbox-community/netbox/issues/3145) - Add a "decommissioning" cable status * [#4173](https://github.com/netbox-community/netbox/issues/4173) - Return graceful error message when webhook queuing fails +* [#4237](https://github.com/netbox-community/netbox/issues/4237) - Support Jinja2 templating for webhook payload and headers ## Bug Fixes