diff --git a/README.md b/README.md index 478f37e5e..5167c53c4 100644 --- a/README.md +++ b/README.md @@ -26,8 +26,12 @@ or join us in the #netbox Slack channel on [NetworkToCode](https://networktocode ![Screenshot of main page](docs/media/screenshot1.png "Main page") +--- + ![Screenshot of rack elevation](docs/media/screenshot2.png "Rack elevation") +--- + ![Screenshot of prefix hierarchy](docs/media/screenshot3.png "Prefix hierarchy") # Installation diff --git a/docs/additional-features/custom-scripts.md b/docs/additional-features/custom-scripts.md index cf98a6290..0904f8c82 100644 --- a/docs/additional-features/custom-scripts.md +++ b/docs/additional-features/custom-scripts.md @@ -27,11 +27,17 @@ class MyScript(Script): var2 = IntegerVar(...) var3 = ObjectVar(...) - def run(self, data): + def run(self, data, commit): ... ``` -The `run()` method is passed a single argument: a dictionary containing all of the variable data passed via the web form. Your script can reference this data during execution. +The `run()` method should accept two arguments: + +* `data` - A dictionary containing all of the variable data passed via the web form. +* `commit` - A boolean indicating whether database changes will be committed. + +!!! note + The `commit` argument was introduced in NetBox v2.7.8. Backward compatibility is maintained for scripts which accept only the `data` argument, however moving forward scripts should accept both arguments. Defining variables is optional: You may create a script with only a `run()` method if no user input is needed. @@ -196,7 +202,7 @@ These variables are presented as a web form to be completed by the user. Once su ``` from django.utils.text import slugify -from dcim.constants import * +from dcim.choices import DeviceStatusChoices, SiteStatusChoices from dcim.models import Device, DeviceRole, DeviceType, Site from extras.scripts import * @@ -222,13 +228,13 @@ class NewBranchScript(Script): ) ) - def run(self, data): + def run(self, data, commit): # Create the new site site = Site( name=data['site_name'], slug=slugify(data['site_name']), - status=SITE_STATUS_PLANNED + status=SiteStatusChoices.STATUS_PLANNED ) site.save() self.log_success("Created new site: {}".format(site)) @@ -240,7 +246,7 @@ class NewBranchScript(Script): device_type=data['switch_model'], name='{}-switch{}'.format(site.slug, i), site=site, - status=DEVICE_STATUS_PLANNED, + status=DeviceStatusChoices.STATUS_PLANNED, device_role=switch_role ) switch.save() diff --git a/docs/additional-features/webhooks.md b/docs/additional-features/webhooks.md index 9a02449f8..310e67bf5 100644 --- a/docs/additional-features/webhooks.md +++ b/docs/additional-features/webhooks.md @@ -1,61 +1,73 @@ # Webhooks -A webhook defines an HTTP request that is sent to an external application when certain types of objects are created, updated, and/or deleted in NetBox. When a webhook is triggered, a POST request is sent to its configured URL. This request will include a full representation of the object being modified for consumption by the receiver. Webhooks are configured via the admin UI under Extras > Webhooks. +A webhook is a mechanism for conveying to some external system a change that took place in NetBox. For example, you may want to notify a monitoring system whenever a device status is changed in NetBox. This can be done by creating a webhook for the device model in NetBox. When NetBox detects a change to a device, an HTTP request containing the details of the change and who made it be sent to the specified receiver. Webhooks are configured in the admin UI under Extras > Webhooks. -An optional secret key can be configured for each webhook. This will append a `X-Hook-Signature` header to the request, consisting of a HMAC (SHA-512) hex digest of the request body using the secret as the key. This digest can be used by the receiver to authenticate the request's content. +## Configuration -## Requests +* **Name** - A unique name for the webhook. The name is not included with outbound messages. +* **Object type(s)** - The type or types of NetBox object that will trigger the webhook. +* **Enabled** - If unchecked, the webhook will be inactive. +* **Events** - A webhook may trigger on any combination of create, update, and delete events. At least one event type must be selected. +* **HTTP method** - The type of HTTP request to send. Options include GET, POST, PUT, PATCH, and DELETE. +* **URL** - The fuly-qualified URL of the request to be sent. This may specify a destination port number if needed. +* **HTTP content type** - The value of the request's `Content-Type` header. (Defaults to `application/json`) +* **Additional headers** - Any additional headers to include with the request (optional). Add one header per line in the format `Name: Value`. Jinja2 templating is supported for this field (see below). +* **Body template** - The content of the request being sent (optional). Jinja2 templating is supported for this field (see below). If blank, NetBox will populate the request body with a raw dump of the webhook context. (If the HTTP cotent type is set to `application/json`, this will be formatted as a JSON object.) +* **Secret** - A secret string used to prove authenticity of the request (optional). This will append a `X-Hook-Signature` header to the request, consisting of a HMAC (SHA-512) hex digest of the request body using the secret as the key. +* **SSL verification** - Uncheck this option to disable validation of the receiver's SSL certificate. (Disable with caution!) +* **CA file path** - The file path to a particular certificate authority (CA) file to use when validating the receiver's SSL certificate (optional). -The webhook POST request is structured as so (assuming `application/json` as the Content-Type): +## Jinja2 Template Support + +[Jinja2 templating](https://jinja.palletsprojects.com/) is supported for the `additional_headers` and `body_template` fields. This enables the user to convey change data in the request headers as well as to craft a customized request body. Request content can be crafted to enable the direct interaction with external systems by ensuring the outgoing message is in a format the receiver expects and understands. + +For example, you might create a NetBox webhook to [trigger a Slack message](https://api.slack.com/messaging/webhooks) any time an IP address is created. You can accomplish this using the following configuration: + +* Object type: IPAM > IP address +* HTTP method: POST +* URL: +* HTTP content type: `application/json` +* Body template: `{"text": "IP address {{ data['address'] }} was created by {{ username }}!"}` + +### Available Context + +The following data is available as context for Jinja2 templates: + +* `event` - The type of event which triggered the webhook: created, updated, or deleted. +* `model` - The NetBox model which triggered the change. +* `timestamp` - The time at which the event occurred (in [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601) format). +* `username` - The name of the user account associated with the change. +* `request_id` - The unique request ID. This may be used to correlate multiple changes associated with a single request. +* `data` - A serialized representation of the object _after_ the change was made. This is typically equivalent to the model's representation in NetBox's REST API. + +### Default Request Body + +If no body template is specified, the request body will be populated with a JSON object containing the context data. For example, a newly created site might appear as follows: ```no-highlight { "event": "created", - "timestamp": "2019-10-12 12:51:29.746944", - "username": "admin", + "timestamp": "2020-02-25 15:10:26.010582+00:00", "model": "site", - "request_id": "43d8e212-94c7-4f67-b544-0dcde4fc0f43", + "username": "jstretch", + "request_id": "fdbca812-3142-4783-b364-2e2bd5c16c6a", "data": { + "id": 19, + "name": "Site 1", + "slug": "site-1", + "status": + "value": "active", + "label": "Active", + "id": 1 + }, + "region": null, ... } } ``` -`data` is the serialized representation of the model instance(s) from the event. The same serializers from the NetBox API are used. So an example of the payload for a Site delete event would be: +## Webhook Processing -```no-highlight -{ - "event": "deleted", - "timestamp": "2019-10-12 12:55:44.030750", - "username": "johnsmith", - "model": "site", - "request_id": "e9bb83b2-ebe4-4346-b13f-07144b1a00b4", - "data": { - "asn": None, - "comments": "", - "contact_email": "", - "contact_name": "", - "contact_phone": "", - "count_circuits": 0, - "count_devices": 0, - "count_prefixes": 0, - "count_racks": 0, - "count_vlans": 0, - "custom_fields": {}, - "facility": "", - "id": 54, - "name": "test", - "physical_address": "", - "region": None, - "shipping_address": "", - "slug": "test", - "tenant": None - } -} -``` +When a change is detected, any resulting webhooks are placed into a Redis queue for processing. This allows the user's request to complete without needing to wait for the outgoing webhook(s) to be processed. The webhooks are then extracted from the queue by the `rqworker` process and HTTP requests are sent to their respective destinations. The current webhook queue and any failed webhooks can be inspected in the admin UI under Django RQ > Queues. -A request is considered successful if the response status code is any one of a list of "good" statuses defined in the [requests library](https://github.com/requests/requests/blob/205755834d34a8a6ecf2b0b5b2e9c3e6a7f4e4b6/requests/models.py#L688), otherwise the request is marked as having failed. The user may manually retry a failed request. - -## Backend Status - -Django-rq includes a status page in the admin site which can be used to view the result of processed webhooks and manually retry any failed webhooks. Access it from http://netbox.local/admin/webhook-backend-status/. +A request is considered successful if the response has a 2XX status code; otherwise, the request is marked as having failed. Failed requests may be retried manually via the admin UI. diff --git a/docs/installation/3-http-daemon.md b/docs/installation/3-http-daemon.md index cc1065fef..4225f8413 100644 --- a/docs/installation/3-http-daemon.md +++ b/docs/installation/3-http-daemon.md @@ -99,6 +99,9 @@ Save the contents of the above example in `/etc/apache2/sites-available/netbox.c To enable SSL, consider this guide on [securing Apache with Let's Encrypt](https://www.digitalocean.com/community/tutorials/how-to-secure-apache-with-let-s-encrypt-on-ubuntu-16-04). +!!! note + Certain components of NetBox (such as the display of rack elevation diagrams) rely on the use of embedded objects. Ensure that your HTTP server configuration does not override the `X-Frame-Options` response header set by NetBox. + # gunicorn Installation Install gunicorn: diff --git a/docs/media/screenshot1.png b/docs/media/screenshot1.png index 3939199d9..e25a5ed93 100644 Binary files a/docs/media/screenshot1.png and b/docs/media/screenshot1.png differ diff --git a/docs/media/screenshot2.png b/docs/media/screenshot2.png index 3a73fe048..bce0a3f5d 100644 Binary files a/docs/media/screenshot2.png and b/docs/media/screenshot2.png differ diff --git a/docs/media/screenshot3.png b/docs/media/screenshot3.png index d2acb9773..6cf67d8f7 100644 Binary files a/docs/media/screenshot3.png and b/docs/media/screenshot3.png differ diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index 95cdb70dd..f9557bf34 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -1,3 +1,32 @@ +# v2.7.8 (2020-02-25) + +## Enhancements + +* [#3145](https://github.com/netbox-community/netbox/issues/3145) - Add a "decommissioning" cable status +* [#4173](https://github.com/netbox-community/netbox/issues/4173) - Return graceful error message when webhook queuing fails +* [#4227](https://github.com/netbox-community/netbox/issues/4227) - Omit internal fields from the change log data +* [#4237](https://github.com/netbox-community/netbox/issues/4237) - Support Jinja2 templating for webhook payload and headers +* [#4262](https://github.com/netbox-community/netbox/issues/4262) - Extend custom scripts to pass the `commit` value via `run()` +* [#4267](https://github.com/netbox-community/netbox/issues/4267) - Denote rack role on rack elevations list + +## Bug Fixes + +* [#4221](https://github.com/netbox-community/netbox/issues/4221) - Fix exception when deleting a device with interface connections when an interfaces webhook is defined +* [#4222](https://github.com/netbox-community/netbox/issues/4222) - Escape double quotes on encapsulated values during CSV export +* [#4224](https://github.com/netbox-community/netbox/issues/4224) - Fix display of rear device image if front image is not defined +* [#4228](https://github.com/netbox-community/netbox/issues/4228) - Improve fit of device images in rack elevations +* [#4230](https://github.com/netbox-community/netbox/issues/4230) - Fix rack units filtering on elevation endpoint +* [#4232](https://github.com/netbox-community/netbox/issues/4232) - Enforce consistent background striping in rack elevations +* [#4235](https://github.com/netbox-community/netbox/issues/4235) - Fix API representation of `content_type` for export templates +* [#4239](https://github.com/netbox-community/netbox/issues/4239) - Fix exception when selecting all filtered objects during bulk edit +* [#4240](https://github.com/netbox-community/netbox/issues/4240) - Fix exception when filtering foreign keys by NULL +* [#4241](https://github.com/netbox-community/netbox/issues/4241) - Correct IP address hyperlinks on interface view +* [#4246](https://github.com/netbox-community/netbox/issues/4246) - Fix duplication of field attributes when multiple IPNetworkVars are present in a script +* [#4252](https://github.com/netbox-community/netbox/issues/4252) - Fix power port assignment for power outlet templates created via REST API +* [#4272](https://github.com/netbox-community/netbox/issues/4272) - Interface type should be required by API serializer + +--- + # v2.7.7 (2020-02-20) **Note:** This release fixes a bug affecting the natural ordering of interfaces. If any interfaces appear unordered in @@ -5,7 +34,7 @@ NetBox, run the following management command to recalculate their naturalized va ``` python3 manage.py renaturalize dcim.Interface -``` +``` ## Enhancements diff --git a/netbox/dcim/api/nested_serializers.py b/netbox/dcim/api/nested_serializers.py index 30d84f703..bb2d61faa 100644 --- a/netbox/dcim/api/nested_serializers.py +++ b/netbox/dcim/api/nested_serializers.py @@ -3,8 +3,8 @@ from rest_framework import serializers from dcim.constants import CONNECTION_STATUS_CHOICES from dcim.models import ( Cable, ConsolePort, ConsoleServerPort, Device, DeviceBay, DeviceType, DeviceRole, FrontPort, FrontPortTemplate, - Interface, Manufacturer, Platform, PowerFeed, PowerOutlet, PowerPanel, PowerPort, Rack, RackGroup, RackRole, - RearPort, RearPortTemplate, Region, Site, VirtualChassis, + Interface, Manufacturer, Platform, PowerFeed, PowerOutlet, PowerPanel, PowerPort, PowerPortTemplate, Rack, + RackGroup, RackRole, RearPort, RearPortTemplate, Region, Site, VirtualChassis, ) from utilities.api import ChoiceField, WritableNestedSerializer @@ -25,6 +25,7 @@ __all__ = [ 'NestedPowerOutletSerializer', 'NestedPowerPanelSerializer', 'NestedPowerPortSerializer', + 'NestedPowerPortTemplateSerializer', 'NestedRackGroupSerializer', 'NestedRackRoleSerializer', 'NestedRackSerializer', @@ -111,6 +112,14 @@ class NestedDeviceTypeSerializer(WritableNestedSerializer): fields = ['id', 'url', 'manufacturer', 'model', 'slug', 'display_name', 'device_count'] +class NestedPowerPortTemplateSerializer(WritableNestedSerializer): + url = serializers.HyperlinkedIdentityField(view_name='dcim-api:powerporttemplate-detail') + + class Meta: + model = PowerPortTemplate + fields = ['id', 'url', 'name'] + + class NestedRearPortTemplateSerializer(WritableNestedSerializer): url = serializers.HyperlinkedIdentityField(view_name='dcim-api:rearporttemplate-detail') diff --git a/netbox/dcim/api/serializers.py b/netbox/dcim/api/serializers.py index 34d9a7890..5483904f5 100644 --- a/netbox/dcim/api/serializers.py +++ b/netbox/dcim/api/serializers.py @@ -172,6 +172,10 @@ class RackReservationSerializer(ValidatedModelSerializer): class RackElevationDetailFilterSerializer(serializers.Serializer): + q = serializers.CharField( + required=False, + default=None + ) face = serializers.ChoiceField( choices=DeviceFaceChoices, default=DeviceFaceChoices.FACE_FRONT @@ -278,7 +282,7 @@ class PowerOutletTemplateSerializer(ValidatedModelSerializer): allow_blank=True, required=False ) - power_port = PowerPortTemplateSerializer( + power_port = NestedPowerPortTemplateSerializer( required=False ) feed_leg = ChoiceField( @@ -294,7 +298,7 @@ class PowerOutletTemplateSerializer(ValidatedModelSerializer): class InterfaceTemplateSerializer(ValidatedModelSerializer): device_type = NestedDeviceTypeSerializer() - type = ChoiceField(choices=InterfaceTypeChoices, required=False) + type = ChoiceField(choices=InterfaceTypeChoices) class Meta: model = InterfaceTemplate @@ -514,7 +518,7 @@ class PowerPortSerializer(TaggitSerializer, ConnectedEndpointSerializer): class InterfaceSerializer(TaggitSerializer, ConnectedEndpointSerializer): device = NestedDeviceSerializer() - type = ChoiceField(choices=InterfaceTypeChoices, required=False) + type = ChoiceField(choices=InterfaceTypeChoices) lag = NestedInterfaceSerializer(required=False, allow_null=True) mode = ChoiceField(choices=InterfaceModeChoices, allow_blank=True, required=False) untagged_vlan = NestedVLANSerializer(required=False, allow_null=True) diff --git a/netbox/dcim/api/views.py b/netbox/dcim/api/views.py index f8297fe46..c4a159126 100644 --- a/netbox/dcim/api/views.py +++ b/netbox/dcim/api/views.py @@ -237,6 +237,11 @@ class RackViewSet(CustomFieldModelViewSet): expand_devices=data['expand_devices'] ) + # Enable filtering rack units by ID + q = data['q'] + if q: + elevation = [u for u in elevation if q in str(u['id']) or q in str(u['name'])] + page = self.paginate_queryset(elevation) if page is not None: rack_units = serializers.RackUnitSerializer(page, many=True, context={'request': request}) diff --git a/netbox/dcim/choices.py b/netbox/dcim/choices.py index e5b77dbaf..c1d7b4053 100644 --- a/netbox/dcim/choices.py +++ b/netbox/dcim/choices.py @@ -973,10 +973,12 @@ class CableStatusChoices(ChoiceSet): STATUS_CONNECTED = 'connected' STATUS_PLANNED = 'planned' + STATUS_DECOMMISSIONING = 'decommissioning' CHOICES = ( (STATUS_CONNECTED, 'Connected'), (STATUS_PLANNED, 'Planned'), + (STATUS_DECOMMISSIONING, 'Decommissioning'), ) LEGACY_MAP = { diff --git a/netbox/dcim/constants.py b/netbox/dcim/constants.py index 13a5052e4..78a418283 100644 --- a/netbox/dcim/constants.py +++ b/netbox/dcim/constants.py @@ -9,10 +9,10 @@ from .choices import InterfaceTypeChoices RACK_U_HEIGHT_DEFAULT = 42 +RACK_ELEVATION_BORDER_WIDTH = 2 RACK_ELEVATION_LEGEND_WIDTH_DEFAULT = 30 - -RACK_ELEVATION_UNIT_WIDTH_DEFAULT = 230 -RACK_ELEVATION_UNIT_HEIGHT_DEFAULT = 20 +RACK_ELEVATION_UNIT_WIDTH_DEFAULT = 220 +RACK_ELEVATION_UNIT_HEIGHT_DEFAULT = 22 # @@ -61,13 +61,10 @@ POWERFEED_MAX_UTILIZATION_DEFAULT = 80 # Percentage # Cabling and connections # -# TODO: Replace with CableStatusChoices? # Console/power/interface connection statuses -CONNECTION_STATUS_PLANNED = False -CONNECTION_STATUS_CONNECTED = True CONNECTION_STATUS_CHOICES = [ - [CONNECTION_STATUS_PLANNED, 'Planned'], - [CONNECTION_STATUS_CONNECTED, 'Connected'], + [False, 'Not Connected'], + [True, 'Connected'], ] # Cable endpoint types diff --git a/netbox/dcim/elevations.py b/netbox/dcim/elevations.py index ed4a3d10a..a3672f60d 100644 --- a/netbox/dcim/elevations.py +++ b/netbox/dcim/elevations.py @@ -6,6 +6,7 @@ from django.utils.http import urlencode from utilities.utils import foreground_color from .choices import DeviceFaceChoices +from .constants import RACK_ELEVATION_BORDER_WIDTH class RackElevationSVG: @@ -22,8 +23,8 @@ class RackElevationSVG: @staticmethod def _add_gradient(drawing, id_, color): gradient = drawing.linearGradient( - start=('0', '0%'), - end=('0', '5%'), + start=(0, 0), + end=(0, 25), spreadMethod='repeat', id_=id_, gradientTransform='rotate(45, 0, 0)', @@ -75,7 +76,7 @@ class RackElevationSVG: if self.include_images and device.device_type.front_image: url = device.device_type.front_image.url image = drawing.image(href=url, insert=start, size=end, class_='device-image') - image.stretch() + image.fit(scale='slice') link.add(image) def _draw_device_rear(self, drawing, device, start, end, text): @@ -88,10 +89,10 @@ class RackElevationSVG: drawing.add(drawing.text(str(device), insert=text)) # Embed rear device type image if one exists - if self.include_images and device.device_type.front_image: + if self.include_images and device.device_type.rear_image: url = device.device_type.rear_image.url image = drawing.image(href=url, insert=start, size=end, class_='device-image') - image.stretch() + image.fit(scale='slice') drawing.add(image) @staticmethod @@ -134,13 +135,16 @@ class RackElevationSVG: """ Return an SVG document representing a rack elevation. """ - drawing = self._setup_drawing(unit_width + legend_width, unit_height * self.rack.u_height) + drawing = self._setup_drawing( + unit_width + legend_width + RACK_ELEVATION_BORDER_WIDTH * 2, + unit_height * self.rack.u_height + RACK_ELEVATION_BORDER_WIDTH * 2 + ) reserved_units = self.rack.get_reserved_units() unit_cursor = 0 for ru in range(0, self.rack.u_height): start_y = ru * unit_height - position_coordinates = (legend_width / 2, start_y + unit_height / 2 + 2) + position_coordinates = (legend_width / 2, start_y + unit_height / 2 + RACK_ELEVATION_BORDER_WIDTH) unit = ru + 1 if self.rack.desc_units else self.rack.u_height - ru drawing.add( drawing.text(str(unit), position_coordinates, class_="unit") @@ -153,11 +157,12 @@ class RackElevationSVG: height = unit.get('height', 1) # Setup drawing coordinates - start_y = unit_cursor * unit_height + x_offset = legend_width + RACK_ELEVATION_BORDER_WIDTH + y_offset = unit_cursor * unit_height + RACK_ELEVATION_BORDER_WIDTH end_y = unit_height * height - start_cordinates = (legend_width, start_y) - end_cordinates = (legend_width + unit_width, end_y) - text_cordinates = (legend_width + (unit_width / 2), start_y + end_y / 2) + start_cordinates = (x_offset, y_offset) + end_cordinates = (unit_width, end_y) + text_cordinates = (x_offset + (unit_width / 2), y_offset + end_y / 2) # Draw the device if device and device.face == face: @@ -187,6 +192,13 @@ class RackElevationSVG: unit_cursor += height # Wrap the drawing with a border - drawing.add(drawing.rect((legend_width, 1), (unit_width - 1, self.rack.u_height * unit_height - 2), class_='rack')) + border_width = RACK_ELEVATION_BORDER_WIDTH + border_offset = RACK_ELEVATION_BORDER_WIDTH / 2 + frame = drawing.rect( + insert=(legend_width + border_offset, border_offset), + size=(unit_width + border_width, self.rack.u_height * unit_height + border_width), + class_='rack' + ) + drawing.add(frame) return drawing diff --git a/netbox/dcim/models/__init__.py b/netbox/dcim/models/__init__.py index 5848a6201..821b8eeb7 100644 --- a/netbox/dcim/models/__init__.py +++ b/netbox/dcim/models/__init__.py @@ -20,10 +20,10 @@ from dcim.choices import * from dcim.constants import * from dcim.fields import ASNField from dcim.elevations import RackElevationSVG -from extras.models import ConfigContextModel, CustomFieldModel, TaggedItem +from extras.models import ConfigContextModel, CustomFieldModel, ObjectChange, TaggedItem from utilities.fields import ColorField, NaturalOrderingField from utilities.models import ChangeLoggedModel -from utilities.utils import to_meters +from utilities.utils import serialize_object, to_meters from .device_component_templates import ( ConsolePortTemplate, ConsoleServerPortTemplate, DeviceBayTemplate, FrontPortTemplate, InterfaceTemplate, PowerOutletTemplate, PowerPortTemplate, RearPortTemplate, @@ -118,6 +118,15 @@ class Region(MPTTModel, ChangeLoggedModel): Q(region__in=self.get_descendants()) ).count() + def to_objectchange(self, action): + # Remove MPTT-internal fields + return ObjectChange( + changed_object=self, + object_repr=str(self), + action=action, + object_data=serialize_object(self, exclude=['level', 'lft', 'rght', 'tree_id']) + ) + # # Sites @@ -1956,6 +1965,7 @@ class Cable(ChangeLoggedModel): STATUS_CLASS_MAP = { CableStatusChoices.STATUS_CONNECTED: 'success', CableStatusChoices.STATUS_PLANNED: 'info', + CableStatusChoices.STATUS_DECOMMISSIONING: 'warning', } class Meta: @@ -2116,14 +2126,14 @@ class Cable(ChangeLoggedModel): b_path = self.termination_a.trace() # Determine overall path status (connected or planned) - if self.status == CableStatusChoices.STATUS_PLANNED: - path_status = CONNECTION_STATUS_PLANNED - else: - path_status = CONNECTION_STATUS_CONNECTED + if self.status == CableStatusChoices.STATUS_CONNECTED: + path_status = True for segment in a_path[1:] + b_path[1:]: - if segment[1] is None or segment[1].status == CableStatusChoices.STATUS_PLANNED: - path_status = CONNECTION_STATUS_PLANNED + if segment[1] is None or segment[1].status != CableStatusChoices.STATUS_CONNECTED: + path_status = False break + else: + path_status = False a_endpoint = a_path[-1][2] b_endpoint = b_path[-1][2] diff --git a/netbox/dcim/models/device_components.py b/netbox/dcim/models/device_components.py index a41eda576..9a3d608d7 100644 --- a/netbox/dcim/models/device_components.py +++ b/netbox/dcim/models/device_components.py @@ -360,9 +360,21 @@ class PowerPort(CableTermination, ComponentModel): @property def connected_endpoint(self): - if self._connected_poweroutlet: - return self._connected_poweroutlet - return self._connected_powerfeed + """ + Return the connected PowerOutlet, if it exists, or the connected PowerFeed, if it exists. We have to check for + ObjectDoesNotExist in case the referenced object has been deleted from the database. + """ + try: + if self._connected_poweroutlet: + return self._connected_poweroutlet + except ObjectDoesNotExist: + pass + try: + if self._connected_powerfeed: + return self._connected_powerfeed + except ObjectDoesNotExist: + pass + return None @connected_endpoint.setter def connected_endpoint(self, value): @@ -717,9 +729,21 @@ class Interface(CableTermination, ComponentModel): @property def connected_endpoint(self): - if self._connected_interface: - return self._connected_interface - return self._connected_circuittermination + """ + Return the connected Interface, if it exists, or the connected CircuitTermination, if it exists. We have to + check for ObjectDoesNotExist in case the referenced object has been deleted from the database. + """ + try: + if self._connected_interface: + return self._connected_interface + except ObjectDoesNotExist: + pass + try: + if self._connected_circuittermination: + return self._connected_circuittermination + except ObjectDoesNotExist: + pass + return None @connected_endpoint.setter def connected_endpoint(self, value): diff --git a/netbox/dcim/tests/test_api.py b/netbox/dcim/tests/test_api.py index a3a072bc9..ad893bec6 100644 --- a/netbox/dcim/tests/test_api.py +++ b/netbox/dcim/tests/test_api.py @@ -596,6 +596,28 @@ class RackTest(APITestCase): self.assertEqual(response.data['count'], 42) + def test_get_elevation_rack_units(self): + + url = '{}?q=3'.format(reverse('dcim-api:rack-elevation', kwargs={'pk': self.rack1.pk})) + response = self.client.get(url, **self.header) + + self.assertEqual(response.data['count'], 13) + + url = '{}?q=U3'.format(reverse('dcim-api:rack-elevation', kwargs={'pk': self.rack1.pk})) + response = self.client.get(url, **self.header) + + self.assertEqual(response.data['count'], 11) + + url = '{}?q=10'.format(reverse('dcim-api:rack-elevation', kwargs={'pk': self.rack1.pk})) + response = self.client.get(url, **self.header) + + self.assertEqual(response.data['count'], 1) + + url = '{}?q=U20'.format(reverse('dcim-api:rack-elevation', kwargs={'pk': self.rack1.pk})) + response = self.client.get(url, **self.header) + + self.assertEqual(response.data['count'], 1) + def test_get_rack_elevation(self): url = reverse('dcim-api:rack-elevation', kwargs={'pk': self.rack1.pk}) @@ -1448,13 +1470,13 @@ class InterfaceTemplateTest(APITestCase): manufacturer=self.manufacturer, model='Test Device Type 1', slug='test-device-type-1' ) self.interfacetemplate1 = InterfaceTemplate.objects.create( - device_type=self.devicetype, name='Test Interface Template 1' + device_type=self.devicetype, name='Test Interface Template 1', type='1000base-t' ) self.interfacetemplate2 = InterfaceTemplate.objects.create( - device_type=self.devicetype, name='Test Interface Template 2' + device_type=self.devicetype, name='Test Interface Template 2', type='1000base-t' ) self.interfacetemplate3 = InterfaceTemplate.objects.create( - device_type=self.devicetype, name='Test Interface Template 3' + device_type=self.devicetype, name='Test Interface Template 3', type='1000base-t' ) def test_get_interfacetemplate(self): @@ -1476,6 +1498,7 @@ class InterfaceTemplateTest(APITestCase): data = { 'device_type': self.devicetype.pk, 'name': 'Test Interface Template 4', + 'type': '1000base-t', } url = reverse('dcim-api:interfacetemplate-list') @@ -1493,14 +1516,17 @@ class InterfaceTemplateTest(APITestCase): { 'device_type': self.devicetype.pk, 'name': 'Test Interface Template 4', + 'type': '1000base-t', }, { 'device_type': self.devicetype.pk, 'name': 'Test Interface Template 5', + 'type': '1000base-t', }, { 'device_type': self.devicetype.pk, 'name': 'Test Interface Template 6', + 'type': '1000base-t', }, ] @@ -1518,6 +1544,7 @@ class InterfaceTemplateTest(APITestCase): data = { 'device_type': self.devicetype.pk, 'name': 'Test Interface Template X', + 'type': '1000base-x-gbic', } url = reverse('dcim-api:interfacetemplate-detail', kwargs={'pk': self.interfacetemplate1.pk}) @@ -2628,9 +2655,9 @@ class InterfaceTest(APITestCase): self.device = Device.objects.create( device_type=devicetype, device_role=devicerole, name='Test Device 1', site=site ) - self.interface1 = Interface.objects.create(device=self.device, name='Test Interface 1') - self.interface2 = Interface.objects.create(device=self.device, name='Test Interface 2') - self.interface3 = Interface.objects.create(device=self.device, name='Test Interface 3') + self.interface1 = Interface.objects.create(device=self.device, name='Test Interface 1', type='1000base-t') + self.interface2 = Interface.objects.create(device=self.device, name='Test Interface 2', type='1000base-t') + self.interface3 = Interface.objects.create(device=self.device, name='Test Interface 3', type='1000base-t') self.vlan1 = VLAN.objects.create(name="Test VLAN 1", vid=1) self.vlan2 = VLAN.objects.create(name="Test VLAN 2", vid=2) @@ -2691,6 +2718,7 @@ class InterfaceTest(APITestCase): data = { 'device': self.device.pk, 'name': 'Test Interface 4', + 'type': '1000base-t', } url = reverse('dcim-api:interface-list') @@ -2707,6 +2735,7 @@ class InterfaceTest(APITestCase): data = { 'device': self.device.pk, 'name': 'Test Interface 4', + 'type': '1000base-t', 'mode': InterfaceModeChoices.MODE_TAGGED, 'untagged_vlan': self.vlan3.id, 'tagged_vlans': [self.vlan1.id, self.vlan2.id], @@ -2728,14 +2757,17 @@ class InterfaceTest(APITestCase): { 'device': self.device.pk, 'name': 'Test Interface 4', + 'type': '1000base-t', }, { 'device': self.device.pk, 'name': 'Test Interface 5', + 'type': '1000base-t', }, { 'device': self.device.pk, 'name': 'Test Interface 6', + 'type': '1000base-t', }, ] @@ -2754,6 +2786,7 @@ class InterfaceTest(APITestCase): { 'device': self.device.pk, 'name': 'Test Interface 4', + 'type': '1000base-t', 'mode': InterfaceModeChoices.MODE_TAGGED, 'untagged_vlan': self.vlan2.id, 'tagged_vlans': [self.vlan1.id], @@ -2761,6 +2794,7 @@ class InterfaceTest(APITestCase): { 'device': self.device.pk, 'name': 'Test Interface 5', + 'type': '1000base-t', 'mode': InterfaceModeChoices.MODE_TAGGED, 'untagged_vlan': self.vlan2.id, 'tagged_vlans': [self.vlan1.id], @@ -2768,6 +2802,7 @@ class InterfaceTest(APITestCase): { 'device': self.device.pk, 'name': 'Test Interface 6', + 'type': '1000base-t', 'mode': InterfaceModeChoices.MODE_TAGGED, 'untagged_vlan': self.vlan2.id, 'tagged_vlans': [self.vlan1.id], @@ -2793,6 +2828,7 @@ class InterfaceTest(APITestCase): data = { 'device': self.device.pk, 'name': 'Test Interface X', + 'type': '1000base-x-gbic', 'lag': lag_interface.pk, } diff --git a/netbox/dcim/tests/test_models.py b/netbox/dcim/tests/test_models.py index 32d864a51..c9d5c4c20 100644 --- a/netbox/dcim/tests/test_models.py +++ b/netbox/dcim/tests/test_models.py @@ -2,7 +2,6 @@ from django.core.exceptions import ValidationError from django.test import TestCase from dcim.choices import * -from dcim.constants import CONNECTION_STATUS_CONNECTED, CONNECTION_STATUS_PLANNED from dcim.models import * from tenancy.models import Tenant @@ -522,14 +521,14 @@ class CablePathTestCase(TestCase): cable3.save() interface1 = Interface.objects.get(pk=self.interface1.pk) self.assertEqual(interface1.connected_endpoint, self.interface2) - self.assertEqual(interface1.connection_status, CONNECTION_STATUS_PLANNED) + self.assertFalse(interface1.connection_status) # Switch third segment from planned to connected cable3.status = CableStatusChoices.STATUS_CONNECTED cable3.save() interface1 = Interface.objects.get(pk=self.interface1.pk) self.assertEqual(interface1.connected_endpoint, self.interface2) - self.assertEqual(interface1.connection_status, CONNECTION_STATUS_CONNECTED) + self.assertTrue(interface1.connection_status) def test_path_teardown(self): @@ -542,7 +541,7 @@ class CablePathTestCase(TestCase): cable3.save() interface1 = Interface.objects.get(pk=self.interface1.pk) self.assertEqual(interface1.connected_endpoint, self.interface2) - self.assertEqual(interface1.connection_status, CONNECTION_STATUS_CONNECTED) + self.assertTrue(interface1.connection_status) # Remove a cable cable2.delete() diff --git a/netbox/dcim/views.py b/netbox/dcim/views.py index 91b32bc70..8f9da2d68 100644 --- a/netbox/dcim/views.py +++ b/netbox/dcim/views.py @@ -357,7 +357,7 @@ class RackElevationListView(PermissionRequiredMixin, View): def get(self, request): - racks = Rack.objects.prefetch_related('site', 'group', 'tenant', 'role', 'devices__device_type') + racks = Rack.objects.prefetch_related('role') racks = filters.RackFilterSet(request.GET, racks).qs total_count = racks.count() diff --git a/netbox/extras/admin.py b/netbox/extras/admin.py index 2a39c207e..f66cc248f 100644 --- a/netbox/extras/admin.py +++ b/netbox/extras/admin.py @@ -26,7 +26,7 @@ class WebhookForm(forms.ModelForm): class Meta: model = Webhook - exclude = [] + exclude = () def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -38,13 +38,35 @@ class WebhookForm(forms.ModelForm): @admin.register(Webhook, site=admin_site) class WebhookAdmin(admin.ModelAdmin): list_display = [ - 'name', 'models', 'payload_url', 'http_content_type', 'enabled', 'type_create', 'type_update', - 'type_delete', 'ssl_verification', + 'name', 'models', 'payload_url', 'http_content_type', 'enabled', 'type_create', 'type_update', 'type_delete', + 'ssl_verification', ] list_filter = [ 'enabled', 'type_create', 'type_update', 'type_delete', 'obj_type', ] form = WebhookForm + fieldsets = ( + (None, { + 'fields': ( + 'name', 'obj_type', 'enabled', + ) + }), + ('Events', { + 'fields': ( + 'type_create', 'type_update', 'type_delete', + ) + }), + ('HTTP Request', { + 'fields': ( + 'payload_url', 'http_method', 'http_content_type', 'additional_headers', 'body_template', 'secret', + ) + }), + ('SSL', { + 'fields': ( + 'ssl_verification', 'ca_file_path', + ) + }) + ) def models(self, obj): return ', '.join([ct.name for ct in obj.obj_type.all()]) diff --git a/netbox/extras/api/serializers.py b/netbox/extras/api/serializers.py index 58433df25..40606ed8e 100644 --- a/netbox/extras/api/serializers.py +++ b/netbox/extras/api/serializers.py @@ -40,10 +40,14 @@ class GraphSerializer(ValidatedModelSerializer): class RenderedGraphSerializer(serializers.ModelSerializer): - embed_url = serializers.SerializerMethodField() - embed_link = serializers.SerializerMethodField() + embed_url = serializers.SerializerMethodField( + read_only=True + ) + embed_link = serializers.SerializerMethodField( + read_only=True + ) type = ContentTypeField( - queryset=ContentType.objects.all() + read_only=True ) class Meta: @@ -62,6 +66,9 @@ class RenderedGraphSerializer(serializers.ModelSerializer): # class ExportTemplateSerializer(ValidatedModelSerializer): + content_type = ContentTypeField( + queryset=ContentType.objects.filter(EXPORTTEMPLATE_MODELS), + ) template_language = ChoiceField( choices=TemplateLanguageChoices, default=TemplateLanguageChoices.LANGUAGE_JINJA2 diff --git a/netbox/extras/choices.py b/netbox/extras/choices.py index 0ae53f03d..9811cc0b0 100644 --- a/netbox/extras/choices.py +++ b/netbox/extras/choices.py @@ -124,17 +124,18 @@ class TemplateLanguageChoices(ChoiceSet): # Webhooks # -class WebhookContentTypeChoices(ChoiceSet): +class WebhookHttpMethodChoices(ChoiceSet): - CONTENTTYPE_JSON = 'application/json' - CONTENTTYPE_FORMDATA = 'application/x-www-form-urlencoded' + METHOD_GET = 'GET' + METHOD_POST = 'POST' + METHOD_PUT = 'PUT' + METHOD_PATCH = 'PATCH' + METHOD_DELETE = 'DELETE' CHOICES = ( - (CONTENTTYPE_JSON, 'JSON'), - (CONTENTTYPE_FORMDATA, 'Form data'), + (METHOD_GET, 'GET'), + (METHOD_POST, 'POST'), + (METHOD_PUT, 'PUT'), + (METHOD_PATCH, 'PATCH'), + (METHOD_DELETE, 'DELETE'), ) - - LEGACY_MAP = { - CONTENTTYPE_JSON: 1, - CONTENTTYPE_FORMDATA: 2, - } diff --git a/netbox/extras/constants.py b/netbox/extras/constants.py index b12bc2f2c..7bb026d34 100644 --- a/netbox/extras/constants.py +++ b/netbox/extras/constants.py @@ -138,6 +138,8 @@ LOG_LEVEL_CODES = { LOG_FAILURE: 'failure', } +HTTP_CONTENT_TYPE_JSON = 'application/json' + # Models which support registered webhooks WEBHOOK_MODELS = Q( Q(app_label='circuits', model__in=[ diff --git a/netbox/extras/middleware.py b/netbox/extras/middleware.py index 3624e11a5..c8be8755f 100644 --- a/netbox/extras/middleware.py +++ b/netbox/extras/middleware.py @@ -5,11 +5,14 @@ from copy import deepcopy from datetime import timedelta from django.conf import settings +from django.contrib import messages from django.db.models.signals import pre_delete, post_save from django.utils import timezone from django_prometheus.models import model_deletes, model_inserts, model_updates +from redis.exceptions import RedisError from extras.utils import is_taggable +from utilities.api import is_api_request from utilities.querysets import DummyQuerySet from .choices import ObjectChangeActionChoices from .models import ObjectChange @@ -98,7 +101,12 @@ class ObjectChangeMiddleware(object): if not _thread_locals.changed_objects: return response + # Disconnect our receivers from the post_save and post_delete signals. + post_save.disconnect(handle_changed_object, dispatch_uid='handle_changed_object') + pre_delete.disconnect(handle_deleted_object, dispatch_uid='handle_deleted_object') + # Create records for any cached objects that were changed. + redis_failed = False for instance, action in _thread_locals.changed_objects: # Refresh cached custom field values @@ -114,7 +122,16 @@ class ObjectChangeMiddleware(object): objectchange.save() # Enqueue webhooks - enqueue_webhooks(instance, request.user, request.id, action) + try: + enqueue_webhooks(instance, request.user, request.id, action) + except RedisError as e: + if not redis_failed and not is_api_request(request): + messages.error( + request, + "There was an error processing webhooks for this request. Check that the Redis service is " + "running and reachable. The full error details were: {}".format(e) + ) + redis_failed = True # Increment metric counters if action == ObjectChangeActionChoices.ACTION_CREATE: diff --git a/netbox/extras/migrations/0038_webhook_template_support.py b/netbox/extras/migrations/0038_webhook_template_support.py new file mode 100644 index 000000000..7d563820f --- /dev/null +++ b/netbox/extras/migrations/0038_webhook_template_support.py @@ -0,0 +1,48 @@ +import json + +from django.db import migrations, models + + +def json_to_text(apps, schema_editor): + """ + Convert a JSON representation of HTTP headers to key-value pairs (one header per line) + """ + Webhook = apps.get_model('extras', 'Webhook') + for webhook in Webhook.objects.exclude(additional_headers=''): + data = json.loads(webhook.additional_headers) + headers = ['{}: {}'.format(k, v) for k, v in data.items()] + Webhook.objects.filter(pk=webhook.pk).update(additional_headers='\n'.join(headers)) + + +class Migration(migrations.Migration): + + dependencies = [ + ('extras', '0037_configcontexts_clusters'), + ] + + operations = [ + migrations.AddField( + model_name='webhook', + name='http_method', + field=models.CharField(default='POST', max_length=30), + ), + migrations.AddField( + model_name='webhook', + name='body_template', + field=models.TextField(blank=True), + ), + migrations.AlterField( + model_name='webhook', + name='additional_headers', + field=models.TextField(blank=True, default=''), + preserve_default=False, + ), + migrations.AlterField( + model_name='webhook', + name='http_content_type', + field=models.CharField(default='application/json', max_length=100), + ), + migrations.RunPython( + code=json_to_text + ), + ] diff --git a/netbox/extras/models.py b/netbox/extras/models.py index 5d175d172..d81fbeab9 100644 --- a/netbox/extras/models.py +++ b/netbox/extras/models.py @@ -1,3 +1,4 @@ +import json from collections import OrderedDict from datetime import date @@ -12,6 +13,7 @@ from django.http import HttpResponse from django.template import Template, Context from django.urls import reverse from django.utils.text import slugify +from rest_framework.utils.encoders import JSONEncoder from taggit.models import TagBase, GenericTaggedItemBase from utilities.fields import ColorField @@ -52,7 +54,6 @@ class Webhook(models.Model): delete in NetBox. The request will contain a representation of the object, which the remote application can act on. Each Webhook can be limited to firing only on certain actions or certain object types. """ - obj_type = models.ManyToManyField( to=ContentType, related_name='webhooks', @@ -81,17 +82,33 @@ class Webhook(models.Model): verbose_name='URL', help_text="A POST will be sent to this URL when the webhook is called." ) - http_content_type = models.CharField( - max_length=50, - choices=WebhookContentTypeChoices, - default=WebhookContentTypeChoices.CONTENTTYPE_JSON, - verbose_name='HTTP content type' + enabled = models.BooleanField( + default=True ) - additional_headers = JSONField( - null=True, + http_method = models.CharField( + max_length=30, + choices=WebhookHttpMethodChoices, + default=WebhookHttpMethodChoices.METHOD_POST, + verbose_name='HTTP method' + ) + http_content_type = models.CharField( + max_length=100, + default=HTTP_CONTENT_TYPE_JSON, + verbose_name='HTTP content type', + help_text='The complete list of official content types is available ' + 'here.' + ) + additional_headers = models.TextField( blank=True, - help_text="User supplied headers which should be added to the request in addition to the HTTP content type. " - "Headers are supplied as key/value pairs in a JSON object." + help_text="User-supplied HTTP headers to be sent with the request in addition to the HTTP content type. " + "Headers should be defined in the format Name: Value. Jinja2 template processing is " + "support with the same context as the request body (below)." + ) + body_template = models.TextField( + blank=True, + help_text='Jinja2 template for a custom request body. If blank, a JSON object representing the change will be ' + 'included. Available context data includes: event, model, ' + 'timestamp, username, request_id, and data.' ) secret = models.CharField( max_length=255, @@ -101,9 +118,6 @@ class Webhook(models.Model): "the secret as the key. The secret is not transmitted in " "the request." ) - enabled = models.BooleanField( - default=True - ) ssl_verification = models.BooleanField( default=True, verbose_name='SSL verification', @@ -126,9 +140,6 @@ class Webhook(models.Model): return self.name def clean(self): - """ - Validate model - """ if not self.type_create and not self.type_delete and not self.type_update: raise ValidationError( "You must select at least one type: create, update, and/or delete." @@ -136,14 +147,30 @@ class Webhook(models.Model): if not self.ssl_verification and self.ca_file_path: raise ValidationError({ - 'ca_file_path': 'Do not specify a CA certificate file if SSL verification is dissabled.' + 'ca_file_path': 'Do not specify a CA certificate file if SSL verification is disabled.' }) - # Verify that JSON data is provided as an object - if self.additional_headers and type(self.additional_headers) is not dict: - raise ValidationError({ - 'additional_headers': 'Header JSON data must be in object form. Example: {"X-API-KEY": "abc123"}' - }) + def render_headers(self, context): + """ + Render additional_headers and return a dict of Header: Value pairs. + """ + if not self.additional_headers: + return {} + ret = {} + data = render_jinja2(self.additional_headers, context) + for line in data.splitlines(): + header, value = line.split(':') + ret[header.strip()] = value.strip() + return ret + + def render_body(self, context): + """ + Render the body template, if defined. Otherwise, jump the context as a JSON object. + """ + if self.body_template: + return render_jinja2(self.body_template, context) + else: + return json.dumps(context, cls=JSONEncoder) # diff --git a/netbox/extras/scripts.py b/netbox/extras/scripts.py index e5a32bde6..97fc50ea0 100644 --- a/netbox/extras/scripts.py +++ b/netbox/extras/scripts.py @@ -63,10 +63,6 @@ class ScriptVariable: self.field_attrs['widget'] = widget self.field_attrs['required'] = required - # Initialize the list of optional validators if none have already been defined - if 'validators' not in self.field_attrs: - self.field_attrs['validators'] = [] - def as_field(self): """ Render the variable as a Django form field. @@ -227,14 +223,12 @@ class IPNetworkVar(ScriptVariable): An IPv4 or IPv6 prefix. """ form_field = IPNetworkFormField - field_attrs = { - 'validators': [prefix_validator] - } def __init__(self, min_prefix_length=None, max_prefix_length=None, *args, **kwargs): super().__init__(*args, **kwargs) - # Optional minimum/maximum prefix lengths + # Set prefix validator and optional minimum/maximum prefix lengths + self.field_attrs['validators'] = [prefix_validator] if min_prefix_length is not None: self.field_attrs['validators'].append( MinPrefixLengthValidator(min_prefix_length) @@ -292,7 +286,7 @@ class BaseScript: return vars - def run(self, data): + def run(self, data, commit): raise NotImplementedError("The script must define a run() method.") def as_form(self, data=None, files=None, initial=None): @@ -389,10 +383,17 @@ def run_script(script, data, request, commit=True): # Add the current request as a property of the script script.request = request + # Determine whether the script accepts a 'commit' argument (this was introduced in v2.7.8) + kwargs = { + 'data': data + } + if 'commit' in inspect.signature(script.run).parameters: + kwargs['commit'] = commit + try: with transaction.atomic(): start_time = time.time() - output = script.run(data) + output = script.run(**kwargs) end_time = time.time() if not commit: raise AbortTransaction() diff --git a/netbox/extras/tests/test_api.py b/netbox/extras/tests/test_api.py index b15553a49..223e5a530 100644 --- a/netbox/extras/tests/test_api.py +++ b/netbox/extras/tests/test_api.py @@ -163,17 +163,17 @@ class ExportTemplateTest(APITestCase): super().setUp() - self.content_type = ContentType.objects.get_for_model(Device) + content_type = ContentType.objects.get_for_model(Device) self.exporttemplate1 = ExportTemplate.objects.create( - content_type=self.content_type, name='Test Export Template 1', + content_type=content_type, name='Test Export Template 1', template_code='{% for obj in queryset %}{{ obj.name }}\n{% endfor %}' ) self.exporttemplate2 = ExportTemplate.objects.create( - content_type=self.content_type, name='Test Export Template 2', + content_type=content_type, name='Test Export Template 2', template_code='{% for obj in queryset %}{{ obj.name }}\n{% endfor %}' ) self.exporttemplate3 = ExportTemplate.objects.create( - content_type=self.content_type, name='Test Export Template 3', + content_type=content_type, name='Test Export Template 3', template_code='{% for obj in queryset %}{{ obj.name }}\n{% endfor %}' ) @@ -194,7 +194,7 @@ class ExportTemplateTest(APITestCase): def test_create_exporttemplate(self): data = { - 'content_type': self.content_type.pk, + 'content_type': 'dcim.device', 'name': 'Test Export Template 4', 'template_code': '{% for obj in queryset %}{{ obj.name }}\n{% endfor %}', } @@ -205,7 +205,7 @@ class ExportTemplateTest(APITestCase): self.assertHttpStatus(response, status.HTTP_201_CREATED) self.assertEqual(ExportTemplate.objects.count(), 4) exporttemplate4 = ExportTemplate.objects.get(pk=response.data['id']) - self.assertEqual(exporttemplate4.content_type_id, data['content_type']) + self.assertEqual(exporttemplate4.content_type, ContentType.objects.get_for_model(Device)) self.assertEqual(exporttemplate4.name, data['name']) self.assertEqual(exporttemplate4.template_code, data['template_code']) @@ -213,17 +213,17 @@ class ExportTemplateTest(APITestCase): data = [ { - 'content_type': self.content_type.pk, + 'content_type': 'dcim.device', 'name': 'Test Export Template 4', 'template_code': '{% for obj in queryset %}{{ obj.name }}\n{% endfor %}', }, { - 'content_type': self.content_type.pk, + 'content_type': 'dcim.device', 'name': 'Test Export Template 5', 'template_code': '{% for obj in queryset %}{{ obj.name }}\n{% endfor %}', }, { - 'content_type': self.content_type.pk, + 'content_type': 'dcim.device', 'name': 'Test Export Template 6', 'template_code': '{% for obj in queryset %}{{ obj.name }}\n{% endfor %}', }, @@ -241,7 +241,7 @@ class ExportTemplateTest(APITestCase): def test_update_exporttemplate(self): data = { - 'content_type': self.content_type.pk, + 'content_type': 'dcim.device', 'name': 'Test Export Template X', 'template_code': '{% for obj in queryset %}{{ obj.name }}\n{% endfor %}', } diff --git a/netbox/extras/tests/test_webhooks.py b/netbox/extras/tests/test_webhooks.py index 026a82bb8..06b4f7c7e 100644 --- a/netbox/extras/tests/test_webhooks.py +++ b/netbox/extras/tests/test_webhooks.py @@ -34,7 +34,7 @@ class WebhookTest(APITestCase): DUMMY_SECRET = "LOOKATMEIMASECRETSTRING" webhooks = Webhook.objects.bulk_create(( - Webhook(name='Site Create Webhook', type_create=True, payload_url=DUMMY_URL, secret=DUMMY_SECRET, additional_headers={'X-Foo': 'Bar'}), + Webhook(name='Site Create Webhook', type_create=True, payload_url=DUMMY_URL, secret=DUMMY_SECRET, additional_headers='X-Foo: Bar'), Webhook(name='Site Update Webhook', type_update=True, payload_url=DUMMY_URL, secret=DUMMY_SECRET), Webhook(name='Site Delete Webhook', type_delete=True, payload_url=DUMMY_URL, secret=DUMMY_SECRET), )) diff --git a/netbox/extras/webhooks.py b/netbox/extras/webhooks.py index cfa05d0f6..8b20641d7 100644 --- a/netbox/extras/webhooks.py +++ b/netbox/extras/webhooks.py @@ -1,4 +1,3 @@ -import datetime import hashlib import hmac diff --git a/netbox/extras/webhooks_worker.py b/netbox/extras/webhooks_worker.py index e48d8a2d7..1b1b76dd9 100644 --- a/netbox/extras/webhooks_worker.py +++ b/netbox/extras/webhooks_worker.py @@ -1,19 +1,21 @@ -import json +import logging import requests from django_rq import job -from rest_framework.utils.encoders import JSONEncoder +from jinja2.exceptions import TemplateError -from .choices import ObjectChangeActionChoices, WebhookContentTypeChoices +from .choices import ObjectChangeActionChoices from .webhooks import generate_signature +logger = logging.getLogger('netbox.webhooks_worker') + @job('default') def process_webhook(webhook, data, model_name, event, timestamp, username, request_id): """ Make a POST request to the defined Webhook """ - payload = { + context = { 'event': dict(ObjectChangeActionChoices)[event].lower(), 'timestamp': timestamp, 'model': model_name, @@ -21,29 +23,48 @@ def process_webhook(webhook, data, model_name, event, timestamp, username, reque 'request_id': request_id, 'data': data } + + # Build the headers for the HTTP request headers = { 'Content-Type': webhook.http_content_type, } - if webhook.additional_headers: - headers.update(webhook.additional_headers) + try: + headers.update(webhook.render_headers(context)) + except (TemplateError, ValueError) as e: + logger.error("Error parsing HTTP headers for webhook {}: {}".format(webhook, e)) + raise e + # Render the request body + try: + body = webhook.render_body(context) + except TemplateError as e: + logger.error("Error rendering request body for webhook {}: {}".format(webhook, e)) + raise e + + # Prepare the HTTP request params = { - 'method': 'POST', + 'method': webhook.http_method, 'url': webhook.payload_url, - 'headers': headers + 'headers': headers, + 'data': body, } + logger.info( + "Sending {} request to {} ({} {})".format( + params['method'], params['url'], context['model'], context['event'] + ) + ) + logger.debug(params) + try: + prepared_request = requests.Request(**params).prepare() + except requests.exceptions.RequestException as e: + logger.error("Error forming HTTP request: {}".format(e)) + raise e - if webhook.http_content_type == WebhookContentTypeChoices.CONTENTTYPE_JSON: - params.update({'data': json.dumps(payload, cls=JSONEncoder)}) - elif webhook.http_content_type == WebhookContentTypeChoices.CONTENTTYPE_FORMDATA: - params.update({'data': payload}) - - prepared_request = requests.Request(**params).prepare() - + # If a secret key is defined, sign the request with a hash of the key and its content if webhook.secret != '': - # Sign the request with a hash of the secret key and its content. prepared_request.headers['X-Hook-Signature'] = generate_signature(prepared_request.body, webhook.secret) + # Send the request with requests.Session() as session: session.verify = webhook.ssl_verification if webhook.ca_file_path: @@ -51,8 +72,10 @@ def process_webhook(webhook, data, model_name, event, timestamp, username, reque response = session.send(prepared_request) if 200 <= response.status_code <= 299: + logger.info("Request succeeded; response status {}".format(response.status_code)) return 'Status {} returned, webhook successfully processed.'.format(response.status_code) else: + logger.warning("Request failed; response status {}: {}".format(response.status_code, response.content)) raise requests.exceptions.RequestException( "Status {} returned with content '{}', webhook FAILED to process.".format( response.status_code, response.content diff --git a/netbox/ipam/forms.py b/netbox/ipam/forms.py index 2b7fb2a6b..8156ae4aa 100644 --- a/netbox/ipam/forms.py +++ b/netbox/ipam/forms.py @@ -276,6 +276,7 @@ class PrefixForm(BootstrapMixin, TenancyForm, CustomFieldModelForm): vrf = DynamicModelChoiceField( queryset=VRF.objects.all(), required=False, + label='VRF', widget=APISelect( api_url="/api/ipam/vrfs/", ) diff --git a/netbox/ipam/tables.py b/netbox/ipam/tables.py index 8f059c652..4dcb0a6c3 100644 --- a/netbox/ipam/tables.py +++ b/netbox/ipam/tables.py @@ -385,7 +385,7 @@ class InterfaceIPAddressTable(BaseTable): """ List IP addresses assigned to a specific Interface. """ - address = tables.TemplateColumn(IPADDRESS_ASSIGN_LINK, verbose_name='IP Address') + address = tables.LinkColumn(verbose_name='IP Address') vrf = tables.TemplateColumn(VRF_LINK, verbose_name='VRF') status = tables.TemplateColumn(STATUS_LABEL) tenant = tables.TemplateColumn(template_code=TENANT_LINK) diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 6c33e5b54..3bc673796 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -12,7 +12,7 @@ from django.core.exceptions import ImproperlyConfigured # Environment setup # -VERSION = '2.7.7' +VERSION = '2.7.8' # Hostname HOSTNAME = platform.node() diff --git a/netbox/project-static/css/base.css b/netbox/project-static/css/base.css index 281e5d9b7..ec45f43e5 100644 --- a/netbox/project-static/css/base.css +++ b/netbox/project-static/css/base.css @@ -179,9 +179,9 @@ nav ul.pagination { /* Racks */ div.rack_header { - margin-left: 30px; + margin-left: 32px; text-align: center; - width: 230px; + width: 220px; } /* Devices */ diff --git a/netbox/templates/dcim/inc/rack_elevation.html b/netbox/templates/dcim/inc/rack_elevation.html index feced6a22..e020c44d6 100644 --- a/netbox/templates/dcim/inc/rack_elevation.html +++ b/netbox/templates/dcim/inc/rack_elevation.html @@ -1,4 +1,6 @@ -{% load helpers %} -
- + +
+ + Save SVG +
diff --git a/netbox/templates/dcim/inc/rack_elevation_header.html b/netbox/templates/dcim/inc/rack_elevation_header.html new file mode 100644 index 000000000..36ac11398 --- /dev/null +++ b/netbox/templates/dcim/inc/rack_elevation_header.html @@ -0,0 +1,10 @@ +{% load helpers %} +
+ {{ rack.name }} + {% if rack.role %} +
{{ rack.role }} + {% endif %} + {% if rack.facility_id %} +
{{ rack.facility_id }} + {% endif %} +
diff --git a/netbox/templates/dcim/rack_elevation_list.html b/netbox/templates/dcim/rack_elevation_list.html index 2ff622f0f..f48b81e06 100644 --- a/netbox/templates/dcim/rack_elevation_list.html +++ b/netbox/templates/dcim/rack_elevation_list.html @@ -17,16 +17,10 @@
{% for rack in page %}
-
- {{ rack.name|truncatechars:"25" }} -

{{ rack.facility_id|truncatechars:"30" }}

-
+ {% include 'dcim/inc/rack_elevation_header.html' %} {% include 'dcim/inc/rack_elevation.html' with face=rack_face %}
-
- {{ rack.name|truncatechars:"25" }} -

{{ rack.facility_id|truncatechars:"30" }}

-
+ {% include 'dcim/inc/rack_elevation_header.html' %}
{% endfor %}
diff --git a/netbox/templates/utilities/obj_list.html b/netbox/templates/utilities/obj_list.html index fe70edd3b..e917819c6 100644 --- a/netbox/templates/utilities/obj_list.html +++ b/netbox/templates/utilities/obj_list.html @@ -17,7 +17,7 @@

{% block title %}{{ content_type.model_class|model_name_plural|bettertitle }}{% endblock %}

-
+
{% with bulk_edit_url=content_type.model_class|url_name:"bulk_edit" bulk_delete_url=content_type.model_class|url_name:"bulk_delete" %} {% if permissions.change or permissions.delete %}
@@ -34,12 +34,12 @@
{% if bulk_edit_url and permissions.change %} - {% endif %} {% if bulk_delete_url and permissions.delete %} - {% endif %} @@ -51,12 +51,12 @@
{% block bulk_buttons %}{% endblock %} {% if bulk_edit_url and permissions.change %} - {% endif %} {% if bulk_delete_url and permissions.delete %} - {% endif %} @@ -69,11 +69,11 @@ {% include 'inc/paginator.html' with paginator=table.paginator page=table.page %}
-
- {% if filter_form %} + {% if filter_form %} +
{% include 'inc/search_panel.html' %} - {% endif %} - {% block sidebar %}{% endblock %} -
+ {% block sidebar %}{% endblock %} +
+ {% endif %}
{% endblock %} diff --git a/netbox/utilities/api.py b/netbox/utilities/api.py index 95de2a25d..72a5735de 100644 --- a/netbox/utilities/api.py +++ b/netbox/utilities/api.py @@ -6,6 +6,7 @@ from django.contrib.contenttypes.models import ContentType from django.core.exceptions import FieldError, MultipleObjectsReturned, ObjectDoesNotExist from django.db.models import ManyToManyField, ProtectedError from django.http import Http404 +from django.urls import reverse from rest_framework.exceptions import APIException from rest_framework.permissions import BasePermission from rest_framework.relations import PrimaryKeyRelatedField, RelatedField @@ -41,6 +42,14 @@ def get_serializer_for_model(model, prefix=''): ) +def is_api_request(request): + """ + Return True of the request is being made via the REST API. + """ + api_path = reverse('api-root') + return request.path_info.startswith(api_path) + + # # Authentication # diff --git a/netbox/utilities/forms.py b/netbox/utilities/forms.py index a7ee63eaa..c57996a43 100644 --- a/netbox/utilities/forms.py +++ b/netbox/utilities/forms.py @@ -2,8 +2,9 @@ import csv import json import re from io import StringIO -import yaml +import django_filters +import yaml from django import forms from django.conf import settings from django.contrib.postgres.forms.jsonb import JSONField as _JSONField, InvalidJSONInput @@ -564,18 +565,17 @@ class TagFilterField(forms.MultipleChoiceField): class DynamicModelChoiceMixin: - field_modifier = '' + filter = django_filters.ModelChoiceFilter def get_bound_field(self, form, field_name): bound_field = BoundField(form, self, field_name) # Modify the QuerySet of the field before we return it. Limit choices to any data already bound: Options # will be populated on-demand via the APISelect widget. - field_name = '{}{}'.format(self.to_field_name or 'pk', self.field_modifier) - if bound_field.data: - self.queryset = self.queryset.filter(**{field_name: self.prepare_value(bound_field.data)}) - elif bound_field.initial: - self.queryset = self.queryset.filter(**{field_name: self.prepare_value(bound_field.initial)}) + data = self.prepare_value(bound_field.data or bound_field.initial) + if data: + filter = self.filter(field_name=self.to_field_name or 'pk', queryset=self.queryset) + self.queryset = filter.filter(self.queryset, data) else: self.queryset = self.queryset.none() @@ -594,7 +594,7 @@ class DynamicModelMultipleChoiceField(DynamicModelChoiceMixin, forms.ModelMultip """ A multiple-choice version of DynamicModelChoiceField. """ - field_modifier = '__in' + filter = django_filters.ModelMultipleChoiceFilter class LaxURLField(forms.URLField): diff --git a/netbox/utilities/middleware.py b/netbox/utilities/middleware.py index 564771821..6cbf4ed17 100644 --- a/netbox/utilities/middleware.py +++ b/netbox/utilities/middleware.py @@ -5,6 +5,7 @@ from django.db import ProgrammingError from django.http import Http404, HttpResponseRedirect from django.urls import reverse +from .api import is_api_request from .views import server_error @@ -38,9 +39,8 @@ class APIVersionMiddleware(object): self.get_response = get_response def __call__(self, request): - api_path = reverse('api-root') response = self.get_response(request) - if request.path_info.startswith(api_path): + if is_api_request(request): response['API-Version'] = settings.REST_FRAMEWORK_VERSION return response diff --git a/netbox/utilities/testing/testcases.py b/netbox/utilities/testing/testcases.py index 8d1b1a1be..de8b93232 100644 --- a/netbox/utilities/testing/testcases.py +++ b/netbox/utilities/testing/testcases.py @@ -172,24 +172,29 @@ class ViewTestCases: @override_settings(EXEMPT_VIEW_PERMISSIONS=[]) def test_create_object(self): + + # Try GET without permission + with disable_warnings('django.request'): + self.assertHttpStatus(self.client.post(self._get_url('add')), 403) + + # Try GET with permission + self.add_permissions( + '{}.add_{}'.format(self.model._meta.app_label, self.model._meta.model_name) + ) + response = self.client.get(path=self._get_url('add')) + self.assertHttpStatus(response, 200) + + # Try POST with permission initial_count = self.model.objects.count() request = { 'path': self._get_url('add'), 'data': post_data(self.form_data), 'follow': False, # Do not follow 302 redirects } - - # Attempt to make the request without required permissions - with disable_warnings('django.request'): - self.assertHttpStatus(self.client.post(**request), 403) - - # Assign the required permission and submit again - self.add_permissions( - '{}.add_{}'.format(self.model._meta.app_label, self.model._meta.model_name) - ) response = self.client.post(**request) self.assertHttpStatus(response, 302) + # Validate object creation self.assertEqual(initial_count + 1, self.model.objects.count()) instance = self.model.objects.order_by('-pk').first() self.assertInstanceEqual(instance, self.form_data) @@ -204,23 +209,27 @@ class ViewTestCases: def test_edit_object(self): instance = self.model.objects.first() + # Try GET without permission + with disable_warnings('django.request'): + self.assertHttpStatus(self.client.post(self._get_url('edit', instance)), 403) + + # Try GET with permission + self.add_permissions( + '{}.change_{}'.format(self.model._meta.app_label, self.model._meta.model_name) + ) + response = self.client.get(path=self._get_url('edit', instance)) + self.assertHttpStatus(response, 200) + + # Try POST with permission request = { 'path': self._get_url('edit', instance), 'data': post_data(self.form_data), 'follow': False, # Do not follow 302 redirects } - - # Attempt to make the request without required permissions - with disable_warnings('django.request'): - self.assertHttpStatus(self.client.post(**request), 403) - - # Assign the required permission and submit again - self.add_permissions( - '{}.change_{}'.format(self.model._meta.app_label, self.model._meta.model_name) - ) response = self.client.post(**request) self.assertHttpStatus(response, 302) + # Validate object modifications instance = self.model.objects.get(pk=instance.pk) self.assertInstanceEqual(instance, self.form_data) @@ -232,23 +241,26 @@ class ViewTestCases: def test_delete_object(self): instance = self.model.objects.first() + # Try GET without permissions + with disable_warnings('django.request'): + self.assertHttpStatus(self.client.post(self._get_url('delete', instance)), 403) + + # Try GET with permission + self.add_permissions( + '{}.delete_{}'.format(self.model._meta.app_label, self.model._meta.model_name) + ) + response = self.client.get(path=self._get_url('delete', instance)) + self.assertHttpStatus(response, 200) + request = { 'path': self._get_url('delete', instance), 'data': {'confirm': True}, 'follow': False, # Do not follow 302 redirects } - - # Attempt to make the request without required permissions - with disable_warnings('django.request'): - self.assertHttpStatus(self.client.post(**request), 403) - - # Assign the required permission and submit again - self.add_permissions( - '{}.delete_{}'.format(self.model._meta.app_label, self.model._meta.model_name) - ) response = self.client.post(**request) self.assertHttpStatus(response, 302) + # Validate object deletion with self.assertRaises(ObjectDoesNotExist): self.model.objects.get(pk=instance.pk) @@ -314,6 +326,20 @@ class ViewTestCases: @override_settings(EXEMPT_VIEW_PERMISSIONS=[]) def test_import_objects(self): + + # Test GET without permission + with disable_warnings('django.request'): + self.assertHttpStatus(self.client.get(self._get_url('import')), 403) + + # Test GET with permission + self.add_permissions( + '{}.view_{}'.format(self.model._meta.app_label, self.model._meta.model_name), + '{}.add_{}'.format(self.model._meta.app_label, self.model._meta.model_name) + ) + response = self.client.get(self._get_url('import')) + self.assertHttpStatus(response, 200) + + # Test POST with permission initial_count = self.model.objects.count() request = { 'path': self._get_url('import'), @@ -321,19 +347,10 @@ class ViewTestCases: 'csv': '\n'.join(self.csv_data) } } - - # Attempt to make the request without required permissions - with disable_warnings('django.request'): - self.assertHttpStatus(self.client.post(**request), 403) - - # Assign the required permission and submit again - self.add_permissions( - '{}.view_{}'.format(self.model._meta.app_label, self.model._meta.model_name), - '{}.add_{}'.format(self.model._meta.app_label, self.model._meta.model_name) - ) response = self.client.post(**request) self.assertHttpStatus(response, 200) + # Validate import of new objects self.assertEqual(self.model.objects.count(), initial_count + len(self.csv_data) - 1) class BulkEditObjectsViewTestCase(ModelViewTestCase): diff --git a/netbox/utilities/utils.py b/netbox/utilities/utils.py index 6969a60e9..371a0530b 100644 --- a/netbox/utilities/utils.py +++ b/netbox/utilities/utils.py @@ -31,8 +31,9 @@ def csv_format(data): if not isinstance(value, str): value = '{}'.format(value) - # Double-quote the value if it contains a comma + # Double-quote the value if it contains a comma or line break if ',' in value or '\n' in value: + value = value.replace('"', '""') # Escape double-quotes csv.append('"{}"'.format(value)) else: csv.append('{}'.format(value)) @@ -80,10 +81,12 @@ def get_subquery(model, field): return subquery -def serialize_object(obj, extra=None): +def serialize_object(obj, extra=None, exclude=None): """ Return a generic JSON representation of an object using Django's built-in serializer. (This is used for things like - change logging, not the REST API.) Optionally include a dictionary to supplement the object data. + change logging, not the REST API.) Optionally include a dictionary to supplement the object data. A list of keys + can be provided to exclude them from the returned dictionary. Private fields (prefaced with an underscore) are + implicitly excluded. """ json_str = serialize('json', [obj]) data = json.loads(json_str)[0]['fields'] @@ -102,6 +105,16 @@ def serialize_object(obj, extra=None): if extra is not None: data.update(extra) + # Copy keys to list to avoid 'dictionary changed size during iteration' exception + for key in list(data): + # Private fields shouldn't be logged in the object change + if isinstance(key, str) and key.startswith('_'): + data.pop(key) + + # Explicitly excluded keys + if isinstance(exclude, (list, tuple)) and key in exclude: + data.pop(key) + return data diff --git a/netbox/utilities/views.py b/netbox/utilities/views.py index 19a6d655e..3d9b638fb 100644 --- a/netbox/utilities/views.py +++ b/netbox/utilities/views.py @@ -626,12 +626,13 @@ class BulkEditView(GetReturnURLMixin, View): model = self.queryset.model - # Create a mutable copy of the POST data - post_data = request.POST.copy() - # If we are editing *all* objects in the queryset, replace the PK list with all matched objects. - if post_data.get('_all') and self.filterset is not None: - post_data['pk'] = [obj.pk for obj in self.filterset(request.GET, model.objects.only('pk')).qs] + if request.POST.get('_all') and self.filterset is not None: + pk_list = [ + obj.pk for obj in self.filterset(request.GET, model.objects.only('pk')).qs + ] + else: + pk_list = request.POST.getlist('pk') if '_apply' in request.POST: form = self.form(model, request.POST) @@ -715,12 +716,10 @@ class BulkEditView(GetReturnURLMixin, View): messages.error(self.request, "{} failed validation: {}".format(obj, e)) else: - # Pass the PK list as initial data to avoid binding the form - initial_data = querydict_to_dict(post_data) - form = self.form(model, initial=initial_data) + form = self.form(model, initial={'pk': pk_list}) # Retrieve objects being edited - table = self.table(self.queryset.filter(pk__in=post_data.getlist('pk')), orderable=False) + table = self.table(self.queryset.filter(pk__in=pk_list), orderable=False) if not table.rows: messages.warning(request, "No {} were selected.".format(model._meta.verbose_name_plural)) return redirect(self.get_return_url(request))