From d250fd91a9dc94126cd3a048944c1a4d95541a16 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Tue, 24 Nov 2020 16:48:55 -0500 Subject: [PATCH 01/19] Post-release version bump --- netbox/netbox/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 022980dcf..adae93bcc 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -16,7 +16,7 @@ from django.core.validators import URLValidator # Environment setup # -VERSION = '2.9.10' +VERSION = '2.9.11-dev' # Hostname HOSTNAME = platform.node() From f8b523b97b347f59abb5701d4c98a4abf4b50bba Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 25 Nov 2020 11:44:44 -0500 Subject: [PATCH 02/19] Extend assertInstanceEqual to accept a list of data fields to exclude from comparison with the instance --- netbox/utilities/testing/api.py | 11 ++++++++++- netbox/utilities/testing/views.py | 19 ++++++++++++------- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/netbox/utilities/testing/api.py b/netbox/utilities/testing/api.py index 2f3e2aa19..856af8ea1 100644 --- a/netbox/utilities/testing/api.py +++ b/netbox/utilities/testing/api.py @@ -174,6 +174,7 @@ class APIViewTestCases: class CreateObjectViewTestCase(APITestCase): create_data = [] + validation_excluded_fields = [] def test_create_object_without_permission(self): """ @@ -205,6 +206,7 @@ class APIViewTestCases: self.assertInstanceEqual( self._get_queryset().get(pk=response.data['id']), self.create_data[0], + exclude=self.validation_excluded_fields, api=True ) @@ -229,11 +231,13 @@ class APIViewTestCases: self.assertInstanceEqual( self._get_queryset().get(pk=obj['id']), self.create_data[i], + exclude=self.validation_excluded_fields, api=True ) class UpdateObjectViewTestCase(APITestCase): update_data = {} + validation_excluded_fields = [] def test_update_object_without_permission(self): """ @@ -266,7 +270,12 @@ class APIViewTestCases: response = self.client.patch(url, update_data, format='json', **self.header) self.assertHttpStatus(response, status.HTTP_200_OK) instance.refresh_from_db() - self.assertInstanceEqual(instance, update_data, api=True) + self.assertInstanceEqual( + instance, + update_data, + exclude=self.validation_excluded_fields, + api=True + ) class DeleteObjectViewTestCase(APITestCase): diff --git a/netbox/utilities/testing/views.py b/netbox/utilities/testing/views.py index afbcbc057..d19848985 100644 --- a/netbox/utilities/testing/views.py +++ b/netbox/utilities/testing/views.py @@ -126,20 +126,25 @@ class TestCase(_TestCase): err_message = f"Expected HTTP status {expected_status}; received {response.status_code}: {err}" self.assertEqual(response.status_code, expected_status, err_message) - def assertInstanceEqual(self, instance, data, api=False): + def assertInstanceEqual(self, instance, data, exclude=None, api=False): """ Compare a model instance to a dictionary, checking that its attribute values match those specified in the dictionary. - :instance: Python object instance - :data: Dictionary of test data used to define the instance - :api: Set to True is the data is a JSON representation of the instance + :param instance: Python object instance + :param data: Dictionary of test data used to define the instance + :param exclude: List of fields to exclude from comparison (e.g. passwords, which get hashed) + :param api: Set to True is the data is a JSON representation of the instance """ - model_dict = self.model_to_dict(instance, fields=data.keys(), api=api) + if exclude is None: + exclude = [] - # Omit any dictionary keys which are not instance attributes + fields = [k for k in data.keys() if k not in exclude] + model_dict = self.model_to_dict(instance, fields=fields, api=api) + + # Omit any dictionary keys which are not instance attributes or have been excluded relevant_data = { - k: v for k, v in data.items() if hasattr(instance, k) + k: v for k, v in data.items() if hasattr(instance, k) and k not in exclude } self.assertDictEqual(model_dict, relevant_data) From 77bbe5730be9fdac354aad751de5d1e7d7e1dcf3 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 25 Nov 2020 11:47:53 -0500 Subject: [PATCH 03/19] Fixes #5383: Fix setting user password via REST API --- docs/release-notes/version-2.9.md | 8 ++++++++ netbox/users/api/serializers.py | 18 ++++++++++++++++-- netbox/users/tests/test_api.py | 4 ++++ 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/docs/release-notes/version-2.9.md b/docs/release-notes/version-2.9.md index 7e7ddd249..563220cb5 100644 --- a/docs/release-notes/version-2.9.md +++ b/docs/release-notes/version-2.9.md @@ -1,5 +1,13 @@ # NetBox v2.9 +## v2.9.11 (FUTURE) + +### Bug Fixes + +* [#5383](https://github.com/netbox-community/netbox/issues/5383) - Fix setting user password via REST API + +--- + ## v2.9.10 (2020-11-24) ### Enhancements diff --git a/netbox/users/api/serializers.py b/netbox/users/api/serializers.py index 1f338d6e4..a3028f3cb 100644 --- a/netbox/users/api/serializers.py +++ b/netbox/users/api/serializers.py @@ -19,9 +19,23 @@ class UserSerializer(ValidatedModelSerializer): class Meta: model = User fields = ( - 'id', 'url', 'username', 'first_name', 'last_name', 'email', 'is_staff', 'is_active', 'date_joined', - 'groups', + 'id', 'url', 'username', 'password', 'first_name', 'last_name', 'email', 'is_staff', 'is_active', + 'date_joined', 'groups', ) + extra_kwargs = { + 'password': {'write_only': True} + } + + def create(self, validated_data): + """ + Extract the password from validated data and set it separately to ensure proper hash generation. + """ + password = validated_data.pop('password') + user = super().create(validated_data) + user.set_password(password) + user.save() + + return user class GroupSerializer(ValidatedModelSerializer): diff --git a/netbox/users/tests/test_api.py b/netbox/users/tests/test_api.py index c4229bff9..2e670b558 100644 --- a/netbox/users/tests/test_api.py +++ b/netbox/users/tests/test_api.py @@ -22,15 +22,19 @@ class UserTest(APIViewTestCases.APIViewTestCase): model = User view_namespace = 'users' brief_fields = ['id', 'url', 'username'] + validation_excluded_fields = ['password'] create_data = [ { 'username': 'User_4', + 'password': 'password4', }, { 'username': 'User_5', + 'password': 'password5', }, { 'username': 'User_6', + 'password': 'password6', }, ] From f55e966c8f9c343c844ea3e757229f1356a27556 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 25 Nov 2020 15:49:18 -0500 Subject: [PATCH 04/19] Closes #4865: Replace all Count() annotations with subqueries (#5385) * Convert circuits to use subqueries * Convert dcim to use subqueries * Convert extras to use subqueries * Convert ipam to use subqueries * Convert secrets to use subqueries * Convert virtualization to use subqueries * Update global search view to use subqueries where appropriate * Remove extraneous order_by() calls --- netbox/circuits/api/views.py | 11 ++++---- netbox/circuits/views.py | 22 +++++++++++----- netbox/dcim/api/views.py | 28 ++++++++++---------- netbox/dcim/views.py | 42 ++++++++++++++++-------------- netbox/extras/api/views.py | 9 +++---- netbox/extras/views.py | 18 ++++++------- netbox/ipam/api/views.py | 19 +++++++------- netbox/ipam/views.py | 21 ++++++++------- netbox/netbox/views.py | 22 +++++++++------- netbox/secrets/api/views.py | 6 ++--- netbox/secrets/views.py | 10 ++++--- netbox/virtualization/api/views.py | 11 ++++---- netbox/virtualization/views.py | 18 +++++++++---- 13 files changed, 134 insertions(+), 103 deletions(-) diff --git a/netbox/circuits/api/views.py b/netbox/circuits/api/views.py index 746ee02f6..32adc66ae 100644 --- a/netbox/circuits/api/views.py +++ b/netbox/circuits/api/views.py @@ -1,4 +1,4 @@ -from django.db.models import Count, Prefetch +from django.db.models import Prefetch from django.shortcuts import get_object_or_404 from rest_framework.decorators import action from rest_framework.response import Response @@ -10,6 +10,7 @@ from extras.api.serializers import RenderedGraphSerializer from extras.api.views import CustomFieldModelViewSet from extras.models import Graph from utilities.api import ModelViewSet +from utilities.utils import get_subquery from . import serializers @@ -27,8 +28,8 @@ class CircuitsRootView(APIRootView): class ProviderViewSet(CustomFieldModelViewSet): queryset = Provider.objects.prefetch_related('tags').annotate( - circuit_count=Count('circuits') - ).order_by(*Provider._meta.ordering) + circuit_count=get_subquery(Circuit, 'provider') + ) serializer_class = serializers.ProviderSerializer filterset_class = filters.ProviderFilterSet @@ -49,8 +50,8 @@ class ProviderViewSet(CustomFieldModelViewSet): class CircuitTypeViewSet(ModelViewSet): queryset = CircuitType.objects.annotate( - circuit_count=Count('circuits') - ).order_by(*CircuitType._meta.ordering) + circuit_count=get_subquery(Circuit, 'type') + ) serializer_class = serializers.CircuitTypeSerializer filterset_class = filters.CircuitTypeFilterSet diff --git a/netbox/circuits/views.py b/netbox/circuits/views.py index 6c29d41f1..2722bda67 100644 --- a/netbox/circuits/views.py +++ b/netbox/circuits/views.py @@ -1,12 +1,12 @@ from django.contrib import messages from django.db import transaction -from django.db.models import Count from django.shortcuts import get_object_or_404, redirect, render from django_tables2 import RequestConfig from extras.models import Graph from utilities.forms import ConfirmationForm from utilities.paginator import EnhancedPaginator, get_paginate_count +from utilities.utils import get_subquery from utilities.views import ( BulkDeleteView, BulkEditView, BulkImportView, ObjectView, ObjectDeleteView, ObjectEditView, ObjectListView, ) @@ -20,7 +20,9 @@ from .models import Circuit, CircuitTermination, CircuitType, Provider # class ProviderListView(ObjectListView): - queryset = Provider.objects.annotate(count_circuits=Count('circuits')).order_by(*Provider._meta.ordering) + queryset = Provider.objects.annotate( + count_circuits=get_subquery(Circuit, 'provider') + ) filterset = filters.ProviderFilterSet filterset_form = forms.ProviderFilterForm table = tables.ProviderTable @@ -72,14 +74,18 @@ class ProviderBulkImportView(BulkImportView): class ProviderBulkEditView(BulkEditView): - queryset = Provider.objects.annotate(count_circuits=Count('circuits')).order_by(*Provider._meta.ordering) + queryset = Provider.objects.annotate( + count_circuits=get_subquery(Circuit, 'provider') + ) filterset = filters.ProviderFilterSet table = tables.ProviderTable form = forms.ProviderBulkEditForm class ProviderBulkDeleteView(BulkDeleteView): - queryset = Provider.objects.annotate(count_circuits=Count('circuits')).order_by(*Provider._meta.ordering) + queryset = Provider.objects.annotate( + count_circuits=get_subquery(Circuit, 'provider') + ) filterset = filters.ProviderFilterSet table = tables.ProviderTable @@ -89,7 +95,9 @@ class ProviderBulkDeleteView(BulkDeleteView): # class CircuitTypeListView(ObjectListView): - queryset = CircuitType.objects.annotate(circuit_count=Count('circuits')).order_by(*CircuitType._meta.ordering) + queryset = CircuitType.objects.annotate( + circuit_count=get_subquery(Circuit, 'type') + ) table = tables.CircuitTypeTable @@ -109,7 +117,9 @@ class CircuitTypeBulkImportView(BulkImportView): class CircuitTypeBulkDeleteView(BulkDeleteView): - queryset = CircuitType.objects.annotate(circuit_count=Count('circuits')).order_by(*CircuitType._meta.ordering) + queryset = CircuitType.objects.annotate( + circuit_count=get_subquery(Circuit, 'type') + ) table = tables.CircuitTypeTable diff --git a/netbox/dcim/api/views.py b/netbox/dcim/api/views.py index 2d7107ef4..0e0e074e1 100644 --- a/netbox/dcim/api/views.py +++ b/netbox/dcim/api/views.py @@ -2,7 +2,7 @@ import socket from collections import OrderedDict from django.conf import settings -from django.db.models import Count, F +from django.db.models import F from django.http import HttpResponseForbidden, HttpResponse from django.shortcuts import get_object_or_404 from drf_yasg import openapi @@ -109,7 +109,7 @@ class SiteViewSet(CustomFieldModelViewSet): vlan_count=get_subquery(VLAN, 'site'), circuit_count=get_subquery(Circuit, 'terminations__site'), virtualmachine_count=get_subquery(VirtualMachine, 'cluster__site'), - ).order_by(*Site._meta.ordering) + ) serializer_class = serializers.SiteSerializer filterset_class = filters.SiteFilterSet @@ -146,8 +146,8 @@ class RackGroupViewSet(ModelViewSet): class RackRoleViewSet(ModelViewSet): queryset = RackRole.objects.annotate( - rack_count=Count('racks') - ).order_by(*RackRole._meta.ordering) + rack_count=get_subquery(Rack, 'role') + ) serializer_class = serializers.RackRoleSerializer filterset_class = filters.RackRoleFilterSet @@ -162,7 +162,7 @@ class RackViewSet(CustomFieldModelViewSet): ).annotate( device_count=get_subquery(Device, 'rack'), powerfeed_count=get_subquery(PowerFeed, 'rack') - ).order_by(*Rack._meta.ordering) + ) serializer_class = serializers.RackSerializer filterset_class = filters.RackFilterSet @@ -237,7 +237,7 @@ class ManufacturerViewSet(ModelViewSet): devicetype_count=get_subquery(DeviceType, 'manufacturer'), inventoryitem_count=get_subquery(InventoryItem, 'manufacturer'), platform_count=get_subquery(Platform, 'manufacturer') - ).order_by(*Manufacturer._meta.ordering) + ) serializer_class = serializers.ManufacturerSerializer filterset_class = filters.ManufacturerFilterSet @@ -248,8 +248,8 @@ class ManufacturerViewSet(ModelViewSet): class DeviceTypeViewSet(CustomFieldModelViewSet): queryset = DeviceType.objects.prefetch_related('manufacturer', 'tags').annotate( - device_count=Count('instances') - ).order_by(*DeviceType._meta.ordering) + device_count=get_subquery(Device, 'device_type') + ) serializer_class = serializers.DeviceTypeSerializer filterset_class = filters.DeviceTypeFilterSet @@ -314,7 +314,7 @@ class DeviceRoleViewSet(ModelViewSet): queryset = DeviceRole.objects.annotate( device_count=get_subquery(Device, 'device_role'), virtualmachine_count=get_subquery(VirtualMachine, 'role') - ).order_by(*DeviceRole._meta.ordering) + ) serializer_class = serializers.DeviceRoleSerializer filterset_class = filters.DeviceRoleFilterSet @@ -327,7 +327,7 @@ class PlatformViewSet(ModelViewSet): queryset = Platform.objects.annotate( device_count=get_subquery(Device, 'platform'), virtualmachine_count=get_subquery(VirtualMachine, 'platform') - ).order_by(*Platform._meta.ordering) + ) serializer_class = serializers.PlatformSerializer filterset_class = filters.PlatformFilterSet @@ -619,8 +619,8 @@ class CableViewSet(ModelViewSet): class VirtualChassisViewSet(ModelViewSet): queryset = VirtualChassis.objects.prefetch_related('tags').annotate( - member_count=Count('members', distinct=True) - ).order_by(*VirtualChassis._meta.ordering) + member_count=get_subquery(Device, 'virtual_chassis') + ) serializer_class = serializers.VirtualChassisSerializer filterset_class = filters.VirtualChassisFilterSet @@ -633,8 +633,8 @@ class PowerPanelViewSet(ModelViewSet): queryset = PowerPanel.objects.prefetch_related( 'site', 'rack_group' ).annotate( - powerfeed_count=Count('powerfeeds') - ).order_by(*PowerPanel._meta.ordering) + powerfeed_count=get_subquery(PowerFeed, 'power_panel') + ) serializer_class = serializers.PowerPanelSerializer filterset_class = filters.PowerPanelFilterSet diff --git a/netbox/dcim/views.py b/netbox/dcim/views.py index 42175d782..200581f4a 100644 --- a/netbox/dcim/views.py +++ b/netbox/dcim/views.py @@ -4,7 +4,7 @@ from django.contrib import messages from django.contrib.contenttypes.models import ContentType from django.core.paginator import EmptyPage, PageNotAnInteger from django.db import transaction -from django.db.models import Count, F, Prefetch +from django.db.models import F, Prefetch from django.forms import ModelMultipleChoiceField, MultipleHiddenInput, modelformset_factory from django.shortcuts import get_object_or_404, redirect, render from django.urls import reverse @@ -263,7 +263,9 @@ class RackGroupBulkDeleteView(BulkDeleteView): # class RackRoleListView(ObjectListView): - queryset = RackRole.objects.annotate(rack_count=Count('racks')).order_by(*RackRole._meta.ordering) + queryset = RackRole.objects.annotate( + rack_count=get_subquery(Rack, 'role') + ) table = tables.RackRoleTable @@ -283,7 +285,9 @@ class RackRoleBulkImportView(BulkImportView): class RackRoleBulkDeleteView(BulkDeleteView): - queryset = RackRole.objects.annotate(rack_count=Count('racks')).order_by(*RackRole._meta.ordering) + queryset = RackRole.objects.annotate( + rack_count=get_subquery(Rack, 'role') + ) table = tables.RackRoleTable @@ -295,8 +299,8 @@ class RackListView(ObjectListView): queryset = Rack.objects.prefetch_related( 'site', 'group', 'tenant', 'role', 'devices__device_type' ).annotate( - device_count=Count('devices') - ).order_by(*Rack._meta.ordering) + device_count=get_subquery(Device, 'rack') + ) filterset = filters.RackFilterSet filterset_form = forms.RackFilterForm table = tables.RackDetailTable @@ -507,8 +511,8 @@ class ManufacturerBulkImportView(BulkImportView): class ManufacturerBulkDeleteView(BulkDeleteView): queryset = Manufacturer.objects.annotate( - devicetype_count=Count('device_types') - ).order_by(*Manufacturer._meta.ordering) + devicetype_count=get_subquery(DeviceType, 'manufacturer') + ) table = tables.ManufacturerTable @@ -518,8 +522,8 @@ class ManufacturerBulkDeleteView(BulkDeleteView): class DeviceTypeListView(ObjectListView): queryset = DeviceType.objects.prefetch_related('manufacturer').annotate( - instance_count=Count('instances') - ).order_by(*DeviceType._meta.ordering) + instance_count=get_subquery(Device, 'device_type') + ) filterset = filters.DeviceTypeFilterSet filterset_form = forms.DeviceTypeFilterForm table = tables.DeviceTypeTable @@ -628,8 +632,8 @@ class DeviceTypeImportView(ObjectImportView): class DeviceTypeBulkEditView(BulkEditView): queryset = DeviceType.objects.prefetch_related('manufacturer').annotate( - instance_count=Count('instances') - ).order_by(*DeviceType._meta.ordering) + instance_count=get_subquery(Device, 'device_type') + ) filterset = filters.DeviceTypeFilterSet table = tables.DeviceTypeTable form = forms.DeviceTypeBulkEditForm @@ -637,8 +641,8 @@ class DeviceTypeBulkEditView(BulkEditView): class DeviceTypeBulkDeleteView(BulkDeleteView): queryset = DeviceType.objects.prefetch_related('manufacturer').annotate( - instance_count=Count('instances') - ).order_by(*DeviceType._meta.ordering) + instance_count=get_subquery(Device, 'device_type') + ) filterset = filters.DeviceTypeFilterSet table = tables.DeviceTypeTable @@ -2198,8 +2202,8 @@ class InterfaceConnectionsListView(ObjectListView): class VirtualChassisListView(ObjectListView): queryset = VirtualChassis.objects.prefetch_related('master').annotate( - member_count=Count('members', distinct=True) - ).order_by(*VirtualChassis._meta.ordering) + member_count=get_subquery(Device, 'virtual_chassis') + ) table = tables.VirtualChassisTable filterset = filters.VirtualChassisFilterSet filterset_form = forms.VirtualChassisFilterForm @@ -2430,8 +2434,8 @@ class PowerPanelListView(ObjectListView): queryset = PowerPanel.objects.prefetch_related( 'site', 'rack_group' ).annotate( - powerfeed_count=Count('powerfeeds') - ).order_by(*PowerPanel._meta.ordering) + powerfeed_count=get_subquery(PowerFeed, 'power_panel') + ) filterset = filters.PowerPanelFilterSet filterset_form = forms.PowerPanelFilterForm table = tables.PowerPanelTable @@ -2482,8 +2486,8 @@ class PowerPanelBulkDeleteView(BulkDeleteView): queryset = PowerPanel.objects.prefetch_related( 'site', 'rack_group' ).annotate( - rack_count=Count('powerfeeds') - ).order_by(*PowerPanel._meta.ordering) + powerfeed_count=get_subquery(PowerFeed, 'power_panel') + ) filterset = filters.PowerPanelFilterSet table = tables.PowerPanelTable diff --git a/netbox/extras/api/views.py b/netbox/extras/api/views.py index 74c9ea889..6efcc65a6 100644 --- a/netbox/extras/api/views.py +++ b/netbox/extras/api/views.py @@ -1,7 +1,6 @@ from collections import OrderedDict from django.contrib.contenttypes.models import ContentType -from django.db.models import Count from django.http import Http404 from django_rq.queues import get_connection from rest_framework import status @@ -15,14 +14,14 @@ from rq import Worker from extras import filters from extras.choices import JobResultStatusChoices from extras.models import ( - ConfigContext, CustomFieldChoice, ExportTemplate, Graph, ImageAttachment, ObjectChange, JobResult, Tag, + ConfigContext, CustomFieldChoice, ExportTemplate, Graph, ImageAttachment, ObjectChange, JobResult, Tag, TaggedItem, ) from extras.reports import get_report, get_reports, run_report from extras.scripts import get_script, get_scripts, run_script from utilities.api import IsAuthenticatedOrLoginNotRequired, ModelViewSet from utilities.exceptions import RQWorkerNotRunningException from utilities.metadata import ContentTypeMetadata -from utilities.utils import copy_safe_request +from utilities.utils import copy_safe_request, get_subquery from . import serializers @@ -149,8 +148,8 @@ class ExportTemplateViewSet(ModelViewSet): class TagViewSet(ModelViewSet): queryset = Tag.objects.annotate( - tagged_items=Count('extras_taggeditem_items') - ).order_by(*Tag._meta.ordering) + tagged_items=get_subquery(TaggedItem, 'tag') + ) serializer_class = serializers.TagSerializer filterset_class = filters.TagFilterSet diff --git a/netbox/extras/views.py b/netbox/extras/views.py index f2c31652c..4aa2969df 100644 --- a/netbox/extras/views.py +++ b/netbox/extras/views.py @@ -1,7 +1,7 @@ from django import template from django.contrib import messages from django.contrib.contenttypes.models import ContentType -from django.db.models import Count, Prefetch, Q +from django.db.models import Prefetch, Q from django.http import Http404, HttpResponseForbidden from django.shortcuts import get_object_or_404, redirect, render from django.views.generic import View @@ -13,7 +13,7 @@ from dcim.models import DeviceRole, Platform, Region, Site from tenancy.models import Tenant, TenantGroup from utilities.forms import ConfirmationForm from utilities.paginator import EnhancedPaginator, get_paginate_count -from utilities.utils import copy_safe_request, shallow_compare_dict +from utilities.utils import copy_safe_request, get_subquery, shallow_compare_dict from utilities.views import ( BulkDeleteView, BulkEditView, BulkImportView, ObjectView, ObjectDeleteView, ObjectEditView, ObjectListView, ContentTypePermissionRequiredMixin, @@ -21,7 +21,7 @@ from utilities.views import ( from virtualization.models import Cluster, ClusterGroup from . import filters, forms, tables from .choices import JobResultStatusChoices -from .models import ConfigContext, ImageAttachment, ObjectChange, JobResult, Tag +from .models import ConfigContext, ImageAttachment, ObjectChange, JobResult, Tag, TaggedItem from .reports import get_report, get_reports, run_report from .scripts import get_scripts, run_script @@ -32,8 +32,8 @@ from .scripts import get_scripts, run_script class TagListView(ObjectListView): queryset = Tag.objects.annotate( - items=Count('extras_taggeditem_items') - ).order_by(*Tag._meta.ordering) + items=get_subquery(TaggedItem, 'tag') + ) filterset = filters.TagFilterSet filterset_form = forms.TagFilterForm table = tables.TagTable @@ -57,16 +57,16 @@ class TagBulkImportView(BulkImportView): class TagBulkEditView(BulkEditView): queryset = Tag.objects.annotate( - items=Count('extras_taggeditem_items') - ).order_by(*Tag._meta.ordering) + items=get_subquery(TaggedItem, 'tag') + ) table = tables.TagTable form = forms.TagBulkEditForm class TagBulkDeleteView(BulkDeleteView): queryset = Tag.objects.annotate( - items=Count('extras_taggeditem_items') - ).order_by(*Tag._meta.ordering) + items=get_subquery(TaggedItem, 'tag') + ) table = tables.TagTable diff --git a/netbox/ipam/api/views.py b/netbox/ipam/api/views.py index c296fd912..eab1af929 100644 --- a/netbox/ipam/api/views.py +++ b/netbox/ipam/api/views.py @@ -1,5 +1,4 @@ from django.conf import settings -from django.db.models import Count from django.shortcuts import get_object_or_404 from django_pglocks import advisory_lock from drf_yasg.utils import swagger_auto_schema @@ -33,7 +32,7 @@ class VRFViewSet(CustomFieldModelViewSet): queryset = VRF.objects.prefetch_related('tenant').prefetch_related('tags').annotate( ipaddress_count=get_subquery(IPAddress, 'vrf'), prefix_count=get_subquery(Prefix, 'vrf') - ).order_by(*VRF._meta.ordering) + ) serializer_class = serializers.VRFSerializer filterset_class = filters.VRFFilterSet @@ -44,8 +43,8 @@ class VRFViewSet(CustomFieldModelViewSet): class RIRViewSet(ModelViewSet): queryset = RIR.objects.annotate( - aggregate_count=Count('aggregates') - ).order_by(*RIR._meta.ordering) + aggregate_count=get_subquery(Aggregate, 'rir') + ) serializer_class = serializers.RIRSerializer filterset_class = filters.RIRFilterSet @@ -68,7 +67,7 @@ class RoleViewSet(ModelViewSet): queryset = Role.objects.annotate( prefix_count=get_subquery(Prefix, 'role'), vlan_count=get_subquery(VLAN, 'role') - ).order_by(*Role._meta.ordering) + ) serializer_class = serializers.RoleSerializer filterset_class = filters.RoleFilterSet @@ -80,7 +79,7 @@ class RoleViewSet(ModelViewSet): class PrefixViewSet(CustomFieldModelViewSet): queryset = Prefix.objects.prefetch_related( 'site', 'vrf__tenant', 'tenant', 'vlan', 'role', 'tags' - ).order_by(*Prefix._meta.ordering) + ) serializer_class = serializers.PrefixSerializer filterset_class = filters.PrefixFilterSet @@ -250,7 +249,7 @@ class PrefixViewSet(CustomFieldModelViewSet): class IPAddressViewSet(CustomFieldModelViewSet): queryset = IPAddress.objects.prefetch_related( 'vrf__tenant', 'tenant', 'nat_inside', 'nat_outside', 'tags', 'assigned_object' - ).order_by(*IPAddress._meta.ordering) + ) serializer_class = serializers.IPAddressSerializer filterset_class = filters.IPAddressFilterSet @@ -261,8 +260,8 @@ class IPAddressViewSet(CustomFieldModelViewSet): class VLANGroupViewSet(ModelViewSet): queryset = VLANGroup.objects.prefetch_related('site').annotate( - vlan_count=Count('vlans') - ).order_by(*VLANGroup._meta.ordering) + vlan_count=get_subquery(VLAN, 'group') + ) serializer_class = serializers.VLANGroupSerializer filterset_class = filters.VLANGroupFilterSet @@ -276,7 +275,7 @@ class VLANViewSet(CustomFieldModelViewSet): 'site', 'group', 'tenant', 'role', 'tags' ).annotate( prefix_count=get_subquery(Prefix, 'vlan') - ).order_by(*VLAN._meta.ordering) + ) serializer_class = serializers.VLANSerializer filterset_class = filters.VLANFilterSet diff --git a/netbox/ipam/views.py b/netbox/ipam/views.py index 5ed2478ba..8dad91d09 100644 --- a/netbox/ipam/views.py +++ b/netbox/ipam/views.py @@ -1,6 +1,5 @@ import netaddr -from django.conf import settings -from django.db.models import Count, Prefetch +from django.db.models import Prefetch from django.db.models.expressions import RawSQL from django.shortcuts import get_object_or_404, redirect, render from django_tables2 import RequestConfig @@ -79,7 +78,9 @@ class VRFBulkDeleteView(BulkDeleteView): # class RIRListView(ObjectListView): - queryset = RIR.objects.annotate(aggregate_count=Count('aggregates')).order_by(*RIR._meta.ordering) + queryset = RIR.objects.annotate( + aggregate_count=get_subquery(Aggregate, 'rir') + ) filterset = filters.RIRFilterSet filterset_form = forms.RIRFilterForm table = tables.RIRDetailTable @@ -172,7 +173,9 @@ class RIRBulkImportView(BulkImportView): class RIRBulkDeleteView(BulkDeleteView): - queryset = RIR.objects.annotate(aggregate_count=Count('aggregates')).order_by(*RIR._meta.ordering) + queryset = RIR.objects.annotate( + aggregate_count=get_subquery(Aggregate, 'rir') + ) filterset = filters.RIRFilterSet table = tables.RIRTable @@ -184,7 +187,7 @@ class RIRBulkDeleteView(BulkDeleteView): class AggregateListView(ObjectListView): queryset = Aggregate.objects.prefetch_related('rir').annotate( child_count=RawSQL('SELECT COUNT(*) FROM ipam_prefix WHERE ipam_prefix.prefix <<= ipam_aggregate.prefix', ()) - ).order_by(*Aggregate._meta.ordering) + ) filterset = filters.AggregateFilterSet filterset_form = forms.AggregateFilterForm table = tables.AggregateDetailTable @@ -652,8 +655,8 @@ class IPAddressBulkDeleteView(BulkDeleteView): class VLANGroupListView(ObjectListView): queryset = VLANGroup.objects.prefetch_related('site').annotate( - vlan_count=Count('vlans') - ).order_by(*VLANGroup._meta.ordering) + vlan_count=get_subquery(VLAN, 'group') + ) filterset = filters.VLANGroupFilterSet filterset_form = forms.VLANGroupFilterForm table = tables.VLANGroupTable @@ -676,8 +679,8 @@ class VLANGroupBulkImportView(BulkImportView): class VLANGroupBulkDeleteView(BulkDeleteView): queryset = VLANGroup.objects.prefetch_related('site').annotate( - vlan_count=Count('vlans') - ).order_by(*VLANGroup._meta.ordering) + vlan_count=get_subquery(VLAN, 'group') + ) filterset = filters.VLANGroupFilterSet table = tables.VLANGroupTable diff --git a/netbox/netbox/views.py b/netbox/netbox/views.py index b2bee0f96..e58d214ce 100644 --- a/netbox/netbox/views.py +++ b/netbox/netbox/views.py @@ -48,8 +48,8 @@ SEARCH_TYPES = OrderedDict(( # Circuits ('provider', { 'queryset': Provider.objects.annotate( - count_circuits=Count('circuits') - ).order_by(*Provider._meta.ordering), + count_circuits=get_subquery(Circuit, 'provider') + ), 'filterset': ProviderFilterSet, 'table': ProviderTable, 'url': 'circuits:provider_list', @@ -76,17 +76,21 @@ SEARCH_TYPES = OrderedDict(( 'url': 'dcim:rack_list', }), ('rackgroup', { - 'queryset': RackGroup.objects.prefetch_related('site').annotate( - rack_count=Count('racks') - ).order_by(*RackGroup._meta.ordering), + 'queryset': RackGroup.objects.add_related_count( + RackGroup.objects.all(), + Rack, + 'group', + 'rack_count', + cumulative=True + ).prefetch_related('site'), 'filterset': RackGroupFilterSet, 'table': RackGroupTable, 'url': 'dcim:rackgroup_list', }), ('devicetype', { 'queryset': DeviceType.objects.prefetch_related('manufacturer').annotate( - instance_count=Count('instances') - ).order_by(*DeviceType._meta.ordering), + instance_count=get_subquery(Device, 'device_type') + ), 'filterset': DeviceTypeFilterSet, 'table': DeviceTypeTable, 'url': 'dcim:devicetype_list', @@ -101,8 +105,8 @@ SEARCH_TYPES = OrderedDict(( }), ('virtualchassis', { 'queryset': VirtualChassis.objects.prefetch_related('master').annotate( - member_count=Count('members', distinct=True) - ).order_by(*VirtualChassis._meta.ordering), + member_count=get_subquery(Device, 'virtual_chassis') + ), 'filterset': VirtualChassisFilterSet, 'table': VirtualChassisTable, 'url': 'dcim:virtualchassis_list', diff --git a/netbox/secrets/api/views.py b/netbox/secrets/api/views.py index 7db6f92b6..8147809b0 100644 --- a/netbox/secrets/api/views.py +++ b/netbox/secrets/api/views.py @@ -1,7 +1,6 @@ import base64 from Crypto.PublicKey import RSA -from django.db.models import Count from django.http import HttpResponseBadRequest from rest_framework.exceptions import ValidationError from rest_framework.permissions import IsAuthenticated @@ -13,6 +12,7 @@ from secrets import filters from secrets.exceptions import InvalidKey from secrets.models import Secret, SecretRole, SessionKey, UserKey from utilities.api import ModelViewSet +from utilities.utils import get_subquery from . import serializers ERR_USERKEY_MISSING = "No UserKey found for the current user." @@ -35,8 +35,8 @@ class SecretsRootView(APIRootView): class SecretRoleViewSet(ModelViewSet): queryset = SecretRole.objects.annotate( - secret_count=Count('secrets') - ).order_by(*SecretRole._meta.ordering) + secret_count=get_subquery(Secret, 'role') + ) serializer_class = serializers.SecretRoleSerializer filterset_class = filters.SecretRoleFilterSet diff --git a/netbox/secrets/views.py b/netbox/secrets/views.py index 2872616b8..e95392dfc 100644 --- a/netbox/secrets/views.py +++ b/netbox/secrets/views.py @@ -2,11 +2,11 @@ import base64 import logging from django.contrib import messages -from django.db.models import Count from django.shortcuts import get_object_or_404, redirect, render from django.utils.html import escape from django.utils.safestring import mark_safe +from utilities.utils import get_subquery from utilities.views import ( BulkDeleteView, BulkEditView, BulkImportView, ObjectView, ObjectDeleteView, ObjectEditView, ObjectListView, ) @@ -29,7 +29,9 @@ def get_session_key(request): # class SecretRoleListView(ObjectListView): - queryset = SecretRole.objects.annotate(secret_count=Count('secrets')).order_by(*SecretRole._meta.ordering) + queryset = SecretRole.objects.annotate( + secret_count=get_subquery(Secret, 'role') + ) table = tables.SecretRoleTable @@ -49,7 +51,9 @@ class SecretRoleBulkImportView(BulkImportView): class SecretRoleBulkDeleteView(BulkDeleteView): - queryset = SecretRole.objects.annotate(secret_count=Count('secrets')).order_by(*SecretRole._meta.ordering) + queryset = SecretRole.objects.annotate( + secret_count=get_subquery(Secret, 'role') + ) table = tables.SecretRoleTable diff --git a/netbox/virtualization/api/views.py b/netbox/virtualization/api/views.py index 55393b110..f805adb2e 100644 --- a/netbox/virtualization/api/views.py +++ b/netbox/virtualization/api/views.py @@ -1,4 +1,3 @@ -from django.db.models import Count from django.shortcuts import get_object_or_404 from rest_framework.decorators import action from rest_framework.response import Response @@ -29,16 +28,16 @@ class VirtualizationRootView(APIRootView): class ClusterTypeViewSet(ModelViewSet): queryset = ClusterType.objects.annotate( - cluster_count=Count('clusters') - ).order_by(*ClusterType._meta.ordering) + cluster_count=get_subquery(Cluster, 'type') + ) serializer_class = serializers.ClusterTypeSerializer filterset_class = filters.ClusterTypeFilterSet class ClusterGroupViewSet(ModelViewSet): queryset = ClusterGroup.objects.annotate( - cluster_count=Count('clusters') - ).order_by(*ClusterGroup._meta.ordering) + cluster_count=get_subquery(Cluster, 'group') + ) serializer_class = serializers.ClusterGroupSerializer filterset_class = filters.ClusterGroupFilterSet @@ -49,7 +48,7 @@ class ClusterViewSet(CustomFieldModelViewSet): ).annotate( device_count=get_subquery(Device, 'cluster'), virtualmachine_count=get_subquery(VirtualMachine, 'cluster') - ).order_by(*Cluster._meta.ordering) + ) serializer_class = serializers.ClusterSerializer filterset_class = filters.ClusterFilterSet diff --git a/netbox/virtualization/views.py b/netbox/virtualization/views.py index 4cabfedeb..2b7ae3a13 100644 --- a/netbox/virtualization/views.py +++ b/netbox/virtualization/views.py @@ -1,6 +1,6 @@ from django.contrib import messages from django.db import transaction -from django.db.models import Count, Prefetch +from django.db.models import Prefetch from django.shortcuts import get_object_or_404, redirect, render from django.urls import reverse @@ -23,7 +23,9 @@ from .models import Cluster, ClusterGroup, ClusterType, VirtualMachine, VMInterf # class ClusterTypeListView(ObjectListView): - queryset = ClusterType.objects.annotate(cluster_count=Count('clusters')).order_by(*ClusterType._meta.ordering) + queryset = ClusterType.objects.annotate( + cluster_count=get_subquery(Cluster, 'type') + ) table = tables.ClusterTypeTable @@ -43,7 +45,9 @@ class ClusterTypeBulkImportView(BulkImportView): class ClusterTypeBulkDeleteView(BulkDeleteView): - queryset = ClusterType.objects.annotate(cluster_count=Count('clusters')).order_by(*ClusterType._meta.ordering) + queryset = ClusterType.objects.annotate( + cluster_count=get_subquery(Cluster, 'type') + ) table = tables.ClusterTypeTable @@ -52,7 +56,9 @@ class ClusterTypeBulkDeleteView(BulkDeleteView): # class ClusterGroupListView(ObjectListView): - queryset = ClusterGroup.objects.annotate(cluster_count=Count('clusters')).order_by(*ClusterGroup._meta.ordering) + queryset = ClusterGroup.objects.annotate( + cluster_count=get_subquery(Cluster, 'group') + ) table = tables.ClusterGroupTable @@ -72,7 +78,9 @@ class ClusterGroupBulkImportView(BulkImportView): class ClusterGroupBulkDeleteView(BulkDeleteView): - queryset = ClusterGroup.objects.annotate(cluster_count=Count('clusters')).order_by(*ClusterGroup._meta.ordering) + queryset = ClusterGroup.objects.annotate( + cluster_count=get_subquery(Cluster, 'group') + ) table = tables.ClusterGroupTable From 6ee161bdc63cb726a7e603ef17c6e3246b7dfc4f Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 3 Dec 2020 12:11:44 -0500 Subject: [PATCH 05/19] Add references to GitHub discussions --- CONTRIBUTING.md | 28 +++++++++++++++++++++++----- README.md | 4 ++-- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 136d9f493..85bb561dd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,15 +4,33 @@ If you encounter any issues installing or using NetBox, try one of the following resources to get assistance. Please **do not** open a GitHub issue except to report bugs or request features. +### GitHub Discussions + +GitHub's discussions are the best place to get help or propose rough ideas for +new functionality. Their integration with GitHub allows for easily cross- +referencing and converting posts to issues as needed. There are several +categories for discussions: + +* **General** - General community discussion +* **Ideas** - Ideas for new functionality that isn't yet ready for a formal + feature request +* **Q&A** - Request help with installing or using NetBox +* **Show and tell** - Share a plugin, script, or something else you've made + using NetBox + ### Mailing List -We have established a Google Groups Mailing List for issues and general -discussion. This is the best forum for obtaining assistance with NetBox -installation. You can find us [here](https://groups.google.com/g/netbox-discuss). +We also have a Google Groups [mailing list](https://groups.google.com/g/netbox-discuss) +for general discussion, however we're encouraging people to use GitHub +discussions where possible, as it's much easier for newcomers to review past +discussions. ### Slack -For real-time discussion, you can join the #netbox Slack channel on [NetworkToCode](https://slack.networktocode.com/). +For real-time chat, you can join the **#netbox** Slack channel on [NetworkToCode](https://slack.networktocode.com/). +Unfortunately, the Slack channel does not provide long-term retention of chat +history, so try to avoid it for any discussions would benefit from being +preserved for future reference. ## Reporting Bugs @@ -171,7 +189,7 @@ overlooked. * Official channels for communication include: - * GitHub issues/pull requests + * GitHub issues, pull requests, and discussions * The [netbox-discuss](https://groups.google.com/g/netbox-discuss) mailing list * The **#netbox** channel on [NetworkToCode Slack](https://networktocode.slack.com/) diff --git a/README.md b/README.md index 284c5a87c..56c71af78 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,8 @@ complete list of requirements, see `requirements.txt`. The code is available [on The complete documentation for NetBox can be found at [Read the Docs](http://netbox.readthedocs.io/en/stable/). -Questions? Comments? Please subscribe to [the netbox-discuss mailing list](https://groups.google.com/g/netbox-discuss), -or join us in the #netbox Slack channel on [NetworkToCode](https://networktocode.slack.com/)! +Questions? Comments? Please start a [discussion on GitHub](https://github.com/netbox-community/netbox/discussions), +or join us in the **#netbox** Slack channel on [NetworkToCode](https://networktocode.slack.com/)! ### Build Status From a4b2e1f2e2b618c0cdcfd6e62c50ecc6eaaa55d9 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 3 Dec 2020 12:13:08 -0500 Subject: [PATCH 06/19] Fix typo (from PR #5393) --- docs/installation/5-http-server.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation/5-http-server.md b/docs/installation/5-http-server.md index 5b4e3dfae..5bd04253a 100644 --- a/docs/installation/5-http-server.md +++ b/docs/installation/5-http-server.md @@ -3,7 +3,7 @@ This documentation provides example configurations for both [nginx](https://www.nginx.com/resources/wiki/) and [Apache](http://httpd.apache.org/docs/2.4), though any HTTP server which supports WSGI should be compatible. !!! info - For the sake of brevity, only Ubuntu 18.04 instructions are provided here, these tasks not unique to NetBox and should carry over to other distributions with mininal changes. Please consult your distribution's documentation for assistance if needed. + For the sake of brevity, only Ubuntu 18.04 instructions are provided here, these tasks are not unique to NetBox and should carry over to other distributions with minimal changes. Please consult your distribution's documentation for assistance if needed. ## Obtain an SSL Certificate From 22bb700f94d13c818c87445e0c170c72b2e05a80 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 3 Dec 2020 13:40:42 -0500 Subject: [PATCH 07/19] Fixes #5396: Fix uniqueness constraint for virtual machine names --- docs/release-notes/version-2.9.md | 1 + netbox/virtualization/models.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/release-notes/version-2.9.md b/docs/release-notes/version-2.9.md index 563220cb5..e755e7b82 100644 --- a/docs/release-notes/version-2.9.md +++ b/docs/release-notes/version-2.9.md @@ -5,6 +5,7 @@ ### Bug Fixes * [#5383](https://github.com/netbox-community/netbox/issues/5383) - Fix setting user password via REST API +* [#5396](https://github.com/netbox-community/netbox/issues/5396) - Fix uniqueness constraint for virtual machine names --- diff --git a/netbox/virtualization/models.py b/netbox/virtualization/models.py index 7a8bd7595..72d84a908 100644 --- a/netbox/virtualization/models.py +++ b/netbox/virtualization/models.py @@ -319,10 +319,10 @@ class VirtualMachine(ChangeLoggedModel, ConfigContextModel, CustomFieldModel): # because Django does not consider two NULL fields to be equal, and thus will not trigger a violation # of the uniqueness constraint without manual intervention. if self.tenant is None and VirtualMachine.objects.exclude(pk=self.pk).filter( - name=self.name, tenant__isnull=True + name=self.name, cluster=self.cluster, tenant__isnull=True ): raise ValidationError({ - 'name': 'A virtual machine with this name already exists.' + 'name': 'A virtual machine with this name already exists in the assigned cluster.' }) super().validate_unique(exclude) From d75696b30a489d57375e037712ec3fe1038f5cb7 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 3 Dec 2020 14:15:48 -0500 Subject: [PATCH 08/19] Fixes #5407: Add direct link to secret on secrets list --- docs/release-notes/version-2.9.md | 1 + netbox/secrets/tables.py | 14 +++++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/docs/release-notes/version-2.9.md b/docs/release-notes/version-2.9.md index e755e7b82..652f6d254 100644 --- a/docs/release-notes/version-2.9.md +++ b/docs/release-notes/version-2.9.md @@ -6,6 +6,7 @@ * [#5383](https://github.com/netbox-community/netbox/issues/5383) - Fix setting user password via REST API * [#5396](https://github.com/netbox-community/netbox/issues/5396) - Fix uniqueness constraint for virtual machine names +* [#5407](https://github.com/netbox-community/netbox/issues/5407) - Add direct link to secret on secrets list --- diff --git a/netbox/secrets/tables.py b/netbox/secrets/tables.py index 5e8c5a8b4..3302d0d19 100644 --- a/netbox/secrets/tables.py +++ b/netbox/secrets/tables.py @@ -28,12 +28,20 @@ class SecretRoleTable(BaseTable): class SecretTable(BaseTable): pk = ToggleColumn() - device = tables.LinkColumn() + id = tables.Column( + linkify=True + ) + device = tables.Column( + linkify=True + ) + role = tables.Column( + linkify=True + ) tags = TagColumn( url_name='secrets:secret_list' ) class Meta(BaseTable.Meta): model = Secret - fields = ('pk', 'device', 'role', 'name', 'last_updated', 'hash', 'tags') - default_columns = ('pk', 'device', 'role', 'name', 'last_updated') + fields = ('pk', 'id', 'device', 'role', 'name', 'last_updated', 'hash', 'tags') + default_columns = ('pk', 'id', 'device', 'role', 'name', 'last_updated') From 584b8109a0c8881d0195dccfc39f9571f380a80b Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 3 Dec 2020 15:22:44 -0500 Subject: [PATCH 09/19] Fixes #5408: Fix updating secrets without setting new plaintext --- docs/release-notes/version-2.9.md | 1 + netbox/secrets/views.py | 26 +++++++++++++------------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/docs/release-notes/version-2.9.md b/docs/release-notes/version-2.9.md index 652f6d254..7411ab149 100644 --- a/docs/release-notes/version-2.9.md +++ b/docs/release-notes/version-2.9.md @@ -7,6 +7,7 @@ * [#5383](https://github.com/netbox-community/netbox/issues/5383) - Fix setting user password via REST API * [#5396](https://github.com/netbox-community/netbox/issues/5396) - Fix uniqueness constraint for virtual machine names * [#5407](https://github.com/netbox-community/netbox/issues/5407) - Add direct link to secret on secrets list +* [#5408](https://github.com/netbox-community/netbox/issues/5408) - Fix updating secrets without setting new plaintext --- diff --git a/netbox/secrets/views.py b/netbox/secrets/views.py index e95392dfc..24138e0e4 100644 --- a/netbox/secrets/views.py +++ b/netbox/secrets/views.py @@ -108,13 +108,14 @@ class SecretEditView(ObjectEditView): if form.is_valid(): logger.debug("Form validation was successful") + secret = form.save(commit=False) - # We must have a session key in order to create a secret or update the plaintext of an existing secret - if (form.cleaned_data['plaintext'] or secret.pk is None) and session_key is None: + # We must have a session key in order to set the plaintext of a Secret + if form.cleaned_data['plaintext'] and session_key is None: logger.debug("Unable to proceed: No session key was provided with the request") form.add_error(None, "No session key was provided with the request. Unable to encrypt secret data.") - else: + elif form.cleaned_data['plaintext']: master_key = None try: sk = SessionKey.objects.get(userkey__user=request.user) @@ -125,19 +126,18 @@ class SecretEditView(ObjectEditView): if master_key is not None: logger.debug("Successfully resolved master key for encryption") - secret = form.save(commit=False) - if form.cleaned_data['plaintext']: - secret.plaintext = str(form.cleaned_data['plaintext']) + secret.plaintext = str(form.cleaned_data['plaintext']) secret.encrypt(master_key) - secret.save() - form.save_m2m() - msg = '{} secret'.format('Created' if not form.instance.pk else 'Modified') - logger.info(f"{msg} {secret} (PK: {secret.pk})") - msg = '{} {}'.format(msg, secret.get_absolute_url(), escape(secret)) - messages.success(request, mark_safe(msg)) + secret.save() + form.save_m2m() - return redirect(self.get_return_url(request, secret)) + msg = '{} secret'.format('Created' if not form.instance.pk else 'Modified') + logger.info(f"{msg} {secret} (PK: {secret.pk})") + msg = f'{msg} {escape(secret)}' + messages.success(request, mark_safe(msg)) + + return redirect(self.get_return_url(request, secret)) else: logger.debug("Form validation failed") From e7f64334c06748b4b85c54d881f5e2b03b9463b5 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Fri, 4 Dec 2020 08:57:19 -0500 Subject: [PATCH 10/19] Fixes #5410: Restore tags field on cable connection forms --- docs/release-notes/version-2.9.md | 1 + netbox/dcim/forms.py | 18 +++++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/docs/release-notes/version-2.9.md b/docs/release-notes/version-2.9.md index 7411ab149..a2b00dc12 100644 --- a/docs/release-notes/version-2.9.md +++ b/docs/release-notes/version-2.9.md @@ -8,6 +8,7 @@ * [#5396](https://github.com/netbox-community/netbox/issues/5396) - Fix uniqueness constraint for virtual machine names * [#5407](https://github.com/netbox-community/netbox/issues/5407) - Add direct link to secret on secrets list * [#5408](https://github.com/netbox-community/netbox/issues/5408) - Fix updating secrets without setting new plaintext +* [#5410](https://github.com/netbox-community/netbox/issues/5410) - Restore tags field on cable connection forms --- diff --git a/netbox/dcim/forms.py b/netbox/dcim/forms.py index 2132214a7..aa7537c7b 100644 --- a/netbox/dcim/forms.py +++ b/netbox/dcim/forms.py @@ -3502,12 +3502,16 @@ class ConnectCableToDeviceForm(BootstrapMixin, forms.ModelForm): 'rack_id': '$termination_b_rack', } ) + tags = DynamicModelMultipleChoiceField( + queryset=Tag.objects.all(), + required=False + ) class Meta: model = Cable fields = [ 'termination_b_region', 'termination_b_site', 'termination_b_rack', 'termination_b_device', - 'termination_b_id', 'type', 'status', 'label', 'color', 'length', 'length_unit', + 'termination_b_id', 'type', 'status', 'label', 'color', 'length', 'length_unit', 'tags', ] widgets = { 'status': StaticSelect2, @@ -3635,12 +3639,16 @@ class ConnectCableToCircuitTerminationForm(BootstrapMixin, forms.ModelForm): 'circuit_id': '$termination_b_circuit' } ) + tags = DynamicModelMultipleChoiceField( + queryset=Tag.objects.all(), + required=False + ) class Meta: model = Cable fields = [ 'termination_b_provider', 'termination_b_region', 'termination_b_site', 'termination_b_circuit', - 'termination_b_id', 'type', 'status', 'label', 'color', 'length', 'length_unit', + 'termination_b_id', 'type', 'status', 'label', 'color', 'length', 'length_unit', 'tags', ] def clean_termination_b_id(self): @@ -3688,12 +3696,16 @@ class ConnectCableToPowerFeedForm(BootstrapMixin, forms.ModelForm): 'power_panel_id': '$termination_b_powerpanel' } ) + tags = DynamicModelMultipleChoiceField( + queryset=Tag.objects.all(), + required=False + ) class Meta: model = Cable fields = [ 'termination_b_rackgroup', 'termination_b_powerpanel', 'termination_b_id', 'type', 'status', 'label', - 'color', 'length', 'length_unit', + 'color', 'length', 'length_unit', 'tags', ] def clean_termination_b_id(self): From cc5c000a6d027a7d60c9fe637113616413a94062 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Tue, 8 Dec 2020 14:55:06 -0500 Subject: [PATCH 11/19] Move CI to GitHub Actions (#5431) Replaces Travis CI with GitHub Actions --- .github/workflows/ci.yml | 50 +++++++++++++++++++++ .travis.yml | 19 -------- README.md | 4 +- netbox/netbox/configuration.testing.py | 4 +- scripts/cibuild.sh | 60 -------------------------- 5 files changed, 54 insertions(+), 83 deletions(-) create mode 100644 .github/workflows/ci.yml delete mode 100644 .travis.yml delete mode 100755 scripts/cibuild.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..232325774 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,50 @@ +name: CI +on: push +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.6, 3.7] + services: + redis: + image: redis + ports: + - 6379:6379 + postgres: + image: postgres + env: + POSTGRES_USER: netbox + POSTGRES_PASSWORD: netbox + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + + steps: + - name: Check out repo + uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies & set up configuration + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install pycodestyle coverage + ln -s configuration.testing.py netbox/netbox/configuration.py + + - name: Check PEP8 compliance + run: pycodestyle --ignore=W504,E501 netbox/ + + - name: Run tests + run: coverage run --source="netbox/" netbox/manage.py test netbox/ + + - name: Show coverage report + run: coverage report --skip-covered --omit *migrations* diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 0dcbd9ee1..000000000 --- a/.travis.yml +++ /dev/null @@ -1,19 +0,0 @@ -sudo: required -services: - - postgresql - - redis-server -addons: - postgresql: "9.6" -language: python -python: - - "3.6" - - "3.7" -install: - - pip install -r requirements.txt - - pip install pycodestyle - - pip install coverage -before_script: - - psql --version - - psql -U postgres -c 'SELECT version();' -script: - - ./scripts/cibuild.sh diff --git a/README.md b/README.md index 56c71af78..9d27844c2 100644 --- a/README.md +++ b/README.md @@ -19,8 +19,8 @@ or join us in the **#netbox** Slack channel on [NetworkToCode](https://networkto | | status | |-------------|------------| -| **master** | [![Build Status](https://travis-ci.org/netbox-community/netbox.svg?branch=master)](https://travis-ci.com/netbox-community/netbox/) | -| **develop** | [![Build Status](https://travis-ci.org/netbox-community/netbox.svg?branch=develop)](https://travis-ci.com/netbox-community/netbox/) | +| **master** | ![Build status](https://github.com/netbox-community/netbox/workflows/CI/badge.svg?branch=master) | +| **develop** | ![Build status](https://github.com/netbox-community/netbox/workflows/CI/badge.svg?branch=develop) | ### Screenshots diff --git a/netbox/netbox/configuration.testing.py b/netbox/netbox/configuration.testing.py index 066f94841..59529b80c 100644 --- a/netbox/netbox/configuration.testing.py +++ b/netbox/netbox/configuration.testing.py @@ -7,8 +7,8 @@ ALLOWED_HOSTS = ['*'] DATABASE = { 'NAME': 'netbox', - 'USER': '', - 'PASSWORD': '', + 'USER': 'netbox', + 'PASSWORD': 'netbox', 'HOST': 'localhost', 'PORT': '', 'CONN_MAX_AGE': 300, diff --git a/scripts/cibuild.sh b/scripts/cibuild.sh deleted file mode 100755 index 6a0422308..000000000 --- a/scripts/cibuild.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash - -# Exit code starts at 0 but is modified if any checks fail -EXIT=0 - -# Output a line prefixed with a timestamp -info() -{ - echo "$(date +'%F %T') |" -} - -# Track number of seconds required to run script -START=$(date +%s) -echo "$(info) starting build checks." - -# Syntax check all python source files -SYNTAX=$(find . -name "*.py" -type f -exec python -m py_compile {} \; 2>&1) -if [[ ! -z $SYNTAX ]]; then - echo -e "$SYNTAX" - echo -e "\n$(info) detected one or more syntax errors, failing build." - EXIT=1 -fi - -# Check all python source files for PEP 8 compliance, but explicitly -# ignore: -# - W504: line break after binary operator -# - E501: line greater than 80 characters in length -pycodestyle \ - --ignore=W504,E501 \ - netbox/ -RC=$? -if [[ $RC != 0 ]]; then - echo -e "\n$(info) one or more PEP 8 errors detected, failing build." - EXIT=$RC -fi - -# Point to the testing configuration file for use in CI -ln -s configuration.testing.py netbox/netbox/configuration.py - -# Run NetBox tests -coverage run --source="netbox/" netbox/manage.py test netbox/ -RC=$? -if [[ $RC != 0 ]]; then - echo -e "\n$(info) one or more tests failed, failing build." - EXIT=$RC -fi - -# Show code coverage report -coverage report --skip-covered --omit *migrations* -RC=$? -if [[ $RC != 0 ]]; then - echo -e "\n$(info) failed to generate code coverage report." - EXIT=$RC -fi - -# Show build duration -END=$(date +%s) -echo "$(info) exiting with code $EXIT after $(($END - $START)) seconds." - -exit $EXIT From 27e27788cd307fea3f0030aafbe81f9472ab652f Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Tue, 8 Dec 2020 16:14:52 -0500 Subject: [PATCH 12/19] Closes #5424: Allow passing Python code to nbshell using --command --- docs/release-notes/version-2.9.md | 4 ++++ netbox/extras/management/commands/nbshell.py | 17 ++++++++++++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/docs/release-notes/version-2.9.md b/docs/release-notes/version-2.9.md index a2b00dc12..b976c7fcd 100644 --- a/docs/release-notes/version-2.9.md +++ b/docs/release-notes/version-2.9.md @@ -2,6 +2,10 @@ ## v2.9.11 (FUTURE) +### Enhancements + +* [#5424](https://github.com/netbox-community/netbox/issues/5424) - Allow passing Python code to `nbshell` using `--command` + ### Bug Fixes * [#5383](https://github.com/netbox-community/netbox/issues/5383) - Fix setting user password via REST API diff --git a/netbox/extras/management/commands/nbshell.py b/netbox/extras/management/commands/nbshell.py index 48da46525..bcb5568b7 100644 --- a/netbox/extras/management/commands/nbshell.py +++ b/netbox/extras/management/commands/nbshell.py @@ -25,12 +25,18 @@ class Command(BaseCommand): help = "Start the Django shell with all NetBox models already imported" django_models = {} + def add_arguments(self, parser): + parser.add_argument( + '-c', '--command', + help='Python code to execute (instead of starting an interactive shell)', + ) + def _lsmodels(self): for app, models in self.django_models.items(): app_name = apps.get_app_config(app).verbose_name - print('{}:'.format(app_name)) + print(f'{app_name}:') for m in models: - print(' {}'.format(m)) + print(f' {m}') def get_namespace(self): namespace = {} @@ -46,7 +52,7 @@ class Command(BaseCommand): # Constants try: - app_constants = sys.modules['{}.constants'.format(app)] + app_constants = sys.modules[f'{app}.constants'] for name in dir(app_constants): namespace[name] = getattr(app_constants, name) except KeyError: @@ -64,5 +70,10 @@ class Command(BaseCommand): return namespace def handle(self, **options): + # If Python code has been passed, execute it and exit. + if options['command']: + exec(options['command'], self.get_namespace()) + return + shell = code.interact(banner=BANNER_TEXT, local=self.get_namespace()) return shell From e062cbb79f12dc26603c7aaa6288a7560097a7c1 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 10 Dec 2020 08:52:55 -0500 Subject: [PATCH 13/19] Fixes #5436: Show assigned IP addresses in interfaces list --- docs/release-notes/version-2.9.md | 1 + netbox/dcim/tables.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/release-notes/version-2.9.md b/docs/release-notes/version-2.9.md index b976c7fcd..9187f9ff7 100644 --- a/docs/release-notes/version-2.9.md +++ b/docs/release-notes/version-2.9.md @@ -13,6 +13,7 @@ * [#5407](https://github.com/netbox-community/netbox/issues/5407) - Add direct link to secret on secrets list * [#5408](https://github.com/netbox-community/netbox/issues/5408) - Fix updating secrets without setting new plaintext * [#5410](https://github.com/netbox-community/netbox/issues/5410) - Restore tags field on cable connection forms +* [#5436](https://github.com/netbox-community/netbox/issues/5436) - Show assigned IP addresses in interfaces list --- diff --git a/netbox/dcim/tables.py b/netbox/dcim/tables.py index 2d8195f85..08235997d 100644 --- a/netbox/dcim/tables.py +++ b/netbox/dcim/tables.py @@ -141,7 +141,7 @@ POWERPANEL_POWERFEED_COUNT = """ """ INTERFACE_IPADDRESSES = """ -{% for ip in record.ip_addresses.unrestricted %} +{% for ip in record.ip_addresses.all %} {{ ip }}
{% endfor %} """ From 5df01ed3e68be8770dc6b1c773f34c2939a585dc Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 10 Dec 2020 15:59:42 -0500 Subject: [PATCH 14/19] Encourage people to use GitHub discussions in place of the mailing list --- .github/ISSUE_TEMPLATE/bug_report.md | 8 +++----- .github/ISSUE_TEMPLATE/feature_request.md | 6 ++---- README.md | 15 +++++++++------ docs/development/index.md | 3 ++- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index fef891803..5df769b94 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -7,11 +7,9 @@ about: Report a reproducible bug in the current release of NetBox