From 1af8a7e68adc373ff195cecd7ec29e24600d8ee3 Mon Sep 17 00:00:00 2001 From: Alyssa Bigley Date: Tue, 25 May 2021 11:09:33 -0400 Subject: [PATCH 01/48] CSV Upload as second field in existing form --- netbox/netbox/views/generic.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/netbox/netbox/views/generic.py b/netbox/netbox/views/generic.py index 8a2bae4fa..8228f6eff 100644 --- a/netbox/netbox/views/generic.py +++ b/netbox/netbox/views/generic.py @@ -7,7 +7,7 @@ from django.contrib.contenttypes.models import ContentType from django.core.exceptions import FieldDoesNotExist, ObjectDoesNotExist, ValidationError from django.db import transaction, IntegrityError from django.db.models import ManyToManyField, ProtectedError -from django.forms import Form, ModelMultipleChoiceField, MultipleHiddenInput, Textarea +from django.forms import Form, ModelMultipleChoiceField, MultipleHiddenInput, Textarea, FileField, CharField from django.http import HttpResponse from django.shortcuts import get_object_or_404, redirect, render from django.utils.html import escape @@ -665,7 +665,10 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): from_form=self.model_form, widget=Textarea(attrs=self.widget_attrs) ) - + Upload_CSV = FileField( + required=False + ) + # this is where the import form is created -- add csv upload here or create new form? return ImportForm(*args, **kwargs) def _save_obj(self, obj_form, request): @@ -690,6 +693,7 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): logger = logging.getLogger('netbox.views.BulkImportView') new_objs = [] form = self._import_form(request.POST) + # this is where the csv data is handled if form.is_valid(): logger.debug("Form validation was successful") @@ -697,6 +701,9 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): try: # Iterate through CSV data and bind each row to a new model form instance. with transaction.atomic(): + # some prints trying to figure out how the form works + print(type(form["Upload_CSV"])) + print(form["Upload_CSV"].data) headers, records = form.cleaned_data['csv'] for row, data in enumerate(records, start=1): obj_form = self.model_form(data, headers=headers) From 2a67c83712eaa8ddd441eaa0a4ef900ef090fc3a Mon Sep 17 00:00:00 2001 From: Alyssa Bigley Date: Thu, 27 May 2021 10:31:51 -0400 Subject: [PATCH 02/48] working csv upload first draft --- netbox/netbox/views/generic.py | 31 ++++++++++++++++--- .../templates/generic/object_bulk_import.html | 2 +- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/netbox/netbox/views/generic.py b/netbox/netbox/views/generic.py index 8228f6eff..c01f69783 100644 --- a/netbox/netbox/views/generic.py +++ b/netbox/netbox/views/generic.py @@ -701,10 +701,33 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): try: # Iterate through CSV data and bind each row to a new model form instance. with transaction.atomic(): - # some prints trying to figure out how the form works - print(type(form["Upload_CSV"])) - print(form["Upload_CSV"].data) - headers, records = form.cleaned_data['csv'] + + if len(request.FILES) != 0: + csv_file = request.FILES["Upload_CSV"] + csv_file.seek(0) + csv_str = csv_file.read().decode('utf-8') + + csv_list = csv_str.split('\n') + header_row = csv_list[0] + csv_list.pop(0) + header_list = header_row.split(',') + headers = {} + for elt in header_list: + headers[elt] = None + records = [] + for row in csv_list: + if row != "": + row_str = (',').join(row) + row_list = row.split(',') + row_dict = {} + for i, elt in enumerate(row_list): + if elt == '': + row_dict[header_list[i]] = None + else: + row_dict[header_list[i]] = elt + records.append(row_dict) + else: + headers, records = form.cleaned_data['csv'] for row, data in enumerate(records, start=1): obj_form = self.model_form(data, headers=headers) restrict_form_fields(obj_form, request.user) diff --git a/netbox/templates/generic/object_bulk_import.html b/netbox/templates/generic/object_bulk_import.html index 170cf3665..e9972e919 100644 --- a/netbox/templates/generic/object_bulk_import.html +++ b/netbox/templates/generic/object_bulk_import.html @@ -20,7 +20,7 @@
-
+ {% csrf_token %} {% render_form form %}
From 92d9dc4f5cec515bf8136723afef163aa1d8b34e Mon Sep 17 00:00:00 2001 From: Alyssa Bigley Date: Wed, 2 Jun 2021 10:12:26 -0400 Subject: [PATCH 03/48] csv parse using python csv library --- netbox/netbox/views/generic.py | 37 ++++++++++++++-------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/netbox/netbox/views/generic.py b/netbox/netbox/views/generic.py index c01f69783..c665c3630 100644 --- a/netbox/netbox/views/generic.py +++ b/netbox/netbox/views/generic.py @@ -1,5 +1,6 @@ import logging import re +import csv from copy import deepcopy from django.contrib import messages @@ -7,7 +8,7 @@ from django.contrib.contenttypes.models import ContentType from django.core.exceptions import FieldDoesNotExist, ObjectDoesNotExist, ValidationError from django.db import transaction, IntegrityError from django.db.models import ManyToManyField, ProtectedError -from django.forms import Form, ModelMultipleChoiceField, MultipleHiddenInput, Textarea, FileField, CharField +from django.forms import Form, ModelMultipleChoiceField, MultipleHiddenInput, Textarea, FileField from django.http import HttpResponse from django.shortcuts import get_object_or_404, redirect, render from django.utils.html import escape @@ -668,7 +669,6 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): Upload_CSV = FileField( required=False ) - # this is where the import form is created -- add csv upload here or create new form? return ImportForm(*args, **kwargs) def _save_obj(self, obj_form, request): @@ -693,7 +693,6 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): logger = logging.getLogger('netbox.views.BulkImportView') new_objs = [] form = self._import_form(request.POST) - # this is where the csv data is handled if form.is_valid(): logger.debug("Form validation was successful") @@ -701,33 +700,27 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): try: # Iterate through CSV data and bind each row to a new model form instance. with transaction.atomic(): - if len(request.FILES) != 0: csv_file = request.FILES["Upload_CSV"] csv_file.seek(0) csv_str = csv_file.read().decode('utf-8') - - csv_list = csv_str.split('\n') - header_row = csv_list[0] - csv_list.pop(0) - header_list = header_row.split(',') + reader = csv.reader(csv_str.splitlines()) + headers_list = next(reader) headers = {} - for elt in header_list: - headers[elt] = None + for header in headers_list: + headers[header] = None records = [] - for row in csv_list: - if row != "": - row_str = (',').join(row) - row_list = row.split(',') - row_dict = {} - for i, elt in enumerate(row_list): - if elt == '': - row_dict[header_list[i]] = None - else: - row_dict[header_list[i]] = elt - records.append(row_dict) + for row in reader: + row_dict = {} + for i, elt in enumerate(row): + if elt == '': + row_dict[headers_list[i]] = None + else: + row_dict[headers_list[i]] = elt + records.append(row_dict) else: headers, records = form.cleaned_data['csv'] + print("headers:", headers, "records:", records) for row, data in enumerate(records, start=1): obj_form = self.model_form(data, headers=headers) restrict_form_fields(obj_form, request.user) From c00b7abad388591f56f50ac3ea8bd0df39091f50 Mon Sep 17 00:00:00 2001 From: Alyssa Bigley Date: Thu, 3 Jun 2021 15:08:47 -0400 Subject: [PATCH 04/48] cleaned up csv parsing --- netbox/netbox/views/generic.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/netbox/netbox/views/generic.py b/netbox/netbox/views/generic.py index c665c3630..0d1734c08 100644 --- a/netbox/netbox/views/generic.py +++ b/netbox/netbox/views/generic.py @@ -666,7 +666,7 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): from_form=self.model_form, widget=Textarea(attrs=self.widget_attrs) ) - Upload_CSV = FileField( + upload_csv = FileField( required=False ) return ImportForm(*args, **kwargs) @@ -692,7 +692,7 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): def post(self, request): logger = logging.getLogger('netbox.views.BulkImportView') new_objs = [] - form = self._import_form(request.POST) + form = self._import_form(request.POST, request.FILES) if form.is_valid(): logger.debug("Form validation was successful") @@ -700,8 +700,8 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): try: # Iterate through CSV data and bind each row to a new model form instance. with transaction.atomic(): - if len(request.FILES) != 0: - csv_file = request.FILES["Upload_CSV"] + if request.FILES: + csv_file = request.FILES["upload_csv"] csv_file.seek(0) csv_str = csv_file.read().decode('utf-8') reader = csv.reader(csv_str.splitlines()) From 12c71257f26a5a84d58cb6b689376357d5dff523 Mon Sep 17 00:00:00 2001 From: Alyssa Bigley Date: Fri, 4 Jun 2021 10:27:19 -0400 Subject: [PATCH 05/48] CSV import implemented using CSVFileField --- netbox/netbox/views/generic.py | 27 +++--------- netbox/utilities/forms/fields.py | 72 ++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 22 deletions(-) diff --git a/netbox/netbox/views/generic.py b/netbox/netbox/views/generic.py index 0d1734c08..24d459219 100644 --- a/netbox/netbox/views/generic.py +++ b/netbox/netbox/views/generic.py @@ -1,6 +1,5 @@ import logging import re -import csv from copy import deepcopy from django.contrib import messages @@ -8,7 +7,7 @@ from django.contrib.contenttypes.models import ContentType from django.core.exceptions import FieldDoesNotExist, ObjectDoesNotExist, ValidationError from django.db import transaction, IntegrityError from django.db.models import ManyToManyField, ProtectedError -from django.forms import Form, ModelMultipleChoiceField, MultipleHiddenInput, Textarea, FileField +from django.forms import Form, ModelMultipleChoiceField, MultipleHiddenInput, Textarea from django.http import HttpResponse from django.shortcuts import get_object_or_404, redirect, render from django.utils.html import escape @@ -21,7 +20,7 @@ from extras.models import CustomField, ExportTemplate from utilities.error_handlers import handle_protectederror from utilities.exceptions import AbortTransaction from utilities.forms import ( - BootstrapMixin, BulkRenameForm, ConfirmationForm, CSVDataField, ImportForm, TableConfigForm, restrict_form_fields, + BootstrapMixin, BulkRenameForm, ConfirmationForm, CSVDataField, ImportForm, TableConfigForm, restrict_form_fields, CSVFileField ) from utilities.permissions import get_permission_for_model from utilities.tables import paginate_table @@ -666,7 +665,8 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): from_form=self.model_form, widget=Textarea(attrs=self.widget_attrs) ) - upload_csv = FileField( + upload_csv = CSVFileField( + from_form=self.model_form, required=False ) return ImportForm(*args, **kwargs) @@ -701,26 +701,9 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): # Iterate through CSV data and bind each row to a new model form instance. with transaction.atomic(): if request.FILES: - csv_file = request.FILES["upload_csv"] - csv_file.seek(0) - csv_str = csv_file.read().decode('utf-8') - reader = csv.reader(csv_str.splitlines()) - headers_list = next(reader) - headers = {} - for header in headers_list: - headers[header] = None - records = [] - for row in reader: - row_dict = {} - for i, elt in enumerate(row): - if elt == '': - row_dict[headers_list[i]] = None - else: - row_dict[headers_list[i]] = elt - records.append(row_dict) + headers, records = form.cleaned_data['upload_csv'] else: headers, records = form.cleaned_data['csv'] - print("headers:", headers, "records:", records) for row, data in enumerate(records, start=1): obj_form = self.model_form(data, headers=headers) restrict_form_fields(obj_form, request.user) diff --git a/netbox/utilities/forms/fields.py b/netbox/utilities/forms/fields.py index 93b9e6c44..1d8e299e0 100644 --- a/netbox/utilities/forms/fields.py +++ b/netbox/utilities/forms/fields.py @@ -26,6 +26,7 @@ __all__ = ( 'CSVChoiceField', 'CSVContentTypeField', 'CSVDataField', + 'CSVFileField', 'CSVModelChoiceField', 'CSVTypedChoiceField', 'DynamicModelChoiceField', @@ -221,6 +222,77 @@ class CSVDataField(forms.CharField): return value +class CSVFileField(forms.FileField): + """ + A CharField (rendered as a Textarea) which accepts CSV-formatted data. It returns data as a two-tuple: The first + item is a dictionary of column headers, mapping field names to the attribute by which they match a related object + (where applicable). The second item is a list of dictionaries, each representing a discrete row of CSV data. + + :param from_form: The form from which the field derives its validation rules. + """ + + def __init__(self, from_form, *args, **kwargs): + + form = from_form() + self.model = form.Meta.model + self.fields = form.fields + self.required_fields = [ + name for name, field in form.fields.items() if field.required + ] + + super().__init__(*args, **kwargs) + + def to_python(self, file): + + records = [] + file.seek(0) + csv_str = file.read().decode('utf-8') + reader = csv.reader(csv_str.splitlines()) + + # Consume the first line of CSV data as column headers. Create a dictionary mapping each header to an optional + # "to" field specifying how the related object is being referenced. For example, importing a Device might use a + # `site.slug` header, to indicate the related site is being referenced by its slug. + + headers = {} + for header in next(reader): + if '.' in header: + field, to_field = header.split('.', 1) + headers[field] = to_field + else: + headers[header] = None + + # Parse CSV rows into a list of dictionaries mapped from the column headers. + for i, row in enumerate(reader, start=1): + if len(row) != len(headers): + raise forms.ValidationError( + f"Row {i}: Expected {len(headers)} columns but found {len(row)}" + ) + row = [col.strip() for col in row] + record = dict(zip(headers.keys(), row)) + records.append(record) + + return headers, records + + def validate(self, value): + headers, records = value + + # Validate provided column headers + for field, to_field in headers.items(): + if field not in self.fields: + raise forms.ValidationError(f'Unexpected column header "{field}" found.') + if to_field and not hasattr(self.fields[field], 'to_field_name'): + raise forms.ValidationError(f'Column "{field}" is not a related object; cannot use dots') + if to_field and not hasattr(self.fields[field].queryset.model, to_field): + raise forms.ValidationError(f'Invalid related object attribute for column "{field}": {to_field}') + + # Validate required fields + for f in self.required_fields: + if f not in headers: + raise forms.ValidationError(f'Required column header "{f}" not found.') + + return value + + class CSVChoiceField(forms.ChoiceField): """ Invert the provided set of choices to take the human-friendly label as input, and return the database value. From bc70b23279ffd16229e7cbcec48605ff5a0f0b5f Mon Sep 17 00:00:00 2001 From: Alyssa Bigley Date: Mon, 7 Jun 2021 11:52:31 -0400 Subject: [PATCH 06/48] edited docstring for CSVFileField --- netbox/utilities/forms/fields.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/netbox/utilities/forms/fields.py b/netbox/utilities/forms/fields.py index 1d8e299e0..a8383de6e 100644 --- a/netbox/utilities/forms/fields.py +++ b/netbox/utilities/forms/fields.py @@ -224,9 +224,10 @@ class CSVDataField(forms.CharField): class CSVFileField(forms.FileField): """ - A CharField (rendered as a Textarea) which accepts CSV-formatted data. It returns data as a two-tuple: The first - item is a dictionary of column headers, mapping field names to the attribute by which they match a related object - (where applicable). The second item is a list of dictionaries, each representing a discrete row of CSV data. + A FileField (rendered as a file input button) which accepts a file containing CSV-formatted data. It returns + data as a two-tuple: The first item is a dictionary of column headers, mapping field names to the attribute + by which they match a related object (where applicable). The second item is a list of dictionaries, each + representing a discrete row of CSV data. :param from_form: The form from which the field derives its validation rules. """ From 24bd8549dce9a1578db17c96d130db5b6b1b85db Mon Sep 17 00:00:00 2001 From: Alyssa Bigley Date: Mon, 7 Jun 2021 14:29:38 -0400 Subject: [PATCH 07/48] removed unnecessary use of seek() --- netbox/utilities/forms/fields.py | 1 - 1 file changed, 1 deletion(-) diff --git a/netbox/utilities/forms/fields.py b/netbox/utilities/forms/fields.py index a8383de6e..0f5b0b6dd 100644 --- a/netbox/utilities/forms/fields.py +++ b/netbox/utilities/forms/fields.py @@ -246,7 +246,6 @@ class CSVFileField(forms.FileField): def to_python(self, file): records = [] - file.seek(0) csv_str = file.read().decode('utf-8') reader = csv.reader(csv_str.splitlines()) From 2b95b626e5c02b9d8b3f66f0437d064a2e23fc5f Mon Sep 17 00:00:00 2001 From: Alyssa Bigley Date: Thu, 10 Jun 2021 14:41:33 -0400 Subject: [PATCH 08/48] changed name of csv_file variable and started work on ValidationError --- netbox/netbox/views/generic.py | 12 +++++++--- netbox/utilities/forms/fields.py | 38 ++++++++++++++++++-------------- 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/netbox/netbox/views/generic.py b/netbox/netbox/views/generic.py index 24d459219..7ff980750 100644 --- a/netbox/netbox/views/generic.py +++ b/netbox/netbox/views/generic.py @@ -665,10 +665,16 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): from_form=self.model_form, widget=Textarea(attrs=self.widget_attrs) ) - upload_csv = CSVFileField( + csv_file = CSVFileField( + label="CSV file", from_form=self.model_form, required=False ) + def used_both_methods(self): + if self.cleaned_data['csv_file'][1] and self.cleaned_data['csv'][1]: + raise ValidationError('') + return False + return ImportForm(*args, **kwargs) def _save_obj(self, obj_form, request): @@ -694,14 +700,14 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): new_objs = [] form = self._import_form(request.POST, request.FILES) - if form.is_valid(): + if form.is_valid() and not form.used_both_methods(): logger.debug("Form validation was successful") try: # Iterate through CSV data and bind each row to a new model form instance. with transaction.atomic(): if request.FILES: - headers, records = form.cleaned_data['upload_csv'] + headers, records = form.cleaned_data['csv_file'] else: headers, records = form.cleaned_data['csv'] for row, data in enumerate(records, start=1): diff --git a/netbox/utilities/forms/fields.py b/netbox/utilities/forms/fields.py index 0f5b0b6dd..2954f9c73 100644 --- a/netbox/utilities/forms/fields.py +++ b/netbox/utilities/forms/fields.py @@ -246,35 +246,39 @@ class CSVFileField(forms.FileField): def to_python(self, file): records = [] - csv_str = file.read().decode('utf-8') - reader = csv.reader(csv_str.splitlines()) + if file: + csv_str = file.read().decode('utf-8') + reader = csv.reader(csv_str.splitlines()) # Consume the first line of CSV data as column headers. Create a dictionary mapping each header to an optional # "to" field specifying how the related object is being referenced. For example, importing a Device might use a # `site.slug` header, to indicate the related site is being referenced by its slug. headers = {} - for header in next(reader): - if '.' in header: - field, to_field = header.split('.', 1) - headers[field] = to_field - else: - headers[header] = None + if file: + for header in next(reader): + if '.' in header: + field, to_field = header.split('.', 1) + headers[field] = to_field + else: + headers[header] = None - # Parse CSV rows into a list of dictionaries mapped from the column headers. - for i, row in enumerate(reader, start=1): - if len(row) != len(headers): - raise forms.ValidationError( - f"Row {i}: Expected {len(headers)} columns but found {len(row)}" - ) - row = [col.strip() for col in row] - record = dict(zip(headers.keys(), row)) - records.append(record) + # Parse CSV rows into a list of dictionaries mapped from the column headers. + for i, row in enumerate(reader, start=1): + if len(row) != len(headers): + raise forms.ValidationError( + f"Row {i}: Expected {len(headers)} columns but found {len(row)}" + ) + row = [col.strip() for col in row] + record = dict(zip(headers.keys(), row)) + records.append(record) return headers, records def validate(self, value): headers, records = value + if not headers and not records: + return value # Validate provided column headers for field, to_field in headers.items(): From 7d48043937305682b78caf76983a294b7bcecaaf Mon Sep 17 00:00:00 2001 From: Alyssa Bigley Date: Fri, 11 Jun 2021 13:42:26 -0400 Subject: [PATCH 09/48] Caught and handled ValidationError --- netbox/netbox/views/generic.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/netbox/netbox/views/generic.py b/netbox/netbox/views/generic.py index 7ff980750..00841f4b9 100644 --- a/netbox/netbox/views/generic.py +++ b/netbox/netbox/views/generic.py @@ -670,9 +670,10 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): from_form=self.model_form, required=False ) - def used_both_methods(self): + + def used_both_csv_fields(self): if self.cleaned_data['csv_file'][1] and self.cleaned_data['csv'][1]: - raise ValidationError('') + return True return False return ImportForm(*args, **kwargs) @@ -700,10 +701,13 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): new_objs = [] form = self._import_form(request.POST, request.FILES) - if form.is_valid() and not form.used_both_methods(): + if form.is_valid(): logger.debug("Form validation was successful") try: + if form.used_both_csv_fields(): + form.add_error('csv_file', "Choose one of two import methods") + raise ValidationError("") # Iterate through CSV data and bind each row to a new model form instance. with transaction.atomic(): if request.FILES: From e34e92b7b998c4926258c257f702122abe4ebfb0 Mon Sep 17 00:00:00 2001 From: Alyssa Bigley Date: Mon, 14 Jun 2021 14:07:37 -0400 Subject: [PATCH 10/48] moved duplicated code in CSV Fields into functions in forms/utils.py --- netbox/utilities/forms/fields.py | 80 +++----------------------------- netbox/utilities/forms/utils.py | 53 +++++++++++++++++++++ 2 files changed, 59 insertions(+), 74 deletions(-) diff --git a/netbox/utilities/forms/fields.py b/netbox/utilities/forms/fields.py index 2954f9c73..2b9b57ea3 100644 --- a/netbox/utilities/forms/fields.py +++ b/netbox/utilities/forms/fields.py @@ -17,7 +17,7 @@ from utilities.utils import content_type_name from utilities.validators import EnhancedURLValidator from . import widgets from .constants import * -from .utils import expand_alphanumeric_pattern, expand_ipaddress_pattern +from .utils import expand_alphanumeric_pattern, expand_ipaddress_pattern, parse_csv, validate_csv __all__ = ( 'CommentField', @@ -175,49 +175,13 @@ class CSVDataField(forms.CharField): 'in double quotes.' def to_python(self, value): - - records = [] reader = csv.reader(StringIO(value.strip())) - # Consume the first line of CSV data as column headers. Create a dictionary mapping each header to an optional - # "to" field specifying how the related object is being referenced. For example, importing a Device might use a - # `site.slug` header, to indicate the related site is being referenced by its slug. - headers = {} - for header in next(reader): - if '.' in header: - field, to_field = header.split('.', 1) - headers[field] = to_field - else: - headers[header] = None - - # Parse CSV rows into a list of dictionaries mapped from the column headers. - for i, row in enumerate(reader, start=1): - if len(row) != len(headers): - raise forms.ValidationError( - f"Row {i}: Expected {len(headers)} columns but found {len(row)}" - ) - row = [col.strip() for col in row] - record = dict(zip(headers.keys(), row)) - records.append(record) - - return headers, records + return parse_csv(reader) def validate(self, value): headers, records = value - - # Validate provided column headers - for field, to_field in headers.items(): - if field not in self.fields: - raise forms.ValidationError(f'Unexpected column header "{field}" found.') - if to_field and not hasattr(self.fields[field], 'to_field_name'): - raise forms.ValidationError(f'Column "{field}" is not a related object; cannot use dots') - if to_field and not hasattr(self.fields[field].queryset.model, to_field): - raise forms.ValidationError(f'Invalid related object attribute for column "{field}": {to_field}') - - # Validate required fields - for f in self.required_fields: - if f not in headers: - raise forms.ValidationError(f'Required column header "{f}" not found.') + validate_csv(headers, self.fields, self.required_fields) return value @@ -244,34 +208,14 @@ class CSVFileField(forms.FileField): super().__init__(*args, **kwargs) def to_python(self, file): - - records = [] if file: csv_str = file.read().decode('utf-8') reader = csv.reader(csv_str.splitlines()) - # Consume the first line of CSV data as column headers. Create a dictionary mapping each header to an optional - # "to" field specifying how the related object is being referenced. For example, importing a Device might use a - # `site.slug` header, to indicate the related site is being referenced by its slug. - headers = {} + records = [] if file: - for header in next(reader): - if '.' in header: - field, to_field = header.split('.', 1) - headers[field] = to_field - else: - headers[header] = None - - # Parse CSV rows into a list of dictionaries mapped from the column headers. - for i, row in enumerate(reader, start=1): - if len(row) != len(headers): - raise forms.ValidationError( - f"Row {i}: Expected {len(headers)} columns but found {len(row)}" - ) - row = [col.strip() for col in row] - record = dict(zip(headers.keys(), row)) - records.append(record) + headers, records = parse_csv(reader) return headers, records @@ -280,19 +224,7 @@ class CSVFileField(forms.FileField): if not headers and not records: return value - # Validate provided column headers - for field, to_field in headers.items(): - if field not in self.fields: - raise forms.ValidationError(f'Unexpected column header "{field}" found.') - if to_field and not hasattr(self.fields[field], 'to_field_name'): - raise forms.ValidationError(f'Column "{field}" is not a related object; cannot use dots') - if to_field and not hasattr(self.fields[field].queryset.model, to_field): - raise forms.ValidationError(f'Invalid related object attribute for column "{field}": {to_field}') - - # Validate required fields - for f in self.required_fields: - if f not in headers: - raise forms.ValidationError(f'Required column header "{f}" not found.') + validate_csv(headers, self.fields, self.required_fields) return value diff --git a/netbox/utilities/forms/utils.py b/netbox/utilities/forms/utils.py index dc001be1a..50a6a416e 100644 --- a/netbox/utilities/forms/utils.py +++ b/netbox/utilities/forms/utils.py @@ -14,6 +14,8 @@ __all__ = ( 'parse_alphanumeric_range', 'parse_numeric_range', 'restrict_form_fields', + 'parse_csv', + 'validate_csv', ) @@ -134,3 +136,54 @@ def restrict_form_fields(form, user, action='view'): for field in form.fields.values(): if hasattr(field, 'queryset') and issubclass(field.queryset.__class__, RestrictedQuerySet): field.queryset = field.queryset.restrict(user, action) + + +def parse_csv(reader): + """ + Parse a csv_reader object into a headers dictionary and a list of records dictionaries. Raise an error + if the records are formatted incorrectly. Return headers and records as a tuple. + """ + records = [] + headers = {} + + # Consume the first line of CSV data as column headers. Create a dictionary mapping each header to an optional + # "to" field specifying how the related object is being referenced. For example, importing a Device might use a + # `site.slug` header, to indicate the related site is being referenced by its slug. + + for header in next(reader): + if '.' in header: + field, to_field = header.split('.', 1) + headers[field] = to_field + else: + headers[header] = None + + # Parse CSV rows into a list of dictionaries mapped from the column headers. + for i, row in enumerate(reader, start=1): + if len(row) != len(headers): + raise forms.ValidationError( + f"Row {i}: Expected {len(headers)} columns but found {len(row)}" + ) + row = [col.strip() for col in row] + record = dict(zip(headers.keys(), row)) + records.append(record) + return headers, records + + +def validate_csv(headers, fields, required_fields): + """ + Validate that parsed csv data conforms to the object's available fields. Raise validation errors + if parsed csv data contains invalid headers or does not contain required headers. + """ + # Validate provided column headers + for field, to_field in headers.items(): + if field not in fields: + raise forms.ValidationError(f'Unexpected column header "{field}" found.') + if to_field and not hasattr(fields[field], 'to_field_name'): + raise forms.ValidationError(f'Column "{field}" is not a related object; cannot use dots') + if to_field and not hasattr(fields[field].queryset.model, to_field): + raise forms.ValidationError(f'Invalid related object attribute for column "{field}": {to_field}') + + # Validate required fields + for f in required_fields: + if f not in headers: + raise forms.ValidationError(f'Required column header "{f}" not found.') From 34dab01072c0831f37a7bd8c8fc5d86f47de4526 Mon Sep 17 00:00:00 2001 From: Alyssa Bigley Date: Mon, 14 Jun 2021 15:23:42 -0400 Subject: [PATCH 11/48] cleaned up validation error method --- netbox/netbox/views/generic.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/netbox/netbox/views/generic.py b/netbox/netbox/views/generic.py index 00841f4b9..7616a8c06 100644 --- a/netbox/netbox/views/generic.py +++ b/netbox/netbox/views/generic.py @@ -672,9 +672,7 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): ) def used_both_csv_fields(self): - if self.cleaned_data['csv_file'][1] and self.cleaned_data['csv'][1]: - return True - return False + return self.cleaned_data['csv_file'][1] and self.cleaned_data['csv'][1] return ImportForm(*args, **kwargs) From 9b2534b6a2c9e080117ae2284eb3deffd13abbe2 Mon Sep 17 00:00:00 2001 From: Hans Erasmus Date: Tue, 22 Jun 2021 09:59:01 +0200 Subject: [PATCH 12/48] Update installation Just separated it so the user can easily click the copy button, and only be presented with the command. --- docs/installation/3-netbox.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/installation/3-netbox.md b/docs/installation/3-netbox.md index 7a8e0bc80..958cd4e46 100644 --- a/docs/installation/3-netbox.md +++ b/docs/installation/3-netbox.md @@ -73,6 +73,9 @@ Next, clone the **master** branch of the NetBox GitHub repository into the curre ```no-highlight $ sudo git clone -b master https://github.com/netbox-community/netbox.git . +``` +The screen below should be the result: +``` Cloning into '.'... remote: Counting objects: 1994, done. remote: Compressing objects: 100% (150/150), done. From 4c586ad2ba586b1ccc9ba7898906ec522ed9857e Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Thu, 8 Jul 2021 09:21:35 -0400 Subject: [PATCH 13/48] PRVB --- netbox/netbox/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 2db448360..7d4b570c2 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -16,7 +16,7 @@ from django.core.validators import URLValidator # Environment setup # -VERSION = '2.11.9' +VERSION = '2.11.10-dev' # Hostname HOSTNAME = platform.node() From 911cbf2b183ab12f7fc1d72d5b6bc37fa4fa873a Mon Sep 17 00:00:00 2001 From: Tobias Genannt Date: Thu, 10 Jun 2021 08:02:13 +0200 Subject: [PATCH 14/48] Fixes #5442: Use LDAP groups to find permissions When AUTH_LDAP_FIND_GROUP_PERMS is set to true the filter to find the users permissions is extended to search for all permissions assigned to groups in which the LDAP user is. --- netbox/netbox/authentication.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/netbox/netbox/authentication.py b/netbox/netbox/authentication.py index 0eee2c13e..e696333ab 100644 --- a/netbox/netbox/authentication.py +++ b/netbox/netbox/authentication.py @@ -11,7 +11,7 @@ from users.models import ObjectPermission from utilities.permissions import permission_is_exempt, resolve_permission, resolve_permission_ct -class ObjectPermissionBackend(ModelBackend): +class ObjectPermissionMixin(): def get_all_permissions(self, user_obj, obj=None): if not user_obj.is_active or user_obj.is_anonymous: @@ -20,13 +20,16 @@ class ObjectPermissionBackend(ModelBackend): user_obj._object_perm_cache = self.get_object_permissions(user_obj) return user_obj._object_perm_cache + def get_permission_filter(self, user_obj): + return Q(users=user_obj) | Q(groups__user=user_obj) + def get_object_permissions(self, user_obj): """ Return all permissions granted to the user by an ObjectPermission. """ # Retrieve all assigned and enabled ObjectPermissions object_permissions = ObjectPermission.objects.filter( - Q(users=user_obj) | Q(groups__user=user_obj), + self.get_permission_filter(user_obj), enabled=True ).prefetch_related('object_types') @@ -86,6 +89,10 @@ class ObjectPermissionBackend(ModelBackend): return model.objects.filter(constraints, pk=obj.pk).exists() +class ObjectPermissionBackend(ObjectPermissionMixin, ModelBackend): + pass + + class RemoteUserBackend(_RemoteUserBackend): """ Custom implementation of Django's RemoteUserBackend which provides configuration hooks for basic customization. @@ -163,8 +170,15 @@ class LDAPBackend: "Required parameter AUTH_LDAP_SERVER_URI is missing from ldap_config.py." ) - # Create a new instance of django-auth-ldap's LDAPBackend - obj = LDAPBackend_() + # Create a new instance of django-auth-ldap's LDAPBackend with our own ObjectPermissions + class NBLDAPBackend(ObjectPermissionMixin, LDAPBackend_): + def get_permission_filter(self, user_obj): + permission_filter = Q(users=user_obj) | Q(groups__user=user_obj) + if self.settings.FIND_GROUP_PERMS: + permission_filter = permission_filter | Q(groups__name__in=user_obj.ldap_user.group_names) + return permission_filter + + obj = NBLDAPBackend() # Read LDAP configuration parameters from ldap_config.py instead of settings.py settings = LDAPSettings() From f55cf993dda9860d387e819779e079374a43297b Mon Sep 17 00:00:00 2001 From: Tobias Genannt Date: Thu, 10 Jun 2021 16:13:43 +0200 Subject: [PATCH 15/48] Use method from parent class Co-authored-by: Jeremy Stretch --- netbox/netbox/authentication.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/netbox/authentication.py b/netbox/netbox/authentication.py index e696333ab..b20091d53 100644 --- a/netbox/netbox/authentication.py +++ b/netbox/netbox/authentication.py @@ -173,7 +173,7 @@ class LDAPBackend: # Create a new instance of django-auth-ldap's LDAPBackend with our own ObjectPermissions class NBLDAPBackend(ObjectPermissionMixin, LDAPBackend_): def get_permission_filter(self, user_obj): - permission_filter = Q(users=user_obj) | Q(groups__user=user_obj) + permission_filter = super().get_permission_filter(user_obj) if self.settings.FIND_GROUP_PERMS: permission_filter = permission_filter | Q(groups__name__in=user_obj.ldap_user.group_names) return permission_filter From fa94b80a13bb70a5dd56129aa8894d9c37938b15 Mon Sep 17 00:00:00 2001 From: Tobias Genannt Date: Tue, 15 Jun 2021 08:49:41 +0200 Subject: [PATCH 16/48] Fix error when running scripts This fixes the error Can't pickle local object 'LDAPBackend.__new__..NBLDAPBackend' --- netbox/netbox/authentication.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/netbox/netbox/authentication.py b/netbox/netbox/authentication.py index b20091d53..03241e522 100644 --- a/netbox/netbox/authentication.py +++ b/netbox/netbox/authentication.py @@ -140,11 +140,25 @@ class RemoteUserBackend(_RemoteUserBackend): return False +# Create a new instance of django-auth-ldap's LDAPBackend with our own ObjectPermissions +try: + from django_auth_ldap.backend import LDAPBackend as LDAPBackend_ + + class NBLDAPBackend(ObjectPermissionMixin, LDAPBackend_): + def get_permission_filter(self, user_obj): + permission_filter = super().get_permission_filter(user_obj) + if self.settings.FIND_GROUP_PERMS: + permission_filter = permission_filter | Q(groups__name__in=user_obj.ldap_user.group_names) + return permission_filter +except ModuleNotFoundError: + pass + + class LDAPBackend: def __new__(cls, *args, **kwargs): try: - from django_auth_ldap.backend import LDAPBackend as LDAPBackend_, LDAPSettings + from django_auth_ldap.backend import LDAPSettings import ldap except ModuleNotFoundError as e: if getattr(e, 'name') == 'django_auth_ldap': @@ -170,14 +184,6 @@ class LDAPBackend: "Required parameter AUTH_LDAP_SERVER_URI is missing from ldap_config.py." ) - # Create a new instance of django-auth-ldap's LDAPBackend with our own ObjectPermissions - class NBLDAPBackend(ObjectPermissionMixin, LDAPBackend_): - def get_permission_filter(self, user_obj): - permission_filter = super().get_permission_filter(user_obj) - if self.settings.FIND_GROUP_PERMS: - permission_filter = permission_filter | Q(groups__name__in=user_obj.ldap_user.group_names) - return permission_filter - obj = NBLDAPBackend() # Read LDAP configuration parameters from ldap_config.py instead of settings.py From c0ddb2092c141952efabd9fe2d1e7ffcd66da695 Mon Sep 17 00:00:00 2001 From: Tobias Genannt Date: Fri, 2 Jul 2021 07:55:13 +0200 Subject: [PATCH 17/48] Fixed bug for users authenticated with API token This prevents a crash when the current user has authenticated himself with an API token. In this case the user will not have the permissions given to his LDAP groups. --- netbox/netbox/authentication.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/netbox/netbox/authentication.py b/netbox/netbox/authentication.py index 03241e522..2c843f076 100644 --- a/netbox/netbox/authentication.py +++ b/netbox/netbox/authentication.py @@ -147,7 +147,9 @@ try: class NBLDAPBackend(ObjectPermissionMixin, LDAPBackend_): def get_permission_filter(self, user_obj): permission_filter = super().get_permission_filter(user_obj) - if self.settings.FIND_GROUP_PERMS: + if (self.settings.FIND_GROUP_PERMS and + hasattr(user_obj, "ldap_user") and + hasattr(user_obj.ldap_user, "group_names")): permission_filter = permission_filter | Q(groups__name__in=user_obj.ldap_user.group_names) return permission_filter except ModuleNotFoundError: From 3b28ef7680a002742fc2c5223702360f62f82500 Mon Sep 17 00:00:00 2001 From: Tobias Genannt Date: Mon, 5 Jul 2021 12:31:52 +0200 Subject: [PATCH 18/48] Load LDAP groups for API token authenticated users When users are authenticated with an API token not all permissions where assigned to the session because the LDAP group memberships where not available. Now the information is loaded from the directory if the user is found. If not the local group memberships are used. --- netbox/netbox/api/authentication.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/netbox/netbox/api/authentication.py b/netbox/netbox/api/authentication.py index 1cb32c1e4..76bb0f983 100644 --- a/netbox/netbox/api/authentication.py +++ b/netbox/netbox/api/authentication.py @@ -25,6 +25,16 @@ class TokenAuthentication(authentication.TokenAuthentication): if not token.user.is_active: raise exceptions.AuthenticationFailed("User inactive") + # When LDAP authentication is active try to load user data from LDAP directory + if (settings.REMOTE_AUTH_ENABLED and + settings.REMOTE_AUTH_BACKEND == 'netbox.authentication.LDAPBackend'): + from netbox.authentication import LDAPBackend + ldap_backend = LDAPBackend() + user = ldap_backend.populate_user(token.user.username) + # If the user is found in the LDAP directory use it, if not fallback to the local user + if user: + return user, token + return token.user, token From 15bc6c74b205facef35a2801a7993c16ee26eed6 Mon Sep 17 00:00:00 2001 From: Tobias Genannt Date: Fri, 9 Jul 2021 08:13:02 +0200 Subject: [PATCH 19/48] Only check REMOTE_AUTH_BACKEND in API token auth --- netbox/netbox/api/authentication.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/netbox/netbox/api/authentication.py b/netbox/netbox/api/authentication.py index 76bb0f983..7f8bee318 100644 --- a/netbox/netbox/api/authentication.py +++ b/netbox/netbox/api/authentication.py @@ -26,8 +26,7 @@ class TokenAuthentication(authentication.TokenAuthentication): raise exceptions.AuthenticationFailed("User inactive") # When LDAP authentication is active try to load user data from LDAP directory - if (settings.REMOTE_AUTH_ENABLED and - settings.REMOTE_AUTH_BACKEND == 'netbox.authentication.LDAPBackend'): + if settings.REMOTE_AUTH_BACKEND == 'netbox.authentication.LDAPBackend': from netbox.authentication import LDAPBackend ldap_backend = LDAPBackend() user = ldap_backend.populate_user(token.user.username) From 239a661a2a078c855afaab25f20a907d16631c3f Mon Sep 17 00:00:00 2001 From: Hans Erasmus Date: Fri, 9 Jul 2021 11:43:50 +0200 Subject: [PATCH 20/48] Update 3-netbox.md --- docs/installation/3-netbox.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/installation/3-netbox.md b/docs/installation/3-netbox.md index 958cd4e46..2a493d835 100644 --- a/docs/installation/3-netbox.md +++ b/docs/installation/3-netbox.md @@ -74,7 +74,9 @@ Next, clone the **master** branch of the NetBox GitHub repository into the curre ```no-highlight $ sudo git clone -b master https://github.com/netbox-community/netbox.git . ``` + The screen below should be the result: + ``` Cloning into '.'... remote: Counting objects: 1994, done. From 63b325b0f6c02497f210bd6f642191c5c3cec60f Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Mon, 12 Jul 2021 09:31:19 -0400 Subject: [PATCH 21/48] Changelog for #5442 --- docs/release-notes/version-2.11.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/release-notes/version-2.11.md b/docs/release-notes/version-2.11.md index 1ff9a8483..3c4b071bf 100644 --- a/docs/release-notes/version-2.11.md +++ b/docs/release-notes/version-2.11.md @@ -1,5 +1,13 @@ # NetBox v2.11 +## v2.11.10 (FUTURE) + +### Bug Fixes + +* [#5442](https://github.com/netbox-community/netbox/issues/5442) - Fix assignment of permissions based on LDAP groups + +--- + ## v2.11.9 (2021-07-08) ### Bug Fixes From ccc3eee6e7d754554500fcad8580c2636c606a94 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Wed, 14 Jul 2021 10:23:31 -0400 Subject: [PATCH 22/48] Updated issue staling timers --- CONTRIBUTING.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5707f4ad2..7a3b1f002 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -160,17 +160,20 @@ accumulating a large backlog of work. The core maintainers group has chosen to make use of GitHub's [Stale bot](https://github.com/apps/stale) to aid in issue management. -* Issues will be marked as stale after 45 days of no activity. -* Then after 15 more days of inactivity, the issue will be closed. +* Issues will be marked as stale after 60 days of no activity. +* If the stable label is not removed in the following 30 days, the issue will + be closed automatically. * Any issue bearing one of the following labels will be exempt from all Stale bot actions: * `status: accepted` * `status: blocked` * `status: needs milestone` -It is natural that some new issues get more attention than others. Stale bot -helps bring renewed attention to potentially valuable issues that may have been -overlooked. +It is natural that some new issues get more attention than others. The stale +bot helps bring renewed attention to potentially valuable issues that may have +been overlooked. **Do not** comment on an issue that has been marked stale in +an effort to circumvent the bot: Doing so will not remove the stale label. +(Stale labels can be removed only by maintainers.) ## Maintainer Guidance From 904be9f9e953374543483806f42fb958c4115133 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Wed, 14 Jul 2021 10:43:18 -0400 Subject: [PATCH 23/48] Closes #6753: Add plugin removal instructions to the docs --- docs/plugins/index.md | 55 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/docs/plugins/index.md b/docs/plugins/index.md index 202e0a96b..c2d62330f 100644 --- a/docs/plugins/index.md +++ b/docs/plugins/index.md @@ -89,3 +89,58 @@ Restart the WSGI service to load the new plugin: ```no-highlight # sudo systemctl restart netbox ``` + +## Removing Plugins + +Follow these steps to completely remove a plugin. + +### Update Configuration + +Remove the plugin from the `PLUGINS` list in `configuration.py`. Also remove any relevant configuration parameters from `PLUGINS_CONFIG`. + +### Remove the Python Package + +Use `pip` to remove the installed plugin: + +```no-highlight +$ source /opt/netbox/venv/bin/activate +(venv) $ pip uninstall +``` + +### Restart WSGI Service + +Restart the WSGI service: + +```no-highlight +# sudo systemctl restart netbox +``` + +### Drop Database Tables + +!!! note + This step is necessary only for plugin which have created one or more database tables (generally through the introduction of new models). Check your plugin's documentation if unsure. + +Enter the PostgreSQL database shell to determine if the plugin has created any SQL tables. Substitute `pluginname` in the example below for the name of the plugin being removed. (You can also run the `\dt` command without a pattern to list _all_ tables.) + +```no-highlight +netbox=> \dt pluginname_* + List of relations + List of relations + Schema | Name | Type | Owner +--------+----------------+-------+-------- + public | pluginname_foo | table | netbox + public | pluginname_bar | table | netbox +(2 rows) +``` + +!!! warning + Exercise extreme caution when removing tables. Users are strongly encouraged to perform a backup of their database immediately before taking these actions. + +Drop each of the listed tables to remove it from the database: + +```no-highlight +netbox=> DROP TABLE pluginname_foo; +DROP TABLE +netbox=> DROP TABLE pluginname_bar; +DROP TABLE +``` From 5d625867055d0216cca303269ac3246639c1b6a0 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Tue, 20 Jul 2021 17:00:13 -0400 Subject: [PATCH 24/48] Fixes #6773: Add missing display field to rack unit serializer --- docs/release-notes/version-2.11.md | 1 + netbox/dcim/api/serializers.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/docs/release-notes/version-2.11.md b/docs/release-notes/version-2.11.md index 3c4b071bf..b8cd02ea7 100644 --- a/docs/release-notes/version-2.11.md +++ b/docs/release-notes/version-2.11.md @@ -5,6 +5,7 @@ ### Bug Fixes * [#5442](https://github.com/netbox-community/netbox/issues/5442) - Fix assignment of permissions based on LDAP groups +* [#6773](https://github.com/netbox-community/netbox/issues/6773) - Add missing `display` field to rack unit serializer --- diff --git a/netbox/dcim/api/serializers.py b/netbox/dcim/api/serializers.py index d9b36e9f2..c9d69fd00 100644 --- a/netbox/dcim/api/serializers.py +++ b/netbox/dcim/api/serializers.py @@ -210,6 +210,10 @@ class RackUnitSerializer(serializers.Serializer): face = ChoiceField(choices=DeviceFaceChoices, read_only=True) device = NestedDeviceSerializer(read_only=True) occupied = serializers.BooleanField(read_only=True) + display = serializers.SerializerMethodField(read_only=True) + + def get_display(self, obj): + return obj['name'] class RackReservationSerializer(PrimaryModelSerializer): From 79f3eb4fb72b07561498274f7cdaeb892492dcc6 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Wed, 21 Jul 2021 15:44:14 -0400 Subject: [PATCH 25/48] Fixes #6778: Rack reservation should display rack's location --- docs/release-notes/version-2.11.md | 1 + netbox/templates/dcim/rackreservation.html | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/release-notes/version-2.11.md b/docs/release-notes/version-2.11.md index b8cd02ea7..0afe81ffd 100644 --- a/docs/release-notes/version-2.11.md +++ b/docs/release-notes/version-2.11.md @@ -6,6 +6,7 @@ * [#5442](https://github.com/netbox-community/netbox/issues/5442) - Fix assignment of permissions based on LDAP groups * [#6773](https://github.com/netbox-community/netbox/issues/6773) - Add missing `display` field to rack unit serializer +* [#6778](https://github.com/netbox-community/netbox/issues/6778) - Rack reservation should display rack's location --- diff --git a/netbox/templates/dcim/rackreservation.html b/netbox/templates/dcim/rackreservation.html index 06e14e7db..8d4fa735c 100644 --- a/netbox/templates/dcim/rackreservation.html +++ b/netbox/templates/dcim/rackreservation.html @@ -39,10 +39,10 @@ - Group + Location - {% if rack.group %} - {{ rack.group }} + {% if rack.location %} + {{ rack.location }} {% else %} None {% endif %} From 9df10983373d24cc8e26cacf653fd51247f0a8f8 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Wed, 21 Jul 2021 15:49:01 -0400 Subject: [PATCH 26/48] Fixes #6780: Include rack location in navigation breadcrumbs --- docs/release-notes/version-2.11.md | 1 + netbox/templates/dcim/rack.html | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/release-notes/version-2.11.md b/docs/release-notes/version-2.11.md index 0afe81ffd..31607eb45 100644 --- a/docs/release-notes/version-2.11.md +++ b/docs/release-notes/version-2.11.md @@ -7,6 +7,7 @@ * [#5442](https://github.com/netbox-community/netbox/issues/5442) - Fix assignment of permissions based on LDAP groups * [#6773](https://github.com/netbox-community/netbox/issues/6773) - Add missing `display` field to rack unit serializer * [#6778](https://github.com/netbox-community/netbox/issues/6778) - Rack reservation should display rack's location +* [#6780](https://github.com/netbox-community/netbox/issues/6780) - Include rack location in navigation breadcrumbs --- diff --git a/netbox/templates/dcim/rack.html b/netbox/templates/dcim/rack.html index ada93518f..94f0ea24c 100644 --- a/netbox/templates/dcim/rack.html +++ b/netbox/templates/dcim/rack.html @@ -9,11 +9,11 @@ {% block breadcrumbs %}
  • Racks
  • {{ object.site }}
  • - {% if object.group %} - {% for group in object.group.get_ancestors %} -
  • {{ group }}
  • + {% if object.location %} + {% for location in object.location.get_ancestors %} +
  • {{ location }}
  • {% endfor %} -
  • {{ object.group }}
  • +
  • {{ object.location }}
  • {% endif %}
  • {{ object }}
  • {% endblock %} From d7a6947975c7631e593c63e5f280acfef2f2ddf7 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Wed, 21 Jul 2021 16:02:32 -0400 Subject: [PATCH 27/48] Fixes #6777: Fix default value validation for custom text fields --- docs/release-notes/version-2.11.md | 1 + netbox/extras/models/customfields.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/release-notes/version-2.11.md b/docs/release-notes/version-2.11.md index 31607eb45..953f42d5c 100644 --- a/docs/release-notes/version-2.11.md +++ b/docs/release-notes/version-2.11.md @@ -6,6 +6,7 @@ * [#5442](https://github.com/netbox-community/netbox/issues/5442) - Fix assignment of permissions based on LDAP groups * [#6773](https://github.com/netbox-community/netbox/issues/6773) - Add missing `display` field to rack unit serializer +* [#6777](https://github.com/netbox-community/netbox/issues/6777) - Fix default value validation for custom text fields * [#6778](https://github.com/netbox-community/netbox/issues/6778) - Rack reservation should display rack's location * [#6780](https://github.com/netbox-community/netbox/issues/6780) - Include rack location in navigation breadcrumbs diff --git a/netbox/extras/models/customfields.py b/netbox/extras/models/customfields.py index 60c6adce9..a433a3f81 100644 --- a/netbox/extras/models/customfields.py +++ b/netbox/extras/models/customfields.py @@ -149,7 +149,8 @@ class CustomField(BigIDModel): # Validate the field's default value (if any) if self.default is not None: try: - self.validate(self.default) + default_value = str(self.default) if self.type == CustomFieldTypeChoices.TYPE_TEXT else self.default + self.validate(default_value) except ValidationError as err: raise ValidationError({ 'default': f'Invalid default value "{self.default}": {err.message}' From e8d7f55b6673a0919c7fb0d540423cdc39644c63 Mon Sep 17 00:00:00 2001 From: Brian Ellwood Date: Thu, 22 Jul 2021 19:04:34 -0400 Subject: [PATCH 28/48] Add AC Hardwire option to PowerPortTypeChoices Resolves FR #6785 --- netbox/dcim/choices.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/netbox/dcim/choices.py b/netbox/dcim/choices.py index 63f44ea37..06d800623 100644 --- a/netbox/dcim/choices.py +++ b/netbox/dcim/choices.py @@ -341,6 +341,8 @@ class PowerPortTypeChoices(ChoiceSet): TYPE_DC = 'dc-terminal' # Proprietary TYPE_SAF_D_GRID = 'saf-d-grid' + # Other + TYPE_OTHER = 'other' CHOICES = ( ('IEC 60320', ( @@ -447,6 +449,9 @@ class PowerPortTypeChoices(ChoiceSet): ('Proprietary', ( (TYPE_SAF_D_GRID, 'Saf-D-Grid'), )), + ('Other', ( + (TYPE_OTHER, 'Other'), + )), ) From faa406433189c9ec6fdd5decb126c3e0d13a01c2 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Fri, 23 Jul 2021 11:13:21 -0400 Subject: [PATCH 29/48] Fixes #6774: Fix A/Z assignment when swapping circuit terminations --- docs/release-notes/version-2.11.md | 1 + netbox/circuits/views.py | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/release-notes/version-2.11.md b/docs/release-notes/version-2.11.md index 953f42d5c..2e5209fb0 100644 --- a/docs/release-notes/version-2.11.md +++ b/docs/release-notes/version-2.11.md @@ -6,6 +6,7 @@ * [#5442](https://github.com/netbox-community/netbox/issues/5442) - Fix assignment of permissions based on LDAP groups * [#6773](https://github.com/netbox-community/netbox/issues/6773) - Add missing `display` field to rack unit serializer +* [#6774](https://github.com/netbox-community/netbox/issues/6774) - Fix A/Z assignment when swapping circuit terminations * [#6777](https://github.com/netbox-community/netbox/issues/6777) - Fix default value validation for custom text fields * [#6778](https://github.com/netbox-community/netbox/issues/6778) - Rack reservation should display rack's location * [#6780](https://github.com/netbox-community/netbox/issues/6780) - Include rack location in navigation breadcrumbs diff --git a/netbox/circuits/views.py b/netbox/circuits/views.py index 15e6bed2f..e7bb889e0 100644 --- a/netbox/circuits/views.py +++ b/netbox/circuits/views.py @@ -287,6 +287,10 @@ class CircuitSwapTerminations(generic.ObjectEditView): termination_z.save() termination_a.term_side = 'Z' termination_a.save() + circuit.refresh_from_db() + circuit.termination_a = termination_z + circuit.termination_z = termination_a + circuit.save() elif termination_a: termination_a.term_side = 'Z' termination_a.save() @@ -300,9 +304,6 @@ class CircuitSwapTerminations(generic.ObjectEditView): circuit.termination_z = None circuit.save() - print(f'term A: {circuit.termination_a}') - print(f'term Z: {circuit.termination_z}') - messages.success(request, f"Swapped terminations for circuit {circuit}.") return redirect('circuits:circuit', pk=circuit.pk) From c0f37f0b0305cfa67acf878b19b7b75f1a4b5dae Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Fri, 23 Jul 2021 11:18:50 -0400 Subject: [PATCH 30/48] Fixes #6794: Fix device name display on device status view --- docs/release-notes/version-2.11.md | 1 + netbox/templates/dcim/device/status.html | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/release-notes/version-2.11.md b/docs/release-notes/version-2.11.md index 2e5209fb0..c2883a2a8 100644 --- a/docs/release-notes/version-2.11.md +++ b/docs/release-notes/version-2.11.md @@ -10,6 +10,7 @@ * [#6777](https://github.com/netbox-community/netbox/issues/6777) - Fix default value validation for custom text fields * [#6778](https://github.com/netbox-community/netbox/issues/6778) - Rack reservation should display rack's location * [#6780](https://github.com/netbox-community/netbox/issues/6780) - Include rack location in navigation breadcrumbs +* [#6794](https://github.com/netbox-community/netbox/issues/6794) - Fix device name display on device status view --- diff --git a/netbox/templates/dcim/device/status.html b/netbox/templates/dcim/device/status.html index b7f8d0eaa..594fb0784 100644 --- a/netbox/templates/dcim/device/status.html +++ b/netbox/templates/dcim/device/status.html @@ -1,7 +1,7 @@ {% extends 'dcim/device/base.html' %} {% load static %} -{% block title %}{{ device }} - Status{% endblock %} +{% block title %}{{ object }} - Status{% endblock %} {% block content %} {% include 'inc/ajax_loader.html' %} From 8db603309468148ab6462a839202e972458162fb Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Fri, 23 Jul 2021 11:24:32 -0400 Subject: [PATCH 31/48] Fixes #6759: Fix assignment of parent interfaces for bulk import --- docs/release-notes/version-2.11.md | 1 + netbox/dcim/forms.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/docs/release-notes/version-2.11.md b/docs/release-notes/version-2.11.md index c2883a2a8..72cb90590 100644 --- a/docs/release-notes/version-2.11.md +++ b/docs/release-notes/version-2.11.md @@ -5,6 +5,7 @@ ### Bug Fixes * [#5442](https://github.com/netbox-community/netbox/issues/5442) - Fix assignment of permissions based on LDAP groups +* [#6759](https://github.com/netbox-community/netbox/issues/6759) - Fix assignment of parent interfaces for bulk import * [#6773](https://github.com/netbox-community/netbox/issues/6773) - Add missing `display` field to rack unit serializer * [#6774](https://github.com/netbox-community/netbox/issues/6774) - Fix A/Z assignment when swapping circuit terminations * [#6777](https://github.com/netbox-community/netbox/issues/6777) - Fix default value validation for custom text fields diff --git a/netbox/dcim/forms.py b/netbox/dcim/forms.py index 5bcf135a1..f2d005ed6 100644 --- a/netbox/dcim/forms.py +++ b/netbox/dcim/forms.py @@ -3378,13 +3378,18 @@ class InterfaceCSVForm(CustomFieldModelCSVForm): Q(device=device) | Q(device__virtual_chassis=device.virtual_chassis), type=InterfaceTypeChoices.TYPE_LAG ) + self.fields['parent'].queryset = Interface.objects.filter( + Q(device=device) | Q(device__virtual_chassis=device.virtual_chassis) + ) elif device: self.fields['lag'].queryset = Interface.objects.filter( device=device, type=InterfaceTypeChoices.TYPE_LAG ) + self.fields['parent'].queryset = Interface.objects.filter(device=device) else: self.fields['lag'].queryset = Interface.objects.none() + self.fields['parent'].queryset = Interface.objects.none() def clean_enabled(self): # Make sure enabled is True when it's not included in the uploaded data From f2a440d0ae61bd189ec90c48dac463b10a7bf1c9 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Fri, 23 Jul 2021 11:34:24 -0400 Subject: [PATCH 32/48] Closes #6781: Disable database query caching by default --- docs/additional-features/caching.md | 5 ++++- docs/configuration/optional-settings.md | 4 ++-- docs/release-notes/version-2.11.md | 4 ++++ netbox/netbox/configuration.example.py | 4 ++-- netbox/netbox/settings.py | 10 ++-------- 5 files changed, 14 insertions(+), 13 deletions(-) diff --git a/docs/additional-features/caching.md b/docs/additional-features/caching.md index 18c9dca68..ebe91f37d 100644 --- a/docs/additional-features/caching.md +++ b/docs/additional-features/caching.md @@ -1,6 +1,9 @@ # Caching -NetBox supports database query caching using [django-cacheops](https://github.com/Suor/django-cacheops) and Redis. When a query is made, the results are cached in Redis for a short period of time, as defined by the [CACHE_TIMEOUT](../configuration/optional-settings.md#cache_timeout) parameter (15 minutes by default). Within that time, all recurrences of that specific query will return the pre-fetched results from the cache. +NetBox supports database query caching using [django-cacheops](https://github.com/Suor/django-cacheops) and Redis. When a query is made, the results are cached in Redis for a short period of time, as defined by the [CACHE_TIMEOUT](../configuration/optional-settings.md#cache_timeout) parameter. Within that time, all recurrences of that specific query will return the pre-fetched results from the cache. + +!!! warning + In NetBox v2.11.10 and later queryset caching is disabled by default, and must be configured. If a change is made to any of the objects returned by the query within that time, or if the timeout expires, the results are automatically invalidated and the next request for those results will be sent to the database. diff --git a/docs/configuration/optional-settings.md b/docs/configuration/optional-settings.md index 927bf9f37..6c62fc6d1 100644 --- a/docs/configuration/optional-settings.md +++ b/docs/configuration/optional-settings.md @@ -54,9 +54,9 @@ BASE_PATH = 'netbox/' ## CACHE_TIMEOUT -Default: 900 +Default: 0 (disabled) -The number of seconds that cache entries will be retained before expiring. +The number of seconds that cached database queries will be retained before expiring. --- diff --git a/docs/release-notes/version-2.11.md b/docs/release-notes/version-2.11.md index 72cb90590..f7eb561b7 100644 --- a/docs/release-notes/version-2.11.md +++ b/docs/release-notes/version-2.11.md @@ -13,6 +13,10 @@ * [#6780](https://github.com/netbox-community/netbox/issues/6780) - Include rack location in navigation breadcrumbs * [#6794](https://github.com/netbox-community/netbox/issues/6794) - Fix device name display on device status view +### Other Changes + +* [#6781](https://github.com/netbox-community/netbox/issues/6781) - Database query caching is now disabled by default + --- ## v2.11.9 (2021-07-08) diff --git a/netbox/netbox/configuration.example.py b/netbox/netbox/configuration.example.py index 461d7f4cd..a5c5521f3 100644 --- a/netbox/netbox/configuration.example.py +++ b/netbox/netbox/configuration.example.py @@ -89,8 +89,8 @@ BANNER_LOGIN = '' # BASE_PATH = 'netbox/' BASE_PATH = '' -# Cache timeout in seconds. Set to 0 to dissable caching. Defaults to 900 (15 minutes) -CACHE_TIMEOUT = 900 +# Cache timeout in seconds. Defaults to zero (disabled). +CACHE_TIMEOUT = 0 # Maximum number of days to retain logged changes. Set to 0 to retain changes indefinitely. (Default: 90) CHANGELOG_RETENTION = 90 diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 7d4b570c2..dff38c530 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -75,7 +75,7 @@ BANNER_TOP = getattr(configuration, 'BANNER_TOP', '') BASE_PATH = getattr(configuration, 'BASE_PATH', '') if BASE_PATH: BASE_PATH = BASE_PATH.strip('/') + '/' # Enforce trailing slash only -CACHE_TIMEOUT = getattr(configuration, 'CACHE_TIMEOUT', 900) +CACHE_TIMEOUT = getattr(configuration, 'CACHE_TIMEOUT', 0) CHANGELOG_RETENTION = getattr(configuration, 'CHANGELOG_RETENTION', 90) CORS_ORIGIN_ALLOW_ALL = getattr(configuration, 'CORS_ORIGIN_ALLOW_ALL', False) CORS_ORIGIN_REGEX_WHITELIST = getattr(configuration, 'CORS_ORIGIN_REGEX_WHITELIST', []) @@ -417,13 +417,7 @@ else: 'ssl': CACHING_REDIS_SSL, 'ssl_cert_reqs': None if CACHING_REDIS_SKIP_TLS_VERIFY else 'required', } - -if not CACHE_TIMEOUT: - CACHEOPS_ENABLED = False -else: - CACHEOPS_ENABLED = True - - +CACHEOPS_ENABLED = bool(CACHE_TIMEOUT) CACHEOPS_DEFAULTS = { 'timeout': CACHE_TIMEOUT } From 67157724f48bdfa5d31a58988cbce2b4f6007530 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Fri, 23 Jul 2021 13:43:33 -0400 Subject: [PATCH 33/48] Introduce "adding models" section to development docs --- docs/development/adding-models.md | 85 +++++++++++++++++++++++++++++++ mkdocs.yml | 1 + 2 files changed, 86 insertions(+) create mode 100644 docs/development/adding-models.md diff --git a/docs/development/adding-models.md b/docs/development/adding-models.md new file mode 100644 index 000000000..6b778d886 --- /dev/null +++ b/docs/development/adding-models.md @@ -0,0 +1,85 @@ +# Adding Models + +## 1. Define the model class + +Models within each app are stored in either `models.py` or within a submodule under the `models/` directory. When creating a model, be sure to subclass the [appropriate base model](models.md) from `netbox.models`. This will typically be PrimaryModel or OrganizationalModel. Remember to add the model class to the `__all__` listing for the module. + +Each model should define, at a minimum: + +* A `__str__()` method returning a user-friendly string representation of the instance +* A `get_absolute_url()` method returning an instance's direct URL (using `reverse()`) +* A `Meta` class specifying a deterministic ordering (if ordered by fields other than the primary ID) + +## 2. Define field choices + +If the model has one or more fields with static choices, define those choices in `choices.py` by subclassing `utilities.choices.ChoiceSet`. + +## 3. Generate database migrations + +Once your model definition is complete, generate database migrations by running `manage.py -n $NAME --no-header`. Always specify a short unique name when generating migrations. + +!!! info + Set `DEVELOPER = True` in your NetBox configuration to enable the creation of new migrations. + +## 4. Add all standard views + +Most models will need view classes created in `views.py` to serve the following operations: + +* List view +* Detail view +* Edit view +* Delete view +* Bulk import +* Bulk edit +* Bulk delete + +## 5. Add URL paths + +Add the relevant URL path for each view created in the previous step to `urls.py`. + +## 6. Create the FilterSet + +Each model should have a corresponding FilterSet class defined. This is used to filter UI and API queries. Subclass the appropriate class from `netbox.filtersets` that matches the model's parent class. + +Every model FilterSet should define a `q` filter to support general search queries. + +## 7. Create the table + +Create a table class for the model in `tables.py` by subclassing `utilities.tables.BaseTable`. Under the table's `Meta` class, be sure to list both the fields and default columns. + +## 8. Create the object template + +Create the HTML template for the object view. (The other views each typically employ a generic template.) This template should extend `generic/object.html`. + +## 9. Add the model to the navigation menu + +For NetBox releases prior to v3.0, add the relevant link(s) to the navigation menu template. For later releases, add the relevant items in `netbox/netbox/navigation_menu.py`. + +## 10. REST API components + +Create the following for each model: + +* Detailed (full) model serializer in `api/serializers.py` +* Nested serializer in `api/nested_serializers.py` +* API view in `api/views.py` +* Endpoint route in `api/urls.py` + +## 11. GraphQL API components (v3.0+) + +Create a Graphene object type for the model in `graphql/types.py` by subclassing the appropriate class from `netbox.graphql.types`. + +Also extend the schema class defined in `graphql/schema.py` with the individual object and object list fields per the established convention. + +## 12. Add tests + +Add tests for the following: + +* UI views +* API views +* Filter sets + +## 13. Documentation + +Create a new documentation page for the model in `docs/models//.md`. Include this file under the "features" documentation where appropriate. + +Also add your model to the index in `docs/development/models.md`. diff --git a/mkdocs.yml b/mkdocs.yml index fb5cf1890..d26799caf 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -76,6 +76,7 @@ nav: - Getting Started: 'development/getting-started.md' - Style Guide: 'development/style-guide.md' - Models: 'development/models.md' + - Adding Models: 'development/adding-models.md' - Extending Models: 'development/extending-models.md' - Application Registry: 'development/application-registry.md' - User Preferences: 'development/user-preferences.md' From 0f066c1c0a1a265b3dc26ce07f174befceebd4ef Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Fri, 23 Jul 2021 13:45:56 -0400 Subject: [PATCH 34/48] Exclude NPM files from git (v3.0+) --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 95e4ff702..1dea89c21 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ *.swp /netbox/netbox/configuration.py /netbox/netbox/ldap_config.py +/netbox/project-static/.cache +/netbox/project-static/node_modules /netbox/reports/* !/netbox/reports/__init__.py /netbox/scripts/* From cf9e437f3188246cbe5487cb5e1f5bb6240c1b1e Mon Sep 17 00:00:00 2001 From: tamaszl <85541319+tamaszl@users.noreply.github.com> Date: Sun, 25 Jul 2021 18:44:21 -0700 Subject: [PATCH 35/48] Update 6-ldap.md - AUTH_LDAP_USER_DN_TEMPLATE to none for windows 2012+ changed When using Windows Server 2012, `AUTH_LDAP_USER_DN_TEMPLATE` should be set to None. to Windows Server 2012+ --- docs/installation/6-ldap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation/6-ldap.md b/docs/installation/6-ldap.md index 27bdb0b40..86114dfb0 100644 --- a/docs/installation/6-ldap.md +++ b/docs/installation/6-ldap.md @@ -74,7 +74,7 @@ STARTTLS can be configured by setting `AUTH_LDAP_START_TLS = True` and using the ### User Authentication !!! info - When using Windows Server 2012, `AUTH_LDAP_USER_DN_TEMPLATE` should be set to None. + When using Windows Server 2012+, `AUTH_LDAP_USER_DN_TEMPLATE` should be set to None. ```python from django_auth_ldap.config import LDAPSearch From 44bdb6cac705c36821d9d268c302b4ff39848eb4 Mon Sep 17 00:00:00 2001 From: bluikko <14869000+bluikko@users.noreply.github.com> Date: Tue, 27 Jul 2021 00:26:46 +0700 Subject: [PATCH 36/48] Add dev server firewall configuration for EL distros (#6772) * Add dev server firewall configuration for EL distros * Fix typo in previous * Indent the firewall block in install docs --- docs/installation/3-netbox.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/installation/3-netbox.md b/docs/installation/3-netbox.md index 2a493d835..e54bf6f3e 100644 --- a/docs/installation/3-netbox.md +++ b/docs/installation/3-netbox.md @@ -267,6 +267,13 @@ Starting development server at http://0.0.0.0:8000/ Quit the server with CONTROL-C. ``` +!!! note + By default RHEL based distros will likely block your testing attempts with firewalld. The development server port can be opened with `firewall-cmd` (add `--permanent` if you want the rule to survive server restarts): + + ```no-highlight + firewall-cmd --zone=public --add-port=8000/tcp + ``` + Next, connect to the name or IP of the server (as defined in `ALLOWED_HOSTS`) on port 8000; for example, . You should be greeted with the NetBox home page. !!! danger From 6867a840b21ea63780ca0807b893c4c086e89962 Mon Sep 17 00:00:00 2001 From: Brian Ellwood Date: Mon, 26 Jul 2021 15:03:43 -0400 Subject: [PATCH 37/48] Update choices.py --- netbox/dcim/choices.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/netbox/dcim/choices.py b/netbox/dcim/choices.py index 06d800623..a53f357f8 100644 --- a/netbox/dcim/choices.py +++ b/netbox/dcim/choices.py @@ -342,7 +342,7 @@ class PowerPortTypeChoices(ChoiceSet): # Proprietary TYPE_SAF_D_GRID = 'saf-d-grid' # Other - TYPE_OTHER = 'other' + TYPE_HARDWIRED = 'hardwired' CHOICES = ( ('IEC 60320', ( @@ -450,7 +450,7 @@ class PowerPortTypeChoices(ChoiceSet): (TYPE_SAF_D_GRID, 'Saf-D-Grid'), )), ('Other', ( - (TYPE_OTHER, 'Other'), + (TYPE_HARDWIRED, 'Hardwired'), )), ) From d79b3cad1b10c24c722e059d81dac96fdc253240 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Tue, 27 Jul 2021 16:04:51 -0400 Subject: [PATCH 38/48] Fixes #6822: Use consistent maximum value for interface MTU --- docs/release-notes/version-2.11.md | 1 + netbox/dcim/constants.py | 2 +- netbox/dcim/forms.py | 12 ++++++------ netbox/dcim/models/device_components.py | 5 ++++- netbox/dcim/tests/test_views.py | 2 +- netbox/virtualization/forms.py | 14 ++++---------- netbox/virtualization/tests/test_views.py | 2 +- 7 files changed, 18 insertions(+), 20 deletions(-) diff --git a/docs/release-notes/version-2.11.md b/docs/release-notes/version-2.11.md index f7eb561b7..4737c460c 100644 --- a/docs/release-notes/version-2.11.md +++ b/docs/release-notes/version-2.11.md @@ -12,6 +12,7 @@ * [#6778](https://github.com/netbox-community/netbox/issues/6778) - Rack reservation should display rack's location * [#6780](https://github.com/netbox-community/netbox/issues/6780) - Include rack location in navigation breadcrumbs * [#6794](https://github.com/netbox-community/netbox/issues/6794) - Fix device name display on device status view +* [#6822](https://github.com/netbox-community/netbox/issues/6822) - Use consistent maximum value for interface MTU ### Other Changes diff --git a/netbox/dcim/constants.py b/netbox/dcim/constants.py index 42ed7c7d0..2a4d368f4 100644 --- a/netbox/dcim/constants.py +++ b/netbox/dcim/constants.py @@ -29,7 +29,7 @@ REARPORT_POSITIONS_MAX = 1024 # INTERFACE_MTU_MIN = 1 -INTERFACE_MTU_MAX = 32767 # Max value of a signed 16-bit integer +INTERFACE_MTU_MAX = 65536 VIRTUAL_IFACE_TYPES = [ InterfaceTypeChoices.TYPE_VIRTUAL, diff --git a/netbox/dcim/forms.py b/netbox/dcim/forms.py index f2d005ed6..565a75c45 100644 --- a/netbox/dcim/forms.py +++ b/netbox/dcim/forms.py @@ -102,6 +102,12 @@ class InterfaceCommonForm(forms.Form): required=False, label='MAC address' ) + mtu = forms.IntegerField( + required=False, + min_value=INTERFACE_MTU_MIN, + max_value=INTERFACE_MTU_MAX, + label='MTU' + ) def clean(self): super().clean() @@ -3173,12 +3179,6 @@ class InterfaceCreateForm(ComponentCreateForm, InterfaceCommonForm): 'type': 'lag', } ) - mtu = forms.IntegerField( - required=False, - min_value=INTERFACE_MTU_MIN, - max_value=INTERFACE_MTU_MAX, - label='MTU' - ) mac_address = forms.CharField( required=False, label='MAC Address' diff --git a/netbox/dcim/models/device_components.py b/netbox/dcim/models/device_components.py index bd7f4ac55..6ac0d7753 100644 --- a/netbox/dcim/models/device_components.py +++ b/netbox/dcim/models/device_components.py @@ -483,7 +483,10 @@ class BaseInterface(models.Model): mtu = models.PositiveIntegerField( blank=True, null=True, - validators=[MinValueValidator(1), MaxValueValidator(65536)], + validators=[ + MinValueValidator(INTERFACE_MTU_MIN), + MaxValueValidator(INTERFACE_MTU_MAX) + ], verbose_name='MTU' ) mode = models.CharField( diff --git a/netbox/dcim/tests/test_views.py b/netbox/dcim/tests/test_views.py index 94cf2c9b3..4942b27c2 100644 --- a/netbox/dcim/tests/test_views.py +++ b/netbox/dcim/tests/test_views.py @@ -1464,7 +1464,7 @@ class InterfaceTestCase(ViewTestCases.DeviceComponentViewTestCase): 'enabled': False, 'lag': interfaces[3].pk, 'mac_address': EUI('01:02:03:04:05:06'), - 'mtu': 2000, + 'mtu': 65000, 'mgmt_only': True, 'description': 'A front port', 'mode': InterfaceModeChoices.MODE_TAGGED, diff --git a/netbox/virtualization/forms.py b/netbox/virtualization/forms.py index f7b241c1a..0819882c3 100644 --- a/netbox/virtualization/forms.py +++ b/netbox/virtualization/forms.py @@ -16,10 +16,10 @@ from ipam.models import IPAddress, VLAN from tenancy.forms import TenancyFilterForm, TenancyForm from tenancy.models import Tenant from utilities.forms import ( - add_blank_choice, BootstrapMixin, BulkEditForm, BulkEditNullBooleanSelect, BulkRenameForm, CommentField, - ConfirmationForm, CSVChoiceField, CSVModelChoiceField, CSVModelForm, DynamicModelChoiceField, - DynamicModelMultipleChoiceField, ExpandableNameField, form_from_model, JSONField, SlugField, SmallTextarea, - StaticSelect2, StaticSelect2Multiple, TagFilterField, BOOLEAN_WITH_BLANK_CHOICES, + add_blank_choice, BootstrapMixin, BulkEditNullBooleanSelect, BulkRenameForm, CommentField, ConfirmationForm, + CSVChoiceField, CSVModelChoiceField, DynamicModelChoiceField, DynamicModelMultipleChoiceField, ExpandableNameField, + form_from_model, JSONField, SlugField, SmallTextarea, StaticSelect2, StaticSelect2Multiple, TagFilterField, + BOOLEAN_WITH_BLANK_CHOICES, ) from .choices import * from .models import Cluster, ClusterGroup, ClusterType, VirtualMachine, VMInterface @@ -680,12 +680,6 @@ class VMInterfaceCreateForm(BootstrapMixin, CustomFieldForm, InterfaceCommonForm 'virtual_machine_id': '$virtual_machine', } ) - mtu = forms.IntegerField( - required=False, - min_value=INTERFACE_MTU_MIN, - max_value=INTERFACE_MTU_MAX, - label='MTU' - ) mac_address = forms.CharField( required=False, label='MAC Address' diff --git a/netbox/virtualization/tests/test_views.py b/netbox/virtualization/tests/test_views.py index 56c9cf280..86be5159f 100644 --- a/netbox/virtualization/tests/test_views.py +++ b/netbox/virtualization/tests/test_views.py @@ -263,7 +263,7 @@ class VMInterfaceTestCase(ViewTestCases.DeviceComponentViewTestCase): 'name': 'Interface X', 'enabled': False, 'mac_address': EUI('01-02-03-04-05-06'), - 'mtu': 2000, + 'mtu': 65000, 'description': 'New description', 'mode': InterfaceModeChoices.MODE_TAGGED, 'untagged_vlan': vlans[0].pk, From 968a41fe7f83491252d2be99bbb0f8a61bb29e4b Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Tue, 27 Jul 2021 16:17:59 -0400 Subject: [PATCH 39/48] Changelog for #6785 --- docs/release-notes/version-2.11.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/release-notes/version-2.11.md b/docs/release-notes/version-2.11.md index 4737c460c..93ad02de6 100644 --- a/docs/release-notes/version-2.11.md +++ b/docs/release-notes/version-2.11.md @@ -2,6 +2,10 @@ ## v2.11.10 (FUTURE) +### Enhancements + +* [#6785](https://github.com/netbox-community/netbox/issues/6785) - Add "hardwired" type for power port types + ### Bug Fixes * [#5442](https://github.com/netbox-community/netbox/issues/5442) - Fix assignment of permissions based on LDAP groups From 481cd1796578b01b604cd2fd242bff6a19e231dc Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Tue, 27 Jul 2021 16:21:56 -0400 Subject: [PATCH 40/48] Fixes #5627: Fix filtering of interface connections list --- docs/release-notes/version-2.11.md | 1 + netbox/dcim/api/views.py | 4 +--- netbox/dcim/views.py | 6 +----- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/docs/release-notes/version-2.11.md b/docs/release-notes/version-2.11.md index 93ad02de6..1b116366c 100644 --- a/docs/release-notes/version-2.11.md +++ b/docs/release-notes/version-2.11.md @@ -9,6 +9,7 @@ ### Bug Fixes * [#5442](https://github.com/netbox-community/netbox/issues/5442) - Fix assignment of permissions based on LDAP groups +* [#5627](https://github.com/netbox-community/netbox/issues/5627) - Fix filtering of interface connections list * [#6759](https://github.com/netbox-community/netbox/issues/6759) - Fix assignment of parent interfaces for bulk import * [#6773](https://github.com/netbox-community/netbox/issues/6773) - Add missing `display` field to rack unit serializer * [#6774](https://github.com/netbox-community/netbox/issues/6774) - Fix A/Z assignment when swapping circuit terminations diff --git a/netbox/dcim/api/views.py b/netbox/dcim/api/views.py index 47ab26828..744d16e0a 100644 --- a/netbox/dcim/api/views.py +++ b/netbox/dcim/api/views.py @@ -592,11 +592,9 @@ class PowerConnectionViewSet(ListModelMixin, GenericViewSet): class InterfaceConnectionViewSet(ListModelMixin, GenericViewSet): queryset = Interface.objects.prefetch_related('device', '_path').filter( - # Avoid duplicate connections by only selecting the lower PK in a connected pair _path__destination_type__app_label='dcim', _path__destination_type__model='interface', - _path__destination_id__isnull=False, - pk__lt=F('_path__destination_id') + _path__destination_id__isnull=False ) serializer_class = serializers.InterfaceConnectionSerializer filterset_class = filtersets.InterfaceConnectionFilterSet diff --git a/netbox/dcim/views.py b/netbox/dcim/views.py index 50b55ee3f..6b776ed87 100644 --- a/netbox/dcim/views.py +++ b/netbox/dcim/views.py @@ -2564,11 +2564,7 @@ class PowerConnectionsListView(generic.ObjectListView): class InterfaceConnectionsListView(generic.ObjectListView): - queryset = Interface.objects.filter( - # Avoid duplicate connections by only selecting the lower PK in a connected pair - _path__isnull=False, - pk__lt=F('_path__destination_id') - ).order_by('device') + queryset = Interface.objects.filter(_path__isnull=False).order_by('device') filterset = filtersets.InterfaceConnectionFilterSet filterset_form = forms.InterfaceConnectionFilterForm table = tables.InterfaceConnectionTable From fa1dc3f0aaf18ecdee47bf1e440e22217bc37550 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Wed, 28 Jul 2021 09:55:40 -0400 Subject: [PATCH 41/48] Fixes #6812: Limit reported prefix utilization to 100% --- docs/release-notes/version-2.11.md | 1 + netbox/ipam/models/ip.py | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/release-notes/version-2.11.md b/docs/release-notes/version-2.11.md index 1b116366c..cd92c755f 100644 --- a/docs/release-notes/version-2.11.md +++ b/docs/release-notes/version-2.11.md @@ -17,6 +17,7 @@ * [#6778](https://github.com/netbox-community/netbox/issues/6778) - Rack reservation should display rack's location * [#6780](https://github.com/netbox-community/netbox/issues/6780) - Include rack location in navigation breadcrumbs * [#6794](https://github.com/netbox-community/netbox/issues/6794) - Fix device name display on device status view +* [#6812](https://github.com/netbox-community/netbox/issues/6812) - Limit reported prefix utilization to 100% * [#6822](https://github.com/netbox-community/netbox/issues/6822) - Use consistent maximum value for interface MTU ### Other Changes diff --git a/netbox/ipam/models/ip.py b/netbox/ipam/models/ip.py index 1f3766e3a..cd5b89cfe 100644 --- a/netbox/ipam/models/ip.py +++ b/netbox/ipam/models/ip.py @@ -181,7 +181,9 @@ class Aggregate(PrimaryModel): """ queryset = Prefix.objects.filter(prefix__net_contained_or_equal=str(self.prefix)) child_prefixes = netaddr.IPSet([p.prefix for p in queryset]) - return int(float(child_prefixes.size) / self.prefix.size * 100) + utilization = int(float(child_prefixes.size) / self.prefix.size * 100) + + return min(utilization, 100) @extras_features('custom_fields', 'custom_links', 'export_templates', 'webhooks') @@ -502,14 +504,16 @@ class Prefix(PrimaryModel): vrf=self.vrf ) child_prefixes = netaddr.IPSet([p.prefix for p in queryset]) - return int(float(child_prefixes.size) / self.prefix.size * 100) + utilization = int(float(child_prefixes.size) / self.prefix.size * 100) else: # Compile an IPSet to avoid counting duplicate IPs child_count = netaddr.IPSet([ip.address.ip for ip in self.get_child_ips()]).size prefix_size = self.prefix.size if self.prefix.version == 4 and self.prefix.prefixlen < 31 and not self.is_pool: prefix_size -= 2 - return int(float(child_count) / prefix_size * 100) + utilization = int(float(child_count) / prefix_size * 100) + + return min(utilization, 100) @extras_features('custom_fields', 'custom_links', 'export_templates', 'tags', 'webhooks') From b26c560b24be59dff5d54913a518bd5fc77e7925 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Wed, 28 Jul 2021 10:25:52 -0400 Subject: [PATCH 42/48] Fixes #6771: Add count of inventory items to manufacturer view --- docs/release-notes/version-2.11.md | 1 + netbox/dcim/views.py | 4 ++++ netbox/templates/dcim/manufacturer.html | 6 ++++++ 3 files changed, 11 insertions(+) diff --git a/docs/release-notes/version-2.11.md b/docs/release-notes/version-2.11.md index cd92c755f..b5308e46d 100644 --- a/docs/release-notes/version-2.11.md +++ b/docs/release-notes/version-2.11.md @@ -22,6 +22,7 @@ ### Other Changes +* [#6771](https://github.com/netbox-community/netbox/issues/6771) - Add count of inventory items to manufacturer view * [#6781](https://github.com/netbox-community/netbox/issues/6781) - Database query caching is now disabled by default --- diff --git a/netbox/dcim/views.py b/netbox/dcim/views.py index 6b776ed87..5afae3ced 100644 --- a/netbox/dcim/views.py +++ b/netbox/dcim/views.py @@ -695,6 +695,9 @@ class ManufacturerView(generic.ObjectView): ).annotate( instance_count=count_related(Device, 'device_type') ) + inventory_items = InventoryItem.objects.restrict(request.user, 'view').filter( + manufacturer=instance + ) devicetypes_table = tables.DeviceTypeTable(devicetypes) devicetypes_table.columns.hide('manufacturer') @@ -702,6 +705,7 @@ class ManufacturerView(generic.ObjectView): return { 'devicetypes_table': devicetypes_table, + 'inventory_item_count': inventory_items.count(), } diff --git a/netbox/templates/dcim/manufacturer.html b/netbox/templates/dcim/manufacturer.html index b2ecacbb1..9ea6f3fe1 100644 --- a/netbox/templates/dcim/manufacturer.html +++ b/netbox/templates/dcim/manufacturer.html @@ -29,6 +29,12 @@ {{ devicetypes_table.rows|length }} + + Inventory Items + + {{ inventory_item_count }} + +
    {% plugin_left_page object %} From 7e14ebe7186eb70f8b72800f2ad53fc7bb5c9b59 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Wed, 28 Jul 2021 10:31:59 -0400 Subject: [PATCH 43/48] Closes #6702: Update reference nginx config to support IPv6 --- contrib/nginx.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/nginx.conf b/contrib/nginx.conf index 1230f3ce4..34821cd52 100644 --- a/contrib/nginx.conf +++ b/contrib/nginx.conf @@ -1,5 +1,5 @@ server { - listen 443 ssl; + listen [::]:443 ssl ipv6only=off; # CHANGE THIS TO YOUR SERVER'S NAME server_name netbox.example.com; @@ -23,7 +23,7 @@ server { server { # Redirect HTTP traffic to HTTPS - listen 80; + listen [::]:80 ipv6only=off; server_name _; return 301 https://$host$request_uri; } From 5fc52fbd4d59d02a5bdf63abe5663e8fb43380e2 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Wed, 28 Jul 2021 11:44:13 -0400 Subject: [PATCH 44/48] Changelog and cleanup for #6560 --- docs/release-notes/version-2.11.md | 3 +- netbox/netbox/views/generic.py | 18 +- .../templates/generic/object_bulk_import.html | 194 +++++++++--------- netbox/utilities/forms/fields.py | 18 +- netbox/utilities/forms/utils.py | 1 + 5 files changed, 122 insertions(+), 112 deletions(-) diff --git a/docs/release-notes/version-2.11.md b/docs/release-notes/version-2.11.md index b5308e46d..6d6bd321b 100644 --- a/docs/release-notes/version-2.11.md +++ b/docs/release-notes/version-2.11.md @@ -4,6 +4,8 @@ ### Enhancements +* [#6560](https://github.com/netbox-community/netbox/issues/6560) - Enable CSV import via uploaded file +* [#6771](https://github.com/netbox-community/netbox/issues/6771) - Add count of inventory items to manufacturer view * [#6785](https://github.com/netbox-community/netbox/issues/6785) - Add "hardwired" type for power port types ### Bug Fixes @@ -22,7 +24,6 @@ ### Other Changes -* [#6771](https://github.com/netbox-community/netbox/issues/6771) - Add count of inventory items to manufacturer view * [#6781](https://github.com/netbox-community/netbox/issues/6781) - Database query caching is now disabled by default --- diff --git a/netbox/netbox/views/generic.py b/netbox/netbox/views/generic.py index a312441e5..bd3d6300a 100644 --- a/netbox/netbox/views/generic.py +++ b/netbox/netbox/views/generic.py @@ -20,7 +20,8 @@ from extras.models import CustomField, ExportTemplate from utilities.error_handlers import handle_protectederror from utilities.exceptions import AbortTransaction, PermissionsViolation from utilities.forms import ( - BootstrapMixin, BulkRenameForm, ConfirmationForm, CSVDataField, ImportForm, TableConfigForm, restrict_form_fields, CSVFileField + BootstrapMixin, BulkRenameForm, ConfirmationForm, CSVDataField, CSVFileField, ImportForm, TableConfigForm, + restrict_form_fields, ) from utilities.permissions import get_permission_for_model from utilities.tables import paginate_table @@ -673,8 +674,16 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): required=False ) - def used_both_csv_fields(self): - return self.cleaned_data['csv_file'][1] and self.cleaned_data['csv'][1] + def clean(self): + csv_rows = self.cleaned_data['csv'][1] + csv_file = self.files.get('csv_file') + + # Check that the user has not submitted both text data and a file + if csv_rows and csv_file: + raise ValidationError( + "Cannot process CSV text and file attachment simultaneously. Please choose only one import " + "method." + ) return ImportForm(*args, **kwargs) @@ -705,9 +714,6 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): logger.debug("Form validation was successful") try: - if form.used_both_csv_fields(): - form.add_error('csv_file', "Choose one of two import methods") - raise ValidationError("") # Iterate through CSV data and bind each row to a new model form instance. with transaction.atomic(): if request.FILES: diff --git a/netbox/templates/generic/object_bulk_import.html b/netbox/templates/generic/object_bulk_import.html index e9972e919..ec6dddf14 100644 --- a/netbox/templates/generic/object_bulk_import.html +++ b/netbox/templates/generic/object_bulk_import.html @@ -16,103 +16,107 @@
    {% endif %} -
    -
    - - {% csrf_token %} - {% render_form form %} -
    -
    - - {% if return_url %} - Cancel - {% endif %} -
    -
    - -
    -

    - {% if fields %} -
    -
    - CSV Field Options -
    - - - - - - - - {% for name, field in fields.items %} - - - - - - - {% endfor %} -
    FieldRequiredAccessorDescription
    - {{ name }} - - {% if field.required %} - - {% else %} - - {% endif %} - - {% if field.to_field_name %} - {{ field.to_field_name }} - {% else %} - - {% endif %} - - {% if field.STATIC_CHOICES %} - - - {% endif %} - {% if field.help_text %} - {{ field.help_text }}
    - {% elif field.label %} - {{ field.label }}
    - {% endif %} - {% if field|widget_type == 'dateinput' %} - Format: YYYY-MM-DD - {% elif field|widget_type == 'checkboxinput' %} - Specify "true" or "false" - {% endif %} -
    -
    -

    - Required fields must be specified for all - objects. -

    -

    - Related objects may be referenced by any unique attribute. - For example, vrf.rd would identify a VRF by its route distinguisher. -

    - {% endif %} +
    + {% csrf_token %} +
    +
    + {% render_field form.csv %} +
    +
    + {% render_field form.csv_file %} +
    -
    +
    +
    + + {% if return_url %} + Cancel + {% endif %} +
    +
    + +
    +

    + {% if fields %} +
    +
    + CSV Field Options +
    + + + + + + + + {% for name, field in fields.items %} + + + + + + + {% endfor %} +
    FieldRequiredAccessorDescription
    + {{ name }} + + {% if field.required %} + + {% else %} + + {% endif %} + + {% if field.to_field_name %} + {{ field.to_field_name }} + {% else %} + + {% endif %} + + {% if field.STATIC_CHOICES %} + + + {% endif %} + {% if field.help_text %} + {{ field.help_text }}
    + {% elif field.label %} + {{ field.label }}
    + {% endif %} + {% if field|widget_type == 'dateinput' %} + Format: YYYY-MM-DD + {% elif field|widget_type == 'checkboxinput' %} + Specify "true" or "false" + {% endif %} +
    +
    +

    + Required fields must be specified for all + objects. +

    +

    + Related objects may be referenced by any unique attribute. + For example, vrf.rd would identify a VRF by its route distinguisher. +

    + {% endif %}
    {% endblock %} diff --git a/netbox/utilities/forms/fields.py b/netbox/utilities/forms/fields.py index 2b9b57ea3..7cb2cd705 100644 --- a/netbox/utilities/forms/fields.py +++ b/netbox/utilities/forms/fields.py @@ -208,22 +208,20 @@ class CSVFileField(forms.FileField): super().__init__(*args, **kwargs) def to_python(self, file): - if file: - csv_str = file.read().decode('utf-8') - reader = csv.reader(csv_str.splitlines()) + if file is None: + return None - headers = {} - records = [] - if file: - headers, records = parse_csv(reader) + csv_str = file.read().decode('utf-8').strip() + reader = csv.reader(csv_str.splitlines()) + headers, records = parse_csv(reader) return headers, records def validate(self, value): - headers, records = value - if not headers and not records: - return value + if value is None: + return None + headers, records = value validate_csv(headers, self.fields, self.required_fields) return value diff --git a/netbox/utilities/forms/utils.py b/netbox/utilities/forms/utils.py index 50a6a416e..503a2e8a0 100644 --- a/netbox/utilities/forms/utils.py +++ b/netbox/utilities/forms/utils.py @@ -166,6 +166,7 @@ def parse_csv(reader): row = [col.strip() for col in row] record = dict(zip(headers.keys(), row)) records.append(record) + return headers, records From f923477c532a03e22c2866d5a72763e8348eebdb Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Wed, 28 Jul 2021 11:54:25 -0400 Subject: [PATCH 45/48] Closes #6644: Add 6P/4P pass-through port types --- docs/release-notes/version-2.11.md | 1 + netbox/dcim/choices.py | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/docs/release-notes/version-2.11.md b/docs/release-notes/version-2.11.md index 6d6bd321b..4ba284d0b 100644 --- a/docs/release-notes/version-2.11.md +++ b/docs/release-notes/version-2.11.md @@ -5,6 +5,7 @@ ### Enhancements * [#6560](https://github.com/netbox-community/netbox/issues/6560) - Enable CSV import via uploaded file +* [#6644](https://github.com/netbox-community/netbox/issues/6644) - Add 6P/4P pass-through port types * [#6771](https://github.com/netbox-community/netbox/issues/6771) - Add count of inventory items to manufacturer view * [#6785](https://github.com/netbox-community/netbox/issues/6785) - Add "hardwired" type for power port types diff --git a/netbox/dcim/choices.py b/netbox/dcim/choices.py index a53f357f8..9a12e6a19 100644 --- a/netbox/dcim/choices.py +++ b/netbox/dcim/choices.py @@ -922,6 +922,11 @@ class PortTypeChoices(ChoiceSet): TYPE_8P6C = '8p6c' TYPE_8P4C = '8p4c' TYPE_8P2C = '8p2c' + TYPE_6P6C = '6p6c' + TYPE_6P4C = '6p4c' + TYPE_6P2C = '6p2c' + TYPE_4P4C = '4p4c' + TYPE_4P2C = '4p2c' TYPE_GG45 = 'gg45' TYPE_TERA4P = 'tera-4p' TYPE_TERA2P = 'tera-2p' @@ -953,6 +958,11 @@ class PortTypeChoices(ChoiceSet): (TYPE_8P6C, '8P6C'), (TYPE_8P4C, '8P4C'), (TYPE_8P2C, '8P2C'), + (TYPE_6P6C, '6P6C'), + (TYPE_6P4C, '6P4C'), + (TYPE_6P2C, '6P2C'), + (TYPE_4P4C, '4P4C'), + (TYPE_4P2C, '4P2C'), (TYPE_GG45, 'GG45'), (TYPE_TERA4P, 'TERA 4P'), (TYPE_TERA2P, 'TERA 2P'), From 5b9e2234ad80953b691b6de624e37d0aabf8897e Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Wed, 28 Jul 2021 15:07:46 -0400 Subject: [PATCH 46/48] Tweak GitHub repo icon & name in docs --- mkdocs.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index d26799caf..d6e60b255 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,11 +1,14 @@ site_name: NetBox Documentation site_url: https://netbox.readthedocs.io/ +repo_name: netbox-community/netbox repo_url: https://github.com/netbox-community/netbox python: install: - requirements: docs/requirements.txt theme: - name: material + name: material + icon: + repo: fontawesome/brands/github extra_css: - extra.css markdown_extensions: From 82d12edb9224e1fa57f92a8abe72a7921a00b1f9 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Wed, 28 Jul 2021 15:12:17 -0400 Subject: [PATCH 47/48] Shrink NetBox logo on docs main page --- docs/index.md | 2 +- mkdocs.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index 5cdf871d9..0fc0dc0b7 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,4 +1,4 @@ -![NetBox](netbox_logo.svg "NetBox logo") +![NetBox](netbox_logo.svg "NetBox logo"){style="height: 100px; margin-bottom: 3em"} # What is NetBox? diff --git a/mkdocs.yml b/mkdocs.yml index d6e60b255..16b345b96 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -13,6 +13,7 @@ extra_css: - extra.css markdown_extensions: - admonition + - attr_list - markdown_include.include: headingOffset: 1 - pymdownx.emoji: From 861f8338e218ca547cd105e8213db380bcf87515 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Wed, 28 Jul 2021 15:17:45 -0400 Subject: [PATCH 48/48] Release v2.11.10 --- .github/ISSUE_TEMPLATE/bug_report.yaml | 2 +- .github/ISSUE_TEMPLATE/feature_request.yaml | 2 +- docs/release-notes/version-2.11.md | 2 +- netbox/netbox/settings.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yaml b/.github/ISSUE_TEMPLATE/bug_report.yaml index 97b55b285..ef31324fe 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yaml +++ b/.github/ISSUE_TEMPLATE/bug_report.yaml @@ -17,7 +17,7 @@ body: What version of NetBox are you currently running? (If you don't have access to the most recent NetBox release, consider testing on our [demo instance](https://demo.netbox.dev/) before opening a bug report to see if your issue has already been addressed.) - placeholder: v2.11.9 + placeholder: v2.11.10 validations: required: true - type: dropdown diff --git a/.github/ISSUE_TEMPLATE/feature_request.yaml b/.github/ISSUE_TEMPLATE/feature_request.yaml index bfb0cc7aa..319538cda 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yaml +++ b/.github/ISSUE_TEMPLATE/feature_request.yaml @@ -14,7 +14,7 @@ body: attributes: label: NetBox version description: What version of NetBox are you currently running? - placeholder: v2.11.9 + placeholder: v2.11.10 validations: required: true - type: dropdown diff --git a/docs/release-notes/version-2.11.md b/docs/release-notes/version-2.11.md index 4ba284d0b..736807e2c 100644 --- a/docs/release-notes/version-2.11.md +++ b/docs/release-notes/version-2.11.md @@ -1,6 +1,6 @@ # NetBox v2.11 -## v2.11.10 (FUTURE) +## v2.11.10 (2021-07-28) ### Enhancements diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index dff38c530..f28f72a27 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -16,7 +16,7 @@ from django.core.validators import URLValidator # Environment setup # -VERSION = '2.11.10-dev' +VERSION = '2.11.10' # Hostname HOSTNAME = platform.node()