From 7ebfa4c1d1890fafb6393d97e88aa76229368f67 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Thu, 2 Feb 2023 15:41:24 -0500 Subject: [PATCH 01/38] PRVB --- docs/release-notes/version-3.4.md | 4 ++++ netbox/netbox/settings.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/release-notes/version-3.4.md b/docs/release-notes/version-3.4.md index 1581ce681..15b84436d 100644 --- a/docs/release-notes/version-3.4.md +++ b/docs/release-notes/version-3.4.md @@ -1,5 +1,9 @@ # NetBox v3.4 +## v3.4.5 (FUTURE) + +--- + ## v3.4.4 (2023-02-02) ### Enhancements diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 8517efca1..cda6ee643 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -24,7 +24,7 @@ from netbox.constants import RQ_QUEUE_DEFAULT, RQ_QUEUE_HIGH, RQ_QUEUE_LOW # Environment setup # -VERSION = '3.4.4' +VERSION = '3.4.5-dev' # Hostname HOSTNAME = platform.node() From 5e1bb20f3208d34321fa218f901f71aaa6f439d5 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Tue, 7 Feb 2023 16:49:07 -0500 Subject: [PATCH 02/38] Display login message as success --- netbox/users/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/users/views.py b/netbox/users/views.py index 832a4e592..a82620914 100644 --- a/netbox/users/views.py +++ b/netbox/users/views.py @@ -96,7 +96,7 @@ class LoginView(View): # Authenticate user auth_login(request, form.get_user()) logger.info(f"User {request.user} successfully authenticated") - messages.info(request, f"Logged in as {request.user}.") + messages.success(request, f"Logged in as {request.user}.") # Ensure the user has a UserConfig defined. (This should normally be handled by # create_userconfig() on user creation.) From edbd597bf2687ef31d1fbaa394b2080bd57f95a9 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Tue, 7 Feb 2023 16:52:54 -0500 Subject: [PATCH 03/38] Update housekeeping command docs --- docs/administration/housekeeping.md | 1 + netbox/extras/management/commands/housekeeping.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/administration/housekeeping.md b/docs/administration/housekeeping.md index da1a5443b..fcc3aa04e 100644 --- a/docs/administration/housekeeping.md +++ b/docs/administration/housekeeping.md @@ -5,6 +5,7 @@ NetBox includes a `housekeeping` management command that should be run nightly. * Clearing expired authentication sessions from the database * Deleting changelog records older than the configured [retention time](../configuration/miscellaneous.md#changelog_retention) * Deleting job result records older than the configured [retention time](../configuration/miscellaneous.md#jobresult_retention) +* Check for new NetBox releases (if [`RELEASE_CHECK_URL`](../configuration/miscellaneous.md#release_check_url) is set) This command can be invoked directly, or by using the shell script provided at `/opt/netbox/contrib/netbox-housekeeping.sh`. This script can be linked from your cron scheduler's daily jobs directory (e.g. `/etc/cron.daily`) or referenced directly within the cron configuration file. diff --git a/netbox/extras/management/commands/housekeeping.py b/netbox/extras/management/commands/housekeeping.py index 42690568d..172e26bf2 100644 --- a/netbox/extras/management/commands/housekeeping.py +++ b/netbox/extras/management/commands/housekeeping.py @@ -37,7 +37,7 @@ class Command(BaseCommand): f"clearing sessions; skipping." ) - # Delete expired ObjectRecords + # Delete expired ObjectChanges if options['verbosity']: self.stdout.write("[*] Checking for expired changelog records") if config.CHANGELOG_RETENTION: From 3f28d6aef3b40726ee6c9fade941c31272271d70 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Tue, 7 Feb 2023 16:55:50 -0500 Subject: [PATCH 04/38] Add step for creating search index --- docs/development/adding-models.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/development/adding-models.md b/docs/development/adding-models.md index aef11d666..7de897a97 100644 --- a/docs/development/adding-models.md +++ b/docs/development/adding-models.md @@ -54,15 +54,19 @@ Each model should have a corresponding FilterSet class defined. This is used to Create a table class for the model in `tables.py` by subclassing `utilities.tables.BaseTable`. Under the table's `Meta` class, be sure to list both the fields and default columns. -## 9. Create the object template +## 9. Create a SearchIndex subclass + +If this model will be included in global search results, create a subclass of `netbox.search.SearchIndex` for it and specify the fields to be indexed. + +## 10. Create the object template Create the HTML template for the object view. (The other views each typically employ a generic template.) This template should extend `generic/object.html`. -## 10. Add the model to the navigation menu +## 11. Add the model to the navigation menu Add the relevant navigation menu items in `netbox/netbox/navigation/menu.py`. -## 11. REST API components +## 12. REST API components Create the following for each model: @@ -71,13 +75,13 @@ Create the following for each model: * API view in `api/views.py` * Endpoint route in `api/urls.py` -## 12. GraphQL API components +## 13. GraphQL API components Create a Graphene object type for the model in `graphql/types.py` by subclassing the appropriate class from `netbox.graphql.types`. Also extend the schema class defined in `graphql/schema.py` with the individual object and object list fields per the established convention. -## 13. Add tests +## 14. Add tests Add tests for the following: @@ -85,7 +89,7 @@ Add tests for the following: * API views * Filter sets -## 14. Documentation +## 15. Documentation Create a new documentation page for the model in `docs/models//.md`. Include this file under the "features" documentation where appropriate. From 56c7a238a4905d3220b63d8b33b07ca52df7efd1 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Tue, 7 Feb 2023 17:24:26 -0500 Subject: [PATCH 05/38] Fixes #11683: Fix CSV header attribute detection when auto-detecting import format --- docs/release-notes/version-3.4.md | 4 ++++ netbox/netbox/views/generic/bulk_views.py | 4 ++-- netbox/utilities/forms/forms.py | 2 ++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/release-notes/version-3.4.md b/docs/release-notes/version-3.4.md index 15b84436d..b23251a70 100644 --- a/docs/release-notes/version-3.4.md +++ b/docs/release-notes/version-3.4.md @@ -2,6 +2,10 @@ ## v3.4.5 (FUTURE) +### Bug Fixes + +* [#11683](https://github.com/netbox-community/netbox/issues/11683) - Fix CSV header attribute detection when auto-detecting import format + --- ## v3.4.4 (2023-02-02) diff --git a/netbox/netbox/views/generic/bulk_views.py b/netbox/netbox/views/generic/bulk_views.py index ab3e8f100..6060475d8 100644 --- a/netbox/netbox/views/generic/bulk_views.py +++ b/netbox/netbox/views/generic/bulk_views.py @@ -384,8 +384,8 @@ class BulkImportView(GetReturnURLMixin, BaseMultiObjectView): 'data': record, 'instance': instance, } - if form.cleaned_data['format'] == ImportFormatChoices.CSV: - model_form_kwargs['headers'] = form._csv_headers + if hasattr(form, '_csv_headers'): + model_form_kwargs['headers'] = form._csv_headers # Add CSV headers model_form = self.model_form(**model_form_kwargs) # When updating, omit all form fields other than those specified in the record. (No diff --git a/netbox/utilities/forms/forms.py b/netbox/utilities/forms/forms.py index 99d03f2a6..9884ffac5 100644 --- a/netbox/utilities/forms/forms.py +++ b/netbox/utilities/forms/forms.py @@ -197,6 +197,8 @@ class ImportForm(BootstrapMixin, forms.Form): self.cleaned_data['data'] = self._clean_json(data) elif format == ImportFormatChoices.YAML: self.cleaned_data['data'] = self._clean_yaml(data) + else: + raise forms.ValidationError(f"Unknown data format: {format}") def _detect_format(self, data): """ From 91705aa9fdfb4b730aaad3b438f65b853ff00107 Mon Sep 17 00:00:00 2001 From: kkthxbye <400797+kkthxbye-code@users.noreply.github.com> Date: Wed, 8 Feb 2023 20:36:20 +0100 Subject: [PATCH 06/38] Fixes #11032 - Replication fields broken in custom validation (#11698) * Fixes #11032 - Replication fields broken in custom validation * Use getattr instead of hasattr to make sure custom validation is triggered as normal --------- Co-authored-by: kkthxbye-code <> --- netbox/netbox/models/features.py | 4 ++++ netbox/netbox/views/generic/object_views.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/netbox/netbox/models/features.py b/netbox/netbox/models/features.py index 8e5af0ab5..f041d016d 100644 --- a/netbox/netbox/models/features.py +++ b/netbox/netbox/models/features.py @@ -257,6 +257,10 @@ class CustomValidationMixin(models.Model): def clean(self): super().clean() + # If the instance is a base for replications, skip custom validation + if getattr(self, '_replicated_base', False): + return + # Send the post_clean signal post_clean.send(sender=self.__class__, instance=self) diff --git a/netbox/netbox/views/generic/object_views.py b/netbox/netbox/views/generic/object_views.py index d855490d1..475cca9d3 100644 --- a/netbox/netbox/views/generic/object_views.py +++ b/netbox/netbox/views/generic/object_views.py @@ -436,6 +436,10 @@ class ComponentCreateView(GetReturnURLMixin, BaseObjectView): form = self.initialize_form(request) instance = self.alter_object(self.queryset.model(), request) + # Note that the form instance is a replicated field base + # This is needed to avoid running custom validators multiple times + form.instance._replicated_base = hasattr(self.form, "replication_fields") + if form.is_valid(): new_components = [] data = deepcopy(request.POST) From 3c970c331ceb7fef0f78d05c6e500c177ab6d037 Mon Sep 17 00:00:00 2001 From: kkthxbye-code Date: Wed, 8 Feb 2023 09:33:06 +0100 Subject: [PATCH 07/38] Fixes #11582: Fix missing VC form errors ### Fixes: #11582 Not sure if this is the correct fix or not. The reason that the custom field errors were not shown is that messages.html only shows non_field_errors if the form passed to the context is named form. This is probably an issue in more places, but not sure how to make it generic. A change to messages.html would also need to support formsets. Any input appreciated @jeremystretch or @arthanson --- netbox/templates/dcim/virtualchassis_add_member.html | 2 ++ netbox/templates/dcim/virtualchassis_edit.html | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/netbox/templates/dcim/virtualchassis_add_member.html b/netbox/templates/dcim/virtualchassis_add_member.html index 17ffd64d9..bc2ba2f55 100644 --- a/netbox/templates/dcim/virtualchassis_add_member.html +++ b/netbox/templates/dcim/virtualchassis_add_member.html @@ -5,6 +5,8 @@ {% block content %}
+ {% render_errors membership_form %} + {% csrf_token %}
Add New Member
diff --git a/netbox/templates/dcim/virtualchassis_edit.html b/netbox/templates/dcim/virtualchassis_edit.html index f98a9fe64..433837cf5 100644 --- a/netbox/templates/dcim/virtualchassis_edit.html +++ b/netbox/templates/dcim/virtualchassis_edit.html @@ -8,6 +8,10 @@
+ {% for form in formset %} + {% render_errors form %} + {% endfor %} + {% csrf_token %} {{ pk_form.pk }} {{ formset.management_form }} From f9237285fdfa1d412e2ff8de8a4ee7475d634c74 Mon Sep 17 00:00:00 2001 From: kkthxbye-code Date: Wed, 8 Feb 2023 10:09:07 +0100 Subject: [PATCH 08/38] Fixes #11601 - Add partial lookup to IPRangeFilterSet --- netbox/ipam/filtersets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/netbox/ipam/filtersets.py b/netbox/ipam/filtersets.py index d069eed27..c312b02ff 100644 --- a/netbox/ipam/filtersets.py +++ b/netbox/ipam/filtersets.py @@ -441,9 +441,9 @@ class IPRangeFilterSet(TenancyFilterSet, NetBoxModelFilterSet): def search(self, queryset, name, value): if not value.strip(): return queryset - qs_filter = Q(description__icontains=value) + qs_filter = Q(description__icontains=value) | Q(start_address__contains=value) | Q(end_address__contains=value) try: - ipaddress = str(netaddr.IPNetwork(value.strip()).cidr) + ipaddress = str(netaddr.IPNetwork(value.strip())) qs_filter |= Q(start_address=ipaddress) qs_filter |= Q(end_address=ipaddress) except (AddrFormatError, ValueError): From b5da383a179be36312c4def9fc6de29bde0a3df0 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Wed, 8 Feb 2023 14:56:14 -0500 Subject: [PATCH 09/38] Changelog for #11032, #11582, #11601 --- docs/release-notes/version-3.4.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/release-notes/version-3.4.md b/docs/release-notes/version-3.4.md index b23251a70..b8ebe4a33 100644 --- a/docs/release-notes/version-3.4.md +++ b/docs/release-notes/version-3.4.md @@ -4,6 +4,9 @@ ### Bug Fixes +* [#11032](https://github.com/netbox-community/netbox/issues/11032) - Fix false custom validation errors during component creation +* [#11582](https://github.com/netbox-community/netbox/issues/11582) - Ensure form validation errors are displayed when adding virtual chassis members +* [#11601](https://github.com/netbox-community/netbox/issues/11601) - Fix partial matching of start/end addresses for IP range search * [#11683](https://github.com/netbox-community/netbox/issues/11683) - Fix CSV header attribute detection when auto-detecting import format --- From df499ea8aca4bbdd5589bba4a3b0265f148ca8b9 Mon Sep 17 00:00:00 2001 From: kkthxbye <400797+kkthxbye-code@users.noreply.github.com> Date: Mon, 13 Feb 2023 23:44:35 +0100 Subject: [PATCH 10/38] Fixes #11459 - Allow using null in conditions (#11722) * Fixes #11459 - Allow using null in conditions - Update docs to reflect this - Change docs example from primary_ip to primary_ip4 as computed properties are not serialized when queuing webhooks * Update netbox/extras/conditions.py --------- Co-authored-by: Simon Toft Co-authored-by: Jeremy Stretch --- docs/reference/conditions.md | 6 +++--- netbox/extras/conditions.py | 3 ++- netbox/extras/tests/test_conditions.py | 10 ++++++++++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/docs/reference/conditions.md b/docs/reference/conditions.md index fb8b66139..514006b01 100644 --- a/docs/reference/conditions.md +++ b/docs/reference/conditions.md @@ -97,7 +97,7 @@ Multiple conditions can be combined into nested sets using AND or OR logic. This ### Examples -`status` is "active" and `primary_ip` is defined _or_ the "exempt" tag is applied. +`status` is "active" and `primary_ip4` is defined _or_ the "exempt" tag is applied. ```json { @@ -109,8 +109,8 @@ Multiple conditions can be combined into nested sets using AND or OR logic. This "value": "active" }, { - "attr": "primary_ip", - "value": "", + "attr": "primary_ip4", + "value": null, "negate": true } ] diff --git a/netbox/extras/conditions.py b/netbox/extras/conditions.py index 965488c3a..c6744e524 100644 --- a/netbox/extras/conditions.py +++ b/netbox/extras/conditions.py @@ -44,7 +44,8 @@ class Condition: bool: (EQ, CONTAINS), int: (EQ, GT, GTE, LT, LTE, CONTAINS), float: (EQ, GT, GTE, LT, LTE, CONTAINS), - list: (EQ, IN, CONTAINS) + list: (EQ, IN, CONTAINS), + type(None): (EQ,) } def __init__(self, attr, value, op=EQ, negate=False): diff --git a/netbox/extras/tests/test_conditions.py b/netbox/extras/tests/test_conditions.py index 8e02eb75d..e7275482a 100644 --- a/netbox/extras/tests/test_conditions.py +++ b/netbox/extras/tests/test_conditions.py @@ -126,6 +126,16 @@ class ConditionSetTest(TestCase): with self.assertRaises(ValueError): ConditionSet({'foo': []}) + def test_null_value(self): + cs = ConditionSet({ + 'and': [ + {'attr': 'a', 'value': None, 'op': 'eq', 'negate': True}, + ] + }) + self.assertFalse(cs.eval({'a': None})) + self.assertTrue(cs.eval({'a': "string"})) + self.assertTrue(cs.eval({'a': {"key": "value"}})) + def test_and_single_depth(self): cs = ConditionSet({ 'and': [ From d748851027783998df542dd3ab6e8f55284d4fca Mon Sep 17 00:00:00 2001 From: kkthxbye <400797+kkthxbye-code@users.noreply.github.com> Date: Mon, 13 Feb 2023 23:49:08 +0100 Subject: [PATCH 11/38] Fixes #11711 - Use CSVModelMultipleChoiceField when importing custom multiple object fields (#11712) * Fixes #11711 - Use CSVModelMultipleChoiceField when importing custom multiple object fields * Fix pep8 --------- Co-authored-by: kkthxbye-code <> --- netbox/extras/models/customfields.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/netbox/extras/models/customfields.py b/netbox/extras/models/customfields.py index 4842c0654..c7a19e4b8 100644 --- a/netbox/extras/models/customfields.py +++ b/netbox/extras/models/customfields.py @@ -21,7 +21,7 @@ from netbox.models.features import CloningMixin, ExportTemplatesMixin, WebhooksM from netbox.search import FieldTypes from utilities import filters from utilities.forms import ( - CSVChoiceField, CSVMultipleChoiceField, DatePicker, DynamicModelChoiceField, DynamicModelMultipleChoiceField, + CSVChoiceField, CSVModelMultipleChoiceField, CSVMultipleChoiceField, DatePicker, DynamicModelChoiceField, DynamicModelMultipleChoiceField, JSONField, LaxURLField, StaticSelectMultiple, StaticSelect, add_blank_choice, ) from utilities.querysets import RestrictedQuerySet @@ -422,10 +422,12 @@ class CustomField(CloningMixin, ExportTemplatesMixin, WebhooksMixin, ChangeLogge # Multiple objects elif self.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: model = self.object_type.model_class() - field = DynamicModelMultipleChoiceField( + field_class = CSVModelMultipleChoiceField if for_csv_import else DynamicModelMultipleChoiceField + + field = field_class( queryset=model.objects.all(), required=required, - initial=initial + initial=initial, ) # Text From 9f91b89467d5eae9ab63e9256c6f760b7341eee5 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Mon, 13 Feb 2023 17:53:01 -0500 Subject: [PATCH 12/38] #11711: Use CSVModelChoiceField for custom object fields during CSV import --- netbox/extras/models/customfields.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/netbox/extras/models/customfields.py b/netbox/extras/models/customfields.py index c7a19e4b8..fa16b8501 100644 --- a/netbox/extras/models/customfields.py +++ b/netbox/extras/models/customfields.py @@ -20,10 +20,12 @@ from netbox.models import ChangeLoggedModel from netbox.models.features import CloningMixin, ExportTemplatesMixin, WebhooksMixin from netbox.search import FieldTypes from utilities import filters -from utilities.forms import ( - CSVChoiceField, CSVModelMultipleChoiceField, CSVMultipleChoiceField, DatePicker, DynamicModelChoiceField, DynamicModelMultipleChoiceField, - JSONField, LaxURLField, StaticSelectMultiple, StaticSelect, add_blank_choice, +from utilities.forms.fields import ( + CSVChoiceField, CSVModelChoiceField, CSVModelMultipleChoiceField, CSVMultipleChoiceField, DynamicModelChoiceField, + DynamicModelMultipleChoiceField, JSONField, LaxURLField, ) +from utilities.forms.widgets import DatePicker, StaticSelectMultiple, StaticSelect +from utilities.forms.utils import add_blank_choice from utilities.querysets import RestrictedQuerySet from utilities.validators import validate_regex @@ -413,7 +415,8 @@ class CustomField(CloningMixin, ExportTemplatesMixin, WebhooksMixin, ChangeLogge # Object elif self.type == CustomFieldTypeChoices.TYPE_OBJECT: model = self.object_type.model_class() - field = DynamicModelChoiceField( + field_class = CSVModelChoiceField if for_csv_import else DynamicModelChoiceField + field = field_class( queryset=model.objects.all(), required=required, initial=initial @@ -423,7 +426,6 @@ class CustomField(CloningMixin, ExportTemplatesMixin, WebhooksMixin, ChangeLogge elif self.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: model = self.object_type.model_class() field_class = CSVModelMultipleChoiceField if for_csv_import else DynamicModelMultipleChoiceField - field = field_class( queryset=model.objects.all(), required=required, From 3150c1f8b3c38f11dc182d22071da3b9bd62b15e Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Mon, 13 Feb 2023 17:58:41 -0500 Subject: [PATCH 13/38] Changelog for #11459, #11711 --- docs/release-notes/version-3.4.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/release-notes/version-3.4.md b/docs/release-notes/version-3.4.md index b8ebe4a33..58c3aea49 100644 --- a/docs/release-notes/version-3.4.md +++ b/docs/release-notes/version-3.4.md @@ -5,9 +5,11 @@ ### Bug Fixes * [#11032](https://github.com/netbox-community/netbox/issues/11032) - Fix false custom validation errors during component creation +* [#11459](https://github.com/netbox-community/netbox/issues/11459) - Enable evaluating null values in custom validation rules * [#11582](https://github.com/netbox-community/netbox/issues/11582) - Ensure form validation errors are displayed when adding virtual chassis members * [#11601](https://github.com/netbox-community/netbox/issues/11601) - Fix partial matching of start/end addresses for IP range search * [#11683](https://github.com/netbox-community/netbox/issues/11683) - Fix CSV header attribute detection when auto-detecting import format +* [#11711](https://github.com/netbox-community/netbox/issues/11711) - Fix CSV import for multiple-object custom fields --- From c78022a74cbce0d3e05e41a3d22b74675f7c645b Mon Sep 17 00:00:00 2001 From: kkthxbye-code Date: Thu, 16 Feb 2023 12:33:08 +0100 Subject: [PATCH 14/38] Change the way we invalidate the module cache to support reloading code from subpackages --- netbox/extras/scripts.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/netbox/extras/scripts.py b/netbox/extras/scripts.py index 77c96de56..313058d57 100644 --- a/netbox/extras/scripts.py +++ b/netbox/extras/scripts.py @@ -524,27 +524,39 @@ def get_scripts(use_names=False): defined name in place of the actual module name. """ scripts = {} - # Iterate through all modules within the scripts path. These are the user-created files in which reports are + + # Get all modules within the scripts path. These are the user-created files in which scripts are # defined. - for importer, module_name, _ in pkgutil.iter_modules([settings.SCRIPTS_ROOT]): - # Use a lock as removing and loading modules is not thread safe - with lock: - # Remove cached module to ensure consistency with filesystem - if module_name in sys.modules: + modules = list(pkgutil.iter_modules([settings.SCRIPTS_ROOT])) + modules_bases = set([name.split(".")[0] for _, name, _ in modules]) + + # Deleting from sys.modules needs to done behind a lock to prevent race conditions where a module is + # removed from sys.modules while another thread is importing + with lock: + for module_name in list(sys.modules.keys()): + # Everything sharing a base module path with a module in the script folder is removed. + # We also remove all modules with a base module called "scripts". This allows modifying imported + # non-script modules without having to reload the RQ worker. + module_base = module_name.split(".")[0] + if module_base == "scripts" or module_base in modules_bases: del sys.modules[module_name] - module = importer.find_module(module_name).load_module(module_name) + for importer, module_name, _ in modules: + module = importer.find_module(module_name).load_module(module_name) if use_names and hasattr(module, 'name'): module_name = module.name + module_scripts = {} script_order = getattr(module, "script_order", ()) ordered_scripts = [cls for cls in script_order if is_script(cls)] unordered_scripts = [cls for _, cls in inspect.getmembers(module, is_script) if cls not in script_order] + for cls in [*ordered_scripts, *unordered_scripts]: # For scripts in submodules use the full import path w/o the root module as the name script_name = cls.full_name.split(".", maxsplit=1)[1] module_scripts[script_name] = cls + if module_scripts: scripts[module_name] = module_scripts From 959404980464c7b6f9b80a17f9d938aa9cdcb968 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?aron=20bergur=20j=C3=B3hannsson?= Date: Tue, 14 Feb 2023 23:58:41 +0000 Subject: [PATCH 15/38] Fixes #11473 graphql invalid tag filter returns all devices/interfaces --- netbox/netbox/graphql/fields.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/netbox/netbox/graphql/fields.py b/netbox/netbox/graphql/fields.py index 5b8e6cc5b..1e215c947 100644 --- a/netbox/netbox/graphql/fields.py +++ b/netbox/netbox/graphql/fields.py @@ -60,6 +60,8 @@ class ObjectListField(DjangoListField): filterset_class = django_object_type._meta.filterset_class if filterset_class: filterset = filterset_class(data=args, queryset=queryset, request=info.context) + if not filterset.is_valid(): + return [] return filterset.qs return queryset From eee1a0e10a9f54dcacaa541ee978f35893991eec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?aron=20bergur=20j=C3=B3hannsson?= Date: Thu, 16 Feb 2023 09:08:30 +0000 Subject: [PATCH 16/38] change empty list to qs.none() --- netbox/netbox/graphql/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/netbox/graphql/fields.py b/netbox/netbox/graphql/fields.py index 1e215c947..7c359e82e 100644 --- a/netbox/netbox/graphql/fields.py +++ b/netbox/netbox/graphql/fields.py @@ -61,7 +61,7 @@ class ObjectListField(DjangoListField): if filterset_class: filterset = filterset_class(data=args, queryset=queryset, request=info.context) if not filterset.is_valid(): - return [] + return queryset.none() return filterset.qs return queryset From 2db181ea4989d6d55a28f0600afdeac29b25d796 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aron=20Bergur=20J=C3=B3hannsson?= <71449504+aronbj20@users.noreply.github.com> Date: Thu, 16 Feb 2023 16:26:22 +0000 Subject: [PATCH 17/38] Closes #11592: Expose FILE_UPLOAD_MAX_MEMORY_SIZE as a setting (#11742) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Closes #11592: Expose FILE_UPLOAD_MAX_MEMOMORY_SIZE as a setting * change configuration settings to alphabetic order * Small example and documentation --------- Co-authored-by: aron bergur jóhannsson --- docs/configuration/miscellaneous.md | 8 ++++++++ netbox/netbox/configuration_example.py | 4 ++++ netbox/netbox/settings.py | 1 + 3 files changed, 13 insertions(+) diff --git a/docs/configuration/miscellaneous.md b/docs/configuration/miscellaneous.md index 4eb090554..eac5d0a2f 100644 --- a/docs/configuration/miscellaneous.md +++ b/docs/configuration/miscellaneous.md @@ -69,6 +69,14 @@ By default, NetBox will permit users to create duplicate prefixes and IP address --- +## FILE_UPLOAD_MAX_MEMORY_SIZE + +Default: 2621440 (i.e. 2.5 MB). + +The maximum size (in bytes) that an upload will be before it gets streamed to the file system. Changing this setting can be useful for example to be able to upload files bigger than 2.5MB to custom scripts for processing. + +--- + ## GRAPHQL_ENABLED !!! tip "Dynamic Configuration Parameter" diff --git a/netbox/netbox/configuration_example.py b/netbox/netbox/configuration_example.py index 14fcde022..7158308af 100644 --- a/netbox/netbox/configuration_example.py +++ b/netbox/netbox/configuration_example.py @@ -217,6 +217,10 @@ RQ_DEFAULT_TIMEOUT = 300 # this setting is derived from the installed location. # SCRIPTS_ROOT = '/opt/netbox/netbox/scripts' +# The maximum size (in bytes) that an upload will be before it gets streamed to the file system. +# Useful to be able to upload files bigger than 2.5Mbyte to custom scripts for processing. +# FILE_UPLOAD_MAX_MEMORY_SIZE = 2621440 + # The name to use for the csrf token cookie. CSRF_COOKIE_NAME = 'csrftoken' diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index cda6ee643..f6ce7ff33 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -91,6 +91,7 @@ DOCS_ROOT = getattr(configuration, 'DOCS_ROOT', os.path.join(os.path.dirname(BAS EMAIL = getattr(configuration, 'EMAIL', {}) EXEMPT_VIEW_PERMISSIONS = getattr(configuration, 'EXEMPT_VIEW_PERMISSIONS', []) FIELD_CHOICES = getattr(configuration, 'FIELD_CHOICES', {}) +FILE_UPLOAD_MAX_MEMORY_SIZE = getattr(configuration, 'FILE_UPLOAD_MAX_MEMORY_SIZE', 2621440) HTTP_PROXIES = getattr(configuration, 'HTTP_PROXIES', None) INTERNAL_IPS = getattr(configuration, 'INTERNAL_IPS', ('127.0.0.1', '::1')) JINJA2_FILTERS = getattr(configuration, 'JINJA2_FILTERS', {}) From 3a4fee4e6e6ecc40141d9b6dec6c59ed1a1da96d Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Thu, 16 Feb 2023 20:15:48 -0500 Subject: [PATCH 18/38] Changelog for #11226, #11335, #11473, #11592 --- docs/release-notes/version-3.4.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/release-notes/version-3.4.md b/docs/release-notes/version-3.4.md index 58c3aea49..9eb0f0cec 100644 --- a/docs/release-notes/version-3.4.md +++ b/docs/release-notes/version-3.4.md @@ -2,10 +2,17 @@ ## v3.4.5 (FUTURE) +### Enhancements + +* [#11592](https://github.com/netbox-community/netbox/issues/11592) - Introduce `FILE_UPLOAD_MAX_MEMORY_SIZE` configuration parameter + ### Bug Fixes * [#11032](https://github.com/netbox-community/netbox/issues/11032) - Fix false custom validation errors during component creation +* [#11226](https://github.com/netbox-community/netbox/issues/11226) - Ensure scripts and reports within submodules are automatically reloaded +* [#11335](https://github.com/netbox-community/netbox/issues/11335) - Avoid exception when rendering change log after uninstalling a plugin * [#11459](https://github.com/netbox-community/netbox/issues/11459) - Enable evaluating null values in custom validation rules +* [#11473](https://github.com/netbox-community/netbox/issues/11473) - GraphQL requests specifying an invalid filter should return an empty queryset * [#11582](https://github.com/netbox-community/netbox/issues/11582) - Ensure form validation errors are displayed when adding virtual chassis members * [#11601](https://github.com/netbox-community/netbox/issues/11601) - Fix partial matching of start/end addresses for IP range search * [#11683](https://github.com/netbox-community/netbox/issues/11683) - Fix CSV header attribute detection when auto-detecting import format From c36e7a1d0b3372844d1cfd1ad8c6482a5fc154fb Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Fri, 17 Feb 2023 10:11:39 -0500 Subject: [PATCH 19/38] Update introduction doc --- docs/introduction.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/introduction.md b/docs/introduction.md index fe82e68aa..bfa0900cb 100644 --- a/docs/introduction.md +++ b/docs/introduction.md @@ -4,7 +4,7 @@ NetBox was originally developed by its lead maintainer, [Jeremy Stretch](https://github.com/jeremystretch), while he was working as a network engineer at [DigitalOcean](https://www.digitalocean.com/) in 2015 as part of an effort to automate their network provisioning. Recognizing the new tool's potential, DigitalOcean agreed to release it as an open source project in June 2016. -Since then, thousands of organizations around the world have embraced NetBox as their central network source of truth to empower both network operators and automation. +Since then, thousands of organizations around the world have embraced NetBox as their central network source of truth to empower both network operators and automation. Today, the open source project is stewarded by [NetBox Labs](https://netboxlabs.com/) and a team of volunteer maintainers. Beyond the core product, myriad [plugins](https://netbox.dev/plugins/) have been developed by the NetBox community to enhance and expand its feature set. ## Key Features @@ -17,6 +17,7 @@ NetBox was built specifically to serve the needs of network engineers and operat * AS number (ASN) management * Rack elevations with SVG rendering * Device modeling using pre-defined types +* Virtual chassis and device contexts * Network, power, and console cabling with SVG traces * Power distribution modeling * Data circuit and provider tracking @@ -29,12 +30,13 @@ NetBox was built specifically to serve the needs of network engineers and operat * Tenant ownership assignment * Device & VM configuration contexts for advanced configuration rendering * Custom fields for data model extension -* Support for custom validation rules +* Custom validation rules * Custom reports & scripts executable directly within the UI * Extensive plugin framework for adding custom functionality * Single sign-on (SSO) authentication * Robust object-based permissions * Detailed, automatic change logging +* Global search engine * NAPALM integration ## What NetBox Is Not From c031951f4b06ce63f6fb55bf06374f391cc6f91c Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Fri, 17 Feb 2023 16:50:10 -0500 Subject: [PATCH 20/38] Closes #11110: Add start_address and end_address filters for IP ranges --- docs/release-notes/version-3.4.md | 1 + netbox/ipam/filtersets.py | 14 ++++++++++++++ netbox/ipam/tests/test_filtersets.py | 8 ++++++++ 3 files changed, 23 insertions(+) diff --git a/docs/release-notes/version-3.4.md b/docs/release-notes/version-3.4.md index 9eb0f0cec..edce381c0 100644 --- a/docs/release-notes/version-3.4.md +++ b/docs/release-notes/version-3.4.md @@ -4,6 +4,7 @@ ### Enhancements +* [#11110](https://github.com/netbox-community/netbox/issues/11110) - Add `start_address` and `end_address` filters for IP ranges * [#11592](https://github.com/netbox-community/netbox/issues/11592) - Introduce `FILE_UPLOAD_MAX_MEMORY_SIZE` configuration parameter ### Bug Fixes diff --git a/netbox/ipam/filtersets.py b/netbox/ipam/filtersets.py index c312b02ff..2e9f56bbc 100644 --- a/netbox/ipam/filtersets.py +++ b/netbox/ipam/filtersets.py @@ -405,6 +405,14 @@ class IPRangeFilterSet(TenancyFilterSet, NetBoxModelFilterSet): field_name='start_address', lookup_expr='family' ) + start_address = MultiValueCharFilter( + method='filter_address', + label=_('Address'), + ) + end_address = MultiValueCharFilter( + method='filter_address', + label=_('Address'), + ) contains = django_filters.CharFilter( method='search_contains', label=_('Ranges which contain this prefix or IP'), @@ -461,6 +469,12 @@ class IPRangeFilterSet(TenancyFilterSet, NetBoxModelFilterSet): except (AddrFormatError, ValueError): return queryset.none() + def filter_address(self, queryset, name, value): + try: + return queryset.filter(**{f'{name}__net_in': value}) + except ValidationError: + return queryset.none() + class IPAddressFilterSet(NetBoxModelFilterSet, TenancyFilterSet): family = django_filters.NumberFilter( diff --git a/netbox/ipam/tests/test_filtersets.py b/netbox/ipam/tests/test_filtersets.py index 711009a7e..13b3ae163 100644 --- a/netbox/ipam/tests/test_filtersets.py +++ b/netbox/ipam/tests/test_filtersets.py @@ -680,6 +680,14 @@ class IPRangeTestCase(TestCase, ChangeLoggedFilterSetTests): params = {'family': '6'} self.assertEqual(self.filterset(params, self.queryset).qs.count(), 4) + def test_start_address(self): + params = {'start_address': ['10.0.1.100', '10.0.2.100']} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + + def test_end_address(self): + params = {'end_address': ['10.0.1.199', '10.0.2.199']} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + def test_contains(self): params = {'contains': '10.0.1.150/24'} self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) From 126f9ba05f5690dcebc00a2c9de2d4f310f00b6a Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Fri, 17 Feb 2023 16:57:52 -0500 Subject: [PATCH 21/38] Raise stale timers from 60/30 to 90/30 --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index ab259af2a..3b37aae56 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -24,7 +24,7 @@ jobs: necessary. close-pr-message: > This PR has been automatically closed due to lack of activity. - days-before-stale: 60 + days-before-stale: 90 days-before-close: 30 exempt-issue-labels: 'status: accepted,status: blocked,status: needs milestone' operations-per-run: 100 From afc752b4ce5d87702ffd1c9a6f600bcf82db8680 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Fri, 17 Feb 2023 21:31:19 -0500 Subject: [PATCH 22/38] Fixes #11723: Circuit terminations should link to their associated circuits (rather than site or provider network) --- docs/release-notes/version-3.4.md | 1 + netbox/circuits/models/circuits.py | 6 ++---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/release-notes/version-3.4.md b/docs/release-notes/version-3.4.md index edce381c0..636755ac4 100644 --- a/docs/release-notes/version-3.4.md +++ b/docs/release-notes/version-3.4.md @@ -18,6 +18,7 @@ * [#11601](https://github.com/netbox-community/netbox/issues/11601) - Fix partial matching of start/end addresses for IP range search * [#11683](https://github.com/netbox-community/netbox/issues/11683) - Fix CSV header attribute detection when auto-detecting import format * [#11711](https://github.com/netbox-community/netbox/issues/11711) - Fix CSV import for multiple-object custom fields +* [#11723](https://github.com/netbox-community/netbox/issues/11723) - Circuit terminations should link to their associated circuits (rather than site or provider network) --- diff --git a/netbox/circuits/models/circuits.py b/netbox/circuits/models/circuits.py index 8ef5761fd..eba7f4de0 100644 --- a/netbox/circuits/models/circuits.py +++ b/netbox/circuits/models/circuits.py @@ -196,12 +196,10 @@ class CircuitTermination( ) def __str__(self): - return f'Termination {self.term_side}: {self.site or self.provider_network}' + return f'{self.circuit}: Termination {self.term_side}' def get_absolute_url(self): - if self.site: - return self.site.get_absolute_url() - return self.provider_network.get_absolute_url() + return self.circuit.get_absolute_url() def clean(self): super().clean() From 315371bf7c5f51c49609dd8b393b0f348a36e10e Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Sun, 19 Feb 2023 16:17:57 -0500 Subject: [PATCH 23/38] Fixes #11786: List only applicable object types in form widget when filtering custom fields --- docs/release-notes/version-3.4.md | 1 + netbox/extras/forms/filtersets.py | 18 ++++++------------ 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/docs/release-notes/version-3.4.md b/docs/release-notes/version-3.4.md index 636755ac4..731400a92 100644 --- a/docs/release-notes/version-3.4.md +++ b/docs/release-notes/version-3.4.md @@ -19,6 +19,7 @@ * [#11683](https://github.com/netbox-community/netbox/issues/11683) - Fix CSV header attribute detection when auto-detecting import format * [#11711](https://github.com/netbox-community/netbox/issues/11711) - Fix CSV import for multiple-object custom fields * [#11723](https://github.com/netbox-community/netbox/issues/11723) - Circuit terminations should link to their associated circuits (rather than site or provider network) +* [#11786](https://github.com/netbox-community/netbox/issues/11786) - List only applicable object types in form widget when filtering custom fields --- diff --git a/netbox/extras/forms/filtersets.py b/netbox/extras/forms/filtersets.py index e6a9089bc..22c7364db 100644 --- a/netbox/extras/forms/filtersets.py +++ b/netbox/extras/forms/filtersets.py @@ -38,8 +38,7 @@ class CustomFieldFilterForm(SavedFiltersMixin, FilterForm): ('Attributes', ('type', 'content_type_id', 'group_name', 'weight', 'required', 'ui_visibility')), ) content_type_id = ContentTypeMultipleChoiceField( - queryset=ContentType.objects.all(), - limit_choices_to=FeatureQuery('custom_fields'), + queryset=ContentType.objects.filter(FeatureQuery('custom_fields').get_query()), required=False, label=_('Object type') ) @@ -79,8 +78,7 @@ class JobResultFilterForm(SavedFiltersMixin, FilterForm): ) obj_type = ContentTypeChoiceField( label=_('Object Type'), - queryset=ContentType.objects.all(), - limit_choices_to=FeatureQuery('job_results'), # TODO: This doesn't actually work + queryset=ContentType.objects.filter(FeatureQuery('job_results').get_query()), required=False, ) status = MultipleChoiceField( @@ -135,8 +133,7 @@ class CustomLinkFilterForm(SavedFiltersMixin, FilterForm): ('Attributes', ('content_types', 'enabled', 'new_window', 'weight')), ) content_types = ContentTypeMultipleChoiceField( - queryset=ContentType.objects.all(), - limit_choices_to=FeatureQuery('custom_links'), + queryset=ContentType.objects.filter(FeatureQuery('custom_links').get_query()), required=False ) enabled = forms.NullBooleanField( @@ -162,8 +159,7 @@ class ExportTemplateFilterForm(SavedFiltersMixin, FilterForm): ('Attributes', ('content_types', 'mime_type', 'file_extension', 'as_attachment')), ) content_types = ContentTypeMultipleChoiceField( - queryset=ContentType.objects.all(), - limit_choices_to=FeatureQuery('export_templates'), + queryset=ContentType.objects.filter(FeatureQuery('export_templates').get_query()), required=False ) mime_type = forms.CharField( @@ -187,8 +183,7 @@ class SavedFilterFilterForm(SavedFiltersMixin, FilterForm): ('Attributes', ('content_types', 'enabled', 'shared', 'weight')), ) content_types = ContentTypeMultipleChoiceField( - queryset=ContentType.objects.all(), - limit_choices_to=FeatureQuery('export_templates'), + queryset=ContentType.objects.filter(FeatureQuery('export_templates').get_query()), required=False ) enabled = forms.NullBooleanField( @@ -215,8 +210,7 @@ class WebhookFilterForm(SavedFiltersMixin, FilterForm): ('Events', ('type_create', 'type_update', 'type_delete')), ) content_type_id = ContentTypeMultipleChoiceField( - queryset=ContentType.objects.all(), - limit_choices_to=FeatureQuery('webhooks'), + queryset=ContentType.objects.filter(FeatureQuery('webhooks').get_query()), required=False, label=_('Object type') ) From ce166b12ce1144fc110a974d053902c4c7be323c Mon Sep 17 00:00:00 2001 From: kkthxbye-code Date: Mon, 6 Feb 2023 14:00:34 +0100 Subject: [PATCH 24/38] Proof of concept for showing containing prefixes when searching for ip-addresses. --- netbox/extras/lookups.py | 15 ++++++++++++++- netbox/netbox/search/__init__.py | 7 +++++++ netbox/netbox/search/backends.py | 23 +++++++++++++++-------- 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/netbox/extras/lookups.py b/netbox/extras/lookups.py index 7197efcfc..4cdda52b4 100644 --- a/netbox/extras/lookups.py +++ b/netbox/extras/lookups.py @@ -1,4 +1,4 @@ -from django.db.models import CharField, Lookup +from django.db.models import CharField, TextField, Lookup class Empty(Lookup): @@ -14,4 +14,17 @@ class Empty(Lookup): return 'CAST(LENGTH(%s) AS BOOLEAN) != %s' % (lhs, rhs), params +class NetContainsOrEquals(Lookup): + """ + This lookup has the same functionality as the one from the ipam app except lhs is cast to inet + """ + lookup_name = 'net_contains_or_equals' + + def as_sql(self, qn, connection): + lhs, lhs_params = self.process_lhs(qn, connection) + rhs, rhs_params = self.process_rhs(qn, connection) + params = lhs_params + rhs_params + return 'CAST(%s as inet) >>= %s' % (lhs, rhs), params + CharField.register_lookup(Empty) +TextField.register_lookup(NetContainsOrEquals) diff --git a/netbox/netbox/search/__init__.py b/netbox/netbox/search/__init__.py index 1eec8e097..f4cc07c4d 100644 --- a/netbox/netbox/search/__init__.py +++ b/netbox/netbox/search/__init__.py @@ -2,6 +2,7 @@ from collections import namedtuple from django.db import models +from ipam.fields import IPAddressField, IPNetworkField from netbox.registry import registry ObjectFieldValue = namedtuple('ObjectFieldValue', ('name', 'type', 'weight', 'value')) @@ -11,6 +12,8 @@ class FieldTypes: FLOAT = 'float' INTEGER = 'int' STRING = 'str' + INET = 'inet' + CIDR = 'cidr' class LookupTypes: @@ -43,6 +46,10 @@ class SearchIndex: field_cls = instance._meta.get_field(field_name).__class__ if issubclass(field_cls, (models.FloatField, models.DecimalField)): return FieldTypes.FLOAT + if issubclass(field_cls, IPAddressField): + return FieldTypes.INET + if issubclass(field_cls, (IPNetworkField)): + return FieldTypes.CIDR if issubclass(field_cls, models.IntegerField): return FieldTypes.INTEGER return FieldTypes.STRING diff --git a/netbox/netbox/search/backends.py b/netbox/netbox/search/backends.py index d659a7abb..6bbfdd7d1 100644 --- a/netbox/netbox/search/backends.py +++ b/netbox/netbox/search/backends.py @@ -3,10 +3,12 @@ from collections import defaultdict from django.conf import settings from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ImproperlyConfigured -from django.db.models import F, Window +from django.db.models import F, Window, Q from django.db.models.functions import window from django.db.models.signals import post_delete, post_save from django.utils.module_loading import import_string +import netaddr +from netaddr.core import AddrFormatError from extras.models import CachedValue, CustomField from netbox.registry import registry @@ -95,18 +97,23 @@ class CachedValueSearchBackend(SearchBackend): def search(self, value, user=None, object_types=None, lookup=DEFAULT_LOOKUP_TYPE): - # Define the search parameters - params = { - f'value__{lookup}': value - } + query_filter = Q(**{f'value__{lookup}': value}) + if lookup in (LookupTypes.STARTSWITH, LookupTypes.ENDSWITH): # Partial string matches are valid only on string values - params['type'] = FieldTypes.STRING + query_filter &= Q(type=FieldTypes.STRING) if object_types: - params['object_type__in'] = object_types + query_filter &= Q(object_typeo__in=object_types) + + if lookup == LookupTypes.PARTIAL: + try: + address = str(netaddr.IPNetwork(value.strip()).cidr) + query_filter |= Q(type=FieldTypes.CIDR) & Q(value__net_contains_or_equals=address) + except (AddrFormatError, ValueError): + pass # Construct the base queryset to retrieve matching results - queryset = CachedValue.objects.filter(**params).annotate( + queryset = CachedValue.objects.filter(query_filter).annotate( # Annotate the rank of each result for its object according to its weight row_number=Window( expression=window.RowNumber(), From a61e7e7c04bde1a7642a264d46430c1dbc06f4e5 Mon Sep 17 00:00:00 2001 From: kkthxbye-code Date: Mon, 6 Feb 2023 21:25:24 +0100 Subject: [PATCH 25/38] Fix typo in search query --- netbox/netbox/search/backends.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/netbox/search/backends.py b/netbox/netbox/search/backends.py index 6bbfdd7d1..2e550a879 100644 --- a/netbox/netbox/search/backends.py +++ b/netbox/netbox/search/backends.py @@ -103,7 +103,7 @@ class CachedValueSearchBackend(SearchBackend): # Partial string matches are valid only on string values query_filter &= Q(type=FieldTypes.STRING) if object_types: - query_filter &= Q(object_typeo__in=object_types) + query_filter &= Q(object_type__in=object_types) if lookup == LookupTypes.PARTIAL: try: From eed1b8f4126d5cca6d7aa5a03e8d2e2882fce7c3 Mon Sep 17 00:00:00 2001 From: kkthxbye-code Date: Tue, 7 Feb 2023 09:40:25 +0100 Subject: [PATCH 26/38] Create CachedValueField to contain search specific lookups --- netbox/extras/fields.py | 7 +++++++ netbox/extras/lookups.py | 4 ++-- .../0085_change_cachedvalue_value_type.py | 19 +++++++++++++++++++ netbox/extras/models/search.py | 3 ++- 4 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 netbox/extras/fields.py create mode 100644 netbox/extras/migrations/0085_change_cachedvalue_value_type.py diff --git a/netbox/extras/fields.py b/netbox/extras/fields.py new file mode 100644 index 000000000..ffd66801d --- /dev/null +++ b/netbox/extras/fields.py @@ -0,0 +1,7 @@ +from django.db.models import TextField + +class CachedValueField(TextField): + """ + Currently a dummy field to prevent custom lookups being applied globally to TextField. + """ + pass \ No newline at end of file diff --git a/netbox/extras/lookups.py b/netbox/extras/lookups.py index 4cdda52b4..a52ef2e8d 100644 --- a/netbox/extras/lookups.py +++ b/netbox/extras/lookups.py @@ -1,5 +1,5 @@ from django.db.models import CharField, TextField, Lookup - +from .fields import CachedValueField class Empty(Lookup): """ @@ -27,4 +27,4 @@ class NetContainsOrEquals(Lookup): return 'CAST(%s as inet) >>= %s' % (lhs, rhs), params CharField.register_lookup(Empty) -TextField.register_lookup(NetContainsOrEquals) +CachedValueField.register_lookup(NetContainsOrEquals) diff --git a/netbox/extras/migrations/0085_change_cachedvalue_value_type.py b/netbox/extras/migrations/0085_change_cachedvalue_value_type.py new file mode 100644 index 000000000..e6512753e --- /dev/null +++ b/netbox/extras/migrations/0085_change_cachedvalue_value_type.py @@ -0,0 +1,19 @@ +# Generated by Django 4.1.6 on 2023-02-07 08:21 + +from django.db import migrations +import extras.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('extras', '0084_staging'), + ] + + operations = [ + migrations.AlterField( + model_name='cachedvalue', + name='value', + field=extras.fields.CachedValueField(), + ), + ] diff --git a/netbox/extras/models/search.py b/netbox/extras/models/search.py index 7c5860e00..6d088abb0 100644 --- a/netbox/extras/models/search.py +++ b/netbox/extras/models/search.py @@ -4,6 +4,7 @@ from django.contrib.contenttypes.models import ContentType from django.db import models from utilities.fields import RestrictedGenericForeignKey +from ..fields import CachedValueField __all__ = ( 'CachedValue', @@ -36,7 +37,7 @@ class CachedValue(models.Model): type = models.CharField( max_length=30 ) - value = models.TextField() + value = CachedValueField() weight = models.PositiveSmallIntegerField( default=1000 ) From 18ea7d1e13c71679c0f03d1c79fcd6ac78c8779c Mon Sep 17 00:00:00 2001 From: kkthxbye-code Date: Tue, 7 Feb 2023 13:24:57 +0100 Subject: [PATCH 27/38] pep8 fixes --- netbox/extras/fields.py | 3 ++- netbox/extras/lookups.py | 2 ++ netbox/netbox/search/backends.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/netbox/extras/fields.py b/netbox/extras/fields.py index ffd66801d..6cd44432f 100644 --- a/netbox/extras/fields.py +++ b/netbox/extras/fields.py @@ -1,7 +1,8 @@ from django.db.models import TextField + class CachedValueField(TextField): """ Currently a dummy field to prevent custom lookups being applied globally to TextField. """ - pass \ No newline at end of file + pass diff --git a/netbox/extras/lookups.py b/netbox/extras/lookups.py index a52ef2e8d..d4ed2b6a4 100644 --- a/netbox/extras/lookups.py +++ b/netbox/extras/lookups.py @@ -1,6 +1,7 @@ from django.db.models import CharField, TextField, Lookup from .fields import CachedValueField + class Empty(Lookup): """ Filter on whether a string is empty. @@ -26,5 +27,6 @@ class NetContainsOrEquals(Lookup): params = lhs_params + rhs_params return 'CAST(%s as inet) >>= %s' % (lhs, rhs), params + CharField.register_lookup(Empty) CachedValueField.register_lookup(NetContainsOrEquals) diff --git a/netbox/netbox/search/backends.py b/netbox/netbox/search/backends.py index 2e550a879..53a4fe683 100644 --- a/netbox/netbox/search/backends.py +++ b/netbox/netbox/search/backends.py @@ -108,7 +108,7 @@ class CachedValueSearchBackend(SearchBackend): if lookup == LookupTypes.PARTIAL: try: address = str(netaddr.IPNetwork(value.strip()).cidr) - query_filter |= Q(type=FieldTypes.CIDR) & Q(value__net_contains_or_equals=address) + query_filter |= Q(type=FieldTypes.CIDR) & Q(value__net_contains_or_equals=address) except (AddrFormatError, ValueError): pass From fc7cb106c1889a0a2a27b8bbdb6ad49118a10eb4 Mon Sep 17 00:00:00 2001 From: kkthxbye-code Date: Sat, 18 Feb 2023 19:29:57 +0100 Subject: [PATCH 28/38] Address feedback --- netbox/extras/lookups.py | 2 +- netbox/netbox/search/__init__.py | 2 +- netbox/netbox/search/backends.py | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/netbox/extras/lookups.py b/netbox/extras/lookups.py index d4ed2b6a4..77fe2301e 100644 --- a/netbox/extras/lookups.py +++ b/netbox/extras/lookups.py @@ -25,7 +25,7 @@ class NetContainsOrEquals(Lookup): lhs, lhs_params = self.process_lhs(qn, connection) rhs, rhs_params = self.process_rhs(qn, connection) params = lhs_params + rhs_params - return 'CAST(%s as inet) >>= %s' % (lhs, rhs), params + return 'CAST(%s AS INET) >>= %s' % (lhs, rhs), params CharField.register_lookup(Empty) diff --git a/netbox/netbox/search/__init__.py b/netbox/netbox/search/__init__.py index f4cc07c4d..6d53e9a97 100644 --- a/netbox/netbox/search/__init__.py +++ b/netbox/netbox/search/__init__.py @@ -48,7 +48,7 @@ class SearchIndex: return FieldTypes.FLOAT if issubclass(field_cls, IPAddressField): return FieldTypes.INET - if issubclass(field_cls, (IPNetworkField)): + if issubclass(field_cls, IPNetworkField): return FieldTypes.CIDR if issubclass(field_cls, models.IntegerField): return FieldTypes.INTEGER diff --git a/netbox/netbox/search/backends.py b/netbox/netbox/search/backends.py index 53a4fe683..10e164c09 100644 --- a/netbox/netbox/search/backends.py +++ b/netbox/netbox/search/backends.py @@ -99,11 +99,12 @@ class CachedValueSearchBackend(SearchBackend): query_filter = Q(**{f'value__{lookup}': value}) + if object_types: + query_filter &= Q(object_type__in=object_types) + if lookup in (LookupTypes.STARTSWITH, LookupTypes.ENDSWITH): # Partial string matches are valid only on string values query_filter &= Q(type=FieldTypes.STRING) - if object_types: - query_filter &= Q(object_type__in=object_types) if lookup == LookupTypes.PARTIAL: try: From 25278becef6c6d6d3c8556931444a333c033e179 Mon Sep 17 00:00:00 2001 From: kkthxbye-code Date: Sat, 18 Feb 2023 19:30:29 +0100 Subject: [PATCH 29/38] Change Prefix and Aggregate search index weights to better order search results. --- netbox/ipam/search.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/netbox/ipam/search.py b/netbox/ipam/search.py index fd6db6a63..ad4403321 100644 --- a/netbox/ipam/search.py +++ b/netbox/ipam/search.py @@ -6,7 +6,7 @@ from netbox.search import SearchIndex, register_search class AggregateIndex(SearchIndex): model = models.Aggregate fields = ( - ('prefix', 100), + ('prefix', 120), ('description', 500), ('date_added', 2000), ('comments', 5000), @@ -70,7 +70,7 @@ class L2VPNIndex(SearchIndex): class PrefixIndex(SearchIndex): model = models.Prefix fields = ( - ('prefix', 100), + ('prefix', 110), ('description', 500), ('comments', 5000), ) From 9efc4689cca381ec699048ef87ed669b3faabb5b Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Sun, 19 Feb 2023 18:52:26 -0500 Subject: [PATCH 30/38] Changelog for #11685 --- docs/release-notes/version-3.4.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/release-notes/version-3.4.md b/docs/release-notes/version-3.4.md index 731400a92..485b85719 100644 --- a/docs/release-notes/version-3.4.md +++ b/docs/release-notes/version-3.4.md @@ -6,6 +6,7 @@ * [#11110](https://github.com/netbox-community/netbox/issues/11110) - Add `start_address` and `end_address` filters for IP ranges * [#11592](https://github.com/netbox-community/netbox/issues/11592) - Introduce `FILE_UPLOAD_MAX_MEMORY_SIZE` configuration parameter +* [#11685](https://github.com/netbox-community/netbox/issues/11685) - Match on containing prefixes and aggregates when querying for IP addresses using global search ### Bug Fixes From e635e3e9595e15dd197bebe0a9d885afcb3e4921 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Sun, 19 Feb 2023 18:53:46 -0500 Subject: [PATCH 31/38] Fixes #11658: Remove reindex command call from search migration --- netbox/extras/migrations/0083_search.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/netbox/extras/migrations/0083_search.py b/netbox/extras/migrations/0083_search.py index 0c53de638..349066918 100644 --- a/netbox/extras/migrations/0083_search.py +++ b/netbox/extras/migrations/0083_search.py @@ -1,27 +1,10 @@ -import sys import uuid import django.db.models.deletion import django.db.models.lookups -from django.core import management from django.db import migrations, models -def reindex(apps, schema_editor): - # Build the search index (except during tests) - if 'test' not in sys.argv: - management.call_command( - 'reindex', - 'circuits', - 'dcim', - 'extras', - 'ipam', - 'tenancy', - 'virtualization', - 'wireless', - ) - - class Migration(migrations.Migration): dependencies = [ @@ -57,8 +40,4 @@ class Migration(migrations.Migration): 'ordering': ('weight', 'object_type', 'object_id'), }, ), - migrations.RunPython( - code=reindex, - reverse_code=migrations.RunPython.noop - ), ] From cd09501d4d0765f03b7c1dc243e88905386559f0 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Sun, 19 Feb 2023 20:08:57 -0500 Subject: [PATCH 32/38] #11685: Omit no-op migration --- netbox/extras/migrations/0083_search.py | 3 ++- .../0085_change_cachedvalue_value_type.py | 19 ------------------- 2 files changed, 2 insertions(+), 20 deletions(-) delete mode 100644 netbox/extras/migrations/0085_change_cachedvalue_value_type.py diff --git a/netbox/extras/migrations/0083_search.py b/netbox/extras/migrations/0083_search.py index 349066918..4c7ae1084 100644 --- a/netbox/extras/migrations/0083_search.py +++ b/netbox/extras/migrations/0083_search.py @@ -3,6 +3,7 @@ import uuid import django.db.models.deletion import django.db.models.lookups from django.db import migrations, models +import extras.fields class Migration(migrations.Migration): @@ -32,7 +33,7 @@ class Migration(migrations.Migration): ('object_id', models.PositiveBigIntegerField()), ('field', models.CharField(max_length=200)), ('type', models.CharField(max_length=30)), - ('value', models.TextField()), + ('value', extras.fields.CachedValueField()), ('weight', models.PositiveSmallIntegerField(default=1000)), ('object_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='contenttypes.contenttype')), ], diff --git a/netbox/extras/migrations/0085_change_cachedvalue_value_type.py b/netbox/extras/migrations/0085_change_cachedvalue_value_type.py deleted file mode 100644 index e6512753e..000000000 --- a/netbox/extras/migrations/0085_change_cachedvalue_value_type.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 4.1.6 on 2023-02-07 08:21 - -from django.db import migrations -import extras.fields - - -class Migration(migrations.Migration): - - dependencies = [ - ('extras', '0084_staging'), - ] - - operations = [ - migrations.AlterField( - model_name='cachedvalue', - name='value', - field=extras.fields.CachedValueField(), - ), - ] From 0855ff8b420b80356e24791c7264af18c84d0d50 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Fri, 17 Feb 2023 20:30:35 -0500 Subject: [PATCH 33/38] Skip clearing cache when handling new objects --- netbox/netbox/search/backends.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/netbox/netbox/search/backends.py b/netbox/netbox/search/backends.py index 10e164c09..14b5a987c 100644 --- a/netbox/netbox/search/backends.py +++ b/netbox/netbox/search/backends.py @@ -54,11 +54,11 @@ class SearchBackend: """ raise NotImplementedError - def caching_handler(self, sender, instance, **kwargs): + def caching_handler(self, sender, instance, created, **kwargs): """ Receiver for the post_save signal, responsible for caching object creation/changes. """ - self.cache(instance) + self.cache(instance, remove_existing=not created) def removal_handler(self, sender, instance, **kwargs): """ From 3e946c78d03d0166401364b76de925063a0b320c Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Mon, 20 Feb 2023 09:02:58 -0500 Subject: [PATCH 34/38] #11685: Clear cached search records for relevant IPAM objects --- .../migrations/0064_clear_search_cache.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 netbox/ipam/migrations/0064_clear_search_cache.py diff --git a/netbox/ipam/migrations/0064_clear_search_cache.py b/netbox/ipam/migrations/0064_clear_search_cache.py new file mode 100644 index 000000000..92d2b2cd7 --- /dev/null +++ b/netbox/ipam/migrations/0064_clear_search_cache.py @@ -0,0 +1,28 @@ +from django.db import migrations + + +def clear_cache(apps, schema_editor): + """ + Clear existing CachedValues referencing IPAddressFields or IPNetworkFields. (#11658 + introduced new cache record types for these.) + """ + ContentType = apps.get_model('contenttypes', 'ContentType') + CachedValue = apps.get_model('extras', 'CachedValue') + + for model_name in ('Aggregate', 'IPAddress', 'IPRange', 'Prefix'): + content_type = ContentType.objects.get(app_label='ipam', model=model_name.lower()) + CachedValue.objects.filter(object_type=content_type).delete() + + +class Migration(migrations.Migration): + + dependencies = [ + ('ipam', '0063_standardize_description_comments'), + ] + + operations = [ + migrations.RunPython( + code=clear_cache, + reverse_code=migrations.RunPython.noop + ), + ] From 5a4d8a71075a4529be8d5ef22d76d69861240f45 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Mon, 20 Feb 2023 09:39:33 -0500 Subject: [PATCH 35/38] Closes #11787: Rebuild any missing search cache entires after upgrade --- docs/release-notes/version-3.4.md | 2 ++ netbox/extras/management/commands/reindex.py | 31 ++++++++++++++------ netbox/netbox/search/backends.py | 14 ++++++++- upgrade.sh | 5 ++++ 4 files changed, 42 insertions(+), 10 deletions(-) diff --git a/docs/release-notes/version-3.4.md b/docs/release-notes/version-3.4.md index 485b85719..a006eb7bf 100644 --- a/docs/release-notes/version-3.4.md +++ b/docs/release-notes/version-3.4.md @@ -7,6 +7,7 @@ * [#11110](https://github.com/netbox-community/netbox/issues/11110) - Add `start_address` and `end_address` filters for IP ranges * [#11592](https://github.com/netbox-community/netbox/issues/11592) - Introduce `FILE_UPLOAD_MAX_MEMORY_SIZE` configuration parameter * [#11685](https://github.com/netbox-community/netbox/issues/11685) - Match on containing prefixes and aggregates when querying for IP addresses using global search +* [#11787](https://github.com/netbox-community/netbox/issues/11787) - Upgrade script will automatically rebuild missing search cache ### Bug Fixes @@ -20,6 +21,7 @@ * [#11683](https://github.com/netbox-community/netbox/issues/11683) - Fix CSV header attribute detection when auto-detecting import format * [#11711](https://github.com/netbox-community/netbox/issues/11711) - Fix CSV import for multiple-object custom fields * [#11723](https://github.com/netbox-community/netbox/issues/11723) - Circuit terminations should link to their associated circuits (rather than site or provider network) +* [#11775](https://github.com/netbox-community/netbox/issues/11775) - Skip checking for old search cache records when creating a new object * [#11786](https://github.com/netbox-community/netbox/issues/11786) - List only applicable object types in form widget when filtering custom fields --- diff --git a/netbox/extras/management/commands/reindex.py b/netbox/extras/management/commands/reindex.py index b601a1ac1..9a29c54f5 100644 --- a/netbox/extras/management/commands/reindex.py +++ b/netbox/extras/management/commands/reindex.py @@ -15,6 +15,11 @@ class Command(BaseCommand): nargs='*', help='One or more apps or models to reindex', ) + parser.add_argument( + '--lazy', + action='store_true', + help="For each model, reindex objects only if no cache entries already exist" + ) def _get_indexers(self, *model_names): indexers = {} @@ -60,14 +65,15 @@ class Command(BaseCommand): raise CommandError("No indexers found!") self.stdout.write(f'Reindexing {len(indexers)} models.') - # Clear all cached values for the specified models - self.stdout.write('Clearing cached values... ', ending='') - self.stdout.flush() - content_types = [ - ContentType.objects.get_for_model(model) for model in indexers.keys() - ] - deleted_count = search_backend.clear(content_types) - self.stdout.write(f'{deleted_count} entries deleted.') + # Clear all cached values for the specified models (if not being lazy) + if not kwargs['lazy']: + self.stdout.write('Clearing cached values... ', ending='') + self.stdout.flush() + content_types = [ + ContentType.objects.get_for_model(model) for model in indexers.keys() + ] + deleted_count = search_backend.clear(content_types) + self.stdout.write(f'{deleted_count} entries deleted.') # Index models self.stdout.write('Indexing models') @@ -76,11 +82,18 @@ class Command(BaseCommand): model_name = model._meta.model_name self.stdout.write(f' {app_label}.{model_name}... ', ending='') self.stdout.flush() + + if kwargs['lazy']: + content_type = ContentType.objects.get_for_model(model) + if cached_count := search_backend.count(object_types=[content_type]): + self.stdout.write(f'Skipping (found {cached_count} existing).') + continue + i = search_backend.cache(model.objects.iterator(), remove_existing=False) if i: self.stdout.write(f'{i} entries cached.') else: - self.stdout.write(f'None found.') + self.stdout.write(f'No objects found.') msg = f'Completed.' if total_count := search_backend.size: diff --git a/netbox/netbox/search/backends.py b/netbox/netbox/search/backends.py index 14b5a987c..f428842f5 100644 --- a/netbox/netbox/search/backends.py +++ b/netbox/netbox/search/backends.py @@ -80,7 +80,13 @@ class SearchBackend: def clear(self, object_types=None): """ - Delete *all* cached data. + Delete *all* cached data (optionally filtered by object type). + """ + raise NotImplementedError + + def count(self, object_types=None): + """ + Return a count of all cache entries (optionally filtered by object type). """ raise NotImplementedError @@ -218,6 +224,12 @@ class CachedValueSearchBackend(SearchBackend): # Call _raw_delete() on the queryset to avoid first loading instances into memory return qs._raw_delete(using=qs.db) + def count(self, object_types=None): + qs = CachedValue.objects.all() + if object_types: + qs = qs.filter(object_type__in=object_types) + return qs.count() + @property def size(self): return CachedValue.objects.count() diff --git a/upgrade.sh b/upgrade.sh index 161d65e32..cac046a9f 100755 --- a/upgrade.sh +++ b/upgrade.sh @@ -103,6 +103,11 @@ COMMAND="python3 netbox/manage.py remove_stale_contenttypes --no-input" echo "Removing stale content types ($COMMAND)..." eval $COMMAND || exit 1 +# Rebuild the search cache (lazily) +COMMAND="python3 netbox/manage.py reindex --lazy" +echo "Rebuilding search cache ($COMMAND)..." +eval $COMMAND || exit 1 + # Delete any expired user sessions COMMAND="python3 netbox/manage.py clearsessions" echo "Removing expired user sessions ($COMMAND)..." From 972ba7bfdc16b42f68b3172ad605a3e113e4ea79 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Mon, 20 Feb 2023 10:27:30 -0500 Subject: [PATCH 36/38] #11685: Fix migration --- netbox/ipam/migrations/0064_clear_search_cache.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/netbox/ipam/migrations/0064_clear_search_cache.py b/netbox/ipam/migrations/0064_clear_search_cache.py index 92d2b2cd7..856fe99e1 100644 --- a/netbox/ipam/migrations/0064_clear_search_cache.py +++ b/netbox/ipam/migrations/0064_clear_search_cache.py @@ -10,8 +10,11 @@ def clear_cache(apps, schema_editor): CachedValue = apps.get_model('extras', 'CachedValue') for model_name in ('Aggregate', 'IPAddress', 'IPRange', 'Prefix'): - content_type = ContentType.objects.get(app_label='ipam', model=model_name.lower()) - CachedValue.objects.filter(object_type=content_type).delete() + try: + content_type = ContentType.objects.get(app_label='ipam', model=model_name.lower()) + CachedValue.objects.filter(object_type=content_type).delete() + except ContentType.DoesNotExist: + pass class Migration(migrations.Migration): From 3586cf79d4cebcff81ab46fc471b32930ce2d0f1 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Tue, 21 Feb 2023 08:42:39 -0500 Subject: [PATCH 37/38] Arrange parameters alphabetically --- netbox/netbox/configuration_example.py | 34 ++++++++++++-------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/netbox/netbox/configuration_example.py b/netbox/netbox/configuration_example.py index 7158308af..92f6133a3 100644 --- a/netbox/netbox/configuration_example.py +++ b/netbox/netbox/configuration_example.py @@ -107,6 +107,9 @@ CORS_ORIGIN_REGEX_WHITELIST = [ # r'^(https?://)?(\w+\.)?example\.com$', ] +# The name to use for the CSRF token cookie. +CSRF_COOKIE_NAME = 'csrftoken' + # Set to True to enable server debugging. WARNING: Debugging introduces a substantial performance penalty and may reveal # sensitive information about your installation. Only enable debugging while performing testing. Never enable debugging # on a production system. @@ -127,6 +130,9 @@ EMAIL = { 'FROM_EMAIL': '', } +# Localization +ENABLE_LOCALIZATION = False + # Exempt certain models from the enforcement of view permissions. Models listed here will be viewable by all users and # by anonymous users. List models in the form `.`. Add '*' to this list to exempt all models. EXEMPT_VIEW_PERMISSIONS = [ @@ -168,16 +174,6 @@ LOGOUT_REDIRECT_URL = 'home' # the default value of this setting is derived from the installed location. # MEDIA_ROOT = '/opt/netbox/netbox/media' -# By default uploaded media is stored on the local filesystem. Using Django-storages is also supported. Provide the -# class path of the storage driver in STORAGE_BACKEND and any configuration options in STORAGE_CONFIG. For example: -# STORAGE_BACKEND = 'storages.backends.s3boto3.S3Boto3Storage' -# STORAGE_CONFIG = { -# 'AWS_ACCESS_KEY_ID': 'Key ID', -# 'AWS_SECRET_ACCESS_KEY': 'Secret', -# 'AWS_STORAGE_BUCKET_NAME': 'netbox', -# 'AWS_S3_REGION_NAME': 'eu-west-1', -# } - # Expose Prometheus monitoring metrics at the HTTP endpoint '/metrics' METRICS_ENABLED = False @@ -217,13 +213,6 @@ RQ_DEFAULT_TIMEOUT = 300 # this setting is derived from the installed location. # SCRIPTS_ROOT = '/opt/netbox/netbox/scripts' -# The maximum size (in bytes) that an upload will be before it gets streamed to the file system. -# Useful to be able to upload files bigger than 2.5Mbyte to custom scripts for processing. -# FILE_UPLOAD_MAX_MEMORY_SIZE = 2621440 - -# The name to use for the csrf token cookie. -CSRF_COOKIE_NAME = 'csrftoken' - # The name to use for the session cookie. SESSION_COOKIE_NAME = 'sessionid' @@ -232,8 +221,15 @@ SESSION_COOKIE_NAME = 'sessionid' # database access.) Note that the user as which NetBox runs must have read and write permissions to this path. SESSION_FILE_PATH = None -# Localization -ENABLE_LOCALIZATION = False +# By default, uploaded media is stored on the local filesystem. Using Django-storages is also supported. Provide the +# class path of the storage driver in STORAGE_BACKEND and any configuration options in STORAGE_CONFIG. For example: +# STORAGE_BACKEND = 'storages.backends.s3boto3.S3Boto3Storage' +# STORAGE_CONFIG = { +# 'AWS_ACCESS_KEY_ID': 'Key ID', +# 'AWS_SECRET_ACCESS_KEY': 'Secret', +# 'AWS_STORAGE_BUCKET_NAME': 'netbox', +# 'AWS_S3_REGION_NAME': 'eu-west-1', +# } # Time zone (default: UTC) TIME_ZONE = 'UTC' From c280ca35d69f1c968b8d454db04ae23e20ef338e Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Tue, 21 Feb 2023 08:45:52 -0500 Subject: [PATCH 38/38] Release v3.4.5 --- .github/ISSUE_TEMPLATE/bug_report.yaml | 2 +- .github/ISSUE_TEMPLATE/feature_request.yaml | 2 +- docs/configuration/miscellaneous.md | 6 +++--- docs/release-notes/version-3.4.md | 3 +-- netbox/netbox/settings.py | 2 +- requirements.txt | 12 ++++++------ 6 files changed, 13 insertions(+), 14 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yaml b/.github/ISSUE_TEMPLATE/bug_report.yaml index 9ed740fff..99c060e8a 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yaml +++ b/.github/ISSUE_TEMPLATE/bug_report.yaml @@ -14,7 +14,7 @@ body: attributes: label: NetBox version description: What version of NetBox are you currently running? - placeholder: v3.4.4 + placeholder: v3.4.5 validations: required: true - type: dropdown diff --git a/.github/ISSUE_TEMPLATE/feature_request.yaml b/.github/ISSUE_TEMPLATE/feature_request.yaml index 8e4ab54a5..1338be9c1 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yaml +++ b/.github/ISSUE_TEMPLATE/feature_request.yaml @@ -14,7 +14,7 @@ body: attributes: label: NetBox version description: What version of NetBox are you currently running? - placeholder: v3.4.4 + placeholder: v3.4.5 validations: required: true - type: dropdown diff --git a/docs/configuration/miscellaneous.md b/docs/configuration/miscellaneous.md index eac5d0a2f..8550564d8 100644 --- a/docs/configuration/miscellaneous.md +++ b/docs/configuration/miscellaneous.md @@ -69,11 +69,11 @@ By default, NetBox will permit users to create duplicate prefixes and IP address --- -## FILE_UPLOAD_MAX_MEMORY_SIZE +## `FILE_UPLOAD_MAX_MEMORY_SIZE` -Default: 2621440 (i.e. 2.5 MB). +Default: `2621440` (2.5 MB). -The maximum size (in bytes) that an upload will be before it gets streamed to the file system. Changing this setting can be useful for example to be able to upload files bigger than 2.5MB to custom scripts for processing. +The maximum amount (in bytes) of uploaded data that will be held in memory before being written to the filesystem. Changing this setting can be useful for example to be able to upload files bigger than 2.5MB to custom scripts for processing. --- diff --git a/docs/release-notes/version-3.4.md b/docs/release-notes/version-3.4.md index a006eb7bf..11df3d47a 100644 --- a/docs/release-notes/version-3.4.md +++ b/docs/release-notes/version-3.4.md @@ -1,6 +1,6 @@ # NetBox v3.4 -## v3.4.5 (FUTURE) +## v3.4.5 (2023-02-21) ### Enhancements @@ -13,7 +13,6 @@ * [#11032](https://github.com/netbox-community/netbox/issues/11032) - Fix false custom validation errors during component creation * [#11226](https://github.com/netbox-community/netbox/issues/11226) - Ensure scripts and reports within submodules are automatically reloaded -* [#11335](https://github.com/netbox-community/netbox/issues/11335) - Avoid exception when rendering change log after uninstalling a plugin * [#11459](https://github.com/netbox-community/netbox/issues/11459) - Enable evaluating null values in custom validation rules * [#11473](https://github.com/netbox-community/netbox/issues/11473) - GraphQL requests specifying an invalid filter should return an empty queryset * [#11582](https://github.com/netbox-community/netbox/issues/11582) - Ensure form validation errors are displayed when adding virtual chassis members diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index f6ce7ff33..2d56a025a 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -24,7 +24,7 @@ from netbox.constants import RQ_QUEUE_DEFAULT, RQ_QUEUE_HIGH, RQ_QUEUE_LOW # Environment setup # -VERSION = '3.4.5-dev' +VERSION = '3.4.5' # Hostname HOSTNAME = platform.node() diff --git a/requirements.txt b/requirements.txt index 3cb2529a8..8bbb80d1e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ bleach==5.0.1 -Django==4.1.6 +Django==4.1.7 django-cors-headers==3.13.0 django-debug-toolbar==3.8.1 django-filter==22.1 @@ -9,23 +9,23 @@ django-pglocks==1.0.4 django-prometheus==2.2.0 django-redis==5.2.0 django-rich==1.4.0 -django-rq==2.6.0 -django-tables2==2.5.1 +django-rq==2.7.0 +django-tables2==2.5.2 django-taggit==3.1.0 django-timezone-field==5.0 djangorestframework==3.14.0 -drf-yasg[validation]==1.21.4 +drf-yasg[validation]==1.21.5 graphene-django==3.0.0 gunicorn==20.1.0 Jinja2==3.1.2 Markdown==3.3.7 -mkdocs-material==9.0.10 +mkdocs-material==9.0.13 mkdocstrings[python-legacy]==0.20.0 netaddr==0.8.0 Pillow==9.4.0 psycopg2-binary==2.9.5 PyYAML==6.0 -sentry-sdk==1.14.0 +sentry-sdk==1.15.0 social-auth-app-django==5.0.0 social-auth-core[openidconnect]==4.3.0 svgwrite==1.4.3