From ae999c1760df39ed51eebdbf6bacbadf44b6a21b Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Thu, 2 Feb 2023 15:41:24 -0500 Subject: [PATCH 01/38] PRVB --- docs/release-notes/version-3.4.md | 4 ++++ netbox/netbox/settings.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/release-notes/version-3.4.md b/docs/release-notes/version-3.4.md index 1581ce681..15b84436d 100644 --- a/docs/release-notes/version-3.4.md +++ b/docs/release-notes/version-3.4.md @@ -1,5 +1,9 @@ # NetBox v3.4 +## v3.4.5 (FUTURE) + +--- + ## v3.4.4 (2023-02-02) ### Enhancements diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 8517efca1..cda6ee643 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -24,7 +24,7 @@ from netbox.constants import RQ_QUEUE_DEFAULT, RQ_QUEUE_HIGH, RQ_QUEUE_LOW # Environment setup # -VERSION = '3.4.4' +VERSION = '3.4.5-dev' # Hostname HOSTNAME = platform.node() From d2cad0b87e8cf8acd45a96ec9818035cba459756 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Tue, 7 Feb 2023 16:49:07 -0500 Subject: [PATCH 02/38] Display login message as success --- netbox/users/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/users/views.py b/netbox/users/views.py index 832a4e592..a82620914 100644 --- a/netbox/users/views.py +++ b/netbox/users/views.py @@ -96,7 +96,7 @@ class LoginView(View): # Authenticate user auth_login(request, form.get_user()) logger.info(f"User {request.user} successfully authenticated") - messages.info(request, f"Logged in as {request.user}.") + messages.success(request, f"Logged in as {request.user}.") # Ensure the user has a UserConfig defined. (This should normally be handled by # create_userconfig() on user creation.) From 7d12b5ad8888560c40c737761e2811a833023836 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Tue, 7 Feb 2023 16:52:54 -0500 Subject: [PATCH 03/38] Update housekeeping command docs --- docs/administration/housekeeping.md | 1 + netbox/extras/management/commands/housekeeping.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/administration/housekeeping.md b/docs/administration/housekeeping.md index da1a5443b..fcc3aa04e 100644 --- a/docs/administration/housekeeping.md +++ b/docs/administration/housekeeping.md @@ -5,6 +5,7 @@ NetBox includes a `housekeeping` management command that should be run nightly. * Clearing expired authentication sessions from the database * Deleting changelog records older than the configured [retention time](../configuration/miscellaneous.md#changelog_retention) * Deleting job result records older than the configured [retention time](../configuration/miscellaneous.md#jobresult_retention) +* Check for new NetBox releases (if [`RELEASE_CHECK_URL`](../configuration/miscellaneous.md#release_check_url) is set) This command can be invoked directly, or by using the shell script provided at `/opt/netbox/contrib/netbox-housekeeping.sh`. This script can be linked from your cron scheduler's daily jobs directory (e.g. `/etc/cron.daily`) or referenced directly within the cron configuration file. diff --git a/netbox/extras/management/commands/housekeeping.py b/netbox/extras/management/commands/housekeeping.py index 42690568d..172e26bf2 100644 --- a/netbox/extras/management/commands/housekeeping.py +++ b/netbox/extras/management/commands/housekeeping.py @@ -37,7 +37,7 @@ class Command(BaseCommand): f"clearing sessions; skipping." ) - # Delete expired ObjectRecords + # Delete expired ObjectChanges if options['verbosity']: self.stdout.write("[*] Checking for expired changelog records") if config.CHANGELOG_RETENTION: From 35a218bd7a8f662b0e0b4460d72a70ddc1476f08 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Tue, 7 Feb 2023 16:55:50 -0500 Subject: [PATCH 04/38] Add step for creating search index --- docs/development/adding-models.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/development/adding-models.md b/docs/development/adding-models.md index aef11d666..7de897a97 100644 --- a/docs/development/adding-models.md +++ b/docs/development/adding-models.md @@ -54,15 +54,19 @@ Each model should have a corresponding FilterSet class defined. This is used to Create a table class for the model in `tables.py` by subclassing `utilities.tables.BaseTable`. Under the table's `Meta` class, be sure to list both the fields and default columns. -## 9. Create the object template +## 9. Create a SearchIndex subclass + +If this model will be included in global search results, create a subclass of `netbox.search.SearchIndex` for it and specify the fields to be indexed. + +## 10. Create the object template Create the HTML template for the object view. (The other views each typically employ a generic template.) This template should extend `generic/object.html`. -## 10. Add the model to the navigation menu +## 11. Add the model to the navigation menu Add the relevant navigation menu items in `netbox/netbox/navigation/menu.py`. -## 11. REST API components +## 12. REST API components Create the following for each model: @@ -71,13 +75,13 @@ Create the following for each model: * API view in `api/views.py` * Endpoint route in `api/urls.py` -## 12. GraphQL API components +## 13. GraphQL API components Create a Graphene object type for the model in `graphql/types.py` by subclassing the appropriate class from `netbox.graphql.types`. Also extend the schema class defined in `graphql/schema.py` with the individual object and object list fields per the established convention. -## 13. Add tests +## 14. Add tests Add tests for the following: @@ -85,7 +89,7 @@ Add tests for the following: * API views * Filter sets -## 14. Documentation +## 15. Documentation Create a new documentation page for the model in `docs/models//.md`. Include this file under the "features" documentation where appropriate. From 77fa999c404558af57b412e45a0bf5712c06bcd2 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Tue, 7 Feb 2023 17:24:26 -0500 Subject: [PATCH 05/38] Fixes #11683: Fix CSV header attribute detection when auto-detecting import format --- docs/release-notes/version-3.4.md | 4 ++++ netbox/netbox/views/generic/bulk_views.py | 4 ++-- netbox/utilities/forms/forms.py | 2 ++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/release-notes/version-3.4.md b/docs/release-notes/version-3.4.md index 15b84436d..b23251a70 100644 --- a/docs/release-notes/version-3.4.md +++ b/docs/release-notes/version-3.4.md @@ -2,6 +2,10 @@ ## v3.4.5 (FUTURE) +### Bug Fixes + +* [#11683](https://github.com/netbox-community/netbox/issues/11683) - Fix CSV header attribute detection when auto-detecting import format + --- ## v3.4.4 (2023-02-02) diff --git a/netbox/netbox/views/generic/bulk_views.py b/netbox/netbox/views/generic/bulk_views.py index ab3e8f100..6060475d8 100644 --- a/netbox/netbox/views/generic/bulk_views.py +++ b/netbox/netbox/views/generic/bulk_views.py @@ -384,8 +384,8 @@ class BulkImportView(GetReturnURLMixin, BaseMultiObjectView): 'data': record, 'instance': instance, } - if form.cleaned_data['format'] == ImportFormatChoices.CSV: - model_form_kwargs['headers'] = form._csv_headers + if hasattr(form, '_csv_headers'): + model_form_kwargs['headers'] = form._csv_headers # Add CSV headers model_form = self.model_form(**model_form_kwargs) # When updating, omit all form fields other than those specified in the record. (No diff --git a/netbox/utilities/forms/forms.py b/netbox/utilities/forms/forms.py index 99d03f2a6..9884ffac5 100644 --- a/netbox/utilities/forms/forms.py +++ b/netbox/utilities/forms/forms.py @@ -197,6 +197,8 @@ class ImportForm(BootstrapMixin, forms.Form): self.cleaned_data['data'] = self._clean_json(data) elif format == ImportFormatChoices.YAML: self.cleaned_data['data'] = self._clean_yaml(data) + else: + raise forms.ValidationError(f"Unknown data format: {format}") def _detect_format(self, data): """ From a881efbde1a9e919f21390486505f77d96181c6a Mon Sep 17 00:00:00 2001 From: kkthxbye <400797+kkthxbye-code@users.noreply.github.com> Date: Wed, 8 Feb 2023 20:36:20 +0100 Subject: [PATCH 06/38] Fixes #11032 - Replication fields broken in custom validation (#11698) * Fixes #11032 - Replication fields broken in custom validation * Use getattr instead of hasattr to make sure custom validation is triggered as normal --------- Co-authored-by: kkthxbye-code <> --- netbox/netbox/models/features.py | 4 ++++ netbox/netbox/views/generic/object_views.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/netbox/netbox/models/features.py b/netbox/netbox/models/features.py index 8e5af0ab5..f041d016d 100644 --- a/netbox/netbox/models/features.py +++ b/netbox/netbox/models/features.py @@ -257,6 +257,10 @@ class CustomValidationMixin(models.Model): def clean(self): super().clean() + # If the instance is a base for replications, skip custom validation + if getattr(self, '_replicated_base', False): + return + # Send the post_clean signal post_clean.send(sender=self.__class__, instance=self) diff --git a/netbox/netbox/views/generic/object_views.py b/netbox/netbox/views/generic/object_views.py index d855490d1..475cca9d3 100644 --- a/netbox/netbox/views/generic/object_views.py +++ b/netbox/netbox/views/generic/object_views.py @@ -436,6 +436,10 @@ class ComponentCreateView(GetReturnURLMixin, BaseObjectView): form = self.initialize_form(request) instance = self.alter_object(self.queryset.model(), request) + # Note that the form instance is a replicated field base + # This is needed to avoid running custom validators multiple times + form.instance._replicated_base = hasattr(self.form, "replication_fields") + if form.is_valid(): new_components = [] data = deepcopy(request.POST) From a8b356a07802bfcbb46ee2e54e703c5c7d4fcddb Mon Sep 17 00:00:00 2001 From: kkthxbye-code Date: Wed, 8 Feb 2023 09:33:06 +0100 Subject: [PATCH 07/38] Fixes #11582: Fix missing VC form errors ### Fixes: #11582 Not sure if this is the correct fix or not. The reason that the custom field errors were not shown is that messages.html only shows non_field_errors if the form passed to the context is named form. This is probably an issue in more places, but not sure how to make it generic. A change to messages.html would also need to support formsets. Any input appreciated @jeremystretch or @arthanson --- netbox/templates/dcim/virtualchassis_add_member.html | 2 ++ netbox/templates/dcim/virtualchassis_edit.html | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/netbox/templates/dcim/virtualchassis_add_member.html b/netbox/templates/dcim/virtualchassis_add_member.html index 17ffd64d9..bc2ba2f55 100644 --- a/netbox/templates/dcim/virtualchassis_add_member.html +++ b/netbox/templates/dcim/virtualchassis_add_member.html @@ -5,6 +5,8 @@ {% block content %}
+ {% render_errors membership_form %} + {% csrf_token %}
Add New Member
diff --git a/netbox/templates/dcim/virtualchassis_edit.html b/netbox/templates/dcim/virtualchassis_edit.html index f98a9fe64..433837cf5 100644 --- a/netbox/templates/dcim/virtualchassis_edit.html +++ b/netbox/templates/dcim/virtualchassis_edit.html @@ -8,6 +8,10 @@
+ {% for form in formset %} + {% render_errors form %} + {% endfor %} + {% csrf_token %} {{ pk_form.pk }} {{ formset.management_form }} From f870931e4f81818862e3e8b7cecf770573d24658 Mon Sep 17 00:00:00 2001 From: kkthxbye-code Date: Wed, 8 Feb 2023 10:09:07 +0100 Subject: [PATCH 08/38] Fixes #11601 - Add partial lookup to IPRangeFilterSet --- netbox/ipam/filtersets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/netbox/ipam/filtersets.py b/netbox/ipam/filtersets.py index d069eed27..c312b02ff 100644 --- a/netbox/ipam/filtersets.py +++ b/netbox/ipam/filtersets.py @@ -441,9 +441,9 @@ class IPRangeFilterSet(TenancyFilterSet, NetBoxModelFilterSet): def search(self, queryset, name, value): if not value.strip(): return queryset - qs_filter = Q(description__icontains=value) + qs_filter = Q(description__icontains=value) | Q(start_address__contains=value) | Q(end_address__contains=value) try: - ipaddress = str(netaddr.IPNetwork(value.strip()).cidr) + ipaddress = str(netaddr.IPNetwork(value.strip())) qs_filter |= Q(start_address=ipaddress) qs_filter |= Q(end_address=ipaddress) except (AddrFormatError, ValueError): From 1b9fd1cf3acd48927c77ec9b699519d1a519ad44 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Wed, 8 Feb 2023 14:56:14 -0500 Subject: [PATCH 09/38] Changelog for #11032, #11582, #11601 --- docs/release-notes/version-3.4.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/release-notes/version-3.4.md b/docs/release-notes/version-3.4.md index b23251a70..b8ebe4a33 100644 --- a/docs/release-notes/version-3.4.md +++ b/docs/release-notes/version-3.4.md @@ -4,6 +4,9 @@ ### Bug Fixes +* [#11032](https://github.com/netbox-community/netbox/issues/11032) - Fix false custom validation errors during component creation +* [#11582](https://github.com/netbox-community/netbox/issues/11582) - Ensure form validation errors are displayed when adding virtual chassis members +* [#11601](https://github.com/netbox-community/netbox/issues/11601) - Fix partial matching of start/end addresses for IP range search * [#11683](https://github.com/netbox-community/netbox/issues/11683) - Fix CSV header attribute detection when auto-detecting import format --- From 5eeee728ef20806167cf0031b12fb23d1e438057 Mon Sep 17 00:00:00 2001 From: kkthxbye <400797+kkthxbye-code@users.noreply.github.com> Date: Mon, 13 Feb 2023 23:44:35 +0100 Subject: [PATCH 10/38] Fixes #11459 - Allow using null in conditions (#11722) * Fixes #11459 - Allow using null in conditions - Update docs to reflect this - Change docs example from primary_ip to primary_ip4 as computed properties are not serialized when queuing webhooks * Update netbox/extras/conditions.py --------- Co-authored-by: Simon Toft Co-authored-by: Jeremy Stretch --- docs/reference/conditions.md | 6 +++--- netbox/extras/conditions.py | 3 ++- netbox/extras/tests/test_conditions.py | 10 ++++++++++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/docs/reference/conditions.md b/docs/reference/conditions.md index fb8b66139..514006b01 100644 --- a/docs/reference/conditions.md +++ b/docs/reference/conditions.md @@ -97,7 +97,7 @@ Multiple conditions can be combined into nested sets using AND or OR logic. This ### Examples -`status` is "active" and `primary_ip` is defined _or_ the "exempt" tag is applied. +`status` is "active" and `primary_ip4` is defined _or_ the "exempt" tag is applied. ```json { @@ -109,8 +109,8 @@ Multiple conditions can be combined into nested sets using AND or OR logic. This "value": "active" }, { - "attr": "primary_ip", - "value": "", + "attr": "primary_ip4", + "value": null, "negate": true } ] diff --git a/netbox/extras/conditions.py b/netbox/extras/conditions.py index 965488c3a..c6744e524 100644 --- a/netbox/extras/conditions.py +++ b/netbox/extras/conditions.py @@ -44,7 +44,8 @@ class Condition: bool: (EQ, CONTAINS), int: (EQ, GT, GTE, LT, LTE, CONTAINS), float: (EQ, GT, GTE, LT, LTE, CONTAINS), - list: (EQ, IN, CONTAINS) + list: (EQ, IN, CONTAINS), + type(None): (EQ,) } def __init__(self, attr, value, op=EQ, negate=False): diff --git a/netbox/extras/tests/test_conditions.py b/netbox/extras/tests/test_conditions.py index 8e02eb75d..e7275482a 100644 --- a/netbox/extras/tests/test_conditions.py +++ b/netbox/extras/tests/test_conditions.py @@ -126,6 +126,16 @@ class ConditionSetTest(TestCase): with self.assertRaises(ValueError): ConditionSet({'foo': []}) + def test_null_value(self): + cs = ConditionSet({ + 'and': [ + {'attr': 'a', 'value': None, 'op': 'eq', 'negate': True}, + ] + }) + self.assertFalse(cs.eval({'a': None})) + self.assertTrue(cs.eval({'a': "string"})) + self.assertTrue(cs.eval({'a': {"key": "value"}})) + def test_and_single_depth(self): cs = ConditionSet({ 'and': [ From f57a1577cf85f5f3dadad5aae8983f2c79a98247 Mon Sep 17 00:00:00 2001 From: kkthxbye <400797+kkthxbye-code@users.noreply.github.com> Date: Mon, 13 Feb 2023 23:49:08 +0100 Subject: [PATCH 11/38] Fixes #11711 - Use CSVModelMultipleChoiceField when importing custom multiple object fields (#11712) * Fixes #11711 - Use CSVModelMultipleChoiceField when importing custom multiple object fields * Fix pep8 --------- Co-authored-by: kkthxbye-code <> --- netbox/extras/models/customfields.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/netbox/extras/models/customfields.py b/netbox/extras/models/customfields.py index 4842c0654..c7a19e4b8 100644 --- a/netbox/extras/models/customfields.py +++ b/netbox/extras/models/customfields.py @@ -21,7 +21,7 @@ from netbox.models.features import CloningMixin, ExportTemplatesMixin, WebhooksM from netbox.search import FieldTypes from utilities import filters from utilities.forms import ( - CSVChoiceField, CSVMultipleChoiceField, DatePicker, DynamicModelChoiceField, DynamicModelMultipleChoiceField, + CSVChoiceField, CSVModelMultipleChoiceField, CSVMultipleChoiceField, DatePicker, DynamicModelChoiceField, DynamicModelMultipleChoiceField, JSONField, LaxURLField, StaticSelectMultiple, StaticSelect, add_blank_choice, ) from utilities.querysets import RestrictedQuerySet @@ -422,10 +422,12 @@ class CustomField(CloningMixin, ExportTemplatesMixin, WebhooksMixin, ChangeLogge # Multiple objects elif self.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: model = self.object_type.model_class() - field = DynamicModelMultipleChoiceField( + field_class = CSVModelMultipleChoiceField if for_csv_import else DynamicModelMultipleChoiceField + + field = field_class( queryset=model.objects.all(), required=required, - initial=initial + initial=initial, ) # Text From 46004b528e5190851bfa573e8fa878fffb97f6fc Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Mon, 13 Feb 2023 17:53:01 -0500 Subject: [PATCH 12/38] #11711: Use CSVModelChoiceField for custom object fields during CSV import --- netbox/extras/models/customfields.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/netbox/extras/models/customfields.py b/netbox/extras/models/customfields.py index c7a19e4b8..fa16b8501 100644 --- a/netbox/extras/models/customfields.py +++ b/netbox/extras/models/customfields.py @@ -20,10 +20,12 @@ from netbox.models import ChangeLoggedModel from netbox.models.features import CloningMixin, ExportTemplatesMixin, WebhooksMixin from netbox.search import FieldTypes from utilities import filters -from utilities.forms import ( - CSVChoiceField, CSVModelMultipleChoiceField, CSVMultipleChoiceField, DatePicker, DynamicModelChoiceField, DynamicModelMultipleChoiceField, - JSONField, LaxURLField, StaticSelectMultiple, StaticSelect, add_blank_choice, +from utilities.forms.fields import ( + CSVChoiceField, CSVModelChoiceField, CSVModelMultipleChoiceField, CSVMultipleChoiceField, DynamicModelChoiceField, + DynamicModelMultipleChoiceField, JSONField, LaxURLField, ) +from utilities.forms.widgets import DatePicker, StaticSelectMultiple, StaticSelect +from utilities.forms.utils import add_blank_choice from utilities.querysets import RestrictedQuerySet from utilities.validators import validate_regex @@ -413,7 +415,8 @@ class CustomField(CloningMixin, ExportTemplatesMixin, WebhooksMixin, ChangeLogge # Object elif self.type == CustomFieldTypeChoices.TYPE_OBJECT: model = self.object_type.model_class() - field = DynamicModelChoiceField( + field_class = CSVModelChoiceField if for_csv_import else DynamicModelChoiceField + field = field_class( queryset=model.objects.all(), required=required, initial=initial @@ -423,7 +426,6 @@ class CustomField(CloningMixin, ExportTemplatesMixin, WebhooksMixin, ChangeLogge elif self.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: model = self.object_type.model_class() field_class = CSVModelMultipleChoiceField if for_csv_import else DynamicModelMultipleChoiceField - field = field_class( queryset=model.objects.all(), required=required, From ddb019ce7a6dcc16bf0fb7b7d323cfc4832bfd7b Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Mon, 13 Feb 2023 17:58:41 -0500 Subject: [PATCH 13/38] Changelog for #11459, #11711 --- docs/release-notes/version-3.4.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/release-notes/version-3.4.md b/docs/release-notes/version-3.4.md index b8ebe4a33..58c3aea49 100644 --- a/docs/release-notes/version-3.4.md +++ b/docs/release-notes/version-3.4.md @@ -5,9 +5,11 @@ ### Bug Fixes * [#11032](https://github.com/netbox-community/netbox/issues/11032) - Fix false custom validation errors during component creation +* [#11459](https://github.com/netbox-community/netbox/issues/11459) - Enable evaluating null values in custom validation rules * [#11582](https://github.com/netbox-community/netbox/issues/11582) - Ensure form validation errors are displayed when adding virtual chassis members * [#11601](https://github.com/netbox-community/netbox/issues/11601) - Fix partial matching of start/end addresses for IP range search * [#11683](https://github.com/netbox-community/netbox/issues/11683) - Fix CSV header attribute detection when auto-detecting import format +* [#11711](https://github.com/netbox-community/netbox/issues/11711) - Fix CSV import for multiple-object custom fields --- From dcc9dea6f91f4d79b67c7c1e29230c72cb72007a Mon Sep 17 00:00:00 2001 From: kkthxbye-code Date: Thu, 16 Feb 2023 12:33:08 +0100 Subject: [PATCH 14/38] Change the way we invalidate the module cache to support reloading code from subpackages --- netbox/extras/scripts.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/netbox/extras/scripts.py b/netbox/extras/scripts.py index 77c96de56..313058d57 100644 --- a/netbox/extras/scripts.py +++ b/netbox/extras/scripts.py @@ -524,27 +524,39 @@ def get_scripts(use_names=False): defined name in place of the actual module name. """ scripts = {} - # Iterate through all modules within the scripts path. These are the user-created files in which reports are + + # Get all modules within the scripts path. These are the user-created files in which scripts are # defined. - for importer, module_name, _ in pkgutil.iter_modules([settings.SCRIPTS_ROOT]): - # Use a lock as removing and loading modules is not thread safe - with lock: - # Remove cached module to ensure consistency with filesystem - if module_name in sys.modules: + modules = list(pkgutil.iter_modules([settings.SCRIPTS_ROOT])) + modules_bases = set([name.split(".")[0] for _, name, _ in modules]) + + # Deleting from sys.modules needs to done behind a lock to prevent race conditions where a module is + # removed from sys.modules while another thread is importing + with lock: + for module_name in list(sys.modules.keys()): + # Everything sharing a base module path with a module in the script folder is removed. + # We also remove all modules with a base module called "scripts". This allows modifying imported + # non-script modules without having to reload the RQ worker. + module_base = module_name.split(".")[0] + if module_base == "scripts" or module_base in modules_bases: del sys.modules[module_name] - module = importer.find_module(module_name).load_module(module_name) + for importer, module_name, _ in modules: + module = importer.find_module(module_name).load_module(module_name) if use_names and hasattr(module, 'name'): module_name = module.name + module_scripts = {} script_order = getattr(module, "script_order", ()) ordered_scripts = [cls for cls in script_order if is_script(cls)] unordered_scripts = [cls for _, cls in inspect.getmembers(module, is_script) if cls not in script_order] + for cls in [*ordered_scripts, *unordered_scripts]: # For scripts in submodules use the full import path w/o the root module as the name script_name = cls.full_name.split(".", maxsplit=1)[1] module_scripts[script_name] = cls + if module_scripts: scripts[module_name] = module_scripts From e30a1477129ce76b1a1a26e4d8c3c3fb556ed9bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?aron=20bergur=20j=C3=B3hannsson?= Date: Tue, 14 Feb 2023 23:58:41 +0000 Subject: [PATCH 15/38] Fixes #11473 graphql invalid tag filter returns all devices/interfaces --- netbox/netbox/graphql/fields.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/netbox/netbox/graphql/fields.py b/netbox/netbox/graphql/fields.py index 5b8e6cc5b..1e215c947 100644 --- a/netbox/netbox/graphql/fields.py +++ b/netbox/netbox/graphql/fields.py @@ -60,6 +60,8 @@ class ObjectListField(DjangoListField): filterset_class = django_object_type._meta.filterset_class if filterset_class: filterset = filterset_class(data=args, queryset=queryset, request=info.context) + if not filterset.is_valid(): + return [] return filterset.qs return queryset From a8dc28536145e6873ad8ec1f3c6afc14b00395d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?aron=20bergur=20j=C3=B3hannsson?= Date: Thu, 16 Feb 2023 09:08:30 +0000 Subject: [PATCH 16/38] change empty list to qs.none() --- netbox/netbox/graphql/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/netbox/graphql/fields.py b/netbox/netbox/graphql/fields.py index 1e215c947..7c359e82e 100644 --- a/netbox/netbox/graphql/fields.py +++ b/netbox/netbox/graphql/fields.py @@ -61,7 +61,7 @@ class ObjectListField(DjangoListField): if filterset_class: filterset = filterset_class(data=args, queryset=queryset, request=info.context) if not filterset.is_valid(): - return [] + return queryset.none() return filterset.qs return queryset From 567e353be2403853aa8394326e2a9736b18dbd3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aron=20Bergur=20J=C3=B3hannsson?= <71449504+aronbj20@users.noreply.github.com> Date: Thu, 16 Feb 2023 16:26:22 +0000 Subject: [PATCH 17/38] Closes #11592: Expose FILE_UPLOAD_MAX_MEMORY_SIZE as a setting (#11742) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Closes #11592: Expose FILE_UPLOAD_MAX_MEMOMORY_SIZE as a setting * change configuration settings to alphabetic order * Small example and documentation --------- Co-authored-by: aron bergur jóhannsson --- docs/configuration/miscellaneous.md | 8 ++++++++ netbox/netbox/configuration_example.py | 4 ++++ netbox/netbox/settings.py | 1 + 3 files changed, 13 insertions(+) diff --git a/docs/configuration/miscellaneous.md b/docs/configuration/miscellaneous.md index 4eb090554..eac5d0a2f 100644 --- a/docs/configuration/miscellaneous.md +++ b/docs/configuration/miscellaneous.md @@ -69,6 +69,14 @@ By default, NetBox will permit users to create duplicate prefixes and IP address --- +## FILE_UPLOAD_MAX_MEMORY_SIZE + +Default: 2621440 (i.e. 2.5 MB). + +The maximum size (in bytes) that an upload will be before it gets streamed to the file system. Changing this setting can be useful for example to be able to upload files bigger than 2.5MB to custom scripts for processing. + +--- + ## GRAPHQL_ENABLED !!! tip "Dynamic Configuration Parameter" diff --git a/netbox/netbox/configuration_example.py b/netbox/netbox/configuration_example.py index 14fcde022..7158308af 100644 --- a/netbox/netbox/configuration_example.py +++ b/netbox/netbox/configuration_example.py @@ -217,6 +217,10 @@ RQ_DEFAULT_TIMEOUT = 300 # this setting is derived from the installed location. # SCRIPTS_ROOT = '/opt/netbox/netbox/scripts' +# The maximum size (in bytes) that an upload will be before it gets streamed to the file system. +# Useful to be able to upload files bigger than 2.5Mbyte to custom scripts for processing. +# FILE_UPLOAD_MAX_MEMORY_SIZE = 2621440 + # The name to use for the csrf token cookie. CSRF_COOKIE_NAME = 'csrftoken' diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index cda6ee643..f6ce7ff33 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -91,6 +91,7 @@ DOCS_ROOT = getattr(configuration, 'DOCS_ROOT', os.path.join(os.path.dirname(BAS EMAIL = getattr(configuration, 'EMAIL', {}) EXEMPT_VIEW_PERMISSIONS = getattr(configuration, 'EXEMPT_VIEW_PERMISSIONS', []) FIELD_CHOICES = getattr(configuration, 'FIELD_CHOICES', {}) +FILE_UPLOAD_MAX_MEMORY_SIZE = getattr(configuration, 'FILE_UPLOAD_MAX_MEMORY_SIZE', 2621440) HTTP_PROXIES = getattr(configuration, 'HTTP_PROXIES', None) INTERNAL_IPS = getattr(configuration, 'INTERNAL_IPS', ('127.0.0.1', '::1')) JINJA2_FILTERS = getattr(configuration, 'JINJA2_FILTERS', {}) From 3c3e3d0b4bfea7730ce1b82ae05057383c43745c Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Thu, 16 Feb 2023 20:15:48 -0500 Subject: [PATCH 18/38] Changelog for #11226, #11335, #11473, #11592 --- docs/release-notes/version-3.4.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/release-notes/version-3.4.md b/docs/release-notes/version-3.4.md index 58c3aea49..9eb0f0cec 100644 --- a/docs/release-notes/version-3.4.md +++ b/docs/release-notes/version-3.4.md @@ -2,10 +2,17 @@ ## v3.4.5 (FUTURE) +### Enhancements + +* [#11592](https://github.com/netbox-community/netbox/issues/11592) - Introduce `FILE_UPLOAD_MAX_MEMORY_SIZE` configuration parameter + ### Bug Fixes * [#11032](https://github.com/netbox-community/netbox/issues/11032) - Fix false custom validation errors during component creation +* [#11226](https://github.com/netbox-community/netbox/issues/11226) - Ensure scripts and reports within submodules are automatically reloaded +* [#11335](https://github.com/netbox-community/netbox/issues/11335) - Avoid exception when rendering change log after uninstalling a plugin * [#11459](https://github.com/netbox-community/netbox/issues/11459) - Enable evaluating null values in custom validation rules +* [#11473](https://github.com/netbox-community/netbox/issues/11473) - GraphQL requests specifying an invalid filter should return an empty queryset * [#11582](https://github.com/netbox-community/netbox/issues/11582) - Ensure form validation errors are displayed when adding virtual chassis members * [#11601](https://github.com/netbox-community/netbox/issues/11601) - Fix partial matching of start/end addresses for IP range search * [#11683](https://github.com/netbox-community/netbox/issues/11683) - Fix CSV header attribute detection when auto-detecting import format From dcfbf799ed970e90837a6adc3a6e3e350369d900 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Fri, 17 Feb 2023 10:11:39 -0500 Subject: [PATCH 19/38] Update introduction doc --- docs/introduction.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/introduction.md b/docs/introduction.md index fe82e68aa..bfa0900cb 100644 --- a/docs/introduction.md +++ b/docs/introduction.md @@ -4,7 +4,7 @@ NetBox was originally developed by its lead maintainer, [Jeremy Stretch](https://github.com/jeremystretch), while he was working as a network engineer at [DigitalOcean](https://www.digitalocean.com/) in 2015 as part of an effort to automate their network provisioning. Recognizing the new tool's potential, DigitalOcean agreed to release it as an open source project in June 2016. -Since then, thousands of organizations around the world have embraced NetBox as their central network source of truth to empower both network operators and automation. +Since then, thousands of organizations around the world have embraced NetBox as their central network source of truth to empower both network operators and automation. Today, the open source project is stewarded by [NetBox Labs](https://netboxlabs.com/) and a team of volunteer maintainers. Beyond the core product, myriad [plugins](https://netbox.dev/plugins/) have been developed by the NetBox community to enhance and expand its feature set. ## Key Features @@ -17,6 +17,7 @@ NetBox was built specifically to serve the needs of network engineers and operat * AS number (ASN) management * Rack elevations with SVG rendering * Device modeling using pre-defined types +* Virtual chassis and device contexts * Network, power, and console cabling with SVG traces * Power distribution modeling * Data circuit and provider tracking @@ -29,12 +30,13 @@ NetBox was built specifically to serve the needs of network engineers and operat * Tenant ownership assignment * Device & VM configuration contexts for advanced configuration rendering * Custom fields for data model extension -* Support for custom validation rules +* Custom validation rules * Custom reports & scripts executable directly within the UI * Extensive plugin framework for adding custom functionality * Single sign-on (SSO) authentication * Robust object-based permissions * Detailed, automatic change logging +* Global search engine * NAPALM integration ## What NetBox Is Not From 22089565964539c4a0030c23250796a44f82a19f Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Fri, 17 Feb 2023 16:50:10 -0500 Subject: [PATCH 20/38] Closes #11110: Add start_address and end_address filters for IP ranges --- docs/release-notes/version-3.4.md | 1 + netbox/ipam/filtersets.py | 14 ++++++++++++++ netbox/ipam/tests/test_filtersets.py | 8 ++++++++ 3 files changed, 23 insertions(+) diff --git a/docs/release-notes/version-3.4.md b/docs/release-notes/version-3.4.md index 9eb0f0cec..edce381c0 100644 --- a/docs/release-notes/version-3.4.md +++ b/docs/release-notes/version-3.4.md @@ -4,6 +4,7 @@ ### Enhancements +* [#11110](https://github.com/netbox-community/netbox/issues/11110) - Add `start_address` and `end_address` filters for IP ranges * [#11592](https://github.com/netbox-community/netbox/issues/11592) - Introduce `FILE_UPLOAD_MAX_MEMORY_SIZE` configuration parameter ### Bug Fixes diff --git a/netbox/ipam/filtersets.py b/netbox/ipam/filtersets.py index c312b02ff..2e9f56bbc 100644 --- a/netbox/ipam/filtersets.py +++ b/netbox/ipam/filtersets.py @@ -405,6 +405,14 @@ class IPRangeFilterSet(TenancyFilterSet, NetBoxModelFilterSet): field_name='start_address', lookup_expr='family' ) + start_address = MultiValueCharFilter( + method='filter_address', + label=_('Address'), + ) + end_address = MultiValueCharFilter( + method='filter_address', + label=_('Address'), + ) contains = django_filters.CharFilter( method='search_contains', label=_('Ranges which contain this prefix or IP'), @@ -461,6 +469,12 @@ class IPRangeFilterSet(TenancyFilterSet, NetBoxModelFilterSet): except (AddrFormatError, ValueError): return queryset.none() + def filter_address(self, queryset, name, value): + try: + return queryset.filter(**{f'{name}__net_in': value}) + except ValidationError: + return queryset.none() + class IPAddressFilterSet(NetBoxModelFilterSet, TenancyFilterSet): family = django_filters.NumberFilter( diff --git a/netbox/ipam/tests/test_filtersets.py b/netbox/ipam/tests/test_filtersets.py index 711009a7e..13b3ae163 100644 --- a/netbox/ipam/tests/test_filtersets.py +++ b/netbox/ipam/tests/test_filtersets.py @@ -680,6 +680,14 @@ class IPRangeTestCase(TestCase, ChangeLoggedFilterSetTests): params = {'family': '6'} self.assertEqual(self.filterset(params, self.queryset).qs.count(), 4) + def test_start_address(self): + params = {'start_address': ['10.0.1.100', '10.0.2.100']} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + + def test_end_address(self): + params = {'end_address': ['10.0.1.199', '10.0.2.199']} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + def test_contains(self): params = {'contains': '10.0.1.150/24'} self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) From a91a6e721e2bb72c6767ad5d0e209a262ece30f3 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Fri, 17 Feb 2023 16:57:52 -0500 Subject: [PATCH 21/38] Raise stale timers from 60/30 to 90/30 --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index ab259af2a..3b37aae56 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -24,7 +24,7 @@ jobs: necessary. close-pr-message: > This PR has been automatically closed due to lack of activity. - days-before-stale: 60 + days-before-stale: 90 days-before-close: 30 exempt-issue-labels: 'status: accepted,status: blocked,status: needs milestone' operations-per-run: 100 From 10c8b7478626ed60090c788e9711c8df76a9e7d6 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Fri, 17 Feb 2023 21:31:19 -0500 Subject: [PATCH 22/38] Fixes #11723: Circuit terminations should link to their associated circuits (rather than site or provider network) --- docs/release-notes/version-3.4.md | 1 + netbox/circuits/models/circuits.py | 6 ++---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/release-notes/version-3.4.md b/docs/release-notes/version-3.4.md index edce381c0..636755ac4 100644 --- a/docs/release-notes/version-3.4.md +++ b/docs/release-notes/version-3.4.md @@ -18,6 +18,7 @@ * [#11601](https://github.com/netbox-community/netbox/issues/11601) - Fix partial matching of start/end addresses for IP range search * [#11683](https://github.com/netbox-community/netbox/issues/11683) - Fix CSV header attribute detection when auto-detecting import format * [#11711](https://github.com/netbox-community/netbox/issues/11711) - Fix CSV import for multiple-object custom fields +* [#11723](https://github.com/netbox-community/netbox/issues/11723) - Circuit terminations should link to their associated circuits (rather than site or provider network) --- diff --git a/netbox/circuits/models/circuits.py b/netbox/circuits/models/circuits.py index 8ef5761fd..eba7f4de0 100644 --- a/netbox/circuits/models/circuits.py +++ b/netbox/circuits/models/circuits.py @@ -196,12 +196,10 @@ class CircuitTermination( ) def __str__(self): - return f'Termination {self.term_side}: {self.site or self.provider_network}' + return f'{self.circuit}: Termination {self.term_side}' def get_absolute_url(self): - if self.site: - return self.site.get_absolute_url() - return self.provider_network.get_absolute_url() + return self.circuit.get_absolute_url() def clean(self): super().clean() From a12fb6f7c6174984e4dcf4149cbf28cc73ffdceb Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Sun, 19 Feb 2023 16:17:57 -0500 Subject: [PATCH 23/38] Fixes #11786: List only applicable object types in form widget when filtering custom fields --- docs/release-notes/version-3.4.md | 1 + netbox/extras/forms/filtersets.py | 18 ++++++------------ 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/docs/release-notes/version-3.4.md b/docs/release-notes/version-3.4.md index 636755ac4..731400a92 100644 --- a/docs/release-notes/version-3.4.md +++ b/docs/release-notes/version-3.4.md @@ -19,6 +19,7 @@ * [#11683](https://github.com/netbox-community/netbox/issues/11683) - Fix CSV header attribute detection when auto-detecting import format * [#11711](https://github.com/netbox-community/netbox/issues/11711) - Fix CSV import for multiple-object custom fields * [#11723](https://github.com/netbox-community/netbox/issues/11723) - Circuit terminations should link to their associated circuits (rather than site or provider network) +* [#11786](https://github.com/netbox-community/netbox/issues/11786) - List only applicable object types in form widget when filtering custom fields --- diff --git a/netbox/extras/forms/filtersets.py b/netbox/extras/forms/filtersets.py index e6a9089bc..22c7364db 100644 --- a/netbox/extras/forms/filtersets.py +++ b/netbox/extras/forms/filtersets.py @@ -38,8 +38,7 @@ class CustomFieldFilterForm(SavedFiltersMixin, FilterForm): ('Attributes', ('type', 'content_type_id', 'group_name', 'weight', 'required', 'ui_visibility')), ) content_type_id = ContentTypeMultipleChoiceField( - queryset=ContentType.objects.all(), - limit_choices_to=FeatureQuery('custom_fields'), + queryset=ContentType.objects.filter(FeatureQuery('custom_fields').get_query()), required=False, label=_('Object type') ) @@ -79,8 +78,7 @@ class JobResultFilterForm(SavedFiltersMixin, FilterForm): ) obj_type = ContentTypeChoiceField( label=_('Object Type'), - queryset=ContentType.objects.all(), - limit_choices_to=FeatureQuery('job_results'), # TODO: This doesn't actually work + queryset=ContentType.objects.filter(FeatureQuery('job_results').get_query()), required=False, ) status = MultipleChoiceField( @@ -135,8 +133,7 @@ class CustomLinkFilterForm(SavedFiltersMixin, FilterForm): ('Attributes', ('content_types', 'enabled', 'new_window', 'weight')), ) content_types = ContentTypeMultipleChoiceField( - queryset=ContentType.objects.all(), - limit_choices_to=FeatureQuery('custom_links'), + queryset=ContentType.objects.filter(FeatureQuery('custom_links').get_query()), required=False ) enabled = forms.NullBooleanField( @@ -162,8 +159,7 @@ class ExportTemplateFilterForm(SavedFiltersMixin, FilterForm): ('Attributes', ('content_types', 'mime_type', 'file_extension', 'as_attachment')), ) content_types = ContentTypeMultipleChoiceField( - queryset=ContentType.objects.all(), - limit_choices_to=FeatureQuery('export_templates'), + queryset=ContentType.objects.filter(FeatureQuery('export_templates').get_query()), required=False ) mime_type = forms.CharField( @@ -187,8 +183,7 @@ class SavedFilterFilterForm(SavedFiltersMixin, FilterForm): ('Attributes', ('content_types', 'enabled', 'shared', 'weight')), ) content_types = ContentTypeMultipleChoiceField( - queryset=ContentType.objects.all(), - limit_choices_to=FeatureQuery('export_templates'), + queryset=ContentType.objects.filter(FeatureQuery('export_templates').get_query()), required=False ) enabled = forms.NullBooleanField( @@ -215,8 +210,7 @@ class WebhookFilterForm(SavedFiltersMixin, FilterForm): ('Events', ('type_create', 'type_update', 'type_delete')), ) content_type_id = ContentTypeMultipleChoiceField( - queryset=ContentType.objects.all(), - limit_choices_to=FeatureQuery('webhooks'), + queryset=ContentType.objects.filter(FeatureQuery('webhooks').get_query()), required=False, label=_('Object type') ) From 11a5905785cb94e3903751ae9a0b67a8732a2182 Mon Sep 17 00:00:00 2001 From: kkthxbye-code Date: Mon, 6 Feb 2023 14:00:34 +0100 Subject: [PATCH 24/38] Proof of concept for showing containing prefixes when searching for ip-addresses. --- netbox/extras/lookups.py | 15 ++++++++++++++- netbox/netbox/search/__init__.py | 7 +++++++ netbox/netbox/search/backends.py | 23 +++++++++++++++-------- 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/netbox/extras/lookups.py b/netbox/extras/lookups.py index 7197efcfc..4cdda52b4 100644 --- a/netbox/extras/lookups.py +++ b/netbox/extras/lookups.py @@ -1,4 +1,4 @@ -from django.db.models import CharField, Lookup +from django.db.models import CharField, TextField, Lookup class Empty(Lookup): @@ -14,4 +14,17 @@ class Empty(Lookup): return 'CAST(LENGTH(%s) AS BOOLEAN) != %s' % (lhs, rhs), params +class NetContainsOrEquals(Lookup): + """ + This lookup has the same functionality as the one from the ipam app except lhs is cast to inet + """ + lookup_name = 'net_contains_or_equals' + + def as_sql(self, qn, connection): + lhs, lhs_params = self.process_lhs(qn, connection) + rhs, rhs_params = self.process_rhs(qn, connection) + params = lhs_params + rhs_params + return 'CAST(%s as inet) >>= %s' % (lhs, rhs), params + CharField.register_lookup(Empty) +TextField.register_lookup(NetContainsOrEquals) diff --git a/netbox/netbox/search/__init__.py b/netbox/netbox/search/__init__.py index 1eec8e097..f4cc07c4d 100644 --- a/netbox/netbox/search/__init__.py +++ b/netbox/netbox/search/__init__.py @@ -2,6 +2,7 @@ from collections import namedtuple from django.db import models +from ipam.fields import IPAddressField, IPNetworkField from netbox.registry import registry ObjectFieldValue = namedtuple('ObjectFieldValue', ('name', 'type', 'weight', 'value')) @@ -11,6 +12,8 @@ class FieldTypes: FLOAT = 'float' INTEGER = 'int' STRING = 'str' + INET = 'inet' + CIDR = 'cidr' class LookupTypes: @@ -43,6 +46,10 @@ class SearchIndex: field_cls = instance._meta.get_field(field_name).__class__ if issubclass(field_cls, (models.FloatField, models.DecimalField)): return FieldTypes.FLOAT + if issubclass(field_cls, IPAddressField): + return FieldTypes.INET + if issubclass(field_cls, (IPNetworkField)): + return FieldTypes.CIDR if issubclass(field_cls, models.IntegerField): return FieldTypes.INTEGER return FieldTypes.STRING diff --git a/netbox/netbox/search/backends.py b/netbox/netbox/search/backends.py index d659a7abb..6bbfdd7d1 100644 --- a/netbox/netbox/search/backends.py +++ b/netbox/netbox/search/backends.py @@ -3,10 +3,12 @@ from collections import defaultdict from django.conf import settings from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ImproperlyConfigured -from django.db.models import F, Window +from django.db.models import F, Window, Q from django.db.models.functions import window from django.db.models.signals import post_delete, post_save from django.utils.module_loading import import_string +import netaddr +from netaddr.core import AddrFormatError from extras.models import CachedValue, CustomField from netbox.registry import registry @@ -95,18 +97,23 @@ class CachedValueSearchBackend(SearchBackend): def search(self, value, user=None, object_types=None, lookup=DEFAULT_LOOKUP_TYPE): - # Define the search parameters - params = { - f'value__{lookup}': value - } + query_filter = Q(**{f'value__{lookup}': value}) + if lookup in (LookupTypes.STARTSWITH, LookupTypes.ENDSWITH): # Partial string matches are valid only on string values - params['type'] = FieldTypes.STRING + query_filter &= Q(type=FieldTypes.STRING) if object_types: - params['object_type__in'] = object_types + query_filter &= Q(object_typeo__in=object_types) + + if lookup == LookupTypes.PARTIAL: + try: + address = str(netaddr.IPNetwork(value.strip()).cidr) + query_filter |= Q(type=FieldTypes.CIDR) & Q(value__net_contains_or_equals=address) + except (AddrFormatError, ValueError): + pass # Construct the base queryset to retrieve matching results - queryset = CachedValue.objects.filter(**params).annotate( + queryset = CachedValue.objects.filter(query_filter).annotate( # Annotate the rank of each result for its object according to its weight row_number=Window( expression=window.RowNumber(), From c7a6aa742a534df7b47a80d4e5dfd2f9e229ded5 Mon Sep 17 00:00:00 2001 From: kkthxbye-code Date: Mon, 6 Feb 2023 21:25:24 +0100 Subject: [PATCH 25/38] Fix typo in search query --- netbox/netbox/search/backends.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/netbox/search/backends.py b/netbox/netbox/search/backends.py index 6bbfdd7d1..2e550a879 100644 --- a/netbox/netbox/search/backends.py +++ b/netbox/netbox/search/backends.py @@ -103,7 +103,7 @@ class CachedValueSearchBackend(SearchBackend): # Partial string matches are valid only on string values query_filter &= Q(type=FieldTypes.STRING) if object_types: - query_filter &= Q(object_typeo__in=object_types) + query_filter &= Q(object_type__in=object_types) if lookup == LookupTypes.PARTIAL: try: From 3db48c069ea147a233ab3ca8a3824f1d291d9da8 Mon Sep 17 00:00:00 2001 From: kkthxbye-code Date: Tue, 7 Feb 2023 09:40:25 +0100 Subject: [PATCH 26/38] Create CachedValueField to contain search specific lookups --- netbox/extras/fields.py | 7 +++++++ netbox/extras/lookups.py | 4 ++-- .../0085_change_cachedvalue_value_type.py | 19 +++++++++++++++++++ netbox/extras/models/search.py | 3 ++- 4 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 netbox/extras/fields.py create mode 100644 netbox/extras/migrations/0085_change_cachedvalue_value_type.py diff --git a/netbox/extras/fields.py b/netbox/extras/fields.py new file mode 100644 index 000000000..ffd66801d --- /dev/null +++ b/netbox/extras/fields.py @@ -0,0 +1,7 @@ +from django.db.models import TextField + +class CachedValueField(TextField): + """ + Currently a dummy field to prevent custom lookups being applied globally to TextField. + """ + pass \ No newline at end of file diff --git a/netbox/extras/lookups.py b/netbox/extras/lookups.py index 4cdda52b4..a52ef2e8d 100644 --- a/netbox/extras/lookups.py +++ b/netbox/extras/lookups.py @@ -1,5 +1,5 @@ from django.db.models import CharField, TextField, Lookup - +from .fields import CachedValueField class Empty(Lookup): """ @@ -27,4 +27,4 @@ class NetContainsOrEquals(Lookup): return 'CAST(%s as inet) >>= %s' % (lhs, rhs), params CharField.register_lookup(Empty) -TextField.register_lookup(NetContainsOrEquals) +CachedValueField.register_lookup(NetContainsOrEquals) diff --git a/netbox/extras/migrations/0085_change_cachedvalue_value_type.py b/netbox/extras/migrations/0085_change_cachedvalue_value_type.py new file mode 100644 index 000000000..e6512753e --- /dev/null +++ b/netbox/extras/migrations/0085_change_cachedvalue_value_type.py @@ -0,0 +1,19 @@ +# Generated by Django 4.1.6 on 2023-02-07 08:21 + +from django.db import migrations +import extras.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('extras', '0084_staging'), + ] + + operations = [ + migrations.AlterField( + model_name='cachedvalue', + name='value', + field=extras.fields.CachedValueField(), + ), + ] diff --git a/netbox/extras/models/search.py b/netbox/extras/models/search.py index 7c5860e00..6d088abb0 100644 --- a/netbox/extras/models/search.py +++ b/netbox/extras/models/search.py @@ -4,6 +4,7 @@ from django.contrib.contenttypes.models import ContentType from django.db import models from utilities.fields import RestrictedGenericForeignKey +from ..fields import CachedValueField __all__ = ( 'CachedValue', @@ -36,7 +37,7 @@ class CachedValue(models.Model): type = models.CharField( max_length=30 ) - value = models.TextField() + value = CachedValueField() weight = models.PositiveSmallIntegerField( default=1000 ) From 9cd30ac09b0ff62f287624988cd950fb8a28b041 Mon Sep 17 00:00:00 2001 From: kkthxbye-code Date: Tue, 7 Feb 2023 13:24:57 +0100 Subject: [PATCH 27/38] pep8 fixes --- netbox/extras/fields.py | 3 ++- netbox/extras/lookups.py | 2 ++ netbox/netbox/search/backends.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/netbox/extras/fields.py b/netbox/extras/fields.py index ffd66801d..6cd44432f 100644 --- a/netbox/extras/fields.py +++ b/netbox/extras/fields.py @@ -1,7 +1,8 @@ from django.db.models import TextField + class CachedValueField(TextField): """ Currently a dummy field to prevent custom lookups being applied globally to TextField. """ - pass \ No newline at end of file + pass diff --git a/netbox/extras/lookups.py b/netbox/extras/lookups.py index a52ef2e8d..d4ed2b6a4 100644 --- a/netbox/extras/lookups.py +++ b/netbox/extras/lookups.py @@ -1,6 +1,7 @@ from django.db.models import CharField, TextField, Lookup from .fields import CachedValueField + class Empty(Lookup): """ Filter on whether a string is empty. @@ -26,5 +27,6 @@ class NetContainsOrEquals(Lookup): params = lhs_params + rhs_params return 'CAST(%s as inet) >>= %s' % (lhs, rhs), params + CharField.register_lookup(Empty) CachedValueField.register_lookup(NetContainsOrEquals) diff --git a/netbox/netbox/search/backends.py b/netbox/netbox/search/backends.py index 2e550a879..53a4fe683 100644 --- a/netbox/netbox/search/backends.py +++ b/netbox/netbox/search/backends.py @@ -108,7 +108,7 @@ class CachedValueSearchBackend(SearchBackend): if lookup == LookupTypes.PARTIAL: try: address = str(netaddr.IPNetwork(value.strip()).cidr) - query_filter |= Q(type=FieldTypes.CIDR) & Q(value__net_contains_or_equals=address) + query_filter |= Q(type=FieldTypes.CIDR) & Q(value__net_contains_or_equals=address) except (AddrFormatError, ValueError): pass From 9d110db41bbcbc21427f54e02fb2e76138feff4f Mon Sep 17 00:00:00 2001 From: kkthxbye-code Date: Sat, 18 Feb 2023 19:29:57 +0100 Subject: [PATCH 28/38] Address feedback --- netbox/extras/lookups.py | 2 +- netbox/netbox/search/__init__.py | 2 +- netbox/netbox/search/backends.py | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/netbox/extras/lookups.py b/netbox/extras/lookups.py index d4ed2b6a4..77fe2301e 100644 --- a/netbox/extras/lookups.py +++ b/netbox/extras/lookups.py @@ -25,7 +25,7 @@ class NetContainsOrEquals(Lookup): lhs, lhs_params = self.process_lhs(qn, connection) rhs, rhs_params = self.process_rhs(qn, connection) params = lhs_params + rhs_params - return 'CAST(%s as inet) >>= %s' % (lhs, rhs), params + return 'CAST(%s AS INET) >>= %s' % (lhs, rhs), params CharField.register_lookup(Empty) diff --git a/netbox/netbox/search/__init__.py b/netbox/netbox/search/__init__.py index f4cc07c4d..6d53e9a97 100644 --- a/netbox/netbox/search/__init__.py +++ b/netbox/netbox/search/__init__.py @@ -48,7 +48,7 @@ class SearchIndex: return FieldTypes.FLOAT if issubclass(field_cls, IPAddressField): return FieldTypes.INET - if issubclass(field_cls, (IPNetworkField)): + if issubclass(field_cls, IPNetworkField): return FieldTypes.CIDR if issubclass(field_cls, models.IntegerField): return FieldTypes.INTEGER diff --git a/netbox/netbox/search/backends.py b/netbox/netbox/search/backends.py index 53a4fe683..10e164c09 100644 --- a/netbox/netbox/search/backends.py +++ b/netbox/netbox/search/backends.py @@ -99,11 +99,12 @@ class CachedValueSearchBackend(SearchBackend): query_filter = Q(**{f'value__{lookup}': value}) + if object_types: + query_filter &= Q(object_type__in=object_types) + if lookup in (LookupTypes.STARTSWITH, LookupTypes.ENDSWITH): # Partial string matches are valid only on string values query_filter &= Q(type=FieldTypes.STRING) - if object_types: - query_filter &= Q(object_type__in=object_types) if lookup == LookupTypes.PARTIAL: try: From bead016544fe20afeb6530c9ef17b354c859e1e4 Mon Sep 17 00:00:00 2001 From: kkthxbye-code Date: Sat, 18 Feb 2023 19:30:29 +0100 Subject: [PATCH 29/38] Change Prefix and Aggregate search index weights to better order search results. --- netbox/ipam/search.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/netbox/ipam/search.py b/netbox/ipam/search.py index fd6db6a63..ad4403321 100644 --- a/netbox/ipam/search.py +++ b/netbox/ipam/search.py @@ -6,7 +6,7 @@ from netbox.search import SearchIndex, register_search class AggregateIndex(SearchIndex): model = models.Aggregate fields = ( - ('prefix', 100), + ('prefix', 120), ('description', 500), ('date_added', 2000), ('comments', 5000), @@ -70,7 +70,7 @@ class L2VPNIndex(SearchIndex): class PrefixIndex(SearchIndex): model = models.Prefix fields = ( - ('prefix', 100), + ('prefix', 110), ('description', 500), ('comments', 5000), ) From cf20d04cbeb0c847746e4e61e1f18af4acf1b60f Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Sun, 19 Feb 2023 18:52:26 -0500 Subject: [PATCH 30/38] Changelog for #11685 --- docs/release-notes/version-3.4.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/release-notes/version-3.4.md b/docs/release-notes/version-3.4.md index 731400a92..485b85719 100644 --- a/docs/release-notes/version-3.4.md +++ b/docs/release-notes/version-3.4.md @@ -6,6 +6,7 @@ * [#11110](https://github.com/netbox-community/netbox/issues/11110) - Add `start_address` and `end_address` filters for IP ranges * [#11592](https://github.com/netbox-community/netbox/issues/11592) - Introduce `FILE_UPLOAD_MAX_MEMORY_SIZE` configuration parameter +* [#11685](https://github.com/netbox-community/netbox/issues/11685) - Match on containing prefixes and aggregates when querying for IP addresses using global search ### Bug Fixes From 1c4cb74fdb675d005ba1b62cadd74d94b9bf838e Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Sun, 19 Feb 2023 18:53:46 -0500 Subject: [PATCH 31/38] Fixes #11658: Remove reindex command call from search migration --- netbox/extras/migrations/0083_search.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/netbox/extras/migrations/0083_search.py b/netbox/extras/migrations/0083_search.py index 0c53de638..349066918 100644 --- a/netbox/extras/migrations/0083_search.py +++ b/netbox/extras/migrations/0083_search.py @@ -1,27 +1,10 @@ -import sys import uuid import django.db.models.deletion import django.db.models.lookups -from django.core import management from django.db import migrations, models -def reindex(apps, schema_editor): - # Build the search index (except during tests) - if 'test' not in sys.argv: - management.call_command( - 'reindex', - 'circuits', - 'dcim', - 'extras', - 'ipam', - 'tenancy', - 'virtualization', - 'wireless', - ) - - class Migration(migrations.Migration): dependencies = [ @@ -57,8 +40,4 @@ class Migration(migrations.Migration): 'ordering': ('weight', 'object_type', 'object_id'), }, ), - migrations.RunPython( - code=reindex, - reverse_code=migrations.RunPython.noop - ), ] From f6324d1a8c53b2951de5752972eb4471eb8e17be Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Sun, 19 Feb 2023 20:08:57 -0500 Subject: [PATCH 32/38] #11685: Omit no-op migration --- netbox/extras/migrations/0083_search.py | 3 ++- .../0085_change_cachedvalue_value_type.py | 19 ------------------- 2 files changed, 2 insertions(+), 20 deletions(-) delete mode 100644 netbox/extras/migrations/0085_change_cachedvalue_value_type.py diff --git a/netbox/extras/migrations/0083_search.py b/netbox/extras/migrations/0083_search.py index 349066918..4c7ae1084 100644 --- a/netbox/extras/migrations/0083_search.py +++ b/netbox/extras/migrations/0083_search.py @@ -3,6 +3,7 @@ import uuid import django.db.models.deletion import django.db.models.lookups from django.db import migrations, models +import extras.fields class Migration(migrations.Migration): @@ -32,7 +33,7 @@ class Migration(migrations.Migration): ('object_id', models.PositiveBigIntegerField()), ('field', models.CharField(max_length=200)), ('type', models.CharField(max_length=30)), - ('value', models.TextField()), + ('value', extras.fields.CachedValueField()), ('weight', models.PositiveSmallIntegerField(default=1000)), ('object_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='contenttypes.contenttype')), ], diff --git a/netbox/extras/migrations/0085_change_cachedvalue_value_type.py b/netbox/extras/migrations/0085_change_cachedvalue_value_type.py deleted file mode 100644 index e6512753e..000000000 --- a/netbox/extras/migrations/0085_change_cachedvalue_value_type.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 4.1.6 on 2023-02-07 08:21 - -from django.db import migrations -import extras.fields - - -class Migration(migrations.Migration): - - dependencies = [ - ('extras', '0084_staging'), - ] - - operations = [ - migrations.AlterField( - model_name='cachedvalue', - name='value', - field=extras.fields.CachedValueField(), - ), - ] From cc3ee78f1750eb44510cf1cb429f7843461f86ca Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Fri, 17 Feb 2023 20:30:35 -0500 Subject: [PATCH 33/38] Skip clearing cache when handling new objects --- netbox/netbox/search/backends.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/netbox/netbox/search/backends.py b/netbox/netbox/search/backends.py index 10e164c09..14b5a987c 100644 --- a/netbox/netbox/search/backends.py +++ b/netbox/netbox/search/backends.py @@ -54,11 +54,11 @@ class SearchBackend: """ raise NotImplementedError - def caching_handler(self, sender, instance, **kwargs): + def caching_handler(self, sender, instance, created, **kwargs): """ Receiver for the post_save signal, responsible for caching object creation/changes. """ - self.cache(instance) + self.cache(instance, remove_existing=not created) def removal_handler(self, sender, instance, **kwargs): """ From 043bc880cc08aae936166ceeefb3da2870baeaf7 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Mon, 20 Feb 2023 09:02:58 -0500 Subject: [PATCH 34/38] #11685: Clear cached search records for relevant IPAM objects --- .../migrations/0064_clear_search_cache.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 netbox/ipam/migrations/0064_clear_search_cache.py diff --git a/netbox/ipam/migrations/0064_clear_search_cache.py b/netbox/ipam/migrations/0064_clear_search_cache.py new file mode 100644 index 000000000..92d2b2cd7 --- /dev/null +++ b/netbox/ipam/migrations/0064_clear_search_cache.py @@ -0,0 +1,28 @@ +from django.db import migrations + + +def clear_cache(apps, schema_editor): + """ + Clear existing CachedValues referencing IPAddressFields or IPNetworkFields. (#11658 + introduced new cache record types for these.) + """ + ContentType = apps.get_model('contenttypes', 'ContentType') + CachedValue = apps.get_model('extras', 'CachedValue') + + for model_name in ('Aggregate', 'IPAddress', 'IPRange', 'Prefix'): + content_type = ContentType.objects.get(app_label='ipam', model=model_name.lower()) + CachedValue.objects.filter(object_type=content_type).delete() + + +class Migration(migrations.Migration): + + dependencies = [ + ('ipam', '0063_standardize_description_comments'), + ] + + operations = [ + migrations.RunPython( + code=clear_cache, + reverse_code=migrations.RunPython.noop + ), + ] From de8029d9d138037532f76390b347bf0dc2c14e46 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Mon, 20 Feb 2023 09:39:33 -0500 Subject: [PATCH 35/38] Closes #11787: Rebuild any missing search cache entires after upgrade --- docs/release-notes/version-3.4.md | 2 ++ netbox/extras/management/commands/reindex.py | 31 ++++++++++++++------ netbox/netbox/search/backends.py | 14 ++++++++- upgrade.sh | 5 ++++ 4 files changed, 42 insertions(+), 10 deletions(-) diff --git a/docs/release-notes/version-3.4.md b/docs/release-notes/version-3.4.md index 485b85719..a006eb7bf 100644 --- a/docs/release-notes/version-3.4.md +++ b/docs/release-notes/version-3.4.md @@ -7,6 +7,7 @@ * [#11110](https://github.com/netbox-community/netbox/issues/11110) - Add `start_address` and `end_address` filters for IP ranges * [#11592](https://github.com/netbox-community/netbox/issues/11592) - Introduce `FILE_UPLOAD_MAX_MEMORY_SIZE` configuration parameter * [#11685](https://github.com/netbox-community/netbox/issues/11685) - Match on containing prefixes and aggregates when querying for IP addresses using global search +* [#11787](https://github.com/netbox-community/netbox/issues/11787) - Upgrade script will automatically rebuild missing search cache ### Bug Fixes @@ -20,6 +21,7 @@ * [#11683](https://github.com/netbox-community/netbox/issues/11683) - Fix CSV header attribute detection when auto-detecting import format * [#11711](https://github.com/netbox-community/netbox/issues/11711) - Fix CSV import for multiple-object custom fields * [#11723](https://github.com/netbox-community/netbox/issues/11723) - Circuit terminations should link to their associated circuits (rather than site or provider network) +* [#11775](https://github.com/netbox-community/netbox/issues/11775) - Skip checking for old search cache records when creating a new object * [#11786](https://github.com/netbox-community/netbox/issues/11786) - List only applicable object types in form widget when filtering custom fields --- diff --git a/netbox/extras/management/commands/reindex.py b/netbox/extras/management/commands/reindex.py index b601a1ac1..9a29c54f5 100644 --- a/netbox/extras/management/commands/reindex.py +++ b/netbox/extras/management/commands/reindex.py @@ -15,6 +15,11 @@ class Command(BaseCommand): nargs='*', help='One or more apps or models to reindex', ) + parser.add_argument( + '--lazy', + action='store_true', + help="For each model, reindex objects only if no cache entries already exist" + ) def _get_indexers(self, *model_names): indexers = {} @@ -60,14 +65,15 @@ class Command(BaseCommand): raise CommandError("No indexers found!") self.stdout.write(f'Reindexing {len(indexers)} models.') - # Clear all cached values for the specified models - self.stdout.write('Clearing cached values... ', ending='') - self.stdout.flush() - content_types = [ - ContentType.objects.get_for_model(model) for model in indexers.keys() - ] - deleted_count = search_backend.clear(content_types) - self.stdout.write(f'{deleted_count} entries deleted.') + # Clear all cached values for the specified models (if not being lazy) + if not kwargs['lazy']: + self.stdout.write('Clearing cached values... ', ending='') + self.stdout.flush() + content_types = [ + ContentType.objects.get_for_model(model) for model in indexers.keys() + ] + deleted_count = search_backend.clear(content_types) + self.stdout.write(f'{deleted_count} entries deleted.') # Index models self.stdout.write('Indexing models') @@ -76,11 +82,18 @@ class Command(BaseCommand): model_name = model._meta.model_name self.stdout.write(f' {app_label}.{model_name}... ', ending='') self.stdout.flush() + + if kwargs['lazy']: + content_type = ContentType.objects.get_for_model(model) + if cached_count := search_backend.count(object_types=[content_type]): + self.stdout.write(f'Skipping (found {cached_count} existing).') + continue + i = search_backend.cache(model.objects.iterator(), remove_existing=False) if i: self.stdout.write(f'{i} entries cached.') else: - self.stdout.write(f'None found.') + self.stdout.write(f'No objects found.') msg = f'Completed.' if total_count := search_backend.size: diff --git a/netbox/netbox/search/backends.py b/netbox/netbox/search/backends.py index 14b5a987c..f428842f5 100644 --- a/netbox/netbox/search/backends.py +++ b/netbox/netbox/search/backends.py @@ -80,7 +80,13 @@ class SearchBackend: def clear(self, object_types=None): """ - Delete *all* cached data. + Delete *all* cached data (optionally filtered by object type). + """ + raise NotImplementedError + + def count(self, object_types=None): + """ + Return a count of all cache entries (optionally filtered by object type). """ raise NotImplementedError @@ -218,6 +224,12 @@ class CachedValueSearchBackend(SearchBackend): # Call _raw_delete() on the queryset to avoid first loading instances into memory return qs._raw_delete(using=qs.db) + def count(self, object_types=None): + qs = CachedValue.objects.all() + if object_types: + qs = qs.filter(object_type__in=object_types) + return qs.count() + @property def size(self): return CachedValue.objects.count() diff --git a/upgrade.sh b/upgrade.sh index 161d65e32..cac046a9f 100755 --- a/upgrade.sh +++ b/upgrade.sh @@ -103,6 +103,11 @@ COMMAND="python3 netbox/manage.py remove_stale_contenttypes --no-input" echo "Removing stale content types ($COMMAND)..." eval $COMMAND || exit 1 +# Rebuild the search cache (lazily) +COMMAND="python3 netbox/manage.py reindex --lazy" +echo "Rebuilding search cache ($COMMAND)..." +eval $COMMAND || exit 1 + # Delete any expired user sessions COMMAND="python3 netbox/manage.py clearsessions" echo "Removing expired user sessions ($COMMAND)..." From 1ce4a198e8cbb36e304ef9c8d22e281ce800c304 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Mon, 20 Feb 2023 10:27:30 -0500 Subject: [PATCH 36/38] #11685: Fix migration --- netbox/ipam/migrations/0064_clear_search_cache.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/netbox/ipam/migrations/0064_clear_search_cache.py b/netbox/ipam/migrations/0064_clear_search_cache.py index 92d2b2cd7..856fe99e1 100644 --- a/netbox/ipam/migrations/0064_clear_search_cache.py +++ b/netbox/ipam/migrations/0064_clear_search_cache.py @@ -10,8 +10,11 @@ def clear_cache(apps, schema_editor): CachedValue = apps.get_model('extras', 'CachedValue') for model_name in ('Aggregate', 'IPAddress', 'IPRange', 'Prefix'): - content_type = ContentType.objects.get(app_label='ipam', model=model_name.lower()) - CachedValue.objects.filter(object_type=content_type).delete() + try: + content_type = ContentType.objects.get(app_label='ipam', model=model_name.lower()) + CachedValue.objects.filter(object_type=content_type).delete() + except ContentType.DoesNotExist: + pass class Migration(migrations.Migration): From a046c508ddf941dc0824aaf67db27bd425f470fc Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Tue, 21 Feb 2023 08:42:39 -0500 Subject: [PATCH 37/38] Arrange parameters alphabetically --- netbox/netbox/configuration_example.py | 34 ++++++++++++-------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/netbox/netbox/configuration_example.py b/netbox/netbox/configuration_example.py index 7158308af..92f6133a3 100644 --- a/netbox/netbox/configuration_example.py +++ b/netbox/netbox/configuration_example.py @@ -107,6 +107,9 @@ CORS_ORIGIN_REGEX_WHITELIST = [ # r'^(https?://)?(\w+\.)?example\.com$', ] +# The name to use for the CSRF token cookie. +CSRF_COOKIE_NAME = 'csrftoken' + # Set to True to enable server debugging. WARNING: Debugging introduces a substantial performance penalty and may reveal # sensitive information about your installation. Only enable debugging while performing testing. Never enable debugging # on a production system. @@ -127,6 +130,9 @@ EMAIL = { 'FROM_EMAIL': '', } +# Localization +ENABLE_LOCALIZATION = False + # Exempt certain models from the enforcement of view permissions. Models listed here will be viewable by all users and # by anonymous users. List models in the form `.`. Add '*' to this list to exempt all models. EXEMPT_VIEW_PERMISSIONS = [ @@ -168,16 +174,6 @@ LOGOUT_REDIRECT_URL = 'home' # the default value of this setting is derived from the installed location. # MEDIA_ROOT = '/opt/netbox/netbox/media' -# By default uploaded media is stored on the local filesystem. Using Django-storages is also supported. Provide the -# class path of the storage driver in STORAGE_BACKEND and any configuration options in STORAGE_CONFIG. For example: -# STORAGE_BACKEND = 'storages.backends.s3boto3.S3Boto3Storage' -# STORAGE_CONFIG = { -# 'AWS_ACCESS_KEY_ID': 'Key ID', -# 'AWS_SECRET_ACCESS_KEY': 'Secret', -# 'AWS_STORAGE_BUCKET_NAME': 'netbox', -# 'AWS_S3_REGION_NAME': 'eu-west-1', -# } - # Expose Prometheus monitoring metrics at the HTTP endpoint '/metrics' METRICS_ENABLED = False @@ -217,13 +213,6 @@ RQ_DEFAULT_TIMEOUT = 300 # this setting is derived from the installed location. # SCRIPTS_ROOT = '/opt/netbox/netbox/scripts' -# The maximum size (in bytes) that an upload will be before it gets streamed to the file system. -# Useful to be able to upload files bigger than 2.5Mbyte to custom scripts for processing. -# FILE_UPLOAD_MAX_MEMORY_SIZE = 2621440 - -# The name to use for the csrf token cookie. -CSRF_COOKIE_NAME = 'csrftoken' - # The name to use for the session cookie. SESSION_COOKIE_NAME = 'sessionid' @@ -232,8 +221,15 @@ SESSION_COOKIE_NAME = 'sessionid' # database access.) Note that the user as which NetBox runs must have read and write permissions to this path. SESSION_FILE_PATH = None -# Localization -ENABLE_LOCALIZATION = False +# By default, uploaded media is stored on the local filesystem. Using Django-storages is also supported. Provide the +# class path of the storage driver in STORAGE_BACKEND and any configuration options in STORAGE_CONFIG. For example: +# STORAGE_BACKEND = 'storages.backends.s3boto3.S3Boto3Storage' +# STORAGE_CONFIG = { +# 'AWS_ACCESS_KEY_ID': 'Key ID', +# 'AWS_SECRET_ACCESS_KEY': 'Secret', +# 'AWS_STORAGE_BUCKET_NAME': 'netbox', +# 'AWS_S3_REGION_NAME': 'eu-west-1', +# } # Time zone (default: UTC) TIME_ZONE = 'UTC' From 4329e0eb92d82388efb91b181f12dd073902fa42 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Tue, 21 Feb 2023 08:45:52 -0500 Subject: [PATCH 38/38] Release v3.4.5 --- .github/ISSUE_TEMPLATE/bug_report.yaml | 2 +- .github/ISSUE_TEMPLATE/feature_request.yaml | 2 +- docs/configuration/miscellaneous.md | 6 +++--- docs/release-notes/version-3.4.md | 3 +-- netbox/netbox/settings.py | 2 +- requirements.txt | 12 ++++++------ 6 files changed, 13 insertions(+), 14 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yaml b/.github/ISSUE_TEMPLATE/bug_report.yaml index 9ed740fff..99c060e8a 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yaml +++ b/.github/ISSUE_TEMPLATE/bug_report.yaml @@ -14,7 +14,7 @@ body: attributes: label: NetBox version description: What version of NetBox are you currently running? - placeholder: v3.4.4 + placeholder: v3.4.5 validations: required: true - type: dropdown diff --git a/.github/ISSUE_TEMPLATE/feature_request.yaml b/.github/ISSUE_TEMPLATE/feature_request.yaml index 8e4ab54a5..1338be9c1 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yaml +++ b/.github/ISSUE_TEMPLATE/feature_request.yaml @@ -14,7 +14,7 @@ body: attributes: label: NetBox version description: What version of NetBox are you currently running? - placeholder: v3.4.4 + placeholder: v3.4.5 validations: required: true - type: dropdown diff --git a/docs/configuration/miscellaneous.md b/docs/configuration/miscellaneous.md index eac5d0a2f..8550564d8 100644 --- a/docs/configuration/miscellaneous.md +++ b/docs/configuration/miscellaneous.md @@ -69,11 +69,11 @@ By default, NetBox will permit users to create duplicate prefixes and IP address --- -## FILE_UPLOAD_MAX_MEMORY_SIZE +## `FILE_UPLOAD_MAX_MEMORY_SIZE` -Default: 2621440 (i.e. 2.5 MB). +Default: `2621440` (2.5 MB). -The maximum size (in bytes) that an upload will be before it gets streamed to the file system. Changing this setting can be useful for example to be able to upload files bigger than 2.5MB to custom scripts for processing. +The maximum amount (in bytes) of uploaded data that will be held in memory before being written to the filesystem. Changing this setting can be useful for example to be able to upload files bigger than 2.5MB to custom scripts for processing. --- diff --git a/docs/release-notes/version-3.4.md b/docs/release-notes/version-3.4.md index a006eb7bf..11df3d47a 100644 --- a/docs/release-notes/version-3.4.md +++ b/docs/release-notes/version-3.4.md @@ -1,6 +1,6 @@ # NetBox v3.4 -## v3.4.5 (FUTURE) +## v3.4.5 (2023-02-21) ### Enhancements @@ -13,7 +13,6 @@ * [#11032](https://github.com/netbox-community/netbox/issues/11032) - Fix false custom validation errors during component creation * [#11226](https://github.com/netbox-community/netbox/issues/11226) - Ensure scripts and reports within submodules are automatically reloaded -* [#11335](https://github.com/netbox-community/netbox/issues/11335) - Avoid exception when rendering change log after uninstalling a plugin * [#11459](https://github.com/netbox-community/netbox/issues/11459) - Enable evaluating null values in custom validation rules * [#11473](https://github.com/netbox-community/netbox/issues/11473) - GraphQL requests specifying an invalid filter should return an empty queryset * [#11582](https://github.com/netbox-community/netbox/issues/11582) - Ensure form validation errors are displayed when adding virtual chassis members diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index f6ce7ff33..2d56a025a 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -24,7 +24,7 @@ from netbox.constants import RQ_QUEUE_DEFAULT, RQ_QUEUE_HIGH, RQ_QUEUE_LOW # Environment setup # -VERSION = '3.4.5-dev' +VERSION = '3.4.5' # Hostname HOSTNAME = platform.node() diff --git a/requirements.txt b/requirements.txt index 3cb2529a8..8bbb80d1e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ bleach==5.0.1 -Django==4.1.6 +Django==4.1.7 django-cors-headers==3.13.0 django-debug-toolbar==3.8.1 django-filter==22.1 @@ -9,23 +9,23 @@ django-pglocks==1.0.4 django-prometheus==2.2.0 django-redis==5.2.0 django-rich==1.4.0 -django-rq==2.6.0 -django-tables2==2.5.1 +django-rq==2.7.0 +django-tables2==2.5.2 django-taggit==3.1.0 django-timezone-field==5.0 djangorestframework==3.14.0 -drf-yasg[validation]==1.21.4 +drf-yasg[validation]==1.21.5 graphene-django==3.0.0 gunicorn==20.1.0 Jinja2==3.1.2 Markdown==3.3.7 -mkdocs-material==9.0.10 +mkdocs-material==9.0.13 mkdocstrings[python-legacy]==0.20.0 netaddr==0.8.0 Pillow==9.4.0 psycopg2-binary==2.9.5 PyYAML==6.0 -sentry-sdk==1.14.0 +sentry-sdk==1.15.0 social-auth-app-django==5.0.0 social-auth-core[openidconnect]==4.3.0 svgwrite==1.4.3