diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index ab259af2a..3b37aae56 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -24,7 +24,7 @@ jobs: necessary. close-pr-message: > This PR has been automatically closed due to lack of activity. - days-before-stale: 60 + days-before-stale: 90 days-before-close: 30 exempt-issue-labels: 'status: accepted,status: blocked,status: needs milestone' operations-per-run: 100 diff --git a/docs/administration/housekeeping.md b/docs/administration/housekeeping.md index da1a5443b..fcc3aa04e 100644 --- a/docs/administration/housekeeping.md +++ b/docs/administration/housekeeping.md @@ -5,6 +5,7 @@ NetBox includes a `housekeeping` management command that should be run nightly. * Clearing expired authentication sessions from the database * Deleting changelog records older than the configured [retention time](../configuration/miscellaneous.md#changelog_retention) * Deleting job result records older than the configured [retention time](../configuration/miscellaneous.md#jobresult_retention) +* Check for new NetBox releases (if [`RELEASE_CHECK_URL`](../configuration/miscellaneous.md#release_check_url) is set) This command can be invoked directly, or by using the shell script provided at `/opt/netbox/contrib/netbox-housekeeping.sh`. This script can be linked from your cron scheduler's daily jobs directory (e.g. `/etc/cron.daily`) or referenced directly within the cron configuration file. diff --git a/docs/configuration/miscellaneous.md b/docs/configuration/miscellaneous.md index 4eb090554..eac5d0a2f 100644 --- a/docs/configuration/miscellaneous.md +++ b/docs/configuration/miscellaneous.md @@ -69,6 +69,14 @@ By default, NetBox will permit users to create duplicate prefixes and IP address --- +## FILE_UPLOAD_MAX_MEMORY_SIZE + +Default: 2621440 (i.e. 2.5 MB). + +The maximum size (in bytes) that an upload will be before it gets streamed to the file system. Changing this setting can be useful for example to be able to upload files bigger than 2.5MB to custom scripts for processing. + +--- + ## GRAPHQL_ENABLED !!! tip "Dynamic Configuration Parameter" diff --git a/docs/development/adding-models.md b/docs/development/adding-models.md index aef11d666..7de897a97 100644 --- a/docs/development/adding-models.md +++ b/docs/development/adding-models.md @@ -54,15 +54,19 @@ Each model should have a corresponding FilterSet class defined. This is used to Create a table class for the model in `tables.py` by subclassing `utilities.tables.BaseTable`. Under the table's `Meta` class, be sure to list both the fields and default columns. -## 9. Create the object template +## 9. Create a SearchIndex subclass + +If this model will be included in global search results, create a subclass of `netbox.search.SearchIndex` for it and specify the fields to be indexed. + +## 10. Create the object template Create the HTML template for the object view. (The other views each typically employ a generic template.) This template should extend `generic/object.html`. -## 10. Add the model to the navigation menu +## 11. Add the model to the navigation menu Add the relevant navigation menu items in `netbox/netbox/navigation/menu.py`. -## 11. REST API components +## 12. REST API components Create the following for each model: @@ -71,13 +75,13 @@ Create the following for each model: * API view in `api/views.py` * Endpoint route in `api/urls.py` -## 12. GraphQL API components +## 13. GraphQL API components Create a Graphene object type for the model in `graphql/types.py` by subclassing the appropriate class from `netbox.graphql.types`. Also extend the schema class defined in `graphql/schema.py` with the individual object and object list fields per the established convention. -## 13. Add tests +## 14. Add tests Add tests for the following: @@ -85,7 +89,7 @@ Add tests for the following: * API views * Filter sets -## 14. Documentation +## 15. Documentation Create a new documentation page for the model in `docs/models//.md`. Include this file under the "features" documentation where appropriate. diff --git a/docs/development/application-registry.md b/docs/development/application-registry.md index c2f894711..fe2c08d56 100644 --- a/docs/development/application-registry.md +++ b/docs/development/application-registry.md @@ -8,6 +8,14 @@ The registry can be inspected by importing `registry` from `extras.registry`. ## Stores +### `data_backends` + +A dictionary mapping data backend types to their respective classes. These are used to interact with [remote data sources](../models/core/datasource.md). + +### `denormalized_fields` + +Stores registration made using `netbox.denormalized.register()`. For each model, a list of related models and their field mappings is maintained to facilitate automatic updates. + ### `model_features` A dictionary of particular features (e.g. custom fields) mapped to the NetBox models which support them, arranged by app. For example: @@ -20,38 +28,23 @@ A dictionary of particular features (e.g. custom fields) mapped to the NetBox mo ... }, 'webhooks': { - ... + 'extras': ['configcontext', 'tag', ...], + 'dcim': ['site', 'rack', 'devicetype', ...], }, ... } ``` -### `plugin_menu_items` +Supported model features are listed in the [features matrix](./models.md#features-matrix). -Navigation menu items provided by NetBox plugins. Each plugin is registered as a key with the list of menu items it provides. An example: +### `plugins` -```python -{ - 'Plugin A': ( - , , , - ), - 'Plugin B': ( - , , , - ), -} -``` +This store maintains all registered items for plugins, such as navigation menus, template extensions, etc. -### `plugin_template_extensions` +### `search` -Plugin content that gets embedded into core NetBox templates. The store comprises NetBox models registered as dictionary keys, each pointing to a list of applicable template extension classes that exist. An example: +A dictionary mapping each model (identified by its app and label) to its search index class, if one has been registered for it. -```python -{ - 'dcim.site': [ - , , , - ], - 'dcim.rack': [ - , , - ], -} -``` +### `views` + +A hierarchical mapping of registered views for each model. Mappings are added using the `register_model_view()` decorator, and URLs paths can be generated from these using `get_model_urls()`. diff --git a/docs/development/models.md b/docs/development/models.md index af11617c8..6f3998977 100644 --- a/docs/development/models.md +++ b/docs/development/models.md @@ -2,38 +2,43 @@ ## Model Types -A NetBox model represents a discrete object type such as a device or IP address. Per [Django convention](https://docs.djangoproject.com/en/stable/topics/db/models/), each model is defined as a Python class and has its own SQL table. All NetBox data models can be categorized by type. +A NetBox model represents a discrete object type such as a device or IP address. Per [Django convention](https://docs.djangoproject.com/en/stable/topics/db/models/), each model is defined as a Python class and has its own table in the PostgreSQL database. All NetBox data models can be categorized by type. -The Django [content types](https://docs.djangoproject.com/en/stable/ref/contrib/contenttypes/) framework can be used to reference models within the database. A ContentType instance references a model by its `app_label` and `name`: For example, the Site model is referred to as `dcim.site`. The content type combined with an object's primary key form a globally unique identifier for the object (e.g. `dcim.site:123`). +The Django [content types](https://docs.djangoproject.com/en/stable/ref/contrib/contenttypes/) framework is used to map Django models to database tables. A ContentType instance references a model by its `app_label` and `name`: For example, the Site model within the DCIM app is referred to as `dcim.site`. The content type combined with an object's primary key form a globally unique identifier for the object (e.g. `dcim.site:123`). ### Features Matrix -* [Change logging](../features/change-logging.md) - Changes to these objects are automatically recorded in the change log -* [Webhooks](../integrations/webhooks.md) - NetBox is capable of generating outgoing webhooks for these objects -* [Custom fields](../customization/custom-fields.md) - These models support the addition of user-defined fields -* [Export templates](../customization/export-templates.md) - Users can create custom export templates for these models -* [Tagging](../models/extras/tag.md) - The models can be tagged with user-defined tags -* [Journaling](../features/journaling.md) - These models support persistent historical commentary -* Nesting - These models can be nested recursively to create a hierarchy +Depending on its classification, each NetBox model may support various features which enhance its operation. Each feature is enabled by inheriting from its designated mixin class, and some features also make use of the [application registry](./application-registry.md#model_features). -| Type | Change Logging | Webhooks | Custom Fields | Export Templates | Tags | Journaling | Nesting | -| ------------------ | ---------------- | ---------------- |------------------| ---------------- | ---------------- | ---------------- | ---------------- | -| Primary | :material-check: | :material-check: | :material-check: | :material-check: | :material-check: | :material-check: | | -| Organizational | :material-check: | :material-check: | :material-check: | :material-check: | :material-check: | | | -| Nested Group | :material-check: | :material-check: | :material-check: | :material-check: | :material-check: | | :material-check: | -| Component | :material-check: | :material-check: | :material-check: | :material-check: | :material-check: | | | -| Component Template | :material-check: | :material-check: | | | | | | +| Feature | Feature Mixin | Registry Key | Description | +|------------------------------------------------------------|-------------------------|--------------------|--------------------------------------------------------------------------------| +| [Change logging](../features/change-logging.md) | `ChangeLoggingMixin` | - | Changes to these objects are automatically recorded in the change log | +| Cloning | `CloningMixin` | - | Provides the `clone()` method to prepare a copy | +| [Custom fields](../customization/custom-fields.md) | `CustomFieldsMixin` | `custom_fields` | These models support the addition of user-defined fields | +| [Custom links](../customization/custom-links.md) | `CustomLinksMixin` | `custom_links` | These models support the assignment of custom links | +| [Custom validation](../customization/custom-validation.md) | `CustomValidationMixin` | - | Supports the enforcement of custom validation rules | +| [Export templates](../customization/export-templates.md) | `ExportTemplatesMixin` | `export_templates` | Users can create custom export templates for these models | +| [Job results](../features/background-jobs.md) | `JobResultsMixin` | `job_results` | Users can create custom export templates for these models | +| [Journaling](../features/journaling.md) | `JournalingMixin` | `journaling` | These models support persistent historical commentary | +| [Synchronized data](../integrations/synchronized-data.md) | `SyncedDataMixin` | `synced_data` | Certain model data can be automatically synchronized from a remote data source | +| [Tagging](../models/extras/tag.md) | `TagsMixin` | `tags` | The models can be tagged with user-defined tags | +| [Webhooks](../integrations/webhooks.md) | `WebhooksMixin` | `webhooks` | NetBox is capable of generating outgoing webhooks for these objects | ## Models Index ### Primary Models +These are considered the "core" application models which are used to model network infrastructure. + * [circuits.Circuit](../models/circuits/circuit.md) * [circuits.Provider](../models/circuits/provider.md) * [circuits.ProviderNetwork](../models/circuits/providernetwork.md) +* [core.DataSource](../models/core/datasource.md) * [dcim.Cable](../models/dcim/cable.md) * [dcim.Device](../models/dcim/device.md) * [dcim.DeviceType](../models/dcim/devicetype.md) +* [dcim.Module](../models/dcim/module.md) +* [dcim.ModuleType](../models/dcim/moduletype.md) * [dcim.PowerFeed](../models/dcim/powerfeed.md) * [dcim.PowerPanel](../models/dcim/powerpanel.md) * [dcim.Rack](../models/dcim/rack.md) @@ -47,10 +52,10 @@ The Django [content types](https://docs.djangoproject.com/en/stable/ref/contrib/ * [ipam.IPAddress](../models/ipam/ipaddress.md) * [ipam.IPRange](../models/ipam/iprange.md) * [ipam.L2VPN](../models/ipam/l2vpn.md) -* [ipam.L2VPNTermination](../models/ipam/l2vpntermination.md) * [ipam.Prefix](../models/ipam/prefix.md) * [ipam.RouteTarget](../models/ipam/routetarget.md) * [ipam.Service](../models/ipam/service.md) +* [ipam.ServiceTemplate](../models/ipam/servicetemplate.md) * [ipam.VLAN](../models/ipam/vlan.md) * [ipam.VRF](../models/ipam/vrf.md) * [tenancy.Contact](../models/tenancy/contact.md) @@ -62,6 +67,8 @@ The Django [content types](https://docs.djangoproject.com/en/stable/ref/contrib/ ### Organizational Models +Organization models are used to organize and classify primary models. + * [circuits.CircuitType](../models/circuits/circuittype.md) * [dcim.DeviceRole](../models/dcim/devicerole.md) * [dcim.Manufacturer](../models/dcim/manufacturer.md) @@ -76,6 +83,8 @@ The Django [content types](https://docs.djangoproject.com/en/stable/ref/contrib/ ### Nested Group Models +Nested group models behave like organizational model, but self-nest within a recursive hierarchy. For example, the Region model can be used to represent a hierarchy of countries, states, and cities. + * [dcim.Location](../models/dcim/location.md) (formerly RackGroup) * [dcim.Region](../models/dcim/region.md) * [dcim.SiteGroup](../models/dcim/sitegroup.md) @@ -85,12 +94,15 @@ The Django [content types](https://docs.djangoproject.com/en/stable/ref/contrib/ ### Component Models +Component models represent individual physical or virtual components belonging to a device or virtual machine. + * [dcim.ConsolePort](../models/dcim/consoleport.md) * [dcim.ConsoleServerPort](../models/dcim/consoleserverport.md) * [dcim.DeviceBay](../models/dcim/devicebay.md) * [dcim.FrontPort](../models/dcim/frontport.md) * [dcim.Interface](../models/dcim/interface.md) * [dcim.InventoryItem](../models/dcim/inventoryitem.md) +* [dcim.ModuleBay](../models/dcim/modulebay.md) * [dcim.PowerOutlet](../models/dcim/poweroutlet.md) * [dcim.PowerPort](../models/dcim/powerport.md) * [dcim.RearPort](../models/dcim/rearport.md) @@ -98,11 +110,15 @@ The Django [content types](https://docs.djangoproject.com/en/stable/ref/contrib/ ### Component Template Models +These function as templates to effect the replication of device and virtual machine components. Component template models support a limited feature set, including change logging, custom validation, and webhooks. + * [dcim.ConsolePortTemplate](../models/dcim/consoleporttemplate.md) * [dcim.ConsoleServerPortTemplate](../models/dcim/consoleserverporttemplate.md) * [dcim.DeviceBayTemplate](../models/dcim/devicebaytemplate.md) * [dcim.FrontPortTemplate](../models/dcim/frontporttemplate.md) * [dcim.InterfaceTemplate](../models/dcim/interfacetemplate.md) +* [dcim.InventoryItemTemplate](../models/dcim/inventoryitemtemplate.md) +* [dcim.ModuleBayTemplate](../models/dcim/modulebaytemplate.md) * [dcim.PowerOutletTemplate](../models/dcim/poweroutlettemplate.md) * [dcim.PowerPortTemplate](../models/dcim/powerporttemplate.md) * [dcim.RearPortTemplate](../models/dcim/rearporttemplate.md) diff --git a/docs/features/background-jobs.md b/docs/features/background-jobs.md new file mode 100644 index 000000000..a36192ab3 --- /dev/null +++ b/docs/features/background-jobs.md @@ -0,0 +1,13 @@ +# Background Jobs + +NetBox includes the ability to execute certain functions as background tasks. These include: + +* [Report](../customization/reports.md) execution +* [Custom script](../customization/custom-scripts.md) execution +* Synchronization of [remote data sources](../integrations/synchronized-data.md) + +Additionally, NetBox plugins can enqueue their own background tasks. This is accomplished using the [JobResult model](../models/extras/jobresult.md). Background tasks are executed by the `rqworker` process(es). + +## Scheduled Jobs + +Background jobs can be configured to run immediately, or at a set time in the future. Scheduled jobs can also be configured to repeat at a set interval. diff --git a/docs/integrations/synchronized-data.md b/docs/integrations/synchronized-data.md new file mode 100644 index 000000000..805cbe15b --- /dev/null +++ b/docs/integrations/synchronized-data.md @@ -0,0 +1,9 @@ +# Synchronized Data + +Some NetBox models support automatic synchronization of certain attributes from remote [data sources](../models/core/datasource.md), such as a git repository hosted on GitHub or GitLab. Data from the authoritative remote source is synchronized locally in NetBox as [data files](../models/core/datafile.md). + +The following features support the use of synchronized data: + +* [Configuration templates](../features/configuration-rendering.md) +* [Configuration context data](../features/context-data.md) +* [Export templates](../customization/export-templates.md) diff --git a/docs/introduction.md b/docs/introduction.md index fe82e68aa..bfa0900cb 100644 --- a/docs/introduction.md +++ b/docs/introduction.md @@ -4,7 +4,7 @@ NetBox was originally developed by its lead maintainer, [Jeremy Stretch](https://github.com/jeremystretch), while he was working as a network engineer at [DigitalOcean](https://www.digitalocean.com/) in 2015 as part of an effort to automate their network provisioning. Recognizing the new tool's potential, DigitalOcean agreed to release it as an open source project in June 2016. -Since then, thousands of organizations around the world have embraced NetBox as their central network source of truth to empower both network operators and automation. +Since then, thousands of organizations around the world have embraced NetBox as their central network source of truth to empower both network operators and automation. Today, the open source project is stewarded by [NetBox Labs](https://netboxlabs.com/) and a team of volunteer maintainers. Beyond the core product, myriad [plugins](https://netbox.dev/plugins/) have been developed by the NetBox community to enhance and expand its feature set. ## Key Features @@ -17,6 +17,7 @@ NetBox was built specifically to serve the needs of network engineers and operat * AS number (ASN) management * Rack elevations with SVG rendering * Device modeling using pre-defined types +* Virtual chassis and device contexts * Network, power, and console cabling with SVG traces * Power distribution modeling * Data circuit and provider tracking @@ -29,12 +30,13 @@ NetBox was built specifically to serve the needs of network engineers and operat * Tenant ownership assignment * Device & VM configuration contexts for advanced configuration rendering * Custom fields for data model extension -* Support for custom validation rules +* Custom validation rules * Custom reports & scripts executable directly within the UI * Extensive plugin framework for adding custom functionality * Single sign-on (SSO) authentication * Robust object-based permissions * Detailed, automatic change logging +* Global search engine * NAPALM integration ## What NetBox Is Not diff --git a/docs/models/extras/jobresult.md b/docs/models/extras/jobresult.md new file mode 100644 index 000000000..81ab75745 --- /dev/null +++ b/docs/models/extras/jobresult.md @@ -0,0 +1,54 @@ +# Job Results + +The JobResult model is used to schedule and record the execution of [background tasks](../../features/background-jobs.md). + +## Fields + +### Name + +The name or other identifier of the NetBox object with which the job is associated. + +## Object Type + +The type of object (model) associated with this job. + +### Created + +The date and time at which the job itself was created. + +### Scheduled + +The date and time at which the job is/was scheduled to execute (if not submitted for immediate execution at the time of creation). + +### Interval + +The interval (in minutes) at which a scheduled job should re-execute. + +### Completed + +The date and time at which the job completed (if complete). + +### User + +The user who created the job. + +### Status + +The job's current status. Potential values include: + +| Value | Description | +|-------|-------------| +| Pending | Awaiting execution by an RQ worker process | +| Scheduled | Scheduled for a future date/time | +| Running | Currently executing | +| Completed | Successfully completed | +| Failed | The job did not complete successfully | +| Errored | An unexpected error was encountered during execution | + +### Data + +Any data associated with the execution of the job, such as log output. + +### Job ID + +The job's UUID, used for unique identification within a queue. diff --git a/docs/reference/conditions.md b/docs/reference/conditions.md index fb8b66139..514006b01 100644 --- a/docs/reference/conditions.md +++ b/docs/reference/conditions.md @@ -97,7 +97,7 @@ Multiple conditions can be combined into nested sets using AND or OR logic. This ### Examples -`status` is "active" and `primary_ip` is defined _or_ the "exempt" tag is applied. +`status` is "active" and `primary_ip4` is defined _or_ the "exempt" tag is applied. ```json { @@ -109,8 +109,8 @@ Multiple conditions can be combined into nested sets using AND or OR logic. This "value": "active" }, { - "attr": "primary_ip", - "value": "", + "attr": "primary_ip4", + "value": null, "negate": true } ] diff --git a/docs/release-notes/version-3.4.md b/docs/release-notes/version-3.4.md index 15b84436d..485b85719 100644 --- a/docs/release-notes/version-3.4.md +++ b/docs/release-notes/version-3.4.md @@ -2,6 +2,26 @@ ## v3.4.5 (FUTURE) +### Enhancements + +* [#11110](https://github.com/netbox-community/netbox/issues/11110) - Add `start_address` and `end_address` filters for IP ranges +* [#11592](https://github.com/netbox-community/netbox/issues/11592) - Introduce `FILE_UPLOAD_MAX_MEMORY_SIZE` configuration parameter +* [#11685](https://github.com/netbox-community/netbox/issues/11685) - Match on containing prefixes and aggregates when querying for IP addresses using global search + +### Bug Fixes + +* [#11032](https://github.com/netbox-community/netbox/issues/11032) - Fix false custom validation errors during component creation +* [#11226](https://github.com/netbox-community/netbox/issues/11226) - Ensure scripts and reports within submodules are automatically reloaded +* [#11335](https://github.com/netbox-community/netbox/issues/11335) - Avoid exception when rendering change log after uninstalling a plugin +* [#11459](https://github.com/netbox-community/netbox/issues/11459) - Enable evaluating null values in custom validation rules +* [#11473](https://github.com/netbox-community/netbox/issues/11473) - GraphQL requests specifying an invalid filter should return an empty queryset +* [#11582](https://github.com/netbox-community/netbox/issues/11582) - Ensure form validation errors are displayed when adding virtual chassis members +* [#11601](https://github.com/netbox-community/netbox/issues/11601) - Fix partial matching of start/end addresses for IP range search +* [#11683](https://github.com/netbox-community/netbox/issues/11683) - Fix CSV header attribute detection when auto-detecting import format +* [#11711](https://github.com/netbox-community/netbox/issues/11711) - Fix CSV import for multiple-object custom fields +* [#11723](https://github.com/netbox-community/netbox/issues/11723) - Circuit terminations should link to their associated circuits (rather than site or provider network) +* [#11786](https://github.com/netbox-community/netbox/issues/11786) - List only applicable object types in form widget when filtering custom fields + --- ## v3.4.4 (2023-02-02) diff --git a/mkdocs.yml b/mkdocs.yml index fcfe0d21d..2487176d3 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -77,6 +77,7 @@ nav: - Configuration Rendering: 'features/configuration-rendering.md' - Change Logging: 'features/change-logging.md' - Journaling: 'features/journaling.md' + - Background Jobs: 'features/background-jobs.md' - Auth & Permissions: 'features/authentication-permissions.md' - API & Integration: 'features/api-integration.md' - Customization: 'features/customization.md' @@ -117,6 +118,7 @@ nav: - REST API: 'integrations/rest-api.md' - GraphQL API: 'integrations/graphql-api.md' - Webhooks: 'integrations/webhooks.md' + - Synchronized Data: 'integrations/synchronized-data.md' - NAPALM: 'integrations/napalm.md' - Prometheus Metrics: 'integrations/prometheus-metrics.md' - Plugins: @@ -153,6 +155,9 @@ nav: - Circuit Type: 'models/circuits/circuittype.md' - Provider: 'models/circuits/provider.md' - Provider Network: 'models/circuits/providernetwork.md' + - Core: + - DataFile: 'models/core/datafile.md' + - DataSource: 'models/core/datasource.md' - DCIM: - Cable: 'models/dcim/cable.md' - ConsolePort: 'models/dcim/consoleport.md' @@ -202,6 +207,7 @@ nav: - CustomLink: 'models/extras/customlink.md' - ExportTemplate: 'models/extras/exporttemplate.md' - ImageAttachment: 'models/extras/imageattachment.md' + - JobResult: 'models/extras/jobresult.md' - JournalEntry: 'models/extras/journalentry.md' - SavedFilter: 'models/extras/savedfilter.md' - StagedChange: 'models/extras/stagedchange.md' diff --git a/netbox/circuits/models/circuits.py b/netbox/circuits/models/circuits.py index 53192d39d..a04d78d9f 100644 --- a/netbox/circuits/models/circuits.py +++ b/netbox/circuits/models/circuits.py @@ -193,12 +193,10 @@ class CircuitTermination( ) def __str__(self): - return f'Termination {self.term_side}: {self.site or self.provider_network}' + return f'{self.circuit}: Termination {self.term_side}' def get_absolute_url(self): - if self.site: - return self.site.get_absolute_url() - return self.provider_network.get_absolute_url() + return self.circuit.get_absolute_url() def clean(self): super().clean() diff --git a/netbox/core/forms/model_forms.py b/netbox/core/forms/model_forms.py index e9cc962cd..a3a478be5 100644 --- a/netbox/core/forms/model_forms.py +++ b/netbox/core/forms/model_forms.py @@ -5,7 +5,7 @@ from django import forms from core.models import * from netbox.forms import NetBoxModelForm from netbox.registry import registry -from utilities.forms import CommentField +from utilities.forms import CommentField, get_field_value __all__ = ( 'DataSourceForm', @@ -44,24 +44,18 @@ class DataSourceForm(NetBoxModelForm): ] if self.backend_fields: fieldsets.append( - ('Backend', self.backend_fields) + ('Backend Parameters', self.backend_fields) ) return fieldsets def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + # Determine the selected backend type + backend_type = get_field_value(self, 'type') + backend = registry['data_backends'].get(backend_type) - backend_classes = registry['data_backends'] - - if self.is_bound and self.data.get('type') in backend_classes: - type_ = self.data['type'] - elif self.initial and self.initial.get('type') in backend_classes: - type_ = self.initial['type'] - else: - type_ = self.fields['type'].initial - backend = backend_classes.get(type_) - + # Add backend-specific form fields self.backend_fields = [] for name, form_field in backend.parameters.items(): field_name = f'backend_{name}' diff --git a/netbox/dcim/forms/common.py b/netbox/dcim/forms/common.py index a2243ce2d..f047d621b 100644 --- a/netbox/dcim/forms/common.py +++ b/netbox/dcim/forms/common.py @@ -3,6 +3,7 @@ from django.utils.translation import gettext as _ from dcim.choices import * from dcim.constants import * +from utilities.forms.utils import get_field_value __all__ = ( 'InterfaceCommonForm', @@ -23,6 +24,20 @@ class InterfaceCommonForm(forms.Form): label=_('MTU') ) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # Determine the selected 802.1Q mode + interface_mode = get_field_value(self, 'mode') + + # Delete VLAN tagging fields which are not relevant for the selected mode + if interface_mode in (InterfaceModeChoices.MODE_ACCESS, InterfaceModeChoices.MODE_TAGGED_ALL): + del self.fields['tagged_vlans'] + elif not interface_mode: + del self.fields['vlan_group'] + del self.fields['untagged_vlan'] + del self.fields['tagged_vlans'] + def clean(self): super().clean() diff --git a/netbox/dcim/forms/model_forms.py b/netbox/dcim/forms/model_forms.py index 498fda949..74e697dde 100644 --- a/netbox/dcim/forms/model_forms.py +++ b/netbox/dcim/forms/model_forms.py @@ -452,7 +452,6 @@ class PlatformForm(NetBoxModelForm): fieldsets = ( ('Platform', ( 'name', 'slug', 'manufacturer', 'config_template', 'description', 'tags', - )), ) @@ -1364,6 +1363,13 @@ class InterfaceForm(InterfaceCommonForm, ModularDeviceComponentForm): ] widgets = { 'speed': SelectSpeedWidget(), + 'mode': forms.Select( + attrs={ + 'hx-get': '.', + 'hx-include': '#form_fields input', + 'hx-target': '#form_fields', + } + ), } labels = { 'mode': '802.1Q Mode', diff --git a/netbox/dcim/tables/devices.py b/netbox/dcim/tables/devices.py index 40dc3a244..bed32251c 100644 --- a/netbox/dcim/tables/devices.py +++ b/netbox/dcim/tables/devices.py @@ -133,8 +133,8 @@ class PlatformTable(NetBoxTable): class Meta(NetBoxTable.Meta): model = models.Platform fields = ( - 'pk', 'id', 'name', 'manufacturer', 'device_count', 'vm_count', 'slug', 'config_template', - 'description', 'tags', 'actions', 'created', 'last_updated', + 'pk', 'id', 'name', 'manufacturer', 'device_count', 'vm_count', 'slug', 'config_template', 'description', + 'tags', 'actions', 'created', 'last_updated', ) default_columns = ( 'pk', 'name', 'manufacturer', 'device_count', 'vm_count', 'description', diff --git a/netbox/extras/conditions.py b/netbox/extras/conditions.py index 965488c3a..c6744e524 100644 --- a/netbox/extras/conditions.py +++ b/netbox/extras/conditions.py @@ -44,7 +44,8 @@ class Condition: bool: (EQ, CONTAINS), int: (EQ, GT, GTE, LT, LTE, CONTAINS), float: (EQ, GT, GTE, LT, LTE, CONTAINS), - list: (EQ, IN, CONTAINS) + list: (EQ, IN, CONTAINS), + type(None): (EQ,) } def __init__(self, attr, value, op=EQ, negate=False): diff --git a/netbox/extras/constants.py b/netbox/extras/constants.py index 7c7fe331e..d65fb9612 100644 --- a/netbox/extras/constants.py +++ b/netbox/extras/constants.py @@ -1,14 +1,2 @@ # Webhook content types HTTP_CONTENT_TYPE_JSON = 'application/json' - -# Registerable extras features -EXTRAS_FEATURES = [ - 'custom_fields', - 'custom_links', - 'export_templates', - 'job_results', - 'journaling', - 'synced_data', - 'tags', - 'webhooks' -] diff --git a/netbox/extras/fields.py b/netbox/extras/fields.py new file mode 100644 index 000000000..6cd44432f --- /dev/null +++ b/netbox/extras/fields.py @@ -0,0 +1,8 @@ +from django.db.models import TextField + + +class CachedValueField(TextField): + """ + Currently a dummy field to prevent custom lookups being applied globally to TextField. + """ + pass diff --git a/netbox/extras/forms/filtersets.py b/netbox/extras/forms/filtersets.py index c3609f98b..114eb1a59 100644 --- a/netbox/extras/forms/filtersets.py +++ b/netbox/extras/forms/filtersets.py @@ -40,8 +40,7 @@ class CustomFieldFilterForm(SavedFiltersMixin, FilterForm): ('Attributes', ('type', 'content_type_id', 'group_name', 'weight', 'required', 'ui_visibility')), ) content_type_id = ContentTypeMultipleChoiceField( - queryset=ContentType.objects.all(), - limit_choices_to=FeatureQuery('custom_fields'), + queryset=ContentType.objects.filter(FeatureQuery('custom_fields').get_query()), required=False, label=_('Object type') ) @@ -80,8 +79,7 @@ class JobResultFilterForm(SavedFiltersMixin, FilterForm): ) obj_type = ContentTypeChoiceField( label=_('Object Type'), - queryset=ContentType.objects.all(), - limit_choices_to=FeatureQuery('job_results'), # TODO: This doesn't actually work + queryset=ContentType.objects.filter(FeatureQuery('job_results').get_query()), required=False, ) status = forms.MultipleChoiceField( @@ -136,8 +134,7 @@ class CustomLinkFilterForm(SavedFiltersMixin, FilterForm): ('Attributes', ('content_types', 'enabled', 'new_window', 'weight')), ) content_types = ContentTypeMultipleChoiceField( - queryset=ContentType.objects.all(), - limit_choices_to=FeatureQuery('custom_links'), + queryset=ContentType.objects.filter(FeatureQuery('custom_links').get_query()), required=False ) enabled = forms.NullBooleanField( @@ -177,8 +174,7 @@ class ExportTemplateFilterForm(SavedFiltersMixin, FilterForm): } ) content_types = ContentTypeMultipleChoiceField( - queryset=ContentType.objects.all(), - limit_choices_to=FeatureQuery('export_templates'), + queryset=ContentType.objects.filter(FeatureQuery('export_templates').get_query()), required=False ) mime_type = forms.CharField( @@ -202,8 +198,7 @@ class SavedFilterFilterForm(SavedFiltersMixin, FilterForm): ('Attributes', ('content_types', 'enabled', 'shared', 'weight')), ) content_types = ContentTypeMultipleChoiceField( - queryset=ContentType.objects.all(), - limit_choices_to=FeatureQuery('export_templates'), + queryset=ContentType.objects.filter(FeatureQuery('export_templates').get_query()), required=False ) enabled = forms.NullBooleanField( @@ -230,8 +225,7 @@ class WebhookFilterForm(SavedFiltersMixin, FilterForm): ('Events', ('type_create', 'type_update', 'type_delete')), ) content_type_id = ContentTypeMultipleChoiceField( - queryset=ContentType.objects.all(), - limit_choices_to=FeatureQuery('webhooks'), + queryset=ContentType.objects.filter(FeatureQuery('webhooks').get_query()), required=False, label=_('Object type') ) diff --git a/netbox/extras/lookups.py b/netbox/extras/lookups.py index 7197efcfc..77fe2301e 100644 --- a/netbox/extras/lookups.py +++ b/netbox/extras/lookups.py @@ -1,4 +1,5 @@ -from django.db.models import CharField, Lookup +from django.db.models import CharField, TextField, Lookup +from .fields import CachedValueField class Empty(Lookup): @@ -14,4 +15,18 @@ class Empty(Lookup): return 'CAST(LENGTH(%s) AS BOOLEAN) != %s' % (lhs, rhs), params +class NetContainsOrEquals(Lookup): + """ + This lookup has the same functionality as the one from the ipam app except lhs is cast to inet + """ + lookup_name = 'net_contains_or_equals' + + def as_sql(self, qn, connection): + lhs, lhs_params = self.process_lhs(qn, connection) + rhs, rhs_params = self.process_rhs(qn, connection) + params = lhs_params + rhs_params + return 'CAST(%s AS INET) >>= %s' % (lhs, rhs), params + + CharField.register_lookup(Empty) +CachedValueField.register_lookup(NetContainsOrEquals) diff --git a/netbox/extras/management/commands/housekeeping.py b/netbox/extras/management/commands/housekeeping.py index 42690568d..172e26bf2 100644 --- a/netbox/extras/management/commands/housekeeping.py +++ b/netbox/extras/management/commands/housekeeping.py @@ -37,7 +37,7 @@ class Command(BaseCommand): f"clearing sessions; skipping." ) - # Delete expired ObjectRecords + # Delete expired ObjectChanges if options['verbosity']: self.stdout.write("[*] Checking for expired changelog records") if config.CHANGELOG_RETENTION: diff --git a/netbox/extras/migrations/0083_search.py b/netbox/extras/migrations/0083_search.py index 0c53de638..4c7ae1084 100644 --- a/netbox/extras/migrations/0083_search.py +++ b/netbox/extras/migrations/0083_search.py @@ -1,25 +1,9 @@ -import sys import uuid import django.db.models.deletion import django.db.models.lookups -from django.core import management from django.db import migrations, models - - -def reindex(apps, schema_editor): - # Build the search index (except during tests) - if 'test' not in sys.argv: - management.call_command( - 'reindex', - 'circuits', - 'dcim', - 'extras', - 'ipam', - 'tenancy', - 'virtualization', - 'wireless', - ) +import extras.fields class Migration(migrations.Migration): @@ -49,7 +33,7 @@ class Migration(migrations.Migration): ('object_id', models.PositiveBigIntegerField()), ('field', models.CharField(max_length=200)), ('type', models.CharField(max_length=30)), - ('value', models.TextField()), + ('value', extras.fields.CachedValueField()), ('weight', models.PositiveSmallIntegerField(default=1000)), ('object_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='contenttypes.contenttype')), ], @@ -57,8 +41,4 @@ class Migration(migrations.Migration): 'ordering': ('weight', 'object_type', 'object_id'), }, ), - migrations.RunPython( - code=reindex, - reverse_code=migrations.RunPython.noop - ), ] diff --git a/netbox/extras/models/customfields.py b/netbox/extras/models/customfields.py index d482d7f8d..8141ca76d 100644 --- a/netbox/extras/models/customfields.py +++ b/netbox/extras/models/customfields.py @@ -1,6 +1,6 @@ +import decimal import re from datetime import datetime, date -import decimal import django_filters from django import forms @@ -20,14 +20,15 @@ from netbox.models import ChangeLoggedModel from netbox.models.features import CloningMixin, ExportTemplatesMixin from netbox.search import FieldTypes from utilities import filters -from utilities.forms import ( - CSVChoiceField, CSVMultipleChoiceField, DatePicker, DynamicModelChoiceField, DynamicModelMultipleChoiceField, - JSONField, LaxURLField, add_blank_choice, +from utilities.forms.fields import ( + CSVChoiceField, CSVModelChoiceField, CSVModelMultipleChoiceField, CSVMultipleChoiceField, DynamicModelChoiceField, + DynamicModelMultipleChoiceField, JSONField, LaxURLField, ) +from utilities.forms.utils import add_blank_choice +from utilities.forms.widgets import DatePicker from utilities.querysets import RestrictedQuerySet from utilities.validators import validate_regex - __all__ = ( 'CustomField', 'CustomFieldManager', @@ -409,7 +410,8 @@ class CustomField(CloningMixin, ExportTemplatesMixin, ChangeLoggedModel): # Object elif self.type == CustomFieldTypeChoices.TYPE_OBJECT: model = self.object_type.model_class() - field = DynamicModelChoiceField( + field_class = CSVModelChoiceField if for_csv_import else DynamicModelChoiceField + field = field_class( queryset=model.objects.all(), required=required, initial=initial @@ -418,10 +420,11 @@ class CustomField(CloningMixin, ExportTemplatesMixin, ChangeLoggedModel): # Multiple objects elif self.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: model = self.object_type.model_class() - field = DynamicModelMultipleChoiceField( + field_class = CSVModelMultipleChoiceField if for_csv_import else DynamicModelMultipleChoiceField + field = field_class( queryset=model.objects.all(), required=required, - initial=initial + initial=initial, ) # Text diff --git a/netbox/extras/models/search.py b/netbox/extras/models/search.py index 7c5860e00..6d088abb0 100644 --- a/netbox/extras/models/search.py +++ b/netbox/extras/models/search.py @@ -4,6 +4,7 @@ from django.contrib.contenttypes.models import ContentType from django.db import models from utilities.fields import RestrictedGenericForeignKey +from ..fields import CachedValueField __all__ = ( 'CachedValue', @@ -36,7 +37,7 @@ class CachedValue(models.Model): type = models.CharField( max_length=30 ) - value = models.TextField() + value = CachedValueField() weight = models.PositiveSmallIntegerField( default=1000 ) diff --git a/netbox/extras/plugins/__init__.py b/netbox/extras/plugins/__init__.py index b56113ca1..ee74ad88e 100644 --- a/netbox/extras/plugins/__init__.py +++ b/netbox/extras/plugins/__init__.py @@ -14,13 +14,13 @@ from .registration import * from .templates import * # Initialize plugin registry -registry['plugins'] = { +registry['plugins'].update({ 'graphql_schemas': [], 'menus': [], 'menu_items': {}, 'preferences': {}, 'template_extensions': collections.defaultdict(list), -} +}) DEFAULT_RESOURCE_PATHS = { 'search_indexes': 'search.indexes', diff --git a/netbox/extras/scripts.py b/netbox/extras/scripts.py index 77c96de56..313058d57 100644 --- a/netbox/extras/scripts.py +++ b/netbox/extras/scripts.py @@ -524,27 +524,39 @@ def get_scripts(use_names=False): defined name in place of the actual module name. """ scripts = {} - # Iterate through all modules within the scripts path. These are the user-created files in which reports are + + # Get all modules within the scripts path. These are the user-created files in which scripts are # defined. - for importer, module_name, _ in pkgutil.iter_modules([settings.SCRIPTS_ROOT]): - # Use a lock as removing and loading modules is not thread safe - with lock: - # Remove cached module to ensure consistency with filesystem - if module_name in sys.modules: + modules = list(pkgutil.iter_modules([settings.SCRIPTS_ROOT])) + modules_bases = set([name.split(".")[0] for _, name, _ in modules]) + + # Deleting from sys.modules needs to done behind a lock to prevent race conditions where a module is + # removed from sys.modules while another thread is importing + with lock: + for module_name in list(sys.modules.keys()): + # Everything sharing a base module path with a module in the script folder is removed. + # We also remove all modules with a base module called "scripts". This allows modifying imported + # non-script modules without having to reload the RQ worker. + module_base = module_name.split(".")[0] + if module_base == "scripts" or module_base in modules_bases: del sys.modules[module_name] - module = importer.find_module(module_name).load_module(module_name) + for importer, module_name, _ in modules: + module = importer.find_module(module_name).load_module(module_name) if use_names and hasattr(module, 'name'): module_name = module.name + module_scripts = {} script_order = getattr(module, "script_order", ()) ordered_scripts = [cls for cls in script_order if is_script(cls)] unordered_scripts = [cls for _, cls in inspect.getmembers(module, is_script) if cls not in script_order] + for cls in [*ordered_scripts, *unordered_scripts]: # For scripts in submodules use the full import path w/o the root module as the name script_name = cls.full_name.split(".", maxsplit=1)[1] module_scripts[script_name] = cls + if module_scripts: scripts[module_name] = module_scripts diff --git a/netbox/extras/tests/test_conditions.py b/netbox/extras/tests/test_conditions.py index 8e02eb75d..e7275482a 100644 --- a/netbox/extras/tests/test_conditions.py +++ b/netbox/extras/tests/test_conditions.py @@ -126,6 +126,16 @@ class ConditionSetTest(TestCase): with self.assertRaises(ValueError): ConditionSet({'foo': []}) + def test_null_value(self): + cs = ConditionSet({ + 'and': [ + {'attr': 'a', 'value': None, 'op': 'eq', 'negate': True}, + ] + }) + self.assertFalse(cs.eval({'a': None})) + self.assertTrue(cs.eval({'a': "string"})) + self.assertTrue(cs.eval({'a': {"key": "value"}})) + def test_and_single_depth(self): cs = ConditionSet({ 'and': [ diff --git a/netbox/extras/utils.py b/netbox/extras/utils.py index 268bf9e80..f90858bcf 100644 --- a/netbox/extras/utils.py +++ b/netbox/extras/utils.py @@ -2,7 +2,6 @@ from django.db.models import Q from django.utils.deconstruct import deconstructible from taggit.managers import _TaggableManager -from extras.constants import EXTRAS_FEATURES from netbox.registry import registry @@ -18,7 +17,7 @@ def is_taggable(obj): def image_upload(instance, filename): """ - Return a path for uploading image attchments. + Return a path for uploading image attachments. """ path = 'image-attachments/' @@ -56,8 +55,14 @@ class FeatureQuery: def register_features(model, features): + """ + Register model features in the application registry. + """ + app_label, model_name = model._meta.label_lower.split('.') for feature in features: - if feature not in EXTRAS_FEATURES: - raise ValueError(f"{feature} is not a valid extras feature!") - app_label, model_name = model._meta.label_lower.split('.') - registry['model_features'][feature][app_label].add(model_name) + try: + registry['model_features'][feature][app_label].add(model_name) + except KeyError: + raise KeyError( + f"{feature} is not a valid model feature! Valid keys are: {registry['model_features'].keys()}" + ) diff --git a/netbox/ipam/filtersets.py b/netbox/ipam/filtersets.py index d069eed27..2e9f56bbc 100644 --- a/netbox/ipam/filtersets.py +++ b/netbox/ipam/filtersets.py @@ -405,6 +405,14 @@ class IPRangeFilterSet(TenancyFilterSet, NetBoxModelFilterSet): field_name='start_address', lookup_expr='family' ) + start_address = MultiValueCharFilter( + method='filter_address', + label=_('Address'), + ) + end_address = MultiValueCharFilter( + method='filter_address', + label=_('Address'), + ) contains = django_filters.CharFilter( method='search_contains', label=_('Ranges which contain this prefix or IP'), @@ -441,9 +449,9 @@ class IPRangeFilterSet(TenancyFilterSet, NetBoxModelFilterSet): def search(self, queryset, name, value): if not value.strip(): return queryset - qs_filter = Q(description__icontains=value) + qs_filter = Q(description__icontains=value) | Q(start_address__contains=value) | Q(end_address__contains=value) try: - ipaddress = str(netaddr.IPNetwork(value.strip()).cidr) + ipaddress = str(netaddr.IPNetwork(value.strip())) qs_filter |= Q(start_address=ipaddress) qs_filter |= Q(end_address=ipaddress) except (AddrFormatError, ValueError): @@ -461,6 +469,12 @@ class IPRangeFilterSet(TenancyFilterSet, NetBoxModelFilterSet): except (AddrFormatError, ValueError): return queryset.none() + def filter_address(self, queryset, name, value): + try: + return queryset.filter(**{f'{name}__net_in': value}) + except ValidationError: + return queryset.none() + class IPAddressFilterSet(NetBoxModelFilterSet, TenancyFilterSet): family = django_filters.NumberFilter( diff --git a/netbox/ipam/search.py b/netbox/ipam/search.py index fd6db6a63..ad4403321 100644 --- a/netbox/ipam/search.py +++ b/netbox/ipam/search.py @@ -6,7 +6,7 @@ from netbox.search import SearchIndex, register_search class AggregateIndex(SearchIndex): model = models.Aggregate fields = ( - ('prefix', 100), + ('prefix', 120), ('description', 500), ('date_added', 2000), ('comments', 5000), @@ -70,7 +70,7 @@ class L2VPNIndex(SearchIndex): class PrefixIndex(SearchIndex): model = models.Prefix fields = ( - ('prefix', 100), + ('prefix', 110), ('description', 500), ('comments', 5000), ) diff --git a/netbox/ipam/tests/test_filtersets.py b/netbox/ipam/tests/test_filtersets.py index 711009a7e..13b3ae163 100644 --- a/netbox/ipam/tests/test_filtersets.py +++ b/netbox/ipam/tests/test_filtersets.py @@ -680,6 +680,14 @@ class IPRangeTestCase(TestCase, ChangeLoggedFilterSetTests): params = {'family': '6'} self.assertEqual(self.filterset(params, self.queryset).qs.count(), 4) + def test_start_address(self): + params = {'start_address': ['10.0.1.100', '10.0.2.100']} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + + def test_end_address(self): + params = {'end_address': ['10.0.1.199', '10.0.2.199']} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + def test_contains(self): params = {'contains': '10.0.1.150/24'} self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) diff --git a/netbox/netbox/configuration_example.py b/netbox/netbox/configuration_example.py index 14fcde022..7158308af 100644 --- a/netbox/netbox/configuration_example.py +++ b/netbox/netbox/configuration_example.py @@ -217,6 +217,10 @@ RQ_DEFAULT_TIMEOUT = 300 # this setting is derived from the installed location. # SCRIPTS_ROOT = '/opt/netbox/netbox/scripts' +# The maximum size (in bytes) that an upload will be before it gets streamed to the file system. +# Useful to be able to upload files bigger than 2.5Mbyte to custom scripts for processing. +# FILE_UPLOAD_MAX_MEMORY_SIZE = 2621440 + # The name to use for the csrf token cookie. CSRF_COOKIE_NAME = 'csrftoken' diff --git a/netbox/netbox/graphql/fields.py b/netbox/netbox/graphql/fields.py index 5b8e6cc5b..7c359e82e 100644 --- a/netbox/netbox/graphql/fields.py +++ b/netbox/netbox/graphql/fields.py @@ -60,6 +60,8 @@ class ObjectListField(DjangoListField): filterset_class = django_object_type._meta.filterset_class if filterset_class: filterset = filterset_class(data=args, queryset=queryset, request=info.context) + if not filterset.is_valid(): + return queryset.none() return filterset.qs return queryset diff --git a/netbox/netbox/models/features.py b/netbox/netbox/models/features.py index f31b6e071..e70d3df7b 100644 --- a/netbox/netbox/models/features.py +++ b/netbox/netbox/models/features.py @@ -12,6 +12,7 @@ from taggit.managers import TaggableManager from extras.choices import CustomFieldVisibilityChoices, ObjectChangeActionChoices from extras.utils import is_taggable, register_features +from netbox.registry import registry from netbox.signals import post_clean from utilities.json import CustomFieldJSONEncoder from utilities.utils import serialize_object @@ -259,6 +260,10 @@ class CustomValidationMixin(models.Model): def clean(self): super().clean() + # If the instance is a base for replications, skip custom validation + if getattr(self, '_replicated_base', False): + return + # Send the post_clean signal post_clean.send(sender=self.__class__, instance=self) @@ -384,22 +389,26 @@ class SyncedDataMixin(models.Model): raise NotImplementedError(f"{self.__class__} must implement a sync_data() method.") -FEATURES_MAP = ( - ('custom_fields', CustomFieldsMixin), - ('custom_links', CustomLinksMixin), - ('export_templates', ExportTemplatesMixin), - ('job_results', JobResultsMixin), - ('journaling', JournalingMixin), - ('synced_data', SyncedDataMixin), - ('tags', TagsMixin), - ('webhooks', WebhooksMixin), -) +FEATURES_MAP = { + 'custom_fields': CustomFieldsMixin, + 'custom_links': CustomLinksMixin, + 'export_templates': ExportTemplatesMixin, + 'job_results': JobResultsMixin, + 'journaling': JournalingMixin, + 'synced_data': SyncedDataMixin, + 'tags': TagsMixin, + 'webhooks': WebhooksMixin, +} + +registry['model_features'].update({ + feature: defaultdict(set) for feature in FEATURES_MAP.keys() +}) @receiver(class_prepared) def _register_features(sender, **kwargs): features = { - feature for feature, cls in FEATURES_MAP if issubclass(sender, cls) + feature for feature, cls in FEATURES_MAP.items() if issubclass(sender, cls) } register_features(sender, features) diff --git a/netbox/netbox/registry.py b/netbox/netbox/registry.py index 670bca683..e37ee0d0c 100644 --- a/netbox/netbox/registry.py +++ b/netbox/netbox/registry.py @@ -1,12 +1,10 @@ import collections -from extras.constants import EXTRAS_FEATURES - class Registry(dict): """ - Central registry for registration of functionality. Once a store (key) is defined, it cannot be overwritten or - deleted (although its value may be manipulated). + Central registry for registration of functionality. Once a Registry is initialized, keys cannot be added or + removed (though the value of each key is mutable). """ def __getitem__(self, key): try: @@ -15,20 +13,18 @@ class Registry(dict): raise KeyError(f"Invalid store: {key}") def __setitem__(self, key, value): - if key in self: - raise KeyError(f"Store already set: {key}") - super().__setitem__(key, value) + raise TypeError("Cannot add stores to registry after initialization") def __delitem__(self, key): raise TypeError("Cannot delete stores from registry") # Initialize the global registry -registry = Registry() -registry['data_backends'] = dict() -registry['denormalized_fields'] = collections.defaultdict(list) -registry['model_features'] = { - feature: collections.defaultdict(set) for feature in EXTRAS_FEATURES -} -registry['search'] = dict() -registry['views'] = collections.defaultdict(dict) +registry = Registry({ + 'data_backends': dict(), + 'denormalized_fields': collections.defaultdict(list), + 'model_features': dict(), + 'plugins': dict(), + 'search': dict(), + 'views': collections.defaultdict(dict), +}) diff --git a/netbox/netbox/search/__init__.py b/netbox/netbox/search/__init__.py index 1eec8e097..6d53e9a97 100644 --- a/netbox/netbox/search/__init__.py +++ b/netbox/netbox/search/__init__.py @@ -2,6 +2,7 @@ from collections import namedtuple from django.db import models +from ipam.fields import IPAddressField, IPNetworkField from netbox.registry import registry ObjectFieldValue = namedtuple('ObjectFieldValue', ('name', 'type', 'weight', 'value')) @@ -11,6 +12,8 @@ class FieldTypes: FLOAT = 'float' INTEGER = 'int' STRING = 'str' + INET = 'inet' + CIDR = 'cidr' class LookupTypes: @@ -43,6 +46,10 @@ class SearchIndex: field_cls = instance._meta.get_field(field_name).__class__ if issubclass(field_cls, (models.FloatField, models.DecimalField)): return FieldTypes.FLOAT + if issubclass(field_cls, IPAddressField): + return FieldTypes.INET + if issubclass(field_cls, IPNetworkField): + return FieldTypes.CIDR if issubclass(field_cls, models.IntegerField): return FieldTypes.INTEGER return FieldTypes.STRING diff --git a/netbox/netbox/search/backends.py b/netbox/netbox/search/backends.py index d659a7abb..10e164c09 100644 --- a/netbox/netbox/search/backends.py +++ b/netbox/netbox/search/backends.py @@ -3,10 +3,12 @@ from collections import defaultdict from django.conf import settings from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ImproperlyConfigured -from django.db.models import F, Window +from django.db.models import F, Window, Q from django.db.models.functions import window from django.db.models.signals import post_delete, post_save from django.utils.module_loading import import_string +import netaddr +from netaddr.core import AddrFormatError from extras.models import CachedValue, CustomField from netbox.registry import registry @@ -95,18 +97,24 @@ class CachedValueSearchBackend(SearchBackend): def search(self, value, user=None, object_types=None, lookup=DEFAULT_LOOKUP_TYPE): - # Define the search parameters - params = { - f'value__{lookup}': value - } + query_filter = Q(**{f'value__{lookup}': value}) + + if object_types: + query_filter &= Q(object_type__in=object_types) + if lookup in (LookupTypes.STARTSWITH, LookupTypes.ENDSWITH): # Partial string matches are valid only on string values - params['type'] = FieldTypes.STRING - if object_types: - params['object_type__in'] = object_types + query_filter &= Q(type=FieldTypes.STRING) + + if lookup == LookupTypes.PARTIAL: + try: + address = str(netaddr.IPNetwork(value.strip()).cidr) + query_filter |= Q(type=FieldTypes.CIDR) & Q(value__net_contains_or_equals=address) + except (AddrFormatError, ValueError): + pass # Construct the base queryset to retrieve matching results - queryset = CachedValue.objects.filter(**params).annotate( + queryset = CachedValue.objects.filter(query_filter).annotate( # Annotate the rank of each result for its object according to its weight row_number=Window( expression=window.RowNumber(), diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index c22adf6c2..22849e6ba 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -91,6 +91,7 @@ DOCS_ROOT = getattr(configuration, 'DOCS_ROOT', os.path.join(os.path.dirname(BAS EMAIL = getattr(configuration, 'EMAIL', {}) EXEMPT_VIEW_PERMISSIONS = getattr(configuration, 'EXEMPT_VIEW_PERMISSIONS', []) FIELD_CHOICES = getattr(configuration, 'FIELD_CHOICES', {}) +FILE_UPLOAD_MAX_MEMORY_SIZE = getattr(configuration, 'FILE_UPLOAD_MAX_MEMORY_SIZE', 2621440) HTTP_PROXIES = getattr(configuration, 'HTTP_PROXIES', None) INTERNAL_IPS = getattr(configuration, 'INTERNAL_IPS', ('127.0.0.1', '::1')) JINJA2_FILTERS = getattr(configuration, 'JINJA2_FILTERS', {}) diff --git a/netbox/netbox/tests/test_registry.py b/netbox/netbox/tests/test_registry.py index 25f9e43ec..e834c4356 100644 --- a/netbox/netbox/tests/test_registry.py +++ b/netbox/netbox/tests/test_registry.py @@ -5,29 +5,23 @@ from netbox.registry import Registry class RegistryTest(TestCase): - def test_add_store(self): - reg = Registry() - reg['foo'] = 123 + def test_set_store(self): + reg = Registry({ + 'foo': 123, + }) + with self.assertRaises(TypeError): + reg['bar'] = 456 - self.assertEqual(reg['foo'], 123) - - def test_manipulate_store(self): - reg = Registry() - reg['foo'] = [1, 2] + def test_mutate_store(self): + reg = Registry({ + 'foo': [1, 2], + }) reg['foo'].append(3) - self.assertListEqual(reg['foo'], [1, 2, 3]) - def test_overwrite_store(self): - reg = Registry() - reg['foo'] = 123 - - with self.assertRaises(KeyError): - reg['foo'] = 456 - def test_delete_store(self): - reg = Registry() - reg['foo'] = 123 - + reg = Registry({ + 'foo': 123, + }) with self.assertRaises(TypeError): del reg['foo'] diff --git a/netbox/netbox/views/generic/bulk_views.py b/netbox/netbox/views/generic/bulk_views.py index 8dfd2c1e3..325d10338 100644 --- a/netbox/netbox/views/generic/bulk_views.py +++ b/netbox/netbox/views/generic/bulk_views.py @@ -389,8 +389,8 @@ class BulkImportView(GetReturnURLMixin, BaseMultiObjectView): 'data': record, 'instance': instance, } - if form.cleaned_data['format'] == ImportFormatChoices.CSV: - model_form_kwargs['headers'] = form._csv_headers + if hasattr(form, '_csv_headers'): + model_form_kwargs['headers'] = form._csv_headers # Add CSV headers model_form = self.model_form(**model_form_kwargs) # When updating, omit all form fields other than those specified in the record. (No diff --git a/netbox/netbox/views/generic/object_views.py b/netbox/netbox/views/generic/object_views.py index c8773d685..1ba789cf1 100644 --- a/netbox/netbox/views/generic/object_views.py +++ b/netbox/netbox/views/generic/object_views.py @@ -431,6 +431,12 @@ class ComponentCreateView(GetReturnURLMixin, BaseObjectView): form = self.initialize_form(request) instance = self.alter_object(self.queryset.model(), request) + # If this is an HTMX request, return only the rendered form HTML + if is_htmx(request): + return render(request, 'htmx/form.html', { + 'form': form, + }) + return render(request, self.template_name, { 'object': instance, 'form': form, @@ -442,6 +448,10 @@ class ComponentCreateView(GetReturnURLMixin, BaseObjectView): form = self.initialize_form(request) instance = self.alter_object(self.queryset.model(), request) + # Note that the form instance is a replicated field base + # This is needed to avoid running custom validators multiple times + form.instance._replicated_base = hasattr(self.form, "replication_fields") + if form.is_valid(): new_components = [] data = deepcopy(request.POST) diff --git a/netbox/project-static/dist/netbox.js b/netbox/project-static/dist/netbox.js index 937249e84..d0058eae9 100644 Binary files a/netbox/project-static/dist/netbox.js and b/netbox/project-static/dist/netbox.js differ diff --git a/netbox/project-static/dist/netbox.js.map b/netbox/project-static/dist/netbox.js.map index 1c72bdb37..5481e38a3 100644 Binary files a/netbox/project-static/dist/netbox.js.map and b/netbox/project-static/dist/netbox.js.map differ diff --git a/netbox/project-static/src/forms/index.ts b/netbox/project-static/src/forms/index.ts index 1ef8540fd..f166c75a8 100644 --- a/netbox/project-static/src/forms/index.ts +++ b/netbox/project-static/src/forms/index.ts @@ -1,10 +1,9 @@ import { initFormElements } from './elements'; import { initSpeedSelector } from './speedSelector'; import { initScopeSelector } from './scopeSelector'; -import { initVlanTags } from './vlanTags'; export function initForms(): void { - for (const func of [initFormElements, initSpeedSelector, initScopeSelector, initVlanTags]) { + for (const func of [initFormElements, initSpeedSelector, initScopeSelector]) { func(); } } diff --git a/netbox/project-static/src/forms/vlanTags.ts b/netbox/project-static/src/forms/vlanTags.ts deleted file mode 100644 index 4ad97c363..000000000 --- a/netbox/project-static/src/forms/vlanTags.ts +++ /dev/null @@ -1,148 +0,0 @@ -import { all, getElement, resetSelect, toggleVisibility as _toggleVisibility } from '../util'; - -/** - * Get a select element's containing `.row` element. - * - * @param element Select element. - * @returns Containing row element. - */ -function fieldContainer(element: Nullable): Nullable { - const container = element?.parentElement?.parentElement ?? null; - if (container !== null && container.classList.contains('row')) { - return container; - } - return null; -} - -/** - * Toggle visibility of the select element's container and disable the select element itself. - * - * @param element Select element. - * @param action 'show' or 'hide' - */ -function toggleVisibility>( - element: E, - action: 'show' | 'hide', -): void { - // Find the select element's containing element. - const parent = fieldContainer(element); - if (element !== null && parent !== null) { - // Toggle container visibility to visually remove it from the form. - _toggleVisibility(parent, action); - // Create a new event so that the APISelect instance properly handles the enable/disable - // action. - const event = new Event(`netbox.select.disabled.${element.name}`); - switch (action) { - case 'hide': - // Disable the native select element and dispatch the event APISelect is listening for. - element.disabled = true; - element.dispatchEvent(event); - break; - case 'show': - // Enable the native select element and dispatch the event APISelect is listening for. - element.disabled = false; - element.dispatchEvent(event); - } - } -} - -/** - * Toggle element visibility when the mode field does not have a value. - */ -function handleModeNone(): void { - const elements = [ - getElement('id_tagged_vlans'), - getElement('id_untagged_vlan'), - getElement('id_vlan_group'), - ]; - - if (all(elements)) { - const [taggedVlans, untaggedVlan] = elements; - resetSelect(untaggedVlan); - resetSelect(taggedVlans); - for (const element of elements) { - toggleVisibility(element, 'hide'); - } - } -} - -/** - * Toggle element visibility when the mode field's value is Access. - */ -function handleModeAccess(): void { - const elements = [ - getElement('id_tagged_vlans'), - getElement('id_untagged_vlan'), - getElement('id_vlan_group'), - ]; - if (all(elements)) { - const [taggedVlans, untaggedVlan, vlanGroup] = elements; - resetSelect(taggedVlans); - toggleVisibility(vlanGroup, 'show'); - toggleVisibility(untaggedVlan, 'show'); - toggleVisibility(taggedVlans, 'hide'); - } -} - -/** - * Toggle element visibility when the mode field's value is Tagged. - */ -function handleModeTagged(): void { - const elements = [ - getElement('id_tagged_vlans'), - getElement('id_untagged_vlan'), - getElement('id_vlan_group'), - ]; - if (all(elements)) { - const [taggedVlans, untaggedVlan, vlanGroup] = elements; - toggleVisibility(taggedVlans, 'show'); - toggleVisibility(vlanGroup, 'show'); - toggleVisibility(untaggedVlan, 'show'); - } -} - -/** - * Toggle element visibility when the mode field's value is Tagged (All). - */ -function handleModeTaggedAll(): void { - const elements = [ - getElement('id_tagged_vlans'), - getElement('id_untagged_vlan'), - getElement('id_vlan_group'), - ]; - if (all(elements)) { - const [taggedVlans, untaggedVlan, vlanGroup] = elements; - resetSelect(taggedVlans); - toggleVisibility(vlanGroup, 'show'); - toggleVisibility(untaggedVlan, 'show'); - toggleVisibility(taggedVlans, 'hide'); - } -} - -/** - * Reset field visibility when the mode field's value changes. - */ -function handleModeChange(element: HTMLSelectElement): void { - switch (element.value) { - case 'access': - handleModeAccess(); - break; - case 'tagged': - handleModeTagged(); - break; - case 'tagged-all': - handleModeTaggedAll(); - break; - case '': - handleModeNone(); - break; - } -} - -export function initVlanTags(): void { - const element = getElement('id_mode'); - if (element !== null) { - element.addEventListener('change', () => handleModeChange(element)); - handleModeChange(element); - } -} diff --git a/netbox/templates/dcim/interface_edit.html b/netbox/templates/dcim/interface_edit.html deleted file mode 100644 index a044de660..000000000 --- a/netbox/templates/dcim/interface_edit.html +++ /dev/null @@ -1,101 +0,0 @@ -{% extends 'generic/object_edit.html' %} -{% load form_helpers %} - -{% block form %} - {# Render hidden fields #} - {% for field in form.hidden_fields %} - {{ field }} - {% endfor %} - -
-
-
Interface
-
- {% if form.instance.device %} -
- -
- -
-
- {% endif %} - {% render_field form.module %} - {% render_field form.name %} - {% render_field form.type %} - {% render_field form.speed %} - {% render_field form.duplex %} - {% render_field form.label %} - {% render_field form.description %} - {% render_field form.tags %} -
- -
-
-
Addressing
-
- {% render_field form.vrf %} - {% render_field form.mac_address %} - {% render_field form.wwn %} -
- -
-
-
Operation
-
- {% render_field form.mtu %} - {% render_field form.tx_power %} - {% render_field form.enabled %} - {% render_field form.mgmt_only %} - {% render_field form.mark_connected %} -
- -
-
-
Related Interfaces
-
- {% render_field form.parent %} - {% render_field form.bridge %} - {% render_field form.lag %} -
- - {% if form.instance.is_wireless %} -
-
-
Wireless
-
- {% render_field form.rf_role %} - {% render_field form.rf_channel %} - {% render_field form.rf_channel_frequency %} - {% render_field form.rf_channel_width %} - {% render_field form.wireless_lan_group %} - {% render_field form.wireless_lans %} -
- {% endif %} - -
-
-
Power over Ethernet (PoE)
-
- {% render_field form.poe_mode %} - {% render_field form.poe_type %} -
- -
-
-
802.1Q Switching
-
- {% render_field form.mode %} - {% render_field form.vlan_group %} - {% render_field form.untagged_vlan %} - {% render_field form.tagged_vlans %} -
- - {% if form.custom_fields %} -
-
-
Custom Fields
-
- {% render_custom_fields form %} -
- {% endif %} -{% endblock %} diff --git a/netbox/templates/dcim/virtualchassis_add_member.html b/netbox/templates/dcim/virtualchassis_add_member.html index 17ffd64d9..bc2ba2f55 100644 --- a/netbox/templates/dcim/virtualchassis_add_member.html +++ b/netbox/templates/dcim/virtualchassis_add_member.html @@ -5,6 +5,8 @@ {% block content %}
+ {% render_errors membership_form %} + {% csrf_token %}
Add New Member
diff --git a/netbox/templates/dcim/virtualchassis_edit.html b/netbox/templates/dcim/virtualchassis_edit.html index f98a9fe64..433837cf5 100644 --- a/netbox/templates/dcim/virtualchassis_edit.html +++ b/netbox/templates/dcim/virtualchassis_edit.html @@ -8,6 +8,10 @@
+ {% for form in formset %} + {% render_errors form %} + {% endfor %} + {% csrf_token %} {{ pk_form.pk }} {{ formset.management_form }} diff --git a/netbox/templates/htmx/form.html b/netbox/templates/htmx/form.html index e5a2ab6c6..e15df4706 100644 --- a/netbox/templates/htmx/form.html +++ b/netbox/templates/htmx/form.html @@ -17,7 +17,7 @@ {% endif %} {% for name in fields %} {% with field=form|getfield:name %} - {% if not field.field.widget.is_hidden %} + {% if field and not field.field.widget.is_hidden %} {% render_field field %} {% endif %} {% endwith %} diff --git a/netbox/users/views.py b/netbox/users/views.py index 832a4e592..a82620914 100644 --- a/netbox/users/views.py +++ b/netbox/users/views.py @@ -96,7 +96,7 @@ class LoginView(View): # Authenticate user auth_login(request, form.get_user()) logger.info(f"User {request.user} successfully authenticated") - messages.info(request, f"Logged in as {request.user}.") + messages.success(request, f"Logged in as {request.user}.") # Ensure the user has a UserConfig defined. (This should normally be handled by # create_userconfig() on user creation.) diff --git a/netbox/utilities/custom_inspectors.py b/netbox/utilities/custom_inspectors.py index 25358535d..7cf9fe02f 100644 --- a/netbox/utilities/custom_inspectors.py +++ b/netbox/utilities/custom_inspectors.py @@ -18,7 +18,7 @@ class NetBoxSwaggerAutoSchema(SwaggerAutoSchema): if not operation_id: # Overwrite the action for bulk update/bulk delete views to ensure they get an operation ID that's # unique from their single-object counterparts (see #3436) - if operation_keys[-1] in ('delete', 'partial_update', 'update') and not self.view.detail: + if operation_keys[-1] in ('delete', 'partial_update', 'update') and not getattr(self.view, 'detail', None): operation_keys[-1] = f'bulk_{operation_keys[-1]}' operation_id = '_'.join(operation_keys) diff --git a/netbox/utilities/forms/forms.py b/netbox/utilities/forms/forms.py index 5dd953cfe..a008a2c8c 100644 --- a/netbox/utilities/forms/forms.py +++ b/netbox/utilities/forms/forms.py @@ -200,6 +200,8 @@ class ImportForm(BootstrapMixin, forms.Form): self.cleaned_data['data'] = self._clean_json(data) elif format == ImportFormatChoices.YAML: self.cleaned_data['data'] = self._clean_yaml(data) + else: + raise forms.ValidationError(f"Unknown data format: {format}") def _detect_format(self, data): """ diff --git a/netbox/utilities/forms/utils.py b/netbox/utilities/forms/utils.py index 1a2f62b2e..2f08a3cce 100644 --- a/netbox/utilities/forms/utils.py +++ b/netbox/utilities/forms/utils.py @@ -12,6 +12,7 @@ __all__ = ( 'expand_alphanumeric_pattern', 'expand_ipaddress_pattern', 'form_from_model', + 'get_field_value', 'get_selected_values', 'parse_alphanumeric_range', 'parse_numeric_range', @@ -113,6 +114,21 @@ def expand_ipaddress_pattern(string, family): yield ''.join([lead, format(i, 'x' if family == 6 else 'd'), remnant]) +def get_field_value(form, field_name): + """ + Return the current bound or initial value associated with a form field, prior to calling + clean() for the form. + """ + field = form.fields[field_name] + + if form.is_bound: + if data := form.data.get(field_name): + if field.valid_value(data): + return data + + return form.get_initial_for_field(field, field_name) + + def get_selected_values(form, field_name): """ Return the list of selected human-friendly values for a form field diff --git a/netbox/utilities/templatetags/form_helpers.py b/netbox/utilities/templatetags/form_helpers.py index 089a3ced9..a3523a7cc 100644 --- a/netbox/utilities/templatetags/form_helpers.py +++ b/netbox/utilities/templatetags/form_helpers.py @@ -11,9 +11,12 @@ register = template.Library() @register.filter() def getfield(form, fieldname): """ - Return the specified field of a Form. + Return the specified bound field of a Form. """ - return form[fieldname] + try: + return form[fieldname] + except KeyError: + return None @register.filter(name='widget_type') diff --git a/netbox/virtualization/forms/model_forms.py b/netbox/virtualization/forms/model_forms.py index 403a04d91..e461eac8a 100644 --- a/netbox/virtualization/forms/model_forms.py +++ b/netbox/virtualization/forms/model_forms.py @@ -349,6 +349,15 @@ class VMInterfaceForm(InterfaceCommonForm, NetBoxModelForm): labels = { 'mode': '802.1Q Mode', } + widgets = { + 'mode': forms.Select( + attrs={ + 'hx-get': '.', + 'hx-include': '#form_fields input', + 'hx-target': '#form_fields', + } + ), + } help_texts = { 'mode': INTERFACE_MODE_HELP_TEXT, }