diff --git a/.gitignore b/.gitignore index d859bad28..36c6d3fa8 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ /netbox/netbox/ldap_config.py /netbox/reports/* !/netbox/reports/__init__.py +/netbox/scripts/* +!/netbox/scripts/__init__.py /netbox/static .idea /*.sh diff --git a/docs/additional-features/custom-scripts.md b/docs/additional-features/custom-scripts.md new file mode 100644 index 000000000..328bb617d --- /dev/null +++ b/docs/additional-features/custom-scripts.md @@ -0,0 +1,213 @@ +# Custom Scripts + +Custom scripting was introduced in NetBox v2.7 to provide a way for users to execute custom logic from within the NetBox UI. Custom scripts enable the user to directly and conveniently manipulate NetBox data in a prescribed fashion. They can be used to accomplish myriad tasks, such as: + +* Automatically populate new devices and cables in preparation for a new site deployment +* Create a range of new reserved prefixes or IP addresses +* Fetch data from an external source and import it to NetBox + +Custom scripts are Python code and exist outside of the official NetBox code base, so they can be updated and changed without interfering with the core NetBox installation. And because they're written from scratch, a custom script can be used to accomplish just about anything. + +## Writing Custom Scripts + +All custom scripts must inherit from the `extras.scripts.Script` base class. This class provides the functionality necessary to generate forms and log activity. + +``` +from extras.scripts import Script + +class MyScript(Script): + .. +``` + +Scripts comprise two core components: variables and a `run()` method. Variables allow your script to accept user input via the NetBox UI. The `run()` method is where your script's execution logic lives. (Note that your script can have as many methods as needed: this is merely the point of invocation for NetBox.) + +``` +class MyScript(Script): + var1 = StringVar(...) + var2 = IntegerVar(...) + var3 = ObjectVar(...) + + def run(self, data): + ... +``` + +The `run()` method is passed a single argument: a dictionary containing all of the variable data passed via the web form. Your script can reference this data during execution. + +Defining variables is optional: You may create a script with only a `run()` method if no user input is needed. + +Returning output from your script is optional. Any raw output generated by the script will be displayed under the "output" tab in the UI. + +## Module Attributes + +### `name` + +You can define `name` within a script module (the Python file which contains one or more scripts) to set the module name. If `name` is not defined, the filename will be used. + +## Script Attributes + +Script attributes are defined under a class named `Meta` within the script. These are optional, but encouraged. + +### `name` + +This is the human-friendly names of your script. If omitted, the class name will be used. + +### `description` + +A human-friendly description of what your script does. + +### `field_order` + +A list of field names indicating the order in which the form fields should appear. This is optional, however on Python 3.5 and earlier the fields will appear in random order. (Declarative ordering is preserved on Python 3.6 and above.) For example: + +``` +field_order = ['var1', 'var2', 'var3'] +``` + +## Reading Data from Files + +The Script class provides two convenience methods for reading data from files: + +* `load_yaml` +* `load_json` + +These two methods will load data in YAML or JSON format, respectively, from files within the local path (i.e. `SCRIPTS_ROOT`). + +## Logging + +The Script object provides a set of convenient functions for recording messages at different severity levels: + +* `log_debug` +* `log_success` +* `log_info` +* `log_warning` +* `log_failure` + +Log messages are returned to the user upon execution of the script. Markdown rendering is supported for log messages. + +## Variable Reference + +### StringVar + +Stores a string of characters (i.e. a line of text). Options include: + +* `min_length` - Minimum number of characters +* `max_length` - Maximum number of characters +* `regex` - A regular expression against which the provided value must match + +Note: `min_length` and `max_length` can be set to the same number to effect a fixed-length field. + +### TextVar + +Arbitrary text of any length. Renders as multi-line text input field. + +### IntegerVar + +Stored a numeric integer. Options include: + +* `min_value:` - Minimum value +* `max_value` - Maximum value + +### BooleanVar + +A true/false flag. This field has no options beyond the defaults. + +### ObjectVar + +A NetBox object. The list of available objects is defined by the queryset parameter. Each instance of this variable is limited to a single object type. + +* `queryset` - A [Django queryset](https://docs.djangoproject.com/en/stable/topics/db/queries/) + +### FileVar + +An uploaded file. Note that uploaded files are present in memory only for the duration of the script's execution: They will not be save for future use. + +### IPNetworkVar + +An IPv4 or IPv6 network with a mask. + +### Default Options + +All variables support the following default options: + +* `label` - The name of the form field +* `description` - A brief description of the field +* `default` - The field's default value +* `required` - Indicates whether the field is mandatory (default: true) + +## Example + +Below is an example script that creates new objects for a planned site. The user is prompted for three variables: + +* The name of the new site +* The device model (a filtered list of defined device types) +* The number of access switches to create + +These variables are presented as a web form to be completed by the user. Once submitted, the script's `run()` method is called to create the appropriate objects. + +``` +from django.utils.text import slugify + +from dcim.constants import * +from dcim.models import Device, DeviceRole, DeviceType, Site +from extras.scripts import * + + +class NewBranchScript(Script): + + class Meta: + name = "New Branch" + description = "Provision a new branch site" + fields = ['site_name', 'switch_count', 'switch_model'] + + site_name = StringVar( + description="Name of the new site" + ) + switch_count = IntegerVar( + description="Number of access switches to create" + ) + switch_model = ObjectVar( + description="Access switch model", + queryset = DeviceType.objects.filter( + manufacturer__name='Cisco', + model__in=['Catalyst 3560X-48T', 'Catalyst 3750X-48T'] + ) + ) + + def run(self, data): + + # Create the new site + site = Site( + name=data['site_name'], + slug=slugify(data['site_name']), + status=SITE_STATUS_PLANNED + ) + site.save() + self.log_success("Created new site: {}".format(site)) + + # Create access switches + switch_role = DeviceRole.objects.get(name='Access Switch') + for i in range(1, data['switch_count'] + 1): + switch = Device( + device_type=data['switch_model'], + name='{}-switch{}'.format(site.slug, i), + site=site, + status=DEVICE_STATUS_PLANNED, + device_role=switch_role + ) + switch.save() + self.log_success("Created new switch: {}".format(switch)) + + # Generate a CSV table of new devices + output = [ + 'name,make,model' + ] + for switch in Device.objects.filter(site=site): + attrs = [ + switch.name, + switch.device_type.manufacturer.name, + switch.device_type.model + ] + output.append(','.join(attrs)) + + return '\n'.join(output) +``` diff --git a/docs/configuration/optional-settings.md b/docs/configuration/optional-settings.md index 4ebb56290..b532c9757 100644 --- a/docs/configuration/optional-settings.md +++ b/docs/configuration/optional-settings.md @@ -277,6 +277,14 @@ The file path to the location where custom reports will be kept. By default, thi --- +## SCRIPTS_ROOT + +Default: $BASE_DIR/netbox/scripts/ + +The file path to the location where custom scripts will be kept. By default, this is the `netbox/scripts/` directory within the base NetBox installation path. + +--- + ## SESSION_FILE_PATH Default: None diff --git a/netbox/extras/forms.py b/netbox/extras/forms.py index eeb7921e1..d4cda76d8 100644 --- a/netbox/extras/forms.py +++ b/netbox/extras/forms.py @@ -384,3 +384,34 @@ class ObjectChangeFilterForm(BootstrapMixin, forms.Form): widget=ContentTypeSelect(), label='Object Type' ) + + +# +# Scripts +# + +class ScriptForm(BootstrapMixin, forms.Form): + _commit = forms.BooleanField( + required=False, + initial=True, + label="Commit changes", + help_text="Commit changes to the database (uncheck for a dry-run)" + ) + + def __init__(self, vars, *args, **kwargs): + + super().__init__(*args, **kwargs) + + # Dynamically populate fields for variables + for name, var in vars.items(): + self.fields[name] = var.as_field() + + # Move _commit to the end of the form + self.fields.move_to_end('_commit', True) + + @property + def requires_input(self): + """ + A boolean indicating whether the form requires user input (ignore the _commit field). + """ + return bool(len(self.fields) > 1) diff --git a/netbox/extras/middleware.py b/netbox/extras/middleware.py index b0b5a014d..79d543907 100644 --- a/netbox/extras/middleware.py +++ b/netbox/extras/middleware.py @@ -9,11 +9,12 @@ from django.utils import timezone from django.utils.functional import curry from django_prometheus.models import model_deletes, model_inserts, model_updates -from extras.webhooks import enqueue_webhooks from .constants import ( OBJECTCHANGE_ACTION_CREATE, OBJECTCHANGE_ACTION_DELETE, OBJECTCHANGE_ACTION_UPDATE, ) from .models import ObjectChange +from .signals import purge_changelog +from .webhooks import enqueue_webhooks _thread_locals = threading.local() @@ -30,6 +31,10 @@ def cache_changed_object(instance, **kwargs): def _record_object_deleted(request, instance, **kwargs): + # TODO: Can we cache deletions for later processing like we do for saves? Currently this will trigger an exception + # when trying to serialize ManyToMany relations after the object has been deleted. This should be doable if we alter + # log_change() to return ObjectChanges to be saved rather than saving them directly. + # Record that the object was deleted if hasattr(instance, 'log_change'): instance.log_change(request.user, request.id, OBJECTCHANGE_ACTION_DELETE) @@ -41,6 +46,13 @@ def _record_object_deleted(request, instance, **kwargs): model_deletes.labels(instance._meta.model_name).inc() +def purge_objectchange_cache(sender, **kwargs): + """ + Delete any queued object changes waiting to be written. + """ + _thread_locals.changed_objects = None + + class ObjectChangeMiddleware(object): """ This middleware performs three functions in response to an object being created, updated, or deleted: @@ -74,9 +86,21 @@ class ObjectChangeMiddleware(object): post_save.connect(cache_changed_object, dispatch_uid='record_object_saved') post_delete.connect(record_object_deleted, dispatch_uid='record_object_deleted') + # Provide a hook for purging the change cache + purge_changelog.connect(purge_objectchange_cache) + # Process the request response = self.get_response(request) + # If the change cache has been purged (e.g. due to an exception) abort the logging of all changes resulting from + # this request. + if _thread_locals.changed_objects is None: + + # Delete ObjectChanges representing deletions, since these have already been written + ObjectChange.objects.filter(request_id=request.id).delete() + + return response + # Create records for any cached objects that were created/updated. for obj, action in _thread_locals.changed_objects: diff --git a/netbox/extras/migrations/0024_scripts.py b/netbox/extras/migrations/0024_scripts.py new file mode 100644 index 000000000..82d0afdc9 --- /dev/null +++ b/netbox/extras/migrations/0024_scripts.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2 on 2019-08-12 15:28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('extras', '0023_fix_tag_sequences'), + ] + + operations = [ + migrations.CreateModel( + name='Script', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False)), + ], + options={ + 'permissions': (('run_script', 'Can run script'),), + 'managed': False, + }, + ), + ] diff --git a/netbox/extras/models.py b/netbox/extras/models.py index 02dd235b6..60723f699 100644 --- a/netbox/extras/models.py +++ b/netbox/extras/models.py @@ -826,6 +826,21 @@ class ConfigContextModel(models.Model): return data +# +# Custom scripts +# + +class Script(models.Model): + """ + Dummy model used to generate permissions for custom scripts. Does not exist in the database. + """ + class Meta: + managed = False + permissions = ( + ('run_script', 'Can run script'), + ) + + # # Report results # diff --git a/netbox/extras/scripts.py b/netbox/extras/scripts.py new file mode 100644 index 000000000..9462ee5bd --- /dev/null +++ b/netbox/extras/scripts.py @@ -0,0 +1,343 @@ +from collections import OrderedDict +import inspect +import json +import os +import pkgutil +import time +import traceback +import yaml + +from django import forms +from django.conf import settings +from django.core.validators import RegexValidator +from django.db import transaction +from mptt.forms import TreeNodeChoiceField +from mptt.models import MPTTModel + +from ipam.formfields import IPFormField +from utilities.exceptions import AbortTransaction +from .constants import LOG_DEFAULT, LOG_FAILURE, LOG_INFO, LOG_SUCCESS, LOG_WARNING +from .forms import ScriptForm +from .signals import purge_changelog + + +__all__ = [ + 'BaseScript', + 'BooleanVar', + 'FileVar', + 'IntegerVar', + 'IPNetworkVar', + 'ObjectVar', + 'Script', + 'StringVar', + 'TextVar', +] + + +# +# Script variables +# + +class ScriptVariable: + """ + Base model for script variables + """ + form_field = forms.CharField + + def __init__(self, label='', description='', default=None, required=True): + + # Default field attributes + self.field_attrs = { + 'help_text': description, + 'required': required + } + if label: + self.field_attrs['label'] = label + if default: + self.field_attrs['initial'] = default + + def as_field(self): + """ + Render the variable as a Django form field. + """ + form_field = self.form_field(**self.field_attrs) + form_field.widget.attrs['class'] = 'form-control' + + return form_field + + +class StringVar(ScriptVariable): + """ + Character string representation. Can enforce minimum/maximum length and/or regex validation. + """ + def __init__(self, min_length=None, max_length=None, regex=None, *args, **kwargs): + super().__init__(*args, **kwargs) + + # Optional minimum/maximum lengths + if min_length: + self.field_attrs['min_length'] = min_length + if max_length: + self.field_attrs['max_length'] = max_length + + # Optional regular expression validation + if regex: + self.field_attrs['validators'] = [ + RegexValidator( + regex=regex, + message='Invalid value. Must match regex: {}'.format(regex), + code='invalid' + ) + ] + + +class TextVar(ScriptVariable): + """ + Free-form text data. Renders as a