From 0a62f75a401a5282b6a77140c7a6270088c320e1 Mon Sep 17 00:00:00 2001 From: Daniel Sheppard Date: Thu, 28 Oct 2021 15:14:42 -0500 Subject: [PATCH 1/5] #6529 - Add CLI to run scripts --- .../extras/management/commands/runscript.py | 160 ++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 netbox/extras/management/commands/runscript.py diff --git a/netbox/extras/management/commands/runscript.py b/netbox/extras/management/commands/runscript.py new file mode 100644 index 000000000..36da9d719 --- /dev/null +++ b/netbox/extras/management/commands/runscript.py @@ -0,0 +1,160 @@ +import json +import logging +import sys +import traceback +import uuid + +from django.contrib.auth.models import User +from django.contrib.contenttypes.models import ContentType +from django.core.management.base import BaseCommand +from django.db import transaction + +from extras.api.serializers import ScriptOutputSerializer +from extras.choices import JobResultStatusChoices +from extras.context_managers import change_logging +from extras.models import JobResult +from extras.scripts import get_scripts +from utilities.exceptions import AbortTransaction +from utilities.utils import NetBoxFakeRequest + + +class Command(BaseCommand): + help = "Run a script in Netbox" + + def add_arguments(self, parser): + parser.add_argument( + '--loglevel', + help="Logging Level (default: info)", + dest='loglevel', + default='info', + choices=['debug', 'info', 'warning', 'error']) + parser.add_argument('--script', help="Script to run", dest='script') + parser.add_argument('--user', help="Data as a json blob", dest='user') + parser.add_argument('data', help="Data as a json blob") + + @staticmethod + def _get_script(module, name): + scripts = get_scripts() + return scripts[module][name]() + + def handle(self, *args, **options): + def _run_script(): + """ + Core script execution task. We capture this within a subfunction to allow for conditionally wrapping it with + the change_logging context manager (which is bypassed if commit == False). + """ + try: + with transaction.atomic(): + script.output = script.run(data=data, commit=commit) + job_result.set_status(JobResultStatusChoices.STATUS_COMPLETED) + + if not commit: + raise AbortTransaction() + + except AbortTransaction: + script.log_info("Database changes have been reverted automatically.") + + except Exception as e: + stacktrace = traceback.format_exc() + script.log_failure( + f"An exception occurred: `{type(e).__name__}: {e}`\n```\n{stacktrace}\n```" + ) + script.log_info("Database changes have been reverted due to error.") + logger.error(f"Exception raised during script execution: {e}") + job_result.set_status(JobResultStatusChoices.STATUS_ERRORED) + + finally: + job_result.data = ScriptOutputSerializer(script).data + job_result.save() + + logger.info(f"Script completed in {job_result.duration}") + + # Params + script = options['script'] + loglevel = options['loglevel'] + data = json.loads(options['data']) + + module, name = script.split('.', 1) + + # Take user from command line if provided and exists, other + if options['user']: + try: + user = User.objects.get(username=options['user']) + except User.DoesNotExist: + user = User.objects.filter(is_superuser=True).order_by('pk')[0] + else: + user = User.objects.filter(is_superuser=True).order_by('pk')[0] + + # Setup logging to Stdout + formatter = logging.Formatter(f'[%(asctime)s][%(levelname)s] - %(message)s') + stdouthandler = logging.StreamHandler(sys.stdout) + stdouthandler.setLevel(logging.DEBUG) + stdouthandler.setFormatter(formatter) + + logger = logging.getLogger(f"netbox.scripts.{module}.{name}") + logger.addHandler(stdouthandler) + + if loglevel == 'debug': + logger.setLevel(logging.DEBUG) + elif loglevel == 'info': + logger.setLevel(logging.INFO) + elif loglevel == 'warning': + logger.setLevel(logging.WARNING) + elif loglevel == 'error': + logger.setLevel(logging.ERROR) + else: + logger.setLevel(logging.INFO) + + # Get the script + script = self._get_script(module, name) + # Parse the parameters + form = script.as_form(data, None) + + script_content_type = ContentType.objects.get(app_label='extras', model='script') + + # Create the job result + job_result = JobResult.objects.create( + name=script.full_name, + obj_type=script_content_type, + user=User.objects.filter(is_superuser=True).order_by('pk')[0], + job_id=uuid.uuid4() + ) + + request = NetBoxFakeRequest({ + 'META': {}, + 'POST': data, + 'GET': {}, + 'FILES': {}, + 'user': user, + 'path': '', + 'id': job_result.job_id + }) + + if form.is_valid(): + job_result.status = JobResultStatusChoices.STATUS_RUNNING + job_result.save() + + commit = form.cleaned_data.pop('_commit') + + logger.info(f"Running script (commit={commit})") + script.request = request + + # Execute the script. If commit is True, wrap it with the change_logging context manager to ensure we process + # change logging, webhooks, etc. + if commit: + with change_logging(request): + _run_script() + else: + _run_script() + + # Delete any previous terminal state results + JobResult.objects.filter( + obj_type=job_result.obj_type, + name=job_result.name, + status__in=JobResultStatusChoices.TERMINAL_STATE_CHOICES + ).exclude( + pk=job_result.pk + ).delete() + else: + job_result.status = JobResultStatusChoices.STATUS_ERRORED + job_result.save() From b7c0e8b71f42a8bf5477c429c78bce75d7ae1ca5 Mon Sep 17 00:00:00 2001 From: Daniel Sheppard Date: Tue, 2 Nov 2021 13:12:12 -0500 Subject: [PATCH 2/5] #6529 - Streamline code and resolve some issues --- .../extras/management/commands/runscript.py | 71 +++++++++---------- 1 file changed, 33 insertions(+), 38 deletions(-) diff --git a/netbox/extras/management/commands/runscript.py b/netbox/extras/management/commands/runscript.py index 36da9d719..8a6dbc989 100644 --- a/netbox/extras/management/commands/runscript.py +++ b/netbox/extras/management/commands/runscript.py @@ -6,14 +6,14 @@ import uuid from django.contrib.auth.models import User from django.contrib.contenttypes.models import ContentType -from django.core.management.base import BaseCommand +from django.core.management.base import BaseCommand, CommandError from django.db import transaction from extras.api.serializers import ScriptOutputSerializer from extras.choices import JobResultStatusChoices from extras.context_managers import change_logging from extras.models import JobResult -from extras.scripts import get_scripts +from extras.scripts import get_script from utilities.exceptions import AbortTransaction from utilities.utils import NetBoxFakeRequest @@ -27,15 +27,11 @@ class Command(BaseCommand): help="Logging Level (default: info)", dest='loglevel', default='info', - choices=['debug', 'info', 'warning', 'error']) - parser.add_argument('--script', help="Script to run", dest='script') - parser.add_argument('--user', help="Data as a json blob", dest='user') - parser.add_argument('data', help="Data as a json blob") - - @staticmethod - def _get_script(module, name): - scripts = get_scripts() - return scripts[module][name]() + choices=['debug', 'info', 'warning', 'error', 'critical']) + parser.add_argument('--script', help="Script to run", dest='script', required=True) + parser.add_argument('--commit', help="Commit this script to database", dest='commit') + parser.add_argument('--user', help="User script is running as", dest='user') + parser.add_argument('data', help="Data as a JSON blob") def handle(self, *args, **options): def _run_script(): @@ -72,7 +68,8 @@ class Command(BaseCommand): # Params script = options['script'] loglevel = options['loglevel'] - data = json.loads(options['data']) + data = json.loads(options['data']) if options['data'] is not None else None + commit = True if options['commit'] in ['1', 'true', 'True'] else False module, name = script.split('.', 1) @@ -94,24 +91,32 @@ class Command(BaseCommand): logger = logging.getLogger(f"netbox.scripts.{module}.{name}") logger.addHandler(stdouthandler) - if loglevel == 'debug': - logger.setLevel(logging.DEBUG) - elif loglevel == 'info': - logger.setLevel(logging.INFO) - elif loglevel == 'warning': - logger.setLevel(logging.WARNING) - elif loglevel == 'error': - logger.setLevel(logging.ERROR) - else: - logger.setLevel(logging.INFO) + try: + logger.setLevel({ + 'critical': logging.CRITICAL, + 'debug': logging.DEBUG, + 'error': logging.ERROR, + 'fatal': logging.FATAL, + 'info': logging.INFO, + 'warning': logging.WARNING, + }[loglevel]) + except KeyError: + raise CommandError(f"Invalid log level: {loglevel}") # Get the script - script = self._get_script(module, name) + script = get_script(module, name)() # Parse the parameters form = script.as_form(data, None) script_content_type = ContentType.objects.get(app_label='extras', model='script') + # Delete any previous terminal state results + JobResult.objects.filter( + obj_type=script_content_type, + name=script.full_name, + status__in=JobResultStatusChoices.TERMINAL_STATE_CHOICES + ).delete() + # Create the job result job_result = JobResult.objects.create( name=script.full_name, @@ -134,27 +139,17 @@ class Command(BaseCommand): job_result.status = JobResultStatusChoices.STATUS_RUNNING job_result.save() - commit = form.cleaned_data.pop('_commit') - logger.info(f"Running script (commit={commit})") script.request = request # Execute the script. If commit is True, wrap it with the change_logging context manager to ensure we process # change logging, webhooks, etc. - if commit: - with change_logging(request): - _run_script() - else: + with change_logging(request): _run_script() - - # Delete any previous terminal state results - JobResult.objects.filter( - obj_type=job_result.obj_type, - name=job_result.name, - status__in=JobResultStatusChoices.TERMINAL_STATE_CHOICES - ).exclude( - pk=job_result.pk - ).delete() else: + logger.error('Data is not valid:') + for field, errors in form.errors.get_json_data().items(): + for error in errors: + logger.error(f'\t{field}: {error.get("message")}') job_result.status = JobResultStatusChoices.STATUS_ERRORED job_result.save() From 7c3318df923558574b6fe5c244122287e5a6bcf7 Mon Sep 17 00:00:00 2001 From: Daniel Sheppard Date: Tue, 2 Nov 2021 15:56:42 -0500 Subject: [PATCH 3/5] #6529 - Adjusted the arguments. Added documentation --- docs/customization/custom-scripts.md | 14 ++++++++++++++ netbox/extras/management/commands/runscript.py | 10 +++++----- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/docs/customization/custom-scripts.md b/docs/customization/custom-scripts.md index cf052f918..ef74ed7ee 100644 --- a/docs/customization/custom-scripts.md +++ b/docs/customization/custom-scripts.md @@ -259,6 +259,20 @@ http://netbox/api/extras/scripts/example.MyReport/ \ --data '{"data": {"foo": "somevalue", "bar": 123}, "commit": true}' ``` +### Via the CLI + +Scripts can be run on the CLI by invoking the management command: + +``` +python3 manage.py runscript [--commit] [--loglevel {debug,info,warning,error,critical}] --script .