From 5ec7badd3323d3d7b01ffb341b69dc1189a9fc93 Mon Sep 17 00:00:00 2001 From: "Charles \"Chuck\" Butler" Date: Fri, 23 Jul 2021 08:59:25 -0500 Subject: [PATCH] VIO-1216 netbox (#52) * VIO-1216 Bump ci config to consume ci-shared - This isn't actually consumign the python pipeline, however we are hijacking the build routine and consuming internal methods. This may fail. * adding continue field in extras migration model for custom_field_data Co-authored-by: Jinal Shah --- .jenkins | 162 ++++++------------ Makefile | 39 +++++ docker-compose.yml | 4 + .../migrations/0051_migrate_customfields.py | 3 + 4 files changed, 95 insertions(+), 113 deletions(-) create mode 100644 Makefile diff --git a/.jenkins b/.jenkins index ff8ec98af..f74530e75 100644 --- a/.jenkins +++ b/.jenkins @@ -1,134 +1,70 @@ -def getSafeBranchName() { - return "${env.BRANCH_NAME}".replace('/', '-') -} +#!/usr/bin/env groovy -def getTagName() { - def branchName = getSafeBranchName() - return "${branchName}.${env.BUILD_NUMBER}" -} +// Include this shared CI repository to load script helpers and libraries. +library identifier: 'vapor@1.15.7', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/vapor-ware/ci-shared.git', + credentialsId: 'vio-bot-gh', +]) -pipeline { - // disallow unconfigured stages - // each stage will have to declare an agent block and direct the pipeline - // how to execute. This skips the implicit agent scheduling. - agent none - environment { - TAG = getTagName() - IMAGE = 'vaporio/netbox' - } - stages { - stage('Test') { - - /* - # Setup an agent dynamically using the following podspec. Netbox requires - # redis and postgres by default (they've disabled all the other backend drivers - # so we'll tack those on to the pods with some sane defaults. - # Note: this targets units on the vapor-build cluster (implicit) This may not be - # desireable in the case of building docker images. - */ - agent { - kubernetes { - defaultContainer 'jnlp' - yaml """ +pythonPipeline([ + 'image': 'vaporio/netbox', + 'pythonVersion': '3.7', + 'skipDocs': true, + 'skipLint': true, + 'skipIntegrationTest': true, + 'skipPrivateRepo': true, + 'podTemplate': """ apiVersion: v1 kind: Pod -metadata: - labels: - jenkins/job: netbox spec: + imagePullSecrets: + - name: dockerhub-vaporvecrobot containers: - name: python - image: vaporio/jenkins-agent-python36:latest + image: vaporio/jenkins-agent-python37:master + imagePullPolicy: Always command: - cat tty: true + resources: + requests: + memory: 1Gi + cpu: 500m + - name: deploy + image: vaporio/deployment-tools:latest + imagePullPolicy: Always + command: + - cat + tty: true + resources: + requests: + memory: 512Mi + cpu: 250m - name: postgres - image: postgres:10 + image: postgres:11 env: - name: POSTGRES_USER - value: netbox + value: postgres - name: POSTGRES_PASSWORD - value: netbox + value: 12345 + resources: + requests: + memory: 256Mi + cpu: 250m - name: redis image: redis:5 command: - redis-server args: - --appendonly yes - - --requirepass netbox -""" - } - } - steps { - container('python') { - /* - # in the netbox/netbox path there is an example configuration file - # clone this file and set up a permissive configuration for CI - # using the values we declared in the podspec - */ - dir('netbox/netbox') { - sh """ - cp configuration.example.py configuration.py - sed -i -e "s/ALLOWED_HOSTS = .*/ALLOWED_HOSTS = ['*']/g" configuration.py - sed -i -e "s/SECRET_KEY = .*/SECRET_KEY = 'netboxci'/g" configuration.py - sed -i -e "s/USER': .*/USER': 'netbox',/g" configuration.py - sed -i -e "s/PASSWORD': .*/PASSWORD': 'netbox',/g" configuration.py - sed -i -e "s/PLUGINS = .*/PLUGINS = ['netbox_virtual_circuit_plugin']/g" configuration.py - """ - } - // finally, kick off tox to run the entire test suite - sh 'tox -v' - } - } - } - stage('The Great British Baking Show') { - /* the docker-build agent is statically enlisted in jenkins. it runs - on the micro-k8s unit in vaporio/foundation:latest and has a uid1000 - accessible docker */ - when { - not { - changeRequest() - } - } - environment { - BASE_TAG = getSafeBranchName() - } - agent { - label 'docker-build' - } - steps { - container('docker') { - // embed tags from build env to do image tracing later - sh ''' - docker build . \ - -f Dockerfile \ - --build-arg BUILD_DATE=$(date -u +%Y-%m-%dT%T 2> /dev/null) \ - --build-arg VCS_REF=${GIT_COMMIT} \ - --build-arg BUILD_VERSION=${BUILD_TAG} \ - --build-arg BRANCH=${BRANCH_NAME} \ - -t ${IMAGE}:${TAG} - ''' - sh 'docker tag ${IMAGE}:${TAG} ${IMAGE}:${BASE_TAG}' - withDockerRegistry(registry: [credentialsId: 'vio-docker-hub']) { - sh "docker push ${env.IMAGE}:${env.TAG}" - sh "docker push ${env.IMAGE}:${env.BASE_TAG}" - } - } - } - } -/* Muting this until Helm 3 chart is updated and tested - stage('Deployment'){ - when { - branch 'develop' - } - steps { - build( - job: '/vapor-xyz/netbox-deploy-dev', - wait: true, - parameters: [string(name: 'IMAGE_TAG', value: "${env.TAG}")] - ) - } - } */ - } -} + - --requirepass 12345 + resources: + requests: + memory: 128Mi + cpu: 250m + imagePullSecrets: + - name: dockerhub-vaporvecrobot +""".stripIndent(), +]) diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..d6226e054 --- /dev/null +++ b/Makefile @@ -0,0 +1,39 @@ +# This task clones the netbox configuration example, and populates for CI automated testing +# Note: the openssl command will be required in the runtime to populate the SECRETKEY. +# brew install openssl and/or apt-get install openssl +setup: + rm -rfv netbox/netbox/configuration.py + cp netbox/netbox/configuration.example.py netbox/netbox/configuration.py + sed -i -e "s/ALLOWED_HOSTS = .*/ALLOWED_HOSTS = ['*']/g" netbox/netbox/configuration.py + sed -i -e "s/SECRET_KEY = .*/SECRET_KEY = '$(shell openssl rand -hex 32)'/g" netbox/netbox/configuration.py + sed -i -e "s/USER': .*/USER': 'postgres',/g" netbox/netbox/configuration.py + sed -i -e "s/PASSWORD': .*/PASSWORD': '12345',/g" netbox/netbox/configuration.py + sed -i -e "s/PLUGINS = .*/PLUGINS = ['netbox_virtual_circuit_plugin']/g" netbox/netbox/configuration.py + +# spin up the required stack components to run the test suite +local-test-deps: reset-volumes + docker-compose up -d postgres + docker-compose up -d redis + +local-test: setup local-test-deps + tox -v + +# Invoked by CI - so ensure the supporting env has been created if +# executing locally +unit-test: + tox -v + +# Reset will kill any running containers and remove them +reset: + docker-compose kill + docker-compose rm -f + +# Remove all local volumes to ensure a restart from scratch +# docker-compose likes to "enable data caching" by not purging +# volumes between resets. +reset-volumes: reset + docker volume rm -f netbox_netbox-media-files + docker volume rm -f netbox_netbox-nginx-config + docker volume rm -f netbox_netbox-postgres-data + docker volume rm -f netbox_netbox-redis-data + docker volume rm -f netbox_netbox-static-files diff --git a/docker-compose.yml b/docker-compose.yml index b6a9c5611..3d5cff207 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -55,6 +55,10 @@ services: - netbox-postgres-data:/var/lib/postgresql/data redis: image: redis:5-alpine + command: + - redis-server + - --appendonly yes + - --requirepass 12345 ports: - 6379:6379 volumes: diff --git a/netbox/extras/migrations/0051_migrate_customfields.py b/netbox/extras/migrations/0051_migrate_customfields.py index 41b2febe7..0ea292623 100644 --- a/netbox/extras/migrations/0051_migrate_customfields.py +++ b/netbox/extras/migrations/0051_migrate_customfields.py @@ -65,6 +65,9 @@ def migrate_customfieldvalues(apps, schema_editor): # Read and update custom field value for each instance # TODO: This can be done more efficiently once .update() is supported for JSON fields cf_data = model.objects.filter(pk=cfv.obj_id).values('custom_field_data').first() + if cf_data is None: + print(f'{cfv.field.name} ({cfv.field.type}): {cfv.serialized_value} ({cfv.pk}): references non-existent {model.__name__} {cfv.obj_id}') + continue try: cf_data['custom_field_data'][cfv.field.name] = deserialize_value(cfv.field, cfv.serialized_value) except Exception as e: