VIO-1216 netbox (#52)

* VIO-1216 Bump ci config to consume ci-shared

- This isn't actually consumign the python pipeline, however we are
  hijacking the build routine and consuming internal methods. This may
fail.

* adding continue field in extras migration model for custom_field_data

Co-authored-by: Jinal Shah <jinalshah1911@gmail.com>

.jenkins

@@ -1,134 +1,70 @@
-def getSafeBranchName() {
+#!/usr/bin/env groovy
-    return "${env.BRANCH_NAME}".replace('/', '-')
-}
 
-def getTagName() {
+// Include this shared CI repository to load script helpers and libraries.
-    def branchName = getSafeBranchName()
+library identifier: 'vapor@1.15.7', retriever: modernSCM([
-    return "${branchName}.${env.BUILD_NUMBER}"
+  $class: 'GitSCMSource',
-}
+  remote: 'https://github.com/vapor-ware/ci-shared.git',
+  credentialsId: 'vio-bot-gh',
+])
 
 
-pipeline {
+pythonPipeline([
-  // disallow unconfigured stages
+  'image': 'vaporio/netbox',
-  // each stage will have to declare an agent block and direct the pipeline
+  'pythonVersion': '3.7',
-  // how to execute. This skips the implicit agent scheduling.
+  'skipDocs': true,
-  agent none
+  'skipLint': true,
-  environment {
+  'skipIntegrationTest': true,
-    TAG = getTagName()
+  'skipPrivateRepo': true,
-    IMAGE = 'vaporio/netbox'
+  'podTemplate': """
-  }
-  stages {
-    stage('Test') {
-
-      /*
-      # Setup an agent dynamically using the following podspec. Netbox requires
-      # redis and postgres by default (they've disabled all the other backend drivers
-      # so we'll tack those on to the pods with some sane defaults.
-      # Note: this targets units on the vapor-build cluster (implicit) This may not be
-      # desireable in the case of building docker images.
-      */
-      agent {
-        kubernetes {
-          defaultContainer 'jnlp'
-          yaml """
 apiVersion: v1
 kind: Pod
-metadata:
-  labels:
-    jenkins/job: netbox
 spec:
+  imagePullSecrets:
+  - name: dockerhub-vaporvecrobot
   containers:
   - name: python
-    image: vaporio/jenkins-agent-python36:latest
+    image: vaporio/jenkins-agent-python37:master
+    imagePullPolicy: Always
     command:
     - cat
     tty: true
+    resources:
+      requests:
+        memory: 1Gi
+        cpu: 500m
+  - name: deploy
+    image: vaporio/deployment-tools:latest
+    imagePullPolicy: Always
+    command:
+    - cat
+    tty: true
+    resources:
+      requests:
+        memory: 512Mi
+        cpu: 250m
   - name: postgres
-    image: postgres:10
+    image: postgres:11
     env:
     - name: POSTGRES_USER
-      value: netbox
+      value: postgres
     - name: POSTGRES_PASSWORD
-      value: netbox
+      value: 12345
+    resources:
+      requests:
+        memory: 256Mi
+        cpu: 250m
   - name: redis
     image: redis:5
     command:
     - redis-server
     args:
     - --appendonly yes
-    - --requirepass netbox
+    - --requirepass 12345
-"""
+    resources:
-        }
+      requests:
-      }
+        memory: 128Mi
-      steps {
+        cpu: 250m
-        container('python') {
+  imagePullSecrets:
-          /*
+    - name: dockerhub-vaporvecrobot
-          # in the netbox/netbox path there is an example configuration file
+""".stripIndent(),
-          # clone this file and set up a permissive configuration for CI
+])
-          # using the values we declared in the podspec
-          */
-          dir('netbox/netbox') {
-            sh """
-              cp configuration.example.py configuration.py
-              sed -i -e "s/ALLOWED_HOSTS = .*/ALLOWED_HOSTS = ['*']/g" configuration.py
-              sed -i -e "s/SECRET_KEY = .*/SECRET_KEY = 'netboxci'/g" configuration.py
-              sed -i -e "s/USER': .*/USER': 'netbox',/g" configuration.py
-              sed -i -e "s/PASSWORD': .*/PASSWORD': 'netbox',/g" configuration.py
-              sed -i -e "s/PLUGINS = .*/PLUGINS = ['netbox_virtual_circuit_plugin']/g" configuration.py
-            """
-          }
-          // finally, kick off tox to run the entire test suite
-          sh 'tox -v'
-        }
-      }
-    }
-    stage('The Great British Baking Show') {
-      /* the docker-build agent is statically enlisted in jenkins. it runs
-      on the micro-k8s unit in vaporio/foundation:latest and has a uid1000
-      accessible docker */
-      when {
-        not {
-          changeRequest()
-        }
-      }
-      environment {
-        BASE_TAG = getSafeBranchName()
-      }
-      agent {
-        label 'docker-build'
-      }
-      steps {
-        container('docker') {
-          // embed tags from build env to do image tracing later
-          sh '''
-            docker build . \
-              -f Dockerfile \
-              --build-arg BUILD_DATE=$(date -u +%Y-%m-%dT%T 2> /dev/null) \
-              --build-arg VCS_REF=${GIT_COMMIT} \
-              --build-arg BUILD_VERSION=${BUILD_TAG} \
-              --build-arg BRANCH=${BRANCH_NAME} \
-              -t ${IMAGE}:${TAG}
-          '''
-          sh 'docker tag ${IMAGE}:${TAG} ${IMAGE}:${BASE_TAG}'
-          withDockerRegistry(registry: [credentialsId: 'vio-docker-hub']) {
-            sh "docker push ${env.IMAGE}:${env.TAG}"
-            sh "docker push ${env.IMAGE}:${env.BASE_TAG}"
-          }
-        }
-      }
-    }
-/* Muting this until Helm 3 chart is updated and tested
-    stage('Deployment'){
-      when {
-        branch 'develop'
-      }
-      steps {
-        build(
-          job: '/vapor-xyz/netbox-deploy-dev',
-          wait: true,
-          parameters: [string(name: 'IMAGE_TAG', value: "${env.TAG}")]
-        )
-      }
-    } */
-  }
-}

Makefile

@@ -0,0 +1,39 @@
+# This task clones the netbox configuration example, and populates for CI automated testing
+# Note: the openssl command will be required in the runtime to populate the SECRETKEY.
+# brew install openssl and/or apt-get install openssl
+setup:
+	rm -rfv netbox/netbox/configuration.py
+	cp netbox/netbox/configuration.example.py netbox/netbox/configuration.py
+	sed -i -e "s/ALLOWED_HOSTS = .*/ALLOWED_HOSTS = ['*']/g" netbox/netbox/configuration.py
+	sed -i -e "s/SECRET_KEY = .*/SECRET_KEY = '$(shell openssl rand -hex 32)'/g" netbox/netbox/configuration.py
+	sed -i -e "s/USER': .*/USER': 'postgres',/g" netbox/netbox/configuration.py
+	sed -i -e "s/PASSWORD': .*/PASSWORD': '12345',/g" netbox/netbox/configuration.py
+	sed -i -e "s/PLUGINS = .*/PLUGINS = ['netbox_virtual_circuit_plugin']/g" netbox/netbox/configuration.py
+
+# spin up the required stack components to run the test suite
+local-test-deps: reset-volumes
+	docker-compose up -d postgres
+	docker-compose up -d redis
+
+local-test: setup local-test-deps
+	tox -v
+
+# Invoked by CI - so ensure the supporting env has been created if
+# executing locally
+unit-test:
+	tox -v
+
+# Reset will kill any running containers and remove them
+reset:
+	docker-compose kill
+	docker-compose rm -f
+
+# Remove all local volumes to ensure a restart from scratch
+# docker-compose likes to "enable data caching" by not purging
+# volumes between resets.
+reset-volumes: reset
+	docker volume rm -f netbox_netbox-media-files
+	docker volume rm -f netbox_netbox-nginx-config
+	docker volume rm -f netbox_netbox-postgres-data
+	docker volume rm -f netbox_netbox-redis-data
+	docker volume rm -f netbox_netbox-static-files

docker-compose.yml

@@ -55,6 +55,10 @@ services:
     - netbox-postgres-data:/var/lib/postgresql/data
   redis:
     image: redis:5-alpine
+    command:
+    - redis-server
+    - --appendonly yes
+    - --requirepass 12345
     ports:
     - 6379:6379
     volumes:

netbox/extras/migrations/0051_migrate_customfields.py

@@ -65,6 +65,9 @@ def migrate_customfieldvalues(apps, schema_editor):
         # Read and update custom field value for each instance
         # TODO: This can be done more efficiently once .update() is supported for JSON fields
         cf_data = model.objects.filter(pk=cfv.obj_id).values('custom_field_data').first()
+        if cf_data is None:
+            print(f'{cfv.field.name} ({cfv.field.type}): {cfv.serialized_value} ({cfv.pk}): references non-existent {model.__name__} {cfv.obj_id}')
+            continue
         try:
             cf_data['custom_field_data'][cfv.field.name] = deserialize_value(cfv.field, cfv.serialized_value)
         except Exception as e: