Cirrus: Use Makefile for image-building

The packer tool takes JSON as input for the details of producing VM images to be used for PR CI-testing. JSON is not a very human-friendly format, without support for comments and frequently containing lots of duplicate data. Fix this by using a Makefile + simple python one-liner to convert from a human-friendly YAML format into packer-native JSON. This allows use of anchors/aliases to reduce duplication, and allows inline comments for easier maintainability. This also allows separating the 'test' action from the 'build' action, for earlier and better syntax problem detection. Lastly, there are some minor ``lib.sh`` and ``integration_test.sh`` updates to support future work, and slightly improve the build and test environments. Signed-off-by: Chris Evich <cevich@redhat.com>
2018-11-07 09:12:15 -05:00 · 2018-11-07 09:12:15 -05:00 · 8d7fdc7d79
parent fb3ceeb385
commit 8d7fdc7d79
10 changed files with 174 additions and 155 deletions
--- a/.cirrus.yml
+++ b/.cirrus.yml
@ -113,13 +113,12 @@ build_vm_images_task:
    env:
        # CSV of packer builder names to enable (see $PACKER_BASE/libpod_images.json)
        PACKER_BUILDS: "ubuntu-18"
-        # TODO: Make these work (also full_vm_testing_task above)
-        # PACKER_BUILDS: "rhel-7,centos-7,fedora-28,ubuntu-18"
-        CENTOS_BASE_IMAGE: "centos-7-v20180911"
-        RHEL_BASE_IMAGE: "rhel-server-ec2-7-5-165-1"
-        FEDORA_BASE_IMAGE: "fedora-cloud-base-28-1-1-7"
+        # TODO: PACKER_BUILDS: "rhel-7,centos-7,fedora-29,fah-29,ubuntu-18"
        UBUNTU_BASE_IMAGE: "ubuntu-1804-bionic-v20180911"
-        # low-level base VM image name inputs to packer
+        CENTOS_BASE_IMAGE: "centos-7-v20180911"
+        RHEL_BASE_IMAGE: "rhel-server-ec2-7-5-165-1"  # Manually imported into GCE
+        FEDORA_BASE_IMAGE:  "fedora-cloud-base-29-1-2-1541186745"  # see $PACKER_BASE/Makefile
+        FAH_BASE_IMAGE:  "fedora-atomichost-29-20181025-1"  # See $PACKER_BASE/Makefile

        # Command to register a RHEL VM
        RHSM_COMMAND: ENCRYPTED[5caa5ff8c5370c3d25c7a1a28168501ab0fa2e5e3b627926f6eaba02b3fed965a7638a6151657809661f8c905c7dc187]
--- a/1
+++ b/1
@ -172,6 +172,7 @@ testunit: libpodimage

 localunit: test/goecho/goecho varlink_generate
 	$(GO) test -tags "$(BUILDTAGS)" -cover $(PACKAGES)
+	$(MAKE) -C contrib/cirrus/packer test

 ginkgo:
 	ginkgo -v -tags "$(BUILDTAGS)" -cover -flakeAttempts 3 -progress -trace -noColor test/e2e/.
--- a/contrib/cirrus/build_vm_images.sh
+++ b/contrib/cirrus/build_vm_images.sh
@ -11,9 +11,9 @@ PACKER_BUILDS $PACKER_BUILDS
 CENTOS_BASE_IMAGE $CENTOS_BASE_IMAGE
 UBUNTU_BASE_IMAGE $UBUNTU_BASE_IMAGE
 FEDORA_BASE_IMAGE $FEDORA_BASE_IMAGE
+FAH_BASE_IMAGE $FAH_BASE_IMAGE
 RHEL_BASE_IMAGE $RHEL_BASE_IMAGE
 RHSM_COMMAND $RHSM_COMMAND
-BUILT_IMAGE_SUFFIX $BUILT_IMAGE_SUFFIX
 SERVICE_ACCOUNT $SERVICE_ACCOUNT
 GCE_SSH_USERNAME $GCE_SSH_USERNAME
 GCP_PROJECT_ID $GCP_PROJECT_ID
@ -43,13 +43,10 @@ fi

 set -x

-cd "$GOSRC"
-# N/B: /usr/sbin/packer is a DIFFERENT tool, and will exit 0 given the args below :(
-TEMPLATE="./$PACKER_BASE/libpod_images.json"
-
-$HOME/packer/packer inspect "$TEMPLATE"
-
-#$HOME/packer/packer build -machine-readable "-only=$PACKER_BUILDS" "$TEMPLATE" | tee /tmp/packer_log.csv
-$HOME/packer/packer build "-only=$PACKER_BUILDS" "$TEMPLATE"
-
-# TODO: Report back to PR names of built images
+cd "$GOSRC/$PACKER_BASE"
+make libpod_images \
+    PACKER_BUILDS=$PACKER_BUILDS \
+    PACKER_VER=$PACKER_VER \
+    GOSRC=$GOSRC \
+    SCRIPT_BASE=$SCRIPT_BASE \
+    PACKER_BASE=$PACKER_BASE
--- a/contrib/cirrus/integration_test.sh
+++ b/contrib/cirrus/integration_test.sh
@ -22,7 +22,9 @@ case "${OS_RELEASE_ID}-${OS_RELEASE_VER}" in
    fedora-28) ;&  # Continue to the next item
    centos-7) ;&
    rhel-7)
-        stub 'integration testing not working on $OS_RELEASE_ID'
+        make install PREFIX=/usr ETCDIR=/etc
+        make test-binaries
+        make localintegration
        ;;
    *) bad_os_id_ver ;;
 esac
--- a/contrib/cirrus/lib.sh
+++ b/contrib/cirrus/lib.sh
@ -4,8 +4,8 @@
 # to be sourced by other scripts, not called directly.

 # Under some contexts these values are not set, make sure they are.
-USER="$(whoami)"
-HOME="$(getent passwd $USER | cut -d : -f 6)"
+export USER="$(whoami)"
+export HOME="$(getent passwd $USER | cut -d : -f 6)"
 if ! [[ "$PATH" =~ "/usr/local/bin" ]]
 then
    export PATH="$PATH:/usr/local/bin"
@ -274,6 +274,10 @@ _finalize(){
    sudo rm -rf /var/lib/cloud/instance?
    sudo rm -rf /root/.ssh/*
    sudo rm -rf /home/*
+    sudo rm -rf /tmp/*
+    sudo rm -rf /tmp/.??*
+    sync
+    sudo fstrim -av
 }

 rh_finalize(){
--- a/contrib/cirrus/packer/.gitignore
+++ b/contrib/cirrus/packer/.gitignore
@ -0,0 +1,3 @@
+*json
+packer
+packer*zip
--- a/contrib/cirrus/packer/Makefile
+++ b/contrib/cirrus/packer/Makefile
@ -0,0 +1,56 @@
+
+# N/B: PACKER_BUILDS variable is required.  Should contain CSV of
+# builder name(s) from applicable YAML file,
+# e.g for names see libpod_images.yml
+
+PACKER_VER ?= 1.3.1
+PACKER_DIST_FILENAME := packer_${PACKER_VER}_linux_amd64.zip
+
+# Only needed for libpod_base_images target
+TIMESTAMP := $(shell date +%s)
+GOSRC ?= $(shell realpath "./../../../")
+PACKER_BASE ?= contrib/cirrus/packer
+SCRIPT_BASE ?= contrib/cirrus
+
+# For debugging nested-virt, use
+#TTYDEV := $(shell tty)
+TTYDEV := /dev/null
+
+.PHONY: all
+all: libpod_images
+
+%.json: %.yml
+	@python3 -c 'import json,yaml; json.dump( yaml.load(open("$<").read()), open("$@","w"), indent=2);'
+
+${PACKER_DIST_FILENAME}:
+	@curl -L --silent --show-error \
+		-O https://releases.hashicorp.com/packer/${PACKER_VER}/${PACKER_DIST_FILENAME}
+
+packer: ${PACKER_DIST_FILENAME}
+	@curl -L --silent --show-error \
+		https://releases.hashicorp.com/packer/${PACKER_VER}/packer_${PACKER_VER}_SHA256SUMS \
+		| grep 'linux_amd64' > /tmp/packer_sha256sums
+	@sha256sum --check /tmp/packer_sha256sums
+	@unzip -o ${PACKER_DIST_FILENAME}
+	@touch --reference=Makefile ${PACKER_DIST_FILENAME}
+
+.PHONY: test
+test: libpod_base_images.json libpod_images.json packer
+	./packer inspect libpod_base_images.json > /dev/null
+	./packer inspect libpod_images.json > /dev/null
+	@echo "All good"
+
+.PHONY: libpod_images
+libpod_images: libpod_images.json packer
+ifndef PACKER_BUILDS
+	$(error PACKER_BUILDS is undefined, expected builder-names CSV)
+endif
+	./packer build -only=${PACKER_BUILDS} \
+		-var GOSRC=$(GOSRC) \
+		-var PACKER_BASE=$(PACKER_BASE) \
+		-var SCRIPT_BASE=$(SCRIPT_BASE) \
+		libpod_images.json
+	@echo ""
+	@echo "Finished.  The images mentioned above, and in packer-manifest.json"
+	@echo "can be used in .cirrus.yml as values for the 'image_name' keys"
+	@echo ""
--- a/contrib/cirrus/packer/libpod_images.json
+++ b/contrib/cirrus/packer/libpod_images.json
@ -1,130 +0,0 @@
-{
-  "variables": {
-    "FEDORA_CNI_COMMIT": "{{env `FEDORA_CNI_COMMIT`}}",
-    "CNI_COMMIT": "{{env `CNI_COMMIT`}}",
-    "CRIO_COMMIT": "{{env `CRIO_COMMIT`}}",
-    "CRIU_COMMIT": "{{env `CRIU_COMMIT`}}",
-    "RUNC_COMMIT": "{{env `RUNC_COMMIT`}}",
-
-    "CENTOS_BASE_IMAGE": "{{env `CENTOS_BASE_IMAGE`}}" ,
-    "UBUNTU_BASE_IMAGE": "{{env `UBUNTU_BASE_IMAGE`}}",
-    "FEDORA_BASE_IMAGE": "{{env `FEDORA_BASE_IMAGE`}}",
-    "RHEL_BASE_IMAGE": "{{env `RHEL_BASE_IMAGE`}}",
-
-    "GOSRC": "{{env `GOSRC`}}",
-    "PACKER_BASE": "{{env `PACKER_BASE`}}",
-    "SCRIPT_BASE": "{{env `SCRIPT_BASE`}}",
-
-    "SERVICE_ACCOUNT": "{{env `SERVICE_ACCOUNT`}}",
-    "GCP_PROJECT_ID": "{{env `GCP_PROJECT_ID`}}",
-    "BUILT_IMAGE_SUFFIX": "{{env `BUILT_IMAGE_SUFFIX`}}",
-    "GCE_SSH_USERNAME": "{{env `GCE_SSH_USERNAME`}}",
-    "RHSM_COMMAND": "{{env `RHSM_COMMAND`}}"
-  },
-  "sensitive-variables": [
-    "GCP_PROJECT_ID", "SERVICE_ACCOUNT", "GCE_SSH_USERNAME", "RHSM_COMMAND"
-  ],
-  "builders": [
-    {
-      "name": "rhel-7",
-      "type": "googlecompute",
-      "project_id": "{{user `GCP_PROJECT_ID`}}",
-      "zone": "us-central1-a",
-      "source_image": "{{user `RHEL_BASE_IMAGE`}}",
-      "image_name": "{{user `RHEL_BASE_IMAGE`}}{{user `BUILT_IMAGE_SUFFIX`}}",
-      "image_family": "{{user `RHEL_BASE_IMAGE`}}-libpod",
-      "service_account_email": "{{user `SERVICE_ACCOUNT`}}",
-      "communicator": "ssh",
-      "ssh_username": "ec2-user",
-      "ssh_pty": "true"
-    },{
-      "name": "centos-7",
-      "type": "googlecompute",
-      "project_id": "{{user `GCP_PROJECT_ID`}}",
-      "zone": "us-central1-a",
-      "source_image": "{{user `CENTOS_BASE_IMAGE`}}",
-      "image_name": "{{user `CENTOS_BASE_IMAGE`}}{{user `BUILT_IMAGE_SUFFIX`}}",
-      "image_family": "{{user `CENTOS_BASE_IMAGE`}}-libpod",
-      "service_account_email": "{{user `SERVICE_ACCOUNT`}}",
-      "communicator": "ssh",
-      "ssh_username": "{{user `GCE_SSH_USERNAME`}}",
-      "ssh_pty": "true"
-    },{
-      "name": "fedora-28",
-      "type": "googlecompute",
-      "project_id": "{{user `GCP_PROJECT_ID`}}",
-      "zone": "us-central1-a",
-      "source_image": "{{user `FEDORA_BASE_IMAGE`}}",
-      "image_name": "{{user `FEDORA_BASE_IMAGE`}}{{user `BUILT_IMAGE_SUFFIX`}}",
-      "image_family": "{{user `FEDORA_BASE_IMAGE`}}-libpod",
-      "service_account_email": "{{user `SERVICE_ACCOUNT`}}",
-      "communicator": "ssh",
-      "ssh_username": "fedora",
-      "ssh_pty": "true"
-    },{
-      "name": "ubuntu-18",
-      "type": "googlecompute",
-      "project_id": "{{user `GCP_PROJECT_ID`}}",
-      "zone": "us-central1-a",
-      "source_image": "{{user `UBUNTU_BASE_IMAGE`}}",
-      "image_name": "{{user `UBUNTU_BASE_IMAGE`}}{{user `BUILT_IMAGE_SUFFIX`}}",
-      "image_family": "{{user `UBUNTU_BASE_IMAGE`}}-libpod",
-      "service_account_email": "{{user `SERVICE_ACCOUNT`}}",
-      "communicator": "ssh",
-      "ssh_username": "{{user `GCE_SSH_USERNAME`}}",
-      "ssh_pty": "true"
-    }
-  ],
-  "provisioners": [
-    {
-      "type": "file",
-      "source": "{{user `GOSRC`}}",
-      "destination": "/tmp/libpod"
-    },{
-      "type": "shell",
-      "only": ["rhel-7"],
-      "script": "{{user `GOSRC`}}/{{user `PACKER_BASE`}}/rhel_setup.sh",
-      "environment_vars": [
-        "SCRIPT_BASE={{user `SCRIPT_BASE`}}",
-        "CNI_COMMIT={{user `CNI_COMMIT`}}",
-        "CRIO_COMMIT={{user `CRIO_COMMIT`}}",
-        "CRIU_COMMIT={{user `CRIU_COMMIT`}}",
-        "RUNC_COMMIT={{user `RUNC_COMMIT`}}",
-        "RHSM_COMMAND={{user `RHSM_COMMAND`}}"
-      ]
-    },{
-      "type": "shell",
-      "only": ["centos-7"],
-      "script": "{{user `GOSRC`}}/{{user `PACKER_BASE`}}/centos_setup.sh",
-      "environment_vars": [
-        "SCRIPT_BASE={{user `SCRIPT_BASE`}}",
-        "CNI_COMMIT={{user `CNI_COMMIT`}}",
-        "CRIO_COMMIT={{user `CRIO_COMMIT`}}",
-        "CRIU_COMMIT={{user `CRIU_COMMIT`}}",
-        "RUNC_COMMIT={{user `RUNC_COMMIT`}}"
-      ]
-    },{
-      "type": "shell",
-      "only": ["fedora-28"],
-      "script": "{{user `GOSRC`}}/{{user `PACKER_BASE`}}/fedora_setup.sh",
-      "environment_vars": [
-        "SCRIPT_BASE={{user `SCRIPT_BASE`}}",
-        "CNI_COMMIT={{user `FEDORA_CNI_COMMIT`}}",
-        "CRIO_COMMIT={{user `CRIO_COMMIT`}}",
-        "CRIU_COMMIT={{user `CRIU_COMMIT`}}",
-        "RUNC_COMMIT={{user `RUNC_COMMIT`}}"
-      ]
-    },{
-      "type": "shell",
-      "only": ["ubuntu-18"],
-      "script": "{{user `GOSRC`}}/{{user `PACKER_BASE`}}/ubuntu_setup.sh",
-      "environment_vars": [
-        "SCRIPT_BASE={{user `SCRIPT_BASE`}}",
-        "CNI_COMMIT={{user `CNI_COMMIT`}}",
-        "CRIO_COMMIT={{user `CRIO_COMMIT`}}",
-        "CRIU_COMMIT={{user `CRIU_COMMIT`}}",
-        "RUNC_COMMIT={{user `RUNC_COMMIT`}}"
-      ]
-    }
-  ]
-}
--- a/contrib/cirrus/packer/libpod_images.yml
+++ b/contrib/cirrus/packer/libpod_images.yml
@ -0,0 +1,89 @@
+---
+
+# All of these are required
+variables:
+    # Names of GCE Base images to start from, in .cirrus.yml
+    RHEL_BASE_IMAGE: '{{env `RHEL_BASE_IMAGE`}}'
+    CENTOS_BASE_IMAGE: '{{env `CENTOS_BASE_IMAGE`}}'
+    UBUNTU_BASE_IMAGE: '{{env `UBUNTU_BASE_IMAGE`}}'
+    FEDORA_BASE_IMAGE: '{{env `FEDORA_BASE_IMAGE`}}'
+    FAH_BASE_IMAGE: '{{env `FAH_BASE_IMAGE`}}'
+
+    # libpod dependencies to build and install into images
+    FEDORA_CNI_COMMIT: "{{env `FEDORA_CNI_COMMIT`}}"
+    CNI_COMMIT: "{{env `CNI_COMMIT`}}"
+    CRIO_COMMIT: "{{env `CRIO_COMMIT`}}"
+    CRIU_COMMIT: "{{env `CRIU_COMMIT`}}"
+    RUNC_COMMIT: "{{env `RUNC_COMMIT`}}"
+
+    CIRRUS_BUILD_ID: '{{env `CIRRUS_BUILD_ID`}}'
+    GOSRC: '{{env `GOSRC`}}'
+    PACKER_BASE: '{{env `PACKER_BASE`}}'
+    SCRIPT_BASE: '{{env `SCRIPT_BASE`}}'
+
+    # Protected credentials, decrypted by Cirrus at runtime
+    GCE_SSH_USERNAME: '{{env `GCE_SSH_USERNAME`}}'
+    GCP_PROJECT_ID: '{{env `GCP_PROJECT_ID`}}'
+    RHSM_COMMAND: '{{env `RHSM_COMMAND`}}'
+    SERVICE_ACCOUNT: '{{env `SERVICE_ACCOUNT`}}'
+    GOOGLE_APPLICATION_CREDENTIALS: '{{env `GOOGLE_APPLICATION_CREDENTIALS`}}'
+
+# Don't leak sensitive values in error messages / output
+sensitive-variables:
+    - 'GCE_SSH_USERNAME'
+    - 'GCP_PROJECT_ID'
+    - 'RHSM_COMMAND'
+    - 'SERVICE_ACCOUNT'
+
+# What images to produce in which cloud
+builders:
+    #  v----- is a YAML anchor, allows referencing this object by name (below)
+    - &gce_hosted_image
+      name: 'ubuntu-18'
+      type: 'googlecompute'
+      image_name: '{{user `UBUNTU_BASE_IMAGE`}}-libpod-{{user `CIRRUS_BUILD_ID`}}'
+      image_family: '{{user `UBUNTU_BASE_IMAGE`}}-libpod'
+      source_image: '{{user `UBUNTU_BASE_IMAGE`}}'
+      project_id: '{{user `GCP_PROJECT_ID`}}'
+      service_account_email: '{{user `SERVICE_ACCOUNT`}}'
+      communicator: 'ssh'
+      ssh_username: '{{user `GCE_SSH_USERNAME`}}'
+      ssh_pty: 'true'
+      # The only supported zone in Cirrus-CI, as of addition of this comment
+      zone: 'us-central1-a'
+
+    #      v----- is a YAML alias, allows partial re-use of the anchor object
+    - <<: *gce_hosted_image
+      name: 'rhel-7'
+      image_name: '{{user `RHEL_BASE_IMAGE`}}-libpod-{{user `CIRRUS_BUILD_ID`}}'
+      image_family: '{{user `RHEL_BASE_IMAGE`}}-libpod'
+      source_image: '{{user `RHEL_BASE_IMAGE`}}'
+      ssh_username: 'ec2-user'
+
+    - <<: *gce_hosted_image
+      name: 'centos-7'
+      image_family: '{{user `CENTOS_BASE_IMAGE`}}-libpod'
+      image_name: '{{user `CENTOS_BASE_IMAGE`}}-libpod-{{user `CIRRUS_BUILD_ID`}}'
+      source_image: '{{user `CENTOS_BASE_IMAGE`}}'
+
+
+# The brains of the operation, making actual modifications to the base-image.
+provisioners:
+    - type: 'file'
+      source: '{{user `GOSRC`}}'
+      destination: '/tmp/libpod'
+
+    - type: 'shell'
+      script: '{{user `GOSRC`}}/{{user `PACKER_BASE`}}/{{split build_name "-" 0}}_setup.sh'
+      environment_vars:
+          - 'SCRIPT_BASE={{user `SCRIPT_BASE`}}'
+          - 'CNI_COMMIT={{user `CNI_COMMIT`}}'
+          - 'FEDORA_CNI_COMMIT={{user `FEDORA_CNI_COMMIT`}}'
+          - 'CRIO_COMMIT={{user `CRIO_COMMIT`}}'
+          - 'CRIU_COMMIT={{user `CRIU_COMMIT`}}'
+          - 'RUNC_COMMIT={{user `RUNC_COMMIT`}}'
+          - 'RHSM_COMMAND={{user `RHSM_COMMAND`}}'
+
+post-processors:
+    - - type: 'manifest'
+        output: '/tmp/packer-manifest.json'
--- a/contrib/cirrus/unit_test.sh
+++ b/contrib/cirrus/unit_test.sh
@ -18,13 +18,11 @@ case "${OS_RELEASE_ID}-${OS_RELEASE_VER}" in
        make localunit "BUILDTAGS=$BUILDTAGS"
        make "BUILDTAGS=$BUILDTAGS"
        ;;
-    fedora-28)
+    fedora-28) ;&  # Continue to the next item
+    centos-7) ;&
+    rhel-7)
        make localunit
        make
        ;;
-    centos-7) ;&  # Continue to the next item
-    rhel-7)
-        stub 'unit testing not working on $OS_RELEASE_ID'
-        ;;
    *) bad_os_id_ver ;;
 esac