Merge pull request #1788 from cevich/cirrus_base_images

Codify and document base-image production + Enable testing with Fedora

.cirrus.yml

@@ -5,48 +5,118 @@
 # and storage.
 gcp_credentials: ENCRYPTED[885c6e4297dd8d6f67593c42b810353af0c505a7a670e2c6fd830c56e86bbb2debcc3c18f942d0d46ab36b63521061d4]
 
-# Default VM to use for testing, unless values overriden by specific tasks (below)
+# Default timeout for each task
-gce_instance:
+timeout_in: 120m
-    image_project: "libpod-218412"
-    zone: "us-central1-a"  # Required by Cirrus for the time being
-    cpu: 2
-    memory: "4Gb"
-    disk: 40
 
-# Main collection of env. varss to set for all scripts. All others
+# Main collection of env. vars to set for all tasks and scripts.
-# are cooked in by $SCRIPT_BASE/setup_environment.sh
 env:
-    FEDORA_CNI_COMMIT: "412b6d31280682bb4fab4446f113c22ff1886554"
+    ####
-    CNI_COMMIT: "7480240de9749f9a0a5c8614b17f1f03e0c06ab9"
+    #### Global variables used for all tasks
-    CRIO_COMMIT: "7a283c391abb7bd25086a8ff91dbb36ebdd24466"
+    ####
-    CRIU_COMMIT: "c74b83cd49c00589c0c0468ba5fe685b67fdbd0a"
-    RUNC_COMMIT: "96ec2177ae841256168fcf76954f7177af9446eb"
     # File to update in home-dir with task-specific env. var values
     ENVLIB: ".bash_profile"
     # Overrides default location (/tmp/cirrus) for repo clone
-    CIRRUS_WORKING_DIR: "/go/src/github.com/containers/libpod"
+    CIRRUS_WORKING_DIR: "/var/tmp/go/src/github.com/containers/libpod"
     # Required so $ENVLIB gets loaded
     CIRRUS_SHELL: "/bin/bash"
     # Save a little typing (path relative to $CIRRUS_WORKING_DIR)
     SCRIPT_BASE: "./contrib/cirrus"
     PACKER_BASE: "./contrib/cirrus/packer"
-    IRCID: ENCRYPTED[e87bba62a8e924dc70bdb2b66b16f6ab4a60d2870e6e5534ae9e2b0076f483c71c84091c655ca239101e6816c5ec0883]
 
-# Every *_task runs in parallel in separate VMs. The name prefix only for reference
+    ####
-# in WebUI, and will be followed by matrix details.  This task does all the
+    #### Variables for composing new cache-images (used in PR testing) from
-# per-pr unit/integration testing.
+    #### base-images (pre-existing in GCE)
-full_vm_testing_task:
+    ####
+    # Git commits to use while building dependencies into cache-images
+    FEDORA_CNI_COMMIT: "412b6d31280682bb4fab4446f113c22ff1886554"
+    CNI_COMMIT: "7480240de9749f9a0a5c8614b17f1f03e0c06ab9"
+    CRIO_COMMIT: "7a283c391abb7bd25086a8ff91dbb36ebdd24466"
+    CRIU_COMMIT: "c74b83cd49c00589c0c0468ba5fe685b67fdbd0a"
+    RUNC_COMMIT: "25f3f893c86d07426df93b7aa172f33fdf093fbd"
+    # CSV of cache-image names to build (see $PACKER_BASE/libpod_images.json)
+    PACKER_BUILDS: "ubuntu-18,fedora-29"  # TODO: fah-29,rhel-7,centos-7
+    # Version of packer to use
+    PACKER_VER: "1.3.1"
+    # Google-maintained base-image names
+    UBUNTU_BASE_IMAGE: "ubuntu-1804-bionic-v20181203a"
+    CENTOS_BASE_IMAGE: "centos-7-v20181113"
+    # Manually produced base-image names (see $SCRIPT_BASE/README.md)
+    FEDORA_BASE_IMAGE:  "fedora-cloud-base-29-1-2-1541789245"
+    FAH_BASE_IMAGE:  "fedora-atomichost-29-20181025-1-1541787861"
+    # RHEL image must be imported, google bills extra for their native image.
+    RHEL_BASE_IMAGE: "rhel-guest-image-7-6-210-x86-64-qcow2-1541783972"
+
+    ####
+    #### Credentials and other secret-sauces, decrypted at runtime when authorized.
+    ####
+    # Freenode IRC credentials for posting status messages
+    IRCID: ENCRYPTED[e87bba62a8e924dc70bdb2b66b16f6ab4a60d2870e6e5534ae9e2b0076f483c71c84091c655ca239101e6816c5ec0883]
+    # Command to register a RHEL VM to install/update packages
+    RHSM_COMMAND: ENCRYPTED[5caa5ff8c5370c3d25c7a1a28168501ab0fa2e5e3b627926f6eaba02b3fed965a7638a6151657809661f8c905c7dc187]
+    # Needed to build GCE images, within a GCE VM
+    SERVICE_ACCOUNT: ENCRYPTED[99e9a0b1c23f8dd29e83dfdf164f064cfd17afd9b895ca3b5e4c41170bd4290a8366fe2ad8e7a210b9f751711d1d002a]
+    # User ID for cirrus to ssh into VMs
+    GCE_SSH_USERNAME: ENCRYPTED[a7706b9e4b8bbb47f76358df7407f4fffa2e8552531190cc0b3315180c4b50588f560c4f85731e99cb5f43a396778277]
+    # Name where this repositories cloud resources are located
+    GCP_PROJECT_ID: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f]
+
+    # Space separated list of environment variables to unset before testing
+    UNSET_ENV_VARS: >-
+        GCP_PROJECT_ID GCE_SSH_USERNAME SERVICE_ACCOUNT RHSM_COMMAND BUILT_IMAGE_SUFFIX
+        IRCID RHEL_BASE_IMAGE FAH_BASE_IMAGE FEDORA_BASE_IMAGE CENTOS_BASE_IMAGE
+        UBUNTU_BASE_IMAGE PACKER_VER PACKER_BUILDS RUNC_COMMIT CRIU_COMMIT
+        CRIO_COMMIT CNI_COMMIT FEDORA_CNI_COMMIT PACKER_BASE SCRIPT_BASE
+        CIRRUS_SHELL CIRRUS_WORKING_DIR ENVLIB BUILT_IMAGE_SUFFIX CIRRUS_CI
+        CI_NODE_INDEX CI_NODE_TOTAL CIRRUS_BASE_BRANCH CIRRUS_BASE_SHA
+        CIRRUS_BRANCH CIRRUS_BUILD_ID CIRRUS_CHANGE_IN_REPO CIRRUS_CLONE_DEPTH
+        CIRRUS_COMMIT_MESSAGE CIRRUS_CHANGE_MESSAGE CIRRUS_REPO_CLONE_HOST
+        CIRRUS_DEFAULT_BRANCH CIRRUS_PR CIRRUS_TAG CIRRUS_OS CIRRUS_TASK_NAME
+        CIRRUS_TASK_ID CIRRUS_REPO_NAME CIRRUS_REPO_OWNER CIRRUS_REPO_FULL_NAME
+        CIRRUS_REPO_CLONE_URL CIRRUS_SHELL CIRRUS_USER_COLLABORATOR CIRRUS_USER_PERMISSION
+        CIRRUS_WORKING_DIR CIRRUS_HTTP_CACHE_HOST PACKER_BUILDS BUILT_IMAGE_SUFFIX
+        XDG_DATA_DIRS XDG_RUNTIME_DIR XDG_SESSION_ID
+
+# Every *_task runs in parallel in separate VMsd. The name prefix only for reference
+# in WebUI, and will be followed by matrix details.  This task gates all others with
+# quick format, lint, and unit tests on the standard platform.
+gating_task:
+
+    env:
+        CIRRUS_WORKING_DIR: "/usr/src/libpod"
+
+    # Runs within Cirrus's "community cluster"
+    container:
+        image: "quay.io/libpod/gate:latest"
+        cpu: 4
+        memory: 12
+
+    gate_script:
+        - '/usr/local/bin/entrypoint.sh validate'
+        - '/usr/local/bin/entrypoint.sh lint'
+
+
+# This task does the unit and integration testing for every platform
+testing_task:
+
+    depends_on:
+        - "gating"
 
     gce_instance:
-        # Generate multiple 'test' tasks, covering all possible
+        image_project: "libpod-218412"
-        # 'matrix' combinations.  All run in parallel.
+        zone: "us-central1-a"  # Required by Cirrus for the time being
+        cpu: 2
+        memory: "4Gb"
+        disk: 40
+        # Generate multiple parallel tasks, covering all possible
+        # 'matrix' combinations.
         matrix:
-            # Images are generated separetly, from build_images_task (below)
+            # Images are generated separately, from build_images_task (below)
             image_name: "ubuntu-18-libpod-0c954a67"
-            # TODO: Make these work (also build_images_task below)
+            image_name: "fedora-29-libpod-0c954a67"
-            #image_name: "rhel-server-ec2-7-5-165-1-libpod-fce09afe"
+            # TODO: tests fail
-            #image_name: "centos-7-v20180911-libpod-fce09afe"
+            # image_name: "rhel-7-something-something"
-            #image_name: "fedora-cloud-base-28-1-1-7-libpod-fce09afe"
+            # image_name: "centos-7-something-something"
+            # image_name: "fah-29-libpod-5070733157859328"
 
     timeout_in: 120m
 
@@ -55,11 +125,9 @@ full_vm_testing_task:
     setup_environment_script: $SCRIPT_BASE/setup_environment.sh
 
     # ...or lists of strings
-    verify_source_script:
+    unit_test_script:
-        - whoami  # root!
+        - go version
-        - $SCRIPT_BASE/verify_source.sh
+        - $SCRIPT_BASE/unit_test.sh
-
-    unit_test_script: $SCRIPT_BASE/unit_test.sh
 
     integration_test_script: $SCRIPT_BASE/integration_test.sh
 
@@ -68,8 +136,7 @@ full_vm_testing_task:
 
 # Because system tests are stored within the repository, it is sometimes
 # necessary to execute them within a PR to validate changes.
-
+optional_testing_task:
-optional_system_testing_task:
 
     # Only run system tests in PRs (not on merge) if magic string is present
     # in the PR description.  Post-merge system testing is assumed to happen
@@ -79,12 +146,13 @@ optional_system_testing_task:
         $CIRRUS_CHANGE_MESSAGE =~ '.*\*\*\*\s*CIRRUS:\s*SYSTEM\s*TEST\s*\*\*\*.*'
 
     gce_instance:
+        image_project: "libpod-218412"
         matrix:
-            image_name: "ubuntu-1804-bionic-v20180911-libpod-e8d18305"
+            image_name: "ubuntu-18-libpod-0c954a67"
+            image_name: "fedora-29-libpod-0c954a67"
             # TODO: Make these work (also build_images_task below)
             #image_name: "rhel-server-ec2-7-5-165-1-libpod-fce09afe"
             #image_name: "centos-7-v20180911-libpod-fce09afe"
-            #image_name: "fedora-cloud-base-28-1-1-7-libpod-fce09afe"
 
     timeout_in: 60m
 
@@ -93,13 +161,11 @@ optional_system_testing_task:
     success_script: $SCRIPT_BASE/success.sh
 
 
-# This task builds new cache-images for future PR testing.  These images save
+# Build new cache-images for future PR testing, but only after a PR merge.
-# time installing/setting up the environment while an engineer is waiting.
+# The cache-images save install/setup time needed test every PR.  The 'active' images
-# The 'active' cache-images for full_vm_testing are selected by the
+# are selected by the 'image_name' items tasks above.  Currently this requires
-# 'image_name' keys.  Updating those items requires manually modification,
+# manually updating the names, but this could be automated (see comment below).
-# but this could be automated (see comment at end of build_vm_images_task).
+cache_images_task:
-
-build_vm_images_task:
     # Only produce new cache-images after a PR merge, and if a magic string
     # is present in the most recent commit-message.
     only_if: >-
@@ -108,45 +174,27 @@ build_vm_images_task:
 
     # Require tests to pass first.
     depends_on:
-        - full_vm_testing  # i.e. 'full_vm_testing_task'
+        - "gating"
-
+        - "testing"
-    env:
-        # CSV of packer builder names to enable (see $PACKER_BASE/libpod_images.json)
-        PACKER_BUILDS: "ubuntu-18"
-        # TODO: Make these work (also full_vm_testing_task above)
-        # PACKER_BUILDS: "rhel-7,centos-7,fedora-28,ubuntu-18"
-        CENTOS_BASE_IMAGE: "centos-7-v20180911"
-        RHEL_BASE_IMAGE: "rhel-server-ec2-7-5-165-1"
-        FEDORA_BASE_IMAGE: "fedora-cloud-base-28-1-1-7"
-        UBUNTU_BASE_IMAGE: "ubuntu-1804-bionic-v20180911"
-        # low-level base VM image name inputs to packer
-
-        # Command to register a RHEL VM
-        RHSM_COMMAND: ENCRYPTED[5caa5ff8c5370c3d25c7a1a28168501ab0fa2e5e3b627926f6eaba02b3fed965a7638a6151657809661f8c905c7dc187]
-        # Additional environment variables needed to build GCE images, within a GCE VM
-        SERVICE_ACCOUNT: ENCRYPTED[99e9a0b1c23f8dd29e83dfdf164f064cfd17afd9b895ca3b5e4c41170bd4290a8366fe2ad8e7a210b9f751711d1d002a]
-        GCE_SSH_USERNAME: ENCRYPTED[a7706b9e4b8bbb47f76358df7407f4fffa2e8552531190cc0b3315180c4b50588f560c4f85731e99cb5f43a396778277]
-        GCP_PROJECT_ID: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f]
-        # Version of packer to use
-        PACKER_VER: "1.3.1"
 
     # VMs created by packer are not cleaned up by cirrus
     auto_cancellation: $CI != "true"
 
     gce_instance:
-        image_name: "image-builder-image"  # Simply CentOS 7 + packer dependencies
+        image_project: "libpod-218412"
+        zone: "us-central1-a"  # Required by Cirrus for the time being
+        cpu: 4
+        memory: "4Gb"
+        disk: 20
+        image_name: "image-builder-image-1541772081"  # Simply CentOS 7 + packer dependencies
         # Additional permissions for building GCE images, within a GCE VM
         scopes:
             - compute
             - devstorage.full_control
-        # Doesn't need many local resources to run
-        cpu: 2
-        memory: "2Gb"
-        disk: 20
     environment_script: $SCRIPT_BASE/setup_environment.sh
     build_vm_images_script: $SCRIPT_BASE/build_vm_images.sh
 
-    # TODO,Continuous Delivery: Automaticly open a libpod PR after using 'sed' to replace
+    # TODO,Continuous Delivery: Automatically open a libpod PR after using 'sed' to replace
     #                           the image_names with the new (just build) images.  That will
     #                           cause a new round of testing to happen (via the PR) using
     #                           the new images.  When all is good, the PR may be manually

Makefile

@@ -172,6 +172,7 @@ testunit: libpodimage
 
 localunit: test/goecho/goecho varlink_generate
 	$(GO) test -tags "$(BUILDTAGS)" -cover $(PACKAGES)
+	$(MAKE) -C contrib/cirrus/packer test
 
 ginkgo:
 	ginkgo -v -tags "$(BUILDTAGS)" -cover -flakeAttempts 3 -progress -trace -noColor test/e2e/.

contrib/cirrus/README.md

@@ -5,6 +5,7 @@
 Similar to other integrated github CI/CD services, Cirrus utilizes a simple
 YAML-based configuration/description file: ``.cirrus.yml``.  Ref: https://cirrus-ci.org/
 
+
 ## Workflow
 
 All tasks execute in parallel, unless there are conditions or dependencies
@@ -12,34 +13,47 @@ which alter this behavior.  Within each task, each script executes in sequence,
 so long as any previous script exited successfully.  The overall state of each
 task (pass or fail) is set based on the exit status of the last script to execute.
 
-### ``full_vm_testing`` Task
 
-1. Unconditionally, spin up one VM per ``matrix: image_name`` item defined
+### ``gating`` Task
-   in ``.cirrus.yml``.  Once accessible, ``ssh`` into each VM and run the following
-   scripts.
 
-2. ``setup_environment.sh``: Configure root's ``.bash_profile``
+***N/B: Steps below are performed by automation***
+
+1. Launch a purpose-built container in Cirrus's community cluster.
+   For container image details, please see
+   [the contributors guide](https://github.com/containers/libpod/blob/master/CONTRIBUTING.md#go-format-and-lint).
+
+3. ``validate``: Perform standard `make validate` source verification,
+   Should run for less than a minute or two.
+
+4. ``lint``: Execute regular `make lint` to check for any code cruft.
+   Should also run for less than a few minutes.
+
+
+### ``testing`` Task
+
+***N/B: Steps below are performed by automation***
+
+1. After `gating` passes, spin up one VM per
+   `matrix: image_name` item. Once accessible, ``ssh``
+   into each VM as the `root` user.
+
+2. ``setup_environment.sh``: Configure root's `.bash_profile`
     for all subsequent scripts (each run in a new shell).  Any
     distribution-specific environment variables are also defined
     here.  For example, setting tags/flags to use compiling.
 
-3. ``verify_source.sh``: Perform per-distribution source
-   verification, lint-checking, etc.  This acts as a minimal
-   gate, blocking extended use of VMs when a PR's code or commits
-   would otherwise not be accepted.  Should run for less than a minute.
-
-4. ``unit_test.sh``: Execute unit-testing, as defined by the ``Makefile``.
-   This should execute within 10-minutes, but often much faster.
-
 5. ``integration_test.sh``: Execute integration-testing.  This is
    much more involved, and relies on access to external
    resources like container images and code from other repositories.
    Total execution time is capped at 2-hours (includes all the above)
    but this script normally completes in less than an hour.
 
-### ``optional_system_testing`` Task
 
-1. Optionally executes in parallel with ``full_vm_testing``.  Requires
+### ``optional_testing`` Task
+
+***N/B: Steps below are performed by automation***
+
+1. Optionally executes in parallel with ``testing``.  Requires
     **prior** to job-start, the magic string ``***CIRRUS: SYSTEM TEST***``
    is found in the pull-request *description*.  The *description* is the first
    text-box under the main *summary* line in the github WebUI.
@@ -49,16 +63,17 @@ task (pass or fail) is set based on the exit status of the last script to execut
 3. ``system_test.sh``: Build both dependencies and libpod, install them,
    then execute `make localsystem` from the repository root.
 
-### ``build_vm_images`` Task
 
-1. When a PR is merged (``$CIRRUS_BRANCH`` == ``master``), Cirrus
+### ``cache_images`` Task
-   checks the last commit message. If it contains the magic string
-   ``***CIRRUS: REBUILD IMAGES***``, then this task continues.
 
-2. Execute run another round of the ``full_vm_testing`` task (above).
+***N/B: Steps below are performed by automation***
-   After the tests pass (post-merge), spin up a special VM
+
-   (from the `image-builder-image`) capable of communicating with the
+1. When a PR is merged (``$CIRRUS_BRANCH`` == ``master``), run another
-   GCE API.  Once accessible, ``ssh`` into the VM and run the following scripts.
+   round of the ``gating`` and ``testing`` tasks (above).
+
+2. Assuming tests pass, if the commit message contains the magic string
+   ``***CIRRUS: REBUILD IMAGES***``, then this task continues.  Otherwise
+   simply mark the master branch as 'passed'.
 
 3. ``setup_environment.sh``: Same as for other tasks.
 
@@ -70,13 +85,108 @@ task (pass or fail) is set based on the exit status of the last script to execut
     1. On a base-image VM, as root, copy the current state of the repository
        into ``/tmp/libpod``.
     2. Execute distribution-specific scripts to prepare the image for
-       use by the ``full_vm_testing`` task (above).  These scripts all
+       use by the ``integration_testing`` task (above).  For example,
-       end with the suffix `_setup.sh` within the `$PACKER_BASE` directory.
+       ``fedora_setup.sh``.
     3. If successful, shut down each VM and create a new GCE Image
-       named after the base image and the commit sha of the merge.
+       named with the base image, and the commit sha of the merge.
 
 ***Note:*** The ``.cirrus.yml`` file must be manually updated with the new
 images names, then the change sent in via a secondary pull-request.  This
-ensures that all the ``full_vm_testing`` tasks can pass with the new images,
+ensures that all the ``integration_testing`` tasks can pass with the new images,
 before subjecting all future PRs to them.  A workflow to automate this
 process is described in comments at the end of the ``.cirrus.yml`` file.
+
+### Base-images
+
+Base-images are VM disk-images specially prepared for executing as GCE VMs.
+In particular, they run services on startup similar in purpose/function
+as the standard 'cloud-init' services.
+
+*  The google services are required for full support of ssh-key management
+   and GCE OAuth capabilities.  Google provides native images in GCE
+   with services pre-installed, for many platforms. For example,
+   RHEL, CentOS, and Ubuntu.
+
+*  Google does ***not*** provide any images for Fedora or Fedora Atomic
+   Host (as of 11/2018), nor do they provide a base-image prepared to
+   run packer for creating other images in the ``build_vm_images`` Task
+   (above).
+
+*  Base images do not need to be produced often, but doing so completely
+   manually would be time-consuming and error-prone.  Therefor a special
+   semi-automatic *Makefile* target is provided to assist with producing
+   all the base-images: ``libpod_base_images``
+
+To produce new base-images, including an `image-builder-image` (used by
+the ``cache_images`` Task) some input parameters are required:
+
+    *  ``GCP_PROJECT_ID``: The complete GCP project ID string e.g. foobar-12345
+       identifying where the images will be stored.
+
+    *  ``GOOGLE_APPLICATION_CREDENTIALS``: A *JSON* file containing
+       credentials for a GCE service account.  This can be [a service
+       account](https://cloud.google.com/docs/authentication/production#obtaining_and_providing_service_account_credentials_manually)
+       or [end-user
+       credentials](https://cloud.google.com/docs/authentication/end-user#creating_your_client_credentials]
+
+    *  ``RHEL_IMAGE_FILE`` and ``RHEL_CSUM_FILE`` complete paths
+       to a `rhel-server-ec2-*.raw.xz` and it's cooresponding
+       checksum file.  These must be supplied manually because
+       they're not available directly via URL like other images.
+
+    * ``RHSM_COMMAND`` contains the complete string needed to register
+      the VM for installing package dependencies.  The VM will be de-registered
+      upon completion.
+
+    *  Optionally, CSV's may be specified to ``PACKER_BUILDS``
+       to limit the base-images produced.  For example,
+       ``PACKER_BUILDS=fedora,image-builder-image``.
+
+The following process should be performed on a bare-metal CentOS 7 machine
+with network access to GCE.  Software dependencies can be obtained from
+the ``packer/image-builder-image_base_setup.sh`` script.
+
+Alternatively, an existing image-builder-image may be used from within GCE.
+However it must be created with elevated cloud privileges.  For example,
+
+```
+$ alias pgcloud='sudo podman run -it --rm -e AS_ID=$UID
+    -e AS_USER=$USER -v /home/$USER:/home/$USER:z cevich/gcloud_centos:latest'
+
+$ URL=https://www.googleapis.com/auth
+$ SCOPES=$URL/userinfo.email,$URL/compute,$URL/devstorage.full_control
+
+$ pgcloud compute instances create $USER-making-images \
+    --image-family image-builder-image \
+    --boot-disk-size "200GB" \
+    --min-cpu-platform "Intel Haswell" \
+    --machine-type n1-standard-2 \
+    --scopes $SCOPES
+
+$ pgcloud compute ssh centos@$USER-making-images
+...
+```
+
+When ready, change to the ``packer`` sub-directory, and run:
+
+```
+$ make libpod_base_images GCP_PROJECT_ID=<VALUE> \
+    GOOGLE_APPLICATION_CREDENTIALS=<VALUE> \
+    RHEL_IMAGE_FILE=<VALUE> \
+    RHEL_CSUM_FILE=<VALUE> \
+    PACKER_BUILDS=<OPTIONAL>
+```
+
+Assuming this is successful (hence the semi-automatic part), packer will
+produce a ``packer-manifest.json`` output file.  This contains the base-image
+names suitable for updating in ``.cirrus.yml``, `env` keys ``*_BASE_IMAGE``.
+
+On failure, it should be possible to determine the problem from the packer
+output.  The only exception is for the Fedora and FAH builds, which utilize
+local qemu-kvm virtualisation.  To observe the serial-port output from those
+builds, set the ``TTYDEV`` parameter to your current device.  For example:
+
+```
+$ make libpod_base_images ... TTYDEV=$(tty)
+  ...
+```

contrib/cirrus/build_vm_images.sh

@@ -8,12 +8,13 @@ CNI_COMMIT $CNI_COMMIT
 CRIO_COMMIT $CRIO_COMMIT
 RUNC_COMMIT $RUNC_COMMIT
 PACKER_BUILDS $PACKER_BUILDS
+BUILT_IMAGE_SUFFIX $BUILT_IMAGE_SUFFIX
 CENTOS_BASE_IMAGE $CENTOS_BASE_IMAGE
 UBUNTU_BASE_IMAGE $UBUNTU_BASE_IMAGE
 FEDORA_BASE_IMAGE $FEDORA_BASE_IMAGE
+FAH_BASE_IMAGE $FAH_BASE_IMAGE
 RHEL_BASE_IMAGE $RHEL_BASE_IMAGE
 RHSM_COMMAND $RHSM_COMMAND
-BUILT_IMAGE_SUFFIX $BUILT_IMAGE_SUFFIX
 SERVICE_ACCOUNT $SERVICE_ACCOUNT
 GCE_SSH_USERNAME $GCE_SSH_USERNAME
 GCP_PROJECT_ID $GCP_PROJECT_ID
@@ -28,28 +29,24 @@ show_env_vars
 # Assume basic dependencies are all met, but there could be a newer version
 # of the packer binary
 PACKER_FILENAME="packer_${PACKER_VER}_linux_amd64.zip"
-mkdir -p "$HOME/packer"
+if [[ -d "$HOME/packer" ]]
-cd "$HOME/packer"
-# image_builder_image has packer pre-installed, check if same version requested
-if ! [[ -r "$PACKER_FILENAME" ]]
 then
-    curl -L -O https://releases.hashicorp.com/packer/$PACKER_VER/$PACKER_FILENAME
+    cd "$HOME/packer"
-    curl -L https://releases.hashicorp.com/packer/${PACKER_VER}/packer_${PACKER_VER}_SHA256SUMS | \
+    # image_builder_image has packer pre-installed, check if same version requested
-        grep 'linux_amd64' > ./sha256sums
+    if [[ -r "$PACKER_FILENAME" ]]
-    sha256sum --check ./sha256sums
+    then
-    unzip -o $PACKER_FILENAME
+        cp $PACKER_FILENAME "$GOSRC/$PACKER_BASE/"
-    ./packer --help &> /dev/null # verify exit(0)
+        cp packer "$GOSRC/$PACKER_BASE/"
+    fi
 fi
 
 set -x
 
-cd "$GOSRC"
+cd "$GOSRC/$PACKER_BASE"
-# N/B: /usr/sbin/packer is a DIFFERENT tool, and will exit 0 given the args below :(
+make libpod_images \
-TEMPLATE="./$PACKER_BASE/libpod_images.json"
+    PACKER_BUILDS=$PACKER_BUILDS \
-
+    PACKER_VER=$PACKER_VER \
-$HOME/packer/packer inspect "$TEMPLATE"
+    GOSRC=$GOSRC \
-
+    SCRIPT_BASE=$SCRIPT_BASE \
-#$HOME/packer/packer build -machine-readable "-only=$PACKER_BUILDS" "$TEMPLATE" | tee /tmp/packer_log.csv
+    PACKER_BASE=$PACKER_BASE \
-$HOME/packer/packer build "-only=$PACKER_BUILDS" "$TEMPLATE"
+    BUILT_IMAGE_SUFFIX=$BUILT_IMAGE_SUFFIX
-
-# TODO: Report back to PR names of built images

contrib/cirrus/integration_test.sh

@@ -9,7 +9,7 @@ OS_RELEASE_ID $OS_RELEASE_ID
 OS_RELEASE_VER $OS_RELEASE_VER
 "
 
-show_env_vars
+clean_env
 
 set -x
 cd "$GOSRC"
@@ -19,10 +19,13 @@ case "${OS_RELEASE_ID}-${OS_RELEASE_VER}" in
         make test-binaries "BUILDTAGS=$BUILDTAGS"
         SKIP_USERNS=1 make localintegration "BUILDTAGS=$BUILDTAGS"
         ;;
-    fedora-28) ;&  # Continue to the next item
+    fedora-29) ;&  # Continue to the next item
+    fedora-28) ;&
     centos-7) ;&
     rhel-7)
-        stub 'integration testing not working on $OS_RELEASE_ID'
+        make install PREFIX=/usr ETCDIR=/etc
+        make test-binaries
+        make localintegration
         ;;
     *) bad_os_id_ver ;;
 esac

contrib/cirrus/lib.sh

@@ -4,8 +4,8 @@
 # to be sourced by other scripts, not called directly.
 
 # Under some contexts these values are not set, make sure they are.
-USER="$(whoami)"
+export USER="$(whoami)"
-HOME="$(getent passwd $USER | cut -d : -f 6)"
+export HOME="$(getent passwd $USER | cut -d : -f 6)"
 if ! [[ "$PATH" =~ "/usr/local/bin" ]]
 then
     export PATH="$PATH:/usr/local/bin"
@@ -73,6 +73,18 @@ PACKER_BUILDS $PACKER_BUILDS
     do
         [[ -z "$NAME" ]] || echo "export $NAME=\"$VALUE\""
     done
+    echo ""
+    echo "##### $(go version) #####"
+    echo ""
+}
+
+# Unset environment variables not needed for testing purposes
+clean_env() {
+    req_env_var "
+        UNSET_ENV_VARS $UNSET_ENV_VARS
+    "
+    echo "Unsetting $(echo $UNSET_ENV_VARS | wc -w) environment variables"
+    unset -v UNSET_ENV_VARS $UNSET_ENV_VARS || true  # don't fail on read-only
 }
 
 # Return a GCE image-name compatible string representation of distribution name
@@ -269,21 +281,29 @@ install_varlink(){
 }
 
 _finalize(){
+    set +e  # Don't fail at the very end
+    set +e  # make errors non-fatal
     echo "Removing leftover giblets from cloud-init"
     cd /
     sudo rm -rf /var/lib/cloud/instance?
     sudo rm -rf /root/.ssh/*
     sudo rm -rf /home/*
+    sudo rm -rf /tmp/*
+    sudo rm -rf /tmp/.??*
+    sync
+    sudo fstrim -av
 }
 
 rh_finalize(){
+    set +e  # Don't fail at the very end
     # Allow root ssh-logins
     if [[ -r /etc/cloud/cloud.cfg ]]
     then
         sudo sed -re 's/^disable_root:.*/disable_root: 0/g' -i /etc/cloud/cloud.cfg
     fi
     echo "Resetting to fresh-state for usage as cloud-image."
-    sudo $(type -P dnf || type -P yum) clean all
+    PKG=$(type -P dnf || type -P yum || echo "")
+    [[ -z "$PKG" ]] || sudo $PKG clean all  # not on atomic
     sudo rm -rf /var/cache/{yum,dnf}
     sudo rm -f /etc/udev/rules.d/*-persistent-*.rules
     sudo touch /.unconfigured  # force firstboot to run
@@ -291,7 +311,35 @@ rh_finalize(){
 }
 
 ubuntu_finalize(){
+    set +e  # Don't fail at the very end
     echo "Resetting to fresh-state for usage as cloud-image."
     sudo rm -rf /var/cache/apt
     _finalize
 }
+
+rhel_exit_handler() {
+    set +ex
+    req_env_var "
+        GOPATH $GOPATH
+        RHSMCMD $RHSMCMD
+    "
+    cd /
+    sudo rm -rf "$RHSMCMD"
+    sudo rm -rf "$GOPATH"
+    sudo subscription-manager remove --all
+    sudo subscription-manager unregister
+    sudo subscription-manager clean
+}
+
+rhsm_enable() {
+    req_env_var "
+        RHSM_COMMAND $RHSM_COMMAND
+    "
+    export GOPATH="$(mktemp -d)"
+    export RHSMCMD="$(mktemp)"
+    trap "rhel_exit_handler" EXIT
+    # Avoid logging sensitive details
+    echo "$RHSM_COMMAND" > "$RHSMCMD"
+    ooe.sh sudo bash "$RHSMCMD"
+    sudo rm -rf "$RHSMCMD"
+}

contrib/cirrus/packer/.gitignore

@@ -0,0 +1,7 @@
+*json
+packer
+packer*zip
+packer_cache
+cidata*
+meta-data
+user-data

contrib/cirrus/packer/Makefile

@@ -0,0 +1,108 @@
+
+# N/B: PACKER_BUILDS variable is required.  Should contain CSV of
+# builder name(s) from applicable YAML file,
+# e.g for names see libpod_images.yml
+
+PACKER_VER ?= 1.3.1
+PACKER_DIST_FILENAME := packer_${PACKER_VER}_linux_amd64.zip
+
+# Only needed for libpod_base_images target
+TIMESTAMP := $(shell date +%s)
+GOSRC ?= $(shell realpath "./../../../")
+PACKER_BASE ?= contrib/cirrus/packer
+SCRIPT_BASE ?= contrib/cirrus
+
+# For debugging nested-virt, use
+#TTYDEV := $(shell tty)
+TTYDEV := /dev/null
+
+.PHONY: all
+all: libpod_images
+
+%.json: %.yml
+	@python3 -c 'import json,yaml; json.dump( yaml.load(open("$<").read()), open("$@","w"), indent=2);'
+
+${PACKER_DIST_FILENAME}:
+	@curl -L --silent --show-error \
+		-O https://releases.hashicorp.com/packer/${PACKER_VER}/${PACKER_DIST_FILENAME}
+
+packer: ${PACKER_DIST_FILENAME}
+	@curl -L --silent --show-error \
+		https://releases.hashicorp.com/packer/${PACKER_VER}/packer_${PACKER_VER}_SHA256SUMS \
+		| grep 'linux_amd64' > /tmp/packer_sha256sums
+	@sha256sum --check /tmp/packer_sha256sums
+	@unzip -o ${PACKER_DIST_FILENAME}
+	@touch --reference=Makefile ${PACKER_DIST_FILENAME}
+
+.PHONY: test
+test: libpod_base_images.json libpod_images.json packer
+	./packer inspect libpod_base_images.json > /dev/null
+	./packer inspect libpod_images.json > /dev/null
+	@echo "All good"
+
+.PHONY: libpod_images
+libpod_images: libpod_images.json packer
+ifndef PACKER_BUILDS
+	$(error PACKER_BUILDS is undefined, expected builder-names CSV)
+endif
+	./packer build -only=${PACKER_BUILDS} \
+		-var GOSRC=$(GOSRC) \
+		-var PACKER_BASE=$(PACKER_BASE) \
+		-var SCRIPT_BASE=$(SCRIPT_BASE) \
+		libpod_images.json
+	@echo ""
+	@echo "Finished.  The images mentioned above, and in packer-manifest.json"
+	@echo "can be used in .cirrus.yml as values for the 'image_name' keys"
+	@echo ""
+
+cidata.ssh:
+	ssh-keygen -f $@ -P "" -q
+
+cidata.ssh.pub: cidata.ssh
+	touch $@
+
+meta-data:
+	echo "local-hostname: localhost.localdomain" > $@
+
+user-data: cidata.ssh.pub
+	bash make-user-data.sh
+
+cidata.iso: user-data meta-data
+	genisoimage -output cidata.iso -volid cidata -input-charset utf-8 -joliet -rock user-data meta-data
+
+# This is intended to be run by a human, with admin access to the libpod GCE project.
+.PHONY: libpod_base_images
+libpod_base_images: libpod_base_images.json cidata.iso cidata.ssh packer
+ifndef GCP_PROJECT_ID
+	$(error GCP_PROJECT_ID is undefined, expected complete GCP project ID string e.g. foobar-12345)
+endif
+ifndef GOOGLE_APPLICATION_CREDENTIALS
+	$(error GOOGLE_APPLICATION_CREDENTIALS is undefined, expected absolute path to JSON file, like $HOME/.config/gcloud/legacy_credentials/*/adc.json)
+endif
+ifndef RHEL_IMAGE_FILE
+	$(error RHEL_IMAGE_FILE is undefined, expected full path to a rhel-server-ec2-*.raw.xz file)
+endif
+ifndef RHEL_CSUM_FILE
+	$(error RHEL_CSUM_FILE is undefined, expected full path to a rhel-server-ec2-*.raw.xz.SHA256SUM file)
+endif
+ifndef RHSM_COMMAND
+	$(error RHSM_COMMAND is undefined, expected string required for temporarily registering VM)
+endif
+	PACKER_CACHE_DIR=/tmp ./packer build \
+		-var TIMESTAMP=$(TIMESTAMP) \
+		-var TTYDEV=$(TTYDEV) \
+		-var GCP_PROJECT_ID=$(GCP_PROJECT_ID) \
+		-var GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) \
+		-var GOSRC=$(GOSRC) \
+		-var PACKER_BASE=$(PACKER_BASE) \
+		-var SCRIPT_BASE=$(SCRIPT_BASE) \
+		-var RHEL_BASE_IMAGE_NAME=$(shell basename $(RHEL_IMAGE_FILE) | tr -d '[[:space:]]' | sed -r -e 's/\.x86_64\.raw\.xz//' | tr '[[:upper:]]' '[[:lower:]]' | tr '[[:punct:]]' '-') \
+		-var RHEL_IMAGE_FILE=$(RHEL_IMAGE_FILE) \
+		-var RHEL_CSUM_FILE=$(RHEL_CSUM_FILE) \
+	    -var 'RHSM_COMMAND=$(RHSM_COMMAND)' \
+		-only $(PACKER_BUILDS) \
+		libpod_base_images.json
+	@echo ""
+	@echo "Finished.  The images mentioned above, and in packer-manifest.json"
+	@echo "can be used in .cirrus.yml as values for the *_BASE_IMAGE keys."
+	@echo ""

contrib/cirrus/packer/README.md

@@ -1,2 +1,3 @@
 These are definitions and scripts consumed by packer to produce the
-various distribution images used for CI testing.
+various distribution images used for CI testing.  For more details
+see the [Cirrus CI documentation](../README.md)

contrib/cirrus/packer/fah_base-setup.sh

@@ -0,0 +1,45 @@
+
+# N/B: This script is not intended to be run by humans.  It is used to configure the
+# FAH base image for importing, so that it will boot in GCE.
+
+set -e
+
+# Load in library (copied by packer, before this script was run)
+source $GOSRC/$SCRIPT_BASE/lib.sh
+
+install_ooe
+
+if [[ "$1" == "pre" ]]
+then
+    echo "Upgrading Atomic Host"
+    setenforce 0
+    ooe.sh atomic host upgrade
+
+    echo "Configuring Repositories"
+    ooe.sh sudo tee /etc/yum.repos.d/ngompa-gce-oslogin.repo <<EOF
+[ngompa-gce-oslogin]
+name=Copr repo for gce-oslogin owned by ngompa
+baseurl=https://copr-be.cloud.fedoraproject.org/results/ngompa/gce-oslogin/fedora-\$releasever-\$basearch/
+type=rpm-md
+skip_if_unavailable=True
+gpgcheck=1
+gpgkey=https://copr-be.cloud.fedoraproject.org/results/ngompa/gce-oslogin/pubkey.gpg
+repo_gpgcheck=0
+enabled=1
+enabled_metadata=1
+EOF
+    echo "Installing necessary packages and  google services"
+    # Google services are enabled by default, upon install.
+    ooe.sh rpm-ostree install rng-tools google-compute-engine google-compute-engine-oslogin
+    echo "Rebooting..."
+    systemctl reboot  # Required for upgrade + package installs to be active
+elif [[ "$1" == "post" ]]
+then
+    echo "Enabling necessary services"
+    systemctl enable rngd  # Must reboot before enabling
+    rh_finalize
+    echo "SUCCESS!"
+else
+    echo "Expected to be called with 'pre' or 'post'"
+    exit 6
+fi

contrib/cirrus/packer/fah_setup.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# This script is called by packer on the subject fah VM, to setup the podman
+# build/test environment.  It's not intended to be used outside of this context.
+
+set -e
+
+# Load in library (copied by packer, before this script was run)
+source /tmp/libpod/$SCRIPT_BASE/lib.sh
+
+req_env_var "
+SCRIPT_BASE $SCRIPT_BASE
+"
+
+install_ooe
+
+ooe.sh sudo atomic host upgrade
+
+ooe.sh sudo rpm-ostree uninstall cloud-init
+
+rh_finalize
+
+echo "SUCCESS!"

contrib/cirrus/packer/fedora_base-setup.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# N/B: This script is not intended to be run by humans.  It is used to configure the
+# fedora base image for importing, so that it will boot in GCE
+
+set -e
+
+# Load in library (copied by packer, before this script was run)
+source $GOSRC/$SCRIPT_BASE/lib.sh
+
+[[ "$1" == "post" ]] || exit 0  # nothing to do
+
+install_ooe
+
+echo "Updating packages"
+ooe.sh dnf -y update
+
+echo "Installing necessary packages and  google services"
+ooe.sh dnf -y copr enable ngompa/gce-oslogin
+ooe.sh dnf -y install rng-tools google-compute-engine google-compute-engine-oslogin
+
+echo "Enabling services"
+ooe.sh systemctl enable rngd
+
+rh_finalize
+
+echo "SUCCESS!"

contrib/cirrus/packer/fedora_setup.sh

@@ -10,6 +10,7 @@ source /tmp/libpod/$SCRIPT_BASE/lib.sh
 
 req_env_var "
 SCRIPT_BASE $SCRIPT_BASE
+FEDORA_CNI_COMMIT $FEDORA_CNI_COMMIT
 CNI_COMMIT $CNI_COMMIT
 CRIO_COMMIT $CRIO_COMMIT
 CRIU_COMMIT $CRIU_COMMIT
@@ -65,11 +66,13 @@ ooe.sh sudo dnf install -y \
     runc \
     skopeo-containers \
     slirp4netns \
+    unzip \
     which \
     xz
 
 install_varlink
 
+CNI_COMMIT=$FEDORA_CNI_COMMIT
 install_cni_plugins
 
 install_buildah

contrib/cirrus/packer/image-builder-image_base-setup.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+# This script is called by packer on a vanilla CentOS VM, to setup the image
+# used for building images FROM base images. It's not intended to be used
+# outside of this context.
+
+set -e
+
+[[ "$1" == "post" ]] || exit 0  # pre stage not needed
+
+# Load in library (copied by packer, before this script was run)
+source $GOSRC/$SCRIPT_BASE/lib.sh
+
+req_env_var "
+    TIMESTAMP $TIMESTAMP
+    GOSRC $GOSRC
+    SCRIPT_BASE $SCRIPT_BASE
+    PACKER_BASE $PACKER_BASE
+"
+
+install_ooe
+
+echo "Updating packages"
+ooe.sh sudo yum -y update
+
+echo "Configuring repositories"
+ooe.sh sudo yum -y install centos-release-scl epel-release
+
+echo "Installing packages"
+ooe.sh sudo yum -y install \
+    genisoimage \
+    golang \
+    google-cloud-sdk \
+    libvirt \
+    libvirt-admin \
+    libvirt-client \
+    libvirt-daemon \
+    make \
+    python34 \
+    python34 \
+    python34-PyYAML \
+    python34-PyYAML \
+    qemu-img \
+    qemu-kvm \
+    qemu-kvm-tools \
+    qemu-user \
+    rsync \
+    unzip \
+    util-linux \
+    vim
+
+sudo ln -s /usr/libexec/qemu-kvm /usr/bin/
+
+sudo tee /etc/modprobe.d/kvm-nested.conf <<EOF
+options kvm-intel nested=1
+options kvm-intel enable_shadow_vmcs=1
+options kvm-intel enable_apicv=1
+options kvm-intel ept=1
+EOF
+
+echo "Installing packer"
+sudo mkdir -p /root/$(basename $PACKER_BASE)
+sudo cp $GOSRC/$PACKER_BASE/*packer* /root/$(basename $PACKER_BASE)
+sudo mkdir -p /root/$(basename $SCRIPT_BASE)
+sudo cp $GOSRC/$SCRIPT_BASE/*.sh /root/$(basename $SCRIPT_BASE)
+
+install_scl_git
+
+echo "Cleaning up"
+cd /
+rm -rf $GOSRC
+
+rh_finalize
+
+echo "SUCCESS!"

contrib/cirrus/packer/libpod_base_images.yml

@@ -0,0 +1,179 @@
+---
+
+variables:
+    # Complete local path to this repository (Required)
+    GOSRC:
+    # Relative path to this (packer) subdirectory (Required)
+    PACKER_BASE:
+    # Relative path to cirrus scripts subdirectory (Required)
+    SCRIPT_BASE:
+    # Unique ID for naming new base-images (required)
+    TIMESTAMP:
+    # Required for output from qemu builders
+    TTYDEV:
+    # RHEL images require click-through agreements to obtain (required)
+    RHEL_BASE_IMAGE_NAME:
+    RHEL_IMAGE_FILE:
+    RHEL_CSUM_FILE:
+    # RHEL requires a subscription to install/update packages
+    RHSM_COMMAND:
+
+    # Fedora images are obtainable by direct download
+    FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/29/Cloud/x86_64/images/Fedora-Cloud-Base-29-1.2.x86_64.qcow2"
+    FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/29/Cloud/x86_64/images/Fedora-Cloud-29-1.2-x86_64-CHECKSUM"
+    FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-29-1-2'  # Name to use in GCE
+    FAH_IMAGE_URL: "https://dl.fedoraproject.org/pub/alt/atomic/stable/Fedora-Atomic-29-20181025.1/AtomicHost/x86_64/images/Fedora-AtomicHost-29-20181025.1.x86_64.qcow2"
+    FAH_CSUM_URL: "https://dl.fedoraproject.org/pub/alt/atomic/stable/Fedora-Atomic-29-20181025.1/AtomicHost/x86_64/images/Fedora-AtomicHost-29-20181025.1-x86_64-CHECKSUM"
+    FAH_BASE_IMAGE_NAME: 'fedora-atomichost-29-20181025-1'  # Name to use in GCE
+
+    # The name of the image in GCE used for packer build libpod_images.yml
+    IBI_BASE_NAME: 'image-builder-image'
+    CIDATA_ISO: 'cidata.iso'  # produced by Makefile
+
+    # Path to json file (required, likely ~/.config/gcloud/legacy_credentials/*/adc.json)
+    GOOGLE_APPLICATION_CREDENTIALS:
+    # The complete project ID (required, not the short name)
+    GCP_PROJECT_ID:
+    # Pre-existing storage bucket w/ lifecycle-enabled
+    XFERBUCKET: "packer-import"  # pre-created, globally unique, lifecycle-enabled
+
+# Don't leak sensitive values in error messages / output
+sensitive-variables:
+    - 'GOOGLE_APPLICATION_CREDENTIALS'
+    - 'GCP_PROJECT_ID'
+    - 'RHSM_COMMAND'
+
+# What images to produce in which cloud
+builders:
+    - name: '{{user `IBI_BASE_NAME`}}'
+      type: 'googlecompute'
+      image_name: '{{user `IBI_BASE_NAME`}}-{{user `TIMESTAMP`}}'
+      image_family: '{{user `IBI_BASE_NAME`}}'
+      source_image_project_id: 'centos-cloud'
+      source_image_family: 'centos-7'
+      project_id: '{{user `GCP_PROJECT_ID`}}'
+      account_file: '{{user `GOOGLE_APPLICATION_CREDENTIALS`}}'
+      communicator: 'ssh'
+      ssh_username: 'centos'
+      ssh_pty: 'true'
+      # The only supported zone in Cirrus-CI, as of addition of this comment
+      zone: 'us-central1-a'
+      # Enable nested virtualization in case it's ever needed
+      image_licenses:
+        - 'https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx'
+      min_cpu_platform: "Intel Broadwell"  # nested-virt requirement
+
+    - &nested_virt
+      name: 'fedora'
+      type: 'qemu'
+      accelerator: "kvm"
+      iso_url: '{{user `FEDORA_IMAGE_URL`}}'
+      disk_image: true
+      format: "raw"
+      disk_size: 5120
+      iso_checksum_url: '{{user `FEDORA_CSUM_URL`}}'
+      iso_checksum_type: "sha256"
+      output_directory: '/tmp/{{build_name}}'
+      vm_name: "disk.raw"  # actually qcow2, name required for post-processing
+      boot_wait: '5s'
+      shutdown_command: 'shutdown -h now'
+      headless: true
+      qemu_binary: "/usr/libexec/qemu-kvm"
+      qemuargs:  # List-of-list format required to override packer-generated args
+        - - "-m"
+          - "1024"
+        - - "-cpu"
+          - "host"
+        - - "-device"
+          - "virtio-rng-pci"
+        - - "-chardev"
+          - "tty,id=pts,path={{user `TTYDEV`}}"
+        - - "-device"
+          - "isa-serial,chardev=pts"
+        - - "-cdrom"
+          - "{{user `CIDATA_ISO`}}"
+        - - "-netdev"
+          - "user,id=net0,hostfwd=tcp::{{ .SSHHostPort }}-:22"
+        - - "-device"
+          - "virtio-net,netdev=net0"
+      communicator: 'ssh'
+      ssh_private_key_file: 'cidata.ssh'
+      ssh_username: 'root'
+
+    - <<: *nested_virt
+      name: 'fah'
+      iso_url: '{{user `FAH_IMAGE_URL`}}'
+      iso_checksum_url: '{{user `FAH_CSUM_URL`}}'
+      disk_size: 10240
+
+    - <<: *nested_virt
+      name: 'rhel'
+      iso_url: 'file://{{user `RHEL_IMAGE_FILE`}}'
+      iso_checksum_url: 'file://{{user `RHEL_CSUM_FILE`}}'
+      disk_size: 10240
+
+provisioners:
+    - type: 'shell'
+      inline:
+        - 'mkdir -p /tmp/libpod/{{user `SCRIPT_BASE`}}'
+        - 'mkdir -p /tmp/libpod/{{user `PACKER_BASE`}}'
+
+    - type: 'file'
+      source: '{{user `GOSRC`}}/.cirrus.yml'
+      destination: '/tmp/libpod/.cirrus.yml'
+
+    - type: 'file'
+      source: '{{user `GOSRC`}}/{{user `SCRIPT_BASE`}}/'
+      destination: '/tmp/libpod/{{user `SCRIPT_BASE`}}/'
+
+    - type: 'file'
+      source: '{{user `GOSRC`}}/{{user `PACKER_BASE`}}/'
+      destination: '/tmp/libpod/{{user `PACKER_BASE`}}/'
+
+    - &shell_script
+      type: 'shell'
+      inline:
+        - 'chmod +x /tmp/libpod/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh'
+        - '/tmp/libpod/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh pre'
+      expect_disconnect: true  # Allow this to reboot the VM
+      environment_vars:
+          - 'TIMESTAMP={{user `TIMESTAMP`}}'
+          - 'GOSRC=/tmp/libpod'
+          - 'SCRIPT_BASE={{user `SCRIPT_BASE`}}'
+          - 'PACKER_BASE={{user `PACKER_BASE`}}'
+          - 'RHSM_COMMAND={{user `RHSM_COMMAND`}}'
+
+    - <<: *shell_script
+      inline: ['{{user `GOSRC`}}/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh']
+      expect_disconnect: false
+      pause_before: '10s'
+      inline:
+        - '/tmp/libpod/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh post'
+
+post-processors:
+    - - type: "compress"
+        only: ['fedora', 'fah', 'rhel']
+        output: '/tmp/{{build_name}}/disk.raw.tar.gz'
+        format: '.tar.gz'
+        compression_level: 9
+      - &gcp_import
+        only: ['fedora']
+        type: "googlecompute-import"
+        project_id: '{{user `GCP_PROJECT_ID`}}'
+        account_file: '{{user `GOOGLE_APPLICATION_CREDENTIALS`}}'
+        bucket: '{{user `XFERBUCKET`}}'
+        gcs_object_name: '{{build_name}}-{{user `TIMESTAMP`}}-{{uuid}}.tar.gz'
+        image_name: "{{user `FEDORA_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}"
+        image_description: 'Based on {{user `FEDORA_IMAGE_URL`}}'
+        image_family: '{{user `FEDORA_BASE_IMAGE_NAME`}}'
+      - <<: *gcp_import
+        only: ['fah']
+        image_name: "{{user `FAH_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}"
+        image_description: 'Based on {{user `FAH_IMAGE_URL`}}'
+        image_family: '{{user `FAH_BASE_IMAGE_NAME`}}'
+      - <<: *gcp_import
+        only: ['rhel']
+        image_name: "{{user `RHEL_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}"
+        image_description: 'Based on {{user `RHEL_IMAGE_FILE`}}'
+        image_family: '{{user `RHEL_BASE_IMAGE_NAME`}}'
+      - type: 'manifest'

contrib/cirrus/packer/libpod_images.json

@@ -1,130 +0,0 @@
-{
-  "variables": {
-    "FEDORA_CNI_COMMIT": "{{env `FEDORA_CNI_COMMIT`}}",
-    "CNI_COMMIT": "{{env `CNI_COMMIT`}}",
-    "CRIO_COMMIT": "{{env `CRIO_COMMIT`}}",
-    "CRIU_COMMIT": "{{env `CRIU_COMMIT`}}",
-    "RUNC_COMMIT": "{{env `RUNC_COMMIT`}}",
-
-    "CENTOS_BASE_IMAGE": "{{env `CENTOS_BASE_IMAGE`}}" ,
-    "UBUNTU_BASE_IMAGE": "{{env `UBUNTU_BASE_IMAGE`}}",
-    "FEDORA_BASE_IMAGE": "{{env `FEDORA_BASE_IMAGE`}}",
-    "RHEL_BASE_IMAGE": "{{env `RHEL_BASE_IMAGE`}}",
-
-    "GOSRC": "{{env `GOSRC`}}",
-    "PACKER_BASE": "{{env `PACKER_BASE`}}",
-    "SCRIPT_BASE": "{{env `SCRIPT_BASE`}}",
-
-    "SERVICE_ACCOUNT": "{{env `SERVICE_ACCOUNT`}}",
-    "GCP_PROJECT_ID": "{{env `GCP_PROJECT_ID`}}",
-    "BUILT_IMAGE_SUFFIX": "{{env `BUILT_IMAGE_SUFFIX`}}",
-    "GCE_SSH_USERNAME": "{{env `GCE_SSH_USERNAME`}}",
-    "RHSM_COMMAND": "{{env `RHSM_COMMAND`}}"
-  },
-  "sensitive-variables": [
-    "GCP_PROJECT_ID", "SERVICE_ACCOUNT", "GCE_SSH_USERNAME", "RHSM_COMMAND"
-  ],
-  "builders": [
-    {
-      "name": "rhel-7",
-      "type": "googlecompute",
-      "project_id": "{{user `GCP_PROJECT_ID`}}",
-      "zone": "us-central1-a",
-      "source_image": "{{user `RHEL_BASE_IMAGE`}}",
-      "image_name": "{{user `RHEL_BASE_IMAGE`}}{{user `BUILT_IMAGE_SUFFIX`}}",
-      "image_family": "{{user `RHEL_BASE_IMAGE`}}-libpod",
-      "service_account_email": "{{user `SERVICE_ACCOUNT`}}",
-      "communicator": "ssh",
-      "ssh_username": "ec2-user",
-      "ssh_pty": "true"
-    },{
-      "name": "centos-7",
-      "type": "googlecompute",
-      "project_id": "{{user `GCP_PROJECT_ID`}}",
-      "zone": "us-central1-a",
-      "source_image": "{{user `CENTOS_BASE_IMAGE`}}",
-      "image_name": "{{user `CENTOS_BASE_IMAGE`}}{{user `BUILT_IMAGE_SUFFIX`}}",
-      "image_family": "{{user `CENTOS_BASE_IMAGE`}}-libpod",
-      "service_account_email": "{{user `SERVICE_ACCOUNT`}}",
-      "communicator": "ssh",
-      "ssh_username": "{{user `GCE_SSH_USERNAME`}}",
-      "ssh_pty": "true"
-    },{
-      "name": "fedora-28",
-      "type": "googlecompute",
-      "project_id": "{{user `GCP_PROJECT_ID`}}",
-      "zone": "us-central1-a",
-      "source_image": "{{user `FEDORA_BASE_IMAGE`}}",
-      "image_name": "{{user `FEDORA_BASE_IMAGE`}}{{user `BUILT_IMAGE_SUFFIX`}}",
-      "image_family": "{{user `FEDORA_BASE_IMAGE`}}-libpod",
-      "service_account_email": "{{user `SERVICE_ACCOUNT`}}",
-      "communicator": "ssh",
-      "ssh_username": "fedora",
-      "ssh_pty": "true"
-    },{
-      "name": "ubuntu-18",
-      "type": "googlecompute",
-      "project_id": "{{user `GCP_PROJECT_ID`}}",
-      "zone": "us-central1-a",
-      "source_image": "{{user `UBUNTU_BASE_IMAGE`}}",
-      "image_name": "{{user `UBUNTU_BASE_IMAGE`}}{{user `BUILT_IMAGE_SUFFIX`}}",
-      "image_family": "{{user `UBUNTU_BASE_IMAGE`}}-libpod",
-      "service_account_email": "{{user `SERVICE_ACCOUNT`}}",
-      "communicator": "ssh",
-      "ssh_username": "{{user `GCE_SSH_USERNAME`}}",
-      "ssh_pty": "true"
-    }
-  ],
-  "provisioners": [
-    {
-      "type": "file",
-      "source": "{{user `GOSRC`}}",
-      "destination": "/tmp/libpod"
-    },{
-      "type": "shell",
-      "only": ["rhel-7"],
-      "script": "{{user `GOSRC`}}/{{user `PACKER_BASE`}}/rhel_setup.sh",
-      "environment_vars": [
-        "SCRIPT_BASE={{user `SCRIPT_BASE`}}",
-        "CNI_COMMIT={{user `CNI_COMMIT`}}",
-        "CRIO_COMMIT={{user `CRIO_COMMIT`}}",
-        "CRIU_COMMIT={{user `CRIU_COMMIT`}}",
-        "RUNC_COMMIT={{user `RUNC_COMMIT`}}",
-        "RHSM_COMMAND={{user `RHSM_COMMAND`}}"
-      ]
-    },{
-      "type": "shell",
-      "only": ["centos-7"],
-      "script": "{{user `GOSRC`}}/{{user `PACKER_BASE`}}/centos_setup.sh",
-      "environment_vars": [
-        "SCRIPT_BASE={{user `SCRIPT_BASE`}}",
-        "CNI_COMMIT={{user `CNI_COMMIT`}}",
-        "CRIO_COMMIT={{user `CRIO_COMMIT`}}",
-        "CRIU_COMMIT={{user `CRIU_COMMIT`}}",
-        "RUNC_COMMIT={{user `RUNC_COMMIT`}}"
-      ]
-    },{
-      "type": "shell",
-      "only": ["fedora-28"],
-      "script": "{{user `GOSRC`}}/{{user `PACKER_BASE`}}/fedora_setup.sh",
-      "environment_vars": [
-        "SCRIPT_BASE={{user `SCRIPT_BASE`}}",
-        "CNI_COMMIT={{user `FEDORA_CNI_COMMIT`}}",
-        "CRIO_COMMIT={{user `CRIO_COMMIT`}}",
-        "CRIU_COMMIT={{user `CRIU_COMMIT`}}",
-        "RUNC_COMMIT={{user `RUNC_COMMIT`}}"
-      ]
-    },{
-      "type": "shell",
-      "only": ["ubuntu-18"],
-      "script": "{{user `GOSRC`}}/{{user `PACKER_BASE`}}/ubuntu_setup.sh",
-      "environment_vars": [
-        "SCRIPT_BASE={{user `SCRIPT_BASE`}}",
-        "CNI_COMMIT={{user `CNI_COMMIT`}}",
-        "CRIO_COMMIT={{user `CRIO_COMMIT`}}",
-        "CRIU_COMMIT={{user `CRIU_COMMIT`}}",
-        "RUNC_COMMIT={{user `RUNC_COMMIT`}}"
-      ]
-    }
-  ]
-}

contrib/cirrus/packer/libpod_images.yml

@@ -0,0 +1,91 @@
+---
+
+# All of these are required
+variables:
+    # Names of GCE Base images to start from, in .cirrus.yml
+    RHEL_BASE_IMAGE: '{{env `RHEL_BASE_IMAGE`}}'
+    CENTOS_BASE_IMAGE: '{{env `CENTOS_BASE_IMAGE`}}'
+    UBUNTU_BASE_IMAGE: '{{env `UBUNTU_BASE_IMAGE`}}'
+    FEDORA_BASE_IMAGE: '{{env `FEDORA_BASE_IMAGE`}}'
+    FAH_BASE_IMAGE: '{{env `FAH_BASE_IMAGE`}}'
+
+    # libpod dependencies to build and install into images
+    FEDORA_CNI_COMMIT: "{{env `FEDORA_CNI_COMMIT`}}"
+    CNI_COMMIT: "{{env `CNI_COMMIT`}}"
+    CRIO_COMMIT: "{{env `CRIO_COMMIT`}}"
+    CRIU_COMMIT: "{{env `CRIU_COMMIT`}}"
+    RUNC_COMMIT: "{{env `RUNC_COMMIT`}}"
+
+    BUILT_IMAGE_SUFFIX: '{{env `BUILT_IMAGE_SUFFIX`}}'
+    GOSRC: '{{env `GOSRC`}}'
+    PACKER_BASE: '{{env `PACKER_BASE`}}'
+    SCRIPT_BASE: '{{env `SCRIPT_BASE`}}'
+
+    # Protected credentials, decrypted by Cirrus at runtime
+    GCE_SSH_USERNAME: '{{env `GCE_SSH_USERNAME`}}'
+    GCP_PROJECT_ID: '{{env `GCP_PROJECT_ID`}}'
+    RHSM_COMMAND: '{{env `RHSM_COMMAND`}}'
+    SERVICE_ACCOUNT: '{{env `SERVICE_ACCOUNT`}}'
+    GOOGLE_APPLICATION_CREDENTIALS: '{{env `GOOGLE_APPLICATION_CREDENTIALS`}}'
+
+# Don't leak sensitive values in error messages / output
+sensitive-variables:
+    - 'GCE_SSH_USERNAME'
+    - 'GCP_PROJECT_ID'
+    - 'RHSM_COMMAND'
+    - 'SERVICE_ACCOUNT'
+
+# What images to produce in which cloud
+builders:
+    #  v----- is a YAML anchor, allows referencing this object by name (below)
+    - &gce_hosted_image
+      name: 'ubuntu-18'
+      type: 'googlecompute'
+      image_name: '{{build_name}}{{user `BUILT_IMAGE_SUFFIX`}}'
+      image_family: '{{build_name}}-libpod'
+      source_image: '{{user `UBUNTU_BASE_IMAGE`}}'
+      disk_size: 20
+      project_id: '{{user `GCP_PROJECT_ID`}}'
+      service_account_email: '{{user `SERVICE_ACCOUNT`}}'
+      communicator: 'ssh'
+      ssh_username: '{{user `GCE_SSH_USERNAME`}}'
+      ssh_pty: 'true'
+      # The only supported zone in Cirrus-CI, as of addition of this comment
+      zone: 'us-central1-a'
+
+    #      v----- is a YAML alias, allows partial re-use of the anchor object
+    - <<: *gce_hosted_image
+      name: 'rhel-7'
+      source_image: '{{user `RHEL_BASE_IMAGE`}}'
+
+    - <<: *gce_hosted_image
+      name: 'centos-7'
+      source_image: '{{user `CENTOS_BASE_IMAGE`}}'
+
+    - <<: *gce_hosted_image
+      name: 'fedora-29'
+      source_image: '{{user `FEDORA_BASE_IMAGE`}}'
+
+    - <<: *gce_hosted_image
+      name: 'fah-29'
+      source_image: '{{user `FAH_BASE_IMAGE`}}'
+
+# The brains of the operation, making actual modifications to the base-image.
+provisioners:
+    - type: 'file'
+      source: '{{user `GOSRC`}}'
+      destination: '/tmp/libpod'
+
+    - type: 'shell'
+      script: '{{user `GOSRC`}}/{{user `PACKER_BASE`}}/{{split build_name "-" 0}}_setup.sh'
+      environment_vars:
+          - 'SCRIPT_BASE={{user `SCRIPT_BASE`}}'
+          - 'CNI_COMMIT={{user `CNI_COMMIT`}}'
+          - 'FEDORA_CNI_COMMIT={{user `FEDORA_CNI_COMMIT`}}'
+          - 'CRIO_COMMIT={{user `CRIO_COMMIT`}}'
+          - 'CRIU_COMMIT={{user `CRIU_COMMIT`}}'
+          - 'RUNC_COMMIT={{user `RUNC_COMMIT`}}'
+          - 'RHSM_COMMAND={{user `RHSM_COMMAND`}}'
+
+post-processors:
+    - - type: 'manifest'

contrib/cirrus/packer/make-user-data.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# This script is utilized by Makefile, it's not intended to be run by humans
+
+cat <<EOF > user-data
+#cloud-config
+timezone: US/Eastern
+growpart:
+    mode: auto
+disable_root: false
+ssh_pwauth: True
+ssh_import_id: [root]
+ssh_authorized_keys:
+    - $(cat cidata.ssh.pub)
+users:
+   - name: root
+     primary-group: root
+     homedir: /root
+     system: true
+EOF

contrib/cirrus/packer/rhel_base-setup.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+# N/B: This script is not intended to be run by humans.  It is used to configure the
+# rhel base image for importing, so that it will boot in GCE
+
+set -e
+
+[[ "$1" == "post" ]] || exit 0  # pre stage is not needed
+
+# Load in library (copied by packer, before this script was run)
+source $GOSRC/$SCRIPT_BASE/lib.sh
+
+req_env_var "
+    RHSM_COMMAND $RHSM_COMMAND
+"
+
+install_ooe
+
+echo "Setting up repos"
+# Frequently needed
+ooe.sh sudo yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+
+# Required for google to manage ssh keys
+ooe.sh sudo tee /etc/yum.repos.d/google-cloud-sdk.repo << EOM
+[google-cloud-compute]
+name=google-cloud-compute
+baseurl=https://packages.cloud.google.com/yum/repos/google-cloud-compute-el7-x86_64
+enabled=1
+gpgcheck=1
+repo_gpgcheck=1
+gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg
+       https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
+EOM
+
+rhsm_enable
+
+echo "Installing/removing packages"
+ooe.sh sudo yum -y install google-compute-engine google-compute-engine-oslogin
+ooe.sh sudo yum -y erase "cloud-init" "rh-amazon-rhui-client*" || true
+ooe.sh sudo systemctl enable \
+    google-accounts-daemon \
+    google-clock-skew-daemon \
+    google-instance-setup \
+    google-network-daemon \
+    google-shutdown-scripts \
+    google-startup-scripts
+
+rhel_exit_handler  # release subscription!
+
+rh_finalize
+
+echo "SUCCESS!"

contrib/cirrus/packer/rhel_setup.sh

@@ -18,24 +18,7 @@ RHSM_COMMAND $RHSM_COMMAND
 
 install_ooe
 
-export GOPATH="$(mktemp -d)"
+rhsm_enable
-export RHSMCMD="$(mktemp)"
-
-exit_handler() {
-    set +ex
-    cd /
-    sudo rm -rf "$RHSMCMD"
-    sudo rm -rf "$GOPATH"
-    sudo subscription-manager remove --all
-    sudo subscription-manager unregister
-    sudo subscription-manager clean
-}
-trap "exit_handler" EXIT
-
-# Avoid logging sensitive details
-echo "$RHSM_COMMAND" > "$RHSMCMD"
-ooe.sh sudo bash "$RHSMCMD"
-sudo rm -rf "$RHSMCMD"
 
 ooe.sh sudo yum -y erase "rh-amazon-rhui-client*"
 ooe.sh sudo subscription-manager repos "--disable=*"
@@ -47,21 +30,6 @@ ooe.sh sudo subscription-manager repos \
 
 ooe.sh sudo yum -y update
 
-# Frequently needed
-ooe.sh sudo yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
-
-# Required for google to manage ssh keys
-sudo tee -a /etc/yum.repos.d/google-cloud-sdk.repo << EOM
-[google-cloud-compute]
-name=google-cloud-compute
-baseurl=https://packages.cloud.google.com/yum/repos/google-cloud-compute-el7-x86_64
-enabled=1
-gpgcheck=1
-repo_gpgcheck=1
-gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg
-       https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
-EOM
-
 ooe.sh sudo yum -y install \
     atomic-registries \
     btrfs-progs-devel \
@@ -74,8 +42,6 @@ ooe.sh sudo yum -y install \
     golang \
     golang-github-cpuguy83-go-md2man \
     golang-github-cpuguy83-go-md2man \
-    google-compute-engine \
-    google-compute-engine-oslogin \
     gpgme-devel \
     iptables \
     libassuan-devel \
@@ -118,7 +84,7 @@ install_criu
 
 install_packer_copied_files
 
-exit_handler  # release subscription!
+rhel_exit_handler  # release subscription!
 
 rh_finalize
 

contrib/cirrus/packer/ubuntu_setup.sh

@@ -21,6 +21,7 @@ install_ooe
 export GOPATH="$(mktemp -d)"
 trap "sudo rm -rf $GOPATH" EXIT
 
+# Avoid getting stuck waiting for user input
 export DEBIAN_FRONTEND=noninteractive
 
 # Try twice as workaround for minor networking problems
@@ -56,6 +57,8 @@ ooe.sh sudo -E apt-get -qq install --no-install-recommends \
     libostree-dev \
     libprotobuf-c0-dev \
     libprotobuf-dev \
+    libseccomp-dev \
+    libseccomp2 \
     libtool \
     libudev-dev \
     lsof \

contrib/cirrus/setup_environment.sh

@@ -16,12 +16,11 @@ CIRRUS_BUILD_ID $CIRRUS_BUILD_ID"
 cd "$CIRRUS_WORKING_DIR"  # for clarity of initial conditions
 
 # Verify basic dependencies
-for depbin in go rsync unzip sha256sum curl make
+for depbin in go rsync unzip sha256sum curl make python3 git
 do
     if ! type -P "$depbin" &> /dev/null
     then
-        echo "ERROR: $depbin binary not found in $PATH"
+        echo "***** WARNING: $depbin binary not found in $PATH *****"
-        exit 2
     fi
 done
 
@@ -35,14 +34,15 @@ then
     # N/B: Single-quote items evaluated every time, double-quotes only once (right now).
     for envstr in \
         "$MARK" \
+        "export EPOCH_TEST_COMMIT=\"$CIRRUS_BASE_SHA\"" \
         "export HEAD=\"$CIRRUS_CHANGE_IN_REPO\"" \
         "export TRAVIS=\"1\"" \
         "export GOSRC=\"$CIRRUS_WORKING_DIR\"" \
         "export OS_RELEASE_ID=\"$(os_release_id)\"" \
         "export OS_RELEASE_VER=\"$(os_release_ver)\"" \
-        "export OS_REL_VER=\"${OS_RELEASE_ID}-${OS_RELEASE_VER}\"" \
+        "export OS_REL_VER=\"$(os_release_id)-$(os_release_ver)\"" \
         "export BUILT_IMAGE_SUFFIX=\"-$CIRRUS_REPO_NAME-${CIRRUS_CHANGE_IN_REPO:0:8}\"" \
-        "export GOPATH=\"/go\"" \
+        "export GOPATH=\"/var/tmp/go\"" \
         'export PATH="$HOME/bin:$GOPATH/bin:/usr/local/bin:$PATH"' \
         'export LD_LIBRARY_PATH="/usr/local/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"'
     do
@@ -57,7 +57,8 @@ then
             install_runc_from_git
             envstr='export BUILDTAGS="seccomp $($GOSRC/hack/btrfs_tag.sh) $($GOSRC/hack/btrfs_installed_tag.sh) $($GOSRC/hack/ostree_tag.sh) varlink exclude_graphdriver_devicemapper"'
             ;;
-        fedora-28) ;&  # Continue to the next item
+        fedora-29) ;&  # Continue to the next item
+        fedora-28) ;&
         centos-7) ;&
         rhel-7)
             envstr='unset BUILDTAGS'  # Use default from Makefile

contrib/cirrus/system_test.sh

@@ -9,7 +9,7 @@ OS_RELEASE_ID $OS_RELEASE_ID
 OS_RELEASE_VER $OS_RELEASE_VER
 "
 
-show_env_vars
+clean_env
 
 set -x
 cd "$GOSRC"

contrib/cirrus/unit_test.sh

@@ -9,22 +9,22 @@ OS_RELEASE_ID $OS_RELEASE_ID
 OS_RELEASE_VER $OS_RELEASE_VER
 "
 
-show_env_vars
+clean_env
 
 set -x
 cd "$GOSRC"
 case "${OS_RELEASE_ID}-${OS_RELEASE_VER}" in
     ubuntu-18)
+        make install.tools "BUILDTAGS=$BUILDTAGS"
         make localunit "BUILDTAGS=$BUILDTAGS"
         make "BUILDTAGS=$BUILDTAGS"
         ;;
-    fedora-28)
+    fedora-29) ;&  # Continue to the next item
+    centos-7) ;&
+    rhel-7)
+        make install.tools
         make localunit
         make
         ;;
-    centos-7) ;&  # Continue to the next item
-    rhel-7)
-        stub 'unit testing not working on $OS_RELEASE_ID'
-        ;;
     *) bad_os_id_ver ;;
 esac

contrib/cirrus/verify_source.sh

@@ -1,30 +0,0 @@
-#!/bin/bash
-
-set -e
-source $(dirname $0)/lib.sh
-
-req_env_var "
-OS_RELEASE_ID $OS_RELEASE_ID
-OS_RELEASE_VER $OS_RELEASE_VER
-"
-
-show_env_vars
-
-set -x
-cd "$GOSRC"
-
-case "${OS_RELEASE_ID}-${OS_RELEASE_VER}" in
-    ubuntu-18)
-        make install.tools "BUILDTAGS=$BUILDTAGS"
-        make validate "BUILDTAGS=$BUILDTAGS"
-        # make lint "BUILDTAGS=$BUILDTAGS"
-        ;;
-    fedora-28) ;&
-    centos-7) ;&
-    rhel-7)
-        make install.tools
-        make validate
-        # make lint
-        ;;
-    *) bad_os_id_ver ;;
-esac