containers
/
automation-tests
mirror of https://github.com/containers/automation-tests.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2561 from cevich/cirrus_test_images 

Cirrus: Support testing of VM cache-image changes
This commit is contained in:
OpenShift Merge Robot 2019-05-21 22:08:00 +02:00 committed by GitHub
parent cec56b0075 bd0ac639bf
commit 8f43d08d96
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 562 additions and 946 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

239
.cirrus.yml
View File

@ -13,61 +13,40 @@ env:
####
#### Global variables used for all tasks
####
# File to update in home-dir with task-specific env. var values
ENVLIB: ".bash_profile"
# Overrides default location (/tmp/cirrus) for repo clone
GOPATH: "/var/tmp/go"
GOSRC: "/var/tmp/go/src/github.com/containers/libpod"
CIRRUS_WORKING_DIR: "/var/tmp/go/src/github.com/containers/libpod"
# Required so $ENVLIB gets loaded
# The default is 'sh' if unspecified
CIRRUS_SHELL: "/bin/bash"
# Save a little typing (path relative to $CIRRUS_WORKING_DIR)
SCRIPT_BASE: "./contrib/cirrus"
PACKER_BASE: "./contrib/cirrus/packer"
CIRRUS_CLONE_DEPTH: 200
# Command to prefix every output line with a timestamp
# (can't do inline awk script, Cirrus-CI or YAML mangles quoting)
TIMESTAMP: "awk --file ${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/timestamp.awk"
# Command to log critical filesystems, types, and sizes.
DFCMD: "df -lhTx tmpfs"
CIRRUS_CLONE_DEPTH: 50
####
#### Cache-image names to test with
###
ACTIVE_CACHE_IMAGE_NAMES: >-
fedora-29-libpod-548c1c05
fedora-28-libpod-548c1c05
ubuntu-18-libpod-548c1c05
rhel-7-libpod-548c1c05
image-builder-image-1541772081
FEDORA_CACHE_IMAGE_NAME: "fedora-29-libpod-548c1c05"
PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-28-libpod-548c1c05"
UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-libpod-548c1c05"
PRIOR_RHEL_CACHE_IMAGE_NAME: "rhel-7-libpod-548c1c05"
# RHEL_CACHE_IMAGE_NAME: "rhel-8-notready"
# CENTOS_CACHE_IMAGE_NAME: "centos-7-notready"
FEDORA_CACHE_IMAGE_NAME: "fedora-29-libpod-5170730531028992"
PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-28-libpod-5170730531028992"
UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-libpod-5170730531028992"
####
#### Variables for composing new cache-images (used in PR testing) from
#### base-images (pre-existing in GCE)
####
BUILT_IMAGE_SUFFIX: "-${CIRRUS_REPO_NAME}-${CIRRUS_BUILD_ID}"
# Git commits to use while building dependencies into cache-images
FEDORA_CNI_COMMIT: "412b6d31280682bb4fab4446f113c22ff1886554"
CNI_COMMIT: "7480240de9749f9a0a5c8614b17f1f03e0c06ab9"
CRIO_COMMIT: "7a283c391abb7bd25086a8ff91dbb36ebdd24466"
CONMON_COMMIT: "f02c053eb37010fc76d1e2966de7f2cb9f969ef2"
CRIU_COMMIT: "c74b83cd49c00589c0c0468ba5fe685b67fdbd0a"
RUNC_COMMIT: "029124da7af7360afa781a0234d1b083550f797c"
# CSV of cache-image names to build (see $PACKER_BASE/libpod_images.json)
PACKER_BUILDS: "ubuntu-18,fedora-29,fedora-28,rhel-7" # TODO: rhel-8,centos-7
# Version of packer to use
PACKER_VER: "1.3.2"
# Special image w/ nested-libvirt + tools for creating new cache and base images
IMAGE_BUILDER_CACHE_IMAGE_NAME: "image-builder-image-1541772081"
# Google-maintained base-image names
UBUNTU_BASE_IMAGE: "ubuntu-1804-bionic-v20181203a"
CENTOS_BASE_IMAGE: "centos-7-v20181113"
# Manually produced base-image names (see $SCRIPT_BASE/README.md)
FEDORA_BASE_IMAGE: "fedora-cloud-base-29-1-2-1541789245"
PRIOR_FEDORA_BASE_IMAGE: "fedora-cloud-base-28-1-1-1544474897"
FAH_BASE_IMAGE: "fedora-atomichost-29-20181025-1-1541787861"
# RHEL image must be imported, google bills extra for their native image.
RHEL_BASE_IMAGE: "rhel-guest-image-7-6-210-x86-64-qcow2-1548099756"
####
#### Default to NOT operating in any special-case testing mode
@ -80,8 +59,6 @@ env:
####
# Freenode IRC credentials for posting status messages
IRCID: ENCRYPTED[e87bba62a8e924dc70bdb2b66b16f6ab4a60d2870e6e5534ae9e2b0076f483c71c84091c655ca239101e6816c5ec0883]
# Command to register a RHEL VM to install/update packages
RHSM_COMMAND: ENCRYPTED[5caa5ff8c5370c3d25c7a1a28168501ab0fa2e5e3b627926f6eaba02b3fed965a7638a6151657809661f8c905c7dc187]
# Needed to build GCE images, within a GCE VM
SERVICE_ACCOUNT: ENCRYPTED[99e9a0b1c23f8dd29e83dfdf164f064cfd17afd9b895ca3b5e4c41170bd4290a8366fe2ad8e7a210b9f751711d1d002a]
# User ID for cirrus to ssh into VMs
@ -89,21 +66,16 @@ env:
# Name where this repositories cloud resources are located
GCP_PROJECT_ID: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f]
# Space separated list of environment variables to unset before testing
UNSET_ENV_VARS: >-
GCP_PROJECT_ID GCE_SSH_USERNAME SERVICE_ACCOUNT RHSM_COMMAND BUILT_IMAGE_SUFFIX
IRCID RHEL_BASE_IMAGE FAH_BASE_IMAGE FEDORA_BASE_IMAGE CENTOS_BASE_IMAGE
UBUNTU_BASE_IMAGE PACKER_VER PACKER_BUILDS RUNC_COMMIT CRIU_COMMIT
CRIO_COMMIT CNI_COMMIT FEDORA_CNI_COMMIT PACKER_BASE SCRIPT_BASE
CIRRUS_SHELL CIRRUS_WORKING_DIR ENVLIB BUILT_IMAGE_SUFFIX CIRRUS_CI
CI_NODE_INDEX CI_NODE_TOTAL CIRRUS_BASE_BRANCH CIRRUS_BASE_SHA
CIRRUS_BRANCH CIRRUS_BUILD_ID CIRRUS_CHANGE_IN_REPO CIRRUS_CLONE_DEPTH
CIRRUS_COMMIT_MESSAGE CIRRUS_CHANGE_MESSAGE CIRRUS_REPO_CLONE_HOST
CIRRUS_DEFAULT_BRANCH CIRRUS_PR CIRRUS_TAG CIRRUS_OS CIRRUS_TASK_NAME
CIRRUS_TASK_ID CIRRUS_REPO_NAME CIRRUS_REPO_OWNER CIRRUS_REPO_FULL_NAME
CIRRUS_REPO_CLONE_URL CIRRUS_SHELL CIRRUS_USER_COLLABORATOR CIRRUS_USER_PERMISSION
CIRRUS_WORKING_DIR CIRRUS_HTTP_CACHE_HOST PACKER_BUILDS BUILT_IMAGE_SUFFIX
XDG_DATA_DIRS XDG_RUNTIME_DIR XDG_SESSION_ID ROOTLESS_USER
# Default VM to use unless set or modified by task
gce_instance:
image_project: "libpod-218412"
zone: "us-central1-a" # Required by Cirrus for the time being
cpu: 2
memory: "4Gb"
disk: 200
# A matrix could be used here, for now just one VM
image_name: "${FEDORA_CACHE_IMAGE_NAME}"
# Every *_task runs in parallel in separate VMsd. The name prefix only for reference
@ -113,6 +85,7 @@ gating_task:
env:
CIRRUS_WORKING_DIR: "/usr/src/libpod"
GOPATH: "/go"
GOSRC: "/go/src/github.com/containers/libpod"
# Runs within Cirrus's "community cluster"
@ -131,7 +104,6 @@ gating_task:
- '/usr/local/bin/entrypoint.sh install.tools |& ${TIMESTAMP}'
- '/usr/local/bin/entrypoint.sh validate |& ${TIMESTAMP}'
- '/usr/local/bin/entrypoint.sh lint |& ${TIMESTAMP}'
- '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/test/test_dot_cirrus_yaml.py |& ${TIMESTAMP}'
# This task builds Podman with different buildtags to ensure the build does
# not break. It also verifies all sub-commands have man pages.
@ -157,11 +129,15 @@ gating_task:
# in sync at all times.
vendor_task:
only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*'
depends_on:
- "gating"
env:
CIRRUS_WORKING_DIR: "/usr/src/libpod"
GOPATH: "/go"
GOSRC: "/go/src/github.com/containers/libpod"
# Runs within Cirrus's "community cluster"
container:
@ -174,7 +150,7 @@ vendor_task:
vendor_script:
- '/usr/local/bin/entrypoint.sh .install.vndr |& ${TIMESTAMP}'
- '/usr/local/bin/entrypoint.sh vendor |& ${TIMESTAMP}'
- 'cd /go/src/github.com/containers/libpod && ./hack/tree_status.sh |& ${TIMESTAMP}'
- 'cd ${GOSRC} && ./hack/tree_status.sh |& ${TIMESTAMP}'
on_failure:
failed_master_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_master_failure.sh |& ${TIMESTAMP}'
@ -184,11 +160,15 @@ vendor_task:
# whether the git tree is clean.
varlink_api_task:
only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*'
depends_on:
- "gating"
env:
CIRRUS_WORKING_DIR: "/usr/src/libpod"
GOPATH: "/go"
GOSRC: "/go/src/github.com/containers/libpod"
# Used by tree_status.sh
SUGGESTION: 'remove API.md, then "make varlink_api_generate" and commit changes.'
@ -200,9 +180,9 @@ varlink_api_task:
timeout_in: 10m
vendor_script:
- '/usr/local/bin/entrypoint.sh varlink_api_generate'
- 'cd /go/src/github.com/containers/libpod && ./hack/tree_status.sh'
api_md_script:
- '/usr/local/bin/entrypoint.sh varlink_api_generate |& ${TIMESTAMP}'
- 'cd ${GOSRC} && ./hack/tree_status.sh |& ${TIMESTAMP}'
on_failure:
failed_master_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_master_failure.sh'
@ -216,7 +196,8 @@ build_each_commit_task:
- "varlink_api"
# $CIRRUS_BASE_BRANCH is only set when testing a PR
only_if: $CIRRUS_BRANCH != 'master'
only_if: $CIRRUS_BRANCH != 'master' &&
$CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*'
gce_instance:
image_project: "libpod-218412"
@ -250,7 +231,11 @@ meta_task:
env:
# Space-separated list of images used by this repository state
IMGNAMES: "${ACTIVE_CACHE_IMAGE_NAMES}"
IMGNAMES: >-
${FEDORA_CACHE_IMAGE_NAME}
${PRIOR_FEDORA_CACHE_IMAGE_NAME}
${UBUNTU_CACHE_IMAGE_NAME}
${IMAGE_BUILDER_CACHE_IMAGE_NAME}
BUILDID: "${CIRRUS_BUILD_ID}"
REPOREF: "${CIRRUS_CHANGE_IN_REPO}"
GCPJSON: ENCRYPTED[950d9c64ad78f7b1f0c7e499b42dc058d2b23aa67e38b315e68f557f2aba0bf83068d4734f7b1e1bdd22deabe99629df]
@ -258,6 +243,8 @@ meta_task:
GCPPROJECT: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f]
CIRRUS_CLONE_DEPTH: 1 # source not used
timeout_in: 10m
script: '/usr/local/bin/entrypoint.sh |& ${TIMESTAMP}'
@ -270,38 +257,28 @@ testing_task:
- "vendor"
- "build_each_commit"
env:
matrix:
TEST_REMOTE_CLIENT: true
TEST_REMOTE_CLIENT: false
# Only test build cache-images, if that's what's requested
only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*'
gce_instance:
image_project: "libpod-218412"
zone: "us-central1-a" # Required by Cirrus for the time being
cpu: 2
memory: "4Gb"
disk: 200 # see https://developers.google.com/compute/docs/disks#performance
# Generate multiple parallel tasks, covering all possible
# 'matrix' combinations.
matrix:
# Images are generated separately, from build_images_task (below)
image_name: "${FEDORA_CACHE_IMAGE_NAME}"
image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}"
image_name: "${UBUNTU_CACHE_IMAGE_NAME}"
# TODO: Make these work (also optional_testing_task below)
# image_name: "${PRIOR_RHEL_CACHE_IMAGE_NAME}"
# image_name: "${RHEL_CACHE_IMAGE_NAME}"
# image_name: "${CENTOS_CACHE_IMAGE_NAME}"
timeout_in: 120m
# Every *_script runs in sequence, for each task. The name prefix is for
# WebUI reference. The values may be strings...
env:
matrix:
TEST_REMOTE_CLIENT: true
TEST_REMOTE_CLIENT: false
setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}'
unit_test_script: '$SCRIPT_BASE/unit_test.sh |& ${TIMESTAMP}'
integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}'
ginkgo_node_logs_script: 'cat $CIRRUS_WORKING_DIR/test/e2e/ginkgo-node-*.log || echo "Ginkgo node logs not found"'
df_script: '${DFCMD}'
audit_log_script: 'cat /var/log/audit/audit.log || cat /var/log/kern.log'
journalctl_b_script: 'journalctl -b'
@ -309,6 +286,7 @@ testing_task:
failed_master_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_master_failure.sh'
# Job has already failed, don't fail again and miss collecting data
failed_ginkgo_node_logs_script: 'cat $CIRRUS_WORKING_DIR/test/e2e/ginkgo-node-*.log || echo "Ginkgo node logs not found"'
failed_df_script: '${DFCMD}'
failed_audit_log_script: 'cat /var/log/audit/audit.log || cat /var/log/kern.log || echo "Uh oh, cat audit.log failed"'
failed_journalctl_b_script: 'journalctl -b || echo "Uh oh, journalctl -b failed"'
@ -322,30 +300,25 @@ special_testing_task:
- "vendor"
- "build_each_commit"
gce_instance:
image_project: "libpod-218412"
zone: "us-central1-a" # Required by Cirrus for the time being
cpu: 2
memory: "4Gb"
disk: 200
# A matrix could be used here, for now just one VM
image_name: "${FEDORA_CACHE_IMAGE_NAME}"
only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*'
env:
matrix:
SPECIALMODE: 'rootless' # See docs
SPECIALMODE: 'in_podman' # See docs
timeout_in: 120m
timeout_in: 60m
setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}'
integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}'
df_script: '${DFCMD}'
audit_log_script: 'cat /var/log/audit/audit.log || cat /var/log/kern.log'
journalctl_b_script: 'journalctl -b'
on_failure:
failed_master_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_master_failure.sh'
# Job has already failed, don't fail again and miss collecting data
failed_df_script: '${DFCMD}'
failed_audit_log_script: 'cat /var/log/audit/audit.log || cat /var/log/kern.log || echo "Uh oh, cat audit.log failed"'
failed_journalctl_b_script: 'journalctl -b || echo "Uh oh, journalctl -b failed"'
@ -354,23 +327,22 @@ special_testing_task:
# necessary to execute them within a PR to validate changes.
optional_testing_task:
depends_on:
- "gating"
# Only run system tests in PRs (not on merge) if magic string is present
# in the PR description. Post-merge system testing is assumed to happen
# later from OS distribution's build systems.
only_if: >-
$CIRRUS_BRANCH != 'master' &&
$CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' &&
$CIRRUS_CHANGE_MESSAGE =~ '.*\*\*\*\s*CIRRUS:\s*SYSTEM\s*TEST\s*\*\*\*.*'
gce_instance:
image_project: "libpod-218412"
matrix:
image_name: "${FEDORA_CACHE_IMAGE_NAME}"
image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}"
image_name: "${UBUNTU_CACHE_IMAGE_NAME}"
# TODO: Make these work (also testing_task above)
# image_name: "${RHEL_CACHE_IMAGE_NAME}"
# image_name: "${PRIOR_RHEL_CACHE_IMAGE_NAME}"
# image_name: "${CENTOS_CACHE_IMAGE_NAME}"
timeout_in: 60m
@ -378,13 +350,85 @@ optional_testing_task:
system_test_script: '$SCRIPT_BASE/system_test.sh |& ${TIMESTAMP}'
# Test building of new cache-images for future PR testing, in this PR.
test_build_cache_images_task:
only_if: >-
$CIRRUS_BRANCH != 'master' &&
$CIRRUS_CHANGE_MESSAGE =~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' &&
$CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*SYSTEM\s*TEST\s*\*\*\*.*'
depends_on:
- "gating"
# VMs created by packer are not cleaned up by cirrus, must allow task to complete
auto_cancellation: $CI != "true"
gce_instance:
image_project: "libpod-218412"
zone: "us-central1-a"
cpu: 4
memory: "4Gb"
disk: 200
image_name: "${IMAGE_BUILDER_CACHE_IMAGE_NAME}"
scopes: # required for image building
- compute
- devstorage.full_control
environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}'
build_vm_images_script: '$SCRIPT_BASE/build_vm_images.sh |& ${TIMESTAMP}'
# Test building of new cache-images for future PR testing, in this PR.
verify_test_built_images_task:
only_if: >-
$CIRRUS_BRANCH != 'master' &&
$CIRRUS_CHANGE_MESSAGE =~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' &&
$CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*SYSTEM\s*TEST\s*\*\*\*.*'
depends_on:
- "gating"
- "test_build_cache_images"
gce_instance:
matrix:
# Images are generated separately, from build_images_task (below)
image_name: "fedora-28${BUILT_IMAGE_SUFFIX}"
image_name: "fedora-29${BUILT_IMAGE_SUFFIX}"
image_name: "ubuntu-18${BUILT_IMAGE_SUFFIX}"
env:
matrix:
TEST_REMOTE_CLIENT: true
TEST_REMOTE_CLIENT: false
# Note: A truncated form of normal testing. It only needs to confirm new images
# "probably" work. A full round of testing will happen again after $*_CACHE_IMAGE_NAME
# are updated in this or another PR (w/o '***CIRRUS: TEST IMAGES***').
environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}'
integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}'
ginkgo_node_logs_script: 'cat $CIRRUS_WORKING_DIR/test/e2e/ginkgo-node-*.log || echo "Ginkgo node logs not found"'
df_script: '${DFCMD}'
audit_log_script: 'cat /var/log/audit/audit.log || cat /var/log/kern.log'
journalctl_b_script: 'journalctl -b'
on_failure:
# Job has already failed, don't fail again and miss collecting data
failed_ginkgo_node_logs_script: 'cat $CIRRUS_WORKING_DIR/test/e2e/ginkgo-node-*.log || echo "Ginkgo node logs not found"'
failed_df_script: '${DFCMD}'
failed_audit_log_script: 'cat /var/log/audit/audit.log || cat /var/log/kern.log || echo "Uh oh, cat audit.log failed"'
failed_journalctl_b_script: 'journalctl -b || echo "Uh oh, journalctl -b failed"'
# Build new cache-images for future PR testing, but only after a PR merge.
# The cache-images save install/setup time needed test every PR. The 'active' images
# are selected by the 'image_name' items tasks above. Currently this requires
# manually updating the names, but this could be automated (see comment below).
cache_images_task:
build_cache_images_task:
# Only produce new cache-images after a PR merge, and if a magic string
# is present in the most recent commit-message.
# is present in the most recent ___commit-message___.
only_if: >-
$CIRRUS_BRANCH == 'master' &&
$CIRRUS_CHANGE_MESSAGE =~ '.*\*\*\*\s*CIRRUS:\s*REBUILD\s*IMAGES\s*\*\*\*.*'
@ -393,6 +437,7 @@ cache_images_task:
depends_on:
- "gating"
- "testing"
- "rootless_testing"
# VMs created by packer are not cleaned up by cirrus
auto_cancellation: $CI != "true"
@ -408,6 +453,7 @@ cache_images_task:
scopes:
- compute
- devstorage.full_control
environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}'
build_vm_images_script: '$SCRIPT_BASE/build_vm_images.sh |& ${TIMESTAMP}'
@ -433,19 +479,24 @@ success_task:
depends_on: # ignores any dependent task conditions
- "gating"
- "varlink_api"
- "build_each_commit"
- "vendor"
- "build_each_commit_task"
- "varlink_api"
- "testing"
- "rootless_testing_task"
- "special_testing"
- "optional_testing"
- "test_build_cache_images"
- "verify_test_built_images"
- "build_cache_images"
env:
CIRRUS_WORKING_DIR: "/usr/src/libpod"
GOPATH: "/go"
GOSRC: "/go/src/github.com/containers/libpod"
container:
image: "quay.io/libpod/gate:latest"
cpu: 1
memory: 1
success_script: '$SCRIPT_BASE/success.sh |& ${TIMESTAMP}'
success_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/success.sh |& ${TIMESTAMP}'

109
contrib/cirrus/README.md
View File

@ -99,43 +99,88 @@ contents of the ``$SPECIALMODE`` environment variable.
then execute `make localsystem` from the repository root.
### ``cache_images`` Task
### ``test_build_cache_images_task`` Task
Modifying the contents of cache-images is done by making changes to
one or more of the ``./contrib/cirrus/packer/*_setup.sh`` files. Testing
those changes currently requires adding a temporary commit to a PR that
updates ``.cirrus.yml``:
* Remove all task sections except ``cache_images_task``.
* Remove the ``only_if`` condition and ``depends_on`` dependencies
The new image names will be displayed at the end of output, assuming the build
is successful, at that point the temporary commit may be removed. Finally,
the new names may be used as ``image_name`` values in ``.cirrus.yml``.
Modifying the contents of cache-images is tested by making changes to
one or more of the ``./contrib/cirrus/packer/*_setup.sh`` files. Then
in the PR description, add the magic string: ``***CIRRUS: TEST IMAGES***``
***N/B: Steps below are performed by automation***
1. When a PR is merged (``$CIRRUS_BRANCH`` == ``master``), run another
round of the ``gating`` and ``testing`` tasks (above).
1. ``setup_environment.sh``: Same as for other tasks.
2. Assuming tests pass, if the commit message contains the magic string
``***CIRRUS: REBUILD IMAGES***``, then this task continues. Otherwise
simply mark the master branch as 'passed'.
3. ``setup_environment.sh``: Same as for other tasks.
4. ``build_vm_images.sh``: Utilize [the packer tool](http://packer.io/docs/)
2. ``build_vm_images.sh``: Utilize [the packer tool](http://packer.io/docs/)
to produce new VM images. Create a new VM from each base-image, connect
to them with ``ssh``, and perform the steps as defined by the
``$PACKER_BASE/libpod_images.json`` file:
``$PACKER_BASE/libpod_images.yml`` file:
1. On a base-image VM, as root, copy the current state of the repository
into ``/tmp/libpod``.
2. Execute distribution-specific scripts to prepare the image for
use by the ``integration_testing`` task (above). For example,
``fedora_setup.sh``.
3. If successful, shut down each VM and create a new GCE Image
named with the base image, and the commit sha of the merge.
use. For example, ``fedora_setup.sh``.
3. If successful, shut down each VM and record the names, and dates
into a json manifest file.
4. Move the manifest file, into a google storage bucket object.
This is a retained as a secondary method for tracking/auditing
creation of VM images, should it ever be needed.
### ``verify_test_built_images`` Task
Only runs following successful ``test_build_cache_images_task`` task. Uses
images following the standard naming format; ***however, only runs a limited
sub-set of automated tests***. Validating newly built images fully, requires
updating ``.cirrus.yml``.
***Manual Steps:*** Assuming `verify_test_built_images` passes, then
you'll find the new image names displayed at the end of the
`test_build_cache_images_task` in the `build_vm_images` output.
For example:
```
...cut...
==> Builds finished. The artifacts of successful builds are:
--> ubuntu-18: A disk image was created: ubuntu-18-libpod-5699523102900224
--> ubuntu-18:
--> fedora-29: A disk image was created: fedora-29-libpod-5699523102900224
--> fedora-29:
--> fedora-28: A disk image was created: fedora-28-libpod-5699523102900224
```
Now edit `.cirrus.yml`, updating the `*_IMAGE_NAME` lines to reflect the
images from above:
```yaml
env:
...cut...
####
#### Cache-image names to test with
###
FEDORA_CACHE_IMAGE_NAME: "fedora-29-libpod-5699523102900224"
PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-28-libpod-5699523102900224"
UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-libpod-5699523102900224"
...cut...
```
***NOTE:*** If re-using the same PR with new images in `.cirrus.yml`,
take care to also *update the PR description* to remove
the magic ``***CIRRUS: TEST IMAGES***`` string. Keeping it and
`--force` pushing would needlessly cause Cirrus-CI to build
and test images again.
### ``build_cache_images`` Task *(Deprecated)*
Exactly the same as ``test_build_cache_images_task`` task, but only runs on
the master branch. Requires a magic string to be in the `HEAD`
commit message: ``***CIRRUS: BUILD IMAGES***``
When successful, the manifest file along with all VM disks, are moved
into a dedicated google storage bucket, separate from the one used by
`test_build_cache_images_task`. These may be used to create new cache-images for
PR testing by manually importing them as described above.
### Base-images
@ -170,15 +215,6 @@ the ``cache_images`` Task) some input parameters are required:
or [end-user
credentials](https://cloud.google.com/docs/authentication/end-user#creating_your_client_credentials)
* ``RHEL_IMAGE_FILE`` and ``RHEL_CSUM_FILE`` complete paths
to a `rhel-server-ec2-*.raw.xz` and it's cooresponding
checksum file. These must be supplied manually because
they're not available directly via URL like other images.
* ``RHSM_COMMAND`` contains the complete string needed to register
the VM for installing package dependencies. The VM will be de-registered
upon completion.
* Optionally, CSV's may be specified to ``PACKER_BUILDS``
to limit the base-images produced. For example,
``PACKER_BUILDS=fedora,image-builder-image``.
@ -224,9 +260,6 @@ When ready, change to the ``packer`` sub-directory, and build the images:
$ cd libpod/contrib/cirrus/packer
$ make libpod_base_images GCP_PROJECT_ID=<VALUE> \
GOOGLE_APPLICATION_CREDENTIALS=<VALUE> \
RHEL_IMAGE_FILE=<VALUE> \
RHEL_CSUM_FILE=<VALUE> \
RHSM_COMMAND=<VALUE> \
PACKER_BUILDS=<OPTIONAL>
```

10
contrib/cirrus/build_vm_images.sh
View File

@ -3,7 +3,10 @@
set -e
source $(dirname $0)/lib.sh
req_env_var CNI_COMMIT CRIO_COMMIT RUNC_COMMIT PACKER_BUILDS BUILT_IMAGE_SUFFIX CENTOS_BASE_IMAGE UBUNTU_BASE_IMAGE FEDORA_BASE_IMAGE FAH_BASE_IMAGE RHEL_BASE_IMAGE RHSM_COMMAND SERVICE_ACCOUNT GCE_SSH_USERNAME GCP_PROJECT_ID PACKER_VER SCRIPT_BASE PACKER_BASE
ENV_VARS='CNI_COMMIT CONMON_COMMIT PACKER_BUILDS BUILT_IMAGE_SUFFIX UBUNTU_BASE_IMAGE FEDORA_BASE_IMAGE PRIOR_FEDORA_BASE_IMAGE SERVICE_ACCOUNT GCE_SSH_USERNAME GCP_PROJECT_ID PACKER_VER SCRIPT_BASE PACKER_BASE'
req_env_var $ENV_VARS
# Must also be made available through make, into packer process
export $ENV_VARS
show_env_vars
@ -46,7 +49,4 @@ make libpod_images \
URI="gs://packer-import${POST_MERGE_BUCKET_SUFFIX}/manifest${BUILT_IMAGE_SUFFIX}.json"
gsutil cp packer-manifest.json "$URI"
echo "Finished."
echo "Any tarball URI's referenced above at at $URI"
echo "may be used to create VM images suitable for use in"
echo ".cirrus.yml as values for the 'image_name' keys."
echo "Finished. A JSON manifest of produced images is available at $URI"

26
contrib/cirrus/integration_test.sh
View File

@ -9,7 +9,6 @@ cd "$GOSRC"
if [[ "$SPECIALMODE" == "in_podman" ]]
then
set -x
${CONTAINER_RUNTIME} run --rm --privileged --net=host \
-v $GOSRC:$GOSRC:Z \
--workdir $GOSRC \
@ -26,30 +25,19 @@ then
elif [[ "$SPECIALMODE" == "rootless" ]]
then
req_env_var ROOTLESS_USER
set -x
ssh $ROOTLESS_USER@localhost \
if [[ "$USER" == "$ROOTLESS_USER" ]]
then
$GOSRC/$SCRIPT_BASE/rootless_test.sh
else
ssh $ROOTLESS_USER@localhost \
-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no \
$GOSRC/$SCRIPT_BASE/rootless_test.sh
exit $?
fi
else
set -x
make
make install PREFIX=/usr ETCDIR=/etc
make test-binaries
make install.tools
clean_env
case "${OS_RELEASE_ID}-${OS_RELEASE_VER}" in
ubuntu-18) ;;
fedora-29) ;& # Continue to the next item
fedora-28) ;&
centos-7) ;&
rhel-7)
make podman-remote
install bin/podman-remote /usr/bin
;;
*) bad_os_id_ver ;;
esac
if [[ "$TEST_REMOTE_CLIENT" == "true" ]]
then
make remoteintegration

266
contrib/cirrus/lib.sh
View File

@ -3,34 +3,91 @@
# Library of common, shared utility functions. This file is intended
# to be sourced by other scripts, not called directly.
# Under some contexts these values are not set, make sure they are.
export USER="$(whoami)"
export HOME="$(getent passwd $USER | cut -d : -f 6)"
# Global details persist here
source /etc/environment # not always loaded under all circumstances
# These are normally set by cirrus, but can't be for VMs setup by hack/get_ci_vm.sh
# Pick some reasonable defaults
ENVLIB=${ENVLIB:-.bash_profile}
CIRRUS_WORKING_DIR="${CIRRUS_WORKING_DIR:-/var/tmp/go/src/github.com/containers/libpod}"
GOSRC="${GOSRC:-$CIRRUS_WORKING_DIR}"
# Under some contexts these values are not set, make sure they are.
USER="$(whoami)"
HOME="$(getent passwd $USER | cut -d : -f 6)"
[[ -n "$UID" ]] || UID=$(getent passwd $USER | cut -d : -f 3)
GID=$(getent passwd $USER | cut -d : -f 4)
# Essential default paths, many are overriden when executing under Cirrus-CI
export GOPATH="${GOPATH:-/var/tmp/go}"
if type -P go &> /dev/null
then
# required for go 1.12+
export GOCACHE="${GOCACHE:-$HOME/.cache/go-build}"
eval "$(go env)"
# required by make and other tools
export $(go env | cut -d '=' -f 1)
# Ensure compiled tooling is reachable
export PATH="$PATH:$GOPATH/bin"
fi
CIRRUS_WORKING_DIR="${CIRRUS_WORKING_DIR:-$GOPATH/src/github.com/containers/libpod}"
export GOSRC="${GOSRC:-$CIRRUS_WORKING_DIR}"
export PATH="$HOME/bin:$GOPATH/bin:/usr/local/bin:$PATH"
export LD_LIBRARY_PATH="/usr/local/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
TIMESTAMPS_FILEPATH="${TIMESTAMPS_FILEPATH:-/var/tmp/timestamps}"
SETUP_MARKER_FILEPATH="${SETUP_MARKER_FILEPATH:-/var/tmp/.setup_environment_sh_complete}"
# Saves typing / in case location ever moves
SCRIPT_BASE=${SCRIPT_BASE:-./contrib/cirrus}
PACKER_BASE=${PACKER_BASE:-./contrib/cirrus/packer}
CIRRUS_BUILD_ID=${CIRRUS_BUILD_ID:-DEADBEEF} # a human
CIRRUS_BASE_SHA=${CIRRUS_BASE_SHA:-HEAD}
CIRRUS_CHANGE_IN_REPO=${CIRRUS_CHANGE_IN_REPO:-FETCH_HEAD}
cd $GOSRC
if type -P git &> /dev/null
then
CIRRUS_CHANGE_IN_REPO=${CIRRUS_CHANGE_IN_REPO:-$(git show-ref --hash=8 HEAD || date +%s)}
else # pick something unique and obviously not from Cirrus
CIRRUS_CHANGE_IN_REPO=${CIRRUS_CHANGE_IN_REPO:-no_git_$(date +%s)}
fi
# Defaults when not running under CI
export CI="${CI:-false}"
CIRRUS_CI="${CIRRUS_CI:-false}"
CONTINUOUS_INTEGRATION="${CONTINUOUS_INTEGRATION:-false}"
CIRRUS_REPO_NAME=${CIRRUS_REPO_NAME:-libpod}
CIRRUS_BASE_SHA=${CIRRUS_BASE_SHA:-unknown$(date +%s)} # difficult to reliably discover
CIRRUS_BUILD_ID=${CIRRUS_BUILD_ID:-$RANDOM$(date +%s)} # must be short and unique
# Vars. for image-building
PACKER_VER="1.3.5"
# CSV of cache-image names to build (see $PACKER_BASE/libpod_images.json)
# Base-images rarely change, define them here so they're out of the way.
PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,fedora-29,fedora-28}"
# Google-maintained base-image names
UBUNTU_BASE_IMAGE="ubuntu-1804-bionic-v20181203a"
# Manually produced base-image names (see $SCRIPT_BASE/README.md)
FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1541789245"
# FEDORA_BASE_IMAGE: "fedora-cloud-base-30-1-2-1556821664"
PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-28-1-1-1544474897"
# PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1541789245"
BUILT_IMAGE_SUFFIX="${BUILT_IMAGE_SUFFIX:--$CIRRUS_REPO_NAME-${CIRRUS_BUILD_ID}}"
# Safe env. vars. to transfer from root -> $ROOTLESS_USER (go env handled separetly)
ROOTLESS_ENV_RE='(CIRRUS_.+)|(ROOTLESS_.+)|(.+_IMAGE.*)|(.+_BASE)|(.*DIRPATH)|(.*FILEPATH)|(SOURCE.*)|(DEPEND.*)|(.+_DEPS_.+)|(OS_REL.*)|(.+_ENV_RE)|(TRAVIS)|(CI.+)'
# Unsafe env. vars for display
SECRET_ENV_RE='(IRCID)|(ACCOUNT)|(^GC[EP]..+)|(SSH)'
SPECIALMODE="${SPECIALMODE:-none}"
TEST_REMOTE_CLIENT="${TEST_REMOTE_CLIENT:-false}"
export CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-podman}
if ! [[ "$PATH" =~ "/usr/local/bin" ]]
# When running as root, this may be empty or not, as a user, it MUST be set.
if [[ "$USER" == "root" ]]
then
export PATH="$PATH:/usr/local/bin"
ROOTLESS_USER="${ROOTLESS_USER:-}"
else
ROOTLESS_USER="${ROOTLESS_USER:-$USER}"
fi
# In ci/testing environment, ensure variables are always loaded
if [[ -r "$HOME/$ENVLIB" ]] && [[ -n "$CI" ]]
then
# Make sure this is always loaded
source "$HOME/$ENVLIB"
fi
# GCE image-name compatible string representation of distribution name
OS_RELEASE_ID="$(source /etc/os-release; echo $ID)"
# GCE image-name compatible string representation of distribution _major_ version
OS_RELEASE_VER="$(source /etc/os-release; echo $VERSION_ID | cut -d '.' -f 1)"
# Combined to ease soe usage
OS_REL_VER="${OS_RELEASE_ID}-${OS_RELEASE_VER}"
# Pass in a list of one or more envariable names; exit non-zero with
# helpful error message if any value is empty
@ -57,81 +114,30 @@ req_env_var() {
done
}
# Some env. vars may contain secrets. Display values for known "safe"
# and useful variables.
# ref: https://cirrus-ci.org/guide/writing-tasks/#environment-variables
show_env_vars() {
# This is almost always multi-line, print it separately
echo "export CIRRUS_CHANGE_MESSAGE=$CIRRUS_CHANGE_MESSAGE"
echo "
BUILDTAGS $BUILDTAGS
BUILT_IMAGE_SUFFIX $BUILT_IMAGE_SUFFIX
ROOTLESS_USER $ROOTLESS_USER
CI $CI
CIRRUS_CI $CIRRUS_CI
CI_NODE_INDEX $CI_NODE_INDEX
CI_NODE_TOTAL $CI_NODE_TOTAL
CONTINUOUS_INTEGRATION $CONTINUOUS_INTEGRATION
CIRRUS_BASE_BRANCH $CIRRUS_BASE_BRANCH
CIRRUS_BASE_SHA $CIRRUS_BASE_SHA
CIRRUS_BRANCH $CIRRUS_BRANCH
CIRRUS_BUILD_ID $CIRRUS_BUILD_ID
CIRRUS_CHANGE_IN_REPO $CIRRUS_CHANGE_IN_REPO
CIRRUS_CLONE_DEPTH $CIRRUS_CLONE_DEPTH
CIRRUS_DEFAULT_BRANCH $CIRRUS_DEFAULT_BRANCH
CIRRUS_PR $CIRRUS_PR
CIRRUS_TAG $CIRRUS_TAG
CIRRUS_OS $CIRRUS_OS
OS $OS
CIRRUS_TASK_NAME $CIRRUS_TASK_NAME
CIRRUS_TASK_ID $CIRRUS_TASK_ID
CIRRUS_REPO_NAME $CIRRUS_REPO_NAME
CIRRUS_REPO_OWNER $CIRRUS_REPO_OWNER
CIRRUS_REPO_FULL_NAME $CIRRUS_REPO_FULL_NAME
CIRRUS_REPO_CLONE_URL $CIRRUS_REPO_CLONE_URL
CIRRUS_SHELL $CIRRUS_SHELL
CIRRUS_USER_COLLABORATOR $CIRRUS_USER_COLLABORATOR
CIRRUS_USER_PERMISSION $CIRRUS_USER_PERMISSION
CIRRUS_WORKING_DIR $CIRRUS_WORKING_DIR
CIRRUS_HTTP_CACHE_HOST $CIRRUS_HTTP_CACHE_HOST
SPECIALMODE $SPECIALMODE
$(go env)
PACKER_BUILDS $PACKER_BUILDS
" | while read NAME VALUE
echo "Showing selection of environment variable definitions:"
_ENV_VAR_NAMES=$(awk 'BEGIN{for(v in ENVIRON) print v}' | \
egrep -v "(^PATH$)|(^BASH_FUNC)|(^[[:punct:][:space:]]+)|$SECRET_ENV_RE" | \
sort -u)
for _env_var_name in $_ENV_VAR_NAMES
do
[[ -z "$NAME" ]] || echo "export $NAME=\"$VALUE\""
# Supports older BASH versions
printf " ${_env_var_name}=%q\n" "$(printenv $_env_var_name)"
done
echo ""
echo "##### $(go version) #####"
echo ""
}
# Unset environment variables not needed for testing purposes
clean_env() {
req_env_var UNSET_ENV_VARS
echo "Unsetting $(echo $UNSET_ENV_VARS | wc -w) environment variables"
unset -v UNSET_ENV_VARS $UNSET_ENV_VARS || true # don't fail on read-only
}
die() {
echo "${2:-FATAL ERROR (but no message given!) in ${FUNCNAME[1]}()}"
echo "************************************************"
echo ">>>>> ${2:-FATAL ERROR (but no message given!) in ${FUNCNAME[1]}()}"
echo "************************************************"
exit ${1:-1}
}
# Return a GCE image-name compatible string representation of distribution name
os_release_id() {
eval "$(egrep -m 1 '^ID=' /etc/os-release | tr -d \' | tr -d \")"
echo "$ID"
}
# Return a GCE image-name compatible string representation of distribution major version
os_release_ver() {
eval "$(egrep -m 1 '^VERSION_ID=' /etc/os-release | tr -d \' | tr -d \")"
echo "$VERSION_ID" | cut -d '.' -f 1
}
bad_os_id_ver() {
echo "Unknown/Unsupported distro. $OS_RELEASE_ID and/or version $OS_RELEASE_VER for $ARGS"
echo "Unknown/Unsupported distro. $OS_RELEASE_ID and/or version $OS_RELEASE_VER for $(basename $0)"
exit 42
}
@ -140,8 +146,8 @@ stub() {
}
ircmsg() {
req_env_var CIRRUS_TASK_ID
[[ -n "$*" ]] || die 9 "ircmsg() invoked without args"
req_env_var CIRRUS_TASK_ID IRCID
[[ -n "$*" ]] || die 9 "ircmsg() invoked without message text argument"
# Sometimes setup_environment.sh didn't run
SCRIPT="$(dirname $0)/podbot.py"
NICK="podbot_$CIRRUS_TASK_ID"
@ -153,8 +159,9 @@ ircmsg() {
}
setup_rootless() {
req_env_var ROOTLESS_USER GOSRC ENVLIB
req_env_var ROOTLESS_USER GOSRC
# Only do this once
if passwd --status $ROOTLESS_USER
then
echo "Updating $ROOTLESS_USER user permissions on possibly changed libpod code"
@ -162,12 +169,7 @@ setup_rootless() {
return 0
fi
# Only do this once
cd $GOSRC
make install.catatonit
go get github.com/onsi/ginkgo/ginkgo
go get github.com/onsi/gomega/...
# Guarantee independence from specific values
ROOTLESS_UID=$[RANDOM+1000]
ROOTLESS_GID=$[RANDOM+1000]
@ -177,7 +179,8 @@ setup_rootless() {
chown -R $ROOTLESS_USER:$ROOTLESS_USER "$GOSRC"
echo "creating ssh keypair for $USER"
ssh-keygen -P "" -f $HOME/.ssh/id_rsa
[[ -r "$HOME/.ssh/id_rsa" ]] || \
ssh-keygen -P "" -f "$HOME/.ssh/id_rsa"
echo "Allowing ssh key for $ROOTLESS_USER"
(umask 077 && mkdir "/home/$ROOTLESS_USER/.ssh")
@ -192,16 +195,19 @@ setup_rootless() {
echo "${ROOTLESS_USER}:$[ROOTLESS_UID * 100]:65536" | \
tee -a /etc/subuid >> /etc/subgid
echo "Copying $HOME/$ENVLIB"
install -o $ROOTLESS_USER -g $ROOTLESS_USER -m 0700 \
"$HOME/$ENVLIB" "/home/$ROOTLESS_USER/$ENVLIB"
echo "Configuring user's go environment variables"
su --login --command 'go env' $ROOTLESS_USER | \
while read envline
do
X=$(echo "export $envline" | tee -a "/home/$ROOTLESS_USER/$ENVLIB") && echo "$X"
done
# Env. vars set by Cirrus and setup_environment.sh must be explicitly
# transfered to the test-user.
echo "Configuring rootless user's environment variables:"
echo "# Added by $GOSRC/$SCRIPT_PATH/lib.sh setup_rootless()"
_ENV_VAR_NAMES=$(awk 'BEGIN{for(v in ENVIRON) print v}' | \
egrep -v "(^PATH$)|(^BASH_FUNC)|(^[[:punct:][:space:]]+)|$SECRET_ENV_RE" | \
egrep "$ROOTLESS_ENV_RE" | \
sort -u)
for _env_var_name in $_ENV_VAR_NAMES
do
# Works with older versions of bash
printf "${_env_var_name}=%q\n" "$(printenv $_env_var_name)" >> "/home/$ROOTLESS_USER/.bashrc"
done
}
# Helper/wrapper script to only show stderr/stdout on non-zero exit
@ -239,6 +245,7 @@ install_cni_plugins() {
}
install_runc_from_git(){
req_env_var GOPATH OS_RELEASE_ID RUNC_COMMIT
wd=$(pwd)
DEST="$GOPATH/src/github.com/opencontainers/runc"
rm -rf "$DEST"
@ -246,13 +253,17 @@ install_runc_from_git(){
cd "$DEST"
ooe.sh git fetch origin --tags
ooe.sh git checkout -q "$RUNC_COMMIT"
ooe.sh make static BUILDTAGS="seccomp apparmor selinux"
if [[ "${OS_RELEASE_ID}" == "ubuntu" ]]
then
ooe.sh make static BUILDTAGS="seccomp apparmor"
else
ooe.sh make BUILDTAGS="seccomp selinux"
fi
sudo install -m 755 runc /usr/bin/runc
cd $wd
}
install_runc(){
OS_RELEASE_ID=$(os_release_id)
echo "Installing RunC from commit $RUNC_COMMIT"
echo "Platform is $OS_RELEASE_ID"
req_env_var GOPATH RUNC_COMMIT OS_RELEASE_ID
@ -298,8 +309,6 @@ install_conmon(){
}
install_criu(){
OS_RELEASE_ID=$(os_release_id)
OS_RELEASE_VER=$(os_release_ver)
echo "Installing CRIU"
echo "Installing CRIU from commit $CRIU_COMMIT"
echo "Platform is $OS_RELEASE_ID"
@ -309,21 +318,6 @@ install_criu(){
ooe.sh sudo -E add-apt-repository -y ppa:criu/ppa
ooe.sh sudo -E apt-get -qq -y update
ooe.sh sudo -E apt-get -qq -y install criu
elif [[ ( "$OS_RELEASE_ID" =~ "centos" || "$OS_RELEASE_ID" =~ "rhel" ) && "$OS_RELEASE_VER" =~ "7"* ]]; then
echo "Configuring Repositories for latest CRIU"
ooe.sh sudo tee /etc/yum.repos.d/adrian-criu-el7.repo <<EOF
[adrian-criu-el7]
name=Copr repo for criu-el7 owned by adrian
baseurl=https://copr-be.cloud.fedoraproject.org/results/adrian/criu-el7/epel-7-$basearch/
type=rpm-md
skip_if_unavailable=True
gpgcheck=1
gpgkey=https://copr-be.cloud.fedoraproject.org/results/adrian/criu-el7/pubkey.gpg
repo_gpgcheck=0
enabled=1
enabled_metadata=1
EOF
ooe.sh sudo yum -y install criu
elif [[ "$OS_RELEASE_ID" =~ "fedora" ]]; then
echo "Using CRIU from distribution"
else
@ -338,16 +332,6 @@ EOF
fi
}
install_packer_copied_files(){
# Install cni config, policy and registry config
sudo install -D -m 755 /tmp/libpod/cni/87-podman-bridge.conflist \
/etc/cni/net.d/87-podman-bridge.conflist
sudo install -D -m 755 /tmp/libpod/test/policy.json \
/etc/containers/policy.json
sudo install -D -m 755 /tmp/libpod/test/redhat_sigstore.yaml \
/etc/containers/registries.d/registry.access.redhat.com.yaml
}
install_varlink() {
echo "Installing varlink from the cheese-factory"
ooe.sh sudo -H pip3 install varlink
@ -376,7 +360,7 @@ rh_finalize(){
fi
echo "Resetting to fresh-state for usage as cloud-image."
PKG=$(type -P dnf || type -P yum || echo "")
[[ -z "$PKG" ]] || sudo $PKG clean all # not on atomic
sudo $PKG clean all
sudo rm -rf /var/cache/{yum,dnf}
sudo rm -f /etc/udev/rules.d/*-persistent-*.rules
sudo touch /.unconfigured # force firstboot to run
@ -389,25 +373,3 @@ ubuntu_finalize(){
sudo rm -rf /var/cache/apt
_finalize
}
rhel_exit_handler() {
set +ex
req_env_var GOPATH RHSMCMD
cd /
sudo rm -rf "$RHSMCMD"
sudo rm -rf "$GOPATH"
sudo subscription-manager remove --all
sudo subscription-manager unregister
sudo subscription-manager clean
}
rhsm_enable() {
req_env_var RHSM_COMMAND
export GOPATH="$(mktemp -d)"
export RHSMCMD="$(mktemp)"
trap "rhel_exit_handler" EXIT
# Avoid logging sensitive details
echo "$RHSM_COMMAND" > "$RHSMCMD"
ooe.sh sudo bash "$RHSMCMD"
sudo rm -rf "$RHSMCMD"
}

17
contrib/cirrus/lib.sh.t
View File

@ -10,12 +10,13 @@ rc=0
function check_result {
testnum=$(expr $testnum + 1)
if [ "$1" = "$2" ]; then
echo "ok $testnum $3 = $1"
MSG=$(echo "$1" | tr -d '*>\012'|sed -e 's/^ \+//')
if [ "$MSG" = "$2" ]; then
echo "ok $testnum $3 = $MSG"
else
echo "not ok $testnum $3"
echo "# expected: $2"
echo "# actual: $1"
echo "# actual: $MSG"
rc=1
fi
}
@ -64,11 +65,17 @@ test_rev '' 1 'FATAL: req_env_var: invoked without arguments'
unset FOO BAR
test_rev FOO 9 'FATAL: test_rev() requires $FOO to be non-empty'
test_rev BAR 9 'FATAL: test_rev() requires $BAR to be non-empty'
# OK if desired envariable is unset
# OK if desired envariable was unset
FOO=1
test_rev FOO 0 ''
# OK if multiple vars are non-empty
FOO="stuff"
BAR="things"
ENV_VARS="FOO BAR"
test_rev "$ENV_VARS" 0 ''
unset BAR
# ...but error if any single desired one is unset
test_rev "FOO BAR" 9 'FATAL: test_rev() requires $BAR to be non-empty'

2
contrib/cirrus/notice_master_failure.sh
View File

@ -12,7 +12,7 @@ NOR="$(echo -n -e '\x0f')"
if [[ "$CIRRUS_BRANCH" =~ "master" ]]
then
BURL="https://cirrus-ci.com/build/$CIRRUS_BUILD_ID"
ircmsg "${RED}[Action Recommended]: ${NOR}Post-merge testing ${RED}$CIRRUS_BRANCH failed${NOR} in $CIRRUS_TASK_NAME on $(os_release_id)-$(os_release_ver): $BURL. Please investigate, and re-run if appropriate."
ircmsg "${RED}[Action Recommended]: ${NOR}Post-merge testing ${RED}$CIRRUS_BRANCH failed${NOR} in $CIRRUS_TASK_NAME on $(OS_RELEASE_ID)-$(OS_RELEASE_VER): $BURL. Please investigate, and re-run if appropriate."
fi
# This script assumed to be executed on failure

19
contrib/cirrus/packer/Makefile
View File

@ -3,7 +3,7 @@
# builder name(s) from applicable YAML file,
# e.g for names see libpod_images.yml
PACKER_VER ?= 1.3.2
PACKER_VER ?= 1.3.5
GOARCH=$(shell go env GOARCH)
ARCH=$(uname -m)
PACKER_DIST_FILENAME := packer_${PACKER_VER}_linux_${GOARCH}.zip
@ -49,6 +49,7 @@ ifndef PACKER_BUILDS
$(error PACKER_BUILDS is undefined, expected builder-names CSV)
endif
./packer build -only=${PACKER_BUILDS} \
-force \
-var GOSRC=$(GOSRC) \
-var PACKER_BASE=$(PACKER_BASE) \
-var SCRIPT_BASE=$(SCRIPT_BASE) \
@ -78,16 +79,11 @@ endif
ifndef GOOGLE_APPLICATION_CREDENTIALS
$(error GOOGLE_APPLICATION_CREDENTIALS is undefined, expected absolute path to JSON file, like $HOME/.config/gcloud/legacy_credentials/*/adc.json)
endif
ifndef RHEL_IMAGE_FILE
$(error RHEL_IMAGE_FILE is undefined, expected full path to a rhel-server-ec2-*.raw.xz file)
endif
ifndef RHEL_CSUM_FILE
$(error RHEL_CSUM_FILE is undefined, expected full path to a rhel-server-ec2-*.raw.xz.SHA256SUM file)
endif
ifndef RHSM_COMMAND
$(error RHSM_COMMAND is undefined, expected string required for temporarily registering VM)
ifndef PACKER_BUILDS
$(error PACKER_BUILDS is undefined, expected builder-names CSV)
endif
PACKER_CACHE_DIR=/tmp ./packer build \
-force \
-var TIMESTAMP=$(TIMESTAMP) \
-var TTYDEV=$(TTYDEV) \
-var GCP_PROJECT_ID=$(GCP_PROJECT_ID) \
@ -95,10 +91,5 @@ endif
-var GOSRC=$(GOSRC) \
-var PACKER_BASE=$(PACKER_BASE) \
-var SCRIPT_BASE=$(SCRIPT_BASE) \
-var RHEL_BASE_IMAGE_NAME=$(shell basename $(RHEL_IMAGE_FILE) | tr -d '[[:space:]]' | sed -r -e 's/\.${ARCH}\.raw\.xz//' | tr '[[:upper:]]' '[[:lower:]]' | tr '[[:punct:]]' '-') \
-var RHEL_IMAGE_FILE=$(RHEL_IMAGE_FILE) \
-var RHEL_CSUM_FILE=$(RHEL_CSUM_FILE) \
-var 'RHSM_COMMAND=$(RHSM_COMMAND)' \
-var POST_MERGE_BUCKET_SUFFIX=$(POST_MERGE_BUCKET_SUFFIX) \
-only $(PACKER_BUILDS) \
libpod_base_images.json

82
contrib/cirrus/packer/centos_setup.sh
View File

@ -1,82 +0,0 @@
#!/bin/bash
# This script is called by packer on the subject CentOS VM, to setup the podman
# build/test environment. It's not intended to be used outside of this context.
set -e
# Load in library (copied by packer, before this script was run)
source /tmp/libpod/$SCRIPT_BASE/lib.sh
req_env_var SCRIPT_BASE CNI_COMMIT CRIO_COMMIT CRIU_COMMIT
install_ooe
export GOPATH="$(mktemp -d)"
trap "sudo rm -rf $GOPATH" EXIT
ooe.sh sudo yum -y update
ooe.sh sudo yum -y install centos-release-scl epel-release
ooe.sh sudo yum -y install \
PyYAML \
atomic-registries \
bats \
btrfs-progs-devel \
bzip2 \
device-mapper-devel \
emacs-nox \
findutils \
glib2-devel \
glibc-static \
gnupg \
golang \
golang-github-cpuguy83-go-md2man \
golang-github-cpuguy83-go-md2man \
gpgme-devel \
iptables \
libassuan-devel \
libcap-devel \
libnet \
libnet-devel \
libnl3-devel \
libseccomp-devel \
libselinux-devel \
lsof \
make \
nmap-ncat \
ostree-devel \
protobuf \
protobuf-c \
protobuf-c-devel \
protobuf-compiler \
protobuf-devel \
protobuf-python \
python \
python2-future \
python3-dateutil \
python3-psutil \
python3-pytoml \
runc \
skopeo-containers \
unzip \
vim \
which \
xz
install_scl_git
install_cni_plugins
install_buildah
install_conmon
install_criu
install_packer_copied_files
rh_finalize
echo "SUCCESS!"

45
contrib/cirrus/packer/fah_base-setup.sh
View File

@ -1,45 +0,0 @@
# N/B: This script is not intended to be run by humans. It is used to configure the
# FAH base image for importing, so that it will boot in GCE.
set -e
# Load in library (copied by packer, before this script was run)
source $GOSRC/$SCRIPT_BASE/lib.sh
install_ooe
if [[ "$1" == "pre" ]]
then
echo "Upgrading Atomic Host"
setenforce 0
ooe.sh atomic host upgrade
echo "Configuring Repositories"
ooe.sh sudo tee /etc/yum.repos.d/ngompa-gce-oslogin.repo <<EOF
[ngompa-gce-oslogin]
name=Copr repo for gce-oslogin owned by ngompa
baseurl=https://copr-be.cloud.fedoraproject.org/results/ngompa/gce-oslogin/fedora-\$releasever-\$basearch/
type=rpm-md
skip_if_unavailable=True
gpgcheck=1
gpgkey=https://copr-be.cloud.fedoraproject.org/results/ngompa/gce-oslogin/pubkey.gpg
repo_gpgcheck=0
enabled=1
enabled_metadata=1
EOF
echo "Installing necessary packages and google services"
# Google services are enabled by default, upon install.
ooe.sh rpm-ostree install rng-tools google-compute-engine google-compute-engine-oslogin
echo "Rebooting..."
systemctl reboot # Required for upgrade + package installs to be active
elif [[ "$1" == "post" ]]
then
echo "Enabling necessary services"
systemctl enable rngd # Must reboot before enabling
rh_finalize
echo "SUCCESS!"
else
echo "Expected to be called with 'pre' or 'post'"
exit 6
fi

21
contrib/cirrus/packer/fah_setup.sh
View File

@ -1,21 +0,0 @@
#!/bin/bash
# This script is called by packer on the subject fah VM, to setup the podman
# build/test environment. It's not intended to be used outside of this context.
set -e
# Load in library (copied by packer, before this script was run)
source /tmp/libpod/$SCRIPT_BASE/lib.sh
req_env_var SCRIPT_BASE
install_ooe
ooe.sh sudo atomic host upgrade
ooe.sh sudo rpm-ostree uninstall cloud-init
rh_finalize
echo "SUCCESS!"

5
contrib/cirrus/packer/fedora_base-setup.sh
View File

@ -15,9 +15,8 @@ install_ooe
echo "Updating packages"
ooe.sh dnf -y update
echo "Installing necessary packages and google services"
ooe.sh dnf -y copr enable ngompa/gce-oslogin
ooe.sh dnf -y install rng-tools google-compute-engine google-compute-engine-oslogin
echo "Installing necessary packages and google services"
ooe.sh dnf -y install rng-tools google-compute-engine-tools google-compute-engine-oslogin
echo "Enabling services"
ooe.sh systemctl enable rngd

12
contrib/cirrus/packer/fedora_setup.sh
View File

@ -8,7 +8,7 @@ set -e
# Load in library (copied by packer, before this script was run)
source /tmp/libpod/$SCRIPT_BASE/lib.sh
req_env_var SCRIPT_BASE FEDORA_CNI_COMMIT CNI_COMMIT CRIO_COMMIT CRIU_COMMIT RUNC_COMMIT
req_env_var SCRIPT_BASE FEDORA_CNI_COMMIT CNI_COMMIT CONMON_COMMIT CRIU_COMMIT
install_ooe
@ -22,6 +22,7 @@ ooe.sh sudo dnf install -y \
bats \
btrfs-progs-devel \
bzip2 \
criu \
device-mapper-devel \
emacs-nox \
findutils \
@ -35,6 +36,7 @@ ooe.sh sudo dnf install -y \
gpgme-devel \
iptables \
iproute \
jq \
libassuan-devel \
libcap-devel \
libnet \
@ -68,16 +70,14 @@ ooe.sh sudo dnf install -y \
install_varlink
install_conmon
CNI_COMMIT=$FEDORA_CNI_COMMIT
install_cni_plugins
install_buildah
install_conmon
install_criu
install_packer_copied_files
sudo /tmp/libpod/hack/install_catatonit.sh
rh_finalize # N/B: Halts system!

66
contrib/cirrus/packer/libpod_base_images.yml
View File

@ -11,24 +11,16 @@ variables:
TIMESTAMP:
# Required for output from qemu builders
TTYDEV:
# RHEL images require click-through agreements to obtain (required)
RHEL_BASE_IMAGE_NAME:
RHEL_IMAGE_FILE:
RHEL_CSUM_FILE:
# RHEL requires a subscription to install/update packages
RHSM_COMMAND:
# Latest Fedora release
FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/29/Cloud/x86_64/images/Fedora-Cloud-Base-29-1.2.x86_64.qcow2"
FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/29/Cloud/x86_64/images/Fedora-Cloud-29-1.2-x86_64-CHECKSUM"
FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-29-1-2' # Name to use in GCE
FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/30/Cloud/x86_64/images/Fedora-Cloud-Base-30-1.2.x86_64.qcow2"
FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/30/Cloud/x86_64/images/Fedora-Cloud-30-1.2-x86_64-CHECKSUM"
FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-30-1-2'
# Prior Fedora release
PRIOR_FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/28/Cloud/x86_64/images/Fedora-Cloud-Base-28-1.1.x86_64.qcow2"
PRIOR_FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/28/Cloud/x86_64/images/Fedora-Cloud-28-1.1-x86_64-CHECKSUM"
PRIOR_FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-28-1-1' # Name to use in GCE
FAH_IMAGE_URL: "https://dl.fedoraproject.org/pub/alt/atomic/stable/Fedora-Atomic-29-20181025.1/AtomicHost/x86_64/images/Fedora-AtomicHost-29-20181025.1.x86_64.qcow2"
FAH_CSUM_URL: "https://dl.fedoraproject.org/pub/alt/atomic/stable/Fedora-Atomic-29-20181025.1/AtomicHost/x86_64/images/Fedora-AtomicHost-29-20181025.1-x86_64-CHECKSUM"
FAH_BASE_IMAGE_NAME: 'fedora-atomichost-29-20181025-1' # Name to use in GCE
PRIOR_FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/29/Cloud/x86_64/images/Fedora-Cloud-Base-29-1.2.x86_64.qcow2"
PRIOR_FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/29/Cloud/x86_64/images/Fedora-Cloud-29-1.2-x86_64-CHECKSUM"
PRIOR_FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-29-1-2' # Name to use in GCE
# The name of the image in GCE used for packer build libpod_images.yml
IBI_BASE_NAME: 'image-builder-image'
@ -45,28 +37,9 @@ variables:
sensitive-variables:
- 'GOOGLE_APPLICATION_CREDENTIALS'
- 'GCP_PROJECT_ID'
- 'RHSM_COMMAND'
# What images to produce in which cloud
builders:
- name: '{{user `IBI_BASE_NAME`}}'
type: 'googlecompute'
image_name: '{{user `IBI_BASE_NAME`}}-{{user `TIMESTAMP`}}'
image_family: '{{user `IBI_BASE_NAME`}}'
source_image_project_id: 'centos-cloud'
source_image_family: 'centos-7'
project_id: '{{user `GCP_PROJECT_ID`}}'
account_file: '{{user `GOOGLE_APPLICATION_CREDENTIALS`}}'
communicator: 'ssh'
ssh_username: 'centos'
ssh_pty: 'true'
# The only supported zone in Cirrus-CI, as of addition of this comment
zone: 'us-central1-a'
# Enable nested virtualization in case it's ever needed
image_licenses:
- 'https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx'
min_cpu_platform: "Intel Broadwell" # nested-virt requirement
- &nested_virt
name: 'fedora'
type: 'qemu'
@ -109,18 +82,6 @@ builders:
iso_url: '{{user `PRIOR_FEDORA_IMAGE_URL`}}'
iso_checksum_url: '{{user `PRIOR_FEDORA_CSUM_URL`}}'
- <<: *nested_virt
name: 'fah'
iso_url: '{{user `FAH_IMAGE_URL`}}'
iso_checksum_url: '{{user `FAH_CSUM_URL`}}'
disk_size: 10240
- <<: *nested_virt
name: 'rhel'
iso_url: 'file://{{user `RHEL_IMAGE_FILE`}}'
iso_checksum_url: 'file://{{user `RHEL_CSUM_FILE`}}'
disk_size: 10240
provisioners:
- type: 'shell'
inline:
@ -150,7 +111,6 @@ provisioners:
- 'GOSRC=/tmp/libpod'
- 'SCRIPT_BASE={{user `SCRIPT_BASE`}}'
- 'PACKER_BASE={{user `PACKER_BASE`}}'
- 'RHSM_COMMAND={{user `RHSM_COMMAND`}}'
- <<: *shell_script
inline: ['{{user `GOSRC`}}/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh']
@ -161,7 +121,7 @@ provisioners:
post-processors:
- - type: "compress"
only: ['fedora', 'prior-fedora', 'fah', 'rhel']
only: ['fedora', 'prior-fedora']
output: '/tmp/{{build_name}}/disk.raw.tar.gz'
format: '.tar.gz'
compression_level: 9
@ -180,14 +140,4 @@ post-processors:
image_name: "{{user `PRIOR_FEDORA_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}"
image_description: 'Based on {{user `PRIOR_FEDORA_IMAGE_URL`}}'
image_family: '{{user `PRIOR_FEDORA_BASE_IMAGE_NAME`}}'
- <<: *gcp_import
only: ['fah']
image_name: "{{user `FAH_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}"
image_description: 'Based on {{user `FAH_IMAGE_URL`}}'
image_family: '{{user `FAH_BASE_IMAGE_NAME`}}'
- <<: *gcp_import
only: ['rhel']
image_name: "{{user `RHEL_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}"
image_description: 'Based on {{user `RHEL_IMAGE_FILE`}}'
image_family: '{{user `RHEL_BASE_IMAGE_NAME`}}'
- type: 'manifest'

28
contrib/cirrus/packer/libpod_images.yml
View File

@ -3,17 +3,14 @@
# All of these are required
variables:
# Names of GCE Base images to start from, in .cirrus.yml
RHEL_BASE_IMAGE: '{{env `RHEL_BASE_IMAGE`}}'
CENTOS_BASE_IMAGE: '{{env `CENTOS_BASE_IMAGE`}}'
UBUNTU_BASE_IMAGE: '{{env `UBUNTU_BASE_IMAGE`}}'
FEDORA_BASE_IMAGE: '{{env `FEDORA_BASE_IMAGE`}}'
PRIOR_FEDORA_BASE_IMAGE: '{{env `PRIOR_FEDORA_BASE_IMAGE`}}'
FAH_BASE_IMAGE: '{{env `FAH_BASE_IMAGE`}}'
# libpod dependencies to build and install into images
FEDORA_CNI_COMMIT: "{{env `FEDORA_CNI_COMMIT`}}"
CNI_COMMIT: "{{env `CNI_COMMIT`}}"
CRIO_COMMIT: "{{env `CRIO_COMMIT`}}"
CONMON_COMMIT: "{{env `CONMON_COMMIT`}}"
CRIU_COMMIT: "{{env `CRIU_COMMIT`}}"
RUNC_COMMIT: "{{env `RUNC_COMMIT`}}"
@ -25,7 +22,6 @@ variables:
# Protected credentials, decrypted by Cirrus at runtime
GCE_SSH_USERNAME: '{{env `GCE_SSH_USERNAME`}}'
GCP_PROJECT_ID: '{{env `GCP_PROJECT_ID`}}'
RHSM_COMMAND: '{{env `RHSM_COMMAND`}}'
SERVICE_ACCOUNT: '{{env `SERVICE_ACCOUNT`}}'
GOOGLE_APPLICATION_CREDENTIALS: '{{env `GOOGLE_APPLICATION_CREDENTIALS`}}'
@ -37,7 +33,6 @@ variables:
sensitive-variables:
- 'GCE_SSH_USERNAME'
- 'GCP_PROJECT_ID'
- 'RHSM_COMMAND'
- 'SERVICE_ACCOUNT'
# What images to produce in which cloud
@ -59,14 +54,6 @@ builders:
zone: 'us-central1-a'
# v----- is a YAML alias, allows partial re-use of the anchor object
- <<: *gce_hosted_image
name: 'rhel-7'
source_image: '{{user `RHEL_BASE_IMAGE`}}'
- <<: *gce_hosted_image
name: 'centos-7'
source_image: '{{user `CENTOS_BASE_IMAGE`}}'
- <<: *gce_hosted_image
name: 'fedora-29'
source_image: '{{user `FEDORA_BASE_IMAGE`}}'
@ -75,10 +62,6 @@ builders:
name: 'fedora-28'
source_image: '{{user `PRIOR_FEDORA_BASE_IMAGE`}}'
- <<: *gce_hosted_image
name: 'fah-29'
source_image: '{{user `FAH_BASE_IMAGE`}}'
# The brains of the operation, making actual modifications to the base-image.
provisioners:
- type: 'file'
@ -91,15 +74,10 @@ provisioners:
- 'GOSRC=/tmp/libpod'
- 'CNI_COMMIT={{user `CNI_COMMIT`}}'
- 'FEDORA_CNI_COMMIT={{user `FEDORA_CNI_COMMIT`}}'
- 'CRIO_COMMIT={{user `CRIO_COMMIT`}}'
- 'CONMON_COMMIT={{user `CONMON_COMMIT`}}'
- 'CRIU_COMMIT={{user `CRIU_COMMIT`}}'
- 'RUNC_COMMIT={{user `RUNC_COMMIT`}}'
- 'SCRIPT_BASE={{user `SCRIPT_BASE`}}'
- 'RHSM_COMMAND={{user `RHSM_COMMAND`}}'
post-processors:
# Store VM disk in GCP storage, where it will expire based on a defined
# lifecycle. This prevents GCE from filling with disused images.
- - type: 'googlecompute-export'
paths: ['gs://packer-import{{user `POST_MERGE_BUCKET_SUFFIX`}}/{{build_name}}{{user `BUILT_IMAGE_SUFFIX`}}.tar.gz']
- type: 'manifest' # writes packer-manifest.json
- - type: 'manifest' # writes packer-manifest.json

78
contrib/cirrus/packer/rhel_base-setup.sh
View File

@ -1,78 +0,0 @@
#!/bin/bash
# N/B: This script is not intended to be run by humans. It is used to configure the
# rhel base image for importing, so that it will boot in GCE
set -e
[[ "$1" == "post" ]] || exit 0 # pre stage is not needed
# Load in library (copied by packer, before this script was run)
source $GOSRC/$SCRIPT_BASE/lib.sh
req_env_var RHSM_COMMAND
install_ooe
rhsm_enable
echo "Setting up repos"
# Frequently needed
ooe.sh sudo yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
# Required for google to manage ssh keys
ooe.sh sudo tee /etc/yum.repos.d/google-cloud-sdk.repo << EOM
[google-cloud-compute]
name=google-cloud-compute
baseurl=https://packages.cloud.google.com/yum/repos/google-cloud-compute-el7-x86_64
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg
https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
EOM
echo "Updating all packages"
ooe.sh sudo yum -y update
echo "Installing/removing packages"
ooe.sh sudo yum -y install rng-tools google-compute-engine google-compute-engine-oslogin
echo "Enabling critical services"
ooe.sh sudo systemctl enable \
rngd \
google-accounts-daemon \
google-clock-skew-daemon \
google-instance-setup \
google-network-daemon \
google-shutdown-scripts \
google-startup-scripts
rhel_exit_handler # release subscription!
echo "Configuring boot"
cat << "EOF" | sudo tee /etc/default/grub
GRUB_TIMEOUT=0
GRUB_DISTRIBUTOR="$(sed 's, release .*$,,g' /etc/system-release)"
GRUB_DEFAULT=saved
GRUB_DISABLE_SUBMENU=true
GRUB_TERMINAL="serial console"
GRUB_SERIAL_COMMAND="serial --speed=38400"
GRUB_CMDLINE_LINUX="crashkernel=auto console=ttyS0,38400n8"
GRUB_DISABLE_RECOVERY="true"
EOF
sudo grub2-mkconfig -o /boot/grub2/grub.cfg
echo "Configuring networking"
ooe.sh sudo nmcli connection modify 'System eth0' 802-3-ethernet.mtu 1460
ooe.sh sudo nmcli connection modify 'System eth0' connection.autoconnect yes
ooe.sh sudo nmcli connection modify 'System eth0' connection.autoconnect-priority
ooe.sh sudo nmcli connection modify 'System eth0' ipv4.method auto
ooe.sh sudo nmcli connection modify 'System eth0' ipv4.dhcp-send-hostname yes
ooe.sh sudo nmcli connection modify 'System eth0' ipv4.dhcp-timeout 0
ooe.sh sudo nmcli connection modify 'System eth0' ipv4.never-default no
ooe.sh /usr/bin/google_instance_setup
rh_finalize
echo "SUCCESS!"

91
contrib/cirrus/packer/rhel_setup.sh
View File

@ -1,91 +0,0 @@
#!/bin/bash
# This script is called by packer on the subject CentOS VM, to setup the podman
# build/test environment. It's not intended to be used outside of this context.
set -e
# Load in library (copied by packer, before this script was run)
source /tmp/libpod/$SCRIPT_BASE/lib.sh
req_env_var SCRIPT_BASE CNI_COMMIT CRIO_COMMIT CRIU_COMMIT RHSM_COMMAND
install_ooe
rhsm_enable
ooe.sh sudo yum -y erase "rh-amazon-rhui-client*"
ooe.sh sudo subscription-manager repos "--disable=*"
ooe.sh sudo subscription-manager repos \
--enable=rhel-7-server-rpms \
--enable=rhel-7-server-optional-rpms \
--enable=rhel-7-server-extras-rpms \
--enable=rhel-server-rhscl-7-rpms
ooe.sh sudo yum -y update
ooe.sh sudo yum -y install \
PyYAML \
atomic-registries \
bats \
btrfs-progs-devel \
bzip2 \
device-mapper-devel \
emacs-nox \
findutils \
glib2-devel \
glibc-static \
gnupg \
golang \
golang-github-cpuguy83-go-md2man \
golang-github-cpuguy83-go-md2man \
gpgme-devel \
iptables \
libassuan-devel \
libcap-devel \
libnet \
libnet-devel \
libnl3-devel \
libseccomp-devel \
libselinux-devel \
lsof \
make \
nmap-ncat \
ostree-devel \
protobuf \
protobuf-c \
protobuf-c-devel \
protobuf-compiler \
protobuf-devel \
protobuf-python \
python \
python2-future \
python2-pyyaml \
python34-dateutil \
python34-psutil \
python34-pytoml \
python34-PyYAML \
runc \
skopeo-containers \
unzip \
vim \
which \
xz
install_scl_git
install_cni_plugins
install_buildah
install_conmon
install_criu
install_packer_copied_files
rhel_exit_handler # release subscription!
rh_finalize
echo "SUCCESS!"

14
contrib/cirrus/packer/ubuntu_setup.sh
View File

@ -8,7 +8,7 @@ set -e
# Load in library (copied by packer, before this script was run)
source /tmp/libpod/$SCRIPT_BASE/lib.sh
req_env_var SCRIPT_BASE CNI_COMMIT CRIO_COMMIT CRIU_COMMIT RUNC_COMMIT
req_env_var SCRIPT_BASE CNI_COMMIT CONMON_COMMIT CRIU_COMMIT
install_ooe
@ -26,6 +26,8 @@ ooe.sh sudo -E apt-get -qq install software-properties-common
# Required to have Go 1.11 on Ubuntu 18.0.4
ooe.sh sudo -E add-apt-repository --yes ppa:longsleep/golang-backports
ooe.sh sudo -E add-apt-repository --yes ppa:projectatomic/ppa
ooe.sh sudo -E add-apt-repository --yes ppa:criu/ppa
ooe.sh sudo -E apt-get -qq update || sudo -E apt-get -qq update
ooe.sh sudo -E apt-get -qq install \
@ -36,6 +38,8 @@ ooe.sh sudo -E apt-get -qq install \
bison \
btrfs-tools \
build-essential \
cri-o-runc \
criu \
curl \
e2fslibs-dev \
emacs-nox \
@ -45,6 +49,7 @@ ooe.sh sudo -E apt-get -qq install \
golang \
iproute2 \
iptables \
jq \
libaio-dev \
libapparmor-dev \
libcap-dev \
@ -89,20 +94,17 @@ ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub.d/*
ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub
ooe.sh sudo update-grub
install_runc
install_conmon
install_criu
install_cni_plugins
install_buildah
install_packer_copied_files
sudo /tmp/libpod/hack/install_catatonit.sh
install_varlink
sudo mkdir -p /etc/containers
sudo curl https://raw.githubusercontent.com/projectatomic/registries/master/registries.fedora\
-o /etc/containers/registries.conf

12
contrib/cirrus/rootless_test.sh
View File

@ -1,22 +1,26 @@
#!/bin/bash
set -e
source $HOME/.bash_profile
cd $GOSRC
source $(dirname $0)/lib.sh
req_env_var GOSRC OS_RELEASE_ID OS_RELEASE_VER
if [[ "$UID" == "0" ]]
then
echo "Error: Expected to be running as a regular user"
exit 1
fi
# Ensure environment setup correctly
req_env_var GOSRC ROOTLESS_USER
echo "."
echo "Hello, my name is $USER and I live in $PWD can I be your friend?"
echo "."
export PODMAN_VARLINK_ADDRESS=unix:/tmp/podman-$(id -u)
show_env_vars
set -x
cd "$GOSRC"
make
make varlink_generate

142
contrib/cirrus/setup_environment.sh
View File

@ -4,11 +4,18 @@ set -e
source $(dirname $0)/lib.sh
req_env_var USER HOME ENVLIB SCRIPT_BASE CIRRUS_BUILD_ID
req_env_var USER HOME GOSRC SCRIPT_BASE SETUP_MARKER_FILEPATH
[[ "$SHELL" =~ "bash" ]] || chsh -s /bin/bash
cd "$CIRRUS_WORKING_DIR" # for clarity of initial conditions
# Ensure this script only executes successfully once and always logs ending timestamp
[[ ! -e "$SETUP_MARKER_FILEPATH" ]] || exit 0
exithandler() {
RET=$?
set +e
show_env_vars
echo "$(basename $0) exit status: $RET"
[[ "$RET" -eq "0" ]] && date +%s >> "SETUP_MARKER_FILEPATH"
}
trap exithandler EXIT
# Verify basic dependencies
for depbin in go rsync unzip sha256sum curl make python3 git
@ -19,71 +26,70 @@ do
fi
done
# Setup env. vars common to all tasks/scripts/platforms and
# ensure they return for every following script execution.
MARK="# Added by $0, manual changes will be lost."
touch "$HOME/$ENVLIB"
if ! grep -q "$MARK" "$HOME/$ENVLIB"
then
cp "$HOME/$ENVLIB" "$HOME/${ENVLIB}_original"
# N/B: Single-quote items evaluated every time, double-quotes only once (right now).
for envstr in \
"$MARK" \
"export EPOCH_TEST_COMMIT=\"$CIRRUS_BASE_SHA\"" \
"export HEAD=\"$CIRRUS_CHANGE_IN_REPO\"" \
"export TRAVIS=\"1\"" \
"export GOSRC=\"$CIRRUS_WORKING_DIR\"" \
"export OS_RELEASE_ID=\"$(os_release_id)\"" \
"export OS_RELEASE_VER=\"$(os_release_ver)\"" \
"export OS_REL_VER=\"$(os_release_id)-$(os_release_ver)\"" \
"export TEST_REMOTE_CLIENT=\"$TEST_REMOTE_CLIENT\"" \
"export BUILT_IMAGE_SUFFIX=\"-$CIRRUS_REPO_NAME-${CIRRUS_CHANGE_IN_REPO:0:8}\"" \
"export GOPATH=\"/var/tmp/go\"" \
'export PATH="$HOME/bin:$GOPATH/bin:/usr/local/bin:$PATH"' \
'export LD_LIBRARY_PATH="/usr/local/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"'
do
# Make permanent in later shells, and set in current shell
X=$(echo "$envstr" | tee -a "$HOME/$ENVLIB") && eval "$X" && echo "$X"
done
# Sometimes environment setup needs to vary between distros
# Note: This should only be used for environment variables, and temporary workarounds.
# Anything externally dependent, should be made fixed-in-time by adding to
# contrib/cirrus/packer/*_setup.sh to be incorporated into VM cache-images
# (see docs).
case "${OS_REL_VER}" in
ubuntu-18) ;;
fedora-29)
# Occasionally, and seemingly only on F29 the root disk fails to expand
# upon boot. When this happens, any number of failures could occur if
# space runs out. Until there is time to investigate the actual cause,
# workaround this problem by detecting it and acting accordingly.
REMAINING=$(df /dev/sda1 | tail -1 | awk '{print $4}')
if [[ "$REMAINING" -lt "100000000" ]] # .cirrus.yml specifies 200gig
then
echo "Fixing failure to expand root filesystem"
growpart /dev/sda 1 # device guaranteed by cloud provider
resize2fs /dev/sda1 # growpart & resuze guaranteed by base-image
fi
;;
fedora-28) ;;
centos-7) # Current VM is an image-builder-image no local podman/testing
echo "No further setup required for VM image building"
exit 0
;;
*) bad_os_id_ver ;;
esac
# Some setup needs to vary between distros
case "${OS_RELEASE_ID}-${OS_RELEASE_VER}" in
ubuntu-18)
# Always install runc on Ubuntu
install_runc_from_git
;;
fedora-29)
CON_SEL="https://kojipkgs.fedoraproject.org/packages/container-selinux/2.100/1.git3b78187.fc29/noarch/container-selinux-2.100-1.git3b78187.fc29.noarch.rpm"
echo ">>>>> OVERRIDING container-selinux WITH $CON_SEL <<<<<"
dnf -y install $CON_SEL
echo ">>>>> OVERRIDING criu and selinux-policy with latest package <<<<<"
dnf -y upgrade criu selinux-policy
;& # Continue to the next item
fedora-28)
echo ">>>>> OVERRIDING source-built runc with latest package <<<<<"
dnf update -y runc
;& # Continue to the next item
centos-7) ;&
rhel-7)
;;
*) bad_os_id_ver ;;
esac
cd "${GOSRC}/"
# Reload to incorporate any changes from above
source "$SCRIPT_BASE/lib.sh"
cd "${GOSRC}/"
# Reload to incorporate any changes from above
source "$SCRIPT_BASE/lib.sh"
echo "Installing cni config, policy and registry config"
req_env_var GOSRC
sudo install -D -m 755 $GOSRC/cni/87-podman-bridge.conflist \
/etc/cni/net.d/87-podman-bridge.conflist
sudo install -D -m 755 $GOSRC/test/policy.json \
/etc/containers/policy.json
sudo install -D -m 755 $GOSRC/test/registries.conf \
/etc/containers/registries.conf
# cri-o if installed will mess with testing in non-obvious ways
rm -f /etc/cni/net.d/*cri*
case "$SPECIALMODE" in
rootless)
X=$(echo "export ROOTLESS_USER='some${RANDOM}dude'" | \
tee -a "$HOME/$ENVLIB") && eval "$X" && echo "$X"
make install.tools
case "$SPECIALMODE" in
none) ;; # Do the normal thing
rootless)
# Only do this once, even if ROOTLESS_USER (somehow) changes
if ! grep -q 'ROOTLESS_USER' /etc/environment
then
X=$(echo "export ROOTLESS_USER='${ROOTLESS_USER:-some${RANDOM}dude}'" | \
tee -a /etc/environment) && eval "$X" && echo "$X"
X=$(echo "export SPECIALMODE='${SPECIALMODE}'" | \
tee -a /etc/environment) && eval "$X" && echo "$X"
X=$(echo "export TEST_REMOTE_CLIENT='${TEST_REMOTE_CLIENT}'" | \
tee -a /etc/environment) && eval "$X" && echo "$X"
setup_rootless
;;
in_podman) # Assumed to be Fedora
dnf install -y podman buildah
$SCRIPT_BASE/setup_container_environment.sh
;;
esac
fi
show_env_vars
fi
;;
in_podman) # Assumed to be Fedora
dnf install -y podman buildah
$SCRIPT_BASE/setup_container_environment.sh
;;
*)
die 111 "Unsupported \$SPECIAL_MODE: $SPECIALMODE"
esac

11
contrib/cirrus/system_test.sh
View File

@ -5,17 +5,12 @@ source $(dirname $0)/lib.sh
req_env_var GOSRC OS_RELEASE_ID OS_RELEASE_VER
clean_env
set -x
cd "$GOSRC"
case "${OS_RELEASE_ID}-${OS_RELEASE_VER}" in
ubuntu-18) ;& # Continue to the next item
fedora-28) ;&
fedora-29) ;&
centos-7) ;&
rhel-7)
case "${OS_RELEASE_ID}" in
ubuntu) ;& # Continue to the next item
fedora)
make install.tools
make
make test-binaries

78
contrib/cirrus/test/test_dot_cirrus_yaml.py
View File

@ -1,78 +0,0 @@
#!/bin/env python3
import sys
import os
import os.path
import unittest
import warnings
import yaml
class TestCaseBase(unittest.TestCase):
SCRIPT_PATH = os.path.realpath((os.path.dirname(sys.argv[0])))
CIRRUS_WORKING_DIR = os.environ.get('CIRRUS_WORKING_DIR',
'{0}/../../../'.format(SCRIPT_PATH))
def setUp(self):
os.chdir(self.CIRRUS_WORKING_DIR)
class TestCirrusYAML(TestCaseBase):
IMAGE_NAME_SUFFIX = '_CACHE_IMAGE_NAME'
ACTIVE_IMAGES_NAME = 'ACTIVE_CACHE_IMAGE_NAMES'
def setUp(self):
TestCirrusYAML._cirrus = None
super().setUp()
@property
def cirrus(self):
if TestCirrusYAML._cirrus is None:
with warnings.catch_warnings():
warnings.filterwarnings("ignore",category=DeprecationWarning)
with open('.cirrus.yml', "r") as dot_cirrus_dot_yaml:
TestCirrusYAML._cirrus = yaml.load(dot_cirrus_dot_yaml)
return TestCirrusYAML._cirrus
def _assert_get_cache_image_names(self, env):
inames = set([key for key in env.keys()
if key.endswith(self.IMAGE_NAME_SUFFIX)])
self.assertNotEqual(inames, set())
ivalues = set([value for key, value in env.items()
if key in inames])
self.assertNotEqual(ivalues, set())
return ivalues
def _assert_get_subdct(self, key, dct):
self.assertIn(key, dct)
return dct[key]
def test_parse_yaml(self):
self.assertIsInstance(self.cirrus, dict)
def test_active_cache_image_names(self):
env = self._assert_get_subdct('env', self.cirrus)
acin = self._assert_get_subdct(self.ACTIVE_IMAGES_NAME, env)
for ivalue in self._assert_get_cache_image_names(env):
self.assertIn(ivalue, acin,
"The '{}' sub-key of 'env' should contain this among"
" its space-separated values."
"".format(self.ACTIVE_IMAGES_NAME))
def test_cache_image_names_active(self):
env = self._assert_get_subdct('env', self.cirrus)
ivalues = self._assert_get_cache_image_names(env)
for avalue in set(self._assert_get_subdct(self.ACTIVE_IMAGES_NAME, env).split()):
self.assertIn(avalue, ivalues,
"All space-separated values in the '{}' sub-key"
" of 'env' must also be used in a key with a '{}' suffix."
"".format(self.ACTIVE_IMAGES_NAME, self.IMAGE_NAME_SUFFIX))
if __name__ == '__main__':
unittest.main(failfast=True, catchbreak=True, verbosity=0)

4
contrib/cirrus/unit_test.sh
View File

@ -3,9 +3,7 @@
set -e
source $(dirname $0)/lib.sh
req_env_var GOSRC OS_RELEASE_ID OS_RELEASE_VER
clean_env
req_env_var GOSRC
set -x
cd "$GOSRC"

129
hack/get_ci_vm.sh
View File

@ -19,7 +19,6 @@ PROJECT="libpod-218412"
GOSRC="/var/tmp/go/src/github.com/containers/libpod"
GCLOUD_IMAGE=${GCLOUD_IMAGE:-quay.io/cevich/gcloud_centos:latest}
GCLOUD_SUDO=${GCLOUD_SUDO-sudo}
ROOTLESS_USER="madcowdog"
# Shared tmp directory between container and us
TMPDIR=$(mktemp -d --tmpdir $(basename $0)_tmpdir_XXXXXX)
@ -48,11 +47,15 @@ showrun() {
}
cleanup() {
RET=$?
set +e
wait
# set GCLOUD_DEBUG to leave tmpdir behind for postmortem
test -z "$GCLOUD_DEBUG" && rm -rf $TMPDIR
# Not always called from an exit handler, but should always exit when called
exit $RET
}
trap cleanup EXIT
@ -67,14 +70,15 @@ delvm() {
image_hints() {
egrep '[[:space:]]+[[:alnum:]].+_CACHE_IMAGE_NAME:[[:space:]+"[[:print:]]+"' \
"$LIBPODROOT/.cirrus.yml" | cut -d: -f 2 | tr -d '"[:blank:]' | \
grep -v 'notready' | grep -v 'image-builder' | sort -u
grep -v 'notready' | sort -u
}
show_usage() {
echo -e "\n${RED}ERROR: $1${NOR}"
echo -e "${YEL}Usage: $(basename $0) [-s | -p | -r] <image_name>${NOR}"
echo "Use -s / -p to select source or package based dependencies"
echo -e "Use -r to setup and run tests as a regular user.\n"
echo -e "${YEL}Usage: $(basename $0) [-m <SPECIALMODE>] [-u <ROOTLESS_USER> ] <image_name>${NOR}"
echo "Use -m <SPECIALMODE> with a supported value documented in contrib/cirrus/README.md."
echo "With '-m rootless' must also specify -u <ROOTLESS_USER> with name of user to create & use"
echo ""
if [[ -r ".cirrus.yml" ]]
then
echo -e "${YEL}Some possible image_name values (from .cirrus.yml):${NOR}"
@ -87,7 +91,7 @@ show_usage() {
get_env_vars() {
python -c '
import yaml
env=yaml.load(open(".cirrus.yml"))["env"]
env=yaml.load(open(".cirrus.yml"), Loader=yaml.SafeLoader)["env"]
keys=[k for k in env if "ENCRYPTED" not in str(env[k])]
for k,v in env.items():
v=str(v)
@ -99,28 +103,56 @@ for k,v in env.items():
parse_args(){
echo -e "$USAGE_WARNING"
if [[ -z "$1" ]]
if [[ "$USER" =~ "root" ]]
then
show_usage "Must specify at least one command-line parameter."
elif [[ "$1" == "-p" ]]
then
echo -e "${YEL}Hint: Use -p for package-based dependencies or -s for source-based.${NOR}"
DEPS="PACKAGE_DEPS=true SOURCE_DEPS=false"
IMAGE_NAME="$2"
show_usage "This script must be run as a regular user."
fi
elif [[ "$1" == "-s" ]]
ENVS="$(get_env_vars)"
[[ "$#" -ge "1" ]] || \
show_usage "Must specify at least one command-line parameter."
IMAGE_NAME=""
ROOTLESS_USER=""
SPECIALMODE="none"
for arg
do
if [[ "$SPECIALMODE" == "GRABNEXT" ]] && [[ "${arg:0:1}" != "-" ]]
then
SPECIALMODE="$arg"
echo -e "${YEL}Using \$SPECIALMODE=$SPECIALMODE.${NOR}"
continue
elif [[ "$ROOTLESS_USER" == "GRABNEXT" ]] && [[ "${arg:0:1}" != "-" ]]
then
ROOTLESS_USER="$arg"
echo -e "${YEL}Using \$ROOTLESS_USER=$ROOTLESS_USER.${NOR}"
continue
fi
case "$arg" in
-m)
SPECIALMODE="GRABNEXT"
;;
-u)
ROOTLESS_USER="GRABNEXT"
;;
*)
[[ "${arg:0:1}" != "-" ]] || \
show_usage "Unknown command-line option '$arg'."
[[ -z "$IMAGE_NAME" ]] || \
show_usage "Must specify exactly one image name, got '$IMAGE_NAME' and '$arg'."
IMAGE_NAME="$arg"
;;
esac
done
if [[ "$SPECIALMODE" == "GRABNEXT" ]]
then
echo -e "${RED}Using source-based dependencies.${NOR}"
DEPS="PACKAGE_DEPS=false SOURCE_DEPS=true"
IMAGE_NAME="$2"
elif [[ "$1" == "-r" ]]
show_usage "Must specify argument to -m option."
fi
if [[ "$ROOTLESS_USER" == "GRABNEXT" ]]
then
DEPS="ROOTLESS_USER=$ROOTLESS_USER"
IMAGE_NAME="$2"
else # no -s or -p
echo -e "${RED}Using package-based dependencies.${NOR}"
DEPS="$(get_env_vars)"
IMAGE_NAME="$1"
show_usage "Must specify argument to -u option."
fi
if [[ -z "$IMAGE_NAME" ]]
@ -128,15 +160,33 @@ parse_args(){
show_usage "No image-name specified."
fi
if [[ "$USER" =~ "root" ]]
if [[ "$SPECIALMODE" == "rootless" ]] && [[ -z "$ROOTLESS_USER" ]]
then
show_usage "This script must be run as a regular user."
show_usage "With '-m rootless' must also pass -u <username> of rootless user."
fi
SETUP_CMD="env $DEPS $GOSRC/contrib/cirrus/setup_environment.sh"
if echo "$IMAGE_NAME" | grep -q "image-builder-image"
then
echo -e "Creating an image-builder VM, I hope you know what you're doing.\n"
IBI_ARGS="--scopes=compute-rw,storage-rw,userinfo-email \"--min-cpu-platform=Intel Haswell\""
SSHUSER="centos"
else
unset IBI_ARGS
SSHUSER="root"
fi
ENVS="$ENVS SPECIALMODE=\"$SPECIALMODE\""
[[ -z "$ROOTLESS_USER" ]] || \
ENVS="$ENVS ROOTLESS_USER=$ROOTLESS_USER"
SETUP_CMD="env $ENVS $GOSRC/contrib/cirrus/setup_environment.sh"
VMNAME="${VMNAME:-${USER}-${IMAGE_NAME}}"
CREATE_CMD="$PGCLOUD compute instances create --zone=$ZONE --image=${IMAGE_NAME} --custom-cpu=$CPUS --custom-memory=$MEMORY --boot-disk-size=$DISK --labels=in-use-by=$USER $VMNAME"
SSH_CMD="$PGCLOUD compute ssh root@$VMNAME"
CREATE_CMD="$PGCLOUD compute instances create --zone=$ZONE --image=${IMAGE_NAME} --custom-cpu=$CPUS --custom-memory=$MEMORY --boot-disk-size=$DISK --labels=in-use-by=$USER $IBI_ARGS $VMNAME"
SSH_CMD="$PGCLOUD compute ssh $SSHUSER@$VMNAME"
CLEANUP_CMD="$PGCLOUD compute instances delete --zone $ZONE --delete-disks=all $VMNAME"
}
@ -147,7 +197,7 @@ parse_args(){
cd "$LIBPODROOT"
parse_args $@
parse_args "$@"
# Ensure mount-points and data directories exist on host as $USER. Also prevents
# permission-denied errors during cleanup() b/c `sudo podman` created mount-points
@ -188,7 +238,7 @@ showrun --background tar cjf $TMPDIR/$TARBALL --warning=no-file-changed -C $LIBP
trap delvm INT # Allow deleting VM if CTRL-C during create
# This fails if VM already exists: permit this usage to re-init
echo -e "\n${YEL}Trying to creating a VM named $VMNAME ${RED}(might take a minute/two. Errors ignored).${NOR}"
echo -e "\n${YEL}Trying to creating a VM named $VMNAME\n${RED}(might take a minute/two. Errors ignored).${NOR}"
showrun $CREATE_CMD || true # allow re-running commands below when "delete: N"
# Any subsequent failure should prompt for VM deletion
@ -208,19 +258,13 @@ then
fi
echo -e "${YEL}Got it${NOR}"
if $SSH_CMD --command "test -r /root/.bash_profile_original"
then
echo -e "\n${YEL}Resetting environment configuration${NOR}"
showrun $SSH_CMD --command "cp /root/.bash_profile_original /root/.bash_profile"
fi
echo -e "\n${YEL}Removing and re-creating $GOSRC on $VMNAME.${NOR}"
showrun $SSH_CMD --command "rm -rf $GOSRC"
showrun $SSH_CMD --command "mkdir -p $GOSRC"
echo -e "\n${YEL}Transfering tarball to $VMNAME.${NOR}"
wait
showrun $SCP_CMD $HOME/$TARBALL root@$VMNAME:/tmp/$TARBALL
showrun $SCP_CMD $HOME/$TARBALL $SSHUSER@$VMNAME:/tmp/$TARBALL
echo -e "\n${YEL}Unpacking tarball into $GOSRC on $VMNAME.${NOR}"
showrun $SSH_CMD --command "tar xjf /tmp/$TARBALL -C $GOSRC"
@ -231,9 +275,12 @@ showrun $SSH_CMD --command "rm -f /tmp/$TARBALL"
echo -e "\n${YEL}Executing environment setup${NOR}"
showrun $SSH_CMD --command "$SETUP_CMD"
echo -e "\n${YEL}Connecting to $VMNAME ${RED}(option to delete VM upon logout).${NOR}\n"
if [[ "$1" == "-r" ]]
echo -e "\n${YEL}Connecting to $VMNAME\n${RED}(option to delete VM upon logout).${NOR}\n"
if [[ -n "$ROOTLESS_USER" ]]
then
echo "Re-chowning source files after transfer"
showrun $SSH_CMD --command "chown -R $ROOTLESS_USER $GOSRC"
echo "Connecting as user $ROOTLESS_USER"
SSH_CMD="$PGCLOUD compute ssh $ROOTLESS_USER@$VMNAME"
fi
showrun $SSH_CMD -- -t "cd $GOSRC && exec env $DEPS bash -il"
showrun $SSH_CMD -- -t "cd $GOSRC && exec env $ENVS bash -il"

2
test/registries.conf
View File

@ -1,5 +1,5 @@
[registries.search]
registries = ['docker.io', 'quay.io']
registries = ['docker.io', 'quay.io', 'registry.fedoraproject.org']
[registries.insecure]
registries = []