diff --git a/.cirrus.Dockerfile b/.cirrus.Dockerfile new file mode 100644 index 00000000..b2226195 --- /dev/null +++ b/.cirrus.Dockerfile @@ -0,0 +1,15 @@ +ARG BASE_IMAGE=ubuntu:latest +FROM $BASE_IMAGE +ARG GOPATH=/var/tmp/go +ARG TEST_USER=testuser + +RUN apt-get -qq update && \ + apt-get install -y sudo docker.io libdevmapper-dev libgpgme-dev libostree-dev + +RUN adduser --shell=/bin/bash --disabled-password \ + --gecos "$TEST_USER" "$TEST_USER" && \ + mkdir -p "$GOPATH" && \ + chown -R $TEST_USER:$TEST_USER "$GOPATH" && \ + find "$GOPATH" -type d -exec chmod 2770 '{}' + +ENV PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$GOPATH/bin" +USER $TEST_USER diff --git a/.cirrus.yml b/.cirrus.yml index f442f113..f4477c91 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -1,13 +1,216 @@ -container: - image: golang:1.13 +--- +# Main collection of env. vars to set for all tasks and scripts. env: - GOPROXY: https://proxy.golang.org + #### + #### Global variables used for all tasks + #### + # Name of the ultimate destination branch for this CI run + DEST_BRANCH: "master" + # CI container image tag (c/skopeo branch name) + SKOPEO_CI_TAG: "master" + # Use GO module mirror (reason unknown, travis did it this way) + GOPROXY: https://proxy.golang.org + # Overrides default location (/tmp/cirrus) for repo clone + GOPATH: "/var/tmp/go" + GOBIN: "${GOPATH}/bin" + GOCACHE: "${GOPATH}/cache" + GOSRC: &gosrc "/var/tmp/go/src/github.com/containers/image" + CIRRUS_WORKING_DIR: *gosrc + # The default is 'sh' if unspecified + CIRRUS_SHELL: "/usr/bin/bash" + # Save a little typing (path relative to $CIRRUS_WORKING_DIR) + SCRIPT_BASE: "./contrib/cirrus" + + #### + #### Cache-image names to test with (double-quotes around names are critical) + #### + FEDORA_NAME: "fedora-33" + PRIOR_FEDORA_NAME: "fedora-32" + UBUNTU_NAME: "ubuntu-20" + PRIOR_UBUNTU_NAME: "ubuntu-19" + + # Google-cloud VM Images + IMAGE_SUFFIX: "c5402398833246208" + FEDORA_CACHE_IMAGE_NAME: "fedora-${IMAGE_SUFFIX}" + PRIOR_FEDORA_CACHE_IMAGE_NAME: "prior-fedora-${IMAGE_SUFFIX}" + UBUNTU_CACHE_IMAGE_NAME: "ubuntu-${IMAGE_SUFFIX}" + PRIOR_UBUNTU_CACHE_IMAGE_NAME: "prior-ubuntu-${IMAGE_SUFFIX}" + + # Container FQIN's (include bleeding-edge development-level container deps.) + FEDORA_CONTAINER_FQIN: "quay.io/libpod/fedora_podman:${IMAGE_SUFFIX}" + PRIOR_FEDORA_CONTAINER_FQIN: "quay.io/libpod/prior-fedora_podman:${IMAGE_SUFFIX}" + UBUNTU_CONTAINER_FQIN: "quay.io/libpod/ubuntu_podman:${IMAGE_SUFFIX}" + PRIOR_UBUNTU_CONTAINER_FQIN: "quay.io/libpod/prior-ubuntu_podman:${IMAGE_SUFFIX}" + # Automatically built on quay.io when skopeo ${SKOPEO_CI_TAG} branch changes + SKOPEO_CI_CONTAINER_FQIN: "quay.io/skopeo/ci:${SKOPEO_CI_TAG}" + + +gcp_credentials: ENCRYPTED[38c860dd789c68bd4f38b24d4fa5ddb525346f7ebe02c8bc91532d625f033cb357f9b4a22f09a8299c92bfdad7556ae5] + validate_task: - validate_script: - - apt-get update -qq - - apt-get install -qq -y libgpgme-dev libdevmapper-dev btrfs-tools libbtrfs-dev - - make tools - - make .gitvalidation - - make validate + # The git-validation tool doesn't work well on branch or tag push, + # under Cirrus-CI, due to challenges obtaining the starting commit ID. + # Only do validation for PRs. + only_if: $CIRRUS_PR != '' + # https://cirrus-ci.org/guide/docker-builder-vm/#dockerfile-as-a-ci-environment + container: + dockerfile: .cirrus.Dockerfile + # golangci-lint tool requires heavy cpu/memory or gets OOM'd + cpu: 6 + memory: 22 + docker_arguments: + BASE_IMAGE: '${UBUNTU_CONTAINER_FQIN}' + script: | + git remote update + make tools + ${GOBIN}/git-validation -q -run DCO,short-subject,dangling-whitespace -range $(git merge-base ${DEST_BRANCH:-master} HEAD)..${CIRRUS_CHANGE_IN_REPO} + make validate + + +cross_task: + container: &std_container + dockerfile: .cirrus.Dockerfile + # golangci-lint tool requires heavy cpu/memory or gets OOM'd + cpu: 2 + memory: 4 + docker_arguments: + BASE_IMAGE: '${UBUNTU_CONTAINER_FQIN}' + script: | + make cross + + +test_task: + alias: test + depends_on: + - cross + - validate + container: *std_container + matrix: + - name: "Test" + env: + BUILDTAGS: 'btrfs_noversion libdm_no_deferred_remove' + - name: "Test w/ opengpg" + env: + BUILDTAGS: 'btrfs_noversion libdm_no_deferred_remove containers_image_openpgp' + script: | + make tools + make test BUILDTAGS="$BUILDTAGS" + + +test_skopeo_task: + alias: test_skopeo + depends_on: + - cross + - validate + gce_instance: + image_project: libpod-218412 + zone: "us-central1-f" + cpu: 2 + memory: "4Gb" + # Required to be 200gig, do not modify - has i/o performance impact + # according to gcloud CLI tool warning messages. + disk: 200 + image_name: ${FEDORA_CACHE_IMAGE_NAME} + matrix: + - name: "Skopeo" + env: + BUILDTAGS: 'btrfs_noversion libdm_no_deferred_remove' + - name: "Skopeo w/ opengpg" + env: + BUILDTAGS: 'btrfs_noversion libdm_no_deferred_remove containers_image_openpgp' + env: + SKOPEO_PATH: "${GOPATH}/src/github.com/containers/skopeo" + # Normally set to an empty-string. + # Temporarily set this to a containers/skopeo PR number to test changes. + SKOPEO_PR: + # TODO: Remove test technical-debt requiring integration tests to run + # inside a container and with --privileged. This is hiding bugs! + PODMANMAKE: >- + podman run -it --rm --privileged + -e GOPATH=$GOPATH + -v $GOPATH:$GOPATH:Z + -w $SKOPEO_PATH + $SKOPEO_CI_CONTAINER_FQIN + make + setup_script: | + # This is required as part of the standard Fedora VM setup + growpart /dev/sda 1 + resize2fs /dev/sda1 + + # VM's come with the distro. skopeo pre-installed + dnf erase -y skopeo + + export "PATH=$PATH:$GOPATH/bin" + make tools + project_module=$(GO111MODULE="on" go list .) + rm -rf "${SKOPEO_PATH}" + git clone -b ${SKOPEO_CI_TAG} \ + https://github.com/containers/skopeo.git ${SKOPEO_PATH} + cd "${SKOPEO_PATH}" + if [[ -n "$SKOPEO_PR" ]] && [[ $SKOPEO_PR -gt 1000 ]]; then + git fetch origin "+refs/pull/$SKOPEO_PR/head" + git checkout FETCH_HEAD + fi + GO111MODULE="on" go mod edit -replace ${project_module}=$GOSRC + echo "root:0:1" | tee -a /etc/subuid >> /etc/subgid + # Note: We _HAVE_ to spam BUILDTAGS in this way due to how Cirrus-CI + # mangles quoted strings when performing in-line 'env' substitution. + main_script: | + cd "${SKOPEO_PATH}" + set -x + $PODMANMAKE vendor BUILDTAGS="$BUILDTAGS" + $PODMANMAKE bin/skopeo BUILDTAGS="$BUILDTAGS" + $PODMANMAKE validate-local BUILDTAGS="$BUILDTAGS" + $PODMANMAKE test-unit-local BUILDTAGS="$BUILDTAGS" + $PODMANMAKE test-integration-local BUILDTAGS="$BUILDTAGS" + # Do not run inside podman, uses containers internally. + make test-system-local BUILDTAGS="$BUILDTAGS" + + +# This task is critical. It updates the "last-used by" timestamp stored +# in metadata for all VM images. This mechanism functions in tandem with +# an out-of-band pruning operation to remove disused VM images. +meta_task: + name: "VM img. keepalive" + alias: meta + container: &smallcontainer + cpu: 2 + memory: 2 + image: quay.io/libpod/imgts:$IMAGE_SUFFIX + env: + # Space-separated list of images used by this repository state + IMGNAMES: >- + ${FEDORA_CACHE_IMAGE_NAME} + ${PRIOR_FEDORA_CACHE_IMAGE_NAME} + ${UBUNTU_CACHE_IMAGE_NAME} + ${PRIOR_UBUNTU_CACHE_IMAGE_NAME} + BUILDID: "${CIRRUS_BUILD_ID}" + REPOREF: "${CIRRUS_REPO_NAME}" + GCPJSON: ENCRYPTED[04306103eee1933f87deb8a5af6514a7e3164aa589d6079abc0451eb2360879430ed020d6e025ca64ef667138ce9d786] + GCPNAME: ENCRYPTED[574c8afac5115af72e6722d7c1d1c7f9fca7a5586f3caad45251c1745d9b82d3c012b5e2f914e19fca0de56ce2c10f5d] + GCPPROJECT: libpod-218412 + clone_script: &noop mkdir -p "$CIRRUS_WORKING_DIR" + script: /usr/local/bin/entrypoint.sh + + +# Status aggregator for all tests. This task simply ensures a defined +# set of tasks all passed, and allows confirming that based on the status +# of this task. +success_task: + name: "Total Success" + alias: success + # N/B: ALL tasks must be listed here, minus their '_task' suffix. + depends_on: + - validate + - cross + - test + - test_skopeo + - meta + container: *smallcontainer + env: + CTR_FQIN: ${FEDORA_CONTAINER_FQIN} + TEST_ENVIRON: container + clone_script: *noop + script: /bin/true diff --git a/.travis.Dockerfile b/.travis.Dockerfile deleted file mode 100644 index 0910cc03..00000000 --- a/.travis.Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM ubuntu:18.04 - -RUN apt-get -qq update && \ - apt-get install -y sudo docker.io git make btrfs-tools libdevmapper-dev libgpgme-dev libostree-dev curl - -ADD https://golang.org/dl/go1.13.13.linux-amd64.tar.gz /tmp - -RUN tar -C /usr/local -xzf /tmp/go1.13.13.linux-amd64.tar.gz && \ - rm /tmp/go1.13.13.linux-amd64.tar.gz && \ - ln -s /usr/local/go/bin/* /usr/local/bin/ diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index b5963f3a..00000000 --- a/.travis.yml +++ /dev/null @@ -1,28 +0,0 @@ ---- -language: go -sudo: required -notifications: - email: false -dist: xenial -services: - - docker -os: - - linux -before_install: - - sudo docker build -t image-test -f .travis.Dockerfile . - - sudo chown -R $(id -u):$(id -g) $HOME/gopath - -env: - - BUILDTAGS='btrfs_noversion libdm_no_deferred_remove' - - BUILDTAGS='btrfs_noversion libdm_no_deferred_remove containers_image_openpgp' - -script: > - sudo docker run --privileged -ti --rm --user $(id -u):$(id -g) - -e TRAVIS=$TRAVIS -e TRAVIS_COMMIT_RANGE=$TRAVIS_COMMIT_RANGE - -e TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST -e TRAVIS_REPO_SLUG=$TRAVIS_REPO_SLUG - -e TRAVIS_BRANCH=$TRAVIS_BRANCH -e TRAVIS_COMMIT=$TRAVIS_COMMIT - -e GOPATH=/gopath -e TRASH_CACHE=/gopath/.trashcache - -e GOCACHE=/tmp/gocache - -v /etc/passwd:/etc/passwd -v /etc/sudoers:/etc/sudoers -v /etc/sudoers.d:/etc/sudoers.d - -v /var/run:/var/run:z -v $HOME/gopath:/gopath:Z - -w /gopath/src/github.com/containers/image image-test bash -c "PATH=$PATH:/gopath/bin make cross tools test test-skopeo SUDO=sudo BUILDTAGS=\"$BUILDTAGS\"" diff --git a/Makefile b/Makefile index a54c9d1a..7a0ea7d7 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ export GOPROXY=https://proxy.golang.org # Which github repository and branch to use for testing with skopeo SKOPEO_REPO = containers/skopeo -SKOPEO_BRANCH = master +SKOPEO_BRANCH ?= master # Set SUDO=sudo to run container integration tests using sudo. SUDO = @@ -83,40 +83,19 @@ clean: test: @$(GPGME_ENV) GO111MODULE="on" go test $(BUILDFLAGS) -cover ./... -# This is not run as part of (make all), but Travis CI does run this. -# Demonstrating a working version of skopeo (possibly with modified SKOPEO_REPO/SKOPEO_BRANCH, e.g. -# make test-skopeo SKOPEO_REPO=runcom/skopeo-1 SKOPEO_BRANCH=oci-3 SUDO=sudo -# ) is a requirement before merging; note that Travis will only test -# the master branch of the upstream repo. -test-skopeo: - @echo === Testing skopeo build - @project_path=$$(pwd) && project_module=$$(GO111MODULE="on" go list .) && export GOPATH=$$(mktemp -d) && \ - skopeo_path=$${GOPATH}/src/github.com/containers/skopeo && \ - git clone -b $(SKOPEO_BRANCH) https://github.com/$(SKOPEO_REPO) $${skopeo_path} && \ - cd $${skopeo_path} && \ - GO111MODULE="on" go mod edit -replace $${project_module}=$${project_path} && \ - make vendor && \ - make BUILDTAGS="$(BUILDTAGS)" bin/skopeo test-unit-local && \ - $(SUDO) make BUILDTAGS="$(BUILDTAGS)" check && \ - rm -rf $${skopeo_path} - fmt: - @gofmt -l -s -w $(SOURCE_DIRS) + @go fmt -l -s -w $(SOURCE_DIRS) validate: lint @GO111MODULE="on" go vet ./... - @test -z "$$(gofmt -s -l . | grep -ve '^vendor' | tee /dev/stderr)" + @test -z "$$(go fmt -s -l . | grep -ve '^vendor' | tee /dev/stderr)" lint: $(GOBIN)/golangci-lint run --build-tags "$(BUILDTAGS)" -# When this is running in travis, it will only check the travis commit range +# When this is running in CI, it will only check the CI commit range .gitvalidation: @which $(GOBIN)/git-validation > /dev/null 2>/dev/null || (echo "ERROR: git-validation not found. Consider 'make clean && make tools'" && false) -ifeq ($(TRAVIS),true) - $(GOBIN)/git-validation -q -run DCO,short-subject,dangling-whitespace -else git fetch -q "https://github.com/containers/image.git" "refs/heads/master" upstream="$$(git rev-parse --verify FETCH_HEAD)" ; \ $(GOBIN)/git-validation -q -run DCO,short-subject,dangling-whitespace -range $$upstream..HEAD -endif diff --git a/README.md b/README.md index 5a671596..ca9f4317 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![GoDoc](https://godoc.org/github.com/containers/image?status.svg)](https://godoc.org/github.com/containers/image/v5) [![Build Status](https://travis-ci.org/containers/image.svg?branch=master)](https://travis-ci.org/containers/image) +[![GoDoc](https://godoc.org/github.com/containers/image?status.svg)](https://godoc.org/github.com/containers/image) [![Build Status](https://api.cirrus-ci.com/github/containers/image.svg)](https://cirrus-ci.com/github/containers/image) = `image` is a set of Go libraries aimed at working in various way with