From 61d7dedd98cb60c1025d50e627b465c033a5b465 Mon Sep 17 00:00:00 2001 From: Ali Ariff Date: Wed, 5 Aug 2020 20:14:01 +0200 Subject: [PATCH] Build ARM docker images (#4794) Build ARM docker images in the release workflow. # Changes: - Add a new env key `DOCKER_MULTIARCH` and `DOCKER_PUSH`. When set, it will build multi-arch images and push them to the registry. See https://github.com/docker/buildx/issues/59 for why it must be pushed to the registry. - Usage of `crazy-max/ghaction-docker-buildx ` is necessary as it already configured with the ability to perform cross-compilation (using QEMU) so we can just use it, instead of manually set up it. - Usage of `buildx` now make default global arguments. (See: https://docs.docker.com/engine/reference/builder/#automatic-platform-args-in-the-global-scope) # Follow-up: - Releasing the CLI binary file in ARM architecture. The docker images resulting from these changes already build in the ARM arch. Still, we need to make another adjustment like how to retrieve those binaries and to name it correctly as part of Github Release artifacts. Signed-off-by: Ali Ariff --- .github/workflows/release.yml | 34 ++++++++++++++++----------------- Dockerfile-proxy | 16 ++++++++++------ bin/_docker.sh | 36 +++++++++++++++++++++++++++++++++-- bin/fetch-proxy | 3 ++- bin/install-deps | 3 ++- cli/Dockerfile-bin | 16 ++++++++++------ cni-plugin/Dockerfile | 10 +++++++--- controller/Dockerfile | 11 +++++++---- web/Dockerfile | 13 ++++++++----- 9 files changed, 96 insertions(+), 46 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5f72f5a08..8630bd62a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -37,12 +37,22 @@ jobs: key: ${{ runner.os }}-buildx-${{ matrix.target }}-${{ env.TAG }} restore-keys: | ${{ runner.os }}-buildx-${{ matrix.target }}- - - name: Build docker images + - name: Configure gcloud + # linkerd/linkerd2-action-gcloud@v1.0.1 + uses: linkerd/linkerd2-action-gcloud@308c4df + with: + cloud_sdk_service_account_key: ${{ secrets.CLOUD_SDK_SERVICE_ACCOUNT_KEY }} + gcp_project: ${{ secrets.GCP_PROJECT }} + gcp_zone: ${{ secrets.GCP_ZONE }} + - name: Set up Docker Buildx + # crazy-max/ghaction-docker-buildx@v3.2.0 + uses: crazy-max/ghaction-docker-buildx@552c9de + - name: Build & Push Multi Arch Images env: DOCKER_TRACE: 1 - run: | - docker buildx create --driver docker-container --use - bin/docker-build-${{ matrix.target }} + DOCKER_MULTIARCH: 1 + DOCKER_PUSH: 1 + run: bin/docker-build-${{ matrix.target }} - name: Prune docker layers cache # changes generate new images while the existing ones don't get removed # so we manually do that to avoid bloating the cache @@ -54,8 +64,9 @@ jobs: env: ARCHIVES: /home/runner/archives run: | + bin/docker-pull-binaries $TAG mkdir -p $ARCHIVES - cp -r ./target/cli/windows/linkerd $ARCHIVES/linkerd-windows.exe + cp -r $PWD/target/release/linkerd2-cli-$TAG-windows.exe $ARCHIVES/linkerd-windows.exe # `with.path` values do not support environment variables yet, so an # absolute path is used here. # @@ -67,19 +78,6 @@ jobs: with: name: image-archives path: /home/runner/archives - - name: Configure gcloud - # linkerd/linkerd2-action-gcloud@v1.0.1 - uses: linkerd/linkerd2-action-gcloud@308c4df - with: - cloud_sdk_service_account_key: ${{ secrets.CLOUD_SDK_SERVICE_ACCOUNT_KEY }} - gcp_project: ${{ secrets.GCP_PROJECT }} - gcp_zone: ${{ secrets.GCP_ZONE }} - - name: Push docker images to registry - run: | - . bin/_docker.sh - docker_push "${{ matrix.target }}" "$TAG" - docker_retag "${{ matrix.target }}" "$TAG" main - docker_push "${{ matrix.target }}" main # todo: Keep in sync with `kind_integration.yml` windows_static_cli_tests: name: Static CLI tests (windows) diff --git a/Dockerfile-proxy b/Dockerfile-proxy index 5c3531461..871bacdf1 100644 --- a/Dockerfile-proxy +++ b/Dockerfile-proxy @@ -1,19 +1,22 @@ ARG RUNTIME_IMAGE=debian:buster-20200514-slim +ARG BUILDPLATFORM=linux/amd64 # Precompile key slow-to-build dependencies -FROM golang:1.14.2-alpine as go-deps +FROM --platform=$BUILDPLATFORM golang:1.14.2-alpine as go-deps WORKDIR /linkerd-build COPY go.mod go.sum ./ COPY bin/install-deps bin/ RUN go mod download -RUN ./bin/install-deps +ARG TARGETARCH +RUN ./bin/install-deps $TARGETARCH -FROM debian:buster-20200514-slim as fetch +FROM --platform=$BUILDPLATFORM $RUNTIME_IMAGE as fetch RUN apt-get update && apt-get install -y ca-certificates curl jq WORKDIR /build COPY bin/fetch-proxy bin/fetch-proxy COPY .proxy-version proxy-version -RUN (proxy=$(bin/fetch-proxy $(cat proxy-version)) && \ +ARG TARGETARCH +RUN (proxy=$(bin/fetch-proxy $(cat proxy-version) $TARGETARCH) && \ mv "$proxy" linkerd2-proxy) ## compile proxy-identity agent @@ -22,9 +25,10 @@ WORKDIR /linkerd-build COPY pkg/flags pkg/flags COPY pkg/tls pkg/tls COPY pkg/version pkg/version -RUN CGO_ENABLED=0 GOOS=linux go build -mod=readonly ./pkg/... +ARG TARGETARCH +RUN CGO_ENABLED=0 GOOS=linux GOARCH=$TARGETARCH go build -mod=readonly ./pkg/... COPY proxy-identity proxy-identity -RUN CGO_ENABLED=0 GOOS=linux go build -o /out/proxy-identity -mod=readonly -ldflags "-s -w" ./proxy-identity +RUN CGO_ENABLED=0 GOOS=linux GOARCH=$TARGETARCH go build -o /out/proxy-identity -mod=readonly -ldflags "-s -w" ./proxy-identity FROM $RUNTIME_IMAGE as runtime COPY --from=fetch /build/target/proxy/LICENSE /usr/lib/linkerd/LICENSE diff --git a/bin/_docker.sh b/bin/_docker.sh index 4bd89513a..cfd163d1b 100644 --- a/bin/_docker.sh +++ b/bin/_docker.sh @@ -20,6 +20,15 @@ export DOCKER_BUILDKIT=${DOCKER_BUILDKIT:-} # buildx cache directory. Needed if DOCKER_BUILDKIT is used export DOCKER_BUILDKIT_CACHE=${DOCKER_BUILDKIT_CACHE:-} +# When set together with DOCKER_BUILDKIT, it will build the multi-arch images. Currently DOCKER_PUSH is also required +export DOCKER_MULTIARCH=${DOCKER_MULTIARCH:-} + +# When set together with DOCKER_MULTIARCH, it will push the multi-arch images to the registry +export DOCKER_PUSH=${DOCKER_PUSH:-} + +# Default supported docker image architectures +export SUPPORTED_ARCHS=linux/amd64,linux/arm64,linux/arm/v7 + docker_repo() { repo=$1 @@ -53,10 +62,33 @@ docker_build() { if [ -n "$DOCKER_BUILDKIT_CACHE" ]; then cache_params="--cache-from type=local,src=${DOCKER_BUILDKIT_CACHE} --cache-to type=local,dest=${DOCKER_BUILDKIT_CACHE},mode=max" fi - log_debug " :; docker buildx $rootdir $cache_params --load -t $repo:$tag -f $file $*" + + output_params="--load" + if [ -n "$DOCKER_MULTIARCH" ]; then + + # Pushing multi-arch images to gcr.io with the same tag that already exists is not possible + # The issue is on gcr as pushing the same tag in docker hub works fine + # Related issues: https://github.com/eclipse/che/issues/16983, https://github.com/open-policy-agent/gatekeeper/issues/665 + if (docker buildx imagetools inspect "$repo:$tag"); then + echo "Build skipped. Image already exists" + exit 0 + fi + + output_params="--platform $SUPPORTED_ARCHS" + if [ -n "$DOCKER_PUSH" ]; then + output_params+=" --push" + else + echo "Error: env DOCKER_PUSH=1 is missing" + echo "When building the multi-arch images it is required to push the images to the registry" + echo "See https://github.com/docker/buildx/issues/59 for more details" + exit 1 + fi + fi + + log_debug " :; docker buildx $rootdir $cache_params $output_params -t $repo:$tag -f $file $*" # shellcheck disable=SC2086 docker buildx build "$rootdir" $cache_params \ - --load \ + $output_params \ -t "$repo:$tag" \ -f "$file" \ "$@" \ diff --git a/bin/fetch-proxy b/bin/fetch-proxy index 849444cd4..41fb158aa 100755 --- a/bin/fetch-proxy +++ b/bin/fetch-proxy @@ -17,7 +17,8 @@ if [ "$version" = latest ]; then fi assetbase="https://github.com/linkerd/linkerd2-proxy/releases/download/release%2F${version}" -pkgname="linkerd2-proxy-${version}-amd64" +arch=${2:-amd64} +pkgname="linkerd2-proxy-${version}-${arch}" pkgfile="${pkgname}.tar.gz" shafile="${pkgname}.txt" diff --git a/bin/install-deps b/bin/install-deps index a50a41000..3e76ee2df 100755 --- a/bin/install-deps +++ b/bin/install-deps @@ -7,9 +7,10 @@ set -eu bindir=$( cd "${0%/*}" && pwd ) rootdir=$( cd "$bindir"/.. && pwd ) +arch=${1:-amd64} cd "$rootdir" -CGO_ENABLED=0 GOOS=linux go install -mod=readonly \ +CGO_ENABLED=0 GOOS=linux GOARCH=$arch go install -mod=readonly \ github.com/golang/protobuf/jsonpb \ github.com/grpc-ecosystem/go-grpc-prometheus \ github.com/prometheus/client_golang/api \ diff --git a/cli/Dockerfile-bin b/cli/Dockerfile-bin index c2a3a3791..841d4e157 100644 --- a/cli/Dockerfile-bin +++ b/cli/Dockerfile-bin @@ -1,10 +1,13 @@ +ARG BUILDPLATFORM=linux/amd64 + # Precompile key slow-to-build dependencies -FROM golang:1.14.2-alpine as go-deps +FROM --platform=$BUILDPLATFORM golang:1.14.2-alpine as go-deps WORKDIR /linkerd-build COPY go.mod go.sum ./ COPY bin/install-deps bin/ RUN go mod download -RUN ./bin/install-deps +ARG TARGETARCH +RUN ./bin/install-deps $TARGETARCH ## compile binaries FROM go-deps as golang @@ -23,14 +26,15 @@ RUN mkdir -p /out RUN go generate -mod=readonly ./pkg/charts/static # Cache builds without version info -RUN CGO_ENABLED=0 GOOS=darwin go build -o /out/linkerd-darwin -tags prod -mod=readonly -ldflags "-s -w" ./cli -RUN CGO_ENABLED=0 GOOS=linux go build -o /out/linkerd-linux -tags prod -mod=readonly -ldflags "-s -w" ./cli +ARG TARGETARCH +RUN CGO_ENABLED=0 GOOS=darwin go build -o /out/linkerd-darwin -tags prod -mod=readonly -ldflags "-s -w" ./cli +RUN CGO_ENABLED=0 GOOS=linux GOARCH=$TARGETARCH go build -o /out/linkerd-linux -tags prod -mod=readonly -ldflags "-s -w" ./cli RUN CGO_ENABLED=0 GOOS=windows go build -o /out/linkerd-windows -tags prod -mod=readonly -ldflags "-s -w" ./cli ARG LINKERD_VERSION ENV GO_LDFLAGS="-s -w -X github.com/linkerd/linkerd2/pkg/version.Version=${LINKERD_VERSION}" -RUN CGO_ENABLED=0 GOOS=darwin go build -o /out/linkerd-darwin -tags prod -mod=readonly -ldflags "${GO_LDFLAGS}" ./cli -RUN CGO_ENABLED=0 GOOS=linux go build -o /out/linkerd-linux -tags prod -mod=readonly -ldflags "${GO_LDFLAGS}" ./cli +RUN CGO_ENABLED=0 GOOS=darwin go build -o /out/linkerd-darwin -tags prod -mod=readonly -ldflags "${GO_LDFLAGS}" ./cli +RUN CGO_ENABLED=0 GOOS=linux GOARCH=$TARGETARCH go build -o /out/linkerd-linux -tags prod -mod=readonly -ldflags "${GO_LDFLAGS}" ./cli RUN CGO_ENABLED=0 GOOS=windows go build -o /out/linkerd-windows -tags prod -mod=readonly -ldflags "${GO_LDFLAGS}" ./cli ## export without sources & dependencies diff --git a/cni-plugin/Dockerfile b/cni-plugin/Dockerfile index a2d76b2d7..f7922bb85 100644 --- a/cni-plugin/Dockerfile +++ b/cni-plugin/Dockerfile @@ -1,10 +1,13 @@ +ARG BUILDPLATFORM=linux/amd64 + # Precompile key slow-to-build dependencies -FROM golang:1.14.2-alpine as go-deps +FROM --platform=$BUILDPLATFORM golang:1.14.2-alpine as go-deps WORKDIR /linkerd-build COPY go.mod go.sum ./ COPY bin/install-deps bin/ RUN go mod download -RUN ./bin/install-deps +ARG TARGETARCH +RUN ./bin/install-deps $TARGETARCH ## compile cni-plugin utility FROM go-deps as golang @@ -12,7 +15,8 @@ WORKDIR /linkerd-build COPY pkg pkg COPY controller controller COPY cni-plugin cni-plugin -RUN CGO_ENABLED=0 GOOS=linux go build -o /go/bin/linkerd-cni -v -mod=readonly ./cni-plugin/ +ARG TARGETARCH +RUN CGO_ENABLED=0 GOOS=linux GOARCH=$TARGETARCH go build -o /go/bin/linkerd-cni -v -mod=readonly ./cni-plugin/ FROM debian:buster-20200514-slim WORKDIR /linkerd diff --git a/controller/Dockerfile b/controller/Dockerfile index dd2b70d74..afada3985 100644 --- a/controller/Dockerfile +++ b/controller/Dockerfile @@ -1,10 +1,13 @@ +ARG BUILDPLATFORM=linux/amd64 + # Precompile key slow-to-build dependencies -FROM golang:1.14.2-alpine as go-deps +FROM --platform=$BUILDPLATFORM golang:1.14.2-alpine as go-deps WORKDIR /linkerd-build COPY go.mod go.sum ./ COPY bin/install-deps bin/ RUN go mod download -RUN ./bin/install-deps +ARG TARGETARCH +RUN ./bin/install-deps $TARGETARCH ## compile controller service FROM go-deps as golang @@ -18,8 +21,8 @@ COPY charts/partials charts/partials # Generate static templates # TODO: `go generate` does not honor -mod=readonly RUN go generate -mod=readonly ./pkg/charts/static - -RUN CGO_ENABLED=0 GOOS=linux go build -o /out/controller -tags prod -mod=readonly -ldflags "-s -w" ./controller/cmd +ARG TARGETARCH +RUN CGO_ENABLED=0 GOOS=linux GOARCH=$TARGETARCH go build -o /out/controller -tags prod -mod=readonly -ldflags "-s -w" ./controller/cmd ## package runtime FROM scratch diff --git a/web/Dockerfile b/web/Dockerfile index 9b5a4eedc..04a691e9f 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -1,13 +1,16 @@ +ARG BUILDPLATFORM=linux/amd64 + # Precompile key slow-to-build dependencies -FROM golang:1.14.2-alpine as go-deps +FROM --platform=$BUILDPLATFORM golang:1.14.2-alpine as go-deps WORKDIR /linkerd-build COPY go.mod go.sum ./ COPY bin/install-deps bin/ RUN go mod download -RUN ./bin/install-deps +ARG TARGETARCH +RUN ./bin/install-deps $TARGETARCH ## bundle web assets -FROM node:14-buster as webpack-bundle +FROM --platform=$BUILDPLATFORM node:14-buster as webpack-bundle RUN curl -o- -L https://yarnpkg.com/install.sh | bash -s -- --version 1.21.1 --network-concurrency 1 ENV PATH /root/.yarn/bin:$PATH @@ -36,8 +39,8 @@ COPY web/main.go web COPY web/srv web/srv COPY controller controller COPY pkg pkg - -RUN CGO_ENABLED=0 GOOS=linux go build -mod=readonly -o web/web -ldflags "-s -w" ./web +ARG TARGETARCH +RUN CGO_ENABLED=0 GOOS=linux GOARCH=$TARGETARCH go build -mod=readonly -o web/web -ldflags "-s -w" ./web ## package it all up FROM debian:buster-20200514-slim