Build ARM docker images (#4794)

Build ARM docker images in the release workflow.

# Changes:
- Add a new env key `DOCKER_MULTIARCH` and `DOCKER_PUSH`. When set, it will build multi-arch images and push them to the registry. See https://github.com/docker/buildx/issues/59 for why it must be pushed to the registry.
- Usage of `crazy-max/ghaction-docker-buildx ` is necessary as it already configured with the ability to perform cross-compilation (using QEMU) so we can just use it, instead of manually set up it.
- Usage of `buildx` now make default global arguments. (See: https://docs.docker.com/engine/reference/builder/#automatic-platform-args-in-the-global-scope)

# Follow-up:
- Releasing the CLI binary file in ARM architecture. The docker images resulting from these changes already build in the ARM arch. Still, we need to make another adjustment like how to retrieve those binaries and to name it correctly as part of Github Release artifacts.

Signed-off-by: Ali Ariff <ali.ariff12@gmail.com>

.github/workflows/release.yml

@@ -37,12 +37,22 @@ jobs:
         key: ${{ runner.os }}-buildx-${{ matrix.target }}-${{ env.TAG }}
         restore-keys: |
           ${{ runner.os }}-buildx-${{ matrix.target }}-
-    - name: Build docker images
+    - name: Configure gcloud
+      # linkerd/linkerd2-action-gcloud@v1.0.1
+      uses: linkerd/linkerd2-action-gcloud@308c4df
+      with:
+        cloud_sdk_service_account_key: ${{ secrets.CLOUD_SDK_SERVICE_ACCOUNT_KEY }}
+        gcp_project: ${{ secrets.GCP_PROJECT }}
+        gcp_zone: ${{ secrets.GCP_ZONE }}
+    - name: Set up Docker Buildx
+      # crazy-max/ghaction-docker-buildx@v3.2.0
+      uses: crazy-max/ghaction-docker-buildx@552c9de
+    - name: Build & Push Multi Arch Images
       env:
         DOCKER_TRACE: 1
-      run: |
+        DOCKER_MULTIARCH: 1
-        docker buildx create --driver docker-container --use
+        DOCKER_PUSH: 1
-        bin/docker-build-${{ matrix.target }}
+      run: bin/docker-build-${{ matrix.target }}
     - name: Prune docker layers cache
       # changes generate new images while the existing ones don't get removed
       # so we manually do that to avoid bloating the cache
@@ -54,8 +64,9 @@ jobs:
       env:
         ARCHIVES: /home/runner/archives
       run: |
+        bin/docker-pull-binaries $TAG
         mkdir -p $ARCHIVES
-        cp -r ./target/cli/windows/linkerd $ARCHIVES/linkerd-windows.exe
+        cp -r $PWD/target/release/linkerd2-cli-$TAG-windows.exe $ARCHIVES/linkerd-windows.exe
     # `with.path` values do not support environment variables yet, so an
     # absolute path is used here.
     #
@@ -67,19 +78,6 @@ jobs:
       with:
         name: image-archives
         path: /home/runner/archives
-    - name: Configure gcloud
-      # linkerd/linkerd2-action-gcloud@v1.0.1
-      uses: linkerd/linkerd2-action-gcloud@308c4df
-      with:
-        cloud_sdk_service_account_key: ${{ secrets.CLOUD_SDK_SERVICE_ACCOUNT_KEY }}
-        gcp_project: ${{ secrets.GCP_PROJECT }}
-        gcp_zone: ${{ secrets.GCP_ZONE }}
-    - name: Push docker images to registry
-      run: |
-        . bin/_docker.sh
-        docker_push "${{ matrix.target }}" "$TAG"
-        docker_retag "${{ matrix.target }}" "$TAG" main
-        docker_push "${{ matrix.target }}" main
   # todo: Keep in sync with `kind_integration.yml`
   windows_static_cli_tests:
     name: Static CLI tests (windows)

Dockerfile-proxy

@@ -1,19 +1,22 @@
 ARG RUNTIME_IMAGE=debian:buster-20200514-slim
+ARG BUILDPLATFORM=linux/amd64
 
 # Precompile key slow-to-build dependencies
-FROM golang:1.14.2-alpine as go-deps
+FROM --platform=$BUILDPLATFORM golang:1.14.2-alpine as go-deps
 WORKDIR /linkerd-build
 COPY go.mod go.sum ./
 COPY bin/install-deps bin/
 RUN go mod download
-RUN ./bin/install-deps
+ARG TARGETARCH
+RUN ./bin/install-deps $TARGETARCH
 
-FROM debian:buster-20200514-slim as fetch
+FROM --platform=$BUILDPLATFORM $RUNTIME_IMAGE as fetch
 RUN apt-get update && apt-get install -y ca-certificates curl jq
 WORKDIR /build
 COPY bin/fetch-proxy bin/fetch-proxy
 COPY .proxy-version proxy-version
-RUN (proxy=$(bin/fetch-proxy $(cat proxy-version)) && \
+ARG TARGETARCH
+RUN (proxy=$(bin/fetch-proxy $(cat proxy-version) $TARGETARCH) && \
     mv "$proxy" linkerd2-proxy)
 
 ## compile proxy-identity agent
@@ -22,9 +25,10 @@ WORKDIR /linkerd-build
 COPY pkg/flags pkg/flags
 COPY pkg/tls pkg/tls
 COPY pkg/version pkg/version
-RUN CGO_ENABLED=0 GOOS=linux go build -mod=readonly ./pkg/...
+ARG TARGETARCH
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=$TARGETARCH go build -mod=readonly ./pkg/...
 COPY proxy-identity proxy-identity
-RUN CGO_ENABLED=0 GOOS=linux go build -o /out/proxy-identity -mod=readonly -ldflags "-s -w" ./proxy-identity
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=$TARGETARCH go build -o /out/proxy-identity -mod=readonly -ldflags "-s -w" ./proxy-identity
 
 FROM $RUNTIME_IMAGE as runtime
 COPY --from=fetch /build/target/proxy/LICENSE /usr/lib/linkerd/LICENSE

bin/_docker.sh

@@ -20,6 +20,15 @@ export DOCKER_BUILDKIT=${DOCKER_BUILDKIT:-}
 # buildx cache directory. Needed if DOCKER_BUILDKIT is used
 export DOCKER_BUILDKIT_CACHE=${DOCKER_BUILDKIT_CACHE:-}
 
+# When set together with DOCKER_BUILDKIT, it will build the multi-arch images. Currently DOCKER_PUSH is also required
+export DOCKER_MULTIARCH=${DOCKER_MULTIARCH:-}
+
+# When set together with DOCKER_MULTIARCH, it will push the multi-arch images to the registry
+export DOCKER_PUSH=${DOCKER_PUSH:-}
+
+# Default supported docker image architectures
+export SUPPORTED_ARCHS=linux/amd64,linux/arm64,linux/arm/v7
+
 docker_repo() {
     repo=$1
 
@@ -53,10 +62,33 @@ docker_build() {
       if [ -n "$DOCKER_BUILDKIT_CACHE" ]; then
         cache_params="--cache-from type=local,src=${DOCKER_BUILDKIT_CACHE} --cache-to type=local,dest=${DOCKER_BUILDKIT_CACHE},mode=max"
       fi
-      log_debug "  :; docker buildx $rootdir $cache_params --load -t $repo:$tag -f $file $*"
+
+      output_params="--load"
+      if [ -n "$DOCKER_MULTIARCH" ]; then
+
+        # Pushing multi-arch images to gcr.io with the same tag that already exists is not possible
+        # The issue is on gcr as pushing the same tag in docker hub works fine
+        # Related issues: https://github.com/eclipse/che/issues/16983, https://github.com/open-policy-agent/gatekeeper/issues/665
+        if (docker buildx imagetools inspect "$repo:$tag"); then
+          echo "Build skipped. Image already exists"
+          exit 0
+        fi
+
+        output_params="--platform $SUPPORTED_ARCHS"
+        if [ -n "$DOCKER_PUSH" ]; then
+          output_params+=" --push"
+        else
+          echo "Error: env DOCKER_PUSH=1 is missing"
+          echo "When building the multi-arch images it is required to push the images to the registry"
+          echo "See https://github.com/docker/buildx/issues/59 for more details"
+          exit 1
+        fi
+      fi
+
+      log_debug "  :; docker buildx $rootdir $cache_params $output_params -t $repo:$tag -f $file $*"
       # shellcheck disable=SC2086
       docker buildx build "$rootdir" $cache_params \
-          --load \
+          $output_params \
           -t "$repo:$tag" \
           -f "$file" \
           "$@" \

bin/fetch-proxy

@@ -17,7 +17,8 @@ if [ "$version" = latest ]; then
 fi
 
 assetbase="https://github.com/linkerd/linkerd2-proxy/releases/download/release%2F${version}"
-pkgname="linkerd2-proxy-${version}-amd64"
+arch=${2:-amd64}
+pkgname="linkerd2-proxy-${version}-${arch}"
 pkgfile="${pkgname}.tar.gz"
 shafile="${pkgname}.txt"
 

bin/install-deps

@@ -7,9 +7,10 @@ set -eu
 
 bindir=$( cd "${0%/*}" && pwd )
 rootdir=$( cd "$bindir"/.. && pwd )
+arch=${1:-amd64}
 cd "$rootdir"
 
-CGO_ENABLED=0 GOOS=linux go install -mod=readonly \
+CGO_ENABLED=0 GOOS=linux GOARCH=$arch go install -mod=readonly \
     github.com/golang/protobuf/jsonpb \
     github.com/grpc-ecosystem/go-grpc-prometheus \
     github.com/prometheus/client_golang/api \

cli/Dockerfile-bin

@@ -1,10 +1,13 @@
+ARG BUILDPLATFORM=linux/amd64
+
 # Precompile key slow-to-build dependencies
-FROM golang:1.14.2-alpine as go-deps
+FROM --platform=$BUILDPLATFORM golang:1.14.2-alpine as go-deps
 WORKDIR /linkerd-build
 COPY go.mod go.sum ./
 COPY bin/install-deps bin/
 RUN go mod download
-RUN ./bin/install-deps
+ARG TARGETARCH
+RUN ./bin/install-deps $TARGETARCH
 
 ## compile binaries
 FROM go-deps as golang
@@ -23,14 +26,15 @@ RUN mkdir -p /out
 RUN go generate -mod=readonly ./pkg/charts/static
 
 # Cache builds without version info
+ARG TARGETARCH
 RUN CGO_ENABLED=0 GOOS=darwin go build -o /out/linkerd-darwin -tags prod -mod=readonly -ldflags "-s -w" ./cli
-RUN CGO_ENABLED=0 GOOS=linux   go build -o /out/linkerd-linux   -tags prod -mod=readonly -ldflags "-s -w" ./cli
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=$TARGETARCH go build -o /out/linkerd-linux -tags prod -mod=readonly -ldflags "-s -w" ./cli
 RUN CGO_ENABLED=0 GOOS=windows go build -o /out/linkerd-windows -tags prod -mod=readonly -ldflags "-s -w" ./cli
 
 ARG LINKERD_VERSION
 ENV GO_LDFLAGS="-s -w -X github.com/linkerd/linkerd2/pkg/version.Version=${LINKERD_VERSION}"
 RUN CGO_ENABLED=0 GOOS=darwin go build -o /out/linkerd-darwin -tags prod -mod=readonly -ldflags "${GO_LDFLAGS}" ./cli
-RUN CGO_ENABLED=0 GOOS=linux   go build -o /out/linkerd-linux   -tags prod -mod=readonly -ldflags "${GO_LDFLAGS}" ./cli
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=$TARGETARCH go build -o /out/linkerd-linux -tags prod -mod=readonly -ldflags "${GO_LDFLAGS}" ./cli
 RUN CGO_ENABLED=0 GOOS=windows go build -o /out/linkerd-windows -tags prod -mod=readonly -ldflags "${GO_LDFLAGS}" ./cli
 
 ## export without sources & dependencies

cni-plugin/Dockerfile

@@ -1,10 +1,13 @@
+ARG BUILDPLATFORM=linux/amd64
+
 # Precompile key slow-to-build dependencies
-FROM golang:1.14.2-alpine as go-deps
+FROM --platform=$BUILDPLATFORM golang:1.14.2-alpine as go-deps
 WORKDIR /linkerd-build
 COPY go.mod go.sum ./
 COPY bin/install-deps bin/
 RUN go mod download
-RUN ./bin/install-deps
+ARG TARGETARCH
+RUN ./bin/install-deps $TARGETARCH
 
 ## compile cni-plugin utility
 FROM go-deps as golang
@@ -12,7 +15,8 @@ WORKDIR /linkerd-build
 COPY pkg pkg
 COPY controller controller
 COPY cni-plugin cni-plugin
-RUN CGO_ENABLED=0 GOOS=linux go build -o /go/bin/linkerd-cni -v -mod=readonly ./cni-plugin/
+ARG TARGETARCH
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=$TARGETARCH go build -o /go/bin/linkerd-cni -v -mod=readonly ./cni-plugin/
 
 FROM debian:buster-20200514-slim
 WORKDIR /linkerd

controller/Dockerfile

@@ -1,10 +1,13 @@
+ARG BUILDPLATFORM=linux/amd64
+
 # Precompile key slow-to-build dependencies
-FROM golang:1.14.2-alpine as go-deps
+FROM --platform=$BUILDPLATFORM golang:1.14.2-alpine as go-deps
 WORKDIR /linkerd-build
 COPY go.mod go.sum ./
 COPY bin/install-deps bin/
 RUN go mod download
-RUN ./bin/install-deps
+ARG TARGETARCH
+RUN ./bin/install-deps $TARGETARCH
 
 ## compile controller service
 FROM go-deps as golang
@@ -18,8 +21,8 @@ COPY charts/partials charts/partials
 # Generate static templates
 # TODO: `go generate` does not honor -mod=readonly
 RUN go generate -mod=readonly ./pkg/charts/static
-
+ARG TARGETARCH
-RUN CGO_ENABLED=0 GOOS=linux go build -o /out/controller -tags prod -mod=readonly -ldflags "-s -w" ./controller/cmd
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=$TARGETARCH go build -o /out/controller -tags prod -mod=readonly -ldflags "-s -w" ./controller/cmd
 
 ## package runtime
 FROM scratch

web/Dockerfile

@@ -1,13 +1,16 @@
+ARG BUILDPLATFORM=linux/amd64
+
 # Precompile key slow-to-build dependencies
-FROM golang:1.14.2-alpine as go-deps
+FROM --platform=$BUILDPLATFORM golang:1.14.2-alpine as go-deps
 WORKDIR /linkerd-build
 COPY go.mod go.sum ./
 COPY bin/install-deps bin/
 RUN go mod download
-RUN ./bin/install-deps
+ARG TARGETARCH
+RUN ./bin/install-deps $TARGETARCH
 
 ## bundle web assets
-FROM node:14-buster as webpack-bundle
+FROM --platform=$BUILDPLATFORM node:14-buster as webpack-bundle
 RUN curl -o- -L https://yarnpkg.com/install.sh | bash -s -- --version 1.21.1 --network-concurrency 1
 
 ENV PATH /root/.yarn/bin:$PATH
@@ -36,8 +39,8 @@ COPY web/main.go web
 COPY web/srv web/srv
 COPY controller controller
 COPY pkg pkg
-
+ARG TARGETARCH
-RUN CGO_ENABLED=0 GOOS=linux go build -mod=readonly -o web/web -ldflags "-s -w" ./web
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=$TARGETARCH go build -mod=readonly -o web/web -ldflags "-s -w" ./web
 
 ## package it all up
 FROM debian:buster-20200514-slim