Automatically release container image for ARM (#2457)

.devcontainer/Dockerfile

@@ -70,6 +70,7 @@ RUN apt-get update \
     && add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/$(lsb_release -is | tr '[:upper:]' '[:lower:]') $(lsb_release -cs) stable" \
     && apt-get update \
     && apt-get install -y docker-ce-cli \
+    && docker buildx create --use \
     #
     # Install pip & pre-commit
     && apt-get -y install python3-pip \

.github/workflows/main-build.yml

@@ -3,10 +3,9 @@ on:
   push:
     branches:
       - main
-concurrency: e2e-tests
 jobs:
-  validate:
-    name: Validate
+  build:
+    name: build
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -55,7 +54,7 @@ jobs:
           registry: ghcr.io
 
       - name: Publish on GitHub Container Registry
-        run: make publish
+        run: make publish-multiarch
 
       # https://github.com/sigstore/cosign-installer
       - name: Install Cosign
@@ -70,6 +69,19 @@ jobs:
         # against the sigstore community Fulcio instance.
         run: make sign-images
 
+  validate:
+    needs: build
+    name: validate
+    runs-on: ubuntu-latest
+    # build-tools is built from ../../tools/build-tools.Dockerfile
+    container: ghcr.io/kedacore/build-tools:main
+    concurrency: e2e-tests
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+
       - name: Run end to end tests
         env:
           AZURE_APP_INSIGHTS_APP_ID: ${{ secrets.AZURE_APP_INSIGHTS_APP_ID }}
@@ -111,3 +123,27 @@ jobs:
           AZURE_SP_KEY: ${{ secrets.AZURE_SP_KEY }}
           AZURE_SP_TENANT: ${{ secrets.AZURE_SP_TENANT }}
           AZURE_SUBSCRIPTION: ${{ secrets.AZURE_SUBSCRIPTION }}
+
+  validate-arm64:
+    needs: build
+    name: validate-arm64
+    runs-on: ARM64
+    container: arm64v8/ubuntu:focal
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+
+      - name: Create k8s v1.23 Kind Cluster
+        uses: helm/kind-action@v1.2.0
+        with:
+          node_image: kindest/node:v1.23.0@sha256:49824ab1727c04e56a21a5d8372a402fcd32ea51ac96a2706a12af38934f81ac
+
+      - name: Install dependecies
+        run : |
+          apt update
+          apt install npm -y
+
+      - name: Run smoke test
+        run: make arm-smoke-test

.github/workflows/release-build.yml

@@ -60,7 +60,7 @@ jobs:
           VERSION: ${{ steps.get_version.outputs.VERSION }}
 
       - name: Publish KEDA images on GitHub Container Registry
-        run: make publish
+        run: make publish-multiarch
         env:
           VERSION: ${{ steps.get_version.outputs.VERSION }}
 

CHANGELOG.md

@@ -30,7 +30,7 @@
 
 ### New
 
-- TODO ([#XXX](https://github.com/kedacore/keda/pull/XXX))
+- **General:** Automatically release container image for ARM ([#2263]https://github.com/kedacore/keda/issues/2263))
 
 ### Improvements
 

Dockerfile

@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM golang:1.17.3 as builder
+FROM --platform=$BUILDPLATFORM golang:1.17.3 AS builder
 
 ARG BUILD_VERSION=main
 ARG GIT_COMMIT=HEAD
@@ -26,7 +26,9 @@ COPY controllers/ controllers/
 COPY pkg/ pkg/
 
 # Build
-RUN VERSION=${BUILD_VERSION} GIT_COMMIT=${GIT_COMMIT} GIT_VERSION=${GIT_VERSION} make manager
+# https://www.docker.com/blog/faster-multi-platform-builds-dockerfile-cross-compilation-guide/
+ARG TARGETOS TARGETARCH
+RUN VERSION=${BUILD_VERSION} GIT_COMMIT=${GIT_COMMIT} GIT_VERSION=${GIT_VERSION} TARGET_OS=$TARGETOS ARCH=$TARGETARCH make manager
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details

Dockerfile.adapter

@@ -1,5 +1,5 @@
 # Build the adapter binary
-FROM golang:1.17.3 as builder
+FROM --platform=$BUILDPLATFORM golang:1.17.3 as builder
 
 ARG BUILD_VERSION=main
 ARG GIT_COMMIT=HEAD
@@ -28,7 +28,9 @@ COPY pkg/ pkg/
 RUN mkdir -p /apiserver.local.config/certificates && chmod -R 777 /apiserver.local.config
 
 # Build
-RUN VERSION=${BUILD_VERSION} GIT_COMMIT=${GIT_COMMIT} GIT_VERSION=${GIT_VERSION} make adapter
+# https://www.docker.com/blog/faster-multi-platform-builds-dockerfile-cross-compilation-guide/
+ARG TARGETOS TARGETARCH
+RUN VERSION=${BUILD_VERSION} GIT_COMMIT=${GIT_COMMIT} GIT_VERSION=${GIT_VERSION} TARGET_OS=$TARGETOS ARCH=$TARGETARCH make adapter
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details

Makefile

@@ -94,6 +94,11 @@ e2e-test-local: ## Run e2e tests against Kubernetes cluster configured in ~/.kub
 e2e-test-clean: get-cluster-context ## Delete all namespaces labeled with type=e2e
 	kubectl delete ns -l type=e2e
 
+.PHONY: arm-smoke-test
+arm-smoke-test: ## Run e2e tests against Kubernetes cluster configured in ~/.kube/config.
+	npm install --prefix tests
+	./tests/run-arm-smoke-tests.sh
+
 ##################################################
 # Development                                    #
 ##################################################
@@ -183,6 +188,10 @@ publish: docker-build ## Push images on to Container Registry (default: ghcr.io)
 	docker push $(IMAGE_CONTROLLER)
 	docker push $(IMAGE_ADAPTER)
 
+publish-multiarch:
+	docker buildx build --push --platform=linux/amd64,linux/arm64 . -t ${IMAGE_CONTROLLER} --build-arg BUILD_VERSION=${VERSION} --build-arg GIT_VERSION=${GIT_VERSION} --build-arg GIT_COMMIT=${GIT_COMMIT}
+	docker buildx build --push --platform=linux/amd64,linux/arm64 -f Dockerfile.adapter -t ${IMAGE_ADAPTER} . --build-arg BUILD_VERSION=${VERSION} --build-arg GIT_VERSION=${GIT_VERSION} --build-arg GIT_COMMIT=${GIT_COMMIT}
+
 release: manifests kustomize set-version ## Produce new KEDA release in keda-$(VERSION).yaml file.
 	cd config/manager && \
 	$(KUSTOMIZE) edit set image ghcr.io/kedacore/keda=${IMAGE_CONTROLLER}

tests/run-arm-smoke-tests.sh

@@ -0,0 +1,102 @@
+#! /bin/bash
+set -eu
+
+DIR=$(dirname "$0")
+cd $DIR
+
+test_files=(
+    "influxdb.test.ts" #ScaledObject
+    "mongodb.test.ts" #ScaledJob
+)
+
+concurrent_tests_limit=5
+pids=()
+lookup=()
+failed_count=0
+failed_lookup=()
+counter=0
+
+function run_setup {
+    ./node_modules/.bin/ava setup.test.ts
+}
+
+function run_tests {
+    counter=0
+
+    for scaler in ${test_files[@]}
+    do
+        test_case="scalers/${scaler}"
+        counter=$((counter+1))
+        ./node_modules/.bin/ava $test_case > "${test_case}.log" 2>&1 &
+        pid=$!
+        echo "Running $test_case with pid: $pid"
+        pids+=($pid)
+        lookup[$pid]=$test_case
+        # limit concurrent runs
+        if [[ "$counter" -gt "$concurrent_tests_limit" ]]; then
+            wait_for_jobs
+            counter=0
+            pids=()
+        fi
+    done
+}
+
+function mark_failed {
+    failed_lookup[$1]=${lookup[$1]}
+    let "failed_count+=1"
+}
+
+function wait_for_jobs {
+    for job in "${pids[@]}"; do
+        wait $job || mark_failed $job
+        echo "Job $job finished"
+    done
+
+    printf "\n$failed_count jobs failed\n"
+    printf '%s\n' "${failed_lookup[@]}"
+}
+
+function print_logs {
+    for test_log in $(find scalers -name "*.log")
+    do
+        echo ">>> $test_log <<<"
+        cat $test_log
+        printf "\n\n##############################################\n"
+        printf "##############################################\n\n"
+    done
+
+    echo ">>> KEDA Operator log <<<"
+    kubectl get pods --no-headers -n keda | awk '{print $1}' | grep keda-operator | xargs kubectl -n keda logs
+    printf "\n\n##############################################\n"
+    printf "##############################################\n\n"
+
+    echo ">>> KEDA Metrics Server log <<<"
+    kubectl get pods --no-headers -n keda | awk '{print $1}' | grep keda-metrics-apiserver | xargs kubectl -n keda logs
+    printf "\n\n##############################################\n"
+    printf "##############################################\n\n"
+}
+
+function run_cleanup {
+    ./node_modules/.bin/ava cleanup.test.ts
+}
+
+function print_failed {
+    echo "$failed_count e2e tests failed"
+    for failed_test in "${failed_lookup[@]}"; do
+        echo $failed_test
+    done
+}
+
+run_setup
+run_tests
+wait_for_jobs
+print_logs
+run_cleanup
+
+if [ "$failed_count" == "0" ];
+then
+    exit 0
+else
+    print_failed
+    exit 1
+fi

tools/build-tools.Dockerfile

@@ -29,7 +29,8 @@ RUN curl -LO https://download.docker.com/linux/static/stable/x86_64/docker-19.03
     echo "$docker_sha256 docker-19.03.2.tgz" | sha256sum -c - && \
     tar xvzf docker-19.03.2.tgz && \
     mv docker/* /usr/local/bin && \
-    rm -rf docker docker-19.03.2.tgz
+    rm -rf docker docker-19.03.2.tgz && \
+    docker buildx create --use
 
 # Install golang
 RUN GO_VERSION=1.17.3 && \