Refactor Github action build workflow to build for both arm64/amd64 (#781)

Why this change? Now that we can efficiently build Docker images targeted at `linux/arm64`, we will start to release images for `linux/arm64` in the same way we do for `linux/amd64` images. Images released for `linux/amd64` are tagged as follows: 1. discourse/base:2.0.\<datetime\>-slim 2. discourse/base:slim 3. discourse/base:2.0.\<datetime\> 4. discourse/base:release For `linux/arm64`, the images are tagged as follows: 1. discourse/base:2.0.\<datetime\>-slim-arm64 2. discourse/base:slim-arm64 3. discourse/base:2.0.\<datetime\>-arm64 4. discourse/base:release-arm64 5. discourse/base:aarch64 (For backwards compatibility) For `linux/arm64`, we unfortunately cannot install chrome because chrome does not currently release binaries for the arch. Therefore, we install chromium which chrome is based off and also install the chromedriver binary for `linux/arm64` released by the electron project.
2024-03-20 06:26:36 +08:00 · 2024-03-20 06:26:36 +08:00 · 4bece5f47e
parent bbefa1e5f3
commit 4bece5f47e
4 changed files with 90 additions and 69 deletions
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@ -15,36 +15,76 @@ env:

 jobs:
  base:
-    runs-on: ubuntu-20.04${{ ((github.event_name != 'schedule') && '-8core') || '' }}
-    timeout-minutes: 90
+    # `unbuntu-20.04-8core` for arch amd64 non-scheduled builds
+    # `unbuntu-20.04` for arch amd64 scheduled builds
+    # `unbuntu-20.04-8core-arm` for arch arm64 non-scheduled builds
+    # `unbuntu-20.04-2core-arm` for arch arm64 scheduled builds
+    runs-on: ubuntu-20.04${{ ((github.event_name != 'schedule') && '-8core') || (( matrix.arch == 'arm64' && '-2core' ) || '') }}${{ (matrix.arch == 'arm64') && '-arm' || '' }}
+    strategy:
+      matrix:
+        arch: [amd64, arm64]
+    timeout-minutes: 30
    steps:
+      - name: Install Docker
+        if: ${{ matrix.arch == 'arm64' }}
+        run: |
+          # Add Docker's official GPG key:
+          sudo apt-get update
+          sudo apt-get install ca-certificates curl
+          sudo install -m 0755 -d /etc/apt/keyrings
+          sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
+          sudo chmod a+r /etc/apt/keyrings/docker.asc
+
+          # Add the repository to Apt sources:
+          echo \
+            "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
+            $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
+            sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+          sudo apt-get update
+
+          # Install Docker
+          sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin
+
+          # Give the current user permission to run docker without sudo
+          sudo usermod -aG docker $USER
+          sudo apt-get install -y acl
+          sudo setfacl --modify user:$USER:rw /var/run/docker.sock
+      - name: Install Ruby
+        if: ${{ matrix.arch == 'arm64' }}
+        run: |
+          sudo apt-get install -y ruby
      - uses: actions/checkout@v3
        with:
          fetch-depth: 1
+      - name: Set arch helper output
+        id: arch-helper
+        run: |
+          echo "arch_postfix_dash=${{ (matrix.arch == 'arm64' && '-arm64') || '' }}" >> $GITHUB_OUTPUT
+          echo "arch_postfix_underscore=${{ (matrix.arch == 'arm64' && '_arm64') || '' }}" >> $GITHUB_OUTPUT
      - name: build slim image
        run: |
-          cd image && ruby auto_build.rb base_slim
+          cd image && ruby auto_build.rb base_slim${{ steps.arch-helper.outputs.arch_postfix_underscore }}
      - name: tag slim images
        id: tag-images
        run: |
          TAG=`date +%Y%m%d-%H%M`
          echo "tag=$(echo $TAG)" >> $GITHUB_OUTPUT
-          docker tag discourse/base:build_slim discourse/base:2.0.$TAG-slim
-          docker tag discourse/base:build_slim discourse/base:slim
+          docker tag discourse/base:build_slim${{ steps.arch-helper.outputs.arch_postfix_underscore }} discourse/base:2.0.$TAG-slim${{ steps.arch-helper.outputs.arch_postfix_dash }}
+          docker tag discourse/base:build_slim${{ steps.arch-helper.outputs.arch_postfix_underscore }} discourse/base:slim${{ steps.arch-helper.outputs.arch_postfix_dash }}
      - name: build release image
        run: |
-          cd image && ruby auto_build.rb base
-      - name: tag release images
+          cd image && ruby auto_build.rb base${{ steps.arch-helper.outputs.arch_postfix_underscore }}
+      - name: tag amd64 release images
        run: |
          TAG=${{ steps.tag-images.outputs.tag }}
-          docker tag discourse/base:build discourse/base:2.0.$TAG
-          docker tag discourse/base:build discourse/base:release
+          docker tag discourse/base:build${{ steps.arch-helper.outputs.arch_postfix_underscore }} discourse/base:2.0.$TAG${{ steps.arch-helper.outputs.arch_postfix_dash }}
+          docker tag discourse/base:build${{ steps.arch-helper.outputs.arch_postfix_underscore }} discourse/base:release${{ steps.arch-helper.outputs.arch_postfix_dash }}
      - name: build test_build image
        run: |
-          cd image && ruby auto_build.rb discourse_test_build
+          cd image && ruby auto_build.rb discourse_test_build${{ steps.arch-helper.outputs.arch_postfix_underscore }}
      - name: run specs
        run: |
-          docker run --rm -e RUBY_ONLY=1 -e USE_TURBO=1 -e SKIP_PLUGINS=1 -e SKIP_LINT=1 discourse/discourse_test:build
+          docker run --rm -e RUBY_ONLY=1 -e USE_TURBO=1 -e SKIP_PLUGINS=1 -e SKIP_LINT=1 discourse/discourse_test:build${{ steps.arch-helper.outputs.arch_postfix_underscore }}
      - name: Print summary
        run: |
          docker images discourse/base
@ -55,10 +95,15 @@ jobs:
        run: |
          TAG=${{ steps.tag-images.outputs.tag }}
          docker login --username discoursebuild --password $DOCKERHUB_PASSWORD
-          docker push discourse/base:2.0.$TAG-slim
-          docker push discourse/base:slim
-          docker push discourse/base:2.0.$TAG
-          docker push discourse/base:release
+          docker push discourse/base:2.0.$TAG-slim${{ steps.arch-helper.outputs.arch_postfix_dash }}
+          docker push discourse/base:slim${{ steps.arch-helper.outputs.arch_postfix_dash }}
+          docker push discourse/base:2.0.$TAG${{ steps.arch-helper.outputs.arch_postfix_dash }}
+          docker push discourse/base:release${{ steps.arch-helper.outputs.arch_postfix_dash }}
+      - name: Push discourse/base:aarch64 image for backwards compatibility
+        if: success() && (github.ref == 'refs/heads/main') && (matrix.arch == 'arm64')
+        run: |
+          docker tag discourse/base:release${{ steps.arch-helper.outputs.arch_postfix_dash }} discourse/base:aarch64
+          docker push discourse/base:aarch64
  test:
    runs-on: ubuntu-20.04${{ ((github.event_name != 'schedule') && '-8core') || '' }}
    timeout-minutes: 30
@ -119,52 +164,3 @@ jobs:
          docker tag discourse/discourse_dev:build discourse/discourse_dev:release
          docker login --username discoursebuild --password $DOCKERHUB_PASSWORD
          docker push discourse/discourse_dev:release
-  aarch64:
-    runs-on: ubuntu-20.04${{ ((github.event_name != 'schedule') && '-8core') || '-2core' }}-arm
-    needs: base
-    steps:
-      - name: Install Docker
-        run: |
-          # Add Docker's official GPG key:
-          sudo apt-get update
-          sudo apt-get install ca-certificates curl
-          sudo install -m 0755 -d /etc/apt/keyrings
-          sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
-          sudo chmod a+r /etc/apt/keyrings/docker.asc
-          # Add the repository to Apt sources:
-          echo \
-            "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
-            $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
-            sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
-          sudo apt-get update
-          # Install Docker
-          sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin
-          # Give the current user permission to run docker without sudo
-          sudo usermod -aG docker $USER
-          sudo apt-get install -y acl
-          sudo setfacl --modify user:$USER:rw /var/run/docker.sock
-      - name: Install Ruby
-        run: |
-          sudo apt-get install -y ruby
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 1
-      - name: build slim image
-        run: |
-          cd image && ruby auto_build.rb base_slim_arm64
-      - name: build release image
-        run: |
-          cd image && ruby auto_build.rb base_arm64
-      - name: tag release image as release
-        working-directory: image/base
-        run: |
-          docker tag discourse/base:build_arm64 discourse/base:aarch64
-      - name: Print summary
-        run: docker images discourse/base
-      - name: push to dockerhub
-        if: success() && (github.ref == 'refs/heads/main')
-        env:
-          DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
-        run: |
-          docker login --username discoursebuild --password $DOCKERHUB_PASSWORD
-          docker push discourse/base:aarch64
--- a/image/auto_build.rb
+++ b/image/auto_build.rb
@ -28,6 +28,11 @@ images = {
    name: "discourse_test",
    tag: "discourse/discourse_test:build",
  },
+  discourse_test_build_arm64: {
+    name: "discourse_test",
+    tag: "discourse/discourse_test:build_arm64",
+    extra_args: "--platform linux/arm64 --build-arg=\"from_tag=build_arm64\"",
+  },
  discourse_dev: {
    name: "discourse_dev",
    tag: "discourse/discourse_dev:build",
--- a/image/discourse_test/Dockerfile
+++ b/image/discourse_test/Dockerfile
@ -17,10 +17,10 @@ RUN chown -R discourse . &&\

 FROM base AS with_browsers

-RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo apt-key add - &&\
-    echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list &&\
+ADD install-chrome /tmp/install-chrome
+RUN /tmp/install-chrome &&\
    apt update &&\
-    apt install -y libgconf-2-4 libxss1 google-chrome-stable firefox-esr &&\
+    apt install -y libgconf-2-4 libxss1 firefox-esr &&\
    cd /tmp && wget -q "https://download.mozilla.org/?product=firefox-latest-ssl&os=linux64&lang=en-US" -O firefox.tar.bz2 &&\
    tar xjvf firefox.tar.bz2 && mv /tmp/firefox /opt/firefox-evergreen &&\
    apt clean
@ -28,7 +28,7 @@ RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo ap
 FROM with_browsers AS release

 RUN cd /var/www/discourse &&\
-    sudo -u discourse bundle install --jobs=4 &&\
+    sudo -u discourse bundle install &&\
    sudo -E -u discourse -H yarn install &&\
    sudo -u discourse yarn cache clean

@ -36,4 +36,4 @@ RUN cd /var/www/discourse && sudo -E -u discourse -H bundle exec rake plugin:ins
    LOAD_PLUGINS=1 sudo -E -u discourse -H bundle exec rake plugin:install_all_gems &&\
    sudo -E -u discourse -H bundle exec ruby script/install_minio_binaries.rb

-ENTRYPOINT ["sudo", "-E", "-u", "discourse", "-H", "ruby", "script/docker_test.rb"]
+ENTRYPOINT ["sudo", "-E", "-u", "discourse", "-H", "ruby", "script/docker_test.rb"]
--- a/image/discourse_test/install-chrome
+++ b/image/discourse_test/install-chrome
@ -0,0 +1,20 @@
+#!/bin/bash
+set -e
+
+# https://googlechromelabs.github.io/chrome-for-testing/ doesn't provide linux/arm64 binaries for chrome or chromedriver
+# yet. Therefore on arm64, we install chromium instead of chrome and install a chromedriver for linux/arm64 from
+# https://github.com/electron/electron/releases/.
+#
+# On the current debian Chromium 120.0.6099.224 is installed so we have to install a chromedriver that is of the same version.
+if [ "$(dpkg --print-architecture)" = "arm64" ]; then
+  apt update && apt install -y chromium unzip &&\
+    wget -q -O /tmp/chromedriver.zip https://github.com/electron/electron/releases/download/v28.2.2/chromedriver-v28.2.2-linux-arm64.zip &&\
+    unzip /tmp/chromedriver.zip -d /tmp/chromedriver &&\
+    mv /tmp/chromedriver/chromedriver /usr/bin &&\
+    rm -rf /tmp/chromedriver /tmp/chromedriver.zip
+else
+  wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo apt-key add - &&\
+    echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list &&\
+    apt update &&\
+    apt install -y google-chrome-stable
+fi