Improve emulated linux aarch64 tests, without protoc artifact build

Based on my testing, the tests are currently flaky, so it would be better to first merge it and start running it continuously to asses the situation.

buildscripts/grpc-java-linux-arm64-tests/Dockerfile

@@ -1,10 +0,0 @@
-# This docker image uses magic to seamlessly run its contents using qemu ARM emulator on x86 host.
-# See https://ownyourbits.com/2018/06/27/running-and-building-arm-docker-containers-in-x86/
-
-# We want to test on arm64v8
-FROM arm64v8/debian:buster
-
-# Make the host's qemu-arm-static emulator available from inside the docker container.
-COPY qemu-arm-static /usr/bin
-
-RUN apt-get update && apt-get install -y --no-install-recommends openjdk-11-jdk-headless && apt-get clean

buildscripts/kokoro/linux_aarch64.cfg

@@ -0,0 +1,12 @@
+# Config file for internal CI
+
+# Location of the continuous shell script in repository.
+build_file: "grpc-java/buildscripts/kokoro/linux_aarch64.sh"
+timeout_mins: 60
+
+action {
+  define_artifacts {
+    regex: "github/grpc-java/mvn-artifacts/**"
+    regex: "github/grpc-java/artifacts/**"
+  }
+}

buildscripts/kokoro/linux_aarch64.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -veux -o pipefail
+
+if [[ -f /VERSION ]]; then
+  cat /VERSION
+fi
+
+cd github/grpc-java
+
+buildscripts/qemu_helpers/prepare_qemu.sh
+
+buildscripts/run_arm64_tests_in_docker.sh

buildscripts/qemu_helpers/prepare_qemu.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+#
+# Setup and configure qemu userspace emulator on kokoro worker so that we can seamlessly emulate processes running
+# inside docker containers.
+
+set -ex
+
+# show pre-existing qemu registration
+cat /proc/sys/fs/binfmt_misc/qemu-aarch64
+
+# Kokoro ubuntu1604 workers have already qemu-user and qemu-user-static packages installed, but it's and old version that:
+# * prints warning about some syscalls (e.g "qemu: Unsupported syscall: 278")
+# * doesn't register with binfmt_misc with the persistent ("F") flag we need (see below)
+#
+# To overcome the above limitations, we use the https://github.com/multiarch/qemu-user-static
+# docker image to provide a new enough version of qemu-user-static and register it with
+# the desired binfmt_misc flags. The most important flag we need is "F" (set by "--persistent yes"),
+# which allows the qemu-aarch64-static binary to be loaded eagerly at the time of registration with binfmt_misc.
+# That way, we can emulate aarch64 binaries running inside docker containers transparently, without needing the emulator
+# binary to be accessible from the docker image we're emulating.
+# Note that on newer distributions (such as glinux), simply "apt install qemu-user-static" is sufficient
+# to install qemu-user-static with the right flags.
+docker run --rm --privileged multiarch/qemu-user-static:5.2.0-2 --reset --credential yes --persistent yes
+
+# Print current qemu reqistration to make sure everything is setup correctly.
+cat /proc/sys/fs/binfmt_misc/qemu-aarch64

buildscripts/run_arm64_tests_in_docker.sh

@@ -3,14 +3,6 @@ set -ex
 
 readonly grpc_java_dir="$(dirname "$(readlink -f "$0")")/.."
 
-# Build magic docker image that can run on x86 host, but looks like an ARM machine
-# from the inside (qemu-user-static is used for emulation)
-# Run "sudo apt install qemu-user-static binfmt-support" install the emulator
-# on the host machine.
-# Also see https://ownyourbits.com/2018/06/27/running-and-building-arm-docker-containers-in-x86/
-cp /usr/bin/qemu-arm-static "${grpc_java_dir}/buildscripts/grpc-java-linux-arm64-tests"
-docker build -t grpc-java-linux-arm64-tests "${grpc_java_dir}/buildscripts/grpc-java-linux-arm64-tests"
-
 if [[ -t 0 ]]; then
   DOCKER_ARGS="-it"
 else
@@ -18,9 +10,32 @@ else
   DOCKER_ARGS=
 fi
 
+# build under x64 docker image to save time over building everything under
+# aarch64 emulator. We've already built and tested the protoc binaries
+# so for the rest of the build we will be using "-PskipCodegen=true"
+# avoid further complicating the build.
+docker run $DOCKER_ARGS --rm=true -v "${grpc_java_dir}":/grpc-java -w /grpc-java \
+  --user "$(id -u):$(id -g)" \
+  -e "JAVA_OPTS=-Duser.home=/grpc-java/.current-user-home -Djava.util.prefs.userRoot=/grpc-java/.current-user-home/.java/.userPrefs" \
+  openjdk:11-jdk-slim-buster \
+  ./gradlew build -x test -PskipAndroid=true -PskipCodegen=true
+
+# Build and run java tests under aarch64 image.
+# To be able to run this docker container on x64 machine, one needs to have
+# qemu-user-static properly registered with binfmt_misc.
+# The most important flag binfmt_misc flag we need is "F" (set by "--persistent yes"),
+# which allows the qemu-aarch64-static binary to be loaded eagerly at the time of registration with binfmt_misc.
+# That way, we can emulate aarch64 binaries running inside docker containers transparently, without needing the emulator
+# binary to be accessible from the docker image we're emulating.
+# Note that on newer distributions (such as glinux), simply "apt install qemu-user-static" is sufficient
+# to install qemu-user-static with the right flags.
+# A note on the "docker run" args used:
 # - run docker container under current user's UID to avoid polluting the workspace
 # - set the user.home property to avoid creating a "?" directory under grpc-java
-exec docker run $DOCKER_ARGS --rm=true -v "${grpc_java_dir}":/grpc-java -w /grpc-java \
+# TODO(jtattermusch): avoid skipping ":grpc-netty:test" once https://github.com/grpc/grpc-java/issues/7830 is fixed.
-  --user "$(id -u):$(id -g)" -e "JAVA_OPTS=-Duser.home=/grpc-java/.current-user-home" \
+# TODO(jtattermusch): avoid skipping "grpc-netty-shaded:testShadow" once it's stable
-  grpc-java-linux-arm64-tests \
+docker run $DOCKER_ARGS --rm=true -v "${grpc_java_dir}":/grpc-java -w /grpc-java \
-  bash -c "./gradlew build -PskipAndroid=true -PskipCodegen=true"
+  --user "$(id -u):$(id -g)" \
+  -e "JAVA_OPTS=-Duser.home=/grpc-java/.current-user-home -Djava.util.prefs.userRoot=/grpc-java/.current-user-home/.java/.userPrefs" \
+  arm64v8/openjdk:11-jdk-slim-buster \
+  ./gradlew build -PskipAndroid=true -PskipCodegen=true -x :grpc-netty:test -x :grpc-netty-shaded:testShadow