diff --git a/buildscripts/grpc-java-linux-arm64-tests/Dockerfile b/buildscripts/grpc-java-linux-arm64-tests/Dockerfile
deleted file mode 100644
index 47cbd18ebc..0000000000
--- a/buildscripts/grpc-java-linux-arm64-tests/Dockerfile
+++ /dev/null
@@ -1,10 +0,0 @@
-# This docker image uses magic to seamlessly run its contents using qemu ARM emulator on x86 host.
-# See https://ownyourbits.com/2018/06/27/running-and-building-arm-docker-containers-in-x86/
-
-# We want to test on arm64v8
-FROM arm64v8/debian:buster
-
-# Make the host's qemu-arm-static emulator available from inside the docker container.
-COPY qemu-arm-static /usr/bin
-
-RUN apt-get update && apt-get install -y --no-install-recommends openjdk-11-jdk-headless && apt-get clean
diff --git a/buildscripts/kokoro/linux_aarch64.cfg b/buildscripts/kokoro/linux_aarch64.cfg
new file mode 100644
index 0000000000..f91fbe4d70
--- /dev/null
+++ b/buildscripts/kokoro/linux_aarch64.cfg
@@ -0,0 +1,12 @@
+# Config file for internal CI
+
+# Location of the continuous shell script in repository.
+build_file: "grpc-java/buildscripts/kokoro/linux_aarch64.sh"
+timeout_mins: 60
+
+action {
+  define_artifacts {
+    regex: "github/grpc-java/mvn-artifacts/**"
+    regex: "github/grpc-java/artifacts/**"
+  }
+}
\ No newline at end of file
diff --git a/buildscripts/kokoro/linux_aarch64.sh b/buildscripts/kokoro/linux_aarch64.sh
new file mode 100755
index 0000000000..11c3f8ae3a
--- /dev/null
+++ b/buildscripts/kokoro/linux_aarch64.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+set -veux -o pipefail
+
+if [[ -f /VERSION ]]; then
+  cat /VERSION
+fi
+
+cd github/grpc-java
+
+buildscripts/qemu_helpers/prepare_qemu.sh
+
+buildscripts/run_arm64_tests_in_docker.sh
diff --git a/buildscripts/qemu_helpers/prepare_qemu.sh b/buildscripts/qemu_helpers/prepare_qemu.sh
new file mode 100755
index 0000000000..f61320222a
--- /dev/null
+++ b/buildscripts/qemu_helpers/prepare_qemu.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+#
+# Setup and configure qemu userspace emulator on kokoro worker so that we can seamlessly emulate processes running
+# inside docker containers.
+
+set -ex
+
+# show pre-existing qemu registration
+cat /proc/sys/fs/binfmt_misc/qemu-aarch64
+
+# Kokoro ubuntu1604 workers have already qemu-user and qemu-user-static packages installed, but it's and old version that:
+# * prints warning about some syscalls (e.g "qemu: Unsupported syscall: 278")
+# * doesn't register with binfmt_misc with the persistent ("F") flag we need (see below)
+#
+# To overcome the above limitations, we use the https://github.com/multiarch/qemu-user-static
+# docker image to provide a new enough version of qemu-user-static and register it with
+# the desired binfmt_misc flags. The most important flag we need is "F" (set by "--persistent yes"),
+# which allows the qemu-aarch64-static binary to be loaded eagerly at the time of registration with binfmt_misc.
+# That way, we can emulate aarch64 binaries running inside docker containers transparently, without needing the emulator
+# binary to be accessible from the docker image we're emulating.
+# Note that on newer distributions (such as glinux), simply "apt install qemu-user-static" is sufficient
+# to install qemu-user-static with the right flags.
+docker run --rm --privileged multiarch/qemu-user-static:5.2.0-2 --reset --credential yes --persistent yes
+
+# Print current qemu reqistration to make sure everything is setup correctly.
+cat /proc/sys/fs/binfmt_misc/qemu-aarch64
diff --git a/buildscripts/run_arm64_tests_in_docker.sh b/buildscripts/run_arm64_tests_in_docker.sh
index f6f8c14cbb..6333d056fe 100755
--- a/buildscripts/run_arm64_tests_in_docker.sh
+++ b/buildscripts/run_arm64_tests_in_docker.sh
@@ -3,14 +3,6 @@ set -ex
 
 readonly grpc_java_dir="$(dirname "$(readlink -f "$0")")/.."
 
-# Build magic docker image that can run on x86 host, but looks like an ARM machine
-# from the inside (qemu-user-static is used for emulation)
-# Run "sudo apt install qemu-user-static binfmt-support" install the emulator
-# on the host machine.
-# Also see https://ownyourbits.com/2018/06/27/running-and-building-arm-docker-containers-in-x86/
-cp /usr/bin/qemu-arm-static "${grpc_java_dir}/buildscripts/grpc-java-linux-arm64-tests"
-docker build -t grpc-java-linux-arm64-tests "${grpc_java_dir}/buildscripts/grpc-java-linux-arm64-tests"
-
 if [[ -t 0 ]]; then
   DOCKER_ARGS="-it"
 else
@@ -18,9 +10,32 @@ else
   DOCKER_ARGS=
 fi
 
+# build under x64 docker image to save time over building everything under
+# aarch64 emulator. We've already built and tested the protoc binaries
+# so for the rest of the build we will be using "-PskipCodegen=true"
+# avoid further complicating the build.
+docker run $DOCKER_ARGS --rm=true -v "${grpc_java_dir}":/grpc-java -w /grpc-java \
+  --user "$(id -u):$(id -g)" \
+  -e "JAVA_OPTS=-Duser.home=/grpc-java/.current-user-home -Djava.util.prefs.userRoot=/grpc-java/.current-user-home/.java/.userPrefs" \
+  openjdk:11-jdk-slim-buster \
+  ./gradlew build -x test -PskipAndroid=true -PskipCodegen=true
+
+# Build and run java tests under aarch64 image.
+# To be able to run this docker container on x64 machine, one needs to have
+# qemu-user-static properly registered with binfmt_misc.
+# The most important flag binfmt_misc flag we need is "F" (set by "--persistent yes"),
+# which allows the qemu-aarch64-static binary to be loaded eagerly at the time of registration with binfmt_misc.
+# That way, we can emulate aarch64 binaries running inside docker containers transparently, without needing the emulator
+# binary to be accessible from the docker image we're emulating.
+# Note that on newer distributions (such as glinux), simply "apt install qemu-user-static" is sufficient
+# to install qemu-user-static with the right flags.
+# A note on the "docker run" args used:
 # - run docker container under current user's UID to avoid polluting the workspace
 # - set the user.home property to avoid creating a "?" directory under grpc-java
-exec docker run $DOCKER_ARGS --rm=true -v "${grpc_java_dir}":/grpc-java -w /grpc-java \
-  --user "$(id -u):$(id -g)" -e "JAVA_OPTS=-Duser.home=/grpc-java/.current-user-home" \
-  grpc-java-linux-arm64-tests \
-  bash -c "./gradlew build -PskipAndroid=true -PskipCodegen=true"
+# TODO(jtattermusch): avoid skipping ":grpc-netty:test" once https://github.com/grpc/grpc-java/issues/7830 is fixed.
+# TODO(jtattermusch): avoid skipping "grpc-netty-shaded:testShadow" once it's stable
+docker run $DOCKER_ARGS --rm=true -v "${grpc_java_dir}":/grpc-java -w /grpc-java \
+  --user "$(id -u):$(id -g)" \
+  -e "JAVA_OPTS=-Duser.home=/grpc-java/.current-user-home -Djava.util.prefs.userRoot=/grpc-java/.current-user-home/.java/.userPrefs" \
+  arm64v8/openjdk:11-jdk-slim-buster \
+  ./gradlew build -PskipAndroid=true -PskipCodegen=true -x :grpc-netty:test -x :grpc-netty-shaded:testShadow