diff --git a/smoke-tests/matrix/build.gradle b/smoke-tests/matrix/build.gradle
index 225af5afd9..11ceccf0ea 100644
--- a/smoke-tests/matrix/build.gradle
+++ b/smoke-tests/matrix/build.gradle
@@ -30,67 +30,80 @@ tasks.create("pushMatrix", DockerPushImage) {
   images.set(matrix)
 }
 
+// Each line under appserver describes one matrix of (version x vm x jdk), dockerfile key overrides
+// Dockerfile name, args key passes raw arguments to docker build
 def targets = [
-  "jetty"  : [
-    "9.4.35": ["8", "11", "15"],
-    "10.0.0": ["11", "15"],
+  "jetty": [
+    [version: ["9.4.35"], vm: ["hotspot", "openj9"], jdk: ["8", "11", "15"]],
+    [version: ["10.0.0"], vm: ["hotspot", "openj9"], jdk: ["11", "15"]],
   ],
-  "tomcat" : [
-    "7.0.107": ["8"],
-    "8.5.60" : ["8", "11"],
-    "9.0.40" : ["8", "11"],
-    "10.0.0" : ["8", "11"]
+  "tomcat": [
+    [version: ["7.0.107"], vm: ["hotspot", "openj9"], jdk: ["8"]],
+    [version: ["8.5.60", "9.0.40", "10.0.0"], vm: ["hotspot", "openj9"], jdk: ["8", "11"]]
   ],
-  "tomee" : [
-    "7.0.0": ["8"],
-    "8.0.6": ["8", "11"],
+  "tomee": [
+    [version: ["7.0.0"], vm: ["hotspot"], jdk: ["8"]],
+    [version: ["8.0.6"], vm: ["hotspot"], jdk: ["8", "11"]],
+    [version: ["7.0.0"], vm: ["openj9"], jdk: ["8"], dockerfile: "tomee-custom"],
+    [version: ["8.0.6"], vm: ["openj9"], jdk: ["8", "11"], dockerfile: "tomee-custom"]
   ],
-  "payara" : [
-    "5.2020.6"      : ["8"],
-    "5.2020.6-jdk11": ["11"]
+  "payara": [
+    [version: ["5.2020.6"], vm: ["hotspot"], jdk: ["8"], args: [tagSuffix: ""]],
+    [version: ["5.2020.6"], vm: ["hotspot"], jdk: ["11"], args: [tagSuffix: "-jdk11"]],
+    [version: ["5.2020.6"], vm: ["openj9"], jdk: ["8", "11"], dockerfile: "payara-custom-5.2020.6"]
   ],
   "wildfly": [
-    "13.0.0.Final": ["8"],
-    "17.0.1.Final": ["8", "11", "15"],
-    "21.0.0.Final": ["8", "11", "15"]
+    [version: ["13.0.0.Final"], vm: ["hotspot", "openj9"], jdk: ["8"]],
+    [version: ["17.0.1.Final", "21.0.0.Final"], vm: ["hotspot", "openj9"], jdk: ["8", "11", "15"]]
   ],
   "liberty": [
-    "20.0.0.12": ["8", "11", "15", "8-jdk-openj9", "11-jdk-openj9", "15-jdk-openj9"]
+    [version: ["20.0.0.12"], vm: ["hotspot", "openj9"], jdk: ["8", "11", "15"]]
   ]
 ]
 
-def dockerWorkingDir = new File(project.buildDir, "docker")
+def configureImage(server, dockerfile, version, vm, jdk, Map<String, String> extraArgs) {
+  def dockerWorkingDir = new File(project.buildDir, "docker")
 
-targets.each { server, data ->
-  data.forEach { version, jdks ->
-    jdks.forEach { jdk ->
-      def dockerfile = "${server}.dockerfile"
+  def prepareTask = tasks.register("${server}ImagePrepare-$version-jdk$jdk-$vm", Copy) {
+    def warTask = project.tasks.war
+    it.dependsOn(warTask)
+    it.into(dockerWorkingDir)
+    it.from("src")
+    it.from(warTask.archiveFile) {
+      rename { _ -> "app.war" }
+    }
+  }
 
-      def prepareTask = tasks.register("${server}ImagePrepare-$version-jdk$jdk", Copy) {
-        def warTask = project.tasks.war
-        it.dependsOn(warTask)
-        it.into(dockerWorkingDir)
-        it.from("src")
-        it.from(warTask.archiveFile) {
-          rename { _ -> "app.war" }
+  def extraTag = findProperty("extraTag") ?: new Date().format("yyyyMMdd.HHmmSS")
+  def vmSuffix = vm == "hotspot" ? "" : "-$vm"
+  def image = "ghcr.io/open-telemetry/java-test-containers:$server-$version-jdk$jdk$vmSuffix-$extraTag"
+
+  def buildTask = tasks.register("${server}Image-$version-jdk$jdk$vmSuffix", DockerBuildImage) {
+    it.dependsOn(prepareTask)
+    group = "build"
+    description = "Builds Docker image with $server $version on JDK $jdk"
+
+    it.inputDir.set(dockerWorkingDir)
+    it.images.add(image)
+    it.dockerFile.set(new File(dockerWorkingDir, dockerfile))
+    it.buildArgs.set(extraArgs + [jdk: jdk, vm: vm, version: version])
+  }
+
+  project.tasks.buildMatrix.dependsOn(buildTask)
+  return image
+}
+
+targets.each { server, matrices ->
+  matrices.forEach { entry ->
+    def dockerfile = (entry["dockerfile"]?.toString() ?: server) + ".dockerfile"
+    def extraArgs = (entry["args"] ?: [:]) as Map<String, String>
+
+    entry.version.forEach { version ->
+      entry.vm.forEach { vm ->
+        entry.jdk.forEach { jdk ->
+          matrix.add(configureImage(server, dockerfile, version, vm, jdk, extraArgs))
         }
       }
-
-      def extraTag = findProperty("extraTag") ?: new Date().format("yyyyMMdd.HHmmSS")
-      def image = "ghcr.io/open-telemetry/java-test-containers:$server-$version-jdk$jdk-$extraTag"
-      matrix.add(image)
-      def buildTask = tasks.register("${server}Image-$version-jdk$jdk", DockerBuildImage) {
-        it.dependsOn(prepareTask)
-        group = "build"
-        description = "Builds Docker image with $server $version on JDK $jdk"
-
-        it.inputDir.set(dockerWorkingDir)
-        it.images.add(image)
-        it.dockerFile.set(new File(dockerWorkingDir, dockerfile))
-        it.buildArgs.set(["version": version, "jdk": jdk])
-      }
-
-      buildMatrixTask.dependsOn(buildTask)
     }
   }
 }
diff --git a/smoke-tests/matrix/src/jetty.dockerfile b/smoke-tests/matrix/src/jetty.dockerfile
index 2f155f0c76..29725aadb5 100644
--- a/smoke-tests/matrix/src/jetty.dockerfile
+++ b/smoke-tests/matrix/src/jetty.dockerfile
@@ -1,8 +1,9 @@
 ARG version
 ARG jdk
+ARG vm
 FROM jetty:${version}-jre11-slim as jetty
 
-FROM adoptopenjdk:${jdk}
+FROM adoptopenjdk:${jdk}-jdk-${vm}
 ENV JETTY_HOME /usr/local/jetty
 ENV JETTY_BASE /var/lib/jetty
 ENV TMPDIR /tmp/jetty
diff --git a/smoke-tests/matrix/src/liberty.dockerfile b/smoke-tests/matrix/src/liberty.dockerfile
index 443414e0ef..a66661a695 100644
--- a/smoke-tests/matrix/src/liberty.dockerfile
+++ b/smoke-tests/matrix/src/liberty.dockerfile
@@ -1,8 +1,9 @@
 ARG version
 ARG jdk
+ARG vm
 FROM open-liberty:${version}-full-java11-openj9 as liberty
 
-FROM adoptopenjdk:${jdk}
+FROM adoptopenjdk:${jdk}-jdk-${vm}
 ENV CONFIG /config
 ENV LIBERTY /opt/ol
 ENV PATH=/opt/ol/wlp/bin:/opt/ol/docker/:/opt/ol/helpers/build:$PATH \
diff --git a/smoke-tests/matrix/src/payara-custom-5.2020.6.dockerfile b/smoke-tests/matrix/src/payara-custom-5.2020.6.dockerfile
new file mode 100644
index 0000000000..ae983f57f9
--- /dev/null
+++ b/smoke-tests/matrix/src/payara-custom-5.2020.6.dockerfile
@@ -0,0 +1,33 @@
+ARG version
+ARG jdk
+ARG vm
+
+FROM payara/server-full:${version} as default
+ENV HOME_DIR=$HOME_DIR
+
+FROM adoptopenjdk:${jdk}-jdk-${vm}
+
+# These environment variables have been confirmed to work with 5.2020.6 only
+ENV HOME_DIR=/opt/payara
+ENV PAYARA_DIR="${HOME_DIR}/appserver" \
+    SCRIPT_DIR="${HOME_DIR}/scripts" \
+    CONFIG_DIR="${HOME_DIR}/config" \
+    DEPLOY_DIR="${HOME_DIR}/deployments" \
+    PASSWORD_FILE="${HOME_DIR}/passwordFile" \
+    ADMIN_USER="admin" \
+    ADMIN_PASSWORD="admin" \
+    MEM_MAX_RAM_PERCENTAGE=70.0 \
+    MEM_XSS=512k \
+    DOMAIN_NAME="production" \
+    PREBOOT_COMMANDS="${HOME_DIR}/config/pre-boot-commands.asadmin" \
+    POSTBOOT_COMMANDS="${HOME_DIR}/config/post-boot-commands.asadmin" \
+    PATH="${PATH}:${HOME_DIR}/scripts"
+
+COPY --from=default $HOME_DIR $HOME_DIR
+RUN rm ${PAYARA_DIR}/glassfish/modules/phonehome-bootstrap.jar
+
+WORKDIR $HOME_DIR
+
+EXPOSE 8080
+CMD ["entrypoint.sh"]
+COPY app.war $DEPLOY_DIR
diff --git a/smoke-tests/matrix/src/payara.dockerfile b/smoke-tests/matrix/src/payara.dockerfile
index 49c994a141..33e3c752c7 100644
--- a/smoke-tests/matrix/src/payara.dockerfile
+++ b/smoke-tests/matrix/src/payara.dockerfile
@@ -1,7 +1,7 @@
 ARG version
 ARG jdk
 
-FROM payara/server-full:${version}
+FROM payara/server-full:${version}${tagSuffix}
 
 RUN rm ${PAYARA_DIR}/glassfish/modules/phonehome-bootstrap.jar
 
diff --git a/smoke-tests/matrix/src/tomcat.dockerfile b/smoke-tests/matrix/src/tomcat.dockerfile
index 39cc90b6a6..3f159df474 100644
--- a/smoke-tests/matrix/src/tomcat.dockerfile
+++ b/smoke-tests/matrix/src/tomcat.dockerfile
@@ -1,6 +1,7 @@
 ARG version
 ARG jdk
+ARG vm
 
-FROM tomcat:${version}-jdk${jdk}-adoptopenjdk-hotspot
+FROM tomcat:${version}-jdk${jdk}-adoptopenjdk-${vm}
 
 COPY app.war /usr/local/tomcat/webapps/
\ No newline at end of file
diff --git a/smoke-tests/matrix/src/tomee-custom.dockerfile b/smoke-tests/matrix/src/tomee-custom.dockerfile
new file mode 100644
index 0000000000..e86d416796
--- /dev/null
+++ b/smoke-tests/matrix/src/tomee-custom.dockerfile
@@ -0,0 +1,15 @@
+ARG version
+ARG jdk
+ARG vm
+
+FROM tomee:${jdk}-jre-${version}-webprofile as default
+
+FROM adoptopenjdk:${jdk}-jdk-${vm}
+
+ENV SERVER_BASE=/usr/local/tomee
+COPY --from=default $SERVER_BASE $SERVER_BASE
+WORKDIR $SERVER_BASE
+
+EXPOSE 8080
+CMD ["bin/catalina.sh", "run"]
+COPY app.war $SERVER_BASE/webapps/
diff --git a/smoke-tests/matrix/src/wildfly.dockerfile b/smoke-tests/matrix/src/wildfly.dockerfile
index f54e52db76..9a929ea18a 100644
--- a/smoke-tests/matrix/src/wildfly.dockerfile
+++ b/smoke-tests/matrix/src/wildfly.dockerfile
@@ -1,5 +1,6 @@
 ARG jdk
-FROM adoptopenjdk:${jdk}
+ARG vm
+FROM adoptopenjdk:${jdk}-jdk-${vm}
 
 # Create a user and group used to launch processes
 # The user ID 1000 is the default for the first "regular" user on Fedora/RHEL,