diff --git a/.github/workflows/update-quarkus-platform.yaml b/.github/workflows/update-quarkus-platform.yaml
index 1e7cdc1c..69928996 100644
--- a/.github/workflows/update-quarkus-platform.yaml
+++ b/.github/workflows/update-quarkus-platform.yaml
@@ -17,6 +17,9 @@ jobs:
       - uses: actions/setup-node@v3
         with:
           node-version: 18
+      - uses: actions/setup-java@v1
+        with:
+          java-version: 17
       - name: Install NPM deps.
         run: npm install axios xml2js octokit
       - name: Create PR
diff --git a/hack/update-quarkus-platform.js b/hack/update-quarkus-platform.js
index be810c1b..222647b4 100644
--- a/hack/update-quarkus-platform.js
+++ b/hack/update-quarkus-platform.js
@@ -84,6 +84,19 @@ const updatePlatformInPom = async (pomPath, newPlatform) => {
     await writeFile(pomPath, newPomData)
 }
 
+const smokeTest = () => {
+    const subproc = spawn("make", ["test-quarkus"], {stdio: ['inherit', 'inherit', 'inherit']})
+    return new Promise((resolve, reject) => {
+        subproc.on('exit', code => {
+            if (code === 0) {
+                resolve()
+                return
+            }
+            reject(new Error("smoke test failed: non-zero exit code"))
+        })
+    })
+}
+
 const main = async () => {
     const latestPlatform = await getLatestPlatform()
     const prTitle = `chore: update Quarkus platform version to ${latestPlatform}`
@@ -103,6 +116,7 @@ const main = async () => {
 
     await updatePlatformInPom(cePomPath, latestPlatform)
     await updatePlatformInPom(httpPomPath, latestPlatform)
+    await smokeTest()
     await prepareBranch(branchName, prTitle)
     await octokit.rest.pulls.create({
         owner: owner,