Work around jvm crash on early 1.8 (#4345)

* Work around jvm crash on early 1.8

* skip retransform if class was already transformed during load

* fix imports after rebase

* add test

* disable test on windows

javaagent-bootstrap/src/main/java/io/opentelemetry/javaagent/OpenTelemetryAgent.java

@@ -41,14 +41,18 @@ import java.util.jar.Manifest;
 public final class OpenTelemetryAgent {
 
   public static void premain(String agentArgs, Instrumentation inst) {
-    agentmain(agentArgs, inst);
+    startAgent(inst, true);
   }
 
   public static void agentmain(String agentArgs, Instrumentation inst) {
+    startAgent(inst, false);
+  }
+
+  private static void startAgent(Instrumentation inst, boolean fromPremain) {
     try {
       File javaagentFile = installBootstrapJar(inst);
       InstrumentationHolder.setInstrumentation(inst);
-      AgentInitializer.initialize(inst, javaagentFile);
+      AgentInitializer.initialize(inst, javaagentFile, fromPremain);
     } catch (Throwable ex) {
       // Don't rethrow.  We don't have a log manager here, so just print.
       System.err.println("ERROR " + OpenTelemetryAgent.class.getName());
@@ -74,7 +78,7 @@ public final class OpenTelemetryAgent {
 
     // passing verify false for vendors who sign the agent jar, because jar file signature
     // verification is very slow before the JIT compiler starts up, which on Java 8 is not until
-    // after premain executes
+    // after premain execution completes
     JarFile agentJar = new JarFile(javaagentFile, false);
     verifyJarManifestMainClassIsThis(javaagentFile, agentJar);
     inst.appendToBootstrapClassLoaderSearch(agentJar);

javaagent-bootstrap/src/main/java/io/opentelemetry/javaagent/bootstrap/AgentInitializer.java

@@ -7,6 +7,7 @@ package io.opentelemetry.javaagent.bootstrap;
 
 import java.io.File;
 import java.lang.instrument.Instrumentation;
+import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import javax.annotation.Nullable;
@@ -21,32 +22,73 @@ import javax.annotation.Nullable;
  */
 public final class AgentInitializer {
 
-  // Accessed via reflection from tests.
-  // fields must be managed under class lock
   @Nullable private static ClassLoader agentClassLoader = null;
+  @Nullable private static AgentStarter agentStarter = null;
 
-  // called via reflection in the OpenTelemetryAgent class
-  public static void initialize(Instrumentation inst, File javaagentFile) throws Exception {
-    if (agentClassLoader == null) {
-      agentClassLoader = createAgentClassLoader("inst", javaagentFile);
+  public static void initialize(Instrumentation inst, File javaagentFile, boolean fromPremain)
+      throws Exception {
+    if (agentClassLoader != null) {
+      return;
+    }
 
-      Class<?> agentInstallerClass =
-          agentClassLoader.loadClass("io.opentelemetry.javaagent.tooling.AgentInstaller");
-      Method agentInstallerMethod =
-          agentInstallerClass.getMethod("installBytebuddyAgent", Instrumentation.class);
-      ClassLoader savedContextClassLoader = Thread.currentThread().getContextClassLoader();
-      try {
-        Thread.currentThread().setContextClassLoader(agentClassLoader);
-        agentInstallerMethod.invoke(null, inst);
-      } finally {
-        Thread.currentThread().setContextClassLoader(savedContextClassLoader);
-      }
+    agentClassLoader = createAgentClassLoader("inst", javaagentFile);
+    agentStarter = createAgentStarter(agentClassLoader, inst, javaagentFile);
+    if (!fromPremain || !delayAgentStart()) {
+      agentStarter.start();
     }
   }
 
-  // TODO misleading name
-  public static synchronized ClassLoader getAgentClassLoader() {
-    return agentClassLoader;
+  /**
+   * Test whether we are running on oracle 1.8 before 1.8.0_40.
+   *
+   * @return true for oracle 1.8 before 1.8.0_40
+   */
+  private static boolean isEarlyOracle18() {
+    // Java HotSpot(TM) 64-Bit Server VM
+    String vmName = System.getProperty("java.vm.name");
+    if (!vmName.contains("HotSpot")) {
+      return false;
+    }
+    // 1.8.0_31
+    String javaVersion = System.getProperty("java.version");
+    if (!javaVersion.startsWith("1.8")) {
+      return false;
+    }
+    int index = javaVersion.indexOf('_');
+    if (index == -1) {
+      return false;
+    }
+    String minorVersion = javaVersion.substring(index + 1);
+    try {
+      int version = Integer.parseInt(minorVersion);
+      if (version >= 40) {
+        return false;
+      }
+    } catch (NumberFormatException exception) {
+      return false;
+    }
+
+    return true;
+  }
+
+  private static boolean delayAgentStart() {
+    if (!isEarlyOracle18()) {
+      return false;
+    }
+
+    return agentStarter.delayStart();
+  }
+
+  /**
+   * Call to this method is inserted into {@code sun.launcher.LauncherHelper.checkAndLoadMain()}.
+   */
+  @SuppressWarnings("unused")
+  public static void delayedStartHook() {
+    agentStarter.start();
+  }
+
+  public static ClassLoader getExtensionsClassLoader() {
+    return agentStarter.getExtensionClassLoader();
   }
 
   /**
@@ -67,15 +109,7 @@ public final class AgentInitializer {
       agentParent = getPlatformClassLoader();
     }
 
-    ClassLoader agentClassLoader =
-        new AgentClassLoader(javaagentFile, innerJarFilename, agentParent);
-
-    Class<?> extensionClassLoaderClass =
-        agentClassLoader.loadClass("io.opentelemetry.javaagent.tooling.ExtensionClassLoader");
-    return (ClassLoader)
-        extensionClassLoaderClass
-            .getDeclaredMethod("getInstance", ClassLoader.class, File.class)
-            .invoke(null, agentClassLoader, javaagentFile);
+    return new AgentClassLoader(javaagentFile, innerJarFilename, agentParent);
   }
 
   private static ClassLoader getPlatformClassLoader()
@@ -92,5 +126,15 @@ public final class AgentInitializer {
     return System.getProperty("java.version").startsWith("1.");
   }
 
+  private static AgentStarter createAgentStarter(
+      ClassLoader agentClassLoader, Instrumentation instrumentation, File javaagentFile)
+      throws Exception {
+    Class<?> starterClass =
+        agentClassLoader.loadClass("io.opentelemetry.javaagent.tooling.AgentStarterImpl");
+    Constructor<?> constructor =
+        starterClass.getDeclaredConstructor(Instrumentation.class, File.class);
+    return (AgentStarter) constructor.newInstance(instrumentation, javaagentFile);
+  }
+
   private AgentInitializer() {}
 }

javaagent-bootstrap/src/main/java/io/opentelemetry/javaagent/bootstrap/AgentStarter.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javaagent.bootstrap;
+
+public interface AgentStarter {
+
+  /**
+   * When running on oracle jdk8 before 1.8.0_40 loading lambda classes inside agent premain will
+   * cause jvm to crash later when lambdas get jit compiled. To circumvent this crash we delay agent
+   * initialization to right before main method is called where loading lambda classes work fine.
+   *
+   * @return true when agent initialization will continue from a callback
+   */
+  boolean delayStart();
+
+  /** Transfer control to startup logic in agent class loader. */
+  void start();
+
+  /**
+   * Get extension class loader.
+   *
+   * @return class loader that is capable of loading configured extensions
+   */
+  ClassLoader getExtensionClassLoader();
+}

javaagent-tooling/src/main/java/io/opentelemetry/javaagent/tooling/AgentStarterImpl.java

@@ -0,0 +1,127 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javaagent.tooling;
+
+import io.opentelemetry.javaagent.bootstrap.AgentInitializer;
+import io.opentelemetry.javaagent.bootstrap.AgentStarter;
+import java.io.File;
+import java.lang.instrument.ClassFileTransformer;
+import java.lang.instrument.Instrumentation;
+import java.lang.instrument.UnmodifiableClassException;
+import java.security.ProtectionDomain;
+import org.objectweb.asm.ClassReader;
+import org.objectweb.asm.ClassVisitor;
+import org.objectweb.asm.ClassWriter;
+import org.objectweb.asm.MethodVisitor;
+import org.objectweb.asm.Opcodes;
+import org.objectweb.asm.Type;
+
+/**
+ * Main entry point into code that is running inside agent class loader, used reflectively from
+ * {@code io.opentelemetry.javaagent.bootstrap.AgentInitializer}.
+ */
+public class AgentStarterImpl implements AgentStarter {
+  private final Instrumentation instrumentation;
+  private final File javaagentFile;
+  private ClassLoader extensionClassLoader;
+
+  public AgentStarterImpl(Instrumentation instrumentation, File javaagentFile) {
+    this.instrumentation = instrumentation;
+    this.javaagentFile = javaagentFile;
+  }
+
+  @Override
+  public boolean delayStart() {
+    LaunchHelperClassFileTransformer transformer = new LaunchHelperClassFileTransformer();
+    instrumentation.addTransformer(transformer, true);
+
+    try {
+      Class<?> clazz = Class.forName("sun.launcher.LauncherHelper", false, null);
+      if (transformer.transformed) {
+        // LauncherHelper was loaded and got transformed
+        return transformer.hookInserted;
+      }
+      // LauncherHelper was already loaded before we set up transformer
+      instrumentation.retransformClasses(clazz);
+      return transformer.hookInserted;
+    } catch (ClassNotFoundException | UnmodifiableClassException ignore) {
+      // ignore
+    } finally {
+      instrumentation.removeTransformer(transformer);
+    }
+
+    return false;
+  }
+
+  @Override
+  public void start() {
+    extensionClassLoader = createExtensionClassLoader(getClass().getClassLoader(), javaagentFile);
+    ClassLoader savedContextClassLoader = Thread.currentThread().getContextClassLoader();
+    try {
+      Thread.currentThread().setContextClassLoader(extensionClassLoader);
+      AgentInstaller.installBytebuddyAgent(instrumentation);
+    } finally {
+      Thread.currentThread().setContextClassLoader(savedContextClassLoader);
+    }
+  }
+
+  @Override
+  public ClassLoader getExtensionClassLoader() {
+    return extensionClassLoader;
+  }
+
+  private static ClassLoader createExtensionClassLoader(
+      ClassLoader agentClassLoader, File javaagentFile) {
+    return ExtensionClassLoader.getInstance(agentClassLoader, javaagentFile);
+  }
+
+  private static class LaunchHelperClassFileTransformer implements ClassFileTransformer {
+    boolean hookInserted = false;
+    boolean transformed = false;
+
+    @Override
+    public byte[] transform(
+        ClassLoader loader,
+        String className,
+        Class<?> classBeingRedefined,
+        ProtectionDomain protectionDomain,
+        byte[] classfileBuffer) {
+      if (!"sun/launcher/LauncherHelper".equals(className)) {
+        return null;
+      }
+      transformed = true;
+      ClassReader cr = new ClassReader(classfileBuffer);
+      ClassWriter cw = new ClassWriter(cr, 0);
+      ClassVisitor cv =
+          new ClassVisitor(Opcodes.ASM7, cw) {
+            @Override
+            public MethodVisitor visitMethod(
+                int access, String name, String descriptor, String signature, String[] exceptions) {
+              MethodVisitor mv = super.visitMethod(access, name, descriptor, signature, exceptions);
+              if ("checkAndLoadMain".equals(name)) {
+                return new MethodVisitor(api, mv) {
+                  @Override
+                  public void visitCode() {
+                    super.visitCode();
+                    hookInserted = true;
+                    mv.visitMethodInsn(
+                        Opcodes.INVOKESTATIC,
+                        Type.getInternalName(AgentInitializer.class),
+                        "delayedStartHook",
+                        "()V",
+                        false);
+                  }
+                };
+              }
+              return mv;
+            }
+          };
+      cr.accept(cv, 0);
+
+      return hookInserted ? cw.toByteArray() : null;
+    }
+  }
+}

javaagent-tooling/src/main/java/io/opentelemetry/javaagent/tooling/Utils.java

@@ -24,7 +24,7 @@ public class Utils {
   }
 
   public static ClassLoader getExtensionsClassLoader() {
-    return AgentInitializer.getAgentClassLoader();
+    return AgentInitializer.getExtensionsClassLoader();
   }
 
   /** Return a classloader which can be used to look up bootstrap resources. */

smoke-tests/src/test/groovy/io/opentelemetry/smoketest/CrashEarlyJdk8Test.groovy

@@ -0,0 +1,59 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.smoketest
+
+import java.time.Duration
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+import org.testcontainers.containers.Container
+import org.testcontainers.containers.GenericContainer
+import org.testcontainers.containers.output.Slf4jLogConsumer
+import org.testcontainers.containers.wait.strategy.Wait
+import org.testcontainers.utility.DockerImageName
+import org.testcontainers.utility.MountableFile
+import spock.lang.IgnoreIf
+import spock.lang.Specification
+
+// Hotspot versions before 8u40 crash in jit compiled lambdas when javaagent initializes
+// java.lang.invoke.CallSite
+// This test verifies that such jvm does not crash with opentelemetry agent
+@IgnoreIf({ os.windows })
+class CrashEarlyJdk8Test extends Specification {
+  private static final Logger logger = LoggerFactory.getLogger(CrashEarlyJdk8Test)
+
+  private static final String TARGET_AGENT_FILENAME = "opentelemetry-javaagent.jar"
+  private static final String agentPath = System.getProperty("io.opentelemetry.smoketest.agent.shadowJar.path")
+
+  def "test crash on early jdk8"() {
+    setup:
+    GenericContainer target =
+      new GenericContainer<>(DockerImageName.parse("azul/zulu-openjdk:8u31"))
+        .withStartupTimeout(Duration.ofMinutes(5))
+        .withLogConsumer(new Slf4jLogConsumer(logger))
+        .withCopyFileToContainer(
+          MountableFile.forHostPath(agentPath), "/" + TARGET_AGENT_FILENAME)
+        .withCopyFileToContainer(
+          MountableFile.forClasspathResource("crashearlyjdk8/CrashEarlyJdk8.java"), "/CrashEarlyJdk8.java")
+        .withCopyFileToContainer(
+          MountableFile.forClasspathResource("crashearlyjdk8/start.sh", 777), "/start.sh")
+        .withCopyFileToContainer(
+          MountableFile.forClasspathResource("crashearlyjdk8/test.sh", 777), "/test.sh")
+        .waitingFor(
+          Wait.forLogMessage(".*started.*\\n", 1)
+        )
+        .withCommand("/bin/sh", "-c", "/start.sh")
+    target.start()
+
+    when:
+    Container.ExecResult result = target.execInContainer("/bin/sh", "-c", "/test.sh")
+    then:
+    result.getExitCode() == 0
+
+    cleanup:
+    System.err.println(result.toString())
+    target.stop()
+  }
+}

smoke-tests/src/test/resources/crashearlyjdk8/CrashEarlyJdk8.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import java.util.function.Supplier;
+
+public class CrashEarlyJdk8 {
+
+  public static void main(String... args) {
+    System.out.println("start test program");
+    CrashEarlyJdk8 crash = new CrashEarlyJdk8();
+    crash.test();
+    System.out.println("test program completed successfully");
+  }
+
+  public void test() {
+    // run loop enough times for jit compiler to kick in
+    for (int i = 0; i < 10_000; i++) {
+      this.bar(this::foo);
+    }
+  }
+
+  public int foo() {
+    return 1;
+  }
+
+  @SuppressWarnings("ReturnValueIgnored")
+  public void bar(Supplier<Integer> supplier) {
+    supplier.get();
+  }
+}

smoke-tests/src/test/resources/crashearlyjdk8/start.sh

@@ -0,0 +1,7 @@
+#/bin/sh
+
+echo "started"
+while :
+do
+  sleep 1
+done

smoke-tests/src/test/resources/crashearlyjdk8/test.sh

@@ -0,0 +1,7 @@
+#/bin/sh
+
+echo "compiling"
+javac CrashEarlyJdk8.java
+echo "finish compiling"
+echo "executing"
+java -javaagent:opentelemetry-javaagent.jar CrashEarlyJdk8

testing-common/src/main/java/io/opentelemetry/javaagent/testing/common/AgentClassLoaderAccess.java

@@ -5,7 +5,7 @@
 
 package io.opentelemetry.javaagent.testing.common;
 
-import java.lang.reflect.Field;
+import java.lang.reflect.Method;
 
 public final class AgentClassLoaderAccess {
 
@@ -16,18 +16,14 @@ public final class AgentClassLoaderAccess {
       Class<?> agentInitializerClass =
           ClassLoader.getSystemClassLoader()
               .loadClass("io.opentelemetry.javaagent.bootstrap.AgentInitializer");
-      Field agentClassLoaderField = agentInitializerClass.getDeclaredField("agentClassLoader");
-      agentClassLoaderField.setAccessible(true);
-      agentClassLoader = (ClassLoader) agentClassLoaderField.get(null);
+      Method getExtensionsClassLoader =
+          agentInitializerClass.getDeclaredMethod("getExtensionsClassLoader");
+      agentClassLoader = (ClassLoader) getExtensionsClassLoader.invoke(null);
     } catch (Throwable t) {
       throw new AssertionError("Could not access agent classLoader", t);
     }
   }
 
-  public static ClassLoader getAgentClassLoader() {
-    return agentClassLoader;
-  }
-
   static Class<?> loadClass(String name) {
     try {
       return agentClassLoader.loadClass(name);