Tooling and jmxfetch temp jars removed

Enhanced the DatadogClassloader to work with jars inside other jars.  No need to create the jars in a temp directory anymore

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/DatadogClassLoader.java

@@ -1,9 +1,28 @@
 package datadog.trace.bootstrap;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLClassLoader;
+import java.net.URLConnection;
+import java.net.URLStreamHandler;
+import java.nio.file.NoSuchFileException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.jar.JarEntry;
+import java.util.jar.JarInputStream;
+import lombok.extern.slf4j.Slf4j;
 
-/** Classloader used to run the core datadog agent. */
+/**
+ * Classloader used to run the core datadog agent.
+ *
+ * <p>It is built around the concept of a jar inside another jar. This classloader loads the files
+ * of the internal jar to load classes and resources.
+ */
+@Slf4j
 public class DatadogClassLoader extends URLClassLoader {
   static {
     ClassLoader.registerAsParallelCapable();
@@ -19,17 +38,41 @@ public class DatadogClassLoader extends URLClassLoader {
    * Construct a new DatadogClassLoader
    *
    * @param bootstrapJarLocation Used for resource lookups.
-   * @param agentJarLocation Classpath of this classloader.
+   * @param internalJarFileName File name of the internal jar
    * @param parent Classloader parent. Should null (bootstrap), or the platform classloader for java
    *     9+.
    */
-  public DatadogClassLoader(URL bootstrapJarLocation, URL agentJarLocation, ClassLoader parent) {
+  public DatadogClassLoader(
-    super(new URL[] {agentJarLocation}, parent);
+      final URL bootstrapJarLocation,
-    bootstrapProxy = new BootstrapClassLoaderProxy(new URL[] {bootstrapJarLocation}, null);
+      final String internalJarFileName,
+      final ClassLoader classloaderForJarResource,
+      final ClassLoader parent) {
+    super(new URL[] {}, parent);
+    bootstrapProxy = new BootstrapClassLoaderProxy(new URL[] {bootstrapJarLocation});
+
+    if (classloaderForJarResource != null) { // Some tests pass null
+      try {
+        // The fields of the URL are mostly dummy.  InternalJarURLHandler is the only important
+        // field.  If extending this class from Classloader instead of URLClassloader required less
+        // boilerplate it could be used and the need for dummy fields would be reduced
+        final URL internalJarURL =
+            new URL(
+                "x-internal-jar",
+                null,
+                0,
+                "/",
+                new InternalJarURLHandler(internalJarFileName, classloaderForJarResource));
+
+        addURL(internalJarURL);
+      } catch (final MalformedURLException e) {
+        // This can't happen with current URL constructor
+        log.error("URL malformed.  Unsupported JDK?", e);
+      }
+    }
   }
 
   @Override
-  public URL getResource(String resourceName) {
+  public URL getResource(final String resourceName) {
     final URL bootstrapResource = bootstrapProxy.getResource(resourceName);
     if (null == bootstrapResource) {
       return super.getResource(resourceName);
@@ -61,18 +104,103 @@ public class DatadogClassLoader extends URLClassLoader {
       ClassLoader.registerAsParallelCapable();
     }
 
-    public BootstrapClassLoaderProxy(URL[] urls, ClassLoader parent) {
+    public BootstrapClassLoaderProxy(final URL[] urls) {
-      super(urls, parent);
+      super(urls);
     }
 
     @Override
-    public void addURL(URL url) {
+    public void addURL(final URL url) {
       super.addURL(url);
     }
 
     @Override
-    protected Class<?> findClass(String name) throws ClassNotFoundException {
+    protected Class<?> findClass(final String name) throws ClassNotFoundException {
       throw new ClassNotFoundException(name);
     }
   }
+
+  private static class InternalJarURLHandler extends URLStreamHandler {
+    private final Map<String, byte[]> filenameToBytes = new HashMap<>();
+
+    public InternalJarURLHandler(
+        final String internalJarFileName, final ClassLoader classloaderForJarResource) {
+
+      final InputStream jarStream =
+          classloaderForJarResource.getResourceAsStream(internalJarFileName);
+
+      if (jarStream != null) {
+        try (final JarInputStream inputStream = new JarInputStream(jarStream)) {
+          JarEntry entry = inputStream.getNextJarEntry();
+
+          while (entry != null) {
+            filenameToBytes.put("/" + entry.getName(), getBytes(inputStream));
+
+            entry = inputStream.getNextJarEntry();
+          }
+
+        } catch (final IOException e) {
+          log.error("Unable to read internal jar", e);
+        }
+      } else {
+        log.error("Internal jar not found");
+      }
+    }
+
+    @Override
+    protected URLConnection openConnection(final URL url) throws IOException {
+      final byte[] bytes = filenameToBytes.get(url.getFile());
+
+      if (bytes == null) {
+        throw new NoSuchFileException(url.getFile());
+      }
+
+      return new InternalJarURLConnection(url, bytes);
+    }
+  }
+
+  private static class InternalJarURLConnection extends URLConnection {
+    private final byte[] bytes;
+
+    private InternalJarURLConnection(final URL url, final byte[] bytes) {
+      super(url);
+      this.bytes = bytes;
+    }
+
+    @Override
+    public void connect() throws IOException {
+      connected = true;
+    }
+
+    @Override
+    public InputStream getInputStream() throws IOException {
+      return new ByteArrayInputStream(bytes);
+    }
+  }
+
+  /**
+   * Standard "copy InputStream to byte[]" implementation using a ByteArrayOutputStream
+   *
+   * <p>IOUtils.toByteArray() or Java 9's InputStream.readAllBytes() could be replacements if they
+   * were available
+   *
+   * <p>This can be optimized using the JarEntry's size(), but its not always available
+   *
+   * @param inputStream
+   * @return a byte[] from the inputstream
+   * @throws IOException
+   */
+  private static byte[] getBytes(final InputStream inputStream) throws IOException {
+    final byte[] buffer = new byte[4096];
+
+    int bytesRead = inputStream.read(buffer, 0, buffer.length);
+    try (final ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) {
+      while (bytesRead != -1) {
+        outputStream.write(buffer, 0, bytesRead);
+
+        bytesRead = inputStream.read(buffer, 0, buffer.length);
+      }
+
+      return outputStream.toByteArray();
+    }
+  }
 }

dd-java-agent/agent-bootstrap/src/test/groovy/datadog/trace/bootstrap/DatadogClassLoaderTest.groovy

@@ -13,7 +13,7 @@ class DatadogClassLoaderTest extends Specification {
     def className1 = 'some/class/Name1'
     def className2 = 'some/class/Name2'
     final URL loc = getClass().getProtectionDomain().getCodeSource().getLocation()
-    final DatadogClassLoader ddLoader = new DatadogClassLoader(loc, loc, (ClassLoader) null)
+    final DatadogClassLoader ddLoader = new DatadogClassLoader(loc, null, null, null)
     final Phaser threadHoldLockPhase = new Phaser(2)
     final Phaser acquireLockFromMainThreadPhase = new Phaser(2)
 

dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/Utils.java

@@ -15,7 +15,7 @@ public class Utils {
   private static Method findLoadedClassMethod = null;
 
   private static final BootstrapClassLoaderProxy unitTestBootstrapProxy =
-      new BootstrapClassLoaderProxy(new URL[0], null);
+      new BootstrapClassLoaderProxy(new URL[0]);
 
   static {
     try {

dd-java-agent/agent-tooling/src/test/groovy/datadog/trace/agent/test/ClassLoaderMatcherTest.groovy

@@ -16,7 +16,7 @@ class ClassLoaderMatcherTest extends Specification {
   def "skips agent classloader"() {
     setup:
     URL root = new URL("file://")
-    final URLClassLoader agentLoader = new DatadogClassLoader(root, root, null)
+    final URLClassLoader agentLoader = new DatadogClassLoader(root, null, null, null)
     expect:
     ClassLoaderMatcher.skipClassLoader().matches(agentLoader)
   }

dd-java-agent/src/main/java/datadog/trace/agent/TracingAgent.java

@@ -31,8 +31,6 @@ public class TracingAgent {
   private static ClassLoader AGENT_CLASSLOADER = null;
   private static ClassLoader JMXFETCH_CLASSLOADER = null;
   private static File bootstrapJar = null;
-  private static File toolingJar = null;
-  private static File jmxFetchJar = null;
 
   public static void premain(final String agentArgs, final Instrumentation inst) throws Exception {
     agentmain(agentArgs, inst);
@@ -81,7 +79,8 @@ public class TracingAgent {
     if (AGENT_CLASSLOADER == null) {
       // bootstrap jar must be appended before agent classloader is created.
       inst.appendToBootstrapClassLoaderSearch(new JarFile(bootstrapJar));
-      final ClassLoader agentClassLoader = createDatadogClassLoader(bootstrapJar, toolingJar);
+      final ClassLoader agentClassLoader =
+          createDatadogClassLoader(bootstrapJar, "agent-tooling-and-instrumentation.jar.zip");
       final ClassLoader contextLoader = Thread.currentThread().getContextClassLoader();
       try {
         Thread.currentThread().setContextClassLoader(agentClassLoader);
@@ -111,7 +110,8 @@ public class TracingAgent {
   public static synchronized void startJmxFetch() throws Exception {
     initializeJars();
     if (JMXFETCH_CLASSLOADER == null) {
-      final ClassLoader jmxFetchClassLoader = createDatadogClassLoader(bootstrapJar, jmxFetchJar);
+      final ClassLoader jmxFetchClassLoader =
+          createDatadogClassLoader(bootstrapJar, "agent-jmxfetch.jar.zip");
       final ClassLoader contextLoader = Thread.currentThread().getContextClassLoader();
       try {
         Thread.currentThread().setContextClassLoader(jmxFetchClassLoader);
@@ -151,18 +151,6 @@ public class TracingAgent {
               "agent-bootstrap.jar.zip",
               "agent-bootstrap.jar");
     }
-    if (toolingJar == null) {
-      toolingJar =
-          extractToTmpFile(
-              TracingAgent.class.getClassLoader(),
-              "agent-tooling-and-instrumentation.jar.zip",
-              "agent-tooling-and-instrumentation.jar");
-    }
-    if (jmxFetchJar == null) {
-      jmxFetchJar =
-          extractToTmpFile(
-              TracingAgent.class.getClassLoader(), "agent-jmxfetch.jar.zip", "agent-jmxfetch.jar");
-    }
   }
 
   /**
@@ -170,11 +158,12 @@ public class TracingAgent {
    * the bootstrap classpath.
    *
    * @param bootstrapJar datadog bootstrap jar which has been appended to the bootstrap loader
-   * @param toolingJar jar to use for the classpath of the datadog classloader
+   * @param innerJarFilename Filename of internal jar to use for the classpath of the datadog
+   *     classloader
    * @return Datadog Classloader
    */
   private static ClassLoader createDatadogClassLoader(
-      final File bootstrapJar, final File toolingJar) throws Exception {
+      final File bootstrapJar, final String innerJarFilename) throws Exception {
     final ClassLoader agentParent;
     final String javaVersion = System.getProperty("java.version");
     if (javaVersion.startsWith("1.7") || javaVersion.startsWith("1.8")) {
@@ -183,13 +172,18 @@ public class TracingAgent {
       // platform classloader is parent of system in java 9+
       agentParent = getPlatformClassLoader();
     }
+
     final Class<?> loaderClass =
         ClassLoader.getSystemClassLoader().loadClass("datadog.trace.bootstrap.DatadogClassLoader");
     final Constructor constructor =
-        loaderClass.getDeclaredConstructor(URL.class, URL.class, ClassLoader.class);
+        loaderClass.getDeclaredConstructor(
+            URL.class, String.class, ClassLoader.class, ClassLoader.class);
     return (ClassLoader)
         constructor.newInstance(
-            bootstrapJar.toURI().toURL(), toolingJar.toURI().toURL(), agentParent);
+            bootstrapJar.toURI().toURL(),
+            innerJarFilename,
+            TracingAgent.class.getClassLoader(),
+            agentParent);
   }
 
   /** Extract sourcePath out of loader to a temporary file named destName. */