Merge pull request #110 from DataDog/tyler/spring-boot

Ensure rules are loaded on the right classpath for Spring Boot

dd-java-agent/dd-java-agent.gradle

@@ -37,7 +37,6 @@ dependencies {
     exclude(group: 'org.mongodb', module: 'mongodb-driver-async')
     exclude(group: 'org.mongodb', module: 'mongo-java-driver')
   }
-  compile group: 'io.opentracing.contrib', name: 'opentracing-jdbc', version: '0.0.2'
   compile(group: 'io.opentracing.contrib', name: 'opentracing-okhttp3', version: '0.0.5') {
     exclude(group: 'com.squareup.okhttp3', module: 'okhttp')
   }
@@ -69,6 +68,9 @@ dependencies {
   compileOnly group: 'com.amazonaws', name: 'aws-java-sdk', version: '1.11.119'
   compileOnly group: 'com.datastax.cassandra', name: 'cassandra-driver-core', version: '3.2.0'
   compileOnly group: 'org.apache.httpcomponents', name: 'httpclient', version: '4.5.3'
+
+  // Not bundled in with the agent.  Usage requires being on the app's classpath (eg. Spring Boot's executable jar)
+  compileOnly group: 'io.opentracing.contrib', name: 'opentracing-jdbc', version: '0.0.3'
 }
 
 jar {
@@ -95,7 +97,10 @@ shadowJar {
 
   if (!project.hasProperty("disableShadowRelocate") || !disableShadowRelocate) {
     // Don't relocate slf4j or opentracing deps.
-    relocate 'com.fasterxml', 'dd.deps.com.fasterxml'
+    relocate('com.fasterxml', 'dd.deps.com.fasterxml') {
+      exclude 'com.fasterxml.jackson.core.type.TypeReference'
+      exclude 'com.fasterxml.jackson.annotation.*'
+    }
     relocate 'javassist', 'dd.deps.javassist'
     relocate 'org.reflections', 'dd.deps.org.reflections'
     relocate 'org.yaml', 'dd.deps.org.yaml'

dd-java-agent/src/main/java/com/datadoghq/trace/agent/InstrumentationChecker.java

@@ -1,7 +1,6 @@
 package com.datadoghq.trace.agent;
 
 import com.datadoghq.trace.resolver.FactoryUtils;
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.core.type.TypeReference;
 import java.io.File;
@@ -180,6 +179,10 @@ public class InstrumentationChecker {
   }
 
   private boolean isClassPresent(final String identifyingPresentClass) {
+    return isClassPresent(identifyingPresentClass, classLoader);
+  }
+
+  static boolean isClassPresent(final String identifyingPresentClass, ClassLoader classLoader) {
     try {
       return identifyingPresentClass != null
           && Class.forName(identifyingPresentClass, false, classLoader) != null;
@@ -189,7 +192,6 @@ public class InstrumentationChecker {
   }
 
   @Data
-  @JsonIgnoreProperties("check")
   static class ArtifactSupport {
     private String artifact;
 

dd-java-agent/src/main/java/com/datadoghq/trace/agent/TraceAnnotationsManager.java

@@ -56,17 +56,18 @@ public class TraceAnnotationsManager {
           + "span.setTag(io.opentracing.tag.Tags.ERROR.getKey(),\"true\");\n"
           + "span.deactivate();\n";
   private static Retransformer transformer;
+  private static AgentTracerConfig agentTracerConfig;
 
   /**
    * This method initializes the manager.
    *
    * @param trans The ByteMan retransformer
    */
-  public static void initialize(final Retransformer trans) throws Exception {
+  public static void initialize(final Retransformer trans) {
     log.debug("Initializing {}", TraceAnnotationsManager.class.getSimpleName());
     transformer = trans;
     //Load configuration
-    final AgentTracerConfig agentTracerConfig =
+    agentTracerConfig =
         FactoryUtils.loadConfigFromFilePropertyOrResource(
             DDTracerFactory.SYSTEM_PROPERTY_CONFIG_PATH,
             DDTracerFactory.CONFIG_PATH,
@@ -74,7 +75,25 @@ public class TraceAnnotationsManager {
 
     log.debug("Configuration: {}", agentTracerConfig.toString());
 
-    final List<String> loadedScripts = loadRules(ClassLoader.getSystemClassLoader());
+    if (InstrumentationChecker.isClassPresent(
+        "org.springframework.boot.loader.LaunchedURLClassLoader",
+        ClassLoader.getSystemClassLoader())) {
+      log.info(
+          "Running in the context of a Spring Boot executable jar.  Deferring rule loading to run in the LaunchedURLClassLoader.");
+      loadRules("spring-boot-rule.btm", ClassLoader.getSystemClassLoader());
+    } else {
+      finishInitialization(ClassLoader.getSystemClassLoader());
+    }
+  }
+
+  /**
+   * This method is separated out from initialize to allow Spring Boot's LaunchedURLClassLoader to
+   * call it once it is loaded.
+   *
+   * @param classLoader
+   */
+  public static void finishInitialization(ClassLoader classLoader) {
+    final List<String> loadedScripts = loadRules(AGENT_RULES, classLoader);
 
     //Check if some rules have to be uninstalled
     final List<String> uninstallScripts =
@@ -85,7 +104,12 @@ public class TraceAnnotationsManager {
         uninstallScripts.addAll(disabledInstrumentations);
       }
     }
-    uninstallScripts(loadedScripts, uninstallScripts);
+
+    try {
+      uninstallScripts(loadedScripts, uninstallScripts);
+    } catch (Exception e) {
+      log.warn("Error uninstalling scripts", e);
+    }
 
     //Check if annotations are enabled
     if (agentTracerConfig != null
@@ -130,10 +154,13 @@ public class TraceAnnotationsManager {
    *
    * @param classLoader The classloader
    */
-  public static List<String> loadRules(final ClassLoader classLoader) {
+  public static List<String> loadRules(String rulesFileName, final ClassLoader classLoader) {
     final List<String> scripts = new ArrayList<>();
     if (transformer == null) {
-      log.warn("Attempt to load OpenTracing agent rules before transformer initialized");
+      log.warn(
+          "Attempt to load rules file {} on classloader {} before transformer initialized",
+          rulesFileName,
+          classLoader == null ? "bootstrap" : classLoader);
       return scripts;
     }
 
@@ -143,7 +170,7 @@ public class TraceAnnotationsManager {
 
     // Load default and custom rules
     try {
-      final Enumeration<URL> iter = classLoader.getResources(AGENT_RULES);
+      final Enumeration<URL> iter = classLoader.getResources(rulesFileName);
       while (iter.hasMoreElements()) {
         loadRules(iter.nextElement().toURI(), scriptNames, scripts);
       }
@@ -158,10 +185,10 @@ public class TraceAnnotationsManager {
       }
       log.debug(sw.toString());
     } catch (IOException | URISyntaxException e) {
-      log.warn("Failed to load OpenTracing agent rules", e);
+      log.warn("Failed to load rules", e);
     }
 
-    log.debug("OpenTracing Agent rules loaded");
+    log.debug("Rules loaded from {} on classloader {}", rulesFileName, classLoader);
     if (log.isTraceEnabled()) {
       for (final String rule : scripts) {
         log.trace("Loading rule: {}", rule);

dd-java-agent/src/main/resources/dd-trace-supported-framework.yaml

@@ -55,8 +55,7 @@ opentracing-mongo-driver:
       - com.mongodb.operation.AsyncReadOperation
       - com.mongodb.client.model.MapReduceAction
 
-  - check:
-    artifact: mongodb-driver-async
+  - artifact: mongodb-driver-async
     supported_version: 3\..*
     identifying_present_classes:
       - com.mongodb.operation.AsyncReadOperation
@@ -69,8 +68,7 @@ opentracing-mongo-driver-helper:
       - com.mongodb.operation.AsyncReadOperation
       - com.mongodb.client.model.MapReduceAction
 
-  - check:
-    artifact: mongodb-driver-async
+  - artifact: mongodb-driver-async
     supported_version: 3\..*
     identifying_present_classes:
       - com.mongodb.operation.AsyncReadOperation

dd-java-agent/src/main/resources/otarules.btm

@@ -44,7 +44,7 @@ HELPER com.datadoghq.trace.agent.integration.CassandraHelper
 AT EXIT
 IF TRUE
 DO
-   $! = patch($!);
+  $! = patch($!);
 ENDRULE
 
 
@@ -57,7 +57,7 @@ HELPER com.datadoghq.trace.agent.integration.JMSMessageProducerHelper
 AT EXIT
 IF TRUE
 DO
-   $! = patch($!);
+  $! = patch($!);
 ENDRULE
 
 
@@ -68,7 +68,7 @@ HELPER com.datadoghq.trace.agent.integration.JMSMessageConsumerHelper
 AT EXIT
 IF TRUE
 DO
-   $! = patch($!);
+  $! = patch($!);
 ENDRULE
 
 
@@ -104,7 +104,7 @@ HELPER com.datadoghq.trace.agent.integration.OkHttpHelper
 AT ENTRY
 IF TRUE
 DO
-    patch($this)
+  patch($this)
 ENDRULE
 
 
@@ -119,7 +119,7 @@ HELPER com.datadoghq.trace.agent.integration.JettyServletHelper
 AT EXIT
 IF getState($0.getServletContext()) == 0
 DO
-    patch($this)
+  patch($this)
 ENDRULE
 
 
@@ -128,10 +128,10 @@ ENDRULE
 RULE opentracing-web-servlet-filter_tomcat
 CLASS org.apache.catalina.core.ApplicationContext
 METHOD <init>
+COMPILE
 HELPER com.datadoghq.trace.agent.integration.TomcatServletHelper
 AT EXIT
 IF TRUE
 DO
-    patch($this)
+  patch($this)
 ENDRULE
-

dd-java-agent/src/main/resources/spring-boot-rule.btm

@@ -0,0 +1,13 @@
+# This rule is required to force the loading of instrumentation rules in otarules.btm when the Spring Boot classloader
+# is first instantiated, before any of the application components have been loaded. Scanning
+# the classloaders resources for OpenTracing agent rule files is far easier this way, then
+# having to scan the nested jar files.
+
+RULE Load rules via Spring Boot Classloader
+CLASS org.springframework.boot.loader.LaunchedURLClassLoader
+METHOD <init>
+AT EXIT
+IF TRUE
+DO
+  com.datadoghq.trace.agent.TraceAnnotationsManager.finishInitialization($0);
+ENDRULE

dd-trace-examples/dropwizard-mongo-client/README.md

@@ -12,7 +12,7 @@ auto-instrumentation for all endpoints. Manual instrumentation has been added as
 Be sure to build the project so that the latest version of ``dd-trace-java`` components are used. You can build
 all libraries and examples launching from the ``dd-trace-java`` root folder:
 ```bash
-./gradlew clean shadowJar
+./gradlew clean shadowJar installDist
 ```
 
 Then you can start all services via Docker:
@@ -30,6 +30,11 @@ Launch the application using the run wrapper you've built during the ``installDi
 JAVA_OPTS=-javaagent:../../dd-java-agent/build/libs/dd-java-agent-{version}.jar build/install/dropwizard-mongo-client/bin/dropwizard-mongo-client server
 ```
 
+Or as an executable jar:
+```bash
+java -javaagent:../../dd-java-agent/build/libs/dd-java-agent-{version}.jar -jar build/libs/dropwizard-mongo-client-demo-all.jar server
+```
+
 ``0.2.0-SNAPSHOT`` is an example of what ``{version}`` looks like.
 
 ### Generate traces

dd-trace-examples/dropwizard-mongo-client/dropwizard-mongo-client.gradle

@@ -1,3 +1,7 @@
+plugins {
+  id "com.github.johnrengelman.shadow" version "2.0.1"
+}
+
 apply plugin: 'application'
 apply from: "${rootDir}/gradle/java.gradle"
 apply from: "${rootDir}/gradle/jacoco.gradle"
@@ -21,6 +25,17 @@ dependencies {
   compile group: 'com.squareup.okhttp3', name: 'okhttp', version: '3.7.0'
 }
 
+jar {
+  manifest {
+    attributes 'Main-Class': 'com.datadoghq.example.dropwizard.BookApplication'
+  }
+}
+
+shadowJar {
+  mergeServiceFiles()
+}
+
+
 task wrapper(type: Wrapper) {
   gradleVersion = '4.0'
 }

dd-trace-examples/spring-boot-jdbc/README.md

@@ -12,7 +12,7 @@ to trace the endpoints.
 Be sure to build the project so that the latest version of ``dd-trace-java`` components are used. You can build
 all libraries and examples launching from the ``dd-trace-java`` root folder:
 ```bash
-./gradlew clean shadowJar
+./gradlew clean shadowJar bootRepackage
 ```
 
 Then you can launch the Datadog agent as follows:
@@ -30,9 +30,14 @@ To launch the application, just:
 ./gradlew bootRun
 ```
 
-The ``bootRun`` Gradle command appends automatically the ``-javaagent`` argument, so that you don't need to specify
+*Note: The ``bootRun`` Gradle command appends automatically the ``-javaagent`` argument, so that you don't need to specify
 the path of the Java Agent. Gradle executes the ``:dd-trace-examples:spring-boot-jdbc:bootRun`` task until you
-stop it.
+stop it.*
+
+Or as an executable jar:
+```bash
+java -javaagent:../../dd-java-agent/build/libs/dd-java-agent-{version}.jar -jar build/libs/spring-boot-jdbc-demo.jar
+```
 
 ### Generate traces
 
@@ -55,6 +60,6 @@ The Java Agent embeds the [OpenTracing Java Agent](https://github.com/opentracin
 
 #### Note for JDBC tracing configuration
 
-JDBC is not automatically instrumented by the Java Agent, so we changed the `application.properties`
-[file](src/main/resources/application.properties) to use the OpenTracing Driver. Without this step in your
-applications, the JDBC driver will not be instrumented.
+[JDBC is not automatically instrumented by the Java Agent](../../README.md#jdbc), so we changed the `application.properties`
+[file](src/main/resources/application.properties) to use the OpenTracing Driver and included it as a dependency in `spring-boot-jdbc.gradle`.
+Without these steps in your applications, the TracingDriver will not work.

dd-trace-examples/spring-boot-jdbc/spring-boot-jdbc.gradle

@@ -9,6 +9,7 @@ version = 'demo'
 description = 'spring-boot-jdbc'
 
 dependencies {
+  compile group: 'io.opentracing.contrib', name: 'opentracing-jdbc', version: '0.0.3'
   compile group: 'com.h2database', name: 'h2', version: '1.4.196'
   compile group: 'org.springframework.boot', name: 'spring-boot-starter-web', version: '1.5.4.RELEASE'
   compile group: 'org.springframework.boot', name: 'spring-boot-starter-data-jpa', version: '1.5.4.RELEASE'

dd-trace/src/main/java/com/datadoghq/trace/resolver/FactoryUtils.java

@@ -30,7 +30,7 @@ public class FactoryUtils {
   }
 
   public static <A> A loadConfigFromFilePropertyOrResource(
-      final String systemProperty, final String resourceName, final TypeReference type) {
+      final String systemProperty, final String resourceName, final TypeReference<A> type) {
     final String filePath = System.getProperty(systemProperty);
     if (filePath != null) {
       try {
@@ -74,7 +74,8 @@ public class FactoryUtils {
     return config;
   }
 
-  public static <A> A loadConfigFromResource(final String resourceName, final TypeReference type) {
+  public static <A> A loadConfigFromResource(
+      final String resourceName, final TypeReference<A> type) {
     A config = null;
 
     // Try loading both suffixes