From 9ff937557e0886cae31c7321c2d0fc8db96d9840 Mon Sep 17 00:00:00 2001 From: Tyler Benson Date: Wed, 23 Aug 2017 14:30:18 -0700 Subject: [PATCH 1/2] Ensure rules are loaded on the right classpath for Spring Boot --- dd-java-agent/dd-java-agent.gradle | 4 +- .../trace/agent/InstrumentationChecker.java | 4 ++ .../trace/agent/TraceAnnotationsManager.java | 45 +++++++++++++++---- dd-java-agent/src/main/resources/otarules.btm | 14 +++--- .../src/main/resources/spring-boot-rule.btm | 13 ++++++ dd-trace-examples/spring-boot-jdbc/README.md | 6 +-- .../spring-boot-jdbc/spring-boot-jdbc.gradle | 1 + 7 files changed, 67 insertions(+), 20 deletions(-) create mode 100644 dd-java-agent/src/main/resources/spring-boot-rule.btm diff --git a/dd-java-agent/dd-java-agent.gradle b/dd-java-agent/dd-java-agent.gradle index ffca0e0e10..0843c9946c 100644 --- a/dd-java-agent/dd-java-agent.gradle +++ b/dd-java-agent/dd-java-agent.gradle @@ -37,7 +37,6 @@ dependencies { exclude(group: 'org.mongodb', module: 'mongodb-driver-async') exclude(group: 'org.mongodb', module: 'mongo-java-driver') } - compile group: 'io.opentracing.contrib', name: 'opentracing-jdbc', version: '0.0.2' compile(group: 'io.opentracing.contrib', name: 'opentracing-okhttp3', version: '0.0.5') { exclude(group: 'com.squareup.okhttp3', module: 'okhttp') } @@ -69,6 +68,9 @@ dependencies { compileOnly group: 'com.amazonaws', name: 'aws-java-sdk', version: '1.11.119' compileOnly group: 'com.datastax.cassandra', name: 'cassandra-driver-core', version: '3.2.0' compileOnly group: 'org.apache.httpcomponents', name: 'httpclient', version: '4.5.3' + + // Not bundled in with the agent. Usage requires being on the app's classpath (eg. Spring Boot's executable jar) + compileOnly group: 'io.opentracing.contrib', name: 'opentracing-jdbc', version: '0.0.3' } jar { diff --git a/dd-java-agent/src/main/java/com/datadoghq/trace/agent/InstrumentationChecker.java b/dd-java-agent/src/main/java/com/datadoghq/trace/agent/InstrumentationChecker.java index 1d2483b87e..e0ac560b43 100644 --- a/dd-java-agent/src/main/java/com/datadoghq/trace/agent/InstrumentationChecker.java +++ b/dd-java-agent/src/main/java/com/datadoghq/trace/agent/InstrumentationChecker.java @@ -180,6 +180,10 @@ public class InstrumentationChecker { } private boolean isClassPresent(final String identifyingPresentClass) { + return isClassPresent(identifyingPresentClass, classLoader); + } + + static boolean isClassPresent(final String identifyingPresentClass, ClassLoader classLoader) { try { return identifyingPresentClass != null && Class.forName(identifyingPresentClass, false, classLoader) != null; diff --git a/dd-java-agent/src/main/java/com/datadoghq/trace/agent/TraceAnnotationsManager.java b/dd-java-agent/src/main/java/com/datadoghq/trace/agent/TraceAnnotationsManager.java index 404466dd50..f62a056a39 100644 --- a/dd-java-agent/src/main/java/com/datadoghq/trace/agent/TraceAnnotationsManager.java +++ b/dd-java-agent/src/main/java/com/datadoghq/trace/agent/TraceAnnotationsManager.java @@ -56,17 +56,18 @@ public class TraceAnnotationsManager { + "span.setTag(io.opentracing.tag.Tags.ERROR.getKey(),\"true\");\n" + "span.deactivate();\n"; private static Retransformer transformer; + private static AgentTracerConfig agentTracerConfig; /** * This method initializes the manager. * * @param trans The ByteMan retransformer */ - public static void initialize(final Retransformer trans) throws Exception { + public static void initialize(final Retransformer trans) { log.debug("Initializing {}", TraceAnnotationsManager.class.getSimpleName()); transformer = trans; //Load configuration - final AgentTracerConfig agentTracerConfig = + agentTracerConfig = FactoryUtils.loadConfigFromFilePropertyOrResource( DDTracerFactory.SYSTEM_PROPERTY_CONFIG_PATH, DDTracerFactory.CONFIG_PATH, @@ -74,7 +75,25 @@ public class TraceAnnotationsManager { log.debug("Configuration: {}", agentTracerConfig.toString()); - final List loadedScripts = loadRules(ClassLoader.getSystemClassLoader()); + if (InstrumentationChecker.isClassPresent( + "org.springframework.boot.loader.LaunchedURLClassLoader", + ClassLoader.getSystemClassLoader())) { + log.info( + "Running in the context of a Spring Boot executable jar. Deferring rule loading to run in the LaunchedURLClassLoader."); + loadRules("spring-boot-rule.btm", ClassLoader.getSystemClassLoader()); + } else { + finishInitialization(ClassLoader.getSystemClassLoader()); + } + } + + /** + * This method is separated out from initialize to allow Spring Boot's LaunchedURLClassLoader to + * call it once it is loaded. + * + * @param classLoader + */ + public static void finishInitialization(ClassLoader classLoader) { + final List loadedScripts = loadRules(AGENT_RULES, classLoader); //Check if some rules have to be uninstalled final List uninstallScripts = @@ -85,7 +104,12 @@ public class TraceAnnotationsManager { uninstallScripts.addAll(disabledInstrumentations); } } - uninstallScripts(loadedScripts, uninstallScripts); + + try { + uninstallScripts(loadedScripts, uninstallScripts); + } catch (Exception e) { + log.warn("Error uninstalling scripts", e); + } //Check if annotations are enabled if (agentTracerConfig != null @@ -130,10 +154,13 @@ public class TraceAnnotationsManager { * * @param classLoader The classloader */ - public static List loadRules(final ClassLoader classLoader) { + public static List loadRules(String rulesFileName, final ClassLoader classLoader) { final List scripts = new ArrayList<>(); if (transformer == null) { - log.warn("Attempt to load OpenTracing agent rules before transformer initialized"); + log.warn( + "Attempt to load rules file {} on classloader {} before transformer initialized", + rulesFileName, + classLoader == null ? "bootstrap" : classLoader); return scripts; } @@ -143,7 +170,7 @@ public class TraceAnnotationsManager { // Load default and custom rules try { - final Enumeration iter = classLoader.getResources(AGENT_RULES); + final Enumeration iter = classLoader.getResources(rulesFileName); while (iter.hasMoreElements()) { loadRules(iter.nextElement().toURI(), scriptNames, scripts); } @@ -158,10 +185,10 @@ public class TraceAnnotationsManager { } log.debug(sw.toString()); } catch (IOException | URISyntaxException e) { - log.warn("Failed to load OpenTracing agent rules", e); + log.warn("Failed to load rules", e); } - log.debug("OpenTracing Agent rules loaded"); + log.debug("Rules loaded from {} on classloader {}", rulesFileName, classLoader); if (log.isTraceEnabled()) { for (final String rule : scripts) { log.trace("Loading rule: {}", rule); diff --git a/dd-java-agent/src/main/resources/otarules.btm b/dd-java-agent/src/main/resources/otarules.btm index 1f50d23637..244b5f2025 100644 --- a/dd-java-agent/src/main/resources/otarules.btm +++ b/dd-java-agent/src/main/resources/otarules.btm @@ -44,7 +44,7 @@ HELPER com.datadoghq.trace.agent.integration.CassandraHelper AT EXIT IF TRUE DO - $! = patch($!); + $! = patch($!); ENDRULE @@ -57,7 +57,7 @@ HELPER com.datadoghq.trace.agent.integration.JMSMessageProducerHelper AT EXIT IF TRUE DO - $! = patch($!); + $! = patch($!); ENDRULE @@ -68,7 +68,7 @@ HELPER com.datadoghq.trace.agent.integration.JMSMessageConsumerHelper AT EXIT IF TRUE DO - $! = patch($!); + $! = patch($!); ENDRULE @@ -104,7 +104,7 @@ HELPER com.datadoghq.trace.agent.integration.OkHttpHelper AT ENTRY IF TRUE DO - patch($this) + patch($this) ENDRULE @@ -119,7 +119,7 @@ HELPER com.datadoghq.trace.agent.integration.JettyServletHelper AT EXIT IF getState($0.getServletContext()) == 0 DO - patch($this) + patch($this) ENDRULE @@ -128,10 +128,10 @@ ENDRULE RULE opentracing-web-servlet-filter_tomcat CLASS org.apache.catalina.core.ApplicationContext METHOD +COMPILE HELPER com.datadoghq.trace.agent.integration.TomcatServletHelper AT EXIT IF TRUE DO - patch($this) + patch($this) ENDRULE - diff --git a/dd-java-agent/src/main/resources/spring-boot-rule.btm b/dd-java-agent/src/main/resources/spring-boot-rule.btm new file mode 100644 index 0000000000..e470d2d41d --- /dev/null +++ b/dd-java-agent/src/main/resources/spring-boot-rule.btm @@ -0,0 +1,13 @@ +# This rule is required to force the loading of instrumentation rules in otarules.btm when the Spring Boot classloader +# is first instantiated, before any of the application components have been loaded. Scanning +# the classloaders resources for OpenTracing agent rule files is far easier this way, then +# having to scan the nested jar files. + +RULE Load rules via Spring Boot Classloader +CLASS org.springframework.boot.loader.LaunchedURLClassLoader +METHOD +AT EXIT +IF TRUE +DO + com.datadoghq.trace.agent.TraceAnnotationsManager.finishInitialization($0); +ENDRULE diff --git a/dd-trace-examples/spring-boot-jdbc/README.md b/dd-trace-examples/spring-boot-jdbc/README.md index 7052d4a4c0..b38a662227 100644 --- a/dd-trace-examples/spring-boot-jdbc/README.md +++ b/dd-trace-examples/spring-boot-jdbc/README.md @@ -55,6 +55,6 @@ The Java Agent embeds the [OpenTracing Java Agent](https://github.com/opentracin #### Note for JDBC tracing configuration -JDBC is not automatically instrumented by the Java Agent, so we changed the `application.properties` -[file](src/main/resources/application.properties) to use the OpenTracing Driver. Without this step in your -applications, the JDBC driver will not be instrumented. +[JDBC is not automatically instrumented by the Java Agent](../../README.md#jdbc), so we changed the `application.properties` +[file](src/main/resources/application.properties) to use the OpenTracing Driver and included it as a dependency in `spring-boot-jdbc.gradle`. +Without these steps in your applications, the TracingDriver will not work. diff --git a/dd-trace-examples/spring-boot-jdbc/spring-boot-jdbc.gradle b/dd-trace-examples/spring-boot-jdbc/spring-boot-jdbc.gradle index 0b9f7a3ffe..cc29156f4c 100644 --- a/dd-trace-examples/spring-boot-jdbc/spring-boot-jdbc.gradle +++ b/dd-trace-examples/spring-boot-jdbc/spring-boot-jdbc.gradle @@ -9,6 +9,7 @@ version = 'demo' description = 'spring-boot-jdbc' dependencies { + compile group: 'io.opentracing.contrib', name: 'opentracing-jdbc', version: '0.0.3' compile group: 'com.h2database', name: 'h2', version: '1.4.196' compile group: 'org.springframework.boot', name: 'spring-boot-starter-web', version: '1.5.4.RELEASE' compile group: 'org.springframework.boot', name: 'spring-boot-starter-data-jpa', version: '1.5.4.RELEASE' From ab7598804e8789a91689e1849ee73e241150c1d2 Mon Sep 17 00:00:00 2001 From: Tyler Benson Date: Wed, 23 Aug 2017 15:29:34 -0700 Subject: [PATCH 2/2] Update dropwizard and spring boot readmes. --- dd-java-agent/dd-java-agent.gradle | 5 ++++- .../trace/agent/InstrumentationChecker.java | 2 -- .../resources/dd-trace-supported-framework.yaml | 6 ++---- .../dropwizard-mongo-client/README.md | 7 ++++++- .../dropwizard-mongo-client.gradle | 15 +++++++++++++++ dd-trace-examples/spring-boot-jdbc/README.md | 11 ++++++++--- .../datadoghq/trace/resolver/FactoryUtils.java | 5 +++-- 7 files changed, 38 insertions(+), 13 deletions(-) diff --git a/dd-java-agent/dd-java-agent.gradle b/dd-java-agent/dd-java-agent.gradle index 0843c9946c..bedf7efc06 100644 --- a/dd-java-agent/dd-java-agent.gradle +++ b/dd-java-agent/dd-java-agent.gradle @@ -97,7 +97,10 @@ shadowJar { if (!project.hasProperty("disableShadowRelocate") || !disableShadowRelocate) { // Don't relocate slf4j or opentracing deps. - relocate 'com.fasterxml', 'dd.deps.com.fasterxml' + relocate('com.fasterxml', 'dd.deps.com.fasterxml') { + exclude 'com.fasterxml.jackson.core.type.TypeReference' + exclude 'com.fasterxml.jackson.annotation.*' + } relocate 'javassist', 'dd.deps.javassist' relocate 'org.reflections', 'dd.deps.org.reflections' relocate 'org.yaml', 'dd.deps.org.yaml' diff --git a/dd-java-agent/src/main/java/com/datadoghq/trace/agent/InstrumentationChecker.java b/dd-java-agent/src/main/java/com/datadoghq/trace/agent/InstrumentationChecker.java index e0ac560b43..603b63bdec 100644 --- a/dd-java-agent/src/main/java/com/datadoghq/trace/agent/InstrumentationChecker.java +++ b/dd-java-agent/src/main/java/com/datadoghq/trace/agent/InstrumentationChecker.java @@ -1,7 +1,6 @@ package com.datadoghq.trace.agent; import com.datadoghq.trace.resolver.FactoryUtils; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.type.TypeReference; import java.io.File; @@ -193,7 +192,6 @@ public class InstrumentationChecker { } @Data - @JsonIgnoreProperties("check") static class ArtifactSupport { private String artifact; diff --git a/dd-java-agent/src/main/resources/dd-trace-supported-framework.yaml b/dd-java-agent/src/main/resources/dd-trace-supported-framework.yaml index 3c98c42095..789052d8b8 100644 --- a/dd-java-agent/src/main/resources/dd-trace-supported-framework.yaml +++ b/dd-java-agent/src/main/resources/dd-trace-supported-framework.yaml @@ -55,8 +55,7 @@ opentracing-mongo-driver: - com.mongodb.operation.AsyncReadOperation - com.mongodb.client.model.MapReduceAction - - check: - artifact: mongodb-driver-async + - artifact: mongodb-driver-async supported_version: 3\..* identifying_present_classes: - com.mongodb.operation.AsyncReadOperation @@ -69,8 +68,7 @@ opentracing-mongo-driver-helper: - com.mongodb.operation.AsyncReadOperation - com.mongodb.client.model.MapReduceAction - - check: - artifact: mongodb-driver-async + - artifact: mongodb-driver-async supported_version: 3\..* identifying_present_classes: - com.mongodb.operation.AsyncReadOperation diff --git a/dd-trace-examples/dropwizard-mongo-client/README.md b/dd-trace-examples/dropwizard-mongo-client/README.md index 71b4293760..47a771043d 100644 --- a/dd-trace-examples/dropwizard-mongo-client/README.md +++ b/dd-trace-examples/dropwizard-mongo-client/README.md @@ -12,7 +12,7 @@ auto-instrumentation for all endpoints. Manual instrumentation has been added as Be sure to build the project so that the latest version of ``dd-trace-java`` components are used. You can build all libraries and examples launching from the ``dd-trace-java`` root folder: ```bash -./gradlew clean shadowJar +./gradlew clean shadowJar installDist ``` Then you can start all services via Docker: @@ -30,6 +30,11 @@ Launch the application using the run wrapper you've built during the ``installDi JAVA_OPTS=-javaagent:../../dd-java-agent/build/libs/dd-java-agent-{version}.jar build/install/dropwizard-mongo-client/bin/dropwizard-mongo-client server ``` +Or as an executable jar: +```bash +java -javaagent:../../dd-java-agent/build/libs/dd-java-agent-{version}.jar -jar build/libs/dropwizard-mongo-client-demo-all.jar server +``` + ``0.2.0-SNAPSHOT`` is an example of what ``{version}`` looks like. ### Generate traces diff --git a/dd-trace-examples/dropwizard-mongo-client/dropwizard-mongo-client.gradle b/dd-trace-examples/dropwizard-mongo-client/dropwizard-mongo-client.gradle index 720e7f21f5..dbf0a1ff76 100644 --- a/dd-trace-examples/dropwizard-mongo-client/dropwizard-mongo-client.gradle +++ b/dd-trace-examples/dropwizard-mongo-client/dropwizard-mongo-client.gradle @@ -1,3 +1,7 @@ +plugins { + id "com.github.johnrengelman.shadow" version "2.0.1" +} + apply plugin: 'application' apply from: "${rootDir}/gradle/java.gradle" apply from: "${rootDir}/gradle/jacoco.gradle" @@ -21,6 +25,17 @@ dependencies { compile group: 'com.squareup.okhttp3', name: 'okhttp', version: '3.7.0' } +jar { + manifest { + attributes 'Main-Class': 'com.datadoghq.example.dropwizard.BookApplication' + } +} + +shadowJar { + mergeServiceFiles() +} + + task wrapper(type: Wrapper) { gradleVersion = '4.0' } diff --git a/dd-trace-examples/spring-boot-jdbc/README.md b/dd-trace-examples/spring-boot-jdbc/README.md index b38a662227..66c0854d5b 100644 --- a/dd-trace-examples/spring-boot-jdbc/README.md +++ b/dd-trace-examples/spring-boot-jdbc/README.md @@ -12,7 +12,7 @@ to trace the endpoints. Be sure to build the project so that the latest version of ``dd-trace-java`` components are used. You can build all libraries and examples launching from the ``dd-trace-java`` root folder: ```bash -./gradlew clean shadowJar +./gradlew clean shadowJar bootRepackage ``` Then you can launch the Datadog agent as follows: @@ -30,9 +30,14 @@ To launch the application, just: ./gradlew bootRun ``` -The ``bootRun`` Gradle command appends automatically the ``-javaagent`` argument, so that you don't need to specify +*Note: The ``bootRun`` Gradle command appends automatically the ``-javaagent`` argument, so that you don't need to specify the path of the Java Agent. Gradle executes the ``:dd-trace-examples:spring-boot-jdbc:bootRun`` task until you -stop it. +stop it.* + +Or as an executable jar: +```bash +java -javaagent:../../dd-java-agent/build/libs/dd-java-agent-{version}.jar -jar build/libs/spring-boot-jdbc-demo.jar +``` ### Generate traces diff --git a/dd-trace/src/main/java/com/datadoghq/trace/resolver/FactoryUtils.java b/dd-trace/src/main/java/com/datadoghq/trace/resolver/FactoryUtils.java index 5580994960..288573cd9e 100644 --- a/dd-trace/src/main/java/com/datadoghq/trace/resolver/FactoryUtils.java +++ b/dd-trace/src/main/java/com/datadoghq/trace/resolver/FactoryUtils.java @@ -30,7 +30,7 @@ public class FactoryUtils { } public static A loadConfigFromFilePropertyOrResource( - final String systemProperty, final String resourceName, final TypeReference type) { + final String systemProperty, final String resourceName, final TypeReference type) { final String filePath = System.getProperty(systemProperty); if (filePath != null) { try { @@ -74,7 +74,8 @@ public class FactoryUtils { return config; } - public static A loadConfigFromResource(final String resourceName, final TypeReference type) { + public static A loadConfigFromResource( + final String resourceName, final TypeReference type) { A config = null; // Try loading both suffixes