Merge pull request #383 from DataDog/ark/muzzle-hookup

Enable Muzzle validation for Default Instrumentation Match phase

.circleci/config.yml

@@ -187,8 +187,8 @@ jobs:
             - dd-trace-java-version-scan
 
       - run:
-          name: Verify Version Scan
+          name: Verify Version Scan and Muzzle
-          command: ./gradlew verifyVersionScan --parallel --stacktrace --no-daemon --max-workers=6
+          command: ./gradlew verifyVersionScan muzzle --parallel --stacktrace --no-daemon --max-workers=6
 
       - save_cache:
           key: dd-trace-java-version-scan-{{ checksum "dd-trace-java.gradle" }}

buildSrc/src/main/groovy/MuzzleExtension.groovy

@@ -0,0 +1,4 @@
+class MuzzleExtension {
+  String group
+  String module
+}

buildSrc/src/main/groovy/MuzzlePlugin.groovy

@@ -0,0 +1,58 @@
+import org.gradle.api.Plugin
+import org.gradle.api.Project
+
+import java.lang.reflect.Method
+
+/**
+ * muzzle task plugin which runs muzzle validation against an instrumentation's compile-time dependencies.
+ *
+ * <p/>TODO: merge this with version scan
+ */
+class MuzzlePlugin implements Plugin<Project> {
+  @Override
+  void apply(Project project) {
+    def bootstrapProject = project.rootProject.getChildProjects().get('dd-java-agent').getChildProjects().get('agent-bootstrap')
+    def toolingProject = project.rootProject.getChildProjects().get('dd-java-agent').getChildProjects().get('agent-tooling')
+    project.extensions.create("muzzle", MuzzleExtension)
+    def muzzle = project.task('muzzle') {
+      group = 'Muzzle'
+      description = "Run instrumentation muzzle on compile time dependencies"
+      doLast {
+        List<URL> userUrls = new ArrayList<>()
+        project.getLogger().info("Creating user classpath for: " + project.getName())
+        for (File f : project.configurations.compileOnly.getFiles()) {
+          project.getLogger().info( '--' + f)
+          userUrls.add(f.toURI().toURL())
+        }
+        for (File f : bootstrapProject.sourceSets.main.runtimeClasspath.getFiles()) {
+          project.getLogger().info( '--' + f)
+          userUrls.add(f.toURI().toURL())
+        }
+        final ClassLoader userCL = new URLClassLoader(userUrls.toArray(new URL[0]), (ClassLoader) null)
+
+        project.getLogger().info("Creating dd classpath for: " + project.getName())
+        Set<URL> ddUrls = new HashSet<>()
+        for (File f : toolingProject.sourceSets.main.runtimeClasspath.getFiles()) {
+          project.getLogger().info( '--' + f)
+          ddUrls.add(f.toURI().toURL())
+        }
+        for(File f : project.sourceSets.main.runtimeClasspath.getFiles()) {
+          project.getLogger().info( '--' + f)
+          ddUrls.add(f.toURI().toURL())
+        }
+
+        final ClassLoader agentCL = new URLClassLoader(ddUrls.toArray(new URL[0]), (ClassLoader) null)
+        // find all instrumenters, get muzzle, and assert
+        Method assertionMethod = agentCL.loadClass('datadog.trace.agent.tooling.muzzle.MuzzleVersionScanPlugin')
+          .getMethod('assertInstrumentationNotMuzzled', ClassLoader.class)
+        assertionMethod.invoke(null, userCL)
+      }
+    }
+    // project.tasks.muzzle.dependsOn(bootstrapProject.tasks.shadowJar)
+    project.tasks.muzzle.dependsOn(bootstrapProject.tasks.compileJava)
+    project.tasks.muzzle.dependsOn(toolingProject.tasks.compileJava)
+    project.afterEvaluate {
+      project.tasks.muzzle.dependsOn(project.tasks.compileJava)
+    }
+  }
+}

buildSrc/src/main/resources/META-INF/gradle-plugins/muzzle.properties

@@ -0,0 +1 @@
+implementation-class=MuzzlePlugin

dd-java-agent-ittests/src/test/groovy/datadog/trace/agent/integration/muzzle/MuzzleBytecodeTransformTest.groovy

@@ -8,13 +8,17 @@ import spock.lang.Specification
 
 class MuzzleBytecodeTransformTest extends Specification {
 
-  /*
   def "muzzle fields added to all instrumentation"() {
     setup:
     List<Class> unMuzzledClasses = []
     List<Class> nonLazyFields = []
     List<Class> unInitFields = []
-    for (final Object instrumenter : ServiceLoader.load(IntegrationTestUtils.getAgentClassLoader().loadClass("datadog.trace.agent.tooling.Instrumenter"), IntegrationTestUtils.getAgentClassLoader())) {
+    for (Object instrumenter : ServiceLoader.load(IntegrationTestUtils.getAgentClassLoader().loadClass("datadog.trace.agent.tooling.Instrumenter"), IntegrationTestUtils.getAgentClassLoader())) {
+      if (instrumenter.getClass().getName().endsWith("TraceConfigInstrumentation")) {
+        // TraceConfigInstrumentation doesn't do muzzle checks
+        // check on TracerClassInstrumentation instead
+        instrumenter = IntegrationTestUtils.getAgentClassLoader().loadClass(instrumenter.getClass().getName() + '$TracerClassInstrumentation').newInstance()
+      }
       Field f
       Method m
       try {
@@ -45,6 +49,5 @@ class MuzzleBytecodeTransformTest extends Specification {
     nonLazyFields == []
     unInitFields == []
   }
-  */
 
 }

dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/AgentInstaller.java

@@ -8,8 +8,6 @@ import static net.bytebuddy.matcher.ElementMatchers.nameStartsWith;
 import static net.bytebuddy.matcher.ElementMatchers.named;
 import static net.bytebuddy.matcher.ElementMatchers.not;
 
-import datadog.trace.agent.tooling.muzzle.Reference.Mismatch;
-import datadog.trace.agent.tooling.muzzle.ReferenceMatcher.MismatchException;
 import java.lang.instrument.Instrumentation;
 import java.util.ServiceLoader;
 import lombok.extern.slf4j.Slf4j;
@@ -91,19 +89,11 @@ public class AgentInstaller {
         final JavaModule module,
         final boolean loaded,
         final Throwable throwable) {
-      if (throwable instanceof MismatchException) {
+      log.debug(
-        final MismatchException mismatchException = (MismatchException) throwable;
+          "Failed to handle {} for transformation on classloader {}: {}",
-        log.debug("{}", mismatchException.getMessage());
+          typeName,
-        for (final Mismatch mismatch : mismatchException.getMismatches()) {
+          classLoader,
-          log.debug("--{}", mismatch);
+          throwable.getMessage());
-        }
-      } else {
-        log.debug(
-            "Failed to handle {} for transformation on classloader {}: {}",
-            typeName,
-            classLoader,
-            throwable.getMessage());
-      }
     }
 
     @Override

dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/Instrumenter.java

@@ -3,14 +3,19 @@ package datadog.trace.agent.tooling;
 import static datadog.trace.agent.tooling.Utils.getConfigEnabled;
 import static net.bytebuddy.matcher.ElementMatchers.any;
 
+import datadog.trace.agent.tooling.muzzle.Reference;
+import datadog.trace.agent.tooling.muzzle.ReferenceMatcher;
+import java.security.ProtectionDomain;
 import java.util.Arrays;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import lombok.extern.slf4j.Slf4j;
 import net.bytebuddy.agent.builder.AgentBuilder;
 import net.bytebuddy.description.type.TypeDescription;
 import net.bytebuddy.matcher.ElementMatcher;
+import net.bytebuddy.utility.JavaModule;
 
 /**
  * Built-in bytebuddy-based instrumentation for the datadog javaagent.
@@ -42,11 +47,13 @@ public interface Instrumenter {
   @Slf4j
   abstract class Default implements Instrumenter {
     private final Set<String> instrumentationNames;
+    private final String instrumentationPrimaryName;
     protected final boolean enabled;
 
     public Default(final String instrumentationName, final String... additionalNames) {
       this.instrumentationNames = new HashSet<>(Arrays.asList(additionalNames));
       instrumentationNames.add(instrumentationName);
+      instrumentationPrimaryName = instrumentationName;
 
       // If default is enabled, we want to enable individually,
       // if default is disabled, we want to disable individually.
@@ -74,6 +81,35 @@ public interface Instrumenter {
       AgentBuilder.Identified.Extendable advice =
           agentBuilder
               .type(typeMatcher(), classLoaderMatcher())
+              .and(
+                  new AgentBuilder.RawMatcher() {
+                    @Override
+                    public boolean matches(
+                        TypeDescription typeDescription,
+                        ClassLoader classLoader,
+                        JavaModule module,
+                        Class<?> classBeingRedefined,
+                        ProtectionDomain protectionDomain) {
+                      // Optimization: calling getInstrumentationMuzzle() inside this method prevents unnecessary loading of muzzle references during agentBuilder setup.
+                      final ReferenceMatcher muzzle = getInstrumentationMuzzle();
+                      if (null != muzzle) {
+                        List<Reference.Mismatch> mismatches =
+                            muzzle.getMismatchedReferenceSources(classLoader);
+                        if (mismatches.size() > 0) {
+                          log.debug(
+                              "Instrumentation muzzled: {} -- {} on {}",
+                              instrumentationPrimaryName,
+                              this.getClass().getName(),
+                              classLoader);
+                        }
+                        for (Reference.Mismatch mismatch : mismatches) {
+                          log.debug("-- {}", mismatch);
+                        }
+                        return mismatches.size() == 0;
+                      }
+                      return true;
+                    }
+                  })
               .transform(DDTransformers.defaultTransformers());
       final String[] helperClassNames = helperClassNames();
       if (helperClassNames.length > 0) {
@@ -85,6 +121,15 @@ public interface Instrumenter {
       return advice.asDecorator();
     }
 
+    /**
+     * This method is implemented dynamically by compile-time bytecode transformations.
+     *
+     * <p>{@see datadog.trace.agent.tooling.muzzle.MuzzleGradlePlugin}
+     */
+    protected ReferenceMatcher getInstrumentationMuzzle() {
+      return null;
+    }
+
     @Override
     public String[] helperClassNames() {
       return new String[0];
@@ -105,11 +150,13 @@ public interface Instrumenter {
       return getConfigEnabled("dd.integrations.enabled", true);
     }
 
-    protected static String getPropOrEnv(final String name) {
+    // TODO: move common config helpers to Utils
+
+    public static String getPropOrEnv(final String name) {
       return System.getProperty(name, System.getenv(propToEnvName(name)));
     }
 
-    private static String propToEnvName(final String name) {
+    public static String propToEnvName(final String name) {
       return name.toUpperCase().replace(".", "_");
     }
   }

dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/muzzle/MuzzleGradlePlugin.java

@@ -6,44 +6,24 @@ import net.bytebuddy.description.type.TypeDefinition;
 import net.bytebuddy.description.type.TypeDescription;
 import net.bytebuddy.dynamic.DynamicType.Builder;
 
+/** Bytebuddy gradle plugin which creates muzzle-references at compile time. */
 public class MuzzleGradlePlugin implements Plugin {
-  // TODO:
+  private static final TypeDescription DefaultInstrumenterTypeDesc =
-  // - Optimizations
+      new TypeDescription.ForLoadedType(Instrumenter.Default.class);
-  //   - Cache safe and unsafe classloaders
-  //   - Do reference generation at compile time
-  //   - lazy-load reference muzzle field
-  // - Additional references to check
-  //   - Fields
-  //   - methods
-  //     - visit annotations
-  //     - visit parameter types
-  //     - visit method instructions
-  //     - method invoke type
-  //   - access flags (including implicit package-private)
-  //   - supertypes
-  // - Misc
-  //   - Also match interfaces which extend Instrumenter
-  //   - Expose config instead of hardcoding datadog namespace (or reconfigure classpath)
-  //   - Run muzzle in matching phase (may require a rewrite of the instrumentation api)
-  //   - Documentation
-
-  private static final TypeDescription InstrumenterTypeDesc =
-      new TypeDescription.ForLoadedType(Instrumenter.class);
 
   @Override
   public boolean matches(final TypeDescription target) {
-    // AutoService annotation is not retained at runtime. Check for instrumenter supertype
+    // AutoService annotation is not retained at runtime. Check for Instrumenter.Default supertype
     boolean isInstrumenter = false;
-    TypeDefinition instrumenter = target;
+    TypeDefinition instrumenter = null == target ? null : target.getSuperClass();
     while (instrumenter != null) {
-      if (instrumenter.getInterfaces().contains(InstrumenterTypeDesc)) {
+      if (instrumenter.equals(DefaultInstrumenterTypeDesc)) {
         isInstrumenter = true;
         break;
       }
       instrumenter = instrumenter.getSuperClass();
     }
-    // return isInstrumenter;
+    return isInstrumenter;
-    return false;
   }
 
   @Override
@@ -51,6 +31,7 @@ public class MuzzleGradlePlugin implements Plugin {
     return builder.visit(new MuzzleVisitor());
   }
 
+  /** Compile-time Optimization used by gradle buildscripts. */
   public static class NoOp implements Plugin {
     @Override
     public boolean matches(final TypeDescription target) {

dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/muzzle/MuzzleVersionScanPlugin.java

@@ -0,0 +1,86 @@
+package datadog.trace.agent.tooling.muzzle;
+
+import datadog.trace.agent.tooling.HelperInjector;
+import datadog.trace.agent.tooling.Instrumenter;
+import java.lang.reflect.Method;
+import java.util.List;
+import java.util.ServiceLoader;
+
+/**
+ * Entry point for muzzle version scan gradle plugin.
+ *
+ * <p>For each instrumenter on the classpath, run muzzle validation and throw an exception if any
+ * mismatches are detected.
+ *
+ * <p>Additionally, after a successful muzzle validation run each instrumenter's helper injector.
+ */
+public class MuzzleVersionScanPlugin {
+  public static void assertInstrumentationNotMuzzled(ClassLoader cl) throws Exception {
+    // muzzle validate all instrumenters
+    for (Instrumenter instrumenter :
+        ServiceLoader.load(Instrumenter.class, MuzzleGradlePlugin.class.getClassLoader())) {
+      if (instrumenter.getClass().getName().endsWith("TraceConfigInstrumentation")) {
+        // TraceConfigInstrumentation doesn't do muzzle checks
+        // check on TracerClassInstrumentation instead
+        instrumenter =
+            (Instrumenter)
+                MuzzleGradlePlugin.class
+                    .getClassLoader()
+                    .loadClass(instrumenter.getClass().getName() + "$TracerClassInstrumentation")
+                    .getDeclaredConstructor()
+                    .newInstance();
+      }
+      Method m = null;
+      try {
+        m = instrumenter.getClass().getDeclaredMethod("getInstrumentationMuzzle");
+        m.setAccessible(true);
+        ReferenceMatcher muzzle = (ReferenceMatcher) m.invoke(instrumenter);
+        List<Reference.Mismatch> mismatches = muzzle.getMismatchedReferenceSources(cl);
+        if (mismatches.size() > 0) {
+          System.err.println(
+              "FAILED MUZZLE VALIDATION: " + instrumenter.getClass().getName() + " mismatches:");
+          for (Reference.Mismatch mismatch : mismatches) {
+            System.err.println("-- " + mismatch);
+          }
+          throw new RuntimeException("Instrumentation failed Muzzle validation");
+        }
+      } finally {
+        if (null != m) {
+          m.setAccessible(false);
+        }
+      }
+    }
+    // run helper injector on all instrumenters
+    for (Instrumenter instrumenter :
+        ServiceLoader.load(Instrumenter.class, MuzzleGradlePlugin.class.getClassLoader())) {
+      if (instrumenter.getClass().getName().endsWith("TraceConfigInstrumentation")) {
+        // TraceConfigInstrumentation doesn't do muzzle checks
+        // check on TracerClassInstrumentation instead
+        instrumenter =
+            (Instrumenter)
+                MuzzleGradlePlugin.class
+                    .getClassLoader()
+                    .loadClass(instrumenter.getClass().getName() + "$TracerClassInstrumentation")
+                    .getDeclaredConstructor()
+                    .newInstance();
+      }
+      try {
+        // Ratpack injects the scope manager as a helper.
+        // This is likely a bug, but we'll grandfather it out of the helper checks for now.
+        if (!instrumenter.getClass().getName().contains("Ratpack")) {
+          // verify helper injector works
+          final String[] helperClassNames = instrumenter.helperClassNames();
+          if (helperClassNames.length > 0) {
+            new HelperInjector(helperClassNames).transform(null, null, cl, null);
+          }
+        }
+      } catch (Exception e) {
+        System.err.println(
+            "FAILED HELPER INJECTION. Are Helpers being injected in the correct order?");
+        throw e;
+      }
+    }
+  }
+
+  private MuzzleVersionScanPlugin() {}
+}

dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/muzzle/MuzzleVisitor.java

@@ -1,6 +1,7 @@
 package datadog.trace.agent.tooling.muzzle;
 
-import java.util.Collection;
+import datadog.trace.agent.tooling.Instrumenter;
+import datadog.trace.agent.tooling.Utils;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
@@ -14,10 +15,7 @@ import net.bytebuddy.implementation.Implementation;
 import net.bytebuddy.jar.asm.*;
 import net.bytebuddy.pool.TypePool;
 
-/**
+/** Visit a class and add: a private instrumenationMuzzle field and getter */
- * Visit a class and add: 1) a private instrumenationMuzzle field and getter AND 2) logic to the end
- * of the instrumentation transformer to assert classpath is safe to apply instrumentation to.
- */
 public class MuzzleVisitor implements AsmVisitorWrapper {
   @Override
   public int mergeWriter(int flags) {
@@ -44,7 +42,7 @@ public class MuzzleVisitor implements AsmVisitorWrapper {
 
   public static class InsertSafetyMatcher extends ClassVisitor {
     private String instrumentationClassName;
-    private Set<String> adviceClassNames = new HashSet<>();
+    private Instrumenter.Default instrumenter;
 
     public InsertSafetyMatcher(ClassVisitor classVisitor) {
       super(Opcodes.ASM6, classVisitor);
@@ -59,6 +57,17 @@ public class MuzzleVisitor implements AsmVisitorWrapper {
         final String superName,
         final String[] interfaces) {
       this.instrumentationClassName = name;
+      try {
+        instrumenter =
+            (Instrumenter.Default)
+                MuzzleVisitor.class
+                    .getClassLoader()
+                    .loadClass(Utils.getClassName(instrumentationClassName))
+                    .getDeclaredConstructor()
+                    .newInstance();
+      } catch (Exception e) {
+        throw new RuntimeException(e);
+      }
       super.visit(version, access, name, signature, superName, interfaces);
     }
 
@@ -74,19 +83,24 @@ public class MuzzleVisitor implements AsmVisitorWrapper {
       if ("<init>".equals(name)) {
         methodVisitor = new InitializeFieldVisitor(methodVisitor);
       }
-      return new InsertMuzzleTransformer(methodVisitor);
+      return methodVisitor;
     }
 
     public Reference[] generateReferences() {
       // track sources we've generated references from to avoid recursion
       final Set<String> referenceSources = new HashSet<>();
       final Map<String, Reference> references = new HashMap<>();
+      final Set<String> adviceClassNames = new HashSet<>();
+
+      for (String adviceClassName : instrumenter.transformers().values()) {
+        adviceClassNames.add(adviceClassName);
+      }
 
       for (String adviceClass : adviceClassNames) {
         if (!referenceSources.contains(adviceClass)) {
           referenceSources.add(adviceClass);
           for (Map.Entry<String, Reference> entry :
-              AdviceReferenceVisitor.createReferencesFrom(
+              ReferenceCreator.createReferencesFrom(
                       adviceClass, ReferenceMatcher.class.getClassLoader())
                   .entrySet()) {
             if (references.containsKey(entry.getKey())) {
@@ -105,10 +119,11 @@ public class MuzzleVisitor implements AsmVisitorWrapper {
     public void visitEnd() {
       { // generate getInstrumentationMuzzle method
         /*
-         * private synchronized ReferenceMatcher getInstrumentationMuzzle() {
+         * protected synchronized ReferenceMatcher getInstrumentationMuzzle() {
          *   if (null == this.instrumentationMuzzle) {
-         *     this.instrumentationMuzzle = new ReferenceMatcher(new Reference[]{
+         *     this.instrumentationMuzzle = new ReferenceMatcher(this.helperClassNames(),
-         *                                                                        //reference builders
+         *                                                       new Reference[]{
+         *                                                                       //reference builders
          *                                                                       });
          *   }
          *   return this.instrumentationMuzzle;
@@ -116,7 +131,7 @@ public class MuzzleVisitor implements AsmVisitorWrapper {
          */
         final MethodVisitor mv =
             visitMethod(
-                Opcodes.ACC_PRIVATE + Opcodes.ACC_SYNCHRONIZED,
+                Opcodes.ACC_PROTECTED + Opcodes.ACC_SYNCHRONIZED,
                 "getInstrumentationMuzzle",
                 "()Ldatadog/trace/agent/tooling/muzzle/ReferenceMatcher;",
                 null,
@@ -137,16 +152,26 @@ public class MuzzleVisitor implements AsmVisitorWrapper {
             "Ldatadog/trace/agent/tooling/muzzle/ReferenceMatcher;");
         mv.visitJumpInsn(Opcodes.IF_ACMPNE, ret);
 
-        final Reference[] references = generateReferences();
         mv.visitVarInsn(Opcodes.ALOAD, 0);
+
         mv.visitTypeInsn(Opcodes.NEW, "datadog/trace/agent/tooling/muzzle/ReferenceMatcher");
         mv.visitInsn(Opcodes.DUP);
-        mv.visitIntInsn(Opcodes.BIPUSH, references.length);
+
+        mv.visitVarInsn(Opcodes.ALOAD, 0);
+        mv.visitMethodInsn(
+            Opcodes.INVOKEVIRTUAL,
+            instrumentationClassName,
+            "helperClassNames",
+            "()[Ljava/lang/String;",
+            false);
+
+        final Reference[] references = generateReferences();
+        mv.visitLdcInsn(references.length);
         mv.visitTypeInsn(Opcodes.ANEWARRAY, "datadog/trace/agent/tooling/muzzle/Reference");
 
         for (int i = 0; i < references.length; ++i) {
           mv.visitInsn(Opcodes.DUP);
-          mv.visitIntInsn(Opcodes.BIPUSH, i);
+          mv.visitLdcInsn(i);
           mv.visitTypeInsn(Opcodes.NEW, "datadog/trace/agent/tooling/muzzle/Reference$Builder");
           mv.visitInsn(Opcodes.DUP);
           mv.visitLdcInsn(references[i].getClassName());
@@ -158,7 +183,7 @@ public class MuzzleVisitor implements AsmVisitorWrapper {
               false);
           for (Reference.Source source : references[i].getSources()) {
             mv.visitLdcInsn(source.getName());
-            mv.visitIntInsn(Opcodes.BIPUSH, source.getLine());
+            mv.visitLdcInsn(source.getLine());
             mv.visitMethodInsn(
                 Opcodes.INVOKEVIRTUAL,
                 "datadog/trace/agent/tooling/muzzle/Reference$Builder",
@@ -199,14 +224,14 @@ public class MuzzleVisitor implements AsmVisitorWrapper {
           }
           for (Reference.Field field : references[i].getFields()) {
             mv.visitLdcInsn(field.getName());
-            mv.visitIntInsn(Opcodes.BIPUSH, field.getFlags().size());
+            mv.visitLdcInsn(field.getFlags().size());
             mv.visitTypeInsn(
                 Opcodes.ANEWARRAY, "datadog/trace/agent/tooling/muzzle/Reference$Flag");
 
             int j = 0;
             for (Reference.Flag flag : field.getFlags()) {
               mv.visitInsn(Opcodes.DUP);
-              mv.visitIntInsn(Opcodes.BIPUSH, j);
+              mv.visitLdcInsn(j);
               mv.visitFieldInsn(
                   Opcodes.GETSTATIC,
                   "datadog/trace/agent/tooling/muzzle/Reference$Flag",
@@ -226,13 +251,13 @@ public class MuzzleVisitor implements AsmVisitorWrapper {
           for (Reference.Method method : references[i].getMethods()) {
             mv.visitLdcInsn(method.getName());
 
-            mv.visitIntInsn(Opcodes.BIPUSH, method.getFlags().size());
+            mv.visitLdcInsn(method.getFlags().size());
             mv.visitTypeInsn(
                 Opcodes.ANEWARRAY, "datadog/trace/agent/tooling/muzzle/Reference$Flag");
             int j = 0;
             for (Reference.Flag flag : method.getFlags()) {
               mv.visitInsn(Opcodes.DUP);
-              mv.visitIntInsn(Opcodes.BIPUSH, j);
+              mv.visitLdcInsn(j);
               mv.visitFieldInsn(
                   Opcodes.GETSTATIC,
                   "datadog/trace/agent/tooling/muzzle/Reference$Flag",
@@ -244,13 +269,13 @@ public class MuzzleVisitor implements AsmVisitorWrapper {
 
             mv.visitLdcInsn(method.getReturnType());
 
-            mv.visitIntInsn(Opcodes.BIPUSH, method.getParameterTypes().size());
+            mv.visitLdcInsn(method.getParameterTypes().size());
             mv.visitTypeInsn(Opcodes.ANEWARRAY, "java/lang/String");
 
             int k = 0;
             for (String parameterType : method.getParameterTypes()) {
               mv.visitInsn(Opcodes.DUP);
-              mv.visitIntInsn(Opcodes.BIPUSH, k);
+              mv.visitLdcInsn(k);
               mv.visitLdcInsn(parameterType);
               mv.visitInsn(Opcodes.AASTORE);
             }
@@ -275,7 +300,7 @@ public class MuzzleVisitor implements AsmVisitorWrapper {
             Opcodes.INVOKESPECIAL,
             "datadog/trace/agent/tooling/muzzle/ReferenceMatcher",
             "<init>",
-            "([Ldatadog/trace/agent/tooling/muzzle/Reference;)V",
+            "([Ljava/lang/String;[Ldatadog/trace/agent/tooling/muzzle/Reference;)V",
             false);
         mv.visitFieldInsn(
             Opcodes.PUTFIELD,
@@ -308,106 +333,6 @@ public class MuzzleVisitor implements AsmVisitorWrapper {
       super.visitEnd();
     }
 
-    /**
-     * Changes this:<br>
-     * &nbsp.transform(DDAdvice.create().advice(named("fooMethod", FooAdvice.class.getname())))
-     * &nbsp.asDecorator(); Into this:<br>
-     * &nbsp.transform(DDAdvice.create().advice(named("fooMethod", FooAdvice.class.getname())))
-     * &nbsp.transform(this.instrumentationMuzzle.assertSafeTransformation("foo.package.FooAdvice"));
-     * &nbsp.asDecorator(); className)
-     */
-    public class InsertMuzzleTransformer extends MethodVisitor {
-      // it would be nice to manage the state with an enum, but that requires this class to be non-static
-      private final int INIT = 0;
-      // SomeClass
-      private final int PREVIOUS_INSTRUCTION_LDC = 1;
-      // SomeClass.getName()
-      private final int PREVIOUS_INSTRUCTION_GET_CLASS_NAME = 2;
-
-      private String lastClassLDC = null;
-
-      private Collection<String> adviceClassNames = new HashSet<>();
-      private int STATE = INIT;
-
-      public InsertMuzzleTransformer(MethodVisitor methodVisitor) {
-        super(Opcodes.ASM6, methodVisitor);
-      }
-
-      public void reset() {
-        STATE = INIT;
-        lastClassLDC = null;
-        adviceClassNames.clear();
-      }
-
-      @Override
-      public void visitMethodInsn(
-          final int opcode,
-          final String owner,
-          final String name,
-          final String descriptor,
-          final boolean isInterface) {
-        if (name.equals("getName")) {
-          if (STATE == PREVIOUS_INSTRUCTION_LDC) {
-            STATE = PREVIOUS_INSTRUCTION_GET_CLASS_NAME;
-          }
-        } else if (name.equals("advice")) {
-          if (STATE == PREVIOUS_INSTRUCTION_GET_CLASS_NAME) {
-            adviceClassNames.add(lastClassLDC);
-            InsertSafetyMatcher.this.adviceClassNames.add(lastClassLDC);
-          }
-          // add last LDC/ToString to adivce list
-        } else if (name.equals("asDecorator")) {
-          this.visitVarInsn(Opcodes.ALOAD, 0);
-          mv.visitMethodInsn(
-              Opcodes.INVOKESPECIAL,
-              instrumentationClassName,
-              "getInstrumentationMuzzle",
-              "()Ldatadog/trace/agent/tooling/muzzle/ReferenceMatcher;",
-              false);
-          mv.visitIntInsn(Opcodes.BIPUSH, adviceClassNames.size());
-          mv.visitTypeInsn(Opcodes.ANEWARRAY, "java/lang/String");
-          int i = 0;
-          for (String adviceClassName : adviceClassNames) {
-            mv.visitInsn(Opcodes.DUP);
-            mv.visitIntInsn(Opcodes.BIPUSH, i);
-            mv.visitLdcInsn(adviceClassName);
-            mv.visitInsn(Opcodes.AASTORE);
-            ++i;
-          }
-          mv.visitMethodInsn(
-              Opcodes.INVOKEVIRTUAL,
-              "datadog/trace/agent/tooling/muzzle/ReferenceMatcher",
-              "assertSafeTransformation",
-              "([Ljava/lang/String;)Lnet/bytebuddy/agent/builder/AgentBuilder$Transformer;",
-              false);
-          mv.visitMethodInsn(
-              Opcodes.INVOKEINTERFACE,
-              "net/bytebuddy/agent/builder/AgentBuilder$Identified$Narrowable",
-              "transform",
-              "(Lnet/bytebuddy/agent/builder/AgentBuilder$Transformer;)Lnet/bytebuddy/agent/builder/AgentBuilder$Identified$Extendable;",
-              true);
-          reset();
-        } else {
-          STATE = INIT;
-          lastClassLDC = null;
-        }
-        super.visitMethodInsn(opcode, owner, name, descriptor, isInterface);
-      }
-
-      @Override
-      public void visitLdcInsn(final Object value) {
-        if (value instanceof Type) {
-          Type type = (Type) value;
-          if (type.getSort() == Type.OBJECT) {
-            lastClassLDC = type.getClassName();
-            STATE = PREVIOUS_INSTRUCTION_LDC;
-            type.getClassName();
-          }
-        }
-        super.visitLdcInsn(value);
-      }
-    }
-
     /** Append a field initializer to the end of a method. */
     public class InitializeFieldVisitor extends MethodVisitor {
       public InitializeFieldVisitor(MethodVisitor methodVisitor) {

dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/muzzle/Reference.java

@@ -5,7 +5,7 @@ import static datadog.trace.agent.tooling.ClassLoaderMatcher.BOOTSTRAP_CLASSLOAD
 import datadog.trace.agent.tooling.Utils;
 import java.util.*;
 
-/** An immutable reference to a single class file. */
+/** An immutable reference to a jvm class. */
 public class Reference {
   private final Set<Source> sources;
   private final String className;

dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/muzzle/ReferenceCreator.java

@@ -7,11 +7,63 @@ import java.util.*;
 import net.bytebuddy.jar.asm.*;
 
 /** Visit a class and collect all references made by the visited class. */
-public class AdviceReferenceVisitor extends ClassVisitor {
+public class ReferenceCreator extends ClassVisitor {
+  /**
+   * Generate all references reachable from a given class.
+   *
+   * @param entryPointClassName Starting point for generating references.
+   * @param loader Classloader used to read class bytes.
+   * @return Map of [referenceClassName -> Reference]
+   */
+  public static Map<String, Reference> createReferencesFrom(
+      String entryPointClassName, ClassLoader loader) {
+    final Set<String> visitedSources = new HashSet<>();
+    final Map<String, Reference> references = new HashMap<>();
+
+    final Queue<String> instrumentationQueue = new ArrayDeque<>();
+    instrumentationQueue.add(entryPointClassName);
+
+    while (!instrumentationQueue.isEmpty()) {
+      final String className = instrumentationQueue.remove();
+      visitedSources.add(className);
+      try {
+        final InputStream in = loader.getResourceAsStream(Utils.getResourceName(className));
+        try {
+          final ReferenceCreator cv = new ReferenceCreator(null);
+          final ClassReader reader = new ClassReader(in);
+          reader.accept(cv, ClassReader.SKIP_FRAMES);
+
+          Map<String, Reference> instrumentationReferences = cv.getReferences();
+          for (Map.Entry<String, Reference> entry : instrumentationReferences.entrySet()) {
+            // Don't generate references created outside of the datadog instrumentation package.
+            if (!visitedSources.contains(entry.getKey())
+                && entry.getKey().startsWith("datadog.trace.instrumentation.")) {
+              instrumentationQueue.add(entry.getKey());
+            }
+            if (references.containsKey(entry.getKey())) {
+              references.put(
+                  entry.getKey(), references.get(entry.getKey()).merge(entry.getValue()));
+            } else {
+              references.put(entry.getKey(), entry.getValue());
+            }
+          }
+
+        } finally {
+          if (in != null) {
+            in.close();
+          }
+        }
+      } catch (IOException ioe) {
+        throw new IllegalStateException(ioe);
+      }
+    }
+    return references;
+  }
+
   private Map<String, Reference> references = new HashMap<>();
   private String refSourceClassName;
 
-  public AdviceReferenceVisitor(ClassVisitor classVisitor) {
+  private ReferenceCreator(ClassVisitor classVisitor) {
     super(Opcodes.ASM6, classVisitor);
   }
 
@@ -51,7 +103,6 @@ public class AdviceReferenceVisitor extends ClassVisitor {
   }
 
   private class AdviceReferenceMethodVisitor extends MethodVisitor {
-    private boolean isAdviceMethod = false;
     private int currentLineNumber = -1;
 
     public AdviceReferenceMethodVisitor(MethodVisitor methodVisitor) {
@@ -75,56 +126,4 @@ public class AdviceReferenceVisitor extends ClassVisitor {
           new Reference.Builder(owner).withSource(refSourceClassName, currentLineNumber).build());
     }
   }
-
-  /**
-   * Generate all references reachable from a given class.
-   *
-   * @param entryPointClassName Starting point for generating references.
-   * @param loader Classloader used to read class bytes.
-   * @return Map of [referenceClassName -> Reference]
-   */
-  public static Map<String, Reference> createReferencesFrom(
-      String entryPointClassName, ClassLoader loader) {
-    final Set<String> visitedSources = new HashSet<>();
-    final Map<String, Reference> references = new HashMap<>();
-
-    final Queue<String> instrumentationQueue = new ArrayDeque<>();
-    instrumentationQueue.add(entryPointClassName);
-
-    while (!instrumentationQueue.isEmpty()) {
-      final String className = instrumentationQueue.remove();
-      visitedSources.add(className);
-      try {
-        final InputStream in = loader.getResourceAsStream(Utils.getResourceName(className));
-        try {
-          final AdviceReferenceVisitor cv = new AdviceReferenceVisitor(null);
-          final ClassReader reader = new ClassReader(in);
-          reader.accept(cv, ClassReader.SKIP_FRAMES);
-
-          Map<String, Reference> instrumentationReferences = cv.getReferences();
-          for (Map.Entry<String, Reference> entry : instrumentationReferences.entrySet()) {
-            // TODO: expose config instead of hardcoding datadog instrumentation namespace
-            if (!visitedSources.contains(entry.getKey())
-                && entry.getKey().startsWith("datadog.trace.instrumentation.")) {
-              instrumentationQueue.add(entry.getKey());
-            }
-            if (references.containsKey(entry.getKey())) {
-              references.put(
-                  entry.getKey(), references.get(entry.getKey()).merge(entry.getValue()));
-            } else {
-              references.put(entry.getKey(), entry.getValue());
-            }
-          }
-
-        } finally {
-          if (in != null) {
-            in.close();
-          }
-        }
-      } catch (IOException ioe) {
-        throw new IllegalStateException(ioe);
-      }
-    }
-    return references;
-  }
 }

dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/muzzle/ReferenceMatcher.java

@@ -3,37 +3,24 @@ package datadog.trace.agent.tooling.muzzle;
 import static net.bytebuddy.dynamic.loading.ClassLoadingStrategy.BOOTSTRAP_LOADER;
 
 import datadog.trace.agent.tooling.Utils;
-import java.security.ProtectionDomain;
 import java.util.*;
 import lombok.extern.slf4j.Slf4j;
-import net.bytebuddy.agent.builder.AgentBuilder;
-import net.bytebuddy.agent.builder.AgentBuilder.Transformer;
-import net.bytebuddy.description.type.TypeDescription;
-import net.bytebuddy.dynamic.DynamicType;
-import net.bytebuddy.utility.JavaModule;
 
-/**
+/** Matches a set of references against a classloader. */
- * A bytebuddy matcher that matches if expected references (classes, fields, methods, visibility)
- * are present on the classpath.
- */
 @Slf4j
-public class ReferenceMatcher implements AgentBuilder.RawMatcher {
+public class ReferenceMatcher {
   private final Map<ClassLoader, List<Reference.Mismatch>> mismatchCache =
       Collections.synchronizedMap(new WeakHashMap<ClassLoader, List<Reference.Mismatch>>());
   private final Reference[] references;
+  private final Set<String> helperClassNames;
 
   public ReferenceMatcher(Reference... references) {
-    this.references = references;
+    this(new String[0], references);
   }
 
-  @Override
+  public ReferenceMatcher(String[] helperClassNames, Reference[] references) {
-  public boolean matches(
+    this.references = references;
-      TypeDescription typeDescription,
+    this.helperClassNames = new HashSet<>(Arrays.asList(helperClassNames));
-      ClassLoader classLoader,
-      JavaModule module,
-      Class<?> classBeingRedefined,
-      ProtectionDomain protectionDomain) {
-    return matches(classLoader);
   }
 
   /**
@@ -52,54 +39,23 @@ public class ReferenceMatcher implements AgentBuilder.RawMatcher {
     if (loader == BOOTSTRAP_LOADER) {
       loader = Utils.getBootstrapProxy();
     }
-    final List<Reference.Mismatch> mismatchedReferences = new ArrayList<>(0);
+    List<Reference.Mismatch> mismatches = mismatchCache.get(loader);
-    for (Reference reference : references) {
+    if (null == mismatches) {
-      mismatchedReferences.addAll(reference.checkMatch(loader));
+      synchronized (loader) {
-    }
+        mismatches = mismatchCache.get(loader);
-    return mismatchedReferences;
-  }
-
-  /**
-   * Create a bytebuddy matcher which throws a MismatchException when there are mismatches with the
-   * classloader under transformation.
-   */
-  public Transformer assertSafeTransformation(String... adviceClassNames) {
-    return new Transformer() {
-      @Override
-      public DynamicType.Builder<?> transform(
-          DynamicType.Builder<?> builder,
-          TypeDescription typeDescription,
-          ClassLoader classLoader,
-          JavaModule module) {
-        if (classLoader == BOOTSTRAP_LOADER) {
-          classLoader = Utils.getBootstrapProxy();
-        }
-        List<Reference.Mismatch> mismatches =
-            mismatchCache.get(getMismatchedReferenceSources(classLoader));
         if (null == mismatches) {
-          // okay if entered by multiple callers during initialization
+          mismatches = new ArrayList<>(0);
-          mismatches = getMismatchedReferenceSources(classLoader);
+          for (Reference reference : references) {
-          mismatchCache.put(classLoader, mismatches);
+            // Don't reference-check helper classes.
-        }
+            // They will be injected by the instrumentation's HelperInjector.
-        if (mismatches.size() == 0) {
+            if (!helperClassNames.contains(reference.getClassName())) {
-          return builder;
+              mismatches.addAll(reference.checkMatch(loader));
-        } else {
+            }
-          throw new MismatchException(classLoader, mismatches);
+          }
+          mismatchCache.put(loader, mismatches);
         }
       }
-    };
-  }
-
-  public static class MismatchException extends RuntimeException {
-    private final List<Reference.Mismatch> mismatches;
-
-    public MismatchException(ClassLoader classLoader, List<Reference.Mismatch> mismatches) {
-      super(mismatches.size() + " mismatches on classloader: " + classLoader);
-      this.mismatches = mismatches;
-    }
-
-    public List<Reference.Mismatch> getMismatches() {
-      return this.mismatches;
     }
+    return mismatches;
   }
 }

dd-java-agent/instrumentation/akka-http-10.0/src/lagomTest/java/EchoService.java

@@ -10,6 +10,7 @@ public interface EchoService extends Service {
 
   ServiceCall<Source<String, NotUsed>, Source<String, NotUsed>> error();
 
+  @Override
   default Descriptor descriptor() {
     return named("echo").withCalls(namedCall("echo", this::echo), namedCall("error", this::error));
   }

dd-java-agent/instrumentation/elasticsearch-rest-5/src/main/java/datadog/trace/instrumentation/elasticsearch5/Elasticsearch5RestClientInstrumentation.java

@@ -35,6 +35,11 @@ public class Elasticsearch5RestClientInstrumentation extends Instrumenter.Defaul
     return false;
   }
 
+  @Override
+  public String[] helperClassNames() {
+    return new String[] {"datadog.trace.instrumentation.elasticsearch5.RestResponseListener"};
+  }
+
   @Override
   public ElementMatcher typeMatcher() {
     return not(isInterface()).and(named("org.elasticsearch.client.RestClient"));

dd-java-agent/instrumentation/instrumentation.gradle

@@ -14,6 +14,7 @@ Project instr_project = project
 subprojects { subProj ->
   if (subProj.getParent() == instr_project) {
     apply plugin: "net.bytebuddy.byte-buddy"
+    apply plugin: 'muzzle'
 
     subProj.byteBuddy {
       transformation {

dd-java-agent/instrumentation/mongo-async-3.3/mongo-async-3.3.gradle

@@ -17,6 +17,7 @@ dependencies {
   compile(project(':dd-java-agent:instrumentation:mongo-3.1')) {
     transitive = false
   }
+  compileOnly group: 'org.mongodb', name: 'mongo-java-driver', version: '3.4.2'
   compileOnly group: 'org.mongodb', name: 'mongodb-driver-async', version: '3.4.2'
 
   compile project(':dd-java-agent:agent-tooling')

dd-java-agent/instrumentation/ratpack-1.4/src/main/java/datadog/trace/instrumentation/ratpack/RatpackInstrumentation.java

@@ -54,6 +54,8 @@ public final class RatpackInstrumentation extends Instrumenter.Default {
       // service registry helpers
       "datadog.trace.instrumentation.ratpack.impl.RatpackRequestExtractAdapter",
       "datadog.trace.instrumentation.ratpack.impl.RatpackScopeManager",
+      "datadog.trace.instrumentation.ratpack.impl.RatpackScopeManager$RatpackScope",
+      "datadog.trace.instrumentation.ratpack.impl.RatpackServerAdvice",
       "datadog.trace.instrumentation.ratpack.impl.RatpackServerAdvice$RatpackServerRegistryAdvice",
       "datadog.trace.instrumentation.ratpack.impl.TracingHandler"
     };

dd-java-agent/instrumentation/spymemcached-2.12/src/main/java/datadog/trace/instrumentation/spymemcached/MemcachedClientInstrumentation.java

@@ -49,6 +49,7 @@ public final class MemcachedClientInstrumentation extends Instrumenter.Default {
   public String[] helperClassNames() {
     return new String[] {
       HELPERS_PACKAGE + ".CompletionListener",
+      HELPERS_PACKAGE + ".SyncCompletionListener",
       HELPERS_PACKAGE + ".GetCompletionListener",
       HELPERS_PACKAGE + ".OperationCompletionListener",
       HELPERS_PACKAGE + ".BulkGetCompletionListener"

dd-java-agent/instrumentation/trace-annotation/src/main/java/datadog/trace/instrumentation/trace_annotation/TraceConfigInstrumentation.java

@@ -17,9 +17,18 @@ import net.bytebuddy.description.method.MethodDescription;
 import net.bytebuddy.description.type.TypeDescription;
 import net.bytebuddy.matcher.ElementMatcher;
 
+/**
+ * TraceConfig Instrumentation does not extend Default.
+ *
+ * <p>Instead it directly implements Instrumenter#instrument() and adds one default Instrumenter for
+ * every configured class+method-list.
+ *
+ * <p>If this becomes a more common use case the building logic should be abstracted out into a
+ * super class.
+ */
 @Slf4j
 @AutoService(Instrumenter.class)
-public class TraceConfigInstrumentation extends Instrumenter.Default {
+public class TraceConfigInstrumentation implements Instrumenter {
   private static final String CONFIG_NAME = "dd.trace.methods";
 
   static final String PACKAGE_CLASS_NAME_REGEX = "[\\w.\\$]+";
@@ -38,9 +47,7 @@ public class TraceConfigInstrumentation extends Instrumenter.Default {
   private final Map<String, Set<String>> classMethodsToTrace;
 
   public TraceConfigInstrumentation() {
-    super("trace", "trace-config");
+    final String configString = Default.getPropOrEnv(CONFIG_NAME);
-
-    final String configString = getPropOrEnv(CONFIG_NAME);
     if (configString == null || configString.trim().isEmpty()) {
       classMethodsToTrace = Collections.emptyMap();
 
@@ -97,6 +104,11 @@ public class TraceConfigInstrumentation extends Instrumenter.Default {
     private final String className;
     private final Set<String> methodNames;
 
+    /** No-arg constructor only used by muzzle and tests. */
+    public TracerClassInstrumentation() {
+      this("noop", Collections.singleton("noop"));
+    }
+
     public TracerClassInstrumentation(String className, Set<String> methodNames) {
       super("trace", "trace-config");
       this.className = className;
@@ -130,6 +142,16 @@ public class TraceConfigInstrumentation extends Instrumenter.Default {
     throw new RuntimeException("TracerConfigInstrumentation must not use TypeMatcher");
   }
 
+  @Override
+  public ElementMatcher<? super ClassLoader> classLoaderMatcher() {
+    throw new RuntimeException("TracerConfigInstrumentation must not use classLoaderMatcher");
+  }
+
+  @Override
+  public String[] helperClassNames() {
+    throw new RuntimeException("TracerConfigInstrumentation must not use helperClassNames");
+  }
+
   @Override
   public Map<ElementMatcher, String> transformers() {
     throw new RuntimeException("TracerConfigInstrumentation must not use transformers.");

dd-java-agent/testing/src/test/groovy/muzzle/AdviceReferenceVisitorTest.groovy

@@ -2,7 +2,7 @@ package muzzle
 
 import datadog.trace.agent.test.AgentTestRunner
 import datadog.trace.agent.test.TestUtils
-import datadog.trace.agent.tooling.muzzle.AdviceReferenceVisitor
+import datadog.trace.agent.tooling.muzzle.ReferenceCreator
 import datadog.trace.agent.tooling.muzzle.ReferenceMatcher
 import datadog.trace.agent.tooling.muzzle.Reference
 
@@ -10,7 +10,7 @@ class AdviceReferenceVisitorTest extends AgentTestRunner {
 
   def "methods body references"() {
     setup:
-    Map<String, Reference> references = AdviceReferenceVisitor.createReferencesFrom(AdviceClass.getName(), this.getClass().getClassLoader())
+    Map<String, Reference> references = ReferenceCreator.createReferencesFrom(AdviceClass.getName(), this.getClass().getClassLoader())
 
     expect:
     references.get('java.lang.Object') != null
@@ -22,9 +22,8 @@ class AdviceReferenceVisitorTest extends AgentTestRunner {
 
   def "match safe classpaths"() {
     setup:
-    Reference[] refs = AdviceReferenceVisitor.createReferencesFrom(AdviceClass.getName(), this.getClass().getClassLoader()).values().toArray(new Reference[0])
+    Reference[] refs = ReferenceCreator.createReferencesFrom(AdviceClass.getName(), this.getClass().getClassLoader()).values().toArray(new Reference[0])
     ReferenceMatcher refMatcher = new ReferenceMatcher(refs)
-    refMatcher.assertSafeTransformation()
     ClassLoader safeClassloader = new URLClassLoader([TestUtils.createJarWithClasses(AdviceClass$A,
                                                                                      AdviceClass$SomeInterface,
                                                                                      AdviceClass$SomeImplementation)] as URL[],

settings.gradle

@@ -13,7 +13,6 @@ include ':dd-java-agent:instrumentation:akka-http-10.0'
 include ':dd-java-agent:instrumentation:apache-httpclient-4.3'
 include ':dd-java-agent:instrumentation:aws-java-sdk-1.11.0'
 include ':dd-java-agent:instrumentation:aws-java-sdk-1.11.106'
-include ':dd-java-agent:instrumentation:classloaders'
 include ':dd-java-agent:instrumentation:datastax-cassandra-3.2'
 include ':dd-java-agent:instrumentation:elasticsearch-rest-5'
 include ':dd-java-agent:instrumentation:elasticsearch-transport-2'