Prevent duplicate telemetry when using both library and auto instrumentation (#2661)

* Prevent duplicate telemetry when using both library and auto instrumentation

* Add unit test

* Fix Oshi tests

* Fix couchbase 3.1 tests

docs/contributing/muzzle.md

@@ -21,13 +21,15 @@ The compile-time reference collection and code generation process is implemented
 plugin (called `MuzzleCodeGenerationPlugin`).
 
 For each instrumentation module the ByteBuddy plugin collects symbols referring to both internal and
-third party APIs used by the currently processed module's type instrumentations (`InstrumentationModule#typeInstrumentations()`).
-The reference collection process starts from advice classes (values of the map returned by the
-`TypeInstrumentation#transformers()`method) and traverses the class graph until it encounters
-a reference to a non-instrumentation class (determined by `InstrumentationClassPredicate`).
-Aside from references, the collection process also builds a graph of dependencies between internal
-instrumentation helper classes - this dependency graph is later used to construct a list of helper
-classes that will be injected to the application classloader (`InstrumentationModule#getMuzzleHelperClassNames()`).
+third party APIs used by the currently processed module's type
+instrumentations (`InstrumentationModule#typeInstrumentations()`). The reference collection process
+starts from advice classes (values of the map returned by the
+`TypeInstrumentation#transformers()` method) and traverses the class graph until it encounters a
+reference to a non-instrumentation class (determined by `InstrumentationClassPredicate` and
+the `InstrumentationModule#isLibraryInstrumentationClass(String)` predicate). Aside from references,
+the collection process also builds a graph of dependencies between internal instrumentation helper
+classes - this dependency graph is later used to construct a list of helper classes that will be
+injected to the application classloader (`InstrumentationModule#getMuzzleHelperClassNames()`).
 
 All collected references are then used to create a `ReferenceMatcher` instance. This matcher
 is stored in the instrumentation module class in the method `InstrumentationModule#getMuzzleReferenceMatcher()`

examples/distro/gradle/shadow.gradle

@@ -4,8 +4,8 @@ ext.relocatePackages = { shadowJar ->
   // rewrite dependencies calling Logger.getLogger
   shadowJar.relocate 'java.util.logging.Logger', 'io.opentelemetry.javaagent.bootstrap.PatchLogger'
 
-  // rewrite library instrumentation dependencies
-  shadowJar.relocate "io.opentelemetry.instrumentation", "io.opentelemetry.javaagent.shaded.instrumentation"
+  // prevents conflict with library instrumentation, which uses its own instrumentation-api
+  shadowJar.relocate 'io.opentelemetry.instrumentation.api', 'io.opentelemetry.javaagent.shaded.instrumentation.api'
 
   // relocate OpenTelemetry API usage
   shadowJar.relocate "io.opentelemetry.api", "io.opentelemetry.javaagent.shaded.io.opentelemetry.api"

gradle/instrumentation.gradle

@@ -79,15 +79,12 @@ shadowJar {
 
   archiveFileName = 'agent-testing.jar'
 
-  // rewrite library instrumentation dependencies
-  relocate "io.opentelemetry.instrumentation", "io.opentelemetry.javaagent.shaded.instrumentation"
-
   // Prevents conflict with other SLF4J instances. Important for premain.
   relocate 'org.slf4j', 'io.opentelemetry.javaagent.slf4j'
   // rewrite dependencies calling Logger.getLogger
   relocate 'java.util.logging.Logger', 'io.opentelemetry.javaagent.bootstrap.PatchLogger'
 
-  // prevents conflict with library instrumentation
+  // prevents conflict with library instrumentation, which uses its own instrumentation-api
   relocate 'io.opentelemetry.instrumentation.api', 'io.opentelemetry.javaagent.shaded.instrumentation.api'
 
   // relocate OpenTelemetry API usage

instrumentation/apache-dubbo/apache-dubbo-2.7/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/apachedubbo/v2_7/DubboInstrumentationModule.java

@@ -32,6 +32,11 @@ public class DubboInstrumentationModule extends InstrumentationModule {
     return hasClassesNamed("org.apache.dubbo.rpc.Filter");
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("io.opentelemetry.instrumentation.apachedubbo.v2_7");
+  }
+
   @Override
   public List<TypeInstrumentation> typeInstrumentations() {
     return Collections.singletonList(new DubboInstrumentation());

instrumentation/armeria-1.3/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/armeria/v1_3/ArmeriaInstrumentationModule.java

@@ -26,6 +26,11 @@ public class ArmeriaInstrumentationModule extends InstrumentationModule {
     return hasClassesNamed("com.linecorp.armeria.server.metric.PrometheusExpositionServiceBuilder");
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("io.opentelemetry.instrumentation.armeria.v1_3");
+  }
+
   @Override
   public List<TypeInstrumentation> typeInstrumentations() {
     return asList(

instrumentation/aws-lambda-1.0/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/awslambda/v1_0/AwsLambdaInstrumentationModule.java

@@ -23,6 +23,11 @@ public class AwsLambdaInstrumentationModule extends InstrumentationModule {
     return new String[] {"io.opentelemetry.extension.aws.AwsXrayPropagator"};
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("io.opentelemetry.instrumentation.awslambda.v1_0");
+  }
+
   @Override
   public List<TypeInstrumentation> typeInstrumentations() {
     return singletonList(new AwsLambdaRequestHandlerInstrumentation());

instrumentation/aws-sdk/aws-sdk-1.11/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/awssdk/v1_11/AwsSdkInstrumentationModule.java

@@ -23,6 +23,11 @@ public class AwsSdkInstrumentationModule extends InstrumentationModule {
     return new String[] {"io.opentelemetry.extension.aws.AwsXrayPropagator"};
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("io.opentelemetry.instrumentation.awssdk.v1_11");
+  }
+
   @Override
   public List<TypeInstrumentation> typeInstrumentations() {
     return asList(

instrumentation/aws-sdk/aws-sdk-2.2/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/awssdk/v2_2/AwsSdkInstrumentationModule.java

@@ -43,6 +43,11 @@ public class AwsSdkInstrumentationModule extends InstrumentationModule {
     return hasClassesNamed("software.amazon.awssdk.core.interceptor.ExecutionInterceptor");
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("io.opentelemetry.instrumentation.awssdk.v2_2");
+  }
+
   @Override
   public List<TypeInstrumentation> typeInstrumentations() {
     return Collections.singletonList(new AwsSdkInitializationInstrumentation());

instrumentation/couchbase/couchbase-3.1/javaagent/couchbase-3.1-javaagent.gradle

@@ -21,4 +21,9 @@ dependencies {
   testLibrary group: "com.couchbase.client", name: "java-client", version: "3.1.0"
 
   testImplementation group: "org.testcontainers", name: "couchbase", version: versions.testcontainers
-}
+}
+
+configurations {
+  // exclude lib from test classpath, otherwise the javaagent would be skipped
+  testImplementation.exclude group: 'com.couchbase.client', module: 'tracing-opentelemetry'
+}

instrumentation/couchbase/couchbase-3.1/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/couchbase/v3_1/CouchbaseInstrumentationModule.java

@@ -20,11 +20,6 @@ public class CouchbaseInstrumentationModule extends InstrumentationModule {
     super("couchbase", "couchbase-3.1");
   }
 
-  @Override
-  public boolean isHelperClass(String className) {
-    return className.startsWith("com.couchbase.client.tracing.opentelemetry");
-  }
-
   @Override
   public ElementMatcher.Junction<ClassLoader> classLoaderMatcher() {
     // New class introduced in 3.1, the minimum version we support.
@@ -33,6 +28,11 @@ public class CouchbaseInstrumentationModule extends InstrumentationModule {
     return hasClassesNamed("com.couchbase.client.core.cnc.TracingIdentifiers");
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("com.couchbase.client.tracing.opentelemetry");
+  }
+
   @Override
   public List<TypeInstrumentation> typeInstrumentations() {
     return Collections.singletonList(new CouchbaseEnvironmentInstrumentation());

instrumentation/grpc-1.5/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/grpc/v1_5/GrpcInstrumentationModule.java

@@ -18,6 +18,11 @@ public class GrpcInstrumentationModule extends InstrumentationModule {
     super("grpc", "grpc-1.5");
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("io.opentelemetry.instrumentation.grpc.v1_5");
+  }
+
   @Override
   public List<TypeInstrumentation> typeInstrumentations() {
     return asList(

instrumentation/instrumentation.gradle

@@ -61,12 +61,12 @@ shadowJar {
     exclude(project(':javaagent-api'))
   }
 
-  // rewrite library instrumentation dependencies
-  relocate "io.opentelemetry.instrumentation", "io.opentelemetry.javaagent.shaded.instrumentation"
-
   // rewrite dependencies calling Logger.getLogger
   relocate 'java.util.logging.Logger', 'io.opentelemetry.javaagent.bootstrap.PatchLogger'
 
+  // prevents conflict with library instrumentation, which uses its own instrumentation-api
+  relocate 'io.opentelemetry.instrumentation.api', 'io.opentelemetry.javaagent.shaded.instrumentation.api'
+
   // relocate OpenTelemetry API usage
   relocate "io.opentelemetry.api", "io.opentelemetry.javaagent.shaded.io.opentelemetry.api"
   relocate "io.opentelemetry.semconv", "io.opentelemetry.javaagent.shaded.io.opentelemetry.semconv"

instrumentation/jaxws/jaxws-2.0-axis2-1.6/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/axis2/Axis2InstrumentationModule.java

@@ -26,6 +26,11 @@ public class Axis2InstrumentationModule extends InstrumentationModule {
     return hasClassesNamed("org.apache.axis2.jaxws.api.MessageAccessor");
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("io.opentelemetry.instrumentation.axis2");
+  }
+
   @Override
   public List<TypeInstrumentation> typeInstrumentations() {
     return Collections.singletonList(new InvocationListenerRegistryTypeInstrumentation());

instrumentation/jaxws/jaxws-2.0-cxf-3.0/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/cxf/CxfInstrumentationModule.java

@@ -17,6 +17,11 @@ public class CxfInstrumentationModule extends InstrumentationModule {
     super("cxf", "cxf-3.0");
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("io.opentelemetry.instrumentation.cxf");
+  }
+
   @Override
   public List<TypeInstrumentation> typeInstrumentations() {
     return Collections.singletonList(new EndpointImplTypeInstrumentation());

instrumentation/lettuce/lettuce-5.1/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/lettuce/v5_1/LettuceInstrumentationModule.java

@@ -36,6 +36,11 @@ public class LettuceInstrumentationModule extends InstrumentationModule {
     return hasClassesNamed("io.lettuce.core.tracing.Tracing");
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("io.opentelemetry.instrumentation.lettuce.v5_1");
+  }
+
   @Override
   public List<TypeInstrumentation> typeInstrumentations() {
     return singletonList(new DefaultClientResourcesInstrumentation());

instrumentation/log4j/log4j-2.13.2/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/log4j/v2_13_2/Log4j2InstrumentationModule.java

@@ -37,6 +37,11 @@ public class Log4j2InstrumentationModule extends InstrumentationModule {
     return hasClassesNamed("org.apache.logging.log4j.core.util.ContextDataProvider");
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("io.opentelemetry.instrumentation.log4j.v2_13_2");
+  }
+
   @Override
   public List<TypeInstrumentation> typeInstrumentations() {
     return Arrays.asList(new BugFixingInstrumentation(), new EmptyTypeInstrumentation());

instrumentation/logback/logback-1.0/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/logback/v1_0/LogbackInstrumentationModule.java

@@ -26,6 +26,11 @@ public class LogbackInstrumentationModule extends InstrumentationModule {
     return asList(new LoggerInstrumentation(), new LoggingEventInstrumentation());
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("io.opentelemetry.instrumentation.logback.v1_0");
+  }
+
   @Override
   public Map<String, String> contextStore() {
     return singletonMap("ch.qos.logback.classic.spi.ILoggingEvent", Span.class.getName());

instrumentation/okhttp/okhttp-3.0/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/okhttp/v3_0/OkHttp3InstrumentationModule.java

@@ -29,6 +29,11 @@ public class OkHttp3InstrumentationModule extends InstrumentationModule {
     super("okhttp", "okhttp-3.0");
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("io.opentelemetry.instrumentation.okhttp.v3_0");
+  }
+
   @Override
   public List<TypeInstrumentation> typeInstrumentations() {
     return singletonList(new OkHttpClientInstrumentation());

instrumentation/oshi/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/oshi/OshiInstrumentationModule.java

@@ -31,6 +31,11 @@ public class OshiInstrumentationModule extends InstrumentationModule {
     super("oshi");
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("io.opentelemetry.instrumentation.oshi");
+  }
+
   @Override
   public List<TypeInstrumentation> typeInstrumentations() {
     return singletonList(new SystemInfoInstrumentation());

instrumentation/oshi/javaagent/src/test/groovy/io/opentelemetry/javaagent/instrumentation/oshi/OshiTest.groovy

@@ -21,13 +21,13 @@ class OshiTest extends AgentInstrumentationSpecification {
     expect:
     platform != null
     // TODO (trask) is this the instrumentation library name we want?
-    findMetric("io.opentelemetry.javaagent.shaded.instrumentation.oshi", "system.disk.io") != null
-    findMetric("io.opentelemetry.javaagent.shaded.instrumentation.oshi", "system.disk.operations") != null
-    findMetric("io.opentelemetry.javaagent.shaded.instrumentation.oshi", "system.memory.usage") != null
-    findMetric("io.opentelemetry.javaagent.shaded.instrumentation.oshi", "system.memory.utilization") != null
-    findMetric("io.opentelemetry.javaagent.shaded.instrumentation.oshi", "system.network.errors") != null
-    findMetric("io.opentelemetry.javaagent.shaded.instrumentation.oshi", "system.network.io") != null
-    findMetric("io.opentelemetry.javaagent.shaded.instrumentation.oshi", "system.network.packets") != null
+    findMetric("io.opentelemetry.instrumentation.oshi", "system.disk.io") != null
+    findMetric("io.opentelemetry.instrumentation.oshi", "system.disk.operations") != null
+    findMetric("io.opentelemetry.instrumentation.oshi", "system.memory.usage") != null
+    findMetric("io.opentelemetry.instrumentation.oshi", "system.memory.utilization") != null
+    findMetric("io.opentelemetry.instrumentation.oshi", "system.network.errors") != null
+    findMetric("io.opentelemetry.instrumentation.oshi", "system.network.io") != null
+    findMetric("io.opentelemetry.instrumentation.oshi", "system.network.packets") != null
   }
 
   def findMetric(instrumentationName, metricName) {

instrumentation/reactor-3.1/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/reactor/ReactorInstrumentationModule.java

@@ -26,6 +26,11 @@ public class ReactorInstrumentationModule extends InstrumentationModule {
     super("reactor", "reactor-3.1");
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("io.opentelemetry.instrumentation.reactor");
+  }
+
   @Override
   public List<TypeInstrumentation> typeInstrumentations() {
     return singletonList(new HooksInstrumentation());

instrumentation/rocketmq-client-4.8/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/rocketmq/RocketMqInstrumentationModule.java

@@ -18,6 +18,11 @@ public class RocketMqInstrumentationModule extends InstrumentationModule {
     super("rocketmq-client", "rocketmq-client-4.8");
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("io.opentelemetry.instrumentation.rocketmq");
+  }
+
   @Override
   public List<TypeInstrumentation> typeInstrumentations() {
     return asList(new RocketMqProducerInstrumentation(), new RocketMqConsumerInstrumentation());

instrumentation/rxjava-2.0/javaagent/src/main/java/io/opentelemetry/instrumentation/rxjava2/RxJava2InstrumentationModule.java

@@ -26,6 +26,11 @@ public class RxJava2InstrumentationModule extends InstrumentationModule {
     super("rxjava2");
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("io.opentelemetry.instrumentation.rxjava2");
+  }
+
   @Override
   public List<TypeInstrumentation> typeInstrumentations() {
     return Collections.singletonList(new PluginInstrumentation());

instrumentation/spring/spring-webflux-5.0/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/spring/webflux/client/WebfluxClientInstrumentationModule.java

@@ -29,6 +29,11 @@ public class WebfluxClientInstrumentationModule extends InstrumentationModule {
     super("spring-webflux", "spring-webflux-5.0", "spring-webflux-client");
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("io.opentelemetry.instrumentation.spring.webflux.client");
+  }
+
   @Override
   public List<TypeInstrumentation> typeInstrumentations() {
     return singletonList(new WebClientBuilderInstrumentation());

instrumentation/spring/spring-webmvc-3.1/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/springwebmvc/SpringWebMvcInstrumentationModule.java

@@ -18,6 +18,11 @@ public class SpringWebMvcInstrumentationModule extends InstrumentationModule {
     super("spring-webmvc", "spring-webmvc-3.1");
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("io.opentelemetry.instrumentation.spring.webmvc");
+  }
+
   @Override
   public List<TypeInstrumentation> typeInstrumentations() {
     return asList(

javaagent-tooling/src/main/java/io/opentelemetry/javaagent/tooling/InstrumentationModule.java

@@ -5,7 +5,9 @@
 
 package io.opentelemetry.javaagent.tooling;
 
+import static io.opentelemetry.javaagent.tooling.bytebuddy.matcher.AgentElementMatchers.callWhenTrue;
 import static io.opentelemetry.javaagent.tooling.bytebuddy.matcher.AgentElementMatchers.failSafe;
+import static io.opentelemetry.javaagent.tooling.bytebuddy.matcher.ClassLoaderMatcher.hasAnyClassesNamed;
 import static java.util.Arrays.asList;
 import static net.bytebuddy.matcher.ElementMatchers.any;
 import static net.bytebuddy.matcher.ElementMatchers.isAnnotatedWith;
@@ -29,6 +31,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.function.Predicate;
+import java.util.stream.Collectors;
 import net.bytebuddy.agent.builder.AgentBuilder;
 import net.bytebuddy.description.annotation.AnnotationSource;
 import net.bytebuddy.description.method.MethodDescription;
@@ -130,7 +133,8 @@ public abstract class InstrumentationModule {
       return parentAgentBuilder;
     }
 
-    ElementMatcher.Junction<ClassLoader> moduleClassLoaderMatcher = classLoaderMatcher();
+    ElementMatcher.Junction<ClassLoader> moduleClassLoaderMatcher =
+        createModuleClassLoaderMatcher(helperClassNames);
     MuzzleMatcher muzzleMatcher = new MuzzleMatcher();
     HelperInjector helperInjector =
         new HelperInjector(mainInstrumentationName(), helperClassNames, helperResourceNames);
@@ -175,6 +179,40 @@ public abstract class InstrumentationModule {
     return helperClassNames;
   }
 
+  /**
+   * Returns the matcher produced by {@link #classLoaderMatcher()} with an additional condition
+   * added: no library instrumentation classes defined by this module can be present in the
+   * application classloader. This disables the javaagent instrumentation if the library
+   * instrumentation is already used by the application.
+   */
+  private ElementMatcher.Junction<ClassLoader> createModuleClassLoaderMatcher(
+      List<String> helperClassNames) {
+    // TODO: optimization: refactor matcher caching, only the matcher returned by this method should
+    // cache results, the intermediary ones don't need to cache anything
+    // right now every matcher returned from ClassLoaderMatcher caches separately
+    // this will also make those "Skipping ..." logs appear less
+
+    List<String> libraryHelperClasses =
+        helperClassNames.stream()
+            .filter(this::isLibraryInstrumentationClass)
+            .collect(Collectors.toList());
+
+    if (libraryHelperClasses.isEmpty()) {
+      return classLoaderMatcher();
+    }
+
+    ElementMatcher.Junction<ClassLoader> libraryMatcher =
+        callWhenTrue(
+            hasAnyClassesNamed(libraryHelperClasses), this::logLibraryInstrumentationDetected);
+    return classLoaderMatcher().and(not(libraryMatcher));
+  }
+
+  private void logLibraryInstrumentationDetected() {
+    log.debug(
+        "Skipping instrumentation {} because library instrumentation was detected on classpath",
+        mainInstrumentationName());
+  }
+
   private AgentBuilder.Identified.Extendable applyInstrumentationTransformers(
       Map<? extends ElementMatcher<? super MethodDescription>, String> transformers,
       AgentBuilder.Identified.Extendable agentBuilder) {
@@ -258,23 +296,8 @@ public abstract class InstrumentationModule {
    * Java helper function here.
    */
   @SuppressWarnings("unused")
-  protected final Predicate<String> additionalLibraryInstrumentationPackage() {
-    return this::isHelperClass;
-  }
-
-  /**
-   * Instrumentation modules can override this method to specify additional packages (or classes)
-   * that should be treated as "library instrumentation" packages. Classes from those packages will
-   * be treated by muzzle as instrumentation helper classes: they will be scanned for references and
-   * automatically injected into the application classloader if they're used in any type
-   * instrumentation. The classes for which this predicate returns {@code true} will be treated as
-   * helper classes, in addition to the default ones defined in {@link
-   * InstrumentationClassPredicate}.
-   *
-   * @param className The name of the class that may or may not be a helper class.
-   */
-  public boolean isHelperClass(String className) {
-    return false;
+  protected final Predicate<String> isLibraryInstrumentationClassPredicate() {
+    return this::isLibraryInstrumentationClass;
   }
 
   /**
@@ -304,9 +327,9 @@ public abstract class InstrumentationModule {
 
   /**
    * Instrumentation modules can override this method to provide additional helper classes that are
-   * not located in instrumentation packages described in {@link InstrumentationClassPredicate} (and
-   * not automatically detected by muzzle). These additional classes will be injected into the
-   * application classloader first.
+   * not located in instrumentation packages described in {@link InstrumentationClassPredicate} and
+   * {@link #isLibraryInstrumentationClass(String)} (and not automatically detected by muzzle).
+   * These additional classes will be injected into the application classloader first.
    */
   protected String[] additionalHelperClassNames() {
     return EMPTY;
@@ -342,6 +365,26 @@ public abstract class InstrumentationModule {
     return any();
   }
 
+  /**
+   * Instrumentation modules that use existing library instrumentation should override this method
+   * to specify additional packages (or classes) that should be treated as "library instrumentation"
+   * packages/classes.
+   *
+   * <p>Classes marked as library instrumentation classes will be treated by muzzle as
+   * instrumentation helper classes: they will be scanned for references and automatically injected
+   * into the application classloader if they're used in any type instrumentation.
+   *
+   * <p>In addition to that, the javaagent will prevent the instrumentations from this module from
+   * being applied when it detects that the application classloader already contains one of the
+   * library classes. This behavior prevents interference between library and javaagent
+   * instrumentation (for example, duplicate telemetry).
+   *
+   * @param className The name of the class that may or may not be a library instrumentation class.
+   */
+  public boolean isLibraryInstrumentationClass(String className) {
+    return false;
+  }
+
   /** Returns a list of all individual type instrumentation in this module. */
   public abstract List<TypeInstrumentation> typeInstrumentations();
 

javaagent-tooling/src/main/java/io/opentelemetry/javaagent/tooling/bytebuddy/matcher/AgentElementMatchers.java

@@ -92,4 +92,10 @@ public class AgentElementMatchers {
       }
     }
   }
+
+  /** Execute {@code callback} action whenever decorated {@code matcher} returns true. */
+  public static <T> ElementMatcher.Junction<T> callWhenTrue(
+      ElementMatcher<? super T> matcher, Runnable callback) {
+    return new CallWhenTrueDecorator<>(matcher, callback);
+  }
 }

javaagent-tooling/src/main/java/io/opentelemetry/javaagent/tooling/bytebuddy/matcher/CallWhenTrueDecorator.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javaagent.tooling.bytebuddy.matcher;
+
+import net.bytebuddy.matcher.ElementMatcher;
+
+final class CallWhenTrueDecorator<T> extends ElementMatcher.Junction.AbstractBase<T> {
+  private final ElementMatcher<? super T> delegate;
+  private final Runnable callback;
+
+  CallWhenTrueDecorator(ElementMatcher<? super T> delegate, Runnable callback) {
+    this.delegate = delegate;
+    this.callback = callback;
+  }
+
+  @Override
+  public boolean matches(T target) {
+    if (delegate.matches(target)) {
+      callback.run();
+      return true;
+    }
+    return false;
+  }
+}

javaagent-tooling/src/main/java/io/opentelemetry/javaagent/tooling/bytebuddy/matcher/ClassLoaderMatcher.java

@@ -5,8 +5,13 @@
 
 package io.opentelemetry.javaagent.tooling.bytebuddy.matcher;
 
+import static java.util.Arrays.asList;
+
 import io.opentelemetry.instrumentation.api.caching.Cache;
 import io.opentelemetry.javaagent.instrumentation.api.internal.InClassLoaderMatcher;
+import io.opentelemetry.javaagent.tooling.Utils;
+import java.util.List;
+import java.util.stream.Collectors;
 import net.bytebuddy.matcher.ElementMatcher;
 
 public final class ClassLoaderMatcher {
@@ -19,45 +24,40 @@ public final class ClassLoaderMatcher {
   }
 
   /**
-   * NOTICE: Does not match the bootstrap classpath. Don't use with classes expected to be on the
-   * bootstrap.
+   * Creates a matcher that checks if all of the passed classes are in the passed {@link
+   * ClassLoader}. If {@code classNames} is empty the matcher will always return {@code true}.
    *
-   * @param classNames list of names to match. returns true if empty.
-   * @return true if class is available as a resource and not the bootstrap classloader.
+   * <p>NOTICE: Does not match the bootstrap classpath. Don't use with classes expected to be on the
+   * bootstrap.
    */
-  public static ElementMatcher.Junction.AbstractBase<ClassLoader> hasClassesNamed(
-      String... classNames) {
-    return new ClassLoaderHasClassesNamedMatcher(classNames);
+  public static ElementMatcher.Junction<ClassLoader> hasClassesNamed(String... classNames) {
+    return new ClassLoaderHasAllClassesNamedMatcher(asList(classNames));
   }
 
-  private static class ClassLoaderHasClassesNamedMatcher
+  /**
+   * Creates a matcher that checks if any of the passed classes are in the passed {@link
+   * ClassLoader}. If {@code classNames} is empty the matcher will always return {@code false}.
+   *
+   * <p>NOTICE: Does not match the bootstrap classpath. Don't use with classes expected to be on the
+   * bootstrap.
+   */
+  public static ElementMatcher.Junction<ClassLoader> hasAnyClassesNamed(List<String> classNames) {
+    return new ClassLoaderHasAnyClassesNamedMatcher(classNames);
+  }
+
+  private abstract static class AbstractClassLoaderMatcher
       extends ElementMatcher.Junction.AbstractBase<ClassLoader> {
 
     private final Cache<ClassLoader, Boolean> cache =
         Cache.newBuilder().setWeakKeys().setMaximumSize(25).build();
 
-    private final String[] resources;
+    protected final List<String> resources;
 
-    private ClassLoaderHasClassesNamedMatcher(String... classNames) {
-      resources = classNames;
-      for (int i = 0; i < resources.length; i++) {
-        resources[i] = resources[i].replace(".", "/") + ".class";
-      }
+    private AbstractClassLoaderMatcher(List<String> classNames) {
+      resources = classNames.stream().map(Utils::getResourceName).collect(Collectors.toList());
     }
 
-    private boolean hasResources(ClassLoader cl) {
-      boolean priorValue = InClassLoaderMatcher.getAndSet(true);
-      try {
-        for (String resource : resources) {
-          if (cl.getResource(resource) == null) {
-            return false;
-          }
-        }
-      } finally {
-        InClassLoaderMatcher.set(priorValue);
-      }
-      return true;
-    }
+    protected abstract boolean doMatch(ClassLoader cl);
 
     @Override
     public boolean matches(ClassLoader cl) {
@@ -67,5 +67,48 @@ public final class ClassLoaderMatcher {
       }
       return cache.computeIfAbsent(cl, this::hasResources);
     }
+
+    private boolean hasResources(ClassLoader cl) {
+      boolean priorValue = InClassLoaderMatcher.getAndSet(true);
+      boolean value;
+      try {
+        value = doMatch(cl);
+      } finally {
+        InClassLoaderMatcher.set(priorValue);
+      }
+      return value;
+    }
+  }
+
+  private static class ClassLoaderHasAllClassesNamedMatcher extends AbstractClassLoaderMatcher {
+    private ClassLoaderHasAllClassesNamedMatcher(List<String> classNames) {
+      super(classNames);
+    }
+
+    @Override
+    protected boolean doMatch(ClassLoader cl) {
+      for (String resource : resources) {
+        if (cl.getResource(resource) == null) {
+          return false;
+        }
+      }
+      return true;
+    }
+  }
+
+  private static class ClassLoaderHasAnyClassesNamedMatcher extends AbstractClassLoaderMatcher {
+    private ClassLoaderHasAnyClassesNamedMatcher(List<String> classNames) {
+      super(classNames);
+    }
+
+    @Override
+    protected boolean doMatch(ClassLoader cl) {
+      for (String resource : resources) {
+        if (cl.getResource(resource) != null) {
+          return true;
+        }
+      }
+      return false;
+    }
   }
 }

javaagent-tooling/src/main/java/io/opentelemetry/javaagent/tooling/muzzle/InstrumentationClassPredicate.java

@@ -14,8 +14,9 @@ public final class InstrumentationClassPredicate {
   private static final String JAVAAGENT_API_PACKAGE =
       "io.opentelemetry.javaagent.instrumentation.api.";
 
-  // library instrumentation packages (both shaded in the agent)
+  // library instrumentation packages
   private static final String LIBRARY_INSTRUMENTATION_PACKAGE = "io.opentelemetry.instrumentation.";
+  // note that instrumentation-api is shaded in the agent
   private static final String INSTRUMENTATION_API_PACKAGE = "io.opentelemetry.instrumentation.api.";
 
   private final Predicate<String> additionalLibraryInstrumentationPredicate;

javaagent-tooling/src/main/java/io/opentelemetry/javaagent/tooling/muzzle/collector/MuzzleCodeGenerator.java

@@ -147,7 +147,8 @@ class MuzzleCodeGenerator implements AsmVisitorWrapper {
               .flatMap(typeInstrumentation -> typeInstrumentation.transformers().values().stream())
               .collect(Collectors.toSet());
 
-      ReferenceCollector collector = new ReferenceCollector(instrumentationModule::isHelperClass);
+      ReferenceCollector collector =
+          new ReferenceCollector(instrumentationModule::isLibraryInstrumentationClass);
       for (String adviceClass : adviceClassNames) {
         collector.collectReferencesFromAdvice(adviceClass);
       }
@@ -206,7 +207,7 @@ class MuzzleCodeGenerator implements AsmVisitorWrapper {
        *                                                        new Reference[]{
        *                                                          // reference builders
        *                                                        },
-       *                                                        this.additionalLibraryInstrumentationPackage());
+       *                                                        this.isLibraryInstrumentationClassPredicate());
        *   }
        *   return this.muzzleReferenceMatcher;
        * }
@@ -472,7 +473,7 @@ class MuzzleCodeGenerator implements AsmVisitorWrapper {
         mv.visitMethodInsn(
             Opcodes.INVOKEVIRTUAL,
             instrumentationClassName,
-            "additionalLibraryInstrumentationPackage",
+            "isLibraryInstrumentationClassPredicate",
             "()Ljava/util/function/Predicate;",
             false);
 

javaagent-tooling/src/test/java/io/opentelemetry/javaagent/tooling/bytebuddy/matcher/CallWhenTrueDecoratorTest.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javaagent.tooling.bytebuddy.matcher;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.BDDMockito.given;
+import static org.mockito.BDDMockito.then;
+import static org.mockito.Mockito.never;
+
+import net.bytebuddy.matcher.ElementMatcher;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+@ExtendWith(MockitoExtension.class)
+class CallWhenTrueDecoratorTest {
+  @Mock ElementMatcher<String> delegateMatcher;
+  @Mock Runnable callback;
+
+  @Test
+  void shouldExecuteCallbackWhenMatcherReturnsTrue() {
+    // given
+    ElementMatcher<String> matcher = new CallWhenTrueDecorator<>(delegateMatcher, callback);
+
+    given(delegateMatcher.matches("true")).willReturn(true);
+
+    // when
+    boolean result = matcher.matches("true");
+
+    // then
+    assertTrue(result);
+    then(callback).should().run();
+  }
+
+  @Test
+  void shouldNotExecuteCallbackWhenMatcherReturnsFalse() {
+    // given
+    ElementMatcher<String> matcher = new CallWhenTrueDecorator<>(delegateMatcher, callback);
+
+    // when
+    boolean result = matcher.matches("not really true");
+
+    // then
+    assertFalse(result);
+    then(callback).should(never()).run();
+  }
+}

javaagent/javaagent.gradle

@@ -87,7 +87,7 @@ tasks.withType(ShadowJar).configureEach {
   // rewrite dependencies calling Logger.getLogger
   relocate 'java.util.logging.Logger', 'io.opentelemetry.javaagent.bootstrap.PatchLogger'
 
-  // prevents conflict with library instrumentation
+  // prevents conflict with library instrumentation, which uses its own instrumentation-api
   relocate 'io.opentelemetry.instrumentation.api', 'io.opentelemetry.javaagent.shaded.instrumentation.api'
 
   // relocate OpenTelemetry API

testing/agent-exporter/agent-exporter.gradle

@@ -43,12 +43,12 @@ shadowJar {
   // Prevents conflict with other SLF4J instances. Important for premain.
   relocate 'org.slf4j', 'io.opentelemetry.javaagent.slf4j'
 
-  // rewrite library instrumentation dependencies
-  relocate "io.opentelemetry.instrumentation", "io.opentelemetry.javaagent.shaded.instrumentation"
-
   // rewrite dependencies calling Logger.getLogger
   relocate 'java.util.logging.Logger', 'io.opentelemetry.javaagent.bootstrap.PatchLogger'
 
+  // prevents conflict with library instrumentation, which uses its own instrumentation-api
+  relocate 'io.opentelemetry.instrumentation.api', 'io.opentelemetry.javaagent.shaded.instrumentation.api'
+
   // relocate OpenTelemetry API usage
   relocate "io.opentelemetry.api", "io.opentelemetry.javaagent.shaded.io.opentelemetry.api"
   relocate "io.opentelemetry.semconv", "io.opentelemetry.javaagent.shaded.io.opentelemetry.semconv"

testing/agent-for-testing/agent-for-testing.gradle

@@ -59,7 +59,7 @@ shadowJar {
   // rewrite dependencies calling Logger.getLogger
   relocate 'java.util.logging.Logger', 'io.opentelemetry.javaagent.bootstrap.PatchLogger'
 
-  // prevents conflict with library instrumentation
+  // prevents conflict with library instrumentation, which uses its own instrumentation-api
   relocate 'io.opentelemetry.instrumentation.api', 'io.opentelemetry.javaagent.shaded.instrumentation.api'
 
   // relocate OpenTelemetry API