Implement MDC auto-instrumentation for log4j2 (#1200)

* Implement MDC auto-instrumentation for log4j2

* Implement MDC auto-instrumentation for log4j2 2.7

* Implement MDC auto-instrumentation for log4j2

* Implement MDC auto-instrumentation for log4j2

* Implement MDC auto-instrumentation for log4j2

* Implement MDC auto-instrumentation for log4j1

* Implement MDC auto-instrumentation for log4j2
This commit is contained in:
Mateusz Rzeszutek 2020-09-18 16:53:43 +02:00 committed by GitHub
parent beb1901f5f
commit d89ce818ef
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 414 additions and 39 deletions

View File

@ -204,6 +204,7 @@ provide the path to a JAR file including an SPI implementation using the system
| [khttp](https://khttp.readthedocs.io) | 0.1+ |
| [Kubernetes Client](https://github.com/kubernetes-client/java) | 7.0+ |
| [Lettuce](https://github.com/lettuce-io/lettuce-core) | 4.0+ |
| [Log4j 2](https://logging.apache.org/log4j/2.x/) | 2.7+ |
| [Logback](http://logback.qos.ch/) | 1.0+ |
| [MongoDB Drivers](https://mongodb.github.io/mongo-java-driver/) | 3.3+ |
| [Netty](https://github.com/netty/netty) | 3.8+ |

View File

@ -0,0 +1,39 @@
/*
* Copyright The OpenTelemetry Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.opentelemetry.instrumentation.api.log;
/**
* This class contains several constants used in logging libraries' Mapped Diagnostic Context
* instrumentations.
*
* @see org.slf4j.MDC
* @see org.apache.logging.log4j.ThreadContext
* @see org.apache.log4j.MDC
*/
public final class LoggingContextConstants {
/** Key under which the current trace id will be injected into the context data. */
public static final String TRACE_ID = "traceId";
/** Key under which the current span id will be injected into the context data. */
public static final String SPAN_ID = "spanId";
/**
* Key under which a boolean indicating whether current span is sampled will be injected into the
* context data.
*/
public static final String SAMPLED = "sampled";
private LoggingContextConstants() {}
}

View File

@ -16,6 +16,9 @@
package io.opentelemetry.instrumentation.auto.log4j.v1_2;
import static io.opentelemetry.instrumentation.api.log.LoggingContextConstants.SAMPLED;
import static io.opentelemetry.instrumentation.api.log.LoggingContextConstants.SPAN_ID;
import static io.opentelemetry.instrumentation.api.log.LoggingContextConstants.TRACE_ID;
import static java.util.Collections.singletonMap;
import static net.bytebuddy.matcher.ElementMatchers.isMethod;
import static net.bytebuddy.matcher.ElementMatchers.isPublic;
@ -79,7 +82,7 @@ public class Log4j1LoggingEventInstrumentation extends Instrumenter.Default {
@Advice.This LoggingEvent event,
@Advice.Argument(0) String key,
@Advice.Return(readOnly = false) Object value) {
if ("traceId".equals(key) || "spanId".equals(key) || "sampled".equals(key)) {
if (TRACE_ID.equals(key) || SPAN_ID.equals(key) || SAMPLED.equals(key)) {
if (value != null) {
// Assume already instrumented event if traceId/spanId/sampled is present.
return;
@ -92,13 +95,13 @@ public class Log4j1LoggingEventInstrumentation extends Instrumenter.Default {
SpanContext spanContext = span.getContext();
switch (key) {
case "traceId":
case TRACE_ID:
value = spanContext.getTraceIdAsHexString();
break;
case "spanId":
case SPAN_ID:
value = spanContext.getSpanIdAsHexString();
break;
case "sampled":
case SAMPLED:
if (spanContext.isSampled()) {
value = "true";
}
@ -128,14 +131,14 @@ public class Log4j1LoggingEventInstrumentation extends Instrumenter.Default {
}
// Assume already instrumented event if traceId is present.
if (!mdc.contains("traceId")) {
if (!mdc.contains(TRACE_ID)) {
Span span = InstrumentationContext.get(LoggingEvent.class, Span.class).get(event);
if (span != null && span.getContext().isValid()) {
SpanContext spanContext = span.getContext();
mdc.put("traceId", spanContext.getTraceIdAsHexString());
mdc.put("spanId", spanContext.getSpanIdAsHexString());
mdc.put(TRACE_ID, spanContext.getTraceIdAsHexString());
mdc.put(SPAN_ID, spanContext.getSpanIdAsHexString());
if (spanContext.isSampled()) {
mdc.put("sampled", "true");
mdc.put(SAMPLED, "true");
}
}
}

View File

@ -0,0 +1,15 @@
apply from: "$rootDir/gradle/java.gradle"
dependencies {
api project(':testing-common')
api group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.7'
implementation deps.guava
implementation deps.groovy
implementation deps.opentelemetryApi
implementation deps.spock
annotationProcessor group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.7'
}

View File

@ -14,24 +14,22 @@
* limitations under the License.
*/
package io.opentelemetry.instrumentation.log4j.v2_13_2
import io.opentelemetry.auto.test.InstrumentationSpecification
import io.opentelemetry.auto.test.utils.TraceUtils
import io.opentelemetry.instrumentation.log4j.v2_13_2.ListAppender
import io.opentelemetry.trace.Span
import io.opentelemetry.trace.TracingContextUtils
import org.apache.logging.log4j.LogManager
import org.apache.logging.log4j.Logger
import spock.lang.Specification
class Log4j2Test extends Specification {
private static final Logger logger = LogManager.getLogger("TestLogger")
abstract class Log4j2Test extends InstrumentationSpecification {
def cleanup() {
ListAppender.get().clearEvents()
}
def "no ids when no span"() {
given:
def logger = LogManager.getLogger("TestLogger")
when:
logger.info("log message 1")
logger.info("log message 2")
@ -52,6 +50,9 @@ class Log4j2Test extends Specification {
}
def "ids when span"() {
given:
def logger = LogManager.getLogger("TestLogger")
when:
Span span1
TraceUtils.runUnderTrace("test") {

View File

@ -20,17 +20,15 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.apache.logging.log4j.core.Appender;
import org.apache.logging.log4j.core.Core;
import org.apache.logging.log4j.core.LogEvent;
import org.apache.logging.log4j.core.appender.AbstractAppender;
import org.apache.logging.log4j.core.config.Property;
import org.apache.logging.log4j.core.config.plugins.Plugin;
import org.apache.logging.log4j.core.config.plugins.PluginAttribute;
import org.apache.logging.log4j.core.config.plugins.PluginFactory;
@Plugin(
name = "ListAppender",
category = Core.CATEGORY_NAME,
category = "Core",
elementType = Appender.ELEMENT_TYPE,
printObject = true)
public class ListAppender extends AbstractAppender {
@ -41,10 +39,10 @@ public class ListAppender extends AbstractAppender {
private static final ListAppender INSTANCE = new ListAppender();
private final List<LogEvent> events = Collections.synchronizedList(new ArrayList<>());
private final List<LogEvent> events = Collections.synchronizedList(new ArrayList<LogEvent>());
public ListAppender() {
super("ListAppender", null, null, true, Property.EMPTY_ARRAY);
super("ListAppender", null, null, true);
}
public List<LogEvent> getEvents() {

View File

@ -0,0 +1,18 @@
apply from: "$rootDir/gradle/instrumentation.gradle"
muzzle {
pass {
group = "org.apache.logging.log4j"
module = "log4j-core"
versions = "[2.13.2,)"
assertInverse = true
}
}
dependencies {
library group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.13.2'
implementation project(':instrumentation:log4j:log4j-2.13.2:library')
testImplementation project(':instrumentation:log4j:log4j-2-testing')
}

View File

@ -0,0 +1,69 @@
/*
* Copyright The OpenTelemetry Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.opentelemetry.instrumentation.auto.log4j.v2_13_2;
import static io.opentelemetry.javaagent.tooling.ClassLoaderMatcher.hasClassesNamed;
import static net.bytebuddy.matcher.ElementMatchers.named;
import com.google.auto.service.AutoService;
import io.opentelemetry.javaagent.tooling.Instrumenter;
import java.util.Collections;
import java.util.Map;
import net.bytebuddy.description.method.MethodDescription;
import net.bytebuddy.description.type.TypeDescription;
import net.bytebuddy.matcher.ElementMatcher;
@AutoService(Instrumenter.class)
public final class Log4j2MdcInstrumentation extends Instrumenter.Default {
public Log4j2MdcInstrumentation() {
super("log4j2", "log4j", "log4j-2.13.2");
}
@Override
public ElementMatcher<ClassLoader> classLoaderMatcher() {
return hasClassesNamed("org.apache.logging.log4j.core.util.ContextDataProvider");
}
@Override
public ElementMatcher<? super TypeDescription> typeMatcher() {
// we cannot use ContextDataProvider here because one of the classes that we inject implements
// this interface, causing the interface to be loaded while it's being transformed, which leads
// to duplicate class definition error after the interface is transformed and the triggering
// class loader tries to load it.
return named("org.apache.logging.log4j.core.impl.ThreadContextDataInjector");
}
@Override
public String[] helperResourceNames() {
return new String[] {
"META-INF/services/org.apache.logging.log4j.core.util.ContextDataProvider",
};
}
@Override
public String[] helperClassNames() {
return new String[] {
"io.opentelemetry.instrumentation.log4j.v2_13_2.OpenTelemetryContextDataProvider"
};
}
@Override
public Map<? extends ElementMatcher<? super MethodDescription>, String> transformers() {
// Nothing to instrument, injecting helper resource & class is enough
return Collections.emptyMap();
}
}

View File

@ -0,0 +1,20 @@
/*
* Copyright The OpenTelemetry Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import io.opentelemetry.auto.test.AgentTestTrait
class AutoLog4j2Test extends Log4j2Test implements AgentTestTrait {
}

View File

@ -31,7 +31,7 @@ a log statement is made when a span is active.
- `traceId`
- `spanId`
- `traceFlags`
- `sampled`
You can use these keys when defining an appender in your `log4j.xml` configuration, for example

View File

@ -7,8 +7,5 @@ apply from: "$rootDir/gradle/instrumentation-library.gradle"
dependencies {
library group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.13.2'
annotationProcessor deps.autoservice
compileOnly deps.autoservice
testAnnotationProcessor group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.13.2'
testImplementation project(':instrumentation:log4j:log4j-2-testing')
}

View File

@ -16,7 +16,10 @@
package io.opentelemetry.instrumentation.log4j.v2_13_2;
import com.google.auto.service.AutoService;
import static io.opentelemetry.instrumentation.api.log.LoggingContextConstants.SAMPLED;
import static io.opentelemetry.instrumentation.api.log.LoggingContextConstants.SPAN_ID;
import static io.opentelemetry.instrumentation.api.log.LoggingContextConstants.TRACE_ID;
import io.opentelemetry.trace.Span;
import io.opentelemetry.trace.SpanContext;
import io.opentelemetry.trace.TracingContextUtils;
@ -29,7 +32,6 @@ import org.apache.logging.log4j.core.util.ContextDataProvider;
* Implementation of Log4j 2's {@link ContextDataProvider} which is loaded via SPI. {@link
* #supplyContextData()} is called when a log entry is created.
*/
@AutoService(ContextDataProvider.class)
public class OpenTelemetryContextDataProvider implements ContextDataProvider {
/**
@ -47,10 +49,10 @@ public class OpenTelemetryContextDataProvider implements ContextDataProvider {
Map<String, String> contextData = new HashMap<>();
SpanContext spanContext = currentSpan.getContext();
contextData.put("traceId", spanContext.getTraceIdAsHexString());
contextData.put("spanId", spanContext.getSpanIdAsHexString());
contextData.put(TRACE_ID, spanContext.getTraceIdAsHexString());
contextData.put(SPAN_ID, spanContext.getSpanIdAsHexString());
if (spanContext.isSampled()) {
contextData.put("sampled", "true");
contextData.put(SAMPLED, "true");
}
return contextData;
}

View File

@ -0,0 +1 @@
io.opentelemetry.instrumentation.log4j.v2_13_2.OpenTelemetryContextDataProvider

View File

@ -0,0 +1,20 @@
/*
* Copyright The OpenTelemetry Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import io.opentelemetry.auto.test.InstrumentationTestTrait
class LibraryLog4j2Test extends Log4j2Test implements InstrumentationTestTrait {
}

View File

@ -0,0 +1,15 @@
apply from: "$rootDir/gradle/instrumentation.gradle"
muzzle {
pass {
group = "org.apache.logging.log4j"
module = "log4j-core"
versions = "[2.7,2.13.2)"
}
}
dependencies {
library group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.7'
testImplementation project(':instrumentation:log4j:log4j-2-testing')
}

View File

@ -0,0 +1,80 @@
/*
* Copyright The OpenTelemetry Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.opentelemetry.instrumentation.auto.log4j.v2_7;
import static io.opentelemetry.javaagent.tooling.ClassLoaderMatcher.hasClassesNamed;
import static net.bytebuddy.matcher.ElementMatchers.isMethod;
import static net.bytebuddy.matcher.ElementMatchers.isPublic;
import static net.bytebuddy.matcher.ElementMatchers.isStatic;
import static net.bytebuddy.matcher.ElementMatchers.named;
import static net.bytebuddy.matcher.ElementMatchers.not;
import static net.bytebuddy.matcher.ElementMatchers.returns;
import com.google.auto.service.AutoService;
import io.opentelemetry.javaagent.tooling.Instrumenter;
import java.util.Collections;
import java.util.Map;
import net.bytebuddy.asm.Advice;
import net.bytebuddy.description.method.MethodDescription;
import net.bytebuddy.description.type.TypeDescription;
import net.bytebuddy.implementation.bytecode.assign.Assigner.Typing;
import net.bytebuddy.matcher.ElementMatcher;
import org.apache.logging.log4j.core.ContextDataInjector;
@AutoService(Instrumenter.class)
public class Log4j27MdcInstrumentation extends Instrumenter.Default {
public Log4j27MdcInstrumentation() {
super("log4j2", "log4j", "log4j-2.7");
}
@Override
public String[] helperClassNames() {
return new String[] {
"io.opentelemetry.instrumentation.auto.log4j.v2_7.SpanDecoratingContextDataInjector"
};
}
@Override
public ElementMatcher<ClassLoader> classLoaderMatcher() {
return hasClassesNamed("org.apache.logging.log4j.core.impl.ContextDataInjectorFactory")
.and(not(hasClassesNamed("org.apache.logging.log4j.core.util.ContextDataProvider")));
}
@Override
public ElementMatcher<? super TypeDescription> typeMatcher() {
return named("org.apache.logging.log4j.core.impl.ContextDataInjectorFactory");
}
@Override
public Map<? extends ElementMatcher<? super MethodDescription>, String> transformers() {
return Collections.singletonMap(
isMethod()
.and(isPublic())
.and(isStatic())
.and(named("createInjector"))
.and(returns(named("org.apache.logging.log4j.core.ContextDataInjector"))),
Log4j27MdcInstrumentation.class.getName() + "$CreateInjectorAdvice");
}
public static class CreateInjectorAdvice {
@Advice.OnMethodExit(suppress = Throwable.class)
public static void onExit(
@Advice.Return(typing = Typing.DYNAMIC, readOnly = false) ContextDataInjector injector) {
injector = new SpanDecoratingContextDataInjector(injector);
}
}
}

View File

@ -0,0 +1,66 @@
/*
* Copyright The OpenTelemetry Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.opentelemetry.instrumentation.auto.log4j.v2_7;
import static io.opentelemetry.instrumentation.api.log.LoggingContextConstants.SAMPLED;
import static io.opentelemetry.instrumentation.api.log.LoggingContextConstants.SPAN_ID;
import static io.opentelemetry.instrumentation.api.log.LoggingContextConstants.TRACE_ID;
import io.opentelemetry.trace.SpanContext;
import io.opentelemetry.trace.TracingContextUtils;
import java.util.List;
import org.apache.logging.log4j.core.ContextDataInjector;
import org.apache.logging.log4j.core.config.Property;
import org.apache.logging.log4j.util.ReadOnlyStringMap;
import org.apache.logging.log4j.util.SortedArrayStringMap;
import org.apache.logging.log4j.util.StringMap;
public final class SpanDecoratingContextDataInjector implements ContextDataInjector {
private final ContextDataInjector delegate;
public SpanDecoratingContextDataInjector(ContextDataInjector delegate) {
this.delegate = delegate;
}
@Override
public StringMap injectContextData(List<Property> list, StringMap stringMap) {
StringMap contextData = delegate.injectContextData(list, stringMap);
if (contextData.containsKey(TRACE_ID)) {
// Assume already instrumented event if traceId is present.
return contextData;
}
SpanContext currentContext = TracingContextUtils.getCurrentSpan().getContext();
if (!currentContext.isValid()) {
return contextData;
}
StringMap newContextData = new SortedArrayStringMap(contextData);
newContextData.putValue(TRACE_ID, currentContext.getTraceIdAsHexString());
newContextData.putValue(SPAN_ID, currentContext.getSpanIdAsHexString());
if (currentContext.isSampled()) {
newContextData.putValue(SAMPLED, "true");
}
return newContextData;
}
@Override
public ReadOnlyStringMap rawContextData() {
return delegate.rawContextData();
}
}

View File

@ -0,0 +1,20 @@
/*
* Copyright The OpenTelemetry Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import io.opentelemetry.auto.test.AgentTestTrait
class Log4j27Test extends Log4j2Test implements AgentTestTrait {
}

View File

@ -16,6 +16,9 @@
package io.opentelemetry.instrumentation.auto.logback.v1_0_0;
import static io.opentelemetry.instrumentation.api.log.LoggingContextConstants.SAMPLED;
import static io.opentelemetry.instrumentation.api.log.LoggingContextConstants.SPAN_ID;
import static io.opentelemetry.instrumentation.api.log.LoggingContextConstants.TRACE_ID;
import static io.opentelemetry.javaagent.tooling.bytebuddy.matcher.AgentElementMatchers.implementsInterface;
import static java.util.Collections.singletonMap;
import static net.bytebuddy.matcher.ElementMatchers.isMethod;
@ -78,7 +81,7 @@ public class LoggingEventInstrumentation extends Instrumenter.Default {
public static void onExit(
@Advice.This ILoggingEvent event,
@Advice.Return(typing = Typing.DYNAMIC, readOnly = false) Map<String, String> contextData) {
if (contextData != null && contextData.containsKey("traceId")) {
if (contextData != null && contextData.containsKey(TRACE_ID)) {
// Assume already instrumented event if traceId is present.
return;
}
@ -90,10 +93,10 @@ public class LoggingEventInstrumentation extends Instrumenter.Default {
Map<String, String> spanContextData = new HashMap<>();
SpanContext spanContext = currentSpan.getContext();
spanContextData.put("traceId", spanContext.getTraceIdAsHexString());
spanContextData.put("spanId", spanContext.getSpanIdAsHexString());
spanContextData.put(TRACE_ID, spanContext.getTraceIdAsHexString());
spanContextData.put(SPAN_ID, spanContext.getSpanIdAsHexString());
if (spanContext.isSampled()) {
spanContextData.put("sampled", "true");
spanContextData.put(SAMPLED, "true");
}
if (contextData == null) {

View File

@ -16,6 +16,10 @@
package io.opentelemetry.instrumentation.logback.v1_0_0;
import static io.opentelemetry.instrumentation.api.log.LoggingContextConstants.SAMPLED;
import static io.opentelemetry.instrumentation.api.log.LoggingContextConstants.SPAN_ID;
import static io.opentelemetry.instrumentation.api.log.LoggingContextConstants.TRACE_ID;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.Appender;
import ch.qos.logback.core.UnsynchronizedAppenderBase;
@ -41,17 +45,17 @@ public class OpenTelemetryAppender extends UnsynchronizedAppenderBase<ILoggingEv
}
Map<String, String> eventContext = event.getMDCPropertyMap();
if (eventContext != null && eventContext.containsKey("traceId")) {
if (eventContext != null && eventContext.containsKey(TRACE_ID)) {
// Assume already instrumented event if traceId is present.
return event;
}
Map<String, String> contextData = new HashMap<>();
SpanContext spanContext = currentSpan.getContext();
contextData.put("traceId", spanContext.getTraceIdAsHexString());
contextData.put("spanId", spanContext.getSpanIdAsHexString());
contextData.put(TRACE_ID, spanContext.getTraceIdAsHexString());
contextData.put(SPAN_ID, spanContext.getSpanIdAsHexString());
if (spanContext.isSampled()) {
contextData.put("sampled", "true");
contextData.put(SAMPLED, "true");
}
if (eventContext == null) {

View File

@ -127,7 +127,10 @@ include ':instrumentation:lettuce:lettuce-4.0'
include ':instrumentation:lettuce:lettuce-5.0'
include ':instrumentation:lettuce:lettuce-5.1'
include ':instrumentation:log4j:log4j-1.2'
include ':instrumentation:log4j:log4j-2.7'
include ':instrumentation:log4j:log4j-2.13.2:auto'
include ':instrumentation:log4j:log4j-2.13.2:library'
include ':instrumentation:log4j:log4j-2-testing'
include ':instrumentation:logback:logback-1.0.0:auto'
include ':instrumentation:logback:logback-1.0.0:library'
include ':instrumentation:logback:logback-1.0.0:testing'