Add config to enable individual executors

Or all executors, bypassing the allow list.

`dd.trace.executor=com.MyCustomExecutor,com.OtherExecutor`
`dd.trace.executors.all=true`

Turns out in many cases, executors that we say we’re skipping, are still being traced because they extend from an already instrumented executor.
This commit is contained in:
Tyler Benson 2019-05-13 17:33:40 -07:00
parent ad2663d840
commit cc23fee614
3 changed files with 210 additions and 68 deletions

View File

@ -6,10 +6,12 @@ import static net.bytebuddy.matcher.ElementMatchers.named;
import static net.bytebuddy.matcher.ElementMatchers.not; import static net.bytebuddy.matcher.ElementMatchers.not;
import datadog.trace.agent.tooling.Instrumenter; import datadog.trace.agent.tooling.Instrumenter;
import datadog.trace.api.Config;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collection; import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.HashSet; import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.Executor; import java.util.concurrent.Executor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import net.bytebuddy.description.type.TypeDescription; import net.bytebuddy.description.type.TypeDescription;
@ -20,11 +22,15 @@ public abstract class AbstractExecutorInstrumentation extends Instrumenter.Defau
public static final String EXEC_NAME = "java_concurrent"; public static final String EXEC_NAME = "java_concurrent";
private static final boolean TRACE_ALL_EXECUTORS =
Config.getBooleanSettingFromEnvironment("trace.executors.all", false);
/** /**
* Only apply executor instrumentation to whitelisted executors. In the future, this restriction * Only apply executor instrumentation to whitelisted executors. To apply to all executors, use
* may be lifted to include all executors. * override setting above.
*/ */
private static final Collection<String> WHITELISTED_EXECUTORS; private static final Collection<String> WHITELISTED_EXECUTORS;
/** /**
* Some frameworks have their executors defined as anon classes inside other classes. Referencing * Some frameworks have their executors defined as anon classes inside other classes. Referencing
* anon classes by name would be fragile, so instead we will use list of class prefix names. Since * anon classes by name would be fragile, so instead we will use list of class prefix names. Since
@ -33,51 +39,62 @@ public abstract class AbstractExecutorInstrumentation extends Instrumenter.Defau
private static final Collection<String> WHITELISTED_EXECUTORS_PREFIXES; private static final Collection<String> WHITELISTED_EXECUTORS_PREFIXES;
static { static {
final String[] whitelist = { if (TRACE_ALL_EXECUTORS) {
"java.util.concurrent.AbstractExecutorService", log.info("Tracing all executors enabled.");
"java.util.concurrent.ThreadPoolExecutor", WHITELISTED_EXECUTORS = Collections.emptyList();
"java.util.concurrent.ScheduledThreadPoolExecutor", WHITELISTED_EXECUTORS_PREFIXES = Collections.emptyList();
"java.util.concurrent.ForkJoinPool", } else {
"java.util.concurrent.Executors$FinalizableDelegatedExecutorService", final String[] whitelist = {
"java.util.concurrent.Executors$DelegatedExecutorService", "java.util.concurrent.AbstractExecutorService",
"javax.management.NotificationBroadcasterSupport$1", "java.util.concurrent.ThreadPoolExecutor",
"kotlinx.coroutines.scheduling.CoroutineScheduler", "java.util.concurrent.ScheduledThreadPoolExecutor",
"scala.concurrent.Future$InternalCallbackExecutor$", "java.util.concurrent.ForkJoinPool",
"scala.concurrent.impl.ExecutionContextImpl", "java.util.concurrent.Executors$FinalizableDelegatedExecutorService",
"scala.concurrent.impl.ExecutionContextImpl$$anon$1", "java.util.concurrent.Executors$DelegatedExecutorService",
"scala.concurrent.forkjoin.ForkJoinPool", "javax.management.NotificationBroadcasterSupport$1",
"scala.concurrent.impl.ExecutionContextImpl$$anon$3", "kotlinx.coroutines.scheduling.CoroutineScheduler",
"akka.dispatch.MessageDispatcher", "scala.concurrent.Future$InternalCallbackExecutor$",
"akka.dispatch.Dispatcher", "scala.concurrent.impl.ExecutionContextImpl",
"akka.dispatch.Dispatcher$LazyExecutorServiceDelegate", "scala.concurrent.impl.ExecutionContextImpl$$anon$1",
"akka.actor.ActorSystemImpl$$anon$1", "scala.concurrent.forkjoin.ForkJoinPool",
"akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool", "scala.concurrent.impl.ExecutionContextImpl$$anon$3",
"akka.dispatch.forkjoin.ForkJoinPool", "akka.dispatch.MessageDispatcher",
"akka.dispatch.BalancingDispatcher", "akka.dispatch.Dispatcher",
"akka.dispatch.ThreadPoolConfig$ThreadPoolExecutorServiceFactory$$anon$1", "akka.dispatch.Dispatcher$LazyExecutorServiceDelegate",
"akka.dispatch.PinnedDispatcher", "akka.actor.ActorSystemImpl$$anon$1",
"akka.dispatch.ExecutionContexts$sameThreadExecutionContext$", "akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool",
"play.api.libs.streams.Execution$trampoline$", "akka.dispatch.forkjoin.ForkJoinPool",
"io.netty.channel.MultithreadEventLoopGroup", "akka.dispatch.BalancingDispatcher",
"io.netty.util.concurrent.MultithreadEventExecutorGroup", "akka.dispatch.ThreadPoolConfig$ThreadPoolExecutorServiceFactory$$anon$1",
"io.netty.util.concurrent.AbstractEventExecutorGroup", "akka.dispatch.PinnedDispatcher",
"io.netty.channel.epoll.EpollEventLoopGroup", "akka.dispatch.ExecutionContexts$sameThreadExecutionContext$",
"io.netty.channel.nio.NioEventLoopGroup", "play.api.libs.streams.Execution$trampoline$",
"io.netty.util.concurrent.GlobalEventExecutor", "io.netty.channel.MultithreadEventLoopGroup",
"io.netty.util.concurrent.AbstractScheduledEventExecutor", "io.netty.util.concurrent.MultithreadEventExecutorGroup",
"io.netty.util.concurrent.AbstractEventExecutor", "io.netty.util.concurrent.AbstractEventExecutorGroup",
"io.netty.util.concurrent.SingleThreadEventExecutor", "io.netty.channel.epoll.EpollEventLoopGroup",
"io.netty.channel.nio.NioEventLoop", "io.netty.channel.nio.NioEventLoopGroup",
"io.netty.channel.SingleThreadEventLoop", "io.netty.util.concurrent.GlobalEventExecutor",
"com.google.common.util.concurrent.AbstractListeningExecutorService", "io.netty.util.concurrent.AbstractScheduledEventExecutor",
"com.google.common.util.concurrent.MoreExecutors$ListeningDecorator", "io.netty.util.concurrent.AbstractEventExecutor",
"com.google.common.util.concurrent.MoreExecutors$ScheduledListeningDecorator", "io.netty.util.concurrent.SingleThreadEventExecutor",
}; "io.netty.channel.nio.NioEventLoop",
WHITELISTED_EXECUTORS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(whitelist))); "io.netty.channel.SingleThreadEventLoop",
"com.google.common.util.concurrent.AbstractListeningExecutorService",
"com.google.common.util.concurrent.MoreExecutors$ListeningDecorator",
"com.google.common.util.concurrent.MoreExecutors$ScheduledListeningDecorator",
};
final String[] whitelistPrefixes = {"slick.util.AsyncExecutor$"}; final Set<String> executors =
WHITELISTED_EXECUTORS_PREFIXES = new HashSet<>(Config.getListSettingFromEnvironment("trace.executors", ""));
Collections.unmodifiableCollection(Arrays.asList(whitelistPrefixes)); executors.addAll(Arrays.asList(whitelist));
WHITELISTED_EXECUTORS = Collections.unmodifiableSet(executors);
final String[] whitelistPrefixes = {"slick.util.AsyncExecutor$"};
WHITELISTED_EXECUTORS_PREFIXES =
Collections.unmodifiableCollection(Arrays.asList(whitelistPrefixes));
}
} }
public AbstractExecutorInstrumentation(final String... additionalNames) { public AbstractExecutorInstrumentation(final String... additionalNames) {
@ -86,30 +103,33 @@ public abstract class AbstractExecutorInstrumentation extends Instrumenter.Defau
@Override @Override
public ElementMatcher<TypeDescription> typeMatcher() { public ElementMatcher<TypeDescription> typeMatcher() {
return not(isInterface()) final ElementMatcher.Junction<TypeDescription> matcher =
.and(safeHasSuperType(named(Executor.class.getName()))) not(isInterface()).and(safeHasSuperType(named(Executor.class.getName())));
.and( if (TRACE_ALL_EXECUTORS) {
new ElementMatcher<TypeDescription>() { return matcher;
@Override }
public boolean matches(final TypeDescription target) { return matcher.and(
boolean whitelisted = WHITELISTED_EXECUTORS.contains(target.getName()); new ElementMatcher<TypeDescription>() {
@Override
public boolean matches(final TypeDescription target) {
boolean whitelisted = WHITELISTED_EXECUTORS.contains(target.getName());
// Check for possible prefixes match only if not whitelisted already // Check for possible prefixes match only if not whitelisted already
if (!whitelisted) { if (!whitelisted) {
for (final String name : WHITELISTED_EXECUTORS_PREFIXES) { for (final String name : WHITELISTED_EXECUTORS_PREFIXES) {
if (target.getName().startsWith(name)) { if (target.getName().startsWith(name)) {
whitelisted = true; whitelisted = true;
break; break;
}
}
} }
if (!whitelisted) {
log.debug("Skipping executor instrumentation for {}", target.getName());
}
return whitelisted;
} }
}); }
if (!whitelisted) {
log.debug("Skipping executor instrumentation for {}", target.getName());
}
return whitelisted;
}
});
} }
@Override @Override

View File

@ -8,18 +8,26 @@ import io.opentracing.util.GlobalTracer
import spock.lang.Shared import spock.lang.Shared
import java.lang.reflect.InvocationTargetException import java.lang.reflect.InvocationTargetException
import java.util.concurrent.AbstractExecutorService
import java.util.concurrent.ArrayBlockingQueue import java.util.concurrent.ArrayBlockingQueue
import java.util.concurrent.Callable import java.util.concurrent.Callable
import java.util.concurrent.ExecutionException
import java.util.concurrent.ForkJoinPool import java.util.concurrent.ForkJoinPool
import java.util.concurrent.ForkJoinTask import java.util.concurrent.ForkJoinTask
import java.util.concurrent.Future import java.util.concurrent.Future
import java.util.concurrent.LinkedBlockingQueue
import java.util.concurrent.RejectedExecutionException import java.util.concurrent.RejectedExecutionException
import java.util.concurrent.ScheduledThreadPoolExecutor import java.util.concurrent.ScheduledThreadPoolExecutor
import java.util.concurrent.ThreadPoolExecutor import java.util.concurrent.ThreadPoolExecutor
import java.util.concurrent.TimeUnit import java.util.concurrent.TimeUnit
import java.util.concurrent.TimeoutException
class ExecutorInstrumentationTest extends AgentTestRunner { class ExecutorInstrumentationTest extends AgentTestRunner {
static {
System.setProperty("dd.trace.executors", "ExecutorInstrumentationTest\$CustomThreadPoolExecutor")
}
@Shared @Shared
def executeRunnable = { e, c -> e.execute((Runnable) c) } def executeRunnable = { e, c -> e.execute((Runnable) c) }
@Shared @Shared
@ -108,6 +116,15 @@ class ExecutorInstrumentationTest extends AgentTestRunner {
"invokeAll with timeout" | invokeAllTimeout | new ForkJoinPool() "invokeAll with timeout" | invokeAllTimeout | new ForkJoinPool()
"invokeAny" | invokeAny | new ForkJoinPool() "invokeAny" | invokeAny | new ForkJoinPool()
"invokeAny with timeout" | invokeAnyTimeout | new ForkJoinPool() "invokeAny with timeout" | invokeAnyTimeout | new ForkJoinPool()
// CustomThreadPoolExecutor would normally be disabled except enabled above.
"execute Runnable" | executeRunnable | new CustomThreadPoolExecutor()
"submit Runnable" | submitRunnable | new CustomThreadPoolExecutor()
"submit Callable" | submitCallable | new CustomThreadPoolExecutor()
"invokeAll" | invokeAll | new CustomThreadPoolExecutor()
"invokeAll with timeout" | invokeAllTimeout | new CustomThreadPoolExecutor()
"invokeAny" | invokeAny | new CustomThreadPoolExecutor()
"invokeAny with timeout" | invokeAnyTimeout | new CustomThreadPoolExecutor()
} }
def "#poolImpl '#name' disabled wrapping"() { def "#poolImpl '#name' disabled wrapping"() {
@ -216,4 +233,105 @@ class ExecutorInstrumentationTest extends AgentTestRunner {
"submit Runnable" | submitRunnable | new ForkJoinPool() "submit Runnable" | submitRunnable | new ForkJoinPool()
"submit Callable" | submitCallable | new ForkJoinPool() "submit Callable" | submitCallable | new ForkJoinPool()
} }
static class CustomThreadPoolExecutor extends AbstractExecutorService {
volatile running = true
def workQueue = new LinkedBlockingQueue<Runnable>(10)
def worker = new Runnable() {
void run() {
try {
while (running) {
def runnable = workQueue.take()
runnable.run()
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt()
} catch (Exception e) {
e.printStackTrace()
}
}
}
def workerThread = new Thread(worker, "ExecutorTestThread")
private CustomThreadPoolExecutor() {
workerThread.start()
}
@Override
void shutdown() {
running = false
workerThread.interrupt()
}
@Override
List<Runnable> shutdownNow() {
running = false
workerThread.interrupt()
return []
}
@Override
boolean isShutdown() {
return !running
}
@Override
boolean isTerminated() {
return workerThread.isAlive()
}
@Override
boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException {
workerThread.join(unit.toMillis(timeout))
return true
}
@Override
def <T> Future<T> submit(Callable<T> task) {
def future = newTaskFor(task)
execute(future)
return future
}
@Override
def <T> Future<T> submit(Runnable task, T result) {
def future = newTaskFor(task, result)
execute(future)
return future
}
@Override
Future<?> submit(Runnable task) {
def future = newTaskFor(task, null)
execute(future)
return future
}
@Override
def <T> List<Future<T>> invokeAll(Collection<? extends Callable<T>> tasks) throws InterruptedException {
return super.invokeAll(tasks)
}
@Override
def <T> List<Future<T>> invokeAll(Collection<? extends Callable<T>> tasks, long timeout, TimeUnit unit) throws InterruptedException {
return super.invokeAll(tasks)
}
@Override
def <T> T invokeAny(Collection<? extends Callable<T>> tasks) throws InterruptedException, ExecutionException {
return super.invokeAny(tasks)
}
@Override
def <T> T invokeAny(Collection<? extends Callable<T>> tasks, long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
return super.invokeAny(tasks)
}
@Override
void execute(Runnable command) {
workQueue.put(command)
}
}
} }

View File

@ -399,7 +399,11 @@ public class Config {
return parseMap(getSettingFromEnvironment(name, defaultValue), PREFIX + name); return parseMap(getSettingFromEnvironment(name, defaultValue), PREFIX + name);
} }
private static List<String> getListSettingFromEnvironment( /**
* Calls {@link #getSettingFromEnvironment(String, String)} and converts the result to a list by
* splitting on `,`.
*/
public static List<String> getListSettingFromEnvironment(
final String name, final String defaultValue) { final String name, final String defaultValue) {
return parseList(getSettingFromEnvironment(name, defaultValue)); return parseList(getSettingFromEnvironment(name, defaultValue));
} }