diff --git a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/AgentInstaller.java b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/AgentInstaller.java
index f3e1d62e07..f9ceb3839b 100644
--- a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/AgentInstaller.java
+++ b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/AgentInstaller.java
@@ -2,7 +2,6 @@ package datadog.trace.agent.tooling;
 
 import static datadog.trace.agent.tooling.ClassLoaderMatcher.skipClassLoader;
 import static net.bytebuddy.matcher.ElementMatchers.any;
-import static net.bytebuddy.matcher.ElementMatchers.isAnnotatedWith;
 import static net.bytebuddy.matcher.ElementMatchers.nameContains;
 import static net.bytebuddy.matcher.ElementMatchers.nameMatches;
 import static net.bytebuddy.matcher.ElementMatchers.nameStartsWith;
@@ -24,7 +23,6 @@ import net.bytebuddy.agent.builder.ResettableClassFileTransformer;
 import net.bytebuddy.description.type.TypeDescription;
 import net.bytebuddy.dynamic.DynamicType;
 import net.bytebuddy.matcher.ElementMatcher;
-import net.bytebuddy.matcher.ElementMatchers;
 import net.bytebuddy.utility.JavaModule;
 
 @Slf4j
@@ -66,10 +64,11 @@ public class AgentInstaller {
             // https://github.com/raphw/byte-buddy/issues/558
             // .with(AgentBuilder.LambdaInstrumentationStrategy.ENABLED)
             .ignore(any(), skipClassLoader())
-            // Unlikely to ever need to instrument an annotation:
-            .or(ElementMatchers.<TypeDescription>isAnnotation())
-            // Unlikely to ever need to instrument an enum:
-            .or(ElementMatchers.<TypeDescription>isEnum())
+            /**
+             * Be very careful about the types of matchers used in this section as they are called
+             * on every class load, so they must be fast. Generally speaking try to only use name
+             * matchers as they don't have to load additional info.
+             */
             .or(
                 nameStartsWith("datadog.trace.")
                     // FIXME: We should remove this once
@@ -134,7 +133,6 @@ public class AgentInstaller {
             .or(nameContains(".asm."))
             .or(nameContains("$__sisu"))
             .or(nameMatches("com\\.mchange\\.v2\\.c3p0\\..*Proxy"))
-            .or(isAnnotatedWith(named("javax.decorator.Decorator")))
             .or(matchesConfiguredExcludes());
 
     if (log.isDebugEnabled()) {
diff --git a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/Instrumenter.java b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/Instrumenter.java
index fc1e8fd1b3..9f4975f10d 100644
--- a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/Instrumenter.java
+++ b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/Instrumenter.java
@@ -2,6 +2,9 @@ package datadog.trace.agent.tooling;
 
 import static datadog.trace.agent.tooling.ByteBuddyElementMatchers.failSafe;
 import static net.bytebuddy.matcher.ElementMatchers.any;
+import static net.bytebuddy.matcher.ElementMatchers.isAnnotatedWith;
+import static net.bytebuddy.matcher.ElementMatchers.named;
+import static net.bytebuddy.matcher.ElementMatchers.not;
 
 import datadog.trace.agent.tooling.context.FieldBackedProvider;
 import datadog.trace.agent.tooling.context.InstrumentationContextProvider;
@@ -78,6 +81,9 @@ public interface Instrumenter {
                       classLoaderMatcher(),
                       "Instrumentation class loader matcher unexpected exception: "
                           + getClass().getName()))
+              // Added here instead of AgentInstaller's ignores because it's relatively
+              // expensive. https://github.com/DataDog/dd-trace-java/pull/1045
+              .and(not(isAnnotatedWith(named("javax.decorator.Decorator"))))
               .and(new MuzzleMatcher())
               .and(new PostMatchHook())
               .transform(DDTransformers.defaultTransformers());
diff --git a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/context/FieldBackedProvider.java b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/context/FieldBackedProvider.java
index 3c77fb422e..e3dce626f2 100644
--- a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/context/FieldBackedProvider.java
+++ b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/context/FieldBackedProvider.java
@@ -2,6 +2,7 @@ package datadog.trace.agent.tooling.context;
 
 import static datadog.trace.agent.tooling.ByteBuddyElementMatchers.safeHasSuperType;
 import static datadog.trace.agent.tooling.ClassLoaderMatcher.BOOTSTRAP_CLASSLOADER;
+import static net.bytebuddy.matcher.ElementMatchers.isAnnotatedWith;
 import static net.bytebuddy.matcher.ElementMatchers.isInterface;
 import static net.bytebuddy.matcher.ElementMatchers.named;
 import static net.bytebuddy.matcher.ElementMatchers.not;
@@ -354,9 +355,12 @@ public class FieldBackedProvider implements InstrumentationContextProvider {
         builder =
             builder
                 .type(
-                    safeHasSuperType(named(entry.getKey())).and(not(isInterface())),
+                    not(isInterface()).and(safeHasSuperType(named(entry.getKey()))),
                     instrumenter.classLoaderMatcher())
                 .and(safeToInjectFieldsMatcher())
+                // Added here instead of AgentInstaller's ignores because it's relatively
+                // expensive. https://github.com/DataDog/dd-trace-java/pull/1045
+                .and(not(isAnnotatedWith(named("javax.decorator.Decorator"))))
                 .transform(AgentBuilder.Transformer.NoOp.INSTANCE);
 
         /**
diff --git a/dd-java-agent/instrumentation/cdi-1.2/src/test/groovy/CDIContainerTest.groovy b/dd-java-agent/instrumentation/cdi-1.2/src/test/groovy/CDIContainerTest.groovy
index 6acb386213..f7878bd704 100644
--- a/dd-java-agent/instrumentation/cdi-1.2/src/test/groovy/CDIContainerTest.groovy
+++ b/dd-java-agent/instrumentation/cdi-1.2/src/test/groovy/CDIContainerTest.groovy
@@ -1,5 +1,4 @@
 import datadog.trace.agent.test.AgentTestRunner
-import datadog.trace.instrumentation.TestBean
 import org.jboss.weld.environment.se.Weld
 import org.jboss.weld.environment.se.WeldContainer
 import org.jboss.weld.environment.se.threading.RunnableDecorator
diff --git a/dd-java-agent/instrumentation/cdi-1.2/src/test/java/datadog/trace/instrumentation/TestBean.java b/dd-java-agent/instrumentation/cdi-1.2/src/test/java/TestBean.java
similarity index 83%
rename from dd-java-agent/instrumentation/cdi-1.2/src/test/java/datadog/trace/instrumentation/TestBean.java
rename to dd-java-agent/instrumentation/cdi-1.2/src/test/java/TestBean.java
index 3a7699aaa6..c68ad16a12 100644
--- a/dd-java-agent/instrumentation/cdi-1.2/src/test/java/datadog/trace/instrumentation/TestBean.java
+++ b/dd-java-agent/instrumentation/cdi-1.2/src/test/java/TestBean.java
@@ -1,5 +1,3 @@
-package datadog.trace.instrumentation;
-
 public class TestBean {
 
   private String someField;