Merge pull request #487 from DataDog/gary/muzzle-aws-sdk

Muzzle AWS SDK Instrumentation

dd-java-agent/instrumentation/aws-java-sdk-1.11.0/aws-java-sdk-1.11.0.gradle

@@ -1,39 +1,76 @@
-apply plugin: 'version-scan'
+// compiling against 1.11.0, but instrumentation should work against 1.10.33 with varying effects,
+// depending on the version's implementation. (i.e. DeleteOptionGroup may have less handlerCounts than
+// expected in 1.11.84. Testing against 1.11.0 instead of 1.10.33 because the RequestHandler class
+// used in testing is abstract in 1.10.33
+// keeping base test version on 1.11.0 because RequestHandler2 is abstract in 1.10.33,
+// therefore keeping base version as 1.11.0 even though the instrumentation probably
+// is able to support up to 1.10.33
+muzzle {
+  pass {
+    group = "com.amazonaws"
+    module = "aws-java-sdk-core"
+    versions = "[1.10.33, 1.11.0)"
+  }
 
-// Commented out because version scan doesn't catch the combination of tests.
+  pass {
-// HttpClientFactory is only present starting in 1.11.0
+    group = "com.amazonaws"
-// HandlerContextAware is added in 1.11.106
+    module = "aws-java-sdk-core"
-// The combination of the two allow us to filter the ranges.
+    versions = "[1.11.0, 1.11.50)"
-//
+  }
 
-//versionScan {
+  pass {
-//  group = "com.amazonaws"
+    group = "com.amazonaws"
-//  module = "aws-java-sdk-core"
+    module = "aws-java-sdk-core"
-//  versions = "[1.11.0,1.11.106)"
+    versions = "[1.11.50, 1.11.100)"
-//  verifyPresent = [
+  }
-//    "com.amazonaws.http.client.HttpClientFactory": null,
-//  ]
-//  verifyMissing = [
-//    "com.amazonaws.HandlerContextAware",
-//  ]
-//}
 
-//versionScan {
+  pass {
-//  group = "com.amazonaws"
+    group = "com.amazonaws"
-//  module = "aws-java-sdk-core"
+    module = "aws-java-sdk-core"
-//  versions = "[1.11.0,)"
+    versions = "[1.11.100, 1.11.150)"
-//  verifyPresent = [
+  }
-//    "com.amazonaws.http.client.HttpClientFactory": null,
-//  ]
-//}
 
-versionScan {
+  pass {
-  group = "com.amazonaws"
+    group = "com.amazonaws"
-  module = "aws-java-sdk-core"
+    module = "aws-java-sdk-core"
-  versions = "[,1.11.106)"
+    versions = "[1.11.150, 1.11.200)"
-  verifyMissing = [
+  }
-    "com.amazonaws.HandlerContextAware",
+
-  ]
+  pass {
+    group = "com.amazonaws"
+    module = "aws-java-sdk-core"
+    versions = "[1.11.200, 1.11.250)"
+  }
+
+  pass {
+    group = "com.amazonaws"
+    module = "aws-java-sdk-core"
+    versions = "[1.11.250, 1.11.300)"
+  }
+
+  pass {
+    group = "com.amazonaws"
+    module = "aws-java-sdk-core"
+    versions = "[1.11.300, 1.11.350)"
+  }
+
+  pass {
+    group = "com.amazonaws"
+    module = "aws-java-sdk-core"
+    versions = "[1.11.350, 1.11.400)"
+  }
+
+  pass {
+    group = "com.amazonaws"
+    module = "aws-java-sdk-core"
+    versions = "[1.11.400,)"
+  }
+
+//  fail {
+//    group = "com.amazonaws"
+//    module = "aws-java-sdk-core"
+//    versions = "[,1.10.33)"
+//  }
 }
 
 apply from: "${rootDir}/gradle/java.gradle"
@@ -41,11 +78,30 @@ apply from: "${rootDir}/gradle/java.gradle"
 apply plugin: 'org.unbroken-dome.test-sets'
 
 testSets {
+  // features used in test_1_11_106 (builder) is available since 1.11.84, but
+  // using 1.11.106 because of previous concerns with byte code differences
+  // in 1.11.106, also, the DeleteOptionGroup request generates more spans
+  // in 1.11.106 than 1.11.84
+  test_1_11_106
+
   latestDepTest {
-    dirName = 'test'
+    dirName = 'test_1_11_106'
   }
 }
 
+configurations.test_1_11_106Compile {
+  resolutionStrategy {
+    force group: 'com.amazonaws', name: 'aws-java-sdk', version: '1.11.106'
+  }
+}
+
+configurations.latestDepTestCompile {
+  resolutionStrategy {
+    force group: 'com.amazonaws', name: 'aws-java-sdk', version: '+'
+  }
+}
+
+
 dependencies {
   compileOnly group: 'com.amazonaws', name: 'aws-java-sdk-core', version: '1.11.0'
 
@@ -60,10 +116,12 @@ dependencies {
   // Include httpclient instrumentation for testing because it is a dependency for aws-sdk.
   testCompile project(':dd-java-agent:instrumentation:apache-httpclient-4')
   testCompile group: 'com.amazonaws', name: 'aws-java-sdk', version: '1.11.0'
-}
 
-configurations.latestDepTestCompile {
+  test_1_11_106Compile project(':dd-java-agent:testing')
-  resolutionStrategy {
+  test_1_11_106Compile project(':dd-java-agent:instrumentation:apache-httpclient-4')
-    force group: 'com.amazonaws', name: 'aws-java-sdk', version: '1.11.105'
+  test_1_11_106Compile group: 'com.amazonaws', name: 'aws-java-sdk', version: '1.11.106'
-  }
+
+  latestDepTestCompile project(':dd-java-agent:testing')
+  latestDepTestCompile project(':dd-java-agent:instrumentation:apache-httpclient-4')
+  latestDepTestCompile group: 'com.amazonaws', name: 'aws-java-sdk', version: '+'
 }

dd-java-agent/instrumentation/aws-java-sdk-1.11.0/src/main/java/datadog/trace/instrumentation/aws/v0/AWSClientInstrumentation.java

@@ -1,10 +1,8 @@
 package datadog.trace.instrumentation.aws.v0;
 
-import static datadog.trace.agent.tooling.ClassLoaderMatcher.classLoaderHasClasses;
 import static net.bytebuddy.matcher.ElementMatchers.declaresField;
 import static net.bytebuddy.matcher.ElementMatchers.isConstructor;
 import static net.bytebuddy.matcher.ElementMatchers.named;
-import static net.bytebuddy.matcher.ElementMatchers.not;
 
 import com.amazonaws.handlers.RequestHandler2;
 import com.google.auto.service.AutoService;
@@ -34,15 +32,6 @@ public final class AWSClientInstrumentation extends Instrumenter.Default {
         .and(declaresField(named("requestHandler2s")));
   }
 
-  @Override
-  public ElementMatcher<ClassLoader> classLoaderMatcher() {
-    return classLoaderHasClasses("com.amazonaws.http.client.HttpClientFactory")
-        .and(
-            not(
-                classLoaderHasClasses(
-                    "com.amazonaws.client.builder.AwsClientBuilder$EndpointConfiguration")));
-  }
-
   @Override
   public String[] helperClassNames() {
     return new String[] {

dd-java-agent/instrumentation/aws-java-sdk-1.11.106/aws-java-sdk-1.11.106.gradle

@@ -1,42 +0,0 @@
-apply plugin: 'version-scan'
-
-versionScan {
-  group = "com.amazonaws"
-  module = "aws-java-sdk-core"
-  versions = "[1.11.106,)"
-  verifyPresent = [
-    "com.amazonaws.HandlerContextAware": null,
-  ]
-}
-
-apply from: "${rootDir}/gradle/java.gradle"
-
-apply plugin: 'org.unbroken-dome.test-sets'
-
-testSets {
-  latestDepTest {
-    dirName = 'test'
-  }
-}
-
-dependencies {
-  compileOnly group: 'com.amazonaws', name: 'aws-java-sdk-core', version: '1.11.106'
-
-  compile project(':dd-java-agent:agent-tooling')
-
-  compile deps.bytebuddy
-  compile deps.opentracing
-  annotationProcessor deps.autoservice
-  implementation deps.autoservice
-
-  testCompile project(':dd-java-agent:testing')
-  // Include httpclient instrumentation for testing because it is a dependency for aws-sdk.
-  testCompile project(':dd-java-agent:instrumentation:apache-httpclient-4')
-  testCompile group: 'com.amazonaws', name: 'aws-java-sdk', version: '1.11.106'
-}
-
-configurations.latestDepTestCompile {
-  resolutionStrategy {
-    force group: 'com.amazonaws', name: 'aws-java-sdk', version: '+'
-  }
-}

dd-java-agent/instrumentation/aws-java-sdk-1.11.106/src/main/java/datadog/trace/instrumentation/aws/v106/AWSClientInstrumentation.java

@@ -1,77 +0,0 @@
-package datadog.trace.instrumentation.aws.v106;
-
-import static datadog.trace.agent.tooling.ClassLoaderMatcher.classLoaderHasClasses;
-import static net.bytebuddy.matcher.ElementMatchers.declaresField;
-import static net.bytebuddy.matcher.ElementMatchers.isAbstract;
-import static net.bytebuddy.matcher.ElementMatchers.isConstructor;
-import static net.bytebuddy.matcher.ElementMatchers.named;
-
-import com.amazonaws.handlers.RequestHandler2;
-import com.google.auto.service.AutoService;
-import datadog.trace.agent.tooling.Instrumenter;
-import io.opentracing.util.GlobalTracer;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import net.bytebuddy.asm.Advice;
-import net.bytebuddy.description.type.TypeDescription;
-import net.bytebuddy.matcher.ElementMatcher;
-
-/**
- * The interface for com.amazonaws.Request changed in 106. The method addHandlerContext which is
- * used in TracingRequestHandler moved to a parent interface, which makes it source, but not
- * bytecode compatible. The instrumentation is the same, but the compiled output is different.
- */
-@AutoService(Instrumenter.class)
-public final class AWSClientInstrumentation extends Instrumenter.Default {
-
-  public AWSClientInstrumentation() {
-    super("aws-sdk");
-  }
-
-  @Override
-  public ElementMatcher<TypeDescription> typeMatcher() {
-    return isAbstract()
-        .and(
-            named("com.amazonaws.AmazonWebServiceClient")
-                .and(declaresField(named("requestHandler2s"))));
-  }
-
-  @Override
-  public ElementMatcher<ClassLoader> classLoaderMatcher() {
-    return classLoaderHasClasses("com.amazonaws.HandlerContextAware");
-  }
-
-  @Override
-  public String[] helperClassNames() {
-    return new String[] {
-      "datadog.trace.instrumentation.aws.v106.TracingRequestHandler",
-      "datadog.trace.instrumentation.aws.v106.SpanDecorator"
-    };
-  }
-
-  @Override
-  public Map<ElementMatcher, String> transformers() {
-    final Map<ElementMatcher, String> transformers = new HashMap<>();
-    transformers.put(isConstructor(), AWSClientAdvice.class.getName());
-    return transformers;
-  }
-
-  public static class AWSClientAdvice {
-    // Since we're instrumenting the constructor, we can't add onThrowable.
-    @Advice.OnMethodExit(suppress = Throwable.class)
-    public static void addHandler(
-        @Advice.FieldValue("requestHandler2s") final List<RequestHandler2> handlers) {
-      boolean hasDDHandler = false;
-      for (final RequestHandler2 handler : handlers) {
-        if (handler instanceof TracingRequestHandler) {
-          hasDDHandler = true;
-          break;
-        }
-      }
-      if (!hasDDHandler) {
-        handlers.add(new TracingRequestHandler(GlobalTracer.get()));
-      }
-    }
-  }
-}

dd-java-agent/instrumentation/aws-java-sdk-1.11.106/src/main/java/datadog/trace/instrumentation/aws/v106/SpanDecorator.java

@@ -1,81 +0,0 @@
-/*
- * Copyright 2017-2018 The OpenTracing Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
- * in compliance with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-package datadog.trace.instrumentation.aws.v106;
-
-import static io.opentracing.log.Fields.ERROR_OBJECT;
-
-import com.amazonaws.AmazonWebServiceResponse;
-import com.amazonaws.Request;
-import com.amazonaws.Response;
-import datadog.trace.api.DDSpanTypes;
-import datadog.trace.api.DDTags;
-import io.opentracing.Span;
-import io.opentracing.tag.Tags;
-import java.util.Collections;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-class SpanDecorator {
-  static final String COMPONENT_NAME = "java-aws-sdk";
-
-  private static final Map<String, String> SERVICE_NAMES = new ConcurrentHashMap<>();
-  private static final Map<Class, String> OPERATION_NAMES = new ConcurrentHashMap<>();
-
-  static void onRequest(final Request<?> request, final Span span) {
-    Tags.COMPONENT.set(span, COMPONENT_NAME);
-    Tags.HTTP_METHOD.set(span, request.getHttpMethod().name());
-    Tags.HTTP_URL.set(span, request.getEndpoint().toString());
-
-    final String awsServiceName = request.getServiceName();
-    final Class<?> awsOperation = request.getOriginalRequest().getClass();
-
-    span.setTag("aws.agent", COMPONENT_NAME);
-    span.setTag("aws.service", awsServiceName);
-    span.setTag("aws.operation", awsOperation.getSimpleName());
-    span.setTag("aws.endpoint", request.getEndpoint().toString());
-
-    span.setTag(DDTags.SERVICE_NAME, COMPONENT_NAME);
-    span.setTag(
-        DDTags.RESOURCE_NAME,
-        remapServiceName(awsServiceName) + "." + remapOperationName(awsOperation));
-    span.setTag(DDTags.SPAN_TYPE, DDSpanTypes.HTTP_CLIENT);
-  }
-
-  static void onResponse(final Response response, final Span span) {
-    Tags.HTTP_STATUS.set(span, response.getHttpResponse().getStatusCode());
-    if (response.getAwsResponse() instanceof AmazonWebServiceResponse) {
-      final AmazonWebServiceResponse awsResp = (AmazonWebServiceResponse) response.getAwsResponse();
-      span.setTag("aws.requestId", awsResp.getRequestId());
-    }
-  }
-
-  static void onError(final Throwable throwable, final Span span) {
-    Tags.ERROR.set(span, Boolean.TRUE);
-    span.log(Collections.singletonMap(ERROR_OBJECT, throwable));
-  }
-
-  private static String remapServiceName(final String serviceName) {
-    if (!SERVICE_NAMES.containsKey(serviceName)) {
-      SERVICE_NAMES.put(serviceName, serviceName.replace("Amazon", "").trim());
-    }
-    return SERVICE_NAMES.get(serviceName);
-  }
-
-  private static String remapOperationName(final Class<?> awsOperation) {
-    if (!OPERATION_NAMES.containsKey(awsOperation)) {
-      OPERATION_NAMES.put(awsOperation, awsOperation.getSimpleName().replace("Request", ""));
-    }
-    return OPERATION_NAMES.get(awsOperation);
-  }
-}

dd-java-agent/instrumentation/aws-java-sdk-1.11.106/src/main/java/datadog/trace/instrumentation/aws/v106/TracingRequestHandler.java

@@ -1,92 +0,0 @@
-/*
- * Copyright 2017-2018 The OpenTracing Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
- * in compliance with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-package datadog.trace.instrumentation.aws.v106;
-
-import com.amazonaws.AmazonWebServiceRequest;
-import com.amazonaws.Request;
-import com.amazonaws.Response;
-import com.amazonaws.handlers.HandlerContextKey;
-import com.amazonaws.handlers.RequestHandler2;
-import io.opentracing.Span;
-import io.opentracing.SpanContext;
-import io.opentracing.Tracer;
-import io.opentracing.propagation.Format;
-import io.opentracing.propagation.TextMapInjectAdapter;
-import io.opentracing.tag.Tags;
-
-/** Tracing Request Handler */
-public class TracingRequestHandler extends RequestHandler2 {
-
-  private final HandlerContextKey<Span> contextKey = new HandlerContextKey<>("span");
-  private final SpanContext parentContext; // for Async Client
-  private final Tracer tracer;
-
-  public TracingRequestHandler(final Tracer tracer) {
-    this.parentContext = null;
-    this.tracer = tracer;
-  }
-
-  /**
-   * In case of Async Client: beforeRequest runs in separate thread therefore we need to inject
-   * parent context to build chain
-   *
-   * @param parentContext parent context
-   */
-  public TracingRequestHandler(final SpanContext parentContext, final Tracer tracer) {
-    this.parentContext = parentContext;
-    this.tracer = tracer;
-  }
-
-  @Override
-  public AmazonWebServiceRequest beforeMarshalling(final AmazonWebServiceRequest request) {
-    return request;
-  }
-
-  /** {@inheritDoc} */
-  @Override
-  public void beforeRequest(final Request<?> request) {
-    final Tracer.SpanBuilder spanBuilder =
-        tracer.buildSpan("aws.command").withTag(Tags.SPAN_KIND.getKey(), Tags.SPAN_KIND_CLIENT);
-
-    if (parentContext != null) {
-      spanBuilder.asChildOf(parentContext);
-    }
-
-    final Span span = spanBuilder.start();
-    SpanDecorator.onRequest(request, span);
-
-    tracer.inject(
-        span.context(),
-        Format.Builtin.HTTP_HEADERS,
-        new TextMapInjectAdapter(request.getHeaders()));
-
-    request.addHandlerContext(contextKey, span);
-  }
-
-  /** {@inheritDoc} */
-  @Override
-  public void afterResponse(final Request<?> request, final Response<?> response) {
-    final Span span = request.getHandlerContext(contextKey);
-    SpanDecorator.onResponse(response, span);
-    span.finish();
-  }
-
-  /** {@inheritDoc} */
-  @Override
-  public void afterError(final Request<?> request, final Response<?> response, final Exception e) {
-    final Span span = request.getHandlerContext(contextKey);
-    SpanDecorator.onError(e, span);
-    span.finish();
-  }
-}

settings.gradle

@@ -13,7 +13,6 @@ include ':dd-trace-api'
 include ':dd-java-agent:instrumentation:akka-http-10.0'
 include ':dd-java-agent:instrumentation:apache-httpclient-4'
 include ':dd-java-agent:instrumentation:aws-java-sdk-1.11.0'
-include ':dd-java-agent:instrumentation:aws-java-sdk-1.11.106'
 include ':dd-java-agent:instrumentation:couchbase-2.0'
 include ':dd-java-agent:instrumentation:datastax-cassandra-2.3'
 include ':dd-java-agent:instrumentation:dropwizard'