Merge pull request #479 from DataDog/tyler/broken-jax-rs-client

Handle connection failures for Jersey and Resteasy

dd-java-agent/instrumentation/instrumentation.gradle

@@ -9,34 +9,31 @@ plugins {
 }
 apply from: "${rootDir}/gradle/java.gradle"
 
-// add all subprojects under 'instrumentation' to the agent's dependencies
 Project instr_project = project
 subprojects { subProj ->
-  if (subProj.getParent() == instr_project) {
+  apply plugin: "net.bytebuddy.byte-buddy"
-    apply plugin: "net.bytebuddy.byte-buddy"
+  apply plugin: 'muzzle'
-    apply plugin: 'muzzle'
 
+  subProj.byteBuddy {
+    transformation {
+      // Applying NoOp optimizes build by applying bytebuddy plugin to only compileJava task
+      tasks = ['compileJava', 'compileScala']
+      plugin = 'datadog.trace.agent.tooling.muzzle.MuzzleGradlePlugin$NoOp'
+    }
+  }
+
+  subProj.afterEvaluate {
     subProj.byteBuddy {
       transformation {
-        // Applying NoOp optimizes build by applying bytebuddy plugin to only compileJava task
         tasks = ['compileJava', 'compileScala']
-        plugin = 'datadog.trace.agent.tooling.muzzle.MuzzleGradlePlugin$NoOp'
+        plugin = 'datadog.trace.agent.tooling.muzzle.MuzzleGradlePlugin'
+        classPath = project(':dd-java-agent:agent-tooling').configurations.instrumentationMuzzle + subProj.configurations.compile + subProj.sourceSets.main.output
       }
     }
+  }
 
-    subProj.afterEvaluate {
+  instr_project.dependencies {
-      subProj.byteBuddy {
+    compile(project(subProj.getPath()))
-        transformation {
-          tasks = ['compileJava', 'compileScala']
-          plugin = 'datadog.trace.agent.tooling.muzzle.MuzzleGradlePlugin'
-          classPath = project(':dd-java-agent:agent-tooling').configurations.instrumentationMuzzle + subProj.configurations.compile + subProj.sourceSets.main.output
-        }
-      }
-    }
-
-    instr_project.dependencies {
-      compile(project(subProj.getPath()))
-    }
   }
 }
 

dd-java-agent/instrumentation/java-concurrent/akka-testing/akka-testing.gradle

@@ -1,14 +1,14 @@
 apply from: "${rootDir}/gradle/java.gradle"
-apply plugin: 'scala'
+apply from: "${rootDir}/gradle/test-with-scala.gradle"
 
 excludedClassesConverage += ['*']
 
 dependencies {
-  compile project(':dd-trace-api')
+  testCompile project(':dd-trace-api')
-  compile project(':dd-trace-ot')
+  testCompile project(':dd-trace-ot')
-  compile group: 'org.scala-lang', name: 'scala-library', version: '2.11.12'
+  testCompile group: 'org.scala-lang', name: 'scala-library', version: '2.11.12'
-  compile group: 'com.typesafe.akka', name: 'akka-actor_2.11', version: '2.3.16'
+  testCompile group: 'com.typesafe.akka', name: 'akka-actor_2.11', version: '2.3.16'
-  compile group: 'com.typesafe.akka', name: 'akka-testkit_2.11', version: '2.3.16'
+  testCompile group: 'com.typesafe.akka', name: 'akka-testkit_2.11', version: '2.3.16'
 
   testCompile project(':dd-java-agent:testing')
   testCompile project(':dd-java-agent:instrumentation:java-concurrent')

dd-java-agent/instrumentation/java-concurrent/scala-testing/scala-testing.gradle

@@ -2,9 +2,9 @@ apply from: "${rootDir}/gradle/java.gradle"
 apply from: "${rootDir}/gradle/test-with-scala.gradle"
 
 dependencies {
-  compile project(':dd-trace-api')
+  testCompile project(':dd-trace-api')
-  compile project(':dd-trace-ot')
+  testCompile project(':dd-trace-ot')
-  compile group: 'org.scala-lang', name: 'scala-library', version: '2.11.12'
+  testCompile group: 'org.scala-lang', name: 'scala-library', version: '2.11.12'
 
   testCompile project(':dd-java-agent:testing')
   testCompile project(':dd-java-agent:instrumentation:java-concurrent')

dd-java-agent/instrumentation/jax-rs-client/connection-error-handling-jersey/connection-error-handling-jersey.gradle

@@ -0,0 +1,22 @@
+muzzle {
+  pass {
+    group = "org.glassfish.jersey.core"
+    module = "jersey-client"
+    versions = "[2.0,)"
+  }
+}
+
+apply from: "${rootDir}/gradle/java.gradle"
+
+dependencies {
+  compileOnly group: 'org.glassfish.jersey.core', name: 'jersey-client', version: '2.0'
+
+  compile deps.bytebuddy
+  compile deps.opentracing
+  annotationProcessor deps.autoservice
+  implementation deps.autoservice
+
+  compile project(':dd-java-agent:agent-tooling')
+
+  compileOnly project(':dd-java-agent:instrumentation:jax-rs-client')
+}

dd-java-agent/instrumentation/jax-rs-client/connection-error-handling-jersey/src/main/java/datadog/trace/instrumentation/connection_error/jersey/JerseyClientConnectionErrorInstrumentation.java

@@ -0,0 +1,146 @@
+package datadog.trace.instrumentation.connection_error.jersey;
+
+import static net.bytebuddy.matcher.ElementMatchers.isMethod;
+import static net.bytebuddy.matcher.ElementMatchers.isPublic;
+import static net.bytebuddy.matcher.ElementMatchers.named;
+import static net.bytebuddy.matcher.ElementMatchers.returns;
+
+import com.google.auto.service.AutoService;
+import datadog.trace.agent.tooling.Instrumenter;
+import datadog.trace.instrumentation.jaxrs.ClientTracingFilter;
+import io.opentracing.Span;
+import io.opentracing.log.Fields;
+import io.opentracing.tag.Tags;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import net.bytebuddy.asm.Advice;
+import net.bytebuddy.description.type.TypeDescription;
+import net.bytebuddy.matcher.ElementMatcher;
+import org.glassfish.jersey.client.ClientRequest;
+
+/**
+ * JAX-RS Client API doesn't define a good point where we can handle connection failures, so we must
+ * handle these errors at the implementation level.
+ */
+@AutoService(Instrumenter.class)
+public final class JerseyClientConnectionErrorInstrumentation extends Instrumenter.Default {
+
+  public JerseyClientConnectionErrorInstrumentation() {
+    super("jax-rs", "jaxrs", "jax-rs-client");
+  }
+
+  @Override
+  public ElementMatcher<TypeDescription> typeMatcher() {
+    return named("org.glassfish.jersey.client.JerseyInvocation");
+  }
+
+  @Override
+  public String[] helperClassNames() {
+    return new String[] {getClass().getName() + "$WrappedFuture"};
+  }
+
+  @Override
+  public Map<ElementMatcher, String> transformers() {
+    final Map<ElementMatcher, String> transformers = new HashMap<>();
+    transformers.put(isMethod().and(isPublic()).and(named("invoke")), InvokeAdvice.class.getName());
+    transformers.put(
+        isMethod().and(isPublic()).and(named("submit")).and(returns(Future.class)),
+        SubmitAdvice.class.getName());
+    return transformers;
+  }
+
+  public static class InvokeAdvice {
+
+    @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class)
+    public static void handleError(
+        @Advice.FieldValue("requestContext") final ClientRequest context,
+        @Advice.Thrown final Throwable throwable) {
+      if (throwable != null) {
+
+        final Object prop = context.getProperty(ClientTracingFilter.SPAN_PROPERTY_NAME);
+        if (prop instanceof Span) {
+          final Span span = (Span) prop;
+          Tags.ERROR.set(span, true);
+          span.log(Collections.singletonMap(Fields.ERROR_OBJECT, throwable));
+          span.finish();
+        }
+      }
+    }
+  }
+
+  public static class SubmitAdvice {
+
+    @Advice.OnMethodExit(suppress = Throwable.class)
+    public static void handleError(
+        @Advice.FieldValue("requestContext") final ClientRequest context,
+        @Advice.Return(readOnly = false) Future future) {
+      if (!(future instanceof WrappedFuture)) {
+        future = new WrappedFuture(future, context);
+      }
+    }
+  }
+
+  public static class WrappedFuture<T> implements Future<T> {
+
+    private final Future<T> wrapped;
+    private final ClientRequest context;
+
+    public WrappedFuture(final Future<T> wrapped, final ClientRequest context) {
+      this.wrapped = wrapped;
+      this.context = context;
+    }
+
+    @Override
+    public boolean cancel(final boolean mayInterruptIfRunning) {
+      return wrapped.cancel(mayInterruptIfRunning);
+    }
+
+    @Override
+    public boolean isCancelled() {
+      return wrapped.isCancelled();
+    }
+
+    @Override
+    public boolean isDone() {
+      return wrapped.isDone();
+    }
+
+    @Override
+    public T get() throws InterruptedException, ExecutionException {
+      try {
+        return wrapped.get();
+      } catch (final ExecutionException e) {
+        final Object prop = context.getProperty(ClientTracingFilter.SPAN_PROPERTY_NAME);
+        if (prop instanceof Span) {
+          final Span span = (Span) prop;
+          Tags.ERROR.set(span, true);
+          span.log(Collections.singletonMap(Fields.ERROR_OBJECT, e.getCause()));
+          span.finish();
+        }
+        throw e;
+      }
+    }
+
+    @Override
+    public T get(final long timeout, final TimeUnit unit)
+        throws InterruptedException, ExecutionException, TimeoutException {
+      try {
+        return wrapped.get(timeout, unit);
+      } catch (final ExecutionException e) {
+        final Object prop = context.getProperty(ClientTracingFilter.SPAN_PROPERTY_NAME);
+        if (prop instanceof Span) {
+          final Span span = (Span) prop;
+          Tags.ERROR.set(span, true);
+          span.log(Collections.singletonMap(Fields.ERROR_OBJECT, e.getCause()));
+          span.finish();
+        }
+        throw e;
+      }
+    }
+  }
+}

dd-java-agent/instrumentation/jax-rs-client/connection-error-handling-resteasy/connection-error-handling-resteasy.gradle

@@ -0,0 +1,22 @@
+muzzle {
+  pass {
+    group = "org.jboss.resteasy"
+    module = "resteasy-client"
+    versions = "[2.0,)"
+  }
+}
+
+apply from: "${rootDir}/gradle/java.gradle"
+
+dependencies {
+  compileOnly group: 'org.jboss.resteasy', name: 'resteasy-client', version: '3.0.0.Final'
+
+  compile deps.bytebuddy
+  compile deps.opentracing
+  annotationProcessor deps.autoservice
+  implementation deps.autoservice
+
+  compile project(':dd-java-agent:agent-tooling')
+
+  compileOnly project(':dd-java-agent:instrumentation:jax-rs-client')
+}

dd-java-agent/instrumentation/jax-rs-client/connection-error-handling-resteasy/src/main/java/datadog/trace/instrumentation/connection_error/resteasy/ResteasyClientConnectionErrorInstrumentation.java

@@ -0,0 +1,146 @@
+package datadog.trace.instrumentation.connection_error.resteasy;
+
+import static io.opentracing.log.Fields.ERROR_OBJECT;
+import static net.bytebuddy.matcher.ElementMatchers.isMethod;
+import static net.bytebuddy.matcher.ElementMatchers.isPublic;
+import static net.bytebuddy.matcher.ElementMatchers.named;
+import static net.bytebuddy.matcher.ElementMatchers.returns;
+
+import com.google.auto.service.AutoService;
+import datadog.trace.agent.tooling.Instrumenter;
+import datadog.trace.instrumentation.jaxrs.ClientTracingFilter;
+import io.opentracing.Span;
+import io.opentracing.log.Fields;
+import io.opentracing.tag.Tags;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import net.bytebuddy.asm.Advice;
+import net.bytebuddy.description.type.TypeDescription;
+import net.bytebuddy.matcher.ElementMatcher;
+import org.jboss.resteasy.client.jaxrs.internal.ClientConfiguration;
+
+/**
+ * JAX-RS Client API doesn't define a good point where we can handle connection failures, so we must
+ * handle these errors at the implementation level.
+ */
+@AutoService(Instrumenter.class)
+public final class ResteasyClientConnectionErrorInstrumentation extends Instrumenter.Default {
+
+  public ResteasyClientConnectionErrorInstrumentation() {
+    super("jax-rs", "jaxrs", "jax-rs-client");
+  }
+
+  @Override
+  public ElementMatcher<TypeDescription> typeMatcher() {
+    return named("org.jboss.resteasy.client.jaxrs.internal.ClientInvocation");
+  }
+
+  @Override
+  public String[] helperClassNames() {
+    return new String[] {getClass().getName() + "$WrappedFuture"};
+  }
+
+  @Override
+  public Map<ElementMatcher, String> transformers() {
+    final Map<ElementMatcher, String> transformers = new HashMap<>();
+    transformers.put(isMethod().and(isPublic()).and(named("invoke")), InvokeAdvice.class.getName());
+    transformers.put(
+        isMethod().and(isPublic()).and(named("submit")).and(returns(Future.class)),
+        SubmitAdvice.class.getName());
+    return transformers;
+  }
+
+  public static class InvokeAdvice {
+
+    @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class)
+    public static void handleError(
+        @Advice.FieldValue("configuration") final ClientConfiguration context,
+        @Advice.Thrown final Throwable throwable) {
+      if (throwable != null) {
+        final Object prop = context.getProperty(ClientTracingFilter.SPAN_PROPERTY_NAME);
+        if (prop instanceof Span) {
+          final Span span = (Span) prop;
+          Tags.ERROR.set(span, true);
+          span.log(Collections.singletonMap(ERROR_OBJECT, throwable));
+          span.finish();
+        }
+      }
+    }
+  }
+
+  public static class SubmitAdvice {
+
+    @Advice.OnMethodExit(suppress = Throwable.class)
+    public static void handleError(
+        @Advice.FieldValue("configuration") final ClientConfiguration context,
+        @Advice.Return(readOnly = false) Future future) {
+      if (!(future instanceof WrappedFuture)) {
+        future = new WrappedFuture(future, context);
+      }
+    }
+  }
+
+  public static class WrappedFuture<T> implements Future<T> {
+
+    private final Future<T> wrapped;
+    private final ClientConfiguration context;
+
+    public WrappedFuture(final Future<T> wrapped, final ClientConfiguration context) {
+      this.wrapped = wrapped;
+      this.context = context;
+    }
+
+    @Override
+    public boolean cancel(final boolean mayInterruptIfRunning) {
+      return wrapped.cancel(mayInterruptIfRunning);
+    }
+
+    @Override
+    public boolean isCancelled() {
+      return wrapped.isCancelled();
+    }
+
+    @Override
+    public boolean isDone() {
+      return wrapped.isDone();
+    }
+
+    @Override
+    public T get() throws InterruptedException, ExecutionException {
+      try {
+        return wrapped.get();
+      } catch (final ExecutionException e) {
+        final Object prop = context.getProperty(ClientTracingFilter.SPAN_PROPERTY_NAME);
+        if (prop instanceof Span) {
+          final Span span = (Span) prop;
+          Tags.ERROR.set(span, true);
+          span.log(Collections.singletonMap(Fields.ERROR_OBJECT, e.getCause()));
+          span.finish();
+        }
+        throw e;
+      }
+    }
+
+    @Override
+    public T get(final long timeout, final TimeUnit unit)
+        throws InterruptedException, ExecutionException, TimeoutException {
+      try {
+        return wrapped.get(timeout, unit);
+      } catch (final ExecutionException e) {
+        final Object prop = context.getProperty(ClientTracingFilter.SPAN_PROPERTY_NAME);
+        if (prop instanceof Span) {
+          final Span span = (Span) prop;
+          Tags.ERROR.set(span, true);
+          span.log(Collections.singletonMap(Fields.ERROR_OBJECT, e.getCause()));
+          span.finish();
+        }
+        throw e;
+      }
+    }
+  }
+}

dd-java-agent/instrumentation/jax-rs-client/jax-rs-client.gradle

@@ -8,11 +8,18 @@ muzzle {
 
 apply from: "${rootDir}/gradle/java.gradle"
 
+apply plugin: 'org.unbroken-dome.test-sets'
+
+testSets {
+  latestDepTest {
+    dirName = 'test'
+  }
+}
+
 dependencies {
   compileOnly group: 'javax.ws.rs', name: 'javax.ws.rs-api', version: '2.0.1'
   compileOnly group: 'javax.annotation', name: 'javax.annotation-api', version: '1.2'
 
-
   compile deps.bytebuddy
   compile deps.opentracing
   annotationProcessor deps.autoservice
@@ -22,15 +29,21 @@ dependencies {
 
   testCompile project(':dd-java-agent:testing')
 
+  testCompile project(':dd-java-agent:instrumentation:jax-rs-client:connection-error-handling-jersey')
+  testCompile project(':dd-java-agent:instrumentation:jax-rs-client:connection-error-handling-resteasy')
+
   testCompile group: 'javax.ws.rs', name: 'javax.ws.rs-api', version: '2.0.1'
 
-  testCompile group: 'org.glassfish.jersey.core', name: 'jersey-client', version: '2.25.1'
+  testCompile group: 'org.glassfish.jersey.core', name: 'jersey-client', version: '2.0'
-  testCompile group: 'org.apache.cxf', name: 'cxf-rt-rs-client', version: '3.1.16'
+  testCompile group: 'org.jboss.resteasy', name: 'resteasy-client', version: '3.0.0.Final'
-  testCompile group: 'org.jboss.resteasy', name: 'resteasy-client', version: '3.0.26.Final'
+  testCompile group: 'org.apache.cxf', name: 'cxf-rt-rs-client', version: '3.1.0'
-
+  // Doesn't work with CXF 3.0.x because their context is wrong:
-//  testCompile group: 'com.sun.jersey', name: 'jersey-core', version: '1.19.4'
+  // https://github.com/apache/cxf/commit/335c7bad2436f08d6d54180212df5a52157c9f21
-//  testCompile group: 'com.sun.jersey', name: 'jersey-servlet', version: '1.19.4'
-//  testCompile group: 'io.dropwizard', name: 'dropwizard-testing', version: '0.7.1'
 
   testCompile group: 'javax.xml.bind', name: 'jaxb-api', version: '2.2.3'
+
+  latestDepTestCompile group: 'org.glassfish.jersey.inject', name: 'jersey-hk2', version: '2.27'
+  latestDepTestCompile group: 'org.glassfish.jersey.core', name: 'jersey-client', version: '2.27'
+  latestDepTestCompile group: 'org.apache.cxf', name: 'cxf-rt-rs-client', version: '3.2.6'
+  latestDepTestCompile group: 'org.jboss.resteasy', name: 'resteasy-client', version: '3.0.26.Final'
 }

dd-java-agent/instrumentation/jax-rs-client/src/main/java/datadog/trace/instrumentation/jaxrs/ClientTracingFilter.java

@@ -6,7 +6,6 @@ import io.opentracing.Span;
 import io.opentracing.propagation.Format;
 import io.opentracing.tag.Tags;
 import io.opentracing.util.GlobalTracer;
-import java.io.IOException;
 import javax.annotation.Priority;
 import javax.ws.rs.Priorities;
 import javax.ws.rs.client.ClientRequestContext;
@@ -18,10 +17,10 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 @Priority(Priorities.HEADER_DECORATOR)
 public class ClientTracingFilter implements ClientRequestFilter, ClientResponseFilter {
-  private static final String PROPERTY_NAME = ClientTracingFilter.class.getName() + ".span";
+  public static final String SPAN_PROPERTY_NAME = "datadog.trace.jax-rs-client.span";
 
   @Override
-  public void filter(final ClientRequestContext requestContext) throws IOException {
+  public void filter(final ClientRequestContext requestContext) {
 
     final Span span =
         GlobalTracer.get()
@@ -41,14 +40,14 @@ public class ClientTracingFilter implements ClientRequestFilter, ClientResponseF
             span.context(),
             Format.Builtin.HTTP_HEADERS,
             new InjectAdapter(requestContext.getHeaders()));
-    requestContext.setProperty(PROPERTY_NAME, span);
+
+    requestContext.setProperty(SPAN_PROPERTY_NAME, span);
   }
 
   @Override
   public void filter(
-      final ClientRequestContext requestContext, final ClientResponseContext responseContext)
+      final ClientRequestContext requestContext, final ClientResponseContext responseContext) {
-      throws IOException {
+    final Object spanObj = requestContext.getProperty(SPAN_PROPERTY_NAME);
-    final Object spanObj = requestContext.getProperty(PROPERTY_NAME);
     if (spanObj instanceof Span) {
       final Span span = (Span) spanObj;
       Tags.HTTP_STATUS.set(span, responseContext.getStatus());

dd-java-agent/instrumentation/jax-rs-client/src/test/groovy/JaxRsClientTest.groovy

@@ -1,4 +1,5 @@
 import datadog.trace.agent.test.AgentTestRunner
+import datadog.trace.agent.test.TestUtils
 import datadog.trace.api.DDSpanTypes
 import datadog.trace.api.DDTags
 import io.opentracing.tag.Tags
@@ -8,17 +9,23 @@ import org.jboss.resteasy.client.jaxrs.ResteasyClientBuilder
 import spock.lang.AutoCleanup
 import spock.lang.Shared
 
+import javax.ws.rs.ProcessingException
 import javax.ws.rs.client.AsyncInvoker
 import javax.ws.rs.client.Client
 import javax.ws.rs.client.Invocation
 import javax.ws.rs.client.WebTarget
 import javax.ws.rs.core.MediaType
 import javax.ws.rs.core.Response
+import java.util.concurrent.ExecutionException
 
+import static datadog.trace.agent.test.asserts.ListWriterAssert.assertTraces
 import static datadog.trace.agent.test.server.http.TestHttpServer.httpServer
 
 class JaxRsClientTest extends AgentTestRunner {
 
+  @Shared
+  def emptyPort = TestUtils.randomOpenPort()
+
   @AutoCleanup
   @Shared
   def server = httpServer {
@@ -32,7 +39,7 @@ class JaxRsClientTest extends AgentTestRunner {
   def "#lib request creates spans and sends headers"() {
     setup:
     Client client = builder.build()
-    WebTarget service = client.target("http://localhost:$server.address.port/ping")
+    WebTarget service = client.target("$server.address/ping")
     Response response
     if (async) {
       AsyncInvoker request = service.request(MediaType.TEXT_PLAIN).async()
@@ -45,35 +52,31 @@ class JaxRsClientTest extends AgentTestRunner {
     expect:
     response.readEntity(String) == "pong"
 
-    TEST_WRITER.size() == 1
+    assertTraces(TEST_WRITER, 1) {
+      trace(0, 1) {
+        span(0) {
+          serviceName "unnamed-java-app"
+          resourceName "GET /ping"
+          operationName "jax-rs.client.call"
+          spanType "http"
+          parent()
+          errored false
+          tags {
 
-    def trace = TEST_WRITER.firstTrace()
+            "$Tags.COMPONENT.key" "jax-rs.client"
-    trace.size() == 1
+            "$Tags.SPAN_KIND.key" Tags.SPAN_KIND_CLIENT
+            "$Tags.HTTP_METHOD.key" "GET"
+            "$Tags.HTTP_STATUS.key" 200
+            "$Tags.HTTP_URL.key" "$server.address/ping"
+            "$DDTags.SPAN_TYPE" DDSpanTypes.HTTP_CLIENT
+            defaultTags()
+          }
+        }
+      }
+    }
 
-    and:
+    server.lastRequest.headers.get("x-datadog-trace-id") == TEST_WRITER[0][0].traceId
-    def span = trace[0]
+    server.lastRequest.headers.get("x-datadog-parent-id") == TEST_WRITER[0][0].spanId
-
-    span.context().operationName == "jax-rs.client.call"
-    span.serviceName == "unnamed-java-app"
-    span.resourceName == "GET /ping"
-    span.type == "http"
-    !span.context().getErrorFlag()
-    span.context().parentId == "0"
-
-
-    def tags = span.context().tags
-    tags[Tags.COMPONENT.key] == "jax-rs.client"
-    tags[Tags.SPAN_KIND.key] == Tags.SPAN_KIND_CLIENT
-    tags[Tags.HTTP_METHOD.key] == "GET"
-    tags[Tags.HTTP_STATUS.key] == 200
-    tags[Tags.HTTP_URL.key] == "http://localhost:$server.address.port/ping"
-    tags[DDTags.SPAN_TYPE] == DDSpanTypes.HTTP_CLIENT
-    tags[DDTags.THREAD_NAME] != null
-    tags[DDTags.THREAD_ID] != null
-    tags.size() == 8
-
-    server.lastRequest.headers.get("x-datadog-trace-id") == "$span.traceId"
-    server.lastRequest.headers.get("x-datadog-parent-id") == "$span.spanId"
 
     where:
     builder                     | async | lib
@@ -84,4 +87,50 @@ class JaxRsClientTest extends AgentTestRunner {
     new ClientBuilderImpl()     | true  | "cxf async"
     new ResteasyClientBuilder() | true  | "resteasy async"
   }
+
+  def "#lib connection failure creates errored span"() {
+    when:
+    Client client = builder.build()
+    WebTarget service = client.target("http://localhost:$emptyPort/ping")
+    if (async) {
+      AsyncInvoker request = service.request(MediaType.TEXT_PLAIN).async()
+      request.get().get()
+    } else {
+      Invocation.Builder request = service.request(MediaType.TEXT_PLAIN)
+      request.get()
+    }
+
+    then:
+    thrown async ? ExecutionException : ProcessingException
+
+    assertTraces(TEST_WRITER, 1) {
+      trace(0, 1) {
+        span(0) {
+          serviceName "unnamed-java-app"
+          resourceName "GET /ping"
+          operationName "jax-rs.client.call"
+          spanType "http"
+          parent()
+          errored true
+          tags {
+            "$Tags.COMPONENT.key" "jax-rs.client"
+            "$Tags.SPAN_KIND.key" Tags.SPAN_KIND_CLIENT
+            "$Tags.HTTP_METHOD.key" "GET"
+            "$Tags.HTTP_URL.key" "http://localhost:$emptyPort/ping"
+            "$DDTags.SPAN_TYPE" DDSpanTypes.HTTP_CLIENT
+            errorTags ProcessingException, String
+            defaultTags()
+          }
+        }
+      }
+    }
+
+    where:
+    builder                     | async | lib
+    new JerseyClientBuilder()   | false | "jersey"
+    new ResteasyClientBuilder() | false | "resteasy"
+    new JerseyClientBuilder()   | true  | "jersey async"
+    new ResteasyClientBuilder() | true  | "resteasy async"
+    // Unfortunately there's not a good way to instrument this for CXF.
+  }
 }

settings.gradle

@@ -25,6 +25,8 @@ include ':dd-java-agent:instrumentation:http-url-connection'
 include ':dd-java-agent:instrumentation:hystrix-1.4'
 include ':dd-java-agent:instrumentation:jax-rs-annotations'
 include ':dd-java-agent:instrumentation:jax-rs-client'
+include ':dd-java-agent:instrumentation:jax-rs-client:connection-error-handling-jersey'
+include ':dd-java-agent:instrumentation:jax-rs-client:connection-error-handling-resteasy'
 include ':dd-java-agent:instrumentation:java-concurrent'
 include ':dd-java-agent:instrumentation:java-concurrent:scala-testing'
 include ':dd-java-agent:instrumentation:java-concurrent:akka-testing'