Jetty-9 httpclient instrumentation, comments please (#3079)

* feat: jetty-9 http client, commit for PR comments

* chore: compensate for build break due to new TypeTransformer

* WIP: jetty9 with wrappers working

* feat: moved structure to match 9.2

* chore: fix package names again, now 9.2, clean up Nullables and Muzzle range

* chore: added latestDepTestLibrary for 10+ but class loader issues due to underlying jetty-server from the test framework

* chore: migrate to new instrumenter api

* chore: adjust to all the gradle plugin changes

* chore: added layers of builders

* chore: update final easy code review comments

* chore: remove comments in gradle

* chore: code cleanup, package cleanup, reduce surface area, iterators

* chore: cleanup of null checks, using HttpFlavorVals enum

* chore: null check removal again

* chore: set muzzle plugin correctly for 9.2 up to 9.4.+

* chore: convert gradle plugins to not use apply syntax

* Rename to build.gradle

* chore: move TypeInstrumention into standalone class

Co-authored-by: Anuraag Agrawal <aanuraag@amazon.co.jp>

instrumentation/jetty-httpclient/jetty-httpclient-9.2/javaagent/build.gradle

@@ -0,0 +1,28 @@
+plugins {
+  id("otel.javaagent-instrumentation")
+}
+
+muzzle {
+  pass {
+    group = "org.eclipse.jetty"
+    module = 'jetty-client'
+    versions = "[9.2,9.4.+)"
+  }
+}
+
+
+//Jetty client 9.2 is the best starting point, HttpClient.send() is stable there
+def jettyVers_base9 = '9.2.0.v20140526'
+
+
+dependencies {
+  implementation project(':instrumentation:jetty-httpclient:jetty-httpclient-9.2:library')
+
+  library "org.eclipse.jetty:jetty-client:${jettyVers_base9}"
+  latestDepTestLibrary "org.eclipse.jetty:jetty-client:9.+"
+
+  testImplementation project(':instrumentation:jetty-httpclient:jetty-httpclient-9.2:testing')
+  testImplementation("org.eclipse.jetty:jetty-server:${jettyVers_base9}") {
+    exclude group: 'org.eclipse.jetty', module: 'jetty-client'
+  }
+}

instrumentation/jetty-httpclient/jetty-httpclient-9.2/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/jetty/httpclient/v9_2/JettyHttpClient9Instrumentation.java

@@ -0,0 +1,87 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javaagent.instrumentation.jetty.httpclient.v9_2;
+
+import static io.opentelemetry.instrumentation.jetty.httpclient.v9_2.internal.JettyClientWrapUtil.wrapResponseListeners;
+import static io.opentelemetry.javaagent.instrumentation.api.Java8BytecodeBridge.currentContext;
+import static io.opentelemetry.javaagent.instrumentation.jetty.httpclient.v9_2.JettyHttpClientSingletons.instrumenter;
+import static net.bytebuddy.matcher.ElementMatchers.isMethod;
+import static net.bytebuddy.matcher.ElementMatchers.named;
+import static net.bytebuddy.matcher.ElementMatchers.takesArgument;
+
+import io.opentelemetry.context.Context;
+import io.opentelemetry.context.Scope;
+import io.opentelemetry.instrumentation.jetty.httpclient.v9_2.internal.JettyHttpClient9TracingInterceptor;
+import io.opentelemetry.javaagent.extension.instrumentation.TypeInstrumentation;
+import io.opentelemetry.javaagent.extension.instrumentation.TypeTransformer;
+import java.util.List;
+import net.bytebuddy.asm.Advice;
+import net.bytebuddy.description.type.TypeDescription;
+import net.bytebuddy.matcher.ElementMatcher;
+import org.eclipse.jetty.client.HttpRequest;
+import org.eclipse.jetty.client.api.Response;
+
+public class JettyHttpClient9Instrumentation implements TypeInstrumentation {
+  @Override
+  public ElementMatcher<TypeDescription> typeMatcher() {
+    return named("org.eclipse.jetty.client.HttpClient");
+  }
+
+  @Override
+  public void transform(TypeTransformer transformer) {
+    transformer.applyAdviceToMethod(
+        isMethod()
+            .and(named("send"))
+            .and(takesArgument(0, named("org.eclipse.jetty.client.HttpRequest")))
+            .and(takesArgument(1, List.class)),
+        JettyHttpClient9Instrumentation.class.getName() + "$JettyHttpClient9Advice");
+  }
+
+  @SuppressWarnings("unused")
+  public static class JettyHttpClient9Advice {
+
+    @Advice.OnMethodEnter(suppress = Throwable.class)
+    public static void addTracingEnter(
+        @Advice.Argument(value = 0) HttpRequest httpRequest,
+        @Advice.Argument(value = 1, readOnly = false) List<Response.ResponseListener> listeners,
+        @Advice.Local("otelContext") Context context,
+        @Advice.Local("otelScope") Scope scope) {
+      Context parentContext = currentContext();
+      if (!instrumenter().shouldStart(parentContext, httpRequest)) {
+        return;
+      }
+
+      // First step is to attach the tracer to the Jetty request. Request listeners are wrapped here
+      JettyHttpClient9TracingInterceptor requestInterceptor =
+          new JettyHttpClient9TracingInterceptor(parentContext, instrumenter());
+      requestInterceptor.attachToRequest(httpRequest);
+
+      // Second step is to wrap all the important result callback
+      listeners = wrapResponseListeners(parentContext, listeners);
+
+      context = requestInterceptor.getContext();
+      scope = context.makeCurrent();
+    }
+
+    @Advice.OnMethodExit(suppress = Throwable.class, onThrowable = Throwable.class)
+    public static void exitTracingInterceptor(
+        @Advice.Argument(value = 0) HttpRequest httpRequest,
+        @Advice.Thrown Throwable throwable,
+        @Advice.Local("otelContext") Context context,
+        @Advice.Local("otelScope") Scope scope) {
+
+      if (scope == null) {
+        return;
+      }
+
+      // not ending span here unless error, span ended in the interceptor
+      scope.close();
+      if (throwable != null) {
+        instrumenter().end(context, httpRequest, null, throwable);
+      }
+    }
+  }
+}

instrumentation/jetty-httpclient/jetty-httpclient-9.2/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/jetty/httpclient/v9_2/JettyHttpClient9InstrumentationModule.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javaagent.instrumentation.jetty.httpclient.v9_2;
+
+import static io.opentelemetry.javaagent.extension.matcher.ClassLoaderMatcher.hasClassesNamed;
+import static java.util.Collections.singletonList;
+
+import com.google.auto.service.AutoService;
+import io.opentelemetry.javaagent.extension.instrumentation.InstrumentationModule;
+import io.opentelemetry.javaagent.extension.instrumentation.TypeInstrumentation;
+import java.util.List;
+import net.bytebuddy.matcher.ElementMatcher;
+
+@AutoService(InstrumentationModule.class)
+public class JettyHttpClient9InstrumentationModule extends InstrumentationModule {
+
+  public JettyHttpClient9InstrumentationModule() {
+    super("jetty-httpclient", "jetty-httpclient-9.2");
+  }
+
+  @Override
+  public List<TypeInstrumentation> typeInstrumentations() {
+    return singletonList(new JettyHttpClient9Instrumentation());
+  }
+
+  @Override
+  public ElementMatcher.Junction<ClassLoader> classLoaderMatcher() {
+    // AbstractTypedContentProvider  showed up in version Jetty Client 9.2 on to 10.x
+    return hasClassesNamed("org.eclipse.jetty.client.util.AbstractTypedContentProvider");
+  }
+}

instrumentation/jetty-httpclient/jetty-httpclient-9.2/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/jetty/httpclient/v9_2/JettyHttpClientSingletons.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javaagent.instrumentation.jetty.httpclient.v9_2;
+
+import io.opentelemetry.api.GlobalOpenTelemetry;
+import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter;
+import io.opentelemetry.instrumentation.jetty.httpclient.v9_2.internal.JettyClientInstrumenterBuilder;
+import io.opentelemetry.instrumentation.jetty.httpclient.v9_2.internal.JettyHttpClientNetAttributesExtractor;
+import io.opentelemetry.javaagent.instrumentation.api.instrumenter.PeerServiceAttributesExtractor;
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.client.api.Response;
+
+public class JettyHttpClientSingletons {
+
+  private static final Instrumenter<Request, Response> INSTRUMENTER;
+
+  private JettyHttpClientSingletons() {}
+
+  static {
+    JettyClientInstrumenterBuilder builder =
+        new JettyClientInstrumenterBuilder(GlobalOpenTelemetry.get());
+
+    PeerServiceAttributesExtractor<Request, Response> peerServiceAttributesExtractor =
+        PeerServiceAttributesExtractor.create(new JettyHttpClientNetAttributesExtractor());
+    INSTRUMENTER = builder.addAttributeExtractor(peerServiceAttributesExtractor).build();
+  }
+
+  public static Instrumenter<Request, Response> instrumenter() {
+    return INSTRUMENTER;
+  }
+}

instrumentation/jetty-httpclient/jetty-httpclient-9.2/javaagent/src/test/groovy/io/opentelemetry/javaagent/instrumentation/jetty/httpclient/v9_2/JettyHttpClient9AgentTest.groovy

@@ -0,0 +1,24 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javaagent.instrumentation.jetty.httpclient.v9_2
+
+import io.opentelemetry.instrumentation.jetty.httpclient.v9_2.AbstractJettyClient9Test
+import io.opentelemetry.instrumentation.test.AgentTestTrait
+import org.eclipse.jetty.client.HttpClient
+import org.eclipse.jetty.util.ssl.SslContextFactory
+
+class JettyHttpClient9AgentTest extends AbstractJettyClient9Test implements AgentTestTrait {
+
+  @Override
+  HttpClient createStandardClient() {
+    return new HttpClient()
+  }
+
+  @Override
+  HttpClient createHttpsClient(SslContextFactory sslContextFactory) {
+    return new HttpClient(sslContextFactory)
+  }
+}

instrumentation/jetty-httpclient/jetty-httpclient-9.2/library/build.gradle

@@ -0,0 +1,17 @@
+plugins {
+  id("otel.library-instrumentation")
+  id("net.ltgt.errorprone")
+}
+
+
+//Jetty client 9.2 is the best starting point, HttpClient.send() is stable there
+def jettyVers_base9 = '9.2.0.v20140526'
+
+dependencies {
+  library "org.eclipse.jetty:jetty-client:${jettyVers_base9}"
+  latestDepTestLibrary "org.eclipse.jetty:jetty-client:9.+"
+  testImplementation project(':instrumentation:jetty-httpclient::jetty-httpclient-9.2:testing')
+
+  implementation "org.slf4j:slf4j-api"
+}
+

instrumentation/jetty-httpclient/jetty-httpclient-9.2/library/src/main/java/io/opentelemetry/instrumentation/jetty/httpclient/v9_2/JettyClientTracing.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.jetty.httpclient.v9_2;
+
+import io.opentelemetry.api.OpenTelemetry;
+import org.eclipse.jetty.client.HttpClient;
+
+/** JettyClientTracing, the Entrypoint for tracing Jetty client. */
+public final class JettyClientTracing {
+
+  private final HttpClient httpClient;
+
+  public static JettyClientTracing create(OpenTelemetry openTelemetry) {
+    JettyClientTracingBuilder builder = newBuilder(openTelemetry);
+    return builder.build();
+  }
+
+  public static JettyClientTracingBuilder newBuilder(OpenTelemetry openTelemetry) {
+    return new JettyClientTracingBuilder(openTelemetry);
+  }
+
+  public HttpClient getHttpClient() {
+    return httpClient;
+  }
+
+  JettyClientTracing(HttpClient httpClient) {
+    this.httpClient = httpClient;
+  }
+}

instrumentation/jetty-httpclient/jetty-httpclient-9.2/library/src/main/java/io/opentelemetry/instrumentation/jetty/httpclient/v9_2/JettyClientTracingBuilder.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.jetty.httpclient.v9_2;
+
+import io.opentelemetry.api.OpenTelemetry;
+import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter;
+import io.opentelemetry.instrumentation.jetty.httpclient.v9_2.internal.JettyClientInstrumenterBuilder;
+import org.eclipse.jetty.client.HttpClientTransport;
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.client.api.Response;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
+
+public final class JettyClientTracingBuilder {
+
+  private final OpenTelemetry openTelemetry;
+  private HttpClientTransport httpClientTransport;
+  private SslContextFactory sslContextFactory;
+
+  public JettyClientTracingBuilder(OpenTelemetry openTelemetry) {
+    this.openTelemetry = openTelemetry;
+  }
+
+  public JettyClientTracingBuilder setHttpClientTransport(HttpClientTransport httpClientTransport) {
+    this.httpClientTransport = httpClientTransport;
+    return this;
+  }
+
+  public JettyClientTracingBuilder setSslContextFactory(SslContextFactory sslContextFactory) {
+    this.sslContextFactory = sslContextFactory;
+    return this;
+  }
+
+  public JettyClientTracing build() {
+    JettyClientInstrumenterBuilder instrumenterBuilder =
+        new JettyClientInstrumenterBuilder(this.openTelemetry);
+    Instrumenter<Request, Response> instrumenter = instrumenterBuilder.build();
+
+    TracingHttpClient tracingHttpClient =
+        TracingHttpClient.buildNew(instrumenter, this.sslContextFactory, this.httpClientTransport);
+
+    return new JettyClientTracing(tracingHttpClient);
+  }
+}

instrumentation/jetty-httpclient/jetty-httpclient-9.2/library/src/main/java/io/opentelemetry/instrumentation/jetty/httpclient/v9_2/TracingHttpClient.java

@@ -0,0 +1,69 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.jetty.httpclient.v9_2;
+
+import static io.opentelemetry.instrumentation.jetty.httpclient.v9_2.internal.JettyClientWrapUtil.wrapResponseListeners;
+
+import io.opentelemetry.context.Context;
+import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter;
+import io.opentelemetry.instrumentation.jetty.httpclient.v9_2.internal.JettyHttpClient9TracingInterceptor;
+import java.util.List;
+import org.eclipse.jetty.client.HttpClient;
+import org.eclipse.jetty.client.HttpClientTransport;
+import org.eclipse.jetty.client.HttpRequest;
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.client.api.Response;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
+
+class TracingHttpClient extends HttpClient {
+
+  private final Instrumenter<Request, Response> instrumenter;
+
+  TracingHttpClient(Instrumenter<Request, Response> instrumenter) {
+    super();
+    this.instrumenter = instrumenter;
+  }
+
+  TracingHttpClient(
+      Instrumenter<Request, Response> instrumenter, SslContextFactory sslContextFactory) {
+    super(sslContextFactory);
+    this.instrumenter = instrumenter;
+  }
+
+  TracingHttpClient(
+      Instrumenter<Request, Response> instrumenter,
+      HttpClientTransport transport,
+      SslContextFactory sslContextFactory) {
+    super(transport, sslContextFactory);
+    this.instrumenter = instrumenter;
+  }
+
+  static TracingHttpClient buildNew(
+      Instrumenter<Request, Response> instrumenter,
+      SslContextFactory sslContextFactory,
+      HttpClientTransport httpClientTransport) {
+    TracingHttpClient tracingHttpClient = null;
+    if (sslContextFactory != null && httpClientTransport != null) {
+      tracingHttpClient =
+          new TracingHttpClient(instrumenter, httpClientTransport, sslContextFactory);
+    } else if (sslContextFactory != null) {
+      tracingHttpClient = new TracingHttpClient(instrumenter, sslContextFactory);
+    } else {
+      tracingHttpClient = new TracingHttpClient(instrumenter);
+    }
+    return tracingHttpClient;
+  }
+
+  @Override
+  protected void send(HttpRequest request, List<Response.ResponseListener> listeners) {
+    Context parentContext = Context.current();
+    JettyHttpClient9TracingInterceptor requestInterceptor =
+        new JettyHttpClient9TracingInterceptor(parentContext, this.instrumenter);
+    requestInterceptor.attachToRequest(request);
+    List<Response.ResponseListener> wrapped = wrapResponseListeners(parentContext, listeners);
+    super.send(request, wrapped);
+  }
+}

instrumentation/jetty-httpclient/jetty-httpclient-9.2/library/src/main/java/io/opentelemetry/instrumentation/jetty/httpclient/v9_2/internal/HttpHeaderSetter.java

@@ -0,0 +1,20 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.jetty.httpclient.v9_2.internal;
+
+import io.opentelemetry.context.propagation.TextMapSetter;
+import org.eclipse.jetty.client.api.Request;
+
+final class HttpHeaderSetter implements TextMapSetter<Request> {
+
+  @Override
+  public void set(Request request, String key, String value) {
+    if (request != null) {
+      // dedupe header fields here with a put()
+      request.getHeaders().put(key, value);
+    }
+  }
+}

instrumentation/jetty-httpclient/jetty-httpclient-9.2/library/src/main/java/io/opentelemetry/instrumentation/jetty/httpclient/v9_2/internal/JettyClientHttpAttributesExtractor.java

@@ -0,0 +1,156 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.jetty.httpclient.v9_2.internal;
+
+import static io.opentelemetry.semconv.trace.attributes.SemanticAttributes.HttpFlavorValues.HTTP_1_0;
+import static io.opentelemetry.semconv.trace.attributes.SemanticAttributes.HttpFlavorValues.HTTP_1_1;
+import static io.opentelemetry.semconv.trace.attributes.SemanticAttributes.HttpFlavorValues.HTTP_2_0;
+
+import io.opentelemetry.instrumentation.api.instrumenter.http.HttpAttributesExtractor;
+import org.checkerframework.checker.nullness.qual.Nullable;
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.client.api.Response;
+import org.eclipse.jetty.http.HttpField;
+import org.eclipse.jetty.http.HttpHeader;
+import org.eclipse.jetty.http.HttpVersion;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+final class JettyClientHttpAttributesExtractor extends HttpAttributesExtractor<Request, Response> {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(JettyClientHttpAttributesExtractor.class);
+
+  @Override
+  @Nullable
+  protected String method(Request request) {
+    return request.getMethod();
+  }
+
+  @Override
+  @Nullable
+  protected String url(Request request) {
+    return request.getURI().toString();
+  }
+
+  @Override
+  @Nullable
+  protected String target(Request request) {
+    String queryString = request.getQuery();
+    return queryString != null ? request.getPath() + "?" + queryString : request.getPath();
+  }
+
+  @Override
+  @Nullable
+  protected String host(Request request) {
+    return request.getHost();
+  }
+
+  @Override
+  @Nullable
+  protected String route(Request request) {
+    return null;
+  }
+
+  @Override
+  @Nullable
+  protected String scheme(Request request) {
+    return request.getScheme();
+  }
+
+  @Override
+  @Nullable
+  protected String userAgent(Request request) {
+    HttpField agentField = request.getHeaders().getField(HttpHeader.USER_AGENT);
+    return agentField != null ? agentField.getValue() : null;
+  }
+
+  @Override
+  @Nullable
+  protected Long requestContentLength(Request request, @Nullable Response response) {
+    HttpField requestContentLengthField = request.getHeaders().getField(HttpHeader.CONTENT_LENGTH);
+    return getLongFromJettyHttpField(requestContentLengthField);
+  }
+
+  @Override
+  @Nullable
+  protected Long requestContentLengthUncompressed(Request request, @Nullable Response response) {
+    return null;
+  }
+
+  @Override
+  @Nullable
+  protected String flavor(Request request, @Nullable Response response) {
+
+    if (response == null) {
+      return HTTP_1_1;
+    }
+    HttpVersion httpVersion = response.getVersion();
+    httpVersion = (httpVersion != null) ? httpVersion : HttpVersion.HTTP_1_1;
+    switch (httpVersion) {
+      case HTTP_0_9:
+      case HTTP_1_0:
+        return HTTP_1_0;
+      case HTTP_1_1:
+        return HTTP_1_1;
+      default:
+        // version 2.0 enum name difference in later versions 9.2 and 9.4 versions
+        if (httpVersion.toString().endsWith("2.0")) {
+          return HTTP_2_0;
+        }
+
+        return HTTP_1_1;
+    }
+  }
+
+  @Override
+  @Nullable
+  protected String serverName(Request request, @Nullable Response response) {
+    return null;
+  }
+
+  @Override
+  @Nullable
+  protected String clientIp(Request request, @Nullable Response response) {
+    return null;
+  }
+
+  @Override
+  @Nullable
+  protected Integer statusCode(Request request, Response response) {
+    return response.getStatus();
+  }
+
+  @Override
+  @Nullable
+  protected Long responseContentLength(Request request, Response response) {
+    Long respContentLength = null;
+    if (response != null) {
+      HttpField requestContentLengthField =
+          response.getHeaders().getField(HttpHeader.CONTENT_LENGTH);
+      respContentLength = getLongFromJettyHttpField(requestContentLengthField);
+    }
+    return respContentLength;
+  }
+
+  @Override
+  @Nullable
+  protected Long responseContentLengthUncompressed(Request request, Response response) {
+    return null;
+  }
+
+  private static Long getLongFromJettyHttpField(HttpField httpField) {
+    Long longFromField = null;
+    try {
+      longFromField = httpField != null ? Long.getLong(httpField.getValue()) : null;
+    } catch (NumberFormatException t) {
+      LOG.debug(
+          "Value {} is not  not valid number format for header field: {}",
+          httpField.getValue(),
+          httpField.getName());
+    }
+    return longFromField;
+  }
+}

instrumentation/jetty-httpclient/jetty-httpclient-9.2/library/src/main/java/io/opentelemetry/instrumentation/jetty/httpclient/v9_2/internal/JettyClientInstrumenterBuilder.java

@@ -0,0 +1,60 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.jetty.httpclient.v9_2.internal;
+
+import io.opentelemetry.api.OpenTelemetry;
+import io.opentelemetry.instrumentation.api.instrumenter.AttributesExtractor;
+import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter;
+import io.opentelemetry.instrumentation.api.instrumenter.SpanNameExtractor;
+import io.opentelemetry.instrumentation.api.instrumenter.SpanStatusExtractor;
+import io.opentelemetry.instrumentation.api.instrumenter.http.HttpAttributesExtractor;
+import io.opentelemetry.instrumentation.api.instrumenter.http.HttpSpanNameExtractor;
+import io.opentelemetry.instrumentation.api.instrumenter.http.HttpSpanStatusExtractor;
+import java.util.ArrayList;
+import java.util.List;
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.client.api.Response;
+
+public final class JettyClientInstrumenterBuilder {
+
+  private static final String INSTRUMENTATION_NAME = "io.opentelemetry.jetty-httpclient-9.2";
+
+  private final OpenTelemetry openTelemetry;
+
+  private final List<AttributesExtractor<? super Request, ? super Response>> additionalExtractors =
+      new ArrayList<>();
+
+  public JettyClientInstrumenterBuilder addAttributeExtractor(
+      AttributesExtractor<? super Request, ? super Response> attributesExtractor) {
+    additionalExtractors.add(attributesExtractor);
+    return this;
+  }
+
+  public JettyClientInstrumenterBuilder(OpenTelemetry openTelemetry) {
+    this.openTelemetry = openTelemetry;
+  }
+
+  public Instrumenter<Request, Response> build() {
+    HttpAttributesExtractor<Request, Response> httpAttributesExtractor =
+        new JettyClientHttpAttributesExtractor();
+    SpanNameExtractor<Request> spanNameExtractor =
+        HttpSpanNameExtractor.create(httpAttributesExtractor);
+    SpanStatusExtractor<Request, Response> spanStatusExtractor =
+        HttpSpanStatusExtractor.create(httpAttributesExtractor);
+    JettyHttpClientNetAttributesExtractor netAttributesExtractor =
+        new JettyHttpClientNetAttributesExtractor();
+
+    Instrumenter<Request, Response> instrumenter =
+        Instrumenter.<Request, Response>newBuilder(
+                this.openTelemetry, INSTRUMENTATION_NAME, spanNameExtractor)
+            .setSpanStatusExtractor(spanStatusExtractor)
+            .addAttributesExtractor(httpAttributesExtractor)
+            .addAttributesExtractor(netAttributesExtractor)
+            .addAttributesExtractors(additionalExtractors)
+            .newClientInstrumenter(new HttpHeaderSetter());
+    return instrumenter;
+  }
+}

instrumentation/jetty-httpclient/jetty-httpclient-9.2/library/src/main/java/io/opentelemetry/instrumentation/jetty/httpclient/v9_2/internal/JettyClientWrapUtil.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.jetty.httpclient.v9_2.internal;
+
+import static java.util.stream.Collectors.toList;
+
+import io.opentelemetry.context.Context;
+import io.opentelemetry.context.Scope;
+import java.util.List;
+import org.eclipse.jetty.client.api.Response;
+
+public final class JettyClientWrapUtil {
+
+  private JettyClientWrapUtil() {}
+
+  /**
+   * Utility to wrap the response listeners only, this includes the important CompleteListener.
+   *
+   * @param parentContext top level context that is above the Jetty client span context
+   * @param listeners all listeners passed to Jetty client send() method
+   * @return list of wrapped ResponseListeners
+   */
+  public static List<Response.ResponseListener> wrapResponseListeners(
+      Context parentContext, List<Response.ResponseListener> listeners) {
+
+    return listeners.stream()
+        .map(listener -> wrapTheListener(listener, parentContext))
+        .collect(toList());
+  }
+
+  private static Response.ResponseListener wrapTheListener(
+      Response.ResponseListener listener, Context context) {
+    Response.ResponseListener wrappedListener = listener;
+    if (listener instanceof Response.CompleteListener
+        && !(listener instanceof JettyHttpClient9TracingInterceptor)) {
+      wrappedListener =
+          (Response.CompleteListener)
+              result -> {
+                try (Scope ignored = context.makeCurrent()) {
+                  ((Response.CompleteListener) listener).onComplete(result);
+                }
+              };
+    }
+    return wrappedListener;
+  }
+}

instrumentation/jetty-httpclient/jetty-httpclient-9.2/library/src/main/java/io/opentelemetry/instrumentation/jetty/httpclient/v9_2/internal/JettyHttpClient9TracingInterceptor.java

@@ -0,0 +1,154 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.jetty.httpclient.v9_2.internal;
+
+import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.context.Context;
+import io.opentelemetry.context.Scope;
+import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter;
+import io.opentelemetry.semconv.trace.attributes.SemanticAttributes;
+import java.util.List;
+import java.util.ListIterator;
+import org.checkerframework.checker.nullness.qual.Nullable;
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.client.api.Response;
+import org.eclipse.jetty.client.api.Result;
+import org.eclipse.jetty.http.HttpField;
+import org.eclipse.jetty.http.HttpHeader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * JettyHttpClient9TracingInterceptor does three jobs stimulated from the Jetty Request object from
+ * attachToRequest() 1. Start the CLIENT span and create the tracer 2. Set the listener callbacks
+ * for each important lifecycle actions that would cause the start and close of the span 3. Set
+ * callback wrappers on two important request-based callbacks
+ */
+public class JettyHttpClient9TracingInterceptor
+    implements Request.BeginListener,
+        Request.FailureListener,
+        Response.SuccessListener,
+        Response.FailureListener,
+        Response.CompleteListener {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(JettyHttpClient9TracingInterceptor.class);
+
+  @Nullable private Context context;
+
+  @Nullable
+  public Context getContext() {
+    return this.context;
+  }
+
+  private final Context parentContext;
+
+  private final Instrumenter<Request, Response> instrumenter;
+
+  public JettyHttpClient9TracingInterceptor(
+      Context parentCtx, Instrumenter<Request, Response> instrumenter) {
+    this.parentContext = parentCtx;
+    this.instrumenter = instrumenter;
+  }
+
+  public void attachToRequest(Request jettyRequest) {
+    List<JettyHttpClient9TracingInterceptor> current =
+        jettyRequest.getRequestListeners(JettyHttpClient9TracingInterceptor.class);
+
+    if (!current.isEmpty()) {
+      LOG.warn("A tracing interceptor is already in place for this request! ");
+      return;
+    }
+    startSpan(jettyRequest);
+
+    // wrap all important request-based listeners that may already be attached, null should ensure
+    // are returned here
+    List<Request.RequestListener> existingListeners = jettyRequest.getRequestListeners(null);
+    wrapRequestListeners(existingListeners);
+
+    jettyRequest
+        .onRequestBegin(this)
+        .onRequestFailure(this)
+        .onResponseFailure(this)
+        .onResponseSuccess(this);
+  }
+
+  private void wrapRequestListeners(List<Request.RequestListener> requestListeners) {
+
+    ListIterator<Request.RequestListener> iterator = requestListeners.listIterator();
+    while (iterator.hasNext()) {
+      Request.RequestListener requestListener = iterator.next();
+      if (requestListener instanceof Request.FailureListener) {
+        iterator.set(
+            (Request.FailureListener)
+                (request, throwable) -> {
+                  try (Scope ignore = context.makeCurrent()) {
+                    ((Request.FailureListener) requestListener).onFailure(request, throwable);
+                  }
+                });
+      }
+      if (requestListener instanceof Request.BeginListener) {
+        iterator.set(
+            (Request.FailureListener)
+                (request, throwable) -> {
+                  try (Scope ignore = context.makeCurrent()) {
+                    ((Request.BeginListener) requestListener).onBegin(request);
+                  }
+                });
+      }
+    }
+  }
+
+  private void startSpan(Request request) {
+
+    if (!instrumenter.shouldStart(this.parentContext, request)) {
+      return;
+    }
+    this.context = instrumenter.start(this.parentContext, request);
+  }
+
+  @Override
+  public void onBegin(Request request) {
+    if (this.context != null) {
+      Span span = Span.fromContext(this.context);
+      HttpField agentField = request.getHeaders().getField(HttpHeader.USER_AGENT);
+      span.setAttribute(SemanticAttributes.HTTP_USER_AGENT, agentField.getValue());
+    }
+  }
+
+  @Override
+  public void onComplete(Result result) {
+    closeIfPossible(result.getResponse());
+  }
+
+  @Override
+  public void onSuccess(Response response) {
+    closeIfPossible(response);
+  }
+
+  @Override
+  public void onFailure(Request request, Throwable t) {
+    if (this.context != null) {
+      instrumenter.end(this.context, request, null, t);
+    }
+  }
+
+  @Override
+  public void onFailure(Response response, Throwable t) {
+    if (this.context != null) {
+      instrumenter.end(this.context, response.getRequest(), response, t);
+    }
+  }
+
+  private void closeIfPossible(Response response) {
+
+    if (this.context != null) {
+      instrumenter.end(this.context, response.getRequest(), response, null);
+    } else {
+      LOG.debug("onComplete - could not find an otel context");
+    }
+  }
+}

instrumentation/jetty-httpclient/jetty-httpclient-9.2/library/src/main/java/io/opentelemetry/instrumentation/jetty/httpclient/v9_2/internal/JettyHttpClientNetAttributesExtractor.java

@@ -0,0 +1,41 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.jetty.httpclient.v9_2.internal;
+
+import io.opentelemetry.instrumentation.api.instrumenter.net.NetAttributesExtractor;
+import io.opentelemetry.semconv.trace.attributes.SemanticAttributes;
+import org.checkerframework.checker.nullness.qual.Nullable;
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.client.api.Response;
+
+public class JettyHttpClientNetAttributesExtractor
+    extends NetAttributesExtractor<Request, Response> {
+
+  @Override
+  public String transport(Request request) {
+    return SemanticAttributes.NetTransportValues.IP_TCP;
+  }
+
+  @Override
+  @Nullable
+  public String peerName(Request request, @Nullable Response response) {
+    return request.getHost();
+  }
+
+  @Override
+  @Nullable
+  public Integer peerPort(Request request, @Nullable Response response) {
+    return request.getPort();
+  }
+
+  @Override
+  @Nullable
+  public String peerIp(Request request, @Nullable Response response) {
+    // Return null unless the library supports resolution to something similar to SocketAddress
+    // https://github.com/open-telemetry/opentelemetry-java-instrumentation/pull/3012/files#r633188645
+    return null;
+  }
+}

instrumentation/jetty-httpclient/jetty-httpclient-9.2/library/src/test/groovy/io/opentelemetry/instrumentation/jetty/httpclient/v9_2/JettyHttpClient9LibraryTest.groovy

@@ -0,0 +1,36 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.jetty.httpclient.v9_2
+
+
+import io.opentelemetry.instrumentation.test.LibraryTestTrait
+import org.eclipse.jetty.client.HttpClient
+import org.eclipse.jetty.util.ssl.SslContextFactory
+
+class JettyHttpClient9LibraryTest extends AbstractJettyClient9Test implements LibraryTestTrait {
+
+
+  @Override
+  boolean testWithClientParent() {
+    //The client parent test does not work well in the context of library only tests.
+    false
+  }
+
+  @Override
+  HttpClient createStandardClient() {
+    JettyClientTracingBuilder jettyClientTracingBuilder = new JettyClientTracingBuilder(getOpenTelemetry())
+    return jettyClientTracingBuilder.build().getHttpClient()
+  }
+
+  @Override
+  HttpClient createHttpsClient(SslContextFactory sslContextFactory) {
+    JettyClientTracingBuilder jettyClientTracingBuilder = new JettyClientTracingBuilder(getOpenTelemetry())
+    return jettyClientTracingBuilder
+      .setSslContextFactory(sslContextFactory)
+      .build()
+      .getHttpClient()
+  }
+}

instrumentation/jetty-httpclient/jetty-httpclient-9.2/testing/build.gradle

@@ -0,0 +1,22 @@
+plugins {
+  id("otel.java-conventions")
+}
+
+//Jetty client 9.2 is the best starting point, HttpClient.send() is stable there
+def jettyVers_base9 = '9.2.0.v20140526'
+
+dependencies {
+  api(project(':testing-common')) {
+    exclude group: 'org.eclipse.jetty', module: 'jetty-client'
+    exclude group: 'org.eclipse.jetty', module: 'jetty-server'
+  }
+
+
+  api "org.eclipse.jetty:jetty-client:${jettyVers_base9}"
+
+  implementation "org.junit.jupiter:junit-jupiter-api"
+
+  implementation "org.codehaus.groovy:groovy-all"
+  implementation "io.opentelemetry:opentelemetry-api"
+  implementation "org.spockframework:spock-core"
+}

instrumentation/jetty-httpclient/jetty-httpclient-9.2/testing/src/main/groovy/io/opentelemetry/instrumentation/jetty/httpclient/v9_2/AbstractJettyClient9Test.groovy

@@ -0,0 +1,150 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.jetty.httpclient.v9_2
+
+import io.opentelemetry.api.common.AttributeKey
+import io.opentelemetry.instrumentation.test.base.HttpClientTest
+import io.opentelemetry.semconv.trace.attributes.SemanticAttributes
+import org.eclipse.jetty.client.HttpClient
+import org.eclipse.jetty.client.api.ContentResponse
+import org.eclipse.jetty.client.api.Request
+import org.eclipse.jetty.client.api.Response
+import org.eclipse.jetty.client.api.Result
+import org.eclipse.jetty.http.HttpMethod
+import org.eclipse.jetty.util.ssl.SslContextFactory
+import org.junit.Rule
+import org.junit.rules.TestName
+import spock.lang.Shared
+
+import java.util.concurrent.TimeUnit
+
+abstract class AbstractJettyClient9Test extends HttpClientTest<Request> {
+
+  abstract HttpClient createStandardClient()
+
+  abstract HttpClient createHttpsClient(SslContextFactory sslContextFactory)
+
+
+  @Shared
+  def client = createStandardClient()
+  @Shared
+  def httpsClient = null
+
+  @Rule
+  TestName name = new TestName()
+
+  Request jettyRequest = null
+
+  def setupSpec() {
+
+    //Start the main Jetty HttpClient and a https client
+    client.start()
+
+    SslContextFactory tlsCtx = new SslContextFactory()
+    httpsClient = createHttpsClient(tlsCtx)
+    httpsClient.setFollowRedirects(false)
+    httpsClient.start()
+  }
+
+  @Override
+  Request buildRequest(String method, URI uri, Map<String, String> headers) {
+
+    HttpClient theClient = uri.scheme == 'https' ? httpsClient : client
+
+    Request request = theClient.newRequest(uri)
+
+    HttpMethod methodObj = HttpMethod.valueOf(method)
+    request.method(methodObj)
+    request.timeout(CONNECT_TIMEOUT_MS, TimeUnit.MILLISECONDS)
+
+    jettyRequest = request
+
+    return request
+  }
+
+  @Override
+  String userAgent() {
+    if (name.methodName.startsWith('connection error') && jettyRequest.getAgent() == null) {
+      return null
+    }
+    return "Jetty"
+  }
+
+  @Override
+  int sendRequest(Request request, String method, URI uri, Map<String, String> headers) {
+    headers.each { k, v ->
+      request.header(k, v)
+    }
+
+    ContentResponse response = request.send()
+
+    return response.status
+  }
+
+  private static class JettyClientListener implements Request.FailureListener, Response.FailureListener {
+
+    volatile Throwable failure
+
+    @Override
+    void onFailure(Request requestF, Throwable failure) {
+      this.failure = failure
+
+    }
+
+    @Override
+    void onFailure(Response responseF, Throwable failure) {
+      this.failure = failure
+    }
+
+  }
+
+  @Override
+  void sendRequestWithCallback(Request request, String method, URI uri, Map<String, String> headers, RequestResult requestResult) {
+
+    JettyClientListener jcl = new JettyClientListener()
+
+    request.onRequestFailure(jcl)
+    request.onResponseFailure(jcl)
+    headers.each { k, v ->
+      request.header(k, v)
+    }
+
+    request.send(new Response.CompleteListener() {
+      @Override
+      void onComplete(Result result) {
+
+        if (jcl.failure != null) {
+          requestResult.complete(jcl.failure)
+          return
+        }
+
+        requestResult.complete(result.response.status)
+      }
+    })
+  }
+
+
+  @Override
+  boolean testRedirects() {
+    false
+  }
+
+  @Override
+  boolean testCausality() {
+    true
+  }
+
+  @Override
+  Set<AttributeKey<?>> httpAttributes(URI uri) {
+    Set<AttributeKey<?>> extra = [
+      SemanticAttributes.HTTP_SCHEME,
+      SemanticAttributes.HTTP_TARGET,
+      SemanticAttributes.HTTP_HOST
+    ]
+    super.httpAttributes(uri) + extra
+  }
+
+}

settings.gradle

@@ -185,6 +185,9 @@ include ':instrumentation:jedis:jedis-3.0:javaagent'
 include ':instrumentation:jetty:jetty-8.0:javaagent'
 include ':instrumentation:jetty:jetty-11.0:javaagent'
 include ':instrumentation:jetty:jetty-common:javaagent'
+include ':instrumentation:jetty-httpclient:jetty-httpclient-9.2:javaagent'
+include ':instrumentation:jetty-httpclient:jetty-httpclient-9.2:library'
+include ':instrumentation:jetty-httpclient:jetty-httpclient-9.2:testing'
 include ':instrumentation:jms-1.1:javaagent'
 include ':instrumentation:jms-1.1:javaagent-unit-tests'
 include ':instrumentation:jsf:jsf-common:library'