Trace SSL handshakes in netty 4.1 (#4604)

* Trace SSL handshakes in netty 4.1

* Update testing-common/src/main/java/io/opentelemetry/instrumentation/testing/junit/http/HttpClientTestServer.java

Co-authored-by: Trask Stalnaker <trask.stalnaker@gmail.com>

* remove unneeded bit of code

Co-authored-by: Trask Stalnaker <trask.stalnaker@gmail.com>
This commit is contained in:
Mateusz Rzeszutek 2021-11-10 22:43:59 +01:00 committed by GitHub
parent 4e39f1ad5a
commit 4719e4cc79
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 515 additions and 6 deletions

View File

@ -21,11 +21,13 @@ public final class NettyClientInstrumenterFactory {
private final String instrumentationName;
private final boolean alwaysCreateConnectSpan;
private final boolean sslTelemetryEnabled;
public NettyClientInstrumenterFactory(
String instrumentationName, boolean alwaysCreateConnectSpan) {
String instrumentationName, boolean alwaysCreateConnectSpan, boolean sslTelemetryEnabled) {
this.instrumentationName = instrumentationName;
this.alwaysCreateConnectSpan = alwaysCreateConnectSpan;
this.sslTelemetryEnabled = sslTelemetryEnabled;
}
public Instrumenter<HttpRequestAndChannel, HttpResponse> createHttpInstrumenter() {
@ -66,4 +68,24 @@ public final class NettyClientInstrumenterFactory {
? new NettyConnectionInstrumenterImpl(instrumenter)
: new NettyErrorOnlyConnectionInstrumenter(instrumenter);
}
public NettySslInstrumenter createSslInstrumenter() {
NettySslNetAttributesExtractor netAttributesExtractor = new NettySslNetAttributesExtractor();
Instrumenter<NettySslRequest, Void> instrumenter =
Instrumenter.<NettySslRequest, Void>builder(
GlobalOpenTelemetry.get(), instrumentationName, NettySslRequest::spanName)
.addAttributesExtractor(netAttributesExtractor)
.addAttributesExtractor(PeerServiceAttributesExtractor.create(netAttributesExtractor))
.setTimeExtractors(
request -> request.timer().startTime(),
(request, channel, error) -> request.timer().now())
.newInstrumenter(
sslTelemetryEnabled
? SpanKindExtractor.alwaysInternal()
: SpanKindExtractor.alwaysClient());
return sslTelemetryEnabled
? new NettySslInstrumenterImpl(instrumenter)
: new NettySslErrorOnlyInstrumenter(instrumenter);
}
}

View File

@ -0,0 +1,38 @@
/*
* Copyright The OpenTelemetry Authors
* SPDX-License-Identifier: Apache-2.0
*/
package io.opentelemetry.javaagent.instrumentation.netty.common.client;
import io.opentelemetry.context.Context;
import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter;
import javax.annotation.Nullable;
final class NettySslErrorOnlyInstrumenter implements NettySslInstrumenter {
private final Instrumenter<NettySslRequest, Void> instrumenter;
NettySslErrorOnlyInstrumenter(Instrumenter<NettySslRequest, Void> instrumenter) {
this.instrumenter = instrumenter;
}
@Override
public boolean shouldStart(Context parentContext, NettySslRequest request) {
// the "real" check is done on end() anyway
return true;
}
@Override
public Context start(Context parentContext, NettySslRequest request) {
return parentContext;
}
@Override
public void end(Context context, NettySslRequest request, @Nullable Throwable error) {
if (error != null && instrumenter.shouldStart(context, request)) {
Context connectContext = instrumenter.start(context, request);
instrumenter.end(connectContext, request, null, error);
}
}
}

View File

@ -0,0 +1,74 @@
/*
* Copyright The OpenTelemetry Authors
* SPDX-License-Identifier: Apache-2.0
*/
package io.opentelemetry.javaagent.instrumentation.netty.common.client;
import io.netty.channel.ChannelDuplexHandler;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelPromise;
import io.netty.handler.ssl.SslHandshakeCompletionEvent;
import io.opentelemetry.context.Context;
import java.net.SocketAddress;
// inspired by reactor-netty SslProvider.SslReadHandler
public final class NettySslInstrumentationHandler extends ChannelDuplexHandler {
private final NettySslInstrumenter instrumenter;
private Context parentContext;
private NettySslRequest request;
private Context context;
public NettySslInstrumentationHandler(NettySslInstrumenter instrumenter) {
this.instrumenter = instrumenter;
}
@Override
public void channelRegistered(ChannelHandlerContext ctx) {
// remember the parent context from the time of channel registration;
// this happens inside Bootstrap#connect()
parentContext = Context.current();
ctx.fireChannelRegistered();
}
@Override
public void connect(
ChannelHandlerContext ctx,
SocketAddress remoteAddress,
SocketAddress localAddress,
ChannelPromise promise) {
// netty SslHandler starts the handshake after it receives the channelActive() signal; this
// happens just after the connection is established
// this makes connect() promise a good place to start the SSL handshake span
promise.addListener(
future -> {
// there won't be any SSL handshake if the channel fails to connect
if (!future.isSuccess()) {
return;
}
request = NettySslRequest.create(ctx.channel());
if (instrumenter.shouldStart(parentContext, request)) {
context = instrumenter.start(parentContext, request);
}
});
ctx.connect(remoteAddress, localAddress, promise);
}
@Override
public void userEventTriggered(ChannelHandlerContext ctx, Object evt) {
if (evt instanceof SslHandshakeCompletionEvent) {
if (ctx.pipeline().context(this) != null) {
ctx.pipeline().remove(this);
}
SslHandshakeCompletionEvent handshake = (SslHandshakeCompletionEvent) evt;
if (context != null) {
instrumenter.end(context, request, handshake.cause());
}
}
ctx.fireUserEventTriggered(evt);
}
}

View File

@ -0,0 +1,18 @@
/*
* Copyright The OpenTelemetry Authors
* SPDX-License-Identifier: Apache-2.0
*/
package io.opentelemetry.javaagent.instrumentation.netty.common.client;
import io.opentelemetry.context.Context;
import javax.annotation.Nullable;
public interface NettySslInstrumenter {
boolean shouldStart(Context parentContext, NettySslRequest request);
Context start(Context parentContext, NettySslRequest request);
void end(Context context, NettySslRequest request, @Nullable Throwable error);
}

View File

@ -0,0 +1,34 @@
/*
* Copyright The OpenTelemetry Authors
* SPDX-License-Identifier: Apache-2.0
*/
package io.opentelemetry.javaagent.instrumentation.netty.common.client;
import io.opentelemetry.context.Context;
import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter;
import javax.annotation.Nullable;
final class NettySslInstrumenterImpl implements NettySslInstrumenter {
private final Instrumenter<NettySslRequest, Void> instrumenter;
NettySslInstrumenterImpl(Instrumenter<NettySslRequest, Void> instrumenter) {
this.instrumenter = instrumenter;
}
@Override
public boolean shouldStart(Context parentContext, NettySslRequest request) {
return instrumenter.shouldStart(parentContext, request);
}
@Override
public Context start(Context parentContext, NettySslRequest request) {
return instrumenter.start(parentContext, request);
}
@Override
public void end(Context context, NettySslRequest request, @Nullable Throwable error) {
instrumenter.end(context, request, null, error);
}
}

View File

@ -0,0 +1,32 @@
/*
* Copyright The OpenTelemetry Authors
* SPDX-License-Identifier: Apache-2.0
*/
package io.opentelemetry.javaagent.instrumentation.netty.common.client;
import static io.opentelemetry.semconv.trace.attributes.SemanticAttributes.NetTransportValues.IP_TCP;
import static io.opentelemetry.semconv.trace.attributes.SemanticAttributes.NetTransportValues.IP_UDP;
import io.netty.channel.socket.DatagramChannel;
import io.opentelemetry.instrumentation.api.instrumenter.net.InetSocketAddressNetClientAttributesExtractor;
import java.net.InetSocketAddress;
import javax.annotation.Nullable;
final class NettySslNetAttributesExtractor
extends InetSocketAddressNetClientAttributesExtractor<NettySslRequest, Void> {
@Nullable
@Override
public InetSocketAddress getAddress(NettySslRequest request, @Nullable Void unused) {
if (request.remoteAddress() instanceof InetSocketAddress) {
return (InetSocketAddress) request.remoteAddress();
}
return null;
}
@Override
public String transport(NettySslRequest request, @Nullable Void unused) {
return request.channel() instanceof DatagramChannel ? IP_UDP : IP_TCP;
}
}

View File

@ -0,0 +1,31 @@
/*
* Copyright The OpenTelemetry Authors
* SPDX-License-Identifier: Apache-2.0
*/
package io.opentelemetry.javaagent.instrumentation.netty.common.client;
import com.google.auto.value.AutoValue;
import io.netty.channel.Channel;
import io.opentelemetry.javaagent.instrumentation.netty.common.Timer;
import java.net.SocketAddress;
import javax.annotation.Nullable;
@AutoValue
public abstract class NettySslRequest {
static NettySslRequest create(Channel channel) {
return new AutoValue_NettySslRequest(Timer.start(), channel, channel.remoteAddress());
}
String spanName() {
return "SSL handshake";
}
abstract Timer timer();
abstract Channel channel();
@Nullable
abstract SocketAddress remoteAddress();
}

View File

@ -22,7 +22,8 @@ public final class NettyClientSingletons {
static {
NettyClientInstrumenterFactory factory =
new NettyClientInstrumenterFactory("io.opentelemetry.netty-4.0", alwaysCreateConnectSpan);
new NettyClientInstrumenterFactory(
"io.opentelemetry.netty-4.0", alwaysCreateConnectSpan, false);
INSTRUMENTER = factory.createHttpInstrumenter();
CONNECTION_INSTRUMENTER = factory.createConnectionInstrumenter();
}

View File

@ -46,10 +46,12 @@ tasks {
val testConnectionSpan by registering(Test::class) {
filter {
includeTestsMatching("Netty41ConnectionSpanTest")
includeTestsMatching("Netty41ClientSslTest")
isFailOnNoMatchingTests = false
}
include("**/Netty41ConnectionSpanTest.*")
include("**/Netty41ConnectionSpanTest.*", "**/Netty41ClientSslTest.*")
jvmArgs("-Dotel.instrumentation.netty.always-create-connect-span=true")
jvmArgs("-Dotel.instrumentation.netty.ssl-telemetry.enabled=true")
}
test {
@ -58,6 +60,7 @@ tasks {
dependsOn(testConnectionSpan)
filter {
excludeTestsMatching("Netty41ConnectionSpanTest")
excludeTestsMatching("Netty41ClientSslTest")
isFailOnNoMatchingTests = false
}
}

View File

@ -5,6 +5,7 @@
package io.opentelemetry.javaagent.instrumentation.netty.v4_1;
import static io.opentelemetry.javaagent.instrumentation.netty.v4_1.client.NettyClientSingletons.sslInstrumenter;
import static net.bytebuddy.matcher.ElementMatchers.isMethod;
import static net.bytebuddy.matcher.ElementMatchers.nameStartsWith;
import static net.bytebuddy.matcher.ElementMatchers.named;
@ -23,6 +24,7 @@ import io.opentelemetry.instrumentation.api.field.VirtualField;
import io.opentelemetry.javaagent.extension.instrumentation.TypeTransformer;
import io.opentelemetry.javaagent.instrumentation.api.CallDepth;
import io.opentelemetry.javaagent.instrumentation.netty.common.AbstractNettyChannelPipelineInstrumentation;
import io.opentelemetry.javaagent.instrumentation.netty.common.client.NettySslInstrumentationHandler;
import io.opentelemetry.javaagent.instrumentation.netty.v4_1.client.HttpClientRequestTracingHandler;
import io.opentelemetry.javaagent.instrumentation.netty.v4_1.client.HttpClientResponseTracingHandler;
import io.opentelemetry.javaagent.instrumentation.netty.v4_1.client.HttpClientTracingHandler;
@ -115,6 +117,10 @@ public class NettyChannelPipelineInstrumentation
ourHandler = new HttpClientRequestTracingHandler();
} else if (handler instanceof HttpResponseDecoder) {
ourHandler = new HttpClientResponseTracingHandler();
// the SslHandler lives in the netty-handler module, using class name comparison to avoid
// adding a dependency
} else if (handler.getClass().getName().equals("io.netty.handler.ssl.SslHandler")) {
ourHandler = new NettySslInstrumentationHandler(sslInstrumenter());
}
if (ourHandler != null) {

View File

@ -12,6 +12,7 @@ import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter;
import io.opentelemetry.javaagent.instrumentation.netty.common.HttpRequestAndChannel;
import io.opentelemetry.javaagent.instrumentation.netty.common.client.NettyClientInstrumenterFactory;
import io.opentelemetry.javaagent.instrumentation.netty.common.client.NettyConnectionInstrumenter;
import io.opentelemetry.javaagent.instrumentation.netty.common.client.NettySslInstrumenter;
public final class NettyClientSingletons {
@ -22,15 +23,20 @@ public final class NettyClientSingletons {
private static final boolean alwaysCreateConnectSpan =
Config.get().getBoolean("otel.instrumentation.netty.always-create-connect-span", false);
private static final boolean sslTelemetryEnabled =
Config.get().getBoolean("otel.instrumentation.netty.ssl-telemetry.enabled", false);
private static final Instrumenter<HttpRequestAndChannel, HttpResponse> INSTRUMENTER;
private static final NettyConnectionInstrumenter CONNECTION_INSTRUMENTER;
private static final NettySslInstrumenter SSL_INSTRUMENTER;
static {
NettyClientInstrumenterFactory factory =
new NettyClientInstrumenterFactory("io.opentelemetry.netty-4.1", alwaysCreateConnectSpan);
new NettyClientInstrumenterFactory(
"io.opentelemetry.netty-4.1", alwaysCreateConnectSpan, sslTelemetryEnabled);
INSTRUMENTER = factory.createHttpInstrumenter();
CONNECTION_INSTRUMENTER = factory.createConnectionInstrumenter();
SSL_INSTRUMENTER = factory.createSslInstrumenter();
}
public static Instrumenter<HttpRequestAndChannel, HttpResponse> instrumenter() {
@ -41,5 +47,9 @@ public final class NettyClientSingletons {
return CONNECTION_INSTRUMENTER;
}
public static NettySslInstrumenter sslInstrumenter() {
return SSL_INSTRUMENTER;
}
private NettyClientSingletons() {}
}

View File

@ -0,0 +1,225 @@
/*
* Copyright The OpenTelemetry Authors
* SPDX-License-Identifier: Apache-2.0
*/
import io.netty.bootstrap.Bootstrap
import io.netty.buffer.Unpooled
import io.netty.channel.Channel
import io.netty.channel.ChannelInitializer
import io.netty.channel.ChannelPipeline
import io.netty.channel.EventLoopGroup
import io.netty.channel.nio.NioEventLoopGroup
import io.netty.channel.socket.SocketChannel
import io.netty.channel.socket.nio.NioSocketChannel
import io.netty.handler.codec.http.DefaultFullHttpRequest
import io.netty.handler.codec.http.HttpClientCodec
import io.netty.handler.codec.http.HttpHeaderNames
import io.netty.handler.codec.http.HttpMethod
import io.netty.handler.codec.http.HttpVersion
import io.netty.handler.ssl.SslContext
import io.netty.handler.ssl.SslContextBuilder
import io.netty.handler.ssl.SslHandler
import io.opentelemetry.instrumentation.test.AgentInstrumentationSpecification
import io.opentelemetry.instrumentation.testing.junit.http.HttpClientTestServer
import io.opentelemetry.semconv.trace.attributes.SemanticAttributes
import spock.lang.Shared
import javax.net.ssl.SSLEngine
import javax.net.ssl.SSLHandshakeException
import java.util.concurrent.CompletableFuture
import java.util.concurrent.ExecutionException
import java.util.concurrent.TimeUnit
import static io.opentelemetry.api.trace.SpanKind.CLIENT
import static io.opentelemetry.api.trace.SpanKind.INTERNAL
import static io.opentelemetry.api.trace.SpanKind.SERVER
import static io.opentelemetry.api.trace.StatusCode.ERROR
import static io.opentelemetry.semconv.trace.attributes.SemanticAttributes.NetTransportValues.IP_TCP
class Netty41ClientSslTest extends AgentInstrumentationSpecification {
@Shared
HttpClientTestServer server
@Shared
EventLoopGroup eventLoopGroup
def setupSpec() {
server = new HttpClientTestServer(openTelemetry)
server.start()
eventLoopGroup = new NioEventLoopGroup()
}
def cleanupSpec() {
server.stop().get(10, TimeUnit.SECONDS)
eventLoopGroup.shutdownGracefully().sync()
}
def "should fail SSL handshake"() {
given:
def bootstrap = createBootstrap(eventLoopGroup, ["SSLv3"])
def uri = server.resolveHttpsAddress("/success")
def request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, uri.toString(), Unpooled.EMPTY_BUFFER)
request.headers().set(HttpHeaderNames.HOST, uri.host)
when:
Channel channel = null
runWithSpan("parent") {
channel = bootstrap.connect(uri.host, uri.port).sync().channel()
def result = new CompletableFuture<Integer>()
channel.pipeline().addLast(new ClientHandler(result))
channel.writeAndFlush(request).get(10, TimeUnit.SECONDS)
result.get(10, TimeUnit.SECONDS)
}
then:
Throwable thrownException = thrown()
if (thrownException instanceof ExecutionException) {
thrownException = thrownException.cause
}
assertTraces(1) {
trace(0, 4) {
span(0) {
name "parent"
status ERROR
errorEvent(thrownException.class, thrownException.message)
}
span(1) {
name "RESOLVE"
kind INTERNAL
childOf span(0)
attributes {
"${SemanticAttributes.NET_TRANSPORT.key}" IP_TCP
"${SemanticAttributes.NET_PEER_NAME.key}" uri.host
"${SemanticAttributes.NET_PEER_PORT.key}" uri.port
}
}
span(2) {
name "CONNECT"
kind INTERNAL
childOf span(0)
attributes {
"${SemanticAttributes.NET_TRANSPORT.key}" IP_TCP
"${SemanticAttributes.NET_PEER_NAME.key}" uri.host
"${SemanticAttributes.NET_PEER_PORT.key}" uri.port
"${SemanticAttributes.NET_PEER_IP.key}" { it == null || it == "127.0.0.1" }
}
}
span(3) {
name "SSL handshake"
kind INTERNAL
childOf span(0)
status ERROR
// netty swallows the exception, it doesn't make any sense to hard-code the message
errorEventWithAnyMessage(SSLHandshakeException)
attributes {
"${SemanticAttributes.NET_TRANSPORT.key}" IP_TCP
"${SemanticAttributes.NET_PEER_NAME.key}" uri.host
"${SemanticAttributes.NET_PEER_PORT.key}" uri.port
"${SemanticAttributes.NET_PEER_IP.key}" { it == null || it == "127.0.0.1" }
}
}
}
}
cleanup:
channel?.close()?.sync()
}
def "should successfully establish SSL handshake"() {
given:
def bootstrap = createBootstrap(eventLoopGroup)
def uri = server.resolveHttpsAddress("/success")
def request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, uri.toString(), Unpooled.EMPTY_BUFFER)
request.headers().set(HttpHeaderNames.HOST, uri.host)
when:
Channel channel = null
runWithSpan("parent") {
channel = bootstrap.connect(uri.host, uri.port).sync().channel()
def result = new CompletableFuture<Integer>()
channel.pipeline().addLast(new ClientHandler(result))
channel.writeAndFlush(request).get(10, TimeUnit.SECONDS)
result.get(10, TimeUnit.SECONDS)
}
then:
assertTraces(1) {
trace(0, 6) {
span(0) {
name "parent"
}
span(1) {
name "RESOLVE"
kind INTERNAL
childOf span(0)
attributes {
"${SemanticAttributes.NET_TRANSPORT.key}" IP_TCP
"${SemanticAttributes.NET_PEER_NAME.key}" uri.host
"${SemanticAttributes.NET_PEER_PORT.key}" uri.port
}
}
span(2) {
name "CONNECT"
kind INTERNAL
childOf span(0)
attributes {
"${SemanticAttributes.NET_TRANSPORT.key}" IP_TCP
"${SemanticAttributes.NET_PEER_NAME.key}" uri.host
"${SemanticAttributes.NET_PEER_PORT.key}" uri.port
"${SemanticAttributes.NET_PEER_IP.key}" { it == null || it == "127.0.0.1" }
}
}
span(3) {
name "SSL handshake"
kind INTERNAL
childOf span(0)
attributes {
"${SemanticAttributes.NET_TRANSPORT.key}" IP_TCP
"${SemanticAttributes.NET_PEER_NAME.key}" uri.host
"${SemanticAttributes.NET_PEER_PORT.key}" uri.port
"${SemanticAttributes.NET_PEER_IP.key}" { it == null || it == "127.0.0.1" }
}
}
span(4) {
name "HTTP GET"
kind CLIENT
childOf(span(0))
}
span(5) {
name "test-http-server"
kind SERVER
childOf(span(4))
}
}
}
cleanup:
channel?.close()?.sync()
}
private static Bootstrap createBootstrap(EventLoopGroup eventLoopGroup, List<String> enabledProtocols = null) {
def bootstrap = new Bootstrap()
bootstrap.group(eventLoopGroup)
.channel(NioSocketChannel)
.handler(new ChannelInitializer<SocketChannel>() {
@Override
protected void initChannel(SocketChannel socketChannel) throws Exception {
ChannelPipeline pipeline = socketChannel.pipeline()
SslContext sslContext = SslContextBuilder.forClient().build()
SSLEngine sslEngine = sslContext.newEngine(socketChannel.alloc())
if (enabledProtocols != null) {
sslEngine.setEnabledProtocols(enabledProtocols as String[])
}
pipeline.addLast(new SslHandler(sslEngine))
pipeline.addLast(new HttpClientCodec())
}
})
bootstrap
}
}

View File

@ -20,7 +20,7 @@ public final class ReactorNettySingletons {
static {
NettyClientInstrumenterFactory instrumenterFactory =
new NettyClientInstrumenterFactory(
"io.opentelemetry.reactor-netty-1.0", alwaysCreateConnectSpan);
"io.opentelemetry.reactor-netty-1.0", alwaysCreateConnectSpan, false);
CONNECTION_INSTRUMENTER = instrumenterFactory.createConnectionInstrumenter();
}

View File

@ -135,6 +135,17 @@ class SpanAssert {
errorEvent(expectedClass, null)
}
def errorEventWithAnyMessage(Class<Throwable> expectedClass) {
event(0) {
eventName(SemanticAttributes.EXCEPTION_EVENT_NAME)
attributes {
"${SemanticAttributes.EXCEPTION_TYPE.key}" expectedClass.canonicalName
"${SemanticAttributes.EXCEPTION_STACKTRACE.key}" String
"${SemanticAttributes.EXCEPTION_MESSAGE.key}" { it != null }
}
}
}
def errorEvent(Class<Throwable> expectedClass, expectedMessage) {
errorEvent(expectedClass, expectedMessage, 0)
}

View File

@ -116,7 +116,11 @@ public final class HttpClientTestServer extends ServerExtension {
.decorator(LoggingService.newDecorator());
}
URI resolveAddress(String path) {
public URI resolveAddress(String path) {
return URI.create("http://localhost:" + httpPort() + path);
}
public URI resolveHttpsAddress(String path) {
return URI.create("https://localhost:" + httpsPort() + path);
}
}