open-telemetry
/
opentelemetry-java-instrumentation
mirror of https://github.com/open-telemetry/opentelemetry-java-instrumentation.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Enhance AWS SDK Instrumentation with Detailed HTTP Error Information (#9448)

This commit is contained in:
Ping Xiang 2023-09-29 09:16:11 -07:00 committed by GitHub
parent 87b2ae09c1
commit 2724a87743
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 368 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
instrumentation/aws-sdk/README.md
View File

@ -5,7 +5,8 @@ For more information, see the respective public setters in the `AwsSdkTelemetryB
- [SDK v1](./aws-sdk-1.11/library/src/main/java/io/opentelemetry/instrumentation/awssdk/v1_11/AwsSdkTelemetryBuilder.java)
- [SDK v2](./aws-sdk-2.2/library/src/main/java/io/opentelemetry/instrumentation/awssdk/v2_2/AwsSdkTelemetryBuilder.java)
| System property | Type | Default | Description |
| ------------------------------------------------------------------------ | ------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------- |
| `otel.instrumentation.aws-sdk.experimental-span-attributes` | Boolean | `false` | Enable the capture of experimental span attributes. |
| System property | Type | Default | Description |
|--------------------------------------------------------------------------| ------- | ------- |---------------------------------------------------------------------------------------------|
| `otel.instrumentation.aws-sdk.experimental-span-attributes` | Boolean | `false` | Enable the capture of experimental span attributes. |
| `otel.instrumentation.aws-sdk.experimental-use-propagator-for-messaging` | Boolean | `false` | v2 only, inject into SNS/SQS attributes with configured propagator: See [v2 README](aws-sdk-2.2/library/README.md#trace-propagation). |
| `otel.instrumentation.aws-sdk.experimental-record-individual-http-error` | Boolean | `false` | v2 only, record errors returned by each individual HTTP request as events for the SDK span. |

3
instrumentation/aws-sdk/aws-sdk-2.2/javaagent/build.gradle.kts
View File

@ -10,6 +10,7 @@ muzzle {
// Used by all SDK services, the only case it isn't is an SDK extension such as a custom HTTP
// client, which is not target of instrumentation anyways.
extraDependency("software.amazon.awssdk:protocol-core")
excludeInstrumentationName("aws-sdk-2.2-sqs")
excludeInstrumentationName("aws-sdk-2.2-sns")
@ -95,6 +96,7 @@ testing {
} else {
implementation("software.amazon.awssdk:s3:2.10.12")
}
implementation(project(":instrumentation:aws-sdk:aws-sdk-2.2:library"))
}
}
}
@ -115,6 +117,7 @@ tasks {
withType<Test>().configureEach {
// TODO run tests both with and without experimental span attributes
systemProperty("otel.instrumentation.aws-sdk.experimental-span-attributes", "true")
systemProperty("otel.instrumentation.aws-sdk.experimental-record-individual-http-error", "true")
}
withType<com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar>().configureEach {

25
instrumentation/aws-sdk/aws-sdk-2.2/javaagent/src/test/java/Aws2ClientRecordHttpErrorTest.java Normal file
View File

@ -0,0 +1,25 @@
/*
* Copyright The OpenTelemetry Authors
* SPDX-License-Identifier: Apache-2.0
*/
import io.opentelemetry.instrumentation.awssdk.v2_2.AbstractAws2ClientRecordHttpErrorTest;
import io.opentelemetry.instrumentation.testing.junit.AgentInstrumentationExtension;
import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension;
import org.junit.jupiter.api.extension.RegisterExtension;
import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
public class Aws2ClientRecordHttpErrorTest extends AbstractAws2ClientRecordHttpErrorTest {
@RegisterExtension
private final AgentInstrumentationExtension testing = AgentInstrumentationExtension.create();
@Override
public ClientOverrideConfiguration.Builder createOverrideConfigurationBuilder() {
return ClientOverrideConfiguration.builder();
}
@Override
protected InstrumentationExtension getTesting() {
return testing;
}
}

1
instrumentation/aws-sdk/aws-sdk-2.2/library-autoconfigure/build.gradle.kts
View File

@ -25,5 +25,6 @@ tasks {
test {
systemProperty("otel.instrumentation.aws-sdk.experimental-span-attributes", true)
systemProperty("otel.instrumentation.aws-sdk.experimental-use-propagator-for-messaging", true)
systemProperty("otel.instrumentation.aws-sdk.experimental-record-individual-http-error", true)
}
}

5
instrumentation/aws-sdk/aws-sdk-2.2/library-autoconfigure/src/main/java/io/opentelemetry/instrumentation/awssdk/v2_2/autoconfigure/TracingExecutionInterceptor.java
View File

@ -36,10 +36,15 @@ public class TracingExecutionInterceptor implements ExecutionInterceptor {
ConfigPropertiesUtil.getBoolean(
"otel.instrumentation.aws-sdk.experimental-use-propagator-for-messaging", false);
private static final boolean RECORD_INDIVIDUAL_HTTP_ERROR =
ConfigPropertiesUtil.getBoolean(
"otel.instrumentation.aws-sdk.experimental-record-individual-http-error", false);
private final ExecutionInterceptor delegate =
AwsSdkTelemetry.builder(GlobalOpenTelemetry.get())
.setCaptureExperimentalSpanAttributes(CAPTURE_EXPERIMENTAL_SPAN_ATTRIBUTES)
.setUseConfiguredPropagatorForMessaging(USE_MESSAGING_PROPAGATOR)
.setRecordIndividualHttpError(RECORD_INDIVIDUAL_HTTP_ERROR)
.build()
.newExecutionInterceptor();

8
instrumentation/aws-sdk/aws-sdk-2.2/library/src/main/java/io/opentelemetry/instrumentation/awssdk/v2_2/AwsSdkTelemetry.java
View File

@ -46,12 +46,14 @@ public class AwsSdkTelemetry {
private final boolean captureExperimentalSpanAttributes;
@Nullable private final TextMapPropagator messagingPropagator;
private final boolean useXrayPropagator;
private final boolean recordIndividualHttpError;
AwsSdkTelemetry(
OpenTelemetry openTelemetry,
boolean captureExperimentalSpanAttributes,
boolean useMessagingPropagator,
boolean useXrayPropagator) {
boolean useXrayPropagator,
boolean recordIndividualHttpError) {
this.useXrayPropagator = useXrayPropagator;
this.requestInstrumenter =
AwsSdkInstrumenterFactory.requestInstrumenter(
@ -62,6 +64,7 @@ public class AwsSdkTelemetry {
this.captureExperimentalSpanAttributes = captureExperimentalSpanAttributes;
this.messagingPropagator =
useMessagingPropagator ? openTelemetry.getPropagators().getTextMapPropagator() : null;
this.recordIndividualHttpError = recordIndividualHttpError;
}
/**
@ -74,6 +77,7 @@ public class AwsSdkTelemetry {
consumerInstrumenter,
captureExperimentalSpanAttributes,
messagingPropagator,
useXrayPropagator);
useXrayPropagator,
recordIndividualHttpError);
}
}

19
instrumentation/aws-sdk/aws-sdk-2.2/library/src/main/java/io/opentelemetry/instrumentation/awssdk/v2_2/AwsSdkTelemetryBuilder.java
View File

@ -17,6 +17,8 @@ public final class AwsSdkTelemetryBuilder {
private boolean useMessagingPropagator;
private boolean recordIndividualHttpError;
private boolean useXrayPropagator = true;
AwsSdkTelemetryBuilder(OpenTelemetry openTelemetry) {
@ -57,6 +59,20 @@ public final class AwsSdkTelemetryBuilder {
return this;
}
/**
* Sets whether errors returned by each individual HTTP request should be recorded as events for
* the SDK span.
*
* <p>This option is off by default. If enabled, the HTTP error code and the error message will be
* captured and associated with the span. This provides detailed insights into errors on a
* per-request basis.
*/
@CanIgnoreReturnValue
public AwsSdkTelemetryBuilder setRecordIndividualHttpError(boolean recordIndividualHttpError) {
this.recordIndividualHttpError = recordIndividualHttpError;
return this;
}
/**
* This setter implemented package-private for testing the messaging propagator, it does not seem
* too useful in general. The option is on by default.
@ -79,6 +95,7 @@ public final class AwsSdkTelemetryBuilder {
openTelemetry,
captureExperimentalSpanAttributes,
useMessagingPropagator,
useXrayPropagator);
useXrayPropagator,
recordIndividualHttpError);
}
}

61
instrumentation/aws-sdk/aws-sdk-2.2/library/src/main/java/io/opentelemetry/instrumentation/awssdk/v2_2/TracingExecutionInterceptor.java
View File

@ -7,6 +7,7 @@ package io.opentelemetry.instrumentation.awssdk.v2_2;
import static io.opentelemetry.instrumentation.awssdk.v2_2.AwsSdkRequestType.DYNAMODB;
import io.opentelemetry.api.common.AttributeKey;
import io.opentelemetry.api.common.Attributes;
import io.opentelemetry.api.common.AttributesBuilder;
import io.opentelemetry.api.trace.Span;
@ -15,7 +16,14 @@ import io.opentelemetry.context.propagation.TextMapPropagator;
import io.opentelemetry.contrib.awsxray.propagator.AwsXrayPropagator;
import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter;
import io.opentelemetry.semconv.SemanticAttributes;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
import software.amazon.awssdk.auth.signer.AwsSignerExecutionAttribute;
import software.amazon.awssdk.awscore.AwsResponse;
@ -50,6 +58,10 @@ final class TracingExecutionInterceptor implements ExecutionInterceptor {
private final Instrumenter<ExecutionAttributes, SdkHttpResponse> consumerInstrumenter;
private final boolean captureExperimentalSpanAttributes;
static final AttributeKey<String> HTTP_ERROR_MSG =
AttributeKey.stringKey("aws.http.error_message");
static final String HTTP_FAILURE_EVENT = "HTTP request failure";
Instrumenter<ExecutionAttributes, SdkHttpResponse> getConsumerInstrumenter() {
return consumerInstrumenter;
}
@ -65,6 +77,7 @@ final class TracingExecutionInterceptor implements ExecutionInterceptor {
@Nullable private final TextMapPropagator messagingPropagator;
private final boolean useXrayPropagator;
private final boolean recordIndividualHttpError;
private final FieldMapper fieldMapper;
TracingExecutionInterceptor(
@ -72,12 +85,14 @@ final class TracingExecutionInterceptor implements ExecutionInterceptor {
Instrumenter<ExecutionAttributes, SdkHttpResponse> consumerInstrumenter,
boolean captureExperimentalSpanAttributes,
TextMapPropagator messagingPropagator,
boolean useXrayPropagator) {
boolean useXrayPropagator,
boolean recordIndividualHttpError) {
this.requestInstrumenter = requestInstrumenter;
this.consumerInstrumenter = consumerInstrumenter;
this.captureExperimentalSpanAttributes = captureExperimentalSpanAttributes;
this.messagingPropagator = messagingPropagator;
this.useXrayPropagator = useXrayPropagator;
this.recordIndividualHttpError = recordIndividualHttpError;
this.fieldMapper = new FieldMapper();
}
@ -222,6 +237,19 @@ final class TracingExecutionInterceptor implements ExecutionInterceptor {
return builder.build();
}
@Override
public Optional<InputStream> modifyHttpResponseContent(
Context.ModifyHttpResponse context, ExecutionAttributes executionAttributes) {
Optional<InputStream> responseBody = context.responseBody();
if (recordIndividualHttpError) {
String errorMsg = extractHttpErrorAsEvent(context, executionAttributes);
if (errorMsg != null) {
return Optional.of(new ByteArrayInputStream(errorMsg.getBytes(Charset.defaultCharset())));
}
}
return responseBody;
}
private void populateRequestAttributes(
Span span,
AwsSdkRequest awsSdkRequest,
@ -289,6 +317,37 @@ final class TracingExecutionInterceptor implements ExecutionInterceptor {
}
}
private static String extractHttpErrorAsEvent(
Context.AfterTransmission context, ExecutionAttributes executionAttributes) {
io.opentelemetry.context.Context otelContext = getContext(executionAttributes);
if (otelContext != null) {
Span span = Span.fromContext(otelContext);
SdkHttpResponse response = context.httpResponse();
if (response != null && !response.isSuccessful()) {
int errorCode = response.statusCode();
// we want to record the error message from http response
Optional<InputStream> responseBody = context.responseBody();
if (responseBody.isPresent()) {
String errorMsg =
new BufferedReader(
new InputStreamReader(responseBody.get(), Charset.defaultCharset()))
.lines()
.collect(Collectors.joining("\n"));
Attributes attributes =
Attributes.of(
SemanticAttributes.HTTP_RESPONSE_STATUS_CODE,
Long.valueOf(errorCode),
HTTP_ERROR_MSG,
errorMsg);
span.addEvent(HTTP_FAILURE_EVENT, attributes);
return errorMsg;
}
}
}
return null;
}
@Override
public void onExecutionFailure(
Context.FailedExecution context, ExecutionAttributes executionAttributes) {

37
instrumentation/aws-sdk/aws-sdk-2.2/library/src/test/java/io/opentelemetry/instrumentation/awssdk/v2_2/Aws2ClientNotRecordHttpErrorTest.java Normal file
View File

@ -0,0 +1,37 @@
/*
* Copyright The OpenTelemetry Authors
* SPDX-License-Identifier: Apache-2.0
*/
package io.opentelemetry.instrumentation.awssdk.v2_2;
import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension;
import io.opentelemetry.instrumentation.testing.junit.LibraryInstrumentationExtension;
import org.junit.jupiter.api.extension.RegisterExtension;
import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
public class Aws2ClientNotRecordHttpErrorTest extends AbstractAws2ClientRecordHttpErrorTest {
@RegisterExtension
public final LibraryInstrumentationExtension testing = LibraryInstrumentationExtension.create();
@Override
public ClientOverrideConfiguration.Builder createOverrideConfigurationBuilder() {
return ClientOverrideConfiguration.builder()
.addExecutionInterceptor(
AwsSdkTelemetry.builder(testing.getOpenTelemetry())
.setCaptureExperimentalSpanAttributes(true)
.setRecordIndividualHttpError(isRecordIndividualHttpErrorEnabled())
.build()
.newExecutionInterceptor());
}
@Override
public boolean isRecordIndividualHttpErrorEnabled() {
return false;
}
@Override
protected InstrumentationExtension getTesting() {
return testing;
}
}

209
instrumentation/aws-sdk/aws-sdk-2.2/testing/src/main/java/io/opentelemetry/instrumentation/awssdk/v2_2/AbstractAws2ClientRecordHttpErrorTest.java Normal file
View File

@ -0,0 +1,209 @@
/*
* Copyright The OpenTelemetry Authors
* SPDX-License-Identifier: Apache-2.0
*/
package io.opentelemetry.instrumentation.awssdk.v2_2;
import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.assertThat;
import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.equalTo;
import io.opentelemetry.api.common.AttributeKey;
import io.opentelemetry.api.trace.SpanKind;
import io.opentelemetry.instrumentation.api.internal.ConfigPropertiesUtil;
import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension;
import io.opentelemetry.semconv.SemanticAttributes;
import io.opentelemetry.testing.internal.armeria.common.HttpResponse;
import io.opentelemetry.testing.internal.armeria.common.HttpStatus;
import io.opentelemetry.testing.internal.armeria.common.MediaType;
import io.opentelemetry.testing.internal.armeria.testing.junit5.server.mock.MockWebServerExtension;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInstance;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
import software.amazon.awssdk.core.interceptor.Context;
import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
import software.amazon.awssdk.http.SdkHttpResponse;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.dynamodb.DynamoDbClientBuilder;
import software.amazon.awssdk.services.dynamodb.model.PutItemRequest;
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
public abstract class AbstractAws2ClientRecordHttpErrorTest {
private static final StaticCredentialsProvider CREDENTIALS_PROVIDER =
StaticCredentialsProvider.create(
AwsBasicCredentials.create("my-access-key", "my-secret-key"));
private static final MockWebServerExtension server = new MockWebServerExtension();
protected static List<String> httpErrorMessages = new ArrayList<>();
@BeforeAll
public static void setupSpec() {
server.start();
}
public static void cleanupSpec() {
server.stop();
}
public abstract ClientOverrideConfiguration.Builder createOverrideConfigurationBuilder();
protected abstract InstrumentationExtension getTesting();
// Introducing a new ExecutionInterceptor that's registered with the AWS SDK.
// It's positioned to execute after the TracingExecutionInterceptor used for SDK instrumentation.
// The purpose of this interceptor is to inspect the response body of failed HTTP requests.
// We aim to ensure that the HTTP error message remains accessible in the response body's
// InputStream
// even after the TracingExecutionInterceptor has processed it.
static class ResponseCheckInterceptor implements ExecutionInterceptor {
@Override
public Optional<InputStream> modifyHttpResponseContent(
Context.ModifyHttpResponse context, ExecutionAttributes executionAttributes) {
Optional<InputStream> responseBody = context.responseBody();
String errorMsg = extractHttpErrorMessage(context, executionAttributes);
if (errorMsg != null) {
return Optional.of(new ByteArrayInputStream(errorMsg.getBytes(Charset.defaultCharset())));
}
return responseBody;
}
private static String extractHttpErrorMessage(
Context.AfterTransmission context, ExecutionAttributes executionAttributes) {
SdkHttpResponse response = context.httpResponse();
if (executionAttributes == null) {
return "";
}
if (response != null && !response.isSuccessful()) {
Optional<InputStream> responseBody = context.responseBody();
if (responseBody.isPresent()) {
String errorMsg =
new BufferedReader(
new InputStreamReader(responseBody.get(), Charset.defaultCharset()))
.lines()
.collect(Collectors.joining("\n"));
httpErrorMessages.add(errorMsg);
return errorMsg;
}
}
return null;
}
}
private static void cleanResponses() {
httpErrorMessages.clear();
}
public boolean isRecordIndividualHttpErrorEnabled() {
// See io.opentelemetry.instrumentation.awssdk.v2_2.autoconfigure.TracingExecutionInterceptor
return ConfigPropertiesUtil.getBoolean(
"otel.instrumentation.aws-sdk.experimental-record-individual-http-error", false);
}
@Test
// Suppressing deprecation because we use some deprecated attributes in the test
@SuppressWarnings("deprecation")
public void testSendDynamoDbRequestWithRetries() {
cleanResponses();
// Setup and configuration
String service = "DynamoDb";
String operation = "PutItem";
String method = "POST";
String requestId = "UNKNOWN";
DynamoDbClientBuilder builder = DynamoDbClient.builder();
ClientOverrideConfiguration.Builder overrideConfigBuilder =
createOverrideConfigurationBuilder()
.addExecutionInterceptor(new ResponseCheckInterceptor());
builder.overrideConfiguration(overrideConfigBuilder.build());
DynamoDbClient client =
builder
.endpointOverride(server.httpUri())
.region(Region.AP_NORTHEAST_1)
.credentialsProvider(CREDENTIALS_PROVIDER)
.build();
// Mocking server responses
server.enqueue(
HttpResponse.of(
HttpStatus.INTERNAL_SERVER_ERROR,
MediaType.PLAIN_TEXT_UTF_8,
"DynamoDB could not process your request"));
server.enqueue(
HttpResponse.of(
HttpStatus.SERVICE_UNAVAILABLE,
MediaType.PLAIN_TEXT_UTF_8,
"DynamoDB is currently unavailable"));
server.enqueue(HttpResponse.of(HttpStatus.OK, MediaType.PLAIN_TEXT_UTF_8, ""));
// Making the call
client.putItem(PutItemRequest.builder().tableName("sometable").build());
getTesting()
.waitAndAssertTraces(
trace -> {
trace.hasSpansSatisfyingExactly(
span -> {
span.hasKind(SpanKind.CLIENT);
span.hasNoParent();
span.hasAttributesSatisfying(
attributes -> {
assertThat(attributes)
.containsEntry(SemanticAttributes.NET_PEER_NAME, "127.0.0.1")
.containsEntry(SemanticAttributes.NET_PEER_PORT, server.httpPort())
.containsEntry(SemanticAttributes.HTTP_METHOD, method)
.containsEntry(SemanticAttributes.HTTP_STATUS_CODE, 200)
.containsEntry(SemanticAttributes.RPC_SYSTEM, "aws-api")
.containsEntry(SemanticAttributes.RPC_SERVICE, service)
.containsEntry(SemanticAttributes.RPC_METHOD, operation)
.containsEntry("aws.agent", "java-aws-sdk")
.containsEntry("aws.requestId", requestId)
.containsEntry("aws.table.name", "sometable")
.containsEntry(SemanticAttributes.DB_SYSTEM, "dynamodb")
.containsEntry(SemanticAttributes.DB_OPERATION, operation);
});
if (isRecordIndividualHttpErrorEnabled()) {
span.hasEventsSatisfyingExactly(
event ->
event
.hasName("HTTP request failure")
.hasAttributesSatisfyingExactly(
equalTo(SemanticAttributes.HTTP_RESPONSE_STATUS_CODE, 500),
equalTo(
AttributeKey.stringKey("aws.http.error_message"),
"DynamoDB could not process your request")),
event ->
event
.hasName("HTTP request failure")
.hasAttributesSatisfyingExactly(
equalTo(SemanticAttributes.HTTP_RESPONSE_STATUS_CODE, 503),
equalTo(
AttributeKey.stringKey("aws.http.error_message"),
"DynamoDB is currently unavailable")));
} else {
span.hasEventsSatisfying(events -> assertThat(events.size()).isEqualTo(0));
}
});
});
// make sure the response body input stream is still available and check its content to be
// expected
assertThat(httpErrorMessages.size()).isEqualTo(2);
assertThat(httpErrorMessages.get(0)).isEqualTo("DynamoDB could not process your request");
assertThat(httpErrorMessages.get(1)).isEqualTo("DynamoDB is currently unavailable");
}
}