Record internal metric for SQL cache misses (#2747)

* Record internal metric for SQL cache misses

And use `SupportabilityMetrics` in `Instrumenter`

* Fix broken shouldStart() logic

* Code review comments
This commit is contained in:
Mateusz Rzeszutek 2021-04-08 17:50:48 +02:00 committed by GitHub
parent c943c5da93
commit 42cf3bc077
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 95 additions and 36 deletions

View File

@ -6,17 +6,17 @@
package io.opentelemetry.instrumentation.api.db; package io.opentelemetry.instrumentation.api.db;
import static io.opentelemetry.instrumentation.api.db.StatementSanitizationConfig.isStatementSanitizationEnabled; import static io.opentelemetry.instrumentation.api.db.StatementSanitizationConfig.isStatementSanitizationEnabled;
import static io.opentelemetry.instrumentation.api.internal.SupportabilityMetrics.CounterNames.SQL_STATEMENT_SANITIZER_CACHE_MISS;
import io.opentelemetry.instrumentation.api.caching.Cache; import io.opentelemetry.instrumentation.api.caching.Cache;
import org.slf4j.Logger; import io.opentelemetry.instrumentation.api.internal.SupportabilityMetrics;
import org.slf4j.LoggerFactory;
/** /**
* This class is responsible for masking potentially sensitive parameters in SQL (and SQL-like) * This class is responsible for masking potentially sensitive parameters in SQL (and SQL-like)
* statements and queries. * statements and queries.
*/ */
public final class SqlStatementSanitizer { public final class SqlStatementSanitizer {
private static final Logger log = LoggerFactory.getLogger(SqlStatementSanitizer.class); private static final SupportabilityMetrics supportability = SupportabilityMetrics.instance();
private static final Cache<String, SqlStatementInfo> sqlToStatementInfoCache = private static final Cache<String, SqlStatementInfo> sqlToStatementInfoCache =
Cache.newBuilder().setMaximumSize(1000).build(); Cache.newBuilder().setMaximumSize(1000).build();
@ -28,7 +28,7 @@ public final class SqlStatementSanitizer {
return sqlToStatementInfoCache.computeIfAbsent( return sqlToStatementInfoCache.computeIfAbsent(
statement, statement,
k -> { k -> {
log.trace("SQL statement cache miss"); supportability.incrementCounter(SQL_STATEMENT_SANITIZER_CACHE_MISS);
return AutoSqlSanitizer.sanitize(statement); return AutoSqlSanitizer.sanitize(statement);
}); });
} }

View File

@ -17,6 +17,7 @@ final class ClientInstrumenter<REQUEST, RESPONSE> extends Instrumenter<REQUEST,
private final TextMapSetter<REQUEST> setter; private final TextMapSetter<REQUEST> setter;
ClientInstrumenter( ClientInstrumenter(
String instrumentationName,
Tracer tracer, Tracer tracer,
SpanNameExtractor<? super REQUEST> spanNameExtractor, SpanNameExtractor<? super REQUEST> spanNameExtractor,
SpanKindExtractor<? super REQUEST> spanKindExtractor, SpanKindExtractor<? super REQUEST> spanKindExtractor,
@ -26,6 +27,7 @@ final class ClientInstrumenter<REQUEST, RESPONSE> extends Instrumenter<REQUEST,
ContextPropagators propagators, ContextPropagators propagators,
TextMapSetter<REQUEST> setter) { TextMapSetter<REQUEST> setter) {
super( super(
instrumentationName,
tracer, tracer,
spanNameExtractor, spanNameExtractor,
spanKindExtractor, spanKindExtractor,

View File

@ -14,6 +14,7 @@ import io.opentelemetry.api.trace.SpanBuilder;
import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.api.trace.SpanKind;
import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.api.trace.Tracer;
import io.opentelemetry.context.Context; import io.opentelemetry.context.Context;
import io.opentelemetry.instrumentation.api.internal.SupportabilityMetrics;
import io.opentelemetry.instrumentation.api.tracer.ClientSpan; import io.opentelemetry.instrumentation.api.tracer.ClientSpan;
import io.opentelemetry.instrumentation.api.tracer.ServerSpan; import io.opentelemetry.instrumentation.api.tracer.ServerSpan;
import java.util.List; import java.util.List;
@ -46,6 +47,9 @@ public class Instrumenter<REQUEST, RESPONSE> {
return new InstrumenterBuilder<>(openTelemetry, instrumentationName, spanNameExtractor); return new InstrumenterBuilder<>(openTelemetry, instrumentationName, spanNameExtractor);
} }
private static final SupportabilityMetrics supportability = SupportabilityMetrics.instance();
private final String instrumentationName;
private final Tracer tracer; private final Tracer tracer;
private final SpanNameExtractor<? super REQUEST> spanNameExtractor; private final SpanNameExtractor<? super REQUEST> spanNameExtractor;
private final SpanKindExtractor<? super REQUEST> spanKindExtractor; private final SpanKindExtractor<? super REQUEST> spanKindExtractor;
@ -54,12 +58,14 @@ public class Instrumenter<REQUEST, RESPONSE> {
private final ErrorCauseExtractor errorCauseExtractor; private final ErrorCauseExtractor errorCauseExtractor;
Instrumenter( Instrumenter(
String instrumentationName,
Tracer tracer, Tracer tracer,
SpanNameExtractor<? super REQUEST> spanNameExtractor, SpanNameExtractor<? super REQUEST> spanNameExtractor,
SpanKindExtractor<? super REQUEST> spanKindExtractor, SpanKindExtractor<? super REQUEST> spanKindExtractor,
SpanStatusExtractor<? super REQUEST, ? super RESPONSE> spanStatusExtractor, SpanStatusExtractor<? super REQUEST, ? super RESPONSE> spanStatusExtractor,
List<? extends AttributesExtractor<? super REQUEST, ? super RESPONSE>> extractors, List<? extends AttributesExtractor<? super REQUEST, ? super RESPONSE>> extractors,
ErrorCauseExtractor errorCauseExtractor) { ErrorCauseExtractor errorCauseExtractor) {
this.instrumentationName = instrumentationName;
this.tracer = tracer; this.tracer = tracer;
this.spanNameExtractor = spanNameExtractor; this.spanNameExtractor = spanNameExtractor;
this.spanKindExtractor = spanKindExtractor; this.spanKindExtractor = spanKindExtractor;
@ -70,21 +76,27 @@ public class Instrumenter<REQUEST, RESPONSE> {
/** /**
* Returns whether instrumentation should be applied for the {@link REQUEST}. If {@code true}, * Returns whether instrumentation should be applied for the {@link REQUEST}. If {@code true},
* call {@link #start(Context, Object)} and {@link #end(Context, Object, Object, Throwable)} arond * call {@link #start(Context, Object)} and {@link #end(Context, Object, Object, Throwable)}
* the operation being instrumented, or if {@code false} execute the operation directly without * around the operation being instrumented, or if {@code false} execute the operation directly
* calling those methods. * without calling those methods.
*/ */
public boolean shouldStart(Context parentContext, REQUEST request) { public boolean shouldStart(Context parentContext, REQUEST request) {
boolean suppressed = false;
SpanKind spanKind = spanKindExtractor.extract(request); SpanKind spanKind = spanKindExtractor.extract(request);
switch (spanKind) { switch (spanKind) {
case SERVER: case SERVER:
return ServerSpan.fromContextOrNull(parentContext) == null; suppressed = ServerSpan.fromContextOrNull(parentContext) != null;
break;
case CLIENT: case CLIENT:
return ClientSpan.fromContextOrNull(parentContext) == null; suppressed = ClientSpan.fromContextOrNull(parentContext) != null;
break;
default: default:
// fallthrough break;
} }
return true; if (suppressed) {
supportability.recordSuppressedSpan(spanKind, instrumentationName);
}
return !suppressed;
} }
/** /**

View File

@ -114,6 +114,7 @@ public final class InstrumenterBuilder<REQUEST, RESPONSE> {
InstrumenterConstructor<REQUEST, RESPONSE> constructor, InstrumenterConstructor<REQUEST, RESPONSE> constructor,
SpanKindExtractor<? super REQUEST> spanKindExtractor) { SpanKindExtractor<? super REQUEST> spanKindExtractor) {
return constructor.create( return constructor.create(
instrumentationName,
openTelemetry.getTracer(instrumentationName, InstrumentationVersion.VERSION), openTelemetry.getTracer(instrumentationName, InstrumentationVersion.VERSION),
spanNameExtractor, spanNameExtractor,
spanKindExtractor, spanKindExtractor,
@ -124,6 +125,7 @@ public final class InstrumenterBuilder<REQUEST, RESPONSE> {
private interface InstrumenterConstructor<RQ, RS> { private interface InstrumenterConstructor<RQ, RS> {
Instrumenter<RQ, RS> create( Instrumenter<RQ, RS> create(
String instrumentationName,
Tracer tracer, Tracer tracer,
SpanNameExtractor<? super RQ> spanNameExtractor, SpanNameExtractor<? super RQ> spanNameExtractor,
SpanKindExtractor<? super RQ> spanKindExtractor, SpanKindExtractor<? super RQ> spanKindExtractor,
@ -137,8 +139,15 @@ public final class InstrumenterBuilder<REQUEST, RESPONSE> {
static <RQ, RS> InstrumenterConstructor<RQ, RS> propagatingToDownstream( static <RQ, RS> InstrumenterConstructor<RQ, RS> propagatingToDownstream(
ContextPropagators propagators, TextMapSetter<RQ> setter) { ContextPropagators propagators, TextMapSetter<RQ> setter) {
return (tracer, spanName, spanKind, spanStatus, attributes, errorCauseExtractor) -> return (instrumentationName,
tracer,
spanName,
spanKind,
spanStatus,
attributes,
errorCauseExtractor) ->
new ClientInstrumenter<>( new ClientInstrumenter<>(
instrumentationName,
tracer, tracer,
spanName, spanName,
spanKind, spanKind,
@ -151,8 +160,15 @@ public final class InstrumenterBuilder<REQUEST, RESPONSE> {
static <RQ, RS> InstrumenterConstructor<RQ, RS> propagatingFromUpstream( static <RQ, RS> InstrumenterConstructor<RQ, RS> propagatingFromUpstream(
ContextPropagators propagators, TextMapGetter<RQ> getter) { ContextPropagators propagators, TextMapGetter<RQ> getter) {
return (tracer, spanName, spanKind, spanStatus, attributes, errorCauseExtractor) -> return (instrumentationName,
tracer,
spanName,
spanKind,
spanStatus,
attributes,
errorCauseExtractor) ->
new ServerInstrumenter<>( new ServerInstrumenter<>(
instrumentationName,
tracer, tracer,
spanName, spanName,
spanKind, spanKind,

View File

@ -17,6 +17,7 @@ final class ServerInstrumenter<REQUEST, RESPONSE> extends Instrumenter<REQUEST,
private final TextMapGetter<REQUEST> getter; private final TextMapGetter<REQUEST> getter;
ServerInstrumenter( ServerInstrumenter(
String instrumentationName,
Tracer tracer, Tracer tracer,
SpanNameExtractor<? super REQUEST> spanNameExtractor, SpanNameExtractor<? super REQUEST> spanNameExtractor,
SpanKindExtractor<? super REQUEST> spanKindExtractor, SpanKindExtractor<? super REQUEST> spanKindExtractor,
@ -26,6 +27,7 @@ final class ServerInstrumenter<REQUEST, RESPONSE> extends Instrumenter<REQUEST,
ContextPropagators propagators, ContextPropagators propagators,
TextMapGetter<REQUEST> getter) { TextMapGetter<REQUEST> getter) {
super( super(
instrumentationName,
tracer, tracer,
spanNameExtractor, spanNameExtractor,
spanKindExtractor, spanKindExtractor,

View File

@ -3,7 +3,7 @@
* SPDX-License-Identifier: Apache-2.0 * SPDX-License-Identifier: Apache-2.0
*/ */
package io.opentelemetry.instrumentation.api.tracer; package io.opentelemetry.instrumentation.api.internal;
import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.api.trace.SpanKind;
import io.opentelemetry.instrumentation.api.config.Config; import io.opentelemetry.instrumentation.api.config.Config;
@ -16,15 +16,19 @@ import java.util.function.Consumer;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
class SupportabilityMetrics { public final class SupportabilityMetrics {
private static final Logger log = LoggerFactory.getLogger(SupportabilityMetrics.class); private static final Logger log = LoggerFactory.getLogger(SupportabilityMetrics.class);
private final boolean agentDebugEnabled; private final boolean agentDebugEnabled;
private final Consumer<String> reporter; private final Consumer<String> reporter;
private final ConcurrentMap<String, KindCounters> suppressionCounters = new ConcurrentHashMap<>(); private final ConcurrentMap<String, KindCounters> suppressionCounters = new ConcurrentHashMap<>();
private final ConcurrentMap<String, LongAdder> counters = new ConcurrentHashMap<>();
public SupportabilityMetrics(Config config) { private static final SupportabilityMetrics INSTANCE =
this(config, log::debug); new SupportabilityMetrics(Config.get(), log::debug).start();
public static SupportabilityMetrics instance() {
return INSTANCE;
} }
// visible for testing // visible for testing
@ -33,7 +37,7 @@ class SupportabilityMetrics {
this.reporter = reporter; this.reporter = reporter;
} }
void recordSuppressedSpan(SpanKind kind, String instrumentationName) { public void recordSuppressedSpan(SpanKind kind, String instrumentationName) {
if (!agentDebugEnabled) { if (!agentDebugEnabled) {
return; return;
} }
@ -43,6 +47,14 @@ class SupportabilityMetrics {
.increment(kind); .increment(kind);
} }
public void incrementCounter(String counterName) {
if (!agentDebugEnabled) {
return;
}
counters.computeIfAbsent(counterName, k -> new LongAdder()).increment();
}
// visible for testing // visible for testing
void report() { void report() {
suppressionCounters.forEach( suppressionCounters.forEach(
@ -55,6 +67,13 @@ class SupportabilityMetrics {
} }
} }
}); });
counters.forEach(
(counterName, counter) -> {
long value = counter.sumThenReset();
if (value > 0) {
reporter.accept("Counter '" + counterName + "' : " + value);
}
});
} }
SupportabilityMetrics start() { SupportabilityMetrics start() {
@ -72,6 +91,13 @@ class SupportabilityMetrics {
return this; return this;
} }
public static final class CounterNames {
public static final String SQL_STATEMENT_SANITIZER_CACHE_MISS =
"SqlStatementSanitizer cache miss";
private CounterNames() {}
}
// this class is threadsafe. // this class is threadsafe.
private static class KindCounters { private static class KindCounters {
private final LongAdder server = new LongAdder(); private final LongAdder server = new LongAdder();

View File

@ -18,8 +18,8 @@ import io.opentelemetry.context.propagation.TextMapGetter;
import io.opentelemetry.context.propagation.TextMapPropagator; import io.opentelemetry.context.propagation.TextMapPropagator;
import io.opentelemetry.context.propagation.TextMapSetter; import io.opentelemetry.context.propagation.TextMapSetter;
import io.opentelemetry.instrumentation.api.InstrumentationVersion; import io.opentelemetry.instrumentation.api.InstrumentationVersion;
import io.opentelemetry.instrumentation.api.config.Config;
import io.opentelemetry.instrumentation.api.context.ContextPropagationDebug; import io.opentelemetry.instrumentation.api.context.ContextPropagationDebug;
import io.opentelemetry.instrumentation.api.internal.SupportabilityMetrics;
import java.lang.reflect.InvocationTargetException; import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method; import java.lang.reflect.Method;
import java.lang.reflect.UndeclaredThrowableException; import java.lang.reflect.UndeclaredThrowableException;
@ -48,9 +48,7 @@ import org.checkerframework.checker.nullness.qual.Nullable;
* are able to see those attributes in the {@code onStart()} method and can freely read/modify them. * are able to see those attributes in the {@code onStart()} method and can freely read/modify them.
*/ */
public abstract class BaseTracer { public abstract class BaseTracer {
// should we make this injectable? private static final SupportabilityMetrics supportability = SupportabilityMetrics.instance();
private static final SupportabilityMetrics supportability =
new SupportabilityMetrics(Config.get()).start();
private final Tracer tracer; private final Tracer tracer;
private final ContextPropagators propagators; private final ContextPropagators propagators;

View File

@ -3,15 +3,13 @@
* SPDX-License-Identifier: Apache-2.0 * SPDX-License-Identifier: Apache-2.0
*/ */
package io.opentelemetry.instrumentation.api.tracer; package io.opentelemetry.instrumentation.api.internal;
import static java.util.Collections.singletonList;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.api.trace.SpanKind;
import io.opentelemetry.instrumentation.api.config.Config; import io.opentelemetry.instrumentation.api.config.Config;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
@ -28,6 +26,9 @@ class SupportabilityMetricsTest {
metrics.recordSuppressedSpan(SpanKind.SERVER, "favoriteInstrumentation"); metrics.recordSuppressedSpan(SpanKind.SERVER, "favoriteInstrumentation");
metrics.recordSuppressedSpan(SpanKind.CLIENT, "favoriteInstrumentation"); metrics.recordSuppressedSpan(SpanKind.CLIENT, "favoriteInstrumentation");
metrics.recordSuppressedSpan(SpanKind.INTERNAL, "otherInstrumentation"); metrics.recordSuppressedSpan(SpanKind.INTERNAL, "otherInstrumentation");
metrics.incrementCounter("some counter");
metrics.incrementCounter("another counter");
metrics.incrementCounter("some counter");
metrics.report(); metrics.report();
@ -45,17 +46,19 @@ class SupportabilityMetricsTest {
metrics.recordSuppressedSpan(SpanKind.SERVER, "favoriteInstrumentation"); metrics.recordSuppressedSpan(SpanKind.SERVER, "favoriteInstrumentation");
metrics.recordSuppressedSpan(SpanKind.CLIENT, "favoriteInstrumentation"); metrics.recordSuppressedSpan(SpanKind.CLIENT, "favoriteInstrumentation");
metrics.recordSuppressedSpan(SpanKind.INTERNAL, "otherInstrumentation"); metrics.recordSuppressedSpan(SpanKind.INTERNAL, "otherInstrumentation");
metrics.incrementCounter("some counter");
metrics.incrementCounter("another counter");
metrics.incrementCounter("some counter");
metrics.report(); metrics.report();
assertThat(reports) assertThat(reports)
.isNotEmpty() .containsExactlyInAnyOrder(
.hasSize(3) "Suppressed Spans by 'favoriteInstrumentation' (CLIENT) : 2",
.hasSameElementsAs( "Suppressed Spans by 'favoriteInstrumentation' (SERVER) : 1",
Arrays.asList( "Suppressed Spans by 'otherInstrumentation' (INTERNAL) : 1",
"Suppressed Spans by 'favoriteInstrumentation' (CLIENT) : 2", "Counter 'some counter' : 2",
"Suppressed Spans by 'favoriteInstrumentation' (SERVER) : 1", "Counter 'another counter' : 1");
"Suppressed Spans by 'otherInstrumentation' (INTERNAL) : 1"));
} }
@Test @Test
@ -66,14 +69,14 @@ class SupportabilityMetricsTest {
Config.create(Collections.singletonMap("otel.javaagent.debug", "true")), reports::add); Config.create(Collections.singletonMap("otel.javaagent.debug", "true")), reports::add);
metrics.recordSuppressedSpan(SpanKind.CLIENT, "favoriteInstrumentation"); metrics.recordSuppressedSpan(SpanKind.CLIENT, "favoriteInstrumentation");
metrics.incrementCounter("some counter");
metrics.report(); metrics.report();
metrics.report(); metrics.report();
assertThat(reports) assertThat(reports)
.isNotEmpty() .containsExactlyInAnyOrder(
.hasSize(1) "Suppressed Spans by 'favoriteInstrumentation' (CLIENT) : 1",
.hasSameElementsAs( "Counter 'some counter' : 1");
singletonList("Suppressed Spans by 'favoriteInstrumentation' (CLIENT) : 1"));
} }
} }