From fe07cff3baabf24dca687df3f78b7fe7478254fd Mon Sep 17 00:00:00 2001 From: John Watson Date: Fri, 16 Oct 2020 10:09:52 -0700 Subject: [PATCH] Add a new Gauge metric type, and hook it up to Prometheus and OTLP exporters. (#1788) --- .../exporters/otlp/MetricAdapter.java | 35 ++-- .../exporters/otlp/MetricAdapterTest.java | 184 +++++++++++++++++- .../exporters/prometheus/MetricAdapter.java | 16 +- .../prometheus/MetricAdapterTest.java | 26 +++ .../sdk/metrics/data/MetricData.java | 17 ++ .../sdk/metrics/view/Aggregations.java | 5 +- .../metrics/DoubleValueObserverSdkTest.java | 6 +- .../sdk/metrics/LongValueObserverSdkTest.java | 6 +- 8 files changed, 265 insertions(+), 30 deletions(-) diff --git a/exporters/otlp/src/main/java/io/opentelemetry/exporters/otlp/MetricAdapter.java b/exporters/otlp/src/main/java/io/opentelemetry/exporters/otlp/MetricAdapter.java index caedff96fa..e9155fa3a6 100644 --- a/exporters/otlp/src/main/java/io/opentelemetry/exporters/otlp/MetricAdapter.java +++ b/exporters/otlp/src/main/java/io/opentelemetry/exporters/otlp/MetricAdapter.java @@ -9,16 +9,17 @@ import static io.opentelemetry.proto.metrics.v1.AggregationTemporality.AGGREGATI import static io.opentelemetry.proto.metrics.v1.AggregationTemporality.AGGREGATION_TEMPORALITY_DELTA; import static io.opentelemetry.proto.metrics.v1.AggregationTemporality.AGGREGATION_TEMPORALITY_UNSPECIFIED; -import io.opentelemetry.common.LabelConsumer; import io.opentelemetry.common.Labels; import io.opentelemetry.proto.common.v1.StringKeyValue; import io.opentelemetry.proto.metrics.v1.AggregationTemporality; import io.opentelemetry.proto.metrics.v1.DoubleDataPoint; +import io.opentelemetry.proto.metrics.v1.DoubleGauge; import io.opentelemetry.proto.metrics.v1.DoubleHistogram; import io.opentelemetry.proto.metrics.v1.DoubleHistogramDataPoint; import io.opentelemetry.proto.metrics.v1.DoubleSum; import io.opentelemetry.proto.metrics.v1.InstrumentationLibraryMetrics; import io.opentelemetry.proto.metrics.v1.IntDataPoint; +import io.opentelemetry.proto.metrics.v1.IntGauge; import io.opentelemetry.proto.metrics.v1.IntSum; import io.opentelemetry.proto.metrics.v1.Metric; import io.opentelemetry.proto.metrics.v1.ResourceMetrics; @@ -37,6 +38,7 @@ import java.util.List; import java.util.Map; final class MetricAdapter { + static List toProtoResourceMetrics(Collection metricData) { Map>> resourceAndLibraryMap = groupByResourceAndLibrary(metricData); @@ -74,11 +76,9 @@ final class MetricAdapter { libraryInfoListMap = new HashMap<>(); result.put(resource, libraryInfoListMap); } - List metricList = libraryInfoListMap.get(metricData.getInstrumentationLibraryInfo()); - if (metricList == null) { - metricList = new ArrayList<>(); - libraryInfoListMap.put(metricData.getInstrumentationLibraryInfo(), metricList); - } + List metricList = + libraryInfoListMap.computeIfAbsent( + metricData.getInstrumentationLibraryInfo(), k -> new ArrayList<>()); metricList.add(toProtoMetric(metricData)); } return result; @@ -129,6 +129,18 @@ final class MetricAdapter { .addAllDataPoints(toSummaryDataPoints(metricData.getPoints())) .build()); break; + case GAUGE_LONG: + builder.setIntGauge( + IntGauge.newBuilder() + .addAllDataPoints(toIntDataPoints(metricData.getPoints())) + .build()); + break; + case GAUGE_DOUBLE: + builder.setDoubleGauge( + DoubleGauge.newBuilder() + .addAllDataPoints(toDoubleDataPoints(metricData.getPoints())) + .build()); + break; } return builder.build(); } @@ -142,8 +154,9 @@ final class MetricAdapter { return AGGREGATION_TEMPORALITY_CUMULATIVE; case SUMMARY: return AGGREGATION_TEMPORALITY_DELTA; + default: + return AGGREGATION_TEMPORALITY_UNSPECIFIED; } - return AGGREGATION_TEMPORALITY_UNSPECIFIED; } static List toIntDataPoints(Collection points) { @@ -229,12 +242,8 @@ final class MetricAdapter { } final List result = new ArrayList<>(labels.size()); labels.forEach( - new LabelConsumer() { - @Override - public void consume(String key, String value) { - result.add(StringKeyValue.newBuilder().setKey(key).setValue(value).build()); - } - }); + (key, value) -> + result.add(StringKeyValue.newBuilder().setKey(key).setValue(value).build())); return result; } diff --git a/exporters/otlp/src/test/java/io/opentelemetry/exporters/otlp/MetricAdapterTest.java b/exporters/otlp/src/test/java/io/opentelemetry/exporters/otlp/MetricAdapterTest.java index e04cad0002..2a16f63037 100644 --- a/exporters/otlp/src/test/java/io/opentelemetry/exporters/otlp/MetricAdapterTest.java +++ b/exporters/otlp/src/test/java/io/opentelemetry/exporters/otlp/MetricAdapterTest.java @@ -7,6 +7,8 @@ package io.opentelemetry.exporters.otlp; import static io.opentelemetry.common.AttributeKey.stringKey; import static io.opentelemetry.proto.metrics.v1.AggregationTemporality.AGGREGATION_TEMPORALITY_CUMULATIVE; +import static io.opentelemetry.proto.metrics.v1.AggregationTemporality.AGGREGATION_TEMPORALITY_DELTA; +import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; @@ -18,10 +20,13 @@ import io.opentelemetry.proto.common.v1.InstrumentationLibrary; import io.opentelemetry.proto.common.v1.KeyValue; import io.opentelemetry.proto.common.v1.StringKeyValue; import io.opentelemetry.proto.metrics.v1.DoubleDataPoint; +import io.opentelemetry.proto.metrics.v1.DoubleGauge; +import io.opentelemetry.proto.metrics.v1.DoubleHistogram; import io.opentelemetry.proto.metrics.v1.DoubleHistogramDataPoint; import io.opentelemetry.proto.metrics.v1.DoubleSum; import io.opentelemetry.proto.metrics.v1.InstrumentationLibraryMetrics; import io.opentelemetry.proto.metrics.v1.IntDataPoint; +import io.opentelemetry.proto.metrics.v1.IntGauge; import io.opentelemetry.proto.metrics.v1.IntSum; import io.opentelemetry.proto.metrics.v1.Metric; import io.opentelemetry.proto.metrics.v1.ResourceMetrics; @@ -196,7 +201,7 @@ class MetricAdapterTest { } @Test - void toProtoMetric() { + void toProtoMetric_monotonic() { assertThat( MetricAdapter.toProtoMetric( MetricData.create( @@ -266,6 +271,183 @@ class MetricAdapterTest { .build()); } + @Test + void toProtoMetric_nonMonotonic() { + assertThat( + MetricAdapter.toProtoMetric( + MetricData.create( + Resource.getEmpty(), + InstrumentationLibraryInfo.getEmpty(), + "name", + "description", + "1", + MetricData.Type.NON_MONOTONIC_LONG, + singletonList(MetricData.LongPoint.create(123, 456, Labels.of("k", "v"), 5))))) + .isEqualTo( + Metric.newBuilder() + .setName("name") + .setDescription("description") + .setUnit("1") + .setIntSum( + IntSum.newBuilder() + .setIsMonotonic(false) + .setAggregationTemporality(AGGREGATION_TEMPORALITY_CUMULATIVE) + .addDataPoints( + IntDataPoint.newBuilder() + .setStartTimeUnixNano(123) + .setTimeUnixNano(456) + .addAllLabels( + singletonList( + StringKeyValue.newBuilder() + .setKey("k") + .setValue("v") + .build())) + .setValue(5) + .build()) + .build()) + .build()); + assertThat( + MetricAdapter.toProtoMetric( + MetricData.create( + Resource.getEmpty(), + InstrumentationLibraryInfo.getEmpty(), + "name", + "description", + "1", + MetricData.Type.NON_MONOTONIC_DOUBLE, + singletonList( + MetricData.DoublePoint.create(123, 456, Labels.of("k", "v"), 5.1))))) + .isEqualTo( + Metric.newBuilder() + .setName("name") + .setDescription("description") + .setUnit("1") + .setDoubleSum( + DoubleSum.newBuilder() + .setIsMonotonic(false) + .setAggregationTemporality(AGGREGATION_TEMPORALITY_CUMULATIVE) + .addDataPoints( + DoubleDataPoint.newBuilder() + .setStartTimeUnixNano(123) + .setTimeUnixNano(456) + .addAllLabels( + singletonList( + StringKeyValue.newBuilder() + .setKey("k") + .setValue("v") + .build())) + .setValue(5.1) + .build()) + .build()) + .build()); + } + + @Test + void toProtoMetric_gauges() { + assertThat( + MetricAdapter.toProtoMetric( + MetricData.create( + Resource.getEmpty(), + InstrumentationLibraryInfo.getEmpty(), + "name", + "description", + "1", + MetricData.Type.GAUGE_LONG, + singletonList(MetricData.LongPoint.create(123, 456, Labels.of("k", "v"), 5))))) + .isEqualTo( + Metric.newBuilder() + .setName("name") + .setDescription("description") + .setUnit("1") + .setIntGauge( + IntGauge.newBuilder() + .addDataPoints( + IntDataPoint.newBuilder() + .setStartTimeUnixNano(123) + .setTimeUnixNano(456) + .addAllLabels( + singletonList( + StringKeyValue.newBuilder() + .setKey("k") + .setValue("v") + .build())) + .setValue(5) + .build()) + .build()) + .build()); + assertThat( + MetricAdapter.toProtoMetric( + MetricData.create( + Resource.getEmpty(), + InstrumentationLibraryInfo.getEmpty(), + "name", + "description", + "1", + MetricData.Type.GAUGE_DOUBLE, + singletonList( + MetricData.DoublePoint.create(123, 456, Labels.of("k", "v"), 5.1))))) + .isEqualTo( + Metric.newBuilder() + .setName("name") + .setDescription("description") + .setUnit("1") + .setDoubleGauge( + DoubleGauge.newBuilder() + .addDataPoints( + DoubleDataPoint.newBuilder() + .setStartTimeUnixNano(123) + .setTimeUnixNano(456) + .addAllLabels( + singletonList( + StringKeyValue.newBuilder() + .setKey("k") + .setValue("v") + .build())) + .setValue(5.1) + .build()) + .build()) + .build()); + } + + @Test + void toProtoMetric_summary() { + assertThat( + MetricAdapter.toProtoMetric( + MetricData.create( + Resource.getEmpty(), + InstrumentationLibraryInfo.getEmpty(), + "name", + "description", + "1", + MetricData.Type.SUMMARY, + singletonList( + MetricData.SummaryPoint.create( + 123, 456, Labels.of("k", "v"), 5, 33d, emptyList()))))) + .isEqualTo( + Metric.newBuilder() + .setName("name") + .setDescription("description") + .setUnit("1") + .setDoubleHistogram( + DoubleHistogram.newBuilder() + .setAggregationTemporality(AGGREGATION_TEMPORALITY_DELTA) + .addDataPoints( + DoubleHistogramDataPoint.newBuilder() + .setStartTimeUnixNano(123) + .setTimeUnixNano(456) + .addAllLabels( + singletonList( + StringKeyValue.newBuilder() + .setKey("k") + .setValue("v") + .build())) + .setCount(5) + .setSum(33d) + .build()) + .build()) + .build()); + } + @Test void toProtoResourceMetrics() { Resource resource = Resource.create(Attributes.of(stringKey("ka"), "va")); diff --git a/exporters/prometheus/src/main/java/io/opentelemetry/exporters/prometheus/MetricAdapter.java b/exporters/prometheus/src/main/java/io/opentelemetry/exporters/prometheus/MetricAdapter.java index 73331b64cc..04f6cd7317 100644 --- a/exporters/prometheus/src/main/java/io/opentelemetry/exporters/prometheus/MetricAdapter.java +++ b/exporters/prometheus/src/main/java/io/opentelemetry/exporters/prometheus/MetricAdapter.java @@ -63,6 +63,8 @@ final class MetricAdapter { switch (type) { case NON_MONOTONIC_LONG: case NON_MONOTONIC_DOUBLE: + case GAUGE_LONG: + case GAUGE_DOUBLE: return Collector.Type.GAUGE; case MONOTONIC_LONG: case MONOTONIC_DOUBLE: @@ -92,11 +94,13 @@ final class MetricAdapter { switch (type) { case MONOTONIC_DOUBLE: case NON_MONOTONIC_DOUBLE: + case GAUGE_DOUBLE: DoublePoint doublePoint = (DoublePoint) point; samples.add(new Sample(name, labelNames, labelValues, doublePoint.getValue())); break; case MONOTONIC_LONG: case NON_MONOTONIC_LONG: + case GAUGE_LONG: LongPoint longPoint = (LongPoint) point; samples.add(new Sample(name, labelNames, labelValues, longPoint.getValue())); break; @@ -154,15 +158,9 @@ final class MetricAdapter { } private static int estimateNumSamples(int numPoints, MetricData.Type type) { - switch (type) { - case NON_MONOTONIC_LONG: - case NON_MONOTONIC_DOUBLE: - case MONOTONIC_LONG: - case MONOTONIC_DOUBLE: - return numPoints; - case SUMMARY: - // count + sum + estimated 2 percentiles (default MinMaxSumCount aggregator). - return numPoints * 4; + if (type == MetricData.Type.SUMMARY) { + // count + sum + estimated 2 percentiles (default MinMaxSumCount aggregator). + return numPoints * 4; } return numPoints; } diff --git a/exporters/prometheus/src/test/java/io/opentelemetry/exporters/prometheus/MetricAdapterTest.java b/exporters/prometheus/src/test/java/io/opentelemetry/exporters/prometheus/MetricAdapterTest.java index 54c24dd3c0..c66c2d433b 100644 --- a/exporters/prometheus/src/test/java/io/opentelemetry/exporters/prometheus/MetricAdapterTest.java +++ b/exporters/prometheus/src/test/java/io/opentelemetry/exporters/prometheus/MetricAdapterTest.java @@ -35,6 +35,10 @@ class MetricAdapterTest { .isEqualTo(Collector.Type.COUNTER); assertThat(MetricAdapter.toMetricFamilyType(MetricData.Type.SUMMARY)) .isEqualTo(Collector.Type.SUMMARY); + assertThat(MetricAdapter.toMetricFamilyType(MetricData.Type.GAUGE_DOUBLE)) + .isEqualTo(Collector.Type.GAUGE); + assertThat(MetricAdapter.toMetricFamilyType(MetricData.Type.GAUGE_LONG)) + .isEqualTo(Collector.Type.GAUGE); } @Test @@ -72,6 +76,17 @@ class MetricAdapterTest { .containsExactly( new Sample("full_name", Collections.emptyList(), Collections.emptyList(), 5), new Sample("full_name", ImmutableList.of("kp"), ImmutableList.of("vp"), 7)); + + assertThat( + MetricAdapter.toSamples( + "full_name", + MetricData.Type.GAUGE_LONG, + ImmutableList.of( + MetricData.LongPoint.create(123, 456, Labels.empty(), 5), + MetricData.LongPoint.create(321, 654, Labels.of("kp", "vp"), 7)))) + .containsExactly( + new Sample("full_name", Collections.emptyList(), Collections.emptyList(), 5), + new Sample("full_name", ImmutableList.of("kp"), ImmutableList.of("vp"), 7)); } @Test @@ -100,6 +115,17 @@ class MetricAdapterTest { .containsExactly( new Sample("full_name", Collections.emptyList(), Collections.emptyList(), 5), new Sample("full_name", ImmutableList.of("kp"), ImmutableList.of("vp"), 7)); + + assertThat( + MetricAdapter.toSamples( + "full_name", + MetricData.Type.GAUGE_DOUBLE, + ImmutableList.of( + MetricData.DoublePoint.create(123, 456, Labels.empty(), 5), + MetricData.DoublePoint.create(321, 654, Labels.of("kp", "vp"), 7)))) + .containsExactly( + new Sample("full_name", Collections.emptyList(), Collections.emptyList(), 5), + new Sample("full_name", ImmutableList.of("kp"), ImmutableList.of("vp"), 7)); } @Test diff --git a/sdk/metrics/src/main/java/io/opentelemetry/sdk/metrics/data/MetricData.java b/sdk/metrics/src/main/java/io/opentelemetry/sdk/metrics/data/MetricData.java index 260c3af875..d0e9db4284 100644 --- a/sdk/metrics/src/main/java/io/opentelemetry/sdk/metrics/data/MetricData.java +++ b/sdk/metrics/src/main/java/io/opentelemetry/sdk/metrics/data/MetricData.java @@ -19,6 +19,7 @@ import javax.annotation.concurrent.Immutable; @Immutable @AutoValue public abstract class MetricData { + MetricData() {} /** The kind of metric. It describes how the data is reported. */ @@ -41,6 +42,18 @@ public abstract class MetricData { * recorded. */ SUMMARY, + + /** + * A Gauge represents a measurement of a long value at a moment in time. Generally only one + * instance of a given Gauge metric will be reported per reporting interval. + */ + GAUGE_LONG, + + /** + * A Gauge represents a measurement of a double value at a moment in time. Generally only one + * instance of a given Gauge metric will be reported per reporting interval. + */ + GAUGE_DOUBLE } /** @@ -110,6 +123,7 @@ public abstract class MetricData { @Immutable public abstract static class Point { + Point() {} /** @@ -145,6 +159,7 @@ public abstract class MetricData { @Immutable @AutoValue public abstract static class LongPoint extends Point { + LongPoint() {} /** @@ -167,6 +182,7 @@ public abstract class MetricData { @Immutable @AutoValue public abstract static class DoublePoint extends Point { + DoublePoint() {} /** @@ -229,6 +245,7 @@ public abstract class MetricData { @Immutable @AutoValue public abstract static class ValueAtPercentile { + ValueAtPercentile() {} /** diff --git a/sdk/metrics/src/main/java/io/opentelemetry/sdk/metrics/view/Aggregations.java b/sdk/metrics/src/main/java/io/opentelemetry/sdk/metrics/view/Aggregations.java index c91e96890c..9b27d9d5f6 100644 --- a/sdk/metrics/src/main/java/io/opentelemetry/sdk/metrics/view/Aggregations.java +++ b/sdk/metrics/src/main/java/io/opentelemetry/sdk/metrics/view/Aggregations.java @@ -222,11 +222,14 @@ public class Aggregations { return instrumentValueType == InstrumentValueType.LONG ? MetricData.Type.MONOTONIC_LONG : MetricData.Type.MONOTONIC_DOUBLE; - case VALUE_OBSERVER: case UP_DOWN_SUM_OBSERVER: return instrumentValueType == InstrumentValueType.LONG ? MetricData.Type.NON_MONOTONIC_LONG : MetricData.Type.NON_MONOTONIC_DOUBLE; + case VALUE_OBSERVER: + return instrumentValueType == InstrumentValueType.LONG + ? MetricData.Type.GAUGE_LONG + : MetricData.Type.GAUGE_DOUBLE; default: // Do not change this unless the limitations of the current LastValueAggregator are fixed. throw new IllegalArgumentException( diff --git a/sdk/metrics/src/test/java/io/opentelemetry/sdk/metrics/DoubleValueObserverSdkTest.java b/sdk/metrics/src/test/java/io/opentelemetry/sdk/metrics/DoubleValueObserverSdkTest.java index cfed78420f..ac2c4b9081 100644 --- a/sdk/metrics/src/test/java/io/opentelemetry/sdk/metrics/DoubleValueObserverSdkTest.java +++ b/sdk/metrics/src/test/java/io/opentelemetry/sdk/metrics/DoubleValueObserverSdkTest.java @@ -64,7 +64,7 @@ class DoubleValueObserverSdkTest { "testObserver", "My own DoubleValueObserver", "ms", - MetricData.Type.NON_MONOTONIC_DOUBLE, + MetricData.Type.GAUGE_DOUBLE, Collections.emptyList())); } @@ -82,7 +82,7 @@ class DoubleValueObserverSdkTest { "testObserver", "", "1", - MetricData.Type.NON_MONOTONIC_DOUBLE, + MetricData.Type.GAUGE_DOUBLE, Collections.singletonList( DoublePoint.create( testClock.now() - SECOND_NANOS, @@ -98,7 +98,7 @@ class DoubleValueObserverSdkTest { "testObserver", "", "1", - MetricData.Type.NON_MONOTONIC_DOUBLE, + MetricData.Type.GAUGE_DOUBLE, Collections.singletonList( DoublePoint.create( testClock.now() - SECOND_NANOS, diff --git a/sdk/metrics/src/test/java/io/opentelemetry/sdk/metrics/LongValueObserverSdkTest.java b/sdk/metrics/src/test/java/io/opentelemetry/sdk/metrics/LongValueObserverSdkTest.java index f3e00ab0cd..70ada279be 100644 --- a/sdk/metrics/src/test/java/io/opentelemetry/sdk/metrics/LongValueObserverSdkTest.java +++ b/sdk/metrics/src/test/java/io/opentelemetry/sdk/metrics/LongValueObserverSdkTest.java @@ -63,7 +63,7 @@ class LongValueObserverSdkTest { "testObserver", "My own LongValueObserver", "ms", - MetricData.Type.NON_MONOTONIC_LONG, + MetricData.Type.GAUGE_LONG, Collections.emptyList())); } @@ -81,7 +81,7 @@ class LongValueObserverSdkTest { "testObserver", "", "1", - MetricData.Type.NON_MONOTONIC_LONG, + MetricData.Type.GAUGE_LONG, Collections.singletonList( LongPoint.create( testClock.now() - SECOND_NANOS, @@ -97,7 +97,7 @@ class LongValueObserverSdkTest { "testObserver", "", "1", - MetricData.Type.NON_MONOTONIC_LONG, + MetricData.Type.GAUGE_LONG, Collections.singletonList( LongPoint.create( testClock.now() - SECOND_NANOS,