Add opt-in support for metric overflow attribute (#4737)

This commit is contained in:
Utkarsh Umesan Pillai 2023-08-08 18:12:07 -07:00 committed by GitHub
parent e227d0fb89
commit 10a898932a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 651 additions and 32 deletions

View File

@ -2,6 +2,17 @@
## Unreleased
* **Experimental Feature** Added an opt-in feature to aggregate any metric
measurements that were dropped due to reaching the [max MetricPoints
limit](https://github.com/open-telemetry/opentelemetry-dotnet/tree/core-1.6.0-alpha.1/docs/metrics/customizing-the-sdk).
When this feature is enabled, SDK would aggregate such measurements using a
reserved MetricPoint with a single tag with key as `otel.metric.overflow` and
value as `true`. The feature is turned-off by default. You can enable it by
setting the environment variable
`OTEL_DOTNET_EXPERIMENTAL_METRICS_EMIT_OVERFLOW_ATTRIBUTE` to `true` before
setting up the `MeterProvider`.
([#4737](https://github.com/open-telemetry/opentelemetry-dotnet/pull/4737))
## 1.6.0-alpha.1
Released 2023-Jul-12

View File

@ -22,9 +22,11 @@ namespace OpenTelemetry.Metrics;
internal sealed class AggregatorStore
{
private static readonly string MetricPointCapHitFixMessage = "Modify instrumentation to reduce the number of unique key/value pair combinations. Or use Views to drop unwanted tags. Or use MeterProviderBuilder.SetMaxMetricPointsPerMetricStream to set higher limit.";
private static readonly string MetricPointCapHitFixMessage = "Consider opting in for the experimental SDK feature to emit all the throttled metrics under the overflow attribute by setting env variable OTEL_DOTNET_EXPERIMENTAL_METRICS_EMIT_OVERFLOW_ATTRIBUTE = true. You could also modify instrumentation to reduce the number of unique key/value pair combinations. Or use Views to drop unwanted tags. Or use MeterProviderBuilder.SetMaxMetricPointsPerMetricStream to set higher limit.";
private static readonly Comparison<KeyValuePair<string, object>> DimensionComparisonDelegate = (x, y) => x.Key.CompareTo(y.Key);
private readonly object lockZeroTags = new();
private readonly object lockOverflowTag = new();
private readonly HashSet<string> tagKeysInteresting;
private readonly int tagsKeysInterestingCount;
@ -43,17 +45,21 @@ internal sealed class AggregatorStore
private readonly UpdateLongDelegate updateLongCallback;
private readonly UpdateDoubleDelegate updateDoubleCallback;
private readonly int maxMetricPoints;
private readonly bool emitOverflowAttribute;
private readonly ExemplarFilter exemplarFilter;
private int metricPointIndex = 0;
private int batchSize = 0;
private int metricCapHitMessageLogged;
private bool zeroTagMetricPointInitialized;
private bool overflowTagMetricPointInitialized;
internal AggregatorStore(
MetricStreamIdentity metricStreamIdentity,
AggregationType aggType,
AggregationTemporality temporality,
int maxMetricPoints,
bool emitOverflowAttribute,
ExemplarFilter exemplarFilter = null)
{
this.name = metricStreamIdentity.InstrumentName;
@ -81,6 +87,15 @@ internal sealed class AggregatorStore
this.tagKeysInteresting = hs;
this.tagsKeysInterestingCount = hs.Count;
}
this.emitOverflowAttribute = emitOverflowAttribute;
if (emitOverflowAttribute)
{
// Setting metricPointIndex to 1 as we would reserve the metricPoints[1] for overflow attribute.
// Newer attributes should be added starting at the index: 2
this.metricPointIndex = 1;
}
}
private delegate void UpdateLongDelegate(long value, ReadOnlySpan<KeyValuePair<string, object>> tags);
@ -197,6 +212,22 @@ internal sealed class AggregatorStore
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void InitializeOverflowTagPointIfNotInitialized()
{
if (!this.overflowTagMetricPointInitialized)
{
lock (this.lockOverflowTag)
{
if (!this.overflowTagMetricPointInitialized)
{
this.metricPoints[1] = new MetricPoint(this, this.aggType, new KeyValuePair<string, object>[] { new("otel.metric.overflow", true) }, this.histogramBounds, this.exponentialHistogramMaxSize, this.exponentialHistogramMaxScale);
this.overflowTagMetricPointInitialized = true;
}
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int LookupAggregatorStore(KeyValuePair<string, object>[] tagKeysAndValues, int length)
{
@ -329,12 +360,21 @@ internal sealed class AggregatorStore
var index = this.FindMetricAggregatorsDefault(tags);
if (index < 0)
{
if (Interlocked.CompareExchange(ref this.metricCapHitMessageLogged, 1, 0) == 0)
if (this.emitOverflowAttribute)
{
OpenTelemetrySdkEventSource.Log.MeasurementDropped(this.name, this.metricPointCapHitMessage, MetricPointCapHitFixMessage);
this.InitializeOverflowTagPointIfNotInitialized();
this.metricPoints[1].Update(value);
return;
}
else
{
if (Interlocked.CompareExchange(ref this.metricCapHitMessageLogged, 1, 0) == 0)
{
OpenTelemetrySdkEventSource.Log.MeasurementDropped(this.name, this.metricPointCapHitMessage, MetricPointCapHitFixMessage);
}
return;
return;
}
}
// TODO: can special case built-in filters to be bit faster.
@ -361,12 +401,21 @@ internal sealed class AggregatorStore
var index = this.FindMetricAggregatorsCustomTag(tags);
if (index < 0)
{
if (Interlocked.CompareExchange(ref this.metricCapHitMessageLogged, 1, 0) == 0)
if (this.emitOverflowAttribute)
{
OpenTelemetrySdkEventSource.Log.MeasurementDropped(this.name, this.metricPointCapHitMessage, MetricPointCapHitFixMessage);
this.InitializeOverflowTagPointIfNotInitialized();
this.metricPoints[1].Update(value);
return;
}
else
{
if (Interlocked.CompareExchange(ref this.metricCapHitMessageLogged, 1, 0) == 0)
{
OpenTelemetrySdkEventSource.Log.MeasurementDropped(this.name, this.metricPointCapHitMessage, MetricPointCapHitFixMessage);
}
return;
return;
}
}
// TODO: can special case built-in filters to be bit faster.
@ -393,12 +442,21 @@ internal sealed class AggregatorStore
var index = this.FindMetricAggregatorsDefault(tags);
if (index < 0)
{
if (Interlocked.CompareExchange(ref this.metricCapHitMessageLogged, 1, 0) == 0)
if (this.emitOverflowAttribute)
{
OpenTelemetrySdkEventSource.Log.MeasurementDropped(this.name, this.metricPointCapHitMessage, MetricPointCapHitFixMessage);
this.InitializeOverflowTagPointIfNotInitialized();
this.metricPoints[1].Update(value);
return;
}
else
{
if (Interlocked.CompareExchange(ref this.metricCapHitMessageLogged, 1, 0) == 0)
{
OpenTelemetrySdkEventSource.Log.MeasurementDropped(this.name, this.metricPointCapHitMessage, MetricPointCapHitFixMessage);
}
return;
return;
}
}
// TODO: can special case built-in filters to be bit faster.
@ -425,12 +483,21 @@ internal sealed class AggregatorStore
var index = this.FindMetricAggregatorsCustomTag(tags);
if (index < 0)
{
if (Interlocked.CompareExchange(ref this.metricCapHitMessageLogged, 1, 0) == 0)
if (this.emitOverflowAttribute)
{
OpenTelemetrySdkEventSource.Log.MeasurementDropped(this.name, this.metricPointCapHitMessage, MetricPointCapHitFixMessage);
this.InitializeOverflowTagPointIfNotInitialized();
this.metricPoints[1].Update(value);
return;
}
else
{
if (Interlocked.CompareExchange(ref this.metricCapHitMessageLogged, 1, 0) == 0)
{
OpenTelemetrySdkEventSource.Log.MeasurementDropped(this.name, this.metricPointCapHitMessage, MetricPointCapHitFixMessage);
}
return;
return;
}
}
// TODO: can special case built-in filters to be bit faster.

View File

@ -19,6 +19,7 @@
using System.Diagnostics;
using System.Diagnostics.Metrics;
using System.Text;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using OpenTelemetry.Internal;
using OpenTelemetry.Resources;
@ -32,6 +33,8 @@ internal sealed class MeterProviderSdk : MeterProvider
internal int ShutdownCount;
internal bool Disposed;
private const string EmitOverFlowAttributeConfigKey = "OTEL_DOTNET_EXPERIMENTAL_METRICS_EMIT_OVERFLOW_ATTRIBUTE";
private readonly List<object> instrumentations = new();
private readonly List<Func<Instrument, MetricStreamConfiguration?>> viewConfigs;
private readonly object collectLock = new();
@ -48,6 +51,9 @@ internal sealed class MeterProviderSdk : MeterProvider
var state = serviceProvider!.GetRequiredService<MeterProviderBuilderSdk>();
state.RegisterProvider(this);
var config = serviceProvider!.GetRequiredService<IConfiguration>();
_ = config.TryGetBoolValue(EmitOverFlowAttributeConfigKey, out bool isEmitOverflowAttributeKeySet);
this.ServiceProvider = serviceProvider!;
if (ownsServiceProvider)
@ -79,7 +85,7 @@ internal sealed class MeterProviderSdk : MeterProvider
reader.SetParentProvider(this);
reader.SetMaxMetricStreams(state.MaxMetricStreams);
reader.SetMaxMetricPointsPerMetricStream(state.MaxMetricPointsPerMetricStream);
reader.SetMaxMetricPointsPerMetricStream(state.MaxMetricPointsPerMetricStream, isEmitOverflowAttributeKeySet);
reader.SetExemplarFilter(state.ExemplarFilter);
if (this.reader == null)

View File

@ -35,6 +35,7 @@ public sealed class Metric
MetricStreamIdentity instrumentIdentity,
AggregationTemporality temporality,
int maxMetricPointsPerMetricStream,
bool emitOverflowAttribute,
ExemplarFilter exemplarFilter = null)
{
this.InstrumentIdentity = instrumentIdentity;
@ -141,7 +142,7 @@ public sealed class Metric
throw new NotSupportedException($"Unsupported Instrument Type: {instrumentIdentity.InstrumentType.FullName}");
}
this.aggStore = new AggregatorStore(instrumentIdentity, aggType, temporality, maxMetricPointsPerMetricStream, exemplarFilter);
this.aggStore = new AggregatorStore(instrumentIdentity, aggType, temporality, maxMetricPointsPerMetricStream, emitOverflowAttribute, exemplarFilter);
this.Temporality = temporality;
this.InstrumentDisposed = false;
}

View File

@ -33,6 +33,7 @@ public abstract partial class MetricReader
private Metric[] metrics;
private Metric[] metricsCurrentBatch;
private int metricIndex = -1;
private bool emitOverflowAttribute;
private ExemplarFilter exemplarFilter;
@ -71,7 +72,7 @@ public abstract partial class MetricReader
Metric metric = null;
try
{
metric = new Metric(metricStreamIdentity, this.GetAggregationTemporality(metricStreamIdentity.InstrumentType), this.maxMetricPointsPerMetricStream, exemplarFilter: this.exemplarFilter);
metric = new Metric(metricStreamIdentity, this.GetAggregationTemporality(metricStreamIdentity.InstrumentType), this.maxMetricPointsPerMetricStream, this.emitOverflowAttribute, this.exemplarFilter);
}
catch (NotSupportedException nse)
{
@ -156,7 +157,7 @@ public abstract partial class MetricReader
}
else
{
Metric metric = new(metricStreamIdentity, this.GetAggregationTemporality(metricStreamIdentity.InstrumentType), this.maxMetricPointsPerMetricStream, this.exemplarFilter);
Metric metric = new(metricStreamIdentity, this.GetAggregationTemporality(metricStreamIdentity.InstrumentType), this.maxMetricPointsPerMetricStream, this.emitOverflowAttribute, this.exemplarFilter);
this.instrumentIdentityToMetric[metricStreamIdentity] = metric;
this.metrics[index] = metric;
@ -230,9 +231,18 @@ public abstract partial class MetricReader
this.exemplarFilter = exemplarFilter;
}
internal void SetMaxMetricPointsPerMetricStream(int maxMetricPointsPerMetricStream)
internal void SetMaxMetricPointsPerMetricStream(int maxMetricPointsPerMetricStream, bool isEmitOverflowAttributeKeySet)
{
this.maxMetricPointsPerMetricStream = maxMetricPointsPerMetricStream;
if (isEmitOverflowAttributeKeySet)
{
// We need at least two metric points. One is reserved for zero tags and the other one for overflow attribute
if (maxMetricPointsPerMetricStream > 1)
{
this.emitOverflowAttribute = true;
}
}
}
private Batch<Metric> GetMetricsBatch()

View File

@ -95,6 +95,26 @@ internal static class ConfigurationExtensions
return true;
}
public static bool TryGetBoolValue(
this IConfiguration configuration,
string key,
out bool value)
{
if (!configuration.TryGetStringValue(key, out var stringValue))
{
value = default;
return false;
}
if (!bool.TryParse(stringValue, out value))
{
LogInvalidEnvironmentVariable?.Invoke(key, stringValue!);
return false;
}
return true;
}
public static bool TryGetValue<T>(
this IConfiguration configuration,
string key,

View File

@ -1,4 +1,4 @@
// <copyright file="AggregatorTest.cs" company="OpenTelemetry Authors">
// <copyright file="AggregatorTestsBase.cs" company="OpenTelemetry Authors">
// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
@ -19,13 +19,27 @@ using Xunit;
namespace OpenTelemetry.Metrics.Tests;
public class AggregatorTest
#pragma warning disable SA1402
public abstract class AggregatorTestsBase
{
private static readonly Meter Meter = new("testMeter");
private static readonly Instrument Instrument = Meter.CreateHistogram<long>("testInstrument");
private static readonly ExplicitBucketHistogramConfiguration HistogramConfiguration = new() { Boundaries = Metric.DefaultHistogramBounds };
private static readonly MetricStreamIdentity MetricStreamIdentity = new(Instrument, HistogramConfiguration);
private readonly AggregatorStore aggregatorStore = new(MetricStreamIdentity, AggregationType.HistogramWithBuckets, AggregationTemporality.Cumulative, 1024);
private readonly bool emitOverflowAttribute;
private readonly AggregatorStore aggregatorStore;
protected AggregatorTestsBase(bool emitOverflowAttribute)
{
if (emitOverflowAttribute)
{
this.emitOverflowAttribute = emitOverflowAttribute;
}
this.aggregatorStore = new(MetricStreamIdentity, AggregationType.HistogramWithBuckets, AggregationTemporality.Cumulative, 1024, emitOverflowAttribute);
}
[Fact]
public void HistogramDistributeToAllBucketsDefault()
@ -284,6 +298,7 @@ public class AggregatorTest
aggregationType,
aggregationTemporality,
maxMetricPoints: 1024,
this.emitOverflowAttribute,
exemplarsEnabled ? new AlwaysOnExemplarFilter() : null);
var expectedHistogram = new Base2ExponentialBucketHistogram();
@ -391,7 +406,8 @@ public class AggregatorTest
metricStreamIdentity,
AggregationType.Base2ExponentialHistogram,
AggregationTemporality.Cumulative,
maxMetricPoints: 1024);
maxMetricPoints: 1024,
this.emitOverflowAttribute);
aggregatorStore.Update(10, Array.Empty<KeyValuePair<string, object>>());
@ -463,3 +479,19 @@ public class AggregatorTest
public double SumOfDelta;
}
}
public class AggregatorTests : AggregatorTestsBase
{
public AggregatorTests()
: base(false)
{
}
}
public class AggregatorTestsWithOverflowAttribute : AggregatorTestsBase
{
public AggregatorTestsWithOverflowAttribute()
: base(true)
{
}
}

View File

@ -1,4 +1,4 @@
// <copyright file="MetricAPITest.cs" company="OpenTelemetry Authors">
// <copyright file="MetricApiTestsBase.cs" company="OpenTelemetry Authors">
// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
@ -24,7 +24,9 @@ using Xunit.Abstractions;
namespace OpenTelemetry.Metrics.Tests;
public class MetricApiTest : MetricTestsBase
#pragma warning disable SA1402
public abstract class MetricApiTestsBase : MetricTestsBase, IDisposable
{
private const int MaxTimeToAllowForFlush = 10000;
private static readonly int NumberOfThreads = Environment.ProcessorCount;
@ -33,9 +35,14 @@ public class MetricApiTest : MetricTestsBase
private static readonly int NumberOfMetricUpdateByEachThread = 100000;
private readonly ITestOutputHelper output;
public MetricApiTest(ITestOutputHelper output)
protected MetricApiTestsBase(ITestOutputHelper output, bool emitOverflowAttribute)
{
this.output = output;
if (emitOverflowAttribute)
{
Environment.SetEnvironmentVariable(EmitOverFlowAttributeConfigKey, "true");
}
}
[Fact]
@ -1518,6 +1525,11 @@ public class MetricApiTest : MetricTestsBase
Assert.Empty(exportedItems);
}
public void Dispose()
{
Environment.SetEnvironmentVariable(EmitOverFlowAttributeConfigKey, null);
}
private static void CounterUpdateThread<T>(object obj)
where T : struct, IComparable
{
@ -1689,3 +1701,19 @@ public class MetricApiTest : MetricTestsBase
public T[] ValuesToRecord;
}
}
public class MetricApiTest : MetricApiTestsBase
{
public MetricApiTest(ITestOutputHelper output)
: base(output, false)
{
}
}
public class MetricApiTestWithOverflowAttribute : MetricApiTestsBase
{
public MetricApiTestWithOverflowAttribute(ITestOutputHelper output)
: base(output, true)
{
}
}

View File

@ -0,0 +1,409 @@
// <copyright file="MetricOverflowAttributeTests.cs" company="OpenTelemetry Authors">
// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// </copyright>
using System.Diagnostics.Metrics;
using System.Reflection;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using OpenTelemetry.Tests;
using Xunit;
namespace OpenTelemetry.Metrics.Tests;
public class MetricOverflowAttributeTests
{
[Theory]
[InlineData("false", false)]
[InlineData("False", false)]
[InlineData("FALSE", false)]
[InlineData("true", true)]
[InlineData("True", true)]
[InlineData("TRUE", true)]
public void TestEmitOverflowAttributeConfigWithEnvVar(string value, bool isEmitOverflowAttributeKeySet)
{
try
{
Environment.SetEnvironmentVariable(MetricTestsBase.EmitOverFlowAttributeConfigKey, value);
var exportedItems = new List<Metric>();
var meter = new Meter(Utils.GetCurrentMethodName());
var counter = meter.CreateCounter<long>("TestCounter");
using var meterProvider = Sdk.CreateMeterProviderBuilder()
.AddMeter(meter.Name)
.AddInMemoryExporter(exportedItems)
.Build();
counter.Add(10);
meterProvider.ForceFlush();
Assert.Single(exportedItems);
var metric = exportedItems[0];
var aggregatorStore = typeof(Metric).GetField("aggStore", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(metric) as AggregatorStore;
var emitOverflowAttribute = (bool)typeof(AggregatorStore).GetField("emitOverflowAttribute", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(aggregatorStore);
Assert.Equal(isEmitOverflowAttributeKeySet, emitOverflowAttribute);
}
finally
{
Environment.SetEnvironmentVariable(MetricTestsBase.EmitOverFlowAttributeConfigKey, null);
}
}
[Theory]
[InlineData("false", false)]
[InlineData("False", false)]
[InlineData("FALSE", false)]
[InlineData("true", true)]
[InlineData("True", true)]
[InlineData("TRUE", true)]
public void TestEmitOverflowAttributeConfigWithOtherConfigProvider(string value, bool isEmitOverflowAttributeKeySet)
{
try
{
var exportedItems = new List<Metric>();
var meter = new Meter(Utils.GetCurrentMethodName());
var counter = meter.CreateCounter<long>("TestCounter");
using var meterProvider = Sdk.CreateMeterProviderBuilder()
.ConfigureServices(services =>
{
var configuration = new ConfigurationBuilder()
.AddInMemoryCollection(new Dictionary<string, string> { [MetricTestsBase.EmitOverFlowAttributeConfigKey] = value })
.Build();
services.AddSingleton<IConfiguration>(configuration);
})
.AddMeter(meter.Name)
.AddInMemoryExporter(exportedItems)
.Build();
counter.Add(10);
meterProvider.ForceFlush();
Assert.Single(exportedItems);
var metric = exportedItems[0];
var aggregatorStore = typeof(Metric).GetField("aggStore", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(metric) as AggregatorStore;
var emitOverflowAttribute = (bool)typeof(AggregatorStore).GetField("emitOverflowAttribute", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(aggregatorStore);
Assert.Equal(isEmitOverflowAttributeKeySet, emitOverflowAttribute);
}
finally
{
Environment.SetEnvironmentVariable(MetricTestsBase.EmitOverFlowAttributeConfigKey, null);
}
}
[Theory]
[InlineData(1, false)]
[InlineData(2, true)]
[InlineData(10, true)]
public void EmitOverflowAttributeIsOnlySetWhenMaxMetricPointsIsGreaterThanOne(int maxMetricPoints, bool isEmitOverflowAttributeKeySet)
{
try
{
Environment.SetEnvironmentVariable(MetricTestsBase.EmitOverFlowAttributeConfigKey, "true");
var exportedItems = new List<Metric>();
var meter = new Meter(Utils.GetCurrentMethodName());
var counter = meter.CreateCounter<long>("TestCounter");
using var meterProvider = Sdk.CreateMeterProviderBuilder()
.SetMaxMetricPointsPerMetricStream(maxMetricPoints)
.AddMeter(meter.Name)
.AddInMemoryExporter(exportedItems)
.Build();
counter.Add(10);
meterProvider.ForceFlush();
Assert.Single(exportedItems);
var metric = exportedItems[0];
var aggregatorStore = typeof(Metric).GetField("aggStore", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(metric) as AggregatorStore;
var emitOverflowAttribute = (bool)typeof(AggregatorStore).GetField("emitOverflowAttribute", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(aggregatorStore);
Assert.Equal(isEmitOverflowAttributeKeySet, emitOverflowAttribute);
}
finally
{
Environment.SetEnvironmentVariable(MetricTestsBase.EmitOverFlowAttributeConfigKey, null);
}
}
[Theory]
[InlineData(MetricReaderTemporalityPreference.Delta)]
[InlineData(MetricReaderTemporalityPreference.Cumulative)]
public void MetricOverflowAttributeIsRecordedCorrectlyForCounter(MetricReaderTemporalityPreference temporalityPreference)
{
try
{
Environment.SetEnvironmentVariable(MetricTestsBase.EmitOverFlowAttributeConfigKey, "true");
var exportedItems = new List<Metric>();
var meter = new Meter(Utils.GetCurrentMethodName());
var counter = meter.CreateCounter<long>("TestCounter");
using var meterProvider = Sdk.CreateMeterProviderBuilder()
.AddMeter(meter.Name)
.AddInMemoryExporter(exportedItems, metricReaderOptions => metricReaderOptions.TemporalityPreference = temporalityPreference)
.Build();
// There are two reserved MetricPoints
// 1. For zero tags
// 2. For metric overflow attribute when user opts-in for this feature
// Max number for MetricPoints available for use when emitted with tags
int maxMetricPointsForUse = MeterProviderBuilderSdk.MaxMetricPointsPerMetricDefault - 2;
for (int i = 0; i < maxMetricPointsForUse; i++)
{
// Emit unique key-value pairs to use up the available MetricPoints
// Once this loop is run, we have used up all available MetricPoints for metrics emitted with tags
counter.Add(10, new KeyValuePair<string, object>("Key", i));
}
meterProvider.ForceFlush();
Assert.Single(exportedItems);
var metric = exportedItems[0];
var metricPoints = new List<MetricPoint>();
foreach (ref readonly var mp in metric.GetMetricPoints())
{
metricPoints.Add(mp);
}
MetricPoint overflowMetricPoint;
// We still have not exceeded the max MetricPoint limit
Assert.DoesNotContain(metricPoints, mp => mp.Tags.KeyAndValues[0].Key == "otel.metric.overflow");
exportedItems.Clear();
metricPoints.Clear();
counter.Add(5, new KeyValuePair<string, object>("Key", 9999)); // Emit a metric to exceed the max MetricPoint limit
meterProvider.ForceFlush();
metric = exportedItems[0];
foreach (ref readonly var mp in metric.GetMetricPoints())
{
metricPoints.Add(mp);
}
overflowMetricPoint = metricPoints.Single(mp => mp.Tags.KeyAndValues[0].Key == "otel.metric.overflow");
Assert.Equal(true, overflowMetricPoint.Tags.KeyAndValues[0].Value);
Assert.Equal(1, overflowMetricPoint.Tags.Count);
Assert.Equal(5, overflowMetricPoint.GetSumLong());
exportedItems.Clear();
metricPoints.Clear();
// Emit 50 more newer MetricPoints with distinct dimension combinations
for (int i = 10000; i < 10050; i++)
{
counter.Add(5, new KeyValuePair<string, object>("Key", i));
}
meterProvider.ForceFlush();
metric = exportedItems[0];
foreach (ref readonly var mp in metric.GetMetricPoints())
{
metricPoints.Add(mp);
}
overflowMetricPoint = metricPoints.Single(mp => mp.Tags.KeyAndValues[0].Key == "otel.metric.overflow");
if (temporalityPreference == MetricReaderTemporalityPreference.Delta)
{
Assert.Equal(250, overflowMetricPoint.GetSumLong()); // 50 * 5
}
else
{
Assert.Equal(255, overflowMetricPoint.GetSumLong()); // 5 + (50 * 5)
}
exportedItems.Clear();
metricPoints.Clear();
// Test that the SDK continues to correctly aggregate the previously registered measurements even after overflow has occurred
counter.Add(15, new KeyValuePair<string, object>("Key", 0));
meterProvider.ForceFlush();
metric = exportedItems[0];
foreach (ref readonly var mp in metric.GetMetricPoints())
{
metricPoints.Add(mp);
}
var metricPoint = metricPoints.Single(mp => mp.Tags.KeyAndValues[0].Key == "Key" && (int)mp.Tags.KeyAndValues[0].Value == 0);
if (temporalityPreference == MetricReaderTemporalityPreference.Delta)
{
Assert.Equal(15, metricPoint.GetSumLong());
}
else
{
overflowMetricPoint = metricPoints.Single(mp => mp.Tags.KeyAndValues[0].Key == "otel.metric.overflow");
Assert.Equal(25, metricPoint.GetSumLong()); // 10 + 15
Assert.Equal(255, overflowMetricPoint.GetSumLong());
}
}
finally
{
Environment.SetEnvironmentVariable(MetricTestsBase.EmitOverFlowAttributeConfigKey, null);
}
}
[Theory]
[InlineData(MetricReaderTemporalityPreference.Delta)]
[InlineData(MetricReaderTemporalityPreference.Cumulative)]
public void MetricOverflowAttributeIsRecordedCorrectlyForHistogram(MetricReaderTemporalityPreference temporalityPreference)
{
try
{
Environment.SetEnvironmentVariable(MetricTestsBase.EmitOverFlowAttributeConfigKey, "true");
var exportedItems = new List<Metric>();
var meter = new Meter(Utils.GetCurrentMethodName());
var histogram = meter.CreateHistogram<long>("TestHistogram");
using var meterProvider = Sdk.CreateMeterProviderBuilder()
.AddMeter(meter.Name)
.AddInMemoryExporter(exportedItems, metricReaderOptions => metricReaderOptions.TemporalityPreference = temporalityPreference)
.Build();
// There are two reserved MetricPoints
// 1. For zero tags
// 2. For metric overflow attribute when user opts-in for this feature
// Max number for MetricPoints available for use when emitted with tags
int maxMetricPointsForUse = MeterProviderBuilderSdk.MaxMetricPointsPerMetricDefault - 2;
for (int i = 0; i < maxMetricPointsForUse; i++)
{
// Emit unique key-value pairs to use up the available MetricPoints
// Once this loop is run, we have used up all available MetricPoints for metrics emitted with tags
histogram.Record(10, new KeyValuePair<string, object>("Key", i));
}
meterProvider.ForceFlush();
Assert.Single(exportedItems);
var metric = exportedItems[0];
var metricPoints = new List<MetricPoint>();
foreach (ref readonly var mp in metric.GetMetricPoints())
{
metricPoints.Add(mp);
}
MetricPoint overflowMetricPoint;
// We still have not exceeded the max MetricPoint limit
Assert.DoesNotContain(metricPoints, mp => mp.Tags.KeyAndValues[0].Key == "otel.metric.overflow");
exportedItems.Clear();
metricPoints.Clear();
histogram.Record(5, new KeyValuePair<string, object>("Key", 9999)); // Emit a metric to exceed the max MetricPoint limit
meterProvider.ForceFlush();
metric = exportedItems[0];
foreach (ref readonly var mp in metric.GetMetricPoints())
{
metricPoints.Add(mp);
}
overflowMetricPoint = metricPoints.Single(mp => mp.Tags.KeyAndValues[0].Key == "otel.metric.overflow");
Assert.Equal(true, overflowMetricPoint.Tags.KeyAndValues[0].Value);
Assert.Equal(1, overflowMetricPoint.GetHistogramCount());
Assert.Equal(5, overflowMetricPoint.GetHistogramSum());
exportedItems.Clear();
metricPoints.Clear();
// Emit 50 more newer MetricPoints with distinct dimension combinations
for (int i = 10000; i < 10050; i++)
{
histogram.Record(5, new KeyValuePair<string, object>("Key", i));
}
meterProvider.ForceFlush();
metric = exportedItems[0];
foreach (ref readonly var mp in metric.GetMetricPoints())
{
metricPoints.Add(mp);
}
overflowMetricPoint = metricPoints.Single(mp => mp.Tags.KeyAndValues[0].Key == "otel.metric.overflow");
if (temporalityPreference == MetricReaderTemporalityPreference.Delta)
{
Assert.Equal(50, overflowMetricPoint.GetHistogramCount());
Assert.Equal(250, overflowMetricPoint.GetHistogramSum()); // 50 * 5
}
else
{
Assert.Equal(51, overflowMetricPoint.GetHistogramCount());
Assert.Equal(255, overflowMetricPoint.GetHistogramSum()); // 5 + (50 * 5)
}
exportedItems.Clear();
metricPoints.Clear();
// Test that the SDK continues to correctly aggregate the previously registered measurements even after overflow has occurred
histogram.Record(15, new KeyValuePair<string, object>("Key", 0));
meterProvider.ForceFlush();
metric = exportedItems[0];
foreach (ref readonly var mp in metric.GetMetricPoints())
{
metricPoints.Add(mp);
}
var metricPoint = metricPoints.Single(mp => mp.Tags.KeyAndValues[0].Key == "Key" && (int)mp.Tags.KeyAndValues[0].Value == 0);
if (temporalityPreference == MetricReaderTemporalityPreference.Delta)
{
Assert.Equal(1, metricPoint.GetHistogramCount());
Assert.Equal(15, metricPoint.GetHistogramSum());
}
else
{
overflowMetricPoint = metricPoints.Single(mp => mp.Tags.KeyAndValues[0].Key == "otel.metric.overflow");
Assert.Equal(2, metricPoint.GetHistogramCount());
Assert.Equal(25, metricPoint.GetHistogramSum()); // 10 + 15
Assert.Equal(255, overflowMetricPoint.GetHistogramSum());
}
}
finally
{
Environment.SetEnvironmentVariable(MetricTestsBase.EmitOverFlowAttributeConfigKey, null);
}
}
}

View File

@ -1,4 +1,4 @@
// <copyright file="MetricSnapshotTests.cs" company="OpenTelemetry Authors">
// <copyright file="MetricSnapshotTestsBase.cs" company="OpenTelemetry Authors">
// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
@ -22,8 +22,18 @@ using Xunit;
namespace OpenTelemetry.Metrics.Tests;
public class MetricSnapshotTests
#pragma warning disable SA1402
public abstract class MetricSnapshotTestsBase : IDisposable
{
protected MetricSnapshotTestsBase(bool emitOverflowAttribute)
{
if (emitOverflowAttribute)
{
Environment.SetEnvironmentVariable(MetricTestsBase.EmitOverFlowAttributeConfigKey, "true");
}
}
[Fact]
public void VerifySnapshot_Counter()
{
@ -87,7 +97,9 @@ public class MetricSnapshotTests
// Verify Snapshot 2
Assert.Equal(2, exportedSnapshots.Count);
var snapshot2 = exportedSnapshots[1];
Assert.Single(snapshot2.MetricPoints);
Assert.Equal(15, snapshot2.MetricPoints[0].GetSumLong());
}
@ -214,7 +226,7 @@ public class MetricSnapshotTests
metricPoint1.TryGetHistogramMinMaxValues(out var min, out var max);
Assert.Equal(10, min);
Assert.Equal(10, max);
AggregatorTest.AssertExponentialBucketsAreCorrect(expectedHistogram, metricPoint1.GetExponentialHistogramData());
AggregatorTestsBase.AssertExponentialBucketsAreCorrect(expectedHistogram, metricPoint1.GetExponentialHistogramData());
// Verify Snapshot 1
Assert.Single(exportedSnapshots);
@ -225,7 +237,7 @@ public class MetricSnapshotTests
snapshot1.MetricPoints[0].TryGetHistogramMinMaxValues(out min, out max);
Assert.Equal(10, min);
Assert.Equal(10, max);
AggregatorTest.AssertExponentialBucketsAreCorrect(expectedHistogram, snapshot1.MetricPoints[0].GetExponentialHistogramData());
AggregatorTestsBase.AssertExponentialBucketsAreCorrect(expectedHistogram, snapshot1.MetricPoints[0].GetExponentialHistogramData());
// Verify Metric == Snapshot
Assert.Equal(metric1.Name, snapshot1.Name);
@ -259,7 +271,7 @@ public class MetricSnapshotTests
metricPoint1.TryGetHistogramMinMaxValues(out min, out max);
Assert.Equal(5, min);
Assert.Equal(10, max);
AggregatorTest.AssertExponentialBucketsAreCorrect(expectedHistogram, metricPoint2.GetExponentialHistogramData());
AggregatorTestsBase.AssertExponentialBucketsAreCorrect(expectedHistogram, metricPoint2.GetExponentialHistogramData());
// Verify Snapshot 1 after second export
// This value is expected to be unchanged.
@ -278,6 +290,27 @@ public class MetricSnapshotTests
snapshot2.MetricPoints[0].TryGetHistogramMinMaxValues(out min, out max);
Assert.Equal(5, min);
Assert.Equal(10, max);
AggregatorTest.AssertExponentialBucketsAreCorrect(expectedHistogram, snapshot2.MetricPoints[0].GetExponentialHistogramData());
AggregatorTestsBase.AssertExponentialBucketsAreCorrect(expectedHistogram, snapshot2.MetricPoints[0].GetExponentialHistogramData());
}
public void Dispose()
{
Environment.SetEnvironmentVariable(MetricTestsBase.EmitOverFlowAttributeConfigKey, null);
}
}
public class MetricSnapshotTests : MetricSnapshotTestsBase
{
public MetricSnapshotTests()
: base(false)
{
}
}
public class MetricSnapshotTestsWithOverflowAttribute : MetricSnapshotTestsBase
{
public MetricSnapshotTestsWithOverflowAttribute()
: base(true)
{
}
}

View File

@ -20,6 +20,8 @@ namespace OpenTelemetry.Metrics.Tests;
public class MetricTestsBase
{
public const string EmitOverFlowAttributeConfigKey = "OTEL_DOTNET_EXPERIMENTAL_METRICS_EMIT_OVERFLOW_ATTRIBUTE";
public static void ValidateMetricPointTags(List<KeyValuePair<string, object>> expectedTags, ReadOnlyTagCollection actualTags)
{
int tagIndex = 0;

View File

@ -636,7 +636,7 @@ public class MetricViewTests : MetricTestsBase
var count = metricPoint.GetHistogramCount();
var sum = metricPoint.GetHistogramSum();
AggregatorTest.AssertExponentialBucketsAreCorrect(expectedHistogram, metricPoint.GetExponentialHistogramData());
AggregatorTestsBase.AssertExponentialBucketsAreCorrect(expectedHistogram, metricPoint.GetExponentialHistogramData());
Assert.Equal(50, sum);
Assert.Equal(6, count);
}