Merge main-metrics to main (#4217)

Co-authored-by: Yun-Ting Lin <yunl@microsoft.com>

.gitignore

@@ -345,3 +345,6 @@ ASALocalRun/
 /.sonarqube
 
 /src/LastMajorVersionBinaries
+
+# Tempo files
+tempo-data/

build/Common.prod.props

@@ -6,6 +6,7 @@
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
+
     <PackageReference Include="Microsoft.CodeAnalysis.PublicApiAnalyzers" Version="$(MicrosoftCodeAnalysisAnalyzersPkgVer)" Condition=" $(OS) == 'Windows_NT'">
       <PrivateAssets>All</PrivateAssets>
     </PackageReference>

docs/metrics/customizing-the-sdk/README.md

@@ -403,6 +403,61 @@ AnotherFruitCounter.Add(4, new("name", "mango"), new("color", "yellow")); // Not
 streams. There is no ability to apply different limits for each instrument at
 this moment.
 
+### Exemplars
+
+Exemplars are example data points for aggregated data. They provide access to
+the raw measurement value, time stamp when measurement was made, and trace
+context, if any. It also provides "Filtered Tags", which are attributes (Tags)
+that are [dropped by a view](#select-specific-tags). Exemplars are an opt-in
+feature, and allow customization via ExemplarFilter and ExemplarReservoir.
+
+#### ExemplarFilter
+
+`ExemplarFilter` determines which measurements are eligible to become an
+Exemplar. i.e. `ExemplarFilter` determines which measurements are offered to
+`ExemplarReservoir`, which makes the final decision about whether the offered
+measurement gets stored as an exemplar. They can be used to control the noise
+and overhead associated with Exemplar collection.
+
+OpenTelemetry SDK comes with the following Filters:
+
+* `AlwaysOnExemplarFilter` - makes all measurements eligible for being an Exemplar.
+* `AlwaysOffExemplarFilter` - makes no measurements eligible for being an
+  Exemplar. Use this to turn-off Exemplar feature.
+* `TraceBasedExemplarFilter` - makes those measurements eligible for being an
+Exemplar, which are recorded in the context of a sampled parent `Activity`
+(span).
+
+`SetExemplarFilter` method on `MeterProviderBuilder` can be used to set the
+desired `ExemplarFilter`.
+
+The snippet below shows how to set `ExemplarFilter`.
+
+```csharp
+using OpenTelemetry;
+using OpenTelemetry.Metrics;
+
+using var meterProvider = Sdk.CreateMeterProviderBuilder()
+    // rest of config not shown
+    .SetExemplarFilter(new TraceBasedExemplarFilter())
+    .Build();
+```
+
+> **Note**
+> As of today, there is no separate toggle for enable/disable Exemplar
+feature. It can be turned off by using `AlwaysOffExemplarFilter`.
+
+#### ExemplarReservoir
+
+`ExemplarReservoir` receives the measurements sampled in by the `ExemplarFilter`
+and is responsible for storing Exemplars.
+`AlignedHistogramBucketExemplarReservoir` is the default reservoir used for
+Histograms with buckets, and it stores one exemplar per histogram bucket. The
+exemplar stored is the last measurement recorded - i.e. any new measurement
+overwrites the previous one.
+
+Currently there is no ability to change the Reservoir used.
+
 ### Instrumentation
 
 // TODO

docs/metrics/exemplars/README.md

@@ -0,0 +1,99 @@
+# Using Exemplars in OpenTelemetry .NET
+
+Exemplars are example data points for aggregated data. They provide specific
+context to otherwise general aggregations. One common use case is to gain
+ability to correlate metrics to traces (and logs). While OpenTelemetry .NET
+supports Exemplars, it is only useful if the telemetry backend also supports the
+capabilities. This tutorial uses well known open source backends to demonstrate
+the concept. The following are the components involved:
+
+* Test App - We use existing example app from the repo. This app is already
+instrumented with OpenTelemetry for logs, metrics and traces, and is configured
+to export them to the configured OTLP end point.
+* OpenTelemetry Collector - An instance of collector is run, which receives
+telemetry from the above app using OTLP. The collector then exports metrics to
+Prometheus, traces to Tempo.
+* Prometheus - Prometheus is used as the Metric backend.
+* Tempo - Tempo is used as the Tracing backend.
+* Grafana - UI to query metrics from Prometheus, traces from Tempo, and to
+  navigate between metrics and traces using Exemplar.
+
+All these components except the test app require additional configuration to
+enable Exemplar feature. To make it easy for users, these components are
+pre-configured to enable Exemplars, and a docker-compose is provided to spun
+ them all up, in the required configurations.
+
+## Pre-requisite
+
+Install docker: <https://docs.docker.com/get-docker/>
+
+## Setup
+
+As mentioned in the intro, this tutorial uses OTel Collector, Prometheus, Tempo,
+and Grafana, and they must be up and running before proceeding. The following
+spins all of them with the correct configurations to support Exemplars.
+
+Navigate to current directory and run the following:
+
+```sh
+docker-compose up -d
+```
+
+If the above step succeeds, all dependencies would be spun up and ready now. To
+test, navigate to Grafana running at: "http://localhost:3000/".
+
+## Run test app
+
+Now that the required dependencies are ready, lets run the demo app.
+This tutorial is using the existing ASP.NET Core app from the repo.
+
+Navigate to [Example Asp.Net Core App](../../../examples/AspNetCore/Program.cs)
+directory and run the following command:
+
+```sh
+dotnet run
+```
+
+Once the application is running, navigate to
+[http://localhost:5000/weatherforecast]("http://localhost:5000/weatherforecast")
+from a web browser. You may use the following Powershell script to generate load
+to the application.
+
+```powershell
+while($true)
+{
+    Invoke-WebRequest http://localhost:5000/weatherforecast
+    Start-Sleep -Milliseconds 500
+}
+```
+
+## Use Exemplars to navigate from Metrics to Traces
+
+The application sends metrics (with exemplars), and traces to the OTel
+Collector, which export metrics and traces to Prometheus and Tempo
+respectively.
+
+Please wait for 2 minutes before continuing so that enough data is generated
+and exported.
+
+Open Grafana, select Explore, and select Prometheus as the source. Select the
+metric named "http_server_duration_bucket", and plot the chart. Toggle on the
+"Exemplar" option from the UI and hit refresh.
+
+![Enable Exemplar](https://user-images.githubusercontent.com/16979322/218627781-9886f837-11ae-4d52-94d3-f1821503209c.png)
+
+The Exemplars appear as special "diamond shaped dots" along with the metric
+charts in the UI. Select any Exemplar to see the exemplar data, which includes
+the timestamp when the measurement was recorded, the raw value, and trace
+context when the recording was done. The "trace_id" enables jumping to the
+tracing backed (tempo). Click on the "Query with Tempo" button next to the
+"trace_id" field to open the corresponding `Trace` in Tempo.
+
+![Navigate to trace with exemplar](https://user-images.githubusercontent.com/16979322/218629999-1d1cd6ba-2385-4683-975a-d4797df8361a.png)
+
+## References
+
+* [Exemplar specification](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/sdk.md#exemplar)
+* [Exemplars in Prometheus](https://prometheus.io/docs/prometheus/latest/feature_flags/#exemplars-storage)
+* [Exemplars in Grafana](https://grafana.com/docs/grafana/latest/fundamentals/exemplars/)
+* [Tempo](https://github.com/grafana/tempo)

docs/metrics/exemplars/docker-compose.yaml

@@ -0,0 +1,51 @@
+version: "3"
+services:
+
+  # OTEL Collector to receive logs, metrics and traces from the application
+  otel-collector:
+    image: otel/opentelemetry-collector:0.70.0
+    command: [ "--config=/etc/otel-collector.yaml" ]
+    volumes:
+      - ./otel-collector.yaml:/etc/otel-collector.yaml
+    ports:
+      - "4317:4317"
+      - "4318:4318"
+      - "9201:9201"
+
+  # Exports Traces to Tempo
+  tempo:
+    image: grafana/tempo:latest
+    command: [ "-config.file=/etc/tempo.yaml" ]
+    volumes:
+      - ./tempo.yaml:/etc/tempo.yaml
+      - ./tempo-data:/tmp/tempo
+    ports:
+      - "3200"  # tempo
+      - "4317"  # otlp grpc
+      - "4318"  # otlp http
+
+# Exports Metrics to Prometheus
+  prometheus:
+    image: prom/prometheus:latest
+    command:
+      - --config.file=/etc/prometheus.yaml
+      - --web.enable-remote-write-receiver
+      - --enable-feature=exemplar-storage
+    volumes:
+      - ./prometheus.yaml:/etc/prometheus.yaml
+    ports:
+      - "9090:9090"
+
+# UI to query traces and metrics
+  grafana:
+    image: grafana/grafana:9.3.2
+    volumes:
+      - ./grafana-datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml
+    environment:
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+      - GF_AUTH_DISABLE_LOGIN_FORM=true
+      - GF_FEATURE_TOGGLES_ENABLE=traceqlEditor
+    ports:
+      - "3000:3000"
+

docs/metrics/exemplars/grafana-datasources.yaml

@@ -0,0 +1,33 @@
+apiVersion: 1
+
+datasources:
+- name: Prometheus
+  type: prometheus
+  uid: prometheus
+  access: proxy
+  orgId: 1
+  url: http://prometheus:9090
+  basicAuth: false
+  isDefault: true
+  version: 1
+  editable: false
+  jsonData:
+    httpMethod: GET
+    exemplarTraceIdDestinations:
+    - name: trace_id
+      datasourceUid: Tempo
+- name: Tempo
+  type: tempo
+  access: proxy
+  orgId: 1
+  url: http://tempo:3200
+  basicAuth: false
+  isDefault: false
+  version: 1
+  editable: false
+  apiVersion: 1
+  uid: tempo
+  jsonData:
+    httpMethod: GET
+    serviceMap:
+      datasourceUid: prometheus

docs/metrics/exemplars/otel-collector.yaml

@@ -0,0 +1,30 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+      http:
+
+exporters:
+  logging:
+    loglevel: debug
+  prometheus:
+    endpoint: ":9201"
+    send_timestamps: true
+    metric_expiration: 180m
+    enable_open_metrics: true
+  otlp:
+    endpoint: tempo:4317
+    tls:
+      insecure: true
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      exporters: [logging,otlp]
+    metrics:
+      receivers: [otlp]
+      exporters: [logging,prometheus]
+    logs:
+      receivers: [otlp]
+      exporters: [logging]

docs/metrics/exemplars/prometheus.yaml

@@ -0,0 +1,8 @@
+global:
+  scrape_interval:     15s
+  evaluation_interval: 15s
+
+scrape_configs:
+  - job_name: 'otel'
+    static_configs:
+      - targets: [ 'otel-collector:9201' ]

docs/metrics/exemplars/tempo.yaml

@@ -0,0 +1,17 @@
+server:
+  http_listen_port: 3200
+
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        http:
+        grpc:
+
+storage:
+  trace:
+    backend: local
+    wal:
+      path: /tmp/tempo/wal
+    local:
+      path: /tmp/tempo/blocks

examples/AspNetCore/Program.cs

@@ -106,6 +106,7 @@ appBuilder.Services.AddOpenTelemetry()
         // Ensure the MeterProvider subscribes to any custom Meters.
         builder
             .AddMeter(Instrumentation.MeterName)
+            .SetExemplarFilter(new TraceBasedExemplarFilter())
             .AddRuntimeInstrumentation()
             .AddHttpClientInstrumentation()
             .AddAspNetCoreInstrumentation();

examples/AspNetCore/appsettings.json

@@ -28,5 +28,15 @@
   },
   "AspNetCoreInstrumentation": {
     "RecordException": "true"
+  },
+  "Kestrel": {
+    "Endpoints": {
+      "Http": {
+        "Url": "http://localhost:5000"
+      },
+      "Https": {
+        "Url": "https://localhost:5001"
+      }
+    }
   }
 }

src/OpenTelemetry.Exporter.Console/ConsoleMetricExporter.cs

@@ -167,6 +167,38 @@ namespace OpenTelemetry.Exporter
                         }
                     }
 
+                    var exemplarString = new StringBuilder();
+                    foreach (var exemplar in metricPoint.GetExemplars())
+                    {
+                        if (exemplar.Timestamp != default)
+                        {
+                            exemplarString.Append("Value: ");
+                            exemplarString.Append(exemplar.DoubleValue);
+                            exemplarString.Append(" Timestamp: ");
+                            exemplarString.Append(exemplar.Timestamp.ToString("yyyy-MM-ddTHH:mm:ss.fffffffZ", CultureInfo.InvariantCulture));
+                            exemplarString.Append(" TraceId: ");
+                            exemplarString.Append(exemplar.TraceId);
+                            exemplarString.Append(" SpanId: ");
+                            exemplarString.Append(exemplar.SpanId);
+
+                            if (exemplar.FilteredTags != null && exemplar.FilteredTags.Count > 0)
+                            {
+                                exemplarString.Append(" Filtered Tags : ");
+
+                                foreach (var tag in exemplar.FilteredTags)
+                                {
+                                    if (ConsoleTagTransformer.Instance.TryTransformTag(tag, out var result))
+                                    {
+                                        exemplarString.Append(result);
+                                        exemplarString.Append(' ');
+                                    }
+                                }
+                            }
+
+                            exemplarString.AppendLine();
+                        }
+                    }
+
                     msg = new StringBuilder();
                     msg.Append('(');
                     msg.Append(metricPoint.StartTime.ToString("yyyy-MM-ddTHH:mm:ss.fffffffZ", CultureInfo.InvariantCulture));
@@ -182,6 +214,14 @@ namespace OpenTelemetry.Exporter
                     msg.Append(metric.MetricType);
                     msg.AppendLine();
                     msg.Append($"Value: {valueDisplay}");
+
+                    if (exemplarString.Length > 0)
+                    {
+                        msg.AppendLine();
+                        msg.AppendLine("Exemplars");
+                        msg.Append(exemplarString.ToString());
+                    }
+
                     this.WriteLine(msg.ToString());
                 }
             }

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/MetricItemExtensions.cs

@@ -16,6 +16,7 @@
 
 using System.Collections.Concurrent;
 using System.Runtime.CompilerServices;
+using Google.Protobuf;
 using Google.Protobuf.Collections;
 using OpenTelemetry.Metrics;
 using OtlpCollector = OpenTelemetry.Proto.Collector.Metrics.V1;
@@ -266,6 +267,40 @@ namespace OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation
                                 }
                             }
 
+                            var exemplars = metricPoint.GetExemplars();
+                            foreach (var examplar in exemplars)
+                            {
+                                if (examplar.Timestamp != default)
+                                {
+                                    byte[] traceIdBytes = new byte[16];
+                                    examplar.TraceId?.CopyTo(traceIdBytes);
+
+                                    byte[] spanIdBytes = new byte[8];
+                                    examplar.SpanId?.CopyTo(spanIdBytes);
+
+                                    var otlpExemplar = new OtlpMetrics.Exemplar
+                                    {
+                                        TimeUnixNano = (ulong)examplar.Timestamp.ToUnixTimeNanoseconds(),
+                                        TraceId = UnsafeByteOperations.UnsafeWrap(traceIdBytes),
+                                        SpanId = UnsafeByteOperations.UnsafeWrap(spanIdBytes),
+                                        AsDouble = examplar.DoubleValue,
+                                    };
+
+                                    if (examplar.FilteredTags != null)
+                                    {
+                                        foreach (var tag in examplar.FilteredTags)
+                                        {
+                                            if (OtlpKeyValueTransformer.Instance.TryTransformTag(tag, out var result))
+                                            {
+                                                otlpExemplar.FilteredAttributes.Add(result);
+                                            }
+                                        }
+                                    }
+
+                                    dataPoint.Exemplars.Add(otlpExemplar);
+                                }
+                            }
+
                             histogram.DataPoints.Add(dataPoint);
                         }
 

src/OpenTelemetry/.publicApi/net462/PublicAPI.Unshipped.txt

@@ -0,0 +1,25 @@
+OpenTelemetry.Metrics.AlwaysOffExemplarFilter
+OpenTelemetry.Metrics.AlwaysOffExemplarFilter.AlwaysOffExemplarFilter() -> void
+OpenTelemetry.Metrics.AlwaysOnExemplarFilter
+OpenTelemetry.Metrics.AlwaysOnExemplarFilter.AlwaysOnExemplarFilter() -> void
+OpenTelemetry.Metrics.Exemplar
+OpenTelemetry.Metrics.Exemplar.DoubleValue.get -> double
+OpenTelemetry.Metrics.Exemplar.Exemplar() -> void
+OpenTelemetry.Metrics.Exemplar.SpanId.get -> System.Diagnostics.ActivitySpanId?
+OpenTelemetry.Metrics.Exemplar.Timestamp.get -> System.DateTime
+OpenTelemetry.Metrics.Exemplar.TraceId.get -> System.Diagnostics.ActivityTraceId?
+OpenTelemetry.Metrics.ExemplarFilter
+OpenTelemetry.Metrics.ExemplarFilter.ExemplarFilter() -> void
+OpenTelemetry.Metrics.TraceBasedExemplarFilter
+OpenTelemetry.Metrics.TraceBasedExemplarFilter.TraceBasedExemplarFilter() -> void
+static OpenTelemetry.Metrics.MeterProviderBuilderExtensions.SetExemplarFilter(this OpenTelemetry.Metrics.MeterProviderBuilder! meterProviderBuilder, OpenTelemetry.Metrics.ExemplarFilter! exemplarFilter) -> OpenTelemetry.Metrics.MeterProviderBuilder!
+~abstract OpenTelemetry.Metrics.ExemplarFilter.ShouldSample(double value, System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string, object>> tags) -> bool
+~abstract OpenTelemetry.Metrics.ExemplarFilter.ShouldSample(long value, System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string, object>> tags) -> bool
+~OpenTelemetry.Metrics.Exemplar.FilteredTags.get -> System.Collections.Generic.List<System.Collections.Generic.KeyValuePair<string, object>>
+~OpenTelemetry.Metrics.MetricPoint.GetExemplars() -> OpenTelemetry.Metrics.Exemplar[]
+~override OpenTelemetry.Metrics.AlwaysOffExemplarFilter.ShouldSample(double value, System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string, object>> tags) -> bool
+~override OpenTelemetry.Metrics.AlwaysOffExemplarFilter.ShouldSample(long value, System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string, object>> tags) -> bool
+~override OpenTelemetry.Metrics.AlwaysOnExemplarFilter.ShouldSample(double value, System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string, object>> tags) -> bool
+~override OpenTelemetry.Metrics.AlwaysOnExemplarFilter.ShouldSample(long value, System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string, object>> tags) -> bool
+~override OpenTelemetry.Metrics.TraceBasedExemplarFilter.ShouldSample(double value, System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string, object>> tags) -> bool
+~override OpenTelemetry.Metrics.TraceBasedExemplarFilter.ShouldSample(long value, System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string, object>> tags) -> bool

src/OpenTelemetry/.publicApi/net6.0/PublicAPI.Unshipped.txt

@@ -0,0 +1,25 @@
+OpenTelemetry.Metrics.AlwaysOffExemplarFilter
+OpenTelemetry.Metrics.AlwaysOffExemplarFilter.AlwaysOffExemplarFilter() -> void
+OpenTelemetry.Metrics.AlwaysOnExemplarFilter
+OpenTelemetry.Metrics.AlwaysOnExemplarFilter.AlwaysOnExemplarFilter() -> void
+OpenTelemetry.Metrics.Exemplar
+OpenTelemetry.Metrics.Exemplar.DoubleValue.get -> double
+OpenTelemetry.Metrics.Exemplar.Exemplar() -> void
+OpenTelemetry.Metrics.Exemplar.SpanId.get -> System.Diagnostics.ActivitySpanId?
+OpenTelemetry.Metrics.Exemplar.Timestamp.get -> System.DateTime
+OpenTelemetry.Metrics.Exemplar.TraceId.get -> System.Diagnostics.ActivityTraceId?
+OpenTelemetry.Metrics.ExemplarFilter
+OpenTelemetry.Metrics.ExemplarFilter.ExemplarFilter() -> void
+OpenTelemetry.Metrics.TraceBasedExemplarFilter
+OpenTelemetry.Metrics.TraceBasedExemplarFilter.TraceBasedExemplarFilter() -> void
+static OpenTelemetry.Metrics.MeterProviderBuilderExtensions.SetExemplarFilter(this OpenTelemetry.Metrics.MeterProviderBuilder! meterProviderBuilder, OpenTelemetry.Metrics.ExemplarFilter! exemplarFilter) -> OpenTelemetry.Metrics.MeterProviderBuilder!
+~abstract OpenTelemetry.Metrics.ExemplarFilter.ShouldSample(double value, System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string, object>> tags) -> bool
+~abstract OpenTelemetry.Metrics.ExemplarFilter.ShouldSample(long value, System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string, object>> tags) -> bool
+~OpenTelemetry.Metrics.Exemplar.FilteredTags.get -> System.Collections.Generic.List<System.Collections.Generic.KeyValuePair<string, object>>
+~OpenTelemetry.Metrics.MetricPoint.GetExemplars() -> OpenTelemetry.Metrics.Exemplar[]
+~override OpenTelemetry.Metrics.AlwaysOffExemplarFilter.ShouldSample(double value, System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string, object>> tags) -> bool
+~override OpenTelemetry.Metrics.AlwaysOffExemplarFilter.ShouldSample(long value, System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string, object>> tags) -> bool
+~override OpenTelemetry.Metrics.AlwaysOnExemplarFilter.ShouldSample(double value, System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string, object>> tags) -> bool
+~override OpenTelemetry.Metrics.AlwaysOnExemplarFilter.ShouldSample(long value, System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string, object>> tags) -> bool
+~override OpenTelemetry.Metrics.TraceBasedExemplarFilter.ShouldSample(double value, System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string, object>> tags) -> bool
+~override OpenTelemetry.Metrics.TraceBasedExemplarFilter.ShouldSample(long value, System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string, object>> tags) -> bool

src/OpenTelemetry/.publicApi/netstandard2.0/PublicAPI.Unshipped.txt

@@ -0,0 +1,25 @@
+OpenTelemetry.Metrics.AlwaysOffExemplarFilter
+OpenTelemetry.Metrics.AlwaysOffExemplarFilter.AlwaysOffExemplarFilter() -> void
+OpenTelemetry.Metrics.AlwaysOnExemplarFilter
+OpenTelemetry.Metrics.AlwaysOnExemplarFilter.AlwaysOnExemplarFilter() -> void
+OpenTelemetry.Metrics.Exemplar
+OpenTelemetry.Metrics.Exemplar.DoubleValue.get -> double
+OpenTelemetry.Metrics.Exemplar.Exemplar() -> void
+OpenTelemetry.Metrics.Exemplar.SpanId.get -> System.Diagnostics.ActivitySpanId?
+OpenTelemetry.Metrics.Exemplar.Timestamp.get -> System.DateTime
+OpenTelemetry.Metrics.Exemplar.TraceId.get -> System.Diagnostics.ActivityTraceId?
+OpenTelemetry.Metrics.ExemplarFilter
+OpenTelemetry.Metrics.ExemplarFilter.ExemplarFilter() -> void
+OpenTelemetry.Metrics.TraceBasedExemplarFilter
+OpenTelemetry.Metrics.TraceBasedExemplarFilter.TraceBasedExemplarFilter() -> void
+static OpenTelemetry.Metrics.MeterProviderBuilderExtensions.SetExemplarFilter(this OpenTelemetry.Metrics.MeterProviderBuilder! meterProviderBuilder, OpenTelemetry.Metrics.ExemplarFilter! exemplarFilter) -> OpenTelemetry.Metrics.MeterProviderBuilder!
+~abstract OpenTelemetry.Metrics.ExemplarFilter.ShouldSample(double value, System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string, object>> tags) -> bool
+~abstract OpenTelemetry.Metrics.ExemplarFilter.ShouldSample(long value, System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string, object>> tags) -> bool
+~OpenTelemetry.Metrics.Exemplar.FilteredTags.get -> System.Collections.Generic.List<System.Collections.Generic.KeyValuePair<string, object>>
+~OpenTelemetry.Metrics.MetricPoint.GetExemplars() -> OpenTelemetry.Metrics.Exemplar[]
+~override OpenTelemetry.Metrics.AlwaysOffExemplarFilter.ShouldSample(double value, System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string, object>> tags) -> bool
+~override OpenTelemetry.Metrics.AlwaysOffExemplarFilter.ShouldSample(long value, System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string, object>> tags) -> bool
+~override OpenTelemetry.Metrics.AlwaysOnExemplarFilter.ShouldSample(double value, System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string, object>> tags) -> bool
+~override OpenTelemetry.Metrics.AlwaysOnExemplarFilter.ShouldSample(long value, System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string, object>> tags) -> bool
+~override OpenTelemetry.Metrics.TraceBasedExemplarFilter.ShouldSample(double value, System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string, object>> tags) -> bool
+~override OpenTelemetry.Metrics.TraceBasedExemplarFilter.ShouldSample(long value, System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string, object>> tags) -> bool

src/OpenTelemetry/.publicApi/netstandard2.1/PublicAPI.Unshipped.txt

@@ -0,0 +1,25 @@
+OpenTelemetry.Metrics.AlwaysOffExemplarFilter
+OpenTelemetry.Metrics.AlwaysOffExemplarFilter.AlwaysOffExemplarFilter() -> void
+OpenTelemetry.Metrics.AlwaysOnExemplarFilter
+OpenTelemetry.Metrics.AlwaysOnExemplarFilter.AlwaysOnExemplarFilter() -> void
+OpenTelemetry.Metrics.Exemplar
+OpenTelemetry.Metrics.Exemplar.DoubleValue.get -> double
+OpenTelemetry.Metrics.Exemplar.Exemplar() -> void
+OpenTelemetry.Metrics.Exemplar.SpanId.get -> System.Diagnostics.ActivitySpanId?
+OpenTelemetry.Metrics.Exemplar.Timestamp.get -> System.DateTime
+OpenTelemetry.Metrics.Exemplar.TraceId.get -> System.Diagnostics.ActivityTraceId?
+OpenTelemetry.Metrics.ExemplarFilter
+OpenTelemetry.Metrics.ExemplarFilter.ExemplarFilter() -> void
+OpenTelemetry.Metrics.TraceBasedExemplarFilter
+OpenTelemetry.Metrics.TraceBasedExemplarFilter.TraceBasedExemplarFilter() -> void
+static OpenTelemetry.Metrics.MeterProviderBuilderExtensions.SetExemplarFilter(this OpenTelemetry.Metrics.MeterProviderBuilder! meterProviderBuilder, OpenTelemetry.Metrics.ExemplarFilter! exemplarFilter) -> OpenTelemetry.Metrics.MeterProviderBuilder!
+~abstract OpenTelemetry.Metrics.ExemplarFilter.ShouldSample(double value, System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string, object>> tags) -> bool
+~abstract OpenTelemetry.Metrics.ExemplarFilter.ShouldSample(long value, System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string, object>> tags) -> bool
+~OpenTelemetry.Metrics.Exemplar.FilteredTags.get -> System.Collections.Generic.List<System.Collections.Generic.KeyValuePair<string, object>>
+~OpenTelemetry.Metrics.MetricPoint.GetExemplars() -> OpenTelemetry.Metrics.Exemplar[]
+~override OpenTelemetry.Metrics.AlwaysOffExemplarFilter.ShouldSample(double value, System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string, object>> tags) -> bool
+~override OpenTelemetry.Metrics.AlwaysOffExemplarFilter.ShouldSample(long value, System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string, object>> tags) -> bool
+~override OpenTelemetry.Metrics.AlwaysOnExemplarFilter.ShouldSample(double value, System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string, object>> tags) -> bool
+~override OpenTelemetry.Metrics.AlwaysOnExemplarFilter.ShouldSample(long value, System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string, object>> tags) -> bool
+~override OpenTelemetry.Metrics.TraceBasedExemplarFilter.ShouldSample(double value, System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string, object>> tags) -> bool
+~override OpenTelemetry.Metrics.TraceBasedExemplarFilter.ShouldSample(long value, System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string, object>> tags) -> bool

src/OpenTelemetry/Metrics/AggregatorStore.cs

@@ -41,6 +41,7 @@ namespace OpenTelemetry.Metrics
         private readonly UpdateLongDelegate updateLongCallback;
         private readonly UpdateDoubleDelegate updateDoubleCallback;
         private readonly int maxMetricPoints;
+        private readonly ExemplarFilter exemplarFilter;
         private int metricPointIndex = 0;
         private int batchSize = 0;
         private int metricCapHitMessageLogged;
@@ -52,7 +53,8 @@ namespace OpenTelemetry.Metrics
             AggregationTemporality temporality,
             int maxMetricPoints,
             double[] histogramBounds,
-            string[] tagKeysInteresting = null)
+            string[] tagKeysInteresting = null,
+            ExemplarFilter exemplarFilter = null)
         {
             this.name = name;
             this.maxMetricPoints = maxMetricPoints;
@@ -63,6 +65,8 @@ namespace OpenTelemetry.Metrics
             this.outputDelta = temporality == AggregationTemporality.Delta;
             this.histogramBounds = histogramBounds;
             this.StartTimeExclusive = DateTimeOffset.UtcNow;
+
+            this.exemplarFilter = exemplarFilter ?? new AlwaysOffExemplarFilter();
             if (tagKeysInteresting == null)
             {
                 this.updateLongCallback = this.UpdateLong;
@@ -86,6 +90,13 @@ namespace OpenTelemetry.Metrics
 
         internal DateTimeOffset EndTimeInclusive { get; private set; }
 
+        internal bool IsExemplarEnabled()
+        {
+            // Using this filter to indicate On/Off
+            // instead of another separate flag.
+            return this.exemplarFilter is not AlwaysOffExemplarFilter;
+        }
+
         internal void Update(long value, ReadOnlySpan<KeyValuePair<string, object>> tags)
         {
             this.updateLongCallback(value, tags);
@@ -309,7 +320,15 @@ namespace OpenTelemetry.Metrics
                     return;
                 }
 
-                this.metricPoints[index].Update(value);
+                // TODO: can special case built-in filters to be bit faster.
+                if (this.exemplarFilter.ShouldSample(value, tags))
+                {
+                    this.metricPoints[index].UpdateWithExemplar(value, tags: default);
+                }
+                else
+                {
+                    this.metricPoints[index].Update(value);
+                }
             }
             catch (Exception)
             {
@@ -332,7 +351,15 @@ namespace OpenTelemetry.Metrics
                     return;
                 }
 
-                this.metricPoints[index].Update(value);
+                // TODO: can special case built-in filters to be bit faster.
+                if (this.exemplarFilter.ShouldSample(value, tags))
+                {
+                    this.metricPoints[index].UpdateWithExemplar(value, tags);
+                }
+                else
+                {
+                    this.metricPoints[index].Update(value);
+                }
             }
             catch (Exception)
             {
@@ -355,7 +382,15 @@ namespace OpenTelemetry.Metrics
                     return;
                 }
 
-                this.metricPoints[index].Update(value);
+                // TODO: can special case built-in filters to be bit faster.
+                if (this.exemplarFilter.ShouldSample(value, tags))
+                {
+                    this.metricPoints[index].UpdateWithExemplar(value, tags: default);
+                }
+                else
+                {
+                    this.metricPoints[index].Update(value);
+                }
             }
             catch (Exception)
             {
@@ -378,7 +413,15 @@ namespace OpenTelemetry.Metrics
                     return;
                 }
 
-                this.metricPoints[index].Update(value);
+                // TODO: can special case built-in filters to be bit faster.
+                if (this.exemplarFilter.ShouldSample(value, tags))
+                {
+                    this.metricPoints[index].UpdateWithExemplar(value, tags);
+                }
+                else
+                {
+                    this.metricPoints[index].Update(value);
+                }
             }
             catch (Exception)
             {

src/OpenTelemetry/Metrics/Builder/MeterProviderBuilderExtensions.cs

@@ -303,6 +303,26 @@ namespace OpenTelemetry.Metrics
             return meterProviderBuilder;
         }
 
+        /// <summary>
+        /// Sets the <see cref="ExemplarFilter"/> to be used for this provider.
+        /// This is applied to all the metrics from this provider.
+        /// </summary>
+        /// <param name="meterProviderBuilder"><see cref="MeterProviderBuilder"/>.</param>
+        /// <param name="exemplarFilter"><see cref="ExemplarFilter"/> ExemplarFilter to use.</param>
+        /// <returns>The supplied <see cref="MeterProviderBuilder"/> for chaining.</returns>
+        public static MeterProviderBuilder SetExemplarFilter(this MeterProviderBuilder meterProviderBuilder, ExemplarFilter exemplarFilter)
+        {
+            meterProviderBuilder.ConfigureBuilder((sp, builder) =>
+            {
+                if (builder is MeterProviderBuilderSdk meterProviderBuilderSdk)
+                {
+                    meterProviderBuilderSdk.SetExemplarFilter(exemplarFilter);
+                }
+            });
+
+            return meterProviderBuilder;
+        }
+
         /// <summary>
         /// Run the given actions to initialize the <see cref="MeterProvider"/>.
         /// </summary>

src/OpenTelemetry/Metrics/Builder/MeterProviderBuilderSdk.cs

@@ -54,6 +54,8 @@ namespace OpenTelemetry.Metrics
 
         public ResourceBuilder? ResourceBuilder { get; private set; }
 
+        public ExemplarFilter? ExemplarFilter { get; private set; }
+
         public MeterProvider? Provider => this.meterProvider;
 
         public List<MetricReader> Readers { get; } = new();
@@ -160,6 +162,15 @@ namespace OpenTelemetry.Metrics
             return this;
         }
 
+        public MeterProviderBuilder SetExemplarFilter(ExemplarFilter exemplarFilter)
+        {
+            Debug.Assert(exemplarFilter != null, "exemplarFilter was null");
+
+            this.ExemplarFilter = exemplarFilter;
+
+            return this;
+        }
+
         public override MeterProviderBuilder AddMeter(params string[] names)
         {
             Debug.Assert(names != null, "names was null");

src/OpenTelemetry/Metrics/Exemplar/AlignedHistogramBucketExemplarReservoir.cs

@@ -0,0 +1,112 @@
+// <copyright file="AlignedHistogramBucketExemplarReservoir.cs" company="OpenTelemetry Authors">
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// </copyright>
+
+using System.Diagnostics;
+
+namespace OpenTelemetry.Metrics;
+
+/// <summary>
+/// The AlignedHistogramBucketExemplarReservoir implementation.
+/// </summary>
+internal sealed class AlignedHistogramBucketExemplarReservoir : ExemplarReservoir
+{
+    private readonly int length;
+    private readonly Exemplar[] runningExemplars;
+    private readonly Exemplar[] tempExemplars;
+
+    public AlignedHistogramBucketExemplarReservoir(int length)
+    {
+        this.length = length;
+        this.runningExemplars = new Exemplar[length + 1];
+        this.tempExemplars = new Exemplar[length + 1];
+    }
+
+    public override void Offer(long value, ReadOnlySpan<KeyValuePair<string, object>> tags, int index = default)
+    {
+        this.OfferAtBoundary(value, tags, index);
+    }
+
+    public override void Offer(double value, ReadOnlySpan<KeyValuePair<string, object>> tags, int index = default)
+    {
+        this.OfferAtBoundary(value, tags, index);
+    }
+
+    public override Exemplar[] Collect(ReadOnlyTagCollection actualTags, bool reset)
+    {
+        for (int i = 0; i < this.runningExemplars.Length; i++)
+        {
+            this.tempExemplars[i] = this.runningExemplars[i];
+            if (this.runningExemplars[i].FilteredTags != null)
+            {
+                // TODO: Better data structure to avoid this Linq.
+                // This is doing filtered = alltags - storedtags.
+                // TODO: At this stage, this logic is done inside Reservoir.
+                // Kinda hard for end users who write own reservoirs.
+                // Evaluate if this logic can be moved elsewhere.
+                // TODO: The cost is paid irrespective of whether the
+                // Exporter supports Exemplar or not. One idea is to
+                // defer this until first exporter attempts read.
+                this.tempExemplars[i].FilteredTags = this.runningExemplars[i].FilteredTags.Except(actualTags.KeyAndValues.ToList()).ToList();
+            }
+
+            if (reset)
+            {
+                this.runningExemplars[i].Timestamp = default;
+            }
+        }
+
+        return this.tempExemplars;
+    }
+
+    private void OfferAtBoundary(double value, ReadOnlySpan<KeyValuePair<string, object>> tags, int index)
+    {
+        ref var exemplar = ref this.runningExemplars[index];
+        exemplar.Timestamp = DateTime.UtcNow;
+        exemplar.DoubleValue = value;
+        exemplar.TraceId = Activity.Current?.TraceId;
+        exemplar.SpanId = Activity.Current?.SpanId;
+
+        if (tags == default)
+        {
+            // default tag is used to indicate
+            // the special case where all tags provided at measurement
+            // recording time are stored.
+            // In this case, Exemplars does not have to store any tags.
+            // In other words, FilteredTags will be empty.
+            return;
+        }
+
+        if (exemplar.FilteredTags == null)
+        {
+            exemplar.FilteredTags = new List<KeyValuePair<string, object>>(tags.Length);
+        }
+        else
+        {
+            // Keep the list, but clear contents.
+            exemplar.FilteredTags.Clear();
+        }
+
+        // Though only those tags that are filtered need to be
+        // stored, finding filtered list from the full tag list
+        // is expensive. So all the tags are stored in hot path (this).
+        // During snapshot, the filtered list is calculated.
+        // TODO: Evaluate alternative approaches based on perf.
+        foreach (var tag in tags)
+        {
+            exemplar.FilteredTags.Add(tag);
+        }
+    }
+}

src/OpenTelemetry/Metrics/Exemplar/AlwaysOffExemplarFilter.cs

@@ -0,0 +1,34 @@
+// <copyright file="AlwaysOffExemplarFilter.cs" company="OpenTelemetry Authors">
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// </copyright>
+
+namespace OpenTelemetry.Metrics;
+
+/// <summary>
+/// An ExemplarFilter which makes no measurements eligible for being an Exemplar.
+/// Using this ExemplarFilter is as good as disabling Exemplar feature.
+/// </summary>
+public sealed class AlwaysOffExemplarFilter : ExemplarFilter
+{
+    public override bool ShouldSample(long value, ReadOnlySpan<KeyValuePair<string, object>> tags)
+    {
+        return false;
+    }
+
+    public override bool ShouldSample(double value, ReadOnlySpan<KeyValuePair<string, object>> tags)
+    {
+        return false;
+    }
+}

src/OpenTelemetry/Metrics/Exemplar/AlwaysOnExemplarFilter.cs

@@ -0,0 +1,33 @@
+// <copyright file="AlwaysOnExemplarFilter.cs" company="OpenTelemetry Authors">
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// </copyright>
+
+namespace OpenTelemetry.Metrics;
+
+/// <summary>
+/// An ExemplarFilter which makes all measurements eligible for being an Exemplar.
+/// </summary>
+public sealed class AlwaysOnExemplarFilter : ExemplarFilter
+{
+    public override bool ShouldSample(long value, ReadOnlySpan<KeyValuePair<string, object>> tags)
+    {
+        return true;
+    }
+
+    public override bool ShouldSample(double value, ReadOnlySpan<KeyValuePair<string, object>> tags)
+    {
+        return true;
+    }
+}

src/OpenTelemetry/Metrics/Exemplar/Exemplar.cs

@@ -0,0 +1,54 @@
+// <copyright file="Exemplar.cs" company="OpenTelemetry Authors">
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// </copyright>
+
+using System.Diagnostics;
+
+namespace OpenTelemetry.Metrics
+{
+    /// <summary>
+    /// Represents an Exemplar data.
+    /// </summary>
+    public struct Exemplar
+    {
+        /// <summary>
+        /// Gets the timestamp.
+        /// </summary>
+        public DateTime Timestamp { get; internal set; }
+
+        /// <summary>
+        /// Gets the TraceId.
+        /// </summary>
+        public ActivityTraceId? TraceId { get; internal set; }
+
+        /// <summary>
+        /// Gets the SpanId.
+        /// </summary>
+        public ActivitySpanId? SpanId { get; internal set; }
+
+        // TODO: Leverage MetricPointValueStorage
+        // and allow double/long instead of double only.
+
+        /// <summary>
+        /// Gets the double value.
+        /// </summary>
+        public double DoubleValue { get; internal set; }
+
+        /// <summary>
+        /// Gets the FilteredTags (i.e any tags that were dropped during aggregation).
+        /// </summary>
+        public List<KeyValuePair<string, object>> FilteredTags { get; internal set; }
+    }
+}

src/OpenTelemetry/Metrics/Exemplar/ExemplarFilter.cs

@@ -0,0 +1,58 @@
+// <copyright file="ExemplarFilter.cs" company="OpenTelemetry Authors">
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// </copyright>
+namespace OpenTelemetry.Metrics;
+
+/// <summary>
+/// The base class for defining Exemplar Filter.
+/// </summary>
+public abstract class ExemplarFilter
+{
+    /// <summary>
+    /// Determines if a given measurement is eligible for being
+    /// considered for becoming Exemplar.
+    /// </summary>
+    /// <param name="value">The value of the measurement.</param>
+    /// <param name="tags">The complete set of tags provided with the measurement.</param>
+    /// <returns>
+    /// Returns
+    /// <c>true</c> to indicate this measurement is eligible to become Exemplar
+    /// and will be given to an ExemplarReservoir.
+    /// Reservoir may further sample, so a true here does not mean that this
+    /// measurement will become an exemplar, it just means it'll be
+    /// eligible for being Exemplar.
+    /// <c>false</c> to indicate this measurement is not eligible to become Exemplar
+    /// and will not be given to the ExemplarReservoir.
+    /// </returns>
+    public abstract bool ShouldSample(long value, ReadOnlySpan<KeyValuePair<string, object>> tags);
+
+    /// <summary>
+    /// Determines if a given measurement is eligible for being
+    /// considered for becoming Exemplar.
+    /// </summary>
+    /// <param name="value">The value of the measurement.</param>
+    /// <param name="tags">The complete set of tags provided with the measurement.</param>
+    /// <returns>
+    /// Returns
+    /// <c>true</c> to indicate this measurement is eligible to become Exemplar
+    /// and will be given to an ExemplarReservoir.
+    /// Reservoir may further sample, so a true here does not mean that this
+    /// measurement will become an exemplar, it just means it'll be
+    /// eligible for being Exemplar.
+    /// <c>false</c> to indicate this measurement is not eligible to become Exemplar
+    /// and will not be given to the ExemplarReservoir.
+    /// </returns>
+    public abstract bool ShouldSample(double value, ReadOnlySpan<KeyValuePair<string, object>> tags);
+}

src/OpenTelemetry/Metrics/Exemplar/ExemplarReservoir.cs

@@ -0,0 +1,42 @@
+// <copyright file="ExemplarReservoir.cs" company="OpenTelemetry Authors">
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// </copyright>
+namespace OpenTelemetry.Metrics;
+
+/// <summary>
+/// The base class for defining Exemplar Reservoir.
+/// </summary>
+internal abstract class ExemplarReservoir
+{
+    /// <summary>
+    /// Offers measurement to the reservoir.
+    /// </summary>
+    /// <param name="value">The value of the measurement.</param>
+    /// <param name="tags">The complete set of tags provided with the measurement.</param>
+    /// <param name="index">The histogram bucket index where this measurement is going to be stored.
+    /// This is optional and is only relevant for Histogram with buckets.</param>
+    public abstract void Offer(long value, ReadOnlySpan<KeyValuePair<string, object>> tags, int index = default);
+
+    /// <summary>
+    /// Offers measurement to the reservoir.
+    /// </summary>
+    /// <param name="value">The value of the measurement.</param>
+    /// <param name="tags">The complete set of tags provided with the measurement.</param>
+    /// <param name="index">The histogram bucket index where this measurement is going to be stored.
+    /// This is optional and is only relevant for Histogram with buckets.</param>
+    public abstract void Offer(double value, ReadOnlySpan<KeyValuePair<string, object>> tags, int index = default);
+
+    public abstract Exemplar[] Collect(ReadOnlyTagCollection actualTags, bool reset);
+}

src/OpenTelemetry/Metrics/Exemplar/TraceBasedExemplarFilter.cs

@@ -0,0 +1,36 @@
+// <copyright file="TraceBasedExemplarFilter.cs" company="OpenTelemetry Authors">
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// </copyright>
+
+using System.Diagnostics;
+
+namespace OpenTelemetry.Metrics;
+
+/// <summary>
+/// An ExemplarFilter which makes those measurements eligible for being an Exemplar,
+/// which are recorded in the context of a sampled parent activity (span).
+/// </summary>
+public sealed class TraceBasedExemplarFilter : ExemplarFilter
+{
+    public override bool ShouldSample(long value, ReadOnlySpan<KeyValuePair<string, object>> tags)
+    {
+        return Activity.Current?.Recorded ?? false;
+    }
+
+    public override bool ShouldSample(double value, ReadOnlySpan<KeyValuePair<string, object>> tags)
+    {
+        return Activity.Current?.Recorded ?? false;
+    }
+}

src/OpenTelemetry/Metrics/HistogramBuckets.cs

@@ -32,6 +32,8 @@ namespace OpenTelemetry.Metrics
         internal readonly long[] RunningBucketCounts;
         internal readonly long[] SnapshotBucketCounts;
 
+        internal readonly ExemplarReservoir ExemplarReservoir;
+
         internal double RunningSum;
         internal double SnapshotSum;
 
@@ -43,11 +45,13 @@ namespace OpenTelemetry.Metrics
 
         internal int IsCriticalSectionOccupied = 0;
 
+        internal Exemplar[] Exemplars;
+
         private readonly BucketLookupNode bucketLookupTreeRoot;
 
         private readonly Func<double, int> findHistogramBucketIndex;
 
-        internal HistogramBuckets(double[] explicitBounds)
+        internal HistogramBuckets(double[] explicitBounds, bool enableExemplar = false)
         {
             this.ExplicitBounds = explicitBounds;
             this.findHistogramBucketIndex = this.FindBucketIndexLinear;
@@ -77,6 +81,10 @@ namespace OpenTelemetry.Metrics
 
             this.RunningBucketCounts = explicitBounds != null ? new long[explicitBounds.Length + 1] : null;
             this.SnapshotBucketCounts = explicitBounds != null ? new long[explicitBounds.Length + 1] : new long[0];
+            if (explicitBounds != null && enableExemplar)
+            {
+                this.ExemplarReservoir = new AlignedHistogramBucketExemplarReservoir(explicitBounds.Length);
+            }
         }
 
         public Enumerator GetEnumerator() => new(this);

src/OpenTelemetry/Metrics/MeterProviderSdk.cs

@@ -80,6 +80,7 @@ namespace OpenTelemetry.Metrics
                 reader.SetParentProvider(this);
                 reader.SetMaxMetricStreams(state.MaxMetricStreams);
                 reader.SetMaxMetricPointsPerMetricStream(state.MaxMetricPointsPerMetricStream);
+                reader.SetExemplarFilter(state.ExemplarFilter);
 
                 if (this.reader == null)
                 {

src/OpenTelemetry/Metrics/Metric.cs

@@ -33,7 +33,8 @@ namespace OpenTelemetry.Metrics
             int maxMetricPointsPerMetricStream,
             double[] histogramBounds = null,
             string[] tagKeysInteresting = null,
-            bool histogramRecordMinMax = true)
+            bool histogramRecordMinMax = true,
+            ExemplarFilter exemplarFilter = null)
         {
             this.InstrumentIdentity = instrumentIdentity;
 
@@ -126,7 +127,7 @@ namespace OpenTelemetry.Metrics
                 throw new NotSupportedException($"Unsupported Instrument Type: {instrumentIdentity.InstrumentType.FullName}");
             }
 
-            this.aggStore = new AggregatorStore(instrumentIdentity.InstrumentName, aggType, temporality, maxMetricPointsPerMetricStream, histogramBounds ?? DefaultHistogramBounds, tagKeysInteresting);
+            this.aggStore = new AggregatorStore(instrumentIdentity.InstrumentName, aggType, temporality, maxMetricPointsPerMetricStream, histogramBounds ?? DefaultHistogramBounds, tagKeysInteresting, exemplarFilter);
             this.Temporality = temporality;
             this.InstrumentDisposed = false;
         }

src/OpenTelemetry/Metrics/MetricPoint.cs

@@ -28,7 +28,7 @@ namespace OpenTelemetry.Metrics
 
         private readonly AggregationType aggType;
 
-        private HistogramBuckets histogramBuckets;
+        private MetricPointOptionalComponents mpComponents;
 
         // Represents temporality adjusted "value" for double/long metric types or "count" when histogram
         private MetricPointValueStorage runningValue;
@@ -57,16 +57,18 @@ namespace OpenTelemetry.Metrics
             if (this.aggType == AggregationType.HistogramWithBuckets ||
                 this.aggType == AggregationType.HistogramWithMinMaxBuckets)
             {
-                this.histogramBuckets = new HistogramBuckets(histogramExplicitBounds);
+                this.mpComponents = new MetricPointOptionalComponents();
+                this.mpComponents.HistogramBuckets = new HistogramBuckets(histogramExplicitBounds, aggregatorStore.IsExemplarEnabled());
             }
             else if (this.aggType == AggregationType.Histogram ||
                      this.aggType == AggregationType.HistogramWithMinMax)
             {
-                this.histogramBuckets = new HistogramBuckets(null);
+                this.mpComponents = new MetricPointOptionalComponents();
+                this.mpComponents.HistogramBuckets = new HistogramBuckets(null);
             }
             else
             {
-                this.histogramBuckets = null;
+                this.mpComponents = null;
             }
 
             // Note: Intentionally set last because this is used to detect valid MetricPoints.
@@ -214,7 +216,7 @@ namespace OpenTelemetry.Metrics
                 this.ThrowNotSupportedMetricTypeException(nameof(this.GetHistogramSum));
             }
 
-            return this.histogramBuckets.SnapshotSum;
+            return this.mpComponents.HistogramBuckets.SnapshotSum;
         }
 
         /// <summary>
@@ -235,7 +237,7 @@ namespace OpenTelemetry.Metrics
                 this.ThrowNotSupportedMetricTypeException(nameof(this.GetHistogramBuckets));
             }
 
-            return this.histogramBuckets;
+            return this.mpComponents.HistogramBuckets;
         }
 
         /// <summary>
@@ -250,10 +252,10 @@ namespace OpenTelemetry.Metrics
             if (this.aggType == AggregationType.HistogramWithMinMax ||
                             this.aggType == AggregationType.HistogramWithMinMaxBuckets)
             {
-                Debug.Assert(this.histogramBuckets != null, "histogramBuckets was null");
+                Debug.Assert(this.mpComponents.HistogramBuckets != null, "histogramBuckets was null");
 
-                min = this.histogramBuckets.SnapshotMin;
-                max = this.histogramBuckets.SnapshotMax;
+                min = this.mpComponents.HistogramBuckets.SnapshotMin;
+                max = this.mpComponents.HistogramBuckets.SnapshotMax;
                 return true;
             }
 
@@ -262,10 +264,21 @@ namespace OpenTelemetry.Metrics
             return false;
         }
 
+        /// <summary>
+        /// Gets the exemplars associated with the metric point.
+        /// </summary>
+        /// <returns><see cref="Exemplar"/>.</returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public readonly Exemplar[] GetExemplars()
+        {
+            // TODO: Do not expose Exemplar data structure (array now)
+            return this.mpComponents.HistogramBuckets?.Exemplars ?? Array.Empty<Exemplar>();
+        }
+
         internal readonly MetricPoint Copy()
         {
             MetricPoint copy = this;
-            copy.histogramBuckets = this.histogramBuckets?.Copy();
+            copy.mpComponents = this.mpComponents?.Copy();
             return copy;
         }
 
@@ -330,6 +343,67 @@ namespace OpenTelemetry.Metrics
             this.MetricPointStatus = MetricPointStatus.CollectPending;
         }
 
+        internal void UpdateWithExemplar(long number, ReadOnlySpan<KeyValuePair<string, object>> tags)
+        {
+            switch (this.aggType)
+            {
+                case AggregationType.LongSumIncomingDelta:
+                    {
+                        Interlocked.Add(ref this.runningValue.AsLong, number);
+                        break;
+                    }
+
+                case AggregationType.LongSumIncomingCumulative:
+                    {
+                        Interlocked.Exchange(ref this.runningValue.AsLong, number);
+                        break;
+                    }
+
+                case AggregationType.LongGauge:
+                    {
+                        Interlocked.Exchange(ref this.runningValue.AsLong, number);
+                        break;
+                    }
+
+                case AggregationType.Histogram:
+                    {
+                        this.UpdateHistogram((double)number);
+                        break;
+                    }
+
+                case AggregationType.HistogramWithMinMax:
+                    {
+                        this.UpdateHistogramWithMinMax((double)number);
+                        break;
+                    }
+
+                case AggregationType.HistogramWithBuckets:
+                    {
+                        this.UpdateHistogramWithBuckets((double)number, tags, true);
+                        break;
+                    }
+
+                case AggregationType.HistogramWithMinMaxBuckets:
+                    {
+                        this.UpdateHistogramWithBucketsAndMinMax((double)number, tags, true);
+                        break;
+                    }
+            }
+
+            // There is a race with Snapshot:
+            // Update() updates the value
+            // Snapshot snapshots the value
+            // Snapshot sets status to NoCollectPending
+            // Update sets status to CollectPending -- this is not right as the Snapshot
+            // already included the updated value.
+            // In the absence of any new Update call until next Snapshot,
+            // this results in exporting an Update even though
+            // it had no update.
+            // TODO: For Delta, this can be mitigated
+            // by ignoring Zero points
+            this.MetricPointStatus = MetricPointStatus.CollectPending;
+        }
+
         internal void Update(double number)
         {
             switch (this.aggType)
@@ -409,6 +483,85 @@ namespace OpenTelemetry.Metrics
             this.MetricPointStatus = MetricPointStatus.CollectPending;
         }
 
+        internal void UpdateWithExemplar(double number, ReadOnlySpan<KeyValuePair<string, object>> tags)
+        {
+            switch (this.aggType)
+            {
+                case AggregationType.DoubleSumIncomingDelta:
+                    {
+                        double initValue, newValue;
+                        var sw = default(SpinWait);
+                        while (true)
+                        {
+                            initValue = this.runningValue.AsDouble;
+
+                            unchecked
+                            {
+                                newValue = initValue + number;
+                            }
+
+                            if (initValue == Interlocked.CompareExchange(ref this.runningValue.AsDouble, newValue, initValue))
+                            {
+                                break;
+                            }
+
+                            sw.SpinOnce();
+                        }
+
+                        break;
+                    }
+
+                case AggregationType.DoubleSumIncomingCumulative:
+                    {
+                        Interlocked.Exchange(ref this.runningValue.AsDouble, number);
+                        break;
+                    }
+
+                case AggregationType.DoubleGauge:
+                    {
+                        Interlocked.Exchange(ref this.runningValue.AsDouble, number);
+                        break;
+                    }
+
+                case AggregationType.Histogram:
+                    {
+                        this.UpdateHistogram(number);
+                        break;
+                    }
+
+                case AggregationType.HistogramWithMinMax:
+                    {
+                        this.UpdateHistogramWithMinMax(number);
+                        break;
+                    }
+
+                case AggregationType.HistogramWithBuckets:
+                    {
+                        this.UpdateHistogramWithBuckets(number, tags, true);
+                        break;
+                    }
+
+                case AggregationType.HistogramWithMinMaxBuckets:
+                    {
+                        this.UpdateHistogramWithBucketsAndMinMax(number, tags, true);
+                        break;
+                    }
+            }
+
+            // There is a race with Snapshot:
+            // Update() updates the value
+            // Snapshot snapshots the value
+            // Snapshot sets status to NoCollectPending
+            // Update sets status to CollectPending -- this is not right as the Snapshot
+            // already included the updated value.
+            // In the absence of any new Update call until next Snapshot,
+            // this results in exporting an Update even though
+            // it had no update.
+            // TODO: For Delta, this can be mitigated
+            // by ignoring Zero points
+            this.MetricPointStatus = MetricPointStatus.CollectPending;
+        }
+
         internal void TakeSnapshot(bool outputDelta)
         {
             switch (this.aggType)
@@ -510,34 +663,37 @@ namespace OpenTelemetry.Metrics
 
                 case AggregationType.HistogramWithBuckets:
                     {
+                        var histogramBuckets = this.mpComponents.HistogramBuckets;
                         var sw = default(SpinWait);
                         while (true)
                         {
-                            if (Interlocked.Exchange(ref this.histogramBuckets.IsCriticalSectionOccupied, 1) == 0)
+                            if (Interlocked.Exchange(ref histogramBuckets.IsCriticalSectionOccupied, 1) == 0)
                             {
                                 // Lock acquired
                                 this.snapshotValue.AsLong = this.runningValue.AsLong;
-                                this.histogramBuckets.SnapshotSum = this.histogramBuckets.RunningSum;
+                                histogramBuckets.SnapshotSum = histogramBuckets.RunningSum;
 
                                 if (outputDelta)
                                 {
                                     this.runningValue.AsLong = 0;
-                                    this.histogramBuckets.RunningSum = 0;
+                                    histogramBuckets.RunningSum = 0;
                                 }
 
-                                for (int i = 0; i < this.histogramBuckets.RunningBucketCounts.Length; i++)
+                                for (int i = 0; i < histogramBuckets.RunningBucketCounts.Length; i++)
                                 {
-                                    this.histogramBuckets.SnapshotBucketCounts[i] = this.histogramBuckets.RunningBucketCounts[i];
+                                    histogramBuckets.SnapshotBucketCounts[i] = histogramBuckets.RunningBucketCounts[i];
                                     if (outputDelta)
                                     {
-                                        this.histogramBuckets.RunningBucketCounts[i] = 0;
+                                        histogramBuckets.RunningBucketCounts[i] = 0;
                                     }
                                 }
 
+                                histogramBuckets.Exemplars = histogramBuckets.ExemplarReservoir?.Collect(this.Tags, outputDelta);
+
                                 this.MetricPointStatus = MetricPointStatus.NoCollectPending;
 
                                 // Release lock
-                                Interlocked.Exchange(ref this.histogramBuckets.IsCriticalSectionOccupied, 0);
+                                Interlocked.Exchange(ref histogramBuckets.IsCriticalSectionOccupied, 0);
                                 break;
                             }
 
@@ -549,25 +705,26 @@ namespace OpenTelemetry.Metrics
 
                 case AggregationType.Histogram:
                     {
+                        var histogramBuckets = this.mpComponents.HistogramBuckets;
                         var sw = default(SpinWait);
                         while (true)
                         {
-                            if (Interlocked.Exchange(ref this.histogramBuckets.IsCriticalSectionOccupied, 1) == 0)
+                            if (Interlocked.Exchange(ref histogramBuckets.IsCriticalSectionOccupied, 1) == 0)
                             {
                                 // Lock acquired
                                 this.snapshotValue.AsLong = this.runningValue.AsLong;
-                                this.histogramBuckets.SnapshotSum = this.histogramBuckets.RunningSum;
+                                histogramBuckets.SnapshotSum = histogramBuckets.RunningSum;
 
                                 if (outputDelta)
                                 {
                                     this.runningValue.AsLong = 0;
-                                    this.histogramBuckets.RunningSum = 0;
+                                    histogramBuckets.RunningSum = 0;
                                 }
 
                                 this.MetricPointStatus = MetricPointStatus.NoCollectPending;
 
                                 // Release lock
-                                Interlocked.Exchange(ref this.histogramBuckets.IsCriticalSectionOccupied, 0);
+                                Interlocked.Exchange(ref histogramBuckets.IsCriticalSectionOccupied, 0);
                                 break;
                             }
 
@@ -579,38 +736,40 @@ namespace OpenTelemetry.Metrics
 
                 case AggregationType.HistogramWithMinMaxBuckets:
                     {
+                        var histogramBuckets = this.mpComponents.HistogramBuckets;
                         var sw = default(SpinWait);
                         while (true)
                         {
-                            if (Interlocked.Exchange(ref this.histogramBuckets.IsCriticalSectionOccupied, 1) == 0)
+                            if (Interlocked.Exchange(ref histogramBuckets.IsCriticalSectionOccupied, 1) == 0)
                             {
                                 // Lock acquired
                                 this.snapshotValue.AsLong = this.runningValue.AsLong;
-                                this.histogramBuckets.SnapshotSum = this.histogramBuckets.RunningSum;
-                                this.histogramBuckets.SnapshotMin = this.histogramBuckets.RunningMin;
-                                this.histogramBuckets.SnapshotMax = this.histogramBuckets.RunningMax;
+                                histogramBuckets.SnapshotSum = histogramBuckets.RunningSum;
+                                histogramBuckets.SnapshotMin = histogramBuckets.RunningMin;
+                                histogramBuckets.SnapshotMax = histogramBuckets.RunningMax;
 
                                 if (outputDelta)
                                 {
                                     this.runningValue.AsLong = 0;
-                                    this.histogramBuckets.RunningSum = 0;
-                                    this.histogramBuckets.RunningMin = double.PositiveInfinity;
-                                    this.histogramBuckets.RunningMax = double.NegativeInfinity;
+                                    histogramBuckets.RunningSum = 0;
+                                    histogramBuckets.RunningMin = double.PositiveInfinity;
+                                    histogramBuckets.RunningMax = double.NegativeInfinity;
                                 }
 
-                                for (int i = 0; i < this.histogramBuckets.RunningBucketCounts.Length; i++)
+                                for (int i = 0; i < histogramBuckets.RunningBucketCounts.Length; i++)
                                 {
-                                    this.histogramBuckets.SnapshotBucketCounts[i] = this.histogramBuckets.RunningBucketCounts[i];
+                                    histogramBuckets.SnapshotBucketCounts[i] = histogramBuckets.RunningBucketCounts[i];
                                     if (outputDelta)
                                     {
-                                        this.histogramBuckets.RunningBucketCounts[i] = 0;
+                                        histogramBuckets.RunningBucketCounts[i] = 0;
                                     }
                                 }
 
+                                histogramBuckets.Exemplars = histogramBuckets.ExemplarReservoir?.Collect(this.Tags, outputDelta);
                                 this.MetricPointStatus = MetricPointStatus.NoCollectPending;
 
                                 // Release lock
-                                Interlocked.Exchange(ref this.histogramBuckets.IsCriticalSectionOccupied, 0);
+                                Interlocked.Exchange(ref histogramBuckets.IsCriticalSectionOccupied, 0);
                                 break;
                             }
 
@@ -622,29 +781,30 @@ namespace OpenTelemetry.Metrics
 
                 case AggregationType.HistogramWithMinMax:
                     {
+                        var histogramBuckets = this.mpComponents.HistogramBuckets;
                         var sw = default(SpinWait);
                         while (true)
                         {
-                            if (Interlocked.Exchange(ref this.histogramBuckets.IsCriticalSectionOccupied, 1) == 0)
+                            if (Interlocked.Exchange(ref histogramBuckets.IsCriticalSectionOccupied, 1) == 0)
                             {
                                 // Lock acquired
                                 this.snapshotValue.AsLong = this.runningValue.AsLong;
-                                this.histogramBuckets.SnapshotSum = this.histogramBuckets.RunningSum;
-                                this.histogramBuckets.SnapshotMin = this.histogramBuckets.RunningMin;
-                                this.histogramBuckets.SnapshotMax = this.histogramBuckets.RunningMax;
+                                histogramBuckets.SnapshotSum = histogramBuckets.RunningSum;
+                                histogramBuckets.SnapshotMin = histogramBuckets.RunningMin;
+                                histogramBuckets.SnapshotMax = histogramBuckets.RunningMax;
 
                                 if (outputDelta)
                                 {
                                     this.runningValue.AsLong = 0;
-                                    this.histogramBuckets.RunningSum = 0;
-                                    this.histogramBuckets.RunningMin = double.PositiveInfinity;
-                                    this.histogramBuckets.RunningMax = double.NegativeInfinity;
+                                    histogramBuckets.RunningSum = 0;
+                                    histogramBuckets.RunningMin = double.PositiveInfinity;
+                                    histogramBuckets.RunningMax = double.NegativeInfinity;
                                 }
 
                                 this.MetricPointStatus = MetricPointStatus.NoCollectPending;
 
                                 // Release lock
-                                Interlocked.Exchange(ref this.histogramBuckets.IsCriticalSectionOccupied, 0);
+                                Interlocked.Exchange(ref histogramBuckets.IsCriticalSectionOccupied, 0);
                                 break;
                             }
 
@@ -658,20 +818,21 @@ namespace OpenTelemetry.Metrics
 
         private void UpdateHistogram(double number)
         {
+            var histogramBuckets = this.mpComponents.HistogramBuckets;
             var sw = default(SpinWait);
             while (true)
             {
-                if (Interlocked.Exchange(ref this.histogramBuckets.IsCriticalSectionOccupied, 1) == 0)
+                if (Interlocked.Exchange(ref histogramBuckets.IsCriticalSectionOccupied, 1) == 0)
                 {
                     // Lock acquired
                     unchecked
                     {
                         this.runningValue.AsLong++;
-                        this.histogramBuckets.RunningSum += number;
+                        histogramBuckets.RunningSum += number;
                     }
 
                     // Release lock
-                    Interlocked.Exchange(ref this.histogramBuckets.IsCriticalSectionOccupied, 0);
+                    Interlocked.Exchange(ref histogramBuckets.IsCriticalSectionOccupied, 0);
                     break;
                 }
 
@@ -681,22 +842,23 @@ namespace OpenTelemetry.Metrics
 
         private void UpdateHistogramWithMinMax(double number)
         {
+            var histogramBuckets = this.mpComponents.HistogramBuckets;
             var sw = default(SpinWait);
             while (true)
             {
-                if (Interlocked.Exchange(ref this.histogramBuckets.IsCriticalSectionOccupied, 1) == 0)
+                if (Interlocked.Exchange(ref histogramBuckets.IsCriticalSectionOccupied, 1) == 0)
                 {
                     // Lock acquired
                     unchecked
                     {
                         this.runningValue.AsLong++;
-                        this.histogramBuckets.RunningSum += number;
-                        this.histogramBuckets.RunningMin = Math.Min(this.histogramBuckets.RunningMin, number);
-                        this.histogramBuckets.RunningMax = Math.Max(this.histogramBuckets.RunningMax, number);
+                        histogramBuckets.RunningSum += number;
+                        histogramBuckets.RunningMin = Math.Min(histogramBuckets.RunningMin, number);
+                        histogramBuckets.RunningMax = Math.Max(histogramBuckets.RunningMax, number);
                     }
 
                     // Release lock
-                    Interlocked.Exchange(ref this.histogramBuckets.IsCriticalSectionOccupied, 0);
+                    Interlocked.Exchange(ref histogramBuckets.IsCriticalSectionOccupied, 0);
                     break;
                 }
 
@@ -704,25 +866,30 @@ namespace OpenTelemetry.Metrics
             }
         }
 
-        private void UpdateHistogramWithBuckets(double number)
+        private void UpdateHistogramWithBuckets(double number, ReadOnlySpan<KeyValuePair<string, object>> tags = default, bool reportExemplar = false)
         {
-            int i = this.histogramBuckets.FindBucketIndex(number);
+            var histogramBuckets = this.mpComponents.HistogramBuckets;
+            int i = histogramBuckets.FindBucketIndex(number);
 
             var sw = default(SpinWait);
             while (true)
             {
-                if (Interlocked.Exchange(ref this.histogramBuckets.IsCriticalSectionOccupied, 1) == 0)
+                if (Interlocked.Exchange(ref histogramBuckets.IsCriticalSectionOccupied, 1) == 0)
                 {
                     // Lock acquired
                     unchecked
                     {
                         this.runningValue.AsLong++;
-                        this.histogramBuckets.RunningSum += number;
-                        this.histogramBuckets.RunningBucketCounts[i]++;
+                        histogramBuckets.RunningSum += number;
+                        histogramBuckets.RunningBucketCounts[i]++;
+                        if (reportExemplar)
+                        {
+                            histogramBuckets.ExemplarReservoir.Offer(number, tags, i);
+                        }
                     }
 
                     // Release lock
-                    Interlocked.Exchange(ref this.histogramBuckets.IsCriticalSectionOccupied, 0);
+                    Interlocked.Exchange(ref histogramBuckets.IsCriticalSectionOccupied, 0);
                     break;
                 }
 
@@ -730,27 +897,33 @@ namespace OpenTelemetry.Metrics
             }
         }
 
-        private void UpdateHistogramWithBucketsAndMinMax(double number)
+        private void UpdateHistogramWithBucketsAndMinMax(double number, ReadOnlySpan<KeyValuePair<string, object>> tags = default, bool reportExemplar = false)
         {
-            int i = this.histogramBuckets.FindBucketIndex(number);
+            var histogramBuckets = this.mpComponents.HistogramBuckets;
+            int i = histogramBuckets.FindBucketIndex(number);
 
             var sw = default(SpinWait);
             while (true)
             {
-                if (Interlocked.Exchange(ref this.histogramBuckets.IsCriticalSectionOccupied, 1) == 0)
+                if (Interlocked.Exchange(ref histogramBuckets.IsCriticalSectionOccupied, 1) == 0)
                 {
                     // Lock acquired
                     unchecked
                     {
                         this.runningValue.AsLong++;
-                        this.histogramBuckets.RunningSum += number;
-                        this.histogramBuckets.RunningBucketCounts[i]++;
-                        this.histogramBuckets.RunningMin = Math.Min(this.histogramBuckets.RunningMin, number);
-                        this.histogramBuckets.RunningMax = Math.Max(this.histogramBuckets.RunningMax, number);
+                        histogramBuckets.RunningSum += number;
+                        histogramBuckets.RunningBucketCounts[i]++;
+                        if (reportExemplar)
+                        {
+                            histogramBuckets.ExemplarReservoir.Offer(number, tags, i);
+                        }
+
+                        histogramBuckets.RunningMin = Math.Min(histogramBuckets.RunningMin, number);
+                        histogramBuckets.RunningMax = Math.Max(histogramBuckets.RunningMax, number);
                     }
 
                     // Release lock
-                    Interlocked.Exchange(ref this.histogramBuckets.IsCriticalSectionOccupied, 0);
+                    Interlocked.Exchange(ref histogramBuckets.IsCriticalSectionOccupied, 0);
                     break;
                 }
 

src/OpenTelemetry/Metrics/MetricPointOptionalComponents.cs

@@ -0,0 +1,38 @@
+// <copyright file="MetricPointOptionalComponents.cs" company="OpenTelemetry Authors">
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// </copyright>
+
+namespace OpenTelemetry.Metrics
+{
+    /// <summary>
+    /// Stores optional components of a metric point.
+    /// Histogram, Exemplar are current components.
+    /// ExponentialHistogram is a future component.
+    /// This is done to keep the MetricPoint (struct)
+    /// size in control.
+    /// </summary>
+    internal sealed class MetricPointOptionalComponents
+    {
+        public HistogramBuckets HistogramBuckets;
+
+        internal MetricPointOptionalComponents Copy()
+        {
+            MetricPointOptionalComponents copy = new MetricPointOptionalComponents();
+            copy.HistogramBuckets = this.HistogramBuckets.Copy();
+
+            return copy;
+        }
+    }
+}

src/OpenTelemetry/Metrics/MetricReaderExt.cs

@@ -34,6 +34,8 @@ namespace OpenTelemetry.Metrics
         private Metric[] metricsCurrentBatch;
         private int metricIndex = -1;
 
+        private ExemplarFilter exemplarFilter;
+
         internal AggregationTemporality GetAggregationTemporality(Type instrumentType)
         {
             return this.temporalityFunc(instrumentType);
@@ -69,7 +71,7 @@ namespace OpenTelemetry.Metrics
                     Metric metric = null;
                     try
                     {
-                        metric = new Metric(metricStreamIdentity, this.GetAggregationTemporality(metricStreamIdentity.InstrumentType), this.maxMetricPointsPerMetricStream);
+                        metric = new Metric(metricStreamIdentity, this.GetAggregationTemporality(metricStreamIdentity.InstrumentType), this.maxMetricPointsPerMetricStream, exemplarFilter: this.exemplarFilter);
                     }
                     catch (NotSupportedException nse)
                     {
@@ -154,7 +156,7 @@ namespace OpenTelemetry.Metrics
                     }
                     else
                     {
-                        Metric metric = new(metricStreamIdentity, this.GetAggregationTemporality(metricStreamIdentity.InstrumentType), this.maxMetricPointsPerMetricStream, metricStreamIdentity.HistogramBucketBounds, metricStreamIdentity.TagKeys, metricStreamIdentity.HistogramRecordMinMax);
+                        Metric metric = new(metricStreamIdentity, this.GetAggregationTemporality(metricStreamIdentity.InstrumentType), this.maxMetricPointsPerMetricStream, metricStreamIdentity.HistogramBucketBounds, metricStreamIdentity.TagKeys, metricStreamIdentity.HistogramRecordMinMax, this.exemplarFilter);
 
                         this.instrumentIdentityToMetric[metricStreamIdentity] = metric;
                         this.metrics[index] = metric;
@@ -223,6 +225,11 @@ namespace OpenTelemetry.Metrics
             this.metricsCurrentBatch = new Metric[maxMetricStreams];
         }
 
+        internal void SetExemplarFilter(ExemplarFilter exemplarFilter)
+        {
+            this.exemplarFilter = exemplarFilter;
+        }
+
         internal void SetMaxMetricPointsPerMetricStream(int maxMetricPointsPerMetricStream)
         {
             this.maxMetricPointsPerMetricStream = maxMetricPointsPerMetricStream;

src/OpenTelemetry/ReadOnlyTagCollection.cs

@@ -25,17 +25,17 @@ namespace OpenTelemetry
     // prevent accidental boxing.
     public readonly struct ReadOnlyTagCollection
     {
-        private readonly KeyValuePair<string, object>[] keyAndValues;
+        internal readonly KeyValuePair<string, object>[] KeyAndValues;
 
         internal ReadOnlyTagCollection(KeyValuePair<string, object>[]? keyAndValues)
         {
-            this.keyAndValues = keyAndValues ?? Array.Empty<KeyValuePair<string, object>>();
+            this.KeyAndValues = keyAndValues ?? Array.Empty<KeyValuePair<string, object>>();
         }
 
         /// <summary>
         /// Gets the number of tags in the collection.
         /// </summary>
-        public int Count => this.keyAndValues.Length;
+        public int Count => this.KeyAndValues.Length;
 
         /// <summary>
         /// Returns an enumerator that iterates through the tags.
@@ -78,7 +78,7 @@ namespace OpenTelemetry
 
                 if (index < this.source.Count)
                 {
-                    this.Current = this.source.keyAndValues[index];
+                    this.Current = this.source.KeyAndValues[index];
 
                     this.index++;
                     return true;

test/Benchmarks/Metrics/ExemplarBenchmarks.cs

@@ -0,0 +1,116 @@
+// <copyright file="ExemplarBenchmarks.cs" company="OpenTelemetry Authors">
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// </copyright>
+
+using System.Diagnostics;
+using System.Diagnostics.Metrics;
+using BenchmarkDotNet.Attributes;
+using OpenTelemetry;
+using OpenTelemetry.Metrics;
+using OpenTelemetry.Tests;
+
+/*
+// * Summary *
+BenchmarkDotNet=v0.13.3, OS=Windows 11 (10.0.22621.1265)
+11th Gen Intel Core i7-1185G7 3.00GHz, 1 CPU, 8 logical and 4 physical cores
+.NET SDK=7.0.103
+  [Host]     : .NET 7.0.3 (7.0.323.6910), X64 RyuJIT AVX2
+  DefaultJob : .NET 7.0.3 (7.0.323.6910), X64 RyuJIT AVX2
+
+
+|                    Method | EnableExemplar |     Mean |   Error |  StdDev | Allocated |
+|-------------------------- |--------------- |---------:|--------:|--------:|----------:|
+|   HistogramNoTagReduction |          False | 256.5 ns | 4.84 ns | 4.53 ns |         - |
+| HistogramWithTagReduction |          False | 246.6 ns | 4.90 ns | 4.81 ns |         - |
+|   HistogramNoTagReduction |           True | 286.4 ns | 5.30 ns | 7.25 ns |         - |
+| HistogramWithTagReduction |           True | 293.6 ns | 5.77 ns | 7.09 ns |         - |
+
+*/
+
+namespace Benchmarks.Metrics
+{
+    public class ExemplarBenchmarks
+    {
+        private static readonly ThreadLocal<Random> ThreadLocalRandom = new(() => new Random());
+        private readonly string[] dimensionValues = new string[] { "DimVal1", "DimVal2", "DimVal3", "DimVal4", "DimVal5", "DimVal6", "DimVal7", "DimVal8", "DimVal9", "DimVal10" };
+        private Histogram<long> histogramWithoutTagReduction;
+
+        private Histogram<long> histogramWithTagReduction;
+
+        private MeterProvider provider;
+        private Meter meter;
+
+        [Params(true, false)]
+        public bool EnableExemplar { get; set; }
+
+        [GlobalSetup]
+        public void Setup()
+        {
+            this.meter = new Meter(Utils.GetCurrentMethodName());
+            this.histogramWithoutTagReduction = this.meter.CreateHistogram<long>("HistogramWithoutTagReduction");
+            this.histogramWithTagReduction = this.meter.CreateHistogram<long>("HistogramWithTagReduction");
+            var exportedItems = new List<Metric>();
+
+            this.provider = Sdk.CreateMeterProviderBuilder()
+                .AddMeter(this.meter.Name)
+                .SetExemplarFilter(this.EnableExemplar ? new AlwaysOnExemplarFilter() : new AlwaysOffExemplarFilter())
+                .AddView("HistogramWithTagReduction", new MetricStreamConfiguration() { TagKeys = new string[] { "DimName1", "DimName2", "DimName3" } })
+                .AddInMemoryExporter(exportedItems, metricReaderOptions =>
+                {
+                    metricReaderOptions.PeriodicExportingMetricReaderOptions.ExportIntervalMilliseconds = 1000;
+                })
+                .Build();
+        }
+
+        [GlobalCleanup]
+        public void Cleanup()
+        {
+            this.meter?.Dispose();
+            this.provider?.Dispose();
+        }
+
+        [Benchmark]
+        public void HistogramNoTagReduction()
+        {
+            var random = ThreadLocalRandom.Value;
+            var tags = new TagList
+            {
+                { "DimName1", this.dimensionValues[random.Next(0, 2)] },
+                { "DimName2", this.dimensionValues[random.Next(0, 2)] },
+                { "DimName3", this.dimensionValues[random.Next(0, 5)] },
+                { "DimName4", this.dimensionValues[random.Next(0, 5)] },
+                { "DimName5", this.dimensionValues[random.Next(0, 10)] },
+            };
+
+            this.histogramWithoutTagReduction.Record(random.Next(1000), tags);
+        }
+
+        [Benchmark]
+        public void HistogramWithTagReduction()
+        {
+            var random = ThreadLocalRandom.Value;
+            var tags = new TagList
+            {
+                { "DimName1", this.dimensionValues[random.Next(0, 2)] },
+                { "DimName2", this.dimensionValues[random.Next(0, 2)] },
+                { "DimName3", this.dimensionValues[random.Next(0, 5)] },
+                { "DimName4", this.dimensionValues[random.Next(0, 5)] },
+                { "DimName5", this.dimensionValues[random.Next(0, 10)] },
+            };
+
+            this.histogramWithTagReduction.Record(random.Next(1000), tags);
+        }
+    }
+}