[docs] Improve exemplars tutorial (#5636)
This commit is contained in:
Reiley Yang
GitHub
parent
498b0d00a1
commit
3cff6dbfd1
|
|
@ -0,0 +1,66 @@
|
|||
// Copyright The OpenTelemetry Authors
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
using System.Diagnostics;
|
||||
using System.Diagnostics.Metrics;
|
||||
using OpenTelemetry;
|
||||
using OpenTelemetry.Exporter;
|
||||
using OpenTelemetry.Metrics;
|
||||
using OpenTelemetry.Trace;
|
||||
|
||||
public class Program
|
||||
{
|
||||
private static readonly ActivitySource MyActivitySource = new("OpenTelemetry.Demo.Exemplar");
|
||||
private static readonly Meter MyMeter = new("OpenTelemetry.Demo.Exemplar");
|
||||
private static readonly Histogram<double> MyHistogram = MyMeter.CreateHistogram<double>("MyHistogram");
|
||||
|
||||
public static void Main()
|
||||
{
|
||||
var tracerProvider = Sdk.CreateTracerProviderBuilder()
|
||||
.AddSource("OpenTelemetry.Demo.Exemplar")
|
||||
.AddOtlpExporter()
|
||||
.Build();
|
||||
|
||||
var meterProvider = Sdk.CreateMeterProviderBuilder()
|
||||
.AddMeter("OpenTelemetry.Demo.Exemplar")
|
||||
.SetExemplarFilter(ExemplarFilterType.TraceBased)
|
||||
.AddOtlpExporter((exporterOptions, metricReaderOptions) =>
|
||||
{
|
||||
exporterOptions.Endpoint = new Uri("http://localhost:9090/api/v1/otlp/v1/metrics");
|
||||
exporterOptions.Protocol = OtlpExportProtocol.HttpProtobuf;
|
||||
metricReaderOptions.PeriodicExportingMetricReaderOptions.ExportIntervalMilliseconds = 1000;
|
||||
})
|
||||
.Build();
|
||||
|
||||
var random = new Random();
|
||||
|
||||
Console.WriteLine("Press any key to exit");
|
||||
|
||||
while (!Console.KeyAvailable)
|
||||
{
|
||||
using (var parent = MyActivitySource.StartActivity("Parent Operation"))
|
||||
{
|
||||
parent?.SetTag("key1", "value1");
|
||||
parent?.SetTag("key2", "value2");
|
||||
|
||||
using (var child = MyActivitySource.StartActivity("Child Operation"))
|
||||
{
|
||||
child?.SetTag("key3", "value3");
|
||||
child?.SetTag("key4", "value4");
|
||||
|
||||
MyHistogram.Record(random.NextDouble() * 100, new("tag1", "value1"), new("tag2", "value2"));
|
||||
}
|
||||
}
|
||||
|
||||
Thread.Sleep(300);
|
||||
}
|
||||
|
||||
// Dispose meter provider before the application ends.
|
||||
// This will flush the remaining metrics and shutdown the metrics pipeline.
|
||||
meterProvider.Dispose();
|
||||
|
||||
// Dispose tracer provider before the application ends.
|
||||
// This will flush the remaining spans and shutdown the tracing pipeline.
|
||||
tracerProvider.Dispose();
|
||||
}
|
||||
}
|
||||
|
|
@ -1,99 +1,166 @@
|
|||
# Using Exemplars in OpenTelemetry .NET
|
||||
|
||||
Exemplars are example data points for aggregated data. They provide specific
|
||||
context to otherwise general aggregations. One common use case is to gain
|
||||
ability to correlate metrics to traces (and logs). While OpenTelemetry .NET
|
||||
supports Exemplars, it is only useful if the telemetry backend also supports the
|
||||
capabilities. This tutorial uses well known open source backends to demonstrate
|
||||
the concept. The following are the components involved:
|
||||
<details>
|
||||
<summary>Table of Contents</summary>
|
||||
|
||||
* Test App - We use existing example app from the repo. This app is already
|
||||
instrumented with OpenTelemetry for logs, metrics and traces, and is configured
|
||||
to export them to the configured OTLP end point.
|
||||
* OpenTelemetry Collector - An instance of collector is run, which receives
|
||||
telemetry from the above app using OTLP. The collector then exports metrics to
|
||||
Prometheus, traces to Tempo.
|
||||
* Prometheus - Prometheus is used as the Metric backend.
|
||||
* Tempo - Tempo is used as the Tracing backend.
|
||||
* Grafana - UI to query metrics from Prometheus, traces from Tempo, and to
|
||||
navigate between metrics and traces using Exemplar.
|
||||
* [Install and run Jaeger](#install-and-run-jaeger)
|
||||
* [Install and run Prometheus](#install-and-run-prometheus)
|
||||
* [Install and configure Grafana](#install-and-configure-grafana)
|
||||
* [Export metrics and traces from the
|
||||
application](#export-metrics-and-traces-from-the-application)
|
||||
* [Use exemplars to navigate from metrics to
|
||||
traces](#use-exemplars-to-navigate-from-metrics-to-traces)
|
||||
* [Learn more](#learn-more)
|
||||
|
||||
All these components except the test app require additional configuration to
|
||||
enable Exemplar feature. To make it easy for users, these components are
|
||||
pre-configured to enable Exemplars, and a docker compose file is provided to
|
||||
spun them all up, in the required configurations.
|
||||
</details>
|
||||
|
||||
## Pre-requisite
|
||||
[Exemplars](../customizing-the-sdk/README.md#exemplars) are example data points
|
||||
for aggregated data. They provide specific context to otherwise general
|
||||
aggregations. One common use case is to gain ability to correlate metrics to
|
||||
traces (and logs). While OpenTelemetry .NET supports Exemplars, it is only
|
||||
useful if the telemetry backend also supports the capabilities. This tutorial
|
||||
uses well known open-source backends to demonstrate the concept. The following
|
||||
components are involved:
|
||||
|
||||
Install docker: <https://docs.docker.com/get-docker/>
|
||||
* [Program.cs](./Program.cs) - this application is instrumented with
|
||||
OpenTelemetry, it sends metrics to Prometheus, and traces to Jaeger.
|
||||
* [Prometheus](#install-and-run-prometheus) - Prometheus is used as the metrics
|
||||
backend.
|
||||
* [Jaeger](#install-and-run-jaeger) - Jaeger is used as the distributed tracing
|
||||
backend.
|
||||
* [Grafana](#install-and-configure-grafana) - UI to query metrics from
|
||||
Prometheus, traces from Jaeger, and to navigate between metrics and traces
|
||||
using Exemplars.
|
||||
|
||||
## Setup
|
||||
## Install and run Jaeger
|
||||
|
||||
As mentioned in the intro, this tutorial uses OTel Collector, Prometheus, Tempo,
|
||||
and Grafana, and they must be up and running before proceeding. The following
|
||||
spins all of them with the correct configurations to support Exemplars.
|
||||
Download the [latest binary distribution
|
||||
archive](https://www.jaegertracing.io/download/) of Jaeger.
|
||||
|
||||
Navigate to current directory and run the following:
|
||||
After finished downloading, extract it to a local location that's easy to
|
||||
access. Run the `jaeger-all-in-one(.exe)` executable:
|
||||
|
||||
```sh
|
||||
docker compose up -d
|
||||
./jaeger-all-in-one --collector.otlp.enabled
|
||||
```
|
||||
|
||||
If the above step succeeds, all dependencies would be spun up and ready now. To
|
||||
test, navigate to Grafana running at: `http://localhost:3000/`.
|
||||
## Install and run Prometheus
|
||||
|
||||
## Run test app
|
||||
Follow the [first steps](https://prometheus.io/docs/introduction/first_steps/)
|
||||
to download the [latest release](https://prometheus.io/download/) of Prometheus.
|
||||
|
||||
Now that the required dependencies are ready, lets run the demo app.
|
||||
This tutorial is using the existing ASP.NET Core app from the repo.
|
||||
|
||||
Navigate to [Example Asp.Net Core App](../../../examples/AspNetCore/Program.cs)
|
||||
directory and run the following command:
|
||||
After finished downloading, extract it to a local location that's easy to
|
||||
access. Run the `prometheus(.exe)` server executable with feature flags
|
||||
[exemplars
|
||||
storage](https://prometheus.io/docs/prometheus/latest/feature_flags/#exemplars-storage)
|
||||
and
|
||||
[otlp-receiver](https://prometheus.io/docs/prometheus/latest/feature_flags/#otlp-receiver)
|
||||
enabled:
|
||||
|
||||
```sh
|
||||
./prometheus --enable-feature=exemplar-storage --enable-feature=otlp-write-receiver
|
||||
```
|
||||
|
||||
## Install and configure Grafana
|
||||
|
||||
Follow the operating system specific instructions to [download and install
|
||||
Grafana](https://grafana.com/docs/grafana/latest/setup-grafana/installation/#supported-operating-systems).
|
||||
|
||||
After installation, start the standalone Grafana server (`grafana-server.exe` or
|
||||
`./bin/grafana-server`, depending on the operating system). Then, use a
|
||||
[supported web
|
||||
browser](https://grafana.com/docs/grafana/latest/setup-grafana/installation/#supported-web-browsers)
|
||||
to navigate to [http://localhost:3000/](http://localhost:3000/).
|
||||
|
||||
Follow the instructions in the Grafana getting started
|
||||
[doc](https://grafana.com/docs/grafana/latest/getting-started/getting-started/#step-2-log-in)
|
||||
to log in.
|
||||
|
||||
After successfully logging in, hover on the Configuration icon
|
||||
on the panel at the left hand side, and click on Plugins.
|
||||
|
||||
Find and click on the Jaeger plugin. Next click on `Create a Jaeger data source`
|
||||
button. Make the following changes:
|
||||
|
||||
1. Set "URL" to `http://localhost:16686/`.
|
||||
2. At the bottom of the page click `Save & test` to ensure the data source is
|
||||
working.
|
||||
|
||||
|
||||
|
||||
Find and click on the Prometheus plugin. Next click on
|
||||
`Create a Prometheus data source` button. Make the following changes:
|
||||
|
||||
1. Set "URL" to `http://localhost:9090`.
|
||||
2. Under the "Exemplars" section, enable "Internal link", set "Data source" to
|
||||
`Jaeger`, and set "Label name" to `trace_id`.
|
||||
3. At the bottom of the page click `Save & test` to ensure the data source is
|
||||
working.
|
||||
|
||||
|
||||
|
||||
## Export metrics and traces from the application
|
||||
|
||||
Create a new console application and run it:
|
||||
|
||||
```sh
|
||||
dotnet new console --output exemplars
|
||||
cd exemplars
|
||||
dotnet run
|
||||
```
|
||||
|
||||
Once the application is running, navigate to
|
||||
[http://localhost:5000/weatherforecast]("http://localhost:5000/weatherforecast")
|
||||
from a web browser. You may use the following Powershell script to generate load
|
||||
to the application.
|
||||
Add reference to [OTLP
|
||||
Exporter](../../../src/OpenTelemetry.Exporter.OpenTelemetryProtocol/README.md):
|
||||
|
||||
```powershell
|
||||
while($true)
|
||||
{
|
||||
Invoke-WebRequest http://localhost:5000/weatherforecast
|
||||
Start-Sleep -Milliseconds 500
|
||||
}
|
||||
```sh
|
||||
dotnet add package OpenTelemetry.Exporter.OpenTelemetryProtocol
|
||||
```
|
||||
|
||||
## Use Exemplars to navigate from Metrics to Traces
|
||||
Now copy the code from [Program.cs](./Program.cs) and run the application again.
|
||||
The application will start sending metrics to Prometheus and traces to Jaeger.
|
||||
|
||||
The application sends metrics (with exemplars), and traces to the OTel
|
||||
Collector, which export metrics and traces to Prometheus and Tempo
|
||||
respectively.
|
||||
The application is configured with trace-based exemplar filter, which enables
|
||||
the OpenTelemetry SDK to attach exemplars to metrics:
|
||||
|
||||
Please wait for 2 minutes before continuing so that enough data is generated
|
||||
and exported.
|
||||
```csharp
|
||||
var meterProvider = Sdk.CreateMeterProviderBuilder()
|
||||
...
|
||||
.SetExemplarFilter(ExemplarFilterType.TraceBased)
|
||||
...
|
||||
```
|
||||
|
||||
For more details about the `SetExemplarFilter` API see: [Customizing
|
||||
OpenTelemetry .NET SDK for Metrics >
|
||||
ExemplarFilter](../customizing-the-sdk/README.md#exemplarfilter).
|
||||
|
||||
## Use exemplars to navigate from metrics to traces
|
||||
|
||||
Open Grafana, select Explore, and select Prometheus as the source. Select the
|
||||
metric named "http_server_duration_bucket", and plot the chart. Toggle on the
|
||||
"Exemplar" option from the UI and hit refresh.
|
||||
metric named `MyHistogram_bucket`, and plot the chart. Toggle on the "Exemplars"
|
||||
option from the UI and hit refresh.
|
||||
|
||||
|
||||
|
||||
|
||||
The Exemplars appear as special "diamond shaped dots" along with the metric
|
||||
charts in the UI. Select any Exemplar to see the exemplar data, which includes
|
||||
charts in the UI. Select any exemplar to see the exemplar data, which includes
|
||||
the timestamp when the measurement was recorded, the raw value, and trace
|
||||
context when the recording was done. The "trace_id" enables jumping to the
|
||||
tracing backed (tempo). Click on the "Query with Tempo" button next to the
|
||||
"trace_id" field to open the corresponding `Trace` in Tempo.
|
||||
tracing backed (Jaeger in this case). Click on the "Query with Jaeger" button
|
||||
next to the "trace_id" field to open the corresponding trace in Jaeger.
|
||||
|
||||
|
||||
|
||||
|
||||
## References
|
||||
## Learn more
|
||||
|
||||
* [Exemplar specification](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/sdk.md#exemplar)
|
||||
* [Exemplars in Prometheus](https://prometheus.io/docs/prometheus/latest/feature_flags/#exemplars-storage)
|
||||
* [Exemplars in Grafana](https://grafana.com/docs/grafana/latest/fundamentals/exemplars/)
|
||||
* [Tempo](https://github.com/grafana/tempo)
|
||||
* [Exemplar
|
||||
specification](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/sdk.md#exemplar)
|
||||
* [What is Prometheus?](https://prometheus.io/docs/introduction/overview/)
|
||||
* [Prometheus now supports OpenTelemetry
|
||||
Metrics](https://horovits.medium.com/prometheus-now-supports-opentelemetry-metrics-83f85878e46a)
|
||||
* [Jaeger Tracing](https://www.jaegertracing.io/)
|
||||
* [Grafana support for
|
||||
Prometheus](https://prometheus.io/docs/visualization/grafana/#creating-a-prometheus-graph)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,5 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="$(RepoRoot)\src\OpenTelemetry.Exporter.OpenTelemetryProtocol\OpenTelemetry.Exporter.OpenTelemetryProtocol.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
Loading…
Reference in New Issue