CustomProperty improvements (#1261)

* CustomProperty improvements

updating tests

updating to RuntimeContextSlot

updating to runtimeContext

adding runtime/subscibe/flag benchmark

updating aspnet and http

updating ramaining instrumentation

* adding try/catch&log

* improve docs for enrich action

* Checking enrichment with sampler

* removing unneeded using

* updating changelog

* updating changelog

Co-authored-by: Cijo Thomas <cithomas@microsoft.com>

src/OpenTelemetry.Instrumentation.AspNet/AspNetInstrumentationOptions.cs

@@ -15,6 +15,7 @@
 // </copyright>
 
 using System;
+using System.Diagnostics;
 using System.Web;
 using OpenTelemetry.Context.Propagation;
 
@@ -41,5 +42,16 @@ namespace OpenTelemetry.Instrumentation.AspNet
         /// If Filter returns false or throw exception, the request is filtered out.
         /// </summary>
         public Func<HttpContext, bool> Filter { get; set; }
+
+        /// <summary>
+        /// Gets or sets an action to enrich an Activity.
+        /// </summary>
+        /// <remarks>
+        /// <para><see cref="Activity"/>: the activity being enriched.</para>
+        /// <para>string: the name of the event.</para>
+        /// <para>object: the raw object from which additional information can be extracted to enrich the activity.
+        /// The type of this object depends on the event, which is given by the above parameter.</para>
+        /// </remarks>
+        public Action<Activity, string, object> Enrich { get; set; }
     }
 }

src/OpenTelemetry.Instrumentation.AspNet/CHANGELOG.md

@@ -2,6 +2,11 @@
 
 ## Unreleased
 
+* Instrumntation no longer store raw objects like `HttpRequest` in
+  Activity.CustomProperty. To enrich activity, use the Enrich action on the
+  instrumentation.
+  ([#1261](https://github.com/open-telemetry/opentelemetry-dotnet/pull/1261))
+
 ## 0.6.0-beta.1
 
 Released 2020-Sep-15
@@ -10,8 +15,8 @@ Released 2020-Sep-15
 
 Released 2020-08-28
 
-* Added Filter public API on AspNetInstrumentationOptions to allow
-  filtering of instrumentation based on HttpContext.
+* Added Filter public API on AspNetInstrumentationOptions to allow filtering of
+  instrumentation based on HttpContext.
 
 * Asp.Net Instrumentation automatically populates HttpRequest, HttpResponse in
   Activity custom property

src/OpenTelemetry.Instrumentation.AspNet/Implementation/AspNetInstrumentationEventSource.cs

@@ -54,5 +54,20 @@ namespace OpenTelemetry.Instrumentation.AspNet.Implementation
         {
             this.WriteEvent(3, exception);
         }
+
+        [NonEvent]
+        public void EnrichmentException(Exception ex)
+        {
+            if (this.IsEnabled(EventLevel.Error, (EventKeywords)(-1)))
+            {
+                this.EnrichmentException(ex.ToInvariantString());
+            }
+        }
+
+        [Event(4, Message = "Enrichment threw exception. Exception {0}.", Level = EventLevel.Error)]
+        public void EnrichmentException(string exception)
+        {
+            this.WriteEvent(4, exception);
+        }
     }
 }

src/OpenTelemetry.Instrumentation.AspNet/Implementation/HttpInListener.cs

@@ -109,7 +109,15 @@ namespace OpenTelemetry.Instrumentation.AspNet.Implementation
 
             if (activity.IsAllDataRequested)
             {
-                activity.SetCustomProperty(RequestCustomPropertyName, request);
+                try
+                {
+                    this.options.Enrich?.Invoke(activity, "OnStartActivity", request);
+                }
+                catch (Exception ex)
+                {
+                    AspNetInstrumentationEventSource.Log.EnrichmentException(ex);
+                }
+
                 if (request.Url.Port == 80 || request.Url.Port == 443)
                 {
                     activity.SetTag(SemanticConventions.AttributeHttpHost, request.Url.Host);
@@ -159,7 +167,15 @@ namespace OpenTelemetry.Instrumentation.AspNet.Implementation
 
                 var response = context.Response;
 
-                activityToEnrich.SetCustomProperty(ResponseCustomPropertyName, response);
+                try
+                {
+                    this.options.Enrich?.Invoke(activity, "OnStopActivity", response);
+                }
+                catch (Exception ex)
+                {
+                    AspNetInstrumentationEventSource.Log.EnrichmentException(ex);
+                }
+
                 activityToEnrich.SetTag(SemanticConventions.AttributeHttpStatusCode, response.StatusCode);
 
                 activityToEnrich.SetStatus(

src/OpenTelemetry.Instrumentation.AspNetCore/AspNetCoreInstrumentationOptions.cs

@@ -15,6 +15,7 @@
 // </copyright>
 
 using System;
+using System.Diagnostics;
 using Microsoft.AspNetCore.Http;
 using OpenTelemetry.Context.Propagation;
 
@@ -41,5 +42,16 @@ namespace OpenTelemetry.Instrumentation.AspNetCore
         /// If Filter returns false or throw exception, the request is filtered out.
         /// </summary>
         public Func<HttpContext, bool> Filter { get; set; }
+
+        /// <summary>
+        /// Gets or sets an action to enrich an Activity.
+        /// </summary>
+        /// <remarks>
+        /// <para><see cref="Activity"/>: the activity being enriched.</para>
+        /// <para>string: the name of the event.</para>
+        /// <para>object: the raw object from which additional information can be extracted to enrich the activity.
+        /// The type of this object depends on the event, which is given by the above parameter.</para>
+        /// </remarks>
+        public Action<Activity, string, object> Enrich { get; set; }
     }
 }

src/OpenTelemetry.Instrumentation.AspNetCore/CHANGELOG.md

@@ -2,6 +2,11 @@
 
 ## Unreleased
 
+* Instrumntation no longer store raw objects like `HttpRequest` in
+  Activity.CustomProperty. To enrich activity, use the Enrich action on the
+  instrumentation.
+  ([#1261](https://github.com/open-telemetry/opentelemetry-dotnet/pull/1261))
+
 ## 0.6.0-beta.1
 
 Released 2020-Sep-15
@@ -16,11 +21,11 @@ Released 2020-Sep-15
 
 Released 2020-08-28
 
-* Added Filter public API on AspNetCoreInstrumentationOptions to allow
-  filtering of instrumentation based on HttpContext.
+* Added Filter public API on AspNetCoreInstrumentationOptions to allow filtering
+  of instrumentation based on HttpContext.
 
-* Asp.Net Core Instrumentation automatically populates HttpRequest,
-  HttpResponse in Activity custom property
+* Asp.Net Core Instrumentation automatically populates HttpRequest, HttpResponse
+  in Activity custom property
 
 * Changed the default propagation to support W3C Baggage
   ([#1048](https://github.com/open-telemetry/opentelemetry-dotnet/pull/1048))

src/OpenTelemetry.Instrumentation.AspNetCore/Implementation/AspNetCoreInstrumentationEventSource.cs

@@ -54,5 +54,20 @@ namespace OpenTelemetry.Instrumentation.AspNetCore.Implementation
         {
             this.WriteEvent(3, exception);
         }
+
+        [NonEvent]
+        public void EnrichmentException(Exception ex)
+        {
+            if (this.IsEnabled(EventLevel.Error, (EventKeywords)(-1)))
+            {
+                this.EnrichmentException(ex.ToInvariantString());
+            }
+        }
+
+        [Event(4, Message = "Enrichment threw exception. Exception {0}.", Level = EventLevel.Error)]
+        public void EnrichmentException(string exception)
+        {
+            this.WriteEvent(4, exception);
+        }
     }
 }

src/OpenTelemetry.Instrumentation.AspNetCore/Implementation/HttpInListener.cs

@@ -112,7 +112,14 @@ namespace OpenTelemetry.Instrumentation.AspNetCore.Implementation
 
             if (activity.IsAllDataRequested)
             {
-                activity.SetCustomProperty(RequestCustomPropertyName, request);
+                try
+                {
+                    this.options.Enrich?.Invoke(activity, "OnStartActivity", request);
+                }
+                catch (Exception ex)
+                {
+                    AspNetCoreInstrumentationEventSource.Log.EnrichmentException(ex);
+                }
 
                 var path = (request.PathBase.HasValue || request.Path.HasValue) ? (request.PathBase + request.Path).ToString() : "/";
                 activity.DisplayName = path;
@@ -152,7 +159,16 @@ namespace OpenTelemetry.Instrumentation.AspNetCore.Implementation
                 }
 
                 var response = context.Response;
-                activity.SetCustomProperty(ResponseCustomPropertyName, response);
+
+                try
+                {
+                    this.options.Enrich?.Invoke(activity, "OnStopActivity", response);
+                }
+                catch (Exception ex)
+                {
+                    AspNetCoreInstrumentationEventSource.Log.EnrichmentException(ex);
+                }
+
                 activity.SetTag(SemanticConventions.AttributeHttpStatusCode, response.StatusCode);
 
                 if (TryGetGrpcMethod(activity, out var grpcMethod))

src/OpenTelemetry.Instrumentation.GrpcNetClient/CHANGELOG.md

@@ -2,22 +2,26 @@
 
 ## Unreleased
 
+* Instrumntation no longer store raw objects like `HttpRequestMessage` in
+  Activity.CustomProperty. To enrich activity, use the Enrich action on the
+  instrumentation.
+  ([#1261](https://github.com/open-telemetry/opentelemetry-dotnet/pull/1261))
+
 ## 0.6.0-beta.1
 
 Released 2020-Sep-15
 
-* The `grpc.method` and `grpc.status_code` attributes
-  added by the library are removed from the span. The information from these
-  attributes is contained in other attributes that follow the conventions of
-  OpenTelemetry.
+* The `grpc.method` and `grpc.status_code` attributes added by the library are
+  removed from the span. The information from these attributes is contained in
+  other attributes that follow the conventions of OpenTelemetry.
   ([#1260](https://github.com/open-telemetry/opentelemetry-dotnet/pull/1260)).
 
 ## 0.5.0-beta.2
 
 Released 2020-08-28
 
-* NuGet package renamed to OpenTelemetry.Instrumentation.GrpcNetClient to
-  more clearly indicate that this package is specifically for gRPC client
+* NuGet package renamed to OpenTelemetry.Instrumentation.GrpcNetClient to more
+  clearly indicate that this package is specifically for gRPC client
   instrumentation. The package was previously named
   OpenTelemetry.Instrumentation.Grpc.
   ([#1136](https://github.com/open-telemetry/opentelemetry-dotnet/pull/1136))

src/OpenTelemetry.Instrumentation.GrpcNetClient/GrpcClientInstrumentationOptions.cs

@@ -14,6 +14,9 @@
 // limitations under the License.
 // </copyright>
 
+using System;
+using System.Diagnostics;
+
 namespace OpenTelemetry.Instrumentation.GrpcNetClient
 {
     /// <summary>
@@ -25,5 +28,16 @@ namespace OpenTelemetry.Instrumentation.GrpcNetClient
         /// Gets or sets a value indicating whether down stream instrumentation is suppressed (disabled).
         /// </summary>
         public bool SuppressDownstreamInstrumentation { get; set; }
+
+        /// <summary>
+        /// Gets or sets an action to enrich an Activity.
+        /// </summary>
+        /// <remarks>
+        /// <para><see cref="Activity"/>: the activity being enriched.</para>
+        /// <para>string: the name of the event.</para>
+        /// <para>object: the raw object from which additional information can be extracted to enrich the activity.
+        /// The type of this object depends on the event, which is given by the above parameter.</para>
+        /// </remarks>
+        public Action<Activity, string, object> Enrich { get; set; }
     }
 }

src/OpenTelemetry.Instrumentation.GrpcNetClient/Implementation/GrpcClientDiagnosticListener.cs

@@ -61,7 +61,15 @@ namespace OpenTelemetry.Instrumentation.GrpcNetClient.Implementation
 
             if (activity.IsAllDataRequested)
             {
-                activity.SetCustomProperty(RequestCustomPropertyName, request);
+                try
+                {
+                    this.options.Enrich?.Invoke(activity, "OnStartActivity", request);
+                }
+                catch (Exception ex)
+                {
+                    GrpcInstrumentationEventSource.Log.EnrichmentException(ex);
+                }
+
                 activity.SetTag(SemanticConventions.AttributeRpcSystem, GrpcTagHelper.RpcSystemGrpc);
 
                 if (GrpcTagHelper.TryParseRpcServiceAndRpcMethod(grpcMethod, out var rpcService, out var rpcMethod))

src/OpenTelemetry.Instrumentation.GrpcNetClient/Implementation/GrpcInstrumentationEventSource.cs

@@ -14,7 +14,9 @@
 // limitations under the License.
 // </copyright>
 
+using System;
 using System.Diagnostics.Tracing;
+using OpenTelemetry.Internal;
 
 namespace OpenTelemetry.Instrumentation.GrpcNetClient.Implementation
 {
@@ -31,5 +33,20 @@ namespace OpenTelemetry.Instrumentation.GrpcNetClient.Implementation
         {
             this.WriteEvent(1, handlerName, eventName);
         }
+
+        [NonEvent]
+        public void EnrichmentException(Exception ex)
+        {
+            if (this.IsEnabled(EventLevel.Error, (EventKeywords)(-1)))
+            {
+                this.EnrichmentException(ex.ToInvariantString());
+            }
+        }
+
+        [Event(2, Message = "Enrichment thrw exception. Exception {0}.", Level = EventLevel.Error)]
+        public void EnrichmentException(string exception)
+        {
+            this.WriteEvent(2, exception);
+        }
     }
 }

src/OpenTelemetry.Instrumentation.GrpcNetClient/OpenTelemetry.Instrumentation.GrpcNetClient.csproj

@@ -6,6 +6,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <Compile Include="$(RepoRoot)\src\OpenTelemetry.Api\Internal\ExceptionExtensions.cs" Link="Internal\ExceptionExtensions.cs" />
     <Compile Include="$(RepoRoot)\src\OpenTelemetry.Api\Trace\SemanticConventions.cs" />
   </ItemGroup>
 

src/OpenTelemetry.Instrumentation.Http/CHANGELOG.md

@@ -2,6 +2,11 @@
 
 ## Unreleased
 
+* Instrumntation no longer store raw objects like `HttpRequestMessage` in
+  Activity.CustomProperty. To enrich activity, use the Enrich action on the
+  instrumentation.
+  ([#1261](https://github.com/open-telemetry/opentelemetry-dotnet/pull/1261))
+
 ## 0.6.0-beta.1
 
 Released 2020-Sep-15

src/OpenTelemetry.Instrumentation.Http/HttpClientInstrumentationOptions.cs

@@ -15,6 +15,7 @@
 // </copyright>
 
 using System;
+using System.Diagnostics;
 using System.Net.Http;
 using System.Runtime.CompilerServices;
 using OpenTelemetry.Context.Propagation;
@@ -50,6 +51,17 @@ namespace OpenTelemetry.Instrumentation.Http
         /// </summary>
         public Func<HttpRequestMessage, bool> Filter { get; set; }
 
+        /// <summary>
+        /// Gets or sets an action to enrich an Activity.
+        /// </summary>
+        /// <remarks>
+        /// <para><see cref="Activity"/>: the activity being enriched.</para>
+        /// <para>string: the name of the event.</para>
+        /// <para>object: the raw object from which additional information can be extracted to enrich the activity.
+        /// The type of this object depends on the event, which is given by the above parameter.</para>
+        /// </remarks>
+        public Action<Activity, string, object> Enrich { get; set; }
+
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         internal bool EventFilter(string activityName, object arg1)
         {

src/OpenTelemetry.Instrumentation.Http/Implementation/HttpHandlerDiagnosticListener.cs

@@ -99,7 +99,15 @@ namespace OpenTelemetry.Instrumentation.Http.Implementation
 
             if (activity.IsAllDataRequested)
             {
-                activity.SetCustomProperty(RequestCustomPropertyName, request);
+                try
+                {
+                    this.options.Enrich?.Invoke(activity, "OnStartActivity", request);
+                }
+                catch (Exception ex)
+                {
+                    HttpInstrumentationEventSource.Log.EnrichmentException(ex);
+                }
+
                 activity.SetTag(SemanticConventions.AttributeHttpMethod, HttpTagHelper.GetNameForHttpMethod(request.Method));
                 activity.SetTag(SemanticConventions.AttributeHttpHost, HttpTagHelper.GetHostTagValueFromRequestUri(request.RequestUri));
                 activity.SetTag(SemanticConventions.AttributeHttpUrl, request.RequestUri.OriginalString);
@@ -139,7 +147,14 @@ namespace OpenTelemetry.Instrumentation.Http.Implementation
 
                 if (this.stopResponseFetcher.Fetch(payload) is HttpResponseMessage response)
                 {
-                    activity.SetCustomProperty(ResponseCustomPropertyName, response);
+                    try
+                    {
+                        this.options.Enrich?.Invoke(activity, "OnStopActivity", response);
+                    }
+                    catch (Exception ex)
+                    {
+                        HttpInstrumentationEventSource.Log.EnrichmentException(ex);
+                    }
 
                     activity.SetTag(SemanticConventions.AttributeHttpStatusCode, (int)response.StatusCode);
 
@@ -163,7 +178,14 @@ namespace OpenTelemetry.Instrumentation.Http.Implementation
                     return;
                 }
 
-                activity.SetCustomProperty(ExceptionCustomPropertyName, exc);
+                try
+                {
+                    this.options.Enrich?.Invoke(activity, "OnException", exc);
+                }
+                catch (Exception ex)
+                {
+                    HttpInstrumentationEventSource.Log.EnrichmentException(ex);
+                }
 
                 if (exc is HttpRequestException)
                 {

src/OpenTelemetry.Instrumentation.Http/Implementation/HttpInstrumentationEventSource.cs

@@ -78,5 +78,20 @@ namespace OpenTelemetry.Instrumentation.Http.Implementation
         {
             this.WriteEvent(4, exception);
         }
+
+        [NonEvent]
+        public void EnrichmentException(Exception ex)
+        {
+            if (this.IsEnabled(EventLevel.Error, (EventKeywords)(-1)))
+            {
+                this.EnrichmentException(ex.ToInvariantString());
+            }
+        }
+
+        [Event(5, Message = "Enrichment thrw exception. Exception {0}.", Level = EventLevel.Error)]
+        public void EnrichmentException(string exception)
+        {
+            this.WriteEvent(5, exception);
+        }
     }
 }

src/OpenTelemetry.Instrumentation.SqlClient/CHANGELOG.md

@@ -2,6 +2,11 @@
 
 ## Unreleased
 
+* Instrumntation no longer store raw objects like `object` in
+  Activity.CustomProperty. To enrich activity, use the Enrich action on the
+  instrumentation.
+  ([#1261](https://github.com/open-telemetry/opentelemetry-dotnet/pull/1261))
+
 ## 0.6.0-beta.1
 
 Released 2020-Sep-15

src/OpenTelemetry.Instrumentation.SqlClient/Implementation/SqlClientDiagnosticListener.cs

@@ -89,7 +89,15 @@ namespace OpenTelemetry.Instrumentation.SqlClient.Implementation
                             var database = this.databaseFetcher.Fetch(connection);
 
                             activity.DisplayName = (string)database;
-                            activity.SetCustomProperty(CommandCustomPropertyName, command);
+                            try
+                            {
+                                this.options.Enrich?.Invoke(activity, "OnCustom", command);
+                            }
+                            catch (Exception ex)
+                            {
+                                SqlClientInstrumentationEventSource.Log.EnrichmentException(ex);
+                            }
+
                             var dataSource = this.dataSourceFetcher.Fetch(connection);
                             var commandText = this.commandTextFetcher.Fetch(command);
 

src/OpenTelemetry.Instrumentation.SqlClient/Implementation/SqlClientInstrumentationEventSource.cs

@@ -60,5 +60,20 @@ namespace OpenTelemetry.Instrumentation.SqlClient.Implementation
         {
             this.WriteEvent(4, handlerName, eventName);
         }
+
+        [NonEvent]
+        public void EnrichmentException(Exception ex)
+        {
+            if (this.IsEnabled(EventLevel.Error, (EventKeywords)(-1)))
+            {
+                this.EnrichmentException(ex.ToInvariantString());
+            }
+        }
+
+        [Event(5, Message = "Enrichment thrw exception. Exception {0}.", Level = EventLevel.Error)]
+        public void EnrichmentException(string exception)
+        {
+            this.WriteEvent(5, exception);
+        }
     }
 }

src/OpenTelemetry.Instrumentation.SqlClient/SqlClientInstrumentationOptions.cs

@@ -13,6 +13,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 // </copyright>
+
 using System;
 using System.Collections.Concurrent;
 using System.Data;
@@ -56,6 +57,17 @@ namespace OpenTelemetry.Instrumentation.SqlClient
         /// </remarks>
         public bool EnableConnectionLevelAttributes { get; set; }
 
+        /// <summary>
+        /// Gets or sets an action to enrich an Activity.
+        /// </summary>
+        /// <remarks>
+        /// <para><see cref="Activity"/>: the activity being enriched.</para>
+        /// <para>string: the name of the event.</para>
+        /// <para>object: the raw object from which additional information can be extracted to enrich the activity.
+        /// The type of this object depends on the event, which is given by the above parameter.</para>
+        /// </remarks>
+        public Action<Activity, string, object> Enrich { get; set; }
+
         internal static SqlConnectionDetails ParseDataSource(string dataSource)
         {
             Match match = DataSourceRegex.Match(dataSource);

src/OpenTelemetry.Instrumentation.StackExchangeRedis/StackExchangeRedisCallsInstrumentationOptions.cs

@@ -13,6 +13,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 // </copyright>
+
 using System;
 using System.Diagnostics;
 

test/OpenTelemetry.Instrumentation.AspNet.Tests/HttpInListenerTests.cs

@@ -58,7 +58,6 @@ namespace OpenTelemetry.Instrumentation.AspNet.Tests
         [InlineData("http://localhost/api/value", 0, null, "TraceContext", "/api/value")] // Request will be filtered
         [InlineData("http://localhost/api/value", 0, null, "TraceContext", "{ThrowException}")] // Filter user code will throw an exception
         [InlineData("http://localhost/api/value/2", 0, null, "CustomContext", "/api/value")] // Request will not be filtered
-        [InlineData("http://localhost/api/value/2", 0, null, "CustomContext", "/api/value", true)] // Request will not be filtered
         public void AspNetRequestsAreCollectedSuccessfully(
             string url,
             int routeType,
@@ -162,6 +161,8 @@ namespace OpenTelemetry.Instrumentation.AspNet.Tests
                     {
                         options.Propagator = propagator.Object;
                     }
+
+                    options.Enrich = ActivityEnrichment;
                 })
             .SetResource(expectedResource)
             .AddProcessor(activityProcessor.Object).Build())
@@ -251,13 +252,23 @@ namespace OpenTelemetry.Instrumentation.AspNet.Tests
             Assert.Equal(HttpContext.Current.Request.UserAgent, span.GetTagValue(SemanticConventions.AttributeHttpUserAgent) as string);
 
             Assert.Equal(expectedResource, span.GetResource());
-            var request = span.GetCustomProperty(HttpInListener.RequestCustomPropertyName);
-            Assert.NotNull(request);
-            Assert.True(request is HttpRequest);
+        }
 
-            var response = span.GetCustomProperty(HttpInListener.ResponseCustomPropertyName);
-            Assert.NotNull(response);
-            Assert.True(response is HttpResponse);
+        private static void ActivityEnrichment(Activity activity, string method, object obj)
+        {
+            switch (method)
+            {
+                case "OnStartActivity":
+                    Assert.True(obj is HttpRequest);
+                    break;
+
+                case "OnStopActivity":
+                    Assert.True(obj is HttpResponse);
+                    break;
+
+                default:
+                    break;
+            }
         }
 
         private class FakeAspNetDiagnosticSource : IDisposable

test/OpenTelemetry.Instrumentation.AspNetCore.Tests/BasicTests.cs

@@ -58,15 +58,23 @@ namespace OpenTelemetry.Instrumentation.AspNetCore.Tests
             Assert.Throws<ArgumentNullException>(() => builder.AddAspNetCoreInstrumentation());
         }
 
-        [Fact]
-        public async Task SuccessfulTemplateControllerCallGeneratesASpan()
+        [Theory]
+        [InlineData(true)]
+        [InlineData(false)]
+        public async Task SuccessfulTemplateControllerCallGeneratesASpan(bool shouldEnrich)
         {
             var expectedResource = Resources.Resources.CreateServiceResource("test-service");
             var activityProcessor = new Mock<ActivityProcessor>();
             void ConfigureTestServices(IServiceCollection services)
             {
                 this.openTelemetrySdk = Sdk.CreateTracerProviderBuilder()
-                    .AddAspNetCoreInstrumentation()
+                    .AddAspNetCoreInstrumentation(options =>
+                    {
+                        if (shouldEnrich)
+                        {
+                            options.Enrich = ActivityEnrichment;
+                        }
+                    })
                     .SetResource(expectedResource)
                     .AddProcessor(activityProcessor.Object)
                     .Build();
@@ -309,13 +317,23 @@ namespace OpenTelemetry.Instrumentation.AspNetCore.Tests
             Assert.Equal(ActivityKind.Server, activityToValidate.Kind);
             Assert.Equal(expectedHttpPath, activityToValidate.GetTagValue(SpanAttributeConstants.HttpPathKey) as string);
             Assert.Equal(expectedResource, activityToValidate.GetResource());
-            var request = activityToValidate.GetCustomProperty(HttpInListener.RequestCustomPropertyName);
-            Assert.NotNull(request);
-            Assert.True(request is HttpRequest);
+        }
 
-            var response = activityToValidate.GetCustomProperty(HttpInListener.ResponseCustomPropertyName);
-            Assert.NotNull(response);
-            Assert.True(response is HttpResponse);
+        private static void ActivityEnrichment(Activity activity, string method, object obj)
+        {
+            switch (method)
+            {
+                case "OnStartActivity":
+                    Assert.True(obj is HttpRequest);
+                    break;
+
+                case "OnStopActivity":
+                    Assert.True(obj is HttpResponse);
+                    break;
+
+                default:
+                    break;
+            }
         }
     }
 }

test/OpenTelemetry.Instrumentation.Grpc.Tests/GrpcTests.client.cs

@@ -32,9 +32,12 @@ namespace OpenTelemetry.Instrumentation.Grpc.Tests
     {
         [Theory]
         [InlineData("http://localhost")]
+        [InlineData("http://localhost", false)]
         [InlineData("http://127.0.0.1")]
+        [InlineData("http://127.0.0.1", false)]
         [InlineData("http://[::1]")]
-        public void GrpcClientCallsAreCollectedSuccessfully(string baseAddress)
+        [InlineData("http://[::1]", false)]
+        public void GrpcClientCallsAreCollectedSuccessfully(string baseAddress, bool shouldEnrich = true)
         {
             var uri = new Uri($"{baseAddress}:{this.fixture.Port}");
             var uriHostNameType = Uri.CheckHostName(uri.Host);
@@ -47,7 +50,13 @@ namespace OpenTelemetry.Instrumentation.Grpc.Tests
 
             using (Sdk.CreateTracerProviderBuilder()
                     .SetSampler(new AlwaysOnSampler())
-                    .AddGrpcClientInstrumentation()
+                    .AddGrpcClientInstrumentation(options =>
+                    {
+                        if (shouldEnrich)
+                        {
+                            options.Enrich = ActivityEnrichment;
+                        }
+                    })
                     .SetResource(expectedResource)
                     .AddProcessor(processor.Object)
                     .Build())
@@ -91,8 +100,10 @@ namespace OpenTelemetry.Instrumentation.Grpc.Tests
             Assert.Null(activity.GetTagValue(GrpcTagHelper.GrpcStatusCodeTagName));
         }
 
-        [Fact]
-        public void GrpcAndHttpClientInstrumentationIsInvoked()
+        [Theory]
+        [InlineData(true)]
+        [InlineData(false)]
+        public void GrpcAndHttpClientInstrumentationIsInvoked(bool shouldEnrich)
         {
             var uri = new Uri($"http://localhost:{this.fixture.Port}");
             var expectedResource = Resources.Resources.CreateServiceResource("test-service");
@@ -104,12 +115,18 @@ namespace OpenTelemetry.Instrumentation.Grpc.Tests
             using (Sdk.CreateTracerProviderBuilder()
                     .SetSampler(new AlwaysOnSampler())
                     .SetResource(expectedResource)
-                    .AddGrpcClientInstrumentation()
+                    .AddGrpcClientInstrumentation(options =>
+                    {
+                        if (shouldEnrich)
+                        {
+                            options.Enrich = ActivityEnrichment;
+                        }
+                    })
                     .AddHttpClientInstrumentation()
                     .AddProcessor(processor.Object)
                     .Build())
             {
-                var channel = GrpcChannel.ForAddress(uri);
+                using var channel = GrpcChannel.ForAddress(uri);
                 var client = new Greeter.GreeterClient(channel);
                 var rs = client.SayHello(new HelloRequest());
             }
@@ -124,8 +141,10 @@ namespace OpenTelemetry.Instrumentation.Grpc.Tests
             Assert.Equal(grpcSpan.SpanId, httpSpan.ParentSpanId);
         }
 
-        [Fact]
-        public void GrpcAndHttpClientInstrumentationWithSuppressInstrumentation()
+        [Theory]
+        [InlineData(true)]
+        [InlineData(false)]
+        public void GrpcAndHttpClientInstrumentationWithSuppressInstrumentation(bool shouldEnrich)
         {
             var uri = new Uri($"http://localhost:{this.fixture.Port}");
             var expectedResource = Resources.Resources.CreateServiceResource("test-service");
@@ -137,7 +156,14 @@ namespace OpenTelemetry.Instrumentation.Grpc.Tests
             using (Sdk.CreateTracerProviderBuilder()
                     .SetSampler(new AlwaysOnSampler())
                     .SetResource(expectedResource)
-                    .AddGrpcClientInstrumentation(o => { o.SuppressDownstreamInstrumentation = true; })
+                    .AddGrpcClientInstrumentation(o =>
+                    {
+                        o.SuppressDownstreamInstrumentation = true;
+                        if (shouldEnrich)
+                        {
+                            o.Enrich = ActivityEnrichment;
+                        }
+                    })
                     .AddHttpClientInstrumentation()
                     .AddProcessor(processor.Object)
                     .Build())
@@ -186,13 +212,23 @@ namespace OpenTelemetry.Instrumentation.Grpc.Tests
         {
             Assert.Equal(ActivityKind.Client, activityToValidate.Kind);
             Assert.Equal(expectedResource, activityToValidate.GetResource());
-            var request = activityToValidate.GetCustomProperty(GrpcClientDiagnosticListener.RequestCustomPropertyName);
-            Assert.NotNull(request);
-            Assert.True(request is HttpRequestMessage);
+        }
 
-            var response = activityToValidate.GetCustomProperty(GrpcClientDiagnosticListener.RequestCustomPropertyName);
-            Assert.NotNull(response);
-            Assert.True(response is HttpRequestMessage);
+        private static void ActivityEnrichment(Activity activity, string method, object obj)
+        {
+            switch (method)
+            {
+                case "OnStartActivity":
+                    Assert.True(obj is HttpRequestMessage);
+                    break;
+
+                case "OnStopActivity":
+                    Assert.True(obj is HttpResponseMessage);
+                    break;
+
+                default:
+                    break;
+            }
         }
     }
 }

test/OpenTelemetry.Instrumentation.Grpc.Tests/GrpcTests.server.cs

@@ -15,6 +15,7 @@
 // </copyright>
 using System;
 using System.Diagnostics;
+using System.Linq;
 using System.Net;
 using System.Threading;
 using Greet;
@@ -43,14 +44,15 @@ namespace OpenTelemetry.Instrumentation.Grpc.Tests
             var uri = new Uri($"http://localhost:{this.fixture.Port}");
             var processor = this.fixture.GrpcServerSpanProcessor;
 
-            var channel = GrpcChannel.ForAddress(uri);
+            using var channel = GrpcChannel.ForAddress(uri);
             var client = new Greeter.GreeterClient(channel);
             client.SayHello(new HelloRequest());
 
             WaitForProcessorInvocations(processor, 2);
 
-            Assert.Equal(2, processor.Invocations.Count); // begin and end was called
-            var activity = (Activity)processor.Invocations[1].Arguments[0];
+            Assert.InRange(processor.Invocations.Count, 2, 6); // begin and end was called
+            var activity = (Activity)processor.Invocations.FirstOrDefault(invo =>
+                invo.Method.Name == "OnEnd" && (invo.Arguments[0] as Activity).OperationName == "Microsoft.AspNetCore.Hosting.HttpRequestIn").Arguments[0];
 
             Assert.Equal(ActivityKind.Server, activity.Kind);
             Assert.Equal("grpc", activity.GetTagValue(SemanticConventions.AttributeRpcSystem));

test/OpenTelemetry.Instrumentation.Http.Tests/HttpClientTests.Basic.netcore31.cs

@@ -56,8 +56,10 @@ namespace OpenTelemetry.Instrumentation.Http.Tests
             Assert.Throws<ArgumentNullException>(() => builder.AddHttpClientInstrumentation());
         }
 
-        [Fact]
-        public async Task HttpClientInstrumentationInjectsHeadersAsync()
+        [Theory]
+        [InlineData(true)]
+        [InlineData(false)]
+        public async Task HttpClientInstrumentationInjectsHeadersAsync(bool shouldEnrich)
         {
             var processor = new Mock<ActivityProcessor>();
             var request = new HttpRequestMessage
@@ -93,7 +95,14 @@ namespace OpenTelemetry.Instrumentation.Http.Tests
             //         });
 
             using (Sdk.CreateTracerProviderBuilder()
-                        .AddHttpClientInstrumentation(o => o.Propagator = mockPropagator.Object)
+                        .AddHttpClientInstrumentation(o =>
+                        {
+                            o.Propagator = mockPropagator.Object;
+                            if (shouldEnrich)
+                            {
+                                o.Enrich = ActivityEnrichment;
+                            }
+                        })
                         .AddProcessor(processor.Object)
                         .Build())
             {
@@ -104,7 +113,7 @@ namespace OpenTelemetry.Instrumentation.Http.Tests
             Assert.Equal(4, processor.Invocations.Count); // OnStart/OnEnd/OnShutdown/Dispose called.
             var activity = (Activity)processor.Invocations[1].Arguments[0];
 
-            ValidateHttpClientActivity(activity, true);
+            Assert.Equal(ActivityKind.Client, activity.Kind);
             Assert.Equal(parent.TraceId, activity.Context.TraceId);
             Assert.Equal(parent.SpanId, activity.ParentSpanId);
             Assert.NotEqual(parent.SpanId, activity.Context.SpanId);
@@ -119,8 +128,10 @@ namespace OpenTelemetry.Instrumentation.Http.Tests
             Assert.Equal("k1=v1,k2=v2", tracestates.Single());
         }
 
-        [Fact]
-        public async Task HttpClientInstrumentationInjectsHeadersAsync_CustomFormat()
+        [Theory]
+        [InlineData(true)]
+        [InlineData(false)]
+        public async Task HttpClientInstrumentationInjectsHeadersAsync_CustomFormat(bool shouldEnrich)
         {
             var propagator = new Mock<IPropagator>();
             propagator.Setup(m => m.Inject<HttpRequestMessage>(It.IsAny<PropagationContext>(), It.IsAny<HttpRequestMessage>(), It.IsAny<Action<HttpRequestMessage, string, string>>()))
@@ -145,7 +156,14 @@ namespace OpenTelemetry.Instrumentation.Http.Tests
             parent.ActivityTraceFlags = ActivityTraceFlags.Recorded;
 
             using (Sdk.CreateTracerProviderBuilder()
-                   .AddHttpClientInstrumentation((opt) => opt.Propagator = propagator.Object)
+                   .AddHttpClientInstrumentation((opt) =>
+                   {
+                       opt.Propagator = propagator.Object;
+                       if (shouldEnrich)
+                       {
+                           opt.Enrich = ActivityEnrichment;
+                       }
+                   })
                    .AddProcessor(processor.Object)
                    .Build())
             {
@@ -156,7 +174,7 @@ namespace OpenTelemetry.Instrumentation.Http.Tests
             Assert.Equal(4, processor.Invocations.Count); // OnStart/OnEnd/OnShutdown/Dispose called.
             var activity = (Activity)processor.Invocations[1].Arguments[0];
 
-            ValidateHttpClientActivity(activity, true);
+            Assert.Equal(ActivityKind.Client, activity.Kind);
             Assert.Equal(parent.TraceId, activity.Context.TraceId);
             Assert.Equal(parent.SpanId, activity.ParentSpanId);
             Assert.NotEqual(parent.SpanId, activity.Context.SpanId);
@@ -286,7 +304,7 @@ namespace OpenTelemetry.Instrumentation.Http.Tests
             Baggage.SetBaggage("k2", "v2");
 
             using (Sdk.CreateTracerProviderBuilder()
-                .AddHttpClientInstrumentation()
+                .AddHttpClientInstrumentation(options => options.Enrich = ActivityEnrichment)
                 .AddProcessor(activityProcessor.Object)
                 .Build())
             {
@@ -295,18 +313,6 @@ namespace OpenTelemetry.Instrumentation.Http.Tests
             }
 
             Assert.Equal(4, activityProcessor.Invocations.Count);
-
-            var activity = (Activity)activityProcessor.Invocations[1].Arguments[0];
-
-            HttpRequestMessage thisRequest = (HttpRequestMessage)activity.GetCustomProperty(HttpHandlerDiagnosticListener.RequestCustomPropertyName);
-
-            string[] correlationContext = thisRequest.Headers.GetValues("Correlation-Context").First().Split(',');
-            Assert.Single(correlationContext);
-            Assert.Contains("k1=v1", correlationContext);
-
-            string[] baggage = thisRequest.Headers.GetValues("Baggage").First().Split(',');
-            Assert.Single(baggage);
-            Assert.Contains("k2=v2", baggage);
         }
 
         public void Dispose()
@@ -315,18 +321,24 @@ namespace OpenTelemetry.Instrumentation.Http.Tests
             Activity.Current = null;
         }
 
-        private static void ValidateHttpClientActivity(Activity activityToValidate, bool responseExpected)
+        private static void ActivityEnrichment(Activity activity, string method, object obj)
         {
-            Assert.Equal(ActivityKind.Client, activityToValidate.Kind);
-            var request = activityToValidate.GetCustomProperty(HttpHandlerDiagnosticListener.RequestCustomPropertyName);
-            Assert.NotNull(request);
-            Assert.True(request is HttpRequestMessage);
-
-            if (responseExpected)
+            switch (method)
             {
-                var response = activityToValidate.GetCustomProperty(HttpHandlerDiagnosticListener.ResponseCustomPropertyName);
-                Assert.NotNull(response);
-                Assert.True(response is HttpResponseMessage);
+                case "OnStartActivity":
+                    Assert.True(obj is HttpRequestMessage);
+                    break;
+
+                case "OnStopActivity":
+                    Assert.True(obj is HttpResponseMessage);
+                    break;
+
+                case "OnException":
+                    Assert.True(obj is Exception);
+                    break;
+
+                default:
+                    break;
             }
         }
     }

test/OpenTelemetry.Instrumentation.Http.Tests/HttpClientTests.netcore31.cs

@@ -25,7 +25,6 @@ using System.Reflection;
 using System.Threading.Tasks;
 using Moq;
 using Newtonsoft.Json;
-using OpenTelemetry.Instrumentation.Http.Implementation;
 using OpenTelemetry.Tests;
 using OpenTelemetry.Trace;
 using Xunit;
@@ -34,6 +33,8 @@ namespace OpenTelemetry.Instrumentation.Http.Tests
 {
     public partial class HttpClientTests
     {
+        public static int Counter;
+
         public static IEnumerable<object[]> TestData => HttpTestData.ReadTestCases();
 
         [Theory]
@@ -56,7 +57,11 @@ namespace OpenTelemetry.Instrumentation.Http.Tests
             using (serverLifeTime)
 
             using (Sdk.CreateTracerProviderBuilder()
-                               .AddHttpClientInstrumentation((opt) => opt.SetHttpFlavor = tc.SetHttpFlavor)
+                               .AddHttpClientInstrumentation((opt) =>
+                               {
+                                   opt.SetHttpFlavor = tc.SetHttpFlavor;
+                                   opt.Enrich = ActivityEnrichment;
+                               })
                                .AddProcessor(processor.Object)
                                .SetResource(expectedResource)
                                .Build())
@@ -90,7 +95,7 @@ namespace OpenTelemetry.Instrumentation.Http.Tests
             Assert.Equal(4, processor.Invocations.Count); // OnStart/OnEnd/OnShutdown/Dispose called.
             var activity = (Activity)processor.Invocations[1].Arguments[0];
 
-            ValidateHttpClientActivity(activity, tc.ResponseExpected);
+            Assert.Equal(ActivityKind.Client, activity.Kind);
             Assert.Equal(tc.SpanName, activity.DisplayName);
 
             var d = new Dictionary<string, string>()
@@ -165,6 +170,35 @@ namespace OpenTelemetry.Instrumentation.Http.Tests
             var t = (Task)this.GetType().InvokeMember(nameof(this.HttpOutCallsAreCollectedSuccessfullyAsync), BindingFlags.InvokeMethod, null, this, HttpTestData.GetArgumentsFromTestCaseObject(input).First());
             await t;
         }
+
+        [Fact]
+        public async Task CheckEnrichmentWhenSampling()
+        {
+            await CheckEnrichment(new AlwaysOffSampler(), 0).ConfigureAwait(false);
+            await CheckEnrichment(new AlwaysOnSampler(), 2).ConfigureAwait(false);
+        }
+
+        private static async Task CheckEnrichment(Sampler sampler, int expect)
+        {
+            Counter = 0;
+            var processor = new Mock<ActivityProcessor>();
+            using (Sdk.CreateTracerProviderBuilder()
+                .SetSampler(sampler)
+                .AddHttpClientInstrumentation(options => options.Enrich = ActivityEnrichmentCounter)
+                .AddProcessor(processor.Object)
+                .Build())
+            {
+                using var c = new HttpClient();
+                using var r = await c.GetAsync("https://opentelemetry.io/").ConfigureAwait(false);
+            }
+
+            Assert.Equal(expect, Counter);
+        }
+
+        private static void ActivityEnrichmentCounter(Activity activity, string method, object obj)
+        {
+            Counter++;
+        }
     }
 }
 #endif

test/OpenTelemetry.Instrumentation.SqlClient.Tests/SqlClientTests.cs

@@ -71,6 +71,7 @@ namespace OpenTelemetry.Instrumentation.SqlClient.Tests
         [InlineData(CommandType.Text, "select 1/1", false, true)]
 #endif
         [InlineData(CommandType.Text, "select 1/0", false, false, true)]
+        [InlineData(CommandType.Text, "select 1/0", false, false, true, false)]
         [InlineData(CommandType.StoredProcedure, "sp_who", false)]
         [InlineData(CommandType.StoredProcedure, "sp_who", true)]
         public void SuccessfulCommandTest(
@@ -78,7 +79,8 @@ namespace OpenTelemetry.Instrumentation.SqlClient.Tests
             string commandText,
             bool captureStoredProcedureCommandName,
             bool captureTextCommandContent = false,
-            bool isFailure = false)
+            bool isFailure = false,
+            bool shouldEnrich = true)
         {
             var activityProcessor = new Mock<ActivityProcessor>();
             using var shutdownSignal = Sdk.CreateTracerProviderBuilder()
@@ -87,6 +89,10 @@ namespace OpenTelemetry.Instrumentation.SqlClient.Tests
                 {
                     options.SetStoredProcedureCommandName = captureStoredProcedureCommandName;
                     options.SetTextCommandContent = captureTextCommandContent;
+                    if (shouldEnrich)
+                    {
+                        options.Enrich = ActivityEnrichment;
+                    }
                 })
                 .Build();
 
@@ -120,16 +126,21 @@ namespace OpenTelemetry.Instrumentation.SqlClient.Tests
 
         [Theory]
         [InlineData(SqlClientDiagnosticListener.SqlDataBeforeExecuteCommand, SqlClientDiagnosticListener.SqlDataAfterExecuteCommand, CommandType.StoredProcedure, "SP_GetOrders", true, false)]
+        [InlineData(SqlClientDiagnosticListener.SqlDataBeforeExecuteCommand, SqlClientDiagnosticListener.SqlDataAfterExecuteCommand, CommandType.StoredProcedure, "SP_GetOrders", true, false, false)]
         [InlineData(SqlClientDiagnosticListener.SqlDataBeforeExecuteCommand, SqlClientDiagnosticListener.SqlDataAfterExecuteCommand, CommandType.Text, "select * from sys.databases", true, false)]
+        [InlineData(SqlClientDiagnosticListener.SqlDataBeforeExecuteCommand, SqlClientDiagnosticListener.SqlDataAfterExecuteCommand, CommandType.Text, "select * from sys.databases", true, false, false)]
         [InlineData(SqlClientDiagnosticListener.SqlMicrosoftBeforeExecuteCommand, SqlClientDiagnosticListener.SqlMicrosoftAfterExecuteCommand, CommandType.StoredProcedure, "SP_GetOrders", false, true)]
+        [InlineData(SqlClientDiagnosticListener.SqlMicrosoftBeforeExecuteCommand, SqlClientDiagnosticListener.SqlMicrosoftAfterExecuteCommand, CommandType.StoredProcedure, "SP_GetOrders", false, true, false)]
         [InlineData(SqlClientDiagnosticListener.SqlMicrosoftBeforeExecuteCommand, SqlClientDiagnosticListener.SqlMicrosoftAfterExecuteCommand, CommandType.Text, "select * from sys.databases", false, true)]
+        [InlineData(SqlClientDiagnosticListener.SqlMicrosoftBeforeExecuteCommand, SqlClientDiagnosticListener.SqlMicrosoftAfterExecuteCommand, CommandType.Text, "select * from sys.databases", false, true, false)]
         public void SqlClientCallsAreCollectedSuccessfully(
             string beforeCommand,
             string afterCommand,
             CommandType commandType,
             string commandText,
             bool captureStoredProcedureCommandName,
-            bool captureTextCommandContent)
+            bool captureTextCommandContent,
+            bool shouldEnrich = true)
         {
             using var sqlConnection = new SqlConnection(TestConnectionString);
             using var sqlCommand = sqlConnection.CreateCommand();
@@ -141,6 +152,10 @@ namespace OpenTelemetry.Instrumentation.SqlClient.Tests
                         {
                             opt.SetTextCommandContent = captureTextCommandContent;
                             opt.SetStoredProcedureCommandName = captureStoredProcedureCommandName;
+                            if (shouldEnrich)
+                            {
+                                opt.Enrich = ActivityEnrichment;
+                            }
                         })
                     .AddProcessor(processor.Object)
                     .Build())
@@ -174,20 +189,35 @@ namespace OpenTelemetry.Instrumentation.SqlClient.Tests
 
             Assert.Equal(4, processor.Invocations.Count); // OnStart/OnEnd/OnShutdown/Dispose called.
 
-            VerifyActivityData(sqlCommand.CommandType, sqlCommand.CommandText, captureStoredProcedureCommandName, captureTextCommandContent, false, sqlConnection.DataSource, (Activity)processor.Invocations[1].Arguments[0]);
+            VerifyActivityData(
+                sqlCommand.CommandType,
+                sqlCommand.CommandText,
+                captureStoredProcedureCommandName,
+                captureTextCommandContent,
+                false,
+                sqlConnection.DataSource,
+                (Activity)processor.Invocations[1].Arguments[0]);
         }
 
         [Theory]
         [InlineData(SqlClientDiagnosticListener.SqlDataBeforeExecuteCommand, SqlClientDiagnosticListener.SqlDataWriteCommandError)]
+        [InlineData(SqlClientDiagnosticListener.SqlDataBeforeExecuteCommand, SqlClientDiagnosticListener.SqlDataWriteCommandError, false)]
         [InlineData(SqlClientDiagnosticListener.SqlMicrosoftBeforeExecuteCommand, SqlClientDiagnosticListener.SqlMicrosoftWriteCommandError)]
-        public void SqlClientErrorsAreCollectedSuccessfully(string beforeCommand, string errorCommand)
+        [InlineData(SqlClientDiagnosticListener.SqlMicrosoftBeforeExecuteCommand, SqlClientDiagnosticListener.SqlMicrosoftWriteCommandError, false)]
+        public void SqlClientErrorsAreCollectedSuccessfully(string beforeCommand, string errorCommand, bool shouldEnrich = true)
         {
             using var sqlConnection = new SqlConnection(TestConnectionString);
             using var sqlCommand = sqlConnection.CreateCommand();
 
             var processor = new Mock<ActivityProcessor>();
             using (Sdk.CreateTracerProviderBuilder()
-                .AddSqlClientInstrumentation()
+                .AddSqlClientInstrumentation(options =>
+                {
+                    if (shouldEnrich)
+                    {
+                        options.Enrich = ActivityEnrichment;
+                    }
+                })
                 .AddProcessor(processor.Object)
                 .Build())
             {
@@ -221,7 +251,14 @@ namespace OpenTelemetry.Instrumentation.SqlClient.Tests
 
             Assert.Equal(4, processor.Invocations.Count); // OnStart/OnEnd/OnShutdown/Dispose called.
 
-            VerifyActivityData(sqlCommand.CommandType, sqlCommand.CommandText, true, false, true, sqlConnection.DataSource, (Activity)processor.Invocations[1].Arguments[0]);
+            VerifyActivityData(
+                sqlCommand.CommandType,
+                sqlCommand.CommandText,
+                true,
+                false,
+                true,
+                sqlConnection.DataSource,
+                (Activity)processor.Invocations[1].Arguments[0]);
         }
 
         private static void VerifyActivityData(
@@ -235,9 +272,6 @@ namespace OpenTelemetry.Instrumentation.SqlClient.Tests
         {
             Assert.Equal("master", activity.DisplayName);
             Assert.Equal(ActivityKind.Client, activity.Kind);
-            var command = activity.GetCustomProperty(SqlClientDiagnosticListener.CommandCustomPropertyName);
-            Assert.NotNull(command);
-            Assert.True(command is SqlCommand);
 
             if (!isFailure)
             {
@@ -283,6 +317,19 @@ namespace OpenTelemetry.Instrumentation.SqlClient.Tests
             Assert.Equal(dataSource, activity.GetTagValue(SemanticConventions.AttributePeerService));
         }
 
+        private static void ActivityEnrichment(Activity activity, string method, object obj)
+        {
+            switch (method)
+            {
+                case "OnCustom":
+                    Assert.True(obj is SqlCommand);
+                    break;
+
+                default:
+                    break;
+            }
+        }
+
         private class FakeSqlClientDiagnosticSource : IDisposable
         {
             private readonly DiagnosticListener listener;