Fix issue with auto-injection of Correlation Identifiers (Serilog) (#472)

Scenario: Some time after the first span is activated, we have set the
logging context properties dd.trace_id=0 and dd.span_id=0. Application
code then sets their own logging context properties. Then, a new span is
created and previously-set properties no longer exist in the logging context.

Problem: Our OnSpanActivated event immediately disposes the previously-set
correlation identifier properties before re-adding the new values.
Serilog has a strict correctness guarantee that requires properties be
modified in stack order. Since we remove properties further down the stack,
we end up losing properties.

Fix: For Serilog, do not add efault values of TraceId=0 and SpanId=0. Also,
add and remove the two properties on SpanOpened and on SpanClosed.
This ensures that properties only get added to/removed from the stack once.

reproductions/DataDogThreadTest/Program.cs

@@ -142,12 +142,11 @@ namespace DataDogThreadTest
                 logger.Info(NonTraceMessage);
 
                 var lastLog = RelevantLogs().Last();
-                var expectedOutOfTraceLog = "TraceId: 0, SpanId: 0";
                 var lastLogTraceId = lastLog.Properties[TraceIdKey];
                 var lastLogSpanIdId = lastLog.Properties[SpanIdKey];
                 var actual = $"TraceId: {lastLogTraceId}, SpanId: {lastLogSpanIdId}";
 
-                if (!actual.Equals(expectedOutOfTraceLog))
+                if (!actual.Equals(NonTraceMessage))
                 {
                     throw new Exception($"Unexpected TraceId or SpanId: {actual}");
                 }

src/Datadog.Trace/AsyncLocalScopeManager.cs

@@ -49,17 +49,18 @@ namespace Datadog.Trace
             if (current == null || current != scope)
             {
                 // This is not the current scope for this context, bail out
+                SpanClosed?.Invoke(this, new SpanEventArgs(scope.Span));
                 return;
             }
 
             // if the scope that was just closed was the active scope,
             // set its parent as the new active scope
-            _activeScope.Set(current.Parent);
-            SpanDeactivated?.Invoke(this, new SpanEventArgs(current.Span));
+            _activeScope.Set(scope.Parent);
+            SpanDeactivated?.Invoke(this, new SpanEventArgs(scope.Span));
 
             if (!isRootSpan)
             {
-                SpanActivated?.Invoke(this, new SpanEventArgs(current.Parent.Span));
+                SpanActivated?.Invoke(this, new SpanEventArgs(scope.Parent.Span));
             }
 
             SpanClosed?.Invoke(this, new SpanEventArgs(scope.Span));

src/Datadog.Trace/Logging/LibLogScopeEventSubscriber.cs

@@ -1,54 +1,113 @@
 using System;
 using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Diagnostics;
+using Datadog.Trace.Logging.LogProviders;
 
 namespace Datadog.Trace.Logging
 {
+    /// <summary>
+    /// Subscriber to ScopeManager events that sets/unsets correlation identifier
+    /// properties in the application's logging context.
+    /// </summary>
     internal class LibLogScopeEventSubscriber : IDisposable
     {
         private readonly IScopeManager _scopeManager;
+        private readonly ILogProvider _logProvider;
 
         // Each mapped context sets a key-value pair into the logging context
-        // Disposing the context unsets the key-value pair
+        // Disposing the returned context unsets the key-value pair
+        // Keep a stack to retain the history of our correlation identifier properties
+        // (the stack is particularly important for Serilog, see below).
         //
-        // IMPORTANT: The contexts must be closed in reverse-order of opening,
-        //            so by convention always open the TraceId context before
-        //            opening the SpanId context, and close the contexts in
-        //            the opposite order
+        // IMPORTANT: Serilog -- The logging contexts (throughout the entire application)
+        //            are maintained in a stack, as opposed to a map, and must be closed
+        //            in reverse-order of opening. When operating on the stack-based model,
+        //            it is only valid to add the properties once unset them once.
         private readonly ConcurrentStack<IDisposable> _contextDisposalStack = new ConcurrentStack<IDisposable>();
 
         public LibLogScopeEventSubscriber(IScopeManager scopeManager)
         {
             _scopeManager = scopeManager;
-            _scopeManager.SpanActivated += OnSpanActivated;
-            _scopeManager.TraceEnded += OnTraceEnded;
-            SetDefaultValues();
+
+            _logProvider = LogProvider.CurrentLogProvider ?? LogProvider.ResolveLogProvider();
+            if (_logProvider is SerilogLogProvider)
+            {
+                // Do not set default values for Serilog because it is unsafe to set
+                // except at the application startup, but this would require auto-instrumentation
+                _scopeManager.SpanOpened += StackOnSpanOpened;
+                _scopeManager.SpanClosed += StackOnSpanClosed;
+            }
+            else
+            {
+                SetDefaultValues();
+                _scopeManager.SpanActivated += MapOnSpanActivated;
+                _scopeManager.TraceEnded += MapOnTraceEnded;
+            }
         }
 
-        public void OnSpanActivated(object sender, SpanEventArgs spanEventArgs)
+        public void StackOnSpanOpened(object sender, SpanEventArgs spanEventArgs)
         {
-            DisposeAll();
-            SetLoggingValues(spanEventArgs.Span.TraceId, spanEventArgs.Span.SpanId);
+            SetCorrelationIdentifierContext(spanEventArgs.Span.TraceId, spanEventArgs.Span.SpanId);
         }
 
-        public void OnTraceEnded(object sender, SpanEventArgs spanEventArgs)
+        public void StackOnSpanClosed(object sender, SpanEventArgs spanEventArgs)
         {
-            DisposeAll();
+            RemoveLastCorrelationIdentifierContext();
+        }
+
+        public void MapOnSpanActivated(object sender, SpanEventArgs spanEventArgs)
+        {
+            RemoveAllCorrelationIdentifierContexts();
+            SetCorrelationIdentifierContext(spanEventArgs.Span.TraceId, spanEventArgs.Span.SpanId);
+        }
+
+        public void MapOnTraceEnded(object sender, SpanEventArgs spanEventArgs)
+        {
+            RemoveAllCorrelationIdentifierContexts();
             SetDefaultValues();
         }
 
         public void Dispose()
         {
-            _scopeManager.SpanActivated -= OnSpanActivated;
-            _scopeManager.TraceEnded -= OnTraceEnded;
-            DisposeAll();
+            if (_logProvider is SerilogLogProvider)
+            {
+                _scopeManager.SpanOpened -= StackOnSpanOpened;
+                _scopeManager.SpanClosed -= StackOnSpanClosed;
+            }
+            else
+            {
+                _scopeManager.SpanActivated -= MapOnSpanActivated;
+                _scopeManager.TraceEnded -= MapOnTraceEnded;
+            }
+
+            RemoveAllCorrelationIdentifierContexts();
         }
 
         private void SetDefaultValues()
         {
-            SetLoggingValues(0, 0);
+            SetCorrelationIdentifierContext(0, 0);
         }
 
-        private void DisposeAll()
+        private void RemoveLastCorrelationIdentifierContext()
+        {
+            for (int i = 0; i < 2; i++)
+            {
+                if (_contextDisposalStack.TryPop(out IDisposable ctxDisposable))
+                {
+                    ctxDisposable.Dispose();
+                }
+                else
+                {
+                    // There is nothing left to pop so do nothing.
+                    // Though we are in a strange circumstance if we did not balance
+                    // the stack properly
+                    Debug.Fail($"{nameof(RemoveLastCorrelationIdentifierContext)} call failed. Too few items on the context stack.");
+                }
+            }
+        }
+
+        private void RemoveAllCorrelationIdentifierContexts()
         {
             while (_contextDisposalStack.TryPop(out IDisposable ctxDisposable))
             {
@@ -56,7 +115,7 @@ namespace Datadog.Trace.Logging
             }
         }
 
-        private void SetLoggingValues(ulong traceId, ulong spanId)
+        private void SetCorrelationIdentifierContext(ulong traceId, ulong spanId)
         {
             _contextDisposalStack.Push(
                 LogProvider.OpenMappedContext(

test/Datadog.Trace.Tests/Logging/Log4NetLogProviderTests.cs

@@ -1,3 +1,4 @@
+using System.Collections.Generic;
 using System.Reflection;
 using Datadog.Trace.Logging;
 using Datadog.Trace.Logging.LogProviders;
@@ -11,8 +12,9 @@ namespace Datadog.Trace.Tests.Logging
     [Collection(nameof(Datadog.Trace.Tests.Logging))]
     public class Log4NetLogProviderTests
     {
+        private readonly ILogProvider _logProvider;
+        private readonly ILog _logger;
         private readonly MemoryAppender _memoryAppender;
-        private ILog _logger;
 
         public Log4NetLogProviderTests()
         {
@@ -20,8 +22,9 @@ namespace Datadog.Trace.Tests.Logging
             var repository = log4net.LogManager.GetRepository(Assembly.GetAssembly(typeof(log4net.LogManager)));
             BasicConfigurator.Configure(repository, _memoryAppender);
 
-            LogProvider.SetCurrentLogProvider(new Log4NetLogProvider());
-            _logger = LogProvider.GetLogger(typeof(Log4NetLogProviderTests));
+            _logProvider = new Log4NetLogProvider();
+            LogProvider.SetCurrentLogProvider(_logProvider);
+            _logger = new LoggerExecutionWrapper(_logProvider.GetLogger("Test"));
         }
 
         [Fact]
@@ -32,39 +35,54 @@ namespace Datadog.Trace.Tests.Logging
 
             // Instantiate a tracer for this test with default settings and set LogsInjectionEnabled to TRUE
             var tracer = LoggingProviderTestHelpers.InitializeTracer(enableLogsInjection: true);
-            LoggingProviderTestHelpers.PerformParentChildScopeSequence(tracer, _logger, out var parentScope, out var childScope);
+            LoggingProviderTestHelpers.PerformParentChildScopeSequence(tracer, _logger, _logProvider.OpenMappedContext, out var parentScope, out var childScope);
+
+            // Filter the logs
+            List<LoggingEvent> filteredLogs = new List<LoggingEvent>(_memoryAppender.GetEvents());
+            filteredLogs.RemoveAll(log => !log.MessageObject.ToString().Contains(LoggingProviderTestHelpers.LogPrefix));
 
             int logIndex = 0;
-            var logEvents = _memoryAppender.GetEvents();
             LoggingEvent logEvent;
 
-            // Verify the log event is decorated with the parent scope properties
-            logEvent = logEvents[logIndex++];
-            Assert.Contains(CorrelationIdentifier.SpanIdKey, logEvent.Properties.GetKeys());
-            Assert.Equal<ulong>(parentScope.Span.SpanId, ulong.Parse(logEvent.Properties[CorrelationIdentifier.SpanIdKey].ToString()));
-            Assert.Contains(CorrelationIdentifier.TraceIdKey, logEvent.Properties.GetKeys());
-            Assert.Equal<ulong>(parentScope.Span.TraceId, ulong.Parse(logEvent.Properties[CorrelationIdentifier.TraceIdKey].ToString()));
+            // Scope: Parent scope
+            // Custom property: N/A
+            logEvent = filteredLogs[logIndex++];
+            logEvent.Contains(parentScope);
+            Assert.DoesNotContain(LoggingProviderTestHelpers.CustomPropertyName, logEvent.Properties.GetKeys());
 
-            // Verify the log event is decorated with the child scope properties
-            logEvent = logEvents[logIndex++];
-            Assert.Contains(CorrelationIdentifier.SpanIdKey, logEvent.Properties.GetKeys());
-            Assert.Equal<ulong>(childScope.Span.SpanId, ulong.Parse(logEvent.Properties[CorrelationIdentifier.SpanIdKey].ToString()));
-            Assert.Contains(CorrelationIdentifier.TraceIdKey, logEvent.Properties.GetKeys());
-            Assert.Equal<ulong>(childScope.Span.TraceId, ulong.Parse(logEvent.Properties[CorrelationIdentifier.TraceIdKey].ToString()));
+            // Scope: Parent scope
+            // Custom property: SET
+            logEvent = filteredLogs[logIndex++];
+            logEvent.Contains(parentScope);
+            Assert.Contains(LoggingProviderTestHelpers.CustomPropertyName, logEvent.Properties.GetKeys());
+            Assert.Equal<int>(LoggingProviderTestHelpers.CustomPropertyValue, int.Parse(logEvent.Properties[LoggingProviderTestHelpers.CustomPropertyName].ToString()));
 
-            // Verify the log event is decorated with the parent scope properties
-            logEvent = logEvents[logIndex++];
-            Assert.Contains(CorrelationIdentifier.SpanIdKey, logEvent.Properties.GetKeys());
-            Assert.Equal<ulong>(parentScope.Span.SpanId, ulong.Parse(logEvent.Properties[CorrelationIdentifier.SpanIdKey].ToString()));
-            Assert.Contains(CorrelationIdentifier.TraceIdKey, logEvent.Properties.GetKeys());
-            Assert.Equal<ulong>(parentScope.Span.TraceId, ulong.Parse(logEvent.Properties[CorrelationIdentifier.TraceIdKey].ToString()));
+            // Scope: Child scope
+            // Custom property: SET
+            logEvent = filteredLogs[logIndex++];
+            logEvent.Contains(childScope);
+            Assert.Contains(LoggingProviderTestHelpers.CustomPropertyName, logEvent.Properties.GetKeys());
+            Assert.Equal<int>(LoggingProviderTestHelpers.CustomPropertyValue, int.Parse(logEvent.Properties[LoggingProviderTestHelpers.CustomPropertyName].ToString()));
 
-            // Verify the log event is decorated with zero values
-            logEvent = logEvents[logIndex++];
-            Assert.Contains(CorrelationIdentifier.SpanIdKey, logEvent.Properties.GetKeys());
-            Assert.Equal<ulong>(0, ulong.Parse(logEvent.Properties[CorrelationIdentifier.SpanIdKey].ToString()));
-            Assert.Contains(CorrelationIdentifier.TraceIdKey, logEvent.Properties.GetKeys());
-            Assert.Equal<ulong>(0, ulong.Parse(logEvent.Properties[CorrelationIdentifier.TraceIdKey].ToString()));
+            // Scope: Parent scope
+            // Custom property: SET
+            logEvent = filteredLogs[logIndex++];
+            logEvent.Contains(parentScope);
+            Assert.Contains(LoggingProviderTestHelpers.CustomPropertyName, logEvent.Properties.GetKeys());
+            Assert.Equal<int>(LoggingProviderTestHelpers.CustomPropertyValue, int.Parse(logEvent.Properties[LoggingProviderTestHelpers.CustomPropertyName].ToString()));
+
+            // EXISTING: Verify the log event is decorated with the parent scope properties
+            // Scope: Parent scope
+            // Custom property: N/A
+            logEvent = filteredLogs[logIndex++];
+            logEvent.Contains(parentScope);
+            Assert.DoesNotContain(LoggingProviderTestHelpers.CustomPropertyName, logEvent.Properties.GetKeys());
+
+            // Scope: Default values of TraceId=0,SpanId=0
+            // Custom property: N/A
+            logEvent = filteredLogs[logIndex++];
+            logEvent.Contains(traceId: 0, spanId: 0);
+            Assert.DoesNotContain(LoggingProviderTestHelpers.CustomPropertyName, logEvent.Properties.GetKeys());
         }
 
         [Fact]
@@ -75,31 +93,59 @@ namespace Datadog.Trace.Tests.Logging
 
             // Instantiate a tracer for this test with default settings and set LogsInjectionEnabled to TRUE
             var tracer = LoggingProviderTestHelpers.InitializeTracer(enableLogsInjection: false);
-            LoggingProviderTestHelpers.PerformParentChildScopeSequence(tracer, _logger, out var parentScope, out var childScope);
+            LoggingProviderTestHelpers.PerformParentChildScopeSequence(tracer, _logger, _logProvider.OpenMappedContext, out var parentScope, out var childScope);
+
+            // Filter the logs
+            List<LoggingEvent> filteredLogs = new List<LoggingEvent>(_memoryAppender.GetEvents());
+            filteredLogs.RemoveAll(log => !log.MessageObject.ToString().Contains(LoggingProviderTestHelpers.LogPrefix));
 
             int logIndex = 0;
-            var logEvents = _memoryAppender.GetEvents();
             LoggingEvent logEvent;
 
-            // Verify the log event is not decorated with the properties
-            logEvent = logEvents[logIndex++];
+            // Scope: N/A
+            // Custom property: N/A
+            logEvent = filteredLogs[logIndex++];
             Assert.DoesNotContain(CorrelationIdentifier.SpanIdKey, logEvent.Properties.GetKeys());
             Assert.DoesNotContain(CorrelationIdentifier.TraceIdKey, logEvent.Properties.GetKeys());
+            Assert.DoesNotContain(LoggingProviderTestHelpers.CustomPropertyName, logEvent.Properties.GetKeys());
 
-            // Verify the log event is not decorated with the properties
-            logEvent = logEvents[logIndex++];
+            // Scope: N/A
+            // Custom property: SET
+            logEvent = filteredLogs[logIndex++];
             Assert.DoesNotContain(CorrelationIdentifier.SpanIdKey, logEvent.Properties.GetKeys());
             Assert.DoesNotContain(CorrelationIdentifier.TraceIdKey, logEvent.Properties.GetKeys());
+            Assert.Contains(LoggingProviderTestHelpers.CustomPropertyName, logEvent.Properties.GetKeys());
+            Assert.Equal<int>(LoggingProviderTestHelpers.CustomPropertyValue, int.Parse(logEvent.Properties[LoggingProviderTestHelpers.CustomPropertyName].ToString()));
 
-            // Verify the log event is not decorated with the properties
-            logEvent = logEvents[logIndex++];
+            // Scope: N/A
+            // Custom property: SET
+            logEvent = filteredLogs[logIndex++];
             Assert.DoesNotContain(CorrelationIdentifier.SpanIdKey, logEvent.Properties.GetKeys());
             Assert.DoesNotContain(CorrelationIdentifier.TraceIdKey, logEvent.Properties.GetKeys());
+            Assert.Contains(LoggingProviderTestHelpers.CustomPropertyName, logEvent.Properties.GetKeys());
+            Assert.Equal<int>(LoggingProviderTestHelpers.CustomPropertyValue, int.Parse(logEvent.Properties[LoggingProviderTestHelpers.CustomPropertyName].ToString()));
 
-            // Verify the log event is not decorated with the properties
-            logEvent = logEvents[logIndex++];
+            // Scope: N/A
+            // Custom property: SET
+            logEvent = filteredLogs[logIndex++];
             Assert.DoesNotContain(CorrelationIdentifier.SpanIdKey, logEvent.Properties.GetKeys());
             Assert.DoesNotContain(CorrelationIdentifier.TraceIdKey, logEvent.Properties.GetKeys());
+            Assert.Contains(LoggingProviderTestHelpers.CustomPropertyName, logEvent.Properties.GetKeys());
+            Assert.Equal<int>(LoggingProviderTestHelpers.CustomPropertyValue, int.Parse(logEvent.Properties[LoggingProviderTestHelpers.CustomPropertyName].ToString()));
+
+            // Scope: N/A
+            // Custom property: N/A
+            logEvent = filteredLogs[logIndex++];
+            Assert.DoesNotContain(CorrelationIdentifier.SpanIdKey, logEvent.Properties.GetKeys());
+            Assert.DoesNotContain(CorrelationIdentifier.TraceIdKey, logEvent.Properties.GetKeys());
+            Assert.DoesNotContain(LoggingProviderTestHelpers.CustomPropertyName, logEvent.Properties.GetKeys());
+
+            // Scope: N/A
+            // Custom property: N/A
+            logEvent = filteredLogs[logIndex++];
+            Assert.DoesNotContain(CorrelationIdentifier.SpanIdKey, logEvent.Properties.GetKeys());
+            Assert.DoesNotContain(CorrelationIdentifier.TraceIdKey, logEvent.Properties.GetKeys());
+            Assert.DoesNotContain(LoggingProviderTestHelpers.CustomPropertyName, logEvent.Properties.GetKeys());
         }
     }
 }

test/Datadog.Trace.Tests/Logging/LoggingProviderTestHelpers.cs

@@ -1,13 +1,19 @@
+using System;
 using Datadog.Trace.Agent;
 using Datadog.Trace.Configuration;
 using Datadog.Trace.Logging;
 using Datadog.Trace.Sampling;
 using Moq;
+using Xunit;
 
 namespace Datadog.Trace.Tests.Logging
 {
-    internal class LoggingProviderTestHelpers
+    internal static class LoggingProviderTestHelpers
     {
+        internal static readonly string CustomPropertyName = "custom";
+        internal static readonly int CustomPropertyValue = 1;
+        internal static readonly string LogPrefix = "[Datadog.Trace.Tests.Logging]";
+
         internal static Tracer InitializeTracer(bool enableLogsInjection)
         {
             var settings = new TracerSettings();
@@ -19,19 +25,51 @@ namespace Datadog.Trace.Tests.Logging
             return new Tracer(settings, writerMock.Object, samplerMock.Object, null);
         }
 
-        internal static void PerformParentChildScopeSequence(Tracer tracer, ILog logger, out Scope parentScope, out Scope childScope)
+        internal static void PerformParentChildScopeSequence(Tracer tracer, ILog logger, Func<string, object, bool, IDisposable> openMappedContext, out Scope parentScope, out Scope childScope)
         {
             parentScope = tracer.StartActive("parent");
-            logger.Log(LogLevel.Info, () => "Started and activated parent scope.");
+            logger.Log(LogLevel.Info, () => $"{LogPrefix}Started and activated parent scope.");
+
+            var customPropertyContext = openMappedContext(CustomPropertyName, CustomPropertyValue, false);
+            logger.Log(LogLevel.Info, () => $"{LogPrefix}Added custom property to MDC");
 
             childScope = tracer.StartActive("child");
-            logger.Log(LogLevel.Info, () => "Started and activated child scope.");
+            logger.Log(LogLevel.Info, () => $"{LogPrefix}Started and activated child scope.");
 
             childScope.Close();
-            logger.Log(LogLevel.Info, () => "Closed child scope and reactivated parent scope.");
+            logger.Log(LogLevel.Info, () => $"{LogPrefix}Closed child scope and reactivated parent scope.");
+
+            customPropertyContext.Dispose();
+            logger.Log(LogLevel.Info, () => $"{LogPrefix}Removed custom property from MDC");
 
             parentScope.Close();
-            logger.Log(LogLevel.Info, () => "Closed child scope so there is no active scope.");
+            logger.Log(LogLevel.Info, () => $"{LogPrefix}Closed child scope so there is no active scope.");
+        }
+
+        internal static void Contains(this log4net.Core.LoggingEvent logEvent, Scope scope)
+        {
+            logEvent.Contains(scope.Span.TraceId, scope.Span.SpanId);
+        }
+
+        internal static void Contains(this log4net.Core.LoggingEvent logEvent, ulong traceId, ulong spanId)
+        {
+            Assert.Contains(CorrelationIdentifier.TraceIdKey, logEvent.Properties.GetKeys());
+            Assert.Equal<ulong>(traceId, ulong.Parse(logEvent.Properties[CorrelationIdentifier.TraceIdKey].ToString()));
+            Assert.Contains(CorrelationIdentifier.SpanIdKey, logEvent.Properties.GetKeys());
+            Assert.Equal<ulong>(spanId, ulong.Parse(logEvent.Properties[CorrelationIdentifier.SpanIdKey].ToString()));
+        }
+
+        internal static void Contains(this Serilog.Events.LogEvent logEvent, Scope scope)
+        {
+            logEvent.Contains(scope.Span.TraceId, scope.Span.SpanId);
+        }
+
+        internal static void Contains(this Serilog.Events.LogEvent logEvent, ulong traceId, ulong spanId)
+        {
+            Assert.True(logEvent.Properties.ContainsKey(CorrelationIdentifier.TraceIdKey));
+            Assert.Equal<ulong>(traceId, ulong.Parse(logEvent.Properties[CorrelationIdentifier.TraceIdKey].ToString()));
+            Assert.True(logEvent.Properties.ContainsKey(CorrelationIdentifier.SpanIdKey));
+            Assert.Equal<ulong>(spanId, ulong.Parse(logEvent.Properties[CorrelationIdentifier.SpanIdKey].ToString()));
         }
     }
 }

test/Datadog.Trace.Tests/Logging/NLogLogProviderTests.cs

@@ -1,3 +1,4 @@
+using System.Collections.Generic;
 using Datadog.Trace.Logging;
 using Datadog.Trace.Logging.LogProviders;
 using NLog;
@@ -10,6 +11,7 @@ namespace Datadog.Trace.Tests.Logging
     [Collection(nameof(Datadog.Trace.Tests.Logging))]
     public class NLogLogProviderTests
     {
+        private readonly ILogProvider _logProvider;
         private readonly ILog _logger;
         private readonly MemoryTarget _target;
 
@@ -18,7 +20,7 @@ namespace Datadog.Trace.Tests.Logging
             var config = new LoggingConfiguration();
             _target = new MemoryTarget
             {
-                Layout = string.Format("${{level:uppercase=true}}|{0}=${{mdc:item={0}}}|{1}=${{mdc:item={1}}}|${{message}}", CorrelationIdentifier.SpanIdKey, CorrelationIdentifier.TraceIdKey)
+                Layout = string.Format("${{level:uppercase=true}}|{0}=${{mdc:item={0}}}|{1}=${{mdc:item={1}}}|{2}=${{mdc:item={2}}}|${{message}}", CorrelationIdentifier.SpanIdKey, CorrelationIdentifier.TraceIdKey, LoggingProviderTestHelpers.CustomPropertyName)
             };
 
             config.AddTarget("memory", _target);
@@ -26,8 +28,9 @@ namespace Datadog.Trace.Tests.Logging
             LogManager.Configuration = config;
             SimpleConfigurator.ConfigureForTargetLogging(_target, NLog.LogLevel.Trace);
 
-            LogProvider.SetCurrentLogProvider(new NLogLogProvider());
-            _logger = LogProvider.GetLogger(typeof(NLogLogProviderTests));
+            _logProvider = new NLogLogProvider();
+            LogProvider.SetCurrentLogProvider(_logProvider);
+            _logger = new LoggerExecutionWrapper(_logProvider.GetLogger("test"));
         }
 
         [Fact]
@@ -38,30 +41,56 @@ namespace Datadog.Trace.Tests.Logging
 
             // Instantiate a tracer for this test with default settings and set LogsInjectionEnabled to TRUE
             var tracer = LoggingProviderTestHelpers.InitializeTracer(enableLogsInjection: true);
-            LoggingProviderTestHelpers.PerformParentChildScopeSequence(tracer, _logger, out var parentScope, out var childScope);
+            LoggingProviderTestHelpers.PerformParentChildScopeSequence(tracer, _logger, _logProvider.OpenMappedContext, out var parentScope, out var childScope);
+
+            // Filter the logs
+            List<string> filteredLogs = new List<string>(_target.Logs);
+            filteredLogs.RemoveAll(log => !log.Contains(LoggingProviderTestHelpers.LogPrefix));
 
             int logIndex = 0;
             string logString;
 
-            // Verify the log event is decorated with the parent scope properties
-            logString = _target.Logs[logIndex++];
+            // Scope: Parent scope
+            // Custom property: N/A
+            logString = filteredLogs[logIndex++];
             Assert.Contains($"{CorrelationIdentifier.SpanIdKey}={parentScope.Span.SpanId}", logString);
             Assert.Contains($"{CorrelationIdentifier.TraceIdKey}={parentScope.Span.TraceId}", logString);
+            Assert.Contains($"{LoggingProviderTestHelpers.CustomPropertyName}=", logString);
 
-            // Verify the log event is decorated with the child scope properties
-            logString = _target.Logs[logIndex++];
+            // Scope: Parent scope
+            // Custom property: SET
+            logString = filteredLogs[logIndex++];
+            Assert.Contains($"{CorrelationIdentifier.SpanIdKey}={parentScope.Span.SpanId}", logString);
+            Assert.Contains($"{CorrelationIdentifier.TraceIdKey}={parentScope.Span.TraceId}", logString);
+            Assert.Contains($"{LoggingProviderTestHelpers.CustomPropertyName}={LoggingProviderTestHelpers.CustomPropertyValue}", logString);
+
+            // Scope: Child scope
+            // Custom property: SET
+            logString = filteredLogs[logIndex++];
             Assert.Contains($"{CorrelationIdentifier.SpanIdKey}={childScope.Span.SpanId}", logString);
             Assert.Contains($"{CorrelationIdentifier.TraceIdKey}={childScope.Span.TraceId}", logString);
+            Assert.Contains($"{LoggingProviderTestHelpers.CustomPropertyName}={LoggingProviderTestHelpers.CustomPropertyValue}", logString);
 
-            // Verify the log event is decorated with the parent scope properties
-            logString = _target.Logs[logIndex++];
+            // Scope: Parent scope
+            // Custom property: SET
+            logString = filteredLogs[logIndex++];
             Assert.Contains($"{CorrelationIdentifier.SpanIdKey}={parentScope.Span.SpanId}", logString);
             Assert.Contains($"{CorrelationIdentifier.TraceIdKey}={parentScope.Span.TraceId}", logString);
+            Assert.Contains($"{LoggingProviderTestHelpers.CustomPropertyName}={LoggingProviderTestHelpers.CustomPropertyValue}", logString);
 
-            // Verify the log event is decorated with zero values
-            logString = _target.Logs[logIndex++];
+            // Scope: Parent scope
+            // Custom property: N/A
+            logString = filteredLogs[logIndex++];
+            Assert.Contains($"{CorrelationIdentifier.SpanIdKey}={parentScope.Span.SpanId}", logString);
+            Assert.Contains($"{CorrelationIdentifier.TraceIdKey}={parentScope.Span.TraceId}", logString);
+            Assert.Contains($"{LoggingProviderTestHelpers.CustomPropertyName}=", logString);
+
+            // Scope: Default values of TraceId=0,SpanId=0
+            // Custom property: N/A
+            logString = filteredLogs[logIndex++];
             Assert.Contains($"{CorrelationIdentifier.SpanIdKey}=0", logString);
             Assert.Contains($"{CorrelationIdentifier.TraceIdKey}=0", logString);
+            Assert.Contains($"{LoggingProviderTestHelpers.CustomPropertyName}=", logString);
         }
 
         [Fact]
@@ -72,30 +101,56 @@ namespace Datadog.Trace.Tests.Logging
 
             // Instantiate a tracer for this test with default settings and set LogsInjectionEnabled to TRUE
             var tracer = LoggingProviderTestHelpers.InitializeTracer(enableLogsInjection: false);
-            LoggingProviderTestHelpers.PerformParentChildScopeSequence(tracer, _logger, out var parentScope, out var childScope);
+            LoggingProviderTestHelpers.PerformParentChildScopeSequence(tracer, _logger, _logProvider.OpenMappedContext, out var parentScope, out var childScope);
+
+            // Filter the logs
+            List<string> filteredLogs = new List<string>(_target.Logs);
+            filteredLogs.RemoveAll(log => !log.Contains(LoggingProviderTestHelpers.LogPrefix));
 
             int logIndex = 0;
             string logString;
 
-            // Verify the log event is not decorated with the properties
-            logString = _target.Logs[logIndex++];
+            // Scope: N/A
+            // Custom property: N/A
+            logString = filteredLogs[logIndex++];
             Assert.Contains($"{CorrelationIdentifier.SpanIdKey}=", logString);
             Assert.Contains($"{CorrelationIdentifier.TraceIdKey}=", logString);
+            Assert.Contains($"{LoggingProviderTestHelpers.CustomPropertyName}=", logString);
 
-            // Verify the log event is not decorated with the properties
-            logString = _target.Logs[logIndex++];
+            // Scope: N/A
+            // Custom property: SET
+            logString = filteredLogs[logIndex++];
             Assert.Contains($"{CorrelationIdentifier.SpanIdKey}=", logString);
             Assert.Contains($"{CorrelationIdentifier.TraceIdKey}=", logString);
+            Assert.Contains($"{LoggingProviderTestHelpers.CustomPropertyName}={LoggingProviderTestHelpers.CustomPropertyValue}", logString);
 
-            // Verify the log event is not decorated with the properties
-            logString = _target.Logs[logIndex++];
+            // Scope: N/A
+            // Custom property: SET
+            logString = filteredLogs[logIndex++];
             Assert.Contains($"{CorrelationIdentifier.SpanIdKey}=", logString);
             Assert.Contains($"{CorrelationIdentifier.TraceIdKey}=", logString);
+            Assert.Contains($"{LoggingProviderTestHelpers.CustomPropertyName}={LoggingProviderTestHelpers.CustomPropertyValue}", logString);
 
-            // Verify the log event is not decorated with the properties
-            logString = _target.Logs[logIndex++];
+            // Scope: N/A
+            // Custom property: SET
+            logString = filteredLogs[logIndex++];
             Assert.Contains($"{CorrelationIdentifier.SpanIdKey}=", logString);
             Assert.Contains($"{CorrelationIdentifier.TraceIdKey}=", logString);
+            Assert.Contains($"{LoggingProviderTestHelpers.CustomPropertyName}={LoggingProviderTestHelpers.CustomPropertyValue}", logString);
+
+            // Scope: N/A
+            // Custom property: N/A
+            logString = filteredLogs[logIndex++];
+            Assert.Contains($"{CorrelationIdentifier.SpanIdKey}=", logString);
+            Assert.Contains($"{CorrelationIdentifier.TraceIdKey}=", logString);
+            Assert.Contains($"{LoggingProviderTestHelpers.CustomPropertyName}=", logString);
+
+            // Scope: N/A
+            // Custom property: N/A
+            logString = filteredLogs[logIndex++];
+            Assert.Contains($"{CorrelationIdentifier.SpanIdKey}=", logString);
+            Assert.Contains($"{CorrelationIdentifier.TraceIdKey}=", logString);
+            Assert.Contains($"{LoggingProviderTestHelpers.CustomPropertyName}=", logString);
         }
     }
 }

test/Datadog.Trace.Tests/Logging/SerilogLogProviderTests.cs

@@ -11,6 +11,7 @@ namespace Datadog.Trace.Tests.Logging
     [Collection(nameof(Datadog.Trace.Tests.Logging))]
     public class SerilogLogProviderTests
     {
+        private readonly ILogProvider _logProvider;
         private readonly ILog _logger;
         private readonly List<LogEvent> _logEvents;
 
@@ -20,10 +21,11 @@ namespace Datadog.Trace.Tests.Logging
                 .Enrich.FromLogContext()
                 .WriteTo.Observers(obs => obs.Subscribe(logEvent => _logEvents.Add(logEvent)))
                 .CreateLogger();
-
-            LogProvider.SetCurrentLogProvider(new SerilogLogProvider());
-            _logger = LogProvider.GetLogger(typeof(SerilogLogProviderTests));
             _logEvents = new List<LogEvent>();
+
+            _logProvider = new SerilogLogProvider();
+            LogProvider.SetCurrentLogProvider(_logProvider);
+            _logger = new LoggerExecutionWrapper(_logProvider.GetLogger("Test"));
         }
 
         [Fact]
@@ -34,38 +36,53 @@ namespace Datadog.Trace.Tests.Logging
 
             // Instantiate a tracer for this test with default settings and set LogsInjectionEnabled to TRUE
             var tracer = LoggingProviderTestHelpers.InitializeTracer(enableLogsInjection: true);
-            LoggingProviderTestHelpers.PerformParentChildScopeSequence(tracer, _logger, out var parentScope, out var childScope);
+            LoggingProviderTestHelpers.PerformParentChildScopeSequence(tracer, _logger, _logProvider.OpenMappedContext, out var parentScope, out var childScope);
+
+            // Filter the logs
+            _logEvents.RemoveAll(log => !log.MessageTemplate.ToString().Contains(LoggingProviderTestHelpers.LogPrefix));
 
             var logIndex = 0;
             LogEvent logEvent;
 
-            // Verify the log event is decorated with the parent scope properties
+            // Scope: Parent scope
+            // Custom property: N/A
             logEvent = _logEvents[logIndex++];
-            Assert.True(logEvent.Properties.ContainsKey(CorrelationIdentifier.SpanIdKey));
-            Assert.Equal<ulong>(parentScope.Span.SpanId, ulong.Parse(logEvent.Properties[CorrelationIdentifier.SpanIdKey].ToString()));
-            Assert.True(logEvent.Properties.ContainsKey(CorrelationIdentifier.TraceIdKey));
-            Assert.Equal<ulong>(parentScope.Span.TraceId, ulong.Parse(logEvent.Properties[CorrelationIdentifier.TraceIdKey].ToString()));
+            logEvent.Contains(parentScope);
+            Assert.False(logEvent.Properties.ContainsKey(LoggingProviderTestHelpers.CustomPropertyName));
 
-            // Verify the log event is decorated with the child scope properties
+            // Scope: Parent scope
+            // Custom property: SET
             logEvent = _logEvents[logIndex++];
-            Assert.True(logEvent.Properties.ContainsKey(CorrelationIdentifier.SpanIdKey));
-            Assert.Equal<ulong>(childScope.Span.SpanId, ulong.Parse(logEvent.Properties[CorrelationIdentifier.SpanIdKey].ToString()));
-            Assert.True(logEvent.Properties.ContainsKey(CorrelationIdentifier.TraceIdKey));
-            Assert.Equal<ulong>(childScope.Span.TraceId, ulong.Parse(logEvent.Properties[CorrelationIdentifier.TraceIdKey].ToString()));
+            logEvent.Contains(parentScope);
+            Assert.True(logEvent.Properties.ContainsKey(LoggingProviderTestHelpers.CustomPropertyName));
+            Assert.Equal<int>(LoggingProviderTestHelpers.CustomPropertyValue, int.Parse(logEvent.Properties[LoggingProviderTestHelpers.CustomPropertyName].ToString()));
 
-            // Verify the log event is decorated with the parent scope properties
+            // Scope: Child scope
+            // Custom property: SET
             logEvent = _logEvents[logIndex++];
-            Assert.True(logEvent.Properties.ContainsKey(CorrelationIdentifier.SpanIdKey));
-            Assert.Equal<ulong>(parentScope.Span.SpanId, ulong.Parse(logEvent.Properties[CorrelationIdentifier.SpanIdKey].ToString()));
-            Assert.True(logEvent.Properties.ContainsKey(CorrelationIdentifier.TraceIdKey));
-            Assert.Equal<ulong>(parentScope.Span.TraceId, ulong.Parse(logEvent.Properties[CorrelationIdentifier.TraceIdKey].ToString()));
+            logEvent.Contains(childScope);
+            Assert.True(logEvent.Properties.ContainsKey(LoggingProviderTestHelpers.CustomPropertyName));
+            Assert.Equal<int>(LoggingProviderTestHelpers.CustomPropertyValue, int.Parse(logEvent.Properties[LoggingProviderTestHelpers.CustomPropertyName].ToString()));
 
-            // Verify the log event is decorated with zero values
+            // Scope: Parent scope
+            // Custom property: SET
             logEvent = _logEvents[logIndex++];
-            Assert.True(logEvent.Properties.ContainsKey(CorrelationIdentifier.SpanIdKey));
-            Assert.Equal<ulong>(0, ulong.Parse(logEvent.Properties[CorrelationIdentifier.SpanIdKey].ToString()));
-            Assert.True(logEvent.Properties.ContainsKey(CorrelationIdentifier.TraceIdKey));
-            Assert.Equal<ulong>(0, ulong.Parse(logEvent.Properties[CorrelationIdentifier.TraceIdKey].ToString()));
+            logEvent.Contains(parentScope);
+            Assert.True(logEvent.Properties.ContainsKey(LoggingProviderTestHelpers.CustomPropertyName));
+            Assert.Equal<int>(LoggingProviderTestHelpers.CustomPropertyValue, int.Parse(logEvent.Properties[LoggingProviderTestHelpers.CustomPropertyName].ToString()));
+
+            // Scope: Parent scope
+            // Custom property: N/A
+            logEvent = _logEvents[logIndex++];
+            logEvent.Contains(parentScope);
+            Assert.False(logEvent.Properties.ContainsKey(LoggingProviderTestHelpers.CustomPropertyName));
+
+            // Scope: N/A
+            // Custom property: N/A
+            logEvent = _logEvents[logIndex++];
+            Assert.False(logEvent.Properties.ContainsKey(CorrelationIdentifier.SpanIdKey));
+            Assert.False(logEvent.Properties.ContainsKey(CorrelationIdentifier.TraceIdKey));
+            Assert.False(logEvent.Properties.ContainsKey(LoggingProviderTestHelpers.CustomPropertyName));
         }
 
         [Fact]
@@ -76,30 +93,58 @@ namespace Datadog.Trace.Tests.Logging
 
             // Instantiate a tracer for this test with default settings and set LogsInjectionEnabled to TRUE
             var tracer = LoggingProviderTestHelpers.InitializeTracer(enableLogsInjection: false);
-            LoggingProviderTestHelpers.PerformParentChildScopeSequence(tracer, _logger, out var parentScope, out var childScope);
+            LoggingProviderTestHelpers.PerformParentChildScopeSequence(tracer, _logger, _logProvider.OpenMappedContext, out var parentScope, out var childScope);
+
+            // Filter the logs
+            _logEvents.RemoveAll(log => !log.MessageTemplate.ToString().Contains(LoggingProviderTestHelpers.LogPrefix));
 
             int logIndex = 0;
             LogEvent logEvent;
 
-            // Verify the log event is not decorated with the properties
+            // Scope: N/A
+            // Custom property: N/A
             logEvent = _logEvents[logIndex++];
             Assert.False(logEvent.Properties.ContainsKey(CorrelationIdentifier.SpanIdKey));
             Assert.False(logEvent.Properties.ContainsKey(CorrelationIdentifier.TraceIdKey));
+            Assert.False(logEvent.Properties.ContainsKey(LoggingProviderTestHelpers.CustomPropertyName));
 
-            // Verify the log event is not decorated with the properties
+            // Scope: N/A
+            // Custom property: SET
             logEvent = _logEvents[logIndex++];
             Assert.False(logEvent.Properties.ContainsKey(CorrelationIdentifier.SpanIdKey));
             Assert.False(logEvent.Properties.ContainsKey(CorrelationIdentifier.TraceIdKey));
+            Assert.True(logEvent.Properties.ContainsKey(LoggingProviderTestHelpers.CustomPropertyName));
+            Assert.Equal<int>(LoggingProviderTestHelpers.CustomPropertyValue, int.Parse(logEvent.Properties[LoggingProviderTestHelpers.CustomPropertyName].ToString()));
 
-            // Verify the log event is not decorated with the properties
+            // Scope: N/A
+            // Custom property: SET
             logEvent = _logEvents[logIndex++];
             Assert.False(logEvent.Properties.ContainsKey(CorrelationIdentifier.SpanIdKey));
             Assert.False(logEvent.Properties.ContainsKey(CorrelationIdentifier.TraceIdKey));
+            Assert.True(logEvent.Properties.ContainsKey(LoggingProviderTestHelpers.CustomPropertyName));
+            Assert.Equal<int>(LoggingProviderTestHelpers.CustomPropertyValue, int.Parse(logEvent.Properties[LoggingProviderTestHelpers.CustomPropertyName].ToString()));
 
-            // Verify the log event is not decorated with the properties
+            // Scope: N/A
+            // Custom property: SET
             logEvent = _logEvents[logIndex++];
             Assert.False(logEvent.Properties.ContainsKey(CorrelationIdentifier.SpanIdKey));
             Assert.False(logEvent.Properties.ContainsKey(CorrelationIdentifier.TraceIdKey));
+            Assert.True(logEvent.Properties.ContainsKey(LoggingProviderTestHelpers.CustomPropertyName));
+            Assert.Equal<int>(LoggingProviderTestHelpers.CustomPropertyValue, int.Parse(logEvent.Properties[LoggingProviderTestHelpers.CustomPropertyName].ToString()));
+
+            // Scope: N/A
+            // Custom property: N/A
+            logEvent = _logEvents[logIndex++];
+            Assert.False(logEvent.Properties.ContainsKey(CorrelationIdentifier.SpanIdKey));
+            Assert.False(logEvent.Properties.ContainsKey(CorrelationIdentifier.TraceIdKey));
+            Assert.False(logEvent.Properties.ContainsKey(LoggingProviderTestHelpers.CustomPropertyName));
+
+            // Scope: N/A
+            // Custom property: N/A
+            logEvent = _logEvents[logIndex++];
+            Assert.False(logEvent.Properties.ContainsKey(CorrelationIdentifier.SpanIdKey));
+            Assert.False(logEvent.Properties.ContainsKey(CorrelationIdentifier.TraceIdKey));
+            Assert.False(logEvent.Properties.ContainsKey(LoggingProviderTestHelpers.CustomPropertyName));
         }
     }
 }