Add diagnostic messages and hang dumps for IntegrationTests (#1298)

* Add xunit diagnosticMessages for IntegrationTests

* Enable internalDiagnosticMessages

* Improve integration tests output

* --blame-hang-timeout 3m

.gitignore

@@ -302,5 +302,8 @@ src/OpenTelemetry.AutoInstrumentation.Native/cmake_install.cmake
 # ignore verify .received files
 *.received.*
 
+# profiler and test logs
+/build_data
+
 # install.sh downloaded artifacts
 /otel-dotnet-auto/

build/nuke/Build.Steps.cs

@@ -339,6 +339,7 @@ partial class Build
                     .SetConfiguration(BuildConfiguration)
                     .SetTargetPlatform(Platform)
                     .SetFilter(AndFilter(TestNameFilter(), ContainersFilter()))
+                    .SetBlameHangTimeout("3m")
                     .EnableTrxLogOutput(GetResultsDirectory(project))
                     .SetProjectFile(project)
                     .EnableNoRestore()

test/IntegrationTests/IntegrationTests.csproj

@@ -26,6 +26,10 @@
     <PackageReference Include="xunit.abstractions" Version="2.0.3" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Content Include="xunit.runner.json" CopyToOutputDirectory="PreserveNewest" />
+  </ItemGroup>
+
   <ItemGroup>
     <ProjectReference Include="..\..\src\OpenTelemetry.AutoInstrumentation\OpenTelemetry.AutoInstrumentation.csproj" />
   </ItemGroup>

test/IntegrationTests/VerboseTestFramework.cs

@@ -0,0 +1,138 @@
+// <copyright file="VerboseTestFramework.cs" company="OpenTelemetry Authors">
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// </copyright>
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+using System.Threading;
+using System.Threading.Tasks;
+using Xunit;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+[assembly: TestFramework("IntegrationTests.Helpers.VerboseTestFramework", "IntegrationTests")]
+
+namespace IntegrationTests.Helpers;
+
+public class VerboseTestFramework : XunitTestFramework
+{
+    public VerboseTestFramework(IMessageSink messageSink)
+        : base(messageSink)
+    {
+    }
+
+    protected override ITestFrameworkExecutor CreateExecutor(AssemblyName assemblyName)
+    {
+        return new VerboseTestExecutor(assemblyName, SourceInformationProvider, DiagnosticMessageSink);
+    }
+
+    private class VerboseTestExecutor : XunitTestFrameworkExecutor
+    {
+        public VerboseTestExecutor(AssemblyName assemblyName, ISourceInformationProvider sourceInformationProvider, IMessageSink diagnosticMessageSink)
+            : base(assemblyName, sourceInformationProvider, diagnosticMessageSink)
+        {
+        }
+
+        protected override async void RunTestCases(IEnumerable<IXunitTestCase> testCases, IMessageSink executionMessageSink, ITestFrameworkExecutionOptions executionOptions)
+        {
+            using (var assemblyRunner = new VerboseTestAssemblyRunner(TestAssembly, testCases, DiagnosticMessageSink, executionMessageSink, executionOptions))
+            {
+                await assemblyRunner.RunAsync();
+            }
+        }
+    }
+
+    private class VerboseTestAssemblyRunner : XunitTestAssemblyRunner
+    {
+        public VerboseTestAssemblyRunner(ITestAssembly testAssembly, IEnumerable<IXunitTestCase> testCases, IMessageSink diagnosticMessageSink, IMessageSink executionMessageSink, ITestFrameworkExecutionOptions executionOptions)
+            : base(testAssembly, testCases, diagnosticMessageSink, executionMessageSink, executionOptions)
+        {
+        }
+
+        protected override Task<RunSummary> RunTestCollectionAsync(IMessageBus messageBus, ITestCollection testCollection, IEnumerable<IXunitTestCase> testCases, CancellationTokenSource cancellationTokenSource)
+        {
+            return new VerboseTestCollectionRunner(testCollection, testCases, DiagnosticMessageSink, messageBus, TestCaseOrderer, new ExceptionAggregator(Aggregator), cancellationTokenSource).RunAsync();
+        }
+    }
+
+    private class VerboseTestCollectionRunner : XunitTestCollectionRunner
+    {
+        private readonly IMessageSink _diagnosticMessageSink;
+
+        public VerboseTestCollectionRunner(ITestCollection testCollection, IEnumerable<IXunitTestCase> testCases, IMessageSink diagnosticMessageSink, IMessageBus messageBus, ITestCaseOrderer testCaseOrderer, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource)
+            : base(testCollection, testCases, diagnosticMessageSink, messageBus, testCaseOrderer, aggregator, cancellationTokenSource)
+        {
+            _diagnosticMessageSink = diagnosticMessageSink;
+        }
+
+        protected override Task<RunSummary> RunTestClassAsync(ITestClass testClass, IReflectionTypeInfo @class, IEnumerable<IXunitTestCase> testCases)
+        {
+            return new VerboseTestClassRunner(testClass, @class, testCases, _diagnosticMessageSink, MessageBus, TestCaseOrderer, new ExceptionAggregator(Aggregator), CancellationTokenSource, CollectionFixtureMappings)
+               .RunAsync();
+        }
+    }
+
+    private class VerboseTestClassRunner : XunitTestClassRunner
+    {
+        public VerboseTestClassRunner(ITestClass testClass, IReflectionTypeInfo @class, IEnumerable<IXunitTestCase> testCases, IMessageSink diagnosticMessageSink, IMessageBus messageBus, ITestCaseOrderer testCaseOrderer, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource, IDictionary<Type, object> collectionFixtureMappings)
+            : base(testClass, @class, testCases, diagnosticMessageSink, messageBus, testCaseOrderer, aggregator, cancellationTokenSource, collectionFixtureMappings)
+        {
+        }
+
+        protected override Task<RunSummary> RunTestMethodAsync(ITestMethod testMethod, IReflectionMethodInfo method, IEnumerable<IXunitTestCase> testCases, object[] constructorArguments)
+        {
+            return new VerboseTestMethodRunner(testMethod, this.Class, method, testCases, this.DiagnosticMessageSink, this.MessageBus, new ExceptionAggregator(this.Aggregator), this.CancellationTokenSource, constructorArguments)
+               .RunAsync();
+        }
+    }
+
+    private class VerboseTestMethodRunner : XunitTestMethodRunner
+    {
+        private readonly IMessageSink _diagnosticMessageSink;
+
+        public VerboseTestMethodRunner(ITestMethod testMethod, IReflectionTypeInfo @class, IReflectionMethodInfo method, IEnumerable<IXunitTestCase> testCases, IMessageSink diagnosticMessageSink, IMessageBus messageBus, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource, object[] constructorArguments)
+            : base(testMethod, @class, method, testCases, diagnosticMessageSink, messageBus, aggregator, cancellationTokenSource, constructorArguments)
+        {
+            _diagnosticMessageSink = diagnosticMessageSink;
+        }
+
+        protected override async Task<RunSummary> RunTestCaseAsync(IXunitTestCase testCase)
+        {
+            var parameters = string.Empty;
+            if (testCase.TestMethodArguments != null)
+            {
+                parameters = string.Join(", ", testCase.TestMethodArguments.Select(a => a?.ToString() ?? "null"));
+            }
+
+            var test = $"{TestMethod.TestClass.Class.Name}.{TestMethod.Method.Name}({parameters})";
+            _diagnosticMessageSink.OnMessage(new DiagnosticMessage($"=== RUN   {test}"));
+
+            try
+            {
+                var result = await base.RunTestCaseAsync(testCase);
+                var status = result.Failed > 0 ? "FAIL" : "PASS";
+                _diagnosticMessageSink.OnMessage(new DiagnosticMessage($"--- {status}: {test} ({result.Time:0.00}s)"));
+                return result;
+            }
+            catch (Exception ex)
+            {
+                _diagnosticMessageSink.OnMessage(new DiagnosticMessage($"--- EXPT: {test} ({ex.Message})"));
+                throw;
+            }
+        }
+    }
+}

test/IntegrationTests/xunit.runner.json

@@ -0,0 +1,6 @@
+{
+  "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
+  "diagnosticMessages": true,
+  "longRunningTestSeconds": 60,
+  "parallelizeTestCollections": false
+}