Improve Integration Test by Generating TLS/mTLS Certificates via MSBuild (#5818)

Co-authored-by: Piotr Kiełkowicz <pkiekowicz@splunk.com>
Co-authored-by: Alan West <3676547+alanwest@users.noreply.github.com>

test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/IntegrationTest/.gitignore

@@ -1,3 +1,9 @@
 # Self-signed cert generated by integration test
 otel-collector.crt
 otel-collector.key
+otel-client.crt
+otel-client.key
+otel-untrusted-collector.crt
+otel-untrusted-collector.key
+certs/*
+certs

test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/IntegrationTest/Dockerfile

@@ -2,12 +2,12 @@
 # This should be run from the root of the repo:
 # docker build --file test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/IntegrationTest/Dockerfile
 
-ARG BUILD_SDK_VERSION=8.0
-ARG TEST_SDK_VERSION=8.0
+ARG BUILD_SDK_VERSION=9.0
+ARG TEST_SDK_VERSION=9.0
 
 FROM mcr.microsoft.com/dotnet/sdk:${BUILD_SDK_VERSION} AS build
 ARG PUBLISH_CONFIGURATION=Release
-ARG PUBLISH_FRAMEWORK=net8.0
+ARG PUBLISH_FRAMEWORK=net9.0
 WORKDIR /repo
 COPY . ./
 WORKDIR "/repo/test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests"

test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/IntegrationTest/create-cert.sh

@@ -1,21 +0,0 @@
-#!/bin/bash
-set -e
-
-# Generate self-signed certificate for the collector
-openssl req -new -newkey rsa:2048 -days 365 -nodes -x509 \
-    -subj "/CN=otel-collector" \
-    -keyout /otel-collector.key  -out /otel-collector.crt
-
-# Copy the certificate and private key file to shared volume that the collector
-# container and test container can access
-cp /otel-collector.crt /otel-collector.key /cfg
-
-chmod 644 /cfg/otel-collector.key
-
-# The integration test is run via docker-compose with the --exit-code-from
-# option. The --exit-code-from option implies --abort-on-container-exit
-# which means when any container exits then all containers are stopped.
-# Since the container running this script would be otherwise short-lived
-# we sleep here. If the test does not finish within this time then the test
-# container will be stopped and have a non-zero exit code.
-sleep 300

test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/IntegrationTest/docker-compose.yml

@@ -5,11 +5,20 @@
 version: '3.7'
 
 services:
-  create-cert:
-    image: mcr.microsoft.com/dotnet/sdk:7.0
+  init-service:
+    image: otel-test-image
+    build:
+      context: .
+      dockerfile: ./test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/IntegrationTest/Dockerfile
     volumes:
       - ./test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/IntegrationTest:/cfg
-    command: /cfg/create-cert.sh
+    command: >
+      sh -c "
+      mkdir -p /cfg/certs;
+      cp /test/*.pem /cfg/certs/;
+      chmod 644 /cfg/certs/*;
+      sleep 1000;
+      "
 
   otel-collector:
     image: otel/opentelemetry-collector
@@ -17,9 +26,10 @@ services:
       - ./test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/IntegrationTest:/cfg
     command: --config=/cfg/otel-collector-config.yaml
     depends_on:
-      - create-cert
+      - init-service
 
   tests:
+    image: otel-test-image
     build:
       context: .
       dockerfile: ./test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/IntegrationTest/Dockerfile
@@ -27,7 +37,7 @@ services:
       - ./test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/IntegrationTest:/cfg
     command: /cfg/run-test.sh
     environment:
-      - OTEL_COLLECTOR_HOSTNAME=otel-collector
-      - OTEL_MOCK_COLLECTOR_HOSTNAME=mock-otel-collector
+      OTEL_COLLECTOR_HOSTNAME: otel-collector
+      OTEL_MOCK_COLLECTOR_HOSTNAME: mock-otel-collector
     depends_on:
       - otel-collector

test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/IntegrationTest/otel-collector-config.yaml

@@ -16,13 +16,39 @@ receivers:
       grpc:
         endpoint: 0.0.0.0:5317
         tls:
-          cert_file: /cfg/otel-collector.crt
-          key_file: /cfg/otel-collector.key
+          cert_file: /cfg/certs/otel-test-server-cert.pem
+          key_file: /cfg/certs/otel-test-server-key.pem
       http:
         endpoint: 0.0.0.0:5318
         tls:
-          cert_file: /cfg/otel-collector.crt
-          key_file: /cfg/otel-collector.key
+          cert_file: /cfg/certs/otel-test-server-cert.pem
+          key_file: /cfg/certs/otel-test-server-key.pem
+  otlp/untrustedtls:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:6317
+        tls:
+          cert_file: /cfg/certs/otel-untrusted-collector-cert.pem
+          key_file: /cfg/certs/otel-untrusted-collector-key.pem
+      http:
+        endpoint: 0.0.0.0:6318
+        tls:
+          cert_file: /cfg/certs/otel-untrusted-collector-cert.pem
+          key_file: /cfg/certs/otel-untrusted-collector-key.pem
+  otlp/mtls:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:7317
+        tls:
+          cert_file: /cfg/certs/otel-test-server-cert.pem
+          key_file: /cfg/certs/otel-test-server-key.pem
+          client_ca_file: /cfg/certs/otel-test-ca-cert.pem
+      http:
+        endpoint: 0.0.0.0:7318
+        tls:
+          cert_file: /cfg/certs/otel-test-server-cert.pem
+          key_file: /cfg/certs/otel-test-server-key.pem
+          client_ca_file: /cfg/certs/otel-test-ca-cert.pem
 
 exporters:
   debug:
@@ -31,11 +57,11 @@ exporters:
 service:
   pipelines:
     traces:
-      receivers: [otlp, otlp/tls]
+      receivers: [otlp, otlp/tls, otlp/untrustedtls, otlp/mtls]
       exporters: [debug]
     metrics:
-      receivers: [otlp, otlp/tls]
+      receivers: [otlp, otlp/tls, otlp/untrustedtls, otlp/mtls]
       exporters: [debug]
     logs:
-      receivers: [otlp, otlp/tls]
+      receivers: [otlp, otlp/tls, otlp/untrustedtls, otlp/mtls]
       exporters: [debug]

test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/IntegrationTest/run-test.sh

@@ -1,8 +1,8 @@
 #!/bin/bash
 set -e
 
-# Trust the self-signed certificated used by the collector
-cp /cfg/otel-collector.crt /usr/local/share/ca-certificates/
+# Trust the self-signed certificate used by the collector
+cp /cfg/certs/otel-test-ca-cert.pem /usr/local/share/ca-certificates/otel-test-ca-cert.crt
 update-ca-certificates --verbose
 
 dotnet test OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests.dll --TestCaseFilter:CategoryName=CollectorIntegrationTests --logger "console;verbosity=detailed"

test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests.csproj

@@ -4,6 +4,25 @@
     <TargetFrameworks>$(TargetFrameworksForTests)</TargetFrameworks>
   </PropertyGroup>
 
+  <!-- Add MSBuild Task to Generate Certificates -->
+  <Target Name="GenerateTestCertificates" BeforeTargets="Publish">
+    <Exec Condition="$(OS) == 'Unix'"
+      Command="/bin/bash gen_test_cert.sh $(IntermediateOutputPath)"
+      ConsoleToMsBuild="true"
+      WorkingDirectory="$(ProjectDir)" />
+
+    <Exec Condition="$(OS) == 'Windows_NT'"
+      Command="pwsh -NonInteractive -executionpolicy Unrestricted -command &quot;&amp; { ./gen_test_cert.ps1 -OutDir $(IntermediateOutputPath) } &quot;"
+      ConsoleToMsBuild="true"
+      WorkingDirectory="$(ProjectDir)" />
+
+    <ItemGroup>
+      <TestCertificates Include="$(IntermediateOutputPath)*.pem"/>
+    </ItemGroup>
+
+    <Copy SourceFiles="@(TestCertificates)" DestinationFolder="$(PublishDir)/%(RecursiveDir)"/>
+  </Target>
+
   <ItemGroup>
     <PackageReference Include="Grpc.AspNetCore.Server" Condition="'$(TargetFrameworkIdentifier)' == '.NETCoreApp'" />
     <PackageReference Include="Microsoft.AspNetCore.TestHost" Condition="'$(TargetFrameworkIdentifier)' == '.NETCoreApp'" />

test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/gen_test_cert.ps1

@@ -0,0 +1,90 @@
+using namespace System.Security.Cryptography;
+using namespace System.Security.Cryptography.X509Certificates;
+
+param (
+  [string] $OutDir
+)
+
+function Write-Certificate {
+  param (
+    [X509Certificate2] $Cert,
+    [string] $Name,
+    [string] $Dir
+  )
+  
+  # write cert content
+  $certPem = $Cert.ExportCertificatePem();
+  $certPemPath = Join-Path $Dir -ChildPath "$Name-cert.pem";
+  [System.IO.File]::WriteAllText($certPemPath, $certPem);
+
+  # write pkey
+  [AsymmetricAlgorithm] $pkey = [RSACertificateExtensions]::GetRSAPrivateKey($Cert);
+  [string] $pkeyPem = $null;
+
+  if ($null -ne $pkey) {
+    $pkeyPem = $pkey.ExportRSAPrivateKeyPem();
+  }
+
+  if ($null -eq $pkey) {
+    $pkey = [ECDsaCertificateExtensions]::GetECDsaPrivateKey($Cert);
+    $pkeyPem = $pkey.ExportECPrivateKeyPem();
+  }
+
+  if ($null -eq $pkeyPem) {
+    return;
+  }
+
+  
+  $pKeyPath = Join-Path $Dir -ChildPath "$Name-key.pem";
+  [System.IO.File]::WriteAllText($pKeyPath, $pkeyPem);
+}
+
+$ca = New-SelfSignedCertificate -CertStoreLocation 'Cert:\CurrentUser\My' `
+  -DnsName "otel-test-ca" `
+  -NotAfter (Get-Date).AddYears(20) `
+  -FriendlyName "otel-test-ca" `
+  -KeyAlgorithm ECDSA_nistP256 `
+  -KeyExportPolicy Exportable `
+  -KeyUsageProperty All -KeyUsage CertSign, CRLSign, DigitalSignature;
+
+  
+try {
+  Write-Certificate -Cert $ca  -Name "otel-test-ca" -Dir $OutDir;
+  $serverCert = New-SelfSignedCertificate -CertStoreLocation 'Cert:\CurrentUser\My' `
+    -DnsName "otel-collector" `
+    -Signer $ca `
+    -NotAfter (Get-Date).AddYears(20) `
+    -FriendlyName "otel-test-server" `
+    -KeyAlgorithm ECDSA_nistP256 `
+    -KeyUsageProperty All `
+    -KeyExportPolicy Exportable `
+    -KeyUsage CertSign, CRLSign, DigitalSignature `
+    -TextExtension @("2.5.29.19={text}CA=1&pathlength=1", "2.5.29.37={text}1.3.6.1.5.5.7.3.1");
+
+  try {
+    Write-Certificate -Cert $serverCert  -Name "otel-test-server" -Dir $OutDir;
+
+    $clientCert = New-SelfSignedCertificate -CertStoreLocation 'Cert:\CurrentUser\My' `
+      -DnsName "otel-test-client" `
+      -Signer $ca `
+      -NotAfter (Get-Date).AddYears(20) `
+      -FriendlyName "otel-test-client" `
+      -KeyAlgorithm ECDSA_nistP256 `
+      -KeyUsageProperty All `
+      -KeyExportPolicy Exportable `
+      -KeyUsage CertSign, CRLSign, DigitalSignature `
+      -TextExtension @("2.5.29.19={text}CA=1&pathlength=1", "2.5.29.37={text}1.3.6.1.5.5.7.3.2");
+    try {
+      Write-Certificate -Cert $clientCert  -Name "otel-test-client" -Dir $OutDir;
+    }
+    finally {
+      Get-Item -Path "Cert:\CurrentUser\My\$($clientCert.Thumbprint)" | Remove-Item;
+    }
+  }
+  finally {
+    Get-Item -Path "Cert:\CurrentUser\My\$($serverCert.Thumbprint)" | Remove-Item;
+  }
+}
+finally {
+  Get-Item -Path "Cert:\CurrentUser\My\$($ca.Thumbprint)" | Remove-Item;
+}

test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/gen_test_cert.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+# Set output directory, default is the current directory
+OUT_DIR=${1:-"."}
+
+# Create output directory if it doesn't exist
+mkdir -p "$OUT_DIR"
+
+# Generate CA certificate (Certificate Authority)
+openssl req -new -newkey rsa:2048 -days 3650 -nodes -x509 \
+    -subj "/CN=otel-test-ca" \
+    -keyout "$OUT_DIR/otel-test-ca-key.pem" -out "$OUT_DIR/otel-test-ca-cert.pem"
+
+# Create the extension configuration file for the server certificate
+cat > "$OUT_DIR/server_cert_ext.cnf" <<EOF
+[ v3_req ]
+basicConstraints = CA:FALSE
+nsCertType = server
+nsComment = "OpenSSL Generated Server Certificate"
+subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid,issuer:always
+keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment
+extendedKeyUsage = serverAuth
+subjectAltName = @alt_names
+
+[ alt_names ]
+DNS.1 = otel-collector
+EOF
+
+# Generate server certificate private key and CSR (Certificate Signing Request)
+openssl req -new -newkey rsa:2048 -sha256 -nodes \
+    -keyout "$OUT_DIR/otel-test-server-key.pem" -out "$OUT_DIR/otel-test-server-csr.pem" \
+    -subj "/CN=otel-collector"
+
+# Sign the server certificate using the CA certificate
+openssl x509 -req -in "$OUT_DIR/otel-test-server-csr.pem" \
+    -extfile "$OUT_DIR/server_cert_ext.cnf" -extensions v3_req \
+    -CA "$OUT_DIR/otel-test-ca-cert.pem" -CAkey "$OUT_DIR/otel-test-ca-key.pem" -CAcreateserial \
+    -out "$OUT_DIR/otel-test-server-cert.pem" \
+    -days 3650 -sha256
+
+# Create the extension configuration file for the client certificate
+cat > "$OUT_DIR/client_cert_ext.cnf" <<EOF
+[ v3_client ]
+basicConstraints = CA:FALSE
+nsCertType = client, email
+nsComment = "OpenSSL Generated Client Certificate"
+subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid,issuer
+keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment
+extendedKeyUsage = clientAuth, emailProtection
+EOF
+
+# Generate client certificate private key and CSR
+openssl req -new -newkey rsa:2048 -sha256 -nodes \
+    -keyout "$OUT_DIR/otel-test-client-key.pem" -out "$OUT_DIR/otel-test-client-csr.pem" \
+    -subj "/CN=otel-test-client"
+
+# Sign the client certificate using the CA certificate
+openssl x509 -req -in "$OUT_DIR/otel-test-client-csr.pem" \
+    -extfile "$OUT_DIR/client_cert_ext.cnf" -extensions v3_client \
+    -CA "$OUT_DIR/otel-test-ca-cert.pem" -CAkey "$OUT_DIR/otel-test-ca-key.pem" -CAcreateserial \
+    -out "$OUT_DIR/otel-test-client-cert.pem" \
+    -days 3650 -sha256
+
+# Generate an untrusted self-signed certificate (not signed by the CA)
+openssl req -new -newkey rsa:2048 -days 365 -nodes -x509 \
+    -subj "/CN=otel-untrusted-collector" \
+    -keyout "$OUT_DIR/otel-untrusted-collector-key.pem" -out "$OUT_DIR/otel-untrusted-collector-cert.pem"