Fix flaky InstrumentGarbageCollectionBenchmarkTest (#6221)

.github/workflows/build.yml

@@ -32,6 +32,7 @@ jobs:
           - os: ubuntu-20.04
             test-java-version: 17
             coverage: true
+            jmh-based-tests: true
     steps:
       - uses: actions/checkout@v4
 
@@ -58,6 +59,8 @@ jobs:
             -Porg.gradle.java.installations.paths=${{ steps.setup-java-test.outputs.path }},${{ steps.setup-java.outputs.path }}
         env:
           GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GRADLE_ENTERPRISE_ACCESS_KEY }}
+          # JMH-based tests run only  if this environment variable is set to true
+          RUN_JMH_BASED_TESTS: ${{ matrix.jmh-based-tests }}
 
       - name: Check for diff
         # The jApiCmp diff compares current to latest, which isn't appropriate for release branches

docs/apidiffs/current_vs_latest/opentelemetry-exporter-zipkin.txt

@@ -1,9 +1,2 @@
 Comparing source compatibility of  against 
-***  MODIFIED CLASS: PUBLIC FINAL io.opentelemetry.exporter.zipkin.ZipkinSpanExporterBuilder  (not serializable)
-	===  CLASS FILE FORMAT VERSION: 52.0 <- 52.0
-	===  UNCHANGED METHOD: PUBLIC io.opentelemetry.exporter.zipkin.ZipkinSpanExporterBuilder setEncoder(zipkin2.codec.BytesEncoder<zipkin2.Span><zipkin2.Span>)
-		+++  NEW ANNOTATION: java.lang.Deprecated
-	+++  NEW METHOD: PUBLIC(+) io.opentelemetry.exporter.zipkin.ZipkinSpanExporterBuilder setEncoder(zipkin2.reporter.BytesEncoder<zipkin2.Span>)
-	===  UNCHANGED METHOD: PUBLIC io.opentelemetry.exporter.zipkin.ZipkinSpanExporterBuilder setSender(zipkin2.reporter.Sender)
-		+++  NEW ANNOTATION: java.lang.Deprecated
-	+++  NEW METHOD: PUBLIC(+) io.opentelemetry.exporter.zipkin.ZipkinSpanExporterBuilder setSender(zipkin2.reporter.BytesMessageSender)
+No changes.

sdk/metrics/src/jmhBasedTest/java/io/opentelemetry/sdk/metrics/internal/state/InstrumentGarbageCollectionBenchmarkTest.java

@@ -45,11 +45,12 @@ public class InstrumentGarbageCollectionBenchmarkTest {
   @SuppressWarnings("rawtypes")
   @Test
   public void normalizedAllocationRateTest() throws RunnerException {
-    // GitHub CI has an environment variable (CI=true). We can use it to skip
-    // this test since it's a lengthy one (roughly 10 seconds) and have it running
-    // only in GitHub CI
+    // OTel GitHub CI Workflow (see .github/) sets an environment variable
+    // (RUN_JMH_BASED_TESTS=true).
+    // We set it only there since it's a lengthy test (roughly 2.5min)
+    // and we want to run it only in CI.
     Assumptions.assumeTrue(
-        "true".equals(System.getenv("CI")),
+        "true".equals(System.getenv("RUN_JMH_BASED_TESTS")),
         "This test should only run in GitHub CI since it's long");
 
     // Runs InstrumentGarbageCollectionBenchmark
@@ -91,7 +92,7 @@ public class InstrumentGarbageCollectionBenchmarkTest {
     }
 
     testInstrumentTypeResultsMap.forEach(
-        (testInstrumentType, testInstrumentTypeResults) -> {
+        (testInstrumentTypeString, testInstrumentTypeResults) -> {
           Map<String, Map<String, Double>> resultMap =
               testInstrumentTypeResults.aggregationTemporalityToMemoryModeResult;
           assertThat(resultMap).hasSameSizeAs(AggregationTemporality.values());
@@ -108,9 +109,11 @@ public class InstrumentGarbageCollectionBenchmarkTest {
                 assertThat(immutableDataAllocRate).isNotNull().isNotZero();
                 assertThat(reusableDataAllocRate).isNotNull().isNotZero();
 
+                TestInstrumentType testInstrumentType =
+                    TestInstrumentType.valueOf(testInstrumentTypeString);
                 float dataAllocRateReductionPercentage =
-                    TestInstrumentType.valueOf(testInstrumentType)
-                        .getDataAllocRateReductionPercentage();
+                    testInstrumentType.getDataAllocRateReductionPercentage();
+                double allowedOffset = testInstrumentType.getAllowedPercentOffset();
 
                 // If this test suddenly fails for you this means you have changed the code in a way
                 // that allocates more memory than before. You can find out where, by running
@@ -119,8 +122,8 @@ public class InstrumentGarbageCollectionBenchmarkTest {
                 assertThat(100 - (reusableDataAllocRate / immutableDataAllocRate) * 100)
                     .describedAs(
                         "Aggregation temporality = %s, testInstrumentType = %s",
-                        aggregationTemporality, testInstrumentType)
-                    .isCloseTo(dataAllocRateReductionPercentage, Offset.offset(2.0));
+                        aggregationTemporality, testInstrumentTypeString)
+                    .isCloseTo(dataAllocRateReductionPercentage, Offset.offset(allowedOffset));
               });
         });
   }

sdk/metrics/src/jmhBasedTest/java/io/opentelemetry/sdk/metrics/internal/state/TestInstrumentType.java

@@ -24,31 +24,52 @@ public enum TestInstrumentType {
   ASYNC_COUNTER(AsyncCounterTester::new),
   EXPONENTIAL_HISTOGRAM(ExponentialHistogramTester::new),
   EXPLICIT_BUCKET(ExplicitBucketHistogramTester::new),
-  LONG_SUM(LongSumTester::new, /* dataAllocRateReductionPercentage= */ 97.3f),
-  DOUBLE_SUM(DoubleSumTester::new, /* dataAllocRateReductionPercentage= */ 97.3f),
-  LONG_LAST_VALUE(LongLastValueTester::new, /* dataAllocRateReductionPercentage= */ 97.3f),
-  DOUBLE_LAST_VALUE(DoubleLastValueTester::new, /* dataAllocRateReductionPercentage= */ 97.3f);
+  LONG_SUM(
+      LongSumTester::new,
+      /* dataAllocRateReductionPercentage= */ 97.3f,
+      /* allowedPercentOffset= */ 4.0f),
+  DOUBLE_SUM(
+      DoubleSumTester::new,
+      /* dataAllocRateReductionPercentage= */ 97.3f,
+      /* allowedPercentOffset= */ 2.0f),
+  LONG_LAST_VALUE(
+      LongLastValueTester::new,
+      /* dataAllocRateReductionPercentage= */ 97.3f,
+      /* allowedPercentOffset= */ 4.0f),
+  DOUBLE_LAST_VALUE(
+      DoubleLastValueTester::new,
+      /* dataAllocRateReductionPercentage= */ 97.3f,
+      /* allowedPercentOffset= */ 4.0f);
 
   private final Supplier<? extends InstrumentTester> instrumentTesterInitializer;
   private final float dataAllocRateReductionPercentage;
+  private final double allowedPercentOffset;
 
+  @SuppressWarnings("unused")
   TestInstrumentType(Supplier<? extends InstrumentTester> instrumentTesterInitializer) {
     this.dataAllocRateReductionPercentage = 99.8f; // default
     this.instrumentTesterInitializer = instrumentTesterInitializer;
+    this.allowedPercentOffset = 2.0f;
   }
 
   // Some instruments have different reduction percentage.
   TestInstrumentType(
       Supplier<? extends InstrumentTester> instrumentTesterInitializer,
-      float dataAllocRateReductionPercentage) {
+      float dataAllocRateReductionPercentage,
+      float allowedPercentOffset) {
     this.instrumentTesterInitializer = instrumentTesterInitializer;
     this.dataAllocRateReductionPercentage = dataAllocRateReductionPercentage;
+    this.allowedPercentOffset = allowedPercentOffset;
   }
 
   float getDataAllocRateReductionPercentage() {
     return dataAllocRateReductionPercentage;
   }
 
+  public double getAllowedPercentOffset() {
+    return allowedPercentOffset;
+  }
+
   InstrumentTester createInstrumentTester() {
     return instrumentTesterInitializer.get();
   }