Update JMH benchmarks (#1242)

Update JMH benchmarks

dd-java-agent/benchmark/benchmark.gradle

@@ -6,68 +6,44 @@ apply from: "${rootDir}/gradle/java.gradle"
 
 dependencies {
   jmh project(':dd-trace-api')
-  jmh group: 'net.bytebuddy', name: 'byte-buddy-agent', version: '1.7.6'
+  jmh deps.bytebuddyagent
-
-  // Add a bunch of dependencies so instrumentation is not disabled.
-  jmh group: 'javax.jms', name: 'javax.jms-api', version: '2.0.1'
-  jmh group: 'javax.servlet', name: 'javax.servlet-api', version: '3.0.1'
-  jmh group: 'org.mongodb', name: 'mongo-java-driver', version: '3.4.2'
-  jmh group: 'org.mongodb', name: 'mongodb-driver-async', version: '3.4.2'
-  jmh(group: 'com.amazonaws', name: 'aws-java-sdk', version: '1.11.119') {
-    exclude(module: 'httpclient')
-    exclude(module: 'jackson-databind')
-    exclude(module: 'jackson-dataformat-cbor')
 }
-  jmh group: 'com.squareup.okhttp3', name: 'okhttp', version: '3.6.0'
-  jmh group: 'org.apache.httpcomponents', name: 'httpclient', version: '4.5.3'
-  jmh(group: 'com.datastax.cassandra', name: 'cassandra-driver-core', version: '3.2.0')
-}
-
-configurations.testRuntimeClasspath.dependencies.clear()
-
 
 jmh {
   timeUnit = 'us' // Output time unit. Available time units are: [m, s, ms, us, ns].
-  benchmarkMode = ['thrpt', 'avgt']
+  benchmarkMode = ['avgt']
-//  timeOnIteration = '5s'
+  timeOnIteration = '20s'
-  iterations = 5 // Number of measurement iterations to do.
+  iterations = 1 // Number of measurement iterations to do.
   fork = 1 // How many times to forks a single benchmark. Use 0 to disable forking altogether
-//  jvmArgs = ["-Dasdf=123"]
+  jvmArgs = ["-Ddd.jmxfetch.enabled=false", "-Ddd.writer.type=LoggingWriter"]
-//  jvmArgs = ["-javaagent:${project(':dd-java-agent').shadowJar.archivePath}"]
+//  jvmArgs += ["-XX:+UnlockDiagnosticVMOptions", "-XX:+DebugNonSafepoints", "-XX:StartFlightRecording=delay=5s,dumponexit=true,name=jmh-benchmark,filename=${rootDir}/dd-java-agent/benchmark/build/reports/jmh/jmh-benchmark.jfr"]
+//  jvmArgs += ["-agentpath:${rootDir}/dd-java-agent/benchmark/src/jmh/resources/libasyncProfiler.so=start,collapsed,file=${rootDir}/dd-java-agent/benchmark/build/reports/jmh/profiler.txt"]
   failOnError = true // Should JMH fail immediately if any benchmark had experienced the unrecoverable error?
-//  warmup = '2s' // Time to spend at each warmup iteration.
+  warmup = '5s' // Time to spend at each warmup iteration.
-//  warmupIterations = 2 // Number of warmup iterations to do.
+//  warmupBatchSize = 10 // Warmup batch size: number of benchmark method calls per operation.
-//  warmupForks = 0 // How many warmup forks to make for a single benchmark. 0 to disable warmup forks.
+  warmupForks = 0 // How many warmup forks to make for a single benchmark. 0 to disable warmup forks.
+  warmupIterations = 1 // Number of warmup iterations to do.
 
-//  profilers = ['stack']
+//  profilers = ['stack:lines=5;detailLine=true;period=5;excludePackages=true']
   // Use profilers to collect additional data. Supported profilers: [cl, comp, gc, stack, perf, perfnorm, perfasm, xperf, xperfasm, hs_cl, hs_comp, hs_gc, hs_rt, hs_thr]
 
 //  humanOutputFile = project.file("${project.buildDir}/reports/jmh/human.txt") // human-readable output file
 //  operationsPerInvocation = 10 // Operations per invocation.
 //  synchronizeIterations = false // Synchronize iterations?
-  timeout = '1s' // Timeout for benchmark iteration.
+  timeout = '5s' // Timeout for benchmark iteration.
-  includeTests = false
+//  includeTests = false
   // Allows to include test sources into generate JMH jar, i.e. use it when benchmarks depend on the test classes.
 
-  duplicateClassesStrategy = 'fail'
+  duplicateClassesStrategy = DuplicatesStrategy.EXCLUDE
-  jmhVersion = '1.20' // Specifies JMH version
+  jmhVersion = '1.23' // Specifies JMH version
 }
 
-// configured as a separate task since the 'jmh' task did not like adding a javaagent argument.
+tasks.jmh.dependsOn project(':dd-java-agent').shadowJar
-tasks.register("jmhAgent", JavaExec) {
-  classpath = files(project.jmhCompileGeneratedClasses.destinationDir)
-  classpath += sourceSets.jmh.runtimeClasspath
-  main = "org.openjdk.jmh.Main"
-  args += ["-tu", "us"]
-  args += ["-bm", "avgt"]
-//  args += ["-prof", "stack:lines=5;detailLine=true;period=5;excludePackages=true"]
-  args += ["-f", "1"]
-  args += ["-foe", "true"]
 
-//  args += ["-wi", "2"]
+/*
-//  args += ["-i", "5"]
+If using libasyncProfiler, use the following to generate nice svg flamegraphs.
-  dependsOn project.tasks.jmhCompileGeneratedClasses
+sed '/unknown/d' dd-java-agent/benchmark/build/reports/jmh/profiler.txt | sed '/^thread_start/d' | sed '/not_walkable/d' > dd-java-agent/benchmark/build/reports/jmh/profiler-cleaned.txt
-}
+(using https://github.com/brendangregg/FlameGraph)
-
+./flamegraph.pl --color=java dd-java-agent/benchmark/build/reports/jmh/profiler-cleaned.txt > dd-java-agent/benchmark/build/reports/jmh/jmh-master.svg
-tasks.jmhAgent.dependsOn project(':dd-java-agent').shadowJar
+ */
 

dd-java-agent/benchmark/src/jmh/java/datadog/benchmark/ClassRetransformingBenchmark.java

@@ -4,49 +4,17 @@ import datadog.benchmark.classes.TracedClass;
 import datadog.benchmark.classes.UntracedClass;
 import java.lang.instrument.Instrumentation;
 import java.lang.instrument.UnmodifiableClassException;
-import java.lang.reflect.Field;
-import java.lang.reflect.Method;
-import java.nio.file.Paths;
 import net.bytebuddy.agent.ByteBuddyAgent;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.Fork;
 import org.openjdk.jmh.annotations.Scope;
-import org.openjdk.jmh.annotations.Setup;
 import org.openjdk.jmh.annotations.State;
-import org.openjdk.jmh.annotations.TearDown;
 
 public class ClassRetransformingBenchmark {
-  public static final String BENCHMARK_HOME =
-      Paths.get(".").toAbsolutePath().normalize().toString();
-
-  static {
-    if (!BENCHMARK_HOME.endsWith("benchmark")) {
-      throw new IllegalArgumentException("Invalid Home directory: " + BENCHMARK_HOME);
-    }
-  }
 
   @State(Scope.Benchmark)
   public static class BenchmarkState {
     private final Instrumentation inst = ByteBuddyAgent.install();
-
-    @Setup
-    public void initializeInstrumentation() {
-      // loading TracedClass will initialize helper injection
-      TracedClass.class.getName();
-    }
-
-    @TearDown
-    public void stopAgent() {
-      try {
-        final Class<?> gt = Class.forName("io.opentracing.util.GlobalTracer");
-        final Field tracerField = gt.getDeclaredField("tracer");
-        tracerField.setAccessible(true);
-        final Object tracer = tracerField.get(null);
-        final Method close = tracer.getClass().getMethod("close");
-        close.invoke(tracer);
-      } catch (final Exception e) {
-      }
-    }
   }
 
   @Benchmark
@@ -60,54 +28,11 @@ public class ClassRetransformingBenchmark {
     state.inst.retransformClasses(TracedClass.class);
   }
 
-  @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.2.2.jar")
+  @Fork(jvmArgsAppend = "-javaagent:/path/to/dd-java-agent-master.jar")
-  public static class WithAgent022 extends ClassRetransformingBenchmark {}
+  public static class WithAgentMaster extends ClassRetransformingBenchmark {}
 
-  @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.2.4.jar")
+  @Fork(
-  public static class WithAgent024 extends ClassRetransformingBenchmark {}
+      jvmArgsAppend =
-
+          "-javaagent:/path/to/dd-trace-java/dd-java-agent/build/libs/dd-java-agent.jar")
-  @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.2.6.jar")
-  public static class WithAgent026 extends ClassRetransformingBenchmark {}
-
-  @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.2.7.jar")
-  public static class WithAgent027 extends ClassRetransformingBenchmark {}
-
-  @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.2.8.jar")
-  public static class WithAgent028 extends ClassRetransformingBenchmark {}
-
-  @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.2.9.jar")
-  public static class WithAgent029 extends ClassRetransformingBenchmark {}
-
-  @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.2.10.jar")
-  public static class WithAgent0210 extends ClassRetransformingBenchmark {}
-
-  @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.2.11.jar")
-  public static class WithAgent0211 extends ClassRetransformingBenchmark {}
-
-  @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.2.12.jar")
-  public static class WithAgent0212 extends ClassRetransformingBenchmark {}
-
-  @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.3.0.jar")
-  public static class WithAgent030 extends ClassRetransformingBenchmark {}
-
-  @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.3.1.jar")
-  public static class WithAgent031 extends ClassRetransformingBenchmark {}
-
-  @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.3.2.jar")
-  public static class WithAgent032 extends ClassRetransformingBenchmark {}
-
-  @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.3.3.jar")
-  public static class WithAgent033 extends ClassRetransformingBenchmark {}
-
-  @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.4.0.jar")
-  public static class WithAgent040 extends ClassRetransformingBenchmark {}
-
-  @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.4.1.jar")
-  public static class WithAgent041 extends ClassRetransformingBenchmark {}
-
-  @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.5.0.jar")
-  public static class WithAgent050 extends ClassRetransformingBenchmark {}
-
-  @Fork(jvmArgsAppend = "-javaagent:../build/libs/dd-java-agent.jar")
   public static class WithAgent extends ClassRetransformingBenchmark {}
 }

dd-java-agent/benchmark/src/jmh/java/datadog/benchmark/classes/C.java

@@ -1,5 +1,5 @@
 package datadog.benchmark.classes;
 
-public interface C extends B {
+public interface C extends A, B {
   void c();
 }

dd-java-agent/benchmark/src/jmh/java/datadog/benchmark/classes/D.java

@@ -1,5 +1,5 @@
 package datadog.benchmark.classes;
 
-public interface D extends C {
+public interface D extends A, B, C {
   void d();
 }

dd-java-agent/benchmark/src/jmh/java/datadog/benchmark/classes/E.java

@@ -1,5 +1,5 @@
 package datadog.benchmark.classes;
 
-public interface E extends D {
+public interface E extends B, C, D {
   void e();
 }

dd-java-agent/benchmark/src/jmh/resources/dd-trace.yaml

@@ -1 +0,0 @@
-enableCustomAnnotationTracingOver: ["datadog.benchmark"]

dd-trace-ot/dd-trace-ot.gradle

@@ -139,7 +139,7 @@ jmh {
 //  warmupBenchmarks = ['.*Warmup'] // Warmup benchmarks to include in the run in addition to already selected. JMH will not measure these benchmarks, but only use them for the warmup.
 
 //  zip64 = true // Use ZIP64 format for bigger archives
-  jmhVersion = '1.21' // Specifies JMH version
+  jmhVersion = '1.23' // Specifies JMH version
 //  includeTests = true // Allows to include test sources into generate JMH jar, i.e. use it when benchmarks depend on the test classes.
   duplicateClassesStrategy = 'warn'
   // Strategy to apply when encountring duplicate classes during creation of the fat jar (i.e. while executing jmhJar task)