From e1964e526e72edd5b93c5f17524cee3e3db1422b Mon Sep 17 00:00:00 2001 From: Tyler Benson Date: Thu, 20 Feb 2020 16:03:41 -0800 Subject: [PATCH] Update JMH benchmarks Benchmark is run with `./gradlew :dd-java-agent:benchmark:jmh` -- after updating the absolute paths to the various agents in `ClassRetransformingBenchmark`. Unfortunately I was unable to have it launch correctly with a relative path. --- dd-java-agent/benchmark/benchmark.gradle | 70 +++++---------- .../ClassRetransformingBenchmark.java | 85 ++----------------- .../jmh/java/datadog/benchmark/classes/C.java | 2 +- .../jmh/java/datadog/benchmark/classes/D.java | 2 +- .../jmh/java/datadog/benchmark/classes/E.java | 2 +- .../benchmark/src/jmh/resources/dd-trace.yaml | 1 - dd-trace-ot/dd-trace-ot.gradle | 2 +- 7 files changed, 32 insertions(+), 132 deletions(-) delete mode 100644 dd-java-agent/benchmark/src/jmh/resources/dd-trace.yaml diff --git a/dd-java-agent/benchmark/benchmark.gradle b/dd-java-agent/benchmark/benchmark.gradle index 8d0205b087..7fcce09041 100644 --- a/dd-java-agent/benchmark/benchmark.gradle +++ b/dd-java-agent/benchmark/benchmark.gradle @@ -6,68 +6,44 @@ apply from: "${rootDir}/gradle/java.gradle" dependencies { jmh project(':dd-trace-api') - jmh group: 'net.bytebuddy', name: 'byte-buddy-agent', version: '1.7.6' - - // Add a bunch of dependencies so instrumentation is not disabled. - jmh group: 'javax.jms', name: 'javax.jms-api', version: '2.0.1' - jmh group: 'javax.servlet', name: 'javax.servlet-api', version: '3.0.1' - jmh group: 'org.mongodb', name: 'mongo-java-driver', version: '3.4.2' - jmh group: 'org.mongodb', name: 'mongodb-driver-async', version: '3.4.2' - jmh(group: 'com.amazonaws', name: 'aws-java-sdk', version: '1.11.119') { - exclude(module: 'httpclient') - exclude(module: 'jackson-databind') - exclude(module: 'jackson-dataformat-cbor') - } - jmh group: 'com.squareup.okhttp3', name: 'okhttp', version: '3.6.0' - jmh group: 'org.apache.httpcomponents', name: 'httpclient', version: '4.5.3' - jmh(group: 'com.datastax.cassandra', name: 'cassandra-driver-core', version: '3.2.0') + jmh deps.bytebuddyagent } -configurations.testRuntimeClasspath.dependencies.clear() - - jmh { timeUnit = 'us' // Output time unit. Available time units are: [m, s, ms, us, ns]. - benchmarkMode = ['thrpt', 'avgt'] -// timeOnIteration = '5s' - iterations = 5 // Number of measurement iterations to do. + benchmarkMode = ['avgt'] + timeOnIteration = '20s' + iterations = 1 // Number of measurement iterations to do. fork = 1 // How many times to forks a single benchmark. Use 0 to disable forking altogether -// jvmArgs = ["-Dasdf=123"] -// jvmArgs = ["-javaagent:${project(':dd-java-agent').shadowJar.archivePath}"] + jvmArgs = ["-Ddd.jmxfetch.enabled=false", "-Ddd.writer.type=LoggingWriter"] +// jvmArgs += ["-XX:+UnlockDiagnosticVMOptions", "-XX:+DebugNonSafepoints", "-XX:StartFlightRecording=delay=5s,dumponexit=true,name=jmh-benchmark,filename=${rootDir}/dd-java-agent/benchmark/build/reports/jmh/jmh-benchmark.jfr"] +// jvmArgs += ["-agentpath:${rootDir}/dd-java-agent/benchmark/src/jmh/resources/libasyncProfiler.so=start,collapsed,file=${rootDir}/dd-java-agent/benchmark/build/reports/jmh/profiler.txt"] failOnError = true // Should JMH fail immediately if any benchmark had experienced the unrecoverable error? -// warmup = '2s' // Time to spend at each warmup iteration. -// warmupIterations = 2 // Number of warmup iterations to do. -// warmupForks = 0 // How many warmup forks to make for a single benchmark. 0 to disable warmup forks. + warmup = '5s' // Time to spend at each warmup iteration. +// warmupBatchSize = 10 // Warmup batch size: number of benchmark method calls per operation. + warmupForks = 0 // How many warmup forks to make for a single benchmark. 0 to disable warmup forks. + warmupIterations = 1 // Number of warmup iterations to do. -// profilers = ['stack'] +// profilers = ['stack:lines=5;detailLine=true;period=5;excludePackages=true'] // Use profilers to collect additional data. Supported profilers: [cl, comp, gc, stack, perf, perfnorm, perfasm, xperf, xperfasm, hs_cl, hs_comp, hs_gc, hs_rt, hs_thr] // humanOutputFile = project.file("${project.buildDir}/reports/jmh/human.txt") // human-readable output file // operationsPerInvocation = 10 // Operations per invocation. // synchronizeIterations = false // Synchronize iterations? - timeout = '1s' // Timeout for benchmark iteration. - includeTests = false + timeout = '5s' // Timeout for benchmark iteration. +// includeTests = false // Allows to include test sources into generate JMH jar, i.e. use it when benchmarks depend on the test classes. - duplicateClassesStrategy = 'fail' - jmhVersion = '1.20' // Specifies JMH version + duplicateClassesStrategy = DuplicatesStrategy.EXCLUDE + jmhVersion = '1.23' // Specifies JMH version } -// configured as a separate task since the 'jmh' task did not like adding a javaagent argument. -tasks.register("jmhAgent", JavaExec) { - classpath = files(project.jmhCompileGeneratedClasses.destinationDir) - classpath += sourceSets.jmh.runtimeClasspath - main = "org.openjdk.jmh.Main" - args += ["-tu", "us"] - args += ["-bm", "avgt"] -// args += ["-prof", "stack:lines=5;detailLine=true;period=5;excludePackages=true"] - args += ["-f", "1"] - args += ["-foe", "true"] +tasks.jmh.dependsOn project(':dd-java-agent').shadowJar -// args += ["-wi", "2"] -// args += ["-i", "5"] - dependsOn project.tasks.jmhCompileGeneratedClasses -} - -tasks.jmhAgent.dependsOn project(':dd-java-agent').shadowJar +/* +If using libasyncProfiler, use the following to generate nice svg flamegraphs. +sed '/unknown/d' dd-java-agent/benchmark/build/reports/jmh/profiler.txt | sed '/^thread_start/d' | sed '/not_walkable/d' > dd-java-agent/benchmark/build/reports/jmh/profiler-cleaned.txt +(using https://github.com/brendangregg/FlameGraph) +./flamegraph.pl --color=java dd-java-agent/benchmark/build/reports/jmh/profiler-cleaned.txt > dd-java-agent/benchmark/build/reports/jmh/jmh-master.svg + */ diff --git a/dd-java-agent/benchmark/src/jmh/java/datadog/benchmark/ClassRetransformingBenchmark.java b/dd-java-agent/benchmark/src/jmh/java/datadog/benchmark/ClassRetransformingBenchmark.java index baec44ec36..e8c8986a45 100644 --- a/dd-java-agent/benchmark/src/jmh/java/datadog/benchmark/ClassRetransformingBenchmark.java +++ b/dd-java-agent/benchmark/src/jmh/java/datadog/benchmark/ClassRetransformingBenchmark.java @@ -4,49 +4,17 @@ import datadog.benchmark.classes.TracedClass; import datadog.benchmark.classes.UntracedClass; import java.lang.instrument.Instrumentation; import java.lang.instrument.UnmodifiableClassException; -import java.lang.reflect.Field; -import java.lang.reflect.Method; -import java.nio.file.Paths; import net.bytebuddy.agent.ByteBuddyAgent; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.Fork; import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.TearDown; public class ClassRetransformingBenchmark { - public static final String BENCHMARK_HOME = - Paths.get(".").toAbsolutePath().normalize().toString(); - - static { - if (!BENCHMARK_HOME.endsWith("benchmark")) { - throw new IllegalArgumentException("Invalid Home directory: " + BENCHMARK_HOME); - } - } @State(Scope.Benchmark) public static class BenchmarkState { private final Instrumentation inst = ByteBuddyAgent.install(); - - @Setup - public void initializeInstrumentation() { - // loading TracedClass will initialize helper injection - TracedClass.class.getName(); - } - - @TearDown - public void stopAgent() { - try { - final Class gt = Class.forName("io.opentracing.util.GlobalTracer"); - final Field tracerField = gt.getDeclaredField("tracer"); - tracerField.setAccessible(true); - final Object tracer = tracerField.get(null); - final Method close = tracer.getClass().getMethod("close"); - close.invoke(tracer); - } catch (final Exception e) { - } - } } @Benchmark @@ -60,54 +28,11 @@ public class ClassRetransformingBenchmark { state.inst.retransformClasses(TracedClass.class); } - @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.2.2.jar") - public static class WithAgent022 extends ClassRetransformingBenchmark {} + @Fork(jvmArgsAppend = "-javaagent:/path/to/dd-java-agent-master.jar") + public static class WithAgentMaster extends ClassRetransformingBenchmark {} - @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.2.4.jar") - public static class WithAgent024 extends ClassRetransformingBenchmark {} - - @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.2.6.jar") - public static class WithAgent026 extends ClassRetransformingBenchmark {} - - @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.2.7.jar") - public static class WithAgent027 extends ClassRetransformingBenchmark {} - - @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.2.8.jar") - public static class WithAgent028 extends ClassRetransformingBenchmark {} - - @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.2.9.jar") - public static class WithAgent029 extends ClassRetransformingBenchmark {} - - @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.2.10.jar") - public static class WithAgent0210 extends ClassRetransformingBenchmark {} - - @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.2.11.jar") - public static class WithAgent0211 extends ClassRetransformingBenchmark {} - - @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.2.12.jar") - public static class WithAgent0212 extends ClassRetransformingBenchmark {} - - @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.3.0.jar") - public static class WithAgent030 extends ClassRetransformingBenchmark {} - - @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.3.1.jar") - public static class WithAgent031 extends ClassRetransformingBenchmark {} - - @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.3.2.jar") - public static class WithAgent032 extends ClassRetransformingBenchmark {} - - @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.3.3.jar") - public static class WithAgent033 extends ClassRetransformingBenchmark {} - - @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.4.0.jar") - public static class WithAgent040 extends ClassRetransformingBenchmark {} - - @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.4.1.jar") - public static class WithAgent041 extends ClassRetransformingBenchmark {} - - @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.5.0.jar") - public static class WithAgent050 extends ClassRetransformingBenchmark {} - - @Fork(jvmArgsAppend = "-javaagent:../build/libs/dd-java-agent.jar") + @Fork( + jvmArgsAppend = + "-javaagent:/path/to/dd-trace-java/dd-java-agent/build/libs/dd-java-agent.jar") public static class WithAgent extends ClassRetransformingBenchmark {} } diff --git a/dd-java-agent/benchmark/src/jmh/java/datadog/benchmark/classes/C.java b/dd-java-agent/benchmark/src/jmh/java/datadog/benchmark/classes/C.java index d124f05c42..b7a256697c 100644 --- a/dd-java-agent/benchmark/src/jmh/java/datadog/benchmark/classes/C.java +++ b/dd-java-agent/benchmark/src/jmh/java/datadog/benchmark/classes/C.java @@ -1,5 +1,5 @@ package datadog.benchmark.classes; -public interface C extends B { +public interface C extends A, B { void c(); } diff --git a/dd-java-agent/benchmark/src/jmh/java/datadog/benchmark/classes/D.java b/dd-java-agent/benchmark/src/jmh/java/datadog/benchmark/classes/D.java index ea8ffba11a..a52a49aa62 100644 --- a/dd-java-agent/benchmark/src/jmh/java/datadog/benchmark/classes/D.java +++ b/dd-java-agent/benchmark/src/jmh/java/datadog/benchmark/classes/D.java @@ -1,5 +1,5 @@ package datadog.benchmark.classes; -public interface D extends C { +public interface D extends A, B, C { void d(); } diff --git a/dd-java-agent/benchmark/src/jmh/java/datadog/benchmark/classes/E.java b/dd-java-agent/benchmark/src/jmh/java/datadog/benchmark/classes/E.java index 3ac6694578..9846b8802c 100644 --- a/dd-java-agent/benchmark/src/jmh/java/datadog/benchmark/classes/E.java +++ b/dd-java-agent/benchmark/src/jmh/java/datadog/benchmark/classes/E.java @@ -1,5 +1,5 @@ package datadog.benchmark.classes; -public interface E extends D { +public interface E extends B, C, D { void e(); } diff --git a/dd-java-agent/benchmark/src/jmh/resources/dd-trace.yaml b/dd-java-agent/benchmark/src/jmh/resources/dd-trace.yaml deleted file mode 100644 index 22d9698b6f..0000000000 --- a/dd-java-agent/benchmark/src/jmh/resources/dd-trace.yaml +++ /dev/null @@ -1 +0,0 @@ -enableCustomAnnotationTracingOver: ["datadog.benchmark"] diff --git a/dd-trace-ot/dd-trace-ot.gradle b/dd-trace-ot/dd-trace-ot.gradle index 3d5772654e..a44b148d24 100644 --- a/dd-trace-ot/dd-trace-ot.gradle +++ b/dd-trace-ot/dd-trace-ot.gradle @@ -138,7 +138,7 @@ jmh { // warmupBenchmarks = ['.*Warmup'] // Warmup benchmarks to include in the run in addition to already selected. JMH will not measure these benchmarks, but only use them for the warmup. // zip64 = true // Use ZIP64 format for bigger archives - jmhVersion = '1.21' // Specifies JMH version + jmhVersion = '1.23' // Specifies JMH version // includeTests = true // Allows to include test sources into generate JMH jar, i.e. use it when benchmarks depend on the test classes. duplicateClassesStrategy = 'warn' // Strategy to apply when encountring duplicate classes during creation of the fat jar (i.e. while executing jmhJar task)