open-telemetry
/
opentelemetry-java-instrumentation
mirror of https://github.com/open-telemetry/opentelemetry-java-instrumentation.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update JMH benchmarks (#1242) 

Update JMH benchmarks
This commit is contained in:
Tyler Benson 2020-02-21 09:38:34 -08:00 committed by GitHub
parent e92d326052 e1964e526e
commit e4d29e37ef
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 32 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
dd-java-agent/benchmark/benchmark.gradle
View File

@ -6,68 +6,44 @@ apply from: "${rootDir}/gradle/java.gradle"
dependencies {
jmh project(':dd-trace-api')
jmh group: 'net.bytebuddy', name: 'byte-buddy-agent', version: '1.7.6'
// Add a bunch of dependencies so instrumentation is not disabled.
jmh group: 'javax.jms', name: 'javax.jms-api', version: '2.0.1'
jmh group: 'javax.servlet', name: 'javax.servlet-api', version: '3.0.1'
jmh group: 'org.mongodb', name: 'mongo-java-driver', version: '3.4.2'
jmh group: 'org.mongodb', name: 'mongodb-driver-async', version: '3.4.2'
jmh(group: 'com.amazonaws', name: 'aws-java-sdk', version: '1.11.119') {
exclude(module: 'httpclient')
exclude(module: 'jackson-databind')
exclude(module: 'jackson-dataformat-cbor')
}
jmh group: 'com.squareup.okhttp3', name: 'okhttp', version: '3.6.0'
jmh group: 'org.apache.httpcomponents', name: 'httpclient', version: '4.5.3'
jmh(group: 'com.datastax.cassandra', name: 'cassandra-driver-core', version: '3.2.0')
jmh deps.bytebuddyagent
}
configurations.testRuntimeClasspath.dependencies.clear()
jmh {
timeUnit = 'us' // Output time unit. Available time units are: [m, s, ms, us, ns].
benchmarkMode = ['thrpt', 'avgt']
// timeOnIteration = '5s'
iterations = 5 // Number of measurement iterations to do.
benchmarkMode = ['avgt']
timeOnIteration = '20s'
iterations = 1 // Number of measurement iterations to do.
fork = 1 // How many times to forks a single benchmark. Use 0 to disable forking altogether
// jvmArgs = ["-Dasdf=123"]
// jvmArgs = ["-javaagent:${project(':dd-java-agent').shadowJar.archivePath}"]
jvmArgs = ["-Ddd.jmxfetch.enabled=false", "-Ddd.writer.type=LoggingWriter"]
// jvmArgs += ["-XX:+UnlockDiagnosticVMOptions", "-XX:+DebugNonSafepoints", "-XX:StartFlightRecording=delay=5s,dumponexit=true,name=jmh-benchmark,filename=${rootDir}/dd-java-agent/benchmark/build/reports/jmh/jmh-benchmark.jfr"]
// jvmArgs += ["-agentpath:${rootDir}/dd-java-agent/benchmark/src/jmh/resources/libasyncProfiler.so=start,collapsed,file=${rootDir}/dd-java-agent/benchmark/build/reports/jmh/profiler.txt"]
failOnError = true // Should JMH fail immediately if any benchmark had experienced the unrecoverable error?
// warmup = '2s' // Time to spend at each warmup iteration.
// warmupIterations = 2 // Number of warmup iterations to do.
// warmupForks = 0 // How many warmup forks to make for a single benchmark. 0 to disable warmup forks.
warmup = '5s' // Time to spend at each warmup iteration.
// warmupBatchSize = 10 // Warmup batch size: number of benchmark method calls per operation.
warmupForks = 0 // How many warmup forks to make for a single benchmark. 0 to disable warmup forks.
warmupIterations = 1 // Number of warmup iterations to do.
// profilers = ['stack']
// profilers = ['stack:lines=5;detailLine=true;period=5;excludePackages=true']
// Use profilers to collect additional data. Supported profilers: [cl, comp, gc, stack, perf, perfnorm, perfasm, xperf, xperfasm, hs_cl, hs_comp, hs_gc, hs_rt, hs_thr]
// humanOutputFile = project.file("${project.buildDir}/reports/jmh/human.txt") // human-readable output file
// operationsPerInvocation = 10 // Operations per invocation.
// synchronizeIterations = false // Synchronize iterations?
timeout = '1s' // Timeout for benchmark iteration.
includeTests = false
timeout = '5s' // Timeout for benchmark iteration.
// includeTests = false
// Allows to include test sources into generate JMH jar, i.e. use it when benchmarks depend on the test classes.
duplicateClassesStrategy = 'fail'
jmhVersion = '1.20' // Specifies JMH version
duplicateClassesStrategy = DuplicatesStrategy.EXCLUDE
jmhVersion = '1.23' // Specifies JMH version
}
// configured as a separate task since the 'jmh' task did not like adding a javaagent argument.
tasks.register("jmhAgent", JavaExec) {
classpath = files(project.jmhCompileGeneratedClasses.destinationDir)
classpath += sourceSets.jmh.runtimeClasspath
main = "org.openjdk.jmh.Main"
args += ["-tu", "us"]
args += ["-bm", "avgt"]
// args += ["-prof", "stack:lines=5;detailLine=true;period=5;excludePackages=true"]
args += ["-f", "1"]
args += ["-foe", "true"]
tasks.jmh.dependsOn project(':dd-java-agent').shadowJar
// args += ["-wi", "2"]
// args += ["-i", "5"]
dependsOn project.tasks.jmhCompileGeneratedClasses
}
tasks.jmhAgent.dependsOn project(':dd-java-agent').shadowJar
/*
If using libasyncProfiler, use the following to generate nice svg flamegraphs.
sed '/unknown/d' dd-java-agent/benchmark/build/reports/jmh/profiler.txt | sed '/^thread_start/d' | sed '/not_walkable/d' > dd-java-agent/benchmark/build/reports/jmh/profiler-cleaned.txt
(using https://github.com/brendangregg/FlameGraph)
./flamegraph.pl --color=java dd-java-agent/benchmark/build/reports/jmh/profiler-cleaned.txt > dd-java-agent/benchmark/build/reports/jmh/jmh-master.svg
*/

85
dd-java-agent/benchmark/src/jmh/java/datadog/benchmark/ClassRetransformingBenchmark.java
View File

@ -4,49 +4,17 @@ import datadog.benchmark.classes.TracedClass;
import datadog.benchmark.classes.UntracedClass;
import java.lang.instrument.Instrumentation;
import java.lang.instrument.UnmodifiableClassException;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.nio.file.Paths;
import net.bytebuddy.agent.ByteBuddyAgent;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.Fork;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.annotations.TearDown;
public class ClassRetransformingBenchmark {
public static final String BENCHMARK_HOME =
Paths.get(".").toAbsolutePath().normalize().toString();
static {
if (!BENCHMARK_HOME.endsWith("benchmark")) {
throw new IllegalArgumentException("Invalid Home directory: " + BENCHMARK_HOME);
}
}
@State(Scope.Benchmark)
public static class BenchmarkState {
private final Instrumentation inst = ByteBuddyAgent.install();
@Setup
public void initializeInstrumentation() {
// loading TracedClass will initialize helper injection
TracedClass.class.getName();
}
@TearDown
public void stopAgent() {
try {
final Class<?> gt = Class.forName("io.opentracing.util.GlobalTracer");
final Field tracerField = gt.getDeclaredField("tracer");
tracerField.setAccessible(true);
final Object tracer = tracerField.get(null);
final Method close = tracer.getClass().getMethod("close");
close.invoke(tracer);
} catch (final Exception e) {
}
}
}
@Benchmark
@ -60,54 +28,11 @@ public class ClassRetransformingBenchmark {
state.inst.retransformClasses(TracedClass.class);
}
@Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.2.2.jar")
public static class WithAgent022 extends ClassRetransformingBenchmark {}
@Fork(jvmArgsAppend = "-javaagent:/path/to/dd-java-agent-master.jar")
public static class WithAgentMaster extends ClassRetransformingBenchmark {}
@Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.2.4.jar")
public static class WithAgent024 extends ClassRetransformingBenchmark {}
@Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.2.6.jar")
public static class WithAgent026 extends ClassRetransformingBenchmark {}
@Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.2.7.jar")
public static class WithAgent027 extends ClassRetransformingBenchmark {}
@Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.2.8.jar")
public static class WithAgent028 extends ClassRetransformingBenchmark {}
@Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.2.9.jar")
public static class WithAgent029 extends ClassRetransformingBenchmark {}
@Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.2.10.jar")
public static class WithAgent0210 extends ClassRetransformingBenchmark {}
@Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.2.11.jar")
public static class WithAgent0211 extends ClassRetransformingBenchmark {}
@Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.2.12.jar")
public static class WithAgent0212 extends ClassRetransformingBenchmark {}
@Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.3.0.jar")
public static class WithAgent030 extends ClassRetransformingBenchmark {}
@Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.3.1.jar")
public static class WithAgent031 extends ClassRetransformingBenchmark {}
@Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.3.2.jar")
public static class WithAgent032 extends ClassRetransformingBenchmark {}
@Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.3.3.jar")
public static class WithAgent033 extends ClassRetransformingBenchmark {}
@Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.4.0.jar")
public static class WithAgent040 extends ClassRetransformingBenchmark {}
@Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.4.1.jar")
public static class WithAgent041 extends ClassRetransformingBenchmark {}
@Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.5.0.jar")
public static class WithAgent050 extends ClassRetransformingBenchmark {}
@Fork(jvmArgsAppend = "-javaagent:../build/libs/dd-java-agent.jar")
@Fork(
jvmArgsAppend =
"-javaagent:/path/to/dd-trace-java/dd-java-agent/build/libs/dd-java-agent.jar")
public static class WithAgent extends ClassRetransformingBenchmark {}
}

2
dd-java-agent/benchmark/src/jmh/java/datadog/benchmark/classes/C.java
View File

@ -1,5 +1,5 @@
package datadog.benchmark.classes;
public interface C extends B {
public interface C extends A, B {
void c();
}

2
dd-java-agent/benchmark/src/jmh/java/datadog/benchmark/classes/D.java
View File

@ -1,5 +1,5 @@
package datadog.benchmark.classes;
public interface D extends C {
public interface D extends A, B, C {
void d();
}

2
dd-java-agent/benchmark/src/jmh/java/datadog/benchmark/classes/E.java
View File

@ -1,5 +1,5 @@
package datadog.benchmark.classes;
public interface E extends D {
public interface E extends B, C, D {
void e();
}

1
dd-java-agent/benchmark/src/jmh/resources/dd-trace.yaml
View File

@ -1 +0,0 @@
enableCustomAnnotationTracingOver: ["datadog.benchmark"]

2
dd-trace-ot/dd-trace-ot.gradle
View File

@ -139,7 +139,7 @@ jmh {
// warmupBenchmarks = ['.*Warmup'] // Warmup benchmarks to include in the run in addition to already selected. JMH will not measure these benchmarks, but only use them for the warmup.
// zip64 = true // Use ZIP64 format for bigger archives
jmhVersion = '1.21' // Specifies JMH version
jmhVersion = '1.23' // Specifies JMH version
// includeTests = true // Allows to include test sources into generate JMH jar, i.e. use it when benchmarks depend on the test classes.
duplicateClassesStrategy = 'warn'
// Strategy to apply when encountring duplicate classes during creation of the fat jar (i.e. while executing jmhJar task)