Overhead benchmarks: Perform a warmup phase. (#3815)

* perform a warmup phase.

* simplify approach to starting JFR recording.

* remove JMX

* use liberica jvm

benchmark-overhead/Dockerfile-petclinic-base

@@ -11,8 +11,7 @@ WORKDIR /app/spring-petclinic-rest
 RUN ./mvnw package -Dmaven.test.skip=true
 RUN cp target/spring-petclinic-rest*.jar /app/spring-petclinic-rest.jar
 
-FROM adoptopenjdk:11-jre
-
+FROM bellsoft/liberica-openjdk-alpine:11
 COPY --from=app-build /app/spring-petclinic-rest.jar /app/spring-petclinic-rest.jar
 WORKDIR /app
 EXPOSE 9966

benchmark-overhead/src/test/java/io/opentelemetry/OverheadTests.java

@@ -30,6 +30,9 @@ import org.junit.jupiter.api.DynamicTest;
 import org.junit.jupiter.api.TestFactory;
 import org.testcontainers.containers.GenericContainer;
 import org.testcontainers.containers.Network;
+import org.testcontainers.containers.startupcheck.OneShotStartupCheckStrategy;
+import org.testcontainers.utility.DockerImageName;
+import org.testcontainers.utility.MountableFile;
 
 public class OverheadTests {
 
@@ -76,6 +79,12 @@ public class OverheadTests {
     petclinic.start();
     writeStartupTimeFile(agent, start);
 
+    if(config.getWarmupSeconds() > 0){
+      doWarmupPhase(config);
+    }
+
+    startRecording(agent, petclinic);
+
     GenericContainer<?> k6 = new K6Container(NETWORK, agent, config, namingConventions).build();
     k6.start();
 
@@ -88,6 +97,28 @@ public class OverheadTests {
     postgres.stop();
   }
 
+  private void startRecording(Agent agent, GenericContainer<?> petclinic) throws Exception {
+    Path outFile = namingConventions.container.jfrFile(agent);
+    String[] command = {"jcmd", "1", "JFR.start", "settings=profile", "dumponexit=true", "name=petclinic", "filename=" + outFile};
+    petclinic.execInContainer(command);
+  }
+
+  private void doWarmupPhase(TestConfig testConfig) {
+    long start = System.currentTimeMillis();
+    System.out.println("Performing startup warming phase for " + testConfig.getWarmupSeconds() + " seconds...");
+    while(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - start) < testConfig.getWarmupSeconds()){
+      GenericContainer<?> k6 = new GenericContainer<>(
+          DockerImageName.parse("loadimpact/k6"))
+          .withNetwork(NETWORK)
+          .withCopyFileToContainer(
+              MountableFile.forHostPath("./k6"), "/app")
+          .withCommand("run", "-u", "5", "-i", "25", "/app/basic.js")
+          .withStartupCheckStrategy(new OneShotStartupCheckStrategy());
+      k6.start();
+    }
+    System.out.println("Warmup complete.");
+  }
+
   private void writeStartupTimeFile(Agent agent, long start) throws IOException {
     long delta = System.currentTimeMillis() - start;
     Path startupPath = namingConventions.local.startupDurationFile(agent);

benchmark-overhead/src/test/java/io/opentelemetry/config/Configs.java

@@ -17,18 +17,21 @@ public enum Configs {
       .name("release")
       .description("compares the latest stable release to no agent")
       .withAgents(Agent.NONE, Agent.LATEST_RELEASE)
+      .warmupSeconds(30)
       .build()
   ),
   SNAPSHOT(TestConfig.builder()
       .name("snapshot")
       .description("compares the latest snapshot to no agent")
       .withAgents(Agent.NONE, Agent.LATEST_SNAPSHOT)
+      .warmupSeconds(30)
       .build()
   ),
   SNAPSHOT_REGRESSION(TestConfig.builder()
       .name("snapshot-regression")
       .description("compares the latest snapshot to the latest stable release")
       .withAgents(Agent.LATEST_RELEASE, Agent.LATEST_SNAPSHOT)
+      .warmupSeconds(30)
       .build()
   )
   ;

benchmark-overhead/src/test/java/io/opentelemetry/config/TestConfig.java

@@ -17,7 +17,7 @@ public class TestConfig {
 
   private final static int DEFAULT_MAX_REQUEST_RATE = 0;  // none
   private final static int DEFAULT_CONCURRENT_CONNECTIONS = 5;
-  private final static int DEFAULT_TOTAL_ITERATIONS = 500;
+  private final static int DEFAULT_TOTAL_ITERATIONS = 5000;
 
   private final String name;
   private final String description;
@@ -25,6 +25,7 @@ public class TestConfig {
   private final int maxRequestRate;
   private final int concurrentConnections;
   private final int totalIterations;
+  private final int warmupSeconds;
 
   public TestConfig(Builder builder) {
     this.name = builder.name;
@@ -33,6 +34,7 @@ public class TestConfig {
     this.maxRequestRate = builder.maxRequestRate;
     this.concurrentConnections = builder.concurrentConnections;
     this.totalIterations = builder.totalIterations;
+    this.warmupSeconds = builder.warmupSeconds;
   }
 
   public String getName() {
@@ -59,7 +61,11 @@ public class TestConfig {
     return totalIterations;
   }
 
-  static Builder builder() {
+  public int getWarmupSeconds() {
+    return warmupSeconds;
+  }
+
+  public static Builder builder() {
     return new Builder();
   }
 
@@ -70,6 +76,7 @@ public class TestConfig {
     private int maxRequestRate = DEFAULT_MAX_REQUEST_RATE;
     private int concurrentConnections = DEFAULT_CONCURRENT_CONNECTIONS;
     private int totalIterations = DEFAULT_TOTAL_ITERATIONS;
+    public int warmupSeconds = 0;
 
     Builder name(String name) {
       this.name = name;
@@ -101,6 +108,11 @@ public class TestConfig {
       return this;
     }
 
+    Builder warmupSeconds(int warmupSeconds){
+      this.warmupSeconds = warmupSeconds;
+      return this;
+    }
+
     TestConfig build(){
       return new TestConfig(this);
     }

benchmark-overhead/src/test/java/io/opentelemetry/containers/PetClinicRestContainer.java

@@ -71,11 +71,8 @@ public class PetClinicRestContainer {
 
   @NotNull
   private String[] buildCommandline(Optional<Path> agentJar) {
-    Path jfrFile = namingConventions.container.jfrFile(this.agent);
     List<String> result = new ArrayList<>(Arrays.asList(
         "java",
-        "-XX:StartFlightRecording:dumponexit=true,disk=true,settings=profile,name=petclinic,filename="
-            + jfrFile,
         "-Dotel.traces.exporter=otlp",
         "-Dotel.imr.export.interval=5000",
         "-Dotel.exporter.otlp.insecure=true",