Merge pull request #2545 from mrueg/fix-benchmarks

chore: Improve benchmarks

.github/workflows/ci.yml

@@ -164,7 +164,17 @@ jobs:
 
     - name: Benchmark tests
       run: |
-        make test-benchmark-compare
+        BENCHSTAT_OUTPUT_FILE=result.txt make test-benchmark-compare
+    - run: |
+        echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY"
+        cat result.txt >> "$GITHUB_STEP_SUMMARY"
+        echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY"
+        cat <<EOL >> "$GITHUB_STEP_SUMMARY"
+        <hr />
+        The table shows the median and 95% confidence interval (CI) summaries for each benchmark comparing the HEAD and the BASE, and an A/B comparison under "vs base". The last column shows the statistical p-value with ten runs (n=10).
+        The last row has the Geometric Mean (geomean) for the given rows in the table.
+		Refer to <a href="https://pkg.go.dev/golang.org/x/perf/cmd/benchstat">benchstat's documentation</a> for more help.
+        EOL
 
   ci-build-kube-state-metrics:
     name: ci-build-kube-state-metrics

Makefile

@@ -97,8 +97,8 @@ validate-template: generate-template
 # the two.
 test-benchmark-compare:
 	@git fetch
-	./tests/compare_benchmarks.sh main
-	./tests/compare_benchmarks.sh ${LATEST_RELEASE_BRANCH}
+	./tests/compare_benchmarks.sh main 2
+	./tests/compare_benchmarks.sh ${LATEST_RELEASE_BRANCH} 2
 
 all: all-container
 

pkg/app/server_test.go

@@ -19,6 +19,7 @@ package app
 import (
 	"bytes"
 	"context"
+	"flag"
 	"fmt"
 	"io"
 	"net/http/httptest"
@@ -40,6 +41,7 @@ import (
 	"k8s.io/client-go/kubernetes/fake"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/cache"
+	"k8s.io/klog/v2"
 	samplev1alpha1 "k8s.io/sample-controller/pkg/apis/samplecontroller/v1alpha1"
 	samplefake "k8s.io/sample-controller/pkg/generated/clientset/versioned/fake"
 
@@ -70,6 +72,10 @@ func BenchmarkKubeStateMetrics(b *testing.B) {
 		b.Errorf("error injecting resources: %v", err)
 	}
 	ctx, cancel := context.WithCancel(context.Background())
+
+	klogFlags := flag.NewFlagSet("klog", flag.ExitOnError)
+	klog.InitFlags(klogFlags)
+	klogFlags.Set("logtostderr", "false")
 	defer cancel()
 	reg := prometheus.NewRegistry()
 

tests/compare_benchmarks.sh

@@ -6,10 +6,14 @@ set -o pipefail
 # error on unset variables
 set -u
 
-[[ "$#" -eq 1 ]] || echo "One argument required, $# provided."
+[[ "$#" -le 2 ]] || echo "At least one argument required, $# provided."
 
 REF_CURRENT="$(git rev-parse --abbrev-ref HEAD)"
-REF_TO_COMPARE=$1
+REF_TO_COMPARE="${1}"
+
+COUNT=${2:-"1"}
+
+echo "Running benchmarks ${COUNT} time(s)"
 
 RESULT_CURRENT="$(mktemp)-${REF_CURRENT}"
 RESULT_TO_COMPARE="$(mktemp)-${REF_TO_COMPARE}"
@@ -17,7 +21,7 @@ RESULT_TO_COMPARE="$(mktemp)-${REF_TO_COMPARE}"
 echo ""
 echo "### Testing ${REF_CURRENT}"
 
-go test -benchmem -run=NONE -bench=. ./... | tee "${RESULT_CURRENT}"
+go test -benchmem -run=NONE -bench=. -count="${COUNT}" ./... | tee "${RESULT_CURRENT}"
 
 echo ""
 echo "### Done testing ${REF_CURRENT}"
@@ -27,7 +31,7 @@ echo "### Testing ${REF_TO_COMPARE}"
 
 git checkout "${REF_TO_COMPARE}"
 
-go test -benchmem -run=NONE -bench=. ./... | tee "${RESULT_TO_COMPARE}"
+go test -benchmem -run=NONE -bench=. -count="${COUNT}" ./... | tee "${RESULT_TO_COMPARE}"
 
 echo ""
 echo "### Done testing ${REF_TO_COMPARE}"
@@ -38,4 +42,8 @@ echo ""
 echo "### Result"
 echo "old=${REF_TO_COMPARE} new=${REF_CURRENT}"
 
-benchstat "${RESULT_TO_COMPARE}" "${RESULT_CURRENT}"
+if [[ -z "${BENCHSTAT_OUTPUT_FILE}" ]]; then
+	benchstat "${REF_TO_COMPARE}=${RESULT_TO_COMPARE}" "${REF_CURRENT}=${RESULT_CURRENT}"
+else
+	benchstat "${REF_TO_COMPARE}=${RESULT_TO_COMPARE}" "${REF_CURRENT}=${RESULT_CURRENT}" | tee -a "${BENCHSTAT_OUTPUT_FILE}"
+fi