From 950e3910183bc4644fb4cacd28355f4c8f8e3508 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Fri, 10 Feb 2023 14:59:07 -0800
Subject: [PATCH 1/4] e2e: set RUNS to repeat tests

---
 test_e2e.sh | 47 ++++++++++++++++++++++++++++++-----------------
 1 file changed, 30 insertions(+), 17 deletions(-)

diff --git a/test_e2e.sh b/test_e2e.sh
index 17a89e5..9ab8cfc 100755
--- a/test_e2e.sh
+++ b/test_e2e.sh
@@ -2697,29 +2697,42 @@ export GIT_CONFIG_SYSTEM=/dev/null
 
 # Iterate over the chosen tests and run them.
 FAILS=()
-RET=0
+FINAL_RET=0
+RUNS="${RUNS:-1}"
 for t; do
-    clean_root
-    clean_work
-    init_repo
+    TEST_RET=0
+    RUN=0
+    while [[ "${RUN}" < "${RUNS}" ]]; do
+        clean_root
+        clean_work
+        init_repo
 
-    echo -n "testcase $t: "
-
-    # See comments on run_test for details.
-    TESTRET=0
-    run_test TESTRET "e2e::${t}" "${DIR}/log.$t"
-    if [[ "$TESTRET" == 0 ]]; then
-        pass
-    else
-        RET=1
-        if [[ "$TESTRET" != 42 ]]; then
-            echo "FAIL: unknown error"
+        sfx=""
+        if [[ "${RUNS}" > 1 ]]; then
+            sfx=" ($((RUN+1))/${RUNS})"
         fi
+        echo -n "testcase ${t}${sfx}: "
+
+        # See comments on run_test for details.
+        RUN_RET=0
+        run_test RUN_RET "e2e::${t}" "${DIR}/log.$t"
+        if [[ "$RUN_RET" == 0 ]]; then
+            pass
+        else
+            TEST_RET=1
+            if [[ "$RUN_RET" != 42 ]]; then
+                echo "FAIL: unknown error"
+            fi
+        fi
+        remove_containers || true
+        RUN=$((RUN+1))
+    done
+    if [[ "$TEST_RET" != 0 ]]; then
+        FINAL_RET=1
         FAILS+=("$t")
     fi
-    remove_containers || true
 done
-if [[ "$RET" != 0 ]]; then
+if [[ "$FINAL_RET" != 0 ]]; then
     echo
     echo "the following tests failed:"
     for f in "${FAILS[@]}"; do

From 759b8ebdfee9376095983e7a5e20f3e706f036f1 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Fri, 10 Feb 2023 15:06:06 -0800
Subject: [PATCH 2/4] e2e: print path to logs on failure

---
 test_e2e.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/test_e2e.sh b/test_e2e.sh
index 9ab8cfc..6525bc1 100755
--- a/test_e2e.sh
+++ b/test_e2e.sh
@@ -2715,7 +2715,8 @@ for t; do
 
         # See comments on run_test for details.
         RUN_RET=0
-        run_test RUN_RET "e2e::${t}" "${DIR}/log.$t"
+        LOG="${DIR}/log.$t"
+        run_test RUN_RET "e2e::${t}" "${LOG}.${RUN}"
         if [[ "$RUN_RET" == 0 ]]; then
             pass
         else
@@ -2729,7 +2730,7 @@ for t; do
     done
     if [[ "$TEST_RET" != 0 ]]; then
         FINAL_RET=1
-        FAILS+=("$t")
+        FAILS+=("$t  (log: ${LOG}.*)")
     fi
 done
 if [[ "$FINAL_RET" != 0 ]]; then

From 14422f8bacd2ef55067c0c72b5886eb88019f1b7 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Fri, 10 Feb 2023 16:05:38 -0800
Subject: [PATCH 3/4] e2e: set MAXWAIT to change wait times

---
 test_e2e.sh | 131 +++++++++++++++++++++++++++-------------------------
 1 file changed, 69 insertions(+), 62 deletions(-)

diff --git a/test_e2e.sh b/test_e2e.sh
index 6525bc1..0b92663 100755
--- a/test_e2e.sh
+++ b/test_e2e.sh
@@ -154,11 +154,18 @@ function clean_root() {
     mkdir -p "$ROOT"
 }
 
+# How long we wait for sync operations to happen between test steps, in seconds
+MAXWAIT="${MAXWAIT:-3}"
+
 # INTERLOCK is a file, under $ROOT, used to coordinate tests and syncs.
 INTERLOCK="_sync_lock"
 function wait_for_sync() {
+    if [[ -z "$1" ]]; then
+        echo "usage: $0 <max-wait-seconds>"
+        return 1
+    fi
     local path="$ROOT/$INTERLOCK"
-    wait_for_file_exists "$path" $1
+    wait_for_file_exists "$path" "$1"
     rm -f "$path"
 }
 
@@ -472,7 +479,7 @@ function e2e::sync_default_branch() {
         --root="$ROOT" \
         --link="link" \
         >> "$1" 2>&1 &
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
@@ -480,14 +487,14 @@ function e2e::sync_default_branch() {
     # Move forward
     echo "$FUNCNAME 2" > "$REPO"/file
     git -C "$REPO" commit -qam "$FUNCNAME 2"
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 2"
 
     # Move backward
     git -C "$REPO" reset -q --hard HEAD^
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
@@ -509,7 +516,7 @@ function e2e::sync_head() {
         --root="$ROOT" \
         --link="link" \
         >> "$1" 2>&1 &
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
@@ -517,14 +524,14 @@ function e2e::sync_head() {
     # Move HEAD forward
     echo "$FUNCNAME 2" > "$REPO"/file
     git -C "$REPO" commit -qam "$FUNCNAME 2"
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 2"
 
     # Move HEAD backward
     git -C "$REPO" reset -q --hard HEAD^
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
@@ -547,7 +554,7 @@ function e2e::worktree_cleanup() {
         >> "$1" 2>&1 &
 
     # wait for first sync
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME"
@@ -573,7 +580,7 @@ function e2e::worktree_cleanup() {
             docker unpause "$CTR" >/dev/null
         done
 
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_file_exists "$ROOT"/link/file2
     assert_file_eq "$ROOT"/link/file2 "$FUNCNAME-ok"
 }
@@ -594,20 +601,20 @@ function e2e::readlink() {
         --root="$ROOT" \
         --link="link" \
         >> "$1" 2>&1 &
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_link_basename_eq "$ROOT"/link $(git -C "$REPO" rev-parse HEAD)
 
     # Move HEAD forward
     echo "$FUNCNAME 2" > "$REPO"/file
     git -C "$REPO" commit -qam "$FUNCNAME 2"
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_link_basename_eq "$ROOT"/link $(git -C "$REPO" rev-parse HEAD)
 
     # Move HEAD backward
     git -C "$REPO" reset -q --hard HEAD^
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_link_basename_eq "$ROOT"/link $(git -C "$REPO" rev-parse HEAD)
 }
@@ -631,7 +638,7 @@ function e2e::sync_named_branch() {
         --root="$ROOT" \
         --link="link" \
         >> "$1" 2>&1 &
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
@@ -641,7 +648,7 @@ function e2e::sync_named_branch() {
     echo "$FUNCNAME 2" > "$REPO"/file
     git -C "$REPO" commit -qam "$FUNCNAME 2"
     git -C "$REPO" checkout -q "$MAIN_BRANCH"
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 2"
@@ -650,7 +657,7 @@ function e2e::sync_named_branch() {
     git -C "$REPO" checkout -q "$OTHER_BRANCH"
     git -C "$REPO" reset -q --hard HEAD^
     git -C "$REPO" checkout -q "$MAIN_BRANCH"
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
@@ -713,7 +720,7 @@ function e2e::sync_tag() {
         --root="$ROOT" \
         --link="link" \
         >> "$1" 2>&1 &
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
@@ -722,7 +729,7 @@ function e2e::sync_tag() {
     echo "$FUNCNAME 2" > "$REPO"/file
     git -C "$REPO" commit -qam "$FUNCNAME 2"
     git -C "$REPO" tag -f "$TAG" >/dev/null
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 2"
@@ -730,7 +737,7 @@ function e2e::sync_tag() {
     # Move the tag backward
     git -C "$REPO" reset -q --hard HEAD^
     git -C "$REPO" tag -f "$TAG" >/dev/null
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
@@ -763,7 +770,7 @@ function e2e::sync_annotated_tag() {
         --root="$ROOT" \
         --link="link" \
         >> "$1" 2>&1 &
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
@@ -772,7 +779,7 @@ function e2e::sync_annotated_tag() {
     echo "$FUNCNAME 2" > "$REPO"/file
     git -C "$REPO" commit -qam "$FUNCNAME 2"
     git -C "$REPO" tag -af "$TAG" -m "$FUNCNAME 2" >/dev/null
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 2"
@@ -780,7 +787,7 @@ function e2e::sync_annotated_tag() {
     # Move the tag backward
     git -C "$REPO" reset -q --hard HEAD^
     git -C "$REPO" tag -af "$TAG" -m "$FUNCNAME 3" >/dev/null
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
@@ -811,7 +818,7 @@ function e2e::sync_sha() {
         --root="$ROOT" \
         --link="link" \
         >> "$1" 2>&1 &
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
@@ -1050,7 +1057,7 @@ function e2e::sync_slow_git_long_timeout() {
         --root="$ROOT" \
         --link="link" \
         >> "$1" 2>&1 &
-    wait_for_sync 10
+    wait_for_sync "$((MAXWAIT * 3))"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
@@ -1058,7 +1065,7 @@ function e2e::sync_slow_git_long_timeout() {
     # Move forward
     echo "$FUNCNAME 2" > "$REPO"/file
     git -C "$REPO" commit -qam "$FUNCNAME 2"
-    wait_for_sync 10
+    wait_for_sync "$((MAXWAIT * 3))"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 2"
@@ -1081,7 +1088,7 @@ function e2e::sync_branch_depth_shallow() {
         --root="$ROOT" \
         --link="link" \
         >> "$1" 2>&1 &
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
@@ -1093,7 +1100,7 @@ function e2e::sync_branch_depth_shallow() {
     # Move forward
     echo "$FUNCNAME 2" > "$REPO"/file
     git -C "$REPO" commit -qam "$FUNCNAME 2"
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 2"
@@ -1104,7 +1111,7 @@ function e2e::sync_branch_depth_shallow() {
 
     # Move backward
     git -C "$REPO" reset -q --hard HEAD^
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
@@ -1137,7 +1144,7 @@ function e2e::sync_tag_depth_shallow_out_of_range() {
         --root="$ROOT" \
         --link="link" \
         >> "$1" 2>&1 &
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
@@ -1153,7 +1160,7 @@ function e2e::sync_tag_depth_shallow_out_of_range() {
     echo "$FUNCNAME 4" > "$REPO"/file
     git -C "$REPO" commit -qam "$FUNCNAME 4"
     git -C "$REPO" tag -af "$TAG" -m "$FUNCNAME 3" "$SHA" >/dev/null
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 3"
@@ -1182,7 +1189,7 @@ function e2e::sync_fetch_skip_depth_1() {
         >> "$1" 2>&1 &
 
     # wait for first sync which does a clone followed by an artifically slowed fetch
-    wait_for_sync 8
+    wait_for_sync "$((MAXWAIT * 3))"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME"
@@ -1199,7 +1206,7 @@ function e2e::sync_fetch_skip_depth_1() {
     echo "$FUNCNAME-ok" > "$REPO"/file3
     git -C "$REPO" add file3
     git -C "$REPO" commit -qam "$FUNCNAME third file"
-    wait_for_sync 10
+    wait_for_sync "$((MAXWAIT * 3))"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file3
     assert_file_eq "$ROOT"/link/file3 "$FUNCNAME-ok"
@@ -1248,7 +1255,7 @@ function e2e::auth_password_correct_password() {
         --root="$ROOT" \
         --link="link" \
         >> "$1" 2>&1 &
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
@@ -1256,14 +1263,14 @@ function e2e::auth_password_correct_password() {
     # Move HEAD forward
     echo "$FUNCNAME 2" > "$REPO"/file
     git -C "$REPO" commit -qam "$FUNCNAME 2"
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 2"
 
     # Move HEAD backward
     git -C "$REPO" reset -q --hard HEAD^
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
@@ -1334,7 +1341,7 @@ function e2e::auth_askpass_url_correct_password() {
         --root="$ROOT" \
         --link="link" \
         >> "$1" 2>&1 &
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
@@ -1342,14 +1349,14 @@ function e2e::auth_askpass_url_correct_password() {
     # Move HEAD forward
     echo "$FUNCNAME 2" > "$REPO"/file
     git -C "$REPO" commit -qam "$FUNCNAME 2"
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 2"
 
     # Move HEAD backward
     git -C "$REPO" reset -q --hard HEAD^
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
@@ -1395,7 +1402,7 @@ function e2e::auth_askpass_url_flaky() {
         --root="$ROOT" \
         --link="link" \
         >> "$1" 2>&1 &
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
@@ -1403,14 +1410,14 @@ function e2e::auth_askpass_url_flaky() {
     # Move HEAD forward
     echo "$FUNCNAME 2" > "$REPO"/file
     git -C "$REPO" commit -qam "$FUNCNAME 2"
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 2"
 
     # Move HEAD backward
     git -C "$REPO" reset -q --hard HEAD^
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
@@ -1432,7 +1439,7 @@ function e2e::exechook_success() {
         --link="link" \
         --exechook-command="$EXECHOOK_COMMAND" \
         >> "$1" 2>&1 &
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_exists "$ROOT"/link/exechook
@@ -1445,7 +1452,7 @@ function e2e::exechook_success() {
     # Move forward
     echo "$FUNCNAME 2" > "$REPO"/file
     git -C "$REPO" commit -qam "$FUNCNAME 2"
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_exists "$ROOT"/link/exechook
@@ -1502,7 +1509,7 @@ function e2e::exechook_success_once() {
         --exechook-command="$EXECHOOK_COMMAND_SLEEPY" \
         >> "$1" 2>&1
 
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_exists "$ROOT"/link/exechook
@@ -1581,7 +1588,7 @@ function e2e::webhook_success() {
         >> "$1" 2>&1 &
 
     # check that basic call works
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     sleep 1 # webhooks are async
     HITS=$(cat "$HITLOG" | wc -l)
     if [[ "$HITS" < 1 ]]; then
@@ -1594,7 +1601,7 @@ function e2e::webhook_success() {
     git -C "$REPO" commit -qam "$FUNCNAME 2"
 
     # check that another call works
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     sleep 1 # webhooks are async
     HITS=$(cat "$HITLOG" | wc -l)
     if [[ "$HITS" < 1 ]]; then
@@ -1632,7 +1639,7 @@ function e2e::webhook_fail_retry() {
         >> "$1" 2>&1 &
 
     # Check that webhook was called
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     sleep 1 # webhooks are async
     HITS=$(cat "$HITLOG" | wc -l)
     if [[ "$HITS" < 1 ]]; then
@@ -1772,7 +1779,7 @@ function e2e::webhook_fire_and_forget() {
         >> "$1" 2>&1 &
 
     # check that basic call works
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     sleep 1 # webhooks are async
     HITS=$(cat "$HITLOG" | wc -l)
     if [[ "$HITS" < 1 ]]; then
@@ -1817,7 +1824,7 @@ function e2e::expose_http() {
     if [[ $(curl --write-out %{http_code} --silent --output /dev/null http://localhost:$BINDPORT) -ne 503 ]] ; then
         fail "health endpoint should have failed: $(curl --write-out %{http_code} --silent --output /dev/null http://localhost:$BINDPORT)"
     fi
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
 
     # check that health endpoint is alive
     if [[ $(curl --write-out %{http_code} --silent --output /dev/null http://localhost:$BINDPORT) -ne 200 ]] ; then
@@ -1921,7 +1928,7 @@ function e2e::submodule_sync_default() {
         --root="$ROOT" \
         --link="link" \
         >> "$1" 2>&1 &
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_exists "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule
@@ -1932,7 +1939,7 @@ function e2e::submodule_sync_default() {
     git -C "$SUBMODULE" commit -qam "$FUNCNAME 2"
     git -C "$REPO" -c protocol.file.allow=always submodule update --recursive --remote > /dev/null 2>&1
     git -C "$REPO" commit -qam "$FUNCNAME 2"
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_exists "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule
@@ -1942,7 +1949,7 @@ function e2e::submodule_sync_default() {
     git -C "$SUBMODULE" reset -q --hard HEAD^
     git -C "$REPO" -c protocol.file.allow=always submodule update --recursive --remote > /dev/null 2>&1
     git -C "$REPO" commit -qam "$FUNCNAME 3"
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_exists "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule
@@ -1953,7 +1960,7 @@ function e2e::submodule_sync_default() {
     git -C "$SUBMODULE" commit -aqm "add nested submodule"
     git -C "$REPO" -c protocol.file.allow=always submodule update --recursive --remote > /dev/null 2>&1
     git -C "$REPO" commit -qam "$FUNCNAME 4"
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_exists "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule
@@ -1967,7 +1974,7 @@ function e2e::submodule_sync_default() {
     git -C "$SUBMODULE" commit -aqm "delete nested submodule"
     git -C "$REPO" -c protocol.file.allow=always submodule update --recursive --remote > /dev/null 2>&1
     git -C "$REPO" commit -qam "$FUNCNAME 5"
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_exists "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule
@@ -1978,7 +1985,7 @@ function e2e::submodule_sync_default() {
     rm -rf "$REPO"/.git/modules/$SUBMODULE_REPO_NAME
     git -C "$REPO" rm -qf $SUBMODULE_REPO_NAME
     git -C "$REPO" commit -aqm "delete submodule"
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_absent "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule
@@ -2015,7 +2022,7 @@ function e2e::submodule_sync_depth() {
         --root="$ROOT" \
         --link="link" \
         >> "$1" 2>&1 &
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule
     assert_file_eq "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule "$FUNCNAME 1"
@@ -2033,7 +2040,7 @@ function e2e::submodule_sync_depth() {
     git -C "$SUBMODULE" commit -aqm "submodule $FUNCNAME 2"
     git -C "$REPO" -c protocol.file.allow=always submodule update --recursive --remote > /dev/null 2>&1
     git -C "$REPO" commit -qam "$FUNCNAME 2"
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule
     assert_file_eq "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule "$FUNCNAME 2"
@@ -2050,7 +2057,7 @@ function e2e::submodule_sync_depth() {
     git -C "$SUBMODULE" reset -q --hard HEAD^
     git -C "$REPO" -c protocol.file.allow=always submodule update --recursive --remote  > /dev/null 2>&1
     git -C "$REPO" commit -qam "$FUNCNAME 3"
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule
     assert_file_eq "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule "$FUNCNAME 1"
@@ -2091,7 +2098,7 @@ function e2e::submodule_sync_off() {
         --link="link" \
         --submodules=off \
         >> "$1" 2>&1 &
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_file_absent "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule
     rm -rf $SUBMODULE
 }
@@ -2134,7 +2141,7 @@ function e2e::submodule_sync_shallow() {
         --link="link" \
         --submodules=shallow \
         >> "$1" 2>&1 &
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_exists "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule
@@ -2171,7 +2178,7 @@ function e2e::auth_ssh() {
         --ssh \
         --ssh-known-hosts=false \
         >> "$1" 2>&1 &
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
@@ -2179,14 +2186,14 @@ function e2e::auth_ssh() {
     # Move HEAD forward
     echo "$FUNCNAME 2" > "$REPO"/file
     git -C "$REPO" commit -qam "$FUNCNAME 2"
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 2"
 
     # Move HEAD backward
     git -C "$REPO" reset -q --hard HEAD^
-    wait_for_sync 3
+    wait_for_sync "${MAXWAIT}"
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"

From bbd0f56d11ced1b040e7c191b89dedaac830a3cd Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Fri, 10 Feb 2023 21:12:24 -0800
Subject: [PATCH 4/4] e2e: assert metrics in tests

---
 test_e2e.sh | 92 +++++++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 76 insertions(+), 16 deletions(-)

diff --git a/test_e2e.sh b/test_e2e.sh
index 0b92663..4027457 100755
--- a/test_e2e.sh
+++ b/test_e2e.sh
@@ -84,6 +84,17 @@ function assert_file_contains() {
     fail "$1 does not contain '$2': $(cat $1)"
 }
 
+function assert_metric_eq() {
+    local val
+    val="$(curl --silent "http://localhost:$HTTP_PORT/metrics" \
+        | grep "^$1 " \
+        | awk '{print $NF}')"
+    if [[ "${val}" == "$2" ]]; then
+        return
+    fi
+    fail "metric $1 was expected to be '$2': ${val}"
+}
+
 # Helper: run a docker container.
 function docker_run() {
     RM="--rm"
@@ -187,6 +198,8 @@ EXECHOOK_ENVVAL=envval
 RUNLOG="$DIR/runlog"
 rm -f $RUNLOG
 touch $RUNLOG
+HTTP_PORT=9376
+METRIC_GOOD_SYNC_COUNT='git_sync_count_total{status="success"}'
 
 function GIT_SYNC() {
     #./bin/linux_amd64/git-sync "$@"
@@ -219,6 +232,9 @@ function GIT_SYNC() {
             --add-user \
             --touch-file="$INTERLOCK" \
             --git-config='protocol.file.allow:always' \
+            --http-bind=":$HTTP_PORT" \
+            --http-metrics \
+            --http-pprof \
             "$@"
 }
 
@@ -483,6 +499,7 @@ function e2e::sync_default_branch() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1
 
     # Move forward
     echo "$FUNCNAME 2" > "$REPO"/file
@@ -491,6 +508,7 @@ function e2e::sync_default_branch() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 2"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 2
 
     # Move backward
     git -C "$REPO" reset -q --hard HEAD^
@@ -498,6 +516,7 @@ function e2e::sync_default_branch() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 3
 }
 
 ##############################################
@@ -520,6 +539,7 @@ function e2e::sync_head() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1
 
     # Move HEAD forward
     echo "$FUNCNAME 2" > "$REPO"/file
@@ -528,6 +548,7 @@ function e2e::sync_head() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 2"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 2
 
     # Move HEAD backward
     git -C "$REPO" reset -q --hard HEAD^
@@ -535,6 +556,7 @@ function e2e::sync_head() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 3
 }
 
 ##############################################
@@ -558,6 +580,7 @@ function e2e::worktree_cleanup() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1
 
     # suspend time so we can fake corruption
     docker ps --filter label="git-sync-e2e=$RUNID" --format="{{.ID}}" \
@@ -583,6 +606,7 @@ function e2e::worktree_cleanup() {
     wait_for_sync "${MAXWAIT}"
     assert_file_exists "$ROOT"/link/file2
     assert_file_eq "$ROOT"/link/file2 "$FUNCNAME-ok"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 2
 }
 
 ##############################################
@@ -642,6 +666,7 @@ function e2e::sync_named_branch() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1
 
     # Add to the branch.
     git -C "$REPO" checkout -q "$OTHER_BRANCH"
@@ -652,6 +677,7 @@ function e2e::sync_named_branch() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 2"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 2
 
     # Move the branch backward
     git -C "$REPO" checkout -q "$OTHER_BRANCH"
@@ -661,6 +687,7 @@ function e2e::sync_named_branch() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 3
 }
 
 ##############################################
@@ -724,6 +751,7 @@ function e2e::sync_tag() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1
 
     # Add something and move the tag forward
     echo "$FUNCNAME 2" > "$REPO"/file
@@ -733,6 +761,7 @@ function e2e::sync_tag() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 2"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 2
 
     # Move the tag backward
     git -C "$REPO" reset -q --hard HEAD^
@@ -741,6 +770,7 @@ function e2e::sync_tag() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 3
 
     # Add something after the tag
     echo "$FUNCNAME 3" > "$REPO"/file
@@ -749,6 +779,7 @@ function e2e::sync_tag() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 3
 }
 
 ##############################################
@@ -774,6 +805,7 @@ function e2e::sync_annotated_tag() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1
 
     # Add something and move the tag forward
     echo "$FUNCNAME 2" > "$REPO"/file
@@ -783,6 +815,7 @@ function e2e::sync_annotated_tag() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 2"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 2
 
     # Move the tag backward
     git -C "$REPO" reset -q --hard HEAD^
@@ -791,6 +824,7 @@ function e2e::sync_annotated_tag() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 3
 
     # Add something after the tag
     echo "$FUNCNAME 3" > "$REPO"/file
@@ -799,6 +833,7 @@ function e2e::sync_annotated_tag() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 3
 }
 
 ##############################################
@@ -822,6 +857,7 @@ function e2e::sync_sha() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1
 
     # Commit something new
     echo "$FUNCNAME 2" > "$REPO"/file
@@ -830,6 +866,7 @@ function e2e::sync_sha() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1
 
     # Revert the last change
     git -C "$REPO" reset -q --hard HEAD^
@@ -837,6 +874,7 @@ function e2e::sync_sha() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1
 }
 
 ##############################################
@@ -1061,6 +1099,7 @@ function e2e::sync_slow_git_long_timeout() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1
 
     # Move forward
     echo "$FUNCNAME 2" > "$REPO"/file
@@ -1069,6 +1108,7 @@ function e2e::sync_slow_git_long_timeout() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 2"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 2
 }
 
 ##############################################
@@ -1092,6 +1132,7 @@ function e2e::sync_branch_depth_shallow() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1
     depth=$(GIT_DIR="$ROOT"/link/.git git log | grep commit | wc -l)
     if [[ $expected_depth != $depth ]]; then
         fail "initial depth mismatch expected=$expected_depth actual=$depth"
@@ -1104,6 +1145,7 @@ function e2e::sync_branch_depth_shallow() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 2"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 2
     depth=$(GIT_DIR="$ROOT"/link/.git git log | grep commit | wc -l)
     if [[ $expected_depth != $depth ]]; then
         fail "forward depth mismatch expected=$expected_depth actual=$depth"
@@ -1115,6 +1157,7 @@ function e2e::sync_branch_depth_shallow() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 3
     depth=$(GIT_DIR="$ROOT"/link/.git git log | grep commit | wc -l)
     if [[ $expected_depth != $depth ]]; then
         fail "backward depth mismatch expected=$expected_depth actual=$depth"
@@ -1148,6 +1191,7 @@ function e2e::sync_tag_depth_shallow_out_of_range() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1
     depth=$(GIT_DIR="$ROOT"/link/.git git log | grep commit | wc -l)
     if [[ $expected_depth != $depth ]]; then
         fail "initial depth mismatch expected=$expected_depth actual=$depth"
@@ -1164,6 +1208,7 @@ function e2e::sync_tag_depth_shallow_out_of_range() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 3"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 2
     depth=$(GIT_DIR="$ROOT"/link/.git git log | grep commit | wc -l)
     if [[ $expected_depth != $depth ]]; then
         fail "forward depth mismatch expected=$expected_depth actual=$depth"
@@ -1193,6 +1238,7 @@ function e2e::sync_fetch_skip_depth_1() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1
 
     # make a second commit to trigger a sync with shallow fetch
     echo "$FUNCNAME-ok" > "$REPO"/file2
@@ -1210,6 +1256,7 @@ function e2e::sync_fetch_skip_depth_1() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file3
     assert_file_eq "$ROOT"/link/file3 "$FUNCNAME-ok"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 2
 }
 
 ##############################################
@@ -1259,6 +1306,7 @@ function e2e::auth_password_correct_password() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1
 
     # Move HEAD forward
     echo "$FUNCNAME 2" > "$REPO"/file
@@ -1267,6 +1315,7 @@ function e2e::auth_password_correct_password() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 2"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 2
 
     # Move HEAD backward
     git -C "$REPO" reset -q --hard HEAD^
@@ -1274,6 +1323,7 @@ function e2e::auth_password_correct_password() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 3
 }
 
 ##############################################
@@ -1345,6 +1395,7 @@ function e2e::auth_askpass_url_correct_password() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1
 
     # Move HEAD forward
     echo "$FUNCNAME 2" > "$REPO"/file
@@ -1353,6 +1404,7 @@ function e2e::auth_askpass_url_correct_password() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 2"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 2
 
     # Move HEAD backward
     git -C "$REPO" reset -q --hard HEAD^
@@ -1360,6 +1412,7 @@ function e2e::auth_askpass_url_correct_password() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 3
 }
 
 ##############################################
@@ -1406,6 +1459,7 @@ function e2e::auth_askpass_url_flaky() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1
 
     # Move HEAD forward
     echo "$FUNCNAME 2" > "$REPO"/file
@@ -1414,6 +1468,7 @@ function e2e::auth_askpass_url_flaky() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 2"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 2
 
     # Move HEAD backward
     git -C "$REPO" reset -q --hard HEAD^
@@ -1421,6 +1476,7 @@ function e2e::auth_askpass_url_flaky() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 3
 }
 
 ##############################################
@@ -1791,8 +1847,6 @@ function e2e::webhook_fire_and_forget() {
 # Test http handler
 ##############################################
 function e2e::expose_http() {
-    BINDPORT=8888
-
     # First sync
     echo "$FUNCNAME 1" > "$REPO"/file
     git -C "$REPO" commit -qam "$FUNCNAME 1"
@@ -1803,16 +1857,13 @@ function e2e::expose_http() {
         --repo="file://$REPO" \
         --branch="$MAIN_BRANCH" \
         --root="$ROOT" \
-        --http-bind=":$BINDPORT" \
-        --http-metrics \
-        --http-pprof \
         --link="link" \
         >> "$1" 2>&1 &
 
     # do nothing, just wait for the HTTP to come up
     for i in $(seq 1 5); do
         sleep 1
-        if curl --silent --output /dev/null http://localhost:$BINDPORT; then
+        if curl --silent --output /dev/null http://localhost:$HTTP_PORT; then
             break
         fi
         if [[ "$i" == 5 ]]; then
@@ -1821,23 +1872,23 @@ function e2e::expose_http() {
     done
 
     # check that health endpoint fails
-    if [[ $(curl --write-out %{http_code} --silent --output /dev/null http://localhost:$BINDPORT) -ne 503 ]] ; then
-        fail "health endpoint should have failed: $(curl --write-out %{http_code} --silent --output /dev/null http://localhost:$BINDPORT)"
+    if [[ $(curl --write-out %{http_code} --silent --output /dev/null http://localhost:$HTTP_PORT) -ne 503 ]] ; then
+        fail "health endpoint should have failed: $(curl --write-out %{http_code} --silent --output /dev/null http://localhost:$HTTP_PORT)"
     fi
     wait_for_sync "${MAXWAIT}"
 
     # check that health endpoint is alive
-    if [[ $(curl --write-out %{http_code} --silent --output /dev/null http://localhost:$BINDPORT) -ne 200 ]] ; then
+    if [[ $(curl --write-out %{http_code} --silent --output /dev/null http://localhost:$HTTP_PORT) -ne 200 ]] ; then
         fail "health endpoint failed"
     fi
 
     # check that the metrics endpoint exists
-    if [[ $(curl --write-out %{http_code} --silent --output /dev/null http://localhost:$BINDPORT/metrics) -ne 200 ]] ; then
+    if [[ $(curl --write-out %{http_code} --silent --output /dev/null http://localhost:$HTTP_PORT/metrics) -ne 200 ]] ; then
         fail "metrics endpoint failed"
     fi
 
     # check that the pprof endpoint exists
-    if [[ $(curl --write-out %{http_code} --silent --output /dev/null http://localhost:$BINDPORT/debug/pprof/) -ne 200 ]] ; then
+    if [[ $(curl --write-out %{http_code} --silent --output /dev/null http://localhost:$HTTP_PORT/debug/pprof/) -ne 200 ]] ; then
         fail "pprof endpoint failed"
     fi
 }
@@ -1846,8 +1897,6 @@ function e2e::expose_http() {
 # Test http handler after restart
 ##############################################
 function e2e::expose_http_after_restart() {
-    BINDPORT=8888
-
     echo "$FUNCNAME" > "$REPO"/file
     git -C "$REPO" commit -qam "$FUNCNAME 1"
 
@@ -1868,13 +1917,12 @@ function e2e::expose_http_after_restart() {
         --repo="file://$REPO" \
         --branch="$MAIN_BRANCH" \
         --root="$ROOT" \
-        --http-bind=":$BINDPORT" \
         --link="link" \
         >> "$1" 2>&1 &
     # do nothing, just wait for the HTTP to come up
     for i in $(seq 1 5); do
         sleep 1
-        if curl --silent --output /dev/null http://localhost:$BINDPORT; then
+        if curl --silent --output /dev/null http://localhost:$HTTP_PORT; then
             break
         fi
         if [[ "$i" == 5 ]]; then
@@ -1885,7 +1933,7 @@ function e2e::expose_http_after_restart() {
     sleep 2 # wait for first loop to confirm synced
 
     # check that health endpoint is alive
-    if [[ $(curl --write-out %{http_code} --silent --output /dev/null http://localhost:$BINDPORT) -ne 200 ]] ; then
+    if [[ $(curl --write-out %{http_code} --silent --output /dev/null http://localhost:$HTTP_PORT) -ne 200 ]] ; then
         fail "health endpoint failed"
     fi
     assert_link_exists "$ROOT"/link
@@ -1933,6 +1981,7 @@ function e2e::submodule_sync_default() {
     assert_file_exists "$ROOT"/link/file
     assert_file_exists "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule
     assert_file_eq "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule "submodule"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1
 
     # Make change in submodule repo
     echo "$FUNCNAME 2" > "$SUBMODULE"/submodule
@@ -1944,6 +1993,7 @@ function e2e::submodule_sync_default() {
     assert_file_exists "$ROOT"/link/file
     assert_file_exists "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule
     assert_file_eq "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule "$FUNCNAME 2"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 2
 
     # Move backward in submodule repo
     git -C "$SUBMODULE" reset -q --hard HEAD^
@@ -1954,6 +2004,7 @@ function e2e::submodule_sync_default() {
     assert_file_exists "$ROOT"/link/file
     assert_file_exists "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule
     assert_file_eq "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule "submodule"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 3
 
     # Add nested submodule to submodule repo
     git -C "$SUBMODULE" -c protocol.file.allow=always submodule add -q file://$NESTED_SUBMODULE
@@ -1966,6 +2017,7 @@ function e2e::submodule_sync_default() {
     assert_file_exists "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule
     assert_file_exists "$ROOT"/link/$SUBMODULE_REPO_NAME/$NESTED_SUBMODULE_REPO_NAME/nested-submodule
     assert_file_eq "$ROOT"/link/$SUBMODULE_REPO_NAME/$NESTED_SUBMODULE_REPO_NAME/nested-submodule "nested-submodule"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 4
 
     # Remove nested submodule
     git -C "$SUBMODULE" submodule deinit -q $NESTED_SUBMODULE_REPO_NAME
@@ -1979,6 +2031,7 @@ function e2e::submodule_sync_default() {
     assert_file_exists "$ROOT"/link/file
     assert_file_exists "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule
     assert_file_absent "$ROOT"/link/$SUBMODULE_REPO_NAME/$NESTED_SUBMODULE_REPO_NAME/nested-submodule
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 5
 
     # Remove submodule
     git -C "$REPO" submodule deinit -q $SUBMODULE_REPO_NAME
@@ -1989,6 +2042,7 @@ function e2e::submodule_sync_default() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_absent "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 6
 
     rm -rf $SUBMODULE
     rm -rf $NESTED_SUBMODULE
@@ -2026,6 +2080,7 @@ function e2e::submodule_sync_depth() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule
     assert_file_eq "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule "$FUNCNAME 1"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1
     depth=$(GIT_DIR="$ROOT"/link/.git git log | grep commit | wc -l)
     if [[ $expected_depth != $depth ]]; then
         fail "initial depth mismatch expected=$expected_depth actual=$depth"
@@ -2044,6 +2099,7 @@ function e2e::submodule_sync_depth() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule
     assert_file_eq "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule "$FUNCNAME 2"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 2
     depth=$(GIT_DIR="$ROOT"/link/.git git log | grep commit | wc -l)
     if [[ $expected_depth != $depth ]]; then
         fail "forward depth mismatch expected=$expected_depth actual=$depth"
@@ -2061,6 +2117,7 @@ function e2e::submodule_sync_depth() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule
     assert_file_eq "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule "$FUNCNAME 1"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 3
     depth=$(GIT_DIR="$ROOT"/link/.git git log | grep commit | wc -l)
     if [[ $expected_depth != $depth ]]; then
         fail "initial depth mismatch expected=$expected_depth actual=$depth"
@@ -2182,6 +2239,7 @@ function e2e::auth_ssh() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1
 
     # Move HEAD forward
     echo "$FUNCNAME 2" > "$REPO"/file
@@ -2190,6 +2248,7 @@ function e2e::auth_ssh() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 2"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 2
 
     # Move HEAD backward
     git -C "$REPO" reset -q --hard HEAD^
@@ -2197,6 +2256,7 @@ function e2e::auth_ssh() {
     assert_link_exists "$ROOT"/link
     assert_file_exists "$ROOT"/link/file
     assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 3
 }
 
 ##############################################