diff --git a/test_e2e.sh b/test_e2e.sh index 17a89e5..4027457 100755 --- a/test_e2e.sh +++ b/test_e2e.sh @@ -84,6 +84,17 @@ function assert_file_contains() { fail "$1 does not contain '$2': $(cat $1)" } +function assert_metric_eq() { + local val + val="$(curl --silent "http://localhost:$HTTP_PORT/metrics" \ + | grep "^$1 " \ + | awk '{print $NF}')" + if [[ "${val}" == "$2" ]]; then + return + fi + fail "metric $1 was expected to be '$2': ${val}" +} + # Helper: run a docker container. function docker_run() { RM="--rm" @@ -154,11 +165,18 @@ function clean_root() { mkdir -p "$ROOT" } +# How long we wait for sync operations to happen between test steps, in seconds +MAXWAIT="${MAXWAIT:-3}" + # INTERLOCK is a file, under $ROOT, used to coordinate tests and syncs. INTERLOCK="_sync_lock" function wait_for_sync() { + if [[ -z "$1" ]]; then + echo "usage: $0 " + return 1 + fi local path="$ROOT/$INTERLOCK" - wait_for_file_exists "$path" $1 + wait_for_file_exists "$path" "$1" rm -f "$path" } @@ -180,6 +198,8 @@ EXECHOOK_ENVVAL=envval RUNLOG="$DIR/runlog" rm -f $RUNLOG touch $RUNLOG +HTTP_PORT=9376 +METRIC_GOOD_SYNC_COUNT='git_sync_count_total{status="success"}' function GIT_SYNC() { #./bin/linux_amd64/git-sync "$@" @@ -212,6 +232,9 @@ function GIT_SYNC() { --add-user \ --touch-file="$INTERLOCK" \ --git-config='protocol.file.allow:always' \ + --http-bind=":$HTTP_PORT" \ + --http-metrics \ + --http-pprof \ "$@" } @@ -472,25 +495,28 @@ function e2e::sync_default_branch() { --root="$ROOT" \ --link="link" \ >> "$1" 2>&1 & - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 1" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1 # Move forward echo "$FUNCNAME 2" > "$REPO"/file git -C "$REPO" commit -qam "$FUNCNAME 2" - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 2" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 2 # Move backward git -C "$REPO" reset -q --hard HEAD^ - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 1" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 3 } ############################################## @@ -509,25 +535,28 @@ function e2e::sync_head() { --root="$ROOT" \ --link="link" \ >> "$1" 2>&1 & - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 1" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1 # Move HEAD forward echo "$FUNCNAME 2" > "$REPO"/file git -C "$REPO" commit -qam "$FUNCNAME 2" - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 2" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 2 # Move HEAD backward git -C "$REPO" reset -q --hard HEAD^ - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 1" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 3 } ############################################## @@ -547,10 +576,11 @@ function e2e::worktree_cleanup() { >> "$1" 2>&1 & # wait for first sync - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1 # suspend time so we can fake corruption docker ps --filter label="git-sync-e2e=$RUNID" --format="{{.ID}}" \ @@ -573,9 +603,10 @@ function e2e::worktree_cleanup() { docker unpause "$CTR" >/dev/null done - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_file_exists "$ROOT"/link/file2 assert_file_eq "$ROOT"/link/file2 "$FUNCNAME-ok" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 2 } ############################################## @@ -594,20 +625,20 @@ function e2e::readlink() { --root="$ROOT" \ --link="link" \ >> "$1" 2>&1 & - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_link_basename_eq "$ROOT"/link $(git -C "$REPO" rev-parse HEAD) # Move HEAD forward echo "$FUNCNAME 2" > "$REPO"/file git -C "$REPO" commit -qam "$FUNCNAME 2" - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_link_basename_eq "$ROOT"/link $(git -C "$REPO" rev-parse HEAD) # Move HEAD backward git -C "$REPO" reset -q --hard HEAD^ - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_link_basename_eq "$ROOT"/link $(git -C "$REPO" rev-parse HEAD) } @@ -631,29 +662,32 @@ function e2e::sync_named_branch() { --root="$ROOT" \ --link="link" \ >> "$1" 2>&1 & - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 1" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1 # Add to the branch. git -C "$REPO" checkout -q "$OTHER_BRANCH" echo "$FUNCNAME 2" > "$REPO"/file git -C "$REPO" commit -qam "$FUNCNAME 2" git -C "$REPO" checkout -q "$MAIN_BRANCH" - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 2" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 2 # Move the branch backward git -C "$REPO" checkout -q "$OTHER_BRANCH" git -C "$REPO" reset -q --hard HEAD^ git -C "$REPO" checkout -q "$MAIN_BRANCH" - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 1" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 3 } ############################################## @@ -713,27 +747,30 @@ function e2e::sync_tag() { --root="$ROOT" \ --link="link" \ >> "$1" 2>&1 & - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 1" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1 # Add something and move the tag forward echo "$FUNCNAME 2" > "$REPO"/file git -C "$REPO" commit -qam "$FUNCNAME 2" git -C "$REPO" tag -f "$TAG" >/dev/null - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 2" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 2 # Move the tag backward git -C "$REPO" reset -q --hard HEAD^ git -C "$REPO" tag -f "$TAG" >/dev/null - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 1" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 3 # Add something after the tag echo "$FUNCNAME 3" > "$REPO"/file @@ -742,6 +779,7 @@ function e2e::sync_tag() { assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 1" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 3 } ############################################## @@ -763,27 +801,30 @@ function e2e::sync_annotated_tag() { --root="$ROOT" \ --link="link" \ >> "$1" 2>&1 & - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 1" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1 # Add something and move the tag forward echo "$FUNCNAME 2" > "$REPO"/file git -C "$REPO" commit -qam "$FUNCNAME 2" git -C "$REPO" tag -af "$TAG" -m "$FUNCNAME 2" >/dev/null - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 2" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 2 # Move the tag backward git -C "$REPO" reset -q --hard HEAD^ git -C "$REPO" tag -af "$TAG" -m "$FUNCNAME 3" >/dev/null - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 1" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 3 # Add something after the tag echo "$FUNCNAME 3" > "$REPO"/file @@ -792,6 +833,7 @@ function e2e::sync_annotated_tag() { assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 1" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 3 } ############################################## @@ -811,10 +853,11 @@ function e2e::sync_sha() { --root="$ROOT" \ --link="link" \ >> "$1" 2>&1 & - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 1" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1 # Commit something new echo "$FUNCNAME 2" > "$REPO"/file @@ -823,6 +866,7 @@ function e2e::sync_sha() { assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 1" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1 # Revert the last change git -C "$REPO" reset -q --hard HEAD^ @@ -830,6 +874,7 @@ function e2e::sync_sha() { assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 1" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1 } ############################################## @@ -1050,18 +1095,20 @@ function e2e::sync_slow_git_long_timeout() { --root="$ROOT" \ --link="link" \ >> "$1" 2>&1 & - wait_for_sync 10 + wait_for_sync "$((MAXWAIT * 3))" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 1" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1 # Move forward echo "$FUNCNAME 2" > "$REPO"/file git -C "$REPO" commit -qam "$FUNCNAME 2" - wait_for_sync 10 + wait_for_sync "$((MAXWAIT * 3))" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 2" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 2 } ############################################## @@ -1081,10 +1128,11 @@ function e2e::sync_branch_depth_shallow() { --root="$ROOT" \ --link="link" \ >> "$1" 2>&1 & - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 1" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1 depth=$(GIT_DIR="$ROOT"/link/.git git log | grep commit | wc -l) if [[ $expected_depth != $depth ]]; then fail "initial depth mismatch expected=$expected_depth actual=$depth" @@ -1093,10 +1141,11 @@ function e2e::sync_branch_depth_shallow() { # Move forward echo "$FUNCNAME 2" > "$REPO"/file git -C "$REPO" commit -qam "$FUNCNAME 2" - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 2" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 2 depth=$(GIT_DIR="$ROOT"/link/.git git log | grep commit | wc -l) if [[ $expected_depth != $depth ]]; then fail "forward depth mismatch expected=$expected_depth actual=$depth" @@ -1104,10 +1153,11 @@ function e2e::sync_branch_depth_shallow() { # Move backward git -C "$REPO" reset -q --hard HEAD^ - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 1" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 3 depth=$(GIT_DIR="$ROOT"/link/.git git log | grep commit | wc -l) if [[ $expected_depth != $depth ]]; then fail "backward depth mismatch expected=$expected_depth actual=$depth" @@ -1137,10 +1187,11 @@ function e2e::sync_tag_depth_shallow_out_of_range() { --root="$ROOT" \ --link="link" \ >> "$1" 2>&1 & - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 1" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1 depth=$(GIT_DIR="$ROOT"/link/.git git log | grep commit | wc -l) if [[ $expected_depth != $depth ]]; then fail "initial depth mismatch expected=$expected_depth actual=$depth" @@ -1153,10 +1204,11 @@ function e2e::sync_tag_depth_shallow_out_of_range() { echo "$FUNCNAME 4" > "$REPO"/file git -C "$REPO" commit -qam "$FUNCNAME 4" git -C "$REPO" tag -af "$TAG" -m "$FUNCNAME 3" "$SHA" >/dev/null - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 3" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 2 depth=$(GIT_DIR="$ROOT"/link/.git git log | grep commit | wc -l) if [[ $expected_depth != $depth ]]; then fail "forward depth mismatch expected=$expected_depth actual=$depth" @@ -1182,10 +1234,11 @@ function e2e::sync_fetch_skip_depth_1() { >> "$1" 2>&1 & # wait for first sync which does a clone followed by an artifically slowed fetch - wait_for_sync 8 + wait_for_sync "$((MAXWAIT * 3))" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1 # make a second commit to trigger a sync with shallow fetch echo "$FUNCNAME-ok" > "$REPO"/file2 @@ -1199,10 +1252,11 @@ function e2e::sync_fetch_skip_depth_1() { echo "$FUNCNAME-ok" > "$REPO"/file3 git -C "$REPO" add file3 git -C "$REPO" commit -qam "$FUNCNAME third file" - wait_for_sync 10 + wait_for_sync "$((MAXWAIT * 3))" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file3 assert_file_eq "$ROOT"/link/file3 "$FUNCNAME-ok" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 2 } ############################################## @@ -1248,25 +1302,28 @@ function e2e::auth_password_correct_password() { --root="$ROOT" \ --link="link" \ >> "$1" 2>&1 & - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 1" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1 # Move HEAD forward echo "$FUNCNAME 2" > "$REPO"/file git -C "$REPO" commit -qam "$FUNCNAME 2" - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 2" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 2 # Move HEAD backward git -C "$REPO" reset -q --hard HEAD^ - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 1" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 3 } ############################################## @@ -1334,25 +1391,28 @@ function e2e::auth_askpass_url_correct_password() { --root="$ROOT" \ --link="link" \ >> "$1" 2>&1 & - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 1" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1 # Move HEAD forward echo "$FUNCNAME 2" > "$REPO"/file git -C "$REPO" commit -qam "$FUNCNAME 2" - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 2" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 2 # Move HEAD backward git -C "$REPO" reset -q --hard HEAD^ - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 1" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 3 } ############################################## @@ -1395,25 +1455,28 @@ function e2e::auth_askpass_url_flaky() { --root="$ROOT" \ --link="link" \ >> "$1" 2>&1 & - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 1" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1 # Move HEAD forward echo "$FUNCNAME 2" > "$REPO"/file git -C "$REPO" commit -qam "$FUNCNAME 2" - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 2" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 2 # Move HEAD backward git -C "$REPO" reset -q --hard HEAD^ - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 1" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 3 } ############################################## @@ -1432,7 +1495,7 @@ function e2e::exechook_success() { --link="link" \ --exechook-command="$EXECHOOK_COMMAND" \ >> "$1" 2>&1 & - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_exists "$ROOT"/link/exechook @@ -1445,7 +1508,7 @@ function e2e::exechook_success() { # Move forward echo "$FUNCNAME 2" > "$REPO"/file git -C "$REPO" commit -qam "$FUNCNAME 2" - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_exists "$ROOT"/link/exechook @@ -1502,7 +1565,7 @@ function e2e::exechook_success_once() { --exechook-command="$EXECHOOK_COMMAND_SLEEPY" \ >> "$1" 2>&1 - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_exists "$ROOT"/link/exechook @@ -1581,7 +1644,7 @@ function e2e::webhook_success() { >> "$1" 2>&1 & # check that basic call works - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" sleep 1 # webhooks are async HITS=$(cat "$HITLOG" | wc -l) if [[ "$HITS" < 1 ]]; then @@ -1594,7 +1657,7 @@ function e2e::webhook_success() { git -C "$REPO" commit -qam "$FUNCNAME 2" # check that another call works - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" sleep 1 # webhooks are async HITS=$(cat "$HITLOG" | wc -l) if [[ "$HITS" < 1 ]]; then @@ -1632,7 +1695,7 @@ function e2e::webhook_fail_retry() { >> "$1" 2>&1 & # Check that webhook was called - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" sleep 1 # webhooks are async HITS=$(cat "$HITLOG" | wc -l) if [[ "$HITS" < 1 ]]; then @@ -1772,7 +1835,7 @@ function e2e::webhook_fire_and_forget() { >> "$1" 2>&1 & # check that basic call works - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" sleep 1 # webhooks are async HITS=$(cat "$HITLOG" | wc -l) if [[ "$HITS" < 1 ]]; then @@ -1784,8 +1847,6 @@ function e2e::webhook_fire_and_forget() { # Test http handler ############################################## function e2e::expose_http() { - BINDPORT=8888 - # First sync echo "$FUNCNAME 1" > "$REPO"/file git -C "$REPO" commit -qam "$FUNCNAME 1" @@ -1796,16 +1857,13 @@ function e2e::expose_http() { --repo="file://$REPO" \ --branch="$MAIN_BRANCH" \ --root="$ROOT" \ - --http-bind=":$BINDPORT" \ - --http-metrics \ - --http-pprof \ --link="link" \ >> "$1" 2>&1 & # do nothing, just wait for the HTTP to come up for i in $(seq 1 5); do sleep 1 - if curl --silent --output /dev/null http://localhost:$BINDPORT; then + if curl --silent --output /dev/null http://localhost:$HTTP_PORT; then break fi if [[ "$i" == 5 ]]; then @@ -1814,23 +1872,23 @@ function e2e::expose_http() { done # check that health endpoint fails - if [[ $(curl --write-out %{http_code} --silent --output /dev/null http://localhost:$BINDPORT) -ne 503 ]] ; then - fail "health endpoint should have failed: $(curl --write-out %{http_code} --silent --output /dev/null http://localhost:$BINDPORT)" + if [[ $(curl --write-out %{http_code} --silent --output /dev/null http://localhost:$HTTP_PORT) -ne 503 ]] ; then + fail "health endpoint should have failed: $(curl --write-out %{http_code} --silent --output /dev/null http://localhost:$HTTP_PORT)" fi - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" # check that health endpoint is alive - if [[ $(curl --write-out %{http_code} --silent --output /dev/null http://localhost:$BINDPORT) -ne 200 ]] ; then + if [[ $(curl --write-out %{http_code} --silent --output /dev/null http://localhost:$HTTP_PORT) -ne 200 ]] ; then fail "health endpoint failed" fi # check that the metrics endpoint exists - if [[ $(curl --write-out %{http_code} --silent --output /dev/null http://localhost:$BINDPORT/metrics) -ne 200 ]] ; then + if [[ $(curl --write-out %{http_code} --silent --output /dev/null http://localhost:$HTTP_PORT/metrics) -ne 200 ]] ; then fail "metrics endpoint failed" fi # check that the pprof endpoint exists - if [[ $(curl --write-out %{http_code} --silent --output /dev/null http://localhost:$BINDPORT/debug/pprof/) -ne 200 ]] ; then + if [[ $(curl --write-out %{http_code} --silent --output /dev/null http://localhost:$HTTP_PORT/debug/pprof/) -ne 200 ]] ; then fail "pprof endpoint failed" fi } @@ -1839,8 +1897,6 @@ function e2e::expose_http() { # Test http handler after restart ############################################## function e2e::expose_http_after_restart() { - BINDPORT=8888 - echo "$FUNCNAME" > "$REPO"/file git -C "$REPO" commit -qam "$FUNCNAME 1" @@ -1861,13 +1917,12 @@ function e2e::expose_http_after_restart() { --repo="file://$REPO" \ --branch="$MAIN_BRANCH" \ --root="$ROOT" \ - --http-bind=":$BINDPORT" \ --link="link" \ >> "$1" 2>&1 & # do nothing, just wait for the HTTP to come up for i in $(seq 1 5); do sleep 1 - if curl --silent --output /dev/null http://localhost:$BINDPORT; then + if curl --silent --output /dev/null http://localhost:$HTTP_PORT; then break fi if [[ "$i" == 5 ]]; then @@ -1878,7 +1933,7 @@ function e2e::expose_http_after_restart() { sleep 2 # wait for first loop to confirm synced # check that health endpoint is alive - if [[ $(curl --write-out %{http_code} --silent --output /dev/null http://localhost:$BINDPORT) -ne 200 ]] ; then + if [[ $(curl --write-out %{http_code} --silent --output /dev/null http://localhost:$HTTP_PORT) -ne 200 ]] ; then fail "health endpoint failed" fi assert_link_exists "$ROOT"/link @@ -1921,44 +1976,48 @@ function e2e::submodule_sync_default() { --root="$ROOT" \ --link="link" \ >> "$1" 2>&1 & - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_exists "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule assert_file_eq "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule "submodule" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1 # Make change in submodule repo echo "$FUNCNAME 2" > "$SUBMODULE"/submodule git -C "$SUBMODULE" commit -qam "$FUNCNAME 2" git -C "$REPO" -c protocol.file.allow=always submodule update --recursive --remote > /dev/null 2>&1 git -C "$REPO" commit -qam "$FUNCNAME 2" - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_exists "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule assert_file_eq "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule "$FUNCNAME 2" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 2 # Move backward in submodule repo git -C "$SUBMODULE" reset -q --hard HEAD^ git -C "$REPO" -c protocol.file.allow=always submodule update --recursive --remote > /dev/null 2>&1 git -C "$REPO" commit -qam "$FUNCNAME 3" - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_exists "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule assert_file_eq "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule "submodule" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 3 # Add nested submodule to submodule repo git -C "$SUBMODULE" -c protocol.file.allow=always submodule add -q file://$NESTED_SUBMODULE git -C "$SUBMODULE" commit -aqm "add nested submodule" git -C "$REPO" -c protocol.file.allow=always submodule update --recursive --remote > /dev/null 2>&1 git -C "$REPO" commit -qam "$FUNCNAME 4" - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_exists "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule assert_file_exists "$ROOT"/link/$SUBMODULE_REPO_NAME/$NESTED_SUBMODULE_REPO_NAME/nested-submodule assert_file_eq "$ROOT"/link/$SUBMODULE_REPO_NAME/$NESTED_SUBMODULE_REPO_NAME/nested-submodule "nested-submodule" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 4 # Remove nested submodule git -C "$SUBMODULE" submodule deinit -q $NESTED_SUBMODULE_REPO_NAME @@ -1967,21 +2026,23 @@ function e2e::submodule_sync_default() { git -C "$SUBMODULE" commit -aqm "delete nested submodule" git -C "$REPO" -c protocol.file.allow=always submodule update --recursive --remote > /dev/null 2>&1 git -C "$REPO" commit -qam "$FUNCNAME 5" - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_exists "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule assert_file_absent "$ROOT"/link/$SUBMODULE_REPO_NAME/$NESTED_SUBMODULE_REPO_NAME/nested-submodule + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 5 # Remove submodule git -C "$REPO" submodule deinit -q $SUBMODULE_REPO_NAME rm -rf "$REPO"/.git/modules/$SUBMODULE_REPO_NAME git -C "$REPO" rm -qf $SUBMODULE_REPO_NAME git -C "$REPO" commit -aqm "delete submodule" - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_absent "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 6 rm -rf $SUBMODULE rm -rf $NESTED_SUBMODULE @@ -2015,10 +2076,11 @@ function e2e::submodule_sync_depth() { --root="$ROOT" \ --link="link" \ >> "$1" 2>&1 & - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule assert_file_eq "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule "$FUNCNAME 1" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1 depth=$(GIT_DIR="$ROOT"/link/.git git log | grep commit | wc -l) if [[ $expected_depth != $depth ]]; then fail "initial depth mismatch expected=$expected_depth actual=$depth" @@ -2033,10 +2095,11 @@ function e2e::submodule_sync_depth() { git -C "$SUBMODULE" commit -aqm "submodule $FUNCNAME 2" git -C "$REPO" -c protocol.file.allow=always submodule update --recursive --remote > /dev/null 2>&1 git -C "$REPO" commit -qam "$FUNCNAME 2" - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule assert_file_eq "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule "$FUNCNAME 2" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 2 depth=$(GIT_DIR="$ROOT"/link/.git git log | grep commit | wc -l) if [[ $expected_depth != $depth ]]; then fail "forward depth mismatch expected=$expected_depth actual=$depth" @@ -2050,10 +2113,11 @@ function e2e::submodule_sync_depth() { git -C "$SUBMODULE" reset -q --hard HEAD^ git -C "$REPO" -c protocol.file.allow=always submodule update --recursive --remote > /dev/null 2>&1 git -C "$REPO" commit -qam "$FUNCNAME 3" - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule assert_file_eq "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule "$FUNCNAME 1" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 3 depth=$(GIT_DIR="$ROOT"/link/.git git log | grep commit | wc -l) if [[ $expected_depth != $depth ]]; then fail "initial depth mismatch expected=$expected_depth actual=$depth" @@ -2091,7 +2155,7 @@ function e2e::submodule_sync_off() { --link="link" \ --submodules=off \ >> "$1" 2>&1 & - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_file_absent "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule rm -rf $SUBMODULE } @@ -2134,7 +2198,7 @@ function e2e::submodule_sync_shallow() { --link="link" \ --submodules=shallow \ >> "$1" 2>&1 & - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_exists "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule @@ -2171,25 +2235,28 @@ function e2e::auth_ssh() { --ssh \ --ssh-known-hosts=false \ >> "$1" 2>&1 & - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 1" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1 # Move HEAD forward echo "$FUNCNAME 2" > "$REPO"/file git -C "$REPO" commit -qam "$FUNCNAME 2" - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 2" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 2 # Move HEAD backward git -C "$REPO" reset -q --hard HEAD^ - wait_for_sync 3 + wait_for_sync "${MAXWAIT}" assert_link_exists "$ROOT"/link assert_file_exists "$ROOT"/link/file assert_file_eq "$ROOT"/link/file "$FUNCNAME 1" + assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 3 } ############################################## @@ -2697,29 +2764,43 @@ export GIT_CONFIG_SYSTEM=/dev/null # Iterate over the chosen tests and run them. FAILS=() -RET=0 +FINAL_RET=0 +RUNS="${RUNS:-1}" for t; do - clean_root - clean_work - init_repo + TEST_RET=0 + RUN=0 + while [[ "${RUN}" < "${RUNS}" ]]; do + clean_root + clean_work + init_repo - echo -n "testcase $t: " - - # See comments on run_test for details. - TESTRET=0 - run_test TESTRET "e2e::${t}" "${DIR}/log.$t" - if [[ "$TESTRET" == 0 ]]; then - pass - else - RET=1 - if [[ "$TESTRET" != 42 ]]; then - echo "FAIL: unknown error" + sfx="" + if [[ "${RUNS}" > 1 ]]; then + sfx=" ($((RUN+1))/${RUNS})" fi - FAILS+=("$t") + echo -n "testcase ${t}${sfx}: " + + # See comments on run_test for details. + RUN_RET=0 + LOG="${DIR}/log.$t" + run_test RUN_RET "e2e::${t}" "${LOG}.${RUN}" + if [[ "$RUN_RET" == 0 ]]; then + pass + else + TEST_RET=1 + if [[ "$RUN_RET" != 42 ]]; then + echo "FAIL: unknown error" + fi + fi + remove_containers || true + RUN=$((RUN+1)) + done + if [[ "$TEST_RET" != 0 ]]; then + FINAL_RET=1 + FAILS+=("$t (log: ${LOG}.*)") fi - remove_containers || true done -if [[ "$RET" != 0 ]]; then +if [[ "$FINAL_RET" != 0 ]]; then echo echo "the following tests failed:" for f in "${FAILS[@]}"; do