Use docker to run helper servers in e2e

Reliably cleaning up leftover things like `nc` processes is surprisingly
difficult in pure shell.  e2e passes on the master branch now.

Makefile

@@ -181,6 +181,7 @@ test: $(BUILD_DIRS)
 
 test-tools:
 	@docker build -t $(REGISTRY)/test/test-sshd _test_tools/sshd
+	@docker build -t $(REGISTRY)/test/test-ncsvr _test_tools/ncsvr
 
 # Help set up multi-arch build tools.  This assumes you have the tools
 # installed.  If you already have a buildx builder available, you don't need

_test_tools/ncsvr/Dockerfile

@@ -0,0 +1,25 @@
+# Stolen from https://github.com/linuxkit/linuxkit/tree/master/pkg/sshd/
+
+FROM alpine AS base
+
+RUN mkdir -p /out/etc/apk && cp -r /etc/apk/* /out/etc/apk/
+RUN apk add --no-cache --initdb -p /out \
+    alpine-baselayout \
+    apk-tools \
+    busybox \
+    util-linux \
+    netcat-openbsd \
+    && true
+
+###############
+
+FROM scratch
+
+ENTRYPOINT []
+WORKDIR /
+
+COPY --from=base /out/ /
+
+COPY ncsvr.sh /
+
+ENTRYPOINT ["/ncsvr.sh"]

_test_tools/ncsvr/README.md

@@ -0,0 +1,38 @@
+# A simple server for tests
+
+DO NOT USE THIS FOR ANYTHING BUT TESTING!!!
+
+## How to use it
+
+Build yourself a test image.  We use example.com so you can't accidentally push
+it.
+
+```
+$ docker build -t example.com/test/test-ncvsr .
+...lots of output...
+Successfully tagged example.com/test/test-ncsvr:latest
+```
+
+Run it.
+
+```
+$ docker run -d example.com/test/test-ncsvr 9376 "echo hello"
+6d05b4111b03c66907031e3cd7587763f0e4fab6c50fac33c4a8284732b448ae
+```
+
+Find your IP.
+
+```
+$ docker inspect 6d05b4111b03c66907031e3cd7587763f0e4fab6c50fac33c4a8284732b448ae | jq -r .[0].NetworkSettings.IPAddress
+192.168.1.2
+```
+
+Connect to it.
+
+```
+$ echo "" | nc 192.168.9.2 9376
+hello
+```
+
+If you want to know how many times it was accessed, mount a file on
+/var/log/hits.  This will log one line per hit.

_test_tools/ncsvr/ncsvr.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+if [ -z "$1" -o -z "$2" ]; then
+    echo "usage: $0 <port> <shell-command>"
+    exit 1
+fi
+
+while true; do
+    sh -c "$2" | nc -l -p "$1" -N -w0 >/dev/null
+    date >> /var/log/hits
+done

test_e2e.sh

@@ -52,12 +52,31 @@ function assert_file_eq() {
     fail "file $1 does not contain '$2': $(cat $1)"
 }
 
-NCPORT=8888
+# Helper: run a docker container.
-function freencport() {
+function docker_run() {
-  while :; do
+    docker run \
-    NCPORT=$((RANDOM+2000))
+        -d \
-    ss -lpn | grep -q ":$NCPORT " || break
+        --rm \
-  done
+        --label git-sync-e2e="$RUNID" \
+        "$@"
+    sleep 2 # wait for it to come up
+}
+
+# Helper: get the IP of a docker container.
+function docker_ip() {
+    if [ -z "$1" ]; then
+        echo "usage: $0 <id>"
+        return 1
+    fi
+    docker inspect "$1" | jq -r .[0].NetworkSettings.IPAddress
+}
+
+function docker_kill() {
+    if [ -z "$1" ]; then
+        echo "usage: $0 <id>"
+        return 1
+    fi
+    docker kill "$1" >/dev/null
 }
 
 # #####################
@@ -78,7 +97,9 @@ if [[ -z "$DIR" ]]; then
     echo "Failed to make a temp dir"
     exit 1
 fi
+echo
 echo "test root is $DIR"
+echo
 
 REPO="$DIR/repo"
 function init_repo() {
@@ -104,7 +125,8 @@ cat "$DOT_SSH/id_test.pub" > "$DOT_SSH/authorized_keys"
 
 function finish() {
   if [ $? -ne 0 ]; then
-    echo "The directory $DIR was not removed as it contains"\
+    echo
+    echo "the directory $DIR was not removed as it contains"\
          "log files useful for debugging"
   fi
   remove_containers
@@ -641,19 +663,15 @@ pass
 ##############################################
 testcase "askpass_url"
 echo "$TESTCASE 1" > "$REPO"/file
-freencport
 git -C "$REPO" commit -qam "$TESTCASE 1"
 # run the askpass_url service with wrong password
-{ (
+CTR=$(docker_run \
-    for i in 1 2; do
+    e2e/test/test-ncsvr \
-        echo -e 'HTTP/1.1 200 OK\r\n\r\nusername=my-username\npassword=wrong' \
+    80 'echo -e "HTTP/1.1 200 OK\r\n\r\nusername=my-username\npassword=wrong"')
-            | nc -N -l $NCPORT > /dev/null;
+IP=$(docker_ip "$CTR")
-    done
-  ) &
-}
 GIT_SYNC \
     --git="$ASKPASS_GIT" \
-    --askpass-url="http://localhost:$NCPORT/git_askpass" \
+    --askpass-url="http://$IP/git_askpass" \
     --one-time \
     --repo="file://$REPO" \
     --branch=master \
@@ -661,26 +679,26 @@ GIT_SYNC \
     --root="$ROOT" \
     --link="link" \
     > "$DIR"/log."$TESTCASE" 2>&1 || true
+docker_kill "$CTR"
 # check for failure
 assert_file_absent "$ROOT"/link/file
+
 # run with askpass_url service with correct password
-{ (
+CTR=$(docker_run \
-    for i in 1 2; do
+    e2e/test/test-ncsvr \
-        echo -e 'HTTP/1.1 200 OK\r\n\r\nusername=my-username\npassword=my-password' \
+    80 'echo -e "HTTP/1.1 200 OK\r\n\r\nusername=my-username\npassword=my-password"')
-            | nc -N -l $NCPORT > /dev/null;
+IP=$(docker_ip "$CTR")
-    done
-  ) &
-}
 GIT_SYNC \
     --git="$ASKPASS_GIT" \
-    --askpass-url="http://localhost:$NCPORT/git_askpass" \
+    --askpass-url="http://$IP/git_askpass" \
     --one-time \
     --repo="file://$REPO" \
     --branch=master \
     --rev=HEAD \
     --root="$ROOT" \
     --link="link" \
-    > "$DIR"/log."$TESTCASE" 2>&1
+    >> "$DIR"/log."$TESTCASE" 2>&1
+docker_kill "$CTR"
 assert_link_exists "$ROOT"/link
 assert_file_exists "$ROOT"/link/file
 assert_file_eq "$ROOT"/link/file "$TESTCASE 1"
@@ -723,42 +741,86 @@ pass
 # Test webhook success
 ##############################################
 testcase "webhook-success"
-freencport
+HITLOG="$DIR/hitlog.$TESTCASE"
 # First sync
+cat /dev/null > "$HITLOG"
+CTR=$(docker_run \
+    -v "$HITLOG":/var/log/hits \
+    e2e/test/test-ncsvr \
+    80 'echo -e "HTTP/1.1 200 OK\r\n"')
+IP=$(docker_ip "$CTR")
 echo "$TESTCASE 1" > "$REPO"/file
 git -C "$REPO" commit -qam "$TESTCASE 1"
 GIT_SYNC \
     --period=100ms \
     --repo="file://$REPO" \
     --root="$ROOT" \
-    --webhook-url="http://127.0.0.1:$NCPORT" \
+    --webhook-url="http://$IP" \
     --webhook-success-status=200 \
     --link="link" \
     > "$DIR"/log."$TESTCASE" 2>&1 &
 # check that basic call works
-{ (echo -e "HTTP/1.1 200 OK\r\n" | nc -q1 -l $NCPORT > /dev/null) &}
+sleep 2
-NCPID=$!
+HITS=$(cat "$HITLOG" | wc -l)
-sleep 3
+if [ "$HITS" -lt 1 ]; then
-if kill -0 $NCPID > /dev/null 2>&1; then
+    fail "webhook 1 called $HITS times"
-    fail "webhook 1 not called, server still running"
 fi
 # Move forward
+cat /dev/null > "$HITLOG"
 echo "$TESTCASE 2" > "$REPO"/file
 git -C "$REPO" commit -qam "$TESTCASE 2"
-# return a failure to ensure that we try again
+# check that another call works
-{ (echo -e "HTTP/1.1 500 Internal Server Error\r\n" | nc -q1 -l $NCPORT > /dev/null) &}
+sleep 2
-NCPID=$!
+HITS=$(cat "$HITLOG" | wc -l)
-sleep 3
+if [ "$HITS" -lt 1 ]; then
-if kill -0 $NCPID > /dev/null 2>&1; then
+    fail "webhook 2 called $HITS times"
-    fail "webhook 2 not called, server still running"
 fi
+docker_kill "$CTR"
+# Wrap up
+pass
+
+##############################################
+# Test webhook fail-retry
+##############################################
+testcase "webhook-fail-retry"
+HITLOG="$DIR/hitlog.$TESTCASE"
+# First sync - return a failure to ensure that we try again
+cat /dev/null > "$HITLOG"
+CTR=$(docker_run \
+    -v "$HITLOG":/var/log/hits \
+    e2e/test/test-ncsvr \
+    80 'echo -e "HTTP/1.1 500 Internal Server Error\r\n"')
+IP=$(docker_ip "$CTR")
+echo "$TESTCASE 1" > "$REPO"/file
+git -C "$REPO" commit -qam "$TESTCASE 1"
+GIT_SYNC \
+    --period=100ms \
+    --repo="file://$REPO" \
+    --root="$ROOT" \
+    --webhook-url="http://$IP" \
+    --webhook-success-status=200 \
+    --link="link" \
+    > "$DIR"/log."$TESTCASE" 2>&1 &
+# Check that webhook was called
+sleep 2
+HITS=$(cat "$HITLOG" | wc -l)
+if [ "$HITS" -lt 1 ]; then
+    fail "webhook 1 called $HITS times"
+fi
+docker_kill "$CTR"
 # Now return 200, ensure that it gets called
-{ (echo -e "HTTP/1.1 200 OK\r\n" | nc -q1 -l $NCPORT > /dev/null) &}
+cat /dev/null > "$HITLOG"
-NCPID=$!
+CTR=$(docker_run \
-sleep 3
+    --ip="$IP" \
-if kill -0 $NCPID > /dev/null 2>&1; then
+    -v "$HITLOG":/var/log/hits \
-    fail "webhook 3 not called, server still running"
+    e2e/test/test-ncsvr \
+    80 'echo -e "HTTP/1.1 200 OK\r\n"')
+sleep 2
+HITS=$(cat "$HITLOG" | wc -l)
+if [ "$HITS" -lt 1 ]; then
+    fail "webhook 2 called $HITS times"
 fi
+docker_kill "$CTR"
 # Wrap up
 pass
 
@@ -766,7 +828,14 @@ pass
 # Test webhook fire-and-forget
 ##############################################
 testcase "webhook-fire-and-forget"
-freencport
+HITLOG="$DIR/hitlog.$TESTCASE"
+# First sync
+cat /dev/null > "$HITLOG"
+CTR=$(docker_run \
+    -v "$HITLOG":/var/log/hits \
+    e2e/test/test-ncsvr \
+    80 'echo -e "HTTP/1.1 404 Not Found\r\n"')
+IP=$(docker_ip "$CTR")
 # First sync
 echo "$TESTCASE 1" > "$REPO"/file
 git -C "$REPO" commit -qam "$TESTCASE 1"
@@ -774,17 +843,17 @@ GIT_SYNC \
     --period=100ms \
     --repo="file://$REPO" \
     --root="$ROOT" \
-    --webhook-url="http://127.0.0.1:$NCPORT" \
+    --webhook-url="http://$IP" \
     --webhook-success-status=-1 \
     --link="link" \
     > "$DIR"/log."$TESTCASE" 2>&1 &
 # check that basic call works
-{ (echo -e "HTTP/1.1 404 Not Found\r\n" | nc -q1 -l $NCPORT > /dev/null) &}
+sleep 2
-NCPID=$!
+HITS=$(cat "$HITLOG" | wc -l)
-sleep 3
+if [ "$HITS" -lt 1 ]; then
-if kill -0 $NCPID > /dev/null 2>&1; then
+    fail "webhook called $HITS times"
-    fail "webhook 1 not called, server still running"
 fi
+docker_kill "$CTR"
 # Wrap up
 pass
 
@@ -1084,15 +1153,11 @@ pass
 testcase "ssh"
 echo "$TESTCASE" > "$REPO"/file
 # Run a git-over-SSH server
-CTR=$(docker run \
+CTR=$(docker_run \
-    -d \
-    --rm \
-    --label git-sync-e2e="$RUNID" \
     -v "$DOT_SSH":/dot_ssh:ro \
     -v "$REPO":/src:ro \
     e2e/test/test-sshd)
-sleep 3 # wait for sshd to come up
+IP=$(docker_ip "$CTR")
-IP=$(docker inspect "$CTR" | jq -r .[0].NetworkSettings.IPAddress)
 git -C "$REPO" commit -qam "$TESTCASE"
 GIT_SYNC \
     --one-time \
@@ -1110,5 +1175,6 @@ assert_file_eq "$ROOT"/link/file "$TESTCASE"
 # Wrap up
 pass
 
-echo "cleaning up $DIR"
+echo
+echo "all tests passed: cleaning up $DIR"
 rm -rf "$DIR"