Merge pull request #413 from barney-s/412

Check for rev to be present after a fetch

cmd/git-sync/main.go

@@ -700,6 +700,14 @@ func addWorktreeAndSwap(ctx context.Context, gitRoot, dest, branch, rev string,
 		return err
 	}
 
+	// With shallow fetches, it's possible to race with the upstream repo and
+	// end up NOT fetching the hash we wanted. If we can't resolve that hash
+	// to a commit we can just end early and leave it for the next sync period.
+	if _, err := revIsHash(ctx, hash, gitRoot); err != nil {
+		log.Error(err, "can't resolve commit, will retry", "rev", rev, "hash", hash)
+		return nil
+	}
+
 	// GC clone
 	if _, err := runCommand(ctx, gitRoot, *flGitCmd, "gc", "--prune=all"); err != nil {
 		return err

slow_git_fetch.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+#
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+if [ "$1" != "fetch" ]; then
+  git "$@"
+  exit $?
+fi
+
+sleep 5
+git "$@"

test_e2e.sh

@@ -161,7 +161,8 @@ function finish() {
 }
 trap finish INT EXIT
 
-SLOW_GIT=/slow_git.sh
+SLOW_GIT_CLONE=/slow_git_clone.sh
+SLOW_GIT_FETCH=/slow_git_fetch.sh
 ASKPASS_GIT=/askpass_git.sh
 SYNC_HOOK_COMMAND=/test_sync_hook_command.sh
 
@@ -174,7 +175,8 @@ function GIT_SYNC() {
         --network="host" \
         -u $(id -u):$(id -g) \
         -v "$DIR":"$DIR":rw \
-        -v "$(pwd)/slow_git.sh":"$SLOW_GIT":ro \
+        -v "$(pwd)/slow_git_clone.sh":"$SLOW_GIT_CLONE":ro \
+        -v "$(pwd)/slow_git_fetch.sh":"$SLOW_GIT_FETCH":ro \
         -v "$(pwd)/askpass_git.sh":"$ASKPASS_GIT":ro \
         -v "$(pwd)/test_sync_hook_command.sh":"$SYNC_HOOK_COMMAND":ro \
         -v "$DOT_SSH/id_test":"/etc/git-secret/ssh":ro \
@@ -609,7 +611,7 @@ testcase "sync-loop-timeout"
 echo "$TESTCASE 1" > "$REPO"/file
 git -C "$REPO" commit -qam "$TESTCASE 1"
 GIT_SYNC \
-    --git="$SLOW_GIT" \
+    --git="$SLOW_GIT_CLONE" \
     --one-time \
     --timeout=1 \
     --repo="file://$REPO" \
@@ -621,7 +623,7 @@ GIT_SYNC \
 assert_file_absent "$ROOT"/link/file
 # run with slow_git but without timing out
 GIT_SYNC \
-    --git="$SLOW_GIT" \
+    --git="$SLOW_GIT_CLONE" \
     --wait=0.1 \
     --timeout=16 \
     --repo="file://$REPO" \
@@ -691,6 +693,48 @@ fi
 # Wrap up
 pass
 
+##############################################
+# Test fetch skipping commit
+##############################################
+testcase "fetch-skip-depth-1"
+echo "$TESTCASE" > "$REPO"/file
+git -C "$REPO" commit -qam "$TESTCASE"
+GIT_SYNC \
+    --git="$SLOW_GIT_FETCH" \
+    --wait=0.1 \
+    --depth=1 \
+    --repo="file://$REPO" \
+    --branch=e2e-branch \
+    --rev=HEAD \
+    --root="$ROOT" \
+    --dest="link" \
+    > "$DIR"/log."$TESTCASE" 2>&1 &
+
+# wait for first sync which does a clone followed by an artifically slowed fetch
+sleep 8
+assert_link_exists "$ROOT"/link
+assert_file_exists "$ROOT"/link/file
+assert_file_eq "$ROOT"/link/file "$TESTCASE"
+
+# make a second commit to trigger a sync with shallow fetch
+echo "$TESTCASE-ok" > "$REPO"/file2
+git -C "$REPO" add file2
+git -C "$REPO" commit -qam "$TESTCASE new file"
+
+# Give time for ls-remote to detect the commit and slowed fetch to start
+sleep 2
+
+# make a third commit to insert the commit between ls-remote and fetch
+echo "$TESTCASE-ok" > "$REPO"/file3
+git -C "$REPO" add file3
+git -C "$REPO" commit -qam "$TESTCASE third file"
+sleep 10
+assert_link_exists "$ROOT"/link
+assert_file_exists "$ROOT"/link/file3
+assert_file_eq "$ROOT"/link/file3 "$TESTCASE-ok"
+
+pass
+
 ##############################################
 # Test password
 ##############################################
@@ -945,7 +989,7 @@ BINDPORT=8888
 echo "$TESTCASE 1" > "$REPO"/file
 git -C "$REPO" commit -qam "$TESTCASE 1"
 GIT_SYNC \
-    --git="$SLOW_GIT" \
+    --git="$SLOW_GIT_CLONE" \
     --repo="file://$REPO" \
     --branch=e2e-branch \
     --root="$ROOT" \