diff --git a/cmd/git-sync/main.go b/cmd/git-sync/main.go index 788c075..d0e6b03 100644 --- a/cmd/git-sync/main.go +++ b/cmd/git-sync/main.go @@ -425,7 +425,10 @@ func main() { os.Exit(1) } - if err := os.MkdirAll(*flRoot, 0700); err != nil { + // Make sure the root exists. 0755 ensures that this is usable as a volume + // when the consumer isn't running as the same UID. We do this very early + // so that we can normalize the path even when there are symlinks in play. + if err := os.MkdirAll(*flRoot, 0755); err != nil { log.Error(err, "ERROR: can't make root dir", "path", *flRoot) os.Exit(1) } @@ -549,6 +552,14 @@ func main() { for { start := time.Now() ctx, cancel := context.WithTimeout(context.Background(), *flSyncTimeout) + if initialSync { + err := git.InitRepo(ctx) + if err != nil { + log.Error(err, "can't init root", absRoot) + os.Exit(1) + } + } + if changed, hash, err := git.SyncRepo(ctx); err != nil { updateSyncMetrics(metricKeyError, start) if *flMaxSyncFailures != -1 && failCount >= *flMaxSyncFailures { @@ -605,6 +616,92 @@ func normalizePath(path string) (string, error) { return abs, nil } +// initRepo looks at the git root and initializes it if needed. This assumes +// the root dir already exists. +func (git *repoSync) InitRepo(ctx context.Context) error { + // Check out the git root, and see if it is already usable. + if _, err := os.Stat(git.root); err != nil { + return err + } + + // Make sure the directory we found is actually usable. + if git.SanityCheck(ctx) { + log.V(0).Info("root directory is valid", "path", git.root) + return nil + } + + // Maybe a previous run crashed? Git won't use this dir. + log.V(0).Info("root directory exists but failed checks, cleaning up", "path", git.root) + + // We remove the contents rather than the dir itself, because a common + // use-case is to have a volume mounted at git.root, which makes removing + // it impossible. + if err := removeDirContents(git.root); err != nil { + return fmt.Errorf("can't remove unusable git root: %w", err) + } + + return nil +} + +// sanityCheck tries to make sure that the dir is a valid git repository. +func (git *repoSync) SanityCheck(ctx context.Context) bool { + log.V(0).Info("sanity-checking git repo", "repo", git.root) + + // If it is empty, we are done. + if empty, err := dirIsEmpty(git.root); err != nil { + log.Error(err, "can't list repo directory", "repo", git.root) + return false + } else if empty { + log.V(0).Info("git repo is empty", "repo", git.root) + return true + } + + // Check that this is actually the root of the repo. + if root, err := runCommand(ctx, git.root, git.cmd, "rev-parse", "--show-toplevel"); err != nil { + log.Error(err, "can't get repo toplevel", "repo", git.root) + return false + } else { + root = strings.TrimSpace(root) + if root != git.root { + log.V(0).Info("git repo is under another repo", "repo", git.root, "parent", root) + return false + } + } + + // Consistency-check the repo. + if _, err := runCommand(ctx, git.root, git.cmd, "fsck", "--no-progress", "--connectivity-only"); err != nil { + log.Error(err, "repo sanity check failed", "repo", git.root) + return false + } + + return true +} + +func dirIsEmpty(dir string) (bool, error) { + dirents, err := ioutil.ReadDir(dir) + if err != nil { + return false, err + } + return len(dirents) == 0, nil +} + +func removeDirContents(dir string) error { + dirents, err := ioutil.ReadDir(dir) + if err != nil { + return err + } + + for _, fi := range dirents { + p := filepath.Join(dir, fi.Name()) + log.V(2).Info("removing path recursively", "path", p, "isDir", fi.IsDir()) + if err := os.RemoveAll(p); err != nil { + return err + } + } + + return nil +} + func updateSyncMetrics(key string, start time.Time) { syncDuration.WithLabelValues(key).Observe(time.Since(start).Seconds()) syncCount.WithLabelValues(key).Inc() diff --git a/cmd/git-sync/main_test.go b/cmd/git-sync/main_test.go index 65df7f7..facc1bd 100644 --- a/cmd/git-sync/main_test.go +++ b/cmd/git-sync/main_test.go @@ -17,7 +17,9 @@ limitations under the License. package main import ( + "io/ioutil" "os" + "path/filepath" "reflect" "strings" "testing" @@ -239,3 +241,123 @@ func TestParseGitConfigs(t *testing.T) { }) } } + +func TestDirIsEmpty(t *testing.T) { + root, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("failed to make a temp dir: %v", err) + } + + // Brand new should be empty. + if empty, err := dirIsEmpty(root); err != nil { + t.Fatalf("unexpected error: %v", err) + } else if !empty { + t.Errorf("expected %q to be deemed empty", root) + } + + // Holding normal files should not be empty. + dir := filepath.Join(root, "files") + if err := os.Mkdir(dir, 0755); err != nil { + t.Fatalf("failed to make a temp subdir: %v", err) + } + for _, file := range []string{"a", "b", "c"} { + path := filepath.Join(dir, file) + if err := ioutil.WriteFile(path, []byte{}, 0755); err != nil { + t.Fatalf("failed to write a file: %v", err) + } + if empty, err := dirIsEmpty(dir); err != nil { + t.Fatalf("unexpected error: %v", err) + } else if empty { + t.Errorf("expected %q to be deemed not-empty", dir) + } + } + + // Holding dot-files should not be empty. + dir = filepath.Join(root, "dot-files") + if err := os.Mkdir(dir, 0755); err != nil { + t.Fatalf("failed to make a temp subdir: %v", err) + } + for _, file := range []string{".a", ".b", ".c"} { + path := filepath.Join(dir, file) + if err := ioutil.WriteFile(path, []byte{}, 0755); err != nil { + t.Fatalf("failed to write a file: %v", err) + } + if empty, err := dirIsEmpty(dir); err != nil { + t.Fatalf("unexpected error: %v", err) + } else if empty { + t.Errorf("expected %q to be deemed not-empty", dir) + } + } + + // Holding dirs should not be empty. + dir = filepath.Join(root, "dirs") + if err := os.Mkdir(dir, 0755); err != nil { + t.Fatalf("failed to make a temp subdir: %v", err) + } + for _, subdir := range []string{"a", "b", "c"} { + path := filepath.Join(dir, subdir) + if err := os.Mkdir(path, 0755); err != nil { + t.Fatalf("failed to make a subdir: %v", err) + } + if empty, err := dirIsEmpty(dir); err != nil { + t.Fatalf("unexpected error: %v", err) + } else if empty { + t.Errorf("expected %q to be deemed not-empty", dir) + } + } + + // Test error path. + if _, err := dirIsEmpty(filepath.Join(root, "does-not-exist")); err == nil { + t.Errorf("unexpected success for non-existent dir") + } +} + +func TestRemoveDirContents(t *testing.T) { + root, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("failed to make a temp dir: %v", err) + } + + // Brand new should be empty. + if empty, err := dirIsEmpty(root); err != nil { + t.Fatalf("unexpected error: %v", err) + } else if !empty { + t.Errorf("expected %q to be deemed empty", root) + } + + // Test removal. + if err := removeDirContents(root); err != nil { + t.Errorf("unexpected error: %v", err) + } + + // Populate the dir. + for _, file := range []string{"f1", "f2", ".f3", ".f4"} { + path := filepath.Join(root, file) + if err := ioutil.WriteFile(path, []byte{}, 0755); err != nil { + t.Fatalf("failed to write a file: %v", err) + } + } + for _, subdir := range []string{"d1", "d2", "d3"} { + path := filepath.Join(root, subdir) + if err := os.Mkdir(path, 0755); err != nil { + t.Fatalf("failed to make a subdir: %v", err) + } + } + + // It should be deemed not-empty + if empty, err := dirIsEmpty(root); err != nil { + t.Fatalf("unexpected error: %v", err) + } else if empty { + t.Errorf("expected %q to be deemed not-empty", root) + } + + // Test removal. + if err := removeDirContents(root); err != nil { + t.Errorf("unexpected error: %v", err) + } + + // Test error path. + if err := removeDirContents(filepath.Join(root, "does-not-exist")); err == nil { + t.Errorf("unexpected success for non-existent dir") + } +} diff --git a/test_e2e.sh b/test_e2e.sh index e37d3f1..660bcae 100755 --- a/test_e2e.sh +++ b/test_e2e.sh @@ -194,9 +194,9 @@ assert_file_eq "$ROOT"/link/file "$TESTCASE" pass ############################################## -# Test HEAD one-time when root exists +# Test HEAD one-time when root exists and is empty ############################################## -testcase "head-once-root-exists" +testcase "head-once-root-exists-empty" echo "$TESTCASE" > "$REPO"/file git -C "$REPO" commit -qam "$TESTCASE" GIT_SYNC \ @@ -281,6 +281,60 @@ ln -s "$ROOT" "$DIR/rootlink" # symlink to test # Wrap up pass +############################################## +# Test HEAD one-time when root is under a git repo +############################################## +testcase "head-once-root-exists-but-is-not-git-root" +echo "$TESTCASE" > "$REPO"/file +git -C "$REPO" commit -qam "$TESTCASE" +# Make a parent dir that is a git repo. +mkdir -p "$ROOT/subdir/root" +date > "$ROOT/subdir/root/file" # so it is not empty +git -C "$ROOT/subdir" init >/dev/null +GIT_SYNC \ + --one-time \ + --repo="file://$REPO" \ + --branch=master \ + --rev=HEAD \ + --root="$ROOT/subdir/root" \ + --link="link" \ + > "$DIR"/log."$TESTCASE" 2>&1 +assert_link_exists "$ROOT"/subdir/root/link +assert_file_exists "$ROOT"/subdir/root/link/file +assert_file_eq "$ROOT"/subdir/root/link/file "$TESTCASE" +# Wrap up +pass + +############################################## +# Test HEAD one-time when root fails sanity +############################################## +testcase "head-once-root-exists-but-fails-sanity" +echo "$TESTCASE" > "$REPO"/file +git -C "$REPO" commit -qam "$TESTCASE" +SHA=$(git -C "$REPO" rev-parse HEAD) +# Make an invalid git repo. +mkdir -p "$ROOT" +git -C "$ROOT" init >/dev/null +echo "ref: refs/heads/nonexist" > "$ROOT/.git/HEAD" +GIT_SYNC \ + --one-time \ + --repo="file://$REPO" \ + --branch=master \ + --rev="HEAD" \ + --root="$ROOT" \ + --link="link" \ + > "$DIR"/log."$TESTCASE" 2>&1 +assert_link_exists "$ROOT"/link +assert_file_exists "$ROOT"/link/file +assert_file_eq "$ROOT"/link/file "$TESTCASE" +# Wrap up +pass + +## FIXME: test when repo is valid git, but wrong remote +## FIXME: test when repo is valid git, but not ar ref we need +## FIXME: test when repo is valid git, and is already correct +exit 42 + ############################################## # Test default syncing (master) ##############################################