kubernetes
/
git-sync
mirror of https://github.com/kubernetes/git-sync.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add --stale-worktree-timeout option

This commit is contained in:
Sacha Viscaino 2023-05-14 00:28:29 +01:00
parent 4b7b68d34c
commit 622ced3864
2 changed files with 327 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

140
cmd/git-sync/main.go
View File

@ -21,8 +21,10 @@ package main // import "k8s.io/git-sync/cmd/git-sync"
import ( import (
"context" "context"
"crypto/md5" "crypto/md5"
"errors"
"fmt" "fmt"
"io" "io"
"io/fs"
"net" "net"
"net/http" "net/http"
"net/http/pprof" "net/http/pprof"
@ -166,6 +168,9 @@ var flAskPassURL = pflag.String("askpass-url",
envString("", "GITSYNC_ASKPASS_URL", "GIT_SYNC_ASKPASS_URL", "GIT_ASKPASS_URL"), envString("", "GITSYNC_ASKPASS_URL", "GIT_SYNC_ASKPASS_URL", "GIT_ASKPASS_URL"),
"a URL to query for git credentials (username=<value> and password=<value>)") "a URL to query for git credentials (username=<value> and password=<value>)")
var flStaleWorktreeTimeout = pflag.Duration("stale-worktree-timeout", envDuration(0, "GITSYNC_STALE_WORKTREE_TIMEOUT"),
"how long to retain non-current worktrees")
var flGitCmd = pflag.String("git", var flGitCmd = pflag.String("git",
envString("git", "GITSYNC_GIT", "GIT_SYNC_GIT"), envString("git", "GITSYNC_GIT", "GIT_SYNC_GIT"),
"the git command to run (subject to PATH search, mostly for testing)") "the git command to run (subject to PATH search, mostly for testing)")
@ -475,19 +480,20 @@ func (abs absPath) Base() string {
// repoSync represents the remote repo and the local sync of it. // repoSync represents the remote repo and the local sync of it.
type repoSync struct { type repoSync struct {
cmd string // the git command to run cmd string // the git command to run
root absPath // absolute path to the root directory root absPath // absolute path to the root directory
repo string // remote repo to sync repo string // remote repo to sync
ref string // the ref to sync ref string // the ref to sync
depth int // for shallow sync depth int // for shallow sync
submodules submodulesMode // how to handle submodules submodules submodulesMode // how to handle submodules
gc gcMode // garbage collection gc gcMode // garbage collection
link absPath // absolute path to the symlink to publish link absPath // absolute path to the symlink to publish
authURL string // a URL to re-fetch credentials, or "" authURL string // a URL to re-fetch credentials, or ""
sparseFile string // path to a sparse-checkout file sparseFile string // path to a sparse-checkout file
syncCount int // how many times have we synced? syncCount int // how many times have we synced?
log *logging.Logger log *logging.Logger
run cmd.Runner run cmd.Runner
staleTimeout time.Duration // time for worktrees to be cleaned up
} }
func main() { func main() {
@ -791,18 +797,19 @@ func main() {
// Capture the various git parameters. // Capture the various git parameters.
git := &repoSync{ git := &repoSync{
cmd: *flGitCmd, cmd: *flGitCmd,
root: absRoot, root: absRoot,
repo: *flRepo, repo: *flRepo,
ref: *flRef, ref: *flRef,
depth: *flDepth, depth: *flDepth,
submodules: submodulesMode(*flSubmodules), submodules: submodulesMode(*flSubmodules),
gc: gcMode(*flGitGC), gc: gcMode(*flGitGC),
link: absLink, link: absLink,
authURL: *flAskPassURL, authURL: *flAskPassURL,
sparseFile: *flSparseCheckoutFile, sparseFile: *flSparseCheckoutFile,
log: log, log: log,
run: cmdRunner, run: cmdRunner,
staleTimeout: *flStaleWorktreeTimeout,
} }
// This context is used only for git credentials initialization. There are // This context is used only for git credentials initialization. There are
@ -963,6 +970,12 @@ func main() {
start := time.Now() start := time.Now()
ctx, cancel := context.WithTimeout(context.Background(), *flSyncTimeout) ctx, cancel := context.WithTimeout(context.Background(), *flSyncTimeout)
if git.staleTimeout > 0 {
if err := git.cleanupStaleWorktrees(); err != nil {
log.Error(err, "failed to clean up stale worktrees")
}
}
if changed, hash, err := git.SyncRepo(ctx, refreshCreds); err != nil { if changed, hash, err := git.SyncRepo(ctx, refreshCreds); err != nil {
failCount++ failCount++
updateSyncMetrics(metricKeyError, start) updateSyncMetrics(metricKeyError, start)
@ -1068,11 +1081,15 @@ func touch(path absPath) error {
if err := os.MkdirAll(dir, defaultDirMode); err != nil { if err := os.MkdirAll(dir, defaultDirMode); err != nil {
return err return err
} }
file, err := os.Create(path.String()) if err := os.Chtimes(path.String(), time.Now(), time.Now()); errors.Is(err, fs.ErrNotExist) {
if err != nil { file, createErr := os.Create(path.String())
if createErr != nil {
return createErr
}
return file.Close()
} else {
return err return err
} }
return file.Close()
} }
const redactedString = "REDACTED" const redactedString = "REDACTED"
@ -1272,6 +1289,21 @@ func (git *repoSync) initRepo(ctx context.Context) error {
return nil return nil
} }
func (git *repoSync) cleanupStaleWorktrees() error {
currentWorktree, err := git.currentWorktree()
if err != nil {
return err
}
err = removeDirContentsIf(git.worktreeFor("").Path(), git.log, func(fi os.FileInfo) (bool, error) {
// delete files that are over the stale time out, and make sure to never delete the current worktree
return fi.Name() != currentWorktree.Hash() && time.Since(fi.ModTime()) > git.staleTimeout, nil
})
if err != nil {
return err
}
return nil
}
// sanityCheckRepo tries to make sure that the repo dir is a valid git repository. // sanityCheckRepo tries to make sure that the repo dir is a valid git repository.
func (git *repoSync) sanityCheckRepo(ctx context.Context) bool { func (git *repoSync) sanityCheckRepo(ctx context.Context) bool {
git.log.V(3).Info("sanity-checking git repo", "repo", git.root) git.log.V(3).Info("sanity-checking git repo", "repo", git.root)
@ -1340,27 +1372,36 @@ func dirIsEmpty(dir absPath) (bool, error) {
return len(dirents) == 0, nil return len(dirents) == 0, nil
} }
// removeDirContents iterated the specified dir and removes all contents, // removeDirContents iterated the specified dir and removes all contents
// except entries which are specifically excepted. func removeDirContents(dir absPath, log *logging.Logger) error {
func removeDirContents(dir absPath, log *logging.Logger, except ...string) error { return removeDirContentsIf(dir, log, func(fi os.FileInfo) (bool, error) {
return true, nil
})
}
func removeDirContentsIf(dir absPath, log *logging.Logger, fn func(fi os.FileInfo) (bool, error)) error {
dirents, err := os.ReadDir(dir.String()) dirents, err := os.ReadDir(dir.String())
if err != nil { if err != nil {
return err return err
} }
exceptMap := map[string]bool{}
for _, x := range except {
exceptMap[x] = true
}
// Save errors until the end. // Save errors until the end.
var errs multiError var errs multiError
for _, fi := range dirents { for _, fi := range dirents {
name := fi.Name() name := fi.Name()
if exceptMap[name] { p := filepath.Join(dir.String(), name)
stat, err := os.Stat(p)
if err != nil {
log.Error(err, "failed to stat path, skipping", "path", p)
continue
}
if shouldDelete, err := fn(stat); err != nil {
log.Error(err, "failed to evaluate shouldDelete function, skipping", "path", p)
continue
} else if !shouldDelete {
log.V(3).Info("skipping path that should not be removed", "path", p)
continue continue
} }
p := filepath.Join(dir.String(), name)
if log != nil { if log != nil {
log.V(3).Info("removing path recursively", "path", p, "isDir", fi.IsDir()) log.V(3).Info("removing path recursively", "path", p, "isDir", fi.IsDir())
} }
@ -1539,12 +1580,12 @@ func (git *repoSync) configureWorktree(ctx context.Context, worktree worktree) e
// cleanup removes old worktrees and runs git's garbage collection. The // cleanup removes old worktrees and runs git's garbage collection. The
// specified worktree is preserved. // specified worktree is preserved.
func (git *repoSync) cleanup(ctx context.Context, worktree worktree) error { func (git *repoSync) cleanup(ctx context.Context) error {
// Save errors until the end. // Save errors until the end.
var cleanupErrs multiError var cleanupErrs multiError
// Clean up previous worktree(s). // Clean up previous worktree(s).
if err := removeDirContents(git.worktreeFor("").Path(), git.log, worktree.Hash()); err != nil { if err := git.cleanupStaleWorktrees(); err != nil {
cleanupErrs = append(cleanupErrs, err) cleanupErrs = append(cleanupErrs, err)
} }
@ -1639,7 +1680,7 @@ func (git *repoSync) IsKnownHash(ctx context.Context, ref string) (bool, error)
return strings.HasPrefix(line, ref), nil return strings.HasPrefix(line, ref), nil
} }
// worktree represents a git worktree (which may or may not exisat on disk). // worktree represents a git worktree (which may or may not exist on disk).
type worktree absPath type worktree absPath
// Hash returns the intended commit hash for this worktree. // Hash returns the intended commit hash for this worktree.
@ -1781,6 +1822,14 @@ func (git *repoSync) SyncRepo(ctx context.Context, refreshCreds func(context.Con
if err != nil { if err != nil {
return false, "", err return false, "", err
} }
if currentWorktree != "" {
// Touch the old worktree -- which will make cleanupStaleWorktrees delete it in the future,
// if the stale timeout option is enabled
err = touch(currentWorktree.Path())
if err != nil {
git.log.Error(err, "Couldn't change stale worktree mtime", "path", currentWorktree.Path())
}
}
} }
// Mark ourselves as "ready". // Mark ourselves as "ready".
@ -1793,7 +1842,7 @@ func (git *repoSync) SyncRepo(ctx context.Context, refreshCreds func(context.Con
// not get caught by the normal cleanup. // not get caught by the normal cleanup.
os.RemoveAll(currentWorktree.Path().String()) os.RemoveAll(currentWorktree.Path().String())
} }
if err := git.cleanup(ctx, newWorktree); err != nil { if err := git.cleanup(ctx); err != nil {
git.log.Error(err, "git cleanup failed", "newWorktree", newWorktree) git.log.Error(err, "git cleanup failed", "newWorktree", newWorktree)
} }
} }
@ -2412,6 +2461,13 @@ OPTIONS
The known_hosts file to use when --ssh-known-hosts is specified. The known_hosts file to use when --ssh-known-hosts is specified.
If not specified, this defaults to "/etc/git-secret/known_hosts". If not specified, this defaults to "/etc/git-secret/known_hosts".
--stale-worktree-timeout <duration>, $GITSYNC_STALE_WORKTREE_TIMEOUT
The length of time to retain stale (not the current link target)
worktrees before being removed. Once this duration has elapsed,
a stale worktree will be removed during the next sync attempt
(as determined by --sync-timeout). If not specified, this defaults
to 0, meaning that stale worktrees will be removed immediately.
--submodules <string>, $GITSYNC_SUBMODULES --submodules <string>, $GITSYNC_SUBMODULES
The git submodule behavior: one of "recursive", "shallow", or The git submodule behavior: one of "recursive", "shallow", or
"off". If not specified, this defaults to "recursive". "off". If not specified, this defaults to "recursive".

229
test_e2e.sh
View File

@ -591,6 +591,235 @@ function e2e::worktree_cleanup() {
assert_file_absent "$ROOT/.worktrees/not_a_directory" assert_file_absent "$ROOT/.worktrees/not_a_directory"
} }
##############################################
# Test stale-worktree-timeout
##############################################
function e2e::stale_worktree_timeout() {
echo "$FUNCNAME 1" > "$REPO"/file
git -C "$REPO" commit -qam "$FUNCNAME"
WT1=$(git -C "$REPO" rev-list -n1 HEAD)
GIT_SYNC \
--period=100ms \
--repo="file://$REPO" \
--root="$ROOT" \
--link="link" \
--stale-worktree-timeout="5s" \
&
# wait for first sync
wait_for_sync "${MAXWAIT}"
assert_link_exists "$ROOT"/link
assert_file_exists "$ROOT"/link/file
assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
# wait 2 seconds and make another commit
sleep 2
echo "$FUNCNAME 2" > "$REPO"/file2
git -C "$REPO" add file2
git -C "$REPO" commit -qam "$FUNCNAME new file"
WT2=$(git -C "$REPO" rev-list -n1 HEAD)
# wait for second sync
wait_for_sync "${MAXWAIT}"
# at this point both WT1 and WT2 should exist, with
# link pointing to the new WT2
assert_link_exists "$ROOT"/link
assert_file_exists "$ROOT"/link/file
assert_file_exists "$ROOT"/link/file2
assert_file_exists "$ROOT"/.worktrees/$WT1/file
assert_file_absent "$ROOT"/.worktrees/$WT1/file2
# wait 2 seconds and make a third commit
sleep 2
echo "$FUNCNAME 3" > "$REPO"/file3
git -C "$REPO" add file3
git -C "$REPO" commit -qam "$FUNCNAME new file"
WT3=$(git -C "$REPO" rev-list -n1 HEAD)
wait_for_sync "${MAXWAIT}"
# at this point WT1, WT2, WT3 should exist, with
# link pointing to WT3
assert_link_exists "$ROOT"/link
assert_file_exists "$ROOT"/link/file
assert_file_exists "$ROOT"/link/file2
assert_file_exists "$ROOT"/link/file3
assert_file_exists "$ROOT"/.worktrees/$WT1/file
assert_file_absent "$ROOT"/.worktrees/$WT1/file2
assert_file_absent "$ROOT"/.worktrees/$WT1/file3
assert_file_exists "$ROOT"/.worktrees/$WT2/file
assert_file_exists "$ROOT"/.worktrees/$WT2/file2
assert_file_absent "$ROOT"/.worktrees/$WT2/file3
assert_file_exists "$ROOT"/.worktrees/$WT3/file
assert_file_exists "$ROOT"/.worktrees/$WT3/file2
assert_file_exists "$ROOT"/.worktrees/$WT3/file3
# wait for WT1 to go stale
sleep 4
# now WT1 should be stale and deleted,
# WT2 and WT3 should still exist
assert_link_exists "$ROOT"/link
assert_file_exists "$ROOT"/link/file
assert_file_exists "$ROOT"/link/file2
assert_file_exists "$ROOT"/link/file3
assert_file_absent "$ROOT"/.worktrees/$WT1/file
assert_file_absent "$ROOT"/.worktrees/$WT1/file2
assert_file_absent "$ROOT"/.worktrees/$WT1/file3
assert_file_exists "$ROOT"/.worktrees/$WT2/file
assert_file_exists "$ROOT"/.worktrees/$WT2/file2
assert_file_absent "$ROOT"/.worktrees/$WT2/file3
assert_file_exists "$ROOT"/.worktrees/$WT3/file
assert_file_exists "$ROOT"/.worktrees/$WT3/file2
assert_file_exists "$ROOT"/.worktrees/$WT3/file3
# wait for WT2 to go stale
sleep 2
# now both WT1 and WT2 are stale, WT3 should be the only
# worktree left
assert_link_exists "$ROOT"/link
assert_file_exists "$ROOT"/link/file
assert_file_exists "$ROOT"/link/file2
assert_file_exists "$ROOT"/link/file3
assert_file_absent "$ROOT"/.worktrees/$WT1/file
assert_file_absent "$ROOT"/.worktrees/$WT1/file2
assert_file_absent "$ROOT"/.worktrees/$WT1/file3
assert_file_absent "$ROOT"/.worktrees/$WT2/file
assert_file_absent "$ROOT"/.worktrees/$WT2/file2
assert_file_absent "$ROOT"/.worktrees/$WT2/file3
assert_file_exists "$ROOT"/.worktrees/$WT3/file
assert_file_exists "$ROOT"/.worktrees/$WT3/file2
assert_file_exists "$ROOT"/.worktrees/$WT3/file3
}
##############################################
# Test stale-worktree-timeout with restarts
##############################################
function e2e::stale_worktree_timeout_restart() {
echo "$FUNCNAME 1" > "$REPO"/file
git -C "$REPO" commit -qam "$FUNCNAME"
WT1=$(git -C "$REPO" rev-list -n1 HEAD)
GIT_SYNC \
--period=100ms \
--repo="file://$REPO" \
--root="$ROOT" \
--link="link" \
--one-time
assert_link_exists "$ROOT"/link
assert_file_exists "$ROOT"/link/file
assert_file_eq "$ROOT"/link/file "$FUNCNAME 1"
# wait 2 seconds and make another commit
sleep 2
echo "$FUNCNAME 2" > "$REPO"/file2
git -C "$REPO" add file2
git -C "$REPO" commit -qam "$FUNCNAME new file"
WT2=$(git -C "$REPO" rev-list -n1 HEAD)
# restart git-sync
GIT_SYNC \
--period=100ms \
--repo="file://$REPO" \
--root="$ROOT" \
--link="link" \
--stale-worktree-timeout="10s" \
--one-time
# at this point both WT1 and WT2 should exist, with
# link pointing to the new WT2
assert_link_exists "$ROOT"/link
assert_file_exists "$ROOT"/link/file
assert_file_exists "$ROOT"/link/file2
assert_file_exists "$ROOT"/.worktrees/$WT1/file
assert_file_absent "$ROOT"/.worktrees/$WT1/file2
# wait 2 seconds and make a third commit
sleep 4
echo "$FUNCNAME 3" > "$REPO"/file3
git -C "$REPO" add file3
git -C "$REPO" commit -qam "$FUNCNAME new file"
WT3=$(git -C "$REPO" rev-list -n1 HEAD)
# restart git-sync
GIT_SYNC \
--period=100ms \
--repo="file://$REPO" \
--root="$ROOT" \
--link="link" \
--stale-worktree-timeout="10s" \
--one-time
# at this point WT1, WT2, WT3 should exist, with
# link pointing to WT3
assert_link_exists "$ROOT"/link
assert_file_exists "$ROOT"/link/file
assert_file_exists "$ROOT"/link/file2
assert_file_exists "$ROOT"/link/file3
assert_file_exists "$ROOT"/.worktrees/$WT1/file
assert_file_absent "$ROOT"/.worktrees/$WT1/file2
assert_file_absent "$ROOT"/.worktrees/$WT1/file3
assert_file_exists "$ROOT"/.worktrees/$WT2/file
assert_file_exists "$ROOT"/.worktrees/$WT2/file2
assert_file_absent "$ROOT"/.worktrees/$WT2/file3
assert_file_exists "$ROOT"/.worktrees/$WT3/file
assert_file_exists "$ROOT"/.worktrees/$WT3/file2
assert_file_exists "$ROOT"/.worktrees/$WT3/file3
# wait for WT1 to go stale and restart git-sync
sleep 8
GIT_SYNC \
--period=100ms \
--repo="file://$REPO" \
--root="$ROOT" \
--link="link" \
--stale-worktree-timeout="10s" \
--one-time
# now WT1 should be stale and deleted,
# WT2 and WT3 should still exist
assert_link_exists "$ROOT"/link
assert_file_exists "$ROOT"/link/file
assert_file_exists "$ROOT"/link/file2
assert_file_exists "$ROOT"/link/file3
assert_file_absent "$ROOT"/.worktrees/$WT1/file
assert_file_absent "$ROOT"/.worktrees/$WT1/file2
assert_file_absent "$ROOT"/.worktrees/$WT1/file3
assert_file_exists "$ROOT"/.worktrees/$WT2/file
assert_file_exists "$ROOT"/.worktrees/$WT2/file2
assert_file_absent "$ROOT"/.worktrees/$WT2/file3
assert_file_exists "$ROOT"/.worktrees/$WT3/file
assert_file_exists "$ROOT"/.worktrees/$WT3/file2
assert_file_exists "$ROOT"/.worktrees/$WT3/file3
# wait for WT2 to go stale and restart git-sync
sleep 4
GIT_SYNC \
--period=100ms \
--repo="file://$REPO" \
--root="$ROOT" \
--link="link" \
--stale-worktree-timeout="10s" \
--one-time
# now both WT1 and WT2 are stale, WT3 should be the only
# worktree left
assert_link_exists "$ROOT"/link
assert_file_exists "$ROOT"/link/file
assert_file_exists "$ROOT"/link/file2
assert_file_exists "$ROOT"/link/file3
assert_file_absent "$ROOT"/.worktrees/$WT1/file
assert_file_absent "$ROOT"/.worktrees/$WT1/file2
assert_file_absent "$ROOT"/.worktrees/$WT1/file3
assert_file_absent "$ROOT"/.worktrees/$WT2/file
assert_file_absent "$ROOT"/.worktrees/$WT2/file2
assert_file_absent "$ROOT"/.worktrees/$WT2/file3
assert_file_exists "$ROOT"/.worktrees/$WT3/file
assert_file_exists "$ROOT"/.worktrees/$WT3/file2
assert_file_exists "$ROOT"/.worktrees/$WT3/file3
}
############################################## ##############################################
# Test v3->v4 upgrade # Test v3->v4 upgrade
############################################## ##############################################