From daa5333f67217ee4a00f7eb148de06403b8d5a55 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Mon, 11 May 2020 11:21:57 -0700 Subject: [PATCH] Use new go-git functionality to fetch commits This avoids shelling out by using the implementation from https://github.com/go-git/go-git/commit/8ecd388ae101a0dd88b78dc47ebfef9fe51699df, which is going to be much more performant. --- cmd/bashbrew/git.go | 73 ++++++++++++++++++++++----------------------- go.mod | 5 +++- go.sum | 11 +++++-- 3 files changed, 49 insertions(+), 40 deletions(-) diff --git a/cmd/bashbrew/git.go b/cmd/bashbrew/git.go index 67825be..33273e1 100644 --- a/cmd/bashbrew/git.go +++ b/cmd/bashbrew/git.go @@ -180,8 +180,10 @@ func (r Repo) fetchGitRepo(arch string, entry *manifest.Manifest2822Entry) (stri } } - fetchString := entry.ArchGitFetch(arch) + ":" - if entry.ArchGitCommit(arch) == "FETCH_HEAD" { + fetchStrings := []string{ + entry.ArchGitFetch(arch) + ":", + } + if entryArchGitCommit := entry.ArchGitCommit(arch); entryArchGitCommit == "FETCH_HEAD" { // fetch remote tag references to a local tag ref so that we can cache them and not re-fetch every time localRef := "refs/tags/" + gitNormalizeForTagUsage(cacheKey) commit, err := getGitCommit(localRef) @@ -190,7 +192,7 @@ func (r Repo) fetchGitRepo(arch string, entry *manifest.Manifest2822Entry) (stri entry.SetGitCommit(arch, commit) return commit, nil } - fetchString += localRef + fetchStrings[0] += localRef } else { // we create a temporary remote dir so that we can clean it up completely afterwards refBase := "refs/remotes" @@ -210,10 +212,19 @@ func (r Repo) fetchGitRepo(arch string, entry *manifest.Manifest2822Entry) (stri tempRef := path.Join(refBase, filepath.Base(tempRefDir)) if entry.ArchGitFetch(arch) == manifest.DefaultLineBasedFetch { // backwards compat (see manifest/line-based.go in go-dockerlibrary) - fetchString += tempRef + "/*" + fetchStrings[0] += tempRef + "/*" } else { - fetchString += tempRef + "/temp" + fetchStrings[0] += tempRef + "/temp" } + + fetchStrings = append([]string{ + // Git (and more recently, GitHub) support "git fetch"ing a specific commit directly! + // (The "actions/checkout@v2" GitHub action uses this to fetch commits for running workflows even after branches are deleted!) + // https://github.com/git/git/commit/f8edeaa05d8623a9f6dad408237496c51101aad8 + // https://github.com/go-git/go-git/pull/58 + // If that works, we want to prefer it (since it'll be much more efficient at getting us the commit we care about), so we prepend it to our list of "things to try fetching" + entryArchGitCommit + ":" + tempRef + "/temp", + }, fetchStrings...) } if strings.HasPrefix(entry.ArchGitRepo(arch), "git://github.com/") { @@ -229,27 +240,32 @@ func (r Repo) fetchGitRepo(arch string, entry *manifest.Manifest2822Entry) (stri return "", err } - err = gitRemote.Fetch(&goGit.FetchOptions{ - RefSpecs: []goGitConfig.RefSpec{goGitConfig.RefSpec(fetchString)}, - Tags: goGit.NoTags, + var commit string + fetchErrors := []error{} + for _, fetchString := range fetchStrings { + err := gitRemote.Fetch(&goGit.FetchOptions{ + RefSpecs: []goGitConfig.RefSpec{goGitConfig.RefSpec(fetchString)}, + Tags: goGit.NoTags, - //Progress: os.Stdout, - }) - if err != nil { - if fetchErr := fetchGitCommit(arch, entry, gitRemote.Config().URLs[0], fetchString); fetchErr != nil { - return "", cli.NewMultiError(err, fetchErr) + //Progress: os.Stdout, + }) + if err != nil { + fetchErrors = append(fetchErrors, err) + continue } - } - commit, err := getGitCommit(entry.ArchGitCommit(arch)) - if err != nil { - if fetchErr := fetchGitCommit(arch, entry, gitRemote.Config().URLs[0], fetchString); fetchErr != nil { - return "", cli.NewMultiError(err, fetchErr) - } commit, err = getGitCommit(entry.ArchGitCommit(arch)) if err != nil { - return "", err + fetchErrors = append(fetchErrors, err) + continue } + + fetchErrors = nil + break + } + + if len(fetchErrors) > 0 { + return "", cli.NewMultiError(fetchErrors...) } gitTag := gitNormalizeForTagUsage(path.Join(arch, namespace, r.RepoName, entry.Tags[0])) @@ -263,20 +279,3 @@ func (r Repo) fetchGitRepo(arch string, entry *manifest.Manifest2822Entry) (stri entry.SetGitCommit(arch, commit) return commit, nil } - -// this is used as a fallback if using github.com/go-git/go-git/v5 to fetch the branch fails to find the commit -// Git (and more recently, GitHub) support "git fetch"ing a specific commit directly! -// (The "actions/checkout@v2" GitHub action uses this to fetch commits for running workflows even after branches are deleted!) -// https://github.com/git/git/commit/f8edeaa05d8623a9f6dad408237496c51101aad8 -// (Unfortunately, github.com/go-git/go-git/v5 does not support fetching a commit like this from what I can figure [https://github.com/go-git/go-git/issues/56], so we have to shell out.) -func fetchGitCommit(arch string, entry *manifest.Manifest2822Entry, gitRemote, fetchString string) error { - commit := entry.ArchGitCommit(arch) - if commit == "FETCH_HEAD" { - return fmt.Errorf("cannot fetch line-based entry commit when fetching by tag") - } - - fetchString = "+" + commit + ":" + strings.SplitN(fetchString, ":", 2)[1] - - _, err := git(`fetch`, `--quiet`, gitRemote, fetchString) - return err -} diff --git a/go.mod b/go.mod index 4a93e5c..870fe5c 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,10 @@ go 1.11 require ( github.com/codegangsta/cli v1.20.0 github.com/docker-library/go-dockerlibrary v0.0.0-20200415185511-8f28c0fe22db - github.com/go-git/go-git/v5 v5.0.0 + github.com/go-git/go-git/v5 v5.0.1-0.20200510222821-568154cab876 + golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 // indirect + golang.org/x/net v0.0.0-20200506145744-7e3656a0809f // indirect + golang.org/x/sys v0.0.0-20200509044756-6aff5f38e54f // indirect pault.ag/go/debian v0.0.0-20190109175134-a131cb0ae041 pault.ag/go/topsort v0.0.0-20160530003732-f98d2ad46e1a ) diff --git a/go.sum b/go.sum index 7314ccf..46119ae 100644 --- a/go.sum +++ b/go.sum @@ -24,8 +24,8 @@ github.com/go-git/go-billy/v5 v5.0.0 h1:7NQHvd9FVid8VL4qVUMm8XifBK+2xCoZ2lSk0agR github.com/go-git/go-billy/v5 v5.0.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= github.com/go-git/go-git-fixtures/v4 v4.0.1 h1:q+IFMfLx200Q3scvt2hN79JsEzy4AmBTp/pqnefH+Bc= github.com/go-git/go-git-fixtures/v4 v4.0.1/go.mod h1:m+ICp2rF3jDhFgEZ/8yziagdT1C+ZpZcrJjappBCDSw= -github.com/go-git/go-git/v5 v5.0.0 h1:k5RWPm4iJwYtfWoxIJy4wJX9ON7ihPeZZYC1fLYDnpg= -github.com/go-git/go-git/v5 v5.0.0/go.mod h1:oYD8y9kWsGINPFJoLdaScGCN6dlKg23blmClfZwtUVA= +github.com/go-git/go-git/v5 v5.0.1-0.20200510222821-568154cab876 h1:+NeGJnaRbcjcD3wDtc8HOJveibQbgJMKwotDaFPi/w4= +github.com/go-git/go-git/v5 v5.0.1-0.20200510222821-568154cab876/go.mod h1:oYD8y9kWsGINPFJoLdaScGCN6dlKg23blmClfZwtUVA= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= @@ -62,15 +62,22 @@ golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 h1:xMPOj6Pz6UipU1wXLkrtqpHbR0AVFnyPEQq/wRWz9lM= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 h1:cg5LA/zNPRzIXIWSCxQW10Rvpy94aQh3LT/ShoCpkHw= +golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f h1:QBjCr1Fz5kw158VqdE9JfI9cJnl/ymnJWAdMuinqL7Y= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 h1:uYVVQ9WP/Ds2ROhcaGPeIdVq0RIXVLwsHlnvJ+cT1So= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200509044756-6aff5f38e54f h1:mOhmO9WsBaJCNmaZHPtHs9wOcdqdKCjF6OPJlmDM3KI= +golang.org/x/sys v0.0.0-20200509044756-6aff5f38e54f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=