fluxcd
/
source-controller
mirror of https://github.com/fluxcd/source-controller.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #524 from racdev/libgit2-proxy-support

This commit is contained in:
Hidde Beydals 2022-01-19 12:00:34 +01:00 committed by GitHub
parent be575864e1 3d276b679b
commit 3ca05e1986
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 300 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
go.mod
View File

@ -9,6 +9,7 @@ require (
github.com/Masterminds/semver/v3 v3.1.1
github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7
github.com/cyphar/filepath-securejoin v0.2.2
github.com/elazarl/goproxy v0.0.0-20211114080932-d06c3be7c11b
github.com/fluxcd/pkg/apis/meta v0.10.2
github.com/fluxcd/pkg/gittestserver v0.5.0
github.com/fluxcd/pkg/gitutil v0.1.0

6
go.sum
View File

@ -267,8 +267,11 @@ github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3
github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
github.com/elazarl/goproxy v0.0.0-20211114080932-d06c3be7c11b h1:1XqENn2YoYZd6w3Awx+7oa+aR87DFIZJFLF2n1IojA0=
github.com/elazarl/goproxy v0.0.0-20211114080932-d06c3be7c11b/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 h1:dWB6v3RcOy03t/bUadywsbyrQwCqZeNIEX6M1OtSZOM=
github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
@ -806,6 +809,7 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=

4
pkg/git/libgit2/checkout.go
View File

@ -64,6 +64,7 @@ func (c *CheckoutBranch) Checkout(ctx context.Context, path, url string, opts *g
FetchOptions: &git2go.FetchOptions{
DownloadTags: git2go.DownloadTagsNone,
RemoteCallbacks: RemoteCallbacks(ctx, opts),
ProxyOptions: git2go.ProxyOptions{Type: git2go.ProxyTypeAuto},
},
CheckoutBranch: c.Branch,
})
@ -93,6 +94,7 @@ func (c *CheckoutTag) Checkout(ctx context.Context, path, url string, opts *git.
FetchOptions: &git2go.FetchOptions{
DownloadTags: git2go.DownloadTagsAll,
RemoteCallbacks: RemoteCallbacks(ctx, opts),
ProxyOptions: git2go.ProxyOptions{Type: git2go.ProxyTypeAuto},
},
})
if err != nil {
@ -116,6 +118,7 @@ func (c *CheckoutCommit) Checkout(ctx context.Context, path, url string, opts *g
FetchOptions: &git2go.FetchOptions{
DownloadTags: git2go.DownloadTagsNone,
RemoteCallbacks: RemoteCallbacks(ctx, opts),
ProxyOptions: git2go.ProxyOptions{Type: git2go.ProxyTypeAuto},
},
})
if err != nil {
@ -147,6 +150,7 @@ func (c *CheckoutSemVer) Checkout(ctx context.Context, path, url string, opts *g
FetchOptions: &git2go.FetchOptions{
DownloadTags: git2go.DownloadTagsAll,
RemoteCallbacks: RemoteCallbacks(ctx, opts),
ProxyOptions: git2go.ProxyOptions{Type: git2go.ProxyTypeAuto},
},
})
if err != nil {

290
pkg/git/strategy/proxy/strategy_proxy_test.go Normal file
View File

@ -0,0 +1,290 @@
/*
Copyright 2021 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package proxy
import (
"context"
"fmt"
"net"
"net/http"
"net/url"
"os"
"strings"
"testing"
"time"
"github.com/elazarl/goproxy"
"github.com/fluxcd/pkg/gittestserver"
. "github.com/onsi/gomega"
"github.com/fluxcd/source-controller/pkg/git"
"github.com/fluxcd/source-controller/pkg/git/gogit"
"github.com/fluxcd/source-controller/pkg/git/libgit2"
"github.com/fluxcd/source-controller/pkg/git/strategy"
)
// These tests are run in a different _test.go file because go-git uses the ProxyFromEnvironment function of the net/http package
// which caches the Proxy settings, hence not including other tests in the same file ensures a clean proxy setup for the tests to run.
func TestCheckoutStrategyForImplementation_Proxied(t *testing.T) {
type cleanupFunc func()
type testCase struct {
name string
gitImpl git.Implementation
url string
branch string
setupGitProxy func(g *WithT, proxy *goproxy.ProxyHttpServer, proxyGotRequest *bool) (*git.AuthOptions, cleanupFunc)
shortTimeout bool
wantUsedProxy bool
wantError bool
}
g := NewWithT(t)
// Get a free port for proxy to use.
l, err := net.Listen("tcp", ":0")
g.Expect(err).ToNot(HaveOccurred())
proxyAddr := fmt.Sprintf("localhost:%d", l.Addr().(*net.TCPAddr).Port)
g.Expect(l.Close()).ToNot(HaveOccurred())
// Note there is no libgit2 HTTP_PROXY test as libgit2 doesnt support proxied HTTP requests.
cases := []testCase{
{
name: "libgit2_HTTPS_PROXY",
gitImpl: libgit2.Implementation,
url: "https://example.com/bar/test-reponame",
branch: "main",
setupGitProxy: func(g *WithT, proxy *goproxy.ProxyHttpServer, proxyGotRequest *bool) (*git.AuthOptions, cleanupFunc) {
// Create the git server.
gitServer, err := gittestserver.NewTempGitServer()
g.Expect(err).ToNot(HaveOccurred())
username := "test-user"
password := "test-password"
gitServer.Auth(username, password)
gitServer.KeyDir(gitServer.Root())
// Start the HTTPS server.
examplePublicKey, err := os.ReadFile("../testdata/certs/server.pem")
g.Expect(err).ToNot(HaveOccurred())
examplePrivateKey, err := os.ReadFile("../testdata/certs/server-key.pem")
g.Expect(err).ToNot(HaveOccurred())
exampleCA, err := os.ReadFile("../testdata/certs/ca.pem")
g.Expect(err).ToNot(HaveOccurred())
err = gitServer.StartHTTPS(examplePublicKey, examplePrivateKey, exampleCA, "example.com")
g.Expect(err).ToNot(HaveOccurred())
// Initialize a git repo.
repoPath := "bar/test-reponame"
err = gitServer.InitRepo("../testdata/repo1", "main", repoPath)
g.Expect(err).ToNot(HaveOccurred())
u, err := url.Parse(gitServer.HTTPAddress())
g.Expect(err).ToNot(HaveOccurred())
// The request is being forwarded to the local test git server in this handler.
// The certificate used here is valid for both example.com and localhost.
var proxyHandler goproxy.FuncHttpsHandler = func(host string, ctx *goproxy.ProxyCtx) (*goproxy.ConnectAction, string) {
// Check if the host matches with the git server address and the user-agent is the expected git client.
userAgent := ctx.Req.Header.Get("User-Agent")
if strings.Contains(host, "example.com") && strings.Contains(userAgent, "libgit2") {
*proxyGotRequest = true
return goproxy.OkConnect, u.Host
}
// Reject if it isn't our request.
return goproxy.RejectConnect, host
}
proxy.OnRequest().HandleConnect(proxyHandler)
return &git.AuthOptions{
Transport: git.HTTPS,
Username: username,
Password: password,
CAFile: exampleCA,
}, func() {
os.RemoveAll(gitServer.Root())
gitServer.StopHTTP()
}
},
shortTimeout: false,
wantUsedProxy: true,
wantError: false,
},
{
name: "gogit_HTTP_PROXY",
gitImpl: gogit.Implementation,
url: "http://example.com/bar/test-reponame",
branch: "main",
setupGitProxy: func(g *WithT, proxy *goproxy.ProxyHttpServer, proxyGotRequest *bool) (*git.AuthOptions, cleanupFunc) {
// Create the git server.
gitServer, err := gittestserver.NewTempGitServer()
g.Expect(err).ToNot(HaveOccurred())
username := "test-user"
password := "test-password"
gitServer.Auth(username, password)
gitServer.KeyDir(gitServer.Root())
g.Expect(gitServer.StartHTTP()).ToNot(HaveOccurred())
// Initialize a git repo.
err = gitServer.InitRepo("../testdata/repo1", "main", "bar/test-reponame")
g.Expect(err).ToNot(HaveOccurred())
u, err := url.Parse(gitServer.HTTPAddress())
g.Expect(err).ToNot(HaveOccurred())
// The request is being forwarded to the local test git server in this handler.
var proxyHandler goproxy.FuncReqHandler = func(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
userAgent := req.Header.Get("User-Agent")
if strings.Contains(req.Host, "example.com") && strings.Contains(userAgent, "git") {
*proxyGotRequest = true
req.Host = u.Host
req.URL.Host = req.Host
return req, nil
}
// Reject if it isnt our request.
return req, goproxy.NewResponse(req, goproxy.ContentTypeText, http.StatusForbidden, "")
}
proxy.OnRequest().Do(proxyHandler)
return &git.AuthOptions{
Transport: git.HTTP,
Username: username,
Password: password,
}, func() {
os.RemoveAll(gitServer.Root())
gitServer.StopHTTP()
}
},
shortTimeout: false,
wantUsedProxy: true,
wantError: false,
},
{
name: "gogit_HTTPS_PROXY",
gitImpl: gogit.Implementation,
url: "https://github.com/git-fixtures/basic",
branch: "master",
setupGitProxy: func(g *WithT, proxy *goproxy.ProxyHttpServer, proxyGotRequest *bool) (*git.AuthOptions, cleanupFunc) {
var proxyHandler goproxy.FuncHttpsHandler = func(host string, ctx *goproxy.ProxyCtx) (*goproxy.ConnectAction, string) {
// We don't check for user agent as this handler is only going to process CONNECT requests, and because Go's net/http
// is the one making such a request on behalf of go-git, adding a check for the go net/http user agent (Go-http-client)
// would only allow false positives from any request originating from Go's net/http.
if strings.Contains(host, "github.com") {
*proxyGotRequest = true
return goproxy.OkConnect, host
}
// Reject if it isnt our request.
return goproxy.RejectConnect, host
}
proxy.OnRequest().HandleConnect(proxyHandler)
// go-git does not allow to use an HTTPS proxy and a custom root CA at the same time.
// See https://github.com/fluxcd/source-controller/pull/524#issuecomment-1006673163.
return nil, func() {}
},
shortTimeout: false,
wantUsedProxy: true,
wantError: false,
},
{
name: "gogit_NO_PROXY",
gitImpl: gogit.Implementation,
url: "https://192.0.2.1/bar/test-reponame",
branch: "main",
setupGitProxy: func(g *WithT, proxy *goproxy.ProxyHttpServer, proxyGotRequest *bool) (*git.AuthOptions, cleanupFunc) {
var proxyHandler goproxy.FuncHttpsHandler = func(host string, ctx *goproxy.ProxyCtx) (*goproxy.ConnectAction, string) {
// We shouldn't hit the proxy so we just want to check for any interaction, then reject.
*proxyGotRequest = true
return goproxy.RejectConnect, host
}
proxy.OnRequest().HandleConnect(proxyHandler)
return nil, func() {}
},
shortTimeout: true,
wantUsedProxy: false,
wantError: true,
},
// TODO: Add a NO_PROXY test for libgit2 once the version of libgit2 used by the source controller is updated to a version that includes
// the NO_PROXY functionality
// This PR introduces the functionality in libgit2: https://github.com/libgit2/libgit2/pull/6026
}
for _, tt := range cases {
t.Run(tt.name, func(t *testing.T) {
g := NewWithT(t)
// Run a proxy server.
proxy := goproxy.NewProxyHttpServer()
proxy.Verbose = true
proxyGotRequest := false
authOpts, cleanup := tt.setupGitProxy(g, proxy, &proxyGotRequest)
defer cleanup()
proxyServer := http.Server{
Addr: proxyAddr,
Handler: proxy,
}
l, err := net.Listen("tcp", proxyServer.Addr)
g.Expect(err).ToNot(HaveOccurred())
go proxyServer.Serve(l)
defer proxyServer.Close()
// Set the proxy env vars for both HTTP and HTTPS because go-git caches them.
os.Setenv("HTTPS_PROXY", fmt.Sprintf("http://%s", proxyAddr))
defer os.Unsetenv("HTTPS_PROXY")
os.Setenv("HTTP_PROXY", fmt.Sprintf("http://%s", proxyAddr))
defer os.Unsetenv("HTTP_PROXY")
os.Setenv("NO_PROXY", "*.0.2.1")
defer os.Unsetenv("NO_PROXY")
// Checkout the repo.
checkoutStrategy, err := strategy.CheckoutStrategyForImplementation(context.TODO(), tt.gitImpl, git.CheckoutOptions{
Branch: tt.branch,
})
g.Expect(err).ToNot(HaveOccurred())
tmpDir, err := os.MkdirTemp("", "test-checkout")
g.Expect(err).ToNot(HaveOccurred())
defer os.RemoveAll(tmpDir)
// for the NO_PROXY test we dont want to wait the 30s for it to timeout/fail, so shorten the timeout
checkoutCtx := context.TODO()
if tt.shortTimeout {
var cancel context.CancelFunc
checkoutCtx, cancel = context.WithTimeout(context.TODO(), 1*time.Second)
defer cancel()
}
_, err = checkoutStrategy.Checkout(checkoutCtx, tmpDir, tt.url, authOpts)
if tt.wantError {
g.Expect(err).To(HaveOccurred())
} else {
g.Expect(err).ToNot(HaveOccurred())
}
g.Expect(proxyGotRequest).To(Equal(tt.wantUsedProxy))
})
}
}