Merge pull request #524 from racdev/libgit2-proxy-support

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/Masterminds/semver/v3 v3.1.1
 	github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7
 	github.com/cyphar/filepath-securejoin v0.2.2
+	github.com/elazarl/goproxy v0.0.0-20211114080932-d06c3be7c11b
 	github.com/fluxcd/pkg/apis/meta v0.10.2
 	github.com/fluxcd/pkg/gittestserver v0.5.0
 	github.com/fluxcd/pkg/gitutil v0.1.0

go.sum

@@ -267,8 +267,11 @@ github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
-github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
+github.com/elazarl/goproxy v0.0.0-20211114080932-d06c3be7c11b h1:1XqENn2YoYZd6w3Awx+7oa+aR87DFIZJFLF2n1IojA0=
+github.com/elazarl/goproxy v0.0.0-20211114080932-d06c3be7c11b/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
+github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 h1:dWB6v3RcOy03t/bUadywsbyrQwCqZeNIEX6M1OtSZOM=
+github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
@@ -806,6 +809,7 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
+github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=

pkg/git/libgit2/checkout.go

@@ -64,6 +64,7 @@ func (c *CheckoutBranch) Checkout(ctx context.Context, path, url string, opts *g
 		FetchOptions: &git2go.FetchOptions{
 			DownloadTags:    git2go.DownloadTagsNone,
 			RemoteCallbacks: RemoteCallbacks(ctx, opts),
+			ProxyOptions:    git2go.ProxyOptions{Type: git2go.ProxyTypeAuto},
 		},
 		CheckoutBranch: c.Branch,
 	})
@@ -93,6 +94,7 @@ func (c *CheckoutTag) Checkout(ctx context.Context, path, url string, opts *git.
 		FetchOptions: &git2go.FetchOptions{
 			DownloadTags:    git2go.DownloadTagsAll,
 			RemoteCallbacks: RemoteCallbacks(ctx, opts),
+			ProxyOptions:    git2go.ProxyOptions{Type: git2go.ProxyTypeAuto},
 		},
 	})
 	if err != nil {
@@ -116,6 +118,7 @@ func (c *CheckoutCommit) Checkout(ctx context.Context, path, url string, opts *g
 		FetchOptions: &git2go.FetchOptions{
 			DownloadTags:    git2go.DownloadTagsNone,
 			RemoteCallbacks: RemoteCallbacks(ctx, opts),
+			ProxyOptions:    git2go.ProxyOptions{Type: git2go.ProxyTypeAuto},
 		},
 	})
 	if err != nil {
@@ -147,6 +150,7 @@ func (c *CheckoutSemVer) Checkout(ctx context.Context, path, url string, opts *g
 		FetchOptions: &git2go.FetchOptions{
 			DownloadTags:    git2go.DownloadTagsAll,
 			RemoteCallbacks: RemoteCallbacks(ctx, opts),
+			ProxyOptions:    git2go.ProxyOptions{Type: git2go.ProxyTypeAuto},
 		},
 	})
 	if err != nil {

pkg/git/strategy/proxy/strategy_proxy_test.go

@@ -0,0 +1,290 @@
+/*
+Copyright 2021 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package proxy
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/elazarl/goproxy"
+	"github.com/fluxcd/pkg/gittestserver"
+	. "github.com/onsi/gomega"
+
+	"github.com/fluxcd/source-controller/pkg/git"
+	"github.com/fluxcd/source-controller/pkg/git/gogit"
+	"github.com/fluxcd/source-controller/pkg/git/libgit2"
+	"github.com/fluxcd/source-controller/pkg/git/strategy"
+)
+
+// These tests are run in a different _test.go file because go-git uses the ProxyFromEnvironment function of the net/http package
+// which caches the Proxy settings, hence not including other tests in the same file ensures a clean proxy setup for the tests to run.
+func TestCheckoutStrategyForImplementation_Proxied(t *testing.T) {
+
+	type cleanupFunc func()
+
+	type testCase struct {
+		name          string
+		gitImpl       git.Implementation
+		url           string
+		branch        string
+		setupGitProxy func(g *WithT, proxy *goproxy.ProxyHttpServer, proxyGotRequest *bool) (*git.AuthOptions, cleanupFunc)
+		shortTimeout  bool
+		wantUsedProxy bool
+		wantError     bool
+	}
+
+	g := NewWithT(t)
+
+	// Get a free port for proxy to use.
+	l, err := net.Listen("tcp", ":0")
+	g.Expect(err).ToNot(HaveOccurred())
+	proxyAddr := fmt.Sprintf("localhost:%d", l.Addr().(*net.TCPAddr).Port)
+	g.Expect(l.Close()).ToNot(HaveOccurred())
+
+	// Note there is no libgit2 HTTP_PROXY test as libgit2 doesnt support proxied HTTP requests.
+	cases := []testCase{
+		{
+			name:    "libgit2_HTTPS_PROXY",
+			gitImpl: libgit2.Implementation,
+			url:     "https://example.com/bar/test-reponame",
+			branch:  "main",
+			setupGitProxy: func(g *WithT, proxy *goproxy.ProxyHttpServer, proxyGotRequest *bool) (*git.AuthOptions, cleanupFunc) {
+				// Create the git server.
+				gitServer, err := gittestserver.NewTempGitServer()
+				g.Expect(err).ToNot(HaveOccurred())
+
+				username := "test-user"
+				password := "test-password"
+				gitServer.Auth(username, password)
+				gitServer.KeyDir(gitServer.Root())
+
+				// Start the HTTPS server.
+				examplePublicKey, err := os.ReadFile("../testdata/certs/server.pem")
+				g.Expect(err).ToNot(HaveOccurred())
+				examplePrivateKey, err := os.ReadFile("../testdata/certs/server-key.pem")
+				g.Expect(err).ToNot(HaveOccurred())
+				exampleCA, err := os.ReadFile("../testdata/certs/ca.pem")
+				g.Expect(err).ToNot(HaveOccurred())
+				err = gitServer.StartHTTPS(examplePublicKey, examplePrivateKey, exampleCA, "example.com")
+				g.Expect(err).ToNot(HaveOccurred())
+
+				// Initialize a git repo.
+				repoPath := "bar/test-reponame"
+				err = gitServer.InitRepo("../testdata/repo1", "main", repoPath)
+				g.Expect(err).ToNot(HaveOccurred())
+
+				u, err := url.Parse(gitServer.HTTPAddress())
+				g.Expect(err).ToNot(HaveOccurred())
+
+				// The request is being forwarded to the local test git server in this handler.
+				// The certificate used here is valid for both example.com and localhost.
+				var proxyHandler goproxy.FuncHttpsHandler = func(host string, ctx *goproxy.ProxyCtx) (*goproxy.ConnectAction, string) {
+					// Check if the host matches with the git server address and the user-agent is the expected git client.
+					userAgent := ctx.Req.Header.Get("User-Agent")
+					if strings.Contains(host, "example.com") && strings.Contains(userAgent, "libgit2") {
+						*proxyGotRequest = true
+						return goproxy.OkConnect, u.Host
+					}
+					// Reject if it isn't our request.
+					return goproxy.RejectConnect, host
+				}
+				proxy.OnRequest().HandleConnect(proxyHandler)
+
+				return &git.AuthOptions{
+						Transport: git.HTTPS,
+						Username:  username,
+						Password:  password,
+						CAFile:    exampleCA,
+					}, func() {
+						os.RemoveAll(gitServer.Root())
+						gitServer.StopHTTP()
+					}
+			},
+			shortTimeout:  false,
+			wantUsedProxy: true,
+			wantError:     false,
+		},
+		{
+			name:    "gogit_HTTP_PROXY",
+			gitImpl: gogit.Implementation,
+			url:     "http://example.com/bar/test-reponame",
+			branch:  "main",
+			setupGitProxy: func(g *WithT, proxy *goproxy.ProxyHttpServer, proxyGotRequest *bool) (*git.AuthOptions, cleanupFunc) {
+				// Create the git server.
+				gitServer, err := gittestserver.NewTempGitServer()
+				g.Expect(err).ToNot(HaveOccurred())
+
+				username := "test-user"
+				password := "test-password"
+				gitServer.Auth(username, password)
+				gitServer.KeyDir(gitServer.Root())
+
+				g.Expect(gitServer.StartHTTP()).ToNot(HaveOccurred())
+
+				// Initialize a git repo.
+				err = gitServer.InitRepo("../testdata/repo1", "main", "bar/test-reponame")
+				g.Expect(err).ToNot(HaveOccurred())
+
+				u, err := url.Parse(gitServer.HTTPAddress())
+				g.Expect(err).ToNot(HaveOccurred())
+
+				// The request is being forwarded to the local test git server in this handler.
+				var proxyHandler goproxy.FuncReqHandler = func(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
+					userAgent := req.Header.Get("User-Agent")
+					if strings.Contains(req.Host, "example.com") && strings.Contains(userAgent, "git") {
+						*proxyGotRequest = true
+						req.Host = u.Host
+						req.URL.Host = req.Host
+						return req, nil
+					}
+					// Reject if it isnt our request.
+					return req, goproxy.NewResponse(req, goproxy.ContentTypeText, http.StatusForbidden, "")
+				}
+				proxy.OnRequest().Do(proxyHandler)
+
+				return &git.AuthOptions{
+						Transport: git.HTTP,
+						Username:  username,
+						Password:  password,
+					}, func() {
+						os.RemoveAll(gitServer.Root())
+						gitServer.StopHTTP()
+					}
+			},
+			shortTimeout:  false,
+			wantUsedProxy: true,
+			wantError:     false,
+		},
+		{
+			name:    "gogit_HTTPS_PROXY",
+			gitImpl: gogit.Implementation,
+			url:     "https://github.com/git-fixtures/basic",
+			branch:  "master",
+			setupGitProxy: func(g *WithT, proxy *goproxy.ProxyHttpServer, proxyGotRequest *bool) (*git.AuthOptions, cleanupFunc) {
+				var proxyHandler goproxy.FuncHttpsHandler = func(host string, ctx *goproxy.ProxyCtx) (*goproxy.ConnectAction, string) {
+					// We don't check for user agent as this handler is only going to process CONNECT requests, and because Go's net/http
+					// is the one making such a request on behalf of go-git, adding a check for the go net/http user agent (Go-http-client)
+					// would only allow false positives from any request originating from Go's net/http.
+					if strings.Contains(host, "github.com") {
+						*proxyGotRequest = true
+						return goproxy.OkConnect, host
+					}
+					// Reject if it isnt our request.
+					return goproxy.RejectConnect, host
+				}
+				proxy.OnRequest().HandleConnect(proxyHandler)
+
+				// go-git does not allow to use an HTTPS proxy and a custom root CA at the same time.
+				// See https://github.com/fluxcd/source-controller/pull/524#issuecomment-1006673163.
+				return nil, func() {}
+			},
+			shortTimeout:  false,
+			wantUsedProxy: true,
+			wantError:     false,
+		},
+		{
+			name:    "gogit_NO_PROXY",
+			gitImpl: gogit.Implementation,
+			url:     "https://192.0.2.1/bar/test-reponame",
+			branch:  "main",
+			setupGitProxy: func(g *WithT, proxy *goproxy.ProxyHttpServer, proxyGotRequest *bool) (*git.AuthOptions, cleanupFunc) {
+				var proxyHandler goproxy.FuncHttpsHandler = func(host string, ctx *goproxy.ProxyCtx) (*goproxy.ConnectAction, string) {
+					// We shouldn't hit the proxy so we just want to check for any interaction, then reject.
+					*proxyGotRequest = true
+					return goproxy.RejectConnect, host
+				}
+				proxy.OnRequest().HandleConnect(proxyHandler)
+
+				return nil, func() {}
+			},
+			shortTimeout:  true,
+			wantUsedProxy: false,
+			wantError:     true,
+		},
+		// TODO: Add a NO_PROXY test for libgit2 once the version of libgit2 used by the source controller is updated to a version that includes
+		// the NO_PROXY functionality
+		// This PR introduces the functionality in libgit2: https://github.com/libgit2/libgit2/pull/6026
+	}
+
+	for _, tt := range cases {
+		t.Run(tt.name, func(t *testing.T) {
+			g := NewWithT(t)
+
+			// Run a proxy server.
+			proxy := goproxy.NewProxyHttpServer()
+			proxy.Verbose = true
+
+			proxyGotRequest := false
+			authOpts, cleanup := tt.setupGitProxy(g, proxy, &proxyGotRequest)
+			defer cleanup()
+
+			proxyServer := http.Server{
+				Addr:    proxyAddr,
+				Handler: proxy,
+			}
+			l, err := net.Listen("tcp", proxyServer.Addr)
+			g.Expect(err).ToNot(HaveOccurred())
+			go proxyServer.Serve(l)
+			defer proxyServer.Close()
+
+			// Set the proxy env vars for both HTTP and HTTPS because go-git caches them.
+			os.Setenv("HTTPS_PROXY", fmt.Sprintf("http://%s", proxyAddr))
+			defer os.Unsetenv("HTTPS_PROXY")
+
+			os.Setenv("HTTP_PROXY", fmt.Sprintf("http://%s", proxyAddr))
+			defer os.Unsetenv("HTTP_PROXY")
+
+			os.Setenv("NO_PROXY", "*.0.2.1")
+			defer os.Unsetenv("NO_PROXY")
+
+			// Checkout the repo.
+			checkoutStrategy, err := strategy.CheckoutStrategyForImplementation(context.TODO(), tt.gitImpl, git.CheckoutOptions{
+				Branch: tt.branch,
+			})
+			g.Expect(err).ToNot(HaveOccurred())
+
+			tmpDir, err := os.MkdirTemp("", "test-checkout")
+			g.Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpDir)
+
+			// for the NO_PROXY test we dont want to wait the 30s for it to timeout/fail, so shorten the timeout
+			checkoutCtx := context.TODO()
+			if tt.shortTimeout {
+				var cancel context.CancelFunc
+				checkoutCtx, cancel = context.WithTimeout(context.TODO(), 1*time.Second)
+				defer cancel()
+			}
+
+			_, err = checkoutStrategy.Checkout(checkoutCtx, tmpDir, tt.url, authOpts)
+			if tt.wantError {
+				g.Expect(err).To(HaveOccurred())
+			} else {
+				g.Expect(err).ToNot(HaveOccurred())
+			}
+
+			g.Expect(proxyGotRequest).To(Equal(tt.wantUsedProxy))
+
+		})
+	}
+}