feat: Add useProxies to registryMirror allowing to mirror more anything (#965)

* feat: add useProxies to registryMirror to allow mirroring anything

Signed-off-by: momiji <teq1uila-free01@yahoo.fr>

client/config/peerhost.go

@@ -527,6 +527,9 @@ type RegistryMirror struct {
 
 	// Request the remote registry directly.
 	Direct bool `yaml:"direct" mapstructure:"direct"`
+
+	// Whether to use proxies to decide when to use dragonfly
+	UseProxies bool `yaml:"useProxies" mapstructure:"useProxies"`
 }
 
 // TLSConfig returns the tls.Config used to communicate with the mirror.

client/daemon/proxy/proxy.go

@@ -269,8 +269,11 @@ func (proxy *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
+	// check direct request
+	directRequest := r.Method != http.MethodConnect && r.URL.Scheme == ""
+
 	// check whiteList
-	if !proxy.checkWhiteList(r) {
+	if !directRequest && !proxy.checkWhiteList(r) {
 		status := http.StatusUnauthorized
 		http.Error(w, http.StatusText(status), status)
 		logger.Debugf("not in whitelist: %s, url：%s", r.Host, r.URL.String())
@@ -556,7 +559,13 @@ func (proxy *Proxy) shouldUseDragonfly(req *http.Request) bool {
 // shouldUseDragonflyForMirror returns whether we should use dragonfly to proxy a request
 // when we use registry mirror.
 func (proxy *Proxy) shouldUseDragonflyForMirror(req *http.Request) bool {
-	return proxy.registry != nil && !proxy.registry.Direct && transport.NeedUseDragonfly(req)
+	if proxy.registry == nil || proxy.registry.Direct {
+		return false
+	}
+	if proxy.registry.UseProxies {
+		return proxy.shouldUseDragonfly(req)
+	}
+	return transport.NeedUseDragonfly(req)
 }
 
 // tunnelHTTPS handles a CONNECT request and proxy an https request through an

client/daemon/proxy/proxy_test.go

@@ -56,7 +56,7 @@ func (tc *testCase) WithRule(regx string, direct bool, useHTTPS bool, redirect s
 	return tc
 }
 
-func (tc *testCase) WithRegistryMirror(rawURL string, direct bool, dynamic bool) *testCase {
+func (tc *testCase) WithRegistryMirror(rawURL string, direct bool, dynamic bool, useProxies bool) *testCase {
 	if tc.Error != nil {
 		return tc
 	}
@@ -67,6 +67,7 @@ func (tc *testCase) WithRegistryMirror(rawURL string, direct bool, dynamic bool)
 		Remote:        &config.URL{URL: u},
 		DynamicRemote: dynamic,
 		Direct:        direct,
+		UseProxies:    useProxies,
 	}
 	return tc
 }
@@ -109,7 +110,7 @@ func (tc *testCase) TestMirror(t *testing.T) {
 	if !a.Nil(tc.Error) {
 		return
 	}
-	tp, err := NewProxy(WithRegistryMirror(tc.RegistryMirror))
+	tp, err := NewProxy(WithRegistryMirror(tc.RegistryMirror), WithRules(tc.Rules))
 	if !a.Nil(err) {
 		return
 	}
@@ -154,17 +155,39 @@ func TestMatch(t *testing.T) {
 		TestMirror(t)
 
 	newTestCase().
-		WithRegistryMirror("http://index.docker.io", false, false).
+		WithRegistryMirror("http://index.docker.io", false, false, false).
 		WithTest("http://h/a", true, false, "").
 		TestMirror(t)
 
 	newTestCase().
-		WithRegistryMirror("http://index.docker.io", false, false).
+		WithRegistryMirror("http://index.docker.io", false, false, false).
 		WithTest("http://index.docker.io/v2/blobs/sha256/xxx", false, false, "").
 		TestMirror(t)
 
 	newTestCase().
-		WithRegistryMirror("http://index.docker.io", true, false).
+		WithRegistryMirror("http://index.docker.io", true, false, false).
+		WithTest("http://index.docker.io/v2/blobs/sha256/xxx", true, false, "").
+		TestMirror(t)
+}
+
+func TestMatchWithUseProxies(t *testing.T) {
+	// should direct as registry is set with direct=false and no proxies are defined
+	newTestCase().
+		WithRegistryMirror("http://index.docker.io", false, false, true).
+		WithTest("http://index.docker.io/v2/blobs/sha256/xxx", true, false, "").
+		TestMirror(t)
+
+	// should cache as registry is set with direct=false, and one proxy matches
+	newTestCase().
+		WithRegistryMirror("http://index.docker.io", false, false, true).
+		WithRule("/blobs/sha256/", false, false, "").
+		WithTest("http://index.docker.io/v2/blobs/sha256/xxx", false, false, "").
+		TestMirror(t)
+
+	// should direct as registry is set with direct=true, even if one proxy matches
+	newTestCase().
+		WithRegistryMirror("http://index.docker.io", true, false, true).
+		WithRule("/blobs/sha256/", false, false, "").
 		WithTest("http://index.docker.io/v2/blobs/sha256/xxx", true, false, "").
 		TestMirror(t)
 }

docs/en/deployment/configuration/dfget.yaml

@@ -221,6 +221,8 @@ proxy:
     certs: []
     # whether to request the remote registry directly
     direct: false
+    # whether to use proxies to decide if dragonfly should be used
+    useProxies: false
 
   proxies:
     # proxy all http image layer download requests with dfget

docs/zh-CN/deployment/configuration/dfget.yaml

@@ -194,6 +194,8 @@ proxy:
     certs: []
     # 是否直连镜像中心，true 的话，流量不再走 p2p
     direct: false
+    # whether to use proxies to decide if dragonfly should be used
+    useProxies: false
 
   proxies:
     # 代理镜像 blobs 信息