chore: enable cache list metadata e2e (#1829)

Signed-off-by: Jim Ma <majinjing3@gmail.com>

.github/workflows/e2e.yml

@@ -34,8 +34,7 @@ jobs:
           - "grpc-tls-ipv6"
           - "ipv6"
           - "split-running-tasks"
-          # TODO enable the follow module after this PR merged https://github.com/dragonflyoss/Dragonfly2/pull/1822
-          # - "cache-list-metadata"
+          - "cache-list-metadata"
         include:
           - module: normal
             charts-config: test/testdata/charts/config.yaml
@@ -58,9 +57,9 @@ jobs:
           - module: split-running-tasks
             charts-config: test/testdata/charts/config-split-running-tasks.yaml
             skip: ""
-          # - module: cache-list-metadata
-          #   charts-config: test/testdata/charts/config-cache-list-metadata.yaml
-          #   skip: ""
+          - module: cache-list-metadata
+            charts-config: test/testdata/charts/config-cache-list-metadata.yaml
+            skip: ""
     steps:
       - name: Checkout code
         uses: actions/checkout@v3

pkg/source/clients/s3protocol/s3_source_client.go

@@ -256,7 +256,7 @@ func (s *s3SourceClient) isDirectory(client *s3.S3, request *source.Request) (bo
 		request.Context(),
 		&s3.ListObjectsV2Input{
 			Bucket:    aws.String(request.URL.Host),
-			Prefix:    aws.String(addTrailingSlash(request.URL.Path)),
+			Prefix:    aws.String(uPath),
 			MaxKeys:   aws.Int64(1),
 			Delimiter: &delimiter,
 		})

pkg/source/list_metadata.go

@@ -42,7 +42,7 @@ var (
 )
 
 func init() {
-	tracer = otel.Tracer("dfget-rpcserver")
+	tracer = otel.Tracer("back-source")
 
 	lmc := &ListMetadataClient{}
 

test/testdata/charts/config-cache-list-metadata.yaml

@@ -78,7 +78,8 @@ dfdaemon:
     metrics: 127.0.0.1:8888
     download:
       prefetch: true
-      recursiveConcurrent: 30
+      recursiveConcurrent:
+        goroutineCount: 30
       cacheRecursiveMetadata: 10m
     scheduler:
       disableAutoBackSource: true

test/tools/download-grpc-test/main.go

@@ -21,6 +21,8 @@ var (
 )
 
 func main() {
+	flag.Parse()
+
 	dialer := func(ctx context.Context, addr string) (net.Conn, error) {
 		return net.Dial("unix", addr)
 	}
@@ -49,7 +51,7 @@ func main() {
 	request := dfdaemonv1.DownRequest{
 		Uuid:              "95305fa2-138b-4466-acec-62865ab6403c",
 		Url:               "s3://minio-test-bucket/dragonfly-test/usr/" + *subDir,
-		Output:            "/var/lib/dragonfly-grpc-test/usr" + *subDir,
+		Output:            "/var/lib/dragonfly-grpc-test/usr/" + *subDir,
 		Recursive:         true, // recursive download
 		Timeout:           0,
 		Limit:             0,