fix: e2e test dfget recursive (#2458)

Signed-off-by: Jim Ma <majinjing3@gmail.com>

pkg/source/clients/s3protocol/s3_source_client.go

@@ -190,9 +190,9 @@ func (s *s3SourceClient) GetLastModified(request *source.Request) (int64, error)
 func (s *s3SourceClient) List(request *source.Request) (urls []source.URLEntry, err error) {
 	client, err := s.newAWSS3Client(request)
 	if err != nil {
-		return nil, fmt.Errorf("get oss client: %w", err)
+		return nil, fmt.Errorf("get s3 client: %w", err)
 	}
-	// if it's a object, just return it.
+	// if it's an object, just return it.
 	isDir, err := s.isDirectory(client, request)
 	if err != nil {
 		return nil, err
@@ -255,7 +255,7 @@ func (s *s3SourceClient) isDirectory(client *s3.S3, request *source.Request) (bo
 			Delimiter: &delimiter,
 		})
 	if err != nil {
-		return false, fmt.Errorf("list oss object %s/%s: %w", request.URL.Host, uPath, err)
+		return false, fmt.Errorf("list s3 object %s/%s: %w", request.URL.Host, uPath, err)
 	}
 	if len(output.Contents)+len(output.CommonPrefixes) > 0 {
 		return true, nil

test/e2e/dfget_test.go

@@ -162,10 +162,10 @@ func singleDfgetTest(name, ns, label, podNamePrefix, container string) {
 			return
 		}
 
-		// prepaired data in minio pod
+		// prepared data in minio pod
 		// test bucket minio-test-bucket
 		// test path /dragonfly-test/usr
-		// test subdirs (no empty dirs)
+		// test sub dirs (no empty dirs)
 		// sha256sum txt: /host/tmp/dragonfly-test.sha256sum.txt
 		subDirs := []string{"bin", "lib64", "libexec", "sbin"}
 
@@ -227,10 +227,10 @@ func singleDfgetTest(name, ns, label, podNamePrefix, container string) {
 			return
 		}
 
-		// prepaired data in minio pod
+		// prepared data in minio pod
 		// test bucket minio-test-bucket
 		// test path /dragonfly-test/usr
-		// test subdirs (no empty dirs)
+		// test sub dirs (no empty dirs)
 		// sha256sum txt: /host/tmp/dragonfly-test.sha256sum.txt
 		subDirs := []string{"bin", "lib64", "libexec", "sbin"}
 

test/e2e/e2e_test.go

@@ -117,8 +117,9 @@ var _ = AfterSuite(func() {
 			out, err = pod.Command("sh", "-c", fmt.Sprintf(`
               set -x
               cp -r /var/log/dragonfly/%s /tmp/artifact/%s-%d
+              cp -r /var/log/dragonfly/dfget /tmp/artifact/%s-%d-dfget
               find /tmp/artifact -type d -exec chmod 777 {} \;
-              `, server.logDirName, server.name, i)).CombinedOutput()
+              `, server.logDirName, server.name, i, server.name, i)).CombinedOutput()
 			if err != nil {
 				fmt.Printf("copy log output: %q, error: %s, retry after 10s\n", string(out), err)
 				// pod maybe restart due to panic, wait it running again

test/testdata/k8s/minio.yaml

@@ -46,7 +46,7 @@ spec:
       containers:
       - name: minio
         # Pull image with dragonfly
-        image: minio/minio:latest
+        image: minio/minio:RELEASE.2022-11-11T03-44-20Z
         imagePullPolicy: "IfNotPresent"
         command:
         - /opt/bin/minio