dragonflyoss
/
dragonfly
mirror of https://github.com/dragonflyoss/dragonfly.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

chore: enable cache list metadata e2e (#1829) 

Signed-off-by: Jim Ma <majinjing3@gmail.com>
This commit is contained in:
Jim Ma 2022-11-14 18:03:46 +08:00 committed by Gaius
parent 42223d994f
commit fdd189e661
No known key found for this signature in database
GPG Key ID: 8B4E5D1290FA2FFB
5 changed files with 11 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
.github/workflows/e2e.yml vendored
View File

@ -34,8 +34,7 @@ jobs:
- "grpc-tls-ipv6" - "grpc-tls-ipv6"
- "ipv6" - "ipv6"
- "split-running-tasks" - "split-running-tasks"
# TODO enable the follow module after this PR merged https://github.com/dragonflyoss/Dragonfly2/pull/1822 - "cache-list-metadata"
# - "cache-list-metadata"
include: include:
- module: normal - module: normal
charts-config: test/testdata/charts/config.yaml charts-config: test/testdata/charts/config.yaml
@ -58,9 +57,9 @@ jobs:
- module: split-running-tasks - module: split-running-tasks
charts-config: test/testdata/charts/config-split-running-tasks.yaml charts-config: test/testdata/charts/config-split-running-tasks.yaml
skip: "" skip: ""
# - module: cache-list-metadata - module: cache-list-metadata
# charts-config: test/testdata/charts/config-cache-list-metadata.yaml charts-config: test/testdata/charts/config-cache-list-metadata.yaml
# skip: "" skip: ""
steps: steps:
- name: Checkout code - name: Checkout code
uses: actions/checkout@v3 uses: actions/checkout@v3

2
pkg/source/clients/s3protocol/s3_source_client.go
View File

@ -256,7 +256,7 @@ func (s *s3SourceClient) isDirectory(client *s3.S3, request *source.Request) (bo
request.Context(), request.Context(),
&s3.ListObjectsV2Input{ &s3.ListObjectsV2Input{
Bucket: aws.String(request.URL.Host), Bucket: aws.String(request.URL.Host),
Prefix: aws.String(addTrailingSlash(request.URL.Path)), Prefix: aws.String(uPath),
MaxKeys: aws.Int64(1), MaxKeys: aws.Int64(1),
Delimiter: &delimiter, Delimiter: &delimiter,
}) })

2
pkg/source/list_metadata.go
View File

@ -42,7 +42,7 @@ var (
) )
func init() { func init() {
tracer = otel.Tracer("dfget-rpcserver") tracer = otel.Tracer("back-source")
lmc := &ListMetadataClient{} lmc := &ListMetadataClient{}

3
test/testdata/charts/config-cache-list-metadata.yaml vendored
View File

@ -78,7 +78,8 @@ dfdaemon:
metrics: 127.0.0.1:8888 metrics: 127.0.0.1:8888
download: download:
prefetch: true prefetch: true
recursiveConcurrent: 30 recursiveConcurrent:
goroutineCount: 30
cacheRecursiveMetadata: 10m cacheRecursiveMetadata: 10m
scheduler: scheduler:
disableAutoBackSource: true disableAutoBackSource: true

4
test/tools/download-grpc-test/main.go
View File

@ -21,6 +21,8 @@ var (
) )
func main() { func main() {
flag.Parse()
dialer := func(ctx context.Context, addr string) (net.Conn, error) { dialer := func(ctx context.Context, addr string) (net.Conn, error) {
return net.Dial("unix", addr) return net.Dial("unix", addr)
} }
@ -49,7 +51,7 @@ func main() {
request := dfdaemonv1.DownRequest{ request := dfdaemonv1.DownRequest{
Uuid: "95305fa2-138b-4466-acec-62865ab6403c", Uuid: "95305fa2-138b-4466-acec-62865ab6403c",
Url: "s3://minio-test-bucket/dragonfly-test/usr/" + *subDir, Url: "s3://minio-test-bucket/dragonfly-test/usr/" + *subDir,
Output: "/var/lib/dragonfly-grpc-test/usr" + *subDir, Output: "/var/lib/dragonfly-grpc-test/usr/" + *subDir,
Recursive: true, // recursive download Recursive: true, // recursive download
Timeout: 0, Timeout: 0,
Limit: 0, Limit: 0,