From 56e668bb0d12eee23f2e2a54e886d08967307166 Mon Sep 17 00:00:00 2001 From: Jim Ma Date: Thu, 22 Dec 2022 10:56:11 +0800 Subject: [PATCH] chore: optimize download log (#1944) Signed-off-by: Jim Ma --- client/daemon/rpcserver/rpcserver.go | 6 +++++- pkg/source/list_metadata.go | 11 ++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/client/daemon/rpcserver/rpcserver.go b/client/daemon/rpcserver/rpcserver.go index d56f6ff62..d8b953516 100644 --- a/client/daemon/rpcserver/rpcserver.go +++ b/client/daemon/rpcserver/rpcserver.go @@ -397,9 +397,11 @@ func (s *server) recursiveDownload(ctx context.Context, req *dfdaemonv1.DownRequ // only try to cache metadata when cacheRecursiveMetadata is set if s.cacheRecursiveMetadata > 0 { - if err := s.recursiveDownloadWithP2PMetadata(ctx, log, span, traceID, req, stream); err == nil { + log.Infof("start to download with p2p metadata") + if err = s.recursiveDownloadWithP2PMetadata(ctx, log, span, traceID, req, stream); err == nil { return nil } + log.Warnf("download with p2p metadata failed, fallback to direct metadata") } return s.recursiveDownloadWithDirectMetadata(ctx, log, span, traceID, req, stream) @@ -455,6 +457,7 @@ func (s *server) recursiveDownloadWithP2PMetadata( span.RecordError(err) return err } + log.Infof("url entries count: %d", len(listMetadata.URLEntries)) loop: for _, urlEntry := range listMetadata.URLEntries { @@ -497,6 +500,7 @@ loop: } // wait all sent tasks done or error + log.Info("all request sent, wait all tasks processed") wg.Wait() if len(downloadErrors()) > 0 { // just return first error diff --git a/pkg/source/list_metadata.go b/pkg/source/list_metadata.go index f4c21f93e..7cc373102 100644 --- a/pkg/source/list_metadata.go +++ b/pkg/source/list_metadata.go @@ -132,7 +132,7 @@ func (lm *ListMetadataClient) Download(request *Request) (*Response, error) { } cost := time.Now().Sub(listStart).Milliseconds() listMilliseconds += cost - log.Infof("list dir %s cost: %dms", request.URL, cost) + log.Infof("list dir %s cost: %dms, entries(include dir): %d", parentReq.URL, cost, len(urlEntries)) for _, urlEntry := range urlEntries { if urlEntry.IsDir { @@ -146,18 +146,19 @@ func (lm *ListMetadataClient) Download(request *Request) (*Response, error) { } } - log.Infof("list dirs cost: %dms, download cost: %dms", listMilliseconds, time.Now().Sub(start).Milliseconds()) - metadata := ListMetadata{ URLEntries: allURLEntries, } data, _ := json.Marshal(metadata) + + log.Infof("list dirs cost: %dms, entries: %d, metadata size: %d, download cost: %dms", + listMilliseconds, len(allURLEntries), len(data), time.Now().Sub(start).Milliseconds()) return &Response{ Status: "OK", StatusCode: http.StatusOK, - Header: Header(map[string][]string{ + Header: map[string][]string{ headers.Expires: {request.Header.Get(ListMetadataExpire)}, - }), + }, Body: io.NopCloser(bytes.NewBuffer(data)), ContentLength: int64(len(data)), Validate: func() error {