From 94e35fa9df57c59a7a217928668deda7d237b15f Mon Sep 17 00:00:00 2001 From: Jim Ma Date: Wed, 11 Aug 2021 17:11:06 +0800 Subject: [PATCH] chore: optimize compute piece size function (#528) Signed-off-by: Jim Ma --- cdnsystem/cdnutil/cdn_util.go | 18 ++++++++++++++++ cdnsystem/daemon/task/manager_util.go | 24 ++++----------------- client/daemon/peer/peertask_file_test.go | 10 +++++---- client/daemon/peer/peertask_manager_test.go | 3 ++- client/daemon/peer/peertask_stream_test.go | 5 +++-- client/daemon/peer/piece_manager.go | 22 ++----------------- test/{ => tools}/stress/README.md | 0 test/{ => tools}/stress/main.go | 4 ++-- 8 files changed, 37 insertions(+), 49 deletions(-) rename test/{ => tools}/stress/README.md (100%) rename test/{ => tools}/stress/main.go (100%) diff --git a/cdnsystem/cdnutil/cdn_util.go b/cdnsystem/cdnutil/cdn_util.go index 7c8f32efb..9e301e57e 100644 --- a/cdnsystem/cdnutil/cdn_util.go +++ b/cdnsystem/cdnutil/cdn_util.go @@ -19,9 +19,27 @@ package cdnutil import ( "fmt" + "d7y.io/dragonfly/v2/cdnsystem/config" "d7y.io/dragonfly/v2/pkg/util/net/iputils" ) func GenCDNPeerID(taskID string) string { return fmt.Sprintf("%s-%s_%s", iputils.HostName, taskID, "CDN") } + +// ComputePieceSize computes the piece size with specified fileLength. +// +// If the fileLength<=0, which means failed to get fileLength +// and then use the DefaultPieceSize. +func ComputePieceSize(length int64) int32 { + if length <= 0 || length <= 200*1024*1024 { + return config.DefaultPieceSize + } + + gapCount := length / int64(100*1024*1024) + mpSize := (gapCount-2)*1024*1024 + config.DefaultPieceSize + if mpSize > config.DefaultPieceSizeLimit { + return config.DefaultPieceSizeLimit + } + return int32(mpSize) +} diff --git a/cdnsystem/daemon/task/manager_util.go b/cdnsystem/daemon/task/manager_util.go index 374137af7..1883fd0d6 100644 --- a/cdnsystem/daemon/task/manager_util.go +++ b/cdnsystem/daemon/task/manager_util.go @@ -22,7 +22,9 @@ import ( "reflect" "time" - "d7y.io/dragonfly/v2/cdnsystem/config" + "github.com/pkg/errors" + + "d7y.io/dragonfly/v2/cdnsystem/cdnutil" cdnerrors "d7y.io/dragonfly/v2/cdnsystem/errors" "d7y.io/dragonfly/v2/cdnsystem/types" logger "d7y.io/dragonfly/v2/internal/dflog" @@ -30,7 +32,6 @@ import ( "d7y.io/dragonfly/v2/pkg/synclock" "d7y.io/dragonfly/v2/pkg/util/net/urlutils" "d7y.io/dragonfly/v2/pkg/util/stringutils" - "github.com/pkg/errors" ) const ( @@ -108,7 +109,7 @@ func (tm *Manager) addOrUpdateTask(ctx context.Context, request *types.TaskRegis // calculate piece size and update the PieceSize and PieceTotal if task.PieceSize <= 0 { - pieceSize := computePieceSize(task.SourceFileLength) + pieceSize := cdnutil.ComputePieceSize(task.SourceFileLength) task.PieceSize = pieceSize } tm.taskStore.Add(task.TaskID, task) @@ -193,20 +194,3 @@ func isSameTask(task1, task2 *types.SeedTask) bool { return true } - -// computePieceSize computes the piece size with specified fileLength. -// -// If the fileLength<=0, which means failed to get fileLength -// and then use the DefaultPieceSize. -func computePieceSize(length int64) int32 { - if length <= 0 || length <= 200*1024*1024 { - return config.DefaultPieceSize - } - - gapCount := length / int64(100*1024*1024) - mpSize := (gapCount-2)*1024*1024 + config.DefaultPieceSize - if mpSize > config.DefaultPieceSizeLimit { - return config.DefaultPieceSizeLimit - } - return int32(mpSize) -} diff --git a/client/daemon/peer/peertask_file_test.go b/client/daemon/peer/peertask_file_test.go index 9ad34baca..02b062ea2 100644 --- a/client/daemon/peer/peertask_file_test.go +++ b/client/daemon/peer/peertask_file_test.go @@ -26,17 +26,19 @@ import ( "testing" "time" + "d7y.io/dragonfly/v2/cdnsystem/cdnutil" "d7y.io/dragonfly/v2/pkg/rpc/base" rangers "d7y.io/dragonfly/v2/pkg/util/rangeutils" + "github.com/golang/mock/gomock" + testifyassert "github.com/stretchr/testify/assert" + "d7y.io/dragonfly/v2/client/clientutil" "d7y.io/dragonfly/v2/client/config" "d7y.io/dragonfly/v2/client/daemon/test" "d7y.io/dragonfly/v2/pkg/rpc/scheduler" "d7y.io/dragonfly/v2/pkg/source" sourceMock "d7y.io/dragonfly/v2/pkg/source/mock" - "github.com/golang/mock/gomock" - testifyassert "github.com/stretchr/testify/assert" ) func TestFilePeerTask_BackSource_WithContentLength(t *testing.T) { @@ -100,7 +102,7 @@ func TestFilePeerTask_BackSource_WithContentLength(t *testing.T) { pieceManager: &pieceManager{ storageManager: storageManager, pieceDownloader: downloader, - computePieceSize: computePieceSize, + computePieceSize: cdnutil.ComputePieceSize, }, storageManager: storageManager, schedulerClient: schedulerClient, @@ -219,7 +221,7 @@ func TestFilePeerTask_BackSource_WithoutContentLength(t *testing.T) { pieceManager: &pieceManager{ storageManager: storageManager, pieceDownloader: downloader, - computePieceSize: computePieceSize, + computePieceSize: cdnutil.ComputePieceSize, }, storageManager: storageManager, schedulerClient: schedulerClient, diff --git a/client/daemon/peer/peertask_manager_test.go b/client/daemon/peer/peertask_manager_test.go index 92222f248..65ed9b5b3 100644 --- a/client/daemon/peer/peertask_manager_test.go +++ b/client/daemon/peer/peertask_manager_test.go @@ -35,6 +35,7 @@ import ( testifyassert "github.com/stretchr/testify/assert" "google.golang.org/grpc" + "d7y.io/dragonfly/v2/cdnsystem/cdnutil" "d7y.io/dragonfly/v2/client/clientutil" "d7y.io/dragonfly/v2/client/config" "d7y.io/dragonfly/v2/client/daemon/storage" @@ -362,7 +363,7 @@ func TestPeerTaskManager_StartStreamPeerTask_BackSource(t *testing.T) { pieceManager: &pieceManager{ storageManager: storageManager, pieceDownloader: NewMockPieceDownloader(ctrl), - computePieceSize: computePieceSize, + computePieceSize: cdnutil.ComputePieceSize, }, storageManager: storageManager, schedulerClient: sched, diff --git a/client/daemon/peer/peertask_stream_test.go b/client/daemon/peer/peertask_stream_test.go index 0ca482899..932aa1cef 100644 --- a/client/daemon/peer/peertask_stream_test.go +++ b/client/daemon/peer/peertask_stream_test.go @@ -28,6 +28,7 @@ import ( "github.com/golang/mock/gomock" testifyassert "github.com/stretchr/testify/assert" + "d7y.io/dragonfly/v2/cdnsystem/cdnutil" "d7y.io/dragonfly/v2/client/clientutil" "d7y.io/dragonfly/v2/client/config" "d7y.io/dragonfly/v2/client/daemon/test" @@ -97,7 +98,7 @@ func TestStreamPeerTask_BackSource_WithContentLength(t *testing.T) { pieceManager: &pieceManager{ storageManager: storageManager, pieceDownloader: downloader, - computePieceSize: computePieceSize, + computePieceSize: cdnutil.ComputePieceSize, }, storageManager: storageManager, schedulerClient: schedulerClient, @@ -204,7 +205,7 @@ func TestStreamPeerTask_BackSource_WithoutContentLength(t *testing.T) { pieceManager: &pieceManager{ storageManager: storageManager, pieceDownloader: downloader, - computePieceSize: computePieceSize, + computePieceSize: cdnutil.ComputePieceSize, }, storageManager: storageManager, schedulerClient: schedulerClient, diff --git a/client/daemon/peer/piece_manager.go b/client/daemon/peer/piece_manager.go index da1004d4f..fbdbbca90 100644 --- a/client/daemon/peer/piece_manager.go +++ b/client/daemon/peer/piece_manager.go @@ -24,7 +24,7 @@ import ( "golang.org/x/time/rate" - cdnconfig "d7y.io/dragonfly/v2/cdnsystem/config" + "d7y.io/dragonfly/v2/cdnsystem/cdnutil" "d7y.io/dragonfly/v2/client/clientutil" "d7y.io/dragonfly/v2/client/config" "d7y.io/dragonfly/v2/client/daemon/storage" @@ -56,7 +56,7 @@ var _ PieceManager = (*pieceManager)(nil) func NewPieceManager(s storage.TaskStorageDriver, opts ...func(*pieceManager)) (PieceManager, error) { pm := &pieceManager{ storageManager: s, - computePieceSize: computePieceSize, + computePieceSize: cdnutil.ComputePieceSize, calculateDigest: true, } for _, opt := range opts { @@ -386,21 +386,3 @@ func (pm *pieceManager) DownloadSource(ctx context.Context, pt Task, request *sc log.Infof("download from source ok") return nil } - -// TODO copy from cdnsystem/daemon/mgr/task/manager_util.go -// computePieceSize computes the piece size with specified fileLength. -// -// If the fileLength<=0, which means failed to get fileLength -// and then use the DefaultPieceSize. -func computePieceSize(length int64) int32 { - if length <= 0 || length <= 200*1024*1024 { - return cdnconfig.DefaultPieceSize - } - - gapCount := length / int64(100*1024*1024) - mpSize := (gapCount-2)*1024*1024 + cdnconfig.DefaultPieceSize - if mpSize > cdnconfig.DefaultPieceSizeLimit { - return cdnconfig.DefaultPieceSizeLimit - } - return int32(mpSize) -} diff --git a/test/stress/README.md b/test/tools/stress/README.md similarity index 100% rename from test/stress/README.md rename to test/tools/stress/README.md diff --git a/test/stress/main.go b/test/tools/stress/main.go similarity index 100% rename from test/stress/main.go rename to test/tools/stress/main.go index 816e53b1b..297e320e0 100644 --- a/test/stress/main.go +++ b/test/tools/stress/main.go @@ -33,13 +33,13 @@ import ( "syscall" "time" - "d7y.io/dragonfly/v2/pkg/unit" - "d7y.io/dragonfly/v2/pkg/util/net/iputils" "github.com/go-echarts/statsview" "github.com/go-echarts/statsview/viewer" "github.com/montanaflynn/stats" "d7y.io/dragonfly/v2/client/config" + "d7y.io/dragonfly/v2/pkg/unit" + "d7y.io/dragonfly/v2/pkg/util/net/iputils" ) var (