/* * Copyright 2020 The Dragonfly Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package cdn import ( "context" "crypto/md5" "hash" "io" "io/ioutil" "sort" "strings" "testing" "time" cdnerrors "d7y.io/dragonfly/v2/cdnsystem/errors" "d7y.io/dragonfly/v2/cdnsystem/storedriver" "d7y.io/dragonfly/v2/cdnsystem/supervisor/cdn/storage" storageMock "d7y.io/dragonfly/v2/cdnsystem/supervisor/cdn/storage/mock" "d7y.io/dragonfly/v2/cdnsystem/types" "d7y.io/dragonfly/v2/pkg/source" sourceMock "d7y.io/dragonfly/v2/pkg/source/mock" "d7y.io/dragonfly/v2/pkg/util/digestutils" "d7y.io/dragonfly/v2/pkg/util/rangeutils" "github.com/golang/mock/gomock" "github.com/stretchr/testify/suite" ) func TestCacheDetectorSuite(t *testing.T) { suite.Run(t, new(CacheDetectorTestSuite)) } type CacheDetectorTestSuite struct { detector *cacheDetector suite.Suite } func (suite *CacheDetectorTestSuite) SetupSuite() { ctrl := gomock.NewController(suite.T()) sourceClient := sourceMock.NewMockResourceClient(ctrl) source.Register("http", sourceClient) storageMgr := storageMock.NewMockManager(ctrl) cacheDataManager := newCacheDataManager(storageMgr) suite.detector = newCacheDetector(cacheDataManager) storageMgr.EXPECT().ReadFileMetaData(fullExpiredCache.taskID).Return(fullExpiredCache.fileMeta, nil).AnyTimes() storageMgr.EXPECT().ReadFileMetaData(fullNoExpiredCache.taskID).Return(fullNoExpiredCache.fileMeta, nil).AnyTimes() storageMgr.EXPECT().ReadFileMetaData(partialNotSupportRangeCache.taskID).Return(partialNotSupportRangeCache.fileMeta, nil).AnyTimes() storageMgr.EXPECT().ReadFileMetaData(partialSupportRangeCache.taskID).Return(partialSupportRangeCache.fileMeta, nil).AnyTimes() storageMgr.EXPECT().ReadFileMetaData(noCache.taskID).Return(noCache.fileMeta, cdnerrors.ErrFileNotExist{}).AnyTimes() storageMgr.EXPECT().ReadDownloadFile(fullNoExpiredCache.taskID).DoAndReturn( func(taskID string) (io.ReadCloser, error) { content, err := ioutil.ReadFile("../../testdata/cdn/go.html") suite.Nil(err) return ioutil.NopCloser(strings.NewReader(string(content))), nil }).AnyTimes() storageMgr.EXPECT().ReadDownloadFile(partialNotSupportRangeCache.taskID).DoAndReturn( func(taskID string) (io.ReadCloser, error) { content, err := ioutil.ReadFile("../../testdata/cdn/go.html") suite.Nil(err) return ioutil.NopCloser(strings.NewReader(string(content))), nil }).AnyTimes() storageMgr.EXPECT().ReadDownloadFile(partialSupportRangeCache.taskID).DoAndReturn( func(taskID string) (io.ReadCloser, error) { content, err := ioutil.ReadFile("../../testdata/cdn/go.html") suite.Nil(err) return ioutil.NopCloser(strings.NewReader(string(content))), nil }).AnyTimes() storageMgr.EXPECT().ReadDownloadFile(noCache.taskID).Return(nil, cdnerrors.ErrFileNotExist{}).AnyTimes() storageMgr.EXPECT().ReadPieceMetaRecords(fullNoExpiredCache.taskID).Return(fullNoExpiredCache.pieces, nil).AnyTimes() storageMgr.EXPECT().ReadPieceMetaRecords(partialNotSupportRangeCache.taskID).Return(partialNotSupportRangeCache.pieces, nil).AnyTimes() storageMgr.EXPECT().ReadPieceMetaRecords(partialSupportRangeCache.taskID).Return(partialSupportRangeCache.pieces, nil).AnyTimes() storageMgr.EXPECT().ReadPieceMetaRecords(noCache.taskID).Return(nil, cdnerrors.ErrFileNotExist{}).AnyTimes() storageMgr.EXPECT().StatDownloadFile(fullNoExpiredCache.taskID).Return(&storedriver.StorageInfo{ Path: "", Size: 9789, CreateTime: time.Time{}, ModTime: time.Time{}, }, nil).AnyTimes() sourceClient.EXPECT().IsExpired(gomock.Any(), expiredAndSupportURL, gomock.Any(), gomock.Any()).Return(true, nil).AnyTimes() sourceClient.EXPECT().IsSupportRange(gomock.Any(), expiredAndSupportURL, gomock.Any()).Return(true, nil).AnyTimes() sourceClient.EXPECT().IsExpired(gomock.Any(), expiredAndNotSupportURL, gomock.Any(), gomock.Any()).Return(true, nil).AnyTimes() sourceClient.EXPECT().IsSupportRange(gomock.Any(), expiredAndNotSupportURL, gomock.Any()).Return(false, nil).AnyTimes() sourceClient.EXPECT().IsExpired(gomock.Any(), noExpiredAndSupportURL, gomock.Any(), gomock.Any()).Return(false, nil).AnyTimes() sourceClient.EXPECT().IsSupportRange(gomock.Any(), noExpiredAndSupportURL, gomock.Any()).Return(true, nil).AnyTimes() sourceClient.EXPECT().IsExpired(gomock.Any(), noExpiredAndNotSupportURL, gomock.Any(), gomock.Any()).Return(false, nil).AnyTimes() sourceClient.EXPECT().IsSupportRange(gomock.Any(), noExpiredAndNotSupportURL, gomock.Any()).Return(false, nil).AnyTimes() } var noCacheTask, partialAndSupportCacheTask, partialAndNotSupportCacheTask, fullCacheExpiredTask, fullCacheNotExpiredTask = "noCache", "partialSupportCache", "partialNotSupportCache", "fullCache", "fullCacheNotExpired" var expiredAndSupportURL, expiredAndNotSupportURL, noExpiredAndSupportURL, noExpiredAndNotSupportURL = "http://expiredsupport.com", "http://expiredNotsupport.com", "http://noexpiredAndsupport.com", "http://noexpiredAndnotsupport.com" type mockData struct { taskID string pieces []*storage.PieceMetaRecord fileMeta *storage.FileMetaData reader io.ReadCloser } var fullNoExpiredCache = mockData{ taskID: fullCacheNotExpiredTask, pieces: fullPieceMetaRecords, fileMeta: newCompletedFileMeta(fullCacheNotExpiredTask, noExpiredAndNotSupportURL, true), } var fullExpiredCache = mockData{ taskID: fullCacheExpiredTask, pieces: fullPieceMetaRecords, fileMeta: newCompletedFileMeta(fullCacheExpiredTask, noExpiredAndSupportURL, true), } var partialSupportRangeCache = mockData{ taskID: partialAndSupportCacheTask, pieces: partialPieceMetaRecords, fileMeta: newPartialFileMeta(partialAndSupportCacheTask, noExpiredAndSupportURL), } var partialNotSupportRangeCache = mockData{ taskID: partialAndNotSupportCacheTask, pieces: partialPieceMetaRecords, fileMeta: newPartialFileMeta(partialAndNotSupportCacheTask, noExpiredAndNotSupportURL), } var noCache = mockData{ taskID: noCacheTask, pieces: nil, fileMeta: nil, } var partialPieceMetaRecords = []*storage.PieceMetaRecord{ { PieceNum: 1, PieceLen: 2000, Md5: "67e186642cc5d1b43713379955af82bd", Range: &rangeutils.Range{ StartIndex: 2000, EndIndex: 3999, }, OriginRange: &rangeutils.Range{ StartIndex: 2000, EndIndex: 3999, }, PieceStyle: 1, }, { PieceNum: 0, PieceLen: 2000, Md5: "4a6cf46821d4fb237bc2179bb5bedfa6", Range: &rangeutils.Range{ StartIndex: 0, EndIndex: 1999, }, OriginRange: &rangeutils.Range{ StartIndex: 0, EndIndex: 1999, }, PieceStyle: 1, }, } var fullPieceMetaRecords = append(partialPieceMetaRecords, &storage.PieceMetaRecord{ PieceNum: 2, PieceLen: 2000, Md5: "5ca91ba695d24ad36f25ea350750c9fe", Range: &rangeutils.Range{ StartIndex: 4000, EndIndex: 5999, }, OriginRange: &rangeutils.Range{ StartIndex: 4000, EndIndex: 5999, }, PieceStyle: 1, }, &storage.PieceMetaRecord{ PieceNum: 3, PieceLen: 2000, Md5: "0408118a35af5084043eabcea19c8695", Range: &rangeutils.Range{ StartIndex: 6000, EndIndex: 7999, }, OriginRange: &rangeutils.Range{ StartIndex: 6000, EndIndex: 7999, }, PieceStyle: 1, }, &storage.PieceMetaRecord{ PieceNum: 4, PieceLen: 1789, Md5: "04de99cd9b578ff0e4a8ed7f382316e0", Range: &rangeutils.Range{ StartIndex: 8000, EndIndex: 9788, }, OriginRange: &rangeutils.Range{ StartIndex: 8000, EndIndex: 9788, }, PieceStyle: 1, }) func newCompletedFileMeta(taskID string, URL string, success bool) *storage.FileMetaData { return &storage.FileMetaData{ TaskID: taskID, TaskURL: URL, PieceSize: 2000, SourceFileLen: 9789, AccessTime: 1624126443284, Interval: 0, CdnFileLength: 9789, SourceRealDigest: "", PieceMd5Sign: "98166bdfebb7b71dd5c6d47492d844f4421d90199641ca11fd8ce3111894115a", ExpireInfo: nil, Finish: true, Success: success, TotalPieceCount: 5, } } func newPartialFileMeta(taskID string, URL string) *storage.FileMetaData { return &storage.FileMetaData{ TaskID: taskID, TaskURL: URL, PieceSize: 2000, SourceFileLen: 9789, AccessTime: 1624126443284, Interval: 0, CdnFileLength: 0, SourceRealDigest: "", PieceMd5Sign: "", ExpireInfo: nil, Finish: false, Success: false, TotalPieceCount: 0, } } func (suite *CacheDetectorTestSuite) TestDetectCache() { type args struct { task *types.SeedTask } tests := []struct { name string args args want *cacheResult wantErr bool }{ { name: "no cache", args: args{ task: &types.SeedTask{ TaskID: noCacheTask, URL: noExpiredAndSupportURL, TaskURL: noExpiredAndSupportURL, }, }, want: nil, wantErr: true, }, { name: "partial cache and support range", args: args{ task: &types.SeedTask{ TaskID: partialAndSupportCacheTask, URL: noExpiredAndSupportURL, TaskURL: noExpiredAndSupportURL, SourceFileLength: 9789, PieceSize: 2000, }, }, want: &cacheResult{ breakPoint: 4000, pieceMetaRecords: partialPieceMetaRecords, fileMetaData: newPartialFileMeta(partialAndSupportCacheTask, noExpiredAndSupportURL), }, wantErr: false, }, { name: "partial cache and not support range", args: args{ task: &types.SeedTask{ TaskID: partialAndNotSupportCacheTask, URL: noExpiredAndNotSupportURL, TaskURL: noExpiredAndNotSupportURL, SourceFileLength: 9789, PieceSize: 2000, }, }, want: nil, wantErr: true, }, { name: "full cache and not expire", args: args{ task: &types.SeedTask{ TaskID: fullCacheNotExpiredTask, URL: noExpiredAndNotSupportURL, TaskURL: noExpiredAndNotSupportURL, SourceFileLength: 9789, PieceSize: 2000, }, }, want: &cacheResult{ breakPoint: -1, pieceMetaRecords: fullPieceMetaRecords, fileMetaData: newCompletedFileMeta(fullCacheNotExpiredTask, noExpiredAndNotSupportURL, true), }, wantErr: false, }, { name: "full cache and expired", args: args{ task: &types.SeedTask{ TaskID: fullCacheExpiredTask, URL: expiredAndSupportURL, TaskURL: expiredAndNotSupportURL, }, }, want: nil, wantErr: true, }, } for _, tt := range tests { suite.Run(tt.name, func() { digest := md5.New() got, err := suite.detector.doDetect(context.Background(), tt.args.task, digest) suite.Equal(tt.want, got) suite.Equal(err != nil, tt.wantErr) }) } } func (suite *CacheDetectorTestSuite) TestParseByReadFile() { type args struct { taskID string metaData *storage.FileMetaData } tests := []struct { name string args args want *cacheResult wantErr bool }{ { name: "partial And SupportCacheTask", args: args{ taskID: partialSupportRangeCache.taskID, metaData: partialSupportRangeCache.fileMeta, }, want: &cacheResult{ breakPoint: 4000, pieceMetaRecords: partialSupportRangeCache.pieces, fileMetaData: partialSupportRangeCache.fileMeta, }, wantErr: false, }, } for _, tt := range tests { suite.Run(tt.name, func() { got, err := suite.detector.parseByReadFile(tt.args.taskID, tt.args.metaData, md5.New()) suite.Equal(tt.want, got) suite.Equal(tt.wantErr, err != nil) }) } } func (suite *CacheDetectorTestSuite) TestParseByReadMetaFile() { type args struct { taskID string fileMetaData *storage.FileMetaData } tests := []struct { name string args args want *cacheResult wantErr bool }{ { name: "parse full cache file meta", args: args{ taskID: fullNoExpiredCache.taskID, fileMetaData: fullNoExpiredCache.fileMeta, }, want: &cacheResult{ breakPoint: -1, pieceMetaRecords: fullNoExpiredCache.pieces, fileMetaData: fullNoExpiredCache.fileMeta, }, wantErr: false, }, } for _, tt := range tests { suite.Run(tt.name, func() { got, err := suite.detector.parseByReadMetaFile(tt.args.taskID, tt.args.fileMetaData) suite.Equal(tt.wantErr, err != nil) suite.Equal(tt.want, got) }) } } func (suite *CacheDetectorTestSuite) TestCheckPieceContent() { content, err := ioutil.ReadFile("../../testdata/cdn/go.html") suite.Nil(err) type args struct { reader io.Reader pieceRecords []*storage.PieceMetaRecord fileMd5 hash.Hash } tests := []struct { name string args args wantFileMd5 string }{ { name: "check partial cache piece content", args: args{ reader: strings.NewReader(string(content)), pieceRecords: partialSupportRangeCache.pieces, fileMd5: md5.New(), }, wantFileMd5: "ddff04669628a76b52d32322e24a9dd8", }, } for _, tt := range tests { suite.Run(tt.name, func() { // sort piece meta records by pieceNum sort.Slice(tt.args.pieceRecords, func(i, j int) bool { return tt.args.pieceRecords[i].PieceNum < tt.args.pieceRecords[j].PieceNum }) for _, pieceRecord := range tt.args.pieceRecords { err := checkPieceContent(tt.args.reader, pieceRecord, tt.args.fileMd5) suite.Nil(err) } suite.Equal(tt.wantFileMd5, digestutils.ToHashString(tt.args.fileMd5)) }) } }