dragonfly/cdnsystem/storedriver/local/local_driver_test.go

789 lines
16 KiB
Go

/*
* Copyright 2020 The Dragonfly Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package local
import (
"context"
"io"
"io/ioutil"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"d7y.io/dragonfly/v2/cdnsystem/cdnerrors"
"d7y.io/dragonfly/v2/cdnsystem/storedriver"
"d7y.io/dragonfly/v2/pkg/unit"
"d7y.io/dragonfly/v2/pkg/util/statutils"
"github.com/stretchr/testify/suite"
)
func TestStorageSuite(t *testing.T) {
suite.Run(t, new(StorageTestSuite))
}
type StorageTestSuite struct {
workHome string
storedriver.Driver
suite.Suite
}
func (s *StorageTestSuite) SetupSuite() {
s.workHome, _ = ioutil.TempDir("/tmp", "cdn-StoreTestSuite-repo")
store, err := NewStorageDriver(&storedriver.Config{
BaseDir: "/tmp/download"})
s.Nil(err)
s.NotNil(store)
s.Driver = store
}
func (s *StorageTestSuite) TeardownSuite() {
if s.workHome != "" {
os.RemoveAll(s.workHome)
}
}
func (s *StorageTestSuite) TestGetPutBytes() {
var cases = []struct {
name string
putRaw *storedriver.Raw
getRaw *storedriver.Raw
data []byte
getErrCheck func(error) bool
expected string
}{
{
putRaw: &storedriver.Raw{
Bucket: "GetPut",
Key: "foo1",
},
getRaw: &storedriver.Raw{
Bucket: "GetPut",
Key: "foo1",
},
data: []byte("hello foo"),
getErrCheck: isNilError,
expected: "hello foo",
}, {
putRaw: &storedriver.Raw{
Bucket: "GetPut",
Key: "foo2",
},
getRaw: &storedriver.Raw{
Bucket: "GetPut",
Key: "foo2",
Offset: 0,
Length: 5,
},
getErrCheck: isNilError,
data: []byte("hello foo"),
expected: "hello",
}, {
putRaw: &storedriver.Raw{
Bucket: "GetPut",
Key: "foo3",
},
getRaw: &storedriver.Raw{
Bucket: "GetPut",
Key: "foo3",
Offset: 0,
Length: 0,
},
getErrCheck: isNilError,
data: []byte("hello foo"),
expected: "hello foo",
}, {
putRaw: &storedriver.Raw{
Bucket: "GetPut",
Key: "foo4",
},
getRaw: &storedriver.Raw{
Bucket: "GetPut",
Key: "foo4",
Offset: 0,
Length: -1,
},
getErrCheck: cdnerrors.IsInvalidValue,
data: []byte("hello foo"),
expected: "",
}, {
putRaw: &storedriver.Raw{
Bucket: "GetPut",
Key: "foo5",
Length: 5,
},
getRaw: &storedriver.Raw{
Bucket: "GetPut",
Key: "foo5",
},
getErrCheck: isNilError,
data: []byte("hello foo"),
expected: "hello",
}, {
putRaw: &storedriver.Raw{
Bucket: "GetPut",
Key: "foo6",
},
getRaw: &storedriver.Raw{
Bucket: "GetPut",
Key: "foo6",
Offset: -1,
},
data: []byte("hello foo"),
getErrCheck: cdnerrors.IsInvalidValue,
expected: "",
}, {
name: "offset不从0开始",
putRaw: &storedriver.Raw{
Bucket: "GetPut",
Key: "foo7",
Offset: 3,
},
getRaw: &storedriver.Raw{
Bucket: "GetPut",
Key: "foo7",
Offset: 3,
},
data: []byte("hello foo"),
getErrCheck: isNilError,
expected: "hello foo",
},
}
for _, v := range cases {
s.Run(v.name, func() {
// put
err := s.PutBytes(v.putRaw, v.data)
s.Nil(err)
// get
result, err := s.GetBytes(v.getRaw)
s.True(v.getErrCheck(err))
s.Equal(v.expected, string(result))
// stat
s.checkStat(v.getRaw)
// remove
s.checkRemove(v.putRaw)
})
}
}
func (s *StorageTestSuite) TestGetPut() {
var cases = []struct {
name string
putRaw *storedriver.Raw
getRaw *storedriver.Raw
data io.Reader
getErrCheck func(error) bool
expected string
}{
{
putRaw: &storedriver.Raw{
Key: "foo0.meta",
Length: 15,
},
getRaw: &storedriver.Raw{
Key: "foo0.meta",
},
data: strings.NewReader("hello meta file"),
getErrCheck: isNilError,
expected: "hello meta file",
}, {
putRaw: &storedriver.Raw{
Key: "foo1.meta",
},
getRaw: &storedriver.Raw{
Key: "foo1.meta",
},
data: strings.NewReader("hello meta file"),
getErrCheck: isNilError,
expected: "hello meta file",
}, {
putRaw: &storedriver.Raw{
Key: "foo2.meta",
Length: 6,
},
getRaw: &storedriver.Raw{
Key: "foo2.meta",
},
data: strings.NewReader("hello meta file"),
getErrCheck: isNilError,
expected: "hello ",
}, {
putRaw: &storedriver.Raw{
Key: "foo3.meta",
},
getRaw: &storedriver.Raw{
Key: "foo3.meta",
Offset: 2,
Length: 5,
},
data: strings.NewReader("hello meta file"),
getErrCheck: isNilError,
expected: "llo m",
}, {
putRaw: &storedriver.Raw{
Key: "foo4.meta",
},
getRaw: &storedriver.Raw{
Key: "foo4.meta",
Offset: 2,
Length: -1,
},
getErrCheck: cdnerrors.IsInvalidValue,
data: strings.NewReader("hello meta file"),
expected: "",
}, {
putRaw: &storedriver.Raw{
Key: "foo5.meta",
},
getRaw: &storedriver.Raw{
Key: "foo5.meta",
Offset: 30,
Length: 5,
},
getErrCheck: cdnerrors.IsInvalidValue,
data: strings.NewReader("hello meta file"),
expected: "",
},
}
for _, v := range cases {
s.Run(v.name, func() {
// put
err := s.Put(v.putRaw, v.data)
s.Nil(err)
// get
r, err := s.Get(v.getRaw)
s.True(v.getErrCheck(err))
if err == nil {
result, err := ioutil.ReadAll(r)
s.Nil(err)
s.Equal(v.expected, string(result))
}
// stat
s.checkStat(v.putRaw)
// remove
s.checkRemove(v.putRaw)
})
}
}
func (s *StorageTestSuite) TestAppendBytes() {
var cases = []struct {
name string
putRaw *storedriver.Raw
appendRaw *storedriver.Raw
getRaw *storedriver.Raw
data io.Reader
appData io.Reader
getErrCheck func(error) bool
expected string
}{
{
putRaw: &storedriver.Raw{
Key: "foo0.meta",
Length: 13,
},
appendRaw: &storedriver.Raw{
Key: "foo0.meta",
Offset: 2,
Length: 3,
Append: true,
},
getRaw: &storedriver.Raw{
Key: "foo0.meta",
},
data: strings.NewReader("hello meta fi"),
appData: strings.NewReader("append data"),
getErrCheck: isNilError,
expected: "hello meta fiapp",
}, {
putRaw: &storedriver.Raw{
Key: "foo1.meta",
},
appendRaw: &storedriver.Raw{
Key: "foo1.meta",
Offset: 29,
Length: 3,
Append: true,
},
getRaw: &storedriver.Raw{
Key: "foo1.meta",
},
data: strings.NewReader("hello meta file"),
appData: strings.NewReader("append data"),
getErrCheck: isNilError,
expected: "hello meta fileapp",
},
//{
// putRaw: &store.Raw{
// Key: "foo2.meta",
// Length: 6,
// },
// appendRaw: &store.Raw{
// Key: "foo0.meta",
// Offset: 29,
// Length: 3,
// Append: true,
// },
// getRaw: &store.Raw{
// Key: "foo2.meta",
// },
// data: strings.NewReader("hello meta file"),
// getErrCheck: isNilError,
// expected: "hello ",
//}, {
// putRaw: &store.Raw{
// Key: "foo3.meta",
// },
// getRaw: &store.Raw{
// Key: "foo3.meta",
// Offset: 2,
// Length: 5,
// },
// data: strings.NewReader("hello meta file"),
// getErrCheck: isNilError,
// expected: "llo m",
//}, {
// putRaw: &store.Raw{
// Key: "foo4.meta",
// },
// getRaw: &store.Raw{
// Key: "foo4.meta",
// Offset: 2,
// Length: -1,
// },
// getErrCheck: cdnerrors.IsInvalidValue,
// data: strings.NewReader("hello meta file"),
// expected: "",
//}, {
// putRaw: &store.Raw{
// Key: "foo5.meta",
// },
// getRaw: &store.Raw{
// Key: "foo5.meta",
// Offset: 30,
// Length: 5,
// },
// getErrCheck: cdnerrors.IsInvalidValue,
// data: strings.NewReader("hello meta file"),
// expected: "",
//},
}
for _, v := range cases {
s.Run(v.name, func() {
// put
err := s.Put(v.putRaw, v.data)
s.Nil(err)
err = s.Put(v.appendRaw, v.appData)
s.Nil(err)
// get
r, err := s.Get(v.getRaw)
s.True(v.getErrCheck(err))
if err == nil {
result, err := ioutil.ReadAll(r)
s.Nil(err)
s.Equal(v.expected, string(result))
}
// stat
s.checkStat(v.putRaw)
// remove
s.checkRemove(v.putRaw)
})
}
}
func (s *StorageTestSuite) TestPutTrunc() {
originRaw := &storedriver.Raw{
Key: "fooTrunc.meta",
Offset: 0,
Trunc: true,
}
originData := "hello world"
var cases = []struct {
truncRaw *storedriver.Raw
getErrCheck func(error) bool
data io.Reader
expectedData string
}{
{
truncRaw: &storedriver.Raw{
Key: "fooTrunc.meta",
Offset: 0,
Trunc: true,
},
data: strings.NewReader("hello"),
getErrCheck: isNilError,
expectedData: "hello",
}, {
truncRaw: &storedriver.Raw{
Key: "fooTrunc.meta",
Offset: 6,
Trunc: true,
},
data: strings.NewReader("golang"),
getErrCheck: isNilError,
expectedData: "\x00\x00\x00\x00\x00\x00golang",
}, {
truncRaw: &storedriver.Raw{
Key: "fooTrunc.meta",
Offset: 0,
Trunc: false,
},
data: strings.NewReader("foo"),
getErrCheck: isNilError,
expectedData: "foolo world",
}, {
truncRaw: &storedriver.Raw{
Key: "fooTrunc.meta",
Offset: 6,
Trunc: false,
},
data: strings.NewReader("foo"),
getErrCheck: isNilError,
expectedData: "hello foold",
},
}
for _, v := range cases {
err := s.Put(originRaw, strings.NewReader(originData))
s.Nil(err)
err = s.Put(v.truncRaw, v.data)
s.Nil(err)
r, err := s.Get(&storedriver.Raw{
Key: "fooTrunc.meta",
})
s.Nil(err)
if err == nil {
result, err := ioutil.ReadAll(r)
s.Nil(err)
s.Equal(string(result[:]), v.expectedData)
}
}
}
func (s *StorageTestSuite) TestPutParallel() {
var key = "fooPutParallel"
var routineCount = 4
var testStr = "hello"
var testStrLength = len(testStr)
var wg sync.WaitGroup
for k := 0; k < routineCount; k++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
s.Put(&storedriver.Raw{
Key: key,
Offset: int64(i) * int64(testStrLength),
}, strings.NewReader(testStr))
}(k)
}
wg.Wait()
info, err := s.Stat(&storedriver.Raw{Key: key})
s.Nil(err)
s.Equal(info.Size, int64(routineCount)*int64(testStrLength))
}
func (s *StorageTestSuite) TestRemove() {
type fields struct {
BaseDir string
}
type args struct {
raw *storedriver.Raw
}
tests := []struct {
name string
fields fields
args args
wantErr bool
}{
{},
}
for _, tt := range tests {
err := s.Remove(tt.args.raw)
s.Equal(err != nil, tt.wantErr)
}
}
func (s *StorageTestSuite) TestStat() {
type fields struct {
BaseDir string
}
type args struct {
raw *storedriver.Raw
}
tests := []struct {
name string
fields fields
args args
want *storedriver.StorageInfo
wantErr bool
}{
{},
}
for _, tt := range tests {
got, err := s.Stat(tt.args.raw)
s.Equal(err, tt.wantErr)
s.EqualValues(got, tt.want)
}
}
func Test_diskStorage_CreateBaseDir(t *testing.T) {
type fields struct {
BaseDir string
}
type args struct {
ctx context.Context
}
tests := []struct {
name string
fields fields
args args
wantErr bool
}{
// TODO: Add test cases.
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ds := &driver{
BaseDir: tt.fields.BaseDir,
}
if err := ds.CreateBaseDir(); (err != nil) != tt.wantErr {
t.Errorf("CreateBaseDir() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
func Test_diskStorage_Exits(t *testing.T) {
type fields struct {
BaseDir string
}
type args struct {
ctx context.Context
raw *storedriver.Raw
}
tests := []struct {
name string
fields fields
args args
want bool
}{
// TODO: Add test cases.
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ds := &driver{
BaseDir: tt.fields.BaseDir,
}
if got := ds.Exits(tt.args.raw); got != tt.want {
t.Errorf("Exits() = %v, want %v", got, tt.want)
}
})
}
}
func Test_diskStorage_GetHomePath(t *testing.T) {
type fields struct {
BaseDir string
}
type args struct {
ctx context.Context
}
tests := []struct {
name string
fields fields
args args
want string
}{
// TODO: Add test cases.
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ds := &driver{
BaseDir: tt.fields.BaseDir,
}
if got := ds.GetHomePath(); got != tt.want {
t.Errorf("GetHomePath() = %v, want %v", got, tt.want)
}
})
}
}
func Test_diskStorage_GetPath(t *testing.T) {
type fields struct {
BaseDir string
}
type args struct {
raw *storedriver.Raw
}
tests := []struct {
name string
fields fields
args args
want string
}{
// TODO: Add test cases.
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ds := &driver{
BaseDir: tt.fields.BaseDir,
}
if got := ds.GetPath(tt.args.raw); got != tt.want {
t.Errorf("GetPath() = %v, want %v", got, tt.want)
}
})
}
}
func Test_diskStorage_GetTotalAndFreeSpace(t *testing.T) {
type fields struct {
BaseDir string
}
type args struct {
ctx context.Context
}
tests := []struct {
name string
fields fields
args args
want unit.Bytes
want1 unit.Bytes
wantErr bool
}{
// TODO: Add test cases.
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ds := &driver{
BaseDir: tt.fields.BaseDir,
}
got, got1, err := ds.GetTotalAndFreeSpace()
if (err != nil) != tt.wantErr {
t.Errorf("GetTotalAndFreeSpace() error = %v, wantErr %v", err, tt.wantErr)
return
}
if got != tt.want {
t.Errorf("GetTotalAndFreeSpace() got = %v, want %v", got, tt.want)
}
if got1 != tt.want1 {
t.Errorf("GetTotalAndFreeSpace() got1 = %v, want %v", got1, tt.want1)
}
})
}
}
func Test_diskStorage_GetTotalSpace(t *testing.T) {
type fields struct {
BaseDir string
}
type args struct {
ctx context.Context
}
tests := []struct {
name string
fields fields
args args
want unit.Bytes
wantErr bool
}{
// TODO: Add test cases.
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ds := &driver{
BaseDir: tt.fields.BaseDir,
}
got, err := ds.GetTotalSpace()
if (err != nil) != tt.wantErr {
t.Errorf("GetTotalSpace() error = %v, wantErr %v", err, tt.wantErr)
return
}
if got != tt.want {
t.Errorf("GetTotalSpace() got = %v, want %v", got, tt.want)
}
})
}
}
func (s *StorageTestSuite) BenchmarkPutParallel() {
var wg sync.WaitGroup
for k := 0; k < 1000; k++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
s.Put(&storedriver.Raw{
Key: "foo.bech",
Offset: int64(i) * 5,
}, strings.NewReader("hello"))
}(k)
}
wg.Wait()
}
func (s *StorageTestSuite) BenchmarkPutSerial() {
for k := 0; k < 1000; k++ {
s.Put(&storedriver.Raw{
Key: "foo1.bech",
Offset: int64(k) * 5,
}, strings.NewReader("hello"))
}
}
// -----------------------------------------------------------------------------
// helper function
func (s *StorageTestSuite) checkStat(raw *storedriver.Raw) {
info, err := s.Stat(raw)
s.Equal(isNilError(err), true)
pathTemp := filepath.Join(s.workHome, raw.Bucket, raw.Key)
f, _ := os.Stat(pathTemp)
s.EqualValues(info, &storedriver.StorageInfo{
Path: filepath.Join(raw.Bucket, raw.Key),
Size: f.Size(),
ModTime: f.ModTime(),
CreateTime: statutils.Ctime(f),
})
}
func (s *StorageTestSuite) checkRemove(raw *storedriver.Raw) {
err := s.Remove(raw)
s.Equal(isNilError(err), true)
_, err = s.Stat(raw)
s.Equal(cdnerrors.IsFileNotExist(err), true)
}
func isNilError(err error) bool {
return err == nil
}