297 lines
10 KiB
Go
297 lines
10 KiB
Go
/*
|
||
* Copyright 2020 The Dragonfly Authors
|
||
*
|
||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||
* you may not use this file except in compliance with the License.
|
||
* You may obtain a copy of the License at
|
||
*
|
||
* http://www.apache.org/licenses/LICENSE-2.0
|
||
*
|
||
* Unless required by applicable law or agreed to in writing, software
|
||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
* See the License for the specific language governing permissions and
|
||
* limitations under the License.
|
||
*/
|
||
//go:generate mockgen -destination ./mock/mock_storage_mgr.go -package mock d7y.io/dragonfly/v2/cdnsystem/supervisor/cdn/storage Manager
|
||
|
||
package storage
|
||
|
||
import (
|
||
"fmt"
|
||
"io"
|
||
"reflect"
|
||
"strconv"
|
||
"strings"
|
||
"time"
|
||
|
||
"d7y.io/dragonfly/v2/cdnsystem/plugins"
|
||
"d7y.io/dragonfly/v2/cdnsystem/storedriver"
|
||
"d7y.io/dragonfly/v2/cdnsystem/supervisor"
|
||
"d7y.io/dragonfly/v2/cdnsystem/types"
|
||
"d7y.io/dragonfly/v2/pkg/unit"
|
||
"d7y.io/dragonfly/v2/pkg/util/rangeutils"
|
||
"github.com/mitchellh/mapstructure"
|
||
"github.com/pkg/errors"
|
||
"gopkg.in/yaml.v3"
|
||
)
|
||
|
||
type Manager interface {
|
||
Initialize(taskMgr supervisor.SeedTaskMgr)
|
||
|
||
// ResetRepo reset the storage of task
|
||
ResetRepo(*types.SeedTask) error
|
||
|
||
// StatDownloadFile stat download file info
|
||
StatDownloadFile(taskID string) (*storedriver.StorageInfo, error)
|
||
|
||
// WriteDownloadFile write data to download file
|
||
WriteDownloadFile(taskID string, offset int64, len int64, data io.Reader) error
|
||
|
||
// ReadDownloadFile return reader of download file
|
||
ReadDownloadFile(taskID string) (io.ReadCloser, error)
|
||
|
||
// CreateUploadLink create a upload link to download file
|
||
CreateUploadLink(taskID string) error
|
||
|
||
// ReadFileMetaData return meta data of download file
|
||
ReadFileMetaData(taskID string) (*FileMetaData, error)
|
||
|
||
// WriteFileMetaData write file meta to storage
|
||
WriteFileMetaData(taskID string, meta *FileMetaData) error
|
||
|
||
// WritePieceMetaRecords write piece meta records to storage
|
||
WritePieceMetaRecords(taskID string, metaRecords []*PieceMetaRecord) error
|
||
|
||
// AppendPieceMetaData append piece meta data to storage
|
||
AppendPieceMetaData(taskID string, metaRecord *PieceMetaRecord) error
|
||
|
||
// ReadPieceMetaRecords read piece meta records from storage
|
||
ReadPieceMetaRecords(taskID string) ([]*PieceMetaRecord, error)
|
||
|
||
// DeleteTask delete task from storage
|
||
DeleteTask(taskID string) error
|
||
|
||
// TryFreeSpace checks if there is enough space for the file, return true while we are sure that there is enough space.
|
||
TryFreeSpace(fileLength int64) (bool, error)
|
||
}
|
||
|
||
// FileMetaData meta data of task
|
||
type FileMetaData struct {
|
||
TaskID string `json:"taskId"`
|
||
TaskURL string `json:"taskUrl"`
|
||
PieceSize int32 `json:"pieceSize"`
|
||
SourceFileLen int64 `json:"sourceFileLen"`
|
||
AccessTime int64 `json:"accessTime"`
|
||
Interval int64 `json:"interval"`
|
||
CdnFileLength int64 `json:"cdnFileLength"`
|
||
SourceRealDigest string `json:"sourceRealDigest"`
|
||
PieceMd5Sign string `json:"pieceMd5Sign"`
|
||
ExpireInfo map[string]string `json:"expireInfo"`
|
||
Finish bool `json:"finish"`
|
||
Success bool `json:"success"`
|
||
TotalPieceCount int32 `json:"totalPieceCount"`
|
||
//PieceMetaDataSign string `json:"pieceMetaDataSign"`
|
||
}
|
||
|
||
// PieceMetaRecord meta data of piece
|
||
type PieceMetaRecord struct {
|
||
PieceNum int32 `json:"pieceNum"` // piece Num start from 0
|
||
PieceLen int32 `json:"pieceLen"` // 存储到存储介质的真实长度
|
||
Md5 string `json:"md5"` // for transported piece content，不是origin source 的 md5，是真是存储到存储介质后的md5（为了读取数据文件时方便校验完整性）
|
||
Range *rangeutils.Range `json:"range"` // 下载存储到磁盘的range，不是origin source的range.提供给客户端发送下载请求,for transported piece content
|
||
OriginRange *rangeutils.Range `json:"originRange"` // piece's real offset in the file
|
||
PieceStyle types.PieceFormat `json:"pieceStyle"` // 1: PlainUnspecified
|
||
}
|
||
|
||
const fieldSeparator = ":"
|
||
|
||
func (record PieceMetaRecord) String() string {
|
||
return fmt.Sprint(record.PieceNum, fieldSeparator, record.PieceLen, fieldSeparator, record.Md5, fieldSeparator, record.Range, fieldSeparator,
|
||
record.OriginRange, fieldSeparator, record.PieceStyle)
|
||
}
|
||
|
||
func ParsePieceMetaRecord(value string) (record *PieceMetaRecord, err error) {
|
||
defer func() {
|
||
if msg := recover(); msg != nil {
|
||
err = errors.Errorf("%v", msg)
|
||
}
|
||
}()
|
||
fields := strings.Split(value, fieldSeparator)
|
||
pieceNum, err := strconv.ParseInt(fields[0], 10, 32)
|
||
if err != nil {
|
||
return nil, errors.Wrapf(err, "invalid pieceNum: %s", fields[0])
|
||
}
|
||
pieceLen, err := strconv.ParseInt(fields[1], 10, 32)
|
||
if err != nil {
|
||
return nil, errors.Wrapf(err, "invalid pieceLen: %s", fields[1])
|
||
}
|
||
md5 := fields[2]
|
||
pieceRange, err := rangeutils.ParseRange(fields[3])
|
||
if err != nil {
|
||
return nil, errors.Wrapf(err, "invalid piece range: %s", fields[3])
|
||
}
|
||
originRange, err := rangeutils.ParseRange(fields[4])
|
||
if err != nil {
|
||
return nil, errors.Wrapf(err, "invalid origin range: %s", fields[4])
|
||
}
|
||
pieceStyle, err := strconv.ParseInt(fields[5], 10, 8)
|
||
if err != nil {
|
||
return nil, errors.Wrapf(err, "invalid pieceStyle: %s", fields[5])
|
||
}
|
||
return &PieceMetaRecord{
|
||
PieceNum: int32(pieceNum),
|
||
PieceLen: int32(pieceLen),
|
||
Md5: md5,
|
||
Range: pieceRange,
|
||
OriginRange: originRange,
|
||
PieceStyle: types.PieceFormat(pieceStyle),
|
||
}, nil
|
||
}
|
||
|
||
type managerPlugin struct {
|
||
// name is a unique identifier, you can also name it ID.
|
||
name string
|
||
// instance holds a manger instant which implements the interface of Manager.
|
||
instance Manager
|
||
}
|
||
|
||
func (m *managerPlugin) Type() plugins.PluginType {
|
||
return plugins.StorageManagerPlugin
|
||
}
|
||
|
||
func (m *managerPlugin) Name() string {
|
||
return m.name
|
||
}
|
||
|
||
func (m *managerPlugin) ResetRepo(task *types.SeedTask) error {
|
||
return m.instance.ResetRepo(task)
|
||
}
|
||
|
||
func (m *managerPlugin) StatDownloadFile(path string) (*storedriver.StorageInfo, error) {
|
||
return m.instance.StatDownloadFile(path)
|
||
}
|
||
|
||
func (m *managerPlugin) WriteDownloadFile(taskID string, offset int64, len int64, data io.Reader) error {
|
||
return m.instance.WriteDownloadFile(taskID, offset, len, data)
|
||
}
|
||
|
||
func (m *managerPlugin) ReadDownloadFile(taskID string) (io.ReadCloser, error) {
|
||
return m.instance.ReadDownloadFile(taskID)
|
||
}
|
||
|
||
func (m *managerPlugin) CreateUploadLink(taskID string) error {
|
||
return m.instance.CreateUploadLink(taskID)
|
||
}
|
||
|
||
func (m *managerPlugin) ReadFileMetaData(taskID string) (*FileMetaData, error) {
|
||
return m.instance.ReadFileMetaData(taskID)
|
||
}
|
||
|
||
func (m *managerPlugin) WriteFileMetaData(taskID string, data *FileMetaData) error {
|
||
return m.instance.WriteFileMetaData(taskID, data)
|
||
}
|
||
|
||
func (m *managerPlugin) WritePieceMetaRecords(taskID string, records []*PieceMetaRecord) error {
|
||
return m.instance.WritePieceMetaRecords(taskID, records)
|
||
}
|
||
|
||
func (m *managerPlugin) AppendPieceMetaData(taskID string, record *PieceMetaRecord) error {
|
||
return m.instance.AppendPieceMetaData(taskID, record)
|
||
}
|
||
|
||
func (m *managerPlugin) ReadPieceMetaRecords(taskID string) ([]*PieceMetaRecord, error) {
|
||
return m.instance.ReadPieceMetaRecords(taskID)
|
||
}
|
||
|
||
func (m *managerPlugin) DeleteTask(taskID string) error {
|
||
return m.instance.DeleteTask(taskID)
|
||
}
|
||
|
||
// ManagerBuilder is a function that creates a new storage manager plugin instant with the giving conf.
|
||
type ManagerBuilder func(cfg *Config) (Manager, error)
|
||
|
||
// Register defines an interface to register a storage manager with specified name.
|
||
// All storage managers should call this function to register itself to the storage manager factory.
|
||
func Register(name string, builder ManagerBuilder) error {
|
||
name = strings.ToLower(name)
|
||
// plugin builder
|
||
var f = func(conf interface{}) (plugins.Plugin, error) {
|
||
cfg := &Config{}
|
||
decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
|
||
DecodeHook: mapstructure.ComposeDecodeHookFunc(func(from, to reflect.Type, v interface{}) (interface{}, error) {
|
||
switch to {
|
||
case reflect.TypeOf(unit.B),
|
||
reflect.TypeOf(time.Second):
|
||
b, _ := yaml.Marshal(v)
|
||
p := reflect.New(to)
|
||
if err := yaml.Unmarshal(b, p.Interface()); err != nil {
|
||
return nil, err
|
||
}
|
||
return p.Interface(), nil
|
||
default:
|
||
return v, nil
|
||
}
|
||
}),
|
||
Result: cfg,
|
||
})
|
||
if err != nil {
|
||
return nil, fmt.Errorf("create decoder: %v", err)
|
||
}
|
||
err = decoder.Decode(conf)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("parse config: %v", err)
|
||
}
|
||
return newManagerPlugin(name, builder, cfg)
|
||
}
|
||
return plugins.RegisterPluginBuilder(plugins.StorageManagerPlugin, name, f)
|
||
}
|
||
|
||
func newManagerPlugin(name string, builder ManagerBuilder, cfg *Config) (plugins.Plugin, error) {
|
||
if name == "" || builder == nil {
|
||
return nil, fmt.Errorf("storage manager plugin's name and builder cannot be nil")
|
||
}
|
||
|
||
instant, err := builder(cfg)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("init storage manager %s: %v", name, err)
|
||
}
|
||
|
||
return &managerPlugin{
|
||
name: name,
|
||
instance: instant,
|
||
}, nil
|
||
}
|
||
|
||
// Get a storage manager from manager with specified name.
|
||
func Get(name string) (Manager, bool) {
|
||
v, ok := plugins.GetPlugin(plugins.StorageManagerPlugin, strings.ToLower(name))
|
||
if !ok {
|
||
return nil, false
|
||
}
|
||
return v.(*managerPlugin).instance, true
|
||
}
|
||
|
||
type Config struct {
|
||
GCInitialDelay time.Duration `yaml:"gcInitialDelay"`
|
||
GCInterval time.Duration `yaml:"gcInterval"`
|
||
DriverConfigs map[string]*DriverConfig `yaml:"driverConfigs"`
|
||
}
|
||
|
||
type DriverConfig struct {
|
||
GCConfig *GCConfig `yaml:"gcConfig"`
|
||
}
|
||
|
||
// GCConfig gc config
|
||
type GCConfig struct {
|
||
YoungGCThreshold unit.Bytes `yaml:"youngGCThreshold"`
|
||
FullGCThreshold unit.Bytes `yaml:"fullGCThreshold"`
|
||
CleanRatio int `yaml:"cleanRatio"`
|
||
IntervalThreshold time.Duration `yaml:"intervalThreshold"`
|
||
}
|
||
|
||
const (
|
||
HybridStorageMode = "hybrid"
|
||
DiskStorageMode = "disk"
|
||
)
|