dragonfly/cdnsystem/storedriver/local/local_driver.go

/*
 *     Copyright 2020 The Dragonfly Authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package local

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	"d7y.io/dragonfly/v2/cdnsystem/cdnerrors"
	"d7y.io/dragonfly/v2/cdnsystem/storedriver"
	logger "d7y.io/dragonfly/v2/pkg/dflog"
	"d7y.io/dragonfly/v2/pkg/synclock"
	"d7y.io/dragonfly/v2/pkg/unit"
	"d7y.io/dragonfly/v2/pkg/util/fileutils"
	"d7y.io/dragonfly/v2/pkg/util/statutils"
	"github.com/pkg/errors"
)

// Ensure driver implements the storedriver.Driver interface
var _ storedriver.Driver = (*driver)(nil)

const (
	DiskDriverName   = "disk"
	MemoryDriverName = "memory"
)

var fileLocker = synclock.NewLockerPool()

func init() {
	storedriver.Register(DiskDriverName, NewStorageDriver)
	storedriver.Register(MemoryDriverName, NewStorageDriver)
}

// driver is one of the implementations of storage Driver using local file system.
type driver struct {
	// BaseDir is the dir that local storage driver will store content based on it.
	BaseDir string
}

// NewStorageDriver performs initialization for disk Storage and return a storage Driver.
func NewStorageDriver(cfg *storedriver.Config) (storedriver.Driver, error) {
	return &driver{
		BaseDir: cfg.BaseDir,
	}, nil
}

func (ds *driver) GetTotalSpace() (unit.Bytes, error) {
	path := ds.BaseDir
	lock(path, -1, true)
	defer unLock(path, -1, true)
	return fileutils.GetTotalSpace(path)
}

func (ds *driver) GetHomePath() string {
	return ds.BaseDir
}

func (ds *driver) CreateBaseDir() error {
	return os.MkdirAll(ds.BaseDir, os.ModePerm)
}

func (ds *driver) MoveFile(src string, dst string) error {
	return fileutils.MoveFile(src, dst)
}

// Get the content of key from storage and return in io stream.
func (ds *driver) Get(raw *storedriver.Raw) (io.ReadCloser, error) {
	path, info, err := ds.statPath(raw.Bucket, raw.Key)
	if err != nil {
		return nil, err
	}

	if err := storedriver.CheckGetRaw(raw, info.Size()); err != nil {
		return nil, err
	}

	r, w := io.Pipe()
	go func(w *io.PipeWriter) {
		defer w.Close()

		lock(path, raw.Offset, true)
		defer unLock(path, raw.Offset, true)

		f, err := os.Open(path)
		if err != nil {
			return
		}
		defer func() {
			if err := f.Close(); err != nil {
				logger.Error("failed to close file %s: %v", f, err)
			}
		}()

		if _, err := f.Seek(raw.Offset, io.SeekStart); err != nil {
			logger.Errorf("failed to seek file %s: %v", f, err)
		}
		var reader io.Reader
		reader = f
		if raw.Length > 0 {
			reader = io.LimitReader(f, raw.Length)
		}
		buf := make([]byte, 256*1024)
		if _, err := io.CopyBuffer(w, reader, buf); err != nil {
			logger.Errorf("failed to copy buffer from file %s: %v", f, err)
		}
	}(w)
	return r, nil
}

// GetBytes gets the content of key from storage and return in bytes.
func (ds *driver) GetBytes(raw *storedriver.Raw) (data []byte, err error) {
	path, info, err := ds.statPath(raw.Bucket, raw.Key)
	if err != nil {
		return nil, err
	}

	if err := storedriver.CheckGetRaw(raw, info.Size()); err != nil {
		return nil, err
	}

	lock(path, raw.Offset, true)
	defer unLock(path, raw.Offset, true)

	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer func() {
		if err := f.Close(); err != nil {
			logger.Errorf("failed to close file %s: %v", f, err)
		}
	}()

	if _, err := f.Seek(raw.Offset, io.SeekStart); err != nil {
		return nil, err
	}
	if raw.Length == 0 {
		data, err = ioutil.ReadAll(f)
	} else {
		data = make([]byte, raw.Length)
		_, err = f.Read(data)
	}
	if err != nil {
		return nil, err
	}
	return data, nil
}

// Put reads the content from reader and put it into storage.
func (ds *driver) Put(raw *storedriver.Raw, data io.Reader) error {
	if err := storedriver.CheckPutRaw(raw); err != nil {
		return err
	}

	path, err := ds.preparePath(raw.Bucket, raw.Key)
	if err != nil {
		return err
	}

	if data == nil {
		return nil
	}

	lock(path, raw.Offset, false)
	defer unLock(path, raw.Offset, false)

	var f *os.File
	if raw.Trunc {
		if err = storedriver.CheckTrunc(raw); err != nil {
			return err
		}
		if f, err = fileutils.OpenFile(path, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0644); err != nil {
			return err
		}
	} else if raw.Append {
		if f, err = fileutils.OpenFile(path, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644); err != nil {
			return err
		}
	} else {
		if f, err = fileutils.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0644); err != nil {
			return err
		}
	}
	defer func() {
		if err := f.Close(); err != nil {
			logger.Errorf("failed to close file %s: %v", f, err)
		}
	}()
	if raw.Trunc {
		if err = f.Truncate(raw.TruncSize); err != nil {
			return err
		}
	}
	if _, err := f.Seek(raw.Offset, io.SeekStart); err != nil {
		return err
	}
	if raw.Length > 0 {
		if _, err = io.CopyN(f, data, raw.Length); err != nil {
			return err
		}
		return nil
	}

	buf := make([]byte, 256*1024)
	if _, err = io.CopyBuffer(f, data, buf); err != nil {
		return err
	}

	return nil
}

// PutBytes puts the content of key from storage with bytes.
func (ds *driver) PutBytes(raw *storedriver.Raw, data []byte) error {
	if err := storedriver.CheckPutRaw(raw); err != nil {
		return err
	}

	path, err := ds.preparePath(raw.Bucket, raw.Key)
	if err != nil {
		return err
	}

	lock(path, raw.Offset, false)
	defer unLock(path, raw.Offset, false)

	var f *os.File
	if raw.Trunc {
		f, err = fileutils.OpenFile(path, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0644)
	} else if raw.Append {
		f, err = fileutils.OpenFile(path, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
	} else {
		f, err = fileutils.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0644)
	}
	if err != nil {
		return err
	}
	defer func() {
		if err := f.Close(); err != nil {
			logger.Errorf("failed to close file %s: %v", f, err)
		}
	}()
	if raw.Trunc {
		if err = f.Truncate(raw.TruncSize); err != nil {
			return err
		}
	}
	if _, err := f.Seek(raw.Offset, io.SeekStart); err != nil {
		return err
	}
	if raw.Length > 0 {
		if _, err := f.Write(data[:raw.Length]); err != nil {
			return err
		}
		return nil
	}
	if _, err := f.Write(data); err != nil {
		return err
	}
	return nil
}

// Stat determines whether the file exists.
func (ds *driver) Stat(raw *storedriver.Raw) (*storedriver.StorageInfo, error) {
	_, fileInfo, err := ds.statPath(raw.Bucket, raw.Key)
	if err != nil {
		return nil, err
	}
	return &storedriver.StorageInfo{
		Path:       filepath.Join(raw.Bucket, raw.Key),
		Size:       fileInfo.Size(),
		CreateTime: statutils.Ctime(fileInfo),
		ModTime:    fileInfo.ModTime(),
	}, nil
}

// Exits if filepath exists, include symbol link
func (ds *driver) Exits(raw *storedriver.Raw) bool {
	filePath := filepath.Join(ds.BaseDir, raw.Bucket, raw.Key)
	return fileutils.PathExist(filePath)
}

// Remove delete a file or dir.
// It will force delete the file or dir when the raw.Trunc is true.
func (ds *driver) Remove(raw *storedriver.Raw) error {
	path, info, err := ds.statPath(raw.Bucket, raw.Key)
	if err != nil {
		return err
	}

	lock(path, -1, false)
	defer unLock(path, -1, false)

	if raw.Trunc || !info.IsDir() {
		return os.RemoveAll(path)
	}
	empty, err := fileutils.IsEmptyDir(path)
	if empty {
		return os.RemoveAll(path)
	}
	return err
}

// GetAvailSpace returns the available disk space in Byte.
func (ds *driver) GetAvailSpace() (unit.Bytes, error) {
	path := ds.BaseDir
	lock(path, -1, true)
	defer unLock(path, -1, true)
	return fileutils.GetFreeSpace(path)
}

func (ds *driver) GetTotalAndFreeSpace() (unit.Bytes, unit.Bytes, error) {
	path := ds.BaseDir
	lock(path, -1, true)
	defer unLock(path, -1, true)
	return fileutils.GetTotalAndFreeSpace(path)
}

// Walk walks the file tree rooted at root which determined by raw.Bucket and raw.Key,
// calling walkFn for each file or directory in the tree, including root.
func (ds *driver) Walk(raw *storedriver.Raw) error {
	path, _, err := ds.statPath(raw.Bucket, raw.Key)
	if err != nil {
		return err
	}

	lock(path, -1, true)
	defer unLock(path, -1, true)

	return filepath.Walk(path, raw.WalkFn)
}

func (ds *driver) GetPath(raw *storedriver.Raw) string {
	return filepath.Join(ds.BaseDir, raw.Bucket, raw.Key)
}

// helper function

// preparePath gets the target path and creates the upper directory if it does not exist.
func (ds *driver) preparePath(bucket, key string) (string, error) {
	dir := filepath.Join(ds.BaseDir, bucket)
	if err := fileutils.MkdirAll(dir); err != nil {
		return "", err
	}
	target := filepath.Join(dir, key)
	return target, nil
}

// statPath determines whether the target file exists and returns an fileMutex if so.
func (ds *driver) statPath(bucket, key string) (string, os.FileInfo, error) {
	filePath := filepath.Join(ds.BaseDir, bucket, key)
	f, err := os.Stat(filePath)
	if err != nil {
		if os.IsNotExist(err) {
			return "", nil, errors.Wrapf(cdnerrors.ErrFileNotExist, "no such file or directory:%s exists", filePath)
		}
		return "", nil, err
	}
	return filePath, f, nil
}

func lock(path string, offset int64, ro bool) {
	if offset != -1 {
		fileLocker.Lock(LockKey(path, -1), true)
	}

	fileLocker.Lock(LockKey(path, offset), ro)
}

func unLock(path string, offset int64, ro bool) {
	if offset != -1 {
		fileLocker.UnLock(LockKey(path, -1), true)
	}

	fileLocker.UnLock(LockKey(path, offset), ro)
}

func LockKey(path string, offset int64) string {
	return fmt.Sprintf("%s:%d", path, offset)
}