dragonfly/client/config/dfget.go

/*
 *     Copyright 2020 The Dragonfly Authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Package config holds all DaemonConfig of dfget.
package config

import (
	"encoding/json"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strings"
	"syscall"
	"time"

	"d7y.io/dragonfly/v2/client/util"
	"d7y.io/dragonfly/v2/cmd/dependency/base"
	"d7y.io/dragonfly/v2/internal/dferrors"
	logger "d7y.io/dragonfly/v2/internal/dflog"
	"d7y.io/dragonfly/v2/pkg/net/url"
	"d7y.io/dragonfly/v2/pkg/os/user"
	pkgstrings "d7y.io/dragonfly/v2/pkg/strings"
	"d7y.io/dragonfly/v2/pkg/unit"
)

type DfgetConfig = ClientOption

// ClientOption holds all the runtime config information.
type ClientOption struct {
	base.Options `yaml:",inline" mapstructure:",squash"`
	// URL download URL.
	URL string `yaml:"url,omitempty" mapstructure:"url,omitempty"`

	// Output full output path.
	Output string `yaml:"output,omitempty" mapstructure:"output,omitempty"`

	// Timeout download timeout(second).
	Timeout time.Duration `yaml:"timeout,omitempty" mapstructure:"timeout,omitempty"`

	BenchmarkRate unit.Bytes `yaml:"benchmarkRate,omitempty" mapstructure:"benchmarkRate,omitempty"`

	// Md5 expected file md5.
	// Deprecated: Md5 is deprecated, use DigestMethod with DigestValue instead
	Md5    string `yaml:"md5,omitempty" mapstructure:"md5,omitempty"`
	Digest string `yaml:"digest,omitempty" mapstructure:"digest,omitempty"`
	// DigestMethod indicates digest method, like md5, sha256
	DigestMethod string `yaml:"digestMethod,omitempty" mapstructure:"digestMethod,omitempty"`

	// DigestValue indicates digest value
	DigestValue string `yaml:"digestValue,omitempty" mapstructure:"digestValue,omitempty"`

	// Tag identify download task, it is available merely when md5 param not exist.
	Tag string `yaml:"tag,omitempty" mapstructure:"tag,omitempty"`

	// Application name that executes dfget.
	Application string `yaml:"application,omitempty" mapstructure:"application,omitempty"`

	// DaemonSock is daemon download socket path.
	DaemonSock string `yaml:"daemonSock,omitempty" mapstructure:"daemon-sock,omitempty"`

	// Priority scheduler will schedule tasks according to priority
	Priority int32 `yaml:"priority,omitempty" mapstructure:"priority,omitempty"`

	// CA certificate to verify when supernode interact with the source.
	Cacerts []string `yaml:"cacert,omitempty" mapstructure:"cacert,omitempty"`

	// Filter filter some query params of url, use char '&' to separate different params.
	// eg: -f 'key&sign' will filter 'key' and 'sign' query param.
	// in this way, different urls correspond one same download task that can use p2p mode.
	Filter string `yaml:"filter,omitempty" mapstructure:"filter,omitempty"`

	// Header of http request.
	// eg: --header='Accept: *' --header='Host: abc'.
	Header []string `yaml:"header,omitempty" mapstructure:"header,omitempty"`

	// DisableBackSource indicates whether to not back source to download when p2p fails.
	DisableBackSource bool `yaml:"disableBackSource,omitempty" mapstructure:"disable-back-source,omitempty"`

	// Insecure indicates whether skip secure verify when supernode interact with the source.
	Insecure bool `yaml:"insecure,omitempty" mapstructure:"insecure,omitempty"`

	// ShowProgress shows progress bar, it's conflict with `--console`.
	ShowProgress bool `yaml:"show-progress,omitempty" mapstructure:"show-progress,omitempty"`

	// LogDir is log directory of dfget.
	LogDir string `yaml:"logDir,omitempty" mapstructure:"logDir,omitempty"`

	// WorkHome is working directory of dfget.
	WorkHome string `yaml:"workHome,omitempty" mapstructure:"workHome,omitempty"`

	RateLimit util.RateLimit `yaml:"rateLimit,omitempty" mapstructure:"rateLimit,omitempty"`

	// Config file paths,
	// default:["/etc/dragonfly/dfget.yaml","/etc/dragonfly.conf"].
	//
	// NOTE: It is recommended to use `/etc/dragonfly/dfget.yaml` as default,
	// and the `/etc/dragonfly.conf` is just to ensure compatibility with previous versions.
	//ConfigFiles []string `json:"-"`

	// MoreDaemonOptions indicates more options passed to daemon by command line.
	MoreDaemonOptions string `yaml:"moreDaemonOptions,omitempty" mapstructure:"moreDaemonOptions,omitempty"`

	// Recursive indicates to download all resources in target url, the target source client must support list action
	Recursive bool `yaml:"recursive,omitempty" mapstructure:"recursive,omitempty"`

	// RecursiveList indicates to list all resources in target url, the target source client must support list action
	RecursiveList bool `yaml:"recursiveList,omitempty" mapstructure:"list,omitempty"`

	// RecursiveLevel indicates to the maximum number of subdirectories that dfget will recurse into
	RecursiveLevel uint `yaml:"recursiveLevel,omitempty" mapstructure:"level,omitempty"`

	RecursiveAcceptRegex string `yaml:"acceptRegex,omitempty" mapstructure:"accept-regex,omitempty"`

	RecursiveRejectRegex string `yaml:"rejectRegex,omitempty" mapstructure:"reject-regex,omitempty"`

	KeepOriginalOffset bool `yaml:"keepOriginalOffset,omitempty" mapstructure:"original-offset,omitempty"`

	// Range stands download range for url, like: 0-9, will download 10 bytes from 0 to 9 ([0:9])
	Range string `yaml:"range,omitempty" mapstructure:"range,omitempty"`
}

func NewDfgetConfig() *ClientOption {
	return &dfgetConfig
}

func (cfg *ClientOption) Validate() error {
	if cfg == nil {
		return fmt.Errorf("runtime config: %w", dferrors.ErrInvalidArgument)
	}

	if !url.IsValid(cfg.URL) {
		return fmt.Errorf("url %s: %w", cfg.URL, dferrors.ErrInvalidArgument)
	}

	if _, err := regexp.Compile(cfg.RecursiveAcceptRegex); err != nil {
		return err
	}

	if _, err := regexp.Compile(cfg.RecursiveRejectRegex); err != nil {
		return err
	}

	if err := cfg.checkOutput(); err != nil {
		return fmt.Errorf("output %s: %w", err.Error(), dferrors.ErrInvalidArgument)
	}

	if err := cfg.checkHeader(); err != nil {
		return fmt.Errorf("output %s: %w", err.Error(), dferrors.ErrInvalidHeader)
	}

	if int64(cfg.RateLimit.Limit) < DefaultMinRate.ToNumber() {
		return fmt.Errorf("rate limit must be greater than %s: %w", DefaultMinRate.String(), dferrors.ErrInvalidArgument)
	}

	return nil
}

func (cfg *ClientOption) Convert(args []string) error {
	if pkgstrings.IsBlank(cfg.Output) {
		url := strings.TrimRight(cfg.URL, "/")
		idx := strings.LastIndexByte(url, '/')
		if idx < 0 {
			return fmt.Errorf("get output from url[%s] error", cfg.URL)
		}
		cfg.Output = url[idx+1:]
	}

	if !filepath.IsAbs(cfg.Output) {
		absPath, err := filepath.Abs(cfg.Output)
		if err != nil {
			return fmt.Errorf("get absolute path[%s] error: %v", cfg.Output, err)
		}
		cfg.Output = absPath
	}
	if cfg.URL == "" && len(args) > 0 {
		cfg.URL = args[0]
	}

	if cfg.Digest != "" {
		cfg.Tag = ""
	}

	if cfg.Console {
		cfg.ShowProgress = false
	}
	return nil
}

func (cfg *ClientOption) String() string {
	data, _ := json.Marshal(cfg)
	return string(data)
}

// checkHeader is for checking the header format
func (cfg *ClientOption) checkHeader() error {
	if len(cfg.Header) == 0 {
		return nil
	}

	for _, header := range cfg.Header {
		if !strings.Contains(header, ":") {
			return fmt.Errorf("header format error: %v", header)
		}
		idx := strings.Index(header, ":")
		if len(strings.TrimSpace(header[:idx])) == 0 || len(strings.TrimSpace(header[idx+1:])) == 0 {
			return fmt.Errorf("header format error: %v", header)
		}
	}

	return nil
}

// This function must be called after checkURL
func (cfg *ClientOption) checkOutput() error {
	if !filepath.IsAbs(cfg.Output) {
		return fmt.Errorf("path[%s] is not absolute path", cfg.Output)
	}
	outputDir, _ := path.Split(cfg.Output)
	if err := MkdirAll(outputDir, 0777, os.Getuid(), os.Getgid()); err != nil {
		return err
	}

	f, err := os.Stat(cfg.Output)
	// when not recursive download, need a file
	if !cfg.Recursive && err == nil && f.IsDir() {
		return fmt.Errorf("path[%s] is directory but requires file path", cfg.Output)
	}

	// check permission
	for dir := cfg.Output; !pkgstrings.IsBlank(dir); dir = filepath.Dir(dir) {
		if err := syscall.Access(dir, syscall.O_RDWR); err == nil {
			break
		} else if os.IsPermission(err) || dir == "/" {
			return fmt.Errorf("user[%s] path[%s] %v", user.Username(), cfg.Output, err)
		}
	}
	return nil
}

// MkdirAll make directories recursive, and changes uid, gid to the latest directory.
// For example: the path /data/x exists, uid=1, gid=1
// when call MkdirAll("/data/x/y/z", 0755, 2, 2)
// MkdirAll creates /data/x/y and change owner to 2:2, creates /data/x/y/z and change owner to 2:2
func MkdirAll(dir string, perm os.FileMode, uid, gid int) error {
	var (
		// directories to create
		dirs []string
		err  error
	)
	subDir := dir
	// find not exist directories from bottom to top
	for {
		if subDir == "" {
			break
		}
		_, err = os.Stat(subDir)
		// directory exists
		if err == nil {
			break
		}
		if os.IsNotExist(err) {
			dirs = append(dirs, subDir)
		} else {
			logger.Errorf("stat error: %s", err)
			return err
		}
		subDir = path.Dir(subDir)
	}

	// no directory to create
	if len(dirs) == 0 {
		return nil
	}

	err = os.MkdirAll(dir, perm)
	if err != nil {
		logger.Errorf("mkdir error: %s", err)
		return err
	}

	// update owner from top to bottom
	for _, d := range dirs {
		err = os.Chown(d, uid, gid)
		if err != nil {
			logger.Errorf("chown error: %s", err)
			return err
		}
	}
	return nil
}