From f592032f72959fc66e918686a71b57eb07f1f71c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8A=A0=E8=8F=B2?= Date: Mon, 17 May 2021 21:56:22 +0800 Subject: [PATCH] Refactor cmd (#217) * refator:cmd and path Signed-off-by: zuozheng.hzz --- cdnsystem/config/config.go | 34 +- cdnsystem/config/plugin_type.go | 6 +- client/config/dfget.go | 69 ++- client/config/dfget_darwin.go | 6 +- client/config/dfget_linux.go | 2 - client/config/peerhost.go | 145 +++-- client/config/peerhost_darwin.go | 14 +- client/config/peerhost_linux.go | 14 +- client/config/peerhost_test.go | 6 - client/config/testdata/config/daemon.yaml | 1 - client/daemon/peer/peertask_stream.go | 2 +- client/daemon/peerhost.go | 31 +- client/dfget/dfget.go | 189 +++++++ client/pidfile/pidfile.go | 74 --- client/pidfile/pidfile_test.go | 54 -- cmd/cdn/cmd/root.go | 15 +- cmd/common/common.go | 127 ----- cmd/dependency/base/option.go | 24 + cmd/dependency/dependency.go | 247 +++++++++ cmd/{common => dependency}/doc_cmd.go | 2 +- cmd/{common => dependency}/version_cmd.go | 2 +- cmd/dfget/cmd/daemon.go | 255 +++------ cmd/dfget/cmd/root.go | 518 +++++------------- cmd/manager/cmd/root.go | 15 +- cmd/scheduler/cmd/root.go | 15 +- docs/en/config/cdn.yaml | 3 - go.mod | 11 +- go.sum | 266 +-------- .../dfpath}/config_darwin.go | 6 +- .../dfpath}/config_linux.go | 4 +- internal/dfpath/dfpath.go | 48 ++ internal/dfpath/home_darwin.go | 25 + internal/dfpath/home_linux.go | 19 + .../logcore => internal/dfpath}/log_darwin.go | 6 +- .../logcore => internal/dfpath}/log_linux.go | 4 +- manager/config/config.go | 48 +- pkg/basic/dfnet/type.go | 4 +- pkg/basic/user.go | 12 +- pkg/dflog/logcore/init.go | 9 +- pkg/pidfile/pidfile.go | 85 +++ pkg/rpc/dfdaemon/client/client.go | 2 +- scheduler/config/config.go | 52 +- scheduler/config/config_darwin.go | 2 - scheduler/config/config_linux.go | 2 - scheduler/config/config_test.go | 4 +- 45 files changed, 1133 insertions(+), 1346 deletions(-) create mode 100644 client/dfget/dfget.go delete mode 100644 client/pidfile/pidfile.go delete mode 100644 client/pidfile/pidfile_test.go delete mode 100644 cmd/common/common.go create mode 100644 cmd/dependency/base/option.go create mode 100644 cmd/dependency/dependency.go rename cmd/{common => dependency}/doc_cmd.go (98%) rename cmd/{common => dependency}/version_cmd.go (98%) rename {cmd/common => internal/dfpath}/config_darwin.go (84%) rename {cmd/common => internal/dfpath}/config_linux.go (91%) create mode 100644 internal/dfpath/dfpath.go create mode 100644 internal/dfpath/home_darwin.go create mode 100644 internal/dfpath/home_linux.go rename {pkg/dflog/logcore => internal/dfpath}/log_darwin.go (84%) rename {pkg/dflog/logcore => internal/dfpath}/log_linux.go (91%) create mode 100644 pkg/pidfile/pidfile.go diff --git a/cdnsystem/config/config.go b/cdnsystem/config/config.go index 0ec569c7f..5436f12b2 100644 --- a/cdnsystem/config/config.go +++ b/cdnsystem/config/config.go @@ -21,6 +21,7 @@ import ( "io/ioutil" "time" + "d7y.io/dragonfly/v2/cmd/dependency/base" "d7y.io/dragonfly/v2/pkg/unit" "d7y.io/dragonfly/v2/pkg/util/net/iputils" "gopkg.in/yaml.v3" @@ -36,12 +37,10 @@ func New() *Config { // Config contains all configuration of cdn node. type Config struct { - Console bool `yaml:"console"` - Verbose bool `yaml:"verbose"` - PProfPort int `yaml:"pprofPort"` - ConfigServer string `yaml:"configServer"` - *BaseProperties `yaml:"base"` - Plugins map[PluginType][]*PluginProperties `yaml:"plugins"` + base.Options `yaml:",inline" mapstructure:",squash"` + *BaseProperties `yaml:"base" mapstructure:"base"` + Plugins map[PluginType][]*PluginProperties `yaml:"plugins" mapstructure:"plugins"` + ConfigServer string `yaml:"configServer" mapstructure:"configServer"` } // Load loads config properties from the giving file. @@ -117,50 +116,49 @@ func NewDefaultBaseProperties() *BaseProperties { // BaseProperties contains all basic properties of cdn system. type BaseProperties struct { - // ListenPort is the port cdn server listens on. // default: 8002 - ListenPort int `yaml:"listenPort"` + ListenPort int `yaml:"listenPort" mapstructure:"listenPort"` // DownloadPort is the port for download files from cdn. // default: 8001 - DownloadPort int `yaml:"downloadPort"` + DownloadPort int `yaml:"downloadPort" mapstructure:"downloadPort"` // SystemReservedBandwidth is the network bandwidth reserved for system software. // default: 20 MB, in format of G(B)/g/M(B)/m/K(B)/k/B, pure number will also be parsed as Byte. - SystemReservedBandwidth unit.Bytes `yaml:"systemReservedBandwidth"` + SystemReservedBandwidth unit.Bytes `yaml:"systemReservedBandwidth" mapstructure:"systemReservedBandwidth"` // MaxBandwidth is the network bandwidth that cdn system can use. // default: 200 MB, in format of G(B)/g/M(B)/m/K(B)/k/B, pure number will also be parsed as Byte. - MaxBandwidth unit.Bytes `yaml:"maxBandwidth"` + MaxBandwidth unit.Bytes `yaml:"maxBandwidth" mapstructure:"maxBandwidth"` // AdvertiseIP is used to set the ip that we advertise to other peer in the p2p-network. // By default, the first non-loop address is advertised. - AdvertiseIP string `yaml:"advertiseIP"` + AdvertiseIP string `yaml:"advertiseIP" mapstructure:"advertiseIP"` // FailAccessInterval is the interval time after failed to access the URL. // unit: minutes // default: 3 - FailAccessInterval time.Duration `yaml:"failAccessInterval"` + FailAccessInterval time.Duration `yaml:"failAccessInterval" mapstructure:"failAccessInterval"` // gc related // GCInitialDelay is the delay time from the start to the first GC execution. // default: 6s - GCInitialDelay time.Duration `yaml:"gcInitialDelay"` + GCInitialDelay time.Duration `yaml:"gcInitialDelay" mapstructure:"gcInitialDelay"` // GCMetaInterval is the interval time to execute GC meta. // default: 2min - GCMetaInterval time.Duration `yaml:"gcMetaInterval"` + GCMetaInterval time.Duration `yaml:"gcMetaInterval" mapstructure:"gcMetaInterval"` // GCStorageInterval is the interval time to execute GC storage. // default: 15s - GCStorageInterval time.Duration `yaml:"gcStorageInterval"` + GCStorageInterval time.Duration `yaml:"gcStorageInterval" mapstructure:"gcStorageInterval"` // TaskExpireTime when a task is not accessed within the taskExpireTime, // and it will be treated to be expired. // default: 3min - TaskExpireTime time.Duration `yaml:"taskExpireTime"` + TaskExpireTime time.Duration `yaml:"taskExpireTime" mapstructure:"taskExpireTime"` // StoragePattern disk/hybrid/memory - StoragePattern string `yaml:"storagePattern"` + StoragePattern string `yaml:"storagePattern" mapstructure:"storagePattern"` } diff --git a/cdnsystem/config/plugin_type.go b/cdnsystem/config/plugin_type.go index d0871a89c..de7d2acb5 100644 --- a/cdnsystem/config/plugin_type.go +++ b/cdnsystem/config/plugin_type.go @@ -34,7 +34,7 @@ var PluginTypes = []PluginType{ // PluginProperties the properties of a plugin. type PluginProperties struct { - Name string `yaml:"name"` - Enable bool `yaml:"enable"` - Config interface{} `yaml:"config"` + Name string `yaml:"name" mapstructure:"name"` + Enable bool `yaml:"enable" mapstructure:"enable"` + Config interface{} `yaml:"config" mapstructure:"config"` } diff --git a/client/config/dfget.go b/client/config/dfget.go index a31139019..3bcfbf6ed 100644 --- a/client/config/dfget.go +++ b/client/config/dfget.go @@ -27,8 +27,11 @@ import ( "syscall" "time" + "d7y.io/dragonfly/v2/cmd/dependency/base" logger "d7y.io/dragonfly/v2/pkg/dflog" + "d7y.io/dragonfly/v2/pkg/unit" "github.com/pkg/errors" + "golang.org/x/time/rate" "d7y.io/dragonfly/v2/pkg/basic" "d7y.io/dragonfly/v2/pkg/dferrors" @@ -36,67 +39,67 @@ import ( "d7y.io/dragonfly/v2/pkg/util/stringutils" ) +type DfgetConfig = ClientOption + // ClientOption holds all the runtime config information. type ClientOption struct { + base.Options `yaml:",inline" mapstructure:",squash"` // URL download URL. - URL string `json:"url"` + URL string `yaml:"url,omitempty" mapstructure:"url,omitempty"` // Lock file location - LockFile string `json:"lock_file" yaml:"lock_file"` + LockFile string `yaml:"lock_file,omitempty" mapstructure:"lock_file,omitempty"` // Output full output path. - Output string `json:"output"` + Output string `yaml:"output,omitempty" mapstructure:"output,omitempty"` // Timeout download timeout(second). - Timeout time.Duration `json:"timeout,omitempty"` + Timeout time.Duration `yaml:"timeout,omitempty" mapstructure:"timeout,omitempty"` + + BenchmarkRate unit.Bytes `yaml:"benchmark-rate,omitempty" mapstructure:"benchmark-rate,omitempty"` // Md5 expected file md5. // Deprecated: Md5 is deprecated, use DigestMethod with DigestValue instead - Md5 string `json:"md5,omitempty"` - + Md5 string `yaml:"md5,omitempty" mapstructure:"md5,omitempty"` + Digest string `yaml:"digest,omitempty" mapstructure:"digest,omitempty"` // DigestMethod indicates digest method, like md5, sha256 - DigestMethod string `json:"digest_method,omitempty"` + DigestMethod string `yaml:"digest_method,omitempty" mapstructure:"digest_method,omitempty"` // DigestValue indicates digest value - DigestValue string `json:"digest_value,omitempty"` + DigestValue string `yaml:"digest_value,omitempty" mapstructure:"digest_value,omitempty"` // Identifier identify download task, it is available merely when md5 param not exist. - Identifier string `json:"identifier,omitempty"` + Identifier string `yaml:"identifier,omitempty" mapstructure:"identifier,omitempty"` // CallSystem system name that executes dfget. - CallSystem string `json:"call_system,omitempty"` + CallSystem string `yaml:"call_system,omitempty" mapstructure:"call_system,omitempty"` // Pattern download pattern, must be 'p2p' or 'cdn' or 'source', // default:`p2p`. - Pattern string `json:"pattern,omitempty"` + Pattern string `yaml:"pattern,omitempty" mapstructure:"pattern,omitempty"` // CA certificate to verify when supernode interact with the source. - Cacerts []string `json:"cacert,omitempty"` + Cacerts []string `yaml:"cacert,omitempty" mapstructure:"cacert,omitempty"` // Filter filter some query params of url, use char '&' to separate different params. // eg: -f 'key&sign' will filter 'key' and 'sign' query param. // in this way, different urls correspond one same download task that can use p2p mode. - Filter []string `json:"filter,omitempty"` + Filter []string `yaml:"filter,omitempty" mapstructure:"filter,omitempty"` // Header of http request. // eg: --header='Accept: *' --header='Host: abc'. - Header []string `json:"header,omitempty"` + Header []string `yaml:"header,omitempty" mapstructure:"header,omitempty"` // NotBackSource indicates whether to not back source to download when p2p fails. - NotBackSource bool `json:"not_back_source,omitempty"` + NotBackSource bool `yaml:"not_back_source,omitempty" mapstructure:"not_back_source,omitempty"` // Insecure indicates whether skip secure verify when supernode interact with the source. - Insecure bool `json:"insecure,omitempty"` + Insecure bool `yaml:"insecure,omitempty" mapstructure:"insecure,omitempty"` // ShowBar shows progress bar, it's conflict with `--console`. - ShowBar bool `json:"show_bar,omitempty"` + ShowBar bool `yaml:"show_bar,omitempty" mapstructure:"show_bar,omitempty"` - // Console shows log on console, it's conflict with `--showbar`. - Console bool `json:"console,omitempty" yaml:"console,omitempty"` - - // Verbose indicates whether to be verbose. - // If set true, log level will be 'debug'. - Verbose bool `json:"verbose,omitempty"` + RateLimit rate.Limit `yaml:"rate-limit,omitempty" mapstructure:"rate-limit,omitempty"` // Config file paths, // default:["/etc/dragonfly/dfget.yaml","/etc/dragonfly.conf"]. @@ -106,10 +109,10 @@ type ClientOption struct { //ConfigFiles []string `json:"-"` // MoreDaemonOptions indicates more options passed to daemon by command line. - MoreDaemonOptions string `json:"more_daemon_options,omitempty"` + MoreDaemonOptions string `yaml:"more_daemon_options,omitempty" mapstructure:"more_daemon_options,omitempty"` } -func NewClientOption() *ClientOption { +func NewDfgetConfig() *ClientOption { return &dfgetConfig } @@ -129,6 +132,22 @@ func (cfg *ClientOption) Validate() error { return nil } +func (cfg *ClientOption) Convert(args []string) error { + if cfg.URL == "" && len(args) > 0 { + cfg.URL = args[0] + } + + if cfg.Digest != "" { + cfg.Identifier = "" + } + + if cfg.Console { + cfg.ShowBar = false + } + + return nil +} + func (cfg *ClientOption) String() string { js, _ := json.Marshal(cfg) return string(js) diff --git a/client/config/dfget_darwin.go b/client/config/dfget_darwin.go index 9541353cd..192e232d6 100644 --- a/client/config/dfget_darwin.go +++ b/client/config/dfget_darwin.go @@ -18,11 +18,15 @@ package config +import "d7y.io/dragonfly/v2/pkg/unit" + var dfgetConfig = ClientOption{ URL: "", LockFile: "/tmp/dfget.lock", Output: "", Timeout: 0, + BenchmarkRate: 128 * unit.KB, + RateLimit: 0, Md5: "", DigestMethod: "", DigestValue: "", @@ -35,6 +39,4 @@ var dfgetConfig = ClientOption{ NotBackSource: false, Insecure: false, ShowBar: false, - Console: false, - Verbose: false, } diff --git a/client/config/dfget_linux.go b/client/config/dfget_linux.go index 49acf5b0a..1712c8564 100644 --- a/client/config/dfget_linux.go +++ b/client/config/dfget_linux.go @@ -35,6 +35,4 @@ var dfgetConfig = ClientOption{ NotBackSource: false, Insecure: false, ShowBar: false, - Console: false, - Verbose: false, } diff --git a/client/config/peerhost.go b/client/config/peerhost.go index a10a7af49..43b6d07b7 100644 --- a/client/config/peerhost.go +++ b/client/config/peerhost.go @@ -30,6 +30,7 @@ import ( "strings" "time" + "d7y.io/dragonfly/v2/cmd/dependency/base" "d7y.io/dragonfly/v2/pkg/util/stringutils" "github.com/pkg/errors" "gopkg.in/yaml.v3" @@ -39,35 +40,35 @@ import ( "d7y.io/dragonfly/v2/pkg/util/net/iputils" ) +type DaemonConfig = PeerHostOption type PeerHostOption struct { + base.Options `yaml:",inline" mapstructure:",squash"` // AliveTime indicates alive duration for which daemon keeps no accessing by any uploading and download requests, // after this period daemon will automatically exit // when AliveTime == 0, will run infinitely - AliveTime clientutil.Duration `json:"alive_time" yaml:"alive_time"` - GCInterval clientutil.Duration `json:"gc_interval" yaml:"gc_interval"` + AliveTime clientutil.Duration `mapstructure:"alive_time" yaml:"alive_time"` + GCInterval clientutil.Duration `mapstructure:"gc_interval" yaml:"gc_interval"` // Pid file location PidFile string `json:"pid_file" yaml:"pid_file"` // Lock file location LockFile string `json:"lock_file" yaml:"lock_file"` - DataDir string `json:"data_dir" yaml:"data_dir"` - WorkHome string `json:"work_home" yaml:"work_home"` - KeepStorage bool `json:"keep_storage" yaml:"keep_storage"` - Verbose bool `yaml:"verbose" json:"verbose"` - Console bool `json:"console" yaml:"console"` + DataDir string `mapstructure:"data_dir" yaml:"data_dir"` + WorkHome string `mapstructure:"work_home" yaml:"work_home"` + KeepStorage bool `mapstructure:"keep_storage" yaml:"keep_storage"` - Scheduler SchedulerOption `json:"scheduler" yaml:"scheduler"` - Host HostOption `json:"host" yaml:"host"` - Download DownloadOption `json:"download" yaml:"download"` - Proxy *ProxyOption `json:"proxy" yaml:"proxy"` - Upload UploadOption `json:"upload" yaml:"upload"` - Storage StorageOption `json:"storage" yaml:"storage"` - Telemetry TelemetryOption `json:"telemetry" yaml:"telemetry"` - ConfigServer string `json:"configServer" yaml:"configServer"` + Scheduler SchedulerOption `mapstructure:"scheduler" yaml:"scheduler"` + Host HostOption `mapstructure:"host" yaml:"host"` + Download DownloadOption `mapstructure:"download" yaml:"download"` + Proxy *ProxyOption `mapstructure:"proxy" yaml:"proxy"` + Upload UploadOption `mapstructure:"upload" yaml:"upload"` + Storage StorageOption `mapstructure:"storage" yaml:"storage"` + Telemetry TelemetryOption `mapstructure:"telemetry" yaml:"telemetry"` + ConfigServer string `mapstructure:"configServer" yaml:"configServer"` } -func NewPeerHostOption() *PeerHostOption { +func NewDaemonConfig() *PeerHostOption { return &peerHostConfig } @@ -120,44 +121,44 @@ func (p *PeerHostOption) Validate() error { type SchedulerOption struct { // NetAddrs is scheduler addresses. - NetAddrs []dfnet.NetAddr `json:"net_addrs" yaml:"net_addrs"` + NetAddrs []dfnet.NetAddr `mapstructure:"net_addrs" yaml:"net_addrs"` // ScheduleTimeout is request timeout. - ScheduleTimeout clientutil.Duration `json:"schedule_timeout" yaml:"schedule_timeout"` + ScheduleTimeout clientutil.Duration `mapstructure:"schedule_timeout" yaml:"schedule_timeout"` } type HostOption struct { // SecurityDomain is the security domain - SecurityDomain string `json:"security_domain" yaml:"security_domain"` + SecurityDomain string `mapstructure:"security_domain" yaml:"security_domain"` // Peerhost location for scheduler - Location string `json:"location" yaml:"location"` + Location string `mapstructure:"location" yaml:"location"` // Peerhost idc for scheduler - IDC string `json:"idc" yaml:"idc"` + IDC string `mapstructure:"idc" yaml:"idc"` // Peerhost net topology for scheduler - NetTopology string `json:"net_topology" yaml:"net_topology"` + NetTopology string `mapstructure:"net_topology" yaml:"net_topology"` // The listen ip for all tcp services of daemon - ListenIP string `json:"listen_ip" yaml:"listen_ip"` + ListenIP string `mapstructure:"listen_ip" yaml:"listen_ip"` // The ip report to scheduler, normal same with listen ip - AdvertiseIP string `json:"advertise_ip" yaml:"advertise_ip"` + AdvertiseIP string `mapstructure:"advertise_ip" yaml:"advertise_ip"` } type DownloadOption struct { - TotalRateLimit clientutil.RateLimit `json:"total_rate_limit" yaml:"total_rate_limit"` - PerPeerRateLimit clientutil.RateLimit `json:"per_peer_rate_limit" yaml:"per_peer_rate_limit"` - DownloadGRPC ListenOption `json:"download_grpc" yaml:"download_grpc"` - PeerGRPC ListenOption `json:"peer_grpc" yaml:"peer_grpc"` - CalculateDigest bool `json:"calculate_digest" yaml:"calculate_digest"` + TotalRateLimit clientutil.RateLimit `mapstructure:"total_rate_limit" yaml:"total_rate_limit"` + PerPeerRateLimit clientutil.RateLimit `mapstructure:"per_peer_rate_limit" yaml:"per_peer_rate_limit"` + DownloadGRPC ListenOption `mapstructure:"download_grpc" yaml:"download_grpc"` + PeerGRPC ListenOption `mapstructure:"peer_grpc" yaml:"peer_grpc"` + CalculateDigest bool `mapstructure:"calculate_digest" yaml:"calculate_digest"` } type ProxyOption struct { // WARNING: when add more option, please update ProxyOption.unmarshal function - ListenOption `json:",inline" yaml:",inline"` - DefaultFilter string `json:"default_filter" yaml:"default_filter"` - MaxConcurrency int64 `json:"max_concurrency" yaml:"max_concurrency"` - RegistryMirror *RegistryMirror `json:"registry_mirror" yaml:"registry_mirror"` - WhiteList []*WhiteList `json:"white_list" yaml:"white_list"` - Proxies []*Proxy `json:"proxies" yaml:"proxies"` - HijackHTTPS *HijackConfig `json:"hijack_https" yaml:"hijack_https"` + ListenOption `mapstructure:",squash" yaml:",inline"` + DefaultFilter string `mapstructure:"default_filter" yaml:"default_filter"` + MaxConcurrency int64 `mapstructure:"max_concurrency" yaml:"max_concurrency"` + RegistryMirror *RegistryMirror `mapstructure:"registry_mirror" yaml:"registry_mirror"` + WhiteList []*WhiteList `mapstructure:"white_list" yaml:"white_list"` + Proxies []*Proxy `mapstructure:"proxies" yaml:"proxies"` + HijackHTTPS *HijackConfig `mapstructure:"hijack_https" yaml:"hijack_https"` } func (p *ProxyOption) UnmarshalJSON(b []byte) error { @@ -259,19 +260,19 @@ func (p *ProxyOption) unmarshal(unmarshal func(in []byte, out interface{}) (err } type UploadOption struct { - ListenOption `yaml:",inline"` - RateLimit clientutil.RateLimit `json:"rate_limit" yaml:"rate_limit"` + ListenOption `yaml:",inline" mapstructure:",squash"` + RateLimit clientutil.RateLimit `mapstructure:"rate_limit" yaml:"rate_limit"` } type ListenOption struct { - Security SecurityOption `json:"security" yaml:"security"` - TCPListen *TCPListenOption `json:"tcp_listen,omitempty" yaml:"tcp_listen,omitempty"` - UnixListen *UnixListenOption `json:"unix_listen,omitempty" yaml:"unix_listen,omitempty"` + Security SecurityOption `mapstructure:"security" yaml:"security"` + TCPListen *TCPListenOption `mapstructure:"tcp_listen,omitempty" yaml:"tcp_listen,omitempty"` + UnixListen *UnixListenOption `mapstructure:"unix_listen,omitempty" yaml:"unix_listen,omitempty"` } type TCPListenOption struct { // Listen stands listen interface, like: 0.0.0.0, 192.168.0.1 - Listen string `json:"listen" yaml:"listen"` + Listen string `mapstructure:"listen" yaml:"listen"` // PortRange stands listen port // yaml example 1: @@ -280,7 +281,7 @@ type TCPListenOption struct { // port: // start: 12345 // end: 12346 - PortRange TCPListenPortRange `json:"port" yaml:"port"` + PortRange TCPListenPortRange `mapstructure:"port" yaml:"port"` } type TCPListenPortRange struct { @@ -363,28 +364,26 @@ func (t *TCPListenPortRange) unmarshal(v interface{}) error { } type UnixListenOption struct { - Socket string `json:"socket" yaml:"socket"` + Socket string `mapstructure:"socket" yaml:"socket"` } type SecurityOption struct { // Insecure indicate enable tls or not - Insecure bool `json:"insecure" yaml:"insecure"` - CACert string `json:"ca_cert" yaml:"ca_cert"` - Cert string `json:"cert" yaml:"cert"` - Key string `json:"key" yaml:"key"` - TLSConfig *tls.Config `json:"tls_config" yaml:"tls_config"` + Insecure bool `mapstructure:"insecure" yaml:"insecure"` + CACert string `mapstructure:"ca_cert" yaml:"ca_cert"` + Cert string `mapstructure:"cert" yaml:"cert"` + Key string `mapstructure:"key" yaml:"key"` + TLSConfig *tls.Config `mapstructure:"tls_config" yaml:"tls_config"` } type StorageOption struct { // DataPath indicates directory which stores temporary files for p2p uploading - DataPath string `json:"data_path" yaml:"data_path"` + DataPath string `mapstructure:"data_path" yaml:"data_path"` // TaskExpireTime indicates caching duration for which cached file keeps no accessed by any process, // after this period cache file will be gc - TaskExpireTime clientutil.Duration `json:"task_expire_time" yaml:"task_expire_time"` - // DiskGCThreshold indicates the threshold to gc the oldest tasks - DiskGCThreshold clientutil.StorageSize `json:"disk_gc_threshold" yaml:"disk_gc_threshold"` - - StoreStrategy StoreStrategy `json:"strategy" yaml:"strategy"` + TaskExpireTime clientutil.Duration `mapstructure:"task_expire_time" yaml:"task_expire_time"` + StoreStrategy StoreStrategy `mapstructure:"strategy" yaml:"strategy"` + DiskGCThreshold clientutil.StorageSize `mapstructure:"disk_gc_threshold" yaml:"disk_gc_threshold"` } type StoreStrategy string @@ -461,16 +460,16 @@ func (t *TLSConfig) UnmarshalJSON(b []byte) error { // RegistryMirror configures the mirror of the official docker registry type RegistryMirror struct { // Remote url for the registry mirror, default is https://index.docker.io - Remote *URL `yaml:"url" json:"url"` + Remote *URL `yaml:"url" mapstructure:"url"` // Optional certificates if the mirror uses self-signed certificates - Certs *CertPool `yaml:"certs" json:"certs"` + Certs *CertPool `yaml:"certs" mapstructure:"certs"` // Whether to ignore certificates errors for the registry - Insecure bool `yaml:"insecure" json:"insecure"` + Insecure bool `yaml:"insecure" mapstructure:"insecure"` // Request the remote registry directly. - Direct bool `yaml:"direct" json:"direct"` + Direct bool `yaml:"direct" mapstructure:"direct"` } // TLSConfig returns the tls.Config used to communicate with the mirror. @@ -590,12 +589,12 @@ func certPoolFromFiles(files ...string) (*x509.CertPool, error) { // Proxy describes a regular expression matching rule for how to proxy a request. type Proxy struct { - Regx *Regexp `yaml:"regx" json:"regx"` - UseHTTPS bool `yaml:"use_https" json:"use_https"` - Direct bool `yaml:"direct" json:"direct"` + Regx *Regexp `yaml:"regx" mapstructure:"regx"` + UseHTTPS bool `yaml:"use_https" mapstructure:"use_https"` + Direct bool `yaml:"direct" mapstructure:"direct"` // Redirect is the host to redirect to, if not empty - Redirect string `yaml:"redirect" json:"redirect"` + Redirect string `yaml:"redirect" mapstructure:"redirect"` } func NewProxy(regx string, useHTTPS bool, direct bool, redirect string) (*Proxy, error) { @@ -665,25 +664,25 @@ func (r *Regexp) MarshalYAML() (interface{}, error) { // HijackConfig represents how dfdaemon hijacks http requests. type HijackConfig struct { - Cert string `yaml:"cert" json:"cert"` - Key string `yaml:"key" json:"key"` - Hosts []*HijackHost `yaml:"hosts" json:"hosts"` + Cert string `yaml:"cert" mapstructure:"cert"` + Key string `yaml:"key" mapstructure:"key"` + Hosts []*HijackHost `yaml:"hosts" mapstructure:"hosts"` } // HijackHost is a hijack rule for the hosts that matches Regx. type HijackHost struct { - Regx *Regexp `yaml:"regx" json:"regx"` - Insecure bool `yaml:"insecure" json:"insecure"` - Certs *CertPool `yaml:"certs" json:"certs"` + Regx *Regexp `yaml:"regx" mapstructure:"regx"` + Insecure bool `yaml:"insecure" mapstructure:"insecure"` + Certs *CertPool `yaml:"certs" mapstructure:"certs"` } // TelemetryOption is the option for telemetry type TelemetryOption struct { - Jaeger string `yaml:"jaeger" json:"jaeger"` + Jaeger string `yaml:"jaeger" mapstructure:"jaeger"` } type WhiteList struct { - Host string `yaml:"host" json:"host"` - Regx *Regexp `yaml:"regx" json:"regx"` - Ports []string `yaml:"ports" json:"ports"` + Host string `yaml:"host" mapstructure:"host"` + Regx *Regexp `yaml:"regx" mapstructure:"regx"` + Ports []string `yaml:"ports" mapstructure:"ports"` } diff --git a/client/config/peerhost_darwin.go b/client/config/peerhost_darwin.go index 057f8a319..af1a2f8c9 100644 --- a/client/config/peerhost_darwin.go +++ b/client/config/peerhost_darwin.go @@ -36,15 +36,13 @@ var ( ) var peerHostConfig = PeerHostOption{ - DataDir: peerHostDataDir, - WorkHome: peerHostWorkHome, - AliveTime: clientutil.Duration{Duration: DefaultDaemonAliveTime}, - GCInterval: clientutil.Duration{Duration: DefaultGCInterval}, - PidFile: "/tmp/dfdaemon.pid", - LockFile: "/tmp/dfdaemon.lock", + DataDir: peerHostDataDir, + WorkHome: peerHostWorkHome, + AliveTime: clientutil.Duration{Duration: DefaultDaemonAliveTime}, + GCInterval: clientutil.Duration{Duration: DefaultGCInterval}, + //PidFile: "/tmp/dfdaemon.pid", + //LockFile: "/tmp/dfdaemon.lock", KeepStorage: false, - Verbose: false, - Console: false, Scheduler: SchedulerOption{ NetAddrs: nil, ScheduleTimeout: clientutil.Duration{Duration: DefaultScheduleTimeout}, diff --git a/client/config/peerhost_linux.go b/client/config/peerhost_linux.go index 784b9da7c..c313c571c 100644 --- a/client/config/peerhost_linux.go +++ b/client/config/peerhost_linux.go @@ -36,15 +36,13 @@ var ( ) var peerHostConfig = PeerHostOption{ - DataDir: peerHostDataDir, - WorkHome: peerHostWorkHome, - AliveTime: clientutil.Duration{Duration: DefaultDaemonAliveTime}, - GCInterval: clientutil.Duration{Duration: DefaultGCInterval}, - PidFile: "/var/run/dfdaemon.pid", - LockFile: "/var/run/dfdaemon.lock", + DataDir: peerHostDataDir, + WorkHome: peerHostWorkHome, + AliveTime: clientutil.Duration{Duration: DefaultDaemonAliveTime}, + GCInterval: clientutil.Duration{Duration: DefaultGCInterval}, + //PidFile: "/var/run/dfdaemon.pid", + //LockFile: "/var/run/dfdaemon.lock", KeepStorage: false, - Verbose: false, - Console: false, Scheduler: SchedulerOption{ NetAddrs: nil, ScheduleTimeout: clientutil.Duration{Duration: DefaultScheduleTimeout}, diff --git a/client/config/peerhost_test.go b/client/config/peerhost_test.go index 586a20290..faa6417b7 100644 --- a/client/config/peerhost_test.go +++ b/client/config/peerhost_test.go @@ -293,7 +293,6 @@ func TestPeerHostOption_Load(t *testing.T) { DataDir: "/tmp/dragonfly/dfdaemon/", WorkHome: "/tmp/dragonfly/dfdaemon/", KeepStorage: false, - Verbose: true, Scheduler: SchedulerOption{ NetAddrs: []dfnet.NetAddr{ { @@ -428,9 +427,4 @@ func TestPeerHostOption_Load(t *testing.T) { assert.EqualValues(peerHostOption, peerHostOptionYAML) - peerHostOptionJSON := &PeerHostOption{} - if err := peerHostOptionJSON.Load("./testdata/config/daemon.json"); err != nil { - t.Fatal(err) - } - assert.EqualValues(peerHostOption, peerHostOptionJSON) } diff --git a/client/config/testdata/config/daemon.yaml b/client/config/testdata/config/daemon.yaml index b38e1604a..08eea5131 100644 --- a/client/config/testdata/config/daemon.yaml +++ b/client/config/testdata/config/daemon.yaml @@ -10,7 +10,6 @@ scheduler: - type: tcp addr: 127.0.0.1:8002 schedule_timeout: 0 -verbose: true host: listen_ip: 0.0.0.0 diff --git a/client/daemon/peer/peertask_stream.go b/client/daemon/peer/peertask_stream.go index 07dff6d81..ef1c7becb 100644 --- a/client/daemon/peer/peertask_stream.go +++ b/client/daemon/peer/peertask_stream.go @@ -252,7 +252,7 @@ func (s *streamPeerTask) Start(ctx context.Context) (io.Reader, map[string]strin case first := <-s.successPieceCh: //if !ok { // s.Warnf("successPieceCh closed unexpect") - // return nil, nil, errors.New("early done") + // return nil, nil, errors.NewDaemonConfig("early done") //} firstPiece = first } diff --git a/client/daemon/peerhost.go b/client/daemon/peerhost.go index d867b48b2..9ddbfef3b 100644 --- a/client/daemon/peerhost.go +++ b/client/daemon/peerhost.go @@ -28,6 +28,9 @@ import ( "sync" "time" + "d7y.io/dragonfly/v2/internal/dfpath" + "d7y.io/dragonfly/v2/pkg/idgen" + "d7y.io/dragonfly/v2/pkg/util/net/iputils" "github.com/pkg/errors" "golang.org/x/sync/errgroup" "golang.org/x/time/rate" @@ -72,16 +75,21 @@ type peerHost struct { PieceManager peer.PieceManager } -func NewPeerHost(host *scheduler.PeerHost, opt config.PeerHostOption) (PeerHost, error) { - var ( - sched schedulerclient.SchedulerClient - err error - ) - if len(opt.Scheduler.NetAddrs) > 0 { - sched, err = schedulerclient.GetClientByAddr(opt.Scheduler.NetAddrs) - } else { - sched, err = schedulerclient.GetSchedulerByConfigServer(opt.ConfigServer) +func New(opt *config.PeerHostOption) (PeerHost, error) { + + host := &scheduler.PeerHost{ + Uuid: idgen.UUIDString(), + Ip: opt.Host.AdvertiseIP, + RpcPort: int32(opt.Download.PeerGRPC.TCPListen.PortRange.Start), + DownPort: 0, + HostName: iputils.HostName, + SecurityDomain: opt.Host.SecurityDomain, + Location: opt.Host.Location, + Idc: opt.Host.IDC, + NetTopology: opt.Host.NetTopology, } + + sched, err := schedulerclient.GetClientByAddr(opt.Scheduler.NetAddrs) if err != nil { return nil, errors.Wrap(err, "failed to get schedulers") } @@ -155,7 +163,7 @@ func NewPeerHost(host *scheduler.PeerHost, opt config.PeerHostOption) (PeerHost, once: &sync.Once{}, done: make(chan bool), schedPeerHost: host, - Option: opt, + Option: *opt, ServiceManager: serviceManager, PeerTaskManager: peerTaskManager, @@ -248,7 +256,8 @@ func (ph *peerHost) prepareTCPListener(opt config.ListenOption, withTLS bool) (n func (ph *peerHost) Serve() error { ph.GCManager.Start() - + // todo remove this field, and use directly dfpath.DaemonSockPath + ph.Option.Download.DownloadGRPC.UnixListen.Socket = dfpath.DaemonSockPath // prepare download service listen if ph.Option.Download.DownloadGRPC.UnixListen == nil { return errors.New("download grpc unix listen option is empty") diff --git a/client/dfget/dfget.go b/client/dfget/dfget.go new file mode 100644 index 000000000..5163c62f1 --- /dev/null +++ b/client/dfget/dfget.go @@ -0,0 +1,189 @@ +/* + * Copyright 2020 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dfget + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "d7y.io/dragonfly/v2/cdnsystem/source" + "d7y.io/dragonfly/v2/client/clientutil/progressbar" + "d7y.io/dragonfly/v2/client/config" + "d7y.io/dragonfly/v2/pkg/basic" + "d7y.io/dragonfly/v2/pkg/dferrors" + logger "d7y.io/dragonfly/v2/pkg/dflog" + "d7y.io/dragonfly/v2/pkg/rpc/base" + dfdaemongrpc "d7y.io/dragonfly/v2/pkg/rpc/dfdaemon" + _ "d7y.io/dragonfly/v2/pkg/rpc/dfdaemon/client" + dfclient "d7y.io/dragonfly/v2/pkg/rpc/dfdaemon/client" + "github.com/go-http-utils/headers" +) + +var filter string + +func Download(cfg *config.DfgetConfig, client dfclient.DaemonClient) error { + var ( + ctx = context.Background() + cancel context.CancelFunc + hdr = parseHeader(cfg.Header) + ) + + if client == nil { + return downloadFromSource(cfg, hdr) + } + + output, err := filepath.Abs(cfg.Output) + if err != nil { + return err + } + + if cfg.Timeout > 0 { + ctx, cancel = context.WithTimeout(ctx, cfg.Timeout) + defer cancel() + } else { + ctx, cancel = context.WithCancel(ctx) + defer cancel() + } + + request := &dfdaemongrpc.DownRequest{ + Url: cfg.URL, + UrlMeta: &base.UrlMeta{ + Md5: cfg.Md5, + Range: hdr[headers.Range], + Header: hdr, + }, + Output: output, + BizId: cfg.CallSystem, + Filter: filter, + Uid: int64(basic.UserId), + Gid: int64(basic.UserGroup), + } + var ( + start = time.Now() + end time.Time + ) + down, err := client.Download(ctx, request) + if err != nil { + return err + } + var ( + result *dfdaemongrpc.DownResult + ) + // todo using progressbar when showBar is true + pb := progressbar.DefaultBytes(-1, "Downloading") + for { + result, err = down.Recv() + if err != nil { + if de, ok := err.(*dferrors.DfError); ok { + logger.Errorf("dragonfly daemon returns error code %d/%s", de.Code, de.Message) + } else { + logger.Errorf("dragonfly daemon returns error %s", err) + } + break + } + if result.CompletedLength > 0 { + pb.Set64(int64(result.CompletedLength)) + } + if result.Done { + pb.Describe("Downloaded") + pb.Finish() + end = time.Now() + fmt.Printf("Task: %s\nPeer: %s\n", result.TaskId, result.PeerId) + fmt.Printf("Download success, time cost: %dms, length: %d\n", end.Sub(start).Milliseconds(), result.CompletedLength) + break + } + } + if err != nil { + logger.Errorf("download by dragonfly error: %s", err) + return downloadFromSource(cfg, hdr) + } + return err +} + +func downloadFromSource(cfg *config.DfgetConfig, hdr map[string]string) (err error) { + if cfg.NotBackSource { + err = fmt.Errorf("dfget download error, and back source disabled") + logger.Warnf("%s", err) + return err + } + + var ( + start = time.Now() + end time.Time + ) + + fmt.Println("dfget download error, try to download from source") + var ( + resourceClient source.ResourceClient + target *os.File + response io.ReadCloser + _ map[string]string + written int64 + ) + + resourceClient, err = source.NewSourceClient() + if err != nil { + logger.Errorf("init source client error: %s", err) + return err + } + + response, _, err = resourceClient.Download(cfg.URL, hdr) + if err != nil { + logger.Errorf("download from source error: %s", err) + return err + } + defer response.Close() + + target, err = os.OpenFile(cfg.Output, os.O_RDWR|os.O_CREATE, 0644) + if err != nil { + logger.Errorf("open %s error: %s", cfg.Output, err) + return err + } + + written, err = io.Copy(target, response) + if err == nil { + logger.Infof("copied %d bytes to %s", written, cfg.Output) + end = time.Now() + fmt.Printf("Download from source success, time cost: %dms\n", end.Sub(start).Milliseconds()) + // change permission + logger.Infof("change own to uid %d gid %d", basic.UserId, basic.UserGroup) + if err = os.Chown(cfg.Output, basic.UserId, basic.UserGroup); err != nil { + logger.Errorf("change own failed: %s", err) + return err + } + return nil + } + logger.Errorf("copied %d bytes to %s, with error: %s", + written, cfg.Output, err) + return err +} + +func parseHeader(s []string) map[string]string { + hdr := map[string]string{} + for _, h := range s { + idx := strings.Index(h, ":") + if idx > 0 { + hdr[h[:idx]] = strings.TrimLeft(h[idx:], " ") + } + } + return hdr +} diff --git a/client/pidfile/pidfile.go b/client/pidfile/pidfile.go deleted file mode 100644 index f2828bc0b..000000000 --- a/client/pidfile/pidfile.go +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright 2020 The Dragonfly Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Package pidfile provides structure and helper functions to create and remove -// PID file. A PID file is usually a file used to store the process ID of a -// running process. -package pidfile // copy from "github.com/docker/docker/pkg/pidfile" - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - "strconv" - "strings" - - "github.com/shirou/gopsutil/v3/process" -) - -// PIDFile is a file used to store the process ID of a running process. -type PIDFile struct { - path string -} - -func IsProcessExistsByPIDFile(path string) (bool, error) { - pidByte, err := ioutil.ReadFile(path) - if os.IsNotExist(err) { - return false, nil - } - if err != nil { - return false, err - } - pid, err := strconv.ParseInt(strings.TrimSpace(string(pidByte)), 10, 32) - if err != nil { - return false, err - } - return process.PidExists(int32(pid)) -} - -// New creates a PIDfile using the specified path. -func New(path string) (*PIDFile, error) { - if ok, err := IsProcessExistsByPIDFile(path); ok { - return nil, fmt.Errorf("process already exists") - } else if err != nil { - return nil, err - } - // Note MkdirAll returns nil if a directory already exists - if err := os.MkdirAll(filepath.Dir(path), os.FileMode(0755)); err != nil { - return nil, err - } - if err := ioutil.WriteFile(path, []byte(fmt.Sprintf("%d", os.Getpid())), 0644); err != nil { - return nil, err - } - - return &PIDFile{path: path}, nil -} - -// Remove removes the PIDFile. -func (file PIDFile) Remove() error { - return os.Remove(file.path) -} diff --git a/client/pidfile/pidfile_test.go b/client/pidfile/pidfile_test.go deleted file mode 100644 index e6e0facf4..000000000 --- a/client/pidfile/pidfile_test.go +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2020 The Dragonfly Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package pidfile // import "github.com/docker/docker/pkg/pidfile" - -import ( - "io/ioutil" - "os" - "path/filepath" - "testing" -) - -func TestNewAndRemove(t *testing.T) { - dir, err := ioutil.TempDir(os.TempDir(), "test-pidfile") - if err != nil { - t.Fatal("Could not create test directory") - } - - path := filepath.Join(dir, "testfile") - file, err := New(path) - if err != nil { - t.Fatal("Could not create test file", err) - } - - _, err = New(path) - if err == nil { - t.Fatal("Test file creation not blocked") - } - - if err := file.Remove(); err != nil { - t.Fatal("Could not delete created test file") - } -} - -func TestRemoveInvalidPath(t *testing.T) { - file := PIDFile{path: filepath.Join("foo", "bar")} - - if err := file.Remove(); err == nil { - t.Fatal("Non-existing file doesn't give an error on delete") - } -} diff --git a/cmd/cdn/cmd/root.go b/cmd/cdn/cmd/root.go index d16c9660d..c47e15c49 100644 --- a/cmd/cdn/cmd/root.go +++ b/cmd/cdn/cmd/root.go @@ -21,7 +21,7 @@ import ( "d7y.io/dragonfly/v2/cdnsystem/config" "d7y.io/dragonfly/v2/cdnsystem/server" - "d7y.io/dragonfly/v2/cmd/common" + "d7y.io/dragonfly/v2/cmd/dependency" logger "d7y.io/dragonfly/v2/pkg/dflog" "d7y.io/dragonfly/v2/pkg/dflog/logcore" "github.com/pkg/errors" @@ -30,11 +30,7 @@ import ( ) var ( - cfg *config.Config -) - -const ( - cdnSystemEnvPrefix = "cdn" + cfg *config.Config ) // rootCmd represents the base command when called without any subcommands @@ -68,9 +64,8 @@ func Execute() { func init() { // Initialize default cdn system config cfg = config.New() - // Initialize cobra - common.InitCobra(rootCmd, cdnSystemEnvPrefix, cfg) + dependency.InitCobra(rootCmd, true, cfg) } func runCdnSystem() error { @@ -78,8 +73,8 @@ func runCdnSystem() error { s, _ := yaml.Marshal(cfg) logger.Infof("cdn system configuration:\n%s", string(s)) - // initialize verbose mode - common.InitVerboseMode(cfg.Verbose, cfg.PProfPort) + ff := dependency.InitMonitor(cfg.Verbose, cfg.PProfPort, cfg.Jaeger) + defer ff() if svr, err := server.New(cfg); err != nil { return err diff --git a/cmd/common/common.go b/cmd/common/common.go deleted file mode 100644 index 49bec7580..000000000 --- a/cmd/common/common.go +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright 2020 The Dragonfly Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package common - -import ( - "fmt" - "reflect" - - logger "d7y.io/dragonfly/v2/pkg/dflog" - "d7y.io/dragonfly/v2/pkg/dflog/logcore" - "d7y.io/dragonfly/v2/pkg/unit" - "github.com/go-echarts/statsview" - "github.com/go-echarts/statsview/viewer" - "github.com/mitchellh/mapstructure" - "github.com/phayes/freeport" - "github.com/pkg/errors" - "github.com/spf13/cobra" - "github.com/spf13/viper" - "go.uber.org/zap/zapcore" - "gopkg.in/yaml.v3" -) - -// InitCobra initializes flags binding and common sub cmds. -// cfgFile is a pointer to configuration path, config is a pointer to configuration struct. -func InitCobra(cmd *cobra.Command, envPrefix string, config interface{}) { - var cfgFile string - cobra.OnInitialize(func() { initConfig(&cfgFile, envPrefix, config) }) - - // Add flags - flagSet := cmd.Flags() - flagSet.Bool("console", false, "whether print log on the terminal") - flagSet.Bool("verbose", false, "whether use debug level logger and enable pprof") - flagSet.Int("pprofPort", 0, "listen port for pprof, only valid when the verbose option is true, default is random port") - flagSet.String("configServer", "", "the service address that provides the configuration item") - flagSet.StringVarP(&cfgFile, "config", "f", "", "the path of configuration file") - - if err := viper.BindPFlags(flagSet); err != nil { - panic(errors.Wrap(err, "bind flags to viper")) - } - - // Add common cmds - cmd.AddCommand(VersionCmd) - cmd.AddCommand(newDocCommand(cmd.Name())) -} - -func InitVerboseMode(verbose bool, pprofPort int) { - if !verbose { - return - } - - logcore.SetCoreLevel(zapcore.DebugLevel) - logcore.SetGrpcLevel(zapcore.DebugLevel) - - // Enable go pprof and statsview - go func() { - if pprofPort == 0 { - pprofPort, _ = freeport.GetFreePort() - } - - debugAddr := fmt.Sprintf("localhost:%d", pprofPort) - viewer.SetConfiguration(viewer.WithAddr(debugAddr)) - - logger.With("pprof", fmt.Sprintf("http://%s/debug/pprof", debugAddr), - "statsview", fmt.Sprintf("http://%s/debug/statsview", debugAddr)). - Infof("enable pprof at %s", debugAddr) - - if err := statsview.New().Start(); err != nil { - logger.Warnf("serve pprof error:%v", err) - } - }() -} - -// initConfig reads in config file and ENV variables if set. -func initConfig(cfgFile *string, envPrefix string, config interface{}) { - if *cfgFile != "" { - // Use config file from the flag. - viper.SetConfigFile(*cfgFile) - } else { - viper.AddConfigPath(defaultConfigDir) - viper.SetConfigName(envPrefix) - viper.SetConfigType("yaml") - } - - viper.SetEnvPrefix(envPrefix) - viper.AutomaticEnv() // read in environment variables that match - - // If a config file is found, read it in. - if err := viper.ReadInConfig(); err == nil { - fmt.Println("using config file:", viper.ConfigFileUsed()) - } - - if err := viper.Unmarshal(config, initDecoderConfig); err != nil { - panic(errors.Wrap(err, "unmarshal config to struct")) - } -} - -func initDecoderConfig(dc *mapstructure.DecoderConfig) { - dc.TagName = "yaml" - dc.DecodeHook = mapstructure.ComposeDecodeHookFunc(dc.DecodeHook, func(from, to reflect.Type, v interface{}) (interface{}, error) { - switch to { - case reflect.TypeOf(unit.B): - b, _ := yaml.Marshal(v) - p := reflect.New(to) - if err := yaml.Unmarshal(b, p.Interface()); err != nil { - return nil, err - } else { - return p.Interface(), nil - } - default: - return v, nil - } - }) -} diff --git a/cmd/dependency/base/option.go b/cmd/dependency/base/option.go new file mode 100644 index 000000000..f3465f6c0 --- /dev/null +++ b/cmd/dependency/base/option.go @@ -0,0 +1,24 @@ +/* + * Copyright 2020 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package base + +type Options struct { + Console bool `yaml:"console" mapstructure:"console"` + Verbose bool `yaml:"verbose" mapstructure:"verbose"` + PProfPort int `yaml:"pprof-port" mapstructure:"pprof-port"` + Jaeger string `yaml:"jaeger" mapstructure:"jaeger"` +} diff --git a/cmd/dependency/dependency.go b/cmd/dependency/dependency.go new file mode 100644 index 000000000..afb3ba36e --- /dev/null +++ b/cmd/dependency/dependency.go @@ -0,0 +1,247 @@ +/* + * Copyright 2020 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dependency + +import ( + "context" + "fmt" + "os" + "os/signal" + "path/filepath" + "reflect" + "strings" + "syscall" + "time" + + "d7y.io/dragonfly/v2/client/clientutil" + "d7y.io/dragonfly/v2/client/config" + "d7y.io/dragonfly/v2/internal/dfpath" + "d7y.io/dragonfly/v2/pkg/basic/dfnet" + logger "d7y.io/dragonfly/v2/pkg/dflog" + "d7y.io/dragonfly/v2/pkg/dflog/logcore" + "d7y.io/dragonfly/v2/pkg/unit" + "d7y.io/dragonfly/v2/pkg/util/net/iputils" + "d7y.io/dragonfly/v2/version" + "github.com/go-echarts/statsview" + "github.com/go-echarts/statsview/viewer" + "github.com/mitchellh/mapstructure" + "github.com/phayes/freeport" + "github.com/pkg/errors" + "github.com/spf13/cobra" + "github.com/spf13/viper" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/trace/jaeger" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/semconv" + "go.uber.org/zap/zapcore" + "gopkg.in/yaml.v3" +) + +// InitCobra initializes flags binding and common sub cmds. +// config is a pointer to configuration struct. +func InitCobra(cmd *cobra.Command, useConfigFile bool, config interface{}) { + rootName := cmd.Root().Name() + cobra.OnInitialize(func() { initConfig(useConfigFile, rootName, config) }) + + // Add common flags + flags := cmd.Flags() + flags.Bool("console", false, "whether logger output records to the stdout") + flags.Bool("verbose", false, "whether logger use debug level") + flags.Int("pprof-port", -1, "listen port for pprof, 0 represents random port") + flags.String("jaeger", "", "jaeger endpoint url, like: http://localhost:14250/api/traces") + flags.String("config", "", fmt.Sprintf("the path of configuration file with yaml extension name, default is %s, it can also be set by env var:%s", filepath.Join(dfpath.DefaultConfigDir, rootName+".yaml"), strings.ToUpper(rootName+"_config"))) + + // Bind common flags + if err := viper.BindPFlags(flags); err != nil { + panic(errors.Wrap(err, "bind common flags to viper")) + } + + // Config for binding env + viper.SetEnvPrefix(rootName) + viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + _ = viper.BindEnv("config") + + // Add common cmds only on root cmd + if !cmd.HasParent() { + cmd.AddCommand(VersionCmd) + cmd.AddCommand(newDocCommand(cmd.Name())) + } +} + +// InitMonitor initialize monitor and return final handler +func InitMonitor(verbose bool, pprofPort int, jaeger string) func() { + var fc = make(chan func(), 5) + + if verbose { + logcore.SetCoreLevel(zapcore.DebugLevel) + logcore.SetGrpcLevel(zapcore.DebugLevel) + } + + if pprofPort >= 0 { + // Enable go pprof and statsview + go func() { + if pprofPort == 0 { + pprofPort, _ = freeport.GetFreePort() + } + + debugAddr := fmt.Sprintf("localhost:%d", pprofPort) + viewer.SetConfiguration(viewer.WithAddr(debugAddr)) + + logger.With("pprof", fmt.Sprintf("http://%s/debug/pprof", debugAddr), + "statsview", fmt.Sprintf("http://%s/debug/statsview", debugAddr)). + Infof("enable pprof at %s", debugAddr) + + vm := statsview.New() + if err := vm.Start(); err != nil { + logger.Warnf("serve pprof error:%v", err) + } else { + fc <- func() { vm.Stop() } + } + }() + } + + if jaeger != "" { + if ff, err := initJaegerTracer(jaeger); err != nil { + logger.Warnf("init jaeger tracer error:%v", err) + } else { + fc <- ff + } + } + + return func() { + logger.Infof("do %d monitor finalizer", len(fc)) + for { + select { + case f := <-fc: + f() + default: + return + } + } + } +} + +func SetupQuitSignalHandler(handler func()) { + signals := make(chan os.Signal, 1) + signal.Notify(signals, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM) + + go func() { + var done bool + for { + select { + case sig := <-signals: + logger.Warnf("receive signal:%v", sig) + if !done { + done = true + handler() + logger.Warnf("handle signal:%v finish", sig) + } + } + } + }() +} + +// initConfig reads in config file and ENV variables if set. +func initConfig(useConfigFile bool, name string, config interface{}) { + // Use config file and read once. + if useConfigFile { + cfgFile := viper.GetString("config") + if cfgFile != "" { + // Use config file from the flag. + viper.SetConfigFile(cfgFile) + } else { + viper.AddConfigPath(dfpath.DefaultConfigDir) + viper.SetConfigName(name) + viper.SetConfigType("yaml") + } + + // If a config file is found, read it in. + if err := viper.ReadInConfig(); err != nil { + ignoreErr := false + if _, ok := err.(viper.ConfigFileNotFoundError); ok { + if cfgFile == "" { + ignoreErr = true + } + } + if !ignoreErr { + panic(errors.Wrap(err, "viper read config")) + } + } + } + + if err := viper.Unmarshal(config, initDecoderConfig); err != nil { + panic(errors.Wrap(err, "unmarshal config to struct")) + } +} + +func initDecoderConfig(dc *mapstructure.DecoderConfig) { + dc.DecodeHook = mapstructure.ComposeDecodeHookFunc(func(from, to reflect.Type, v interface{}) (interface{}, error) { + switch to { + case reflect.TypeOf(unit.B), + reflect.TypeOf(dfnet.NetAddr{}), + reflect.TypeOf(clientutil.RateLimit{}), + reflect.TypeOf(clientutil.Duration{}), + reflect.TypeOf(config.ProxyOption{}), + reflect.TypeOf(config.TCPListenPortRange{}), + reflect.TypeOf(config.FileString("")), + reflect.TypeOf(config.URL{}), + reflect.TypeOf(config.CertPool{}), + reflect.TypeOf(config.Regexp{}), + reflect.TypeOf(clientutil.StorageSize{}): + b, _ := yaml.Marshal(v) + p := reflect.New(to) + if err := yaml.Unmarshal(b, p.Interface()); err != nil { + return nil, err + } else { + return p.Interface(), nil + } + default: + return v, nil + } + }, mapstructure.StringToSliceHookFunc("&"), dc.DecodeHook) +} + +// initTracer creates a new trace provider instance and registers it as global trace provider. +func initJaegerTracer(url string) (func(), error) { + exp, err := jaeger.NewRawExporter(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(url))) + if err != nil { + return nil, err + } + + tp := sdktrace.NewTracerProvider( + // Always be sure to batch in production. + sdktrace.WithBatcher(exp), + sdktrace.WithSampler(sdktrace.AlwaysSample()), + // Record information about this application in an Resource. + sdktrace.WithResource(resource.NewWithAttributes( + semconv.ServiceNameKey.String("dragonfly"), + semconv.ServiceInstanceIDKey.String(fmt.Sprintf("%s|%s", iputils.HostName, iputils.HostIp)), + semconv.ServiceVersionKey.String(version.GitVersion))), + ) + + // Register our TracerProvider as the global so any imported + // instrumentation in the future will default to using it. + otel.SetTracerProvider(tp) + + return func() { + // Do not make the application hang when it is shutdown. + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + _ = tp.Shutdown(ctx) + }, nil +} diff --git a/cmd/common/doc_cmd.go b/cmd/dependency/doc_cmd.go similarity index 98% rename from cmd/common/doc_cmd.go rename to cmd/dependency/doc_cmd.go index 1490ec0d5..f6e87b162 100644 --- a/cmd/common/doc_cmd.go +++ b/cmd/dependency/doc_cmd.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package common +package dependency import ( "fmt" diff --git a/cmd/common/version_cmd.go b/cmd/dependency/version_cmd.go similarity index 98% rename from cmd/common/version_cmd.go rename to cmd/dependency/version_cmd.go index f49738d36..3c6afc98a 100644 --- a/cmd/common/version_cmd.go +++ b/cmd/dependency/version_cmd.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package common +package dependency import ( "fmt" diff --git a/cmd/dfget/cmd/daemon.go b/cmd/dfget/cmd/daemon.go index ccd87f368..4c4f56c11 100644 --- a/cmd/dfget/cmd/daemon.go +++ b/cmd/dfget/cmd/daemon.go @@ -17,216 +17,129 @@ package cmd import ( - "fmt" + "context" "os" - "os/signal" - "syscall" + "time" "d7y.io/dragonfly/v2/client/config" - "d7y.io/dragonfly/v2/client/daemon" - "d7y.io/dragonfly/v2/client/pidfile" + server "d7y.io/dragonfly/v2/client/daemon" + "d7y.io/dragonfly/v2/cmd/dependency" + "d7y.io/dragonfly/v2/internal/dfpath" + "d7y.io/dragonfly/v2/pkg/basic/dfnet" logger "d7y.io/dragonfly/v2/pkg/dflog" "d7y.io/dragonfly/v2/pkg/dflog/logcore" - _ "d7y.io/dragonfly/v2/pkg/rpc/dfdaemon/server" - "d7y.io/dragonfly/v2/pkg/rpc/scheduler" - "d7y.io/dragonfly/v2/pkg/util/net/iputils" + "d7y.io/dragonfly/v2/pkg/rpc/dfdaemon/client" "github.com/gofrs/flock" - "github.com/google/uuid" "github.com/pkg/errors" "github.com/spf13/cobra" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/exporters/trace/jaeger" - "go.opentelemetry.io/otel/sdk/resource" - sdktrace "go.opentelemetry.io/otel/sdk/trace" - "go.opentelemetry.io/otel/semconv" + "github.com/spf13/viper" "gopkg.in/yaml.v3" ) -var daemonConfig *config.PeerHostOption +var ( + cfg *config.DaemonConfig +) +// daemonCmd represents the daemon command var daemonCmd = &cobra.Command{ - Use: "daemon", - Short: "Launch a peer daemon for downloading and uploading files.", - SilenceUsage: true, - + Use: "daemon", + Short: "start the client daemon of dragonfly", + Long: `client daemon is mainly responsible for transmitting blocks between peers +and putting the completed file into the specified target path. at the same time, +it supports container engine, wget and other downloading tools through proxy function.`, + Args: cobra.NoArgs, + DisableAutoGenTag: true, + SilenceUsage: true, + FParseErrWhitelist: cobra.FParseErrWhitelist{UnknownFlags: true}, RunE: func(cmd *cobra.Command, args []string) error { + if err := logcore.InitDaemon(cfg.Console); err != nil { + return errors.Wrap(err, "init client daemon logger") + } + // Convert config - if err := daemonConfig.Convert(); err != nil { + if err := cfg.Convert(); err != nil { return err } // Validate config - if err := daemonConfig.Validate(); err != nil { + if err := cfg.Validate(); err != nil { return err } - // Initialize logger - if err := logcore.InitDaemon(daemonConfig.Console); err != nil { - return errors.Wrap(err, "init daemon logger") - } - - // Initialize telemetry - if daemonConfig.Telemetry.Jaeger != "" { - flush, err := initTracer(daemonConfig.Telemetry.Jaeger) - if err != nil { - logger.Errorf("initialize trace for jaeger error: %s", err) - } else { - logger.Infof("initialize trace for jaeger at %s", daemonConfig.Telemetry.Jaeger) - defer flush() - } - } - - // Serve daemon return runDaemon() }, } func init() { - // Initialize default daemon config - daemonConfig = config.NewPeerHostOption() - - // Initialize cobra - initDaemonConfig(config.PeerHostConfigPath) - - // Add flags - flagSet := daemonCmd.Flags() - flagSet.StringVar(&daemonConfig.DataDir, "data", daemonConfig.DataDir, "local directory which stores temporary files for p2p uploading") - flagSet.DurationVar(&daemonConfig.GCInterval.Duration, "gc-interval", daemonConfig.GCInterval.Duration, "gc interval") - flagSet.BoolVar(&daemonConfig.KeepStorage, "keep-storage", daemonConfig.KeepStorage, "keep storage after daemon exit") - flagSet.BoolVar(&daemonConfig.Verbose, "verbose", daemonConfig.Verbose, "print verbose log and enable golang debug info") - flagSet.BoolVar(&daemonConfig.Console, "console", daemonConfig.Console, "console shows log on console") - flagSet.StringVar(&daemonConfig.Host.AdvertiseIP, "advertise-ip", daemonConfig.Host.AdvertiseIP, "the ip report to scheduler, normal same with listen ip") - flagSet.StringVar(&daemonConfig.Download.DownloadGRPC.UnixListen.Socket, "grpc-unix-listen", daemonConfig.Download.DownloadGRPC.UnixListen.Socket, "the local unix domain socket listen address for grpc with dfget") - flagSet.IntVar(&daemonConfig.Download.PeerGRPC.TCPListen.PortRange.Start, "grpc-port", daemonConfig.Download.PeerGRPC.TCPListen.PortRange.Start, "the listen address for grpc with other peers") - flagSet.IntVar(&daemonConfig.Download.PeerGRPC.TCPListen.PortRange.End, "grpc-port-end", daemonConfig.Download.PeerGRPC.TCPListen.PortRange.End, "the listen address for grpc with other peers") - flagSet.IntVar(&daemonConfig.Upload.ListenOption.TCPListen.PortRange.Start, "upload-port", daemonConfig.Upload.ListenOption.TCPListen.PortRange.Start, "the address that daemon will listen on for peer upload") - flagSet.IntVar(&daemonConfig.Upload.ListenOption.TCPListen.PortRange.End, "upload-port-end", daemonConfig.Upload.ListenOption.TCPListen.PortRange.End, "the address that daemon will listen on for peer upload") - flagSet.StringVar(&daemonConfig.PidFile, "pid", daemonConfig.PidFile, "dfdaemon pid file location") - flagSet.StringVar(&daemonConfig.LockFile, "lock", daemonConfig.LockFile, "dfdaemon lock file location") - flagSet.StringVar(&daemonConfig.Host.SecurityDomain, "security-domain", daemonConfig.Host.SecurityDomain, "peer security domain for scheduler") - flagSet.StringVar(&daemonConfig.Host.Location, "location", daemonConfig.Host.Location, "peer location for scheduler") - flagSet.StringVar(&daemonConfig.Host.IDC, "idc", daemonConfig.Host.IDC, "peer idc for scheduler") - flagSet.StringVar(&daemonConfig.Host.NetTopology, "net-topology", daemonConfig.Host.NetTopology, "peer net topology for scheduler") - flagSet.Var(config.NewLimitRateValue(&daemonConfig.Download.TotalRateLimit), "download-rate", "total download rate limit for other peers and back source") - flagSet.Var(config.NewLimitRateValue(&daemonConfig.Download.PerPeerRateLimit), "per-peer-download-rate", "per peer download rate limit for other peers and back source") - flagSet.Var(config.NewLimitRateValue(&daemonConfig.Upload.RateLimit), "upload-rate", "upload rate limit for other peers") - flagSet.DurationVar(&daemonConfig.Scheduler.ScheduleTimeout.Duration, "schedule-timeout", daemonConfig.Scheduler.ScheduleTimeout.Duration, "schedule timeout") - flagSet.StringVar(&daemonConfig.Telemetry.Jaeger, "jaeger", daemonConfig.Telemetry.Jaeger, "jaeger addr, like: http://localhost:14268") - flagSet.StringVar(&daemonConfig.ConfigServer, "configServer", daemonConfig.ConfigServer, "specify config server") - flagSet.String("config", config.PeerHostConfigPath, "daemon config file location") - - // Add command + // Add the command to parent rootCmd.AddCommand(daemonCmd) -} -// initDaemonConfig reads in config file if set -func initDaemonConfig(cfgPath string) { - var flagPath string - for i, v := range os.Args { - if v == "--config" && i+1 < len(os.Args) { - flagPath = os.Args[i+1] - } + if len(os.Args) > 1 && os.Args[1] == daemonCmd.Name() { + // Initialize default daemon config + cfg = config.NewDaemonConfig() + // Initialize cobra + dependency.InitCobra(daemonCmd, true, cfg) + + flags := daemonCmd.Flags() + flags.Int("launcher", -1, "pid of process launching daemon, a negative number implies that the daemon is started directly by the user") + flags.Lookup("launcher").Hidden = true + _ = viper.BindPFlags(flags) } - - if flagPath != "" { - cfgPath = flagPath - } - - _, err := os.Stat(cfgPath) - if err != nil { - if os.IsNotExist(err) { - return - } - - fmt.Println(err) - os.Exit(1) - } - - // Load from config file - if err := daemonConfig.Load(cfgPath); err != nil { - fmt.Println(err) - os.Exit(1) - } -} - -// initTracer creates a new trace provider instance and registers it as global trace provider. -func initTracer(addr string) (func(), error) { - // Create and install Jaeger export pipeline. - flush, err := jaeger.InstallNewPipeline( - jaeger.WithCollectorEndpoint(fmt.Sprintf("%s/api/traces", addr)), - jaeger.WithSDKOptions( - sdktrace.WithSampler(sdktrace.AlwaysSample()), - sdktrace.WithResource(resource.NewWithAttributes( - semconv.ServiceNameKey.String("dragonfly"), - attribute.String("exporter", "jaeger"), - )), - ), - ) - if err != nil { - return nil, err - } - return flush, nil } func runDaemon() error { - // Daemon config values - data, _ := yaml.Marshal(daemonConfig) - logger.Infof("loaded daemon option(debug only, can not use as config): \n%s", string(data)) - - // Initialize lock file - lock := flock.New(daemonConfig.LockFile) - if ok, err := lock.TryLock(); err != nil { + target := dfnet.NetAddr{Type: dfnet.UNIX, Addr: dfpath.DaemonSockPath} + daemonClient, err := client.GetClientByAddr([]dfnet.NetAddr{target}) + if err != nil { return err - } else if !ok { - return fmt.Errorf("lock file %s failed, other daemon is already running", daemonConfig.LockFile) + } + + // Checking Steps: + // + // 1. Try to lock + // + // 2. If lock successfully, start the client daemon and then return + // + // 3. If lock fail, checking whether the daemon has been started. If true, return directly. + // Otherwise, wait 50 ms and execute again from 1 + // 4. Checking timeout about 5s + lock := flock.New(dfpath.DaemonLockPath) + times := 0 + limit := 100 // 100 * 50ms = 5s + interval := 50 * time.Millisecond + for { + if ok, err := lock.TryLock(); err != nil { + return err + } else if !ok { + if daemonClient.CheckHealth(context.Background(), target) == nil { + return errors.New("the daemon is running, so there is no need to start it again") + } + } else { + break + } + + times++ + if times > limit { + return errors.New("the daemon is unhealthy") + } + + time.Sleep(interval) } defer lock.Unlock() - // Initialize pid file - pid, err := pidfile.New(daemonConfig.PidFile) - if err != nil { - return fmt.Errorf("check pid failed: %s, please check %s", err, daemonConfig.PidFile) - } - defer pid.Remove() + logger.Infof("daemon is launched by pid:%d", viper.GetInt("launcher")) - // Initialize verbose mode - initVerboseMode(daemonConfig.Verbose) + // daemon config values + s, _ := yaml.Marshal(cfg) + logger.Infof("client daemon configuration:\n%s", string(s)) - ph, err := daemon.NewPeerHost(&scheduler.PeerHost{ - Uuid: uuid.New().String(), - Ip: daemonConfig.Host.AdvertiseIP, - RpcPort: int32(daemonConfig.Download.PeerGRPC.TCPListen.PortRange.Start), - DownPort: 0, - HostName: iputils.HostName, - SecurityDomain: daemonConfig.Host.SecurityDomain, - Location: daemonConfig.Host.Location, - Idc: daemonConfig.Host.IDC, - NetTopology: daemonConfig.Host.NetTopology, - }, *daemonConfig) - if err != nil { - logger.Errorf("init peer host failed: %s", err) + ff := dependency.InitMonitor(cfg.Verbose, cfg.PProfPort, cfg.Jaeger) + defer ff() + + if svr, err := server.New(cfg); err != nil { return err + } else { + dependency.SetupQuitSignalHandler(func() { svr.Stop() }) + return svr.Serve() } - - setupSignalHandler(ph) - return ph.Serve() -} - -func setupSignalHandler(ph daemon.PeerHost) { - sigs := make(chan os.Signal, 1) - signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) - - go func() { - var done bool - for { - select { - case sig := <-sigs: - logger.Infof("receive %s signal", sig) - if !done { - ph.Stop() - done = true - } - } - } - }() } diff --git a/cmd/dfget/cmd/root.go b/cmd/dfget/cmd/root.go index 7cb513111..89e51e0b9 100644 --- a/cmd/dfget/cmd/root.go +++ b/cmd/dfget/cmd/root.go @@ -18,477 +18,209 @@ package cmd import ( "context" - "encoding/json" - "fmt" - "io" "os" "os/exec" - "path/filepath" "strconv" "strings" "syscall" "time" - "d7y.io/dragonfly/v2/cmd/common" - "github.com/go-echarts/statsview" - "github.com/go-echarts/statsview/viewer" - "github.com/go-http-utils/headers" - "github.com/gofrs/flock" - "github.com/phayes/freeport" - "github.com/spf13/cobra" - "go.uber.org/zap/zapcore" - - "d7y.io/dragonfly/v2/cdnsystem/source" - "d7y.io/dragonfly/v2/client/clientutil/progressbar" "d7y.io/dragonfly/v2/client/config" - "d7y.io/dragonfly/v2/client/pidfile" - "d7y.io/dragonfly/v2/pkg/basic" + "d7y.io/dragonfly/v2/client/dfget" + "d7y.io/dragonfly/v2/cmd/dependency" + "d7y.io/dragonfly/v2/internal/dfpath" "d7y.io/dragonfly/v2/pkg/basic/dfnet" - "d7y.io/dragonfly/v2/pkg/dferrors" logger "d7y.io/dragonfly/v2/pkg/dflog" "d7y.io/dragonfly/v2/pkg/dflog/logcore" - "d7y.io/dragonfly/v2/pkg/rpc/base" - dfdaemongrpc "d7y.io/dragonfly/v2/pkg/rpc/dfdaemon" - _ "d7y.io/dragonfly/v2/pkg/rpc/dfdaemon/client" - dfclient "d7y.io/dragonfly/v2/pkg/rpc/dfdaemon/client" + "d7y.io/dragonfly/v2/pkg/rpc/dfdaemon/client" + "d7y.io/dragonfly/v2/pkg/unit" + "github.com/gofrs/flock" + "github.com/pkg/errors" + "github.com/spf13/cobra" + "github.com/spf13/viper" + "gopkg.in/yaml.v3" ) -var filter string +var ( + dfgetConfig *config.DfgetConfig +) -var dfgetConfig *config.ClientOption - -// dfgetDescription is used to describe dfget command in details. -var dfgetDescription = `dfget is the client of Dragonfly which takes a role of peer in a P2P network. +var dfgetDescription = `dfget is the client of dragonfly which takes a role of peer in a P2P network. When user triggers a file downloading task, dfget will download the pieces of file from other peers. Meanwhile, it will act as an uploader to support other peers to download pieces from it if it owns them. In addition, dfget has the abilities to provide more advanced functionality, such as network bandwidth limit, transmission encryption and so on.` -// dfgetExample shows examples in dfget command, and is used in auto-generated cli docs. -var dfgetExample = ` -$ dfget -u https://example.com/1G -o /tmp/d7y.test -⠼ Downloaded (1.0 GB, 509.902 MB/s) -Task: 4d07b1df273af9c830296903f0ba0cc2290dc630b26f634d6ac95cddfce6a0ef -Peer: 10.0.0.1-30-59c54ceb-868a-4897-9832-577d2b347cce -Download success, time cost: 2008ms, length: 1073741824 -` - -var deprecatedFlags struct { - nodes config.SupernodesValue - version bool - - commonString string - commonBool bool - commonInt int -} - +// rootCmd represents the base command when called without any subcommands var rootCmd = &cobra.Command{ - Use: "dfget", - Short: "client of Dragonfly used to download and upload files", - SilenceUsage: true, - Long: dfgetDescription, - DisableAutoGenTag: true, // disable displaying auto generation tag in cli docs - Example: dfgetExample, + Use: "dfget url -O path", + Short: "the P2P client of dragonfly", + Long: dfgetDescription, + Args: cobra.MaximumNArgs(1), + DisableAutoGenTag: true, + FParseErrWhitelist: cobra.FParseErrWhitelist{UnknownFlags: true}, RunE: func(cmd *cobra.Command, args []string) error { - if deprecatedFlags.version { - common.VersionCmd.Run(nil, nil) - return nil + if err := logcore.InitDfget(dfgetConfig.Console); err != nil { + return errors.Wrap(err, "init client dfget logger") } - // Convent deprecated flags - convertDeprecatedFlags() - // Dfget config validate + // Convert config + if err := dfgetConfig.Convert(args); err != nil { + return err + } + + // Validate config if err := dfgetConfig.Validate(); err != nil { return err } - // Init logger - logcore.InitDfget(dfgetConfig.Console) - - // Serve dfget + // do get file return runDfget() }, } -// Execute will process dfget. +// Execute adds all child commands to the root command and sets flags appropriately. +// This is called by main.main(). It only needs to happen once to the rootCmd. func Execute() { if err := rootCmd.Execute(); err != nil { - logger.Errorf("Execute error: %s", err) + logger.Error(err) os.Exit(1) } } func init() { // Initialize default dfget config - dfgetConfig = config.NewClientOption() + dfgetConfig = config.NewDfgetConfig() + // Initialize cobra + dependency.InitCobra(rootCmd, false, dfgetConfig) // Add flags flagSet := rootCmd.Flags() - persistentflagSet := rootCmd.PersistentFlags() - flagSet.StringVarP(&dfgetConfig.URL, "url", "u", "", "URL of user requested downloading file(only HTTP/HTTPs supported)") - flagSet.StringVarP(&dfgetConfig.Output, "output", "o", "", - "destination path which is used to store the requested downloading file. It must contain detailed directory and specific filename, for example, '/tmp/file.mp4'") - flagSet.StringVarP(&dfgetConfig.Output, "", "O", "", "Deprecated, keep for backward compatibility, use --output or -o instead") - flagSet.Var(config.NewLimitRateValue(&daemonConfig.Download.TotalRateLimit), "totallimit", - "network bandwidth rate limit for the whole host, in format of G(B)/g/M(B)/m/K(B)/k/B, pure number will also be parsed as Byte") - flagSet.VarP(config.NewDurationValue(&dfgetConfig.Timeout), "timeout", "e", - "timeout for file downloading task. If dfget has not finished downloading all pieces of file before --timeout, the dfget will throw an error and exit") - flagSet.Var(config.NewDurationValue(&dfgetConfig.Timeout), "exceed", - "timeout for file downloading task. If dfget has not finished downloading all pieces of file before --timeout, the dfget will throw an error and exit") - flagSet.StringVarP(&dfgetConfig.Md5, "md5", "m", "", - "md5 value input from user for the requested downloading file to enhance security") - flagSet.StringVarP(&dfgetConfig.Identifier, "identifier", "i", "", - "the usage of identifier is making different downloading tasks generate different downloading task IDs even if they have the same URLs. conflict with --md5.") - flagSet.StringVar(&dfgetConfig.CallSystem, "callsystem", "", - "the name of dfget caller which is for debugging. Once set, it will be passed to all components around the request to make debugging easy") - flagSet.StringSliceVar(&dfgetConfig.Cacerts, "cacerts", nil, - "the cacert file which is used to verify remote server when supernode interact with the source.") - flagSet.StringVarP(&dfgetConfig.Pattern, "pattern", "p", "p2p", - "download pattern, must be p2p/cdn/source, cdn and source do not support flag --totallimit") - flagSet.StringVarP(&filter, "filter", "f", "", - "filter some query params of URL, use char '&' to separate different params"+ - "\neg: -f 'key&sign' will filter 'key' and 'sign' query param"+ - "\nin this way, different but actually the same URLs can reuse the same downloading task") - flagSet.StringArrayVar(&dfgetConfig.Header, "header", nil, - "http header, eg: --header='Accept: *' --header='Host: abc'") - flagSet.VarP(&deprecatedFlags.nodes, "node", "n", - "deprecated, please use schedulers instead. specify the addresses(host:port=weight) of supernodes where the host is necessary, the port(default: 8002) and the weight(default:1) are optional. And the type of weight must be integer") - flagSet.BoolVar(&dfgetConfig.NotBackSource, "notbacksource", false, - "disable back source downloading for requested file when p2p fails to download it") - flagSet.BoolVar(&dfgetConfig.NotBackSource, "notbs", false, - "disable back source downloading for requested file when p2p fails to download it") - flagSet.BoolVar(&deprecatedFlags.commonBool, "dfdaemon", false, - "identify whether the request is from dfdaemon") - flagSet.BoolVar(&dfgetConfig.Insecure, "insecure", false, - "identify whether supernode should skip secure verify when interact with the source.") - flagSet.IntVar(&deprecatedFlags.commonInt, "clientqueue", 0, - "specify the size of client queue which controls the number of pieces that can be processed simultaneously") - flagSet.BoolVarP(&dfgetConfig.ShowBar, "showbar", "b", false, - "show progress bar, it is conflict with '--console'") - flagSet.BoolVar(&dfgetConfig.Console, "console", false, - "show log on console, it's conflict with '--showbar'") - flagSet.BoolVar(&dfgetConfig.Verbose, "verbose", true, - "enable verbose mode, all debug log will be display") - persistentflagSet.StringVar(&daemonConfig.WorkHome, "home", daemonConfig.WorkHome, - "the work home directory") - persistentflagSet.StringVar(&daemonConfig.Host.ListenIP, "ip", daemonConfig.Host.ListenIP, - "IP address that server will listen on") - flagSet.IntVar(&daemonConfig.Upload.ListenOption.TCPListen.PortRange.Start, "port", daemonConfig.Upload.ListenOption.TCPListen.PortRange.Start, - "port number that server will listen on") - persistentflagSet.DurationVar(&daemonConfig.Storage.TaskExpireTime.Duration, "expiretime", daemonConfig.Storage.TaskExpireTime.Duration, - "caching duration for which cached file keeps no accessed by any process, after this period cache file will be deleted") - persistentflagSet.DurationVar(&daemonConfig.AliveTime.Duration, "alivetime", daemonConfig.AliveTime.Duration, - "alive duration for which uploader keeps no accessing by any uploading requests, after this period uploader will automatically exit") - flagSet.StringVar(&daemonConfig.Download.DownloadGRPC.UnixListen.Socket, "daemon-sock", - daemonConfig.Download.DownloadGRPC.UnixListen.Socket, "the unix domain socket address for grpc with daemon") - flagSet.StringVar(&daemonConfig.PidFile, "daemon-pid", daemonConfig.PidFile, "the daemon pid") - persistentflagSet.VarP(config.NewNetAddrsValue(&daemonConfig.Scheduler.NetAddrs), "scheduler", "s", "the scheduler addresses") - flagSet.StringVar(&dfgetConfig.MoreDaemonOptions, "more-daemon-options", "", - "more options passed to daemon by command line, please confirm your options with \"dfget daemon --help\"") + flagSet.StringP("url", "u", dfgetConfig.URL, + "download a file from the url, equivalent to the command's first position argument") - // backward compatibility - flagSet.BoolVar(&deprecatedFlags.commonBool, "cachefirst", false, "deprecated") - flagSet.BoolVar(&deprecatedFlags.commonBool, "check", false, "deprecated") - flagSet.BoolVar(&deprecatedFlags.commonBool, "createmeta", false, "deprecated") - flagSet.BoolVar(&deprecatedFlags.commonBool, "notmd5", false, "deprecated") - flagSet.BoolVar(&deprecatedFlags.commonBool, "showcenter", false, "deprecated") - flagSet.BoolVar(&deprecatedFlags.commonBool, "usewrap", false, "deprecated") - flagSet.StringVar(&deprecatedFlags.commonString, "locallimit", "", "deprecated") - flagSet.StringVar(&deprecatedFlags.commonString, "minrate", "", "deprecated") - flagSet.StringVarP(&deprecatedFlags.commonString, "tasktype", "t", "", "deprecated") - flagSet.StringVarP(&deprecatedFlags.commonString, "center", "c", "", "deprecated") - flagSet.BoolVarP(&deprecatedFlags.version, "version", "v", false, "deprecated") + flagSet.StringP("output", "O", dfgetConfig.Output, + "destination path which is used to store the downloaded file. It must be a full path, for example, '/tmp/file.mp4'") - flagSet.MarkDeprecated("exceed", "please use '--timeout' or '-e' instead") - flagSet.MarkDeprecated("clientqueue", "controlled by Manager and Scheduler") - flagSet.MarkDeprecated("dfdaemon", "not used anymore") - flagSet.MarkDeprecated("version", "Please use 'dfget version' instead") - flagSet.MarkShorthandDeprecated("v", "Please use 'dfget version' instead") + flagSet.DurationP("timeout", "e", dfgetConfig.Timeout, + "timeout for file downloading task. If dfget has not finished downloading all pieces of file "+ + "before --timeout, the dfget will throw an error and exit, default see --benchmark-rate") - // Add command - rootCmd.AddCommand(common.VersionCmd) -} + flagSet.String("benchmark-rate", dfgetConfig.BenchmarkRate.String(), + "benchmark rate in format of G(B)/g/M(B)/m/K(B)/k/B which is used to calculate the default --timeout, "+ + "calculation formula: fileLength/benchmark-rate") -// Convert flags -func convertDeprecatedFlags() { - for _, node := range deprecatedFlags.nodes.Nodes { - daemonConfig.Scheduler.NetAddrs = append(daemonConfig.Scheduler.NetAddrs, dfnet.NetAddr{ - Type: dfnet.TCP, - Addr: node, - }) + flagSet.String("limit", unit.Bytes(dfgetConfig.RateLimit).String(), + "network bandwidth rate limit in format of G(B)/g/M(B)/m/K(B)/k/B, pure number will be parsed as Byte, 0 is infinite") + + flagSet.String("digest", dfgetConfig.Digest, + "digest is used to check the integrity of the downloaded file, in format of md5:xxx or sha256:yyy") + + flagSet.StringP("identifier", "i", dfgetConfig.Identifier, + "different identifiers for the same url will be divided into different P2P tasks, it conflicts with --digest") + + flagSet.StringP("filter", "f", strings.Join(dfgetConfig.Filter, "&"), + "filter some query params of url, use char '&' to separate different params, eg: -f 'key&sign' "+ + "will filter 'key' and 'sign' query param. in this way, different urls can correspond to the same P2P task") + + flagSet.Bool("not-back-source", dfgetConfig.NotBackSource, + "disable dfget downloading file directly from url source when peer fails to download file") + + flagSet.StringP("pattern", "p", dfgetConfig.Pattern, "downloading pattern, must be p2p/cdn/source") + + flagSet.StringArrayP("header", "H", dfgetConfig.Header, "url header, eg: --header='Accept: *' --header='Host: abc'") + + flagSet.StringArray("cacerts", dfgetConfig.Cacerts, + "cacert files is used to verify CA for remote server when dragonfly interacts with the url source") + + flagSet.Bool("insecure", dfgetConfig.Insecure, + "identify whether dragonfly should skip CA verification for remote server when it interacts with the url source") + + flagSet.BoolP("show-progress", "b", dfgetConfig.ShowBar, "show progress bar, it conflicts with --console") + + flagSet.String("callsystem", dfgetConfig.CallSystem, "the system name of dfget caller which is mainly used for statistics and access control") + + // Bind cmd flags + if err := viper.BindPFlags(flagSet); err != nil { + panic(errors.Wrap(err, "bind dfget flags to viper")) } } // runDfget does some init operations and starts to download. func runDfget() error { // Dfget config values - s, _ := json.MarshalIndent(dfgetConfig, "", " ") - logger.Debugf("dfget option(debug only, can not use as config):\n%s", string(s)) + s, _ := yaml.Marshal(dfgetConfig) + logger.Infof("client dfget configuration:\n%s", string(s)) - var addr = dfnet.NetAddr{ - Type: dfnet.UNIX, - Addr: daemonConfig.Download.DownloadGRPC.UnixListen.Socket, - } - var ( - ctx = context.Background() - cancel context.CancelFunc - hdr = parseHeader(dfgetConfig.Header) - ) + ff := dependency.InitMonitor(dfgetConfig.Verbose, dfgetConfig.PProfPort, dfgetConfig.Jaeger) + defer ff() - // Initialize verbose mode - initVerboseMode(dfgetConfig.Verbose) - - // Check df daemon state, start a new daemon if necessary - daemonClient, err := checkAndSpawnDaemon(addr) + logger.Info("start to check and spawn daemon") + daemonClient, err := checkAndSpawnDaemon() if err != nil { - logger.Errorf("connect daemon error: %s", err) - return downloadFromSource(hdr, err) - } - - output, err := filepath.Abs(dfgetConfig.Output) - if err != nil { - return err - } - - if dfgetConfig.Timeout > 0 { - ctx, cancel = context.WithTimeout(ctx, dfgetConfig.Timeout) - defer cancel() + logger.Errorf("check and spawn daemon error:%v", err) } else { - ctx, cancel = context.WithCancel(ctx) - defer cancel() + logger.Info("check and spawn daemon success") } - request := &dfdaemongrpc.DownRequest{ - Url: dfgetConfig.URL, - UrlMeta: &base.UrlMeta{ - Md5: dfgetConfig.Md5, - Range: hdr[headers.Range], - Header: hdr, - }, - Output: output, - BizId: dfgetConfig.CallSystem, - Filter: filter, - Uid: int64(basic.UserId), - Gid: int64(basic.UserGroup), - } - var ( - start = time.Now() - end time.Time - ) - down, err := daemonClient.Download(ctx, request) - if err != nil { - return err - } - var ( - result *dfdaemongrpc.DownResult - ) - pb := progressbar.DefaultBytes(-1, "Downloading") - for { - result, err = down.Recv() - if err != nil { - if de, ok := err.(*dferrors.DfError); ok { - logger.Errorf("dragonfly daemon returns error code %d/%s", de.Code, de.Message) - } else { - logger.Errorf("dragonfly daemon returns error %s", err) - } - break - } - if result.CompletedLength > 0 { - pb.Set64(int64(result.CompletedLength)) - } - if result.Done { - pb.Describe("Downloaded") - pb.Finish() - end = time.Now() - fmt.Printf("Task: %s\nPeer: %s\n", result.TaskId, result.PeerId) - fmt.Printf("Download success, time cost: %dms, length: %d\n", end.Sub(start).Milliseconds(), result.CompletedLength) - break - } - } - if err != nil { - logger.Errorf("download by dragonfly error: %s", err) - return downloadFromSource(hdr, err) - } - return err + return dfget.Download(dfgetConfig, daemonClient) } -func initVerboseMode(verbose bool) { - if !verbose { - return - } - - logcore.SetCoreLevel(zapcore.DebugLevel) - logcore.SetGrpcLevel(zapcore.DebugLevel) - - go func() { - // enable go pprof and statsview - port, _ := strconv.Atoi(os.Getenv("D7Y_PPROF_PORT")) - if port == 0 { - port, _ = freeport.GetFreePort() - } - - debugListen := fmt.Sprintf("localhost:%d", port) - viewer.SetConfiguration(viewer.WithAddr(debugListen)) - - logger.With("pprof", fmt.Sprintf("http://%s/debug/pprof", debugListen), - "statsview", fmt.Sprintf("http://%s/debug/statsview", debugListen)). - Infof("enable debug at http://%s", debugListen) - - if err := statsview.New().Start(); err != nil { - logger.Warnf("serve go pprof error: %s", err) - } - }() -} - -func downloadFromSource(hdr map[string]string, dferr error) (err error) { - if dfgetConfig.NotBackSource { - err = fmt.Errorf("dfget download error: %s, and back source disabled", dferr) - logger.Warnf("%s", err) - return err - } - - var ( - start = time.Now() - end time.Time - ) - - fmt.Printf("dfget download error: %s, try to download from source", dferr) - var ( - resourceClient source.ResourceClient - target *os.File - response io.ReadCloser - _ map[string]string - written int64 - ) - - resourceClient, err = source.NewSourceClient() - if err != nil { - logger.Errorf("init source client error: %s", err) - return err - } - - response, _, err = resourceClient.Download(dfgetConfig.URL, hdr) - if err != nil { - logger.Errorf("download from source error: %s", err) - return err - } - defer response.Close() - - target, err = os.OpenFile(dfgetConfig.Output, os.O_RDWR|os.O_CREATE, 0644) - if err != nil { - logger.Errorf("open %s error: %s", dfgetConfig.Output, err) - return err - } - - written, err = io.Copy(target, response) - if err == nil { - logger.Infof("copied %d bytes to %s", written, dfgetConfig.Output) - end = time.Now() - fmt.Printf("Download from source success, time cost: %dms\n", end.Sub(start).Milliseconds()) - // change permission - logger.Infof("change own to uid %d gid %d", basic.UserId, basic.UserGroup) - if err = os.Chown(dfgetConfig.Output, basic.UserId, basic.UserGroup); err != nil { - logger.Errorf("change own failed: %s", err) - return err - } - return nil - } - logger.Errorf("copied %d bytes to %s, with error: %s", - written, dfgetConfig.Output, err) - return err -} - -func checkAndSpawnDaemon(addr dfnet.NetAddr) (dfclient.DaemonClient, error) { - // Check pid - if ok, err := pidfile.IsProcessExistsByPIDFile(daemonConfig.PidFile); err != nil || !ok { - logger.Infof("daemon pid not found, try to start daemon") - if err = spawnDaemon(); err != nil { - return nil, fmt.Errorf("start daemon error: %s", err) - } - } - - // Check socket - _, err := os.Stat(addr.Addr) - if os.IsNotExist(err) { - logger.Warnf("daemon addr not found, try to start daemon again") - if err = spawnDaemon(); err != nil { - return nil, fmt.Errorf("start daemon error: %s", err) - } - } else if err != nil { - return nil, fmt.Errorf("unknown error when stat daemon socket: %s", err) - } - - // Check daemon health - return probeDaemon(addr) -} - -func probeDaemon(addr dfnet.NetAddr) (dfclient.DaemonClient, error) { - dc, err := dfclient.GetClientByAddr([]dfnet.NetAddr{addr}) +// checkAndSpawnDaemon do checking at four checkpoints +func checkAndSpawnDaemon() (client.DaemonClient, error) { + target := dfnet.NetAddr{Type: dfnet.UNIX, Addr: dfpath.DaemonSockPath} + daemonClient, err := client.GetClientByAddr([]dfnet.NetAddr{target}) if err != nil { return nil, err } - err = dc.CheckHealth(context.Background(), addr) - if err != nil { - return nil, err + // 1.Check without lock + if daemonClient.CheckHealth(context.Background(), target) == nil { + return daemonClient, nil } - return dc, nil -} - -func spawnDaemon() error { - // Initialize lock file - lock := flock.New(dfgetConfig.LockFile) + lock := flock.New(dfpath.DfgetLockPath) lock.Lock() defer lock.Unlock() - // Initialize daemon args - var args = []string{ - "daemon", - "--download-rate", fmt.Sprintf("%f", daemonConfig.Download.TotalRateLimit.Limit), - "--upload-port", fmt.Sprintf("%d", daemonConfig.Upload.TCPListen.PortRange.Start), - "--home", daemonConfig.WorkHome, - "--ip", daemonConfig.Host.ListenIP, - "--expiretime", daemonConfig.Storage.TaskExpireTime.String(), - "--alivetime", daemonConfig.AliveTime.String(), - "--grpc-unix-listen", daemonConfig.Download.DownloadGRPC.UnixListen.Socket, - "--pid", daemonConfig.PidFile, - } - - // Set more daemon options - if dfgetConfig.MoreDaemonOptions != "" { - args = append(args, strings.Split(dfgetConfig.MoreDaemonOptions, " ")...) - } - cmd := exec.Command(os.Args[0], args...) - - // Set verbose - if dfgetConfig.Verbose { - cmd.Args = append(cmd.Args, "--verbose") - } - - // Set scheduler - for _, s := range daemonConfig.Scheduler.NetAddrs { - cmd.Args = append(cmd.Args, "--scheduler", s.Addr) + // 2.Check with lock + if daemonClient.CheckHealth(context.Background(), target) == nil { + return daemonClient, nil } + cmd := exec.Command(os.Args[0], "daemon", "--launcher", strconv.Itoa(os.Getpid())) cmd.Stdin = nil cmd.Stdout = nil cmd.Stderr = nil cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true} - logger.Infof("start daemon with cmd: %s", strings.Join(cmd.Args, " ")) - return cmd.Start() -} + logger.Info("do start daemon") -func parseHeader(s []string) map[string]string { - hdr := map[string]string{} - for _, h := range s { - idx := strings.Index(h, ":") - if idx > 0 { - hdr[h[:idx]] = strings.TrimLeft(h[idx:], " ") - } + err = cmd.Start() + if err != nil { + return nil, err + } + + // 3.First check since starting + if daemonClient.CheckHealth(context.Background(), target) == nil { + return daemonClient, nil + } + + times := 0 + limit := 100 + interval := 50 * time.Millisecond + for { + // 4.Cycle check with 5s timeout + if daemonClient.CheckHealth(context.Background(), target) == nil { + return daemonClient, nil + } + + times++ + if times > limit { + return nil, errors.New("the daemon is unhealthy") + } + time.Sleep(interval) } - return hdr } diff --git a/cmd/manager/cmd/root.go b/cmd/manager/cmd/root.go index 4da8c0d21..4852bd8da 100644 --- a/cmd/manager/cmd/root.go +++ b/cmd/manager/cmd/root.go @@ -19,7 +19,7 @@ package cmd import ( "os" - "d7y.io/dragonfly/v2/cmd/common" + "d7y.io/dragonfly/v2/cmd/dependency" "d7y.io/dragonfly/v2/manager/config" "d7y.io/dragonfly/v2/manager/server" logger "d7y.io/dragonfly/v2/pkg/dflog" @@ -30,11 +30,7 @@ import ( ) var ( - cfg *config.Config -) - -const ( - managerEnvPrefix = "manager" + cfg *config.Config ) // rootCmd represents the base command when called without any subcommands @@ -67,9 +63,8 @@ func Execute() { func init() { // Initialize default manager config cfg = config.New() - // Initialize cobra - common.InitCobra(rootCmd, managerEnvPrefix, cfg) + dependency.InitCobra(rootCmd, true, cfg) } func runManager() error { @@ -77,8 +72,8 @@ func runManager() error { s, _ := yaml.Marshal(cfg) logger.Infof("manager configuration:\n%s", string(s)) - // initialize verbose mode - common.InitVerboseMode(cfg.Verbose, cfg.PProfPort) + ff := dependency.InitMonitor(cfg.Verbose, cfg.PProfPort, cfg.Jaeger) + defer ff() if svr, err := server.New(cfg); err != nil { return err diff --git a/cmd/scheduler/cmd/root.go b/cmd/scheduler/cmd/root.go index f06e2ab6f..d74a7d16f 100644 --- a/cmd/scheduler/cmd/root.go +++ b/cmd/scheduler/cmd/root.go @@ -19,7 +19,7 @@ package cmd import ( "os" - "d7y.io/dragonfly/v2/cmd/common" + "d7y.io/dragonfly/v2/cmd/dependency" logger "d7y.io/dragonfly/v2/pkg/dflog" "d7y.io/dragonfly/v2/pkg/dflog/logcore" "d7y.io/dragonfly/v2/scheduler/config" @@ -30,11 +30,7 @@ import ( ) var ( - cfg *config.Config -) - -const ( - schedulerEnvPrefix = "scheduler" + cfg *config.Config ) // rootCmd represents the base command when called without any subcommands @@ -67,9 +63,8 @@ func Execute() { func init() { // Initialize default scheduler config cfg = config.New() - // Initialize cobra - common.InitCobra(rootCmd, schedulerEnvPrefix, cfg) + dependency.InitCobra(rootCmd, true, cfg) } func runScheduler() error { @@ -77,8 +72,8 @@ func runScheduler() error { s, _ := yaml.Marshal(cfg) logger.Infof("scheduler configuration:\n%s", string(s)) - // initialize verbose mode - common.InitVerboseMode(cfg.Verbose, cfg.PProfPort) + ff := dependency.InitMonitor(cfg.Verbose, cfg.PProfPort, cfg.Jaeger) + defer ff() if svr, err := server.New(cfg); err != nil { return err diff --git a/docs/en/config/cdn.yaml b/docs/en/config/cdn.yaml index b337852b4..e413dc5e1 100644 --- a/docs/en/config/cdn.yaml +++ b/docs/en/config/cdn.yaml @@ -44,9 +44,6 @@ base: # StoragePattern is the pattern of storage policy, [disk/hybrid] storagePattern: disk - # Console shows log on console - console: false - plugins: storage: - name: disk diff --git a/go.mod b/go.mod index 148fc8d2a..5507daf4c 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/satori/go.uuid v1.2.0 // indirect github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b - github.com/shirou/gopsutil/v3 v3.20.12 + github.com/shirou/gopsutil/v3 v3.21.4 github.com/sirupsen/logrus v1.2.0 github.com/spf13/afero v1.2.2 // indirect github.com/spf13/cobra v1.1.1 @@ -41,10 +41,10 @@ require ( github.com/swaggo/swag v1.7.0 github.com/valyala/fasthttp v1.22.0 github.com/willf/bitset v1.1.11 - go.opentelemetry.io/otel v0.19.0 - go.opentelemetry.io/otel/exporters/trace/jaeger v0.19.0 - go.opentelemetry.io/otel/sdk v0.19.0 - go.opentelemetry.io/otel/trace v0.19.0 + go.opentelemetry.io/otel v0.20.0 + go.opentelemetry.io/otel/exporters/trace/jaeger v0.20.0 + go.opentelemetry.io/otel/sdk v0.20.0 + go.opentelemetry.io/otel/trace v0.20.0 go.uber.org/atomic v1.6.0 go.uber.org/zap v1.16.0 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 @@ -57,7 +57,6 @@ require ( gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/errgo.v2 v2.1.0 gopkg.in/natefinch/lumberjack.v2 v2.0.0 - gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 gorm.io/driver/mysql v1.0.4 gorm.io/gorm v1.21.3 diff --git a/go.sum b/go.sum index b0e7156d5..33a257709 100644 --- a/go.sum +++ b/go.sum @@ -5,35 +5,11 @@ cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6A cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -71,12 +47,7 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -102,8 +73,6 @@ github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -128,8 +97,6 @@ github.com/go-echarts/go-echarts/v2 v2.2.3/go.mod h1:6TOomEztzGDVDkOSCFBq3ed7xOY github.com/go-echarts/statsview v0.3.4 h1:CCuytRAutdnF901NrR4BzSjHXjUp8OyA3/iopgG/1/Y= github.com/go-echarts/statsview v0.3.4/go.mod h1:AehKjL9cTFMeIo5QdV8sQO43vFmfY65X5GMWa3XMciY= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a h1:v6zMvHuY9yue4+QkG/HQ/W67wvtQmWJ4SDo9aK/GIno= github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a/go.mod h1:I79BieaU4fxrw4LMXby6q5OS9XnoR9UIKLOzDFjUmuw= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -181,25 +148,17 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -215,29 +174,15 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -277,8 +222,6 @@ github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0m github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= @@ -294,7 +237,6 @@ github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= @@ -403,8 +345,8 @@ github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdh github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b h1:h+3JX2VoWTFuyQEo87pStk/a99dzIO1mM9KxIyLPGTU= github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b/go.mod h1:/yeG0My1xr/u+HZrFQ1tOQQQQrOawfyMUH13ai5brBc= -github.com/shirou/gopsutil/v3 v3.20.12 h1:abpcjSQRHdb3thCge/UyJty9CnvvmUHljTSrjtFU+Og= -github.com/shirou/gopsutil/v3 v3.20.12/go.mod h1:igHnfak0qnw1biGeI2qKQvu0ZkwvEkUcCLlYhZzdr/4= +github.com/shirou/gopsutil/v3 v3.21.4 h1:XB/+p+kVnyYLuPHCfa99lxz2aJyvVhnyd+FxZqH/k7M= +github.com/shirou/gopsutil/v3 v3.21.4/go.mod h1:ghfMypLDrFSWN2c9cDYFLHyynQ+QUht0cv/18ZqVczw= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -451,6 +393,10 @@ github.com/swaggo/gin-swagger v1.3.0/go.mod h1:oy1BRA6WvgtCp848lhxce7BnWH4C8Bxa0 github.com/swaggo/swag v1.5.1/go.mod h1:1Bl9F/ZBpVWh22nY0zmYyASPO1lI/zIwRDrpZU+tv8Y= github.com/swaggo/swag v1.7.0 h1:5bCA/MTLQoIqDXXyHfOpMeDvL9j68OY/udlK4pQoo4E= github.com/swaggo/swag v1.7.0/go.mod h1:BdPIL73gvS9NBsdi7M1JOxLvlbfvNRaBP8m6WT6Aajo= +github.com/tklauser/go-sysconf v0.3.4 h1:HT8SVixZd3IzLdfs/xlpq0jeSfTX57g1v6wB1EuzV7M= +github.com/tklauser/go-sysconf v0.3.4/go.mod h1:Cl2c8ZRWfHD5IrfHo9VN+FX9kCFjIOyVklgXycLB6ek= +github.com/tklauser/numcpus v0.2.1 h1:ct88eFm+Q7m2ZfXJdan1xYoXKlmwsfP+k88q05KvlZc= +github.com/tklauser/numcpus v0.2.1/go.mod h1:9aU+wOc6WjUIZEwWMP62PL/41d65P+iks1gBkr4QyP8= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= @@ -470,30 +416,22 @@ github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV github.com/willf/bitset v1.1.11 h1:N7Z7E9UvjW+sGsEl7k/SJrvY2reP1A07MrGuCjIOjRE= github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/otel v0.19.0 h1:Lenfy7QHRXPZVsw/12CWpxX6d/JkrX8wrx2vO8G80Ng= -go.opentelemetry.io/otel v0.19.0/go.mod h1:j9bF567N9EfomkSidSfmMwIwIBuP37AMAIzVW85OxSg= -go.opentelemetry.io/otel/exporters/trace/jaeger v0.19.0 h1:qU7sGQoDrlAvNYryR2SJT4ULP/q+5tE38W+h31WEE/M= -go.opentelemetry.io/otel/exporters/trace/jaeger v0.19.0/go.mod h1:BliRm9d7rH44N6CzBQ0OPEPfMqSzf4WvFFvyoocOW9Y= -go.opentelemetry.io/otel/metric v0.19.0 h1:dtZ1Ju44gkJkYvo+3qGqVXmf88tc+a42edOywypengg= -go.opentelemetry.io/otel/metric v0.19.0/go.mod h1:8f9fglJPRnXuskQmKpnad31lcLJ2VmNNqIsx/uIwBSc= -go.opentelemetry.io/otel/oteltest v0.19.0 h1:YVfA0ByROYqTwOxqHVZYZExzEpfZor+MU1rU+ip2v9Q= -go.opentelemetry.io/otel/oteltest v0.19.0/go.mod h1:tI4yxwh8U21v7JD6R3BcA/2+RBoTKFexE/PJ/nSO7IA= -go.opentelemetry.io/otel/sdk v0.19.0 h1:13pQquZyGbIvGxBWcVzUqe8kg5VGbTBiKKKXpYCylRM= -go.opentelemetry.io/otel/sdk v0.19.0/go.mod h1:ouO7auJYMivDjywCHA6bqTI7jJMVQV1HdKR5CmH8DGo= -go.opentelemetry.io/otel/trace v0.19.0 h1:1ucYlenXIDA1OlHVLDZKX0ObXV5RLaq06DtUKz5e5zc= -go.opentelemetry.io/otel/trace v0.19.0/go.mod h1:4IXiNextNOpPnRlI4ryK69mn5iC84bjBWZQA5DXz/qg= +go.opentelemetry.io/otel v0.20.0 h1:eaP0Fqu7SXHwvjiqDq83zImeehOHX8doTvU9AwXON8g= +go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= +go.opentelemetry.io/otel/exporters/trace/jaeger v0.20.0 h1:FoclOadJNul1vUiKnZU0sKFWOZtZQq3jUzSbrX2jwNM= +go.opentelemetry.io/otel/exporters/trace/jaeger v0.20.0/go.mod h1:10qwvAmKpvwRO5lL3KQ8EWznPp89uGfhcbK152LFWsQ= +go.opentelemetry.io/otel/metric v0.20.0 h1:4kzhXFP+btKm4jwxpjIqjs41A7MakRFUS86bqLHTIw8= +go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= +go.opentelemetry.io/otel/oteltest v0.20.0 h1:HiITxCawalo5vQzdHfKeZurV8x7ljcqAgiWzF6Vaeaw= +go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= +go.opentelemetry.io/otel/sdk v0.20.0 h1:JsxtGXd06J8jrnya7fdI/U/MR6yXA5DtbZy+qoHQlr8= +go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= +go.opentelemetry.io/otel/trace v0.20.0 h1:1DL6EXUdcg95gukhuRRvLDO/4X5THh/5dIV52lqtnbw= +go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -521,11 +459,6 @@ golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -534,23 +467,14 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1 h1:Kvvh58BN8Y9/lBi7hTekvtMpm07eUZ0ck5pRHpsMWrY= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -569,27 +493,10 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226101413-39120d07d75e/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= @@ -597,23 +504,12 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -634,44 +530,21 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201024232916-9f70ab9862d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf h1:MZ2shdL+ZM/XzY3ZGOnh4Nlpnxz5GSOhOmtHo3iPU6M= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -681,7 +554,6 @@ golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 h1:Hir2P/De0WpUhtrKGGjvSb2YxUgyZ7EFOSLIcSSpiwE= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -708,39 +580,9 @@ golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201120155355-20be4ac4bd6e h1:t96dS3DO8DGjawSLJL/HIdz8CycAd2v07XxqB3UPTi0= golang.org/x/tools v0.0.0-20201120155355-20be4ac4bd6e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -751,29 +593,10 @@ google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0 h1:12aHIhhQCpWtd3Rcp2WwbboB5W72tJHcjzyA9MCoHAw= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -783,50 +606,14 @@ google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb h1:hcskBH5qZCOa7WpTUFUFvoebnSFZBYpjykLtjIp9DVk= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0 h1:o1bcQ6imQMIOpdrO3SWf2z5RV72WbDwdXuK0MDlc8As= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -837,7 +624,6 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -880,10 +666,8 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/apimachinery v0.20.1 h1:LAhz8pKbgR8tUwn7boK+b2HZdt7MiTu2mkYtFMUjTRQ= k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/client-go v11.0.0+incompatible h1:LBbX2+lOwY9flffWlJM7f1Ct8V2SRNiMRDFeiwnJo9o= @@ -894,8 +678,6 @@ k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/cmd/common/config_darwin.go b/internal/dfpath/config_darwin.go similarity index 84% rename from cmd/common/config_darwin.go rename to internal/dfpath/config_darwin.go index b602e4ca3..8d07ad1bc 100644 --- a/cmd/common/config_darwin.go +++ b/internal/dfpath/config_darwin.go @@ -14,12 +14,10 @@ * limitations under the License. */ -package common +package dfpath import ( "path/filepath" - - "d7y.io/dragonfly/v2/pkg/basic" ) -var defaultConfigDir = filepath.Join(basic.HomeDir, ".dragonfly") +var DefaultConfigDir = filepath.Join(WorkHome, "config") diff --git a/cmd/common/config_linux.go b/internal/dfpath/config_linux.go similarity index 91% rename from cmd/common/config_linux.go rename to internal/dfpath/config_linux.go index ca2aec037..8d09a1bd2 100644 --- a/cmd/common/config_linux.go +++ b/internal/dfpath/config_linux.go @@ -14,6 +14,6 @@ * limitations under the License. */ -package common +package dfpath -var defaultConfigDir = "/etc/dragonfly" +var DefaultConfigDir = "/etc/dragonfly" diff --git a/internal/dfpath/dfpath.go b/internal/dfpath/dfpath.go new file mode 100644 index 000000000..24ec651e0 --- /dev/null +++ b/internal/dfpath/dfpath.go @@ -0,0 +1,48 @@ +/* + * Copyright 2020 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dfpath + +import ( + "path/filepath" + + "d7y.io/dragonfly/v2/pkg/util/fileutils" +) + +var ( + DefaultDataDir = filepath.Join(WorkHome, "data") + DaemonSockPath = filepath.Join(WorkHome, "daemon.sock") + DaemonLockPath = filepath.Join(WorkHome, "daemon.lock") + DfgetLockPath = filepath.Join(WorkHome, "dfget.lock") +) + +func init() { + if err := fileutils.MkdirAll(WorkHome); err != nil { + panic(err) + } + + if err := fileutils.MkdirAll(DefaultConfigDir); err != nil { + panic(err) + } + + if err := fileutils.MkdirAll(LogDir); err != nil { + panic(err) + } + + if err := fileutils.MkdirAll(DefaultDataDir); err != nil { + panic(err) + } +} diff --git a/internal/dfpath/home_darwin.go b/internal/dfpath/home_darwin.go new file mode 100644 index 000000000..84420fb62 --- /dev/null +++ b/internal/dfpath/home_darwin.go @@ -0,0 +1,25 @@ +/* + * Copyright 2020 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dfpath + +import ( + "path/filepath" + + "d7y.io/dragonfly/v2/pkg/basic" +) + +var WorkHome = filepath.Join(basic.HomeDir, ".dragonfly") diff --git a/internal/dfpath/home_linux.go b/internal/dfpath/home_linux.go new file mode 100644 index 000000000..a9496b914 --- /dev/null +++ b/internal/dfpath/home_linux.go @@ -0,0 +1,19 @@ +/* + * Copyright 2020 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dfpath + +var WorkHome = "/usr/local/dragonfly" diff --git a/pkg/dflog/logcore/log_darwin.go b/internal/dfpath/log_darwin.go similarity index 84% rename from pkg/dflog/logcore/log_darwin.go rename to internal/dfpath/log_darwin.go index db7c883a9..a2dc944ae 100644 --- a/pkg/dflog/logcore/log_darwin.go +++ b/internal/dfpath/log_darwin.go @@ -14,12 +14,10 @@ * limitations under the License. */ -package logcore +package dfpath import ( "path/filepath" - - "d7y.io/dragonfly/v2/pkg/basic" ) -var clientLogDir = filepath.Join(basic.HomeDir, ".dragonfly/logs") \ No newline at end of file +var LogDir = filepath.Join(WorkHome, "logs") diff --git a/pkg/dflog/logcore/log_linux.go b/internal/dfpath/log_linux.go similarity index 91% rename from pkg/dflog/logcore/log_linux.go rename to internal/dfpath/log_linux.go index ca90a3fa9..92eba5c47 100644 --- a/pkg/dflog/logcore/log_linux.go +++ b/internal/dfpath/log_linux.go @@ -14,6 +14,6 @@ * limitations under the License. */ -package logcore +package dfpath -var clientLogDir = "/var/log/dragonfly" +var LogDir = "/var/log/dragonfly" diff --git a/manager/config/config.go b/manager/config/config.go index b67d85982..2e4b973e7 100644 --- a/manager/config/config.go +++ b/manager/config/config.go @@ -1,54 +1,52 @@ package config -const ( - DefaultConfigFilePath string = "/etc/dragonfly/manager.yaml" +import ( + "d7y.io/dragonfly/v2/cmd/dependency/base" ) type Config struct { - Console bool `yaml:"console"` - Verbose bool `yaml:"verbose"` - PProfPort int `yaml:"pprofPort"` - Server *ServerConfig `yaml:"server"` - Configure *ConfigureConfig `yaml:"configure"` - Stores []*StoreConfig `yaml:"stores"` - HostService *HostService `yaml:"host-service"` + base.Options `yaml:",inline" mapstructure:",squash"` + Server *ServerConfig `yaml:"server" mapstructure:"server"` + Configure *ConfigureConfig `yaml:"configure" mapstructure:"configure"` + Stores []*StoreConfig `yaml:"stores" mapstructure:"stores"` + HostService *HostService `yaml:"host-service" mapstructure:"host-service"` } type ServerConfig struct { - IP string `yaml:"ip"` - Port int `yaml:"port"` + IP string `yaml:"ip" mapstructure:"ip"` + Port int `yaml:"port" mapstructure:"port"` } type ConfigureConfig struct { - StoreName string `yaml:"store-name"` + StoreName string `yaml:"store-name" mapstructure:"store-name"` } type MysqlConfig struct { - User string `yaml:"user"` - Password string `yaml:"password"` - IP string `yaml:"ip"` - Port int `yaml:"port"` - Db string `yaml:"db"` + User string `yaml:"user" mapstructure:"user"` + Password string `yaml:"password" mapstructure:"password"` + IP string `yaml:"ip" mapstructure:"ip"` + Port int `yaml:"port" mapstructure:"port"` + Db string `yaml:"db" mapstructure:"db"` } type OssConfig struct { } type StoreConfig struct { - Name string `yaml:"name"` - Type string `yaml:"type"` - Mysql *MysqlConfig `yaml:"mysql,omitempty"` - Oss *OssConfig `yaml:"oss,omitempty"` + Name string `yaml:"name" mapstructure:"name"` + Type string `yaml:"type" mapstructure:"type"` + Mysql *MysqlConfig `yaml:"mysql,omitempty" mapstructure:"mysql,omitempty"` + Oss *OssConfig `yaml:"oss,omitempty" mapstructure:"oss,omitempty"` } type HostService struct { } type SkylineService struct { - Domain string `yaml:"domain"` - AppName string `yaml:"app-name"` - Account string `yaml:"account"` - AccessKey string `yaml:"access-key"` + Domain string `yaml:"domain" mapstructure:"domain"` + AppName string `yaml:"app-name" mapstructure:"app-name"` + Account string `yaml:"account" mapstructure:"account"` + AccessKey string `yaml:"access-key" mapstructure:"access-key"` } func New() *Config { diff --git a/pkg/basic/dfnet/type.go b/pkg/basic/dfnet/type.go index 05a545524..f093ef1b1 100644 --- a/pkg/basic/dfnet/type.go +++ b/pkg/basic/dfnet/type.go @@ -31,9 +31,9 @@ const ( ) type NetAddr struct { - Type NetworkType `json:"type" yaml:"type"` + Type NetworkType `mapstructure:"type" yaml:"type"` // see https://github.com/grpc/grpc/blob/master/doc/naming.md - Addr string `json:"addr" yaml:"addr"` + Addr string `mapstructure:"addr" yaml:"addr"` } func (n NetAddr) GetEndpoint() string { diff --git a/pkg/basic/user.go b/pkg/basic/user.go index 4d76ec37a..0104fc9da 100644 --- a/pkg/basic/user.go +++ b/pkg/basic/user.go @@ -40,26 +40,18 @@ func init() { } Username = u.Username - UserId, err = strconv.Atoi(u.Uid) - UserGroup, err = strconv.Atoi(u.Gid) + UserId, _ = strconv.Atoi(u.Uid) + UserGroup, _ = strconv.Atoi(u.Gid) HomeDir = u.HomeDir HomeDir = strings.TrimSpace(HomeDir) if stringutils.IsBlank(HomeDir) { panic("home dir is empty") } - HomeDir = strings.TrimRight(HomeDir, "/") - if stringutils.IsBlank(HomeDir) { - HomeDir = "/" - } TmpDir = os.TempDir() TmpDir = strings.TrimSpace(TmpDir) if stringutils.IsBlank(TmpDir) { TmpDir = "/tmp" } - TmpDir = strings.TrimRight(TmpDir, "/") - if stringutils.IsBlank(TmpDir) { - TmpDir = "/" - } } diff --git a/pkg/dflog/logcore/init.go b/pkg/dflog/logcore/init.go index 27160031c..0afab7b07 100644 --- a/pkg/dflog/logcore/init.go +++ b/pkg/dflog/logcore/init.go @@ -20,6 +20,7 @@ import ( "fmt" "path" + "d7y.io/dragonfly/v2/internal/dfpath" "d7y.io/dragonfly/v2/pkg/basic" "d7y.io/dragonfly/v2/pkg/dflog" ) @@ -136,19 +137,19 @@ func InitDaemon(console bool) error { return nil } - if coreLogger, err := CreateLogger(path.Join(clientLogDir, fmt.Sprintf("dfdaemon-%s", CoreLogFileName)), 100, 7, 14, false, false); err != nil { + if coreLogger, err := CreateLogger(path.Join(dfpath.LogDir, fmt.Sprintf("dfdaemon-%s", CoreLogFileName)), 100, 7, 14, false, false); err != nil { return err } else { logger.SetCoreLogger(coreLogger.Sugar()) } - if grpcLogger, err := CreateLogger(path.Join(clientLogDir, fmt.Sprintf("dfdaemon-%s", GrpcLogFileName)), 100, 7, 14, false, false); err != nil { + if grpcLogger, err := CreateLogger(path.Join(dfpath.LogDir, fmt.Sprintf("dfdaemon-%s", GrpcLogFileName)), 100, 7, 14, false, false); err != nil { return err } else { logger.SetGrpcLogger(grpcLogger.Sugar()) } - if gcLogger, err := CreateLogger(path.Join(clientLogDir, "gc.log"), 100, 7, 14, false, false); err != nil { + if gcLogger, err := CreateLogger(path.Join(dfpath.LogDir, "gc.log"), 100, 7, 14, false, false); err != nil { return err } else { logger.SetGcLogger(gcLogger.Sugar()) @@ -162,7 +163,7 @@ func InitDfget(console bool) error { return nil } - if dfgetLogger, err := CreateLogger(path.Join(clientLogDir, "dfget.log"), 300, -1, -1, false, false); err != nil { + if dfgetLogger, err := CreateLogger(path.Join(dfpath.LogDir, "dfget.log"), 300, -1, -1, false, false); err != nil { return err } else { log := dfgetLogger.Sugar() diff --git a/pkg/pidfile/pidfile.go b/pkg/pidfile/pidfile.go new file mode 100644 index 000000000..24b795da7 --- /dev/null +++ b/pkg/pidfile/pidfile.go @@ -0,0 +1,85 @@ +/* + * Copyright 2020 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package pidfile + +import ( + "fmt" + "io/ioutil" + "os" + "strconv" + "strings" + + "d7y.io/dragonfly/v2/pkg/util/fileutils" + "github.com/pkg/errors" + "github.com/shirou/gopsutil/v3/process" +) + +// PIDFile is a file used to store the process ID and cmdline of a running process. +type PIDFile struct { + path string + pid int + cmdline string +} + +func IsProcessExistsByPIDFile(path string) (bool, error) { + bytes, err := ioutil.ReadFile(path) + if err != nil { + return false, err + } + + content := strings.TrimSpace(string(bytes)) + index := strings.LastIndex(content, "@") + if index == -1 { + return false, errors.New("pid file content is invalid") + } + + pid, err := strconv.Atoi(content[index+1:]) + if err != nil { + return false, err + } + p, err := process.NewProcess(int32(pid)) + if err != nil { + return false, err + } + cmdline, _ := p.Cmdline() + return strings.TrimSpace(cmdline+"@"+strconv.Itoa(pid)) == content, nil +} + +// New creates a PIDFile using the specified path. +func New(path string) (*PIDFile, error) { + //if ok, _ := IsProcessExistsByPIDFile(path); ok { + // return nil, errors.Errorf("process already exists") + //} + + p, err := process.NewProcess(int32(os.Getpid())) + if err != nil { + return nil, err + } + + cmdline, _ := p.Cmdline() + + if err := ioutil.WriteFile(path, []byte(fmt.Sprintf("%s@%d", cmdline, p.Pid)), 0644); err != nil { + return nil, err + } + + return &PIDFile{path: path, pid: int(p.Pid), cmdline: cmdline}, nil +} + +// Remove removes the PIDFile. +func (pf *PIDFile) Remove() error { + return fileutils.DeleteFile(pf.path) +} diff --git a/pkg/rpc/dfdaemon/client/client.go b/pkg/rpc/dfdaemon/client/client.go index a13d659be..34d996874 100644 --- a/pkg/rpc/dfdaemon/client/client.go +++ b/pkg/rpc/dfdaemon/client/client.go @@ -97,7 +97,7 @@ func (dc *daemonClient) getDaemonClientWithTarget(target string) (dfdaemon.Daemo func (dc *daemonClient) Download(ctx context.Context, req *dfdaemon.DownRequest, opts ...grpc.CallOption) (*DownResultStream, error) { req.Uuid = uuid.New().String() - // 生成taskId + // generate taskId taskId := idgen.TaskID(req.Url, req.Filter, req.UrlMeta, req.BizId) return newDownResultStream(dc, ctx, taskId, req, opts) } diff --git a/scheduler/config/config.go b/scheduler/config/config.go index 2fd7d0255..22fb94a04 100644 --- a/scheduler/config/config.go +++ b/scheduler/config/config.go @@ -16,50 +16,52 @@ package config +import ( + "d7y.io/dragonfly/v2/cmd/dependency/base" +) + type Config struct { - Console bool `yaml:"console"` - Verbose bool `yaml:"verbose"` - PProfPort int `yaml:"pprofPort"` - ConfigServer string `yaml:"configServer"` - Scheduler SchedulerConfig `yaml:"scheduler"` - Server ServerConfig `yaml:"server"` - Worker SchedulerWorkerConfig `yaml:"worker"` - CDN CDNConfig `yaml:"cdn"` - GC GCConfig `yaml:"gc"` + base.Options `yaml:",inline" mapstructure:",squash"` + ConfigServer string `yaml:"configServer" mapstructure:"configServer"` + Scheduler SchedulerConfig `yaml:"scheduler" mapstructure:"scheduler"` + Server ServerConfig `yaml:"server" mapstructure:"server"` + Worker SchedulerWorkerConfig `yaml:"worker" mapstructure:"worker"` + CDN CDNConfig `yaml:"cdn" mapstructure:"cdn"` + GC GCConfig `yaml:"gc" mapstructure:"gc"` } type SchedulerConfig struct { - ABTest bool `yaml:"abtest"` - AScheduler string `yaml:"ascheduler"` - BScheduler string `yaml:"bscheduler"` + ABTest bool `yaml:"abtest" mapstructure:"abtest"` + AScheduler string `yaml:"ascheduler" mapstructure:"ascheduler"` + BScheduler string `yaml:"bscheduler" mapstructure:"bscheduler"` } type ServerConfig struct { - IP string `yaml:"ip"` - Port int `yaml:"port"` + IP string `yaml:"ip" mapstructure:"ip"` + Port int `yaml:"port" mapstructure:"port"` } type SchedulerWorkerConfig struct { - WorkerNum int `yaml:"workerNum"` - WorkerJobPoolSize int `yaml:"workerJobPoolSize"` - SenderNum int `yaml:"senderNum"` - SenderJobPoolSize int `yaml:"senderJobPoolSize"` + WorkerNum int `yaml:"workerNum" mapstructure:"workerNum"` + WorkerJobPoolSize int `yaml:"workerJobPoolSize" mapstructure:"workerJobPoolSize"` + SenderNum int `yaml:"senderNum" mapstructure:"senderNum"` + SenderJobPoolSize int `yaml:"senderJobPoolSize" mapstructure:"senderJobPoolSize"` } type CDNServerConfig struct { - Name string `yaml:"name"` - IP string `yaml:"ip"` - RpcPort int32 `yaml:"rpcPort"` - DownloadPort int32 `yaml:"downloadPort"` + Name string `yaml:"name" mapstructure:"name"` + IP string `yaml:"ip" mapstructure:"ip"` + RpcPort int32 `yaml:"rpcPort" mapstructure:"rpcPort"` + DownloadPort int32 `yaml:"downloadPort" mapstructure:"downloadPort"` } type CDNConfig struct { - Servers []CDNServerConfig `yaml:"servers"` + Servers []CDNServerConfig `yaml:"servers" mapstructure:"servers"` } type GCConfig struct { - PeerTaskDelay int64 `yaml:"peerTaskDelay"` - TaskDelay int64 `yaml:"taskDelay"` + PeerTaskDelay int64 `yaml:"peerTaskDelay" mapstructure:"peerTaskDelay"` + TaskDelay int64 `yaml:"taskDelay" mapstructure:"taskDelay"` } func New() *Config { diff --git a/scheduler/config/config_darwin.go b/scheduler/config/config_darwin.go index 052fbec38..a58ff9729 100644 --- a/scheduler/config/config_darwin.go +++ b/scheduler/config/config_darwin.go @@ -13,8 +13,6 @@ var ( ) var config = Config{ - Console: false, - Verbose: true, Server: ServerConfig{ Port: 8002, }, diff --git a/scheduler/config/config_linux.go b/scheduler/config/config_linux.go index 4ead42647..0f3ba149c 100644 --- a/scheduler/config/config_linux.go +++ b/scheduler/config/config_linux.go @@ -9,8 +9,6 @@ var ( ) var config = Config{ - Console: false, - Verbose: true, Server: ServerConfig{ Port: 8002, }, diff --git a/scheduler/config/config_test.go b/scheduler/config/config_test.go index 313ecab18..50129c11f 100644 --- a/scheduler/config/config_test.go +++ b/scheduler/config/config_test.go @@ -23,15 +23,13 @@ import ( "github.com/mitchellh/mapstructure" testifyassert "github.com/stretchr/testify/assert" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) func TestSchedulerConfig_Load(t *testing.T) { assert := testifyassert.New(t) config := &Config{ - Console: true, - Verbose: true, Scheduler: SchedulerConfig{ ABTest: true, AScheduler: "a-scheduler",