400 lines
10 KiB
Go
400 lines
10 KiB
Go
/*
|
|
* Copyright 2020 The Dragonfly Authors
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
package config
|
|
|
|
import (
|
|
"time"
|
|
|
|
"github.com/pkg/errors"
|
|
|
|
"d7y.io/dragonfly/v2/cmd/dependency/base"
|
|
"d7y.io/dragonfly/v2/pkg/util/hostutils"
|
|
"d7y.io/dragonfly/v2/pkg/util/net/iputils"
|
|
"d7y.io/dragonfly/v2/scheduler/storage"
|
|
)
|
|
|
|
type Config struct {
|
|
// Base options.
|
|
base.Options `yaml:",inline" mapstructure:",squash"`
|
|
|
|
// Scheduler configuration.
|
|
Scheduler *SchedulerConfig `yaml:"scheduler" mapstructure:"scheduler"`
|
|
|
|
// Server configuration.
|
|
Server *ServerConfig `yaml:"server" mapstructure:"server"`
|
|
|
|
// Dynconfig configuration.
|
|
DynConfig *DynConfig `yaml:"dynConfig" mapstructure:"dynConfig"`
|
|
|
|
// Manager configuration.
|
|
Manager *ManagerConfig `yaml:"manager" mapstructure:"manager"`
|
|
|
|
// SeedPeer configuration.
|
|
SeedPeer *SeedPeerConfig `yaml:"seedPeer" mapstructure:"seedPeer"`
|
|
|
|
// Host configuration.
|
|
Host *HostConfig `yaml:"host" mapstructure:"host"`
|
|
|
|
// Job configuration.
|
|
Job *JobConfig `yaml:"job" mapstructure:"job"`
|
|
|
|
// Storage configuration.
|
|
Storage *StorageConfig `yaml:"storage" mapstructure:"storage"`
|
|
|
|
// Metrics configuration.
|
|
Metrics *MetricsConfig `yaml:"metrics" mapstructure:"metrics"`
|
|
}
|
|
|
|
// New default configuration.
|
|
func New() *Config {
|
|
return &Config{
|
|
Server: &ServerConfig{
|
|
IP: iputils.IPv4,
|
|
Host: hostutils.FQDNHostname,
|
|
Listen: "0.0.0.0",
|
|
Port: 8002,
|
|
},
|
|
Scheduler: &SchedulerConfig{
|
|
Algorithm: "default",
|
|
BackSourceCount: 3,
|
|
RetryBackSourceLimit: 5,
|
|
RetryLimit: 10,
|
|
RetryInterval: 50 * time.Millisecond,
|
|
GC: &GCConfig{
|
|
PeerGCInterval: 10 * time.Minute,
|
|
PeerTTL: 24 * time.Hour,
|
|
TaskGCInterval: 10 * time.Minute,
|
|
TaskTTL: 24 * time.Hour,
|
|
HostGCInterval: 30 * time.Minute,
|
|
HostTTL: 48 * time.Hour,
|
|
},
|
|
},
|
|
DynConfig: &DynConfig{
|
|
RefreshInterval: 10 * time.Second,
|
|
},
|
|
Host: &HostConfig{},
|
|
Manager: &ManagerConfig{
|
|
SchedulerClusterID: 1,
|
|
KeepAlive: KeepAliveConfig{
|
|
Interval: 5 * time.Second,
|
|
},
|
|
},
|
|
SeedPeer: &SeedPeerConfig{
|
|
Enable: true,
|
|
},
|
|
Job: &JobConfig{
|
|
Enable: true,
|
|
GlobalWorkerNum: 10,
|
|
SchedulerWorkerNum: 10,
|
|
LocalWorkerNum: 10,
|
|
Redis: &RedisConfig{
|
|
Port: 6379,
|
|
BrokerDB: 1,
|
|
BackendDB: 2,
|
|
},
|
|
},
|
|
Storage: &StorageConfig{
|
|
MaxSize: storage.DefaultMaxSize,
|
|
MaxBackups: storage.DefaultMaxBackups,
|
|
BufferSize: storage.DefaultBufferSize,
|
|
},
|
|
Metrics: &MetricsConfig{
|
|
Enable: false,
|
|
EnablePeerHost: false,
|
|
},
|
|
}
|
|
}
|
|
|
|
// Validate config parameters.
|
|
func (c *Config) Validate() error {
|
|
if c.Server == nil {
|
|
return errors.New("server requires parameter server")
|
|
}
|
|
|
|
if c.Server.IP == "" {
|
|
return errors.New("server requires parameter ip")
|
|
}
|
|
|
|
if c.Server.Host == "" {
|
|
return errors.New("server requires parameter host")
|
|
}
|
|
|
|
if c.Server.Port <= 0 {
|
|
return errors.New("server requires parameter port")
|
|
}
|
|
|
|
if c.Server.Listen == "" {
|
|
return errors.New("server requires parameter listen")
|
|
}
|
|
|
|
if c.Scheduler.Algorithm == "" {
|
|
return errors.New("scheduler requires parameter algorithm")
|
|
}
|
|
|
|
if c.Scheduler.RetryLimit <= 0 {
|
|
return errors.New("scheduler requires parameter retryLimit")
|
|
}
|
|
|
|
if c.Scheduler.RetryInterval <= 0 {
|
|
return errors.New("scheduler requires parameter retryInterval")
|
|
}
|
|
|
|
if c.Scheduler.GC == nil {
|
|
return errors.New("scheduler requires parameter gc")
|
|
}
|
|
|
|
if c.Scheduler.GC.PeerGCInterval <= 0 {
|
|
return errors.New("scheduler requires parameter peerGCInterval")
|
|
}
|
|
|
|
if c.Scheduler.GC.PeerTTL <= 0 {
|
|
return errors.New("scheduler requires parameter peerTTL")
|
|
}
|
|
|
|
if c.Scheduler.GC.TaskGCInterval <= 0 {
|
|
return errors.New("scheduler requires parameter taskGCInterval")
|
|
}
|
|
|
|
if c.Scheduler.GC.TaskTTL <= 0 {
|
|
return errors.New("scheduler requires parameter taskTTL")
|
|
}
|
|
|
|
if c.DynConfig.RefreshInterval <= 0 {
|
|
return errors.New("dynconfig requires parameter refreshInterval")
|
|
}
|
|
|
|
if c.Manager.Addr == "" {
|
|
return errors.New("manager requires parameter addr")
|
|
}
|
|
|
|
if c.Manager.SchedulerClusterID == 0 {
|
|
return errors.New("manager requires parameter schedulerClusterID")
|
|
}
|
|
|
|
if c.Manager.KeepAlive.Interval <= 0 {
|
|
return errors.New("manager requires parameter keepAlive interval")
|
|
}
|
|
|
|
if c.Job != nil && c.Job.Enable {
|
|
if c.Job.GlobalWorkerNum == 0 {
|
|
return errors.New("job requires parameter globalWorkerNum")
|
|
}
|
|
|
|
if c.Job.SchedulerWorkerNum == 0 {
|
|
return errors.New("job requires parameter schedulerWorkerNum")
|
|
}
|
|
|
|
if c.Job.LocalWorkerNum == 0 {
|
|
return errors.New("job requires parameter localWorkerNum")
|
|
}
|
|
|
|
if c.Job.Redis.Host == "" {
|
|
return errors.New("job requires parameter redis host")
|
|
}
|
|
|
|
if c.Job.Redis.Port <= 0 {
|
|
return errors.New("job requires parameter redis port")
|
|
}
|
|
|
|
if c.Job.Redis.BrokerDB <= 0 {
|
|
return errors.New("job requires parameter redis brokerDB")
|
|
}
|
|
|
|
if c.Job.Redis.BackendDB <= 0 {
|
|
return errors.New("job requires parameter redis backendDB")
|
|
}
|
|
}
|
|
|
|
if c.Storage == nil {
|
|
return errors.New("server requires parameter storage")
|
|
}
|
|
|
|
if c.Storage.MaxSize <= 0 {
|
|
return errors.New("storage requires parameter maxSize")
|
|
}
|
|
|
|
if c.Storage.MaxBackups <= 0 {
|
|
return errors.New("storage requires parameter maxBackups")
|
|
}
|
|
|
|
if c.Storage.BufferSize <= 0 {
|
|
return errors.New("storage requires parameter bufferSize")
|
|
}
|
|
|
|
if c.Metrics != nil && c.Metrics.Enable {
|
|
if c.Metrics.Addr == "" {
|
|
return errors.New("metrics requires parameter addr")
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
type ServerConfig struct {
|
|
// Server ip.
|
|
IP string `yaml:"ip" mapstructure:"ip"`
|
|
|
|
// Server hostname.
|
|
Host string `yaml:"host" mapstructure:"host"`
|
|
|
|
// Listen stands listen interface, like: 0.0.0.0, 192.168.0.1.
|
|
Listen string `yaml:"listen" mapstructure:"listen"`
|
|
|
|
// Server port.
|
|
Port int `yaml:"port" mapstructure:"port"`
|
|
|
|
// Server dynamic config cache directory.
|
|
CacheDir string `yaml:"cacheDir" mapstructure:"cacheDir"`
|
|
|
|
// Server log directory.
|
|
LogDir string `yaml:"logDir" mapstructure:"logDir"`
|
|
}
|
|
|
|
type SchedulerConfig struct {
|
|
// Scheduling algorithm used by the scheduler.
|
|
Algorithm string `yaml:"algorithm" mapstructure:"algorithm"`
|
|
|
|
// Single task allows the client to back-to-source count.
|
|
BackSourceCount int `yaml:"backSourceCount" mapstructure:"backSourceCount"`
|
|
|
|
// Retry scheduling back-to-source limit times.
|
|
RetryBackSourceLimit int `yaml:"retryBackSourceLimit" mapstructure:"retryBackSourceLimit"`
|
|
|
|
// Retry scheduling limit times.
|
|
RetryLimit int `yaml:"retryLimit" mapstructure:"retryLimit"`
|
|
|
|
// Retry scheduling interval.
|
|
RetryInterval time.Duration `yaml:"retryInterval" mapstructure:"retryInterval"`
|
|
|
|
// Task and peer gc configuration.
|
|
GC *GCConfig `yaml:"gc" mapstructure:"gc"`
|
|
}
|
|
|
|
type GCConfig struct {
|
|
// Peer gc interval.
|
|
PeerGCInterval time.Duration `yaml:"peerGCInterval" mapstructure:"peerGCInterval"`
|
|
|
|
// Peer time to live.
|
|
PeerTTL time.Duration `yaml:"peerTTL" mapstructure:"peerTTL"`
|
|
|
|
// Task gc interval.
|
|
TaskGCInterval time.Duration `yaml:"taskGCInterval" mapstructure:"taskGCInterval"`
|
|
|
|
// Task time to live.
|
|
TaskTTL time.Duration `yaml:"taskTTL" mapstructure:"taskTTL"`
|
|
|
|
// Host gc interval.
|
|
HostGCInterval time.Duration `yaml:"hostGCInterval" mapstructure:"hostGCInterval"`
|
|
|
|
// Host time to live.
|
|
HostTTL time.Duration `yaml:"hostTTL" mapstructure:"hostTTL"`
|
|
}
|
|
|
|
type DynConfig struct {
|
|
// RefreshInterval is refresh interval for manager cache.
|
|
RefreshInterval time.Duration `yaml:"refreshInterval" mapstructure:"refreshInterval"`
|
|
}
|
|
|
|
type HostConfig struct {
|
|
// IDC for scheduler.
|
|
IDC string `mapstructure:"idc" yaml:"idc"`
|
|
|
|
// NetTopology for scheduler.
|
|
NetTopology string `mapstructure:"netTopology" yaml:"netTopology"`
|
|
|
|
// Location for scheduler.
|
|
Location string `mapstructure:"location" yaml:"location"`
|
|
}
|
|
|
|
type ManagerConfig struct {
|
|
// Addr is manager address.
|
|
Addr string `yaml:"addr" mapstructure:"addr"`
|
|
|
|
// SchedulerClusterID is scheduler cluster id.
|
|
SchedulerClusterID uint `yaml:"schedulerClusterID" mapstructure:"schedulerClusterID"`
|
|
|
|
// KeepAlive configuration.
|
|
KeepAlive KeepAliveConfig `yaml:"keepAlive" mapstructure:"keepAlive"`
|
|
}
|
|
|
|
type SeedPeerConfig struct {
|
|
// Enable is to enable seed peer as P2P peer.
|
|
Enable bool `yaml:"enable" mapstructure:"enable"`
|
|
}
|
|
|
|
type KeepAliveConfig struct {
|
|
// Keep alive interval.
|
|
Interval time.Duration `yaml:"interval" mapstructure:"interval"`
|
|
}
|
|
|
|
type JobConfig struct {
|
|
// Enable job service.
|
|
Enable bool `yaml:"enable" mapstructure:"enable"`
|
|
|
|
// Number of workers in global queue.
|
|
GlobalWorkerNum uint `yaml:"globalWorkerNum" mapstructure:"globalWorkerNum"`
|
|
|
|
// Number of workers in scheduler queue.
|
|
SchedulerWorkerNum uint `yaml:"schedulerWorkerNum" mapstructure:"schedulerWorkerNum"`
|
|
|
|
// Number of workers in local queue.
|
|
LocalWorkerNum uint `yaml:"localWorkerNum" mapstructure:"localWorkerNum"`
|
|
|
|
// Redis configuration.
|
|
Redis *RedisConfig `yaml:"redis" mapstructure:"redis"`
|
|
}
|
|
|
|
type StorageConfig struct {
|
|
// MaxSize sets the maximum size in megabytes of storage file.
|
|
MaxSize int `yaml:"maxSize" mapstructure:"maxSize"`
|
|
|
|
// MaxBackups sets the maximum number of storage files to retain.
|
|
MaxBackups int `yaml:"maxBackups" mapstructure:"maxBackups"`
|
|
|
|
// BufferSize sets the size of buffer container,
|
|
// if the buffer is full, write all the records in the buffer to the file.
|
|
BufferSize int `yaml:"bufferSize" mapstructure:"bufferSize"`
|
|
}
|
|
|
|
type RedisConfig struct {
|
|
// Server hostname.
|
|
Host string `yaml:"host" mapstructure:"host"`
|
|
|
|
// Server port.
|
|
Port int `yaml:"port" mapstructure:"port"`
|
|
|
|
// Server password.
|
|
Password string `yaml:"password" mapstructure:"password"`
|
|
|
|
// Broker database name.
|
|
BrokerDB int `yaml:"brokerDB" mapstructure:"brokerDB"`
|
|
|
|
// Backend database name.
|
|
BackendDB int `yaml:"backendDB" mapstructure:"backendDB"`
|
|
}
|
|
|
|
type MetricsConfig struct {
|
|
// Enable metrics service.
|
|
Enable bool `yaml:"enable" mapstructure:"enable"`
|
|
|
|
// Metrics service address.
|
|
Addr string `yaml:"addr" mapstructure:"addr"`
|
|
|
|
// Enable peer host metrics.
|
|
EnablePeerHost bool `yaml:"enablePeerHost" mapstructure:"enablePeerHost"`
|
|
}
|