docs: Update documents (#595)
* docs: Add zh docs (#592) Co-authored-by: santong <244372610@qq.com> Co-authored-by: Jim Ma <majinjing3@gmail.com> Co-authored-by: yxxhero <11087727+yxxhero@users.noreply.github.com>
This commit is contained in:
zzy987
Gaius
parent
f46f370b32
commit
36a32460d5
|
|
@ -8,3 +8,9 @@ We encourage users to experience Dragonfly in different ways. When doing this, t
|
|||
[swagger2markup](https://github.com/Swagger2Markup/swagger2markup). Please **DO NOT** edit [api.md](../docs/en/api-reference/api.md) directly.
|
||||
|
||||
Directory `/api` mainly describes the second part **Dragonfly Manager's Raw API**.
|
||||
|
||||
Currently, the description of swagger format is provided, which can be viewed in `manager`. It contains two format files of swagger, JSON and yaml. You can
|
||||
browse through relevant viewing tools to obtain the specific call method of API, or view online through the `/swagger/doc.json` endpoint of manager service.
|
||||
|
||||
Through manager's API, you can integrate with your own services, so you can automatically manage dragonfly and improve the level of Devops.
|
||||
|
||||
|
|
|
|||
|
|
@ -19,14 +19,15 @@ package cmd
|
|||
import (
|
||||
"os"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/cobra"
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"d7y.io/dragonfly/v2/cmd/dependency"
|
||||
logger "d7y.io/dragonfly/v2/internal/dflog"
|
||||
"d7y.io/dragonfly/v2/internal/dflog/logcore"
|
||||
"d7y.io/dragonfly/v2/scheduler"
|
||||
"d7y.io/dragonfly/v2/scheduler/config"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/cobra"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
@ -37,7 +38,8 @@ var (
|
|||
var rootCmd = &cobra.Command{
|
||||
Use: "scheduler",
|
||||
Short: "the scheduler of dragonfly",
|
||||
Long: `scheduler is a long-running process and is mainly responsible for deciding which peers transmit blocks to each other.`,
|
||||
Long: `Scheduler is a long-running process which receives and manages download tasks from the client, notify the CDN to return to the source,
|
||||
generate and maintain a P2P network during the download process, and push suitable download nodes to the client`,
|
||||
Args: cobra.NoArgs,
|
||||
DisableAutoGenTag: true,
|
||||
SilenceUsage: true,
|
||||
|
|
|
|||
|
|
@ -2,5 +2,5 @@
|
|||
|
||||
[English](en/README.md)
|
||||
|
||||
[TODO 简体中文](./zh-CN/README.md)
|
||||
[简体中文](zh-CN/README.md)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# Dragonfly Document
|
||||
|
||||
Dragonfly Document is written, drawn, memorialized representation of all things about Dragonfly. For those who are generally interested in Dragonfly, README.md of this repo is sufficient. While for end users, all details contained in `/docs` is the best guide all should have. And for developers, contents in Developer Guide part is that all need.
|
||||
Dragonfly Document is written, drawn, memorialized representation of all things about Dragonfly. For those who are generally interested in Dragonfly, README.md of this repo is sufficient. While for end users, all details contained in `/docs` is the best guide all should have. And for developers, contents in [Developer Guide](#Developer Guide) part is that all need.
|
||||
|
||||
Organization of document is as following:
|
||||
|
||||
|
|
@ -10,8 +10,8 @@ Organization of document is as following:
|
|||
* [WIP dfget](cli-reference/dfget.md)
|
||||
* [WIP cdnsystem](cli-reference/cdn.md)
|
||||
* [WIP scheduler](cli-reference/scheduler.md)
|
||||
* [TODO manager](cli-reference/manager.md)
|
||||
* [TODO API Reference](#API-Reference)
|
||||
* [manager](cli-reference/manager.md)
|
||||
* [API Reference](#API-Reference)
|
||||
* [Ecosystem](#Ecosystem)
|
||||
* [Kubernetes Integration](ecosystem/Kubernetes-with-Dragonfly.md)
|
||||
* [WIP Harbor Integration](ecosystem/Harbor-with-Dragonfly.md)
|
||||
|
|
@ -39,7 +39,7 @@ understand Dragonfly better would be included as well.
|
|||
|
||||
For almost all users, commandline is the first reference you may need. Document in directory [CLI Reference](cli-reference) is about command detailed usage of Dragonfly CLI including `dfget`, `cdnsystem`, `scheduler` and `manager`. You can get introductions, synopsis, examples, options about command. Last but not least, Dragonfly can guarantee commandline docs is strongly consistent with Dragonfly CLI's source code. What's more, all commandline docs are auto generated via source code.
|
||||
|
||||
## [TODO] API Reference
|
||||
## API Reference
|
||||
|
||||
Commandline is the easiest way to experience Dragonfly's ability. API extension will bring more further experience of Dragonfly. Commandline is just one kind of combination usage of API, if you wish to hack or take more advantages of Dragonfly, please see [API Reference](./api-reference). Like command line document, all API docs are auto generated via source code.
|
||||
|
||||
|
|
@ -55,11 +55,12 @@ better understanding of how Dragonfly is designed, developer could learn source
|
|||
|
||||
### [TODO] Design Doc
|
||||
|
||||
[Design Doc](./design) is content all about design of Dragonfly. It includes all things taken into consideration at the very beginning, the architecture designed for all components in Dragonfly, the interactive workflow between components, all APIs in Dragonfly and some technical things else.
|
||||
[Design Doc](./design/architecture.md) is content all about design of Dragonfly. It includes all things taken into consideration at the very beginning, the
|
||||
architecture designed for all components in Dragonfly, the interactive workflow between components, all APIs in Dragonfly and some technical things else.
|
||||
|
||||
### [TODO] Test Guide
|
||||
|
||||
[Test Guide](./test-guide) is the best reference helping contributors get aware of how to setup testing environment and do it. Currently we can divide test of Dragonfly into four dimensions:
|
||||
[Test Guide](./test-guide/test-guide.md) is the best reference helping contributors get aware of how to setup testing environment and do it. Currently we can divide test of Dragonfly into four dimensions:
|
||||
|
||||
* unit test;
|
||||
* API integration test;
|
||||
|
|
|
|||
|
|
@ -1,26 +1,31 @@
|
|||
## CDN
|
||||
# CDN
|
||||
|
||||
CDN is a long-running process which caches downloaded data from source to avoid downloading the same files from source repeatedly
|
||||
|
||||
## Try it
|
||||
```
|
||||
go run cmd/cdnsystem/main.go [Option]
|
||||
```
|
||||
## Log configuration
|
||||
set environment variable console=true if you want to print logs to Terminal
|
||||
|
||||
## Runtime metrics monitoring
|
||||
```
|
||||
go run cmd/cdnsystem/main.go --profiler
|
||||
```
|
||||
### Options
|
||||
## Usage
|
||||
|
||||
```
|
||||
--config string the path of configuration file with yaml extension name, default is /Users/${USER_HOME}/.dragonfly/config/cdn.yaml, it can also be
|
||||
set by env var:CDN_CONFIG，The settings and uses of each configuration item can refer to cdn.yaml in config directory
|
||||
cdn [flags]
|
||||
cdn [command]
|
||||
```
|
||||
|
||||
## Available Commands:
|
||||
|
||||
```
|
||||
doc generate documents
|
||||
help Help about any command
|
||||
version show version
|
||||
```
|
||||
|
||||
## Flags
|
||||
|
||||
```
|
||||
--config string the path of configuration file with yaml extension name, default is /Users/${USER_HOME}/.dragonfly/config/cdn.yaml, it can
|
||||
also be set by environment variable cdn_config
|
||||
--console whether logger output records to the stdout
|
||||
-h, --help help for cdn
|
||||
-h, --help help for scheduler
|
||||
--jaeger string jaeger endpoint url, like: http://localhost:14250/api/traces
|
||||
--pprof-port int listen port for pprof, 0 represents random port (default -1)
|
||||
--service-name string name of the service for tracer (default "dragonfly-cdn")
|
||||
--verbose whether logger use debug level
|
||||
```
|
||||
|
|
|
|||
|
|
@ -3,26 +3,55 @@
|
|||
Manager is a process that runs in the background and plays the role of the brain of each subsystem cluster in Dragonfly. It is used to manage the dynamic
|
||||
configuration of each system module and provide functions such as heartbeat keeping alive, monitoring the market, and product functions.
|
||||
|
||||
## Try it
|
||||
### Try it
|
||||
|
||||
```
|
||||
go run cmd/manager/main.go [Option]
|
||||
manager [Option]
|
||||
```
|
||||
|
||||
## Log configuration
|
||||
set environment variable console=local if you want to print logs to Terminal
|
||||
### Log configuration
|
||||
|
||||
## Runtime metrics monitoring
|
||||
```
|
||||
go run cmd/scheduler/main.go --profiler
|
||||
1. set option --console if you want to print logs to Terminal
|
||||
2. log path: /var/log/dragonfly/manager/
|
||||
```
|
||||
|
||||
### Runtime metrics monitoring
|
||||
|
||||
```
|
||||
manager --pprof-port port
|
||||
```
|
||||
|
||||
### Swagger support
|
||||
|
||||
endpoint: /swagger/doc.json
|
||||
|
||||
### Prometheus metrics monitoring
|
||||
|
||||
endpoint: /metrics
|
||||
|
||||
### HealthCheck
|
||||
|
||||
endpoint: /healthy/
|
||||
|
||||
|
||||
### Enable jaeger
|
||||
|
||||
```
|
||||
manager --jaeger http://localhost:14250/api/traces
|
||||
```
|
||||
|
||||
### Options
|
||||
|
||||
```
|
||||
--config string the path of configuration file with yaml extension name, default is /Users/${USER_HOME}/.dragonfly/config/scheduler.yaml, it can
|
||||
also be set by env var:SCHEDULER_CONFIG，The settings and uses of each configuration item can refer to scheduler.yaml in config directory
|
||||
--config string the path of configuration file with yaml extension name, default is /etc/dragonfly/manager.yaml, it can also be set by env var: MANAGER_CONFIG
|
||||
--console whether logger output records to the stdout
|
||||
-h, --help help for cdn
|
||||
-h, --help help for manager
|
||||
--jaeger string jaeger endpoint url, like: http://localhost:14250/api/traces
|
||||
--pprof-port int listen port for pprof, 0 represents random port (default -1)
|
||||
--service-name string name of the service for tracer (default "dragonfly-manager")
|
||||
--verbose whether logger use debug level
|
||||
```
|
||||
|
||||
### Manager Config
|
||||
[Config Example](../config/manager.yaml)
|
||||
|
|
@ -1,26 +1,32 @@
|
|||
## Scheduler
|
||||
# Scheduler
|
||||
|
||||
Scheduler is a long-running process which receives and manages download tasks from the client, notify the CDN to return to the source,
|
||||
generate and maintain a P2P network during the download process, and push suitable download nodes to the client
|
||||
## Try it
|
||||
```
|
||||
go run cmd/scheduler/main.go [Option]
|
||||
```
|
||||
## Log configuration
|
||||
set environment variable console=true if you want to print logs to Terminal
|
||||
Scheduler is a long-running process which receives and manages download tasks from the client, notify the CDN to return to the source, generate and maintain a
|
||||
P2P network during the download process, and push suitable download nodes to the client
|
||||
|
||||
## Usage
|
||||
|
||||
## Runtime metrics monitoring
|
||||
```
|
||||
go run cmd/scheduler/main.go --profiler
|
||||
scheduler [flags]
|
||||
scheduler [command]
|
||||
```
|
||||
### Options
|
||||
|
||||
## Available Commands:
|
||||
|
||||
```
|
||||
doc generate documents
|
||||
help Help about any command
|
||||
version show version
|
||||
```
|
||||
|
||||
## Flags
|
||||
|
||||
```
|
||||
--config string the path of configuration file with yaml extension name, default is /Users/${USER_HOME}/.dragonfly/config/scheduler.yaml, it can
|
||||
also be set by env var:SCHEDULER_CONFIG，The settings and uses of each configuration item can refer to scheduler.yaml in config directory
|
||||
also be set by environment variable scheduler_config
|
||||
--console whether logger output records to the stdout
|
||||
-h, --help help for cdn
|
||||
-h, --help help for scheduler
|
||||
--jaeger string jaeger endpoint url, like: http://localhost:14250/api/traces
|
||||
--pprof-port int listen port for pprof, 0 represents random port (default -1)
|
||||
--service-name string name of the service for tracer (default "dragonfly-scheduler")
|
||||
--verbose whether logger use debug level
|
||||
```
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
# This file is the template of cdn system configuration file.
|
||||
# You can configure your cdn system by change the parameter according your requirement.
|
||||
---
|
||||
base:
|
||||
# listenPort is the port cdn server listens on.
|
||||
# default: 8003
|
||||
|
|
@ -19,6 +18,10 @@ base:
|
|||
# default: 1G, in format of G(B)/g/M(B)/m/K(B)/k/B, pure number will also be parsed as Byte.
|
||||
maxBandwidth: 1G
|
||||
|
||||
# AdvertiseIP is used to set the ip that we advertise to other peer in the p2p-network.
|
||||
# By default, the first non-loop address is advertised.
|
||||
advertiseIP:
|
||||
|
||||
# FailAccessInterval is the interval time after failed to access the URL.
|
||||
# If a task failed to be downloaded from the source, it will not be retried in the time since the last failure.
|
||||
# default: 3m
|
||||
|
|
@ -42,51 +45,57 @@ base:
|
|||
|
||||
# manager configuration
|
||||
manager:
|
||||
addr: 127.0.0.1:65003
|
||||
cdnClusterID: 1
|
||||
addr: ""
|
||||
cdnClusterID: 0
|
||||
keepAlive:
|
||||
interval: 5s
|
||||
|
||||
# host configuration
|
||||
host:
|
||||
location:
|
||||
idc:
|
||||
|
||||
plugins:
|
||||
storageDriver:
|
||||
storagedriver:
|
||||
- name: disk
|
||||
enable: true
|
||||
config:
|
||||
baseDir: /tmp/cdnsystem2
|
||||
baseDir: /Users/${USER_HOME}/ftp
|
||||
- name: memory
|
||||
enable: true
|
||||
enable: false
|
||||
config:
|
||||
baseDir: /tmp/memory/dragonfly
|
||||
|
||||
storageManager:
|
||||
baseDir: /dev/shm/dragonfly
|
||||
storagemanager:
|
||||
- name: disk
|
||||
enable: true
|
||||
config:
|
||||
gcInitialDelay: 5s
|
||||
gcInitialDelay: 0s
|
||||
gcInterval: 15s
|
||||
driverConfigs:
|
||||
disk:
|
||||
gcConfig:
|
||||
youngGCThreshold: 100G
|
||||
fullGCThreshold: 5G
|
||||
youngGCThreshold: 100.0GB
|
||||
fullGCThreshold: 5.0GB
|
||||
cleanRatio: 1
|
||||
intervalThreshold: 2h
|
||||
intervalThreshold: 2h0m0s
|
||||
- name: hybrid
|
||||
enable: false
|
||||
config:
|
||||
gcInitialDelay: 5s
|
||||
gcInitialDelay: 0s
|
||||
gcInterval: 15s
|
||||
driverConfigs:
|
||||
disk:
|
||||
gcConfig:
|
||||
youngGCThreshold: 100G
|
||||
fullGCThreshold: 5G
|
||||
youngGCThreshold: 100.0GB
|
||||
fullGCThreshold: 5.0GB
|
||||
cleanRatio: 1
|
||||
intervalThreshold: 2h
|
||||
intervalThreshold: 2h0m0s
|
||||
memory:
|
||||
gcConfig:
|
||||
youngGCThreshold: 100G
|
||||
fullGCThreshold: 5G
|
||||
youngGCThreshold: 100.0GB
|
||||
fullGCThreshold: 5.0GB
|
||||
cleanRatio: 3
|
||||
intervalThreshold: 2h
|
||||
intervalThreshold: 2h0m0s
|
||||
|
||||
# Console shows log on console
|
||||
# default: false
|
||||
|
|
@ -97,6 +106,12 @@ console: false
|
|||
verbose: false
|
||||
|
||||
# listen port for pprof, only valid when the verbose option is true
|
||||
# default is random port
|
||||
pprofPort: 0
|
||||
# default is -1. If it is 0, pprof will use a random port.
|
||||
pprofPort: -1
|
||||
|
||||
# jaeger endpoint url
|
||||
jaeger:
|
||||
|
||||
# service name used in tracer
|
||||
# default: dragonfly-cdn
|
||||
service-name: dragonfly-cdn
|
||||
|
|
|
|||
|
|
@ -37,11 +37,11 @@ database:
|
|||
cache:
|
||||
# redis cache configure
|
||||
redis:
|
||||
# cache ttl configure
|
||||
# cache ttl configure unit[nanosecond]
|
||||
ttl: 30000000000
|
||||
# local cache configure
|
||||
local:
|
||||
# lfu cache size
|
||||
size: 10000
|
||||
# cache ttl configure
|
||||
# cache ttl configure unit[nanosecond]
|
||||
ttl: 30000000000
|
||||
|
|
|
|||
|
|
@ -1,29 +1,138 @@
|
|||
# This file is the template of scheduler configuration file.
|
||||
# You can configure your scheduler by change the parameter according your requirement.
|
||||
---
|
||||
debug: false
|
||||
|
||||
# scheduler policy configuration
|
||||
scheduler:
|
||||
# abtest whether to open ab test
|
||||
# default: false
|
||||
abtest: false
|
||||
# aevaluator specify which evaluator is used when taskID suffix with _A
|
||||
# only take effect when abtest is true
|
||||
# default: ""
|
||||
aevaluator: ""
|
||||
# bevaluator specify which evaluator is used when taskID suffix with _B
|
||||
# only take effect when abtest is true
|
||||
# default: ""
|
||||
bevaluator: ""
|
||||
# workerNum is the number of goroutines that perform scheduling tasks
|
||||
# default:
|
||||
workerNum: 12
|
||||
# backSourceCount is the number of back-to-origin clients when the CDN is disable or unavailable
|
||||
# default: 3
|
||||
backSourceCount: 3
|
||||
# accessWindow is access interval window that can trigger CDN back to the source again
|
||||
# default: 3m
|
||||
accessWindow: 3m
|
||||
# candidateParentCount is number of candidate parent nodes
|
||||
# default: 10
|
||||
candidateParentCount: 10
|
||||
# scheduler is currently effective scheduling policy
|
||||
# default: basic
|
||||
scheduler: basic
|
||||
# cdnLoad is CDN host can provide the maximum upload load
|
||||
# default: 100
|
||||
cdnLoad: 100
|
||||
# clientLoad is client host can provide the maximum upload load
|
||||
# default: 10
|
||||
clientLoad: 10
|
||||
# openMonitor Whether to enable monitoring, currently only the current peer list status information is monitored
|
||||
# default: false
|
||||
openMonitor: false
|
||||
# gc metadata configuration
|
||||
gc:
|
||||
# peerGCInterval peer's gc interval
|
||||
peerGCInterval: 1m
|
||||
# peerTTL peer's TTL duration
|
||||
peerTTL: 10m
|
||||
# peerTTI peer's TTI duration
|
||||
peerTTI: 3m
|
||||
# taskGCInterval task's gc interval
|
||||
taskGCInterval: 1m
|
||||
# taskTTL task's TTL duration
|
||||
taskTTL: 10m
|
||||
# taskTTI task's TTI duration
|
||||
taskTTI: 3m
|
||||
|
||||
# server scheduler instance configuration
|
||||
server:
|
||||
# ip
|
||||
ip:
|
||||
# host
|
||||
host:
|
||||
# ListenPort is the ip and port scheduler server listens on.
|
||||
# default: 8002
|
||||
port: 8002
|
||||
port:
|
||||
|
||||
scheduler:
|
||||
|
||||
worker:
|
||||
worker-num: 1
|
||||
worker-job-pool-size: 10000
|
||||
sender-num: 10
|
||||
sender-job-pool-size: 10000
|
||||
|
||||
cdn:
|
||||
servers:
|
||||
- name: "B-M75UMD6M-2153.local"
|
||||
ip: "127.0.0.1"
|
||||
rpc-port: 8003
|
||||
download-port: 8001
|
||||
# dynamic data configuration
|
||||
dynConfig:
|
||||
# type
|
||||
type:
|
||||
# expireTime
|
||||
expireTime:
|
||||
# cdnDirPath
|
||||
cdnDirPath:
|
||||
# data
|
||||
data: ""
|
||||
|
||||
# manager configuration
|
||||
manager:
|
||||
# addr manager access address
|
||||
addr: 127.0.0.1:65003
|
||||
# schedulerClusterID cluster id to which scheduler instance belongs
|
||||
schedulerClusterID: 1
|
||||
# keepAlive keep alive configuration
|
||||
keepAlive:
|
||||
# interval
|
||||
interval: 5s
|
||||
|
||||
# host scheduler host configuration
|
||||
host:
|
||||
# location is the location of scheduler instance
|
||||
location: ""
|
||||
# idc is the idc of scheduler instance
|
||||
idc: ""
|
||||
|
||||
# machinery async job configuration, see https://github.com/RichardKnop/machinery
|
||||
job:
|
||||
# globalWorkerNum
|
||||
globalWorkerNum: 2
|
||||
# schedulerWorkerNum
|
||||
schedulerWorkerNum: 3
|
||||
# localWorkerNum
|
||||
localWorkerNum: 3
|
||||
# redis configuration
|
||||
redis:
|
||||
# host
|
||||
host: ""
|
||||
# port
|
||||
port:
|
||||
# password
|
||||
password: ""
|
||||
# brokerDB
|
||||
brokerDB: ""
|
||||
# backendDB
|
||||
backendDB: ""
|
||||
|
||||
# Whether to disable CDN
|
||||
# default: false
|
||||
disableCDN: false
|
||||
|
||||
# Console shows log on console
|
||||
# default: false
|
||||
console: false
|
||||
|
||||
# Whether to enable debug level logger and enable pprof
|
||||
# default: false
|
||||
verbose: false
|
||||
|
||||
# listen port for pprof, only valid when the verbose option is true
|
||||
# default is -1. If it is 0, pprof will use a random port.
|
||||
pprofPort: -1
|
||||
|
||||
# jaeger endpoint url
|
||||
# default: ""
|
||||
jaeger: ""
|
||||
|
||||
# service name used in tracer
|
||||
# default: dragonfly-scheduler
|
||||
service-name: dragonfly-scheduler
|
||||
|
|
@ -0,0 +1,105 @@
|
|||
# Product Thinking and Design
|
||||
|
||||
## Positioning
|
||||
Provide enterprise-level (efficient, stable, secure, low-cost, product-oriented) file distribution and management services, and become the cloud-native best practice and standard solution in this realm.
|
||||
|
||||
## What problems are solved
|
||||
|
||||
**Architecture design flaws:** The existing architecture is difficult to meet the growing business needs of file distribution, and has gradually exposed deficiencies in stability, efficiency, and security, and is facing more and more challenges
|
||||
|
||||
**Insufficient value penetration:** Currently only supports HTTP back-to-source protocol, and lacks adaptation to other types of storage (HDFS, storage services of various cloud vendors, Maven, YUM, etc.)
|
||||
|
||||
**Single distribution mode:** Currently only supports active pull mode, lack of active push and active synchronization capabilities
|
||||
|
||||
**Lack of product capabilities:** No well-experienced console functions are capabilities, such as distribution task management and control, data market, multi-tenancy, and permission control, etc.
|
||||
|
||||
## Core Competence
|
||||
|
||||
- Through a unified back-to-source adaptation layer and realization of P2P file distribution capabilities that support various types of storage (HDFS,
|
||||
storage services of various cloud vendors, Maven, YUM, etc.)
|
||||
- Support more distribution modes: active pull, active push, real-time synchronization, remote copy, automatic warm-up, cross-cloud transmission, etc.
|
||||
- Separation and decoupling between systems, scheduling and CDN plug-in, and support on-demand deployment, light or heavy, internal or external, flexible to
|
||||
meet the actual needs of different scenarios
|
||||
- Based on the newly designed P2P protocol framework of grpc, with better efficiency and stability
|
||||
- Support encrypted transmission, account-based transmission authentication and current limiting, and multi-tenant isolation mechanism
|
||||
- Support more efficient IO methods: multi-threaded IO, memory mapping, DMA, etc.
|
||||
- Support dynamic compression, memory file system and more efficient scheduling algorithms to improve distribution efficiency
|
||||
- The client supports third-party software's native integration of Dragonfly's P2P capabilities through the C/S mode
|
||||
- Production capabilities: support file upload, task management of various distribution modes, data views, and global management and control functions
|
||||
- One set of internal and external, core features are shared with each other, and non-common features are extended separately
|
||||
- Further integration with the ecosystem: Harbor, Nydus (image downloading solution on demand), warehouse services of various cloud vendors, etc.
|
||||
|
||||
## Architecture Design
|
||||
|
||||
### Overall structure
|
||||
![alt][arch]
|
||||
|
||||
### Entity relationship
|
||||
|
||||
![alt][association]
|
||||
|
||||
### Image file download process
|
||||
|
||||
![alt][download-process]
|
||||
|
||||
### Sub-system architecture
|
||||
|
||||
#### CDN system
|
||||
|
||||
![cdn][cdn]
|
||||
|
||||
### Sub-system function
|
||||
|
||||
**Manager**
|
||||
- Dynamic configuration management
|
||||
- Data Big Disk & Bandwidth Measurement
|
||||
- File upload & distribution management and control
|
||||
- File synchronization management
|
||||
- Account & access control
|
||||
- Subscription & notification
|
||||
- Multi-tenant management
|
||||
- Command channel service (channel integration)
|
||||
|
||||
**Scheduler**
|
||||
- Multi-dimensional P2P scheduling strategy
|
||||
- Intelligent scheduling algorithm (advanced)
|
||||
- The scheduling results can be quantitatively compared and support A/B Testing
|
||||
- Scheduling strategy plug-in and CDN subsystem plug-in
|
||||
- Meta-information distribution pipeline
|
||||
- Client download results and back-to-source results statistics and docking with monitoring services
|
||||
|
||||
**CDN**
|
||||
- Multi-source adaptation capabilities, including catalog download and range download functions
|
||||
- DMA read and write
|
||||
- Transfer back to the source
|
||||
- Distribute warm-up
|
||||
- Dynamic compression
|
||||
- Storage management (seed storage, disk cleaning, block-level storage)
|
||||
- Memory file system
|
||||
- Secure transmission (symmetric encryption, integrity check, etc.)
|
||||
- Storage media plug-in
|
||||
|
||||
**Dfget daemon**
|
||||
- Multi-source adaptation capabilities, including catalog download and range download functions
|
||||
- Efficient IO mechanism
|
||||
- IO scheduling of multiple download tasks (high IO throughput)
|
||||
- Reduce the number of file reads and writes: improve the storage of temporary files and improve file integrity verification
|
||||
- DMA read and write
|
||||
- Streaming
|
||||
- Client download Proxy capability (http & https)
|
||||
- Single service process startup (CS mode) and resident and non-resident modes
|
||||
- Failed back to source processing
|
||||
- Local cache capability (client seeder mode)
|
||||
- Client elegant upgrade and deployment plan
|
||||
- Command execution
|
||||
|
||||
**Framework**
|
||||
- High availability, high performance, easy integration
|
||||
- Consistent Hash algorithm selection
|
||||
- Client connection management
|
||||
|
||||
|
||||
[arch]: ../images/arch.png
|
||||
[association]: ../../zh-CN/images/association.png
|
||||
[download-process]: ../../zh-CN/images/download-process.png
|
||||
[cdn]: ../../zh-CN/images/cdn.png
|
||||
|
|
@ -1,5 +1,8 @@
|
|||
[install-client](./install-client.md) is the installation instructions of client
|
||||
|
||||
[install-manager](./install-manager.md) is the installation instructions of manager
|
||||
|
||||
[install-scheduler](./install-scheduler.md) is the installation instructions of scheduler
|
||||
|
||||
[install-cdn](./install-cdn.md) is the installation instructions of CDN
|
||||
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ You can get it from [DockerHub](https://hub.docker.com/) directly.
|
|||
1. Obtain the latest Docker image of the cdn.
|
||||
|
||||
```sh
|
||||
docker pull d7yio/cdn
|
||||
docker pull dragonflyoss/cdn
|
||||
```
|
||||
|
||||
Or you can build your own cdn image.
|
||||
|
|
@ -118,16 +118,16 @@ Or you can build your own cdn executable file.
|
|||
### Start cdn
|
||||
|
||||
```sh
|
||||
cdnHomeDir=/home/admin/cdn
|
||||
cdnHomeDir=/home/admin
|
||||
cdnDownloadPort=8001
|
||||
cdn --home-dir=$cdnHomeDir --port=8003 --download-port=$cdnDownloadPort
|
||||
cdn --port=8003 --download-port=$cdnDownloadPort
|
||||
```
|
||||
|
||||
### Start file server
|
||||
|
||||
You can start a file server in any way. However, the following conditions must be met:
|
||||
|
||||
- It must be rooted at `${cdnHomeDir}/ftp` which is defined in the previous step.
|
||||
- It must be rooted at `cdnHomeDir/ftp` which is defined in the previous step.
|
||||
- It must listen on the port `cdnDownloadPort` which is defined in the previous step.
|
||||
|
||||
Let's take nginx as an example.
|
||||
|
|
@ -139,8 +139,8 @@ Let's take nginx as an example.
|
|||
# Must be ${cdnDownloadPort}
|
||||
listen 8001;
|
||||
location / {
|
||||
# Must be ${cdnHomeDir}/repo
|
||||
root /home/admin/cdn/repo;
|
||||
# Must be ${cdnHomeDir}/ftp
|
||||
root /home/admin/ftp;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ You can get it from [DockerHub](https://hub.docker.com/) directly.
|
|||
1. Obtain the latest Docker image of the scheduler.
|
||||
|
||||
```sh
|
||||
docker pull d7yio/scheduler
|
||||
docker pull dragonflyoss/scheduler
|
||||
```
|
||||
|
||||
Or you can build your own scheduler image.
|
||||
|
|
|
|||
|
|
@ -87,7 +87,7 @@ Environment="HTTP_PROXY=http://127.0.0.1:65001"
|
|||
Environment="HTTPS_PROXY=http://127.0.0.1:65001"
|
||||
```
|
||||
|
||||
### Step 5: Pull images with proxy
|
||||
### Step 4: Pull images with proxy
|
||||
|
||||
Through the above steps, we can start to validate if Dragonfly works as expected.
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
# [WIP] Dragonfly Quick Start
|
||||
# Dragonfly Quick Start
|
||||
|
||||
Dragonfly Quick Start document aims to help you to quick start Dragonfly journey. This experiment is quite easy and
|
||||
simplified.
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
# FAQ
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
# 蜻蜓文档
|
||||
|
||||
Dragonfly Document 是关于 Dragonfly 的介绍。对于一般对 Dragonfly 感兴趣的人，这个 repo 的 README.md 就足够了。对于最终用户来说，`/docs` 中包含的所有详细信息是最好的指南。而对于开发者来说，[开发者指南](#开发者指南)部分的内容才是最需要的。
|
||||
|
||||
文档文件组织如下：
|
||||
|
||||
* [快速开始](#快速开始)
|
||||
* [用户指南](#用户指南)
|
||||
* [CLI参考](#CLI参考)
|
||||
* [dfget](cli-reference/dfget.md)
|
||||
* [cdnsystem](cli-reference/cdn.md)
|
||||
* [scheduler](cli-reference/scheduler.md)
|
||||
* [manager](cli-reference/manager.md)
|
||||
* [API 参考](#API参考)
|
||||
* [生态](#生态)
|
||||
* [Kubernetes 集成](ecosystem/Kubernetes-with-Dragonfly.md)
|
||||
* [Harbor 集成](ecosystem/Harbor-with-Dragonfly.md)
|
||||
* [开发者指南](#开发者指南)
|
||||
* [设计文档](#设计文档)
|
||||
* [测试指南](#测试指南)
|
||||
|
||||
## 如何贡献文档
|
||||
|
||||
在此页面中找到 `WIP` 或 `TODO` 并参考 [CONTRIBUTING](../../CONTRIBUTING.md)。
|
||||
|
||||
## 快速开始
|
||||
|
||||
[快速开始](user-guide/quick-start.md) 如果您想尝试 Dragonfly，这正是您所需要的。本文件包括您需要做什么准备工作，
|
||||
如何安装 Dragonfly 以及如何体验 Dragonfly 的使用。
|
||||
|
||||
## 用户指南
|
||||
|
||||
[用户指南](user-guide/README.md) 帮助用户体验 Dragonfly 所需的各种指南。不仅仅是非常简短的 [快速开始]
|
||||
(user-guide/quick-start.md)，而且包括详细的安装和配置说明。此外，任何有助于用户更好地理解 Dragonfly 的概念也将包括在内。
|
||||
|
||||
## CLI参考
|
||||
|
||||
对于几乎所有用户，命令行是您可能需要的第一个参考。 [CLI Reference](cli-reference)目录下的文档是关于Dragonfly CLI的命令详细使用，包括`dfget`、`cdnsystem`、`scheduler`和`manager`。您可以获得有关命令的介绍、概要、示例和选项。最后但同样重要的是，Dragonfly 可以保证命令行文档与 Dragonfly CLI 的源代码高度一致。更重要的是，所有命令行文档都是通过源代码自动生成的。
|
||||
|
||||
## API参考
|
||||
|
||||
命令行是体验 Dragonfly 能力的最简单方法。 API参考将为 Dragonfly 带来更多更进一步的体验。 Commandline只是API的一种组合用法，如果你想破解或利用Dragonfly的更多优势，请参见[API参考](./api-reference)。与命令行文档一样，所有 API 文档都是通过源代码自动生成的。
|
||||
|
||||
## 生态
|
||||
|
||||
生态文档显示了 Dragonfly 与云原生生态系统中流行的工具或系统之间的联系。他们指导最终用户如何体验云
|
||||
使用 Dragonfly 的原生系统，例如其他 CNCF 项目 [Kubernetes](ecosystem/Kubernetes-with-Dragonfly.md) 和 [Harbor](ecosystem/Harbor-with-Dragonfly.md)。
|
||||
|
||||
## 开发者指南
|
||||
|
||||
[开发者指南](development/local.md) 帮助（潜在的）开发者/贡献者理解 Dragonfly 内部的理论而不是它公开的接口。和
|
||||
更好地理解 Dragonfly 的设计方式，开发人员可以更轻松地学习 Dragonfly 的源代码，并知道如何调试、测试和破解。
|
||||
|
||||
### [TODO] 设计文档
|
||||
|
||||
[Design Doc](./design/architecture.md) 是关于 Dragonfly 设计的内容。它包括一开始就考虑到的所有事情，为 Dragonfly 中所有组件设计的架构，组件之间的交互工作流，Dragonfly 中的所有 API 以及其他一些技术内容。
|
||||
|
||||
### [TODO] 测试指南
|
||||
|
||||
[测试指南](./test-guide/test-guide.md) 是帮助贡献者了解如何设置测试环境并进行操作的最佳参考。目前我们可以将 Dragonfly 的测试分为四个维度：
|
||||
|
||||
* 单元测试;
|
||||
* API集成测试；
|
||||
* CLI集成测试；
|
||||
* 节点 e2e 测试。
|
||||
|
||||
更多详情请参考[测试](./test-guide)。
|
||||
|
||||
## 写在后面
|
||||
|
||||
文件夹`/docs` 没有完全包含有关蜻蜓的所有文档，在此 repo 的其他路径中还有其他非常有用的文档，例如：
|
||||
|
||||
* [TODO FAQ.md](./FAQ.md)
|
||||
* [CHANGELOG.md](../../CHANGELOG.md)
|
||||
* [TODO ROADMAP.md](./ROADMAP.md)
|
||||
* 其他。
|
||||
|
||||
如果您正在搜索某个文档但没有找到，请不要犹豫 [提交问题](https://github.com/dragonflyoss/Dragonfly2/issues/new/choose) 寻求帮助。另外，如果您发现文档中有一些不正确的地方或错别字，请发起 Pull Request 以更正。
|
||||
|
|
@ -0,0 +1 @@
|
|||
# Roadmap
|
||||
|
|
@ -1,26 +1,31 @@
|
|||
## CDN
|
||||
# CDN
|
||||
|
||||
CDN 是一个长时间运行的服务进程，它缓存从源下载的数据，以避免重复从源下载相同的文件
|
||||
|
||||
## 用法
|
||||
```
|
||||
go run cmd/cdnsystem/main.go [Option]
|
||||
```
|
||||
## 输出日志配置
|
||||
如果你期望把系统执行日志输出到命令终端，设置环境变量 console=true
|
||||
|
||||
## Runtime metrics monitoring
|
||||
```
|
||||
go run cmd/cdnsystem/main.go --profiler
|
||||
```
|
||||
### 可选参数
|
||||
|
||||
```
|
||||
--config string the path of configuration file with yaml extension name, default is /Users/${USER_HOME}/.dragonfly/config/cdn.yaml, it can also be
|
||||
set by env var:CDN_CONFIG，The settings and uses of each configuration item can refer to cdn.yaml in config directory
|
||||
--console whether logger output records to the stdout
|
||||
-h, --help help for cdn
|
||||
--jaeger string jaeger endpoint url, like: http://localhost:14250/api/traces
|
||||
--pprof-port int listen port for pprof, 0 represents random port (default -1)
|
||||
--verbose whether logger use debug level
|
||||
cdn [flags]
|
||||
cdn [command]
|
||||
```
|
||||
|
||||
## 可用子命令
|
||||
|
||||
```
|
||||
doc 生成帮助文档
|
||||
help 命令的帮助文档
|
||||
version 查看 cdn 当前版本
|
||||
```
|
||||
|
||||
## 可选参数
|
||||
|
||||
```
|
||||
--config string 配置文件地址 (默认值为/Users/${USER_HOME}/.dragonfly/config/cdn.yaml)，也可以使用环境变量 cdn_config 进行配置
|
||||
--console logger 日志是否输出记录到标准输出
|
||||
-h, --help 显示 cdn 命令帮助信息
|
||||
--jaeger string jaeger 收集器 endpoint url, 例如: http://localhost:14250/api/traces
|
||||
--pprof-port int pprof 监听的端口，为 0 时使用随机端口（默认值为 -1）
|
||||
--service-name string tracer分布式追踪日志中的服务名称 (默认 "dragonfly-scheduler")
|
||||
--verbose 是否使用debug级别的日志输出
|
||||
|
||||
```
|
||||
|
|
@ -0,0 +1,97 @@
|
|||
# dfget
|
||||
|
||||
`dfget` 是Dragonfly用以下载和上传文件的客户端。
|
||||
|
||||
### Synopsis
|
||||
|
||||
dfget 是 Dragonfly 的一个客户端，也是p2p网络中的一个peer。当用户发起文件下载请求时，
|
||||
dfget 将从其他peer下载文件。同时，它也能作为上传者，以支持其他peer对它已拥有对部分对下载。
|
||||
此外，dfget 还提供了一些高级功能，如网络带宽限制、加密传输等。
|
||||
|
||||
```
|
||||
dfget [flags]
|
||||
```
|
||||
|
||||
### Example
|
||||
|
||||
```
|
||||
dfget --schedulers 127.0.0.1:8002 -o /path/to/output -u "http://example.com/object"
|
||||
```
|
||||
|
||||
## Log configuration
|
||||
|
||||
如果您想要在Terminal中查看日志，请使用 --console 将环境变量 console 设置为 true。
|
||||
|
||||
### Options
|
||||
|
||||
```
|
||||
--alivetime duration alive duration for which uploader keeps no accessing by any uploading requests, after this period uploader will automatically exit (default 5m0s)
|
||||
--cacerts strings the cacert file which is used to verify remote server when supernode interact with the source.
|
||||
--callsystem string the name of dfget caller which is for debugging. Once set, it will be passed to all components around the request to make debugging easy
|
||||
--clientqueue int specify the size of client queue which controls the number of pieces that can be processed simultaneously
|
||||
--console show log on console, it's conflict with '--showbar'
|
||||
--daemon-pid string the daemon pid (default "/tmp/dfdaemon.pid")
|
||||
--daemon-sock string the unix domain socket address for grpc with daemon (default "/tmp/dfdamon.sock")
|
||||
--dfdaemon identify whether the request is from dfdaemon
|
||||
--expiretime duration caching duration for which cached file keeps no accessed by any process, after this period cache file will be deleted (default 3m0s)
|
||||
-f, --filter string filter some query params of URL, use char '&' to separate different params
|
||||
eg: -f 'key&sign' will filter 'key' and 'sign' query param
|
||||
in this way, different but actually the same URLs can reuse the same downloading task
|
||||
--header stringArray http header, eg: --header='Accept: *' --header='Host: abc'
|
||||
-h, --help help for dfget
|
||||
--home string the work home directory of dfget (default "/Users/jim/.dragonfly/dfdaemon/")
|
||||
-i, --identifier string the usage of identifier is making different downloading tasks generate different downloading task IDs even if they have the same URLs. conflict with --md5.
|
||||
--insecure identify whether supernode should skip secure verify when interact with the source.
|
||||
--ip string IP address that server will listen on (default "0.0.0.0")
|
||||
-m, --md5 string md5 value input from user for the requested downloading file to enhance security
|
||||
--more-daemon-options string more options passed to daemon by command line, please confirm your options with "dfget daemon --help"
|
||||
-n, --node supernodes deprecated, please use schedulers instead. specify the addresses(host:port=weight) of supernodes where the host is necessary, the port(default: 8002) and the weight(default:1) are optional. And the type of weight must be integer
|
||||
--notbacksource disable back source downloading for requested file when p2p fails to download it
|
||||
-o, --output string destination path which is used to store the requested downloading file. It must contain detailed directory and specific filename, for example, '/tmp/file.mp4'
|
||||
-p, --pattern string download pattern, must be p2p/cdn/source, cdn and source do not support flag --totallimit (default "p2p")
|
||||
--port int port number that server will listen on (default 65002)
|
||||
--schedulers schedulers the scheduler addresses
|
||||
-b, --showbar show progress bar, it is conflict with '--console'
|
||||
-e, --timeout duration timeout set for file downloading task. If dfget has not finished downloading all pieces of file before --timeout, the dfget will throw an error and exit
|
||||
--totallimit ratelimit network bandwidth rate limit for the whole host, in format of G(B)/g/M(B)/m/K(B)/k/B, pure number will also be parsed as Byte (default 104857600.000000)
|
||||
-u, --url string URL of user requested downloading file(only HTTP/HTTPs supported)
|
||||
--verbose enable verbose mode, all debug log will be display
|
||||
|
||||
```
|
||||
|
||||
# dfget daemon
|
||||
|
||||
## Log configuration
|
||||
|
||||
如果您想要在Terminal中查看日志，请使用 --console 将环境变量 console 设置为 true。
|
||||
|
||||
### Options
|
||||
|
||||
```
|
||||
--advertise-ip string the ip report to scheduler, normal same with listen ip (default "10.15.232.63")
|
||||
--alivetime duration alive duration for which uploader keeps no accessing by any uploading requests, after this period uploader will automatically exit (default 5m0s)
|
||||
--data string local directory which stores temporary files for p2p uploading
|
||||
--download-rate ratelimit download rate limit for other peers and back source (default 104857600.000000)
|
||||
--expiretime duration caching duration for which cached file keeps no accessed by any process, after this period cache file will be deleted (default 3m0s)
|
||||
--gc-interval duration gc interval (default 1m0s)
|
||||
--grpc-port int the listen address for grpc with other peers (default 65000)
|
||||
--grpc-port-end int the listen address for grpc with other peers (default 65000)
|
||||
--grpc-unix-listen string the local unix domain socket listen address for grpc with dfget (default "/tmp/dfdamon.sock")
|
||||
-h, --help help for daemon
|
||||
--home string the work home directory of dfget daemon
|
||||
--idc string peer idc for scheduler
|
||||
--keep-storage keep storage after daemon exit
|
||||
--ip string IP address that server will listen on
|
||||
--location string peer location for scheduler
|
||||
--lock string dfdaemon lock file location (default "/tmp/dfdaemon.lock")
|
||||
--net-topology string peer net topology for scheduler
|
||||
--pid string dfdaemon pid file location (default "/tmp/dfdaemon.pid")
|
||||
--proxy-port int the address that daemon will listen on for proxy service (default 65001)
|
||||
--proxy-port-end int the address that daemon will listen on for proxy service (default 65001)
|
||||
-s, --schedulers schedulers schedulers
|
||||
--security-domain string peer security domain for scheduler
|
||||
--upload-port int the address that daemon will listen on for peer upload (default 65002)
|
||||
--upload-port-end int the address that daemon will listen on for peer upload (default 65002)
|
||||
--upload-rate ratelimit upload rate limit for other peers (default 104857600.000000)
|
||||
--verbose print verbose log and enable golang debug info
|
||||
```
|
||||
|
|
@ -1,20 +1,55 @@
|
|||
## Scheduler
|
||||
## Manager
|
||||
|
||||
Manager 是一个常驻后台运行的进程，它在蜻蜓中扮演各个子系统集群大脑的角色， 用于管理各个系统模块依赖的动态配置，以及提供心跳保活、监控大盘和产品化的功能。
|
||||
|
||||
### 用法
|
||||
|
||||
```
|
||||
go run cmd/manager/main.go [Option]
|
||||
manager [Option]
|
||||
```
|
||||
|
||||
### 日志配置
|
||||
|
||||
```
|
||||
1. 终端日志输出： --console
|
||||
2. 日志目录: /var/log/dragonfly/manager/
|
||||
```
|
||||
|
||||
### 指标输出
|
||||
|
||||
```
|
||||
启动时配置参数: --pprof-port port
|
||||
```
|
||||
|
||||
### Swagger支持
|
||||
|
||||
端点: /swagger/doc.json
|
||||
|
||||
### Prometheus集成
|
||||
|
||||
端点: /metrics
|
||||
|
||||
### 健康检查
|
||||
|
||||
端点: /healthy/
|
||||
|
||||
### jaeger集成
|
||||
|
||||
```
|
||||
启动时配置 --jaeger http://localhost:14250/api/traces
|
||||
```
|
||||
|
||||
### 可选参数
|
||||
|
||||
```
|
||||
--config string the path of scheduler's configuration file (default "conf/scheduler.yaml")
|
||||
-h, --help help for scheduler
|
||||
--port int port is the port that scheduler server listens on (default 8002)
|
||||
--sender-job-pool-size int sender-job-pool-size is used for scheduler and do not change it (default 10000)
|
||||
--sender-num int sender-num is used for scheduler and do not change it (default 50)
|
||||
--worker-job-pool-size int worker-job-pool-size is used for scheduler and do not change it (default 10000)
|
||||
--worker-num int worker-num is used for scheduler and do not change it (default 12)
|
||||
--config string the path of configuration file with yaml extension name, default is /etc/dragonfly/manager.yaml, it can also be set by env var: MANAGER_CONFIG
|
||||
--console whether logger output records to the stdout
|
||||
-h, --help help for manager
|
||||
--jaeger string jaeger endpoint url, like: http://localhost:14250/api/traces
|
||||
--pprof-port int listen port for pprof, 0 represents random port (default -1)
|
||||
--service-name string name of the service for tracer (default "dragonfly-manager")
|
||||
--verbose whether logger use debug level
|
||||
```
|
||||
|
||||
### 配置文件
|
||||
[配置样例](../config/manager.yaml)
|
||||
|
|
@ -1,20 +1,30 @@
|
|||
## Scheduler
|
||||
# Scheduler
|
||||
|
||||
Scheduler 是一个常驻后台运行的进程，用于接收和管理客户端的下载任务，通知CDN进行回源， 在下载过程中生成维护P2P网络，给客户端推送适合的下载节点
|
||||
|
||||
### 用法
|
||||
```
|
||||
go run cmd/scheduler/main.go [Option]
|
||||
```
|
||||
|
||||
### 可选参数
|
||||
## 用法
|
||||
|
||||
```
|
||||
--config string the path of scheduler's configuration file (default "conf/scheduler.yaml")
|
||||
-h, --help help for scheduler
|
||||
--port int port is the port that scheduler server listens on (default 8002)
|
||||
--sender-job-pool-size int sender-job-pool-size is used for scheduler and do not change it (default 10000)
|
||||
--sender-num int sender-num is used for scheduler and do not change it (default 50)
|
||||
--worker-job-pool-size int worker-job-pool-size is used for scheduler and do not change it (default 10000)
|
||||
--worker-num int worker-num is used for scheduler and do not change it (default 12)
|
||||
scheduler [flags]
|
||||
scheduler [command]
|
||||
```
|
||||
|
||||
## 可用子命令
|
||||
|
||||
```
|
||||
doc 生成帮助文档
|
||||
help 命令的帮助文档
|
||||
version 查看 scheduler 当前版本
|
||||
```
|
||||
|
||||
## 可选参数
|
||||
|
||||
```
|
||||
--config string 配置文件地址 (默认值为/Users/${USER_HOME}/.dragonfly/config/scheduler.yaml)，也可以使用环境变量 scheduler_config 进行配置
|
||||
--console logger 日志是否输出记录到标准输出
|
||||
-h, --help 显示 scheduler 命令帮助信息
|
||||
--jaeger string jaeger 收集器 endpoint url, 例如: http://localhost:14250/api/traces
|
||||
--pprof-port int pprof 监听的端口，为 0 时使用随机端口（默认值为 -1）
|
||||
--service-name string tracer分布式追踪日志中的服务名称 (默认 "dragonfly-scheduler")
|
||||
--verbose 是否使用debug级别的日志输出
|
||||
```
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
worker_rlimit_nofile 100000;
|
||||
|
||||
events {
|
||||
use epoll;
|
||||
worker_connections 20480;
|
||||
}
|
||||
|
||||
http {
|
||||
include mime.types;
|
||||
default_type application/octet-stream;
|
||||
root /home/admin/cai/htdocs;
|
||||
sendfile on;
|
||||
tcp_nopush on;
|
||||
|
||||
server_tokens off;
|
||||
keepalive_timeout 5;
|
||||
|
||||
client_header_timeout 1m;
|
||||
send_timeout 1m;
|
||||
client_max_body_size 3m;
|
||||
|
||||
index index.html index.htm;
|
||||
access_log off;
|
||||
log_not_found off;
|
||||
|
||||
gzip on;
|
||||
gzip_http_version 1.0;
|
||||
gzip_comp_level 6;
|
||||
gzip_min_length 1024;
|
||||
gzip_proxied any;
|
||||
gzip_vary on;
|
||||
gzip_disable msie6;
|
||||
gzip_buffers 96 8k;
|
||||
gzip_types text/xml text/plain text/css application/javascript application/x-javascript application/rss+xml application/json;
|
||||
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header Web-Server-Type nginx;
|
||||
proxy_set_header WL-Proxy-Client-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_redirect off;
|
||||
proxy_buffers 128 8k;
|
||||
proxy_intercept_errors on;
|
||||
|
||||
server {
|
||||
listen 8001;
|
||||
location / {
|
||||
root /home/admin/ftp;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,81 +1,125 @@
|
|||
# 这个文件是 CDN 系统的配置模版
|
||||
# 你可以通过修改这里的配置项来自定义你的 CDN 系统
|
||||
---
|
||||
base: # 基础配置项
|
||||
# listenPort cdn 服务的监听端口.
|
||||
# default: 8003
|
||||
# CDN 配置模版
|
||||
# 你可以通过修改这里的配置项来自定义你的 CDN
|
||||
base:
|
||||
# CDN 服务监听的端口
|
||||
# 默认值：8003
|
||||
listenPort: 8003
|
||||
|
||||
# DownloadPort CDN提供文件下载的端口.
|
||||
# 你需要先启动一个文件服务器，并且该文件服务器监听该下载端口。.
|
||||
# default: 8001
|
||||
# CDN 提供文件下载服务的端口
|
||||
# 你需要先启动一个文件服务器，并且该文件服务器监听该下载端口。
|
||||
# 默认值：8001
|
||||
downloadPort: 8001
|
||||
|
||||
# SystemReservedBandwidth is the network bandwidth reserved for system software.
|
||||
# default: 20 MB, in format of G(B)/g/M(B)/m/K(B)/k/B, pure number will also be parsed as Byte.
|
||||
# 为系统软件预留的网络带宽
|
||||
# 接收的输入格式以 G(B)/g/M(B)/m/K(B)/k/B 结尾作为单位。如果您输入一个不带单位的整数，它的单位会被认为是 B(Byte)。
|
||||
# 默认值：20 MB
|
||||
systemReservedBandwidth: 20M
|
||||
|
||||
# MaxBandwidth is the network bandwidth that cdn can use.
|
||||
# default: 1G, in format of G(B)/g/M(B)/m/K(B)/k/B, pure number will also be parsed as Byte.
|
||||
# CDN 可以使用的最大网络带宽
|
||||
# 接收的输入格式以 G(B)/g/M(B)/m/K(B)/k/B 结尾作为单位。如果您输入一个不带单位的整数，它的单位会被认为是 B(Byte)。
|
||||
# 默认值：1G
|
||||
maxBandwidth: 1G
|
||||
|
||||
# FailAccessInterval is the interval time after failed to access the URL.
|
||||
# If a task failed to be downloaded from the source, it will not be retried in the time since the last failure.
|
||||
# default: 3m
|
||||
# CDN 暴露给 p2p 网络中的其他 peer 的 IP 地址
|
||||
# 默认值：首个非本地回环IP。
|
||||
advertiseIP:
|
||||
|
||||
# CDN 请求某个 URL 失败后，不再对该 URL 发起请求的时间间隔长度
|
||||
# 换句话说，如果一个回源下载任务失败了，在这段时间里它将不会被重试。
|
||||
# 默认值：3m
|
||||
failAccessInterval: 3m
|
||||
|
||||
# GCInitialDelay is the delay time from the start to the first GC execution.
|
||||
# default: 6s
|
||||
# CDN 启动后到启动第一次垃圾回收的时间间隔
|
||||
# 默认值：6s
|
||||
gcInitialDelay: 6s
|
||||
|
||||
# GCMetaInterval is the interval time to execute GC meta.
|
||||
# default: 2m0s
|
||||
# 进行元数据信息回收的时间间隔
|
||||
# 每隔一个该时间间隔，CDN 就会启动一次元数据信息回收。
|
||||
# 默认值：2m0s
|
||||
gcMetaInterval: 2m
|
||||
|
||||
# gcStorageInterval is the interval time to execute GC storage.
|
||||
# default: 15s
|
||||
gcStorageInterval: 15s
|
||||
|
||||
# TaskExpireTime when a task is not accessed within the taskExpireTime,
|
||||
# and it will be treated to be expired.
|
||||
# default: 3m0s
|
||||
# 任务失效时间
|
||||
# 如果一个任务的信息距离上次访问过去了一个该时间间隔，该任务信息将被认为已经失效。
|
||||
# 默认值：3m0s
|
||||
taskExpireTime: 3m
|
||||
|
||||
# StoragePattern is the pattern of storage policy, [disk/hybrid]
|
||||
storagePattern: disk
|
||||
# CDN 使用的存储插件，可选[disk/hybrid]。disk 完全使用硬盘存储，hybrid 同时使用内存和硬盘进行存储。
|
||||
# 默认值：disk
|
||||
storageMode: disk
|
||||
|
||||
plugins: # 插件配置
|
||||
storage: # 存储插件配置
|
||||
# CDN 连接的 manager，可以不指定。
|
||||
# 各项配置默认值如下。如果 addr 为空字符串，CDN将不会连接manager。
|
||||
manager:
|
||||
addr: ""
|
||||
cdnClusterID: 0
|
||||
keepAlive:
|
||||
interval: 5s
|
||||
|
||||
# 主机信息
|
||||
host:
|
||||
# 地理位置
|
||||
location: ""
|
||||
# IDC(Internet Data Center)，互联网数据中心
|
||||
idc: ""
|
||||
|
||||
plugins:
|
||||
storagedriver:
|
||||
- name: disk
|
||||
enable: true
|
||||
config:
|
||||
baseDir: /tmp/cdnsystem2
|
||||
gcConfig:
|
||||
youngGCThreshold: 100G
|
||||
fullGCThreshold: 5G
|
||||
cleanRatio: 1
|
||||
intervalThreshold: 2h
|
||||
baseDir: /Users/${USER_HOME}/ftp
|
||||
- name: memory
|
||||
enable: false
|
||||
config:
|
||||
baseDir: /dev/shm/dragonfly
|
||||
storagemanager:
|
||||
- name: disk
|
||||
enable: true
|
||||
config:
|
||||
baseDir: /tmp/memory/dragonfly
|
||||
gcInitialDelay: 0s
|
||||
gcInterval: 15s
|
||||
driverConfigs:
|
||||
disk:
|
||||
gcConfig:
|
||||
youngGCThreshold: 100G
|
||||
fullGCThreshold: 5G
|
||||
youngGCThreshold: 100.0GB
|
||||
fullGCThreshold: 5.0GB
|
||||
cleanRatio: 1
|
||||
intervalThreshold: 2h0m0s
|
||||
- name: hybrid
|
||||
enable: false
|
||||
config:
|
||||
gcInitialDelay: 0s
|
||||
gcInterval: 15s
|
||||
driverConfigs:
|
||||
disk:
|
||||
gcConfig:
|
||||
youngGCThreshold: 100.0GB
|
||||
fullGCThreshold: 5.0GB
|
||||
cleanRatio: 1
|
||||
intervalThreshold: 2h0m0s
|
||||
memory:
|
||||
gcConfig:
|
||||
youngGCThreshold: 100.0GB
|
||||
fullGCThreshold: 5.0GB
|
||||
cleanRatio: 3
|
||||
intervalThreshold: 2h
|
||||
intervalThreshold: 2h0m0s
|
||||
|
||||
# Console shows log on console
|
||||
# default: false
|
||||
# 是否在控制台程序中显示日志
|
||||
# 默认值：false
|
||||
console: false
|
||||
|
||||
# Whether to enable debug level logger and enable pprof
|
||||
# default: false
|
||||
# 是否使用调试级别的日志、是否启用 pprof
|
||||
# 默认值：false
|
||||
verbose: false
|
||||
|
||||
# listen port for pprof, only valid when the verbose option is true
|
||||
# default is random port
|
||||
pprofPort: 0
|
||||
# pprof 监听的端口，仅在 verbose 为 true 时可用
|
||||
# 默认值：-1（表示不启动），为 0 时使用随机值
|
||||
pprofPort: -1
|
||||
|
||||
# the service address that provides the configuration item
|
||||
configServer: 127.0.0.1:8004
|
||||
# jaeger 地址
|
||||
# 默认使用空字符串（不配置 jaeger）
|
||||
jaeger:
|
||||
|
||||
# tracer 中使用的 service-name
|
||||
# 默认值：dragonfly-cdn
|
||||
service-name: dragonfly-cdn
|
||||
|
|
|
|||
|
|
@ -0,0 +1,45 @@
|
|||
# 此文件是 manager 的配置文件模板，你可以通过根据需要改变对应的值来配置 manager 服务。
|
||||
---
|
||||
# 当前的服务配置
|
||||
server:
|
||||
# grpc 服务配置
|
||||
grpc:
|
||||
# 监听的 ip 地址
|
||||
listen: 127.0.0.1
|
||||
# 监听的端口, manager 会从 start 到 end 之间的按顺序中选择一个可用端口
|
||||
port:
|
||||
start: 65003
|
||||
end: 65003
|
||||
# rest 服务配置
|
||||
rest:
|
||||
# 标准的 rest 服务地址: ip:port, ip 不配置则默认为0.0.0.0
|
||||
addr: :8080
|
||||
|
||||
# 数据库配置, 当前只支持 mysql 以及 redis
|
||||
database:
|
||||
# mysql 配置
|
||||
mysql:
|
||||
user: dragonfly
|
||||
password: dragonfly
|
||||
host: dragonfly
|
||||
port: 3306
|
||||
dbname: manager
|
||||
# redis 配置
|
||||
redis:
|
||||
password: dragonfly
|
||||
host: dragonfly
|
||||
port: 6379
|
||||
db: 0
|
||||
|
||||
# 缓存配置
|
||||
cache:
|
||||
# redis 缓存配置
|
||||
redis:
|
||||
# ttl 配置，单位[纳秒]
|
||||
ttl: 30000000000
|
||||
# 本地缓存配置
|
||||
local:
|
||||
# LFU 缓存大小
|
||||
size: 10000
|
||||
# ttl 配置，单位[纳秒]
|
||||
ttl: 3000000000
|
||||
|
|
@ -0,0 +1,139 @@
|
|||
# scheduler 配置文件模版, 您可以根据您的需求更改这里的参数来配置您的 Scheduler
|
||||
|
||||
# scheduler 调度策略配置
|
||||
scheduler:
|
||||
# abtest 是否开启ab测试
|
||||
# default: false
|
||||
abtest: false
|
||||
# aevaluator 指定当开启AB测试时，后缀为 _A 的 taskID 使用的 evaluator 计算方法
|
||||
# 只有当开启了 AB 测试的情况下该配置才生效
|
||||
# default: ""
|
||||
aevaluator: ""
|
||||
# bevaluator 指定当开启AB测试时，后缀为 _B 的 taskID 使用的 evaluator 计算方法
|
||||
# 只有当开启了 AB 测试的情况下该配置才生效
|
||||
# default: ""
|
||||
bevaluator: ""
|
||||
# workerNum 执行调度任务处理的 goroutine 数量
|
||||
# default: 默认机器可用的 CPU 数量
|
||||
workerNum: 12
|
||||
# backSourceCount 当 CDN 禁用或不可用时，回源客户端的数量
|
||||
# default: 3
|
||||
backSourceCount: 3
|
||||
# accessWindow 可触发 CDN 回源的时间间隔窗口
|
||||
# default: 3m
|
||||
accessWindow: 3m
|
||||
# candidateParentCount 候选父节点数量，
|
||||
# default: 10
|
||||
candidateParentCount: 10
|
||||
# scheduler 当前生效的 scheduler 调度策略
|
||||
# default: basic
|
||||
scheduler: basic
|
||||
# cdnLoad CDN 节点可以提供上传的最大负载
|
||||
# default: 100
|
||||
cdnLoad: 100
|
||||
# clientLoad 普通客户端节点可以提供上传的最大负载
|
||||
# default: 10
|
||||
clientLoad: 10
|
||||
# openMonitor 是否打开监控，目前只监控了当前的 peer 列表状态信息
|
||||
# default: false
|
||||
openMonitor: false
|
||||
# gc 元数据回收策略
|
||||
gc:
|
||||
# peerGCInterval peer 的回收间隔
|
||||
peerGCInterval: 1m
|
||||
# peerTTL peer 的离开时间, 距离上次 peer 的访问时间超过改值则 peer 会被设置成 leave 状态
|
||||
peerTTL: 10m
|
||||
# peerTTI peer 的 TTI 时间，距离上次 peer 的访问时间超过改值则 peer 会被设置成 zombie 状态
|
||||
peerTTI: 3m
|
||||
# taskGCInterval task 的回收间隔
|
||||
taskGCInterval: 1m
|
||||
# taskTTL task 的离开时间, 距离上次 task 的访问时间超过改值则 task 会被设置成 leave 状态
|
||||
taskTTL: 10m
|
||||
# taskTTI 的 TTI 时间，距离上次 task 的访问时间超过改值则 task 会被设置成 zombie 状态
|
||||
taskTTI: 3m
|
||||
|
||||
# server scheduler 服务实例配置信息
|
||||
server:
|
||||
# ip
|
||||
ip:
|
||||
# host
|
||||
host:
|
||||
# ListenPort is the ip and port scheduler server listens on.
|
||||
# default: 8002
|
||||
port:
|
||||
|
||||
# 动态数据配置
|
||||
dynConfig:
|
||||
# type 动态数据来源类型 manager 从远端manager读取 / local 从本地文件读取
|
||||
# default: local
|
||||
type: local
|
||||
# expireTime 缓存的过期时间
|
||||
# default 30s
|
||||
expireTime: 30s
|
||||
# cdnDirPath cdn 实例列表文件存放的目录
|
||||
cdnDirPath:
|
||||
# data 本地 CDN 动态配置
|
||||
data: ""
|
||||
|
||||
# manager 配置
|
||||
manager:
|
||||
# addr manager 访问地址
|
||||
addr: ""
|
||||
# schedulerClusterID 要注册的 scheduler 集群 ID
|
||||
schedulerClusterID:
|
||||
# keepAlive 和 manager 保持心跳的配置
|
||||
keepAlive:
|
||||
# interval 保持心跳的时间间隔
|
||||
interval: 5s
|
||||
|
||||
# scheduler 自身主机信息
|
||||
host:
|
||||
# location scheduler 实例所在的地理位置信息
|
||||
location: ""
|
||||
# idc scheduler 实例所在机房
|
||||
idc: ""
|
||||
|
||||
# machinery 异步任务配置，配置参考 https://github.com/RichardKnop/machinery
|
||||
job:
|
||||
# globalWorkerNum
|
||||
globalWorkerNum: 2
|
||||
# schedulerWorkerNum
|
||||
schedulerWorkerNum: 3
|
||||
# localWorkerNum
|
||||
localWorkerNum: 3
|
||||
# redis configuration
|
||||
redis:
|
||||
# host redis 服务实例地址
|
||||
host: ""
|
||||
# port redis 服务实例端口
|
||||
port:
|
||||
# password
|
||||
password: ""
|
||||
# brokerDB
|
||||
brokerDB: ""
|
||||
# backendDB
|
||||
backendDB: ""
|
||||
|
||||
# disableCDN 是否停用 CDN
|
||||
# default: false
|
||||
disableCDN: false
|
||||
|
||||
# Console 是否在控制台程序中显示日志
|
||||
# default: false
|
||||
console: false
|
||||
|
||||
# verbose 是否使用调试级别的日志、是否启用 pprof。
|
||||
# default: false
|
||||
verbose: false
|
||||
|
||||
# pprofPort pprof 监听的端口，仅在 verbose 为 true 时可用
|
||||
# default is -1. If it is 0, pprof will use a random port.
|
||||
pprofPort: -1
|
||||
|
||||
# jaeger 地址
|
||||
# 默认使用空字符串（不配置 jaeger）
|
||||
jaeger:
|
||||
|
||||
# tracer 中使用的 service-name
|
||||
# 默认值：dragonfly-cdn
|
||||
service-name: dragonfly-scheduler
|
||||
|
|
@ -44,7 +44,7 @@
|
|||
|
||||
#### CDN模块
|
||||
|
||||
|
||||
![cdn][cdn]
|
||||
|
||||
### 子系统功能
|
||||
|
||||
|
|
@ -100,3 +100,4 @@
|
|||
[arch]: ../images/arch.png
|
||||
[association]: ../images/association.png
|
||||
[download-process]: ../images/download-process.png
|
||||
[cdn]: ../images/cdn.png
|
||||
|
|
|
|||
|
|
@ -0,0 +1,62 @@
|
|||
# 开发
|
||||
|
||||
快速搭建本地开发环境
|
||||
|
||||
### 第一步：安装 docker 和 docker compose
|
||||
|
||||
详见文档 [docs.docker.com]
|
||||
|
||||
### 第二步：启动 dragonfly
|
||||
|
||||
打开 dragonfly 文档、启动 docker-compose.
|
||||
|
||||
```bash
|
||||
$ docker-compose up
|
||||
Creating network "dragonfly2_default" with the default driver
|
||||
Creating cdn ... done
|
||||
Creating scheduler ... done
|
||||
Creating dfdaemon ... done
|
||||
Attaching to cdn, scheduler, dfdaemon
|
||||
```
|
||||
|
||||
### 第三步：日志分析
|
||||
|
||||
查看 dragonfly 日志
|
||||
|
||||
```bash
|
||||
$ tail -f log/**/*.log
|
||||
==> log/dragonfly/cdn/core.log <==
|
||||
{"level":"info","ts":"2021-02-26 05:43:36.896","caller":"cmd/root.go:90","msg":"success to init local ip of cdn, start to run cdn system, use ip: 172.21.0.2"}
|
||||
{"level":"info","ts":"2021-02-26 05:43:36.901","caller":"plugins/plugins.go:51","msg":"add plugin[storage][local]"}
|
||||
{"level":"info","ts":"2021-02-26 05:43:36.902","caller":"plugins/plugins.go:37","msg":"plugin[sourceclient][http] is disabled"}
|
||||
|
||||
==> log/dragonfly/dfdaemon/core.log <==
|
||||
{"level":"info","ts":"2021-02-26 05:43:37.797","caller":"proxy/proxy_manager.go:63","msg":"registry mirror: https://index.docker.io"}
|
||||
{"level":"info","ts":"2021-02-26 05:43:37.798","caller":"proxy/proxy_manager.go:68","msg":"load 1 proxy rules"}
|
||||
{"level":"info","ts":"2021-02-26 05:43:37.799","caller":"proxy/proxy_manager.go:78","msg":"[1] proxy blobs/sha256.* with dragonfly "}
|
||||
{"level":"debug","ts":"2021-02-26 05:43:37.799","caller":"rpc/server_listen.go:31","msg":"start to listen port: 0.0.0.0:65000"}
|
||||
{"level":"debug","ts":"2021-02-26 05:43:37.800","caller":"rpc/server_listen.go:31","msg":"start to listen port: 0.0.0.0:65002"}
|
||||
{"level":"debug","ts":"2021-02-26 05:43:37.800","caller":"rpc/server_listen.go:31","msg":"start to listen port: 0.0.0.0:65001"}
|
||||
{"level":"info","ts":"2021-02-26 05:43:37.800","caller":"daemon/peerhost.go:274","msg":"serve download grpc at unix:///tmp/dfdamon.sock"}
|
||||
{"level":"info","ts":"2021-02-26 05:43:37.801","caller":"daemon/peerhost.go:285","msg":"serve peer grpc at tcp://[::]:65000"}
|
||||
{"level":"info","ts":"2021-02-26 05:43:37.801","caller":"daemon/peerhost.go:320","msg":"serve upload service at tcp://[::]:65002"}
|
||||
{"level":"info","ts":"2021-02-26 05:43:37.801","caller":"daemon/peerhost.go:306","msg":"serve proxy at tcp://0.0.0.0:65001"}
|
||||
|
||||
==> log/dragonfly/scheduler/core.log <==
|
||||
{"level":"info","ts":"2021-02-26 05:43:37.332","caller":"cmd/root.go:57","msg":"start to run scheduler"}
|
||||
{"level":"info","ts":"2021-02-26 05:43:37.338","caller":"server/server.go:35","msg":"start server at port %!s(int=8002)"}
|
||||
{"level":"info","ts":"2021-02-26 05:43:37.342","caller":"worker/sender.go:49","msg":"start sender worker : 50"}
|
||||
{"level":"info","ts":"2021-02-26 05:43:37.343","caller":"worker/worker_group.go:64","msg":"start scheduler worker number:6"}
|
||||
```
|
||||
|
||||
### 第四步：关闭 dragonfly
|
||||
|
||||
```bash
|
||||
$ docker-compose down
|
||||
Removing dfdaemon ... done
|
||||
Removing scheduler ... done
|
||||
Removing cdn ... done
|
||||
Removing network dragonfly2_default
|
||||
```
|
||||
|
||||
[docs.docker.com]: https://docs.docker.com
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
[install-client](./install-client.md) 是 client 的安装指南
|
||||
|
||||
[install-manager](./install-manager.md) 是 manager 的安装指南
|
||||
|
||||
[install-scheduler](./install-scheduler.md) 是 scheduler 的安装指南
|
||||
|
||||
[install-cdn](./install-cdn.md) 是 CDN 的安装指南
|
||||
|
|
@ -0,0 +1,161 @@
|
|||
# 安装 Dragonfly CDN
|
||||
|
||||
本文档阐述如何安装并启动 Dragonfly CDN。
|
||||
|
||||
## 部署方式
|
||||
|
||||
用下列方式之一部署 CDN：
|
||||
|
||||
- 通过 Docker 部署
|
||||
- 直接在物理机上部署：推荐用于生产用途
|
||||
|
||||
## 环境要求
|
||||
|
||||
使用 Docker 部署时，以下条件必须满足：
|
||||
|
||||
所需软件 | 版本要求
|
||||
---|---
|
||||
Git|1.9.1+
|
||||
Docker|1.12.0+
|
||||
|
||||
直接在物理机上部署时，以下条件必须满足：
|
||||
|
||||
所需软件 | 版本要求
|
||||
---|---
|
||||
Git|1.9.1+
|
||||
Golang|1.12.x
|
||||
Nginx|0.8+
|
||||
|
||||
## 使用 Docker 部署
|
||||
|
||||
### 获取 CDN 镜像
|
||||
|
||||
您可以直接从 [DockerHub](https://hub.docker.com/) 获取 CDN 镜像。
|
||||
|
||||
1. 获取最新的 CDN 镜像
|
||||
|
||||
```sh
|
||||
docker pull dragonflyoss/cdn
|
||||
```
|
||||
|
||||
或者您可以构建自己的 CDN 镜像
|
||||
|
||||
1. 获取 Dragonfly 的源码
|
||||
|
||||
```sh
|
||||
git clone https://github.com/dragonflyoss/Dragonfly2.git
|
||||
```
|
||||
|
||||
2. 打开项目文件夹
|
||||
|
||||
```sh
|
||||
cd Dragonfly2
|
||||
```
|
||||
|
||||
3. 构建 CDN 的 Docker 镜像
|
||||
|
||||
```sh
|
||||
TAG="1.0.0"
|
||||
make docker-build-cdn D7Y_VERSION=$TAG
|
||||
```
|
||||
|
||||
4. 获取最新的 CDN 镜像 ID
|
||||
|
||||
```sh
|
||||
docker image ls | grep 'cdn' | awk '{print $3}' | head -n1
|
||||
```
|
||||
|
||||
### 启动 cdn
|
||||
|
||||
**注意：** 需要使用上述步骤获得的 ID 替换 ${cdnDockerImageId}。
|
||||
|
||||
```sh
|
||||
docker run -d --name cdn --restart=always -p 8001:8001 -p 8003:8003 -v /home/admin/ftp:/home/admin/ftp ${cdnDockerImageId}
|
||||
--download-port=8001
|
||||
```
|
||||
|
||||
## 在物理机上部署
|
||||
|
||||
### 获取 CDN 可执行文件
|
||||
|
||||
1. 下载 Dragonfly 项目的压缩包。您可以从 [github releases page](https://github.
|
||||
com/dragonflyoss/Dragonfly2/releases) 下载一个已发布的最近版本
|
||||
|
||||
```sh
|
||||
version=1.0.0
|
||||
wget https://github.com/dragonflyoss/Dragonfly2/releases/download/v$version/Dragonfly2_$version_linux_amd64.tar.gz
|
||||
```
|
||||
|
||||
2. 解压压缩包
|
||||
|
||||
```bash
|
||||
# Replace `xxx` with the installation directory.
|
||||
tar -zxf Dragonfly2_1.0.0_linux_amd64.tar.gz -C xxx
|
||||
```
|
||||
|
||||
3. 把 `cdn` 移动到环境变量 `PATH` 下以确保您可以直接使用 `cdn` 命令
|
||||
|
||||
或者您可以编译生成自己的 CDN 可执行文件。
|
||||
|
||||
1. 获取 Dragonfly 的源码
|
||||
|
||||
```sh
|
||||
git clone https://github.com/dragonflyoss/Dragonfly2.git
|
||||
```
|
||||
|
||||
2. 打开项目文件夹
|
||||
|
||||
```sh
|
||||
cd Dragonfly2
|
||||
```
|
||||
|
||||
3. 编译源码
|
||||
|
||||
```sh
|
||||
make build-cdn && make install-cdn
|
||||
```
|
||||
|
||||
### 启动 cdn
|
||||
|
||||
```sh
|
||||
cdnHomeDir=/home/admin
|
||||
cdnDownloadPort=8001
|
||||
cdn --port=8003 --download-port=$cdnDownloadPort
|
||||
```
|
||||
|
||||
### 启动 file server
|
||||
|
||||
您可以在满足以下条件的基础上用任何方式启动 file server：
|
||||
|
||||
- 必须挂载在前面步骤中已经定义的目录 `${cdnHomeDir}/ftp` 上。
|
||||
- 必须监听前面步骤中已经定义的 `cdnDownloadPort` 端口。
|
||||
|
||||
以 nginx 为例：
|
||||
|
||||
1. 将下面的配置添加到 Nginx 配置文件中
|
||||
|
||||
```conf
|
||||
server {
|
||||
# Must be ${cdnDownloadPort}
|
||||
listen 8001;
|
||||
location / {
|
||||
# Must be ${cdnHomeDir}/ftp
|
||||
root /home/admin/ftp;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
2. 启动 Nginx.
|
||||
|
||||
```sh
|
||||
sudo nginx
|
||||
```
|
||||
|
||||
## 检查
|
||||
|
||||
- CDN 部署完成之后，运行以下命令以检查 Nginx 和 **cdn** 是否正在运行，以及 `8001` 和 `8003` 端口是否可用。
|
||||
|
||||
```sh
|
||||
telnet 127.0.0.1 8001
|
||||
telnet 127.0.0.1 8003
|
||||
```
|
||||
|
|
@ -0,0 +1,129 @@
|
|||
# 安装 Dragonfly Scheduler
|
||||
|
||||
本文档阐述如何安装并启动 Dragonfly Scheduler。
|
||||
|
||||
## 部署方式
|
||||
|
||||
用下列方式之一部署 scheduler：
|
||||
|
||||
- 通过 Docker 部署：推荐用于生产用途
|
||||
- 直接在物理机上部署
|
||||
|
||||
## 环境要求
|
||||
|
||||
使用 Docker 部署时，以下条件必须满足：
|
||||
|
||||
所需软件 | 版本要求
|
||||
---|---
|
||||
Git|1.9.1+
|
||||
Docker|1.12.0+
|
||||
|
||||
直接在物理机上部署时，以下条件必须满足：
|
||||
|
||||
所需软件 | 版本要求
|
||||
---|---
|
||||
Git|1.9.1+
|
||||
Golang|1.12.x
|
||||
Nginx|0.8+
|
||||
|
||||
## 使用 Docker 部署
|
||||
|
||||
### 获取 scheduler 镜像
|
||||
|
||||
您可以直接从 [DockerHub](https://hub.docker.com/) 获取 scheduler 镜像。
|
||||
|
||||
1. 获取最新的 scheduler 镜像
|
||||
|
||||
```sh
|
||||
docker pull dragonflyoss/scheduler
|
||||
```
|
||||
|
||||
或者您可以构建自己的 scheduler 镜像
|
||||
|
||||
1. 获取 Dragonfly 的源码
|
||||
|
||||
```sh
|
||||
git clone https://github.com/dragonflyoss/Dragonfly2.git
|
||||
```
|
||||
|
||||
2. 打开项目文件夹
|
||||
|
||||
```sh
|
||||
cd Dragonfly2
|
||||
```
|
||||
|
||||
3. 构建 scheduler 的 Docker 镜像
|
||||
|
||||
```sh
|
||||
TAG="1.0.0"
|
||||
make docker-build-scheduler D7Y_VERSION=$TAG
|
||||
```
|
||||
|
||||
4. 获取最新的 scheduler 镜像 ID
|
||||
|
||||
```sh
|
||||
docker image ls | grep 'scheduler' | awk '{print $3}' | head -n1
|
||||
```
|
||||
|
||||
### 启动 scheduler
|
||||
|
||||
**注意：** 需要使用上述步骤获得的 ID 替换 ${schedulerDockerImageId}。
|
||||
|
||||
```sh
|
||||
docker run -d --name scheduler --restart=always -p 8002 ${schedulerDockerImageId}
|
||||
```
|
||||
|
||||
## 在物理机上部署
|
||||
|
||||
### 获取 scheduler 可执行文件
|
||||
|
||||
1. 下载 Dragonfly 项目的压缩包。您可以从 [github releases page](https://github.
|
||||
com/dragonflyoss/Dragonfly2/releases) 下载一个已发布的最近版本
|
||||
|
||||
```sh
|
||||
version=1.0.0
|
||||
wget https://github.com/dragonflyoss/Dragonfly2/releases/download/v$version/Dragonfly2_$version_linux_amd64.tar.gz
|
||||
```
|
||||
|
||||
2. 解压压缩包
|
||||
|
||||
```bash
|
||||
# Replace `xxx` with the installation directory.
|
||||
tar -zxf Dragonfly2_1.0.0_linux_amd64.tar.gz -C xxx
|
||||
```
|
||||
|
||||
3. 把 `scheduler` 移动到环境变量 `PATH` 下以确保您可以直接使用 `scheduler` 命令
|
||||
|
||||
或者您可以编译生成自己的 scheduler 可执行文件。
|
||||
|
||||
1. 获取 Dragonfly 的源码
|
||||
|
||||
```sh
|
||||
git clone https://github.com/dragonflyoss/Dragonfly2.git
|
||||
```
|
||||
|
||||
2. 打开项目文件夹
|
||||
|
||||
```sh
|
||||
cd Dragonfly2
|
||||
```
|
||||
|
||||
3. 编译源码
|
||||
|
||||
```sh
|
||||
make build-scheduler && make install-scheduler
|
||||
```
|
||||
|
||||
### 启动 scheduler
|
||||
|
||||
```sh
|
||||
scheduler --options
|
||||
```
|
||||
|
||||
## 检查
|
||||
|
||||
- scheduler 部署完成之后，运行以下命令以检查 **scheduler** 是否正在运行，以及 `8002` 端口是否可用。
|
||||
|
||||
```sh
|
||||
telnet 127.0.0.1 8002
|
||||
```
|
||||
|
|
@ -0,0 +1,147 @@
|
|||
# 使用 dfget daemon 作为 containerd 的 http 代理
|
||||
|
||||
目前 containerd 的 `ctr` 命令不支持带有 registry-mirrors 的私有注册表，为此，我们需要为 containerd 使用 HTTP 代理。
|
||||
|
||||
## 快速开始
|
||||
|
||||
### 第一步：为 http 代理生成 CA 证书
|
||||
|
||||
生成一个 CA 证书私钥。
|
||||
|
||||
```bash
|
||||
openssl genrsa -out ca.key 2048
|
||||
```
|
||||
|
||||
打开 openssl 配置文件 `openssl.conf`。设置 `basicConstraints` 为 true，然后您就能修改这些值。
|
||||
|
||||
```text
|
||||
[ req ]
|
||||
#default_bits = 2048
|
||||
#default_md = sha256
|
||||
#default_keyfile = privkey.pem
|
||||
distinguished_name = req_distinguished_name
|
||||
attributes = req_attributes
|
||||
extensions = v3_ca
|
||||
req_extensions = v3_ca
|
||||
|
||||
[ req_distinguished_name ]
|
||||
countryName = Country Name (2 letter code)
|
||||
countryName_min = 2
|
||||
countryName_max = 2
|
||||
stateOrProvinceName = State or Province Name (full name)
|
||||
localityName = Locality Name (eg, city)
|
||||
0.organizationName = Organization Name (eg, company)
|
||||
organizationalUnitName = Organizational Unit Name (eg, section)
|
||||
commonName = Common Name (eg, fully qualified host name)
|
||||
commonName_max = 64
|
||||
emailAddress = Email Address
|
||||
emailAddress_max = 64
|
||||
|
||||
[ req_attributes ]
|
||||
challengePassword = A challenge password
|
||||
challengePassword_min = 4
|
||||
challengePassword_max = 20
|
||||
|
||||
[ v3_ca ]
|
||||
basicConstraints = CA:TRUE
|
||||
```
|
||||
|
||||
生成 CA 证书。
|
||||
|
||||
```bash
|
||||
openssl req -new -key ca.key -nodes -out ca.csr -config openssl.conf
|
||||
openssl x509 -req -days 36500 -extfile openssl.conf -extensions v3_ca -in ca.csr -signkey ca.key -out ca.crt
|
||||
```
|
||||
|
||||
### 第二步：配置 dfget daemon
|
||||
|
||||
为了将 dfget daemon 作为 http 代理使用，首先你需要在 `/etc/dragonfly/dfget.yaml` 中增加一条代理规则，
|
||||
它将会代理 `your.private.registry` 对镜像层的请求：
|
||||
|
||||
```yaml
|
||||
proxy:
|
||||
security:
|
||||
insecure: true
|
||||
tcpListen:
|
||||
listen: 0.0.0.0
|
||||
port: 65001
|
||||
proxies:
|
||||
- regx: blobs/sha256.*
|
||||
hijackHTTPS:
|
||||
# CA certificate's path used to hijack https requests
|
||||
cert: ca.crt
|
||||
key: ca.key
|
||||
hosts:
|
||||
- regx: your.private.registry
|
||||
```
|
||||
|
||||
### 第三步：配置 containerd
|
||||
|
||||
在 `/etc/systemd/system/containerd.service.d/http-proxy.conf` 设置 dfdaemon 为 docker daemon 的
|
||||
`HTTP_PROXY` 和 `HTTPS_PROXY` 代理：
|
||||
|
||||
```
|
||||
[Service]
|
||||
Environment="HTTP_PROXY=http://127.0.0.1:65001"
|
||||
Environment="HTTPS_PROXY=http://127.0.0.1:65001"
|
||||
```
|
||||
|
||||
### 第四步：使用代理拉取镜像
|
||||
|
||||
完成以上的步骤后，我们可以尝试验证 Dragonfly 是否像我们预期的一样正常工作。
|
||||
|
||||
您可以像往常一样拉取镜像，比如：
|
||||
|
||||
```bash
|
||||
ctr image pull your.private.registry/namespace/image:latest
|
||||
```
|
||||
|
||||
## 自定义配置项
|
||||
|
||||
### 使用自签名证书注册
|
||||
|
||||
如果您使用一个自签名证书进行注册，你可以用以下配置来忽略证书错误：
|
||||
|
||||
```yaml
|
||||
proxy:
|
||||
security:
|
||||
insecure: true
|
||||
tcpListen:
|
||||
listen: 0.0.0.0
|
||||
port: 65001
|
||||
proxies:
|
||||
- regx: blobs/sha256.*
|
||||
hijackHTTPS:
|
||||
# CA certificate's path used to hijack https requests
|
||||
cert: ca.crt
|
||||
key: ca.key
|
||||
hosts:
|
||||
- regx: your.private.registry
|
||||
insecure: true
|
||||
```
|
||||
|
||||
也可以使用以下配置提供一个证书：
|
||||
|
||||
```yaml
|
||||
proxy:
|
||||
security:
|
||||
insecure: true
|
||||
tcpListen:
|
||||
listen: 0.0.0.0
|
||||
port: 65001
|
||||
proxies:
|
||||
- regx: blobs/sha256.*
|
||||
hijackHTTPS:
|
||||
# CA certificate's path used to hijack https requests
|
||||
cert: ca.crt
|
||||
key: ca.key
|
||||
hosts:
|
||||
- regx: your.private.registry
|
||||
certs: ["server.crt"]
|
||||
```
|
||||
|
||||
您能使用以下命令获取您服务器的证书：
|
||||
|
||||
```
|
||||
openssl x509 -in <(openssl s_client -showcerts -servername xxx -connect xxx:443 -prexit 2>/dev/null)
|
||||
```
|
||||
|
|
@ -0,0 +1,156 @@
|
|||
# 使用 dfget daemon 作为 docker daemon 的 http 代理
|
||||
|
||||
目前 docker 不支持带有 registry-mirrors 的私有注册表，为此，我们需要为 docker daemon 使用 HTTP 代理。
|
||||
|
||||
## 快速开始
|
||||
|
||||
### 第一步：为 http 代理生成 CA 证书
|
||||
|
||||
生成一个 CA 证书私钥。
|
||||
|
||||
```bash
|
||||
openssl genrsa -out ca.key 2048
|
||||
```
|
||||
|
||||
打开 openssl 配置文件 `openssl.conf`。设置 `basicConstraints` 为 true，然后您就能修改这些值。
|
||||
|
||||
```text
|
||||
[ req ]
|
||||
#default_bits = 2048
|
||||
#default_md = sha256
|
||||
#default_keyfile = privkey.pem
|
||||
distinguished_name = req_distinguished_name
|
||||
attributes = req_attributes
|
||||
extensions = v3_ca
|
||||
req_extensions = v3_ca
|
||||
[ req_distinguished_name ]
|
||||
countryName = Country Name (2 letter code)
|
||||
countryName_min = 2
|
||||
countryName_max = 2
|
||||
stateOrProvinceName = State or Province Name (full name)
|
||||
localityName = Locality Name (eg, city)
|
||||
0.organizationName = Organization Name (eg, company)
|
||||
organizationalUnitName = Organizational Unit Name (eg, section)
|
||||
commonName = Common Name (eg, fully qualified host name)
|
||||
commonName_max = 64
|
||||
emailAddress = Email Address
|
||||
emailAddress_max = 64
|
||||
[ req_attributes ]
|
||||
challengePassword = A challenge password
|
||||
challengePassword_min = 4
|
||||
challengePassword_max = 20
|
||||
[ v3_ca ]
|
||||
basicConstraints = CA:TRUE
|
||||
```
|
||||
|
||||
生成 CA 证书。
|
||||
|
||||
```bash
|
||||
openssl req -new -key ca.key -nodes -out ca.csr -config openssl.conf
|
||||
openssl x509 -req -days 36500 -extfile openssl.conf -extensions v3_ca -in ca.csr -signkey ca.key -out ca.crt
|
||||
```
|
||||
|
||||
### 第二步：配置 dfget daemon
|
||||
|
||||
为了将 dfget daemon 作为 http 代理使用，首先你需要在 `/var/log/dragonfly/dfget.yaml` 中增加一条代理规则，
|
||||
它将会代理 `your.private.registry` 对镜像层的请求：
|
||||
|
||||
```yaml
|
||||
proxy:
|
||||
security:
|
||||
insecure: true
|
||||
tcpListen:
|
||||
listen: 0.0.0.0
|
||||
port: 65001
|
||||
proxies:
|
||||
- regx: blobs/sha256.*
|
||||
hijackHTTPS:
|
||||
# CA certificate's path used to hijack https requests
|
||||
cert: ca.crt
|
||||
key: ca.key
|
||||
hosts:
|
||||
- regx: your.private.registry
|
||||
```
|
||||
|
||||
### 第三步：配置 Docker daemon
|
||||
|
||||
为了忽略您的证书错误，您需要在
|
||||
`/etc/docker/daemon.json` 中把 `insecure-registries` 设置为您的私有代理：
|
||||
|
||||
```json
|
||||
{
|
||||
"insecure-registries": ["your.private.registry"]
|
||||
}
|
||||
```
|
||||
|
||||
### 第四步：继续配置 Docker daemon
|
||||
|
||||
在 `/etc/systemd/system/docker.service.d/http-proxy.conf` 设置 dfdaemon 为 docker daemon 的
|
||||
`HTTP_PROXY` 和 `HTTPS_PROXY` 代理：
|
||||
|
||||
```
|
||||
[Service]
|
||||
Environment="HTTP_PROXY=http://127.0.0.1:65001"
|
||||
Environment="HTTPS_PROXY=http://127.0.0.1:65001"
|
||||
```
|
||||
|
||||
### 第五步：使用代理拉取镜像
|
||||
|
||||
完成以上的步骤后，我们可以尝试验证 Dragonfly 是否像我们预期的一样正常工作。
|
||||
|
||||
您可以像往常一样拉取镜像，比如：
|
||||
|
||||
```bash
|
||||
docker pull your.private.registry/namespace/image:latest
|
||||
```
|
||||
|
||||
## 自定义配置项
|
||||
|
||||
### 使用自签名证书注册
|
||||
|
||||
如果您使用一个自签名证书进行注册，你可以用以下配置来忽略证书错误：
|
||||
|
||||
```yaml
|
||||
proxy:
|
||||
security:
|
||||
insecure: true
|
||||
tcpListen:
|
||||
listen: 0.0.0.0
|
||||
port: 65001
|
||||
proxies:
|
||||
- regx: blobs/sha256.*
|
||||
hijackHTTPS:
|
||||
# CA certificate's path used to hijack https requests
|
||||
cert: ca.crt
|
||||
key: ca.key
|
||||
hosts:
|
||||
- regx: your.private.registry
|
||||
insecure: true
|
||||
```
|
||||
|
||||
也可以使用以下配置提供一个证书：
|
||||
|
||||
```yaml
|
||||
proxy:
|
||||
security:
|
||||
insecure: true
|
||||
tcpListen:
|
||||
listen: 0.0.0.0
|
||||
port: 65001
|
||||
proxies:
|
||||
- regx: blobs/sha256.*
|
||||
hijackHTTPS:
|
||||
# CA certificate's path used to hijack https requests
|
||||
cert: ca.crt
|
||||
key: ca.key
|
||||
hosts:
|
||||
- regx: your.private.registry
|
||||
certs: ["server.crt"]
|
||||
```
|
||||
|
||||
您能使用以下命令获取您服务器的证书：
|
||||
|
||||
```
|
||||
openssl x509 -in <(openssl s_client -showcerts -servername xxx -connect xxx:443 -prexit 2>/dev/null)
|
||||
```
|
||||
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
# Dragonfly 快速开始
|
||||
|
||||
Dragonfly 快速开始文档的目标是帮助您快速开始使用 Dragonfly。
|
||||
|
||||
您可以根据 [Kubernetes-with-Dragonfly](../ecosystem/Kubernetes-with-Dragonfly.md) 文档中的内容快速搭建 Dragonfly 的 Kubernetes 集群。
|
||||
|
||||
下表列出了一些容器的运行时、版本和文档。
|
||||
|
||||
| Runtime | Version | Document | CRI Support | Pull Command |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| Containerd without CRI | All | [Link](./proxy/containerd.md) | No | ctr image pull docker.io/library/alpine |
|
||||
| Containerd with CRI | v1.1.0+ | [Link](./registry-mirror/cri-containerd.md) | Yes | crictl pull docker.io/library/alpine:latest |
|
||||
| CRI-O | All | [Link](./registry-mirror/cri-o.md) | Yes | crictl pull docker.io/library/alpine:latest |
|
||||
|
||||
当在 kubernetes 集群中使用 Dragonfly 时，我们推荐使用 `Containerd with CRI` 和 `CRI-O`，参考文档是
|
||||
[Kubernetes-with-Dragonfly](../ecosystem/Kubernetes-with-Dragonfly.md)。
|
||||
|
||||
## 相关文档
|
||||
|
||||
- [install manager](../user-guide/install/install-manager.md) - 安装 Dragonfly manager
|
||||
- [install cdn](../user-guide/install/install-cdn.md) - 安装 Dragonfly cdn
|
||||
- [install scheduler](../user-guide/install/install-scheduler.md) - 安装 Dragonfly scheduler
|
||||
- [proxy](../user-guide/proxy/containerd.md) - 使用 Dragonfly 作为 docker daemon 的 HTTP 代理
|
||||
- Container Runtimes
|
||||
- [cri-o mirror](../user-guide/registry-mirror/cri-o.md) - 使用 Dragonfly 作为 CRIO daemon 的 Registry Mirror
|
||||
- [cri-containerd_mirror](../user-guide/registry-mirror/cri-containerd.md) - 使用 Dragonfly 作为 containerd daemon 的 Registry Mirror
|
||||
|
|
@ -42,8 +42,8 @@ func New() *Config {
|
|||
return &Config{
|
||||
Scheduler: &SchedulerConfig{
|
||||
ABTest: false,
|
||||
AScheduler: "",
|
||||
BScheduler: "",
|
||||
AEvaluator: "",
|
||||
BEvaluator: "",
|
||||
WorkerNum: runtime.GOMAXPROCS(0),
|
||||
BackSourceCount: 3,
|
||||
AccessWindow: 3 * time.Minute,
|
||||
|
|
@ -51,7 +51,7 @@ func New() *Config {
|
|||
Scheduler: "basic",
|
||||
CDNLoad: 100,
|
||||
ClientLoad: 10,
|
||||
OpenMonitor: true,
|
||||
OpenMonitor: false,
|
||||
GC: &GCConfig{
|
||||
PeerGCInterval: 1 * time.Minute,
|
||||
TaskGCInterval: 1 * time.Minute,
|
||||
|
|
@ -180,8 +180,8 @@ type DynConfig struct {
|
|||
|
||||
type SchedulerConfig struct {
|
||||
ABTest bool `yaml:"abtest" mapstructure:"abtest"`
|
||||
AScheduler string `yaml:"ascheduler" mapstructure:"ascheduler"`
|
||||
BScheduler string `yaml:"bscheduler" mapstructure:"bscheduler"`
|
||||
AEvaluator string `yaml:"aevaluator" mapstructure:"aevaluator"`
|
||||
BEvaluator string `yaml:"bevaluator" mapstructure:"bevaluator"`
|
||||
WorkerNum int `yaml:"workerNum" mapstructure:"workerNum"`
|
||||
BackSourceCount int32 `yaml:"backSourceCount" mapstructure:"backSourceCount"`
|
||||
// AccessWindow should less than CDN task expireTime
|
||||
|
|
|
|||
|
|
@ -40,8 +40,8 @@ func TestSchedulerConfig_Load(t *testing.T) {
|
|||
},
|
||||
Scheduler: &SchedulerConfig{
|
||||
ABTest: true,
|
||||
AScheduler: "a-scheduler",
|
||||
BScheduler: "b-scheduler",
|
||||
AEvaluator: "a-evaluator",
|
||||
BEvaluator: "b-evaluator",
|
||||
WorkerNum: 8,
|
||||
},
|
||||
Server: &ServerConfig{
|
||||
|
|
|
|||
|
|
@ -5,8 +5,8 @@ dynconfig:
|
|||
|
||||
scheduler:
|
||||
abtest: true
|
||||
ascheduler: "a-scheduler"
|
||||
bscheduler: "b-scheduler"
|
||||
aevaluator: "a-evaluator"
|
||||
bevaluator: "b-evaluator"
|
||||
workerNum: 8
|
||||
workerJobPoolSize: 10000
|
||||
senderNum: 10
|
||||
|
|
|
|||
|
|
@ -46,8 +46,8 @@ type Factory struct {
|
|||
cache map[string]Evaluator
|
||||
cacheClearFunc sync.Once
|
||||
abtest bool
|
||||
ascheduler string
|
||||
bscheduler string
|
||||
aEvaluator string
|
||||
bEvaluator string
|
||||
}
|
||||
|
||||
var _ Evaluator = (*Factory)(nil)
|
||||
|
|
@ -70,8 +70,8 @@ func NewEvaluatorFactory(cfg *config.SchedulerConfig) *Factory {
|
|||
getEvaluatorFuncs: map[int]getEvaluatorFunc{},
|
||||
cache: map[string]Evaluator{},
|
||||
abtest: cfg.ABTest,
|
||||
ascheduler: cfg.AScheduler,
|
||||
bscheduler: cfg.BScheduler,
|
||||
aEvaluator: cfg.AEvaluator,
|
||||
bEvaluator: cfg.BEvaluator,
|
||||
}
|
||||
return factory
|
||||
}
|
||||
|
|
@ -104,12 +104,12 @@ func (ef *Factory) get(taskID string) Evaluator {
|
|||
if ef.abtest {
|
||||
name := ""
|
||||
if strings.HasSuffix(taskID, idgen.TwinsBSuffix) {
|
||||
if ef.bscheduler != "" {
|
||||
name = ef.bscheduler
|
||||
if ef.bEvaluator != "" {
|
||||
name = ef.bEvaluator
|
||||
}
|
||||
} else {
|
||||
if ef.ascheduler != "" {
|
||||
name = ef.ascheduler
|
||||
if ef.aEvaluator != "" {
|
||||
name = ef.aEvaluator
|
||||
}
|
||||
}
|
||||
if name != "" {
|
||||
|
|
|
|||
Loading…
Reference in New Issue