mirror of https://github.com/dragonflyoss/api.git
feat: add grpc proto files
Signed-off-by: Gaius <gaius.qi@gmail.com>
This commit is contained in:
Gaius
parent
9308208216
commit
f2f6989afe
|
|
@ -0,0 +1,185 @@
|
|||
/*
|
||||
* Copyright 2022 The Dragonfly Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
syntax = "proto3";
|
||||
|
||||
package common;
|
||||
|
||||
import "validate/validate.proto";
|
||||
|
||||
option go_package = "d7y.io/api/pkg/apis/common/v1";
|
||||
|
||||
enum Code{
|
||||
X_UNSPECIFIED = 0;
|
||||
// success code 200-299
|
||||
Success = 200;
|
||||
// framework can not find server node
|
||||
ServerUnavailable = 500;
|
||||
|
||||
// common response error 1000-1999
|
||||
// client can be migrated to another scheduler/CDN
|
||||
ResourceLacked = 1000;
|
||||
BackToSourceAborted = 1001;
|
||||
BadRequest = 1400;
|
||||
PeerTaskNotFound = 1404;
|
||||
UnknownError = 1500;
|
||||
RequestTimeOut = 1504;
|
||||
|
||||
// client response error 4000-4999
|
||||
ClientError = 4000;
|
||||
ClientPieceRequestFail = 4001; // get piece task from other peer error
|
||||
ClientScheduleTimeout = 4002; // wait scheduler response timeout
|
||||
ClientContextCanceled = 4003;
|
||||
ClientWaitPieceReady = 4004; // when target peer downloads from source slowly, should wait
|
||||
ClientPieceDownloadFail = 4005;
|
||||
ClientRequestLimitFail = 4006;
|
||||
ClientConnectionError = 4007;
|
||||
ClientBackSourceError = 4008;
|
||||
ClientPieceNotFound = 4404;
|
||||
|
||||
// scheduler response error 5000-5999
|
||||
SchedError = 5000;
|
||||
SchedNeedBackSource = 5001; // client should try to download from source
|
||||
SchedPeerGone = 5002; // client should disconnect from scheduler
|
||||
SchedPeerNotFound = 5004; // peer not found in scheduler
|
||||
SchedPeerPieceResultReportFail = 5005; // report piece
|
||||
SchedTaskStatusError = 5006; // task status is fail
|
||||
|
||||
// cdnsystem response error 6000-6999
|
||||
CDNTaskRegistryFail = 6001;
|
||||
CDNTaskNotFound = 6404;
|
||||
|
||||
// manager response error 7000-7999
|
||||
InvalidResourceType = 7001;
|
||||
}
|
||||
|
||||
enum PieceStyle{
|
||||
PLAIN = 0;
|
||||
}
|
||||
|
||||
enum SizeScope{
|
||||
// size > one piece size
|
||||
NORMAL = 0;
|
||||
// 128 byte < size <= one piece size and be plain type
|
||||
SMALL = 1;
|
||||
// size <= 128 byte and be plain type
|
||||
TINY = 2;
|
||||
}
|
||||
|
||||
// Pattern represents pattern of task.
|
||||
enum Pattern{
|
||||
// Default pattern, scheduler will use all p2p node
|
||||
// include dfdaemon and seed peers.
|
||||
P2P = 0;
|
||||
|
||||
// Seed peer pattern, scheduler will use only seed peers.
|
||||
SEED_PEER = 1;
|
||||
|
||||
// Source pattern, scheduler will say back source
|
||||
// when there is no available peer in p2p.
|
||||
SOURCE = 2;
|
||||
}
|
||||
|
||||
// TaskType represents type of task.
|
||||
enum TaskType{
|
||||
// Normal is normal type of task,
|
||||
// normal task is a normal p2p task.
|
||||
Normal = 0;
|
||||
|
||||
// DfCache is dfcache type of task,
|
||||
// dfcache task is a cache task, and the task url is fake url.
|
||||
// It can only be used for caching and cannot be downloaded back to source.
|
||||
DfCache = 1;
|
||||
|
||||
// DfStore is dfstore type of task,
|
||||
// dfstore task is a persistent task in backend.
|
||||
DfStore = 2;
|
||||
}
|
||||
|
||||
message GrpcDfError {
|
||||
Code code = 1;
|
||||
string message = 2;
|
||||
}
|
||||
|
||||
// UrlMeta describes url meta info.
|
||||
message UrlMeta{
|
||||
// digest checks integrity of url content, for example md5:xxx or sha256:yyy
|
||||
string digest = 1 [(validate.rules).string = {pattern: "^(md5)|(sha256):[A-Fa-f0-9]+$", ignore_empty:true}];
|
||||
// url tag identifies different task for same url, conflict with digest
|
||||
string tag = 2;
|
||||
// content range for url
|
||||
string range = 3 [(validate.rules).string = {pattern: "^[0-9]+-[0-9]*$", ignore_empty:true}];
|
||||
// filter url used to generate task id
|
||||
string filter = 4;
|
||||
// other url header infos
|
||||
map<string, string> header = 5;
|
||||
}
|
||||
|
||||
message HostLoad{
|
||||
// cpu usage
|
||||
float cpu_ratio = 1 [(validate.rules).float = {gte: 0, lte: 1}];
|
||||
// memory usage
|
||||
float mem_ratio = 2 [(validate.rules).float = {gte: 0, lte: 1}];
|
||||
// disk space usage
|
||||
float disk_ratio = 3 [(validate.rules).float = {gte: 0, lte: 1}];
|
||||
}
|
||||
|
||||
message PieceTaskRequest{
|
||||
string task_id = 1 [(validate.rules).string.min_len = 1];
|
||||
string src_pid = 2 [(validate.rules).string.min_len = 1];
|
||||
string dst_pid = 3 [(validate.rules).string.min_len = 1];
|
||||
// piece number
|
||||
uint32 start_num = 4 [(validate.rules).uint32.gte = 0];
|
||||
// expected piece count, limit = 0 represent request pieces as many shards as possible
|
||||
uint32 limit = 5 [(validate.rules).uint32.gte = 0];
|
||||
}
|
||||
|
||||
message PieceInfo{
|
||||
// piece_num < 0 represent start report piece flag
|
||||
int32 piece_num = 1;
|
||||
uint64 range_start = 2 [(validate.rules).uint64.gte = 0];
|
||||
uint32 range_size = 3 [(validate.rules).uint32.gte = 0];
|
||||
string piece_md5 = 4 [(validate.rules).string = {pattern:"([a-f\\d]{32}|[A-F\\d]{32}|[a-f\\d]{16}|[A-F\\d]{16})", ignore_empty:true}];
|
||||
uint64 piece_offset = 5 [(validate.rules).uint64.gte = 0];
|
||||
base.PieceStyle piece_style = 6;
|
||||
// total time(millisecond) consumed
|
||||
uint64 download_cost = 7 [(validate.rules).uint64.gte = 0];
|
||||
}
|
||||
|
||||
message ExtendAttribute{
|
||||
// task response header, eg: HTTP Response Header
|
||||
map<string, string> header = 1;
|
||||
// task response code, eg: HTTP Status Code
|
||||
int32 status_code = 2;
|
||||
// task response status, eg: HTTP Status
|
||||
string status = 3;
|
||||
}
|
||||
|
||||
message PiecePacket{
|
||||
string task_id = 2 [(validate.rules).string.min_len = 1];
|
||||
string dst_pid = 3 [(validate.rules).string.min_len = 1];
|
||||
// ip:port
|
||||
string dst_addr = 4 [(validate.rules).string.min_len = 1];
|
||||
repeated PieceInfo piece_infos = 5;
|
||||
// total piece count for url, total_piece represent total piece is unknown
|
||||
int32 total_piece = 6;
|
||||
// content_length < 0 represent content length is unknown
|
||||
int64 content_length = 7;
|
||||
// sha256 code of all piece md5
|
||||
string piece_md5_sign = 8;
|
||||
// task extend attribute
|
||||
ExtendAttribute extend_attribute = 9;
|
||||
}
|
||||
|
|
@ -0,0 +1,167 @@
|
|||
/*
|
||||
* Copyright 2022 The Dragonfly Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
syntax = "proto3";
|
||||
|
||||
package dfdaemon;
|
||||
|
||||
import "pkg/apis/common/v1/common.proto";
|
||||
import "google/protobuf/empty.proto";
|
||||
import "validate/validate.proto";
|
||||
|
||||
option go_package = "d7y.io/api/pkg/apis/dfdaemon/v1";
|
||||
|
||||
message SeedRequest{
|
||||
string task_id = 1 [(validate.rules).string.min_len = 1];
|
||||
string url = 2 [(validate.rules).string.uri = true];
|
||||
common.UrlMeta url_meta = 3;
|
||||
}
|
||||
|
||||
// keep piece meta and data separately
|
||||
// check piece md5, md5s sign and total content length
|
||||
message PieceSeed{
|
||||
// peer id for cdn node, need suffix with _CDN
|
||||
string peer_id = 2 [(validate.rules).string.min_len = 1];
|
||||
// cdn host id
|
||||
string host_id = 3 [(validate.rules).string.min_len = 1];
|
||||
common.PieceInfo piece_info = 4;
|
||||
|
||||
// whether or not all seeds are downloaded
|
||||
bool done = 5;
|
||||
// content total length for the url, content_length < 0 represent content length is unknown
|
||||
int64 content_length = 6;
|
||||
// total piece count, -1 represents task is downloading or failed
|
||||
int32 total_piece_count = 7;
|
||||
// begin time for the piece downloading
|
||||
uint64 begin_time = 8;
|
||||
// end time for the piece downloading
|
||||
uint64 end_time = 9;
|
||||
// task extend attribute
|
||||
common.ExtendAttribute extend_attribute = 10;
|
||||
}
|
||||
|
||||
message DownRequest{
|
||||
// Identify one downloading, the framework will fill it automatically.
|
||||
string uuid = 1 [(validate.rules).string.uuid = true];
|
||||
// Download file from the url, not only for http.
|
||||
string url = 2 [(validate.rules).string.uri = true];
|
||||
// Pieces will be written to output path directly,
|
||||
// at the same time, dfdaemon workspace also makes soft link to the output.
|
||||
string output = 3 [(validate.rules).string.min_len = 1];
|
||||
// Timeout duration.
|
||||
uint64 timeout = 4 [(validate.rules).uint64.gte = 0];
|
||||
// Rate limit in bytes per second.
|
||||
double limit = 5 [(validate.rules).double.gte = 0];
|
||||
// Disable back-to-source.
|
||||
bool disable_back_source = 6;
|
||||
// URL meta info.
|
||||
common.UrlMeta url_meta = 7;
|
||||
// Pattern has p2p/seed-peer/source, default is p2p.
|
||||
string pattern = 8 [(validate.rules).string = {in:["p2p", "seed-peer", "source"], ignore_empty:true}];
|
||||
// Call system.
|
||||
string callsystem = 9;
|
||||
// User id.
|
||||
int64 uid = 10;
|
||||
// Group id.
|
||||
int64 gid = 11;
|
||||
// Keep original offset, used for ranged request, only available for hard link, otherwise will failed.
|
||||
bool keep_original_offset = 12;
|
||||
}
|
||||
|
||||
message DownResult{
|
||||
// Task id.
|
||||
string task_id = 2 [(validate.rules).string.min_len = 1];
|
||||
// Peer id.
|
||||
string peer_id = 3 [(validate.rules).string.min_len = 1];
|
||||
// Task has completed length.
|
||||
uint64 completed_length = 4 [(validate.rules).uint64.gte = 0];
|
||||
// Task has been completed.
|
||||
bool done = 5;
|
||||
}
|
||||
|
||||
message StatTaskRequest{
|
||||
// Download url.
|
||||
string url = 1 [(validate.rules).string.min_len = 1];
|
||||
// URL meta info.
|
||||
common.UrlMeta url_meta = 2;
|
||||
// Check local cache only.
|
||||
bool local_only = 3;
|
||||
}
|
||||
|
||||
message ImportTaskRequest{
|
||||
// Download url.
|
||||
string url = 1 [(validate.rules).string.min_len = 1];
|
||||
// URL meta info.
|
||||
common.UrlMeta url_meta = 2;
|
||||
// File to be imported.
|
||||
string path = 3 [(validate.rules).string.min_len = 1];
|
||||
// Task type.
|
||||
common.TaskType type = 4;
|
||||
}
|
||||
|
||||
message ExportTaskRequest{
|
||||
// Download url.
|
||||
string url = 1 [(validate.rules).string.min_len = 1];
|
||||
// Output path of downloaded file.
|
||||
string output = 2 [(validate.rules).string.min_len = 1];
|
||||
// Timeout duration.
|
||||
uint64 timeout = 3 [(validate.rules).uint64.gte = 0];
|
||||
// Rate limit in bytes per second.
|
||||
double limit = 4 [(validate.rules).double.gte = 0];
|
||||
// URL meta info.
|
||||
common.UrlMeta url_meta = 5;
|
||||
// Call system.
|
||||
string callsystem = 6;
|
||||
// User id.
|
||||
int64 uid = 7;
|
||||
// Group id.
|
||||
int64 gid = 8;
|
||||
// Only export from local storage.
|
||||
bool local_only = 9;
|
||||
}
|
||||
|
||||
message DeleteTaskRequest{
|
||||
// Download url.
|
||||
string url = 1 [(validate.rules).string.min_len = 1];
|
||||
// URL meta info.
|
||||
common.UrlMeta url_meta = 2;
|
||||
}
|
||||
|
||||
// Daemon Client RPC Service
|
||||
service Daemon{
|
||||
// Trigger client to download file
|
||||
rpc Download(DownRequest) returns(stream DownResult);
|
||||
// Get piece tasks from other peers
|
||||
rpc GetPieceTasks(common.PieceTaskRequest)returns(common.PiecePacket);
|
||||
// Check daemon health
|
||||
rpc CheckHealth(google.protobuf.Empty)returns(google.protobuf.Empty);
|
||||
// Sync piece tasks with other peers
|
||||
rpc SyncPieceTasks(stream common.PieceTaskRequest)returns(stream common.PiecePacket);
|
||||
// Check if given task exists in P2P cache system
|
||||
rpc StatTask(StatTaskRequest) returns(google.protobuf.Empty);
|
||||
// Import the given file into P2P cache system
|
||||
rpc ImportTask(ImportTaskRequest) returns(google.protobuf.Empty);
|
||||
// Export or download file from P2P cache system
|
||||
rpc ExportTask(ExportTaskRequest) returns(google.protobuf.Empty);
|
||||
// Delete file from P2P cache system
|
||||
rpc DeleteTask(DeleteTaskRequest) returns(google.protobuf.Empty);
|
||||
// Generate seeds and return to scheduler
|
||||
rpc ObtainSeeds(SeedRequest)returns(stream PieceSeed);
|
||||
// Get piece tasks from cdn
|
||||
rpc GetPieceTasks(common.PieceTaskRequest)returns(common.PiecePacket);
|
||||
// Sync piece tasks with other peers
|
||||
rpc SyncPieceTasks(stream common.PieceTaskRequest)returns(stream common.PiecePacket);
|
||||
}
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
/*
|
||||
* Copyright 2022 The Dragonfly Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
syntax = "proto3";
|
||||
|
||||
package errordetails;
|
||||
|
||||
import "pkg/apis/common/v1/common.proto";
|
||||
|
||||
option go_package = "d7y.io/api/pkg/apis/errordetails/v1";
|
||||
|
||||
message SourceError {
|
||||
bool temporary = 1;
|
||||
// source response metadata, eg: HTTP Status Code, HTTP Status, HTTP Header
|
||||
common.ExtendAttribute metadata = 2;
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,321 @@
|
|||
/*
|
||||
* Copyright 2022 The Dragonfly Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
syntax = "proto3";
|
||||
|
||||
package manager;
|
||||
|
||||
import "google/protobuf/empty.proto";
|
||||
import "validate/validate.proto";
|
||||
|
||||
option go_package = "d7y.io/api/pkg/apis/manager/v1";
|
||||
|
||||
// Request source type.
|
||||
enum SourceType {
|
||||
// Scheduler service.
|
||||
SCHEDULER_SOURCE = 0;
|
||||
// Peer service.
|
||||
PEER_SOURCE = 1;
|
||||
// SeedPeer service.
|
||||
SEED_PEER_SOURCE = 2;
|
||||
}
|
||||
|
||||
// SecurityGroup represents security group of cluster.
|
||||
message SecurityGroup {
|
||||
// Group id.
|
||||
uint64 id = 1;
|
||||
// Group name.
|
||||
string name = 2;
|
||||
// Group biography.
|
||||
string bio = 3;
|
||||
// Group domain.
|
||||
string domain = 4;
|
||||
// Group proxy domain.
|
||||
string proxy_domain = 5;
|
||||
}
|
||||
|
||||
// SeedPeerCluster represents cluster of seed peer.
|
||||
message SeedPeerCluster {
|
||||
// Cluster id.
|
||||
uint64 id = 1;
|
||||
// Cluster name.
|
||||
string name = 2;
|
||||
// Cluster biography.
|
||||
string bio = 3;
|
||||
// Cluster configuration.
|
||||
bytes config = 4;
|
||||
// Cluster scopes.
|
||||
bytes scopes = 5;
|
||||
// Security group to which the seed peer cluster belongs.
|
||||
SecurityGroup security_group = 6;
|
||||
}
|
||||
|
||||
// SeedPeer represents seed peer for network.
|
||||
message SeedPeer {
|
||||
// Seed peer id.
|
||||
uint64 id = 1;
|
||||
// Seed peer hostname.
|
||||
string host_name = 2;
|
||||
// Seed peer type.
|
||||
string type = 3;
|
||||
// Seed peer idc.
|
||||
string idc = 5;
|
||||
// Seed peer network topology.
|
||||
string net_topology = 6;
|
||||
// Seed peer location.
|
||||
string location = 7;
|
||||
// Seed peer ip.
|
||||
string ip = 8;
|
||||
// Seed peer grpc port.
|
||||
int32 port = 9;
|
||||
// Seed peer download port.
|
||||
int32 download_port = 10;
|
||||
// Seed peer state.
|
||||
string state = 11;
|
||||
// ID of the cluster to which the seed peer belongs.
|
||||
uint64 seed_peer_cluster_id = 12;
|
||||
// Cluster to which the seed peer belongs.
|
||||
SeedPeerCluster seed_peer_cluster = 13;
|
||||
// Schedulers included in seed peer.
|
||||
repeated Scheduler schedulers = 14;
|
||||
// Seed peer object storage port.
|
||||
int32 object_storage_port = 15;
|
||||
}
|
||||
|
||||
// GetSeedPeerRequest represents request of GetSeedPeer.
|
||||
message GetSeedPeerRequest {
|
||||
// Request source type.
|
||||
SourceType source_type = 1 [(validate.rules).enum.defined_only = true];
|
||||
// Seed peer hostname.
|
||||
string host_name = 2 [(validate.rules).string.hostname = true];
|
||||
// ID of the cluster to which the seed peer belongs.
|
||||
uint64 seed_peer_cluster_id = 3 [(validate.rules).uint64 = {gte: 1}];
|
||||
// Seed peer ip.
|
||||
string ip = 4 [(validate.rules).string = {ip: true, ignore_empty: true}];
|
||||
}
|
||||
|
||||
// UpdateSeedPeerRequest represents request of UpdateSeedPeer.
|
||||
message UpdateSeedPeerRequest {
|
||||
// Request source type.
|
||||
SourceType source_type = 1 [(validate.rules).enum.defined_only = true];
|
||||
// Seed peer hostname.
|
||||
string host_name = 2 [(validate.rules).string.hostname = true];
|
||||
// Seed peer type.
|
||||
string type = 3 [(validate.rules).string = {in: ["super", "strong", "weak"]}];
|
||||
// Seed peer idc.
|
||||
string idc = 5 [(validate.rules).string = {min_len: 1, max_len: 1024, ignore_empty: true}];
|
||||
// Seed peer network topology.
|
||||
string net_topology = 6 [(validate.rules).string = {min_len: 1, max_len: 1024, ignore_empty: true}];
|
||||
// Seed peer location.
|
||||
string location = 7 [(validate.rules).string = {max_len: 1024, ignore_empty: true}];
|
||||
// Seed peer ip.
|
||||
string ip = 8 [(validate.rules).string = {ip: true}];
|
||||
// Seed peer port.
|
||||
int32 port = 9 [(validate.rules).int32 = {gte: 1024, lt: 65535}];
|
||||
// Seed peer download port.
|
||||
int32 download_port = 10 [(validate.rules).int32 = {gte: 1024, lt: 65535}];
|
||||
// ID of the cluster to which the seed peer belongs.
|
||||
uint64 seed_peer_cluster_id = 11 [(validate.rules).uint64 = {gte: 1}];
|
||||
// Seed peer object storage port.
|
||||
int32 object_storage_port = 12 [(validate.rules).int32 = {gte: 1024, lt: 65535, ignore_empty: true}];
|
||||
}
|
||||
|
||||
// SeedPeerCluster represents cluster of scheduler.
|
||||
message SchedulerCluster {
|
||||
// Cluster id.
|
||||
uint64 id = 1;
|
||||
// Cluster name.
|
||||
string name = 2;
|
||||
// Cluster biography.
|
||||
string bio = 3;
|
||||
// Cluster config.
|
||||
bytes config = 4;
|
||||
// Cluster client config.
|
||||
bytes client_config = 5;
|
||||
// Cluster scopes.
|
||||
bytes scopes = 6;
|
||||
// Security group to which the scheduler cluster belongs.
|
||||
SecurityGroup security_group = 7;
|
||||
}
|
||||
|
||||
// SeedPeerCluster represents scheduler for network.
|
||||
message Scheduler {
|
||||
// Scheduler id.
|
||||
uint64 id = 1;
|
||||
// Scheduler hostname.
|
||||
string host_name = 2;
|
||||
// Deprecated: Do not use.
|
||||
string vips = 3;
|
||||
// Scheduler idc.
|
||||
string idc = 4;
|
||||
// Scheduler location.
|
||||
string location = 5;
|
||||
// Deprecated: Use net_topology instead.
|
||||
bytes net_config = 6;
|
||||
// Scheduler ip.
|
||||
string ip = 7;
|
||||
// Scheduler grpc port.
|
||||
int32 port = 8;
|
||||
// Scheduler state.
|
||||
string state = 9;
|
||||
// ID of the cluster to which the scheduler belongs.
|
||||
uint64 scheduler_cluster_id = 10;
|
||||
// Cluster to which the scheduler belongs.
|
||||
SchedulerCluster scheduler_cluster = 11;
|
||||
// Seed peers to which the scheduler belongs.
|
||||
repeated SeedPeer seed_peers = 13;
|
||||
// Scheduler network topology.
|
||||
string net_topology = 14;
|
||||
}
|
||||
|
||||
// GetSchedulerRequest represents request of GetScheduler.
|
||||
message GetSchedulerRequest {
|
||||
// Request source type.
|
||||
SourceType source_type = 1 [(validate.rules).enum.defined_only = true];
|
||||
// Scheduler hostname.
|
||||
string host_name = 2 [(validate.rules).string.hostname = true];
|
||||
// ID of the cluster to which the scheduler belongs.
|
||||
uint64 scheduler_cluster_id = 3 [(validate.rules).uint64 = {gte: 1}];
|
||||
// Scheduler ip.
|
||||
string ip = 4 [(validate.rules).string = {ip: true, ignore_empty: true}];
|
||||
}
|
||||
|
||||
// UpdateSchedulerRequest represents request of UpdateScheduler.
|
||||
message UpdateSchedulerRequest {
|
||||
// Request source type.
|
||||
SourceType source_type = 1 [(validate.rules).enum.defined_only = true];
|
||||
// Scheduler hostname.
|
||||
string host_name = 2 [(validate.rules).string.hostname = true];
|
||||
// ID of the cluster to which the scheduler belongs.
|
||||
uint64 scheduler_cluster_id = 3 [(validate.rules).uint64 = {gte: 1}];
|
||||
// Deprecated: Do not use.
|
||||
string vips = 4 [(validate.rules).string = {min_len: 1, max_len: 1024, ignore_empty: true}];
|
||||
// Scheduler idc.
|
||||
string idc = 5 [(validate.rules).string = {min_len: 1, max_len: 1024, ignore_empty: true}];
|
||||
// Scheduler location.
|
||||
string location = 6 [(validate.rules).string = {min_len: 1, max_len: 1024, ignore_empty: true}];
|
||||
// Deprecated: Use net_topology instead.
|
||||
bytes net_config = 7 [(validate.rules).bytes = {min_len: 1, ignore_empty: true}];
|
||||
// Scheduler ip.
|
||||
string ip = 8 [(validate.rules).string = {ip: true}];
|
||||
// Scheduler port.
|
||||
int32 port = 9 [(validate.rules).int32 = {gte: 1024, lt: 65535}];
|
||||
// Scheduler network topology.
|
||||
string net_topology = 10 [(validate.rules).string = {min_len: 1, max_len: 1024, ignore_empty: true}];
|
||||
}
|
||||
|
||||
// ListSchedulersRequest represents request of ListSchedulers.
|
||||
message ListSchedulersRequest {
|
||||
// Request source type.
|
||||
SourceType source_type = 1 [(validate.rules).enum.defined_only = true];
|
||||
// Source service hostname.
|
||||
string host_name = 2 [(validate.rules).string.hostname = true];
|
||||
// Source service ip.
|
||||
string ip = 3 [(validate.rules).string.ip = true];
|
||||
// Source service host information.
|
||||
map<string, string> host_info = 5 [(validate.rules).map.ignore_empty = true];
|
||||
}
|
||||
|
||||
// ListSchedulersResponse represents response of ListSchedulers.
|
||||
message ListSchedulersResponse {
|
||||
// Schedulers to which the source service belongs.
|
||||
repeated Scheduler schedulers = 1;
|
||||
}
|
||||
|
||||
// ObjectStorage represents config of object storage.
|
||||
message ObjectStorage {
|
||||
// Object storage name of type.
|
||||
string name = 1 [(validate.rules).string = {min_len: 1, max_len: 1024}];
|
||||
// Storage region.
|
||||
string region = 2 [(validate.rules).string = {min_len: 1, max_len: 1024, ignore_empty: true}];
|
||||
// Datacenter endpoint.
|
||||
string endpoint = 3 [(validate.rules).string = {min_len: 1, max_len: 1024, ignore_empty: true}];
|
||||
// Access key id.
|
||||
string access_key = 4 [(validate.rules).string = {min_len: 1, max_len: 1024, ignore_empty: true}];
|
||||
// Access key secret.
|
||||
string secret_key = 5 [(validate.rules).string = {min_len: 1, max_len: 1024, ignore_empty: true}];
|
||||
}
|
||||
|
||||
// GetObjectStorageRequest represents request of GetObjectStorage.
|
||||
message GetObjectStorageRequest {
|
||||
// Request source type.
|
||||
SourceType source_type = 1 [(validate.rules).enum.defined_only = true];
|
||||
// Source service hostname.
|
||||
string host_name = 2 [(validate.rules).string.hostname = true];
|
||||
// Source service ip.
|
||||
string ip = 3 [(validate.rules).string.ip = true];
|
||||
}
|
||||
|
||||
// Bucket represents config of bucket.
|
||||
message Bucket {
|
||||
// Bucket name.
|
||||
string name = 1 [(validate.rules).string = {min_len: 1, max_len: 1024}];
|
||||
}
|
||||
|
||||
// ListSchedulersRequest represents request of ListBuckets.
|
||||
message ListBucketsRequest {
|
||||
// Request source type.
|
||||
SourceType source_type = 1 [(validate.rules).enum.defined_only = true];
|
||||
// Source service hostname.
|
||||
string host_name = 2 [(validate.rules).string.hostname = true];
|
||||
// Source service ip.
|
||||
string ip = 3 [(validate.rules).string.ip = true];
|
||||
}
|
||||
|
||||
// ListBucketsResponse represents response of ListBuckets.
|
||||
message ListBucketsResponse {
|
||||
// Bucket configs.
|
||||
repeated Bucket buckets = 1;
|
||||
}
|
||||
|
||||
// KeepAliveRequest represents request of KeepAlive.
|
||||
message KeepAliveRequest {
|
||||
// Request source type.
|
||||
SourceType source_type = 1 [(validate.rules).enum.defined_only = true];
|
||||
// Source service hostname.
|
||||
string host_name = 2 [(validate.rules).string.hostname = true];
|
||||
// ID of the cluster to which the source service belongs.
|
||||
uint64 cluster_id = 3 [(validate.rules).uint64 = {gte: 1}];
|
||||
// Source service ip.
|
||||
string ip = 4 [(validate.rules).string = {ip: true, ignore_empty: true}];
|
||||
}
|
||||
|
||||
// Manager RPC Service.
|
||||
service Manager {
|
||||
// Get SeedPeer and SeedPeer cluster configuration.
|
||||
rpc GetSeedPeer(GetSeedPeerRequest) returns(SeedPeer);
|
||||
|
||||
// Update SeedPeer configuration.
|
||||
rpc UpdateSeedPeer(UpdateSeedPeerRequest) returns(SeedPeer);
|
||||
|
||||
// Get Scheduler and Scheduler cluster configuration.
|
||||
rpc GetScheduler(GetSchedulerRequest)returns(Scheduler);
|
||||
|
||||
// Update scheduler configuration.
|
||||
rpc UpdateScheduler(UpdateSchedulerRequest) returns(Scheduler);
|
||||
|
||||
// List acitve schedulers configuration.
|
||||
rpc ListSchedulers(ListSchedulersRequest)returns(ListSchedulersResponse);
|
||||
|
||||
// Get ObjectStorage configuration.
|
||||
rpc GetObjectStorage(GetObjectStorageRequest) returns(ObjectStorage);
|
||||
|
||||
// List buckets configuration.
|
||||
rpc ListBuckets(ListBucketsRequest)returns(ListBucketsResponse);
|
||||
|
||||
// KeepAlive with manager.
|
||||
rpc KeepAlive(stream KeepAliveRequest)returns(google.protobuf.Empty);
|
||||
}
|
||||
|
|
@ -0,0 +1,257 @@
|
|||
/*
|
||||
* Copyright 2022 The Dragonfly Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
syntax = "proto3";
|
||||
|
||||
package scheduler;
|
||||
|
||||
import "pkg/apis/common/v1/common.proto";
|
||||
import "pkg/apis/errordetails/v1/errordetails.proto";
|
||||
import "validate/validate.proto";
|
||||
import "google/protobuf/empty.proto";
|
||||
|
||||
option go_package = "d7y.io/api/pkg/apis/scheduler/v1";
|
||||
|
||||
// PeerTaskRequest represents request of RegisterPeerTask.
|
||||
message PeerTaskRequest{
|
||||
// Download url.
|
||||
string url = 1 [(validate.rules).string.uri = true];
|
||||
// URL meta info.
|
||||
common.UrlMeta url_meta = 2 [(validate.rules).message.required = true];
|
||||
// Peer id and it must be global uniqueness.
|
||||
string peer_id = 3 [(validate.rules).string.min_len = 1];
|
||||
// Peer host info.
|
||||
PeerHost peer_host = 4;
|
||||
// Peer host load.
|
||||
common.HostLoad host_load = 5;
|
||||
// Whether this request is caused by migration.
|
||||
bool is_migrating = 6;
|
||||
// Pattern includes p2p, seed-peer and source.
|
||||
common.Pattern pattern = 7;
|
||||
// Task id.
|
||||
string task_id = 8;
|
||||
}
|
||||
|
||||
// RegisterResult represents response of RegisterPeerTask.
|
||||
message RegisterResult{
|
||||
// Task type.
|
||||
common.TaskType task_type = 1;
|
||||
// Task id
|
||||
string task_id = 2 [(validate.rules).string.min_len = 1];
|
||||
// File size scope.
|
||||
common.SizeScope size_scope = 3 [(validate.rules).enum.defined_only = true];
|
||||
// Download the only piece directly for small or tiny file.
|
||||
oneof direct_piece{
|
||||
// Return single piece info when size scope is small.
|
||||
SinglePiece single_piece = 4;
|
||||
// Return task content when size scope is tiny.
|
||||
bytes piece_content = 5;
|
||||
}
|
||||
// Task extend attribute,
|
||||
// only direct_piece will carry extend attribute.
|
||||
common.ExtendAttribute extend_attribute = 6;
|
||||
}
|
||||
|
||||
// SinglePiece represents information of single piece.
|
||||
message SinglePiece{
|
||||
// Destination peer id.
|
||||
string dst_pid = 1 [(validate.rules).string.min_len = 1];
|
||||
// Destination download address.
|
||||
string dst_addr = 2 [(validate.rules).string.min_len = 1];
|
||||
// Piece info.
|
||||
common.PieceInfo piece_info = 3;
|
||||
}
|
||||
|
||||
// PeerHost represents information of peer host.
|
||||
message PeerHost{
|
||||
// Peer host id.
|
||||
string id = 1 [(validate.rules).string.min_len = 1];
|
||||
// peer host ip
|
||||
string ip = 2 [(validate.rules).string.ip = true];
|
||||
// Port of grpc service.
|
||||
int32 rpc_port = 3 [(validate.rules).int32 = {gte: 1024, lt: 65535}];
|
||||
// Port of download server.
|
||||
int32 down_port = 4 [(validate.rules).int32 = {gte: 1024, lt: 65535}];
|
||||
// Peer hostname.
|
||||
string host_name = 5 [(validate.rules).string.hostname = true];
|
||||
// Security domain for network.
|
||||
string security_domain = 6;
|
||||
// Location path(area|country|province|city|...).
|
||||
string location = 7;
|
||||
// IDC where the peer host is located
|
||||
string idc = 8;
|
||||
// Network topology(switch|router|...).
|
||||
string net_topology = 9;
|
||||
}
|
||||
|
||||
// PieceResult represents request of ReportPieceResult.
|
||||
message PieceResult{
|
||||
// Task id.
|
||||
string task_id = 1 [(validate.rules).string.min_len = 1];
|
||||
// Source peer id.
|
||||
string src_pid = 2 [(validate.rules).string.min_len = 1];
|
||||
// Destination peer id.
|
||||
string dst_pid = 3;
|
||||
// Piece info.
|
||||
common.PieceInfo piece_info = 4;
|
||||
// Begin time of the piece downloading.
|
||||
uint64 begin_time = 5;
|
||||
// End time of the piece downloading.
|
||||
uint64 end_time = 6;
|
||||
// Whether the piece downloading is successfully.
|
||||
bool success = 7;
|
||||
// Result code.
|
||||
common.Code code = 8;
|
||||
// Peer host load.
|
||||
common.HostLoad host_load = 9;
|
||||
// Finished count.
|
||||
int32 finished_count = 10;
|
||||
// Task extend attribute,
|
||||
// only first success back source piece will carry extend attribute.
|
||||
common.ExtendAttribute extend_attribute = 11;
|
||||
}
|
||||
|
||||
// PeerPacket represents response of ReportPieceResult.
|
||||
message PeerPacket{
|
||||
message DestPeer{
|
||||
// Destination ip.
|
||||
string ip = 1 [(validate.rules).string.ip = true];
|
||||
// Port of grpc service.
|
||||
int32 rpc_port = 2 [(validate.rules).int32 = {gte: 1024, lt: 65535}];
|
||||
// Destination peer id.
|
||||
string peer_id = 3 [(validate.rules).string.min_len = 1];
|
||||
}
|
||||
|
||||
// Task id.
|
||||
string task_id = 2 [(validate.rules).string.min_len = 1];
|
||||
// Source peer id.
|
||||
string src_pid = 3 [(validate.rules).string.min_len = 1];
|
||||
// Concurrent downloading count from main peer.
|
||||
int32 parallel_count = 4 [(validate.rules).int32.gte = 1];
|
||||
// Main peer.
|
||||
DestPeer main_peer = 5;
|
||||
// Candidate peers.
|
||||
repeated DestPeer candidate_peers = 6;
|
||||
// Result code.
|
||||
common.Code code = 7;
|
||||
// Error detail.
|
||||
oneof errordetails{
|
||||
// Source error.
|
||||
errordetails.SourceError source_error = 8;
|
||||
}
|
||||
}
|
||||
|
||||
// PeerResult represents response of ReportPeerResult.
|
||||
message PeerResult{
|
||||
// Task id.
|
||||
string task_id = 1 [(validate.rules).string.min_len = 1];
|
||||
// Peer id.
|
||||
string peer_id = 2 [(validate.rules).string.min_len = 1];
|
||||
// Source host ip.
|
||||
string src_ip = 3 [(validate.rules).string.ip = true];
|
||||
// Security domain.
|
||||
string security_domain = 4;
|
||||
// IDC where the peer host is located
|
||||
string idc = 5;
|
||||
// Download url.
|
||||
string url = 6 [(validate.rules).string.uri = true];
|
||||
// Total content length.
|
||||
int64 content_length = 7 [(validate.rules).int64.gte = -1];
|
||||
// Total network traffic.
|
||||
uint64 traffic = 8;
|
||||
// Total cost time.
|
||||
uint32 cost = 9;
|
||||
// Whether peer downloading file is successfully.
|
||||
bool success = 10;
|
||||
// Result code.
|
||||
common.Code code = 11;
|
||||
// Task total piece count.
|
||||
int32 total_piece_count = 12 [(validate.rules).int32.gte = -1];
|
||||
// Error detail.
|
||||
oneof errordetails{
|
||||
// Source error.
|
||||
errordetails.SourceError source_error = 13;
|
||||
}
|
||||
}
|
||||
|
||||
// PeerTarget represents request of LeaveTask.
|
||||
message PeerTarget{
|
||||
// Task id.
|
||||
string task_id = 1 [(validate.rules).string.min_len = 1];
|
||||
// Peer id.
|
||||
string peer_id = 2 [(validate.rules).string.min_len = 1];
|
||||
}
|
||||
|
||||
// StatTaskRequest represents request of StatTask.
|
||||
message StatTaskRequest{
|
||||
// Task id.
|
||||
string task_id = 1 [(validate.rules).string.min_len = 1];
|
||||
}
|
||||
|
||||
// Task represents download task.
|
||||
message Task{
|
||||
// Task id.
|
||||
string id = 1 [(validate.rules).string.min_len = 1];
|
||||
// Task type.
|
||||
common.TaskType type = 2;
|
||||
// Task content length.
|
||||
int64 content_length = 3 [(validate.rules).int64.gte = 1];
|
||||
// Task total piece count.
|
||||
int32 total_piece_count = 4 [(validate.rules).int32.gte = 1];
|
||||
// Task state.
|
||||
string state = 5 [(validate.rules).string.min_len = 1];
|
||||
// Task peer count.
|
||||
int32 peer_count = 6 [(validate.rules).int32.gte = 0];
|
||||
// Task contains available peer.
|
||||
bool hasAvailablePeer = 7;
|
||||
}
|
||||
|
||||
// AnnounceTaskRequest represents request of AnnounceTask.
|
||||
message AnnounceTaskRequest{
|
||||
// Task id.
|
||||
string task_id = 1 [(validate.rules).string.min_len = 1];
|
||||
// Download url.
|
||||
string url = 2 [(validate.rules).string = {uri: true, ignore_empty: true}];
|
||||
// URL meta info.
|
||||
common.UrlMeta url_meta = 3 [(validate.rules).message.required = true];
|
||||
// Peer host info.
|
||||
PeerHost peer_host = 4;
|
||||
// Task piece info.
|
||||
common.PiecePacket piece_packet = 5 [(validate.rules).message.required = true];
|
||||
// Task type.
|
||||
common.TaskType task_type = 6;
|
||||
}
|
||||
|
||||
// Scheduler RPC Service.
|
||||
service Scheduler{
|
||||
// RegisterPeerTask registers a peer into task.
|
||||
rpc RegisterPeerTask(PeerTaskRequest)returns(RegisterResult);
|
||||
|
||||
// ReportPieceResult reports piece results and receives peer packets.
|
||||
rpc ReportPieceResult(stream PieceResult)returns(stream PeerPacket);
|
||||
|
||||
// ReportPeerResult reports downloading result for the peer.
|
||||
rpc ReportPeerResult(PeerResult)returns(google.protobuf.Empty);
|
||||
|
||||
// LeaveTask makes the peer leaving from task.
|
||||
rpc LeaveTask(PeerTarget)returns(google.protobuf.Empty);
|
||||
|
||||
// Checks if any peer has the given task.
|
||||
rpc StatTask(StatTaskRequest)returns(Task);
|
||||
|
||||
// A peer announces that it has the announced task to other peers.
|
||||
rpc AnnounceTask(AnnounceTaskRequest) returns(google.protobuf.Empty);
|
||||
}
|
||||
Loading…
Reference in New Issue