api/pkg/apis/scheduler/v1/scheduler.proto

/*
 *     Copyright 2022 The Dragonfly Authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

syntax = "proto3";

package scheduler;

import "pkg/apis/common/v1/common.proto";
import "pkg/apis/errordetails/v1/errordetails.proto";
import "validate/validate.proto";
import "google/protobuf/empty.proto";

option go_package = "d7y.io/api/v2/pkg/apis/scheduler/v1;scheduler";

// PeerTaskRequest represents request of RegisterPeerTask.
message PeerTaskRequest{
  // Download url.
  string url = 1 [(validate.rules).string.uri = true];
  // URL meta info.
  common.UrlMeta url_meta = 2 [(validate.rules).message.required = true];
  // Peer id and it must be global uniqueness.
  string peer_id = 3 [(validate.rules).string.min_len = 1];
  // Peer host info.
  PeerHost peer_host = 4;
  // When requesting for range, it will trigger seed peer to download entire task.
  bool prefetch = 5;
  // Whether this request is caused by migration.
  bool is_migrating = 6;
  // Task id.
  string task_id = 8;
}

// RegisterResult represents response of RegisterPeerTask.
message RegisterResult{
  // Task type.
  common.TaskType task_type = 1;
  // Task id
  string task_id = 2 [(validate.rules).string.min_len = 1];
  // File size scope.
  common.SizeScope size_scope = 3;
  // Download the only piece directly for small or tiny file.
  oneof direct_piece{
    // Return single piece info when size scope is small.
    SinglePiece single_piece = 4;
    // Return task content when size scope is tiny.
    bytes piece_content = 5;
  }
  // Task extend attribute,
  // only direct_piece will carry extend attribute.
  common.ExtendAttribute extend_attribute = 6;
}

// SinglePiece represents information of single piece.
message SinglePiece{
  // Destination peer id.
  string dst_pid = 1 [(validate.rules).string.min_len = 1];
  // Destination download address.
  string dst_addr = 2 [(validate.rules).string.min_len = 1];
  // Piece info.
  common.PieceInfo piece_info = 3;
}

// PeerHost represents information of peer host.
message PeerHost{
  // Peer host id.
  string id = 1 [(validate.rules).string.min_len = 1];
  // peer host ip
  string ip = 2 [(validate.rules).string.ip = true];
  // Port of grpc service.
  int32 rpc_port = 3 [(validate.rules).int32 = {gte: 1024, lt: 65535}];
  // Port of download server.
  int32 down_port = 4 [(validate.rules).int32 = {gte: 1024, lt: 65535}];
  // Peer hostname.
  string hostname = 5 [(validate.rules).string.hostname = true];
  // Location path(area|country|province|city|...).
  string location = 7;
  // IDC where the peer host is located
  string idc = 8;
}

// PieceResult represents request of ReportPieceResult.
message PieceResult{
  // Task id.
  string task_id = 1 [(validate.rules).string.min_len = 1];
  // Source peer id.
  string src_pid = 2 [(validate.rules).string.min_len = 1];
  // Destination peer id.
  string dst_pid = 3;
  // Piece info.
  common.PieceInfo piece_info = 4;
  // Begin time of the piece downloading.
  uint64 begin_time = 5;
  // End time of the piece downloading.
  uint64 end_time = 6;
  // Whether the piece downloading is successfully.
  bool success = 7;
  // Result code.
  common.Code code = 8;
  // Finished count.
  int32 finished_count = 10;
  // Task extend attribute,
  // only first success back source piece will carry extend attribute.
  common.ExtendAttribute extend_attribute = 11;
}

// PeerPacket represents response of ReportPieceResult.
message PeerPacket{
  message DestPeer{
    // Destination ip.
    string ip = 1 [(validate.rules).string.ip = true];
    // Port of grpc service.
    int32 rpc_port = 2 [(validate.rules).int32 = {gte: 1024, lt: 65535}];
    // Destination peer id.
    string peer_id = 3 [(validate.rules).string.min_len = 1];
  }

  // Task id.
  string task_id = 2 [(validate.rules).string.min_len = 1];
  // Source peer id.
  string src_pid = 3 [(validate.rules).string.min_len = 1];
  // Main peer.
  DestPeer main_peer = 5;
  // Candidate peers.
  repeated DestPeer candidate_peers = 6;
  // Result code.
  common.Code code = 7;
  // Error detail.
  oneof errordetails{
    // Source error.
    errordetails.SourceError source_error = 8;
  }
}

// PeerResult represents response of ReportPeerResult.
message PeerResult{
  // Task id.
  string task_id = 1 [(validate.rules).string.min_len = 1];
  // Peer id.
  string peer_id = 2 [(validate.rules).string.min_len = 1];
  // Source host ip.
  string src_ip = 3 [(validate.rules).string.ip = true];
  // IDC where the peer host is located
  string idc = 5;
  // Download url.
  string url = 6 [(validate.rules).string.uri = true];
  // Total content length.
  int64 content_length = 7 [(validate.rules).int64.gte = -1];
  // Total network traffic.
  uint64 traffic = 8;
  // Total cost time.
  uint32 cost = 9;
  // Whether peer downloading file is successfully.
  bool success = 10;
  // Result code.
  common.Code code = 11;
  // Task total piece count.
  int32 total_piece_count = 12 [(validate.rules).int32.gte = -1];
  // Error detail.
  oneof errordetails{
    // Source error.
    errordetails.SourceError source_error = 13;
  }
}

// AnnounceTaskRequest represents request of AnnounceTask.
message AnnounceTaskRequest{
  // Task id.
  string task_id = 1 [(validate.rules).string.min_len = 1];
  // Download url.
  string url = 2 [(validate.rules).string = {uri: true, ignore_empty: true}];
  // URL meta info.
  common.UrlMeta url_meta = 3 [(validate.rules).message.required = true];
  // Peer host info.
  PeerHost peer_host = 4;
  // Task piece info.
  common.PiecePacket piece_packet = 5 [(validate.rules).message.required = true];
  // Task type.
  common.TaskType task_type = 6;
}

// StatTaskRequest represents request of StatTask.
message StatTaskRequest{
  // Task id.
  string task_id = 1 [(validate.rules).string.min_len = 1];
}

// Task represents download task.
message Task{
  // Task id.
  string id = 1 [(validate.rules).string.min_len = 1];
  // Task type.
  common.TaskType type = 2;
  // Task content length.
  int64 content_length = 3 [(validate.rules).int64.gte = 1];
  // Task total piece count.
  int32 total_piece_count = 4 [(validate.rules).int32.gte = 1];
  // Task state.
  string state = 5 [(validate.rules).string.min_len = 1];
  // Task peer count.
  int32 peer_count = 6 [(validate.rules).int32.gte = 0];
  // Task contains available peer.
  bool hasAvailablePeer = 7;
}

// PeerTarget represents request of LeaveTask.
message PeerTarget{
  // Task id.
  string task_id = 1 [(validate.rules).string.min_len = 1];
  // Peer id.
  string peer_id = 2 [(validate.rules).string.min_len = 1];
}

// LeaveHostRequest represents request of LeaveHost.
message LeaveHostRequest{
  // Host id.
  string id = 1 [(validate.rules).string.min_len = 1];
}

// AnnounceHostRequest represents request of AnnounceHost.
message AnnounceHostRequest{
  // Host id.
  string id = 1 [(validate.rules).string.min_len = 1];
  // Host type.
  string type = 2 [(validate.rules).string = {in: ["normal", "super", "strong", "weak"]}];
  // Hostname.
  string hostname = 3 [(validate.rules).string.min_len = 1];
  // Host ip.
  string ip = 4 [(validate.rules).string.ip = true];
  // Port of grpc service.
  int32 port = 5 [(validate.rules).int32 = {gte: 1024, lt: 65535}];
  // Port of download server.
  int32 download_port = 6 [(validate.rules).int32 = {gte: 1024, lt: 65535}];
  // Host OS.
  string os = 7;
  // Host platform.
  string platform = 8;
  // Host platform family.
  string platform_family = 9;
  // Host platform version.
  string platform_version = 10;
  // Host kernel version.
  string kernel_version = 11;
  // CPU Stat.
  CPU cpu = 12 [(validate.rules).message.required = true];
  // Memory Stat.
  Memory memory = 13 [(validate.rules).message.required = true];
  // Network Stat.
  Network network = 14 [(validate.rules).message.required = true];
  // Disk Stat.
  Disk disk = 15 [(validate.rules).message.required = true];
  // Build information.
  Build build = 16 [(validate.rules).message.required = true];
  // ID of the cluster to which the host belongs.
  uint64 scheduler_cluster_id = 17;
  // Port of object storage server.
  int32 object_storage_port = 18 [(validate.rules).int32 = {gte: 1024, lt: 65535, ignore_empty: true}];
}

// CPU Stat.
message CPU {
  // Number of logical cores in the system.
  uint32 logical_count = 1;
  // Number of physical cores in the system
  uint32 physical_count = 2;
  // Percent calculates the percentage of cpu used.
  double percent = 3 [(validate.rules).double.gte = 0];
  // Calculates the percentage of cpu used by process.
  double process_percent = 4 [(validate.rules).double.gte = 0];
  // CPUTimes contains the amounts of time the CPU has spent performing different kinds of work.
  CPUTimes times = 5 [(validate.rules).message.required = true];
}

// CPUTimes contains the amounts of time the CPU has spent performing different
// kinds of work. Time units are in seconds.
message CPUTimes {
  // CPU time of user.
  double user = 1 [(validate.rules).double.gte = 0];
  // CPU time of system.
  double system = 2 [(validate.rules).double.gte = 0];
  // CPU time of idle.
  double idle = 3 [(validate.rules).double.gte = 0];
  // CPU time of nice.
  double nice = 4 [(validate.rules).double.gte = 0];
  // CPU time of iowait.
  double iowait = 5 [(validate.rules).double.gte = 0];
  // CPU time of irq.
  double irq = 6 [(validate.rules).double.gte = 0];
  // CPU time of softirq.
  double softirq = 7 [(validate.rules).double.gte = 0];
  // CPU time of steal.
  double steal = 8 [(validate.rules).double.gte = 0];
  // CPU time of guest.
  double guest = 9 [(validate.rules).double.gte = 0];
  // CPU time of guest nice.
  double guest_nice = 10 [(validate.rules).double.gte = 0];
}

// Memory Stat.
message Memory {
  // Total amount of RAM on this system.
  uint64 total = 1;
  // RAM available for programs to allocate.
  uint64 available = 2;
  // RAM used by programs.
  uint64 used = 3;
  // Percentage of RAM used by programs.
  double used_percent = 4;
  // Calculates the percentage of memory used by process.
  double process_used_percent = 5;
  // This is the kernel's notion of free memory.
  uint64 free = 6;
}

// Network Stat.
message Network {
  // Return count of tcp connections opened and status is ESTABLISHED.
  uint32 tcp_connection_count = 1;
  // Return count of upload tcp connections opened and status is ESTABLISHED.
  uint32 upload_tcp_connection_count = 2;
  // Location path(area|country|province|city|...).
  string location = 4;
  // IDC where the peer host is located
  string idc = 5;
}

// Disk Stat.
message Disk {
  // Total amount of disk on the data path of dragonfly.
  uint64 total = 1;
  // Free amount of disk on the data path of dragonfly.
  uint64 free = 2;
  // Used amount of disk on the data path of dragonfly.
  uint64 used = 3;
  // Used percent of disk on the data path of dragonfly directory.
  double used_percent = 4 [(validate.rules).double = {gte: 0, lte: 100}];
  // Total amount of indoes on the data path of dragonfly directory.
  uint64 inodes_total = 5;
  // Used amount of indoes on the data path of dragonfly directory.
  uint64 inodes_used = 6;
  // Free amount of indoes on the data path of dragonfly directory.
  uint64 inodes_free = 7;
  // Used percent of indoes on the data path of dragonfly directory.
  double inodes_used_percent = 8 [(validate.rules).double = {gte: 0, lte: 100}];
}

// Build information.
message Build {
  // Git version.
  string git_version = 1;
  // Git commit.
  string git_commit = 2;
  // Golang version.
  string go_version = 3;
  // Build platform.
  string platform = 4;
}

// Scheduler RPC Service.
service Scheduler{
  // RegisterPeerTask registers a peer into task.
  rpc RegisterPeerTask(PeerTaskRequest)returns(RegisterResult);

  // ReportPieceResult reports piece results and receives peer packets.
  rpc ReportPieceResult(stream PieceResult)returns(stream PeerPacket);

  // ReportPeerResult reports downloading result for the peer.
  rpc ReportPeerResult(PeerResult)returns(google.protobuf.Empty);

  // A peer announces that it has the announced task to other peers.
  rpc AnnounceTask(AnnounceTaskRequest) returns(google.protobuf.Empty);

  // Checks if any peer has the given task.
  rpc StatTask(StatTaskRequest)returns(Task);

  // LeaveTask makes the peer leaving from task.
  rpc LeaveTask(PeerTarget)returns(google.protobuf.Empty);

  // AnnounceHost announces host to scheduler.
  rpc AnnounceHost(AnnounceHostRequest)returns(google.protobuf.Empty);

  // LeaveHost makes the peers leaving from host.
  rpc LeaveHost(LeaveHostRequest)returns(google.protobuf.Empty);
}