api/pkg/apis/manager/v1/manager.proto

/*
 *     Copyright 2022 The Dragonfly Authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

syntax = "proto3";

package manager;

import "pkg/apis/common/v1/common.proto";
import "google/protobuf/empty.proto";
import "validate/validate.proto";
import "google/protobuf/timestamp.proto";

option go_package = "d7y.io/api/pkg/apis/manager/v1";

// Request source type.
enum SourceType {
  // Scheduler service.
  SCHEDULER_SOURCE = 0;
  // Peer service.
  PEER_SOURCE = 1;
  // SeedPeer service.
  SEED_PEER_SOURCE = 2;
}

// SecurityGroup represents security group of cluster.
message SecurityGroup {
  // Group id.
  uint64 id = 1;
  // Group name.
  string name = 2;
  // Group biography.
  string bio = 3;
  // Group domain.
  string domain = 4;
  // Group proxy domain.
  string proxy_domain = 5;
}

// SeedPeerCluster represents cluster of seed peer.
message SeedPeerCluster {
  // Cluster id.
  uint64 id = 1;
  // Cluster name.
  string name = 2;
  // Cluster biography.
  string bio = 3;
  // Cluster configuration.
  bytes config = 4;
  // Cluster scopes.
  bytes scopes = 5;
  // Security group to which the seed peer cluster belongs.
  SecurityGroup security_group = 6;
}

// SeedPeer represents seed peer for network.
message SeedPeer {
  // Seed peer id.
  uint64 id = 1;
  // Seed peer hostname.
  string host_name = 2;
  // Seed peer type.
  string type = 3;
  // Seed peer idc.
  string idc = 5;
  // Seed peer network topology.
  string net_topology = 6;
  // Seed peer location.
  string location = 7;
  // Seed peer ip.
  string ip = 8;
  // Seed peer grpc port.
  int32 port = 9;
  // Seed peer download port.
  int32 download_port = 10;
  // Seed peer state.
  string state = 11;
  // ID of the cluster to which the seed peer belongs.
  uint64 seed_peer_cluster_id = 12;
  // Cluster to which the seed peer belongs.
  SeedPeerCluster seed_peer_cluster = 13;
  // Schedulers included in seed peer.
  repeated Scheduler schedulers = 14;
  // Seed peer object storage port.
  int32 object_storage_port = 15;
}

// GetSeedPeerRequest represents request of GetSeedPeer.
message GetSeedPeerRequest {
  // Request source type.
  SourceType source_type = 1 [(validate.rules).enum.defined_only = true];
  // Seed peer hostname.
  string host_name = 2 [(validate.rules).string.hostname = true];
  // ID of the cluster to which the seed peer belongs.
  uint64 seed_peer_cluster_id = 3 [(validate.rules).uint64 = {gte: 1}];
  // Seed peer ip.
  string ip = 4 [(validate.rules).string = {ip: true, ignore_empty: true}];
}

// UpdateSeedPeerRequest represents request of UpdateSeedPeer.
message UpdateSeedPeerRequest {
  // Request source type.
  SourceType source_type = 1 [(validate.rules).enum.defined_only = true];
  // Seed peer hostname.
  string host_name = 2 [(validate.rules).string.hostname = true];
  // Seed peer type.
  string type = 3 [(validate.rules).string = {in: ["super", "strong", "weak"]}];
  // Seed peer idc.
  string idc = 5 [(validate.rules).string = {min_len: 1, max_len: 1024, ignore_empty: true}];
  // Seed peer network topology.
  string net_topology = 6 [(validate.rules).string = {min_len: 1, max_len: 1024, ignore_empty: true}];
  // Seed peer location.
  string location = 7 [(validate.rules).string = {max_len: 1024, ignore_empty: true}];
  // Seed peer ip.
  string ip = 8 [(validate.rules).string = {ip: true}];
  // Seed peer port.
  int32 port = 9 [(validate.rules).int32 = {gte: 1024, lt: 65535}];
  // Seed peer download port.
  int32 download_port = 10 [(validate.rules).int32 = {gte: 1024, lt: 65535}];
  // ID of the cluster to which the seed peer belongs.
  uint64 seed_peer_cluster_id = 11 [(validate.rules).uint64 = {gte: 1}];
  // Seed peer object storage port.
  int32 object_storage_port = 12 [(validate.rules).int32 = {gte: 1024, lt: 65535, ignore_empty: true}];
}

// SeedPeerCluster represents cluster of scheduler.
message SchedulerCluster {
  // Cluster id.
  uint64 id = 1;
  // Cluster name.
  string name = 2;
  // Cluster biography.
  string bio = 3;
  // Cluster config.
  bytes config = 4;
  // Cluster client config.
  bytes client_config = 5;
  // Cluster scopes.
  bytes scopes = 6;
  // Security group to which the scheduler cluster belongs.
  SecurityGroup security_group = 7;
}

// SeedPeerCluster represents scheduler for network.
message Scheduler {
  // Scheduler id.
  uint64 id = 1;
  // Scheduler hostname.
  string host_name = 2;
  // Deprecated: Do not use.
  string vips = 3;
  // Scheduler idc.
  string idc = 4;
  // Scheduler location.
  string location = 5;
  // Deprecated: Use net_topology instead.
  bytes net_config = 6;
  // Scheduler ip.
  string ip = 7;
  // Scheduler grpc port.
  int32 port = 8;
  // Scheduler state.
  string state = 9;
  // ID of the cluster to which the scheduler belongs.
  uint64 scheduler_cluster_id = 10;
  // Cluster to which the scheduler belongs.
  SchedulerCluster scheduler_cluster = 11;
  // Seed peers to which the scheduler belongs.
  repeated SeedPeer seed_peers = 13;
  // Scheduler network topology.
  string net_topology = 14;
}

// GetSchedulerRequest represents request of GetScheduler.
message GetSchedulerRequest {
  // Request source type.
  SourceType source_type = 1 [(validate.rules).enum.defined_only = true];
  // Scheduler hostname.
  string host_name = 2 [(validate.rules).string.hostname = true];
  // ID of the cluster to which the scheduler belongs.
  uint64 scheduler_cluster_id = 3 [(validate.rules).uint64 = {gte: 1}];
  // Scheduler ip.
  string ip = 4 [(validate.rules).string = {ip: true, ignore_empty: true}];
}

// UpdateSchedulerRequest represents request of UpdateScheduler.
message UpdateSchedulerRequest {
  // Request source type.
  SourceType source_type = 1 [(validate.rules).enum.defined_only = true];
  // Scheduler hostname.
  string host_name = 2 [(validate.rules).string.hostname = true];
  // ID of the cluster to which the scheduler belongs.
  uint64 scheduler_cluster_id = 3 [(validate.rules).uint64 = {gte: 1}];
  // Deprecated: Do not use.
  string vips = 4 [(validate.rules).string = {min_len: 1, max_len: 1024, ignore_empty: true}];
  // Scheduler idc.
  string idc = 5 [(validate.rules).string = {min_len: 1, max_len: 1024, ignore_empty: true}];
  // Scheduler location.
  string location = 6 [(validate.rules).string = {min_len: 1, max_len: 1024, ignore_empty: true}];
  // Deprecated: Use net_topology instead.
  bytes net_config = 7 [(validate.rules).bytes = {min_len: 1, ignore_empty: true}];
  // Scheduler ip.
  string ip = 8 [(validate.rules).string = {ip: true}];
  // Scheduler port.
  int32 port = 9 [(validate.rules).int32 = {gte: 1024, lt: 65535}];
  // Scheduler network topology.
  string net_topology = 10 [(validate.rules).string = {min_len: 1, max_len: 1024, ignore_empty: true}];
}

// ListSchedulersRequest represents request of ListSchedulers.
message ListSchedulersRequest {
  // Request source type.
  SourceType source_type = 1 [(validate.rules).enum.defined_only = true];
  // Source service hostname.
  string host_name = 2 [(validate.rules).string.hostname = true];
  // Source service ip.
  string ip = 3 [(validate.rules).string.ip = true];
  // Source service host information.
  map<string, string> host_info = 5 [(validate.rules).map.ignore_empty = true];
  // Dfdaemon version.
  string version = 6 [(validate.rules).string = {min_len: 1, max_len: 1024, ignore_empty: true}];
  // Dfdaemon commit.
  string commit = 7 [(validate.rules).string = {min_len: 1, max_len: 1024, ignore_empty: true}];
}

// ListSchedulersResponse represents response of ListSchedulers.
message ListSchedulersResponse {
  // Schedulers to which the source service belongs.
  repeated Scheduler schedulers = 1;
}

// ObjectStorage represents config of object storage.
message ObjectStorage {
  // name is object storage name of type, it can be s3, oss or obs.
  string name = 1 [(validate.rules).string = {min_len: 1, max_len: 1024}];
  // Region is storage region.
  string region = 2 [(validate.rules).string = {min_len: 1, max_len: 1024, ignore_empty: true}];
  // Endpoint is datacenter endpoint.
  string endpoint = 3 [(validate.rules).string = {min_len: 1, max_len: 1024, ignore_empty: true}];
  // AccessKey is access key ID.
  string access_key = 4 [(validate.rules).string = {min_len: 1, max_len: 1024, ignore_empty: true}];
  // SecretKey is access key secret.
  string secret_key = 5 [(validate.rules).string = {min_len: 1, max_len: 1024, ignore_empty: true}];
  // S3ForcePathStyle sets force path style for s3, true by default.
  // Set this to `true` to force the request to use path-style addressing,
  // i.e., `http://s3.amazonaws.com/BUCKET/KEY`. By default, the S3 client
  // will use virtual hosted bucket addressing when possible
  // (`http://BUCKET.s3.amazonaws.com/KEY`).
  // Refer to https://github.com/aws/aws-sdk-go/blob/main/aws/config.go#L118.
  bool s3_force_path_style = 6;
}

// GetObjectStorageRequest represents request of GetObjectStorage.
message GetObjectStorageRequest {
  // Request source type.
  SourceType source_type = 1 [(validate.rules).enum.defined_only = true];
  // Source service hostname.
  string host_name = 2 [(validate.rules).string.hostname = true];
  // Source service ip.
  string ip = 3 [(validate.rules).string.ip = true];
}

// Bucket represents config of bucket.
message Bucket {
  // Bucket name.
  string name = 1 [(validate.rules).string = {min_len: 1, max_len: 1024}];
}

// ListSchedulersRequest represents request of ListBuckets.
message ListBucketsRequest {
  // Request source type.
  SourceType source_type = 1 [(validate.rules).enum.defined_only = true];
  // Source service hostname.
  string host_name = 2 [(validate.rules).string.hostname = true];
  // Source service ip.
  string ip = 3 [(validate.rules).string.ip = true];
}

// ListBucketsResponse represents response of ListBuckets.
message ListBucketsResponse {
  // Bucket configs.
  repeated Bucket buckets = 1;
}

// Model represents information of model.
message Model {
  // Model id.
  string model_id = 1 [(validate.rules).string = {min_len: 1, max_len: 1024}];
  // Model name.
  string name = 2 [(validate.rules).string = {min_len: 1, max_len: 1024}];
  // Model version id.
  string version_id = 3 [(validate.rules).string = {min_len: 1, max_len: 1024}];
  // Scheduler id.
  uint64 scheduler_id = 4 [(validate.rules).uint64 = {gte: 1}];
  // Scheduler hostname.
  string host_name = 5 [(validate.rules).string.hostname = true];
  // Scheduler ip.
  string ip = 6 [(validate.rules).string.ip = true];
  // Model create time.
  google.protobuf.Timestamp created_at = 7 [(validate.rules).timestamp.required = true];
  // Model update time.
  google.protobuf.Timestamp updated_at = 8 [(validate.rules).timestamp.required = true];
}

// ListModelsRequest represents request of ListModels.
message ListModelsRequest {
  // Scheduler id.
  uint64 scheduler_id = 1 [(validate.rules).uint64 = {gte: 1}];
}

// ListModelsResponse represents response of ListModels.
message ListModelsResponse {
  // Model informations.
  repeated Model models = 1;
}

// GetModelRequest represents request of GetModel.
message GetModelRequest {
  // Scheduler id.
  uint64 scheduler_id = 1 [(validate.rules).uint64 = {gte: 1}];
  // Model id.
  string model_id = 2 [(validate.rules).string = {min_len: 1, max_len: 1024}];
}

// CreateModelRequest represents request of CreateModel.
message CreateModelRequest {
  // Model id.
  string model_id = 1 [(validate.rules).string = {min_len: 1, max_len: 1024}];
  // Model name.
  string name = 2 [(validate.rules).string = {min_len: 1, max_len: 1024}];
  // Model version id.
  string version_id = 3 [(validate.rules).string = {min_len: 1, max_len: 1024}];
  // Scheduler id.
  uint64 scheduler_id = 4 [(validate.rules).uint64 = {gte: 1}];
  // Scheduler hostname.
  string host_name = 5 [(validate.rules).string.hostname = true];
  // Scheduler ip.
  string ip = 6 [(validate.rules).string.ip = true];
}

// UpdateModelRequest represents request of UpdateModel.
message UpdateModelRequest {
  // Model id.
  string model_id = 1 [(validate.rules).string = {min_len: 1, max_len: 1024}];
  // Model name.
  string name = 2 [(validate.rules).string = {min_len: 1, max_len: 1024, ignore_empty: true}];
  // Model version id.
  string version_id = 3 [(validate.rules).string = {min_len: 1, max_len: 1024, ignore_empty: true}];
  // Scheduler id.
  uint64 scheduler_id = 4 [(validate.rules).uint64 = {gte: 1}];
  // Scheduler hostname.
  string host_name = 5 [(validate.rules).string = {hostname: true, ignore_empty: true}];
  // Scheduler ip.
  string ip = 6 [(validate.rules).string = {ip: true, ignore_empty: true}];
}

// DeleteModelRequest represents request of DeleteModel.
message DeleteModelRequest {
  // Scheduler id.
  uint64 scheduler_id = 1 [(validate.rules).uint64 = {gte: 1}];
  // Model id.
  string model_id = 2 [(validate.rules).string = {min_len: 1, max_len: 1024}];
}

// ModelVersion represents information of model version.
message ModelVersion {
  // Model version id.
  string version_id = 1 [(validate.rules).string = {min_len: 1, max_len: 1024}];
  // Model version data.
  bytes data = 2 [(validate.rules).bytes.min_len = 1];
  // Model version mae.
  double mae = 3 [(validate.rules).double.gte = 0];
  // Model version mse.
  double mse = 4 [(validate.rules).double.gte = 0];
  // Model version rmse.
  double rmse = 5 [(validate.rules).double.gte = 0];
  // Model version r^2.
  double r2 = 6;
  // Model create time.
  google.protobuf.Timestamp created_at = 7 [(validate.rules).timestamp.required = true];
  // Model update time.
  google.protobuf.Timestamp updated_at = 8 [(validate.rules).timestamp.required = true];
}

// ListModelVersionsRequest represents request of ListModelVersions.
message ListModelVersionsRequest {
  // Scheduler id.
  uint64 scheduler_id = 1 [(validate.rules).uint64 = {gte: 1}];
  // Model id.
  string model_id = 2 [(validate.rules).string = {min_len: 1, max_len: 1024}];
}

// ListModelVersionsResponse represents response of ListModelVersions.
message ListModelVersionsResponse {
  // Model version informations.
  repeated ModelVersion model_versions = 1;
}

// GetModelVersionRequest represents request of GetModelVersion.
message GetModelVersionRequest {
  // Scheduler id.
  uint64 scheduler_id = 1 [(validate.rules).uint64 = {gte: 1}];
  // Model id.
  string model_id = 2 [(validate.rules).string = {min_len: 1, max_len: 1024}];
  // Model version id.
  string version_id = 3 [(validate.rules).string = {min_len: 1, max_len: 1024}];
}

// CreateModelVersionRequest represents request of CreateModelVersion.
message CreateModelVersionRequest {
  // Scheduler id.
  uint64 scheduler_id = 1 [(validate.rules).uint64 = {gte: 1}];
  // Model id.
  string model_id = 2 [(validate.rules).string = {min_len: 1, max_len: 1024}];
  // Model version data.
  bytes data = 3 [(validate.rules).bytes.min_len = 1];
  // Model version mae.
  double mae = 4 [(validate.rules).double.gte = 0];
  // Model version mse.
  double mse = 5 [(validate.rules).double.gte = 0];
  // Model version rmse.
  double rmse = 6 [(validate.rules).double.gte = 0];
  // Model version r^2.
  double r2 = 7;
}

// UpdateModelVersionRequest represents request of UpdateModelVersion.
message UpdateModelVersionRequest {
  // Model version id.
  string version_id = 1 [(validate.rules).string = {min_len: 1, max_len: 1024}];
  // Scheduler id.
  uint64 scheduler_id = 2 [(validate.rules).uint64 = {gte: 1}];
  // Model id.
  string model_id = 3 [(validate.rules).string = {min_len: 1, max_len: 1024}];
  // Model version data.
  bytes data = 4 [(validate.rules).bytes = {min_len: 1, ignore_empty: true}];
  // Model version mae.
  double mae = 5 [(validate.rules).double = {gte: 0, ignore_empty: true}];
  // Model version mse.
  double mse = 6 [(validate.rules).double = {gte: 0, ignore_empty: true}];
  // Model version rmse.
  double rmse = 7 [(validate.rules).double = {gte: 0, ignore_empty: true}];
  // Model version r^2.
  double r2 = 8 [(validate.rules).double = {ignore_empty: true}];
}

// DeleteModelVersionRequest represents request of DeleteModelVersion.
message DeleteModelVersionRequest {
  // Scheduler id.
  uint64 scheduler_id = 1 [(validate.rules).uint64 = {gte: 1}];
  // Model id.
  string model_id = 2 [(validate.rules).string = {min_len: 1, max_len: 1024}];
  // Model version id.
  string version_id = 3 [(validate.rules).string = {min_len: 1, max_len: 1024}];
}

// URLPriority represents config of url priority.
message URLPriority {
  // URL regex.
  string regex = 1 [(validate.rules).string = {min_len: 1}];
  // URL priority value.
  common.Priority value = 2;
}

// ApplicationPriority represents config of application priority.
message ApplicationPriority {
  // Priority value.
  common.Priority value = 1;
  // URL priority.
  repeated URLPriority urls = 2;
}

// Application represents config of application.
message Application {
  // Application id.
  uint64 id = 1 [(validate.rules).uint64 = {gte: 1}];
  // Application name.
  string name = 2 [(validate.rules).string = {min_len: 1, max_len: 1024}];
  // Application url.
  string url = 3 [(validate.rules).string.uri = true];
  // Application biography.
  string bio = 4;
  // Application priority.
  ApplicationPriority priority = 5 [(validate.rules).message.required = true];
}

// ListApplicationsRequest represents request of ListApplications.
message ListApplicationsRequest {
  // Request source type.
  SourceType source_type = 1 [(validate.rules).enum.defined_only = true];
  // Source service hostname.
  string host_name = 2 [(validate.rules).string.hostname = true];
  // Source service ip.
  string ip = 3 [(validate.rules).string.ip = true];
}

// ListApplicationsResponse represents response of ListApplications.
message ListApplicationsResponse {
  // Application configs.
  repeated Application applications = 1;
}

// KeepAliveRequest represents request of KeepAlive.
message KeepAliveRequest {
  // Request source type.
  SourceType source_type = 1 [(validate.rules).enum.defined_only = true];
  // Source service hostname.
  string host_name = 2 [(validate.rules).string.hostname = true];
  // ID of the cluster to which the source service belongs.
  uint64 cluster_id = 3 [(validate.rules).uint64 = {gte: 1}];
  // Source service ip.
  string ip = 4 [(validate.rules).string = {ip: true, ignore_empty: true}];
}

// Manager RPC Service.
service Manager {
  // Get SeedPeer and SeedPeer cluster configuration.
  rpc GetSeedPeer(GetSeedPeerRequest) returns(SeedPeer);

  // Update SeedPeer configuration.
  rpc UpdateSeedPeer(UpdateSeedPeerRequest) returns(SeedPeer);

  // Get Scheduler and Scheduler cluster configuration.
  rpc GetScheduler(GetSchedulerRequest)returns(Scheduler);

  // Update scheduler configuration.
  rpc UpdateScheduler(UpdateSchedulerRequest) returns(Scheduler);

  // List acitve schedulers configuration.
  rpc ListSchedulers(ListSchedulersRequest)returns(ListSchedulersResponse);

  // Get ObjectStorage configuration.
  rpc GetObjectStorage(GetObjectStorageRequest) returns(ObjectStorage);

  // List buckets configuration.
  rpc ListBuckets(ListBucketsRequest)returns(ListBucketsResponse);

  // List models information.
  rpc ListModels(ListModelsRequest)returns(ListModelsResponse);

  // Get model information.
  rpc GetModel(GetModelRequest)returns(Model);

  // Create model information.
  rpc CreateModel(CreateModelRequest)returns(Model);

  // Update model information.
  rpc UpdateModel(UpdateModelRequest)returns(Model);

  // Delete model information.
  rpc DeleteModel(DeleteModelRequest)returns(google.protobuf.Empty);

  // List model versions information.
  rpc ListModelVersions(ListModelVersionsRequest)returns(ListModelVersionsResponse);

  // Get model version information.
  rpc GetModelVersion(GetModelVersionRequest)returns(ModelVersion);

  // Create model version information.
  rpc CreateModelVersion(CreateModelVersionRequest)returns(ModelVersion);

  // Update model version information.
  rpc UpdateModelVersion(UpdateModelVersionRequest)returns(ModelVersion);

  // Delete model version information.
  rpc DeleteModelVersion(DeleteModelVersionRequest)returns(google.protobuf.Empty);

  // List applications configuration.
  rpc ListApplications(ListApplicationsRequest)returns(ListApplicationsResponse);

  // KeepAlive with manager.
  rpc KeepAlive(stream KeepAliveRequest)returns(google.protobuf.Empty);
}