api/proto/dfdaemon.proto

/*
 *     Copyright 2022 The Dragonfly Authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

syntax = "proto3";

package dfdaemon.v2;

import "common.proto";
import "google/protobuf/duration.proto";
import "google/protobuf/empty.proto";

// DownloadTaskRequest represents request of DownloadTask.
message DownloadTaskRequest {
  // Download information.
  common.v2.Download download = 1;
}

// DownloadTaskStartedResponse represents task download started response of DownloadTaskResponse.
message DownloadTaskStartedResponse {
  // Task content length.
  uint64 content_length = 1;

  // Range is url range of request. If protocol is http, range
  // is parsed from http header. If other protocol, range comes
  // from download range field.
  optional common.v2.Range range = 2;

  // Task response headers.
  map<string, string> response_header = 3;

  // Need to download pieces.
  repeated common.v2.Piece pieces = 4;

  // is_finished indicates whether the download task is finished.
  bool is_finished = 5;
}

// DownloadPieceFinishedResponse represents piece download finished response of DownloadTaskResponse.
message DownloadPieceFinishedResponse {
  // Finished piece of task.
  common.v2.Piece piece = 1;
}

// DownloadTaskResponse represents response of DownloadTask.
message DownloadTaskResponse {
  // Host id.
  string host_id = 1;
  // Task id.
  string task_id = 2;
  // Peer id.
  string peer_id = 3;

  oneof response {
    DownloadTaskStartedResponse download_task_started_response = 4;
    DownloadPieceFinishedResponse download_piece_finished_response = 5;
  }
}

// SyncPiecesRequest represents request of SyncPieces.
message SyncPiecesRequest {
  // Host id.
  string host_id = 1;
  // Task id.
  string task_id = 2;
  // Interested piece numbers.
  repeated uint32 interested_piece_numbers = 3;
}

// SyncPiecesResponse represents response of SyncPieces.
message SyncPiecesResponse {
  // Exist piece number.
  uint32 number = 1;
  // Piece offset.
  uint64 offset = 2;
  // Piece length.
  uint64 length = 3;
}

// DownloadPieceRequest represents request of DownloadPiece.
message DownloadPieceRequest{
  // Host id.
  string host_id = 1;
  // Task id.
  string task_id = 2;
  // Piece number.
  uint32 piece_number = 3;
}

// DownloadPieceResponse represents response of DownloadPieces.
message DownloadPieceResponse {
  // Piece information.
  common.v2.Piece piece = 1;
  // Piece metadata digest, it is used to verify the integrity of the piece metadata.
  optional string digest = 2;
}

// StatTaskRequest represents request of StatTask.
message StatTaskRequest {
  // Task id.
  string task_id = 1;
  // Remote IP represents the IP address of the client initiating the stat request.
  optional string remote_ip = 2;
  // local_only specifies whether to query task information from local client only.
  // If true, the query will be restricted to the local client.
  // By default (false), the query may be forwarded to the scheduler
  // for a cluster-wide search.
  bool local_only = 3;
}

// ListTaskEntriesRequest represents request of ListTaskEntries.
message ListTaskEntriesRequest {
  // Task id.
  string task_id = 1;
  // URL to be listed the entries.
  string url = 2;
  // HTTP header to be sent with the request.
  map<string, string> request_header = 3;
  // List timeout.
  optional google.protobuf.Duration timeout = 4;
  // certificate_chain is the client certs with DER format for the backend client to list the entries.
  repeated bytes certificate_chain = 5;
  // Object storage protocol information.
  optional common.v2.ObjectStorage object_storage = 6;
  // HDFS protocol information.
  optional common.v2.HDFS hdfs = 7;
  // Remote IP represents the IP address of the client initiating the list request.
  optional string remote_ip = 8;
}

// ListTaskEntriesResponse represents response of ListTaskEntries.
message ListTaskEntriesResponse {
  // Content length is the content length of the response
  uint64 content_length = 1;
  // HTTP header to be sent with the request.
  map<string, string> response_header = 2;
  // Backend HTTP status code.
  optional int32 status_code = 3;
  /// Entries is the information of the entries in the directory.
  repeated Entry entries = 4;
}

// Entry represents an entry in a directory.
message Entry {
  // URL of the entry.
  string url = 1;
  // Size of the entry.
  uint64 content_length = 2;
  // Is directory or not.
  bool is_dir = 3;
}

// DeleteTaskRequest represents request of DeleteTask.
message DeleteTaskRequest {
  // Task id.
  string task_id = 1;
  // Remote IP represents the IP address of the client initiating the delete request.
  optional string remote_ip = 2;
}

// DownloadCacheTaskRequest represents request of DownloadCacheTask.
message DownloadCacheTaskRequest {
  // Download url.
  string url = 1;
  // Digest of the task digest, for example :xxx or sha256:yyy.
  optional string digest = 2;
  // Range is url range of request. If protocol is http, range
  // will set in request header. If protocol is others, range
  // will set in range field.
  optional common.v2.Range range = 3;
  // Task type.
  common.v2.TaskType type = 4;
  // URL tag identifies different task for same url.
  optional string tag = 5;
  // Application of task.
  optional string application = 6;
  // Peer priority.
  common.v2.Priority priority = 7;
  // Filtered query params to generate the task id.
  // When filter is ["Signature", "Expires", "ns"], for example:
  // http://example.com/xyz?Expires=e1&Signature=s1&ns=docker.io and http://example.com/xyz?Expires=e2&Signature=s2&ns=docker.io
  // will generate the same task id.
  // Default value includes the filtered query params of s3, gcs, oss, obs, cos.
  repeated string filtered_query_params = 8;
  // Task request headers.
  map<string, string> request_header = 9;
  // Task piece length.
  optional uint64 piece_length = 10;
  // File path to be downloaded. If output_path is set, the downloaded file will be saved to the specified path.
  // Dfdaemon will try to create hard link to the output path before starting the download. If hard link creation fails,
  // it will copy the file to the output path after the download is completed.
  // For more details refer to https://github.com/dragonflyoss/design/blob/main/systems-analysis/file-download-workflow-with-hard-link/README.md.
  optional string output_path = 11;
  // Download timeout.
  optional google.protobuf.Duration timeout = 12;
  // Dfdaemon cannot download the task from the source if disable_back_to_source is true.
  bool disable_back_to_source = 13;
  // Scheduler needs to schedule the task downloads from the source if need_back_to_source is true.
  bool need_back_to_source = 14;
  // certificate_chain is the client certs with DER format for the backend client to download back-to-source.
  repeated bytes certificate_chain = 15;
  // Prefetch pre-downloads all pieces of the task when the download task request is a range request.
  bool prefetch = 16;
  // Object storage protocol information.
  optional common.v2.ObjectStorage object_storage = 17;
  // HDFS protocol information.
  optional common.v2.HDFS hdfs = 18;
  // is_prefetch is the flag to indicate whether the request is a prefetch request.
  bool is_prefetch = 19;
  // need_piece_content is the flag to indicate whether the response needs to return piece content.
  bool need_piece_content = 20;
  // content_for_calculating_task_id is the content used to calculate the task id.
  // If content_for_calculating_task_id is set, use its value to calculate the task ID.
  // Otherwise, calculate the task ID based on url, piece_length, tag, application, and filtered_query_params.
  optional string content_for_calculating_task_id = 21;
  // remote_ip represents the IP address of the client initiating the download request.
  // For proxy requests, it is set to the IP address of the request source.
  // For dfget requests, it is set to the IP address of the dfget.
  optional string remote_ip = 22;
}

// DownloadCacheTaskStartedResponse represents cache task download started response of DownloadCacheTaskResponse.
message DownloadCacheTaskStartedResponse {
  // Task content length.
  uint64 content_length = 1;

  // Range is url range of request. If protocol is http, range
  // is parsed from http header. If other protocol, range comes
  // from download range field.
  optional common.v2.Range range = 2;

  // Task response headers.
  map<string, string> response_header = 3;

  // Need to download pieces.
  repeated common.v2.Piece pieces = 4;

  // is_finished indicates whether the download task is finished.
  bool is_finished = 5;
}

// DownloadCacheTaskResponse represents response of DownloadCacheTask.
message DownloadCacheTaskResponse {
  // Host id.
  string host_id = 1;
  // Task id.
  string task_id = 2;
  // Peer id.
  string peer_id = 3;

  oneof response {
    DownloadCacheTaskStartedResponse download_cache_task_started_response = 4;
    DownloadPieceFinishedResponse download_piece_finished_response = 5;
  }
}

// SyncCachePiecesRequest represents request of SyncCachePieces.
message SyncCachePiecesRequest {
  // Host id.
  string host_id = 1;
  // Task id.
  string task_id = 2;
  // Interested piece numbers.
  repeated uint32 interested_cache_piece_numbers = 3;
}

// SyncCachePiecesResponse represents response of SyncCachePieces.
message SyncCachePiecesResponse {
  // Exist piece number.
  uint32 number = 1;
  // Piece offset.
  uint64 offset = 2;
  // Piece length.
  uint64 length = 3;
}

// DownloadCachePieceRequest represents request of DownloadCachePiece.
message DownloadCachePieceRequest{
  // Host id.
  string host_id = 1;
  // Task id.
  string task_id = 2;
  // Piece number.
  uint32 piece_number = 3;
}

// DownloadCachePieceResponse represents response of DownloadCachePieces.
message DownloadCachePieceResponse {
  // Piece information.
  common.v2.Piece piece = 1;
  // Piece metadata digest, it is used to verify the integrity of the piece metadata.
  optional string digest = 2;
}

// StatCacheTaskRequest represents request of StatCacheTask.
message StatCacheTaskRequest {
  // Task id.
  string task_id = 1;
  // Remote IP represents the IP address of the client initiating the stat request.
  optional string remote_ip = 2;
  // local_only specifies whether to query task information from local client only.
  // If true, the query will be restricted to the local client.
  // By default (false), the query may be forwarded to the scheduler
  // for a cluster-wide search.
  bool local_only = 3;
}

// DeleteCacheTaskRequest represents request of DeleteCacheTask.
message DeleteCacheTaskRequest {
  // Task id.
  string task_id = 1;
}

// DownloadPersistentCacheTaskRequest represents request of DownloadPersistentCacheTask.
message DownloadPersistentCacheTaskRequest {
  // Task id.
  string task_id = 1;
  // Persistent represents whether the persistent cache task is persistent.
  // If the persistent cache task is persistent, the persistent cache peer will
  // not be deleted when dfdaemon runs garbage collection.
  bool persistent = 2;
  // Tag is used to distinguish different persistent cache tasks.
  optional string tag = 3;
  // Application of task.
  optional string application = 4;
  // File path to be exported. If output_path is set, the exported file will be saved to the specified path.
  // Dfdaemon will try to create hard link to the output path before starting the export. If hard link creation fails,
  // it will copy the file to the output path after the export is completed.
  // For more details refer to https://github.com/dragonflyoss/design/blob/main/systems-analysis/file-download-workflow-with-hard-link/README.md.
  optional string output_path = 5;
  // Download timeout.
  optional google.protobuf.Duration timeout = 6;
  // need_piece_content is the flag to indicate whether the response needs to return piece content.
  bool need_piece_content = 7;
  // force_hard_link is the flag to indicate whether the exported file must be hard linked to the output path.
  // For more details refer to https://github.com/dragonflyoss/design/blob/main/systems-analysis/file-download-workflow-with-hard-link/README.md.
  bool force_hard_link = 8;
  // Verifies task data integrity after download using a digest. Supports CRC32, SHA256, and SHA512 algorithms.
  // Format: `<algorithm>:<hash>`, e.g., `crc32:xxx`, `sha256:yyy`, `sha512:zzz`.
  // Returns an error if the computed digest mismatches the expected value.
  //
  // Performance
  // Digest calculation increases processing time. Enable only when data integrity verification is critical.
  optional string digest = 9;
  // Remote IP represents the IP address of the client initiating the download request.
  optional string remote_ip = 10;
}

// DownloadPersistentCacheTaskStartedResponse represents task download started response of DownloadPersistentCacheTaskResponse.
message DownloadPersistentCacheTaskStartedResponse {
  // Task content length.
  uint64 content_length = 1;
}

// DownloadPersistentCacheTaskResponse represents response of DownloadPersistentCacheTask.
message DownloadPersistentCacheTaskResponse {
  // Host id.
  string host_id = 1;
  // Task id.
  string task_id = 2;
  // Peer id.
  string peer_id = 3;

  oneof response {
    DownloadPersistentCacheTaskStartedResponse download_persistent_cache_task_started_response = 4;
    DownloadPieceFinishedResponse download_piece_finished_response = 5;
  }
}

// UploadPersistentCacheTaskRequest represents request of UploadPersistentCacheTask.
message UploadPersistentCacheTaskRequest {
  // content_for_calculating_task_id is the content used to calculate the task id.
  // If content_for_calculating_task_id is set, use its value to calculate the task ID.
  // Otherwise, calculate the task ID based on the file content, tag and application by crc32 algorithm`.
  optional string content_for_calculating_task_id = 1;
  // Upload file path of persistent cache task.
  string path = 2;
  // Replica count of the persistent persistent cache task.
  uint64 persistent_replica_count = 3;
  // Tag is used to distinguish different persistent cache tasks.
  optional string tag = 4;
  // Application of the persistent cache task.
  optional string application = 5;
  // Piece length of the persistent cache task, the value needs to be greater than or equal to 4194304(4MiB).
  optional uint64 piece_length = 6;
  // TTL of the persistent cache task.
  google.protobuf.Duration ttl = 7;
  // Download timeout.
  optional google.protobuf.Duration timeout = 8;
  // Remote IP represents the IP address of the client initiating the upload request.
  optional string remote_ip = 9;
}

// UpdatePersistentCacheTaskRequest represents request of UpdatePersistentCacheTask.
message UpdatePersistentCacheTaskRequest {
  // Task id.
  string task_id = 1;
  // Persistent represents whether the persistent cache peer is persistent.
  // If the persistent cache peer is persistent, the persistent cache peer will
  // not be deleted when dfdaemon runs garbage collection. It only be deleted
  // when the task is deleted by the user.
  bool persistent = 2;
  // Remote IP represents the IP address of the client initiating the list request.
  optional string remote_ip = 3;
}

// StatPersistentCacheTaskRequest represents request of StatPersistentCacheTask.
message StatPersistentCacheTaskRequest {
  // Task id.
  string task_id = 1;
  // Remote IP represents the IP address of the client initiating the stat request.
  optional string remote_ip = 2;
}

// DeletePersistentCacheTaskRequest represents request of DeletePersistentCacheTask.
message DeletePersistentCacheTaskRequest {
  // Task id.
  string task_id = 1;
  // Remote IP represents the IP address of the client initiating the delete request.
  optional string remote_ip = 2;
}

// SyncPersistentCachePiecesRequest represents request of SyncPersistentCachePieces.
message SyncPersistentCachePiecesRequest {
  // Host id.
  string host_id = 1;
  // Task id.
  string task_id = 2;
  // Interested piece numbers.
  repeated uint32 interested_piece_numbers = 3;
}

// SyncPersistentCachePiecesResponse represents response of SyncPersistentCachePieces.
message SyncPersistentCachePiecesResponse {
  // Exist piece number.
  uint32 number = 1;
  // Piece offset.
  uint64 offset = 2;
  // Piece length.
  uint64 length = 3;
}

// DownloadPersistentCachePieceRequest represents request of DownloadPersistentCachePiece.
message DownloadPersistentCachePieceRequest{
  // Host id.
  string host_id = 1;
  // Task id.
  string task_id = 2;
  // Piece number.
  uint32 piece_number = 3;
}

// DownloadPersistentCachePieceResponse represents response of DownloadPersistentCachePieces.
message DownloadPersistentCachePieceResponse {
  // Piece information.
  common.v2.Piece piece = 1;
  // Piece metadata digest, it is used to verify the integrity of the piece metadata.
  optional string digest = 2;
}

// SyncHostRequest represents request of SyncHost.
message SyncHostRequest {
  // Host id.
  string host_id = 1;
  // Peer id.
  string peer_id = 2;
}

// IBVerbsQueuePairEndpoint represents queue pair endpoint of IBVerbs.
message IBVerbsQueuePairEndpoint {
  // Number of the queue pair.
  uint32 num = 1;
  // Local identifier of the context.
  uint32 lid = 2;
  // Global identifier of the context.
  bytes gid = 3;
}

// ExchangeIBVerbsQueuePairEndpointRequest represents request of ExchangeIBVerbsQueuePairEndpoint.
message ExchangeIBVerbsQueuePairEndpointRequest {
  // Information of the source's queue pair endpoint of IBVerbs.
  IBVerbsQueuePairEndpoint endpoint = 1;
}

// ExchangeIBVerbsQueuePairEndpointResponse represents response of ExchangeIBVerbsQueuePairEndpoint.
message ExchangeIBVerbsQueuePairEndpointResponse {
  // Information of the destination's queue pair endpoint of IBVerbs.
  IBVerbsQueuePairEndpoint endpoint = 1;
}

// DfdaemonUpload represents upload service of dfdaemon.
service DfdaemonUpload {
  // DownloadTask downloads task from p2p network.
  rpc DownloadTask(DownloadTaskRequest) returns(stream DownloadTaskResponse);

  // StatTask stats task information.
  rpc StatTask(StatTaskRequest) returns(common.v2.Task);

  // DeleteTask deletes task from p2p network.
  rpc DeleteTask(DeleteTaskRequest) returns(google.protobuf.Empty);

  // SyncPieces syncs piece metadatas from remote peer.
  rpc SyncPieces(SyncPiecesRequest) returns(stream SyncPiecesResponse);

  // DownloadPiece downloads piece from the remote peer.
  rpc DownloadPiece(DownloadPieceRequest)returns(DownloadPieceResponse);

  // DownloadCacheTask downloads cache task from p2p network.
  rpc DownloadCacheTask(DownloadCacheTaskRequest) returns(stream DownloadCacheTaskResponse);

  // StatCacheTask stats cache task information.
  rpc StatCacheTask(StatCacheTaskRequest) returns(common.v2.CacheTask);

  // DeleteCacheTask deletes cache task from p2p network.
  rpc DeleteCacheTask(DeleteCacheTaskRequest) returns(google.protobuf.Empty);

  // SyncCachePieces syncs cache piece metadatas from remote peer.
  rpc SyncCachePieces(SyncCachePiecesRequest) returns(stream SyncCachePiecesResponse);

  // DownloadCachePiece downloads cache piece from the remote peer.
  rpc DownloadCachePiece(DownloadCachePieceRequest)returns(DownloadCachePieceResponse);

  // DownloadPersistentCacheTask downloads persistent cache task from p2p network.
  rpc DownloadPersistentCacheTask(DownloadPersistentCacheTaskRequest) returns(stream DownloadPersistentCacheTaskResponse);

  // UpdatePersistentCacheTask updates metadate of thr persistent cache task in p2p network.
  rpc UpdatePersistentCacheTask(UpdatePersistentCacheTaskRequest) returns(google.protobuf.Empty);

  // StatPersistentCacheTask stats persistent cache task information.
  rpc StatPersistentCacheTask(StatPersistentCacheTaskRequest) returns(common.v2.PersistentCacheTask);

  // DeletePersistentCacheTask deletes persistent cache task from p2p network.
  rpc DeletePersistentCacheTask(DeletePersistentCacheTaskRequest) returns(google.protobuf.Empty);

  // SyncPersistentCachePieces syncs persistent cache pieces from remote peer.
  rpc SyncPersistentCachePieces(SyncPersistentCachePiecesRequest) returns(stream SyncPersistentCachePiecesResponse);

  // DownloadPersistentCachePiece downloads persistent cache piece from p2p network.
  rpc DownloadPersistentCachePiece(DownloadPersistentCachePieceRequest)returns(DownloadPersistentCachePieceResponse);

  // SyncHost sync host info from parents.
  rpc SyncHost(SyncHostRequest) returns (stream common.v2.Host);

  // ExchangeIBVerbsQueuePairEndpoint exchanges queue pair endpoint of IBVerbs with remote peer.
  rpc ExchangeIBVerbsQueuePairEndpoint(ExchangeIBVerbsQueuePairEndpointRequest) returns(ExchangeIBVerbsQueuePairEndpointResponse);
}

// DfdaemonDownload represents download service of dfdaemon.
service DfdaemonDownload {
  // DownloadTask downloads task from p2p network.
  rpc DownloadTask(DownloadTaskRequest) returns(stream DownloadTaskResponse);

  // StatTask stats task information.
  rpc StatTask(StatTaskRequest) returns(common.v2.Task);

  // ListTaskEntries lists task entries for downloading directory.
  rpc ListTaskEntries(ListTaskEntriesRequest) returns(ListTaskEntriesResponse);

  // DeleteTask deletes task from p2p network.
  rpc DeleteTask(DeleteTaskRequest) returns(google.protobuf.Empty);

  // DeleteHost releases host in scheduler.
  rpc DeleteHost(google.protobuf.Empty)returns(google.protobuf.Empty);

   // DownloadCacheTask downloads cache task from p2p network.
  rpc DownloadCacheTask(DownloadCacheTaskRequest) returns(stream DownloadCacheTaskResponse);

  // StatCacheTask stats cache task information.
  rpc StatCacheTask(StatCacheTaskRequest) returns(common.v2.CacheTask);

  // DeleteCacheTask deletes cache task from p2p network.
  rpc DeleteCacheTask(DeleteCacheTaskRequest) returns(google.protobuf.Empty);

  // DownloadPersistentCacheTask downloads persistent cache task from p2p network.
  rpc DownloadPersistentCacheTask(DownloadPersistentCacheTaskRequest) returns(stream DownloadPersistentCacheTaskResponse);

  // UploadPersistentCacheTask uploads persistent cache task to p2p network.
  rpc UploadPersistentCacheTask(UploadPersistentCacheTaskRequest) returns(common.v2.PersistentCacheTask);

  // StatPersistentCacheTask stats persistent cache task information.
  rpc StatPersistentCacheTask(StatPersistentCacheTaskRequest) returns(common.v2.PersistentCacheTask);
}