refactor: add dragonfly-client-util crate (#295)

Signed-off-by: Gaius <gaius.qi@gmail.com>

Cargo.lock

@@ -612,18 +612,16 @@ dependencies = [
  "dashmap",
  "dragonfly-api",
  "dragonfly-client-core",
+ "dragonfly-client-util",
  "fs2",
  "fslock",
  "futures",
  "futures-util",
  "hashring",
- "headers 0.4.0",
- "hex",
  "home",
  "hostname",
  "http 1.0.0",
  "http-body-util",
- "http-range-header",
  "humantime",
  "humantime-serde",
  "hyper 1.1.0",
@@ -646,7 +644,6 @@ dependencies = [
  "reqwest",
  "rocksdb",
  "rustls 0.22.2",
- "rustls-pemfile 2.1.1",
  "rustls-pki-types",
  "serde",
  "serde_json",
@@ -654,7 +651,6 @@ dependencies = [
  "serde_yaml",
  "sha2",
  "sysinfo",
- "thiserror",
  "tokio",
  "tokio-rustls 0.25.0",
  "tokio-stream",
@@ -669,7 +665,6 @@ dependencies = [
  "tracing-opentelemetry",
  "tracing-subscriber",
  "url",
- "uuid",
  "validator",
  "warp",
 ]
@@ -698,6 +693,25 @@ dependencies = [
  "warp",
 ]
 
+[[package]]
+name = "dragonfly-client-util"
+version = "0.1.17"
+dependencies = [
+ "dragonfly-api",
+ "dragonfly-client-core",
+ "hex",
+ "http-range-header",
+ "hyper 1.1.0",
+ "rcgen",
+ "reqwest",
+ "rustls-pemfile 2.1.1",
+ "rustls-pki-types",
+ "sha2",
+ "tracing",
+ "url",
+ "uuid",
+]
+
 [[package]]
 name = "either"
 version = "1.9.0"

Cargo.toml

@@ -23,7 +23,7 @@ name = "dfstore"
 path = "src/bin/dfstore/main.rs"
 
 [workspace]
-members = ["dragonfly-client-core"]
+members = ["dragonfly-client-core", "dragonfly-client-util"]
 
 [workspace.package]
 version = "0.1.17"
@@ -35,9 +35,40 @@ license = "Apache-2.0"
 readme = "README.md"
 edition = "2021"
 
-[dependencies]
+[workspace.dependencies]
 dragonfly-client-core = { path = "dragonfly-client-core" }
+dragonfly-client-util = { path = "dragonfly-client-util" }
+thiserror = "1.0"
+dragonfly-api = "2.0.102"
+reqwest = { version = "0.11.24", features = ["stream", "native-tls", "rustls-tls"] }
+rcgen = { version = "0.12.1", features = ["x509-parser"] }
+hyper = { version = "1.1", features = ["full"] }
+hyper-util = { version = "0.1.2", features = ["client", "client-legacy", "tokio", "server-auto", "http1", "http2"] }
+http-range-header = "0.4.0"
+tracing = "0.1"
+url = "2.4.0"
+rustls = "0.22.2"
+rustls-pki-types = "1.2.0"
+rustls-pemfile = "2.1.1"
+sha2 = "0.10"
+uuid = { version = "1.4", features = ["v4"] }
+hex = "0.4"
+rocksdb = "0.22.0"
+serde_yaml = "0.9"
+serde_json = "1.0"
+http = "1"
+tonic = { version = "0.9.2", features = ["gzip"] }
+tokio = { version = "1.29.1", features = ["full"] }
+tokio-stream = "0.1.14"
+validator = { version = "0.16", features = ["derive"] }
+warp = "0.3.5"
+headers = "0.4.0"
 
+[dependencies]
+dragonfly-client-core.workspace = true
+dragonfly-client-util.workspace = true
+rcgen.workspace = true
+hyper-util.workspace = true
 clap = { version = "4.5.1", features = [ "derive" ] }
 home = "0.5.4"
 tracing = "0.1"
@@ -51,7 +82,6 @@ serde_regex = "1.1.0"
 serde_yaml = "0.9"
 serde_json = "1.0"
 validator = { version = "0.16", features = ["derive"] }
-thiserror = "1.0"
 opentelemetry = { version = "0.18.0", default-features = false, features = ["trace", "rt-tokio"] }
 opentelemetry-jaeger = { version = "0.17.0", features = ["rt-tokio"] }
 lazy_static = "1.4"
@@ -74,8 +104,6 @@ dragonfly-api = "2.0.102"
 chrono = { version = "0.4.34", features = ["serde"] }
 sysinfo = "0.29.6"
 sha2 = "0.10"
-hex = "0.4"
-uuid = { version = "1.4", features = ["v4"] }
 url = "2.4.0"
 async-stream = "0.3.5"
 anyhow = "1.0.80"
@@ -94,15 +122,10 @@ openssl = { version = "0.10", features = ["vendored"] }
 humantime-serde = "1.1.1"
 leaky-bucket = "1.0.1"
 hyper = { version = "1.1", features = ["full"] }
-hyper-util = { version = "0.1.2", features = ["client", "client-legacy", "tokio", "server-auto", "http1", "http2"] }
 tokio-rustls = "0.25"
 hyper-rustls = { version = "0.26", features = [ "http1", "http2", "logging" ] }
 http-body-util = "0.1.0"
 regex = "1.10.2"
-http-range-header = "0.4.0"
 futures-util = "0.3.30"
-headers = "0.4.0"
 rustls = "0.22.2"
-rustls-pemfile = "2.1.1"
 rustls-pki-types = "1.2.0"
-rcgen = { version = "0.12.1", features = ["x509-parser"] }

dragonfly-client-core/Cargo.toml

@@ -1,7 +1,6 @@
 [package]
 name = "dragonfly-client-core"
-description = "Core library for the client"
+description = "Core library for the dragonfly client"
-
 version.workspace = true
 authors.workspace = true
 homepage.workspace = true
@@ -11,21 +10,21 @@ license.workspace = true
 edition.workspace = true
 
 [dependencies]
-thiserror = "1.0"
+rcgen.workspace = true
-rocksdb = "0.22.0"
+reqwest.workspace = true
-serde_yaml = "0.9"
+hyper.workspace = true
-serde_json = "1.0"
+http-range-header.workspace = true
-url = "2.4.0"
+url.workspace = true
-hyper = "1.1"
+hyper-util.workspace = true
-hyper-util = "0.1.2"
+thiserror.workspace = true
-rustls = "0.22.2"
+rocksdb.workspace = true
-reqwest = "0.11.24"
+serde_yaml.workspace = true
-rcgen = "0.12.1"
+serde_json.workspace = true
-http = "1"
+rustls.workspace = true
-http-range-header = "0.4.0"
+http.workspace = true
-tonic = "0.9.2"
+tonic.workspace = true
-tokio = "1.29.1"
+tokio.workspace = true
-tokio-stream = "0.1.14"
+tokio-stream.workspace = true
-validator = "0.16"
+validator.workspace = true
-warp = "0.3.5"
+warp.workspace = true
-headers = "0.4.0"
+headers.workspace = true

dragonfly-client-util/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "dragonfly-client-util"
+description = "Utility library for the dragonfly client"
+version.workspace = true
+authors.workspace = true
+homepage.workspace = true
+repository.workspace = true
+keywords.workspace = true
+license.workspace = true
+readme.workspace = true
+edition.workspace = true
+
+[dependencies]
+dragonfly-client-core.workspace = true
+dragonfly-api.workspace = true
+reqwest.workspace = true
+hyper.workspace = true
+http-range-header.workspace = true
+tracing.workspace = true
+url.workspace = true
+rcgen.workspace = true
+rustls-pki-types.workspace = true
+rustls-pemfile.workspace = true
+sha2.workspace = true
+uuid.workspace = true
+hex.workspace = true

dragonfly-client-util/src/digest/mod.rs

@@ -1,5 +1,5 @@
 /*
- *     Copyright 2023 The Dragonfly Authors
+ *     Copyright 2024 The Dragonfly Authors
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dragonfly-client-util/src/http/mod.rs

@@ -1,5 +1,5 @@
 /*
- *     Copyright 2023 The Dragonfly Authors
+ *     Copyright 2024 The Dragonfly Authors
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dragonfly-client-util/src/id_generator/mod.rs

@@ -1,5 +1,5 @@
 /*
- *     Copyright 2023 The Dragonfly Authors
+ *     Copyright 2024 The Dragonfly Authors
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dragonfly-client-util/src/lib.rs

@@ -1,5 +1,5 @@
 /*
- *     Copyright 2023 The Dragonfly Authors
+ *     Copyright 2024 The Dragonfly Authors
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

src/bin/dfdaemon/main.rs

@@ -30,7 +30,7 @@ use dragonfly_client::shutdown;
 use dragonfly_client::storage::Storage;
 use dragonfly_client::task::Task;
 use dragonfly_client::tracing::init_tracing;
-use dragonfly_client::utils::id_generator::IDGenerator;
+use dragonfly_client_util::id_generator::IDGenerator;
 use std::net::SocketAddr;
 use std::path::PathBuf;
 use std::sync::Arc;

src/bin/dfget/main.rs

@@ -21,8 +21,8 @@ use dragonfly_client::config::{self, dfdaemon, dfget};
 use dragonfly_client::grpc::dfdaemon_download::DfdaemonDownloadClient;
 use dragonfly_client::grpc::health::HealthClient;
 use dragonfly_client::tracing::init_tracing;
-use dragonfly_client::utils::http::header_vec_to_hashmap;
 use dragonfly_client_core::Error;
+use dragonfly_client_util::http::header_vec_to_hashmap;
 use fslock::LockFile;
 use indicatif::{ProgressBar, ProgressState, ProgressStyle};
 use std::path::PathBuf;

src/grpc/dfdaemon_download.rs

@@ -16,7 +16,6 @@
 
 use crate::shutdown;
 use crate::task;
-use crate::utils::http::{get_range, hashmap_to_reqwest_headermap, reqwest_headermap_to_hashmap};
 use dragonfly_api::common::v2::Task;
 use dragonfly_api::dfdaemon::v2::{
     dfdaemon_download_client::DfdaemonDownloadClient as DfdaemonDownloadGRPCClient,
@@ -31,6 +30,9 @@ use dragonfly_api::scheduler::v2::{
     LeaveHostRequest as SchedulerLeaveHostRequest, StatTaskRequest as SchedulerStatTaskRequest,
 };
 use dragonfly_client_core::{Error as ClientError, Result as ClientResult};
+use dragonfly_client_util::http::{
+    get_range, hashmap_to_reqwest_headermap, reqwest_headermap_to_hashmap,
+};
 use std::path::Path;
 use std::path::PathBuf;
 use std::sync::Arc;

src/lib.rs

@@ -26,4 +26,3 @@ pub mod shutdown;
 pub mod storage;
 pub mod task;
 pub mod tracing;
-pub mod utils;

src/proxy/mod.rs

@@ -18,13 +18,6 @@ use crate::config::dfdaemon::{Config, Rule};
 use crate::grpc::dfdaemon_download::DfdaemonDownloadClient;
 use crate::shutdown;
 use crate::task::Task;
-use crate::utils::http::{
-    hashmap_to_hyper_header_map, hyper_headermap_to_reqwest_headermap, reqwest_headermap_to_hashmap,
-};
-use crate::utils::tls::{
-    generate_ca_cert_from_pem, generate_certs_from_pem, generate_self_signed_certs_by_ca_cert,
-    generate_simple_self_signed_certs,
-};
 use bytes::Bytes;
 use dragonfly_api::common::v2::{Download, TaskType};
 use dragonfly_api::dfdaemon::v2::{
@@ -32,6 +25,16 @@ use dragonfly_api::dfdaemon::v2::{
 };
 use dragonfly_api::errordetails::v2::Http;
 use dragonfly_client_core::{Error as ClientError, Result as ClientResult};
+use dragonfly_client_util::{
+    http::{
+        hashmap_to_hyper_header_map, hyper_headermap_to_reqwest_headermap,
+        reqwest_headermap_to_hashmap,
+    },
+    tls::{
+        generate_ca_cert_from_pem, generate_certs_from_pem, generate_self_signed_certs_by_ca_cert,
+        generate_simple_self_signed_certs,
+    },
+};
 use futures_util::TryStreamExt;
 use http_body_util::{combinators::BoxBody, BodyExt, Empty, Full, StreamBody};
 use hyper::body::Frame;

src/storage/metadata.rs

@@ -14,9 +14,9 @@
  * limitations under the License.
  */
 
-use crate::utils::http::reqwest_headermap_to_hashmap;
 use chrono::{NaiveDateTime, Utc};
 use dragonfly_client_core::{Error, Result};
+use dragonfly_client_util::http::reqwest_headermap_to_hashmap;
 use reqwest::header::{self, HeaderMap};
 use rocksdb::{
     BlockBasedOptions, Cache, ColumnFamily, IteratorMode, Options, TransactionDB,

src/storage/mod.rs

@@ -15,9 +15,9 @@
  */
 
 use crate::config::dfdaemon::Config;
-use crate::utils::digest::{Algorithm, Digest};
 use dragonfly_api::common::v2::Range;
 use dragonfly_client_core::{Error, Result};
+use dragonfly_client_util::digest::{Algorithm, Digest};
 use reqwest::header::HeaderMap;
 use std::path::Path;
 use std::sync::Arc;

src/task/mod.rs

@@ -18,8 +18,6 @@ use crate::backend::http::{Request as HTTPRequest, HTTP};
 use crate::config::dfdaemon::Config;
 use crate::grpc::{scheduler::SchedulerClient, REQUEST_TIMEOUT};
 use crate::storage::{metadata, Storage};
-use crate::utils::http::{hashmap_to_reqwest_headermap, reqwest_headermap_to_hashmap};
-use crate::utils::id_generator::IDGenerator;
 use dragonfly_api::common::v2::Range;
 use dragonfly_api::common::v2::{Download, Peer, Piece, TrafficType};
 use dragonfly_api::dfdaemon::{
@@ -39,6 +37,10 @@ use dragonfly_api::scheduler::v2::{
 use dragonfly_client_core::{
     DownloadFromRemotePeerFailed, Error, HTTPError, Result as ClientResult,
 };
+use dragonfly_client_util::{
+    http::{hashmap_to_reqwest_headermap, reqwest_headermap_to_hashmap},
+    id_generator::IDGenerator,
+};
 use reqwest::header::HeaderMap;
 use std::path::Path;
 use std::sync::Arc;