diff --git a/Cargo.lock b/Cargo.lock index e1bf3e6c..8ae84379 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -612,18 +612,16 @@ dependencies = [ "dashmap", "dragonfly-api", "dragonfly-client-core", + "dragonfly-client-util", "fs2", "fslock", "futures", "futures-util", "hashring", - "headers 0.4.0", - "hex", "home", "hostname", "http 1.0.0", "http-body-util", - "http-range-header", "humantime", "humantime-serde", "hyper 1.1.0", @@ -646,7 +644,6 @@ dependencies = [ "reqwest", "rocksdb", "rustls 0.22.2", - "rustls-pemfile 2.1.1", "rustls-pki-types", "serde", "serde_json", @@ -654,7 +651,6 @@ dependencies = [ "serde_yaml", "sha2", "sysinfo", - "thiserror", "tokio", "tokio-rustls 0.25.0", "tokio-stream", @@ -669,7 +665,6 @@ dependencies = [ "tracing-opentelemetry", "tracing-subscriber", "url", - "uuid", "validator", "warp", ] @@ -698,6 +693,25 @@ dependencies = [ "warp", ] +[[package]] +name = "dragonfly-client-util" +version = "0.1.17" +dependencies = [ + "dragonfly-api", + "dragonfly-client-core", + "hex", + "http-range-header", + "hyper 1.1.0", + "rcgen", + "reqwest", + "rustls-pemfile 2.1.1", + "rustls-pki-types", + "sha2", + "tracing", + "url", + "uuid", +] + [[package]] name = "either" version = "1.9.0" diff --git a/Cargo.toml b/Cargo.toml index ef240c64..e61d29d8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,7 @@ name = "dfstore" path = "src/bin/dfstore/main.rs" [workspace] -members = ["dragonfly-client-core"] +members = ["dragonfly-client-core", "dragonfly-client-util"] [workspace.package] version = "0.1.17" @@ -35,9 +35,40 @@ license = "Apache-2.0" readme = "README.md" edition = "2021" -[dependencies] +[workspace.dependencies] dragonfly-client-core = { path = "dragonfly-client-core" } +dragonfly-client-util = { path = "dragonfly-client-util" } +thiserror = "1.0" +dragonfly-api = "2.0.102" +reqwest = { version = "0.11.24", features = ["stream", "native-tls", "rustls-tls"] } +rcgen = { version = "0.12.1", features = ["x509-parser"] } +hyper = { version = "1.1", features = ["full"] } +hyper-util = { version = "0.1.2", features = ["client", "client-legacy", "tokio", "server-auto", "http1", "http2"] } +http-range-header = "0.4.0" +tracing = "0.1" +url = "2.4.0" +rustls = "0.22.2" +rustls-pki-types = "1.2.0" +rustls-pemfile = "2.1.1" +sha2 = "0.10" +uuid = { version = "1.4", features = ["v4"] } +hex = "0.4" +rocksdb = "0.22.0" +serde_yaml = "0.9" +serde_json = "1.0" +http = "1" +tonic = { version = "0.9.2", features = ["gzip"] } +tokio = { version = "1.29.1", features = ["full"] } +tokio-stream = "0.1.14" +validator = { version = "0.16", features = ["derive"] } +warp = "0.3.5" +headers = "0.4.0" +[dependencies] +dragonfly-client-core.workspace = true +dragonfly-client-util.workspace = true +rcgen.workspace = true +hyper-util.workspace = true clap = { version = "4.5.1", features = [ "derive" ] } home = "0.5.4" tracing = "0.1" @@ -51,7 +82,6 @@ serde_regex = "1.1.0" serde_yaml = "0.9" serde_json = "1.0" validator = { version = "0.16", features = ["derive"] } -thiserror = "1.0" opentelemetry = { version = "0.18.0", default-features = false, features = ["trace", "rt-tokio"] } opentelemetry-jaeger = { version = "0.17.0", features = ["rt-tokio"] } lazy_static = "1.4" @@ -74,8 +104,6 @@ dragonfly-api = "2.0.102" chrono = { version = "0.4.34", features = ["serde"] } sysinfo = "0.29.6" sha2 = "0.10" -hex = "0.4" -uuid = { version = "1.4", features = ["v4"] } url = "2.4.0" async-stream = "0.3.5" anyhow = "1.0.80" @@ -94,15 +122,10 @@ openssl = { version = "0.10", features = ["vendored"] } humantime-serde = "1.1.1" leaky-bucket = "1.0.1" hyper = { version = "1.1", features = ["full"] } -hyper-util = { version = "0.1.2", features = ["client", "client-legacy", "tokio", "server-auto", "http1", "http2"] } tokio-rustls = "0.25" hyper-rustls = { version = "0.26", features = [ "http1", "http2", "logging" ] } http-body-util = "0.1.0" regex = "1.10.2" -http-range-header = "0.4.0" futures-util = "0.3.30" -headers = "0.4.0" rustls = "0.22.2" -rustls-pemfile = "2.1.1" rustls-pki-types = "1.2.0" -rcgen = { version = "0.12.1", features = ["x509-parser"] } diff --git a/dragonfly-client-core/Cargo.toml b/dragonfly-client-core/Cargo.toml index 9e17f3bb..009e5a83 100644 --- a/dragonfly-client-core/Cargo.toml +++ b/dragonfly-client-core/Cargo.toml @@ -1,7 +1,6 @@ [package] name = "dragonfly-client-core" -description = "Core library for the client" - +description = "Core library for the dragonfly client" version.workspace = true authors.workspace = true homepage.workspace = true @@ -11,21 +10,21 @@ license.workspace = true edition.workspace = true [dependencies] -thiserror = "1.0" -rocksdb = "0.22.0" -serde_yaml = "0.9" -serde_json = "1.0" -url = "2.4.0" -hyper = "1.1" -hyper-util = "0.1.2" -rustls = "0.22.2" -reqwest = "0.11.24" -rcgen = "0.12.1" -http = "1" -http-range-header = "0.4.0" -tonic = "0.9.2" -tokio = "1.29.1" -tokio-stream = "0.1.14" -validator = "0.16" -warp = "0.3.5" -headers = "0.4.0" +rcgen.workspace = true +reqwest.workspace = true +hyper.workspace = true +http-range-header.workspace = true +url.workspace = true +hyper-util.workspace = true +thiserror.workspace = true +rocksdb.workspace = true +serde_yaml.workspace = true +serde_json.workspace = true +rustls.workspace = true +http.workspace = true +tonic.workspace = true +tokio.workspace = true +tokio-stream.workspace = true +validator.workspace = true +warp.workspace = true +headers.workspace = true diff --git a/dragonfly-client-util/Cargo.toml b/dragonfly-client-util/Cargo.toml new file mode 100644 index 00000000..12469161 --- /dev/null +++ b/dragonfly-client-util/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "dragonfly-client-util" +description = "Utility library for the dragonfly client" +version.workspace = true +authors.workspace = true +homepage.workspace = true +repository.workspace = true +keywords.workspace = true +license.workspace = true +readme.workspace = true +edition.workspace = true + +[dependencies] +dragonfly-client-core.workspace = true +dragonfly-api.workspace = true +reqwest.workspace = true +hyper.workspace = true +http-range-header.workspace = true +tracing.workspace = true +url.workspace = true +rcgen.workspace = true +rustls-pki-types.workspace = true +rustls-pemfile.workspace = true +sha2.workspace = true +uuid.workspace = true +hex.workspace = true diff --git a/src/utils/digest.rs b/dragonfly-client-util/src/digest/mod.rs similarity index 98% rename from src/utils/digest.rs rename to dragonfly-client-util/src/digest/mod.rs index bd5513ea..0772bf27 100644 --- a/src/utils/digest.rs +++ b/dragonfly-client-util/src/digest/mod.rs @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Dragonfly Authors + * Copyright 2024 The Dragonfly Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/utils/http.rs b/dragonfly-client-util/src/http/mod.rs similarity index 99% rename from src/utils/http.rs rename to dragonfly-client-util/src/http/mod.rs index 78c7d368..c32e6e5f 100644 --- a/src/utils/http.rs +++ b/dragonfly-client-util/src/http/mod.rs @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Dragonfly Authors + * Copyright 2024 The Dragonfly Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/utils/id_generator.rs b/dragonfly-client-util/src/id_generator/mod.rs similarity index 98% rename from src/utils/id_generator.rs rename to dragonfly-client-util/src/id_generator/mod.rs index 6e427220..218d6f03 100644 --- a/src/utils/id_generator.rs +++ b/dragonfly-client-util/src/id_generator/mod.rs @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Dragonfly Authors + * Copyright 2024 The Dragonfly Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/utils/mod.rs b/dragonfly-client-util/src/lib.rs similarity index 93% rename from src/utils/mod.rs rename to dragonfly-client-util/src/lib.rs index 510f5dce..447dfdcf 100644 --- a/src/utils/mod.rs +++ b/dragonfly-client-util/src/lib.rs @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Dragonfly Authors + * Copyright 2024 The Dragonfly Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/utils/tls.rs b/dragonfly-client-util/src/tls/mod.rs similarity index 100% rename from src/utils/tls.rs rename to dragonfly-client-util/src/tls/mod.rs diff --git a/src/bin/dfdaemon/main.rs b/src/bin/dfdaemon/main.rs index d3155a94..aaf68ee9 100644 --- a/src/bin/dfdaemon/main.rs +++ b/src/bin/dfdaemon/main.rs @@ -30,7 +30,7 @@ use dragonfly_client::shutdown; use dragonfly_client::storage::Storage; use dragonfly_client::task::Task; use dragonfly_client::tracing::init_tracing; -use dragonfly_client::utils::id_generator::IDGenerator; +use dragonfly_client_util::id_generator::IDGenerator; use std::net::SocketAddr; use std::path::PathBuf; use std::sync::Arc; diff --git a/src/bin/dfget/main.rs b/src/bin/dfget/main.rs index 152c12d3..616d2a4d 100644 --- a/src/bin/dfget/main.rs +++ b/src/bin/dfget/main.rs @@ -21,8 +21,8 @@ use dragonfly_client::config::{self, dfdaemon, dfget}; use dragonfly_client::grpc::dfdaemon_download::DfdaemonDownloadClient; use dragonfly_client::grpc::health::HealthClient; use dragonfly_client::tracing::init_tracing; -use dragonfly_client::utils::http::header_vec_to_hashmap; use dragonfly_client_core::Error; +use dragonfly_client_util::http::header_vec_to_hashmap; use fslock::LockFile; use indicatif::{ProgressBar, ProgressState, ProgressStyle}; use std::path::PathBuf; diff --git a/src/grpc/dfdaemon_download.rs b/src/grpc/dfdaemon_download.rs index 3a64d272..147f0f29 100644 --- a/src/grpc/dfdaemon_download.rs +++ b/src/grpc/dfdaemon_download.rs @@ -16,7 +16,6 @@ use crate::shutdown; use crate::task; -use crate::utils::http::{get_range, hashmap_to_reqwest_headermap, reqwest_headermap_to_hashmap}; use dragonfly_api::common::v2::Task; use dragonfly_api::dfdaemon::v2::{ dfdaemon_download_client::DfdaemonDownloadClient as DfdaemonDownloadGRPCClient, @@ -31,6 +30,9 @@ use dragonfly_api::scheduler::v2::{ LeaveHostRequest as SchedulerLeaveHostRequest, StatTaskRequest as SchedulerStatTaskRequest, }; use dragonfly_client_core::{Error as ClientError, Result as ClientResult}; +use dragonfly_client_util::http::{ + get_range, hashmap_to_reqwest_headermap, reqwest_headermap_to_hashmap, +}; use std::path::Path; use std::path::PathBuf; use std::sync::Arc; diff --git a/src/lib.rs b/src/lib.rs index 0ad796c4..363310ff 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -26,4 +26,3 @@ pub mod shutdown; pub mod storage; pub mod task; pub mod tracing; -pub mod utils; diff --git a/src/proxy/mod.rs b/src/proxy/mod.rs index 62ff27be..2645671a 100644 --- a/src/proxy/mod.rs +++ b/src/proxy/mod.rs @@ -18,13 +18,6 @@ use crate::config::dfdaemon::{Config, Rule}; use crate::grpc::dfdaemon_download::DfdaemonDownloadClient; use crate::shutdown; use crate::task::Task; -use crate::utils::http::{ - hashmap_to_hyper_header_map, hyper_headermap_to_reqwest_headermap, reqwest_headermap_to_hashmap, -}; -use crate::utils::tls::{ - generate_ca_cert_from_pem, generate_certs_from_pem, generate_self_signed_certs_by_ca_cert, - generate_simple_self_signed_certs, -}; use bytes::Bytes; use dragonfly_api::common::v2::{Download, TaskType}; use dragonfly_api::dfdaemon::v2::{ @@ -32,6 +25,16 @@ use dragonfly_api::dfdaemon::v2::{ }; use dragonfly_api::errordetails::v2::Http; use dragonfly_client_core::{Error as ClientError, Result as ClientResult}; +use dragonfly_client_util::{ + http::{ + hashmap_to_hyper_header_map, hyper_headermap_to_reqwest_headermap, + reqwest_headermap_to_hashmap, + }, + tls::{ + generate_ca_cert_from_pem, generate_certs_from_pem, generate_self_signed_certs_by_ca_cert, + generate_simple_self_signed_certs, + }, +}; use futures_util::TryStreamExt; use http_body_util::{combinators::BoxBody, BodyExt, Empty, Full, StreamBody}; use hyper::body::Frame; diff --git a/src/storage/metadata.rs b/src/storage/metadata.rs index e4c3e660..009f65a8 100644 --- a/src/storage/metadata.rs +++ b/src/storage/metadata.rs @@ -14,9 +14,9 @@ * limitations under the License. */ -use crate::utils::http::reqwest_headermap_to_hashmap; use chrono::{NaiveDateTime, Utc}; use dragonfly_client_core::{Error, Result}; +use dragonfly_client_util::http::reqwest_headermap_to_hashmap; use reqwest::header::{self, HeaderMap}; use rocksdb::{ BlockBasedOptions, Cache, ColumnFamily, IteratorMode, Options, TransactionDB, diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 690ec82b..5f274ddf 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -15,9 +15,9 @@ */ use crate::config::dfdaemon::Config; -use crate::utils::digest::{Algorithm, Digest}; use dragonfly_api::common::v2::Range; use dragonfly_client_core::{Error, Result}; +use dragonfly_client_util::digest::{Algorithm, Digest}; use reqwest::header::HeaderMap; use std::path::Path; use std::sync::Arc; diff --git a/src/task/mod.rs b/src/task/mod.rs index 25e204a7..09f05e03 100644 --- a/src/task/mod.rs +++ b/src/task/mod.rs @@ -18,8 +18,6 @@ use crate::backend::http::{Request as HTTPRequest, HTTP}; use crate::config::dfdaemon::Config; use crate::grpc::{scheduler::SchedulerClient, REQUEST_TIMEOUT}; use crate::storage::{metadata, Storage}; -use crate::utils::http::{hashmap_to_reqwest_headermap, reqwest_headermap_to_hashmap}; -use crate::utils::id_generator::IDGenerator; use dragonfly_api::common::v2::Range; use dragonfly_api::common::v2::{Download, Peer, Piece, TrafficType}; use dragonfly_api::dfdaemon::{ @@ -39,6 +37,10 @@ use dragonfly_api::scheduler::v2::{ use dragonfly_client_core::{ DownloadFromRemotePeerFailed, Error, HTTPError, Result as ClientResult, }; +use dragonfly_client_util::{ + http::{hashmap_to_reqwest_headermap, reqwest_headermap_to_hashmap}, + id_generator::IDGenerator, +}; use reqwest::header::HeaderMap; use std::path::Path; use std::sync::Arc;