From 2070c8156ef36c52ef5d8b2973adbbdd7be0d6cd Mon Sep 17 00:00:00 2001 From: Gaius Date: Tue, 2 Sep 2025 10:27:20 +0800 Subject: [PATCH] chore(dependencies): update dependencies and remove unused metakeys (#1322) * chore(dependencies): update dependencies and remove unused metakeys Signed-off-by: Gaius * build(deps): update dependencies and remove unused features Signed-off-by: Gaius --------- Signed-off-by: Gaius --- Cargo.lock | 645 ++++++++++++------ Cargo.toml | 18 +- dragonfly-client-backend/src/hdfs.rs | 3 +- dragonfly-client-backend/src/http.rs | 5 + dragonfly-client-backend/src/lib.rs | 6 - .../src/object_storage.rs | 129 ++-- dragonfly-client/src/bin/dfget/main.rs | 8 + 7 files changed, 533 insertions(+), 281 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 005aba20..63da0d7a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -265,6 +265,29 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "aws-lc-rs" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c953fe1ba023e6b7730c0d4b031d06f267f23a46167dcbd40316644b10a17ba" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbfd150b5dbdb988bcc8fb1fe787eb6b7ee6180ca24da683b61ea5405f3d43ff" +dependencies = [ + "bindgen", + "cc", + "cmake", + "dunce", + "fs_extra", +] + [[package]] name = "axum" version = "0.7.5" @@ -287,7 +310,7 @@ dependencies = [ "rustversion", "serde", "sync_wrapper 1.0.1", - "tower", + "tower 0.4.13", "tower-layer", "tower-service", ] @@ -314,13 +337,12 @@ dependencies = [ [[package]] name = "backon" -version = "0.4.4" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d67782c3f868daa71d3533538e98a8e13713231969def7536e8039606fc46bf0" +checksum = "592277618714fbcecda9a02ba7a8781f319d26532a88553bbacc77ba5d2b3a8d" dependencies = [ "fastrand", - "futures-core", - "pin-project", + "gloo-timers", "tokio", ] @@ -368,9 +390,9 @@ dependencies = [ [[package]] name = "bindgen" -version = "0.69.4" +version = "0.69.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" dependencies = [ "bitflags 2.4.2", "cexpr", @@ -378,12 +400,15 @@ dependencies = [ "itertools", "lazy_static", "lazycell", + "log", + "prettyplease", "proc-macro2", "quote", "regex", - "rustc-hash", + "rustc-hash 1.1.0", "shlex", "syn 2.0.90", + "which", ] [[package]] @@ -538,6 +563,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "chrono" version = "0.4.41" @@ -641,6 +672,15 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" +[[package]] +name = "cmake" +version = "0.1.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" +dependencies = [ + "cc", +] + [[package]] name = "colorchoice" version = "1.0.0" @@ -813,6 +853,12 @@ dependencies = [ "itertools", ] +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + [[package]] name = "crossbeam-channel" version = "0.5.15" @@ -978,9 +1024,9 @@ dependencies = [ [[package]] name = "dragonfly-api" -version = "2.1.59" +version = "2.1.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd73e56ce17d3332f61fe4a2536d61877800140b5c970d608a60b819f39f7451" +checksum = "4936a45b8c07b0b6c88fb451bb61909d55a4a0706c637326268da06ebb5240b2" dependencies = [ "prost 0.13.5", "prost-types 0.14.1", @@ -1037,7 +1083,7 @@ dependencies = [ "rcgen", "reqwest", "rolling-file", - "rustls 0.22.4", + "rustls", "rustls-pki-types", "serde", "serde_json", @@ -1047,13 +1093,13 @@ dependencies = [ "termion", "tikv-jemallocator", "tokio", - "tokio-rustls 0.25.0", + "tokio-rustls", "tokio-stream", "tokio-util", "tonic", "tonic-health", "tonic-reflection", - "tower", + "tower 0.4.13", "tracing", "tracing-appender", "tracing-opentelemetry", @@ -1082,12 +1128,12 @@ dependencies = [ "reqwest-middleware", "reqwest-retry", "reqwest-tracing", - "rustls 0.22.4", + "rustls", "rustls-pemfile 2.2.0", "rustls-pki-types", "tempfile", "tokio", - "tokio-rustls 0.25.0", + "tokio-rustls", "tokio-util", "tonic", "tracing", @@ -1206,7 +1252,7 @@ dependencies = [ "rcgen", "reqwest", "rustix 1.0.8", - "rustls 0.22.4", + "rustls", "rustls-pemfile 2.2.0", "rustls-pki-types", "sha2", @@ -1218,6 +1264,12 @@ dependencies = [ "uuid", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "either" version = "1.9.0" @@ -1320,12 +1372,6 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" -[[package]] -name = "flagset" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdeb3aa5e95cf9aabc17f060cfa0ced7b83f042390760ca53bf09df9968acaa1" - [[package]] name = "flate2" version = "1.0.28" @@ -1382,6 +1428,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures" version = "0.3.31" @@ -1471,6 +1523,20 @@ dependencies = [ "slab", ] +[[package]] +name = "generator" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "605183a538e3e2a9c1038635cc5c2d194e2ee8fd0d1b66b8349fad7dbacce5a2" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows 0.60.0", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -1501,8 +1567,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi 0.13.3+wasi-0.2.2", + "wasm-bindgen", "windows-targets 0.52.6", ] @@ -1518,6 +1586,18 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "h2" version = "0.3.26" @@ -1694,9 +1774,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hickory-proto" -version = "0.24.4" +version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92652067c9ce6f66ce53cc38d1169daa36e6e7eb7dd3b63b5103bd9d97117248" +checksum = "f8a6fe56c0038198998a6f217ca4e7ef3a5e51f46163bd6dd60b5c71ca6c6502" dependencies = [ "async-trait", "cfg-if", @@ -1708,8 +1788,9 @@ dependencies = [ "idna 1.0.3", "ipnet", "once_cell", - "rand 0.8.5", - "thiserror 1.0.69", + "rand 0.9.1", + "ring 0.17.7", + "thiserror 2.0.16", "tinyvec", "tokio", "tracing", @@ -1718,21 +1799,21 @@ dependencies = [ [[package]] name = "hickory-resolver" -version = "0.24.4" +version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbb117a1ca520e111743ab2f6688eddee69db4e0ea242545a604dce8a66fd22e" +checksum = "dc62a9a99b0bfb44d2ab95a7208ac952d31060efc16241c87eaf36406fecf87a" dependencies = [ "cfg-if", "futures-util", "hickory-proto", "ipconfig", - "lru-cache", + "moka", "once_cell", "parking_lot 0.12.1", - "rand 0.8.5", + "rand 0.9.1", "resolv-conf", "smallvec", - "thiserror 1.0.69", + "thiserror 2.0.16", "tokio", "tracing", ] @@ -1913,21 +1994,21 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.26.0" +version = "0.27.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ - "futures-util", "http 1.3.1", "hyper 1.6.0", "hyper-util", "log", - "rustls 0.22.4", + "rustls", "rustls-native-certs", "rustls-pki-types", "tokio", - "tokio-rustls 0.25.0", + "tokio-rustls", "tower-service", + "webpki-roots", ] [[package]] @@ -1943,28 +2024,13 @@ dependencies = [ "tower-service", ] -[[package]] -name = "hyper-tls" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" -dependencies = [ - "bytes", - "http-body-util", - "hyper 1.6.0", - "hyper-util", - "native-tls", - "tokio", - "tokio-native-tls", - "tower-service", -] - [[package]] name = "hyper-util" version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d9b05277c7e8da2c93a568989bb6207bef0112e8d17df7a6eda4a3cf143bc5e" dependencies = [ + "base64 0.22.1", "bytes", "futures-channel", "futures-core", @@ -1972,7 +2038,9 @@ dependencies = [ "http 1.3.1", "http-body 1.0.0", "hyper 1.6.0", + "ipnet", "libc", + "percent-encoding", "pin-project-lite", "socket2 0.6.0", "tokio", @@ -2257,7 +2325,7 @@ dependencies = [ "socket2 0.5.9", "widestring", "windows-sys 0.48.0", - "winreg 0.50.0", + "winreg", ] [[package]] @@ -2275,6 +2343,16 @@ dependencies = [ "serde", ] +[[package]] +name = "iri-string" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "is-terminal" version = "0.4.12" @@ -2329,10 +2407,11 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.67" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a1d36f1235bc969acba30b7f5990b864423a6068a10f7c90ae8f0112e3a59d1" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" dependencies = [ + "once_cell", "wasm-bindgen", ] @@ -2437,12 +2516,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "linked-hash-map" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" - [[package]] name = "linux-raw-sys" version = "0.4.14" @@ -2489,6 +2562,19 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + [[package]] name = "lru" version = "0.12.5" @@ -2499,13 +2585,10 @@ dependencies = [ ] [[package]] -name = "lru-cache" +name = "lru-slab" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" -dependencies = [ - "linked-hash-map", -] +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] name = "lz4-sys" @@ -2619,6 +2702,25 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "moka" +version = "0.12.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9321642ca94a4282428e6ea4af8cc2ca4eac48ac7a6a4ea8f33f76d0ce70926" +dependencies = [ + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "loom", + "parking_lot 0.12.1", + "portable-atomic", + "rustc_version", + "smallvec", + "tagptr", + "thiserror 1.0.69", + "uuid", +] + [[package]] name = "multer" version = "2.1.0" @@ -2643,24 +2745,6 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" -[[package]] -name = "native-tls" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" -dependencies = [ - "lazy_static", - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - [[package]] name = "neli" version = "0.6.4" @@ -2876,9 +2960,13 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.19.0" +version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +dependencies = [ + "critical-section", + "portable-atomic", +] [[package]] name = "oorandom" @@ -2888,26 +2976,23 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" [[package]] name = "opendal" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "615d41187deea0ea7fab5b48e9afef6ae8fc742fdcfa248846ee3d92ff71e986" +version = "0.54.0" +source = "git+https://github.com/apache/opendal.git?rev=7c9c505b4880cad14824b2a22090723ef68ecce5#7c9c505b4880cad14824b2a22090723ef68ecce5" dependencies = [ "anyhow", - "async-trait", "backon", "base64 0.22.1", "bytes", "chrono", "crc32c", - "flagset", "futures", "getrandom 0.2.12", "http 1.3.1", + "http-body 1.0.0", "log", "md-5", - "once_cell", "percent-encoding", - "quick-xml 0.36.1", + "quick-xml 0.38.3", "reqsign", "reqwest", "serde", @@ -3389,9 +3474,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.6.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" +checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" [[package]] name = "powerfmt" @@ -3769,9 +3854,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.35.0" +version = "0.37.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86e446ed58cef1bbfe847bc2fda0e2e4ea9f0e57b90c507d4781292590d72a4e" +checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" dependencies = [ "memchr", "serde", @@ -3779,14 +3864,69 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.36.1" +version = "0.38.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96a05e2e8efddfa51a84ca47cec303fac86c8541b686d37cac5efc0e094417bc" +checksum = "42a232e7487fc2ef313d96dde7948e7a3c05101870d8985e4fd8d26aedd27b89" dependencies = [ "memchr", "serde", ] +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash 2.1.1", + "rustls", + "socket2 0.6.0", + "thiserror 2.0.16", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" +dependencies = [ + "bytes", + "getrandom 0.3.1", + "lru-slab", + "rand 0.9.1", + "ring 0.17.7", + "rustc-hash 2.1.1", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.16", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2 0.6.0", + "tracing", + "windows-sys 0.60.2", +] + [[package]] name = "quote" version = "1.0.35" @@ -3952,9 +4092,9 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "reqsign" -version = "0.16.0" +version = "0.16.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03dd4ba7c3901dd43e6b8c7446a760d45bc1ea4301002e1a6fa48f97c3a796fa" +checksum = "43451dbf3590a7590684c25fb8d12ecdcc90ed3ac123433e500447c7d77ed701" dependencies = [ "anyhow", "async-trait", @@ -3970,7 +4110,7 @@ dependencies = [ "log", "once_cell", "percent-encoding", - "quick-xml 0.35.0", + "quick-xml 0.37.5", "rand 0.8.5", "reqwest", "rsa", @@ -3979,50 +4119,45 @@ dependencies = [ "serde_json", "sha1", "sha2", + "tokio", ] [[package]] name = "reqwest" -version = "0.12.4" +version = "0.12.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10" +checksum = "d429f34c8092b2d42c7c93cec323bb4adeb7c67698f70839adec842ec10c7ceb" dependencies = [ "async-compression", "base64 0.22.1", "bytes", - "encoding_rs", "futures-channel", "futures-core", "futures-util", - "h2 0.4.2", "hickory-resolver", "http 1.3.1", "http-body 1.0.0", "http-body-util", "hyper 1.6.0", "hyper-rustls", - "hyper-tls", "hyper-util", - "ipnet", "js-sys", "log", - "mime", - "native-tls", "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.22.4", - "rustls-pemfile 2.2.0", + "quinn", + "rustls", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", - "sync_wrapper 0.1.2", - "system-configuration", + "sync_wrapper 1.0.1", "tokio", - "tokio-native-tls", - "tokio-rustls 0.25.0", + "tokio-rustls", "tokio-util", + "tower 0.5.2", + "tower-http", "tower-service", "url", "wasm-bindgen", @@ -4030,7 +4165,6 @@ dependencies = [ "wasm-streams", "web-sys", "webpki-roots", - "winreg 0.52.0", ] [[package]] @@ -4202,6 +4336,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + [[package]] name = "rustc_version" version = "0.4.1" @@ -4248,24 +4388,11 @@ dependencies = [ [[package]] name = "rustls" -version = "0.22.4" +version = "0.23.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" -dependencies = [ - "log", - "ring 0.17.7", - "rustls-pki-types", - "rustls-webpki", - "subtle", - "zeroize", -] - -[[package]] -name = "rustls" -version = "0.23.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" +checksum = "c0ebcbd2f03de0fc1122ad9bb24b127a5a6cd51d72604a3f3c50ac459762b6cc" dependencies = [ + "aws-lc-rs", "log", "once_cell", "ring 0.17.7", @@ -4277,9 +4404,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.7.3" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5" +checksum = "fcaf18a4f2be7326cd874a5fa579fae794320a0f388d365dca7e480e55f83f8a" dependencies = [ "openssl-probe", "rustls-pemfile 2.2.0", @@ -4312,15 +4439,17 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" dependencies = [ + "web-time", "zeroize", ] [[package]] name = "rustls-webpki" -version = "0.102.7" +version = "0.103.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84678086bd54edf2b415183ed7a94d0efb049f1b646a33e22a36f3794be6ae56" +checksum = "0a17884ae0c1b773f1ccd2bd4a8c72f16da897310a98b0e84bf349ad5ead92fc" dependencies = [ + "aws-lc-rs", "ring 0.17.7", "rustls-pki-types", "untrusted 0.9.0", @@ -4704,6 +4833,9 @@ name = "sync_wrapper" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +dependencies = [ + "futures-core", +] [[package]] name = "synstructure" @@ -4741,27 +4873,6 @@ dependencies = [ "windows 0.57.0", ] -[[package]] -name = "system-configuration" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" -dependencies = [ - "bitflags 1.3.2", - "core-foundation", - "system-configuration-sys", -] - -[[package]] -name = "system-configuration-sys" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "tabled" version = "0.20.0" @@ -4786,6 +4897,12 @@ dependencies = [ "syn 2.0.90", ] +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + [[package]] name = "tempfile" version = "3.20.0" @@ -5008,35 +5125,13 @@ dependencies = [ "syn 2.0.90", ] -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - [[package]] name = "tokio-rustls" -version = "0.25.0" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" +checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" dependencies = [ - "rustls 0.22.4", - "rustls-pki-types", - "tokio", -] - -[[package]] -name = "tokio-rustls" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" -dependencies = [ - "rustls 0.23.12", - "rustls-pki-types", + "rustls", "tokio", ] @@ -5128,9 +5223,9 @@ dependencies = [ "rustls-pemfile 2.2.0", "socket2 0.5.9", "tokio", - "tokio-rustls 0.26.0", + "tokio-rustls", "tokio-stream", - "tower", + "tower 0.4.13", "tower-layer", "tower-service", "tracing", @@ -5195,6 +5290,39 @@ dependencies = [ "tracing", ] +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper 1.0.1", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" +dependencies = [ + "bitflags 2.4.2", + "bytes", + "futures-util", + "http 1.3.1", + "http-body 1.0.0", + "iri-string", + "pin-project-lite", + "tower 0.5.2", + "tower-layer", + "tower-service", +] + [[package]] name = "tower-layer" version = "0.3.3" @@ -5475,12 +5603,14 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.16.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" +checksum = "f33196643e165781c20a5ead5582283a7dacbb87855d867fbc2df3f81eddc1be" dependencies = [ "getrandom 0.3.1", + "js-sys", "serde", + "wasm-bindgen", ] [[package]] @@ -5610,23 +5740,24 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.90" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1223296a201415c7fad14792dbefaace9bd52b62d33453ade1c5b5f07555406" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" dependencies = [ "cfg-if", + "once_cell", + "rustversion", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.90" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcdc935b63408d58a32f8cc9738a0bffd8f05cc7c002086c6ef20b7312ad9dcd" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" dependencies = [ "bumpalo", "log", - "once_cell", "proc-macro2", "quote", "syn 2.0.90", @@ -5647,9 +5778,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.90" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e4c238561b2d428924c49815533a8b9121c664599558a5d9ec51f8a1740a999" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5657,9 +5788,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.90" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", @@ -5670,9 +5801,12 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.90" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] [[package]] name = "wasm-streams" @@ -5724,9 +5858,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.26.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3de34ae270483955a94f4b21bdaaeb83d508bb84a01435f393818edb0012009" +checksum = "7e8983c3ab33d6fb807cfcdad2491c4ea8cbc8ed839181c7dfd9c67c83e261b2" dependencies = [ "rustls-pki-types", ] @@ -5800,6 +5934,28 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddf874e74c7a99773e62b1c671427abf01a425e77c3d3fb9fb1e4883ea934529" +dependencies = [ + "windows-collections", + "windows-core 0.60.1", + "windows-future", + "windows-link", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5467f79cc1ba3f52ebb2ed41dbb459b8e7db636cc3429458d9a852e15bc24dec" +dependencies = [ + "windows-core 0.60.1", +] + [[package]] name = "windows-core" version = "0.52.0" @@ -5815,12 +5971,35 @@ version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d" dependencies = [ - "windows-implement", - "windows-interface", - "windows-result", + "windows-implement 0.57.0", + "windows-interface 0.57.0", + "windows-result 0.1.2", "windows-targets 0.52.6", ] +[[package]] +name = "windows-core" +version = "0.60.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca21a92a9cae9bf4ccae5cf8368dce0837100ddf6e6d57936749e85f152f6247" +dependencies = [ + "windows-implement 0.59.0", + "windows-interface 0.59.1", + "windows-link", + "windows-result 0.3.1", + "windows-strings", +] + +[[package]] +name = "windows-future" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a787db4595e7eb80239b74ce8babfb1363d8e343ab072f2ffe901400c03349f0" +dependencies = [ + "windows-core 0.60.1", + "windows-link", +] + [[package]] name = "windows-implement" version = "0.57.0" @@ -5832,6 +6011,17 @@ dependencies = [ "syn 2.0.90", ] +[[package]] +name = "windows-implement" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83577b051e2f49a058c308f17f273b570a6a758386fc291b5f6a934dd84e48c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "windows-interface" version = "0.57.0" @@ -5843,12 +6033,33 @@ dependencies = [ "syn 2.0.90", ] +[[package]] +name = "windows-interface" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "windows-link" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dccfd733ce2b1753b03b6d3c65edf020262ea35e20ccdf3e288043e6dd620e3" +[[package]] +name = "windows-numerics" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "005dea54e2f6499f2cee279b8f703b3cf3b5734a2d8d21867c8f44003182eeed" +dependencies = [ + "windows-core 0.60.1", + "windows-link", +] + [[package]] name = "windows-result" version = "0.1.2" @@ -5858,6 +6069,24 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-result" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06374efe858fab7e4f881500e6e86ec8bc28f9462c47e5a9941a0142ad86b189" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -6098,16 +6327,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "winreg" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" -dependencies = [ - "cfg-if", - "windows-sys 0.48.0", -] - [[package]] name = "wiremock" version = "0.6.4" diff --git a/Cargo.toml b/Cargo.toml index 010f42b4..75ed66d7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,13 +29,11 @@ dragonfly-client-storage = { path = "dragonfly-client-storage", version = "1.0.1 dragonfly-client-backend = { path = "dragonfly-client-backend", version = "1.0.15" } dragonfly-client-util = { path = "dragonfly-client-util", version = "1.0.15" } dragonfly-client-init = { path = "dragonfly-client-init", version = "1.0.15" } -dragonfly-api = "2.1.59" +dragonfly-api = "2.1.61" thiserror = "2.0" futures = "0.3.31" -reqwest = { version = "0.12.4", features = [ +reqwest = { version = "0.12.22", features = [ "stream", - "native-tls", - "default-tls", "rustls-tls", "gzip", "brotli", @@ -43,7 +41,7 @@ reqwest = { version = "0.12.4", features = [ "deflate", "blocking", "hickory-dns", -] } +], default-features = false } reqwest-middleware = "0.4" rcgen = { version = "0.12.1", features = ["x509-parser"] } hyper = { version = "1.6", features = ["full"] } @@ -55,11 +53,11 @@ hyper-util = { version = "0.1.16", features = [ "http1", "http2", ] } -hyper-rustls = { version = "0.26", features = ["http1", "http2", "logging"] } +hyper-rustls = { version = "0.27.7", features = ["http1", "http2", "logging", "ring"] } http-range-header = "0.4.2" tracing = "0.1" url = "2.5.4" -rustls = { version = "0.22.4", features = ["tls12"] } +rustls = { version = "0.23.31", features = ["tls12", "ring"] } rustls-pki-types = "1.12.0" rustls-pemfile = "2.2.0" sha2 = "0.10" @@ -83,7 +81,7 @@ humantime = "2.1.0" prost-wkt-types = "0.6" chrono = { version = "0.4.41", features = ["serde", "clock"] } openssl = { version = "0.10", features = ["vendored"] } -opendal = { version = "0.48.0", features = [ +opendal = { git = "https://github.com/apache/opendal.git", rev = "7c9c505b4880cad14824b2a22090723ef68ecce5", features = [ "services-s3", "services-azblob", "services-gcs", @@ -100,7 +98,9 @@ bytesize = { version = "1.3.3", features = ["serde"] } bytesize-serde = "0.2.1" percent-encoding = "2.3.2" tempfile = "3.20.0" -tokio-rustls = "0.25.0-alpha.4" +tokio-rustls = { version = "0.26.2", default-features = false, features = [ + "ring", +] } serde_json = "1.0.143" lru = "0.12.5" fs2 = "0.4.3" diff --git a/dragonfly-client-backend/src/hdfs.rs b/dragonfly-client-backend/src/hdfs.rs index f96ecc52..a8e94be5 100644 --- a/dragonfly-client-backend/src/hdfs.rs +++ b/dragonfly-client-backend/src/hdfs.rs @@ -17,7 +17,7 @@ use dragonfly_api::common; use dragonfly_client_core::error::BackendError; use dragonfly_client_core::{Error as ClientError, Result as ClientResult}; -use opendal::{layers::TimeoutLayer, Metakey, Operator}; +use opendal::{layers::TimeoutLayer, Operator}; use percent_encoding::percent_decode_str; use std::time::Duration; use tokio_util::io::StreamReader; @@ -110,7 +110,6 @@ impl super::Backend for Hdfs { operator .list_with(decoded_path.as_str()) .recursive(true) - .metakey(Metakey::ContentLength | Metakey::Mode) .await // Do the list op here. .map_err(|err| { error!( diff --git a/dragonfly-client-backend/src/http.rs b/dragonfly-client-backend/src/http.rs index 18810ba4..6861f79e 100644 --- a/dragonfly-client-backend/src/http.rs +++ b/dragonfly-client-backend/src/http.rs @@ -44,6 +44,9 @@ pub struct HTTP { impl HTTP { /// new returns a new HTTP. pub fn new(scheme: &str) -> Result { + // Initialize the ring as the default TLS provider. + let _ = rustls::crypto::ring::default_provider().install_default(); + // Default TLS client config with no validation. let client_config_builder = rustls::ClientConfig::builder() .dangerous() @@ -408,6 +411,8 @@ TrIVG3cErZoBC6zqBs/Ibe9q3gdHGqS3QLAKy/k= /// Start a https server with given public key and private key. async fn start_https_server(cert_pem: &str, key_pem: &str) -> String { + let _ = rustls::crypto::ring::default_provider().install_default(); + let server_certs = load_certs_from_pem(cert_pem).unwrap(); let server_key = load_key_from_pem(key_pem).unwrap(); diff --git a/dragonfly-client-backend/src/lib.rs b/dragonfly-client-backend/src/lib.rs index 78778f13..99544e1b 100644 --- a/dragonfly-client-backend/src/lib.rs +++ b/dragonfly-client-backend/src/lib.rs @@ -39,12 +39,6 @@ const POOL_MAX_IDLE_PER_HOST: usize = 1024; /// KEEP_ALIVE_INTERVAL is the keep alive interval for TCP connection. const KEEP_ALIVE_INTERVAL: Duration = Duration::from_secs(60); -/// HTTP2_KEEP_ALIVE_INTERVAL is the interval for HTTP2 keep alive. -const HTTP2_KEEP_ALIVE_INTERVAL: Duration = Duration::from_secs(300); - -/// HTTP2_KEEP_ALIVE_TIMEOUT is the timeout for HTTP2 keep alive. -const HTTP2_KEEP_ALIVE_TIMEOUT: Duration = Duration::from_secs(20); - /// MAX_RETRY_TIMES is the max retry times for the request. const MAX_RETRY_TIMES: u32 = 1; diff --git a/dragonfly-client-backend/src/object_storage.rs b/dragonfly-client-backend/src/object_storage.rs index 9b7bc8c2..0928c513 100644 --- a/dragonfly-client-backend/src/object_storage.rs +++ b/dragonfly-client-backend/src/object_storage.rs @@ -17,7 +17,11 @@ use dragonfly_api::common; use dragonfly_client_core::error::BackendError; use dragonfly_client_core::{Error as ClientError, Result as ClientResult}; -use opendal::{layers::TimeoutLayer, raw::HttpClient, Metakey, Operator}; +use opendal::{ + layers::{HttpClientLayer, TimeoutLayer}, + raw::HttpClient, + Operator, +}; use percent_encoding::percent_decode_str; use std::fmt; use std::result::Result; @@ -178,6 +182,9 @@ pub struct ObjectStorage { impl ObjectStorage { /// Returns ObjectStorage that implements the Backend trait. pub fn new(scheme: Scheme) -> ClientResult { + // Initialize the ring as the default TLS provider. + let _ = rustls::crypto::ring::default_provider().install_default(); + // Initialize the reqwest client. let client = reqwest::Client::builder() .gzip(true) @@ -187,9 +194,6 @@ impl ObjectStorage { .hickory_dns(true) .pool_max_idle_per_host(super::POOL_MAX_IDLE_PER_HOST) .tcp_keepalive(super::KEEP_ALIVE_INTERVAL) - .http2_keep_alive_timeout(super::HTTP2_KEEP_ALIVE_TIMEOUT) - .http2_keep_alive_interval(super::HTTP2_KEEP_ALIVE_INTERVAL) - .http2_keep_alive_while_idle(true) .build()?; Ok(Self { scheme, client }) @@ -254,7 +258,6 @@ impl ObjectStorage { builder = builder .access_key_id(access_key_id) .secret_access_key(access_key_secret) - .http_client(HttpClient::with(self.client.clone())) .bucket(&parsed_url.bucket) .region(region); @@ -270,6 +273,7 @@ impl ObjectStorage { Ok(Operator::new(builder)? .finish() + .layer(HttpClientLayer::new(HttpClient::with(self.client.clone()))) .layer(TimeoutLayer::new().with_timeout(timeout))) } @@ -282,9 +286,7 @@ impl ObjectStorage { ) -> ClientResult { // Initialize the GCS operator with the object storage. let mut builder = opendal::services::Gcs::default(); - builder = builder - .http_client(HttpClient::with(self.client.clone())) - .bucket(&parsed_url.bucket); + builder = builder.bucket(&parsed_url.bucket); // Configure the credentials using the local path to the credential file if provided. // Otherwise, configure using the Application Default Credentials (ADC). @@ -304,6 +306,7 @@ impl ObjectStorage { Ok(Operator::new(builder)? .finish() + .layer(HttpClientLayer::new(HttpClient::with(self.client.clone()))) .layer(TimeoutLayer::new().with_timeout(timeout))) } @@ -340,12 +343,12 @@ impl ObjectStorage { builder = builder .account_name(access_key_id) .account_key(access_key_secret) - .http_client(HttpClient::with(self.client.clone())) .container(&parsed_url.bucket) .endpoint(endpoint); Ok(Operator::new(builder)? .finish() + .layer(HttpClientLayer::new(HttpClient::with(self.client.clone()))) .layer(TimeoutLayer::new().with_timeout(timeout))) } @@ -356,40 +359,57 @@ impl ObjectStorage { object_storage: common::v2::ObjectStorage, timeout: Duration, ) -> ClientResult { - // OSS requires the access key id, access key secret, and endpoint. - let (Some(access_key_id), Some(access_key_secret), Some(endpoint)) = ( + if let (Some(access_key_id), Some(access_key_secret), Some(endpoint)) = ( &object_storage.access_key_id, &object_storage.access_key_secret, &object_storage.endpoint, - ) else { - return Err(ClientError::BackendError(Box::new(BackendError { - message: format!( - "{} {}", - self.scheme, - make_need_fields_message!(object_storage { - access_key_id, - access_key_secret, - endpoint - }) - ), - status_code: None, - header: None, - }))); - }; + ) { + // Initialize the OSS operator with the object storage. + let mut builder = opendal::services::Oss::default(); + builder = builder + .access_key_id(access_key_id) + .access_key_secret(access_key_secret) + .endpoint(endpoint) + .root("/") + .bucket(&parsed_url.bucket); - // Initialize the OSS operator with the object storage. - let mut builder = opendal::services::Oss::default(); - builder = builder - .access_key_id(access_key_id) - .access_key_secret(access_key_secret) - .endpoint(endpoint) - .http_client(HttpClient::with(self.client.clone())) - .root("/") - .bucket(&parsed_url.bucket); + return Ok(Operator::new(builder)? + .finish() + .layer(HttpClientLayer::new(HttpClient::with(self.client.clone()))) + .layer(TimeoutLayer::new().with_timeout(timeout))); + } - Ok(Operator::new(builder)? - .finish() - .layer(TimeoutLayer::new().with_timeout(timeout))) + if let (Some(security_token), Some(endpoint)) = + (&object_storage.security_token, &object_storage.endpoint) + { + // Initialize the OSS operator with the object storage. + let mut builder = opendal::services::Oss::default(); + builder = builder + .security_token(security_token) + .endpoint(endpoint) + .root("/") + .bucket(&parsed_url.bucket); + + return Ok(Operator::new(builder)? + .finish() + .layer(HttpClientLayer::new(HttpClient::with(self.client.clone()))) + .layer(TimeoutLayer::new().with_timeout(timeout))); + } + + Err(ClientError::BackendError(Box::new(BackendError { + message: format!( + "{} {}", + self.scheme, + make_need_fields_message!(object_storage { + access_key_id, + access_key_secret, + security_token, + endpoint + }) + ), + status_code: None, + header: None, + }))) } /// obs_operator initializes the OBS operator with the parsed URL and object storage. @@ -426,11 +446,11 @@ impl ObjectStorage { .access_key_id(access_key_id) .secret_access_key(access_key_secret) .endpoint(endpoint) - .http_client(HttpClient::with(self.client.clone())) .bucket(&parsed_url.bucket); Ok(Operator::new(builder)? .finish() + .layer(HttpClientLayer::new(HttpClient::with(self.client.clone()))) .layer(TimeoutLayer::new().with_timeout(timeout))) } @@ -468,11 +488,11 @@ impl ObjectStorage { .secret_id(access_key_id) .secret_key(access_key_secret) .endpoint(endpoint) - .http_client(HttpClient::with(self.client.clone())) .bucket(&parsed_url.bucket); Ok(Operator::new(builder)? .finish() + .layer(HttpClientLayer::new(HttpClient::with(self.client.clone()))) .layer(TimeoutLayer::new().with_timeout(timeout))) } } @@ -513,7 +533,6 @@ impl crate::Backend for ObjectStorage { operator .list_with(&parsed_url.key) .recursive(true) - .metakey(Metakey::ContentLength | Metakey::Mode) .await // Do the list op here. .map_err(|err| { error!( @@ -757,8 +776,17 @@ mod tests { }, ), (Scheme::GCS, ObjectStorageInfo::default()), + // ( + // Scheme::ABS, + // ObjectStorageInfo { + // endpoint: Some("test-endpoint.local".into()), + // access_key_id: Some("access-key-id".into()), + // access_key_secret: Some("access-key-secret".into()), + // ..Default::default() + // }, + // ), ( - Scheme::ABS, + Scheme::OSS, ObjectStorageInfo { endpoint: Some("test-endpoint.local".into()), access_key_id: Some("access-key-id".into()), @@ -770,8 +798,7 @@ mod tests { Scheme::OSS, ObjectStorageInfo { endpoint: Some("test-endpoint.local".into()), - access_key_id: Some("access-key-id".into()), - access_key_secret: Some("access-key-secret".into()), + security_token: Some("security-token".into()), ..Default::default() }, ), @@ -1073,28 +1100,28 @@ mod tests { let test_cases = vec![ ( ObjectStorageInfo::default(), - "backend error oss need access_key_id, access_key_secret, endpoint", + "backend error oss need access_key_id, access_key_secret, security_token, endpoint", ), ( ObjectStorageInfo { access_key_id: Some("access_key_id".into()), ..Default::default() }, - "backend error oss need access_key_secret, endpoint", + "backend error oss need access_key_secret, security_token, endpoint", ), ( ObjectStorageInfo { access_key_secret: Some("access_key_secret".into()), ..Default::default() }, - "backend error oss need access_key_id, endpoint", + "backend error oss need access_key_id, security_token, endpoint", ), ( ObjectStorageInfo { endpoint: Some("test-endpoint.local".into()), ..Default::default() }, - "backend error oss need access_key_id, access_key_secret", + "backend error oss need access_key_id, access_key_secret, security_token", ), ( ObjectStorageInfo { @@ -1102,7 +1129,7 @@ mod tests { access_key_secret: Some("access_key_secret".into()), ..Default::default() }, - "backend error oss need endpoint", + "backend error oss need security_token, endpoint", ), ( ObjectStorageInfo { @@ -1110,7 +1137,7 @@ mod tests { endpoint: Some("test-endpoint.local".into()), ..Default::default() }, - "backend error oss need access_key_secret", + "backend error oss need access_key_secret, security_token", ), ( ObjectStorageInfo { @@ -1118,7 +1145,7 @@ mod tests { endpoint: Some("test-endpoint.local".into()), ..Default::default() }, - "backend error oss need access_key_id", + "backend error oss need access_key_id, security_token", ), ]; diff --git a/dragonfly-client/src/bin/dfget/main.rs b/dragonfly-client/src/bin/dfget/main.rs index 1482685e..89120435 100644 --- a/dragonfly-client/src/bin/dfget/main.rs +++ b/dragonfly-client/src/bin/dfget/main.rs @@ -248,6 +248,12 @@ struct Args { )] storage_predefined_acl: Option, + #[arg( + long, + help = "Specify the temporary STS security token for accessing Object Storage Service(OSS)" + )] + storage_security_token: Option, + #[arg( long, help = "Specify the delegation token for Hadoop Distributed File System(HDFS)" @@ -651,6 +657,7 @@ async fn download_dir(args: Args, download_client: DfdaemonDownloadClient) -> Re access_key_id: args.storage_access_key_id.clone(), access_key_secret: args.storage_access_key_secret.clone(), session_token: args.storage_session_token.clone(), + security_token: args.storage_security_token.clone(), region: args.storage_region.clone(), endpoint: args.storage_endpoint.clone(), credential_path: args.storage_credential_path.clone(), @@ -761,6 +768,7 @@ async fn download( access_key_id: args.storage_access_key_id.clone(), access_key_secret: args.storage_access_key_secret.clone(), session_token: args.storage_session_token.clone(), + security_token: args.storage_security_token.clone(), region: args.storage_region.clone(), endpoint: args.storage_endpoint.clone(), credential_path: args.storage_credential_path.clone(),