feat(tcp): optimize TCP socket configuration for 4M-64M data chunk transfers (#1384)

Signed-off-by: Gaius <gaius.qi@gmail.com>

Cargo.lock

@@ -1018,7 +1018,7 @@ dependencies = [
 
 [[package]]
 name = "dragonfly-client"
-version = "1.0.26"
+version = "1.0.27"
 dependencies = [
  "anyhow",
  "bytes",
@@ -1094,7 +1094,7 @@ dependencies = [
 
 [[package]]
 name = "dragonfly-client-backend"
-version = "1.0.26"
+version = "1.0.27"
 dependencies = [
  "dragonfly-api",
  "dragonfly-client-core",
@@ -1125,7 +1125,7 @@ dependencies = [
 
 [[package]]
 name = "dragonfly-client-config"
-version = "1.0.26"
+version = "1.0.27"
 dependencies = [
  "bytesize",
  "bytesize-serde",
@@ -1155,7 +1155,7 @@ dependencies = [
 
 [[package]]
 name = "dragonfly-client-core"
-version = "1.0.26"
+version = "1.0.27"
 dependencies = [
  "headers 0.4.1",
  "hyper 1.6.0",
@@ -1175,7 +1175,7 @@ dependencies = [
 
 [[package]]
 name = "dragonfly-client-init"
-version = "1.0.26"
+version = "1.0.27"
 dependencies = [
  "anyhow",
  "clap",
@@ -1192,7 +1192,7 @@ dependencies = [
 
 [[package]]
 name = "dragonfly-client-metric"
-version = "1.0.26"
+version = "1.0.27"
 dependencies = [
  "dragonfly-api",
  "dragonfly-client-config",
@@ -1207,7 +1207,7 @@ dependencies = [
 
 [[package]]
 name = "dragonfly-client-storage"
-version = "1.0.26"
+version = "1.0.27"
 dependencies = [
  "bincode",
  "bytes",
@@ -1222,6 +1222,7 @@ dependencies = [
  "dragonfly-client-util",
  "fs2",
  "leaky-bucket",
+ "nix 0.30.1",
  "num_cpus",
  "prost-wkt-types",
  "quinn",
@@ -1241,7 +1242,7 @@ dependencies = [
 
 [[package]]
 name = "dragonfly-client-util"
-version = "1.0.26"
+version = "1.0.27"
 dependencies = [
  "base64 0.22.1",
  "bytes",
@@ -1680,7 +1681,7 @@ dependencies = [
 
 [[package]]
 name = "hdfs"
-version = "1.0.26"
+version = "1.0.27"
 dependencies = [
  "dragonfly-client-backend",
  "dragonfly-client-core",
@@ -2685,6 +2686,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "memoffset"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"
+dependencies = [
+ "autocfg",
+]
+
 [[package]]
 name = "mime"
 version = "0.3.17"
@@ -2830,6 +2840,19 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "nix"
+version = "0.30.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6"
+dependencies = [
+ "bitflags 2.9.4",
+ "cfg-if",
+ "cfg_aliases",
+ "libc",
+ "memoffset",
+]
+
 [[package]]
 name = "no-std-net"
 version = "0.6.0"
@@ -3545,7 +3568,7 @@ dependencies = [
  "inferno",
  "libc",
  "log",
- "nix",
+ "nix 0.26.4",
  "once_cell",
  "protobuf 3.7.2",
  "protobuf-codegen",

Cargo.toml

@@ -13,7 +13,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.0.26"
+version = "1.0.27"
 authors = ["The Dragonfly Developers"]
 homepage = "https://d7y.io/"
 repository = "https://github.com/dragonflyoss/client.git"
@@ -23,14 +23,14 @@ readme = "README.md"
 edition = "2021"
 
 [workspace.dependencies]
-dragonfly-client = { path = "dragonfly-client", version = "1.0.26" }
+dragonfly-client = { path = "dragonfly-client", version = "1.0.27" }
-dragonfly-client-core = { path = "dragonfly-client-core", version = "1.0.26" }
+dragonfly-client-core = { path = "dragonfly-client-core", version = "1.0.27" }
-dragonfly-client-config = { path = "dragonfly-client-config", version = "1.0.26" }
+dragonfly-client-config = { path = "dragonfly-client-config", version = "1.0.27" }
-dragonfly-client-storage = { path = "dragonfly-client-storage", version = "1.0.26" }
+dragonfly-client-storage = { path = "dragonfly-client-storage", version = "1.0.27" }
-dragonfly-client-backend = { path = "dragonfly-client-backend", version = "1.0.26" }
+dragonfly-client-backend = { path = "dragonfly-client-backend", version = "1.0.27" }
-dragonfly-client-metric = { path = "dragonfly-client-metric", version = "1.0.26" }
+dragonfly-client-metric = { path = "dragonfly-client-metric", version = "1.0.27" }
-dragonfly-client-util = { path = "dragonfly-client-util", version = "1.0.26" }
+dragonfly-client-util = { path = "dragonfly-client-util", version = "1.0.27" }
-dragonfly-client-init = { path = "dragonfly-client-init", version = "1.0.26" }
+dragonfly-client-init = { path = "dragonfly-client-init", version = "1.0.27" }
 dragonfly-api = "=2.1.70"
 thiserror = "2.0"
 futures = "0.3.31"

dragonfly-client-backend/src/http.rs

@@ -70,6 +70,13 @@ impl HTTP {
             .use_preconfigured_tls(client_config_builder)
             .pool_max_idle_per_host(super::POOL_MAX_IDLE_PER_HOST)
             .tcp_keepalive(super::KEEP_ALIVE_INTERVAL)
+            .tcp_nodelay(true)
+            .http2_adaptive_window(true)
+            .http2_initial_stream_window_size(Some(super::HTTP2_STREAM_WINDOW_SIZE))
+            .http2_initial_connection_window_size(Some(super::HTTP2_CONNECTION_WINDOW_SIZE))
+            .http2_keep_alive_timeout(super::HTTP2_KEEP_ALIVE_TIMEOUT)
+            .http2_keep_alive_interval(super::HTTP2_KEEP_ALIVE_INTERVAL)
+            .http2_keep_alive_while_idle(true)
             .build()?;
 
         let retry_policy =
@@ -118,6 +125,13 @@ impl HTTP {
                     .no_deflate()
                     .hickory_dns(true)
                     .use_preconfigured_tls(client_config_builder)
+                    .tcp_nodelay(true)
+                    .http2_adaptive_window(true)
+                    .http2_initial_stream_window_size(Some(super::HTTP2_STREAM_WINDOW_SIZE))
+                    .http2_initial_connection_window_size(Some(super::HTTP2_CONNECTION_WINDOW_SIZE))
+                    .http2_keep_alive_timeout(super::HTTP2_KEEP_ALIVE_TIMEOUT)
+                    .http2_keep_alive_interval(super::HTTP2_KEEP_ALIVE_INTERVAL)
+                    .http2_keep_alive_while_idle(true)
                     .build()?;
 
                 let retry_policy =

dragonfly-client-backend/src/lib.rs

@@ -45,6 +45,12 @@ const HTTP2_KEEP_ALIVE_INTERVAL: Duration = Duration::from_secs(300);
 /// HTTP2_KEEP_ALIVE_TIMEOUT is the timeout for HTTP2 keep alive.
 const HTTP2_KEEP_ALIVE_TIMEOUT: Duration = Duration::from_secs(20);
 
+/// HTTP2_STREAM_WINDOW_SIZE is the stream window size for HTTP2 connection.
+const HTTP2_STREAM_WINDOW_SIZE: u32 = 16 * 1024 * 1024;
+
+/// HTTP2_CONNECTION_WINDOW_SIZE is the connection window size for HTTP2 connection.
+const HTTP2_CONNECTION_WINDOW_SIZE: u32 = 16 * 1024 * 1024;
+
 /// MAX_RETRY_TIMES is the max retry times for the request.
 const MAX_RETRY_TIMES: u32 = 1;
 

dragonfly-client-backend/src/object_storage.rs

@@ -187,6 +187,10 @@ impl ObjectStorage {
             .hickory_dns(true)
             .pool_max_idle_per_host(super::POOL_MAX_IDLE_PER_HOST)
             .tcp_keepalive(super::KEEP_ALIVE_INTERVAL)
+            .tcp_nodelay(true)
+            .http2_adaptive_window(true)
+            .http2_initial_stream_window_size(Some(super::HTTP2_STREAM_WINDOW_SIZE))
+            .http2_initial_connection_window_size(Some(super::HTTP2_CONNECTION_WINDOW_SIZE))
             .http2_keep_alive_timeout(super::HTTP2_KEEP_ALIVE_TIMEOUT)
             .http2_keep_alive_interval(super::HTTP2_KEEP_ALIVE_INTERVAL)
             .http2_keep_alive_while_idle(true)

dragonfly-client-config/src/dfdaemon.rs

@@ -1014,11 +1014,11 @@ pub struct Storage {
     )]
     pub write_piece_timeout: Duration,
 
-    /// write_buffer_size is the buffer size for writing piece to disk, default is 128KB.
+    /// write_buffer_size is the buffer size for writing piece to disk, default is 4MiB.
     #[serde(default = "default_storage_write_buffer_size")]
     pub write_buffer_size: usize,
 
-    /// read_buffer_size is the buffer size for reading piece from disk, default is 128KB.
+    /// read_buffer_size is the buffer size for reading piece from disk, default is 4MiB.
     #[serde(default = "default_storage_read_buffer_size")]
     pub read_buffer_size: usize,
 

dragonfly-client-storage/Cargo.toml

@@ -36,6 +36,7 @@ bincode = "1.3.3"
 walkdir = "2.5.0"
 quinn = "0.11.9"
 socket2 = "0.6.0"
+nix = { version = "0.30.1", features = ["socket", "net"] }
 
 [dev-dependencies]
 tempfile.workspace = true

dragonfly-client-storage/src/client/mod.rs

@@ -20,10 +20,10 @@ pub mod tcp;
 use std::time::Duration;
 
 /// DEFAULT_SEND_BUFFER_SIZE is the default size of the send buffer for network connections.
-const DEFAULT_SEND_BUFFER_SIZE: usize = 4 * 1024 * 1024;
+const DEFAULT_SEND_BUFFER_SIZE: usize = 16 * 1024 * 1024;
 
 /// DEFAULT_RECV_BUFFER_SIZE is the default size of the receive buffer for network connections.
-const DEFAULT_RECV_BUFFER_SIZE: usize = 4 * 1024 * 1024;
+const DEFAULT_RECV_BUFFER_SIZE: usize = 16 * 1024 * 1024;
 
 /// DEFAULT_KEEPALIVE_INTERVAL is the default interval for sending keepalive messages.
 const DEFAULT_KEEPALIVE_INTERVAL: Duration = Duration::from_secs(5);

dragonfly-client-storage/src/client/tcp.rs

@@ -179,6 +179,18 @@ impl TCPClient {
         socket.set_tcp_keepalive(
             &TcpKeepalive::new().with_interval(super::DEFAULT_KEEPALIVE_INTERVAL),
         )?;
+        #[cfg(target_os = "linux")]
+        {
+            use nix::sys::socket::{setsockopt, sockopt::TcpFastOpenConnect};
+            use std::os::fd::AsFd;
+            use tracing::{info, warn};
+
+            if let Err(err) = setsockopt(&socket.as_fd(), TcpFastOpenConnect, &true) {
+                warn!("failed to set tcp fast open: {}", err);
+            } else {
+                info!("set tcp fast open to true");
+            }
+        }
 
         let (reader, mut writer) = stream.into_split();
         writer.write_all(&request).await.inspect_err(|err| {

dragonfly-client-storage/src/server/mod.rs

@@ -20,10 +20,10 @@ pub mod tcp;
 use std::time::Duration;
 
 /// DEFAULT_SEND_BUFFER_SIZE is the default size of the send buffer for network connections.
-const DEFAULT_SEND_BUFFER_SIZE: usize = 4 * 1024 * 1024;
+const DEFAULT_SEND_BUFFER_SIZE: usize = 16 * 1024 * 1024;
 
 /// DEFAULT_RECV_BUFFER_SIZE is the default size of the receive buffer for network connections.
-const DEFAULT_RECV_BUFFER_SIZE: usize = 4 * 1024 * 1024;
+const DEFAULT_RECV_BUFFER_SIZE: usize = 16 * 1024 * 1024;
 
 /// DEFAULT_KEEPALIVE_INTERVAL is the default interval for sending keepalive messages.
 const DEFAULT_KEEPALIVE_INTERVAL: Duration = Duration::from_secs(5);

dragonfly-client-storage/src/server/tcp.rs

@@ -99,11 +99,21 @@ impl TCPServer {
         )?;
         #[cfg(target_os = "linux")]
         {
-            use tracing::warn;
+            use nix::sys::socket::{setsockopt, sockopt::TcpFastOpenConnect};
-            if let Err(err) = socket.set_tcp_congestion("bbr".as_bytes()) {
+            use std::os::fd::AsFd;
+            use tracing::{info, warn};
+
+            if let Err(err) = socket.set_tcp_congestion("cubic".as_bytes()) {
                 warn!("failed to set tcp congestion: {}", err);
+            } else {
+                info!("set tcp congestion to cubic");
+            }
+
+            if let Err(err) = setsockopt(&socket.as_fd(), TcpFastOpenConnect, &true) {
+                warn!("failed to set tcp fast open: {}", err);
+            } else {
+                info!("set tcp fast open to true");
             }
-            info!("set tcp congestion to bbr");
         }
 
         socket.bind(&self.addr.into())?;