dragonflyoss
/
client
mirror of https://github.com/dragonflyoss/client.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat(tcp): optimize TCP socket configuration for 4M-64M data chunk transfers (#1384) 

Signed-off-by: Gaius <gaius.qi@gmail.com>
This commit is contained in:
Gaius 2025-09-28 21:47:07 +08:00 committed by GitHub
parent e6a0b9372d
commit 163a57a0e3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 98 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
Cargo.lock generated
View File

@ -1018,7 +1018,7 @@ dependencies = [
[[package]]
name = "dragonfly-client"
version = "1.0.26"
version = "1.0.27"
dependencies = [
"anyhow",
"bytes",
@ -1094,7 +1094,7 @@ dependencies = [
[[package]]
name = "dragonfly-client-backend"
version = "1.0.26"
version = "1.0.27"
dependencies = [
"dragonfly-api",
"dragonfly-client-core",
@ -1125,7 +1125,7 @@ dependencies = [
[[package]]
name = "dragonfly-client-config"
version = "1.0.26"
version = "1.0.27"
dependencies = [
"bytesize",
"bytesize-serde",
@ -1155,7 +1155,7 @@ dependencies = [
[[package]]
name = "dragonfly-client-core"
version = "1.0.26"
version = "1.0.27"
dependencies = [
"headers 0.4.1",
"hyper 1.6.0",
@ -1175,7 +1175,7 @@ dependencies = [
[[package]]
name = "dragonfly-client-init"
version = "1.0.26"
version = "1.0.27"
dependencies = [
"anyhow",
"clap",
@ -1192,7 +1192,7 @@ dependencies = [
[[package]]
name = "dragonfly-client-metric"
version = "1.0.26"
version = "1.0.27"
dependencies = [
"dragonfly-api",
"dragonfly-client-config",
@ -1207,7 +1207,7 @@ dependencies = [
[[package]]
name = "dragonfly-client-storage"
version = "1.0.26"
version = "1.0.27"
dependencies = [
"bincode",
"bytes",
@ -1222,6 +1222,7 @@ dependencies = [
"dragonfly-client-util",
"fs2",
"leaky-bucket",
"nix 0.30.1",
"num_cpus",
"prost-wkt-types",
"quinn",
@ -1241,7 +1242,7 @@ dependencies = [
[[package]]
name = "dragonfly-client-util"
version = "1.0.26"
version = "1.0.27"
dependencies = [
"base64 0.22.1",
"bytes",
@ -1680,7 +1681,7 @@ dependencies = [
[[package]]
name = "hdfs"
version = "1.0.26"
version = "1.0.27"
dependencies = [
"dragonfly-client-backend",
"dragonfly-client-core",
@ -2685,6 +2686,15 @@ dependencies = [
"libc",
]
[[package]]
name = "memoffset"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"
dependencies = [
"autocfg",
]
[[package]]
name = "mime"
version = "0.3.17"
@ -2830,6 +2840,19 @@ dependencies = [
"libc",
]
[[package]]
name = "nix"
version = "0.30.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6"
dependencies = [
"bitflags 2.9.4",
"cfg-if",
"cfg_aliases",
"libc",
"memoffset",
]
[[package]]
name = "no-std-net"
version = "0.6.0"
@ -3545,7 +3568,7 @@ dependencies = [
"inferno",
"libc",
"log",
"nix",
"nix 0.26.4",
"once_cell",
"protobuf 3.7.2",
"protobuf-codegen",

18
Cargo.toml
View File

@ -13,7 +13,7 @@ members = [
]
[workspace.package]
version = "1.0.26"
version = "1.0.27"
authors = ["The Dragonfly Developers"]
homepage = "https://d7y.io/"
repository = "https://github.com/dragonflyoss/client.git"
@ -23,14 +23,14 @@ readme = "README.md"
edition = "2021"
[workspace.dependencies]
dragonfly-client = { path = "dragonfly-client", version = "1.0.26" }
dragonfly-client-core = { path = "dragonfly-client-core", version = "1.0.26" }
dragonfly-client-config = { path = "dragonfly-client-config", version = "1.0.26" }
dragonfly-client-storage = { path = "dragonfly-client-storage", version = "1.0.26" }
dragonfly-client-backend = { path = "dragonfly-client-backend", version = "1.0.26" }
dragonfly-client-metric = { path = "dragonfly-client-metric", version = "1.0.26" }
dragonfly-client-util = { path = "dragonfly-client-util", version = "1.0.26" }
dragonfly-client-init = { path = "dragonfly-client-init", version = "1.0.26" }
dragonfly-client = { path = "dragonfly-client", version = "1.0.27" }
dragonfly-client-core = { path = "dragonfly-client-core", version = "1.0.27" }
dragonfly-client-config = { path = "dragonfly-client-config", version = "1.0.27" }
dragonfly-client-storage = { path = "dragonfly-client-storage", version = "1.0.27" }
dragonfly-client-backend = { path = "dragonfly-client-backend", version = "1.0.27" }
dragonfly-client-metric = { path = "dragonfly-client-metric", version = "1.0.27" }
dragonfly-client-util = { path = "dragonfly-client-util", version = "1.0.27" }
dragonfly-client-init = { path = "dragonfly-client-init", version = "1.0.27" }
dragonfly-api = "=2.1.70"
thiserror = "2.0"
futures = "0.3.31"

14
dragonfly-client-backend/src/http.rs
View File

@ -70,6 +70,13 @@ impl HTTP {
.use_preconfigured_tls(client_config_builder)
.pool_max_idle_per_host(super::POOL_MAX_IDLE_PER_HOST)
.tcp_keepalive(super::KEEP_ALIVE_INTERVAL)
.tcp_nodelay(true)
.http2_adaptive_window(true)
.http2_initial_stream_window_size(Some(super::HTTP2_STREAM_WINDOW_SIZE))
.http2_initial_connection_window_size(Some(super::HTTP2_CONNECTION_WINDOW_SIZE))
.http2_keep_alive_timeout(super::HTTP2_KEEP_ALIVE_TIMEOUT)
.http2_keep_alive_interval(super::HTTP2_KEEP_ALIVE_INTERVAL)
.http2_keep_alive_while_idle(true)
.build()?;
let retry_policy =
@ -118,6 +125,13 @@ impl HTTP {
.no_deflate()
.hickory_dns(true)
.use_preconfigured_tls(client_config_builder)
.tcp_nodelay(true)
.http2_adaptive_window(true)
.http2_initial_stream_window_size(Some(super::HTTP2_STREAM_WINDOW_SIZE))
.http2_initial_connection_window_size(Some(super::HTTP2_CONNECTION_WINDOW_SIZE))
.http2_keep_alive_timeout(super::HTTP2_KEEP_ALIVE_TIMEOUT)
.http2_keep_alive_interval(super::HTTP2_KEEP_ALIVE_INTERVAL)
.http2_keep_alive_while_idle(true)
.build()?;
let retry_policy =

6
dragonfly-client-backend/src/lib.rs
View File

@ -45,6 +45,12 @@ const HTTP2_KEEP_ALIVE_INTERVAL: Duration = Duration::from_secs(300);
/// HTTP2_KEEP_ALIVE_TIMEOUT is the timeout for HTTP2 keep alive.
const HTTP2_KEEP_ALIVE_TIMEOUT: Duration = Duration::from_secs(20);
/// HTTP2_STREAM_WINDOW_SIZE is the stream window size for HTTP2 connection.
const HTTP2_STREAM_WINDOW_SIZE: u32 = 16 * 1024 * 1024;
/// HTTP2_CONNECTION_WINDOW_SIZE is the connection window size for HTTP2 connection.
const HTTP2_CONNECTION_WINDOW_SIZE: u32 = 16 * 1024 * 1024;
/// MAX_RETRY_TIMES is the max retry times for the request.
const MAX_RETRY_TIMES: u32 = 1;

4
dragonfly-client-backend/src/object_storage.rs
View File

@ -187,6 +187,10 @@ impl ObjectStorage {
.hickory_dns(true)
.pool_max_idle_per_host(super::POOL_MAX_IDLE_PER_HOST)
.tcp_keepalive(super::KEEP_ALIVE_INTERVAL)
.tcp_nodelay(true)
.http2_adaptive_window(true)
.http2_initial_stream_window_size(Some(super::HTTP2_STREAM_WINDOW_SIZE))
.http2_initial_connection_window_size(Some(super::HTTP2_CONNECTION_WINDOW_SIZE))
.http2_keep_alive_timeout(super::HTTP2_KEEP_ALIVE_TIMEOUT)
.http2_keep_alive_interval(super::HTTP2_KEEP_ALIVE_INTERVAL)
.http2_keep_alive_while_idle(true)

4
dragonfly-client-config/src/dfdaemon.rs
View File

@ -1014,11 +1014,11 @@ pub struct Storage {
)]
pub write_piece_timeout: Duration,
/// write_buffer_size is the buffer size for writing piece to disk, default is 128KB.
/// write_buffer_size is the buffer size for writing piece to disk, default is 4MiB.
#[serde(default = "default_storage_write_buffer_size")]
pub write_buffer_size: usize,
/// read_buffer_size is the buffer size for reading piece from disk, default is 128KB.
/// read_buffer_size is the buffer size for reading piece from disk, default is 4MiB.
#[serde(default = "default_storage_read_buffer_size")]
pub read_buffer_size: usize,

1
dragonfly-client-storage/Cargo.toml
View File

@ -36,6 +36,7 @@ bincode = "1.3.3"
walkdir = "2.5.0"
quinn = "0.11.9"
socket2 = "0.6.0"
nix = { version = "0.30.1", features = ["socket", "net"] }
[dev-dependencies]
tempfile.workspace = true

4
dragonfly-client-storage/src/client/mod.rs
View File

@ -20,10 +20,10 @@ pub mod tcp;
use std::time::Duration;
/// DEFAULT_SEND_BUFFER_SIZE is the default size of the send buffer for network connections.
const DEFAULT_SEND_BUFFER_SIZE: usize = 4 * 1024 * 1024;
const DEFAULT_SEND_BUFFER_SIZE: usize = 16 * 1024 * 1024;
/// DEFAULT_RECV_BUFFER_SIZE is the default size of the receive buffer for network connections.
const DEFAULT_RECV_BUFFER_SIZE: usize = 4 * 1024 * 1024;
const DEFAULT_RECV_BUFFER_SIZE: usize = 16 * 1024 * 1024;
/// DEFAULT_KEEPALIVE_INTERVAL is the default interval for sending keepalive messages.
const DEFAULT_KEEPALIVE_INTERVAL: Duration = Duration::from_secs(5);

12
dragonfly-client-storage/src/client/tcp.rs
View File

@ -179,6 +179,18 @@ impl TCPClient {
socket.set_tcp_keepalive(
&TcpKeepalive::new().with_interval(super::DEFAULT_KEEPALIVE_INTERVAL),
)?;
#[cfg(target_os = "linux")]
{
use nix::sys::socket::{setsockopt, sockopt::TcpFastOpenConnect};
use std::os::fd::AsFd;
use tracing::{info, warn};
if let Err(err) = setsockopt(&socket.as_fd(), TcpFastOpenConnect, &true) {
warn!("failed to set tcp fast open: {}", err);
} else {
info!("set tcp fast open to true");
}
}
let (reader, mut writer) = stream.into_split();
writer.write_all(&request).await.inspect_err(|err| {

4
dragonfly-client-storage/src/server/mod.rs
View File

@ -20,10 +20,10 @@ pub mod tcp;
use std::time::Duration;
/// DEFAULT_SEND_BUFFER_SIZE is the default size of the send buffer for network connections.
const DEFAULT_SEND_BUFFER_SIZE: usize = 4 * 1024 * 1024;
const DEFAULT_SEND_BUFFER_SIZE: usize = 16 * 1024 * 1024;
/// DEFAULT_RECV_BUFFER_SIZE is the default size of the receive buffer for network connections.
const DEFAULT_RECV_BUFFER_SIZE: usize = 4 * 1024 * 1024;
const DEFAULT_RECV_BUFFER_SIZE: usize = 16 * 1024 * 1024;
/// DEFAULT_KEEPALIVE_INTERVAL is the default interval for sending keepalive messages.
const DEFAULT_KEEPALIVE_INTERVAL: Duration = Duration::from_secs(5);

16
dragonfly-client-storage/src/server/tcp.rs
View File

@ -99,11 +99,21 @@ impl TCPServer {
)?;
#[cfg(target_os = "linux")]
{
use tracing::warn;
if let Err(err) = socket.set_tcp_congestion("bbr".as_bytes()) {
use nix::sys::socket::{setsockopt, sockopt::TcpFastOpenConnect};
use std::os::fd::AsFd;
use tracing::{info, warn};
if let Err(err) = socket.set_tcp_congestion("cubic".as_bytes()) {
warn!("failed to set tcp congestion: {}", err);
} else {
info!("set tcp congestion to cubic");
}
if let Err(err) = setsockopt(&socket.as_fd(), TcpFastOpenConnect, &true) {
warn!("failed to set tcp fast open: {}", err);
} else {
info!("set tcp fast open to true");
}
info!("set tcp congestion to bbr");
}
socket.bind(&self.addr.into())?;