dragonflyoss
/
client
mirror of https://github.com/dragonflyoss/client.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: enable console subscriber layer for tracing spawn tasks on `127.0.0.1:6669` when log level is TRACE (#1275) 

Signed-off-by: Gaius <gaius.qi@gmail.com>
This commit is contained in:
Gaius 2025-08-06 14:58:47 +08:00 committed by GitHub
parent 848737e327
commit e415df936d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 113 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.cargo/config.toml Normal file
View File

@ -0,0 +1,2 @@
[build]
rustflags = ["--cfg", "tokio_unstable"]

72
Cargo.lock generated
View File

@ -660,6 +660,45 @@ dependencies = [
"windows-sys 0.60.2",
]
[[package]]
name = "console-api"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8030735ecb0d128428b64cd379809817e620a40e5001c54465b99ec5feec2857"
dependencies = [
"futures-core",
"prost 0.13.5",
"prost-types 0.13.5",
"tonic",
"tracing-core",
]
[[package]]
name = "console-subscriber"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6539aa9c6a4cd31f4b1c040f860a1eac9aa80e7df6b05d506a6e7179936d6a01"
dependencies = [
"console-api",
"crossbeam-channel",
"crossbeam-utils",
"futures-task",
"hdrhistogram",
"humantime",
"hyper-util",
"prost 0.13.5",
"prost-types 0.13.5",
"serde",
"serde_json",
"thread_local",
"tokio",
"tokio-stream",
"tonic",
"tracing",
"tracing-core",
"tracing-subscriber",
]
[[package]]
name = "const-oid"
version = "0.9.6"
@ -954,13 +993,14 @@ dependencies = [
[[package]]
name = "dragonfly-client"
version = "1.0.9"
version = "1.0.10"
dependencies = [
"anyhow",
"bytes",
"bytesize",
"chrono",
"clap",
"console-subscriber",
"dashmap",
"dragonfly-api",
"dragonfly-client-backend",
@ -1025,7 +1065,7 @@ dependencies = [
[[package]]
name = "dragonfly-client-backend"
version = "1.0.9"
version = "1.0.10"
dependencies = [
"dragonfly-api",
"dragonfly-client-core",
@ -1056,7 +1096,7 @@ dependencies = [
[[package]]
name = "dragonfly-client-config"
version = "1.0.9"
version = "1.0.10"
dependencies = [
"bytesize",
"bytesize-serde",
@ -1086,7 +1126,7 @@ dependencies = [
[[package]]
name = "dragonfly-client-core"
version = "1.0.9"
version = "1.0.10"
dependencies = [
"headers 0.4.1",
"hyper 1.6.0",
@ -1104,7 +1144,7 @@ dependencies = [
[[package]]
name = "dragonfly-client-init"
version = "1.0.9"
version = "1.0.10"
dependencies = [
"anyhow",
"clap",
@ -1121,7 +1161,7 @@ dependencies = [
[[package]]
name = "dragonfly-client-storage"
version = "1.0.9"
version = "1.0.10"
dependencies = [
"bincode",
"bytes",
@ -1148,7 +1188,7 @@ dependencies = [
[[package]]
name = "dragonfly-client-util"
version = "1.0.9"
version = "1.0.10"
dependencies = [
"base64 0.22.1",
"bytesize",
@ -1545,13 +1585,26 @@ dependencies = [
[[package]]
name = "hdfs"
version = "1.0.9"
version = "1.0.10"
dependencies = [
"dragonfly-client-backend",
"dragonfly-client-core",
"tonic",
]
[[package]]
name = "hdrhistogram"
version = "7.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d"
dependencies = [
"base64 0.21.7",
"byteorder",
"flate2",
"nom",
"num-traits",
]
[[package]]
name = "headers"
version = "0.3.9"
@ -1861,7 +1914,7 @@ dependencies = [
"hyper 1.6.0",
"libc",
"pin-project-lite",
"socket2 0.5.9",
"socket2 0.6.0",
"tokio",
"tower-service",
"tracing",
@ -4858,6 +4911,7 @@ dependencies = [
"slab",
"socket2 0.6.0",
"tokio-macros",
"tracing",
"windows-sys 0.59.0",
]

18
Cargo.toml
View File

@ -12,7 +12,7 @@ members = [
]
[workspace.package]
version = "1.0.9"
version = "1.0.10"
authors = ["The Dragonfly Developers"]
homepage = "https://d7y.io/"
repository = "https://github.com/dragonflyoss/client.git"
@ -22,13 +22,13 @@ readme = "README.md"
edition = "2021"
[workspace.dependencies]
dragonfly-client = { path = "dragonfly-client", version = "1.0.9" }
dragonfly-client-core = { path = "dragonfly-client-core", version = "1.0.9" }
dragonfly-client-config = { path = "dragonfly-client-config", version = "1.0.9" }
dragonfly-client-storage = { path = "dragonfly-client-storage", version = "1.0.9" }
dragonfly-client-backend = { path = "dragonfly-client-backend", version = "1.0.9" }
dragonfly-client-util = { path = "dragonfly-client-util", version = "1.0.9" }
dragonfly-client-init = { path = "dragonfly-client-init", version = "1.0.9" }
dragonfly-client = { path = "dragonfly-client", version = "1.0.10" }
dragonfly-client-core = { path = "dragonfly-client-core", version = "1.0.10" }
dragonfly-client-config = { path = "dragonfly-client-config", version = "1.0.10" }
dragonfly-client-storage = { path = "dragonfly-client-storage", version = "1.0.10" }
dragonfly-client-backend = { path = "dragonfly-client-backend", version = "1.0.10" }
dragonfly-client-util = { path = "dragonfly-client-util", version = "1.0.10" }
dragonfly-client-init = { path = "dragonfly-client-init", version = "1.0.10" }
dragonfly-api = "2.1.55"
thiserror = "2.0"
futures = "0.3.31"
@ -71,7 +71,7 @@ serde_yaml = "0.9"
http = "1"
tonic = { version = "0.12.2", features = ["tls"] }
tonic-reflection = "0.12.3"
tokio = { version = "1.47.1", features = ["full"] }
tokio = { version = "1.47.1", features = ["full", "tracing"] }
tokio-util = { version = "0.7.16", features = ["full"] }
tokio-stream = "0.1.17"
validator = { version = "0.16", features = ["derive"] }

4
ci/Dockerfile
View File

@ -7,6 +7,7 @@ RUN apt-get update && apt-get install -y \
&& rm -rf /var/lib/apt/lists/*
COPY Cargo.toml Cargo.lock ./
COPY .cargo ./cargo
COPY dragonfly-client/Cargo.toml ./dragonfly-client/Cargo.toml
COPY dragonfly-client/src ./dragonfly-client/src
@ -40,6 +41,8 @@ RUN case "${TARGETPLATFORM}" in \
esac && \
cargo build --release --verbose --bin dfget --bin dfdaemon --bin dfcache
RUN cargo install tokio-console --locked --root /usr/local
FROM public.ecr.aws/docker/library/alpine:3.20 AS health
ENV GRPC_HEALTH_PROBE_VERSION=v0.4.24
@ -67,6 +70,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends iperf3 fio curl
COPY --from=builder /app/client/target/release/dfget /usr/local/bin/dfget
COPY --from=builder /app/client/target/release/dfdaemon /usr/local/bin/dfdaemon
COPY --from=builder /app/client/target/release/dfcache /usr/local/bin/dfcache
COPY --from=builder /usr/local/bin/tokio-console /usr/local/bin/
COPY --from=pprof /go/bin/pprof /bin/pprof
COPY --from=health /bin/grpc_health_probe /bin/grpc_health_probe

3
ci/Dockerfile.debug
View File

@ -7,6 +7,7 @@ RUN apt-get update && apt-get install -y \
&& rm -rf /var/lib/apt/lists/*
COPY Cargo.toml Cargo.lock ./
COPY .cargo ./cargo
COPY dragonfly-client/Cargo.toml ./dragonfly-client/Cargo.toml
COPY dragonfly-client/src ./dragonfly-client/src
@ -42,6 +43,7 @@ RUN case "${TARGETPLATFORM}" in \
RUN cargo install flamegraph --root /usr/local
RUN cargo install bottom --locked --root /usr/local
RUN cargo install tokio-console --locked --root /usr/local
FROM public.ecr.aws/docker/library/alpine:3.20 AS health
@ -72,6 +74,7 @@ COPY --from=builder /app/client/target/debug/dfdaemon /usr/local/bin/dfdaemon
COPY --from=builder /app/client/target/debug/dfcache /usr/local/bin/dfcache
COPY --from=builder /usr/local/bin/flamegraph /usr/local/bin/
COPY --from=builder /usr/local/bin/btm /usr/local/bin/
COPY --from=builder /usr/local/bin/tokio-console /usr/local/bin/
COPY --from=pprof /go/bin/pprof /bin/pprof
COPY --from=health /bin/grpc_health_probe /bin/grpc_health_probe

1
ci/Dockerfile.dfinit
View File

@ -7,6 +7,7 @@ RUN apt-get update && apt-get install -y \
WORKDIR /app/client
COPY Cargo.toml Cargo.lock ./
COPY .cargo ./cargo
COPY dragonfly-client/Cargo.toml ./dragonfly-client/Cargo.toml
COPY dragonfly-client/src ./dragonfly-client/src

22
dragonfly-client-backend/src/http.rs
View File

@ -50,6 +50,17 @@ impl HTTP {
.with_custom_certificate_verifier(NoVerifier::new())
.with_no_client_auth();
// Disable automatic compression to prevent double-decompression issues.
//
// Problem scenario:
// 1. Origin server supports gzip and returns "content-encoding: gzip" header.
// 2. Backend decompresses the response and stores uncompressed content to disk.
// 3. When user's client downloads via dfdaemon proxy, the original "content-encoding: gzip".
// header is forwarded to it.
// 4. User's client attempts to decompress the already-decompressed content, causing errors.
//
// Solution: Disable all compression formats (gzip, brotli, zstd, deflate) to ensure
// we receive and store uncompressed content, eliminating the double-decompression issue.
let client = reqwest::Client::builder()
.no_gzip()
.no_brotli()
@ -88,6 +99,17 @@ impl HTTP {
.with_root_certificates(root_cert_store)
.with_no_client_auth();
// Disable automatic compression to prevent double-decompression issues.
//
// Problem scenario:
// 1. Origin server supports gzip and returns "content-encoding: gzip" header.
// 2. Backend decompresses the response and stores uncompressed content to disk.
// 3. When user's client downloads via dfdaemon proxy, the original "content-encoding: gzip".
// header is forwarded to it.
// 4. User's client attempts to decompress the already-decompressed content, causing errors.
//
// Solution: Disable all compression formats (gzip, brotli, zstd, deflate) to ensure
// we receive and store uncompressed content, eliminating the double-decompression issue.
let client = reqwest::Client::builder()
.no_gzip()
.no_brotli()

1
dragonfly-client/Cargo.toml
View File

@ -85,6 +85,7 @@ path-absolutize = "3.1.1"
dashmap = "6.1.0"
fastrand = "2.3.0"
glob = "0.3.2"
console-subscriber = "0.4.1"
[dev-dependencies]
tempfile.workspace = true

8
dragonfly-client/src/tracing/mod.rs
View File

@ -102,8 +102,16 @@ pub fn init_tracing(
let env_filter = EnvFilter::try_from_default_env()
.unwrap_or_else(|_| EnvFilter::default().add_directive(log_level.into()));
// Enable console subscriber layer for tracing spawn tasks on `127.0.0.1:6669` when log level is TRACE.
let console_subscriber_layer = if log_level == Level::TRACE {
Some(console_subscriber::spawn())
} else {
None
};
let subscriber = Registry::default()
.with(env_filter)
.with(console_subscriber_layer)
.with(file_logging_layer)
.with(stdout_logging_layer);