diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 00000000..bff29e6e --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[build] +rustflags = ["--cfg", "tokio_unstable"] diff --git a/Cargo.lock b/Cargo.lock index 9967ec9d..90354438 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -660,6 +660,45 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "console-api" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8030735ecb0d128428b64cd379809817e620a40e5001c54465b99ec5feec2857" +dependencies = [ + "futures-core", + "prost 0.13.5", + "prost-types 0.13.5", + "tonic", + "tracing-core", +] + +[[package]] +name = "console-subscriber" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6539aa9c6a4cd31f4b1c040f860a1eac9aa80e7df6b05d506a6e7179936d6a01" +dependencies = [ + "console-api", + "crossbeam-channel", + "crossbeam-utils", + "futures-task", + "hdrhistogram", + "humantime", + "hyper-util", + "prost 0.13.5", + "prost-types 0.13.5", + "serde", + "serde_json", + "thread_local", + "tokio", + "tokio-stream", + "tonic", + "tracing", + "tracing-core", + "tracing-subscriber", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -954,13 +993,14 @@ dependencies = [ [[package]] name = "dragonfly-client" -version = "1.0.9" +version = "1.0.10" dependencies = [ "anyhow", "bytes", "bytesize", "chrono", "clap", + "console-subscriber", "dashmap", "dragonfly-api", "dragonfly-client-backend", @@ -1025,7 +1065,7 @@ dependencies = [ [[package]] name = "dragonfly-client-backend" -version = "1.0.9" +version = "1.0.10" dependencies = [ "dragonfly-api", "dragonfly-client-core", @@ -1056,7 +1096,7 @@ dependencies = [ [[package]] name = "dragonfly-client-config" -version = "1.0.9" +version = "1.0.10" dependencies = [ "bytesize", "bytesize-serde", @@ -1086,7 +1126,7 @@ dependencies = [ [[package]] name = "dragonfly-client-core" -version = "1.0.9" +version = "1.0.10" dependencies = [ "headers 0.4.1", "hyper 1.6.0", @@ -1104,7 +1144,7 @@ dependencies = [ [[package]] name = "dragonfly-client-init" -version = "1.0.9" +version = "1.0.10" dependencies = [ "anyhow", "clap", @@ -1121,7 +1161,7 @@ dependencies = [ [[package]] name = "dragonfly-client-storage" -version = "1.0.9" +version = "1.0.10" dependencies = [ "bincode", "bytes", @@ -1148,7 +1188,7 @@ dependencies = [ [[package]] name = "dragonfly-client-util" -version = "1.0.9" +version = "1.0.10" dependencies = [ "base64 0.22.1", "bytesize", @@ -1545,13 +1585,26 @@ dependencies = [ [[package]] name = "hdfs" -version = "1.0.9" +version = "1.0.10" dependencies = [ "dragonfly-client-backend", "dragonfly-client-core", "tonic", ] +[[package]] +name = "hdrhistogram" +version = "7.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d" +dependencies = [ + "base64 0.21.7", + "byteorder", + "flate2", + "nom", + "num-traits", +] + [[package]] name = "headers" version = "0.3.9" @@ -1861,7 +1914,7 @@ dependencies = [ "hyper 1.6.0", "libc", "pin-project-lite", - "socket2 0.5.9", + "socket2 0.6.0", "tokio", "tower-service", "tracing", @@ -4858,6 +4911,7 @@ dependencies = [ "slab", "socket2 0.6.0", "tokio-macros", + "tracing", "windows-sys 0.59.0", ] diff --git a/Cargo.toml b/Cargo.toml index 6c121b9c..5fe31638 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ members = [ ] [workspace.package] -version = "1.0.9" +version = "1.0.10" authors = ["The Dragonfly Developers"] homepage = "https://d7y.io/" repository = "https://github.com/dragonflyoss/client.git" @@ -22,13 +22,13 @@ readme = "README.md" edition = "2021" [workspace.dependencies] -dragonfly-client = { path = "dragonfly-client", version = "1.0.9" } -dragonfly-client-core = { path = "dragonfly-client-core", version = "1.0.9" } -dragonfly-client-config = { path = "dragonfly-client-config", version = "1.0.9" } -dragonfly-client-storage = { path = "dragonfly-client-storage", version = "1.0.9" } -dragonfly-client-backend = { path = "dragonfly-client-backend", version = "1.0.9" } -dragonfly-client-util = { path = "dragonfly-client-util", version = "1.0.9" } -dragonfly-client-init = { path = "dragonfly-client-init", version = "1.0.9" } +dragonfly-client = { path = "dragonfly-client", version = "1.0.10" } +dragonfly-client-core = { path = "dragonfly-client-core", version = "1.0.10" } +dragonfly-client-config = { path = "dragonfly-client-config", version = "1.0.10" } +dragonfly-client-storage = { path = "dragonfly-client-storage", version = "1.0.10" } +dragonfly-client-backend = { path = "dragonfly-client-backend", version = "1.0.10" } +dragonfly-client-util = { path = "dragonfly-client-util", version = "1.0.10" } +dragonfly-client-init = { path = "dragonfly-client-init", version = "1.0.10" } dragonfly-api = "2.1.55" thiserror = "2.0" futures = "0.3.31" @@ -71,7 +71,7 @@ serde_yaml = "0.9" http = "1" tonic = { version = "0.12.2", features = ["tls"] } tonic-reflection = "0.12.3" -tokio = { version = "1.47.1", features = ["full"] } +tokio = { version = "1.47.1", features = ["full", "tracing"] } tokio-util = { version = "0.7.16", features = ["full"] } tokio-stream = "0.1.17" validator = { version = "0.16", features = ["derive"] } diff --git a/ci/Dockerfile b/ci/Dockerfile index f56c27f3..75c06d79 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -7,6 +7,7 @@ RUN apt-get update && apt-get install -y \ && rm -rf /var/lib/apt/lists/* COPY Cargo.toml Cargo.lock ./ +COPY .cargo ./cargo COPY dragonfly-client/Cargo.toml ./dragonfly-client/Cargo.toml COPY dragonfly-client/src ./dragonfly-client/src @@ -40,6 +41,8 @@ RUN case "${TARGETPLATFORM}" in \ esac && \ cargo build --release --verbose --bin dfget --bin dfdaemon --bin dfcache +RUN cargo install tokio-console --locked --root /usr/local + FROM public.ecr.aws/docker/library/alpine:3.20 AS health ENV GRPC_HEALTH_PROBE_VERSION=v0.4.24 @@ -67,6 +70,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends iperf3 fio curl COPY --from=builder /app/client/target/release/dfget /usr/local/bin/dfget COPY --from=builder /app/client/target/release/dfdaemon /usr/local/bin/dfdaemon COPY --from=builder /app/client/target/release/dfcache /usr/local/bin/dfcache +COPY --from=builder /usr/local/bin/tokio-console /usr/local/bin/ COPY --from=pprof /go/bin/pprof /bin/pprof COPY --from=health /bin/grpc_health_probe /bin/grpc_health_probe diff --git a/ci/Dockerfile.debug b/ci/Dockerfile.debug index 54a79d22..4566b2b1 100644 --- a/ci/Dockerfile.debug +++ b/ci/Dockerfile.debug @@ -7,6 +7,7 @@ RUN apt-get update && apt-get install -y \ && rm -rf /var/lib/apt/lists/* COPY Cargo.toml Cargo.lock ./ +COPY .cargo ./cargo COPY dragonfly-client/Cargo.toml ./dragonfly-client/Cargo.toml COPY dragonfly-client/src ./dragonfly-client/src @@ -42,6 +43,7 @@ RUN case "${TARGETPLATFORM}" in \ RUN cargo install flamegraph --root /usr/local RUN cargo install bottom --locked --root /usr/local +RUN cargo install tokio-console --locked --root /usr/local FROM public.ecr.aws/docker/library/alpine:3.20 AS health @@ -72,6 +74,7 @@ COPY --from=builder /app/client/target/debug/dfdaemon /usr/local/bin/dfdaemon COPY --from=builder /app/client/target/debug/dfcache /usr/local/bin/dfcache COPY --from=builder /usr/local/bin/flamegraph /usr/local/bin/ COPY --from=builder /usr/local/bin/btm /usr/local/bin/ +COPY --from=builder /usr/local/bin/tokio-console /usr/local/bin/ COPY --from=pprof /go/bin/pprof /bin/pprof COPY --from=health /bin/grpc_health_probe /bin/grpc_health_probe diff --git a/ci/Dockerfile.dfinit b/ci/Dockerfile.dfinit index f2623dfa..d331dae9 100644 --- a/ci/Dockerfile.dfinit +++ b/ci/Dockerfile.dfinit @@ -7,6 +7,7 @@ RUN apt-get update && apt-get install -y \ WORKDIR /app/client COPY Cargo.toml Cargo.lock ./ +COPY .cargo ./cargo COPY dragonfly-client/Cargo.toml ./dragonfly-client/Cargo.toml COPY dragonfly-client/src ./dragonfly-client/src diff --git a/dragonfly-client-backend/src/http.rs b/dragonfly-client-backend/src/http.rs index e43a29e1..050b052c 100644 --- a/dragonfly-client-backend/src/http.rs +++ b/dragonfly-client-backend/src/http.rs @@ -50,6 +50,17 @@ impl HTTP { .with_custom_certificate_verifier(NoVerifier::new()) .with_no_client_auth(); + // Disable automatic compression to prevent double-decompression issues. + // + // Problem scenario: + // 1. Origin server supports gzip and returns "content-encoding: gzip" header. + // 2. Backend decompresses the response and stores uncompressed content to disk. + // 3. When user's client downloads via dfdaemon proxy, the original "content-encoding: gzip". + // header is forwarded to it. + // 4. User's client attempts to decompress the already-decompressed content, causing errors. + // + // Solution: Disable all compression formats (gzip, brotli, zstd, deflate) to ensure + // we receive and store uncompressed content, eliminating the double-decompression issue. let client = reqwest::Client::builder() .no_gzip() .no_brotli() @@ -88,6 +99,17 @@ impl HTTP { .with_root_certificates(root_cert_store) .with_no_client_auth(); + // Disable automatic compression to prevent double-decompression issues. + // + // Problem scenario: + // 1. Origin server supports gzip and returns "content-encoding: gzip" header. + // 2. Backend decompresses the response and stores uncompressed content to disk. + // 3. When user's client downloads via dfdaemon proxy, the original "content-encoding: gzip". + // header is forwarded to it. + // 4. User's client attempts to decompress the already-decompressed content, causing errors. + // + // Solution: Disable all compression formats (gzip, brotli, zstd, deflate) to ensure + // we receive and store uncompressed content, eliminating the double-decompression issue. let client = reqwest::Client::builder() .no_gzip() .no_brotli() diff --git a/dragonfly-client/Cargo.toml b/dragonfly-client/Cargo.toml index f9928669..015c25f0 100644 --- a/dragonfly-client/Cargo.toml +++ b/dragonfly-client/Cargo.toml @@ -85,6 +85,7 @@ path-absolutize = "3.1.1" dashmap = "6.1.0" fastrand = "2.3.0" glob = "0.3.2" +console-subscriber = "0.4.1" [dev-dependencies] tempfile.workspace = true diff --git a/dragonfly-client/src/tracing/mod.rs b/dragonfly-client/src/tracing/mod.rs index 2f1c6594..8ce15f48 100644 --- a/dragonfly-client/src/tracing/mod.rs +++ b/dragonfly-client/src/tracing/mod.rs @@ -102,8 +102,16 @@ pub fn init_tracing( let env_filter = EnvFilter::try_from_default_env() .unwrap_or_else(|_| EnvFilter::default().add_directive(log_level.into())); + // Enable console subscriber layer for tracing spawn tasks on `127.0.0.1:6669` when log level is TRACE. + let console_subscriber_layer = if log_level == Level::TRACE { + Some(console_subscriber::spawn()) + } else { + None + }; + let subscriber = Registry::default() .with(env_filter) + .with(console_subscriber_layer) .with(file_logging_layer) .with(stdout_logging_layer);