From e430f7421ff095d8f5fbf1b910438f7d35d9bdf3 Mon Sep 17 00:00:00 2001 From: Reid <61492567+reidliu41@users.noreply.github.com> Date: Thu, 24 Sep 2026 17:22:23 +0800 Subject: [PATCH] [Bugfix][Rust Frontend] Prevent MM timing from enabling debug tracing (#58378) Co-authored-by: Bugen Zhao Co-authored-by: Claude Opus 5.5 Signed-off-by: reidliu41 Signed-off-by: Bugen Zhao --- rust/src/chat/Cargo.toml | 2 +- rust/src/chat/src/multimodal/timing.rs | 6 +- rust/src/tracing/src/timing.rs | 80 +++++++++++++++++++++++--- 3 files changed, 79 insertions(+), 9 deletions(-) diff --git a/rust/src/chat/Cargo.toml b/rust/src/chat/Cargo.toml index 1f1f1974ea..a110020ec1 100644 --- a/rust/src/chat/Cargo.toml +++ b/rust/src/chat/Cargo.toml @@ -28,6 +28,7 @@ thiserror-ext.workspace = true time.workspace = true tokio.workspace = true tracing.workspace = true +tracing-subscriber.workspace = true trait-set.workspace = true uuid.workspace = true vllm-engine-core-client.workspace = true @@ -48,7 +49,6 @@ rmp-serde.workspace = true serial_test.workspace = true tempfile.workspace = true tokio.workspace = true -tracing-subscriber.workspace = true uuid.workspace = true vllm-engine-core-client = { workspace = true, features = ["test-util"] } vllm-tokenizer = { workspace = true, features = ["test-utils"] } diff --git a/rust/src/chat/src/multimodal/timing.rs b/rust/src/chat/src/multimodal/timing.rs index 5bdf207396..c57383c7a3 100644 --- a/rust/src/chat/src/multimodal/timing.rs +++ b/rust/src/chat/src/multimodal/timing.rs @@ -4,6 +4,7 @@ //! Multimodal preprocessing spans and their timing collector configuration. use tracing::{Span, info_span}; +use tracing_subscriber::filter::{Filtered, LevelFilter}; use vllm_tracing::timing::{RequestTimingLayer, RequestTimingStats}; /// Target of the multimodal preprocessing stage spans. @@ -12,7 +13,10 @@ pub(super) const MM_STAGE_TARGET: &str = "mm_processor_timing"; /// Create the timing layer and its stats handle for multimodal preprocessing /// stage spans (`vllm-bench mm-processor`), mirroring the Python /// `TimingContext` / `MultiModalTimingRegistry`. -pub fn mm_timing_layer() -> (RequestTimingLayer, RequestTimingStats) { +pub fn mm_timing_layer() -> ( + Filtered, + RequestTimingStats, +) { RequestTimingLayer::new(MM_STAGE_TARGET) } diff --git a/rust/src/tracing/src/timing.rs b/rust/src/tracing/src/timing.rs index f9a50719fb..899e9eaea2 100644 --- a/rust/src/tracing/src/timing.rs +++ b/rust/src/tracing/src/timing.rs @@ -18,6 +18,7 @@ use std::time::Instant; use tracing::field::{Field, Visit}; use tracing::span::Attributes; use tracing::{Id, Subscriber}; +use tracing_subscriber::filter::{Filtered, LevelFilter}; use tracing_subscriber::layer::{Context, Layer}; use tracing_subscriber::registry::LookupSpan; @@ -66,7 +67,8 @@ impl Visit for SpanFields { /// Timed spans must have the configured target and a `stage` field. The /// `request_id` is taken from the timed span itself or its nearest ancestor /// carrying that field, regardless of the ancestor's target. Spans without -/// a request ID are ignored. Span names are unrestricted. +/// a request ID are ignored. Span names are unrestricted. Both the timed spans +/// and their `request_id` ancestors must be at `INFO` level or above. /// /// Both fields are read at span creation; subsequent [`tracing::Span::record`] /// calls are ignored. String values are used verbatim; other values use their @@ -93,13 +95,18 @@ pub struct RequestTimingStats { impl RequestTimingLayer { /// Create the layer and its stats handle. Only spans whose target equals /// `target` and that carry a `stage` field are timed. - pub fn new(target: &'static str) -> (Self, RequestTimingStats) { + /// + /// The layer is filtered to `INFO` so that, composed with a filtered log + /// layer, it does not raise the subscriber's max level hint and enable + /// `DEBUG`/`TRACE` callsites that the log layer would discard anyway. + pub fn new(target: &'static str) -> (Filtered, RequestTimingStats) { let stats = Arc::new(Mutex::new(HashMap::new())); + let layer = Self { + target, + stats: Arc::clone(&stats), + }; ( - Self { - target, - stats: Arc::clone(&stats), - }, + Filtered::new(layer, LevelFilter::INFO), RequestTimingStats { stats }, ) } @@ -165,8 +172,10 @@ where #[cfg(test)] mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + use super::*; - use tracing::info_span; + use tracing::{debug, debug_span, info_span}; use tracing_subscriber::layer::SubscriberExt as _; const TARGET: &str = "test_request_timing"; @@ -218,6 +227,63 @@ mod tests { assert!(stats.stat().is_empty()); } + #[test] + fn does_not_enable_debug_at_info() { + let (timing_layer, _) = RequestTimingLayer::new(TARGET); + let info_layer = tracing_subscriber::fmt::layer() + .with_writer(std::io::sink) + .with_filter(LevelFilter::INFO); + let subscriber = tracing_subscriber::registry().with(timing_layer).with(info_layer); + let evaluated_debug_fields = AtomicUsize::new(0); + + tracing::subscriber::with_default(subscriber, || { + assert_eq!(LevelFilter::current(), LevelFilter::INFO); + debug!( + value = evaluated_debug_fields.fetch_add(1, Ordering::Relaxed), + "debug event" + ); + assert!(debug_span!("debug span").is_disabled()); + }); + + assert_eq!(evaluated_debug_fields.load(Ordering::Relaxed), 0); + } + + #[test] + fn does_not_disable_explicit_debug_logging() { + let (timing_layer, _) = RequestTimingLayer::new(TARGET); + let debug_layer = tracing_subscriber::fmt::layer() + .with_writer(std::io::sink) + .with_filter(LevelFilter::DEBUG); + let subscriber = tracing_subscriber::registry().with(timing_layer).with(debug_layer); + let evaluated_debug_fields = AtomicUsize::new(0); + + tracing::subscriber::with_default(subscriber, || { + assert_eq!(LevelFilter::current(), LevelFilter::DEBUG); + debug!( + value = evaluated_debug_fields.fetch_add(1, Ordering::Relaxed), + "debug event" + ); + assert!(!debug_span!("debug span").is_disabled()); + }); + + assert_eq!(evaluated_debug_fields.load(Ordering::Relaxed), 1); + } + + #[test] + fn records_stage_timings_when_logging_at_warn() { + let (timing_layer, stats) = RequestTimingLayer::new(TARGET); + let warn_layer = tracing_subscriber::fmt::layer() + .with_writer(std::io::sink) + .with_filter(LevelFilter::WARN); + let subscriber = tracing_subscriber::registry().with(timing_layer).with(warn_layer); + tracing::subscriber::with_default(subscriber, || { + info_span!("request", request_id = "req-1").in_scope(|| { + drop(stage_span("media_fetch")); + }); + }); + assert!(stats.stat()["req-1"].contains_key("media_fetch_secs")); + } + #[test] fn other_targets_are_ignored() { let (layer, stats) = RequestTimingLayer::new(TARGET);