[Rust Frontend] Separate multimodal instrumentation from request timing (#58084)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-09-22 06:57:56 +00:00
committed by GitHub
parent 2af05512cf
commit a6c47fbbf4
9 changed files with 143 additions and 52 deletions
+4 -3
View File
@@ -16,13 +16,14 @@ use tokio::sync::Semaphore;
use vllm_chat::{ use vllm_chat::{
ChatContent, ChatContentPart, ChatLlm, ChatMessage, ChatRequest, LoadModelBackendsOptions, ChatContent, ChatContentPart, ChatLlm, ChatMessage, ChatRequest, LoadModelBackendsOptions,
MultiModalTimingStats, load_model_backends, load_model_backends,
}; };
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, TransportMode}; use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, TransportMode};
use vllm_llm::Llm; use vllm_llm::Llm;
use vllm_managed_engine::cli::ManagedEngineArgs; use vllm_managed_engine::cli::ManagedEngineArgs;
use vllm_managed_engine::{ManagedEngineConfig, ManagedEngineHandle}; use vllm_managed_engine::{ManagedEngineConfig, ManagedEngineHandle};
use vllm_text::TextLlm; use vllm_text::TextLlm;
use vllm_tracing::timing::RequestTimingStats;
use crate::config::RangeRatio; use crate::config::RangeRatio;
use crate::datasets::SampleRequest; use crate::datasets::SampleRequest;
@@ -133,7 +134,7 @@ pub struct MmProcessorArgs {
} }
/// Entry point for `vllm-bench mm-processor`. /// Entry point for `vllm-bench mm-processor`.
pub async fn run_mm_processor(args: MmProcessorArgs, timing: MultiModalTimingStats) -> Result<()> { pub async fn run_mm_processor(args: MmProcessorArgs, timing: RequestTimingStats) -> Result<()> {
if args.engine.data_parallel_size_local == Some(0) { if args.engine.data_parallel_size_local == Some(0) {
anyhow::bail!( anyhow::bail!(
"`--data-parallel-size-local 0` is not supported; the benchmark requires \ "`--data-parallel-size-local 0` is not supported; the benchmark requires \
@@ -179,7 +180,7 @@ pub async fn run_mm_processor(args: MmProcessorArgs, timing: MultiModalTimingSta
async fn run_mm_processor_with_engine( async fn run_mm_processor_with_engine(
args: MmProcessorArgs, args: MmProcessorArgs,
handshake_port: u16, handshake_port: u16,
timing: &MultiModalTimingStats, timing: &RequestTimingStats,
) -> Result<()> { ) -> Result<()> {
let range_ratio = let range_ratio =
RangeRatio::parse(&args.random_range_ratio).context("invalid --random-range-ratio")?; RangeRatio::parse(&args.random_range_ratio).context("invalid --random-range-ratio")?;
+1 -3
View File
@@ -22,6 +22,7 @@ pub use event::{
}; };
use futures::{StreamExt, TryStreamExt as _}; use futures::{StreamExt, TryStreamExt as _};
pub use llm_multimodal::MediaContentPart; pub use llm_multimodal::MediaContentPart;
pub use multimodal::{mm_request_span, mm_timing_layer};
use output::apply_output_grammar; use output::apply_output_grammar;
pub use output::{ pub use output::{
ChatOutputProcessor, DefaultChatOutputProcessor, DynChatOutputProcessor, ChatOutputProcessor, DefaultChatOutputProcessor, DynChatOutputProcessor,
@@ -47,9 +48,6 @@ pub use stream::{ChatEventStream, ChatEventStreamTrait, CollectedAssistantMessag
pub use vllm_engine_core_client::protocol::multimodal::MmFeatures; pub use vllm_engine_core_client::protocol::multimodal::MmFeatures;
pub use vllm_llm::FinishReason; pub use vllm_llm::FinishReason;
pub use vllm_text::GenerationConfigMode; pub use vllm_text::GenerationConfigMode;
pub use vllm_tracing::timing::{
RequestTimingStats as MultiModalTimingStats, mm_request_span, mm_timing_layer,
};
mod backend; mod backend;
mod error; mod error;
+23 -23
View File
@@ -43,11 +43,13 @@ mod input;
mod item; mod item;
mod preprocessed; mod preprocessed;
mod tensor; mod tensor;
mod timing;
mod video; mod video;
use self::expand::expand_prompt_token_ids; use self::expand::expand_prompt_token_ids;
pub use self::input::MultimodalInput; pub use self::input::MultimodalInput;
use vllm_tracing::timing::{mm_request_span, mm_stage_span}; use self::timing::MM_STAGE_TARGET;
pub use self::timing::{mm_request_span, mm_timing_layer};
/// Resolved multimodal support for one loaded model. /// Resolved multimodal support for one loaded model.
#[derive(Clone)] #[derive(Clone)]
@@ -760,13 +762,17 @@ impl MultimodalModelInfo {
if media_parts.is_empty() { if media_parts.is_empty() {
return Ok(Vec::new()); return Ok(Vec::new());
} }
self.prepare_multimodal_timed(media_parts, prompt_token_ids, model_dtype) self.prepare_multimodal_timed(media_parts, prompt_token_ids, model_dtype).await
.instrument(mm_stage_span("preprocessor_total"))
.await
} }
/// Timed body of [`Self::prepare_multimodal`]; each stage is a `tracing` /// Timed body of [`Self::prepare_multimodal`]; each stage is a `tracing`
/// span aggregated per request by the `vllm-tracing` timing layer. /// span aggregated per request by the `vllm-tracing` timing layer.
#[tracing::instrument(
name = "mm_stage",
target = MM_STAGE_TARGET,
skip_all,
fields(stage = "preprocessor_total")
)]
async fn prepare_multimodal_timed( async fn prepare_multimodal_timed(
&self, &self,
media_parts: Vec<MediaContentPart>, media_parts: Vec<MediaContentPart>,
@@ -775,34 +781,22 @@ impl MultimodalModelInfo {
) -> Result<MmFeatures> { ) -> Result<MmFeatures> {
let media_parts_len = media_parts.len(); let media_parts_len = media_parts.len();
self.validate_mm_limits(&media_parts)?; self.validate_mm_limits(&media_parts)?;
let fetched = let fetched = self.fetch_media(media_parts).await?;
self.fetch_media(media_parts).instrument(mm_stage_span("media_fetch")).await?;
let mut prepared = Vec::new(); let mut prepared = Vec::new();
if !fetched.images.is_empty() { if !fetched.images.is_empty() {
prepared.push( prepared
self.prepare_images(fetched.images, fetched.image_uuids, model_dtype) .push(self.prepare_images(fetched.images, fetched.image_uuids, model_dtype).await?);
.instrument(mm_stage_span("preprocess_image"))
.await?,
);
} }
if !fetched.videos.is_empty() { if !fetched.videos.is_empty() {
prepared.push( prepared
self.prepare_videos(fetched.videos, fetched.video_uuids, model_dtype) .push(self.prepare_videos(fetched.videos, fetched.video_uuids, model_dtype).await?);
.instrument(mm_stage_span("preprocess_video"))
.await?,
);
} }
if !fetched.audios.is_empty() { if !fetched.audios.is_empty() {
prepared.push( prepared.push(self.prepare_audios(fetched.audios, fetched.audio_uuids).await?);
self.prepare_audios(fetched.audios, fetched.audio_uuids)
.instrument(mm_stage_span("preprocess_audio"))
.await?,
);
} }
let mut ranges = mm_stage_span("prompt_expansion") let mut ranges = expand_prompt_token_ids(prompt_token_ids, &prepared)?;
.in_scope(|| expand_prompt_token_ids(prompt_token_ids, &prepared))?;
let mut features = Vec::with_capacity(media_parts_len); let mut features = Vec::with_capacity(media_parts_len);
for media in prepared { for media in prepared {
@@ -845,6 +839,12 @@ impl MultimodalModelInfo {
/// Fetch all connector-backed media parts and split them per modality, /// Fetch all connector-backed media parts and split them per modality,
/// preserving their request-order UUID metadata. /// preserving their request-order UUID metadata.
#[tracing::instrument(
name = "mm_stage",
target = MM_STAGE_TARGET,
skip_all,
fields(stage = "media_fetch")
)]
async fn fetch_media(&self, media_parts: Vec<MediaContentPart>) -> Result<FetchedMedia> { async fn fetch_media(&self, media_parts: Vec<MediaContentPart>) -> Result<FetchedMedia> {
let mut tracker = AsyncMultiModalTracker::new(Arc::clone(&self.media_connector)); let mut tracker = AsyncMultiModalTracker::new(Arc::clone(&self.media_connector));
for part in media_parts { for part in media_parts {
+46
View File
@@ -8,6 +8,7 @@ use std::sync::Arc;
use llm_multimodal::{AudioClip, Modality, PreprocessedEncoderInputs}; use llm_multimodal::{AudioClip, Modality, PreprocessedEncoderInputs};
use vllm_engine_core_client::protocol::dtype::ModelDtype; use vllm_engine_core_client::protocol::dtype::ModelDtype;
use super::timing::MM_STAGE_TARGET;
use super::{AudioModalitySupport, MultimodalModelInfo, PreparedMedia, item}; use super::{AudioModalitySupport, MultimodalModelInfo, PreparedMedia, item};
use crate::error::{Error, Result, bail_multimodal, multimodal}; use crate::error::{Error, Result, bail_multimodal, multimodal};
@@ -16,6 +17,12 @@ pub(super) const AUDIO_PRIMARY_KEY: &str = "input_audio_features";
impl MultimodalModelInfo { impl MultimodalModelInfo {
/// Preprocess fetched audio clips as one batch and build per-item features. /// Preprocess fetched audio clips as one batch and build per-item features.
#[tracing::instrument(
name = "mm_stage",
target = MM_STAGE_TARGET,
skip_all,
fields(stage = "preprocess_audio")
)]
pub(super) async fn prepare_audios( pub(super) async fn prepare_audios(
&self, &self,
clips: Vec<Arc<AudioClip>>, clips: Vec<Arc<AudioClip>>,
@@ -236,6 +243,45 @@ mod tests {
)); ));
} }
#[tokio::test]
async fn audio_pipeline_attributes_stage_timings_to_request() {
use tracing::Instrument as _;
use tracing::instrument::WithSubscriber as _;
use tracing_subscriber::layer::SubscriberExt as _;
use crate::{mm_request_span, mm_timing_layer};
let info = qwen3_asr_info();
let (layer, timings) = mm_timing_layer();
let parts = vec![MediaContentPart::AudioData {
data: wav_i16_mono(16_000, &[0; 1_600]),
mime_type: Some("audio/wav".to_string()),
uuid: Some("audio-1".to_string()),
}];
async {
info.prepare_multimodal(parts, &mut vec![AUDIO_PAD_ID], ModelDtype::Float32)
.instrument(mm_request_span("request-1"))
.await
.unwrap();
}
.with_subscriber(tracing_subscriber::registry().with(layer))
.await;
let records = timings.stat();
assert_eq!(records.len(), 1);
let mut stages = records["request-1"].keys().collect::<Vec<_>>();
stages.sort();
expect_test::expect![[r#"
[
"media_fetch_secs",
"preprocess_audio_secs",
"preprocessor_total_secs",
"prompt_expansion_secs",
]
"#]]
.assert_debug_eq(&stages);
}
#[tokio::test] #[tokio::test]
async fn tracker_processor_and_lowering_preserve_audio_contract() { async fn tracker_processor_and_lowering_preserve_audio_contract() {
let info = qwen3_asr_info(); let info = qwen3_asr_info();
+7
View File
@@ -10,6 +10,7 @@ use vllm_engine_core_client::protocol::multimodal::PlaceholderRange;
use vllm_engine_core_client::protocol::tensor::WireTensor; use vllm_engine_core_client::protocol::tensor::WireTensor;
use super::PreparedMedia; use super::PreparedMedia;
use super::timing::MM_STAGE_TARGET;
use crate::error::{Error, Result, bail_multimodal}; use crate::error::{Error, Result, bail_multimodal};
/// One modality's queue of pending placeholder replacements for prompt /// One modality's queue of pending placeholder replacements for prompt
@@ -55,6 +56,12 @@ impl<'a> ExpansionLane<'a> {
/// ///
/// The returned ranges point into the already-expanded prompt, grouped per /// The returned ranges point into the already-expanded prompt, grouped per
/// modality in item order. /// modality in item order.
#[tracing::instrument(
name = "mm_stage",
target = MM_STAGE_TARGET,
skip_all,
fields(stage = "prompt_expansion")
)]
pub(super) fn expand_prompt_token_ids( pub(super) fn expand_prompt_token_ids(
prompt_token_ids: &mut Vec<u32>, prompt_token_ids: &mut Vec<u32>,
prepared: &[PreparedMedia], prepared: &[PreparedMedia],
+7
View File
@@ -9,6 +9,7 @@ use std::sync::Arc;
use llm_multimodal::{ImageFrame, Modality, PreprocessedEncoderInputs}; use llm_multimodal::{ImageFrame, Modality, PreprocessedEncoderInputs};
use vllm_engine_core_client::protocol::dtype::ModelDtype; use vllm_engine_core_client::protocol::dtype::ModelDtype;
use super::timing::MM_STAGE_TARGET;
use super::{ModalitySupport, MultimodalModelInfo, PreparedMedia, item}; use super::{ModalitySupport, MultimodalModelInfo, PreparedMedia, item};
use crate::error::{Error, Result, bail_multimodal, multimodal}; use crate::error::{Error, Result, bail_multimodal, multimodal};
@@ -18,6 +19,12 @@ pub(super) const IMAGE_PRIMARY_KEY: &str = "pixel_values";
impl MultimodalModelInfo { impl MultimodalModelInfo {
/// Preprocess all fetched image frames as one batch and build per-item /// Preprocess all fetched image frames as one batch and build per-item
/// features. /// features.
#[tracing::instrument(
name = "mm_stage",
target = MM_STAGE_TARGET,
skip_all,
fields(stage = "preprocess_image")
)]
pub(super) async fn prepare_images( pub(super) async fn prepare_images(
&self, &self,
frames: Vec<Arc<ImageFrame>>, frames: Vec<Arc<ImageFrame>>,
+22
View File
@@ -0,0 +1,22 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
//! Multimodal preprocessing spans and their timing collector configuration.
use tracing::{Span, info_span};
use vllm_tracing::timing::{RequestTimingLayer, RequestTimingStats};
/// Target of the multimodal preprocessing stage spans.
pub(super) const MM_STAGE_TARGET: &str = "mm_processor_timing";
/// Create the timing layer and its stats handle for multimodal preprocessing
/// stage spans (`vllm-bench mm-processor`), mirroring the Python
/// `TimingContext` / `MultiModalTimingRegistry`.
pub fn mm_timing_layer() -> (RequestTimingLayer, RequestTimingStats) {
RequestTimingLayer::new(MM_STAGE_TARGET)
}
/// Span carrying the `request_id` used to attribute multimodal stage timings.
pub fn mm_request_span(request_id: &str) -> Span {
info_span!("mm_request", request_id)
}
+7
View File
@@ -16,6 +16,7 @@ use vllm_engine_core_client::protocol::multimodal::{
SliceSpec, SliceSpec,
}; };
use super::timing::MM_STAGE_TARGET;
use super::{ModalitySupport, MultimodalModelInfo, PreparedItem, PreparedMedia, tensor}; use super::{ModalitySupport, MultimodalModelInfo, PreparedItem, PreparedMedia, tensor};
use crate::error::{Error, Result, bail_multimodal, multimodal}; use crate::error::{Error, Result, bail_multimodal, multimodal};
@@ -32,6 +33,12 @@ impl MultimodalModelInfo {
/// Unlike images, each clip runs through the preprocessor independently /// Unlike images, each clip runs through the preprocessor independently
/// (a batch of one), so its tensors are complete per item and need no /// (a batch of one), so its tensors are complete per item and need no
/// cross-item slicing. /// cross-item slicing.
#[tracing::instrument(
name = "mm_stage",
target = MM_STAGE_TARGET,
skip_all,
fields(stage = "preprocess_video")
)]
pub(super) async fn prepare_videos( pub(super) async fn prepare_videos(
&self, &self,
clips: Vec<Arc<VideoClip>>, clips: Vec<Arc<VideoClip>>,
+26 -23
View File
@@ -16,8 +16,8 @@ use std::sync::{Arc, Mutex};
use std::time::Instant; use std::time::Instant;
use tracing::field::{Field, Visit}; use tracing::field::{Field, Visit};
use tracing::span::{Attributes, Span}; use tracing::span::Attributes;
use tracing::{Id, Subscriber, info_span}; use tracing::{Id, Subscriber};
use tracing_subscriber::layer::{Context, Layer}; use tracing_subscriber::layer::{Context, Layer};
use tracing_subscriber::registry::LookupSpan; use tracing_subscriber::registry::LookupSpan;
@@ -60,6 +60,26 @@ impl Visit for SpanFields {
/// `tracing` layer that aggregates stage-span timings per request, keyed by /// `tracing` layer that aggregates stage-span timings per request, keyed by
/// the nearest ancestor span with a `request_id` field. /// the nearest ancestor span with a `request_id` field.
///
/// # Span contract
///
/// Timed spans must have the configured target and a `stage` field. The
/// `request_id` is taken from the timed span itself or its nearest ancestor
/// carrying that field, regardless of the ancestor's target. Spans without
/// a request ID are ignored. Span names are unrestricted.
///
/// Both fields are read at span creation; subsequent [`tracing::Span::record`]
/// calls are ignored. String values are used verbatim; other values use their
/// debug representation. Prefer string fields for stable keys.
///
/// # Timing semantics
///
/// Each sample measures wall-clock time from span creation to final close,
/// including async waits and any time the span remains alive through cloned
/// handles. Nested and concurrent spans are measured independently, so their
/// durations can overlap. Repeated stages accumulate under `"{stage}_secs"`
/// for each request. Closing spans are recorded regardless of the operation's
/// success, failure, or cancellation; callers select the samples to report.
pub struct RequestTimingLayer { pub struct RequestTimingLayer {
target: &'static str, target: &'static str,
stats: Arc<Mutex<HashMap<String, StageStats>>>, stats: Arc<Mutex<HashMap<String, StageStats>>>,
@@ -87,6 +107,10 @@ impl RequestTimingLayer {
impl RequestTimingStats { impl RequestTimingStats {
/// Drain and return `{request_id: {stage_secs}}` records. /// Drain and return `{request_id: {stage_secs}}` records.
///
/// Returns accumulated timings since the previous drain. Spans still open
/// contribute to a later drain; finish the measured work before calling
/// this method to collect complete request timings.
pub fn stat(&self) -> HashMap<String, StageStats> { pub fn stat(&self) -> HashMap<String, StageStats> {
std::mem::take(&mut *self.stats.lock().unwrap()) std::mem::take(&mut *self.stats.lock().unwrap())
} }
@@ -139,27 +163,6 @@ where
} }
} }
/// Target of the multimodal preprocessing stage spans.
const MM_STAGE_TARGET: &str = "mm_processor_timing";
/// Create the timing layer and its stats handle for multimodal preprocessing
/// stage spans (`vllm-bench mm-processor`), mirroring the Python
/// `TimingContext` / `MultiModalTimingRegistry`.
pub fn mm_timing_layer() -> (RequestTimingLayer, RequestTimingStats) {
RequestTimingLayer::new(MM_STAGE_TARGET)
}
/// Span carrying the `request_id` used to attribute multimodal stage timings.
pub fn mm_request_span(request_id: &str) -> Span {
info_span!("mm_request", request_id)
}
/// Span for one multimodal preprocessing stage; the elapsed time is recorded
/// on close.
pub fn mm_stage_span(stage: &'static str) -> Span {
info_span!(target: MM_STAGE_TARGET, "mm_stage", stage)
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;