[Rust Frontend] Separate multimodal instrumentation from request timing (#58084)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -22,6 +22,7 @@ pub use event::{
|
||||
};
|
||||
use futures::{StreamExt, TryStreamExt as _};
|
||||
pub use llm_multimodal::MediaContentPart;
|
||||
pub use multimodal::{mm_request_span, mm_timing_layer};
|
||||
use output::apply_output_grammar;
|
||||
pub use output::{
|
||||
ChatOutputProcessor, DefaultChatOutputProcessor, DynChatOutputProcessor,
|
||||
@@ -47,9 +48,6 @@ pub use stream::{ChatEventStream, ChatEventStreamTrait, CollectedAssistantMessag
|
||||
pub use vllm_engine_core_client::protocol::multimodal::MmFeatures;
|
||||
pub use vllm_llm::FinishReason;
|
||||
pub use vllm_text::GenerationConfigMode;
|
||||
pub use vllm_tracing::timing::{
|
||||
RequestTimingStats as MultiModalTimingStats, mm_request_span, mm_timing_layer,
|
||||
};
|
||||
|
||||
mod backend;
|
||||
mod error;
|
||||
|
||||
@@ -43,11 +43,13 @@ mod input;
|
||||
mod item;
|
||||
mod preprocessed;
|
||||
mod tensor;
|
||||
mod timing;
|
||||
mod video;
|
||||
|
||||
use self::expand::expand_prompt_token_ids;
|
||||
pub use self::input::MultimodalInput;
|
||||
use vllm_tracing::timing::{mm_request_span, mm_stage_span};
|
||||
use self::timing::MM_STAGE_TARGET;
|
||||
pub use self::timing::{mm_request_span, mm_timing_layer};
|
||||
|
||||
/// Resolved multimodal support for one loaded model.
|
||||
#[derive(Clone)]
|
||||
@@ -760,13 +762,17 @@ impl MultimodalModelInfo {
|
||||
if media_parts.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
self.prepare_multimodal_timed(media_parts, prompt_token_ids, model_dtype)
|
||||
.instrument(mm_stage_span("preprocessor_total"))
|
||||
.await
|
||||
self.prepare_multimodal_timed(media_parts, prompt_token_ids, model_dtype).await
|
||||
}
|
||||
|
||||
/// Timed body of [`Self::prepare_multimodal`]; each stage is a `tracing`
|
||||
/// span aggregated per request by the `vllm-tracing` timing layer.
|
||||
#[tracing::instrument(
|
||||
name = "mm_stage",
|
||||
target = MM_STAGE_TARGET,
|
||||
skip_all,
|
||||
fields(stage = "preprocessor_total")
|
||||
)]
|
||||
async fn prepare_multimodal_timed(
|
||||
&self,
|
||||
media_parts: Vec<MediaContentPart>,
|
||||
@@ -775,34 +781,22 @@ impl MultimodalModelInfo {
|
||||
) -> Result<MmFeatures> {
|
||||
let media_parts_len = media_parts.len();
|
||||
self.validate_mm_limits(&media_parts)?;
|
||||
let fetched =
|
||||
self.fetch_media(media_parts).instrument(mm_stage_span("media_fetch")).await?;
|
||||
let fetched = self.fetch_media(media_parts).await?;
|
||||
|
||||
let mut prepared = Vec::new();
|
||||
if !fetched.images.is_empty() {
|
||||
prepared.push(
|
||||
self.prepare_images(fetched.images, fetched.image_uuids, model_dtype)
|
||||
.instrument(mm_stage_span("preprocess_image"))
|
||||
.await?,
|
||||
);
|
||||
prepared
|
||||
.push(self.prepare_images(fetched.images, fetched.image_uuids, model_dtype).await?);
|
||||
}
|
||||
if !fetched.videos.is_empty() {
|
||||
prepared.push(
|
||||
self.prepare_videos(fetched.videos, fetched.video_uuids, model_dtype)
|
||||
.instrument(mm_stage_span("preprocess_video"))
|
||||
.await?,
|
||||
);
|
||||
prepared
|
||||
.push(self.prepare_videos(fetched.videos, fetched.video_uuids, model_dtype).await?);
|
||||
}
|
||||
if !fetched.audios.is_empty() {
|
||||
prepared.push(
|
||||
self.prepare_audios(fetched.audios, fetched.audio_uuids)
|
||||
.instrument(mm_stage_span("preprocess_audio"))
|
||||
.await?,
|
||||
);
|
||||
prepared.push(self.prepare_audios(fetched.audios, fetched.audio_uuids).await?);
|
||||
}
|
||||
|
||||
let mut ranges = mm_stage_span("prompt_expansion")
|
||||
.in_scope(|| expand_prompt_token_ids(prompt_token_ids, &prepared))?;
|
||||
let mut ranges = expand_prompt_token_ids(prompt_token_ids, &prepared)?;
|
||||
|
||||
let mut features = Vec::with_capacity(media_parts_len);
|
||||
for media in prepared {
|
||||
@@ -845,6 +839,12 @@ impl MultimodalModelInfo {
|
||||
|
||||
/// Fetch all connector-backed media parts and split them per modality,
|
||||
/// preserving their request-order UUID metadata.
|
||||
#[tracing::instrument(
|
||||
name = "mm_stage",
|
||||
target = MM_STAGE_TARGET,
|
||||
skip_all,
|
||||
fields(stage = "media_fetch")
|
||||
)]
|
||||
async fn fetch_media(&self, media_parts: Vec<MediaContentPart>) -> Result<FetchedMedia> {
|
||||
let mut tracker = AsyncMultiModalTracker::new(Arc::clone(&self.media_connector));
|
||||
for part in media_parts {
|
||||
|
||||
@@ -8,6 +8,7 @@ use std::sync::Arc;
|
||||
use llm_multimodal::{AudioClip, Modality, PreprocessedEncoderInputs};
|
||||
use vllm_engine_core_client::protocol::dtype::ModelDtype;
|
||||
|
||||
use super::timing::MM_STAGE_TARGET;
|
||||
use super::{AudioModalitySupport, MultimodalModelInfo, PreparedMedia, item};
|
||||
use crate::error::{Error, Result, bail_multimodal, multimodal};
|
||||
|
||||
@@ -16,6 +17,12 @@ pub(super) const AUDIO_PRIMARY_KEY: &str = "input_audio_features";
|
||||
|
||||
impl MultimodalModelInfo {
|
||||
/// Preprocess fetched audio clips as one batch and build per-item features.
|
||||
#[tracing::instrument(
|
||||
name = "mm_stage",
|
||||
target = MM_STAGE_TARGET,
|
||||
skip_all,
|
||||
fields(stage = "preprocess_audio")
|
||||
)]
|
||||
pub(super) async fn prepare_audios(
|
||||
&self,
|
||||
clips: Vec<Arc<AudioClip>>,
|
||||
@@ -236,6 +243,45 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn audio_pipeline_attributes_stage_timings_to_request() {
|
||||
use tracing::Instrument as _;
|
||||
use tracing::instrument::WithSubscriber as _;
|
||||
use tracing_subscriber::layer::SubscriberExt as _;
|
||||
|
||||
use crate::{mm_request_span, mm_timing_layer};
|
||||
|
||||
let info = qwen3_asr_info();
|
||||
let (layer, timings) = mm_timing_layer();
|
||||
let parts = vec![MediaContentPart::AudioData {
|
||||
data: wav_i16_mono(16_000, &[0; 1_600]),
|
||||
mime_type: Some("audio/wav".to_string()),
|
||||
uuid: Some("audio-1".to_string()),
|
||||
}];
|
||||
async {
|
||||
info.prepare_multimodal(parts, &mut vec![AUDIO_PAD_ID], ModelDtype::Float32)
|
||||
.instrument(mm_request_span("request-1"))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
.with_subscriber(tracing_subscriber::registry().with(layer))
|
||||
.await;
|
||||
|
||||
let records = timings.stat();
|
||||
assert_eq!(records.len(), 1);
|
||||
let mut stages = records["request-1"].keys().collect::<Vec<_>>();
|
||||
stages.sort();
|
||||
expect_test::expect![[r#"
|
||||
[
|
||||
"media_fetch_secs",
|
||||
"preprocess_audio_secs",
|
||||
"preprocessor_total_secs",
|
||||
"prompt_expansion_secs",
|
||||
]
|
||||
"#]]
|
||||
.assert_debug_eq(&stages);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tracker_processor_and_lowering_preserve_audio_contract() {
|
||||
let info = qwen3_asr_info();
|
||||
|
||||
@@ -10,6 +10,7 @@ use vllm_engine_core_client::protocol::multimodal::PlaceholderRange;
|
||||
use vllm_engine_core_client::protocol::tensor::WireTensor;
|
||||
|
||||
use super::PreparedMedia;
|
||||
use super::timing::MM_STAGE_TARGET;
|
||||
use crate::error::{Error, Result, bail_multimodal};
|
||||
|
||||
/// One modality's queue of pending placeholder replacements for prompt
|
||||
@@ -55,6 +56,12 @@ impl<'a> ExpansionLane<'a> {
|
||||
///
|
||||
/// The returned ranges point into the already-expanded prompt, grouped per
|
||||
/// modality in item order.
|
||||
#[tracing::instrument(
|
||||
name = "mm_stage",
|
||||
target = MM_STAGE_TARGET,
|
||||
skip_all,
|
||||
fields(stage = "prompt_expansion")
|
||||
)]
|
||||
pub(super) fn expand_prompt_token_ids(
|
||||
prompt_token_ids: &mut Vec<u32>,
|
||||
prepared: &[PreparedMedia],
|
||||
|
||||
@@ -9,6 +9,7 @@ use std::sync::Arc;
|
||||
use llm_multimodal::{ImageFrame, Modality, PreprocessedEncoderInputs};
|
||||
use vllm_engine_core_client::protocol::dtype::ModelDtype;
|
||||
|
||||
use super::timing::MM_STAGE_TARGET;
|
||||
use super::{ModalitySupport, MultimodalModelInfo, PreparedMedia, item};
|
||||
use crate::error::{Error, Result, bail_multimodal, multimodal};
|
||||
|
||||
@@ -18,6 +19,12 @@ pub(super) const IMAGE_PRIMARY_KEY: &str = "pixel_values";
|
||||
impl MultimodalModelInfo {
|
||||
/// Preprocess all fetched image frames as one batch and build per-item
|
||||
/// features.
|
||||
#[tracing::instrument(
|
||||
name = "mm_stage",
|
||||
target = MM_STAGE_TARGET,
|
||||
skip_all,
|
||||
fields(stage = "preprocess_image")
|
||||
)]
|
||||
pub(super) async fn prepare_images(
|
||||
&self,
|
||||
frames: Vec<Arc<ImageFrame>>,
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
//! Multimodal preprocessing spans and their timing collector configuration.
|
||||
|
||||
use tracing::{Span, info_span};
|
||||
use vllm_tracing::timing::{RequestTimingLayer, RequestTimingStats};
|
||||
|
||||
/// Target of the multimodal preprocessing stage spans.
|
||||
pub(super) const MM_STAGE_TARGET: &str = "mm_processor_timing";
|
||||
|
||||
/// Create the timing layer and its stats handle for multimodal preprocessing
|
||||
/// stage spans (`vllm-bench mm-processor`), mirroring the Python
|
||||
/// `TimingContext` / `MultiModalTimingRegistry`.
|
||||
pub fn mm_timing_layer() -> (RequestTimingLayer, RequestTimingStats) {
|
||||
RequestTimingLayer::new(MM_STAGE_TARGET)
|
||||
}
|
||||
|
||||
/// Span carrying the `request_id` used to attribute multimodal stage timings.
|
||||
pub fn mm_request_span(request_id: &str) -> Span {
|
||||
info_span!("mm_request", request_id)
|
||||
}
|
||||
@@ -16,6 +16,7 @@ use vllm_engine_core_client::protocol::multimodal::{
|
||||
SliceSpec,
|
||||
};
|
||||
|
||||
use super::timing::MM_STAGE_TARGET;
|
||||
use super::{ModalitySupport, MultimodalModelInfo, PreparedItem, PreparedMedia, tensor};
|
||||
use crate::error::{Error, Result, bail_multimodal, multimodal};
|
||||
|
||||
@@ -32,6 +33,12 @@ impl MultimodalModelInfo {
|
||||
/// Unlike images, each clip runs through the preprocessor independently
|
||||
/// (a batch of one), so its tensors are complete per item and need no
|
||||
/// cross-item slicing.
|
||||
#[tracing::instrument(
|
||||
name = "mm_stage",
|
||||
target = MM_STAGE_TARGET,
|
||||
skip_all,
|
||||
fields(stage = "preprocess_video")
|
||||
)]
|
||||
pub(super) async fn prepare_videos(
|
||||
&self,
|
||||
clips: Vec<Arc<VideoClip>>,
|
||||
|
||||
Reference in New Issue
Block a user