From 7a67941c1d0506d0690fbb4e91b573c660d24981 Mon Sep 17 00:00:00 2001 From: Connor Carpenter Date: Sat, 29 Aug 2026 23:31:37 -0400 Subject: [PATCH] [Rust Frontend][gRPC] Add audio and video media inputs (#53760) Signed-off-by: Connor Carpenter --- rust/src/chat/src/multimodal/expand.rs | 49 +++++++++-- rust/src/server/src/grpc/convert.rs | 87 ++++++++++++-------- tests/model_executor/test_qwen3_asr_mrope.py | 27 ++++++ vllm/model_executor/models/qwen3_asr.py | 11 +-- 4 files changed, 124 insertions(+), 50 deletions(-) create mode 100644 tests/model_executor/test_qwen3_asr_mrope.py diff --git a/rust/src/chat/src/multimodal/expand.rs b/rust/src/chat/src/multimodal/expand.rs index e449de5909..884df2fe44 100644 --- a/rust/src/chat/src/multimodal/expand.rs +++ b/rust/src/chat/src/multimodal/expand.rs @@ -86,20 +86,30 @@ pub(super) fn expand_prompt_token_ids( } let replacement_len = replacement.tokens.len(); + let structural_prefix = replacement.structural_prefix; + if structural_prefix > expanded.len() { + bail_multimodal!( + "placeholder token `{}` declares a structural prefix of {} token(s), but only {} token(s) precede it", + lane.placeholder_token, + structural_prefix, + expanded.len() + ); + } + let range_len = replacement_len.saturating_add(structural_prefix); let is_embed = { - let mask = replacement - .tokens - .iter() - .map(|&token| token as u32 == lane.embed_token_id) - .collect::>(); - WireTensor::from_bool(vec![replacement_len], mask).map_err(Error::Multimodal)? + let mut mask = Vec::with_capacity(range_len); + mask.extend(std::iter::repeat_n(false, structural_prefix)); + mask.extend( + replacement.tokens.iter().map(|&token| token as u32 == lane.embed_token_id), + ); + WireTensor::from_bool(vec![range_len], mask).map_err(Error::Multimodal)? }; - let expanded_offset = expanded.len(); + let expanded_offset = expanded.len() - structural_prefix; expanded.extend(replacement.tokens.iter().map(|&token| token as u32)); ranges.entry(lane.modality).or_default().push(PlaceholderRange { offset: expanded_offset, - length: replacement_len, + length: range_len, is_embed: Some(is_embed), }); } @@ -420,6 +430,29 @@ mod tests { assert_bool_mask(&video_ranges[0], &[true, true, true, true]); } + #[test] + fn expand_prompt_tokens_includes_structural_prefix_in_video_range() { + const VISION_START_ID: u32 = 151652; + const VISION_END_ID: u32 = 151653; + + let mut prompt_token_ids = vec![1, VISION_START_ID, QWEN3_VIDEO_PAD_ID, VISION_END_ID, 2]; + let replacement = PromptReplacement::repeated( + Modality::Video, + "<|video_pad|>", + QWEN3_VIDEO_PAD_ID as TokenId, + 3, + ) + .with_structural_prefix(1); + let prepared = vec![qwen3_video_prepared(vec![replacement])]; + + let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap(); + + let range = &ranges[&Modality::Video][0]; + assert_eq!(range.offset, 1); + assert_eq!(range.length, 4); + assert_bool_mask(range, &[false, true, true, true]); + } + const QWEN3_AUDIO_PAD_ID: u32 = 151_676; fn qwen3_audio_prepared(replacements: Vec) -> PreparedMedia { diff --git a/rust/src/server/src/grpc/convert.rs b/rust/src/server/src/grpc/convert.rs index 06ce8010de..93669bd1b2 100644 --- a/rust/src/server/src/grpc/convert.rs +++ b/rust/src/server/src/grpc/convert.rs @@ -22,49 +22,66 @@ pub fn media_parts_from_request( ) -> Result, Status> { let mut parts = Vec::with_capacity(media.len()); for (index, item) in media.into_iter().enumerate() { - match item.modality() { - pb::Modality::Image => {} - pb::Modality::Unspecified => { - return Err(Status::invalid_argument(format!( - "media[{index}].modality is required" - ))); - } - other => { - return Err(Status::unimplemented(format!( - "media[{index}].modality {other:?} is not supported by the gRPC service" - ))); - } + let modality = item.modality(); + if modality == pb::Modality::Unspecified { + return Err(Status::invalid_argument(format!( + "media[{index}].modality is required" + ))); } let uuid = (!item.uuid.is_empty()).then_some(item.uuid); let mime_type = (!item.mime_type.is_empty()).then_some(item.mime_type); - let part = match item.source { - Some(pb::media_item::Source::Url(url)) => { - validate_media_uri(index, "url", &url, &["http", "https"])?; - MediaContentPart::ImageUrl { - url, - detail: None, - uuid, - } + let source = item.source.ok_or_else(|| { + Status::invalid_argument(format!("media[{index}].source is required")) + })?; + match &source { + pb::media_item::Source::Url(url) => { + validate_media_uri(index, "url", url, &["http", "https"])?; } - Some(pb::media_item::Source::DataUri(uri)) => { - validate_media_uri(index, "data_uri", &uri, &["data"])?; - MediaContentPart::ImageUrl { - url: uri, - detail: None, - uuid, - } + pb::media_item::Source::DataUri(uri) => { + validate_media_uri(index, "data_uri", uri, &["data"])?; } - Some(pb::media_item::Source::RawBytes(bytes)) => MediaContentPart::ImageData { - data: bytes, - mime_type, - uuid, + pb::media_item::Source::RawBytes(_) => {} + } + let part = match (modality, source) { + ( + pb::Modality::Image, + pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url), + ) => MediaContentPart::ImageUrl { + url, detail: None, + uuid, }, - None => { - return Err(Status::invalid_argument(format!( - "media[{index}].source is required" - ))); + (pb::Modality::Image, pb::media_item::Source::RawBytes(data)) => { + MediaContentPart::ImageData { + data, + mime_type, + uuid, + detail: None, + } } + ( + pb::Modality::Video, + pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url), + ) => MediaContentPart::VideoUrl { url, uuid }, + (pb::Modality::Video, pb::media_item::Source::RawBytes(data)) => { + MediaContentPart::VideoData { + data, + mime_type, + uuid, + } + } + ( + pb::Modality::Audio, + pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url), + ) => MediaContentPart::AudioUrl { url, uuid }, + (pb::Modality::Audio, pb::media_item::Source::RawBytes(data)) => { + MediaContentPart::AudioData { + data, + mime_type, + uuid, + } + } + (pb::Modality::Unspecified, _) => unreachable!("modality validated above"), }; parts.push(part); } diff --git a/tests/model_executor/test_qwen3_asr_mrope.py b/tests/model_executor/test_qwen3_asr_mrope.py new file mode 100644 index 0000000000..9cae99f27a --- /dev/null +++ b/tests/model_executor/test_qwen3_asr_mrope.py @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch + +from vllm.model_executor.models.qwen3_asr import ( + Qwen3ASRForConditionalGeneration, +) +from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange + + +def test_mrope_positions_with_stripped_audio_data() -> None: + input_tokens = list(range(7)) + feature = MultiModalFeatureSpec( + data=None, + modality="audio", + identifier="audio", + mm_position=PlaceholderRange(offset=2, length=3), + ) + + positions, delta = Qwen3ASRForConditionalGeneration.get_mrope_input_positions( + None, input_tokens, [feature] + ) + + expected = torch.arange(len(input_tokens), dtype=torch.long).expand(3, -1) + torch.testing.assert_close(positions, expected) + assert delta == 0 diff --git a/vllm/model_executor/models/qwen3_asr.py b/vllm/model_executor/models/qwen3_asr.py index 921d14dd13..e98f37ba37 100644 --- a/vllm/model_executor/models/qwen3_asr.py +++ b/vllm/model_executor/models/qwen3_asr.py @@ -575,13 +575,10 @@ class Qwen3ASRForConditionalGeneration( for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset): offset = mm_feature.mm_position.offset - # Get audio feature length from mm_feature data - audio_feature_length = mm_feature.data["audio_feature_lengths"].data - if isinstance(audio_feature_length, torch.Tensor): - audio_feature_length = audio_feature_length.item() - audio_len = _get_feat_extract_output_lengths( - torch.tensor(audio_feature_length) - ).item() + # The placeholder already contains one token per audio embedding. + # Use its length so M-RoPE can be reconstructed even when a covered + # prefix no longer carries the audio processor tensors. + audio_len = mm_feature.mm_position.length # Text segment before audio (includes audio_start token) text_len = offset - st