[Rust Frontend][gRPC] Add audio and video media inputs (#53760)

Signed-off-by: Connor Carpenter <connorc@nvidia.com>
This commit is contained in:
Connor Carpenter
2026-08-30 11:31:37 +08:00
committed by GitHub
parent b016ed8ea3
commit 7a67941c1d
4 changed files with 124 additions and 50 deletions
+41 -8
View File
@@ -86,20 +86,30 @@ pub(super) fn expand_prompt_token_ids(
} }
let replacement_len = replacement.tokens.len(); let replacement_len = replacement.tokens.len();
let structural_prefix = replacement.structural_prefix;
if structural_prefix > expanded.len() {
bail_multimodal!(
"placeholder token `{}` declares a structural prefix of {} token(s), but only {} token(s) precede it",
lane.placeholder_token,
structural_prefix,
expanded.len()
);
}
let range_len = replacement_len.saturating_add(structural_prefix);
let is_embed = { let is_embed = {
let mask = replacement let mut mask = Vec::with_capacity(range_len);
.tokens mask.extend(std::iter::repeat_n(false, structural_prefix));
.iter() mask.extend(
.map(|&token| token as u32 == lane.embed_token_id) replacement.tokens.iter().map(|&token| token as u32 == lane.embed_token_id),
.collect::<Vec<_>>(); );
WireTensor::from_bool(vec![replacement_len], mask).map_err(Error::Multimodal)? WireTensor::from_bool(vec![range_len], mask).map_err(Error::Multimodal)?
}; };
let expanded_offset = expanded.len(); let expanded_offset = expanded.len() - structural_prefix;
expanded.extend(replacement.tokens.iter().map(|&token| token as u32)); expanded.extend(replacement.tokens.iter().map(|&token| token as u32));
ranges.entry(lane.modality).or_default().push(PlaceholderRange { ranges.entry(lane.modality).or_default().push(PlaceholderRange {
offset: expanded_offset, offset: expanded_offset,
length: replacement_len, length: range_len,
is_embed: Some(is_embed), is_embed: Some(is_embed),
}); });
} }
@@ -420,6 +430,29 @@ mod tests {
assert_bool_mask(&video_ranges[0], &[true, true, true, true]); assert_bool_mask(&video_ranges[0], &[true, true, true, true]);
} }
#[test]
fn expand_prompt_tokens_includes_structural_prefix_in_video_range() {
const VISION_START_ID: u32 = 151652;
const VISION_END_ID: u32 = 151653;
let mut prompt_token_ids = vec![1, VISION_START_ID, QWEN3_VIDEO_PAD_ID, VISION_END_ID, 2];
let replacement = PromptReplacement::repeated(
Modality::Video,
"<|video_pad|>",
QWEN3_VIDEO_PAD_ID as TokenId,
3,
)
.with_structural_prefix(1);
let prepared = vec![qwen3_video_prepared(vec![replacement])];
let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap();
let range = &ranges[&Modality::Video][0];
assert_eq!(range.offset, 1);
assert_eq!(range.length, 4);
assert_bool_mask(range, &[false, true, true, true]);
}
const QWEN3_AUDIO_PAD_ID: u32 = 151_676; const QWEN3_AUDIO_PAD_ID: u32 = 151_676;
fn qwen3_audio_prepared(replacements: Vec<PromptReplacement>) -> PreparedMedia { fn qwen3_audio_prepared(replacements: Vec<PromptReplacement>) -> PreparedMedia {
+52 -35
View File
@@ -22,49 +22,66 @@ pub fn media_parts_from_request(
) -> Result<Vec<MediaContentPart>, Status> { ) -> Result<Vec<MediaContentPart>, Status> {
let mut parts = Vec::with_capacity(media.len()); let mut parts = Vec::with_capacity(media.len());
for (index, item) in media.into_iter().enumerate() { for (index, item) in media.into_iter().enumerate() {
match item.modality() { let modality = item.modality();
pb::Modality::Image => {} if modality == pb::Modality::Unspecified {
pb::Modality::Unspecified => { return Err(Status::invalid_argument(format!(
return Err(Status::invalid_argument(format!( "media[{index}].modality is required"
"media[{index}].modality is required" )));
)));
}
other => {
return Err(Status::unimplemented(format!(
"media[{index}].modality {other:?} is not supported by the gRPC service"
)));
}
} }
let uuid = (!item.uuid.is_empty()).then_some(item.uuid); let uuid = (!item.uuid.is_empty()).then_some(item.uuid);
let mime_type = (!item.mime_type.is_empty()).then_some(item.mime_type); let mime_type = (!item.mime_type.is_empty()).then_some(item.mime_type);
let part = match item.source { let source = item.source.ok_or_else(|| {
Some(pb::media_item::Source::Url(url)) => { Status::invalid_argument(format!("media[{index}].source is required"))
validate_media_uri(index, "url", &url, &["http", "https"])?; })?;
MediaContentPart::ImageUrl { match &source {
url, pb::media_item::Source::Url(url) => {
detail: None, validate_media_uri(index, "url", url, &["http", "https"])?;
uuid,
}
} }
Some(pb::media_item::Source::DataUri(uri)) => { pb::media_item::Source::DataUri(uri) => {
validate_media_uri(index, "data_uri", &uri, &["data"])?; validate_media_uri(index, "data_uri", uri, &["data"])?;
MediaContentPart::ImageUrl {
url: uri,
detail: None,
uuid,
}
} }
Some(pb::media_item::Source::RawBytes(bytes)) => MediaContentPart::ImageData { pb::media_item::Source::RawBytes(_) => {}
data: bytes, }
mime_type, let part = match (modality, source) {
uuid, (
pb::Modality::Image,
pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url),
) => MediaContentPart::ImageUrl {
url,
detail: None, detail: None,
uuid,
}, },
None => { (pb::Modality::Image, pb::media_item::Source::RawBytes(data)) => {
return Err(Status::invalid_argument(format!( MediaContentPart::ImageData {
"media[{index}].source is required" data,
))); mime_type,
uuid,
detail: None,
}
} }
(
pb::Modality::Video,
pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url),
) => MediaContentPart::VideoUrl { url, uuid },
(pb::Modality::Video, pb::media_item::Source::RawBytes(data)) => {
MediaContentPart::VideoData {
data,
mime_type,
uuid,
}
}
(
pb::Modality::Audio,
pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url),
) => MediaContentPart::AudioUrl { url, uuid },
(pb::Modality::Audio, pb::media_item::Source::RawBytes(data)) => {
MediaContentPart::AudioData {
data,
mime_type,
uuid,
}
}
(pb::Modality::Unspecified, _) => unreachable!("modality validated above"),
}; };
parts.push(part); parts.push(part);
} }
@@ -0,0 +1,27 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.models.qwen3_asr import (
Qwen3ASRForConditionalGeneration,
)
from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange
def test_mrope_positions_with_stripped_audio_data() -> None:
input_tokens = list(range(7))
feature = MultiModalFeatureSpec(
data=None,
modality="audio",
identifier="audio",
mm_position=PlaceholderRange(offset=2, length=3),
)
positions, delta = Qwen3ASRForConditionalGeneration.get_mrope_input_positions(
None, input_tokens, [feature]
)
expected = torch.arange(len(input_tokens), dtype=torch.long).expand(3, -1)
torch.testing.assert_close(positions, expected)
assert delta == 0
+4 -7
View File
@@ -575,13 +575,10 @@ class Qwen3ASRForConditionalGeneration(
for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset): for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
offset = mm_feature.mm_position.offset offset = mm_feature.mm_position.offset
# Get audio feature length from mm_feature data # The placeholder already contains one token per audio embedding.
audio_feature_length = mm_feature.data["audio_feature_lengths"].data # Use its length so M-RoPE can be reconstructed even when a covered
if isinstance(audio_feature_length, torch.Tensor): # prefix no longer carries the audio processor tensors.
audio_feature_length = audio_feature_length.item() audio_len = mm_feature.mm_position.length
audio_len = _get_feat_extract_output_lengths(
torch.tensor(audio_feature_length)
).item()
# Text segment before audio (includes audio_start token) # Text segment before audio (includes audio_start token)
text_len = offset - st text_len = offset - st