[Rust Frontend][gRPC] Add audio and video media inputs (#53760)
Signed-off-by: Connor Carpenter <connorc@nvidia.com>
This commit is contained in:
@@ -86,20 +86,30 @@ pub(super) fn expand_prompt_token_ids(
|
|||||||
}
|
}
|
||||||
|
|
||||||
let replacement_len = replacement.tokens.len();
|
let replacement_len = replacement.tokens.len();
|
||||||
|
let structural_prefix = replacement.structural_prefix;
|
||||||
|
if structural_prefix > expanded.len() {
|
||||||
|
bail_multimodal!(
|
||||||
|
"placeholder token `{}` declares a structural prefix of {} token(s), but only {} token(s) precede it",
|
||||||
|
lane.placeholder_token,
|
||||||
|
structural_prefix,
|
||||||
|
expanded.len()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let range_len = replacement_len.saturating_add(structural_prefix);
|
||||||
let is_embed = {
|
let is_embed = {
|
||||||
let mask = replacement
|
let mut mask = Vec::with_capacity(range_len);
|
||||||
.tokens
|
mask.extend(std::iter::repeat_n(false, structural_prefix));
|
||||||
.iter()
|
mask.extend(
|
||||||
.map(|&token| token as u32 == lane.embed_token_id)
|
replacement.tokens.iter().map(|&token| token as u32 == lane.embed_token_id),
|
||||||
.collect::<Vec<_>>();
|
);
|
||||||
WireTensor::from_bool(vec![replacement_len], mask).map_err(Error::Multimodal)?
|
WireTensor::from_bool(vec![range_len], mask).map_err(Error::Multimodal)?
|
||||||
};
|
};
|
||||||
|
|
||||||
let expanded_offset = expanded.len();
|
let expanded_offset = expanded.len() - structural_prefix;
|
||||||
expanded.extend(replacement.tokens.iter().map(|&token| token as u32));
|
expanded.extend(replacement.tokens.iter().map(|&token| token as u32));
|
||||||
ranges.entry(lane.modality).or_default().push(PlaceholderRange {
|
ranges.entry(lane.modality).or_default().push(PlaceholderRange {
|
||||||
offset: expanded_offset,
|
offset: expanded_offset,
|
||||||
length: replacement_len,
|
length: range_len,
|
||||||
is_embed: Some(is_embed),
|
is_embed: Some(is_embed),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -420,6 +430,29 @@ mod tests {
|
|||||||
assert_bool_mask(&video_ranges[0], &[true, true, true, true]);
|
assert_bool_mask(&video_ranges[0], &[true, true, true, true]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn expand_prompt_tokens_includes_structural_prefix_in_video_range() {
|
||||||
|
const VISION_START_ID: u32 = 151652;
|
||||||
|
const VISION_END_ID: u32 = 151653;
|
||||||
|
|
||||||
|
let mut prompt_token_ids = vec![1, VISION_START_ID, QWEN3_VIDEO_PAD_ID, VISION_END_ID, 2];
|
||||||
|
let replacement = PromptReplacement::repeated(
|
||||||
|
Modality::Video,
|
||||||
|
"<|video_pad|>",
|
||||||
|
QWEN3_VIDEO_PAD_ID as TokenId,
|
||||||
|
3,
|
||||||
|
)
|
||||||
|
.with_structural_prefix(1);
|
||||||
|
let prepared = vec![qwen3_video_prepared(vec![replacement])];
|
||||||
|
|
||||||
|
let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap();
|
||||||
|
|
||||||
|
let range = &ranges[&Modality::Video][0];
|
||||||
|
assert_eq!(range.offset, 1);
|
||||||
|
assert_eq!(range.length, 4);
|
||||||
|
assert_bool_mask(range, &[false, true, true, true]);
|
||||||
|
}
|
||||||
|
|
||||||
const QWEN3_AUDIO_PAD_ID: u32 = 151_676;
|
const QWEN3_AUDIO_PAD_ID: u32 = 151_676;
|
||||||
|
|
||||||
fn qwen3_audio_prepared(replacements: Vec<PromptReplacement>) -> PreparedMedia {
|
fn qwen3_audio_prepared(replacements: Vec<PromptReplacement>) -> PreparedMedia {
|
||||||
|
|||||||
@@ -22,49 +22,66 @@ pub fn media_parts_from_request(
|
|||||||
) -> Result<Vec<MediaContentPart>, Status> {
|
) -> Result<Vec<MediaContentPart>, Status> {
|
||||||
let mut parts = Vec::with_capacity(media.len());
|
let mut parts = Vec::with_capacity(media.len());
|
||||||
for (index, item) in media.into_iter().enumerate() {
|
for (index, item) in media.into_iter().enumerate() {
|
||||||
match item.modality() {
|
let modality = item.modality();
|
||||||
pb::Modality::Image => {}
|
if modality == pb::Modality::Unspecified {
|
||||||
pb::Modality::Unspecified => {
|
return Err(Status::invalid_argument(format!(
|
||||||
return Err(Status::invalid_argument(format!(
|
"media[{index}].modality is required"
|
||||||
"media[{index}].modality is required"
|
)));
|
||||||
)));
|
|
||||||
}
|
|
||||||
other => {
|
|
||||||
return Err(Status::unimplemented(format!(
|
|
||||||
"media[{index}].modality {other:?} is not supported by the gRPC service"
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
let uuid = (!item.uuid.is_empty()).then_some(item.uuid);
|
let uuid = (!item.uuid.is_empty()).then_some(item.uuid);
|
||||||
let mime_type = (!item.mime_type.is_empty()).then_some(item.mime_type);
|
let mime_type = (!item.mime_type.is_empty()).then_some(item.mime_type);
|
||||||
let part = match item.source {
|
let source = item.source.ok_or_else(|| {
|
||||||
Some(pb::media_item::Source::Url(url)) => {
|
Status::invalid_argument(format!("media[{index}].source is required"))
|
||||||
validate_media_uri(index, "url", &url, &["http", "https"])?;
|
})?;
|
||||||
MediaContentPart::ImageUrl {
|
match &source {
|
||||||
url,
|
pb::media_item::Source::Url(url) => {
|
||||||
detail: None,
|
validate_media_uri(index, "url", url, &["http", "https"])?;
|
||||||
uuid,
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
Some(pb::media_item::Source::DataUri(uri)) => {
|
pb::media_item::Source::DataUri(uri) => {
|
||||||
validate_media_uri(index, "data_uri", &uri, &["data"])?;
|
validate_media_uri(index, "data_uri", uri, &["data"])?;
|
||||||
MediaContentPart::ImageUrl {
|
|
||||||
url: uri,
|
|
||||||
detail: None,
|
|
||||||
uuid,
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
Some(pb::media_item::Source::RawBytes(bytes)) => MediaContentPart::ImageData {
|
pb::media_item::Source::RawBytes(_) => {}
|
||||||
data: bytes,
|
}
|
||||||
mime_type,
|
let part = match (modality, source) {
|
||||||
uuid,
|
(
|
||||||
|
pb::Modality::Image,
|
||||||
|
pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url),
|
||||||
|
) => MediaContentPart::ImageUrl {
|
||||||
|
url,
|
||||||
detail: None,
|
detail: None,
|
||||||
|
uuid,
|
||||||
},
|
},
|
||||||
None => {
|
(pb::Modality::Image, pb::media_item::Source::RawBytes(data)) => {
|
||||||
return Err(Status::invalid_argument(format!(
|
MediaContentPart::ImageData {
|
||||||
"media[{index}].source is required"
|
data,
|
||||||
)));
|
mime_type,
|
||||||
|
uuid,
|
||||||
|
detail: None,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
(
|
||||||
|
pb::Modality::Video,
|
||||||
|
pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url),
|
||||||
|
) => MediaContentPart::VideoUrl { url, uuid },
|
||||||
|
(pb::Modality::Video, pb::media_item::Source::RawBytes(data)) => {
|
||||||
|
MediaContentPart::VideoData {
|
||||||
|
data,
|
||||||
|
mime_type,
|
||||||
|
uuid,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
(
|
||||||
|
pb::Modality::Audio,
|
||||||
|
pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url),
|
||||||
|
) => MediaContentPart::AudioUrl { url, uuid },
|
||||||
|
(pb::Modality::Audio, pb::media_item::Source::RawBytes(data)) => {
|
||||||
|
MediaContentPart::AudioData {
|
||||||
|
data,
|
||||||
|
mime_type,
|
||||||
|
uuid,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
(pb::Modality::Unspecified, _) => unreachable!("modality validated above"),
|
||||||
};
|
};
|
||||||
parts.push(part);
|
parts.push(part);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,27 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
import torch
|
||||||
|
|
||||||
|
from vllm.model_executor.models.qwen3_asr import (
|
||||||
|
Qwen3ASRForConditionalGeneration,
|
||||||
|
)
|
||||||
|
from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange
|
||||||
|
|
||||||
|
|
||||||
|
def test_mrope_positions_with_stripped_audio_data() -> None:
|
||||||
|
input_tokens = list(range(7))
|
||||||
|
feature = MultiModalFeatureSpec(
|
||||||
|
data=None,
|
||||||
|
modality="audio",
|
||||||
|
identifier="audio",
|
||||||
|
mm_position=PlaceholderRange(offset=2, length=3),
|
||||||
|
)
|
||||||
|
|
||||||
|
positions, delta = Qwen3ASRForConditionalGeneration.get_mrope_input_positions(
|
||||||
|
None, input_tokens, [feature]
|
||||||
|
)
|
||||||
|
|
||||||
|
expected = torch.arange(len(input_tokens), dtype=torch.long).expand(3, -1)
|
||||||
|
torch.testing.assert_close(positions, expected)
|
||||||
|
assert delta == 0
|
||||||
@@ -575,13 +575,10 @@ class Qwen3ASRForConditionalGeneration(
|
|||||||
for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
|
for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
|
||||||
offset = mm_feature.mm_position.offset
|
offset = mm_feature.mm_position.offset
|
||||||
|
|
||||||
# Get audio feature length from mm_feature data
|
# The placeholder already contains one token per audio embedding.
|
||||||
audio_feature_length = mm_feature.data["audio_feature_lengths"].data
|
# Use its length so M-RoPE can be reconstructed even when a covered
|
||||||
if isinstance(audio_feature_length, torch.Tensor):
|
# prefix no longer carries the audio processor tensors.
|
||||||
audio_feature_length = audio_feature_length.item()
|
audio_len = mm_feature.mm_position.length
|
||||||
audio_len = _get_feat_extract_output_lengths(
|
|
||||||
torch.tensor(audio_feature_length)
|
|
||||||
).item()
|
|
||||||
|
|
||||||
# Text segment before audio (includes audio_start token)
|
# Text segment before audio (includes audio_start token)
|
||||||
text_len = offset - st
|
text_len = offset - st
|
||||||
|
|||||||
Reference in New Issue
Block a user