[Rust Frontend][gRPC] Add audio and video media inputs (#53760)

Signed-off-by: Connor Carpenter <connorc@nvidia.com>
This commit is contained in:
Connor Carpenter
2026-08-30 11:31:37 +08:00
committed by GitHub
parent b016ed8ea3
commit 7a67941c1d
4 changed files with 124 additions and 50 deletions
+41 -8
View File
@@ -86,20 +86,30 @@ pub(super) fn expand_prompt_token_ids(
}
let replacement_len = replacement.tokens.len();
let structural_prefix = replacement.structural_prefix;
if structural_prefix > expanded.len() {
bail_multimodal!(
"placeholder token `{}` declares a structural prefix of {} token(s), but only {} token(s) precede it",
lane.placeholder_token,
structural_prefix,
expanded.len()
);
}
let range_len = replacement_len.saturating_add(structural_prefix);
let is_embed = {
let mask = replacement
.tokens
.iter()
.map(|&token| token as u32 == lane.embed_token_id)
.collect::<Vec<_>>();
WireTensor::from_bool(vec![replacement_len], mask).map_err(Error::Multimodal)?
let mut mask = Vec::with_capacity(range_len);
mask.extend(std::iter::repeat_n(false, structural_prefix));
mask.extend(
replacement.tokens.iter().map(|&token| token as u32 == lane.embed_token_id),
);
WireTensor::from_bool(vec![range_len], mask).map_err(Error::Multimodal)?
};
let expanded_offset = expanded.len();
let expanded_offset = expanded.len() - structural_prefix;
expanded.extend(replacement.tokens.iter().map(|&token| token as u32));
ranges.entry(lane.modality).or_default().push(PlaceholderRange {
offset: expanded_offset,
length: replacement_len,
length: range_len,
is_embed: Some(is_embed),
});
}
@@ -420,6 +430,29 @@ mod tests {
assert_bool_mask(&video_ranges[0], &[true, true, true, true]);
}
#[test]
fn expand_prompt_tokens_includes_structural_prefix_in_video_range() {
const VISION_START_ID: u32 = 151652;
const VISION_END_ID: u32 = 151653;
let mut prompt_token_ids = vec![1, VISION_START_ID, QWEN3_VIDEO_PAD_ID, VISION_END_ID, 2];
let replacement = PromptReplacement::repeated(
Modality::Video,
"<|video_pad|>",
QWEN3_VIDEO_PAD_ID as TokenId,
3,
)
.with_structural_prefix(1);
let prepared = vec![qwen3_video_prepared(vec![replacement])];
let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap();
let range = &ranges[&Modality::Video][0];
assert_eq!(range.offset, 1);
assert_eq!(range.length, 4);
assert_bool_mask(range, &[false, true, true, true]);
}
const QWEN3_AUDIO_PAD_ID: u32 = 151_676;
fn qwen3_audio_prepared(replacements: Vec<PromptReplacement>) -> PreparedMedia {