[Rust Frontend][gRPC] Add audio and video media inputs (#53760)
Signed-off-by: Connor Carpenter <connorc@nvidia.com>
This commit is contained in:
@@ -86,20 +86,30 @@ pub(super) fn expand_prompt_token_ids(
|
||||
}
|
||||
|
||||
let replacement_len = replacement.tokens.len();
|
||||
let structural_prefix = replacement.structural_prefix;
|
||||
if structural_prefix > expanded.len() {
|
||||
bail_multimodal!(
|
||||
"placeholder token `{}` declares a structural prefix of {} token(s), but only {} token(s) precede it",
|
||||
lane.placeholder_token,
|
||||
structural_prefix,
|
||||
expanded.len()
|
||||
);
|
||||
}
|
||||
let range_len = replacement_len.saturating_add(structural_prefix);
|
||||
let is_embed = {
|
||||
let mask = replacement
|
||||
.tokens
|
||||
.iter()
|
||||
.map(|&token| token as u32 == lane.embed_token_id)
|
||||
.collect::<Vec<_>>();
|
||||
WireTensor::from_bool(vec![replacement_len], mask).map_err(Error::Multimodal)?
|
||||
let mut mask = Vec::with_capacity(range_len);
|
||||
mask.extend(std::iter::repeat_n(false, structural_prefix));
|
||||
mask.extend(
|
||||
replacement.tokens.iter().map(|&token| token as u32 == lane.embed_token_id),
|
||||
);
|
||||
WireTensor::from_bool(vec![range_len], mask).map_err(Error::Multimodal)?
|
||||
};
|
||||
|
||||
let expanded_offset = expanded.len();
|
||||
let expanded_offset = expanded.len() - structural_prefix;
|
||||
expanded.extend(replacement.tokens.iter().map(|&token| token as u32));
|
||||
ranges.entry(lane.modality).or_default().push(PlaceholderRange {
|
||||
offset: expanded_offset,
|
||||
length: replacement_len,
|
||||
length: range_len,
|
||||
is_embed: Some(is_embed),
|
||||
});
|
||||
}
|
||||
@@ -420,6 +430,29 @@ mod tests {
|
||||
assert_bool_mask(&video_ranges[0], &[true, true, true, true]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expand_prompt_tokens_includes_structural_prefix_in_video_range() {
|
||||
const VISION_START_ID: u32 = 151652;
|
||||
const VISION_END_ID: u32 = 151653;
|
||||
|
||||
let mut prompt_token_ids = vec![1, VISION_START_ID, QWEN3_VIDEO_PAD_ID, VISION_END_ID, 2];
|
||||
let replacement = PromptReplacement::repeated(
|
||||
Modality::Video,
|
||||
"<|video_pad|>",
|
||||
QWEN3_VIDEO_PAD_ID as TokenId,
|
||||
3,
|
||||
)
|
||||
.with_structural_prefix(1);
|
||||
let prepared = vec![qwen3_video_prepared(vec![replacement])];
|
||||
|
||||
let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap();
|
||||
|
||||
let range = &ranges[&Modality::Video][0];
|
||||
assert_eq!(range.offset, 1);
|
||||
assert_eq!(range.length, 4);
|
||||
assert_bool_mask(range, &[false, true, true, true]);
|
||||
}
|
||||
|
||||
const QWEN3_AUDIO_PAD_ID: u32 = 151_676;
|
||||
|
||||
fn qwen3_audio_prepared(replacements: Vec<PromptReplacement>) -> PreparedMedia {
|
||||
|
||||
Reference in New Issue
Block a user