[Rust Frontend] Optimize multimodal prompt expansion (#43670)

Signed-off-by: RickyChen / 陳昭儒 <ricky.chen@infinirc.com>
This commit is contained in:
Chao-Ju Chen
2026-05-28 09:46:18 -07:00
committed by GitHub
parent 9aa131f944
commit 099024762c
+119 -47
View File
@@ -392,53 +392,13 @@ impl MultimodalModelInfo {
prompt_token_ids: &mut Vec<u32>,
replacements: Vec<PromptReplacement>,
) -> Result<Vec<PlaceholderRange>> {
let mut cursor = 0;
let mut ranges = Vec::with_capacity(replacements.len());
for replacement in replacements {
if replacement.modality != Modality::Image {
bail_multimodal!(
"unsupported prompt replacement modality `{}`",
replacement.modality
);
}
let offset = find_next_token(
prompt_token_ids,
self.spec.placeholder_marker_token_id,
cursor,
)
.ok_or_else(|| {
multimodal!(
"placeholder token `{}` was not found in tokenized prompt",
self.spec.placeholder_token
)
})?;
if replacement.tokens.is_empty() {
bail_multimodal!(
"placeholder token `{}` expanded to no tokens",
self.spec.placeholder_token
);
}
let replacement_len = replacement.tokens.len();
let replacement_tokens =
replacement.tokens.iter().map(|&token| token as u32).collect::<Vec<_>>();
let is_embed = {
let mask = replacement_tokens
.iter()
.map(|&token| token == self.spec.placeholder_embed_token_id)
.collect::<Vec<_>>();
WireTensor::from_bool(vec![replacement_len], mask).map_err(Error::Multimodal)?
};
prompt_token_ids.splice(offset..offset + 1, replacement_tokens);
ranges.push(PlaceholderRange {
offset,
length: replacement_len,
is_embed: Some(is_embed),
});
cursor = offset + replacement_len;
}
Ok(ranges)
expand_prompt_token_ids(
prompt_token_ids,
replacements,
self.spec.placeholder_marker_token_id,
self.spec.placeholder_embed_token_id,
&self.spec.placeholder_token,
)
}
/// Convert preprocessed image tensors into engine-core multimodal features.
@@ -516,6 +476,71 @@ impl MultimodalModelInfo {
}
}
fn expand_prompt_token_ids(
prompt_token_ids: &mut Vec<u32>,
replacements: Vec<PromptReplacement>,
placeholder_marker_token_id: u32,
placeholder_embed_token_id: u32,
placeholder_token: &str,
) -> Result<Vec<PlaceholderRange>> {
if replacements.is_empty() {
return Ok(Vec::new());
}
let replacement_growth = replacements.iter().fold(0usize, |total, replacement| {
total.saturating_add(replacement.tokens.len().saturating_sub(1))
});
let mut expanded =
Vec::with_capacity(prompt_token_ids.len().saturating_add(replacement_growth));
let mut ranges = Vec::with_capacity(replacements.len());
let mut cursor = 0usize;
for replacement in replacements {
if replacement.modality != Modality::Image {
bail_multimodal!(
"unsupported prompt replacement modality `{}`",
replacement.modality
);
}
let offset = find_next_token(prompt_token_ids, placeholder_marker_token_id, cursor)
.ok_or_else(|| {
multimodal!(
"placeholder token `{placeholder_token}` was not found in tokenized prompt"
)
})?;
if replacement.tokens.is_empty() {
bail_multimodal!("placeholder token `{placeholder_token}` expanded to no tokens");
}
let replacement_len = replacement.tokens.len();
let is_embed = {
let mask = replacement
.tokens
.iter()
.map(|&token| token as u32 == placeholder_embed_token_id)
.collect::<Vec<_>>();
WireTensor::from_bool(vec![replacement_len], mask).map_err(Error::Multimodal)?
};
expanded.extend_from_slice(&prompt_token_ids[cursor..offset]);
let expanded_offset = expanded.len();
expanded.extend(replacement.tokens.into_iter().map(|token| token as u32));
ranges.push(PlaceholderRange {
offset: expanded_offset,
length: replacement_len,
is_embed: Some(is_embed),
});
cursor = offset + 1;
}
expanded.extend_from_slice(&prompt_token_ids[cursor..]);
*prompt_token_ids = expanded;
Ok(ranges)
}
/// Find `needle` in `haystack`, starting at `start`.
///
/// This is intentionally order-preserving rather than a global replace: each
@@ -736,6 +761,53 @@ mod tests {
assert!(matches!(error, Error::Multimodal(message) if message.contains("not found")));
}
#[test]
fn expand_prompt_tokens_ignores_empty_replacements() {
let info = llama4_info();
let mut prompt_token_ids = vec![1, LLAMA4_IMAGE_ID, 2];
let original_prompt_token_ids = prompt_token_ids.clone();
let ranges = info.expand_prompt_tokens(&mut prompt_token_ids, Vec::new()).unwrap();
assert!(ranges.is_empty());
assert_eq!(prompt_token_ids, original_prompt_token_ids);
}
#[test]
fn expand_prompt_tokens_leaves_prompt_unchanged_when_later_placeholder_missing() {
let info = llama4_info();
let mut prompt_token_ids = vec![1, LLAMA4_IMAGE_ID, 2];
let original_prompt_token_ids = prompt_token_ids.clone();
let replacements = vec![
llama4_single_tile_replacement(),
llama4_single_tile_replacement(),
];
let error = info.expand_prompt_tokens(&mut prompt_token_ids, replacements).unwrap_err();
assert!(matches!(error, Error::Multimodal(message) if message.contains("not found")));
assert_eq!(prompt_token_ids, original_prompt_token_ids);
}
#[test]
fn expand_prompt_tokens_errors_when_replacement_is_empty() {
let info = llama4_info();
let mut prompt_token_ids = vec![1, LLAMA4_IMAGE_ID, 2];
let original_prompt_token_ids = prompt_token_ids.clone();
let replacements = vec![PromptReplacement::sequence(
Modality::Image,
"<|image|>",
Vec::new(),
)];
let error = info.expand_prompt_tokens(&mut prompt_token_ids, replacements).unwrap_err();
assert!(
matches!(error, Error::Multimodal(message) if message.contains("expanded to no tokens"))
);
assert_eq!(prompt_token_ids, original_prompt_token_ids);
}
#[test]
fn expand_prompt_tokens_skips_llama4_image_marker_inside_replacement() {
let info = llama4_info();