From 099024762cc7622890f4887d27e5f547a8e73225 Mon Sep 17 00:00:00 2001 From: Chao-Ju Chen Date: Fri, 29 May 2026 00:46:18 +0800 Subject: [PATCH] [Rust Frontend] Optimize multimodal prompt expansion (#43670) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: RickyChen / 陳昭儒 --- rust/src/chat/src/multimodal.rs | 166 +++++++++++++++++++++++--------- 1 file changed, 119 insertions(+), 47 deletions(-) diff --git a/rust/src/chat/src/multimodal.rs b/rust/src/chat/src/multimodal.rs index ad73040560..8526d4fa0e 100644 --- a/rust/src/chat/src/multimodal.rs +++ b/rust/src/chat/src/multimodal.rs @@ -392,53 +392,13 @@ impl MultimodalModelInfo { prompt_token_ids: &mut Vec, replacements: Vec, ) -> Result> { - let mut cursor = 0; - let mut ranges = Vec::with_capacity(replacements.len()); - for replacement in replacements { - if replacement.modality != Modality::Image { - bail_multimodal!( - "unsupported prompt replacement modality `{}`", - replacement.modality - ); - } - let offset = find_next_token( - prompt_token_ids, - self.spec.placeholder_marker_token_id, - cursor, - ) - .ok_or_else(|| { - multimodal!( - "placeholder token `{}` was not found in tokenized prompt", - self.spec.placeholder_token - ) - })?; - - if replacement.tokens.is_empty() { - bail_multimodal!( - "placeholder token `{}` expanded to no tokens", - self.spec.placeholder_token - ); - } - let replacement_len = replacement.tokens.len(); - let replacement_tokens = - replacement.tokens.iter().map(|&token| token as u32).collect::>(); - let is_embed = { - let mask = replacement_tokens - .iter() - .map(|&token| token == self.spec.placeholder_embed_token_id) - .collect::>(); - WireTensor::from_bool(vec![replacement_len], mask).map_err(Error::Multimodal)? - }; - - prompt_token_ids.splice(offset..offset + 1, replacement_tokens); - ranges.push(PlaceholderRange { - offset, - length: replacement_len, - is_embed: Some(is_embed), - }); - cursor = offset + replacement_len; - } - Ok(ranges) + expand_prompt_token_ids( + prompt_token_ids, + replacements, + self.spec.placeholder_marker_token_id, + self.spec.placeholder_embed_token_id, + &self.spec.placeholder_token, + ) } /// Convert preprocessed image tensors into engine-core multimodal features. @@ -516,6 +476,71 @@ impl MultimodalModelInfo { } } +fn expand_prompt_token_ids( + prompt_token_ids: &mut Vec, + replacements: Vec, + placeholder_marker_token_id: u32, + placeholder_embed_token_id: u32, + placeholder_token: &str, +) -> Result> { + if replacements.is_empty() { + return Ok(Vec::new()); + } + + let replacement_growth = replacements.iter().fold(0usize, |total, replacement| { + total.saturating_add(replacement.tokens.len().saturating_sub(1)) + }); + let mut expanded = + Vec::with_capacity(prompt_token_ids.len().saturating_add(replacement_growth)); + let mut ranges = Vec::with_capacity(replacements.len()); + let mut cursor = 0usize; + + for replacement in replacements { + if replacement.modality != Modality::Image { + bail_multimodal!( + "unsupported prompt replacement modality `{}`", + replacement.modality + ); + } + + let offset = find_next_token(prompt_token_ids, placeholder_marker_token_id, cursor) + .ok_or_else(|| { + multimodal!( + "placeholder token `{placeholder_token}` was not found in tokenized prompt" + ) + })?; + + if replacement.tokens.is_empty() { + bail_multimodal!("placeholder token `{placeholder_token}` expanded to no tokens"); + } + + let replacement_len = replacement.tokens.len(); + let is_embed = { + let mask = replacement + .tokens + .iter() + .map(|&token| token as u32 == placeholder_embed_token_id) + .collect::>(); + WireTensor::from_bool(vec![replacement_len], mask).map_err(Error::Multimodal)? + }; + + expanded.extend_from_slice(&prompt_token_ids[cursor..offset]); + let expanded_offset = expanded.len(); + expanded.extend(replacement.tokens.into_iter().map(|token| token as u32)); + ranges.push(PlaceholderRange { + offset: expanded_offset, + length: replacement_len, + is_embed: Some(is_embed), + }); + cursor = offset + 1; + } + + expanded.extend_from_slice(&prompt_token_ids[cursor..]); + *prompt_token_ids = expanded; + + Ok(ranges) +} + /// Find `needle` in `haystack`, starting at `start`. /// /// This is intentionally order-preserving rather than a global replace: each @@ -736,6 +761,53 @@ mod tests { assert!(matches!(error, Error::Multimodal(message) if message.contains("not found"))); } + #[test] + fn expand_prompt_tokens_ignores_empty_replacements() { + let info = llama4_info(); + let mut prompt_token_ids = vec![1, LLAMA4_IMAGE_ID, 2]; + let original_prompt_token_ids = prompt_token_ids.clone(); + + let ranges = info.expand_prompt_tokens(&mut prompt_token_ids, Vec::new()).unwrap(); + + assert!(ranges.is_empty()); + assert_eq!(prompt_token_ids, original_prompt_token_ids); + } + + #[test] + fn expand_prompt_tokens_leaves_prompt_unchanged_when_later_placeholder_missing() { + let info = llama4_info(); + let mut prompt_token_ids = vec![1, LLAMA4_IMAGE_ID, 2]; + let original_prompt_token_ids = prompt_token_ids.clone(); + let replacements = vec![ + llama4_single_tile_replacement(), + llama4_single_tile_replacement(), + ]; + + let error = info.expand_prompt_tokens(&mut prompt_token_ids, replacements).unwrap_err(); + + assert!(matches!(error, Error::Multimodal(message) if message.contains("not found"))); + assert_eq!(prompt_token_ids, original_prompt_token_ids); + } + + #[test] + fn expand_prompt_tokens_errors_when_replacement_is_empty() { + let info = llama4_info(); + let mut prompt_token_ids = vec![1, LLAMA4_IMAGE_ID, 2]; + let original_prompt_token_ids = prompt_token_ids.clone(); + let replacements = vec![PromptReplacement::sequence( + Modality::Image, + "<|image|>", + Vec::new(), + )]; + + let error = info.expand_prompt_tokens(&mut prompt_token_ids, replacements).unwrap_err(); + + assert!( + matches!(error, Error::Multimodal(message) if message.contains("expanded to no tokens")) + ); + assert_eq!(prompt_token_ids, original_prompt_token_ids); + } + #[test] fn expand_prompt_tokens_skips_llama4_image_marker_inside_replacement() { let info = llama4_info();