[Rust Frontend][Multimodal] Accept preprocessed multimodal gRPC features (#55047)

Signed-off-by: Biswa Panda <biswa.panda@gmail.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Biswa Panda
2026-09-12 02:29:49 +00:00
committed by GitHub
co-authored by Bugen Zhao Codex
parent a0844fa6c6
commit 31759ccb6d
16 changed files with 986 additions and 113 deletions
+14 -5
View File
@@ -130,7 +130,7 @@ impl ChatRequestProcessor {
/// Prepare media for an already-tokenized request.
async fn prepare_media(
&self,
media: Vec<MediaContentPart>,
media: multimodal::MultimodalInput,
token_ids: &mut Vec<u32>,
) -> Result<Option<MmFeatures>> {
if media.is_empty() {
@@ -140,8 +140,15 @@ impl ChatRequestProcessor {
.backend
.multimodal_model_info()
.ok_or(Error::UnsupportedMultimodalRenderer)?;
let model_dtype = self.model_dtype.ok_or(Error::UnsupportedMultimodalRenderer)?;
let features = info.prepare_multimodal(media, token_ids, model_dtype).await?;
let features = match media {
multimodal::MultimodalInput::Raw(parts) => {
let model_dtype = self.model_dtype.ok_or(Error::UnsupportedMultimodalRenderer)?;
info.prepare_multimodal(parts, token_ids, model_dtype).await?
}
multimodal::MultimodalInput::Preprocessed(features) => {
info.prepare_preprocessed(features, token_ids.len())?
}
};
Ok(Some(features))
}
@@ -286,12 +293,14 @@ impl ChatLlm {
}
/// Prepare media for an already-tokenized request.
/// Raw content is preprocessed; inline features are checked against model
/// capabilities, modality limits, and the final prompt positions.
pub async fn prepare_media(
&self,
media: Vec<MediaContentPart>,
media: impl Into<multimodal::MultimodalInput>,
token_ids: &mut Vec<u32>,
) -> Result<Option<MmFeatures>> {
self.processor.prepare_media(media, token_ids).await
self.processor.prepare_media(media.into(), token_ids).await
}
/// Effective tool-call parser name for this model, if parsing is enabled.