[Rust Frontend][gRPC] Add multimodal image inference (#50368)

Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Connor Carpenter <connorc@nvidia.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Connor Carpenter
2026-08-04 09:34:31 +00:00
committed by GitHub
co-authored by Bugen Zhao
parent 0b1c151cbc
commit e98a8774cb
12 changed files with 553 additions and 69 deletions
+29 -1
View File
@@ -21,6 +21,7 @@ pub use event::{
AssistantToolCall, ChatEvent,
};
use futures::{StreamExt, TryStreamExt as _};
pub use llm_multimodal::MediaContentPart;
pub use output::{
ChatOutputProcessor, DefaultChatOutputProcessor, DynChatOutputProcessor,
HarmonyChatOutputProcessor,
@@ -41,6 +42,7 @@ pub use request::{
ChatToolChoice, GenerationPromptMode, ReasoningEffort, SamplingParams,
};
pub use stream::{ChatEventStream, ChatEventStreamTrait, CollectedAssistantMessage};
pub use vllm_engine_core_client::protocol::multimodal::MmFeatures;
pub use vllm_llm::FinishReason;
mod backend;
@@ -55,7 +57,6 @@ mod stream;
use vllm_engine_core_client::EngineCoreClient;
use vllm_engine_core_client::protocol::dtype::ModelDtype;
use vllm_engine_core_client::protocol::multimodal::MmFeatures;
use vllm_engine_core_client::protocol::request::ReasoningParserKwargs;
use vllm_llm::Llm;
use vllm_text::{Prompt, TextLlm, TextRequest};
@@ -136,6 +137,24 @@ impl ChatRequestProcessor {
}
}
/// Prepare media for an already-tokenized request.
async fn prepare_media(
&self,
media: Vec<MediaContentPart>,
token_ids: &mut Vec<u32>,
) -> Result<Option<MmFeatures>> {
if media.is_empty() {
return Ok(None);
}
let info = self
.backend
.multimodal_model_info()
.ok_or(Error::UnsupportedMultimodalRenderer)?;
let model_dtype = self.model_dtype.ok_or(Error::UnsupportedMultimodalRenderer)?;
let features = info.prepare_multimodal(media, token_ids, model_dtype).await?;
Ok(Some(features))
}
/// Prepare one chat request without submitting it to an engine.
pub async fn prepare(
&self,
@@ -262,6 +281,15 @@ impl ChatLlm {
self.processor.backend.multimodal_model_info().is_some()
}
/// Prepare media for an already-tokenized request.
pub async fn prepare_media(
&self,
media: Vec<MediaContentPart>,
token_ids: &mut Vec<u32>,
) -> Result<Option<MmFeatures>> {
self.processor.prepare_media(media, token_ids).await
}
/// Effective tool-call parser name for this model, if parsing is enabled.
pub fn tool_call_parser_name(&self) -> Option<&str> {
match &self.tool_call_parser {
+1 -1
View File
@@ -696,7 +696,7 @@ impl MultimodalModelInfo {
/// `prompt_token_ids` is mutated in place because placeholder expansion
/// changes both the final prompt and the offsets recorded in
/// `PlaceholderRange`.
async fn prepare_multimodal(
pub(crate) async fn prepare_multimodal(
&self,
media_parts: Vec<MediaContentPart>,
prompt_token_ids: &mut Vec<u32>,