[Rust Frontend][gRPC] Add multimodal image inference (#50368)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Connor Carpenter <connorc@nvidia.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Bugen Zhao
parent
0b1c151cbc
commit
e98a8774cb
@@ -21,6 +21,7 @@ pub use event::{
|
||||
AssistantToolCall, ChatEvent,
|
||||
};
|
||||
use futures::{StreamExt, TryStreamExt as _};
|
||||
pub use llm_multimodal::MediaContentPart;
|
||||
pub use output::{
|
||||
ChatOutputProcessor, DefaultChatOutputProcessor, DynChatOutputProcessor,
|
||||
HarmonyChatOutputProcessor,
|
||||
@@ -41,6 +42,7 @@ pub use request::{
|
||||
ChatToolChoice, GenerationPromptMode, ReasoningEffort, SamplingParams,
|
||||
};
|
||||
pub use stream::{ChatEventStream, ChatEventStreamTrait, CollectedAssistantMessage};
|
||||
pub use vllm_engine_core_client::protocol::multimodal::MmFeatures;
|
||||
pub use vllm_llm::FinishReason;
|
||||
|
||||
mod backend;
|
||||
@@ -55,7 +57,6 @@ mod stream;
|
||||
|
||||
use vllm_engine_core_client::EngineCoreClient;
|
||||
use vllm_engine_core_client::protocol::dtype::ModelDtype;
|
||||
use vllm_engine_core_client::protocol::multimodal::MmFeatures;
|
||||
use vllm_engine_core_client::protocol::request::ReasoningParserKwargs;
|
||||
use vllm_llm::Llm;
|
||||
use vllm_text::{Prompt, TextLlm, TextRequest};
|
||||
@@ -136,6 +137,24 @@ impl ChatRequestProcessor {
|
||||
}
|
||||
}
|
||||
|
||||
/// Prepare media for an already-tokenized request.
|
||||
async fn prepare_media(
|
||||
&self,
|
||||
media: Vec<MediaContentPart>,
|
||||
token_ids: &mut Vec<u32>,
|
||||
) -> Result<Option<MmFeatures>> {
|
||||
if media.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
let info = self
|
||||
.backend
|
||||
.multimodal_model_info()
|
||||
.ok_or(Error::UnsupportedMultimodalRenderer)?;
|
||||
let model_dtype = self.model_dtype.ok_or(Error::UnsupportedMultimodalRenderer)?;
|
||||
let features = info.prepare_multimodal(media, token_ids, model_dtype).await?;
|
||||
Ok(Some(features))
|
||||
}
|
||||
|
||||
/// Prepare one chat request without submitting it to an engine.
|
||||
pub async fn prepare(
|
||||
&self,
|
||||
@@ -262,6 +281,15 @@ impl ChatLlm {
|
||||
self.processor.backend.multimodal_model_info().is_some()
|
||||
}
|
||||
|
||||
/// Prepare media for an already-tokenized request.
|
||||
pub async fn prepare_media(
|
||||
&self,
|
||||
media: Vec<MediaContentPart>,
|
||||
token_ids: &mut Vec<u32>,
|
||||
) -> Result<Option<MmFeatures>> {
|
||||
self.processor.prepare_media(media, token_ids).await
|
||||
}
|
||||
|
||||
/// Effective tool-call parser name for this model, if parsing is enabled.
|
||||
pub fn tool_call_parser_name(&self) -> Option<&str> {
|
||||
match &self.tool_call_parser {
|
||||
|
||||
@@ -696,7 +696,7 @@ impl MultimodalModelInfo {
|
||||
/// `prompt_token_ids` is mutated in place because placeholder expansion
|
||||
/// changes both the final prompt and the offsets recorded in
|
||||
/// `PlaceholderRange`.
|
||||
async fn prepare_multimodal(
|
||||
pub(crate) async fn prepare_multimodal(
|
||||
&self,
|
||||
media_parts: Vec<MediaContentPart>,
|
||||
prompt_token_ids: &mut Vec<u32>,
|
||||
|
||||
Reference in New Issue
Block a user