diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 0591fb2368..0a001833e9 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -2290,7 +2290,7 @@ checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" [[package]] name = "llm-multimodal" version = "1.7.1" -source = "git+https://github.com/smg-project/llm-multimodal?rev=24c676dd49d73f05cb4068aade2e60397a5c0afe#24c676dd49d73f05cb4068aade2e60397a5c0afe" +source = "git+https://github.com/smg-project/llm-multimodal?rev=53ca0d6ea829af7477b6b63851cc70bed409bbc8#53ca0d6ea829af7477b6b63851cc70bed409bbc8" dependencies = [ "anyhow", "base64 0.22.1", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index ea8b813619..afcce8d7fd 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -61,7 +61,7 @@ indexmap = "2.13.0" indicatif = "0.18.4" itertools = "0.14.0" libc = "0.2.177" -llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "24c676dd49d73f05cb4068aade2e60397a5c0afe", default-features = false, features = ["native-tls"] } +llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "53ca0d6ea829af7477b6b63851cc70bed409bbc8", default-features = false, features = ["native-tls"] } mimalloc = "0.1.52" minijinja = { version = "2.24", features = ["unstable_machinery", "json", "builtins", "loader", "loop_controls", "preserve_order"] } minijinja-contrib = { version = "2.24", features = ["pycompat"] } diff --git a/rust/src/chat/src/multimodal.rs b/rust/src/chat/src/multimodal.rs index 9b6cebeb44..4af49aa5c6 100644 --- a/rust/src/chat/src/multimodal.rs +++ b/rust/src/chat/src/multimodal.rs @@ -17,12 +17,12 @@ use std::path::Path; use std::sync::{Arc, LazyLock}; use itertools::{Either, izip}; +use llm_multimodal::registry::ModelRegistryError; use llm_multimodal::{ AsyncMultiModalTracker, AudioClip, AudioPreProcessor, EncoderFieldLayouts, FieldLayout, ImageFrame, MediaConnector, MediaConnectorConfig, MediaContentPart, Modality, ModelMetadata, ModelProcessorSpec, ModelRegistry, PreProcessorConfig, PreprocessedEncoderInputs, PromptReplacement, Tokenizer as TokenResolver, TrackedMedia, VideoClip, VisionPreProcessor, - VisionProcessorRegistry, }; use serde::{Deserialize, Serialize}; use thiserror_ext::AsReport as _; @@ -55,8 +55,8 @@ pub use self::timing::{mm_request_span, mm_timing_layer}; #[derive(Clone)] pub struct MultimodalModelInfo { context: MultimodalModelContext, - image: Option, - video: Option, + image: Option, + video: Option, audio: Option, media_connector: Arc, /// Maximum number of input items allowed per prompt for each modality. @@ -133,13 +133,18 @@ impl MultimodalModelContext { REGISTRY.lookup(&self.metadata()) } - /// Resolve a static vision preprocessor for one loaded model. - /// - /// The vision preprocessor serves both the image and video modalities. - fn resolve_vision_processor(&self) -> Option<&'static dyn VisionPreProcessor> { - static REGISTRY: LazyLock = - LazyLock::new(VisionProcessorRegistry::with_defaults); - REGISTRY.find(&self.model_id, self.model_type.as_deref()) + /// Build a vision preprocessor for one loaded model and modality. + fn resolve_vision_processor( + &self, + model_spec: &'static dyn ModelProcessorSpec, + preprocessor_config: &PreProcessorConfig, + modality: Modality, + ) -> Result>> { + match model_spec.vision_processor(&self.metadata(), preprocessor_config, modality) { + Ok(processor) => Ok(Some(Arc::from(processor))), + Err(ModelRegistryError::UnsupportedModality { .. }) => Ok(None), + Err(error) => Err(error.into()), + } } /// Resolve an audio preprocessor for one loaded model. @@ -148,7 +153,7 @@ impl MultimodalModelContext { model_spec: &'static dyn ModelProcessorSpec, preprocessor_config: &PreProcessorConfig, ) -> Option> { - model_spec.audio_processor(&self.config, preprocessor_config).map(Arc::from) + model_spec.audio_processor(&self.metadata(), preprocessor_config).map(Arc::from) } } @@ -242,17 +247,16 @@ impl ResolvedPlaceholder { } } -/// Static per-modality vision preprocessor plus its loaded config, resolved -/// placeholder tokens, and the model's shared tensor-layout spec. +/// Model-owned vision preprocessor plus resolved placeholder tokens and +/// the model's shared tensor-layout spec. #[derive(Clone)] -struct ModalitySupport { +struct VisionModalitySupport { spec: ResolvedMultimodalSpec, placeholder: ResolvedPlaceholder, - processor: &'static dyn VisionPreProcessor, - config: PreProcessorConfig, + processor: Arc, } -/// Static audio preprocessor plus the resolved model contract for its output. +/// Model-owned audio preprocessor plus the resolved model contract for its output. #[derive(Clone)] struct AudioModalitySupport { spec: ResolvedMultimodalSpec, @@ -399,7 +403,7 @@ impl MultimodalModelInfo { &context, preprocessor_config.clone(), video_preprocessor_config, - ); + )?; let audio = Self::resolve_audio_lane(&context, &preprocessor_config)?; if image.is_none() && video.is_none() && audio.is_none() { @@ -430,27 +434,19 @@ impl MultimodalModelInfo { /// /// A missing model spec or vision processor disables both lanes but is /// not an error: unsupported model families are logged and skipped. + /// Processor construction errors propagate to model loading. fn resolve_vision_lanes( context: &MultimodalModelContext, preprocessor_config: PreProcessorConfig, video_preprocessor_config: PreProcessorConfig, - ) -> (Option, Option) { + ) -> Result<(Option, Option)> { let Some(raw_spec) = context.resolve_model_spec() else { warn!( model_id = context.model_id, model_type = context.model_type, "multimodal model spec is not registered; disabling image/video support for this model" ); - return (None, None); - }; - - let Some(processor) = context.resolve_vision_processor() else { - warn!( - model_id = context.model_id, - model_type = context.model_type, - "vision processor is not registered; disabling image/video support for this model" - ); - return (None, None); + return Ok((None, None)); }; // Warn and disable the modality if the placeholder resolution fails. @@ -468,14 +464,19 @@ impl MultimodalModelInfo { } }; - let image = resolve_placeholder(Modality::Image).map(|placeholder| ModalitySupport { - spec: ResolvedMultimodalSpec::new(raw_spec, Modality::Image), - placeholder, - processor, - config: preprocessor_config.clone(), - }); + let image = if let Some(placeholder) = resolve_placeholder(Modality::Image) { + context + .resolve_vision_processor(raw_spec, &preprocessor_config, Modality::Image)? + .map(|processor| VisionModalitySupport { + spec: ResolvedMultimodalSpec::new(raw_spec, Modality::Image), + placeholder, + processor, + }) + } else { + None + }; - let video = resolve_placeholder(Modality::Video).and_then(|placeholder| { + let video = if let Some(placeholder) = resolve_placeholder(Modality::Video) { // Placeholder expansion attributes markers to modalities by token // ID, so a marker shared with the image modality is ambiguous. let image_marker = image.as_ref().map(|image| image.placeholder.marker_token_id); @@ -487,16 +488,23 @@ impl MultimodalModelInfo { ); None } else { - Some(ModalitySupport { - spec: ResolvedMultimodalSpec::new(raw_spec, Modality::Video), - placeholder, - processor, - config: video_preprocessor_config, - }) + context + .resolve_vision_processor( + raw_spec, + &video_preprocessor_config, + Modality::Video, + )? + .map(|processor| VisionModalitySupport { + spec: ResolvedMultimodalSpec::new(raw_spec, Modality::Video), + placeholder, + processor, + }) } - }); + } else { + None + }; - (image, video) + Ok((image, video)) } /// Resolve a spec-backed audio lane when the model advertises audio support. diff --git a/rust/src/chat/src/multimodal/image.rs b/rust/src/chat/src/multimodal/image.rs index 3627d6c30e..3062dae3e6 100644 --- a/rust/src/chat/src/multimodal/image.rs +++ b/rust/src/chat/src/multimodal/image.rs @@ -10,7 +10,7 @@ use llm_multimodal::{ImageFrame, Modality, PreprocessedEncoderInputs}; use vllm_engine_core_client::protocol::dtype::ModelDtype; use super::timing::MM_STAGE_TARGET; -use super::{ModalitySupport, MultimodalModelInfo, PreparedMedia, item}; +use super::{MultimodalModelInfo, PreparedMedia, VisionModalitySupport, item}; use crate::error::{Error, Result, bail_multimodal, multimodal}; /// Forward-kwargs name of the primary image encoder input. @@ -63,15 +63,14 @@ impl MultimodalModelInfo { /// conversion. async fn preprocess_images( &self, - support: &ModalitySupport, + support: &VisionModalitySupport, image_frames: &[Arc], ) -> Result { - let config = support.config.clone(); - let processor = support.processor; + let processor = Arc::clone(&support.processor); let images = image_frames.iter().map(|frame| frame.data().clone()).collect::>(); // TODO: is it still necessary given that we've already in a dedicated runtime? - tokio::task::spawn_blocking(move || Ok(processor.preprocess(&images, &config)?)) + tokio::task::spawn_blocking(move || Ok(processor.preprocess(&images)?)) .await .map_err(|error| multimodal!("image preprocessing task failed: {error}"))? } diff --git a/rust/src/chat/src/multimodal/video.rs b/rust/src/chat/src/multimodal/video.rs index 67273970ad..a6b7856a62 100644 --- a/rust/src/chat/src/multimodal/video.rs +++ b/rust/src/chat/src/multimodal/video.rs @@ -17,7 +17,7 @@ use vllm_engine_core_client::protocol::multimodal::{ }; use super::timing::MM_STAGE_TARGET; -use super::{ModalitySupport, MultimodalModelInfo, PreparedItem, PreparedMedia, tensor}; +use super::{MultimodalModelInfo, PreparedItem, PreparedMedia, VisionModalitySupport, tensor}; use crate::error::{Error, Result, bail_multimodal, multimodal}; /// Forward-kwargs name of the primary video encoder input. @@ -83,18 +83,17 @@ impl MultimodalModelInfo { /// processor. async fn preprocess_video_clip( &self, - support: &ModalitySupport, + support: &VisionModalitySupport, clip: Arc, ) -> Result { - let config = support.config.clone(); - let processor = support.processor; + let processor = Arc::clone(&support.processor); tokio::task::spawn_blocking(move || { // Prefer the borrowed-RGB fast path, which avoids materializing a // `DynamicImage` per sampled frame after media decode. if let Some(rgb_video) = clip.rgb_video() { match rgb_video.frame_refs() { - Ok(frame_refs) => match processor.preprocess_video_rgb(&frame_refs, &config) { + Ok(frame_refs) => match processor.preprocess_video_rgb(&frame_refs) { Ok(preprocessed) => return Ok(preprocessed), Err(error) => warn!( error = %error.as_report(), @@ -109,7 +108,7 @@ impl MultimodalModelInfo { } let frames = clip.materialized_frames().map_err(|error| multimodal!("{error}"))?; - Ok(processor.preprocess_video(&frames, &config)?) + Ok(processor.preprocess_video(&frames)?) }) .await .map_err(|error| multimodal!("video preprocessing task failed: {error}"))? @@ -124,7 +123,7 @@ impl MultimodalModelInfo { /// `flat_from_sizes` treatment of video patches), and batched metadata /// tensors drop their singleton batch axis. fn build_video_item( - support: &ModalitySupport, + support: &VisionModalitySupport, preprocessed: PreprocessedEncoderInputs, hash: String, uuid: Option,