iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Rust Frontend] Construct model-owned vision processors through specs (#58109)

Browse Source 
Co-authored-by: Codex <noreply@openai.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-09-22 10:01:53 +00:00
committed by GitHub
co-authored by Codex
parent 6dc34b6334
commit 81d7293c21
5 changed files with 64 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust/Cargo.lock
Generated
+1 -1
View File
@@ -2290,7 +2290,7 @@ checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092"
[[package]]
name = "llm-multimodal"
version = "1.7.1"
source = "git+https://github.com/smg-project/llm-multimodal?rev=24c676dd49d73f05cb4068aade2e60397a5c0afe#24c676dd49d73f05cb4068aade2e60397a5c0afe"
source = "git+https://github.com/smg-project/llm-multimodal?rev=53ca0d6ea829af7477b6b63851cc70bed409bbc8#53ca0d6ea829af7477b6b63851cc70bed409bbc8"
dependencies = [
"anyhow",
"base64 0.22.1",
rust/Cargo.toml
+1 -1
View File
@@ -61,7 +61,7 @@ indexmap = "2.13.0"
indicatif = "0.18.4"
itertools = "0.14.0"
libc = "0.2.177"
llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "24c676dd49d73f05cb4068aade2e60397a5c0afe", default-features = false, features = ["native-tls"] }
llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "53ca0d6ea829af7477b6b63851cc70bed409bbc8", default-features = false, features = ["native-tls"] }
mimalloc = "0.1.52"
minijinja = { version = "2.24", features = ["unstable_machinery", "json", "builtins", "loader", "loop_controls", "preserve_order"] }
minijinja-contrib = { version = "2.24", features = ["pycompat"] }
rust/src/chat/src/multimodal.rs
+52 -44
View File
@@ -17,12 +17,12 @@ use std::path::Path;
use std::sync::{Arc, LazyLock};
use itertools::{Either, izip};
use llm_multimodal::registry::ModelRegistryError;
use llm_multimodal::{
AsyncMultiModalTracker, AudioClip, AudioPreProcessor, EncoderFieldLayouts, FieldLayout,
ImageFrame, MediaConnector, MediaConnectorConfig, MediaContentPart, Modality, ModelMetadata,
ModelProcessorSpec, ModelRegistry, PreProcessorConfig, PreprocessedEncoderInputs,
PromptReplacement, Tokenizer as TokenResolver, TrackedMedia, VideoClip, VisionPreProcessor,
VisionProcessorRegistry,
};
use serde::{Deserialize, Serialize};
use thiserror_ext::AsReport as _;
@@ -55,8 +55,8 @@ pub use self::timing::{mm_request_span, mm_timing_layer};
#[derive(Clone)]
pub struct MultimodalModelInfo {
context: MultimodalModelContext,
image: Option<ModalitySupport>,
video: Option<ModalitySupport>,
image: Option<VisionModalitySupport>,
video: Option<VisionModalitySupport>,
audio: Option<AudioModalitySupport>,
media_connector: Arc<MediaConnector>,
/// Maximum number of input items allowed per prompt for each modality.
@@ -133,13 +133,18 @@ impl MultimodalModelContext {
REGISTRY.lookup(&self.metadata())
}
/// Resolve a static vision preprocessor for one loaded model.
///
/// The vision preprocessor serves both the image and video modalities.
fn resolve_vision_processor(&self) -> Option<&'static dyn VisionPreProcessor> {
static REGISTRY: LazyLock<VisionProcessorRegistry> =
LazyLock::new(VisionProcessorRegistry::with_defaults);
REGISTRY.find(&self.model_id, self.model_type.as_deref())
/// Build a vision preprocessor for one loaded model and modality.
fn resolve_vision_processor(
&self,
model_spec: &'static dyn ModelProcessorSpec,
preprocessor_config: &PreProcessorConfig,
modality: Modality,
) -> Result<Option<Arc<dyn VisionPreProcessor>>> {
match model_spec.vision_processor(&self.metadata(), preprocessor_config, modality) {
Ok(processor) => Ok(Some(Arc::from(processor))),
Err(ModelRegistryError::UnsupportedModality { .. }) => Ok(None),
Err(error) => Err(error.into()),
}
}
/// Resolve an audio preprocessor for one loaded model.
@@ -148,7 +153,7 @@ impl MultimodalModelContext {
model_spec: &'static dyn ModelProcessorSpec,
preprocessor_config: &PreProcessorConfig,
) -> Option<Arc<dyn AudioPreProcessor>> {
model_spec.audio_processor(&self.config, preprocessor_config).map(Arc::from)
model_spec.audio_processor(&self.metadata(), preprocessor_config).map(Arc::from)
}
}
@@ -242,17 +247,16 @@ impl ResolvedPlaceholder {
}
}
/// Static per-modality vision preprocessor plus its loaded config, resolved
/// placeholder tokens, and the model's shared tensor-layout spec.
/// Model-owned vision preprocessor plus resolved placeholder tokens and
/// the model's shared tensor-layout spec.
#[derive(Clone)]
struct ModalitySupport {
struct VisionModalitySupport {
spec: ResolvedMultimodalSpec,
placeholder: ResolvedPlaceholder,
processor: &'static dyn VisionPreProcessor,
config: PreProcessorConfig,
processor: Arc<dyn VisionPreProcessor>,
}
/// Static audio preprocessor plus the resolved model contract for its output.
/// Model-owned audio preprocessor plus the resolved model contract for its output.
#[derive(Clone)]
struct AudioModalitySupport {
spec: ResolvedMultimodalSpec,
@@ -399,7 +403,7 @@ impl MultimodalModelInfo {
&context,
preprocessor_config.clone(),
video_preprocessor_config,
);
)?;
let audio = Self::resolve_audio_lane(&context, &preprocessor_config)?;
if image.is_none() && video.is_none() && audio.is_none() {
@@ -430,27 +434,19 @@ impl MultimodalModelInfo {
///
/// A missing model spec or vision processor disables both lanes but is
/// not an error: unsupported model families are logged and skipped.
/// Processor construction errors propagate to model loading.
fn resolve_vision_lanes(
context: &MultimodalModelContext,
preprocessor_config: PreProcessorConfig,
video_preprocessor_config: PreProcessorConfig,
) -> (Option<ModalitySupport>, Option<ModalitySupport>) {
) -> Result<(Option<VisionModalitySupport>, Option<VisionModalitySupport>)> {
let Some(raw_spec) = context.resolve_model_spec() else {
warn!(
model_id = context.model_id,
model_type = context.model_type,
"multimodal model spec is not registered; disabling image/video support for this model"
);
return (None, None);
};
let Some(processor) = context.resolve_vision_processor() else {
warn!(
model_id = context.model_id,
model_type = context.model_type,
"vision processor is not registered; disabling image/video support for this model"
);
return (None, None);
return Ok((None, None));
};
// Warn and disable the modality if the placeholder resolution fails.
@@ -468,14 +464,19 @@ impl MultimodalModelInfo {
}
};
let image = resolve_placeholder(Modality::Image).map(|placeholder| ModalitySupport {
spec: ResolvedMultimodalSpec::new(raw_spec, Modality::Image),
placeholder,
processor,
config: preprocessor_config.clone(),
});
let image = if let Some(placeholder) = resolve_placeholder(Modality::Image) {
context
.resolve_vision_processor(raw_spec, &preprocessor_config, Modality::Image)?
.map(|processor| VisionModalitySupport {
spec: ResolvedMultimodalSpec::new(raw_spec, Modality::Image),
placeholder,
processor,
})
} else {
None
};
let video = resolve_placeholder(Modality::Video).and_then(|placeholder| {
let video = if let Some(placeholder) = resolve_placeholder(Modality::Video) {
// Placeholder expansion attributes markers to modalities by token
// ID, so a marker shared with the image modality is ambiguous.
let image_marker = image.as_ref().map(|image| image.placeholder.marker_token_id);
@@ -487,16 +488,23 @@ impl MultimodalModelInfo {
);
None
} else {
Some(ModalitySupport {
spec: ResolvedMultimodalSpec::new(raw_spec, Modality::Video),
placeholder,
processor,
config: video_preprocessor_config,
})
context
.resolve_vision_processor(
raw_spec,
&video_preprocessor_config,
Modality::Video,
)?
.map(|processor| VisionModalitySupport {
spec: ResolvedMultimodalSpec::new(raw_spec, Modality::Video),
placeholder,
processor,
})
}
});
} else {
None
};
(image, video)
Ok((image, video))
}
/// Resolve a spec-backed audio lane when the model advertises audio support.
rust/src/chat/src/multimodal/image.rs
+4 -5
View File
@@ -10,7 +10,7 @@ use llm_multimodal::{ImageFrame, Modality, PreprocessedEncoderInputs};
use vllm_engine_core_client::protocol::dtype::ModelDtype;
use super::timing::MM_STAGE_TARGET;
use super::{ModalitySupport, MultimodalModelInfo, PreparedMedia, item};
use super::{MultimodalModelInfo, PreparedMedia, VisionModalitySupport, item};
use crate::error::{Error, Result, bail_multimodal, multimodal};
/// Forward-kwargs name of the primary image encoder input.
@@ -63,15 +63,14 @@ impl MultimodalModelInfo {
/// conversion.
async fn preprocess_images(
&self,
support: &ModalitySupport,
support: &VisionModalitySupport,
image_frames: &[Arc<ImageFrame>],
) -> Result<PreprocessedEncoderInputs> {
let config = support.config.clone();
let processor = support.processor;
let processor = Arc::clone(&support.processor);
let images = image_frames.iter().map(|frame| frame.data().clone()).collect::<Vec<_>>();
// TODO: is it still necessary given that we've already in a dedicated runtime?
tokio::task::spawn_blocking(move || Ok(processor.preprocess(&images, &config)?))
tokio::task::spawn_blocking(move || Ok(processor.preprocess(&images)?))
.await
.map_err(|error| multimodal!("image preprocessing task failed: {error}"))?
}
rust/src/chat/src/multimodal/video.rs
+6 -7
View File
@@ -17,7 +17,7 @@ use vllm_engine_core_client::protocol::multimodal::{
};
use super::timing::MM_STAGE_TARGET;
use super::{ModalitySupport, MultimodalModelInfo, PreparedItem, PreparedMedia, tensor};
use super::{MultimodalModelInfo, PreparedItem, PreparedMedia, VisionModalitySupport, tensor};
use crate::error::{Error, Result, bail_multimodal, multimodal};
/// Forward-kwargs name of the primary video encoder input.
@@ -83,18 +83,17 @@ impl MultimodalModelInfo {
/// processor.
async fn preprocess_video_clip(
&self,
support: &ModalitySupport,
support: &VisionModalitySupport,
clip: Arc<VideoClip>,
) -> Result<PreprocessedEncoderInputs> {
let config = support.config.clone();
let processor = support.processor;
let processor = Arc::clone(&support.processor);
tokio::task::spawn_blocking(move || {
// Prefer the borrowed-RGB fast path, which avoids materializing a
// `DynamicImage` per sampled frame after media decode.
if let Some(rgb_video) = clip.rgb_video() {
match rgb_video.frame_refs() {
Ok(frame_refs) => match processor.preprocess_video_rgb(&frame_refs, &config) {
Ok(frame_refs) => match processor.preprocess_video_rgb(&frame_refs) {
Ok(preprocessed) => return Ok(preprocessed),
Err(error) => warn!(
error = %error.as_report(),
@@ -109,7 +108,7 @@ impl MultimodalModelInfo {
}
let frames = clip.materialized_frames().map_err(|error| multimodal!("{error}"))?;
Ok(processor.preprocess_video(&frames, &config)?)
Ok(processor.preprocess_video(&frames)?)
})
.await
.map_err(|error| multimodal!("video preprocessing task failed: {error}"))?
@@ -124,7 +123,7 @@ impl MultimodalModelInfo {
/// `flat_from_sizes` treatment of video patches), and batched metadata
/// tensors drop their singleton batch axis.
fn build_video_item(
support: &ModalitySupport,
support: &VisionModalitySupport,
preprocessed: PreprocessedEncoderInputs,
hash: String,
uuid: Option<String>,