[Rust Frontend] Construct model-owned vision processors through specs (#58109)
Co-authored-by: Codex <noreply@openai.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Generated
+1
-1
@@ -2290,7 +2290,7 @@ checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092"
|
|||||||
[[package]]
|
[[package]]
|
||||||
name = "llm-multimodal"
|
name = "llm-multimodal"
|
||||||
version = "1.7.1"
|
version = "1.7.1"
|
||||||
source = "git+https://github.com/smg-project/llm-multimodal?rev=24c676dd49d73f05cb4068aade2e60397a5c0afe#24c676dd49d73f05cb4068aade2e60397a5c0afe"
|
source = "git+https://github.com/smg-project/llm-multimodal?rev=53ca0d6ea829af7477b6b63851cc70bed409bbc8#53ca0d6ea829af7477b6b63851cc70bed409bbc8"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"base64 0.22.1",
|
"base64 0.22.1",
|
||||||
|
|||||||
+1
-1
@@ -61,7 +61,7 @@ indexmap = "2.13.0"
|
|||||||
indicatif = "0.18.4"
|
indicatif = "0.18.4"
|
||||||
itertools = "0.14.0"
|
itertools = "0.14.0"
|
||||||
libc = "0.2.177"
|
libc = "0.2.177"
|
||||||
llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "24c676dd49d73f05cb4068aade2e60397a5c0afe", default-features = false, features = ["native-tls"] }
|
llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "53ca0d6ea829af7477b6b63851cc70bed409bbc8", default-features = false, features = ["native-tls"] }
|
||||||
mimalloc = "0.1.52"
|
mimalloc = "0.1.52"
|
||||||
minijinja = { version = "2.24", features = ["unstable_machinery", "json", "builtins", "loader", "loop_controls", "preserve_order"] }
|
minijinja = { version = "2.24", features = ["unstable_machinery", "json", "builtins", "loader", "loop_controls", "preserve_order"] }
|
||||||
minijinja-contrib = { version = "2.24", features = ["pycompat"] }
|
minijinja-contrib = { version = "2.24", features = ["pycompat"] }
|
||||||
|
|||||||
@@ -17,12 +17,12 @@ use std::path::Path;
|
|||||||
use std::sync::{Arc, LazyLock};
|
use std::sync::{Arc, LazyLock};
|
||||||
|
|
||||||
use itertools::{Either, izip};
|
use itertools::{Either, izip};
|
||||||
|
use llm_multimodal::registry::ModelRegistryError;
|
||||||
use llm_multimodal::{
|
use llm_multimodal::{
|
||||||
AsyncMultiModalTracker, AudioClip, AudioPreProcessor, EncoderFieldLayouts, FieldLayout,
|
AsyncMultiModalTracker, AudioClip, AudioPreProcessor, EncoderFieldLayouts, FieldLayout,
|
||||||
ImageFrame, MediaConnector, MediaConnectorConfig, MediaContentPart, Modality, ModelMetadata,
|
ImageFrame, MediaConnector, MediaConnectorConfig, MediaContentPart, Modality, ModelMetadata,
|
||||||
ModelProcessorSpec, ModelRegistry, PreProcessorConfig, PreprocessedEncoderInputs,
|
ModelProcessorSpec, ModelRegistry, PreProcessorConfig, PreprocessedEncoderInputs,
|
||||||
PromptReplacement, Tokenizer as TokenResolver, TrackedMedia, VideoClip, VisionPreProcessor,
|
PromptReplacement, Tokenizer as TokenResolver, TrackedMedia, VideoClip, VisionPreProcessor,
|
||||||
VisionProcessorRegistry,
|
|
||||||
};
|
};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use thiserror_ext::AsReport as _;
|
use thiserror_ext::AsReport as _;
|
||||||
@@ -55,8 +55,8 @@ pub use self::timing::{mm_request_span, mm_timing_layer};
|
|||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct MultimodalModelInfo {
|
pub struct MultimodalModelInfo {
|
||||||
context: MultimodalModelContext,
|
context: MultimodalModelContext,
|
||||||
image: Option<ModalitySupport>,
|
image: Option<VisionModalitySupport>,
|
||||||
video: Option<ModalitySupport>,
|
video: Option<VisionModalitySupport>,
|
||||||
audio: Option<AudioModalitySupport>,
|
audio: Option<AudioModalitySupport>,
|
||||||
media_connector: Arc<MediaConnector>,
|
media_connector: Arc<MediaConnector>,
|
||||||
/// Maximum number of input items allowed per prompt for each modality.
|
/// Maximum number of input items allowed per prompt for each modality.
|
||||||
@@ -133,13 +133,18 @@ impl MultimodalModelContext {
|
|||||||
REGISTRY.lookup(&self.metadata())
|
REGISTRY.lookup(&self.metadata())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Resolve a static vision preprocessor for one loaded model.
|
/// Build a vision preprocessor for one loaded model and modality.
|
||||||
///
|
fn resolve_vision_processor(
|
||||||
/// The vision preprocessor serves both the image and video modalities.
|
&self,
|
||||||
fn resolve_vision_processor(&self) -> Option<&'static dyn VisionPreProcessor> {
|
model_spec: &'static dyn ModelProcessorSpec,
|
||||||
static REGISTRY: LazyLock<VisionProcessorRegistry> =
|
preprocessor_config: &PreProcessorConfig,
|
||||||
LazyLock::new(VisionProcessorRegistry::with_defaults);
|
modality: Modality,
|
||||||
REGISTRY.find(&self.model_id, self.model_type.as_deref())
|
) -> Result<Option<Arc<dyn VisionPreProcessor>>> {
|
||||||
|
match model_spec.vision_processor(&self.metadata(), preprocessor_config, modality) {
|
||||||
|
Ok(processor) => Ok(Some(Arc::from(processor))),
|
||||||
|
Err(ModelRegistryError::UnsupportedModality { .. }) => Ok(None),
|
||||||
|
Err(error) => Err(error.into()),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Resolve an audio preprocessor for one loaded model.
|
/// Resolve an audio preprocessor for one loaded model.
|
||||||
@@ -148,7 +153,7 @@ impl MultimodalModelContext {
|
|||||||
model_spec: &'static dyn ModelProcessorSpec,
|
model_spec: &'static dyn ModelProcessorSpec,
|
||||||
preprocessor_config: &PreProcessorConfig,
|
preprocessor_config: &PreProcessorConfig,
|
||||||
) -> Option<Arc<dyn AudioPreProcessor>> {
|
) -> Option<Arc<dyn AudioPreProcessor>> {
|
||||||
model_spec.audio_processor(&self.config, preprocessor_config).map(Arc::from)
|
model_spec.audio_processor(&self.metadata(), preprocessor_config).map(Arc::from)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -242,17 +247,16 @@ impl ResolvedPlaceholder {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Static per-modality vision preprocessor plus its loaded config, resolved
|
/// Model-owned vision preprocessor plus resolved placeholder tokens and
|
||||||
/// placeholder tokens, and the model's shared tensor-layout spec.
|
/// the model's shared tensor-layout spec.
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
struct ModalitySupport {
|
struct VisionModalitySupport {
|
||||||
spec: ResolvedMultimodalSpec,
|
spec: ResolvedMultimodalSpec,
|
||||||
placeholder: ResolvedPlaceholder,
|
placeholder: ResolvedPlaceholder,
|
||||||
processor: &'static dyn VisionPreProcessor,
|
processor: Arc<dyn VisionPreProcessor>,
|
||||||
config: PreProcessorConfig,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Static audio preprocessor plus the resolved model contract for its output.
|
/// Model-owned audio preprocessor plus the resolved model contract for its output.
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
struct AudioModalitySupport {
|
struct AudioModalitySupport {
|
||||||
spec: ResolvedMultimodalSpec,
|
spec: ResolvedMultimodalSpec,
|
||||||
@@ -399,7 +403,7 @@ impl MultimodalModelInfo {
|
|||||||
&context,
|
&context,
|
||||||
preprocessor_config.clone(),
|
preprocessor_config.clone(),
|
||||||
video_preprocessor_config,
|
video_preprocessor_config,
|
||||||
);
|
)?;
|
||||||
let audio = Self::resolve_audio_lane(&context, &preprocessor_config)?;
|
let audio = Self::resolve_audio_lane(&context, &preprocessor_config)?;
|
||||||
|
|
||||||
if image.is_none() && video.is_none() && audio.is_none() {
|
if image.is_none() && video.is_none() && audio.is_none() {
|
||||||
@@ -430,27 +434,19 @@ impl MultimodalModelInfo {
|
|||||||
///
|
///
|
||||||
/// A missing model spec or vision processor disables both lanes but is
|
/// A missing model spec or vision processor disables both lanes but is
|
||||||
/// not an error: unsupported model families are logged and skipped.
|
/// not an error: unsupported model families are logged and skipped.
|
||||||
|
/// Processor construction errors propagate to model loading.
|
||||||
fn resolve_vision_lanes(
|
fn resolve_vision_lanes(
|
||||||
context: &MultimodalModelContext,
|
context: &MultimodalModelContext,
|
||||||
preprocessor_config: PreProcessorConfig,
|
preprocessor_config: PreProcessorConfig,
|
||||||
video_preprocessor_config: PreProcessorConfig,
|
video_preprocessor_config: PreProcessorConfig,
|
||||||
) -> (Option<ModalitySupport>, Option<ModalitySupport>) {
|
) -> Result<(Option<VisionModalitySupport>, Option<VisionModalitySupport>)> {
|
||||||
let Some(raw_spec) = context.resolve_model_spec() else {
|
let Some(raw_spec) = context.resolve_model_spec() else {
|
||||||
warn!(
|
warn!(
|
||||||
model_id = context.model_id,
|
model_id = context.model_id,
|
||||||
model_type = context.model_type,
|
model_type = context.model_type,
|
||||||
"multimodal model spec is not registered; disabling image/video support for this model"
|
"multimodal model spec is not registered; disabling image/video support for this model"
|
||||||
);
|
);
|
||||||
return (None, None);
|
return Ok((None, None));
|
||||||
};
|
|
||||||
|
|
||||||
let Some(processor) = context.resolve_vision_processor() else {
|
|
||||||
warn!(
|
|
||||||
model_id = context.model_id,
|
|
||||||
model_type = context.model_type,
|
|
||||||
"vision processor is not registered; disabling image/video support for this model"
|
|
||||||
);
|
|
||||||
return (None, None);
|
|
||||||
};
|
};
|
||||||
|
|
||||||
// Warn and disable the modality if the placeholder resolution fails.
|
// Warn and disable the modality if the placeholder resolution fails.
|
||||||
@@ -468,14 +464,19 @@ impl MultimodalModelInfo {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let image = resolve_placeholder(Modality::Image).map(|placeholder| ModalitySupport {
|
let image = if let Some(placeholder) = resolve_placeholder(Modality::Image) {
|
||||||
spec: ResolvedMultimodalSpec::new(raw_spec, Modality::Image),
|
context
|
||||||
placeholder,
|
.resolve_vision_processor(raw_spec, &preprocessor_config, Modality::Image)?
|
||||||
processor,
|
.map(|processor| VisionModalitySupport {
|
||||||
config: preprocessor_config.clone(),
|
spec: ResolvedMultimodalSpec::new(raw_spec, Modality::Image),
|
||||||
});
|
placeholder,
|
||||||
|
processor,
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
|
||||||
let video = resolve_placeholder(Modality::Video).and_then(|placeholder| {
|
let video = if let Some(placeholder) = resolve_placeholder(Modality::Video) {
|
||||||
// Placeholder expansion attributes markers to modalities by token
|
// Placeholder expansion attributes markers to modalities by token
|
||||||
// ID, so a marker shared with the image modality is ambiguous.
|
// ID, so a marker shared with the image modality is ambiguous.
|
||||||
let image_marker = image.as_ref().map(|image| image.placeholder.marker_token_id);
|
let image_marker = image.as_ref().map(|image| image.placeholder.marker_token_id);
|
||||||
@@ -487,16 +488,23 @@ impl MultimodalModelInfo {
|
|||||||
);
|
);
|
||||||
None
|
None
|
||||||
} else {
|
} else {
|
||||||
Some(ModalitySupport {
|
context
|
||||||
spec: ResolvedMultimodalSpec::new(raw_spec, Modality::Video),
|
.resolve_vision_processor(
|
||||||
placeholder,
|
raw_spec,
|
||||||
processor,
|
&video_preprocessor_config,
|
||||||
config: video_preprocessor_config,
|
Modality::Video,
|
||||||
})
|
)?
|
||||||
|
.map(|processor| VisionModalitySupport {
|
||||||
|
spec: ResolvedMultimodalSpec::new(raw_spec, Modality::Video),
|
||||||
|
placeholder,
|
||||||
|
processor,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
});
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
|
||||||
(image, video)
|
Ok((image, video))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Resolve a spec-backed audio lane when the model advertises audio support.
|
/// Resolve a spec-backed audio lane when the model advertises audio support.
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ use llm_multimodal::{ImageFrame, Modality, PreprocessedEncoderInputs};
|
|||||||
use vllm_engine_core_client::protocol::dtype::ModelDtype;
|
use vllm_engine_core_client::protocol::dtype::ModelDtype;
|
||||||
|
|
||||||
use super::timing::MM_STAGE_TARGET;
|
use super::timing::MM_STAGE_TARGET;
|
||||||
use super::{ModalitySupport, MultimodalModelInfo, PreparedMedia, item};
|
use super::{MultimodalModelInfo, PreparedMedia, VisionModalitySupport, item};
|
||||||
use crate::error::{Error, Result, bail_multimodal, multimodal};
|
use crate::error::{Error, Result, bail_multimodal, multimodal};
|
||||||
|
|
||||||
/// Forward-kwargs name of the primary image encoder input.
|
/// Forward-kwargs name of the primary image encoder input.
|
||||||
@@ -63,15 +63,14 @@ impl MultimodalModelInfo {
|
|||||||
/// conversion.
|
/// conversion.
|
||||||
async fn preprocess_images(
|
async fn preprocess_images(
|
||||||
&self,
|
&self,
|
||||||
support: &ModalitySupport,
|
support: &VisionModalitySupport,
|
||||||
image_frames: &[Arc<ImageFrame>],
|
image_frames: &[Arc<ImageFrame>],
|
||||||
) -> Result<PreprocessedEncoderInputs> {
|
) -> Result<PreprocessedEncoderInputs> {
|
||||||
let config = support.config.clone();
|
let processor = Arc::clone(&support.processor);
|
||||||
let processor = support.processor;
|
|
||||||
let images = image_frames.iter().map(|frame| frame.data().clone()).collect::<Vec<_>>();
|
let images = image_frames.iter().map(|frame| frame.data().clone()).collect::<Vec<_>>();
|
||||||
|
|
||||||
// TODO: is it still necessary given that we've already in a dedicated runtime?
|
// TODO: is it still necessary given that we've already in a dedicated runtime?
|
||||||
tokio::task::spawn_blocking(move || Ok(processor.preprocess(&images, &config)?))
|
tokio::task::spawn_blocking(move || Ok(processor.preprocess(&images)?))
|
||||||
.await
|
.await
|
||||||
.map_err(|error| multimodal!("image preprocessing task failed: {error}"))?
|
.map_err(|error| multimodal!("image preprocessing task failed: {error}"))?
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ use vllm_engine_core_client::protocol::multimodal::{
|
|||||||
};
|
};
|
||||||
|
|
||||||
use super::timing::MM_STAGE_TARGET;
|
use super::timing::MM_STAGE_TARGET;
|
||||||
use super::{ModalitySupport, MultimodalModelInfo, PreparedItem, PreparedMedia, tensor};
|
use super::{MultimodalModelInfo, PreparedItem, PreparedMedia, VisionModalitySupport, tensor};
|
||||||
use crate::error::{Error, Result, bail_multimodal, multimodal};
|
use crate::error::{Error, Result, bail_multimodal, multimodal};
|
||||||
|
|
||||||
/// Forward-kwargs name of the primary video encoder input.
|
/// Forward-kwargs name of the primary video encoder input.
|
||||||
@@ -83,18 +83,17 @@ impl MultimodalModelInfo {
|
|||||||
/// processor.
|
/// processor.
|
||||||
async fn preprocess_video_clip(
|
async fn preprocess_video_clip(
|
||||||
&self,
|
&self,
|
||||||
support: &ModalitySupport,
|
support: &VisionModalitySupport,
|
||||||
clip: Arc<VideoClip>,
|
clip: Arc<VideoClip>,
|
||||||
) -> Result<PreprocessedEncoderInputs> {
|
) -> Result<PreprocessedEncoderInputs> {
|
||||||
let config = support.config.clone();
|
let processor = Arc::clone(&support.processor);
|
||||||
let processor = support.processor;
|
|
||||||
|
|
||||||
tokio::task::spawn_blocking(move || {
|
tokio::task::spawn_blocking(move || {
|
||||||
// Prefer the borrowed-RGB fast path, which avoids materializing a
|
// Prefer the borrowed-RGB fast path, which avoids materializing a
|
||||||
// `DynamicImage` per sampled frame after media decode.
|
// `DynamicImage` per sampled frame after media decode.
|
||||||
if let Some(rgb_video) = clip.rgb_video() {
|
if let Some(rgb_video) = clip.rgb_video() {
|
||||||
match rgb_video.frame_refs() {
|
match rgb_video.frame_refs() {
|
||||||
Ok(frame_refs) => match processor.preprocess_video_rgb(&frame_refs, &config) {
|
Ok(frame_refs) => match processor.preprocess_video_rgb(&frame_refs) {
|
||||||
Ok(preprocessed) => return Ok(preprocessed),
|
Ok(preprocessed) => return Ok(preprocessed),
|
||||||
Err(error) => warn!(
|
Err(error) => warn!(
|
||||||
error = %error.as_report(),
|
error = %error.as_report(),
|
||||||
@@ -109,7 +108,7 @@ impl MultimodalModelInfo {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let frames = clip.materialized_frames().map_err(|error| multimodal!("{error}"))?;
|
let frames = clip.materialized_frames().map_err(|error| multimodal!("{error}"))?;
|
||||||
Ok(processor.preprocess_video(&frames, &config)?)
|
Ok(processor.preprocess_video(&frames)?)
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.map_err(|error| multimodal!("video preprocessing task failed: {error}"))?
|
.map_err(|error| multimodal!("video preprocessing task failed: {error}"))?
|
||||||
@@ -124,7 +123,7 @@ impl MultimodalModelInfo {
|
|||||||
/// `flat_from_sizes` treatment of video patches), and batched metadata
|
/// `flat_from_sizes` treatment of video patches), and batched metadata
|
||||||
/// tensors drop their singleton batch axis.
|
/// tensors drop their singleton batch axis.
|
||||||
fn build_video_item(
|
fn build_video_item(
|
||||||
support: &ModalitySupport,
|
support: &VisionModalitySupport,
|
||||||
preprocessed: PreprocessedEncoderInputs,
|
preprocessed: PreprocessedEncoderInputs,
|
||||||
hash: String,
|
hash: String,
|
||||||
uuid: Option<String>,
|
uuid: Option<String>,
|
||||||
|
|||||||
Reference in New Issue
Block a user