diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 0a001833e9..b3f9084bcf 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -2290,7 +2290,7 @@ checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" [[package]] name = "llm-multimodal" version = "1.7.1" -source = "git+https://github.com/smg-project/llm-multimodal?rev=53ca0d6ea829af7477b6b63851cc70bed409bbc8#53ca0d6ea829af7477b6b63851cc70bed409bbc8" +source = "git+https://github.com/smg-project/llm-multimodal?rev=f0985ef65967615db2c79279aa07818499301bfd#f0985ef65967615db2c79279aa07818499301bfd" dependencies = [ "anyhow", "base64 0.22.1", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index afcce8d7fd..40b48f85bc 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -61,7 +61,7 @@ indexmap = "2.13.0" indicatif = "0.18.4" itertools = "0.14.0" libc = "0.2.177" -llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "53ca0d6ea829af7477b6b63851cc70bed409bbc8", default-features = false, features = ["native-tls"] } +llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "f0985ef65967615db2c79279aa07818499301bfd", default-features = false, features = ["native-tls"] } mimalloc = "0.1.52" minijinja = { version = "2.24", features = ["unstable_machinery", "json", "builtins", "loader", "loop_controls", "preserve_order"] } minijinja-contrib = { version = "2.24", features = ["pycompat"] } diff --git a/rust/src/chat/src/lib.rs b/rust/src/chat/src/lib.rs index 8f41d72004..95c4332f74 100644 --- a/rust/src/chat/src/lib.rs +++ b/rust/src/chat/src/lib.rs @@ -72,6 +72,9 @@ pub struct ChatRequestProcessor { /// Effective model dtype reported by the engine. /// Absent for text-only frontends without an engine handshake. model_dtype: Option, + /// Effective runtime context length reported by the engine. + /// Absent for render-only frontends without an engine handshake. + max_model_len: Option, /// Tool-call parser selection used when preparing generation requests. tool_call_parser: ParserSelection, /// Reasoning parser selection used when preparing generation requests. @@ -83,10 +86,11 @@ pub struct ChatRequestProcessor { impl ChatRequestProcessor { /// Create a processor with multimodal support using the effective model /// dtype reported by the engine. - fn new(backend: DynChatBackend, model_dtype: ModelDtype) -> Self { + fn new(backend: DynChatBackend, model_dtype: ModelDtype, max_model_len: u32) -> Self { Self { backend, model_dtype: Some(model_dtype), + max_model_len: Some(max_model_len as usize), tool_call_parser: ParserSelection::Auto, reasoning_parser: ParserSelection::Auto, tool_strict_level: ToolStrictLevel::Auto, @@ -98,6 +102,7 @@ impl ChatRequestProcessor { Self { backend, model_dtype: None, + max_model_len: None, tool_call_parser: ParserSelection::Auto, reasoning_parser: ParserSelection::Auto, tool_strict_level: ToolStrictLevel::Auto, @@ -137,6 +142,7 @@ impl ChatRequestProcessor { rendered, self.backend.multimodal_model_info(), model_dtype, + self.max_model_len, ) .await } @@ -161,7 +167,8 @@ impl ChatRequestProcessor { let features = match media { multimodal::MultimodalInput::Raw(parts) => { let model_dtype = self.model_dtype.ok_or(Error::UnsupportedMultimodalRenderer)?; - info.prepare_multimodal(parts, token_ids, model_dtype).await? + info.prepare_multimodal(parts, token_ids, model_dtype, self.max_model_len) + .await? } multimodal::MultimodalInput::Preprocessed(features) => { info.prepare_preprocessed(features, token_ids.len())? @@ -256,10 +263,11 @@ impl ChatLlm { /// backend. pub fn new(text: TextLlm, backend: DynChatBackend) -> Self { let model_dtype = text.engine_core_client().model_dtype(); + let max_model_len = text.engine_core_client().max_model_len(); Self { text, - processor: ChatRequestProcessor::new(backend, model_dtype), + processor: ChatRequestProcessor::new(backend, model_dtype, max_model_len), } } diff --git a/rust/src/chat/src/multimodal.rs b/rust/src/chat/src/multimodal.rs index 55acfea93b..4138d39bda 100644 --- a/rust/src/chat/src/multimodal.rs +++ b/rust/src/chat/src/multimodal.rs @@ -23,6 +23,7 @@ use llm_multimodal::{ ImageFrame, MediaConnector, MediaConnectorConfig, MediaContentPart, Modality, ModelMetadata, ModelProcessorSpec, ModelRegistry, PreProcessorConfig, PreprocessedEncoderInputs, PromptReplacement, Tokenizer as TokenResolver, TrackedMedia, VideoClip, VisionPreProcessor, + VisionPreprocessingContext, }; use serde::{Deserialize, Serialize}; use thiserror_ext::AsReport as _; @@ -55,6 +56,8 @@ pub use self::timing::{mm_request_span, mm_timing_layer}; #[derive(Clone)] pub struct MultimodalModelInfo { context: MultimodalModelContext, + /// Rendered placeholder marker IDs for all resolved modalities. + placeholder_token_ids: Vec, image: Option, video: Option, audio: Option, @@ -420,8 +423,18 @@ impl MultimodalModelInfo { MediaConnectorConfig::default(), )?); + let placeholder_token_ids = [ + image.as_ref().map(|support| support.placeholder.marker_token_id), + video.as_ref().map(|support| support.placeholder.marker_token_id), + audio.as_ref().map(|support| support.placeholder.marker_token_id), + ] + .into_iter() + .flatten() + .collect(); + Ok(Some(Self { context, + placeholder_token_ids, image, video, audio, @@ -547,6 +560,20 @@ impl MultimodalModelInfo { Modality::ImageEmbeds => None, } } + + fn vision_preprocessing_context( + &self, + prompt_token_ids: &[u32], + max_model_len: Option, + ) -> VisionPreprocessingContext { + let text_prompt_length = prompt_token_ids + .iter() + .filter(|token_id| !self.placeholder_token_ids.contains(token_id)) + .count(); + VisionPreprocessingContext { + token_budget: max_model_len.map(|limit| limit.saturating_sub(text_prompt_length)), + } + } } /// Finalize a rendered chat prompt into text-generation input. @@ -559,6 +586,7 @@ pub(crate) async fn finalize_rendered_prompt( rendered: RenderedPrompt, info: Option<&MultimodalModelInfo>, model_dtype: ModelDtype, + max_model_len: Option, ) -> Result<(Prompt, Option)> { let media_parts = extract_media_parts(request, rendered.media_order.as_deref())?; if media_parts.is_empty() { @@ -574,7 +602,12 @@ pub(crate) async fn finalize_rendered_prompt( Prompt::TokenIds(token_ids) => token_ids, }; let prepared = info - .prepare_multimodal(media_parts, &mut prompt_token_ids, model_dtype) + .prepare_multimodal( + media_parts, + &mut prompt_token_ids, + model_dtype, + max_model_len, + ) .instrument(mm_request_span(&request.request_id)) .await?; @@ -768,11 +801,13 @@ impl MultimodalModelInfo { media_parts: Vec, prompt_token_ids: &mut Vec, model_dtype: ModelDtype, + max_model_len: Option, ) -> Result { if media_parts.is_empty() { return Ok(Vec::new()); } - self.prepare_multimodal_timed(media_parts, prompt_token_ids, model_dtype).await + self.prepare_multimodal_timed(media_parts, prompt_token_ids, model_dtype, max_model_len) + .await } /// Timed body of [`Self::prepare_multimodal`]; each stage is a `tracing` @@ -788,15 +823,24 @@ impl MultimodalModelInfo { media_parts: Vec, prompt_token_ids: &mut Vec, model_dtype: ModelDtype, + max_model_len: Option, ) -> Result { let media_parts_len = media_parts.len(); self.validate_mm_limits(&media_parts)?; + let vision_context = self.vision_preprocessing_context(prompt_token_ids, max_model_len); let fetched = self.fetch_media(media_parts).await?; let mut prepared = Vec::new(); if !fetched.images.is_empty() { - prepared - .push(self.prepare_images(fetched.images, fetched.image_uuids, model_dtype).await?); + prepared.push( + self.prepare_images( + fetched.images, + fetched.image_uuids, + model_dtype, + vision_context, + ) + .await?, + ); } if !fetched.videos.is_empty() { prepared @@ -1044,6 +1088,100 @@ mod tests { ); } + #[test] + fn vision_preprocessing_context_uses_runtime_budget_and_text_tokens() { + let info = qwen3_vl_info(); + let context = info.vision_preprocessing_context( + &[11, QWEN3_IMAGE_PAD_ID, 12, QWEN3_VIDEO_PAD_ID, 13], + Some(4096), + ); + + assert_eq!(context.token_budget, Some(4096 - 3)); + assert_eq!( + info.vision_preprocessing_context(&[11, 12, 13], Some(2)).token_budget, + Some(0) + ); + assert_eq!( + info.vision_preprocessing_context(&[11], None).token_budget, + None + ); + } + + #[tokio::test] + async fn nemotron_budget_controls_preprocessing_and_engine_features() { + use vllm_engine_core_client::protocol::multimodal::MmKwargValue; + + let tokenizer = TestTokenizer::new() + .with_regular_token("", 1018) + .with_regular_token("", 1019) + .with_regular_token("", 1020); + let config = serde_json::json!({ + "model_type": "nemotron_h_omni", + "img_context_token_id": 1018, + "max_position_embeddings": 512, + "patch_size": 16, + "downsample_ratio": 0.5, + "norm_mean": [0.0, 0.0, 0.0], + "norm_std": [1.0, 1.0, 1.0], + "vision_config": { + "args": {"min_num_patches": 1024, "max_num_patches": 13312} + } + }); + let info = test_info("nemotron_h_omni", config, tokenizer); + let media = MediaContentPart::ImageUrl { + url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=".to_string(), + detail: None, + uuid: None, + }; + let mut prompt = vec![11; 100]; + prompt.push(1018); + + let mut tokens = prompt.clone(); + let features = info + .prepare_multimodal( + vec![media.clone()], + &mut tokens, + ModelDtype::Float32, + Some(512), + ) + .await + .unwrap(); + assert_eq!(features.len(), 1); + let feature = &features[0]; + assert_eq!( + (feature.mm_position.offset, feature.mm_position.length), + (100, 258) + ); + assert_eq!(tokens.len(), 358); + assert_eq!(tokens[100], 1019); + assert!(tokens[101..357].iter().all(|&token| token == 1018)); + assert_eq!(tokens[357], 1020); + let data = feature.data.as_ref().unwrap(); + assert!(matches!( + data["pixel_values_flat"].data.as_ref(), + Some(MmKwargValue::Tensor(tensor)) if tensor.shape == [3, 512, 512] + )); + assert_eq!( + data["num_tokens_per_image"].data, + Some(MmKwargValue::Int(256)) + ); + assert_eq!( + data["imgs_sizes"].data, + Some(MmKwargValue::List(vec![ + MmKwargValue::Int(512), + MmKwargValue::Int(512), + ])) + ); + + let error = info + .prepare_multimodal(vec![media], &mut prompt, ModelDtype::Float32, Some(100)) + .await + .unwrap_err(); + assert!( + matches!(error, Error::Multimodal(message) if message.contains("exceeding token_budget")) + ); + } + #[test] fn qwen3_vl_without_video_token_id_disables_video_support_only() { let config = serde_json::json!({ diff --git a/rust/src/chat/src/multimodal/audio.rs b/rust/src/chat/src/multimodal/audio.rs index c230752f94..6b2fecf78d 100644 --- a/rust/src/chat/src/multimodal/audio.rs +++ b/rust/src/chat/src/multimodal/audio.rs @@ -259,7 +259,7 @@ mod tests { uuid: Some("audio-1".to_string()), }]; async { - info.prepare_multimodal(parts, &mut vec![AUDIO_PAD_ID], ModelDtype::Float32) + info.prepare_multimodal(parts, &mut vec![AUDIO_PAD_ID], ModelDtype::Float32, None) .instrument(mm_request_span("request-1")) .await .unwrap(); diff --git a/rust/src/chat/src/multimodal/image.rs b/rust/src/chat/src/multimodal/image.rs index 3062dae3e6..818bb72f1a 100644 --- a/rust/src/chat/src/multimodal/image.rs +++ b/rust/src/chat/src/multimodal/image.rs @@ -6,7 +6,7 @@ use std::sync::Arc; -use llm_multimodal::{ImageFrame, Modality, PreprocessedEncoderInputs}; +use llm_multimodal::{ImageFrame, Modality, PreprocessedEncoderInputs, VisionPreprocessingContext}; use vllm_engine_core_client::protocol::dtype::ModelDtype; use super::timing::MM_STAGE_TARGET; @@ -30,11 +30,12 @@ impl MultimodalModelInfo { frames: Vec>, uuids: Vec>, model_dtype: ModelDtype, + preprocessing_context: VisionPreprocessingContext, ) -> Result { let support = self.image.as_ref().ok_or_else(|| Error::UnsupportedModality { modality: Modality::Image.to_string(), })?; - let preprocessed = self.preprocess_images(support, &frames).await?; + let preprocessed = self.preprocess_images(support, &frames, preprocessing_context).await?; let replacements = support.spec.prompt_replacements_for(&self.context, &preprocessed)?; if replacements.len() != frames.len() { bail_multimodal!( @@ -65,13 +66,16 @@ impl MultimodalModelInfo { &self, support: &VisionModalitySupport, image_frames: &[Arc], + preprocessing_context: VisionPreprocessingContext, ) -> Result { let processor = Arc::clone(&support.processor); let images = image_frames.iter().map(|frame| frame.data().clone()).collect::>(); // TODO: is it still necessary given that we've already in a dedicated runtime? - tokio::task::spawn_blocking(move || Ok(processor.preprocess(&images)?)) - .await - .map_err(|error| multimodal!("image preprocessing task failed: {error}"))? + tokio::task::spawn_blocking(move || { + Ok(processor.preprocess_with_context(&images, &preprocessing_context)?) + }) + .await + .map_err(|error| multimodal!("image preprocessing task failed: {error}"))? } } diff --git a/rust/src/chat/src/multimodal/tensor.rs b/rust/src/chat/src/multimodal/tensor.rs index 2c564fbda7..3991130989 100644 --- a/rust/src/chat/src/multimodal/tensor.rs +++ b/rust/src/chat/src/multimodal/tensor.rs @@ -18,7 +18,7 @@ use crate::error::{Error, Result, bail_multimodal, multimodal}; pub(super) enum KwargValue { /// Tensor with row-major flat data and shape. Tensor { wire: WireTensor }, - /// Non-tensor kwarg value that is shared or copied as-is. + /// Non-tensor kwarg value preserved in protocol form. Passthrough(ProtocolKwargValue), } @@ -150,7 +150,17 @@ impl KwargValue { /// Batched fields use their first axis as media-item index and drop that /// axis in the per-feature value, matching vLLM's batched-field semantics. pub(super) fn batched_wire_value_at(&self, index: usize) -> Result { - self.wire_value_range(index, index + 1, true) + match self { + Self::Passthrough(ProtocolKwargValue::List(values)) => values + .get(index) + .cloned() + .ok_or_else(|| multimodal!("batched passthrough has no item at index {index}")), + Self::Passthrough(value) if index == 0 => Ok(value.clone()), + Self::Passthrough(_) => Err(multimodal!( + "scalar batched passthrough has no item at index {index}" + )), + Self::Tensor { .. } => self.wire_value_range(index, index + 1, true), + } } /// Convert one media item's flat tensor range directly to wire bytes. @@ -336,6 +346,41 @@ mod tests { ); } + #[test] + fn batched_passthrough_value_at_selects_media_item() { + let image_sizes = KwargValue::from_model_specific( + ModelSpecificValue::TupleVec(vec![(384, 736), (512, 512)]), + ModelDtype::BFloat16, + ) + .unwrap(); + + assert_eq!( + image_sizes.batched_wire_value_at(1).unwrap(), + ProtocolKwargValue::List(vec![ + ProtocolKwargValue::Int(512), + ProtocolKwargValue::Int(512), + ]) + ); + assert!(image_sizes.batched_wire_value_at(2).is_err()); + + let token_counts = KwargValue::from_model_specific( + ModelSpecificValue::IntVec(vec![276, 400]), + ModelDtype::BFloat16, + ) + .unwrap(); + assert_eq!( + token_counts.batched_wire_value_at(1).unwrap(), + ProtocolKwargValue::Int(400) + ); + + let scalar = KwargValue::Passthrough(ProtocolKwargValue::Int(7)); + assert_eq!( + scalar.batched_wire_value_at(0).unwrap(), + ProtocolKwargValue::Int(7) + ); + assert!(scalar.batched_wire_value_at(1).is_err()); + } + #[test] fn flat_wire_value_range_keeps_first_axis() { let value = KwargValue::Tensor { diff --git a/rust/src/text/src/backend/hf/config.rs b/rust/src/text/src/backend/hf/config.rs index 849874bede..0768adf4ae 100644 --- a/rust/src/text/src/backend/hf/config.rs +++ b/rust/src/text/src/backend/hf/config.rs @@ -84,10 +84,11 @@ impl HfSpecialTokens { /// Minimal subset of `config.json` (the model's main HF config). /// -/// This intentionally supports only the two layouts we currently care about in +/// This intentionally supports only the layouts we currently care about in /// the Rust frontend: /// - pure text models that keep text metadata at the top level -/// - composite models that expose a single nested `text_config` +/// - composite models that expose a single nested `text_config` or its +/// `llm_config` alias (used by Nemotron) /// /// We do not support additional entry points such as `decoder`, `generator`, or /// `text_encoder`. @@ -102,6 +103,7 @@ pub struct ModelConfig { n_routed_experts: Option, num_local_experts: Option, block_configs: Vec, + #[serde(alias = "llm_config")] text_config: Option>, } @@ -188,7 +190,7 @@ impl ModelConfig { /// Return the config that the Rust frontend treats as the text/LLM config. /// /// This is deliberately narrower than Python/transformers: we only support - /// either the top-level config itself or a single nested `text_config`. + /// either the top-level config itself or a single nested text config. fn effective_text_config(&self) -> &Self { self.text_config.as_deref().unwrap_or(self) } @@ -233,7 +235,8 @@ impl ModelConfig { /// config. /// /// The only intentional simplification here is how we pick the text config: - /// Rust only looks at the top level or `text_config`, not the broader + /// Rust only looks at the top level or `text_config` (including its + /// `llm_config` alias), not the broader /// transformers composite-config surface. fn num_experts_from_block_configs(&self) -> u32 { self.effective_text_config() @@ -405,6 +408,27 @@ mod tests { assert_eq!(config.vocab_size().unwrap(), 151936); } + #[test] + fn model_config_uses_llm_config_for_nemotron_composite_models() { + let config: ModelConfig = serde_json::from_str( + r#"{ + "model_type": "nemotron_h_omni", + "llm_config": { + "model_type": "nemotron_h", + "vocab_size": 131072, + "n_routed_experts": 256, + "eos_token_id": 2 + } + }"#, + ) + .unwrap(); + + assert_eq!(config.model_type(), Some("nemotron_h_omni")); + assert_eq!(config.vocab_size().unwrap(), 131072); + assert_eq!(config.eos_token_ids(), &[2]); + assert_eq!(config.num_experts(), 256); + } + #[test] fn model_config_reads_top_level_eos_token_ids() { let single: ModelConfig = serde_json::from_str(r#"{"eos_token_id":151645}"#).unwrap();