[Rust Frontend] Pass vision preprocessing context for Nemotron-H (#57634)

Pass the remaining engine context-length budget to model-owned vision processors through VisionPreprocessingContext. Preserve Nemotron batched engine fields and recognize llm_config as a text_config alias.

Use the merged upstream llm-multimodal revision f0985ef65967615db2c79279aa07818499301bfd.

Co-authored-by: Codex <noreply@openai.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Biswa Panda
2026-09-24 14:08:26 +08:00
committed by GitHub
co-authored by Codex
parent ee2f3cdf12
commit cf78797cb7
8 changed files with 240 additions and 21 deletions
+1 -1
View File
@@ -2290,7 +2290,7 @@ checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092"
[[package]]
name = "llm-multimodal"
version = "1.7.1"
source = "git+https://github.com/smg-project/llm-multimodal?rev=53ca0d6ea829af7477b6b63851cc70bed409bbc8#53ca0d6ea829af7477b6b63851cc70bed409bbc8"
source = "git+https://github.com/smg-project/llm-multimodal?rev=f0985ef65967615db2c79279aa07818499301bfd#f0985ef65967615db2c79279aa07818499301bfd"
dependencies = [
"anyhow",
"base64 0.22.1",
+1 -1
View File
@@ -61,7 +61,7 @@ indexmap = "2.13.0"
indicatif = "0.18.4"
itertools = "0.14.0"
libc = "0.2.177"
llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "53ca0d6ea829af7477b6b63851cc70bed409bbc8", default-features = false, features = ["native-tls"] }
llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "f0985ef65967615db2c79279aa07818499301bfd", default-features = false, features = ["native-tls"] }
mimalloc = "0.1.52"
minijinja = { version = "2.24", features = ["unstable_machinery", "json", "builtins", "loader", "loop_controls", "preserve_order"] }
minijinja-contrib = { version = "2.24", features = ["pycompat"] }
+11 -3
View File
@@ -72,6 +72,9 @@ pub struct ChatRequestProcessor {
/// Effective model dtype reported by the engine.
/// Absent for text-only frontends without an engine handshake.
model_dtype: Option<ModelDtype>,
/// Effective runtime context length reported by the engine.
/// Absent for render-only frontends without an engine handshake.
max_model_len: Option<usize>,
/// Tool-call parser selection used when preparing generation requests.
tool_call_parser: ParserSelection,
/// Reasoning parser selection used when preparing generation requests.
@@ -83,10 +86,11 @@ pub struct ChatRequestProcessor {
impl ChatRequestProcessor {
/// Create a processor with multimodal support using the effective model
/// dtype reported by the engine.
fn new(backend: DynChatBackend, model_dtype: ModelDtype) -> Self {
fn new(backend: DynChatBackend, model_dtype: ModelDtype, max_model_len: u32) -> Self {
Self {
backend,
model_dtype: Some(model_dtype),
max_model_len: Some(max_model_len as usize),
tool_call_parser: ParserSelection::Auto,
reasoning_parser: ParserSelection::Auto,
tool_strict_level: ToolStrictLevel::Auto,
@@ -98,6 +102,7 @@ impl ChatRequestProcessor {
Self {
backend,
model_dtype: None,
max_model_len: None,
tool_call_parser: ParserSelection::Auto,
reasoning_parser: ParserSelection::Auto,
tool_strict_level: ToolStrictLevel::Auto,
@@ -137,6 +142,7 @@ impl ChatRequestProcessor {
rendered,
self.backend.multimodal_model_info(),
model_dtype,
self.max_model_len,
)
.await
}
@@ -161,7 +167,8 @@ impl ChatRequestProcessor {
let features = match media {
multimodal::MultimodalInput::Raw(parts) => {
let model_dtype = self.model_dtype.ok_or(Error::UnsupportedMultimodalRenderer)?;
info.prepare_multimodal(parts, token_ids, model_dtype).await?
info.prepare_multimodal(parts, token_ids, model_dtype, self.max_model_len)
.await?
}
multimodal::MultimodalInput::Preprocessed(features) => {
info.prepare_preprocessed(features, token_ids.len())?
@@ -256,10 +263,11 @@ impl ChatLlm {
/// backend.
pub fn new(text: TextLlm, backend: DynChatBackend) -> Self {
let model_dtype = text.engine_core_client().model_dtype();
let max_model_len = text.engine_core_client().max_model_len();
Self {
text,
processor: ChatRequestProcessor::new(backend, model_dtype),
processor: ChatRequestProcessor::new(backend, model_dtype, max_model_len),
}
}
+142 -4
View File
@@ -23,6 +23,7 @@ use llm_multimodal::{
ImageFrame, MediaConnector, MediaConnectorConfig, MediaContentPart, Modality, ModelMetadata,
ModelProcessorSpec, ModelRegistry, PreProcessorConfig, PreprocessedEncoderInputs,
PromptReplacement, Tokenizer as TokenResolver, TrackedMedia, VideoClip, VisionPreProcessor,
VisionPreprocessingContext,
};
use serde::{Deserialize, Serialize};
use thiserror_ext::AsReport as _;
@@ -55,6 +56,8 @@ pub use self::timing::{mm_request_span, mm_timing_layer};
#[derive(Clone)]
pub struct MultimodalModelInfo {
context: MultimodalModelContext,
/// Rendered placeholder marker IDs for all resolved modalities.
placeholder_token_ids: Vec<u32>,
image: Option<VisionModalitySupport>,
video: Option<VisionModalitySupport>,
audio: Option<AudioModalitySupport>,
@@ -420,8 +423,18 @@ impl MultimodalModelInfo {
MediaConnectorConfig::default(),
)?);
let placeholder_token_ids = [
image.as_ref().map(|support| support.placeholder.marker_token_id),
video.as_ref().map(|support| support.placeholder.marker_token_id),
audio.as_ref().map(|support| support.placeholder.marker_token_id),
]
.into_iter()
.flatten()
.collect();
Ok(Some(Self {
context,
placeholder_token_ids,
image,
video,
audio,
@@ -547,6 +560,20 @@ impl MultimodalModelInfo {
Modality::ImageEmbeds => None,
}
}
fn vision_preprocessing_context(
&self,
prompt_token_ids: &[u32],
max_model_len: Option<usize>,
) -> VisionPreprocessingContext {
let text_prompt_length = prompt_token_ids
.iter()
.filter(|token_id| !self.placeholder_token_ids.contains(token_id))
.count();
VisionPreprocessingContext {
token_budget: max_model_len.map(|limit| limit.saturating_sub(text_prompt_length)),
}
}
}
/// Finalize a rendered chat prompt into text-generation input.
@@ -559,6 +586,7 @@ pub(crate) async fn finalize_rendered_prompt(
rendered: RenderedPrompt,
info: Option<&MultimodalModelInfo>,
model_dtype: ModelDtype,
max_model_len: Option<usize>,
) -> Result<(Prompt, Option<MmFeatures>)> {
let media_parts = extract_media_parts(request, rendered.media_order.as_deref())?;
if media_parts.is_empty() {
@@ -574,7 +602,12 @@ pub(crate) async fn finalize_rendered_prompt(
Prompt::TokenIds(token_ids) => token_ids,
};
let prepared = info
.prepare_multimodal(media_parts, &mut prompt_token_ids, model_dtype)
.prepare_multimodal(
media_parts,
&mut prompt_token_ids,
model_dtype,
max_model_len,
)
.instrument(mm_request_span(&request.request_id))
.await?;
@@ -768,11 +801,13 @@ impl MultimodalModelInfo {
media_parts: Vec<MediaContentPart>,
prompt_token_ids: &mut Vec<u32>,
model_dtype: ModelDtype,
max_model_len: Option<usize>,
) -> Result<MmFeatures> {
if media_parts.is_empty() {
return Ok(Vec::new());
}
self.prepare_multimodal_timed(media_parts, prompt_token_ids, model_dtype).await
self.prepare_multimodal_timed(media_parts, prompt_token_ids, model_dtype, max_model_len)
.await
}
/// Timed body of [`Self::prepare_multimodal`]; each stage is a `tracing`
@@ -788,15 +823,24 @@ impl MultimodalModelInfo {
media_parts: Vec<MediaContentPart>,
prompt_token_ids: &mut Vec<u32>,
model_dtype: ModelDtype,
max_model_len: Option<usize>,
) -> Result<MmFeatures> {
let media_parts_len = media_parts.len();
self.validate_mm_limits(&media_parts)?;
let vision_context = self.vision_preprocessing_context(prompt_token_ids, max_model_len);
let fetched = self.fetch_media(media_parts).await?;
let mut prepared = Vec::new();
if !fetched.images.is_empty() {
prepared
.push(self.prepare_images(fetched.images, fetched.image_uuids, model_dtype).await?);
prepared.push(
self.prepare_images(
fetched.images,
fetched.image_uuids,
model_dtype,
vision_context,
)
.await?,
);
}
if !fetched.videos.is_empty() {
prepared
@@ -1044,6 +1088,100 @@ mod tests {
);
}
#[test]
fn vision_preprocessing_context_uses_runtime_budget_and_text_tokens() {
let info = qwen3_vl_info();
let context = info.vision_preprocessing_context(
&[11, QWEN3_IMAGE_PAD_ID, 12, QWEN3_VIDEO_PAD_ID, 13],
Some(4096),
);
assert_eq!(context.token_budget, Some(4096 - 3));
assert_eq!(
info.vision_preprocessing_context(&[11, 12, 13], Some(2)).token_budget,
Some(0)
);
assert_eq!(
info.vision_preprocessing_context(&[11], None).token_budget,
None
);
}
#[tokio::test]
async fn nemotron_budget_controls_preprocessing_and_engine_features() {
use vllm_engine_core_client::protocol::multimodal::MmKwargValue;
let tokenizer = TestTokenizer::new()
.with_regular_token("<image>", 1018)
.with_regular_token("<img>", 1019)
.with_regular_token("</img>", 1020);
let config = serde_json::json!({
"model_type": "nemotron_h_omni",
"img_context_token_id": 1018,
"max_position_embeddings": 512,
"patch_size": 16,
"downsample_ratio": 0.5,
"norm_mean": [0.0, 0.0, 0.0],
"norm_std": [1.0, 1.0, 1.0],
"vision_config": {
"args": {"min_num_patches": 1024, "max_num_patches": 13312}
}
});
let info = test_info("nemotron_h_omni", config, tokenizer);
let media = MediaContentPart::ImageUrl {
url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=".to_string(),
detail: None,
uuid: None,
};
let mut prompt = vec![11; 100];
prompt.push(1018);
let mut tokens = prompt.clone();
let features = info
.prepare_multimodal(
vec![media.clone()],
&mut tokens,
ModelDtype::Float32,
Some(512),
)
.await
.unwrap();
assert_eq!(features.len(), 1);
let feature = &features[0];
assert_eq!(
(feature.mm_position.offset, feature.mm_position.length),
(100, 258)
);
assert_eq!(tokens.len(), 358);
assert_eq!(tokens[100], 1019);
assert!(tokens[101..357].iter().all(|&token| token == 1018));
assert_eq!(tokens[357], 1020);
let data = feature.data.as_ref().unwrap();
assert!(matches!(
data["pixel_values_flat"].data.as_ref(),
Some(MmKwargValue::Tensor(tensor)) if tensor.shape == [3, 512, 512]
));
assert_eq!(
data["num_tokens_per_image"].data,
Some(MmKwargValue::Int(256))
);
assert_eq!(
data["imgs_sizes"].data,
Some(MmKwargValue::List(vec![
MmKwargValue::Int(512),
MmKwargValue::Int(512),
]))
);
let error = info
.prepare_multimodal(vec![media], &mut prompt, ModelDtype::Float32, Some(100))
.await
.unwrap_err();
assert!(
matches!(error, Error::Multimodal(message) if message.contains("exceeding token_budget"))
);
}
#[test]
fn qwen3_vl_without_video_token_id_disables_video_support_only() {
let config = serde_json::json!({
+1 -1
View File
@@ -259,7 +259,7 @@ mod tests {
uuid: Some("audio-1".to_string()),
}];
async {
info.prepare_multimodal(parts, &mut vec![AUDIO_PAD_ID], ModelDtype::Float32)
info.prepare_multimodal(parts, &mut vec![AUDIO_PAD_ID], ModelDtype::Float32, None)
.instrument(mm_request_span("request-1"))
.await
.unwrap();
+9 -5
View File
@@ -6,7 +6,7 @@
use std::sync::Arc;
use llm_multimodal::{ImageFrame, Modality, PreprocessedEncoderInputs};
use llm_multimodal::{ImageFrame, Modality, PreprocessedEncoderInputs, VisionPreprocessingContext};
use vllm_engine_core_client::protocol::dtype::ModelDtype;
use super::timing::MM_STAGE_TARGET;
@@ -30,11 +30,12 @@ impl MultimodalModelInfo {
frames: Vec<Arc<ImageFrame>>,
uuids: Vec<Option<String>>,
model_dtype: ModelDtype,
preprocessing_context: VisionPreprocessingContext,
) -> Result<PreparedMedia> {
let support = self.image.as_ref().ok_or_else(|| Error::UnsupportedModality {
modality: Modality::Image.to_string(),
})?;
let preprocessed = self.preprocess_images(support, &frames).await?;
let preprocessed = self.preprocess_images(support, &frames, preprocessing_context).await?;
let replacements = support.spec.prompt_replacements_for(&self.context, &preprocessed)?;
if replacements.len() != frames.len() {
bail_multimodal!(
@@ -65,13 +66,16 @@ impl MultimodalModelInfo {
&self,
support: &VisionModalitySupport,
image_frames: &[Arc<ImageFrame>],
preprocessing_context: VisionPreprocessingContext,
) -> Result<PreprocessedEncoderInputs> {
let processor = Arc::clone(&support.processor);
let images = image_frames.iter().map(|frame| frame.data().clone()).collect::<Vec<_>>();
// TODO: is it still necessary given that we've already in a dedicated runtime?
tokio::task::spawn_blocking(move || Ok(processor.preprocess(&images)?))
.await
.map_err(|error| multimodal!("image preprocessing task failed: {error}"))?
tokio::task::spawn_blocking(move || {
Ok(processor.preprocess_with_context(&images, &preprocessing_context)?)
})
.await
.map_err(|error| multimodal!("image preprocessing task failed: {error}"))?
}
}
+47 -2
View File
@@ -18,7 +18,7 @@ use crate::error::{Error, Result, bail_multimodal, multimodal};
pub(super) enum KwargValue {
/// Tensor with row-major flat data and shape.
Tensor { wire: WireTensor },
/// Non-tensor kwarg value that is shared or copied as-is.
/// Non-tensor kwarg value preserved in protocol form.
Passthrough(ProtocolKwargValue),
}
@@ -150,7 +150,17 @@ impl KwargValue {
/// Batched fields use their first axis as media-item index and drop that
/// axis in the per-feature value, matching vLLM's batched-field semantics.
pub(super) fn batched_wire_value_at(&self, index: usize) -> Result<ProtocolKwargValue> {
self.wire_value_range(index, index + 1, true)
match self {
Self::Passthrough(ProtocolKwargValue::List(values)) => values
.get(index)
.cloned()
.ok_or_else(|| multimodal!("batched passthrough has no item at index {index}")),
Self::Passthrough(value) if index == 0 => Ok(value.clone()),
Self::Passthrough(_) => Err(multimodal!(
"scalar batched passthrough has no item at index {index}"
)),
Self::Tensor { .. } => self.wire_value_range(index, index + 1, true),
}
}
/// Convert one media item's flat tensor range directly to wire bytes.
@@ -336,6 +346,41 @@ mod tests {
);
}
#[test]
fn batched_passthrough_value_at_selects_media_item() {
let image_sizes = KwargValue::from_model_specific(
ModelSpecificValue::TupleVec(vec![(384, 736), (512, 512)]),
ModelDtype::BFloat16,
)
.unwrap();
assert_eq!(
image_sizes.batched_wire_value_at(1).unwrap(),
ProtocolKwargValue::List(vec![
ProtocolKwargValue::Int(512),
ProtocolKwargValue::Int(512),
])
);
assert!(image_sizes.batched_wire_value_at(2).is_err());
let token_counts = KwargValue::from_model_specific(
ModelSpecificValue::IntVec(vec![276, 400]),
ModelDtype::BFloat16,
)
.unwrap();
assert_eq!(
token_counts.batched_wire_value_at(1).unwrap(),
ProtocolKwargValue::Int(400)
);
let scalar = KwargValue::Passthrough(ProtocolKwargValue::Int(7));
assert_eq!(
scalar.batched_wire_value_at(0).unwrap(),
ProtocolKwargValue::Int(7)
);
assert!(scalar.batched_wire_value_at(1).is_err());
}
#[test]
fn flat_wire_value_range_keeps_first_axis() {
let value = KwargValue::Tensor {
+28 -4
View File
@@ -84,10 +84,11 @@ impl HfSpecialTokens {
/// Minimal subset of `config.json` (the model's main HF config).
///
/// This intentionally supports only the two layouts we currently care about in
/// This intentionally supports only the layouts we currently care about in
/// the Rust frontend:
/// - pure text models that keep text metadata at the top level
/// - composite models that expose a single nested `text_config`
/// - composite models that expose a single nested `text_config` or its
/// `llm_config` alias (used by Nemotron)
///
/// We do not support additional entry points such as `decoder`, `generator`, or
/// `text_encoder`.
@@ -102,6 +103,7 @@ pub struct ModelConfig {
n_routed_experts: Option<OneOrManyExpertCount>,
num_local_experts: Option<OneOrManyExpertCount>,
block_configs: Vec<BlockConfig>,
#[serde(alias = "llm_config")]
text_config: Option<Box<ModelConfig>>,
}
@@ -188,7 +190,7 @@ impl ModelConfig {
/// Return the config that the Rust frontend treats as the text/LLM config.
///
/// This is deliberately narrower than Python/transformers: we only support
/// either the top-level config itself or a single nested `text_config`.
/// either the top-level config itself or a single nested text config.
fn effective_text_config(&self) -> &Self {
self.text_config.as_deref().unwrap_or(self)
}
@@ -233,7 +235,8 @@ impl ModelConfig {
/// config.
///
/// The only intentional simplification here is how we pick the text config:
/// Rust only looks at the top level or `text_config`, not the broader
/// Rust only looks at the top level or `text_config` (including its
/// `llm_config` alias), not the broader
/// transformers composite-config surface.
fn num_experts_from_block_configs(&self) -> u32 {
self.effective_text_config()
@@ -405,6 +408,27 @@ mod tests {
assert_eq!(config.vocab_size().unwrap(), 151936);
}
#[test]
fn model_config_uses_llm_config_for_nemotron_composite_models() {
let config: ModelConfig = serde_json::from_str(
r#"{
"model_type": "nemotron_h_omni",
"llm_config": {
"model_type": "nemotron_h",
"vocab_size": 131072,
"n_routed_experts": 256,
"eos_token_id": 2
}
}"#,
)
.unwrap();
assert_eq!(config.model_type(), Some("nemotron_h_omni"));
assert_eq!(config.vocab_size().unwrap(), 131072);
assert_eq!(config.eos_token_ids(), &[2]);
assert_eq!(config.num_experts(), 256);
}
#[test]
fn model_config_reads_top_level_eos_token_ids() {
let single: ModelConfig = serde_json::from_str(r#"{"eos_token_id":151645}"#).unwrap();