[Rust Frontend] Pass vision preprocessing context for Nemotron-H (#57634)

Pass the remaining engine context-length budget to model-owned vision processors through VisionPreprocessingContext. Preserve Nemotron batched engine fields and recognize llm_config as a text_config alias.

Use the merged upstream llm-multimodal revision f0985ef65967615db2c79279aa07818499301bfd.

Co-authored-by: Codex <noreply@openai.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Biswa Panda
2026-09-24 14:08:26 +08:00
committed by GitHub
co-authored by Codex
parent ee2f3cdf12
commit cf78797cb7
8 changed files with 240 additions and 21 deletions
+28 -4
View File
@@ -84,10 +84,11 @@ impl HfSpecialTokens {
/// Minimal subset of `config.json` (the model's main HF config).
///
/// This intentionally supports only the two layouts we currently care about in
/// This intentionally supports only the layouts we currently care about in
/// the Rust frontend:
/// - pure text models that keep text metadata at the top level
/// - composite models that expose a single nested `text_config`
/// - composite models that expose a single nested `text_config` or its
/// `llm_config` alias (used by Nemotron)
///
/// We do not support additional entry points such as `decoder`, `generator`, or
/// `text_encoder`.
@@ -102,6 +103,7 @@ pub struct ModelConfig {
n_routed_experts: Option<OneOrManyExpertCount>,
num_local_experts: Option<OneOrManyExpertCount>,
block_configs: Vec<BlockConfig>,
#[serde(alias = "llm_config")]
text_config: Option<Box<ModelConfig>>,
}
@@ -188,7 +190,7 @@ impl ModelConfig {
/// Return the config that the Rust frontend treats as the text/LLM config.
///
/// This is deliberately narrower than Python/transformers: we only support
/// either the top-level config itself or a single nested `text_config`.
/// either the top-level config itself or a single nested text config.
fn effective_text_config(&self) -> &Self {
self.text_config.as_deref().unwrap_or(self)
}
@@ -233,7 +235,8 @@ impl ModelConfig {
/// config.
///
/// The only intentional simplification here is how we pick the text config:
/// Rust only looks at the top level or `text_config`, not the broader
/// Rust only looks at the top level or `text_config` (including its
/// `llm_config` alias), not the broader
/// transformers composite-config surface.
fn num_experts_from_block_configs(&self) -> u32 {
self.effective_text_config()
@@ -405,6 +408,27 @@ mod tests {
assert_eq!(config.vocab_size().unwrap(), 151936);
}
#[test]
fn model_config_uses_llm_config_for_nemotron_composite_models() {
let config: ModelConfig = serde_json::from_str(
r#"{
"model_type": "nemotron_h_omni",
"llm_config": {
"model_type": "nemotron_h",
"vocab_size": 131072,
"n_routed_experts": 256,
"eos_token_id": 2
}
}"#,
)
.unwrap();
assert_eq!(config.model_type(), Some("nemotron_h_omni"));
assert_eq!(config.vocab_size().unwrap(), 131072);
assert_eq!(config.eos_token_ids(), &[2]);
assert_eq!(config.num_experts(), 256);
}
#[test]
fn model_config_reads_top_level_eos_token_ids() {
let single: ModelConfig = serde_json::from_str(r#"{"eos_token_id":151645}"#).unwrap();