[Rust Frontend] Pass vision preprocessing context for Nemotron-H (#57634)
Pass the remaining engine context-length budget to model-owned vision processors through VisionPreprocessingContext. Preserve Nemotron batched engine fields and recognize llm_config as a text_config alias. Use the merged upstream llm-multimodal revision f0985ef65967615db2c79279aa07818499301bfd. Co-authored-by: Codex <noreply@openai.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -84,10 +84,11 @@ impl HfSpecialTokens {
|
||||
|
||||
/// Minimal subset of `config.json` (the model's main HF config).
|
||||
///
|
||||
/// This intentionally supports only the two layouts we currently care about in
|
||||
/// This intentionally supports only the layouts we currently care about in
|
||||
/// the Rust frontend:
|
||||
/// - pure text models that keep text metadata at the top level
|
||||
/// - composite models that expose a single nested `text_config`
|
||||
/// - composite models that expose a single nested `text_config` or its
|
||||
/// `llm_config` alias (used by Nemotron)
|
||||
///
|
||||
/// We do not support additional entry points such as `decoder`, `generator`, or
|
||||
/// `text_encoder`.
|
||||
@@ -102,6 +103,7 @@ pub struct ModelConfig {
|
||||
n_routed_experts: Option<OneOrManyExpertCount>,
|
||||
num_local_experts: Option<OneOrManyExpertCount>,
|
||||
block_configs: Vec<BlockConfig>,
|
||||
#[serde(alias = "llm_config")]
|
||||
text_config: Option<Box<ModelConfig>>,
|
||||
}
|
||||
|
||||
@@ -188,7 +190,7 @@ impl ModelConfig {
|
||||
/// Return the config that the Rust frontend treats as the text/LLM config.
|
||||
///
|
||||
/// This is deliberately narrower than Python/transformers: we only support
|
||||
/// either the top-level config itself or a single nested `text_config`.
|
||||
/// either the top-level config itself or a single nested text config.
|
||||
fn effective_text_config(&self) -> &Self {
|
||||
self.text_config.as_deref().unwrap_or(self)
|
||||
}
|
||||
@@ -233,7 +235,8 @@ impl ModelConfig {
|
||||
/// config.
|
||||
///
|
||||
/// The only intentional simplification here is how we pick the text config:
|
||||
/// Rust only looks at the top level or `text_config`, not the broader
|
||||
/// Rust only looks at the top level or `text_config` (including its
|
||||
/// `llm_config` alias), not the broader
|
||||
/// transformers composite-config surface.
|
||||
fn num_experts_from_block_configs(&self) -> u32 {
|
||||
self.effective_text_config()
|
||||
@@ -405,6 +408,27 @@ mod tests {
|
||||
assert_eq!(config.vocab_size().unwrap(), 151936);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn model_config_uses_llm_config_for_nemotron_composite_models() {
|
||||
let config: ModelConfig = serde_json::from_str(
|
||||
r#"{
|
||||
"model_type": "nemotron_h_omni",
|
||||
"llm_config": {
|
||||
"model_type": "nemotron_h",
|
||||
"vocab_size": 131072,
|
||||
"n_routed_experts": 256,
|
||||
"eos_token_id": 2
|
||||
}
|
||||
}"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(config.model_type(), Some("nemotron_h_omni"));
|
||||
assert_eq!(config.vocab_size().unwrap(), 131072);
|
||||
assert_eq!(config.eos_token_ids(), &[2]);
|
||||
assert_eq!(config.num_experts(), 256);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn model_config_reads_top_level_eos_token_ids() {
|
||||
let single: ModelConfig = serde_json::from_str(r#"{"eos_token_id":151645}"#).unwrap();
|
||||
|
||||
Reference in New Issue
Block a user