diff --git a/tests/models/registry.py b/tests/models/registry.py index 92ebac0184..9c15decd8f 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -1038,6 +1038,50 @@ _MULTIMODAL_EXAMPLE_MODELS = { }, trust_remote_code=True, ), + # NemotronH_Nano_Omni_Reasoning_V3 is an alias for NemotronH_Nano_VL_V2 + # Use the same registry test as NemotronH_Nano_VL_V2 above + "NemotronH_Nano_Omni_Reasoning_V3": _HfExamplesInfo( + "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16", + max_model_len=4096, + use_original_num_layers=True, + hf_overrides={ + "vision_config": PretrainedConfig( + args={ + "min_num_patches": 1, + "max_num_patches": 12, + "model": "vit_huge_patch16_224", + }, + video_temporal_patch_size=2, + ), + "text_config": { + "num_hidden_layers": 2, + "hybrid_override_pattern": "M*", + }, + }, + trust_remote_code=True, + ), + # NemotronH_Super_Omni_Reasoning_V3 is an alias for NemotronH_Nano_VL_V2 as well + # Use the same registry test as NemotronH_Nano_VL_V2 above + "NemotronH_Super_Omni_Reasoning_V3": _HfExamplesInfo( + "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16", + max_model_len=4096, + use_original_num_layers=True, + hf_overrides={ + "vision_config": PretrainedConfig( + args={ + "min_num_patches": 1, + "max_num_patches": 12, + "model": "vit_huge_patch16_224", + }, + video_temporal_patch_size=2, + ), + "text_config": { + "num_hidden_layers": 2, + "hybrid_override_pattern": "M*", + }, + }, + trust_remote_code=True, + ), "OpenCUAForConditionalGeneration": _HfExamplesInfo( "xlangai/OpenCUA-7B", trust_remote_code=True ), diff --git a/vllm/config/speculative.py b/vllm/config/speculative.py index 61f311c842..bbe923f68f 100644 --- a/vllm/config/speculative.py +++ b/vllm/config/speculative.py @@ -300,6 +300,10 @@ class SpeculativeConfig: {"n_predict": n_predict, "architectures": ["ErnieMTPModel"]} ) + if hf_config.architectures[0] == "NemotronH_Super_Omni_Reasoning_V3": + # Promote VLM's text_config so MTP detection below fires correctly + hf_config = hf_config.text_config + if ( hf_config.model_type in {"nemotron_h", "nemotron_h_puzzle"} and hasattr(hf_config, "num_nextn_predict_layers") diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index ef89e159fe..01a767b9f3 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -473,6 +473,8 @@ _MULTIMODAL_MODELS = { "MolmoForCausalLM": ("molmo", "MolmoForCausalLM"), "Molmo2ForConditionalGeneration": ("molmo2", "Molmo2ForConditionalGeneration"), "NemotronH_Nano_VL_V2": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"), + "NemotronH_Nano_Omni_Reasoning_V3": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"), + "NemotronH_Super_Omni_Reasoning_V3": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"), "NVLM_D": ("nvlm_d", "NVLM_D_Model"), "OpenCUAForConditionalGeneration": ("opencua", "OpenCUAForConditionalGeneration"), "OpenPanguVLForConditionalGeneration": (