diff --git a/tests/models/registry.py b/tests/models/registry.py index 361253d13a..118336588d 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -7,6 +7,7 @@ from typing import Any, Literal import pytest from packaging.version import Version +from transformers import PretrainedConfig from transformers import __version__ as TRANSFORMERS_VERSION from vllm.config.model import ModelDType, TokenizerMode @@ -1201,6 +1202,35 @@ _MULTIMODAL_EXAMPLE_MODELS = { }, trust_remote_code=True, ), + "NemotronH_Nano_Omni_Reasoning_V3": _HfExamplesInfo( + "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16", + max_model_len=4096, + # NemotronH layers are constructed via `hybrid_override_pattern` + use_original_num_layers=True, + hf_overrides={ + "vision_config": PretrainedConfig( + args={ + "min_num_patches": 1, + "max_num_patches": 12, + "model": "vit_huge_patch16_224", + }, + video_temporal_patch_size=2, + # TODO(nhaber): This is `true` in the official `config.json`, + # but this causes a processor exception in the tests due to a known bug + # with mixed-resolution video when `true`. To be resolved. + video_maintain_aspect_ratio=False, + ), + "text_config": {"num_hidden_layers": 2, "hybrid_override_pattern": "M*"}, + }, + trust_remote_code=True, + ), + "NemotronH_Super_Omni_Reasoning_V3": _HfExamplesInfo( + "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16", is_available_online=False + ), + # TODO: Change repo id once pertinent archs are public. + "NemotronH_Omni_Reasoning_V3": _HfExamplesInfo( + "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16", is_available_online=False + ), "OpenCUAForConditionalGeneration": _HfExamplesInfo( "xlangai/OpenCUA-7B", trust_remote_code=True, diff --git a/vllm/config/speculative.py b/vllm/config/speculative.py index 49823bc031..78940a79b9 100644 --- a/vllm/config/speculative.py +++ b/vllm/config/speculative.py @@ -797,6 +797,13 @@ class SpeculativeConfig: {"n_predict": n_predict, "architectures": ["ErnieMTPModel"]} ) + if hf_config.architectures[0] in ( + "NemotronH_Super_Omni_Reasoning_V3", + "NemotronH_Omni_Reasoning_V3", + ): + # Promote VLM's text_config so MTP detection below fires correctly + hf_config = hf_config.text_config + if ( hf_config.model_type in {"nemotron_h", "nemotron_h_puzzle"} and hasattr(hf_config, "num_nextn_predict_layers") diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index 2f030ec1b7..2b4f781de1 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -520,6 +520,9 @@ _MULTIMODAL_MODELS = { "MossAudioModel": ("moss_audio", "MossAudioModel"), "HfMoondream": ("moondream3", "Moondream3ForCausalLM"), "NemotronH_Nano_VL_V2": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"), + "NemotronH_Nano_Omni_Reasoning_V3": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"), + "NemotronH_Super_Omni_Reasoning_V3": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"), + "NemotronH_Omni_Reasoning_V3": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"), "NVLM_D": ("nvlm_d", "NVLM_D_Model"), "MuseGlimmerForConditionalGeneration": ("muse_glimmer", "MuseGlimmerForCausalLM"), "OpenCUAForConditionalGeneration": ("opencua", "OpenCUAForConditionalGeneration"),