diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md
index d55348ad72..ddc50ad41c 100644
--- a/docs/models/supported_models.md
+++ b/docs/models/supported_models.md
@@ -602,6 +602,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
| `MossTranscribeDiarizeForConditionalGeneration` | MOSS-Transcribe-Diarize | T + A | `OpenMOSS-Team/MOSS-Transcribe-Diarize` | | ✅︎ |
| `Moondream3ForCausalLM` | Moondream3 | T + I | `moondream/moondream3-preview` | | ✅︎ |
| `MuseGlimmerForCausalLM`, `MuseGlimmerForConditionalGeneration` | Muse Glimmer | T + I+ + V+ | `meta-models/Muse-Glimmer-30B` | ✅︎ | ✅︎ |
+| `NemotronH_Nano_Omni_Reasoning_V3`, `NemotronH_Nano_VL_V2` | Nemotron Nano 3 Omni, Nemotron Nano V2 VL | T + IE+ + V+ + A* | `nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16` | ✅︎ | |
| `NVLM_D_Model` | NVLM-D 1.0 | T + I+ | `nvidia/NVLM-D-72B`, etc. | | ✅︎ |
| `OpenCUAForConditionalGeneration` | OpenCUA-7B | T + IE+ | `xlangai/OpenCUA-7B` | ✅︎ | ✅︎ |
| `OpenPanguVLForConditionalGeneration` | openpangu-VL | T + IE+ + VE+ | `FreedomIntelligence/openPangu-VL-7B` | ✅︎ | ✅︎ |
@@ -701,6 +702,12 @@ Some models are supported only via the [Transformers modeling backend](#transfor
`meta-models/Muse-Glimmer-30B-assistant` checkpoint, which vLLM serves through
its [DFlash](../features/speculative_decoding/README.md) path.
+!!! note
+ Both Nemotron Nano architecture names map to the same vLLM implementation.
+ Audio inputs are only supported by the Omni variants; `NemotronH_Nano_VL_V2`
+ checkpoints such as `nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16` accept text,
+ image and video only.
+
!!! note
The official `openbmb/MiniCPM-V-2` doesn't work yet, so we need to use a fork (`HwwwH/MiniCPM-V-2`) for now.
For more details, please see:
diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py
index 2edf56871d..6f5e073bd6 100644
--- a/vllm/model_executor/models/nano_nemotron_vl.py
+++ b/vllm/model_executor/models/nano_nemotron_vl.py
@@ -23,6 +23,7 @@ from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
from vllm.inputs import MultiModalDataDict, MultiModalInput
from vllm.logger import init_logger
+from vllm.lora.layers.base import BaseLayerWithLoRA
from vllm.model_executor.layers.activation import ReLUSquaredActivation
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -30,6 +31,7 @@ from vllm.model_executor.models.interfaces import (
HasInnerState,
IsHybrid,
MultiModalEmbeddings,
+ SupportsLoRA,
SupportsMultiModal,
SupportsMultiModalPruning,
)
@@ -897,11 +899,22 @@ class NanoNemotronVLDummyInputsBuilder(
dummy_inputs=NanoNemotronVLDummyInputsBuilder,
)
class NemotronH_Nano_VL_V2(
- nn.Module, HasInnerState, IsHybrid, SupportsMultiModal, SupportsMultiModalPruning
+ nn.Module,
+ HasInnerState,
+ IsHybrid,
+ SupportsMultiModal,
+ SupportsMultiModalPruning,
+ SupportsLoRA,
):
requires_sequential_video_encoding = True
"""Temporarily needed for dynamic res video w/ conv3d, doesn't support bs>1 yet"""
+ # LoRA covers the language model only
+ is_non_gated_moe = NemotronHForCausalLM.is_non_gated_moe
+ packed_modules_mapping = NemotronHForCausalLM.packed_modules_mapping
+ embedding_modules = NemotronHForCausalLM.embedding_modules
+ lora_skip_prefixes = NemotronHForCausalLM.lora_skip_prefixes
+
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={
"language_model.backbone": "language_model.model",
@@ -1331,7 +1344,19 @@ class NemotronH_Nano_VL_V2(
# Create final video embeddings, merging text embeddings for indicator
# tokens with video embeddings
- text_embeddings = self.get_language_model().embed_input_ids(repl_token_ids)
+
+ # LoRA support -
+ # These replacement tokens are produced inside the encoder, so they are
+ # absent from the request-token batch that the LoRA adapter index
+ # mapping is built from, and there can be more
+ # of them than max_num_batched_tokens.
+ # Embed them with the base weights -
+ # the LoRA delta is undefined for tokens with no mapping entry.
+ embed_tokens = self.get_language_model().model.embed_tokens
+ if isinstance(embed_tokens, BaseLayerWithLoRA):
+ embed_tokens = embed_tokens.base_layer
+ text_embeddings = embed_tokens(repl_token_ids)
+
final_video_embeddings = _merge_multimodal_embeddings(
inputs_embeds=text_embeddings,
multimodal_embeddings=video_embeddings,