[LoRA][Nemotron] Add LoRA support for Nemotron VL models (for the language model only) (#56231)

Signed-off-by: Daniel Serebrenik <daserebrenik@nvidia.com>

docs/models/supported_models.md

@@ -602,6 +602,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
 | `MossTranscribeDiarizeForConditionalGeneration` | MOSS-Transcribe-Diarize | T + A | `OpenMOSS-Team/MOSS-Transcribe-Diarize` | | ✅︎ |
 | `Moondream3ForCausalLM` | Moondream3 | T + I | `moondream/moondream3-preview` | | ✅︎ |
 | `MuseGlimmerForCausalLM`, `MuseGlimmerForConditionalGeneration` | Muse Glimmer | T + I<sup>+</sup> + V<sup>+</sup> | `meta-models/Muse-Glimmer-30B` | ✅︎ | ✅︎ |
+| `NemotronH_Nano_Omni_Reasoning_V3`, `NemotronH_Nano_VL_V2` | Nemotron Nano 3 Omni, Nemotron Nano V2 VL | T + I<sup>E+</sup> + V<sup>+</sup> + A<sup>*</sup> | `nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16` | ✅︎ | |
 | `NVLM_D_Model` | NVLM-D 1.0 | T + I<sup>+</sup> | `nvidia/NVLM-D-72B`, etc. | | ✅︎ |
 | `OpenCUAForConditionalGeneration` | OpenCUA-7B | T + I<sup>E+</sup> | `xlangai/OpenCUA-7B` | ✅︎ | ✅︎ |
 | `OpenPanguVLForConditionalGeneration` | openpangu-VL | T + I<sup>E+</sup> + V<sup>E+</sup> | `FreedomIntelligence/openPangu-VL-7B` | ✅︎ | ✅︎ |
@@ -701,6 +702,12 @@ Some models are supported only via the [Transformers modeling backend](#transfor
     `meta-models/Muse-Glimmer-30B-assistant` checkpoint, which vLLM serves through
     its [DFlash](../features/speculative_decoding/README.md) path.
 
+!!! note
+    Both Nemotron Nano architecture names map to the same vLLM implementation.
+    Audio inputs are only supported by the Omni variants; `NemotronH_Nano_VL_V2`
+    checkpoints such as `nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16` accept text,
+    image and video only.
+
 !!! note
     The official `openbmb/MiniCPM-V-2` doesn't work yet, so we need to use a fork (`HwwwH/MiniCPM-V-2`) for now.
     For more details, please see: <https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630>

vllm/model_executor/models/nano_nemotron_vl.py

@@ -23,6 +23,7 @@ from vllm.config import VllmConfig
 from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
 from vllm.inputs import MultiModalDataDict, MultiModalInput
 from vllm.logger import init_logger
+from vllm.lora.layers.base import BaseLayerWithLoRA
 from vllm.model_executor.layers.activation import ReLUSquaredActivation
 from vllm.model_executor.layers.layernorm import RMSNorm
 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -30,6 +31,7 @@ from vllm.model_executor.models.interfaces import (
     HasInnerState,
     IsHybrid,
     MultiModalEmbeddings,
+    SupportsLoRA,
     SupportsMultiModal,
     SupportsMultiModalPruning,
 )
@@ -897,11 +899,22 @@ class NanoNemotronVLDummyInputsBuilder(
     dummy_inputs=NanoNemotronVLDummyInputsBuilder,
 )
 class NemotronH_Nano_VL_V2(
-    nn.Module, HasInnerState, IsHybrid, SupportsMultiModal, SupportsMultiModalPruning
+    nn.Module,
+    HasInnerState,
+    IsHybrid,
+    SupportsMultiModal,
+    SupportsMultiModalPruning,
+    SupportsLoRA,
 ):
     requires_sequential_video_encoding = True
     """Temporarily needed for dynamic res video w/ conv3d, doesn't support bs>1 yet"""
 
+    # LoRA covers the language model only
+    is_non_gated_moe = NemotronHForCausalLM.is_non_gated_moe
+    packed_modules_mapping = NemotronHForCausalLM.packed_modules_mapping
+    embedding_modules = NemotronHForCausalLM.embedding_modules
+    lora_skip_prefixes = NemotronHForCausalLM.lora_skip_prefixes
+
     hf_to_vllm_mapper = WeightsMapper(
         orig_to_new_prefix={
             "language_model.backbone": "language_model.model",
@@ -1331,7 +1344,19 @@ class NemotronH_Nano_VL_V2(
 
         # Create final video embeddings, merging text embeddings for indicator
         # tokens with video embeddings
-        text_embeddings = self.get_language_model().embed_input_ids(repl_token_ids)
+
+        # LoRA support -
+        # These replacement tokens are produced inside the encoder, so they are
+        # absent from the request-token batch that the LoRA adapter index
+        # mapping is built from, and there can be more
+        # of them than max_num_batched_tokens.
+        # Embed them with the base weights -
+        # the LoRA delta is undefined for tokens with no mapping entry.
+        embed_tokens = self.get_language_model().model.embed_tokens
+        if isinstance(embed_tokens, BaseLayerWithLoRA):
+            embed_tokens = embed_tokens.base_layer
+        text_embeddings = embed_tokens(repl_token_ids)
+
         final_video_embeddings = _merge_multimodal_embeddings(
             inputs_embeds=text_embeddings,
             multimodal_embeddings=video_embeddings,