From a31ec3a68bbbedd4b5d59490c762bcf32abf1a17 Mon Sep 17 00:00:00 2001 From: danisereb Date: Wed, 16 Sep 2026 07:20:28 +0300 Subject: [PATCH] [LoRA][Nemotron] Add LoRA support for Nemotron VL models (for the language model only) (#56231) Signed-off-by: Daniel Serebrenik --- docs/models/supported_models.md | 7 +++++ .../model_executor/models/nano_nemotron_vl.py | 29 +++++++++++++++++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index d55348ad72..ddc50ad41c 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -602,6 +602,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen | `MossTranscribeDiarizeForConditionalGeneration` | MOSS-Transcribe-Diarize | T + A | `OpenMOSS-Team/MOSS-Transcribe-Diarize` | | ✅︎ | | `Moondream3ForCausalLM` | Moondream3 | T + I | `moondream/moondream3-preview` | | ✅︎ | | `MuseGlimmerForCausalLM`, `MuseGlimmerForConditionalGeneration` | Muse Glimmer | T + I+ + V+ | `meta-models/Muse-Glimmer-30B` | ✅︎ | ✅︎ | +| `NemotronH_Nano_Omni_Reasoning_V3`, `NemotronH_Nano_VL_V2` | Nemotron Nano 3 Omni, Nemotron Nano V2 VL | T + IE+ + V+ + A* | `nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16` | ✅︎ | | | `NVLM_D_Model` | NVLM-D 1.0 | T + I+ | `nvidia/NVLM-D-72B`, etc. | | ✅︎ | | `OpenCUAForConditionalGeneration` | OpenCUA-7B | T + IE+ | `xlangai/OpenCUA-7B` | ✅︎ | ✅︎ | | `OpenPanguVLForConditionalGeneration` | openpangu-VL | T + IE+ + VE+ | `FreedomIntelligence/openPangu-VL-7B` | ✅︎ | ✅︎ | @@ -701,6 +702,12 @@ Some models are supported only via the [Transformers modeling backend](#transfor `meta-models/Muse-Glimmer-30B-assistant` checkpoint, which vLLM serves through its [DFlash](../features/speculative_decoding/README.md) path. +!!! note + Both Nemotron Nano architecture names map to the same vLLM implementation. + Audio inputs are only supported by the Omni variants; `NemotronH_Nano_VL_V2` + checkpoints such as `nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16` accept text, + image and video only. + !!! note The official `openbmb/MiniCPM-V-2` doesn't work yet, so we need to use a fork (`HwwwH/MiniCPM-V-2`) for now. For more details, please see: diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py index 2edf56871d..6f5e073bd6 100644 --- a/vllm/model_executor/models/nano_nemotron_vl.py +++ b/vllm/model_executor/models/nano_nemotron_vl.py @@ -23,6 +23,7 @@ from vllm.config import VllmConfig from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions from vllm.inputs import MultiModalDataDict, MultiModalInput from vllm.logger import init_logger +from vllm.lora.layers.base import BaseLayerWithLoRA from vllm.model_executor.layers.activation import ReLUSquaredActivation from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.model_loader.weight_utils import default_weight_loader @@ -30,6 +31,7 @@ from vllm.model_executor.models.interfaces import ( HasInnerState, IsHybrid, MultiModalEmbeddings, + SupportsLoRA, SupportsMultiModal, SupportsMultiModalPruning, ) @@ -897,11 +899,22 @@ class NanoNemotronVLDummyInputsBuilder( dummy_inputs=NanoNemotronVLDummyInputsBuilder, ) class NemotronH_Nano_VL_V2( - nn.Module, HasInnerState, IsHybrid, SupportsMultiModal, SupportsMultiModalPruning + nn.Module, + HasInnerState, + IsHybrid, + SupportsMultiModal, + SupportsMultiModalPruning, + SupportsLoRA, ): requires_sequential_video_encoding = True """Temporarily needed for dynamic res video w/ conv3d, doesn't support bs>1 yet""" + # LoRA covers the language model only + is_non_gated_moe = NemotronHForCausalLM.is_non_gated_moe + packed_modules_mapping = NemotronHForCausalLM.packed_modules_mapping + embedding_modules = NemotronHForCausalLM.embedding_modules + lora_skip_prefixes = NemotronHForCausalLM.lora_skip_prefixes + hf_to_vllm_mapper = WeightsMapper( orig_to_new_prefix={ "language_model.backbone": "language_model.model", @@ -1331,7 +1344,19 @@ class NemotronH_Nano_VL_V2( # Create final video embeddings, merging text embeddings for indicator # tokens with video embeddings - text_embeddings = self.get_language_model().embed_input_ids(repl_token_ids) + + # LoRA support - + # These replacement tokens are produced inside the encoder, so they are + # absent from the request-token batch that the LoRA adapter index + # mapping is built from, and there can be more + # of them than max_num_batched_tokens. + # Embed them with the base weights - + # the LoRA delta is undefined for tokens with no mapping entry. + embed_tokens = self.get_language_model().model.embed_tokens + if isinstance(embed_tokens, BaseLayerWithLoRA): + embed_tokens = embed_tokens.base_layer + text_embeddings = embed_tokens(repl_token_ids) + final_video_embeddings = _merge_multimodal_embeddings( inputs_embeds=text_embeddings, multimodal_embeddings=video_embeddings,