Fix Nano Nemotron text-only weight loading (#41205)

Signed-off-by: sunghoon.baek <sunghoon.baek@connectfy.cloud>
Signed-off-by: Baekpica <35071468+Baekpica@users.noreply.github.com>
Signed-off-by: sunghoon.baek <seanbb93@gmail.com>
Co-authored-by: sunghoon.baek <sunghoon.baek@connectfy.cloud>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Netanel Haber <58652339+netanel-haber@users.noreply.github.com>
This commit is contained in:
Baekpica
2026-05-04 21:43:07 +03:00
committed by GitHub
co-authored by sunghoon.baek OpenAI Codex Netanel Haber
parent e724b0ea8d
commit 8c780943b4
2 changed files with 129 additions and 3 deletions
@@ -0,0 +1,114 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.model_executor.models.nano_nemotron_vl import NemotronH_Nano_VL_V2
class _TextOnlyMultiModalConfig:
def get_limit_per_prompt(self, modality: str) -> int:
return 0
class _ImageOnlyMultiModalConfig:
def get_limit_per_prompt(self, modality: str) -> int:
return 1 if modality == "image" else 0
class _ModelConfig:
multimodal_config = _TextOnlyMultiModalConfig()
class _ImageOnlyModelConfig:
multimodal_config = _ImageOnlyMultiModalConfig()
class _LanguageModel:
def __init__(self) -> None:
self.loaded_weights: list[tuple[str, object]] = []
def load_weights(self, weights):
self.loaded_weights = list(weights)
class _MissingMultiModalModule:
def named_parameters(self):
raise AssertionError("multimodal weights should not be inspected")
def load_weights(self, weights):
raise AssertionError("multimodal weights should not be loaded")
class _AdapterModule:
def named_parameters(self):
return []
class _VisionModel:
def __init__(self) -> None:
self.loaded_weights: list[tuple[str, object]] = []
def load_weights(self, weights):
self.loaded_weights = list(weights)
def test_nano_nemotron_vl_skips_multimodal_weights_in_text_only_mode():
model = object.__new__(NemotronH_Nano_VL_V2)
language_model = _LanguageModel()
object.__setattr__(model, "model_config", _ModelConfig())
object.__setattr__(model, "language_model", language_model)
object.__setattr__(model, "mlp1", _AdapterModule())
object.__setattr__(model, "vision_model", _MissingMultiModalModule())
object.__setattr__(model, "sound_encoder", None)
language_weight = object()
model.load_weights(
[
("language_model.layers.0.weight", language_weight),
("mlp1.0.weight", object()),
("vision_model.radio_model.encoder.weight", object()),
("sound_encoder.encoder.weight", object()),
]
)
assert language_model.loaded_weights == [("layers.0.weight", language_weight)]
def test_nano_nemotron_vl_loads_vision_weights_without_sound_encoder():
model = object.__new__(NemotronH_Nano_VL_V2)
language_model = _LanguageModel()
vision_model = _VisionModel()
object.__setattr__(model, "model_config", _ImageOnlyModelConfig())
object.__setattr__(model, "language_model", language_model)
object.__setattr__(model, "mlp1", _AdapterModule())
object.__setattr__(model, "vision_model", vision_model)
object.__setattr__(model, "sound_encoder", None)
language_weight = object()
vision_weight = object()
model.load_weights(
[
("language_model.layers.0.weight", language_weight),
("vision_model.radio_model.encoder.weight", vision_weight),
]
)
assert language_model.loaded_weights == [("layers.0.weight", language_weight)]
assert vision_model.loaded_weights == [
("radio_model.encoder.weight", vision_weight)
]
def test_nano_nemotron_vl_requires_sound_encoder_for_sound_weights():
model = object.__new__(NemotronH_Nano_VL_V2)
language_model = _LanguageModel()
vision_model = _VisionModel()
object.__setattr__(model, "model_config", _ImageOnlyModelConfig())
object.__setattr__(model, "language_model", language_model)
object.__setattr__(model, "mlp1", _AdapterModule())
object.__setattr__(model, "vision_model", vision_model)
object.__setattr__(model, "sound_encoder", None)
with pytest.raises(AssertionError):
model.load_weights([("sound_encoder.encoder.weight", object())])