diff --git a/tests/models/multimodal/test_nano_nemotron_vl.py b/tests/models/multimodal/test_nano_nemotron_vl.py new file mode 100644 index 0000000000..6922af79c0 --- /dev/null +++ b/tests/models/multimodal/test_nano_nemotron_vl.py @@ -0,0 +1,114 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest + +from vllm.model_executor.models.nano_nemotron_vl import NemotronH_Nano_VL_V2 + + +class _TextOnlyMultiModalConfig: + def get_limit_per_prompt(self, modality: str) -> int: + return 0 + + +class _ImageOnlyMultiModalConfig: + def get_limit_per_prompt(self, modality: str) -> int: + return 1 if modality == "image" else 0 + + +class _ModelConfig: + multimodal_config = _TextOnlyMultiModalConfig() + + +class _ImageOnlyModelConfig: + multimodal_config = _ImageOnlyMultiModalConfig() + + +class _LanguageModel: + def __init__(self) -> None: + self.loaded_weights: list[tuple[str, object]] = [] + + def load_weights(self, weights): + self.loaded_weights = list(weights) + + +class _MissingMultiModalModule: + def named_parameters(self): + raise AssertionError("multimodal weights should not be inspected") + + def load_weights(self, weights): + raise AssertionError("multimodal weights should not be loaded") + + +class _AdapterModule: + def named_parameters(self): + return [] + + +class _VisionModel: + def __init__(self) -> None: + self.loaded_weights: list[tuple[str, object]] = [] + + def load_weights(self, weights): + self.loaded_weights = list(weights) + + +def test_nano_nemotron_vl_skips_multimodal_weights_in_text_only_mode(): + model = object.__new__(NemotronH_Nano_VL_V2) + language_model = _LanguageModel() + object.__setattr__(model, "model_config", _ModelConfig()) + object.__setattr__(model, "language_model", language_model) + object.__setattr__(model, "mlp1", _AdapterModule()) + object.__setattr__(model, "vision_model", _MissingMultiModalModule()) + object.__setattr__(model, "sound_encoder", None) + + language_weight = object() + model.load_weights( + [ + ("language_model.layers.0.weight", language_weight), + ("mlp1.0.weight", object()), + ("vision_model.radio_model.encoder.weight", object()), + ("sound_encoder.encoder.weight", object()), + ] + ) + + assert language_model.loaded_weights == [("layers.0.weight", language_weight)] + + +def test_nano_nemotron_vl_loads_vision_weights_without_sound_encoder(): + model = object.__new__(NemotronH_Nano_VL_V2) + language_model = _LanguageModel() + vision_model = _VisionModel() + object.__setattr__(model, "model_config", _ImageOnlyModelConfig()) + object.__setattr__(model, "language_model", language_model) + object.__setattr__(model, "mlp1", _AdapterModule()) + object.__setattr__(model, "vision_model", vision_model) + object.__setattr__(model, "sound_encoder", None) + + language_weight = object() + vision_weight = object() + model.load_weights( + [ + ("language_model.layers.0.weight", language_weight), + ("vision_model.radio_model.encoder.weight", vision_weight), + ] + ) + + assert language_model.loaded_weights == [("layers.0.weight", language_weight)] + assert vision_model.loaded_weights == [ + ("radio_model.encoder.weight", vision_weight) + ] + + +def test_nano_nemotron_vl_requires_sound_encoder_for_sound_weights(): + model = object.__new__(NemotronH_Nano_VL_V2) + language_model = _LanguageModel() + vision_model = _VisionModel() + object.__setattr__(model, "model_config", _ImageOnlyModelConfig()) + object.__setattr__(model, "language_model", language_model) + object.__setattr__(model, "mlp1", _AdapterModule()) + object.__setattr__(model, "vision_model", vision_model) + object.__setattr__(model, "sound_encoder", None) + + with pytest.raises(AssertionError): + model.load_weights([("sound_encoder.encoder.weight", object())]) diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py index 684ced0a6a..994b52606b 100644 --- a/vllm/model_executor/models/nano_nemotron_vl.py +++ b/vllm/model_executor/models/nano_nemotron_vl.py @@ -1499,6 +1499,11 @@ class NemotronH_Nano_VL_V2( return self.language_model.compute_logits(hidden_states) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): + mm_config = self.model_config.multimodal_config + load_multimodal_weights = not all( + mm_config.get_limit_per_prompt(modality) == 0 + for modality in ("image", "video", "audio") + ) adapter_dict = dict(self.mlp1.named_parameters()) def is_llm(name: str) -> bool: @@ -1523,23 +1528,30 @@ class NemotronH_Nano_VL_V2( # Strip 'language_model.' prefix for LLM weights llm_weights.append((".".join(name.split(".")[1:]), w)) elif is_adapter_weights((name, w)): + if not load_multimodal_weights: + continue # Load vision-language adapter weights directly trimmed_name = ".".join(name.split(".")[1:]) param = adapter_dict[trimmed_name] with torch.no_grad(): default_weight_loader(param, w) elif is_vision_weights(name): + if not load_multimodal_weights: + continue # Convert: vision_model.radio_model.* → radio_model.* hf_key = name[len("vision_model.") :] # Remove "vision_model." prefix vision_weights.append((hf_key, w)) elif is_sound_weights(name): + if not load_multimodal_weights: + continue assert self.sound_encoder is not None sound_weights.append((name, w)) self.language_model.load_weights(llm_weights) - self.vision_model.load_weights(vision_weights) - if self.sound_encoder is not None and len(sound_weights) > 0: - self.sound_encoder.load_weights(sound_weights) + if load_multimodal_weights: + self.vision_model.load_weights(vision_weights) + if self.sound_encoder is not None and len(sound_weights) > 0: + self.sound_encoder.load_weights(sound_weights) def get_vit_model_from_radio_config(self, hf_config): hf_config_vision = hf_config.vision_config