Fix Nano Nemotron text-only weight loading (#41205)
Signed-off-by: sunghoon.baek <sunghoon.baek@connectfy.cloud> Signed-off-by: Baekpica <35071468+Baekpica@users.noreply.github.com> Signed-off-by: sunghoon.baek <seanbb93@gmail.com> Co-authored-by: sunghoon.baek <sunghoon.baek@connectfy.cloud> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Netanel Haber <58652339+netanel-haber@users.noreply.github.com>
This commit is contained in:
co-authored by
sunghoon.baek
OpenAI Codex
Netanel Haber
parent
e724b0ea8d
commit
8c780943b4
@@ -0,0 +1,114 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.model_executor.models.nano_nemotron_vl import NemotronH_Nano_VL_V2
|
||||
|
||||
|
||||
class _TextOnlyMultiModalConfig:
|
||||
def get_limit_per_prompt(self, modality: str) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
class _ImageOnlyMultiModalConfig:
|
||||
def get_limit_per_prompt(self, modality: str) -> int:
|
||||
return 1 if modality == "image" else 0
|
||||
|
||||
|
||||
class _ModelConfig:
|
||||
multimodal_config = _TextOnlyMultiModalConfig()
|
||||
|
||||
|
||||
class _ImageOnlyModelConfig:
|
||||
multimodal_config = _ImageOnlyMultiModalConfig()
|
||||
|
||||
|
||||
class _LanguageModel:
|
||||
def __init__(self) -> None:
|
||||
self.loaded_weights: list[tuple[str, object]] = []
|
||||
|
||||
def load_weights(self, weights):
|
||||
self.loaded_weights = list(weights)
|
||||
|
||||
|
||||
class _MissingMultiModalModule:
|
||||
def named_parameters(self):
|
||||
raise AssertionError("multimodal weights should not be inspected")
|
||||
|
||||
def load_weights(self, weights):
|
||||
raise AssertionError("multimodal weights should not be loaded")
|
||||
|
||||
|
||||
class _AdapterModule:
|
||||
def named_parameters(self):
|
||||
return []
|
||||
|
||||
|
||||
class _VisionModel:
|
||||
def __init__(self) -> None:
|
||||
self.loaded_weights: list[tuple[str, object]] = []
|
||||
|
||||
def load_weights(self, weights):
|
||||
self.loaded_weights = list(weights)
|
||||
|
||||
|
||||
def test_nano_nemotron_vl_skips_multimodal_weights_in_text_only_mode():
|
||||
model = object.__new__(NemotronH_Nano_VL_V2)
|
||||
language_model = _LanguageModel()
|
||||
object.__setattr__(model, "model_config", _ModelConfig())
|
||||
object.__setattr__(model, "language_model", language_model)
|
||||
object.__setattr__(model, "mlp1", _AdapterModule())
|
||||
object.__setattr__(model, "vision_model", _MissingMultiModalModule())
|
||||
object.__setattr__(model, "sound_encoder", None)
|
||||
|
||||
language_weight = object()
|
||||
model.load_weights(
|
||||
[
|
||||
("language_model.layers.0.weight", language_weight),
|
||||
("mlp1.0.weight", object()),
|
||||
("vision_model.radio_model.encoder.weight", object()),
|
||||
("sound_encoder.encoder.weight", object()),
|
||||
]
|
||||
)
|
||||
|
||||
assert language_model.loaded_weights == [("layers.0.weight", language_weight)]
|
||||
|
||||
|
||||
def test_nano_nemotron_vl_loads_vision_weights_without_sound_encoder():
|
||||
model = object.__new__(NemotronH_Nano_VL_V2)
|
||||
language_model = _LanguageModel()
|
||||
vision_model = _VisionModel()
|
||||
object.__setattr__(model, "model_config", _ImageOnlyModelConfig())
|
||||
object.__setattr__(model, "language_model", language_model)
|
||||
object.__setattr__(model, "mlp1", _AdapterModule())
|
||||
object.__setattr__(model, "vision_model", vision_model)
|
||||
object.__setattr__(model, "sound_encoder", None)
|
||||
|
||||
language_weight = object()
|
||||
vision_weight = object()
|
||||
model.load_weights(
|
||||
[
|
||||
("language_model.layers.0.weight", language_weight),
|
||||
("vision_model.radio_model.encoder.weight", vision_weight),
|
||||
]
|
||||
)
|
||||
|
||||
assert language_model.loaded_weights == [("layers.0.weight", language_weight)]
|
||||
assert vision_model.loaded_weights == [
|
||||
("radio_model.encoder.weight", vision_weight)
|
||||
]
|
||||
|
||||
|
||||
def test_nano_nemotron_vl_requires_sound_encoder_for_sound_weights():
|
||||
model = object.__new__(NemotronH_Nano_VL_V2)
|
||||
language_model = _LanguageModel()
|
||||
vision_model = _VisionModel()
|
||||
object.__setattr__(model, "model_config", _ImageOnlyModelConfig())
|
||||
object.__setattr__(model, "language_model", language_model)
|
||||
object.__setattr__(model, "mlp1", _AdapterModule())
|
||||
object.__setattr__(model, "vision_model", vision_model)
|
||||
object.__setattr__(model, "sound_encoder", None)
|
||||
|
||||
with pytest.raises(AssertionError):
|
||||
model.load_weights([("sound_encoder.encoder.weight", object())])
|
||||
@@ -1499,6 +1499,11 @@ class NemotronH_Nano_VL_V2(
|
||||
return self.language_model.compute_logits(hidden_states)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
|
||||
mm_config = self.model_config.multimodal_config
|
||||
load_multimodal_weights = not all(
|
||||
mm_config.get_limit_per_prompt(modality) == 0
|
||||
for modality in ("image", "video", "audio")
|
||||
)
|
||||
adapter_dict = dict(self.mlp1.named_parameters())
|
||||
|
||||
def is_llm(name: str) -> bool:
|
||||
@@ -1523,23 +1528,30 @@ class NemotronH_Nano_VL_V2(
|
||||
# Strip 'language_model.' prefix for LLM weights
|
||||
llm_weights.append((".".join(name.split(".")[1:]), w))
|
||||
elif is_adapter_weights((name, w)):
|
||||
if not load_multimodal_weights:
|
||||
continue
|
||||
# Load vision-language adapter weights directly
|
||||
trimmed_name = ".".join(name.split(".")[1:])
|
||||
param = adapter_dict[trimmed_name]
|
||||
with torch.no_grad():
|
||||
default_weight_loader(param, w)
|
||||
elif is_vision_weights(name):
|
||||
if not load_multimodal_weights:
|
||||
continue
|
||||
# Convert: vision_model.radio_model.* → radio_model.*
|
||||
hf_key = name[len("vision_model.") :] # Remove "vision_model." prefix
|
||||
vision_weights.append((hf_key, w))
|
||||
elif is_sound_weights(name):
|
||||
if not load_multimodal_weights:
|
||||
continue
|
||||
assert self.sound_encoder is not None
|
||||
sound_weights.append((name, w))
|
||||
|
||||
self.language_model.load_weights(llm_weights)
|
||||
self.vision_model.load_weights(vision_weights)
|
||||
if self.sound_encoder is not None and len(sound_weights) > 0:
|
||||
self.sound_encoder.load_weights(sound_weights)
|
||||
if load_multimodal_weights:
|
||||
self.vision_model.load_weights(vision_weights)
|
||||
if self.sound_encoder is not None and len(sound_weights) > 0:
|
||||
self.sound_encoder.load_weights(sound_weights)
|
||||
|
||||
def get_vit_model_from_radio_config(self, hf_config):
|
||||
hf_config_vision = hf_config.vision_config
|
||||
|
||||
Reference in New Issue
Block a user