Avoid silent weights corruption when loading Nemotron Nano VL with reusable-buffer loaders like runai distributed streaming (#42244)
Signed-off-by: Noa Neria <nneria@nvidia.com>
This commit is contained in:
@@ -53,6 +53,19 @@ class _VisionModel:
|
|||||||
self.loaded_weights = list(weights)
|
self.loaded_weights = list(weights)
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeTensor:
|
||||||
|
"""Sentinel stand-in for torch.Tensor in load_weights tests. Supports the
|
||||||
|
.detach().clone() chain used by load_weights for buffered mm weights;
|
||||||
|
both methods return self so identity (and the existing equality
|
||||||
|
assertions) are preserved through cloning."""
|
||||||
|
|
||||||
|
def detach(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def clone(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
def test_nano_nemotron_vl_skips_multimodal_weights_in_text_only_mode():
|
def test_nano_nemotron_vl_skips_multimodal_weights_in_text_only_mode():
|
||||||
model = object.__new__(NemotronH_Nano_VL_V2)
|
model = object.__new__(NemotronH_Nano_VL_V2)
|
||||||
language_model = _LanguageModel()
|
language_model = _LanguageModel()
|
||||||
@@ -86,7 +99,7 @@ def test_nano_nemotron_vl_loads_vision_weights_without_sound_encoder():
|
|||||||
object.__setattr__(model, "sound_encoder", None)
|
object.__setattr__(model, "sound_encoder", None)
|
||||||
|
|
||||||
language_weight = object()
|
language_weight = object()
|
||||||
vision_weight = object()
|
vision_weight = _FakeTensor()
|
||||||
model.load_weights(
|
model.load_weights(
|
||||||
[
|
[
|
||||||
("language_model.layers.0.weight", language_weight),
|
("language_model.layers.0.weight", language_weight),
|
||||||
|
|||||||
@@ -1518,37 +1518,51 @@ class NemotronH_Nano_VL_V2(
|
|||||||
def is_sound_weights(name: str) -> bool:
|
def is_sound_weights(name: str) -> bool:
|
||||||
return name.startswith("sound")
|
return name.startswith("sound")
|
||||||
|
|
||||||
# Separate weights by component
|
# LLM weights (the bulk of the model) are streamed lazily through a
|
||||||
llm_weights = []
|
# generator so each tensor is copied into its parameter before the
|
||||||
vision_weights = []
|
# iterator advances, avoiding stale-reference corruption with
|
||||||
sound_weights = []
|
# reusable-buffer streamers. The smaller mm components (mlp1, vision,
|
||||||
|
# sound) are detach+cloned on append so they are independent of any
|
||||||
|
# reusable buffer the streamer may use, then loaded after the LLM.
|
||||||
|
adapter_weights: list[tuple[str, torch.Tensor]] = []
|
||||||
|
vision_weights: list[tuple[str, torch.Tensor]] = []
|
||||||
|
sound_weights: list[tuple[str, torch.Tensor]] = []
|
||||||
|
|
||||||
for name, w in weights:
|
def llm_weights_gen():
|
||||||
if is_llm(name):
|
for name, w in weights:
|
||||||
# Strip 'language_model.' prefix for LLM weights
|
if is_llm(name):
|
||||||
llm_weights.append((".".join(name.split(".")[1:]), w))
|
# Strip 'language_model.' prefix for LLM weights
|
||||||
elif is_adapter_weights((name, w)):
|
yield ".".join(name.split(".")[1:]), w
|
||||||
if not load_multimodal_weights:
|
elif is_adapter_weights((name, w)):
|
||||||
continue
|
if not load_multimodal_weights:
|
||||||
|
continue
|
||||||
|
trimmed_name = ".".join(name.split(".")[1:])
|
||||||
|
adapter_weights.append((trimmed_name, w.detach().clone()))
|
||||||
|
elif is_vision_weights(name):
|
||||||
|
if not load_multimodal_weights:
|
||||||
|
continue
|
||||||
|
# Convert: vision_model.radio_model.* → radio_model.*
|
||||||
|
hf_key = name[len("vision_model.") :]
|
||||||
|
vision_weights.append((hf_key, w.detach().clone()))
|
||||||
|
elif is_sound_weights(name):
|
||||||
|
if not load_multimodal_weights:
|
||||||
|
continue
|
||||||
|
assert self.sound_encoder is not None
|
||||||
|
sound_weights.append((name, w.detach().clone()))
|
||||||
|
|
||||||
|
# Fully drain the generator so every mm tensor is buffered, even if
|
||||||
|
# the LLM loader stops iterating early.
|
||||||
|
llm_weights_iter = llm_weights_gen()
|
||||||
|
self.language_model.load_weights(llm_weights_iter)
|
||||||
|
for _ in llm_weights_iter:
|
||||||
|
pass
|
||||||
|
|
||||||
|
if load_multimodal_weights:
|
||||||
|
for trimmed_name, w in adapter_weights:
|
||||||
# Load vision-language adapter weights directly
|
# Load vision-language adapter weights directly
|
||||||
trimmed_name = ".".join(name.split(".")[1:])
|
|
||||||
param = adapter_dict[trimmed_name]
|
param = adapter_dict[trimmed_name]
|
||||||
with torch.no_grad():
|
with torch.no_grad():
|
||||||
default_weight_loader(param, w)
|
default_weight_loader(param, w)
|
||||||
elif is_vision_weights(name):
|
|
||||||
if not load_multimodal_weights:
|
|
||||||
continue
|
|
||||||
# Convert: vision_model.radio_model.* → radio_model.*
|
|
||||||
hf_key = name[len("vision_model.") :] # Remove "vision_model." prefix
|
|
||||||
vision_weights.append((hf_key, w))
|
|
||||||
elif is_sound_weights(name):
|
|
||||||
if not load_multimodal_weights:
|
|
||||||
continue
|
|
||||||
assert self.sound_encoder is not None
|
|
||||||
sound_weights.append((name, w))
|
|
||||||
|
|
||||||
self.language_model.load_weights(llm_weights)
|
|
||||||
if load_multimodal_weights:
|
|
||||||
self.vision_model.load_weights(vision_weights)
|
self.vision_model.load_weights(vision_weights)
|
||||||
if self.sound_encoder is not None and len(sound_weights) > 0:
|
if self.sound_encoder is not None and len(sound_weights) > 0:
|
||||||
self.sound_encoder.load_weights(sound_weights)
|
self.sound_encoder.load_weights(sound_weights)
|
||||||
|
|||||||
Reference in New Issue
Block a user