[Bugfix] Load and apply Nemotron Super final vision LayerNorm (#61401)

Signed-off-by: Shaun Kotek <skotek@nvidia.com>
Co-authored-by: Shaun Kotek - Nvidia <skotek@nvidia.com>
This commit is contained in:
Nathan Sala
2026-10-13 12:42:29 +00:00
committed by GitHub
co-authored by Shaun Kotek - Nvidia
parent 7342dc0317
commit 0838cd6593
2 changed files with 91 additions and 1 deletions
@@ -97,6 +97,7 @@ def test_nano_nemotron_vl_skips_multimodal_weights_in_text_only_mode():
object.__setattr__(model, "language_model", language_model)
object.__setattr__(model, "mlp1", _AdapterModule())
object.__setattr__(model, "vision_model", _MissingMultiModalModule())
object.__setattr__(model, "vision_final_layernorm", _MissingMultiModalModule())
object.__setattr__(model, "sound_encoder", None)
language_weight = object()
@@ -106,6 +107,8 @@ def test_nano_nemotron_vl_skips_multimodal_weights_in_text_only_mode():
("mlp1.0.weight", object()),
("vision_model.radio_model.encoder.weight", object()),
("sound_encoder.encoder.weight", object()),
("vision_projector.vision_final_layernorm.weight", object()),
("vision_projector.vision_final_layernorm.bias", object()),
]
)
@@ -120,6 +123,7 @@ def test_nano_nemotron_vl_loads_vision_weights_without_sound_encoder():
object.__setattr__(model, "language_model", language_model)
object.__setattr__(model, "mlp1", _AdapterModule())
object.__setattr__(model, "vision_model", vision_model)
object.__setattr__(model, "vision_final_layernorm", None)
object.__setattr__(model, "sound_encoder", None)
language_weight = object()
@@ -145,12 +149,65 @@ def test_nano_nemotron_vl_requires_sound_encoder_for_sound_weights():
object.__setattr__(model, "language_model", language_model)
object.__setattr__(model, "mlp1", _AdapterModule())
object.__setattr__(model, "vision_model", vision_model)
object.__setattr__(model, "vision_final_layernorm", None)
object.__setattr__(model, "sound_encoder", None)
with pytest.raises(AssertionError):
model.load_weights([("sound_encoder.encoder.weight", object())])
def _vision_norm_model(use_layernorm=True):
model = object.__new__(NemotronH_Nano_VL_V2)
torch.nn.Module.__init__(model)
model.model_config = _ImageOnlyModelConfig()
model.language_model = _LanguageModel()
model.vision_model = _VisionModel()
model.sound_encoder = None
model.mlp1 = torch.nn.Identity()
model.vision_final_layernorm = (
torch.nn.LayerNorm(4, eps=1e-6, dtype=torch.bfloat16) if use_layernorm else None
)
model.patch_size = 2
model.downsample_ratio = 0.5
model.ps_version = "v2"
model.video_temporal_patch_size = 2
return model
@pytest.mark.parametrize("mode", ["image", "dynamic", "video"])
@pytest.mark.parametrize("use_layernorm", [True, False])
def test_vision_layernorm_is_loaded_and_applied_before_pixel_shuffle(
mode, use_layernorm
):
model = _vision_norm_model(use_layernorm)
weight = torch.tensor([2.0, 3.0, 4.0, 5.0], dtype=torch.bfloat16)
bias = torch.tensor([0.5, -0.5, 1.0, -1.0], dtype=torch.bfloat16)
model.load_weights(
[
("vision_projector.vision_final_layernorm.weight", weight),
("vision_projector.vision_final_layernorm.bias", bias),
]
if use_layernorm
else []
)
features = torch.arange(16, dtype=torch.bfloat16).reshape(1, 4, 4)
expected = (
torch.nn.functional.layer_norm(features, (4,), weight, bias, eps=1e-6)
if use_layernorm
else features
)
expected = model.pixel_shuffle(expected.reshape(1, 2, 2, 4)).reshape(1, 1, 16)
model.vision_model = lambda *args, **kwargs: (None, features)
pixels = torch.zeros(2 if mode == "video" else 1, 3, 4, 4)
if mode == "dynamic":
output = model.extract_feature_dynamic(pixels, imgs_sizes=[(4, 4)])
else:
output = model.extract_feature(
pixels, num_frames=2 if mode == "video" else None
)
torch.testing.assert_close(output, expected)
def _make_mm_items_with_video_bytes(
video_bytes: bytes,
) -> MultiModalDataItems: