[Model Loader] Keep norm layers neutral under dummy weight init (#61323)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Woosuk Kwon
2026-10-13 08:54:24 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 12b005eefa
commit ac71b17517
2 changed files with 30 additions and 0 deletions
+22
View File
@@ -352,5 +352,27 @@ def test_dummy_wide_integer_param_is_untouched():
assert torch.equal(param, torch.arange(16, dtype=torch.int32))
def test_dummy_norm_weights_are_neutral(default_vllm_config):
"""Dummy norms stay identity-like, keeping activations at a realistic scale."""
from vllm.model_executor.layers.layernorm import (
GemmaRMSNorm,
LayerNorm,
RMSNorm,
RMSNormGated,
)
expected = {
RMSNorm(16): {"weight": 1.0},
GemmaRMSNorm(16): {"weight": 0.0},
LayerNorm(16): {"weight": 1.0, "bias": 0.0},
RMSNormGated(16): {"weight": 1.0},
}
for norm, values in expected.items():
for name, param in norm.named_parameters():
param.data.uniform_(-5, 5)
initialize_single_dummy_weight(param, name=name)
assert torch.all(param == values[name]), (type(norm).__name__, name)
if __name__ == "__main__":
test_download_weights_from_hf()