fix: fuse weightless RMSNorms at their declared width (#50867)

Signed-off-by: Anuj Bolewar <anujbolewar@gmail.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Anuj Bolewar <anujbolewar@gmail.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Anuj Bolewar
2026-08-04 16:03:09 +00:00
committed by GitHub
co-authored by Anuj Bolewar Harry Mellor
parent 4f819f801b
commit 166f4e2dc3
2 changed files with 29 additions and 23 deletions
@@ -146,26 +146,39 @@ def test_rms_norm_builds_vllm_class(cls, expected, zero_centered, default_vllm_c
from vllm.model_executor.layers.layernorm import GemmaRMSNorm as VLLMGemmaRMSNorm from vllm.model_executor.layers.layernorm import GemmaRMSNorm as VLLMGemmaRMSNorm
from vllm.model_executor.layers.layernorm import RMSNorm as VLLMRMSNorm from vllm.model_executor.layers.layernorm import RMSNorm as VLLMRMSNorm
# `default_vllm_config` supplies the config context the CustomOp needs; the
# weightless path reads hidden size from the model config, so stub it.
vllm_config = SimpleNamespace(
model_config=SimpleNamespace(get_hidden_size=lambda: 16)
)
with torch.device("meta"): with torch.device("meta"):
module = cls() module = cls()
fuser = get_fuser(module) fuser = get_fuser(module)
built = fuser.fuse(module, "norm", vllm_config) built = fuser.fuse(module, "norm", default_vllm_config)
from vllm.model_executor.models.transformers.fusers.rms_norm import ( from vllm.model_executor.models.transformers.fusers.rms_norm import (
TPAwareNormMixin, TPAwareNormMixin,
) )
types_by_name = {"RMSNorm": VLLMRMSNorm, "GemmaRMSNorm": VLLMGemmaRMSNorm} types_by_name = {"RMSNorm": VLLMRMSNorm, "GemmaRMSNorm": VLLMGemmaRMSNorm}
assert isinstance(built, types_by_name[expected]) assert isinstance(built, types_by_name[expected])
assert isinstance(built, TPAwareNormMixin) # fused norms self-correct under TP assert isinstance(built, TPAwareNormMixin)
assert built.variance_epsilon == module.variance_epsilon assert built.variance_epsilon == module.variance_epsilon
assert isinstance(built.weight, nn.Parameter) == ( # The weight states the hidden size (and is what the TP check reads); a norm
getattr(module, "weight", None) is not None # without one has none to state.
) weight = getattr(module, "weight", None)
assert isinstance(built.weight, nn.Parameter) == (weight is not None)
assert built.weight.shape[0] == (weight.size(0) if weight is not None else 0)
def test_weightless_norm_has_no_hidden_size(default_vllm_config):
"""A weightless norm states no hidden size, and the model's (LM) hidden size
would be wrong for one on a sub-dimension: Llama 4's `qk_norm` normalizes
head_dim, so sizing it at hidden_size made the TP gather reject its input.
It fuses with none, and so normalizes the width it is given at any TP size,
matching the unfused norm and vLLM's native `RMSNorm(hidden_size=head_dim)`.
"""
module = WeightlessRMSNorm(72)
built = get_fuser(module).fuse(module, "norm", default_vllm_config)
assert built.hidden_size == 0
built.tp_size = 2 # emulate TP=2 without a real process group
x = torch.randn(4, 72)
torch.testing.assert_close(built(x), module(x))
def test_fused_rms_norm_op_default_eps(default_vllm_config): def test_fused_rms_norm_op_default_eps(default_vllm_config):
@@ -178,8 +191,7 @@ def test_fused_rms_norm_op_default_eps(default_vllm_config):
fuser = get_fuser(module) fuser = get_fuser(module)
assert isinstance(fuser, RMSNormFuser) assert isinstance(fuser, RMSNormFuser)
assert not fuser.zero_centered assert not fuser.zero_centered
mc = SimpleNamespace(get_hidden_size=lambda: 16, dtype=torch.float32) vllm_config = SimpleNamespace(model_config=SimpleNamespace(dtype=torch.float32))
vllm_config = SimpleNamespace(model_config=mc)
built = fuser.fuse(module, "norm", vllm_config) built = fuser.fuse(module, "norm", vllm_config)
assert isinstance(built, VLLMRMSNorm) assert isinstance(built, VLLMRMSNorm)
assert built.variance_epsilon == torch.finfo(torch.float32).eps assert built.variance_epsilon == torch.finfo(torch.float32).eps
@@ -188,18 +200,15 @@ def test_fused_rms_norm_op_default_eps(default_vllm_config):
def test_eps_is_derived_per_instance(default_vllm_config): def test_eps_is_derived_per_instance(default_vllm_config):
"""Two instances of the same norm class with different eps must fuse to their """Two instances of the same norm class with different eps must fuse to their
own eps: the type-cached fuser holds only structure, not this value.""" own eps: the type-cached fuser holds only structure, not this value."""
vllm_config = SimpleNamespace(
model_config=SimpleNamespace(get_hidden_size=lambda: 16)
)
with torch.device("meta"): with torch.device("meta"):
for eps in (1e-5, 1e-6): for eps in (1e-5, 1e-6):
module = RMSNorm(16, eps=eps) module = RMSNorm(16, eps=eps)
built = get_fuser(module).fuse(module, "norm", vllm_config) built = get_fuser(module).fuse(module, "norm", default_vllm_config)
assert built.variance_epsilon == eps assert built.variance_epsilon == eps
def test_fused_norm_is_gather_capable(default_vllm_config): def test_fused_norm_is_gather_capable(default_vllm_config):
"""Every fused norm is emitted gather-capable, so a norm on a head-sharded """Every weighted fused norm is emitted gather-capable, so a norm on a head-sharded
projection (OLMoE-style) self-corrects at runtime with no QKV-specific projection (OLMoE-style) self-corrects at runtime with no QKV-specific
plumbing. A full-width input skips the gather and equals a plain norm.""" plumbing. A full-width input skips the gather and equals a plain norm."""
from vllm.model_executor.layers.layernorm import GemmaRMSNorm, RMSNorm from vllm.model_executor.layers.layernorm import GemmaRMSNorm, RMSNorm
@@ -190,20 +190,17 @@ class RMSNormFuser(BaseFuser):
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig" self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> nn.Module: ) -> nn.Module:
"""Fuse the matched RMSNorm pattern into a vLLM fused RMSNorm CustomOp.""" """Fuse the matched RMSNorm pattern into a vLLM fused RMSNorm CustomOp."""
model_config = vllm_config.model_config
weight = getattr(module, "weight", None) weight = getattr(module, "weight", None)
hidden_size = ( has_weight = weight is not None
weight.size(0) if weight is not None else model_config.get_hidden_size() hidden_size = weight.size(0) if has_weight else 0
)
graph = trace(module) graph = trace(module)
eps = self._eps_from_graph(graph) if graph is not None else None eps = self._eps_from_graph(graph) if graph is not None else None
if eps is None: if eps is None:
# If eps not in graph, match torch behaviour. # If eps not in graph, match torch behaviour.
dtype = weight.dtype if weight is not None else model_config.dtype dtype = weight.dtype if has_weight else vllm_config.model_config.dtype
eps = torch.finfo(dtype).eps eps = torch.finfo(dtype).eps
if self.zero_centered: if self.zero_centered:
return TPAwareGemmaRMSNorm(hidden_size=hidden_size, eps=eps) return TPAwareGemmaRMSNorm(hidden_size=hidden_size, eps=eps)
has_weight = weight is not None
return TPAwareRMSNorm( return TPAwareRMSNorm(
hidden_size=hidden_size, hidden_size=hidden_size,
eps=eps, eps=eps,