fix: fuse weightless RMSNorms at their declared width (#50867)

Signed-off-by: Anuj Bolewar <anujbolewar@gmail.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Anuj Bolewar <anujbolewar@gmail.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Anuj Bolewar
2026-08-04 16:03:09 +00:00
committed by GitHub
co-authored by Anuj Bolewar Harry Mellor
parent 4f819f801b
commit 166f4e2dc3
2 changed files with 29 additions and 23 deletions
@@ -190,20 +190,17 @@ class RMSNormFuser(BaseFuser):
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> nn.Module:
"""Fuse the matched RMSNorm pattern into a vLLM fused RMSNorm CustomOp."""
model_config = vllm_config.model_config
weight = getattr(module, "weight", None)
hidden_size = (
weight.size(0) if weight is not None else model_config.get_hidden_size()
)
has_weight = weight is not None
hidden_size = weight.size(0) if has_weight else 0
graph = trace(module)
eps = self._eps_from_graph(graph) if graph is not None else None
if eps is None:
# If eps not in graph, match torch behaviour.
dtype = weight.dtype if weight is not None else model_config.dtype
dtype = weight.dtype if has_weight else vllm_config.model_config.dtype
eps = torch.finfo(dtype).eps
if self.zero_centered:
return TPAwareGemmaRMSNorm(hidden_size=hidden_size, eps=eps)
has_weight = weight is not None
return TPAwareRMSNorm(
hidden_size=hidden_size,
eps=eps,