[Aiter][ROCm] RMSNormGated+GroupedQuantFP8 fusion (#40710)

Signed-off-by: Tres Popp <tres.popp@amd.com>
Signed-off-by: Tres Popp <trespopp@gmail.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Tres
2026-05-14 15:37:09 -04:00
committed by GitHub
co-authored by Cursor
parent 9898f94abe
commit f887aa1a53
6 changed files with 557 additions and 34 deletions
+46 -31
View File
@@ -229,56 +229,71 @@ class RMSNormGated(CustomOp):
def reset_parameters(self):
torch.nn.init.ones_(self.weight)
def forward_native(
self, x: torch.Tensor, z: torch.Tensor | None = None
@staticmethod
def forward_static(
x: torch.Tensor,
z: torch.Tensor | None,
weight: torch.Tensor,
epsilon: float,
orig_dtype: torch.dtype,
group_size: int | None = None,
norm_before_gate: bool = True,
activation: str = "swish",
) -> torch.Tensor:
"""Pure-PyTorch RMS normalization with optional gating.
This static method contains the full native logic so that both
``forward_native`` and ``MatcherRMSNormGated`` (used by the
compilation pattern matcher) can share the same implementation.
If *z* is not None and *norm_before_gate* is True:
``out = rms_norm(x) * act(z)``
If *z* is not None and *norm_before_gate* is False:
``out = rms_norm(x * act(z))``
"""
Native PyTorch implementation of RMS normalization with gating.
Args:
x: Input tensor
z: Optional gating tensor
Returns:
Normalized (and optionally gated) tensor
If z is not None:
- norm_before_gate=True: out = norm(x) * silu(z)
- norm_before_gate=False: out = norm(x * silu(z))
"""
orig_dtype = x.dtype
x = x.float()
weight = self.weight.float()
z = z.float() if z is not None else None
weight = weight.float()
if z is not None:
z = z.float()
assert self.activation in ["silu", "sigmoid", "swish"]
act_fn = F.sigmoid if self.activation == "sigmoid" else F.silu
assert activation in ["silu", "sigmoid", "swish"]
act_fn = F.sigmoid if activation == "sigmoid" else F.silu
# Apply gating before normalization if needed
if z is not None and not self.norm_before_gate:
if z is not None and not norm_before_gate:
x = x * act_fn(z)
# RMS Normalization
if self.group_size is None:
# Standard RMS norm across the last dimension
if group_size is None:
variance = x.pow(2).mean(dim=-1, keepdim=True)
x_normed = x * torch.rsqrt(variance + self.eps)
x_normed = x * torch.rsqrt(variance + epsilon)
out = x_normed * weight
else:
# Group RMS norm
from einops import rearrange
x_group = rearrange(x, "... (g d) -> ... g d", d=self.group_size)
x_group = rearrange(x, "... (g d) -> ... g d", d=group_size)
variance = x_group.pow(2).mean(dim=-1, keepdim=True)
x_normed = x_group * torch.rsqrt(variance + self.eps)
x_normed = x_group * torch.rsqrt(variance + epsilon)
out = rearrange(x_normed, "... g d -> ... (g d)") * weight
# Apply gating after normalization if needed
if z is not None and self.norm_before_gate:
if z is not None and norm_before_gate:
out = out * act_fn(z)
return out.to(orig_dtype)
def forward_native(
self, x: torch.Tensor, z: torch.Tensor | None = None
) -> torch.Tensor:
"""PyTorch-native implementation equivalent to forward()."""
return self.forward_static(
x,
z,
self.weight,
self.eps,
x.dtype,
group_size=self.group_size,
norm_before_gate=self.norm_before_gate,
activation=self.activation,
)
def forward_cuda(
self, x: torch.Tensor, z: torch.Tensor | None = None
) -> torch.Tensor: