[Aiter][ROCm] RMSNormGated+GroupedQuantFP8 fusion (#40710)
Signed-off-by: Tres Popp <tres.popp@amd.com> Signed-off-by: Tres Popp <trespopp@gmail.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -229,56 +229,71 @@ class RMSNormGated(CustomOp):
|
||||
def reset_parameters(self):
|
||||
torch.nn.init.ones_(self.weight)
|
||||
|
||||
def forward_native(
|
||||
self, x: torch.Tensor, z: torch.Tensor | None = None
|
||||
@staticmethod
|
||||
def forward_static(
|
||||
x: torch.Tensor,
|
||||
z: torch.Tensor | None,
|
||||
weight: torch.Tensor,
|
||||
epsilon: float,
|
||||
orig_dtype: torch.dtype,
|
||||
group_size: int | None = None,
|
||||
norm_before_gate: bool = True,
|
||||
activation: str = "swish",
|
||||
) -> torch.Tensor:
|
||||
"""Pure-PyTorch RMS normalization with optional gating.
|
||||
|
||||
This static method contains the full native logic so that both
|
||||
``forward_native`` and ``MatcherRMSNormGated`` (used by the
|
||||
compilation pattern matcher) can share the same implementation.
|
||||
|
||||
If *z* is not None and *norm_before_gate* is True:
|
||||
``out = rms_norm(x) * act(z)``
|
||||
If *z* is not None and *norm_before_gate* is False:
|
||||
``out = rms_norm(x * act(z))``
|
||||
"""
|
||||
Native PyTorch implementation of RMS normalization with gating.
|
||||
|
||||
Args:
|
||||
x: Input tensor
|
||||
z: Optional gating tensor
|
||||
|
||||
Returns:
|
||||
Normalized (and optionally gated) tensor
|
||||
|
||||
If z is not None:
|
||||
- norm_before_gate=True: out = norm(x) * silu(z)
|
||||
- norm_before_gate=False: out = norm(x * silu(z))
|
||||
"""
|
||||
orig_dtype = x.dtype
|
||||
x = x.float()
|
||||
weight = self.weight.float()
|
||||
z = z.float() if z is not None else None
|
||||
weight = weight.float()
|
||||
if z is not None:
|
||||
z = z.float()
|
||||
|
||||
assert self.activation in ["silu", "sigmoid", "swish"]
|
||||
act_fn = F.sigmoid if self.activation == "sigmoid" else F.silu
|
||||
assert activation in ["silu", "sigmoid", "swish"]
|
||||
act_fn = F.sigmoid if activation == "sigmoid" else F.silu
|
||||
|
||||
# Apply gating before normalization if needed
|
||||
if z is not None and not self.norm_before_gate:
|
||||
if z is not None and not norm_before_gate:
|
||||
x = x * act_fn(z)
|
||||
|
||||
# RMS Normalization
|
||||
if self.group_size is None:
|
||||
# Standard RMS norm across the last dimension
|
||||
if group_size is None:
|
||||
variance = x.pow(2).mean(dim=-1, keepdim=True)
|
||||
x_normed = x * torch.rsqrt(variance + self.eps)
|
||||
x_normed = x * torch.rsqrt(variance + epsilon)
|
||||
out = x_normed * weight
|
||||
else:
|
||||
# Group RMS norm
|
||||
from einops import rearrange
|
||||
|
||||
x_group = rearrange(x, "... (g d) -> ... g d", d=self.group_size)
|
||||
x_group = rearrange(x, "... (g d) -> ... g d", d=group_size)
|
||||
variance = x_group.pow(2).mean(dim=-1, keepdim=True)
|
||||
x_normed = x_group * torch.rsqrt(variance + self.eps)
|
||||
x_normed = x_group * torch.rsqrt(variance + epsilon)
|
||||
out = rearrange(x_normed, "... g d -> ... (g d)") * weight
|
||||
|
||||
# Apply gating after normalization if needed
|
||||
if z is not None and self.norm_before_gate:
|
||||
if z is not None and norm_before_gate:
|
||||
out = out * act_fn(z)
|
||||
|
||||
return out.to(orig_dtype)
|
||||
|
||||
def forward_native(
|
||||
self, x: torch.Tensor, z: torch.Tensor | None = None
|
||||
) -> torch.Tensor:
|
||||
"""PyTorch-native implementation equivalent to forward()."""
|
||||
return self.forward_static(
|
||||
x,
|
||||
z,
|
||||
self.weight,
|
||||
self.eps,
|
||||
x.dtype,
|
||||
group_size=self.group_size,
|
||||
norm_before_gate=self.norm_before_gate,
|
||||
activation=self.activation,
|
||||
)
|
||||
|
||||
def forward_cuda(
|
||||
self, x: torch.Tensor, z: torch.Tensor | None = None
|
||||
) -> torch.Tensor:
|
||||
|
||||
Reference in New Issue
Block a user