[perf] Add gemma RMS AR fusion (#42646)
Signed-off-by: jiahanc <173873397+jiahanc@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Jiangyun Zhu
parent
1bdc60ed53
commit
d0975a4b50
@@ -159,20 +159,10 @@ class GemmaRMSNorm(CustomOp):
|
||||
residual: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
"""PyTorch-native implementation equivalent to forward()."""
|
||||
orig_dtype = x.dtype
|
||||
weight = self.weight.data.float() + 1.0
|
||||
if residual is not None:
|
||||
x = (
|
||||
x.float() + residual.float()
|
||||
if orig_dtype == torch.float16
|
||||
else x + residual
|
||||
)
|
||||
residual = x
|
||||
# ir.ops.rms_norm handles fp32 upcast internally
|
||||
out = ir.ops.rms_norm(x, weight, self.variance_epsilon)
|
||||
return (
|
||||
out.to(orig_dtype) if residual is None else (out.to(orig_dtype), residual)
|
||||
)
|
||||
weight = self.weight.float() + 1.0
|
||||
if residual is None:
|
||||
return ir.ops.rms_norm(x, weight, self.variance_epsilon)
|
||||
return ir.ops.fused_add_rms_norm(x, residual, weight, self.variance_epsilon)
|
||||
|
||||
def forward_cuda(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user