[ROCm][Perf] Fix Gemma AR+RMS prefill fallback (#58547)

Signed-off-by: Mikko Tukiainen <Mikko.Tukiainen@amd.com>
Signed-off-by: Mikko Tukiainen <mikko.tukiainen@amd.com>
This commit is contained in:
Mikko Tukiainen
2026-10-12 19:53:08 +00:00
committed by GitHub
parent fabaecbfa5
commit abad32a718
6 changed files with 220 additions and 13 deletions
+3
View File
@@ -136,6 +136,7 @@ def _rocm_aiter_rmsnorm2d_fwd_with_add_impl(
residual: torch.Tensor,
weight: torch.Tensor,
variance_epsilon: float,
gemma_norm: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]:
from aiter import rmsnorm2d_fwd_with_add
@@ -157,6 +158,7 @@ def _rocm_aiter_rmsnorm2d_fwd_with_add_impl(
residual_out, # residual output
weight,
variance_epsilon,
gemma_norm=gemma_norm,
)
return out.reshape(x_shape), residual_out.reshape(residual_shape)
@@ -166,6 +168,7 @@ def _rocm_aiter_rmsnorm2d_fwd_with_add_fake(
residual: torch.Tensor,
weight: torch.Tensor,
variance_epsilon: float,
gemma_norm: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]:
residual_out = torch.empty_like(residual)
out = torch.empty_like(x)