[ROCm][Perf] Fix Gemma AR+RMS prefill fallback (#58547)

Signed-off-by: Mikko Tukiainen <Mikko.Tukiainen@amd.com>
Signed-off-by: Mikko Tukiainen <mikko.tukiainen@amd.com>
This commit is contained in:
Mikko Tukiainen
2026-10-12 19:53:08 +00:00
committed by GitHub
parent fabaecbfa5
commit abad32a718
6 changed files with 220 additions and 13 deletions
@@ -1149,6 +1149,8 @@ class AllReduceFusionPass(VllmPatternMatcherPass):
# TODO: make BasePattern to inherit from VllmPatternReplacement
class AiterAllreduceFusedRMSNormPattern(BasePattern, VllmPatternReplacement):
gemma_norm = False
def __init__(
self,
epsilon: float,
@@ -1170,7 +1172,8 @@ class AiterAllreduceFusedRMSNormPattern(BasePattern, VllmPatternReplacement):
input: torch.Tensor, weight: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
allreduce_output = tensor_model_parallel_all_reduce(input)
rms = vllm.ir.ops.rms_norm(allreduce_output, weight, self.epsilon)
norm_weight = weight.float() + 1.0 if self.gemma_norm else weight
rms = vllm.ir.ops.rms_norm(allreduce_output, norm_weight, self.epsilon)
return rms, allreduce_output
@@ -1187,13 +1190,22 @@ class AiterAllreduceFusedRMSNormPattern(BasePattern, VllmPatternReplacement):
residual=residual,
weight=weight.to(input.dtype),
epsilon=self.epsilon,
gemma_norm=self.gemma_norm,
)
return allreduce[0], allreduce[1]
return _replacement
class AiterAllreduceFusedGemmaRMSNormPattern(AiterAllreduceFusedRMSNormPattern):
"""Match all-reduce followed by GemmaRMSNorm without a residual."""
gemma_norm = True
class AiterAllreduceFusedAddRMSNormPattern(BasePattern, VllmPatternReplacement):
gemma_norm = False
def __init__(
self,
epsilon: float,
@@ -1216,8 +1228,9 @@ class AiterAllreduceFusedAddRMSNormPattern(BasePattern, VllmPatternReplacement):
residual: torch.Tensor, input: torch.Tensor, weight: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
allreduce_output = tensor_model_parallel_all_reduce(input)
norm_weight = weight.float() + 1.0 if self.gemma_norm else weight
rms, residual = vllm.ir.ops.fused_add_rms_norm(
allreduce_output, residual, weight, self.epsilon
allreduce_output, residual, norm_weight, self.epsilon
)
return rms, residual
@@ -1233,12 +1246,19 @@ class AiterAllreduceFusedAddRMSNormPattern(BasePattern, VllmPatternReplacement):
residual=residual,
weight=weight.to(input.dtype),
epsilon=self.epsilon,
gemma_norm=self.gemma_norm,
)
return allreduce[0], allreduce[1]
return _replacement
class AiterAllreduceFusedAddGemmaRMSNormPattern(AiterAllreduceFusedAddRMSNormPattern):
"""Match all-reduce followed by fused-add GemmaRMSNorm."""
gemma_norm = True
class AiterAllreduceFusedAddRMSNormOutputOnlyPattern(
AiterAllreduceFusedAddRMSNormPattern
):
@@ -1267,6 +1287,14 @@ class AiterAllreduceFusedAddRMSNormOutputOnlyPattern(
return _replacement
class AiterAllreduceFusedAddGemmaRMSNormOutputOnlyPattern(
AiterAllreduceFusedAddRMSNormOutputOnlyPattern
):
"""Match fused-add GemmaRMSNorm when its residual output is dead."""
gemma_norm = True
class AiterAllreduceFusedRMSNormGroupQuantFP8Pattern(
BasePattern, VllmPatternReplacement
):
@@ -1608,7 +1636,7 @@ class RocmAiterAllReduceFusionPass(VllmFusionPatternMatcherPass):
# AR+RMS-only fusion runs first and consumes the all_reduce node,
# leaving the trailing quant op stranded as an unfused kernel.
# Register larger subgraphs first (DeepSeek indexer fan-out, then
# quant-only AR+RMS+quant, then AR+RMS-only).
# quant-only AR+RMS+quant, then Gemma AR+RMS, then AR+RMS-only).
if supports_per_group_quant:
self.register(
AiterAllreduceFusedAddRMSNormGroupQuantWithIndexerPattern(
@@ -1632,6 +1660,30 @@ class RocmAiterAllReduceFusionPass(VllmFusionPatternMatcherPass):
)
)
# Must stay above the non-Gemma AR+RMS patterns below, which would
# otherwise match first.
self.register(
AiterAllreduceFusedGemmaRMSNormPattern(
epsilon,
self.model_dtype,
self.device,
)
)
self.register(
AiterAllreduceFusedAddGemmaRMSNormPattern(
epsilon,
self.model_dtype,
self.device,
)
)
self.register(
AiterAllreduceFusedAddGemmaRMSNormOutputOnlyPattern(
epsilon,
self.model_dtype,
self.device,
)
)
self.register(
AiterAllreduceFusedRMSNormPattern(
epsilon,