[ROCm][Perf] Fix Gemma AR+RMS prefill fallback (#58547)
Signed-off-by: Mikko Tukiainen <Mikko.Tukiainen@amd.com> Signed-off-by: Mikko Tukiainen <mikko.tukiainen@amd.com>
This commit is contained in:
@@ -1149,6 +1149,8 @@ class AllReduceFusionPass(VllmPatternMatcherPass):
|
||||
|
||||
# TODO: make BasePattern to inherit from VllmPatternReplacement
|
||||
class AiterAllreduceFusedRMSNormPattern(BasePattern, VllmPatternReplacement):
|
||||
gemma_norm = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
epsilon: float,
|
||||
@@ -1170,7 +1172,8 @@ class AiterAllreduceFusedRMSNormPattern(BasePattern, VllmPatternReplacement):
|
||||
input: torch.Tensor, weight: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
allreduce_output = tensor_model_parallel_all_reduce(input)
|
||||
rms = vllm.ir.ops.rms_norm(allreduce_output, weight, self.epsilon)
|
||||
norm_weight = weight.float() + 1.0 if self.gemma_norm else weight
|
||||
rms = vllm.ir.ops.rms_norm(allreduce_output, norm_weight, self.epsilon)
|
||||
|
||||
return rms, allreduce_output
|
||||
|
||||
@@ -1187,13 +1190,22 @@ class AiterAllreduceFusedRMSNormPattern(BasePattern, VllmPatternReplacement):
|
||||
residual=residual,
|
||||
weight=weight.to(input.dtype),
|
||||
epsilon=self.epsilon,
|
||||
gemma_norm=self.gemma_norm,
|
||||
)
|
||||
return allreduce[0], allreduce[1]
|
||||
|
||||
return _replacement
|
||||
|
||||
|
||||
class AiterAllreduceFusedGemmaRMSNormPattern(AiterAllreduceFusedRMSNormPattern):
|
||||
"""Match all-reduce followed by GemmaRMSNorm without a residual."""
|
||||
|
||||
gemma_norm = True
|
||||
|
||||
|
||||
class AiterAllreduceFusedAddRMSNormPattern(BasePattern, VllmPatternReplacement):
|
||||
gemma_norm = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
epsilon: float,
|
||||
@@ -1216,8 +1228,9 @@ class AiterAllreduceFusedAddRMSNormPattern(BasePattern, VllmPatternReplacement):
|
||||
residual: torch.Tensor, input: torch.Tensor, weight: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
allreduce_output = tensor_model_parallel_all_reduce(input)
|
||||
norm_weight = weight.float() + 1.0 if self.gemma_norm else weight
|
||||
rms, residual = vllm.ir.ops.fused_add_rms_norm(
|
||||
allreduce_output, residual, weight, self.epsilon
|
||||
allreduce_output, residual, norm_weight, self.epsilon
|
||||
)
|
||||
return rms, residual
|
||||
|
||||
@@ -1233,12 +1246,19 @@ class AiterAllreduceFusedAddRMSNormPattern(BasePattern, VllmPatternReplacement):
|
||||
residual=residual,
|
||||
weight=weight.to(input.dtype),
|
||||
epsilon=self.epsilon,
|
||||
gemma_norm=self.gemma_norm,
|
||||
)
|
||||
return allreduce[0], allreduce[1]
|
||||
|
||||
return _replacement
|
||||
|
||||
|
||||
class AiterAllreduceFusedAddGemmaRMSNormPattern(AiterAllreduceFusedAddRMSNormPattern):
|
||||
"""Match all-reduce followed by fused-add GemmaRMSNorm."""
|
||||
|
||||
gemma_norm = True
|
||||
|
||||
|
||||
class AiterAllreduceFusedAddRMSNormOutputOnlyPattern(
|
||||
AiterAllreduceFusedAddRMSNormPattern
|
||||
):
|
||||
@@ -1267,6 +1287,14 @@ class AiterAllreduceFusedAddRMSNormOutputOnlyPattern(
|
||||
return _replacement
|
||||
|
||||
|
||||
class AiterAllreduceFusedAddGemmaRMSNormOutputOnlyPattern(
|
||||
AiterAllreduceFusedAddRMSNormOutputOnlyPattern
|
||||
):
|
||||
"""Match fused-add GemmaRMSNorm when its residual output is dead."""
|
||||
|
||||
gemma_norm = True
|
||||
|
||||
|
||||
class AiterAllreduceFusedRMSNormGroupQuantFP8Pattern(
|
||||
BasePattern, VllmPatternReplacement
|
||||
):
|
||||
@@ -1608,7 +1636,7 @@ class RocmAiterAllReduceFusionPass(VllmFusionPatternMatcherPass):
|
||||
# AR+RMS-only fusion runs first and consumes the all_reduce node,
|
||||
# leaving the trailing quant op stranded as an unfused kernel.
|
||||
# Register larger subgraphs first (DeepSeek indexer fan-out, then
|
||||
# quant-only AR+RMS+quant, then AR+RMS-only).
|
||||
# quant-only AR+RMS+quant, then Gemma AR+RMS, then AR+RMS-only).
|
||||
if supports_per_group_quant:
|
||||
self.register(
|
||||
AiterAllreduceFusedAddRMSNormGroupQuantWithIndexerPattern(
|
||||
@@ -1632,6 +1660,30 @@ class RocmAiterAllReduceFusionPass(VllmFusionPatternMatcherPass):
|
||||
)
|
||||
)
|
||||
|
||||
# Must stay above the non-Gemma AR+RMS patterns below, which would
|
||||
# otherwise match first.
|
||||
self.register(
|
||||
AiterAllreduceFusedGemmaRMSNormPattern(
|
||||
epsilon,
|
||||
self.model_dtype,
|
||||
self.device,
|
||||
)
|
||||
)
|
||||
self.register(
|
||||
AiterAllreduceFusedAddGemmaRMSNormPattern(
|
||||
epsilon,
|
||||
self.model_dtype,
|
||||
self.device,
|
||||
)
|
||||
)
|
||||
self.register(
|
||||
AiterAllreduceFusedAddGemmaRMSNormOutputOnlyPattern(
|
||||
epsilon,
|
||||
self.model_dtype,
|
||||
self.device,
|
||||
)
|
||||
)
|
||||
|
||||
self.register(
|
||||
AiterAllreduceFusedRMSNormPattern(
|
||||
epsilon,
|
||||
|
||||
Reference in New Issue
Block a user