[ROCm][Perf] Fix Gemma AR+RMS prefill fallback (#58547)

Signed-off-by: Mikko Tukiainen <Mikko.Tukiainen@amd.com>
Signed-off-by: Mikko Tukiainen <mikko.tukiainen@amd.com>
This commit is contained in:
Mikko Tukiainen
2026-10-12 19:53:08 +00:00
committed by GitHub
parent fabaecbfa5
commit abad32a718
6 changed files with 220 additions and 13 deletions
@@ -177,10 +177,12 @@ class TestAllReduceGemmaRMSNormModel(torch.nn.Module):
token_num=16,
eps=1e-6,
dtype: torch.dtype = torch.float16,
use_aiter: bool = False,
):
super().__init__()
self.hidden_size = hidden_size
self.eps = eps
self.use_aiter = use_aiter
self.norm = [GemmaRMSNorm(hidden_size, eps) for _ in range(4)]
# Non-trivial weight (~Gemma range) so (1 + w) exercises the scale path.
for n in self.norm:
@@ -210,6 +212,8 @@ class TestAllReduceGemmaRMSNormModel(torch.nn.Module):
return [torch.ops.vllm.all_reduce.default]
def ops_in_model_after(self):
if self.use_aiter:
return [rocm_aiter_ops.get_fused_allreduce_rmsnorm_op()]
return [torch.ops.vllm.flashinfer_trtllm_fused_allreduce_norm.default]
@@ -383,6 +387,32 @@ class TestAiterAllReduceRMSNormGroupQuantFP8Model(torch.nn.Module):
]
class TestAiterAllReduceGemmaRMSNormGroupQuantFP8Model(
TestAiterAllReduceRMSNormGroupQuantFP8Model
):
"""Gemma norms feeding the per-group FP8 quant sites.
The fused AR+RMS+quant op has no ``gemma_norm`` mode, so these sites must
keep matching the quant-fused patterns with the pre-added ``1 + weight``.
The Gemma AR+RMS-only patterns match a prefix of the same subgraph, so if
they were registered ahead of the quant-fused ones they would consume the
``all_reduce`` and strand ``rocm_aiter_group_fp8_quant`` as a separate
kernel -- which the ``fully_replaced=True`` check below would catch.
"""
def __init__(
self,
hidden_size=128,
token_num=16,
eps=1e-6,
dtype: torch.dtype = torch.bfloat16,
):
super().__init__(hidden_size, token_num, eps, dtype)
self.norm = [GemmaRMSNorm(hidden_size, eps) for _ in range(4)]
for norm in self.norm:
norm.weight.requires_grad_(False)
class TestAllReduceFusedAddRMSNormStaticQuantFP4Model(torch.nn.Module):
def __init__(
self, hidden_size=16, token_num=16, eps=1e-6, dtype: torch.dtype = torch.float16
@@ -450,10 +480,10 @@ class TestAllReduceFusedAddRMSNormStaticQuantFP4Model(torch.nn.Module):
pytest.param(
TestAllReduceGemmaRMSNormModel,
False,
False,
IS_AITER_FOUND,
marks=pytest.mark.skipif(
current_platform.is_rocm(),
reason="Not supported on ROCm platform",
current_platform.is_rocm() and not IS_AITER_FOUND,
reason="AITER is required on ROCm",
),
),
pytest.param(
@@ -646,7 +676,10 @@ def all_reduce_fusion_pass_on_test_model(
)
token_num = batch_size * seq_len
if test_model_cls is TestAllReduceRMSNormModel:
if test_model_cls in (
TestAllReduceRMSNormModel,
TestAllReduceGemmaRMSNormModel,
):
model = test_model_cls(
hidden_size, token_num, dtype=dtype, use_aiter=use_aiter
)
@@ -667,9 +700,13 @@ def all_reduce_fusion_pass_on_test_model(
)
backend.check_before_ops(model.ops_in_model_before(), fully_replaced=False)
backend.check_after_ops(model.ops_in_model_after())
if test_model_cls in (
TestAllReduceGemmaRMSNormModel,
TestAllReduceGemmaRMSNormStaticQuantFP8Model,
if (
test_model_cls
in (
TestAllReduceGemmaRMSNormModel,
TestAllReduceGemmaRMSNormStaticQuantFP8Model,
)
and not use_aiter
):
fused_op = torch.ops.vllm.flashinfer_trtllm_fused_allreduce_norm.default
fused_nodes = list(find_op_nodes(fused_op, backend.graph_post_pass))
@@ -684,6 +721,13 @@ def all_reduce_fusion_pass_on_test_model(
@pytest.mark.parametrize("hidden_size", [128])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("enable_rms_norm_custom_op", [True, False])
@pytest.mark.parametrize(
"test_model_cls",
[
TestAiterAllReduceRMSNormGroupQuantFP8Model,
TestAiterAllReduceGemmaRMSNormGroupQuantFP8Model,
],
)
@pytest.mark.skipif(
not current_platform.is_rocm(),
reason="ROCm AITER AR+RMS+per-group-FP8-quant fusion is ROCm-only",
@@ -695,6 +739,7 @@ def test_rocm_aiter_all_reduce_rmsnorm_group_quant_fp8_fusion_pass_replace(
hidden_size: int,
dtype: torch.dtype,
enable_rms_norm_custom_op: bool,
test_model_cls: type[torch.nn.Module],
monkeypatch: pytest.MonkeyPatch,
):
"""Sibling of ``test_all_reduce_fusion_pass_replace`` for the new
@@ -710,6 +755,10 @@ def test_rocm_aiter_all_reduce_rmsnorm_group_quant_fp8_fusion_pass_replace(
residual, DSv3.2 indexer fan-out; parametrized over
``rocm_aiter_group_fp8_quant``
producer).
Parametrized over plain and Gemma norms: the Gemma AR+RMS-only patterns
match a prefix of these subgraphs, so this also pins their registration
order behind the quant-fused ones.
"""
with monkeypatch.context() as m:
m.setenv("VLLM_ROCM_USE_AITER", "1")
@@ -730,7 +779,7 @@ def test_rocm_aiter_all_reduce_rmsnorm_group_quant_fp8_fusion_pass_replace(
args=(
num_processes,
master_port,
TestAiterAllReduceRMSNormGroupQuantFP8Model,
test_model_cls,
batch_size,
seq_len,
hidden_size,