[ROCm][Perf] Fix Gemma AR+RMS prefill fallback (#58547)

Signed-off-by: Mikko Tukiainen <Mikko.Tukiainen@amd.com>
Signed-off-by: Mikko Tukiainen <mikko.tukiainen@amd.com>
This commit is contained in:
Mikko Tukiainen
2026-10-12 19:53:08 +00:00
committed by GitHub
parent fabaecbfa5
commit abad32a718
6 changed files with 220 additions and 13 deletions
@@ -177,10 +177,12 @@ class TestAllReduceGemmaRMSNormModel(torch.nn.Module):
token_num=16,
eps=1e-6,
dtype: torch.dtype = torch.float16,
use_aiter: bool = False,
):
super().__init__()
self.hidden_size = hidden_size
self.eps = eps
self.use_aiter = use_aiter
self.norm = [GemmaRMSNorm(hidden_size, eps) for _ in range(4)]
# Non-trivial weight (~Gemma range) so (1 + w) exercises the scale path.
for n in self.norm:
@@ -210,6 +212,8 @@ class TestAllReduceGemmaRMSNormModel(torch.nn.Module):
return [torch.ops.vllm.all_reduce.default]
def ops_in_model_after(self):
if self.use_aiter:
return [rocm_aiter_ops.get_fused_allreduce_rmsnorm_op()]
return [torch.ops.vllm.flashinfer_trtllm_fused_allreduce_norm.default]
@@ -383,6 +387,32 @@ class TestAiterAllReduceRMSNormGroupQuantFP8Model(torch.nn.Module):
]
class TestAiterAllReduceGemmaRMSNormGroupQuantFP8Model(
TestAiterAllReduceRMSNormGroupQuantFP8Model
):
"""Gemma norms feeding the per-group FP8 quant sites.
The fused AR+RMS+quant op has no ``gemma_norm`` mode, so these sites must
keep matching the quant-fused patterns with the pre-added ``1 + weight``.
The Gemma AR+RMS-only patterns match a prefix of the same subgraph, so if
they were registered ahead of the quant-fused ones they would consume the
``all_reduce`` and strand ``rocm_aiter_group_fp8_quant`` as a separate
kernel -- which the ``fully_replaced=True`` check below would catch.
"""
def __init__(
self,
hidden_size=128,
token_num=16,
eps=1e-6,
dtype: torch.dtype = torch.bfloat16,
):
super().__init__(hidden_size, token_num, eps, dtype)
self.norm = [GemmaRMSNorm(hidden_size, eps) for _ in range(4)]
for norm in self.norm:
norm.weight.requires_grad_(False)
class TestAllReduceFusedAddRMSNormStaticQuantFP4Model(torch.nn.Module):
def __init__(
self, hidden_size=16, token_num=16, eps=1e-6, dtype: torch.dtype = torch.float16
@@ -450,10 +480,10 @@ class TestAllReduceFusedAddRMSNormStaticQuantFP4Model(torch.nn.Module):
pytest.param(
TestAllReduceGemmaRMSNormModel,
False,
False,
IS_AITER_FOUND,
marks=pytest.mark.skipif(
current_platform.is_rocm(),
reason="Not supported on ROCm platform",
current_platform.is_rocm() and not IS_AITER_FOUND,
reason="AITER is required on ROCm",
),
),
pytest.param(
@@ -646,7 +676,10 @@ def all_reduce_fusion_pass_on_test_model(
)
token_num = batch_size * seq_len
if test_model_cls is TestAllReduceRMSNormModel:
if test_model_cls in (
TestAllReduceRMSNormModel,
TestAllReduceGemmaRMSNormModel,
):
model = test_model_cls(
hidden_size, token_num, dtype=dtype, use_aiter=use_aiter
)
@@ -667,9 +700,13 @@ def all_reduce_fusion_pass_on_test_model(
)
backend.check_before_ops(model.ops_in_model_before(), fully_replaced=False)
backend.check_after_ops(model.ops_in_model_after())
if test_model_cls in (
TestAllReduceGemmaRMSNormModel,
TestAllReduceGemmaRMSNormStaticQuantFP8Model,
if (
test_model_cls
in (
TestAllReduceGemmaRMSNormModel,
TestAllReduceGemmaRMSNormStaticQuantFP8Model,
)
and not use_aiter
):
fused_op = torch.ops.vllm.flashinfer_trtllm_fused_allreduce_norm.default
fused_nodes = list(find_op_nodes(fused_op, backend.graph_post_pass))
@@ -684,6 +721,13 @@ def all_reduce_fusion_pass_on_test_model(
@pytest.mark.parametrize("hidden_size", [128])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("enable_rms_norm_custom_op", [True, False])
@pytest.mark.parametrize(
"test_model_cls",
[
TestAiterAllReduceRMSNormGroupQuantFP8Model,
TestAiterAllReduceGemmaRMSNormGroupQuantFP8Model,
],
)
@pytest.mark.skipif(
not current_platform.is_rocm(),
reason="ROCm AITER AR+RMS+per-group-FP8-quant fusion is ROCm-only",
@@ -695,6 +739,7 @@ def test_rocm_aiter_all_reduce_rmsnorm_group_quant_fp8_fusion_pass_replace(
hidden_size: int,
dtype: torch.dtype,
enable_rms_norm_custom_op: bool,
test_model_cls: type[torch.nn.Module],
monkeypatch: pytest.MonkeyPatch,
):
"""Sibling of ``test_all_reduce_fusion_pass_replace`` for the new
@@ -710,6 +755,10 @@ def test_rocm_aiter_all_reduce_rmsnorm_group_quant_fp8_fusion_pass_replace(
residual, DSv3.2 indexer fan-out; parametrized over
``rocm_aiter_group_fp8_quant``
producer).
Parametrized over plain and Gemma norms: the Gemma AR+RMS-only patterns
match a prefix of these subgraphs, so this also pins their registration
order behind the quant-fused ones.
"""
with monkeypatch.context() as m:
m.setenv("VLLM_ROCM_USE_AITER", "1")
@@ -730,7 +779,7 @@ def test_rocm_aiter_all_reduce_rmsnorm_group_quant_fp8_fusion_pass_replace(
args=(
num_processes,
master_port,
TestAiterAllReduceRMSNormGroupQuantFP8Model,
test_model_cls,
batch_size,
seq_len,
hidden_size,
@@ -0,0 +1,89 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import sys
import types
from types import SimpleNamespace
import pytest
import torch
import vllm._aiter_ops as aiter_ops
import vllm.distributed as vllm_distributed
import vllm.kernels # noqa: F401
def test_rocm_aiter_gemma_rmsnorm_falls_back_after_allreduce(
monkeypatch: pytest.MonkeyPatch,
):
class FakeAiterCA:
world_size = 8
fully_connected = True
def custom_fused_ar_rms(self, *args, **kwargs):
return None
class FakeAiterAllReduce:
aiter_ca = FakeAiterCA()
def use_1stage_fused_ar_rms(self, inp):
return False
class FakeQuickReduce:
disabled = False
def should_quick_allreduce(self, inp):
raise AssertionError("Gemma norm must not use fused QR+RMSNorm")
device_comm = SimpleNamespace(qr_comm=FakeQuickReduce())
monkeypatch.setattr(
vllm_distributed,
"get_tp_group",
lambda: SimpleNamespace(device_communicator=device_comm),
)
monkeypatch.setattr(
aiter_ops.rocm_aiter_ops,
"get_aiter_allreduce",
lambda: FakeAiterAllReduce(),
)
monkeypatch.setattr(
vllm_distributed,
"tensor_model_parallel_all_reduce",
lambda inp: inp + 2,
)
calls = []
fake_aiter = types.SimpleNamespace()
def fake_rmsnorm2d_fwd_with_add(
out,
inp,
residual,
residual_out,
weight,
epsilon,
*,
gemma_norm,
):
calls.append((inp, weight, epsilon, gemma_norm))
residual_out.copy_(inp + residual)
out.copy_(residual_out)
fake_aiter.rmsnorm2d_fwd_with_add = fake_rmsnorm2d_fwd_with_add
monkeypatch.setitem(sys.modules, "aiter", fake_aiter)
inp = torch.randn(8192, 16, dtype=torch.bfloat16)
residual = torch.randn_like(inp)
weight = torch.randn(16, dtype=torch.bfloat16)
out, residual_out = aiter_ops._rocm_aiter_fused_allreduce_rmsnorm_impl(
inp, residual, weight, 1e-6, gemma_norm=True
)
assert len(calls) == 1
assert calls[0][0] is not inp
assert calls[0][1] is weight
assert calls[0][2] == 1e-6
assert calls[0][3] is True
torch.testing.assert_close(residual_out, inp + 2 + residual)
torch.testing.assert_close(out, residual_out)