From 7e917606504f48f22491eb9d04ce50c28618d45f Mon Sep 17 00:00:00 2001 From: jiacao-amd Date: Wed, 9 Sep 2026 23:39:12 -0700 Subject: [PATCH] [ROCm][Perf] Route large DSV4 sparse prefill to AITER OPUS (#54855) Signed-off-by: jiacao-amd Co-authored-by: OpenAI Codex Co-authored-by: Claude Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../attention/test_rocm_triton_attn_dsv4.py | 211 ++++++++++++++++++ .../v1/attention/ops/rocm_aiter_mla_sparse.py | 107 +++++++++ 2 files changed, 318 insertions(+) diff --git a/tests/kernels/attention/test_rocm_triton_attn_dsv4.py b/tests/kernels/attention/test_rocm_triton_attn_dsv4.py index e8cfa70d6a..fb696edd84 100644 --- a/tests/kernels/attention/test_rocm_triton_attn_dsv4.py +++ b/tests/kernels/attention/test_rocm_triton_attn_dsv4.py @@ -381,6 +381,217 @@ def test_sparse_attn_prefill_ragged_kernel() -> None: torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2) +@pytest.mark.parametrize( + ("num_queries", "on_gfx950", "expected"), + [(1023, True, False), (1024, True, True), (1024, False, False)], +) +def test_aiter_sparse_prefill_opus_selection( + num_queries: int, on_gfx950: bool, expected: bool +) -> None: + from vllm.v1.attention.ops.rocm_aiter_mla_sparse import ( + _can_use_aiter_sparse_prefill_opus, + ) + + device = torch.device("cuda") + q = torch.empty(num_queries, 16, HEAD_DIM, dtype=torch.bfloat16, device=device) + kv = torch.empty(num_queries, HEAD_DIM, dtype=torch.bfloat16, device=device) + attn_sink = torch.empty(16, dtype=torch.float32, device=device) + output = torch.empty_like(q) + + assert ( + _can_use_aiter_sparse_prefill_opus( + q, kv, attn_sink, output, on_gfx950=on_gfx950 + ) + is expected + ) + + +def test_aiter_sparse_prefill_opus_selection_rejects_incompatible_inputs() -> None: + from vllm.v1.attention.ops.rocm_aiter_mla_sparse import ( + _can_use_aiter_sparse_prefill_opus, + ) + + device = torch.device("cuda") + q = torch.empty(1024, 16, HEAD_DIM, dtype=torch.bfloat16, device=device) + kv = torch.empty(1024, HEAD_DIM, dtype=torch.bfloat16, device=device) + attn_sink = torch.empty(16, dtype=torch.float32, device=device) + output = torch.empty_like(q) + + assert _can_use_aiter_sparse_prefill_opus(q, kv, attn_sink, output, on_gfx950=True) + assert not _can_use_aiter_sparse_prefill_opus( + q, + kv, + attn_sink, + torch.empty(16, 1024, HEAD_DIM, dtype=q.dtype, device=device).transpose(0, 1), + on_gfx950=True, + ) + assert not _can_use_aiter_sparse_prefill_opus( + q, kv, attn_sink[:-1], output, on_gfx950=True + ) + assert not _can_use_aiter_sparse_prefill_opus( + q.cpu(), kv.cpu(), attn_sink.cpu(), output.cpu(), on_gfx950=True + ) + + +def test_sparse_attn_prefill_aiter_opus_routing(monkeypatch) -> None: + from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod + + q = torch.empty(2, 1, HEAD_DIM, dtype=torch.bfloat16) + kv = torch.empty(2, 1, HEAD_DIM, dtype=torch.bfloat16) + indices = torch.tensor([[0], [1]], dtype=torch.int32) + topk_length = torch.ones(2, dtype=torch.int32) + attn_sink = torch.empty(1, dtype=torch.float32) + output = torch.empty_like(q) + opus_calls = 0 + + def fake_opus(*args, out): + nonlocal opus_calls + opus_calls += 1 + assert args[2].dtype == torch.int32 + assert args[3].dtype == torch.int32 + assert args[5].numel() == 0 + assert torch.count_nonzero(args[6]) == 0 + out.zero_() + return out + + monkeypatch.setattr(mod, "_can_use_aiter_sparse_prefill_opus", lambda *args: True) + monkeypatch.setattr(mod, "_get_aiter_sparse_prefill_opus", lambda: fake_opus) + monkeypatch.setattr( + mod, + "build_ragged_indices_from_dense", + lambda *args, **kwargs: ( + torch.tensor([0, 1], dtype=torch.int32), + torch.tensor([0, 1, 2], dtype=torch.int32), + ), + ) + monkeypatch.setattr( + mod, + "_rocm_sparse_attn_prefill_triton", + lambda *args, **kwargs: pytest.fail("unexpected dense Triton fallback"), + ) + monkeypatch.setattr( + mod, + "_rocm_sparse_attn_prefill_ragged_triton", + lambda *args, **kwargs: pytest.fail("unexpected ragged Triton fallback"), + ) + + mod.rocm_sparse_attn_prefill( + q=q, + kv=kv, + indices=indices, + topk_length=topk_length, + scale=HEAD_DIM**-0.5, + head_dim=HEAD_DIM, + nope_head_dim=NOPE_HEAD_DIM, + rope_head_dim=ROPE_HEAD_DIM, + attn_sink=attn_sink, + output=output, + ) + + assert opus_calls == 1 + assert torch.count_nonzero(output) == 0 + + +def test_sparse_attn_prefill_preserves_dense_triton_fallback(monkeypatch) -> None: + from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod + + q = torch.empty(2, 1, HEAD_DIM, dtype=torch.bfloat16) + kv = torch.empty(2, 1, HEAD_DIM, dtype=torch.bfloat16) + indices = torch.tensor([[0], [1]], dtype=torch.int32) + topk_length = torch.ones(2, dtype=torch.int32) + attn_sink = torch.empty(1, dtype=torch.float32) + output = torch.empty_like(q) + dense_fallback_calls = 0 + + def fake_dense_fallback(*args, **kwargs): + nonlocal dense_fallback_calls + dense_fallback_calls += 1 + return torch.zeros_like(q) + + monkeypatch.setattr(mod, "_can_use_aiter_sparse_prefill_opus", lambda *args: True) + monkeypatch.setattr(mod, "_get_aiter_sparse_prefill_opus", lambda: None) + monkeypatch.setattr(mod, "_rocm_sparse_attn_prefill_triton", fake_dense_fallback) + monkeypatch.setattr( + mod, + "_rocm_sparse_attn_prefill_ragged_triton", + lambda *args, **kwargs: pytest.fail("unexpected ragged Triton fallback"), + ) + + mod.rocm_sparse_attn_prefill( + q=q, + kv=kv, + indices=indices, + topk_length=topk_length, + scale=HEAD_DIM**-0.5, + head_dim=HEAD_DIM, + nope_head_dim=NOPE_HEAD_DIM, + rope_head_dim=ROPE_HEAD_DIM, + attn_sink=attn_sink, + output=output, + ) + + assert dense_fallback_calls == 1 + assert torch.count_nonzero(output) == 0 + + +@requires_gfx950 +@torch.inference_mode() +def test_sparse_attn_prefill_ragged_aiter_opus(monkeypatch) -> None: + opus_mod = pytest.importorskip("aiter.ops.pa_sparse_prefill_opus") + from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod + + monkeypatch.setattr( + mod, "_get_aiter_sparse_prefill_opus", lambda: opus_mod.pa_sparse_prefill_opus + ) + + device = torch.device("cuda") + torch.manual_seed(4) + num_queries = 8 + num_heads = 16 + q = ( + torch.randn( + num_queries, + num_heads, + HEAD_DIM, + dtype=torch.bfloat16, + device=device, + ) + * 0.125 + ) + kv = torch.randn(num_queries, HEAD_DIM, dtype=torch.bfloat16, device=device) * 0.125 + rows = [list(range(query_idx + 1)) for query_idx in range(num_queries)] + indices = torch.tensor( + [index for row in rows for index in row], dtype=torch.int32, device=device + ) + indptr = torch.tensor( + [0] + + [ + sum(len(row) for row in rows[: query_idx + 1]) + for query_idx in range(num_queries) + ], + dtype=torch.int32, + device=device, + ) + attn_sink = torch.linspace( + -0.25, 0.25, num_heads, dtype=torch.float32, device=device + ) + output = torch.empty_like(q) + scale = HEAD_DIM**-0.5 + + assert mod._rocm_sparse_attn_prefill_ragged_aiter_opus( + q=q, + kv=kv, + indices=indices, + indptr=indptr, + scale=scale, + attn_sink=attn_sink, + output=output, + ) + expected = _ref_sparse_prefill_ragged(q, kv, rows, scale, attn_sink) + + torch.testing.assert_close(output, expected, atol=2e-2, rtol=2e-2) + + @torch.inference_mode() def test_sparse_attn_decode_ragged_kernel() -> None: from vllm.v1.attention.ops.rocm_aiter_mla_sparse import ( diff --git a/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py b/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py index 46e1ee0596..45a381d4a3 100644 --- a/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py +++ b/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py @@ -13,6 +13,7 @@ import vllm.envs as envs from vllm.compilation.breakable_cudagraph import eager_break_during_capture from vllm.config import CUDAGraphMode from vllm.forward_context import get_forward_context +from vllm.logger import init_logger from vllm.platforms import current_platform from vllm.triton_utils import tl, triton from vllm.utils.torch_utils import LayerNameType @@ -26,6 +27,8 @@ else: _ON_GFX942 = False _ON_GFX950 = False +logger = init_logger(__name__) + @functools.cache def _get_aiter_topk_ops() -> tuple[Callable[..., None], Callable[..., None]] | None: @@ -39,8 +42,25 @@ def _get_aiter_topk_ops() -> tuple[Callable[..., None], Callable[..., None]] | N return top_k_per_row_prefill, top_k_per_row_decode +@functools.cache +def _get_aiter_sparse_prefill_opus() -> Callable[..., torch.Tensor] | None: + from vllm._aiter_ops import rocm_aiter_ops + + if not rocm_aiter_ops.is_mla_enabled(): + return None + try: + from aiter.ops.pa_sparse_prefill_opus import pa_sparse_prefill_opus + except ImportError: + return None + logger.info_once("Using AITER OPUS for large sparse MLA prefill on gfx950") + return pa_sparse_prefill_opus + + _GFX950_C4A_AITER_MAX_COMPRESSED_SEQ_LEN = 64 * 1024 _GFX950_C4A_NATIVE_MAX_ROWS = 256 +# Conservative perf gate, not a correctness bound: OPUS is correct for any query +# count, but Triton stays faster below this measured crossover. +_GFX950_AITER_SPARSE_PREFILL_OPUS_MIN_QUERIES = 1024 def _get_aiter_top_k_kernel( @@ -2667,6 +2687,66 @@ def _rocm_sparse_attn_prefill_triton( ) +def _can_use_aiter_sparse_prefill_opus( + q: torch.Tensor, + kv: torch.Tensor, + attn_sink: torch.Tensor | None, + output: torch.Tensor, + on_gfx950: bool = _ON_GFX950, +) -> bool: + return ( + on_gfx950 + and q.shape[0] >= _GFX950_AITER_SPARSE_PREFILL_OPUS_MIN_QUERIES + and q.is_cuda + and output.shape == q.shape + and kv.shape[-1] == q.shape[-1] + and q.dtype in (torch.bfloat16, torch.float16) + and kv.dtype == q.dtype + and output.dtype == q.dtype + and kv.device == q.device + and output.device == q.device + and q.stride(-1) == 1 + and kv.stride(-1) == 1 + and output.stride() == q.stride() + and attn_sink is not None + and attn_sink.shape == (q.shape[1],) + and attn_sink.dtype == torch.float32 + and attn_sink.device == q.device + ) + + +def _rocm_sparse_attn_prefill_ragged_aiter_opus( + q: torch.Tensor, + kv: torch.Tensor, + indices: torch.Tensor, + indptr: torch.Tensor, + scale: float, + attn_sink: torch.Tensor, + output: torch.Tensor, +) -> bool: + pa_sparse_prefill_opus = _get_aiter_sparse_prefill_opus() + if pa_sparse_prefill_opus is None: + return False + + indices = _as_int32_contiguous_1d(indices) + indptr = _as_int32_contiguous_1d(indptr) + empty_indices = indices[:0] + empty_indptr = torch.zeros_like(indptr) + pa_sparse_prefill_opus( + q, + kv, + indices, + indptr, + kv[:1], + empty_indices, + empty_indptr, + attn_sink.contiguous(), + float(scale), + out=output, + ) + return True + + @functools.lru_cache def _decode_cu_count() -> int: try: @@ -3170,6 +3250,33 @@ def rocm_sparse_attn_prefill( rope_head_dim, "rocm_sparse_attn_prefill", ) + opus_attn_sink = None if attn_sink is None else attn_sink[: q.shape[1]] + if ( + _can_use_aiter_sparse_prefill_opus(q, kv.squeeze(1), opus_attn_sink, output) + and _get_aiter_sparse_prefill_opus() is not None + ): + if ragged_indices is None or ragged_indptr is None: + assert indices is not None + indices_2d = indices.reshape(indices.shape[0], -1) + ragged_indices, ragged_indptr = build_ragged_indices_from_dense( + indices_2d, + topk_length + if topk_length is not None + else (indices_2d >= 0).sum(dim=-1, dtype=torch.int32), + num_rows=kv.shape[0], + ) + assert opus_attn_sink is not None + if _rocm_sparse_attn_prefill_ragged_aiter_opus( + q=q, + kv=kv.squeeze(1), + indices=ragged_indices, + indptr=ragged_indptr, + scale=scale, + attn_sink=opus_attn_sink, + output=output, + ): + return + if ragged_indices is not None and ragged_indptr is not None: output_chunk = _rocm_sparse_attn_prefill_ragged_triton( q=q,