[ROCm][Perf] Route large DSV4 sparse prefill to AITER OPUS (#54855)

Signed-off-by: jiacao-amd <jiahui.cao@amd.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
jiacao-amd
2026-09-10 14:39:12 +08:00
committed by GitHub
co-authored by OpenAI Codex Claude mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 442d36031c
commit 7e91760650
2 changed files with 318 additions and 0 deletions
@@ -381,6 +381,217 @@ def test_sparse_attn_prefill_ragged_kernel() -> None:
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
@pytest.mark.parametrize(
("num_queries", "on_gfx950", "expected"),
[(1023, True, False), (1024, True, True), (1024, False, False)],
)
def test_aiter_sparse_prefill_opus_selection(
num_queries: int, on_gfx950: bool, expected: bool
) -> None:
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_can_use_aiter_sparse_prefill_opus,
)
device = torch.device("cuda")
q = torch.empty(num_queries, 16, HEAD_DIM, dtype=torch.bfloat16, device=device)
kv = torch.empty(num_queries, HEAD_DIM, dtype=torch.bfloat16, device=device)
attn_sink = torch.empty(16, dtype=torch.float32, device=device)
output = torch.empty_like(q)
assert (
_can_use_aiter_sparse_prefill_opus(
q, kv, attn_sink, output, on_gfx950=on_gfx950
)
is expected
)
def test_aiter_sparse_prefill_opus_selection_rejects_incompatible_inputs() -> None:
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_can_use_aiter_sparse_prefill_opus,
)
device = torch.device("cuda")
q = torch.empty(1024, 16, HEAD_DIM, dtype=torch.bfloat16, device=device)
kv = torch.empty(1024, HEAD_DIM, dtype=torch.bfloat16, device=device)
attn_sink = torch.empty(16, dtype=torch.float32, device=device)
output = torch.empty_like(q)
assert _can_use_aiter_sparse_prefill_opus(q, kv, attn_sink, output, on_gfx950=True)
assert not _can_use_aiter_sparse_prefill_opus(
q,
kv,
attn_sink,
torch.empty(16, 1024, HEAD_DIM, dtype=q.dtype, device=device).transpose(0, 1),
on_gfx950=True,
)
assert not _can_use_aiter_sparse_prefill_opus(
q, kv, attn_sink[:-1], output, on_gfx950=True
)
assert not _can_use_aiter_sparse_prefill_opus(
q.cpu(), kv.cpu(), attn_sink.cpu(), output.cpu(), on_gfx950=True
)
def test_sparse_attn_prefill_aiter_opus_routing(monkeypatch) -> None:
from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod
q = torch.empty(2, 1, HEAD_DIM, dtype=torch.bfloat16)
kv = torch.empty(2, 1, HEAD_DIM, dtype=torch.bfloat16)
indices = torch.tensor([[0], [1]], dtype=torch.int32)
topk_length = torch.ones(2, dtype=torch.int32)
attn_sink = torch.empty(1, dtype=torch.float32)
output = torch.empty_like(q)
opus_calls = 0
def fake_opus(*args, out):
nonlocal opus_calls
opus_calls += 1
assert args[2].dtype == torch.int32
assert args[3].dtype == torch.int32
assert args[5].numel() == 0
assert torch.count_nonzero(args[6]) == 0
out.zero_()
return out
monkeypatch.setattr(mod, "_can_use_aiter_sparse_prefill_opus", lambda *args: True)
monkeypatch.setattr(mod, "_get_aiter_sparse_prefill_opus", lambda: fake_opus)
monkeypatch.setattr(
mod,
"build_ragged_indices_from_dense",
lambda *args, **kwargs: (
torch.tensor([0, 1], dtype=torch.int32),
torch.tensor([0, 1, 2], dtype=torch.int32),
),
)
monkeypatch.setattr(
mod,
"_rocm_sparse_attn_prefill_triton",
lambda *args, **kwargs: pytest.fail("unexpected dense Triton fallback"),
)
monkeypatch.setattr(
mod,
"_rocm_sparse_attn_prefill_ragged_triton",
lambda *args, **kwargs: pytest.fail("unexpected ragged Triton fallback"),
)
mod.rocm_sparse_attn_prefill(
q=q,
kv=kv,
indices=indices,
topk_length=topk_length,
scale=HEAD_DIM**-0.5,
head_dim=HEAD_DIM,
nope_head_dim=NOPE_HEAD_DIM,
rope_head_dim=ROPE_HEAD_DIM,
attn_sink=attn_sink,
output=output,
)
assert opus_calls == 1
assert torch.count_nonzero(output) == 0
def test_sparse_attn_prefill_preserves_dense_triton_fallback(monkeypatch) -> None:
from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod
q = torch.empty(2, 1, HEAD_DIM, dtype=torch.bfloat16)
kv = torch.empty(2, 1, HEAD_DIM, dtype=torch.bfloat16)
indices = torch.tensor([[0], [1]], dtype=torch.int32)
topk_length = torch.ones(2, dtype=torch.int32)
attn_sink = torch.empty(1, dtype=torch.float32)
output = torch.empty_like(q)
dense_fallback_calls = 0
def fake_dense_fallback(*args, **kwargs):
nonlocal dense_fallback_calls
dense_fallback_calls += 1
return torch.zeros_like(q)
monkeypatch.setattr(mod, "_can_use_aiter_sparse_prefill_opus", lambda *args: True)
monkeypatch.setattr(mod, "_get_aiter_sparse_prefill_opus", lambda: None)
monkeypatch.setattr(mod, "_rocm_sparse_attn_prefill_triton", fake_dense_fallback)
monkeypatch.setattr(
mod,
"_rocm_sparse_attn_prefill_ragged_triton",
lambda *args, **kwargs: pytest.fail("unexpected ragged Triton fallback"),
)
mod.rocm_sparse_attn_prefill(
q=q,
kv=kv,
indices=indices,
topk_length=topk_length,
scale=HEAD_DIM**-0.5,
head_dim=HEAD_DIM,
nope_head_dim=NOPE_HEAD_DIM,
rope_head_dim=ROPE_HEAD_DIM,
attn_sink=attn_sink,
output=output,
)
assert dense_fallback_calls == 1
assert torch.count_nonzero(output) == 0
@requires_gfx950
@torch.inference_mode()
def test_sparse_attn_prefill_ragged_aiter_opus(monkeypatch) -> None:
opus_mod = pytest.importorskip("aiter.ops.pa_sparse_prefill_opus")
from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod
monkeypatch.setattr(
mod, "_get_aiter_sparse_prefill_opus", lambda: opus_mod.pa_sparse_prefill_opus
)
device = torch.device("cuda")
torch.manual_seed(4)
num_queries = 8
num_heads = 16
q = (
torch.randn(
num_queries,
num_heads,
HEAD_DIM,
dtype=torch.bfloat16,
device=device,
)
* 0.125
)
kv = torch.randn(num_queries, HEAD_DIM, dtype=torch.bfloat16, device=device) * 0.125
rows = [list(range(query_idx + 1)) for query_idx in range(num_queries)]
indices = torch.tensor(
[index for row in rows for index in row], dtype=torch.int32, device=device
)
indptr = torch.tensor(
[0]
+ [
sum(len(row) for row in rows[: query_idx + 1])
for query_idx in range(num_queries)
],
dtype=torch.int32,
device=device,
)
attn_sink = torch.linspace(
-0.25, 0.25, num_heads, dtype=torch.float32, device=device
)
output = torch.empty_like(q)
scale = HEAD_DIM**-0.5
assert mod._rocm_sparse_attn_prefill_ragged_aiter_opus(
q=q,
kv=kv,
indices=indices,
indptr=indptr,
scale=scale,
attn_sink=attn_sink,
output=output,
)
expected = _ref_sparse_prefill_ragged(q, kv, rows, scale, attn_sink)
torch.testing.assert_close(output, expected, atol=2e-2, rtol=2e-2)
@torch.inference_mode()
def test_sparse_attn_decode_ragged_kernel() -> None:
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
@@ -13,6 +13,7 @@ import vllm.envs as envs
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import CUDAGraphMode
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import LayerNameType
@@ -26,6 +27,8 @@ else:
_ON_GFX942 = False
_ON_GFX950 = False
logger = init_logger(__name__)
@functools.cache
def _get_aiter_topk_ops() -> tuple[Callable[..., None], Callable[..., None]] | None:
@@ -39,8 +42,25 @@ def _get_aiter_topk_ops() -> tuple[Callable[..., None], Callable[..., None]] | N
return top_k_per_row_prefill, top_k_per_row_decode
@functools.cache
def _get_aiter_sparse_prefill_opus() -> Callable[..., torch.Tensor] | None:
from vllm._aiter_ops import rocm_aiter_ops
if not rocm_aiter_ops.is_mla_enabled():
return None
try:
from aiter.ops.pa_sparse_prefill_opus import pa_sparse_prefill_opus
except ImportError:
return None
logger.info_once("Using AITER OPUS for large sparse MLA prefill on gfx950")
return pa_sparse_prefill_opus
_GFX950_C4A_AITER_MAX_COMPRESSED_SEQ_LEN = 64 * 1024
_GFX950_C4A_NATIVE_MAX_ROWS = 256
# Conservative perf gate, not a correctness bound: OPUS is correct for any query
# count, but Triton stays faster below this measured crossover.
_GFX950_AITER_SPARSE_PREFILL_OPUS_MIN_QUERIES = 1024
def _get_aiter_top_k_kernel(
@@ -2667,6 +2687,66 @@ def _rocm_sparse_attn_prefill_triton(
)
def _can_use_aiter_sparse_prefill_opus(
q: torch.Tensor,
kv: torch.Tensor,
attn_sink: torch.Tensor | None,
output: torch.Tensor,
on_gfx950: bool = _ON_GFX950,
) -> bool:
return (
on_gfx950
and q.shape[0] >= _GFX950_AITER_SPARSE_PREFILL_OPUS_MIN_QUERIES
and q.is_cuda
and output.shape == q.shape
and kv.shape[-1] == q.shape[-1]
and q.dtype in (torch.bfloat16, torch.float16)
and kv.dtype == q.dtype
and output.dtype == q.dtype
and kv.device == q.device
and output.device == q.device
and q.stride(-1) == 1
and kv.stride(-1) == 1
and output.stride() == q.stride()
and attn_sink is not None
and attn_sink.shape == (q.shape[1],)
and attn_sink.dtype == torch.float32
and attn_sink.device == q.device
)
def _rocm_sparse_attn_prefill_ragged_aiter_opus(
q: torch.Tensor,
kv: torch.Tensor,
indices: torch.Tensor,
indptr: torch.Tensor,
scale: float,
attn_sink: torch.Tensor,
output: torch.Tensor,
) -> bool:
pa_sparse_prefill_opus = _get_aiter_sparse_prefill_opus()
if pa_sparse_prefill_opus is None:
return False
indices = _as_int32_contiguous_1d(indices)
indptr = _as_int32_contiguous_1d(indptr)
empty_indices = indices[:0]
empty_indptr = torch.zeros_like(indptr)
pa_sparse_prefill_opus(
q,
kv,
indices,
indptr,
kv[:1],
empty_indices,
empty_indptr,
attn_sink.contiguous(),
float(scale),
out=output,
)
return True
@functools.lru_cache
def _decode_cu_count() -> int:
try:
@@ -3170,6 +3250,33 @@ def rocm_sparse_attn_prefill(
rope_head_dim,
"rocm_sparse_attn_prefill",
)
opus_attn_sink = None if attn_sink is None else attn_sink[: q.shape[1]]
if (
_can_use_aiter_sparse_prefill_opus(q, kv.squeeze(1), opus_attn_sink, output)
and _get_aiter_sparse_prefill_opus() is not None
):
if ragged_indices is None or ragged_indptr is None:
assert indices is not None
indices_2d = indices.reshape(indices.shape[0], -1)
ragged_indices, ragged_indptr = build_ragged_indices_from_dense(
indices_2d,
topk_length
if topk_length is not None
else (indices_2d >= 0).sum(dim=-1, dtype=torch.int32),
num_rows=kv.shape[0],
)
assert opus_attn_sink is not None
if _rocm_sparse_attn_prefill_ragged_aiter_opus(
q=q,
kv=kv.squeeze(1),
indices=ragged_indices,
indptr=ragged_indptr,
scale=scale,
attn_sink=opus_attn_sink,
output=output,
):
return
if ragged_indices is not None and ragged_indptr is not None:
output_chunk = _rocm_sparse_attn_prefill_ragged_triton(
q=q,