[ROCm][Perf] Route large DSV4 sparse prefill to AITER OPUS (#54855)
Signed-off-by: jiacao-amd <jiahui.cao@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Claude
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
442d36031c
commit
7e91760650
@@ -381,6 +381,217 @@ def test_sparse_attn_prefill_ragged_kernel() -> None:
|
||||
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_queries", "on_gfx950", "expected"),
|
||||
[(1023, True, False), (1024, True, True), (1024, False, False)],
|
||||
)
|
||||
def test_aiter_sparse_prefill_opus_selection(
|
||||
num_queries: int, on_gfx950: bool, expected: bool
|
||||
) -> None:
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_can_use_aiter_sparse_prefill_opus,
|
||||
)
|
||||
|
||||
device = torch.device("cuda")
|
||||
q = torch.empty(num_queries, 16, HEAD_DIM, dtype=torch.bfloat16, device=device)
|
||||
kv = torch.empty(num_queries, HEAD_DIM, dtype=torch.bfloat16, device=device)
|
||||
attn_sink = torch.empty(16, dtype=torch.float32, device=device)
|
||||
output = torch.empty_like(q)
|
||||
|
||||
assert (
|
||||
_can_use_aiter_sparse_prefill_opus(
|
||||
q, kv, attn_sink, output, on_gfx950=on_gfx950
|
||||
)
|
||||
is expected
|
||||
)
|
||||
|
||||
|
||||
def test_aiter_sparse_prefill_opus_selection_rejects_incompatible_inputs() -> None:
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_can_use_aiter_sparse_prefill_opus,
|
||||
)
|
||||
|
||||
device = torch.device("cuda")
|
||||
q = torch.empty(1024, 16, HEAD_DIM, dtype=torch.bfloat16, device=device)
|
||||
kv = torch.empty(1024, HEAD_DIM, dtype=torch.bfloat16, device=device)
|
||||
attn_sink = torch.empty(16, dtype=torch.float32, device=device)
|
||||
output = torch.empty_like(q)
|
||||
|
||||
assert _can_use_aiter_sparse_prefill_opus(q, kv, attn_sink, output, on_gfx950=True)
|
||||
assert not _can_use_aiter_sparse_prefill_opus(
|
||||
q,
|
||||
kv,
|
||||
attn_sink,
|
||||
torch.empty(16, 1024, HEAD_DIM, dtype=q.dtype, device=device).transpose(0, 1),
|
||||
on_gfx950=True,
|
||||
)
|
||||
assert not _can_use_aiter_sparse_prefill_opus(
|
||||
q, kv, attn_sink[:-1], output, on_gfx950=True
|
||||
)
|
||||
assert not _can_use_aiter_sparse_prefill_opus(
|
||||
q.cpu(), kv.cpu(), attn_sink.cpu(), output.cpu(), on_gfx950=True
|
||||
)
|
||||
|
||||
|
||||
def test_sparse_attn_prefill_aiter_opus_routing(monkeypatch) -> None:
|
||||
from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod
|
||||
|
||||
q = torch.empty(2, 1, HEAD_DIM, dtype=torch.bfloat16)
|
||||
kv = torch.empty(2, 1, HEAD_DIM, dtype=torch.bfloat16)
|
||||
indices = torch.tensor([[0], [1]], dtype=torch.int32)
|
||||
topk_length = torch.ones(2, dtype=torch.int32)
|
||||
attn_sink = torch.empty(1, dtype=torch.float32)
|
||||
output = torch.empty_like(q)
|
||||
opus_calls = 0
|
||||
|
||||
def fake_opus(*args, out):
|
||||
nonlocal opus_calls
|
||||
opus_calls += 1
|
||||
assert args[2].dtype == torch.int32
|
||||
assert args[3].dtype == torch.int32
|
||||
assert args[5].numel() == 0
|
||||
assert torch.count_nonzero(args[6]) == 0
|
||||
out.zero_()
|
||||
return out
|
||||
|
||||
monkeypatch.setattr(mod, "_can_use_aiter_sparse_prefill_opus", lambda *args: True)
|
||||
monkeypatch.setattr(mod, "_get_aiter_sparse_prefill_opus", lambda: fake_opus)
|
||||
monkeypatch.setattr(
|
||||
mod,
|
||||
"build_ragged_indices_from_dense",
|
||||
lambda *args, **kwargs: (
|
||||
torch.tensor([0, 1], dtype=torch.int32),
|
||||
torch.tensor([0, 1, 2], dtype=torch.int32),
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
mod,
|
||||
"_rocm_sparse_attn_prefill_triton",
|
||||
lambda *args, **kwargs: pytest.fail("unexpected dense Triton fallback"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
mod,
|
||||
"_rocm_sparse_attn_prefill_ragged_triton",
|
||||
lambda *args, **kwargs: pytest.fail("unexpected ragged Triton fallback"),
|
||||
)
|
||||
|
||||
mod.rocm_sparse_attn_prefill(
|
||||
q=q,
|
||||
kv=kv,
|
||||
indices=indices,
|
||||
topk_length=topk_length,
|
||||
scale=HEAD_DIM**-0.5,
|
||||
head_dim=HEAD_DIM,
|
||||
nope_head_dim=NOPE_HEAD_DIM,
|
||||
rope_head_dim=ROPE_HEAD_DIM,
|
||||
attn_sink=attn_sink,
|
||||
output=output,
|
||||
)
|
||||
|
||||
assert opus_calls == 1
|
||||
assert torch.count_nonzero(output) == 0
|
||||
|
||||
|
||||
def test_sparse_attn_prefill_preserves_dense_triton_fallback(monkeypatch) -> None:
|
||||
from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod
|
||||
|
||||
q = torch.empty(2, 1, HEAD_DIM, dtype=torch.bfloat16)
|
||||
kv = torch.empty(2, 1, HEAD_DIM, dtype=torch.bfloat16)
|
||||
indices = torch.tensor([[0], [1]], dtype=torch.int32)
|
||||
topk_length = torch.ones(2, dtype=torch.int32)
|
||||
attn_sink = torch.empty(1, dtype=torch.float32)
|
||||
output = torch.empty_like(q)
|
||||
dense_fallback_calls = 0
|
||||
|
||||
def fake_dense_fallback(*args, **kwargs):
|
||||
nonlocal dense_fallback_calls
|
||||
dense_fallback_calls += 1
|
||||
return torch.zeros_like(q)
|
||||
|
||||
monkeypatch.setattr(mod, "_can_use_aiter_sparse_prefill_opus", lambda *args: True)
|
||||
monkeypatch.setattr(mod, "_get_aiter_sparse_prefill_opus", lambda: None)
|
||||
monkeypatch.setattr(mod, "_rocm_sparse_attn_prefill_triton", fake_dense_fallback)
|
||||
monkeypatch.setattr(
|
||||
mod,
|
||||
"_rocm_sparse_attn_prefill_ragged_triton",
|
||||
lambda *args, **kwargs: pytest.fail("unexpected ragged Triton fallback"),
|
||||
)
|
||||
|
||||
mod.rocm_sparse_attn_prefill(
|
||||
q=q,
|
||||
kv=kv,
|
||||
indices=indices,
|
||||
topk_length=topk_length,
|
||||
scale=HEAD_DIM**-0.5,
|
||||
head_dim=HEAD_DIM,
|
||||
nope_head_dim=NOPE_HEAD_DIM,
|
||||
rope_head_dim=ROPE_HEAD_DIM,
|
||||
attn_sink=attn_sink,
|
||||
output=output,
|
||||
)
|
||||
|
||||
assert dense_fallback_calls == 1
|
||||
assert torch.count_nonzero(output) == 0
|
||||
|
||||
|
||||
@requires_gfx950
|
||||
@torch.inference_mode()
|
||||
def test_sparse_attn_prefill_ragged_aiter_opus(monkeypatch) -> None:
|
||||
opus_mod = pytest.importorskip("aiter.ops.pa_sparse_prefill_opus")
|
||||
from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod
|
||||
|
||||
monkeypatch.setattr(
|
||||
mod, "_get_aiter_sparse_prefill_opus", lambda: opus_mod.pa_sparse_prefill_opus
|
||||
)
|
||||
|
||||
device = torch.device("cuda")
|
||||
torch.manual_seed(4)
|
||||
num_queries = 8
|
||||
num_heads = 16
|
||||
q = (
|
||||
torch.randn(
|
||||
num_queries,
|
||||
num_heads,
|
||||
HEAD_DIM,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
* 0.125
|
||||
)
|
||||
kv = torch.randn(num_queries, HEAD_DIM, dtype=torch.bfloat16, device=device) * 0.125
|
||||
rows = [list(range(query_idx + 1)) for query_idx in range(num_queries)]
|
||||
indices = torch.tensor(
|
||||
[index for row in rows for index in row], dtype=torch.int32, device=device
|
||||
)
|
||||
indptr = torch.tensor(
|
||||
[0]
|
||||
+ [
|
||||
sum(len(row) for row in rows[: query_idx + 1])
|
||||
for query_idx in range(num_queries)
|
||||
],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
attn_sink = torch.linspace(
|
||||
-0.25, 0.25, num_heads, dtype=torch.float32, device=device
|
||||
)
|
||||
output = torch.empty_like(q)
|
||||
scale = HEAD_DIM**-0.5
|
||||
|
||||
assert mod._rocm_sparse_attn_prefill_ragged_aiter_opus(
|
||||
q=q,
|
||||
kv=kv,
|
||||
indices=indices,
|
||||
indptr=indptr,
|
||||
scale=scale,
|
||||
attn_sink=attn_sink,
|
||||
output=output,
|
||||
)
|
||||
expected = _ref_sparse_prefill_ragged(q, kv, rows, scale, attn_sink)
|
||||
|
||||
torch.testing.assert_close(output, expected, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_sparse_attn_decode_ragged_kernel() -> None:
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
|
||||
@@ -13,6 +13,7 @@ import vllm.envs as envs
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import LayerNameType
|
||||
@@ -26,6 +27,8 @@ else:
|
||||
_ON_GFX942 = False
|
||||
_ON_GFX950 = False
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def _get_aiter_topk_ops() -> tuple[Callable[..., None], Callable[..., None]] | None:
|
||||
@@ -39,8 +42,25 @@ def _get_aiter_topk_ops() -> tuple[Callable[..., None], Callable[..., None]] | N
|
||||
return top_k_per_row_prefill, top_k_per_row_decode
|
||||
|
||||
|
||||
@functools.cache
|
||||
def _get_aiter_sparse_prefill_opus() -> Callable[..., torch.Tensor] | None:
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
if not rocm_aiter_ops.is_mla_enabled():
|
||||
return None
|
||||
try:
|
||||
from aiter.ops.pa_sparse_prefill_opus import pa_sparse_prefill_opus
|
||||
except ImportError:
|
||||
return None
|
||||
logger.info_once("Using AITER OPUS for large sparse MLA prefill on gfx950")
|
||||
return pa_sparse_prefill_opus
|
||||
|
||||
|
||||
_GFX950_C4A_AITER_MAX_COMPRESSED_SEQ_LEN = 64 * 1024
|
||||
_GFX950_C4A_NATIVE_MAX_ROWS = 256
|
||||
# Conservative perf gate, not a correctness bound: OPUS is correct for any query
|
||||
# count, but Triton stays faster below this measured crossover.
|
||||
_GFX950_AITER_SPARSE_PREFILL_OPUS_MIN_QUERIES = 1024
|
||||
|
||||
|
||||
def _get_aiter_top_k_kernel(
|
||||
@@ -2667,6 +2687,66 @@ def _rocm_sparse_attn_prefill_triton(
|
||||
)
|
||||
|
||||
|
||||
def _can_use_aiter_sparse_prefill_opus(
|
||||
q: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
attn_sink: torch.Tensor | None,
|
||||
output: torch.Tensor,
|
||||
on_gfx950: bool = _ON_GFX950,
|
||||
) -> bool:
|
||||
return (
|
||||
on_gfx950
|
||||
and q.shape[0] >= _GFX950_AITER_SPARSE_PREFILL_OPUS_MIN_QUERIES
|
||||
and q.is_cuda
|
||||
and output.shape == q.shape
|
||||
and kv.shape[-1] == q.shape[-1]
|
||||
and q.dtype in (torch.bfloat16, torch.float16)
|
||||
and kv.dtype == q.dtype
|
||||
and output.dtype == q.dtype
|
||||
and kv.device == q.device
|
||||
and output.device == q.device
|
||||
and q.stride(-1) == 1
|
||||
and kv.stride(-1) == 1
|
||||
and output.stride() == q.stride()
|
||||
and attn_sink is not None
|
||||
and attn_sink.shape == (q.shape[1],)
|
||||
and attn_sink.dtype == torch.float32
|
||||
and attn_sink.device == q.device
|
||||
)
|
||||
|
||||
|
||||
def _rocm_sparse_attn_prefill_ragged_aiter_opus(
|
||||
q: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
indptr: torch.Tensor,
|
||||
scale: float,
|
||||
attn_sink: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
) -> bool:
|
||||
pa_sparse_prefill_opus = _get_aiter_sparse_prefill_opus()
|
||||
if pa_sparse_prefill_opus is None:
|
||||
return False
|
||||
|
||||
indices = _as_int32_contiguous_1d(indices)
|
||||
indptr = _as_int32_contiguous_1d(indptr)
|
||||
empty_indices = indices[:0]
|
||||
empty_indptr = torch.zeros_like(indptr)
|
||||
pa_sparse_prefill_opus(
|
||||
q,
|
||||
kv,
|
||||
indices,
|
||||
indptr,
|
||||
kv[:1],
|
||||
empty_indices,
|
||||
empty_indptr,
|
||||
attn_sink.contiguous(),
|
||||
float(scale),
|
||||
out=output,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
@functools.lru_cache
|
||||
def _decode_cu_count() -> int:
|
||||
try:
|
||||
@@ -3170,6 +3250,33 @@ def rocm_sparse_attn_prefill(
|
||||
rope_head_dim,
|
||||
"rocm_sparse_attn_prefill",
|
||||
)
|
||||
opus_attn_sink = None if attn_sink is None else attn_sink[: q.shape[1]]
|
||||
if (
|
||||
_can_use_aiter_sparse_prefill_opus(q, kv.squeeze(1), opus_attn_sink, output)
|
||||
and _get_aiter_sparse_prefill_opus() is not None
|
||||
):
|
||||
if ragged_indices is None or ragged_indptr is None:
|
||||
assert indices is not None
|
||||
indices_2d = indices.reshape(indices.shape[0], -1)
|
||||
ragged_indices, ragged_indptr = build_ragged_indices_from_dense(
|
||||
indices_2d,
|
||||
topk_length
|
||||
if topk_length is not None
|
||||
else (indices_2d >= 0).sum(dim=-1, dtype=torch.int32),
|
||||
num_rows=kv.shape[0],
|
||||
)
|
||||
assert opus_attn_sink is not None
|
||||
if _rocm_sparse_attn_prefill_ragged_aiter_opus(
|
||||
q=q,
|
||||
kv=kv.squeeze(1),
|
||||
indices=ragged_indices,
|
||||
indptr=ragged_indptr,
|
||||
scale=scale,
|
||||
attn_sink=opus_attn_sink,
|
||||
output=output,
|
||||
):
|
||||
return
|
||||
|
||||
if ragged_indices is not None and ragged_indptr is not None:
|
||||
output_chunk = _rocm_sparse_attn_prefill_ragged_triton(
|
||||
q=q,
|
||||
|
||||
Reference in New Issue
Block a user