[ROCm][Perf][GLM-5.3-Flash] BF16 splitk kernel for sparse MLA decode (#58584)

Signed-off-by: simondanielsson <simon.danielsson99@hotmail.com>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
Simon Danielsson
2026-10-07 21:37:51 -05:00
committed by GitHub
co-authored by Douglas Lehr
parent 3722645539
commit 87d9996abe
5 changed files with 627 additions and 13 deletions
@@ -156,6 +156,8 @@ def test_sparse_mla_sink_matches_ragged_reference(
num_decodes=batch_size,
num_decode_tokens=batch_size,
max_query_len=1,
max_seq_len=max(seq_lens),
topk_tokens=max(seq_lens),
)
sinks = torch.linspace(-2.0, 6.0, real_heads, device=device)
@@ -367,12 +369,15 @@ def test_sparse_mla_sink_matches_dense_attention_with_empty_rows_and_paged_cache
for page, offset in rows
]
lengths = torch.tensor([0, *(len(rows) for rows in selected_rows)])
longest_row = max(len(rows) for rows in selected_rows)
metadata = SimpleNamespace(
block_size=block_size,
num_prefills=0,
num_decodes=len(selected_rows),
num_decode_tokens=len(selected_rows),
max_query_len=1,
max_seq_len=longest_row,
topk_tokens=longest_row,
qo_indptr=torch.arange(
len(selected_rows) + 1, dtype=torch.int32, device="cuda"
),
@@ -479,6 +484,7 @@ def test_sparse_mla_sink_forward_mqa_preserves_split_query(dtype):
num_decodes=num_tokens,
num_decode_tokens=num_tokens,
max_query_len=1,
max_seq_len=kv.shape[0],
block_size=block_size,
topk_tokens=impl.topk_indices_buffer.shape[1],
req_id_per_token=torch.zeros(num_tokens, dtype=torch.int32, device="cuda"),
@@ -596,6 +596,87 @@ def test_sparse_attn_prefill_ragged_kernel() -> None:
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
NOPE_ONLY_HEAD_DIM = 512
def _nope_only_ragged_inputs(
kv_lens: list[int], num_heads: int, device: torch.device
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Build a GLM-like 512+0 NoPE decode batch with ragged KV segments."""
num_kv = 4096
kv = (
torch.randn(num_kv, NOPE_ONLY_HEAD_DIM, dtype=torch.bfloat16, device=device)
* 0.125
)
q = (
torch.randn(
len(kv_lens),
num_heads,
NOPE_ONLY_HEAD_DIM,
dtype=torch.bfloat16,
device=device,
)
* 0.125
)
rows = [torch.randperm(num_kv)[:length].tolist() for length in kv_lens]
indices, indptr = _ragged_from_rows(rows, device)
return q, kv, indices, indptr
# Lengths below the split count leave splits empty; non-multiples force a tail.
_NOPE_ONLY_KV_LENS = [0, 1, 3, 37, 129, 2048]
@requires_split_decode_arch
@pytest.mark.parametrize("num_splits", [2, 32])
@pytest.mark.parametrize("with_sink", [True, False])
@torch.inference_mode()
def test_sparse_attn_decode_bf16_split_k_matches_ragged(
num_splits: int, with_sink: bool
) -> None:
"""Split-K decode must reproduce the single-workgroup ragged kernel."""
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_rocm_sparse_attn_decode_ragged_bf16_triton,
_rocm_sparse_attn_prefill_ragged_triton,
)
device = torch.device("cuda")
set_random_seed(0)
num_heads = 16
q, kv, indices, indptr = _nope_only_ragged_inputs(
_NOPE_ONLY_KV_LENS, num_heads, device
)
attn_sink = (
torch.randn(num_heads, dtype=torch.float32, device=device)
if with_sink
else None
)
scale = NOPE_ONLY_HEAD_DIM**-0.5
expected = _rocm_sparse_attn_prefill_ragged_triton(
q=q,
kv=kv,
indices=indices,
indptr=indptr,
scale=scale,
attn_sink=attn_sink,
nope_head_dim=NOPE_ONLY_HEAD_DIM,
rope_head_dim=0,
)
actual = _rocm_sparse_attn_decode_ragged_bf16_triton(
q=q,
kv=kv,
indices=indices,
indptr=indptr,
scale=scale,
attn_sink=attn_sink,
nope_head_dim=NOPE_ONLY_HEAD_DIM,
rope_head_dim=0,
num_splits=num_splits,
)
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
def _sparse_prefill_ragged_inputs(nope_dim: int, rope_dim: int) -> dict:
device = torch.device("cuda")
set_random_seed(7)
@@ -1105,6 +1186,21 @@ def test_decode_num_splits_gfx950(monkeypatch) -> None:
assert mod._decode_gfx950_num_splits(1, 1, 128, 8192) == 32
assert mod._decode_gfx950_num_splits(17, 1, 128, 32) == 4
assert mod._decode_gfx950_num_splits(512, 1, 128, 7812) == 1
assert mod._decode_gfx950_num_splits(4, 1, 2048.0, 0.0, 32) == 32
def test_sparse_decode_bf16_num_splits_floor_and_tile_clamp(monkeypatch) -> None:
"""Short rows skip splitting, and splits never exceed the tiles they walk."""
from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod
monkeypatch.setattr(mod, "_decode_num_splits", lambda *args: 32)
monkeypatch.setattr(mod, "_decode_gfx950_num_splits", lambda *args: 32)
floor = mod._SPARSE_DECODE_BF16_MIN_SPLIT_LEN
block_k = mod._SPARSE_DECODE_BF16_BLOCK_K
assert mod.rocm_sparse_decode_bf16_num_splits(1, 1, floor - 1) == 1
assert mod.rocm_sparse_decode_bf16_num_splits(1, 1, floor) == floor // block_k
assert mod.rocm_sparse_decode_bf16_num_splits(1, 1, 2048) == 32
@requires_split_decode_arch