[ROCm][Perf][GLM-5.3-Flash] BF16 splitk kernel for sparse MLA decode (#58584)
Signed-off-by: simondanielsson <simon.danielsson99@hotmail.com> Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
co-authored by
Douglas Lehr
parent
3722645539
commit
87d9996abe
@@ -156,6 +156,8 @@ def test_sparse_mla_sink_matches_ragged_reference(
|
||||
num_decodes=batch_size,
|
||||
num_decode_tokens=batch_size,
|
||||
max_query_len=1,
|
||||
max_seq_len=max(seq_lens),
|
||||
topk_tokens=max(seq_lens),
|
||||
)
|
||||
sinks = torch.linspace(-2.0, 6.0, real_heads, device=device)
|
||||
|
||||
@@ -367,12 +369,15 @@ def test_sparse_mla_sink_matches_dense_attention_with_empty_rows_and_paged_cache
|
||||
for page, offset in rows
|
||||
]
|
||||
lengths = torch.tensor([0, *(len(rows) for rows in selected_rows)])
|
||||
longest_row = max(len(rows) for rows in selected_rows)
|
||||
metadata = SimpleNamespace(
|
||||
block_size=block_size,
|
||||
num_prefills=0,
|
||||
num_decodes=len(selected_rows),
|
||||
num_decode_tokens=len(selected_rows),
|
||||
max_query_len=1,
|
||||
max_seq_len=longest_row,
|
||||
topk_tokens=longest_row,
|
||||
qo_indptr=torch.arange(
|
||||
len(selected_rows) + 1, dtype=torch.int32, device="cuda"
|
||||
),
|
||||
@@ -479,6 +484,7 @@ def test_sparse_mla_sink_forward_mqa_preserves_split_query(dtype):
|
||||
num_decodes=num_tokens,
|
||||
num_decode_tokens=num_tokens,
|
||||
max_query_len=1,
|
||||
max_seq_len=kv.shape[0],
|
||||
block_size=block_size,
|
||||
topk_tokens=impl.topk_indices_buffer.shape[1],
|
||||
req_id_per_token=torch.zeros(num_tokens, dtype=torch.int32, device="cuda"),
|
||||
|
||||
@@ -596,6 +596,87 @@ def test_sparse_attn_prefill_ragged_kernel() -> None:
|
||||
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
NOPE_ONLY_HEAD_DIM = 512
|
||||
|
||||
|
||||
def _nope_only_ragged_inputs(
|
||||
kv_lens: list[int], num_heads: int, device: torch.device
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Build a GLM-like 512+0 NoPE decode batch with ragged KV segments."""
|
||||
num_kv = 4096
|
||||
kv = (
|
||||
torch.randn(num_kv, NOPE_ONLY_HEAD_DIM, dtype=torch.bfloat16, device=device)
|
||||
* 0.125
|
||||
)
|
||||
q = (
|
||||
torch.randn(
|
||||
len(kv_lens),
|
||||
num_heads,
|
||||
NOPE_ONLY_HEAD_DIM,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
* 0.125
|
||||
)
|
||||
rows = [torch.randperm(num_kv)[:length].tolist() for length in kv_lens]
|
||||
indices, indptr = _ragged_from_rows(rows, device)
|
||||
return q, kv, indices, indptr
|
||||
|
||||
|
||||
# Lengths below the split count leave splits empty; non-multiples force a tail.
|
||||
_NOPE_ONLY_KV_LENS = [0, 1, 3, 37, 129, 2048]
|
||||
|
||||
|
||||
@requires_split_decode_arch
|
||||
@pytest.mark.parametrize("num_splits", [2, 32])
|
||||
@pytest.mark.parametrize("with_sink", [True, False])
|
||||
@torch.inference_mode()
|
||||
def test_sparse_attn_decode_bf16_split_k_matches_ragged(
|
||||
num_splits: int, with_sink: bool
|
||||
) -> None:
|
||||
"""Split-K decode must reproduce the single-workgroup ragged kernel."""
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_rocm_sparse_attn_decode_ragged_bf16_triton,
|
||||
_rocm_sparse_attn_prefill_ragged_triton,
|
||||
)
|
||||
|
||||
device = torch.device("cuda")
|
||||
set_random_seed(0)
|
||||
num_heads = 16
|
||||
q, kv, indices, indptr = _nope_only_ragged_inputs(
|
||||
_NOPE_ONLY_KV_LENS, num_heads, device
|
||||
)
|
||||
attn_sink = (
|
||||
torch.randn(num_heads, dtype=torch.float32, device=device)
|
||||
if with_sink
|
||||
else None
|
||||
)
|
||||
scale = NOPE_ONLY_HEAD_DIM**-0.5
|
||||
|
||||
expected = _rocm_sparse_attn_prefill_ragged_triton(
|
||||
q=q,
|
||||
kv=kv,
|
||||
indices=indices,
|
||||
indptr=indptr,
|
||||
scale=scale,
|
||||
attn_sink=attn_sink,
|
||||
nope_head_dim=NOPE_ONLY_HEAD_DIM,
|
||||
rope_head_dim=0,
|
||||
)
|
||||
actual = _rocm_sparse_attn_decode_ragged_bf16_triton(
|
||||
q=q,
|
||||
kv=kv,
|
||||
indices=indices,
|
||||
indptr=indptr,
|
||||
scale=scale,
|
||||
attn_sink=attn_sink,
|
||||
nope_head_dim=NOPE_ONLY_HEAD_DIM,
|
||||
rope_head_dim=0,
|
||||
num_splits=num_splits,
|
||||
)
|
||||
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
def _sparse_prefill_ragged_inputs(nope_dim: int, rope_dim: int) -> dict:
|
||||
device = torch.device("cuda")
|
||||
set_random_seed(7)
|
||||
@@ -1105,6 +1186,21 @@ def test_decode_num_splits_gfx950(monkeypatch) -> None:
|
||||
assert mod._decode_gfx950_num_splits(1, 1, 128, 8192) == 32
|
||||
assert mod._decode_gfx950_num_splits(17, 1, 128, 32) == 4
|
||||
assert mod._decode_gfx950_num_splits(512, 1, 128, 7812) == 1
|
||||
assert mod._decode_gfx950_num_splits(4, 1, 2048.0, 0.0, 32) == 32
|
||||
|
||||
|
||||
def test_sparse_decode_bf16_num_splits_floor_and_tile_clamp(monkeypatch) -> None:
|
||||
"""Short rows skip splitting, and splits never exceed the tiles they walk."""
|
||||
from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod
|
||||
|
||||
monkeypatch.setattr(mod, "_decode_num_splits", lambda *args: 32)
|
||||
monkeypatch.setattr(mod, "_decode_gfx950_num_splits", lambda *args: 32)
|
||||
|
||||
floor = mod._SPARSE_DECODE_BF16_MIN_SPLIT_LEN
|
||||
block_k = mod._SPARSE_DECODE_BF16_BLOCK_K
|
||||
assert mod.rocm_sparse_decode_bf16_num_splits(1, 1, floor - 1) == 1
|
||||
assert mod.rocm_sparse_decode_bf16_num_splits(1, 1, floor) == floor // block_k
|
||||
assert mod.rocm_sparse_decode_bf16_num_splits(1, 1, 2048) == 32
|
||||
|
||||
|
||||
@requires_split_decode_arch
|
||||
|
||||
Reference in New Issue
Block a user