[ROCm][Perf] Optimize DSV4.1 K=512 decode top-k on gfx950 (#56743)

Signed-off-by: fai <fangzhouai@gmail.com>
This commit is contained in:
Fangzhou Ai
2026-09-15 18:34:24 +00:00
committed by GitHub
parent 2f46a1d9c6
commit dffbb714e4
3 changed files with 406 additions and 58 deletions
@@ -58,6 +58,7 @@ def _get_aiter_sparse_prefill_opus() -> Callable[..., torch.Tensor] | None:
_GFX950_C4A_AITER_MAX_COMPRESSED_SEQ_LEN = 64 * 1024
_GFX950_C4A_NATIVE_MAX_ROWS = 256
_GFX950_DSV4_NATIVE_MAX_COLUMNS = 1024 * 1024
# Conservative perf gate, not a correctness bound: OPUS is correct for any query
# count, but Triton stays faster below this measured crossover.
_GFX950_AITER_SPARSE_PREFILL_OPUS_MIN_QUERIES = 1024
@@ -69,6 +70,8 @@ def _get_aiter_top_k_kernel(
compress_ratio: int,
num_rows: int,
max_valid_seq_len: int | None = None,
num_columns: int | None = None,
topk_tokens: int = 1024,
on_gfx950: bool = _ON_GFX950,
) -> Callable[..., None] | None:
if compress_ratio <= 1 or not on_gfx950:
@@ -76,6 +79,13 @@ def _get_aiter_top_k_kernel(
if not is_prefill:
assert max_valid_seq_len is not None
if (
topk_tokens == 512
and 0 < num_rows <= 384
and num_columns is not None
and num_columns <= _GFX950_DSV4_NATIVE_MAX_COLUMNS
):
return None
# AITER v0.1.19 decode is one-block only. This measured gfx950
# FP32/k=1024 compressed-row boundary is independent of the native
# split-count boundary in sampler.cu.
@@ -1261,6 +1271,8 @@ def rocm_aiter_sparse_attn_indexer(
compress_ratio=compress_ratio,
num_rows=num_rows,
max_valid_seq_len=max_compressed_seq_len,
num_columns=logits.shape[1],
topk_tokens=topk_tokens,
)
if aiter_topk_kernel is not None:
_launch_aiter_top_k_per_row_decode(