[ROCm][Perf] Optimize DSV4.1 K=512 decode top-k on gfx950 (#56743)
Signed-off-by: fai <fangzhouai@gmail.com>
This commit is contained in:
@@ -58,6 +58,7 @@ def _get_aiter_sparse_prefill_opus() -> Callable[..., torch.Tensor] | None:
|
||||
|
||||
_GFX950_C4A_AITER_MAX_COMPRESSED_SEQ_LEN = 64 * 1024
|
||||
_GFX950_C4A_NATIVE_MAX_ROWS = 256
|
||||
_GFX950_DSV4_NATIVE_MAX_COLUMNS = 1024 * 1024
|
||||
# Conservative perf gate, not a correctness bound: OPUS is correct for any query
|
||||
# count, but Triton stays faster below this measured crossover.
|
||||
_GFX950_AITER_SPARSE_PREFILL_OPUS_MIN_QUERIES = 1024
|
||||
@@ -69,6 +70,8 @@ def _get_aiter_top_k_kernel(
|
||||
compress_ratio: int,
|
||||
num_rows: int,
|
||||
max_valid_seq_len: int | None = None,
|
||||
num_columns: int | None = None,
|
||||
topk_tokens: int = 1024,
|
||||
on_gfx950: bool = _ON_GFX950,
|
||||
) -> Callable[..., None] | None:
|
||||
if compress_ratio <= 1 or not on_gfx950:
|
||||
@@ -76,6 +79,13 @@ def _get_aiter_top_k_kernel(
|
||||
|
||||
if not is_prefill:
|
||||
assert max_valid_seq_len is not None
|
||||
if (
|
||||
topk_tokens == 512
|
||||
and 0 < num_rows <= 384
|
||||
and num_columns is not None
|
||||
and num_columns <= _GFX950_DSV4_NATIVE_MAX_COLUMNS
|
||||
):
|
||||
return None
|
||||
# AITER v0.1.19 decode is one-block only. This measured gfx950
|
||||
# FP32/k=1024 compressed-row boundary is independent of the native
|
||||
# split-count boundary in sampler.cu.
|
||||
@@ -1261,6 +1271,8 @@ def rocm_aiter_sparse_attn_indexer(
|
||||
compress_ratio=compress_ratio,
|
||||
num_rows=num_rows,
|
||||
max_valid_seq_len=max_compressed_seq_len,
|
||||
num_columns=logits.shape[1],
|
||||
topk_tokens=topk_tokens,
|
||||
)
|
||||
if aiter_topk_kernel is not None:
|
||||
_launch_aiter_top_k_per_row_decode(
|
||||
|
||||
Reference in New Issue
Block a user