[ROCm][Perf][GLM-5.3-Flash] "Fit kpool top-k indices to AITER" with a single Triton kernel (#58008)

Signed-off-by: simondanielsson <simon.danielsson99@hotmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Simon Danielsson
2026-09-30 09:45:00 -05:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 643bbef55c
commit 2df122e65e
2 changed files with 110 additions and 13 deletions
@@ -8,6 +8,7 @@ import torch
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.attention.backends.mla import rocm_aiter_mla_sparse as sparse_mod
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
_use_rocm_sparse_triton,
@@ -26,6 +27,26 @@ def _store_sparse_kv_row_offset_kernel(slot_ptr, output_ptr, stride: tl.constexp
tl.store(output_ptr, _sparse_kv_row_offset(slot, stride))
def _fit_kpool_indices_reference(
token_indices: torch.Tensor, topk_tokens: int
) -> torch.Tensor:
history = token_indices[:, :topk_tokens]
tail = token_indices[:, topk_tokens:]
valid_history = (history >= 0).sum(dim=1)
valid_tail = (tail >= 0).sum(dim=1)
keep_history = torch.minimum(valid_history, topk_tokens - valid_tail)
columns = torch.arange(topk_tokens, device=token_indices.device).unsqueeze(0)
tail_offsets = columns - keep_history.unsqueeze(1)
tail_values = torch.gather(
tail, 1, tail_offsets.clamp(min=0, max=tail.shape[1] - 1)
)
output = torch.where(columns < keep_history.unsqueeze(1), history, tail_values)
valid_output = columns < (keep_history + valid_tail).unsqueeze(1)
return torch.where(valid_output, output, -1)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm required")
def test_fit_kpool_indices_preserves_tail_and_best_history():
token_indices = torch.tensor(
[
@@ -34,6 +55,7 @@ def test_fit_kpool_indices_preserves_tail_and_best_history():
[-1, -1, -1, -1, -1, -1, -1, -1],
],
dtype=torch.int32,
device="cuda",
)
fitted = fit_kpool_indices_to_aiter(token_indices, topk_tokens=6)
@@ -45,6 +67,28 @@ def test_fit_kpool_indices_preserves_tail_and_best_history():
]
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm required")
@pytest.mark.parametrize(
("num_tokens", "topk_tokens", "tail_width"),
[(1, 4, 1), (7, 6, 2), (13, 128, 128), (3, 2048, 128)],
)
def test_fit_kpool_indices_matches_eager_reference(num_tokens, topk_tokens, tail_width):
"""The Triton fit must reproduce the eager packing rule exactly."""
set_random_seed(topk_tokens + num_tokens)
width = topk_tokens + tail_width
token_indices = torch.randint(0, 4096, (num_tokens, width), dtype=torch.int32)
valid = torch.rand((num_tokens, width)) < 0.5
valid[0] = False
valid[-1] = True
token_indices = torch.where(valid, token_indices, -1).cuda()
fitted = fit_kpool_indices_to_aiter(token_indices, topk_tokens=topk_tokens)
torch.testing.assert_close(
fitted, _fit_kpool_indices_reference(token_indices, topk_tokens)
)
def test_fit_kpool_indices_exact_width_is_noop():
token_indices = torch.tensor([[3, 2, 1, -1]], dtype=torch.int32)