[ROCm][Perf][GLM-5.3-Flash] Add AITER topk backend for decodes (#58167)

Signed-off-by: simondanielsson <simon.danielsson99@hotmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Simon Danielsson
2026-10-02 11:47:41 +01:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 751fa96a49
commit 98b29aa99a
8 changed files with 705 additions and 240 deletions
@@ -8,10 +8,6 @@ import torch
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.platforms import current_platform
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda(), reason="CUDA-only dispatch"
)
def _require_deep_gemm() -> None:
from vllm.utils.deep_gemm import has_deep_gemm
@@ -20,17 +16,10 @@ def _require_deep_gemm() -> None:
pytest.skip("kpool indexer requires DeepGEMM")
@pytest.mark.parametrize("backend", ["auto", "persistent", "cooperative", "torch"])
def test_kpool_indexer_dispatches_through_shared_topk_backend(backend: str) -> None:
"""The kpool indexer must read kernel_config.sparse_indexer_topk_backend
and hand it to the shared SparseIndexerTopk dispatcher, rather than
hard-coding its own cooperative/persistent/per_row choice."""
_require_deep_gemm()
from vllm.models.glm5next.nvidia.sparse_indexer import SparseAttnIndexerKpool
def _build(indexer_cls, backend: str):
cfg = VllmConfig(kernel_config={"sparse_indexer_topk_backend": backend})
with set_current_vllm_config(cfg):
op = SparseAttnIndexerKpool(
return indexer_cls(
k_cache=None,
quant_block_size=128,
scale_fmt="ue8m0",
@@ -40,4 +29,73 @@ def test_kpool_indexer_dispatches_through_shared_topk_backend(backend: str) -> N
max_total_seq_len=8192,
topk_indices_buffer=torch.empty(8, 2176, dtype=torch.int32, device="cuda"),
)
assert op.topk_backend == backend
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only dispatch")
@pytest.mark.parametrize("backend", ["auto", "persistent", "cooperative", "torch"])
def test_kpool_indexer_dispatches_through_shared_topk_backend(backend: str) -> None:
"""The kpool indexer must read kernel_config.sparse_indexer_topk_backend
and hand it to the shared SparseIndexerTopk dispatcher, rather than
hard-coding its own cooperative/persistent/per_row choice."""
_require_deep_gemm()
from vllm.models.glm5next.nvidia.sparse_indexer import SparseAttnIndexerKpool
assert _build(SparseAttnIndexerKpool, backend).topk_backend == backend
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only dispatch")
@pytest.mark.parametrize("backend", ["auto", "aiter", "per_row", "torch"])
def test_kpool_indexer_dispatches_through_shared_topk_backend_rocm(
backend: str,
) -> None:
"""The AMD kpool indexer must go through the same dispatcher, so the
AITER decode top-k is reachable from kernel_config."""
from vllm.models.glm5next.amd.sparse_indexer import SparseAttnIndexerKpool
assert _build(SparseAttnIndexerKpool, backend).topk_backend == backend
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only dispatch")
@pytest.mark.parametrize(
"configured,full_cudagraph,expected",
[
("auto", False, "aiter"),
("auto", True, "auto"),
("aiter", True, "aiter"),
("per_row", False, "per_row"),
],
)
def test_kpool_decode_topk_backend_never_auto_selects_aiter_under_full_cudagraph(
monkeypatch: pytest.MonkeyPatch,
configured: str,
full_cudagraph: bool,
expected: str,
) -> None:
"""A FULL cudagraph bakes in one kernel for every replay, so "auto" cannot
use the context length to pick AITER and must stay on the in-tree kernel.
Explicit backends keep working, and outside FULL the long-context heuristic
still narrows to AITER."""
from vllm._aiter_ops import rocm_aiter_ops
from vllm.models.glm5next.amd import sparse_indexer
monkeypatch.setattr(
rocm_aiter_ops, "is_indexer_top_k_enabled", lambda: True, raising=False
)
monkeypatch.setattr(
rocm_aiter_ops,
"is_indexer_top_k_supported",
lambda **kwargs: True,
raising=False,
)
assert (
sparse_indexer._kpool_decode_topk_backend(
configured,
num_rows=8,
max_valid_seq_len=64 * 1024,
select_k=512,
index_kpool=4,
full_cudagraph=full_cudagraph,
)
== expected
)