[ROCm][Perf][GLM-5.3-Flash] Add AITER topk backend for decodes (#58167)
Signed-off-by: simondanielsson <simon.danielsson99@hotmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
751fa96a49
commit
98b29aa99a
@@ -8,10 +8,6 @@ import torch
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="CUDA-only dispatch"
|
||||
)
|
||||
|
||||
|
||||
def _require_deep_gemm() -> None:
|
||||
from vllm.utils.deep_gemm import has_deep_gemm
|
||||
@@ -20,17 +16,10 @@ def _require_deep_gemm() -> None:
|
||||
pytest.skip("kpool indexer requires DeepGEMM")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend", ["auto", "persistent", "cooperative", "torch"])
|
||||
def test_kpool_indexer_dispatches_through_shared_topk_backend(backend: str) -> None:
|
||||
"""The kpool indexer must read kernel_config.sparse_indexer_topk_backend
|
||||
and hand it to the shared SparseIndexerTopk dispatcher, rather than
|
||||
hard-coding its own cooperative/persistent/per_row choice."""
|
||||
_require_deep_gemm()
|
||||
from vllm.models.glm5next.nvidia.sparse_indexer import SparseAttnIndexerKpool
|
||||
|
||||
def _build(indexer_cls, backend: str):
|
||||
cfg = VllmConfig(kernel_config={"sparse_indexer_topk_backend": backend})
|
||||
with set_current_vllm_config(cfg):
|
||||
op = SparseAttnIndexerKpool(
|
||||
return indexer_cls(
|
||||
k_cache=None,
|
||||
quant_block_size=128,
|
||||
scale_fmt="ue8m0",
|
||||
@@ -40,4 +29,73 @@ def test_kpool_indexer_dispatches_through_shared_topk_backend(backend: str) -> N
|
||||
max_total_seq_len=8192,
|
||||
topk_indices_buffer=torch.empty(8, 2176, dtype=torch.int32, device="cuda"),
|
||||
)
|
||||
assert op.topk_backend == backend
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only dispatch")
|
||||
@pytest.mark.parametrize("backend", ["auto", "persistent", "cooperative", "torch"])
|
||||
def test_kpool_indexer_dispatches_through_shared_topk_backend(backend: str) -> None:
|
||||
"""The kpool indexer must read kernel_config.sparse_indexer_topk_backend
|
||||
and hand it to the shared SparseIndexerTopk dispatcher, rather than
|
||||
hard-coding its own cooperative/persistent/per_row choice."""
|
||||
_require_deep_gemm()
|
||||
from vllm.models.glm5next.nvidia.sparse_indexer import SparseAttnIndexerKpool
|
||||
|
||||
assert _build(SparseAttnIndexerKpool, backend).topk_backend == backend
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only dispatch")
|
||||
@pytest.mark.parametrize("backend", ["auto", "aiter", "per_row", "torch"])
|
||||
def test_kpool_indexer_dispatches_through_shared_topk_backend_rocm(
|
||||
backend: str,
|
||||
) -> None:
|
||||
"""The AMD kpool indexer must go through the same dispatcher, so the
|
||||
AITER decode top-k is reachable from kernel_config."""
|
||||
from vllm.models.glm5next.amd.sparse_indexer import SparseAttnIndexerKpool
|
||||
|
||||
assert _build(SparseAttnIndexerKpool, backend).topk_backend == backend
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only dispatch")
|
||||
@pytest.mark.parametrize(
|
||||
"configured,full_cudagraph,expected",
|
||||
[
|
||||
("auto", False, "aiter"),
|
||||
("auto", True, "auto"),
|
||||
("aiter", True, "aiter"),
|
||||
("per_row", False, "per_row"),
|
||||
],
|
||||
)
|
||||
def test_kpool_decode_topk_backend_never_auto_selects_aiter_under_full_cudagraph(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
configured: str,
|
||||
full_cudagraph: bool,
|
||||
expected: str,
|
||||
) -> None:
|
||||
"""A FULL cudagraph bakes in one kernel for every replay, so "auto" cannot
|
||||
use the context length to pick AITER and must stay on the in-tree kernel.
|
||||
Explicit backends keep working, and outside FULL the long-context heuristic
|
||||
still narrows to AITER."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.models.glm5next.amd import sparse_indexer
|
||||
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_ops, "is_indexer_top_k_enabled", lambda: True, raising=False
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_ops,
|
||||
"is_indexer_top_k_supported",
|
||||
lambda **kwargs: True,
|
||||
raising=False,
|
||||
)
|
||||
|
||||
assert (
|
||||
sparse_indexer._kpool_decode_topk_backend(
|
||||
configured,
|
||||
num_rows=8,
|
||||
max_valid_seq_len=64 * 1024,
|
||||
select_k=512,
|
||||
index_kpool=4,
|
||||
full_cudagraph=full_cudagraph,
|
||||
)
|
||||
== expected
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user