[GLM-5.3-Flash] Route kpool indexer top-k through the shared SparseIndexerTopk dispatcher (#57546)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
@@ -1871,17 +1871,32 @@ def test_sparse_indexer_topk_backend_resolution() -> None:
|
||||
has_coop = _has_cooperative_topk()
|
||||
has_fi = _has_flashinfer_topk()
|
||||
|
||||
# "auto" is exactly the pre-existing chain: cooperative -> persistent ->
|
||||
# per_row. It must never select the opt-in backends by itself.
|
||||
# "auto" is the chain cooperative -> persistent -> per_row, with the
|
||||
# cooperative/persistent switch at cooperative_topk's cluster-wave row
|
||||
# limit. It must never select the opt-in backends by itself.
|
||||
if has_coop:
|
||||
assert resolve("auto") == "cooperative"
|
||||
from vllm.model_executor.layers.indexer_topk import (
|
||||
AUTO_COOPERATIVE_MAX_ROWS,
|
||||
)
|
||||
|
||||
# cooperative for every batch within its cluster-wave row limit...
|
||||
assert (
|
||||
resolve("auto", k=512, num_rows=AUTO_COOPERATIVE_MAX_ROWS) == "cooperative"
|
||||
)
|
||||
assert resolve("auto", k=512, num_rows=8) == "cooperative"
|
||||
# Past cooperative's 64-row limit -> persistent, even where
|
||||
# DeepSelect would be applicable.
|
||||
# ...persistent past the row limit, even where DeepSelect would be
|
||||
# applicable.
|
||||
assert (
|
||||
resolve("auto", k=512, num_rows=AUTO_COOPERATIVE_MAX_ROWS + 1)
|
||||
== "persistent"
|
||||
)
|
||||
assert resolve("auto", k=512, num_rows=128) == "persistent"
|
||||
# 16B-aligned (cooperative TMA ok) but not DeepSelect-aligned:
|
||||
# still cooperative at <= 64 rows...
|
||||
assert resolve("auto", unaligned_logits, num_rows=64) == "cooperative"
|
||||
# 16B-aligned (cooperative TMA ok) but not DeepSelect-aligned: the
|
||||
# same row-limit logic applies.
|
||||
assert (
|
||||
resolve("auto", unaligned_logits, num_rows=AUTO_COOPERATIVE_MAX_ROWS)
|
||||
== "cooperative"
|
||||
)
|
||||
# ...and persistent past the row limit.
|
||||
assert resolve("auto", unaligned_logits, num_rows=128) == "persistent"
|
||||
# Unsupported topk (outside {512, 1024, 2048} for the workspace kernels)
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Focused unit tests for the GLM decode top-k dispatch."""
|
||||
"""Tests for the GLM kpool indexer's top-k backend wiring."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.glm5next.nvidia.sparse_indexer import _use_cooperative_topk
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
@@ -13,29 +13,31 @@ pytestmark = pytest.mark.skipif(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_rows", [1, 64], ids=["1row", "64rows"])
|
||||
@torch.inference_mode()
|
||||
def test_cooperative_topk_is_selected_for_supported_batch(num_rows: int):
|
||||
logits = torch.empty(num_rows, 8192, dtype=torch.float32, device="cuda")
|
||||
assert _use_cooperative_topk(logits, 512, num_rows)
|
||||
def _require_deep_gemm() -> None:
|
||||
from vllm.utils.deep_gemm import has_deep_gemm
|
||||
|
||||
if not has_deep_gemm():
|
||||
pytest.skip("kpool indexer requires DeepGEMM")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_rows", "select_k", "stride_padding"),
|
||||
[
|
||||
(65, 512, 0),
|
||||
(1, 256, 0),
|
||||
(1, 512, 1),
|
||||
],
|
||||
ids=["too_many_rows", "unsupported_k", "unaligned_stride"],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_cooperative_topk_falls_back_on_unsupported_batch(
|
||||
num_rows: int, select_k: int, stride_padding: int
|
||||
):
|
||||
logits = torch.empty(
|
||||
num_rows, 8192 + stride_padding, dtype=torch.float32, device="cuda"
|
||||
)
|
||||
if stride_padding:
|
||||
logits = logits[:, :8192]
|
||||
assert not _use_cooperative_topk(logits, select_k, num_rows)
|
||||
@pytest.mark.parametrize("backend", ["auto", "persistent", "cooperative", "torch"])
|
||||
def test_kpool_indexer_dispatches_through_shared_topk_backend(backend: str) -> None:
|
||||
"""The kpool indexer must read kernel_config.sparse_indexer_topk_backend
|
||||
and hand it to the shared SparseIndexerTopk dispatcher, rather than
|
||||
hard-coding its own cooperative/persistent/per_row choice."""
|
||||
_require_deep_gemm()
|
||||
from vllm.models.glm5next.nvidia.sparse_indexer import SparseAttnIndexerKpool
|
||||
|
||||
cfg = VllmConfig(kernel_config={"sparse_indexer_topk_backend": backend})
|
||||
with set_current_vllm_config(cfg):
|
||||
op = SparseAttnIndexerKpool(
|
||||
k_cache=None,
|
||||
quant_block_size=128,
|
||||
scale_fmt="ue8m0",
|
||||
topk_tokens=2048,
|
||||
head_dim=128,
|
||||
max_pool_len=4096,
|
||||
max_total_seq_len=8192,
|
||||
topk_indices_buffer=torch.empty(8, 2176, dtype=torch.int32, device="cuda"),
|
||||
)
|
||||
assert op.topk_backend == backend
|
||||
|
||||
Reference in New Issue
Block a user