[Perf][Kernel] Integrate DeepSelect TopK for the DSA sparse indexer (#56464)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Kimi Code <noreply@moonshot.ai>
Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
Jiangyun Zhu
2026-09-13 00:44:37 -07:00
committed by GitHub
co-authored by Kimi Code Roger Wang
parent 1cfd972816
commit fa008bdccf
8 changed files with 829 additions and 52 deletions
+323
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import numpy as np
import pytest
import torch
@@ -21,6 +22,13 @@ def _has_device_capability(major: int) -> bool:
return current_platform.is_cuda() and current_platform.has_device_capability(major)
# DeepSelect is compiled for sm_100a/sm_103a only.
requires_sm100 = pytest.mark.skipif(
not current_platform.is_device_capability_family(100),
reason="DeepSelect requires SM100a/SM103a",
)
COOPERATIVE_TOPK_BACKEND = pytest.param(
"cooperative_topk",
marks=pytest.mark.skipif(
@@ -1315,3 +1323,318 @@ def test_workspace_topk_padded_stride(top_k: int, backend: str) -> None:
f"Row {i}: {backend} with padded stride doesn't match. "
f"seq_len={sl}, stride={padded_stride}"
)
@requires_sm100
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
@pytest.mark.parametrize("batch_size", [1, 8, 64])
@pytest.mark.parametrize("vocab_size", [4096, 131072])
@pytest.mark.parametrize("top_k", [512, 2048])
@torch.inference_mode()
def test_deep_select_topk(
dtype: torch.dtype,
batch_size: int,
vocab_size: int,
top_k: int,
) -> None:
"""DeepSelect wrapper vs torch.topk with variable per-row ends."""
from vllm.model_executor.layers import indexer_topk
set_random_seed(0)
torch.set_default_device("cuda:0")
row_ends = torch.randint(
1, vocab_size + 1, (batch_size,), dtype=torch.int32, device="cuda"
)
# Force one short row to exercise the -1 out-of-bounds fill.
row_ends[0] = min(top_k - 1, vocab_size)
logits = torch.randn(batch_size, vocab_size, dtype=dtype, device="cuda")
# Values beyond each row's end must never be read by the kernel: fill
# them with NaN (production tails are uninitialized; DeepSelect traps on
# NaN with abort_when_nan_found=True, so a wrong read fails loudly).
col_idx = torch.arange(vocab_size, device="cuda")
logits[col_idx[None, :] >= row_ends[:, None]] = float("nan")
indices = indexer_topk.deep_select_topk(logits, top_k, end=row_ends)
torch.accelerator.synchronize()
assert indices.shape == (batch_size, top_k)
row_starts = torch.zeros(batch_size, dtype=torch.int32, device="cuda")
torch_indices = torch.empty((batch_size, top_k), dtype=torch.int32, device="cuda")
for i in range(batch_size):
row_end = int(row_ends[i])
k_i = min(top_k, row_end)
idx = logits[i, :row_end].topk(k_i, dim=-1)[1]
torch_indices[i, :k_i] = idx
assert torch.all(indices[i, k_i:] == -1), (
f"Row {i}: expected -1 fill after {k_i} valid indices"
)
# All selected indices must lie inside [0, row_end).
assert torch.all(indices[i, :k_i] < row_end)
assert compare_top_k_results(
logits, indices, torch_indices, row_starts, row_ends, top_k
), "DeepSelect topk results don't match torch.topk"
@requires_sm100
@torch.inference_mode()
def test_deep_select_topk_preallocated_output() -> None:
"""DeepSelect writes into a preallocated aligned buffer (no `end`)."""
from vllm.model_executor.layers import indexer_topk
set_random_seed(0)
torch.set_default_device("cuda:0")
batch_size = 8
vocab_size = 131072
top_k = 2048
logits = torch.randn(batch_size, vocab_size, dtype=torch.float32, device="cuda")
# Mimic vLLM's topk_indices_buffer: slice of a wider contiguous buffer,
# whose stride(0) is 32B-aligned.
buffer = torch.full((batch_size, 4096), -2, dtype=torch.int32, device="cuda")
output_idx = buffer[:, :top_k]
result = indexer_topk.deep_select_topk(logits, top_k, output_idx=output_idx)
torch.accelerator.synchronize()
assert result.data_ptr() == output_idx.data_ptr()
assert torch.all(buffer[:, top_k:] == -2), "kernel wrote out of bounds"
row_ends = torch.full((batch_size,), vocab_size, dtype=torch.int32, device="cuda")
row_starts = torch.zeros(batch_size, dtype=torch.int32, device="cuda")
torch_indices = torch.empty((batch_size, top_k), dtype=torch.int32, device="cuda")
for i in range(batch_size):
torch_indices[i] = logits[i].topk(top_k, dim=-1)[1]
assert compare_top_k_results(
logits, output_idx, torch_indices, row_starts, row_ends, top_k
), "DeepSelect topk (preallocated output) results don't match torch.topk"
def _has_flashinfer_topk() -> bool:
if not current_platform.is_cuda():
return False
# find_spec only: importing flashinfer initializes CUDA at import time.
import importlib.util
return importlib.util.find_spec("flashinfer") is not None
def _has_cooperative_topk() -> bool:
return _has_device_capability(
90
) and not current_platform.is_device_capability_family(120)
SPARSE_INDEXER_EXPLICIT_BACKENDS = [
pytest.param(
"per_row",
marks=pytest.mark.skipif(
not current_platform.is_cuda(), reason="requires CUDA"
),
),
"torch",
pytest.param(
"persistent",
marks=pytest.mark.skipif(
not current_platform.is_cuda(), reason="requires CUDA"
),
),
pytest.param(
"cooperative",
marks=pytest.mark.skipif(
not _has_cooperative_topk(),
reason="cooperative_topk requires SM90+ (non-SM12x)",
),
),
pytest.param(
"flashinfer",
marks=pytest.mark.skipif(
not _has_flashinfer_topk(), reason="requires flashinfer top-k"
),
),
pytest.param("deep_select", marks=requires_sm100),
]
@pytest.mark.parametrize("next_n", [1, 4])
@pytest.mark.parametrize("backend", SPARSE_INDEXER_EXPLICIT_BACKENDS)
@torch.inference_mode()
def test_sparse_indexer_decode_topk_explicit_backends(
backend: str, next_n: int, workspace_init
) -> None:
"""Every explicit sparse-indexer decode top-k backend must restrict each
row to its seq_len (dirty data past the end must never be selected) and
match torch.topk on the valid region."""
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.indexer_topk import SparseIndexerTopk
set_random_seed(0)
torch.set_default_device("cuda:0")
top_k = 2048
batch_size = 4
num_rows = batch_size * next_n
# Per-row effective lens in the (B, next_n) "native spec decode" form;
# cooperative/persistent_topk require lengths.numel() == num_rows.
row_ends = torch.randint(4000, 16000, (num_rows,), dtype=torch.int32, device="cuda")
seq_lens = row_ends.reshape(batch_size, next_n)
max_seq_len = int(row_ends.max())
# Pad the stride: multiple of 4 for cooperative_topk (TMA) and
# 1024B-aligned (256 fp32) for DeepSelect.
stride = (max_seq_len + 255) // 256 * 256
row_starts = torch.zeros(num_rows, dtype=torch.int32, device="cuda")
logits = torch.randn(num_rows, stride, dtype=torch.float32, device="cuda")
col_idx = torch.arange(stride, device="cuda")
logits[col_idx[None, :] >= row_ends[:, None]] = 1e30 # dirty tail
indices = torch.full((num_rows, top_k), -2, dtype=torch.int32, device="cuda")
cfg = VllmConfig(kernel_config={"sparse_indexer_topk_backend": backend})
with set_current_vllm_config(cfg):
SparseIndexerTopk()(logits, seq_lens, next_n, indices, top_k, max_seq_len)
torch.accelerator.synchronize()
# k_i == top_k for every row here (row_ends >= 3997 > top_k).
assert torch.all(indices >= 0)
torch_indices = torch.empty((num_rows, top_k), dtype=torch.int32, device="cuda")
for i in range(num_rows):
row_end = int(row_ends[i])
torch_indices[i] = logits[i, :row_end].topk(top_k, dim=-1)[1]
assert torch.all(indices[i] < row_end), (
f"{backend}: row {i} selected indices past its seq_len"
)
assert compare_top_k_results(
logits, indices, torch_indices, row_starts, row_ends, top_k
), f"{backend} results don't match torch.topk"
@pytest.mark.parametrize(
"backend",
[
"torch",
pytest.param(
"per_row",
marks=pytest.mark.skipif(
not current_platform.is_cuda(), reason="requires CUDA"
),
),
pytest.param(
"flashinfer",
marks=pytest.mark.skipif(
not _has_flashinfer_topk(), reason="requires flashinfer top-k"
),
),
pytest.param("deep_select", marks=requires_sm100),
],
)
@torch.inference_mode()
def test_sparse_indexer_decode_topk_short_seq_lens(
backend: str, workspace_init
) -> None:
"""seq_len < next_n: derived per-row ends must clamp at 0 (the reference
kernels do max(0, ...)); negative ends are OOB for the ragged kernels."""
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.indexer_topk import SparseIndexerTopk
set_random_seed(0)
next_n = 4
top_k = 512
vocab = 8192 # 1024B-aligned rows for DeepSelect
# 1D (B,) seq_lens: rows of a request share its len and each kernel
# derives per-row ends (with clamping) itself.
seq_lens = torch.tensor([2, 6], dtype=torch.int32, device="cuda")
num_rows = 2 * next_n
logits = torch.randn(num_rows, vocab, dtype=torch.float32, device="cuda")
indices = torch.full((num_rows, top_k), -2, dtype=torch.int32, device="cuda")
cfg = VllmConfig(kernel_config={"sparse_indexer_topk_backend": backend})
with set_current_vllm_config(cfg):
SparseIndexerTopk()(logits, seq_lens, next_n, indices, top_k, vocab)
torch.accelerator.synchronize()
# Derived ends: [2,6] - next_n + 1 + arange(next_n) -> [-1,0,1,2] and
# [3,4,5,6], clamped to [0,0,1,2] and [3,4,5,6].
for row, end in enumerate([0, 0, 1, 2, 3, 4, 5, 6]):
k_i = min(top_k, end)
assert torch.all(indices[row, k_i:] == -1), f"{backend}: row {row}"
if k_i:
ref = logits[row, :end].topk(k_i).indices.sort().values
got = indices[row, :k_i].sort().values
assert torch.equal(got, ref), f"{backend}: row {row}"
@pytest.mark.skipif(not current_platform.is_cuda(), reason="requires CUDA")
@torch.inference_mode()
def test_sparse_indexer_topk_backend_resolution() -> None:
"""Auto heuristic chain and fail-fast validation of explicit backends."""
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.indexer_topk import SparseIndexerTopk
# 1024B-aligned stride: 131072 * 4 bytes.
logits = torch.randn(64, 131072, dtype=torch.float32, device="cuda")
# 4-divisible (cooperative TMA ok) but not 1024B-aligned (DeepSelect no):
# 131076 * 4 = 524304 bytes.
unaligned_logits = torch.randn(128, 131076, dtype=torch.float32, device="cuda")[
:, :131072
]
def resolve(
backend: str,
t: torch.Tensor = logits,
k: int = 2048,
num_rows: int = 64,
) -> str:
cfg = VllmConfig(kernel_config={"sparse_indexer_topk_backend": backend})
with set_current_vllm_config(cfg):
return SparseIndexerTopk().resolve_backend(t, k, num_rows)
is_sm100 = current_platform.is_device_capability_family(100)
has_coop = _has_cooperative_topk()
has_fi = _has_flashinfer_topk()
# "auto" is exactly the pre-existing chain: cooperative -> persistent ->
# per_row. It must never select the opt-in backends by itself.
if has_coop:
assert resolve("auto") == "cooperative"
assert resolve("auto", k=512, num_rows=8) == "cooperative"
# Past cooperative's 64-row limit -> persistent, even where
# DeepSelect would be applicable.
assert resolve("auto", k=512, num_rows=128) == "persistent"
# 16B-aligned (cooperative TMA ok) but not DeepSelect-aligned:
# still cooperative at <= 64 rows...
assert resolve("auto", unaligned_logits, num_rows=64) == "cooperative"
# ...and persistent past the row limit.
assert resolve("auto", unaligned_logits, num_rows=128) == "persistent"
# Unsupported topk (outside {512, 1024, 2048} for the workspace kernels)
# -> per_row fallback.
assert resolve("auto", k=5000) == "per_row"
# Explicit values are returned as-is when constraints are met.
assert resolve("per_row") == "per_row"
assert resolve("torch") == "torch"
assert resolve("persistent") == "persistent"
if has_coop:
assert resolve("cooperative") == "cooperative"
if has_fi:
assert resolve("flashinfer") == "flashinfer"
if is_sm100:
# Explicit deep_select has no row-count requirement.
assert resolve("deep_select", num_rows=1) == "deep_select"
with pytest.raises(RuntimeError, match="constraints"):
resolve("deep_select", unaligned_logits)
# Fail-fast on unmet constraints.
with pytest.raises(RuntimeError, match="num_rows must be <= 64"):
resolve("cooperative", num_rows=128)
with pytest.raises(RuntimeError, match="topk_tokens must be in"):
resolve("persistent", k=3000)
with pytest.raises(RuntimeError, match="topk_tokens must be in"):
resolve("cooperative", k=3000)
if not is_sm100:
with pytest.raises(RuntimeError, match="SM100a/SM103a"):
resolve("deep_select")