[Perf][Kernel] Integrate DeepSelect TopK for the DSA sparse indexer (#56464)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Roger Wang <hey@rogerw.io> Co-authored-by: Kimi Code <noreply@moonshot.ai> Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
co-authored by
Kimi Code
Roger Wang
parent
1cfd972816
commit
fa008bdccf
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
@@ -21,6 +22,13 @@ def _has_device_capability(major: int) -> bool:
|
||||
return current_platform.is_cuda() and current_platform.has_device_capability(major)
|
||||
|
||||
|
||||
# DeepSelect is compiled for sm_100a/sm_103a only.
|
||||
requires_sm100 = pytest.mark.skipif(
|
||||
not current_platform.is_device_capability_family(100),
|
||||
reason="DeepSelect requires SM100a/SM103a",
|
||||
)
|
||||
|
||||
|
||||
COOPERATIVE_TOPK_BACKEND = pytest.param(
|
||||
"cooperative_topk",
|
||||
marks=pytest.mark.skipif(
|
||||
@@ -1315,3 +1323,318 @@ def test_workspace_topk_padded_stride(top_k: int, backend: str) -> None:
|
||||
f"Row {i}: {backend} with padded stride doesn't match. "
|
||||
f"seq_len={sl}, stride={padded_stride}"
|
||||
)
|
||||
|
||||
|
||||
@requires_sm100
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
|
||||
@pytest.mark.parametrize("batch_size", [1, 8, 64])
|
||||
@pytest.mark.parametrize("vocab_size", [4096, 131072])
|
||||
@pytest.mark.parametrize("top_k", [512, 2048])
|
||||
@torch.inference_mode()
|
||||
def test_deep_select_topk(
|
||||
dtype: torch.dtype,
|
||||
batch_size: int,
|
||||
vocab_size: int,
|
||||
top_k: int,
|
||||
) -> None:
|
||||
"""DeepSelect wrapper vs torch.topk with variable per-row ends."""
|
||||
from vllm.model_executor.layers import indexer_topk
|
||||
|
||||
set_random_seed(0)
|
||||
torch.set_default_device("cuda:0")
|
||||
|
||||
row_ends = torch.randint(
|
||||
1, vocab_size + 1, (batch_size,), dtype=torch.int32, device="cuda"
|
||||
)
|
||||
# Force one short row to exercise the -1 out-of-bounds fill.
|
||||
row_ends[0] = min(top_k - 1, vocab_size)
|
||||
logits = torch.randn(batch_size, vocab_size, dtype=dtype, device="cuda")
|
||||
# Values beyond each row's end must never be read by the kernel: fill
|
||||
# them with NaN (production tails are uninitialized; DeepSelect traps on
|
||||
# NaN with abort_when_nan_found=True, so a wrong read fails loudly).
|
||||
col_idx = torch.arange(vocab_size, device="cuda")
|
||||
logits[col_idx[None, :] >= row_ends[:, None]] = float("nan")
|
||||
|
||||
indices = indexer_topk.deep_select_topk(logits, top_k, end=row_ends)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
assert indices.shape == (batch_size, top_k)
|
||||
row_starts = torch.zeros(batch_size, dtype=torch.int32, device="cuda")
|
||||
torch_indices = torch.empty((batch_size, top_k), dtype=torch.int32, device="cuda")
|
||||
for i in range(batch_size):
|
||||
row_end = int(row_ends[i])
|
||||
k_i = min(top_k, row_end)
|
||||
idx = logits[i, :row_end].topk(k_i, dim=-1)[1]
|
||||
torch_indices[i, :k_i] = idx
|
||||
assert torch.all(indices[i, k_i:] == -1), (
|
||||
f"Row {i}: expected -1 fill after {k_i} valid indices"
|
||||
)
|
||||
# All selected indices must lie inside [0, row_end).
|
||||
assert torch.all(indices[i, :k_i] < row_end)
|
||||
|
||||
assert compare_top_k_results(
|
||||
logits, indices, torch_indices, row_starts, row_ends, top_k
|
||||
), "DeepSelect topk results don't match torch.topk"
|
||||
|
||||
|
||||
@requires_sm100
|
||||
@torch.inference_mode()
|
||||
def test_deep_select_topk_preallocated_output() -> None:
|
||||
"""DeepSelect writes into a preallocated aligned buffer (no `end`)."""
|
||||
from vllm.model_executor.layers import indexer_topk
|
||||
|
||||
set_random_seed(0)
|
||||
torch.set_default_device("cuda:0")
|
||||
|
||||
batch_size = 8
|
||||
vocab_size = 131072
|
||||
top_k = 2048
|
||||
logits = torch.randn(batch_size, vocab_size, dtype=torch.float32, device="cuda")
|
||||
|
||||
# Mimic vLLM's topk_indices_buffer: slice of a wider contiguous buffer,
|
||||
# whose stride(0) is 32B-aligned.
|
||||
buffer = torch.full((batch_size, 4096), -2, dtype=torch.int32, device="cuda")
|
||||
output_idx = buffer[:, :top_k]
|
||||
result = indexer_topk.deep_select_topk(logits, top_k, output_idx=output_idx)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
assert result.data_ptr() == output_idx.data_ptr()
|
||||
assert torch.all(buffer[:, top_k:] == -2), "kernel wrote out of bounds"
|
||||
|
||||
row_ends = torch.full((batch_size,), vocab_size, dtype=torch.int32, device="cuda")
|
||||
row_starts = torch.zeros(batch_size, dtype=torch.int32, device="cuda")
|
||||
torch_indices = torch.empty((batch_size, top_k), dtype=torch.int32, device="cuda")
|
||||
for i in range(batch_size):
|
||||
torch_indices[i] = logits[i].topk(top_k, dim=-1)[1]
|
||||
|
||||
assert compare_top_k_results(
|
||||
logits, output_idx, torch_indices, row_starts, row_ends, top_k
|
||||
), "DeepSelect topk (preallocated output) results don't match torch.topk"
|
||||
|
||||
|
||||
def _has_flashinfer_topk() -> bool:
|
||||
if not current_platform.is_cuda():
|
||||
return False
|
||||
# find_spec only: importing flashinfer initializes CUDA at import time.
|
||||
import importlib.util
|
||||
|
||||
return importlib.util.find_spec("flashinfer") is not None
|
||||
|
||||
|
||||
def _has_cooperative_topk() -> bool:
|
||||
return _has_device_capability(
|
||||
90
|
||||
) and not current_platform.is_device_capability_family(120)
|
||||
|
||||
|
||||
SPARSE_INDEXER_EXPLICIT_BACKENDS = [
|
||||
pytest.param(
|
||||
"per_row",
|
||||
marks=pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="requires CUDA"
|
||||
),
|
||||
),
|
||||
"torch",
|
||||
pytest.param(
|
||||
"persistent",
|
||||
marks=pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="requires CUDA"
|
||||
),
|
||||
),
|
||||
pytest.param(
|
||||
"cooperative",
|
||||
marks=pytest.mark.skipif(
|
||||
not _has_cooperative_topk(),
|
||||
reason="cooperative_topk requires SM90+ (non-SM12x)",
|
||||
),
|
||||
),
|
||||
pytest.param(
|
||||
"flashinfer",
|
||||
marks=pytest.mark.skipif(
|
||||
not _has_flashinfer_topk(), reason="requires flashinfer top-k"
|
||||
),
|
||||
),
|
||||
pytest.param("deep_select", marks=requires_sm100),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("next_n", [1, 4])
|
||||
@pytest.mark.parametrize("backend", SPARSE_INDEXER_EXPLICIT_BACKENDS)
|
||||
@torch.inference_mode()
|
||||
def test_sparse_indexer_decode_topk_explicit_backends(
|
||||
backend: str, next_n: int, workspace_init
|
||||
) -> None:
|
||||
"""Every explicit sparse-indexer decode top-k backend must restrict each
|
||||
row to its seq_len (dirty data past the end must never be selected) and
|
||||
match torch.topk on the valid region."""
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.indexer_topk import SparseIndexerTopk
|
||||
|
||||
set_random_seed(0)
|
||||
torch.set_default_device("cuda:0")
|
||||
|
||||
top_k = 2048
|
||||
batch_size = 4
|
||||
num_rows = batch_size * next_n
|
||||
|
||||
# Per-row effective lens in the (B, next_n) "native spec decode" form;
|
||||
# cooperative/persistent_topk require lengths.numel() == num_rows.
|
||||
row_ends = torch.randint(4000, 16000, (num_rows,), dtype=torch.int32, device="cuda")
|
||||
seq_lens = row_ends.reshape(batch_size, next_n)
|
||||
max_seq_len = int(row_ends.max())
|
||||
# Pad the stride: multiple of 4 for cooperative_topk (TMA) and
|
||||
# 1024B-aligned (256 fp32) for DeepSelect.
|
||||
stride = (max_seq_len + 255) // 256 * 256
|
||||
|
||||
row_starts = torch.zeros(num_rows, dtype=torch.int32, device="cuda")
|
||||
|
||||
logits = torch.randn(num_rows, stride, dtype=torch.float32, device="cuda")
|
||||
col_idx = torch.arange(stride, device="cuda")
|
||||
logits[col_idx[None, :] >= row_ends[:, None]] = 1e30 # dirty tail
|
||||
|
||||
indices = torch.full((num_rows, top_k), -2, dtype=torch.int32, device="cuda")
|
||||
cfg = VllmConfig(kernel_config={"sparse_indexer_topk_backend": backend})
|
||||
with set_current_vllm_config(cfg):
|
||||
SparseIndexerTopk()(logits, seq_lens, next_n, indices, top_k, max_seq_len)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
# k_i == top_k for every row here (row_ends >= 3997 > top_k).
|
||||
assert torch.all(indices >= 0)
|
||||
torch_indices = torch.empty((num_rows, top_k), dtype=torch.int32, device="cuda")
|
||||
for i in range(num_rows):
|
||||
row_end = int(row_ends[i])
|
||||
torch_indices[i] = logits[i, :row_end].topk(top_k, dim=-1)[1]
|
||||
assert torch.all(indices[i] < row_end), (
|
||||
f"{backend}: row {i} selected indices past its seq_len"
|
||||
)
|
||||
|
||||
assert compare_top_k_results(
|
||||
logits, indices, torch_indices, row_starts, row_ends, top_k
|
||||
), f"{backend} results don't match torch.topk"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"backend",
|
||||
[
|
||||
"torch",
|
||||
pytest.param(
|
||||
"per_row",
|
||||
marks=pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="requires CUDA"
|
||||
),
|
||||
),
|
||||
pytest.param(
|
||||
"flashinfer",
|
||||
marks=pytest.mark.skipif(
|
||||
not _has_flashinfer_topk(), reason="requires flashinfer top-k"
|
||||
),
|
||||
),
|
||||
pytest.param("deep_select", marks=requires_sm100),
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_sparse_indexer_decode_topk_short_seq_lens(
|
||||
backend: str, workspace_init
|
||||
) -> None:
|
||||
"""seq_len < next_n: derived per-row ends must clamp at 0 (the reference
|
||||
kernels do max(0, ...)); negative ends are OOB for the ragged kernels."""
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.indexer_topk import SparseIndexerTopk
|
||||
|
||||
set_random_seed(0)
|
||||
next_n = 4
|
||||
top_k = 512
|
||||
vocab = 8192 # 1024B-aligned rows for DeepSelect
|
||||
# 1D (B,) seq_lens: rows of a request share its len and each kernel
|
||||
# derives per-row ends (with clamping) itself.
|
||||
seq_lens = torch.tensor([2, 6], dtype=torch.int32, device="cuda")
|
||||
num_rows = 2 * next_n
|
||||
logits = torch.randn(num_rows, vocab, dtype=torch.float32, device="cuda")
|
||||
indices = torch.full((num_rows, top_k), -2, dtype=torch.int32, device="cuda")
|
||||
|
||||
cfg = VllmConfig(kernel_config={"sparse_indexer_topk_backend": backend})
|
||||
with set_current_vllm_config(cfg):
|
||||
SparseIndexerTopk()(logits, seq_lens, next_n, indices, top_k, vocab)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
# Derived ends: [2,6] - next_n + 1 + arange(next_n) -> [-1,0,1,2] and
|
||||
# [3,4,5,6], clamped to [0,0,1,2] and [3,4,5,6].
|
||||
for row, end in enumerate([0, 0, 1, 2, 3, 4, 5, 6]):
|
||||
k_i = min(top_k, end)
|
||||
assert torch.all(indices[row, k_i:] == -1), f"{backend}: row {row}"
|
||||
if k_i:
|
||||
ref = logits[row, :end].topk(k_i).indices.sort().values
|
||||
got = indices[row, :k_i].sort().values
|
||||
assert torch.equal(got, ref), f"{backend}: row {row}"
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="requires CUDA")
|
||||
@torch.inference_mode()
|
||||
def test_sparse_indexer_topk_backend_resolution() -> None:
|
||||
"""Auto heuristic chain and fail-fast validation of explicit backends."""
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.indexer_topk import SparseIndexerTopk
|
||||
|
||||
# 1024B-aligned stride: 131072 * 4 bytes.
|
||||
logits = torch.randn(64, 131072, dtype=torch.float32, device="cuda")
|
||||
# 4-divisible (cooperative TMA ok) but not 1024B-aligned (DeepSelect no):
|
||||
# 131076 * 4 = 524304 bytes.
|
||||
unaligned_logits = torch.randn(128, 131076, dtype=torch.float32, device="cuda")[
|
||||
:, :131072
|
||||
]
|
||||
|
||||
def resolve(
|
||||
backend: str,
|
||||
t: torch.Tensor = logits,
|
||||
k: int = 2048,
|
||||
num_rows: int = 64,
|
||||
) -> str:
|
||||
cfg = VllmConfig(kernel_config={"sparse_indexer_topk_backend": backend})
|
||||
with set_current_vllm_config(cfg):
|
||||
return SparseIndexerTopk().resolve_backend(t, k, num_rows)
|
||||
|
||||
is_sm100 = current_platform.is_device_capability_family(100)
|
||||
has_coop = _has_cooperative_topk()
|
||||
has_fi = _has_flashinfer_topk()
|
||||
|
||||
# "auto" is exactly the pre-existing chain: cooperative -> persistent ->
|
||||
# per_row. It must never select the opt-in backends by itself.
|
||||
if has_coop:
|
||||
assert resolve("auto") == "cooperative"
|
||||
assert resolve("auto", k=512, num_rows=8) == "cooperative"
|
||||
# Past cooperative's 64-row limit -> persistent, even where
|
||||
# DeepSelect would be applicable.
|
||||
assert resolve("auto", k=512, num_rows=128) == "persistent"
|
||||
# 16B-aligned (cooperative TMA ok) but not DeepSelect-aligned:
|
||||
# still cooperative at <= 64 rows...
|
||||
assert resolve("auto", unaligned_logits, num_rows=64) == "cooperative"
|
||||
# ...and persistent past the row limit.
|
||||
assert resolve("auto", unaligned_logits, num_rows=128) == "persistent"
|
||||
# Unsupported topk (outside {512, 1024, 2048} for the workspace kernels)
|
||||
# -> per_row fallback.
|
||||
assert resolve("auto", k=5000) == "per_row"
|
||||
|
||||
# Explicit values are returned as-is when constraints are met.
|
||||
assert resolve("per_row") == "per_row"
|
||||
assert resolve("torch") == "torch"
|
||||
assert resolve("persistent") == "persistent"
|
||||
if has_coop:
|
||||
assert resolve("cooperative") == "cooperative"
|
||||
if has_fi:
|
||||
assert resolve("flashinfer") == "flashinfer"
|
||||
if is_sm100:
|
||||
# Explicit deep_select has no row-count requirement.
|
||||
assert resolve("deep_select", num_rows=1) == "deep_select"
|
||||
with pytest.raises(RuntimeError, match="constraints"):
|
||||
resolve("deep_select", unaligned_logits)
|
||||
|
||||
# Fail-fast on unmet constraints.
|
||||
with pytest.raises(RuntimeError, match="num_rows must be <= 64"):
|
||||
resolve("cooperative", num_rows=128)
|
||||
with pytest.raises(RuntimeError, match="topk_tokens must be in"):
|
||||
resolve("persistent", k=3000)
|
||||
with pytest.raises(RuntimeError, match="topk_tokens must be in"):
|
||||
resolve("cooperative", k=3000)
|
||||
if not is_sm100:
|
||||
with pytest.raises(RuntimeError, match="SM100a/SM103a"):
|
||||
resolve("deep_select")
|
||||
|
||||
Reference in New Issue
Block a user