[Bugfix][DSA] Bound DeepSelect sentinel columns in the sparse top-k remap (#58215)
Signed-off-by: Juntian Liu <Juntianl777@gmail.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
e0f03eb4f4
commit
9ef37771be
@@ -677,6 +677,40 @@ def test_candidate_blocks_to_sparse_indices_matches_reference(
|
||||
assert not end_buf[1 - end_offset :: 2].any()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
||||
def test_sparse_topk_remap_bounds_nan_rows():
|
||||
"""DeepSelect writes an out-of-range sentinel to slot 0 of NaN rows (the
|
||||
rest is left untouched); the remap turns it into -1."""
|
||||
from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import (
|
||||
has_deep_select,
|
||||
sparse_topk_remap,
|
||||
)
|
||||
|
||||
if not has_deep_select():
|
||||
pytest.skip("DeepSelect extension (vllm._deepselect_C) required")
|
||||
rows, num_sparse, sbk, topk = 4, 128, 8, 512
|
||||
width = num_sparse * sbk
|
||||
logits = torch.randn(rows, width, dtype=torch.bfloat16, device="cuda")
|
||||
logits[1:, :16] = float("nan")
|
||||
sparse_indices = torch.arange(num_sparse, dtype=torch.int32, device="cuda")
|
||||
end = torch.full((rows,), width, dtype=torch.int32, device="cuda")
|
||||
out = torch.empty(rows, topk, dtype=torch.int32, device="cuda")
|
||||
cols = torch.zeros_like(out)
|
||||
|
||||
sparse_topk_remap(
|
||||
logits,
|
||||
sparse_indices.repeat(rows, 1),
|
||||
end,
|
||||
torch.zeros_like(end),
|
||||
sbk,
|
||||
topk,
|
||||
out,
|
||||
col_indices=cols,
|
||||
)
|
||||
assert (cols[1:, 0] >= width).all() and (out[1:, 0] == -1).all()
|
||||
assert (out[0] >= 0).all()
|
||||
|
||||
|
||||
def _skip_unless_sm100_sparse_kernels():
|
||||
if not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10:
|
||||
pytest.skip("SM100 required")
|
||||
|
||||
@@ -146,19 +146,21 @@ def _sparse_topk_remap_kernel(
|
||||
out_ptr,
|
||||
out_stride,
|
||||
k,
|
||||
width,
|
||||
SBK: tl.constexpr,
|
||||
K_POW2: tl.constexpr,
|
||||
):
|
||||
"""Remap sparse top-k columns to request-local positions.
|
||||
|
||||
Column ``j * SBK + o`` scores the token at
|
||||
``(si[row, j] - ks // SBK) * SBK + o``; -1 columns stay -1.
|
||||
``(si[row, j] - ks // SBK) * SBK + o``; columns outside ``[0, width)``
|
||||
(including DeepSelect's sentinel for NaN rows) become -1.
|
||||
"""
|
||||
row = tl.program_id(0)
|
||||
ks_base = tl.load(ks_ptr + row * ks_stride) // SBK
|
||||
cols = tl.arange(0, K_POW2)
|
||||
c = tl.load(col_ptr + row * col_stride + cols, mask=cols < k, other=-1)
|
||||
valid = c >= 0
|
||||
valid = (c >= 0) & (c < width)
|
||||
cc = tl.where(valid, c, 0)
|
||||
blocks = tl.load(si_ptr + row * si_stride + cc // SBK)
|
||||
pos = (blocks - ks_base) * SBK + cc % SBK
|
||||
@@ -301,6 +303,7 @@ def sparse_topk_remap(
|
||||
topk_indices,
|
||||
topk_indices.stride(0),
|
||||
topk_tokens,
|
||||
width,
|
||||
SBK=sparse_block_kv,
|
||||
K_POW2=triton.next_power_of_2(topk_tokens),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user