From 64563d0ec4b761745d5babbc5750b263f0783926 Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Thu, 17 Sep 2026 19:00:10 -0700 Subject: [PATCH] [Bugfix][DSv4.1] Preserve NaN-scored candidate block indices (#57454) Signed-off-by: Woosuk Kwon Co-authored-by: Codex --- .../layers/test_mla_short_prefill_indexer.py | 22 ++++++++++++++++++- .../kernels/attention/dsa/candidate_blocks.py | 3 ++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/tests/model_executor/layers/test_mla_short_prefill_indexer.py b/tests/model_executor/layers/test_mla_short_prefill_indexer.py index 41a06d0546..882b745ece 100644 --- a/tests/model_executor/layers/test_mla_short_prefill_indexer.py +++ b/tests/model_executor/layers/test_mla_short_prefill_indexer.py @@ -365,6 +365,26 @@ def test_select_candidate_blocks_tolerates_empty_rows(): assert (out[0] >= 0).all() and (out[2, :2] >= 0).all() +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") +@pytest.mark.parametrize("score", [0.0, float("nan")]) +@pytest.mark.parametrize("start", [0, 3]) +def test_candidate_selection_keeps_valid_blocks_with_nan_scores(score, start): + """NaN warmup scores must not turn real blocks into repeated sparse padding.""" + logits = torch.full((3, 64), score, device="cuda") + starts = torch.full((3,), start, device="cuda", dtype=torch.int32) + lengths = torch.tensor([0, 17, 35], device="cuda", dtype=torch.int32) + out = torch.empty(3, 8, device="cuda", dtype=torch.int32) + + sparse_indexer._select_candidate_blocks(logits, starts, starts + lengths, 8, 8, out) + + for row, count in enumerate([0, 3, 5]): + valid = out[row][out[row] >= 0].sort().values + torch.testing.assert_close( + valid, torch.arange(count, device="cuda", dtype=torch.int32) + ) + assert (out[row] == -1).sum().item() == 8 - count + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") @pytest.mark.parametrize( "width,block_size,k,decode", @@ -413,7 +433,7 @@ def test_candidate_kernels_preserve_packed_bounds_and_padding( top = reduced.topk(min(k, nblocks), dim=-1) expected = torch.full((rows, k), -1, device="cuda", dtype=torch.int32) expected[:, : top.indices.shape[1]] = torch.where( - top.values > -torch.inf, top.indices, -1 + top.values != -torch.inf, top.indices, -1 ).int() actual = torch.empty(rows, k * 2, device="cuda", dtype=torch.int32)[:, ::2] select_candidate_blocks(logits, starts, ends, k, block_size, actual, repeat) diff --git a/vllm/model_executor/kernels/attention/dsa/candidate_blocks.py b/vllm/model_executor/kernels/attention/dsa/candidate_blocks.py index efc7c418ca..b9cdc923b1 100644 --- a/vllm/model_executor/kernels/attention/dsa/candidate_blocks.py +++ b/vllm/model_executor/kernels/attention/dsa/candidate_blocks.py @@ -66,9 +66,10 @@ def _store_candidates_kernel( cols = tl.program_id(1) * TILE + tl.arange(0, TILE) value = tl.load(values + row * k + cols, cols < k, other=-float("inf")) index = tl.load(indices + row * k + cols, cols < k, other=-1) + # NaN scores can occur during warmup; only -inf denotes padding. tl.store( output + row * out_stride_row + cols * out_stride_col, - tl.where(value > -float("inf"), index, -1), + tl.where(value != -float("inf"), index, -1), cols < OUT_K, )