[Bugfix][DSv4.1] Preserve NaN-scored candidate block indices (#57454)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Woosuk Kwon
2026-09-17 19:00:10 -07:00
committed by GitHub
co-authored by Codex
parent 2c88fb131c
commit 64563d0ec4
2 changed files with 23 additions and 2 deletions
@@ -365,6 +365,26 @@ def test_select_candidate_blocks_tolerates_empty_rows():
assert (out[0] >= 0).all() and (out[2, :2] >= 0).all()
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
@pytest.mark.parametrize("score", [0.0, float("nan")])
@pytest.mark.parametrize("start", [0, 3])
def test_candidate_selection_keeps_valid_blocks_with_nan_scores(score, start):
"""NaN warmup scores must not turn real blocks into repeated sparse padding."""
logits = torch.full((3, 64), score, device="cuda")
starts = torch.full((3,), start, device="cuda", dtype=torch.int32)
lengths = torch.tensor([0, 17, 35], device="cuda", dtype=torch.int32)
out = torch.empty(3, 8, device="cuda", dtype=torch.int32)
sparse_indexer._select_candidate_blocks(logits, starts, starts + lengths, 8, 8, out)
for row, count in enumerate([0, 3, 5]):
valid = out[row][out[row] >= 0].sort().values
torch.testing.assert_close(
valid, torch.arange(count, device="cuda", dtype=torch.int32)
)
assert (out[row] == -1).sum().item() == 8 - count
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
@pytest.mark.parametrize(
"width,block_size,k,decode",
@@ -413,7 +433,7 @@ def test_candidate_kernels_preserve_packed_bounds_and_padding(
top = reduced.topk(min(k, nblocks), dim=-1)
expected = torch.full((rows, k), -1, device="cuda", dtype=torch.int32)
expected[:, : top.indices.shape[1]] = torch.where(
top.values > -torch.inf, top.indices, -1
top.values != -torch.inf, top.indices, -1
).int()
actual = torch.empty(rows, k * 2, device="cuda", dtype=torch.int32)[:, ::2]
select_candidate_blocks(logits, starts, ends, k, block_size, actual, repeat)
@@ -66,9 +66,10 @@ def _store_candidates_kernel(
cols = tl.program_id(1) * TILE + tl.arange(0, TILE)
value = tl.load(values + row * k + cols, cols < k, other=-float("inf"))
index = tl.load(indices + row * k + cols, cols < k, other=-1)
# NaN scores can occur during warmup; only -inf denotes padding.
tl.store(
output + row * out_stride_row + cols * out_stride_col,
tl.where(value > -float("inf"), index, -1),
tl.where(value != -float("inf"), index, -1),
cols < OUT_K,
)