[Bugfix][DSv4.1] Preserve NaN-scored candidate block indices (#57454)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -365,6 +365,26 @@ def test_select_candidate_blocks_tolerates_empty_rows():
|
||||
assert (out[0] >= 0).all() and (out[2, :2] >= 0).all()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
||||
@pytest.mark.parametrize("score", [0.0, float("nan")])
|
||||
@pytest.mark.parametrize("start", [0, 3])
|
||||
def test_candidate_selection_keeps_valid_blocks_with_nan_scores(score, start):
|
||||
"""NaN warmup scores must not turn real blocks into repeated sparse padding."""
|
||||
logits = torch.full((3, 64), score, device="cuda")
|
||||
starts = torch.full((3,), start, device="cuda", dtype=torch.int32)
|
||||
lengths = torch.tensor([0, 17, 35], device="cuda", dtype=torch.int32)
|
||||
out = torch.empty(3, 8, device="cuda", dtype=torch.int32)
|
||||
|
||||
sparse_indexer._select_candidate_blocks(logits, starts, starts + lengths, 8, 8, out)
|
||||
|
||||
for row, count in enumerate([0, 3, 5]):
|
||||
valid = out[row][out[row] >= 0].sort().values
|
||||
torch.testing.assert_close(
|
||||
valid, torch.arange(count, device="cuda", dtype=torch.int32)
|
||||
)
|
||||
assert (out[row] == -1).sum().item() == 8 - count
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
||||
@pytest.mark.parametrize(
|
||||
"width,block_size,k,decode",
|
||||
@@ -413,7 +433,7 @@ def test_candidate_kernels_preserve_packed_bounds_and_padding(
|
||||
top = reduced.topk(min(k, nblocks), dim=-1)
|
||||
expected = torch.full((rows, k), -1, device="cuda", dtype=torch.int32)
|
||||
expected[:, : top.indices.shape[1]] = torch.where(
|
||||
top.values > -torch.inf, top.indices, -1
|
||||
top.values != -torch.inf, top.indices, -1
|
||||
).int()
|
||||
actual = torch.empty(rows, k * 2, device="cuda", dtype=torch.int32)[:, ::2]
|
||||
select_candidate_blocks(logits, starts, ends, k, block_size, actual, repeat)
|
||||
|
||||
@@ -66,9 +66,10 @@ def _store_candidates_kernel(
|
||||
cols = tl.program_id(1) * TILE + tl.arange(0, TILE)
|
||||
value = tl.load(values + row * k + cols, cols < k, other=-float("inf"))
|
||||
index = tl.load(indices + row * k + cols, cols < k, other=-1)
|
||||
# NaN scores can occur during warmup; only -inf denotes padding.
|
||||
tl.store(
|
||||
output + row * out_stride_row + cols * out_stride_col,
|
||||
tl.where(value > -float("inf"), index, -1),
|
||||
tl.where(value != -float("inf"), index, -1),
|
||||
cols < OUT_K,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user