[Bugfix][Model Runner V2][Spec Decode] Fix int32 offset overflow in sampler kernels (#46560)

Signed-off-by: xiaojun.wei <jessiewei747@gmail.com>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
JessieWei
2026-06-24 11:00:57 -07:00
committed by GitHub
co-authored by Claude Opus 4.8 Robert Shaw
parent 56ca5997ea
commit 3c43237233
10 changed files with 90 additions and 16 deletions
+34
View File
@@ -1263,3 +1263,37 @@ def test_prompt_logprobs_with_chunking_and_preemption():
assert preemptions > 0, "Test did not trigger any preemptions"
print(f"Test passed with {preemptions} preemptions")
@large_gpu_mark(min_gb=24)
def test_token_logprobs_large_batch_int64_row_offset():
"""Regression: logprob kernel row offset (row * vocab_size) must use int64.
The rejection-sampler logprobs path runs the logprob kernels over the
spec-expanded logits batch, so batch_size * vocab_size can exceed 2**31
(e.g. DFlash drafts K tokens per request). With int32 offset arithmetic the
per-row pointer wraps to a negative address and the kernel hits a CUDA
illegal memory access. Run over a batch where batch_size * vocab_size > 2**31
and check the highest-offset row matches a reference log-softmax.
"""
if not current_platform.is_cuda():
pytest.skip("int32 row-offset overflow is a CUDA kernel issue")
from vllm.v1.worker.gpu.sample.logprob import compute_token_logprobs
device = torch.device("cuda")
vocab_size = 131072
batch_size = 2**31 // vocab_size + 64 # batch_size * vocab_size > 2**31
# logits (the large input) plus small logprob/rank outputs; ~1 GB headroom.
required_bytes = batch_size * vocab_size * 4 + (1 << 30)
if torch.cuda.mem_get_info()[0] < required_bytes:
pytest.skip(f"needs ~{required_bytes / 1e9:.0f} GB of free GPU memory")
logits = torch.randn(batch_size, vocab_size, device=device, dtype=torch.float32)
token_ids = torch.full((batch_size, 1), 7, device=device, dtype=torch.int64)
logprobs = compute_token_logprobs(logits, token_ids)
torch.accelerator.synchronize() # surface any async illegal memory access
last = batch_size - 1
ref = torch.log_softmax(logits[last].float(), dim=-1)[7]
assert torch.allclose(logprobs[last, 0], ref, atol=1e-2), (
f"logprob {logprobs[last, 0].item()} != ref {ref.item()}"
)
+40
View File
@@ -4,6 +4,7 @@ import pytest
import torch
from torch import Generator
from tests.utils import large_gpu_mark
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
from vllm.utils.torch_utils import set_random_seed
@@ -404,6 +405,45 @@ class TestTritonTopkTopp:
self._compare_results(logits, k, p)
@large_gpu_mark(min_gb=24)
def test_large_batch_int64_row_offset(self):
"""Regression: per-row offset (row * vocab_size) must not overflow int32.
Speculative decoding expands the logits batch (e.g. DFlash drafts K
tokens per request), so batch_size * vocab_size can exceed 2**31. With
int32 offset arithmetic the per-row pointer wraps to a negative address
and the kernel hits a CUDA illegal memory access. Use a batch where
batch_size * vocab_size > 2**31 and give the highest-offset row the same
logits as row 0: an overflow there would read a different row and change
the kept set.
"""
from vllm.v1.sample.ops.topk_topp_triton import apply_top_k_top_p_triton
if not current_platform.is_cuda():
pytest.skip("int32 row-offset overflow is a CUDA kernel issue")
vocab_size = 131072
batch_size = 2**31 // vocab_size + 64 # batch_size * vocab_size > 2**31
# logits is modified in place; the only extra device memory is the
# per-SM scratch buffer (~num_sm * vocab), so allow ~1 GB of headroom.
required_bytes = batch_size * vocab_size * 4 + (1 << 30)
if torch.cuda.mem_get_info()[0] < required_bytes:
pytest.skip(f"needs ~{required_bytes / 1e9:.0f} GB of free GPU memory")
logits = torch.randn(
batch_size, vocab_size, generator=self.generator, dtype=torch.float32
)
logits[batch_size - 1] = logits[0]
k = torch.full((batch_size,), 5, dtype=torch.int32)
result = apply_top_k_top_p_triton(logits, k, None)
torch.accelerator.synchronize() # surface any async illegal memory access
kept_first = (result[0] > float("-inf")).nonzero(as_tuple=True)[0]
kept_last = (result[batch_size - 1] > float("-inf")).nonzero(as_tuple=True)[0]
assert kept_first.numel() == 5, f"row 0 kept {kept_first.numel()}, expected 5"
assert torch.equal(kept_first, kept_last), (
"highest-offset row produced a different top-k mask than the "
"identical row 0 (int32 row-offset overflow)"
)
@pytest.mark.parametrize(
"mode",
["topk_only", "topp_only", "topk_and_topp"],