[Bugfix][Model Runner V2][Spec Decode] Fix int32 offset overflow in sampler kernels (#46560)
Signed-off-by: xiaojun.wei <jessiewei747@gmail.com> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
Robert Shaw
parent
56ca5997ea
commit
3c43237233
@@ -1263,3 +1263,37 @@ def test_prompt_logprobs_with_chunking_and_preemption():
|
||||
assert preemptions > 0, "Test did not trigger any preemptions"
|
||||
|
||||
print(f"Test passed with {preemptions} preemptions")
|
||||
|
||||
|
||||
@large_gpu_mark(min_gb=24)
|
||||
def test_token_logprobs_large_batch_int64_row_offset():
|
||||
"""Regression: logprob kernel row offset (row * vocab_size) must use int64.
|
||||
|
||||
The rejection-sampler logprobs path runs the logprob kernels over the
|
||||
spec-expanded logits batch, so batch_size * vocab_size can exceed 2**31
|
||||
(e.g. DFlash drafts K tokens per request). With int32 offset arithmetic the
|
||||
per-row pointer wraps to a negative address and the kernel hits a CUDA
|
||||
illegal memory access. Run over a batch where batch_size * vocab_size > 2**31
|
||||
and check the highest-offset row matches a reference log-softmax.
|
||||
"""
|
||||
if not current_platform.is_cuda():
|
||||
pytest.skip("int32 row-offset overflow is a CUDA kernel issue")
|
||||
from vllm.v1.worker.gpu.sample.logprob import compute_token_logprobs
|
||||
|
||||
device = torch.device("cuda")
|
||||
vocab_size = 131072
|
||||
batch_size = 2**31 // vocab_size + 64 # batch_size * vocab_size > 2**31
|
||||
# logits (the large input) plus small logprob/rank outputs; ~1 GB headroom.
|
||||
required_bytes = batch_size * vocab_size * 4 + (1 << 30)
|
||||
if torch.cuda.mem_get_info()[0] < required_bytes:
|
||||
pytest.skip(f"needs ~{required_bytes / 1e9:.0f} GB of free GPU memory")
|
||||
|
||||
logits = torch.randn(batch_size, vocab_size, device=device, dtype=torch.float32)
|
||||
token_ids = torch.full((batch_size, 1), 7, device=device, dtype=torch.int64)
|
||||
logprobs = compute_token_logprobs(logits, token_ids)
|
||||
torch.accelerator.synchronize() # surface any async illegal memory access
|
||||
last = batch_size - 1
|
||||
ref = torch.log_softmax(logits[last].float(), dim=-1)[7]
|
||||
assert torch.allclose(logprobs[last, 0], ref, atol=1e-2), (
|
||||
f"logprob {logprobs[last, 0].item()} != ref {ref.item()}"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user