From a47f38f82569c236d7d23b7ad0c8792ac6d62247 Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Thu, 2 Jul 2026 07:32:38 -0700 Subject: [PATCH] [Bugfix][Model Runner V2][Spec Decode] Fix int32 offset overflow in block verification kernels (#47383) Signed-off-by: Woosuk Kwon Co-authored-by: Claude Fable 5 --- .../worker/test_gpu_rejection_sampler_i64.py | 144 ++++++++++++++++++ .../spec_decode/rejection_sampler_utils.py | 8 +- 2 files changed, 148 insertions(+), 4 deletions(-) create mode 100644 tests/v1/worker/test_gpu_rejection_sampler_i64.py diff --git a/tests/v1/worker/test_gpu_rejection_sampler_i64.py b/tests/v1/worker/test_gpu_rejection_sampler_i64.py new file mode 100644 index 0000000000..769444ed64 --- /dev/null +++ b/tests/v1/worker/test_gpu_rejection_sampler_i64.py @@ -0,0 +1,144 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""int64 indexing in the block-verification rejection sampler kernels. + +With a GLM-scale vocab (~155k), logit_idx * vocab_stride exceeds int32 once +logit_idx >= ~13.8k (and req_state_idx * draft_stride_0 similarly), so the +block-verification kernels must promote indices to int64 before the stride +multiply. These tests place one request at a high logit/request-state index +and check its outputs match the same request run alone at index 0. + +Each case allocates ~5 GiB of GPU memory. +""" + +import pytest +import torch + +from vllm.platforms import current_platform +from vllm.v1.worker.gpu.spec_decode.rejection_sampler_utils import rejection_sample + +VOCAB_SIZE = 155264 +NUM_SPECULATIVE_STEPS = 2 +LOGITS_PER_REQ = NUM_SPECULATIVE_STEPS + 1 + + +def _run( + target_logits: torch.Tensor, # [LOGITS_PER_REQ, V], the request under test + draft_logits: torch.Tensor, # [NUM_SPECULATIVE_STEPS, V] + draft_tokens: list[int], + num_reqs: int, + max_num_reqs: int, + roi_req: int, + req_state_idx: int, + device: torch.device, +) -> tuple[torch.Tensor, torch.Tensor]: + """Run rejection_sample on a batch of uniform k=2 requests, with the + request under test at request index `roi_req` and draft-logits row + `req_state_idx`. Returns that request's (sampled, num_sampled).""" + num_logits = num_reqs * LOGITS_PER_REQ + roi_row = roi_req * LOGITS_PER_REQ + + full_target = torch.randn( + num_logits, VOCAB_SIZE, dtype=torch.bfloat16, device=device + ) + full_target[roi_row : roi_row + LOGITS_PER_REQ] = target_logits + full_draft = torch.randn( + max_num_reqs, + NUM_SPECULATIVE_STEPS, + VOCAB_SIZE, + dtype=torch.bfloat16, + device=device, + ) + full_draft[req_state_idx] = draft_logits + + # Draft token for step i of request r lives at row r*LOGITS_PER_REQ + i + 1. + draft_sampled = torch.zeros(num_logits, dtype=torch.int64, device=device) + for i, tok in enumerate(draft_tokens): + draft_sampled[roi_row + i + 1] = tok + + cu_num_logits = torch.arange( + 0, num_logits + 1, LOGITS_PER_REQ, dtype=torch.int32, device=device + ) + # Filler requests all map to draft-logits row 0. + idx_mapping = torch.zeros(num_reqs, dtype=torch.int32, device=device) + idx_mapping[roi_req] = req_state_idx + expanded_idx_mapping = idx_mapping.repeat_interleave(LOGITS_PER_REQ) + expanded_local_pos = ( + torch.arange(LOGITS_PER_REQ, dtype=torch.int32, device=device) + .repeat(num_reqs) + .contiguous() + ) + + # Positions only need to match between the reference and big-batch runs + # for the request under test (they key the Gumbel noise and uniforms). + pos = torch.arange(num_logits, dtype=torch.int64, device=device) + pos[roi_row : roi_row + LOGITS_PER_REQ] = torch.arange( + 1000, 1000 + LOGITS_PER_REQ, dtype=torch.int64, device=device + ) + temperature = torch.ones(max_num_reqs, dtype=torch.float32, device=device) + seeds = torch.full((max_num_reqs,), 42, dtype=torch.int64, device=device) + + sampled, num_sampled = rejection_sample( + target_logits=full_target, + draft_logits=full_draft, + draft_sampled=draft_sampled, + cu_num_logits=cu_num_logits, + pos=pos, + idx_mapping=idx_mapping, + expanded_idx_mapping=expanded_idx_mapping, + expanded_local_pos=expanded_local_pos, + temperature=temperature, + seed=seeds, + num_speculative_steps=NUM_SPECULATIVE_STEPS, + use_block_verification=True, + ) + return sampled[roi_req].clone(), num_sampled[roi_req].clone() + + +@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA") +@pytest.mark.parametrize( + # Overflow triggers when roi_req * LOGITS_PER_REQ * VOCAB_SIZE (target + # logits) or req_state_idx * NUM_SPECULATIVE_STEPS * VOCAB_SIZE (draft + # logits) exceeds 2**31. + ("num_reqs", "max_num_reqs", "roi_req", "req_state_idx"), + [ + (5500, 4, 5000, 2), # target-side: logit_idx 15000 * V > 2**31 + (22, 8192, 11, 8000), # draft-side: req_state_idx 8000 * 2V > 2**31 + ], +) +def test_block_verification_i64_indexing( + num_reqs: int, max_num_reqs: int, roi_req: int, req_state_idx: int +): + device = torch.device("cuda:0") + torch.manual_seed(0) + target_logits = torch.randn( + LOGITS_PER_REQ, VOCAB_SIZE, dtype=torch.bfloat16, device=device + ) + draft_logits = torch.randn( + NUM_SPECULATIVE_STEPS, VOCAB_SIZE, dtype=torch.bfloat16, device=device + ) + draft_tokens = [123, 45678] + + sampled_ref, num_sampled_ref = _run( + target_logits, + draft_logits, + draft_tokens, + num_reqs=1, + max_num_reqs=1, + roi_req=0, + req_state_idx=0, + device=device, + ) + sampled, num_sampled = _run( + target_logits, + draft_logits, + draft_tokens, + num_reqs=num_reqs, + max_num_reqs=max_num_reqs, + roi_req=roi_req, + req_state_idx=req_state_idx, + device=device, + ) + assert num_sampled.item() == num_sampled_ref.item() + n = num_sampled_ref.item() + assert torch.equal(sampled[:n], sampled_ref[:n]) diff --git a/vllm/v1/worker/gpu/spec_decode/rejection_sampler_utils.py b/vllm/v1/worker/gpu/spec_decode/rejection_sampler_utils.py index eb23717e43..f700ea846c 100644 --- a/vllm/v1/worker/gpu/spec_decode/rejection_sampler_utils.py +++ b/vllm/v1/worker/gpu/spec_decode/rejection_sampler_utils.py @@ -328,8 +328,8 @@ def _compute_cumulative_log_p_kernel( HAS_DRAFT_LOGITS: tl.constexpr, ): req_idx = tl.program_id(0) - req_state_idx = tl.load(idx_mapping_ptr + req_idx) - start_idx = tl.load(cu_num_logits_ptr + req_idx) + req_state_idx = tl.load(idx_mapping_ptr + req_idx).to(tl.int64) + start_idx = tl.load(cu_num_logits_ptr + req_idx).to(tl.int64) end_idx = tl.load(cu_num_logits_ptr + req_idx + 1) num_draft_tokens = end_idx - start_idx - 1 temp = tl.load(temp_ptr + req_state_idx).to(tl.float32) @@ -405,7 +405,7 @@ def _compute_local_residual_mass_kernel( BLOCK_SIZE: tl.constexpr, PADDED_VOCAB_NUM_BLOCKS: tl.constexpr, ): - logit_idx = tl.program_id(0) + logit_idx = tl.program_id(0).to(tl.int64) draft_step_idx = tl.load(expanded_local_pos_ptr + logit_idx) if draft_step_idx == 0 or draft_step_idx >= num_speculative_steps: # The acceptance threshold, h, looks one position ahead and sums @@ -413,7 +413,7 @@ def _compute_local_residual_mass_kernel( # first and last (bonus) positions aren't needed for this computation. return - req_state_idx = tl.load(expanded_idx_mapping_ptr + logit_idx) + req_state_idx = tl.load(expanded_idx_mapping_ptr + logit_idx).to(tl.int64) temp = tl.load(temp_ptr + req_state_idx).to(tl.float32) if temp == 0.0: return