[Bugfix][Model Runner V2][Spec Decode] Fix int32 offset overflow in block verification kernels (#47383)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
3e158ae62d
commit
a47f38f825
@@ -0,0 +1,144 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""int64 indexing in the block-verification rejection sampler kernels.
|
||||
|
||||
With a GLM-scale vocab (~155k), logit_idx * vocab_stride exceeds int32 once
|
||||
logit_idx >= ~13.8k (and req_state_idx * draft_stride_0 similarly), so the
|
||||
block-verification kernels must promote indices to int64 before the stride
|
||||
multiply. These tests place one request at a high logit/request-state index
|
||||
and check its outputs match the same request run alone at index 0.
|
||||
|
||||
Each case allocates ~5 GiB of GPU memory.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.worker.gpu.spec_decode.rejection_sampler_utils import rejection_sample
|
||||
|
||||
VOCAB_SIZE = 155264
|
||||
NUM_SPECULATIVE_STEPS = 2
|
||||
LOGITS_PER_REQ = NUM_SPECULATIVE_STEPS + 1
|
||||
|
||||
|
||||
def _run(
|
||||
target_logits: torch.Tensor, # [LOGITS_PER_REQ, V], the request under test
|
||||
draft_logits: torch.Tensor, # [NUM_SPECULATIVE_STEPS, V]
|
||||
draft_tokens: list[int],
|
||||
num_reqs: int,
|
||||
max_num_reqs: int,
|
||||
roi_req: int,
|
||||
req_state_idx: int,
|
||||
device: torch.device,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Run rejection_sample on a batch of uniform k=2 requests, with the
|
||||
request under test at request index `roi_req` and draft-logits row
|
||||
`req_state_idx`. Returns that request's (sampled, num_sampled)."""
|
||||
num_logits = num_reqs * LOGITS_PER_REQ
|
||||
roi_row = roi_req * LOGITS_PER_REQ
|
||||
|
||||
full_target = torch.randn(
|
||||
num_logits, VOCAB_SIZE, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
full_target[roi_row : roi_row + LOGITS_PER_REQ] = target_logits
|
||||
full_draft = torch.randn(
|
||||
max_num_reqs,
|
||||
NUM_SPECULATIVE_STEPS,
|
||||
VOCAB_SIZE,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
full_draft[req_state_idx] = draft_logits
|
||||
|
||||
# Draft token for step i of request r lives at row r*LOGITS_PER_REQ + i + 1.
|
||||
draft_sampled = torch.zeros(num_logits, dtype=torch.int64, device=device)
|
||||
for i, tok in enumerate(draft_tokens):
|
||||
draft_sampled[roi_row + i + 1] = tok
|
||||
|
||||
cu_num_logits = torch.arange(
|
||||
0, num_logits + 1, LOGITS_PER_REQ, dtype=torch.int32, device=device
|
||||
)
|
||||
# Filler requests all map to draft-logits row 0.
|
||||
idx_mapping = torch.zeros(num_reqs, dtype=torch.int32, device=device)
|
||||
idx_mapping[roi_req] = req_state_idx
|
||||
expanded_idx_mapping = idx_mapping.repeat_interleave(LOGITS_PER_REQ)
|
||||
expanded_local_pos = (
|
||||
torch.arange(LOGITS_PER_REQ, dtype=torch.int32, device=device)
|
||||
.repeat(num_reqs)
|
||||
.contiguous()
|
||||
)
|
||||
|
||||
# Positions only need to match between the reference and big-batch runs
|
||||
# for the request under test (they key the Gumbel noise and uniforms).
|
||||
pos = torch.arange(num_logits, dtype=torch.int64, device=device)
|
||||
pos[roi_row : roi_row + LOGITS_PER_REQ] = torch.arange(
|
||||
1000, 1000 + LOGITS_PER_REQ, dtype=torch.int64, device=device
|
||||
)
|
||||
temperature = torch.ones(max_num_reqs, dtype=torch.float32, device=device)
|
||||
seeds = torch.full((max_num_reqs,), 42, dtype=torch.int64, device=device)
|
||||
|
||||
sampled, num_sampled = rejection_sample(
|
||||
target_logits=full_target,
|
||||
draft_logits=full_draft,
|
||||
draft_sampled=draft_sampled,
|
||||
cu_num_logits=cu_num_logits,
|
||||
pos=pos,
|
||||
idx_mapping=idx_mapping,
|
||||
expanded_idx_mapping=expanded_idx_mapping,
|
||||
expanded_local_pos=expanded_local_pos,
|
||||
temperature=temperature,
|
||||
seed=seeds,
|
||||
num_speculative_steps=NUM_SPECULATIVE_STEPS,
|
||||
use_block_verification=True,
|
||||
)
|
||||
return sampled[roi_req].clone(), num_sampled[roi_req].clone()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
|
||||
@pytest.mark.parametrize(
|
||||
# Overflow triggers when roi_req * LOGITS_PER_REQ * VOCAB_SIZE (target
|
||||
# logits) or req_state_idx * NUM_SPECULATIVE_STEPS * VOCAB_SIZE (draft
|
||||
# logits) exceeds 2**31.
|
||||
("num_reqs", "max_num_reqs", "roi_req", "req_state_idx"),
|
||||
[
|
||||
(5500, 4, 5000, 2), # target-side: logit_idx 15000 * V > 2**31
|
||||
(22, 8192, 11, 8000), # draft-side: req_state_idx 8000 * 2V > 2**31
|
||||
],
|
||||
)
|
||||
def test_block_verification_i64_indexing(
|
||||
num_reqs: int, max_num_reqs: int, roi_req: int, req_state_idx: int
|
||||
):
|
||||
device = torch.device("cuda:0")
|
||||
torch.manual_seed(0)
|
||||
target_logits = torch.randn(
|
||||
LOGITS_PER_REQ, VOCAB_SIZE, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
draft_logits = torch.randn(
|
||||
NUM_SPECULATIVE_STEPS, VOCAB_SIZE, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
draft_tokens = [123, 45678]
|
||||
|
||||
sampled_ref, num_sampled_ref = _run(
|
||||
target_logits,
|
||||
draft_logits,
|
||||
draft_tokens,
|
||||
num_reqs=1,
|
||||
max_num_reqs=1,
|
||||
roi_req=0,
|
||||
req_state_idx=0,
|
||||
device=device,
|
||||
)
|
||||
sampled, num_sampled = _run(
|
||||
target_logits,
|
||||
draft_logits,
|
||||
draft_tokens,
|
||||
num_reqs=num_reqs,
|
||||
max_num_reqs=max_num_reqs,
|
||||
roi_req=roi_req,
|
||||
req_state_idx=req_state_idx,
|
||||
device=device,
|
||||
)
|
||||
assert num_sampled.item() == num_sampled_ref.item()
|
||||
n = num_sampled_ref.item()
|
||||
assert torch.equal(sampled[:n], sampled_ref[:n])
|
||||
@@ -328,8 +328,8 @@ def _compute_cumulative_log_p_kernel(
|
||||
HAS_DRAFT_LOGITS: tl.constexpr,
|
||||
):
|
||||
req_idx = tl.program_id(0)
|
||||
req_state_idx = tl.load(idx_mapping_ptr + req_idx)
|
||||
start_idx = tl.load(cu_num_logits_ptr + req_idx)
|
||||
req_state_idx = tl.load(idx_mapping_ptr + req_idx).to(tl.int64)
|
||||
start_idx = tl.load(cu_num_logits_ptr + req_idx).to(tl.int64)
|
||||
end_idx = tl.load(cu_num_logits_ptr + req_idx + 1)
|
||||
num_draft_tokens = end_idx - start_idx - 1
|
||||
temp = tl.load(temp_ptr + req_state_idx).to(tl.float32)
|
||||
@@ -405,7 +405,7 @@ def _compute_local_residual_mass_kernel(
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
PADDED_VOCAB_NUM_BLOCKS: tl.constexpr,
|
||||
):
|
||||
logit_idx = tl.program_id(0)
|
||||
logit_idx = tl.program_id(0).to(tl.int64)
|
||||
draft_step_idx = tl.load(expanded_local_pos_ptr + logit_idx)
|
||||
if draft_step_idx == 0 or draft_step_idx >= num_speculative_steps:
|
||||
# The acceptance threshold, h, looks one position ahead and sums
|
||||
@@ -413,7 +413,7 @@ def _compute_local_residual_mass_kernel(
|
||||
# first and last (bonus) positions aren't needed for this computation.
|
||||
return
|
||||
|
||||
req_state_idx = tl.load(expanded_idx_mapping_ptr + logit_idx)
|
||||
req_state_idx = tl.load(expanded_idx_mapping_ptr + logit_idx).to(tl.int64)
|
||||
temp = tl.load(temp_ptr + req_state_idx).to(tl.float32)
|
||||
if temp == 0.0:
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user