[MRV2][Spec Decode] Avoid rejection sampler OOM by chunking (#48630)

Signed-off-by: mgoin <mgoin64@gmail.com>
Signed-off-by: Michael Goin <mgoin64@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Michael Goin
2026-07-23 18:13:13 +08:00
committed by GitHub
co-authored by Nick Hill
parent 521aa80f71
commit ac36a7a1e7
12 changed files with 394 additions and 71 deletions
@@ -412,3 +412,65 @@ def test_block_verification_accepts_at_least_as_many(num_speculative_steps: int)
f"Block verification mean accepted length {mean_block:.4f} is worse "
f"than standard {mean_standard:.4f}."
)
@pytest.mark.parametrize("has_draft_logits", [True, False])
def test_chunked_requests_match_full_batch(has_draft_logits: bool):
torch.manual_seed(7)
device = "cuda"
num_reqs = 5
num_speculative_steps = 3
vocab_size = 257
target_logits = torch.randn(vocab_size, device=device)
draft_logits = torch.randn(vocab_size, device=device)
inputs = _build_rejection_sample_inputs(
target_logits,
draft_logits,
num_speculative_steps,
temperature=0.6,
num_trials=num_reqs,
)
padded_target_logits = torch.empty(
inputs["target_logits"].shape[0], vocab_size + 3, device=device
)
padded_target_logits[:, :vocab_size].copy_(inputs["target_logits"])
inputs["target_logits"] = padded_target_logits[:, :vocab_size]
assert inputs["target_logits"].stride(-1) == 1
assert not inputs["target_logits"].is_contiguous()
if not has_draft_logits:
inputs["draft_logits"] = None
sampled, num_sampled = rejection_sample(
**inputs, num_speculative_steps=num_speculative_steps
)
sampled_chunks = []
num_sampled_chunks = []
for start, end in ((0, 2), (2, 5)):
lo = start * (num_speculative_steps + 1)
hi = end * (num_speculative_steps + 1)
chunk_inputs = dict(inputs)
for name in (
"target_logits",
"draft_sampled",
"pos",
"expanded_idx_mapping",
"expanded_local_pos",
):
chunk_inputs[name] = inputs[name][lo:hi]
chunk_inputs["cu_num_logits"] = inputs["cu_num_logits"][start : end + 1] - lo
chunk_inputs["idx_mapping"] = inputs["idx_mapping"][start:end]
chunk_sampled, chunk_num_sampled = rejection_sample(
**chunk_inputs, num_speculative_steps=num_speculative_steps
)
sampled_chunks.append(chunk_sampled)
num_sampled_chunks.append(chunk_num_sampled)
chunked_sampled = torch.cat(sampled_chunks)
chunked_num_sampled = torch.cat(num_sampled_chunks)
assert torch.equal(chunked_num_sampled, num_sampled)
steps = torch.arange(num_speculative_steps + 1, device=device)
valid = steps.unsqueeze(0) < num_sampled.unsqueeze(1)
assert torch.equal(chunked_sampled[valid], sampled[valid])
+26 -1
View File
@@ -2,7 +2,32 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest import TestCase
from vllm.v1.outputs import LogprobsLists
import torch
from vllm.v1.outputs import LogprobsLists, LogprobsTensors
def test_logprobs_tensors_cat():
first = LogprobsTensors(
torch.tensor([[1, 2]]),
torch.tensor([[0.1, 0.2]]),
torch.tensor([1]),
)
second = LogprobsTensors(
torch.tensor([[3, 4]]),
torch.tensor([[0.3, 0.4]]),
torch.tensor([2]),
)
result = LogprobsTensors.cat([first, second], [0, 1, 2])
assert result.logprob_token_ids.tolist() == [[1, 2], [3, 4]]
assert result.logprobs.tolist() == (
first.logprobs.tolist() + second.logprobs.tolist()
)
assert result.selected_token_ranks.tolist() == [1, 2]
assert result.cu_num_generated_tokens == [0, 1, 2]
assert LogprobsTensors.cat([first]) is first
class TestLogprobsLists(TestCase):
@@ -0,0 +1,109 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import MethodType, SimpleNamespace
from typing import get_args
import numpy as np
import pytest
import torch
from vllm.config.model import PROCESSED_LOGPROBS_MODES, LogprobsMode
from vllm.platforms import current_platform
from vllm.v1.worker.gpu.spec_decode.rejection_sampler import (
RejectionSampler,
_iter_request_chunks,
)
def test_iter_request_chunks_preserves_request_boundaries():
cu_num_logits = np.array([0, 3, 4, 11, 13], dtype=np.int32)
assert list(_iter_request_chunks(cu_num_logits, max_chunk_logits=5)) == [
(0, 2),
(2, 3),
(3, 4),
]
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
@pytest.mark.parametrize("logprobs_mode", get_args(LogprobsMode))
def test_chunked_scores_match_full_batch(logprobs_mode: str):
device = torch.device("cuda")
cu_num_logits_np = np.array([0, 3, 4, 8, 10], dtype=np.int32)
num_logits_per_req = np.diff(cu_num_logits_np)
idx_mapping_np = np.array([7, 2, 9, 1], dtype=np.int32)
input_batch = SimpleNamespace(
num_reqs=4,
cu_num_logits_np=cu_num_logits_np,
cu_num_logits=torch.from_numpy(cu_num_logits_np).to(device),
idx_mapping_np=idx_mapping_np,
idx_mapping=torch.from_numpy(idx_mapping_np).to(device),
expanded_idx_mapping=torch.from_numpy(
np.repeat(idx_mapping_np, num_logits_per_req)
).to(device),
expanded_local_pos=torch.from_numpy(
np.concatenate(
[np.arange(count, dtype=np.int32) for count in num_logits_per_req]
)
).to(device),
)
rejection_sampler = object.__new__(RejectionSampler)
rejection_sampler.sampler = SimpleNamespace(logprobs_mode=logprobs_mode)
rejection_sampler.num_speculative_steps = 3
def fake_verify(
self,
logits,
_draft_logits,
_draft_sampled,
_pos,
cu_num_logits,
idx_mapping,
*_mappings,
):
num_sampled = torch.diff(cu_num_logits).to(torch.int32)
sampled = (
idx_mapping.to(torch.int64).unsqueeze(1) + torch.arange(4, device=device)
) % logits.shape[1]
return logits.float() + 1, sampled, num_sampled
rejection_sampler._verify = MethodType(fake_verify, rejection_sampler)
logits = torch.arange(170, dtype=torch.float32, device=device).view(10, 17)
sampled, num_sampled, chunked_logprobs = rejection_sampler._verify_in_chunks(
logits,
input_batch,
draft_logits=None,
draft_sampled=torch.arange(10, device=device),
pos=torch.arange(10, device=device),
max_chunk_logits=5,
max_num_logprobs=2,
)
score_logits = logits + 1 if logprobs_mode in PROCESSED_LOGPROBS_MODES else logits
full_logprobs = rejection_sampler._get_logprobs_tensors(
sampled,
num_sampled,
score_logits,
input_batch.cu_num_logits,
input_batch.cu_num_logits_np,
max_num_logprobs=2,
)
assert sampled[:, 0].tolist() == idx_mapping_np.tolist()
assert num_sampled.tolist() == num_logits_per_req.tolist()
assert chunked_logprobs is not None
assert full_logprobs is not None
assert torch.equal(
chunked_logprobs.logprob_token_ids,
full_logprobs.logprob_token_ids,
)
assert torch.equal(chunked_logprobs.logprobs, full_logprobs.logprobs)
assert torch.equal(
chunked_logprobs.selected_token_ranks,
full_logprobs.selected_token_ranks,
)
assert (
chunked_logprobs.cu_num_generated_tokens
== full_logprobs.cu_num_generated_tokens
)