[MRV2][Spec Decode] Avoid rejection sampler OOM by chunking (#48630)

Signed-off-by: mgoin <mgoin64@gmail.com>
Signed-off-by: Michael Goin <mgoin64@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Michael Goin
2026-07-23 18:13:13 +08:00
committed by GitHub
co-authored by Nick Hill
parent 521aa80f71
commit ac36a7a1e7
12 changed files with 394 additions and 71 deletions
@@ -1,8 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterator
import numpy as np
import torch
from vllm.config import SpeculativeConfig
from vllm.config.model import PROCESSED_LOGPROBS_MODES
from vllm.triton_utils import tl, triton
from vllm.v1.outputs import LogprobsTensors
from vllm.v1.spec_decode.utils import unconditional_to_conditional_rates
@@ -19,6 +23,29 @@ from vllm.v1.worker.gpu.spec_decode.rejection_sampler_utils import (
rejection_sample,
)
# Cap on the FP32 target-logits buffer materialized by apply_sampling_params.
# TODO(mgoin): Chunking is a workaround. The rejection kernels already upcast
# per vocab block on load and apply ops like temperature and gumbel, so folding
# sampling-param application into those kernels would remove this buffer and
# its traffic entirely.
MAX_CHUNK_BYTES = 2**30 # 1GB
_FP32_BYTES = 4
def _iter_request_chunks(
cu_num_logits: np.ndarray, max_chunk_logits: int
) -> Iterator[tuple[int, int]]:
"""Yield maximally packed request ranges without splitting requests."""
assert max_chunk_logits > 0
num_reqs = cu_num_logits.size - 1
start = 0
while start < num_reqs:
max_logit = int(cu_num_logits[start]) + max_chunk_logits
end = int(np.searchsorted(cu_num_logits, max_logit, side="right") - 1)
end = min(num_reqs, max(start + 1, end))
yield start, end
start = end
@triton.jit
def _flatten_sampled_kernel(
@@ -66,18 +93,17 @@ class RejectionSampler:
def _get_logprobs_tensors(
self,
input_batch: InputBatch,
sampled: torch.Tensor,
num_sampled: torch.Tensor,
logits: torch.Tensor,
cu_num_logits: torch.Tensor,
cu_num_logits_np: np.ndarray,
max_num_logprobs: int,
) -> LogprobsTensors | None:
max_num_logprobs = self.sampler.sampling_states.max_num_logprobs(
input_batch.idx_mapping_np
)
if max_num_logprobs == NO_LOGPROBS:
return None
num_reqs = input_batch.cu_num_logits.shape[0] - 1
num_reqs = cu_num_logits.shape[0] - 1
num_logits = logits.shape[0]
flat_sampled = torch.zeros(
num_logits, dtype=sampled.dtype, device=sampled.device
@@ -87,19 +113,122 @@ class RejectionSampler:
sampled,
sampled.stride(0),
num_sampled,
input_batch.cu_num_logits,
cu_num_logits,
num_warps=1,
)
expanded_logits = num_logits != input_batch.idx_mapping.shape[0]
expanded_logits = num_logits != num_reqs
return compute_topk_scores(
logits,
max_num_logprobs,
flat_sampled,
input_batch.cu_num_logits_np.tolist() if expanded_logits else None,
cu_num_logits_np.tolist() if expanded_logits else None,
logits_mode=self.sampler.logprobs_mode
in ("raw_logits", "processed_logits"),
)
def _verify(
self,
logits: torch.Tensor,
draft_logits: torch.Tensor | None,
draft_sampled: torch.Tensor,
pos: torch.Tensor,
cu_num_logits: torch.Tensor,
idx_mapping: torch.Tensor,
idx_mapping_np: np.ndarray,
expanded_idx_mapping: torch.Tensor,
expanded_local_pos: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
processed_logits = self.sampler.apply_sampling_params(
logits,
expanded_idx_mapping,
idx_mapping_np,
pos,
draft_sampled,
expanded_local_pos,
)
sampled, num_sampled = rejection_sample(
processed_logits,
draft_logits,
draft_sampled,
cu_num_logits,
pos,
idx_mapping,
expanded_idx_mapping,
expanded_local_pos,
self.sampler.sampling_states.temperature.gpu,
self.sampler.sampling_states.seeds.gpu,
self.num_speculative_steps,
self.synthetic_conditional_rates,
use_fp64=self.sampler.use_fp64_gumbel,
use_block_verification=self.use_block_verification,
)
return processed_logits, sampled, num_sampled
def _verify_in_chunks(
self,
logits: torch.Tensor,
input_batch: InputBatch,
draft_logits: torch.Tensor | None,
draft_sampled: torch.Tensor,
pos: torch.Tensor,
max_chunk_logits: int,
max_num_logprobs: int,
) -> tuple[torch.Tensor, torch.Tensor, LogprobsTensors | None]:
cu_num_logits_np = input_batch.cu_num_logits_np
use_processed_logits = self.sampler.logprobs_mode in PROCESSED_LOGPROBS_MODES
sampled_chunks: list[torch.Tensor] = []
num_sampled_chunks: list[torch.Tensor] = []
logprobs_chunks: list[LogprobsTensors] = []
for start, end in _iter_request_chunks(cu_num_logits_np, max_chunk_logits):
lo = int(cu_num_logits_np[start])
hi = int(cu_num_logits_np[end])
chunk_cu_num_logits_np = cu_num_logits_np[start : end + 1] - lo
chunk_cu_num_logits = input_batch.cu_num_logits[start : end + 1] - lo
# draft_logits uses persistent request-state indices and stays global.
processed_logits, sampled, num_sampled = self._verify(
logits[lo:hi],
draft_logits,
draft_sampled[lo:hi],
pos[lo:hi],
chunk_cu_num_logits,
input_batch.idx_mapping[start:end],
input_batch.idx_mapping_np[start:end],
input_batch.expanded_idx_mapping[lo:hi],
input_batch.expanded_local_pos[lo:hi],
)
chunk_logprobs = self._get_logprobs_tensors(
sampled,
num_sampled,
processed_logits if use_processed_logits else logits[lo:hi],
chunk_cu_num_logits,
chunk_cu_num_logits_np,
max_num_logprobs,
)
if chunk_logprobs is not None:
logprobs_chunks.append(chunk_logprobs)
del processed_logits
sampled_chunks.append(sampled)
num_sampled_chunks.append(num_sampled)
if len(sampled_chunks) == 1:
logprobs_tensors = logprobs_chunks[0] if logprobs_chunks else None
return sampled_chunks[0], num_sampled_chunks[0], logprobs_tensors
logprobs_tensors = None
if logprobs_chunks:
expanded_logits = logits.shape[0] != input_batch.num_reqs
logprobs_tensors = LogprobsTensors.cat(
logprobs_chunks,
cu_num_generated_tokens=(
cu_num_logits_np.tolist() if expanded_logits else None
),
)
sampled = torch.cat(sampled_chunks)
num_sampled = torch.cat(num_sampled_chunks)
return sampled, num_sampled, logprobs_tensors
def __call__(
self,
logits: torch.Tensor,
@@ -112,37 +241,19 @@ class RejectionSampler:
draft_sampled = input_batch.input_ids[input_batch.logits_indices]
pos = input_batch.positions[input_batch.logits_indices]
processed_logits = self.sampler.apply_sampling_params(
logits,
input_batch.expanded_idx_mapping,
input_batch.idx_mapping_np,
pos,
draft_sampled,
input_batch.expanded_local_pos,
max_num_logprobs = self.sampler.sampling_states.max_num_logprobs(
input_batch.idx_mapping_np
)
sampled, num_sampled = rejection_sample(
processed_logits,
max_chunk_logits = max(1, MAX_CHUNK_BYTES // (logits.shape[1] * _FP32_BYTES))
sampled, num_sampled, logprobs_tensors = self._verify_in_chunks(
logits,
input_batch,
draft_logits,
draft_sampled,
input_batch.cu_num_logits,
pos,
input_batch.idx_mapping,
input_batch.expanded_idx_mapping,
input_batch.expanded_local_pos,
self.sampler.sampling_states.temperature.gpu,
self.sampler.sampling_states.seeds.gpu,
self.num_speculative_steps,
self.synthetic_conditional_rates,
use_fp64=self.sampler.use_fp64_gumbel,
use_block_verification=self.use_block_verification,
)
logprobs_tensors = self._get_logprobs_tensors(
input_batch,
sampled,
num_sampled,
processed_logits
if self.sampler.logprobs_mode in ("processed_logprobs", "processed_logits")
else logits,
max_chunk_logits,
max_num_logprobs,
)
num_sampled, num_rejected = get_num_sampled_and_rejected(
@@ -888,6 +888,10 @@ def rejection_sample(
use_fp64: bool = False,
use_block_verification: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]:
assert target_logits.ndim == 2 and target_logits.stride(-1) == 1
assert draft_logits is None or (
draft_logits.ndim == 3 and draft_logits.stride(-1) == 1
)
num_reqs = cu_num_logits.shape[0] - 1
num_logits, vocab_size = target_logits.shape
draft_logits_stride_0 = 0