[MRV2][Spec Decode] Avoid rejection sampler OOM by chunking (#48630)
Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Nick Hill
parent
521aa80f71
commit
ac36a7a1e7
@@ -1,8 +1,12 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Iterator
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.config import SpeculativeConfig
|
||||
from vllm.config.model import PROCESSED_LOGPROBS_MODES
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.outputs import LogprobsTensors
|
||||
from vllm.v1.spec_decode.utils import unconditional_to_conditional_rates
|
||||
@@ -19,6 +23,29 @@ from vllm.v1.worker.gpu.spec_decode.rejection_sampler_utils import (
|
||||
rejection_sample,
|
||||
)
|
||||
|
||||
# Cap on the FP32 target-logits buffer materialized by apply_sampling_params.
|
||||
# TODO(mgoin): Chunking is a workaround. The rejection kernels already upcast
|
||||
# per vocab block on load and apply ops like temperature and gumbel, so folding
|
||||
# sampling-param application into those kernels would remove this buffer and
|
||||
# its traffic entirely.
|
||||
MAX_CHUNK_BYTES = 2**30 # 1GB
|
||||
_FP32_BYTES = 4
|
||||
|
||||
|
||||
def _iter_request_chunks(
|
||||
cu_num_logits: np.ndarray, max_chunk_logits: int
|
||||
) -> Iterator[tuple[int, int]]:
|
||||
"""Yield maximally packed request ranges without splitting requests."""
|
||||
assert max_chunk_logits > 0
|
||||
num_reqs = cu_num_logits.size - 1
|
||||
start = 0
|
||||
while start < num_reqs:
|
||||
max_logit = int(cu_num_logits[start]) + max_chunk_logits
|
||||
end = int(np.searchsorted(cu_num_logits, max_logit, side="right") - 1)
|
||||
end = min(num_reqs, max(start + 1, end))
|
||||
yield start, end
|
||||
start = end
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _flatten_sampled_kernel(
|
||||
@@ -66,18 +93,17 @@ class RejectionSampler:
|
||||
|
||||
def _get_logprobs_tensors(
|
||||
self,
|
||||
input_batch: InputBatch,
|
||||
sampled: torch.Tensor,
|
||||
num_sampled: torch.Tensor,
|
||||
logits: torch.Tensor,
|
||||
cu_num_logits: torch.Tensor,
|
||||
cu_num_logits_np: np.ndarray,
|
||||
max_num_logprobs: int,
|
||||
) -> LogprobsTensors | None:
|
||||
max_num_logprobs = self.sampler.sampling_states.max_num_logprobs(
|
||||
input_batch.idx_mapping_np
|
||||
)
|
||||
if max_num_logprobs == NO_LOGPROBS:
|
||||
return None
|
||||
|
||||
num_reqs = input_batch.cu_num_logits.shape[0] - 1
|
||||
num_reqs = cu_num_logits.shape[0] - 1
|
||||
num_logits = logits.shape[0]
|
||||
flat_sampled = torch.zeros(
|
||||
num_logits, dtype=sampled.dtype, device=sampled.device
|
||||
@@ -87,19 +113,122 @@ class RejectionSampler:
|
||||
sampled,
|
||||
sampled.stride(0),
|
||||
num_sampled,
|
||||
input_batch.cu_num_logits,
|
||||
cu_num_logits,
|
||||
num_warps=1,
|
||||
)
|
||||
expanded_logits = num_logits != input_batch.idx_mapping.shape[0]
|
||||
expanded_logits = num_logits != num_reqs
|
||||
return compute_topk_scores(
|
||||
logits,
|
||||
max_num_logprobs,
|
||||
flat_sampled,
|
||||
input_batch.cu_num_logits_np.tolist() if expanded_logits else None,
|
||||
cu_num_logits_np.tolist() if expanded_logits else None,
|
||||
logits_mode=self.sampler.logprobs_mode
|
||||
in ("raw_logits", "processed_logits"),
|
||||
)
|
||||
|
||||
def _verify(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
draft_logits: torch.Tensor | None,
|
||||
draft_sampled: torch.Tensor,
|
||||
pos: torch.Tensor,
|
||||
cu_num_logits: torch.Tensor,
|
||||
idx_mapping: torch.Tensor,
|
||||
idx_mapping_np: np.ndarray,
|
||||
expanded_idx_mapping: torch.Tensor,
|
||||
expanded_local_pos: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
processed_logits = self.sampler.apply_sampling_params(
|
||||
logits,
|
||||
expanded_idx_mapping,
|
||||
idx_mapping_np,
|
||||
pos,
|
||||
draft_sampled,
|
||||
expanded_local_pos,
|
||||
)
|
||||
sampled, num_sampled = rejection_sample(
|
||||
processed_logits,
|
||||
draft_logits,
|
||||
draft_sampled,
|
||||
cu_num_logits,
|
||||
pos,
|
||||
idx_mapping,
|
||||
expanded_idx_mapping,
|
||||
expanded_local_pos,
|
||||
self.sampler.sampling_states.temperature.gpu,
|
||||
self.sampler.sampling_states.seeds.gpu,
|
||||
self.num_speculative_steps,
|
||||
self.synthetic_conditional_rates,
|
||||
use_fp64=self.sampler.use_fp64_gumbel,
|
||||
use_block_verification=self.use_block_verification,
|
||||
)
|
||||
return processed_logits, sampled, num_sampled
|
||||
|
||||
def _verify_in_chunks(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
input_batch: InputBatch,
|
||||
draft_logits: torch.Tensor | None,
|
||||
draft_sampled: torch.Tensor,
|
||||
pos: torch.Tensor,
|
||||
max_chunk_logits: int,
|
||||
max_num_logprobs: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, LogprobsTensors | None]:
|
||||
cu_num_logits_np = input_batch.cu_num_logits_np
|
||||
use_processed_logits = self.sampler.logprobs_mode in PROCESSED_LOGPROBS_MODES
|
||||
sampled_chunks: list[torch.Tensor] = []
|
||||
num_sampled_chunks: list[torch.Tensor] = []
|
||||
logprobs_chunks: list[LogprobsTensors] = []
|
||||
|
||||
for start, end in _iter_request_chunks(cu_num_logits_np, max_chunk_logits):
|
||||
lo = int(cu_num_logits_np[start])
|
||||
hi = int(cu_num_logits_np[end])
|
||||
chunk_cu_num_logits_np = cu_num_logits_np[start : end + 1] - lo
|
||||
chunk_cu_num_logits = input_batch.cu_num_logits[start : end + 1] - lo
|
||||
# draft_logits uses persistent request-state indices and stays global.
|
||||
processed_logits, sampled, num_sampled = self._verify(
|
||||
logits[lo:hi],
|
||||
draft_logits,
|
||||
draft_sampled[lo:hi],
|
||||
pos[lo:hi],
|
||||
chunk_cu_num_logits,
|
||||
input_batch.idx_mapping[start:end],
|
||||
input_batch.idx_mapping_np[start:end],
|
||||
input_batch.expanded_idx_mapping[lo:hi],
|
||||
input_batch.expanded_local_pos[lo:hi],
|
||||
)
|
||||
chunk_logprobs = self._get_logprobs_tensors(
|
||||
sampled,
|
||||
num_sampled,
|
||||
processed_logits if use_processed_logits else logits[lo:hi],
|
||||
chunk_cu_num_logits,
|
||||
chunk_cu_num_logits_np,
|
||||
max_num_logprobs,
|
||||
)
|
||||
if chunk_logprobs is not None:
|
||||
logprobs_chunks.append(chunk_logprobs)
|
||||
del processed_logits
|
||||
sampled_chunks.append(sampled)
|
||||
num_sampled_chunks.append(num_sampled)
|
||||
|
||||
if len(sampled_chunks) == 1:
|
||||
logprobs_tensors = logprobs_chunks[0] if logprobs_chunks else None
|
||||
return sampled_chunks[0], num_sampled_chunks[0], logprobs_tensors
|
||||
|
||||
logprobs_tensors = None
|
||||
if logprobs_chunks:
|
||||
expanded_logits = logits.shape[0] != input_batch.num_reqs
|
||||
logprobs_tensors = LogprobsTensors.cat(
|
||||
logprobs_chunks,
|
||||
cu_num_generated_tokens=(
|
||||
cu_num_logits_np.tolist() if expanded_logits else None
|
||||
),
|
||||
)
|
||||
|
||||
sampled = torch.cat(sampled_chunks)
|
||||
num_sampled = torch.cat(num_sampled_chunks)
|
||||
return sampled, num_sampled, logprobs_tensors
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
@@ -112,37 +241,19 @@ class RejectionSampler:
|
||||
|
||||
draft_sampled = input_batch.input_ids[input_batch.logits_indices]
|
||||
pos = input_batch.positions[input_batch.logits_indices]
|
||||
processed_logits = self.sampler.apply_sampling_params(
|
||||
logits,
|
||||
input_batch.expanded_idx_mapping,
|
||||
input_batch.idx_mapping_np,
|
||||
pos,
|
||||
draft_sampled,
|
||||
input_batch.expanded_local_pos,
|
||||
|
||||
max_num_logprobs = self.sampler.sampling_states.max_num_logprobs(
|
||||
input_batch.idx_mapping_np
|
||||
)
|
||||
sampled, num_sampled = rejection_sample(
|
||||
processed_logits,
|
||||
max_chunk_logits = max(1, MAX_CHUNK_BYTES // (logits.shape[1] * _FP32_BYTES))
|
||||
sampled, num_sampled, logprobs_tensors = self._verify_in_chunks(
|
||||
logits,
|
||||
input_batch,
|
||||
draft_logits,
|
||||
draft_sampled,
|
||||
input_batch.cu_num_logits,
|
||||
pos,
|
||||
input_batch.idx_mapping,
|
||||
input_batch.expanded_idx_mapping,
|
||||
input_batch.expanded_local_pos,
|
||||
self.sampler.sampling_states.temperature.gpu,
|
||||
self.sampler.sampling_states.seeds.gpu,
|
||||
self.num_speculative_steps,
|
||||
self.synthetic_conditional_rates,
|
||||
use_fp64=self.sampler.use_fp64_gumbel,
|
||||
use_block_verification=self.use_block_verification,
|
||||
)
|
||||
logprobs_tensors = self._get_logprobs_tensors(
|
||||
input_batch,
|
||||
sampled,
|
||||
num_sampled,
|
||||
processed_logits
|
||||
if self.sampler.logprobs_mode in ("processed_logprobs", "processed_logits")
|
||||
else logits,
|
||||
max_chunk_logits,
|
||||
max_num_logprobs,
|
||||
)
|
||||
|
||||
num_sampled, num_rejected = get_num_sampled_and_rejected(
|
||||
|
||||
@@ -888,6 +888,10 @@ def rejection_sample(
|
||||
use_fp64: bool = False,
|
||||
use_block_verification: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert target_logits.ndim == 2 and target_logits.stride(-1) == 1
|
||||
assert draft_logits is None or (
|
||||
draft_logits.ndim == 3 and draft_logits.stride(-1) == 1
|
||||
)
|
||||
num_reqs = cu_num_logits.shape[0] - 1
|
||||
num_logits, vocab_size = target_logits.shape
|
||||
draft_logits_stride_0 = 0
|
||||
|
||||
Reference in New Issue
Block a user