[MRV2][Spec Decode] Avoid rejection sampler OOM by chunking (#48630)
Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Nick Hill
parent
521aa80f71
commit
ac36a7a1e7
@@ -132,15 +132,7 @@ class PromptLogprobsWorker:
|
||||
|
||||
if prompt_logprobs_list:
|
||||
# Merge the in-progress logprobs.
|
||||
logprobs = LogprobsTensors(
|
||||
logprob_token_ids=torch.cat(
|
||||
[x.logprob_token_ids for x in prompt_logprobs_list]
|
||||
),
|
||||
logprobs=torch.cat([x.logprobs for x in prompt_logprobs_list]),
|
||||
selected_token_ranks=torch.cat(
|
||||
[x.selected_token_ranks for x in prompt_logprobs_list]
|
||||
),
|
||||
)
|
||||
logprobs = LogprobsTensors.cat(prompt_logprobs_list)
|
||||
prompt_logprobs_list.clear()
|
||||
|
||||
if logprobs is None:
|
||||
|
||||
@@ -5,7 +5,7 @@ import numpy as np
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config.model import LogprobsMode
|
||||
from vllm.config.model import PROCESSED_LOGPROBS_MODES, LogprobsMode
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.sample.ops.topk_topp_sampler import (
|
||||
apply_top_k_top_p,
|
||||
@@ -100,7 +100,7 @@ class Sampler:
|
||||
)
|
||||
|
||||
if return_logprobs:
|
||||
if self.logprobs_mode in ("processed_logprobs", "processed_logits"):
|
||||
if self.logprobs_mode in PROCESSED_LOGPROBS_MODES:
|
||||
logits = processed_logits
|
||||
expanded_logits = logits.shape[0] != idx_mapping_np.shape[0]
|
||||
cu_num_logits = cu_num_logits_np.tolist() if expanded_logits else None
|
||||
@@ -221,10 +221,7 @@ class Sampler:
|
||||
# any greedy requests or per-request seeds, or if post-processed
|
||||
# logprobs need to be returned for any requests.
|
||||
(top_k is None and top_p is None)
|
||||
or (
|
||||
return_logprobs
|
||||
and self.logprobs_mode in ("processed_logprobs", "processed_logits")
|
||||
)
|
||||
or (return_logprobs and self.logprobs_mode in PROCESSED_LOGPROBS_MODES)
|
||||
or self.sampling_states.any_greedy(idx_mapping_np)
|
||||
or self.sampling_states.any_explicit_seed(idx_mapping_np)
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user