[MRV2][Spec Decode] Avoid rejection sampler OOM by chunking (#48630)
Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Nick Hill
parent
521aa80f71
commit
ac36a7a1e7
@@ -10,6 +10,7 @@ from typing import TYPE_CHECKING
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config.model import PROCESSED_LOGPROBS_MODES
|
||||
from vllm.logger import init_logger
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.outputs import LogprobsLists, LogprobsTensors, SamplerOutput
|
||||
@@ -67,10 +68,7 @@ class RejectionSampler(nn.Module):
|
||||
self.sampler = sampler
|
||||
self.use_fp64_gumbel = getattr(sampler, "use_fp64_gumbel", False)
|
||||
logprobs_mode = self.sampler.logprobs_mode
|
||||
self.is_processed_logprobs_mode = logprobs_mode in (
|
||||
"processed_logprobs",
|
||||
"processed_logits",
|
||||
)
|
||||
self.is_processed_logprobs_mode = logprobs_mode in PROCESSED_LOGPROBS_MODES
|
||||
self.is_logits_logprobs_mode = logprobs_mode in (
|
||||
"raw_logits",
|
||||
"processed_logits",
|
||||
|
||||
Reference in New Issue
Block a user