[MRV2][Spec Decode] Avoid rejection sampler OOM by chunking (#48630)
Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Nick Hill
parent
521aa80f71
commit
ac36a7a1e7
@@ -7,7 +7,7 @@ import torch.nn as nn
|
||||
|
||||
from vllm import envs
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config.model import LogprobsMode
|
||||
from vllm.config.model import PROCESSED_LOGPROBS_MODES, LogprobsMode
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
@@ -87,7 +87,7 @@ class TopKTopPSampler(nn.Module):
|
||||
# FlashInfer doesn't expose post-top-k/top-p logits/logprobs,
|
||||
# so it can't be used when the configured mode requires them.
|
||||
can_use_flashinfer = (
|
||||
logprobs_mode not in ("processed_logits", "processed_logprobs")
|
||||
logprobs_mode not in PROCESSED_LOGPROBS_MODES
|
||||
and flashinfer_sampler_supported()
|
||||
)
|
||||
self.forward = (
|
||||
@@ -108,7 +108,7 @@ class TopKTopPSampler(nn.Module):
|
||||
else:
|
||||
self.forward = self.forward_native
|
||||
elif (
|
||||
logprobs_mode not in ("processed_logits", "processed_logprobs")
|
||||
logprobs_mode not in PROCESSED_LOGPROBS_MODES
|
||||
and rocm_aiter_ops.is_enabled()
|
||||
):
|
||||
self.aiter_ops = None
|
||||
@@ -165,7 +165,7 @@ class TopKTopPSampler(nn.Module):
|
||||
return self.forward_native(logits, generators, k, p)
|
||||
if self.use_fp64_gumbel:
|
||||
return self.forward_native(logits, generators, k, p)
|
||||
assert self.logprobs_mode not in ("processed_logits", "processed_logprobs"), (
|
||||
assert self.logprobs_mode not in PROCESSED_LOGPROBS_MODES, (
|
||||
"FlashInfer does not support returning logits/logprobs"
|
||||
)
|
||||
# flashinfer sampling functions expect contiguous logits.
|
||||
@@ -236,10 +236,9 @@ class TopKTopPSampler(nn.Module):
|
||||
return self.forward_native(logits, generators, k, p)
|
||||
if self.use_fp64_gumbel:
|
||||
return self.forward_native(logits, generators, k, p)
|
||||
assert self.logprobs_mode not in (
|
||||
"processed_logits",
|
||||
"processed_logprobs",
|
||||
), "aiter sampler does not support returning logits/logprobs."
|
||||
assert self.logprobs_mode not in PROCESSED_LOGPROBS_MODES, (
|
||||
"aiter sampler does not support returning logits/logprobs."
|
||||
)
|
||||
if self.aiter_ops is None and not self._init_aiter_ops():
|
||||
return self.forward_native(logits, generators, k, p)
|
||||
return self.aiter_sample(logits, k, p, generators), None
|
||||
@@ -300,10 +299,7 @@ class TopKTopPSampler(nn.Module):
|
||||
logits.shape[0], dtype=torch.int64, device=logits.device
|
||||
)
|
||||
logits_to_return = None
|
||||
if (
|
||||
self.logprobs_mode == "processed_logits"
|
||||
or self.logprobs_mode == "processed_logprobs"
|
||||
):
|
||||
if self.logprobs_mode in PROCESSED_LOGPROBS_MODES:
|
||||
logits_to_return = torch.empty_like(logits)
|
||||
|
||||
assert len(generators) != logits.shape[0], (
|
||||
|
||||
@@ -10,6 +10,7 @@ from typing import TYPE_CHECKING
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config.model import PROCESSED_LOGPROBS_MODES
|
||||
from vllm.logger import init_logger
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.outputs import LogprobsLists, LogprobsTensors, SamplerOutput
|
||||
@@ -67,10 +68,7 @@ class RejectionSampler(nn.Module):
|
||||
self.sampler = sampler
|
||||
self.use_fp64_gumbel = getattr(sampler, "use_fp64_gumbel", False)
|
||||
logprobs_mode = self.sampler.logprobs_mode
|
||||
self.is_processed_logprobs_mode = logprobs_mode in (
|
||||
"processed_logprobs",
|
||||
"processed_logits",
|
||||
)
|
||||
self.is_processed_logprobs_mode = logprobs_mode in PROCESSED_LOGPROBS_MODES
|
||||
self.is_logits_logprobs_mode = logprobs_mode in (
|
||||
"raw_logits",
|
||||
"processed_logits",
|
||||
|
||||
Reference in New Issue
Block a user