[MRV2][Spec Decode] Avoid rejection sampler OOM by chunking (#48630)

Signed-off-by: mgoin <mgoin64@gmail.com>
Signed-off-by: Michael Goin <mgoin64@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Michael Goin
2026-07-23 18:13:13 +08:00
committed by GitHub
co-authored by Nick Hill
parent 521aa80f71
commit ac36a7a1e7
12 changed files with 394 additions and 71 deletions
+2 -4
View File
@@ -10,6 +10,7 @@ from typing import TYPE_CHECKING
import torch
import torch.nn as nn
from vllm.config.model import PROCESSED_LOGPROBS_MODES
from vllm.logger import init_logger
from vllm.triton_utils import tl, triton
from vllm.v1.outputs import LogprobsLists, LogprobsTensors, SamplerOutput
@@ -67,10 +68,7 @@ class RejectionSampler(nn.Module):
self.sampler = sampler
self.use_fp64_gumbel = getattr(sampler, "use_fp64_gumbel", False)
logprobs_mode = self.sampler.logprobs_mode
self.is_processed_logprobs_mode = logprobs_mode in (
"processed_logprobs",
"processed_logits",
)
self.is_processed_logprobs_mode = logprobs_mode in PROCESSED_LOGPROBS_MODES
self.is_logits_logprobs_mode = logprobs_mode in (
"raw_logits",
"processed_logits",