[MRV2][Spec Decode] Avoid rejection sampler OOM by chunking (#48630)

Signed-off-by: mgoin <mgoin64@gmail.com>
Signed-off-by: Michael Goin <mgoin64@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Michael Goin
2026-07-23 18:13:13 +08:00
committed by GitHub
co-authored by Nick Hill
parent 521aa80f71
commit ac36a7a1e7
12 changed files with 394 additions and 71 deletions
+2 -4
View File
@@ -37,6 +37,7 @@ from vllm.config import (
update_config,
)
from vllm.config.cache import CacheConfig
from vllm.config.model import PROCESSED_LOGPROBS_MODES
from vllm.distributed.ec_transfer import get_ec_transfer, has_ec_transfer
from vllm.distributed.eplb.eplb_state import EplbState
from vllm.distributed.kv_transfer import get_kv_transfer_group, has_kv_transfer_group
@@ -6215,10 +6216,7 @@ class GPUModelRunner(
# memory during profile_run.
# No .clone() of logits: warmup output is discarded, so any in-place
# mutation by forward_native does not affect correctness.
if self.sampler.logprobs_mode not in (
"processed_logits",
"processed_logprobs",
):
if self.sampler.logprobs_mode not in PROCESSED_LOGPROBS_MODES:
self.sampler(
logits=logits,
sampling_metadata=replace(