[MRV2][Spec Decode] Avoid rejection sampler OOM by chunking (#48630)
Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Nick Hill
parent
521aa80f71
commit
ac36a7a1e7
@@ -37,6 +37,7 @@ from vllm.config import (
|
||||
update_config,
|
||||
)
|
||||
from vllm.config.cache import CacheConfig
|
||||
from vllm.config.model import PROCESSED_LOGPROBS_MODES
|
||||
from vllm.distributed.ec_transfer import get_ec_transfer, has_ec_transfer
|
||||
from vllm.distributed.eplb.eplb_state import EplbState
|
||||
from vllm.distributed.kv_transfer import get_kv_transfer_group, has_kv_transfer_group
|
||||
@@ -6215,10 +6216,7 @@ class GPUModelRunner(
|
||||
# memory during profile_run.
|
||||
# No .clone() of logits: warmup output is discarded, so any in-place
|
||||
# mutation by forward_native does not affect correctness.
|
||||
if self.sampler.logprobs_mode not in (
|
||||
"processed_logits",
|
||||
"processed_logprobs",
|
||||
):
|
||||
if self.sampler.logprobs_mode not in PROCESSED_LOGPROBS_MODES:
|
||||
self.sampler(
|
||||
logits=logits,
|
||||
sampling_metadata=replace(
|
||||
|
||||
Reference in New Issue
Block a user