[Spec Decode] Don't reserve DFlash/DSpark draft slots in the scheduler token budget (#59468)
Signed-off-by: zixi-qi <zixi@inferact.ai> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
338dab077c
commit
7cd7982dfd
@@ -1853,19 +1853,21 @@ class SpeculativeConfig:
|
||||
==================== ============= ======== ================
|
||||
EAGLE3 eagle3 No 0
|
||||
P-EAGLE eagle3 Yes K - 1
|
||||
DFlash dflash Yes K
|
||||
DSpark dspark Yes K - 1
|
||||
DFlash dflash Yes 0
|
||||
DSpark dspark Yes 0
|
||||
MTP mtp No 0
|
||||
N-gram ngram No 0
|
||||
Draft model draft_model No 1
|
||||
PARD draft_model Yes K
|
||||
==================== ============= ======== ================
|
||||
|
||||
DFlash and DSpark draft in a separate query pass rather than adding
|
||||
rows to the target batch.
|
||||
"""
|
||||
num_draft_tokens = self.num_speculative_tokens
|
||||
|
||||
if self.use_dflash():
|
||||
# DFlash uses one bonus query followed by K mask queries.
|
||||
return num_draft_tokens
|
||||
if self.use_dflash() or self.use_dspark():
|
||||
return 0
|
||||
|
||||
if self.parallel_drafting:
|
||||
if self.uses_draft_model():
|
||||
|
||||
+24
-5
@@ -2300,9 +2300,6 @@ class VllmConfig:
|
||||
worker is configured to handle.
|
||||
"""
|
||||
if self.speculative_config is not None:
|
||||
scheduled_token_delta = (
|
||||
self.speculative_config.max_num_new_slots_for_drafting
|
||||
)
|
||||
max_num_batched_tokens = self.scheduler_config.max_num_batched_tokens
|
||||
if self.scheduler_config.max_num_scheduled_tokens is None:
|
||||
self.scheduler_config.max_num_scheduled_tokens = max_num_batched_tokens
|
||||
@@ -2326,12 +2323,34 @@ class VllmConfig:
|
||||
" num_speculative_tokens.",
|
||||
)
|
||||
|
||||
if max_num_batched_tokens <= scheduled_token_delta:
|
||||
# num_lookahead_tokens is at least every method's per-request draft
|
||||
# slots, and covers the DFlash/DSpark draft pass's query rows.
|
||||
if max_num_batched_tokens <= self.num_lookahead_tokens:
|
||||
raise ValueError(
|
||||
"VllmConfig does not have enough slots to schedule a token and"
|
||||
" support the speculative decoding settings."
|
||||
f" Got {max_num_batched_tokens=} and {scheduled_token_delta=}."
|
||||
f" Got {max_num_batched_tokens=} and"
|
||||
f" num_lookahead_tokens={self.num_lookahead_tokens}."
|
||||
)
|
||||
spec = self.speculative_config
|
||||
if spec.use_dflash() or spec.use_dspark():
|
||||
max_num_active_seqs = (
|
||||
self.scheduler_config.max_num_active_seqs
|
||||
or self.scheduler_config.max_num_seqs
|
||||
)
|
||||
max_draft_seqs = max_num_batched_tokens // self.num_lookahead_tokens
|
||||
if max_draft_seqs < max_num_active_seqs:
|
||||
logger.warning_once(
|
||||
"The %s draft pass needs %d query tokens per request within "
|
||||
"max_num_batched_tokens=%d. Overriding max_num_active_seqs "
|
||||
"from %d to %d.",
|
||||
spec.method,
|
||||
self.num_lookahead_tokens,
|
||||
max_num_batched_tokens,
|
||||
max_num_active_seqs,
|
||||
max_draft_seqs,
|
||||
)
|
||||
self.scheduler_config.max_num_active_seqs = max_draft_seqs
|
||||
|
||||
def _set_cudagraph_sizes(self):
|
||||
"""VLLM defines the default candidate list of batch sizes for CUDA graph
|
||||
|
||||
Reference in New Issue
Block a user