[Spec Decode] Don't reserve DFlash/DSpark draft slots in the scheduler token budget (#59468)

Signed-off-by: zixi-qi <zixi@inferact.ai>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
qizixi
2026-10-16 01:23:58 +08:00
committed by GitHub
co-authored by Claude Opus 5.5 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 338dab077c
commit 7cd7982dfd
4 changed files with 73 additions and 12 deletions
+7 -5
View File
@@ -1853,19 +1853,21 @@ class SpeculativeConfig:
==================== ============= ======== ================
EAGLE3 eagle3 No 0
P-EAGLE eagle3 Yes K - 1
DFlash dflash Yes K
DSpark dspark Yes K - 1
DFlash dflash Yes 0
DSpark dspark Yes 0
MTP mtp No 0
N-gram ngram No 0
Draft model draft_model No 1
PARD draft_model Yes K
==================== ============= ======== ================
DFlash and DSpark draft in a separate query pass rather than adding
rows to the target batch.
"""
num_draft_tokens = self.num_speculative_tokens
if self.use_dflash():
# DFlash uses one bonus query followed by K mask queries.
return num_draft_tokens
if self.use_dflash() or self.use_dspark():
return 0
if self.parallel_drafting:
if self.uses_draft_model():
+24 -5
View File
@@ -2300,9 +2300,6 @@ class VllmConfig:
worker is configured to handle.
"""
if self.speculative_config is not None:
scheduled_token_delta = (
self.speculative_config.max_num_new_slots_for_drafting
)
max_num_batched_tokens = self.scheduler_config.max_num_batched_tokens
if self.scheduler_config.max_num_scheduled_tokens is None:
self.scheduler_config.max_num_scheduled_tokens = max_num_batched_tokens
@@ -2326,12 +2323,34 @@ class VllmConfig:
" num_speculative_tokens.",
)
if max_num_batched_tokens <= scheduled_token_delta:
# num_lookahead_tokens is at least every method's per-request draft
# slots, and covers the DFlash/DSpark draft pass's query rows.
if max_num_batched_tokens <= self.num_lookahead_tokens:
raise ValueError(
"VllmConfig does not have enough slots to schedule a token and"
" support the speculative decoding settings."
f" Got {max_num_batched_tokens=} and {scheduled_token_delta=}."
f" Got {max_num_batched_tokens=} and"
f" num_lookahead_tokens={self.num_lookahead_tokens}."
)
spec = self.speculative_config
if spec.use_dflash() or spec.use_dspark():
max_num_active_seqs = (
self.scheduler_config.max_num_active_seqs
or self.scheduler_config.max_num_seqs
)
max_draft_seqs = max_num_batched_tokens // self.num_lookahead_tokens
if max_draft_seqs < max_num_active_seqs:
logger.warning_once(
"The %s draft pass needs %d query tokens per request within "
"max_num_batched_tokens=%d. Overriding max_num_active_seqs "
"from %d to %d.",
spec.method,
self.num_lookahead_tokens,
max_num_batched_tokens,
max_num_active_seqs,
max_draft_seqs,
)
self.scheduler_config.max_num_active_seqs = max_draft_seqs
def _set_cudagraph_sizes(self):
"""VLLM defines the default candidate list of batch sizes for CUDA graph