From 763e3f7c911308fa609fb9437642929efe955798 Mon Sep 17 00:00:00 2001 From: Ting SUN Date: Sun, 18 Oct 2026 00:41:04 +0800 Subject: [PATCH] [Bugfix][Spec Decode] Disable dynamic speculative decoding for fixed-K proposers instead of crashing (#46399) Signed-off-by: Ting Sun Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> --- vllm/config/vllm.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 53b9784dad..050cc56119 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -1056,6 +1056,28 @@ class VllmConfig: ) self.compilation_config.cudagraph_mode = CUDAGraphMode.PIECEWISE + def _maybe_disable_dynamic_sd_for_v1(self) -> None: + speculative_config = self.speculative_config + # V2 truncates drafts centrally. V1 requires proposer support. + if ( + speculative_config is None + or not speculative_config.uses_dynamic_speculative_decoding() + or self.use_v2_model_runner + or speculative_config.method + in ("ngram", "eagle", "eagle3", "mtp", "draft_model", "dflash") + ): + return + + logger.warning_once( + "Dynamic speculative decoding is not supported with the '%s' " + "speculative method on Model Runner V1. " + "Disabling num_speculative_tokens_per_batch_size " + "and falling back to static num_speculative_tokens=%d.", + speculative_config.method, + speculative_config.num_speculative_tokens, + ) + speculative_config.num_speculative_tokens_per_batch_size = None + def _maybe_disable_dynamic_sd_for_data_parallel(self) -> None: speculative_config = self.speculative_config if ( @@ -1829,6 +1851,7 @@ class VllmConfig: ) self._maybe_disable_dynamic_sd_for_data_parallel() + self._maybe_disable_dynamic_sd_for_v1() self._maybe_override_dynamic_sd_cudagraph_mode() if (