[Bugfix][Spec Decode] Disable dynamic speculative decoding for fixed-K proposers instead of crashing (#46399)

Signed-off-by: Ting Sun <suntcrick@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
Ting SUN
2026-10-17 12:41:04 -04:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Wentao Ye
parent e744eacf68
commit 763e3f7c91
+23
View File
@@ -1056,6 +1056,28 @@ class VllmConfig:
)
self.compilation_config.cudagraph_mode = CUDAGraphMode.PIECEWISE
def _maybe_disable_dynamic_sd_for_v1(self) -> None:
speculative_config = self.speculative_config
# V2 truncates drafts centrally. V1 requires proposer support.
if (
speculative_config is None
or not speculative_config.uses_dynamic_speculative_decoding()
or self.use_v2_model_runner
or speculative_config.method
in ("ngram", "eagle", "eagle3", "mtp", "draft_model", "dflash")
):
return
logger.warning_once(
"Dynamic speculative decoding is not supported with the '%s' "
"speculative method on Model Runner V1. "
"Disabling num_speculative_tokens_per_batch_size "
"and falling back to static num_speculative_tokens=%d.",
speculative_config.method,
speculative_config.num_speculative_tokens,
)
speculative_config.num_speculative_tokens_per_batch_size = None
def _maybe_disable_dynamic_sd_for_data_parallel(self) -> None:
speculative_config = self.speculative_config
if (
@@ -1829,6 +1851,7 @@ class VllmConfig:
)
self._maybe_disable_dynamic_sd_for_data_parallel()
self._maybe_disable_dynamic_sd_for_v1()
self._maybe_override_dynamic_sd_cudagraph_mode()
if (