diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 53b9784dad..050cc56119 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -1056,6 +1056,28 @@ class VllmConfig: ) self.compilation_config.cudagraph_mode = CUDAGraphMode.PIECEWISE + def _maybe_disable_dynamic_sd_for_v1(self) -> None: + speculative_config = self.speculative_config + # V2 truncates drafts centrally. V1 requires proposer support. + if ( + speculative_config is None + or not speculative_config.uses_dynamic_speculative_decoding() + or self.use_v2_model_runner + or speculative_config.method + in ("ngram", "eagle", "eagle3", "mtp", "draft_model", "dflash") + ): + return + + logger.warning_once( + "Dynamic speculative decoding is not supported with the '%s' " + "speculative method on Model Runner V1. " + "Disabling num_speculative_tokens_per_batch_size " + "and falling back to static num_speculative_tokens=%d.", + speculative_config.method, + speculative_config.num_speculative_tokens, + ) + speculative_config.num_speculative_tokens_per_batch_size = None + def _maybe_disable_dynamic_sd_for_data_parallel(self) -> None: speculative_config = self.speculative_config if ( @@ -1829,6 +1851,7 @@ class VllmConfig: ) self._maybe_disable_dynamic_sd_for_data_parallel() + self._maybe_disable_dynamic_sd_for_v1() self._maybe_override_dynamic_sd_cudagraph_mode() if (