[Bugfix][Spec Decode] Disable dynamic speculative decoding for fixed-K proposers instead of crashing (#46399)
Signed-off-by: Ting Sun <suntcrick@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Wentao Ye
parent
e744eacf68
commit
763e3f7c91
@@ -1056,6 +1056,28 @@ class VllmConfig:
|
||||
)
|
||||
self.compilation_config.cudagraph_mode = CUDAGraphMode.PIECEWISE
|
||||
|
||||
def _maybe_disable_dynamic_sd_for_v1(self) -> None:
|
||||
speculative_config = self.speculative_config
|
||||
# V2 truncates drafts centrally. V1 requires proposer support.
|
||||
if (
|
||||
speculative_config is None
|
||||
or not speculative_config.uses_dynamic_speculative_decoding()
|
||||
or self.use_v2_model_runner
|
||||
or speculative_config.method
|
||||
in ("ngram", "eagle", "eagle3", "mtp", "draft_model", "dflash")
|
||||
):
|
||||
return
|
||||
|
||||
logger.warning_once(
|
||||
"Dynamic speculative decoding is not supported with the '%s' "
|
||||
"speculative method on Model Runner V1. "
|
||||
"Disabling num_speculative_tokens_per_batch_size "
|
||||
"and falling back to static num_speculative_tokens=%d.",
|
||||
speculative_config.method,
|
||||
speculative_config.num_speculative_tokens,
|
||||
)
|
||||
speculative_config.num_speculative_tokens_per_batch_size = None
|
||||
|
||||
def _maybe_disable_dynamic_sd_for_data_parallel(self) -> None:
|
||||
speculative_config = self.speculative_config
|
||||
if (
|
||||
@@ -1829,6 +1851,7 @@ class VllmConfig:
|
||||
)
|
||||
|
||||
self._maybe_disable_dynamic_sd_for_data_parallel()
|
||||
self._maybe_disable_dynamic_sd_for_v1()
|
||||
self._maybe_override_dynamic_sd_cudagraph_mode()
|
||||
|
||||
if (
|
||||
|
||||
Reference in New Issue
Block a user