Disable dynamic speculative decoding when DP is enabled (#45963)

Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Tyler Michael Smith
2026-07-07 12:52:41 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Claude Opus 4.6
parent 8b745527cd
commit 93e2ab7111
4 changed files with 60 additions and 1 deletions
@@ -73,3 +73,4 @@ VLLM_USE_V2_MODEL_RUNNER=0 vllm serve meta-llama/Llama-3.1-8B-Instruct \
* Tested with Eagle, Eagle-3, and DFlash. Other SD methods may or may not work out of the box
* Full Cudagraph only works with Model Runner V2. MRv1 only supports piece-wise cuda graph with this feature
* Not compatible with data parallelism (`--data-parallel-size > 1`). Each DP rank schedules independently, so ranks can pick different K values, causing DP collective divergence and deadlocks. When DP is enabled, vLLM automatically disables `num_speculative_tokens_per_batch_size` and falls back to the static `num_speculative_tokens` value.
+10 -1
View File
@@ -58,6 +58,8 @@ def create_scheduler(
skip_tokenizer_init: bool = False,
async_scheduling: bool = False,
pipeline_parallel_size: int = 1,
data_parallel_size: int = 1,
num_speculative_tokens_per_batch_size: list[tuple[int, int, int]] | None = None,
use_ec_connector: bool = False,
ec_role: str | None = None,
use_v2_model_runner: bool | None = None,
@@ -130,6 +132,10 @@ def create_scheduler(
spec_kwargs: dict = dict(
model="ngram", num_speculative_tokens=num_speculative_tokens
)
if num_speculative_tokens_per_batch_size is not None:
spec_kwargs["num_speculative_tokens_per_batch_size"] = (
num_speculative_tokens_per_batch_size
)
if speculative_method is not None:
spec_kwargs["method"] = speculative_method
spec_kwargs["prompt_lookup_max"] = num_speculative_tokens
@@ -150,7 +156,10 @@ def create_scheduler(
scheduler_config=scheduler_config,
model_config=model_config,
cache_config=cache_config,
parallel_config=ParallelConfig(pipeline_parallel_size=pipeline_parallel_size),
parallel_config=ParallelConfig(
pipeline_parallel_size=pipeline_parallel_size,
data_parallel_size=data_parallel_size,
),
kv_transfer_config=kv_transfer_config,
speculative_config=speculative_config,
ec_transfer_config=ec_transfer_config,
+29
View File
@@ -2,6 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Regression tests for the Dynamic SD batch-size schedule helpers."""
import logging
import pytest
from tests.v1.core.utils import create_requests, create_scheduler
@@ -187,6 +189,33 @@ def test_scheduler_falls_back_to_static_k_when_dsd_not_configured():
assert output.num_spec_tokens_to_schedule == 3
def test_dynamic_sd_is_disabled_with_data_parallel(caplog_vllm):
with caplog_vllm.at_level(logging.WARNING, logger="vllm"):
scheduler = create_scheduler(
max_num_seqs=256,
max_num_batched_tokens=2560,
num_speculative_tokens=3,
num_speculative_tokens_per_batch_size=[
(1, 16, 3),
(64, 128, 2),
(256, 4096, 0),
],
data_parallel_size=2,
)
speculative_config = scheduler.vllm_config.speculative_config
assert speculative_config is not None
assert speculative_config.num_speculative_tokens_per_batch_size is None
assert scheduler.dynamic_sd_lookup is None
assert "Dynamic speculative decoding is not supported with data parallelism" in (
caplog_vllm.text
)
output = _add_requests_and_schedule(scheduler, 256)
assert len(output.num_scheduled_tokens) == 256
assert output.num_spec_tokens_to_schedule == 3
def test_scheduler_uses_static_k_when_no_requests_are_scheduled():
scheduler = _make_scheduler_with_dynamic_sd(
[(1, 16, 3), (64, 128, 2), (256, 4096, 0)],
+20
View File
@@ -787,6 +787,25 @@ class VllmConfig:
)
self.compilation_config.cudagraph_mode = CUDAGraphMode.PIECEWISE
def _maybe_disable_dynamic_sd_for_data_parallel(self) -> None:
speculative_config = self.speculative_config
if (
speculative_config is None
or not speculative_config.uses_dynamic_speculative_decoding()
or self.parallel_config.data_parallel_size <= 1
):
return
logger.warning_once(
"Dynamic speculative decoding is not supported with data "
"parallelism because data-parallel ranks can select different "
"speculative-token counts, causing DP divergence and deadlocks. "
"Disabling num_speculative_tokens_per_batch_size and falling back "
"to static num_speculative_tokens=%d.",
speculative_config.num_speculative_tokens,
)
speculative_config.num_speculative_tokens_per_batch_size = None
def _post_init_kv_transfer_config(self) -> None:
"""Update KVTransferConfig based on top-level configs in VllmConfig.
@@ -1201,6 +1220,7 @@ class VllmConfig:
"optimization level defaults."
)
self._maybe_disable_dynamic_sd_for_data_parallel()
self._maybe_override_dynamic_sd_cudagraph_mode()
if (