From 93e2ab71119ff08805adc93be75196450382b088 Mon Sep 17 00:00:00 2001 From: Tyler Michael Smith Date: Tue, 7 Jul 2026 08:52:41 -0400 Subject: [PATCH] Disable dynamic speculative decoding when DP is enabled (#45963) Signed-off-by: Tyler Michael Smith Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Claude Opus 4.6 (1M context) --- .../dynamic_speculative_decoding.md | 1 + tests/v1/core/utils.py | 11 ++++++- tests/v1/spec_decode/test_dynamic_sd.py | 29 +++++++++++++++++++ vllm/config/vllm.py | 20 +++++++++++++ 4 files changed, 60 insertions(+), 1 deletion(-) diff --git a/docs/features/speculative_decoding/dynamic_speculative_decoding.md b/docs/features/speculative_decoding/dynamic_speculative_decoding.md index aa55c5f08d..682eaafd29 100644 --- a/docs/features/speculative_decoding/dynamic_speculative_decoding.md +++ b/docs/features/speculative_decoding/dynamic_speculative_decoding.md @@ -73,3 +73,4 @@ VLLM_USE_V2_MODEL_RUNNER=0 vllm serve meta-llama/Llama-3.1-8B-Instruct \ * Tested with Eagle, Eagle-3, and DFlash. Other SD methods may or may not work out of the box * Full Cudagraph only works with Model Runner V2. MRv1 only supports piece-wise cuda graph with this feature +* Not compatible with data parallelism (`--data-parallel-size > 1`). Each DP rank schedules independently, so ranks can pick different K values, causing DP collective divergence and deadlocks. When DP is enabled, vLLM automatically disables `num_speculative_tokens_per_batch_size` and falls back to the static `num_speculative_tokens` value. diff --git a/tests/v1/core/utils.py b/tests/v1/core/utils.py index 3a375ec757..d347a5e2d4 100644 --- a/tests/v1/core/utils.py +++ b/tests/v1/core/utils.py @@ -58,6 +58,8 @@ def create_scheduler( skip_tokenizer_init: bool = False, async_scheduling: bool = False, pipeline_parallel_size: int = 1, + data_parallel_size: int = 1, + num_speculative_tokens_per_batch_size: list[tuple[int, int, int]] | None = None, use_ec_connector: bool = False, ec_role: str | None = None, use_v2_model_runner: bool | None = None, @@ -130,6 +132,10 @@ def create_scheduler( spec_kwargs: dict = dict( model="ngram", num_speculative_tokens=num_speculative_tokens ) + if num_speculative_tokens_per_batch_size is not None: + spec_kwargs["num_speculative_tokens_per_batch_size"] = ( + num_speculative_tokens_per_batch_size + ) if speculative_method is not None: spec_kwargs["method"] = speculative_method spec_kwargs["prompt_lookup_max"] = num_speculative_tokens @@ -150,7 +156,10 @@ def create_scheduler( scheduler_config=scheduler_config, model_config=model_config, cache_config=cache_config, - parallel_config=ParallelConfig(pipeline_parallel_size=pipeline_parallel_size), + parallel_config=ParallelConfig( + pipeline_parallel_size=pipeline_parallel_size, + data_parallel_size=data_parallel_size, + ), kv_transfer_config=kv_transfer_config, speculative_config=speculative_config, ec_transfer_config=ec_transfer_config, diff --git a/tests/v1/spec_decode/test_dynamic_sd.py b/tests/v1/spec_decode/test_dynamic_sd.py index fe9f30ba25..8d46b241c6 100644 --- a/tests/v1/spec_decode/test_dynamic_sd.py +++ b/tests/v1/spec_decode/test_dynamic_sd.py @@ -2,6 +2,8 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Regression tests for the Dynamic SD batch-size schedule helpers.""" +import logging + import pytest from tests.v1.core.utils import create_requests, create_scheduler @@ -187,6 +189,33 @@ def test_scheduler_falls_back_to_static_k_when_dsd_not_configured(): assert output.num_spec_tokens_to_schedule == 3 +def test_dynamic_sd_is_disabled_with_data_parallel(caplog_vllm): + with caplog_vllm.at_level(logging.WARNING, logger="vllm"): + scheduler = create_scheduler( + max_num_seqs=256, + max_num_batched_tokens=2560, + num_speculative_tokens=3, + num_speculative_tokens_per_batch_size=[ + (1, 16, 3), + (64, 128, 2), + (256, 4096, 0), + ], + data_parallel_size=2, + ) + + speculative_config = scheduler.vllm_config.speculative_config + assert speculative_config is not None + assert speculative_config.num_speculative_tokens_per_batch_size is None + assert scheduler.dynamic_sd_lookup is None + assert "Dynamic speculative decoding is not supported with data parallelism" in ( + caplog_vllm.text + ) + + output = _add_requests_and_schedule(scheduler, 256) + assert len(output.num_scheduled_tokens) == 256 + assert output.num_spec_tokens_to_schedule == 3 + + def test_scheduler_uses_static_k_when_no_requests_are_scheduled(): scheduler = _make_scheduler_with_dynamic_sd( [(1, 16, 3), (64, 128, 2), (256, 4096, 0)], diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index e8796a3db8..a1fe109289 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -787,6 +787,25 @@ class VllmConfig: ) self.compilation_config.cudagraph_mode = CUDAGraphMode.PIECEWISE + def _maybe_disable_dynamic_sd_for_data_parallel(self) -> None: + speculative_config = self.speculative_config + if ( + speculative_config is None + or not speculative_config.uses_dynamic_speculative_decoding() + or self.parallel_config.data_parallel_size <= 1 + ): + return + + logger.warning_once( + "Dynamic speculative decoding is not supported with data " + "parallelism because data-parallel ranks can select different " + "speculative-token counts, causing DP divergence and deadlocks. " + "Disabling num_speculative_tokens_per_batch_size and falling back " + "to static num_speculative_tokens=%d.", + speculative_config.num_speculative_tokens, + ) + speculative_config.num_speculative_tokens_per_batch_size = None + def _post_init_kv_transfer_config(self) -> None: """Update KVTransferConfig based on top-level configs in VllmConfig. @@ -1201,6 +1220,7 @@ class VllmConfig: "optimization level defaults." ) + self._maybe_disable_dynamic_sd_for_data_parallel() self._maybe_override_dynamic_sd_cudagraph_mode() if (