Disable dynamic speculative decoding when DP is enabled (#45963)
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Claude Opus 4.6
parent
8b745527cd
commit
93e2ab7111
@@ -73,3 +73,4 @@ VLLM_USE_V2_MODEL_RUNNER=0 vllm serve meta-llama/Llama-3.1-8B-Instruct \
|
||||
|
||||
* Tested with Eagle, Eagle-3, and DFlash. Other SD methods may or may not work out of the box
|
||||
* Full Cudagraph only works with Model Runner V2. MRv1 only supports piece-wise cuda graph with this feature
|
||||
* Not compatible with data parallelism (`--data-parallel-size > 1`). Each DP rank schedules independently, so ranks can pick different K values, causing DP collective divergence and deadlocks. When DP is enabled, vLLM automatically disables `num_speculative_tokens_per_batch_size` and falls back to the static `num_speculative_tokens` value.
|
||||
|
||||
+10
-1
@@ -58,6 +58,8 @@ def create_scheduler(
|
||||
skip_tokenizer_init: bool = False,
|
||||
async_scheduling: bool = False,
|
||||
pipeline_parallel_size: int = 1,
|
||||
data_parallel_size: int = 1,
|
||||
num_speculative_tokens_per_batch_size: list[tuple[int, int, int]] | None = None,
|
||||
use_ec_connector: bool = False,
|
||||
ec_role: str | None = None,
|
||||
use_v2_model_runner: bool | None = None,
|
||||
@@ -130,6 +132,10 @@ def create_scheduler(
|
||||
spec_kwargs: dict = dict(
|
||||
model="ngram", num_speculative_tokens=num_speculative_tokens
|
||||
)
|
||||
if num_speculative_tokens_per_batch_size is not None:
|
||||
spec_kwargs["num_speculative_tokens_per_batch_size"] = (
|
||||
num_speculative_tokens_per_batch_size
|
||||
)
|
||||
if speculative_method is not None:
|
||||
spec_kwargs["method"] = speculative_method
|
||||
spec_kwargs["prompt_lookup_max"] = num_speculative_tokens
|
||||
@@ -150,7 +156,10 @@ def create_scheduler(
|
||||
scheduler_config=scheduler_config,
|
||||
model_config=model_config,
|
||||
cache_config=cache_config,
|
||||
parallel_config=ParallelConfig(pipeline_parallel_size=pipeline_parallel_size),
|
||||
parallel_config=ParallelConfig(
|
||||
pipeline_parallel_size=pipeline_parallel_size,
|
||||
data_parallel_size=data_parallel_size,
|
||||
),
|
||||
kv_transfer_config=kv_transfer_config,
|
||||
speculative_config=speculative_config,
|
||||
ec_transfer_config=ec_transfer_config,
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Regression tests for the Dynamic SD batch-size schedule helpers."""
|
||||
|
||||
import logging
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.v1.core.utils import create_requests, create_scheduler
|
||||
@@ -187,6 +189,33 @@ def test_scheduler_falls_back_to_static_k_when_dsd_not_configured():
|
||||
assert output.num_spec_tokens_to_schedule == 3
|
||||
|
||||
|
||||
def test_dynamic_sd_is_disabled_with_data_parallel(caplog_vllm):
|
||||
with caplog_vllm.at_level(logging.WARNING, logger="vllm"):
|
||||
scheduler = create_scheduler(
|
||||
max_num_seqs=256,
|
||||
max_num_batched_tokens=2560,
|
||||
num_speculative_tokens=3,
|
||||
num_speculative_tokens_per_batch_size=[
|
||||
(1, 16, 3),
|
||||
(64, 128, 2),
|
||||
(256, 4096, 0),
|
||||
],
|
||||
data_parallel_size=2,
|
||||
)
|
||||
|
||||
speculative_config = scheduler.vllm_config.speculative_config
|
||||
assert speculative_config is not None
|
||||
assert speculative_config.num_speculative_tokens_per_batch_size is None
|
||||
assert scheduler.dynamic_sd_lookup is None
|
||||
assert "Dynamic speculative decoding is not supported with data parallelism" in (
|
||||
caplog_vllm.text
|
||||
)
|
||||
|
||||
output = _add_requests_and_schedule(scheduler, 256)
|
||||
assert len(output.num_scheduled_tokens) == 256
|
||||
assert output.num_spec_tokens_to_schedule == 3
|
||||
|
||||
|
||||
def test_scheduler_uses_static_k_when_no_requests_are_scheduled():
|
||||
scheduler = _make_scheduler_with_dynamic_sd(
|
||||
[(1, 16, 3), (64, 128, 2), (256, 4096, 0)],
|
||||
|
||||
@@ -787,6 +787,25 @@ class VllmConfig:
|
||||
)
|
||||
self.compilation_config.cudagraph_mode = CUDAGraphMode.PIECEWISE
|
||||
|
||||
def _maybe_disable_dynamic_sd_for_data_parallel(self) -> None:
|
||||
speculative_config = self.speculative_config
|
||||
if (
|
||||
speculative_config is None
|
||||
or not speculative_config.uses_dynamic_speculative_decoding()
|
||||
or self.parallel_config.data_parallel_size <= 1
|
||||
):
|
||||
return
|
||||
|
||||
logger.warning_once(
|
||||
"Dynamic speculative decoding is not supported with data "
|
||||
"parallelism because data-parallel ranks can select different "
|
||||
"speculative-token counts, causing DP divergence and deadlocks. "
|
||||
"Disabling num_speculative_tokens_per_batch_size and falling back "
|
||||
"to static num_speculative_tokens=%d.",
|
||||
speculative_config.num_speculative_tokens,
|
||||
)
|
||||
speculative_config.num_speculative_tokens_per_batch_size = None
|
||||
|
||||
def _post_init_kv_transfer_config(self) -> None:
|
||||
"""Update KVTransferConfig based on top-level configs in VllmConfig.
|
||||
|
||||
@@ -1201,6 +1220,7 @@ class VllmConfig:
|
||||
"optimization level defaults."
|
||||
)
|
||||
|
||||
self._maybe_disable_dynamic_sd_for_data_parallel()
|
||||
self._maybe_override_dynamic_sd_cudagraph_mode()
|
||||
|
||||
if (
|
||||
|
||||
Reference in New Issue
Block a user