Disable dynamic speculative decoding when DP is enabled (#45963)
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Claude Opus 4.6
parent
8b745527cd
commit
93e2ab7111
+10
-1
@@ -58,6 +58,8 @@ def create_scheduler(
|
||||
skip_tokenizer_init: bool = False,
|
||||
async_scheduling: bool = False,
|
||||
pipeline_parallel_size: int = 1,
|
||||
data_parallel_size: int = 1,
|
||||
num_speculative_tokens_per_batch_size: list[tuple[int, int, int]] | None = None,
|
||||
use_ec_connector: bool = False,
|
||||
ec_role: str | None = None,
|
||||
use_v2_model_runner: bool | None = None,
|
||||
@@ -130,6 +132,10 @@ def create_scheduler(
|
||||
spec_kwargs: dict = dict(
|
||||
model="ngram", num_speculative_tokens=num_speculative_tokens
|
||||
)
|
||||
if num_speculative_tokens_per_batch_size is not None:
|
||||
spec_kwargs["num_speculative_tokens_per_batch_size"] = (
|
||||
num_speculative_tokens_per_batch_size
|
||||
)
|
||||
if speculative_method is not None:
|
||||
spec_kwargs["method"] = speculative_method
|
||||
spec_kwargs["prompt_lookup_max"] = num_speculative_tokens
|
||||
@@ -150,7 +156,10 @@ def create_scheduler(
|
||||
scheduler_config=scheduler_config,
|
||||
model_config=model_config,
|
||||
cache_config=cache_config,
|
||||
parallel_config=ParallelConfig(pipeline_parallel_size=pipeline_parallel_size),
|
||||
parallel_config=ParallelConfig(
|
||||
pipeline_parallel_size=pipeline_parallel_size,
|
||||
data_parallel_size=data_parallel_size,
|
||||
),
|
||||
kv_transfer_config=kv_transfer_config,
|
||||
speculative_config=speculative_config,
|
||||
ec_transfer_config=ec_transfer_config,
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Regression tests for the Dynamic SD batch-size schedule helpers."""
|
||||
|
||||
import logging
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.v1.core.utils import create_requests, create_scheduler
|
||||
@@ -187,6 +189,33 @@ def test_scheduler_falls_back_to_static_k_when_dsd_not_configured():
|
||||
assert output.num_spec_tokens_to_schedule == 3
|
||||
|
||||
|
||||
def test_dynamic_sd_is_disabled_with_data_parallel(caplog_vllm):
|
||||
with caplog_vllm.at_level(logging.WARNING, logger="vllm"):
|
||||
scheduler = create_scheduler(
|
||||
max_num_seqs=256,
|
||||
max_num_batched_tokens=2560,
|
||||
num_speculative_tokens=3,
|
||||
num_speculative_tokens_per_batch_size=[
|
||||
(1, 16, 3),
|
||||
(64, 128, 2),
|
||||
(256, 4096, 0),
|
||||
],
|
||||
data_parallel_size=2,
|
||||
)
|
||||
|
||||
speculative_config = scheduler.vllm_config.speculative_config
|
||||
assert speculative_config is not None
|
||||
assert speculative_config.num_speculative_tokens_per_batch_size is None
|
||||
assert scheduler.dynamic_sd_lookup is None
|
||||
assert "Dynamic speculative decoding is not supported with data parallelism" in (
|
||||
caplog_vllm.text
|
||||
)
|
||||
|
||||
output = _add_requests_and_schedule(scheduler, 256)
|
||||
assert len(output.num_scheduled_tokens) == 256
|
||||
assert output.num_spec_tokens_to_schedule == 3
|
||||
|
||||
|
||||
def test_scheduler_uses_static_k_when_no_requests_are_scheduled():
|
||||
scheduler = _make_scheduler_with_dynamic_sd(
|
||||
[(1, 16, 3), (64, 128, 2), (256, 4096, 0)],
|
||||
|
||||
Reference in New Issue
Block a user