[Bugfix][Spec Decode] Auto-enable async scheduling for draft models (#48341)
Signed-off-by: BWAAEEEK <jooho414@gmail.com>
This commit is contained in:
@@ -407,6 +407,29 @@ def test_reconfigure_for_independent_dp_rank_on_multinode_dense_model():
|
||||
assert parallel_config.world_size == 8
|
||||
|
||||
|
||||
def test_draft_model_enables_async_scheduling_by_default():
|
||||
parallel_config = ParallelConfig(distributed_executor_backend="uni")
|
||||
model_config = ModelConfig("Qwen/Qwen3-0.6B", max_model_len=2048)
|
||||
speculative_config = SpeculativeConfig(
|
||||
method="draft_model",
|
||||
model="Qwen/Qwen3-0.6B",
|
||||
num_speculative_tokens=3,
|
||||
target_model_config=model_config,
|
||||
target_parallel_config=parallel_config,
|
||||
)
|
||||
cfg = VllmConfig(
|
||||
model_config=model_config,
|
||||
scheduler_config=SchedulerConfig(
|
||||
max_model_len=2048,
|
||||
is_encoder_decoder=False,
|
||||
),
|
||||
parallel_config=parallel_config,
|
||||
speculative_config=speculative_config,
|
||||
)
|
||||
|
||||
assert cfg.scheduler_config.async_scheduling is True
|
||||
|
||||
|
||||
@dataclass
|
||||
class _TestConfigFields:
|
||||
a: int
|
||||
|
||||
@@ -25,10 +25,6 @@ from ..utils import (
|
||||
)
|
||||
|
||||
|
||||
class AsyncSchedulingNotEnabledError(AssertionError):
|
||||
"""Raised when draft-model spec decode does not enable async scheduling."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class ArgsTest:
|
||||
target_model: str
|
||||
@@ -93,22 +89,12 @@ cases = [
|
||||
@pytest.mark.parametrize("args", cases)
|
||||
@pytest.mark.parametrize("enforce_eager", [True, False])
|
||||
@single_gpu_only
|
||||
# TODO: Fix async_scheduling & engine initialization issues - see https://github.com/vllm-project/vllm/issues/38929
|
||||
@pytest.mark.xfail(
|
||||
raises=AsyncSchedulingNotEnabledError,
|
||||
reason="draft_model does not yet enable async_scheduling: issue #38929",
|
||||
)
|
||||
def test_draft_model_correctness(args: ArgsTest, enforce_eager: bool):
|
||||
args.enforce_eager = enforce_eager
|
||||
assert_draft_model_correctness(args)
|
||||
|
||||
|
||||
@single_gpu_only
|
||||
# TODO: Fix async_scheduling and engine initialization issues - see https://github.com/vllm-project/vllm/issues/38929
|
||||
@pytest.mark.xfail(
|
||||
raises=AsyncSchedulingNotEnabledError,
|
||||
reason="draft_model does not yet enable async_scheduling: issue #38929",
|
||||
)
|
||||
def test_draft_model_realistic_example():
|
||||
args = ArgsTest(
|
||||
target_model="Qwen/Qwen3-1.7B",
|
||||
@@ -124,11 +110,6 @@ def test_draft_model_realistic_example():
|
||||
|
||||
|
||||
@single_gpu_only
|
||||
# TODO: Fix async_scheduling and engine initialization issues - see https://github.com/vllm-project/vllm/issues/38929
|
||||
@pytest.mark.xfail(
|
||||
raises=AsyncSchedulingNotEnabledError,
|
||||
reason="draft_model does not yet enable async_scheduling: issue #38929",
|
||||
)
|
||||
def test_draft_model_parallel_drafting():
|
||||
args = ArgsTest(
|
||||
target_model="Qwen/Qwen3-1.7B",
|
||||
@@ -155,11 +136,6 @@ def test_draft_model_parallel_drafting():
|
||||
)
|
||||
@pytest.mark.parametrize("enforce_eager", [True, False])
|
||||
@single_gpu_only
|
||||
# TODO: Fix async_scheduling and engine initialization issues - see https://github.com/vllm-project/vllm/issues/38929
|
||||
@pytest.mark.xfail(
|
||||
raises=AsyncSchedulingNotEnabledError,
|
||||
reason="draft_model does not yet enable async_scheduling: issue #38929",
|
||||
)
|
||||
def test_draft_model_quantization(models: tuple[str, str], enforce_eager: bool):
|
||||
tgt_model, draft_model = models
|
||||
sd_case = ArgsTest(
|
||||
@@ -172,11 +148,6 @@ def test_draft_model_quantization(models: tuple[str, str], enforce_eager: bool):
|
||||
|
||||
|
||||
@multi_gpu_only(num_gpus=2)
|
||||
# TODO: Fix async_scheduling and engine initialization issues - see https://github.com/vllm-project/vllm/issues/38929
|
||||
@pytest.mark.xfail(
|
||||
raises=AsyncSchedulingNotEnabledError,
|
||||
reason="draft_model does not yet enable async_scheduling: issue #38929",
|
||||
)
|
||||
def test_draft_model_tensor_parallelism():
|
||||
"""Ensure spec decode works when running with TP > 1."""
|
||||
_skip_if_insufficient_gpus_for_tp(2)
|
||||
@@ -372,17 +343,8 @@ def assert_draft_model_correctness(args: ArgsTest):
|
||||
|
||||
assert acceptance_rate >= args.expected_acceptance_rate
|
||||
assert acceptance_len >= args.expected_acceptance_len
|
||||
# draft_model supports async scheduling; assert it is active by default.
|
||||
# Raise AsyncSchedulingNotEnabledError (a subclass of AssertionError) so that
|
||||
# @pytest.mark.xfail(raises=AsyncSchedulingNotEnabledError) catches only this
|
||||
# specific failure — leaving all other assertion failures (e.g. correctness or
|
||||
# acceptance-rate checks above) visible as real test failures.
|
||||
has_async = spec_llm.llm_engine.vllm_config.scheduler_config.async_scheduling
|
||||
del spec_llm # CLEANUP
|
||||
torch.accelerator.empty_cache()
|
||||
cleanup_dist_env_and_memory()
|
||||
if not has_async:
|
||||
raise AsyncSchedulingNotEnabledError(
|
||||
"Expected async_scheduling=True for draft_model spec decode, got False."
|
||||
" See https://github.com/vllm-project/vllm/issues/38929"
|
||||
)
|
||||
assert has_async, "Expected async_scheduling=True for draft_model spec decode"
|
||||
|
||||
@@ -1126,6 +1126,7 @@ class VllmConfig:
|
||||
self.speculative_config is not None
|
||||
and self.speculative_config.method not in get_args(EagleModelTypes)
|
||||
and self.speculative_config.method not in get_args(NgramGPUTypes)
|
||||
and self.speculative_config.method != "draft_model"
|
||||
and self.speculative_config.method != "dspark"
|
||||
):
|
||||
logger.warning_once(
|
||||
|
||||
Reference in New Issue
Block a user