[Spec Decode] Enable async scheduling for DFlash (#58065)
This commit is contained in:
@@ -1568,6 +1568,71 @@ def test_draft_model_enables_async_scheduling_by_default():
|
||||
assert cfg.scheduler_config.async_scheduling is True
|
||||
|
||||
|
||||
def test_dflash_allows_async_scheduling(tmp_path: Path):
|
||||
"""DFlash must stay async-schedulable: it was the only parallel-drafting
|
||||
method excluded from the allowlist."""
|
||||
from transformers import LlamaConfig
|
||||
|
||||
common = dict(
|
||||
hidden_size=128,
|
||||
intermediate_size=256,
|
||||
num_hidden_layers=2,
|
||||
num_attention_heads=4,
|
||||
num_key_value_heads=2,
|
||||
vocab_size=256,
|
||||
max_position_embeddings=2048,
|
||||
)
|
||||
target_path = tmp_path / "target"
|
||||
draft_path = tmp_path / "draft"
|
||||
_write_json(
|
||||
target_path / "config.json",
|
||||
LlamaConfig(architectures=["LlamaForCausalLM"], **common).to_dict(),
|
||||
)
|
||||
_write_json(
|
||||
draft_path / "config.json",
|
||||
dict(
|
||||
common,
|
||||
architectures=["DFlash2DraftModel"],
|
||||
model_type="qwen3",
|
||||
num_hidden_layers=1,
|
||||
layer_types=["sliding_attention"],
|
||||
sliding_window=128,
|
||||
dflash_config=dict(
|
||||
block_size=4,
|
||||
mask_token_id=255,
|
||||
target_layer_ids=[0],
|
||||
conv_group_size=2,
|
||||
conv_kernel_size=2,
|
||||
selector_rank=8,
|
||||
selector_top_k=4,
|
||||
),
|
||||
),
|
||||
)
|
||||
parallel_config = ParallelConfig(distributed_executor_backend="uni")
|
||||
model_config = ModelConfig(
|
||||
model=str(target_path), tokenizer_mode="skip", max_model_len=2048
|
||||
)
|
||||
speculative_config = SpeculativeConfig(
|
||||
method="dflash",
|
||||
model=str(draft_path),
|
||||
num_speculative_tokens=3,
|
||||
target_model_config=model_config,
|
||||
target_parallel_config=parallel_config,
|
||||
)
|
||||
cfg = VllmConfig(
|
||||
model_config=model_config,
|
||||
scheduler_config=SchedulerConfig(
|
||||
max_model_len=2048,
|
||||
is_encoder_decoder=False,
|
||||
),
|
||||
parallel_config=parallel_config,
|
||||
speculative_config=speculative_config,
|
||||
)
|
||||
|
||||
assert speculative_config.method == "dflash"
|
||||
assert cfg.scheduler_config.async_scheduling is True
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
@pytest.mark.parametrize("tp_size", [1, 2])
|
||||
@pytest.mark.parametrize("target_ep", [False, True], ids=["ep-off", "ep-on"])
|
||||
|
||||
+4
-2
@@ -1457,11 +1457,12 @@ class VllmConfig:
|
||||
and self.speculative_config.method not in get_args(NgramGPUTypes)
|
||||
and self.speculative_config.method != "draft_model"
|
||||
and self.speculative_config.method != "dspark"
|
||||
and self.speculative_config.method != "dflash"
|
||||
):
|
||||
raise ValueError(
|
||||
"Currently, async scheduling is only supported "
|
||||
"with EAGLE/MTP/Draft Model/NGram GPU/DSpark kind of "
|
||||
"speculative decoding"
|
||||
"with EAGLE/MTP/Draft Model/NGram GPU/DSpark/DFlash "
|
||||
"kind of speculative decoding"
|
||||
)
|
||||
if self.speculative_config.disable_padded_drafter_batch:
|
||||
raise ValueError(
|
||||
@@ -1490,6 +1491,7 @@ class VllmConfig:
|
||||
and self.speculative_config.method not in get_args(NgramGPUTypes)
|
||||
and self.speculative_config.method != "draft_model"
|
||||
and self.speculative_config.method != "dspark"
|
||||
and self.speculative_config.method != "dflash"
|
||||
):
|
||||
logger.warning_once(
|
||||
"Async scheduling not supported with %s-based "
|
||||
|
||||
Reference in New Issue
Block a user