[Spec Decode] Enable async scheduling for DFlash (#58065)

This commit is contained in:
Jeff (Junze) Ma
2026-09-21 22:05:44 -07:00
committed by GitHub
parent 42a85a4976
commit 0961bbae28
2 changed files with 69 additions and 2 deletions
+65
View File
@@ -1568,6 +1568,71 @@ def test_draft_model_enables_async_scheduling_by_default():
assert cfg.scheduler_config.async_scheduling is True
def test_dflash_allows_async_scheduling(tmp_path: Path):
"""DFlash must stay async-schedulable: it was the only parallel-drafting
method excluded from the allowlist."""
from transformers import LlamaConfig
common = dict(
hidden_size=128,
intermediate_size=256,
num_hidden_layers=2,
num_attention_heads=4,
num_key_value_heads=2,
vocab_size=256,
max_position_embeddings=2048,
)
target_path = tmp_path / "target"
draft_path = tmp_path / "draft"
_write_json(
target_path / "config.json",
LlamaConfig(architectures=["LlamaForCausalLM"], **common).to_dict(),
)
_write_json(
draft_path / "config.json",
dict(
common,
architectures=["DFlash2DraftModel"],
model_type="qwen3",
num_hidden_layers=1,
layer_types=["sliding_attention"],
sliding_window=128,
dflash_config=dict(
block_size=4,
mask_token_id=255,
target_layer_ids=[0],
conv_group_size=2,
conv_kernel_size=2,
selector_rank=8,
selector_top_k=4,
),
),
)
parallel_config = ParallelConfig(distributed_executor_backend="uni")
model_config = ModelConfig(
model=str(target_path), tokenizer_mode="skip", max_model_len=2048
)
speculative_config = SpeculativeConfig(
method="dflash",
model=str(draft_path),
num_speculative_tokens=3,
target_model_config=model_config,
target_parallel_config=parallel_config,
)
cfg = VllmConfig(
model_config=model_config,
scheduler_config=SchedulerConfig(
max_model_len=2048,
is_encoder_decoder=False,
),
parallel_config=parallel_config,
speculative_config=speculative_config,
)
assert speculative_config.method == "dflash"
assert cfg.scheduler_config.async_scheduling is True
@pytest.mark.skip_global_cleanup
@pytest.mark.parametrize("tp_size", [1, 2])
@pytest.mark.parametrize("target_ep", [False, True], ids=["ep-off", "ep-on"])
+4 -2
View File
@@ -1457,11 +1457,12 @@ class VllmConfig:
and self.speculative_config.method not in get_args(NgramGPUTypes)
and self.speculative_config.method != "draft_model"
and self.speculative_config.method != "dspark"
and self.speculative_config.method != "dflash"
):
raise ValueError(
"Currently, async scheduling is only supported "
"with EAGLE/MTP/Draft Model/NGram GPU/DSpark kind of "
"speculative decoding"
"with EAGLE/MTP/Draft Model/NGram GPU/DSpark/DFlash "
"kind of speculative decoding"
)
if self.speculative_config.disable_padded_drafter_batch:
raise ValueError(
@@ -1490,6 +1491,7 @@ class VllmConfig:
and self.speculative_config.method not in get_args(NgramGPUTypes)
and self.speculative_config.method != "draft_model"
and self.speculative_config.method != "dspark"
and self.speculative_config.method != "dflash"
):
logger.warning_once(
"Async scheduling not supported with %s-based "