[Spec Decode] Don't reserve DFlash/DSpark draft slots in the scheduler token budget (#59468)

Signed-off-by: zixi-qi <zixi@inferact.ai>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
qizixi
2026-10-16 01:23:58 +08:00
committed by GitHub
co-authored by Claude Opus 5.5 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 338dab077c
commit 7cd7982dfd
4 changed files with 73 additions and 12 deletions
+2 -2
View File
@@ -2095,8 +2095,8 @@ def test_draft_parallel_config_preserves_ep_without_model(
[
pytest.param("eagle3", False, 0, id="eagle3"),
pytest.param("eagle3", True, 7, id="p-eagle"),
pytest.param("dflash", True, 8, id="dflash"),
pytest.param("dspark", True, 7, id="dspark"),
pytest.param("dflash", True, 0, id="dflash"),
pytest.param("dspark", True, 0, id="dspark"),
pytest.param("mtp", False, 0, id="mtp"),
pytest.param("ngram", False, 0, id="ngram"),
pytest.param("draft_model", False, 1, id="draft-model"),
+40
View File
@@ -2159,6 +2159,46 @@ def test_draft_slots_budgeted_per_scheduled_request(tmp_path, monkeypatch):
assert scheduler.schedule().num_scheduled_tokens == {"0": 10, "1": 4}
@pytest.mark.parametrize(("method", "max_num_reqs"), [("dflash", 4), ("dspark", 5)])
def test_block_drafters_cap_requests_instead_of_draft_slots(
tmp_path, method, max_num_reqs
):
(tmp_path / "config.json").write_text(
'{"architectures": ["OPTForCausalLM"], "model_type": "opt"}'
)
base = create_scheduler(
model=str(tmp_path),
max_num_batched_tokens=20,
num_speculative_tokens=4,
parallel_drafting=True,
skip_tokenizer_init=True,
)
# DFlash/DSpark need a resolvable draft model, so switch the ngram config in
# place and redo the config step. DSpark then has 4 lookahead tokens, not 5.
vllm_config = base.vllm_config
object.__setattr__(vllm_config.speculative_config, "method", method)
object.__setattr__(
vllm_config.speculative_config,
"draft_model_config",
SimpleNamespace(hf_config=SimpleNamespace()),
)
vllm_config._set_max_num_scheduled_tokens()
scheduler = Scheduler(
vllm_config=vllm_config,
kv_cache_config=base.kv_cache_config,
structured_output_manager=base.structured_output_manager,
block_size=base.block_size,
)
for request in create_requests(num_requests=8, num_tokens=2):
scheduler.add_request(request)
# Admission stops at 20 // num_lookahead_tokens, and no draft slots are
# charged: each admitted prompt is scheduled in full.
assert scheduler.schedule().num_scheduled_tokens == {
str(i): 2 for i in range(max_num_reqs)
}
# Note - these test cases mirror some of those in test_rejection_sampler.py
@pytest.mark.parametrize(
"spec_tokens,output_tokens,expected,expected_per_req",