[Spec Decode] Don't reserve DFlash/DSpark draft slots in the scheduler token budget (#59468)
Signed-off-by: zixi-qi <zixi@inferact.ai> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
338dab077c
commit
7cd7982dfd
@@ -2095,8 +2095,8 @@ def test_draft_parallel_config_preserves_ep_without_model(
|
||||
[
|
||||
pytest.param("eagle3", False, 0, id="eagle3"),
|
||||
pytest.param("eagle3", True, 7, id="p-eagle"),
|
||||
pytest.param("dflash", True, 8, id="dflash"),
|
||||
pytest.param("dspark", True, 7, id="dspark"),
|
||||
pytest.param("dflash", True, 0, id="dflash"),
|
||||
pytest.param("dspark", True, 0, id="dspark"),
|
||||
pytest.param("mtp", False, 0, id="mtp"),
|
||||
pytest.param("ngram", False, 0, id="ngram"),
|
||||
pytest.param("draft_model", False, 1, id="draft-model"),
|
||||
|
||||
@@ -2159,6 +2159,46 @@ def test_draft_slots_budgeted_per_scheduled_request(tmp_path, monkeypatch):
|
||||
assert scheduler.schedule().num_scheduled_tokens == {"0": 10, "1": 4}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("method", "max_num_reqs"), [("dflash", 4), ("dspark", 5)])
|
||||
def test_block_drafters_cap_requests_instead_of_draft_slots(
|
||||
tmp_path, method, max_num_reqs
|
||||
):
|
||||
(tmp_path / "config.json").write_text(
|
||||
'{"architectures": ["OPTForCausalLM"], "model_type": "opt"}'
|
||||
)
|
||||
base = create_scheduler(
|
||||
model=str(tmp_path),
|
||||
max_num_batched_tokens=20,
|
||||
num_speculative_tokens=4,
|
||||
parallel_drafting=True,
|
||||
skip_tokenizer_init=True,
|
||||
)
|
||||
# DFlash/DSpark need a resolvable draft model, so switch the ngram config in
|
||||
# place and redo the config step. DSpark then has 4 lookahead tokens, not 5.
|
||||
vllm_config = base.vllm_config
|
||||
object.__setattr__(vllm_config.speculative_config, "method", method)
|
||||
object.__setattr__(
|
||||
vllm_config.speculative_config,
|
||||
"draft_model_config",
|
||||
SimpleNamespace(hf_config=SimpleNamespace()),
|
||||
)
|
||||
vllm_config._set_max_num_scheduled_tokens()
|
||||
scheduler = Scheduler(
|
||||
vllm_config=vllm_config,
|
||||
kv_cache_config=base.kv_cache_config,
|
||||
structured_output_manager=base.structured_output_manager,
|
||||
block_size=base.block_size,
|
||||
)
|
||||
for request in create_requests(num_requests=8, num_tokens=2):
|
||||
scheduler.add_request(request)
|
||||
|
||||
# Admission stops at 20 // num_lookahead_tokens, and no draft slots are
|
||||
# charged: each admitted prompt is scheduled in full.
|
||||
assert scheduler.schedule().num_scheduled_tokens == {
|
||||
str(i): 2 for i in range(max_num_reqs)
|
||||
}
|
||||
|
||||
|
||||
# Note - these test cases mirror some of those in test_rejection_sampler.py
|
||||
@pytest.mark.parametrize(
|
||||
"spec_tokens,output_tokens,expected,expected_per_req",
|
||||
|
||||
Reference in New Issue
Block a user