[Perf] Adaptive budget for spec scheduled input tokens, ~60% better Kimi K3 DSpark TTFT (#51725)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -1280,6 +1280,30 @@ def test_reset_connector_cache_no_connector_is_no_op_success():
|
||||
assert scheduler.reset_prefix_cache(reset_connector=True) is True
|
||||
|
||||
|
||||
def test_draft_slots_budgeted_per_scheduled_request(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0")
|
||||
(tmp_path / "config.json").write_text(
|
||||
'{"architectures": ["OPTForCausalLM"], "model_type": "opt"}'
|
||||
)
|
||||
scheduler = create_scheduler(
|
||||
model=str(tmp_path),
|
||||
max_num_seqs=16,
|
||||
max_num_batched_tokens=20,
|
||||
num_speculative_tokens=4,
|
||||
parallel_drafting=True,
|
||||
skip_tokenizer_init=True,
|
||||
)
|
||||
speculative_config = scheduler.vllm_config.speculative_config
|
||||
assert speculative_config is not None
|
||||
assert scheduler.max_num_scheduled_tokens == 20
|
||||
assert speculative_config.max_num_new_slots_for_drafting == 3
|
||||
|
||||
for request in create_requests(num_requests=2, num_tokens=10):
|
||||
scheduler.add_request(request)
|
||||
|
||||
assert scheduler.schedule().num_scheduled_tokens == {"0": 10, "1": 4}
|
||||
|
||||
|
||||
# Note - these test cases mirror some of those in test_rejection_sampler.py
|
||||
@pytest.mark.parametrize(
|
||||
"spec_tokens,output_tokens,expected",
|
||||
|
||||
@@ -61,6 +61,7 @@ def create_scheduler(
|
||||
max_model_len: int | None = None,
|
||||
num_speculative_tokens: int | None = None,
|
||||
speculative_method: str | None = None,
|
||||
parallel_drafting: bool = False,
|
||||
skip_tokenizer_init: bool = False,
|
||||
async_scheduling: bool = False,
|
||||
pipeline_parallel_size: int = 1,
|
||||
@@ -151,6 +152,7 @@ def create_scheduler(
|
||||
spec_kwargs["prompt_lookup_max"] = num_speculative_tokens
|
||||
spec_kwargs["prompt_lookup_min"] = 1
|
||||
speculative_config = SpeculativeConfig(**spec_kwargs)
|
||||
speculative_config.parallel_drafting = parallel_drafting
|
||||
|
||||
ec_transfer_config = (
|
||||
ECTransferConfig(
|
||||
|
||||
Reference in New Issue
Block a user