[Perf] Adaptive budget for spec scheduled input tokens, ~60% better Kimi K3 DSpark TTFT (#51725)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-08-10 20:52:21 -07:00
committed by GitHub
parent 0f2ea973be
commit 0914ed2e81
4 changed files with 54 additions and 24 deletions
+2
View File
@@ -61,6 +61,7 @@ def create_scheduler(
max_model_len: int | None = None,
num_speculative_tokens: int | None = None,
speculative_method: str | None = None,
parallel_drafting: bool = False,
skip_tokenizer_init: bool = False,
async_scheduling: bool = False,
pipeline_parallel_size: int = 1,
@@ -151,6 +152,7 @@ def create_scheduler(
spec_kwargs["prompt_lookup_max"] = num_speculative_tokens
spec_kwargs["prompt_lookup_min"] = 1
speculative_config = SpeculativeConfig(**spec_kwargs)
speculative_config.parallel_drafting = parallel_drafting
ec_transfer_config = (
ECTransferConfig(