[Scheduler] Tune --long-prefill-token-threshold adaptiveness (#58459)

Signed-off-by: Robert Shaw <robertgshaw2@gmail.com>
This commit is contained in:
Robert Shaw
2026-09-23 22:23:18 -04:00
committed by GitHub
parent e9f167129c
commit 6697a7dd30
5 changed files with 54 additions and 1 deletions
+5
View File
@@ -58,6 +58,7 @@ def create_scheduler(
enable_chunked_prefill: bool = True,
enable_prefix_caching: bool = False,
long_prefill_token_threshold: int = 0,
long_prefill_token_threshold_adaptive: bool = False,
disable_chunked_mm_input: bool = False,
use_kv_connector: None | bool | str | MockKVConfig = None,
kv_role: str = "kv_both",
@@ -90,6 +91,9 @@ def create_scheduler(
enable_prefix_caching: optionally force APC config
(True/False) or use default
(False)
long_prefill_token_threshold: cap on prefill chunk size
long_prefill_token_threshold_adaptive: floor the cap at a
fair share of the token budget
Returns:
{class}`Scheduler` instance
@@ -115,6 +119,7 @@ def create_scheduler(
max_num_batched_tokens=max_num_batched_tokens,
max_model_len=max_model_len,
long_prefill_token_threshold=long_prefill_token_threshold,
long_prefill_token_threshold_adaptive=(long_prefill_token_threshold_adaptive),
disable_chunked_mm_input=disable_chunked_mm_input,
enable_chunked_prefill=enable_chunked_prefill,
async_scheduling=async_scheduling,