[Scheduler] Tune --long-prefill-token-threshold adaptiveness (#58459)
Signed-off-by: Robert Shaw <robertgshaw2@gmail.com>
This commit is contained in:
@@ -796,6 +796,26 @@ def test_long_prefill_threshold_applies_with_other_requests():
|
||||
assert output.num_scheduled_tokens[short_req.request_id] == 10
|
||||
|
||||
|
||||
def test_long_prefill_threshold_floored_by_fair_share():
|
||||
"""With the adaptive flag, the effective threshold never falls below the
|
||||
fair share of the token budget: max_num_batched_tokens / num queued +
|
||||
running requests."""
|
||||
scheduler = create_scheduler(
|
||||
max_num_batched_tokens=1024,
|
||||
long_prefill_token_threshold=100,
|
||||
long_prefill_token_threshold_adaptive=True,
|
||||
)
|
||||
long_req = create_requests(num_requests=1, num_tokens=2000)[0]
|
||||
short_req = create_requests(num_requests=1, num_tokens=10, req_ids=["short"])[0]
|
||||
for request in [long_req, short_req]:
|
||||
scheduler.add_request(request)
|
||||
|
||||
output = scheduler.schedule()
|
||||
# 100 is below the fair share (1024 // 2 = 512), so the floor binds.
|
||||
assert output.num_scheduled_tokens[long_req.request_id] == 512
|
||||
assert output.num_scheduled_tokens[short_req.request_id] == 10
|
||||
|
||||
|
||||
def test_update_from_output_routes_sampling_masks_by_request():
|
||||
"""Each request receives the sampler row at its own batch index."""
|
||||
scheduler = create_scheduler()
|
||||
|
||||
@@ -58,6 +58,7 @@ def create_scheduler(
|
||||
enable_chunked_prefill: bool = True,
|
||||
enable_prefix_caching: bool = False,
|
||||
long_prefill_token_threshold: int = 0,
|
||||
long_prefill_token_threshold_adaptive: bool = False,
|
||||
disable_chunked_mm_input: bool = False,
|
||||
use_kv_connector: None | bool | str | MockKVConfig = None,
|
||||
kv_role: str = "kv_both",
|
||||
@@ -90,6 +91,9 @@ def create_scheduler(
|
||||
enable_prefix_caching: optionally force APC config
|
||||
(True/False) or use default
|
||||
(False)
|
||||
long_prefill_token_threshold: cap on prefill chunk size
|
||||
long_prefill_token_threshold_adaptive: floor the cap at a
|
||||
fair share of the token budget
|
||||
|
||||
Returns:
|
||||
{class}`Scheduler` instance
|
||||
@@ -115,6 +119,7 @@ def create_scheduler(
|
||||
max_num_batched_tokens=max_num_batched_tokens,
|
||||
max_model_len=max_model_len,
|
||||
long_prefill_token_threshold=long_prefill_token_threshold,
|
||||
long_prefill_token_threshold_adaptive=(long_prefill_token_threshold_adaptive),
|
||||
disable_chunked_mm_input=disable_chunked_mm_input,
|
||||
enable_chunked_prefill=enable_chunked_prefill,
|
||||
async_scheduling=async_scheduling,
|
||||
|
||||
Reference in New Issue
Block a user