[Scheduler] Tune --long-prefill-token-threshold adaptiveness (#58459)

Signed-off-by: Robert Shaw <robertgshaw2@gmail.com>
This commit is contained in:
Robert Shaw
2026-09-23 22:23:18 -04:00
committed by GitHub
parent e9f167129c
commit 6697a7dd30
5 changed files with 54 additions and 1 deletions
+20
View File
@@ -796,6 +796,26 @@ def test_long_prefill_threshold_applies_with_other_requests():
assert output.num_scheduled_tokens[short_req.request_id] == 10
def test_long_prefill_threshold_floored_by_fair_share():
"""With the adaptive flag, the effective threshold never falls below the
fair share of the token budget: max_num_batched_tokens / num queued +
running requests."""
scheduler = create_scheduler(
max_num_batched_tokens=1024,
long_prefill_token_threshold=100,
long_prefill_token_threshold_adaptive=True,
)
long_req = create_requests(num_requests=1, num_tokens=2000)[0]
short_req = create_requests(num_requests=1, num_tokens=10, req_ids=["short"])[0]
for request in [long_req, short_req]:
scheduler.add_request(request)
output = scheduler.schedule()
# 100 is below the fair share (1024 // 2 = 512), so the floor binds.
assert output.num_scheduled_tokens[long_req.request_id] == 512
assert output.num_scheduled_tokens[short_req.request_id] == 10
def test_update_from_output_routes_sampling_masks_by_request():
"""Each request receives the sampler row at its own batch index."""
scheduler = create_scheduler()