[Scheduler] Soften Long Prefill Tokens Threshhold (#57951)

Signed-off-by: Robert Shaw <robertgshaw2@gmail.com>
Signed-off-by: Robert Shaw <robshaw@redhat.com>
Co-authored-by: Robert Shaw <robshaw@redhat.com>
This commit is contained in:
Robert Shaw
2026-09-21 17:19:42 -07:00
committed by GitHub
co-authored by Robert Shaw
parent 78d7836525
commit c64b15cde5
5 changed files with 52 additions and 11 deletions
+1 -1
View File
@@ -417,7 +417,7 @@ def test_abort_mid_prefill_defers_free():
chunk is still in flight must withhold its blocks.
"""
scheduler = create_scheduler(
model=MODEL, async_scheduling=True, long_prefill_token_threshold=16
model=MODEL, async_scheduling=True, max_num_batched_tokens=16
)
scheduler.defer_block_free = True
pool = scheduler.kv_cache_manager.block_pool
+30
View File
@@ -762,6 +762,36 @@ def test_schedule_concurrent_partial_requests(enable_prefix_caching: bool):
assert output2.num_scheduled_tokens[requests[2].request_id] == 800 - 224 - 224
def test_long_prefill_threshold_ignored_when_alone():
"""A lone long prefill is not capped by the threshold: it has no other
request to starve."""
scheduler = create_scheduler(
max_num_batched_tokens=1024,
long_prefill_token_threshold=400,
)
request = create_requests(num_requests=1, num_tokens=2000)[0]
scheduler.add_request(request)
output = scheduler.schedule()
assert output.num_scheduled_tokens[request.request_id] == 1024
def test_long_prefill_threshold_applies_with_other_requests():
"""The threshold caps the prefill as soon as another request is queued."""
scheduler = create_scheduler(
max_num_batched_tokens=1024,
long_prefill_token_threshold=400,
)
long_req = create_requests(num_requests=1, num_tokens=2000)[0]
short_req = create_requests(num_requests=1, num_tokens=10, req_ids=["short"])[0]
for request in [long_req, short_req]:
scheduler.add_request(request)
output = scheduler.schedule()
assert output.num_scheduled_tokens[long_req.request_id] == 400
assert output.num_scheduled_tokens[short_req.request_id] == 10
def test_update_from_output_routes_sampling_masks_by_request():
"""Each request receives the sampler row at its own batch index."""
scheduler = create_scheduler()
@@ -77,13 +77,13 @@ def test_pooling_chunked_prefill(vllm_runner, monkeypatch):
chunk_size = 10
# Set chunking parameters to force chunked prefill
# Note: Chunked prefill is automatically handled by vLLM
# internally based on the model size and prompt
# Cap the token budget to force chunked prefill; the
# long_prefill_token_threshold is not applied to a lone request.
with vllm_runner(
model_id,
runner="pooling",
long_prefill_token_threshold=chunk_size,
max_num_batched_tokens=chunk_size,
max_num_seqs=1,
tensor_parallel_size=1,
enforce_eager=True,
enable_chunked_prefill=True,