[Scheduler] Soften Long Prefill Tokens Threshhold (#57951)

Signed-off-by: Robert Shaw <robertgshaw2@gmail.com>
Signed-off-by: Robert Shaw <robshaw@redhat.com>
Co-authored-by: Robert Shaw <robshaw@redhat.com>
This commit is contained in:
Robert Shaw
2026-09-21 17:19:42 -07:00
committed by GitHub
co-authored by Robert Shaw
parent 78d7836525
commit c64b15cde5
5 changed files with 52 additions and 11 deletions
+1 -1
View File
@@ -417,7 +417,7 @@ def test_abort_mid_prefill_defers_free():
chunk is still in flight must withhold its blocks. chunk is still in flight must withhold its blocks.
""" """
scheduler = create_scheduler( scheduler = create_scheduler(
model=MODEL, async_scheduling=True, long_prefill_token_threshold=16 model=MODEL, async_scheduling=True, max_num_batched_tokens=16
) )
scheduler.defer_block_free = True scheduler.defer_block_free = True
pool = scheduler.kv_cache_manager.block_pool pool = scheduler.kv_cache_manager.block_pool
+30
View File
@@ -762,6 +762,36 @@ def test_schedule_concurrent_partial_requests(enable_prefix_caching: bool):
assert output2.num_scheduled_tokens[requests[2].request_id] == 800 - 224 - 224 assert output2.num_scheduled_tokens[requests[2].request_id] == 800 - 224 - 224
def test_long_prefill_threshold_ignored_when_alone():
"""A lone long prefill is not capped by the threshold: it has no other
request to starve."""
scheduler = create_scheduler(
max_num_batched_tokens=1024,
long_prefill_token_threshold=400,
)
request = create_requests(num_requests=1, num_tokens=2000)[0]
scheduler.add_request(request)
output = scheduler.schedule()
assert output.num_scheduled_tokens[request.request_id] == 1024
def test_long_prefill_threshold_applies_with_other_requests():
"""The threshold caps the prefill as soon as another request is queued."""
scheduler = create_scheduler(
max_num_batched_tokens=1024,
long_prefill_token_threshold=400,
)
long_req = create_requests(num_requests=1, num_tokens=2000)[0]
short_req = create_requests(num_requests=1, num_tokens=10, req_ids=["short"])[0]
for request in [long_req, short_req]:
scheduler.add_request(request)
output = scheduler.schedule()
assert output.num_scheduled_tokens[long_req.request_id] == 400
assert output.num_scheduled_tokens[short_req.request_id] == 10
def test_update_from_output_routes_sampling_masks_by_request(): def test_update_from_output_routes_sampling_masks_by_request():
"""Each request receives the sampler row at its own batch index.""" """Each request receives the sampler row at its own batch index."""
scheduler = create_scheduler() scheduler = create_scheduler()
@@ -77,13 +77,13 @@ def test_pooling_chunked_prefill(vllm_runner, monkeypatch):
chunk_size = 10 chunk_size = 10
# Set chunking parameters to force chunked prefill # Cap the token budget to force chunked prefill; the
# Note: Chunked prefill is automatically handled by vLLM # long_prefill_token_threshold is not applied to a lone request.
# internally based on the model size and prompt
with vllm_runner( with vllm_runner(
model_id, model_id,
runner="pooling", runner="pooling",
long_prefill_token_threshold=chunk_size, max_num_batched_tokens=chunk_size,
max_num_seqs=1,
tensor_parallel_size=1, tensor_parallel_size=1,
enforce_eager=True, enforce_eager=True,
enable_chunked_prefill=True, enable_chunked_prefill=True,
+4 -1
View File
@@ -79,7 +79,10 @@ class SchedulerConfig:
long_prefill_token_threshold: int = Field(default=0, ge=0) long_prefill_token_threshold: int = Field(default=0, ge=0)
"""For chunked prefill, a request is considered long if the prompt is """For chunked prefill, a request is considered long if the prompt is
longer than this number of tokens. 0 disables the cap (default).""" longer than this number of tokens. 0 disables the cap (default).
The cap is not applied when the request is the only one in the batch,
since there is no other request for it to starve."""
max_num_queued_reqs: int | None = Field(default=None, ge=0) max_num_queued_reqs: int | None = Field(default=None, ge=0)
"""Maximum number of requests that can be in-flight (waiting or running) """Maximum number of requests that can be in-flight (waiting or running)
+13 -5
View File
@@ -599,6 +599,15 @@ class Scheduler(SchedulerInterface):
throttle_prefills and not self.prefill_capacity_bound throttle_prefills and not self.prefill_capacity_bound
) and any(not r.is_prefill_chunk for r in self.running) ) and any(not r.is_prefill_chunk for r in self.running)
# `long_prefill_token_threshold` exists to stop a long prefill from
# starving other requests of the token budget. When it is the only
# request there is nobody to starve, so let it use the whole budget.
long_prefill_token_threshold = (
self.scheduler_config.long_prefill_token_threshold
if len(self.running) + len(self.waiting) + len(self.skipped_waiting) > 1
else 0
)
# First, schedule the RUNNING requests. # First, schedule the RUNNING requests.
req_index = 0 req_index = 0
while req_index < len(self.running) and token_budget > 0: while req_index < len(self.running) and token_budget > 0:
@@ -650,8 +659,8 @@ class Scheduler(SchedulerInterface):
+ request.num_output_placeholders + request.num_output_placeholders
- request.num_computed_tokens - request.num_computed_tokens
) )
if 0 < self.scheduler_config.long_prefill_token_threshold < num_new_tokens: if 0 < long_prefill_token_threshold < num_new_tokens:
num_new_tokens = self.scheduler_config.long_prefill_token_threshold num_new_tokens = long_prefill_token_threshold
num_new_tokens = min( num_new_tokens = min(
num_new_tokens, token_budget, input_budget - draft_slots num_new_tokens, token_budget, input_budget - draft_slots
) )
@@ -1090,9 +1099,8 @@ class Scheduler(SchedulerInterface):
num_new_tokens = padded_num_tokens num_new_tokens = padded_num_tokens
pad_spec_decode = True pad_spec_decode = True
threshold = self.scheduler_config.long_prefill_token_threshold if 0 < long_prefill_token_threshold < num_new_tokens:
if 0 < threshold < num_new_tokens: num_new_tokens = long_prefill_token_threshold
num_new_tokens = threshold
# chunked prefill has to be enabled explicitly to allow # chunked prefill has to be enabled explicitly to allow
# pooling requests to be chunked # pooling requests to be chunked