[Scheduler] Soften Long Prefill Tokens Threshhold (#57951)
Signed-off-by: Robert Shaw <robertgshaw2@gmail.com> Signed-off-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Robert Shaw <robshaw@redhat.com>
This commit is contained in:
co-authored by
Robert Shaw
parent
78d7836525
commit
c64b15cde5
@@ -417,7 +417,7 @@ def test_abort_mid_prefill_defers_free():
|
|||||||
chunk is still in flight must withhold its blocks.
|
chunk is still in flight must withhold its blocks.
|
||||||
"""
|
"""
|
||||||
scheduler = create_scheduler(
|
scheduler = create_scheduler(
|
||||||
model=MODEL, async_scheduling=True, long_prefill_token_threshold=16
|
model=MODEL, async_scheduling=True, max_num_batched_tokens=16
|
||||||
)
|
)
|
||||||
scheduler.defer_block_free = True
|
scheduler.defer_block_free = True
|
||||||
pool = scheduler.kv_cache_manager.block_pool
|
pool = scheduler.kv_cache_manager.block_pool
|
||||||
|
|||||||
@@ -762,6 +762,36 @@ def test_schedule_concurrent_partial_requests(enable_prefix_caching: bool):
|
|||||||
assert output2.num_scheduled_tokens[requests[2].request_id] == 800 - 224 - 224
|
assert output2.num_scheduled_tokens[requests[2].request_id] == 800 - 224 - 224
|
||||||
|
|
||||||
|
|
||||||
|
def test_long_prefill_threshold_ignored_when_alone():
|
||||||
|
"""A lone long prefill is not capped by the threshold: it has no other
|
||||||
|
request to starve."""
|
||||||
|
scheduler = create_scheduler(
|
||||||
|
max_num_batched_tokens=1024,
|
||||||
|
long_prefill_token_threshold=400,
|
||||||
|
)
|
||||||
|
request = create_requests(num_requests=1, num_tokens=2000)[0]
|
||||||
|
scheduler.add_request(request)
|
||||||
|
|
||||||
|
output = scheduler.schedule()
|
||||||
|
assert output.num_scheduled_tokens[request.request_id] == 1024
|
||||||
|
|
||||||
|
|
||||||
|
def test_long_prefill_threshold_applies_with_other_requests():
|
||||||
|
"""The threshold caps the prefill as soon as another request is queued."""
|
||||||
|
scheduler = create_scheduler(
|
||||||
|
max_num_batched_tokens=1024,
|
||||||
|
long_prefill_token_threshold=400,
|
||||||
|
)
|
||||||
|
long_req = create_requests(num_requests=1, num_tokens=2000)[0]
|
||||||
|
short_req = create_requests(num_requests=1, num_tokens=10, req_ids=["short"])[0]
|
||||||
|
for request in [long_req, short_req]:
|
||||||
|
scheduler.add_request(request)
|
||||||
|
|
||||||
|
output = scheduler.schedule()
|
||||||
|
assert output.num_scheduled_tokens[long_req.request_id] == 400
|
||||||
|
assert output.num_scheduled_tokens[short_req.request_id] == 10
|
||||||
|
|
||||||
|
|
||||||
def test_update_from_output_routes_sampling_masks_by_request():
|
def test_update_from_output_routes_sampling_masks_by_request():
|
||||||
"""Each request receives the sampler row at its own batch index."""
|
"""Each request receives the sampler row at its own batch index."""
|
||||||
scheduler = create_scheduler()
|
scheduler = create_scheduler()
|
||||||
|
|||||||
@@ -77,13 +77,13 @@ def test_pooling_chunked_prefill(vllm_runner, monkeypatch):
|
|||||||
|
|
||||||
chunk_size = 10
|
chunk_size = 10
|
||||||
|
|
||||||
# Set chunking parameters to force chunked prefill
|
# Cap the token budget to force chunked prefill; the
|
||||||
# Note: Chunked prefill is automatically handled by vLLM
|
# long_prefill_token_threshold is not applied to a lone request.
|
||||||
# internally based on the model size and prompt
|
|
||||||
with vllm_runner(
|
with vllm_runner(
|
||||||
model_id,
|
model_id,
|
||||||
runner="pooling",
|
runner="pooling",
|
||||||
long_prefill_token_threshold=chunk_size,
|
max_num_batched_tokens=chunk_size,
|
||||||
|
max_num_seqs=1,
|
||||||
tensor_parallel_size=1,
|
tensor_parallel_size=1,
|
||||||
enforce_eager=True,
|
enforce_eager=True,
|
||||||
enable_chunked_prefill=True,
|
enable_chunked_prefill=True,
|
||||||
|
|||||||
@@ -79,7 +79,10 @@ class SchedulerConfig:
|
|||||||
|
|
||||||
long_prefill_token_threshold: int = Field(default=0, ge=0)
|
long_prefill_token_threshold: int = Field(default=0, ge=0)
|
||||||
"""For chunked prefill, a request is considered long if the prompt is
|
"""For chunked prefill, a request is considered long if the prompt is
|
||||||
longer than this number of tokens. 0 disables the cap (default)."""
|
longer than this number of tokens. 0 disables the cap (default).
|
||||||
|
|
||||||
|
The cap is not applied when the request is the only one in the batch,
|
||||||
|
since there is no other request for it to starve."""
|
||||||
|
|
||||||
max_num_queued_reqs: int | None = Field(default=None, ge=0)
|
max_num_queued_reqs: int | None = Field(default=None, ge=0)
|
||||||
"""Maximum number of requests that can be in-flight (waiting or running)
|
"""Maximum number of requests that can be in-flight (waiting or running)
|
||||||
|
|||||||
@@ -599,6 +599,15 @@ class Scheduler(SchedulerInterface):
|
|||||||
throttle_prefills and not self.prefill_capacity_bound
|
throttle_prefills and not self.prefill_capacity_bound
|
||||||
) and any(not r.is_prefill_chunk for r in self.running)
|
) and any(not r.is_prefill_chunk for r in self.running)
|
||||||
|
|
||||||
|
# `long_prefill_token_threshold` exists to stop a long prefill from
|
||||||
|
# starving other requests of the token budget. When it is the only
|
||||||
|
# request there is nobody to starve, so let it use the whole budget.
|
||||||
|
long_prefill_token_threshold = (
|
||||||
|
self.scheduler_config.long_prefill_token_threshold
|
||||||
|
if len(self.running) + len(self.waiting) + len(self.skipped_waiting) > 1
|
||||||
|
else 0
|
||||||
|
)
|
||||||
|
|
||||||
# First, schedule the RUNNING requests.
|
# First, schedule the RUNNING requests.
|
||||||
req_index = 0
|
req_index = 0
|
||||||
while req_index < len(self.running) and token_budget > 0:
|
while req_index < len(self.running) and token_budget > 0:
|
||||||
@@ -650,8 +659,8 @@ class Scheduler(SchedulerInterface):
|
|||||||
+ request.num_output_placeholders
|
+ request.num_output_placeholders
|
||||||
- request.num_computed_tokens
|
- request.num_computed_tokens
|
||||||
)
|
)
|
||||||
if 0 < self.scheduler_config.long_prefill_token_threshold < num_new_tokens:
|
if 0 < long_prefill_token_threshold < num_new_tokens:
|
||||||
num_new_tokens = self.scheduler_config.long_prefill_token_threshold
|
num_new_tokens = long_prefill_token_threshold
|
||||||
num_new_tokens = min(
|
num_new_tokens = min(
|
||||||
num_new_tokens, token_budget, input_budget - draft_slots
|
num_new_tokens, token_budget, input_budget - draft_slots
|
||||||
)
|
)
|
||||||
@@ -1090,9 +1099,8 @@ class Scheduler(SchedulerInterface):
|
|||||||
num_new_tokens = padded_num_tokens
|
num_new_tokens = padded_num_tokens
|
||||||
pad_spec_decode = True
|
pad_spec_decode = True
|
||||||
|
|
||||||
threshold = self.scheduler_config.long_prefill_token_threshold
|
if 0 < long_prefill_token_threshold < num_new_tokens:
|
||||||
if 0 < threshold < num_new_tokens:
|
num_new_tokens = long_prefill_token_threshold
|
||||||
num_new_tokens = threshold
|
|
||||||
|
|
||||||
# chunked prefill has to be enabled explicitly to allow
|
# chunked prefill has to be enabled explicitly to allow
|
||||||
# pooling requests to be chunked
|
# pooling requests to be chunked
|
||||||
|
|||||||
Reference in New Issue
Block a user