[Scheduler] Soften Long Prefill Tokens Threshhold (#57951)
Signed-off-by: Robert Shaw <robertgshaw2@gmail.com> Signed-off-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Robert Shaw <robshaw@redhat.com>
This commit is contained in:
co-authored by
Robert Shaw
parent
78d7836525
commit
c64b15cde5
@@ -417,7 +417,7 @@ def test_abort_mid_prefill_defers_free():
|
||||
chunk is still in flight must withhold its blocks.
|
||||
"""
|
||||
scheduler = create_scheduler(
|
||||
model=MODEL, async_scheduling=True, long_prefill_token_threshold=16
|
||||
model=MODEL, async_scheduling=True, max_num_batched_tokens=16
|
||||
)
|
||||
scheduler.defer_block_free = True
|
||||
pool = scheduler.kv_cache_manager.block_pool
|
||||
|
||||
@@ -762,6 +762,36 @@ def test_schedule_concurrent_partial_requests(enable_prefix_caching: bool):
|
||||
assert output2.num_scheduled_tokens[requests[2].request_id] == 800 - 224 - 224
|
||||
|
||||
|
||||
def test_long_prefill_threshold_ignored_when_alone():
|
||||
"""A lone long prefill is not capped by the threshold: it has no other
|
||||
request to starve."""
|
||||
scheduler = create_scheduler(
|
||||
max_num_batched_tokens=1024,
|
||||
long_prefill_token_threshold=400,
|
||||
)
|
||||
request = create_requests(num_requests=1, num_tokens=2000)[0]
|
||||
scheduler.add_request(request)
|
||||
|
||||
output = scheduler.schedule()
|
||||
assert output.num_scheduled_tokens[request.request_id] == 1024
|
||||
|
||||
|
||||
def test_long_prefill_threshold_applies_with_other_requests():
|
||||
"""The threshold caps the prefill as soon as another request is queued."""
|
||||
scheduler = create_scheduler(
|
||||
max_num_batched_tokens=1024,
|
||||
long_prefill_token_threshold=400,
|
||||
)
|
||||
long_req = create_requests(num_requests=1, num_tokens=2000)[0]
|
||||
short_req = create_requests(num_requests=1, num_tokens=10, req_ids=["short"])[0]
|
||||
for request in [long_req, short_req]:
|
||||
scheduler.add_request(request)
|
||||
|
||||
output = scheduler.schedule()
|
||||
assert output.num_scheduled_tokens[long_req.request_id] == 400
|
||||
assert output.num_scheduled_tokens[short_req.request_id] == 10
|
||||
|
||||
|
||||
def test_update_from_output_routes_sampling_masks_by_request():
|
||||
"""Each request receives the sampler row at its own batch index."""
|
||||
scheduler = create_scheduler()
|
||||
|
||||
Reference in New Issue
Block a user