[Scheduler] Soften Long Prefill Tokens Threshhold (#57951)
Signed-off-by: Robert Shaw <robertgshaw2@gmail.com> Signed-off-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Robert Shaw <robshaw@redhat.com>
This commit is contained in:
co-authored by
Robert Shaw
parent
78d7836525
commit
c64b15cde5
@@ -417,7 +417,7 @@ def test_abort_mid_prefill_defers_free():
|
||||
chunk is still in flight must withhold its blocks.
|
||||
"""
|
||||
scheduler = create_scheduler(
|
||||
model=MODEL, async_scheduling=True, long_prefill_token_threshold=16
|
||||
model=MODEL, async_scheduling=True, max_num_batched_tokens=16
|
||||
)
|
||||
scheduler.defer_block_free = True
|
||||
pool = scheduler.kv_cache_manager.block_pool
|
||||
|
||||
@@ -762,6 +762,36 @@ def test_schedule_concurrent_partial_requests(enable_prefix_caching: bool):
|
||||
assert output2.num_scheduled_tokens[requests[2].request_id] == 800 - 224 - 224
|
||||
|
||||
|
||||
def test_long_prefill_threshold_ignored_when_alone():
|
||||
"""A lone long prefill is not capped by the threshold: it has no other
|
||||
request to starve."""
|
||||
scheduler = create_scheduler(
|
||||
max_num_batched_tokens=1024,
|
||||
long_prefill_token_threshold=400,
|
||||
)
|
||||
request = create_requests(num_requests=1, num_tokens=2000)[0]
|
||||
scheduler.add_request(request)
|
||||
|
||||
output = scheduler.schedule()
|
||||
assert output.num_scheduled_tokens[request.request_id] == 1024
|
||||
|
||||
|
||||
def test_long_prefill_threshold_applies_with_other_requests():
|
||||
"""The threshold caps the prefill as soon as another request is queued."""
|
||||
scheduler = create_scheduler(
|
||||
max_num_batched_tokens=1024,
|
||||
long_prefill_token_threshold=400,
|
||||
)
|
||||
long_req = create_requests(num_requests=1, num_tokens=2000)[0]
|
||||
short_req = create_requests(num_requests=1, num_tokens=10, req_ids=["short"])[0]
|
||||
for request in [long_req, short_req]:
|
||||
scheduler.add_request(request)
|
||||
|
||||
output = scheduler.schedule()
|
||||
assert output.num_scheduled_tokens[long_req.request_id] == 400
|
||||
assert output.num_scheduled_tokens[short_req.request_id] == 10
|
||||
|
||||
|
||||
def test_update_from_output_routes_sampling_masks_by_request():
|
||||
"""Each request receives the sampler row at its own batch index."""
|
||||
scheduler = create_scheduler()
|
||||
|
||||
@@ -77,13 +77,13 @@ def test_pooling_chunked_prefill(vllm_runner, monkeypatch):
|
||||
|
||||
chunk_size = 10
|
||||
|
||||
# Set chunking parameters to force chunked prefill
|
||||
# Note: Chunked prefill is automatically handled by vLLM
|
||||
# internally based on the model size and prompt
|
||||
# Cap the token budget to force chunked prefill; the
|
||||
# long_prefill_token_threshold is not applied to a lone request.
|
||||
with vllm_runner(
|
||||
model_id,
|
||||
runner="pooling",
|
||||
long_prefill_token_threshold=chunk_size,
|
||||
max_num_batched_tokens=chunk_size,
|
||||
max_num_seqs=1,
|
||||
tensor_parallel_size=1,
|
||||
enforce_eager=True,
|
||||
enable_chunked_prefill=True,
|
||||
|
||||
Reference in New Issue
Block a user