[Perf] Skip/shrink all_token_ids copy in scheduler for non-async and V2 runner (#45840)
Signed-off-by: amanchugh89 <amanchugh.89@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -144,6 +144,43 @@ def test_async_scheduling_pp_allows_rescheduling_with_output_placeholders():
|
||||
assert req.request_id in output.num_scheduled_tokens
|
||||
|
||||
|
||||
def test_cached_request_data_resumed_all_token_ids_mrv1_only():
|
||||
"""all_token_ids carries a resumed request's token ids to the connector
|
||||
for the V1 model runner, but is skipped entirely for the V2 model runner.
|
||||
"""
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
|
||||
scheduler = create_scheduler()
|
||||
(req,) = create_requests(num_requests=1, num_tokens=8)
|
||||
req.append_output_token_ids([101, 102, 103])
|
||||
|
||||
# A resumed request was not scheduled in the previous step.
|
||||
assert req.request_id not in scheduler.prev_step_scheduled_req_ids
|
||||
|
||||
empty_blocks = KVCacheBlocks(blocks=((),))
|
||||
|
||||
def make_cached():
|
||||
return scheduler._make_cached_request_data(
|
||||
running_reqs=[],
|
||||
resumed_reqs=[req],
|
||||
num_scheduled_tokens={req.request_id: 1},
|
||||
spec_decode_tokens={},
|
||||
req_to_new_blocks={req.request_id: empty_blocks},
|
||||
)
|
||||
|
||||
# V1 model runner: the full token id list is propagated.
|
||||
assert not scheduler.use_v2_model_runner
|
||||
cached = make_cached()
|
||||
assert req.request_id in cached.resumed_req_ids
|
||||
assert cached.all_token_ids[req.request_id] == list(req.all_token_ids)
|
||||
|
||||
# V2 model runner: all_token_ids is skipped entirely.
|
||||
scheduler.use_v2_model_runner = True
|
||||
cached = make_cached()
|
||||
assert req.request_id in cached.resumed_req_ids
|
||||
assert cached.all_token_ids == {}
|
||||
|
||||
|
||||
def test_schedule_partial_requests():
|
||||
"""Test scheduling behavior with partial requests.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user