[Perf] Skip/shrink all_token_ids copy in scheduler for non-async and V2 runner (#45840)
Signed-off-by: amanchugh89 <amanchugh.89@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -144,6 +144,43 @@ def test_async_scheduling_pp_allows_rescheduling_with_output_placeholders():
|
||||
assert req.request_id in output.num_scheduled_tokens
|
||||
|
||||
|
||||
def test_cached_request_data_resumed_all_token_ids_mrv1_only():
|
||||
"""all_token_ids carries a resumed request's token ids to the connector
|
||||
for the V1 model runner, but is skipped entirely for the V2 model runner.
|
||||
"""
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
|
||||
scheduler = create_scheduler()
|
||||
(req,) = create_requests(num_requests=1, num_tokens=8)
|
||||
req.append_output_token_ids([101, 102, 103])
|
||||
|
||||
# A resumed request was not scheduled in the previous step.
|
||||
assert req.request_id not in scheduler.prev_step_scheduled_req_ids
|
||||
|
||||
empty_blocks = KVCacheBlocks(blocks=((),))
|
||||
|
||||
def make_cached():
|
||||
return scheduler._make_cached_request_data(
|
||||
running_reqs=[],
|
||||
resumed_reqs=[req],
|
||||
num_scheduled_tokens={req.request_id: 1},
|
||||
spec_decode_tokens={},
|
||||
req_to_new_blocks={req.request_id: empty_blocks},
|
||||
)
|
||||
|
||||
# V1 model runner: the full token id list is propagated.
|
||||
assert not scheduler.use_v2_model_runner
|
||||
cached = make_cached()
|
||||
assert req.request_id in cached.resumed_req_ids
|
||||
assert cached.all_token_ids[req.request_id] == list(req.all_token_ids)
|
||||
|
||||
# V2 model runner: all_token_ids is skipped entirely.
|
||||
scheduler.use_v2_model_runner = True
|
||||
cached = make_cached()
|
||||
assert req.request_id in cached.resumed_req_ids
|
||||
assert cached.all_token_ids == {}
|
||||
|
||||
|
||||
def test_schedule_partial_requests():
|
||||
"""Test scheduling behavior with partial requests.
|
||||
|
||||
|
||||
@@ -118,8 +118,8 @@ class CachedRequestData:
|
||||
# NOTE(woosuk): new_token_ids is only used for pipeline parallelism.
|
||||
# When PP is not used, new_token_ids will be empty.
|
||||
new_token_ids: list[list[int]]
|
||||
# For requests not scheduled in the last step, propagate the token ids to the
|
||||
# connector. Won't contain requests that were scheduled in the prior step.
|
||||
# MRV1-only: For requests not scheduled in the last step, propagate the token ids
|
||||
# to the connector. Won't contain requests scheduled in the prior step.
|
||||
all_token_ids: dict[str, list[int]]
|
||||
new_block_ids: list[tuple[list[int], ...] | None]
|
||||
num_computed_tokens: list[int]
|
||||
|
||||
@@ -101,6 +101,7 @@ class Scheduler(SchedulerInterface):
|
||||
self.finished_req_ids_dict: dict[int, set[str]] | None = (
|
||||
defaultdict(set) if include_finished_set else None
|
||||
)
|
||||
# Track requests scheduled in prior step (MRV1-only).
|
||||
self.prev_step_scheduled_req_ids: set[str] = set()
|
||||
|
||||
# Scheduling constraints.
|
||||
@@ -1010,8 +1011,8 @@ class Scheduler(SchedulerInterface):
|
||||
|
||||
# Construct the scheduler output.
|
||||
if self.use_v2_model_runner:
|
||||
scheduled_new_reqs = scheduled_new_reqs + scheduled_resumed_reqs
|
||||
scheduled_resumed_reqs = []
|
||||
scheduled_new_reqs.extend(scheduled_resumed_reqs)
|
||||
scheduled_resumed_reqs.clear()
|
||||
new_reqs_data = [
|
||||
NewRequestData.from_request(
|
||||
req,
|
||||
@@ -1037,9 +1038,10 @@ class Scheduler(SchedulerInterface):
|
||||
req_to_new_blocks,
|
||||
)
|
||||
|
||||
# Record the request ids that were scheduled in this step.
|
||||
self.prev_step_scheduled_req_ids.clear()
|
||||
self.prev_step_scheduled_req_ids.update(num_scheduled_tokens.keys())
|
||||
# Record the request ids that were scheduled in this step (MRV1-only).
|
||||
if not self.use_v2_model_runner:
|
||||
self.prev_step_scheduled_req_ids.clear()
|
||||
self.prev_step_scheduled_req_ids.update(num_scheduled_tokens.keys())
|
||||
|
||||
new_block_ids_to_zero = (
|
||||
(self.kv_cache_manager.take_new_block_ids() or None)
|
||||
@@ -1252,12 +1254,11 @@ class Scheduler(SchedulerInterface):
|
||||
req.num_computed_tokens : req.num_computed_tokens + num_tokens
|
||||
]
|
||||
new_token_ids.append(token_ids)
|
||||
scheduled_in_prev_step = req_id in self.prev_step_scheduled_req_ids
|
||||
if idx >= num_running_reqs:
|
||||
assert not scheduled_in_prev_step
|
||||
resumed_req_ids.add(req_id)
|
||||
if not scheduled_in_prev_step:
|
||||
all_token_ids[req_id] = req.all_token_ids.copy()
|
||||
if not self.use_v2_model_runner: # noqa: SIM102
|
||||
if req_id not in self.prev_step_scheduled_req_ids:
|
||||
all_token_ids[req_id] = req.all_token_ids.copy()
|
||||
new_block_ids.append(
|
||||
req_to_new_blocks[req_id].get_block_ids(allow_none=True)
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user