From 6e919960af42f79d6811d84b2d4316212fcf59cb Mon Sep 17 00:00:00 2001 From: aman Date: Sat, 20 Jun 2026 23:36:57 +0100 Subject: [PATCH] [Perf] Skip/shrink all_token_ids copy in scheduler for non-async and V2 runner (#45840) Signed-off-by: amanchugh89 Signed-off-by: Nick Hill Co-authored-by: Claude Co-authored-by: Nick Hill --- tests/v1/core/test_scheduler.py | 37 +++++++++++++++++++++++++++++++++ vllm/v1/core/sched/output.py | 4 ++-- vllm/v1/core/sched/scheduler.py | 19 +++++++++-------- 3 files changed, 49 insertions(+), 11 deletions(-) diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index b2825c34df..6eb5ff5cc4 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -144,6 +144,43 @@ def test_async_scheduling_pp_allows_rescheduling_with_output_placeholders(): assert req.request_id in output.num_scheduled_tokens +def test_cached_request_data_resumed_all_token_ids_mrv1_only(): + """all_token_ids carries a resumed request's token ids to the connector + for the V1 model runner, but is skipped entirely for the V2 model runner. + """ + from vllm.v1.core.kv_cache_manager import KVCacheBlocks + + scheduler = create_scheduler() + (req,) = create_requests(num_requests=1, num_tokens=8) + req.append_output_token_ids([101, 102, 103]) + + # A resumed request was not scheduled in the previous step. + assert req.request_id not in scheduler.prev_step_scheduled_req_ids + + empty_blocks = KVCacheBlocks(blocks=((),)) + + def make_cached(): + return scheduler._make_cached_request_data( + running_reqs=[], + resumed_reqs=[req], + num_scheduled_tokens={req.request_id: 1}, + spec_decode_tokens={}, + req_to_new_blocks={req.request_id: empty_blocks}, + ) + + # V1 model runner: the full token id list is propagated. + assert not scheduler.use_v2_model_runner + cached = make_cached() + assert req.request_id in cached.resumed_req_ids + assert cached.all_token_ids[req.request_id] == list(req.all_token_ids) + + # V2 model runner: all_token_ids is skipped entirely. + scheduler.use_v2_model_runner = True + cached = make_cached() + assert req.request_id in cached.resumed_req_ids + assert cached.all_token_ids == {} + + def test_schedule_partial_requests(): """Test scheduling behavior with partial requests. diff --git a/vllm/v1/core/sched/output.py b/vllm/v1/core/sched/output.py index 0c1b9d34c5..291e73bc64 100644 --- a/vllm/v1/core/sched/output.py +++ b/vllm/v1/core/sched/output.py @@ -118,8 +118,8 @@ class CachedRequestData: # NOTE(woosuk): new_token_ids is only used for pipeline parallelism. # When PP is not used, new_token_ids will be empty. new_token_ids: list[list[int]] - # For requests not scheduled in the last step, propagate the token ids to the - # connector. Won't contain requests that were scheduled in the prior step. + # MRV1-only: For requests not scheduled in the last step, propagate the token ids + # to the connector. Won't contain requests scheduled in the prior step. all_token_ids: dict[str, list[int]] new_block_ids: list[tuple[list[int], ...] | None] num_computed_tokens: list[int] diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 25ccf79bc3..9f7c8d74de 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -101,6 +101,7 @@ class Scheduler(SchedulerInterface): self.finished_req_ids_dict: dict[int, set[str]] | None = ( defaultdict(set) if include_finished_set else None ) + # Track requests scheduled in prior step (MRV1-only). self.prev_step_scheduled_req_ids: set[str] = set() # Scheduling constraints. @@ -1010,8 +1011,8 @@ class Scheduler(SchedulerInterface): # Construct the scheduler output. if self.use_v2_model_runner: - scheduled_new_reqs = scheduled_new_reqs + scheduled_resumed_reqs - scheduled_resumed_reqs = [] + scheduled_new_reqs.extend(scheduled_resumed_reqs) + scheduled_resumed_reqs.clear() new_reqs_data = [ NewRequestData.from_request( req, @@ -1037,9 +1038,10 @@ class Scheduler(SchedulerInterface): req_to_new_blocks, ) - # Record the request ids that were scheduled in this step. - self.prev_step_scheduled_req_ids.clear() - self.prev_step_scheduled_req_ids.update(num_scheduled_tokens.keys()) + # Record the request ids that were scheduled in this step (MRV1-only). + if not self.use_v2_model_runner: + self.prev_step_scheduled_req_ids.clear() + self.prev_step_scheduled_req_ids.update(num_scheduled_tokens.keys()) new_block_ids_to_zero = ( (self.kv_cache_manager.take_new_block_ids() or None) @@ -1252,12 +1254,11 @@ class Scheduler(SchedulerInterface): req.num_computed_tokens : req.num_computed_tokens + num_tokens ] new_token_ids.append(token_ids) - scheduled_in_prev_step = req_id in self.prev_step_scheduled_req_ids if idx >= num_running_reqs: - assert not scheduled_in_prev_step resumed_req_ids.add(req_id) - if not scheduled_in_prev_step: - all_token_ids[req_id] = req.all_token_ids.copy() + if not self.use_v2_model_runner: # noqa: SIM102 + if req_id not in self.prev_step_scheduled_req_ids: + all_token_ids[req_id] = req.all_token_ids.copy() new_block_ids.append( req_to_new_blocks[req_id].get_block_ids(allow_none=True) )