From 82536acc54fc8d828b0d8add9d093aee0da1370e Mon Sep 17 00:00:00 2001 From: Gabriel Wu <13583761+lucifer1004@users.noreply.github.com> Date: Sat, 23 May 2026 14:23:32 +0800 Subject: [PATCH] Keep scheduler alive for delayed KV connector frees (#43433) Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com> --- tests/v1/core/test_scheduler.py | 39 +++++++++++++++++++ .../kv_connector/unit/test_multi_connector.py | 22 ++++++++--- vllm/v1/core/sched/scheduler.py | 11 +++++- vllm/v1/engine/core.py | 9 ++--- 4 files changed, 69 insertions(+), 12 deletions(-) diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index 1c5e1ac4ba..bcbac67a63 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -3928,6 +3928,45 @@ def test_abort_request_finished_recving(): assert not scheduler.finished_recving_kv_req_ids +def test_delayed_kv_connector_free_keeps_scheduler_active(): + scheduler = create_scheduler(use_kv_connector=True) + queued_request, request = create_requests( + num_requests=2, req_ids=["queued", "finished"] + ) + scheduler.add_request(queued_request) + + assert not scheduler.has_finished_requests() + + request.status = RequestStatus.FINISHED_STOPPED + scheduler.requests[request.request_id] = request + scheduler.finished_req_ids = set() + + assert scheduler.has_finished_requests() + assert scheduler.has_requests() + + scheduler_output = SchedulerOutput( + scheduled_new_reqs=[], + scheduled_cached_reqs=CachedRequestData.make_empty(), + num_scheduled_tokens={}, + total_num_scheduled_tokens=0, + scheduled_encoder_inputs={}, + scheduled_spec_decode_tokens={}, + num_common_prefix_blocks=[], + finished_req_ids=set(), + free_encoder_mm_hashes=[], + ) + model_runner_output = ModelRunnerOutput( + req_ids=[], + req_id_to_index={}, + kv_connector_output=KVConnectorOutput(finished_sending={request.request_id}), + ) + + scheduler.update_from_output(scheduler_output, model_runner_output) + + assert request.request_id not in scheduler.requests + assert not scheduler.has_finished_requests() + + # ============================================================================== # Variable-length encoder cross-attention block allocation tests # ============================================================================== diff --git a/tests/v1/kv_connector/unit/test_multi_connector.py b/tests/v1/kv_connector/unit/test_multi_connector.py index f2d9a0be43..3c1f7cfa9a 100644 --- a/tests/v1/kv_connector/unit/test_multi_connector.py +++ b/tests/v1/kv_connector/unit/test_multi_connector.py @@ -258,10 +258,12 @@ def test_multi_example_connector_consistency(): ) events = get_connector_events() + storage1_scheduler_events = _ignore_event_collection(events["storage1-SCHEDULER"]) + storage2_scheduler_events = _ignore_event_collection(events["storage2-SCHEDULER"]) # First event is bind_gpu_block_pool from initialization, then # set_xfer_handshake_metadata, then on_new_request when the request is enqueued, # then get_num_new_matched_tokens and update_state_after_alloc from generate(). - assert events["storage1-SCHEDULER"][:6] == [ + assert storage1_scheduler_events[:6] == [ "bind_gpu_block_pool", "set_xfer_handshake_metadata", "on_new_request", @@ -281,7 +283,7 @@ def test_multi_example_connector_consistency(): "wait_for_layer_load", "save_kv_layer", ] - assert events["storage2-SCHEDULER"][:6] == [ + assert storage2_scheduler_events[:6] == [ "bind_gpu_block_pool", "set_xfer_handshake_metadata", "on_new_request", @@ -312,13 +314,15 @@ def test_multi_example_connector_consistency(): # connector so update_state_after_alloc will be with allocated blocks # on that one but with zero blocks for others (first nonzero match is # chosen). - assert events["storage1-SCHEDULER"][:4] == [ + storage1_scheduler_events = _ignore_event_collection(events["storage1-SCHEDULER"]) + storage2_scheduler_events = _ignore_event_collection(events["storage2-SCHEDULER"]) + assert storage1_scheduler_events[:4] == [ "on_new_request", "get_num_new_matched_tokens 0", "update_state_after_alloc num_blocks=[7] 96", "build_connector_meta", ] - assert events["storage2-SCHEDULER"][:4] == [ + assert storage2_scheduler_events[:4] == [ "on_new_request", "get_num_new_matched_tokens 0", "update_state_after_alloc num_blocks=[0] 0", @@ -340,13 +344,15 @@ def test_multi_example_connector_consistency(): # return 0 from the first connector, but the second connector should have # a hit, so update_state_after_alloc will only be called with allocated # blocks for the second connector. - assert events["storage1-SCHEDULER"][:4] == [ + storage1_scheduler_events = _ignore_event_collection(events["storage1-SCHEDULER"]) + storage2_scheduler_events = _ignore_event_collection(events["storage2-SCHEDULER"]) + assert storage1_scheduler_events[:4] == [ "on_new_request", "get_num_new_matched_tokens 0", "update_state_after_alloc num_blocks=[0] 0", "build_connector_meta", ] - assert events["storage2-SCHEDULER"][:4] == [ + assert storage2_scheduler_events[:4] == [ "on_new_request", "get_num_new_matched_tokens 0", "update_state_after_alloc num_blocks=[7] 96", @@ -358,6 +364,10 @@ def test_multi_example_connector_consistency(): shutil.rmtree(storage_2_path) +def _ignore_event_collection(events: list[str]) -> list[str]: + return [event for event in events if event != "take_events"] + + def get_connector_events() -> dict[str, list[str]]: # Read in connector events and reset the files. import glob diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 45a5169a3e..c69c9a8119 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -1882,7 +1882,16 @@ class Scheduler(SchedulerInterface): return num_waiting + len(self.running) def has_finished_requests(self) -> bool: - return len(self.finished_req_ids) > 0 + if self.finished_req_ids: + return True + if self.connector is None: + return False + # Finished requests waiting on delayed connector cleanup remain in + # self.requests after they have been removed from scheduling queues. + num_in_queues = ( + len(self.waiting) + len(self.skipped_waiting) + len(self.running) + ) + return len(self.requests) > num_in_queues def reset_prefix_cache( self, reset_running_requests: bool = False, reset_connector: bool = False diff --git a/vllm/v1/engine/core.py b/vllm/v1/engine/core.py index 8c5fe71ed2..c9503e46dd 100644 --- a/vllm/v1/engine/core.py +++ b/vllm/v1/engine/core.py @@ -1242,11 +1242,10 @@ class EngineCoreProc(EngineCore): # Post-step hook. self.post_step(model_executed) - # If no model execution happened but there are waiting requests - # (e.g., WAITING_FOR_REMOTE_KVS), yield the GIL briefly to allow - # background threads (like NIXL handshake) to make progress. - # Without this, the tight polling loop can starve background threads. - if not model_executed and self.scheduler.has_unfinished_requests(): + # If no model execution happened but there is still scheduler work + # (e.g. WAITING_FOR_REMOTE_KVS or delayed KV connector frees), yield + # the GIL briefly to allow background transfer threads to make progress. + if not model_executed and self.scheduler.has_requests(): time.sleep(0.001) return model_executed