From 6d8fff5698aea7225792b37c3b075cc64c4a0178 Mon Sep 17 00:00:00 2001 From: Itay Etelis <92247226+Etelis@users.noreply.github.com> Date: Wed, 17 Jun 2026 11:35:07 +0300 Subject: [PATCH] [KV Connector][Offloading] Avoid blocking the engine to flush offloads on idle (#45595) Signed-off-by: Itay Etelis Signed-off-by: Or Ozeri Signed-off-by: Itay Etelis Co-authored-by: Itay Etelis Co-authored-by: Or Ozeri Co-authored-by: Itay Etelis --- .../offloading_connector/test_scheduler.py | 85 +++---------------- .../kv_connector/v1/offloading/scheduler.py | 16 ++-- .../kv_connector/v1/offloading_connector.py | 4 + vllm/v1/kv_offload/base.py | 8 ++ vllm/v1/kv_offload/tiering/base.py | 8 ++ vllm/v1/kv_offload/tiering/manager.py | 9 ++ 6 files changed, 49 insertions(+), 81 deletions(-) diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py index 8bd46184d6..973fcc63e3 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py @@ -262,12 +262,9 @@ def test_concurrent_lookups_of_the_same_prefix(request_runner, async_scheduling: runner.manager.prepare_store.side_effect = lambda keys, req_context: ( generate_store_output(keys) ) - # With sync scheduling, all-finished flush fires within this run. - # With async scheduling, the finish is delayed so flush fires later. runner.run( decoded_tokens=[EOS_TOKEN_ID], expected_stored=(0, 1, 2), - expected_flushed=(0, 1, 2) if not async_scheduling else (), ) # start a request to load the first block, but don't complete @@ -332,7 +329,6 @@ def test_abort_loading_requests(request_runner, async_scheduling: bool): runner.run( decoded_tokens=[EOS_TOKEN_ID], expected_stored=(0, 1, 2), - expected_flushed=(0, 1, 2) if not async_scheduling else (), ) # start a request to load the first block, but don't complete @@ -359,7 +355,6 @@ def test_abort_loading_requests(request_runner, async_scheduling: bool): runner.run( decoded_tokens=[], expected_loaded=(0, 1, 2), - expected_flushed=(0, 1, 2), ) # assert request is deleted @@ -774,7 +769,6 @@ def test_request_level_policy_stores_all_blocks(request_runner, async_scheduling runner.run( decoded_tokens=[EOS_TOKEN_ID], expected_stored=(0, 1, 2), - expected_flushed=(0, 1, 2) if not async_scheduling else (), ) # Reset GPU prefix cache so the next request must load from CPU. @@ -841,13 +835,8 @@ def test_fence_at_update_state_after_alloc(request_runner): runner.manager.prepare_store.side_effect = lambda keys, req_context: ( generate_store_output(keys) ) - runner.run( - decoded_tokens=[EOS_TOKEN_ID], - complete_transfers=False, - expected_stored=(0,), - expected_flushed=(0,), - ) - assert runner.connector_scheduler._block_id_to_pending_jobs == {} + runner.run(decoded_tokens=[EOS_TOKEN_ID], complete_transfers=False) + assert runner.connector_scheduler._block_id_to_pending_jobs runner.scheduler.reset_prefix_cache() runner.new_request(token_ids=[0] * 4) @@ -858,6 +847,8 @@ def test_fence_at_update_state_after_alloc(request_runner): runner.run( decoded_tokens=[], complete_transfers=False, + expected_stored=(0,), + expected_flushed=(0,), ) assert runner.connector_scheduler._block_id_to_pending_jobs == {} @@ -877,13 +868,8 @@ def test_fence_at_build_store_jobs(request_runner): runner.manager.prepare_store.side_effect = lambda keys, req_context: ( generate_store_output(keys) ) - runner.run( - decoded_tokens=[EOS_TOKEN_ID], - complete_transfers=False, - expected_stored=(0,), - expected_flushed=(0,), - ) - assert runner.connector_scheduler._block_id_to_pending_jobs == {} + runner.run(decoded_tokens=[EOS_TOKEN_ID], complete_transfers=False) + assert runner.connector_scheduler._block_id_to_pending_jobs runner.scheduler.reset_prefix_cache() runner.new_request(token_ids=[1] * 4) @@ -893,6 +879,8 @@ def test_fence_at_build_store_jobs(request_runner): ) runner.run( decoded_tokens=[EOS_TOKEN_ID], + expected_stored=(0,), + expected_flushed=(0,), ) assert runner.connector_scheduler._block_id_to_pending_jobs == {} @@ -966,10 +954,10 @@ def test_max_offload_tokens_validation(request_runner, async_scheduling: bool): lambda keys, req_context: generate_store_output(keys) ) - # With sync scheduling, the connector flushes completed stores when the - # request finishes; async scheduling defers the flush to the next step. - flushed_all = all_offsets if not async_scheduling else () - flushed_two = (0, 1, 2, 3, 4, 5) if not async_scheduling else () + # Pending offloads drain via non-blocking stepping, not a flush, so no + # blocks are flushed when the request finishes. + flushed_all: tuple[int, ...] = () + flushed_two: tuple[int, ...] = () # None -> no cap, all 9 offsets stored r = make_runner() @@ -1081,32 +1069,6 @@ def test_offload_prompt_only(request_runner, async_scheduling: bool): assert len(offered_keys) == num_prompt_blocks -def test_flush_all_jobs_when_no_requests_remain(request_runner): - """When all tracked requests are finished, build_connector_meta flushes - all pending jobs since there will be no future step to complete them.""" - block_size = 4 - block_size_factor = 1 - offloaded_block_size = block_size * block_size_factor - - runner = request_runner( - block_size=block_size, - num_gpu_blocks=100, - async_scheduling=False, - block_size_factor=block_size_factor, - ) - - runner.new_request(token_ids=[0] * offloaded_block_size) - runner.manager.prepare_store.side_effect = lambda keys, req_context: ( - generate_store_output(keys) - ) - runner.run( - decoded_tokens=[EOS_TOKEN_ID], - complete_transfers=False, - expected_stored=(0,), - expected_flushed=(0,), - ) - - @pytest.mark.parametrize("async_scheduling", [True, False]) def test_reset_cache(request_runner, async_scheduling: bool): """reset_cache flushes in-flight loads, calls manager.reset_cache(), resets @@ -1131,7 +1093,6 @@ def test_reset_cache(request_runner, async_scheduling: bool): runner.run( decoded_tokens=[EOS_TOKEN_ID], expected_stored=(0, 1, 2), - expected_flushed=(0, 1, 2) if not async_scheduling else (), ) # Reset GPU prefix cache then start a request that loads from CPU. @@ -1381,7 +1342,6 @@ def test_stale_sliding_window_block_after_prepare_store_failure( runner.run( decoded_tokens=[EOS_TOKEN_ID], expected_stored=(2, 3), - expected_flushed=(2, 3) if not async_scheduling else (), ) @@ -1409,7 +1369,6 @@ def test_skip_reading_prefix_cache(request_runner, async_scheduling: bool): runner.run( decoded_tokens=[EOS_TOKEN_ID], expected_stored=(0, 1, 2), - expected_flushed=(0, 1, 2) if not async_scheduling else (), ) # Reset GPU prefix cache so the next request cannot hit locally. @@ -1429,7 +1388,6 @@ def test_skip_reading_prefix_cache(request_runner, async_scheduling: bool): decoded_tokens=[EOS_TOKEN_ID], expected_loaded=(), # no CPU loads must happen expected_stored=(0, 1, 2), # tokens still offloaded to CPU - expected_flushed=(0, 1, 2) if not async_scheduling else (), ) # The external lookup must have been completely skipped. @@ -1936,15 +1894,6 @@ class TestEagle: (1, 0), (1, 1), ), - expected_flushed=( - (0, 0), - (0, 1), - (0, 2), - (1, 0), - (1, 1), - ) - if not async_scheduling - else (), ) @pytest.mark.parametrize("async_scheduling", [True, False]) @@ -1990,7 +1939,6 @@ class TestEagle: runner.run( decoded_tokens=[EOS_TOKEN_ID], expected_stored=((0, 0), (0, 1)), - expected_flushed=((0, 0), (0, 1)) if not async_scheduling else (), ) @pytest.mark.parametrize("async_scheduling", [True, False]) @@ -2087,15 +2035,6 @@ class TestEagle: (1, 0), (1, 1), ), - expected_flushed=( - (0, 0), - (0, 1), - (0, 2), - (1, 0), - (1, 1), - ) - if not async_scheduling - else (), ) runner.scheduler.reset_prefix_cache() diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py index 443d5b28d5..9c3cb7e5a5 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py @@ -1016,14 +1016,6 @@ class OffloadingConnectorScheduler: for jid in self._block_id_to_pending_jobs[bid] ) - # If all tracked requests are finished, flush all pending jobs - # (both store and load) - there might not be a future scheduler - # step to trigger their completion. - if self._req_status and all( - rs.req.is_finished() for rs in self._req_status.values() - ): - self._current_batch_jobs_to_flush.update(self._jobs.keys()) - meta = OffloadingConnectorMetadata( load_jobs=self._current_batch_load_jobs, store_jobs=self._build_store_jobs(scheduler_output), @@ -1034,6 +1026,14 @@ class OffloadingConnectorScheduler: self._current_batch_allocated_block_ids = set() return meta + def has_pending_push_work(self) -> bool: + """Whether the engine must keep stepping. + + While True, build_connector_meta() and update_connector_output() + continue to be called even when no requests are scheduled. + """ + return bool(self._jobs) or self.manager.has_pending_work() + def update_connector_output(self, connector_output: KVConnectorOutput): """ Update KVConnector state from worker-side connectors output. diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py index 7d567ed462..1c5986d515 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py @@ -150,6 +150,10 @@ class OffloadingConnector(KVConnectorBase_V1, SupportsHMA): assert self.connector_scheduler is not None return self.connector_scheduler.build_connector_meta(scheduler_output) + def has_pending_push_work(self) -> bool: + assert self.connector_scheduler is not None + return self.connector_scheduler.has_pending_push_work() + def update_connector_output(self, connector_output: KVConnectorOutput): assert self.connector_scheduler is not None self.connector_scheduler.update_connector_output(connector_output) diff --git a/vllm/v1/kv_offload/base.py b/vllm/v1/kv_offload/base.py index 16f783190f..15781bbc8a 100644 --- a/vllm/v1/kv_offload/base.py +++ b/vllm/v1/kv_offload/base.py @@ -288,6 +288,14 @@ class OffloadingManager(ABC): """ return + def has_pending_work(self) -> bool: + """Whether this manager needs the engine to keep stepping. + + While True, on_schedule_end() and get_finished_jobs() continue + to be called even when no requests are scheduled. + """ + return False + def reset_cache(self) -> None: """Evict all tracked blocks and reset internal state.""" return diff --git a/vllm/v1/kv_offload/tiering/base.py b/vllm/v1/kv_offload/tiering/base.py index dd9178fc7c..f9fbdf9495 100644 --- a/vllm/v1/kv_offload/tiering/base.py +++ b/vllm/v1/kv_offload/tiering/base.py @@ -153,6 +153,14 @@ class SecondaryTierManager(ABC): """ pass + def has_pending_work(self) -> bool: + """Whether this tier needs the engine to keep stepping. + + While True, on_schedule_end() and get_finished_jobs() continue + to be called even when no requests are scheduled. + """ + return False + def touch(self, keys: Collection[OffloadKey], req_context: ReqContext): """ Mark blocks as recently used for eviction policy. diff --git a/vllm/v1/kv_offload/tiering/manager.py b/vllm/v1/kv_offload/tiering/manager.py index fbcccea162..d13e1f1eea 100644 --- a/vllm/v1/kv_offload/tiering/manager.py +++ b/vllm/v1/kv_offload/tiering/manager.py @@ -577,6 +577,15 @@ class TieringOffloadingManager(OffloadingManager): for tier in self.secondary_tiers: tier.on_schedule_end() + @override + def has_pending_work(self) -> bool: + # In-flight primary<->secondary transfers (pending promotions are + # translated to transfer jobs in on_schedule_end), plus any work the + # secondary tiers themselves still have outstanding. + return bool(self._transfer_jobs) or any( + tier.has_pending_work() for tier in self.secondary_tiers + ) + @override def take_events(self) -> Iterable[OffloadingEvent]: """Yield offloading events collected since the last call.