diff --git a/tests/v1/simple_kv_offload/test_scheduler.py b/tests/v1/simple_kv_offload/test_scheduler.py index e59905f504..cff60ea01d 100644 --- a/tests/v1/simple_kv_offload/test_scheduler.py +++ b/tests/v1/simple_kv_offload/test_scheduler.py @@ -1354,3 +1354,177 @@ def test_toctou_cpu_hit_evicted_between_phases_no_crash() -> None: ) assert len(meta_b.load_gpu_blocks) == 2 assert len(meta_b.load_cpu_blocks) == 2 + + +# --------------------------------------------------------------------------- +# Test 12: Reset with pending eager stores waits for completion +# --------------------------------------------------------------------------- +def test_reset_pending_eager_stores() -> None: + """Eager mode: reset() abandons in-flight stores until they complete.""" + fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False) + sched = fix.scheduler + gpu_pool = fix.gpu_block_pool + + num_blocks = 2 + req = make_request(num_blocks=num_blocks) + + kv_blocks = _alloc_and_register(fix, req, num_blocks) + sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) + block_ids = kv_blocks.get_block_ids() + sched_out = make_scheduler_output( + {req.request_id: num_blocks * BLOCK_SIZE}, + new_reqs={req.request_id: block_ids}, + ) + + meta = sched.build_connector_meta(sched_out) + assert meta.store_event >= 0 + assert len(sched._store_event_to_blocks) > 0 + + # GPU blocks should have elevated ref_cnt from touch() + for bid in meta.store_gpu_blocks: + assert gpu_pool.blocks[bid].ref_cnt > 0 + + # Free the request's own block refs (simulates preemption) + gpu_pool.free_blocks(gpu_pool.blocks[bid] for bid in block_ids[0]) + + # Reset should keep DMA refs pinned until the worker reports completion. + assert sched.reset() is False + assert len(sched._store_event_to_blocks) == 0 + assert len(sched._abandoned_store_event_to_blocks) == 1 + assert len(sched._reqs_to_store) == 0 + assert len(sched._store_event_to_reqs) == 0 + + num_used = gpu_pool.num_gpu_blocks - gpu_pool.get_num_free_blocks() + assert num_used > 1 + + simulate_store_completion(sched, meta.store_event) + assert len(sched._abandoned_store_event_to_blocks) == 0 + + # All GPU blocks should now be free (ref_cnt == 0) except null block + num_used = gpu_pool.num_gpu_blocks - gpu_pool.get_num_free_blocks() + assert num_used == 1, f"Expected only null block in use, got {num_used}" + + # GPU prefix cache reset should now succeed + assert gpu_pool.reset_prefix_cache() is True + assert sched.reset() is True + + +# --------------------------------------------------------------------------- +# Test 13: Reset with pending lazy stores waits for completion +# --------------------------------------------------------------------------- +def test_reset_pending_lazy_stores() -> None: + """Lazy mode: reset() abandons in-flight stores until they complete.""" + fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=8, lazy=True) + sched = fix.scheduler + gpu_pool = fix.gpu_block_pool + + num_blocks = 2 + req = make_request(num_blocks=num_blocks) + + # Allocate, hash, and free — blocks move to free queue with hashes + gpu_blocks = _allocate_gpu_blocks(gpu_pool, req, num_blocks, group_id=0) + gpu_pool.free_blocks(gpu_blocks) + + # Push hashed blocks to LRU head + fillers = _flush_old_blocks_to_lru_head(gpu_pool, num_filler_blocks=5) + + # Lazy scanner offloads old hashed blocks + sched_out = make_scheduler_output({}) + meta = sched.build_connector_meta(sched_out) + assert meta.store_event >= 0 + assert len(sched._store_event_to_blocks) > 0 + + gpu_pool.free_blocks(fillers) + + # Reset should keep DMA refs pinned until the worker reports completion. + assert sched.reset() is False + assert len(sched._store_event_to_blocks) == 0 + assert len(sched._abandoned_store_event_to_blocks) == 1 + assert sched._cursor is None + + simulate_store_completion(sched, meta.store_event) + assert len(sched._abandoned_store_event_to_blocks) == 0 + assert sched.reset() is True + + # No CPU cache hits after reset + req2 = Request( + request_id="req-after-lazy-reset", + prompt_token_ids=req.prompt_token_ids, + sampling_params=req.sampling_params, + pooling_params=None, + mm_features=None, + block_hasher=req._block_hasher, + ) + hit_tokens, _ = sched.get_num_new_matched_tokens(req2, num_computed_tokens=0) + assert hit_tokens == 0, "CPU cache should be empty after reset" + + +# --------------------------------------------------------------------------- +# Test 14: Reset with pending loads waits for completion +# --------------------------------------------------------------------------- +def test_reset_pending_loads() -> None: + """reset() abandons in-flight loads until they complete.""" + fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False) + sched = fix.scheduler + gpu_pool = fix.gpu_block_pool + + num_blocks = 2 + + # First store blocks to CPU + req = make_request(num_blocks=num_blocks) + kv_blocks = _alloc_and_register(fix, req, num_blocks) + sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) + block_ids = kv_blocks.get_block_ids() + sched_out = make_scheduler_output( + {req.request_id: num_blocks * BLOCK_SIZE}, + new_reqs={req.request_id: block_ids}, + ) + meta = sched.build_connector_meta(sched_out) + simulate_store_completion(sched, meta.store_event) + + # Start a load — CPU cache hit + req2 = Request( + request_id="req-load-reset", + prompt_token_ids=req.prompt_token_ids, + sampling_params=req.sampling_params, + pooling_params=None, + mm_features=None, + block_hasher=req._block_hasher, + ) + hit_tokens, is_async = sched.get_num_new_matched_tokens(req2, num_computed_tokens=0) + assert hit_tokens > 0 + + gpu_blocks2 = gpu_pool.get_new_blocks(num_blocks) + kv_blocks2 = KVCacheBlocks(blocks=(gpu_blocks2,)) + sched.update_state_after_alloc(req2, kv_blocks2, num_external_tokens=hit_tokens) + + block_ids2 = kv_blocks2.get_block_ids() + sched_out2 = make_scheduler_output( + {req2.request_id: 1}, + new_reqs={req2.request_id: block_ids2}, + ) + meta2 = sched.build_connector_meta(sched_out2) + assert meta2.load_event >= 0 + assert req2.request_id in sched._reqs_to_load + + # Free request block refs (simulates preemption) + gpu_pool.free_blocks(gpu_pool.blocks[bid] for bid in block_ids[0]) + gpu_pool.free_blocks(gpu_pool.blocks[bid] for bid in block_ids2[0]) + + # Reset should keep load touch refs until the worker reports completion. + assert sched.reset() is False + assert len(sched._reqs_to_load) == 0 + assert len(sched._abandoned_reqs_to_load) == 1 + assert len(sched._load_event_to_reqs) == 1 + + num_used = gpu_pool.num_gpu_blocks - gpu_pool.get_num_free_blocks() + assert num_used > 1 + + simulate_load_completion(sched, {req2.request_id}) + assert len(sched._abandoned_reqs_to_load) == 0 + assert len(sched._load_event_to_reqs) == 0 + assert sched.reset() is True + + # All GPU blocks free + num_used = gpu_pool.num_gpu_blocks - gpu_pool.get_num_free_blocks() + assert num_used == 1, f"Expected only null block in use, got {num_used}" diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py index 15904da9e5..f1dac13ca5 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py @@ -245,10 +245,10 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA): return self.scheduler_manager.take_events() return [] + # NOTE: Workers are not contacted. In-flight transfers drain naturally, + # and stale completions are ignored by the guarded + # SimpleCPUOffloadScheduler._process_store_event(). def reset_cache(self) -> bool | None: - raise NotImplementedError( - "SimpleCPUOffloadConnector does not support reset_cache(). " - "reset_prefix_cache() requires synchronizing all pending " - "CPU offload transfers before clearing GPU prefix cache blocks, " - "which is not yet implemented." - ) + if self.scheduler_manager is not None: + return self.scheduler_manager.reset() + return None diff --git a/vllm/v1/simple_kv_offload/manager.py b/vllm/v1/simple_kv_offload/manager.py index f61c4320df..fe984be96a 100644 --- a/vllm/v1/simple_kv_offload/manager.py +++ b/vllm/v1/simple_kv_offload/manager.py @@ -159,10 +159,12 @@ class SimpleCPUOffloadScheduler: else: self._target_free = 0 self._store_event_to_blocks: dict[int, TransferMeta] = {} + self._abandoned_store_event_to_blocks: dict[int, TransferMeta] = {} # Eager mode only self._reqs_to_store: dict[str, StoreRequestState] = {} self._store_event_to_reqs: dict[int, list[str]] = {} self._in_flight_store_gpu_blocks: set[int] = set() + self._abandoned_reqs_to_load: dict[str, LoadRequestState] = {} # Event counters self._load_event_counter: int = 0 @@ -427,7 +429,10 @@ class SimpleCPUOffloadScheduler: load_event=load_event, load_gpu_blocks=load_gpu, load_cpu_blocks=load_cpu, - load_event_to_reqs=self._load_event_to_reqs, + load_event_to_reqs={ + event_idx: list(req_ids) + for event_idx, req_ids in self._load_event_to_reqs.items() + }, store_event=store_event, store_gpu_blocks=store_gpu, store_cpu_blocks=store_cpu, @@ -680,9 +685,17 @@ class SimpleCPUOffloadScheduler: def _process_store_event(self, event_idx: int) -> None: """Process a fully-completed store event.""" - transfer = self._store_event_to_blocks.pop(event_idx) + transfer = self._store_event_to_blocks.pop(event_idx, None) + if transfer is None: + transfer = self._abandoned_store_event_to_blocks.pop(event_idx, None) + if transfer is None: + return # guard stale events from before a reset() call + self._release_transfer_refs(transfer) + return + if not self._lazy_mode: self._in_flight_store_gpu_blocks.difference_update(transfer.gpu_block_ids) + self._process_store_completion(transfer.gpu_block_ids, transfer.cpu_block_ids) logger.debug( "Store event %d completed: cached %d blocks to CPU", @@ -725,9 +738,22 @@ class SimpleCPUOffloadScheduler: self._gpu_block_pool.blocks[bid] for bid in gpu_block_ids ) + def _release_transfer_refs(self, transfer: TransferMeta) -> None: + """Release transfer refs without making copied data cacheable.""" + cpu_blocks = [self.cpu_block_pool.blocks[bid] for bid in transfer.cpu_block_ids] + for cpu_block in cpu_blocks: + cpu_block.reset_hash() + self.cpu_block_pool.free_blocks(cpu_blocks) + assert self._gpu_block_pool is not None + self._gpu_block_pool.free_blocks( + self._gpu_block_pool.blocks[bid] for bid in transfer.gpu_block_ids + ) + def has_pending_stores(self) -> bool: """Return True if there are in-flight store transfers.""" - return bool(self._store_event_to_blocks) + return bool( + self._store_event_to_blocks or self._abandoned_store_event_to_blocks + ) def request_finished( self, @@ -787,6 +813,8 @@ class SimpleCPUOffloadScheduler: and frees CPU/GPU touch refs. """ state = self._reqs_to_load.pop(req_id, None) + if state is None: + state = self._abandoned_reqs_to_load.pop(req_id, None) if state is None: return # Remove from load event mapping (only this req, not whole event) @@ -830,3 +858,43 @@ class SimpleCPUOffloadScheduler: def take_events(self) -> Iterable[KVCacheEvent]: return self.cpu_block_pool.take_events() + + def reset(self) -> bool: + """Abandon pending transfers and reset the CPU cache when safe. + + Worker-side DMA may still be using blocks after reset is requested. + Keep those block refs pinned until the existing completion path reports + the transfer finished, then release refs without caching abandoned + store results. + """ + + self._abandoned_store_event_to_blocks.update(self._store_event_to_blocks) + self._store_event_to_blocks.clear() + self._in_flight_store_gpu_blocks.clear() + + # Loads that have not been sent to the worker cannot have running DMA. + # In-flight loads stay pinned and are cleaned up on completion. + for req_id in list(self._reqs_to_load): + state = self._reqs_to_load.pop(req_id) + if state.load_event is None: + self._reqs_to_load[req_id] = state + self._cleanup_load_request(req_id) + else: + self._abandoned_reqs_to_load[req_id] = state + + self._reqs_to_store.clear() + self._store_event_to_reqs.clear() + self._store_event_pending_counts = { + event_idx: count + for event_idx, count in self._store_event_pending_counts.items() + if event_idx in self._abandoned_store_event_to_blocks + } + self._cursor = None + # NOTE: _load_event_counter / _store_event_counter are not + # reset as they are monotonic and must stay ahead of the workers + # high-water marks to avoid event index collisions + + if self._abandoned_store_event_to_blocks or self._abandoned_reqs_to_load: + return False + + return self.cpu_block_pool.reset_prefix_cache()