From ed938ad7db9c28e3725058037c41285d8f46869e Mon Sep 17 00:00:00 2001 From: Varun Sundar Rabindranath Date: Wed, 17 Jun 2026 22:34:59 -0400 Subject: [PATCH] [CPUOffloading] Guard CPU eviction check (#45757) Signed-off-by: Varun Sundar Rabindranath Co-authored-by: Varun Sundar Rabindranath --- tests/v1/kv_offload/cpu/test_manager.py | 93 +++++++++++++++++++++++++ vllm/v1/kv_offload/cpu/manager.py | 21 ++++++ 2 files changed, 114 insertions(+) diff --git a/tests/v1/kv_offload/cpu/test_manager.py b/tests/v1/kv_offload/cpu/test_manager.py index 8b68855def..6e4cbb1c6b 100644 --- a/tests/v1/kv_offload/cpu/test_manager.py +++ b/tests/v1/kv_offload/cpu/test_manager.py @@ -689,3 +689,96 @@ def test_filter_reused_manager(): assert prepare_store_output.keys_to_store == [] manager.complete_store(to_keys([1]), _EMPTY_REQ_CTX) + + +def test_evictable_cache_block_count(): + """ + Verifies _num_evictable_cache_blocks is maintained correctly through the + full store/load lifecycle, eviction, failed stores, concurrent loads, + reset_cache, and the early-exit fast path in prepare_store. + """ + manager = make_cpu_manager(num_blocks=4, cache_policy="lru") + + # Initially no blocks allocated. + assert manager._num_evictable_cache_blocks == 0 + + # Initial cache state [x, x, x, x] + + # We get 3 blocks from the cache. + manager.prepare_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX) + # cache state [1', 2', 3', x] <- 1', 2', 3' are actively being used. + assert manager._num_evictable_cache_blocks == 0 + + # Completing stores makes them idle. + manager.complete_store(to_keys([1, 2, 3]), _EMPTY_REQ_CTX) + # cache state [1, 2, 3, x] <- 1, 2, 3 blocks are idle. + assert manager._num_evictable_cache_blocks == 3 + + # prepare_load pins a block: idle count decrements once even if the + # same block is loaded by two concurrent callers. + manager.prepare_load(to_keys([1]), _EMPTY_REQ_CTX) + # cache state [1', 2, 3, x] <- 2, 3 blocks are idle. + assert manager._num_evictable_cache_blocks == 2 + manager.prepare_load(to_keys([1]), _EMPTY_REQ_CTX) # 2nd concurrent load + # cache state [1', 2, 3, x] <- 2, 3 blocks are idle. + assert manager._num_evictable_cache_blocks == 2 # no double-decrement + + # First complete_load does not restore idle (ref_cnt still 1). + manager.complete_load(to_keys([1]), _EMPTY_REQ_CTX) + # cache state [1', 2, 3, x] <- 2, 3 blocks are idle. + assert manager._num_evictable_cache_blocks == 2 + # Second complete_load drops ref_cnt to 0 -> block becomes idle again. + manager.complete_load(to_keys([1]), _EMPTY_REQ_CTX) + # cache state [1, 2, 3, x] <- 1, 2, 3 blocks are idle. + assert manager._num_evictable_cache_blocks == 3 + + # Eviction decrements idle count. + # Cache has 3 stored blocks and 1 free slot. Storing 3 new keys needs 2 eviction. + manager.prepare_store(to_keys([4, 5, 6]), _EMPTY_REQ_CTX) + # cache state [1, 4', 5', 6'] <- block 1 is idle + assert manager._num_evictable_cache_blocks == 1 + + # Failed store does not increment idle count (block discarded from cache). + manager.complete_store(to_keys([4, 5, 6]), _EMPTY_REQ_CTX, success=False) + # cache state [1, x, x, x] <- block 1 is idle. Other returned to cache. + assert manager._num_evictable_cache_blocks == 1 + + # reset_cache zeroes the count unconditionally. + manager.reset_cache() + # cache state [x, x, x, x] + assert manager._num_evictable_cache_blocks == 0 + + # setup 3 blocks with loads so idle count drops to 0. + manager.prepare_store(to_keys([10, 11, 12]), _EMPTY_REQ_CTX) + manager.complete_store(to_keys([10, 11, 12]), _EMPTY_REQ_CTX) + manager.prepare_load(to_keys([10, 11, 12]), _EMPTY_REQ_CTX) + # cache state [10', 11', 12', x] + assert manager._num_evictable_cache_blocks == 0 + + # prepare_store requiring eviction must return None immediately (fast exit). + # Spy on policy.evict to confirm the fast path short-circuits before calling it. + evict_called = False + original_evict = manager._policy.evict + + def spy_evict(*args, **kwargs): + nonlocal evict_called + evict_called = True + return original_evict(*args, **kwargs) + + manager._policy.evict = spy_evict # type: ignore[method-assign] + # cache state [10', 11', 12', x] <- cannot evict anything + assert manager.prepare_store(to_keys([14, 15]), _EMPTY_REQ_CTX) is None + assert not evict_called, ( + "_num_evictable_cache_blocks==0 should short-circuit before evict()" + ) + + # After releasing the loads, eviction becomes possible again. + manager.complete_load(to_keys([10, 11, 12]), _EMPTY_REQ_CTX) + # cache state [10, 11, 12, x] <- 10, 11, 12 are idle + assert manager._num_evictable_cache_blocks == 3 + assert manager.prepare_store(to_keys([14, 15]), _EMPTY_REQ_CTX) is not None + # cache state [10, 11, 14', 15'] <- 10, 11 are idle + assert manager._num_evictable_cache_blocks == 2 + manager.complete_store(to_keys([14, 15]), _EMPTY_REQ_CTX) + # cache state [10, 11, 14, 15] <- all blocks idle + assert manager._num_evictable_cache_blocks == 4 diff --git a/vllm/v1/kv_offload/cpu/manager.py b/vllm/v1/kv_offload/cpu/manager.py index 3218e152df..7835d35309 100644 --- a/vllm/v1/kv_offload/cpu/manager.py +++ b/vllm/v1/kv_offload/cpu/manager.py @@ -59,6 +59,9 @@ class CPUOffloadingManager(OffloadingManager): f"Supported: {list(_CACHE_POLICIES)}" ) self._policy: CachePolicy = policy_cls(cache_capacity=num_blocks) + # Track the number of blocks in the cache that are evictable. i.e. ref_cnt 0. + self._num_evictable_cache_blocks: int = 0 + self.store_threshold: int = store_threshold self.max_tracker_size: int = max_tracker_size self.stores_skipped_in_current_batch: int = 0 @@ -133,6 +136,9 @@ class CPUOffloadingManager(OffloadingManager): block = self._policy.get(key) assert block is not None, f"Block {key!r} not found in cache" assert block.is_ready, f"Block {key!r} is not ready for reading" + if block.ref_cnt == 0: + self._num_evictable_cache_blocks -= 1 # ref_cnt 0 -> 1 + assert self._num_evictable_cache_blocks >= 0 block.ref_cnt += 1 blocks.append(block) return self._get_load_store_spec(keys, blocks) @@ -150,6 +156,8 @@ class CPUOffloadingManager(OffloadingManager): assert block is not None, f"Block {key!r} not found" assert block.ref_cnt > 0, f"Block {key!r} ref_cnt is already 0" block.ref_cnt -= 1 + if block.ref_cnt == 0: + self._num_evictable_cache_blocks += 1 # ref_cnt 1 -> 0 @override def prepare_store( @@ -175,12 +183,23 @@ class CPUOffloadingManager(OffloadingManager): to_evict: list[OffloadKey] = [] if num_blocks_to_evict > 0: + if num_blocks_to_evict > self._num_evictable_cache_blocks: + # Eviction will fail. + return None + # There is a still a chance for eviction failure as some of the + # idle blocks might be in the protected list. + # Blocks from the original input are excluded from eviction candidates: # a block that was already stored must remain in the cache after this call. protected = set(keys) evicted = self._policy.evict(num_blocks_to_evict, protected) if evicted is None: return None + + # cache-policy removes only idle blocks. + self._num_evictable_cache_blocks -= len(evicted) + assert self._num_evictable_cache_blocks >= 0 + for key, block in evicted: self._free_block(block) to_evict.append(key) @@ -225,6 +244,7 @@ class CPUOffloadingManager(OffloadingManager): block = self._policy.get(key) if block is not None and not block.is_ready: block.ref_cnt = 0 + self._num_evictable_cache_blocks += 1 stored_keys.append(key) else: for key in keys: @@ -250,6 +270,7 @@ class CPUOffloadingManager(OffloadingManager): # flushes in-flight load job IDs to the workers before any new stores # can begin, preventing a cross-direction data race on reused offload block IDs. self._policy.clear() + self._num_evictable_cache_blocks = 0 self._free_list.clear() self._num_allocated_blocks = 0