diff --git a/tests/v1/kv_offload/cpu/test_manager.py b/tests/v1/kv_offload/cpu/test_manager.py index 5a36be16bf..6b33041791 100644 --- a/tests/v1/kv_offload/cpu/test_manager.py +++ b/tests/v1/kv_offload/cpu/test_manager.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from collections import OrderedDict from collections.abc import Iterable from dataclasses import dataclass @@ -59,6 +60,17 @@ class ExpectedPrepareStoreOutput: evicted_keys: list[int] +class _CountingOrderedDict(OrderedDict): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.items_yielded = 0 + + def items(self): + for item in super().items(): + self.items_yielded += 1 + yield item + + def to_key(int_hash: int) -> OffloadKey: return make_offload_key(str(int_hash).encode(), 0) @@ -686,6 +698,106 @@ class TestARCPolicy: # block 5 should be in T1 assert to_keys([5])[0] in arc_policy.t1 + def test_batch_eviction_scans_t1_and_t2_once(self): + """ARC batch eviction must preserve order without restarting scans.""" + cpu_manager, arc_policy = self._make_manager( + num_blocks=256, enable_events=False + ) + keys = to_keys(list(range(256))) + cpu_manager.prepare_store(keys, _EMPTY_REQ_CTX) + cpu_manager.complete_store(keys, _EMPTY_REQ_CTX) + + cpu_manager.touch(keys[128:], _EMPTY_REQ_CTX) + arc_policy.target_t1_size = 64 + + protected = {keys[0], keys[2], keys[255]} + arc_policy.t1[keys[1]].ref_cnt = 1 + arc_policy.t2[keys[254]].ref_cnt = 1 + + num_evictions = 124 + num_t1_evictions = len(arc_policy.t1) - int(arc_policy.target_t1_size) + 1 + num_t2_evictions = num_evictions - num_t1_evictions + t1_order = list(arc_policy.t1) + t2_order = list(arc_policy.t2) + expected_t1 = [ + key + for key, block in arc_policy.t1.items() + if block.ref_cnt == 0 and key not in protected + ][:num_t1_evictions] + expected_t2 = [ + key + for key, block in arc_policy.t2.items() + if block.ref_cnt == 0 and key not in protected + ][:num_t2_evictions] + expected_t1_scans = t1_order.index(expected_t1[-1]) + 1 + expected_t2_scans = t2_order.index(expected_t2[-1]) + 1 + + counting_t1 = _CountingOrderedDict(arc_policy.t1) + counting_t2 = _CountingOrderedDict(arc_policy.t2) + arc_policy.t1 = counting_t1 + arc_policy.t2 = counting_t2 + + evicted = arc_policy.evict(num_evictions, protected) + + assert evicted is not None + assert [key for key, _ in evicted] == expected_t1 + expected_t2 + assert counting_t1.items_yielded == expected_t1_scans + assert counting_t2.items_yielded == expected_t2_scans + + def test_batch_eviction_falls_back_after_t1_iterator_exhausted(self): + """An exhausted T1 scan must keep falling back to T2.""" + cpu_manager, arc_policy = self._make_manager(num_blocks=8, enable_events=False) + keys = to_keys(list(range(8))) + cpu_manager.prepare_store(keys, _EMPTY_REQ_CTX) + cpu_manager.complete_store(keys, _EMPTY_REQ_CTX) + + cpu_manager.touch(keys[6:], _EMPTY_REQ_CTX) + arc_policy.target_t1_size = 4 + + t1_order = list(arc_policy.t1) + t2_order = list(arc_policy.t2) + protected = set(t1_order[1:3]) + for key in t1_order[3:]: + arc_policy.t1[key].ref_cnt = 1 + + # Selecting the sole eligible T1 entry leaves virtual_t1_size above + # the target, so each remaining selection must retry T1 then use T2. + eligible_t1 = [ + key + for key, block in arc_policy.t1.items() + if block.ref_cnt == 0 and key not in protected + ] + assert eligible_t1 == t1_order[:1] + assert len(t1_order) - 1 >= int(arc_policy.target_t1_size) + + counting_t1 = _CountingOrderedDict(arc_policy.t1) + counting_t2 = _CountingOrderedDict(arc_policy.t2) + arc_policy.t1 = counting_t1 + arc_policy.t2 = counting_t2 + + evicted = arc_policy.evict(3, protected) + + assert evicted is not None + assert [key for key, _ in evicted] == [t1_order[0], *t2_order] + assert counting_t1.items_yielded == len(t1_order) + assert counting_t2.items_yielded == len(t2_order) + + def test_batch_eviction_failure_is_atomic(self): + """Finding only some candidates must not partially evict the cache.""" + cpu_manager, arc_policy = self._make_manager(num_blocks=4, enable_events=False) + keys = to_keys(list(range(4))) + cpu_manager.prepare_store(keys, _EMPTY_REQ_CTX) + cpu_manager.complete_store(keys, _EMPTY_REQ_CTX) + + before_t1 = list(arc_policy.t1.items()) + protected = set(keys[1:]) + + assert arc_policy.evict(2, protected) is None + assert list(arc_policy.t1.items()) == before_t1 + assert not arc_policy.t2 + assert not arc_policy.b1 + assert not arc_policy.b2 + def test_ghost_list_bounds(self): """ Tests that ghost lists (B1, B2) don't grow unbounded. diff --git a/vllm/v1/kv_offload/cpu/policies/arc.py b/vllm/v1/kv_offload/cpu/policies/arc.py index d6569cbcfd..cd34e56060 100644 --- a/vllm/v1/kv_offload/cpu/policies/arc.py +++ b/vllm/v1/kv_offload/cpu/policies/arc.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from collections import OrderedDict -from collections.abc import Iterable +from collections.abc import Iterable, Iterator from typing_extensions import override @@ -120,37 +120,36 @@ class ARCCachePolicy(CachePolicy): candidates: list[ tuple[OffloadKey, BlockStatus, bool] ] = [] # (key, block, from_t1) - already_selected: set[OffloadKey] = set() virtual_t1_size = len(self.t1) + # Keep the scans monotonic: restarting from the LRU end after every + # selection makes a batch eviction quadratic in the number of blocks. + t1_iter = iter(self.t1.items()) + t2_iter = iter(self.t2.items()) + + def next_candidate( + entries: Iterator[tuple[OffloadKey, BlockStatus]], + ) -> tuple[OffloadKey, BlockStatus] | None: + for key, block in entries: + if block.ref_cnt == 0 and key not in protected: + return key, block + return None for _ in range(n): candidate: tuple[OffloadKey, BlockStatus, bool] | None = None if virtual_t1_size >= int(self.target_t1_size): - for key, block in self.t1.items(): - if ( - block.ref_cnt == 0 - and key not in protected - and key not in already_selected - ): - candidate = (key, block, True) - virtual_t1_size -= 1 - break + entry = next_candidate(t1_iter) + if entry is not None: + candidate = (*entry, True) + virtual_t1_size -= 1 if candidate is None: - for key, block in self.t2.items(): - if ( - block.ref_cnt == 0 - and key not in protected - and key not in already_selected - ): - candidate = (key, block, False) - break - if candidate is None: + entry = next_candidate(t2_iter) + if entry is None: return None + candidate = (*entry, False) candidates.append(candidate) - already_selected.add(candidate[0]) # Apply all evictions now that we know n candidates exist. result: list[tuple[OffloadKey, BlockStatus]] = []