[Perf][KV Offload] Avoid quadratic ARC batch eviction (#50992)

Signed-off-by: MINJUN GIL <alswnsrlf12@naver.com>
This commit is contained in:
MINJUN GIL
2026-08-05 10:58:36 +03:00
committed by GitHub
parent 96e333e81a
commit b92352ca2c
2 changed files with 132 additions and 21 deletions
+112
View File
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections import OrderedDict
from collections.abc import Iterable
from dataclasses import dataclass
@@ -59,6 +60,17 @@ class ExpectedPrepareStoreOutput:
evicted_keys: list[int]
class _CountingOrderedDict(OrderedDict):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.items_yielded = 0
def items(self):
for item in super().items():
self.items_yielded += 1
yield item
def to_key(int_hash: int) -> OffloadKey:
return make_offload_key(str(int_hash).encode(), 0)
@@ -686,6 +698,106 @@ class TestARCPolicy:
# block 5 should be in T1
assert to_keys([5])[0] in arc_policy.t1
def test_batch_eviction_scans_t1_and_t2_once(self):
"""ARC batch eviction must preserve order without restarting scans."""
cpu_manager, arc_policy = self._make_manager(
num_blocks=256, enable_events=False
)
keys = to_keys(list(range(256)))
cpu_manager.prepare_store(keys, _EMPTY_REQ_CTX)
cpu_manager.complete_store(keys, _EMPTY_REQ_CTX)
cpu_manager.touch(keys[128:], _EMPTY_REQ_CTX)
arc_policy.target_t1_size = 64
protected = {keys[0], keys[2], keys[255]}
arc_policy.t1[keys[1]].ref_cnt = 1
arc_policy.t2[keys[254]].ref_cnt = 1
num_evictions = 124
num_t1_evictions = len(arc_policy.t1) - int(arc_policy.target_t1_size) + 1
num_t2_evictions = num_evictions - num_t1_evictions
t1_order = list(arc_policy.t1)
t2_order = list(arc_policy.t2)
expected_t1 = [
key
for key, block in arc_policy.t1.items()
if block.ref_cnt == 0 and key not in protected
][:num_t1_evictions]
expected_t2 = [
key
for key, block in arc_policy.t2.items()
if block.ref_cnt == 0 and key not in protected
][:num_t2_evictions]
expected_t1_scans = t1_order.index(expected_t1[-1]) + 1
expected_t2_scans = t2_order.index(expected_t2[-1]) + 1
counting_t1 = _CountingOrderedDict(arc_policy.t1)
counting_t2 = _CountingOrderedDict(arc_policy.t2)
arc_policy.t1 = counting_t1
arc_policy.t2 = counting_t2
evicted = arc_policy.evict(num_evictions, protected)
assert evicted is not None
assert [key for key, _ in evicted] == expected_t1 + expected_t2
assert counting_t1.items_yielded == expected_t1_scans
assert counting_t2.items_yielded == expected_t2_scans
def test_batch_eviction_falls_back_after_t1_iterator_exhausted(self):
"""An exhausted T1 scan must keep falling back to T2."""
cpu_manager, arc_policy = self._make_manager(num_blocks=8, enable_events=False)
keys = to_keys(list(range(8)))
cpu_manager.prepare_store(keys, _EMPTY_REQ_CTX)
cpu_manager.complete_store(keys, _EMPTY_REQ_CTX)
cpu_manager.touch(keys[6:], _EMPTY_REQ_CTX)
arc_policy.target_t1_size = 4
t1_order = list(arc_policy.t1)
t2_order = list(arc_policy.t2)
protected = set(t1_order[1:3])
for key in t1_order[3:]:
arc_policy.t1[key].ref_cnt = 1
# Selecting the sole eligible T1 entry leaves virtual_t1_size above
# the target, so each remaining selection must retry T1 then use T2.
eligible_t1 = [
key
for key, block in arc_policy.t1.items()
if block.ref_cnt == 0 and key not in protected
]
assert eligible_t1 == t1_order[:1]
assert len(t1_order) - 1 >= int(arc_policy.target_t1_size)
counting_t1 = _CountingOrderedDict(arc_policy.t1)
counting_t2 = _CountingOrderedDict(arc_policy.t2)
arc_policy.t1 = counting_t1
arc_policy.t2 = counting_t2
evicted = arc_policy.evict(3, protected)
assert evicted is not None
assert [key for key, _ in evicted] == [t1_order[0], *t2_order]
assert counting_t1.items_yielded == len(t1_order)
assert counting_t2.items_yielded == len(t2_order)
def test_batch_eviction_failure_is_atomic(self):
"""Finding only some candidates must not partially evict the cache."""
cpu_manager, arc_policy = self._make_manager(num_blocks=4, enable_events=False)
keys = to_keys(list(range(4)))
cpu_manager.prepare_store(keys, _EMPTY_REQ_CTX)
cpu_manager.complete_store(keys, _EMPTY_REQ_CTX)
before_t1 = list(arc_policy.t1.items())
protected = set(keys[1:])
assert arc_policy.evict(2, protected) is None
assert list(arc_policy.t1.items()) == before_t1
assert not arc_policy.t2
assert not arc_policy.b1
assert not arc_policy.b2
def test_ghost_list_bounds(self):
"""
Tests that ghost lists (B1, B2) don't grow unbounded.
+20 -21
View File
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections import OrderedDict
from collections.abc import Iterable
from collections.abc import Iterable, Iterator
from typing_extensions import override
@@ -120,37 +120,36 @@ class ARCCachePolicy(CachePolicy):
candidates: list[
tuple[OffloadKey, BlockStatus, bool]
] = [] # (key, block, from_t1)
already_selected: set[OffloadKey] = set()
virtual_t1_size = len(self.t1)
# Keep the scans monotonic: restarting from the LRU end after every
# selection makes a batch eviction quadratic in the number of blocks.
t1_iter = iter(self.t1.items())
t2_iter = iter(self.t2.items())
def next_candidate(
entries: Iterator[tuple[OffloadKey, BlockStatus]],
) -> tuple[OffloadKey, BlockStatus] | None:
for key, block in entries:
if block.ref_cnt == 0 and key not in protected:
return key, block
return None
for _ in range(n):
candidate: tuple[OffloadKey, BlockStatus, bool] | None = None
if virtual_t1_size >= int(self.target_t1_size):
for key, block in self.t1.items():
if (
block.ref_cnt == 0
and key not in protected
and key not in already_selected
):
candidate = (key, block, True)
virtual_t1_size -= 1
break
entry = next_candidate(t1_iter)
if entry is not None:
candidate = (*entry, True)
virtual_t1_size -= 1
if candidate is None:
for key, block in self.t2.items():
if (
block.ref_cnt == 0
and key not in protected
and key not in already_selected
):
candidate = (key, block, False)
break
if candidate is None:
entry = next_candidate(t2_iter)
if entry is None:
return None
candidate = (*entry, False)
candidates.append(candidate)
already_selected.add(candidate[0])
# Apply all evictions now that we know n candidates exist.
result: list[tuple[OffloadKey, BlockStatus]] = []