[Perf][KV Offload] Avoid quadratic ARC batch eviction (#50992)
Signed-off-by: MINJUN GIL <alswnsrlf12@naver.com>
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
|
||||
@@ -59,6 +60,17 @@ class ExpectedPrepareStoreOutput:
|
||||
evicted_keys: list[int]
|
||||
|
||||
|
||||
class _CountingOrderedDict(OrderedDict):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.items_yielded = 0
|
||||
|
||||
def items(self):
|
||||
for item in super().items():
|
||||
self.items_yielded += 1
|
||||
yield item
|
||||
|
||||
|
||||
def to_key(int_hash: int) -> OffloadKey:
|
||||
return make_offload_key(str(int_hash).encode(), 0)
|
||||
|
||||
@@ -686,6 +698,106 @@ class TestARCPolicy:
|
||||
# block 5 should be in T1
|
||||
assert to_keys([5])[0] in arc_policy.t1
|
||||
|
||||
def test_batch_eviction_scans_t1_and_t2_once(self):
|
||||
"""ARC batch eviction must preserve order without restarting scans."""
|
||||
cpu_manager, arc_policy = self._make_manager(
|
||||
num_blocks=256, enable_events=False
|
||||
)
|
||||
keys = to_keys(list(range(256)))
|
||||
cpu_manager.prepare_store(keys, _EMPTY_REQ_CTX)
|
||||
cpu_manager.complete_store(keys, _EMPTY_REQ_CTX)
|
||||
|
||||
cpu_manager.touch(keys[128:], _EMPTY_REQ_CTX)
|
||||
arc_policy.target_t1_size = 64
|
||||
|
||||
protected = {keys[0], keys[2], keys[255]}
|
||||
arc_policy.t1[keys[1]].ref_cnt = 1
|
||||
arc_policy.t2[keys[254]].ref_cnt = 1
|
||||
|
||||
num_evictions = 124
|
||||
num_t1_evictions = len(arc_policy.t1) - int(arc_policy.target_t1_size) + 1
|
||||
num_t2_evictions = num_evictions - num_t1_evictions
|
||||
t1_order = list(arc_policy.t1)
|
||||
t2_order = list(arc_policy.t2)
|
||||
expected_t1 = [
|
||||
key
|
||||
for key, block in arc_policy.t1.items()
|
||||
if block.ref_cnt == 0 and key not in protected
|
||||
][:num_t1_evictions]
|
||||
expected_t2 = [
|
||||
key
|
||||
for key, block in arc_policy.t2.items()
|
||||
if block.ref_cnt == 0 and key not in protected
|
||||
][:num_t2_evictions]
|
||||
expected_t1_scans = t1_order.index(expected_t1[-1]) + 1
|
||||
expected_t2_scans = t2_order.index(expected_t2[-1]) + 1
|
||||
|
||||
counting_t1 = _CountingOrderedDict(arc_policy.t1)
|
||||
counting_t2 = _CountingOrderedDict(arc_policy.t2)
|
||||
arc_policy.t1 = counting_t1
|
||||
arc_policy.t2 = counting_t2
|
||||
|
||||
evicted = arc_policy.evict(num_evictions, protected)
|
||||
|
||||
assert evicted is not None
|
||||
assert [key for key, _ in evicted] == expected_t1 + expected_t2
|
||||
assert counting_t1.items_yielded == expected_t1_scans
|
||||
assert counting_t2.items_yielded == expected_t2_scans
|
||||
|
||||
def test_batch_eviction_falls_back_after_t1_iterator_exhausted(self):
|
||||
"""An exhausted T1 scan must keep falling back to T2."""
|
||||
cpu_manager, arc_policy = self._make_manager(num_blocks=8, enable_events=False)
|
||||
keys = to_keys(list(range(8)))
|
||||
cpu_manager.prepare_store(keys, _EMPTY_REQ_CTX)
|
||||
cpu_manager.complete_store(keys, _EMPTY_REQ_CTX)
|
||||
|
||||
cpu_manager.touch(keys[6:], _EMPTY_REQ_CTX)
|
||||
arc_policy.target_t1_size = 4
|
||||
|
||||
t1_order = list(arc_policy.t1)
|
||||
t2_order = list(arc_policy.t2)
|
||||
protected = set(t1_order[1:3])
|
||||
for key in t1_order[3:]:
|
||||
arc_policy.t1[key].ref_cnt = 1
|
||||
|
||||
# Selecting the sole eligible T1 entry leaves virtual_t1_size above
|
||||
# the target, so each remaining selection must retry T1 then use T2.
|
||||
eligible_t1 = [
|
||||
key
|
||||
for key, block in arc_policy.t1.items()
|
||||
if block.ref_cnt == 0 and key not in protected
|
||||
]
|
||||
assert eligible_t1 == t1_order[:1]
|
||||
assert len(t1_order) - 1 >= int(arc_policy.target_t1_size)
|
||||
|
||||
counting_t1 = _CountingOrderedDict(arc_policy.t1)
|
||||
counting_t2 = _CountingOrderedDict(arc_policy.t2)
|
||||
arc_policy.t1 = counting_t1
|
||||
arc_policy.t2 = counting_t2
|
||||
|
||||
evicted = arc_policy.evict(3, protected)
|
||||
|
||||
assert evicted is not None
|
||||
assert [key for key, _ in evicted] == [t1_order[0], *t2_order]
|
||||
assert counting_t1.items_yielded == len(t1_order)
|
||||
assert counting_t2.items_yielded == len(t2_order)
|
||||
|
||||
def test_batch_eviction_failure_is_atomic(self):
|
||||
"""Finding only some candidates must not partially evict the cache."""
|
||||
cpu_manager, arc_policy = self._make_manager(num_blocks=4, enable_events=False)
|
||||
keys = to_keys(list(range(4)))
|
||||
cpu_manager.prepare_store(keys, _EMPTY_REQ_CTX)
|
||||
cpu_manager.complete_store(keys, _EMPTY_REQ_CTX)
|
||||
|
||||
before_t1 = list(arc_policy.t1.items())
|
||||
protected = set(keys[1:])
|
||||
|
||||
assert arc_policy.evict(2, protected) is None
|
||||
assert list(arc_policy.t1.items()) == before_t1
|
||||
assert not arc_policy.t2
|
||||
assert not arc_policy.b1
|
||||
assert not arc_policy.b2
|
||||
|
||||
def test_ghost_list_bounds(self):
|
||||
"""
|
||||
Tests that ghost lists (B1, B2) don't grow unbounded.
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Iterable
|
||||
from collections.abc import Iterable, Iterator
|
||||
|
||||
from typing_extensions import override
|
||||
|
||||
@@ -120,37 +120,36 @@ class ARCCachePolicy(CachePolicy):
|
||||
candidates: list[
|
||||
tuple[OffloadKey, BlockStatus, bool]
|
||||
] = [] # (key, block, from_t1)
|
||||
already_selected: set[OffloadKey] = set()
|
||||
virtual_t1_size = len(self.t1)
|
||||
# Keep the scans monotonic: restarting from the LRU end after every
|
||||
# selection makes a batch eviction quadratic in the number of blocks.
|
||||
t1_iter = iter(self.t1.items())
|
||||
t2_iter = iter(self.t2.items())
|
||||
|
||||
def next_candidate(
|
||||
entries: Iterator[tuple[OffloadKey, BlockStatus]],
|
||||
) -> tuple[OffloadKey, BlockStatus] | None:
|
||||
for key, block in entries:
|
||||
if block.ref_cnt == 0 and key not in protected:
|
||||
return key, block
|
||||
return None
|
||||
|
||||
for _ in range(n):
|
||||
candidate: tuple[OffloadKey, BlockStatus, bool] | None = None
|
||||
|
||||
if virtual_t1_size >= int(self.target_t1_size):
|
||||
for key, block in self.t1.items():
|
||||
if (
|
||||
block.ref_cnt == 0
|
||||
and key not in protected
|
||||
and key not in already_selected
|
||||
):
|
||||
candidate = (key, block, True)
|
||||
virtual_t1_size -= 1
|
||||
break
|
||||
entry = next_candidate(t1_iter)
|
||||
if entry is not None:
|
||||
candidate = (*entry, True)
|
||||
virtual_t1_size -= 1
|
||||
|
||||
if candidate is None:
|
||||
for key, block in self.t2.items():
|
||||
if (
|
||||
block.ref_cnt == 0
|
||||
and key not in protected
|
||||
and key not in already_selected
|
||||
):
|
||||
candidate = (key, block, False)
|
||||
break
|
||||
if candidate is None:
|
||||
entry = next_candidate(t2_iter)
|
||||
if entry is None:
|
||||
return None
|
||||
candidate = (*entry, False)
|
||||
|
||||
candidates.append(candidate)
|
||||
already_selected.add(candidate[0])
|
||||
|
||||
# Apply all evictions now that we know n candidates exist.
|
||||
result: list[tuple[OffloadKey, BlockStatus]] = []
|
||||
|
||||
Reference in New Issue
Block a user