[kv_offload] Move FilterReusedOffloadingManager logic to CPUOffloadingManager (#41727)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
parent
4b64fc2cbf
commit
4955990f1b
@@ -17,7 +17,6 @@ from vllm.v1.kv_offload.base import (
|
||||
from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec
|
||||
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
|
||||
from vllm.v1.kv_offload.cpu.policies.arc import ARCCachePolicy
|
||||
from vllm.v1.kv_offload.reuse_manager import FilterReusedOffloadingManager
|
||||
|
||||
|
||||
def make_req_context(kv_transfer_params: dict | None = None) -> ReqContext:
|
||||
@@ -565,14 +564,14 @@ class TestARCPolicy:
|
||||
|
||||
def test_filter_reused_manager():
|
||||
"""
|
||||
Tests FilterReusedOffloadingManager with a CPUOffloadingManager.
|
||||
Tests CPUOffloadingManager reuse filtering (store_threshold=2).
|
||||
"""
|
||||
lru_manager = CPUOffloadingManager(
|
||||
num_blocks=4, cache_policy="lru", enable_events=True
|
||||
)
|
||||
|
||||
manager = FilterReusedOffloadingManager(
|
||||
backing=lru_manager, store_threshold=2, max_tracker_size=3
|
||||
manager = CPUOffloadingManager(
|
||||
num_blocks=4,
|
||||
cache_policy="lru",
|
||||
enable_events=True,
|
||||
store_threshold=2,
|
||||
max_tracker_size=3,
|
||||
)
|
||||
|
||||
# Lookup [1, 2] -> 1st time, added to tracker but not eligible for store yet
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Collection, Iterable
|
||||
from typing import Literal
|
||||
|
||||
@@ -37,6 +38,8 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
num_blocks: int,
|
||||
cache_policy: Literal["lru", "arc"] = "lru",
|
||||
enable_events: bool = False,
|
||||
store_threshold: int = 1,
|
||||
max_tracker_size: int = 64_000,
|
||||
):
|
||||
self.medium: str = CPULoadStoreSpec.medium()
|
||||
self._num_blocks: int = num_blocks
|
||||
@@ -50,6 +53,13 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
f"Supported: {list(_CACHE_POLICIES)}"
|
||||
)
|
||||
self._policy: CachePolicy = policy_cls(cache_capacity=num_blocks)
|
||||
self.store_threshold: int = store_threshold
|
||||
self.max_tracker_size: int = max_tracker_size
|
||||
|
||||
# Number of block references. It is ordered so can evict the LRU entry in O(1).
|
||||
self.counts: OrderedDict[OffloadKey, int] | None = (
|
||||
OrderedDict() if store_threshold >= 2 else None
|
||||
)
|
||||
|
||||
# --- block pool ---
|
||||
|
||||
@@ -85,6 +95,14 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
# --- OffloadingManager interface ---
|
||||
|
||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None:
|
||||
if self.counts is not None:
|
||||
if key in self.counts:
|
||||
self.counts.move_to_end(key)
|
||||
self.counts[key] += 1
|
||||
else:
|
||||
if len(self.counts) >= self.max_tracker_size:
|
||||
self.counts.popitem(last=False)
|
||||
self.counts[key] = 1
|
||||
block = self._policy.get(key)
|
||||
if block is None:
|
||||
return False
|
||||
@@ -123,6 +141,8 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
keys: Collection[OffloadKey],
|
||||
req_context: ReqContext,
|
||||
) -> PrepareStoreOutput | None:
|
||||
if self.counts is not None:
|
||||
keys = [k for k in keys if self.counts.get(k, 0) >= self.store_threshold]
|
||||
# filter out blocks that are already stored
|
||||
keys_to_store = [k for k in keys if self._policy.get(k) is None]
|
||||
|
||||
|
||||
@@ -15,7 +15,6 @@ from vllm.v1.kv_offload.base import (
|
||||
from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec
|
||||
from vllm.v1.kv_offload.cpu.gpu_worker import CpuGpuOffloadingHandlers
|
||||
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
|
||||
from vllm.v1.kv_offload.reuse_manager import FilterReusedOffloadingManager
|
||||
from vllm.v1.kv_offload.worker.worker import OffloadingHandler
|
||||
|
||||
|
||||
@@ -61,25 +60,21 @@ class CPUOffloadingSpec(OffloadingSpec):
|
||||
kv_events_config is not None and kv_events_config.enable_kv_cache_events
|
||||
)
|
||||
|
||||
self._manager = CPUOffloadingManager(
|
||||
num_blocks=self.num_blocks,
|
||||
cache_policy=self.eviction_policy, # type: ignore[arg-type]
|
||||
enable_events=enable_events,
|
||||
)
|
||||
|
||||
# store_threshold: how many times a block must appear in lookup()
|
||||
# before it is eligible for CPU offloading. Values < 2 disable
|
||||
# filtering (a threshold of 1 equals no filter; 0 is the default).
|
||||
store_threshold = int(self.extra_config.get("store_threshold", 0))
|
||||
if store_threshold >= 2:
|
||||
max_tracker_size = int(
|
||||
self.extra_config.get("max_tracker_size", 64_000)
|
||||
)
|
||||
self._manager = FilterReusedOffloadingManager(
|
||||
backing=self._manager,
|
||||
store_threshold=store_threshold,
|
||||
max_tracker_size=max_tracker_size,
|
||||
)
|
||||
|
||||
# Maximum entries in the internal tracker's LRU table.
|
||||
max_tracker_size = int(self.extra_config.get("max_tracker_size", 64_000))
|
||||
|
||||
self._manager = CPUOffloadingManager(
|
||||
num_blocks=self.num_blocks,
|
||||
cache_policy=self.eviction_policy, # type: ignore[arg-type]
|
||||
enable_events=enable_events,
|
||||
store_threshold=store_threshold,
|
||||
max_tracker_size=max_tracker_size,
|
||||
)
|
||||
return self._manager
|
||||
|
||||
def get_handlers(
|
||||
|
||||
@@ -1,125 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Reuse-frequency gating for CPU KV-cache offload stores.
|
||||
|
||||
FilterReusedOffloadingManager — OffloadingManager decorator that skips
|
||||
storing blocks that have not yet been seen enough times.
|
||||
"""
|
||||
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Collection, Iterable
|
||||
|
||||
from vllm.v1.kv_offload.base import (
|
||||
LoadStoreSpec,
|
||||
OffloadingEvent,
|
||||
OffloadingManager,
|
||||
OffloadKey,
|
||||
PrepareStoreOutput,
|
||||
ReqContext,
|
||||
)
|
||||
|
||||
|
||||
class FilterReusedOffloadingManager(OffloadingManager):
|
||||
"""An :class:`OffloadingManager` decorator that skips storing blocks
|
||||
whose reuse frequency is below *store_threshold*.
|
||||
|
||||
All methods are delegated to the *backing* manager. Two methods are
|
||||
intercepted:
|
||||
|
||||
* ``prepare_store`` — filters out keys that have not yet
|
||||
* ``lookup`` — records the visited key in an internal LRU
|
||||
counter, then delegates to the backing manager.
|
||||
crossed the threshold *before* calling the backing
|
||||
``prepare_store``.
|
||||
|
||||
Args:
|
||||
backing: The underlying ``OffloadingManager`` to delegate to.
|
||||
store_threshold: A block must be seen at least this many times in
|
||||
``lookup()`` before it is eligible for offloading. Must be >= 2
|
||||
(a value of 1 would be equivalent to no filtering).
|
||||
max_tracker_size: Maximum entries in the internal tracker's LRU table.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
backing: OffloadingManager,
|
||||
store_threshold: int = 2,
|
||||
max_tracker_size: int = 64_000,
|
||||
):
|
||||
if store_threshold < 2:
|
||||
raise ValueError(
|
||||
"FilterReusedOffloadingManager store_threshold must be >= 2, "
|
||||
f"got {store_threshold}"
|
||||
)
|
||||
if max_tracker_size < 1:
|
||||
raise ValueError(
|
||||
"FilterReusedOffloadingManager max_tracker_size must be >= 1, "
|
||||
f"got {max_tracker_size}"
|
||||
)
|
||||
self._backing = backing
|
||||
self.store_threshold = store_threshold
|
||||
self.max_tracker_size = max_tracker_size
|
||||
# Ordered so we can evict the LRU entry in O(1).
|
||||
self.counts: OrderedDict[OffloadKey, int] = OrderedDict()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Intercepted methods
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None:
|
||||
"""Record the key, then delegate lookup to backing manager."""
|
||||
if key in self.counts:
|
||||
self.counts.move_to_end(key)
|
||||
self.counts[key] += 1
|
||||
else:
|
||||
if len(self.counts) >= self.max_tracker_size:
|
||||
self.counts.popitem(last=False) # evict LRU
|
||||
self.counts[key] = 1
|
||||
return self._backing.lookup(key, req_context)
|
||||
|
||||
def prepare_store(
|
||||
self, keys: Collection[OffloadKey], req_context: ReqContext
|
||||
) -> PrepareStoreOutput | None:
|
||||
"""Filter out blocks below threshold, then delegate to backing.
|
||||
|
||||
Filtering is evaluated *before* calling the backing manager's
|
||||
``prepare_store`` so that blocks that would be skipped do not
|
||||
consume any CPU offload capacity.
|
||||
"""
|
||||
eligible = [
|
||||
key for key in keys if self.counts.get(key, 0) >= self.store_threshold
|
||||
]
|
||||
|
||||
# Passing an empty list is intentional and safe — CPUOffloadingManager
|
||||
# handles it correctly, returning a PrepareStoreOutput with empty lists.
|
||||
# Delegate to the backing manager with only the eligible keys.
|
||||
return self._backing.prepare_store(eligible, req_context)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Delegated methods
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def prepare_load(
|
||||
self, keys: Collection[OffloadKey], req_context: ReqContext
|
||||
) -> LoadStoreSpec:
|
||||
return self._backing.prepare_load(keys, req_context)
|
||||
|
||||
def touch(self, keys: Collection[OffloadKey], req_context: ReqContext) -> None:
|
||||
return self._backing.touch(keys, req_context)
|
||||
|
||||
def complete_load(
|
||||
self, keys: Collection[OffloadKey], req_context: ReqContext
|
||||
) -> None:
|
||||
return self._backing.complete_load(keys, req_context)
|
||||
|
||||
def complete_store(
|
||||
self,
|
||||
keys: Collection[OffloadKey],
|
||||
req_context: ReqContext,
|
||||
success: bool = True,
|
||||
) -> None:
|
||||
return self._backing.complete_store(keys, req_context, success)
|
||||
|
||||
def take_events(self) -> Iterable[OffloadingEvent]:
|
||||
return self._backing.take_events()
|
||||
Reference in New Issue
Block a user