[Bugfix][KV Offload] Track cache recency once per request (#51787)
Signed-off-by: mindungil <alswnsrlf12@naver.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Collection, Iterable
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from typing_extensions import override
|
||||
|
||||
@@ -18,6 +19,7 @@ from vllm.v1.kv_offload.base import (
|
||||
PrepareStoreOutput,
|
||||
ReqContext,
|
||||
RequestOffloadingContext,
|
||||
get_offload_group_idx,
|
||||
)
|
||||
from vllm.v1.kv_offload.cpu.common import (
|
||||
CPULoadStoreSpec,
|
||||
@@ -27,6 +29,20 @@ from vllm.v1.kv_offload.cpu.policies.base import CachePolicy, ChunkStatus
|
||||
from vllm.v1.kv_offload.cpu.policies.factory import CachePolicyFactory
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _RequestCacheAccess:
|
||||
"""Cache keys observed by one request, grouped in prefix order."""
|
||||
|
||||
owner: object
|
||||
cache_generation: int
|
||||
key_groups: dict[int, list[OffloadKey]] = field(default_factory=dict)
|
||||
seen_keys: set[OffloadKey] = field(default_factory=set)
|
||||
inserted_keys: set[OffloadKey] = field(default_factory=set)
|
||||
reused_keys: set[OffloadKey] = field(default_factory=set)
|
||||
store_miss_keys: set[OffloadKey] = field(default_factory=set)
|
||||
finished: bool = False
|
||||
|
||||
|
||||
class CPUOffloadingManager(OffloadingManager):
|
||||
"""
|
||||
An OffloadingManager with a pluggable CachePolicy, resolved by name via
|
||||
@@ -66,6 +82,7 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
self.max_tracker_size: int = max_tracker_size
|
||||
self.stores_skipped_in_current_batch: int = 0
|
||||
self.allocation_sizes_in_current_batch: list[int] = []
|
||||
self._cache_generation = 0
|
||||
|
||||
# Number of chunk references. It is ordered so can evict the LRU entry in O(1).
|
||||
self.counts: OrderedDict[OffloadKey, int] | None = (
|
||||
@@ -124,10 +141,48 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
num_unprotected -= 1
|
||||
self.counts[key] = 1
|
||||
|
||||
def _get_request_cache_access(self, req_context: ReqContext) -> _RequestCacheAccess:
|
||||
state = req_context.get_state(_RequestCacheAccess)
|
||||
if (
|
||||
state is None
|
||||
or state.owner is not self
|
||||
or state.cache_generation != self._cache_generation
|
||||
):
|
||||
state = _RequestCacheAccess(
|
||||
owner=self, cache_generation=self._cache_generation
|
||||
)
|
||||
req_context.set_state(state)
|
||||
return state
|
||||
|
||||
def _record_request_cache_access(
|
||||
self,
|
||||
keys: Iterable[OffloadKey],
|
||||
req_context: ReqContext,
|
||||
inserted_keys: Iterable[OffloadKey] = (),
|
||||
reused_keys: Iterable[OffloadKey] = (),
|
||||
) -> None:
|
||||
state = self._get_request_cache_access(req_context)
|
||||
for key in keys:
|
||||
if key in state.seen_keys:
|
||||
continue
|
||||
assert not state.finished, (
|
||||
"New cache keys observed after request finalization"
|
||||
)
|
||||
group_idx = get_offload_group_idx(key)
|
||||
state.key_groups.setdefault(group_idx, []).append(key)
|
||||
state.seen_keys.add(key)
|
||||
state.inserted_keys.update(inserted_keys)
|
||||
# Re-reading a chunk inserted by this request is an internal transfer
|
||||
# (for example, a tiering cascade), not a second cache access.
|
||||
state.reused_keys.update(
|
||||
key for key in reused_keys if key not in state.inserted_keys
|
||||
)
|
||||
|
||||
# --- OffloadingManager interface ---
|
||||
|
||||
@override
|
||||
def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext:
|
||||
self._get_request_cache_access(req_context)
|
||||
return RequestOffloadingContext()
|
||||
|
||||
@override
|
||||
@@ -144,6 +199,15 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
self,
|
||||
keys: Collection[OffloadKey],
|
||||
req_context: ReqContext,
|
||||
) -> LoadStoreSpec:
|
||||
return self._prepare_load(keys, req_context, record_access=True)
|
||||
|
||||
def _prepare_load(
|
||||
self,
|
||||
keys: Collection[OffloadKey],
|
||||
req_context: ReqContext,
|
||||
*,
|
||||
record_access: bool,
|
||||
) -> LoadStoreSpec:
|
||||
chunks = []
|
||||
for key in keys:
|
||||
@@ -156,6 +220,8 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
assert self._num_evictable_cache_chunks >= 0
|
||||
chunk.ref_cnt += 1
|
||||
chunks.append(chunk)
|
||||
if record_access:
|
||||
self._record_request_cache_access(keys, req_context, reused_keys=keys)
|
||||
return self._get_load_store_spec(keys, chunks)
|
||||
|
||||
@override
|
||||
@@ -181,13 +247,52 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
keys: Collection[OffloadKey],
|
||||
req_context: ReqContext,
|
||||
) -> PrepareStoreOutput | None:
|
||||
keys = list(keys)
|
||||
if self.counts is not None:
|
||||
num_keys = len(keys)
|
||||
self._record_accesses(keys)
|
||||
keys = [k for k in keys if self.counts.get(k, 0) >= self.store_threshold]
|
||||
self.stores_skipped_in_current_batch += num_keys - len(keys)
|
||||
# filter out chunks that are already stored
|
||||
keys_to_store = [k for k in keys if self._policy.get(k) is None]
|
||||
# Partition the original offer before admission filtering. Ready
|
||||
# resident chunks are request reuses even if their tracker entry was
|
||||
# aged out; the threshold applies only to new store candidates.
|
||||
keys_to_store: list[OffloadKey] = []
|
||||
ready_existing_keys: list[OffloadKey] = []
|
||||
for key in keys:
|
||||
chunk = self._policy.get(key)
|
||||
if chunk is None:
|
||||
keys_to_store.append(key)
|
||||
else:
|
||||
if chunk.is_ready:
|
||||
ready_existing_keys.append(key)
|
||||
|
||||
if self.counts is not None:
|
||||
num_store_candidates = len(keys_to_store)
|
||||
keys_to_store = [
|
||||
key
|
||||
for key in keys_to_store
|
||||
if self.counts.get(key, 0) >= self.store_threshold
|
||||
]
|
||||
self.stores_skipped_in_current_batch += num_store_candidates - len(
|
||||
keys_to_store
|
||||
)
|
||||
|
||||
state = self._get_request_cache_access(req_context)
|
||||
new_store_misses = [
|
||||
key for key in keys_to_store if key not in state.store_miss_keys
|
||||
]
|
||||
if new_store_misses:
|
||||
# ARC learns from B1/B2 before insert() removes the ghost entry.
|
||||
# Deduplication makes this one policy observation per request.
|
||||
self._policy.on_store_miss(new_store_misses, req_context)
|
||||
state.store_miss_keys.update(new_store_misses)
|
||||
|
||||
recorded_keys = set(keys_to_store)
|
||||
recorded_keys.update(ready_existing_keys)
|
||||
# Record once before any allocation-related early return. Store
|
||||
# candidates are classified as insertions only after allocation succeeds.
|
||||
self._record_request_cache_access(
|
||||
(key for key in keys if key in recorded_keys),
|
||||
req_context,
|
||||
reused_keys=ready_existing_keys,
|
||||
)
|
||||
|
||||
if not keys_to_store:
|
||||
return PrepareStoreOutput(
|
||||
@@ -239,6 +344,7 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
for key, chunk in zip(keys_to_store, chunks):
|
||||
self._policy.insert(key, chunk)
|
||||
self._num_write_pending_chunks += len(keys_to_store)
|
||||
state.inserted_keys.update(keys_to_store)
|
||||
|
||||
# build store specs for allocated chunks
|
||||
store_spec = self._get_load_store_spec(keys_to_store, chunks)
|
||||
@@ -284,6 +390,35 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
)
|
||||
)
|
||||
|
||||
@override
|
||||
def on_request_finished(self, req_context: ReqContext) -> None:
|
||||
state = req_context.get_state(_RequestCacheAccess)
|
||||
if (
|
||||
state is None
|
||||
or state.owner is not self
|
||||
or state.cache_generation != self._cache_generation
|
||||
or state.finished
|
||||
):
|
||||
return
|
||||
state.finished = True
|
||||
key_groups = []
|
||||
for group_idx in sorted(state.key_groups):
|
||||
keys = state.key_groups[group_idx]
|
||||
positions = {
|
||||
key: position
|
||||
for key in keys
|
||||
if (position := req_context.get_offload_key_position(key)) is not None
|
||||
}
|
||||
if len(positions) == len(keys):
|
||||
keys = sorted(keys, key=positions.__getitem__)
|
||||
key_groups.append(tuple(keys))
|
||||
self._policy.on_request_finished(
|
||||
tuple(key_groups),
|
||||
state.inserted_keys - state.reused_keys,
|
||||
state.reused_keys,
|
||||
req_context,
|
||||
)
|
||||
|
||||
@override
|
||||
def reset_cache(self) -> None:
|
||||
# Clear ALL chunks unconditionally. The scheduler's _stale_job_threshold
|
||||
@@ -292,6 +427,7 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
# flushes in-flight load job IDs to the workers before any new stores
|
||||
# can begin, preventing a cross-direction data race on reused offload chunk IDs.
|
||||
self._policy.clear()
|
||||
self._cache_generation += 1
|
||||
self._num_evictable_cache_chunks = 0
|
||||
self._num_write_pending_chunks = 0
|
||||
|
||||
|
||||
Reference in New Issue
Block a user