[kv_offload] Implement reset_cache() for the offloading connector (#41956)

Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Or Ozeri <or@ozery.com>
This commit is contained in:
Martin Hickey
2026-05-14 16:00:10 +03:00
committed by GitHub
co-authored by gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Or Ozeri
parent 6548560496
commit 5bd8c71e79
8 changed files with 153 additions and 0 deletions
@@ -223,6 +223,9 @@ class OffloadingConnectorScheduler:
# Job ID counter shared by loads and stores.
self._job_counter: int = 0
# Threshold value for stale jobs. All job ids >= _stale_job_threshold are
# active jobs.
self._stale_job_threshold: int = 0
self._jobs: dict[int, TransferJobStatus] = {}
# block_id -> pending store job_ids. Used to track jobs that needs
@@ -802,6 +805,13 @@ class OffloadingConnectorScheduler:
meta = OffloadingWorkerMetadata()
for job_id, count in meta.completed_jobs.items():
assert count > 0
if job_id < self._stale_job_threshold:
logger.debug(
"Skipping stale completed job %d (pre-reset counter: %d)",
job_id,
self._stale_job_threshold,
)
continue
job_status = self._jobs[job_id]
job_status.pending_count -= count
if job_status.pending_count > 0:
@@ -882,5 +892,33 @@ class OffloadingConnectorScheduler:
lora_name=None,
)
def reset_cache(self) -> None:
"""Reset the offloading manager cache, evicting all stored blocks."""
# reset_cache cannot be called in the middle of a schedule step
assert not self._current_batch_load_jobs
assert not self._current_batch_jobs_to_flush
# Flush all in-flight jobs
self._current_batch_jobs_to_flush.update(self._jobs.keys())
# Reset offloading manager cache
self.manager.reset_cache()
# Reset store progress so active requests re-offload from block 0
for status in self._req_status.values():
for group_state in status.group_states:
group_state.next_stored_block_idx = 0
# Discard jobs and save job_counter to be able to discard worker responses
self._stale_job_threshold = self._job_counter
self._jobs.clear()
self._block_id_to_pending_jobs.clear()
# Note: _current_batch_jobs_to_flush is intentionally NOT cleared.
# The load flush IDs collected above must be delivered to workers.
if self._blocks_being_loaded is not None:
self._blocks_being_loaded.clear()
def shutdown(self) -> None:
self.manager.shutdown()
@@ -174,6 +174,11 @@ class OffloadingConnector(KVConnectorBase_V1, SupportsHMA):
def get_required_kvcache_layout(cls, vllm_config: VllmConfig) -> str | None:
return "HND"
def reset_cache(self) -> bool | None:
assert self.connector_scheduler is not None
self.connector_scheduler.reset_cache()
return True
def get_kv_connector_stats(self) -> KVConnectorStats | None:
if self.connector_worker is None:
return None # We only emit stats from the worker-side
+4
View File
@@ -218,6 +218,10 @@ class OffloadingManager(ABC):
"""
return ()
def reset_cache(self) -> None:
"""Evict all tracked blocks and reset internal state."""
return
def shutdown(self) -> None:
"""Shutdown the manager and release any resources."""
return
+11
View File
@@ -223,6 +223,17 @@ class CPUOffloadingManager(OffloadingManager):
)
)
def reset_cache(self) -> None:
# Clear ALL blocks unconditionally. The scheduler's _stale_job_threshold
# guarantees that complete_load / complete_store are never called for
# pre-reset jobs, so no lazy cleanup is needed. The scheduler also
# flushes in-flight load job IDs to the workers before any new stores
# can begin, preventing a cross-direction data race on reused offload block IDs.
self._policy.clear()
self._free_list.clear()
self._num_allocated_blocks = 0
def take_events(self) -> Iterable[OffloadingEvent]:
if self.events is not None:
yield from self.events
+7
View File
@@ -94,6 +94,13 @@ class ARCCachePolicy(CachePolicy):
# move to MRU position (end) to keep it fresh in the ghost list
self.b2.move_to_end(key)
def clear(self) -> None:
self.t1.clear()
self.t2.clear()
self.b1.clear()
self.b2.clear()
self.target_t1_size = 0.0
def evict(
self, n: int, protected: set[OffloadKey]
) -> list[tuple[OffloadKey, BlockStatus]] | None:
+8
View File
@@ -74,3 +74,11 @@ class CachePolicy(ABC):
For ARC: ghost list cleanup (trimming to cache_capacity) is performed
at the end of a successful eviction.
"""
@abstractmethod
def clear(self) -> None:
"""
Remove ALL blocks regardless of ref_cnt.
Ghost lists and adaptive state are also reset.
"""
+3
View File
@@ -28,6 +28,9 @@ class LRUCachePolicy(CachePolicy):
if key in self.blocks:
self.blocks.move_to_end(key)
def clear(self) -> None:
self.blocks.clear()
def evict(
self, n: int, protected: set[OffloadKey]
) -> list[tuple[OffloadKey, BlockStatus]] | None: