[KV Offloading] Implement reset_cache for TieringOffloadingManager (#44541)

Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Ronen Schaffer
2026-06-15 18:42:53 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 0a1c5034f5
commit 7e612a0f06
10 changed files with 255 additions and 22 deletions
@@ -111,7 +111,7 @@ class MockSubscriber:
self.sub.close()
def _wait_for_prefix_cache_reset(llm: LLM, reset_connector: bool = False) -> None:
def _wait_for_prefix_cache_reset(llm: LLM) -> None:
"""Wait for async offload transfers to finish so prefix cache can reset.
The GPU-to-CPU offload runs on a CUDA stream asynchronously. While blocks
@@ -119,14 +119,10 @@ def _wait_for_prefix_cache_reset(llm: LLM, reset_connector: bool = False) -> Non
``False``. Between retries we send a dummy single-token prefill to force
the engine to step, which polls the worker for completed transfers and
frees GPU blocks.
Args:
llm: The LLM instance to reset.
reset_connector: If True, also reset the KV connector state.
"""
_dummy_params = SamplingParams(max_tokens=1)
deadline = time.monotonic() + _RESET_CACHE_TIMEOUT
while not llm.reset_prefix_cache(reset_connector=reset_connector):
while not llm.reset_prefix_cache():
if time.monotonic() > deadline:
raise TimeoutError(
"reset_prefix_cache did not succeed within "
@@ -141,9 +137,7 @@ def _wait_for_prefix_cache_reset(llm: LLM, reset_connector: bool = False) -> Non
)
def _latency_test(
llm: LLM, subscriber: MockSubscriber | None, reset_connector: bool = False
):
def _latency_test(llm: LLM, subscriber: MockSubscriber | None):
sampling_params = SamplingParams(max_tokens=1)
num_times_cpu_better_than_cold = 0
@@ -173,7 +167,7 @@ def _latency_test(
# Wait for the async CPU offload to finish, then reset prefix cache
# so the next generate() must reload from CPU rather than GPU.
_wait_for_prefix_cache_reset(llm, reset_connector=reset_connector)
_wait_for_prefix_cache_reset(llm)
# Verify CPU stored events arrived (offload is done before we
# attempt to load from CPU).
@@ -549,7 +543,7 @@ def test_fs_tiering_offloading(tmp_path) -> None:
topic=kv_events_config.topic,
)
try:
_latency_test(llm, subscriber, reset_connector=True)
_latency_test(llm, subscriber)
_accuracy_test(llm, subscriber)
finally:
subscriber.close()
@@ -10,6 +10,7 @@ data integrity throughout the process.
import mmap
import os
import threading
import time
from unittest.mock import MagicMock
@@ -22,6 +23,7 @@ from vllm.v1.kv_offload.tiering.base import JobMetadata
from vllm.v1.kv_offload.tiering.fs.manager import (
FileSystemTierManager,
)
from vllm.v1.kv_offload.tiering.fs.thread_pool import DualQueueThreadPool
# ---------------------------------------------------------------------------
# Helpers
@@ -296,3 +298,23 @@ def test_store_load_data_integrity(fs_tier):
assert torch.allclose(tensor[bid], expected[i]), (
f"Block {bid} data mismatch after store+load"
)
def test_wait_idle_blocks_until_tasks_complete():
"""wait_idle must not return while a task is still in flight."""
pool = DualQueueThreadPool(n_read_threads=1, n_write_threads=1)
gate = threading.Event()
pool.enqueue_store(job_id=1, n_tasks=1, tasks=[lambda: gate.wait(timeout=5.0)])
waiter = threading.Thread(target=pool.wait_idle)
waiter.start()
try:
waiter.join(timeout=0.2)
assert waiter.is_alive(), "wait_idle returned before task completed"
gate.set()
waiter.join(timeout=5.0)
assert not waiter.is_alive(), "wait_idle did not unblock"
finally:
gate.set()
pool.shutdown(wait=True)
waiter.join(timeout=5.0)
@@ -290,6 +290,33 @@ class TestMockObjTierBasic:
assert len(results) == 1
assert results[0].success
def test_drain_jobs_polls_until_transfers_complete(self):
"""drain_jobs must keep polling check_xfer_state until every
in-flight transfer finishes. A buggy implementation that only
polled once would return with _transfers still populated.
"""
call_count = [0]
original = self.agent.check_xfer_state
def delayed(h):
call_count[0] += 1
# Stay in PROC for the first 2 polls, then DONE.
return "PROC" if call_count[0] < 3 else original(h)
self.agent.check_xfer_state = delayed
self.tier.submit_store(make_job(1, [key(1)], [0]))
assert self.tier._transfers # in flight
self.tier.drain_jobs()
assert not self.tier._transfers # fully drained
assert call_count[0] >= 3 # polled past the initial PROC responses
# Result is buffered for the next get_finished_jobs() call.
results = list(self.tier.get_finished_jobs())
assert len(results) == 1
assert results[0].success
class TestMockObjTierMultiBlock:
def test_store_multiple_blocks(self):
@@ -490,6 +490,85 @@ class TestTieringOffloadingManager:
# tier2 (block-level) does not get existing blocks here.
self.secondary_tier2.submit_store.assert_not_called()
def test_reset_cache_clears_all_state(self, manager_setup):
"""reset_cache wipes every kind of orchestrator state and resets
primary tier; pending submissions are dropped without being sent
to the secondary tier."""
# Cascade — populates primary blocks and leaves cascade jobs
# in _transfer_jobs (the synchronous example tier has already
# queued completions); reset_cache's drain loop will pick them up.
blocks = to_keys(range(3))
self.manager.prepare_store(blocks, _CTX)
self.manager.complete_store(blocks, _CTX, success=True)
assert self.manager._transfer_jobs
# Pending promotion submission (deferred — no on_schedule_end after
# the lookup that staged it).
promo_block = to_keys([99])[0]
self.secondary_tier1.blocks[promo_block] = True
assert self.manager.lookup(promo_block, ReqContext(req_id="pending")) is None
assert self.manager._pending_load_submissions
# Request-level tier registration.
self.secondary_tier1.on_new_request = (
lambda req_context: RequestOffloadingContext(
policy=OffloadPolicy.REQUEST_LEVEL
)
)
self.manager.on_new_request(ReqContext(req_id="rl"))
assert self.manager._request_level_tiers
# Mark this step as already polled (reset_cache must clear it).
self.manager._processed_jobs_this_step = True
# Spy: pending submission must NOT reach the tier.
self.secondary_tier1.submit_load = MagicMock(
wraps=self.secondary_tier1.submit_load
)
self.manager.reset_cache()
# Orchestrator state cleared.
assert self.manager._transfer_jobs == {}
assert self.manager._pending_load_submissions == {}
assert self.manager._request_level_tiers == {}
assert self.manager._processed_jobs_this_step is False
# Primary tier reset to a fresh state.
assert self.primary_tier._num_allocated_blocks == 0
assert self.primary_tier._free_list == []
for block in blocks:
assert self.primary_tier.lookup(block, _CTX) is False
# Pending submission was dropped, not submitted.
self.secondary_tier1.submit_load.assert_not_called()
def test_reset_cache_drains_all_tiers(self, manager_setup):
"""reset_cache must drain each secondary tier before resetting
the primary tier so no tier I/O is touching primary memory.
Without the drain, an in-flight transfer could write into, or
read junk from, a primary slot that the post-reset path has
reallocated.
"""
self.secondary_tier1.drain_jobs = MagicMock(
wraps=self.secondary_tier1.drain_jobs
)
self.secondary_tier2.drain_jobs = MagicMock(
wraps=self.secondary_tier2.drain_jobs
)
# Drive a cascade so a job lands in _transfer_jobs.
blocks = to_keys(range(3))
self.manager.prepare_store(blocks, _CTX)
self.manager.complete_store(blocks, _CTX, success=True)
assert self.manager._transfer_jobs
self.manager.reset_cache()
self.secondary_tier1.drain_jobs.assert_called_once()
self.secondary_tier2.drain_jobs.assert_called_once()
assert self.manager._transfer_jobs == {}
class TestTieringOffloadingWithoutSecondaryTiers:
"""Test TieringOffloadingManager with no secondary tiers (backward compat)."""