[SimpleCPUOffloadConnector]: Add support for reset_cache() (#39726)

Signed-off-by: Jonathan Chen <chenleejonathan@gmail.com>
Signed-off-by: Jonathan <chenleejonathan@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Jonathan Chen
2026-06-17 19:47:12 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent ed938ad7db
commit d57888efa4
3 changed files with 251 additions and 9 deletions
@@ -1354,3 +1354,177 @@ def test_toctou_cpu_hit_evicted_between_phases_no_crash() -> None:
)
assert len(meta_b.load_gpu_blocks) == 2
assert len(meta_b.load_cpu_blocks) == 2
# ---------------------------------------------------------------------------
# Test 12: Reset with pending eager stores waits for completion
# ---------------------------------------------------------------------------
def test_reset_pending_eager_stores() -> None:
"""Eager mode: reset() abandons in-flight stores until they complete."""
fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False)
sched = fix.scheduler
gpu_pool = fix.gpu_block_pool
num_blocks = 2
req = make_request(num_blocks=num_blocks)
kv_blocks = _alloc_and_register(fix, req, num_blocks)
sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0)
block_ids = kv_blocks.get_block_ids()
sched_out = make_scheduler_output(
{req.request_id: num_blocks * BLOCK_SIZE},
new_reqs={req.request_id: block_ids},
)
meta = sched.build_connector_meta(sched_out)
assert meta.store_event >= 0
assert len(sched._store_event_to_blocks) > 0
# GPU blocks should have elevated ref_cnt from touch()
for bid in meta.store_gpu_blocks:
assert gpu_pool.blocks[bid].ref_cnt > 0
# Free the request's own block refs (simulates preemption)
gpu_pool.free_blocks(gpu_pool.blocks[bid] for bid in block_ids[0])
# Reset should keep DMA refs pinned until the worker reports completion.
assert sched.reset() is False
assert len(sched._store_event_to_blocks) == 0
assert len(sched._abandoned_store_event_to_blocks) == 1
assert len(sched._reqs_to_store) == 0
assert len(sched._store_event_to_reqs) == 0
num_used = gpu_pool.num_gpu_blocks - gpu_pool.get_num_free_blocks()
assert num_used > 1
simulate_store_completion(sched, meta.store_event)
assert len(sched._abandoned_store_event_to_blocks) == 0
# All GPU blocks should now be free (ref_cnt == 0) except null block
num_used = gpu_pool.num_gpu_blocks - gpu_pool.get_num_free_blocks()
assert num_used == 1, f"Expected only null block in use, got {num_used}"
# GPU prefix cache reset should now succeed
assert gpu_pool.reset_prefix_cache() is True
assert sched.reset() is True
# ---------------------------------------------------------------------------
# Test 13: Reset with pending lazy stores waits for completion
# ---------------------------------------------------------------------------
def test_reset_pending_lazy_stores() -> None:
"""Lazy mode: reset() abandons in-flight stores until they complete."""
fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=8, lazy=True)
sched = fix.scheduler
gpu_pool = fix.gpu_block_pool
num_blocks = 2
req = make_request(num_blocks=num_blocks)
# Allocate, hash, and free — blocks move to free queue with hashes
gpu_blocks = _allocate_gpu_blocks(gpu_pool, req, num_blocks, group_id=0)
gpu_pool.free_blocks(gpu_blocks)
# Push hashed blocks to LRU head
fillers = _flush_old_blocks_to_lru_head(gpu_pool, num_filler_blocks=5)
# Lazy scanner offloads old hashed blocks
sched_out = make_scheduler_output({})
meta = sched.build_connector_meta(sched_out)
assert meta.store_event >= 0
assert len(sched._store_event_to_blocks) > 0
gpu_pool.free_blocks(fillers)
# Reset should keep DMA refs pinned until the worker reports completion.
assert sched.reset() is False
assert len(sched._store_event_to_blocks) == 0
assert len(sched._abandoned_store_event_to_blocks) == 1
assert sched._cursor is None
simulate_store_completion(sched, meta.store_event)
assert len(sched._abandoned_store_event_to_blocks) == 0
assert sched.reset() is True
# No CPU cache hits after reset
req2 = Request(
request_id="req-after-lazy-reset",
prompt_token_ids=req.prompt_token_ids,
sampling_params=req.sampling_params,
pooling_params=None,
mm_features=None,
block_hasher=req._block_hasher,
)
hit_tokens, _ = sched.get_num_new_matched_tokens(req2, num_computed_tokens=0)
assert hit_tokens == 0, "CPU cache should be empty after reset"
# ---------------------------------------------------------------------------
# Test 14: Reset with pending loads waits for completion
# ---------------------------------------------------------------------------
def test_reset_pending_loads() -> None:
"""reset() abandons in-flight loads until they complete."""
fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False)
sched = fix.scheduler
gpu_pool = fix.gpu_block_pool
num_blocks = 2
# First store blocks to CPU
req = make_request(num_blocks=num_blocks)
kv_blocks = _alloc_and_register(fix, req, num_blocks)
sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0)
block_ids = kv_blocks.get_block_ids()
sched_out = make_scheduler_output(
{req.request_id: num_blocks * BLOCK_SIZE},
new_reqs={req.request_id: block_ids},
)
meta = sched.build_connector_meta(sched_out)
simulate_store_completion(sched, meta.store_event)
# Start a load — CPU cache hit
req2 = Request(
request_id="req-load-reset",
prompt_token_ids=req.prompt_token_ids,
sampling_params=req.sampling_params,
pooling_params=None,
mm_features=None,
block_hasher=req._block_hasher,
)
hit_tokens, is_async = sched.get_num_new_matched_tokens(req2, num_computed_tokens=0)
assert hit_tokens > 0
gpu_blocks2 = gpu_pool.get_new_blocks(num_blocks)
kv_blocks2 = KVCacheBlocks(blocks=(gpu_blocks2,))
sched.update_state_after_alloc(req2, kv_blocks2, num_external_tokens=hit_tokens)
block_ids2 = kv_blocks2.get_block_ids()
sched_out2 = make_scheduler_output(
{req2.request_id: 1},
new_reqs={req2.request_id: block_ids2},
)
meta2 = sched.build_connector_meta(sched_out2)
assert meta2.load_event >= 0
assert req2.request_id in sched._reqs_to_load
# Free request block refs (simulates preemption)
gpu_pool.free_blocks(gpu_pool.blocks[bid] for bid in block_ids[0])
gpu_pool.free_blocks(gpu_pool.blocks[bid] for bid in block_ids2[0])
# Reset should keep load touch refs until the worker reports completion.
assert sched.reset() is False
assert len(sched._reqs_to_load) == 0
assert len(sched._abandoned_reqs_to_load) == 1
assert len(sched._load_event_to_reqs) == 1
num_used = gpu_pool.num_gpu_blocks - gpu_pool.get_num_free_blocks()
assert num_used > 1
simulate_load_completion(sched, {req2.request_id})
assert len(sched._abandoned_reqs_to_load) == 0
assert len(sched._load_event_to_reqs) == 0
assert sched.reset() is True
# All GPU blocks free
num_used = gpu_pool.num_gpu_blocks - gpu_pool.get_num_free_blocks()
assert num_used == 1, f"Expected only null block in use, got {num_used}"
@@ -245,10 +245,10 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
return self.scheduler_manager.take_events()
return []
# NOTE: Workers are not contacted. In-flight transfers drain naturally,
# and stale completions are ignored by the guarded
# SimpleCPUOffloadScheduler._process_store_event().
def reset_cache(self) -> bool | None:
raise NotImplementedError(
"SimpleCPUOffloadConnector does not support reset_cache(). "
"reset_prefix_cache() requires synchronizing all pending "
"CPU offload transfers before clearing GPU prefix cache blocks, "
"which is not yet implemented."
)
if self.scheduler_manager is not None:
return self.scheduler_manager.reset()
return None
+71 -3
View File
@@ -159,10 +159,12 @@ class SimpleCPUOffloadScheduler:
else:
self._target_free = 0
self._store_event_to_blocks: dict[int, TransferMeta] = {}
self._abandoned_store_event_to_blocks: dict[int, TransferMeta] = {}
# Eager mode only
self._reqs_to_store: dict[str, StoreRequestState] = {}
self._store_event_to_reqs: dict[int, list[str]] = {}
self._in_flight_store_gpu_blocks: set[int] = set()
self._abandoned_reqs_to_load: dict[str, LoadRequestState] = {}
# Event counters
self._load_event_counter: int = 0
@@ -427,7 +429,10 @@ class SimpleCPUOffloadScheduler:
load_event=load_event,
load_gpu_blocks=load_gpu,
load_cpu_blocks=load_cpu,
load_event_to_reqs=self._load_event_to_reqs,
load_event_to_reqs={
event_idx: list(req_ids)
for event_idx, req_ids in self._load_event_to_reqs.items()
},
store_event=store_event,
store_gpu_blocks=store_gpu,
store_cpu_blocks=store_cpu,
@@ -680,9 +685,17 @@ class SimpleCPUOffloadScheduler:
def _process_store_event(self, event_idx: int) -> None:
"""Process a fully-completed store event."""
transfer = self._store_event_to_blocks.pop(event_idx)
transfer = self._store_event_to_blocks.pop(event_idx, None)
if transfer is None:
transfer = self._abandoned_store_event_to_blocks.pop(event_idx, None)
if transfer is None:
return # guard stale events from before a reset() call
self._release_transfer_refs(transfer)
return
if not self._lazy_mode:
self._in_flight_store_gpu_blocks.difference_update(transfer.gpu_block_ids)
self._process_store_completion(transfer.gpu_block_ids, transfer.cpu_block_ids)
logger.debug(
"Store event %d completed: cached %d blocks to CPU",
@@ -725,9 +738,22 @@ class SimpleCPUOffloadScheduler:
self._gpu_block_pool.blocks[bid] for bid in gpu_block_ids
)
def _release_transfer_refs(self, transfer: TransferMeta) -> None:
"""Release transfer refs without making copied data cacheable."""
cpu_blocks = [self.cpu_block_pool.blocks[bid] for bid in transfer.cpu_block_ids]
for cpu_block in cpu_blocks:
cpu_block.reset_hash()
self.cpu_block_pool.free_blocks(cpu_blocks)
assert self._gpu_block_pool is not None
self._gpu_block_pool.free_blocks(
self._gpu_block_pool.blocks[bid] for bid in transfer.gpu_block_ids
)
def has_pending_stores(self) -> bool:
"""Return True if there are in-flight store transfers."""
return bool(self._store_event_to_blocks)
return bool(
self._store_event_to_blocks or self._abandoned_store_event_to_blocks
)
def request_finished(
self,
@@ -787,6 +813,8 @@ class SimpleCPUOffloadScheduler:
and frees CPU/GPU touch refs.
"""
state = self._reqs_to_load.pop(req_id, None)
if state is None:
state = self._abandoned_reqs_to_load.pop(req_id, None)
if state is None:
return
# Remove from load event mapping (only this req, not whole event)
@@ -830,3 +858,43 @@ class SimpleCPUOffloadScheduler:
def take_events(self) -> Iterable[KVCacheEvent]:
return self.cpu_block_pool.take_events()
def reset(self) -> bool:
"""Abandon pending transfers and reset the CPU cache when safe.
Worker-side DMA may still be using blocks after reset is requested.
Keep those block refs pinned until the existing completion path reports
the transfer finished, then release refs without caching abandoned
store results.
"""
self._abandoned_store_event_to_blocks.update(self._store_event_to_blocks)
self._store_event_to_blocks.clear()
self._in_flight_store_gpu_blocks.clear()
# Loads that have not been sent to the worker cannot have running DMA.
# In-flight loads stay pinned and are cleaned up on completion.
for req_id in list(self._reqs_to_load):
state = self._reqs_to_load.pop(req_id)
if state.load_event is None:
self._reqs_to_load[req_id] = state
self._cleanup_load_request(req_id)
else:
self._abandoned_reqs_to_load[req_id] = state
self._reqs_to_store.clear()
self._store_event_to_reqs.clear()
self._store_event_pending_counts = {
event_idx: count
for event_idx, count in self._store_event_pending_counts.items()
if event_idx in self._abandoned_store_event_to_blocks
}
self._cursor = None
# NOTE: _load_event_counter / _store_event_counter are not
# reset as they are monotonic and must stay ahead of the workers
# high-water marks to avoid event index collisions
if self._abandoned_store_event_to_blocks or self._abandoned_reqs_to_load:
return False
return self.cpu_block_pool.reset_prefix_cache()