[SimpleCPUOffloadConnector]: Add support for reset_cache() (#39726)
Signed-off-by: Jonathan Chen <chenleejonathan@gmail.com> Signed-off-by: Jonathan <chenleejonathan@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
ed938ad7db
commit
d57888efa4
@@ -1354,3 +1354,177 @@ def test_toctou_cpu_hit_evicted_between_phases_no_crash() -> None:
|
||||
)
|
||||
assert len(meta_b.load_gpu_blocks) == 2
|
||||
assert len(meta_b.load_cpu_blocks) == 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 12: Reset with pending eager stores waits for completion
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_reset_pending_eager_stores() -> None:
|
||||
"""Eager mode: reset() abandons in-flight stores until they complete."""
|
||||
fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False)
|
||||
sched = fix.scheduler
|
||||
gpu_pool = fix.gpu_block_pool
|
||||
|
||||
num_blocks = 2
|
||||
req = make_request(num_blocks=num_blocks)
|
||||
|
||||
kv_blocks = _alloc_and_register(fix, req, num_blocks)
|
||||
sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0)
|
||||
block_ids = kv_blocks.get_block_ids()
|
||||
sched_out = make_scheduler_output(
|
||||
{req.request_id: num_blocks * BLOCK_SIZE},
|
||||
new_reqs={req.request_id: block_ids},
|
||||
)
|
||||
|
||||
meta = sched.build_connector_meta(sched_out)
|
||||
assert meta.store_event >= 0
|
||||
assert len(sched._store_event_to_blocks) > 0
|
||||
|
||||
# GPU blocks should have elevated ref_cnt from touch()
|
||||
for bid in meta.store_gpu_blocks:
|
||||
assert gpu_pool.blocks[bid].ref_cnt > 0
|
||||
|
||||
# Free the request's own block refs (simulates preemption)
|
||||
gpu_pool.free_blocks(gpu_pool.blocks[bid] for bid in block_ids[0])
|
||||
|
||||
# Reset should keep DMA refs pinned until the worker reports completion.
|
||||
assert sched.reset() is False
|
||||
assert len(sched._store_event_to_blocks) == 0
|
||||
assert len(sched._abandoned_store_event_to_blocks) == 1
|
||||
assert len(sched._reqs_to_store) == 0
|
||||
assert len(sched._store_event_to_reqs) == 0
|
||||
|
||||
num_used = gpu_pool.num_gpu_blocks - gpu_pool.get_num_free_blocks()
|
||||
assert num_used > 1
|
||||
|
||||
simulate_store_completion(sched, meta.store_event)
|
||||
assert len(sched._abandoned_store_event_to_blocks) == 0
|
||||
|
||||
# All GPU blocks should now be free (ref_cnt == 0) except null block
|
||||
num_used = gpu_pool.num_gpu_blocks - gpu_pool.get_num_free_blocks()
|
||||
assert num_used == 1, f"Expected only null block in use, got {num_used}"
|
||||
|
||||
# GPU prefix cache reset should now succeed
|
||||
assert gpu_pool.reset_prefix_cache() is True
|
||||
assert sched.reset() is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 13: Reset with pending lazy stores waits for completion
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_reset_pending_lazy_stores() -> None:
|
||||
"""Lazy mode: reset() abandons in-flight stores until they complete."""
|
||||
fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=8, lazy=True)
|
||||
sched = fix.scheduler
|
||||
gpu_pool = fix.gpu_block_pool
|
||||
|
||||
num_blocks = 2
|
||||
req = make_request(num_blocks=num_blocks)
|
||||
|
||||
# Allocate, hash, and free — blocks move to free queue with hashes
|
||||
gpu_blocks = _allocate_gpu_blocks(gpu_pool, req, num_blocks, group_id=0)
|
||||
gpu_pool.free_blocks(gpu_blocks)
|
||||
|
||||
# Push hashed blocks to LRU head
|
||||
fillers = _flush_old_blocks_to_lru_head(gpu_pool, num_filler_blocks=5)
|
||||
|
||||
# Lazy scanner offloads old hashed blocks
|
||||
sched_out = make_scheduler_output({})
|
||||
meta = sched.build_connector_meta(sched_out)
|
||||
assert meta.store_event >= 0
|
||||
assert len(sched._store_event_to_blocks) > 0
|
||||
|
||||
gpu_pool.free_blocks(fillers)
|
||||
|
||||
# Reset should keep DMA refs pinned until the worker reports completion.
|
||||
assert sched.reset() is False
|
||||
assert len(sched._store_event_to_blocks) == 0
|
||||
assert len(sched._abandoned_store_event_to_blocks) == 1
|
||||
assert sched._cursor is None
|
||||
|
||||
simulate_store_completion(sched, meta.store_event)
|
||||
assert len(sched._abandoned_store_event_to_blocks) == 0
|
||||
assert sched.reset() is True
|
||||
|
||||
# No CPU cache hits after reset
|
||||
req2 = Request(
|
||||
request_id="req-after-lazy-reset",
|
||||
prompt_token_ids=req.prompt_token_ids,
|
||||
sampling_params=req.sampling_params,
|
||||
pooling_params=None,
|
||||
mm_features=None,
|
||||
block_hasher=req._block_hasher,
|
||||
)
|
||||
hit_tokens, _ = sched.get_num_new_matched_tokens(req2, num_computed_tokens=0)
|
||||
assert hit_tokens == 0, "CPU cache should be empty after reset"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 14: Reset with pending loads waits for completion
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_reset_pending_loads() -> None:
|
||||
"""reset() abandons in-flight loads until they complete."""
|
||||
fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False)
|
||||
sched = fix.scheduler
|
||||
gpu_pool = fix.gpu_block_pool
|
||||
|
||||
num_blocks = 2
|
||||
|
||||
# First store blocks to CPU
|
||||
req = make_request(num_blocks=num_blocks)
|
||||
kv_blocks = _alloc_and_register(fix, req, num_blocks)
|
||||
sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0)
|
||||
block_ids = kv_blocks.get_block_ids()
|
||||
sched_out = make_scheduler_output(
|
||||
{req.request_id: num_blocks * BLOCK_SIZE},
|
||||
new_reqs={req.request_id: block_ids},
|
||||
)
|
||||
meta = sched.build_connector_meta(sched_out)
|
||||
simulate_store_completion(sched, meta.store_event)
|
||||
|
||||
# Start a load — CPU cache hit
|
||||
req2 = Request(
|
||||
request_id="req-load-reset",
|
||||
prompt_token_ids=req.prompt_token_ids,
|
||||
sampling_params=req.sampling_params,
|
||||
pooling_params=None,
|
||||
mm_features=None,
|
||||
block_hasher=req._block_hasher,
|
||||
)
|
||||
hit_tokens, is_async = sched.get_num_new_matched_tokens(req2, num_computed_tokens=0)
|
||||
assert hit_tokens > 0
|
||||
|
||||
gpu_blocks2 = gpu_pool.get_new_blocks(num_blocks)
|
||||
kv_blocks2 = KVCacheBlocks(blocks=(gpu_blocks2,))
|
||||
sched.update_state_after_alloc(req2, kv_blocks2, num_external_tokens=hit_tokens)
|
||||
|
||||
block_ids2 = kv_blocks2.get_block_ids()
|
||||
sched_out2 = make_scheduler_output(
|
||||
{req2.request_id: 1},
|
||||
new_reqs={req2.request_id: block_ids2},
|
||||
)
|
||||
meta2 = sched.build_connector_meta(sched_out2)
|
||||
assert meta2.load_event >= 0
|
||||
assert req2.request_id in sched._reqs_to_load
|
||||
|
||||
# Free request block refs (simulates preemption)
|
||||
gpu_pool.free_blocks(gpu_pool.blocks[bid] for bid in block_ids[0])
|
||||
gpu_pool.free_blocks(gpu_pool.blocks[bid] for bid in block_ids2[0])
|
||||
|
||||
# Reset should keep load touch refs until the worker reports completion.
|
||||
assert sched.reset() is False
|
||||
assert len(sched._reqs_to_load) == 0
|
||||
assert len(sched._abandoned_reqs_to_load) == 1
|
||||
assert len(sched._load_event_to_reqs) == 1
|
||||
|
||||
num_used = gpu_pool.num_gpu_blocks - gpu_pool.get_num_free_blocks()
|
||||
assert num_used > 1
|
||||
|
||||
simulate_load_completion(sched, {req2.request_id})
|
||||
assert len(sched._abandoned_reqs_to_load) == 0
|
||||
assert len(sched._load_event_to_reqs) == 0
|
||||
assert sched.reset() is True
|
||||
|
||||
# All GPU blocks free
|
||||
num_used = gpu_pool.num_gpu_blocks - gpu_pool.get_num_free_blocks()
|
||||
assert num_used == 1, f"Expected only null block in use, got {num_used}"
|
||||
|
||||
@@ -245,10 +245,10 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
return self.scheduler_manager.take_events()
|
||||
return []
|
||||
|
||||
# NOTE: Workers are not contacted. In-flight transfers drain naturally,
|
||||
# and stale completions are ignored by the guarded
|
||||
# SimpleCPUOffloadScheduler._process_store_event().
|
||||
def reset_cache(self) -> bool | None:
|
||||
raise NotImplementedError(
|
||||
"SimpleCPUOffloadConnector does not support reset_cache(). "
|
||||
"reset_prefix_cache() requires synchronizing all pending "
|
||||
"CPU offload transfers before clearing GPU prefix cache blocks, "
|
||||
"which is not yet implemented."
|
||||
)
|
||||
if self.scheduler_manager is not None:
|
||||
return self.scheduler_manager.reset()
|
||||
return None
|
||||
|
||||
@@ -159,10 +159,12 @@ class SimpleCPUOffloadScheduler:
|
||||
else:
|
||||
self._target_free = 0
|
||||
self._store_event_to_blocks: dict[int, TransferMeta] = {}
|
||||
self._abandoned_store_event_to_blocks: dict[int, TransferMeta] = {}
|
||||
# Eager mode only
|
||||
self._reqs_to_store: dict[str, StoreRequestState] = {}
|
||||
self._store_event_to_reqs: dict[int, list[str]] = {}
|
||||
self._in_flight_store_gpu_blocks: set[int] = set()
|
||||
self._abandoned_reqs_to_load: dict[str, LoadRequestState] = {}
|
||||
|
||||
# Event counters
|
||||
self._load_event_counter: int = 0
|
||||
@@ -427,7 +429,10 @@ class SimpleCPUOffloadScheduler:
|
||||
load_event=load_event,
|
||||
load_gpu_blocks=load_gpu,
|
||||
load_cpu_blocks=load_cpu,
|
||||
load_event_to_reqs=self._load_event_to_reqs,
|
||||
load_event_to_reqs={
|
||||
event_idx: list(req_ids)
|
||||
for event_idx, req_ids in self._load_event_to_reqs.items()
|
||||
},
|
||||
store_event=store_event,
|
||||
store_gpu_blocks=store_gpu,
|
||||
store_cpu_blocks=store_cpu,
|
||||
@@ -680,9 +685,17 @@ class SimpleCPUOffloadScheduler:
|
||||
|
||||
def _process_store_event(self, event_idx: int) -> None:
|
||||
"""Process a fully-completed store event."""
|
||||
transfer = self._store_event_to_blocks.pop(event_idx)
|
||||
transfer = self._store_event_to_blocks.pop(event_idx, None)
|
||||
if transfer is None:
|
||||
transfer = self._abandoned_store_event_to_blocks.pop(event_idx, None)
|
||||
if transfer is None:
|
||||
return # guard stale events from before a reset() call
|
||||
self._release_transfer_refs(transfer)
|
||||
return
|
||||
|
||||
if not self._lazy_mode:
|
||||
self._in_flight_store_gpu_blocks.difference_update(transfer.gpu_block_ids)
|
||||
|
||||
self._process_store_completion(transfer.gpu_block_ids, transfer.cpu_block_ids)
|
||||
logger.debug(
|
||||
"Store event %d completed: cached %d blocks to CPU",
|
||||
@@ -725,9 +738,22 @@ class SimpleCPUOffloadScheduler:
|
||||
self._gpu_block_pool.blocks[bid] for bid in gpu_block_ids
|
||||
)
|
||||
|
||||
def _release_transfer_refs(self, transfer: TransferMeta) -> None:
|
||||
"""Release transfer refs without making copied data cacheable."""
|
||||
cpu_blocks = [self.cpu_block_pool.blocks[bid] for bid in transfer.cpu_block_ids]
|
||||
for cpu_block in cpu_blocks:
|
||||
cpu_block.reset_hash()
|
||||
self.cpu_block_pool.free_blocks(cpu_blocks)
|
||||
assert self._gpu_block_pool is not None
|
||||
self._gpu_block_pool.free_blocks(
|
||||
self._gpu_block_pool.blocks[bid] for bid in transfer.gpu_block_ids
|
||||
)
|
||||
|
||||
def has_pending_stores(self) -> bool:
|
||||
"""Return True if there are in-flight store transfers."""
|
||||
return bool(self._store_event_to_blocks)
|
||||
return bool(
|
||||
self._store_event_to_blocks or self._abandoned_store_event_to_blocks
|
||||
)
|
||||
|
||||
def request_finished(
|
||||
self,
|
||||
@@ -787,6 +813,8 @@ class SimpleCPUOffloadScheduler:
|
||||
and frees CPU/GPU touch refs.
|
||||
"""
|
||||
state = self._reqs_to_load.pop(req_id, None)
|
||||
if state is None:
|
||||
state = self._abandoned_reqs_to_load.pop(req_id, None)
|
||||
if state is None:
|
||||
return
|
||||
# Remove from load event mapping (only this req, not whole event)
|
||||
@@ -830,3 +858,43 @@ class SimpleCPUOffloadScheduler:
|
||||
|
||||
def take_events(self) -> Iterable[KVCacheEvent]:
|
||||
return self.cpu_block_pool.take_events()
|
||||
|
||||
def reset(self) -> bool:
|
||||
"""Abandon pending transfers and reset the CPU cache when safe.
|
||||
|
||||
Worker-side DMA may still be using blocks after reset is requested.
|
||||
Keep those block refs pinned until the existing completion path reports
|
||||
the transfer finished, then release refs without caching abandoned
|
||||
store results.
|
||||
"""
|
||||
|
||||
self._abandoned_store_event_to_blocks.update(self._store_event_to_blocks)
|
||||
self._store_event_to_blocks.clear()
|
||||
self._in_flight_store_gpu_blocks.clear()
|
||||
|
||||
# Loads that have not been sent to the worker cannot have running DMA.
|
||||
# In-flight loads stay pinned and are cleaned up on completion.
|
||||
for req_id in list(self._reqs_to_load):
|
||||
state = self._reqs_to_load.pop(req_id)
|
||||
if state.load_event is None:
|
||||
self._reqs_to_load[req_id] = state
|
||||
self._cleanup_load_request(req_id)
|
||||
else:
|
||||
self._abandoned_reqs_to_load[req_id] = state
|
||||
|
||||
self._reqs_to_store.clear()
|
||||
self._store_event_to_reqs.clear()
|
||||
self._store_event_pending_counts = {
|
||||
event_idx: count
|
||||
for event_idx, count in self._store_event_pending_counts.items()
|
||||
if event_idx in self._abandoned_store_event_to_blocks
|
||||
}
|
||||
self._cursor = None
|
||||
# NOTE: _load_event_counter / _store_event_counter are not
|
||||
# reset as they are monotonic and must stay ahead of the workers
|
||||
# high-water marks to avoid event index collisions
|
||||
|
||||
if self._abandoned_store_event_to_blocks or self._abandoned_reqs_to_load:
|
||||
return False
|
||||
|
||||
return self.cpu_block_pool.reset_prefix_cache()
|
||||
|
||||
Reference in New Issue
Block a user