[SimpleCPUOffloadConnector]: Add support for reset_cache() (#39726)

Signed-off-by: Jonathan Chen <chenleejonathan@gmail.com>
Signed-off-by: Jonathan <chenleejonathan@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Jonathan Chen
2026-06-17 19:47:12 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent ed938ad7db
commit d57888efa4
3 changed files with 251 additions and 9 deletions
@@ -1354,3 +1354,177 @@ def test_toctou_cpu_hit_evicted_between_phases_no_crash() -> None:
)
assert len(meta_b.load_gpu_blocks) == 2
assert len(meta_b.load_cpu_blocks) == 2
# ---------------------------------------------------------------------------
# Test 12: Reset with pending eager stores waits for completion
# ---------------------------------------------------------------------------
def test_reset_pending_eager_stores() -> None:
"""Eager mode: reset() abandons in-flight stores until they complete."""
fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False)
sched = fix.scheduler
gpu_pool = fix.gpu_block_pool
num_blocks = 2
req = make_request(num_blocks=num_blocks)
kv_blocks = _alloc_and_register(fix, req, num_blocks)
sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0)
block_ids = kv_blocks.get_block_ids()
sched_out = make_scheduler_output(
{req.request_id: num_blocks * BLOCK_SIZE},
new_reqs={req.request_id: block_ids},
)
meta = sched.build_connector_meta(sched_out)
assert meta.store_event >= 0
assert len(sched._store_event_to_blocks) > 0
# GPU blocks should have elevated ref_cnt from touch()
for bid in meta.store_gpu_blocks:
assert gpu_pool.blocks[bid].ref_cnt > 0
# Free the request's own block refs (simulates preemption)
gpu_pool.free_blocks(gpu_pool.blocks[bid] for bid in block_ids[0])
# Reset should keep DMA refs pinned until the worker reports completion.
assert sched.reset() is False
assert len(sched._store_event_to_blocks) == 0
assert len(sched._abandoned_store_event_to_blocks) == 1
assert len(sched._reqs_to_store) == 0
assert len(sched._store_event_to_reqs) == 0
num_used = gpu_pool.num_gpu_blocks - gpu_pool.get_num_free_blocks()
assert num_used > 1
simulate_store_completion(sched, meta.store_event)
assert len(sched._abandoned_store_event_to_blocks) == 0
# All GPU blocks should now be free (ref_cnt == 0) except null block
num_used = gpu_pool.num_gpu_blocks - gpu_pool.get_num_free_blocks()
assert num_used == 1, f"Expected only null block in use, got {num_used}"
# GPU prefix cache reset should now succeed
assert gpu_pool.reset_prefix_cache() is True
assert sched.reset() is True
# ---------------------------------------------------------------------------
# Test 13: Reset with pending lazy stores waits for completion
# ---------------------------------------------------------------------------
def test_reset_pending_lazy_stores() -> None:
"""Lazy mode: reset() abandons in-flight stores until they complete."""
fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=8, lazy=True)
sched = fix.scheduler
gpu_pool = fix.gpu_block_pool
num_blocks = 2
req = make_request(num_blocks=num_blocks)
# Allocate, hash, and free — blocks move to free queue with hashes
gpu_blocks = _allocate_gpu_blocks(gpu_pool, req, num_blocks, group_id=0)
gpu_pool.free_blocks(gpu_blocks)
# Push hashed blocks to LRU head
fillers = _flush_old_blocks_to_lru_head(gpu_pool, num_filler_blocks=5)
# Lazy scanner offloads old hashed blocks
sched_out = make_scheduler_output({})
meta = sched.build_connector_meta(sched_out)
assert meta.store_event >= 0
assert len(sched._store_event_to_blocks) > 0
gpu_pool.free_blocks(fillers)
# Reset should keep DMA refs pinned until the worker reports completion.
assert sched.reset() is False
assert len(sched._store_event_to_blocks) == 0
assert len(sched._abandoned_store_event_to_blocks) == 1
assert sched._cursor is None
simulate_store_completion(sched, meta.store_event)
assert len(sched._abandoned_store_event_to_blocks) == 0
assert sched.reset() is True
# No CPU cache hits after reset
req2 = Request(
request_id="req-after-lazy-reset",
prompt_token_ids=req.prompt_token_ids,
sampling_params=req.sampling_params,
pooling_params=None,
mm_features=None,
block_hasher=req._block_hasher,
)
hit_tokens, _ = sched.get_num_new_matched_tokens(req2, num_computed_tokens=0)
assert hit_tokens == 0, "CPU cache should be empty after reset"
# ---------------------------------------------------------------------------
# Test 14: Reset with pending loads waits for completion
# ---------------------------------------------------------------------------
def test_reset_pending_loads() -> None:
"""reset() abandons in-flight loads until they complete."""
fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False)
sched = fix.scheduler
gpu_pool = fix.gpu_block_pool
num_blocks = 2
# First store blocks to CPU
req = make_request(num_blocks=num_blocks)
kv_blocks = _alloc_and_register(fix, req, num_blocks)
sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0)
block_ids = kv_blocks.get_block_ids()
sched_out = make_scheduler_output(
{req.request_id: num_blocks * BLOCK_SIZE},
new_reqs={req.request_id: block_ids},
)
meta = sched.build_connector_meta(sched_out)
simulate_store_completion(sched, meta.store_event)
# Start a load — CPU cache hit
req2 = Request(
request_id="req-load-reset",
prompt_token_ids=req.prompt_token_ids,
sampling_params=req.sampling_params,
pooling_params=None,
mm_features=None,
block_hasher=req._block_hasher,
)
hit_tokens, is_async = sched.get_num_new_matched_tokens(req2, num_computed_tokens=0)
assert hit_tokens > 0
gpu_blocks2 = gpu_pool.get_new_blocks(num_blocks)
kv_blocks2 = KVCacheBlocks(blocks=(gpu_blocks2,))
sched.update_state_after_alloc(req2, kv_blocks2, num_external_tokens=hit_tokens)
block_ids2 = kv_blocks2.get_block_ids()
sched_out2 = make_scheduler_output(
{req2.request_id: 1},
new_reqs={req2.request_id: block_ids2},
)
meta2 = sched.build_connector_meta(sched_out2)
assert meta2.load_event >= 0
assert req2.request_id in sched._reqs_to_load
# Free request block refs (simulates preemption)
gpu_pool.free_blocks(gpu_pool.blocks[bid] for bid in block_ids[0])
gpu_pool.free_blocks(gpu_pool.blocks[bid] for bid in block_ids2[0])
# Reset should keep load touch refs until the worker reports completion.
assert sched.reset() is False
assert len(sched._reqs_to_load) == 0
assert len(sched._abandoned_reqs_to_load) == 1
assert len(sched._load_event_to_reqs) == 1
num_used = gpu_pool.num_gpu_blocks - gpu_pool.get_num_free_blocks()
assert num_used > 1
simulate_load_completion(sched, {req2.request_id})
assert len(sched._abandoned_reqs_to_load) == 0
assert len(sched._load_event_to_reqs) == 0
assert sched.reset() is True
# All GPU blocks free
num_used = gpu_pool.num_gpu_blocks - gpu_pool.get_num_free_blocks()
assert num_used == 1, f"Expected only null block in use, got {num_used}"