From 12b9573c98c32932e83fdb22e22c152b248c8c88 Mon Sep 17 00:00:00 2001 From: maithilijoshi20 <97733343+maithilijoshi20@users.noreply.github.com> Date: Wed, 2 Sep 2026 12:23:09 +0900 Subject: [PATCH] [Bugfix][KV Offloading] Fix eager SimpleCPUOffload cache registration and final flush (#53532) Signed-off-by: maithilijoshi20 Signed-off-by: maithilijoshi20 <97733343+maithilijoshi20@users.noreply.github.com> Co-authored-by: maithilijoshi20 Co-authored-by: OpenAI Codex Co-authored-by: Yifan Qiao --- tests/v1/simple_kv_offload/test_scheduler.py | 254 +++++++++++++++++- .../v1/simple_cpu_offload_connector.py | 3 +- vllm/v1/simple_kv_offload/manager.py | 191 +++++++------ 3 files changed, 354 insertions(+), 94 deletions(-) diff --git a/tests/v1/simple_kv_offload/test_scheduler.py b/tests/v1/simple_kv_offload/test_scheduler.py index 714617a4f6..c3439a77e0 100644 --- a/tests/v1/simple_kv_offload/test_scheduler.py +++ b/tests/v1/simple_kv_offload/test_scheduler.py @@ -18,6 +18,9 @@ from vllm.config import ( SchedulerConfig, VllmConfig, ) +from vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector import ( + SimpleCPUOffloadConnector, +) from vllm.utils.hashing import sha256 from vllm.v1.core.block_pool import BlockPool from vllm.v1.core.kv_cache_manager import KVCacheBlocks @@ -39,12 +42,16 @@ from vllm.v1.kv_cache_interface import ( KVCacheConfig, KVCacheGroupSpec, KVCacheTensor, + SlidingWindowSpec, ) from vllm.v1.outputs import KVConnectorOutput from vllm.v1.request import Request from vllm.v1.simple_kv_offload.manager import SimpleCPUOffloadScheduler from vllm.v1.simple_kv_offload.metadata import SimpleCPUOffloadWorkerMetadata +pytestmark = pytest.mark.skip_global_cleanup + + # --------------------------------------------------------------------------- # Constants # --------------------------------------------------------------------------- @@ -77,15 +84,26 @@ def _make_kv_cache_config( ) # Ensure specs are registered for tests for g in range(num_groups): layer_names = [f"layer_{g}"] + spec = ( + FullAttentionSpec( + block_size=BLOCK_SIZE, + num_kv_heads=NUM_KV_HEADS, + head_size=HEAD_SIZE, + dtype=DTYPE, + ) + if g == 0 + else SlidingWindowSpec( + block_size=BLOCK_SIZE, + num_kv_heads=NUM_KV_HEADS, + head_size=HEAD_SIZE, + dtype=DTYPE, + sliding_window=BLOCK_SIZE * 4, + ) + ) groups.append( KVCacheGroupSpec( layer_names, - FullAttentionSpec( - block_size=BLOCK_SIZE, - num_kv_heads=NUM_KV_HEADS, - head_size=HEAD_SIZE, - dtype=DTYPE, - ), + spec, ) ) tensors.append( @@ -412,6 +430,230 @@ def test_eager_store_and_load_roundtrip() -> None: assert len(meta2.load_cpu_blocks) == len(meta2.load_gpu_blocks) +def test_eager_store_preserves_secondary_block_hashes() -> None: + """CPU copies retain fine-grained hashes owned by the GPU block.""" + fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False) + sched = fix.scheduler + req = make_request(num_blocks=1) + kv_blocks = _alloc_and_register(fix, req, num_blocks=1) + gpu_block = kv_blocks.blocks[0][0] + primary_hash = gpu_block.block_hash + primary_num_tokens = gpu_block.block_hash_num_tokens + assert primary_hash is not None + + fine_grained_req = Request( + request_id="req-fine-grained-hash", + prompt_token_ids=req.prompt_token_ids, + sampling_params=req.sampling_params, + pooling_params=None, + mm_features=None, + block_hasher=get_request_block_hasher(BLOCK_SIZE // 2, sha256), + ) + secondary_hash = make_block_hash_with_group_id(fine_grained_req.block_hashes[0], 0) + fix.gpu_block_pool._insert_block_hash( + secondary_hash, gpu_block, num_tokens=BLOCK_SIZE // 2 + ) + assert ( + secondary_hash + in fix.gpu_block_pool.cached_block_hashes_by_block[gpu_block.block_id] + ) + + sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) + block_ids = kv_blocks.get_block_ids() + meta = sched.build_connector_meta( + make_scheduler_output( + {req.request_id: BLOCK_SIZE}, + new_reqs={req.request_id: block_ids}, + ) + ) + simulate_store_completion(sched, meta.store_event) + + cpu_block = sched.cpu_block_pool.cached_block_hash_to_block.get_one_block( + secondary_hash + ) + assert cpu_block is not None + assert cpu_block.block_hash == primary_hash + assert cpu_block.block_hash_num_tokens == primary_num_tokens + assert ( + secondary_hash + in sched.cpu_block_pool.cached_block_hashes_by_block[cpu_block.block_id] + ) + + +# --------------------------------------------------------------------------- +# Test 1b: Finished eager stores +# --------------------------------------------------------------------------- +def test_finished_eager_stores_are_batched() -> None: + """Finished requests in one step retain, complete, and cache every store.""" + fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False) + sched = fix.scheduler + + requests = [make_request(num_blocks=2) for _ in range(2)] + block_ids = [] + for request in requests: + kv_blocks = _alloc_and_register(fix, request, num_blocks=2) + sched.update_state_after_alloc(request, kv_blocks, num_external_tokens=0) + block_ids.append(kv_blocks.get_block_ids()) + + for request, request_block_ids in zip(requests, block_ids): + sched.request_finished_all_groups(request, request_block_ids) + + meta = sched.build_connector_meta(make_scheduler_output({})) + assert len(meta.store_gpu_blocks) == 4 + assert len(meta.store_cpu_blocks) == 4 + simulate_store_completion(sched, meta.store_event) + + for request in requests: + for block_hash in request.block_hashes[:2]: + cached = sched.cpu_block_pool.cached_block_hash_to_block.get_one_block( + make_block_hash_with_group_id(block_hash, 0) + ) + assert cached is not None + + request = requests[0] + cached_request = Request( + request_id="req-finished-eager-load", + prompt_token_ids=request.prompt_token_ids, + sampling_params=request.sampling_params, + pooling_params=None, + mm_features=None, + block_hasher=request._block_hasher, + ) + hit_tokens, is_async = sched.get_num_new_matched_tokens( + cached_request, num_computed_tokens=0 + ) + assert hit_tokens == 2 * BLOCK_SIZE + assert is_async is True + + cached_gpu_blocks = fix.gpu_block_pool.get_new_blocks(2) + sched.update_state_after_alloc( + cached_request, + KVCacheBlocks(blocks=(cached_gpu_blocks,)), + num_external_tokens=hit_tokens, + ) + load_meta = sched.build_connector_meta( + make_scheduler_output( + {cached_request.request_id: 1}, + new_reqs={ + cached_request.request_id: ( + [block.block_id for block in cached_gpu_blocks], + ) + }, + ) + ) + assert load_meta.load_event >= 0 + assert len(load_meta.load_gpu_blocks) == 2 + assert len(load_meta.load_cpu_blocks) == 2 + + +def test_finished_eager_store_skips_existing_cpu_cache_entries() -> None: + """A final flush must not consume CPU capacity for an existing hash.""" + fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False) + sched = fix.scheduler + + first = make_request(num_blocks=2) + first_blocks = _alloc_and_register(fix, first, num_blocks=2) + sched.update_state_after_alloc(first, first_blocks, num_external_tokens=0) + sched.request_finished_all_groups(first, first_blocks.get_block_ids()) + first_meta = sched.build_connector_meta(make_scheduler_output({})) + simulate_store_completion(sched, first_meta.store_event) + free_blocks = get_cpu_free_blocks(sched) + + duplicate = Request( + request_id="req-finished-eager-duplicate", + prompt_token_ids=first.prompt_token_ids, + sampling_params=first.sampling_params, + pooling_params=None, + mm_features=None, + block_hasher=first._block_hasher, + ) + duplicate_blocks = _alloc_and_register(fix, duplicate, num_blocks=2) + sched.update_state_after_alloc(duplicate, duplicate_blocks, num_external_tokens=0) + sched.request_finished_all_groups(duplicate, duplicate_blocks.get_block_ids()) + + duplicate_meta = sched.build_connector_meta(make_scheduler_output({})) + assert duplicate_meta.store_event == -1 + assert get_cpu_free_blocks(sched) == free_blocks + + +def test_finished_eager_store_is_reported_pending_before_metadata_build() -> None: + """Queued finished stores must keep the connector pending until submitted.""" + fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False) + sched = fix.scheduler + request = make_request(num_blocks=2) + kv_blocks = _alloc_and_register(fix, request, num_blocks=2) + connector = SimpleCPUOffloadConnector.__new__(SimpleCPUOffloadConnector) + connector.scheduler_manager = sched + sched.update_state_after_alloc(request, kv_blocks, num_external_tokens=0) + + sched.request_finished_all_groups(request, kv_blocks.get_block_ids()) + + assert sched.has_pending_stores() + assert connector.has_pending_push_work() + meta = sched.build_connector_meta(make_scheduler_output({})) + assert meta.store_event >= 0 + assert not sched._pending_finished_stores + assert connector.has_pending_push_work() + simulate_store_completion(sched, meta.store_event) + assert not sched.has_pending_stores() + assert not connector.has_pending_push_work() + + +def test_finished_eager_store_caches_all_groups() -> None: + """The final flush retains completed blocks from every KV cache group.""" + fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, num_groups=2, lazy=False) + sched = fix.scheduler + request = make_request(num_blocks=2) + group_blocks = tuple( + _allocate_gpu_blocks(fix.gpu_block_pool, request, 2, group_id=group_id) + for group_id in range(2) + ) + request.num_computed_tokens = 2 * BLOCK_SIZE + kv_blocks = KVCacheBlocks(blocks=group_blocks) + sched.update_state_after_alloc(request, kv_blocks, num_external_tokens=0) + + sched.request_finished_all_groups(request, kv_blocks.get_block_ids()) + meta = sched.build_connector_meta(make_scheduler_output({})) + assert len(meta.store_gpu_blocks) == 4 + simulate_store_completion(sched, meta.store_event) + + for group_id in range(2): + for block_hash in request.block_hashes[:2]: + cached = sched.cpu_block_pool.cached_block_hash_to_block.get_one_block( + make_block_hash_with_group_id(block_hash, group_id) + ) + assert cached is not None + + +def test_finished_eager_store_respects_cpu_capacity() -> None: + """A finished request skips excess blocks when the CPU cache is full.""" + fix = make_scheduler(num_cpu_blocks=3, num_gpu_blocks=16, lazy=False) + sched = fix.scheduler + request = make_request(num_blocks=3) + kv_blocks = _alloc_and_register(fix, request, num_blocks=3) + sched.update_state_after_alloc(request, kv_blocks, num_external_tokens=0) + available = get_cpu_free_blocks(sched) + + sched.request_finished_all_groups(request, kv_blocks.get_block_ids()) + meta = sched.build_connector_meta(make_scheduler_output({})) + + assert len(meta.store_gpu_blocks) == available + assert len(meta.store_cpu_blocks) == available + + +def test_reset_releases_unsubmitted_finished_stores() -> None: + """Reset releases finished-store refs that were not sent to a worker.""" + fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False) + sched = fix.scheduler + request = make_request(num_blocks=2) + kv_blocks = _alloc_and_register(fix, request, num_blocks=2) + sched.update_state_after_alloc(request, kv_blocks, num_external_tokens=0) + + sched.request_finished_all_groups(request, kv_blocks.get_block_ids()) + + assert sched.reset() + + # --------------------------------------------------------------------------- # Test 1b: Boundary — max_hit_len cap drops the last full block when the # prompt is an exact multiple of BLOCK_SIZE. diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py index 90a421da0f..47f1e098e4 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py @@ -291,8 +291,7 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA): ) return False, None - # NOTE: New API only for SimpleCPUOffloadConnector. - def has_pending_transfers(self) -> bool: + def has_pending_push_work(self) -> bool: if self.scheduler_manager is not None: return self.scheduler_manager.has_pending_stores() return False diff --git a/vllm/v1/simple_kv_offload/manager.py b/vllm/v1/simple_kv_offload/manager.py index cc3a1fcc36..423aca7955 100644 --- a/vllm/v1/simple_kv_offload/manager.py +++ b/vllm/v1/simple_kv_offload/manager.py @@ -174,6 +174,7 @@ class SimpleCPUOffloadScheduler: self._reqs_to_store: dict[str, StoreRequestState] = {} self._store_event_to_reqs: dict[int, list[str]] = {} self._in_flight_store_gpu_blocks: set[int] = set() + self._pending_finished_stores: list[TransferMeta] = [] self._abandoned_reqs_to_load: dict[str, LoadRequestState] = {} # Event counters @@ -414,6 +415,11 @@ class SimpleCPUOffloadScheduler: # --- Stores --- store_event = -1 store_gpu, store_cpu, store_req_ids = self.prepare_store_specs(scheduler_output) + for transfer in self._pending_finished_stores: + store_gpu.extend(transfer.gpu_block_ids) + store_cpu.extend(transfer.cpu_block_ids) + self._pending_finished_stores.clear() + if store_gpu: store_event = self._store_event_counter self._store_event_counter += 1 @@ -492,7 +498,6 @@ class SimpleCPUOffloadScheduler: self._cursor = None gpu_ids: list[int] = [] - block_hashes: list[bytes] = [] last_visited = self._cursor for covered, node in enumerate(free_queue.iter_blocks_after(self._cursor)): @@ -508,16 +513,13 @@ class SimpleCPUOffloadScheduler: and cpu_pool.cached_block_hash_to_block.get_one_block(bhash) is None ): gpu_ids.append(node.block_id) - block_hashes.append(bhash) self._cursor = last_visited - # Batch-allocate CPU blocks and stamp hashes. + # Batch-allocate CPU blocks. if gpu_ids: cpu_blocks = cpu_pool.get_new_blocks(len(gpu_ids)) cpu_ids = [blk.block_id for blk in cpu_blocks] - for cpu_blk, bhash in zip(cpu_blocks, block_hashes): # type: ignore[assignment] - cpu_blk._block_hash = bhash # type: ignore[assignment] # Touch GPU blocks to prevent eviction during async copy. gpu_pool.touch([gpu_pool.blocks[bid] for bid in gpu_ids]) else: @@ -531,9 +533,8 @@ class SimpleCPUOffloadScheduler: """Identify newly computed blocks to offload from scheduler requests. Only considers blocks whose KV data has been **confirmed computed** by - the GPU. This means blocks from the current step are NOT stored until the - next step. If a request finishes in the same step as its last full block, - that block may be missed. (TODO: flush on finish.) + the GPU. Blocks from the current step are stored on a later step, or by + the finish-time flush if the request completes first. Returns: (gpu_block_ids, cpu_block_ids, req_ids) for the store event. @@ -547,7 +548,6 @@ class SimpleCPUOffloadScheduler: if gpu_block_pool is None: return [], [], [] cpu_block_pool = self.cpu_block_pool - num_free = cpu_block_pool.get_num_free_blocks() kv_cache_groups = self.cpu_kv_cache_config.kv_cache_groups num_groups = len(kv_cache_groups) # Dedup against blocks already scheduled. @@ -575,73 +575,15 @@ class SimpleCPUOffloadScheduler: if not block_ids_by_group: continue - # --- Phase 1: Scan blocks, classify as cached vs to-store --- - gpu_block_ids: list[int] = [] - block_hashes_to_store: list[bytes] = [] - advanced_per_group: list[int] = [0] * num_groups - out_of_space = False - # Confirmed tokens: KV data written and visible to all streams. - req = state.request - confirmed_tokens = req.num_computed_tokens - req.num_output_placeholders - # Cap to blocks with confirmed KV data. - aligned_tokens = confirmed_tokens // self.block_size * self.block_size + gpu_block_ids, advanced_per_group = self._select_eager_blocks_to_store( + state, block_ids_by_group + ) - for g in range(num_groups): - # FIXME (yifan): handle CPU cache eviction, where - # num_stored_blocks can be stale and omit evicted blocks in - # the middle of the request. - already_stored_g = state.num_stored_blocks[g] - group_gpu_ids = block_ids_by_group[g] - - g_block_size = ( - kv_cache_groups[g].kv_cache_spec.block_size * self.cp_world_size - ) - ready_blocks_g = aligned_tokens // g_block_size - scannable = group_gpu_ids[already_stored_g:ready_blocks_g] - - for gpu_block_id in scannable: - gpu_block = gpu_block_pool.blocks[gpu_block_id] - if gpu_block.is_null: - advanced_per_group[g] += 1 - continue - - bhash_with_group = gpu_block.block_hash - if bhash_with_group is None: - # Masked-out SWA position the coordinator chose not to - # hash; it can never serve a prefix-cache hit, so skip. - advanced_per_group[g] += 1 - continue - - # Skip if already scheduled for store or already cached in CPU. - if ( - gpu_block_id in in_flight - or cpu_block_pool.cached_block_hash_to_block.get_one_block( - bhash_with_group - ) - is not None - ): - advanced_per_group[g] += 1 - continue - - if num_free <= 0: - out_of_space = True - break - num_free -= 1 - - gpu_block_ids.append(gpu_block_id) - block_hashes_to_store.append(bhash_with_group) - advanced_per_group[g] += 1 - - if out_of_space: - break - - # --- Phase 2: Batch allocate CPU blocks and stamp hashes --- + # --- Phase 2: Batch allocate CPU blocks --- n_to_alloc = len(gpu_block_ids) if n_to_alloc > 0: cpu_blocks_alloc = cpu_block_pool.get_new_blocks(n_to_alloc) cpu_block_ids = [blk.block_id for blk in cpu_blocks_alloc] - for cpu_blk, bhash in zip(cpu_blocks_alloc, block_hashes_to_store): - cpu_blk._block_hash = bhash # type: ignore[assignment] else: cpu_block_ids = [] @@ -669,6 +611,53 @@ class SimpleCPUOffloadScheduler: return merged_gpu_block_ids, merged_cpu_block_ids, req_ids + def _select_eager_blocks_to_store( + self, + state: StoreRequestState, + block_ids_by_group: tuple[list[int], ...], + ) -> tuple[list[int], list[int]]: + """Return confirmed, uncached eager blocks and per-group cursor advances.""" + assert self._gpu_block_pool is not None + gpu_block_ids: list[int] = [] + advanced_per_group = [0] * len(self.cpu_kv_cache_config.kv_cache_groups) + confirmed_tokens = ( + state.request.num_computed_tokens - state.request.num_output_placeholders + ) + aligned_tokens = confirmed_tokens // self.block_size * self.block_size + num_free = self.cpu_block_pool.get_num_free_blocks() + + for g, group_gpu_ids in enumerate(block_ids_by_group): + if len(gpu_block_ids) >= num_free: + break + # FIXME (yifan): handle CPU cache eviction, where + # num_stored_blocks can be stale and omit evicted blocks in + # the middle of the request. + group_size = ( + self.cpu_kv_cache_config.kv_cache_groups[g].kv_cache_spec.block_size + * self.cp_world_size + ) + ready = min(len(group_gpu_ids), aligned_tokens // group_size) + for gpu_block_id in group_gpu_ids[state.num_stored_blocks[g] : ready]: + gpu_block = self._gpu_block_pool.blocks[gpu_block_id] + if gpu_block.is_null or gpu_block.block_hash is None: + advanced_per_group[g] += 1 + continue + if ( + gpu_block_id in self._in_flight_store_gpu_blocks + or self.cpu_block_pool.cached_block_hash_to_block.get_one_block( + gpu_block.block_hash + ) + is not None + ): + advanced_per_group[g] += 1 + continue + if len(gpu_block_ids) >= num_free: + break + gpu_block_ids.append(gpu_block_id) + advanced_per_group[g] += 1 + + return gpu_block_ids, advanced_per_group + def update_connector_output(self, connector_output: KVConnectorOutput) -> None: """Handle async transfer completions from worker. @@ -726,24 +715,31 @@ class SimpleCPUOffloadScheduler: def _process_store_completion( self, gpu_block_ids: list[int], cpu_block_ids: list[int] ) -> None: - """Cache CPU blocks per-group and release GPU refs. - - Block hashes were stamped on CPU blocks at allocation time (in - ``_prepare_*_store_specs``). Here we just register them in the - cache map so they become discoverable by the load path. - """ + """Register copied blocks in the CPU prefix cache and release refs.""" assert len(cpu_block_ids) == len(gpu_block_ids) cpu_blocks = [self.cpu_block_pool.blocks[bid] for bid in cpu_block_ids] - for cpu_block in cpu_blocks: - bhash = cpu_block.block_hash - assert bhash is not None - self.cpu_block_pool.cached_block_hash_to_block.insert(bhash, cpu_block) + assert self._gpu_block_pool is not None + for gpu_block_id, cpu_block in zip(gpu_block_ids, cpu_blocks): + gpu_block = self._gpu_block_pool.blocks[gpu_block_id] + primary_hash = gpu_block.block_hash + assert primary_hash is not None + self.cpu_block_pool._insert_block_hash( + primary_hash, + cpu_block, + num_tokens=gpu_block.block_hash_num_tokens, + ) + secondary_hashes = self._gpu_block_pool.cached_block_hashes_by_block.get( + gpu_block_id, () + ) + for block_hash in secondary_hashes: + self.cpu_block_pool._insert_block_hash( + block_hash, cpu_block, num_tokens=None + ) # Free CPU and GPU blocks' ref counts to turn them into prefix cache self.cpu_block_pool.free_blocks(cpu_blocks) - assert self._gpu_block_pool is not None self._gpu_block_pool.free_blocks( self._gpu_block_pool.blocks[bid] for bid in gpu_block_ids ) @@ -751,8 +747,6 @@ class SimpleCPUOffloadScheduler: def _release_transfer_refs(self, transfer: TransferMeta) -> None: """Release transfer refs without making copied data cacheable.""" cpu_blocks = [self.cpu_block_pool.blocks[bid] for bid in transfer.cpu_block_ids] - for cpu_block in cpu_blocks: - cpu_block.reset_hash() self.cpu_block_pool.free_blocks(cpu_blocks) assert self._gpu_block_pool is not None self._gpu_block_pool.free_blocks( @@ -760,9 +754,11 @@ class SimpleCPUOffloadScheduler: ) def has_pending_stores(self) -> bool: - """Return True if there are in-flight store transfers.""" + """Return True if a store transfer is queued or in flight.""" return bool( - self._store_event_to_blocks or self._abandoned_store_event_to_blocks + self._pending_finished_stores + or self._store_event_to_blocks + or self._abandoned_store_event_to_blocks ) def request_finished( @@ -804,8 +800,27 @@ class SimpleCPUOffloadScheduler: request: "Request", block_ids: tuple[list[int], ...], ) -> tuple[bool, dict[str, Any] | None]: + self._queue_finished_eager_store(request, block_ids) return self.request_finished(request, block_ids=[]) + def _queue_finished_eager_store( + self, request: "Request", block_ids: tuple[list[int], ...] + ) -> None: + """Queue confirmed eager blocks omitted after a request enters decode.""" + state = self._reqs_to_store.get(request.request_id) + gpu_pool = self._gpu_block_pool + if state is None or gpu_pool is None: + return + gpu_ids, _ = self._select_eager_blocks_to_store(state, block_ids) + if not gpu_ids: + return + cpu_blocks = self.cpu_block_pool.get_new_blocks(len(gpu_ids)) + self._pending_finished_stores.append( + TransferMeta(gpu_ids, [b.block_id for b in cpu_blocks]) + ) + self._in_flight_store_gpu_blocks.update(gpu_ids) + gpu_pool.touch([gpu_pool.blocks[bid] for bid in gpu_ids]) + def _free_pending_cpu_hit(self, pending: tuple) -> None: """Release the temporary CPU block pin taken in get_num_new_matched_tokens().""" cpu_hit_blocks, _ = pending @@ -879,6 +894,10 @@ class SimpleCPUOffloadScheduler: """ self._abandoned_store_event_to_blocks.update(self._store_event_to_blocks) + for transfer in self._pending_finished_stores: + self._release_transfer_refs(transfer) + self._pending_finished_stores.clear() + self._store_event_to_blocks.clear() self._in_flight_store_gpu_blocks.clear()