From 67e5b0acc9988afc50d019db64ad9da0dadaa15e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Lucchesi?= Date: Thu, 17 Sep 2026 21:26:10 +0200 Subject: [PATCH] [Bugfix][HiSparse][NIXL] Import full blocks without tail prefill on D (#57049) Signed-off-by: NickLucche Signed-off-by: Lucas Wilkinson Co-authored-by: OpenAI Codex Co-authored-by: Lucas Wilkinson --- tests/v1/core/test_prefix_caching.py | 104 ++++++++++++++++-- tests/v1/worker/test_utils.py | 65 +++++++++++ .../kv_connector/v1/hisparse/worker.py | 55 +++++++-- .../kv_connector/v1/nixl/base_worker.py | 3 +- .../kv_connector/v1/nixl/pull_worker.py | 10 +- vllm/v1/core/single_type_kv_cache_manager.py | 20 ++-- vllm/v1/hisparse/coordinator.py | 68 ++++++++---- vllm/v1/hisparse/types.py | 5 +- 8 files changed, 277 insertions(+), 53 deletions(-) diff --git a/tests/v1/core/test_prefix_caching.py b/tests/v1/core/test_prefix_caching.py index 01bf73b6e7..229f9c17f9 100644 --- a/tests/v1/core/test_prefix_caching.py +++ b/tests/v1/core/test_prefix_caching.py @@ -325,7 +325,7 @@ def test_hisparse_reports_when_context_is_fully_resident(): def test_hisparse_host_prefix_can_be_completed_by_indexer_offload(): - """Keep indexer-only imports host-backed in the resident block table.""" + """Restore the final host page when completing an indexer-only import.""" manager = make_hisparse_kv_cache_manager( 32, 16, @@ -360,11 +360,15 @@ def test_hisparse_host_prefix_can_be_completed_by_indexer_offload(): source, indexer, resident, hot = manager.get_blocks(resumed.request_id).blocks assert len(source) == len(indexer) == len(resident) == 4 assert len(hot) == 2 - # The local prefix is adopted from shadow pages (GPU-resident), while the - # externally imported page stays host-backed until its tail allocation. assert not any(block.is_null for block in resident[:2]) - assert resident[2].is_null + assert not resident[2].is_null assert not resident[3].is_null + coordinator = get_hisparse_coordinator(manager) + assert not coordinator.build_offload_command().page_transfers + coordinator.finish_host_import(resumed.request_id, failed=False) + transfers = coordinator.build_offload_command().page_transfers + assert len(transfers) == 1 and transfers[0].restore + assert transfers[0].resident_block_ids == (resident[2].block_id,) def test_hisparse_indexer_offload_is_capped_by_missing_host_prefix(): @@ -465,7 +469,8 @@ def test_connector_completes_partial_prefix_without_importing_missing_host_kv( ) allocated = manager.get_blocks(resumed.request_id).blocks assert len(allocated[0]) == host_blocks - assert allocated[2][1].is_null == (external > 0) + assert allocated[2][1].is_null == (host_blocks > 2) + assert not allocated[2][-1].is_null manager.free(resumed) # A surviving indexer offload must not make absent host KV look computed. @@ -490,6 +495,85 @@ def allocate_external_prefix( ) +@pytest.mark.parametrize("num_tokens", [1, 15, 16, 17, 31, 32, 33, 127]) +def test_nixl_hisparse_full_block_import_keeps_a_writable_tail(num_tokens): + """Import every real token and restore the page used by last-token replay.""" + from tests.v1.kv_connector.unit.utils import create_vllm_config + from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import ( + NixlPullConnectorScheduler, + ) + + manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True) + config = create_vllm_config(block_size=HISPARSE_BLOCK_SIZE) + connector = NixlPullConnectorScheduler(config, "test", manager.kv_cache_config) + request = make_request( + "import", list(range(num_tokens)), HISPARSE_BLOCK_SIZE, sha256 + ) + request.kv_transfer_params = {"do_remote_prefill": True} + count, is_async = connector.get_num_new_matched_tokens(request, 0) + assert count == num_tokens and is_async + assert allocate_external_prefix(manager, request, count) is not None + source, _, resident, _ = manager.get_blocks(request.request_id).blocks + assert len(source) == len(resident) == (num_tokens + 15) // 16 + assert all(block.is_null for block in resident[:-1]) + assert not resident[-1].is_null + coordinator = get_hisparse_coordinator(manager) + assert not coordinator.build_offload_command().page_transfers + + coordinator.finish_host_import(request.request_id, failed=False) + transfers = coordinator.build_offload_command().page_transfers + assert len(transfers) == 1 + restore = transfers[0] + assert restore.restore and not restore.after_forward + assert restore.host_block_id == source[-1].block_id + assert restore.resident_block_ids == (resident[-1].block_id,) + request.num_computed_tokens = count + scheduler = SimpleNamespace( + connector=connector, + kv_cache_manager=manager, + failed_recving_kv_req_ids=set(), + finished_recving_kv_req_ids={request.request_id}, + ) + Scheduler._update_waiting_for_remote_kv(scheduler, request) + assert request.num_tokens - request.num_computed_tokens == 1 + assert manager.allocate_slots(request, num_new_tokens=1) is not None + assert coordinator.build_row_mirrors([(request.request_id, num_tokens - 1, 1)]) + # A pending restore must not expose uninitialized GPU copies to prefix hits. + assert all(resident[-1] not in copies for copies in coordinator.copies.values()) + + coordinator.update_spills({restore.transfer_id: 2}, {restore.transfer_id: 1}) + assert resident[-1].ref_cnt == 2 + assert all(resident[-1] not in copies for copies in coordinator.copies.values()) + coordinator.update_spills({}, {restore.transfer_id: 1}) + assert resident[-1].ref_cnt == 1 + assert coordinator.request_states[request.request_id].valid_pages == set( + range(num_tokens // HISPARSE_BLOCK_SIZE) + ) + if num_tokens % HISPARSE_BLOCK_SIZE: + assert source[-1].block_hash is None + else: + assert coordinator.copies[source[-1].block_hash] == (resident[-1],) + + +def test_hisparse_aborted_tail_restore_retains_both_endpoints(): + """An abort cannot recycle storage still used by a queued tail restore.""" + manager = make_hisparse_kv_cache_manager(16, 16) + request = make_request("import", list(range(17)), HISPARSE_BLOCK_SIZE, sha256) + assert allocate_external_prefix(manager, request, 17) is not None + source, _, resident, _ = manager.get_blocks(request.request_id).blocks + host_tail, gpu_tail = source[-1], resident[-1] + coordinator = get_hisparse_coordinator(manager) + coordinator.finish_host_import(request.request_id, failed=False) + restore = coordinator.build_offload_command().page_transfers[0] + manager.free(request) + assert host_tail.ref_cnt == gpu_tail.ref_cnt == 1 + coordinator.update_spills({restore.transfer_id: 2}, {restore.transfer_id: 1}) + assert host_tail.ref_cnt == gpu_tail.ref_cnt == 1 + coordinator.update_spills({}, {restore.transfer_id: 1}) + assert host_tail.ref_cnt == gpu_tail.ref_cnt == 0 + assert not coordinator.has_pending_work() + + @pytest.mark.parametrize("enable_caching", [False, True]) def test_hisparse_keeps_resident_pages_until_hot_buffer_is_allocated(enable_caching): """Later admissions must not remove a scheduled request's only readable tier.""" @@ -669,7 +753,7 @@ def test_hisparse_host_cow_copy_is_drained_without_a_gpu_pool(): def test_hisparse_inflight_host_import_reserves_remaining_gpu_pages(): """An in-flight host import must reserve its unwritten resident pages.""" manager = make_hisparse_kv_cache_manager( - 11, + 12, 16, transfer_device_cache=True, ) @@ -1127,9 +1211,10 @@ def test_hisparse_external_import_uses_hard_gpu_footprint(): assert len(source) == num_prompt_blocks assert len(indexer) == num_prompt_blocks assert len(resident) == num_prompt_blocks - assert all(block.is_null for block in resident) + assert all(block.is_null for block in resident[:-1]) + assert not resident[-1].is_null assert len(hot) == 2 - assert manager.block_pool.get_num_free_blocks() == 1 + assert manager.block_pool.get_num_free_blocks() == 0 def test_hisparse_external_import_survives_capacity_retry(): @@ -1149,7 +1234,8 @@ def test_hisparse_external_import_survives_capacity_retry(): manager.free(first) assert allocate_external_prefix(manager, second, len(tokens)) is not None _, _, resident, hot = manager.get_blocks(second.request_id).blocks - assert all(block.is_null for block in resident) + assert all(block.is_null for block in resident[:-1]) + assert not resident[-1].is_null assert len(hot) == 2 diff --git a/tests/v1/worker/test_utils.py b/tests/v1/worker/test_utils.py index 55cab287fd..9a345e926c 100644 --- a/tests/v1/worker/test_utils.py +++ b/tests/v1/worker/test_utils.py @@ -531,6 +531,71 @@ def test_hisparse_eager_mirror_records_transfer_without_page_copy(): worker._enqueue_transfers.assert_not_called() +@pytest.mark.parametrize("is_host_writer", [False, True]) +@pytest.mark.parametrize("device", ["cpu", "cuda"]) +def test_hisparse_tail_restore_preserves_imported_rows( + monkeypatch, request, is_host_writer, device +): + """Restore externally pinned host rows on every rank under sync checking.""" + if device == "cuda" and not torch.cuda.is_available(): + pytest.skip("CUDA is required for the device copy check") + worker = _make_hisparse_worker() + worker.is_host_writer = is_host_writer + worker.kernel_block_size = 4 + worker.host_caches = tuple( + torch.arange(24, dtype=torch.float32).reshape(12, 2) + layer * 100 + for layer in range(2) + ) + if device == "cuda": + # Match production's cudaHostRegister pool, which torch.is_pinned() + # does not recognize, rather than torch's caching pinned allocator. + host_pool, registered = hisparse_runtime_module.allocate_pinned_host_pool( + sum(cache.nbytes for cache in worker.host_caches) + ) + request.addfinalizer( + lambda: hisparse_runtime_module.release_pinned_state([], [registered]) + ) + host_caches = host_pool.view(torch.float32).view(2, 12, 2).unbind() + for destination, source in zip(host_caches, worker.host_caches): + destination.copy_(source) + worker.host_caches = host_caches + worker.resident_caches = tuple( + torch.full((4, 4, 2), -1.0, device=device) for _ in range(2) + ) + worker.cache_handles = [ + SimpleNamespace( + runtime=SimpleNamespace(resident_source_index=i, eager_host_mirror=True) + ) + for i in range(2) + ] + worker._record_transfer_completion = MagicMock() + stream = torch.cuda.current_stream() if device == "cuda" else MagicMock() + monkeypatch.setattr(hisparse_worker_module, "current_stream", lambda: stream) + transfer = SparseKVPageTransfer(7, 2, (1, 3), False, restore=True) + + restore_pages = worker._restore_pages + if device == "cuda": + import vllm.utils.gpu_sync_debug as gpu_sync_debug + + monkeypatch.setattr(gpu_sync_debug, "_SYNC_CHECK_MODE", "error") + monkeypatch.setattr(gpu_sync_debug, "_sync_check_enabled", False) + gpu_sync_debug.enable_gpu_sync_check() + restore_pages = gpu_sync_debug.with_gpu_sync_check(restore_pages) + restore_pages([transfer]) + + # Appending into unused capacity must preserve every imported prompt row. + for cache, host, block in zip( + worker.resident_caches, worker.host_caches, transfer.resident_block_ids + ): + assert torch.equal(cache[block].cpu(), host[8:12]) + cache[block, 2:] = 999 + assert torch.equal(cache[block, :2].cpu(), host[8:10]) + assert torch.all(cache[0] == -1) + worker._record_transfer_completion.assert_called_once_with( + [transfer], stream=stream + ) + + def test_hisparse_lazy_mirror_copies_transfer_pages(): worker = _make_hisparse_worker() worker.cache_handles = [ diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/worker.py index b7ffe9ec97..b223616401 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/worker.py @@ -22,6 +22,7 @@ from vllm.distributed.parallel_state import ( get_tensor_model_parallel_rank, get_tp_group, ) +from vllm.utils.gpu_sync_debug import gpu_sync_allowed from vllm.utils.torch_utils import current_stream from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy from vllm.v1.hisparse.layout import HISPARSE_HOT_SUFFIX @@ -414,11 +415,18 @@ class HiSparseConnectorWorker: transfers = ( metadata.command.page_transfers if metadata.command is not None else [] ) + self._restore_pages([transfer for transfer in transfers if transfer.restore]) self._post_forward_transfers = [ - transfer for transfer in transfers if transfer.after_forward + transfer + for transfer in transfers + if not transfer.restore and transfer.after_forward ] self._submit_transfers( - [transfer for transfer in transfers if not transfer.after_forward] + [ + transfer + for transfer in transfers + if not transfer.restore and not transfer.after_forward + ] ) self._pending_invalid_block_ids.extend(metadata.source_block_ids) if request_state_indices is not None: @@ -708,11 +716,17 @@ class HiSparseConnectorWorker: self._submitted_mirror_layers.update(pending_layers) def _record_transfer_completion( - self, transfers: list[SparseKVPageTransfer] + self, + transfers: list[SparseKVPageTransfer], + *, + stream: torch.Stream | None = None, ) -> None: - if not transfers or not self.is_host_writer: + if not transfers: return - stream = self.dma_stream + if stream is None: + if not self.is_host_writer: + return + stream = self.dma_stream assert stream is not None completion_event = torch.Event() completion_event.record(stream) @@ -720,6 +734,30 @@ class HiSparseConnectorWorker: self._pending_transfer_events.append((completion_event, transfer_ids)) self._enqueued_transfer_ids.extend(transfer_ids) + def _restore_pages(self, transfers: list[SparseKVPageTransfer]) -> None: + """Restore imported tails on every rank before its forward or graph replay.""" + if not transfers: + return + for layer_index, cache in enumerate(self.cache_handles): + source_index = cache.runtime.resident_source_index + source = self.host_caches[layer_index] + destination = self.resident_caches[layer_index] + for transfer in transfers: + host_start = transfer.host_block_id * self.kernel_block_size + resident_block = transfer.resident_block_ids[source_index] + # Each import pays one H2D page copy and enqueue per layer/rank + # instead of tail prefill on D. Restore the prompt rows before + # decode appends to this page; unused rows do not extend the + # logical sequence. + # cudaHostRegister pins this pool outside PyTorch's allocator, + # so the sync checker cannot recognize it via is_pinned(). + with gpu_sync_allowed(): + destination[resident_block].copy_( + source[host_start : host_start + self.kernel_block_size], + non_blocking=True, + ) + self._record_transfer_completion(transfers, stream=current_stream()) + def _submit_transfers(self, transfers: list[SparseKVPageTransfer]) -> None: if self.cache_handles[0].runtime.eager_host_mirror: self._record_transfer_completion(transfers) @@ -733,15 +771,12 @@ class HiSparseConnectorWorker: descriptor_count = len(transfers) * num_layers descriptors = self._acquire_dma_descriptors(descriptor_count) destination_rows = np.fromiter( - ( - transfer.destination_block_id * self.kernel_block_size - for transfer in transfers - ), + (transfer.host_block_id * self.kernel_block_size for transfer in transfers), dtype=np.int64, count=len(transfers), ) source_blocks_by_transfer = np.asarray( - [transfer.source_block_ids for transfer in transfers], dtype=np.int64 + [transfer.resident_block_ids for transfer in transfers], dtype=np.int64 ) if source_blocks_by_transfer.ndim != 2: raise RuntimeError( diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py index 7bcdd2531b..60cf20303c 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py @@ -3278,7 +3278,8 @@ class NixlBaseConnectorWorker: """Pair an uncached decode suffix with the same prefill regions.""" assert len(decode_block_ids) == len(prefill_block_ids) if not any(decode_block_ids): - return [], prefill_block_ids + empty_regions: list[list[int]] = [[] for _ in decode_block_ids] + return empty_regions, empty_regions.copy() if num_computed_blocks is not None: assert num_remote_blocks is not None diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py index fea3a00ce2..203c6bcfa6 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py @@ -168,8 +168,14 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker): local_block_ids = meta.local_physical_block_ids remote_region_groups = self.dst_region_group_ids[engine_id] local_region_groups = self.region_group_ids or remote_region_groups - groups_differ = local_region_groups != remote_region_groups - if groups_differ: + if not local_block_ids: + # Region expansion cannot index empty groups. Pass empty specs to + # _read_blocks so its existing cache-hit notification path runs. + read_specs = [ + ReadSpec(remote_rank=rank, local_block_ids=[], remote_block_ids=[]) + for rank in plan.all_source_ranks + ] + elif local_region_groups != remote_region_groups: if not self.use_mla or self._has_mamba: raise NotImplementedError( "Different NIXL cache-group layouts are only supported for " diff --git a/vllm/v1/core/single_type_kv_cache_manager.py b/vllm/v1/core/single_type_kv_cache_manager.py index 9ad0a38ed5..835a434c1e 100644 --- a/vllm/v1/core/single_type_kv_cache_manager.py +++ b/vllm/v1/core/single_type_kv_cache_manager.py @@ -2498,12 +2498,14 @@ class HiSparseResidentManager(_HiSparseAuxiliaryManager): del num_tokens_main_model assert not new_computed_blocks if total_computed_tokens > num_local_computed_tokens: - if total_computed_tokens % self.block_size != 0: - raise ValueError( - "A host-only HiSparse import must end on a cache-block boundary." - ) - imported_pages = total_computed_tokens // self.block_size - return max(cdiv(num_tokens, self.block_size) - imported_pages, 0) + # Keep the last imported page writable, including the last-token + # replay on a full prompt hit. Only earlier pages stay host-only. + imported_pages = cdiv(total_computed_tokens, self.block_size) + existing = len(self.req_to_blocks.get(request_id, ())) + needs_tail = self.get_resident_page(request_id, imported_pages - 1) is None + return int(needs_tail) + max( + cdiv(num_tokens, self.block_size) - max(imported_pages, existing), 0 + ) existing = len(self.req_to_blocks.get(request_id, ())) host_pages = cdiv(num_local_computed_tokens, self.block_size) required = cdiv(num_tokens, self.block_size) @@ -2518,13 +2520,15 @@ class HiSparseResidentManager(_HiSparseAuxiliaryManager): num_local_computed_tokens: int, num_external_computed_tokens: int, ) -> None: - """Represent imported host history with null resident pages.""" + """Reserve a writable final page; earlier imported pages stay on host.""" assert num_external_computed_tokens > 0 num_tokens = num_local_computed_tokens + num_external_computed_tokens blocks = self.req_to_blocks[request_id] - tail_page = cdiv(num_tokens, self.block_size) - 1 + tail_page = (num_tokens - 1) // self.block_size if len(blocks) <= tail_page: blocks.extend([self._null_block] * (tail_page + 1 - len(blocks))) + if blocks[tail_page].is_null: + blocks[tail_page] = self.block_pool.get_new_blocks(1)[0] def add_local_computed_blocks( self, diff --git a/vllm/v1/hisparse/coordinator.py b/vllm/v1/hisparse/coordinator.py index ecbc0a270a..056e19479a 100644 --- a/vllm/v1/hisparse/coordinator.py +++ b/vllm/v1/hisparse/coordinator.py @@ -73,6 +73,7 @@ class _PendingSpill: request_state: _HiSparseRequestState host_block: KVCacheBlock resident_blocks: tuple[KVCacheBlock, ...] + restore: bool = False expected_worker_completions: int = 0 worker_completions: int = 0 enqueue_applied: bool = False @@ -271,25 +272,29 @@ class HiSparseCoordinator: num_computed_tokens // self.host_manager.block_size, len(host_blocks) ) for host_idx in range(state.copies_recorded_blocks, num_host_blocks): - host_block = host_blocks[host_idx] - if host_block.is_null or host_block.block_hash is None: + if host_idx in state.pending_pages: continue - blocks: list[KVCacheBlock] = [] - for manager in self.resident_managers: - block = manager.get_resident_page(request_id, host_idx) - if block is None: - break - blocks.append(block) - if len(blocks) != len(self.resident_managers): - continue - self._drop_copy(host_block.block_hash) - self.copies[host_block.block_hash] = tuple(blocks) - for block in blocks: - self._copy_by_block[block.block_id] = host_block.block_hash + self._record_copy(request_id, host_idx, host_blocks[host_idx]) state.copies_recorded_blocks = max( state.copies_recorded_blocks, num_host_blocks ) + def _record_copy( + self, request_id: str, page_idx: int, host_block: KVCacheBlock + ) -> None: + if host_block.is_null or host_block.block_hash is None: + return + blocks: list[KVCacheBlock] = [] + for manager in self.resident_managers: + block = manager.get_resident_page(request_id, page_idx) + if block is None: + return + blocks.append(block) + self._drop_copy(host_block.block_hash) + self.copies[host_block.block_hash] = tuple(blocks) + for block in blocks: + self._copy_by_block[block.block_id] = host_block.block_hash + def _drop_copy(self, host_hash: BlockHashWithGroupId) -> None: blocks = self.copies.pop(host_hash, None) if blocks is None: @@ -406,9 +411,14 @@ class HiSparseCoordinator: assert self.host_manager is not None host_block_size = self.host_manager.block_size num_pages = num_computed_tokens // host_block_size + importing_pages = cdiv( + self._pending_imports.get(request_id, 0), host_block_size + ) state = self._get_request_state(request_id) budget = max(self.max_spill_pages - len(self.spills_to_send), 0) for page_idx in range(num_pages): + if page_idx < importing_pages: + continue if budget == 0: break if page_idx in state.pending_pages: @@ -420,7 +430,7 @@ class HiSparseCoordinator: for manager in self.resident_managers ): continue - if self._plan_spill(request_id, page_idx, after_forward=True): + if self._plan_page_transfer(request_id, page_idx, after_forward=True): budget -= 1 def publish_when_ready( @@ -491,17 +501,27 @@ class HiSparseCoordinator: state = self._get_request_state(request_id) state.valid_pages = set(range(num_pages)) state.ready_prefix_pages = num_pages + if num_computed_tokens: + self._plan_page_transfer( + request_id, + (num_computed_tokens - 1) // block_size, + after_forward=False, + restore=True, + ) self._publish_host_blocks_if_ready(request_id) - def _plan_spill( + def _plan_page_transfer( self, request_id: str, page_idx: int, *, after_forward: bool, + restore: bool = False, ) -> bool: assert self.host_manager is not None state = self._get_request_state(request_id) + if page_idx in state.pending_pages: + return False host_blocks = self.host_manager.req_to_blocks.get(request_id) if host_blocks is None or page_idx >= len(host_blocks): return False @@ -523,9 +543,10 @@ class HiSparseCoordinator: self.next_spill_id += 1 plan = SparseKVPageTransfer( transfer_id=spill_id, - destination_block_id=host_block.block_id, - source_block_ids=tuple(block.block_id for block in blocks), + host_block_id=host_block.block_id, + resident_block_ids=tuple(block.block_id for block in blocks), after_forward=after_forward, + restore=restore, ) self.pending_spills[spill_id] = _PendingSpill( transfer_id=spill_id, @@ -533,6 +554,7 @@ class HiSparseCoordinator: request_state=state, host_block=host_block, resident_blocks=tuple(blocks), + restore=restore, ) state.pending_pages[page_idx] = spill_id self.spills_to_send.append(plan) @@ -666,7 +688,7 @@ class HiSparseCoordinator: self._complete_host_writes() def _apply_enqueued_spills(self) -> None: - """Drop the DMA pins once the worker has enqueued a spill.""" + """Drop the GPU pins once every worker has completed the transfer.""" for pending in self.pending_spills.values(): if ( pending.enqueue_applied @@ -692,8 +714,12 @@ class HiSparseCoordinator: request_id, page_idx = pending.page pending.request_state.pending_pages.pop(page_idx, None) if self.request_states.get(request_id) is pending.request_state: - pending.request_state.valid_pages.add(page_idx) - self._page_became_clean(request_id, pending.request_state, page_idx) + if not pending.restore: + pending.request_state.valid_pages.add(page_idx) + if page_idx in pending.request_state.valid_pages: + self._page_became_clean(request_id, pending.request_state, page_idx) + if pending.restore: + self._record_copy(request_id, page_idx, pending.host_block) completed_request_ids.add(request_id) assert self.host_manager is not None self.host_manager.block_pool.free_blocks([pending.host_block]) diff --git a/vllm/v1/hisparse/types.py b/vllm/v1/hisparse/types.py index 8618f3cb20..269793ea85 100644 --- a/vllm/v1/hisparse/types.py +++ b/vllm/v1/hisparse/types.py @@ -9,9 +9,10 @@ class SparseKVPageTransfer: """Copy one logical KV page between cache-manager and worker-owned tiers.""" transfer_id: int - destination_block_id: int - source_block_ids: tuple[int, ...] + host_block_id: int + resident_block_ids: tuple[int, ...] after_forward: bool + restore: bool = False @dataclass(frozen=True)