[Bugfix][HiSparse][NIXL] Import full blocks without tail prefill on D (#57049)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
Nicolò Lucchesi
2026-09-17 15:26:10 -04:00
committed by GitHub
co-authored by OpenAI Codex Lucas Wilkinson
parent 41f9104fa6
commit 67e5b0acc9
8 changed files with 277 additions and 53 deletions
@@ -22,6 +22,7 @@ from vllm.distributed.parallel_state import (
get_tensor_model_parallel_rank,
get_tp_group,
)
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.torch_utils import current_stream
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
from vllm.v1.hisparse.layout import HISPARSE_HOT_SUFFIX
@@ -414,11 +415,18 @@ class HiSparseConnectorWorker:
transfers = (
metadata.command.page_transfers if metadata.command is not None else []
)
self._restore_pages([transfer for transfer in transfers if transfer.restore])
self._post_forward_transfers = [
transfer for transfer in transfers if transfer.after_forward
transfer
for transfer in transfers
if not transfer.restore and transfer.after_forward
]
self._submit_transfers(
[transfer for transfer in transfers if not transfer.after_forward]
[
transfer
for transfer in transfers
if not transfer.restore and not transfer.after_forward
]
)
self._pending_invalid_block_ids.extend(metadata.source_block_ids)
if request_state_indices is not None:
@@ -708,11 +716,17 @@ class HiSparseConnectorWorker:
self._submitted_mirror_layers.update(pending_layers)
def _record_transfer_completion(
self, transfers: list[SparseKVPageTransfer]
self,
transfers: list[SparseKVPageTransfer],
*,
stream: torch.Stream | None = None,
) -> None:
if not transfers or not self.is_host_writer:
if not transfers:
return
stream = self.dma_stream
if stream is None:
if not self.is_host_writer:
return
stream = self.dma_stream
assert stream is not None
completion_event = torch.Event()
completion_event.record(stream)
@@ -720,6 +734,30 @@ class HiSparseConnectorWorker:
self._pending_transfer_events.append((completion_event, transfer_ids))
self._enqueued_transfer_ids.extend(transfer_ids)
def _restore_pages(self, transfers: list[SparseKVPageTransfer]) -> None:
"""Restore imported tails on every rank before its forward or graph replay."""
if not transfers:
return
for layer_index, cache in enumerate(self.cache_handles):
source_index = cache.runtime.resident_source_index
source = self.host_caches[layer_index]
destination = self.resident_caches[layer_index]
for transfer in transfers:
host_start = transfer.host_block_id * self.kernel_block_size
resident_block = transfer.resident_block_ids[source_index]
# Each import pays one H2D page copy and enqueue per layer/rank
# instead of tail prefill on D. Restore the prompt rows before
# decode appends to this page; unused rows do not extend the
# logical sequence.
# cudaHostRegister pins this pool outside PyTorch's allocator,
# so the sync checker cannot recognize it via is_pinned().
with gpu_sync_allowed():
destination[resident_block].copy_(
source[host_start : host_start + self.kernel_block_size],
non_blocking=True,
)
self._record_transfer_completion(transfers, stream=current_stream())
def _submit_transfers(self, transfers: list[SparseKVPageTransfer]) -> None:
if self.cache_handles[0].runtime.eager_host_mirror:
self._record_transfer_completion(transfers)
@@ -733,15 +771,12 @@ class HiSparseConnectorWorker:
descriptor_count = len(transfers) * num_layers
descriptors = self._acquire_dma_descriptors(descriptor_count)
destination_rows = np.fromiter(
(
transfer.destination_block_id * self.kernel_block_size
for transfer in transfers
),
(transfer.host_block_id * self.kernel_block_size for transfer in transfers),
dtype=np.int64,
count=len(transfers),
)
source_blocks_by_transfer = np.asarray(
[transfer.source_block_ids for transfer in transfers], dtype=np.int64
[transfer.resident_block_ids for transfer in transfers], dtype=np.int64
)
if source_blocks_by_transfer.ndim != 2:
raise RuntimeError(
@@ -3278,7 +3278,8 @@ class NixlBaseConnectorWorker:
"""Pair an uncached decode suffix with the same prefill regions."""
assert len(decode_block_ids) == len(prefill_block_ids)
if not any(decode_block_ids):
return [], prefill_block_ids
empty_regions: list[list[int]] = [[] for _ in decode_block_ids]
return empty_regions, empty_regions.copy()
if num_computed_blocks is not None:
assert num_remote_blocks is not None
@@ -168,8 +168,14 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
local_block_ids = meta.local_physical_block_ids
remote_region_groups = self.dst_region_group_ids[engine_id]
local_region_groups = self.region_group_ids or remote_region_groups
groups_differ = local_region_groups != remote_region_groups
if groups_differ:
if not local_block_ids:
# Region expansion cannot index empty groups. Pass empty specs to
# _read_blocks so its existing cache-hit notification path runs.
read_specs = [
ReadSpec(remote_rank=rank, local_block_ids=[], remote_block_ids=[])
for rank in plan.all_source_ranks
]
elif local_region_groups != remote_region_groups:
if not self.use_mla or self._has_mamba:
raise NotImplementedError(
"Different NIXL cache-group layouts are only supported for "
+12 -8
View File
@@ -2498,12 +2498,14 @@ class HiSparseResidentManager(_HiSparseAuxiliaryManager):
del num_tokens_main_model
assert not new_computed_blocks
if total_computed_tokens > num_local_computed_tokens:
if total_computed_tokens % self.block_size != 0:
raise ValueError(
"A host-only HiSparse import must end on a cache-block boundary."
)
imported_pages = total_computed_tokens // self.block_size
return max(cdiv(num_tokens, self.block_size) - imported_pages, 0)
# Keep the last imported page writable, including the last-token
# replay on a full prompt hit. Only earlier pages stay host-only.
imported_pages = cdiv(total_computed_tokens, self.block_size)
existing = len(self.req_to_blocks.get(request_id, ()))
needs_tail = self.get_resident_page(request_id, imported_pages - 1) is None
return int(needs_tail) + max(
cdiv(num_tokens, self.block_size) - max(imported_pages, existing), 0
)
existing = len(self.req_to_blocks.get(request_id, ()))
host_pages = cdiv(num_local_computed_tokens, self.block_size)
required = cdiv(num_tokens, self.block_size)
@@ -2518,13 +2520,15 @@ class HiSparseResidentManager(_HiSparseAuxiliaryManager):
num_local_computed_tokens: int,
num_external_computed_tokens: int,
) -> None:
"""Represent imported host history with null resident pages."""
"""Reserve a writable final page; earlier imported pages stay on host."""
assert num_external_computed_tokens > 0
num_tokens = num_local_computed_tokens + num_external_computed_tokens
blocks = self.req_to_blocks[request_id]
tail_page = cdiv(num_tokens, self.block_size) - 1
tail_page = (num_tokens - 1) // self.block_size
if len(blocks) <= tail_page:
blocks.extend([self._null_block] * (tail_page + 1 - len(blocks)))
if blocks[tail_page].is_null:
blocks[tail_page] = self.block_pool.get_new_blocks(1)[0]
def add_local_computed_blocks(
self,
+47 -21
View File
@@ -73,6 +73,7 @@ class _PendingSpill:
request_state: _HiSparseRequestState
host_block: KVCacheBlock
resident_blocks: tuple[KVCacheBlock, ...]
restore: bool = False
expected_worker_completions: int = 0
worker_completions: int = 0
enqueue_applied: bool = False
@@ -271,25 +272,29 @@ class HiSparseCoordinator:
num_computed_tokens // self.host_manager.block_size, len(host_blocks)
)
for host_idx in range(state.copies_recorded_blocks, num_host_blocks):
host_block = host_blocks[host_idx]
if host_block.is_null or host_block.block_hash is None:
if host_idx in state.pending_pages:
continue
blocks: list[KVCacheBlock] = []
for manager in self.resident_managers:
block = manager.get_resident_page(request_id, host_idx)
if block is None:
break
blocks.append(block)
if len(blocks) != len(self.resident_managers):
continue
self._drop_copy(host_block.block_hash)
self.copies[host_block.block_hash] = tuple(blocks)
for block in blocks:
self._copy_by_block[block.block_id] = host_block.block_hash
self._record_copy(request_id, host_idx, host_blocks[host_idx])
state.copies_recorded_blocks = max(
state.copies_recorded_blocks, num_host_blocks
)
def _record_copy(
self, request_id: str, page_idx: int, host_block: KVCacheBlock
) -> None:
if host_block.is_null or host_block.block_hash is None:
return
blocks: list[KVCacheBlock] = []
for manager in self.resident_managers:
block = manager.get_resident_page(request_id, page_idx)
if block is None:
return
blocks.append(block)
self._drop_copy(host_block.block_hash)
self.copies[host_block.block_hash] = tuple(blocks)
for block in blocks:
self._copy_by_block[block.block_id] = host_block.block_hash
def _drop_copy(self, host_hash: BlockHashWithGroupId) -> None:
blocks = self.copies.pop(host_hash, None)
if blocks is None:
@@ -406,9 +411,14 @@ class HiSparseCoordinator:
assert self.host_manager is not None
host_block_size = self.host_manager.block_size
num_pages = num_computed_tokens // host_block_size
importing_pages = cdiv(
self._pending_imports.get(request_id, 0), host_block_size
)
state = self._get_request_state(request_id)
budget = max(self.max_spill_pages - len(self.spills_to_send), 0)
for page_idx in range(num_pages):
if page_idx < importing_pages:
continue
if budget == 0:
break
if page_idx in state.pending_pages:
@@ -420,7 +430,7 @@ class HiSparseCoordinator:
for manager in self.resident_managers
):
continue
if self._plan_spill(request_id, page_idx, after_forward=True):
if self._plan_page_transfer(request_id, page_idx, after_forward=True):
budget -= 1
def publish_when_ready(
@@ -491,17 +501,27 @@ class HiSparseCoordinator:
state = self._get_request_state(request_id)
state.valid_pages = set(range(num_pages))
state.ready_prefix_pages = num_pages
if num_computed_tokens:
self._plan_page_transfer(
request_id,
(num_computed_tokens - 1) // block_size,
after_forward=False,
restore=True,
)
self._publish_host_blocks_if_ready(request_id)
def _plan_spill(
def _plan_page_transfer(
self,
request_id: str,
page_idx: int,
*,
after_forward: bool,
restore: bool = False,
) -> bool:
assert self.host_manager is not None
state = self._get_request_state(request_id)
if page_idx in state.pending_pages:
return False
host_blocks = self.host_manager.req_to_blocks.get(request_id)
if host_blocks is None or page_idx >= len(host_blocks):
return False
@@ -523,9 +543,10 @@ class HiSparseCoordinator:
self.next_spill_id += 1
plan = SparseKVPageTransfer(
transfer_id=spill_id,
destination_block_id=host_block.block_id,
source_block_ids=tuple(block.block_id for block in blocks),
host_block_id=host_block.block_id,
resident_block_ids=tuple(block.block_id for block in blocks),
after_forward=after_forward,
restore=restore,
)
self.pending_spills[spill_id] = _PendingSpill(
transfer_id=spill_id,
@@ -533,6 +554,7 @@ class HiSparseCoordinator:
request_state=state,
host_block=host_block,
resident_blocks=tuple(blocks),
restore=restore,
)
state.pending_pages[page_idx] = spill_id
self.spills_to_send.append(plan)
@@ -666,7 +688,7 @@ class HiSparseCoordinator:
self._complete_host_writes()
def _apply_enqueued_spills(self) -> None:
"""Drop the DMA pins once the worker has enqueued a spill."""
"""Drop the GPU pins once every worker has completed the transfer."""
for pending in self.pending_spills.values():
if (
pending.enqueue_applied
@@ -692,8 +714,12 @@ class HiSparseCoordinator:
request_id, page_idx = pending.page
pending.request_state.pending_pages.pop(page_idx, None)
if self.request_states.get(request_id) is pending.request_state:
pending.request_state.valid_pages.add(page_idx)
self._page_became_clean(request_id, pending.request_state, page_idx)
if not pending.restore:
pending.request_state.valid_pages.add(page_idx)
if page_idx in pending.request_state.valid_pages:
self._page_became_clean(request_id, pending.request_state, page_idx)
if pending.restore:
self._record_copy(request_id, page_idx, pending.host_block)
completed_request_ids.add(request_id)
assert self.host_manager is not None
self.host_manager.block_pool.free_blocks([pending.host_block])
+3 -2
View File
@@ -9,9 +9,10 @@ class SparseKVPageTransfer:
"""Copy one logical KV page between cache-manager and worker-owned tiers."""
transfer_id: int
destination_block_id: int
source_block_ids: tuple[int, ...]
host_block_id: int
resident_block_ids: tuple[int, ...]
after_forward: bool
restore: bool = False
@dataclass(frozen=True)