[Bugfix][HiSparse][NIXL] Import full blocks without tail prefill on D (#57049)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
co-authored by
OpenAI Codex
Lucas Wilkinson
parent
41f9104fa6
commit
67e5b0acc9
@@ -22,6 +22,7 @@ from vllm.distributed.parallel_state import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tp_group,
|
||||
)
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.torch_utils import current_stream
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
|
||||
from vllm.v1.hisparse.layout import HISPARSE_HOT_SUFFIX
|
||||
@@ -414,11 +415,18 @@ class HiSparseConnectorWorker:
|
||||
transfers = (
|
||||
metadata.command.page_transfers if metadata.command is not None else []
|
||||
)
|
||||
self._restore_pages([transfer for transfer in transfers if transfer.restore])
|
||||
self._post_forward_transfers = [
|
||||
transfer for transfer in transfers if transfer.after_forward
|
||||
transfer
|
||||
for transfer in transfers
|
||||
if not transfer.restore and transfer.after_forward
|
||||
]
|
||||
self._submit_transfers(
|
||||
[transfer for transfer in transfers if not transfer.after_forward]
|
||||
[
|
||||
transfer
|
||||
for transfer in transfers
|
||||
if not transfer.restore and not transfer.after_forward
|
||||
]
|
||||
)
|
||||
self._pending_invalid_block_ids.extend(metadata.source_block_ids)
|
||||
if request_state_indices is not None:
|
||||
@@ -708,11 +716,17 @@ class HiSparseConnectorWorker:
|
||||
self._submitted_mirror_layers.update(pending_layers)
|
||||
|
||||
def _record_transfer_completion(
|
||||
self, transfers: list[SparseKVPageTransfer]
|
||||
self,
|
||||
transfers: list[SparseKVPageTransfer],
|
||||
*,
|
||||
stream: torch.Stream | None = None,
|
||||
) -> None:
|
||||
if not transfers or not self.is_host_writer:
|
||||
if not transfers:
|
||||
return
|
||||
stream = self.dma_stream
|
||||
if stream is None:
|
||||
if not self.is_host_writer:
|
||||
return
|
||||
stream = self.dma_stream
|
||||
assert stream is not None
|
||||
completion_event = torch.Event()
|
||||
completion_event.record(stream)
|
||||
@@ -720,6 +734,30 @@ class HiSparseConnectorWorker:
|
||||
self._pending_transfer_events.append((completion_event, transfer_ids))
|
||||
self._enqueued_transfer_ids.extend(transfer_ids)
|
||||
|
||||
def _restore_pages(self, transfers: list[SparseKVPageTransfer]) -> None:
|
||||
"""Restore imported tails on every rank before its forward or graph replay."""
|
||||
if not transfers:
|
||||
return
|
||||
for layer_index, cache in enumerate(self.cache_handles):
|
||||
source_index = cache.runtime.resident_source_index
|
||||
source = self.host_caches[layer_index]
|
||||
destination = self.resident_caches[layer_index]
|
||||
for transfer in transfers:
|
||||
host_start = transfer.host_block_id * self.kernel_block_size
|
||||
resident_block = transfer.resident_block_ids[source_index]
|
||||
# Each import pays one H2D page copy and enqueue per layer/rank
|
||||
# instead of tail prefill on D. Restore the prompt rows before
|
||||
# decode appends to this page; unused rows do not extend the
|
||||
# logical sequence.
|
||||
# cudaHostRegister pins this pool outside PyTorch's allocator,
|
||||
# so the sync checker cannot recognize it via is_pinned().
|
||||
with gpu_sync_allowed():
|
||||
destination[resident_block].copy_(
|
||||
source[host_start : host_start + self.kernel_block_size],
|
||||
non_blocking=True,
|
||||
)
|
||||
self._record_transfer_completion(transfers, stream=current_stream())
|
||||
|
||||
def _submit_transfers(self, transfers: list[SparseKVPageTransfer]) -> None:
|
||||
if self.cache_handles[0].runtime.eager_host_mirror:
|
||||
self._record_transfer_completion(transfers)
|
||||
@@ -733,15 +771,12 @@ class HiSparseConnectorWorker:
|
||||
descriptor_count = len(transfers) * num_layers
|
||||
descriptors = self._acquire_dma_descriptors(descriptor_count)
|
||||
destination_rows = np.fromiter(
|
||||
(
|
||||
transfer.destination_block_id * self.kernel_block_size
|
||||
for transfer in transfers
|
||||
),
|
||||
(transfer.host_block_id * self.kernel_block_size for transfer in transfers),
|
||||
dtype=np.int64,
|
||||
count=len(transfers),
|
||||
)
|
||||
source_blocks_by_transfer = np.asarray(
|
||||
[transfer.source_block_ids for transfer in transfers], dtype=np.int64
|
||||
[transfer.resident_block_ids for transfer in transfers], dtype=np.int64
|
||||
)
|
||||
if source_blocks_by_transfer.ndim != 2:
|
||||
raise RuntimeError(
|
||||
|
||||
@@ -3278,7 +3278,8 @@ class NixlBaseConnectorWorker:
|
||||
"""Pair an uncached decode suffix with the same prefill regions."""
|
||||
assert len(decode_block_ids) == len(prefill_block_ids)
|
||||
if not any(decode_block_ids):
|
||||
return [], prefill_block_ids
|
||||
empty_regions: list[list[int]] = [[] for _ in decode_block_ids]
|
||||
return empty_regions, empty_regions.copy()
|
||||
|
||||
if num_computed_blocks is not None:
|
||||
assert num_remote_blocks is not None
|
||||
|
||||
@@ -168,8 +168,14 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
local_block_ids = meta.local_physical_block_ids
|
||||
remote_region_groups = self.dst_region_group_ids[engine_id]
|
||||
local_region_groups = self.region_group_ids or remote_region_groups
|
||||
groups_differ = local_region_groups != remote_region_groups
|
||||
if groups_differ:
|
||||
if not local_block_ids:
|
||||
# Region expansion cannot index empty groups. Pass empty specs to
|
||||
# _read_blocks so its existing cache-hit notification path runs.
|
||||
read_specs = [
|
||||
ReadSpec(remote_rank=rank, local_block_ids=[], remote_block_ids=[])
|
||||
for rank in plan.all_source_ranks
|
||||
]
|
||||
elif local_region_groups != remote_region_groups:
|
||||
if not self.use_mla or self._has_mamba:
|
||||
raise NotImplementedError(
|
||||
"Different NIXL cache-group layouts are only supported for "
|
||||
|
||||
Reference in New Issue
Block a user