[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com>
Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Matthew Bonanni
2026-09-12 00:36:47 -04:00
committed by GitHub
co-authored by OpenAI Codex Lucas Wilkinson Lucas Wilkinson Nicolò Lucchesi OpenAI Codex Claude Opus 5 Lucas Wilkinson
parent 120ec4ebd2
commit d43bb2f37f
124 changed files with 14262 additions and 906 deletions
@@ -197,6 +197,12 @@ KVConnectorFactory.register_connector(
"MultiConnector",
)
KVConnectorFactory.register_connector(
"HiSparseConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector",
"HiSparseConnector",
)
KVConnectorFactory.register_connector(
"MoRIIOConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector",
@@ -60,6 +60,7 @@ class KVOutputAggregator:
# [req_id -> n_remaining_workers]
self._recv_remaining_count = dict[str, int]()
self._send_remaining_count = dict[str, int]()
self._failed_recving_pending = set[str]()
self._expected_finished_count = expected_finished_count
@classmethod
@@ -156,6 +157,10 @@ class KVOutputAggregator:
combined_kv_cache_events.increment_workers(1)
invalid_block_ids |= kv_output.invalid_block_ids
self._failed_recving_pending |= kv_output.failed_recving
failed_recving = self._failed_recving_pending & finished_recving
self._failed_recving_pending -= failed_recving
# select output of the worker specified by output_rank
output = outputs[output_rank]
@@ -168,6 +173,7 @@ class KVOutputAggregator:
kv_cache_events=combined_kv_cache_events or None,
kv_connector_worker_meta=aggregated_kv_connector_worker_meta or None,
invalid_block_ids=invalid_block_ids,
failed_recving=failed_recving,
expected_finished_count=self._expected_finished_count,
)
@@ -34,8 +34,8 @@ The class provides the following primitives:
save_kv_layer() - starts saving KV for layer i (maybe async)
wait_for_save() - blocks until all saves are done
get_finished() - called with ids of finished requests, returns
ids of requests that have completed async sending/recving.
get_transfer_results() - returns async send/receive completions and
receive failures in one snapshot.
build_connector_worker_meta() - builds metadata to be sent
back to the scheduler-side connector
"""
@@ -43,6 +43,7 @@ The class provides the following primitives:
import enum
from abc import ABC, abstractmethod
from collections.abc import Callable, Iterable
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, Literal
import torch
@@ -63,7 +64,7 @@ if TYPE_CHECKING:
)
from vllm.forward_context import ForwardContext
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.request import Request
@@ -82,6 +83,19 @@ CopyBlocksOp = Callable[
logger = init_logger(__name__)
@dataclass
class KVConnectorTransferResults:
"""Asynchronous transfer completions from one worker snapshot.
Failed receives also appear in ``finished_recving`` so the scheduler can
release the request from its transfer wait state.
"""
finished_sending: set[str] = field(default_factory=set)
finished_recving: set[str] = field(default_factory=set)
failed_recving: set[str] = field(default_factory=set)
class SupportsHMA(ABC):
"""
The class that indicates the corresponding connector supports hybrid memory
@@ -211,6 +225,7 @@ class KVConnectorBase_V1(ABC):
else:
raise ValueError("kv_transfer_config must be set for KVConnectorBase_V1")
self._kv_cache_config = kv_cache_config
self._kv_cache_manager: KVCacheManager | None = None
self._role = role
@property
@@ -278,6 +293,15 @@ class KVConnectorBase_V1(ABC):
"""
return
# TODO(NickLucche): group model-runner lifecycle hooks in the interface.
def finish_forward(self) -> None:
"""Notify the connector that the model no longer reads this step's KV."""
return
def reset_capture_state(self) -> None:
"""Reset worker state mutated while capturing CUDA graphs."""
return
def handle_preemptions(self, kv_connector_metadata: KVConnectorMetadata):
"""
Handle preempted requests or evicted blocks BEFORE they are overwritten.
@@ -368,6 +392,16 @@ class KVConnectorBase_V1(ABC):
"""
return None, None
def get_transfer_results(
self, finished_req_ids: set[str]
) -> KVConnectorTransferResults:
"""Return completed sends, receives, and receive failures together."""
finished_sending, finished_recving = self.get_finished(finished_req_ids)
return KVConnectorTransferResults(
finished_sending=set(finished_sending or ()),
finished_recving=set(finished_recving or ()),
)
def get_block_ids_with_load_errors(self) -> set[int]:
"""
Get the set of block IDs that failed to load.
@@ -380,9 +414,9 @@ class KVConnectorBase_V1(ABC):
- Applies to both sync- and async-loading requests.
- Async loading: failed blocks may be reported in any forward pass
up to and including the pass where the request ID is returned by
`get_finished()`. Even if failures occur, the request must still
be reported via `get_finished()`, and the failed block IDs must
appear here no later than that same pass.
`get_transfer_results()`. Even if failures occur, the request
must still be reported as finished receiving, and the failed
block IDs must appear here no later than that same pass.
- Sync loading: failed blocks should be reported in the forward
pass in which they are detected.
"""
@@ -436,6 +470,11 @@ class KVConnectorBase_V1(ABC):
# Scheduler-side methods
# ==============================
def bind_kv_cache_manager(self, kv_cache_manager: "KVCacheManager") -> None:
"""Bind the scheduler's cache manager after it has been constructed."""
self._kv_cache_manager = kv_cache_manager
self.bind_gpu_block_pool(kv_cache_manager.block_pool)
def bind_gpu_block_pool(self, gpu_block_pool: "BlockPool") -> None:
"""
Bind the GPU block pool to the connector for per-GPU block status tracking.
@@ -583,6 +622,10 @@ class KVConnectorBase_V1(ABC):
"""
return ()
def has_pending_block_frees(self) -> bool:
"""Whether pending transfers can release blocks instead of preemption."""
return False
def has_pending_push_work(self) -> bool:
"""Return True if the connector has push-mode work that requires
the engine main loop to keep stepping (e.g. a P-side request whose
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,329 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Connector boundary for HiSparse scheduler and worker state."""
from __future__ import annotations
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
import torch
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
KVConnectorMetadata,
KVConnectorRole,
KVConnectorWorkerMetadata,
SupportsHMA,
)
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.worker import (
HiSparseConnectorWorker,
)
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.hisparse.coordinator import get_hisparse_coordinator
from vllm.v1.hisparse.types import SparseKVOffloadCommand, SparseKVRowMirror
from vllm.v1.outputs import KVConnectorOutput
if TYPE_CHECKING:
from vllm.forward_context import ForwardContext
from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager
from vllm.v1.hisparse.coordinator import HiSparseCoordinator
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.request import Request
@dataclass
class HiSparseConnectorMetadata(KVConnectorMetadata):
command: SparseKVOffloadCommand | None
host_block_copies: tuple[KVCacheBlockCopy, ...]
source_block_ids: tuple[int, ...]
row_mirrors: dict[str, tuple[SparseKVRowMirror, ...]]
all_context_pages_resident: bool
@dataclass
class HiSparseConnectorWorkerMetadata(KVConnectorWorkerMetadata):
enqueued_transfer_counts: dict[int, int]
completed_transfer_counts: dict[int, int]
completed_host_copy_dst_ids: tuple[int, ...] = ()
def aggregate(self, other: KVConnectorWorkerMetadata) -> KVConnectorWorkerMetadata:
assert isinstance(other, HiSparseConnectorWorkerMetadata)
def add_counts(first: dict[int, int], second: dict[int, int]) -> dict[int, int]:
combined = first.copy()
for transfer_id, count in second.items():
combined[transfer_id] = combined.get(transfer_id, 0) + count
return combined
return HiSparseConnectorWorkerMetadata(
enqueued_transfer_counts=add_counts(
self.enqueued_transfer_counts, other.enqueued_transfer_counts
),
completed_transfer_counts=add_counts(
self.completed_transfer_counts, other.completed_transfer_counts
),
# Every worker runs the same copies, so a union is the ack.
completed_host_copy_dst_ids=tuple(
{*self.completed_host_copy_dst_ids, *other.completed_host_copy_dst_ids}
),
)
class HiSparseConnectorScheduler:
def __init__(
self,
*,
async_speculative: bool,
draft_kv_lookahead: int = 0,
) -> None:
self.coordinator: HiSparseCoordinator | None = None
self.async_speculative = async_speculative
self.draft_kv_lookahead = draft_kv_lookahead
self.requests: dict[str, Request] = {}
def bind_coordinator(self, coordinator: HiSparseCoordinator) -> None:
assert self.coordinator is None
self.coordinator = coordinator
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> HiSparseConnectorMetadata:
assert self.coordinator is not None
# HiSparse rebinds worker state every step, so the load must start
# before the forward rather than being deferred to post-forward.
scheduler_output.has_sync_kv_loads = True
scheduler_output.block_table_updates = (
self.coordinator.take_block_table_updates() or None
)
command = self.coordinator.build_offload_command()
host_block_copies = self.coordinator.take_host_block_copies()
source_group_id = self.coordinator.host_group_id
assert source_group_id is not None
source_block_ids = [
block_id
for request in scheduler_output.scheduled_new_reqs
for block_id in request.block_ids[source_group_id]
]
for new_block_ids in scheduler_output.scheduled_cached_reqs.new_block_ids:
if new_block_ids is not None:
source_block_ids.extend(new_block_ids[source_group_id])
num_computed_tokens = {
request.req_id: request.num_computed_tokens
for request in scheduler_output.scheduled_new_reqs
}
num_computed_tokens.update(
zip(
scheduler_output.scheduled_cached_reqs.req_ids,
scheduler_output.scheduled_cached_reqs.num_computed_tokens,
)
)
scheduled_requests = tuple(
(
request_id,
num_computed_tokens[request_id],
scheduled_count,
)
for request_id, scheduled_count in (
scheduler_output.num_scheduled_tokens.items()
)
)
row_mirrors = {}
for request_id, scheduled_start, scheduled_count in scheduled_requests:
mirror_start = scheduled_start
if self.async_speculative:
mirror_start = max(
0,
scheduled_start - self.requests[request_id].num_output_placeholders,
)
row_mirrors[request_id] = self.coordinator.build_row_mirrors(
(
(
request_id,
mirror_start,
scheduled_count
+ scheduled_start
- mirror_start
+ self.draft_kv_lookahead,
),
),
)
return HiSparseConnectorMetadata(
command,
host_block_copies,
tuple(source_block_ids),
row_mirrors,
self.coordinator.all_context_pages_resident(scheduled_requests),
)
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
assert self.coordinator is not None
for request_id in connector_output.finished_recving or ():
self.coordinator.finish_host_import(
request_id, failed=request_id in connector_output.failed_recving
)
metadata = connector_output.kv_connector_worker_meta
if metadata is None:
return
assert isinstance(metadata, HiSparseConnectorWorkerMetadata)
self.coordinator.release_completed_host_copies(
metadata.completed_host_copy_dst_ids
)
self.coordinator.update_spills(
metadata.enqueued_transfer_counts,
metadata.completed_transfer_counts,
)
class HiSparseConnector(KVConnectorBase_V1, SupportsHMA):
"""Join the scheduler coordinator to the worker's transfer engine."""
@classmethod
def get_required_kvcache_layout(cls, vllm_config: VllmConfig) -> str:
return "BLHNC"
def __init__(
self,
vllm_config: VllmConfig,
role: KVConnectorRole,
kv_cache_config: KVCacheConfig,
) -> None:
super().__init__(vllm_config, role, kv_cache_config)
if kv_cache_config.hisparse_host_num_blocks is None:
raise ValueError("HiSparseConnector requires a HiSparse host pool")
self.connector_scheduler: HiSparseConnectorScheduler | None = None
self.connector_worker: HiSparseConnectorWorker | None = None
if role == KVConnectorRole.SCHEDULER:
speculative_config = vllm_config.speculative_config
self.connector_scheduler = HiSparseConnectorScheduler(
async_speculative=bool(
vllm_config.scheduler_config.async_scheduling
and speculative_config is not None
),
draft_kv_lookahead=vllm_config.num_lookahead_tokens,
)
elif role == KVConnectorRole.WORKER:
self.connector_worker = HiSparseConnectorWorker(
vllm_config, kv_cache_config
)
else:
raise ValueError(f"Unsupported KV connector role: {role}")
def bind_kv_cache_manager(self, kv_cache_manager: KVCacheManager) -> None:
assert self.connector_scheduler is not None
self.connector_scheduler.bind_coordinator(
get_hisparse_coordinator(kv_cache_manager)
)
@property
def requires_kv_delivery(self) -> bool:
return False
def has_pending_push_work(self) -> bool:
assert self.connector_scheduler is not None
assert self.connector_scheduler.coordinator is not None
return self.connector_scheduler.coordinator.has_pending_work()
def has_pending_block_frees(self) -> bool:
assert self.connector_scheduler is not None
assert self.connector_scheduler.coordinator is not None
return self.connector_scheduler.coordinator.has_pending_reclamation()
def finish_forward(self) -> None:
assert self.connector_worker is not None
self.connector_worker.finish_forward()
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]) -> None:
assert self.connector_worker is not None
self.connector_worker.register_kv_caches(kv_caches)
def reset_capture_state(self) -> None:
assert self.connector_worker is not None
self.connector_worker.reset_hot_state()
def start_load_kv(self, forward_context: ForwardContext, **kwargs: Any) -> None:
assert self.connector_worker is not None
metadata = self._get_connector_metadata()
assert isinstance(metadata, HiSparseConnectorMetadata)
request_state_indices = kwargs.get("request_state_indices")
assert request_state_indices is None or isinstance(
request_state_indices, torch.Tensor
)
request_ids = kwargs.get("request_ids")
assert request_ids is None or isinstance(request_ids, list)
attn_metadata = kwargs.get("attn_metadata")
assert attn_metadata is None or isinstance(attn_metadata, dict)
self.connector_worker.start_step(
metadata,
request_state_indices,
request_ids,
num_tokens=int(kwargs.get("num_tokens") or 0),
)
self.connector_worker.prepare_forward(attn_metadata)
def wait_for_layer_load(self, layer_name: str) -> None:
return
def save_kv_layer(
self,
layer_name: str,
kv_layer: torch.Tensor,
attn_metadata: AttentionMetadata,
**kwargs: Any,
) -> None:
return
def wait_for_save(self) -> None:
return
def build_connector_worker_meta(self) -> KVConnectorWorkerMetadata | None:
assert self.connector_worker is not None
enqueued, completed = self.connector_worker.take_transfer_updates()
host_copies = self.connector_worker.take_completed_host_copies()
if not enqueued and not completed and not host_copies:
return None
return HiSparseConnectorWorkerMetadata(
enqueued_transfer_counts={transfer_id: 1 for transfer_id in enqueued},
completed_transfer_counts={transfer_id: 1 for transfer_id in completed},
completed_host_copy_dst_ids=tuple(host_copies),
)
def shutdown(self) -> None:
if self.connector_worker is not None:
self.connector_worker.shutdown()
def get_num_new_matched_tokens(
self, request: Request, num_computed_tokens: int
) -> tuple[int | None, bool]:
return 0, False
def update_state_after_alloc(
self,
request: Request,
blocks: KVCacheBlocks,
num_external_tokens: int,
) -> None:
assert self.connector_scheduler is not None
self.connector_scheduler.requests[request.request_id] = request
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> KVConnectorMetadata:
assert self.connector_scheduler is not None
return self.connector_scheduler.build_connector_meta(scheduler_output)
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
assert self.connector_scheduler is not None
self.connector_scheduler.update_connector_output(connector_output)
def request_finished_all_groups(
self,
request: Request,
block_ids: tuple[list[int], ...],
) -> tuple[bool, dict[str, Any] | None]:
assert self.connector_scheduler is not None
self.connector_scheduler.requests.pop(request.request_id, None)
return False, None
@@ -0,0 +1,832 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""HiSparse worker-side host/hot state and data movement."""
from __future__ import annotations
from collections import deque
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from functools import partial
from typing import TYPE_CHECKING, Any, NamedTuple
import numpy as np
import torch
from vllm import _custom_ops as ops
from vllm.config import VllmConfig
from vllm.distributed.parallel_state import (
get_tensor_model_parallel_rank,
get_tp_group,
)
from vllm.utils.torch_utils import current_stream
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
from vllm.v1.hisparse.layout import HISPARSE_HOT_SUFFIX
from vllm.v1.hisparse.runtime import HiSparseCacheHandle, release_pinned_state
from vllm.v1.hisparse.types import SparseKVPageTransfer, SparseKVRowMirror
from vllm.v1.kv_cache_interface import (
HiSparseHotSpec,
KVCacheConfig,
)
from vllm.v1.worker.utils import copy_kv_cache_blocks_inplace
if TYPE_CHECKING:
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import (
HiSparseConnectorMetadata,
)
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
class _DMADescriptors(NamedTuple):
src: torch.Tensor
dst: torch.Tensor
sizes: torch.Tensor
src_np: np.ndarray
dst_np: np.ndarray
sizes_np: np.ndarray
@dataclass
class _SlotMappingStaging:
stream: torch.Stream
event: torch.Event
slots: torch.Tensor
candidates: tuple[SparseKVRowMirror, ...] = ()
num_tokens: int = 0
source_index: int = 0
def _allocate_dma_descriptors(size: int) -> _DMADescriptors:
src, dst, sizes = (torch.empty(size, dtype=torch.int64) for _ in range(3))
return _DMADescriptors(
src,
dst,
sizes,
src.numpy(),
dst.numpy(),
sizes.numpy(),
)
def _get_hisparse_cache(
forward_context: dict[str, Any], layer_name: str
) -> HiSparseCacheHandle:
attention_layer = forward_context[layer_name]
hisparse_cache = attention_layer.hisparse_cache
assert hisparse_cache is not None
return hisparse_cache
def _flatten_row_mirrors(
row_mirrors: Mapping[str, tuple[SparseKVRowMirror, ...]],
request_ids: Sequence[str] | None,
) -> tuple[SparseKVRowMirror, ...]:
ordered_ids = row_mirrors if request_ids is None else request_ids
return tuple(
mirror
for request_id in ordered_ids
for mirror in row_mirrors.get(request_id, ())
)
def _select_written_row_mirrors(
candidates: tuple[SparseKVRowMirror, ...],
source_slots: np.ndarray,
source_index: int,
) -> tuple[SparseKVRowMirror, ...]:
"""Select and coalesce GPU-written rows from a scheduler-owned envelope."""
source_slots = source_slots[source_slots >= 0]
if source_slots.size == 0 or not candidates:
return ()
starts = np.asarray([mirror.source_starts for mirror in candidates], dtype=np.int64)
counts = np.fromiter((mirror.num_rows for mirror in candidates), dtype=np.int64)
destinations = np.fromiter(
[mirror.destination_start for mirror in candidates], dtype=np.int64
)
if source_index >= starts.shape[1]:
raise RuntimeError("HiSparse row-mirror source index is out of range.")
source_starts = starts[:, source_index]
order = np.argsort(source_starts)
matches = np.searchsorted(source_starts[order], source_slots, side="right") - 1
candidate_ids = order[np.maximum(matches, 0)]
offsets = source_slots - source_starts[candidate_ids]
valid = (matches >= 0) & (offsets < counts[candidate_ids])
if not np.all(valid):
# A page the scheduler could not map has no host destination, so its
# rows are simply not mirrorable this step.
source_slots = source_slots[valid]
candidate_ids = candidate_ids[valid]
offsets = offsets[valid]
if source_slots.size == 0:
return ()
sources = starts[candidate_ids] + offsets[:, None]
destinations = destinations[candidate_ids] + offsets
boundaries = (
np.flatnonzero(
(candidate_ids[1:] != candidate_ids[:-1])
| (np.diff(destinations) != 1)
| np.any(np.diff(sources, axis=0) != 1, axis=1)
)
+ 1
)
run_starts = np.concatenate(([0], boundaries))
run_ends = np.concatenate((boundaries, [destinations.size]))
return tuple(
SparseKVRowMirror(
source_starts=tuple(int(value) for value in sources[start]),
destination_start=int(destinations[start]),
num_rows=int(end - start),
)
for start, end in zip(run_starts, run_ends, strict=True)
)
def _create_hisparse_host_events(
shared_host_region: SharedOffloadRegion | None,
is_host_writer: bool,
device: torch.device,
) -> tuple[torch.Event, torch.Event]:
if shared_host_region is None:
return torch.Event(), torch.Event()
events: tuple[torch.Event, torch.Event] | None = None
ipc_handles = None
if is_host_writer:
events = (
torch.cuda.Event(interprocess=True),
torch.cuda.Event(interprocess=True),
)
stream = current_stream()
for event in events:
event.record(stream)
ipc_handles = tuple(event.ipc_handle() for event in events)
ipc_handles = get_tp_group().broadcast_object(ipc_handles, src=0)
if events is None:
events = (
torch.cuda.Event.from_ipc_handle(device, ipc_handles[0]),
torch.cuda.Event.from_ipc_handle(device, ipc_handles[1]),
)
return events
class HiSparseConnectorWorker:
"""Own HiSparse host/hot state and execute its worker-side transfers."""
def __init__(self, vllm_config: VllmConfig, kv_cache_config: KVCacheConfig) -> None:
self.vllm_config = vllm_config
self.kv_cache_config = kv_cache_config
self._initialized = False
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]) -> None:
forward_context = self.vllm_config.compilation_config.static_forward_context
cache_handles: list[HiSparseCacheHandle] = []
cache_layer_names: list[str] = []
for group in self.kv_cache_config.kv_cache_groups:
if not isinstance(group.kv_cache_spec, HiSparseHotSpec):
continue
for cache_name in group.layer_names:
assert cache_name.endswith(HISPARSE_HOT_SUFFIX)
layer_name = cache_name[: -len(HISPARSE_HOT_SUFFIX)]
cache_handles.append(_get_hisparse_cache(forward_context, layer_name))
cache_layer_names.append(layer_name)
if not cache_handles:
raise RuntimeError("HiSparse connector found no hot-cache handles.")
hot_backings: dict[int, torch.Tensor] = {}
registered_host_pools: dict[int, torch.Tensor] = {}
for cache in cache_handles:
hot_backing = cache.runtime.hot_backing
hot_backings[hot_backing.untyped_storage().data_ptr()] = hot_backing
registered_pool = cache.runtime.registered_host_pool
registered_host_pools[registered_pool.data_ptr()] = registered_pool
if len(hot_backings) != 1:
raise RuntimeError("HiSparse hot tensors must share one GPU backing.")
hot_backing = next(iter(hot_backings.values()))
pinned_host_pools = list(registered_host_pools.values())
resident = cache_handles[0].view
assert resident is not None
host_num_blocks = self.kv_cache_config.hisparse_host_num_blocks
assert host_num_blocks is not None
try:
self.initialize(
cache_handles,
cache_layer_names,
hot_backing,
self.vllm_config.scheduler_config.max_num_seqs,
host_num_blocks,
hot_backing.device,
pinned_host_pools,
)
except Exception:
release_pinned_state(
[cache.runtime for cache in cache_handles],
pinned_host_pools,
)
raise
def initialize(
self,
cache_handles: list[HiSparseCacheHandle],
cache_layer_names: list[str],
hot_backing: torch.Tensor,
max_num_reqs: int,
host_num_blocks: int,
device: torch.device,
pinned_host_pools: list[torch.Tensor],
*,
shared_host_region: SharedOffloadRegion | None = None,
is_host_writer: bool = True,
) -> None:
if self._initialized:
raise RuntimeError("HiSparse connector worker is already initialized.")
resident = cache_handles[0].view
assert resident is not None
self.kernel_block_size = resident.block_size
self.host_num_blocks = host_num_blocks
self.pinned_host_pools = pinned_host_pools
self.shared_host_region = shared_host_region
self.is_host_writer = is_host_writer
self.dma_stream = (
torch.cuda.Stream(device=device) if self.is_host_writer else None
)
self._slot_mapping_staging = None
if self.is_host_writer:
max_mirror_rows = (
self.vllm_config.scheduler_config.max_num_batched_tokens
+ max_num_reqs * (self.vllm_config.num_lookahead_tokens + 1)
)
self._slot_mapping_staging = _SlotMappingStaging(
stream=torch.Stream(device=device),
event=torch.Event(),
slots=torch.empty(
max_mirror_rows,
dtype=torch.int64,
pin_memory=True,
),
)
self._dma_free_descriptors: list[_DMADescriptors] = []
self._pending_dma_descriptors: deque[tuple[torch.Event, _DMADescriptors]] = (
deque()
)
self._dma_submitted = False
self._per_layer_mirrored: set[int] = set()
self._submitted_mirror_layers: set[int] = set()
self._layer_ready_events = tuple(torch.Event() for _ in cache_handles)
self._forward_ready_event = torch.Event()
self._set_row_mirrors(())
self.cache_layer_names = cache_layer_names
self._group_leaders = tuple(
(layer_name, cache)
for layer_name, cache in zip(cache_layer_names, cache_handles, strict=True)
if cache.runtime.is_group_leader
)
self.host_write_events = _create_hisparse_host_events(
shared_host_region, is_host_writer, device
)
self.host_write_event = self.host_write_events[1]
self._next_host_write_event = 0
self.cache_handles = cache_handles
self.leader_runtimes = [cache.runtime for _, cache in self._group_leaders]
request_state_indices = {
indices.data_ptr(): indices
for cache in cache_handles
if (indices := cache.runtime.request_state_indices) is not None
}
if len(request_state_indices) != 1:
raise RuntimeError(
"HiSparse runtimes must share one request-state mapping."
)
self.request_state_indices = next(iter(request_state_indices.values()))
if self.request_state_indices.numel() != max_num_reqs:
raise RuntimeError(
"HiSparse request-state mapping does not match max_num_seqs."
)
self.hot_backing = hot_backing
self._pending_invalid_block_ids: list[int] = []
# Destination block ids of host copies this worker has run.
self._completed_host_copy_dst_ids: list[int] = []
self._post_forward_transfers: list[SparseKVPageTransfer] = []
self._enqueued_transfer_ids: list[int] = []
self._pending_transfer_events: deque[tuple[torch.Event, tuple[int, ...]]] = (
deque()
)
self._init_dma()
if self.is_host_writer:
for layer_index, handle in enumerate(cache_handles):
handle.submit_layer_mirror = partial(
self._enqueue_layer_mirror, layer_index
)
self._initialized = True
def set_request_state_indices(self, indices: torch.Tensor) -> None:
if indices.numel() > self.request_state_indices.numel():
raise ValueError(
"HiSparse request-state mapping exceeds max_num_seqs: "
f"{indices.numel()} > {self.request_state_indices.numel()}."
)
if torch.cuda.is_current_stream_capturing():
return
# Attention indexes persistent request state by input-batch row. Refresh
# that indirection after every batch compaction or reorder.
self.request_state_indices.fill_(-1)
self.request_state_indices[: indices.numel()].copy_(indices)
if self._pending_invalid_block_ids:
self.invalidate_blocks(self._pending_invalid_block_ids, indices)
self._pending_invalid_block_ids.clear()
def _init_dma(self) -> None:
host_caches = tuple(cache.runtime.host_cache for cache in self.cache_handles)
resident_caches = []
for cache in self.cache_handles:
assert cache.view is not None and cache.slot_mapping is not None
resident_caches.append(cache.view.cache)
self.host_caches = host_caches
self.resident_caches = tuple(resident_caches)
for resident_cache, host_cache in zip(resident_caches, host_caches):
row_bytes = resident_cache.shape[-1] * resident_cache.element_size()
if (
resident_cache.ndim != 3
or resident_cache.shape[1] != self.kernel_block_size
or resident_cache.stride(1) * resident_cache.element_size() != row_bytes
):
raise RuntimeError("HiSparse DMA requires contiguous resident rows.")
if (
host_cache.ndim != 2
or not host_cache.is_contiguous()
or host_cache.shape[1] * host_cache.element_size() != row_bytes
):
raise RuntimeError("HiSparse DMA requires contiguous host rows.")
def start_step(
self,
metadata: HiSparseConnectorMetadata,
request_state_indices: torch.Tensor | None,
request_ids: list[str] | None = None,
num_tokens: int = 0,
) -> None:
self._stage_row_mirror_mapping(num_tokens)
previous_host_write_event = self.host_write_event
self.host_write_event = self.host_write_events[self._next_host_write_event]
self._next_host_write_event ^= 1
current_stream().wait_event(previous_host_write_event)
self._release_completed_dma_descriptors()
mirrors = _flatten_row_mirrors(metadata.row_mirrors, request_ids)
if self._slot_mapping_staging is not None:
self._slot_mapping_staging.candidates = mirrors
self._set_row_mirrors(mirrors)
self._dma_submitted = False
self._clear_forward_mirror_state()
for handle in self.cache_handles:
handle.all_context_pages_resident = metadata.all_context_pages_resident
handle.mirror_from_resident = True
self._copy_host_blocks(metadata.host_block_copies, previous_host_write_event)
transfers = (
metadata.command.page_transfers if metadata.command is not None else []
)
self._post_forward_transfers = [
transfer for transfer in transfers if transfer.after_forward
]
self._submit_transfers(
[transfer for transfer in transfers if not transfer.after_forward]
)
self._pending_invalid_block_ids.extend(metadata.source_block_ids)
if request_state_indices is not None:
self.set_request_state_indices(request_state_indices)
def _clear_forward_mirror_state(self) -> None:
self._per_layer_mirrored.clear()
self._submitted_mirror_layers.clear()
for handle in self.cache_handles:
handle.decode_batch = False
handle.host_mirror_required = False
handle.num_actual_tokens = 0
handle.num_decode_tokens = 0
handle.req_id_per_token = None
def prepare_forward(self, attn_metadata: Mapping[str, Any] | None) -> None:
if attn_metadata is None:
return
for layer_name, handle in self._group_leaders:
metadata = attn_metadata.get(layer_name)
if metadata is not None:
handle.prepare_group_for_batch(metadata)
def _stage_row_mirror_mapping(self, num_tokens: int) -> None:
"""Snapshot the rows this forward will write, off the compute stream.
The resident slot mapping is a persistent view bound at registration
time, so the rows are already staged by the time the forward launches.
"""
state = self._slot_mapping_staging
if state is None or not num_tokens:
return
handle = self.cache_handles[0]
slots = handle.slot_mapping
assert slots is not None
source_index = handle.runtime.resident_source_index
start = state.num_tokens
end = start + num_tokens
if end > state.slots.shape[0]:
raise ValueError(
"HiSparse row mapping exceeds staging capacity: "
f"{end} > {state.slots.shape[0]}."
)
main_stream = current_stream()
state.stream.wait_stream(main_stream)
with torch.cuda.stream(state.stream):
state.slots[start:end].copy_(
slots[:num_tokens],
non_blocking=True,
)
state.event.record(state.stream)
state.num_tokens = end
state.source_index = source_index
def _resolve_row_mirrors(self, state: _SlotMappingStaging) -> None:
"""Compute the final row mirrors from the staged GPU slot mapping."""
self._set_row_mirrors(
_select_written_row_mirrors(
state.candidates,
state.slots[: state.num_tokens].numpy(),
state.source_index,
)
)
state.num_tokens = 0
def _copy_host_blocks(
self,
host_block_copies: Sequence[KVCacheBlockCopy],
previous_host_write_event: torch.Event,
) -> None:
if not host_block_copies:
return
self._completed_host_copy_dst_ids.extend(
copy.dst_block_id for copy in host_block_copies
)
if self.shared_host_region is None or get_tensor_model_parallel_rank() == 0:
if self.host_caches:
previous_host_write_event.synchronize()
copy_kv_cache_blocks_inplace(
self.host_caches,
self.host_num_blocks,
host_block_copies,
)
if self.shared_host_region is not None:
get_tp_group().barrier()
def invalidate_blocks(
self, block_ids: list[int], request_state_indices: torch.Tensor
) -> None:
"""Invalidate recycled host slots in this worker's leader runtimes."""
if not block_ids:
return
device = self.cache_handles[0].runtime.device
staging = torch.tensor(block_ids, dtype=torch.int32, pin_memory=True)
blocks = staging.to(device, dtype=torch.int32, non_blocking=True)
offsets = torch.arange(self.kernel_block_size, dtype=torch.int32, device=device)
slots = (blocks[:, None] * self.kernel_block_size + offsets[None, :]).flatten()
sorted_slots = torch.sort(slots).values
state_indices = request_state_indices.to(device=device, dtype=torch.long)
for runtime in self.leader_runtimes:
runtime.invalidate_sorted_slots(sorted_slots, state_indices)
def reset_hot_state(self) -> None:
for runtime in self.leader_runtimes:
runtime.reset_hot_state()
def _release_completed_dma_descriptors(self) -> None:
pending = self._pending_dma_descriptors
while pending and pending[0][0].query():
_, descriptors = pending.popleft()
self._dma_free_descriptors.append(descriptors)
def _acquire_dma_descriptors(self, size: int) -> _DMADescriptors:
for index, descriptors in enumerate(self._dma_free_descriptors):
if descriptors.src.numel() >= size:
return self._dma_free_descriptors.pop(index)
return _allocate_dma_descriptors(size)
def _submit_dma_descriptors(
self,
descriptors: _DMADescriptors,
descriptor_count: int,
transfer_ids: tuple[int, ...] = (),
ready_event: torch.Event | None = None,
) -> None:
stream = self.dma_stream
assert stream is not None
if ready_event is None:
stream.wait_stream(current_stream())
else:
stream.wait_event(ready_event)
completion_event = torch.Event()
with torch.cuda.stream(stream):
ops.swap_blocks_batch(
descriptors.src[:descriptor_count],
descriptors.dst[:descriptor_count],
descriptors.sizes[:descriptor_count],
)
self.host_write_event.record(stream)
completion_event.record(stream)
self._pending_dma_descriptors.append((completion_event, descriptors))
self._dma_submitted = True
if transfer_ids:
self._pending_transfer_events.append((completion_event, transfer_ids))
self._enqueued_transfer_ids.extend(transfer_ids)
def _set_row_mirrors(self, mirrors: tuple[SparseKVRowMirror, ...]) -> None:
self._row_mirrors = mirrors
self._row_mirror_destination_starts = np.fromiter(
(mirror.destination_start for mirror in mirrors),
dtype=np.int64,
count=len(mirrors),
)
self._row_mirror_counts = np.fromiter(
(mirror.num_rows for mirror in mirrors),
dtype=np.int64,
count=len(mirrors),
)
if mirrors:
self._row_mirror_source_starts = np.asarray(
[mirror.source_starts for mirror in mirrors], dtype=np.int64
)
if self._row_mirror_source_starts.ndim != 2:
raise RuntimeError("HiSparse DMA source mappings must be rectangular.")
else:
self._row_mirror_source_starts = np.empty((0, 0), dtype=np.int64)
self._row_mirror_num_rows = int(self._row_mirror_counts.sum())
def _enqueue_row_dma(
self, layer_indices: Sequence[int], ready_event: torch.Event | None = None
) -> None:
if (
not layer_indices
or not self._row_mirror_num_rows
or not self.is_host_writer
):
return
mirrors = self._row_mirrors
num_layers = len(layer_indices)
descriptor_count = len(mirrors) * num_layers
descriptors = self._acquire_dma_descriptors(descriptor_count)
destination_starts = self._row_mirror_destination_starts
row_counts = self._row_mirror_counts
for descriptor_offset, layer_index in enumerate(layer_indices):
cache = self.cache_handles[layer_index]
source_index = cache.runtime.resident_source_index
if source_index >= self._row_mirror_source_starts.shape[1]:
raise RuntimeError("HiSparse row DMA source index is out of range.")
source_rows = self._row_mirror_source_starts[:, source_index]
source = self.resident_caches[layer_index]
destination = self.host_caches[layer_index]
row_bytes = source.shape[-1] * source.element_size()
if (
source.stride(1) * source.element_size() != row_bytes
or destination.shape[1] * destination.element_size() != row_bytes
):
raise RuntimeError("HiSparse row DMA requires contiguous rows.")
source_blocks, source_row_offsets = np.divmod(
source_rows, self.kernel_block_size
)
source_out_of_range = (
np.any(source_blocks < 0)
or np.any(source_blocks >= source.shape[0])
or np.any(source_row_offsets + row_counts > self.kernel_block_size)
)
if source_out_of_range:
raise RuntimeError("HiSparse row DMA source is out of range.")
if np.any(destination_starts < 0) or np.any(
destination_starts + row_counts > destination.shape[0]
):
raise RuntimeError("HiSparse row DMA index is out of range.")
descriptor_slice = slice(descriptor_offset, descriptor_count, num_layers)
descriptors.src_np[descriptor_slice] = (
source.data_ptr()
+ source_blocks * source.stride(0) * source.element_size()
+ source_row_offsets * row_bytes
)
descriptors.dst_np[descriptor_slice] = (
destination.data_ptr() + destination_starts * row_bytes
)
descriptors.sizes_np[descriptor_slice] = row_counts * row_bytes
self._submit_dma_descriptors(
descriptors, descriptor_count, ready_event=ready_event
)
def _enqueue_layer_mirror(self, layer_index: int) -> None:
handle = self.cache_handles[layer_index]
if not handle.host_mirror_required:
return
if layer_index in self._per_layer_mirrored:
raise RuntimeError(f"HiSparse layer {layer_index} mirrored twice.")
state = self._slot_mapping_staging
if state is not None and state.num_tokens:
# The staging stream waited on the compute stream before the
# forward launched, so this only drains pre-forward work that is
# already queued ahead of the layer kernels: no GPU bubble.
state.event.synchronize()
self._resolve_row_mirrors(state)
self._per_layer_mirrored.add(layer_index)
next_layer = layer_index + 1
if (
next_layer < len(self.cache_handles)
and self.cache_handles[next_layer].runtime.resident_source_index
== handle.runtime.resident_source_index
):
return
ready_event = self._layer_ready_events[layer_index]
ready_event.record()
pending_layers = tuple(
sorted(self._per_layer_mirrored - self._submitted_mirror_layers)
)
self._enqueue_row_dma(
pending_layers,
ready_event=ready_event,
)
self._submitted_mirror_layers.update(pending_layers)
def _record_transfer_completion(
self, transfers: list[SparseKVPageTransfer]
) -> None:
if not transfers or not self.is_host_writer:
return
stream = self.dma_stream
assert stream is not None
completion_event = torch.Event()
completion_event.record(stream)
transfer_ids = tuple(transfer.transfer_id for transfer in transfers)
self._pending_transfer_events.append((completion_event, transfer_ids))
self._enqueued_transfer_ids.extend(transfer_ids)
def _submit_transfers(self, transfers: list[SparseKVPageTransfer]) -> None:
if self.cache_handles[0].runtime.eager_host_mirror:
self._record_transfer_completion(transfers)
else:
self._enqueue_transfers(transfers)
def _enqueue_transfers(self, transfers: list[SparseKVPageTransfer]) -> None:
if not transfers or not self.is_host_writer:
return
num_layers = len(self.cache_handles)
descriptor_count = len(transfers) * num_layers
descriptors = self._acquire_dma_descriptors(descriptor_count)
destination_rows = np.fromiter(
(
transfer.destination_block_id * self.kernel_block_size
for transfer in transfers
),
dtype=np.int64,
count=len(transfers),
)
source_blocks_by_transfer = np.asarray(
[transfer.source_block_ids for transfer in transfers], dtype=np.int64
)
if source_blocks_by_transfer.ndim != 2:
raise RuntimeError(
"HiSparse spill DMA source mappings must be rectangular."
)
for layer_index, cache in enumerate(self.cache_handles):
source_index = cache.runtime.resident_source_index
if source_index >= source_blocks_by_transfer.shape[1]:
raise RuntimeError("HiSparse spill DMA source index is out of range.")
source_blocks = source_blocks_by_transfer[:, source_index]
source = self.resident_caches[layer_index]
destination = self.host_caches[layer_index]
if np.any(source_blocks < 0) or np.any(source_blocks >= source.shape[0]):
raise RuntimeError("HiSparse spill DMA source is out of range.")
if np.any(destination_rows < 0) or np.any(
destination_rows + self.kernel_block_size > destination.shape[0]
):
raise RuntimeError("HiSparse spill DMA destination is out of range.")
row_bytes = source.shape[-1] * source.element_size()
descriptor_slice = slice(layer_index, descriptor_count, num_layers)
descriptors.src_np[descriptor_slice] = (
source.data_ptr()
+ source_blocks * source.stride(0) * source.element_size()
)
descriptors.dst_np[descriptor_slice] = (
destination.data_ptr() + destination_rows * row_bytes
)
descriptors.sizes_np[descriptor_slice] = self.kernel_block_size * row_bytes
self._submit_dma_descriptors(
descriptors,
descriptor_count,
transfer_ids=tuple(transfer.transfer_id for transfer in transfers),
)
def _enqueue_host_mirror(
self,
ready_event: torch.Event | None = None,
) -> None:
active = [
(index, handle)
for index, handle in enumerate(self.cache_handles)
if handle.num_actual_tokens != 0
]
if not active:
return
def mirror_key(handle: HiSparseCacheHandle) -> tuple:
slots = handle.mirror_slot_mapping
return (
handle.num_actual_tokens,
handle.num_decode_tokens,
handle.decode_batch,
handle.host_mirror_required,
handle.runtime.eager_host_mirror,
None if slots is None else slots.data_ptr(),
)
keys = {mirror_key(handle) for _, handle in active}
if len(keys) > 1:
raise RuntimeError(
f"HiSparse cache layers disagree on mirror metadata: {list(keys)}."
)
cache = active[0][1]
if not cache.host_mirror_required:
return
dst_slots = cache.mirror_slot_mapping
if dst_slots is None:
raise RuntimeError("HiSparse host mirror has no source slot mapping.")
num_rows = min(cache.num_actual_tokens, dst_slots.numel())
if num_rows == 0:
return
if self.is_host_writer:
expected_layers = {index for index, _ in active}
if self._per_layer_mirrored and self._per_layer_mirrored != expected_layers:
raise RuntimeError(
"HiSparse per-layer DMA did not mirror every active layer: "
f"expected {sorted(expected_layers)}, got "
f"{sorted(self._per_layer_mirrored)}."
)
pending_layers = tuple(
sorted(expected_layers - self._submitted_mirror_layers)
)
if pending_layers:
self._enqueue_row_dma(pending_layers, ready_event=ready_event)
self._submitted_mirror_layers.update(pending_layers)
assert cache.req_id_per_token is not None
for _, handle in active:
if handle.runtime.is_group_leader:
handle.runtime.invalidate_written_slots(
dst_slots[:num_rows],
cache.req_id_per_token[:num_rows],
)
def _finish_mirror_phase(self, ready_event: torch.Event | None = None) -> None:
state = self._slot_mapping_staging
if state is not None and state.num_tokens:
# Measured free: blocking here costs no throughput (26.7 vs 27.6
# gen tok/s) and lets every step mirror exactly the written rows.
state.event.synchronize()
self._resolve_row_mirrors(state)
self._enqueue_host_mirror(ready_event)
self._clear_forward_mirror_state()
def finish_forward(self) -> None:
compute_stream = current_stream()
self._forward_ready_event.record()
self._finish_mirror_phase(self._forward_ready_event)
transfers = self._post_forward_transfers
self._post_forward_transfers = []
self._submit_transfers(transfers)
if self.is_host_writer:
if self._dma_submitted:
compute_stream.wait_event(self.host_write_event)
self._dma_submitted = False
else:
self.host_write_event.record(compute_stream)
self._release_completed_dma_descriptors()
def take_completed_host_copies(self) -> list[int]:
"""Drain host copies this worker has enqueued for this step."""
completed = self._completed_host_copy_dst_ids
self._completed_host_copy_dst_ids = []
return completed
def take_transfer_updates(self) -> tuple[list[int], list[int]]:
enqueued = self._enqueued_transfer_ids
self._enqueued_transfer_ids = []
completed: list[int] = []
while self._pending_transfer_events:
event, transfer_ids = self._pending_transfer_events[0]
if not event.query():
break
self._pending_transfer_events.popleft()
completed.extend(transfer_ids)
return enqueued, completed
def shutdown(self) -> None:
if not self._initialized:
return
if self._slot_mapping_staging is not None:
self._slot_mapping_staging.stream.synchronize()
if self.dma_stream is not None:
self.dma_stream.synchronize()
release_pinned_state(
[cache.runtime for cache in self.cache_handles], self.pinned_host_pools
)
self._initialized = False
@@ -702,9 +702,9 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
- Applies to both sync- and async-loading requests.
- Async loading: failed blocks may be reported in any forward pass
up to and including the pass where the request ID is returned by
`get_finished()`. Even if failures occur, the request must still
be reported via `get_finished()`, and the failed block IDs must
appear here no later than that same pass.
`get_transfer_results()`. Even if failures occur, the request
must still be reported as finished receiving, and the failed
block IDs must appear here no later than that same pass.
- Sync loading: failed blocks should be reported in the forward
pass in which they are detected.
"""
@@ -635,6 +635,7 @@ class MooncakeConnectorScheduler:
):
self.vllm_config = vllm_config
self.block_size = vllm_config.cache_config.block_size
self.kv_cache_config = kv_cache_config
assert vllm_config.kv_transfer_config
self.is_kv_producer: bool = (
@@ -649,7 +650,7 @@ class MooncakeConnectorScheduler:
not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
and any(
not isinstance(g.kv_cache_spec, FullAttentionSpec)
for g in kv_cache_config.kv_cache_groups
for g in kv_cache_config.transfer_groups
)
)
# GDN is represented as a MambaSpec in vLLM. This Mooncake MambaSpec
@@ -670,7 +671,7 @@ class MooncakeConnectorScheduler:
(g.kv_cache_spec.sliding_window, g.kv_cache_spec.block_size)
if isinstance(g.kv_cache_spec, SlidingWindowSpec)
else (0, self.block_size)
for g in kv_cache_config.kv_cache_groups
for g in kv_cache_config.transfer_groups
]
# cdiv(n_tokens, block_size) gives blocks/window; add 1 to
# conservatively account for boundary overlap.
@@ -684,11 +685,17 @@ class MooncakeConnectorScheduler:
block_ids: tuple[list[int], ...] | list[list[int]],
) -> list[list[int]]:
"""Clip per-group block IDs to sliding window size."""
if len(block_ids) == 0 or not self._is_hma_required:
return list(block_ids)
if len(block_ids) == 0:
return []
if len(block_ids) == len(self.kv_cache_config.transfer_group_ids):
selected = list(block_ids)
else:
selected = list(self.kv_cache_config.select_transfer_block_ids(block_ids))
if len(selected) == 0 or not self._is_hma_required:
return selected
return [
blocks[-self.blocks_per_sw[i] :] if self.blocks_per_sw[i] > 0 else blocks
for i, blocks in enumerate(block_ids)
for i, blocks in enumerate(selected)
]
def _get_remote_prefill_token_count(self, num_prompt_tokens: int) -> int:
@@ -1046,18 +1053,13 @@ class MooncakeConnectorWorker:
self._tp_size: dict[EngineId, int] = {self.engine_id: self.tp_size}
self._layer_specs: dict[str, KVCacheSpec] = {}
for group in kv_cache_config.kv_cache_groups:
for group in kv_cache_config.transfer_groups:
group_spec = group.kv_cache_spec
specs_by_layer = getattr(group_spec, "kv_cache_specs", {})
for layer_name in group.layer_names:
self._layer_specs[layer_name] = specs_by_layer.get(
layer_name, group_spec
)
self._layer_group_indices: dict[str, int] = {
layer: group_index
for group_index, group in enumerate(kv_cache_config.kv_cache_groups)
for layer in group.layer_names
}
self.transfer_topo = TransferTopology(
tp_rank=self.tp_rank,
tp_size=self.tp_size,
@@ -1441,7 +1443,7 @@ class MooncakeConnectorWorker:
block_arange = np.arange(self._physical_blocks_per_logical_kv_block).reshape(
1, -1
)
group_specs = self.kv_cache_config.kv_cache_groups
group_specs = self.kv_cache_config.transfer_groups
return [
BlockTable.map_to_kernel_blocks(
np.array(group),
@@ -1494,7 +1496,7 @@ class MooncakeConnectorWorker:
local_block_ids_by_group: list[list[int]] = []
remote_block_ids_by_group: list[list[int]] = []
has_block_error = False
group_specs = self.kv_cache_config.kv_cache_groups
group_specs = self.kv_cache_config.transfer_groups
for group_index, (local_group, remote_group) in enumerate(
zip(send_meta.local_block_ids, remote_block_ids_per_group)
):
@@ -1750,7 +1752,7 @@ class MooncakeConnectorWorker:
self.registered_layer_names.append(layer_name)
self.registered_layer_indices.append(layer_index)
self.registered_group_indices.append(
self._layer_group_indices[layer_name]
self.kv_cache_config.transfer_group_index_by_layer[layer_name]
)
storage = cache.untyped_storage()
storage_addr = storage.data_ptr()
@@ -2121,7 +2123,7 @@ class MooncakeConnectorWorker:
) -> list[TransferRegion]:
if not group_indices:
group_indices = [
self._layer_group_indices.get(layer_name, 0)
self.kv_cache_config.transfer_group_index_by_layer.get(layer_name, 0)
for layer_name in layer_names
]
return _expand_transfer_regions(
@@ -3,18 +3,20 @@
"""External-store cache-hit coordinator for MooncakeStoreConnector."""
from collections.abc import Sequence
from typing import cast
from typing import NamedTuple, cast
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
chunk_hashes_for_block_size,
)
from vllm.utils.math_utils import cdiv
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_coordinator import SpecGroup
from vllm.v1.core.kv_cache_utils import (
BlockHash,
KVCacheBlock,
)
from vllm.v1.core.single_type_kv_cache_manager import (
SingleTypeKVCacheManager,
)
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheGroupSpec,
@@ -25,6 +27,13 @@ from vllm.v1.kv_cache_interface import (
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
class StoreSpecGroup(NamedTuple):
spec: KVCacheSpec
group_ids: list[int]
manager_cls: type[SingleTypeKVCacheManager]
use_eagle: bool
class ExternalCachedBlockPool:
"""Duck-typed BlockPool backed by a ``(group_id, hash)`` exists set."""
@@ -121,7 +130,7 @@ class MooncakeStoreCoordinator:
"""Mirrors KVCacheCoordinator.verify_and_split_kv_cache_groups but
dispatches via spec_manager_map (we don't allocate managers).
"""
attention_groups: list[SpecGroup] = []
attention_groups: list[StoreSpecGroup] = []
for i, g in enumerate(self.kv_cache_groups):
# Skip groups that opt out of prefix caching (e.g. GLM-5.3-Flash
# kpool tail): per-request scratch, never shareable, so they must
@@ -143,7 +152,7 @@ class MooncakeStoreCoordinator:
break
else:
attention_groups.append(
SpecGroup(spec, [i], manager_cls, g.is_eagle_group)
StoreSpecGroup(spec, [i], manager_cls, g.is_eagle_group)
)
# Full attention first (matches upstream convergence ordering).
attention_groups.sort(key=lambda g: not isinstance(g.spec, FullAttentionSpec))
@@ -67,6 +67,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.protocol import
from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv
from vllm.utils.network_utils import get_ip, make_zmq_socket
from vllm.utils.torch_utils import is_non_overlapping_and_dense
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
from vllm.v1.core.kv_cache_utils import (
BlockHash,
@@ -1909,21 +1910,31 @@ class MooncakeStoreWorker:
logger.warning("No KV caches to offload.")
return
# Resolve each entry to a representative tensor for storage
# deduplication. For attention layers the value is already a tensor;
# for Mamba layers it is a list of tensors that all share the same
# underlying raw storage, so we take the first one.
def _repr_tensor(v: torch.Tensor | list[torch.Tensor]) -> torch.Tensor:
assert isinstance(v, torch.Tensor | list)
return v if isinstance(v, torch.Tensor) else v[0]
assert self.cache_config.num_gpu_blocks is not None
self.num_blocks = self.cache_config.num_gpu_blocks
use_group_regions = self._kv_cache_config.hisparse_host_num_blocks is not None
seen_storage_ptrs: set[int] = set()
cache_tensors: list[torch.Tensor] = []
for cache in kv_caches.values():
cache = group_kernel_blocks(cache, self.num_blocks)
cache_tensors.append(cache)
cache_storage = cache.untyped_storage()
base_addr = cache_storage.data_ptr()
region_len = cache_storage.nbytes()
if base_addr not in seen_storage_ptrs:
if not use_group_regions:
seen_storage_ptrs: set[int] = set()
cache_tensors = [
group_kernel_blocks(_repr_tensor(cache), self.num_blocks)
for cache in kv_caches.values()
]
for cache in cache_tensors:
cache_storage = cache.untyped_storage()
base_addr = cache_storage.data_ptr()
if base_addr in seen_storage_ptrs:
continue
seen_storage_ptrs.add(base_addr)
region_len = cache_storage.nbytes()
ret = self.store.register_buffer(base_addr, region_len)
if ret != 0:
logger.error(
@@ -1932,17 +1943,110 @@ class MooncakeStoreWorker:
region_len,
ret,
)
for db in self.token_dbs:
db.store_layout.register_kv_caches(cache_tensors, self.num_blocks)
registered_buffers: dict[int, int] = {}
num_host_segments = 0
num_device_segments = 0
for group, db in zip(self._kv_cache_groups, self.token_dbs, strict=True):
if not use_group_regions:
continue
missing_layers = [
layer_name
for layer_name in group.layer_names
if layer_name not in kv_caches
]
if missing_layers:
raise ValueError(
f"Missing transferable KV caches for layers: {missing_layers}"
)
group_caches = [
(layer_name, kv_caches[layer_name]) for layer_name in group.layer_names
]
seen_group_ptrs: set[int] = set()
addrs: list[int] = []
block_lens: list[int] = []
for layer_name, value in group_caches:
cache = _repr_tensor(value)
is_host_resident = group.host_resident
if is_host_resident:
num_blocks = self._kv_cache_config.hisparse_host_num_blocks
assert num_blocks is not None
else:
num_blocks = self._kv_cache_config.num_blocks
cache = group_kernel_blocks(cache, num_blocks)
cache_storage = cache.untyped_storage()
if is_host_resident:
base_addr = cache.data_ptr()
region_len = cache.nbytes
else:
base_addr = cache_storage.data_ptr()
region_len = cache_storage.nbytes()
if region_len % num_blocks != 0:
raise ValueError(
f"KV cache region for {layer_name} has size {region_len}, "
f"which is not divisible by its {num_blocks} blocks."
)
registered_len = registered_buffers.get(base_addr)
if registered_len is None:
ret = self.store.register_buffer(base_addr, region_len)
if ret != 0:
mem_kind = "host" if is_host_resident else "device"
raise RuntimeError(
"Mooncake register_buffer failed for addr "
f"{base_addr:#x} len {region_len} ({mem_kind}): {ret}"
)
registered_buffers[base_addr] = region_len
elif registered_len != region_len:
raise ValueError(
f"KV cache views at {base_addr:#x} expose inconsistent "
f"region sizes: {registered_len} and {region_len}."
)
regions: list[tuple[int, int]] = []
if not is_non_overlapping_and_dense(cache[0]):
for head_idx in range(cache.shape[1]):
head_cache = cache[:, head_idx]
assert is_non_overlapping_and_dense(head_cache[0])
regions.append(
(
head_cache.data_ptr(),
head_cache.stride(0) * head_cache.element_size(),
)
)
elif cache.stride(0) * cache.element_size() * num_blocks == region_len:
regions.append((base_addr, region_len // num_blocks))
else:
regions.append(
(cache.data_ptr(), cache.stride(0) * cache.element_size())
)
for region_addr, block_len in regions:
if region_addr in seen_group_ptrs:
continue
seen_group_ptrs.add(region_addr)
addrs.append(region_addr)
block_lens.append(block_len)
if is_host_resident:
num_host_segments += 1
else:
num_device_segments += 1
db.set_kv_caches_base_addr(addrs)
db.set_block_len(block_lens)
logger.info(
"Registered KV caches: num_groups=%d, num_tensors=%d, num_blocks=%d",
"Registered KV caches: num_groups=%d, num_segments=%d (host=%d, device=%d)",
len(self.token_dbs),
len(cache_tensors),
self.num_blocks,
num_host_segments + num_device_segments,
num_host_segments,
num_device_segments,
)
for db in self.token_dbs:
db.store_layout.register_kv_caches(cache_tensors, self.num_blocks)
# Start transfer threads
if self.can_put:
ready_event_sending = threading.Event()
@@ -2316,6 +2420,19 @@ class MooncakeStoreWorker:
if store is None:
return
self.store = None
deadline = time.monotonic() + 5.0
for thread in (self.kv_send_thread, *self.kv_recv_threads):
if thread is None:
continue
while thread.request_queue.unfinished_tasks and time.monotonic() < deadline:
time.sleep(0.05)
if thread.request_queue.unfinished_tasks:
logger.warning(
"Mooncake store %s still has %d in-flight requests at "
"close; closing anyway.",
thread.name,
thread.request_queue.unfinished_tasks,
)
try:
store.close()
except Exception as e:
@@ -17,6 +17,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorHandshakeMetadata,
KVConnectorMetadata,
KVConnectorRole,
KVConnectorTransferResults,
KVConnectorWorkerMetadata,
SupportsHMA,
)
@@ -35,7 +36,7 @@ if TYPE_CHECKING:
from vllm.distributed.kv_events import KVCacheEvent
from vllm.forward_context import ForwardContext
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.request import Request
@@ -208,6 +209,10 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
# Propagated from scheduler to worker side via the connector metadata.
self._extra_async_saves: dict[str, int] = {}
@property
def sub_connectors(self) -> list[KVConnectorBase_V1]:
return list(self._connectors)
@property
def supports_divergent_local_hybrid_hits(self) -> bool:
return bool(self._connectors) and all(
@@ -248,6 +253,10 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
for c in self._connectors:
c.register_kv_caches(kv_caches)
def bind_kv_cache_manager(self, kv_cache_manager: "KVCacheManager") -> None:
for connector in self._connectors:
connector.bind_kv_cache_manager(kv_cache_manager)
def bind_gpu_block_pool(self, gpu_block_pool: "BlockPool") -> None:
for c in self._connectors:
c.bind_gpu_block_pool(gpu_block_pool)
@@ -292,6 +301,14 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
for c in self._connectors:
c.start_load_kv(forward_context, **kwargs)
def finish_forward(self) -> None:
for c in self._connectors:
c.finish_forward()
def reset_capture_state(self) -> None:
for c in self._connectors:
c.reset_capture_state()
def wait_for_layer_load(self, layer_name: str) -> None:
for c in self._connectors:
c.wait_for_layer_load(layer_name)
@@ -313,29 +330,31 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
finished_sending: set[str] = set()
finished_recving: set[str] = set()
for c in self._connectors:
sending, recving = c.get_finished(finished_req_ids)
if not recving and not sending:
continue
# Aggregate finished recving request ids.
finished_recving.update(recving or ())
# Aggregate finished sending request ids - only include
# once we've drained the "extra" count (for cases where
# more than one connector is async-saving the same request).
for req_id in sending or ():
results = self.get_transfer_results(finished_req_ids)
return (
results.finished_sending or None,
results.finished_recving or None,
)
def get_transfer_results(
self, finished_req_ids: set[str]
) -> KVConnectorTransferResults:
results = KVConnectorTransferResults()
for connector in self._connectors:
child_results = connector.get_transfer_results(finished_req_ids)
results.finished_recving.update(child_results.finished_recving)
results.failed_recving.update(child_results.failed_recving)
for req_id in child_results.finished_sending:
extra_pending = self._extra_async_saves.get(req_id)
if extra_pending is None:
finished_sending.add(req_id)
continue
assert extra_pending > 0
if extra_pending == 1:
del self._extra_async_saves[req_id]
results.finished_sending.add(req_id)
else:
self._extra_async_saves[req_id] = extra_pending - 1
return finished_sending or None, finished_recving or None
assert extra_pending > 0
if extra_pending == 1:
del self._extra_async_saves[req_id]
else:
self._extra_async_saves[req_id] = extra_pending - 1
return results
def get_block_ids_with_load_errors(self) -> set[int]:
agg_block_ids: set[int] = set()
@@ -563,6 +582,9 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
for c in self._connectors:
yield from c.take_events()
def has_pending_block_frees(self) -> bool:
return any(c.has_pending_block_frees() for c in self._connectors)
def has_pending_push_work(self) -> bool:
return any(c.has_pending_push_work() for c in self._connectors)
@@ -630,7 +652,7 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
return MultiKVConnectorStats(data=reconstructed_data)
def get_kv_connector_stats(self) -> MultiKVConnectorStats | None:
def get_kv_connector_stats(self) -> KVConnectorStats | None:
# Group connector stats by connector type.
stats_by_connector: MultiKVConnectorStats | None = None
for c in self._connectors:
@@ -93,10 +93,7 @@ class NixlBaseConnectorScheduler:
for g in kv_cache_config.transfer_groups
)
)
self._has_mamba = any(
isinstance(g.kv_cache_spec, MambaSpec)
for g in kv_cache_config.transfer_groups
)
self._has_mamba = kv_cache_config.has_mamba_layers
logger.info("Initializing NIXL Scheduler %s", engine_id)
if vllm_config.scheduler_config.disable_hybrid_kv_cache_manager:
@@ -2,8 +2,10 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Base worker-side logic for the NIXL connector."""
import contextlib
import itertools
import logging
import math
import os
import queue
import threading
@@ -31,7 +33,10 @@ from vllm.distributed.kv_transfer.kv_connector.utils import (
kv_postprocess_blksize_on_receive,
kv_postprocess_layout_on_receive,
)
from vllm.distributed.kv_transfer.kv_connector.v1.base import CopyBlocksOp
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
CopyBlocksOp,
KVConnectorTransferResults,
)
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
GET_META_MSG,
@@ -186,23 +191,28 @@ class NixlBaseConnectorWorker:
num_blocks = dst_num_blocks
if block_size_ratio is not None:
num_blocks = int(num_blocks * block_size_ratio)
num_regions = self.num_regions or len(self.block_len_per_layer)
if region_num_blocks is None:
region_num_blocks = [num_blocks] * self.num_regions
region_num_blocks = [num_blocks] * num_regions
elif block_size_ratio is not None:
region_num_blocks = [
int(count * block_size_ratio) for count in region_num_blocks
]
assert len(region_num_blocks) == self.num_regions
if region_num_blocks:
num_regions = len(region_num_blocks)
assert len(region_num_blocks) == num_regions
region_offsets = np.cumsum([0, *region_num_blocks[:-1]])
num_fa_descs = sum(region_num_blocks)
# All-attention fast path: single vectorized broadcast.
# All-attention fast path.
if num_ssm_regions == 0:
if region_group_ids is None:
region_group_ids = self.region_group_ids
if uses_region_group_mapping is None:
uses_region_group_mapping = self._uses_region_group_mapping
assert len(region_group_ids) == self.num_regions
if not region_group_ids and num_regions == 1:
region_group_ids = [0]
assert len(region_group_ids) == num_regions
if uses_region_group_mapping is None:
uses_region_group_mapping = len(set(region_group_ids)) > 1
region_group_ids_array = np.asarray(region_group_ids, dtype=np.int32)
@@ -397,8 +407,6 @@ class NixlBaseConnectorWorker:
# Config.
self.vllm_config = vllm_config
# mypy will complain on re-assignment otherwise.
self.block_size: int = cast(int, vllm_config.cache_config.block_size)
if vllm_config.kv_transfer_config is None:
raise ValueError("kv_transfer_config must be set for NixlConnector")
@@ -420,6 +428,17 @@ class NixlBaseConnectorWorker:
)
self.kv_cache_config = kv_cache_config
transfer_block_sizes = [
group.kv_cache_spec.block_size
for group in kv_cache_config.transfer_groups
if get_representative_spec_type(group.kv_cache_spec)
not in (MambaSpec, CircularBufferSpec)
]
self.block_size = (
math.lcm(*transfer_block_sizes)
if transfer_block_sizes
else cast(int, vllm_config.cache_config.block_size)
)
# Per-layer specs, unwrapping UniformTypeKVCacheSpecs group wrappers.
self._layer_specs: dict[str, KVCacheSpec] = {}
for group in kv_cache_config.transfer_groups:
@@ -599,12 +618,15 @@ class NixlBaseConnectorWorker:
# Number of NIXL regions. Currently one region per cache
# (so 1 per layer for MLA, otherwise 2 per layer)
self.num_regions = 0
self.region_mem_types: list[str] = []
self.region_group_ids: list[int] = []
self._uses_region_group_mapping = False
self.region_names: list[str] = []
self.region_num_blocks: list[int] = []
self.region_mem_types: list[str] = []
self._mixed_mem_types = False
self._desc_is_dram_by_block_size: dict[int, np.ndarray] = {}
self._desc_pos_by_block_size: dict[int, np.ndarray] = {}
self._dram_src_handles_by_block_size: dict[int, int] = {}
# PP>1 (push mode): this worker holds a contiguous layer slice and
# transfers into the matching sub-range of a PP=1 remote's regions.
@@ -644,9 +666,6 @@ class NixlBaseConnectorWorker:
self.dst_region_group_ids: dict[EngineId, list[int]] = {}
self.dst_uses_region_group_mapping: dict[EngineId, bool] = {}
self.dst_region_mem_types: dict[EngineId, list[str]] = {}
self._desc_is_dram_by_block_size: dict[int, np.ndarray] = {}
self._desc_pos_by_block_size: dict[int, np.ndarray] = {}
self._dram_src_handles_by_block_size: dict[int, int] = {}
self._registered_descs: list[Any] = []
# In progress transfers.
@@ -667,6 +686,7 @@ class NixlBaseConnectorWorker:
# Uses Queue for thread-safe cross-thread coordination with the
# background handshake thread, matching the _ready_requests pattern.
self._failed_recv_reqs: queue.Queue[ReqId] = queue.Queue()
self._failed_inflight_recvs: set[ReqId] = set()
self._pending_recv_notifs: dict[ReqId, list[tuple[str, bytes]]] = {}
# Handshake metadata of this worker for NIXL transfers.
@@ -690,7 +710,6 @@ class NixlBaseConnectorWorker:
"engine_ttl", 3600.0
)
self.block_size = vllm_config.cache_config.block_size
self.model_config = vllm_config.model_config
self.use_mla = self.model_config.use_mla
@@ -1240,7 +1259,7 @@ class NixlBaseConnectorWorker:
self.use_host_buffer,
)
registration_ranges: dict[tuple[int, str], tuple[int, int, int, str]] = {}
registration_ranges: dict[tuple[int, str], tuple[int, int, int]] = {}
region_mem_types: list[str] = []
seen_base_addresses: list[int] = []
self._ssm_region_indices = []
@@ -1280,10 +1299,16 @@ class NixlBaseConnectorWorker:
# K and V are packed into the content dim, so each attention layer is a
# single NIXL region whose block transfers as one unit. Mamba layers instead
# register separate conv/ssm sub-regions (see `_build_mamba_local`).
# P and D may allocate equivalent transferable layers in different
# cache-group orders. Keep their region lists aligned without putting
# layers.10 before layers.2, which would break PP region slicing.
layer_names = (
xfer_buffers
if self._is_csa_linear
else sorted(xfer_buffers, key=_region_sort_key)
else sorted(
xfer_buffers,
key=_region_sort_key,
)
)
for layer_name in layer_names:
cache = xfer_buffers[layer_name]
@@ -1299,6 +1324,7 @@ class NixlBaseConnectorWorker:
layer_name,
)
continue
group_index = self.kv_cache_config.transfer_group_index_by_layer[layer_name]
# `layer_spec.page_size_bytes` only accounts for logical page_size, that is
# the page_size assuming constant `self._logical_num_blocks`.
physical_page_size = (
@@ -1307,17 +1333,48 @@ class NixlBaseConnectorWorker:
else layer_spec.page_size_bytes
// self._physical_blocks_per_logical_kv_block
)
group_id = self.kv_cache_config.transfer_group_index_by_layer[layer_name]
group = self.kv_cache_config.transfer_groups[group_index]
if group.host_resident:
logical_num_blocks = self.kv_cache_config.hisparse_host_num_blocks
assert logical_num_blocks is not None
else:
logical_num_blocks = self.kv_cache_config.num_blocks
group_id = group_index
num_blocks = (
self._logical_num_blocks
logical_num_blocks
if isinstance(layer_spec, MambaSpec)
else self.num_blocks
else logical_num_blocks * self._physical_blocks_per_logical_kv_block
)
base_addr = cache.data_ptr()
is_mla_region = isinstance(
layer_spec, (MLAAttentionSpec, SlidingWindowMLASpec)
)
logger.debug(
"Registering layer %s with cache shape: %s", layer_name, cache.shape
)
storage = cache.untyped_storage()
storage_addr = storage.data_ptr()
is_host_resident = group.host_resident
if cache.device.type == "cpu":
mem_type = "DRAM"
region_device_id = 0
else:
mem_type = self.nixl_memory_type
region_device_id = max(cache.get_device(), 0)
self.device_id = region_device_id
if is_host_resident:
registration_base = cache.data_ptr()
registration_len = cache.nbytes
else:
registration_base = storage_addr
registration_len = storage.nbytes()
storage_key = (registration_base, mem_type)
if storage_key not in registration_ranges:
registration_ranges[storage_key] = (
registration_base,
registration_base + registration_len,
region_device_id,
)
if isinstance(layer_spec, KpoolTailSpec):
compressed_owner = compressed_region_owners.get(cache.data_ptr())
@@ -1345,22 +1402,6 @@ class NixlBaseConnectorWorker:
)
continue
if cache.device.type == "cpu":
mem_type = "DRAM"
region_device_id = 0
else:
mem_type = self.nixl_memory_type
region_device_id = max(cache.get_device(), 0)
self.device_id = region_device_id
registration_ranges.setdefault(
(storage_addr, mem_type),
(storage_addr, storage.nbytes(), region_device_id, ""),
)
is_mla_region = isinstance(
layer_spec, (MLAAttentionSpec, SlidingWindowMLASpec)
)
if isinstance(layer_spec, MambaSpec):
physical_ratio = self._physical_blocks_per_logical_kv_block
block_len = physical_page_size // physical_ratio
@@ -1381,14 +1422,13 @@ class NixlBaseConnectorWorker:
block_stride = physical_page_size
else:
block_stride = cache.stride(0) * cache.element_size()
storage_is_block_major = num_blocks * block_stride == storage.nbytes()
# A layer whose [H, N, C] interior is dense addresses its own page
# as one chunk. Otherwise the block's whole row is the only
# contiguous transfer unit.
hnc_contiguous = (
cache.ndim == 4
and cache.stride(2) == cache.shape[3]
and cache.stride(1) == cache.shape[2] * cache.shape[3]
storage_is_block_major = num_blocks * block_stride == registration_len
# A view with a contiguous page can own a strided region even
# when sibling layers share the same backing allocation.
page_contiguous = (
cache.ndim > 1
and cache[0].is_contiguous()
and cache[0].nbytes == physical_page_size
)
virtual_transfer_pages = _uses_dense_virtual_transfer_pages(
layer_spec, cache, physical_page_size, num_blocks
@@ -1399,13 +1439,14 @@ class NixlBaseConnectorWorker:
(cache.data_ptr(), physical_page_size, physical_page_size)
]
elif storage_is_block_major and (
(packed_storage and is_mla_region)
or (not hnc_contiguous and not self._is_csa_linear)
not page_contiguous
and ((packed_storage and is_mla_region) or not self._is_csa_linear)
):
# Packed MLA layouts transfer the complete storage row.
storage_block_len = storage.nbytes() // num_blocks
# TODO(Lucas): handle TP slicing for packed_storage; for now
# restrict to MLA (DSv4) where kv is replicated.
storage_block_len = registration_len // num_blocks
region_specs = [
(storage_addr, storage_block_len, storage_block_len)
(registration_base, storage_block_len, storage_block_len)
]
elif storage_is_block_major:
region_specs = [
@@ -1440,6 +1481,10 @@ class NixlBaseConnectorWorker:
region_index = seen_base_addresses.index(base_addr)
assert region_mem_types[region_index] == mem_type
self._region_is_mla[region_index] |= is_mla_region
if is_mla_region:
self.block_len_per_layer[region_index] = block_len
self.block_stride_per_layer[region_index] = block_stride
self.region_num_blocks[region_index] = num_blocks
if self.region_group_ids[region_index] != group_id:
self.region_group_ids[region_index] = _SHARED_REGION_GROUP_ID
else:
@@ -1450,8 +1495,8 @@ class NixlBaseConnectorWorker:
self.region_group_ids.append(group_id)
self.region_names.append(layer_name)
self.region_num_blocks.append(num_blocks)
region_mem_types.append(mem_type)
self._region_is_mla.append(is_mla_region)
region_mem_types.append(mem_type)
if isinstance(layer_spec, MambaSpec):
if layer_spec.tp_replicated:
@@ -1496,7 +1541,6 @@ class NixlBaseConnectorWorker:
== len(self.region_group_ids)
== len(self.region_names)
== len(self.region_num_blocks)
== len(region_mem_types)
)
# Descriptor ids must be region-ordered, matching the remote side.
self._scratch_region_indices.sort()
@@ -1521,16 +1565,23 @@ class NixlBaseConnectorWorker:
# Total local FA descriptors (boundary between FA and mamba descs).
self.num_descs = sum(self.region_num_blocks)
self._mixed_mem_types = len(set(region_mem_types)) > 1
if self._mixed_mem_types:
assert self.use_mla and not self._has_mamba, (
"Mixed-device KV registration is only supported for MLA "
"models without Mamba layers."
)
for mem_type in sorted(set(region_mem_types)):
ranges = [
registration
for (
_,
registration_mem_type,
), registration in registration_ranges.items()
if registration_mem_type == mem_type
ranges_for_mem_type = [
(start, end - start, device_id, "")
for (_, cache_mem_type), (
start,
end,
device_id,
) in registration_ranges.items()
if cache_mem_type == mem_type
]
descs = self.nixl_wrapper.get_reg_descs(ranges, mem_type)
descs = self.nixl_wrapper.get_reg_descs(ranges_for_mem_type, mem_type)
self.nixl_wrapper.register_memory(descs, backends=self.nixl_backends)
self._registered_descs.append(descs)
@@ -1854,7 +1905,11 @@ class NixlBaseConnectorWorker:
if self._mixed_mem_types:
desc_is_dram = np.concatenate(
[
np.full(count * block_size_ratio, mem_type == "DRAM", dtype=bool)
np.full(
count * block_size_ratio,
mem_type == "DRAM",
dtype=bool,
)
for mem_type, count in zip(
self.region_mem_types,
self.region_num_blocks,
@@ -1864,24 +1919,29 @@ class NixlBaseConnectorWorker:
)
assert len(desc_is_dram) == len(blocks_data)
desc_pos = np.empty(len(desc_is_dram), dtype=np.int64)
dram_indices = np.flatnonzero(desc_is_dram)
device_indices = np.flatnonzero(~desc_is_dram)
desc_pos[dram_indices] = np.arange(len(dram_indices), dtype=np.int64)
desc_pos[device_indices] = np.arange(len(device_indices), dtype=np.int64)
dram_idx = np.where(desc_is_dram)[0]
vram_idx = np.where(~desc_is_dram)[0]
desc_pos[dram_idx] = np.arange(len(dram_idx), dtype=np.int64)
desc_pos[vram_idx] = np.arange(len(vram_idx), dtype=np.int64)
self._desc_is_dram_by_block_size[block_size] = desc_is_dram
self._desc_pos_by_block_size[block_size] = desc_pos
blocks_data[desc_is_dram, 2] = 0
dram_descs = self.nixl_wrapper.get_xfer_descs(
blocks_data[desc_is_dram], "DRAM"
)
# DRAM descriptors are registered under CPU device 0.
blocks_data = [
(addr, length, 0) if is_dram else (addr, length, dev)
for (addr, length, dev), is_dram in zip(
blocks_data, desc_is_dram, strict=True
)
]
dram_blocks = [blocks_data[i] for i in dram_idx]
vram_blocks = [blocks_data[i] for i in vram_idx]
dram_descs = self.nixl_wrapper.get_xfer_descs(dram_blocks, "DRAM")
self._dram_src_handles_by_block_size[block_size] = (
self.nixl_wrapper.prep_xfer_dlist("NIXL_INIT_AGENT", dram_descs)
)
device_descs = self.nixl_wrapper.get_xfer_descs(
blocks_data[~desc_is_dram], self.nixl_memory_type
)
descs = self.nixl_wrapper.get_xfer_descs(vram_blocks, self.nixl_memory_type)
return (
self.nixl_wrapper.prep_xfer_dlist("NIXL_INIT_AGENT", device_descs),
self.nixl_wrapper.prep_xfer_dlist("NIXL_INIT_AGENT", descs),
blocks_data,
)
@@ -1968,10 +2028,6 @@ class NixlBaseConnectorWorker:
]
nixl_agent_meta.block_lens = nixl_agent_meta.block_lens[start:end]
nixl_agent_meta.block_strides = nixl_agent_meta.block_strides[start:end]
if nixl_agent_meta.region_num_blocks is not None:
nixl_agent_meta.region_num_blocks = nixl_agent_meta.region_num_blocks[
start:end
]
if nixl_agent_meta.region_group_ids is not None:
nixl_agent_meta.region_group_ids = nixl_agent_meta.region_group_ids[
start:end
@@ -1982,6 +2038,10 @@ class NixlBaseConnectorWorker:
nixl_agent_meta.region_mem_types = nixl_agent_meta.region_mem_types[
start:end
]
if nixl_agent_meta.region_num_blocks is not None:
nixl_agent_meta.region_num_blocks = nixl_agent_meta.region_num_blocks[
start:end
]
### Register remote engine in TransferTopology (idempotent).
assert self.transfer_topo is not None
@@ -2026,11 +2086,12 @@ class NixlBaseConnectorWorker:
nixl_agent_meta.region_num_blocks
or [nixl_agent_meta.num_blocks] * num_remote_regions
)
self.dst_region_group_ids[engine_id] = nixl_agent_meta.region_group_ids or (
remote_region_group_ids = nixl_agent_meta.region_group_ids or (
self.region_group_ids
if len(self.region_group_ids) == num_remote_regions
else [0] * num_remote_regions
)
self.dst_region_group_ids[engine_id] = remote_region_group_ids
self.dst_uses_region_group_mapping[engine_id] = (
len(set(self.dst_region_group_ids[engine_id])) > 1
)
@@ -2138,8 +2199,6 @@ class NixlBaseConnectorWorker:
mamba = self._build_mamba_remote(nixl_agent_meta, tp_ratio, transfer_info)
blocks_data = np.concatenate([blocks_data, mamba])
# A mixed local destination is split into multiple READs below. The
# producer remains a single descriptor list in this prerequisite.
remote_mem_types = set(self.dst_region_mem_types[engine_id])
if len(remote_mem_types) != 1:
raise NotImplementedError(
@@ -2540,9 +2599,10 @@ class NixlBaseConnectorWorker:
indices=indices,
)
def get_finished(self) -> tuple[set[str], set[str]]:
def get_transfer_results(self) -> KVConnectorTransferResults:
"""
Get requests that are done sending or recving on this specific worker.
Get transfers that completed on this specific worker.
The scheduler process (via the MultiprocExecutor) will use this output
to track which workers are done.
"""
@@ -2564,7 +2624,6 @@ class NixlBaseConnectorWorker:
# Add failed requests to done_recving for scheduler tracking
# (blocks are already marked invalid, scheduler will handle recompute)
done_recving.update(failed_recv_reqs)
if len(done_sending) > 0 or len(done_recving) > 0:
logger.debug(
"Rank %s, get_finished: %s requests done sending "
@@ -2577,6 +2636,7 @@ class NixlBaseConnectorWorker:
block_ids_for_blocksize_post_process = defaultdict(list)
block_ids_for_heterogeneous_attn_post_process = list[list[int]]()
direct_device_recving = set[str]()
for req_id in done_recving:
# clean up metadata for completed requests
meta = self._recving_metadata.pop(req_id, None)
@@ -2594,6 +2654,8 @@ class NixlBaseConnectorWorker:
if self.use_host_buffer:
self.sync_recved_kv_to_device(req_id, meta)
direct_device_recving.add(req_id)
# Post processing for heteroblocksize/layout, and for blocks the
# transfer clipped. The latter happens either at remote-block
# granularity (block_size_ratio > 1) or at kernel-block
@@ -2642,7 +2704,7 @@ class NixlBaseConnectorWorker:
for block_ids in block_ids_for_heterogeneous_attn_post_process:
self.post_process_device_kv_on_receive_heterogeneous_attn(block_ids)
self._sync_device_after_mamba_recv(done_recving, failed_recv_reqs)
self._sync_device_after_direct_recv(direct_device_recving)
# Handle timeout to avoid stranding blocks on remote.
now = time.perf_counter()
@@ -2664,20 +2726,21 @@ class NixlBaseConnectorWorker:
del self._reqs_to_send[req_id]
done_sending.add(req_id)
return done_sending, done_recving
return KVConnectorTransferResults(
finished_sending=done_sending,
finished_recving=done_recving,
failed_recving=failed_recv_reqs,
)
def _sync_device_after_mamba_recv(
self,
done_recving: set[str],
failed_recv_reqs: set[str],
) -> None:
"""Synchronize ROCm direct-GPU Mamba receives before model execution."""
if (
not current_platform.is_rocm()
or not self._has_mamba
or self.use_host_buffer
or not (done_recving - failed_recv_reqs)
):
def get_finished(self) -> tuple[set[str], set[str]]:
"""Compatibility wrapper for the legacy completion API."""
results = self.get_transfer_results()
return results.finished_sending, results.finished_recving
def _sync_device_after_direct_recv(self, done_recving: set[str]) -> None:
"""Make direct NIXL writes visible before model execution."""
requires_sync = current_platform.is_rocm() and self._has_mamba
if self.use_host_buffer or not done_recving or not requires_sync:
return
torch.accelerator.synchronize()
@@ -2731,11 +2794,11 @@ class NixlBaseConnectorWorker:
self.nixl_wrapper.release_xfer_handle(handle)
elif xfer_state == "PROC":
in_progress.append(handle)
continue
else:
self._log_failure(
failure_type="transfer_failed",
msg="Marking blocks as invalid",
msg="Deferring request completion until its last "
"xfer is terminal",
req_id=req_id,
xfer_state=xfer_state,
)
@@ -2743,21 +2806,20 @@ class NixlBaseConnectorWorker:
except Exception as e:
self._log_failure(
failure_type="transfer_exception",
msg="Marking blocks as invalid",
msg="Handle is unpollable; treating it as terminal",
req_id=req_id,
error=e,
)
self._handle_failed_transfer(req_id, handle)
if not in_progress:
# Only report request as completed when all transfers are done.
# A request failed in an earlier poll was already reported via
# _failed_recv_reqs and its metadata popped by get_finished();
# don't report it again, just drop the remaining handles.
if req_id in self._recving_metadata:
done_req_ids.add(req_id)
self._send_pending_recv_notifs(req_id)
del transfers[req_id]
done_req_ids.add(req_id)
if req_id in self._failed_inflight_recvs:
self._failed_inflight_recvs.remove(req_id)
self._report_failed_recv(req_id)
else:
self._send_pending_recv_notifs(req_id)
else:
transfers[req_id] = in_progress
return done_req_ids
@@ -2771,22 +2833,25 @@ class NixlBaseConnectorWorker:
req_id: The request ID.
handle: The transfer handle.
"""
# (multi-read) One handle is created per remote rank, and they do not
# all fail in the same _pop_done_transfers poll. The request is
# reported failed on the first one, which pops its metadata in
# get_finished(); on the later failures only the handle cleanup is left.
# TODO (NickLucche) handle failed transfer for HMA.
# A split READ's notification is sent only after every handle succeeds.
# A sibling READ may still be writing these blocks. Retain metadata
# and defer invalidation until every handle is terminal.
self._pending_recv_notifs.pop(req_id, None)
if (meta := self._recving_metadata.get(req_id)) is not None:
if not self._is_hma_required:
self._invalid_block_ids.put(set(meta.local_block_ids[0]))
self._failed_recv_reqs.put(req_id)
if req_id in self._recving_transfers:
self._failed_inflight_recvs.add(req_id)
else:
self._report_failed_recv(req_id)
if handle is not None:
self.nixl_wrapper.release_xfer_handle(handle)
self.xfer_stats.record_failed_transfer()
def _report_failed_recv(self, req_id: str) -> None:
if (meta := self._recving_metadata.get(req_id)) is not None:
if not self._is_hma_required:
self._invalid_block_ids.put(set(meta.local_block_ids[0]))
self._failed_recv_reqs.put(req_id)
def _send_pending_recv_notifs(self, req_id: str) -> None:
"""Send notifications deferred by split DRAM/VRAM reads."""
for agent_name, notif_id in self._pending_recv_notifs.pop(req_id, []):
try:
self.nixl_wrapper.send_notif(agent_name, notif_msg=notif_id)
@@ -2909,10 +2974,10 @@ class NixlBaseConnectorWorker:
return block_ids
block_arange = np.arange(0, ratio).reshape(1, -1)
# Mamba blocks have no logical<>physical discrepancy (block-size=1)
group_specs = self.kv_cache_config.transfer_groups
physical_block_ids = []
for i, group in enumerate(block_ids):
if _is_ssm_spec(get_representative_spec_type(group_specs[i].kv_cache_spec)):
spec = self.kv_cache_config.transfer_groups[i].kv_cache_spec
if _is_ssm_spec(get_representative_spec_type(spec)):
physical_block_ids.append(group)
else:
physical_block_ids.append(
@@ -3215,9 +3280,44 @@ class NixlBaseConnectorWorker:
)
def __del__(self):
self.shutdown()
with contextlib.suppress(Exception):
self.shutdown()
def shutdown(self):
def _finish_shutdown(self) -> None:
self._recving_transfers.clear()
try:
for handle in self.src_xfer_handles_by_block_size.values():
self.nixl_wrapper.release_dlist_handle(handle)
for handles in self.src_xfer_handles_by_tp_ratio.values():
for handle in handles:
self.nixl_wrapper.release_dlist_handle(handle)
for handles in self._dram_src_handles_by_tp_ratio.values():
for handle in handles:
self.nixl_wrapper.release_dlist_handle(handle)
for handle in self._dram_src_handles_by_block_size.values():
self.nixl_wrapper.release_dlist_handle(handle)
except Exception:
logger.exception("NIXL dlist-handle release failed at shutdown.")
self.src_xfer_handles_by_block_size.clear()
self.src_xfer_handles_by_tp_ratio.clear()
self._dram_src_handles_by_tp_ratio.clear()
self._dram_src_handles_by_block_size.clear()
try:
for engine_id in list(self._remote_agents):
self._cleanup_remote_engine(engine_id, log_eviction=False)
except Exception:
logger.exception("NIXL remote-engine cleanup failed at shutdown.")
try:
for desc in self._registered_descs:
self.nixl_wrapper.deregister_memory(desc)
finally:
self._registered_descs.clear()
# Drop cache references before their owners release registered
# host memory; handshake futures may outlive model-runner shutdown.
self.device_kv_caches = {}
self.host_xfer_buffers = {}
def shutdown(self) -> None:
"""Shutdown the connector worker."""
if not hasattr(self, "_handshake_initiation_executor"):
# error happens during init, no need to shutdown
@@ -3227,22 +3327,4 @@ class NixlBaseConnectorWorker:
for handle in handles:
self.nixl_wrapper.release_xfer_handle(handle)
self._recving_transfers.clear()
for handle in self.src_xfer_handles_by_block_size.values():
self.nixl_wrapper.release_dlist_handle(handle)
self.src_xfer_handles_by_block_size.clear()
for handles in self.src_xfer_handles_by_tp_ratio.values():
for handle in handles:
self.nixl_wrapper.release_dlist_handle(handle)
self.src_xfer_handles_by_tp_ratio.clear()
for handles in self._dram_src_handles_by_tp_ratio.values():
for handle in handles:
self.nixl_wrapper.release_dlist_handle(handle)
self._dram_src_handles_by_tp_ratio.clear()
for handle in self._dram_src_handles_by_block_size.values():
self.nixl_wrapper.release_dlist_handle(handle)
self._dram_src_handles_by_block_size.clear()
for engine_id in list(self._remote_agents):
self._cleanup_remote_engine(engine_id, log_eviction=False)
for desc in self._registered_descs:
self.nixl_wrapper.deregister_memory(desc)
self._registered_descs.clear()
self._finish_shutdown()
@@ -26,6 +26,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorHandshakeMetadata,
KVConnectorMetadata,
KVConnectorRole,
KVConnectorTransferResults,
SupportsHMA,
)
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
@@ -234,6 +235,18 @@ class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA):
assert self.connector_worker is not None
return self.connector_worker.get_finished()
def get_transfer_results(
self, finished_req_ids: set[str]
) -> KVConnectorTransferResults:
assert self.connector_worker is not None
results = self.connector_worker.get_transfer_results()
if (
self.kv_transfer_config.kv_role == "kv_producer"
and self.connector_worker.pcp_rank > 0
):
results.finished_sending.clear()
return results
def get_block_ids_with_load_errors(self) -> set[int]:
"""Get block IDs that failed to load via NIXL."""
assert self.connector_worker is not None
@@ -305,7 +305,9 @@ class NixlConnectorMetadata(KVConnectorMetadata):
local_num_computed_blocks: tuple[int, ...] = (),
):
req = self._add_new_req(
local_block_ids, kv_transfer_params, local_num_computed_blocks
local_block_ids,
kv_transfer_params,
local_num_computed_blocks,
)
req.remote = RemoteMeta(
block_ids=kv_transfer_params["remote_block_ids"],
@@ -166,11 +166,10 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
dcp_active = self.dcp_size > 1 or remote_info.remote_dcp_size > 1
local_block_ids = meta.local_physical_block_ids
local_region_groups = getattr(self, "region_group_ids", [])
remote_region_groups = getattr(self, "dst_region_group_ids", {}).get(
engine_id, local_region_groups
)
if local_region_groups != remote_region_groups:
remote_region_groups = self.dst_region_group_ids[engine_id]
local_region_groups = self.region_group_ids or remote_region_groups
groups_differ = local_region_groups != remote_region_groups
if groups_differ:
if not self.use_mla or self._has_mamba:
raise NotImplementedError(
"Different NIXL cache-group layouts are only supported for "
@@ -292,6 +291,9 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
spec.remote_rank
]
# Once a read routes the request to failure reporting, the
# scheduler may free and reuse its blocks, so no sibling READs
# may be posted (and P must not be notified).
if not self._read_blocks(
read_spec=spec,
request_id=req_id,
@@ -329,6 +331,10 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
"""
Post a READ point-to-point xfer request from a single local worker to
a single remote worker.
Returns True when the read was posted (or was unnecessary), False
when the request was routed to failure reporting — the caller must
not post further transfers for it.
"""
assert self.transfer_topo is not None
remote_rank = read_spec.remote_rank
@@ -416,11 +422,11 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
dst_num_blocks=self.dst_num_blocks[dst_engine_id],
block_size_ratio=None,
physical_blocks_per_logical=remote_info.remote_physical_blocks_per_logical,
region_num_blocks=self.dst_region_num_blocks[dst_engine_id],
region_num_blocks=(self.dst_region_num_blocks.get(dst_engine_id) or None),
region_group_ids=(
list(range(self.num_regions))
if read_spec.block_ids_by_region
else self.dst_region_group_ids[dst_engine_id]
else (self.dst_region_group_ids.get(dst_engine_id) or None)
),
uses_region_group_mapping=(
self.num_regions > 1
@@ -433,11 +439,11 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
dst_num_blocks=self.dst_num_blocks[self.engine_id],
block_size_ratio=block_size_ratio,
physical_blocks_per_logical=self._physical_blocks_per_logical_kv_block,
region_num_blocks=self.dst_region_num_blocks[self.engine_id],
region_num_blocks=(self.dst_region_num_blocks.get(self.engine_id) or None),
region_group_ids=(
list(range(self.num_regions))
if read_spec.block_ids_by_region
else self.region_group_ids
else (self.region_group_ids or None)
),
uses_region_group_mapping=(
self.num_regions > 1
@@ -537,11 +543,11 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
self._pending_recv_notifs.setdefault(request_id, []).append(
(notif_agent, notif_id)
)
for index, handle in enumerate(handles):
for i, handle in enumerate(handles):
try:
self.nixl_wrapper.transfer(handle)
except Exception:
for unstarted in handles[index:]:
for unstarted in handles[i:]:
self.nixl_wrapper.release_xfer_handle(unstarted)
raise
self._recving_transfers[request_id].append(handle)
@@ -41,6 +41,9 @@ from typing import TYPE_CHECKING, Any
import msgspec
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorTransferResults,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker import (
NixlBaseConnectorWorker,
)
@@ -145,7 +148,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
self._push_writer_thread.start()
logger.info("nixl-push-writer thread started (rank=%d)", self.tp_rank)
def shutdown(self):
def shutdown(self) -> None:
self._push_writer_stop.set()
# Unblock the writer if it's waiting in the no-active-state branch.
self._push_writer_wake.set()
@@ -522,7 +525,14 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
)
remote_block_ids = meta.remote.block_ids
local_block_ids = meta.local_physical_block_ids
num_groups = len(local_block_ids)
local_region_groups = self.region_group_ids
remote_region_groups = self.dst_region_group_ids[engine_id]
groups_differ = local_region_groups != remote_region_groups
if groups_differ:
raise NotImplementedError(
"NixlPushConnector does not support different producer and "
"consumer cache-group layouts"
)
# MLA latent is replicated across D's TP ranks: the tp-mapping
# collapses it to one rank (fine for reads), but push must WRITE every
@@ -537,6 +547,8 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
else:
write_ranks = list(plan.all_source_ranks)
num_groups = len(local_block_ids)
def group_ids(block_ids: BlockIds, rank: int) -> BlockIds:
return [
list(block_ids[g])
@@ -777,13 +789,14 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
self._reqs_to_send.pop(req_id, None)
return notified_req_ids
def get_finished(self) -> tuple[set[str], set[str]]:
def get_transfer_results(self) -> KVConnectorTransferResults:
# Engine main thread asking for completions: also wake the writer
# so it gets a chance to drain NIXL notifs (heartbeats, completion
# notifs, late PUSH_REGs) even if it had been parked.
self._push_writer_wake.set()
done_sending, done_recving = super().get_finished()
results = super().get_transfer_results()
done_sending = results.finished_sending
# ``_pop_done_transfers`` mutates ``_sending_transfers``; the
# writer thread also appends to it, so guard the pop.
@@ -803,4 +816,4 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
if done_sending:
self._push_writer_wake.set()
return done_sending, done_recving
return results
@@ -12,6 +12,7 @@ from vllm.v1.core.kv_cache_utils import (
from vllm.v1.kv_cache_interface import (
AttentionSpec,
FullAttentionSpec,
KVCacheGroupRole,
KVCacheSpec,
MLAAttentionSpec,
SlidingWindowMLASpec,
@@ -31,6 +32,16 @@ if TYPE_CHECKING:
from vllm.v1.kv_cache_interface import KVCacheConfig
def get_offloading_group_ids(kv_cache_config: "KVCacheConfig") -> tuple[int, ...]:
if kv_cache_config.hisparse_host_num_blocks is None:
return tuple(range(len(kv_cache_config.kv_cache_groups)))
return tuple(
group_id
for group_id, group in enumerate(kv_cache_config.kv_cache_groups)
if group.role is KVCacheGroupRole.HISPARSE_INDEXER
)
def build_offloading_config(
vllm_config: "VllmConfig",
kv_cache_config: "KVCacheConfig",
@@ -43,15 +54,22 @@ def build_offloading_config(
engine_id = kv_transfer_config.engine_id
parallel_config = vllm_config.parallel_config
selected_groups = tuple(
(group_id, kv_cache_config.kv_cache_groups[group_id])
for group_id in get_offloading_group_ids(kv_cache_config)
)
if not selected_groups:
raise ValueError("KV offloading found no eligible cache groups.")
groups = tuple(
OffloadingGroupConfig(
group_id=group_id,
tokens_per_block=resolve_dcp_kv_block_size(
group.kv_cache_spec,
parallel_config.decode_context_parallel_size,
),
layer_names=tuple(group.layer_names),
)
for group in kv_cache_config.kv_cache_groups
for group_id, group in selected_groups
)
_, tokens_per_hash = resolve_kv_cache_block_sizes(kv_cache_config, vllm_config)
@@ -103,11 +121,20 @@ def build_offloading_config(
)
worker_kv_bytes_per_block = 0
if kv_cache_config.num_blocks > 0 and kv_cache_config.kv_cache_tensors:
all_groups_selected = len(selected_groups) == len(kv_cache_config.kv_cache_groups)
if (
all_groups_selected
and kv_cache_config.num_blocks > 0
and kv_cache_config.kv_cache_tensors
):
# Every KVCacheTensor describes placement within the same backing allocation,
# so its size is the total, not a per-tensor share.
total_gpu_kv_bytes = kv_cache_config.kv_cache_tensors[0].size
worker_kv_bytes_per_block = total_gpu_kv_bytes // kv_cache_config.num_blocks
elif kv_cache_config.num_blocks > 0:
worker_kv_bytes_per_block = sum(
group.kv_cache_spec.page_size_bytes for _, group in selected_groups
)
single_group_spec = (
kv_cache_config.kv_cache_groups[0].kv_cache_spec
@@ -152,8 +152,9 @@ def resolve_mamba_align_size(
such groups agree on the same value.
"""
mamba_align_size: int | None = None
for idx, tokens_per_block in enumerate(spec.tokens_per_block):
kv_spec = kv_cache_config.kv_cache_groups[idx].kv_cache_spec
for group in spec.config.groups:
tokens_per_block = group.tokens_per_block
kv_spec = kv_cache_config.kv_cache_groups[group.group_id].kv_cache_spec
if isinstance(kv_spec, MambaSpec) and kv_spec.mamba_cache_mode in (
"align",
"all",
@@ -188,8 +189,9 @@ class SchedulerOffloadConfig(NamedTuple):
# each segment can never serve a load hit. Relevant for hybrid
# architectures like DeepSeek V4 (MLA + SWA groups).
full_attn_tokens_per_chunk: set[int] = set()
for idx, tokens_per_block in enumerate(spec.tokens_per_block):
kv_spec = kv_cache_config.kv_cache_groups[idx].kv_cache_spec
for group in spec.config.groups:
tokens_per_block = group.tokens_per_block
kv_spec = kv_cache_config.kv_cache_groups[group.group_id].kv_cache_spec
sw = get_sliding_window_size_in_chunks(
kv_spec, tokens_per_block * spec.blocks_per_chunk
)
@@ -214,9 +216,9 @@ class SchedulerOffloadConfig(NamedTuple):
retention_interval = vllm_config.cache_config.prefix_cache_retention_interval
eagle_groups = {
idx
for idx, g in enumerate(kv_cache_config.kv_cache_groups)
if g.is_eagle_group
group.group_id
for group in spec.config.groups
if kv_cache_config.kv_cache_groups[group.group_id].is_eagle_group
}
use_eagle_block_drop = (
@@ -249,8 +251,10 @@ class SchedulerOffloadConfig(NamedTuple):
)
kv_group_configs_list: list[GroupOffloadConfig] = []
for idx, tokens_per_block in enumerate(spec.tokens_per_block):
kv_cache_group = kv_cache_config.kv_cache_groups[idx]
for group in spec.config.groups:
group_id = group.group_id
tokens_per_block = group.tokens_per_block
kv_cache_group = kv_cache_config.kv_cache_groups[group_id]
kv_spec = kv_cache_group.kv_cache_spec
sw = get_sliding_window_size_in_chunks(
kv_spec, tokens_per_block * spec.blocks_per_chunk
@@ -261,7 +265,7 @@ class SchedulerOffloadConfig(NamedTuple):
)
kv_group_configs_list.append(
GroupOffloadConfig(
group_idx=idx,
group_idx=group_id,
tokens_per_block=tokens_per_block,
tokens_per_chunk=tokens_per_block * spec.blocks_per_chunk,
hashes_per_chunk=(
@@ -272,7 +276,7 @@ class SchedulerOffloadConfig(NamedTuple):
kv_cache_spec=kv_spec,
manager_cls=manager_cls,
kv_event_group_spec=get_offloading_event_group_spec(kv_cache_group),
is_eagle_group=idx in eagle_groups,
is_eagle_group=group_id in eagle_groups,
requires_cow_source=(
isinstance(kv_spec, MambaSpec)
and kv_spec.mamba_cache_mode == "align"
@@ -408,9 +412,10 @@ class RequestOffloadState:
if new_block_id_groups is None:
return
assert len(new_block_id_groups) == len(self.group_states)
for group_state, new_blocks in zip(self.group_states, new_block_id_groups):
group_state.block_ids.extend(new_blocks)
for group_config, group_state in zip(
self.config.kv_group_configs, self.group_states
):
group_state.block_ids.extend(new_block_id_groups[group_config.group_idx])
def storable_chunks(
self,
@@ -541,11 +546,11 @@ class OffloadingConnectorScheduler:
full_attention_groups: list[int] = []
sliding_window_groups: list[int] = []
for group_config in self.config.kv_group_configs:
for config_idx, group_config in enumerate(self.config.kv_group_configs):
if group_config.sliding_window_size_in_chunks is None:
full_attention_groups.append(group_config.group_idx)
full_attention_groups.append(config_idx)
else:
sliding_window_groups.append(group_config.group_idx)
sliding_window_groups.append(config_idx)
# sort sliding window groups by window size in decreasing order
def _sliding_window_sort_key(i: int) -> int:
@@ -737,7 +742,11 @@ class OffloadingConnectorScheduler:
req_status.req_context,
)
def _lookup_complete_chunks(self, req_status: RequestOffloadState) -> int | None:
def _lookup_complete_chunks(
self,
req_status: RequestOffloadState,
max_num_new_tokens: int | None = None,
) -> int | None:
"""
Find how many tokens beyond num_locally_computed_tokens can be loaded.
@@ -748,6 +757,10 @@ class OffloadingConnectorScheduler:
"""
num_computed_tokens = req_status.num_locally_computed_tokens
max_hit_size_tokens: int = req_status.req.num_tokens
if max_num_new_tokens is not None:
max_hit_size_tokens = min(
max_hit_size_tokens, num_computed_tokens + max_num_new_tokens
)
if self._sliding_window_groups:
# the last prompt token has to be recomputed to get the logprobs
# for sliding window attention, we must reduce by 1 to make sure
@@ -929,8 +942,12 @@ class OffloadingConnectorScheduler:
hash_idx = boundary_tokens // self.config.tokens_per_hash - 1
return make_offload_key(request.block_hashes[hash_idx], group_idx)
def _lookup(self, req_status: RequestOffloadState) -> int | None:
complete_hit = self._lookup_complete_chunks(req_status)
def _lookup(
self,
req_status: RequestOffloadState,
max_num_new_tokens: int | None = None,
) -> int | None:
complete_hit = self._lookup_complete_chunks(req_status, max_num_new_tokens)
req_status.partial_tail_boundary = None
if complete_hit is None or not self.config.supports_partial_tail:
return complete_hit
@@ -939,9 +956,10 @@ class OffloadingConnectorScheduler:
complete_boundary = local_tokens + complete_hit
tokens_per_hash = self.config.tokens_per_hash
block_end = complete_boundary + self._partial_tail_block_size
max_boundary = round_down(
min(req_status.req.num_prompt_tokens - 1, block_end - 1), tokens_per_hash
)
max_boundary = min(req_status.req.num_prompt_tokens - 1, block_end - 1)
if max_num_new_tokens is not None:
max_boundary = min(max_boundary, local_tokens + max_num_new_tokens)
max_boundary = round_down(max_boundary, tokens_per_hash)
if max_boundary <= complete_boundary:
return complete_hit
@@ -990,7 +1008,10 @@ class OffloadingConnectorScheduler:
self._req_status[request.request_id] = req_status
def get_num_new_matched_tokens(
self, request: Request, num_computed_tokens: int
self,
request: Request,
num_computed_tokens: int,
max_num_new_tokens: int | None = None,
) -> tuple[int | None, bool]:
"""
Get number of new tokens that can be loaded beyond the
@@ -1030,7 +1051,7 @@ class OffloadingConnectorScheduler:
num_hit_tokens = 0
else:
lookup_start = time.monotonic()
num_hit_tokens = self._lookup(req_status)
num_hit_tokens = self._lookup(req_status, max_num_new_tokens)
self._connector_stats.observe_histogram(
_ConnectorMetricName.LOOKUP_SYNC_DELAY,
time.monotonic() - lookup_start,
@@ -1065,11 +1086,11 @@ class OffloadingConnectorScheduler:
# per group
group_sizes: list[int] = []
block_indices: list[int] = []
for group_config, group_state, group_blocks in zip(
for group_config, group_state in zip(
self.config.kv_group_configs,
req_status.group_states,
blocks.blocks,
):
group_blocks = blocks.blocks[group_config.group_idx]
self._current_batch_allocated_block_ids.update(
block.block_id for block in group_blocks if block.block_id != 0
)
@@ -1184,7 +1205,8 @@ class OffloadingConnectorScheduler:
for grp_idx in self._sliding_window_groups
)
req_status.update_block_id_groups(new_block_id_groups)
for new_blocks in new_block_id_groups:
for group_config in self.config.kv_group_configs:
new_blocks = new_block_id_groups[group_config.group_idx]
for bid in new_blocks:
if bid != 0:
self._current_batch_allocated_block_ids.add(bid)
@@ -1218,6 +1240,10 @@ class OffloadingConnectorScheduler:
) -> dict[int, TransferJob]:
store_jobs: dict[int, TransferJob] = {}
num_groups = len(self.config.kv_group_configs)
config_idx_by_group = {
config.group_idx: idx
for idx, config in enumerate(self.config.kv_group_configs)
}
for req_id, entries in handoffs.items():
req_status = self._req_status.get(req_id)
if req_status is None:
@@ -1225,7 +1251,10 @@ class OffloadingConnectorScheduler:
req = req_status.req
max_boundary = self._calc_num_offloadable_tokens(req_status, req.num_tokens)
for group_idx, block_id, boundary in entries:
group_config = self.config.kv_group_configs[group_idx]
config_idx = config_idx_by_group.get(group_idx)
if config_idx is None:
continue
group_config = self.config.kv_group_configs[config_idx]
if (
block_id == 0
or boundary > max_boundary
@@ -1254,9 +1283,11 @@ class OffloadingConnectorScheduler:
fenced_block_ids=[block_id],
)
group_sizes = [0] * num_groups
group_sizes[group_idx] = 1
group_sizes[config_idx] = 1
block_indices = [0] * num_groups
block_indices[group_idx] = boundary // group_config.tokens_per_block - 1
block_indices[config_idx] = (
boundary // group_config.tokens_per_block - 1
)
store_jobs[job_id] = TransferJob(
req_id=req_id,
src_spec=GPULoadStoreSpec(
@@ -1300,6 +1331,12 @@ class OffloadingConnectorScheduler:
assert len(boundaries) == 1
boundary = boundaries.pop()
req = req_status.req
group_states = {
group.group_idx: state
for group, state in zip(
self.config.kv_group_configs, req_status.group_states
)
}
max_boundary = min(
req.num_prompt_tokens,
req_status.max_offload_tokens or req.num_prompt_tokens,
@@ -1315,7 +1352,7 @@ class OffloadingConnectorScheduler:
block_idx = boundary // self._partial_tail_block_size
if any(
group.group_idx not in self._cow_source_groups
and block_idx >= len(req_status.group_states[group.group_idx].block_ids)
and block_idx >= len(group_states[group.group_idx].block_ids)
for group in self.config.kv_group_configs
):
continue
@@ -1326,7 +1363,7 @@ class OffloadingConnectorScheduler:
block_ids = [
cow_blocks[group.group_idx]
if group.group_idx in self._cow_source_groups
else req_status.group_states[group.group_idx].block_ids[block_idx]
else group_states[group.group_idx].block_ids[block_idx]
for group in self.config.kv_group_configs
]
assert all(block_id != 0 for block_id in block_ids)
@@ -14,6 +14,9 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
OffloadingWorkerMetadata,
ReqId,
)
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.config import (
get_offloading_group_ids,
)
from vllm.logger import init_logger
from vllm.v1.kv_cache_interface import (
AttentionSpec,
@@ -65,7 +68,10 @@ class OffloadingConnectorWorker:
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
kv_cache_config = self.kv_cache_config
num_blocks = kv_cache_config.num_blocks
selected_group_ids = get_offloading_group_ids(kv_cache_config)
selected_groups = tuple(
kv_cache_config.kv_cache_groups[group_id] for group_id in selected_group_ids
)
mappings = derive_canonical_mappings(
self.vllm_config, kv_cache_config, kv_caches
)
@@ -76,7 +82,9 @@ class OffloadingConnectorWorker:
unpadded_page_size_bytes: dict[str, int] = {}
# layer_name -> size of page in bytes
page_size_bytes: dict[str, int] = {}
for kv_cache_group in kv_cache_config.kv_cache_groups:
for kv_cache_group in selected_groups:
assert not kv_cache_group.host_resident
num_blocks = kv_cache_config.num_blocks
group_layer_names = kv_cache_group.layer_names
group_kv_cache_spec = kv_cache_group.kv_cache_spec
if isinstance(group_kv_cache_spec, UniformTypeKVCacheSpecs):
@@ -131,8 +139,11 @@ class OffloadingConnectorWorker:
),
None,
)
if packed_layer_name is not None:
if packed_layer_name is not None and len(selected_groups) == len(
kv_cache_config.kv_cache_groups
):
(tensor,) = tensors_per_block[packed_layer_name]
num_blocks = tensor.shape[0]
block_stride = tensor.stride(0)
packed_tensor = tensor.as_strided(
(num_blocks, block_stride),
@@ -142,10 +153,7 @@ class OffloadingConnectorWorker:
self._init_worker(
CanonicalKVCaches(
[CanonicalKVCacheTensor(packed_tensor, block_stride)],
[
[CanonicalKVCacheRef(0, block_stride)]
for _ in kv_cache_config.kv_cache_groups
],
[[CanonicalKVCacheRef(0, block_stride)] for _ in selected_groups],
)
)
return
@@ -192,7 +200,7 @@ class OffloadingConnectorWorker:
)
group_data_refs: list[list[CanonicalKVCacheRef]] = []
for kv_cache_group in kv_cache_config.kv_cache_groups:
for kv_cache_group in selected_groups:
group_refs: list[CanonicalKVCacheRef] = []
for layer_name in kv_cache_group.layer_names:
group_refs += block_data_refs[layer_name]
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable
from functools import cached_property
from typing import Any
import torch
@@ -25,6 +26,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
)
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.config import (
build_offloading_config,
get_offloading_group_ids,
)
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
OffloadingConnectorStats,
@@ -38,6 +40,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.worker import (
)
from vllm.forward_context import ForwardContext
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.kv_cache_coordinator import HybridKVCacheCoordinator
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.kv_cache_interface import KVCacheConfig
@@ -47,6 +50,25 @@ from vllm.v1.request import Request
class OffloadingConnector(KVConnectorBase_V1, SupportsHMA):
@cached_property
def _bounding_group_ids(self) -> tuple[int, ...]:
"""Prefix-cacheable groups this connector does not offload.
Never offer tokens for a range some group cannot cover: past such a
group's own cached prefix its KV would be left unwritten.
"""
offloaded = set(get_offloading_group_ids(self._kv_cache_config))
return tuple(
group_id
for group_id, group in enumerate(self._kv_cache_config.kv_cache_groups)
if group.kv_cache_spec.prefix_cacheable and group_id not in offloaded
)
@property
def scheduler(self) -> OffloadingConnectorScheduler:
assert self.connector_scheduler is not None
return self.connector_scheduler
@property
def requires_kv_delivery(self) -> bool:
# Runs as kv_both, but is a best-effort cache: a dropped save is just a
@@ -138,9 +160,30 @@ class OffloadingConnector(KVConnectorBase_V1, SupportsHMA):
) -> tuple[int | None, bool]:
assert self.connector_scheduler is not None
return self.connector_scheduler.get_num_new_matched_tokens(
request, num_computed_tokens
request,
num_computed_tokens,
max_num_new_tokens=self._max_loadable_tokens(request, num_computed_tokens),
)
def _max_loadable_tokens(
self, request: "Request", num_computed_tokens: int
) -> int | None:
"""How far past ``num_computed_tokens`` a load may reach, if bounded.
Bounded by the deepest prefix the groups this connector does not
offload already hold, since nothing refills them beyond it.
"""
if not self._bounding_group_ids:
return None
assert self._kv_cache_manager is not None
coordinator = self._kv_cache_manager.coordinator
assert isinstance(coordinator, HybridKVCacheCoordinator)
_, per_group_hits = coordinator.find_longest_cache_hit_per_group(
request.block_hashes, request.num_tokens - 1
)
bound = min(per_group_hits[group_id] for group_id in self._bounding_group_ids)
return max(0, bound - num_computed_tokens)
def update_state_after_alloc(
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
):
@@ -185,6 +228,8 @@ class OffloadingConnector(KVConnectorBase_V1, SupportsHMA):
@classmethod
def get_required_kvcache_layout(cls, vllm_config: VllmConfig) -> str | None:
if vllm_config.attention_config.hisparse_config is not None:
return "BLHNC"
return "LBHNC"
def reset_cache(self) -> bool | None:
@@ -78,13 +78,11 @@ def ensure_kv_transfer_initialized(
global _KV_CONNECTOR_AGENT
if vllm_config.kv_transfer_config is None:
if _KV_CONNECTOR_AGENT is not None:
return
if (
vllm_config.kv_transfer_config.is_kv_transfer_instance
and _KV_CONNECTOR_AGENT is None
):
kv_transfer_config = vllm_config.kv_transfer_config
if kv_transfer_config is not None and kv_transfer_config.is_kv_transfer_instance:
# NIXL P/D requires an interleave_size equal to block_size.
vllm_config.adjust_dcp_kv_cache_interleave_size(kv_cache_config)
_sync_engine_id_across_tp(vllm_config)