[KVConnector][MoRIIO] Transfer hybrid mamba/KDA recurrent state in READ mode (#51052)

Signed-off-by: Yichao Zhu <Yichao.Zhu@amd.com>
This commit is contained in:
YukioZzz
2026-09-20 17:30:34 -04:00
committed by GitHub
parent 01f1f58f10
commit 9679173788
11 changed files with 2747 additions and 185 deletions
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import contextlib
import math
import os
import threading
import time
@@ -45,6 +46,13 @@ TransferId = str
TransferOffsetsKey = tuple[str, tuple[int, ...], tuple[int, ...], torch.dtype]
def _positive_finite_timeout(name: str, value: Any) -> float:
timeout = float(value)
if not math.isfinite(timeout) or timeout <= 0:
raise ValueError(f"{name} must be finite and greater than zero")
return timeout
class MoRIIOTransferAck(NamedTuple):
transfer_id: TransferId
consumer_tp_size: int = 1
@@ -160,6 +168,20 @@ class MoRIIOMode(Enum):
WRITE = "write"
class TransferBatchState(Enum):
"""Verdict for a group of transfer statuses that belong to one request.
A request's KV transfer is spread over one status per layer (two for a KDA
layer), so a single status never decides the request: PENDING means at
least one is still in flight and none has failed, FAILED means at least
one failed, DONE means all succeeded.
"""
DONE = "done"
FAILED = "failed"
PENDING = "pending"
class MoRIIOError(Exception):
"""Base exception for MoRIIO operations."""
@@ -260,6 +282,7 @@ class MoRIIOConfig:
tp_size: int
transfer_timeout: float
defer_timeout: float
recv_abort_timeout: float
read_mode: bool = False
qp_per_transfer: int = 1
post_batch_size: int = -1
@@ -284,6 +307,8 @@ class MoRIIOConfig:
# raising TransferError (sec).
# defer_timeout -> Timeout before a deferred send with no finished_sending
# notification is reaped and its blocks force-freed (sec).
# recv_abort_timeout -> Timeout before an in-flight recv whose RDMA
# completion never arrived is aborted (sec).
# Knobs for RDMA transfers, ignored if on xgmi backend
# qp_per_transfer -> Number of RDMA Queue Pairs per KV transfer.
@@ -315,13 +340,29 @@ class MoRIIOConfig:
"must be one of 'rdma' or 'xgmi'."
)
transfer_timeout = float(
transfer_timeout = _positive_finite_timeout(
"kv_connector_extra_config.transfer_timeout",
extra_config.get(
"transfer_timeout", MoRIIOConstants.DEFAULT_TRANSFER_TIMEOUT
)
),
)
defer_timeout = float(
extra_config.get("defer_timeout", MoRIIOConstants.DEFAULT_DEFER_TIMEOUT)
defer_timeout = _positive_finite_timeout(
"kv_connector_extra_config.defer_timeout",
extra_config.get("defer_timeout", MoRIIOConstants.DEFAULT_DEFER_TIMEOUT),
)
legacy_recv_abort_timeout = os.environ.get("VLLM_MORIIO_TRANSFER_TIMEOUT_S")
if legacy_recv_abort_timeout is not None:
logger.warning_once(
"The environment variable VLLM_MORIIO_TRANSFER_TIMEOUT_S is "
"deprecated. Set 'recv_abort_timeout' inside "
"kv_transfer_config.kv_connector_extra_config instead."
)
recv_abort_timeout = _positive_finite_timeout(
"kv_connector_extra_config.recv_abort_timeout",
extra_config.get(
"recv_abort_timeout",
legacy_recv_abort_timeout or MoRIIOConstants.DEFAULT_RECV_ABORT_TIMEOUT,
),
)
return cls(
@@ -347,6 +388,7 @@ class MoRIIOConfig:
backend=backend,
transfer_timeout=transfer_timeout,
defer_timeout=defer_timeout,
recv_abort_timeout=recv_abort_timeout,
)
@@ -374,6 +416,10 @@ class MoRIIOConstants:
# notification is reaped and its blocks force-freed.
# Overridable via kv_connector_extra_config["defer_timeout"].
DEFAULT_DEFER_TIMEOUT = 60.0
# Timeout (seconds) before an in-flight recv whose RDMA completion was lost
# is aborted, so the decode worker does not hang on it forever.
# Overridable via kv_connector_extra_config["recv_abort_timeout"].
DEFAULT_RECV_ABORT_TIMEOUT = 120.0
# The router embeds both zmq_addresses in the request_id:
@@ -567,9 +613,12 @@ class MoRIIOConnectorMetadata(KVConnectorMetadata):
remote_handshake_port=int(remote_handshake_port),
remote_notify_port=int(remote_notify_port),
# Remote peer TP degree (used as remote_tp_size downstream). The
# proxy advertises it under "remote_tp_size"; #46332 read "tp_size"
# which is absent on WRITE producer requests -> defaulted to 1 ->
# rank collapse. Read the right key; 0 == unknown (== homogeneous).
# decode side must know the prefiller's TP degree to map producer
# ranks onto the correct decode rank (and vice versa). The old plain
# "tp_size" key (#46332) is absent on WRITE producer requests and
# defaulted to 1, collapsing every producer rank onto decode rank 0
# and corrupting the transfer; prefer "remote_tp_size". 0 == unknown
# (treated as homogeneous downstream).
tp_size=int(
kv_transfer_params.get("remote_tp_size")
or kv_transfer_params.get("tp_size")
File diff suppressed because it is too large Load Diff
@@ -29,6 +29,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import (
MoRIIOError,
MoRIIOTransferAck,
RemoteAllocInfo,
TransferBatchState,
TransferError,
TransferId,
WriteTask,
@@ -58,7 +59,6 @@ try:
except ImportError:
logger.error("MoRIIO is not available")
"""Write task execution logic for MoRIIO connector."""
@@ -395,9 +395,9 @@ class MoRIIOWriter:
)
request_info.transfer_offsets[key] = offsets
# Get session index
layer_names = list(self.worker.layer_name_to_local_kv_cache_metadata.keys())
sess_idx = layer_names.index(task.layer_name)
# One session per registered region; attention layers map to a single
# index (see MoRIIOConnectorWorker._region_session_indices).
sess_idx = self.worker._region_session_indices(task.layer_name)[0]
local_off, remote_off, sizes = offsets
@@ -662,6 +662,25 @@ class MoRIIOWrapper:
)
return transfer_status
def poll_transfer_batch(self, transfer_statuses: list[Any]) -> TransferBatchState:
"""Non-blocking verdict over every status of one request.
Judging a request by its newest status alone calls it done while an
earlier layer is unfinished or already failed; failures do land on
layers other than the last, because _post_read_with_backoff returns a
failed status when the send queue never drains.
This stays a Python scan so it does not drive the backend's progress
callback from the engine thread alongside MoRIIO's own poller.
"""
state = TransferBatchState.DONE
for status in transfer_statuses:
if status.Failed():
return TransferBatchState.FAILED
if not status.Succeeded():
state = TransferBatchState.PENDING
return state
def waiting_for_transfer_complete(self, transfer_statuses: list[Any] | None = None):
if transfer_statuses is None:
with self.lock:
@@ -673,6 +692,15 @@ class MoRIIOWrapper:
if not transfers_to_wait:
return
self._poll_transfers_until_done(transfers_to_wait)
def _poll_transfers_until_done(self, transfers_to_wait: list[Any]) -> None:
"""Fallback for mori builds without the batched wait.
Gives failure precedence over pending: once any status has failed the
request is lost, so waiting out the remaining ones would only delay the
barrier by up to transfer_timeout.
"""
timeout = self._transfer_timeout
deadline = time.monotonic() + timeout
remaining = list(transfers_to_wait)
@@ -698,6 +726,8 @@ class MoRIIOWrapper:
)
else:
still_waiting.append(status)
if errors:
break
remaining = still_waiting
if remaining:
time.sleep(0.001)
@@ -2,14 +2,18 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable, Mapping
from typing import NamedTuple
from typing import NamedTuple, cast
import torch
from vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils import (
MambaConvSplitInfo,
)
from vllm.v1.kv_cache_interface import (
AttentionSpec,
KVCacheConfig,
KVCacheSpec,
MambaSpec,
MLAAttentionSpec,
SlidingWindowMLASpec,
UniformTypeKVCacheSpecs,
@@ -29,6 +33,85 @@ class LayerTransferGeometry(NamedTuple):
split_kv_regions: bool
class MambaTransferGeometry(NamedTuple):
"""Geometry of a hybrid (mamba/KDA) layer's recurrent state.
A KDA layer's packed page is exposed as ``conv_state`` and ``ssm_state``
views. Each is slot-strided, so a slot's bytes live at
``slot * slot_stride * element_size`` and span ``slot_bytes``. The two
tensors are registered as two separate MoRIIO regions per layer.
"""
conv_slot_stride: int
ssm_slot_stride: int
ssm_slot_bytes: int
conv_element_size: int
ssm_element_size: int
conv_region_len: int
ssm_region_len: int
def get_mamba_transfer_geometry(
conv: torch.Tensor,
ssm: torch.Tensor,
) -> MambaTransferGeometry:
"""Derive the slot-strided geometry of a KDA layer's conv + ssm state.
``conv``/``ssm`` are zero-copy views over the layer's packed page. Slot byte
offset is ``slot * slot_stride * element_size`` and each
slot spans ``subtensor[0].numel() * element_size`` bytes; the whole tensor
is registered as one region (``*_region_len`` bytes).
"""
if conv.ndim == 0 or ssm.ndim == 0 or conv.shape[0] == 0 or ssm.shape[0] == 0:
raise ValueError("KDA conv/ssm caches must contain at least one slot")
if conv.shape[0] != ssm.shape[0]:
raise ValueError(
f"KDA conv/ssm slot count mismatch: {conv.shape[0]} != {ssm.shape[0]}"
)
conv_slot_bytes = conv[0].numel() * conv.element_size()
ssm_slot_bytes = ssm[0].numel() * ssm.element_size()
conv_available_bytes = (
conv.untyped_storage().nbytes() - conv.storage_offset() * conv.element_size()
)
ssm_available_bytes = (
ssm.untyped_storage().nbytes() - ssm.storage_offset() * ssm.element_size()
)
if conv.shape[0]:
conv_required_bytes = (conv.shape[0] - 1) * conv.stride(
0
) * conv.element_size() + conv_slot_bytes
ssm_required_bytes = (ssm.shape[0] - 1) * ssm.stride(
0
) * ssm.element_size() + ssm_slot_bytes
if conv_required_bytes > conv_available_bytes:
raise ValueError("KDA conv slots exceed the registered storage extent")
if ssm_required_bytes > ssm_available_bytes:
raise ValueError("KDA ssm slots exceed the registered storage extent")
return MambaTransferGeometry(
conv_slot_stride=conv.stride(0),
ssm_slot_stride=ssm.stride(0),
ssm_slot_bytes=ssm_slot_bytes,
conv_element_size=conv.element_size(),
ssm_element_size=ssm.element_size(),
# Registered region must span the full slot-strided extent: stride(0)
# may exceed the per-slot element count, so numel() would under-size the
# region and truncate the highest-slot transfers. shape[0]*stride(0)
# covers every byte a ``slot * slot_stride * elem`` offset can address.
# Cap at the bytes remaining from each view's storage offset. A packed
# conv+ssm page places ssm at a non-zero intra-buffer offset,
# so do not register past the end of the shared backing allocation.
conv_region_len=min(
conv.shape[0] * conv.stride(0) * conv.element_size(),
conv_available_bytes,
),
ssm_region_len=min(
ssm.shape[0] * ssm.stride(0) * ssm.element_size(),
ssm_available_bytes,
),
)
def build_layer_to_spec(kv_cache_config: KVCacheConfig) -> dict[str, KVCacheSpec]:
layer_to_spec: dict[str, KVCacheSpec] = {}
for group in kv_cache_config.kv_cache_groups:
@@ -99,10 +182,10 @@ def get_layer_transfer_geometry(
layer_to_spec: Mapping[str, KVCacheSpec],
remote_num_blocks: int | None = None,
) -> LayerTransferGeometry:
spec = layer_to_spec[layer_name]
shape = kv_cache.shape
stride = kv_cache.stride()
element_size = kv_cache.element_size()
spec = layer_to_spec[layer_name]
is_mla_cache = is_mla_cache_layer(layer_to_spec, layer_name)
if not is_mla_cache and len(shape) == 5 and shape[0] == 2:
@@ -238,12 +321,60 @@ def get_layer_transfer_geometry(
)
def kda_conv_ssm(
kv_cache: torch.Tensor,
spec: MambaSpec,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Return zero-copy conv and SSM views over a packed two-state page."""
from math import prod
from vllm.utils.torch_utils import get_dtype_size
if len(spec.shapes) != 2 or len(spec.dtypes) != 2:
raise ValueError(
"MoRIIO KDA transfer requires exactly two states (conv and SSM)"
)
if kv_cache.element_size() != 1:
raise ValueError("MoRIIO KDA packed cache must use a byte-sized dtype")
if kv_cache.ndim != 4 or kv_cache.shape[1:3] != (1, 1) or kv_cache.stride(3) != 1:
raise ValueError(
"MoRIIO KDA packed cache must have shape [B, 1, 1, C] with "
"a contiguous byte dimension"
)
# Mirror MambaBase.bind_kv_cache: blocks may be strided within a shared
# multi-layer allocation, while every individual byte page is contiguous.
pages = kv_cache.squeeze(dim=(1, 2))
states: list[torch.Tensor] = []
offset = 0
for shp, dt in zip(spec.shapes, spec.dtypes, strict=True):
nbytes = prod(shp) * get_dtype_size(dt)
if offset + nbytes > pages.shape[1]:
raise ValueError(
"MoRIIO KDA state layout exceeds the packed cache page size"
)
state = pages[:, offset : offset + nbytes].view(dt)
states.append(state.view(-1, *shp))
offset += nbytes
return states[0], states[1]
def iter_layer_registration_regions(
layer_name: str,
kv_cache: torch.Tensor,
layer_to_spec: Mapping[str, KVCacheSpec],
) -> list[tuple[torch.Tensor, int]]:
geometry = get_layer_transfer_geometry(layer_name, kv_cache, layer_to_spec)
spec = layer_to_spec[layer_name]
if isinstance(spec, MambaSpec):
# KDA layer: register the conv and ssm tensors as two whole-tensor
# regions (the conv 3-subprojection split is expressed as transfer
# offset triples, not extra registrations).
conv, ssm = kda_conv_ssm(kv_cache, spec)
geom = get_mamba_transfer_geometry(conv, ssm)
return [(conv, geom.conv_region_len), (ssm, geom.ssm_region_len)]
geometry = cast(
LayerTransferGeometry,
get_layer_transfer_geometry(layer_name, kv_cache, layer_to_spec),
)
region_len = geometry.num_blocks * geometry.regions_per_block * geometry.block_len
if geometry.regions_per_block == 1:
# With padded or interleaved pages the block stride exceeds the
@@ -311,9 +442,36 @@ def compute_block_transfer_offsets(
"local_block_ids longer than remote_block_ids: "
f"{len(local_block_ids)} > {len(remote_block_ids)}"
)
geometry = get_layer_transfer_geometry(
layer_name, kv_cache, layer_to_spec, remote_num_blocks
geometry = cast(
LayerTransferGeometry,
get_layer_transfer_geometry(
layer_name, kv_cache, layer_to_spec, remote_num_blocks
),
)
invalid_local = next(
(
block_id
for block_id in local_block_ids
if not 0 <= block_id < geometry.num_blocks
),
None,
)
if invalid_local is not None:
raise ValueError(
f"local block id {invalid_local} is outside [0, {geometry.num_blocks})"
)
invalid_remote = next(
(
block_id
for block_id in remote_block_ids
if not 0 <= block_id < remote_num_blocks
),
None,
)
if invalid_remote is not None:
raise ValueError(
f"remote block id {invalid_remote} is outside [0, {remote_num_blocks})"
)
element_size = kv_cache.element_size()
transfer_size_byte = geometry.block_len
per_block = geometry.transfers_per_block
@@ -339,3 +497,125 @@ def compute_block_transfer_offsets(
w += 1
return merge_fn(offset_local, offset_remote, sizes)
class MambaOffsetTemplate(NamedTuple):
"""Slot-independent conv+ssm offset decomposition for a KDA layer.
Homogeneous-TP geometry (conv/ssm slot strides, conv sub-projection
offsets, ssm per-slot size) is identical across requests and across the
homogeneous GDN layers, so it is computed once and reused; only the
per-request slot bases vary (see ``apply_mamba_offset_template``).
"""
num_slots: int
conv_slot_stride_bytes: int
ssm_slot_stride_bytes: int
ssm_read_bytes: int
conv_subprojs: tuple[tuple[int, int, int], ...]
def build_mamba_offset_template(
conv: torch.Tensor,
ssm: torch.Tensor,
split_info: MambaConvSplitInfo,
tp_ratio: int,
) -> MambaOffsetTemplate:
"""Precompute the slot-independent conv+ssm offset decomposition.
Only homogeneous TP (``tp_ratio == 1``) is supported; heterogeneous TP is
gated with ``NotImplementedError`` (see the design doc). The returned
template is a pure function of the layer geometry and ``split_info``, so it
can be cached and reused across requests without recomputation.
"""
if tp_ratio != 1:
# Heterogeneous-TP recurrent-state transfer needs the remote page's
# slot stride (which differs from the local page) and, for
# P_TP > D_TP, a multi-rank gather of conv/ssm slices. Both are
# follow-ups. The mamba path bypasses the attention KV-head validator,
# so fail loudly here instead of silently transferring wrong bytes.
raise NotImplementedError(
"heterogeneous-TP mamba/KDA transfer is not supported yet "
f"(tp_ratio={tp_ratio}); use equal prefill/decode TP for "
"hybrid models"
)
geom = get_mamba_transfer_geometry(conv, ssm)
conv_local_offsets = split_info.local_conv_offsets
conv_remote_offsets = split_info.remote_conv_offsets(0, 1)
ssm_read_bytes = geom.ssm_slot_bytes
conv_subprojs = tuple(
(loff, roff, rsz)
for (loff, _lsz), (roff, rsz) in zip(conv_local_offsets, conv_remote_offsets)
)
return MambaOffsetTemplate(
num_slots=conv.shape[0],
conv_slot_stride_bytes=geom.conv_slot_stride * geom.conv_element_size,
ssm_slot_stride_bytes=geom.ssm_slot_stride * geom.ssm_element_size,
ssm_read_bytes=ssm_read_bytes,
conv_subprojs=conv_subprojs,
)
def apply_mamba_offset_template(
template: MambaOffsetTemplate,
local_slots: list[int],
remote_slots: list[int],
) -> tuple[list[int], list[int], list[int]]:
"""Apply per-request slot bases to a cached ``MambaOffsetTemplate``.
Conv sub-projection entries (all slots) come first, followed by one ssm
entry per slot.
"""
if len(local_slots) != len(remote_slots):
raise ValueError(
"local and remote mamba slot counts must match: "
f"{len(local_slots)} != {len(remote_slots)}"
)
invalid_local = next(
(slot for slot in local_slots if not 0 <= slot < template.num_slots), None
)
if invalid_local is not None:
raise ValueError(
f"local mamba slot {invalid_local} is outside [0, {template.num_slots})"
)
invalid_remote = next((slot for slot in remote_slots if slot < 0), None)
if invalid_remote is not None:
raise ValueError(f"remote mamba slot {invalid_remote} must be non-negative")
# The peer's KDA slot count is not part of MoRIIOAgentMetadata yet. Do not
# compare remote slot IDs against the local pool size: two otherwise
# compatible engines may reserve different numbers of slots.
local_offs: list[int] = []
remote_offs: list[int] = []
sizes: list[int] = []
# Conv sub-projections (region 0), all slots first.
conv_slot_stride_bytes = template.conv_slot_stride_bytes
for ls, rs in zip(local_slots, remote_slots):
lbase = ls * conv_slot_stride_bytes
rbase = rs * conv_slot_stride_bytes
for loff, roff, rsz in template.conv_subprojs:
local_offs.append(lbase + loff)
remote_offs.append(rbase + roff)
sizes.append(rsz)
# SSM temporal state (region 1), one entry per slot.
ssm_slot_stride_bytes = template.ssm_slot_stride_bytes
ssm_read_bytes = template.ssm_read_bytes
for ls, rs in zip(local_slots, remote_slots):
local_offs.append(ls * ssm_slot_stride_bytes)
remote_offs.append(rs * ssm_slot_stride_bytes)
sizes.append(ssm_read_bytes)
return local_offs, remote_offs, sizes
def compute_mamba_conv_split_count(
local_slots: list[int],
split_info: MambaConvSplitInfo,
) -> int:
"""Number of leading conv entries in a KDA layer transfer plan.
Entries ``[:count]`` are conv sub-projections (conv region/session);
entries ``[count:]`` are the per-slot ssm state (ssm region/session).
"""
return len(local_slots) * len(split_info.local_conv_offsets)