nixl refactor: new transfer design (#40731)

Signed-off-by: ZhanqiuHu <zhu@redhat.com>
Signed-off-by: NickLucche <nlucches@redhat.com>
Co-authored-by: NickLucche <nlucches@redhat.com>
This commit is contained in:
zhanqiuhu
2026-05-06 06:16:25 -07:00
committed by GitHub
co-authored by NickLucche
parent 242afc6bf4
commit df8e63f4ed
8 changed files with 753 additions and 877 deletions
@@ -386,38 +386,6 @@ class EngineTransferInfo:
"""Physical blocks per logical block."""
@dataclass(frozen=True)
class MambaEngineTransferInfo(EngineTransferInfo):
"""Extends ``EngineTransferInfo`` with Mamba-hybrid transfer geometry.
For hybrid SSM+Attention models, FA and Mamba layers may require
different numbers of reads from different remote ranks. This
dataclass captures that per-engine transfer plan.
"""
remote_fa_source_ranks: tuple[int, ...]
"""Remote ranks carrying unique FA heads for this local rank."""
remote_all_source_ranks: tuple[int, ...]
"""All remote ranks this local rank reads from (FA + Mamba)."""
remote_num_fa_reads: int
"""Number of distinct remote ranks needed for FA data."""
remote_num_mamba_reads: int
"""Number of distinct remote ranks needed for Mamba data."""
remote_fa_descriptor_bytes: int
"""Byte size of one FA K (or V) descriptor entry."""
is_remote_replicated: bool
"""Whether the remote engine has replicated KV heads
(remote_tp_size > total_num_kv_heads)."""
remote_physical_heads: int
"""Physical KV heads stored per remote rank."""
# ---- Transfer topology ----
@@ -439,8 +407,6 @@ class TransferTopology:
self.local_physical_heads = max(1, self.total_num_kv_heads // self.tp_size)
self._engines: dict[EngineId, EngineTransferInfo] = {}
self._fa_source_sets: dict[EngineId, frozenset[int]] = {}
self._fa_source_indices: dict[EngineId, dict[int, int]] = {}
# Figure out whether the first dimension of the cache is K/V
# or num_blocks.
@@ -487,24 +453,12 @@ class TransferTopology:
def register_remote_engine(
self,
remote_engine_id: EngineId,
remote_tp_size: int,
remote_block_size: int,
remote_block_len: int,
remote_physical_blocks_per_logical: int,
*,
local_block_len: int = 0,
info: EngineTransferInfo,
) -> EngineTransferInfo:
"""Register a remote engine, unifying worker dicts state.
Only remote engines should be registered here — the local engine's
identity (tp_size, block_size, etc.) is set via ``__init__`` params.
For Mamba models, also computes the Mamba transfer plan and
builds the FA source lookup caches.
Args:
local_block_len: Local representative block_len (bytes).
Required for Mamba models to compute ``fa_descriptor_bytes``.
The caller (worker) is responsible for computing the info via
the transfer policy. This method only stores and deduplicates.
"""
assert remote_engine_id != self.engine_id, (
f"Cannot register local engine {self.engine_id} as remote. "
@@ -512,29 +466,6 @@ class TransferTopology:
)
if remote_engine_id in self._engines:
return self._engines[remote_engine_id]
info: EngineTransferInfo
if self.is_mamba:
info = self._build_mamba_info(
remote_tp_size=remote_tp_size,
remote_block_size=remote_block_size,
remote_block_len=remote_block_len,
remote_physical_blocks_per_logical=(remote_physical_blocks_per_logical),
local_block_len=local_block_len,
)
assert isinstance(info, MambaEngineTransferInfo)
self._fa_source_sets[remote_engine_id] = frozenset(
info.remote_fa_source_ranks
)
self._fa_source_indices[remote_engine_id] = {
r: i for i, r in enumerate(info.remote_fa_source_ranks)
}
else:
info = EngineTransferInfo(
remote_tp_size=remote_tp_size,
remote_block_len=remote_block_len,
remote_block_size=remote_block_size,
remote_physical_blocks_per_logical=(remote_physical_blocks_per_logical),
)
self._engines[remote_engine_id] = info
return info
@@ -622,14 +553,8 @@ class TransferTopology:
"""Get the remote TP rank(s) that the current local TP rank will
read from. When remote tp_size > local tp_size, reads from
multiple remote ranks.
For Mamba models, returns the precomputed ``all_source_ranks``
(FA + Mamba union).
"""
info = self._engines[remote_engine_id]
if isinstance(info, MambaEngineTransferInfo):
return list(info.remote_all_source_ranks)
tp_ratio = self.tp_ratio(info.remote_tp_size)
if tp_ratio > 0:
return [self.tp_rank // tp_ratio]
@@ -662,309 +587,15 @@ class TransferTopology:
# Regular case: backends like FA register K/V in separate regions
return cache if self.split_k_and_v else [cache]
# ============================================================
# Mamba-specific methods
# ============================================================
def should_skip_fa(self, remote_engine_id: EngineId, remote_rank: int) -> bool:
"""Whether to skip FA groups for this remote rank (mamba-only)."""
return remote_rank not in self._fa_source_sets[remote_engine_id]
def fa_head_slot(self, remote_engine_id: EngineId, remote_rank: int) -> int:
"""Index into local FA block for this remote rank's head data.
For remote ranks in ``fa_source_ranks``, returns 0, 1, …, reads-1.
For ranks NOT in ``fa_source_ranks`` (replicated duplicates),
returns the slot of the matching source rank with the same head.
"""
fa_index = self._fa_source_indices[remote_engine_id]
if remote_rank in fa_index:
return fa_index[remote_rank]
mamba_info = self._engines[remote_engine_id]
assert isinstance(mamba_info, MambaEngineTransferInfo)
K = self.total_num_kv_heads
remote_tp = mamba_info.remote_tp_size
r_head = self._physical_head_range(remote_tp, K, remote_rank)
for target in mamba_info.remote_fa_source_ranks:
t_head = self._physical_head_range(remote_tp, K, target)
if self._range_overlap(r_head, t_head):
return fa_index[target]
return 0
def fa_rank_offset(
self, remote_engine_id: EngineId, remote_kv_block_len: int
) -> int:
"""Byte offset into remote FA block for this local rank.
When local TP is replicated (local_tp > K), multiple local ranks
share a head. Computes offset *relative to the target remote
rank's first head* so it works regardless of how many heads the
remote has. Returns 0 when local does not index into remote.
"""
mamba_info = self._engines[remote_engine_id]
assert isinstance(mamba_info, MambaEngineTransferInfo)
tp_ratio = self.tp_ratio(mamba_info.remote_tp_size)
if self.is_mla or tp_ratio <= 0:
return 0
K = self.total_num_kv_heads
is_local_replicated = self.tp_size > K
if is_local_replicated:
local_head = self.tp_rank * K // self.tp_size
p_rank = mamba_info.remote_fa_source_ranks[0]
p_start = p_rank * K // mamba_info.remote_tp_size
return (local_head - p_start) * remote_kv_block_len
return self.tp_rank % tp_ratio * remote_kv_block_len
def needs_split_handles(self, remote_engine_id: EngineId) -> bool:
"""Whether per-remote-rank split handles are needed.
True when FA and mamba have different read counts, requiring
different splitting factors in the local handle.
"""
mamba_info = self._engines[remote_engine_id]
assert isinstance(mamba_info, MambaEngineTransferInfo)
tp_ratio = self.tp_ratio(mamba_info.remote_tp_size)
return (
tp_ratio < 0
and not self.is_mla
and len(mamba_info.remote_all_source_ranks) > 1
)
def compute_split_handle_data(
self,
remote_engine_id: EngineId,
src_blocks_data: list[tuple[int, int, int]],
num_fa_descs: int,
abs_tp: int,
) -> list[list[tuple[int, int, int]]]:
"""Per-remote-rank (addr, len, dev) triples for Mamba-HMA split
handles.
FA descriptors (indices < num_fa_descs) are sliced by
``remote_num_fa_reads``; mamba descriptors are sliced uniformly
by ``abs_tp``.
"""
mamba_info = self._engines[remote_engine_id]
assert isinstance(mamba_info, MambaEngineTransferInfo)
all_handle_data: list[list[tuple[int, int, int]]] = []
for p_idx, p_rank in enumerate(mamba_info.remote_all_source_ranks):
handle_data: list[tuple[int, int, int]] = []
skip_fa = self.should_skip_fa(remote_engine_id, p_rank)
fa_slot = self.fa_head_slot(remote_engine_id, p_rank) if not skip_fa else 0
for j, (addr, local_len, dev) in enumerate(src_blocks_data):
if j < num_fa_descs:
assert mamba_info.remote_num_fa_reads >= 1
fa_chunk = local_len // mamba_info.remote_num_fa_reads
handle_data.append((addr + fa_slot * fa_chunk, fa_chunk, dev))
else:
mamba_chunk = local_len // abs_tp
handle_data.append((addr + p_idx * mamba_chunk, mamba_chunk, dev))
all_handle_data.append(handle_data)
return all_handle_data
def filter_block_ids_for_rank(
self,
remote_engine_id: EngineId,
remote_rank: int,
local_ids: BlockIds,
remote_ids: BlockIds,
is_mamba_group: list[bool],
) -> tuple[BlockIds, BlockIds]:
"""Zero out FA groups for remote ranks outside ``fa_source_ranks``.
Returns (filtered_local_ids, filtered_remote_ids). When the
remote rank carries FA data for this local rank, returns the
inputs unchanged.
"""
if not self.should_skip_fa(remote_engine_id, remote_rank):
return local_ids, remote_ids
num_groups = len(local_ids)
filtered_local: list[list[int]] = [
[] if not is_mamba_group[g] else local_ids[g] for g in range(num_groups)
]
filtered_remote: list[list[int]] = [
[] if not is_mamba_group[g] else remote_ids[g] for g in range(num_groups)
]
return filtered_local, filtered_remote
def describe(self, remote_engine_id: EngineId) -> str:
"""One-line summary of transfer config for logging."""
info = self._engines[remote_engine_id]
base = (
return (
f"TransferTopology("
f"tp_ratio={self.tp_ratio(info.remote_tp_size)}, "
f"K={self.total_num_kv_heads}, "
f"local_tp={self.tp_size}, "
f"remote_tp={info.remote_tp_size}, "
f"local_rank={self.tp_rank}, "
f"remote_block_len={info.remote_block_len}"
)
if isinstance(info, MambaEngineTransferInfo):
return (
f"TransferTopology.mamba({base}, "
f"fa_reads={info.remote_num_fa_reads}, "
f"mamba_reads={info.remote_num_mamba_reads}, "
f"fa_sources={list(info.remote_fa_source_ranks)}, "
f"all_sources={list(info.remote_all_source_ranks)}, "
f"fa_desc_bytes={info.remote_fa_descriptor_bytes})"
)
return f"TransferTopology({base})"
# ============================================================
# Private helpers
# ============================================================
# Mamba-HMA hetero-TP transfer config:
# With hetero-TP (P_TP > D_TP), FA KV cache may be replicated across
# P ranks (when P_TP > num_kv_heads), but Mamba conv/SSM state is
# almost always uniquely sharded per P rank. So the number of P
# ranks D must read from can differ between FA and Mamba, and they
# must be handled separately.
@staticmethod
def _physical_head_range(tp_size: int, num_heads: int, rank: int) -> range:
"""Physical KV head range stored in a rank's KV cache tensor.
When ``tp_size <= num_heads``: sharded, K/TP contiguous heads per rank.
When ``tp_size > num_heads``: 1 physical head per rank. Heads are
distributed **contiguously** (matching vLLM's GQA weight partitioning):
consecutive ranks share a head before moving to the next one.
"""
if tp_size <= num_heads:
assert num_heads % tp_size == 0
per_rank = num_heads // tp_size
return range(rank * per_rank, (rank + 1) * per_rank)
else:
h = rank * num_heads // tp_size
return range(h, h + 1)
@staticmethod
def _range_overlap(a: range, b: range) -> range:
start = max(a.start, b.start)
stop = min(a.stop, b.stop)
return range(start, max(start, stop))
# ============================================================
# Private: build Mamba transfer info
# ============================================================
def _build_mamba_info(
self,
remote_tp_size: int,
remote_block_size: int,
remote_block_len: int,
remote_physical_blocks_per_logical: int,
local_block_len: int,
) -> MambaEngineTransferInfo:
"""Compute Mamba transfer plan."""
K = self.total_num_kv_heads
local_tp = self.tp_size
local_rank = self.tp_rank
is_remote_replicated = remote_tp_size > K
remote_physical_heads = max(1, K // remote_tp_size)
if local_tp >= remote_tp_size:
assert local_tp % remote_tp_size == 0
tp_ratio = local_tp // remote_tp_size
else:
assert remote_tp_size % local_tp == 0
tp_ratio = -(remote_tp_size // local_tp)
abs_tp = -tp_ratio if tp_ratio < 0 else 1
mamba_range: range | None = None
if tp_ratio < 0:
mamba_range = range(local_rank * abs_tp, (local_rank + 1) * abs_tp)
# ---- FA read targets ----
if self.is_mla or tp_ratio >= 0:
num_fa_reads = 1
fa_source_ranks: list[int] = (
[0]
if self.is_mla
else [local_rank // tp_ratio if tp_ratio > 0 else local_rank]
)
else:
local_needs = self._physical_head_range(local_tp, K, local_rank)
search_range = (
mamba_range if mamba_range is not None else range(remote_tp_size)
)
seen: set[tuple[int, int]] = set()
fa_source_ranks = []
for p in search_range:
p_has = self._physical_head_range(remote_tp_size, K, p)
ov = self._range_overlap(local_needs, p_has)
if len(ov) > 0:
key = (ov.start, ov.stop)
if key not in seen:
seen.add(key)
fa_source_ranks.append(p)
if not fa_source_ranks:
for p in range(remote_tp_size):
p_has = self._physical_head_range(remote_tp_size, K, p)
ov = self._range_overlap(local_needs, p_has)
if len(ov) > 0:
key = (ov.start, ov.stop)
if key not in seen:
seen.add(key)
fa_source_ranks.append(p)
num_fa_reads = len(fa_source_ranks)
# ---- All source ranks (mamba + FA) ----
if mamba_range is not None and abs_tp > num_fa_reads:
num_mamba_reads = abs_tp
all_source_ranks = list(mamba_range)
else:
num_mamba_reads = num_fa_reads
all_source_ranks = list(fa_source_ranks)
# ---- FA descriptor bytes ----
effective_block_len = min(local_block_len, remote_block_len)
if self.is_kv_layout_blocks_first:
fa_descriptor_bytes = effective_block_len // 2
else:
fa_descriptor_bytes = effective_block_len
# ---- Validation ----
is_local_replicated = local_tp > K
if is_local_replicated and is_remote_replicated and tp_ratio > 0:
logger.info(
"Both-replicated hetero-TP: local_tp=%d > remote_tp=%d > K=%d.",
local_tp,
remote_tp_size,
K,
)
tt_set = set(all_source_ranks)
for t in fa_source_ranks:
if t not in tt_set:
logger.error(
"FA source rank %d NOT in all_source_ranks %s.",
t,
all_source_ranks,
)
if self.is_kv_layout_blocks_first and tp_ratio < 0 and num_fa_reads > 0:
local_k_half = local_block_len // 2
remote_k_half = remote_block_len // 2
expected = local_k_half // num_fa_reads
if expected != remote_k_half:
logger.warning(
"FA size mismatch: local_k_half=%d / reads=%d = %d, "
"but remote_k_half=%d.",
local_k_half,
num_fa_reads,
expected,
remote_k_half,
)
return MambaEngineTransferInfo(
remote_tp_size=remote_tp_size,
remote_block_len=remote_block_len,
remote_block_size=remote_block_size,
remote_physical_blocks_per_logical=(remote_physical_blocks_per_logical),
remote_fa_source_ranks=tuple(fa_source_ranks),
remote_all_source_ranks=tuple(all_source_ranks),
remote_num_fa_reads=num_fa_reads,
remote_num_mamba_reads=num_mamba_reads,
remote_fa_descriptor_bytes=fa_descriptor_bytes,
is_remote_replicated=is_remote_replicated,
remote_physical_heads=remote_physical_heads,
f"remote_block_len={info.remote_block_len})"
)
@@ -32,8 +32,9 @@ GET_META_MSG = b"get_meta_msg"
# Version History:
# 1: Initial version with compatibility checking
# 2: Add remote_request_id to kv_transfer_params
# 3: Add physical_blocks_per_logical_kv_block to NixlAgentMetadata
#
NIXL_CONNECTOR_VERSION: int = 2
NIXL_CONNECTOR_VERSION: int = 3
@dataclass
@@ -48,6 +49,7 @@ class NixlAgentMetadata:
block_size: int
ssm_sizes: tuple[int, int]
attn_backend_name: str
physical_blocks_per_logical_kv_block: int
@dataclass
@@ -0,0 +1,142 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""TP mapping computation for NIXL KV cache transfers."""
from __future__ import annotations
from dataclasses import dataclass
import numpy as np
from vllm.distributed.kv_transfer.kv_connector.utils import (
BlockIds,
)
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheSpec, MambaSpec
# ======================================================================
# Data structures
# ======================================================================
@dataclass(frozen=True)
class ReadSpec:
"""Specification for a single remote block read operation."""
remote_rank: int
local_block_ids: BlockIds
remote_block_ids: BlockIds
def _is_attention_spec(spec_type: type[KVCacheSpec]) -> bool:
return issubclass(spec_type, AttentionSpec)
def _is_ssm_spec(spec_type: type[KVCacheSpec]) -> bool:
return issubclass(spec_type, MambaSpec)
@dataclass(frozen=True)
class TPMapping:
"""Complete local-to-remote TP mapping for one remote engine.
Generated once per remote engine during handshake.
"""
# Remote TP ranks that this local rank reads from, per group.
# Position = local piece index.
source_ranks_per_group: tuple[tuple[int, ...], ...]
# Superset of all source ranks (union of all groups).
all_source_ranks: tuple[int, ...]
# Maps each source rank to its FA head slot index.
rank_to_attention_slot: dict[int, int]
# FA head offset factor for hetero-TP (D_TP > P_TP).
rank_offset_factor: int
# ======================================================================
# TP mapping computation
# ======================================================================
def compute_tp_mapping(
tp_rank: int,
tp_size: int,
remote_tp_size: int,
is_mla: bool,
total_num_kv_heads: int,
group_spec_types: tuple[type[KVCacheSpec], ...],
) -> TPMapping:
"""Build the complete local-to-remote TP mapping.
Computes source ranks, head slot assignments, and the rank offset
factor in a single pass.
"""
# --- Attention source ranks ---
if is_mla:
# All heads replicated across all ranks.
attn_ranks = [0]
elif tp_size >= remote_tp_size:
# D (local TP) > P (remote TP): multiple local ranks read different chunks from
# *one* remote rank, corresponding to different kv heads.
attn_ranks = [tp_rank * remote_tp_size // tp_size]
else:
# P (remote TP) > D (local TP): one local rank
# reads from multiple remote ranks.
# GQA dedup: when K < remote_tp_size, several remote ranks
# hold the same KV head. np.unique keeps only the first
# rank per unique head so we don't issue redundant reads.
abs_tp = remote_tp_size // tp_size
start = tp_rank * abs_tp
heads = np.arange(start, start + abs_tp) * total_num_kv_heads // remote_tp_size
_, unique_idx = np.unique(heads, return_index=True)
attn_ranks = (start + np.sort(unique_idx)).tolist()
# --- SSM source ranks ---
has_ssm = any(_is_ssm_spec(t) for t in group_spec_types)
if has_ssm:
if tp_size < remote_tp_size:
abs_tp = remote_tp_size // tp_size
ssm_ranks = list(range(tp_rank * abs_tp, (tp_rank + 1) * abs_tp))
else:
ssm_ranks = list(attn_ranks)
else:
ssm_ranks = []
all_ranks = sorted(set(attn_ranks) | set(ssm_ranks))
# --- Per-group ordered source ranks ---
source_ranks_per_group = tuple(
tuple(ssm_ranks) if _is_ssm_spec(t) else tuple(attn_ranks)
for t in group_spec_types
)
# --- Attention head slots ---
head_to_slot: dict[int, int] = {}
for i, r in enumerate(attn_ranks):
head_to_slot[r * total_num_kv_heads // remote_tp_size] = i
rank_to_attention_slot = {
r: head_to_slot.get(r * total_num_kv_heads // remote_tp_size, 0)
for r in all_ranks
}
# --- Rank offset factor ---
if is_mla or tp_size <= remote_tp_size:
# We don't index into remote for reading, no offset needed.
rank_offset_factor = 0
elif tp_size > total_num_kv_heads:
local_head = tp_rank * total_num_kv_heads // tp_size
p_start = attn_ranks[0] * total_num_kv_heads // remote_tp_size
rank_offset_factor = local_head - p_start
else:
# D TP > P TP: we index into remote to read different heads depending on rank.
rank_offset_factor = tp_rank % (tp_size // remote_tp_size)
return TPMapping(
source_ranks_per_group=source_ranks_per_group,
all_source_ranks=tuple(all_ranks),
rank_to_attention_slot=rank_to_attention_slot,
rank_offset_factor=rank_offset_factor,
)
File diff suppressed because it is too large Load Diff
@@ -31,6 +31,7 @@ class MambaConvSplitInfo:
x_local: int # intermediate_size / TP (columns for x)
b_local: int # groups_ss / TP (columns for B; C is same size)
conv_dtype_size: int # bytes per element (e.g. 2 for float16)
ssm_sizes: tuple[int, int] # (conv_state_bytes, ssm_state_bytes)
@property
def conv_dim_local(self) -> int:
@@ -99,8 +100,8 @@ def derive_mamba_conv_split(
local_tp: this engine's tensor-parallel size.
Returns:
MambaConvSplitInfo with per-rank x_local, b_local, conv_rows, and
conv_dtype_size.
MambaConvSplitInfo with per-rank x_local, b_local, conv_rows,
conv_dtype_size, and ssm_sizes (conv_state_bytes, ssm_state_bytes).
"""
if mamba_spec.mamba_type != "mamba2":
raise NotImplementedError(
@@ -142,12 +143,20 @@ def derive_mamba_conv_split(
dtype=mamba_spec.dtypes[0], # type: ignore[misc]
).element_size()
ssm_dtype_size = torch.tensor(
[],
dtype=mamba_spec.dtypes[1], # type: ignore[misc]
).element_size()
conv_state_bytes = torch.Size(mamba_spec.shapes[0]).numel() * conv_dtype_size
ssm_state_bytes = torch.Size(mamba_spec.shapes[1]).numel() * ssm_dtype_size
# Divide by TP to get per-rank column counts.
return MambaConvSplitInfo(
conv_rows=conv_rows,
x_local=intermediate_size // local_tp,
b_local=groups_ss // local_tp,
conv_dtype_size=conv_dtype_size,
ssm_sizes=(conv_state_bytes, ssm_state_bytes),
)