[NIXL] Prepare KV transfer descriptors as NIXL 1.5 strided lists (#61032)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Ilia Yastrebov <iyastrebov@nvidia.com>
This commit is contained in:
Nick Hill
2026-10-14 18:07:24 +02:00
committed by GitHub
co-authored by Ilia Yastrebov
parent 9e2b579aab
commit eaf25f675a
11 changed files with 343 additions and 227 deletions
@@ -65,7 +65,11 @@ from vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils import
MambaConvSplitInfo,
derive_mamba_conv_split,
)
from vllm.distributed.nixl_utils import NixlWrapper, nixl_agent_config
from vllm.distributed.nixl_utils import (
NixlWrapper,
check_nixl_version,
nixl_agent_config,
)
from vllm.distributed.parallel_state import (
get_pcp_group,
get_tensor_model_parallel_rank,
@@ -277,20 +281,23 @@ class NixlBaseConnectorWorker:
src_blocks_data: np.ndarray,
num_fa_descs: int,
block_size_ratio: int = 1,
) -> Iterator[list[tuple[int, int, int]]]:
) -> Iterator[np.ndarray]:
"""Build split handle data for P_TP > D_TP scenario.
num_fa_descs is the boundary between FA and SSM descriptors.
Split counts are derived from source_ranks_per_group lengths.
FA uses rank_to_attention_slot for the slot offset;
SSM uses the rank's positional index.
Each Nx5 strided desc covers one region, so slicing it slices every block.
"""
fa_idx = next(
i for i, t in enumerate(self._group_spec_types) if _is_attention_spec(t)
)
fa_num_splits = len(plan.source_ranks_per_group[fa_idx])
has_ssm_descs = num_fa_descs < len(src_blocks_data)
starts = self._strided_desc_starts(src_blocks_data)
num_descs = int(src_blocks_data[:, 4].sum())
has_ssm_descs = num_fa_descs < num_descs
ssm_idx = next(
(i for i, t in enumerate(self._group_spec_types) if _is_ssm_spec(t)),
None,
@@ -300,38 +307,34 @@ class NixlBaseConnectorWorker:
if has_ssm_descs and ssm_idx is not None
else 0
)
# Per-FA-descriptor replicate flag, in _build_fa_local emission order.
fa_desc_replicated = self._fa_desc_replicated(num_fa_descs, block_size_ratio)
sharded_desc_end = len(src_blocks_data) - (
sharded_desc_end = num_descs - (
self._logical_num_blocks if self._ple_region_index is not None else 0
)
assert num_fa_descs <= sharded_desc_end
assert block_size_ratio == 1 or fa_num_splits == 1 or all(fa_desc_replicated), (
# REPLICATE (MLA) descs are read whole on every rank; SPLIT (full-attn)
# descs take this rank's head slice. SSM descs before the PLE pages are
# sharded by the rank's positional index; PLE pages are copied whole.
is_fa = starts < num_fa_descs
fa_replicated = self._fa_desc_replicated(
starts[is_fa], num_fa_descs, block_size_ratio
)
assert block_size_ratio == 1 or fa_num_splits == 1 or fa_replicated.all(), (
"Head-sharded attention reads with P_TP > D_TP and heterogeneous "
"block sizes are not supported"
)
src_blocks_list = src_blocks_data.tolist()
fa_split = is_fa.copy()
fa_split[is_fa] = ~fa_replicated
ssm_split = ~is_fa & (starts < sharded_desc_end)
for p_idx, p_rank in enumerate(plan.all_source_ranks):
fa_slot = plan.rank_to_attention_slot.get(p_rank, 0)
handle: list[tuple[int, int, int]] = []
for j, (addr, local_len, dev) in enumerate(src_blocks_list):
if j < num_fa_descs:
if fa_desc_replicated[j]:
# REPLICATE (MLA): whole block written on every rank.
handle.append((addr, local_len, dev))
else:
# SPLIT (full-attn): this rank's head slice.
chunk = local_len // fa_num_splits
handle.append((addr + fa_slot * chunk, chunk, dev))
elif j < sharded_desc_end:
chunk = local_len // ssm_num_splits
handle.append((addr + p_idx * chunk, chunk, dev))
else:
handle.append((addr, local_len, dev))
handle = src_blocks_data.copy()
handle[fa_split, 1] //= fa_num_splits
handle[fa_split, 0] += fa_slot * handle[fa_split, 1]
if ssm_num_splits:
handle[ssm_split, 1] //= ssm_num_splits
handle[ssm_split, 0] += p_idx * handle[ssm_split, 1]
yield handle
def _needs_split_local_xfer_handles(self, tp_ratio: int, plan: TPMapping) -> bool:
@@ -346,25 +349,30 @@ class NixlBaseConnectorWorker:
return tp_ratio < 0 and (not self.use_mla or len(plan.all_source_ranks) > 1)
def _fa_desc_replicated(
self, num_fa_descs: int, block_size_ratio: int = 1
) -> list[bool]:
"""Per-FA-descriptor replicate flag, in _build_fa_local emission order:
region-major, ``region_num_blocks[i] * block_size_ratio`` descs per
region. Length ``num_fa_descs``.
self,
desc_starts: np.ndarray,
num_fa_descs: int,
block_size_ratio: int = 1,
) -> np.ndarray:
"""Replicate flag per FA strided desc, given each one's flat block index.
_build_fa_local emits regions in order, with
``region_num_blocks[i] * block_size_ratio`` blocks per region. A strided
desc never spans regions, so the region owning its first block decides.
"""
assert self.transfer_topo is not None
n_regions = len(self.block_len_per_layer)
if n_regions == 0 or self.num_regions == 0:
return [False] * num_fa_descs
return np.zeros(len(desc_starts), dtype=bool)
region_indices = self._transfer_layer_region_indices or range(n_regions)
flags: list[bool] = []
for i in region_indices:
num_descs = self.region_num_blocks[i] * block_size_ratio
flags.extend([self._is_region_replicated(i)] * num_descs)
assert len(flags) == num_fa_descs, (
f"FA desc flags {len(flags)} != num_fa_descs {num_fa_descs}"
region_ends = np.cumsum(
[self.region_num_blocks[i] * block_size_ratio for i in region_indices]
)
return flags
assert region_ends[-1] == num_fa_descs, (
f"FA region blocks {region_ends[-1]} != num_fa_descs {num_fa_descs}"
)
replicated = np.array([self._is_region_replicated(i) for i in region_indices])
return replicated[np.searchsorted(region_ends, desc_starts, side="right")]
def _is_region_replicated(self, region_idx: int) -> bool:
"""Whether region ``region_idx`` is transferred REPLICATE vs SPLIT.
@@ -512,6 +520,8 @@ class NixlBaseConnectorWorker:
if nixl_wrapper_cls is None:
logger.error("NIXL is not available")
raise RuntimeError("NIXL is not available")
# Descriptor lists are prepared in NIXL's strided (Nx5) form.
check_nixl_version("1.5.0")
logger.info("Initializing NIXL wrapper")
logger.info("Initializing NIXL worker %s", engine_id)
@@ -1492,7 +1502,6 @@ class NixlBaseConnectorWorker:
if isinstance(layer_spec, MambaSpec)
else logical_num_blocks * self._physical_blocks_per_logical_kv_block
)
base_addr = cache.data_ptr()
is_mla_region = isinstance(
layer_spec, (MLAAttentionSpec, SlidingWindowMLASpec)
)
@@ -1827,7 +1836,7 @@ class NixlBaseConnectorWorker:
def _build_mamba_local(self, base_addresses: list[int]) -> np.ndarray:
"""Build desc regions (conv sub-projections + ssm) per layer for
local mamba blocks with DS conv layout, as an Nx3 uint64 array.
local mamba blocks with DS conv layout, as Nx5 uint64 strided descs.
A Mamba block interleaves conv and SSM state, which crucially differ in
size, so the two are indexed as separate sub-regions. Attention blocks
@@ -1865,28 +1874,32 @@ class NixlBaseConnectorWorker:
num_blocks = self._logical_num_blocks
physical_per_logical = self._physical_blocks_per_logical_kv_block
device_id = self.device_id
block_arange = np.arange(num_blocks, dtype=np.uint64)
parts: list[np.ndarray] = []
strided_descs: list[tuple[int, int, int, int, int]] = []
region_indices = self._ssm_region_indices or range(len(base_addresses))
for i in region_indices:
base_addr = base_addresses[i]
block_stride = self.block_stride_per_layer[i] * physical_per_logical
blk_addrs = base_addr + block_arange * block_stride
for off, sz in conv_offsets:
parts.append(self._stack_descs(blk_addrs + off, sz, device_id))
strided_descs.append(
(base_addr + off, sz, device_id, block_stride, num_blocks)
)
# SSM temporal state follows the conv state.
parts.append(self._stack_descs(blk_addrs + conv_size, ssm_size, device_id))
strided_descs.append(
(base_addr + conv_size, ssm_size, device_id, block_stride, num_blocks)
)
if (region_index := self._ple_region_index) is not None:
block_len = self._ple_page_len() * physical_per_logical
block_stride = (
self.block_stride_per_layer[region_index] * physical_per_logical
)
block_addrs = base_addresses[region_index] + block_arange * block_stride
parts.append(self._stack_descs(block_addrs, block_len, self.device_id))
base_addr = base_addresses[region_index]
strided_descs.append(
(base_addr, block_len, device_id, block_stride, num_blocks)
)
return np.concatenate(parts)
return self._pack_strided_descs(strided_descs)
def _build_mamba_remote(
self,
@@ -1896,7 +1909,7 @@ class NixlBaseConnectorWorker:
) -> np.ndarray:
"""Build remote desc regions (conv sub-projections + ssm) per layer.
For hetero-TP, each D rank reads only its sub-projection slice from
the P rank. Returns an Nx3 uint64 array."""
the P rank. Returns Nx5 uint64 strided descs."""
assert nixl_agent_meta.kv_caches_base_addr, (
"Remote KV cache base addresses must not be empty."
)
@@ -1916,9 +1929,8 @@ class NixlBaseConnectorWorker:
remote_physical_per_logical = transfer_info.remote_physical_blocks_per_logical
num_blocks = nixl_agent_meta.num_blocks // remote_physical_per_logical
device_id = nixl_agent_meta.device_id
block_arange = np.arange(num_blocks, dtype=np.uint64)
parts: list[np.ndarray] = []
strided_descs: list[tuple[int, int, int, int, int]] = []
# NOTE (ZhanqiuHu): use per-layer block_lens[i], not [0], in case
# block lengths vary across layers (e.g. MLA).
region_indices = self._ssm_region_indices or range(
@@ -1929,12 +1941,15 @@ class NixlBaseConnectorWorker:
block_stride = (
nixl_agent_meta.block_strides[i] * remote_physical_per_logical
)
blk_addrs = base_addr + block_arange * block_stride
for off, sz in conv_offsets:
parts.append(self._stack_descs(blk_addrs + off, sz, device_id))
strided_descs.append(
(base_addr + off, sz, device_id, block_stride, num_blocks)
)
# SSM temporal state is also TP-sharded on the heads dimension.
ssm_addrs = blk_addrs + conv_size_remote + local_offset * ssm_read_size
parts.append(self._stack_descs(ssm_addrs, ssm_read_size, device_id))
ssm_addr = base_addr + conv_size_remote + local_offset * ssm_read_size
strided_descs.append(
(ssm_addr, ssm_read_size, device_id, block_stride, num_blocks)
)
if (region_index := self._ple_region_index) is not None:
local_block_len = (
@@ -1954,52 +1969,85 @@ class NixlBaseConnectorWorker:
nixl_agent_meta.block_strides[region_index]
* remote_physical_per_logical
)
block_addrs = (
nixl_agent_meta.kv_caches_base_addr[region_index]
+ block_arange * remote_block_stride
addr = nixl_agent_meta.kv_caches_base_addr[region_index]
strided_descs.append(
(addr, remote_block_len, device_id, remote_block_stride, num_blocks)
)
parts.append(self._stack_descs(block_addrs, remote_block_len, device_id))
return np.concatenate(parts)
return self._pack_strided_descs(strided_descs)
@staticmethod
def _stack_descs(addrs: np.ndarray, length: int, device_id: int) -> np.ndarray:
out = np.empty((addrs.shape[0], 3), dtype=np.uint64)
out[:, 0] = addrs
out[:, 1] = length
out[:, 2] = device_id
return out
def _pack_strided_descs(
strided_descs: list[tuple[int, int, int, int, int]],
) -> np.ndarray:
"""Pack (addr, len, dev_id, stride, count) rows into an Nx5 uint64 array.
Each row describes `count` blocks of `len` bytes whose starts are
`stride` bytes apart; NIXL >= 1.5 prepares this layout directly.
"""
return np.array(strided_descs, dtype=np.uint64).reshape(-1, 5)
@staticmethod
def _block_strided_descs(
base_addr: int,
block_len: int,
block_stride: int,
num_blocks: int,
block_size_ratio: int,
device_id: int,
) -> np.ndarray:
"""Nx5 strided descs for `num_blocks` blocks, each split into
`block_size_ratio` sub-blocks of `block_len` bytes.
Evenly spaced sub-blocks collapse into one strided desc. In a padded
block the sub-blocks abut only within the block, so each block gets
its own.
"""
if block_size_ratio == 1 or block_stride == block_size_ratio * block_len:
stride = block_stride // block_size_ratio
count = num_blocks * block_size_ratio
return np.array(
[[base_addr, block_len, device_id, stride, count]], dtype=np.uint64
)
strided_descs = np.empty((num_blocks, 5), dtype=np.uint64)
strided_descs[:, 0] = (
base_addr + np.arange(num_blocks, dtype=np.uint64) * block_stride
)
strided_descs[:, 1] = block_len
strided_descs[:, 2] = device_id
strided_descs[:, 3] = block_len
strided_descs[:, 4] = block_size_ratio
return strided_descs
@staticmethod
def _strided_desc_starts(strided_descs: np.ndarray) -> np.ndarray:
"""Flat block index of each strided desc's first block."""
counts = strided_descs[:, 4]
return np.cumsum(counts) - counts
def _build_fa_local(
self,
base_addresses: list[int],
block_size_ratio: int,
) -> np.ndarray:
"""Build local FA descriptors as an Nx3 uint64 array."""
"""Build local FA descriptors as Nx5 uint64 strided descs."""
assert self.transfer_topo is not None
assert base_addresses, "Local KV cache base addresses must not be empty."
device_id = self.device_id
region_indices = self._transfer_layer_region_indices or range(
len(base_addresses)
)
parts: list[np.ndarray] = []
for i in region_indices:
base_addr = base_addresses[i]
block_len = self.block_len_per_layer[i] // block_size_ratio
logical_blocks = np.repeat(
np.arange(self.region_num_blocks[i], dtype=np.uint64),
block_size_ratio,
)
split_offsets = np.tile(
np.arange(block_size_ratio, dtype=np.uint64),
parts = [
self._block_strided_descs(
base_addresses[i],
self.block_len_per_layer[i] // block_size_ratio,
self.block_stride_per_layer[i],
self.region_num_blocks[i],
block_size_ratio,
device_id,
)
addrs = (
base_addr
+ logical_blocks * self.block_stride_per_layer[i]
+ split_offsets * block_len
)
parts.append(self._stack_descs(addrs, block_len, device_id))
for i in region_indices
]
return np.concatenate(parts)
def _build_fa_remote(
@@ -2008,7 +2056,7 @@ class NixlBaseConnectorWorker:
nixl_agent_meta: NixlAgentMetadata,
block_size_ratio: int,
) -> np.ndarray:
"""Build remote FA descriptors as an Nx3 uint64 array."""
"""Build remote FA descriptors as Nx5 uint64 strided descs."""
assert self.transfer_topo is not None
assert nixl_agent_meta.kv_caches_base_addr, (
"Remote KV cache base addresses must not be empty."
@@ -2024,7 +2072,7 @@ class NixlBaseConnectorWorker:
] * len(nixl_agent_meta.kv_caches_base_addr)
device_id = nixl_agent_meta.device_id
block_strides = nixl_agent_meta.block_strides
parts: list[np.ndarray] = []
strided_descs: list[tuple[int, int, int, int, int]] = []
use_layer_name_routing = bool(self._transfer_layer_region_indices)
for i, base_addr in enumerate(nixl_agent_meta.kv_caches_base_addr):
local_region = (
@@ -2046,10 +2094,12 @@ class NixlBaseConnectorWorker:
)
local_block_len = local_block_len // num_reads
block_arange = np.arange(region_num_blocks[i], dtype=np.uint64)
addrs = base_addr + rank_offset + block_arange * block_strides[i]
parts.append(self._stack_descs(addrs, local_block_len, device_id))
return np.concatenate(parts)
addr = base_addr + rank_offset
count = region_num_blocks[i]
strided_descs.append(
(addr, local_block_len, device_id, block_strides[i], count)
)
return self._pack_strided_descs(strided_descs)
def register_local_xfer_handler(
self,
@@ -2072,13 +2122,13 @@ class NixlBaseConnectorWorker:
blocks_data = self._build_fa_local(local_base_addresses, block_size_ratio)
logger.debug(
"Created %s blocks for src engine %s and rank %s on device id %s",
len(blocks_data),
blocks_data[:, 4].sum(),
self.engine_id,
self.tp_rank,
self.device_id,
)
if self._has_mamba:
assert self.num_descs * block_size_ratio == len(blocks_data)
assert self.num_descs * block_size_ratio == blocks_data[:, 4].sum()
# TODO (ZhanqiuHu): For homogeneous TP (tp_ratio == 1), the 3-descs split
# is unnecessary — a single conv desc per block suffices. Consider
# adding a fast path that falls back to the standard 2-region
@@ -2104,7 +2154,7 @@ class NixlBaseConnectorWorker:
)
]
)
assert len(desc_is_dram) == len(blocks_data)
assert len(desc_is_dram) == blocks_data[:, 4].sum()
desc_pos = np.empty(len(desc_is_dram), dtype=np.int64)
dram_idx = np.where(desc_is_dram)[0]
vram_idx = np.where(~desc_is_dram)[0]
@@ -2112,28 +2162,32 @@ class NixlBaseConnectorWorker:
desc_pos[vram_idx] = np.arange(len(vram_idx), dtype=np.int64)
self._desc_is_dram_by_block_size[block_size] = desc_is_dram
self._desc_pos_by_block_size[block_size] = desc_pos
# DRAM descriptors are registered under CPU device 0.
blocks_data[desc_is_dram, 2] = 0
dram_blocks = blocks_data[dram_idx]
vram_blocks = blocks_data[vram_idx]
# A strided desc never spans regions, so its first block decides.
strided_is_dram = desc_is_dram[self._strided_desc_starts(blocks_data)]
blocks_data[strided_is_dram, 2] = 0
if self._skip_dram_xfer:
self._dram_src_handles_by_block_size[block_size] = None
else:
dram_descs = self.nixl_wrapper.get_xfer_descs(dram_blocks, "DRAM")
self._dram_src_handles_by_block_size[block_size] = (
self.nixl_wrapper.prep_xfer_dlist("NIXL_INIT_AGENT", dram_descs)
self.nixl_wrapper.prep_xfer_dlist(
"NIXL_INIT_AGENT",
blocks_data[strided_is_dram],
mem_type="DRAM",
)
)
descs = self.nixl_wrapper.get_xfer_descs(vram_blocks, self.nixl_memory_type)
return (
self.nixl_wrapper.prep_xfer_dlist("NIXL_INIT_AGENT", descs),
blocks_data,
device_handle = self.nixl_wrapper.prep_xfer_dlist(
"NIXL_INIT_AGENT",
blocks_data[~strided_is_dram],
mem_type=self.nixl_memory_type,
)
return device_handle, blocks_data
mem_type = self.region_mem_types[0]
descs = self.nixl_wrapper.get_xfer_descs(blocks_data, mem_type)
# NIXL_INIT_AGENT to be used for preparations of local descs.
return self.nixl_wrapper.prep_xfer_dlist("NIXL_INIT_AGENT", descs), blocks_data
handle = self.nixl_wrapper.prep_xfer_dlist(
"NIXL_INIT_AGENT", blocks_data, mem_type=self.region_mem_types[0]
)
return handle, blocks_data
def add_remote_agent(
self,
@@ -2350,23 +2404,23 @@ class NixlBaseConnectorWorker:
block_size_ratio,
):
if self._mixed_mem_types:
handle_data = np.asarray(handle_data, dtype=np.uint64)
desc_is_dram = self._desc_is_dram_by_block_size[remote_block_size]
strided_is_dram = desc_is_dram[
self._strided_desc_starts(handle_data)
]
if self._skip_dram_xfer:
dram_handle = None
else:
dram_descs = self.nixl_wrapper.get_xfer_descs(
handle_data[desc_is_dram], "DRAM"
)
dram_handle = self.nixl_wrapper.prep_xfer_dlist(
"NIXL_INIT_AGENT", dram_descs
"NIXL_INIT_AGENT",
handle_data[strided_is_dram],
mem_type="DRAM",
)
self._dram_src_handles_by_tp_ratio[split_key].append(dram_handle)
handle_data = handle_data[~desc_is_dram]
descs = self.nixl_wrapper.get_xfer_descs(
handle_data, self.nixl_memory_type
handle_data = handle_data[~strided_is_dram]
handle = self.nixl_wrapper.prep_xfer_dlist(
"NIXL_INIT_AGENT", handle_data, mem_type=self.nixl_memory_type
)
handle = self.nixl_wrapper.prep_xfer_dlist("NIXL_INIT_AGENT", descs)
self.src_xfer_handles_by_tp_ratio[split_key].append(handle)
### Register remote agent memory regions
@@ -2383,7 +2437,7 @@ class NixlBaseConnectorWorker:
)
logger.debug(
"Created %s blocks for dst engine %s with remote rank %s and local rank %s",
len(blocks_data),
blocks_data[:, 4].sum(),
engine_id,
remote_tp_rank,
self.tp_rank,
@@ -2402,9 +2456,10 @@ class NixlBaseConnectorWorker:
raise NotImplementedError(
"NIXL pull does not yet support a mixed-memory producer"
)
descs = self.nixl_wrapper.get_xfer_descs(blocks_data, remote_mem_types.pop())
self.dst_xfer_side_handles[engine_id][remote_tp_rank] = (
self.nixl_wrapper.prep_xfer_dlist(remote_agent_name, descs)
self.nixl_wrapper.prep_xfer_dlist(
remote_agent_name, blocks_data, mem_type=remote_mem_types.pop()
)
)
return remote_agent_name
+20
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import functools
import importlib
import os
import sys
@@ -91,6 +92,24 @@ def is_nixl_available() -> bool:
return pkg in sys.modules or importlib.util.find_spec(pkg) is not None
@functools.cache
def check_nixl_version(min_version: str) -> None:
"""Raise if the installed NIXL package is older than ``min_version``."""
from importlib.metadata import packages_distributions, version
from packaging.version import Version
pkg = _get_nixl_package_name()
for dist in packages_distributions().get(pkg, ()):
installed = version(dist)
if Version(installed) < Version(min_version):
raise RuntimeError(
f"NIXL >= {min_version} is required, but {dist}=={installed} "
"is installed"
)
return
def alias_nixl_for_ray() -> None:
"""Let Ray's ``nixl._api`` import resolve to the ROCm implementation."""
if not current_platform.is_rocm() or "nixl._api" in sys.modules:
@@ -112,5 +131,6 @@ __all__ = [
"nixl_agent_config",
"nixlXferTelemetry",
"is_nixl_available",
"check_nixl_version",
"alias_nixl_for_ray",
]