Vectorize prep xfer list creation (#48209)

Signed-off-by: Ilia Yastrebov <iyastrebov@nvidia.com>
This commit is contained in:
Ilia Yastrebov
2026-07-16 11:39:05 +02:00
committed by GitHub
parent ea1d65fe6d
commit 9f8cbfd8eb
3 changed files with 85 additions and 74 deletions
@@ -13,6 +13,7 @@ from typing import Any, cast
from unittest.mock import MagicMock, patch
import msgspec
import numpy as np
import pytest
import ray
import torch
@@ -750,7 +751,10 @@ class TestNixlHandshake:
worker.block_len_per_layer = [4096 * worker.block_size]
worker.num_blocks = 1
worker.dst_num_blocks[worker.engine_id] = worker.num_blocks
worker.src_blocks_data = [(0, worker.block_len_per_layer[0], worker.tp_rank)]
worker.src_blocks_data = np.array(
[(0, worker.block_len_per_layer[0], worker.tp_rank)],
dtype=np.uint64,
)
worker.num_descs = len(worker.src_blocks_data)
def check_handshake(remote_tp_size: int):
@@ -1124,8 +1128,8 @@ class TestNixlHandshake:
== worker._mamba_ssm_size[1]
)
assert worker._build_fa_remote(plan, meta, block_size_ratio=1) == [
(0x1000 + local_block_len, local_block_len, 0)
assert worker._build_fa_remote(plan, meta, block_size_ratio=1).tolist() == [
[0x1000 + local_block_len, local_block_len, 0]
]
@patch(
@@ -1158,10 +1162,13 @@ class TestNixlHandshake:
worker._region_is_mla = [False, True]
worker.num_blocks = 1
worker.dst_num_blocks[worker.engine_id] = worker.num_blocks
worker.src_blocks_data = [
(0, fa_len, worker.tp_rank),
(0, idx_len, worker.tp_rank),
]
worker.src_blocks_data = np.array(
[
(0, fa_len, worker.tp_rank),
(0, idx_len, worker.tp_rank),
],
dtype=np.uint64,
)
worker.num_descs = len(worker.src_blocks_data)
# D_TP=2, P_TP=1 -> tp_ratio=2. SPLIT region scales by tp_ratio;
+13 -6
View File
@@ -11,6 +11,7 @@ from __future__ import annotations
from types import SimpleNamespace
import numpy as np
import pytest
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
@@ -102,7 +103,10 @@ class TestBuildSrcSplitHandles:
)
worker = _make_mock_worker_for_splits((FullAttentionSpec,))
src_blocks_data = [(0x2000 + i * 1024, 1024, 0) for i in range(8)]
src_blocks_data = np.array(
[(0x2000 + i * 1024, 1024, 0) for i in range(8)],
dtype=np.uint64,
)
num_descs = len(src_blocks_data)
splits = list(
worker._build_local_splits_from_plan(
@@ -135,11 +139,14 @@ class TestMambaPlanSplitHandles:
worker = _make_mock_worker_for_splits((FullAttentionSpec, MambaSpec))
# 2 FA descs + 1 SSM desc
src_blocks_data = [
(1000, 200, 0), # FA desc 0
(2000, 200, 0), # FA desc 1
(3000, 400, 0), # SSM desc 0
]
src_blocks_data = np.array(
[
(1000, 200, 0), # FA desc 0
(2000, 200, 0), # FA desc 1
(3000, 400, 0), # SSM desc 0
],
dtype=np.uint64,
)
splits = list(worker._build_local_splits_from_plan(plan, src_blocks_data, 2))
@@ -160,7 +160,7 @@ class NixlBaseConnectorWorker:
def _build_local_splits_from_plan(
self,
plan: TPMapping,
src_blocks_data: list[tuple[int, int, int]],
src_blocks_data: np.ndarray,
num_fa_descs: int,
) -> Iterator[list[tuple[int, int, int]]]:
"""Build split handle data for P_TP > D_TP scenario.
@@ -188,12 +188,13 @@ class NixlBaseConnectorWorker:
# Per-FA-descriptor replicate flag, in _build_fa_local emission order.
fa_desc_replicated = self._fa_desc_replicated(num_fa_descs)
src_blocks_list = src_blocks_data.tolist()
for p_idx, p_rank in enumerate(plan.all_source_ranks):
fa_slot = plan.rank_to_attention_slot.get(p_rank, 0)
handle: list[tuple[int, int, int]] = []
for j, (addr, local_len, dev) in enumerate(src_blocks_data):
for j, (addr, local_len, dev) in enumerate(src_blocks_list):
if j < num_fa_descs:
if fa_desc_replicated[j]:
# REPLICATE (MLA): whole block written on every rank.
@@ -1277,51 +1278,48 @@ class NixlBaseConnectorWorker:
self,
base_addresses: list[int],
block_size_ratio: int,
) -> list[tuple[int, int, int]]:
) -> np.ndarray:
"""Build desc regions (conv sub-projections + ssm) per layer for
local mamba blocks with DS conv layout."""
local mamba blocks with DS conv layout, as an Nx3 uint64 array."""
assert block_size_ratio == 1, (
"Mamba 3-read transfer with block_size_ratio != 1 is not tested. "
f"Got block_size_ratio={block_size_ratio}."
)
assert base_addresses, "Local KV cache base addresses must not be empty."
assert self._conv_decomp is not None
conv_offsets = self._conv_decomp.local_conv_offsets
conv_size, ssm_size = self._mamba_ssm_size
num_blocks = self._logical_num_blocks * block_size_ratio
physical_per_logical = self._physical_blocks_per_logical_kv_block
device_id = self.device_id
block_arange = np.arange(num_blocks, dtype=np.uint64)
result: list[tuple[int, int, int]] = []
parts: list[np.ndarray] = []
for i, base_addr in enumerate(base_addresses):
# Jump one page_size, but ssm page_size may be bigger when kernel
# locks block size to a specific value (physical_per_logical scale).
page_stride = (
self.block_len_per_layer[i] // block_size_ratio * physical_per_logical
)
blk_addrs = base_addr + block_arange * page_stride
for off, sz in conv_offsets:
for blk in range(num_blocks):
result.append(
(base_addr + blk * page_stride + off, sz, self.device_id)
)
parts.append(self._stack_descs(blk_addrs + off, sz, device_id))
# SSM temporal state follows the conv state.
for blk in range(num_blocks):
result.append(
(
base_addr + blk * page_stride + conv_size,
ssm_size,
self.device_id,
)
)
return result
parts.append(self._stack_descs(blk_addrs + conv_size, ssm_size, device_id))
return np.concatenate(parts)
def _build_mamba_remote(
self,
nixl_agent_meta: NixlAgentMetadata,
tp_ratio: int,
transfer_info: EngineTransferInfo,
) -> list[tuple[int, int, int]]:
) -> np.ndarray:
"""Build remote desc regions (conv sub-projections + ssm) per layer.
For hetero-TP, each D rank reads only its sub-projection slice from
the P rank."""
the P rank. Returns an Nx3 uint64 array."""
assert nixl_agent_meta.kv_caches_base_addr, (
"Remote KV cache base addresses must not be empty."
)
assert self._conv_decomp is not None
effective_ratio = max(tp_ratio, 1)
# Mamba conv state is always TP-sharded, even when attention KV
@@ -1338,35 +1336,41 @@ class NixlBaseConnectorWorker:
remote_physical_per_logical = transfer_info.remote_physical_blocks_per_logical
num_blocks = nixl_agent_meta.num_blocks // remote_physical_per_logical
device_id = nixl_agent_meta.device_id
block_arange = np.arange(num_blocks, dtype=np.uint64)
result: list[tuple[int, int, int]] = []
parts: list[np.ndarray] = []
# NOTE (ZhanqiuHu): use per-layer block_lens[i], not [0], in case
# block lengths vary across layers (e.g. MLA).
for i, base_addr in enumerate(nixl_agent_meta.kv_caches_base_addr):
page_stride = nixl_agent_meta.block_lens[i] * remote_physical_per_logical
blk_addrs = base_addr + block_arange * page_stride
for off, sz in conv_offsets:
for blk in range(num_blocks):
result.append((base_addr + blk * page_stride + off, sz, device_id))
parts.append(self._stack_descs(blk_addrs + off, sz, device_id))
# SSM temporal state is also TP-sharded on the heads dimension.
for blk in range(num_blocks):
ssm_addr = (
base_addr
+ blk * page_stride
+ conv_size_remote
+ local_offset * ssm_read_size
)
result.append((ssm_addr, ssm_read_size, device_id))
return result
ssm_addrs = blk_addrs + conv_size_remote + local_offset * ssm_read_size
parts.append(self._stack_descs(ssm_addrs, ssm_read_size, device_id))
return np.concatenate(parts)
@staticmethod
def _stack_descs(addrs: np.ndarray, length: int, device_id: int) -> np.ndarray:
out = np.empty((addrs.shape[0], 3), dtype=np.uint64)
out[:, 0] = addrs
out[:, 1] = length
out[:, 2] = device_id
return out
def _build_fa_local(
self,
base_addresses: list[int],
block_size_ratio: int,
) -> list[tuple[int, int, int]]:
"""Build local FA descriptors for all layers."""
) -> np.ndarray:
"""Build local FA descriptors for all layers as an Nx3 uint64 array."""
assert self.transfer_topo is not None
assert base_addresses, "Local KV cache base addresses must not be empty."
num_blocks = self.num_blocks * block_size_ratio
result: list[tuple[int, int, int]] = []
device_id = self.device_id
block_arange = np.arange(num_blocks, dtype=np.uint64)
parts: list[np.ndarray] = []
for i, base_addr in enumerate(base_addresses):
kv_block_len = (
self.get_backend_aware_kv_block_len(
@@ -1375,20 +1379,21 @@ class NixlBaseConnectorWorker:
// block_size_ratio
)
page_stride = self.block_len_per_layer[i] // block_size_ratio
for block_id in range(num_blocks):
block_offset = block_id * page_stride
addr = base_addr + block_offset
result.append((addr, kv_block_len, self.device_id))
return result
addrs = base_addr + block_arange * page_stride
parts.append(self._stack_descs(addrs, kv_block_len, device_id))
return np.concatenate(parts)
def _build_fa_remote(
self,
plan: TPMapping,
nixl_agent_meta: NixlAgentMetadata,
block_size_ratio: int,
) -> list[tuple[int, int, int]]:
"""Build remote FA descriptors for all layers."""
) -> np.ndarray:
"""Build remote FA descriptors for all layers as an Nx3 uint64 array."""
assert self.transfer_topo is not None
assert nixl_agent_meta.kv_caches_base_addr, (
"Remote KV cache base addresses must not be empty."
)
fa_group_idx = next(
i for i, t in enumerate(self._group_spec_types) if _is_attention_spec(t)
)
@@ -1396,7 +1401,9 @@ class NixlBaseConnectorWorker:
# per-rank offset; REPLICATE regions read the whole block once.
split_reads = len(plan.source_ranks_per_group[fa_group_idx])
num_blocks = nixl_agent_meta.num_blocks
result: list[tuple[int, int, int]] = []
device_id = nixl_agent_meta.device_id
block_arange = np.arange(num_blocks, dtype=np.uint64)
parts: list[np.ndarray] = []
for i, base_addr in enumerate(nixl_agent_meta.kv_caches_base_addr):
replicated = self._is_region_replicated(i)
# Read our whole local region size from remote..
@@ -1417,18 +1424,14 @@ class NixlBaseConnectorWorker:
local_block_len = local_block_len // num_reads
page_size = nixl_agent_meta.block_lens[i]
for block_id in range(num_blocks):
block_offset = block_id * page_size
# For each block, grab the kv heads chunk belonging to current local
# tp rank of size local_block_len.
addr = base_addr + block_offset + rank_offset
result.append((addr, local_block_len, nixl_agent_meta.device_id))
return result
addrs = base_addr + rank_offset + block_arange * page_size
parts.append(self._stack_descs(addrs, local_block_len, device_id))
return np.concatenate(parts)
def register_local_xfer_handler(
self,
block_size: int,
) -> tuple[int, list[tuple[int, int, int]]]:
) -> tuple[int, np.ndarray]:
"""
Function used for register local xfer handler with local block_size or
Remote block_size.
@@ -1461,9 +1464,8 @@ class NixlBaseConnectorWorker:
# remote has been seen. Currently we always register 4 regions
# because local descs are created before knowing the remote TP.
logger.debug("Registering local Mamba descriptors (4 regions/layer)")
blocks_data.extend(
self._build_mamba_local(local_base_addresses, block_size_ratio)
)
mamba = self._build_mamba_local(local_base_addresses, block_size_ratio)
blocks_data = np.concatenate([blocks_data, mamba])
descs = self.nixl_wrapper.get_xfer_descs(blocks_data, self.nixl_memory_type)
# NIXL_INIT_AGENT to be used for preparations of local descs.
@@ -1650,13 +1652,8 @@ class NixlBaseConnectorWorker:
engine_id,
remote_tp_rank,
)
blocks_data.extend(
self._build_mamba_remote(
nixl_agent_meta,
tp_ratio,
transfer_info,
)
)
mamba = self._build_mamba_remote(nixl_agent_meta, tp_ratio, transfer_info)
blocks_data = np.concatenate([blocks_data, mamba])
# Register with NIXL.
descs = self.nixl_wrapper.get_xfer_descs(blocks_data, self.nixl_memory_type)