From 9f8cbfd8eb01de1310caa0b657840cbaf6c3b015 Mon Sep 17 00:00:00 2001 From: Ilia Yastrebov Date: Thu, 16 Jul 2026 11:39:05 +0200 Subject: [PATCH] Vectorize prep xfer list creation (#48209) Signed-off-by: Ilia Yastrebov --- .../kv_connector/unit/test_nixl_connector.py | 21 ++-- tests/v1/kv_connector/unit/test_tp_mapping.py | 19 ++- .../kv_connector/v1/nixl/base_worker.py | 119 +++++++++--------- 3 files changed, 85 insertions(+), 74 deletions(-) diff --git a/tests/v1/kv_connector/unit/test_nixl_connector.py b/tests/v1/kv_connector/unit/test_nixl_connector.py index c1088f1c6e..31d34c69b3 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector.py @@ -13,6 +13,7 @@ from typing import Any, cast from unittest.mock import MagicMock, patch import msgspec +import numpy as np import pytest import ray import torch @@ -750,7 +751,10 @@ class TestNixlHandshake: worker.block_len_per_layer = [4096 * worker.block_size] worker.num_blocks = 1 worker.dst_num_blocks[worker.engine_id] = worker.num_blocks - worker.src_blocks_data = [(0, worker.block_len_per_layer[0], worker.tp_rank)] + worker.src_blocks_data = np.array( + [(0, worker.block_len_per_layer[0], worker.tp_rank)], + dtype=np.uint64, + ) worker.num_descs = len(worker.src_blocks_data) def check_handshake(remote_tp_size: int): @@ -1124,8 +1128,8 @@ class TestNixlHandshake: == worker._mamba_ssm_size[1] ) - assert worker._build_fa_remote(plan, meta, block_size_ratio=1) == [ - (0x1000 + local_block_len, local_block_len, 0) + assert worker._build_fa_remote(plan, meta, block_size_ratio=1).tolist() == [ + [0x1000 + local_block_len, local_block_len, 0] ] @patch( @@ -1158,10 +1162,13 @@ class TestNixlHandshake: worker._region_is_mla = [False, True] worker.num_blocks = 1 worker.dst_num_blocks[worker.engine_id] = worker.num_blocks - worker.src_blocks_data = [ - (0, fa_len, worker.tp_rank), - (0, idx_len, worker.tp_rank), - ] + worker.src_blocks_data = np.array( + [ + (0, fa_len, worker.tp_rank), + (0, idx_len, worker.tp_rank), + ], + dtype=np.uint64, + ) worker.num_descs = len(worker.src_blocks_data) # D_TP=2, P_TP=1 -> tp_ratio=2. SPLIT region scales by tp_ratio; diff --git a/tests/v1/kv_connector/unit/test_tp_mapping.py b/tests/v1/kv_connector/unit/test_tp_mapping.py index 5ab6b68400..7c73509823 100644 --- a/tests/v1/kv_connector/unit/test_tp_mapping.py +++ b/tests/v1/kv_connector/unit/test_tp_mapping.py @@ -11,6 +11,7 @@ from __future__ import annotations from types import SimpleNamespace +import numpy as np import pytest from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import ( @@ -102,7 +103,10 @@ class TestBuildSrcSplitHandles: ) worker = _make_mock_worker_for_splits((FullAttentionSpec,)) - src_blocks_data = [(0x2000 + i * 1024, 1024, 0) for i in range(8)] + src_blocks_data = np.array( + [(0x2000 + i * 1024, 1024, 0) for i in range(8)], + dtype=np.uint64, + ) num_descs = len(src_blocks_data) splits = list( worker._build_local_splits_from_plan( @@ -135,11 +139,14 @@ class TestMambaPlanSplitHandles: worker = _make_mock_worker_for_splits((FullAttentionSpec, MambaSpec)) # 2 FA descs + 1 SSM desc - src_blocks_data = [ - (1000, 200, 0), # FA desc 0 - (2000, 200, 0), # FA desc 1 - (3000, 400, 0), # SSM desc 0 - ] + src_blocks_data = np.array( + [ + (1000, 200, 0), # FA desc 0 + (2000, 200, 0), # FA desc 1 + (3000, 400, 0), # SSM desc 0 + ], + dtype=np.uint64, + ) splits = list(worker._build_local_splits_from_plan(plan, src_blocks_data, 2)) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py index d48cb2a5ff..8787a4dc87 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py @@ -160,7 +160,7 @@ class NixlBaseConnectorWorker: def _build_local_splits_from_plan( self, plan: TPMapping, - src_blocks_data: list[tuple[int, int, int]], + src_blocks_data: np.ndarray, num_fa_descs: int, ) -> Iterator[list[tuple[int, int, int]]]: """Build split handle data for P_TP > D_TP scenario. @@ -188,12 +188,13 @@ class NixlBaseConnectorWorker: # Per-FA-descriptor replicate flag, in _build_fa_local emission order. fa_desc_replicated = self._fa_desc_replicated(num_fa_descs) + src_blocks_list = src_blocks_data.tolist() for p_idx, p_rank in enumerate(plan.all_source_ranks): fa_slot = plan.rank_to_attention_slot.get(p_rank, 0) handle: list[tuple[int, int, int]] = [] - for j, (addr, local_len, dev) in enumerate(src_blocks_data): + for j, (addr, local_len, dev) in enumerate(src_blocks_list): if j < num_fa_descs: if fa_desc_replicated[j]: # REPLICATE (MLA): whole block written on every rank. @@ -1277,51 +1278,48 @@ class NixlBaseConnectorWorker: self, base_addresses: list[int], block_size_ratio: int, - ) -> list[tuple[int, int, int]]: + ) -> np.ndarray: """Build desc regions (conv sub-projections + ssm) per layer for - local mamba blocks with DS conv layout.""" + local mamba blocks with DS conv layout, as an Nx3 uint64 array.""" assert block_size_ratio == 1, ( "Mamba 3-read transfer with block_size_ratio != 1 is not tested. " f"Got block_size_ratio={block_size_ratio}." ) + assert base_addresses, "Local KV cache base addresses must not be empty." assert self._conv_decomp is not None conv_offsets = self._conv_decomp.local_conv_offsets conv_size, ssm_size = self._mamba_ssm_size num_blocks = self._logical_num_blocks * block_size_ratio physical_per_logical = self._physical_blocks_per_logical_kv_block + device_id = self.device_id + block_arange = np.arange(num_blocks, dtype=np.uint64) - result: list[tuple[int, int, int]] = [] + parts: list[np.ndarray] = [] for i, base_addr in enumerate(base_addresses): # Jump one page_size, but ssm page_size may be bigger when kernel # locks block size to a specific value (physical_per_logical scale). page_stride = ( self.block_len_per_layer[i] // block_size_ratio * physical_per_logical ) + blk_addrs = base_addr + block_arange * page_stride for off, sz in conv_offsets: - for blk in range(num_blocks): - result.append( - (base_addr + blk * page_stride + off, sz, self.device_id) - ) + parts.append(self._stack_descs(blk_addrs + off, sz, device_id)) # SSM temporal state follows the conv state. - for blk in range(num_blocks): - result.append( - ( - base_addr + blk * page_stride + conv_size, - ssm_size, - self.device_id, - ) - ) - return result + parts.append(self._stack_descs(blk_addrs + conv_size, ssm_size, device_id)) + return np.concatenate(parts) def _build_mamba_remote( self, nixl_agent_meta: NixlAgentMetadata, tp_ratio: int, transfer_info: EngineTransferInfo, - ) -> list[tuple[int, int, int]]: + ) -> np.ndarray: """Build remote desc regions (conv sub-projections + ssm) per layer. For hetero-TP, each D rank reads only its sub-projection slice from - the P rank.""" + the P rank. Returns an Nx3 uint64 array.""" + assert nixl_agent_meta.kv_caches_base_addr, ( + "Remote KV cache base addresses must not be empty." + ) assert self._conv_decomp is not None effective_ratio = max(tp_ratio, 1) # Mamba conv state is always TP-sharded, even when attention KV @@ -1338,35 +1336,41 @@ class NixlBaseConnectorWorker: remote_physical_per_logical = transfer_info.remote_physical_blocks_per_logical num_blocks = nixl_agent_meta.num_blocks // remote_physical_per_logical device_id = nixl_agent_meta.device_id + block_arange = np.arange(num_blocks, dtype=np.uint64) - result: list[tuple[int, int, int]] = [] + parts: list[np.ndarray] = [] # NOTE (ZhanqiuHu): use per-layer block_lens[i], not [0], in case # block lengths vary across layers (e.g. MLA). for i, base_addr in enumerate(nixl_agent_meta.kv_caches_base_addr): page_stride = nixl_agent_meta.block_lens[i] * remote_physical_per_logical + blk_addrs = base_addr + block_arange * page_stride for off, sz in conv_offsets: - for blk in range(num_blocks): - result.append((base_addr + blk * page_stride + off, sz, device_id)) + parts.append(self._stack_descs(blk_addrs + off, sz, device_id)) # SSM temporal state is also TP-sharded on the heads dimension. - for blk in range(num_blocks): - ssm_addr = ( - base_addr - + blk * page_stride - + conv_size_remote - + local_offset * ssm_read_size - ) - result.append((ssm_addr, ssm_read_size, device_id)) - return result + ssm_addrs = blk_addrs + conv_size_remote + local_offset * ssm_read_size + parts.append(self._stack_descs(ssm_addrs, ssm_read_size, device_id)) + return np.concatenate(parts) + + @staticmethod + def _stack_descs(addrs: np.ndarray, length: int, device_id: int) -> np.ndarray: + out = np.empty((addrs.shape[0], 3), dtype=np.uint64) + out[:, 0] = addrs + out[:, 1] = length + out[:, 2] = device_id + return out def _build_fa_local( self, base_addresses: list[int], block_size_ratio: int, - ) -> list[tuple[int, int, int]]: - """Build local FA descriptors for all layers.""" + ) -> np.ndarray: + """Build local FA descriptors for all layers as an Nx3 uint64 array.""" assert self.transfer_topo is not None + assert base_addresses, "Local KV cache base addresses must not be empty." num_blocks = self.num_blocks * block_size_ratio - result: list[tuple[int, int, int]] = [] + device_id = self.device_id + block_arange = np.arange(num_blocks, dtype=np.uint64) + parts: list[np.ndarray] = [] for i, base_addr in enumerate(base_addresses): kv_block_len = ( self.get_backend_aware_kv_block_len( @@ -1375,20 +1379,21 @@ class NixlBaseConnectorWorker: // block_size_ratio ) page_stride = self.block_len_per_layer[i] // block_size_ratio - for block_id in range(num_blocks): - block_offset = block_id * page_stride - addr = base_addr + block_offset - result.append((addr, kv_block_len, self.device_id)) - return result + addrs = base_addr + block_arange * page_stride + parts.append(self._stack_descs(addrs, kv_block_len, device_id)) + return np.concatenate(parts) def _build_fa_remote( self, plan: TPMapping, nixl_agent_meta: NixlAgentMetadata, block_size_ratio: int, - ) -> list[tuple[int, int, int]]: - """Build remote FA descriptors for all layers.""" + ) -> np.ndarray: + """Build remote FA descriptors for all layers as an Nx3 uint64 array.""" assert self.transfer_topo is not None + assert nixl_agent_meta.kv_caches_base_addr, ( + "Remote KV cache base addresses must not be empty." + ) fa_group_idx = next( i for i, t in enumerate(self._group_spec_types) if _is_attention_spec(t) ) @@ -1396,7 +1401,9 @@ class NixlBaseConnectorWorker: # per-rank offset; REPLICATE regions read the whole block once. split_reads = len(plan.source_ranks_per_group[fa_group_idx]) num_blocks = nixl_agent_meta.num_blocks - result: list[tuple[int, int, int]] = [] + device_id = nixl_agent_meta.device_id + block_arange = np.arange(num_blocks, dtype=np.uint64) + parts: list[np.ndarray] = [] for i, base_addr in enumerate(nixl_agent_meta.kv_caches_base_addr): replicated = self._is_region_replicated(i) # Read our whole local region size from remote.. @@ -1417,18 +1424,14 @@ class NixlBaseConnectorWorker: local_block_len = local_block_len // num_reads page_size = nixl_agent_meta.block_lens[i] - for block_id in range(num_blocks): - block_offset = block_id * page_size - # For each block, grab the kv heads chunk belonging to current local - # tp rank of size local_block_len. - addr = base_addr + block_offset + rank_offset - result.append((addr, local_block_len, nixl_agent_meta.device_id)) - return result + addrs = base_addr + rank_offset + block_arange * page_size + parts.append(self._stack_descs(addrs, local_block_len, device_id)) + return np.concatenate(parts) def register_local_xfer_handler( self, block_size: int, - ) -> tuple[int, list[tuple[int, int, int]]]: + ) -> tuple[int, np.ndarray]: """ Function used for register local xfer handler with local block_size or Remote block_size. @@ -1461,9 +1464,8 @@ class NixlBaseConnectorWorker: # remote has been seen. Currently we always register 4 regions # because local descs are created before knowing the remote TP. logger.debug("Registering local Mamba descriptors (4 regions/layer)") - blocks_data.extend( - self._build_mamba_local(local_base_addresses, block_size_ratio) - ) + mamba = self._build_mamba_local(local_base_addresses, block_size_ratio) + blocks_data = np.concatenate([blocks_data, mamba]) descs = self.nixl_wrapper.get_xfer_descs(blocks_data, self.nixl_memory_type) # NIXL_INIT_AGENT to be used for preparations of local descs. @@ -1650,13 +1652,8 @@ class NixlBaseConnectorWorker: engine_id, remote_tp_rank, ) - blocks_data.extend( - self._build_mamba_remote( - nixl_agent_meta, - tp_ratio, - transfer_info, - ) - ) + mamba = self._build_mamba_remote(nixl_agent_meta, tp_ratio, transfer_info) + blocks_data = np.concatenate([blocks_data, mamba]) # Register with NIXL. descs = self.nixl_wrapper.get_xfer_descs(blocks_data, self.nixl_memory_type)