Vectorize prep xfer list creation (#48209)
Signed-off-by: Ilia Yastrebov <iyastrebov@nvidia.com>
This commit is contained in:
@@ -13,6 +13,7 @@ from typing import Any, cast
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import msgspec
|
||||
import numpy as np
|
||||
import pytest
|
||||
import ray
|
||||
import torch
|
||||
@@ -750,7 +751,10 @@ class TestNixlHandshake:
|
||||
worker.block_len_per_layer = [4096 * worker.block_size]
|
||||
worker.num_blocks = 1
|
||||
worker.dst_num_blocks[worker.engine_id] = worker.num_blocks
|
||||
worker.src_blocks_data = [(0, worker.block_len_per_layer[0], worker.tp_rank)]
|
||||
worker.src_blocks_data = np.array(
|
||||
[(0, worker.block_len_per_layer[0], worker.tp_rank)],
|
||||
dtype=np.uint64,
|
||||
)
|
||||
worker.num_descs = len(worker.src_blocks_data)
|
||||
|
||||
def check_handshake(remote_tp_size: int):
|
||||
@@ -1124,8 +1128,8 @@ class TestNixlHandshake:
|
||||
== worker._mamba_ssm_size[1]
|
||||
)
|
||||
|
||||
assert worker._build_fa_remote(plan, meta, block_size_ratio=1) == [
|
||||
(0x1000 + local_block_len, local_block_len, 0)
|
||||
assert worker._build_fa_remote(plan, meta, block_size_ratio=1).tolist() == [
|
||||
[0x1000 + local_block_len, local_block_len, 0]
|
||||
]
|
||||
|
||||
@patch(
|
||||
@@ -1158,10 +1162,13 @@ class TestNixlHandshake:
|
||||
worker._region_is_mla = [False, True]
|
||||
worker.num_blocks = 1
|
||||
worker.dst_num_blocks[worker.engine_id] = worker.num_blocks
|
||||
worker.src_blocks_data = [
|
||||
(0, fa_len, worker.tp_rank),
|
||||
(0, idx_len, worker.tp_rank),
|
||||
]
|
||||
worker.src_blocks_data = np.array(
|
||||
[
|
||||
(0, fa_len, worker.tp_rank),
|
||||
(0, idx_len, worker.tp_rank),
|
||||
],
|
||||
dtype=np.uint64,
|
||||
)
|
||||
worker.num_descs = len(worker.src_blocks_data)
|
||||
|
||||
# D_TP=2, P_TP=1 -> tp_ratio=2. SPLIT region scales by tp_ratio;
|
||||
|
||||
@@ -11,6 +11,7 @@ from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
|
||||
@@ -102,7 +103,10 @@ class TestBuildSrcSplitHandles:
|
||||
)
|
||||
|
||||
worker = _make_mock_worker_for_splits((FullAttentionSpec,))
|
||||
src_blocks_data = [(0x2000 + i * 1024, 1024, 0) for i in range(8)]
|
||||
src_blocks_data = np.array(
|
||||
[(0x2000 + i * 1024, 1024, 0) for i in range(8)],
|
||||
dtype=np.uint64,
|
||||
)
|
||||
num_descs = len(src_blocks_data)
|
||||
splits = list(
|
||||
worker._build_local_splits_from_plan(
|
||||
@@ -135,11 +139,14 @@ class TestMambaPlanSplitHandles:
|
||||
|
||||
worker = _make_mock_worker_for_splits((FullAttentionSpec, MambaSpec))
|
||||
# 2 FA descs + 1 SSM desc
|
||||
src_blocks_data = [
|
||||
(1000, 200, 0), # FA desc 0
|
||||
(2000, 200, 0), # FA desc 1
|
||||
(3000, 400, 0), # SSM desc 0
|
||||
]
|
||||
src_blocks_data = np.array(
|
||||
[
|
||||
(1000, 200, 0), # FA desc 0
|
||||
(2000, 200, 0), # FA desc 1
|
||||
(3000, 400, 0), # SSM desc 0
|
||||
],
|
||||
dtype=np.uint64,
|
||||
)
|
||||
|
||||
splits = list(worker._build_local_splits_from_plan(plan, src_blocks_data, 2))
|
||||
|
||||
|
||||
@@ -160,7 +160,7 @@ class NixlBaseConnectorWorker:
|
||||
def _build_local_splits_from_plan(
|
||||
self,
|
||||
plan: TPMapping,
|
||||
src_blocks_data: list[tuple[int, int, int]],
|
||||
src_blocks_data: np.ndarray,
|
||||
num_fa_descs: int,
|
||||
) -> Iterator[list[tuple[int, int, int]]]:
|
||||
"""Build split handle data for P_TP > D_TP scenario.
|
||||
@@ -188,12 +188,13 @@ class NixlBaseConnectorWorker:
|
||||
|
||||
# Per-FA-descriptor replicate flag, in _build_fa_local emission order.
|
||||
fa_desc_replicated = self._fa_desc_replicated(num_fa_descs)
|
||||
src_blocks_list = src_blocks_data.tolist()
|
||||
|
||||
for p_idx, p_rank in enumerate(plan.all_source_ranks):
|
||||
fa_slot = plan.rank_to_attention_slot.get(p_rank, 0)
|
||||
|
||||
handle: list[tuple[int, int, int]] = []
|
||||
for j, (addr, local_len, dev) in enumerate(src_blocks_data):
|
||||
for j, (addr, local_len, dev) in enumerate(src_blocks_list):
|
||||
if j < num_fa_descs:
|
||||
if fa_desc_replicated[j]:
|
||||
# REPLICATE (MLA): whole block written on every rank.
|
||||
@@ -1277,51 +1278,48 @@ class NixlBaseConnectorWorker:
|
||||
self,
|
||||
base_addresses: list[int],
|
||||
block_size_ratio: int,
|
||||
) -> list[tuple[int, int, int]]:
|
||||
) -> np.ndarray:
|
||||
"""Build desc regions (conv sub-projections + ssm) per layer for
|
||||
local mamba blocks with DS conv layout."""
|
||||
local mamba blocks with DS conv layout, as an Nx3 uint64 array."""
|
||||
assert block_size_ratio == 1, (
|
||||
"Mamba 3-read transfer with block_size_ratio != 1 is not tested. "
|
||||
f"Got block_size_ratio={block_size_ratio}."
|
||||
)
|
||||
assert base_addresses, "Local KV cache base addresses must not be empty."
|
||||
assert self._conv_decomp is not None
|
||||
conv_offsets = self._conv_decomp.local_conv_offsets
|
||||
conv_size, ssm_size = self._mamba_ssm_size
|
||||
num_blocks = self._logical_num_blocks * block_size_ratio
|
||||
physical_per_logical = self._physical_blocks_per_logical_kv_block
|
||||
device_id = self.device_id
|
||||
block_arange = np.arange(num_blocks, dtype=np.uint64)
|
||||
|
||||
result: list[tuple[int, int, int]] = []
|
||||
parts: list[np.ndarray] = []
|
||||
for i, base_addr in enumerate(base_addresses):
|
||||
# Jump one page_size, but ssm page_size may be bigger when kernel
|
||||
# locks block size to a specific value (physical_per_logical scale).
|
||||
page_stride = (
|
||||
self.block_len_per_layer[i] // block_size_ratio * physical_per_logical
|
||||
)
|
||||
blk_addrs = base_addr + block_arange * page_stride
|
||||
for off, sz in conv_offsets:
|
||||
for blk in range(num_blocks):
|
||||
result.append(
|
||||
(base_addr + blk * page_stride + off, sz, self.device_id)
|
||||
)
|
||||
parts.append(self._stack_descs(blk_addrs + off, sz, device_id))
|
||||
# SSM temporal state follows the conv state.
|
||||
for blk in range(num_blocks):
|
||||
result.append(
|
||||
(
|
||||
base_addr + blk * page_stride + conv_size,
|
||||
ssm_size,
|
||||
self.device_id,
|
||||
)
|
||||
)
|
||||
return result
|
||||
parts.append(self._stack_descs(blk_addrs + conv_size, ssm_size, device_id))
|
||||
return np.concatenate(parts)
|
||||
|
||||
def _build_mamba_remote(
|
||||
self,
|
||||
nixl_agent_meta: NixlAgentMetadata,
|
||||
tp_ratio: int,
|
||||
transfer_info: EngineTransferInfo,
|
||||
) -> list[tuple[int, int, int]]:
|
||||
) -> np.ndarray:
|
||||
"""Build remote desc regions (conv sub-projections + ssm) per layer.
|
||||
For hetero-TP, each D rank reads only its sub-projection slice from
|
||||
the P rank."""
|
||||
the P rank. Returns an Nx3 uint64 array."""
|
||||
assert nixl_agent_meta.kv_caches_base_addr, (
|
||||
"Remote KV cache base addresses must not be empty."
|
||||
)
|
||||
assert self._conv_decomp is not None
|
||||
effective_ratio = max(tp_ratio, 1)
|
||||
# Mamba conv state is always TP-sharded, even when attention KV
|
||||
@@ -1338,35 +1336,41 @@ class NixlBaseConnectorWorker:
|
||||
remote_physical_per_logical = transfer_info.remote_physical_blocks_per_logical
|
||||
num_blocks = nixl_agent_meta.num_blocks // remote_physical_per_logical
|
||||
device_id = nixl_agent_meta.device_id
|
||||
block_arange = np.arange(num_blocks, dtype=np.uint64)
|
||||
|
||||
result: list[tuple[int, int, int]] = []
|
||||
parts: list[np.ndarray] = []
|
||||
# NOTE (ZhanqiuHu): use per-layer block_lens[i], not [0], in case
|
||||
# block lengths vary across layers (e.g. MLA).
|
||||
for i, base_addr in enumerate(nixl_agent_meta.kv_caches_base_addr):
|
||||
page_stride = nixl_agent_meta.block_lens[i] * remote_physical_per_logical
|
||||
blk_addrs = base_addr + block_arange * page_stride
|
||||
for off, sz in conv_offsets:
|
||||
for blk in range(num_blocks):
|
||||
result.append((base_addr + blk * page_stride + off, sz, device_id))
|
||||
parts.append(self._stack_descs(blk_addrs + off, sz, device_id))
|
||||
# SSM temporal state is also TP-sharded on the heads dimension.
|
||||
for blk in range(num_blocks):
|
||||
ssm_addr = (
|
||||
base_addr
|
||||
+ blk * page_stride
|
||||
+ conv_size_remote
|
||||
+ local_offset * ssm_read_size
|
||||
)
|
||||
result.append((ssm_addr, ssm_read_size, device_id))
|
||||
return result
|
||||
ssm_addrs = blk_addrs + conv_size_remote + local_offset * ssm_read_size
|
||||
parts.append(self._stack_descs(ssm_addrs, ssm_read_size, device_id))
|
||||
return np.concatenate(parts)
|
||||
|
||||
@staticmethod
|
||||
def _stack_descs(addrs: np.ndarray, length: int, device_id: int) -> np.ndarray:
|
||||
out = np.empty((addrs.shape[0], 3), dtype=np.uint64)
|
||||
out[:, 0] = addrs
|
||||
out[:, 1] = length
|
||||
out[:, 2] = device_id
|
||||
return out
|
||||
|
||||
def _build_fa_local(
|
||||
self,
|
||||
base_addresses: list[int],
|
||||
block_size_ratio: int,
|
||||
) -> list[tuple[int, int, int]]:
|
||||
"""Build local FA descriptors for all layers."""
|
||||
) -> np.ndarray:
|
||||
"""Build local FA descriptors for all layers as an Nx3 uint64 array."""
|
||||
assert self.transfer_topo is not None
|
||||
assert base_addresses, "Local KV cache base addresses must not be empty."
|
||||
num_blocks = self.num_blocks * block_size_ratio
|
||||
result: list[tuple[int, int, int]] = []
|
||||
device_id = self.device_id
|
||||
block_arange = np.arange(num_blocks, dtype=np.uint64)
|
||||
parts: list[np.ndarray] = []
|
||||
for i, base_addr in enumerate(base_addresses):
|
||||
kv_block_len = (
|
||||
self.get_backend_aware_kv_block_len(
|
||||
@@ -1375,20 +1379,21 @@ class NixlBaseConnectorWorker:
|
||||
// block_size_ratio
|
||||
)
|
||||
page_stride = self.block_len_per_layer[i] // block_size_ratio
|
||||
for block_id in range(num_blocks):
|
||||
block_offset = block_id * page_stride
|
||||
addr = base_addr + block_offset
|
||||
result.append((addr, kv_block_len, self.device_id))
|
||||
return result
|
||||
addrs = base_addr + block_arange * page_stride
|
||||
parts.append(self._stack_descs(addrs, kv_block_len, device_id))
|
||||
return np.concatenate(parts)
|
||||
|
||||
def _build_fa_remote(
|
||||
self,
|
||||
plan: TPMapping,
|
||||
nixl_agent_meta: NixlAgentMetadata,
|
||||
block_size_ratio: int,
|
||||
) -> list[tuple[int, int, int]]:
|
||||
"""Build remote FA descriptors for all layers."""
|
||||
) -> np.ndarray:
|
||||
"""Build remote FA descriptors for all layers as an Nx3 uint64 array."""
|
||||
assert self.transfer_topo is not None
|
||||
assert nixl_agent_meta.kv_caches_base_addr, (
|
||||
"Remote KV cache base addresses must not be empty."
|
||||
)
|
||||
fa_group_idx = next(
|
||||
i for i, t in enumerate(self._group_spec_types) if _is_attention_spec(t)
|
||||
)
|
||||
@@ -1396,7 +1401,9 @@ class NixlBaseConnectorWorker:
|
||||
# per-rank offset; REPLICATE regions read the whole block once.
|
||||
split_reads = len(plan.source_ranks_per_group[fa_group_idx])
|
||||
num_blocks = nixl_agent_meta.num_blocks
|
||||
result: list[tuple[int, int, int]] = []
|
||||
device_id = nixl_agent_meta.device_id
|
||||
block_arange = np.arange(num_blocks, dtype=np.uint64)
|
||||
parts: list[np.ndarray] = []
|
||||
for i, base_addr in enumerate(nixl_agent_meta.kv_caches_base_addr):
|
||||
replicated = self._is_region_replicated(i)
|
||||
# Read our whole local region size from remote..
|
||||
@@ -1417,18 +1424,14 @@ class NixlBaseConnectorWorker:
|
||||
local_block_len = local_block_len // num_reads
|
||||
|
||||
page_size = nixl_agent_meta.block_lens[i]
|
||||
for block_id in range(num_blocks):
|
||||
block_offset = block_id * page_size
|
||||
# For each block, grab the kv heads chunk belonging to current local
|
||||
# tp rank of size local_block_len.
|
||||
addr = base_addr + block_offset + rank_offset
|
||||
result.append((addr, local_block_len, nixl_agent_meta.device_id))
|
||||
return result
|
||||
addrs = base_addr + rank_offset + block_arange * page_size
|
||||
parts.append(self._stack_descs(addrs, local_block_len, device_id))
|
||||
return np.concatenate(parts)
|
||||
|
||||
def register_local_xfer_handler(
|
||||
self,
|
||||
block_size: int,
|
||||
) -> tuple[int, list[tuple[int, int, int]]]:
|
||||
) -> tuple[int, np.ndarray]:
|
||||
"""
|
||||
Function used for register local xfer handler with local block_size or
|
||||
Remote block_size.
|
||||
@@ -1461,9 +1464,8 @@ class NixlBaseConnectorWorker:
|
||||
# remote has been seen. Currently we always register 4 regions
|
||||
# because local descs are created before knowing the remote TP.
|
||||
logger.debug("Registering local Mamba descriptors (4 regions/layer)")
|
||||
blocks_data.extend(
|
||||
self._build_mamba_local(local_base_addresses, block_size_ratio)
|
||||
)
|
||||
mamba = self._build_mamba_local(local_base_addresses, block_size_ratio)
|
||||
blocks_data = np.concatenate([blocks_data, mamba])
|
||||
|
||||
descs = self.nixl_wrapper.get_xfer_descs(blocks_data, self.nixl_memory_type)
|
||||
# NIXL_INIT_AGENT to be used for preparations of local descs.
|
||||
@@ -1650,13 +1652,8 @@ class NixlBaseConnectorWorker:
|
||||
engine_id,
|
||||
remote_tp_rank,
|
||||
)
|
||||
blocks_data.extend(
|
||||
self._build_mamba_remote(
|
||||
nixl_agent_meta,
|
||||
tp_ratio,
|
||||
transfer_info,
|
||||
)
|
||||
)
|
||||
mamba = self._build_mamba_remote(nixl_agent_meta, tp_ratio, transfer_info)
|
||||
blocks_data = np.concatenate([blocks_data, mamba])
|
||||
|
||||
# Register with NIXL.
|
||||
descs = self.nixl_wrapper.get_xfer_descs(blocks_data, self.nixl_memory_type)
|
||||
|
||||
Reference in New Issue
Block a user