[NIXL] Prepare KV transfer descriptors as NIXL 1.5 strided lists (#61032)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Ilia Yastrebov <iyastrebov@nvidia.com>
This commit is contained in:
co-authored by
Ilia Yastrebov
parent
9e2b579aab
commit
eaf25f675a
@@ -65,7 +65,11 @@ from vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils import
|
||||
MambaConvSplitInfo,
|
||||
derive_mamba_conv_split,
|
||||
)
|
||||
from vllm.distributed.nixl_utils import NixlWrapper, nixl_agent_config
|
||||
from vllm.distributed.nixl_utils import (
|
||||
NixlWrapper,
|
||||
check_nixl_version,
|
||||
nixl_agent_config,
|
||||
)
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_pcp_group,
|
||||
get_tensor_model_parallel_rank,
|
||||
@@ -277,20 +281,23 @@ class NixlBaseConnectorWorker:
|
||||
src_blocks_data: np.ndarray,
|
||||
num_fa_descs: int,
|
||||
block_size_ratio: int = 1,
|
||||
) -> Iterator[list[tuple[int, int, int]]]:
|
||||
) -> Iterator[np.ndarray]:
|
||||
"""Build split handle data for P_TP > D_TP scenario.
|
||||
|
||||
num_fa_descs is the boundary between FA and SSM descriptors.
|
||||
Split counts are derived from source_ranks_per_group lengths.
|
||||
FA uses rank_to_attention_slot for the slot offset;
|
||||
SSM uses the rank's positional index.
|
||||
Each Nx5 strided desc covers one region, so slicing it slices every block.
|
||||
"""
|
||||
fa_idx = next(
|
||||
i for i, t in enumerate(self._group_spec_types) if _is_attention_spec(t)
|
||||
)
|
||||
fa_num_splits = len(plan.source_ranks_per_group[fa_idx])
|
||||
|
||||
has_ssm_descs = num_fa_descs < len(src_blocks_data)
|
||||
starts = self._strided_desc_starts(src_blocks_data)
|
||||
num_descs = int(src_blocks_data[:, 4].sum())
|
||||
has_ssm_descs = num_fa_descs < num_descs
|
||||
ssm_idx = next(
|
||||
(i for i, t in enumerate(self._group_spec_types) if _is_ssm_spec(t)),
|
||||
None,
|
||||
@@ -300,38 +307,34 @@ class NixlBaseConnectorWorker:
|
||||
if has_ssm_descs and ssm_idx is not None
|
||||
else 0
|
||||
)
|
||||
|
||||
# Per-FA-descriptor replicate flag, in _build_fa_local emission order.
|
||||
fa_desc_replicated = self._fa_desc_replicated(num_fa_descs, block_size_ratio)
|
||||
sharded_desc_end = len(src_blocks_data) - (
|
||||
sharded_desc_end = num_descs - (
|
||||
self._logical_num_blocks if self._ple_region_index is not None else 0
|
||||
)
|
||||
assert num_fa_descs <= sharded_desc_end
|
||||
|
||||
assert block_size_ratio == 1 or fa_num_splits == 1 or all(fa_desc_replicated), (
|
||||
# REPLICATE (MLA) descs are read whole on every rank; SPLIT (full-attn)
|
||||
# descs take this rank's head slice. SSM descs before the PLE pages are
|
||||
# sharded by the rank's positional index; PLE pages are copied whole.
|
||||
is_fa = starts < num_fa_descs
|
||||
fa_replicated = self._fa_desc_replicated(
|
||||
starts[is_fa], num_fa_descs, block_size_ratio
|
||||
)
|
||||
assert block_size_ratio == 1 or fa_num_splits == 1 or fa_replicated.all(), (
|
||||
"Head-sharded attention reads with P_TP > D_TP and heterogeneous "
|
||||
"block sizes are not supported"
|
||||
)
|
||||
src_blocks_list = src_blocks_data.tolist()
|
||||
fa_split = is_fa.copy()
|
||||
fa_split[is_fa] = ~fa_replicated
|
||||
ssm_split = ~is_fa & (starts < sharded_desc_end)
|
||||
|
||||
for p_idx, p_rank in enumerate(plan.all_source_ranks):
|
||||
fa_slot = plan.rank_to_attention_slot.get(p_rank, 0)
|
||||
|
||||
handle: list[tuple[int, int, int]] = []
|
||||
for j, (addr, local_len, dev) in enumerate(src_blocks_list):
|
||||
if j < num_fa_descs:
|
||||
if fa_desc_replicated[j]:
|
||||
# REPLICATE (MLA): whole block written on every rank.
|
||||
handle.append((addr, local_len, dev))
|
||||
else:
|
||||
# SPLIT (full-attn): this rank's head slice.
|
||||
chunk = local_len // fa_num_splits
|
||||
handle.append((addr + fa_slot * chunk, chunk, dev))
|
||||
elif j < sharded_desc_end:
|
||||
chunk = local_len // ssm_num_splits
|
||||
handle.append((addr + p_idx * chunk, chunk, dev))
|
||||
else:
|
||||
handle.append((addr, local_len, dev))
|
||||
handle = src_blocks_data.copy()
|
||||
handle[fa_split, 1] //= fa_num_splits
|
||||
handle[fa_split, 0] += fa_slot * handle[fa_split, 1]
|
||||
if ssm_num_splits:
|
||||
handle[ssm_split, 1] //= ssm_num_splits
|
||||
handle[ssm_split, 0] += p_idx * handle[ssm_split, 1]
|
||||
yield handle
|
||||
|
||||
def _needs_split_local_xfer_handles(self, tp_ratio: int, plan: TPMapping) -> bool:
|
||||
@@ -346,25 +349,30 @@ class NixlBaseConnectorWorker:
|
||||
return tp_ratio < 0 and (not self.use_mla or len(plan.all_source_ranks) > 1)
|
||||
|
||||
def _fa_desc_replicated(
|
||||
self, num_fa_descs: int, block_size_ratio: int = 1
|
||||
) -> list[bool]:
|
||||
"""Per-FA-descriptor replicate flag, in _build_fa_local emission order:
|
||||
region-major, ``region_num_blocks[i] * block_size_ratio`` descs per
|
||||
region. Length ``num_fa_descs``.
|
||||
self,
|
||||
desc_starts: np.ndarray,
|
||||
num_fa_descs: int,
|
||||
block_size_ratio: int = 1,
|
||||
) -> np.ndarray:
|
||||
"""Replicate flag per FA strided desc, given each one's flat block index.
|
||||
|
||||
_build_fa_local emits regions in order, with
|
||||
``region_num_blocks[i] * block_size_ratio`` blocks per region. A strided
|
||||
desc never spans regions, so the region owning its first block decides.
|
||||
"""
|
||||
assert self.transfer_topo is not None
|
||||
n_regions = len(self.block_len_per_layer)
|
||||
if n_regions == 0 or self.num_regions == 0:
|
||||
return [False] * num_fa_descs
|
||||
return np.zeros(len(desc_starts), dtype=bool)
|
||||
region_indices = self._transfer_layer_region_indices or range(n_regions)
|
||||
flags: list[bool] = []
|
||||
for i in region_indices:
|
||||
num_descs = self.region_num_blocks[i] * block_size_ratio
|
||||
flags.extend([self._is_region_replicated(i)] * num_descs)
|
||||
assert len(flags) == num_fa_descs, (
|
||||
f"FA desc flags {len(flags)} != num_fa_descs {num_fa_descs}"
|
||||
region_ends = np.cumsum(
|
||||
[self.region_num_blocks[i] * block_size_ratio for i in region_indices]
|
||||
)
|
||||
return flags
|
||||
assert region_ends[-1] == num_fa_descs, (
|
||||
f"FA region blocks {region_ends[-1]} != num_fa_descs {num_fa_descs}"
|
||||
)
|
||||
replicated = np.array([self._is_region_replicated(i) for i in region_indices])
|
||||
return replicated[np.searchsorted(region_ends, desc_starts, side="right")]
|
||||
|
||||
def _is_region_replicated(self, region_idx: int) -> bool:
|
||||
"""Whether region ``region_idx`` is transferred REPLICATE vs SPLIT.
|
||||
@@ -512,6 +520,8 @@ class NixlBaseConnectorWorker:
|
||||
if nixl_wrapper_cls is None:
|
||||
logger.error("NIXL is not available")
|
||||
raise RuntimeError("NIXL is not available")
|
||||
# Descriptor lists are prepared in NIXL's strided (Nx5) form.
|
||||
check_nixl_version("1.5.0")
|
||||
logger.info("Initializing NIXL wrapper")
|
||||
logger.info("Initializing NIXL worker %s", engine_id)
|
||||
|
||||
@@ -1492,7 +1502,6 @@ class NixlBaseConnectorWorker:
|
||||
if isinstance(layer_spec, MambaSpec)
|
||||
else logical_num_blocks * self._physical_blocks_per_logical_kv_block
|
||||
)
|
||||
base_addr = cache.data_ptr()
|
||||
is_mla_region = isinstance(
|
||||
layer_spec, (MLAAttentionSpec, SlidingWindowMLASpec)
|
||||
)
|
||||
@@ -1827,7 +1836,7 @@ class NixlBaseConnectorWorker:
|
||||
|
||||
def _build_mamba_local(self, base_addresses: list[int]) -> np.ndarray:
|
||||
"""Build desc regions (conv sub-projections + ssm) per layer for
|
||||
local mamba blocks with DS conv layout, as an Nx3 uint64 array.
|
||||
local mamba blocks with DS conv layout, as Nx5 uint64 strided descs.
|
||||
|
||||
A Mamba block interleaves conv and SSM state, which crucially differ in
|
||||
size, so the two are indexed as separate sub-regions. Attention blocks
|
||||
@@ -1865,28 +1874,32 @@ class NixlBaseConnectorWorker:
|
||||
num_blocks = self._logical_num_blocks
|
||||
physical_per_logical = self._physical_blocks_per_logical_kv_block
|
||||
device_id = self.device_id
|
||||
block_arange = np.arange(num_blocks, dtype=np.uint64)
|
||||
parts: list[np.ndarray] = []
|
||||
strided_descs: list[tuple[int, int, int, int, int]] = []
|
||||
|
||||
region_indices = self._ssm_region_indices or range(len(base_addresses))
|
||||
for i in region_indices:
|
||||
base_addr = base_addresses[i]
|
||||
block_stride = self.block_stride_per_layer[i] * physical_per_logical
|
||||
blk_addrs = base_addr + block_arange * block_stride
|
||||
for off, sz in conv_offsets:
|
||||
parts.append(self._stack_descs(blk_addrs + off, sz, device_id))
|
||||
strided_descs.append(
|
||||
(base_addr + off, sz, device_id, block_stride, num_blocks)
|
||||
)
|
||||
# SSM temporal state follows the conv state.
|
||||
parts.append(self._stack_descs(blk_addrs + conv_size, ssm_size, device_id))
|
||||
strided_descs.append(
|
||||
(base_addr + conv_size, ssm_size, device_id, block_stride, num_blocks)
|
||||
)
|
||||
|
||||
if (region_index := self._ple_region_index) is not None:
|
||||
block_len = self._ple_page_len() * physical_per_logical
|
||||
block_stride = (
|
||||
self.block_stride_per_layer[region_index] * physical_per_logical
|
||||
)
|
||||
block_addrs = base_addresses[region_index] + block_arange * block_stride
|
||||
parts.append(self._stack_descs(block_addrs, block_len, self.device_id))
|
||||
base_addr = base_addresses[region_index]
|
||||
strided_descs.append(
|
||||
(base_addr, block_len, device_id, block_stride, num_blocks)
|
||||
)
|
||||
|
||||
return np.concatenate(parts)
|
||||
return self._pack_strided_descs(strided_descs)
|
||||
|
||||
def _build_mamba_remote(
|
||||
self,
|
||||
@@ -1896,7 +1909,7 @@ class NixlBaseConnectorWorker:
|
||||
) -> np.ndarray:
|
||||
"""Build remote desc regions (conv sub-projections + ssm) per layer.
|
||||
For hetero-TP, each D rank reads only its sub-projection slice from
|
||||
the P rank. Returns an Nx3 uint64 array."""
|
||||
the P rank. Returns Nx5 uint64 strided descs."""
|
||||
assert nixl_agent_meta.kv_caches_base_addr, (
|
||||
"Remote KV cache base addresses must not be empty."
|
||||
)
|
||||
@@ -1916,9 +1929,8 @@ class NixlBaseConnectorWorker:
|
||||
remote_physical_per_logical = transfer_info.remote_physical_blocks_per_logical
|
||||
num_blocks = nixl_agent_meta.num_blocks // remote_physical_per_logical
|
||||
device_id = nixl_agent_meta.device_id
|
||||
block_arange = np.arange(num_blocks, dtype=np.uint64)
|
||||
|
||||
parts: list[np.ndarray] = []
|
||||
strided_descs: list[tuple[int, int, int, int, int]] = []
|
||||
# NOTE (ZhanqiuHu): use per-layer block_lens[i], not [0], in case
|
||||
# block lengths vary across layers (e.g. MLA).
|
||||
region_indices = self._ssm_region_indices or range(
|
||||
@@ -1929,12 +1941,15 @@ class NixlBaseConnectorWorker:
|
||||
block_stride = (
|
||||
nixl_agent_meta.block_strides[i] * remote_physical_per_logical
|
||||
)
|
||||
blk_addrs = base_addr + block_arange * block_stride
|
||||
for off, sz in conv_offsets:
|
||||
parts.append(self._stack_descs(blk_addrs + off, sz, device_id))
|
||||
strided_descs.append(
|
||||
(base_addr + off, sz, device_id, block_stride, num_blocks)
|
||||
)
|
||||
# SSM temporal state is also TP-sharded on the heads dimension.
|
||||
ssm_addrs = blk_addrs + conv_size_remote + local_offset * ssm_read_size
|
||||
parts.append(self._stack_descs(ssm_addrs, ssm_read_size, device_id))
|
||||
ssm_addr = base_addr + conv_size_remote + local_offset * ssm_read_size
|
||||
strided_descs.append(
|
||||
(ssm_addr, ssm_read_size, device_id, block_stride, num_blocks)
|
||||
)
|
||||
|
||||
if (region_index := self._ple_region_index) is not None:
|
||||
local_block_len = (
|
||||
@@ -1954,52 +1969,85 @@ class NixlBaseConnectorWorker:
|
||||
nixl_agent_meta.block_strides[region_index]
|
||||
* remote_physical_per_logical
|
||||
)
|
||||
block_addrs = (
|
||||
nixl_agent_meta.kv_caches_base_addr[region_index]
|
||||
+ block_arange * remote_block_stride
|
||||
addr = nixl_agent_meta.kv_caches_base_addr[region_index]
|
||||
strided_descs.append(
|
||||
(addr, remote_block_len, device_id, remote_block_stride, num_blocks)
|
||||
)
|
||||
parts.append(self._stack_descs(block_addrs, remote_block_len, device_id))
|
||||
|
||||
return np.concatenate(parts)
|
||||
return self._pack_strided_descs(strided_descs)
|
||||
|
||||
@staticmethod
|
||||
def _stack_descs(addrs: np.ndarray, length: int, device_id: int) -> np.ndarray:
|
||||
out = np.empty((addrs.shape[0], 3), dtype=np.uint64)
|
||||
out[:, 0] = addrs
|
||||
out[:, 1] = length
|
||||
out[:, 2] = device_id
|
||||
return out
|
||||
def _pack_strided_descs(
|
||||
strided_descs: list[tuple[int, int, int, int, int]],
|
||||
) -> np.ndarray:
|
||||
"""Pack (addr, len, dev_id, stride, count) rows into an Nx5 uint64 array.
|
||||
|
||||
Each row describes `count` blocks of `len` bytes whose starts are
|
||||
`stride` bytes apart; NIXL >= 1.5 prepares this layout directly.
|
||||
"""
|
||||
return np.array(strided_descs, dtype=np.uint64).reshape(-1, 5)
|
||||
|
||||
@staticmethod
|
||||
def _block_strided_descs(
|
||||
base_addr: int,
|
||||
block_len: int,
|
||||
block_stride: int,
|
||||
num_blocks: int,
|
||||
block_size_ratio: int,
|
||||
device_id: int,
|
||||
) -> np.ndarray:
|
||||
"""Nx5 strided descs for `num_blocks` blocks, each split into
|
||||
`block_size_ratio` sub-blocks of `block_len` bytes.
|
||||
|
||||
Evenly spaced sub-blocks collapse into one strided desc. In a padded
|
||||
block the sub-blocks abut only within the block, so each block gets
|
||||
its own.
|
||||
"""
|
||||
if block_size_ratio == 1 or block_stride == block_size_ratio * block_len:
|
||||
stride = block_stride // block_size_ratio
|
||||
count = num_blocks * block_size_ratio
|
||||
return np.array(
|
||||
[[base_addr, block_len, device_id, stride, count]], dtype=np.uint64
|
||||
)
|
||||
strided_descs = np.empty((num_blocks, 5), dtype=np.uint64)
|
||||
strided_descs[:, 0] = (
|
||||
base_addr + np.arange(num_blocks, dtype=np.uint64) * block_stride
|
||||
)
|
||||
strided_descs[:, 1] = block_len
|
||||
strided_descs[:, 2] = device_id
|
||||
strided_descs[:, 3] = block_len
|
||||
strided_descs[:, 4] = block_size_ratio
|
||||
return strided_descs
|
||||
|
||||
@staticmethod
|
||||
def _strided_desc_starts(strided_descs: np.ndarray) -> np.ndarray:
|
||||
"""Flat block index of each strided desc's first block."""
|
||||
counts = strided_descs[:, 4]
|
||||
return np.cumsum(counts) - counts
|
||||
|
||||
def _build_fa_local(
|
||||
self,
|
||||
base_addresses: list[int],
|
||||
block_size_ratio: int,
|
||||
) -> np.ndarray:
|
||||
"""Build local FA descriptors as an Nx3 uint64 array."""
|
||||
"""Build local FA descriptors as Nx5 uint64 strided descs."""
|
||||
assert self.transfer_topo is not None
|
||||
assert base_addresses, "Local KV cache base addresses must not be empty."
|
||||
device_id = self.device_id
|
||||
region_indices = self._transfer_layer_region_indices or range(
|
||||
len(base_addresses)
|
||||
)
|
||||
parts: list[np.ndarray] = []
|
||||
for i in region_indices:
|
||||
base_addr = base_addresses[i]
|
||||
block_len = self.block_len_per_layer[i] // block_size_ratio
|
||||
logical_blocks = np.repeat(
|
||||
np.arange(self.region_num_blocks[i], dtype=np.uint64),
|
||||
block_size_ratio,
|
||||
)
|
||||
split_offsets = np.tile(
|
||||
np.arange(block_size_ratio, dtype=np.uint64),
|
||||
parts = [
|
||||
self._block_strided_descs(
|
||||
base_addresses[i],
|
||||
self.block_len_per_layer[i] // block_size_ratio,
|
||||
self.block_stride_per_layer[i],
|
||||
self.region_num_blocks[i],
|
||||
block_size_ratio,
|
||||
device_id,
|
||||
)
|
||||
addrs = (
|
||||
base_addr
|
||||
+ logical_blocks * self.block_stride_per_layer[i]
|
||||
+ split_offsets * block_len
|
||||
)
|
||||
parts.append(self._stack_descs(addrs, block_len, device_id))
|
||||
for i in region_indices
|
||||
]
|
||||
return np.concatenate(parts)
|
||||
|
||||
def _build_fa_remote(
|
||||
@@ -2008,7 +2056,7 @@ class NixlBaseConnectorWorker:
|
||||
nixl_agent_meta: NixlAgentMetadata,
|
||||
block_size_ratio: int,
|
||||
) -> np.ndarray:
|
||||
"""Build remote FA descriptors as an Nx3 uint64 array."""
|
||||
"""Build remote FA descriptors as Nx5 uint64 strided descs."""
|
||||
assert self.transfer_topo is not None
|
||||
assert nixl_agent_meta.kv_caches_base_addr, (
|
||||
"Remote KV cache base addresses must not be empty."
|
||||
@@ -2024,7 +2072,7 @@ class NixlBaseConnectorWorker:
|
||||
] * len(nixl_agent_meta.kv_caches_base_addr)
|
||||
device_id = nixl_agent_meta.device_id
|
||||
block_strides = nixl_agent_meta.block_strides
|
||||
parts: list[np.ndarray] = []
|
||||
strided_descs: list[tuple[int, int, int, int, int]] = []
|
||||
use_layer_name_routing = bool(self._transfer_layer_region_indices)
|
||||
for i, base_addr in enumerate(nixl_agent_meta.kv_caches_base_addr):
|
||||
local_region = (
|
||||
@@ -2046,10 +2094,12 @@ class NixlBaseConnectorWorker:
|
||||
)
|
||||
local_block_len = local_block_len // num_reads
|
||||
|
||||
block_arange = np.arange(region_num_blocks[i], dtype=np.uint64)
|
||||
addrs = base_addr + rank_offset + block_arange * block_strides[i]
|
||||
parts.append(self._stack_descs(addrs, local_block_len, device_id))
|
||||
return np.concatenate(parts)
|
||||
addr = base_addr + rank_offset
|
||||
count = region_num_blocks[i]
|
||||
strided_descs.append(
|
||||
(addr, local_block_len, device_id, block_strides[i], count)
|
||||
)
|
||||
return self._pack_strided_descs(strided_descs)
|
||||
|
||||
def register_local_xfer_handler(
|
||||
self,
|
||||
@@ -2072,13 +2122,13 @@ class NixlBaseConnectorWorker:
|
||||
blocks_data = self._build_fa_local(local_base_addresses, block_size_ratio)
|
||||
logger.debug(
|
||||
"Created %s blocks for src engine %s and rank %s on device id %s",
|
||||
len(blocks_data),
|
||||
blocks_data[:, 4].sum(),
|
||||
self.engine_id,
|
||||
self.tp_rank,
|
||||
self.device_id,
|
||||
)
|
||||
if self._has_mamba:
|
||||
assert self.num_descs * block_size_ratio == len(blocks_data)
|
||||
assert self.num_descs * block_size_ratio == blocks_data[:, 4].sum()
|
||||
# TODO (ZhanqiuHu): For homogeneous TP (tp_ratio == 1), the 3-descs split
|
||||
# is unnecessary — a single conv desc per block suffices. Consider
|
||||
# adding a fast path that falls back to the standard 2-region
|
||||
@@ -2104,7 +2154,7 @@ class NixlBaseConnectorWorker:
|
||||
)
|
||||
]
|
||||
)
|
||||
assert len(desc_is_dram) == len(blocks_data)
|
||||
assert len(desc_is_dram) == blocks_data[:, 4].sum()
|
||||
desc_pos = np.empty(len(desc_is_dram), dtype=np.int64)
|
||||
dram_idx = np.where(desc_is_dram)[0]
|
||||
vram_idx = np.where(~desc_is_dram)[0]
|
||||
@@ -2112,28 +2162,32 @@ class NixlBaseConnectorWorker:
|
||||
desc_pos[vram_idx] = np.arange(len(vram_idx), dtype=np.int64)
|
||||
self._desc_is_dram_by_block_size[block_size] = desc_is_dram
|
||||
self._desc_pos_by_block_size[block_size] = desc_pos
|
||||
|
||||
# DRAM descriptors are registered under CPU device 0.
|
||||
blocks_data[desc_is_dram, 2] = 0
|
||||
dram_blocks = blocks_data[dram_idx]
|
||||
vram_blocks = blocks_data[vram_idx]
|
||||
# A strided desc never spans regions, so its first block decides.
|
||||
strided_is_dram = desc_is_dram[self._strided_desc_starts(blocks_data)]
|
||||
blocks_data[strided_is_dram, 2] = 0
|
||||
if self._skip_dram_xfer:
|
||||
self._dram_src_handles_by_block_size[block_size] = None
|
||||
else:
|
||||
dram_descs = self.nixl_wrapper.get_xfer_descs(dram_blocks, "DRAM")
|
||||
self._dram_src_handles_by_block_size[block_size] = (
|
||||
self.nixl_wrapper.prep_xfer_dlist("NIXL_INIT_AGENT", dram_descs)
|
||||
self.nixl_wrapper.prep_xfer_dlist(
|
||||
"NIXL_INIT_AGENT",
|
||||
blocks_data[strided_is_dram],
|
||||
mem_type="DRAM",
|
||||
)
|
||||
)
|
||||
descs = self.nixl_wrapper.get_xfer_descs(vram_blocks, self.nixl_memory_type)
|
||||
return (
|
||||
self.nixl_wrapper.prep_xfer_dlist("NIXL_INIT_AGENT", descs),
|
||||
blocks_data,
|
||||
device_handle = self.nixl_wrapper.prep_xfer_dlist(
|
||||
"NIXL_INIT_AGENT",
|
||||
blocks_data[~strided_is_dram],
|
||||
mem_type=self.nixl_memory_type,
|
||||
)
|
||||
return device_handle, blocks_data
|
||||
|
||||
mem_type = self.region_mem_types[0]
|
||||
descs = self.nixl_wrapper.get_xfer_descs(blocks_data, mem_type)
|
||||
# NIXL_INIT_AGENT to be used for preparations of local descs.
|
||||
return self.nixl_wrapper.prep_xfer_dlist("NIXL_INIT_AGENT", descs), blocks_data
|
||||
handle = self.nixl_wrapper.prep_xfer_dlist(
|
||||
"NIXL_INIT_AGENT", blocks_data, mem_type=self.region_mem_types[0]
|
||||
)
|
||||
return handle, blocks_data
|
||||
|
||||
def add_remote_agent(
|
||||
self,
|
||||
@@ -2350,23 +2404,23 @@ class NixlBaseConnectorWorker:
|
||||
block_size_ratio,
|
||||
):
|
||||
if self._mixed_mem_types:
|
||||
handle_data = np.asarray(handle_data, dtype=np.uint64)
|
||||
desc_is_dram = self._desc_is_dram_by_block_size[remote_block_size]
|
||||
strided_is_dram = desc_is_dram[
|
||||
self._strided_desc_starts(handle_data)
|
||||
]
|
||||
if self._skip_dram_xfer:
|
||||
dram_handle = None
|
||||
else:
|
||||
dram_descs = self.nixl_wrapper.get_xfer_descs(
|
||||
handle_data[desc_is_dram], "DRAM"
|
||||
)
|
||||
dram_handle = self.nixl_wrapper.prep_xfer_dlist(
|
||||
"NIXL_INIT_AGENT", dram_descs
|
||||
"NIXL_INIT_AGENT",
|
||||
handle_data[strided_is_dram],
|
||||
mem_type="DRAM",
|
||||
)
|
||||
self._dram_src_handles_by_tp_ratio[split_key].append(dram_handle)
|
||||
handle_data = handle_data[~desc_is_dram]
|
||||
descs = self.nixl_wrapper.get_xfer_descs(
|
||||
handle_data, self.nixl_memory_type
|
||||
handle_data = handle_data[~strided_is_dram]
|
||||
handle = self.nixl_wrapper.prep_xfer_dlist(
|
||||
"NIXL_INIT_AGENT", handle_data, mem_type=self.nixl_memory_type
|
||||
)
|
||||
handle = self.nixl_wrapper.prep_xfer_dlist("NIXL_INIT_AGENT", descs)
|
||||
self.src_xfer_handles_by_tp_ratio[split_key].append(handle)
|
||||
|
||||
### Register remote agent memory regions
|
||||
@@ -2383,7 +2437,7 @@ class NixlBaseConnectorWorker:
|
||||
)
|
||||
logger.debug(
|
||||
"Created %s blocks for dst engine %s with remote rank %s and local rank %s",
|
||||
len(blocks_data),
|
||||
blocks_data[:, 4].sum(),
|
||||
engine_id,
|
||||
remote_tp_rank,
|
||||
self.tp_rank,
|
||||
@@ -2402,9 +2456,10 @@ class NixlBaseConnectorWorker:
|
||||
raise NotImplementedError(
|
||||
"NIXL pull does not yet support a mixed-memory producer"
|
||||
)
|
||||
descs = self.nixl_wrapper.get_xfer_descs(blocks_data, remote_mem_types.pop())
|
||||
self.dst_xfer_side_handles[engine_id][remote_tp_rank] = (
|
||||
self.nixl_wrapper.prep_xfer_dlist(remote_agent_name, descs)
|
||||
self.nixl_wrapper.prep_xfer_dlist(
|
||||
remote_agent_name, blocks_data, mem_type=remote_mem_types.pop()
|
||||
)
|
||||
)
|
||||
|
||||
return remote_agent_name
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import functools
|
||||
import importlib
|
||||
import os
|
||||
import sys
|
||||
@@ -91,6 +92,24 @@ def is_nixl_available() -> bool:
|
||||
return pkg in sys.modules or importlib.util.find_spec(pkg) is not None
|
||||
|
||||
|
||||
@functools.cache
|
||||
def check_nixl_version(min_version: str) -> None:
|
||||
"""Raise if the installed NIXL package is older than ``min_version``."""
|
||||
from importlib.metadata import packages_distributions, version
|
||||
|
||||
from packaging.version import Version
|
||||
|
||||
pkg = _get_nixl_package_name()
|
||||
for dist in packages_distributions().get(pkg, ()):
|
||||
installed = version(dist)
|
||||
if Version(installed) < Version(min_version):
|
||||
raise RuntimeError(
|
||||
f"NIXL >= {min_version} is required, but {dist}=={installed} "
|
||||
"is installed"
|
||||
)
|
||||
return
|
||||
|
||||
|
||||
def alias_nixl_for_ray() -> None:
|
||||
"""Let Ray's ``nixl._api`` import resolve to the ROCm implementation."""
|
||||
if not current_platform.is_rocm() or "nixl._api" in sys.modules:
|
||||
@@ -112,5 +131,6 @@ __all__ = [
|
||||
"nixl_agent_config",
|
||||
"nixlXferTelemetry",
|
||||
"is_nixl_available",
|
||||
"check_nixl_version",
|
||||
"alias_nixl_for_ray",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user