[KV Connector] Support NIXL heterogeneous P/D block sizes for hybrid models (#49612)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Nick Hill
2026-07-28 06:45:02 -07:00
committed by GitHub
co-authored by Claude Fable 5
parent 9b9fc4039c
commit 601fa9a74e
7 changed files with 1014 additions and 111 deletions
@@ -12,6 +12,7 @@ import uuid
from collections import defaultdict
from collections.abc import Iterator
from concurrent.futures import Future, ThreadPoolExecutor
from functools import cached_property
from typing import TYPE_CHECKING, Any, cast
import msgspec
@@ -67,6 +68,7 @@ from vllm.distributed.parallel_state import (
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.network_utils import make_zmq_path
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.attention.backends.utils import get_kv_cache_layout
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
@@ -122,9 +124,11 @@ class NixlBaseConnectorWorker:
return (region_ids * num_blocks + block_arr).flatten()
# Compute desc ids per group using the right stride: FA descs have
# num_blocks entries per region (kernel granularity), SSM descs have
# logical_blocks entries per region (no kernel splitting).
logical_blocks = num_blocks // physical_blocks_per_logical
# num_blocks entries per region (kernel granularity, expanded by
# block_size_ratio for heterogeneous block sizes), SSM descs have
# logical_blocks entries per region (no kernel splitting, and never
# ratio-expanded since state blocks are indivisible).
logical_blocks = dst_num_blocks // physical_blocks_per_logical
all_descs: list[np.ndarray] = []
for i, group in enumerate(block_ids):
group_arr = np.asarray(group)
@@ -161,6 +165,7 @@ class NixlBaseConnectorWorker:
plan: TPMapping,
src_blocks_data: np.ndarray,
num_fa_descs: int,
block_size_ratio: int = 1,
) -> Iterator[list[tuple[int, int, int]]]:
"""Build split handle data for P_TP > D_TP scenario.
@@ -187,6 +192,11 @@ class NixlBaseConnectorWorker:
# Per-FA-descriptor replicate flag, in _build_fa_local emission order.
fa_desc_replicated = self._fa_desc_replicated(num_fa_descs)
assert block_size_ratio == 1 or fa_num_splits == 1 or all(fa_desc_replicated), (
"Head-sharded attention reads with P_TP > D_TP and heterogeneous "
"block sizes are not supported"
)
src_blocks_list = src_blocks_data.tolist()
for p_idx, p_rank in enumerate(plan.all_source_ranks):
@@ -442,9 +452,12 @@ class NixlBaseConnectorWorker:
# nixl_prepped_dlist_handle.
self.src_xfer_handles_by_block_size: dict[int, int] = {}
# Local descriptor arrays per remote block size (block_size_ratio>1),
# kept for building per-tp-ratio splits at the same granularity.
self.src_blocks_data_by_block_size: dict[int, np.ndarray] = {}
# Populated dynamically during handshake based on remote configuration.
# Keep track of regions at different tp_ratio values. tp_ratio->handles
self.src_xfer_handles_by_tp_ratio: dict[int, list[int]] = {}
# Per-source split handles, keyed by (tp_ratio, remote_block_size).
self.src_xfer_handles_by_tp_ratio: dict[tuple[int, int], list[int]] = {}
# Map of engine_id -> {tp_rank: nixl_prepped_dlist_handle (int)}.
self.dst_xfer_side_handles = defaultdict[EngineId, dict[int, int]](dict)
@@ -1258,11 +1271,7 @@ class NixlBaseConnectorWorker:
agent_metadata_bytes=encoder.encode(agent_metadata),
)
def _build_mamba_local(
self,
base_addresses: list[int],
block_size_ratio: int,
) -> np.ndarray:
def _build_mamba_local(self, base_addresses: list[int]) -> np.ndarray:
"""Build desc regions (conv sub-projections + ssm) per layer for
local mamba blocks with DS conv layout, as an Nx3 uint64 array.
@@ -1289,16 +1298,17 @@ class NixlBaseConnectorWorker:
| Key N-1 | Val N-1 | |Conv N-1| SSM N-1 |
+-------------------+ +--------------------+
|1st_split-2nd_split| |1st_split-2nd_split |
Mamba state blocks are indivisible (not token-extent data), so the
descriptors always use the local page geometry regardless of any
attention block-size ratio; their desc ids are likewise never
ratio-expanded (see _compute_desc_ids).
"""
assert block_size_ratio == 1, (
"Mamba 3-read transfer with block_size_ratio != 1 is not tested. "
f"Got block_size_ratio={block_size_ratio}."
)
assert base_addresses, "Local KV cache base addresses must not be empty."
assert self._conv_decomp is not None
conv_offsets = self._conv_decomp.local_conv_offsets
conv_size, ssm_size = self._mamba_ssm_size
num_blocks = self._logical_num_blocks * block_size_ratio
num_blocks = self._logical_num_blocks
physical_per_logical = self._physical_blocks_per_logical_kv_block
device_id = self.device_id
block_arange = np.arange(num_blocks, dtype=np.uint64)
@@ -1307,9 +1317,7 @@ class NixlBaseConnectorWorker:
for i, base_addr in enumerate(base_addresses):
# Jump one page_size, but ssm page_size may be bigger when kernel
# locks block size to a specific value (physical_per_logical scale).
page_stride = (
self.block_len_per_layer[i] // block_size_ratio * physical_per_logical
)
page_stride = self.block_len_per_layer[i] * physical_per_logical
blk_addrs = base_addr + block_arange * page_stride
for off, sz in conv_offsets:
parts.append(self._stack_descs(blk_addrs + off, sz, device_id))
@@ -1459,7 +1467,7 @@ class NixlBaseConnectorWorker:
self.device_id,
)
if self._has_mamba:
assert self.num_descs == len(blocks_data)
assert self.num_descs * block_size_ratio == len(blocks_data)
# TODO (ZhanqiuHu): For homogeneous TP (tp_ratio == 1), the 3-descs split
# is unnecessary — a single conv desc per block suffices. Consider
# adding a fast path that falls back to the standard 2-region
@@ -1467,7 +1475,7 @@ class NixlBaseConnectorWorker:
# remote has been seen. Currently we always register 4 regions
# because local descs are created before knowing the remote TP.
logger.debug("Registering local Mamba descriptors (4 regions/layer)")
mamba = self._build_mamba_local(local_base_addresses, block_size_ratio)
mamba = self._build_mamba_local(local_base_addresses)
blocks_data = np.concatenate([blocks_data, mamba])
descs = self.nixl_wrapper.get_xfer_descs(blocks_data, self.nixl_memory_type)
@@ -1607,29 +1615,44 @@ class NixlBaseConnectorWorker:
plan = self.tp_mappings[engine_id]
### (Optional) Register a local handler at the remote engine's block
### granularity (remote/prefill blocks smaller than local).
remote_block_size = nixl_agent_meta.block_size
src_blocks_data = self.src_blocks_data
if block_size_ratio > 1:
if remote_block_size not in self.src_xfer_handles_by_block_size:
handle, blocks_data = self.register_local_xfer_handler(
remote_block_size
)
self.src_xfer_handles_by_block_size[remote_block_size] = handle
self.src_blocks_data_by_block_size[remote_block_size] = blocks_data
src_blocks_data = self.src_blocks_data_by_block_size[remote_block_size]
### (Optional) Register local agent memory regions. MLA is not split.
split_key = (tp_ratio, remote_block_size)
if (
tp_ratio < 0
and (not self.use_mla or len(plan.all_source_ranks) > 1)
and tp_ratio not in self.src_xfer_handles_by_tp_ratio
and split_key not in self.src_xfer_handles_by_tp_ratio
):
# Remote tp_size > local tp_size: read from multiple remote ranks.
# Logically "split" own regions into per-source chunks. Hybrid
# MLA+SSM also needs this path: MLA is replicated and read once,
# while the SSM state is sharded across every remote TP rank.
# We only do this once per remote tp_size (replica-friendly).
self.src_xfer_handles_by_tp_ratio[tp_ratio] = []
# We only do this once per remote (tp_size, block_size).
self.src_xfer_handles_by_tp_ratio[split_key] = []
for handle_data in self._build_local_splits_from_plan(
plan,
self.src_blocks_data,
self.num_descs,
src_blocks_data,
self.num_descs * block_size_ratio,
block_size_ratio,
):
descs = self.nixl_wrapper.get_xfer_descs(
handle_data, self.nixl_memory_type
)
handle = self.nixl_wrapper.prep_xfer_dlist("NIXL_INIT_AGENT", descs)
self.src_xfer_handles_by_tp_ratio[tp_ratio].append(handle)
self.src_xfer_handles_by_tp_ratio[split_key].append(handle)
### Register remote agent memory regions
# With homogeneous TP, D pulls the whole kv cache from corresponding rank. With
@@ -1665,13 +1688,6 @@ class NixlBaseConnectorWorker:
self.nixl_wrapper.prep_xfer_dlist(remote_agent_name, descs)
)
if block_size_ratio > 1:
# when prefill with smaller block_size, we need to init a
# new handler with same block_len to match
self.src_xfer_handles_by_block_size[nixl_agent_meta.block_size] = (
self.register_local_xfer_handler(nixl_agent_meta.block_size)[0]
)
return remote_agent_name
def _validate_remote_agent_handshake(
@@ -1716,9 +1732,13 @@ class NixlBaseConnectorWorker:
"Disable prefix caching with --no-enable-prefix-caching."
)
if self._is_hma_required:
assert block_size_ratio == 1, (
"HMA does not support different remote block size yet"
if block_size_ratio != 1:
# Heterogeneous block sizes transfer at remote-block granularity;
# the untransferred tail of the last local attention block is
# zeroed in the receive post-process, and mamba state pages
# transfer 1:1 (never sub-split).
assert not self.use_host_buffer, (
"Heterogeneous block sizes are not supported with host buffer"
)
kv_cache_layout = (
self.kv_cache_layout
@@ -1875,27 +1895,57 @@ class NixlBaseConnectorWorker:
"d2h",
)
@cached_property
def _attention_kv_caches(self) -> list[torch.Tensor]:
"""Device KV caches of attention layers (mamba states excluded),
as consumed by the receive post-process."""
assert self.device_kv_caches, (
"_attention_kv_caches accessed before register_kv_caches"
)
mamba_layers = {
name
for g, group in enumerate(self.kv_cache_config.kv_cache_groups)
if _is_ssm_spec(self._group_spec_types[g])
for name in group.layer_names
}
kv_caches = self.device_kv_caches
return [cache for name, cache in kv_caches.items() if name not in mamba_layers]
def post_process_device_kv_on_receive(
self,
block_size_ratio: int,
block_ids_list: list[list[int]],
block_ids_list: list[tuple[list[int], int]],
convert: bool = True,
):
"""
Post process device kv cache after receiving from remote.
3 types of post processing supported:
3 types of conversion supported (``convert``):
* kv_cache_postprocess_layout => convert from HND to NHD
* kv_cache_postprocess_blksize => convert from small block size
to large block size
* kv_cache_postprocess_blksize_and_layout => convert from small
block size to large block size and convert from HND to NHD
The transfer only covers ``covered_sub_blocks`` remote-sized
sub-blocks of each request's local attention blocks; the rest was
clipped, either by remote-block pairing (block-size ratio) or by the
hetero-ppl front trim in ``_apply_prefix_caching``. Those blocks were
excluded from the scheduler's alloc-time KV zeroing (which would race
the RDMA write), so everything past the covered range is zeroed here.
Stale bytes would otherwise surface as garbage or NaNs once decode
grows into the untransferred tail.
"""
if len(self.device_kv_caches) == 0:
return
assert block_size_ratio >= 1, "Only nP < nD supported currently."
assert self.transfer_topo is not None
if self.enable_permute_local_kv and block_size_ratio > 1:
if not convert:
logger.debug(
"Post-processing device kv cache on receive by zeroing "
"untransferred blocks."
)
elif self.enable_permute_local_kv and block_size_ratio > 1:
logger.debug(
"Post-processing device kv cache on receive by converting "
"block_size with %sx bigger and permuting layout from HND"
@@ -1914,18 +1964,45 @@ class NixlBaseConnectorWorker:
block_size_ratio,
)
for block_ids in block_ids_list:
indices = torch.tensor(block_ids, device=self.device_type, dtype=torch.long)
attn_caches = self._attention_kv_caches
device = attn_caches[0].device
for block_ids, covered_sub_blocks in block_ids_list:
# Blocks the transfer didn't write: the token tail of the last
# partially covered block, then everything beyond it.
covered_blocks, sub_blocks_in_last = divmod(
covered_sub_blocks, block_size_ratio
)
first_stale = covered_blocks + (1 if sub_blocks_in_last else 0)
has_stale = first_stale < len(block_ids)
indices = None
if convert or has_stale:
indices = async_tensor_h2d(block_ids, device, torch.long)
for cache in self.device_kv_caches.values():
if self.enable_permute_local_kv and block_size_ratio > 1:
kv_postprocess_blksize_and_layout_on_receive(
cache, indices, block_size_ratio
)
elif self.enable_permute_local_kv:
kv_postprocess_layout_on_receive(cache, indices)
else:
kv_postprocess_blksize_on_receive(cache, indices, block_size_ratio)
if convert:
for cache in attn_caches:
if self.enable_permute_local_kv and block_size_ratio > 1:
kv_postprocess_blksize_and_layout_on_receive(
cache, indices, block_size_ratio
)
elif self.enable_permute_local_kv:
kv_postprocess_layout_on_receive(cache, indices)
else:
kv_postprocess_blksize_on_receive(
cache, indices, block_size_ratio
)
if sub_blocks_in_last:
last_block_id = block_ids[covered_blocks]
for cache in attn_caches:
# Both post-processed layouts leave tokens on dim 1.
sub_block_tokens = cache.shape[1] // block_size_ratio
zero_from = sub_blocks_in_last * sub_block_tokens
cache[last_block_id, zero_from:].zero_()
if has_stale:
assert indices is not None
stale_ids = indices[first_stale:]
for cache in attn_caches:
cache.index_fill_(0, stale_ids, 0)
def post_process_device_kv_on_receive_heterogeneous_attn(
self, block_ids: list[int]
@@ -1995,18 +2072,33 @@ class NixlBaseConnectorWorker:
if self.use_host_buffer:
self.sync_recved_kv_to_device(req_id, meta)
# post processing for heteroblocksize
# Post processing for heteroblocksize/layout, and for blocks the
# transfer clipped. The latter happens either at remote-block
# granularity (block_size_ratio > 1) or at kernel-block
# granularity, when equal kernel pages meet differing logical
# block sizes and _apply_prefix_caching front-trims to the
# minimum count (hybrid heterogeneous TP).
remote_info = self.transfer_topo.get_engine_info(meta.remote.engine_id)
block_size_ratio = self.transfer_topo.block_size_ratio(
remote_info.remote_block_size
)
if not self.use_mla and (
block_size_ratio > 1 or self.enable_permute_local_kv
):
assert not self._is_hma_required
block_ids_for_blocksize_post_process[block_size_ratio].append(
meta.local_physical_block_ids[0]
)
hetero_ppl = (
remote_info.remote_physical_blocks_per_logical
!= self._physical_blocks_per_logical_kv_block
)
if block_size_ratio > 1 or self.enable_permute_local_kv or hetero_ppl:
for g, local_group in enumerate(meta.local_physical_block_ids):
if not local_group or _is_ssm_spec(self._group_spec_types[g]):
continue
# Number of remote-sized sub-blocks the transfer covered;
# everything past this was clipped and must be zeroed.
covered_sub_blocks = min(
len(local_group) * block_size_ratio,
len(meta.remote.block_ids[g]),
)
block_ids_for_blocksize_post_process[block_size_ratio].append(
(local_group, covered_sub_blocks)
)
# post processing for heterogeneous attention
if self.enable_heterogeneous_attn_post_process:
block_ids_for_heterogeneous_attn_post_process.append(
@@ -2016,7 +2108,14 @@ class NixlBaseConnectorWorker:
block_size_ratio,
block_ids_list,
) in block_ids_for_blocksize_post_process.items():
self.post_process_device_kv_on_receive(block_size_ratio, block_ids_list)
# MLA never needs the block-size/layout conversion, but its
# clipped blocks still need zeroing.
convert = not self.use_mla and (
block_size_ratio > 1 or self.enable_permute_local_kv
)
self.post_process_device_kv_on_receive(
block_size_ratio, block_ids_list, convert
)
for block_ids in block_ids_for_heterogeneous_attn_post_process:
self.post_process_device_kv_on_receive_heterogeneous_attn(block_ids)
@@ -2206,6 +2305,45 @@ class NixlBaseConnectorWorker:
return mapped_2d.flatten().astype(np.int64)
def _map_block_ids_for_block_size_ratio(
self,
local_block_ids: BlockIds,
remote_block_ids: BlockIds,
block_size_ratio: int,
) -> tuple[BlockIds, BlockIds]:
"""Map attention-group block ids to remote-block granularity.
Each local attention block is split into ``block_size_ratio``
sub-blocks paired 1:1 with remote blocks. Sub-blocks beyond the
remote list — the untransferred tail of the last local block — are
clipped here and zeroed in the receive post-process. Mamba state
blocks are indivisible and transfer 1:1, unexpanded.
ex: remote (prefill) block ids with block_size 4:
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
Local (decode) block ids with block_size 16: [1, 2, 3] expand to
[4, 5, ..., 15], then clip to the first 10 to pair 1:1 with remote.
"""
mapped_local: list[list[int]] = []
mapped_remote: list[list[int]] = []
for i, remote_group in enumerate(remote_block_ids):
local_group = local_block_ids[i] if local_block_ids else []
if _is_ssm_spec(self._group_spec_types[i]):
mapped_local.append(list(local_group))
mapped_remote.append(list(remote_group))
continue
mapped = self.get_mapped_blocks(
np.asarray(local_group), block_size_ratio
).tolist()
if len(mapped) > len(remote_group):
mapped = mapped[: len(remote_group)]
mapped_local.append(mapped)
mapped_remote.append(list(remote_group))
if not any(mapped_local):
# Full prefix cache hit is indicated with an empty list.
return [], mapped_remote
return mapped_local, mapped_remote
def _logical_to_kernel_block_ids(self, block_ids: BlockIds, ratio: int) -> BlockIds:
"""
Convert block ids to kernel physical block ids.
@@ -2300,13 +2438,18 @@ class NixlBaseConnectorWorker:
remote_block_ids[i] = remote_group[-num_local_blocks:]
else:
# TODO Handle prefix caching with different block_sizes
max_padding = max(
self._physical_blocks_per_logical_kv_block,
remote_physical_per_logical,
# Allocation rounding legitimately leaves up to
# ppl - 1 trailing dead kernel blocks per side (plus one
# extra local block for the recomputed final token), so
# the counts may differ by up to the sum of the two
# ratios; anything larger indicates mismatched lists.
max_padding = (
self._physical_blocks_per_logical_kv_block
+ remote_physical_per_logical
)
assert abs(num_local_blocks - num_remote_blocks) < max_padding, (
assert abs(num_local_blocks - num_remote_blocks) <= max_padding, (
f"Group {i}: |{num_local_blocks} - "
f"{num_remote_blocks}| >= {max_padding}"
f"{num_remote_blocks}| > {max_padding}"
)
num_blocks = min(num_local_blocks, num_remote_blocks)
local_block_ids[i] = local_block_ids[i][:num_blocks]
@@ -5,8 +5,6 @@
import time
from typing import TYPE_CHECKING
import numpy as np
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker import (
NixlBaseConnectorWorker,
)
@@ -178,10 +176,10 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
)
# Get side handles.
if tp_ratio < 0 and (not self.use_mla or len(read_specs) > 1):
assert remote_block_size == self.block_size
# Remote tp_size > local tp_size: we must perform multiple
# reads. Get the memory chunk onto which we will write to.
local_xfer_side_handle = self.src_xfer_handles_by_tp_ratio[tp_ratio][i]
split_key = (tp_ratio, remote_block_size)
local_xfer_side_handle = self.src_xfer_handles_by_tp_ratio[split_key][i]
else:
# Single read from remote, we write to the whole memory region.
# Also handle remote block size different from local block size.
@@ -235,30 +233,11 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
remote_info.remote_block_size
)
if block_size_ratio > 1:
# TODO (NickLucche) assume HMA is off. Change to handle multiple KV groups.
assert not self._is_hma_required
local_block_ids0 = local_block_ids[0] if local_block_ids else []
remote_block_ids0 = remote_block_ids[0]
local_block_ids_mapped = self.get_mapped_blocks(
np.asarray(local_block_ids0), block_size_ratio
).tolist()
if len(local_block_ids_mapped) > len(remote_block_ids0):
# NOTE:
# get_mapped_blocks will always expand block_ids for n times.
# ex:
# prefill block_ids with block_size as 4:
# [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
# Local decode block_ids with block_size as 16: [1, 2, 3]
# expanded decode block_ids with get_mapped_blocks from [1, 2, 3] to
# [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
# Then we clip local to align with prefill
# [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] to
# [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
local_block_ids_mapped = local_block_ids_mapped[
: len(remote_block_ids0)
]
local_block_ids = [local_block_ids_mapped] if local_block_ids_mapped else []
remote_block_ids = [remote_block_ids0]
local_block_ids, remote_block_ids = (
self._map_block_ids_for_block_size_ratio(
local_block_ids, remote_block_ids, block_size_ratio
)
)
# NOTE(rob): having the staging blocks be on the READER side is
# not going to work well (since we will have to call rearrange tensors).
# after we detect the txn is complete (which means we cannot make the
@@ -39,7 +39,6 @@ from concurrent.futures import Future
from typing import TYPE_CHECKING, Any
import msgspec
import numpy as np
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker import (
@@ -553,8 +552,8 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
req_id,
)
if tp_ratio < 0 and not self.use_mla:
assert remote_block_size == self.block_size
local_xfer_side_handle = self.src_xfer_handles_by_tp_ratio[tp_ratio][i]
split_key = (tp_ratio, remote_block_size)
local_xfer_side_handle = self.src_xfer_handles_by_tp_ratio[split_key][i]
else:
local_xfer_side_handle = self.src_xfer_handles_by_block_size[
remote_block_size
@@ -606,18 +605,11 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
remote_info.remote_block_size
)
if block_size_ratio > 1:
assert not self._is_hma_required
local_block_ids0 = local_block_ids[0] if local_block_ids else []
remote_block_ids0 = remote_block_ids[0]
local_block_ids_mapped = self.get_mapped_blocks(
np.asarray(local_block_ids0), block_size_ratio
).tolist()
if len(local_block_ids_mapped) > len(remote_block_ids0):
local_block_ids_mapped = local_block_ids_mapped[
: len(remote_block_ids0)
]
local_block_ids = [local_block_ids_mapped] if local_block_ids_mapped else []
remote_block_ids = [remote_block_ids0]
local_block_ids, remote_block_ids = (
self._map_block_ids_for_block_size_ratio(
local_block_ids, remote_block_ids, block_size_ratio
)
)
notif_id = f"{remote_request_id}:{self.world_size}".encode()