[KVConnector][NIXL] Support attention-HMA layouts in pipeline-parallel push prefill (#50494)

Signed-off-by: zixi-qi <zixi@inferact.ai>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Yiliu Dong <91178480+qianlihuang@users.noreply.github.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
qizixi
2026-09-15 15:43:56 -07:00
committed by GitHub
co-authored by Claude Yiliu Dong Codex
parent c6fa1f05d1
commit 0fefffc934
8 changed files with 713 additions and 49 deletions
@@ -164,6 +164,10 @@ class NixlBaseConnectorWorker:
# Overridden by NixlPushConnectorWorker.
_TRANSFER_MODE: str = "pull"
# Layer-name routing is supported only by NixlPushConnector for HMA KV
# caches under PP.
_supports_pp_hma = False
def _compute_desc_ids(
self,
block_ids: BlockIds,
@@ -191,28 +195,42 @@ class NixlBaseConnectorWorker:
num_blocks = dst_num_blocks
if block_size_ratio is not None:
num_blocks = int(num_blocks * block_size_ratio)
num_regions = self.num_regions or len(self.block_len_per_layer)
use_layer_name_routing = bool(self._transfer_layer_group_ids)
num_desc_regions = (
len(self._transfer_layer_group_ids)
if use_layer_name_routing
else self.num_regions or len(self.block_len_per_layer)
)
if region_num_blocks is None:
region_num_blocks = [num_blocks] * num_regions
region_num_blocks = [num_blocks] * num_desc_regions
elif block_size_ratio is not None:
region_num_blocks = [
int(count * block_size_ratio) for count in region_num_blocks
]
if region_num_blocks:
num_regions = len(region_num_blocks)
assert len(region_num_blocks) == num_regions
if region_num_blocks and not use_layer_name_routing:
num_desc_regions = len(region_num_blocks)
assert len(region_num_blocks) == num_desc_regions
region_offsets = np.cumsum([0, *region_num_blocks[:-1]])
num_fa_descs = sum(region_num_blocks)
if use_layer_name_routing:
# Descriptors in layer order, using each layer's KV-cache group.
return np.concatenate(
[
np.asarray(block_ids[g], dtype=np.int32) + region_offsets[k]
for k, g in enumerate(self._transfer_layer_group_ids)
]
)
# All-attention fast path.
if num_ssm_regions == 0:
if region_group_ids is None:
region_group_ids = self.region_group_ids
if uses_region_group_mapping is None:
uses_region_group_mapping = self._uses_region_group_mapping
if not region_group_ids and num_regions == 1:
if not region_group_ids and num_desc_regions == 1:
region_group_ids = [0]
assert len(region_group_ids) == num_regions
assert len(region_group_ids) == num_desc_regions
if uses_region_group_mapping is None:
uses_region_group_mapping = len(set(region_group_ids)) > 1
region_group_ids_array = np.asarray(region_group_ids, dtype=np.int32)
@@ -392,6 +410,103 @@ class NixlBaseConnectorWorker:
"""
return region_idx < len(self._region_is_mla) and self._region_is_mla[region_idx]
def _set_region_layers(self, region_layers: list[list[str]]) -> None:
layer_names = [layer_name for region in region_layers for layer_name in region]
assert len(layer_names) == len(set(layer_names)), (
"A KV cache layer spans multiple NIXL regions"
)
self.region_members: list[list[str]] = region_layers
if not self._requires_layer_name_routing():
return
group_by_layer = self.kv_cache_config.transfer_group_index_by_layer
ungrouped = [name for name in layer_names if name not in group_by_layer]
assert not ungrouped, f"KV cache layers outside any local group: {ungrouped}"
self._transfer_layer_names = tuple(layer_names)
self._transfer_layer_region_indices = tuple(
region_idx
for region_idx, region in enumerate(region_layers)
for _ in region
)
self._transfer_layer_group_ids = tuple(
group_by_layer[name] for name in layer_names
)
def _requires_layer_name_routing(self) -> bool:
"""Whether PP push must match HMA layers by name, not region index."""
return (
self._supports_pp_hma
and self.pp_size > 1
and self._is_hma_required
and not self._has_mamba
)
def _align_remote_regions_by_layer(
self, nixl_agent_meta: NixlAgentMetadata
) -> None:
"""Select remote regions for this PP stage in local layer order."""
remote_region_layers = nixl_agent_meta.region_members
# Region-index routing here would silently transfer stale KV.
assert remote_region_layers, "Remote advertised no region_members"
assert (
len(nixl_agent_meta.kv_caches_base_addr)
== len(nixl_agent_meta.block_lens)
== len(nixl_agent_meta.block_strides)
== len(remote_region_layers)
), "Remote region metadata lengths disagree"
assert all(
values is None or len(values) == len(remote_region_layers)
for values in (
nixl_agent_meta.region_num_blocks,
nixl_agent_meta.region_group_ids,
nixl_agent_meta.region_names,
nixl_agent_meta.region_mem_types,
)
), "Remote region metadata lengths disagree"
remote_region_by_layer: dict[str, int] = {}
for region_idx, layer_names in enumerate(remote_region_layers):
for layer_name in layer_names:
assert layer_name not in remote_region_by_layer, (
f"Remote advertised layer {layer_name!r} in multiple regions"
)
remote_region_by_layer[layer_name] = region_idx
missing = [
name
for name in self._transfer_layer_names
if name not in remote_region_by_layer
]
assert not missing, f"Remote is missing locally owned layers: {missing}"
remote_regions = [
remote_region_by_layer[name] for name in self._transfer_layer_names
]
nixl_agent_meta.kv_caches_base_addr = [
nixl_agent_meta.kv_caches_base_addr[i] for i in remote_regions
]
nixl_agent_meta.block_lens = [
nixl_agent_meta.block_lens[i] for i in remote_regions
]
nixl_agent_meta.block_strides = [
nixl_agent_meta.block_strides[i] for i in remote_regions
]
if nixl_agent_meta.region_num_blocks is not None:
nixl_agent_meta.region_num_blocks = [
nixl_agent_meta.region_num_blocks[i] for i in remote_regions
]
if nixl_agent_meta.region_group_ids is not None:
nixl_agent_meta.region_group_ids = [
nixl_agent_meta.region_group_ids[i] for i in remote_regions
]
if nixl_agent_meta.region_mem_types is not None:
nixl_agent_meta.region_mem_types = [
nixl_agent_meta.region_mem_types[i] for i in remote_regions
]
if nixl_agent_meta.region_names is not None:
nixl_agent_meta.region_names = list(self._transfer_layer_names)
# One layer per region keeps a second alignment pass a no-op.
nixl_agent_meta.region_members = [[name] for name in self._transfer_layer_names]
def __init__(
self,
vllm_config: "VllmConfig",
@@ -638,18 +753,26 @@ class NixlBaseConnectorWorker:
# transfers into the matching sub-range of a PP=1 remote's regions.
self.pp_size = vllm_config.parallel_config.pipeline_parallel_size
self._remote_region_offset = 0
# PP push slices regions per layer (uniform count); HMA breaks that.
if self.pp_size > 1 and self._is_hma_required:
raise NotImplementedError(
"NixlPushConnector does not support pipeline_parallel_size > 1 "
"with hybrid KV cache layouts (HMA) yet."
)
# Decode-side PP is unsupported (completions counted per consumer rank).
if vllm_config.kv_transfer_config.kv_role == "kv_consumer" and self.pp_size > 1:
raise NotImplementedError(
"NixlPushConnector consumer (decode) does not support "
"pipeline_parallel_size > 1."
)
if self.pp_size > 1:
if self._is_hma_required and not self._supports_pp_hma:
raise NotImplementedError(
"NixlConnector (pull) does not support pipeline_parallel_size "
"> 1 with hybrid KV cache layouts (HMA); use NixlPushConnector "
"for PP + HMA."
)
# PP push routes HMA (hybrid) attention layouts by layer name;
# Mamba/SSM hybrids are not yet supported under PP.
if self._has_mamba:
raise NotImplementedError(
"NixlPushConnector does not support pipeline_parallel_size > 1 "
"with Mamba/SSM hybrid KV cache layouts yet."
)
# Decode-side PP is unsupported (completions counted per consumer rank).
if vllm_config.kv_transfer_config.kv_role == "kv_consumer":
raise NotImplementedError(
"NixlPushConnector consumer (decode) does not support "
"pipeline_parallel_size > 1."
)
# Keep heartbeat handshakes to a PP-sharded producer notif-only.
self._hb_handshake_notif_only = False
@@ -773,6 +896,13 @@ class NixlBaseConnectorWorker:
# within a block (BLHNC/BHLNC), where stride > block_len.
self.block_stride_per_layer = list[int]()
# Region layer membership is populated for HMA layouts.
self.region_members = []
# Local layer order, set by ``_set_region_layers`` only when
# routing by layer name. Empty means region-index routing.
self._transfer_layer_names = tuple[str, ...]()
self._transfer_layer_region_indices = tuple[int, ...]()
self._transfer_layer_group_ids = tuple[int, ...]()
# Per-engine TP mappings. Generated during handshake.
self.tp_mappings: dict[EngineId, TPMapping] = {}
@@ -1306,6 +1436,11 @@ class NixlBaseConnectorWorker:
):
compressed_region_owners.setdefault(cache.data_ptr(), cache)
track_region_layers = (
self._supports_pp_hma and self._is_hma_required and not self._has_mamba
)
region_layers: list[list[str]] = []
# K and V are packed into the content dim, so each attention layer is a
# single NIXL region whose block transfers as one unit. Mamba layers instead
# register separate conv/ssm sub-regions (see `_build_mamba_local`).
@@ -1508,6 +1643,12 @@ class NixlBaseConnectorWorker:
self._region_is_mla.append(is_mla_region)
region_mem_types.append(mem_type)
if track_region_layers:
if region_index == len(region_layers):
region_layers.append([layer_name])
else:
region_layers[region_index].append(layer_name)
if isinstance(layer_spec, MambaSpec):
if layer_spec.tp_replicated:
assert self._is_csa_linear
@@ -1563,7 +1704,9 @@ class NixlBaseConnectorWorker:
if self._has_mamba:
self.region_num_blocks = [self.num_blocks] * self.num_regions
if self.pp_size > 1:
self._set_region_layers(region_layers)
if self.pp_size > 1 and not self._is_hma_required:
start_layer, end_layer = self.model_config.get_layers_start_end_indices(
self.vllm_config.parallel_config
)
@@ -1573,7 +1716,12 @@ class NixlBaseConnectorWorker:
self._remote_region_offset = regions_per_layer * start_layer
# Total local FA descriptors (boundary between FA and mamba descs).
self.num_descs = sum(self.region_num_blocks)
xfer_region_num_blocks = (
[self.region_num_blocks[i] for i in self._transfer_layer_region_indices]
if self._transfer_layer_region_indices
else self.region_num_blocks
)
self.num_descs = sum(xfer_region_num_blocks)
self._mixed_mem_types = len(set(region_mem_types)) > 1
if self._mixed_mem_types:
@@ -1597,7 +1745,7 @@ class NixlBaseConnectorWorker:
self.device_kv_caches = kv_caches
self.dst_num_blocks[self.engine_id] = self.num_blocks
self.dst_region_num_blocks[self.engine_id] = self.region_num_blocks
self.dst_region_num_blocks[self.engine_id] = xfer_region_num_blocks
self.dst_region_group_ids[self.engine_id] = self.region_group_ids
self.dst_uses_region_group_mapping[self.engine_id] = (
self._uses_region_group_mapping
@@ -1647,6 +1795,7 @@ class NixlBaseConnectorWorker:
region_mem_types=self.region_mem_types,
dcp_size=self.dcp_size,
pcp_size=self.pcp_size,
region_members=self.region_members,
)
# Wrap metadata in payload with hash for defensive decoding
assert self.compat_hash is not None
@@ -1805,12 +1954,16 @@ class NixlBaseConnectorWorker:
base_addresses: list[int],
block_size_ratio: int,
) -> np.ndarray:
"""Build local FA descriptors for all layers as an Nx3 uint64 array."""
"""Build local FA descriptors as an Nx3 uint64 array."""
assert self.transfer_topo is not None
assert base_addresses, "Local KV cache base addresses must not be empty."
device_id = self.device_id
region_indices = self._transfer_layer_region_indices or range(
len(base_addresses)
)
parts: list[np.ndarray] = []
for i, base_addr in enumerate(base_addresses):
for i in region_indices:
base_addr = base_addresses[i]
block_len = self.block_len_per_layer[i] // block_size_ratio
logical_blocks = np.repeat(
np.arange(self.region_num_blocks[i], dtype=np.uint64),
@@ -1834,7 +1987,7 @@ class NixlBaseConnectorWorker:
nixl_agent_meta: NixlAgentMetadata,
block_size_ratio: int,
) -> np.ndarray:
"""Build remote FA descriptors for all layers as an Nx3 uint64 array."""
"""Build remote FA descriptors as an Nx3 uint64 array."""
assert self.transfer_topo is not None
assert nixl_agent_meta.kv_caches_base_addr, (
"Remote KV cache base addresses must not be empty."
@@ -1851,10 +2004,14 @@ class NixlBaseConnectorWorker:
device_id = nixl_agent_meta.device_id
block_strides = nixl_agent_meta.block_strides
parts: list[np.ndarray] = []
use_layer_name_routing = bool(self._transfer_layer_region_indices)
for i, base_addr in enumerate(nixl_agent_meta.kv_caches_base_addr):
replicated = self._is_region_replicated(i)
local_region = (
self._transfer_layer_region_indices[i] if use_layer_name_routing else i
)
replicated = self._is_region_replicated(local_region)
# Read our whole local region size from remote..
local_block_len = self.block_len_per_layer[i]
local_block_len = self.block_len_per_layer[local_region]
remote_kv_block_len = local_block_len // block_size_ratio
if block_size_ratio > 1:
# ..using remote kv_block_len as transfer unit
@@ -2021,9 +2178,22 @@ class NixlBaseConnectorWorker:
)
return self._remote_agents[engine_id][(0, remote_tp_rank)]
assert self.transfer_topo is not None
transfer_topo = self.transfer_topo
# Number of physical regions registered locally (one per layer/tensor).
num_local_regions = len(self.block_len_per_layer)
if (
if self._transfer_layer_region_indices:
if self.block_size != nixl_agent_meta.block_size:
raise NotImplementedError(
"Attention-HMA push requires identical P/D block sizes."
)
if remote_tp_size > transfer_topo.tp_size and not self.use_mla:
raise NotImplementedError(
"Attention-HMA push does not support decode TP greater than "
"prefill TP yet"
)
self._align_remote_regions_by_layer(nixl_agent_meta)
elif (
self.pp_size > 1
and len(nixl_agent_meta.kv_caches_base_addr) > num_local_regions
):
@@ -2038,6 +2208,10 @@ class NixlBaseConnectorWorker:
]
nixl_agent_meta.block_lens = nixl_agent_meta.block_lens[start:end]
nixl_agent_meta.block_strides = nixl_agent_meta.block_strides[start:end]
if nixl_agent_meta.region_num_blocks is not None:
nixl_agent_meta.region_num_blocks = nixl_agent_meta.region_num_blocks[
start:end
]
if nixl_agent_meta.region_group_ids is not None:
nixl_agent_meta.region_group_ids = nixl_agent_meta.region_group_ids[
start:end
@@ -2048,14 +2222,8 @@ class NixlBaseConnectorWorker:
nixl_agent_meta.region_mem_types = nixl_agent_meta.region_mem_types[
start:end
]
if nixl_agent_meta.region_num_blocks is not None:
nixl_agent_meta.region_num_blocks = nixl_agent_meta.region_num_blocks[
start:end
]
### Register remote engine in TransferTopology (idempotent).
assert self.transfer_topo is not None
transfer_topo = self.transfer_topo
physical_blocks_per_logical = (
nixl_agent_meta.physical_blocks_per_logical_kv_block
)
@@ -2349,6 +2517,9 @@ class NixlBaseConnectorWorker:
# the per-rank KV head ratio rather than the raw tp_ratio, because GQA
# replication caps per-rank heads at 1 when tp > total_kv_heads
# (issue #45330). Mamba uses the ssm_sizes counterpart, so skip here.
local_regions = self._transfer_layer_region_indices or range(
len(self.block_len_per_layer)
)
if self._has_mamba and self.use_mla:
# Hybrid MLA+SSM (e.g. KimiLinear's KDA+MLA): regions are
# kernel-granularity views of the mamba-unified page. The MLA
@@ -2365,7 +2536,7 @@ class NixlBaseConnectorWorker:
f"remote={nixl_agent_meta.block_lens}."
)
elif not self._has_mamba:
assert len(self.block_len_per_layer) == len(nixl_agent_meta.block_lens), (
assert len(local_regions) == len(nixl_agent_meta.block_lens), (
"Number of KV layers must match between prefill and decode"
)
model_replicated = self.use_mla or self.transfer_topo.is_kv_replicated(
@@ -2374,8 +2545,9 @@ class NixlBaseConnectorWorker:
total_kv_heads = self.transfer_topo.total_num_kv_heads
local_heads = self.transfer_topo.local_physical_heads
remote_heads = max(1, total_kv_heads // remote_tp_size)
for i, local_len in enumerate(self.block_len_per_layer):
replicated = model_replicated or self._is_region_replicated(i)
for i, region_idx in enumerate(local_regions):
local_len = self.block_len_per_layer[region_idx]
replicated = model_replicated or self._is_region_replicated(region_idx)
remote_len = nixl_agent_meta.block_lens[i]
if replicated:
assert local_len // block_size_ratio == remote_len, (
@@ -2406,8 +2578,7 @@ class NixlBaseConnectorWorker:
# TP workers that handhshake with same remote have same #blocks.
assert self.dst_num_blocks[remote_engine_id] == nixl_agent_meta.num_blocks
# Same number of regions/~layers.
assert len(nixl_agent_meta.kv_caches_base_addr) == len(self.block_len_per_layer)
assert len(nixl_agent_meta.kv_caches_base_addr) == len(local_regions)
num_remote_regions = len(nixl_agent_meta.kv_caches_base_addr)
if nixl_agent_meta.region_num_blocks is not None:
assert len(nixl_agent_meta.region_num_blocks) == num_remote_regions
@@ -2423,7 +2594,12 @@ class NixlBaseConnectorWorker:
if nixl_agent_meta.region_group_ids is not None:
assert len(nixl_agent_meta.region_group_ids) == num_remote_regions
if nixl_agent_meta.region_names is not None:
assert nixl_agent_meta.region_names == self.region_names, (
expected_names = (
list(self._transfer_layer_names)
if self._transfer_layer_names
else self.region_names
)
assert nixl_agent_meta.region_names == expected_names, (
"NIXL transfer regions must name the same model caches in the "
"same order"
)
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Metadata dataclasses and helpers for the NIXL connector."""
from dataclasses import dataclass
from dataclasses import dataclass, field
from typing import Any
from vllm.config import VllmConfig
@@ -47,8 +47,9 @@ PUSH_REG_NOTIF_PREFIX = b"PUSH_REG:"
# 9: Add block_strides
# 10: Add dense virtual transfer pages for compressed MLA caches
# 11: Add per-region transfer geometry and memory types to NixlAgentMetadata
# 12: Add per-region member names for PP push
#
NIXL_CONNECTOR_VERSION: int = 11
NIXL_CONNECTOR_VERSION: int = 12
@dataclass
@@ -71,6 +72,8 @@ class NixlAgentMetadata:
region_mem_types: list[str] | None = None
dcp_size: int = 1
pcp_size: int = 1
# Layer names sharing each advertised region, in region order.
region_members: list[list[str]] = field(default_factory=list)
@dataclass
@@ -83,6 +83,8 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
# Distinguishes push from pull in the NIXL compatibility hash.
_TRANSFER_MODE: str = "push"
_supports_pp_hma = True
def __init__(
self,
vllm_config: "VllmConfig",
@@ -530,7 +532,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
local_region_groups = self.region_group_ids
remote_region_groups = self.dst_region_group_ids[engine_id]
groups_differ = local_region_groups != remote_region_groups
if groups_differ:
if groups_differ and not self._transfer_layer_group_ids:
raise NotImplementedError(
"NixlPushConnector does not support different producer and "
"consumer cache-group layouts"