[KVConnector][NIXL] Support packed MLA KV layouts in pipeline-parallel push prefill (#50499)

Signed-off-by: zixi-qi <zixi@inferact.ai>
This commit is contained in:
qizixi
2026-09-28 18:50:46 +02:00
committed by GitHub
parent 94d1462924
commit 6d3ea3c2c9
6 changed files with 357 additions and 28 deletions
@@ -165,8 +165,8 @@ class NixlBaseConnectorWorker:
# Overridden by NixlPushConnectorWorker.
_TRANSFER_MODE: str = "pull"
# Layer-name routing is supported only by NixlPushConnector for HMA KV
# caches under PP.
# Layer-name routing is supported only by NixlPushConnector for HMA or
# packed KV caches under PP.
_supports_pp_hma = False
def _compute_desc_ids(
@@ -432,15 +432,22 @@ class NixlBaseConnectorWorker:
group_by_layer[name] for name in layer_names
)
def _requires_layer_name_routing(self) -> bool:
"""Whether PP push must match HMA layers by name, not region index."""
def _tracks_region_layers(self) -> bool:
"""Whether regions advertise their member layer names.
Push workers do so for HMA and packed layouts, so a PP producer can
match layers by name instead of by region index.
"""
return (
self._supports_pp_hma
and self.pp_size > 1
and self._is_hma_required
and (self._is_hma_required or self._has_packed_cache)
and not self._has_mamba
)
def _requires_layer_name_routing(self) -> bool:
"""Whether PP push must match HMA/packed layers by name, not region index."""
return self.pp_size > 1 and self._tracks_region_layers()
def _align_remote_regions_by_layer(
self, nixl_agent_meta: NixlAgentMetadata
) -> None:
@@ -479,6 +486,10 @@ class NixlBaseConnectorWorker:
]
assert not missing, f"Remote is missing locally owned layers: {missing}"
packed_regions = {
remote_region_by_layer[name]
for name in nixl_agent_meta.packed_member_layouts
}
remote_regions = [
remote_region_by_layer[name] for name in self._transfer_layer_names
]
@@ -505,6 +516,24 @@ class NixlBaseConnectorWorker:
]
if nixl_agent_meta.region_names is not None:
nixl_agent_meta.region_names = list(self._transfer_layer_names)
if nixl_agent_meta.packed_member_layouts:
for i, layer_name in enumerate(self._transfer_layer_names):
if remote_regions[i] not in packed_regions:
continue
assert layer_name in nixl_agent_meta.packed_member_layouts, (
f"Remote packed layer {layer_name!r} has no layout"
)
offset, page_size = nixl_agent_meta.packed_member_layouts[layer_name]
assert (
0 <= offset < offset + page_size <= nixl_agent_meta.block_lens[i]
), f"Remote packed layer {layer_name!r} escapes its block"
local_region = self._transfer_layer_region_indices[i]
assert page_size == self.block_len_per_layer[local_region], (
f"Packed MLA page sizes must match for layer {layer_name!r}"
)
nixl_agent_meta.kv_caches_base_addr[i] += offset
nixl_agent_meta.block_lens[i] = page_size
nixl_agent_meta.packed_member_layouts = {}
# One layer per region keeps a second alignment pass a no-op.
nixl_agent_meta.region_members = [[name] for name in self._transfer_layer_names]
@@ -899,7 +928,10 @@ class NixlBaseConnectorWorker:
# within a block (BLHNC/BHLNC), where stride > block_len.
self.block_stride_per_layer = list[int]()
# Region layer membership is populated for HMA layouts.
# Block-outermost layouts (BLHNC/BHLNC) pack every layer's page into one
# block row of a shared allocation.
self._has_packed_cache = KVCacheLayout[self.kv_cache_layout].is_block_outermost
# Region layer membership is populated for HMA and packed layouts.
self.region_members = []
# Local layer order, set by ``_set_region_layers`` only when
# routing by layer name. Empty means region-index routing.
@@ -1352,6 +1384,8 @@ class NixlBaseConnectorWorker:
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
"""Register the KV Cache data in nixl."""
use_layer_name_routing = self._requires_layer_name_routing()
route_packed_layers = self._has_packed_cache and use_layer_name_routing
self.transfer_topo = TransferTopology(
tp_rank=self.transfer_tp_rank,
tp_size=self.transfer_tp_size,
@@ -1369,9 +1403,15 @@ class NixlBaseConnectorWorker:
else None,
is_mamba=self._has_mamba,
)
backend_name = self.backend_name
if self._supports_pp_hma and self._has_packed_cache:
# PP can reorder the same backends across stages of a packed model.
backend_name = ",".join(
sorted(backend.get_name() for backend in self.attn_backends)
)
self.compat_hash = compute_nixl_compatibility_hash(
self.vllm_config,
self.backend_name,
backend_name,
transfer_mode=self._TRANSFER_MODE,
)
@@ -1413,6 +1453,11 @@ class NixlBaseConnectorWorker:
# CSA-linear needs separate logical regions for attention and state
# aliases even though every view shares one block-major allocation.
packed_storage = packed_storage and not self._is_csa_linear
if route_packed_layers and any(
not isinstance(spec, (MLAAttentionSpec, SlidingWindowMLASpec))
for spec in self._layer_specs.values()
):
raise NotImplementedError("PP push with packed KV caches requires MLA")
layer_specs: dict[str, KVCacheSpec] = {}
compressed_region_owners: dict[int, torch.Tensor] = {}
@@ -1439,10 +1484,9 @@ class NixlBaseConnectorWorker:
):
compressed_region_owners.setdefault(cache.data_ptr(), cache)
track_region_layers = (
self._supports_pp_hma and self._is_hma_required and not self._has_mamba
)
track_region_layers = self._tracks_region_layers()
region_layers: list[list[str]] = []
packed_member_layouts: dict[str, tuple[int, int]] = {}
# K and V are packed into the content dim, so each attention layer is a
# single NIXL region whose block transfers as one unit. Mamba layers instead
@@ -1587,11 +1631,34 @@ class NixlBaseConnectorWorker:
virtual_transfer_pages = _uses_dense_virtual_transfer_pages(
layer_spec, cache, physical_page_size, num_blocks
)
# Push workers address the pages of packed MLA rows by layer.
packed_mla_push = (
track_region_layers
and self._has_packed_cache
and packed_storage
and storage_is_block_major
and is_mla_region
)
if virtual_transfer_pages:
# A compressed kernel row can contain multiple NIXL transfer pages.
region_specs = [
(cache.data_ptr(), physical_page_size, physical_page_size)
]
elif packed_mla_push and use_layer_name_routing:
# PP>1 prefill stage: register only this layer's page of each row.
region_specs = [
(cache.data_ptr(), physical_page_size, block_stride)
]
elif packed_mla_push:
# PP=1 decode or prefill: register whole rows, and advertise where
# this layer's page sits so a PP>1 prefill can write into it.
storage_block_len = registration_len // num_blocks
region_specs = [
(registration_base, storage_block_len, storage_block_len)
]
offset = cache.data_ptr() - storage_addr
assert offset >= 0 and offset + physical_page_size <= block_stride
packed_member_layouts[layer_name] = (offset, physical_page_size)
elif storage_is_block_major and (
not page_contiguous
and ((packed_storage and is_mla_region) or not self._is_csa_linear)
@@ -1633,7 +1700,9 @@ class NixlBaseConnectorWorker:
]
for base_addr, block_len, block_stride in region_specs:
if base_addr in seen_base_addresses:
# A packed PP producer keeps one region per layer, so aliases at
# one address (a layer and its SWA view) stay distinct.
if base_addr in seen_base_addresses and not route_packed_layers:
region_index = seen_base_addresses.index(base_addr)
assert region_mem_types[region_index] == mem_type
self._region_is_mla[region_index] |= is_mla_region
@@ -1717,7 +1786,7 @@ class NixlBaseConnectorWorker:
self._set_region_layers(region_layers)
if self.pp_size > 1 and not self._is_hma_required:
if self.pp_size > 1 and not use_layer_name_routing:
start_layer, end_layer = self.model_config.get_layers_start_end_indices(
self.vllm_config.parallel_config
)
@@ -1807,6 +1876,7 @@ class NixlBaseConnectorWorker:
dcp_size=self.dcp_size,
pcp_size=self.pcp_size,
region_members=self.region_members,
packed_member_layouts=packed_member_layouts,
)
# Wrap metadata in payload with hash for defensive decoding
assert self.compat_hash is not None
@@ -48,8 +48,9 @@ PUSH_REG_NOTIF_PREFIX = b"PUSH_REG:"
# 10: Add dense virtual transfer pages for compressed MLA caches
# 11: Add per-region transfer geometry and memory types to NixlAgentMetadata
# 12: Add per-region member names for PP push
# 13: Add packed-member layouts and order-independent packed-push backend hashes
#
NIXL_CONNECTOR_VERSION: int = 12
NIXL_CONNECTOR_VERSION: int = 13
@dataclass
@@ -74,6 +75,8 @@ class NixlAgentMetadata:
pcp_size: int = 1
# Layer names sharing each advertised region, in region order.
region_members: list[list[str]] = field(default_factory=list)
# Packed member -> (byte offset in its region's block, bytes per page).
packed_member_layouts: dict[str, tuple[int, int]] = field(default_factory=dict)
@dataclass