[KV Connector] Support NIXL P/D for hybrid MLA+SSM models (#49762)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Jared Wen <w13431838023@gmail.com>
This commit is contained in:
Nick Hill
2026-07-29 11:50:30 +08:00
committed by GitHub
co-authored by Jared Wen
parent 32a423ac0a
commit f37f03db4a
5 changed files with 281 additions and 38 deletions
@@ -1093,7 +1093,7 @@ class NixlBaseConnectorWorker:
caches_data = []
# With hybrid allocator, layers can share a kv cache tensor
seen_base_addresses = []
seen_base_addresses: list[int] = []
# K and V are packed into the content dim, so each attention layer is a
# single NIXL region whose block transfers as one unit. Mamba layers instead
@@ -1139,11 +1139,19 @@ class NixlBaseConnectorWorker:
curr_tensor_size_bytes = num_blocks * physical_page_size
base_addr = cache.data_ptr()
is_mla_region = isinstance(
layer_spec, (MLAAttentionSpec, SlidingWindowMLASpec)
)
if base_addr in seen_base_addresses:
# NOTE (NickLucche) HMA employs memory pooling to share tensors
# across groups. This results in skipping all tensors but the ones
# pointed to by group0. Also, generally we will have more blocks
# per tensor but fewer regions.
# A shared tensor may back both SSM and attention layers (e.g.
# KDA+MLA in KimiLinear); the region's FA view is MLA whichever
# layer registered it first.
idx = seen_base_addresses.index(base_addr)
self._region_is_mla[idx] |= is_mla_region
logger.debug("Skipping %s because it's already seen", layer_name)
continue
logger.debug(
@@ -1157,9 +1165,6 @@ class NixlBaseConnectorWorker:
)
else:
self.block_len_per_layer.append(physical_page_size)
is_mla_region = isinstance(
layer_spec, (MLAAttentionSpec, SlidingWindowMLASpec)
)
self._region_is_mla.append(is_mla_region)
if not is_mla_region:
@@ -1801,7 +1806,22 @@ class NixlBaseConnectorWorker:
# the per-rank KV head ratio rather than the raw tp_ratio, because GQA
# replication caps per-rank heads at 1 when tp > total_kv_heads
# (issue #45330). Mamba uses the ssm_sizes counterpart, so skip here.
if not self._has_mamba:
if self._has_mamba and self.use_mla:
# Hybrid MLA+SSM (e.g. KimiLinear's KDA+MLA): regions are
# kernel-granularity views of the mamba-unified page. The MLA
# per-token page is TP-independent, so the block lengths must
# match up to the kernel block size ratio even under
# heterogeneous TP (remote kernel blocks may be smaller).
# SSM geometry is validated via ssm_sizes/conv offsets instead.
assert self.block_len_per_layer == [
block_len * block_size_ratio for block_len in nixl_agent_meta.block_lens
], (
"Hybrid MLA kernel-granularity block lengths must match "
f"between P and D (block_size_ratio={block_size_ratio}): "
f"local={self.block_len_per_layer}, "
f"remote={nixl_agent_meta.block_lens}."
)
elif not self._has_mamba:
assert len(self.block_len_per_layer) == len(nixl_agent_meta.block_lens), (
"Number of KV layers must match between prefill and decode"
)
@@ -52,7 +52,10 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
ReqMeta,
TransferHandle,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import ReadSpec
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
ReadSpec,
_is_attention_spec,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import get_base_request_id
from vllm.logger import init_logger
@@ -505,41 +508,37 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
local_block_ids = meta.local_physical_block_ids
num_groups = len(local_block_ids)
if self.use_mla and tp_ratio < 0:
# MLA latent is replicated across D's TP ranks: the tp-mapping
# collapses to one rank (fine for reads), but push must WRITE every
# D rank or the rest decode stale KV; only the dst differs per rank.
# MLA latent is replicated across D's TP ranks: the tp-mapping
# collapses it to one rank (fine for reads), but push must WRITE every
# D rank or the rest decode stale KV. For hybrid MLA+SSM the sharded
# SSM state already targets every covered D rank, so only the
# attention groups need widening; pure MLA writes to all handshaked
# ranks (only the dst differs per rank).
replicate_attn = self.use_mla and tp_ratio < 0
if replicate_attn and not self._has_mamba:
assert len(plan.all_source_ranks) == 1
mla_local_ids = [list(ids) for ids in local_block_ids]
mla_remote_ids = [list(ids) for ids in remote_block_ids]
read_specs = [
ReadSpec(
remote_rank=rank,
local_block_ids=mla_local_ids,
remote_block_ids=mla_remote_ids,
)
for rank in self.dst_xfer_side_handles[engine_id]
]
write_ranks = sorted(self.dst_xfer_side_handles[engine_id])
else:
read_specs = [
ReadSpec(
remote_rank=rank,
local_block_ids=[
list(local_block_ids[g])
if rank in plan.source_ranks_per_group[g]
else []
for g in range(num_groups)
],
remote_block_ids=[
list(remote_block_ids[g])
if rank in plan.source_ranks_per_group[g]
else []
for g in range(num_groups)
],
)
for rank in plan.all_source_ranks
write_ranks = list(plan.all_source_ranks)
def group_ids(block_ids: BlockIds, rank: int) -> BlockIds:
return [
list(block_ids[g])
if (replicate_attn and _is_attention_spec(self._group_spec_types[g]))
or rank in plan.source_ranks_per_group[g]
else []
for g in range(num_groups)
]
read_specs = [
ReadSpec(
remote_rank=rank,
local_block_ids=group_ids(local_block_ids, rank),
remote_block_ids=group_ids(remote_block_ids, rank),
)
for rank in write_ranks
]
handles: list[int] = []
for i, spec in enumerate(read_specs):
remote_block_size = remote_info.remote_block_size
@@ -551,7 +550,10 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
remote_block_size,
req_id,
)
if tp_ratio < 0 and not self.use_mla:
if tp_ratio < 0 and (not self.use_mla or len(plan.all_source_ranks) > 1):
# Multiple targets: write each rank its chunk of local memory.
# Hybrid MLA+SSM also lands here: its split handles replicate
# the attention descriptors and chunk only the SSM state.
split_key = (tp_ratio, remote_block_size)
local_xfer_side_handle = self.src_xfer_handles_by_tp_ratio[split_key][i]
else: