[KV Connector] Support NIXL P/D for hybrid MLA+SSM models (#49762)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Jared Wen <w13431838023@gmail.com>
This commit is contained in:
@@ -1093,7 +1093,7 @@ class NixlBaseConnectorWorker:
|
||||
|
||||
caches_data = []
|
||||
# With hybrid allocator, layers can share a kv cache tensor
|
||||
seen_base_addresses = []
|
||||
seen_base_addresses: list[int] = []
|
||||
|
||||
# K and V are packed into the content dim, so each attention layer is a
|
||||
# single NIXL region whose block transfers as one unit. Mamba layers instead
|
||||
@@ -1139,11 +1139,19 @@ class NixlBaseConnectorWorker:
|
||||
curr_tensor_size_bytes = num_blocks * physical_page_size
|
||||
|
||||
base_addr = cache.data_ptr()
|
||||
is_mla_region = isinstance(
|
||||
layer_spec, (MLAAttentionSpec, SlidingWindowMLASpec)
|
||||
)
|
||||
if base_addr in seen_base_addresses:
|
||||
# NOTE (NickLucche) HMA employs memory pooling to share tensors
|
||||
# across groups. This results in skipping all tensors but the ones
|
||||
# pointed to by group0. Also, generally we will have more blocks
|
||||
# per tensor but fewer regions.
|
||||
# A shared tensor may back both SSM and attention layers (e.g.
|
||||
# KDA+MLA in KimiLinear); the region's FA view is MLA whichever
|
||||
# layer registered it first.
|
||||
idx = seen_base_addresses.index(base_addr)
|
||||
self._region_is_mla[idx] |= is_mla_region
|
||||
logger.debug("Skipping %s because it's already seen", layer_name)
|
||||
continue
|
||||
logger.debug(
|
||||
@@ -1157,9 +1165,6 @@ class NixlBaseConnectorWorker:
|
||||
)
|
||||
else:
|
||||
self.block_len_per_layer.append(physical_page_size)
|
||||
is_mla_region = isinstance(
|
||||
layer_spec, (MLAAttentionSpec, SlidingWindowMLASpec)
|
||||
)
|
||||
self._region_is_mla.append(is_mla_region)
|
||||
|
||||
if not is_mla_region:
|
||||
@@ -1801,7 +1806,22 @@ class NixlBaseConnectorWorker:
|
||||
# the per-rank KV head ratio rather than the raw tp_ratio, because GQA
|
||||
# replication caps per-rank heads at 1 when tp > total_kv_heads
|
||||
# (issue #45330). Mamba uses the ssm_sizes counterpart, so skip here.
|
||||
if not self._has_mamba:
|
||||
if self._has_mamba and self.use_mla:
|
||||
# Hybrid MLA+SSM (e.g. KimiLinear's KDA+MLA): regions are
|
||||
# kernel-granularity views of the mamba-unified page. The MLA
|
||||
# per-token page is TP-independent, so the block lengths must
|
||||
# match up to the kernel block size ratio even under
|
||||
# heterogeneous TP (remote kernel blocks may be smaller).
|
||||
# SSM geometry is validated via ssm_sizes/conv offsets instead.
|
||||
assert self.block_len_per_layer == [
|
||||
block_len * block_size_ratio for block_len in nixl_agent_meta.block_lens
|
||||
], (
|
||||
"Hybrid MLA kernel-granularity block lengths must match "
|
||||
f"between P and D (block_size_ratio={block_size_ratio}): "
|
||||
f"local={self.block_len_per_layer}, "
|
||||
f"remote={nixl_agent_meta.block_lens}."
|
||||
)
|
||||
elif not self._has_mamba:
|
||||
assert len(self.block_len_per_layer) == len(nixl_agent_meta.block_lens), (
|
||||
"Number of KV layers must match between prefill and decode"
|
||||
)
|
||||
|
||||
@@ -52,7 +52,10 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
ReqMeta,
|
||||
TransferHandle,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import ReadSpec
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
|
||||
ReadSpec,
|
||||
_is_attention_spec,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import get_base_request_id
|
||||
from vllm.logger import init_logger
|
||||
|
||||
@@ -505,41 +508,37 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
local_block_ids = meta.local_physical_block_ids
|
||||
num_groups = len(local_block_ids)
|
||||
|
||||
if self.use_mla and tp_ratio < 0:
|
||||
# MLA latent is replicated across D's TP ranks: the tp-mapping
|
||||
# collapses to one rank (fine for reads), but push must WRITE every
|
||||
# D rank or the rest decode stale KV; only the dst differs per rank.
|
||||
# MLA latent is replicated across D's TP ranks: the tp-mapping
|
||||
# collapses it to one rank (fine for reads), but push must WRITE every
|
||||
# D rank or the rest decode stale KV. For hybrid MLA+SSM the sharded
|
||||
# SSM state already targets every covered D rank, so only the
|
||||
# attention groups need widening; pure MLA writes to all handshaked
|
||||
# ranks (only the dst differs per rank).
|
||||
replicate_attn = self.use_mla and tp_ratio < 0
|
||||
if replicate_attn and not self._has_mamba:
|
||||
assert len(plan.all_source_ranks) == 1
|
||||
mla_local_ids = [list(ids) for ids in local_block_ids]
|
||||
mla_remote_ids = [list(ids) for ids in remote_block_ids]
|
||||
read_specs = [
|
||||
ReadSpec(
|
||||
remote_rank=rank,
|
||||
local_block_ids=mla_local_ids,
|
||||
remote_block_ids=mla_remote_ids,
|
||||
)
|
||||
for rank in self.dst_xfer_side_handles[engine_id]
|
||||
]
|
||||
write_ranks = sorted(self.dst_xfer_side_handles[engine_id])
|
||||
else:
|
||||
read_specs = [
|
||||
ReadSpec(
|
||||
remote_rank=rank,
|
||||
local_block_ids=[
|
||||
list(local_block_ids[g])
|
||||
if rank in plan.source_ranks_per_group[g]
|
||||
else []
|
||||
for g in range(num_groups)
|
||||
],
|
||||
remote_block_ids=[
|
||||
list(remote_block_ids[g])
|
||||
if rank in plan.source_ranks_per_group[g]
|
||||
else []
|
||||
for g in range(num_groups)
|
||||
],
|
||||
)
|
||||
for rank in plan.all_source_ranks
|
||||
write_ranks = list(plan.all_source_ranks)
|
||||
|
||||
def group_ids(block_ids: BlockIds, rank: int) -> BlockIds:
|
||||
return [
|
||||
list(block_ids[g])
|
||||
if (replicate_attn and _is_attention_spec(self._group_spec_types[g]))
|
||||
or rank in plan.source_ranks_per_group[g]
|
||||
else []
|
||||
for g in range(num_groups)
|
||||
]
|
||||
|
||||
read_specs = [
|
||||
ReadSpec(
|
||||
remote_rank=rank,
|
||||
local_block_ids=group_ids(local_block_ids, rank),
|
||||
remote_block_ids=group_ids(remote_block_ids, rank),
|
||||
)
|
||||
for rank in write_ranks
|
||||
]
|
||||
|
||||
handles: list[int] = []
|
||||
for i, spec in enumerate(read_specs):
|
||||
remote_block_size = remote_info.remote_block_size
|
||||
@@ -551,7 +550,10 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
remote_block_size,
|
||||
req_id,
|
||||
)
|
||||
if tp_ratio < 0 and not self.use_mla:
|
||||
if tp_ratio < 0 and (not self.use_mla or len(plan.all_source_ranks) > 1):
|
||||
# Multiple targets: write each rank its chunk of local memory.
|
||||
# Hybrid MLA+SSM also lands here: its split handles replicate
|
||||
# the attention descriptors and chunk only the SSM state.
|
||||
split_key = (tp_ratio, remote_block_size)
|
||||
local_xfer_side_handle = self.src_xfer_handles_by_tp_ratio[split_key][i]
|
||||
else:
|
||||
|
||||
Reference in New Issue
Block a user