[KV Connector] Support NIXL P/D for hybrid MLA+SSM models (#49762)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Jared Wen <w13431838023@gmail.com>
This commit is contained in:
@@ -1380,3 +1380,199 @@ def test_logical_to_kernel_block_ids_with_remote_ratio(
|
||||
assert list(result) == expected_kernel_block_ids, (
|
||||
f"Expected {expected_kernel_block_ids}, got {result}"
|
||||
)
|
||||
|
||||
|
||||
# ── Hybrid MLA+SSM (KimiLinear-shaped KDA+MLA) tests ─────────────────────
|
||||
|
||||
|
||||
def _make_hybrid_mla_kv_cache_config(num_blocks: int = 4):
|
||||
"""KimiLinear-shaped config: one MLA group and two KDA (GDN-typed
|
||||
MambaSpec) groups whose layers share the same HMA tensors, with a
|
||||
mamba-aligned unified page and an MLA kernel block smaller than the
|
||||
logical block."""
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
MambaSpec,
|
||||
MLAAttentionSpec,
|
||||
)
|
||||
|
||||
# 12-token logical blocks over a 4-token MLA kernel block.
|
||||
mla_spec = MLAAttentionSpec(
|
||||
block_size=12, num_kv_heads=1, head_size=6, dtype=torch.float16
|
||||
)
|
||||
unified_page = mla_spec.page_size_bytes
|
||||
kda_spec = MambaSpec(
|
||||
block_size=12,
|
||||
# GDN-decomposable conv (Q|K|V = 2|2|4 cols x 3 rows) + fp32 temporal.
|
||||
shapes=((8, 3), (1, 4, 4)),
|
||||
dtypes=(torch.float16, torch.float32),
|
||||
page_size_padded=unified_page,
|
||||
mamba_type=MambaAttentionBackendEnum.GDN_ATTN,
|
||||
)
|
||||
assert kda_spec.page_size_bytes == unified_page
|
||||
return KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=num_blocks * unified_page,
|
||||
shared_by=[f"mla.{i}", f"kda_a.{i}", f"kda_b.{i}"],
|
||||
)
|
||||
for i in range(2)
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["mla.0", "mla.1"], mla_spec),
|
||||
KVCacheGroupSpec(["kda_a.0", "kda_a.1"], kda_spec),
|
||||
KVCacheGroupSpec(["kda_b.0", "kda_b.1"], kda_spec),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_register_kv_caches_hybrid_mla_dual_purpose_regions():
|
||||
"""Hybrid MLA+KDA registration: HMA tensors shared by both layer types
|
||||
must be flagged as MLA regions even when a KDA layer registers them
|
||||
first, expose TP-independent kernel-granularity block lens, and build
|
||||
FA + mamba descriptors for every region."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker as bw
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
|
||||
NixlConnectorWorker,
|
||||
)
|
||||
|
||||
kv_cache_config = _make_hybrid_mla_kv_cache_config()
|
||||
unified_page = kv_cache_config.kv_cache_groups[0].kv_cache_spec.page_size_bytes
|
||||
vllm_config = create_vllm_config(block_size=12)
|
||||
# kv_buffer_device defaults to the *real* platform's device type, which on
|
||||
# a CPU-only test host would make this a host-buffer worker: host xfer
|
||||
# buffers are per-layer, so the HMA shared tensors would not be
|
||||
# deduplicated. Pin it to the faked device type.
|
||||
vllm_config.kv_transfer_config.kv_buffer_device = "cuda"
|
||||
|
||||
fake_backend = MagicMock()
|
||||
fake_backend.get_supported_kernel_block_sizes.return_value = [4]
|
||||
fake_backend.get_name.return_value = "FLASHMLA"
|
||||
fake_backend.full_cls_name.return_value = "fake.FLASHMLA"
|
||||
fake_platform = MagicMock()
|
||||
fake_platform.device_type = "cuda"
|
||||
fake_platform.get_nixl_memory_type.return_value = "VRAM"
|
||||
|
||||
with (
|
||||
patch.object(bw, "NixlWrapper"),
|
||||
patch.object(bw, "get_tensor_model_parallel_rank", return_value=0),
|
||||
patch.object(bw, "get_tensor_model_parallel_world_size", return_value=1),
|
||||
patch.object(bw, "get_current_attn_backends", return_value=[fake_backend]),
|
||||
patch.object(bw, "current_platform", fake_platform),
|
||||
patch(
|
||||
"vllm.model_executor.layers.mamba.mamba_utils.get_conv_state_layout",
|
||||
return_value="DS",
|
||||
),
|
||||
set_current_vllm_config(vllm_config),
|
||||
):
|
||||
worker = NixlConnectorWorker(vllm_config, "test-engine", kv_cache_config)
|
||||
worker.use_mla = True # opt-125m test config is not MLA; force the flag
|
||||
worker.nixl_wrapper.get_agent_metadata.return_value = b"fake-agent-metadata"
|
||||
|
||||
tensors = [torch.zeros(4 * unified_page, dtype=torch.uint8) for _ in range(2)]
|
||||
# KDA layer first per tensor: exercises the dual-purpose flag merge.
|
||||
worker.register_kv_caches(
|
||||
{
|
||||
"kda_a.0": tensors[0],
|
||||
"mla.0": tensors[0],
|
||||
"kda_b.0": tensors[0],
|
||||
"kda_a.1": tensors[1],
|
||||
"mla.1": tensors[1],
|
||||
"kda_b.1": tensors[1],
|
||||
}
|
||||
)
|
||||
|
||||
# 12-token logical blocks over the 4-token MLA kernel block.
|
||||
assert worker._physical_blocks_per_logical_kv_block == 3
|
||||
assert worker.block_size == 4 and worker.num_blocks == 12
|
||||
# Both shared tensors are dual-purpose: their FA view is MLA even though
|
||||
# a KDA layer registered them first.
|
||||
assert worker._region_is_mla == [True, True]
|
||||
assert worker.num_regions == 2 and worker.num_descs == 24
|
||||
# Kernel-granularity block lens; TP-independent for MLA hybrids.
|
||||
assert worker.block_len_per_layer == [unified_page // 3] * 2
|
||||
# Split handles must replicate every FA descriptor (MLA isn't head-sharded).
|
||||
assert worker._fa_desc_replicated(worker.num_descs) == [True] * 24
|
||||
# FA descs: 2 regions x 12 kernel blocks, page stride = kernel page.
|
||||
# Mamba descs: 2 regions x (3 conv sub-projections + 1 ssm) x 4 blocks.
|
||||
assert worker.src_blocks_data.shape == (24 + 32, 3)
|
||||
fa_descs = worker.src_blocks_data[:24]
|
||||
assert fa_descs[1][0] - fa_descs[0][0] == unified_page // 3
|
||||
assert all(size == unified_page // 3 for size in fa_descs[:, 1])
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_push_write_hybrid_mla_replicates_attention():
|
||||
"""Hybrid MLA+SSM push with P_TP < D_TP: attention blocks must be
|
||||
written to every covered D rank (replicated MLA latent) while SSM state
|
||||
is written per-rank through the split handles."""
|
||||
import threading
|
||||
from collections import defaultdict
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import (
|
||||
NixlPushConnectorWorker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
|
||||
TPMapping,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import MambaSpec, MLAAttentionSpec
|
||||
|
||||
worker = object.__new__(NixlPushConnectorWorker)
|
||||
worker.shutdown = lambda: None # skeleton worker: silence __del__
|
||||
worker.use_mla = True
|
||||
worker._has_mamba = True
|
||||
worker._group_spec_types = (MLAAttentionSpec, MambaSpec)
|
||||
worker.transfer_topo = MagicMock()
|
||||
worker.transfer_topo.tp_ratio.return_value = -2
|
||||
remote_info = MagicMock()
|
||||
remote_info.remote_physical_blocks_per_logical = 1
|
||||
remote_info.remote_block_size = 4
|
||||
worker.transfer_topo.get_engine_info.return_value = remote_info
|
||||
|
||||
engine_id = "remote-engine"
|
||||
# Read-oriented mapping collapses the replicated attention group to one
|
||||
# source rank; the SSM state is sharded across both covered D ranks.
|
||||
worker.tp_mappings = {
|
||||
engine_id: TPMapping(
|
||||
source_ranks_per_group=((0,), (0, 1)),
|
||||
all_source_ranks=(0, 1),
|
||||
rank_to_attention_slot={0: 0, 1: 0},
|
||||
rank_offset_factor=0,
|
||||
)
|
||||
}
|
||||
worker.dst_xfer_side_handles = {engine_id: {0: 100, 1: 101}}
|
||||
worker.src_xfer_handles_by_tp_ratio = {(-2, 4): [200, 201]}
|
||||
worker.src_xfer_handles_by_block_size = {4: 300}
|
||||
worker._sending_transfers = defaultdict(list)
|
||||
worker._sending_transfers_lock = threading.Lock()
|
||||
worker.kv_cache_config = _make_hybrid_mla_kv_cache_config()
|
||||
worker._xfer_blocks = MagicMock(return_value=1)
|
||||
|
||||
meta = MagicMock()
|
||||
meta.remote.engine_id = engine_id
|
||||
meta.remote.block_ids = [[7, 8], [3]]
|
||||
meta.local_physical_block_ids = [[1, 2], [5]]
|
||||
|
||||
worker._xfer_blocks_for_req("req-1", meta)
|
||||
|
||||
calls = worker._xfer_blocks.call_args_list
|
||||
assert len(calls) == 2
|
||||
for call, rank, local_handle, remote_handle in zip(
|
||||
calls, (0, 1), (200, 201), (100, 101)
|
||||
):
|
||||
spec = call.kwargs["read_spec"]
|
||||
assert spec.remote_rank == rank
|
||||
# Attention group replicated to every rank, SSM by membership.
|
||||
assert spec.local_block_ids == [[1, 2], [5]]
|
||||
assert spec.remote_block_ids == [[7, 8], [3]]
|
||||
assert call.kwargs["local_xfer_side_handle"] == local_handle
|
||||
assert call.kwargs["remote_xfer_side_handle"] == remote_handle
|
||||
|
||||
@@ -617,3 +617,24 @@ def test_mla_hybrid_large_ppl_geometry(num_tokens):
|
||||
num_tokens=num_tokens,
|
||||
tp_size=8,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_mismatched_mla_kernel_page_rejected_for_mla_hybrid():
|
||||
"""The MLA per-token page is TP-independent, so kernel block lengths
|
||||
differing by anything other than the block-size ratio must fail the
|
||||
handshake loudly rather than transfer at mismatched geometry."""
|
||||
worker = _make_mla_hybrid_worker(
|
||||
local_block_size=12, kernel_block_size=4, num_logical_blocks=8
|
||||
)
|
||||
meta_r = _make_remote_meta(
|
||||
worker,
|
||||
remote_block_size=8,
|
||||
remote_kernel_block_size=4,
|
||||
remote_num_logical=12,
|
||||
remote_ssm_sizes=(24, 32),
|
||||
)
|
||||
# Equal kernel block sizes (ratio 1), but a half-sized per-token page.
|
||||
meta_r.block_lens = [x // 2 for x in worker.block_len_per_layer]
|
||||
with pytest.raises((AssertionError, RuntimeError)):
|
||||
worker.add_remote_agent(meta_r, remote_tp_rank=0, remote_tp_size=2)
|
||||
|
||||
@@ -43,6 +43,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import (
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import (
|
||||
get_base_request_id,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
|
||||
from .utils import make_nixl_push_scheduler
|
||||
@@ -337,6 +338,9 @@ class _StubWriterWorker(NixlPushConnectorWorker):
|
||||
w.engine_id = "test-decode-engine"
|
||||
w._remote_agents = {}
|
||||
w._physical_blocks_per_logical_kv_block = 1
|
||||
# Single non-hybrid attention group, matching the stub block id lists.
|
||||
w._has_mamba = False
|
||||
w._group_spec_types = (FullAttentionSpec,)
|
||||
|
||||
# Track _do_start_push_kv invocations.
|
||||
calls: list[tuple[str, Any, dict[str, Any]]] = []
|
||||
|
||||
Reference in New Issue
Block a user