From faacc13565312d29e9596d182fc808b452a4508e Mon Sep 17 00:00:00 2001 From: Stary Date: Tue, 29 Sep 2026 22:29:04 +0200 Subject: [PATCH] [Perf][KV Connector][Mooncake] Pack hybrid/MLA KV into coalesced transfer regions (#57952) Signed-off-by: staryxchen Co-authored-by: Cursor Co-authored-by: Yifan Qiao --- .../unit/test_mooncake_connector.py | 1030 ++++++++++++++--- .../test_mooncake_connector_hybrid_mamba.py | 2 + .../v1/mooncake/mooncake_connector.py | 559 ++++++++- 3 files changed, 1384 insertions(+), 207 deletions(-) diff --git a/tests/v1/kv_connector/unit/test_mooncake_connector.py b/tests/v1/kv_connector/unit/test_mooncake_connector.py index be70a46bf7..c306103288 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_connector.py +++ b/tests/v1/kv_connector/unit/test_mooncake_connector.py @@ -23,6 +23,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import ( mooncake_utils, ) from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector import ( + _SHARED_REGION_GROUP_ID, KVConnectorRole, MooncakeConnector, MooncakeConnectorMetadata, @@ -34,7 +35,11 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector im SendBlockMeta, TransferRegion, _align_transfer_regions, + _block_ids_for_region, + _coalesce_contiguous_transfer_regions, _compute_sender_transfer_plan, + _has_opaque_packed_row, + _pp_mismatch_hides_packed_layers, _validate_asymmetric_region_lengths, get_mooncake_bootstrap_addr, should_launch_bootstrap_server, @@ -48,6 +53,7 @@ from vllm.v1.kv_cache_interface import ( KVCacheConfig, KVCacheGroupSpec, KVCacheLayout, + MLAAttentionSpec, ) from vllm.v1.request import RequestStatus @@ -164,6 +170,34 @@ def _make_test_kv_cache_config() -> KVCacheConfig: ) +def _make_packed_mla_kv_cache_config( + num_blocks: int, *, num_groups: int = 1 +) -> KVCacheConfig: + spec = MLAAttentionSpec( + block_size=16, + num_kv_heads=1, + head_size=64, + dtype=torch.uint8, + ) + if num_groups == 1: + groups = [ + KVCacheGroupSpec( + ["model.layers.0.mla_attn", "model.layers.1.indexer"], + spec, + ) + ] + else: + groups = [ + KVCacheGroupSpec(["model.layers.0.mla_attn"], spec), + KVCacheGroupSpec(["model.layers.1.indexer"], spec), + ] + return KVCacheConfig( + num_blocks=num_blocks, + kv_cache_tensors=[], + kv_cache_groups=groups, + ) + + class FakeMooncakeWrapper: """Mock Mooncake TransferEngine for unit testing environments.""" @@ -235,6 +269,37 @@ def test_align_transfer_regions_uses_layer_name_occurrences(): assert [r.base_addr for r in aligned_local] == [0x1000, 0x1100] assert [r.base_addr for r in aligned_remote] == [0xB000, 0xB100] + # Hetero PP with no shared layers is an empty success. Same PP fails closed. + p_regions = [ + _region( + 0x1000, + block_len=2048, + kv_block_len=1024, + row_offset=0, + layer_name="model.layers.2.mla_attn", + layer_index=2, + ) + ] + early_d = [ + _region( + 0xA000, + block_len=2048, + kv_block_len=1024, + row_offset=0, + layer_name="model.layers.0.mla_attn", + layer_index=0, + ) + ] + none_p, none_d, none_err = _align_transfer_regions( + p_regions, early_d, allow_partial_layers=True + ) + assert none_err is None + assert none_p == [] and none_d == [] + _, _, mismatch_err = _align_transfer_regions( + p_regions, early_d, allow_partial_layers=False + ) + assert mismatch_err is not None + @pytest.mark.asyncio async def test_build_transfer_params_separates_prefill_pp_layers(): @@ -323,18 +388,7 @@ async def test_build_transfer_params_separates_prefill_pp_layers(): local_block_ids=[[10, 11]], ready=asyncio.Event(), ) - xfer_meta = MooncakeXferMetadata( - remote_hostname="consumer-host", - remote_port=54321, - remote_tp_size=1, - remote_tp_rank=0, - req_blocks={"d-req-pp": (transfer_id, [[20, 21]])}, - kv_caches_base_addr=[region.base_addr for region in remote_regions], - block_lens=[region.block_len for region in remote_regions], - kv_block_lens=[region.kv_block_len for region in remote_regions], - registered_layer_names=[region.layer_name for region in remote_regions], - registered_layer_indices=[region.layer_index for region in remote_regions], - ) + xfer_meta = _xfer_meta(remote_regions, {"d-req-pp": (transfer_id, [[20, 21]])}) for pp_rank, local_regions in producer_pp_regions.items(): aligned_local, aligned_remote, err = _align_transfer_regions( @@ -411,20 +465,24 @@ async def test_send_kv_to_decode_aligns_consumer_regions_by_layer_metadata( prefill_worker.reqs_need_send[transfer_id] = send_meta send_meta.ready.set() - xfer_meta = MooncakeXferMetadata( - remote_hostname="consumer-host", - remote_port=54321, - remote_tp_size=1, - remote_tp_rank=0, - req_blocks={"d-req-layer-align": (transfer_id, [[20]])}, - kv_caches_base_addr=[0xA000, 0xB000], - block_lens=[block_len, block_len], - kv_block_lens=[block_len, block_len], - registered_layer_names=[ - "model.layers.0.self_attn", - "model.layers.1.self_attn", + xfer_meta = _xfer_meta( + [ + _region( + 0xA000, + block_len=block_len, + row_offset=-1, + layer_name="model.layers.0.self_attn", + layer_index=0, + ), + _region( + 0xB000, + block_len=block_len, + row_offset=-1, + layer_name="model.layers.1.self_attn", + layer_index=1, + ), ], - registered_layer_indices=[0, 1], + {"d-req-layer-align": (transfer_id, [[20]])}, ) mock_socket = AsyncMock(spec=zmq.asyncio.Socket) mock_socket.send_multipart = AsyncMock() @@ -1093,6 +1151,81 @@ def patch_worker_dependencies(): } +def _region( + base: int, + *, + block_len: int = 256, + kv_block_len: int | None = None, + row_offset: int | None = None, + layer_name: str = "layer", + layer_index: int = 0, + group_index: int = 0, + shared_group_ids: tuple[int, ...] = (), +) -> TransferRegion: + """One transfer slice. `row_offset` defaults to `base` for in-row addresses.""" + return TransferRegion( + layer_name=layer_name, + layer_index=layer_index, + base_addr=base, + block_len=block_len, + kv_block_len=block_len if kv_block_len is None else kv_block_len, + group_index=group_index, + shared_group_ids=shared_group_ids, + row_offset=base if row_offset is None else row_offset, + ) + + +def _xfer_meta( + regions: list[TransferRegion], + req_blocks: dict, + **overrides, +) -> MooncakeXferMetadata: + """Handshake metadata derived from the regions under test.""" + fields = dict( + remote_hostname="consumer-host", + remote_port=54321, + remote_tp_size=1, + remote_tp_rank=0, + req_blocks=req_blocks, + kv_caches_base_addr=[region.base_addr for region in regions], + block_lens=[region.block_len for region in regions], + kv_block_lens=[region.kv_block_len for region in regions], + registered_layer_names=[region.layer_name for region in regions], + registered_layer_indices=[region.layer_index for region in regions], + registered_group_indices=[region.group_index for region in regions], + registered_shared_group_ids=[ + list(region.shared_group_ids) for region in regions + ], + registered_row_offsets=[region.row_offset for region in regions], + ) + fields.update(overrides) + return MooncakeXferMetadata(**fields) + + +@contextlib.contextmanager +def mooncake_register_worker(kv_cache_config, *, kv_role: str = "kv_consumer"): + """Build a worker whose engine registration and sender thread are patched.""" + vllm_config = create_vllm_config(kv_connector="MooncakeConnector", kv_role=kv_role) + connector_mod = ( + "vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector" + ) + with ( + set_current_vllm_config(vllm_config), + patch_worker_dependencies(), + patch(f"{connector_mod}.threading.Event"), + patch(f"{connector_mod}.threading.Thread") as mock_thread, + ): + connector = MooncakeConnector( + vllm_config, KVConnectorRole.WORKER, kv_cache_config + ) + worker = connector.connector_worker + mock_thread.return_value.is_alive.return_value = False + with patch.object( + worker.engine, "batch_register_memory", return_value=0 + ) as reg: + yield worker, reg + + @pytest.mark.asyncio @pytest.mark.parametrize( ("local_pp_size", "local_pp_rank", "expected_addrs"), @@ -1211,18 +1344,19 @@ async def test_heterogeneous_pp_waits_for_consumer_without_shared_layers(): patch.object(worker, "_send_blocks", return_value=0) as send, ): for pp_rank in range(2): - metadata = MooncakeXferMetadata( + metadata = _xfer_meta( + [ + _region( + 0x2000, + row_offset=-1, + layer_name=f"model.layers.{pp_rank}.self_attn", + layer_index=pp_rank, + ) + ], + {"d-req": ("transfer", [[2]])}, remote_hostname="consumer", remote_port=1234 + pp_rank, - remote_tp_size=1, - remote_tp_rank=0, remote_pp_size=2, - req_blocks={"d-req": ("transfer", [[2]])}, - kv_caches_base_addr=[0x2000], - block_lens=[256], - kv_block_lens=[256], - registered_layer_names=[f"model.layers.{pp_rank}.self_attn"], - registered_layer_indices=[pp_rank], ) await worker.send_kv_to_decode(b"consumer", sock, metadata) response = worker._xfer_resp_decoder.decode( @@ -1286,17 +1420,17 @@ async def test_kv_producer(monkeypatch): send_meta.ready.set() # Remote consumer request metadata - xfer_meta = MooncakeXferMetadata( - remote_hostname="consumer-host", - remote_port=54321, - remote_tp_size=1, - remote_tp_rank=0, - req_blocks={"d-req-1": (transfer_id, [[20, 21]])}, - kv_caches_base_addr=[0x2000], - block_lens=[block_len], - kv_block_lens=[block_len], - registered_layer_names=["model.layers.0.self_attn"], - registered_layer_indices=[0], + xfer_meta = _xfer_meta( + [ + _region( + 0x2000, + block_len=block_len, + row_offset=-1, + layer_name="model.layers.0.self_attn", + layer_index=0, + ) + ], + {"d-req-1": (transfer_id, [[20, 21]])}, ) mock_socket = AsyncMock(spec=zmq.asyncio.Socket) @@ -1544,139 +1678,703 @@ async def test_worker_get_finished_timeout(monkeypatch): ) def test_register_kv_caches(layout: KVCacheLayout, separate_kv_head_groups: bool): """Tests the memory registration logic with the underlying Mooncake engine.""" - vllm_config = create_vllm_config( - kv_connector="MooncakeConnector", kv_role="kv_consumer" + spec = FullAttentionSpec( + block_size=16, + num_kv_heads=4, + head_size=64, + dtype=torch.float16, + num_head_slots=2 if separate_kv_head_groups else None, + state_content_bytes=4 * 64 * 2 if separate_kv_head_groups else None, ) - - with ( - set_current_vllm_config(vllm_config), - patch_worker_dependencies(), - patch( - "vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector.threading.Event" - ), - patch( - "vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector.threading.Thread" - ) as mock_thread, - ): - connector = MooncakeConnector( - vllm_config, - KVConnectorRole.WORKER, - _make_test_kv_cache_config(), - ) - worker = connector.connector_worker - mock_thread.return_value.is_alive.return_value = False - - spec = FullAttentionSpec( - block_size=16, - num_kv_heads=4, - head_size=64, - dtype=torch.float16, - num_head_slots=2 if separate_kv_head_groups else None, - state_content_bytes=4 * 64 * 2 if separate_kv_head_groups else None, - ) - layer_names = [ - "model.layers.0.self_attn", - "model.layers.1.self_attn", - ] + layer_names = [ + "model.layers.0.self_attn", + "model.layers.1.self_attn", + ] + with mooncake_register_worker(_make_test_kv_cache_config()) as (worker, reg): for layer_name in layer_names: worker._layer_specs[layer_name] = spec raw = torch.zeros(2 * 2 * spec.page_size_bytes, dtype=torch.int8) tensor1, tensor2 = dense_kv_cache_views(raw, spec, 2, 2, layout) kv_caches = dict(zip(layer_names, (tensor1, tensor2))) + worker.register_kv_caches(kv_caches) - with patch.object( - worker.engine, "batch_register_memory", return_value=0 - ) as mock_batch_register: - connector.register_kv_caches(kv_caches) + reg.assert_called_once() + registered_ptrs, registered_lens = reg.call_args[0] + assert registered_ptrs == [raw.data_ptr()] + assert registered_lens == [raw.nbytes] - mock_batch_register.assert_called_once() - registered_ptrs, registered_lens = mock_batch_register.call_args[0] - assert registered_ptrs == [raw.data_ptr()] - assert registered_lens == [raw.nbytes] + n_blocks = tensor1.shape[0] if tensor1.ndim > 1 else 1 + storage_end = raw.data_ptr() + raw.nbytes + for base, stride, kv_len in zip( + worker.kv_caches_base_addr, + worker.block_len_per_layer, + worker.kv_block_len_per_layer, + ): + assert base + (n_blocks - 1) * stride + kv_len <= storage_end - if not layout.is_block_compact: - expected_addrs = [ - cache[:, head_idx].data_ptr() - for cache in (tensor1, tensor2) - for head_idx in range(cache.shape[1]) - ] - head_block_bytes = tensor1.stride(0) * tensor1.element_size() - assert worker.kv_caches_base_addr == expected_addrs - assert worker.block_len_per_layer == [head_block_bytes] * len( - expected_addrs - ) - assert worker.kv_block_len_per_layer == [head_block_bytes] * len( - expected_addrs - ) - assert worker.registered_layer_names == [ - layer_name - for layer_name in layer_names - for _ in range(tensor1.shape[1]) - ] - else: - assert len(worker.block_len_per_layer) == len(kv_caches) - for bl in worker.block_len_per_layer: - assert bl == tensor1.stride(0) * tensor1.element_size() - assert worker.kv_block_len_per_layer == [spec.page_size_bytes] * 2 - assert worker.registered_layer_names == list(kv_caches) - assert worker.registered_layer_indices == [0, 1] + # Expected shape is the layout, not a recomputation of the pack predicate. + kind = { + KVCacheLayout.LBHNC: "page", + KVCacheLayout.BLHNC: "page", + KVCacheLayout.BHLNC: "opaque", + KVCacheLayout.LHBNC: "per_head", + }[layout] + if kind == "opaque": + packed_row = raw.nbytes // n_blocks + assert worker.kv_caches_base_addr == [raw.data_ptr()] + assert worker.block_len_per_layer == [packed_row] + assert worker.kv_block_len_per_layer == [packed_row] + assert worker.registered_layer_names == [layer_names[0]] + elif kind == "per_head": + expected_addrs = [ + cache[:, head_idx].data_ptr() + for cache in (tensor1, tensor2) + for head_idx in range(cache.shape[1]) + ] + head_block_bytes = tensor1.stride(0) * tensor1.element_size() + assert worker.kv_caches_base_addr == expected_addrs + assert worker.block_len_per_layer == [head_block_bytes] * len( + expected_addrs + ) + assert worker.kv_block_len_per_layer == [head_block_bytes] * len( + expected_addrs + ) + assert worker.registered_layer_names == [ + layer_name + for layer_name in layer_names + for _ in range(tensor1.shape[1]) + ] + else: + page_stride = tensor1.stride(0) * tensor1.element_size() + assert len(worker.block_len_per_layer) == len(kv_caches) + assert worker.block_len_per_layer == [page_stride, page_stride] + assert worker.kv_block_len_per_layer == [spec.page_size_bytes] * 2 + assert worker.registered_layer_names == list(kv_caches) + assert worker.registered_layer_indices == [0, 1] + + +def test_register_noncontiguous_packed_allows_matching_pp(): + """Same PP on both sides can still register a non-contiguous packed row. + + A remote PP mismatch is rejected at send time, once the peer PP size is + known. Rejecting every local PP>1 layout at startup is too strict. + """ + spec = FullAttentionSpec( + block_size=16, num_kv_heads=4, head_size=64, dtype=torch.float16 + ) + layer_names = ["model.layers.0.self_attn", "model.layers.1.self_attn"] + with mooncake_register_worker(_make_test_kv_cache_config()) as (worker, _reg): + worker.pp_size = 2 + for layer_name in layer_names: + worker._layer_specs[layer_name] = spec + raw = torch.zeros(2 * 2 * spec.page_size_bytes, dtype=torch.int8) + tensor1, tensor2 = dense_kv_cache_views(raw, spec, 2, 2, KVCacheLayout.BHLNC) + assert not tensor1[0].is_contiguous() + worker.register_kv_caches(dict(zip(layer_names, (tensor1, tensor2)))) + assert worker.kv_caches_base_addr == [raw.data_ptr()] + assert worker.region_row_offsets == [0] def test_register_kv_caches_supports_mixed_mla_and_eagle_shapes(): """Mixed MLA+Eagle caches should register by byte length, not shape.""" - vllm_config = create_vllm_config( - kv_connector="MooncakeConnector", kv_role="kv_consumer" - ) - - with ( - set_current_vllm_config(vllm_config), - patch_worker_dependencies(), - patch( - "vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector.threading.Event" - ), - patch( - "vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector.threading.Thread" - ) as mock_thread, - ): - connector = MooncakeConnector( - vllm_config, - KVConnectorRole.WORKER, - _make_test_kv_cache_config(), - ) - worker = connector.connector_worker - mock_thread.return_value.is_alive.return_value = False - + with mooncake_register_worker(_make_test_kv_cache_config()) as (worker, reg): worker.use_mla = True worker.transfer_topo.is_mla = True - - # MLA cache tensor: shape[-2] is the block size and each block's - # byte stride matches the cache spec page size. mla_cache = torch.zeros((2, 16, 512), dtype=torch.float16) - # Eagle3/GQA-like cache tensor: shape[-2] is num_kv_heads, not block size. eagle_cache = torch.zeros((2, 16, 8, 64), dtype=torch.float16) kv_caches = { "model.layers.0.mla_attn": mla_cache, "model.layers.1.eagle_attn": eagle_cache, } + worker.register_kv_caches(kv_caches) - with patch.object( - worker.engine, "batch_register_memory", return_value=0 - ) as mock_batch_register: - connector.register_kv_caches(kv_caches) + reg.assert_called_once() + registered_ptrs, registered_lens = reg.call_args[0] + assert registered_ptrs == [mla_cache.data_ptr(), eagle_cache.data_ptr()] + assert registered_lens == [mla_cache.nbytes, eagle_cache.nbytes] + assert worker.block_len_per_layer == [ + mla_cache.nbytes // mla_cache.shape[0], + eagle_cache.nbytes // eagle_cache.shape[0], + ] + assert worker.registered_layer_names == [ + "model.layers.0.mla_attn", + "model.layers.1.eagle_attn", + ] + assert worker.registered_layer_indices == [0, 1] + assert len(worker.kv_caches_base_addr) == 2 - mock_batch_register.assert_called_once() - registered_ptrs, registered_lens = mock_batch_register.call_args[0] - assert registered_ptrs == [mla_cache.data_ptr(), eagle_cache.data_ptr()] - assert registered_lens == [mla_cache.nbytes, eagle_cache.nbytes] - assert worker.block_len_per_layer == [ - mla_cache.nbytes // mla_cache.shape[0], - eagle_cache.nbytes // eagle_cache.shape[0], + +def test_packed_and_unpacked_region_lengths_fail_homogeneous_tp_handshake(): + packed = TransferRegion( + layer_name="model.layers.0.mla_attn", + layer_index=0, + base_addr=0x1000, + block_len=2048, + kv_block_len=2048, + group_index=0, + ) + unpacked = TransferRegion( + layer_name="model.layers.0.mla_attn", + layer_index=0, + base_addr=0x1000, + block_len=64, + kv_block_len=64, + group_index=0, + ) + err = _validate_asymmetric_region_lengths( + local_regions=[packed], + remote_regions=[unpacked], + local_tp_size=1, + remote_tp_size=1, + producer_cache_replicated=False, + ) + assert err is not None + assert "length mismatch" in err + + count_err = _validate_asymmetric_region_lengths( + local_regions=[packed], + remote_regions=[unpacked, unpacked], + local_tp_size=1, + remote_tp_size=1, + producer_cache_replicated=False, + ) + assert count_err is not None + assert "region counts" in count_err + + +@pytest.mark.parametrize( + ("num_groups", "expected_group_index"), + [(1, 0), (2, _SHARED_REGION_GROUP_ID)], +) +def test_register_kv_caches_collapses_shared_mla_storage( + num_groups, expected_group_index +): + """Shared backing collapses to one packed region; two groups mark it -1.""" + num_blocks = 4 + packed_row = 256 + with mooncake_register_worker( + _make_packed_mla_kv_cache_config(num_blocks, num_groups=num_groups) + ) as (worker, reg): + backing = torch.zeros((num_blocks, packed_row), dtype=torch.uint8) + worker.register_kv_caches( + { + "model.layers.0.mla_attn": backing, + "model.layers.1.indexer": backing, + } + ) + + reg.assert_called_once() + registered_ptrs, registered_lens = reg.call_args[0] + assert registered_ptrs == [backing.data_ptr()] + assert registered_lens == [backing.untyped_storage().nbytes()] + assert worker.kv_caches_base_addr == [backing.data_ptr()] + assert worker.block_len_per_layer == [packed_row] + assert worker.kv_block_len_per_layer == [packed_row] + assert worker.registered_layer_names == ["model.layers.0.mla_attn"] + assert worker.registered_group_indices == [expected_group_index] + if expected_group_index == _SHARED_REGION_GROUP_ID: + assert worker.region_shared_groups == [(0, 1)] + + +def test_block_ids_for_region_flattens_shared_groups(): + ids = [[10, 11], [12]] + assert _block_ids_for_region(ids, 0) == [10, 11] + assert _block_ids_for_region(ids, 1) == [12] + assert _block_ids_for_region(ids, _SHARED_REGION_GROUP_ID, (0, 1)) == [ + 10, + 11, + 12, + ] + # A group that does not share the allocation stays out. + assert _block_ids_for_region( + [[10, 11], [12], [99]], _SHARED_REGION_GROUP_ID, (0, 1) + ) == [10, 11, 12] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("group_index", "local_ids", "remote_ids", "n_blocks"), + [ + (0, [[10, 11]], [[20, 21]], 2), + (_SHARED_REGION_GROUP_ID, [[10, 11], [12]], [[20, 21], [22]], 3), + ], +) +async def test_build_transfer_params_sends_packed_region_once( + group_index, local_ids, remote_ids, n_blocks +): + """Packed region coalesces contiguous blocks, including shared-group flatten.""" + worker = MooncakeConnectorWorker.__new__(MooncakeConnectorWorker) + worker.async_zmq_ctx = MagicMock() + worker.is_kv_consumer = True + worker.is_kv_producer = True + worker.tp_rank = 0 + worker.tp_size = 1 + worker.use_mla = True + worker.kv_cache_config = _make_packed_mla_kv_cache_config( + num_blocks=4, num_groups=len(local_ids) + ) + worker._physical_blocks_per_logical_kv_block = 1 + worker.transfer_topo = SimpleNamespace( + local_replicates_kv_cache=False, + total_num_kv_heads=1, + ) + + block_len = 256 + shared = (0, 1) if group_index == _SHARED_REGION_GROUP_ID else () + local_region = _region( + 0x1000, + block_len=block_len, + row_offset=0, + layer_name="model.layers.0.mla_attn", + group_index=group_index, + shared_group_ids=shared, + ) + remote_region = _region( + 0xA000, + block_len=block_len, + row_offset=0, + layer_name="model.layers.0.mla_attn", + group_index=group_index, + shared_group_ids=shared, + ) + transfer_id = "xfer-packed" + send_meta = SendBlockMeta( + p_req_id="p-packed", + transfer_id=transfer_id, + local_block_ids=local_ids, + ready=asyncio.Event(), + ) + xfer_meta = _xfer_meta([remote_region], {"d-packed": (transfer_id, remote_ids)}) + + ( + src_ptrs, + dst_ptrs, + lengths, + err_reqs, + err_msg, + ) = await worker._build_transfer_params( + ready_reqs=[("d-packed", send_meta)], + agent_meta=xfer_meta, + local_regions=[local_region], + remote_regions=[remote_region], + ) + + assert err_reqs == [] + assert err_msg is None + assert src_ptrs == [0x1000 + 10 * block_len] + assert dst_ptrs == [0xA000 + 20 * block_len] + assert lengths == [n_blocks * block_len] + + +def test_coalesce_promotes_padding_only_for_a_full_row(): + """A full-row merge includes the padding tail; a partial row does not.""" + page, row = 1024, 2560 + local = [ + _region(0, block_len=row, kv_block_len=page, row_offset=0), + _region(page, block_len=row, kv_block_len=page, row_offset=page), + ] + remote = [ + _region(100, block_len=row, kv_block_len=page, row_offset=0), + _region(100 + page, block_len=row, kv_block_len=page, row_offset=page), + ] + full_local, full_remote = _coalesce_contiguous_transfer_regions( + local, remote, promote_full_row=True + ) + assert len(full_local) == 1 + assert full_local[0].kv_block_len == full_remote[0].kv_block_len == row + + partial_local, _ = _coalesce_contiguous_transfer_regions(local, remote) + assert partial_local[0].kv_block_len == 2 * page + + +def test_coalesce_spans_padding_between_pages(): + """Unpadded pages merge across the padding gap without a full-row lift.""" + row = 2560 + merged, _ = _coalesce_contiguous_transfer_regions( + [ + _region(0, block_len=row, kv_block_len=1000), + _region(1024, block_len=row, kv_block_len=1000), + ], + [ + _region(5000, block_len=row, kv_block_len=1000, row_offset=0), + _region(6024, block_len=row, kv_block_len=1000, row_offset=1024), + ], + ) + assert len(merged) == 1 + assert merged[0].kv_block_len == 2024 + assert merged[0].row_offset == 0 + + +def test_coalesce_promotes_each_group_that_starts_at_row_zero(): + """Groups overlay one row, so each row-start run is its own full-row copy.""" + page, gap, row = 1000, 1024, 4096 + + def side(origin: int) -> list[TransferRegion]: + return [ + _region( + origin + base, + block_len=row, + kv_block_len=page, + row_offset=offset, + layer_name=f"group-{group}", + layer_index=group, + group_index=group, + ) + for group, base, offset in ( + (0, 0, 0), + (0, gap, gap), + (1, 0, 0), + (1, gap, gap), + ) ] - assert worker.registered_layer_names == [ - "model.layers.0.mla_attn", - "model.layers.1.eagle_attn", - ] - assert worker.registered_layer_indices == [0, 1] + + merged_local, merged_remote = _coalesce_contiguous_transfer_regions( + side(0), side(8000), promote_full_row=True + ) + assert [region.group_index for region in merged_local] == [0, 1] + assert [region.kv_block_len for region in merged_local] == [row, row] + assert [region.kv_block_len for region in merged_remote] == [row, row] + assert merged_local[1].base_addr == 0 + assert merged_remote[1].base_addr == 8000 + + +def test_coalesce_does_not_promote_a_mid_row_run(): + """A matched PP slice that does not start at row offset 0 stays partial.""" + page, row = 1000, 4096 + merged, _ = _coalesce_contiguous_transfer_regions( + [ + _region( + 1024, block_len=row, kv_block_len=page, row_offset=1024, layer_index=2 + ), + _region( + 2048, block_len=row, kv_block_len=page, row_offset=2048, layer_index=2 + ), + ], + [ + _region( + 9000 + 1024, + block_len=row, + kv_block_len=page, + row_offset=1024, + layer_index=2, + ), + _region( + 9000 + 2048, + block_len=row, + kv_block_len=page, + row_offset=2048, + layer_index=2, + ), + ], + promote_full_row=True, + ) + assert len(merged) == 1 + assert merged[0].row_offset == 1024 + assert merged[0].kv_block_len == 1024 + page + + +def test_opaque_packed_row_is_full_stride_at_offset_zero(): + """Branch B is row-sized at offset 0. A page view at offset 0 is not.""" + assert _has_opaque_packed_row([0], [4096], [4096]) + assert not _has_opaque_packed_row([0, 1024], [1000, 1000], [4096, 4096]) + assert not _has_opaque_packed_row([], [4096], [4096]) + + +def test_pp_mismatch_rejects_opaque_row_in_both_directions(): + """The smaller PP side can be either the producer or the consumer.""" + page_view = dict( + remote_row_offsets=[0, 1024], + remote_kv_block_lens=[1000, 1000], + remote_block_lens=[4096, 4096], + ) + opaque_row = dict( + remote_row_offsets=[0], + remote_kv_block_lens=[4096], + remote_block_lens=[4096], + ) + # P pp=2, D pp=1, local row is opaque. + assert _pp_mismatch_hides_packed_layers( + 2, 1, local_has_opaque_row=True, **page_view + ) + # P pp=1, D pp=2, remote row is opaque. + assert _pp_mismatch_hides_packed_layers( + 1, 2, local_has_opaque_row=False, **opaque_row + ) + # Matching PP still aligns by the shared first layer name. + assert not _pp_mismatch_hides_packed_layers( + 2, 2, local_has_opaque_row=True, **opaque_row + ) + # Hetero PP with page-contiguous views is not this rejection. + assert not _pp_mismatch_hides_packed_layers( + 2, 1, local_has_opaque_row=False, **page_view + ) + + +def test_prepare_transfer_regions_reuses_success_and_error(): + """A peer layout is built once. Re-registering drops the cache.""" + worker, _, _, _ = _register_sliced_packed_mla([0, 1]) + regions = worker._get_transfer_regions( + worker.kv_caches_base_addr, + worker.block_len_per_layer, + worker.kv_block_len_per_layer, + worker.registered_layer_names, + worker.registered_layer_indices, + worker.registered_group_indices, + [tuple(groups) for groups in worker.region_shared_groups], + worker.region_row_offsets, + ) + meta = _xfer_meta( + regions, + {}, + remote_hostname="peer", + remote_port=1, + remote_tp_size=worker.tp_size, + remote_pp_size=worker.pp_size, + ) + calls = 0 + original = worker._get_transfer_regions + + def counting(*args, **kwargs): + nonlocal calls + calls += 1 + return original(*args, **kwargs) + + worker._get_transfer_regions = counting + prepared = worker._prepare_transfer_regions(meta) + assert prepared[2] is None + assert calls == 2 + assert worker._prepare_transfer_regions(meta) == prepared + assert calls == 2 + + mismatch = _xfer_meta( + [ + _region( + 0x2000, + block_len=1024, + row_offset=0, + layer_name="model.layers.9.mla_attn", + layer_index=9, + ) + ], + {}, + remote_hostname="peer", + remote_port=2, + remote_tp_size=worker.tp_size, + remote_pp_size=worker.pp_size, + ) + failed = worker._prepare_transfer_regions(mismatch) + assert failed[2] is not None + assert calls == 4 + assert worker._prepare_transfer_regions(mismatch)[2] == failed[2] + assert calls == 4 + + page = 1024 + backing = torch.zeros((4, page * 2), dtype=torch.uint8) + kv_caches = { + "model.layers.0.mla_attn": backing[:, :page], + "model.layers.1.mla_attn": backing[:, page:], + } + worker.is_kv_consumer = True + # The first register built a producer. Its thread patch is already gone, + # so stay on the consumer path and return before the sender thread starts. + with patch.object(worker.engine, "batch_register_memory", return_value=0): + worker.register_kv_caches(kv_caches) + assert worker._prepared_transfer_regions == {} + worker._prepare_transfer_regions(meta) + assert calls == 6 + + +def test_coalesce_keeps_a_run_inside_its_starting_row(): + """The row bound is the run start, not the previous slice.""" + row = 2500 + slices = [ + _region(base, block_len=row, kv_block_len=1000) for base in (0, 1000, 2000) + ] + merged, _ = _coalesce_contiguous_transfer_regions(slices, slices) + assert len(merged) == 2 + assert merged[0].kv_block_len == 2000 + assert merged[1].base_addr == 2000 + assert merged[1].kv_block_len == 1000 + + +def _layer_name(idx: int) -> str: + return f"model.layers.{idx}.mla_attn" + + +def _register_sliced_packed_mla( + layer_idxs: list[int], *, page: int = 1024, num_blocks: int = 4 +): + """Register column slices of one MLA packed row. Returns worker fields.""" + spec = MLAAttentionSpec( + block_size=16, + num_kv_heads=1, + head_size=64, + dtype=torch.uint8, + ) + kv_cache_config = KVCacheConfig( + num_blocks=num_blocks, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec([_layer_name(i) for i in layer_idxs], spec), + ], + ) + backing = torch.zeros((num_blocks, page * len(layer_idxs)), dtype=torch.uint8) + kv_caches = { + _layer_name(idx): backing[:, i * page : (i + 1) * page] + for i, idx in enumerate(layer_idxs) + } + with mooncake_register_worker(kv_cache_config, kv_role="kv_producer") as ( + worker, + _, + ): + worker.register_kv_caches(kv_caches) + regions = worker._get_transfer_regions( + worker.kv_caches_base_addr, + worker.block_len_per_layer, + worker.kv_block_len_per_layer, + worker.registered_layer_names, + worker.registered_layer_indices, + worker.registered_group_indices, + row_offsets=worker.region_row_offsets, + ) + return worker, kv_cache_config, regions, backing.data_ptr() + + +def test_sliced_packed_mla_registers_each_layer_view(): + """Each column of a packed MLA row stays its own view, in row order.""" + page = 1024 + layer_idxs = [0, 1] + with mooncake_register_worker( + KVCacheConfig( + num_blocks=4, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec( + [_layer_name(i) for i in layer_idxs], + MLAAttentionSpec( + block_size=16, + num_kv_heads=1, + head_size=64, + dtype=torch.uint8, + ), + ) + ], + ), + kv_role="kv_producer", + ) as (worker, reg): + backing = torch.zeros((4, page * len(layer_idxs)), dtype=torch.uint8) + worker.register_kv_caches( + { + _layer_name(idx): backing[:, i * page : (i + 1) * page] + for i, idx in enumerate(layer_idxs) + } + ) + reg.assert_called_once() + assert reg.call_args[0][0] == [backing.data_ptr()] + assert worker.region_row_offsets == [0, page] + + +async def _copy_packed_slices( + p_layers: list[int], + d_layers: list[int], + *, + allow_partial: bool, + local_ids: list[list[int]], + remote_ids: list[list[int]], + page: int = 1024, +): + p_worker, p_config, p_regions, p_base = _register_sliced_packed_mla( + p_layers, page=page + ) + _, _, d_regions, d_base = _register_sliced_packed_mla(d_layers, page=page) + aligned_p, aligned_d, err = _align_transfer_regions( + p_regions, d_regions, allow_partial_layers=allow_partial + ) + assert err is None + coalesced_p, coalesced_d = _coalesce_contiguous_transfer_regions( + aligned_p, aligned_d + ) + transfer_id = "xfer-packed-slices" + send_meta = SendBlockMeta( + p_req_id="p-slices", + transfer_id=transfer_id, + local_block_ids=local_ids, + ready=asyncio.Event(), + ) + p_worker.kv_cache_config = p_config + src, dst, lengths, err_reqs, err_msg = await p_worker._build_transfer_params( + ready_reqs=[("d-slices", send_meta)], + agent_meta=_xfer_meta(coalesced_d, {"d-slices": (transfer_id, remote_ids)}), + local_regions=coalesced_p, + remote_regions=coalesced_d, + ) + return SimpleNamespace( + p_regions=p_regions, + d_regions=d_regions, + coalesced_p=coalesced_p, + coalesced_d=coalesced_d, + p_base=p_base, + d_base=d_base, + src=src, + dst=dst, + lengths=lengths, + err_reqs=err_reqs, + err_msg=err_msg, + ) + + +@pytest.mark.asyncio +async def test_packed_hetero_pp_keeps_shared_layers(): + """P-PP4/D-PP2 copies the shared MLA span, not either side's full row.""" + page = 1024 + copied = await _copy_packed_slices( + [2, 3], + [0, 1, 2, 3], + allow_partial=True, + local_ids=[[10, 11]], + remote_ids=[[10, 11]], + page=page, + ) + assert [region.layer_index for region in copied.p_regions] == [2, 3] + assert [region.kv_block_len for region in copied.p_regions] == [page, page] + assert copied.p_regions[0].block_len == 2 * page + assert copied.d_regions[0].block_len == 4 * page + assert len(copied.coalesced_p) == 1 + assert copied.coalesced_p[0].kv_block_len == 2 * page + assert copied.coalesced_d[0].base_addr == copied.d_base + 2 * page + assert copied.err_reqs == [] + assert copied.err_msg is None + assert copied.lengths == [2 * page, 2 * page] + assert copied.src == [ + copied.p_base + 10 * (2 * page), + copied.p_base + 11 * (2 * page), + ] + assert copied.dst == [ + copied.d_base + 2 * page + 10 * (4 * page), + copied.d_base + 2 * page + 11 * (4 * page), + ] + + +@pytest.mark.asyncio +async def test_packed_matching_layers_coalesce_to_one_copy(): + """Identical packed layer sets still emit one copy for contiguous blocks.""" + page = 1024 + copied = await _copy_packed_slices( + [0, 1], + [0, 1], + allow_partial=False, + local_ids=[[10, 11]], + remote_ids=[[20, 21]], + page=page, + ) + assert copied.err_reqs == [] + assert copied.err_msg is None + assert len(copied.coalesced_p) == 1 + assert copied.coalesced_p[0].kv_block_len == copied.coalesced_p[0].block_len + block_len = 2 * page + assert copied.src == [copied.p_base + 10 * block_len] + assert copied.dst == [copied.d_base + 20 * block_len] + assert copied.lengths == [2 * block_len] @pytest.mark.asyncio @@ -1760,22 +2458,24 @@ async def test_kv_producer_heterogeneous_tp(monkeypatch, d_tp_size): ) as mock_send_blocks: for d_rank in target_d_ranks: remote_block_ids = d_rank_remote_blocks[d_rank] - xfer_meta = MooncakeXferMetadata( - remote_hostname="consumer-host", - remote_port=54321, - remote_tp_size=d_tp_size, - remote_tp_rank=d_rank, - req_blocks={ + xfer_meta = _xfer_meta( + [ + _region( + 0x2000, + block_len=remote_block_len, + row_offset=-1, + layer_name="model.layers.0.self_attn", + layer_index=0, + ) + ], + { f"d-req-h1-r{d_rank}": ( transfer_id, remote_block_ids, ) }, - kv_caches_base_addr=[0x2000], - block_lens=[remote_block_len], - kv_block_lens=[remote_block_len], - registered_layer_names=["model.layers.0.self_attn"], - registered_layer_indices=[0], + remote_tp_size=d_tp_size, + remote_tp_rank=d_rank, ) mock_send_blocks.reset_mock() diff --git a/tests/v1/kv_connector/unit/test_mooncake_connector_hybrid_mamba.py b/tests/v1/kv_connector/unit/test_mooncake_connector_hybrid_mamba.py index 9376745516..23856b4541 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_connector_hybrid_mamba.py +++ b/tests/v1/kv_connector/unit/test_mooncake_connector_hybrid_mamba.py @@ -242,6 +242,8 @@ async def test_build_transfer_params_uses_non_overlapping_physical_pages(): kv_cache_config = make_hybrid_gdn_kv_cache_config(block_size=16) worker = object.__new__(MooncakeConnectorWorker) worker.shutdown = noop_shutdown + # __new__ skips __init__, where this cache is created. register_kv_caches clears it. + worker._prepared_transfer_regions = {} worker.use_mla = False worker.engine = SimpleNamespace(batch_register_memory=lambda *_: 0) worker.is_kv_consumer = True diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py index e3fb5b95c9..46118f4056 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import asyncio +import itertools import logging import queue import threading @@ -62,6 +63,8 @@ from vllm.v1.kv_cache_interface import ( KpoolTailSpec, KVCacheSpec, MambaSpec, + MLAAttentionSpec, + SlidingWindowMLASpec, SlidingWindowSpec, ) from vllm.v1.request import RequestStatus @@ -90,6 +93,12 @@ if TYPE_CHECKING: ReqId = str # Internal scheduler request ID TransferId = str # KV transfer coordination ID (shared by P/D) +# Follow NIXL (`nixl/base_worker.py`): HMA packed pages can put more than one +# KV group on a single allocation. group_index=-1 flattens every group's block +# ids onto that region. Hybrid KV manager assigns distinct ids per group, so +# flatten does not copy the same block twice. +_SHARED_REGION_GROUP_ID = -1 + @dataclass(frozen=True) class TransferRegion: @@ -99,6 +108,34 @@ class TransferRegion: block_len: int kv_block_len: int group_index: int = 0 + # Groups that share this allocation. Empty unless group_index is -1. + shared_group_ids: tuple[int, ...] = () + # Byte offset of this view from the start of its block row. + row_offset: int = 0 + + +def _block_ids_for_region( + block_ids_by_group: list[list[int]], + group_index: int, + shared_group_ids: tuple[int, ...] = (), +) -> list[int]: + """Follow NIXL `_block_ids_by_region`. + + -1 flattens only the groups that share this allocation. NIXL can flatten + every group because its packed flag means they all share one storage; + Mooncake can still have a separate Mamba group in the same request. + """ + if group_index == _SHARED_REGION_GROUP_ID: + assert shared_group_ids, "Shared packed region is missing its group ids." + return list( + itertools.chain.from_iterable( + block_ids_by_group[group] for group in shared_group_ids + ) + ) + assert 0 <= group_index < len(block_ids_by_group), ( + "Transfer region references a missing KV group." + ) + return list(block_ids_by_group[group_index]) def _get_tp_ratio(local_tp_size: int, remote_tp_size: int) -> int: @@ -129,6 +166,8 @@ def _expand_transfer_regions( layer_names: list[str], layer_indices: list[int], group_indices: list[int] | None = None, + shared_group_ids: list[tuple[int, ...]] | None = None, + row_offsets: list[int] | None = None, ) -> list[TransferRegion]: """Expand registered KV tensors into the regions transferred by Mooncake.""" assert ( @@ -150,6 +189,14 @@ def _expand_transfer_regions( "Mooncake transfer regions require matching group metadata lengths, " f"got group_indices={len(group_indices)}, layer_names={len(layer_names)}." ) + if shared_group_ids is None: + shared_group_ids = [() for _ in layer_names] + assert len(shared_group_ids) == len(layer_names) + # Missing offsets must not look like row starts, or every run would be + # promoted to the full row. + if row_offsets is None: + row_offsets = [-1] * len(layer_names) + assert len(row_offsets) == len(layer_names) regions: list[TransferRegion] = [] for ( base_addr, @@ -158,6 +205,8 @@ def _expand_transfer_regions( layer_name, layer_index, group_index, + region_groups, + row_offset, ) in zip( base_addrs, block_lens, @@ -165,6 +214,8 @@ def _expand_transfer_regions( layer_names, layer_indices, group_indices, + shared_group_ids, + row_offsets, ): regions.append( TransferRegion( @@ -174,6 +225,8 @@ def _expand_transfer_regions( block_len=block_len, kv_block_len=kv_block_len, group_index=group_index, + shared_group_ids=region_groups, + row_offset=row_offset, ) ) return regions @@ -377,12 +430,168 @@ def _align_transfer_regions( f"{remote_region.group_index}." ), ) + if ( + local_region.shared_group_ids + and remote_region.shared_group_ids + and local_region.shared_group_ids != remote_region.shared_group_ids + ): + return ( + [], + [], + ( + "Mooncake shared-group set mismatch for " + f"{local_region.layer_name}: producer=" + f"{local_region.shared_group_ids}, consumer=" + f"{remote_region.shared_group_ids}." + ), + ) aligned_local.append(local_region) aligned_remote.append(remote_region) return aligned_local, aligned_remote, None +def _coalesce_contiguous_transfer_regions( + local_regions: list[TransferRegion], + remote_regions: list[TransferRegion], + *, + promote_full_row: bool = False, +) -> tuple[list[TransferRegion], list[TransferRegion]]: + """Merge aligned packed slices that are adjacent on both sides. + + Page-contiguous views stay addressable by layer name for hetero PP + (#56033). KV groups overlay one row and must stay separate: each group + owns a different block-id space. A run that starts at row offset 0 is + lifted to the full row so consecutive blocks can coalesce. + """ + if len(local_regions) != len(remote_regions): + return local_regions, remote_regions + + def adjacent( + run_start: TransferRegion, prev: TransferRegion, nxt: TransferRegion + ) -> bool: + # Page padding sits between unpadded payloads. Allow that gap, but + # keep the whole run inside the row that `run_start` began. Anchoring + # on `prev` would slide the bound forward as the run grows. + return ( + prev.group_index == nxt.group_index + and prev.shared_group_ids == nxt.shared_group_ids + and prev.block_len == nxt.block_len + and prev.base_addr + prev.kv_block_len <= nxt.base_addr + and nxt.base_addr + nxt.kv_block_len + <= run_start.base_addr + run_start.block_len + ) + + def span(region: TransferRegion, kv_block_len: int) -> TransferRegion: + return TransferRegion( + layer_name=region.layer_name, + layer_index=region.layer_index, + base_addr=region.base_addr, + block_len=region.block_len, + kv_block_len=kv_block_len, + group_index=region.group_index, + shared_group_ids=region.shared_group_ids, + row_offset=region.row_offset, + ) + + def promote( + merged_local: list[TransferRegion], merged_remote: list[TransferRegion] + ) -> tuple[list[TransferRegion], list[TransferRegion]]: + if not promote_full_row: + return merged_local, merged_remote + # Each KV group overlays the row from offset 0. Promoting that run to + # the full row copies bytes the group's block ids exclusively own. + # Equal block_len is the row-structure check: both peers packed the + # same stride, so row_offset does not need a second compare in align. + promoted_local: list[TransferRegion] = [] + promoted_remote: list[TransferRegion] = [] + for local_region, remote_region in zip(merged_local, merged_remote): + if ( + local_region.row_offset == 0 + and remote_region.row_offset == 0 + and local_region.block_len == remote_region.block_len + and local_region.kv_block_len <= local_region.block_len + and remote_region.kv_block_len <= remote_region.block_len + ): + promoted_local.append(span(local_region, local_region.block_len)) + promoted_remote.append(span(remote_region, remote_region.block_len)) + else: + promoted_local.append(local_region) + promoted_remote.append(remote_region) + return promoted_local, promoted_remote + + if len(local_regions) <= 1: + return promote(list(local_regions), list(remote_regions)) + + merged_local: list[TransferRegion] = [] + merged_remote: list[TransferRegion] = [] + run_local = [local_regions[0]] + run_remote = [remote_regions[0]] + + def flush() -> None: + if len(run_local) == 1: + merged_local.append(run_local[0]) + merged_remote.append(run_remote[0]) + return + for merged, run in ((merged_local, run_local), (merged_remote, run_remote)): + span_len = run[-1].base_addr + run[-1].kv_block_len - run[0].base_addr + merged.append(span(run[0], span_len)) + + for local_region, remote_region in zip(local_regions[1:], remote_regions[1:]): + if adjacent(run_local[0], run_local[-1], local_region) and adjacent( + run_remote[0], run_remote[-1], remote_region + ): + run_local.append(local_region) + run_remote.append(remote_region) + continue + flush() + run_local = [local_region] + run_remote = [remote_region] + flush() + return promote(merged_local, merged_remote) + + +def _has_opaque_packed_row( + row_offsets: list[int], kv_block_lens: list[int], block_lens: list[int] +) -> bool: + """Branch B is one row-sized region at offset 0, so later layer names are gone. + + Page-contiguous views keep kv_block_len below the stride until promotion, + and promotion happens after this handshake check. + """ + if ( + not row_offsets + or len(row_offsets) != len(kv_block_lens) + or len(row_offsets) != len(block_lens) + ): + return False + return any( + offset == 0 and length == stride + for offset, length, stride in zip(row_offsets, kv_block_lens, block_lens) + ) + + +def _pp_mismatch_hides_packed_layers( + local_pp_size: int, + remote_pp_size: int, + *, + local_has_opaque_row: bool, + remote_row_offsets: list[int], + remote_kv_block_lens: list[int], + remote_block_lens: list[int], +) -> bool: + """A row that dropped per-layer names cannot be aligned across PP sizes. + + Either side may be the one that collapsed the row. Matching PP still + lines up, because both stages keep the same first layer name. + """ + if local_pp_size == remote_pp_size: + return False + return local_has_opaque_row or _has_opaque_packed_row( + remote_row_offsets, remote_kv_block_lens, remote_block_lens + ) + + def _get_tensor_dense_flag(tensor: torch.Tensor) -> bool | None: is_dense = getattr(tensor, "is_non_overlapping_and_dense", None) if callable(is_dense): @@ -405,6 +614,12 @@ class MooncakeXferMetadata( registered_layer_names: list[str] = msgspec.field(default_factory=list) registered_layer_indices: list[int] = msgspec.field(default_factory=list) registered_group_indices: list[int] = msgspec.field(default_factory=list) + # Parallel to registered regions. Each entry is the groups that share + # that region. Empty means the peer did not send the field. + registered_shared_group_ids: list[list[int]] = msgspec.field(default_factory=list) + # Byte offset of each registered view from its block row. -1 means the + # peer did not send the field, so the run must not be promoted. + registered_row_offsets: list[int] = msgspec.field(default_factory=list) remote_pp_size: int = 1 @@ -999,7 +1214,18 @@ class MooncakeConnectorWorker: self.registered_layer_names: list[str] = [] self.registered_layer_indices: list[int] = [] self.registered_group_indices: list[int] = [] + # Parallel to registered regions. -1 flattens only these groups. + self.region_shared_groups: list[tuple[int, ...]] = [] + # Parallel to registered regions. -1 is not a packed row view. + self.region_row_offsets: list[int] = [] + self.opaque_packed_storages: set[int] = set() self.seen_base_addresses: list[int] = [] + # Aligned regions depend only on the peer's registered layout. + # The third item is an error string when alignment cannot proceed. + self._prepared_transfer_regions: dict[ + tuple, + tuple[list[TransferRegion], list[TransferRegion], str | None], + ] = {} assert (parallel_config := vllm_config.parallel_config) dp_rank = parallel_config.data_parallel_index @@ -1266,39 +1492,35 @@ class MooncakeConnectorWorker: ) await sock.send_multipart((identity, self._encoder.encode(response))) return - local_regions = self._get_transfer_regions( - self.kv_caches_base_addr, - self.block_len_per_layer, - self.kv_block_len_per_layer, - self.registered_layer_names, - self.registered_layer_indices, - self.registered_group_indices, - ) - remote_regions = self._get_transfer_regions( - meta.kv_caches_base_addr, - meta.block_lens, - meta.kv_block_lens, - meta.registered_layer_names, - meta.registered_layer_indices, - meta.registered_group_indices, - ) - local_regions, remote_regions, align_err = _align_transfer_regions( - local_regions, - remote_regions, - allow_partial_layers=( - meta.remote_pp_size > 1 and meta.remote_pp_size != self.pp_size - ), - ) - if align_err is not None: + if _pp_mismatch_hides_packed_layers( + self.pp_size, + meta.remote_pp_size, + local_has_opaque_row=bool(self.opaque_packed_storages), + remote_row_offsets=meta.registered_row_offsets, + remote_kv_block_lens=meta.kv_block_lens, + remote_block_lens=meta.block_lens, + ): + msg = ( + "Mooncake non-page-contiguous packed regions keep only the " + "first layer name, so they cannot be aligned across PP sizes " + f"{self.pp_size} and {meta.remote_pp_size}." + ) + logger.error(msg) response = MooncakeXferResponse( status=MooncakeXferResponseStatus.ERROR, - err_msg=align_err, + err_msg=msg, ) await sock.send_multipart((identity, self._encoder.encode(response))) return - validation_err = self._validate_head_resharding_layout( - meta.remote_tp_size, local_regions - ) or _validate_asymmetric_region_lengths( + local_regions, remote_regions, prep_err = self._prepare_transfer_regions(meta) + if prep_err is not None: + response = MooncakeXferResponse( + status=MooncakeXferResponseStatus.ERROR, + err_msg=prep_err, + ) + await sock.send_multipart((identity, self._encoder.encode(response))) + return + validation_err = _validate_asymmetric_region_lengths( local_regions=local_regions, remote_regions=remote_regions, local_tp_size=self.tp_size, @@ -1600,12 +1822,16 @@ class MooncakeConnectorWorker: "Aligned Mooncake transfer regions must belong to the same " "KV group." ) - group_index = local_region.group_index - assert group_index < len(local_block_ids_by_group), ( - "Transfer region references a missing KV group." + local_block_ids = _block_ids_for_region( + local_block_ids_by_group, + local_region.group_index, + local_region.shared_group_ids, + ) + remote_block_ids = _block_ids_for_region( + remote_block_ids_by_group, + remote_region.group_index, + local_region.shared_group_ids, ) - local_block_ids = local_block_ids_by_group[group_index] - remote_block_ids = remote_block_ids_by_group[group_index] if not local_block_ids: continue @@ -1738,6 +1964,17 @@ class MooncakeConnectorWorker: self.registered_layer_names = [] self.registered_layer_indices = [] self.registered_group_indices = [] + self.region_shared_groups = [] + self.region_row_offsets = [] + self.opaque_packed_storages = set() + self._prepared_transfer_regions.clear() + + packed_storage_to_region: dict[int, int] = {} + packed_view_to_region: dict[tuple[int, int, int], int] = {} + collapsed_views = 0 + num_blocks = ( + self.kv_cache_config.num_blocks * self._physical_blocks_per_logical_kv_block + ) for layer_name, cache in kv_caches.items(): layer_index = extract_layer_index(layer_name) @@ -1751,6 +1988,104 @@ class MooncakeConnectorWorker: # One raw page tensor per layer; for Mamba that page holds all the # recurrent states, unpacked only when binding the cache for execution. self._log_debug_cache_registration(layer_name, cache) + group_index = self.kv_cache_config.transfer_group_index_by_layer[layer_name] + storage = cache.untyped_storage() + storage_addr = storage.data_ptr() + if storage_addr not in seen_storage_ptrs: + seen_storage_ptrs.add(storage_addr) + kv_data_ptrs.append(storage_addr) + kv_data_lens.append(storage.nbytes()) + + is_mla_region = isinstance( + layer_spec, (MLAAttentionSpec, SlidingWindowMLASpec) + ) + tensor_blocks = cache.shape[0] if cache.ndim > 1 else num_blocks + if cache.ndim == 1: + # A flat buffer is block-major only when it is an integral run + # of this layer's pages. NIXL uses the same remainder check. + hnc_contiguous = False + page = ( + layer_spec.page_size_bytes + if isinstance(layer_spec, AttentionSpec) + else 0 + ) + remainder = ( + storage.nbytes() - tensor_blocks * page + if tensor_blocks > 0 and page > 0 + else -1 + ) + if 0 <= remainder < page: + block_stride = page + storage_is_block_major = True + else: + block_stride = 0 + storage_is_block_major = False + else: + block_stride = cache.stride(0) * cache.element_size() + storage_is_block_major = ( + tensor_blocks > 0 + and tensor_blocks * block_stride == storage.nbytes() + ) + hnc_contiguous = ( + cache.ndim == 4 + and cache.stride(2) == cache.shape[3] + and cache.stride(1) == cache.shape[2] * cache.shape[3] + ) + # Follow NIXL packed pages. Page-contiguous slices stay per layer so + # hetero PP can align by name (#56033). A non-contiguous row cannot + # name a sub-span, so it is one region anchored at the allocation. + use_packed_row = ( + not isinstance(layer_spec, (MambaSpec, KpoolTailSpec)) + and storage_is_block_major + and block_stride > 0 + and (is_mla_region or not hnc_contiguous) + ) + if use_packed_row: + page_contiguous = ( + cache.ndim > 1 + and cache[0].is_contiguous() + and 0 < cache[0].nbytes <= block_stride + ) + if page_contiguous: + view_addr = cache.data_ptr() + kv_block_len = cache[0].nbytes + else: + # NIXL branch B: length is the row, base is the allocation. + view_addr = storage_addr + kv_block_len = block_stride + self.opaque_packed_storages.add(storage_addr) + # Same PP on both sides still aligns by the first layer name. + # A remote PP mismatch is rejected at send time, once the + # peer's PP size is known. + dedupe_key = (view_addr, block_stride, kv_block_len) + region_idx = packed_view_to_region.get(dedupe_key) + if region_idx is not None: + collapsed_views += 1 + # -1 means these groups share the view. Flatten only them. + groups = list(self.region_shared_groups[region_idx]) + if group_index not in groups: + groups.append(group_index) + self.region_shared_groups[region_idx] = tuple(groups) + if len(self.region_shared_groups[region_idx]) > 1: + self.registered_group_indices[region_idx] = ( + _SHARED_REGION_GROUP_ID + ) + continue + if not page_contiguous: + self.opaque_packed_storages.add(storage_addr) + if storage_addr not in packed_storage_to_region: + packed_storage_to_region[storage_addr] = len(region_base_addresses) + packed_view_to_region[dedupe_key] = len(region_base_addresses) + region_base_addresses.append(view_addr) + self.block_len_per_layer.append(block_stride) + self.kv_block_len_per_layer.append(kv_block_len) + self.registered_layer_names.append(layer_name) + self.registered_layer_indices.append(layer_index) + self.registered_group_indices.append(group_index) + self.region_shared_groups.append((group_index,)) + self.region_row_offsets.append(view_addr - storage_addr) + continue + block_is_contiguous = is_non_overlapping_and_dense(cache[0]) if not block_is_contiguous: # Non-block-compact layouts scatter a block across per-head @@ -1791,15 +2126,10 @@ class MooncakeConnectorWorker: self.kv_block_len_per_layer.append(kv_block_len) self.registered_layer_names.append(layer_name) self.registered_layer_indices.append(layer_index) - self.registered_group_indices.append( - self.kv_cache_config.transfer_group_index_by_layer[layer_name] - ) - storage = cache.untyped_storage() - storage_addr = storage.data_ptr() - if storage_addr not in seen_storage_ptrs: - seen_storage_ptrs.add(storage_addr) - kv_data_ptrs.append(storage_addr) - kv_data_lens.append(storage.nbytes()) + self.registered_group_indices.append(group_index) + self.region_shared_groups.append((group_index,)) + # Not a view into a shared packed row, so do not promote it. + self.region_row_offsets.append(-1) self.kv_caches_base_addr = region_base_addresses self.seen_base_addresses = kv_data_ptrs @@ -1807,6 +2137,18 @@ class MooncakeConnectorWorker: if not kv_data_ptrs: raise RuntimeError("No KV cache tensors were registered with Mooncake.") + if packed_storage_to_region and ( + collapsed_views + or len(packed_view_to_region) != len(packed_storage_to_region) + ): + logger.info( + "Mooncake packed KV xfer: %s layer regions over %s storage " + "regions (%s duplicate views collapsed).", + len(packed_view_to_region), + len(packed_storage_to_region), + collapsed_views, + ) + ret_value = self.engine.batch_register_memory(kv_data_ptrs, kv_data_lens) if ret_value != 0: raise RuntimeError("Mooncake batch memory registration failed.") @@ -1962,6 +2304,10 @@ class MooncakeConnectorWorker: registered_layer_names=self.registered_layer_names, registered_layer_indices=self.registered_layer_indices, registered_group_indices=self.registered_group_indices, + registered_shared_group_ids=[ + list(groups) for groups in self.region_shared_groups + ], + registered_row_offsets=self.region_row_offsets, ) encoded_data = self._encoder.encode(metadata) @@ -2209,6 +2555,131 @@ class MooncakeConnectorWorker: def _producer_cache_is_replicated(self) -> bool: return self.transfer_topo.local_replicates_kv_cache + def _prepare_transfer_regions( + self, meta: MooncakeXferMetadata + ) -> tuple[list[TransferRegion], list[TransferRegion], str | None]: + """Align and coalesce once per peer layout. + + ``send_kv_to_decode`` runs on every scheduler batch. The registered + layout does not change until the next ``register_kv_caches``. + """ + cache_key = ( + meta.remote_hostname, + meta.remote_port, + meta.remote_tp_size, + meta.remote_tp_rank, + meta.remote_pp_size, + tuple(meta.kv_caches_base_addr), + tuple(meta.block_lens), + tuple(meta.kv_block_lens), + tuple(meta.registered_layer_names), + tuple(meta.registered_layer_indices), + tuple(meta.registered_group_indices), + tuple(tuple(groups) for groups in meta.registered_shared_group_ids), + tuple(meta.registered_row_offsets), + ) + cached = self._prepared_transfer_regions.get(cache_key) + if cached is not None: + return cached + + def finish( + local: list[TransferRegion], + remote: list[TransferRegion], + err: str | None, + ) -> tuple[list[TransferRegion], list[TransferRegion], str | None]: + prepared = (local, remote, err) + self._prepared_transfer_regions[cache_key] = prepared + return prepared + + shared_groups: list[tuple[int, ...]] | None = self.region_shared_groups + if len(self.region_shared_groups) != len(self.registered_layer_names): + if _SHARED_REGION_GROUP_ID in self.registered_group_indices: + return finish( + [], + [], + ( + "Mooncake shared-group metadata does not match " + "registered layers, so a shared packed region cannot " + "be transferred." + ), + ) + shared_groups = None + local_rows: list[int] | None = self.region_row_offsets + if len(self.region_row_offsets) != len(self.registered_layer_names): + local_rows = None + local_regions = self._get_transfer_regions( + self.kv_caches_base_addr, + self.block_len_per_layer, + self.kv_block_len_per_layer, + self.registered_layer_names, + self.registered_layer_indices, + self.registered_group_indices, + shared_groups, + local_rows, + ) + remote_shared_groups = [ + tuple(groups) for groups in meta.registered_shared_group_ids + ] + remote_shared: list[tuple[int, ...]] | None = remote_shared_groups + if len(remote_shared_groups) != len(meta.registered_layer_names): + remote_shared = None + remote_rows = meta.registered_row_offsets + remote_row_offsets: list[int] | None = remote_rows + if len(remote_rows) != len(meta.registered_layer_names): + remote_row_offsets = None + remote_regions = self._get_transfer_regions( + meta.kv_caches_base_addr, + meta.block_lens, + meta.kv_block_lens, + meta.registered_layer_names, + meta.registered_layer_indices, + meta.registered_group_indices, + remote_shared, + remote_row_offsets, + ) + pre_align_local = local_regions + pre_align_remote = remote_regions + local_regions, remote_regions, align_err = _align_transfer_regions( + local_regions, + remote_regions, + allow_partial_layers=meta.remote_pp_size != self.pp_size, + ) + if align_err is not None: + return finish([], [], align_err) + # Head checks must see every layer name. Coalesce keeps only the first. + head_err = self._validate_head_resharding_layout( + meta.remote_tp_size, local_regions + ) + if head_err is not None: + return finish([], [], head_err) + # Hetero TP indexes kv_block_len as one head-sharded payload. A merged + # multi-layer span would land that offset on the wrong layer. + # Replicated KV keeps that offset at 0. MLA is replicated in practice; + # gating on use_mla alone would promote and then fail the length check + # for a non-replicated MLA rank with tp_ratio != 1. + tp_ratio = _get_tp_ratio(self.tp_size, meta.remote_tp_size) + if tp_ratio == 1 or self._producer_cache_is_replicated(): + covered_both_sides = len(local_regions) == len(pre_align_local) and len( + remote_regions + ) == len(pre_align_remote) + n_aligned = len(local_regions) + local_regions, remote_regions = _coalesce_contiguous_transfer_regions( + local_regions, + remote_regions, + promote_full_row=covered_both_sides, + ) + logger.info( + "Mooncake packed coalesce: %s aligned regions -> %s transfer regions.", + n_aligned, + len(local_regions), + ) + self._prepared_transfer_regions[cache_key] = ( + local_regions, + remote_regions, + None, + ) + return local_regions, remote_regions, None + def _validate_head_resharding_layout( self, remote_tp_size: int, local_regions: list[TransferRegion] ) -> str | None: @@ -2242,6 +2713,8 @@ class MooncakeConnectorWorker: layer_names: list[str], layer_indices: list[int], group_indices: list[int] | None = None, + shared_group_ids: list[tuple[int, ...]] | None = None, + row_offsets: list[int] | None = None, ) -> list[TransferRegion]: if not group_indices: group_indices = [ @@ -2255,6 +2728,8 @@ class MooncakeConnectorWorker: layer_names=layer_names, layer_indices=layer_indices, group_indices=group_indices, + shared_group_ids=shared_group_ids, + row_offsets=row_offsets, ) def _get_sender_transfer_plan(