diff --git a/tests/v1/kv_connector/unit/test_mooncake_connector.py b/tests/v1/kv_connector/unit/test_mooncake_connector.py index a10ae1f456..d1ae1c6de9 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_connector.py +++ b/tests/v1/kv_connector/unit/test_mooncake_connector.py @@ -4,22 +4,29 @@ import asyncio import contextlib import time +from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch import pytest import torch import zmq.asyncio +from vllm import envs from vllm.config import set_current_vllm_config from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector import ( KVConnectorRole, MooncakeConnector, MooncakeConnectorMetadata, + MooncakeConnectorWorker, MooncakeXferMetadata, MooncakeXferResponse, MooncakeXferResponseStatus, PullReqMeta, SendBlockMeta, + TransferRegion, + _align_transfer_regions, + get_mooncake_bootstrap_addr, + should_launch_bootstrap_server, ) from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_utils import ( MooncakeBootstrapServer, @@ -57,6 +64,272 @@ class FakeMooncakeWrapper: return 0 +def test_align_transfer_regions_uses_layer_name_occurrences(): + """Repeated layer names should align by occurrence order.""" + + local_regions = [ + TransferRegion( + layer_name="model.layers.1.self_attn", + layer_index=1, + base_addr=0x1000, + block_len=256, + kv_block_len=128, + ), + TransferRegion( + layer_name="model.layers.1.self_attn", + layer_index=1, + base_addr=0x1100, + block_len=256, + kv_block_len=128, + ), + ] + remote_regions = [ + TransferRegion( + layer_name="model.layers.0.self_attn", + layer_index=0, + base_addr=0xA000, + block_len=256, + kv_block_len=128, + ), + TransferRegion( + layer_name="model.layers.1.self_attn", + layer_index=1, + base_addr=0xB000, + block_len=256, + kv_block_len=128, + ), + TransferRegion( + layer_name="model.layers.1.self_attn", + layer_index=1, + base_addr=0xB100, + block_len=256, + kv_block_len=128, + ), + ] + + aligned_local, aligned_remote, err = _align_transfer_regions( + local_regions, remote_regions + ) + + assert err is None + assert [r.base_addr for r in aligned_local] == [0x1000, 0x1100] + assert [r.base_addr for r in aligned_remote] == [0xB000, 0xB100] + + +@pytest.mark.asyncio +async def test_build_transfer_params_separates_prefill_pp_layers(): + """Each producer PP stage should send only its registered layer shard.""" + + worker = MooncakeConnectorWorker.__new__(MooncakeConnectorWorker) + worker.async_zmq_ctx = MagicMock() + worker.is_kv_consumer = True + worker.is_kv_producer = True + worker.tp_rank = 0 + worker.tp_size = 1 + worker.transfer_topo = SimpleNamespace(local_replicates_kv_cache=False) + + block_len = 256 + remote_regions = [ + TransferRegion( + layer_name=f"model.layers.{layer_index}.self_attn", + layer_index=layer_index, + base_addr=base_addr, + block_len=block_len, + kv_block_len=block_len, + ) + for layer_index, base_addr in [ + (0, 0xA000), + (1, 0xB000), + (2, 0xC000), + (3, 0xD000), + ] + ] + producer_pp_regions = { + 0: [ + TransferRegion( + layer_name="model.layers.0.self_attn", + layer_index=0, + base_addr=0x1000, + block_len=block_len, + kv_block_len=block_len, + ), + TransferRegion( + layer_name="model.layers.1.self_attn", + layer_index=1, + base_addr=0x2000, + block_len=block_len, + kv_block_len=block_len, + ), + ], + 1: [ + TransferRegion( + layer_name="model.layers.2.self_attn", + layer_index=2, + base_addr=0x3000, + block_len=block_len, + kv_block_len=block_len, + ), + TransferRegion( + layer_name="model.layers.3.self_attn", + layer_index=3, + base_addr=0x4000, + block_len=block_len, + kv_block_len=block_len, + ), + ], + } + expected_by_pp_rank = { + 0: { + "layers": [0, 1], + "src_ptrs": [0x1000 + 10 * block_len, 0x2000 + 10 * block_len], + "dst_ptrs": [0xA000 + 20 * block_len, 0xB000 + 20 * block_len], + }, + 1: { + "layers": [2, 3], + "src_ptrs": [0x3000 + 10 * block_len, 0x4000 + 10 * block_len], + "dst_ptrs": [0xC000 + 20 * block_len, 0xD000 + 20 * block_len], + }, + } + + transfer_id = "xfer-pp-split" + send_meta = SendBlockMeta( + p_req_id="p-req-pp", + transfer_id=transfer_id, + local_block_ids=[[10, 11]], + ready=asyncio.Event(), + ) + xfer_meta = MooncakeXferMetadata( + remote_hostname="consumer-host", + remote_port=54321, + remote_tp_size=1, + remote_tp_rank=0, + req_blocks={"d-req-pp": (transfer_id, [[20, 21]])}, + kv_caches_base_addr=[region.base_addr for region in remote_regions], + block_lens=[region.block_len for region in remote_regions], + registered_layer_names=[region.layer_name for region in remote_regions], + registered_layer_indices=[region.layer_index for region in remote_regions], + ) + + for pp_rank, local_regions in producer_pp_regions.items(): + aligned_local, aligned_remote, err = _align_transfer_regions( + local_regions, remote_regions + ) + + assert err is None + assert [r.layer_index for r in aligned_local] == ( + expected_by_pp_rank[pp_rank]["layers"] + ) + assert [r.layer_index for r in aligned_remote] == ( + expected_by_pp_rank[pp_rank]["layers"] + ) + + ( + src_ptrs, + dst_ptrs, + lengths, + err_reqs, + err_msg, + ) = await worker._build_transfer_params( + ready_reqs=[("d-req-pp", send_meta)], + agent_meta=xfer_meta, + local_regions=aligned_local, + remote_regions=aligned_remote, + ) + + assert err_reqs == [] + assert err_msg is None + assert src_ptrs == expected_by_pp_rank[pp_rank]["src_ptrs"] + assert dst_ptrs == expected_by_pp_rank[pp_rank]["dst_ptrs"] + assert lengths == [2 * block_len, 2 * block_len] + + +@pytest.mark.asyncio +async def test_send_kv_to_decode_aligns_consumer_regions_by_layer_metadata( + monkeypatch, +): + """Producer sends its PP layer shard to the matching consumer layer address.""" + + monkeypatch.setenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "5") + vllm_config = create_vllm_config( + kv_connector="MooncakeConnector", kv_role="kv_producer" + ) + + with set_current_vllm_config(vllm_config), patch_worker_dependencies(): + prefill_connector = MooncakeConnector( + vllm_config, + KVConnectorRole.WORKER, + _make_test_kv_cache_config(), + ) + prefill_worker = prefill_connector.connector_worker + + block_len = 4096 + kv_half = block_len // 2 + prefill_worker.kv_caches_base_addr = [0x1000] + prefill_worker.block_len_per_layer = [block_len] + prefill_worker.registered_layer_names = ["model.layers.1.self_attn"] + prefill_worker.registered_layer_indices = [1] + + class InlineSenderLoop: + async def run_in_executor(self, executor, func, *args): + return func(*args) + + origin_sender_loop = prefill_worker.sender_loop + prefill_worker.sender_loop = InlineSenderLoop() + + transfer_id = "xfer-layer-align" + send_meta = SendBlockMeta( + p_req_id="p-req-layer-align", + transfer_id=transfer_id, + local_block_ids=[[10]], + ready=asyncio.Event(), + ) + prefill_worker.reqs_need_send[transfer_id] = send_meta + send_meta.ready.set() + + xfer_meta = MooncakeXferMetadata( + remote_hostname="consumer-host", + remote_port=54321, + remote_tp_size=1, + remote_tp_rank=0, + req_blocks={"d-req-layer-align": (transfer_id, [[20]])}, + kv_caches_base_addr=[0xA000, 0xB000], + block_lens=[block_len, block_len], + registered_layer_names=[ + "model.layers.0.self_attn", + "model.layers.1.self_attn", + ], + registered_layer_indices=[0, 1], + ) + mock_socket = AsyncMock(spec=zmq.asyncio.Socket) + mock_socket.send_multipart = AsyncMock() + identity = b"consumer-layer-align" + + with patch.object( + prefill_worker, "_send_blocks", return_value=0 + ) as mock_send_blocks: + await prefill_worker.send_kv_to_decode(identity, mock_socket, xfer_meta) + + src_ptrs, dst_ptrs, lengths = mock_send_blocks.call_args[0][1:] + assert src_ptrs == [ + 0x1000 + 10 * block_len, + 0x1000 + 10 * block_len + kv_half, + ] + assert dst_ptrs == [ + 0xB000 + 20 * block_len, + 0xB000 + 20 * block_len + kv_half, + ] + assert lengths == [kv_half, kv_half] + + sent_identity, sent_payload = mock_socket.send_multipart.call_args[0][0] + assert sent_identity == identity + response = prefill_worker._xfer_resp_decoder.decode(sent_payload) + assert response.status == MooncakeXferResponseStatus.FINISH + assert response.ok_reqs == ["d-req-layer-align"] + + prefill_worker.sender_loop = origin_sender_loop + prefill_worker.shutdown() + + def test_basic_interface(): """Unit test for basic MooncakeConnector interface functionality.""" @@ -174,7 +447,7 @@ async def test_bootstrap_server(bootstrap_server: MooncakeBootstrapServer): assert response.status_code == 200 assert response.json() == {} - # Register a worker + # Register multiple PP workers from the same producer engine. payload1 = { "engine_id": "eng-1", "dp_rank": 0, @@ -187,7 +460,19 @@ async def test_bootstrap_server(bootstrap_server: MooncakeBootstrapServer): assert response.status_code == 200 assert response.json() == {"status": "ok"} - # Query after registration + payload2 = { + "engine_id": "eng-1", + "dp_rank": 0, + "tp_rank": 0, + "pp_rank": 1, + "addr": "tcp://2.2.2.2:2222", + } + async with httpx.AsyncClient() as client: + response = await client.post(f"{base_url}/register", json=payload2) + assert response.status_code == 200 + assert response.json() == {"status": "ok"} + + # Query after registration should preserve the PP dimension. async with httpx.AsyncClient() as client: response = await client.get(f"{base_url}/query") assert response.status_code == 200 @@ -195,6 +480,7 @@ async def test_bootstrap_server(bootstrap_server: MooncakeBootstrapServer): assert "0" in data assert data["0"]["engine_id"] == "eng-1" assert data["0"]["worker_addr"]["0"]["0"] == "tcp://1.1.1.1:1111" + assert data["0"]["worker_addr"]["0"]["1"] == "tcp://2.2.2.2:2222" # Test failure: re-registering the same worker async with httpx.AsyncClient() as client: @@ -216,6 +502,104 @@ async def test_bootstrap_server(bootstrap_server: MooncakeBootstrapServer): assert "Engine ID mismatch" in response.text +def _make_bootstrap_vllm_config( + *, + local_engines_only: bool = False, + data_parallel_rank_local: int = 0, + data_parallel_index: int = 0, + nnodes_within_dp: int = 1, +) -> SimpleNamespace: + return SimpleNamespace( + parallel_config=SimpleNamespace( + local_engines_only=local_engines_only, + data_parallel_rank_local=data_parallel_rank_local, + data_parallel_index=data_parallel_index, + nnodes_within_dp=nnodes_within_dp, + master_addr="model-parallel-master", + data_parallel_master_ip="data-parallel-master", + ) + ) + + +@pytest.mark.parametrize( + ( + "tp_rank", + "pp_rank", + "local_engines_only", + "data_parallel_rank_local", + "data_parallel_index", + "expected", + ), + [ + (1, 0, False, 0, 0, False), + (0, 1, False, 0, 0, False), + (0, 0, True, 0, 1, True), + (0, 0, True, 1, 0, False), + (0, 0, False, 0, 0, True), + (0, 0, False, 0, 1, False), + ], + ids=[ + "nonzero_tp_rank", + "nonzero_pp_rank", + "local_engine_rank_zero", + "local_engine_nonzero_rank", + "internal_lb_first_dp_engine", + "internal_lb_nonzero_dp_engine", + ], +) +def test_should_launch_bootstrap_server_selects_single_owner( + tp_rank: int, + pp_rank: int, + local_engines_only: bool, + data_parallel_rank_local: int, + data_parallel_index: int, + expected: bool, +): + vllm_config = _make_bootstrap_vllm_config( + local_engines_only=local_engines_only, + data_parallel_rank_local=data_parallel_rank_local, + data_parallel_index=data_parallel_index, + ) + with ( + patch( + "vllm.distributed.kv_transfer.kv_connector.v1.mooncake." + "mooncake_connector.get_tensor_model_parallel_rank", + return_value=tp_rank, + ), + patch( + "vllm.distributed.kv_transfer.kv_connector.v1.mooncake." + "mooncake_connector.get_pp_group" + ) as mock_pp_group, + ): + mock_pp_group.return_value.rank_in_group = pp_rank + assert should_launch_bootstrap_server(vllm_config) is expected + + +@pytest.mark.parametrize( + ("local_engines_only", "nnodes_within_dp", "expected_host"), + [ + (True, 2, "127.0.0.1"), + (False, 2, "model-parallel-master"), + (False, 1, "data-parallel-master"), + ], + ids=["local_engine", "multi_node_tp_or_pp", "single_node_internal_lb"], +) +def test_get_mooncake_bootstrap_addr_selects_expected_host( + local_engines_only: bool, + nnodes_within_dp: int, + expected_host: str, +): + vllm_config = _make_bootstrap_vllm_config( + local_engines_only=local_engines_only, + nnodes_within_dp=nnodes_within_dp, + ) + + assert get_mooncake_bootstrap_addr(vllm_config) == ( + expected_host, + envs.VLLM_MOONCAKE_BOOTSTRAP_PORT, + ) + + def test_scheduler_request_finished(): """ Tests the scheduler-side logic when a request finishes. @@ -273,7 +657,6 @@ def patch_worker_dependencies(): patch( "vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector.get_pp_group" ) as mock_pp, - patch("vllm.distributed.parallel_state.is_local_first_rank", return_value=True), patch( "vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector.should_launch_bootstrap_server", return_value=False, @@ -307,6 +690,93 @@ def patch_worker_dependencies(): } +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("local_pp_size", "local_pp_rank", "expected_addrs"), + [ + (1, 0, ["tcp://producer-pp0:1234", "tcp://producer-pp1:1234"]), + (2, 1, ["tcp://producer-pp1:1234"]), + ], + ids=["heterogeneous_pp_pulls_all_remote_pp", "matching_pp_pulls_same_rank"], +) +async def test_receive_kv_selects_remote_pp_workers( + local_pp_size: int, + local_pp_rank: int, + expected_addrs: list[str], +): + """Decode workers should not hard-code producer pp_rank 0.""" + + vllm_config = create_vllm_config( + kv_connector="MooncakeConnector", kv_role="kv_consumer" + ) + + with set_current_vllm_config(vllm_config), patch_worker_dependencies(): + decode_connector = MooncakeConnector( + vllm_config, + KVConnectorRole.WORKER, + _make_test_kv_cache_config(), + ) + decode_worker = decode_connector.connector_worker + decode_worker.pp_size = local_pp_size + decode_worker.pp_rank = local_pp_rank + decode_worker._remote_agents = { + "p-engine": { + 0: { + 0: "tcp://producer-pp0:1234", + 1: "tcp://producer-pp1:1234", + } + } + } + decode_worker._tp_size["p-engine"] = 1 + + pull_metas = { + "d-req-1": PullReqMeta( + d_req_id="d-req-1", + transfer_id="xfer-req-1", + local_block_ids=[[100, 101]], + remote_engine_id="p-engine", + remote_bootstrap_addr="http://bootstrap:33333", + ) + } + seen_addrs: list[str] = [] + + async def fake_receive(worker_addr: str, metas: dict[str, PullReqMeta]): + seen_addrs.append(worker_addr) + for meta in metas.values(): + meta.pull_tasks_count -= 1 + + with patch.object( + decode_worker, + "receive_kv_from_single_worker", + side_effect=fake_receive, + ): + decode_worker.receive_kv("p-engine", pull_metas) + await asyncio.sleep(0) + + assert seen_addrs == expected_addrs + assert pull_metas["d-req-1"].pull_tasks_count == 0 + decode_worker.shutdown() + + +def test_resolve_need_send_accounts_for_remote_tp_fanout(): + """Producer-side completion waits for every paired consumer TP pull.""" + + worker = MooncakeConnectorWorker.__new__(MooncakeConnectorWorker) + worker.async_zmq_ctx = MagicMock() + worker.is_kv_consumer = True + worker.is_kv_producer = True + send_meta = SendBlockMeta( + p_req_id="p-req-1", + transfer_id="xfer-req-1", + local_block_ids=[[1]], + ready=asyncio.Event(), + ) + + worker.resolve_need_send(send_meta, remote_tp_ranks=[0, 1]) + + assert send_meta.need_send == 2 + + @pytest.mark.asyncio @patch( "vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector.TransferEngine", @@ -335,6 +805,8 @@ async def test_kv_producer(monkeypatch): prefill_worker.kv_caches_base_addr = [0x1000] block_len = 4096 prefill_worker.block_len_per_layer = [block_len] + prefill_worker.registered_layer_names = ["model.layers.0.self_attn"] + prefill_worker.registered_layer_indices = [0] # Override loop to use current test loop origin_sender_loop = prefill_worker.sender_loop @@ -360,6 +832,8 @@ async def test_kv_producer(monkeypatch): req_blocks={"d-req-1": (transfer_id, [[20, 21]])}, kv_caches_base_addr=[0x2000], block_lens=[block_len], + registered_layer_names=["model.layers.0.self_attn"], + registered_layer_indices=[0], ) mock_socket = AsyncMock(spec=zmq.asyncio.Socket) @@ -506,6 +980,9 @@ async def test_kv_consumuer(monkeypatch): ) decode_worker = decode_connector.connector_worker decode_worker.kv_caches_base_addr = [0x1000] + decode_worker.block_len_per_layer = [4096] + decode_worker.registered_layer_names = ["model.layers.0.self_attn"] + decode_worker.registered_layer_indices = [0] decode_worker.rpc_port = 54321 # A request to pull data arrives. @@ -547,6 +1024,10 @@ async def test_kv_consumuer(monkeypatch): assert sent_meta.remote_hostname == "127.0.0.1" assert sent_meta.remote_port == 54321 assert sent_meta.req_blocks["d-req-1"] == ("xfer-req-1", [[100, 101]]) + assert sent_meta.kv_caches_base_addr == [0x1000] + assert sent_meta.block_lens == [4096] + assert sent_meta.registered_layer_names == ["model.layers.0.self_attn"] + assert sent_meta.registered_layer_indices == [0] # Verify internal state is updated correctly. assert "d-req-1" in decode_worker.finished_recving_reqs @@ -626,7 +1107,10 @@ def test_register_kv_caches(): ) tensor1 = torch.zeros(*kv_cache_shape, dtype=torch.float16) tensor2 = torch.zeros(*kv_cache_shape, dtype=torch.float16) - kv_caches = {"layer0": tensor1, "layer1": tensor2} + kv_caches = { + "model.layers.0.self_attn": tensor1, + "model.layers.1.self_attn": tensor2, + } with patch.object( worker.engine, "batch_register_memory", return_value=0 @@ -643,6 +1127,8 @@ def test_register_kv_caches(): assert len(worker.block_len_per_layer) == len(registered_ptrs) for bl in worker.block_len_per_layer: assert bl == tensor1.nbytes // tensor1.shape[0] + assert worker.registered_layer_names == list(kv_caches) + assert worker.registered_layer_indices == [0, 1] def test_register_kv_caches_supports_mixed_mla_and_eagle_shapes(): @@ -677,7 +1163,10 @@ def test_register_kv_caches_supports_mixed_mla_and_eagle_shapes(): mla_cache = torch.zeros((2, 16, 96), dtype=torch.float16) # Eagle3/GQA-like cache tensor: shape[-2] is num_kv_heads, not block size. eagle_cache = torch.zeros((2, 16, 8, 64), dtype=torch.float16) - kv_caches = {"mla_layer": mla_cache, "eagle_layer": eagle_cache} + kv_caches = { + "model.layers.0.mla_attn": mla_cache, + "model.layers.1.eagle_attn": eagle_cache, + } with patch.object( worker.engine, "batch_register_memory", return_value=0 @@ -692,6 +1181,11 @@ def test_register_kv_caches_supports_mixed_mla_and_eagle_shapes(): mla_cache.nbytes // mla_cache.shape[0], eagle_cache.nbytes // eagle_cache.shape[0], ] + assert worker.registered_layer_names == [ + "model.layers.0.mla_attn", + "model.layers.1.eagle_attn", + ] + assert worker.registered_layer_indices == [0, 1] @pytest.mark.asyncio @@ -742,6 +1236,8 @@ async def test_kv_producer_heterogeneous_tp(monkeypatch, d_tp_size): prefill_worker.kv_caches_base_addr = [0x1000] prefill_worker.block_len_per_layer = [local_block_len] + prefill_worker.registered_layer_names = ["model.layers.0.self_attn"] + prefill_worker.registered_layer_indices = [0] origin_sender_loop = prefill_worker.sender_loop prefill_worker.sender_loop = asyncio.get_event_loop() @@ -787,6 +1283,8 @@ async def test_kv_producer_heterogeneous_tp(monkeypatch, d_tp_size): }, kv_caches_base_addr=[0x2000], block_lens=[remote_block_len], + registered_layer_names=["model.layers.0.self_attn"], + registered_layer_indices=[0], ) mock_send_blocks.reset_mock() diff --git a/tests/v1/kv_connector/unit/test_mooncake_connector_hma.py b/tests/v1/kv_connector/unit/test_mooncake_connector_hma.py index 8e25df7ca8..f45074fff7 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_connector_hma.py +++ b/tests/v1/kv_connector/unit/test_mooncake_connector_hma.py @@ -261,12 +261,20 @@ async def test_build_transfer_params_multi_group_trimming(monkeypatch): local_regions = [ TransferRegion( - base_addr=0x1000, block_len=block_len, kv_block_len=block_len + layer_name="model.layers.0.self_attn", + layer_index=0, + base_addr=0x1000, + block_len=block_len, + kv_block_len=block_len, ), ] remote_regions = [ TransferRegion( - base_addr=0x2000, block_len=block_len, kv_block_len=block_len + layer_name="model.layers.0.self_attn", + layer_index=0, + base_addr=0x2000, + block_len=block_len, + kv_block_len=block_len, ), ] @@ -344,12 +352,20 @@ async def test_build_transfer_params_group_count_mismatch(monkeypatch): local_regions = [ TransferRegion( - base_addr=0x1000, block_len=block_len, kv_block_len=block_len + layer_name="model.layers.0.self_attn", + layer_index=0, + base_addr=0x1000, + block_len=block_len, + kv_block_len=block_len, ), ] remote_regions = [ TransferRegion( - base_addr=0x2000, block_len=block_len, kv_block_len=block_len + layer_name="model.layers.0.self_attn", + layer_index=0, + base_addr=0x2000, + block_len=block_len, + kv_block_len=block_len, ), ] diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py index ccb7257c88..1bc23cead5 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py @@ -43,10 +43,10 @@ from vllm.distributed.parallel_state import ( get_pp_group, get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size, - is_local_first_rank, ) from vllm.forward_context import ForwardContext from vllm.logger import init_logger +from vllm.model_executor.models.utils import extract_layer_index from vllm.platforms import current_platform from vllm.utils.math_utils import cdiv from vllm.utils.network_utils import get_ip, make_zmq_path, make_zmq_socket @@ -80,6 +80,8 @@ TransferId = str # KV transfer coordination ID (shared by P/D) @dataclass(frozen=True) class TransferRegion: + layer_name: str + layer_index: int base_addr: int block_len: int kv_block_len: int @@ -109,18 +111,28 @@ def _get_tp_ratio(local_tp_size: int, remote_tp_size: int) -> int: def _expand_transfer_regions( base_addrs: list[int], block_lens: list[int], + layer_names: list[str], + layer_indices: list[int], is_kv_layout_blocks_first: bool, ) -> list[TransferRegion]: """Expand registered KV tensors into the regions transferred by Mooncake.""" - assert len(base_addrs) == len(block_lens), ( - "Mooncake transfer regions require matching numbers of base addresses " - f"and block lengths, got {len(base_addrs)} and {len(block_lens)}." + assert ( + len(base_addrs) == len(block_lens) == len(layer_names) == len(layer_indices) + ), ( + "Mooncake transfer regions require matching metadata lengths, got " + f"base_addrs={len(base_addrs)}, block_lens={len(block_lens)}, " + f"layer_names={len(layer_names)}, " + f"layer_indices={len(layer_indices)}." ) regions: list[TransferRegion] = [] - for base_addr, block_len in zip(base_addrs, block_lens): + for base_addr, block_len, layer_name, layer_index in zip( + base_addrs, block_lens, layer_names, layer_indices + ): kv_block_len = block_len // 2 if is_kv_layout_blocks_first else block_len regions.append( TransferRegion( + layer_name=layer_name, + layer_index=layer_index, base_addr=base_addr, block_len=block_len, kv_block_len=kv_block_len, @@ -129,6 +141,8 @@ def _expand_transfer_regions( if is_kv_layout_blocks_first: regions.append( TransferRegion( + layer_name=layer_name, + layer_index=layer_index, base_addr=base_addr + kv_block_len, block_len=block_len, kv_block_len=kv_block_len, @@ -244,6 +258,62 @@ def _validate_asymmetric_region_lengths( return None +def _align_transfer_regions( + local_regions: list[TransferRegion], + remote_regions: list[TransferRegion], +) -> tuple[list[TransferRegion], list[TransferRegion], str | None]: + """Align KV transfer regions by registered layer-name occurrence. + + PP shards own different layer subsets. Positional matching is therefore + wrong once producer and consumer have different PP layouts. Multiple + registered transfer buffers for the same layer are represented by repeated + layer names and matched by occurrence order. + """ + + def keyed_regions( + regions: list[TransferRegion], + ) -> list[tuple[tuple[str, int], TransferRegion]]: + counts: dict[str, int] = defaultdict(int) + keyed: list[tuple[tuple[str, int], TransferRegion]] = [] + for region in regions: + occurrence = counts[region.layer_name] + counts[region.layer_name] += 1 + keyed.append(((region.layer_name, occurrence), region)) + return keyed + + local_keyed = keyed_regions(local_regions) + remote_keyed = keyed_regions(remote_regions) + remote_by_key = dict(remote_keyed) + aligned_local: list[TransferRegion] = [] + aligned_remote: list[TransferRegion] = [] + for key, local_region in local_keyed: + remote_region = remote_by_key.get(key) + if remote_region is None: + return ( + [], + [], + ( + "Mooncake producer registered layer has no matching " + f"consumer occurrence: {key[0]} occurrence {key[1]}." + ), + ) + if local_region.layer_index != remote_region.layer_index: + return ( + [], + [], + ( + "Mooncake registered layer index mismatch for " + f"{local_region.layer_name}: producer=" + f"{local_region.layer_index}, consumer=" + f"{remote_region.layer_index}." + ), + ) + aligned_local.append(local_region) + aligned_remote.append(remote_region) + + return aligned_local, aligned_remote, None + + def _get_tensor_dense_flag(tensor: torch.Tensor) -> bool | None: is_dense = getattr(tensor, "is_non_overlapping_and_dense", None) if callable(is_dense): @@ -262,6 +332,8 @@ class MooncakeXferMetadata( req_blocks: dict[ReqId, tuple[TransferId, list[list[int]]]] kv_caches_base_addr: list[int] block_lens: list[int] + registered_layer_names: list[str] = msgspec.field(default_factory=list) + registered_layer_indices: list[int] = msgspec.field(default_factory=list) class MooncakeXferResponseStatus(IntEnum): @@ -782,17 +854,15 @@ class MooncakeConnectorWorker: self.tp_size = get_tensor_model_parallel_world_size() self.num_blocks = 0 self.block_len_per_layer: list[int] = [] + self.registered_layer_names: list[str] = [] + self.registered_layer_indices: list[int] = [] self.seen_base_addresses: list[int] = [] assert (parallel_config := vllm_config.parallel_config) dp_rank = parallel_config.data_parallel_index dp_local_rank = parallel_config.data_parallel_rank_local self.dp_rank = dp_local_rank if parallel_config.local_engines_only else dp_rank - pp_size = vllm_config.parallel_config.pipeline_parallel_size - if pp_size > 1: - raise ValueError( - "Mooncake Transfer Engine does not support pipeline parallelism yet." - ) + self.pp_size = vllm_config.parallel_config.pipeline_parallel_size self.pp_rank = get_pp_group().rank_in_group self.kv_caches_base_addr: list[int] = [] @@ -1020,11 +1090,27 @@ class MooncakeConnectorWorker: await sock.send_multipart((identity, self._encoder.encode(response))) return local_regions = self._get_transfer_regions( - self.kv_caches_base_addr, self.block_len_per_layer + self.kv_caches_base_addr, + self.block_len_per_layer, + self.registered_layer_names, + self.registered_layer_indices, ) remote_regions = self._get_transfer_regions( - meta.kv_caches_base_addr, meta.block_lens + meta.kv_caches_base_addr, + meta.block_lens, + meta.registered_layer_names, + meta.registered_layer_indices, ) + local_regions, remote_regions, align_err = _align_transfer_regions( + local_regions, remote_regions + ) + if align_err is not None: + response = MooncakeXferResponse( + status=MooncakeXferResponseStatus.ERROR, + err_msg=align_err, + ) + await sock.send_multipart((identity, self._encoder.encode(response))) + return validation_err = _validate_asymmetric_region_lengths( local_regions=local_regions, remote_regions=remote_regions, @@ -1171,11 +1257,15 @@ class MooncakeConnectorWorker: ) await sock.send_multipart((identity, self._encoder.encode(response))) - def resolve_need_send(self, send_meta: SendBlockMeta, remote_tp_ranks: list[int]): + def resolve_need_send( + self, + send_meta: SendBlockMeta, + remote_tp_ranks: list[int], + ): # Prepare for heterogeneous TP (one P pairs to multiple D) send_meta.need_send = len(remote_tp_ranks) logger.debug( - "Mooncake request %s will be served by %d consumer TP workers: %s", + "Mooncake request %s will be served by %d consumer TP workers: TP ranks=%s", send_meta.transfer_id, send_meta.need_send, remote_tp_ranks, @@ -1394,10 +1484,13 @@ class MooncakeConnectorWorker: kv_data_lens = [] seen_base_addresses = [] self.block_len_per_layer = [] + self.registered_layer_names = [] + self.registered_layer_indices = [] split_k_and_v = self.transfer_topo.split_k_and_v tensor_size_bytes = None for layer_name, cache_or_caches in kv_caches.items(): + layer_index = extract_layer_index(layer_name) cache_list = cache_or_caches if split_k_and_v else [cache_or_caches] logger.debug( "registering layer %s with %d cache tensor(s)", @@ -1428,6 +1521,8 @@ class MooncakeConnectorWorker: block_len = cache.stride(0) * cache.element_size() self.block_len_per_layer.append(block_len) + self.registered_layer_names.append(layer_name) + self.registered_layer_indices.append(layer_index) kv_data_ptrs.append(base_addr) kv_data_lens.append(self.num_blocks * block_len) @@ -1547,6 +1642,8 @@ class MooncakeConnectorWorker: }, kv_caches_base_addr=self.kv_caches_base_addr, block_lens=self.block_len_per_layer, + registered_layer_names=self.registered_layer_names, + registered_layer_indices=self.registered_layer_indices, ) encoded_data = self._encoder.encode(metadata) @@ -1648,16 +1745,28 @@ class MooncakeConnectorWorker: remote_tp_ranks = self.transfer_topo.handshake_target_ranks( self._tp_size[remote_engine_id] ) - count = len(remote_tp_ranks) + worker_addrs: list[str] = [] + selected_remote_pp: dict[int, list[int]] = {} + for remote_tp_rank in remote_tp_ranks: + pp_to_addr = self._remote_agents[remote_engine_id][remote_tp_rank] + if self.pp_size == len(pp_to_addr) and self.pp_rank in pp_to_addr: + pp_ranks = [self.pp_rank] + else: + pp_ranks = sorted(pp_to_addr) + selected_remote_pp[remote_tp_rank] = pp_ranks + worker_addrs.extend(pp_to_addr[pp_rank] for pp_rank in pp_ranks) + + count = len(worker_addrs) logger.debug( - "Receiving Mooncake KV for engine %s from producer TP ranks %s", + "Receiving Mooncake KV for engine %s from producer TP ranks %s " + "and PP ranks %s", remote_engine_id, remote_tp_ranks, + selected_remote_pp, ) for pull_meta in pull_metas.values(): pull_meta.pull_tasks_count = count - for remote_tp_rank in remote_tp_ranks: - worker_addr = self._remote_agents[remote_engine_id][remote_tp_rank][0] + for worker_addr in worker_addrs: asyncio.create_task( self.receive_kv_from_single_worker(worker_addr, pull_metas) ) @@ -1740,11 +1849,17 @@ class MooncakeConnectorWorker: return self.transfer_topo.local_replicates_kv_cache def _get_transfer_regions( - self, base_addrs: list[int], block_lens: list[int] + self, + base_addrs: list[int], + block_lens: list[int], + layer_names: list[str], + layer_indices: list[int], ) -> list[TransferRegion]: return _expand_transfer_regions( base_addrs=base_addrs, block_lens=block_lens, + layer_names=layer_names, + layer_indices=layer_indices, is_kv_layout_blocks_first=self.transfer_topo.virtually_split_kv_in_blocks, ) @@ -1816,14 +1931,20 @@ def _async_loop(loop: asyncio.AbstractEventLoop): def should_launch_bootstrap_server(vllm_config: VllmConfig) -> bool: assert (parallel_config := vllm_config.parallel_config) + # Only the TP=0, PP=0 worker of the designated engine should launch it. + if get_tensor_model_parallel_rank() != 0: + return False + if get_pp_group().rank_in_group != 0: + return False + # In hybrid or external LB mode, # each instance should have its own bootstrap server. - # + if parallel_config.local_engines_only: + return parallel_config.data_parallel_rank_local == 0 + # In internal LB mode, - # only the real global first rank need to launch the bootstrap server. - return is_local_first_rank() and ( - parallel_config.local_engines_only or parallel_config.data_parallel_index == 0 - ) + # only the first data-parallel engine should launch the bootstrap server. + return parallel_config.data_parallel_index == 0 def get_mooncake_bootstrap_addr(vllm_config: VllmConfig) -> tuple[str, int]: @@ -1836,6 +1957,10 @@ def get_mooncake_bootstrap_addr(vllm_config: VllmConfig) -> tuple[str, int]: if parallel_config.local_engines_only: # In hybrid or external LB mode, connect to local server. host = "127.0.0.1" + elif parallel_config.nnodes_within_dp > 1: + # Internal LB multi-node TP/PP uses the model-parallel master as the + # single bootstrap endpoint for all ranks in the engine. + host = parallel_config.master_addr else: host = parallel_config.data_parallel_master_ip port = envs.VLLM_MOONCAKE_BOOTSTRAP_PORT diff --git a/vllm/distributed/kv_transfer/kv_transfer_state.py b/vllm/distributed/kv_transfer/kv_transfer_state.py index 67a6b4ca7a..f9209dc3e4 100644 --- a/vllm/distributed/kv_transfer/kv_transfer_state.py +++ b/vllm/distributed/kv_transfer/kv_transfer_state.py @@ -50,8 +50,13 @@ def is_v1_kv_transfer_group(connector: KVConnectorBaseType | None = None) -> boo def _sync_engine_id_across_tp(vllm_config: "VllmConfig") -> None: """Broadcast engine_id from TP rank 0 so all workers in a - multi-node TP group share the same value.""" + multi-node TP group share the same value. + + When PP is enabled, also broadcast across PP ranks so all workers in + the same model-parallel engine share the same value. + """ from vllm.distributed.parallel_state import ( + get_pp_group, get_tp_group, ) @@ -59,6 +64,8 @@ def _sync_engine_id_across_tp(vllm_config: "VllmConfig") -> None: synced_id = get_tp_group().broadcast_object( vllm_config.kv_transfer_config.engine_id, src=0 ) + if vllm_config.parallel_config.pipeline_parallel_size > 1: + synced_id = get_pp_group().broadcast_object(synced_id, src=0) vllm_config.kv_transfer_config.engine_id = synced_id