[KV Connector][Mooncake] Pipeline-parallel support for PD-disaggregated serving with Mooncake connector (#44528)

Signed-off-by: hanhan.hank <hanhan.hank@bytedance.com>
Signed-off-by: Hank Han <hanhan7630@outlook.com>
This commit is contained in:
Hank Han
2026-06-16 04:35:38 -07:00
committed by GitHub
parent ad32608e24
commit d53f4593ce
4 changed files with 680 additions and 34 deletions
@@ -4,22 +4,29 @@
import asyncio
import contextlib
import time
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
import torch
import zmq.asyncio
from vllm import envs
from vllm.config import set_current_vllm_config
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector import (
KVConnectorRole,
MooncakeConnector,
MooncakeConnectorMetadata,
MooncakeConnectorWorker,
MooncakeXferMetadata,
MooncakeXferResponse,
MooncakeXferResponseStatus,
PullReqMeta,
SendBlockMeta,
TransferRegion,
_align_transfer_regions,
get_mooncake_bootstrap_addr,
should_launch_bootstrap_server,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_utils import (
MooncakeBootstrapServer,
@@ -57,6 +64,272 @@ class FakeMooncakeWrapper:
return 0
def test_align_transfer_regions_uses_layer_name_occurrences():
"""Repeated layer names should align by occurrence order."""
local_regions = [
TransferRegion(
layer_name="model.layers.1.self_attn",
layer_index=1,
base_addr=0x1000,
block_len=256,
kv_block_len=128,
),
TransferRegion(
layer_name="model.layers.1.self_attn",
layer_index=1,
base_addr=0x1100,
block_len=256,
kv_block_len=128,
),
]
remote_regions = [
TransferRegion(
layer_name="model.layers.0.self_attn",
layer_index=0,
base_addr=0xA000,
block_len=256,
kv_block_len=128,
),
TransferRegion(
layer_name="model.layers.1.self_attn",
layer_index=1,
base_addr=0xB000,
block_len=256,
kv_block_len=128,
),
TransferRegion(
layer_name="model.layers.1.self_attn",
layer_index=1,
base_addr=0xB100,
block_len=256,
kv_block_len=128,
),
]
aligned_local, aligned_remote, err = _align_transfer_regions(
local_regions, remote_regions
)
assert err is None
assert [r.base_addr for r in aligned_local] == [0x1000, 0x1100]
assert [r.base_addr for r in aligned_remote] == [0xB000, 0xB100]
@pytest.mark.asyncio
async def test_build_transfer_params_separates_prefill_pp_layers():
"""Each producer PP stage should send only its registered layer shard."""
worker = MooncakeConnectorWorker.__new__(MooncakeConnectorWorker)
worker.async_zmq_ctx = MagicMock()
worker.is_kv_consumer = True
worker.is_kv_producer = True
worker.tp_rank = 0
worker.tp_size = 1
worker.transfer_topo = SimpleNamespace(local_replicates_kv_cache=False)
block_len = 256
remote_regions = [
TransferRegion(
layer_name=f"model.layers.{layer_index}.self_attn",
layer_index=layer_index,
base_addr=base_addr,
block_len=block_len,
kv_block_len=block_len,
)
for layer_index, base_addr in [
(0, 0xA000),
(1, 0xB000),
(2, 0xC000),
(3, 0xD000),
]
]
producer_pp_regions = {
0: [
TransferRegion(
layer_name="model.layers.0.self_attn",
layer_index=0,
base_addr=0x1000,
block_len=block_len,
kv_block_len=block_len,
),
TransferRegion(
layer_name="model.layers.1.self_attn",
layer_index=1,
base_addr=0x2000,
block_len=block_len,
kv_block_len=block_len,
),
],
1: [
TransferRegion(
layer_name="model.layers.2.self_attn",
layer_index=2,
base_addr=0x3000,
block_len=block_len,
kv_block_len=block_len,
),
TransferRegion(
layer_name="model.layers.3.self_attn",
layer_index=3,
base_addr=0x4000,
block_len=block_len,
kv_block_len=block_len,
),
],
}
expected_by_pp_rank = {
0: {
"layers": [0, 1],
"src_ptrs": [0x1000 + 10 * block_len, 0x2000 + 10 * block_len],
"dst_ptrs": [0xA000 + 20 * block_len, 0xB000 + 20 * block_len],
},
1: {
"layers": [2, 3],
"src_ptrs": [0x3000 + 10 * block_len, 0x4000 + 10 * block_len],
"dst_ptrs": [0xC000 + 20 * block_len, 0xD000 + 20 * block_len],
},
}
transfer_id = "xfer-pp-split"
send_meta = SendBlockMeta(
p_req_id="p-req-pp",
transfer_id=transfer_id,
local_block_ids=[[10, 11]],
ready=asyncio.Event(),
)
xfer_meta = MooncakeXferMetadata(
remote_hostname="consumer-host",
remote_port=54321,
remote_tp_size=1,
remote_tp_rank=0,
req_blocks={"d-req-pp": (transfer_id, [[20, 21]])},
kv_caches_base_addr=[region.base_addr for region in remote_regions],
block_lens=[region.block_len for region in remote_regions],
registered_layer_names=[region.layer_name for region in remote_regions],
registered_layer_indices=[region.layer_index for region in remote_regions],
)
for pp_rank, local_regions in producer_pp_regions.items():
aligned_local, aligned_remote, err = _align_transfer_regions(
local_regions, remote_regions
)
assert err is None
assert [r.layer_index for r in aligned_local] == (
expected_by_pp_rank[pp_rank]["layers"]
)
assert [r.layer_index for r in aligned_remote] == (
expected_by_pp_rank[pp_rank]["layers"]
)
(
src_ptrs,
dst_ptrs,
lengths,
err_reqs,
err_msg,
) = await worker._build_transfer_params(
ready_reqs=[("d-req-pp", send_meta)],
agent_meta=xfer_meta,
local_regions=aligned_local,
remote_regions=aligned_remote,
)
assert err_reqs == []
assert err_msg is None
assert src_ptrs == expected_by_pp_rank[pp_rank]["src_ptrs"]
assert dst_ptrs == expected_by_pp_rank[pp_rank]["dst_ptrs"]
assert lengths == [2 * block_len, 2 * block_len]
@pytest.mark.asyncio
async def test_send_kv_to_decode_aligns_consumer_regions_by_layer_metadata(
monkeypatch,
):
"""Producer sends its PP layer shard to the matching consumer layer address."""
monkeypatch.setenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "5")
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector", kv_role="kv_producer"
)
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
prefill_connector = MooncakeConnector(
vllm_config,
KVConnectorRole.WORKER,
_make_test_kv_cache_config(),
)
prefill_worker = prefill_connector.connector_worker
block_len = 4096
kv_half = block_len // 2
prefill_worker.kv_caches_base_addr = [0x1000]
prefill_worker.block_len_per_layer = [block_len]
prefill_worker.registered_layer_names = ["model.layers.1.self_attn"]
prefill_worker.registered_layer_indices = [1]
class InlineSenderLoop:
async def run_in_executor(self, executor, func, *args):
return func(*args)
origin_sender_loop = prefill_worker.sender_loop
prefill_worker.sender_loop = InlineSenderLoop()
transfer_id = "xfer-layer-align"
send_meta = SendBlockMeta(
p_req_id="p-req-layer-align",
transfer_id=transfer_id,
local_block_ids=[[10]],
ready=asyncio.Event(),
)
prefill_worker.reqs_need_send[transfer_id] = send_meta
send_meta.ready.set()
xfer_meta = MooncakeXferMetadata(
remote_hostname="consumer-host",
remote_port=54321,
remote_tp_size=1,
remote_tp_rank=0,
req_blocks={"d-req-layer-align": (transfer_id, [[20]])},
kv_caches_base_addr=[0xA000, 0xB000],
block_lens=[block_len, block_len],
registered_layer_names=[
"model.layers.0.self_attn",
"model.layers.1.self_attn",
],
registered_layer_indices=[0, 1],
)
mock_socket = AsyncMock(spec=zmq.asyncio.Socket)
mock_socket.send_multipart = AsyncMock()
identity = b"consumer-layer-align"
with patch.object(
prefill_worker, "_send_blocks", return_value=0
) as mock_send_blocks:
await prefill_worker.send_kv_to_decode(identity, mock_socket, xfer_meta)
src_ptrs, dst_ptrs, lengths = mock_send_blocks.call_args[0][1:]
assert src_ptrs == [
0x1000 + 10 * block_len,
0x1000 + 10 * block_len + kv_half,
]
assert dst_ptrs == [
0xB000 + 20 * block_len,
0xB000 + 20 * block_len + kv_half,
]
assert lengths == [kv_half, kv_half]
sent_identity, sent_payload = mock_socket.send_multipart.call_args[0][0]
assert sent_identity == identity
response = prefill_worker._xfer_resp_decoder.decode(sent_payload)
assert response.status == MooncakeXferResponseStatus.FINISH
assert response.ok_reqs == ["d-req-layer-align"]
prefill_worker.sender_loop = origin_sender_loop
prefill_worker.shutdown()
def test_basic_interface():
"""Unit test for basic MooncakeConnector interface functionality."""
@@ -174,7 +447,7 @@ async def test_bootstrap_server(bootstrap_server: MooncakeBootstrapServer):
assert response.status_code == 200
assert response.json() == {}
# Register a worker
# Register multiple PP workers from the same producer engine.
payload1 = {
"engine_id": "eng-1",
"dp_rank": 0,
@@ -187,7 +460,19 @@ async def test_bootstrap_server(bootstrap_server: MooncakeBootstrapServer):
assert response.status_code == 200
assert response.json() == {"status": "ok"}
# Query after registration
payload2 = {
"engine_id": "eng-1",
"dp_rank": 0,
"tp_rank": 0,
"pp_rank": 1,
"addr": "tcp://2.2.2.2:2222",
}
async with httpx.AsyncClient() as client:
response = await client.post(f"{base_url}/register", json=payload2)
assert response.status_code == 200
assert response.json() == {"status": "ok"}
# Query after registration should preserve the PP dimension.
async with httpx.AsyncClient() as client:
response = await client.get(f"{base_url}/query")
assert response.status_code == 200
@@ -195,6 +480,7 @@ async def test_bootstrap_server(bootstrap_server: MooncakeBootstrapServer):
assert "0" in data
assert data["0"]["engine_id"] == "eng-1"
assert data["0"]["worker_addr"]["0"]["0"] == "tcp://1.1.1.1:1111"
assert data["0"]["worker_addr"]["0"]["1"] == "tcp://2.2.2.2:2222"
# Test failure: re-registering the same worker
async with httpx.AsyncClient() as client:
@@ -216,6 +502,104 @@ async def test_bootstrap_server(bootstrap_server: MooncakeBootstrapServer):
assert "Engine ID mismatch" in response.text
def _make_bootstrap_vllm_config(
*,
local_engines_only: bool = False,
data_parallel_rank_local: int = 0,
data_parallel_index: int = 0,
nnodes_within_dp: int = 1,
) -> SimpleNamespace:
return SimpleNamespace(
parallel_config=SimpleNamespace(
local_engines_only=local_engines_only,
data_parallel_rank_local=data_parallel_rank_local,
data_parallel_index=data_parallel_index,
nnodes_within_dp=nnodes_within_dp,
master_addr="model-parallel-master",
data_parallel_master_ip="data-parallel-master",
)
)
@pytest.mark.parametrize(
(
"tp_rank",
"pp_rank",
"local_engines_only",
"data_parallel_rank_local",
"data_parallel_index",
"expected",
),
[
(1, 0, False, 0, 0, False),
(0, 1, False, 0, 0, False),
(0, 0, True, 0, 1, True),
(0, 0, True, 1, 0, False),
(0, 0, False, 0, 0, True),
(0, 0, False, 0, 1, False),
],
ids=[
"nonzero_tp_rank",
"nonzero_pp_rank",
"local_engine_rank_zero",
"local_engine_nonzero_rank",
"internal_lb_first_dp_engine",
"internal_lb_nonzero_dp_engine",
],
)
def test_should_launch_bootstrap_server_selects_single_owner(
tp_rank: int,
pp_rank: int,
local_engines_only: bool,
data_parallel_rank_local: int,
data_parallel_index: int,
expected: bool,
):
vllm_config = _make_bootstrap_vllm_config(
local_engines_only=local_engines_only,
data_parallel_rank_local=data_parallel_rank_local,
data_parallel_index=data_parallel_index,
)
with (
patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake."
"mooncake_connector.get_tensor_model_parallel_rank",
return_value=tp_rank,
),
patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake."
"mooncake_connector.get_pp_group"
) as mock_pp_group,
):
mock_pp_group.return_value.rank_in_group = pp_rank
assert should_launch_bootstrap_server(vllm_config) is expected
@pytest.mark.parametrize(
("local_engines_only", "nnodes_within_dp", "expected_host"),
[
(True, 2, "127.0.0.1"),
(False, 2, "model-parallel-master"),
(False, 1, "data-parallel-master"),
],
ids=["local_engine", "multi_node_tp_or_pp", "single_node_internal_lb"],
)
def test_get_mooncake_bootstrap_addr_selects_expected_host(
local_engines_only: bool,
nnodes_within_dp: int,
expected_host: str,
):
vllm_config = _make_bootstrap_vllm_config(
local_engines_only=local_engines_only,
nnodes_within_dp=nnodes_within_dp,
)
assert get_mooncake_bootstrap_addr(vllm_config) == (
expected_host,
envs.VLLM_MOONCAKE_BOOTSTRAP_PORT,
)
def test_scheduler_request_finished():
"""
Tests the scheduler-side logic when a request finishes.
@@ -273,7 +657,6 @@ def patch_worker_dependencies():
patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector.get_pp_group"
) as mock_pp,
patch("vllm.distributed.parallel_state.is_local_first_rank", return_value=True),
patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector.should_launch_bootstrap_server",
return_value=False,
@@ -307,6 +690,93 @@ def patch_worker_dependencies():
}
@pytest.mark.asyncio
@pytest.mark.parametrize(
("local_pp_size", "local_pp_rank", "expected_addrs"),
[
(1, 0, ["tcp://producer-pp0:1234", "tcp://producer-pp1:1234"]),
(2, 1, ["tcp://producer-pp1:1234"]),
],
ids=["heterogeneous_pp_pulls_all_remote_pp", "matching_pp_pulls_same_rank"],
)
async def test_receive_kv_selects_remote_pp_workers(
local_pp_size: int,
local_pp_rank: int,
expected_addrs: list[str],
):
"""Decode workers should not hard-code producer pp_rank 0."""
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector", kv_role="kv_consumer"
)
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
decode_connector = MooncakeConnector(
vllm_config,
KVConnectorRole.WORKER,
_make_test_kv_cache_config(),
)
decode_worker = decode_connector.connector_worker
decode_worker.pp_size = local_pp_size
decode_worker.pp_rank = local_pp_rank
decode_worker._remote_agents = {
"p-engine": {
0: {
0: "tcp://producer-pp0:1234",
1: "tcp://producer-pp1:1234",
}
}
}
decode_worker._tp_size["p-engine"] = 1
pull_metas = {
"d-req-1": PullReqMeta(
d_req_id="d-req-1",
transfer_id="xfer-req-1",
local_block_ids=[[100, 101]],
remote_engine_id="p-engine",
remote_bootstrap_addr="http://bootstrap:33333",
)
}
seen_addrs: list[str] = []
async def fake_receive(worker_addr: str, metas: dict[str, PullReqMeta]):
seen_addrs.append(worker_addr)
for meta in metas.values():
meta.pull_tasks_count -= 1
with patch.object(
decode_worker,
"receive_kv_from_single_worker",
side_effect=fake_receive,
):
decode_worker.receive_kv("p-engine", pull_metas)
await asyncio.sleep(0)
assert seen_addrs == expected_addrs
assert pull_metas["d-req-1"].pull_tasks_count == 0
decode_worker.shutdown()
def test_resolve_need_send_accounts_for_remote_tp_fanout():
"""Producer-side completion waits for every paired consumer TP pull."""
worker = MooncakeConnectorWorker.__new__(MooncakeConnectorWorker)
worker.async_zmq_ctx = MagicMock()
worker.is_kv_consumer = True
worker.is_kv_producer = True
send_meta = SendBlockMeta(
p_req_id="p-req-1",
transfer_id="xfer-req-1",
local_block_ids=[[1]],
ready=asyncio.Event(),
)
worker.resolve_need_send(send_meta, remote_tp_ranks=[0, 1])
assert send_meta.need_send == 2
@pytest.mark.asyncio
@patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector.TransferEngine",
@@ -335,6 +805,8 @@ async def test_kv_producer(monkeypatch):
prefill_worker.kv_caches_base_addr = [0x1000]
block_len = 4096
prefill_worker.block_len_per_layer = [block_len]
prefill_worker.registered_layer_names = ["model.layers.0.self_attn"]
prefill_worker.registered_layer_indices = [0]
# Override loop to use current test loop
origin_sender_loop = prefill_worker.sender_loop
@@ -360,6 +832,8 @@ async def test_kv_producer(monkeypatch):
req_blocks={"d-req-1": (transfer_id, [[20, 21]])},
kv_caches_base_addr=[0x2000],
block_lens=[block_len],
registered_layer_names=["model.layers.0.self_attn"],
registered_layer_indices=[0],
)
mock_socket = AsyncMock(spec=zmq.asyncio.Socket)
@@ -506,6 +980,9 @@ async def test_kv_consumuer(monkeypatch):
)
decode_worker = decode_connector.connector_worker
decode_worker.kv_caches_base_addr = [0x1000]
decode_worker.block_len_per_layer = [4096]
decode_worker.registered_layer_names = ["model.layers.0.self_attn"]
decode_worker.registered_layer_indices = [0]
decode_worker.rpc_port = 54321
# A request to pull data arrives.
@@ -547,6 +1024,10 @@ async def test_kv_consumuer(monkeypatch):
assert sent_meta.remote_hostname == "127.0.0.1"
assert sent_meta.remote_port == 54321
assert sent_meta.req_blocks["d-req-1"] == ("xfer-req-1", [[100, 101]])
assert sent_meta.kv_caches_base_addr == [0x1000]
assert sent_meta.block_lens == [4096]
assert sent_meta.registered_layer_names == ["model.layers.0.self_attn"]
assert sent_meta.registered_layer_indices == [0]
# Verify internal state is updated correctly.
assert "d-req-1" in decode_worker.finished_recving_reqs
@@ -626,7 +1107,10 @@ def test_register_kv_caches():
)
tensor1 = torch.zeros(*kv_cache_shape, dtype=torch.float16)
tensor2 = torch.zeros(*kv_cache_shape, dtype=torch.float16)
kv_caches = {"layer0": tensor1, "layer1": tensor2}
kv_caches = {
"model.layers.0.self_attn": tensor1,
"model.layers.1.self_attn": tensor2,
}
with patch.object(
worker.engine, "batch_register_memory", return_value=0
@@ -643,6 +1127,8 @@ def test_register_kv_caches():
assert len(worker.block_len_per_layer) == len(registered_ptrs)
for bl in worker.block_len_per_layer:
assert bl == tensor1.nbytes // tensor1.shape[0]
assert worker.registered_layer_names == list(kv_caches)
assert worker.registered_layer_indices == [0, 1]
def test_register_kv_caches_supports_mixed_mla_and_eagle_shapes():
@@ -677,7 +1163,10 @@ def test_register_kv_caches_supports_mixed_mla_and_eagle_shapes():
mla_cache = torch.zeros((2, 16, 96), dtype=torch.float16)
# Eagle3/GQA-like cache tensor: shape[-2] is num_kv_heads, not block size.
eagle_cache = torch.zeros((2, 16, 8, 64), dtype=torch.float16)
kv_caches = {"mla_layer": mla_cache, "eagle_layer": eagle_cache}
kv_caches = {
"model.layers.0.mla_attn": mla_cache,
"model.layers.1.eagle_attn": eagle_cache,
}
with patch.object(
worker.engine, "batch_register_memory", return_value=0
@@ -692,6 +1181,11 @@ def test_register_kv_caches_supports_mixed_mla_and_eagle_shapes():
mla_cache.nbytes // mla_cache.shape[0],
eagle_cache.nbytes // eagle_cache.shape[0],
]
assert worker.registered_layer_names == [
"model.layers.0.mla_attn",
"model.layers.1.eagle_attn",
]
assert worker.registered_layer_indices == [0, 1]
@pytest.mark.asyncio
@@ -742,6 +1236,8 @@ async def test_kv_producer_heterogeneous_tp(monkeypatch, d_tp_size):
prefill_worker.kv_caches_base_addr = [0x1000]
prefill_worker.block_len_per_layer = [local_block_len]
prefill_worker.registered_layer_names = ["model.layers.0.self_attn"]
prefill_worker.registered_layer_indices = [0]
origin_sender_loop = prefill_worker.sender_loop
prefill_worker.sender_loop = asyncio.get_event_loop()
@@ -787,6 +1283,8 @@ async def test_kv_producer_heterogeneous_tp(monkeypatch, d_tp_size):
},
kv_caches_base_addr=[0x2000],
block_lens=[remote_block_len],
registered_layer_names=["model.layers.0.self_attn"],
registered_layer_indices=[0],
)
mock_send_blocks.reset_mock()
@@ -261,12 +261,20 @@ async def test_build_transfer_params_multi_group_trimming(monkeypatch):
local_regions = [
TransferRegion(
base_addr=0x1000, block_len=block_len, kv_block_len=block_len
layer_name="model.layers.0.self_attn",
layer_index=0,
base_addr=0x1000,
block_len=block_len,
kv_block_len=block_len,
),
]
remote_regions = [
TransferRegion(
base_addr=0x2000, block_len=block_len, kv_block_len=block_len
layer_name="model.layers.0.self_attn",
layer_index=0,
base_addr=0x2000,
block_len=block_len,
kv_block_len=block_len,
),
]
@@ -344,12 +352,20 @@ async def test_build_transfer_params_group_count_mismatch(monkeypatch):
local_regions = [
TransferRegion(
base_addr=0x1000, block_len=block_len, kv_block_len=block_len
layer_name="model.layers.0.self_attn",
layer_index=0,
base_addr=0x1000,
block_len=block_len,
kv_block_len=block_len,
),
]
remote_regions = [
TransferRegion(
base_addr=0x2000, block_len=block_len, kv_block_len=block_len
layer_name="model.layers.0.self_attn",
layer_index=0,
base_addr=0x2000,
block_len=block_len,
kv_block_len=block_len,
),
]
@@ -43,10 +43,10 @@ from vllm.distributed.parallel_state import (
get_pp_group,
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
is_local_first_rank,
)
from vllm.forward_context import ForwardContext
from vllm.logger import init_logger
from vllm.model_executor.models.utils import extract_layer_index
from vllm.platforms import current_platform
from vllm.utils.math_utils import cdiv
from vllm.utils.network_utils import get_ip, make_zmq_path, make_zmq_socket
@@ -80,6 +80,8 @@ TransferId = str # KV transfer coordination ID (shared by P/D)
@dataclass(frozen=True)
class TransferRegion:
layer_name: str
layer_index: int
base_addr: int
block_len: int
kv_block_len: int
@@ -109,18 +111,28 @@ def _get_tp_ratio(local_tp_size: int, remote_tp_size: int) -> int:
def _expand_transfer_regions(
base_addrs: list[int],
block_lens: list[int],
layer_names: list[str],
layer_indices: list[int],
is_kv_layout_blocks_first: bool,
) -> list[TransferRegion]:
"""Expand registered KV tensors into the regions transferred by Mooncake."""
assert len(base_addrs) == len(block_lens), (
"Mooncake transfer regions require matching numbers of base addresses "
f"and block lengths, got {len(base_addrs)} and {len(block_lens)}."
assert (
len(base_addrs) == len(block_lens) == len(layer_names) == len(layer_indices)
), (
"Mooncake transfer regions require matching metadata lengths, got "
f"base_addrs={len(base_addrs)}, block_lens={len(block_lens)}, "
f"layer_names={len(layer_names)}, "
f"layer_indices={len(layer_indices)}."
)
regions: list[TransferRegion] = []
for base_addr, block_len in zip(base_addrs, block_lens):
for base_addr, block_len, layer_name, layer_index in zip(
base_addrs, block_lens, layer_names, layer_indices
):
kv_block_len = block_len // 2 if is_kv_layout_blocks_first else block_len
regions.append(
TransferRegion(
layer_name=layer_name,
layer_index=layer_index,
base_addr=base_addr,
block_len=block_len,
kv_block_len=kv_block_len,
@@ -129,6 +141,8 @@ def _expand_transfer_regions(
if is_kv_layout_blocks_first:
regions.append(
TransferRegion(
layer_name=layer_name,
layer_index=layer_index,
base_addr=base_addr + kv_block_len,
block_len=block_len,
kv_block_len=kv_block_len,
@@ -244,6 +258,62 @@ def _validate_asymmetric_region_lengths(
return None
def _align_transfer_regions(
local_regions: list[TransferRegion],
remote_regions: list[TransferRegion],
) -> tuple[list[TransferRegion], list[TransferRegion], str | None]:
"""Align KV transfer regions by registered layer-name occurrence.
PP shards own different layer subsets. Positional matching is therefore
wrong once producer and consumer have different PP layouts. Multiple
registered transfer buffers for the same layer are represented by repeated
layer names and matched by occurrence order.
"""
def keyed_regions(
regions: list[TransferRegion],
) -> list[tuple[tuple[str, int], TransferRegion]]:
counts: dict[str, int] = defaultdict(int)
keyed: list[tuple[tuple[str, int], TransferRegion]] = []
for region in regions:
occurrence = counts[region.layer_name]
counts[region.layer_name] += 1
keyed.append(((region.layer_name, occurrence), region))
return keyed
local_keyed = keyed_regions(local_regions)
remote_keyed = keyed_regions(remote_regions)
remote_by_key = dict(remote_keyed)
aligned_local: list[TransferRegion] = []
aligned_remote: list[TransferRegion] = []
for key, local_region in local_keyed:
remote_region = remote_by_key.get(key)
if remote_region is None:
return (
[],
[],
(
"Mooncake producer registered layer has no matching "
f"consumer occurrence: {key[0]} occurrence {key[1]}."
),
)
if local_region.layer_index != remote_region.layer_index:
return (
[],
[],
(
"Mooncake registered layer index mismatch for "
f"{local_region.layer_name}: producer="
f"{local_region.layer_index}, consumer="
f"{remote_region.layer_index}."
),
)
aligned_local.append(local_region)
aligned_remote.append(remote_region)
return aligned_local, aligned_remote, None
def _get_tensor_dense_flag(tensor: torch.Tensor) -> bool | None:
is_dense = getattr(tensor, "is_non_overlapping_and_dense", None)
if callable(is_dense):
@@ -262,6 +332,8 @@ class MooncakeXferMetadata(
req_blocks: dict[ReqId, tuple[TransferId, list[list[int]]]]
kv_caches_base_addr: list[int]
block_lens: list[int]
registered_layer_names: list[str] = msgspec.field(default_factory=list)
registered_layer_indices: list[int] = msgspec.field(default_factory=list)
class MooncakeXferResponseStatus(IntEnum):
@@ -782,17 +854,15 @@ class MooncakeConnectorWorker:
self.tp_size = get_tensor_model_parallel_world_size()
self.num_blocks = 0
self.block_len_per_layer: list[int] = []
self.registered_layer_names: list[str] = []
self.registered_layer_indices: list[int] = []
self.seen_base_addresses: list[int] = []
assert (parallel_config := vllm_config.parallel_config)
dp_rank = parallel_config.data_parallel_index
dp_local_rank = parallel_config.data_parallel_rank_local
self.dp_rank = dp_local_rank if parallel_config.local_engines_only else dp_rank
pp_size = vllm_config.parallel_config.pipeline_parallel_size
if pp_size > 1:
raise ValueError(
"Mooncake Transfer Engine does not support pipeline parallelism yet."
)
self.pp_size = vllm_config.parallel_config.pipeline_parallel_size
self.pp_rank = get_pp_group().rank_in_group
self.kv_caches_base_addr: list[int] = []
@@ -1020,11 +1090,27 @@ class MooncakeConnectorWorker:
await sock.send_multipart((identity, self._encoder.encode(response)))
return
local_regions = self._get_transfer_regions(
self.kv_caches_base_addr, self.block_len_per_layer
self.kv_caches_base_addr,
self.block_len_per_layer,
self.registered_layer_names,
self.registered_layer_indices,
)
remote_regions = self._get_transfer_regions(
meta.kv_caches_base_addr, meta.block_lens
meta.kv_caches_base_addr,
meta.block_lens,
meta.registered_layer_names,
meta.registered_layer_indices,
)
local_regions, remote_regions, align_err = _align_transfer_regions(
local_regions, remote_regions
)
if align_err is not None:
response = MooncakeXferResponse(
status=MooncakeXferResponseStatus.ERROR,
err_msg=align_err,
)
await sock.send_multipart((identity, self._encoder.encode(response)))
return
validation_err = _validate_asymmetric_region_lengths(
local_regions=local_regions,
remote_regions=remote_regions,
@@ -1171,11 +1257,15 @@ class MooncakeConnectorWorker:
)
await sock.send_multipart((identity, self._encoder.encode(response)))
def resolve_need_send(self, send_meta: SendBlockMeta, remote_tp_ranks: list[int]):
def resolve_need_send(
self,
send_meta: SendBlockMeta,
remote_tp_ranks: list[int],
):
# Prepare for heterogeneous TP (one P pairs to multiple D)
send_meta.need_send = len(remote_tp_ranks)
logger.debug(
"Mooncake request %s will be served by %d consumer TP workers: %s",
"Mooncake request %s will be served by %d consumer TP workers: TP ranks=%s",
send_meta.transfer_id,
send_meta.need_send,
remote_tp_ranks,
@@ -1394,10 +1484,13 @@ class MooncakeConnectorWorker:
kv_data_lens = []
seen_base_addresses = []
self.block_len_per_layer = []
self.registered_layer_names = []
self.registered_layer_indices = []
split_k_and_v = self.transfer_topo.split_k_and_v
tensor_size_bytes = None
for layer_name, cache_or_caches in kv_caches.items():
layer_index = extract_layer_index(layer_name)
cache_list = cache_or_caches if split_k_and_v else [cache_or_caches]
logger.debug(
"registering layer %s with %d cache tensor(s)",
@@ -1428,6 +1521,8 @@ class MooncakeConnectorWorker:
block_len = cache.stride(0) * cache.element_size()
self.block_len_per_layer.append(block_len)
self.registered_layer_names.append(layer_name)
self.registered_layer_indices.append(layer_index)
kv_data_ptrs.append(base_addr)
kv_data_lens.append(self.num_blocks * block_len)
@@ -1547,6 +1642,8 @@ class MooncakeConnectorWorker:
},
kv_caches_base_addr=self.kv_caches_base_addr,
block_lens=self.block_len_per_layer,
registered_layer_names=self.registered_layer_names,
registered_layer_indices=self.registered_layer_indices,
)
encoded_data = self._encoder.encode(metadata)
@@ -1648,16 +1745,28 @@ class MooncakeConnectorWorker:
remote_tp_ranks = self.transfer_topo.handshake_target_ranks(
self._tp_size[remote_engine_id]
)
count = len(remote_tp_ranks)
worker_addrs: list[str] = []
selected_remote_pp: dict[int, list[int]] = {}
for remote_tp_rank in remote_tp_ranks:
pp_to_addr = self._remote_agents[remote_engine_id][remote_tp_rank]
if self.pp_size == len(pp_to_addr) and self.pp_rank in pp_to_addr:
pp_ranks = [self.pp_rank]
else:
pp_ranks = sorted(pp_to_addr)
selected_remote_pp[remote_tp_rank] = pp_ranks
worker_addrs.extend(pp_to_addr[pp_rank] for pp_rank in pp_ranks)
count = len(worker_addrs)
logger.debug(
"Receiving Mooncake KV for engine %s from producer TP ranks %s",
"Receiving Mooncake KV for engine %s from producer TP ranks %s "
"and PP ranks %s",
remote_engine_id,
remote_tp_ranks,
selected_remote_pp,
)
for pull_meta in pull_metas.values():
pull_meta.pull_tasks_count = count
for remote_tp_rank in remote_tp_ranks:
worker_addr = self._remote_agents[remote_engine_id][remote_tp_rank][0]
for worker_addr in worker_addrs:
asyncio.create_task(
self.receive_kv_from_single_worker(worker_addr, pull_metas)
)
@@ -1740,11 +1849,17 @@ class MooncakeConnectorWorker:
return self.transfer_topo.local_replicates_kv_cache
def _get_transfer_regions(
self, base_addrs: list[int], block_lens: list[int]
self,
base_addrs: list[int],
block_lens: list[int],
layer_names: list[str],
layer_indices: list[int],
) -> list[TransferRegion]:
return _expand_transfer_regions(
base_addrs=base_addrs,
block_lens=block_lens,
layer_names=layer_names,
layer_indices=layer_indices,
is_kv_layout_blocks_first=self.transfer_topo.virtually_split_kv_in_blocks,
)
@@ -1816,14 +1931,20 @@ def _async_loop(loop: asyncio.AbstractEventLoop):
def should_launch_bootstrap_server(vllm_config: VllmConfig) -> bool:
assert (parallel_config := vllm_config.parallel_config)
# Only the TP=0, PP=0 worker of the designated engine should launch it.
if get_tensor_model_parallel_rank() != 0:
return False
if get_pp_group().rank_in_group != 0:
return False
# In hybrid or external LB mode,
# each instance should have its own bootstrap server.
#
if parallel_config.local_engines_only:
return parallel_config.data_parallel_rank_local == 0
# In internal LB mode,
# only the real global first rank need to launch the bootstrap server.
return is_local_first_rank() and (
parallel_config.local_engines_only or parallel_config.data_parallel_index == 0
)
# only the first data-parallel engine should launch the bootstrap server.
return parallel_config.data_parallel_index == 0
def get_mooncake_bootstrap_addr(vllm_config: VllmConfig) -> tuple[str, int]:
@@ -1836,6 +1957,10 @@ def get_mooncake_bootstrap_addr(vllm_config: VllmConfig) -> tuple[str, int]:
if parallel_config.local_engines_only:
# In hybrid or external LB mode, connect to local server.
host = "127.0.0.1"
elif parallel_config.nnodes_within_dp > 1:
# Internal LB multi-node TP/PP uses the model-parallel master as the
# single bootstrap endpoint for all ranks in the engine.
host = parallel_config.master_addr
else:
host = parallel_config.data_parallel_master_ip
port = envs.VLLM_MOONCAKE_BOOTSTRAP_PORT
@@ -50,8 +50,13 @@ def is_v1_kv_transfer_group(connector: KVConnectorBaseType | None = None) -> boo
def _sync_engine_id_across_tp(vllm_config: "VllmConfig") -> None:
"""Broadcast engine_id from TP rank 0 so all workers in a
multi-node TP group share the same value."""
multi-node TP group share the same value.
When PP is enabled, also broadcast across PP ranks so all workers in
the same model-parallel engine share the same value.
"""
from vllm.distributed.parallel_state import (
get_pp_group,
get_tp_group,
)
@@ -59,6 +64,8 @@ def _sync_engine_id_across_tp(vllm_config: "VllmConfig") -> None:
synced_id = get_tp_group().broadcast_object(
vllm_config.kv_transfer_config.engine_id, src=0
)
if vllm_config.parallel_config.pipeline_parallel_size > 1:
synced_id = get_pp_group().broadcast_object(synced_id, src=0)
vllm_config.kv_transfer_config.engine_id = synced_id