[KV Connector][Mooncake] Pipeline-parallel support for PD-disaggregated serving with Mooncake connector (#44528)
Signed-off-by: hanhan.hank <hanhan.hank@bytedance.com> Signed-off-by: Hank Han <hanhan7630@outlook.com>
This commit is contained in:
@@ -4,22 +4,29 @@
|
||||
import asyncio
|
||||
import contextlib
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import zmq.asyncio
|
||||
|
||||
from vllm import envs
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector import (
|
||||
KVConnectorRole,
|
||||
MooncakeConnector,
|
||||
MooncakeConnectorMetadata,
|
||||
MooncakeConnectorWorker,
|
||||
MooncakeXferMetadata,
|
||||
MooncakeXferResponse,
|
||||
MooncakeXferResponseStatus,
|
||||
PullReqMeta,
|
||||
SendBlockMeta,
|
||||
TransferRegion,
|
||||
_align_transfer_regions,
|
||||
get_mooncake_bootstrap_addr,
|
||||
should_launch_bootstrap_server,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_utils import (
|
||||
MooncakeBootstrapServer,
|
||||
@@ -57,6 +64,272 @@ class FakeMooncakeWrapper:
|
||||
return 0
|
||||
|
||||
|
||||
def test_align_transfer_regions_uses_layer_name_occurrences():
|
||||
"""Repeated layer names should align by occurrence order."""
|
||||
|
||||
local_regions = [
|
||||
TransferRegion(
|
||||
layer_name="model.layers.1.self_attn",
|
||||
layer_index=1,
|
||||
base_addr=0x1000,
|
||||
block_len=256,
|
||||
kv_block_len=128,
|
||||
),
|
||||
TransferRegion(
|
||||
layer_name="model.layers.1.self_attn",
|
||||
layer_index=1,
|
||||
base_addr=0x1100,
|
||||
block_len=256,
|
||||
kv_block_len=128,
|
||||
),
|
||||
]
|
||||
remote_regions = [
|
||||
TransferRegion(
|
||||
layer_name="model.layers.0.self_attn",
|
||||
layer_index=0,
|
||||
base_addr=0xA000,
|
||||
block_len=256,
|
||||
kv_block_len=128,
|
||||
),
|
||||
TransferRegion(
|
||||
layer_name="model.layers.1.self_attn",
|
||||
layer_index=1,
|
||||
base_addr=0xB000,
|
||||
block_len=256,
|
||||
kv_block_len=128,
|
||||
),
|
||||
TransferRegion(
|
||||
layer_name="model.layers.1.self_attn",
|
||||
layer_index=1,
|
||||
base_addr=0xB100,
|
||||
block_len=256,
|
||||
kv_block_len=128,
|
||||
),
|
||||
]
|
||||
|
||||
aligned_local, aligned_remote, err = _align_transfer_regions(
|
||||
local_regions, remote_regions
|
||||
)
|
||||
|
||||
assert err is None
|
||||
assert [r.base_addr for r in aligned_local] == [0x1000, 0x1100]
|
||||
assert [r.base_addr for r in aligned_remote] == [0xB000, 0xB100]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_build_transfer_params_separates_prefill_pp_layers():
|
||||
"""Each producer PP stage should send only its registered layer shard."""
|
||||
|
||||
worker = MooncakeConnectorWorker.__new__(MooncakeConnectorWorker)
|
||||
worker.async_zmq_ctx = MagicMock()
|
||||
worker.is_kv_consumer = True
|
||||
worker.is_kv_producer = True
|
||||
worker.tp_rank = 0
|
||||
worker.tp_size = 1
|
||||
worker.transfer_topo = SimpleNamespace(local_replicates_kv_cache=False)
|
||||
|
||||
block_len = 256
|
||||
remote_regions = [
|
||||
TransferRegion(
|
||||
layer_name=f"model.layers.{layer_index}.self_attn",
|
||||
layer_index=layer_index,
|
||||
base_addr=base_addr,
|
||||
block_len=block_len,
|
||||
kv_block_len=block_len,
|
||||
)
|
||||
for layer_index, base_addr in [
|
||||
(0, 0xA000),
|
||||
(1, 0xB000),
|
||||
(2, 0xC000),
|
||||
(3, 0xD000),
|
||||
]
|
||||
]
|
||||
producer_pp_regions = {
|
||||
0: [
|
||||
TransferRegion(
|
||||
layer_name="model.layers.0.self_attn",
|
||||
layer_index=0,
|
||||
base_addr=0x1000,
|
||||
block_len=block_len,
|
||||
kv_block_len=block_len,
|
||||
),
|
||||
TransferRegion(
|
||||
layer_name="model.layers.1.self_attn",
|
||||
layer_index=1,
|
||||
base_addr=0x2000,
|
||||
block_len=block_len,
|
||||
kv_block_len=block_len,
|
||||
),
|
||||
],
|
||||
1: [
|
||||
TransferRegion(
|
||||
layer_name="model.layers.2.self_attn",
|
||||
layer_index=2,
|
||||
base_addr=0x3000,
|
||||
block_len=block_len,
|
||||
kv_block_len=block_len,
|
||||
),
|
||||
TransferRegion(
|
||||
layer_name="model.layers.3.self_attn",
|
||||
layer_index=3,
|
||||
base_addr=0x4000,
|
||||
block_len=block_len,
|
||||
kv_block_len=block_len,
|
||||
),
|
||||
],
|
||||
}
|
||||
expected_by_pp_rank = {
|
||||
0: {
|
||||
"layers": [0, 1],
|
||||
"src_ptrs": [0x1000 + 10 * block_len, 0x2000 + 10 * block_len],
|
||||
"dst_ptrs": [0xA000 + 20 * block_len, 0xB000 + 20 * block_len],
|
||||
},
|
||||
1: {
|
||||
"layers": [2, 3],
|
||||
"src_ptrs": [0x3000 + 10 * block_len, 0x4000 + 10 * block_len],
|
||||
"dst_ptrs": [0xC000 + 20 * block_len, 0xD000 + 20 * block_len],
|
||||
},
|
||||
}
|
||||
|
||||
transfer_id = "xfer-pp-split"
|
||||
send_meta = SendBlockMeta(
|
||||
p_req_id="p-req-pp",
|
||||
transfer_id=transfer_id,
|
||||
local_block_ids=[[10, 11]],
|
||||
ready=asyncio.Event(),
|
||||
)
|
||||
xfer_meta = MooncakeXferMetadata(
|
||||
remote_hostname="consumer-host",
|
||||
remote_port=54321,
|
||||
remote_tp_size=1,
|
||||
remote_tp_rank=0,
|
||||
req_blocks={"d-req-pp": (transfer_id, [[20, 21]])},
|
||||
kv_caches_base_addr=[region.base_addr for region in remote_regions],
|
||||
block_lens=[region.block_len for region in remote_regions],
|
||||
registered_layer_names=[region.layer_name for region in remote_regions],
|
||||
registered_layer_indices=[region.layer_index for region in remote_regions],
|
||||
)
|
||||
|
||||
for pp_rank, local_regions in producer_pp_regions.items():
|
||||
aligned_local, aligned_remote, err = _align_transfer_regions(
|
||||
local_regions, remote_regions
|
||||
)
|
||||
|
||||
assert err is None
|
||||
assert [r.layer_index for r in aligned_local] == (
|
||||
expected_by_pp_rank[pp_rank]["layers"]
|
||||
)
|
||||
assert [r.layer_index for r in aligned_remote] == (
|
||||
expected_by_pp_rank[pp_rank]["layers"]
|
||||
)
|
||||
|
||||
(
|
||||
src_ptrs,
|
||||
dst_ptrs,
|
||||
lengths,
|
||||
err_reqs,
|
||||
err_msg,
|
||||
) = await worker._build_transfer_params(
|
||||
ready_reqs=[("d-req-pp", send_meta)],
|
||||
agent_meta=xfer_meta,
|
||||
local_regions=aligned_local,
|
||||
remote_regions=aligned_remote,
|
||||
)
|
||||
|
||||
assert err_reqs == []
|
||||
assert err_msg is None
|
||||
assert src_ptrs == expected_by_pp_rank[pp_rank]["src_ptrs"]
|
||||
assert dst_ptrs == expected_by_pp_rank[pp_rank]["dst_ptrs"]
|
||||
assert lengths == [2 * block_len, 2 * block_len]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_send_kv_to_decode_aligns_consumer_regions_by_layer_metadata(
|
||||
monkeypatch,
|
||||
):
|
||||
"""Producer sends its PP layer shard to the matching consumer layer address."""
|
||||
|
||||
monkeypatch.setenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "5")
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MooncakeConnector", kv_role="kv_producer"
|
||||
)
|
||||
|
||||
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
|
||||
prefill_connector = MooncakeConnector(
|
||||
vllm_config,
|
||||
KVConnectorRole.WORKER,
|
||||
_make_test_kv_cache_config(),
|
||||
)
|
||||
prefill_worker = prefill_connector.connector_worker
|
||||
|
||||
block_len = 4096
|
||||
kv_half = block_len // 2
|
||||
prefill_worker.kv_caches_base_addr = [0x1000]
|
||||
prefill_worker.block_len_per_layer = [block_len]
|
||||
prefill_worker.registered_layer_names = ["model.layers.1.self_attn"]
|
||||
prefill_worker.registered_layer_indices = [1]
|
||||
|
||||
class InlineSenderLoop:
|
||||
async def run_in_executor(self, executor, func, *args):
|
||||
return func(*args)
|
||||
|
||||
origin_sender_loop = prefill_worker.sender_loop
|
||||
prefill_worker.sender_loop = InlineSenderLoop()
|
||||
|
||||
transfer_id = "xfer-layer-align"
|
||||
send_meta = SendBlockMeta(
|
||||
p_req_id="p-req-layer-align",
|
||||
transfer_id=transfer_id,
|
||||
local_block_ids=[[10]],
|
||||
ready=asyncio.Event(),
|
||||
)
|
||||
prefill_worker.reqs_need_send[transfer_id] = send_meta
|
||||
send_meta.ready.set()
|
||||
|
||||
xfer_meta = MooncakeXferMetadata(
|
||||
remote_hostname="consumer-host",
|
||||
remote_port=54321,
|
||||
remote_tp_size=1,
|
||||
remote_tp_rank=0,
|
||||
req_blocks={"d-req-layer-align": (transfer_id, [[20]])},
|
||||
kv_caches_base_addr=[0xA000, 0xB000],
|
||||
block_lens=[block_len, block_len],
|
||||
registered_layer_names=[
|
||||
"model.layers.0.self_attn",
|
||||
"model.layers.1.self_attn",
|
||||
],
|
||||
registered_layer_indices=[0, 1],
|
||||
)
|
||||
mock_socket = AsyncMock(spec=zmq.asyncio.Socket)
|
||||
mock_socket.send_multipart = AsyncMock()
|
||||
identity = b"consumer-layer-align"
|
||||
|
||||
with patch.object(
|
||||
prefill_worker, "_send_blocks", return_value=0
|
||||
) as mock_send_blocks:
|
||||
await prefill_worker.send_kv_to_decode(identity, mock_socket, xfer_meta)
|
||||
|
||||
src_ptrs, dst_ptrs, lengths = mock_send_blocks.call_args[0][1:]
|
||||
assert src_ptrs == [
|
||||
0x1000 + 10 * block_len,
|
||||
0x1000 + 10 * block_len + kv_half,
|
||||
]
|
||||
assert dst_ptrs == [
|
||||
0xB000 + 20 * block_len,
|
||||
0xB000 + 20 * block_len + kv_half,
|
||||
]
|
||||
assert lengths == [kv_half, kv_half]
|
||||
|
||||
sent_identity, sent_payload = mock_socket.send_multipart.call_args[0][0]
|
||||
assert sent_identity == identity
|
||||
response = prefill_worker._xfer_resp_decoder.decode(sent_payload)
|
||||
assert response.status == MooncakeXferResponseStatus.FINISH
|
||||
assert response.ok_reqs == ["d-req-layer-align"]
|
||||
|
||||
prefill_worker.sender_loop = origin_sender_loop
|
||||
prefill_worker.shutdown()
|
||||
|
||||
|
||||
def test_basic_interface():
|
||||
"""Unit test for basic MooncakeConnector interface functionality."""
|
||||
|
||||
@@ -174,7 +447,7 @@ async def test_bootstrap_server(bootstrap_server: MooncakeBootstrapServer):
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {}
|
||||
|
||||
# Register a worker
|
||||
# Register multiple PP workers from the same producer engine.
|
||||
payload1 = {
|
||||
"engine_id": "eng-1",
|
||||
"dp_rank": 0,
|
||||
@@ -187,7 +460,19 @@ async def test_bootstrap_server(bootstrap_server: MooncakeBootstrapServer):
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {"status": "ok"}
|
||||
|
||||
# Query after registration
|
||||
payload2 = {
|
||||
"engine_id": "eng-1",
|
||||
"dp_rank": 0,
|
||||
"tp_rank": 0,
|
||||
"pp_rank": 1,
|
||||
"addr": "tcp://2.2.2.2:2222",
|
||||
}
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.post(f"{base_url}/register", json=payload2)
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {"status": "ok"}
|
||||
|
||||
# Query after registration should preserve the PP dimension.
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.get(f"{base_url}/query")
|
||||
assert response.status_code == 200
|
||||
@@ -195,6 +480,7 @@ async def test_bootstrap_server(bootstrap_server: MooncakeBootstrapServer):
|
||||
assert "0" in data
|
||||
assert data["0"]["engine_id"] == "eng-1"
|
||||
assert data["0"]["worker_addr"]["0"]["0"] == "tcp://1.1.1.1:1111"
|
||||
assert data["0"]["worker_addr"]["0"]["1"] == "tcp://2.2.2.2:2222"
|
||||
|
||||
# Test failure: re-registering the same worker
|
||||
async with httpx.AsyncClient() as client:
|
||||
@@ -216,6 +502,104 @@ async def test_bootstrap_server(bootstrap_server: MooncakeBootstrapServer):
|
||||
assert "Engine ID mismatch" in response.text
|
||||
|
||||
|
||||
def _make_bootstrap_vllm_config(
|
||||
*,
|
||||
local_engines_only: bool = False,
|
||||
data_parallel_rank_local: int = 0,
|
||||
data_parallel_index: int = 0,
|
||||
nnodes_within_dp: int = 1,
|
||||
) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(
|
||||
local_engines_only=local_engines_only,
|
||||
data_parallel_rank_local=data_parallel_rank_local,
|
||||
data_parallel_index=data_parallel_index,
|
||||
nnodes_within_dp=nnodes_within_dp,
|
||||
master_addr="model-parallel-master",
|
||||
data_parallel_master_ip="data-parallel-master",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"tp_rank",
|
||||
"pp_rank",
|
||||
"local_engines_only",
|
||||
"data_parallel_rank_local",
|
||||
"data_parallel_index",
|
||||
"expected",
|
||||
),
|
||||
[
|
||||
(1, 0, False, 0, 0, False),
|
||||
(0, 1, False, 0, 0, False),
|
||||
(0, 0, True, 0, 1, True),
|
||||
(0, 0, True, 1, 0, False),
|
||||
(0, 0, False, 0, 0, True),
|
||||
(0, 0, False, 0, 1, False),
|
||||
],
|
||||
ids=[
|
||||
"nonzero_tp_rank",
|
||||
"nonzero_pp_rank",
|
||||
"local_engine_rank_zero",
|
||||
"local_engine_nonzero_rank",
|
||||
"internal_lb_first_dp_engine",
|
||||
"internal_lb_nonzero_dp_engine",
|
||||
],
|
||||
)
|
||||
def test_should_launch_bootstrap_server_selects_single_owner(
|
||||
tp_rank: int,
|
||||
pp_rank: int,
|
||||
local_engines_only: bool,
|
||||
data_parallel_rank_local: int,
|
||||
data_parallel_index: int,
|
||||
expected: bool,
|
||||
):
|
||||
vllm_config = _make_bootstrap_vllm_config(
|
||||
local_engines_only=local_engines_only,
|
||||
data_parallel_rank_local=data_parallel_rank_local,
|
||||
data_parallel_index=data_parallel_index,
|
||||
)
|
||||
with (
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake."
|
||||
"mooncake_connector.get_tensor_model_parallel_rank",
|
||||
return_value=tp_rank,
|
||||
),
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake."
|
||||
"mooncake_connector.get_pp_group"
|
||||
) as mock_pp_group,
|
||||
):
|
||||
mock_pp_group.return_value.rank_in_group = pp_rank
|
||||
assert should_launch_bootstrap_server(vllm_config) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("local_engines_only", "nnodes_within_dp", "expected_host"),
|
||||
[
|
||||
(True, 2, "127.0.0.1"),
|
||||
(False, 2, "model-parallel-master"),
|
||||
(False, 1, "data-parallel-master"),
|
||||
],
|
||||
ids=["local_engine", "multi_node_tp_or_pp", "single_node_internal_lb"],
|
||||
)
|
||||
def test_get_mooncake_bootstrap_addr_selects_expected_host(
|
||||
local_engines_only: bool,
|
||||
nnodes_within_dp: int,
|
||||
expected_host: str,
|
||||
):
|
||||
vllm_config = _make_bootstrap_vllm_config(
|
||||
local_engines_only=local_engines_only,
|
||||
nnodes_within_dp=nnodes_within_dp,
|
||||
)
|
||||
|
||||
assert get_mooncake_bootstrap_addr(vllm_config) == (
|
||||
expected_host,
|
||||
envs.VLLM_MOONCAKE_BOOTSTRAP_PORT,
|
||||
)
|
||||
|
||||
|
||||
def test_scheduler_request_finished():
|
||||
"""
|
||||
Tests the scheduler-side logic when a request finishes.
|
||||
@@ -273,7 +657,6 @@ def patch_worker_dependencies():
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector.get_pp_group"
|
||||
) as mock_pp,
|
||||
patch("vllm.distributed.parallel_state.is_local_first_rank", return_value=True),
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector.should_launch_bootstrap_server",
|
||||
return_value=False,
|
||||
@@ -307,6 +690,93 @@ def patch_worker_dependencies():
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("local_pp_size", "local_pp_rank", "expected_addrs"),
|
||||
[
|
||||
(1, 0, ["tcp://producer-pp0:1234", "tcp://producer-pp1:1234"]),
|
||||
(2, 1, ["tcp://producer-pp1:1234"]),
|
||||
],
|
||||
ids=["heterogeneous_pp_pulls_all_remote_pp", "matching_pp_pulls_same_rank"],
|
||||
)
|
||||
async def test_receive_kv_selects_remote_pp_workers(
|
||||
local_pp_size: int,
|
||||
local_pp_rank: int,
|
||||
expected_addrs: list[str],
|
||||
):
|
||||
"""Decode workers should not hard-code producer pp_rank 0."""
|
||||
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MooncakeConnector", kv_role="kv_consumer"
|
||||
)
|
||||
|
||||
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
|
||||
decode_connector = MooncakeConnector(
|
||||
vllm_config,
|
||||
KVConnectorRole.WORKER,
|
||||
_make_test_kv_cache_config(),
|
||||
)
|
||||
decode_worker = decode_connector.connector_worker
|
||||
decode_worker.pp_size = local_pp_size
|
||||
decode_worker.pp_rank = local_pp_rank
|
||||
decode_worker._remote_agents = {
|
||||
"p-engine": {
|
||||
0: {
|
||||
0: "tcp://producer-pp0:1234",
|
||||
1: "tcp://producer-pp1:1234",
|
||||
}
|
||||
}
|
||||
}
|
||||
decode_worker._tp_size["p-engine"] = 1
|
||||
|
||||
pull_metas = {
|
||||
"d-req-1": PullReqMeta(
|
||||
d_req_id="d-req-1",
|
||||
transfer_id="xfer-req-1",
|
||||
local_block_ids=[[100, 101]],
|
||||
remote_engine_id="p-engine",
|
||||
remote_bootstrap_addr="http://bootstrap:33333",
|
||||
)
|
||||
}
|
||||
seen_addrs: list[str] = []
|
||||
|
||||
async def fake_receive(worker_addr: str, metas: dict[str, PullReqMeta]):
|
||||
seen_addrs.append(worker_addr)
|
||||
for meta in metas.values():
|
||||
meta.pull_tasks_count -= 1
|
||||
|
||||
with patch.object(
|
||||
decode_worker,
|
||||
"receive_kv_from_single_worker",
|
||||
side_effect=fake_receive,
|
||||
):
|
||||
decode_worker.receive_kv("p-engine", pull_metas)
|
||||
await asyncio.sleep(0)
|
||||
|
||||
assert seen_addrs == expected_addrs
|
||||
assert pull_metas["d-req-1"].pull_tasks_count == 0
|
||||
decode_worker.shutdown()
|
||||
|
||||
|
||||
def test_resolve_need_send_accounts_for_remote_tp_fanout():
|
||||
"""Producer-side completion waits for every paired consumer TP pull."""
|
||||
|
||||
worker = MooncakeConnectorWorker.__new__(MooncakeConnectorWorker)
|
||||
worker.async_zmq_ctx = MagicMock()
|
||||
worker.is_kv_consumer = True
|
||||
worker.is_kv_producer = True
|
||||
send_meta = SendBlockMeta(
|
||||
p_req_id="p-req-1",
|
||||
transfer_id="xfer-req-1",
|
||||
local_block_ids=[[1]],
|
||||
ready=asyncio.Event(),
|
||||
)
|
||||
|
||||
worker.resolve_need_send(send_meta, remote_tp_ranks=[0, 1])
|
||||
|
||||
assert send_meta.need_send == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector.TransferEngine",
|
||||
@@ -335,6 +805,8 @@ async def test_kv_producer(monkeypatch):
|
||||
prefill_worker.kv_caches_base_addr = [0x1000]
|
||||
block_len = 4096
|
||||
prefill_worker.block_len_per_layer = [block_len]
|
||||
prefill_worker.registered_layer_names = ["model.layers.0.self_attn"]
|
||||
prefill_worker.registered_layer_indices = [0]
|
||||
|
||||
# Override loop to use current test loop
|
||||
origin_sender_loop = prefill_worker.sender_loop
|
||||
@@ -360,6 +832,8 @@ async def test_kv_producer(monkeypatch):
|
||||
req_blocks={"d-req-1": (transfer_id, [[20, 21]])},
|
||||
kv_caches_base_addr=[0x2000],
|
||||
block_lens=[block_len],
|
||||
registered_layer_names=["model.layers.0.self_attn"],
|
||||
registered_layer_indices=[0],
|
||||
)
|
||||
|
||||
mock_socket = AsyncMock(spec=zmq.asyncio.Socket)
|
||||
@@ -506,6 +980,9 @@ async def test_kv_consumuer(monkeypatch):
|
||||
)
|
||||
decode_worker = decode_connector.connector_worker
|
||||
decode_worker.kv_caches_base_addr = [0x1000]
|
||||
decode_worker.block_len_per_layer = [4096]
|
||||
decode_worker.registered_layer_names = ["model.layers.0.self_attn"]
|
||||
decode_worker.registered_layer_indices = [0]
|
||||
decode_worker.rpc_port = 54321
|
||||
|
||||
# A request to pull data arrives.
|
||||
@@ -547,6 +1024,10 @@ async def test_kv_consumuer(monkeypatch):
|
||||
assert sent_meta.remote_hostname == "127.0.0.1"
|
||||
assert sent_meta.remote_port == 54321
|
||||
assert sent_meta.req_blocks["d-req-1"] == ("xfer-req-1", [[100, 101]])
|
||||
assert sent_meta.kv_caches_base_addr == [0x1000]
|
||||
assert sent_meta.block_lens == [4096]
|
||||
assert sent_meta.registered_layer_names == ["model.layers.0.self_attn"]
|
||||
assert sent_meta.registered_layer_indices == [0]
|
||||
|
||||
# Verify internal state is updated correctly.
|
||||
assert "d-req-1" in decode_worker.finished_recving_reqs
|
||||
@@ -626,7 +1107,10 @@ def test_register_kv_caches():
|
||||
)
|
||||
tensor1 = torch.zeros(*kv_cache_shape, dtype=torch.float16)
|
||||
tensor2 = torch.zeros(*kv_cache_shape, dtype=torch.float16)
|
||||
kv_caches = {"layer0": tensor1, "layer1": tensor2}
|
||||
kv_caches = {
|
||||
"model.layers.0.self_attn": tensor1,
|
||||
"model.layers.1.self_attn": tensor2,
|
||||
}
|
||||
|
||||
with patch.object(
|
||||
worker.engine, "batch_register_memory", return_value=0
|
||||
@@ -643,6 +1127,8 @@ def test_register_kv_caches():
|
||||
assert len(worker.block_len_per_layer) == len(registered_ptrs)
|
||||
for bl in worker.block_len_per_layer:
|
||||
assert bl == tensor1.nbytes // tensor1.shape[0]
|
||||
assert worker.registered_layer_names == list(kv_caches)
|
||||
assert worker.registered_layer_indices == [0, 1]
|
||||
|
||||
|
||||
def test_register_kv_caches_supports_mixed_mla_and_eagle_shapes():
|
||||
@@ -677,7 +1163,10 @@ def test_register_kv_caches_supports_mixed_mla_and_eagle_shapes():
|
||||
mla_cache = torch.zeros((2, 16, 96), dtype=torch.float16)
|
||||
# Eagle3/GQA-like cache tensor: shape[-2] is num_kv_heads, not block size.
|
||||
eagle_cache = torch.zeros((2, 16, 8, 64), dtype=torch.float16)
|
||||
kv_caches = {"mla_layer": mla_cache, "eagle_layer": eagle_cache}
|
||||
kv_caches = {
|
||||
"model.layers.0.mla_attn": mla_cache,
|
||||
"model.layers.1.eagle_attn": eagle_cache,
|
||||
}
|
||||
|
||||
with patch.object(
|
||||
worker.engine, "batch_register_memory", return_value=0
|
||||
@@ -692,6 +1181,11 @@ def test_register_kv_caches_supports_mixed_mla_and_eagle_shapes():
|
||||
mla_cache.nbytes // mla_cache.shape[0],
|
||||
eagle_cache.nbytes // eagle_cache.shape[0],
|
||||
]
|
||||
assert worker.registered_layer_names == [
|
||||
"model.layers.0.mla_attn",
|
||||
"model.layers.1.eagle_attn",
|
||||
]
|
||||
assert worker.registered_layer_indices == [0, 1]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -742,6 +1236,8 @@ async def test_kv_producer_heterogeneous_tp(monkeypatch, d_tp_size):
|
||||
|
||||
prefill_worker.kv_caches_base_addr = [0x1000]
|
||||
prefill_worker.block_len_per_layer = [local_block_len]
|
||||
prefill_worker.registered_layer_names = ["model.layers.0.self_attn"]
|
||||
prefill_worker.registered_layer_indices = [0]
|
||||
|
||||
origin_sender_loop = prefill_worker.sender_loop
|
||||
prefill_worker.sender_loop = asyncio.get_event_loop()
|
||||
@@ -787,6 +1283,8 @@ async def test_kv_producer_heterogeneous_tp(monkeypatch, d_tp_size):
|
||||
},
|
||||
kv_caches_base_addr=[0x2000],
|
||||
block_lens=[remote_block_len],
|
||||
registered_layer_names=["model.layers.0.self_attn"],
|
||||
registered_layer_indices=[0],
|
||||
)
|
||||
|
||||
mock_send_blocks.reset_mock()
|
||||
|
||||
@@ -261,12 +261,20 @@ async def test_build_transfer_params_multi_group_trimming(monkeypatch):
|
||||
|
||||
local_regions = [
|
||||
TransferRegion(
|
||||
base_addr=0x1000, block_len=block_len, kv_block_len=block_len
|
||||
layer_name="model.layers.0.self_attn",
|
||||
layer_index=0,
|
||||
base_addr=0x1000,
|
||||
block_len=block_len,
|
||||
kv_block_len=block_len,
|
||||
),
|
||||
]
|
||||
remote_regions = [
|
||||
TransferRegion(
|
||||
base_addr=0x2000, block_len=block_len, kv_block_len=block_len
|
||||
layer_name="model.layers.0.self_attn",
|
||||
layer_index=0,
|
||||
base_addr=0x2000,
|
||||
block_len=block_len,
|
||||
kv_block_len=block_len,
|
||||
),
|
||||
]
|
||||
|
||||
@@ -344,12 +352,20 @@ async def test_build_transfer_params_group_count_mismatch(monkeypatch):
|
||||
|
||||
local_regions = [
|
||||
TransferRegion(
|
||||
base_addr=0x1000, block_len=block_len, kv_block_len=block_len
|
||||
layer_name="model.layers.0.self_attn",
|
||||
layer_index=0,
|
||||
base_addr=0x1000,
|
||||
block_len=block_len,
|
||||
kv_block_len=block_len,
|
||||
),
|
||||
]
|
||||
remote_regions = [
|
||||
TransferRegion(
|
||||
base_addr=0x2000, block_len=block_len, kv_block_len=block_len
|
||||
layer_name="model.layers.0.self_attn",
|
||||
layer_index=0,
|
||||
base_addr=0x2000,
|
||||
block_len=block_len,
|
||||
kv_block_len=block_len,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@@ -43,10 +43,10 @@ from vllm.distributed.parallel_state import (
|
||||
get_pp_group,
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
is_local_first_rank,
|
||||
)
|
||||
from vllm.forward_context import ForwardContext
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.utils import extract_layer_index
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.network_utils import get_ip, make_zmq_path, make_zmq_socket
|
||||
@@ -80,6 +80,8 @@ TransferId = str # KV transfer coordination ID (shared by P/D)
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TransferRegion:
|
||||
layer_name: str
|
||||
layer_index: int
|
||||
base_addr: int
|
||||
block_len: int
|
||||
kv_block_len: int
|
||||
@@ -109,18 +111,28 @@ def _get_tp_ratio(local_tp_size: int, remote_tp_size: int) -> int:
|
||||
def _expand_transfer_regions(
|
||||
base_addrs: list[int],
|
||||
block_lens: list[int],
|
||||
layer_names: list[str],
|
||||
layer_indices: list[int],
|
||||
is_kv_layout_blocks_first: bool,
|
||||
) -> list[TransferRegion]:
|
||||
"""Expand registered KV tensors into the regions transferred by Mooncake."""
|
||||
assert len(base_addrs) == len(block_lens), (
|
||||
"Mooncake transfer regions require matching numbers of base addresses "
|
||||
f"and block lengths, got {len(base_addrs)} and {len(block_lens)}."
|
||||
assert (
|
||||
len(base_addrs) == len(block_lens) == len(layer_names) == len(layer_indices)
|
||||
), (
|
||||
"Mooncake transfer regions require matching metadata lengths, got "
|
||||
f"base_addrs={len(base_addrs)}, block_lens={len(block_lens)}, "
|
||||
f"layer_names={len(layer_names)}, "
|
||||
f"layer_indices={len(layer_indices)}."
|
||||
)
|
||||
regions: list[TransferRegion] = []
|
||||
for base_addr, block_len in zip(base_addrs, block_lens):
|
||||
for base_addr, block_len, layer_name, layer_index in zip(
|
||||
base_addrs, block_lens, layer_names, layer_indices
|
||||
):
|
||||
kv_block_len = block_len // 2 if is_kv_layout_blocks_first else block_len
|
||||
regions.append(
|
||||
TransferRegion(
|
||||
layer_name=layer_name,
|
||||
layer_index=layer_index,
|
||||
base_addr=base_addr,
|
||||
block_len=block_len,
|
||||
kv_block_len=kv_block_len,
|
||||
@@ -129,6 +141,8 @@ def _expand_transfer_regions(
|
||||
if is_kv_layout_blocks_first:
|
||||
regions.append(
|
||||
TransferRegion(
|
||||
layer_name=layer_name,
|
||||
layer_index=layer_index,
|
||||
base_addr=base_addr + kv_block_len,
|
||||
block_len=block_len,
|
||||
kv_block_len=kv_block_len,
|
||||
@@ -244,6 +258,62 @@ def _validate_asymmetric_region_lengths(
|
||||
return None
|
||||
|
||||
|
||||
def _align_transfer_regions(
|
||||
local_regions: list[TransferRegion],
|
||||
remote_regions: list[TransferRegion],
|
||||
) -> tuple[list[TransferRegion], list[TransferRegion], str | None]:
|
||||
"""Align KV transfer regions by registered layer-name occurrence.
|
||||
|
||||
PP shards own different layer subsets. Positional matching is therefore
|
||||
wrong once producer and consumer have different PP layouts. Multiple
|
||||
registered transfer buffers for the same layer are represented by repeated
|
||||
layer names and matched by occurrence order.
|
||||
"""
|
||||
|
||||
def keyed_regions(
|
||||
regions: list[TransferRegion],
|
||||
) -> list[tuple[tuple[str, int], TransferRegion]]:
|
||||
counts: dict[str, int] = defaultdict(int)
|
||||
keyed: list[tuple[tuple[str, int], TransferRegion]] = []
|
||||
for region in regions:
|
||||
occurrence = counts[region.layer_name]
|
||||
counts[region.layer_name] += 1
|
||||
keyed.append(((region.layer_name, occurrence), region))
|
||||
return keyed
|
||||
|
||||
local_keyed = keyed_regions(local_regions)
|
||||
remote_keyed = keyed_regions(remote_regions)
|
||||
remote_by_key = dict(remote_keyed)
|
||||
aligned_local: list[TransferRegion] = []
|
||||
aligned_remote: list[TransferRegion] = []
|
||||
for key, local_region in local_keyed:
|
||||
remote_region = remote_by_key.get(key)
|
||||
if remote_region is None:
|
||||
return (
|
||||
[],
|
||||
[],
|
||||
(
|
||||
"Mooncake producer registered layer has no matching "
|
||||
f"consumer occurrence: {key[0]} occurrence {key[1]}."
|
||||
),
|
||||
)
|
||||
if local_region.layer_index != remote_region.layer_index:
|
||||
return (
|
||||
[],
|
||||
[],
|
||||
(
|
||||
"Mooncake registered layer index mismatch for "
|
||||
f"{local_region.layer_name}: producer="
|
||||
f"{local_region.layer_index}, consumer="
|
||||
f"{remote_region.layer_index}."
|
||||
),
|
||||
)
|
||||
aligned_local.append(local_region)
|
||||
aligned_remote.append(remote_region)
|
||||
|
||||
return aligned_local, aligned_remote, None
|
||||
|
||||
|
||||
def _get_tensor_dense_flag(tensor: torch.Tensor) -> bool | None:
|
||||
is_dense = getattr(tensor, "is_non_overlapping_and_dense", None)
|
||||
if callable(is_dense):
|
||||
@@ -262,6 +332,8 @@ class MooncakeXferMetadata(
|
||||
req_blocks: dict[ReqId, tuple[TransferId, list[list[int]]]]
|
||||
kv_caches_base_addr: list[int]
|
||||
block_lens: list[int]
|
||||
registered_layer_names: list[str] = msgspec.field(default_factory=list)
|
||||
registered_layer_indices: list[int] = msgspec.field(default_factory=list)
|
||||
|
||||
|
||||
class MooncakeXferResponseStatus(IntEnum):
|
||||
@@ -782,17 +854,15 @@ class MooncakeConnectorWorker:
|
||||
self.tp_size = get_tensor_model_parallel_world_size()
|
||||
self.num_blocks = 0
|
||||
self.block_len_per_layer: list[int] = []
|
||||
self.registered_layer_names: list[str] = []
|
||||
self.registered_layer_indices: list[int] = []
|
||||
self.seen_base_addresses: list[int] = []
|
||||
|
||||
assert (parallel_config := vllm_config.parallel_config)
|
||||
dp_rank = parallel_config.data_parallel_index
|
||||
dp_local_rank = parallel_config.data_parallel_rank_local
|
||||
self.dp_rank = dp_local_rank if parallel_config.local_engines_only else dp_rank
|
||||
pp_size = vllm_config.parallel_config.pipeline_parallel_size
|
||||
if pp_size > 1:
|
||||
raise ValueError(
|
||||
"Mooncake Transfer Engine does not support pipeline parallelism yet."
|
||||
)
|
||||
self.pp_size = vllm_config.parallel_config.pipeline_parallel_size
|
||||
self.pp_rank = get_pp_group().rank_in_group
|
||||
|
||||
self.kv_caches_base_addr: list[int] = []
|
||||
@@ -1020,11 +1090,27 @@ class MooncakeConnectorWorker:
|
||||
await sock.send_multipart((identity, self._encoder.encode(response)))
|
||||
return
|
||||
local_regions = self._get_transfer_regions(
|
||||
self.kv_caches_base_addr, self.block_len_per_layer
|
||||
self.kv_caches_base_addr,
|
||||
self.block_len_per_layer,
|
||||
self.registered_layer_names,
|
||||
self.registered_layer_indices,
|
||||
)
|
||||
remote_regions = self._get_transfer_regions(
|
||||
meta.kv_caches_base_addr, meta.block_lens
|
||||
meta.kv_caches_base_addr,
|
||||
meta.block_lens,
|
||||
meta.registered_layer_names,
|
||||
meta.registered_layer_indices,
|
||||
)
|
||||
local_regions, remote_regions, align_err = _align_transfer_regions(
|
||||
local_regions, remote_regions
|
||||
)
|
||||
if align_err is not None:
|
||||
response = MooncakeXferResponse(
|
||||
status=MooncakeXferResponseStatus.ERROR,
|
||||
err_msg=align_err,
|
||||
)
|
||||
await sock.send_multipart((identity, self._encoder.encode(response)))
|
||||
return
|
||||
validation_err = _validate_asymmetric_region_lengths(
|
||||
local_regions=local_regions,
|
||||
remote_regions=remote_regions,
|
||||
@@ -1171,11 +1257,15 @@ class MooncakeConnectorWorker:
|
||||
)
|
||||
await sock.send_multipart((identity, self._encoder.encode(response)))
|
||||
|
||||
def resolve_need_send(self, send_meta: SendBlockMeta, remote_tp_ranks: list[int]):
|
||||
def resolve_need_send(
|
||||
self,
|
||||
send_meta: SendBlockMeta,
|
||||
remote_tp_ranks: list[int],
|
||||
):
|
||||
# Prepare for heterogeneous TP (one P pairs to multiple D)
|
||||
send_meta.need_send = len(remote_tp_ranks)
|
||||
logger.debug(
|
||||
"Mooncake request %s will be served by %d consumer TP workers: %s",
|
||||
"Mooncake request %s will be served by %d consumer TP workers: TP ranks=%s",
|
||||
send_meta.transfer_id,
|
||||
send_meta.need_send,
|
||||
remote_tp_ranks,
|
||||
@@ -1394,10 +1484,13 @@ class MooncakeConnectorWorker:
|
||||
kv_data_lens = []
|
||||
seen_base_addresses = []
|
||||
self.block_len_per_layer = []
|
||||
self.registered_layer_names = []
|
||||
self.registered_layer_indices = []
|
||||
|
||||
split_k_and_v = self.transfer_topo.split_k_and_v
|
||||
tensor_size_bytes = None
|
||||
for layer_name, cache_or_caches in kv_caches.items():
|
||||
layer_index = extract_layer_index(layer_name)
|
||||
cache_list = cache_or_caches if split_k_and_v else [cache_or_caches]
|
||||
logger.debug(
|
||||
"registering layer %s with %d cache tensor(s)",
|
||||
@@ -1428,6 +1521,8 @@ class MooncakeConnectorWorker:
|
||||
block_len = cache.stride(0) * cache.element_size()
|
||||
|
||||
self.block_len_per_layer.append(block_len)
|
||||
self.registered_layer_names.append(layer_name)
|
||||
self.registered_layer_indices.append(layer_index)
|
||||
kv_data_ptrs.append(base_addr)
|
||||
kv_data_lens.append(self.num_blocks * block_len)
|
||||
|
||||
@@ -1547,6 +1642,8 @@ class MooncakeConnectorWorker:
|
||||
},
|
||||
kv_caches_base_addr=self.kv_caches_base_addr,
|
||||
block_lens=self.block_len_per_layer,
|
||||
registered_layer_names=self.registered_layer_names,
|
||||
registered_layer_indices=self.registered_layer_indices,
|
||||
)
|
||||
|
||||
encoded_data = self._encoder.encode(metadata)
|
||||
@@ -1648,16 +1745,28 @@ class MooncakeConnectorWorker:
|
||||
remote_tp_ranks = self.transfer_topo.handshake_target_ranks(
|
||||
self._tp_size[remote_engine_id]
|
||||
)
|
||||
count = len(remote_tp_ranks)
|
||||
worker_addrs: list[str] = []
|
||||
selected_remote_pp: dict[int, list[int]] = {}
|
||||
for remote_tp_rank in remote_tp_ranks:
|
||||
pp_to_addr = self._remote_agents[remote_engine_id][remote_tp_rank]
|
||||
if self.pp_size == len(pp_to_addr) and self.pp_rank in pp_to_addr:
|
||||
pp_ranks = [self.pp_rank]
|
||||
else:
|
||||
pp_ranks = sorted(pp_to_addr)
|
||||
selected_remote_pp[remote_tp_rank] = pp_ranks
|
||||
worker_addrs.extend(pp_to_addr[pp_rank] for pp_rank in pp_ranks)
|
||||
|
||||
count = len(worker_addrs)
|
||||
logger.debug(
|
||||
"Receiving Mooncake KV for engine %s from producer TP ranks %s",
|
||||
"Receiving Mooncake KV for engine %s from producer TP ranks %s "
|
||||
"and PP ranks %s",
|
||||
remote_engine_id,
|
||||
remote_tp_ranks,
|
||||
selected_remote_pp,
|
||||
)
|
||||
for pull_meta in pull_metas.values():
|
||||
pull_meta.pull_tasks_count = count
|
||||
for remote_tp_rank in remote_tp_ranks:
|
||||
worker_addr = self._remote_agents[remote_engine_id][remote_tp_rank][0]
|
||||
for worker_addr in worker_addrs:
|
||||
asyncio.create_task(
|
||||
self.receive_kv_from_single_worker(worker_addr, pull_metas)
|
||||
)
|
||||
@@ -1740,11 +1849,17 @@ class MooncakeConnectorWorker:
|
||||
return self.transfer_topo.local_replicates_kv_cache
|
||||
|
||||
def _get_transfer_regions(
|
||||
self, base_addrs: list[int], block_lens: list[int]
|
||||
self,
|
||||
base_addrs: list[int],
|
||||
block_lens: list[int],
|
||||
layer_names: list[str],
|
||||
layer_indices: list[int],
|
||||
) -> list[TransferRegion]:
|
||||
return _expand_transfer_regions(
|
||||
base_addrs=base_addrs,
|
||||
block_lens=block_lens,
|
||||
layer_names=layer_names,
|
||||
layer_indices=layer_indices,
|
||||
is_kv_layout_blocks_first=self.transfer_topo.virtually_split_kv_in_blocks,
|
||||
)
|
||||
|
||||
@@ -1816,14 +1931,20 @@ def _async_loop(loop: asyncio.AbstractEventLoop):
|
||||
|
||||
def should_launch_bootstrap_server(vllm_config: VllmConfig) -> bool:
|
||||
assert (parallel_config := vllm_config.parallel_config)
|
||||
# Only the TP=0, PP=0 worker of the designated engine should launch it.
|
||||
if get_tensor_model_parallel_rank() != 0:
|
||||
return False
|
||||
if get_pp_group().rank_in_group != 0:
|
||||
return False
|
||||
|
||||
# In hybrid or external LB mode,
|
||||
# each instance should have its own bootstrap server.
|
||||
#
|
||||
if parallel_config.local_engines_only:
|
||||
return parallel_config.data_parallel_rank_local == 0
|
||||
|
||||
# In internal LB mode,
|
||||
# only the real global first rank need to launch the bootstrap server.
|
||||
return is_local_first_rank() and (
|
||||
parallel_config.local_engines_only or parallel_config.data_parallel_index == 0
|
||||
)
|
||||
# only the first data-parallel engine should launch the bootstrap server.
|
||||
return parallel_config.data_parallel_index == 0
|
||||
|
||||
|
||||
def get_mooncake_bootstrap_addr(vllm_config: VllmConfig) -> tuple[str, int]:
|
||||
@@ -1836,6 +1957,10 @@ def get_mooncake_bootstrap_addr(vllm_config: VllmConfig) -> tuple[str, int]:
|
||||
if parallel_config.local_engines_only:
|
||||
# In hybrid or external LB mode, connect to local server.
|
||||
host = "127.0.0.1"
|
||||
elif parallel_config.nnodes_within_dp > 1:
|
||||
# Internal LB multi-node TP/PP uses the model-parallel master as the
|
||||
# single bootstrap endpoint for all ranks in the engine.
|
||||
host = parallel_config.master_addr
|
||||
else:
|
||||
host = parallel_config.data_parallel_master_ip
|
||||
port = envs.VLLM_MOONCAKE_BOOTSTRAP_PORT
|
||||
|
||||
@@ -50,8 +50,13 @@ def is_v1_kv_transfer_group(connector: KVConnectorBaseType | None = None) -> boo
|
||||
|
||||
def _sync_engine_id_across_tp(vllm_config: "VllmConfig") -> None:
|
||||
"""Broadcast engine_id from TP rank 0 so all workers in a
|
||||
multi-node TP group share the same value."""
|
||||
multi-node TP group share the same value.
|
||||
|
||||
When PP is enabled, also broadcast across PP ranks so all workers in
|
||||
the same model-parallel engine share the same value.
|
||||
"""
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_pp_group,
|
||||
get_tp_group,
|
||||
)
|
||||
|
||||
@@ -59,6 +64,8 @@ def _sync_engine_id_across_tp(vllm_config: "VllmConfig") -> None:
|
||||
synced_id = get_tp_group().broadcast_object(
|
||||
vllm_config.kv_transfer_config.engine_id, src=0
|
||||
)
|
||||
if vllm_config.parallel_config.pipeline_parallel_size > 1:
|
||||
synced_id = get_pp_group().broadcast_object(synced_id, src=0)
|
||||
vllm_config.kv_transfer_config.engine_id = synced_id
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user