[Perf][KV Connector][Mooncake] Send the KV layout once per peer (#59825)

Signed-off-by: staryxchen <staryxchen@tencent.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com>
This commit is contained in:
Stary
2026-10-15 05:36:31 +00:00
committed by GitHub
co-authored by Cursor Jiangyun Zhu
parent a400804ce4
commit 6605802904
3 changed files with 412 additions and 70 deletions
@@ -11,6 +11,7 @@ from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import msgspec
import pytest
import torch
import zmq.asyncio
@@ -2229,6 +2230,215 @@ def test_prepare_transfer_regions_reuses_success_and_error():
assert calls == 6
def _short_xfer_meta(worker, layout_id: int, req_blocks: dict) -> MooncakeXferMetadata:
return MooncakeXferMetadata(
remote_hostname="peer",
remote_port=9,
remote_tp_size=worker.tp_size,
remote_tp_rank=0,
remote_pp_size=worker.pp_size,
layout_id=layout_id,
req_blocks=req_blocks,
kv_caches_base_addr=[],
block_lens=[],
kv_block_lens=[],
)
@pytest.mark.asyncio
async def test_steady_pull_omits_layout_and_unknown_id_asks_for_resend():
"""A later pull is much smaller, and an unseen layout_id is not guessed."""
n = 146
names = [f"model.layers.{i}.self_attn" for i in range(n)]
req_blocks = {"d": ("xfer", [[1, 2, 3]])}
full = MooncakeXferMetadata(
remote_hostname="h",
remote_port=1,
remote_tp_size=4,
remote_tp_rank=0,
layout_id=1,
req_blocks=req_blocks,
kv_caches_base_addr=list(range(n)),
block_lens=[4096] * n,
kv_block_lens=[3000] * n,
registered_layer_names=names,
registered_layer_indices=list(range(n)),
registered_group_indices=[i % 5 for i in range(n)],
registered_shared_group_ids=[[] for _ in range(n)],
registered_row_offsets=[i * 100 for i in range(n)],
)
steady = MooncakeXferMetadata(
remote_hostname="h",
remote_port=1,
remote_tp_size=4,
remote_tp_rank=0,
layout_id=1,
req_blocks=req_blocks,
kv_caches_base_addr=[],
block_lens=[],
kv_block_lens=[],
)
encoder = msgspec.msgpack.Encoder()
full_bytes = len(encoder.encode(full))
steady_bytes = len(encoder.encode(steady))
assert steady_bytes < full_bytes * 0.2
worker, _, regions, _ = _register_sliced_packed_mla([0, 1])
calls = 0
original = worker._get_transfer_regions
def counting(*args, **kwargs):
nonlocal calls
calls += 1
return original(*args, **kwargs)
worker._get_transfer_regions = counting
def handshake(layout_id: int) -> MooncakeXferMetadata:
return _xfer_meta(
regions,
req_blocks,
remote_hostname="peer",
remote_port=9,
remote_tp_size=worker.tp_size,
remote_pp_size=worker.pp_size,
layout_id=layout_id,
)
prepared = worker._prepare_transfer_regions(handshake(1))
assert prepared[2] is None
assert calls == 2
stored = worker._layout_by_peer[worker._peer_layout_key(handshake(1))]
assert stored == (prepared[0], prepared[1])
# Same layout under a new id hits the tuple cache and must still be stored.
worker._prepare_transfer_regions(handshake(3))
assert calls == 2
assert worker._peer_layout_key(handshake(3)) in worker._layout_by_peer
sock = MagicMock()
sock.send_multipart = AsyncMock()
await worker.send_kv_to_decode(b"d", sock, _short_xfer_meta(worker, 5, req_blocks))
_, payload = sock.send_multipart.call_args[0][0]
response = worker._xfer_resp_decoder.decode(payload)
assert response.status == MooncakeXferResponseStatus.LAYOUT_MISS
assert calls == 2
@pytest.mark.asyncio
async def test_peer_layouts_evict_least_recently_used(monkeypatch):
"""Stale layout ids are bounded, and an evicted peer is asked to resend."""
monkeypatch.setattr(mooncake_connector, "_MAX_PEER_LAYOUTS", 2)
worker, _, regions, _ = _register_sliced_packed_mla([0])
req_blocks = {"d": ("xfer", [[1]])}
def handshake(layout_id: int) -> MooncakeXferMetadata:
return _xfer_meta(
regions,
req_blocks,
remote_hostname="peer",
remote_port=9,
remote_tp_size=worker.tp_size,
remote_pp_size=worker.pp_size,
layout_id=layout_id,
)
worker._prepare_transfer_regions(handshake(1))
worker._prepare_transfer_regions(handshake(3))
assert worker._lookup_peer_layout(_short_xfer_meta(worker, 1, req_blocks))
worker._prepare_transfer_regions(handshake(5))
assert worker._lookup_peer_layout(_short_xfer_meta(worker, 1, req_blocks))
assert worker._lookup_peer_layout(_short_xfer_meta(worker, 5, req_blocks))
sock = MagicMock()
sock.send_multipart = AsyncMock()
await worker.send_kv_to_decode(b"d", sock, _short_xfer_meta(worker, 3, req_blocks))
_, payload = sock.send_multipart.call_args[0][0]
response = worker._xfer_resp_decoder.decode(payload)
assert response.status == MooncakeXferResponseStatus.LAYOUT_MISS
class _ScriptedSock:
def __init__(self, responses: list[bytes], sent: list[bytes]):
self._responses = responses
self._sent = sent
def setsockopt(self, *_args, **_kwargs):
return None
async def send(self, data: bytes):
self._sent.append(data)
async def recv(self):
return self._responses.pop(0)
def __enter__(self):
return self
def __exit__(self, *_args):
return False
def _single_pull() -> dict[str, PullReqMeta]:
return {
"d": PullReqMeta(
d_req_id="d",
transfer_id="t",
local_block_ids=[[1]],
remote_engine_id="p",
remote_bootstrap_addr="http://bootstrap",
)
}
async def _pull_once(worker, addr: str, responses: list[MooncakeXferResponse]):
sent: list[bytes] = []
encoded = [worker._encoder.encode(response) for response in responses]
with patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake."
"mooncake_connector.make_zmq_socket",
return_value=_ScriptedSock(encoded, sent),
):
await worker.receive_kv_from_single_worker(addr, _single_pull())
return sent
@pytest.mark.asyncio
async def test_pull_resends_layout_when_prefill_forgot_it():
worker, _, _, _ = _register_sliced_packed_mla([0])
addr = "tcp://peer:1"
worker._acked_layout_peers.add(addr)
sent = await _pull_once(
worker,
addr,
[
MooncakeXferResponse(status=MooncakeXferResponseStatus.LAYOUT_MISS),
MooncakeXferResponse(
status=MooncakeXferResponseStatus.FINISH,
ok_reqs=["d"],
layout_id=worker._kv_layout_id,
),
],
)
assert len(sent) == 2
assert len(sent[1]) > len(sent[0])
assert addr in worker._acked_layout_peers
@pytest.mark.asyncio
async def test_pull_keeps_layout_for_prefill_that_does_not_echo_it():
"""A prefill without layout_id support never gets a layout-less pull."""
worker, _, _, _ = _register_sliced_packed_mla([0])
addr = "tcp://old-peer:1"
finish = MooncakeXferResponse(
status=MooncakeXferResponseStatus.FINISH, ok_reqs=["d"]
)
first = await _pull_once(worker, addr, [finish])
assert addr not in worker._acked_layout_peers
second = await _pull_once(worker, addr, [finish])
assert len(second[0]) == len(first[0])
def test_coalesce_keeps_a_run_inside_its_starting_row():
"""The row bound is the run start, not the previous slice."""
row = 2500
@@ -8,6 +8,7 @@ validated by this test module.
"""
import asyncio
from collections import OrderedDict
from types import SimpleNamespace
from unittest.mock import patch
@@ -243,8 +244,10 @@ async def test_build_transfer_params_uses_non_overlapping_physical_pages():
kv_cache_config = make_hybrid_gdn_kv_cache_config(block_size=16)
worker = object.__new__(MooncakeConnectorWorker)
worker.shutdown = noop_shutdown
# __new__ skips __init__, where this cache is created. register_kv_caches clears it.
# __new__ skips __init__, where these caches are created.
worker._prepared_transfer_regions = {}
worker._layout_by_peer = OrderedDict()
worker._acked_layout_peers = set()
worker.use_mla = False
worker.engine = SimpleNamespace(batch_register_memory=lambda *_: 0)
worker.is_kv_consumer = True