[Perf][KV Connector][Mooncake] Send the KV layout once per peer (#59825)

Signed-off-by: staryxchen <staryxchen@tencent.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com>
This commit is contained in:
Stary
2026-10-15 05:36:31 +00:00
committed by GitHub
co-authored by Cursor Jiangyun Zhu
parent a400804ce4
commit 6605802904
3 changed files with 412 additions and 70 deletions
@@ -11,6 +11,7 @@ from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import msgspec
import pytest
import torch
import zmq.asyncio
@@ -2229,6 +2230,215 @@ def test_prepare_transfer_regions_reuses_success_and_error():
assert calls == 6
def _short_xfer_meta(worker, layout_id: int, req_blocks: dict) -> MooncakeXferMetadata:
return MooncakeXferMetadata(
remote_hostname="peer",
remote_port=9,
remote_tp_size=worker.tp_size,
remote_tp_rank=0,
remote_pp_size=worker.pp_size,
layout_id=layout_id,
req_blocks=req_blocks,
kv_caches_base_addr=[],
block_lens=[],
kv_block_lens=[],
)
@pytest.mark.asyncio
async def test_steady_pull_omits_layout_and_unknown_id_asks_for_resend():
"""A later pull is much smaller, and an unseen layout_id is not guessed."""
n = 146
names = [f"model.layers.{i}.self_attn" for i in range(n)]
req_blocks = {"d": ("xfer", [[1, 2, 3]])}
full = MooncakeXferMetadata(
remote_hostname="h",
remote_port=1,
remote_tp_size=4,
remote_tp_rank=0,
layout_id=1,
req_blocks=req_blocks,
kv_caches_base_addr=list(range(n)),
block_lens=[4096] * n,
kv_block_lens=[3000] * n,
registered_layer_names=names,
registered_layer_indices=list(range(n)),
registered_group_indices=[i % 5 for i in range(n)],
registered_shared_group_ids=[[] for _ in range(n)],
registered_row_offsets=[i * 100 for i in range(n)],
)
steady = MooncakeXferMetadata(
remote_hostname="h",
remote_port=1,
remote_tp_size=4,
remote_tp_rank=0,
layout_id=1,
req_blocks=req_blocks,
kv_caches_base_addr=[],
block_lens=[],
kv_block_lens=[],
)
encoder = msgspec.msgpack.Encoder()
full_bytes = len(encoder.encode(full))
steady_bytes = len(encoder.encode(steady))
assert steady_bytes < full_bytes * 0.2
worker, _, regions, _ = _register_sliced_packed_mla([0, 1])
calls = 0
original = worker._get_transfer_regions
def counting(*args, **kwargs):
nonlocal calls
calls += 1
return original(*args, **kwargs)
worker._get_transfer_regions = counting
def handshake(layout_id: int) -> MooncakeXferMetadata:
return _xfer_meta(
regions,
req_blocks,
remote_hostname="peer",
remote_port=9,
remote_tp_size=worker.tp_size,
remote_pp_size=worker.pp_size,
layout_id=layout_id,
)
prepared = worker._prepare_transfer_regions(handshake(1))
assert prepared[2] is None
assert calls == 2
stored = worker._layout_by_peer[worker._peer_layout_key(handshake(1))]
assert stored == (prepared[0], prepared[1])
# Same layout under a new id hits the tuple cache and must still be stored.
worker._prepare_transfer_regions(handshake(3))
assert calls == 2
assert worker._peer_layout_key(handshake(3)) in worker._layout_by_peer
sock = MagicMock()
sock.send_multipart = AsyncMock()
await worker.send_kv_to_decode(b"d", sock, _short_xfer_meta(worker, 5, req_blocks))
_, payload = sock.send_multipart.call_args[0][0]
response = worker._xfer_resp_decoder.decode(payload)
assert response.status == MooncakeXferResponseStatus.LAYOUT_MISS
assert calls == 2
@pytest.mark.asyncio
async def test_peer_layouts_evict_least_recently_used(monkeypatch):
"""Stale layout ids are bounded, and an evicted peer is asked to resend."""
monkeypatch.setattr(mooncake_connector, "_MAX_PEER_LAYOUTS", 2)
worker, _, regions, _ = _register_sliced_packed_mla([0])
req_blocks = {"d": ("xfer", [[1]])}
def handshake(layout_id: int) -> MooncakeXferMetadata:
return _xfer_meta(
regions,
req_blocks,
remote_hostname="peer",
remote_port=9,
remote_tp_size=worker.tp_size,
remote_pp_size=worker.pp_size,
layout_id=layout_id,
)
worker._prepare_transfer_regions(handshake(1))
worker._prepare_transfer_regions(handshake(3))
assert worker._lookup_peer_layout(_short_xfer_meta(worker, 1, req_blocks))
worker._prepare_transfer_regions(handshake(5))
assert worker._lookup_peer_layout(_short_xfer_meta(worker, 1, req_blocks))
assert worker._lookup_peer_layout(_short_xfer_meta(worker, 5, req_blocks))
sock = MagicMock()
sock.send_multipart = AsyncMock()
await worker.send_kv_to_decode(b"d", sock, _short_xfer_meta(worker, 3, req_blocks))
_, payload = sock.send_multipart.call_args[0][0]
response = worker._xfer_resp_decoder.decode(payload)
assert response.status == MooncakeXferResponseStatus.LAYOUT_MISS
class _ScriptedSock:
def __init__(self, responses: list[bytes], sent: list[bytes]):
self._responses = responses
self._sent = sent
def setsockopt(self, *_args, **_kwargs):
return None
async def send(self, data: bytes):
self._sent.append(data)
async def recv(self):
return self._responses.pop(0)
def __enter__(self):
return self
def __exit__(self, *_args):
return False
def _single_pull() -> dict[str, PullReqMeta]:
return {
"d": PullReqMeta(
d_req_id="d",
transfer_id="t",
local_block_ids=[[1]],
remote_engine_id="p",
remote_bootstrap_addr="http://bootstrap",
)
}
async def _pull_once(worker, addr: str, responses: list[MooncakeXferResponse]):
sent: list[bytes] = []
encoded = [worker._encoder.encode(response) for response in responses]
with patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake."
"mooncake_connector.make_zmq_socket",
return_value=_ScriptedSock(encoded, sent),
):
await worker.receive_kv_from_single_worker(addr, _single_pull())
return sent
@pytest.mark.asyncio
async def test_pull_resends_layout_when_prefill_forgot_it():
worker, _, _, _ = _register_sliced_packed_mla([0])
addr = "tcp://peer:1"
worker._acked_layout_peers.add(addr)
sent = await _pull_once(
worker,
addr,
[
MooncakeXferResponse(status=MooncakeXferResponseStatus.LAYOUT_MISS),
MooncakeXferResponse(
status=MooncakeXferResponseStatus.FINISH,
ok_reqs=["d"],
layout_id=worker._kv_layout_id,
),
],
)
assert len(sent) == 2
assert len(sent[1]) > len(sent[0])
assert addr in worker._acked_layout_peers
@pytest.mark.asyncio
async def test_pull_keeps_layout_for_prefill_that_does_not_echo_it():
"""A prefill without layout_id support never gets a layout-less pull."""
worker, _, _, _ = _register_sliced_packed_mla([0])
addr = "tcp://old-peer:1"
finish = MooncakeXferResponse(
status=MooncakeXferResponseStatus.FINISH, ok_reqs=["d"]
)
first = await _pull_once(worker, addr, [finish])
assert addr not in worker._acked_layout_peers
second = await _pull_once(worker, addr, [finish])
assert len(second[0]) == len(first[0])
def test_coalesce_keeps_a_run_inside_its_starting_row():
"""The row bound is the run start, not the previous slice."""
row = 2500
@@ -8,6 +8,7 @@ validated by this test module.
"""
import asyncio
from collections import OrderedDict
from types import SimpleNamespace
from unittest.mock import patch
@@ -243,8 +244,10 @@ async def test_build_transfer_params_uses_non_overlapping_physical_pages():
kv_cache_config = make_hybrid_gdn_kv_cache_config(block_size=16)
worker = object.__new__(MooncakeConnectorWorker)
worker.shutdown = noop_shutdown
# __new__ skips __init__, where this cache is created. register_kv_caches clears it.
# __new__ skips __init__, where these caches are created.
worker._prepared_transfer_regions = {}
worker._layout_by_peer = OrderedDict()
worker._acked_layout_peers = set()
worker.use_mla = False
worker.engine = SimpleNamespace(batch_register_memory=lambda *_: 0)
worker.is_kv_consumer = True
@@ -5,9 +5,10 @@ import itertools
import logging
import os
import queue
import secrets
import threading
import time
from collections import defaultdict
from collections import OrderedDict, defaultdict
from collections.abc import Collection
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
@@ -76,6 +77,9 @@ from vllm.v1.worker.utils import select_common_block_size
logger = init_logger(__name__)
_BOOTSTRAP_MAX_ATTEMPTS: Final[int] = 3
# Decode restarts mint new layout ids, so stale peer layouts pile up. Evicting
# one only costs that peer a LAYOUT_MISS and one layout resend.
_MAX_PEER_LAYOUTS: Final[int] = 1024
try:
from mooncake.engine import TransferEngine
@@ -623,6 +627,9 @@ class MooncakeXferMetadata(
# peer did not send the field, so the run must not be promoted.
registered_row_offsets: list[int] = msgspec.field(default_factory=list)
remote_pp_size: int = 1
# Random per register_kv_caches, so ids never repeat across restarts.
# 0 means the peer sends the layout on every pull.
layout_id: int = 0
class MooncakeXferResponseStatus(IntEnum):
@@ -632,6 +639,9 @@ class MooncakeXferResponseStatus(IntEnum):
CONTINUE = 1
# Something wrong, see err_msg
ERROR = 2
# P does not hold the layout for this layout_id; resend it.
# Only sent for pulls with a nonzero layout_id and no layout.
LAYOUT_MISS = 3
class MooncakeXferResponse(
@@ -642,6 +652,9 @@ class MooncakeXferResponse(
ok_reqs: list[ReqId] | None = None
err_reqs: list[ReqId] | None = None
err_msg: str | None = None
# The pull's layout_id once P holds that layout. D omits the layout only
# after seeing it, so a P that predates this field always gets it.
layout_id: int = 0
@dataclass
@@ -1272,6 +1285,14 @@ class MooncakeConnectorWorker:
tuple,
tuple[list[TransferRegion], list[TransferRegion], str | None],
] = {}
# (host, port, tp_rank, pp_size, tp_size, layout_id) -> regions, in LRU
# order. Filled only after a pull that actually carried the layout.
self._layout_by_peer: OrderedDict[
tuple, tuple[list[TransferRegion], list[TransferRegion]]
] = OrderedDict()
self._kv_layout_id = 0
# Decode side: worker addresses that echoed the current layout_id.
self._acked_layout_peers: set[str] = set()
assert (parallel_config := vllm_config.parallel_config)
dp_rank = parallel_config.data_parallel_index
@@ -1561,14 +1582,30 @@ class MooncakeConnectorWorker:
)
await sock.send_multipart((identity, self._encoder.encode(response)))
return
local_regions, remote_regions, prep_err = self._prepare_transfer_regions(meta)
if prep_err is not None:
response = MooncakeXferResponse(
status=MooncakeXferResponseStatus.ERROR,
err_msg=prep_err,
if meta.layout_id and not meta.kv_caches_base_addr:
stored = self._lookup_peer_layout(meta)
if stored is None:
response = MooncakeXferResponse(
status=MooncakeXferResponseStatus.LAYOUT_MISS,
err_msg=(
f"Mooncake KV layout {meta.layout_id:#x} from "
f"{meta.remote_hostname}:{meta.remote_port} is not cached."
),
)
await sock.send_multipart((identity, self._encoder.encode(response)))
return
local_regions, remote_regions = stored
else:
local_regions, remote_regions, prep_err = self._prepare_transfer_regions(
meta
)
await sock.send_multipart((identity, self._encoder.encode(response)))
return
if prep_err is not None:
response = MooncakeXferResponse(
status=MooncakeXferResponseStatus.ERROR,
err_msg=prep_err,
)
await sock.send_multipart((identity, self._encoder.encode(response)))
return
validation_err = _validate_asymmetric_region_lengths(
local_regions=local_regions,
remote_regions=remote_regions,
@@ -1629,6 +1666,7 @@ class MooncakeConnectorWorker:
status=MooncakeXferResponseStatus.FINISH,
err_reqs=list(pending_reqs),
err_msg="Timeout waiting for P side ready.",
layout_id=meta.layout_id,
)
await sock.send_multipart((identity, self._encoder.encode(response)))
break
@@ -1719,6 +1757,7 @@ class MooncakeConnectorWorker:
ok_reqs=[d_req_id for d_req_id, _ in ok_ready_reqs] or None,
err_reqs=err_reqs or None,
err_msg=err_msg,
layout_id=meta.layout_id,
)
await sock.send_multipart((identity, self._encoder.encode(response)))
@@ -2018,6 +2057,10 @@ class MooncakeConnectorWorker:
self.region_row_offsets = []
self.opaque_packed_storages = set()
self._prepared_transfer_regions.clear()
self._layout_by_peer.clear()
self._acked_layout_peers.clear()
# Odd, so never 0 (the "layout on every pull" marker).
self._kv_layout_id = secrets.randbits(63) | 1
packed_storage_to_region: dict[int, int] = {}
packed_view_to_region: dict[tuple[int, int, int], int] = {}
@@ -2357,64 +2400,74 @@ class MooncakeConnectorWorker:
pull_metas: dict[ReqId, PullReqMeta],
):
req_ids = set(pull_metas)
metadata = MooncakeXferMetadata(
remote_hostname=self.hostname,
remote_port=self.rpc_port,
remote_tp_size=self.tp_size,
remote_tp_rank=self.tp_rank,
remote_pp_size=self.pp_size,
req_blocks={
req_id: (pull_meta.transfer_id, pull_meta.local_block_ids)
for req_id, pull_meta in pull_metas.items()
},
kv_caches_base_addr=self.kv_caches_base_addr,
block_lens=self.block_len_per_layer,
kv_block_lens=self.kv_block_len_per_layer,
registered_layer_names=self.registered_layer_names,
registered_layer_indices=self.registered_layer_indices,
registered_group_indices=self.registered_group_indices,
registered_shared_group_ids=[
list(groups) for groups in self.region_shared_groups
],
registered_row_offsets=self.region_row_offsets,
include_layout = (
not self._kv_layout_id or worker_addr not in self._acked_layout_peers
)
encoded_data = self._encoder.encode(metadata)
logger.debug(
"Size of encoded MooncakeXferMetadata: %d bytes", len(encoded_data)
)
logger.debug(
"Sending kv transfer request for %s on path: %s", req_ids, worker_addr
)
# Send query for the request.
try:
with make_zmq_socket(
self.async_zmq_ctx, worker_addr, zmq.DEALER, bind=False, linger=0
) as sock:
# If something goes wrong, let P wait timeout first (in asyncio.wait()).
sock.setsockopt(
zmq.RCVTIMEO, (envs.VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT + 60) * 1000
# Each pass either returns or breaks out of the recv loop to resend
# the layout after a LAYOUT_MISS.
while True:
metadata = self._build_pull_metadata(
pull_metas, include_layout=include_layout
)
encoded_data = self._encoder.encode(metadata)
logger.debug(
"MooncakeXferMetadata to %s is %d bytes (%s).",
worker_addr,
len(encoded_data),
"layout" if include_layout else "blocks",
)
logger.debug(
"Sending kv transfer request for %s on path: %s", req_ids, worker_addr
)
try:
with make_zmq_socket(
self.async_zmq_ctx, worker_addr, zmq.DEALER, bind=False, linger=0
) as sock:
sock.setsockopt(
zmq.RCVTIMEO,
(envs.VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT + 60) * 1000,
)
await sock.send(encoded_data)
while True:
ret_msg = await sock.recv()
response = self._xfer_resp_decoder.decode(ret_msg)
status = response.status
if status == MooncakeXferResponseStatus.LAYOUT_MISS:
self._acked_layout_peers.discard(worker_addr)
if not include_layout:
logger.info(
"Mooncake prefill %s lost the KV layout; "
"resending it.",
worker_addr,
)
include_layout = True
break
if status in (
MooncakeXferResponseStatus.ERROR,
MooncakeXferResponseStatus.LAYOUT_MISS,
):
self._handle_failed_recv(
pull_metas,
req_ids,
response.err_msg or "transfer error",
)
return
if (
self._kv_layout_id
and response.layout_id == self._kv_layout_id
):
self._acked_layout_peers.add(worker_addr)
self.process_pulling_result(response, pull_metas)
if status == MooncakeXferResponseStatus.FINISH:
return
except zmq.ContextTerminated:
logger.debug(
"ZMQ context terminated, exiting Mooncake receiver thread."
)
await sock.send(encoded_data)
while True:
ret_msg = await sock.recv()
response = self._xfer_resp_decoder.decode(ret_msg)
if response.status == MooncakeXferResponseStatus.ERROR:
self._handle_failed_recv(
pull_metas,
req_ids,
response.err_msg or "transfer error",
)
return
self.process_pulling_result(response, pull_metas)
if response.status == MooncakeXferResponseStatus.FINISH:
break
except zmq.ContextTerminated:
logger.debug("ZMQ context terminated, exiting Mooncake receiver thread.")
except Exception as e:
self._handle_failed_recv(pull_metas, req_ids, f"transfer failed: {e}")
return
return
except Exception as e:
self._handle_failed_recv(pull_metas, req_ids, f"transfer failed: {e}")
return
def _handle_failed_recv(
self,
@@ -2673,6 +2726,8 @@ class MooncakeConnectorWorker:
)
cached = self._prepared_transfer_regions.get(cache_key)
if cached is not None:
# A new layout_id can carry a layout this cache already holds.
self._remember_peer_layout(meta, cached)
return cached
def finish(
@@ -2766,12 +2821,86 @@ class MooncakeConnectorWorker:
n_aligned,
len(local_regions),
)
self._prepared_transfer_regions[cache_key] = (
local_regions,
remote_regions,
None,
prepared = (local_regions, remote_regions, None)
self._prepared_transfer_regions[cache_key] = prepared
self._remember_peer_layout(meta, prepared)
return prepared
def _peer_layout_key(self, meta: MooncakeXferMetadata) -> tuple:
return (
meta.remote_hostname,
meta.remote_port,
meta.remote_tp_rank,
meta.remote_pp_size,
meta.remote_tp_size,
meta.layout_id,
)
def _remember_peer_layout(
self,
meta: MooncakeXferMetadata,
prepared: tuple[list[TransferRegion], list[TransferRegion], str | None],
) -> None:
local_regions, remote_regions, err = prepared
if err is not None or not meta.layout_id:
return
key = self._peer_layout_key(meta)
self._layout_by_peer[key] = (local_regions, remote_regions)
self._layout_by_peer.move_to_end(key)
while len(self._layout_by_peer) > _MAX_PEER_LAYOUTS:
self._layout_by_peer.popitem(last=False)
def _lookup_peer_layout(
self, meta: MooncakeXferMetadata
) -> tuple[list[TransferRegion], list[TransferRegion]] | None:
key = self._peer_layout_key(meta)
stored = self._layout_by_peer.get(key)
if stored is not None:
self._layout_by_peer.move_to_end(key)
return stored
def _build_pull_metadata(
self,
pull_metas: dict[ReqId, PullReqMeta],
*,
include_layout: bool,
) -> MooncakeXferMetadata:
req_blocks = {
req_id: (pull_meta.transfer_id, pull_meta.local_block_ids)
for req_id, pull_meta in pull_metas.items()
}
if not include_layout:
return MooncakeXferMetadata(
remote_hostname=self.hostname,
remote_port=self.rpc_port,
remote_tp_size=self.tp_size,
remote_tp_rank=self.tp_rank,
remote_pp_size=self.pp_size,
layout_id=self._kv_layout_id,
req_blocks=req_blocks,
kv_caches_base_addr=[],
block_lens=[],
kv_block_lens=[],
)
return MooncakeXferMetadata(
remote_hostname=self.hostname,
remote_port=self.rpc_port,
remote_tp_size=self.tp_size,
remote_tp_rank=self.tp_rank,
remote_pp_size=self.pp_size,
layout_id=self._kv_layout_id,
req_blocks=req_blocks,
kv_caches_base_addr=self.kv_caches_base_addr,
block_lens=self.block_len_per_layer,
kv_block_lens=self.kv_block_len_per_layer,
registered_layer_names=self.registered_layer_names,
registered_layer_indices=self.registered_layer_indices,
registered_group_indices=self.registered_group_indices,
registered_shared_group_ids=[
list(groups) for groups in self.region_shared_groups
],
registered_row_offsets=self.region_row_offsets,
)
return local_regions, remote_regions, None
def _validate_head_resharding_layout(
self, remote_tp_size: int, local_regions: list[TransferRegion]