[Perf][KV Connector][Mooncake] Send the KV layout once per peer (#59825)
Signed-off-by: staryxchen <staryxchen@tencent.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com>
This commit is contained in:
co-authored by
Cursor
Jiangyun Zhu
parent
a400804ce4
commit
6605802904
@@ -11,6 +11,7 @@ from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import msgspec
|
||||
import pytest
|
||||
import torch
|
||||
import zmq.asyncio
|
||||
@@ -2229,6 +2230,215 @@ def test_prepare_transfer_regions_reuses_success_and_error():
|
||||
assert calls == 6
|
||||
|
||||
|
||||
def _short_xfer_meta(worker, layout_id: int, req_blocks: dict) -> MooncakeXferMetadata:
|
||||
return MooncakeXferMetadata(
|
||||
remote_hostname="peer",
|
||||
remote_port=9,
|
||||
remote_tp_size=worker.tp_size,
|
||||
remote_tp_rank=0,
|
||||
remote_pp_size=worker.pp_size,
|
||||
layout_id=layout_id,
|
||||
req_blocks=req_blocks,
|
||||
kv_caches_base_addr=[],
|
||||
block_lens=[],
|
||||
kv_block_lens=[],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_steady_pull_omits_layout_and_unknown_id_asks_for_resend():
|
||||
"""A later pull is much smaller, and an unseen layout_id is not guessed."""
|
||||
n = 146
|
||||
names = [f"model.layers.{i}.self_attn" for i in range(n)]
|
||||
req_blocks = {"d": ("xfer", [[1, 2, 3]])}
|
||||
full = MooncakeXferMetadata(
|
||||
remote_hostname="h",
|
||||
remote_port=1,
|
||||
remote_tp_size=4,
|
||||
remote_tp_rank=0,
|
||||
layout_id=1,
|
||||
req_blocks=req_blocks,
|
||||
kv_caches_base_addr=list(range(n)),
|
||||
block_lens=[4096] * n,
|
||||
kv_block_lens=[3000] * n,
|
||||
registered_layer_names=names,
|
||||
registered_layer_indices=list(range(n)),
|
||||
registered_group_indices=[i % 5 for i in range(n)],
|
||||
registered_shared_group_ids=[[] for _ in range(n)],
|
||||
registered_row_offsets=[i * 100 for i in range(n)],
|
||||
)
|
||||
steady = MooncakeXferMetadata(
|
||||
remote_hostname="h",
|
||||
remote_port=1,
|
||||
remote_tp_size=4,
|
||||
remote_tp_rank=0,
|
||||
layout_id=1,
|
||||
req_blocks=req_blocks,
|
||||
kv_caches_base_addr=[],
|
||||
block_lens=[],
|
||||
kv_block_lens=[],
|
||||
)
|
||||
encoder = msgspec.msgpack.Encoder()
|
||||
full_bytes = len(encoder.encode(full))
|
||||
steady_bytes = len(encoder.encode(steady))
|
||||
assert steady_bytes < full_bytes * 0.2
|
||||
|
||||
worker, _, regions, _ = _register_sliced_packed_mla([0, 1])
|
||||
calls = 0
|
||||
original = worker._get_transfer_regions
|
||||
|
||||
def counting(*args, **kwargs):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
return original(*args, **kwargs)
|
||||
|
||||
worker._get_transfer_regions = counting
|
||||
|
||||
def handshake(layout_id: int) -> MooncakeXferMetadata:
|
||||
return _xfer_meta(
|
||||
regions,
|
||||
req_blocks,
|
||||
remote_hostname="peer",
|
||||
remote_port=9,
|
||||
remote_tp_size=worker.tp_size,
|
||||
remote_pp_size=worker.pp_size,
|
||||
layout_id=layout_id,
|
||||
)
|
||||
|
||||
prepared = worker._prepare_transfer_regions(handshake(1))
|
||||
assert prepared[2] is None
|
||||
assert calls == 2
|
||||
stored = worker._layout_by_peer[worker._peer_layout_key(handshake(1))]
|
||||
assert stored == (prepared[0], prepared[1])
|
||||
|
||||
# Same layout under a new id hits the tuple cache and must still be stored.
|
||||
worker._prepare_transfer_regions(handshake(3))
|
||||
assert calls == 2
|
||||
assert worker._peer_layout_key(handshake(3)) in worker._layout_by_peer
|
||||
|
||||
sock = MagicMock()
|
||||
sock.send_multipart = AsyncMock()
|
||||
await worker.send_kv_to_decode(b"d", sock, _short_xfer_meta(worker, 5, req_blocks))
|
||||
_, payload = sock.send_multipart.call_args[0][0]
|
||||
response = worker._xfer_resp_decoder.decode(payload)
|
||||
assert response.status == MooncakeXferResponseStatus.LAYOUT_MISS
|
||||
assert calls == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_peer_layouts_evict_least_recently_used(monkeypatch):
|
||||
"""Stale layout ids are bounded, and an evicted peer is asked to resend."""
|
||||
monkeypatch.setattr(mooncake_connector, "_MAX_PEER_LAYOUTS", 2)
|
||||
worker, _, regions, _ = _register_sliced_packed_mla([0])
|
||||
req_blocks = {"d": ("xfer", [[1]])}
|
||||
|
||||
def handshake(layout_id: int) -> MooncakeXferMetadata:
|
||||
return _xfer_meta(
|
||||
regions,
|
||||
req_blocks,
|
||||
remote_hostname="peer",
|
||||
remote_port=9,
|
||||
remote_tp_size=worker.tp_size,
|
||||
remote_pp_size=worker.pp_size,
|
||||
layout_id=layout_id,
|
||||
)
|
||||
|
||||
worker._prepare_transfer_regions(handshake(1))
|
||||
worker._prepare_transfer_regions(handshake(3))
|
||||
assert worker._lookup_peer_layout(_short_xfer_meta(worker, 1, req_blocks))
|
||||
worker._prepare_transfer_regions(handshake(5))
|
||||
|
||||
assert worker._lookup_peer_layout(_short_xfer_meta(worker, 1, req_blocks))
|
||||
assert worker._lookup_peer_layout(_short_xfer_meta(worker, 5, req_blocks))
|
||||
sock = MagicMock()
|
||||
sock.send_multipart = AsyncMock()
|
||||
await worker.send_kv_to_decode(b"d", sock, _short_xfer_meta(worker, 3, req_blocks))
|
||||
_, payload = sock.send_multipart.call_args[0][0]
|
||||
response = worker._xfer_resp_decoder.decode(payload)
|
||||
assert response.status == MooncakeXferResponseStatus.LAYOUT_MISS
|
||||
|
||||
|
||||
class _ScriptedSock:
|
||||
def __init__(self, responses: list[bytes], sent: list[bytes]):
|
||||
self._responses = responses
|
||||
self._sent = sent
|
||||
|
||||
def setsockopt(self, *_args, **_kwargs):
|
||||
return None
|
||||
|
||||
async def send(self, data: bytes):
|
||||
self._sent.append(data)
|
||||
|
||||
async def recv(self):
|
||||
return self._responses.pop(0)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
|
||||
def _single_pull() -> dict[str, PullReqMeta]:
|
||||
return {
|
||||
"d": PullReqMeta(
|
||||
d_req_id="d",
|
||||
transfer_id="t",
|
||||
local_block_ids=[[1]],
|
||||
remote_engine_id="p",
|
||||
remote_bootstrap_addr="http://bootstrap",
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
async def _pull_once(worker, addr: str, responses: list[MooncakeXferResponse]):
|
||||
sent: list[bytes] = []
|
||||
encoded = [worker._encoder.encode(response) for response in responses]
|
||||
with patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake."
|
||||
"mooncake_connector.make_zmq_socket",
|
||||
return_value=_ScriptedSock(encoded, sent),
|
||||
):
|
||||
await worker.receive_kv_from_single_worker(addr, _single_pull())
|
||||
return sent
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pull_resends_layout_when_prefill_forgot_it():
|
||||
worker, _, _, _ = _register_sliced_packed_mla([0])
|
||||
addr = "tcp://peer:1"
|
||||
worker._acked_layout_peers.add(addr)
|
||||
sent = await _pull_once(
|
||||
worker,
|
||||
addr,
|
||||
[
|
||||
MooncakeXferResponse(status=MooncakeXferResponseStatus.LAYOUT_MISS),
|
||||
MooncakeXferResponse(
|
||||
status=MooncakeXferResponseStatus.FINISH,
|
||||
ok_reqs=["d"],
|
||||
layout_id=worker._kv_layout_id,
|
||||
),
|
||||
],
|
||||
)
|
||||
assert len(sent) == 2
|
||||
assert len(sent[1]) > len(sent[0])
|
||||
assert addr in worker._acked_layout_peers
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pull_keeps_layout_for_prefill_that_does_not_echo_it():
|
||||
"""A prefill without layout_id support never gets a layout-less pull."""
|
||||
worker, _, _, _ = _register_sliced_packed_mla([0])
|
||||
addr = "tcp://old-peer:1"
|
||||
finish = MooncakeXferResponse(
|
||||
status=MooncakeXferResponseStatus.FINISH, ok_reqs=["d"]
|
||||
)
|
||||
first = await _pull_once(worker, addr, [finish])
|
||||
assert addr not in worker._acked_layout_peers
|
||||
second = await _pull_once(worker, addr, [finish])
|
||||
assert len(second[0]) == len(first[0])
|
||||
|
||||
|
||||
def test_coalesce_keeps_a_run_inside_its_starting_row():
|
||||
"""The row bound is the run start, not the previous slice."""
|
||||
row = 2500
|
||||
|
||||
@@ -8,6 +8,7 @@ validated by this test module.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from collections import OrderedDict
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
@@ -243,8 +244,10 @@ async def test_build_transfer_params_uses_non_overlapping_physical_pages():
|
||||
kv_cache_config = make_hybrid_gdn_kv_cache_config(block_size=16)
|
||||
worker = object.__new__(MooncakeConnectorWorker)
|
||||
worker.shutdown = noop_shutdown
|
||||
# __new__ skips __init__, where this cache is created. register_kv_caches clears it.
|
||||
# __new__ skips __init__, where these caches are created.
|
||||
worker._prepared_transfer_regions = {}
|
||||
worker._layout_by_peer = OrderedDict()
|
||||
worker._acked_layout_peers = set()
|
||||
worker.use_mla = False
|
||||
worker.engine = SimpleNamespace(batch_register_memory=lambda *_: 0)
|
||||
worker.is_kv_consumer = True
|
||||
|
||||
Reference in New Issue
Block a user