diff --git a/tests/v1/kv_connector/unit/test_mooncake_connector.py b/tests/v1/kv_connector/unit/test_mooncake_connector.py index 2cbb9cc670..afe626cba8 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_connector.py +++ b/tests/v1/kv_connector/unit/test_mooncake_connector.py @@ -11,6 +11,7 @@ from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch import httpx +import msgspec import pytest import torch import zmq.asyncio @@ -2229,6 +2230,215 @@ def test_prepare_transfer_regions_reuses_success_and_error(): assert calls == 6 +def _short_xfer_meta(worker, layout_id: int, req_blocks: dict) -> MooncakeXferMetadata: + return MooncakeXferMetadata( + remote_hostname="peer", + remote_port=9, + remote_tp_size=worker.tp_size, + remote_tp_rank=0, + remote_pp_size=worker.pp_size, + layout_id=layout_id, + req_blocks=req_blocks, + kv_caches_base_addr=[], + block_lens=[], + kv_block_lens=[], + ) + + +@pytest.mark.asyncio +async def test_steady_pull_omits_layout_and_unknown_id_asks_for_resend(): + """A later pull is much smaller, and an unseen layout_id is not guessed.""" + n = 146 + names = [f"model.layers.{i}.self_attn" for i in range(n)] + req_blocks = {"d": ("xfer", [[1, 2, 3]])} + full = MooncakeXferMetadata( + remote_hostname="h", + remote_port=1, + remote_tp_size=4, + remote_tp_rank=0, + layout_id=1, + req_blocks=req_blocks, + kv_caches_base_addr=list(range(n)), + block_lens=[4096] * n, + kv_block_lens=[3000] * n, + registered_layer_names=names, + registered_layer_indices=list(range(n)), + registered_group_indices=[i % 5 for i in range(n)], + registered_shared_group_ids=[[] for _ in range(n)], + registered_row_offsets=[i * 100 for i in range(n)], + ) + steady = MooncakeXferMetadata( + remote_hostname="h", + remote_port=1, + remote_tp_size=4, + remote_tp_rank=0, + layout_id=1, + req_blocks=req_blocks, + kv_caches_base_addr=[], + block_lens=[], + kv_block_lens=[], + ) + encoder = msgspec.msgpack.Encoder() + full_bytes = len(encoder.encode(full)) + steady_bytes = len(encoder.encode(steady)) + assert steady_bytes < full_bytes * 0.2 + + worker, _, regions, _ = _register_sliced_packed_mla([0, 1]) + calls = 0 + original = worker._get_transfer_regions + + def counting(*args, **kwargs): + nonlocal calls + calls += 1 + return original(*args, **kwargs) + + worker._get_transfer_regions = counting + + def handshake(layout_id: int) -> MooncakeXferMetadata: + return _xfer_meta( + regions, + req_blocks, + remote_hostname="peer", + remote_port=9, + remote_tp_size=worker.tp_size, + remote_pp_size=worker.pp_size, + layout_id=layout_id, + ) + + prepared = worker._prepare_transfer_regions(handshake(1)) + assert prepared[2] is None + assert calls == 2 + stored = worker._layout_by_peer[worker._peer_layout_key(handshake(1))] + assert stored == (prepared[0], prepared[1]) + + # Same layout under a new id hits the tuple cache and must still be stored. + worker._prepare_transfer_regions(handshake(3)) + assert calls == 2 + assert worker._peer_layout_key(handshake(3)) in worker._layout_by_peer + + sock = MagicMock() + sock.send_multipart = AsyncMock() + await worker.send_kv_to_decode(b"d", sock, _short_xfer_meta(worker, 5, req_blocks)) + _, payload = sock.send_multipart.call_args[0][0] + response = worker._xfer_resp_decoder.decode(payload) + assert response.status == MooncakeXferResponseStatus.LAYOUT_MISS + assert calls == 2 + + +@pytest.mark.asyncio +async def test_peer_layouts_evict_least_recently_used(monkeypatch): + """Stale layout ids are bounded, and an evicted peer is asked to resend.""" + monkeypatch.setattr(mooncake_connector, "_MAX_PEER_LAYOUTS", 2) + worker, _, regions, _ = _register_sliced_packed_mla([0]) + req_blocks = {"d": ("xfer", [[1]])} + + def handshake(layout_id: int) -> MooncakeXferMetadata: + return _xfer_meta( + regions, + req_blocks, + remote_hostname="peer", + remote_port=9, + remote_tp_size=worker.tp_size, + remote_pp_size=worker.pp_size, + layout_id=layout_id, + ) + + worker._prepare_transfer_regions(handshake(1)) + worker._prepare_transfer_regions(handshake(3)) + assert worker._lookup_peer_layout(_short_xfer_meta(worker, 1, req_blocks)) + worker._prepare_transfer_regions(handshake(5)) + + assert worker._lookup_peer_layout(_short_xfer_meta(worker, 1, req_blocks)) + assert worker._lookup_peer_layout(_short_xfer_meta(worker, 5, req_blocks)) + sock = MagicMock() + sock.send_multipart = AsyncMock() + await worker.send_kv_to_decode(b"d", sock, _short_xfer_meta(worker, 3, req_blocks)) + _, payload = sock.send_multipart.call_args[0][0] + response = worker._xfer_resp_decoder.decode(payload) + assert response.status == MooncakeXferResponseStatus.LAYOUT_MISS + + +class _ScriptedSock: + def __init__(self, responses: list[bytes], sent: list[bytes]): + self._responses = responses + self._sent = sent + + def setsockopt(self, *_args, **_kwargs): + return None + + async def send(self, data: bytes): + self._sent.append(data) + + async def recv(self): + return self._responses.pop(0) + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + +def _single_pull() -> dict[str, PullReqMeta]: + return { + "d": PullReqMeta( + d_req_id="d", + transfer_id="t", + local_block_ids=[[1]], + remote_engine_id="p", + remote_bootstrap_addr="http://bootstrap", + ) + } + + +async def _pull_once(worker, addr: str, responses: list[MooncakeXferResponse]): + sent: list[bytes] = [] + encoded = [worker._encoder.encode(response) for response in responses] + with patch( + "vllm.distributed.kv_transfer.kv_connector.v1.mooncake." + "mooncake_connector.make_zmq_socket", + return_value=_ScriptedSock(encoded, sent), + ): + await worker.receive_kv_from_single_worker(addr, _single_pull()) + return sent + + +@pytest.mark.asyncio +async def test_pull_resends_layout_when_prefill_forgot_it(): + worker, _, _, _ = _register_sliced_packed_mla([0]) + addr = "tcp://peer:1" + worker._acked_layout_peers.add(addr) + sent = await _pull_once( + worker, + addr, + [ + MooncakeXferResponse(status=MooncakeXferResponseStatus.LAYOUT_MISS), + MooncakeXferResponse( + status=MooncakeXferResponseStatus.FINISH, + ok_reqs=["d"], + layout_id=worker._kv_layout_id, + ), + ], + ) + assert len(sent) == 2 + assert len(sent[1]) > len(sent[0]) + assert addr in worker._acked_layout_peers + + +@pytest.mark.asyncio +async def test_pull_keeps_layout_for_prefill_that_does_not_echo_it(): + """A prefill without layout_id support never gets a layout-less pull.""" + worker, _, _, _ = _register_sliced_packed_mla([0]) + addr = "tcp://old-peer:1" + finish = MooncakeXferResponse( + status=MooncakeXferResponseStatus.FINISH, ok_reqs=["d"] + ) + first = await _pull_once(worker, addr, [finish]) + assert addr not in worker._acked_layout_peers + second = await _pull_once(worker, addr, [finish]) + assert len(second[0]) == len(first[0]) + + def test_coalesce_keeps_a_run_inside_its_starting_row(): """The row bound is the run start, not the previous slice.""" row = 2500 diff --git a/tests/v1/kv_connector/unit/test_mooncake_connector_hybrid_mamba.py b/tests/v1/kv_connector/unit/test_mooncake_connector_hybrid_mamba.py index 65abd910bb..d64c5ae8b5 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_connector_hybrid_mamba.py +++ b/tests/v1/kv_connector/unit/test_mooncake_connector_hybrid_mamba.py @@ -8,6 +8,7 @@ validated by this test module. """ import asyncio +from collections import OrderedDict from types import SimpleNamespace from unittest.mock import patch @@ -243,8 +244,10 @@ async def test_build_transfer_params_uses_non_overlapping_physical_pages(): kv_cache_config = make_hybrid_gdn_kv_cache_config(block_size=16) worker = object.__new__(MooncakeConnectorWorker) worker.shutdown = noop_shutdown - # __new__ skips __init__, where this cache is created. register_kv_caches clears it. + # __new__ skips __init__, where these caches are created. worker._prepared_transfer_regions = {} + worker._layout_by_peer = OrderedDict() + worker._acked_layout_peers = set() worker.use_mla = False worker.engine = SimpleNamespace(batch_register_memory=lambda *_: 0) worker.is_kv_consumer = True diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py index bc70d7cba8..1d12bd1ab1 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py @@ -5,9 +5,10 @@ import itertools import logging import os import queue +import secrets import threading import time -from collections import defaultdict +from collections import OrderedDict, defaultdict from collections.abc import Collection from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass @@ -76,6 +77,9 @@ from vllm.v1.worker.utils import select_common_block_size logger = init_logger(__name__) _BOOTSTRAP_MAX_ATTEMPTS: Final[int] = 3 +# Decode restarts mint new layout ids, so stale peer layouts pile up. Evicting +# one only costs that peer a LAYOUT_MISS and one layout resend. +_MAX_PEER_LAYOUTS: Final[int] = 1024 try: from mooncake.engine import TransferEngine @@ -623,6 +627,9 @@ class MooncakeXferMetadata( # peer did not send the field, so the run must not be promoted. registered_row_offsets: list[int] = msgspec.field(default_factory=list) remote_pp_size: int = 1 + # Random per register_kv_caches, so ids never repeat across restarts. + # 0 means the peer sends the layout on every pull. + layout_id: int = 0 class MooncakeXferResponseStatus(IntEnum): @@ -632,6 +639,9 @@ class MooncakeXferResponseStatus(IntEnum): CONTINUE = 1 # Something wrong, see err_msg ERROR = 2 + # P does not hold the layout for this layout_id; resend it. + # Only sent for pulls with a nonzero layout_id and no layout. + LAYOUT_MISS = 3 class MooncakeXferResponse( @@ -642,6 +652,9 @@ class MooncakeXferResponse( ok_reqs: list[ReqId] | None = None err_reqs: list[ReqId] | None = None err_msg: str | None = None + # The pull's layout_id once P holds that layout. D omits the layout only + # after seeing it, so a P that predates this field always gets it. + layout_id: int = 0 @dataclass @@ -1272,6 +1285,14 @@ class MooncakeConnectorWorker: tuple, tuple[list[TransferRegion], list[TransferRegion], str | None], ] = {} + # (host, port, tp_rank, pp_size, tp_size, layout_id) -> regions, in LRU + # order. Filled only after a pull that actually carried the layout. + self._layout_by_peer: OrderedDict[ + tuple, tuple[list[TransferRegion], list[TransferRegion]] + ] = OrderedDict() + self._kv_layout_id = 0 + # Decode side: worker addresses that echoed the current layout_id. + self._acked_layout_peers: set[str] = set() assert (parallel_config := vllm_config.parallel_config) dp_rank = parallel_config.data_parallel_index @@ -1561,14 +1582,30 @@ class MooncakeConnectorWorker: ) await sock.send_multipart((identity, self._encoder.encode(response))) return - local_regions, remote_regions, prep_err = self._prepare_transfer_regions(meta) - if prep_err is not None: - response = MooncakeXferResponse( - status=MooncakeXferResponseStatus.ERROR, - err_msg=prep_err, + if meta.layout_id and not meta.kv_caches_base_addr: + stored = self._lookup_peer_layout(meta) + if stored is None: + response = MooncakeXferResponse( + status=MooncakeXferResponseStatus.LAYOUT_MISS, + err_msg=( + f"Mooncake KV layout {meta.layout_id:#x} from " + f"{meta.remote_hostname}:{meta.remote_port} is not cached." + ), + ) + await sock.send_multipart((identity, self._encoder.encode(response))) + return + local_regions, remote_regions = stored + else: + local_regions, remote_regions, prep_err = self._prepare_transfer_regions( + meta ) - await sock.send_multipart((identity, self._encoder.encode(response))) - return + if prep_err is not None: + response = MooncakeXferResponse( + status=MooncakeXferResponseStatus.ERROR, + err_msg=prep_err, + ) + await sock.send_multipart((identity, self._encoder.encode(response))) + return validation_err = _validate_asymmetric_region_lengths( local_regions=local_regions, remote_regions=remote_regions, @@ -1629,6 +1666,7 @@ class MooncakeConnectorWorker: status=MooncakeXferResponseStatus.FINISH, err_reqs=list(pending_reqs), err_msg="Timeout waiting for P side ready.", + layout_id=meta.layout_id, ) await sock.send_multipart((identity, self._encoder.encode(response))) break @@ -1719,6 +1757,7 @@ class MooncakeConnectorWorker: ok_reqs=[d_req_id for d_req_id, _ in ok_ready_reqs] or None, err_reqs=err_reqs or None, err_msg=err_msg, + layout_id=meta.layout_id, ) await sock.send_multipart((identity, self._encoder.encode(response))) @@ -2018,6 +2057,10 @@ class MooncakeConnectorWorker: self.region_row_offsets = [] self.opaque_packed_storages = set() self._prepared_transfer_regions.clear() + self._layout_by_peer.clear() + self._acked_layout_peers.clear() + # Odd, so never 0 (the "layout on every pull" marker). + self._kv_layout_id = secrets.randbits(63) | 1 packed_storage_to_region: dict[int, int] = {} packed_view_to_region: dict[tuple[int, int, int], int] = {} @@ -2357,64 +2400,74 @@ class MooncakeConnectorWorker: pull_metas: dict[ReqId, PullReqMeta], ): req_ids = set(pull_metas) - metadata = MooncakeXferMetadata( - remote_hostname=self.hostname, - remote_port=self.rpc_port, - remote_tp_size=self.tp_size, - remote_tp_rank=self.tp_rank, - remote_pp_size=self.pp_size, - req_blocks={ - req_id: (pull_meta.transfer_id, pull_meta.local_block_ids) - for req_id, pull_meta in pull_metas.items() - }, - kv_caches_base_addr=self.kv_caches_base_addr, - block_lens=self.block_len_per_layer, - kv_block_lens=self.kv_block_len_per_layer, - registered_layer_names=self.registered_layer_names, - registered_layer_indices=self.registered_layer_indices, - registered_group_indices=self.registered_group_indices, - registered_shared_group_ids=[ - list(groups) for groups in self.region_shared_groups - ], - registered_row_offsets=self.region_row_offsets, + include_layout = ( + not self._kv_layout_id or worker_addr not in self._acked_layout_peers ) - - encoded_data = self._encoder.encode(metadata) - logger.debug( - "Size of encoded MooncakeXferMetadata: %d bytes", len(encoded_data) - ) - logger.debug( - "Sending kv transfer request for %s on path: %s", req_ids, worker_addr - ) - - # Send query for the request. - try: - with make_zmq_socket( - self.async_zmq_ctx, worker_addr, zmq.DEALER, bind=False, linger=0 - ) as sock: - # If something goes wrong, let P wait timeout first (in asyncio.wait()). - sock.setsockopt( - zmq.RCVTIMEO, (envs.VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT + 60) * 1000 + # Each pass either returns or breaks out of the recv loop to resend + # the layout after a LAYOUT_MISS. + while True: + metadata = self._build_pull_metadata( + pull_metas, include_layout=include_layout + ) + encoded_data = self._encoder.encode(metadata) + logger.debug( + "MooncakeXferMetadata to %s is %d bytes (%s).", + worker_addr, + len(encoded_data), + "layout" if include_layout else "blocks", + ) + logger.debug( + "Sending kv transfer request for %s on path: %s", req_ids, worker_addr + ) + try: + with make_zmq_socket( + self.async_zmq_ctx, worker_addr, zmq.DEALER, bind=False, linger=0 + ) as sock: + sock.setsockopt( + zmq.RCVTIMEO, + (envs.VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT + 60) * 1000, + ) + await sock.send(encoded_data) + while True: + ret_msg = await sock.recv() + response = self._xfer_resp_decoder.decode(ret_msg) + status = response.status + if status == MooncakeXferResponseStatus.LAYOUT_MISS: + self._acked_layout_peers.discard(worker_addr) + if not include_layout: + logger.info( + "Mooncake prefill %s lost the KV layout; " + "resending it.", + worker_addr, + ) + include_layout = True + break + if status in ( + MooncakeXferResponseStatus.ERROR, + MooncakeXferResponseStatus.LAYOUT_MISS, + ): + self._handle_failed_recv( + pull_metas, + req_ids, + response.err_msg or "transfer error", + ) + return + if ( + self._kv_layout_id + and response.layout_id == self._kv_layout_id + ): + self._acked_layout_peers.add(worker_addr) + self.process_pulling_result(response, pull_metas) + if status == MooncakeXferResponseStatus.FINISH: + return + except zmq.ContextTerminated: + logger.debug( + "ZMQ context terminated, exiting Mooncake receiver thread." ) - await sock.send(encoded_data) - while True: - ret_msg = await sock.recv() - response = self._xfer_resp_decoder.decode(ret_msg) - if response.status == MooncakeXferResponseStatus.ERROR: - self._handle_failed_recv( - pull_metas, - req_ids, - response.err_msg or "transfer error", - ) - return - self.process_pulling_result(response, pull_metas) - if response.status == MooncakeXferResponseStatus.FINISH: - break - except zmq.ContextTerminated: - logger.debug("ZMQ context terminated, exiting Mooncake receiver thread.") - except Exception as e: - self._handle_failed_recv(pull_metas, req_ids, f"transfer failed: {e}") - return + return + except Exception as e: + self._handle_failed_recv(pull_metas, req_ids, f"transfer failed: {e}") + return def _handle_failed_recv( self, @@ -2673,6 +2726,8 @@ class MooncakeConnectorWorker: ) cached = self._prepared_transfer_regions.get(cache_key) if cached is not None: + # A new layout_id can carry a layout this cache already holds. + self._remember_peer_layout(meta, cached) return cached def finish( @@ -2766,12 +2821,86 @@ class MooncakeConnectorWorker: n_aligned, len(local_regions), ) - self._prepared_transfer_regions[cache_key] = ( - local_regions, - remote_regions, - None, + prepared = (local_regions, remote_regions, None) + self._prepared_transfer_regions[cache_key] = prepared + self._remember_peer_layout(meta, prepared) + return prepared + + def _peer_layout_key(self, meta: MooncakeXferMetadata) -> tuple: + return ( + meta.remote_hostname, + meta.remote_port, + meta.remote_tp_rank, + meta.remote_pp_size, + meta.remote_tp_size, + meta.layout_id, + ) + + def _remember_peer_layout( + self, + meta: MooncakeXferMetadata, + prepared: tuple[list[TransferRegion], list[TransferRegion], str | None], + ) -> None: + local_regions, remote_regions, err = prepared + if err is not None or not meta.layout_id: + return + key = self._peer_layout_key(meta) + self._layout_by_peer[key] = (local_regions, remote_regions) + self._layout_by_peer.move_to_end(key) + while len(self._layout_by_peer) > _MAX_PEER_LAYOUTS: + self._layout_by_peer.popitem(last=False) + + def _lookup_peer_layout( + self, meta: MooncakeXferMetadata + ) -> tuple[list[TransferRegion], list[TransferRegion]] | None: + key = self._peer_layout_key(meta) + stored = self._layout_by_peer.get(key) + if stored is not None: + self._layout_by_peer.move_to_end(key) + return stored + + def _build_pull_metadata( + self, + pull_metas: dict[ReqId, PullReqMeta], + *, + include_layout: bool, + ) -> MooncakeXferMetadata: + req_blocks = { + req_id: (pull_meta.transfer_id, pull_meta.local_block_ids) + for req_id, pull_meta in pull_metas.items() + } + if not include_layout: + return MooncakeXferMetadata( + remote_hostname=self.hostname, + remote_port=self.rpc_port, + remote_tp_size=self.tp_size, + remote_tp_rank=self.tp_rank, + remote_pp_size=self.pp_size, + layout_id=self._kv_layout_id, + req_blocks=req_blocks, + kv_caches_base_addr=[], + block_lens=[], + kv_block_lens=[], + ) + return MooncakeXferMetadata( + remote_hostname=self.hostname, + remote_port=self.rpc_port, + remote_tp_size=self.tp_size, + remote_tp_rank=self.tp_rank, + remote_pp_size=self.pp_size, + layout_id=self._kv_layout_id, + req_blocks=req_blocks, + kv_caches_base_addr=self.kv_caches_base_addr, + block_lens=self.block_len_per_layer, + kv_block_lens=self.kv_block_len_per_layer, + registered_layer_names=self.registered_layer_names, + registered_layer_indices=self.registered_layer_indices, + registered_group_indices=self.registered_group_indices, + registered_shared_group_ids=[ + list(groups) for groups in self.region_shared_groups + ], + registered_row_offsets=self.region_row_offsets, ) - return local_regions, remote_regions, None def _validate_head_resharding_layout( self, remote_tp_size: int, local_regions: list[TransferRegion]