[Perf][KV Connector][Mooncake] Send the KV layout once per peer (#59825)
Signed-off-by: staryxchen <staryxchen@tencent.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com>
This commit is contained in:
co-authored by
Cursor
Jiangyun Zhu
parent
a400804ce4
commit
6605802904
@@ -11,6 +11,7 @@ from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import msgspec
|
||||
import pytest
|
||||
import torch
|
||||
import zmq.asyncio
|
||||
@@ -2229,6 +2230,215 @@ def test_prepare_transfer_regions_reuses_success_and_error():
|
||||
assert calls == 6
|
||||
|
||||
|
||||
def _short_xfer_meta(worker, layout_id: int, req_blocks: dict) -> MooncakeXferMetadata:
|
||||
return MooncakeXferMetadata(
|
||||
remote_hostname="peer",
|
||||
remote_port=9,
|
||||
remote_tp_size=worker.tp_size,
|
||||
remote_tp_rank=0,
|
||||
remote_pp_size=worker.pp_size,
|
||||
layout_id=layout_id,
|
||||
req_blocks=req_blocks,
|
||||
kv_caches_base_addr=[],
|
||||
block_lens=[],
|
||||
kv_block_lens=[],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_steady_pull_omits_layout_and_unknown_id_asks_for_resend():
|
||||
"""A later pull is much smaller, and an unseen layout_id is not guessed."""
|
||||
n = 146
|
||||
names = [f"model.layers.{i}.self_attn" for i in range(n)]
|
||||
req_blocks = {"d": ("xfer", [[1, 2, 3]])}
|
||||
full = MooncakeXferMetadata(
|
||||
remote_hostname="h",
|
||||
remote_port=1,
|
||||
remote_tp_size=4,
|
||||
remote_tp_rank=0,
|
||||
layout_id=1,
|
||||
req_blocks=req_blocks,
|
||||
kv_caches_base_addr=list(range(n)),
|
||||
block_lens=[4096] * n,
|
||||
kv_block_lens=[3000] * n,
|
||||
registered_layer_names=names,
|
||||
registered_layer_indices=list(range(n)),
|
||||
registered_group_indices=[i % 5 for i in range(n)],
|
||||
registered_shared_group_ids=[[] for _ in range(n)],
|
||||
registered_row_offsets=[i * 100 for i in range(n)],
|
||||
)
|
||||
steady = MooncakeXferMetadata(
|
||||
remote_hostname="h",
|
||||
remote_port=1,
|
||||
remote_tp_size=4,
|
||||
remote_tp_rank=0,
|
||||
layout_id=1,
|
||||
req_blocks=req_blocks,
|
||||
kv_caches_base_addr=[],
|
||||
block_lens=[],
|
||||
kv_block_lens=[],
|
||||
)
|
||||
encoder = msgspec.msgpack.Encoder()
|
||||
full_bytes = len(encoder.encode(full))
|
||||
steady_bytes = len(encoder.encode(steady))
|
||||
assert steady_bytes < full_bytes * 0.2
|
||||
|
||||
worker, _, regions, _ = _register_sliced_packed_mla([0, 1])
|
||||
calls = 0
|
||||
original = worker._get_transfer_regions
|
||||
|
||||
def counting(*args, **kwargs):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
return original(*args, **kwargs)
|
||||
|
||||
worker._get_transfer_regions = counting
|
||||
|
||||
def handshake(layout_id: int) -> MooncakeXferMetadata:
|
||||
return _xfer_meta(
|
||||
regions,
|
||||
req_blocks,
|
||||
remote_hostname="peer",
|
||||
remote_port=9,
|
||||
remote_tp_size=worker.tp_size,
|
||||
remote_pp_size=worker.pp_size,
|
||||
layout_id=layout_id,
|
||||
)
|
||||
|
||||
prepared = worker._prepare_transfer_regions(handshake(1))
|
||||
assert prepared[2] is None
|
||||
assert calls == 2
|
||||
stored = worker._layout_by_peer[worker._peer_layout_key(handshake(1))]
|
||||
assert stored == (prepared[0], prepared[1])
|
||||
|
||||
# Same layout under a new id hits the tuple cache and must still be stored.
|
||||
worker._prepare_transfer_regions(handshake(3))
|
||||
assert calls == 2
|
||||
assert worker._peer_layout_key(handshake(3)) in worker._layout_by_peer
|
||||
|
||||
sock = MagicMock()
|
||||
sock.send_multipart = AsyncMock()
|
||||
await worker.send_kv_to_decode(b"d", sock, _short_xfer_meta(worker, 5, req_blocks))
|
||||
_, payload = sock.send_multipart.call_args[0][0]
|
||||
response = worker._xfer_resp_decoder.decode(payload)
|
||||
assert response.status == MooncakeXferResponseStatus.LAYOUT_MISS
|
||||
assert calls == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_peer_layouts_evict_least_recently_used(monkeypatch):
|
||||
"""Stale layout ids are bounded, and an evicted peer is asked to resend."""
|
||||
monkeypatch.setattr(mooncake_connector, "_MAX_PEER_LAYOUTS", 2)
|
||||
worker, _, regions, _ = _register_sliced_packed_mla([0])
|
||||
req_blocks = {"d": ("xfer", [[1]])}
|
||||
|
||||
def handshake(layout_id: int) -> MooncakeXferMetadata:
|
||||
return _xfer_meta(
|
||||
regions,
|
||||
req_blocks,
|
||||
remote_hostname="peer",
|
||||
remote_port=9,
|
||||
remote_tp_size=worker.tp_size,
|
||||
remote_pp_size=worker.pp_size,
|
||||
layout_id=layout_id,
|
||||
)
|
||||
|
||||
worker._prepare_transfer_regions(handshake(1))
|
||||
worker._prepare_transfer_regions(handshake(3))
|
||||
assert worker._lookup_peer_layout(_short_xfer_meta(worker, 1, req_blocks))
|
||||
worker._prepare_transfer_regions(handshake(5))
|
||||
|
||||
assert worker._lookup_peer_layout(_short_xfer_meta(worker, 1, req_blocks))
|
||||
assert worker._lookup_peer_layout(_short_xfer_meta(worker, 5, req_blocks))
|
||||
sock = MagicMock()
|
||||
sock.send_multipart = AsyncMock()
|
||||
await worker.send_kv_to_decode(b"d", sock, _short_xfer_meta(worker, 3, req_blocks))
|
||||
_, payload = sock.send_multipart.call_args[0][0]
|
||||
response = worker._xfer_resp_decoder.decode(payload)
|
||||
assert response.status == MooncakeXferResponseStatus.LAYOUT_MISS
|
||||
|
||||
|
||||
class _ScriptedSock:
|
||||
def __init__(self, responses: list[bytes], sent: list[bytes]):
|
||||
self._responses = responses
|
||||
self._sent = sent
|
||||
|
||||
def setsockopt(self, *_args, **_kwargs):
|
||||
return None
|
||||
|
||||
async def send(self, data: bytes):
|
||||
self._sent.append(data)
|
||||
|
||||
async def recv(self):
|
||||
return self._responses.pop(0)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
|
||||
def _single_pull() -> dict[str, PullReqMeta]:
|
||||
return {
|
||||
"d": PullReqMeta(
|
||||
d_req_id="d",
|
||||
transfer_id="t",
|
||||
local_block_ids=[[1]],
|
||||
remote_engine_id="p",
|
||||
remote_bootstrap_addr="http://bootstrap",
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
async def _pull_once(worker, addr: str, responses: list[MooncakeXferResponse]):
|
||||
sent: list[bytes] = []
|
||||
encoded = [worker._encoder.encode(response) for response in responses]
|
||||
with patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake."
|
||||
"mooncake_connector.make_zmq_socket",
|
||||
return_value=_ScriptedSock(encoded, sent),
|
||||
):
|
||||
await worker.receive_kv_from_single_worker(addr, _single_pull())
|
||||
return sent
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pull_resends_layout_when_prefill_forgot_it():
|
||||
worker, _, _, _ = _register_sliced_packed_mla([0])
|
||||
addr = "tcp://peer:1"
|
||||
worker._acked_layout_peers.add(addr)
|
||||
sent = await _pull_once(
|
||||
worker,
|
||||
addr,
|
||||
[
|
||||
MooncakeXferResponse(status=MooncakeXferResponseStatus.LAYOUT_MISS),
|
||||
MooncakeXferResponse(
|
||||
status=MooncakeXferResponseStatus.FINISH,
|
||||
ok_reqs=["d"],
|
||||
layout_id=worker._kv_layout_id,
|
||||
),
|
||||
],
|
||||
)
|
||||
assert len(sent) == 2
|
||||
assert len(sent[1]) > len(sent[0])
|
||||
assert addr in worker._acked_layout_peers
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pull_keeps_layout_for_prefill_that_does_not_echo_it():
|
||||
"""A prefill without layout_id support never gets a layout-less pull."""
|
||||
worker, _, _, _ = _register_sliced_packed_mla([0])
|
||||
addr = "tcp://old-peer:1"
|
||||
finish = MooncakeXferResponse(
|
||||
status=MooncakeXferResponseStatus.FINISH, ok_reqs=["d"]
|
||||
)
|
||||
first = await _pull_once(worker, addr, [finish])
|
||||
assert addr not in worker._acked_layout_peers
|
||||
second = await _pull_once(worker, addr, [finish])
|
||||
assert len(second[0]) == len(first[0])
|
||||
|
||||
|
||||
def test_coalesce_keeps_a_run_inside_its_starting_row():
|
||||
"""The row bound is the run start, not the previous slice."""
|
||||
row = 2500
|
||||
|
||||
@@ -8,6 +8,7 @@ validated by this test module.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from collections import OrderedDict
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
@@ -243,8 +244,10 @@ async def test_build_transfer_params_uses_non_overlapping_physical_pages():
|
||||
kv_cache_config = make_hybrid_gdn_kv_cache_config(block_size=16)
|
||||
worker = object.__new__(MooncakeConnectorWorker)
|
||||
worker.shutdown = noop_shutdown
|
||||
# __new__ skips __init__, where this cache is created. register_kv_caches clears it.
|
||||
# __new__ skips __init__, where these caches are created.
|
||||
worker._prepared_transfer_regions = {}
|
||||
worker._layout_by_peer = OrderedDict()
|
||||
worker._acked_layout_peers = set()
|
||||
worker.use_mla = False
|
||||
worker.engine = SimpleNamespace(batch_register_memory=lambda *_: 0)
|
||||
worker.is_kv_consumer = True
|
||||
|
||||
@@ -5,9 +5,10 @@ import itertools
|
||||
import logging
|
||||
import os
|
||||
import queue
|
||||
import secrets
|
||||
import threading
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from collections import OrderedDict, defaultdict
|
||||
from collections.abc import Collection
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
@@ -76,6 +77,9 @@ from vllm.v1.worker.utils import select_common_block_size
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_BOOTSTRAP_MAX_ATTEMPTS: Final[int] = 3
|
||||
# Decode restarts mint new layout ids, so stale peer layouts pile up. Evicting
|
||||
# one only costs that peer a LAYOUT_MISS and one layout resend.
|
||||
_MAX_PEER_LAYOUTS: Final[int] = 1024
|
||||
|
||||
try:
|
||||
from mooncake.engine import TransferEngine
|
||||
@@ -623,6 +627,9 @@ class MooncakeXferMetadata(
|
||||
# peer did not send the field, so the run must not be promoted.
|
||||
registered_row_offsets: list[int] = msgspec.field(default_factory=list)
|
||||
remote_pp_size: int = 1
|
||||
# Random per register_kv_caches, so ids never repeat across restarts.
|
||||
# 0 means the peer sends the layout on every pull.
|
||||
layout_id: int = 0
|
||||
|
||||
|
||||
class MooncakeXferResponseStatus(IntEnum):
|
||||
@@ -632,6 +639,9 @@ class MooncakeXferResponseStatus(IntEnum):
|
||||
CONTINUE = 1
|
||||
# Something wrong, see err_msg
|
||||
ERROR = 2
|
||||
# P does not hold the layout for this layout_id; resend it.
|
||||
# Only sent for pulls with a nonzero layout_id and no layout.
|
||||
LAYOUT_MISS = 3
|
||||
|
||||
|
||||
class MooncakeXferResponse(
|
||||
@@ -642,6 +652,9 @@ class MooncakeXferResponse(
|
||||
ok_reqs: list[ReqId] | None = None
|
||||
err_reqs: list[ReqId] | None = None
|
||||
err_msg: str | None = None
|
||||
# The pull's layout_id once P holds that layout. D omits the layout only
|
||||
# after seeing it, so a P that predates this field always gets it.
|
||||
layout_id: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -1272,6 +1285,14 @@ class MooncakeConnectorWorker:
|
||||
tuple,
|
||||
tuple[list[TransferRegion], list[TransferRegion], str | None],
|
||||
] = {}
|
||||
# (host, port, tp_rank, pp_size, tp_size, layout_id) -> regions, in LRU
|
||||
# order. Filled only after a pull that actually carried the layout.
|
||||
self._layout_by_peer: OrderedDict[
|
||||
tuple, tuple[list[TransferRegion], list[TransferRegion]]
|
||||
] = OrderedDict()
|
||||
self._kv_layout_id = 0
|
||||
# Decode side: worker addresses that echoed the current layout_id.
|
||||
self._acked_layout_peers: set[str] = set()
|
||||
|
||||
assert (parallel_config := vllm_config.parallel_config)
|
||||
dp_rank = parallel_config.data_parallel_index
|
||||
@@ -1561,14 +1582,30 @@ class MooncakeConnectorWorker:
|
||||
)
|
||||
await sock.send_multipart((identity, self._encoder.encode(response)))
|
||||
return
|
||||
local_regions, remote_regions, prep_err = self._prepare_transfer_regions(meta)
|
||||
if prep_err is not None:
|
||||
response = MooncakeXferResponse(
|
||||
status=MooncakeXferResponseStatus.ERROR,
|
||||
err_msg=prep_err,
|
||||
if meta.layout_id and not meta.kv_caches_base_addr:
|
||||
stored = self._lookup_peer_layout(meta)
|
||||
if stored is None:
|
||||
response = MooncakeXferResponse(
|
||||
status=MooncakeXferResponseStatus.LAYOUT_MISS,
|
||||
err_msg=(
|
||||
f"Mooncake KV layout {meta.layout_id:#x} from "
|
||||
f"{meta.remote_hostname}:{meta.remote_port} is not cached."
|
||||
),
|
||||
)
|
||||
await sock.send_multipart((identity, self._encoder.encode(response)))
|
||||
return
|
||||
local_regions, remote_regions = stored
|
||||
else:
|
||||
local_regions, remote_regions, prep_err = self._prepare_transfer_regions(
|
||||
meta
|
||||
)
|
||||
await sock.send_multipart((identity, self._encoder.encode(response)))
|
||||
return
|
||||
if prep_err is not None:
|
||||
response = MooncakeXferResponse(
|
||||
status=MooncakeXferResponseStatus.ERROR,
|
||||
err_msg=prep_err,
|
||||
)
|
||||
await sock.send_multipart((identity, self._encoder.encode(response)))
|
||||
return
|
||||
validation_err = _validate_asymmetric_region_lengths(
|
||||
local_regions=local_regions,
|
||||
remote_regions=remote_regions,
|
||||
@@ -1629,6 +1666,7 @@ class MooncakeConnectorWorker:
|
||||
status=MooncakeXferResponseStatus.FINISH,
|
||||
err_reqs=list(pending_reqs),
|
||||
err_msg="Timeout waiting for P side ready.",
|
||||
layout_id=meta.layout_id,
|
||||
)
|
||||
await sock.send_multipart((identity, self._encoder.encode(response)))
|
||||
break
|
||||
@@ -1719,6 +1757,7 @@ class MooncakeConnectorWorker:
|
||||
ok_reqs=[d_req_id for d_req_id, _ in ok_ready_reqs] or None,
|
||||
err_reqs=err_reqs or None,
|
||||
err_msg=err_msg,
|
||||
layout_id=meta.layout_id,
|
||||
)
|
||||
await sock.send_multipart((identity, self._encoder.encode(response)))
|
||||
|
||||
@@ -2018,6 +2057,10 @@ class MooncakeConnectorWorker:
|
||||
self.region_row_offsets = []
|
||||
self.opaque_packed_storages = set()
|
||||
self._prepared_transfer_regions.clear()
|
||||
self._layout_by_peer.clear()
|
||||
self._acked_layout_peers.clear()
|
||||
# Odd, so never 0 (the "layout on every pull" marker).
|
||||
self._kv_layout_id = secrets.randbits(63) | 1
|
||||
|
||||
packed_storage_to_region: dict[int, int] = {}
|
||||
packed_view_to_region: dict[tuple[int, int, int], int] = {}
|
||||
@@ -2357,64 +2400,74 @@ class MooncakeConnectorWorker:
|
||||
pull_metas: dict[ReqId, PullReqMeta],
|
||||
):
|
||||
req_ids = set(pull_metas)
|
||||
metadata = MooncakeXferMetadata(
|
||||
remote_hostname=self.hostname,
|
||||
remote_port=self.rpc_port,
|
||||
remote_tp_size=self.tp_size,
|
||||
remote_tp_rank=self.tp_rank,
|
||||
remote_pp_size=self.pp_size,
|
||||
req_blocks={
|
||||
req_id: (pull_meta.transfer_id, pull_meta.local_block_ids)
|
||||
for req_id, pull_meta in pull_metas.items()
|
||||
},
|
||||
kv_caches_base_addr=self.kv_caches_base_addr,
|
||||
block_lens=self.block_len_per_layer,
|
||||
kv_block_lens=self.kv_block_len_per_layer,
|
||||
registered_layer_names=self.registered_layer_names,
|
||||
registered_layer_indices=self.registered_layer_indices,
|
||||
registered_group_indices=self.registered_group_indices,
|
||||
registered_shared_group_ids=[
|
||||
list(groups) for groups in self.region_shared_groups
|
||||
],
|
||||
registered_row_offsets=self.region_row_offsets,
|
||||
include_layout = (
|
||||
not self._kv_layout_id or worker_addr not in self._acked_layout_peers
|
||||
)
|
||||
|
||||
encoded_data = self._encoder.encode(metadata)
|
||||
logger.debug(
|
||||
"Size of encoded MooncakeXferMetadata: %d bytes", len(encoded_data)
|
||||
)
|
||||
logger.debug(
|
||||
"Sending kv transfer request for %s on path: %s", req_ids, worker_addr
|
||||
)
|
||||
|
||||
# Send query for the request.
|
||||
try:
|
||||
with make_zmq_socket(
|
||||
self.async_zmq_ctx, worker_addr, zmq.DEALER, bind=False, linger=0
|
||||
) as sock:
|
||||
# If something goes wrong, let P wait timeout first (in asyncio.wait()).
|
||||
sock.setsockopt(
|
||||
zmq.RCVTIMEO, (envs.VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT + 60) * 1000
|
||||
# Each pass either returns or breaks out of the recv loop to resend
|
||||
# the layout after a LAYOUT_MISS.
|
||||
while True:
|
||||
metadata = self._build_pull_metadata(
|
||||
pull_metas, include_layout=include_layout
|
||||
)
|
||||
encoded_data = self._encoder.encode(metadata)
|
||||
logger.debug(
|
||||
"MooncakeXferMetadata to %s is %d bytes (%s).",
|
||||
worker_addr,
|
||||
len(encoded_data),
|
||||
"layout" if include_layout else "blocks",
|
||||
)
|
||||
logger.debug(
|
||||
"Sending kv transfer request for %s on path: %s", req_ids, worker_addr
|
||||
)
|
||||
try:
|
||||
with make_zmq_socket(
|
||||
self.async_zmq_ctx, worker_addr, zmq.DEALER, bind=False, linger=0
|
||||
) as sock:
|
||||
sock.setsockopt(
|
||||
zmq.RCVTIMEO,
|
||||
(envs.VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT + 60) * 1000,
|
||||
)
|
||||
await sock.send(encoded_data)
|
||||
while True:
|
||||
ret_msg = await sock.recv()
|
||||
response = self._xfer_resp_decoder.decode(ret_msg)
|
||||
status = response.status
|
||||
if status == MooncakeXferResponseStatus.LAYOUT_MISS:
|
||||
self._acked_layout_peers.discard(worker_addr)
|
||||
if not include_layout:
|
||||
logger.info(
|
||||
"Mooncake prefill %s lost the KV layout; "
|
||||
"resending it.",
|
||||
worker_addr,
|
||||
)
|
||||
include_layout = True
|
||||
break
|
||||
if status in (
|
||||
MooncakeXferResponseStatus.ERROR,
|
||||
MooncakeXferResponseStatus.LAYOUT_MISS,
|
||||
):
|
||||
self._handle_failed_recv(
|
||||
pull_metas,
|
||||
req_ids,
|
||||
response.err_msg or "transfer error",
|
||||
)
|
||||
return
|
||||
if (
|
||||
self._kv_layout_id
|
||||
and response.layout_id == self._kv_layout_id
|
||||
):
|
||||
self._acked_layout_peers.add(worker_addr)
|
||||
self.process_pulling_result(response, pull_metas)
|
||||
if status == MooncakeXferResponseStatus.FINISH:
|
||||
return
|
||||
except zmq.ContextTerminated:
|
||||
logger.debug(
|
||||
"ZMQ context terminated, exiting Mooncake receiver thread."
|
||||
)
|
||||
await sock.send(encoded_data)
|
||||
while True:
|
||||
ret_msg = await sock.recv()
|
||||
response = self._xfer_resp_decoder.decode(ret_msg)
|
||||
if response.status == MooncakeXferResponseStatus.ERROR:
|
||||
self._handle_failed_recv(
|
||||
pull_metas,
|
||||
req_ids,
|
||||
response.err_msg or "transfer error",
|
||||
)
|
||||
return
|
||||
self.process_pulling_result(response, pull_metas)
|
||||
if response.status == MooncakeXferResponseStatus.FINISH:
|
||||
break
|
||||
except zmq.ContextTerminated:
|
||||
logger.debug("ZMQ context terminated, exiting Mooncake receiver thread.")
|
||||
except Exception as e:
|
||||
self._handle_failed_recv(pull_metas, req_ids, f"transfer failed: {e}")
|
||||
return
|
||||
return
|
||||
except Exception as e:
|
||||
self._handle_failed_recv(pull_metas, req_ids, f"transfer failed: {e}")
|
||||
return
|
||||
|
||||
def _handle_failed_recv(
|
||||
self,
|
||||
@@ -2673,6 +2726,8 @@ class MooncakeConnectorWorker:
|
||||
)
|
||||
cached = self._prepared_transfer_regions.get(cache_key)
|
||||
if cached is not None:
|
||||
# A new layout_id can carry a layout this cache already holds.
|
||||
self._remember_peer_layout(meta, cached)
|
||||
return cached
|
||||
|
||||
def finish(
|
||||
@@ -2766,12 +2821,86 @@ class MooncakeConnectorWorker:
|
||||
n_aligned,
|
||||
len(local_regions),
|
||||
)
|
||||
self._prepared_transfer_regions[cache_key] = (
|
||||
local_regions,
|
||||
remote_regions,
|
||||
None,
|
||||
prepared = (local_regions, remote_regions, None)
|
||||
self._prepared_transfer_regions[cache_key] = prepared
|
||||
self._remember_peer_layout(meta, prepared)
|
||||
return prepared
|
||||
|
||||
def _peer_layout_key(self, meta: MooncakeXferMetadata) -> tuple:
|
||||
return (
|
||||
meta.remote_hostname,
|
||||
meta.remote_port,
|
||||
meta.remote_tp_rank,
|
||||
meta.remote_pp_size,
|
||||
meta.remote_tp_size,
|
||||
meta.layout_id,
|
||||
)
|
||||
|
||||
def _remember_peer_layout(
|
||||
self,
|
||||
meta: MooncakeXferMetadata,
|
||||
prepared: tuple[list[TransferRegion], list[TransferRegion], str | None],
|
||||
) -> None:
|
||||
local_regions, remote_regions, err = prepared
|
||||
if err is not None or not meta.layout_id:
|
||||
return
|
||||
key = self._peer_layout_key(meta)
|
||||
self._layout_by_peer[key] = (local_regions, remote_regions)
|
||||
self._layout_by_peer.move_to_end(key)
|
||||
while len(self._layout_by_peer) > _MAX_PEER_LAYOUTS:
|
||||
self._layout_by_peer.popitem(last=False)
|
||||
|
||||
def _lookup_peer_layout(
|
||||
self, meta: MooncakeXferMetadata
|
||||
) -> tuple[list[TransferRegion], list[TransferRegion]] | None:
|
||||
key = self._peer_layout_key(meta)
|
||||
stored = self._layout_by_peer.get(key)
|
||||
if stored is not None:
|
||||
self._layout_by_peer.move_to_end(key)
|
||||
return stored
|
||||
|
||||
def _build_pull_metadata(
|
||||
self,
|
||||
pull_metas: dict[ReqId, PullReqMeta],
|
||||
*,
|
||||
include_layout: bool,
|
||||
) -> MooncakeXferMetadata:
|
||||
req_blocks = {
|
||||
req_id: (pull_meta.transfer_id, pull_meta.local_block_ids)
|
||||
for req_id, pull_meta in pull_metas.items()
|
||||
}
|
||||
if not include_layout:
|
||||
return MooncakeXferMetadata(
|
||||
remote_hostname=self.hostname,
|
||||
remote_port=self.rpc_port,
|
||||
remote_tp_size=self.tp_size,
|
||||
remote_tp_rank=self.tp_rank,
|
||||
remote_pp_size=self.pp_size,
|
||||
layout_id=self._kv_layout_id,
|
||||
req_blocks=req_blocks,
|
||||
kv_caches_base_addr=[],
|
||||
block_lens=[],
|
||||
kv_block_lens=[],
|
||||
)
|
||||
return MooncakeXferMetadata(
|
||||
remote_hostname=self.hostname,
|
||||
remote_port=self.rpc_port,
|
||||
remote_tp_size=self.tp_size,
|
||||
remote_tp_rank=self.tp_rank,
|
||||
remote_pp_size=self.pp_size,
|
||||
layout_id=self._kv_layout_id,
|
||||
req_blocks=req_blocks,
|
||||
kv_caches_base_addr=self.kv_caches_base_addr,
|
||||
block_lens=self.block_len_per_layer,
|
||||
kv_block_lens=self.kv_block_len_per_layer,
|
||||
registered_layer_names=self.registered_layer_names,
|
||||
registered_layer_indices=self.registered_layer_indices,
|
||||
registered_group_indices=self.registered_group_indices,
|
||||
registered_shared_group_ids=[
|
||||
list(groups) for groups in self.region_shared_groups
|
||||
],
|
||||
registered_row_offsets=self.region_row_offsets,
|
||||
)
|
||||
return local_regions, remote_regions, None
|
||||
|
||||
def _validate_head_resharding_layout(
|
||||
self, remote_tp_size: int, local_regions: list[TransferRegion]
|
||||
|
||||
Reference in New Issue
Block a user