[Perf][KV Connector][Mooncake] Pack hybrid/MLA KV into coalesced transfer regions (#57952)
Signed-off-by: staryxchen <staryxchen@tencent.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
co-authored by
Cursor
Yifan Qiao
parent
5dd693122c
commit
faacc13565
File diff suppressed because it is too large
Load Diff
@@ -242,6 +242,8 @@ async def test_build_transfer_params_uses_non_overlapping_physical_pages():
|
||||
kv_cache_config = make_hybrid_gdn_kv_cache_config(block_size=16)
|
||||
worker = object.__new__(MooncakeConnectorWorker)
|
||||
worker.shutdown = noop_shutdown
|
||||
# __new__ skips __init__, where this cache is created. register_kv_caches clears it.
|
||||
worker._prepared_transfer_regions = {}
|
||||
worker.use_mla = False
|
||||
worker.engine = SimpleNamespace(batch_register_memory=lambda *_: 0)
|
||||
worker.is_kv_consumer = True
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
import itertools
|
||||
import logging
|
||||
import queue
|
||||
import threading
|
||||
@@ -62,6 +63,8 @@ from vllm.v1.kv_cache_interface import (
|
||||
KpoolTailSpec,
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
MLAAttentionSpec,
|
||||
SlidingWindowMLASpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
from vllm.v1.request import RequestStatus
|
||||
@@ -90,6 +93,12 @@ if TYPE_CHECKING:
|
||||
ReqId = str # Internal scheduler request ID
|
||||
TransferId = str # KV transfer coordination ID (shared by P/D)
|
||||
|
||||
# Follow NIXL (`nixl/base_worker.py`): HMA packed pages can put more than one
|
||||
# KV group on a single allocation. group_index=-1 flattens every group's block
|
||||
# ids onto that region. Hybrid KV manager assigns distinct ids per group, so
|
||||
# flatten does not copy the same block twice.
|
||||
_SHARED_REGION_GROUP_ID = -1
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TransferRegion:
|
||||
@@ -99,6 +108,34 @@ class TransferRegion:
|
||||
block_len: int
|
||||
kv_block_len: int
|
||||
group_index: int = 0
|
||||
# Groups that share this allocation. Empty unless group_index is -1.
|
||||
shared_group_ids: tuple[int, ...] = ()
|
||||
# Byte offset of this view from the start of its block row.
|
||||
row_offset: int = 0
|
||||
|
||||
|
||||
def _block_ids_for_region(
|
||||
block_ids_by_group: list[list[int]],
|
||||
group_index: int,
|
||||
shared_group_ids: tuple[int, ...] = (),
|
||||
) -> list[int]:
|
||||
"""Follow NIXL `_block_ids_by_region`.
|
||||
|
||||
-1 flattens only the groups that share this allocation. NIXL can flatten
|
||||
every group because its packed flag means they all share one storage;
|
||||
Mooncake can still have a separate Mamba group in the same request.
|
||||
"""
|
||||
if group_index == _SHARED_REGION_GROUP_ID:
|
||||
assert shared_group_ids, "Shared packed region is missing its group ids."
|
||||
return list(
|
||||
itertools.chain.from_iterable(
|
||||
block_ids_by_group[group] for group in shared_group_ids
|
||||
)
|
||||
)
|
||||
assert 0 <= group_index < len(block_ids_by_group), (
|
||||
"Transfer region references a missing KV group."
|
||||
)
|
||||
return list(block_ids_by_group[group_index])
|
||||
|
||||
|
||||
def _get_tp_ratio(local_tp_size: int, remote_tp_size: int) -> int:
|
||||
@@ -129,6 +166,8 @@ def _expand_transfer_regions(
|
||||
layer_names: list[str],
|
||||
layer_indices: list[int],
|
||||
group_indices: list[int] | None = None,
|
||||
shared_group_ids: list[tuple[int, ...]] | None = None,
|
||||
row_offsets: list[int] | None = None,
|
||||
) -> list[TransferRegion]:
|
||||
"""Expand registered KV tensors into the regions transferred by Mooncake."""
|
||||
assert (
|
||||
@@ -150,6 +189,14 @@ def _expand_transfer_regions(
|
||||
"Mooncake transfer regions require matching group metadata lengths, "
|
||||
f"got group_indices={len(group_indices)}, layer_names={len(layer_names)}."
|
||||
)
|
||||
if shared_group_ids is None:
|
||||
shared_group_ids = [() for _ in layer_names]
|
||||
assert len(shared_group_ids) == len(layer_names)
|
||||
# Missing offsets must not look like row starts, or every run would be
|
||||
# promoted to the full row.
|
||||
if row_offsets is None:
|
||||
row_offsets = [-1] * len(layer_names)
|
||||
assert len(row_offsets) == len(layer_names)
|
||||
regions: list[TransferRegion] = []
|
||||
for (
|
||||
base_addr,
|
||||
@@ -158,6 +205,8 @@ def _expand_transfer_regions(
|
||||
layer_name,
|
||||
layer_index,
|
||||
group_index,
|
||||
region_groups,
|
||||
row_offset,
|
||||
) in zip(
|
||||
base_addrs,
|
||||
block_lens,
|
||||
@@ -165,6 +214,8 @@ def _expand_transfer_regions(
|
||||
layer_names,
|
||||
layer_indices,
|
||||
group_indices,
|
||||
shared_group_ids,
|
||||
row_offsets,
|
||||
):
|
||||
regions.append(
|
||||
TransferRegion(
|
||||
@@ -174,6 +225,8 @@ def _expand_transfer_regions(
|
||||
block_len=block_len,
|
||||
kv_block_len=kv_block_len,
|
||||
group_index=group_index,
|
||||
shared_group_ids=region_groups,
|
||||
row_offset=row_offset,
|
||||
)
|
||||
)
|
||||
return regions
|
||||
@@ -377,12 +430,168 @@ def _align_transfer_regions(
|
||||
f"{remote_region.group_index}."
|
||||
),
|
||||
)
|
||||
if (
|
||||
local_region.shared_group_ids
|
||||
and remote_region.shared_group_ids
|
||||
and local_region.shared_group_ids != remote_region.shared_group_ids
|
||||
):
|
||||
return (
|
||||
[],
|
||||
[],
|
||||
(
|
||||
"Mooncake shared-group set mismatch for "
|
||||
f"{local_region.layer_name}: producer="
|
||||
f"{local_region.shared_group_ids}, consumer="
|
||||
f"{remote_region.shared_group_ids}."
|
||||
),
|
||||
)
|
||||
aligned_local.append(local_region)
|
||||
aligned_remote.append(remote_region)
|
||||
|
||||
return aligned_local, aligned_remote, None
|
||||
|
||||
|
||||
def _coalesce_contiguous_transfer_regions(
|
||||
local_regions: list[TransferRegion],
|
||||
remote_regions: list[TransferRegion],
|
||||
*,
|
||||
promote_full_row: bool = False,
|
||||
) -> tuple[list[TransferRegion], list[TransferRegion]]:
|
||||
"""Merge aligned packed slices that are adjacent on both sides.
|
||||
|
||||
Page-contiguous views stay addressable by layer name for hetero PP
|
||||
(#56033). KV groups overlay one row and must stay separate: each group
|
||||
owns a different block-id space. A run that starts at row offset 0 is
|
||||
lifted to the full row so consecutive blocks can coalesce.
|
||||
"""
|
||||
if len(local_regions) != len(remote_regions):
|
||||
return local_regions, remote_regions
|
||||
|
||||
def adjacent(
|
||||
run_start: TransferRegion, prev: TransferRegion, nxt: TransferRegion
|
||||
) -> bool:
|
||||
# Page padding sits between unpadded payloads. Allow that gap, but
|
||||
# keep the whole run inside the row that `run_start` began. Anchoring
|
||||
# on `prev` would slide the bound forward as the run grows.
|
||||
return (
|
||||
prev.group_index == nxt.group_index
|
||||
and prev.shared_group_ids == nxt.shared_group_ids
|
||||
and prev.block_len == nxt.block_len
|
||||
and prev.base_addr + prev.kv_block_len <= nxt.base_addr
|
||||
and nxt.base_addr + nxt.kv_block_len
|
||||
<= run_start.base_addr + run_start.block_len
|
||||
)
|
||||
|
||||
def span(region: TransferRegion, kv_block_len: int) -> TransferRegion:
|
||||
return TransferRegion(
|
||||
layer_name=region.layer_name,
|
||||
layer_index=region.layer_index,
|
||||
base_addr=region.base_addr,
|
||||
block_len=region.block_len,
|
||||
kv_block_len=kv_block_len,
|
||||
group_index=region.group_index,
|
||||
shared_group_ids=region.shared_group_ids,
|
||||
row_offset=region.row_offset,
|
||||
)
|
||||
|
||||
def promote(
|
||||
merged_local: list[TransferRegion], merged_remote: list[TransferRegion]
|
||||
) -> tuple[list[TransferRegion], list[TransferRegion]]:
|
||||
if not promote_full_row:
|
||||
return merged_local, merged_remote
|
||||
# Each KV group overlays the row from offset 0. Promoting that run to
|
||||
# the full row copies bytes the group's block ids exclusively own.
|
||||
# Equal block_len is the row-structure check: both peers packed the
|
||||
# same stride, so row_offset does not need a second compare in align.
|
||||
promoted_local: list[TransferRegion] = []
|
||||
promoted_remote: list[TransferRegion] = []
|
||||
for local_region, remote_region in zip(merged_local, merged_remote):
|
||||
if (
|
||||
local_region.row_offset == 0
|
||||
and remote_region.row_offset == 0
|
||||
and local_region.block_len == remote_region.block_len
|
||||
and local_region.kv_block_len <= local_region.block_len
|
||||
and remote_region.kv_block_len <= remote_region.block_len
|
||||
):
|
||||
promoted_local.append(span(local_region, local_region.block_len))
|
||||
promoted_remote.append(span(remote_region, remote_region.block_len))
|
||||
else:
|
||||
promoted_local.append(local_region)
|
||||
promoted_remote.append(remote_region)
|
||||
return promoted_local, promoted_remote
|
||||
|
||||
if len(local_regions) <= 1:
|
||||
return promote(list(local_regions), list(remote_regions))
|
||||
|
||||
merged_local: list[TransferRegion] = []
|
||||
merged_remote: list[TransferRegion] = []
|
||||
run_local = [local_regions[0]]
|
||||
run_remote = [remote_regions[0]]
|
||||
|
||||
def flush() -> None:
|
||||
if len(run_local) == 1:
|
||||
merged_local.append(run_local[0])
|
||||
merged_remote.append(run_remote[0])
|
||||
return
|
||||
for merged, run in ((merged_local, run_local), (merged_remote, run_remote)):
|
||||
span_len = run[-1].base_addr + run[-1].kv_block_len - run[0].base_addr
|
||||
merged.append(span(run[0], span_len))
|
||||
|
||||
for local_region, remote_region in zip(local_regions[1:], remote_regions[1:]):
|
||||
if adjacent(run_local[0], run_local[-1], local_region) and adjacent(
|
||||
run_remote[0], run_remote[-1], remote_region
|
||||
):
|
||||
run_local.append(local_region)
|
||||
run_remote.append(remote_region)
|
||||
continue
|
||||
flush()
|
||||
run_local = [local_region]
|
||||
run_remote = [remote_region]
|
||||
flush()
|
||||
return promote(merged_local, merged_remote)
|
||||
|
||||
|
||||
def _has_opaque_packed_row(
|
||||
row_offsets: list[int], kv_block_lens: list[int], block_lens: list[int]
|
||||
) -> bool:
|
||||
"""Branch B is one row-sized region at offset 0, so later layer names are gone.
|
||||
|
||||
Page-contiguous views keep kv_block_len below the stride until promotion,
|
||||
and promotion happens after this handshake check.
|
||||
"""
|
||||
if (
|
||||
not row_offsets
|
||||
or len(row_offsets) != len(kv_block_lens)
|
||||
or len(row_offsets) != len(block_lens)
|
||||
):
|
||||
return False
|
||||
return any(
|
||||
offset == 0 and length == stride
|
||||
for offset, length, stride in zip(row_offsets, kv_block_lens, block_lens)
|
||||
)
|
||||
|
||||
|
||||
def _pp_mismatch_hides_packed_layers(
|
||||
local_pp_size: int,
|
||||
remote_pp_size: int,
|
||||
*,
|
||||
local_has_opaque_row: bool,
|
||||
remote_row_offsets: list[int],
|
||||
remote_kv_block_lens: list[int],
|
||||
remote_block_lens: list[int],
|
||||
) -> bool:
|
||||
"""A row that dropped per-layer names cannot be aligned across PP sizes.
|
||||
|
||||
Either side may be the one that collapsed the row. Matching PP still
|
||||
lines up, because both stages keep the same first layer name.
|
||||
"""
|
||||
if local_pp_size == remote_pp_size:
|
||||
return False
|
||||
return local_has_opaque_row or _has_opaque_packed_row(
|
||||
remote_row_offsets, remote_kv_block_lens, remote_block_lens
|
||||
)
|
||||
|
||||
|
||||
def _get_tensor_dense_flag(tensor: torch.Tensor) -> bool | None:
|
||||
is_dense = getattr(tensor, "is_non_overlapping_and_dense", None)
|
||||
if callable(is_dense):
|
||||
@@ -405,6 +614,12 @@ class MooncakeXferMetadata(
|
||||
registered_layer_names: list[str] = msgspec.field(default_factory=list)
|
||||
registered_layer_indices: list[int] = msgspec.field(default_factory=list)
|
||||
registered_group_indices: list[int] = msgspec.field(default_factory=list)
|
||||
# Parallel to registered regions. Each entry is the groups that share
|
||||
# that region. Empty means the peer did not send the field.
|
||||
registered_shared_group_ids: list[list[int]] = msgspec.field(default_factory=list)
|
||||
# Byte offset of each registered view from its block row. -1 means the
|
||||
# peer did not send the field, so the run must not be promoted.
|
||||
registered_row_offsets: list[int] = msgspec.field(default_factory=list)
|
||||
remote_pp_size: int = 1
|
||||
|
||||
|
||||
@@ -999,7 +1214,18 @@ class MooncakeConnectorWorker:
|
||||
self.registered_layer_names: list[str] = []
|
||||
self.registered_layer_indices: list[int] = []
|
||||
self.registered_group_indices: list[int] = []
|
||||
# Parallel to registered regions. -1 flattens only these groups.
|
||||
self.region_shared_groups: list[tuple[int, ...]] = []
|
||||
# Parallel to registered regions. -1 is not a packed row view.
|
||||
self.region_row_offsets: list[int] = []
|
||||
self.opaque_packed_storages: set[int] = set()
|
||||
self.seen_base_addresses: list[int] = []
|
||||
# Aligned regions depend only on the peer's registered layout.
|
||||
# The third item is an error string when alignment cannot proceed.
|
||||
self._prepared_transfer_regions: dict[
|
||||
tuple,
|
||||
tuple[list[TransferRegion], list[TransferRegion], str | None],
|
||||
] = {}
|
||||
|
||||
assert (parallel_config := vllm_config.parallel_config)
|
||||
dp_rank = parallel_config.data_parallel_index
|
||||
@@ -1266,39 +1492,35 @@ class MooncakeConnectorWorker:
|
||||
)
|
||||
await sock.send_multipart((identity, self._encoder.encode(response)))
|
||||
return
|
||||
local_regions = self._get_transfer_regions(
|
||||
self.kv_caches_base_addr,
|
||||
self.block_len_per_layer,
|
||||
self.kv_block_len_per_layer,
|
||||
self.registered_layer_names,
|
||||
self.registered_layer_indices,
|
||||
self.registered_group_indices,
|
||||
)
|
||||
remote_regions = self._get_transfer_regions(
|
||||
meta.kv_caches_base_addr,
|
||||
meta.block_lens,
|
||||
meta.kv_block_lens,
|
||||
meta.registered_layer_names,
|
||||
meta.registered_layer_indices,
|
||||
meta.registered_group_indices,
|
||||
)
|
||||
local_regions, remote_regions, align_err = _align_transfer_regions(
|
||||
local_regions,
|
||||
remote_regions,
|
||||
allow_partial_layers=(
|
||||
meta.remote_pp_size > 1 and meta.remote_pp_size != self.pp_size
|
||||
),
|
||||
)
|
||||
if align_err is not None:
|
||||
if _pp_mismatch_hides_packed_layers(
|
||||
self.pp_size,
|
||||
meta.remote_pp_size,
|
||||
local_has_opaque_row=bool(self.opaque_packed_storages),
|
||||
remote_row_offsets=meta.registered_row_offsets,
|
||||
remote_kv_block_lens=meta.kv_block_lens,
|
||||
remote_block_lens=meta.block_lens,
|
||||
):
|
||||
msg = (
|
||||
"Mooncake non-page-contiguous packed regions keep only the "
|
||||
"first layer name, so they cannot be aligned across PP sizes "
|
||||
f"{self.pp_size} and {meta.remote_pp_size}."
|
||||
)
|
||||
logger.error(msg)
|
||||
response = MooncakeXferResponse(
|
||||
status=MooncakeXferResponseStatus.ERROR,
|
||||
err_msg=align_err,
|
||||
err_msg=msg,
|
||||
)
|
||||
await sock.send_multipart((identity, self._encoder.encode(response)))
|
||||
return
|
||||
validation_err = self._validate_head_resharding_layout(
|
||||
meta.remote_tp_size, local_regions
|
||||
) or _validate_asymmetric_region_lengths(
|
||||
local_regions, remote_regions, prep_err = self._prepare_transfer_regions(meta)
|
||||
if prep_err is not None:
|
||||
response = MooncakeXferResponse(
|
||||
status=MooncakeXferResponseStatus.ERROR,
|
||||
err_msg=prep_err,
|
||||
)
|
||||
await sock.send_multipart((identity, self._encoder.encode(response)))
|
||||
return
|
||||
validation_err = _validate_asymmetric_region_lengths(
|
||||
local_regions=local_regions,
|
||||
remote_regions=remote_regions,
|
||||
local_tp_size=self.tp_size,
|
||||
@@ -1600,12 +1822,16 @@ class MooncakeConnectorWorker:
|
||||
"Aligned Mooncake transfer regions must belong to the same "
|
||||
"KV group."
|
||||
)
|
||||
group_index = local_region.group_index
|
||||
assert group_index < len(local_block_ids_by_group), (
|
||||
"Transfer region references a missing KV group."
|
||||
local_block_ids = _block_ids_for_region(
|
||||
local_block_ids_by_group,
|
||||
local_region.group_index,
|
||||
local_region.shared_group_ids,
|
||||
)
|
||||
remote_block_ids = _block_ids_for_region(
|
||||
remote_block_ids_by_group,
|
||||
remote_region.group_index,
|
||||
local_region.shared_group_ids,
|
||||
)
|
||||
local_block_ids = local_block_ids_by_group[group_index]
|
||||
remote_block_ids = remote_block_ids_by_group[group_index]
|
||||
if not local_block_ids:
|
||||
continue
|
||||
|
||||
@@ -1738,6 +1964,17 @@ class MooncakeConnectorWorker:
|
||||
self.registered_layer_names = []
|
||||
self.registered_layer_indices = []
|
||||
self.registered_group_indices = []
|
||||
self.region_shared_groups = []
|
||||
self.region_row_offsets = []
|
||||
self.opaque_packed_storages = set()
|
||||
self._prepared_transfer_regions.clear()
|
||||
|
||||
packed_storage_to_region: dict[int, int] = {}
|
||||
packed_view_to_region: dict[tuple[int, int, int], int] = {}
|
||||
collapsed_views = 0
|
||||
num_blocks = (
|
||||
self.kv_cache_config.num_blocks * self._physical_blocks_per_logical_kv_block
|
||||
)
|
||||
|
||||
for layer_name, cache in kv_caches.items():
|
||||
layer_index = extract_layer_index(layer_name)
|
||||
@@ -1751,6 +1988,104 @@ class MooncakeConnectorWorker:
|
||||
# One raw page tensor per layer; for Mamba that page holds all the
|
||||
# recurrent states, unpacked only when binding the cache for execution.
|
||||
self._log_debug_cache_registration(layer_name, cache)
|
||||
group_index = self.kv_cache_config.transfer_group_index_by_layer[layer_name]
|
||||
storage = cache.untyped_storage()
|
||||
storage_addr = storage.data_ptr()
|
||||
if storage_addr not in seen_storage_ptrs:
|
||||
seen_storage_ptrs.add(storage_addr)
|
||||
kv_data_ptrs.append(storage_addr)
|
||||
kv_data_lens.append(storage.nbytes())
|
||||
|
||||
is_mla_region = isinstance(
|
||||
layer_spec, (MLAAttentionSpec, SlidingWindowMLASpec)
|
||||
)
|
||||
tensor_blocks = cache.shape[0] if cache.ndim > 1 else num_blocks
|
||||
if cache.ndim == 1:
|
||||
# A flat buffer is block-major only when it is an integral run
|
||||
# of this layer's pages. NIXL uses the same remainder check.
|
||||
hnc_contiguous = False
|
||||
page = (
|
||||
layer_spec.page_size_bytes
|
||||
if isinstance(layer_spec, AttentionSpec)
|
||||
else 0
|
||||
)
|
||||
remainder = (
|
||||
storage.nbytes() - tensor_blocks * page
|
||||
if tensor_blocks > 0 and page > 0
|
||||
else -1
|
||||
)
|
||||
if 0 <= remainder < page:
|
||||
block_stride = page
|
||||
storage_is_block_major = True
|
||||
else:
|
||||
block_stride = 0
|
||||
storage_is_block_major = False
|
||||
else:
|
||||
block_stride = cache.stride(0) * cache.element_size()
|
||||
storage_is_block_major = (
|
||||
tensor_blocks > 0
|
||||
and tensor_blocks * block_stride == storage.nbytes()
|
||||
)
|
||||
hnc_contiguous = (
|
||||
cache.ndim == 4
|
||||
and cache.stride(2) == cache.shape[3]
|
||||
and cache.stride(1) == cache.shape[2] * cache.shape[3]
|
||||
)
|
||||
# Follow NIXL packed pages. Page-contiguous slices stay per layer so
|
||||
# hetero PP can align by name (#56033). A non-contiguous row cannot
|
||||
# name a sub-span, so it is one region anchored at the allocation.
|
||||
use_packed_row = (
|
||||
not isinstance(layer_spec, (MambaSpec, KpoolTailSpec))
|
||||
and storage_is_block_major
|
||||
and block_stride > 0
|
||||
and (is_mla_region or not hnc_contiguous)
|
||||
)
|
||||
if use_packed_row:
|
||||
page_contiguous = (
|
||||
cache.ndim > 1
|
||||
and cache[0].is_contiguous()
|
||||
and 0 < cache[0].nbytes <= block_stride
|
||||
)
|
||||
if page_contiguous:
|
||||
view_addr = cache.data_ptr()
|
||||
kv_block_len = cache[0].nbytes
|
||||
else:
|
||||
# NIXL branch B: length is the row, base is the allocation.
|
||||
view_addr = storage_addr
|
||||
kv_block_len = block_stride
|
||||
self.opaque_packed_storages.add(storage_addr)
|
||||
# Same PP on both sides still aligns by the first layer name.
|
||||
# A remote PP mismatch is rejected at send time, once the
|
||||
# peer's PP size is known.
|
||||
dedupe_key = (view_addr, block_stride, kv_block_len)
|
||||
region_idx = packed_view_to_region.get(dedupe_key)
|
||||
if region_idx is not None:
|
||||
collapsed_views += 1
|
||||
# -1 means these groups share the view. Flatten only them.
|
||||
groups = list(self.region_shared_groups[region_idx])
|
||||
if group_index not in groups:
|
||||
groups.append(group_index)
|
||||
self.region_shared_groups[region_idx] = tuple(groups)
|
||||
if len(self.region_shared_groups[region_idx]) > 1:
|
||||
self.registered_group_indices[region_idx] = (
|
||||
_SHARED_REGION_GROUP_ID
|
||||
)
|
||||
continue
|
||||
if not page_contiguous:
|
||||
self.opaque_packed_storages.add(storage_addr)
|
||||
if storage_addr not in packed_storage_to_region:
|
||||
packed_storage_to_region[storage_addr] = len(region_base_addresses)
|
||||
packed_view_to_region[dedupe_key] = len(region_base_addresses)
|
||||
region_base_addresses.append(view_addr)
|
||||
self.block_len_per_layer.append(block_stride)
|
||||
self.kv_block_len_per_layer.append(kv_block_len)
|
||||
self.registered_layer_names.append(layer_name)
|
||||
self.registered_layer_indices.append(layer_index)
|
||||
self.registered_group_indices.append(group_index)
|
||||
self.region_shared_groups.append((group_index,))
|
||||
self.region_row_offsets.append(view_addr - storage_addr)
|
||||
continue
|
||||
|
||||
block_is_contiguous = is_non_overlapping_and_dense(cache[0])
|
||||
if not block_is_contiguous:
|
||||
# Non-block-compact layouts scatter a block across per-head
|
||||
@@ -1791,15 +2126,10 @@ class MooncakeConnectorWorker:
|
||||
self.kv_block_len_per_layer.append(kv_block_len)
|
||||
self.registered_layer_names.append(layer_name)
|
||||
self.registered_layer_indices.append(layer_index)
|
||||
self.registered_group_indices.append(
|
||||
self.kv_cache_config.transfer_group_index_by_layer[layer_name]
|
||||
)
|
||||
storage = cache.untyped_storage()
|
||||
storage_addr = storage.data_ptr()
|
||||
if storage_addr not in seen_storage_ptrs:
|
||||
seen_storage_ptrs.add(storage_addr)
|
||||
kv_data_ptrs.append(storage_addr)
|
||||
kv_data_lens.append(storage.nbytes())
|
||||
self.registered_group_indices.append(group_index)
|
||||
self.region_shared_groups.append((group_index,))
|
||||
# Not a view into a shared packed row, so do not promote it.
|
||||
self.region_row_offsets.append(-1)
|
||||
|
||||
self.kv_caches_base_addr = region_base_addresses
|
||||
self.seen_base_addresses = kv_data_ptrs
|
||||
@@ -1807,6 +2137,18 @@ class MooncakeConnectorWorker:
|
||||
if not kv_data_ptrs:
|
||||
raise RuntimeError("No KV cache tensors were registered with Mooncake.")
|
||||
|
||||
if packed_storage_to_region and (
|
||||
collapsed_views
|
||||
or len(packed_view_to_region) != len(packed_storage_to_region)
|
||||
):
|
||||
logger.info(
|
||||
"Mooncake packed KV xfer: %s layer regions over %s storage "
|
||||
"regions (%s duplicate views collapsed).",
|
||||
len(packed_view_to_region),
|
||||
len(packed_storage_to_region),
|
||||
collapsed_views,
|
||||
)
|
||||
|
||||
ret_value = self.engine.batch_register_memory(kv_data_ptrs, kv_data_lens)
|
||||
if ret_value != 0:
|
||||
raise RuntimeError("Mooncake batch memory registration failed.")
|
||||
@@ -1962,6 +2304,10 @@ class MooncakeConnectorWorker:
|
||||
registered_layer_names=self.registered_layer_names,
|
||||
registered_layer_indices=self.registered_layer_indices,
|
||||
registered_group_indices=self.registered_group_indices,
|
||||
registered_shared_group_ids=[
|
||||
list(groups) for groups in self.region_shared_groups
|
||||
],
|
||||
registered_row_offsets=self.region_row_offsets,
|
||||
)
|
||||
|
||||
encoded_data = self._encoder.encode(metadata)
|
||||
@@ -2209,6 +2555,131 @@ class MooncakeConnectorWorker:
|
||||
def _producer_cache_is_replicated(self) -> bool:
|
||||
return self.transfer_topo.local_replicates_kv_cache
|
||||
|
||||
def _prepare_transfer_regions(
|
||||
self, meta: MooncakeXferMetadata
|
||||
) -> tuple[list[TransferRegion], list[TransferRegion], str | None]:
|
||||
"""Align and coalesce once per peer layout.
|
||||
|
||||
``send_kv_to_decode`` runs on every scheduler batch. The registered
|
||||
layout does not change until the next ``register_kv_caches``.
|
||||
"""
|
||||
cache_key = (
|
||||
meta.remote_hostname,
|
||||
meta.remote_port,
|
||||
meta.remote_tp_size,
|
||||
meta.remote_tp_rank,
|
||||
meta.remote_pp_size,
|
||||
tuple(meta.kv_caches_base_addr),
|
||||
tuple(meta.block_lens),
|
||||
tuple(meta.kv_block_lens),
|
||||
tuple(meta.registered_layer_names),
|
||||
tuple(meta.registered_layer_indices),
|
||||
tuple(meta.registered_group_indices),
|
||||
tuple(tuple(groups) for groups in meta.registered_shared_group_ids),
|
||||
tuple(meta.registered_row_offsets),
|
||||
)
|
||||
cached = self._prepared_transfer_regions.get(cache_key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
def finish(
|
||||
local: list[TransferRegion],
|
||||
remote: list[TransferRegion],
|
||||
err: str | None,
|
||||
) -> tuple[list[TransferRegion], list[TransferRegion], str | None]:
|
||||
prepared = (local, remote, err)
|
||||
self._prepared_transfer_regions[cache_key] = prepared
|
||||
return prepared
|
||||
|
||||
shared_groups: list[tuple[int, ...]] | None = self.region_shared_groups
|
||||
if len(self.region_shared_groups) != len(self.registered_layer_names):
|
||||
if _SHARED_REGION_GROUP_ID in self.registered_group_indices:
|
||||
return finish(
|
||||
[],
|
||||
[],
|
||||
(
|
||||
"Mooncake shared-group metadata does not match "
|
||||
"registered layers, so a shared packed region cannot "
|
||||
"be transferred."
|
||||
),
|
||||
)
|
||||
shared_groups = None
|
||||
local_rows: list[int] | None = self.region_row_offsets
|
||||
if len(self.region_row_offsets) != len(self.registered_layer_names):
|
||||
local_rows = None
|
||||
local_regions = self._get_transfer_regions(
|
||||
self.kv_caches_base_addr,
|
||||
self.block_len_per_layer,
|
||||
self.kv_block_len_per_layer,
|
||||
self.registered_layer_names,
|
||||
self.registered_layer_indices,
|
||||
self.registered_group_indices,
|
||||
shared_groups,
|
||||
local_rows,
|
||||
)
|
||||
remote_shared_groups = [
|
||||
tuple(groups) for groups in meta.registered_shared_group_ids
|
||||
]
|
||||
remote_shared: list[tuple[int, ...]] | None = remote_shared_groups
|
||||
if len(remote_shared_groups) != len(meta.registered_layer_names):
|
||||
remote_shared = None
|
||||
remote_rows = meta.registered_row_offsets
|
||||
remote_row_offsets: list[int] | None = remote_rows
|
||||
if len(remote_rows) != len(meta.registered_layer_names):
|
||||
remote_row_offsets = None
|
||||
remote_regions = self._get_transfer_regions(
|
||||
meta.kv_caches_base_addr,
|
||||
meta.block_lens,
|
||||
meta.kv_block_lens,
|
||||
meta.registered_layer_names,
|
||||
meta.registered_layer_indices,
|
||||
meta.registered_group_indices,
|
||||
remote_shared,
|
||||
remote_row_offsets,
|
||||
)
|
||||
pre_align_local = local_regions
|
||||
pre_align_remote = remote_regions
|
||||
local_regions, remote_regions, align_err = _align_transfer_regions(
|
||||
local_regions,
|
||||
remote_regions,
|
||||
allow_partial_layers=meta.remote_pp_size != self.pp_size,
|
||||
)
|
||||
if align_err is not None:
|
||||
return finish([], [], align_err)
|
||||
# Head checks must see every layer name. Coalesce keeps only the first.
|
||||
head_err = self._validate_head_resharding_layout(
|
||||
meta.remote_tp_size, local_regions
|
||||
)
|
||||
if head_err is not None:
|
||||
return finish([], [], head_err)
|
||||
# Hetero TP indexes kv_block_len as one head-sharded payload. A merged
|
||||
# multi-layer span would land that offset on the wrong layer.
|
||||
# Replicated KV keeps that offset at 0. MLA is replicated in practice;
|
||||
# gating on use_mla alone would promote and then fail the length check
|
||||
# for a non-replicated MLA rank with tp_ratio != 1.
|
||||
tp_ratio = _get_tp_ratio(self.tp_size, meta.remote_tp_size)
|
||||
if tp_ratio == 1 or self._producer_cache_is_replicated():
|
||||
covered_both_sides = len(local_regions) == len(pre_align_local) and len(
|
||||
remote_regions
|
||||
) == len(pre_align_remote)
|
||||
n_aligned = len(local_regions)
|
||||
local_regions, remote_regions = _coalesce_contiguous_transfer_regions(
|
||||
local_regions,
|
||||
remote_regions,
|
||||
promote_full_row=covered_both_sides,
|
||||
)
|
||||
logger.info(
|
||||
"Mooncake packed coalesce: %s aligned regions -> %s transfer regions.",
|
||||
n_aligned,
|
||||
len(local_regions),
|
||||
)
|
||||
self._prepared_transfer_regions[cache_key] = (
|
||||
local_regions,
|
||||
remote_regions,
|
||||
None,
|
||||
)
|
||||
return local_regions, remote_regions, None
|
||||
|
||||
def _validate_head_resharding_layout(
|
||||
self, remote_tp_size: int, local_regions: list[TransferRegion]
|
||||
) -> str | None:
|
||||
@@ -2242,6 +2713,8 @@ class MooncakeConnectorWorker:
|
||||
layer_names: list[str],
|
||||
layer_indices: list[int],
|
||||
group_indices: list[int] | None = None,
|
||||
shared_group_ids: list[tuple[int, ...]] | None = None,
|
||||
row_offsets: list[int] | None = None,
|
||||
) -> list[TransferRegion]:
|
||||
if not group_indices:
|
||||
group_indices = [
|
||||
@@ -2255,6 +2728,8 @@ class MooncakeConnectorWorker:
|
||||
layer_names=layer_names,
|
||||
layer_indices=layer_indices,
|
||||
group_indices=group_indices,
|
||||
shared_group_ids=shared_group_ids,
|
||||
row_offsets=row_offsets,
|
||||
)
|
||||
|
||||
def _get_sender_transfer_plan(
|
||||
|
||||
Reference in New Issue
Block a user