[Perf][KV Connector][Mooncake] Pack hybrid/MLA KV into coalesced transfer regions (#57952)

Signed-off-by: staryxchen <staryxchen@tencent.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Stary
2026-09-29 13:29:04 -07:00
committed by GitHub
co-authored by Cursor Yifan Qiao
parent 5dd693122c
commit faacc13565
3 changed files with 1384 additions and 207 deletions
File diff suppressed because it is too large Load Diff
@@ -242,6 +242,8 @@ async def test_build_transfer_params_uses_non_overlapping_physical_pages():
kv_cache_config = make_hybrid_gdn_kv_cache_config(block_size=16)
worker = object.__new__(MooncakeConnectorWorker)
worker.shutdown = noop_shutdown
# __new__ skips __init__, where this cache is created. register_kv_caches clears it.
worker._prepared_transfer_regions = {}
worker.use_mla = False
worker.engine = SimpleNamespace(batch_register_memory=lambda *_: 0)
worker.is_kv_consumer = True
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import itertools
import logging
import queue
import threading
@@ -62,6 +63,8 @@ from vllm.v1.kv_cache_interface import (
KpoolTailSpec,
KVCacheSpec,
MambaSpec,
MLAAttentionSpec,
SlidingWindowMLASpec,
SlidingWindowSpec,
)
from vllm.v1.request import RequestStatus
@@ -90,6 +93,12 @@ if TYPE_CHECKING:
ReqId = str # Internal scheduler request ID
TransferId = str # KV transfer coordination ID (shared by P/D)
# Follow NIXL (`nixl/base_worker.py`): HMA packed pages can put more than one
# KV group on a single allocation. group_index=-1 flattens every group's block
# ids onto that region. Hybrid KV manager assigns distinct ids per group, so
# flatten does not copy the same block twice.
_SHARED_REGION_GROUP_ID = -1
@dataclass(frozen=True)
class TransferRegion:
@@ -99,6 +108,34 @@ class TransferRegion:
block_len: int
kv_block_len: int
group_index: int = 0
# Groups that share this allocation. Empty unless group_index is -1.
shared_group_ids: tuple[int, ...] = ()
# Byte offset of this view from the start of its block row.
row_offset: int = 0
def _block_ids_for_region(
block_ids_by_group: list[list[int]],
group_index: int,
shared_group_ids: tuple[int, ...] = (),
) -> list[int]:
"""Follow NIXL `_block_ids_by_region`.
-1 flattens only the groups that share this allocation. NIXL can flatten
every group because its packed flag means they all share one storage;
Mooncake can still have a separate Mamba group in the same request.
"""
if group_index == _SHARED_REGION_GROUP_ID:
assert shared_group_ids, "Shared packed region is missing its group ids."
return list(
itertools.chain.from_iterable(
block_ids_by_group[group] for group in shared_group_ids
)
)
assert 0 <= group_index < len(block_ids_by_group), (
"Transfer region references a missing KV group."
)
return list(block_ids_by_group[group_index])
def _get_tp_ratio(local_tp_size: int, remote_tp_size: int) -> int:
@@ -129,6 +166,8 @@ def _expand_transfer_regions(
layer_names: list[str],
layer_indices: list[int],
group_indices: list[int] | None = None,
shared_group_ids: list[tuple[int, ...]] | None = None,
row_offsets: list[int] | None = None,
) -> list[TransferRegion]:
"""Expand registered KV tensors into the regions transferred by Mooncake."""
assert (
@@ -150,6 +189,14 @@ def _expand_transfer_regions(
"Mooncake transfer regions require matching group metadata lengths, "
f"got group_indices={len(group_indices)}, layer_names={len(layer_names)}."
)
if shared_group_ids is None:
shared_group_ids = [() for _ in layer_names]
assert len(shared_group_ids) == len(layer_names)
# Missing offsets must not look like row starts, or every run would be
# promoted to the full row.
if row_offsets is None:
row_offsets = [-1] * len(layer_names)
assert len(row_offsets) == len(layer_names)
regions: list[TransferRegion] = []
for (
base_addr,
@@ -158,6 +205,8 @@ def _expand_transfer_regions(
layer_name,
layer_index,
group_index,
region_groups,
row_offset,
) in zip(
base_addrs,
block_lens,
@@ -165,6 +214,8 @@ def _expand_transfer_regions(
layer_names,
layer_indices,
group_indices,
shared_group_ids,
row_offsets,
):
regions.append(
TransferRegion(
@@ -174,6 +225,8 @@ def _expand_transfer_regions(
block_len=block_len,
kv_block_len=kv_block_len,
group_index=group_index,
shared_group_ids=region_groups,
row_offset=row_offset,
)
)
return regions
@@ -377,12 +430,168 @@ def _align_transfer_regions(
f"{remote_region.group_index}."
),
)
if (
local_region.shared_group_ids
and remote_region.shared_group_ids
and local_region.shared_group_ids != remote_region.shared_group_ids
):
return (
[],
[],
(
"Mooncake shared-group set mismatch for "
f"{local_region.layer_name}: producer="
f"{local_region.shared_group_ids}, consumer="
f"{remote_region.shared_group_ids}."
),
)
aligned_local.append(local_region)
aligned_remote.append(remote_region)
return aligned_local, aligned_remote, None
def _coalesce_contiguous_transfer_regions(
local_regions: list[TransferRegion],
remote_regions: list[TransferRegion],
*,
promote_full_row: bool = False,
) -> tuple[list[TransferRegion], list[TransferRegion]]:
"""Merge aligned packed slices that are adjacent on both sides.
Page-contiguous views stay addressable by layer name for hetero PP
(#56033). KV groups overlay one row and must stay separate: each group
owns a different block-id space. A run that starts at row offset 0 is
lifted to the full row so consecutive blocks can coalesce.
"""
if len(local_regions) != len(remote_regions):
return local_regions, remote_regions
def adjacent(
run_start: TransferRegion, prev: TransferRegion, nxt: TransferRegion
) -> bool:
# Page padding sits between unpadded payloads. Allow that gap, but
# keep the whole run inside the row that `run_start` began. Anchoring
# on `prev` would slide the bound forward as the run grows.
return (
prev.group_index == nxt.group_index
and prev.shared_group_ids == nxt.shared_group_ids
and prev.block_len == nxt.block_len
and prev.base_addr + prev.kv_block_len <= nxt.base_addr
and nxt.base_addr + nxt.kv_block_len
<= run_start.base_addr + run_start.block_len
)
def span(region: TransferRegion, kv_block_len: int) -> TransferRegion:
return TransferRegion(
layer_name=region.layer_name,
layer_index=region.layer_index,
base_addr=region.base_addr,
block_len=region.block_len,
kv_block_len=kv_block_len,
group_index=region.group_index,
shared_group_ids=region.shared_group_ids,
row_offset=region.row_offset,
)
def promote(
merged_local: list[TransferRegion], merged_remote: list[TransferRegion]
) -> tuple[list[TransferRegion], list[TransferRegion]]:
if not promote_full_row:
return merged_local, merged_remote
# Each KV group overlays the row from offset 0. Promoting that run to
# the full row copies bytes the group's block ids exclusively own.
# Equal block_len is the row-structure check: both peers packed the
# same stride, so row_offset does not need a second compare in align.
promoted_local: list[TransferRegion] = []
promoted_remote: list[TransferRegion] = []
for local_region, remote_region in zip(merged_local, merged_remote):
if (
local_region.row_offset == 0
and remote_region.row_offset == 0
and local_region.block_len == remote_region.block_len
and local_region.kv_block_len <= local_region.block_len
and remote_region.kv_block_len <= remote_region.block_len
):
promoted_local.append(span(local_region, local_region.block_len))
promoted_remote.append(span(remote_region, remote_region.block_len))
else:
promoted_local.append(local_region)
promoted_remote.append(remote_region)
return promoted_local, promoted_remote
if len(local_regions) <= 1:
return promote(list(local_regions), list(remote_regions))
merged_local: list[TransferRegion] = []
merged_remote: list[TransferRegion] = []
run_local = [local_regions[0]]
run_remote = [remote_regions[0]]
def flush() -> None:
if len(run_local) == 1:
merged_local.append(run_local[0])
merged_remote.append(run_remote[0])
return
for merged, run in ((merged_local, run_local), (merged_remote, run_remote)):
span_len = run[-1].base_addr + run[-1].kv_block_len - run[0].base_addr
merged.append(span(run[0], span_len))
for local_region, remote_region in zip(local_regions[1:], remote_regions[1:]):
if adjacent(run_local[0], run_local[-1], local_region) and adjacent(
run_remote[0], run_remote[-1], remote_region
):
run_local.append(local_region)
run_remote.append(remote_region)
continue
flush()
run_local = [local_region]
run_remote = [remote_region]
flush()
return promote(merged_local, merged_remote)
def _has_opaque_packed_row(
row_offsets: list[int], kv_block_lens: list[int], block_lens: list[int]
) -> bool:
"""Branch B is one row-sized region at offset 0, so later layer names are gone.
Page-contiguous views keep kv_block_len below the stride until promotion,
and promotion happens after this handshake check.
"""
if (
not row_offsets
or len(row_offsets) != len(kv_block_lens)
or len(row_offsets) != len(block_lens)
):
return False
return any(
offset == 0 and length == stride
for offset, length, stride in zip(row_offsets, kv_block_lens, block_lens)
)
def _pp_mismatch_hides_packed_layers(
local_pp_size: int,
remote_pp_size: int,
*,
local_has_opaque_row: bool,
remote_row_offsets: list[int],
remote_kv_block_lens: list[int],
remote_block_lens: list[int],
) -> bool:
"""A row that dropped per-layer names cannot be aligned across PP sizes.
Either side may be the one that collapsed the row. Matching PP still
lines up, because both stages keep the same first layer name.
"""
if local_pp_size == remote_pp_size:
return False
return local_has_opaque_row or _has_opaque_packed_row(
remote_row_offsets, remote_kv_block_lens, remote_block_lens
)
def _get_tensor_dense_flag(tensor: torch.Tensor) -> bool | None:
is_dense = getattr(tensor, "is_non_overlapping_and_dense", None)
if callable(is_dense):
@@ -405,6 +614,12 @@ class MooncakeXferMetadata(
registered_layer_names: list[str] = msgspec.field(default_factory=list)
registered_layer_indices: list[int] = msgspec.field(default_factory=list)
registered_group_indices: list[int] = msgspec.field(default_factory=list)
# Parallel to registered regions. Each entry is the groups that share
# that region. Empty means the peer did not send the field.
registered_shared_group_ids: list[list[int]] = msgspec.field(default_factory=list)
# Byte offset of each registered view from its block row. -1 means the
# peer did not send the field, so the run must not be promoted.
registered_row_offsets: list[int] = msgspec.field(default_factory=list)
remote_pp_size: int = 1
@@ -999,7 +1214,18 @@ class MooncakeConnectorWorker:
self.registered_layer_names: list[str] = []
self.registered_layer_indices: list[int] = []
self.registered_group_indices: list[int] = []
# Parallel to registered regions. -1 flattens only these groups.
self.region_shared_groups: list[tuple[int, ...]] = []
# Parallel to registered regions. -1 is not a packed row view.
self.region_row_offsets: list[int] = []
self.opaque_packed_storages: set[int] = set()
self.seen_base_addresses: list[int] = []
# Aligned regions depend only on the peer's registered layout.
# The third item is an error string when alignment cannot proceed.
self._prepared_transfer_regions: dict[
tuple,
tuple[list[TransferRegion], list[TransferRegion], str | None],
] = {}
assert (parallel_config := vllm_config.parallel_config)
dp_rank = parallel_config.data_parallel_index
@@ -1266,39 +1492,35 @@ class MooncakeConnectorWorker:
)
await sock.send_multipart((identity, self._encoder.encode(response)))
return
local_regions = self._get_transfer_regions(
self.kv_caches_base_addr,
self.block_len_per_layer,
self.kv_block_len_per_layer,
self.registered_layer_names,
self.registered_layer_indices,
self.registered_group_indices,
)
remote_regions = self._get_transfer_regions(
meta.kv_caches_base_addr,
meta.block_lens,
meta.kv_block_lens,
meta.registered_layer_names,
meta.registered_layer_indices,
meta.registered_group_indices,
)
local_regions, remote_regions, align_err = _align_transfer_regions(
local_regions,
remote_regions,
allow_partial_layers=(
meta.remote_pp_size > 1 and meta.remote_pp_size != self.pp_size
),
)
if align_err is not None:
if _pp_mismatch_hides_packed_layers(
self.pp_size,
meta.remote_pp_size,
local_has_opaque_row=bool(self.opaque_packed_storages),
remote_row_offsets=meta.registered_row_offsets,
remote_kv_block_lens=meta.kv_block_lens,
remote_block_lens=meta.block_lens,
):
msg = (
"Mooncake non-page-contiguous packed regions keep only the "
"first layer name, so they cannot be aligned across PP sizes "
f"{self.pp_size} and {meta.remote_pp_size}."
)
logger.error(msg)
response = MooncakeXferResponse(
status=MooncakeXferResponseStatus.ERROR,
err_msg=align_err,
err_msg=msg,
)
await sock.send_multipart((identity, self._encoder.encode(response)))
return
validation_err = self._validate_head_resharding_layout(
meta.remote_tp_size, local_regions
) or _validate_asymmetric_region_lengths(
local_regions, remote_regions, prep_err = self._prepare_transfer_regions(meta)
if prep_err is not None:
response = MooncakeXferResponse(
status=MooncakeXferResponseStatus.ERROR,
err_msg=prep_err,
)
await sock.send_multipart((identity, self._encoder.encode(response)))
return
validation_err = _validate_asymmetric_region_lengths(
local_regions=local_regions,
remote_regions=remote_regions,
local_tp_size=self.tp_size,
@@ -1600,12 +1822,16 @@ class MooncakeConnectorWorker:
"Aligned Mooncake transfer regions must belong to the same "
"KV group."
)
group_index = local_region.group_index
assert group_index < len(local_block_ids_by_group), (
"Transfer region references a missing KV group."
local_block_ids = _block_ids_for_region(
local_block_ids_by_group,
local_region.group_index,
local_region.shared_group_ids,
)
remote_block_ids = _block_ids_for_region(
remote_block_ids_by_group,
remote_region.group_index,
local_region.shared_group_ids,
)
local_block_ids = local_block_ids_by_group[group_index]
remote_block_ids = remote_block_ids_by_group[group_index]
if not local_block_ids:
continue
@@ -1738,6 +1964,17 @@ class MooncakeConnectorWorker:
self.registered_layer_names = []
self.registered_layer_indices = []
self.registered_group_indices = []
self.region_shared_groups = []
self.region_row_offsets = []
self.opaque_packed_storages = set()
self._prepared_transfer_regions.clear()
packed_storage_to_region: dict[int, int] = {}
packed_view_to_region: dict[tuple[int, int, int], int] = {}
collapsed_views = 0
num_blocks = (
self.kv_cache_config.num_blocks * self._physical_blocks_per_logical_kv_block
)
for layer_name, cache in kv_caches.items():
layer_index = extract_layer_index(layer_name)
@@ -1751,6 +1988,104 @@ class MooncakeConnectorWorker:
# One raw page tensor per layer; for Mamba that page holds all the
# recurrent states, unpacked only when binding the cache for execution.
self._log_debug_cache_registration(layer_name, cache)
group_index = self.kv_cache_config.transfer_group_index_by_layer[layer_name]
storage = cache.untyped_storage()
storage_addr = storage.data_ptr()
if storage_addr not in seen_storage_ptrs:
seen_storage_ptrs.add(storage_addr)
kv_data_ptrs.append(storage_addr)
kv_data_lens.append(storage.nbytes())
is_mla_region = isinstance(
layer_spec, (MLAAttentionSpec, SlidingWindowMLASpec)
)
tensor_blocks = cache.shape[0] if cache.ndim > 1 else num_blocks
if cache.ndim == 1:
# A flat buffer is block-major only when it is an integral run
# of this layer's pages. NIXL uses the same remainder check.
hnc_contiguous = False
page = (
layer_spec.page_size_bytes
if isinstance(layer_spec, AttentionSpec)
else 0
)
remainder = (
storage.nbytes() - tensor_blocks * page
if tensor_blocks > 0 and page > 0
else -1
)
if 0 <= remainder < page:
block_stride = page
storage_is_block_major = True
else:
block_stride = 0
storage_is_block_major = False
else:
block_stride = cache.stride(0) * cache.element_size()
storage_is_block_major = (
tensor_blocks > 0
and tensor_blocks * block_stride == storage.nbytes()
)
hnc_contiguous = (
cache.ndim == 4
and cache.stride(2) == cache.shape[3]
and cache.stride(1) == cache.shape[2] * cache.shape[3]
)
# Follow NIXL packed pages. Page-contiguous slices stay per layer so
# hetero PP can align by name (#56033). A non-contiguous row cannot
# name a sub-span, so it is one region anchored at the allocation.
use_packed_row = (
not isinstance(layer_spec, (MambaSpec, KpoolTailSpec))
and storage_is_block_major
and block_stride > 0
and (is_mla_region or not hnc_contiguous)
)
if use_packed_row:
page_contiguous = (
cache.ndim > 1
and cache[0].is_contiguous()
and 0 < cache[0].nbytes <= block_stride
)
if page_contiguous:
view_addr = cache.data_ptr()
kv_block_len = cache[0].nbytes
else:
# NIXL branch B: length is the row, base is the allocation.
view_addr = storage_addr
kv_block_len = block_stride
self.opaque_packed_storages.add(storage_addr)
# Same PP on both sides still aligns by the first layer name.
# A remote PP mismatch is rejected at send time, once the
# peer's PP size is known.
dedupe_key = (view_addr, block_stride, kv_block_len)
region_idx = packed_view_to_region.get(dedupe_key)
if region_idx is not None:
collapsed_views += 1
# -1 means these groups share the view. Flatten only them.
groups = list(self.region_shared_groups[region_idx])
if group_index not in groups:
groups.append(group_index)
self.region_shared_groups[region_idx] = tuple(groups)
if len(self.region_shared_groups[region_idx]) > 1:
self.registered_group_indices[region_idx] = (
_SHARED_REGION_GROUP_ID
)
continue
if not page_contiguous:
self.opaque_packed_storages.add(storage_addr)
if storage_addr not in packed_storage_to_region:
packed_storage_to_region[storage_addr] = len(region_base_addresses)
packed_view_to_region[dedupe_key] = len(region_base_addresses)
region_base_addresses.append(view_addr)
self.block_len_per_layer.append(block_stride)
self.kv_block_len_per_layer.append(kv_block_len)
self.registered_layer_names.append(layer_name)
self.registered_layer_indices.append(layer_index)
self.registered_group_indices.append(group_index)
self.region_shared_groups.append((group_index,))
self.region_row_offsets.append(view_addr - storage_addr)
continue
block_is_contiguous = is_non_overlapping_and_dense(cache[0])
if not block_is_contiguous:
# Non-block-compact layouts scatter a block across per-head
@@ -1791,15 +2126,10 @@ class MooncakeConnectorWorker:
self.kv_block_len_per_layer.append(kv_block_len)
self.registered_layer_names.append(layer_name)
self.registered_layer_indices.append(layer_index)
self.registered_group_indices.append(
self.kv_cache_config.transfer_group_index_by_layer[layer_name]
)
storage = cache.untyped_storage()
storage_addr = storage.data_ptr()
if storage_addr not in seen_storage_ptrs:
seen_storage_ptrs.add(storage_addr)
kv_data_ptrs.append(storage_addr)
kv_data_lens.append(storage.nbytes())
self.registered_group_indices.append(group_index)
self.region_shared_groups.append((group_index,))
# Not a view into a shared packed row, so do not promote it.
self.region_row_offsets.append(-1)
self.kv_caches_base_addr = region_base_addresses
self.seen_base_addresses = kv_data_ptrs
@@ -1807,6 +2137,18 @@ class MooncakeConnectorWorker:
if not kv_data_ptrs:
raise RuntimeError("No KV cache tensors were registered with Mooncake.")
if packed_storage_to_region and (
collapsed_views
or len(packed_view_to_region) != len(packed_storage_to_region)
):
logger.info(
"Mooncake packed KV xfer: %s layer regions over %s storage "
"regions (%s duplicate views collapsed).",
len(packed_view_to_region),
len(packed_storage_to_region),
collapsed_views,
)
ret_value = self.engine.batch_register_memory(kv_data_ptrs, kv_data_lens)
if ret_value != 0:
raise RuntimeError("Mooncake batch memory registration failed.")
@@ -1962,6 +2304,10 @@ class MooncakeConnectorWorker:
registered_layer_names=self.registered_layer_names,
registered_layer_indices=self.registered_layer_indices,
registered_group_indices=self.registered_group_indices,
registered_shared_group_ids=[
list(groups) for groups in self.region_shared_groups
],
registered_row_offsets=self.region_row_offsets,
)
encoded_data = self._encoder.encode(metadata)
@@ -2209,6 +2555,131 @@ class MooncakeConnectorWorker:
def _producer_cache_is_replicated(self) -> bool:
return self.transfer_topo.local_replicates_kv_cache
def _prepare_transfer_regions(
self, meta: MooncakeXferMetadata
) -> tuple[list[TransferRegion], list[TransferRegion], str | None]:
"""Align and coalesce once per peer layout.
``send_kv_to_decode`` runs on every scheduler batch. The registered
layout does not change until the next ``register_kv_caches``.
"""
cache_key = (
meta.remote_hostname,
meta.remote_port,
meta.remote_tp_size,
meta.remote_tp_rank,
meta.remote_pp_size,
tuple(meta.kv_caches_base_addr),
tuple(meta.block_lens),
tuple(meta.kv_block_lens),
tuple(meta.registered_layer_names),
tuple(meta.registered_layer_indices),
tuple(meta.registered_group_indices),
tuple(tuple(groups) for groups in meta.registered_shared_group_ids),
tuple(meta.registered_row_offsets),
)
cached = self._prepared_transfer_regions.get(cache_key)
if cached is not None:
return cached
def finish(
local: list[TransferRegion],
remote: list[TransferRegion],
err: str | None,
) -> tuple[list[TransferRegion], list[TransferRegion], str | None]:
prepared = (local, remote, err)
self._prepared_transfer_regions[cache_key] = prepared
return prepared
shared_groups: list[tuple[int, ...]] | None = self.region_shared_groups
if len(self.region_shared_groups) != len(self.registered_layer_names):
if _SHARED_REGION_GROUP_ID in self.registered_group_indices:
return finish(
[],
[],
(
"Mooncake shared-group metadata does not match "
"registered layers, so a shared packed region cannot "
"be transferred."
),
)
shared_groups = None
local_rows: list[int] | None = self.region_row_offsets
if len(self.region_row_offsets) != len(self.registered_layer_names):
local_rows = None
local_regions = self._get_transfer_regions(
self.kv_caches_base_addr,
self.block_len_per_layer,
self.kv_block_len_per_layer,
self.registered_layer_names,
self.registered_layer_indices,
self.registered_group_indices,
shared_groups,
local_rows,
)
remote_shared_groups = [
tuple(groups) for groups in meta.registered_shared_group_ids
]
remote_shared: list[tuple[int, ...]] | None = remote_shared_groups
if len(remote_shared_groups) != len(meta.registered_layer_names):
remote_shared = None
remote_rows = meta.registered_row_offsets
remote_row_offsets: list[int] | None = remote_rows
if len(remote_rows) != len(meta.registered_layer_names):
remote_row_offsets = None
remote_regions = self._get_transfer_regions(
meta.kv_caches_base_addr,
meta.block_lens,
meta.kv_block_lens,
meta.registered_layer_names,
meta.registered_layer_indices,
meta.registered_group_indices,
remote_shared,
remote_row_offsets,
)
pre_align_local = local_regions
pre_align_remote = remote_regions
local_regions, remote_regions, align_err = _align_transfer_regions(
local_regions,
remote_regions,
allow_partial_layers=meta.remote_pp_size != self.pp_size,
)
if align_err is not None:
return finish([], [], align_err)
# Head checks must see every layer name. Coalesce keeps only the first.
head_err = self._validate_head_resharding_layout(
meta.remote_tp_size, local_regions
)
if head_err is not None:
return finish([], [], head_err)
# Hetero TP indexes kv_block_len as one head-sharded payload. A merged
# multi-layer span would land that offset on the wrong layer.
# Replicated KV keeps that offset at 0. MLA is replicated in practice;
# gating on use_mla alone would promote and then fail the length check
# for a non-replicated MLA rank with tp_ratio != 1.
tp_ratio = _get_tp_ratio(self.tp_size, meta.remote_tp_size)
if tp_ratio == 1 or self._producer_cache_is_replicated():
covered_both_sides = len(local_regions) == len(pre_align_local) and len(
remote_regions
) == len(pre_align_remote)
n_aligned = len(local_regions)
local_regions, remote_regions = _coalesce_contiguous_transfer_regions(
local_regions,
remote_regions,
promote_full_row=covered_both_sides,
)
logger.info(
"Mooncake packed coalesce: %s aligned regions -> %s transfer regions.",
n_aligned,
len(local_regions),
)
self._prepared_transfer_regions[cache_key] = (
local_regions,
remote_regions,
None,
)
return local_regions, remote_regions, None
def _validate_head_resharding_layout(
self, remote_tp_size: int, local_regions: list[TransferRegion]
) -> str | None:
@@ -2242,6 +2713,8 @@ class MooncakeConnectorWorker:
layer_names: list[str],
layer_indices: list[int],
group_indices: list[int] | None = None,
shared_group_ids: list[tuple[int, ...]] | None = None,
row_offsets: list[int] | None = None,
) -> list[TransferRegion]:
if not group_indices:
group_indices = [
@@ -2255,6 +2728,8 @@ class MooncakeConnectorWorker:
layer_names=layer_names,
layer_indices=layer_indices,
group_indices=group_indices,
shared_group_ids=shared_group_ids,
row_offsets=row_offsets,
)
def _get_sender_transfer_plan(