nixl refactor: new transfer design (#40731)
Signed-off-by: ZhanqiuHu <zhu@redhat.com> Signed-off-by: NickLucche <nlucches@redhat.com> Co-authored-by: NickLucche <nlucches@redhat.com>
This commit is contained in:
@@ -386,38 +386,6 @@ class EngineTransferInfo:
|
||||
"""Physical blocks per logical block."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MambaEngineTransferInfo(EngineTransferInfo):
|
||||
"""Extends ``EngineTransferInfo`` with Mamba-hybrid transfer geometry.
|
||||
|
||||
For hybrid SSM+Attention models, FA and Mamba layers may require
|
||||
different numbers of reads from different remote ranks. This
|
||||
dataclass captures that per-engine transfer plan.
|
||||
"""
|
||||
|
||||
remote_fa_source_ranks: tuple[int, ...]
|
||||
"""Remote ranks carrying unique FA heads for this local rank."""
|
||||
|
||||
remote_all_source_ranks: tuple[int, ...]
|
||||
"""All remote ranks this local rank reads from (FA + Mamba)."""
|
||||
|
||||
remote_num_fa_reads: int
|
||||
"""Number of distinct remote ranks needed for FA data."""
|
||||
|
||||
remote_num_mamba_reads: int
|
||||
"""Number of distinct remote ranks needed for Mamba data."""
|
||||
|
||||
remote_fa_descriptor_bytes: int
|
||||
"""Byte size of one FA K (or V) descriptor entry."""
|
||||
|
||||
is_remote_replicated: bool
|
||||
"""Whether the remote engine has replicated KV heads
|
||||
(remote_tp_size > total_num_kv_heads)."""
|
||||
|
||||
remote_physical_heads: int
|
||||
"""Physical KV heads stored per remote rank."""
|
||||
|
||||
|
||||
# ---- Transfer topology ----
|
||||
|
||||
|
||||
@@ -439,8 +407,6 @@ class TransferTopology:
|
||||
self.local_physical_heads = max(1, self.total_num_kv_heads // self.tp_size)
|
||||
|
||||
self._engines: dict[EngineId, EngineTransferInfo] = {}
|
||||
self._fa_source_sets: dict[EngineId, frozenset[int]] = {}
|
||||
self._fa_source_indices: dict[EngineId, dict[int, int]] = {}
|
||||
|
||||
# Figure out whether the first dimension of the cache is K/V
|
||||
# or num_blocks.
|
||||
@@ -487,24 +453,12 @@ class TransferTopology:
|
||||
def register_remote_engine(
|
||||
self,
|
||||
remote_engine_id: EngineId,
|
||||
remote_tp_size: int,
|
||||
remote_block_size: int,
|
||||
remote_block_len: int,
|
||||
remote_physical_blocks_per_logical: int,
|
||||
*,
|
||||
local_block_len: int = 0,
|
||||
info: EngineTransferInfo,
|
||||
) -> EngineTransferInfo:
|
||||
"""Register a remote engine, unifying worker dicts state.
|
||||
|
||||
Only remote engines should be registered here — the local engine's
|
||||
identity (tp_size, block_size, etc.) is set via ``__init__`` params.
|
||||
|
||||
For Mamba models, also computes the Mamba transfer plan and
|
||||
builds the FA source lookup caches.
|
||||
|
||||
Args:
|
||||
local_block_len: Local representative block_len (bytes).
|
||||
Required for Mamba models to compute ``fa_descriptor_bytes``.
|
||||
The caller (worker) is responsible for computing the info via
|
||||
the transfer policy. This method only stores and deduplicates.
|
||||
"""
|
||||
assert remote_engine_id != self.engine_id, (
|
||||
f"Cannot register local engine {self.engine_id} as remote. "
|
||||
@@ -512,29 +466,6 @@ class TransferTopology:
|
||||
)
|
||||
if remote_engine_id in self._engines:
|
||||
return self._engines[remote_engine_id]
|
||||
info: EngineTransferInfo
|
||||
if self.is_mamba:
|
||||
info = self._build_mamba_info(
|
||||
remote_tp_size=remote_tp_size,
|
||||
remote_block_size=remote_block_size,
|
||||
remote_block_len=remote_block_len,
|
||||
remote_physical_blocks_per_logical=(remote_physical_blocks_per_logical),
|
||||
local_block_len=local_block_len,
|
||||
)
|
||||
assert isinstance(info, MambaEngineTransferInfo)
|
||||
self._fa_source_sets[remote_engine_id] = frozenset(
|
||||
info.remote_fa_source_ranks
|
||||
)
|
||||
self._fa_source_indices[remote_engine_id] = {
|
||||
r: i for i, r in enumerate(info.remote_fa_source_ranks)
|
||||
}
|
||||
else:
|
||||
info = EngineTransferInfo(
|
||||
remote_tp_size=remote_tp_size,
|
||||
remote_block_len=remote_block_len,
|
||||
remote_block_size=remote_block_size,
|
||||
remote_physical_blocks_per_logical=(remote_physical_blocks_per_logical),
|
||||
)
|
||||
self._engines[remote_engine_id] = info
|
||||
return info
|
||||
|
||||
@@ -622,14 +553,8 @@ class TransferTopology:
|
||||
"""Get the remote TP rank(s) that the current local TP rank will
|
||||
read from. When remote tp_size > local tp_size, reads from
|
||||
multiple remote ranks.
|
||||
|
||||
For Mamba models, returns the precomputed ``all_source_ranks``
|
||||
(FA + Mamba union).
|
||||
"""
|
||||
info = self._engines[remote_engine_id]
|
||||
if isinstance(info, MambaEngineTransferInfo):
|
||||
return list(info.remote_all_source_ranks)
|
||||
|
||||
tp_ratio = self.tp_ratio(info.remote_tp_size)
|
||||
if tp_ratio > 0:
|
||||
return [self.tp_rank // tp_ratio]
|
||||
@@ -662,309 +587,15 @@ class TransferTopology:
|
||||
# Regular case: backends like FA register K/V in separate regions
|
||||
return cache if self.split_k_and_v else [cache]
|
||||
|
||||
# ============================================================
|
||||
# Mamba-specific methods
|
||||
# ============================================================
|
||||
|
||||
def should_skip_fa(self, remote_engine_id: EngineId, remote_rank: int) -> bool:
|
||||
"""Whether to skip FA groups for this remote rank (mamba-only)."""
|
||||
return remote_rank not in self._fa_source_sets[remote_engine_id]
|
||||
|
||||
def fa_head_slot(self, remote_engine_id: EngineId, remote_rank: int) -> int:
|
||||
"""Index into local FA block for this remote rank's head data.
|
||||
|
||||
For remote ranks in ``fa_source_ranks``, returns 0, 1, …, reads-1.
|
||||
For ranks NOT in ``fa_source_ranks`` (replicated duplicates),
|
||||
returns the slot of the matching source rank with the same head.
|
||||
"""
|
||||
fa_index = self._fa_source_indices[remote_engine_id]
|
||||
if remote_rank in fa_index:
|
||||
return fa_index[remote_rank]
|
||||
mamba_info = self._engines[remote_engine_id]
|
||||
assert isinstance(mamba_info, MambaEngineTransferInfo)
|
||||
K = self.total_num_kv_heads
|
||||
remote_tp = mamba_info.remote_tp_size
|
||||
r_head = self._physical_head_range(remote_tp, K, remote_rank)
|
||||
for target in mamba_info.remote_fa_source_ranks:
|
||||
t_head = self._physical_head_range(remote_tp, K, target)
|
||||
if self._range_overlap(r_head, t_head):
|
||||
return fa_index[target]
|
||||
return 0
|
||||
|
||||
def fa_rank_offset(
|
||||
self, remote_engine_id: EngineId, remote_kv_block_len: int
|
||||
) -> int:
|
||||
"""Byte offset into remote FA block for this local rank.
|
||||
|
||||
When local TP is replicated (local_tp > K), multiple local ranks
|
||||
share a head. Computes offset *relative to the target remote
|
||||
rank's first head* so it works regardless of how many heads the
|
||||
remote has. Returns 0 when local does not index into remote.
|
||||
"""
|
||||
mamba_info = self._engines[remote_engine_id]
|
||||
assert isinstance(mamba_info, MambaEngineTransferInfo)
|
||||
tp_ratio = self.tp_ratio(mamba_info.remote_tp_size)
|
||||
if self.is_mla or tp_ratio <= 0:
|
||||
return 0
|
||||
K = self.total_num_kv_heads
|
||||
is_local_replicated = self.tp_size > K
|
||||
if is_local_replicated:
|
||||
local_head = self.tp_rank * K // self.tp_size
|
||||
p_rank = mamba_info.remote_fa_source_ranks[0]
|
||||
p_start = p_rank * K // mamba_info.remote_tp_size
|
||||
return (local_head - p_start) * remote_kv_block_len
|
||||
return self.tp_rank % tp_ratio * remote_kv_block_len
|
||||
|
||||
def needs_split_handles(self, remote_engine_id: EngineId) -> bool:
|
||||
"""Whether per-remote-rank split handles are needed.
|
||||
|
||||
True when FA and mamba have different read counts, requiring
|
||||
different splitting factors in the local handle.
|
||||
"""
|
||||
mamba_info = self._engines[remote_engine_id]
|
||||
assert isinstance(mamba_info, MambaEngineTransferInfo)
|
||||
tp_ratio = self.tp_ratio(mamba_info.remote_tp_size)
|
||||
return (
|
||||
tp_ratio < 0
|
||||
and not self.is_mla
|
||||
and len(mamba_info.remote_all_source_ranks) > 1
|
||||
)
|
||||
|
||||
def compute_split_handle_data(
|
||||
self,
|
||||
remote_engine_id: EngineId,
|
||||
src_blocks_data: list[tuple[int, int, int]],
|
||||
num_fa_descs: int,
|
||||
abs_tp: int,
|
||||
) -> list[list[tuple[int, int, int]]]:
|
||||
"""Per-remote-rank (addr, len, dev) triples for Mamba-HMA split
|
||||
handles.
|
||||
|
||||
FA descriptors (indices < num_fa_descs) are sliced by
|
||||
``remote_num_fa_reads``; mamba descriptors are sliced uniformly
|
||||
by ``abs_tp``.
|
||||
"""
|
||||
mamba_info = self._engines[remote_engine_id]
|
||||
assert isinstance(mamba_info, MambaEngineTransferInfo)
|
||||
all_handle_data: list[list[tuple[int, int, int]]] = []
|
||||
for p_idx, p_rank in enumerate(mamba_info.remote_all_source_ranks):
|
||||
handle_data: list[tuple[int, int, int]] = []
|
||||
skip_fa = self.should_skip_fa(remote_engine_id, p_rank)
|
||||
fa_slot = self.fa_head_slot(remote_engine_id, p_rank) if not skip_fa else 0
|
||||
for j, (addr, local_len, dev) in enumerate(src_blocks_data):
|
||||
if j < num_fa_descs:
|
||||
assert mamba_info.remote_num_fa_reads >= 1
|
||||
fa_chunk = local_len // mamba_info.remote_num_fa_reads
|
||||
handle_data.append((addr + fa_slot * fa_chunk, fa_chunk, dev))
|
||||
else:
|
||||
mamba_chunk = local_len // abs_tp
|
||||
handle_data.append((addr + p_idx * mamba_chunk, mamba_chunk, dev))
|
||||
all_handle_data.append(handle_data)
|
||||
return all_handle_data
|
||||
|
||||
def filter_block_ids_for_rank(
|
||||
self,
|
||||
remote_engine_id: EngineId,
|
||||
remote_rank: int,
|
||||
local_ids: BlockIds,
|
||||
remote_ids: BlockIds,
|
||||
is_mamba_group: list[bool],
|
||||
) -> tuple[BlockIds, BlockIds]:
|
||||
"""Zero out FA groups for remote ranks outside ``fa_source_ranks``.
|
||||
|
||||
Returns (filtered_local_ids, filtered_remote_ids). When the
|
||||
remote rank carries FA data for this local rank, returns the
|
||||
inputs unchanged.
|
||||
"""
|
||||
if not self.should_skip_fa(remote_engine_id, remote_rank):
|
||||
return local_ids, remote_ids
|
||||
num_groups = len(local_ids)
|
||||
filtered_local: list[list[int]] = [
|
||||
[] if not is_mamba_group[g] else local_ids[g] for g in range(num_groups)
|
||||
]
|
||||
filtered_remote: list[list[int]] = [
|
||||
[] if not is_mamba_group[g] else remote_ids[g] for g in range(num_groups)
|
||||
]
|
||||
return filtered_local, filtered_remote
|
||||
|
||||
def describe(self, remote_engine_id: EngineId) -> str:
|
||||
"""One-line summary of transfer config for logging."""
|
||||
info = self._engines[remote_engine_id]
|
||||
base = (
|
||||
return (
|
||||
f"TransferTopology("
|
||||
f"tp_ratio={self.tp_ratio(info.remote_tp_size)}, "
|
||||
f"K={self.total_num_kv_heads}, "
|
||||
f"local_tp={self.tp_size}, "
|
||||
f"remote_tp={info.remote_tp_size}, "
|
||||
f"local_rank={self.tp_rank}, "
|
||||
f"remote_block_len={info.remote_block_len}"
|
||||
)
|
||||
if isinstance(info, MambaEngineTransferInfo):
|
||||
return (
|
||||
f"TransferTopology.mamba({base}, "
|
||||
f"fa_reads={info.remote_num_fa_reads}, "
|
||||
f"mamba_reads={info.remote_num_mamba_reads}, "
|
||||
f"fa_sources={list(info.remote_fa_source_ranks)}, "
|
||||
f"all_sources={list(info.remote_all_source_ranks)}, "
|
||||
f"fa_desc_bytes={info.remote_fa_descriptor_bytes})"
|
||||
)
|
||||
return f"TransferTopology({base})"
|
||||
|
||||
# ============================================================
|
||||
# Private helpers
|
||||
# ============================================================
|
||||
# Mamba-HMA hetero-TP transfer config:
|
||||
# With hetero-TP (P_TP > D_TP), FA KV cache may be replicated across
|
||||
# P ranks (when P_TP > num_kv_heads), but Mamba conv/SSM state is
|
||||
# almost always uniquely sharded per P rank. So the number of P
|
||||
# ranks D must read from can differ between FA and Mamba, and they
|
||||
# must be handled separately.
|
||||
|
||||
@staticmethod
|
||||
def _physical_head_range(tp_size: int, num_heads: int, rank: int) -> range:
|
||||
"""Physical KV head range stored in a rank's KV cache tensor.
|
||||
|
||||
When ``tp_size <= num_heads``: sharded, K/TP contiguous heads per rank.
|
||||
When ``tp_size > num_heads``: 1 physical head per rank. Heads are
|
||||
distributed **contiguously** (matching vLLM's GQA weight partitioning):
|
||||
consecutive ranks share a head before moving to the next one.
|
||||
"""
|
||||
if tp_size <= num_heads:
|
||||
assert num_heads % tp_size == 0
|
||||
per_rank = num_heads // tp_size
|
||||
return range(rank * per_rank, (rank + 1) * per_rank)
|
||||
else:
|
||||
h = rank * num_heads // tp_size
|
||||
return range(h, h + 1)
|
||||
|
||||
@staticmethod
|
||||
def _range_overlap(a: range, b: range) -> range:
|
||||
start = max(a.start, b.start)
|
||||
stop = min(a.stop, b.stop)
|
||||
return range(start, max(start, stop))
|
||||
|
||||
# ============================================================
|
||||
# Private: build Mamba transfer info
|
||||
# ============================================================
|
||||
|
||||
def _build_mamba_info(
|
||||
self,
|
||||
remote_tp_size: int,
|
||||
remote_block_size: int,
|
||||
remote_block_len: int,
|
||||
remote_physical_blocks_per_logical: int,
|
||||
local_block_len: int,
|
||||
) -> MambaEngineTransferInfo:
|
||||
"""Compute Mamba transfer plan."""
|
||||
K = self.total_num_kv_heads
|
||||
local_tp = self.tp_size
|
||||
local_rank = self.tp_rank
|
||||
|
||||
is_remote_replicated = remote_tp_size > K
|
||||
remote_physical_heads = max(1, K // remote_tp_size)
|
||||
|
||||
if local_tp >= remote_tp_size:
|
||||
assert local_tp % remote_tp_size == 0
|
||||
tp_ratio = local_tp // remote_tp_size
|
||||
else:
|
||||
assert remote_tp_size % local_tp == 0
|
||||
tp_ratio = -(remote_tp_size // local_tp)
|
||||
|
||||
abs_tp = -tp_ratio if tp_ratio < 0 else 1
|
||||
|
||||
mamba_range: range | None = None
|
||||
if tp_ratio < 0:
|
||||
mamba_range = range(local_rank * abs_tp, (local_rank + 1) * abs_tp)
|
||||
|
||||
# ---- FA read targets ----
|
||||
if self.is_mla or tp_ratio >= 0:
|
||||
num_fa_reads = 1
|
||||
fa_source_ranks: list[int] = (
|
||||
[0]
|
||||
if self.is_mla
|
||||
else [local_rank // tp_ratio if tp_ratio > 0 else local_rank]
|
||||
)
|
||||
else:
|
||||
local_needs = self._physical_head_range(local_tp, K, local_rank)
|
||||
search_range = (
|
||||
mamba_range if mamba_range is not None else range(remote_tp_size)
|
||||
)
|
||||
seen: set[tuple[int, int]] = set()
|
||||
fa_source_ranks = []
|
||||
for p in search_range:
|
||||
p_has = self._physical_head_range(remote_tp_size, K, p)
|
||||
ov = self._range_overlap(local_needs, p_has)
|
||||
if len(ov) > 0:
|
||||
key = (ov.start, ov.stop)
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
fa_source_ranks.append(p)
|
||||
if not fa_source_ranks:
|
||||
for p in range(remote_tp_size):
|
||||
p_has = self._physical_head_range(remote_tp_size, K, p)
|
||||
ov = self._range_overlap(local_needs, p_has)
|
||||
if len(ov) > 0:
|
||||
key = (ov.start, ov.stop)
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
fa_source_ranks.append(p)
|
||||
num_fa_reads = len(fa_source_ranks)
|
||||
|
||||
# ---- All source ranks (mamba + FA) ----
|
||||
if mamba_range is not None and abs_tp > num_fa_reads:
|
||||
num_mamba_reads = abs_tp
|
||||
all_source_ranks = list(mamba_range)
|
||||
else:
|
||||
num_mamba_reads = num_fa_reads
|
||||
all_source_ranks = list(fa_source_ranks)
|
||||
|
||||
# ---- FA descriptor bytes ----
|
||||
effective_block_len = min(local_block_len, remote_block_len)
|
||||
if self.is_kv_layout_blocks_first:
|
||||
fa_descriptor_bytes = effective_block_len // 2
|
||||
else:
|
||||
fa_descriptor_bytes = effective_block_len
|
||||
|
||||
# ---- Validation ----
|
||||
is_local_replicated = local_tp > K
|
||||
if is_local_replicated and is_remote_replicated and tp_ratio > 0:
|
||||
logger.info(
|
||||
"Both-replicated hetero-TP: local_tp=%d > remote_tp=%d > K=%d.",
|
||||
local_tp,
|
||||
remote_tp_size,
|
||||
K,
|
||||
)
|
||||
tt_set = set(all_source_ranks)
|
||||
for t in fa_source_ranks:
|
||||
if t not in tt_set:
|
||||
logger.error(
|
||||
"FA source rank %d NOT in all_source_ranks %s.",
|
||||
t,
|
||||
all_source_ranks,
|
||||
)
|
||||
if self.is_kv_layout_blocks_first and tp_ratio < 0 and num_fa_reads > 0:
|
||||
local_k_half = local_block_len // 2
|
||||
remote_k_half = remote_block_len // 2
|
||||
expected = local_k_half // num_fa_reads
|
||||
if expected != remote_k_half:
|
||||
logger.warning(
|
||||
"FA size mismatch: local_k_half=%d / reads=%d = %d, "
|
||||
"but remote_k_half=%d.",
|
||||
local_k_half,
|
||||
num_fa_reads,
|
||||
expected,
|
||||
remote_k_half,
|
||||
)
|
||||
|
||||
return MambaEngineTransferInfo(
|
||||
remote_tp_size=remote_tp_size,
|
||||
remote_block_len=remote_block_len,
|
||||
remote_block_size=remote_block_size,
|
||||
remote_physical_blocks_per_logical=(remote_physical_blocks_per_logical),
|
||||
remote_fa_source_ranks=tuple(fa_source_ranks),
|
||||
remote_all_source_ranks=tuple(all_source_ranks),
|
||||
remote_num_fa_reads=num_fa_reads,
|
||||
remote_num_mamba_reads=num_mamba_reads,
|
||||
remote_fa_descriptor_bytes=fa_descriptor_bytes,
|
||||
is_remote_replicated=is_remote_replicated,
|
||||
remote_physical_heads=remote_physical_heads,
|
||||
f"remote_block_len={info.remote_block_len})"
|
||||
)
|
||||
|
||||
@@ -32,8 +32,9 @@ GET_META_MSG = b"get_meta_msg"
|
||||
# Version History:
|
||||
# 1: Initial version with compatibility checking
|
||||
# 2: Add remote_request_id to kv_transfer_params
|
||||
# 3: Add physical_blocks_per_logical_kv_block to NixlAgentMetadata
|
||||
#
|
||||
NIXL_CONNECTOR_VERSION: int = 2
|
||||
NIXL_CONNECTOR_VERSION: int = 3
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -48,6 +49,7 @@ class NixlAgentMetadata:
|
||||
block_size: int
|
||||
ssm_sizes: tuple[int, int]
|
||||
attn_backend_name: str
|
||||
physical_blocks_per_logical_kv_block: int
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""TP mapping computation for NIXL KV cache transfers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import (
|
||||
BlockIds,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheSpec, MambaSpec
|
||||
|
||||
# ======================================================================
|
||||
# Data structures
|
||||
# ======================================================================
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReadSpec:
|
||||
"""Specification for a single remote block read operation."""
|
||||
|
||||
remote_rank: int
|
||||
local_block_ids: BlockIds
|
||||
remote_block_ids: BlockIds
|
||||
|
||||
|
||||
def _is_attention_spec(spec_type: type[KVCacheSpec]) -> bool:
|
||||
return issubclass(spec_type, AttentionSpec)
|
||||
|
||||
|
||||
def _is_ssm_spec(spec_type: type[KVCacheSpec]) -> bool:
|
||||
return issubclass(spec_type, MambaSpec)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TPMapping:
|
||||
"""Complete local-to-remote TP mapping for one remote engine.
|
||||
|
||||
Generated once per remote engine during handshake.
|
||||
"""
|
||||
|
||||
# Remote TP ranks that this local rank reads from, per group.
|
||||
# Position = local piece index.
|
||||
source_ranks_per_group: tuple[tuple[int, ...], ...]
|
||||
|
||||
# Superset of all source ranks (union of all groups).
|
||||
all_source_ranks: tuple[int, ...]
|
||||
|
||||
# Maps each source rank to its FA head slot index.
|
||||
rank_to_attention_slot: dict[int, int]
|
||||
|
||||
# FA head offset factor for hetero-TP (D_TP > P_TP).
|
||||
rank_offset_factor: int
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# TP mapping computation
|
||||
# ======================================================================
|
||||
|
||||
|
||||
def compute_tp_mapping(
|
||||
tp_rank: int,
|
||||
tp_size: int,
|
||||
remote_tp_size: int,
|
||||
is_mla: bool,
|
||||
total_num_kv_heads: int,
|
||||
group_spec_types: tuple[type[KVCacheSpec], ...],
|
||||
) -> TPMapping:
|
||||
"""Build the complete local-to-remote TP mapping.
|
||||
|
||||
Computes source ranks, head slot assignments, and the rank offset
|
||||
factor in a single pass.
|
||||
"""
|
||||
# --- Attention source ranks ---
|
||||
if is_mla:
|
||||
# All heads replicated across all ranks.
|
||||
attn_ranks = [0]
|
||||
elif tp_size >= remote_tp_size:
|
||||
# D (local TP) > P (remote TP): multiple local ranks read different chunks from
|
||||
# *one* remote rank, corresponding to different kv heads.
|
||||
attn_ranks = [tp_rank * remote_tp_size // tp_size]
|
||||
else:
|
||||
# P (remote TP) > D (local TP): one local rank
|
||||
# reads from multiple remote ranks.
|
||||
# GQA dedup: when K < remote_tp_size, several remote ranks
|
||||
# hold the same KV head. np.unique keeps only the first
|
||||
# rank per unique head so we don't issue redundant reads.
|
||||
abs_tp = remote_tp_size // tp_size
|
||||
start = tp_rank * abs_tp
|
||||
heads = np.arange(start, start + abs_tp) * total_num_kv_heads // remote_tp_size
|
||||
_, unique_idx = np.unique(heads, return_index=True)
|
||||
attn_ranks = (start + np.sort(unique_idx)).tolist()
|
||||
|
||||
# --- SSM source ranks ---
|
||||
has_ssm = any(_is_ssm_spec(t) for t in group_spec_types)
|
||||
if has_ssm:
|
||||
if tp_size < remote_tp_size:
|
||||
abs_tp = remote_tp_size // tp_size
|
||||
ssm_ranks = list(range(tp_rank * abs_tp, (tp_rank + 1) * abs_tp))
|
||||
else:
|
||||
ssm_ranks = list(attn_ranks)
|
||||
else:
|
||||
ssm_ranks = []
|
||||
|
||||
all_ranks = sorted(set(attn_ranks) | set(ssm_ranks))
|
||||
|
||||
# --- Per-group ordered source ranks ---
|
||||
source_ranks_per_group = tuple(
|
||||
tuple(ssm_ranks) if _is_ssm_spec(t) else tuple(attn_ranks)
|
||||
for t in group_spec_types
|
||||
)
|
||||
|
||||
# --- Attention head slots ---
|
||||
head_to_slot: dict[int, int] = {}
|
||||
for i, r in enumerate(attn_ranks):
|
||||
head_to_slot[r * total_num_kv_heads // remote_tp_size] = i
|
||||
rank_to_attention_slot = {
|
||||
r: head_to_slot.get(r * total_num_kv_heads // remote_tp_size, 0)
|
||||
for r in all_ranks
|
||||
}
|
||||
|
||||
# --- Rank offset factor ---
|
||||
if is_mla or tp_size <= remote_tp_size:
|
||||
# We don't index into remote for reading, no offset needed.
|
||||
rank_offset_factor = 0
|
||||
elif tp_size > total_num_kv_heads:
|
||||
local_head = tp_rank * total_num_kv_heads // tp_size
|
||||
p_start = attn_ranks[0] * total_num_kv_heads // remote_tp_size
|
||||
rank_offset_factor = local_head - p_start
|
||||
else:
|
||||
# D TP > P TP: we index into remote to read different heads depending on rank.
|
||||
rank_offset_factor = tp_rank % (tp_size // remote_tp_size)
|
||||
|
||||
return TPMapping(
|
||||
source_ranks_per_group=source_ranks_per_group,
|
||||
all_source_ranks=tuple(all_ranks),
|
||||
rank_to_attention_slot=rank_to_attention_slot,
|
||||
rank_offset_factor=rank_offset_factor,
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -31,6 +31,7 @@ class MambaConvSplitInfo:
|
||||
x_local: int # intermediate_size / TP (columns for x)
|
||||
b_local: int # groups_ss / TP (columns for B; C is same size)
|
||||
conv_dtype_size: int # bytes per element (e.g. 2 for float16)
|
||||
ssm_sizes: tuple[int, int] # (conv_state_bytes, ssm_state_bytes)
|
||||
|
||||
@property
|
||||
def conv_dim_local(self) -> int:
|
||||
@@ -99,8 +100,8 @@ def derive_mamba_conv_split(
|
||||
local_tp: this engine's tensor-parallel size.
|
||||
|
||||
Returns:
|
||||
MambaConvSplitInfo with per-rank x_local, b_local, conv_rows, and
|
||||
conv_dtype_size.
|
||||
MambaConvSplitInfo with per-rank x_local, b_local, conv_rows,
|
||||
conv_dtype_size, and ssm_sizes (conv_state_bytes, ssm_state_bytes).
|
||||
"""
|
||||
if mamba_spec.mamba_type != "mamba2":
|
||||
raise NotImplementedError(
|
||||
@@ -142,12 +143,20 @@ def derive_mamba_conv_split(
|
||||
dtype=mamba_spec.dtypes[0], # type: ignore[misc]
|
||||
).element_size()
|
||||
|
||||
ssm_dtype_size = torch.tensor(
|
||||
[],
|
||||
dtype=mamba_spec.dtypes[1], # type: ignore[misc]
|
||||
).element_size()
|
||||
conv_state_bytes = torch.Size(mamba_spec.shapes[0]).numel() * conv_dtype_size
|
||||
ssm_state_bytes = torch.Size(mamba_spec.shapes[1]).numel() * ssm_dtype_size
|
||||
|
||||
# Divide by TP to get per-rank column counts.
|
||||
return MambaConvSplitInfo(
|
||||
conv_rows=conv_rows,
|
||||
x_local=intermediate_size // local_tp,
|
||||
b_local=groups_ss // local_tp,
|
||||
conv_dtype_size=conv_dtype_size,
|
||||
ssm_sizes=(conv_state_bytes, ssm_state_bytes),
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user