[KVConnector][1/N] PP-aware handshake aggregation and intermediate-PP output plumbing (#43720)
Signed-off-by: zixi-qi <zixi@inferact.ai>
This commit is contained in:
@@ -368,7 +368,7 @@ def get_current_attn_backend(
|
||||
class EngineTransferInfo:
|
||||
"""Common per-remote-engine transfer state, computed at handshake.
|
||||
|
||||
Stored per ``engine_id`` inside ``TransferTopology._engines``.
|
||||
Stored per ``(engine_id, pp_rank)`` inside ``TransferTopology._engines``.
|
||||
"""
|
||||
|
||||
remote_tp_size: int
|
||||
@@ -382,6 +382,15 @@ class EngineTransferInfo:
|
||||
remote_physical_blocks_per_logical: int
|
||||
"""Physical blocks per logical block."""
|
||||
|
||||
remote_pp_rank: int = 0
|
||||
"""Remote producer PP rank for this engine."""
|
||||
|
||||
start_layer: int = 0
|
||||
"""Global index of the first layer owned by this PP rank."""
|
||||
|
||||
end_layer: int = 0
|
||||
"""Exclusive global index after the last layer owned by this PP rank."""
|
||||
|
||||
|
||||
# ---- Transfer topology ----
|
||||
|
||||
@@ -403,7 +412,7 @@ class TransferTopology:
|
||||
def __post_init__(self):
|
||||
self.local_physical_heads = max(1, self.total_num_kv_heads // self.tp_size)
|
||||
|
||||
self._engines: dict[EngineId, EngineTransferInfo] = {}
|
||||
self._engines: dict[tuple[EngineId, int], EngineTransferInfo] = {}
|
||||
|
||||
# Figure out whether the first dimension of the cache is K/V
|
||||
# or num_blocks.
|
||||
@@ -461,13 +470,16 @@ class TransferTopology:
|
||||
f"Cannot register local engine {self.engine_id} as remote. "
|
||||
f"Local identity is set via __init__ params."
|
||||
)
|
||||
if remote_engine_id in self._engines:
|
||||
return self._engines[remote_engine_id]
|
||||
self._engines[remote_engine_id] = info
|
||||
engine_key = (remote_engine_id, info.remote_pp_rank)
|
||||
if engine_key in self._engines:
|
||||
return self._engines[engine_key]
|
||||
self._engines[engine_key] = info
|
||||
return info
|
||||
|
||||
def get_engine_info(self, remote_engine_id: EngineId) -> EngineTransferInfo:
|
||||
return self._engines[remote_engine_id]
|
||||
def get_engine_info(
|
||||
self, remote_engine_id: EngineId, remote_pp_rank: int = 0
|
||||
) -> EngineTransferInfo:
|
||||
return self._engines[(remote_engine_id, remote_pp_rank)]
|
||||
|
||||
# ============================================================
|
||||
# Layout properties
|
||||
@@ -528,15 +540,22 @@ class TransferTopology:
|
||||
)
|
||||
return self.block_size // remote_block_size
|
||||
|
||||
def is_kv_replicated(self, remote_engine_id: EngineId) -> bool:
|
||||
def is_kv_replicated(
|
||||
self, remote_engine_id: EngineId, remote_pp_rank: int = 0
|
||||
) -> bool:
|
||||
"""Whether the KV cache is replicated across TP workers due to the
|
||||
number of TP workers being greater than the number of KV heads.
|
||||
"""
|
||||
return self._engines[remote_engine_id].remote_tp_size > self.total_num_kv_heads
|
||||
return (
|
||||
self._engines[(remote_engine_id, remote_pp_rank)].remote_tp_size
|
||||
> self.total_num_kv_heads
|
||||
)
|
||||
|
||||
def replicates_kv_cache(self, remote_engine_id: EngineId) -> bool:
|
||||
def replicates_kv_cache(
|
||||
self, remote_engine_id: EngineId, remote_pp_rank: int = 0
|
||||
) -> bool:
|
||||
# MLA is always replicated as the hidden dim can't be split.
|
||||
return self.is_mla or self.is_kv_replicated(remote_engine_id)
|
||||
return self.is_mla or self.is_kv_replicated(remote_engine_id, remote_pp_rank)
|
||||
|
||||
@property
|
||||
def local_replicates_kv_cache(self) -> bool:
|
||||
@@ -555,12 +574,14 @@ class TransferTopology:
|
||||
abs_ratio = -tp_ratio
|
||||
return [self.tp_rank * abs_ratio + i for i in range(abs_ratio)]
|
||||
|
||||
def target_remote_ranks(self, remote_engine_id: EngineId) -> list[int]:
|
||||
def target_remote_ranks(
|
||||
self, remote_engine_id: EngineId, remote_pp_rank: int = 0
|
||||
) -> list[int]:
|
||||
"""Get the remote TP rank(s) that the current local TP rank will
|
||||
read from. When remote tp_size > local tp_size, reads from
|
||||
multiple remote ranks.
|
||||
"""
|
||||
info = self._engines[remote_engine_id]
|
||||
info = self._engines[(remote_engine_id, remote_pp_rank)]
|
||||
tp_ratio = self.tp_ratio(info.remote_tp_size)
|
||||
if tp_ratio > 0:
|
||||
return [self.tp_rank // tp_ratio]
|
||||
@@ -593,15 +614,16 @@ class TransferTopology:
|
||||
# Regular case: backends like FA register K/V in separate regions
|
||||
return cache if self.split_k_and_v else [cache]
|
||||
|
||||
def describe(self, remote_engine_id: EngineId) -> str:
|
||||
def describe(self, remote_engine_id: EngineId, remote_pp_rank: int = 0) -> str:
|
||||
"""One-line summary of transfer config for logging."""
|
||||
info = self._engines[remote_engine_id]
|
||||
info = self._engines[(remote_engine_id, remote_pp_rank)]
|
||||
return (
|
||||
f"TransferTopology("
|
||||
f"tp_ratio={self.tp_ratio(info.remote_tp_size)}, "
|
||||
f"num_kv_heads={self.total_num_kv_heads if not self.is_mla else 1}, "
|
||||
f"local_tp={self.tp_size}, "
|
||||
f"remote_tp={info.remote_tp_size}, "
|
||||
f"remote_pp={remote_pp_rank}, "
|
||||
f"local_rank={self.tp_rank}, "
|
||||
f"remote_block_len={info.remote_block_len})"
|
||||
)
|
||||
|
||||
@@ -644,6 +644,23 @@ class KVConnectorBase_V1(ABC):
|
||||
"""
|
||||
return None
|
||||
|
||||
def set_xfer_handshake_metadata_pp_aware(
|
||||
self, metadata: dict[tuple[int, int], KVConnectorHandshakeMetadata]
|
||||
) -> None:
|
||||
"""
|
||||
Set handshake metadata keyed by (pp_rank, tp_rank).
|
||||
- Default implementation assumes pp_rank is always 0
|
||||
- PP-aware connectors override this to consume all PP producer shards.
|
||||
"""
|
||||
if any(pp_rank != 0 for pp_rank, _ in metadata):
|
||||
raise ValueError(
|
||||
f"{type(self).__name__} received pp_rank > 0 handshake metadata "
|
||||
"but does not support PP-disaggregated KV transfer."
|
||||
)
|
||||
self.set_xfer_handshake_metadata(
|
||||
{tp_rank: meta for (_, tp_rank), meta in metadata.items()}
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def build_prom_metrics(
|
||||
cls,
|
||||
|
||||
@@ -471,6 +471,12 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
for c in self._connectors:
|
||||
c.set_xfer_handshake_metadata(metadata)
|
||||
|
||||
def set_xfer_handshake_metadata_pp_aware(
|
||||
self, metadata: dict[tuple[int, int], KVConnectorHandshakeMetadata]
|
||||
) -> None:
|
||||
for c in self._connectors:
|
||||
c.set_xfer_handshake_metadata_pp_aware(metadata)
|
||||
|
||||
def _aggregate_request_finished(
|
||||
self,
|
||||
request: "Request",
|
||||
|
||||
Reference in New Issue
Block a user