[KVConnector][1/N] PP-aware handshake aggregation and intermediate-PP output plumbing (#43720)

Signed-off-by: zixi-qi <zixi@inferact.ai>
This commit is contained in:
qizixi
2026-06-04 22:04:19 -07:00
committed by GitHub
parent da1daf40bf
commit 96229fa99e
9 changed files with 459 additions and 26 deletions
@@ -368,7 +368,7 @@ def get_current_attn_backend(
class EngineTransferInfo:
"""Common per-remote-engine transfer state, computed at handshake.
Stored per ``engine_id`` inside ``TransferTopology._engines``.
Stored per ``(engine_id, pp_rank)`` inside ``TransferTopology._engines``.
"""
remote_tp_size: int
@@ -382,6 +382,15 @@ class EngineTransferInfo:
remote_physical_blocks_per_logical: int
"""Physical blocks per logical block."""
remote_pp_rank: int = 0
"""Remote producer PP rank for this engine."""
start_layer: int = 0
"""Global index of the first layer owned by this PP rank."""
end_layer: int = 0
"""Exclusive global index after the last layer owned by this PP rank."""
# ---- Transfer topology ----
@@ -403,7 +412,7 @@ class TransferTopology:
def __post_init__(self):
self.local_physical_heads = max(1, self.total_num_kv_heads // self.tp_size)
self._engines: dict[EngineId, EngineTransferInfo] = {}
self._engines: dict[tuple[EngineId, int], EngineTransferInfo] = {}
# Figure out whether the first dimension of the cache is K/V
# or num_blocks.
@@ -461,13 +470,16 @@ class TransferTopology:
f"Cannot register local engine {self.engine_id} as remote. "
f"Local identity is set via __init__ params."
)
if remote_engine_id in self._engines:
return self._engines[remote_engine_id]
self._engines[remote_engine_id] = info
engine_key = (remote_engine_id, info.remote_pp_rank)
if engine_key in self._engines:
return self._engines[engine_key]
self._engines[engine_key] = info
return info
def get_engine_info(self, remote_engine_id: EngineId) -> EngineTransferInfo:
return self._engines[remote_engine_id]
def get_engine_info(
self, remote_engine_id: EngineId, remote_pp_rank: int = 0
) -> EngineTransferInfo:
return self._engines[(remote_engine_id, remote_pp_rank)]
# ============================================================
# Layout properties
@@ -528,15 +540,22 @@ class TransferTopology:
)
return self.block_size // remote_block_size
def is_kv_replicated(self, remote_engine_id: EngineId) -> bool:
def is_kv_replicated(
self, remote_engine_id: EngineId, remote_pp_rank: int = 0
) -> bool:
"""Whether the KV cache is replicated across TP workers due to the
number of TP workers being greater than the number of KV heads.
"""
return self._engines[remote_engine_id].remote_tp_size > self.total_num_kv_heads
return (
self._engines[(remote_engine_id, remote_pp_rank)].remote_tp_size
> self.total_num_kv_heads
)
def replicates_kv_cache(self, remote_engine_id: EngineId) -> bool:
def replicates_kv_cache(
self, remote_engine_id: EngineId, remote_pp_rank: int = 0
) -> bool:
# MLA is always replicated as the hidden dim can't be split.
return self.is_mla or self.is_kv_replicated(remote_engine_id)
return self.is_mla or self.is_kv_replicated(remote_engine_id, remote_pp_rank)
@property
def local_replicates_kv_cache(self) -> bool:
@@ -555,12 +574,14 @@ class TransferTopology:
abs_ratio = -tp_ratio
return [self.tp_rank * abs_ratio + i for i in range(abs_ratio)]
def target_remote_ranks(self, remote_engine_id: EngineId) -> list[int]:
def target_remote_ranks(
self, remote_engine_id: EngineId, remote_pp_rank: int = 0
) -> list[int]:
"""Get the remote TP rank(s) that the current local TP rank will
read from. When remote tp_size > local tp_size, reads from
multiple remote ranks.
"""
info = self._engines[remote_engine_id]
info = self._engines[(remote_engine_id, remote_pp_rank)]
tp_ratio = self.tp_ratio(info.remote_tp_size)
if tp_ratio > 0:
return [self.tp_rank // tp_ratio]
@@ -593,15 +614,16 @@ class TransferTopology:
# Regular case: backends like FA register K/V in separate regions
return cache if self.split_k_and_v else [cache]
def describe(self, remote_engine_id: EngineId) -> str:
def describe(self, remote_engine_id: EngineId, remote_pp_rank: int = 0) -> str:
"""One-line summary of transfer config for logging."""
info = self._engines[remote_engine_id]
info = self._engines[(remote_engine_id, remote_pp_rank)]
return (
f"TransferTopology("
f"tp_ratio={self.tp_ratio(info.remote_tp_size)}, "
f"num_kv_heads={self.total_num_kv_heads if not self.is_mla else 1}, "
f"local_tp={self.tp_size}, "
f"remote_tp={info.remote_tp_size}, "
f"remote_pp={remote_pp_rank}, "
f"local_rank={self.tp_rank}, "
f"remote_block_len={info.remote_block_len})"
)
@@ -644,6 +644,23 @@ class KVConnectorBase_V1(ABC):
"""
return None
def set_xfer_handshake_metadata_pp_aware(
self, metadata: dict[tuple[int, int], KVConnectorHandshakeMetadata]
) -> None:
"""
Set handshake metadata keyed by (pp_rank, tp_rank).
- Default implementation assumes pp_rank is always 0
- PP-aware connectors override this to consume all PP producer shards.
"""
if any(pp_rank != 0 for pp_rank, _ in metadata):
raise ValueError(
f"{type(self).__name__} received pp_rank > 0 handshake metadata "
"but does not support PP-disaggregated KV transfer."
)
self.set_xfer_handshake_metadata(
{tp_rank: meta for (_, tp_rank), meta in metadata.items()}
)
@classmethod
def build_prom_metrics(
cls,
@@ -471,6 +471,12 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
for c in self._connectors:
c.set_xfer_handshake_metadata(metadata)
def set_xfer_handshake_metadata_pp_aware(
self, metadata: dict[tuple[int, int], KVConnectorHandshakeMetadata]
) -> None:
for c in self._connectors:
c.set_xfer_handshake_metadata_pp_aware(metadata)
def _aggregate_request_finished(
self,
request: "Request",