[Bugfix] Fix NixlConnector handshake block_len validation for GQA-replicated KV heads (#45879)
Signed-off-by: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Co-authored-by: waynehacking8 <waynehacking8@gmail.com>
This commit is contained in:
co-authored by
waynehacking8
parent
5fd21eb0b2
commit
0d339cf135
@@ -1599,7 +1599,9 @@ class NixlBaseConnectorWorker:
|
||||
# Per-region block_len validation enforcing the P/D invariant.
|
||||
# REPLICATE regions (MLA, or a whole-model MLA / replicated-KV transfer)
|
||||
# only allow the number of blocks to differ; SPLIT regions scale with
|
||||
# tp_ratio. Mamba uses the ssm_sizes counterpart, so skip block_len here.
|
||||
# the per-rank KV head ratio rather than the raw tp_ratio, because GQA
|
||||
# replication caps per-rank heads at 1 when tp > total_kv_heads
|
||||
# (issue #45330). Mamba uses the ssm_sizes counterpart, so skip here.
|
||||
if not self._has_mamba:
|
||||
assert len(self.block_len_per_layer) == len(nixl_agent_meta.block_lens), (
|
||||
"Number of KV layers must match between prefill and decode"
|
||||
@@ -1607,6 +1609,9 @@ class NixlBaseConnectorWorker:
|
||||
model_replicated = self.use_mla or self.transfer_topo.is_kv_replicated(
|
||||
remote_engine_id
|
||||
)
|
||||
total_kv_heads = self.transfer_topo.total_num_kv_heads
|
||||
local_heads = self.transfer_topo.local_physical_heads
|
||||
remote_heads = max(1, total_kv_heads // remote_tp_size)
|
||||
for i, local_len in enumerate(self.block_len_per_layer):
|
||||
replicated = model_replicated or self._is_region_replicated(i)
|
||||
remote_len = nixl_agent_meta.block_lens[i]
|
||||
@@ -1617,9 +1622,13 @@ class NixlBaseConnectorWorker:
|
||||
f"remote={remote_len}, bsr={block_size_ratio})."
|
||||
)
|
||||
elif tp_ratio > 0:
|
||||
assert remote_len == (local_len * tp_ratio) // block_size_ratio, (
|
||||
assert (
|
||||
remote_len
|
||||
== (local_len * remote_heads // local_heads) // block_size_ratio
|
||||
), (
|
||||
f"SPLIT region {i}: remote P KV block_len {remote_len} "
|
||||
f"must equal local {local_len} * tp_ratio {tp_ratio} "
|
||||
f"must equal local {local_len} * remote_heads "
|
||||
f"{remote_heads} // local_heads {local_heads} "
|
||||
f"// block_size_ratio {block_size_ratio}."
|
||||
)
|
||||
else:
|
||||
@@ -1627,10 +1636,10 @@ class NixlBaseConnectorWorker:
|
||||
"Different local/remote block sizes are not supported "
|
||||
"when P TP > D TP."
|
||||
)
|
||||
assert remote_len == local_len // (-tp_ratio), (
|
||||
f"SPLIT region {i}: remote P KV block_len "
|
||||
f"{remote_len} must equal local {local_len} "
|
||||
f"// |tp_ratio| {-tp_ratio}."
|
||||
assert remote_len == local_len * remote_heads // local_heads, (
|
||||
f"SPLIT region {i}: remote P KV block_len {remote_len} "
|
||||
f"must equal local {local_len} * remote_heads "
|
||||
f"{remote_heads} // local_heads {local_heads}."
|
||||
)
|
||||
|
||||
# TP workers that handhshake with same remote have same #blocks.
|
||||
|
||||
Reference in New Issue
Block a user