[PD][Bugfix] Fix NIXL hybrid MLA+mamba heterogeneous TP (#49297)
Signed-off-by: ZeldaHuang <zelda.huanghuang@gmail.com>
This commit is contained in:
@@ -1598,12 +1598,14 @@ class NixlBaseConnectorWorker:
|
||||
### (Optional) Register local agent memory regions. MLA is not split.
|
||||
if (
|
||||
tp_ratio < 0
|
||||
and not self.use_mla
|
||||
and (not self.use_mla or len(plan.all_source_ranks) > 1)
|
||||
and tp_ratio not in self.src_xfer_handles_by_tp_ratio
|
||||
):
|
||||
# Remote tp_size > local tp_size: read from multiple remote ranks.
|
||||
# Logically "split" own regions into |tp_ratio| chunks. Mind that
|
||||
# we only do this once per remote tp_size (replica-friendly).
|
||||
# Logically "split" own regions into per-source chunks. Hybrid
|
||||
# MLA+SSM also needs this path: MLA is replicated and read once,
|
||||
# while the SSM state is sharded across every remote TP rank.
|
||||
# We only do this once per remote tp_size (replica-friendly).
|
||||
self.src_xfer_handles_by_tp_ratio[tp_ratio] = []
|
||||
|
||||
for handle_data in self._build_local_splits_from_plan(
|
||||
|
||||
@@ -161,9 +161,9 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
]
|
||||
|
||||
# D may have to perform multiple reads from different remote ranks.
|
||||
# MLA opt: when P TP > D TP, only a single read is executed for
|
||||
# the first remote rank (cache is duplicated)..
|
||||
if self.use_mla and tp_ratio < 0:
|
||||
# Pure MLA reads once because its cache is replicated. Hybrid
|
||||
# MLA+SSM still needs one read per SSM source rank.
|
||||
if self.use_mla and tp_ratio < 0 and not self._has_mamba:
|
||||
assert len(read_specs) == 1
|
||||
|
||||
for i, spec in enumerate(read_specs):
|
||||
@@ -177,7 +177,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
req_id,
|
||||
)
|
||||
# Get side handles.
|
||||
if tp_ratio < 0 and not self.use_mla:
|
||||
if tp_ratio < 0 and (not self.use_mla or len(read_specs) > 1):
|
||||
assert remote_block_size == self.block_size
|
||||
# Remote tp_size > local tp_size: we must perform multiple
|
||||
# reads. Get the memory chunk onto which we will write to.
|
||||
@@ -203,7 +203,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
remote_xfer_side_handle=remote_xfer_side_handle,
|
||||
)
|
||||
|
||||
if self.use_mla and tp_ratio < 0 and read_specs:
|
||||
if self.use_mla and tp_ratio < 0 and len(read_specs) == 1:
|
||||
# ..but we still need to notify the other remote ranks that we
|
||||
# have the blocks we need so they can update the request state.
|
||||
notif_id = f"{meta.remote.request_id}:{self.world_size}".encode()
|
||||
|
||||
Reference in New Issue
Block a user