[ROCm] Fix misrouting race-condition in multi-decode P/D disagg with mori-io (#51681)

Signed-off-by: Vincent Cave <vincent.cave@amd.com>
Signed-off-by: Shiksha Patel <shikpate@amd.com>
Co-authored-by: Shiksha Patel <shikpate@amd.com>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Vincent Cavé
2026-09-24 19:52:43 +00:00
committed by GitHub
co-authored by Shiksha Patel Douglas Lehr Andreas Karatzas
parent 90a9515006
commit 92e9c6bf13
6 changed files with 343 additions and 49 deletions
@@ -69,6 +69,8 @@ class WriteTask:
event: torch.cuda.Event
remote_notify_port: int
remote_ip: str
multi_pod_hosts: list[str] = field(default_factory=list)
remote_dp_size_local: int = 0
enqueue_time: float = field(default_factory=time.perf_counter)
retried: int = 0
@@ -96,7 +98,7 @@ class RemoteAllocInfo:
writes_expected: int | None = None
decode_dp_rank: int = 0
completion_request_id: str | None = None
completion_remote_notify_port: int | None = None
completion_notify_port: int | None = None
completion_remote_ip: str | None = None
completion_notified: bool = False
transfer_statuses: list[Any] = field(default_factory=list)
@@ -1776,6 +1776,8 @@ class MoRIIOConnectorWorker:
kv_layer: torch.Tensor,
remote_notify_port: int,
remote_ip: str,
multi_pod_hosts: list[str],
remote_dp_size_local: int,
) -> None:
"""Schedule a block write operation.
@@ -1789,6 +1791,8 @@ class MoRIIOConnectorWorker:
kv_layer: KV cache tensor
remote_notify_port: Port for completion notification
remote_ip: IP address of remote node
multi_pod_hosts: List of pod IPs for multi-pod Wide-EP
remote_dp_size_local: Per-pod DP size for multi-pod
"""
# synchronization to prevent dirty reads between
@@ -1810,6 +1814,8 @@ class MoRIIOConnectorWorker:
event=event,
remote_notify_port=remote_notify_port,
remote_ip=remote_ip,
multi_pod_hosts=multi_pod_hosts,
remote_dp_size_local=remote_dp_size_local,
)
self._writer.schedule_write(task)
@@ -3167,18 +3173,17 @@ class MoRIIOConnectorWorker:
)
def _write_blocks_for_req(self, req_id: ReqId, meta: ReqMeta, layer_name, kv_layer):
# Stash multi_pod_hosts + local DP size on the worker so
# MoRIIOEngine._finalize_if_complete (which sees only the WriteTask,
# not ReqMeta) can pick the per-rank pod IP for the completion notify.
# Last-writer-wins is safe: all requests share the same topology.
if meta.multi_pod_hosts:
self.multi_pod_hosts = list(meta.multi_pod_hosts)
else:
self.multi_pod_hosts = [meta.remote_host]
if meta.remote_dp_size_local:
self.remote_dp_size_local = int(meta.remote_dp_size_local)
else:
self.remote_dp_size_local = int(meta.remote_dp_size)
# Compute per-request values to pass through the task (no shared state).
# This avoids race conditions when concurrent requests target different
# decode pods - each WriteTask carries its own routing info.
hosts = (
list(meta.multi_pod_hosts) if meta.multi_pod_hosts else [meta.remote_host]
)
dp_local = (
int(meta.remote_dp_size_local)
if meta.remote_dp_size_local
else int(meta.remote_dp_size)
)
# WRITE does not support HMA, so unwrap blocks into flat lists.
# remote_block_ids can itself be empty so need to be careful when unwrapping.
local_block_ids = meta.local_block_ids[0]
@@ -3193,6 +3198,8 @@ class MoRIIOConnectorWorker:
kv_layer=kv_layer,
remote_notify_port=meta.remote_notify_port,
remote_ip=meta.remote_host,
multi_pod_hosts=hosts,
remote_dp_size_local=dp_local,
)
def merge_contiguous_blocks(
@@ -314,18 +314,27 @@ class MoRIIOWriter:
request_info = self._get_remote_alloc_info(task.transfer_id)
with self._write_state_lock:
request_info.completion_request_id = task.request_id
request_info.completion_remote_notify_port = task.remote_notify_port
# Wide-EP multi-pod: task.remote_ip addresses only the first pod,
# so resolve the per-rank host from multi_pod_hosts (falls back to
# task.remote_ip for single-pod or on any indexing miss).
_remote_ip = task.remote_ip
_hosts = list(getattr(self.worker, "multi_pod_hosts", []) or [])
_dp_local = int(getattr(self.worker, "remote_dp_size_local", 0) or 0)
_hosts = task.multi_pod_hosts
_dp_local = task.remote_dp_size_local
if _hosts and _dp_local > 0:
_pod_idx = int(request_info.decode_dp_rank) // _dp_local
if 0 <= _pod_idx < len(_hosts):
_remote_ip = _hosts[_pod_idx]
request_info.completion_remote_ip = _remote_ip
# Resolve the final notify port under the lock so finalize only
# reads it. The offset uses the per-pod local rank (% dp_local),
# since each pod binds notify sockets only for its local ranks;
# single-pod is bit-identical (modulus is a no-op).
_decode_dp_rank_for_port = int(request_info.decode_dp_rank)
if _dp_local > 0:
_decode_dp_rank_for_port = _decode_dp_rank_for_port % _dp_local
request_info.completion_notify_port = task.remote_notify_port + (
get_port_offset(_decode_dp_rank_for_port, self.worker.tp_rank)
)
if task.transfer_id in self._sealed_writes:
request_info.writes_expected = self._sealed_writes[task.transfer_id]
@@ -461,9 +470,9 @@ class MoRIIOWriter:
if request_info.completion_notified:
return
request_id = request_info.completion_request_id
remote_notify_port = request_info.completion_remote_notify_port
remote_port = request_info.completion_notify_port
remote_ip = request_info.completion_remote_ip
if request_id is None or remote_notify_port is None or remote_ip is None:
if request_id is None or remote_port is None or remote_ip is None:
return
transfer_statuses = list(request_info.transfer_statuses)
request_info.transfer_statuses.clear()
@@ -472,16 +481,6 @@ class MoRIIOWriter:
# Wait for this request's transfers to complete.
self.worker.moriio_wrapper.waiting_for_transfer_complete(transfer_statuses)
# The notify port offset must use the per-pod local rank
# (% dp_local), since each pod binds notify sockets only for its local
# ranks. Single-pod is bit-identical (modulus is a no-op).
_dp_local = int(getattr(self.worker, "remote_dp_size_local", 0) or 0)
_decode_dp_rank_for_port = int(request_info.decode_dp_rank)
if _dp_local > 0:
_decode_dp_rank_for_port = _decode_dp_rank_for_port % _dp_local
remote_port = remote_notify_port + get_port_offset(
_decode_dp_rank_for_port, self.worker.tp_rank
)
# Consider using RDMA immediate data in decode side
# to eliminate the need for this notification.
# Consider including the first gen token from prefill in the notification