[ROCm] Fix misrouting race-condition in multi-decode P/D disagg with mori-io (#51681)
Signed-off-by: Vincent Cave <vincent.cave@amd.com> Signed-off-by: Shiksha Patel <shikpate@amd.com> Co-authored-by: Shiksha Patel <shikpate@amd.com> Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Shiksha Patel
Douglas Lehr
Andreas Karatzas
parent
90a9515006
commit
92e9c6bf13
@@ -69,6 +69,8 @@ class WriteTask:
|
||||
event: torch.cuda.Event
|
||||
remote_notify_port: int
|
||||
remote_ip: str
|
||||
multi_pod_hosts: list[str] = field(default_factory=list)
|
||||
remote_dp_size_local: int = 0
|
||||
enqueue_time: float = field(default_factory=time.perf_counter)
|
||||
retried: int = 0
|
||||
|
||||
@@ -96,7 +98,7 @@ class RemoteAllocInfo:
|
||||
writes_expected: int | None = None
|
||||
decode_dp_rank: int = 0
|
||||
completion_request_id: str | None = None
|
||||
completion_remote_notify_port: int | None = None
|
||||
completion_notify_port: int | None = None
|
||||
completion_remote_ip: str | None = None
|
||||
completion_notified: bool = False
|
||||
transfer_statuses: list[Any] = field(default_factory=list)
|
||||
|
||||
@@ -1776,6 +1776,8 @@ class MoRIIOConnectorWorker:
|
||||
kv_layer: torch.Tensor,
|
||||
remote_notify_port: int,
|
||||
remote_ip: str,
|
||||
multi_pod_hosts: list[str],
|
||||
remote_dp_size_local: int,
|
||||
) -> None:
|
||||
"""Schedule a block write operation.
|
||||
|
||||
@@ -1789,6 +1791,8 @@ class MoRIIOConnectorWorker:
|
||||
kv_layer: KV cache tensor
|
||||
remote_notify_port: Port for completion notification
|
||||
remote_ip: IP address of remote node
|
||||
multi_pod_hosts: List of pod IPs for multi-pod Wide-EP
|
||||
remote_dp_size_local: Per-pod DP size for multi-pod
|
||||
|
||||
"""
|
||||
# synchronization to prevent dirty reads between
|
||||
@@ -1810,6 +1814,8 @@ class MoRIIOConnectorWorker:
|
||||
event=event,
|
||||
remote_notify_port=remote_notify_port,
|
||||
remote_ip=remote_ip,
|
||||
multi_pod_hosts=multi_pod_hosts,
|
||||
remote_dp_size_local=remote_dp_size_local,
|
||||
)
|
||||
self._writer.schedule_write(task)
|
||||
|
||||
@@ -3167,18 +3173,17 @@ class MoRIIOConnectorWorker:
|
||||
)
|
||||
|
||||
def _write_blocks_for_req(self, req_id: ReqId, meta: ReqMeta, layer_name, kv_layer):
|
||||
# Stash multi_pod_hosts + local DP size on the worker so
|
||||
# MoRIIOEngine._finalize_if_complete (which sees only the WriteTask,
|
||||
# not ReqMeta) can pick the per-rank pod IP for the completion notify.
|
||||
# Last-writer-wins is safe: all requests share the same topology.
|
||||
if meta.multi_pod_hosts:
|
||||
self.multi_pod_hosts = list(meta.multi_pod_hosts)
|
||||
else:
|
||||
self.multi_pod_hosts = [meta.remote_host]
|
||||
if meta.remote_dp_size_local:
|
||||
self.remote_dp_size_local = int(meta.remote_dp_size_local)
|
||||
else:
|
||||
self.remote_dp_size_local = int(meta.remote_dp_size)
|
||||
# Compute per-request values to pass through the task (no shared state).
|
||||
# This avoids race conditions when concurrent requests target different
|
||||
# decode pods - each WriteTask carries its own routing info.
|
||||
hosts = (
|
||||
list(meta.multi_pod_hosts) if meta.multi_pod_hosts else [meta.remote_host]
|
||||
)
|
||||
dp_local = (
|
||||
int(meta.remote_dp_size_local)
|
||||
if meta.remote_dp_size_local
|
||||
else int(meta.remote_dp_size)
|
||||
)
|
||||
# WRITE does not support HMA, so unwrap blocks into flat lists.
|
||||
# remote_block_ids can itself be empty so need to be careful when unwrapping.
|
||||
local_block_ids = meta.local_block_ids[0]
|
||||
@@ -3193,6 +3198,8 @@ class MoRIIOConnectorWorker:
|
||||
kv_layer=kv_layer,
|
||||
remote_notify_port=meta.remote_notify_port,
|
||||
remote_ip=meta.remote_host,
|
||||
multi_pod_hosts=hosts,
|
||||
remote_dp_size_local=dp_local,
|
||||
)
|
||||
|
||||
def merge_contiguous_blocks(
|
||||
|
||||
@@ -314,18 +314,27 @@ class MoRIIOWriter:
|
||||
request_info = self._get_remote_alloc_info(task.transfer_id)
|
||||
with self._write_state_lock:
|
||||
request_info.completion_request_id = task.request_id
|
||||
request_info.completion_remote_notify_port = task.remote_notify_port
|
||||
# Wide-EP multi-pod: task.remote_ip addresses only the first pod,
|
||||
# so resolve the per-rank host from multi_pod_hosts (falls back to
|
||||
# task.remote_ip for single-pod or on any indexing miss).
|
||||
_remote_ip = task.remote_ip
|
||||
_hosts = list(getattr(self.worker, "multi_pod_hosts", []) or [])
|
||||
_dp_local = int(getattr(self.worker, "remote_dp_size_local", 0) or 0)
|
||||
_hosts = task.multi_pod_hosts
|
||||
_dp_local = task.remote_dp_size_local
|
||||
if _hosts and _dp_local > 0:
|
||||
_pod_idx = int(request_info.decode_dp_rank) // _dp_local
|
||||
if 0 <= _pod_idx < len(_hosts):
|
||||
_remote_ip = _hosts[_pod_idx]
|
||||
request_info.completion_remote_ip = _remote_ip
|
||||
# Resolve the final notify port under the lock so finalize only
|
||||
# reads it. The offset uses the per-pod local rank (% dp_local),
|
||||
# since each pod binds notify sockets only for its local ranks;
|
||||
# single-pod is bit-identical (modulus is a no-op).
|
||||
_decode_dp_rank_for_port = int(request_info.decode_dp_rank)
|
||||
if _dp_local > 0:
|
||||
_decode_dp_rank_for_port = _decode_dp_rank_for_port % _dp_local
|
||||
request_info.completion_notify_port = task.remote_notify_port + (
|
||||
get_port_offset(_decode_dp_rank_for_port, self.worker.tp_rank)
|
||||
)
|
||||
if task.transfer_id in self._sealed_writes:
|
||||
request_info.writes_expected = self._sealed_writes[task.transfer_id]
|
||||
|
||||
@@ -461,9 +470,9 @@ class MoRIIOWriter:
|
||||
if request_info.completion_notified:
|
||||
return
|
||||
request_id = request_info.completion_request_id
|
||||
remote_notify_port = request_info.completion_remote_notify_port
|
||||
remote_port = request_info.completion_notify_port
|
||||
remote_ip = request_info.completion_remote_ip
|
||||
if request_id is None or remote_notify_port is None or remote_ip is None:
|
||||
if request_id is None or remote_port is None or remote_ip is None:
|
||||
return
|
||||
transfer_statuses = list(request_info.transfer_statuses)
|
||||
request_info.transfer_statuses.clear()
|
||||
@@ -472,16 +481,6 @@ class MoRIIOWriter:
|
||||
# Wait for this request's transfers to complete.
|
||||
self.worker.moriio_wrapper.waiting_for_transfer_complete(transfer_statuses)
|
||||
|
||||
# The notify port offset must use the per-pod local rank
|
||||
# (% dp_local), since each pod binds notify sockets only for its local
|
||||
# ranks. Single-pod is bit-identical (modulus is a no-op).
|
||||
_dp_local = int(getattr(self.worker, "remote_dp_size_local", 0) or 0)
|
||||
_decode_dp_rank_for_port = int(request_info.decode_dp_rank)
|
||||
if _dp_local > 0:
|
||||
_decode_dp_rank_for_port = _decode_dp_rank_for_port % _dp_local
|
||||
remote_port = remote_notify_port + get_port_offset(
|
||||
_decode_dp_rank_for_port, self.worker.tp_rank
|
||||
)
|
||||
# Consider using RDMA immediate data in decode side
|
||||
# to eliminate the need for this notification.
|
||||
# Consider including the first gen token from prefill in the notification
|
||||
|
||||
Reference in New Issue
Block a user