[ROCm] Fix misrouting race-condition in multi-decode P/D disagg with mori-io (#51681)
Signed-off-by: Vincent Cave <vincent.cave@amd.com> Signed-off-by: Shiksha Patel <shikpate@amd.com> Co-authored-by: Shiksha Patel <shikpate@amd.com> Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Shiksha Patel
Douglas Lehr
Andreas Karatzas
parent
90a9515006
commit
92e9c6bf13
@@ -4,8 +4,7 @@
|
||||
import importlib.util
|
||||
import math
|
||||
import threading
|
||||
from collections import OrderedDict, defaultdict
|
||||
from queue import Queue
|
||||
from collections import OrderedDict
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
@@ -19,6 +18,8 @@ from vllm.v1.kv_cache_interface import (
|
||||
MLAAttentionSpec,
|
||||
)
|
||||
|
||||
from .utils import make_moriio_writer
|
||||
|
||||
aiter_available = importlib.util.find_spec("aiter") is not None
|
||||
mori_available = importlib.util.find_spec("mori") is not None
|
||||
|
||||
@@ -90,18 +91,6 @@ def _remote_meta(num_blocks: int = 16) -> SimpleNamespace:
|
||||
return SimpleNamespace(num_blocks=num_blocks)
|
||||
|
||||
|
||||
def _writer_with_fake_worker(fake_worker: Any) -> Any:
|
||||
writer = MoRIIOWriter.__new__(MoRIIOWriter)
|
||||
writer._worker_ref = lambda: fake_worker
|
||||
writer._write_task_q = Queue()
|
||||
writer._write_state_lock = threading.Lock()
|
||||
writer._scheduled_writes = defaultdict(int)
|
||||
writer._scheduled_layers = defaultdict(set)
|
||||
writer._sealed_writes = {}
|
||||
writer.ensure_worker_started = lambda: None
|
||||
return writer
|
||||
|
||||
|
||||
def _wrapper_for_messages() -> Any:
|
||||
wrapper = MoRIIOWrapper.__new__(MoRIIOWrapper)
|
||||
wrapper.lock = threading.Lock()
|
||||
@@ -409,7 +398,7 @@ def test_write_scheduler_deduplicates_layers_and_seals_expected_count():
|
||||
request_info = RemoteAllocInfo(block_ids=[4, 5])
|
||||
wrapper = _wrapper_for_messages()
|
||||
wrapper.done_remote_allocate_req_dict["xfer"] = request_info
|
||||
writer = _writer_with_fake_worker(SimpleNamespace(moriio_wrapper=wrapper))
|
||||
writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper))
|
||||
|
||||
assert writer.schedule_write(_write_task("dense0"))
|
||||
assert not writer.schedule_write(_write_task("dense0"))
|
||||
@@ -459,12 +448,12 @@ def test_write_completion_notifies_once_after_all_sealed_writes_finish():
|
||||
request_info = RemoteAllocInfo(block_ids=[4, 5], writes_expected=2)
|
||||
request_info.transfer_statuses.extend(["status-a", "status-b"])
|
||||
request_info.completion_request_id = "req"
|
||||
request_info.completion_remote_notify_port = 7000
|
||||
# Final notify port is resolved in _execute_write_task; finalize reads it
|
||||
# as-is, so set the already-offset value (base 7000 + tp_rank-2 offset).
|
||||
request_info.completion_notify_port = 7002
|
||||
request_info.completion_remote_ip = "127.0.0.1"
|
||||
wrapper.done_remote_allocate_req_dict["xfer"] = request_info
|
||||
writer = _writer_with_fake_worker(
|
||||
SimpleNamespace(moriio_wrapper=wrapper, tp_rank=2)
|
||||
)
|
||||
writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper, tp_rank=2))
|
||||
writer._scheduled_writes["xfer"] = 2
|
||||
writer._scheduled_layers["xfer"] = {"dense0", "indexer"}
|
||||
writer._sealed_writes["xfer"] = 2
|
||||
@@ -512,7 +501,7 @@ def test_moriio_wrapper_waits_scoped_statuses_without_global_drain():
|
||||
def test_write_failure_marks_terminal_and_clears_scheduled_state():
|
||||
wrapper = _wrapper_for_messages()
|
||||
wrapper.done_remote_allocate_req_dict["xfer"] = RemoteAllocInfo(block_ids=[4, 5])
|
||||
writer = _writer_with_fake_worker(SimpleNamespace(moriio_wrapper=wrapper))
|
||||
writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper))
|
||||
writer._scheduled_writes["xfer"] = 2
|
||||
writer._scheduled_layers["xfer"] = {"dense0", "indexer"}
|
||||
writer._sealed_writes["xfer"] = 2
|
||||
@@ -530,7 +519,7 @@ def test_write_failure_marks_terminal_and_clears_scheduled_state():
|
||||
def test_schedule_write_rejects_terminal_transfer_without_recreating_state():
|
||||
wrapper = _wrapper_for_messages()
|
||||
wrapper.done_remote_allocate_req_dict["xfer"] = RemoteAllocInfo(block_ids=[4, 5])
|
||||
writer = _writer_with_fake_worker(SimpleNamespace(moriio_wrapper=wrapper))
|
||||
writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper))
|
||||
writer._scheduled_writes["xfer"] = 1
|
||||
writer._scheduled_layers["xfer"] = {"dense0"}
|
||||
writer._sealed_writes["xfer"] = 1
|
||||
|
||||
@@ -0,0 +1,271 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for mori-io write_done routing (#51681)."""
|
||||
|
||||
import threading
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import (
|
||||
RemoteAllocInfo,
|
||||
ReqMeta,
|
||||
WriteTask,
|
||||
get_port_offset,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import (
|
||||
MoRIIOConnectorWorker,
|
||||
)
|
||||
|
||||
from .utils import make_moriio_writer
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
def _fake_worker(
|
||||
alloc: dict[str, Any],
|
||||
*,
|
||||
tp_rank: int = 0,
|
||||
set_remote_dp_size_local: int | None = None,
|
||||
) -> Any:
|
||||
wrapper = SimpleNamespace(
|
||||
done_remote_allocate_req_dict=alloc,
|
||||
lock=threading.Lock(),
|
||||
done_req_ids=[],
|
||||
sent_notifies=[],
|
||||
)
|
||||
wrapper.send_notify = lambda transfer_id, remote_ip, remote_port, message_type: (
|
||||
wrapper.sent_notifies.append(
|
||||
(transfer_id, remote_ip, remote_port, message_type)
|
||||
)
|
||||
)
|
||||
wrapper.waiting_for_transfer_complete = lambda statuses: None
|
||||
wrapper._mark_transfer_terminal_locked = lambda transfer_id: None
|
||||
wrapper._is_transfer_terminal_locked = lambda transfer_id: False
|
||||
worker = SimpleNamespace(moriio_wrapper=wrapper, tp_rank=tp_rank)
|
||||
if set_remote_dp_size_local is not None:
|
||||
worker.remote_dp_size_local = set_remote_dp_size_local
|
||||
return worker
|
||||
|
||||
|
||||
def _req_meta(
|
||||
*,
|
||||
transfer_id: str = "tA",
|
||||
remote_host: str = "10.0.0.1",
|
||||
multi_pod_hosts: list[str] | None = None,
|
||||
remote_dp_size: int = 16,
|
||||
remote_dp_size_local: int = 8,
|
||||
) -> ReqMeta:
|
||||
return ReqMeta(
|
||||
transfer_id=transfer_id,
|
||||
local_block_ids=[[1]],
|
||||
remote_block_ids=[[2]],
|
||||
remote_host=remote_host,
|
||||
remote_port=1,
|
||||
remote_handshake_port=2,
|
||||
remote_notify_port=20000,
|
||||
remote_engine_id="remote-engine",
|
||||
tp_size=1,
|
||||
remote_dp_size=remote_dp_size,
|
||||
multi_pod_hosts=list(multi_pod_hosts or []),
|
||||
remote_dp_size_local=remote_dp_size_local,
|
||||
)
|
||||
|
||||
|
||||
def _write_task(
|
||||
transfer_id: str,
|
||||
remote_ip: str,
|
||||
*,
|
||||
multi_pod_hosts: list[str] | None = None,
|
||||
remote_dp_size_local: int = 0,
|
||||
request_id: str = "req",
|
||||
) -> WriteTask:
|
||||
return WriteTask(
|
||||
request_id=request_id,
|
||||
transfer_id=transfer_id,
|
||||
dst_engine_id="remote-engine",
|
||||
local_block_ids=[1, 3],
|
||||
remote_block_ids_hint=None,
|
||||
layer_name="layer0",
|
||||
event=None,
|
||||
remote_notify_port=20000,
|
||||
remote_ip=remote_ip,
|
||||
multi_pod_hosts=multi_pod_hosts or [],
|
||||
remote_dp_size_local=remote_dp_size_local,
|
||||
)
|
||||
|
||||
|
||||
def test_execute_write_task_ip_is_per_task_under_overwrite():
|
||||
alloc = {"tA": RemoteAllocInfo(block_ids=None, decode_dp_rank=0)}
|
||||
worker = _fake_worker(alloc)
|
||||
worker.multi_pod_hosts = ["10.0.0.2"]
|
||||
worker.remote_dp_size_local = 1
|
||||
|
||||
writer = make_moriio_writer(worker)
|
||||
task_a = _write_task(
|
||||
"tA", "10.0.0.1", multi_pod_hosts=["10.0.0.1"], remote_dp_size_local=1
|
||||
)
|
||||
writer._execute_write_task(task_a)
|
||||
|
||||
assert alloc["tA"].completion_remote_ip == "10.0.0.1"
|
||||
assert alloc["tA"].completion_remote_ip != "10.0.0.2"
|
||||
|
||||
|
||||
def test_deferred_task_ip_is_per_task_under_overwrite():
|
||||
alloc = {"tA": RemoteAllocInfo(block_ids=None, decode_dp_rank=0)}
|
||||
worker = _fake_worker(alloc)
|
||||
worker.multi_pod_hosts = ["10.0.0.2"]
|
||||
worker.remote_dp_size_local = 1
|
||||
|
||||
writer = make_moriio_writer(worker)
|
||||
task_a = _write_task(
|
||||
"tA", "10.0.0.1", multi_pod_hosts=["10.0.0.1"], remote_dp_size_local=1
|
||||
)
|
||||
writer._deferred_tasks.append(task_a)
|
||||
writer._process_deferred_tasks()
|
||||
|
||||
assert alloc["tA"].completion_remote_ip == "10.0.0.1"
|
||||
assert writer._deferred_tasks == []
|
||||
|
||||
|
||||
def test_two_requests_resolve_distinct_hosts():
|
||||
alloc = {
|
||||
"tA": RemoteAllocInfo(block_ids=None, decode_dp_rank=0),
|
||||
"tB": RemoteAllocInfo(block_ids=None, decode_dp_rank=3),
|
||||
}
|
||||
worker = _fake_worker(alloc)
|
||||
writer = make_moriio_writer(worker)
|
||||
|
||||
task_a = _write_task(
|
||||
"tA", "10.0.0.1", multi_pod_hosts=["10.0.0.1"], remote_dp_size_local=1
|
||||
)
|
||||
task_b = _write_task(
|
||||
"tB", "10.0.0.2", multi_pod_hosts=["10.0.0.2"], remote_dp_size_local=1
|
||||
)
|
||||
worker.multi_pod_hosts = ["10.0.0.2"]
|
||||
writer._execute_write_task(task_a)
|
||||
writer._execute_write_task(task_b)
|
||||
|
||||
assert alloc["tA"].completion_remote_ip == "10.0.0.1"
|
||||
assert alloc["tB"].completion_remote_ip == "10.0.0.2"
|
||||
|
||||
|
||||
def test_finalize_notify_port_uses_pod_local_rank():
|
||||
base = 20000
|
||||
info = RemoteAllocInfo(block_ids=None, decode_dp_rank=9)
|
||||
alloc = {"tA": info}
|
||||
worker = _fake_worker(alloc, tp_rank=0)
|
||||
writer = make_moriio_writer(worker)
|
||||
|
||||
task = _write_task(
|
||||
"tA",
|
||||
"10.0.0.1",
|
||||
multi_pod_hosts=["10.0.0.1", "10.0.0.2"],
|
||||
remote_dp_size_local=8,
|
||||
)
|
||||
writer._execute_write_task(task)
|
||||
assert info.completion_remote_ip == "10.0.0.2"
|
||||
|
||||
info.writes_expected = 1
|
||||
info.writes_done = 1
|
||||
writer._finalize_if_complete("tA", info)
|
||||
|
||||
assert worker.moriio_wrapper.sent_notifies
|
||||
_tid, ip, port, mtype = worker.moriio_wrapper.sent_notifies[-1]
|
||||
assert mtype == "write_done"
|
||||
assert ip == "10.0.0.2"
|
||||
assert port == base + get_port_offset(9 % 8, 0)
|
||||
|
||||
|
||||
def test_finalize_port_ignores_stale_worker_dp_local():
|
||||
base = 20000
|
||||
info = RemoteAllocInfo(block_ids=None, decode_dp_rank=9)
|
||||
alloc = {"tA": info}
|
||||
worker = _fake_worker(alloc, tp_rank=0, set_remote_dp_size_local=1)
|
||||
writer = make_moriio_writer(worker)
|
||||
|
||||
task = _write_task(
|
||||
"tA",
|
||||
"10.0.0.1",
|
||||
multi_pod_hosts=["10.0.0.1", "10.0.0.2"],
|
||||
remote_dp_size_local=8,
|
||||
)
|
||||
writer._execute_write_task(task)
|
||||
assert info.completion_notify_port == base + get_port_offset(9 % 8, 0)
|
||||
|
||||
info.writes_expected = 1
|
||||
info.writes_done = 1
|
||||
writer._finalize_if_complete("tA", info)
|
||||
|
||||
_tid, _ip, port, mtype = worker.moriio_wrapper.sent_notifies[-1]
|
||||
assert mtype == "write_done"
|
||||
assert port == base + get_port_offset(9 % 8, 0)
|
||||
assert port != base + get_port_offset(9 % 1, 0)
|
||||
|
||||
|
||||
def test_finalize_single_pod_port_uses_global_rank():
|
||||
base = 20000
|
||||
info = RemoteAllocInfo(block_ids=None, decode_dp_rank=3)
|
||||
alloc = {"tA": info}
|
||||
worker = _fake_worker(alloc, tp_rank=2)
|
||||
writer = make_moriio_writer(worker)
|
||||
|
||||
task = _write_task("tA", "10.0.0.1", multi_pod_hosts=[], remote_dp_size_local=0)
|
||||
writer._execute_write_task(task)
|
||||
assert info.completion_remote_ip == "10.0.0.1"
|
||||
assert info.completion_notify_port == base + get_port_offset(3, 2)
|
||||
|
||||
info.writes_expected = 1
|
||||
info.writes_done = 1
|
||||
writer._finalize_if_complete("tA", info)
|
||||
|
||||
_tid, _ip, port, _mtype = worker.moriio_wrapper.sent_notifies[-1]
|
||||
assert port == base + get_port_offset(3, 2)
|
||||
|
||||
|
||||
def test_write_blocks_for_req_forwards_routing_to_schedule_write():
|
||||
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
def _capture_schedule_write_blocks(**kwargs: Any) -> None:
|
||||
captured.update(kwargs)
|
||||
|
||||
worker.schedule_write_blocks = _capture_schedule_write_blocks # type: ignore[method-assign]
|
||||
|
||||
meta = _req_meta(
|
||||
remote_host="10.0.0.1",
|
||||
multi_pod_hosts=["10.0.0.1", "10.0.0.2"],
|
||||
remote_dp_size_local=8,
|
||||
)
|
||||
worker._write_blocks_for_req("req-a", meta, "layer0", kv_layer=None)
|
||||
|
||||
assert captured["multi_pod_hosts"] == ["10.0.0.1", "10.0.0.2"]
|
||||
assert captured["remote_dp_size_local"] == 8
|
||||
assert captured["remote_ip"] == "10.0.0.1"
|
||||
assert captured["local_block_ids"] == [1]
|
||||
assert captured["remote_block_ids"] == [2]
|
||||
assert not hasattr(worker, "multi_pod_hosts")
|
||||
|
||||
|
||||
def test_write_blocks_for_req_falls_back_when_multi_pod_hosts_empty():
|
||||
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
def _capture_schedule_write_blocks(**kwargs: Any) -> None:
|
||||
captured.update(kwargs)
|
||||
|
||||
worker.schedule_write_blocks = _capture_schedule_write_blocks # type: ignore[method-assign]
|
||||
|
||||
meta = _req_meta(
|
||||
remote_host="10.0.0.9",
|
||||
multi_pod_hosts=[],
|
||||
remote_dp_size=4,
|
||||
remote_dp_size_local=0,
|
||||
)
|
||||
worker._write_blocks_for_req("req-b", meta, "layer0", kv_layer=None)
|
||||
|
||||
assert captured["multi_pod_hosts"] == ["10.0.0.9"]
|
||||
assert captured["remote_dp_size_local"] == 4
|
||||
assert captured["local_block_ids"] == [1]
|
||||
assert captured["remote_block_ids"] == [2]
|
||||
@@ -632,3 +632,29 @@ def make_nixl_push_scheduler(
|
||||
sched.blocks_per_sw = []
|
||||
|
||||
return sched
|
||||
|
||||
|
||||
def make_moriio_writer(fake_worker: Any) -> Any:
|
||||
"""Build a MoRIIOWriter with internals stubbed for unit tests.
|
||||
|
||||
Bypasses ``__init__`` and wires only the write/finalize state the tests
|
||||
touch, including the deferred-task fields used by the routing suite.
|
||||
"""
|
||||
import threading
|
||||
from queue import Queue
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_engine import (
|
||||
MoRIIOWriter,
|
||||
)
|
||||
|
||||
writer = MoRIIOWriter.__new__(MoRIIOWriter)
|
||||
writer._worker_ref = lambda: fake_worker
|
||||
writer._write_task_q = Queue()
|
||||
writer._write_state_lock = threading.Lock()
|
||||
writer._scheduled_writes = defaultdict(int)
|
||||
writer._scheduled_layers = defaultdict(set)
|
||||
writer._sealed_writes = {}
|
||||
writer._deferred_tasks = []
|
||||
writer._defer_timeout = 60.0
|
||||
writer.ensure_worker_started = lambda: None
|
||||
return writer
|
||||
|
||||
@@ -69,6 +69,8 @@ class WriteTask:
|
||||
event: torch.cuda.Event
|
||||
remote_notify_port: int
|
||||
remote_ip: str
|
||||
multi_pod_hosts: list[str] = field(default_factory=list)
|
||||
remote_dp_size_local: int = 0
|
||||
enqueue_time: float = field(default_factory=time.perf_counter)
|
||||
retried: int = 0
|
||||
|
||||
@@ -96,7 +98,7 @@ class RemoteAllocInfo:
|
||||
writes_expected: int | None = None
|
||||
decode_dp_rank: int = 0
|
||||
completion_request_id: str | None = None
|
||||
completion_remote_notify_port: int | None = None
|
||||
completion_notify_port: int | None = None
|
||||
completion_remote_ip: str | None = None
|
||||
completion_notified: bool = False
|
||||
transfer_statuses: list[Any] = field(default_factory=list)
|
||||
|
||||
@@ -1776,6 +1776,8 @@ class MoRIIOConnectorWorker:
|
||||
kv_layer: torch.Tensor,
|
||||
remote_notify_port: int,
|
||||
remote_ip: str,
|
||||
multi_pod_hosts: list[str],
|
||||
remote_dp_size_local: int,
|
||||
) -> None:
|
||||
"""Schedule a block write operation.
|
||||
|
||||
@@ -1789,6 +1791,8 @@ class MoRIIOConnectorWorker:
|
||||
kv_layer: KV cache tensor
|
||||
remote_notify_port: Port for completion notification
|
||||
remote_ip: IP address of remote node
|
||||
multi_pod_hosts: List of pod IPs for multi-pod Wide-EP
|
||||
remote_dp_size_local: Per-pod DP size for multi-pod
|
||||
|
||||
"""
|
||||
# synchronization to prevent dirty reads between
|
||||
@@ -1810,6 +1814,8 @@ class MoRIIOConnectorWorker:
|
||||
event=event,
|
||||
remote_notify_port=remote_notify_port,
|
||||
remote_ip=remote_ip,
|
||||
multi_pod_hosts=multi_pod_hosts,
|
||||
remote_dp_size_local=remote_dp_size_local,
|
||||
)
|
||||
self._writer.schedule_write(task)
|
||||
|
||||
@@ -3167,18 +3173,17 @@ class MoRIIOConnectorWorker:
|
||||
)
|
||||
|
||||
def _write_blocks_for_req(self, req_id: ReqId, meta: ReqMeta, layer_name, kv_layer):
|
||||
# Stash multi_pod_hosts + local DP size on the worker so
|
||||
# MoRIIOEngine._finalize_if_complete (which sees only the WriteTask,
|
||||
# not ReqMeta) can pick the per-rank pod IP for the completion notify.
|
||||
# Last-writer-wins is safe: all requests share the same topology.
|
||||
if meta.multi_pod_hosts:
|
||||
self.multi_pod_hosts = list(meta.multi_pod_hosts)
|
||||
else:
|
||||
self.multi_pod_hosts = [meta.remote_host]
|
||||
if meta.remote_dp_size_local:
|
||||
self.remote_dp_size_local = int(meta.remote_dp_size_local)
|
||||
else:
|
||||
self.remote_dp_size_local = int(meta.remote_dp_size)
|
||||
# Compute per-request values to pass through the task (no shared state).
|
||||
# This avoids race conditions when concurrent requests target different
|
||||
# decode pods - each WriteTask carries its own routing info.
|
||||
hosts = (
|
||||
list(meta.multi_pod_hosts) if meta.multi_pod_hosts else [meta.remote_host]
|
||||
)
|
||||
dp_local = (
|
||||
int(meta.remote_dp_size_local)
|
||||
if meta.remote_dp_size_local
|
||||
else int(meta.remote_dp_size)
|
||||
)
|
||||
# WRITE does not support HMA, so unwrap blocks into flat lists.
|
||||
# remote_block_ids can itself be empty so need to be careful when unwrapping.
|
||||
local_block_ids = meta.local_block_ids[0]
|
||||
@@ -3193,6 +3198,8 @@ class MoRIIOConnectorWorker:
|
||||
kv_layer=kv_layer,
|
||||
remote_notify_port=meta.remote_notify_port,
|
||||
remote_ip=meta.remote_host,
|
||||
multi_pod_hosts=hosts,
|
||||
remote_dp_size_local=dp_local,
|
||||
)
|
||||
|
||||
def merge_contiguous_blocks(
|
||||
|
||||
@@ -314,18 +314,27 @@ class MoRIIOWriter:
|
||||
request_info = self._get_remote_alloc_info(task.transfer_id)
|
||||
with self._write_state_lock:
|
||||
request_info.completion_request_id = task.request_id
|
||||
request_info.completion_remote_notify_port = task.remote_notify_port
|
||||
# Wide-EP multi-pod: task.remote_ip addresses only the first pod,
|
||||
# so resolve the per-rank host from multi_pod_hosts (falls back to
|
||||
# task.remote_ip for single-pod or on any indexing miss).
|
||||
_remote_ip = task.remote_ip
|
||||
_hosts = list(getattr(self.worker, "multi_pod_hosts", []) or [])
|
||||
_dp_local = int(getattr(self.worker, "remote_dp_size_local", 0) or 0)
|
||||
_hosts = task.multi_pod_hosts
|
||||
_dp_local = task.remote_dp_size_local
|
||||
if _hosts and _dp_local > 0:
|
||||
_pod_idx = int(request_info.decode_dp_rank) // _dp_local
|
||||
if 0 <= _pod_idx < len(_hosts):
|
||||
_remote_ip = _hosts[_pod_idx]
|
||||
request_info.completion_remote_ip = _remote_ip
|
||||
# Resolve the final notify port under the lock so finalize only
|
||||
# reads it. The offset uses the per-pod local rank (% dp_local),
|
||||
# since each pod binds notify sockets only for its local ranks;
|
||||
# single-pod is bit-identical (modulus is a no-op).
|
||||
_decode_dp_rank_for_port = int(request_info.decode_dp_rank)
|
||||
if _dp_local > 0:
|
||||
_decode_dp_rank_for_port = _decode_dp_rank_for_port % _dp_local
|
||||
request_info.completion_notify_port = task.remote_notify_port + (
|
||||
get_port_offset(_decode_dp_rank_for_port, self.worker.tp_rank)
|
||||
)
|
||||
if task.transfer_id in self._sealed_writes:
|
||||
request_info.writes_expected = self._sealed_writes[task.transfer_id]
|
||||
|
||||
@@ -461,9 +470,9 @@ class MoRIIOWriter:
|
||||
if request_info.completion_notified:
|
||||
return
|
||||
request_id = request_info.completion_request_id
|
||||
remote_notify_port = request_info.completion_remote_notify_port
|
||||
remote_port = request_info.completion_notify_port
|
||||
remote_ip = request_info.completion_remote_ip
|
||||
if request_id is None or remote_notify_port is None or remote_ip is None:
|
||||
if request_id is None or remote_port is None or remote_ip is None:
|
||||
return
|
||||
transfer_statuses = list(request_info.transfer_statuses)
|
||||
request_info.transfer_statuses.clear()
|
||||
@@ -472,16 +481,6 @@ class MoRIIOWriter:
|
||||
# Wait for this request's transfers to complete.
|
||||
self.worker.moriio_wrapper.waiting_for_transfer_complete(transfer_statuses)
|
||||
|
||||
# The notify port offset must use the per-pod local rank
|
||||
# (% dp_local), since each pod binds notify sockets only for its local
|
||||
# ranks. Single-pod is bit-identical (modulus is a no-op).
|
||||
_dp_local = int(getattr(self.worker, "remote_dp_size_local", 0) or 0)
|
||||
_decode_dp_rank_for_port = int(request_info.decode_dp_rank)
|
||||
if _dp_local > 0:
|
||||
_decode_dp_rank_for_port = _decode_dp_rank_for_port % _dp_local
|
||||
remote_port = remote_notify_port + get_port_offset(
|
||||
_decode_dp_rank_for_port, self.worker.tp_rank
|
||||
)
|
||||
# Consider using RDMA immediate data in decode side
|
||||
# to eliminate the need for this notification.
|
||||
# Consider including the first gen token from prefill in the notification
|
||||
|
||||
Reference in New Issue
Block a user