[ROCm] Fix misrouting race-condition in multi-decode P/D disagg with mori-io (#51681)

Signed-off-by: Vincent Cave <vincent.cave@amd.com>
Signed-off-by: Shiksha Patel <shikpate@amd.com>
Co-authored-by: Shiksha Patel <shikpate@amd.com>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Vincent Cavé
2026-09-24 19:52:43 +00:00
committed by GitHub
co-authored by Shiksha Patel Douglas Lehr Andreas Karatzas
parent 90a9515006
commit 92e9c6bf13
6 changed files with 343 additions and 49 deletions
@@ -4,8 +4,7 @@
import importlib.util
import math
import threading
from collections import OrderedDict, defaultdict
from queue import Queue
from collections import OrderedDict
from types import SimpleNamespace
from typing import Any
@@ -19,6 +18,8 @@ from vllm.v1.kv_cache_interface import (
MLAAttentionSpec,
)
from .utils import make_moriio_writer
aiter_available = importlib.util.find_spec("aiter") is not None
mori_available = importlib.util.find_spec("mori") is not None
@@ -90,18 +91,6 @@ def _remote_meta(num_blocks: int = 16) -> SimpleNamespace:
return SimpleNamespace(num_blocks=num_blocks)
def _writer_with_fake_worker(fake_worker: Any) -> Any:
writer = MoRIIOWriter.__new__(MoRIIOWriter)
writer._worker_ref = lambda: fake_worker
writer._write_task_q = Queue()
writer._write_state_lock = threading.Lock()
writer._scheduled_writes = defaultdict(int)
writer._scheduled_layers = defaultdict(set)
writer._sealed_writes = {}
writer.ensure_worker_started = lambda: None
return writer
def _wrapper_for_messages() -> Any:
wrapper = MoRIIOWrapper.__new__(MoRIIOWrapper)
wrapper.lock = threading.Lock()
@@ -409,7 +398,7 @@ def test_write_scheduler_deduplicates_layers_and_seals_expected_count():
request_info = RemoteAllocInfo(block_ids=[4, 5])
wrapper = _wrapper_for_messages()
wrapper.done_remote_allocate_req_dict["xfer"] = request_info
writer = _writer_with_fake_worker(SimpleNamespace(moriio_wrapper=wrapper))
writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper))
assert writer.schedule_write(_write_task("dense0"))
assert not writer.schedule_write(_write_task("dense0"))
@@ -459,12 +448,12 @@ def test_write_completion_notifies_once_after_all_sealed_writes_finish():
request_info = RemoteAllocInfo(block_ids=[4, 5], writes_expected=2)
request_info.transfer_statuses.extend(["status-a", "status-b"])
request_info.completion_request_id = "req"
request_info.completion_remote_notify_port = 7000
# Final notify port is resolved in _execute_write_task; finalize reads it
# as-is, so set the already-offset value (base 7000 + tp_rank-2 offset).
request_info.completion_notify_port = 7002
request_info.completion_remote_ip = "127.0.0.1"
wrapper.done_remote_allocate_req_dict["xfer"] = request_info
writer = _writer_with_fake_worker(
SimpleNamespace(moriio_wrapper=wrapper, tp_rank=2)
)
writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper, tp_rank=2))
writer._scheduled_writes["xfer"] = 2
writer._scheduled_layers["xfer"] = {"dense0", "indexer"}
writer._sealed_writes["xfer"] = 2
@@ -512,7 +501,7 @@ def test_moriio_wrapper_waits_scoped_statuses_without_global_drain():
def test_write_failure_marks_terminal_and_clears_scheduled_state():
wrapper = _wrapper_for_messages()
wrapper.done_remote_allocate_req_dict["xfer"] = RemoteAllocInfo(block_ids=[4, 5])
writer = _writer_with_fake_worker(SimpleNamespace(moriio_wrapper=wrapper))
writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper))
writer._scheduled_writes["xfer"] = 2
writer._scheduled_layers["xfer"] = {"dense0", "indexer"}
writer._sealed_writes["xfer"] = 2
@@ -530,7 +519,7 @@ def test_write_failure_marks_terminal_and_clears_scheduled_state():
def test_schedule_write_rejects_terminal_transfer_without_recreating_state():
wrapper = _wrapper_for_messages()
wrapper.done_remote_allocate_req_dict["xfer"] = RemoteAllocInfo(block_ids=[4, 5])
writer = _writer_with_fake_worker(SimpleNamespace(moriio_wrapper=wrapper))
writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper))
writer._scheduled_writes["xfer"] = 1
writer._scheduled_layers["xfer"] = {"dense0"}
writer._sealed_writes["xfer"] = 1
@@ -0,0 +1,271 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for mori-io write_done routing (#51681)."""
import threading
from types import SimpleNamespace
from typing import Any
import pytest
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import (
RemoteAllocInfo,
ReqMeta,
WriteTask,
get_port_offset,
)
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import (
MoRIIOConnectorWorker,
)
from .utils import make_moriio_writer
pytestmark = pytest.mark.cpu_test
def _fake_worker(
alloc: dict[str, Any],
*,
tp_rank: int = 0,
set_remote_dp_size_local: int | None = None,
) -> Any:
wrapper = SimpleNamespace(
done_remote_allocate_req_dict=alloc,
lock=threading.Lock(),
done_req_ids=[],
sent_notifies=[],
)
wrapper.send_notify = lambda transfer_id, remote_ip, remote_port, message_type: (
wrapper.sent_notifies.append(
(transfer_id, remote_ip, remote_port, message_type)
)
)
wrapper.waiting_for_transfer_complete = lambda statuses: None
wrapper._mark_transfer_terminal_locked = lambda transfer_id: None
wrapper._is_transfer_terminal_locked = lambda transfer_id: False
worker = SimpleNamespace(moriio_wrapper=wrapper, tp_rank=tp_rank)
if set_remote_dp_size_local is not None:
worker.remote_dp_size_local = set_remote_dp_size_local
return worker
def _req_meta(
*,
transfer_id: str = "tA",
remote_host: str = "10.0.0.1",
multi_pod_hosts: list[str] | None = None,
remote_dp_size: int = 16,
remote_dp_size_local: int = 8,
) -> ReqMeta:
return ReqMeta(
transfer_id=transfer_id,
local_block_ids=[[1]],
remote_block_ids=[[2]],
remote_host=remote_host,
remote_port=1,
remote_handshake_port=2,
remote_notify_port=20000,
remote_engine_id="remote-engine",
tp_size=1,
remote_dp_size=remote_dp_size,
multi_pod_hosts=list(multi_pod_hosts or []),
remote_dp_size_local=remote_dp_size_local,
)
def _write_task(
transfer_id: str,
remote_ip: str,
*,
multi_pod_hosts: list[str] | None = None,
remote_dp_size_local: int = 0,
request_id: str = "req",
) -> WriteTask:
return WriteTask(
request_id=request_id,
transfer_id=transfer_id,
dst_engine_id="remote-engine",
local_block_ids=[1, 3],
remote_block_ids_hint=None,
layer_name="layer0",
event=None,
remote_notify_port=20000,
remote_ip=remote_ip,
multi_pod_hosts=multi_pod_hosts or [],
remote_dp_size_local=remote_dp_size_local,
)
def test_execute_write_task_ip_is_per_task_under_overwrite():
alloc = {"tA": RemoteAllocInfo(block_ids=None, decode_dp_rank=0)}
worker = _fake_worker(alloc)
worker.multi_pod_hosts = ["10.0.0.2"]
worker.remote_dp_size_local = 1
writer = make_moriio_writer(worker)
task_a = _write_task(
"tA", "10.0.0.1", multi_pod_hosts=["10.0.0.1"], remote_dp_size_local=1
)
writer._execute_write_task(task_a)
assert alloc["tA"].completion_remote_ip == "10.0.0.1"
assert alloc["tA"].completion_remote_ip != "10.0.0.2"
def test_deferred_task_ip_is_per_task_under_overwrite():
alloc = {"tA": RemoteAllocInfo(block_ids=None, decode_dp_rank=0)}
worker = _fake_worker(alloc)
worker.multi_pod_hosts = ["10.0.0.2"]
worker.remote_dp_size_local = 1
writer = make_moriio_writer(worker)
task_a = _write_task(
"tA", "10.0.0.1", multi_pod_hosts=["10.0.0.1"], remote_dp_size_local=1
)
writer._deferred_tasks.append(task_a)
writer._process_deferred_tasks()
assert alloc["tA"].completion_remote_ip == "10.0.0.1"
assert writer._deferred_tasks == []
def test_two_requests_resolve_distinct_hosts():
alloc = {
"tA": RemoteAllocInfo(block_ids=None, decode_dp_rank=0),
"tB": RemoteAllocInfo(block_ids=None, decode_dp_rank=3),
}
worker = _fake_worker(alloc)
writer = make_moriio_writer(worker)
task_a = _write_task(
"tA", "10.0.0.1", multi_pod_hosts=["10.0.0.1"], remote_dp_size_local=1
)
task_b = _write_task(
"tB", "10.0.0.2", multi_pod_hosts=["10.0.0.2"], remote_dp_size_local=1
)
worker.multi_pod_hosts = ["10.0.0.2"]
writer._execute_write_task(task_a)
writer._execute_write_task(task_b)
assert alloc["tA"].completion_remote_ip == "10.0.0.1"
assert alloc["tB"].completion_remote_ip == "10.0.0.2"
def test_finalize_notify_port_uses_pod_local_rank():
base = 20000
info = RemoteAllocInfo(block_ids=None, decode_dp_rank=9)
alloc = {"tA": info}
worker = _fake_worker(alloc, tp_rank=0)
writer = make_moriio_writer(worker)
task = _write_task(
"tA",
"10.0.0.1",
multi_pod_hosts=["10.0.0.1", "10.0.0.2"],
remote_dp_size_local=8,
)
writer._execute_write_task(task)
assert info.completion_remote_ip == "10.0.0.2"
info.writes_expected = 1
info.writes_done = 1
writer._finalize_if_complete("tA", info)
assert worker.moriio_wrapper.sent_notifies
_tid, ip, port, mtype = worker.moriio_wrapper.sent_notifies[-1]
assert mtype == "write_done"
assert ip == "10.0.0.2"
assert port == base + get_port_offset(9 % 8, 0)
def test_finalize_port_ignores_stale_worker_dp_local():
base = 20000
info = RemoteAllocInfo(block_ids=None, decode_dp_rank=9)
alloc = {"tA": info}
worker = _fake_worker(alloc, tp_rank=0, set_remote_dp_size_local=1)
writer = make_moriio_writer(worker)
task = _write_task(
"tA",
"10.0.0.1",
multi_pod_hosts=["10.0.0.1", "10.0.0.2"],
remote_dp_size_local=8,
)
writer._execute_write_task(task)
assert info.completion_notify_port == base + get_port_offset(9 % 8, 0)
info.writes_expected = 1
info.writes_done = 1
writer._finalize_if_complete("tA", info)
_tid, _ip, port, mtype = worker.moriio_wrapper.sent_notifies[-1]
assert mtype == "write_done"
assert port == base + get_port_offset(9 % 8, 0)
assert port != base + get_port_offset(9 % 1, 0)
def test_finalize_single_pod_port_uses_global_rank():
base = 20000
info = RemoteAllocInfo(block_ids=None, decode_dp_rank=3)
alloc = {"tA": info}
worker = _fake_worker(alloc, tp_rank=2)
writer = make_moriio_writer(worker)
task = _write_task("tA", "10.0.0.1", multi_pod_hosts=[], remote_dp_size_local=0)
writer._execute_write_task(task)
assert info.completion_remote_ip == "10.0.0.1"
assert info.completion_notify_port == base + get_port_offset(3, 2)
info.writes_expected = 1
info.writes_done = 1
writer._finalize_if_complete("tA", info)
_tid, _ip, port, _mtype = worker.moriio_wrapper.sent_notifies[-1]
assert port == base + get_port_offset(3, 2)
def test_write_blocks_for_req_forwards_routing_to_schedule_write():
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
captured: dict[str, Any] = {}
def _capture_schedule_write_blocks(**kwargs: Any) -> None:
captured.update(kwargs)
worker.schedule_write_blocks = _capture_schedule_write_blocks # type: ignore[method-assign]
meta = _req_meta(
remote_host="10.0.0.1",
multi_pod_hosts=["10.0.0.1", "10.0.0.2"],
remote_dp_size_local=8,
)
worker._write_blocks_for_req("req-a", meta, "layer0", kv_layer=None)
assert captured["multi_pod_hosts"] == ["10.0.0.1", "10.0.0.2"]
assert captured["remote_dp_size_local"] == 8
assert captured["remote_ip"] == "10.0.0.1"
assert captured["local_block_ids"] == [1]
assert captured["remote_block_ids"] == [2]
assert not hasattr(worker, "multi_pod_hosts")
def test_write_blocks_for_req_falls_back_when_multi_pod_hosts_empty():
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
captured: dict[str, Any] = {}
def _capture_schedule_write_blocks(**kwargs: Any) -> None:
captured.update(kwargs)
worker.schedule_write_blocks = _capture_schedule_write_blocks # type: ignore[method-assign]
meta = _req_meta(
remote_host="10.0.0.9",
multi_pod_hosts=[],
remote_dp_size=4,
remote_dp_size_local=0,
)
worker._write_blocks_for_req("req-b", meta, "layer0", kv_layer=None)
assert captured["multi_pod_hosts"] == ["10.0.0.9"]
assert captured["remote_dp_size_local"] == 4
assert captured["local_block_ids"] == [1]
assert captured["remote_block_ids"] == [2]
+26
View File
@@ -632,3 +632,29 @@ def make_nixl_push_scheduler(
sched.blocks_per_sw = []
return sched
def make_moriio_writer(fake_worker: Any) -> Any:
"""Build a MoRIIOWriter with internals stubbed for unit tests.
Bypasses ``__init__`` and wires only the write/finalize state the tests
touch, including the deferred-task fields used by the routing suite.
"""
import threading
from queue import Queue
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_engine import (
MoRIIOWriter,
)
writer = MoRIIOWriter.__new__(MoRIIOWriter)
writer._worker_ref = lambda: fake_worker
writer._write_task_q = Queue()
writer._write_state_lock = threading.Lock()
writer._scheduled_writes = defaultdict(int)
writer._scheduled_layers = defaultdict(set)
writer._sealed_writes = {}
writer._deferred_tasks = []
writer._defer_timeout = 60.0
writer.ensure_worker_started = lambda: None
return writer
@@ -69,6 +69,8 @@ class WriteTask:
event: torch.cuda.Event
remote_notify_port: int
remote_ip: str
multi_pod_hosts: list[str] = field(default_factory=list)
remote_dp_size_local: int = 0
enqueue_time: float = field(default_factory=time.perf_counter)
retried: int = 0
@@ -96,7 +98,7 @@ class RemoteAllocInfo:
writes_expected: int | None = None
decode_dp_rank: int = 0
completion_request_id: str | None = None
completion_remote_notify_port: int | None = None
completion_notify_port: int | None = None
completion_remote_ip: str | None = None
completion_notified: bool = False
transfer_statuses: list[Any] = field(default_factory=list)
@@ -1776,6 +1776,8 @@ class MoRIIOConnectorWorker:
kv_layer: torch.Tensor,
remote_notify_port: int,
remote_ip: str,
multi_pod_hosts: list[str],
remote_dp_size_local: int,
) -> None:
"""Schedule a block write operation.
@@ -1789,6 +1791,8 @@ class MoRIIOConnectorWorker:
kv_layer: KV cache tensor
remote_notify_port: Port for completion notification
remote_ip: IP address of remote node
multi_pod_hosts: List of pod IPs for multi-pod Wide-EP
remote_dp_size_local: Per-pod DP size for multi-pod
"""
# synchronization to prevent dirty reads between
@@ -1810,6 +1814,8 @@ class MoRIIOConnectorWorker:
event=event,
remote_notify_port=remote_notify_port,
remote_ip=remote_ip,
multi_pod_hosts=multi_pod_hosts,
remote_dp_size_local=remote_dp_size_local,
)
self._writer.schedule_write(task)
@@ -3167,18 +3173,17 @@ class MoRIIOConnectorWorker:
)
def _write_blocks_for_req(self, req_id: ReqId, meta: ReqMeta, layer_name, kv_layer):
# Stash multi_pod_hosts + local DP size on the worker so
# MoRIIOEngine._finalize_if_complete (which sees only the WriteTask,
# not ReqMeta) can pick the per-rank pod IP for the completion notify.
# Last-writer-wins is safe: all requests share the same topology.
if meta.multi_pod_hosts:
self.multi_pod_hosts = list(meta.multi_pod_hosts)
else:
self.multi_pod_hosts = [meta.remote_host]
if meta.remote_dp_size_local:
self.remote_dp_size_local = int(meta.remote_dp_size_local)
else:
self.remote_dp_size_local = int(meta.remote_dp_size)
# Compute per-request values to pass through the task (no shared state).
# This avoids race conditions when concurrent requests target different
# decode pods - each WriteTask carries its own routing info.
hosts = (
list(meta.multi_pod_hosts) if meta.multi_pod_hosts else [meta.remote_host]
)
dp_local = (
int(meta.remote_dp_size_local)
if meta.remote_dp_size_local
else int(meta.remote_dp_size)
)
# WRITE does not support HMA, so unwrap blocks into flat lists.
# remote_block_ids can itself be empty so need to be careful when unwrapping.
local_block_ids = meta.local_block_ids[0]
@@ -3193,6 +3198,8 @@ class MoRIIOConnectorWorker:
kv_layer=kv_layer,
remote_notify_port=meta.remote_notify_port,
remote_ip=meta.remote_host,
multi_pod_hosts=hosts,
remote_dp_size_local=dp_local,
)
def merge_contiguous_blocks(
@@ -314,18 +314,27 @@ class MoRIIOWriter:
request_info = self._get_remote_alloc_info(task.transfer_id)
with self._write_state_lock:
request_info.completion_request_id = task.request_id
request_info.completion_remote_notify_port = task.remote_notify_port
# Wide-EP multi-pod: task.remote_ip addresses only the first pod,
# so resolve the per-rank host from multi_pod_hosts (falls back to
# task.remote_ip for single-pod or on any indexing miss).
_remote_ip = task.remote_ip
_hosts = list(getattr(self.worker, "multi_pod_hosts", []) or [])
_dp_local = int(getattr(self.worker, "remote_dp_size_local", 0) or 0)
_hosts = task.multi_pod_hosts
_dp_local = task.remote_dp_size_local
if _hosts and _dp_local > 0:
_pod_idx = int(request_info.decode_dp_rank) // _dp_local
if 0 <= _pod_idx < len(_hosts):
_remote_ip = _hosts[_pod_idx]
request_info.completion_remote_ip = _remote_ip
# Resolve the final notify port under the lock so finalize only
# reads it. The offset uses the per-pod local rank (% dp_local),
# since each pod binds notify sockets only for its local ranks;
# single-pod is bit-identical (modulus is a no-op).
_decode_dp_rank_for_port = int(request_info.decode_dp_rank)
if _dp_local > 0:
_decode_dp_rank_for_port = _decode_dp_rank_for_port % _dp_local
request_info.completion_notify_port = task.remote_notify_port + (
get_port_offset(_decode_dp_rank_for_port, self.worker.tp_rank)
)
if task.transfer_id in self._sealed_writes:
request_info.writes_expected = self._sealed_writes[task.transfer_id]
@@ -461,9 +470,9 @@ class MoRIIOWriter:
if request_info.completion_notified:
return
request_id = request_info.completion_request_id
remote_notify_port = request_info.completion_remote_notify_port
remote_port = request_info.completion_notify_port
remote_ip = request_info.completion_remote_ip
if request_id is None or remote_notify_port is None or remote_ip is None:
if request_id is None or remote_port is None or remote_ip is None:
return
transfer_statuses = list(request_info.transfer_statuses)
request_info.transfer_statuses.clear()
@@ -472,16 +481,6 @@ class MoRIIOWriter:
# Wait for this request's transfers to complete.
self.worker.moriio_wrapper.waiting_for_transfer_complete(transfer_statuses)
# The notify port offset must use the per-pod local rank
# (% dp_local), since each pod binds notify sockets only for its local
# ranks. Single-pod is bit-identical (modulus is a no-op).
_dp_local = int(getattr(self.worker, "remote_dp_size_local", 0) or 0)
_decode_dp_rank_for_port = int(request_info.decode_dp_rank)
if _dp_local > 0:
_decode_dp_rank_for_port = _decode_dp_rank_for_port % _dp_local
remote_port = remote_notify_port + get_port_offset(
_decode_dp_rank_for_port, self.worker.tp_rank
)
# Consider using RDMA immediate data in decode side
# to eliminate the need for this notification.
# Consider including the first gen token from prefill in the notification