[ROCm] Fix misrouting race-condition in multi-decode P/D disagg with mori-io (#51681)

Signed-off-by: Vincent Cave <vincent.cave@amd.com>
Signed-off-by: Shiksha Patel <shikpate@amd.com>
Co-authored-by: Shiksha Patel <shikpate@amd.com>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Vincent Cavé
2026-09-24 19:52:43 +00:00
committed by GitHub
co-authored by Shiksha Patel Douglas Lehr Andreas Karatzas
parent 90a9515006
commit 92e9c6bf13
6 changed files with 343 additions and 49 deletions
@@ -4,8 +4,7 @@
import importlib.util
import math
import threading
from collections import OrderedDict, defaultdict
from queue import Queue
from collections import OrderedDict
from types import SimpleNamespace
from typing import Any
@@ -19,6 +18,8 @@ from vllm.v1.kv_cache_interface import (
MLAAttentionSpec,
)
from .utils import make_moriio_writer
aiter_available = importlib.util.find_spec("aiter") is not None
mori_available = importlib.util.find_spec("mori") is not None
@@ -90,18 +91,6 @@ def _remote_meta(num_blocks: int = 16) -> SimpleNamespace:
return SimpleNamespace(num_blocks=num_blocks)
def _writer_with_fake_worker(fake_worker: Any) -> Any:
writer = MoRIIOWriter.__new__(MoRIIOWriter)
writer._worker_ref = lambda: fake_worker
writer._write_task_q = Queue()
writer._write_state_lock = threading.Lock()
writer._scheduled_writes = defaultdict(int)
writer._scheduled_layers = defaultdict(set)
writer._sealed_writes = {}
writer.ensure_worker_started = lambda: None
return writer
def _wrapper_for_messages() -> Any:
wrapper = MoRIIOWrapper.__new__(MoRIIOWrapper)
wrapper.lock = threading.Lock()
@@ -409,7 +398,7 @@ def test_write_scheduler_deduplicates_layers_and_seals_expected_count():
request_info = RemoteAllocInfo(block_ids=[4, 5])
wrapper = _wrapper_for_messages()
wrapper.done_remote_allocate_req_dict["xfer"] = request_info
writer = _writer_with_fake_worker(SimpleNamespace(moriio_wrapper=wrapper))
writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper))
assert writer.schedule_write(_write_task("dense0"))
assert not writer.schedule_write(_write_task("dense0"))
@@ -459,12 +448,12 @@ def test_write_completion_notifies_once_after_all_sealed_writes_finish():
request_info = RemoteAllocInfo(block_ids=[4, 5], writes_expected=2)
request_info.transfer_statuses.extend(["status-a", "status-b"])
request_info.completion_request_id = "req"
request_info.completion_remote_notify_port = 7000
# Final notify port is resolved in _execute_write_task; finalize reads it
# as-is, so set the already-offset value (base 7000 + tp_rank-2 offset).
request_info.completion_notify_port = 7002
request_info.completion_remote_ip = "127.0.0.1"
wrapper.done_remote_allocate_req_dict["xfer"] = request_info
writer = _writer_with_fake_worker(
SimpleNamespace(moriio_wrapper=wrapper, tp_rank=2)
)
writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper, tp_rank=2))
writer._scheduled_writes["xfer"] = 2
writer._scheduled_layers["xfer"] = {"dense0", "indexer"}
writer._sealed_writes["xfer"] = 2
@@ -512,7 +501,7 @@ def test_moriio_wrapper_waits_scoped_statuses_without_global_drain():
def test_write_failure_marks_terminal_and_clears_scheduled_state():
wrapper = _wrapper_for_messages()
wrapper.done_remote_allocate_req_dict["xfer"] = RemoteAllocInfo(block_ids=[4, 5])
writer = _writer_with_fake_worker(SimpleNamespace(moriio_wrapper=wrapper))
writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper))
writer._scheduled_writes["xfer"] = 2
writer._scheduled_layers["xfer"] = {"dense0", "indexer"}
writer._sealed_writes["xfer"] = 2
@@ -530,7 +519,7 @@ def test_write_failure_marks_terminal_and_clears_scheduled_state():
def test_schedule_write_rejects_terminal_transfer_without_recreating_state():
wrapper = _wrapper_for_messages()
wrapper.done_remote_allocate_req_dict["xfer"] = RemoteAllocInfo(block_ids=[4, 5])
writer = _writer_with_fake_worker(SimpleNamespace(moriio_wrapper=wrapper))
writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper))
writer._scheduled_writes["xfer"] = 1
writer._scheduled_layers["xfer"] = {"dense0"}
writer._sealed_writes["xfer"] = 1
@@ -0,0 +1,271 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for mori-io write_done routing (#51681)."""
import threading
from types import SimpleNamespace
from typing import Any
import pytest
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import (
RemoteAllocInfo,
ReqMeta,
WriteTask,
get_port_offset,
)
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import (
MoRIIOConnectorWorker,
)
from .utils import make_moriio_writer
pytestmark = pytest.mark.cpu_test
def _fake_worker(
alloc: dict[str, Any],
*,
tp_rank: int = 0,
set_remote_dp_size_local: int | None = None,
) -> Any:
wrapper = SimpleNamespace(
done_remote_allocate_req_dict=alloc,
lock=threading.Lock(),
done_req_ids=[],
sent_notifies=[],
)
wrapper.send_notify = lambda transfer_id, remote_ip, remote_port, message_type: (
wrapper.sent_notifies.append(
(transfer_id, remote_ip, remote_port, message_type)
)
)
wrapper.waiting_for_transfer_complete = lambda statuses: None
wrapper._mark_transfer_terminal_locked = lambda transfer_id: None
wrapper._is_transfer_terminal_locked = lambda transfer_id: False
worker = SimpleNamespace(moriio_wrapper=wrapper, tp_rank=tp_rank)
if set_remote_dp_size_local is not None:
worker.remote_dp_size_local = set_remote_dp_size_local
return worker
def _req_meta(
*,
transfer_id: str = "tA",
remote_host: str = "10.0.0.1",
multi_pod_hosts: list[str] | None = None,
remote_dp_size: int = 16,
remote_dp_size_local: int = 8,
) -> ReqMeta:
return ReqMeta(
transfer_id=transfer_id,
local_block_ids=[[1]],
remote_block_ids=[[2]],
remote_host=remote_host,
remote_port=1,
remote_handshake_port=2,
remote_notify_port=20000,
remote_engine_id="remote-engine",
tp_size=1,
remote_dp_size=remote_dp_size,
multi_pod_hosts=list(multi_pod_hosts or []),
remote_dp_size_local=remote_dp_size_local,
)
def _write_task(
transfer_id: str,
remote_ip: str,
*,
multi_pod_hosts: list[str] | None = None,
remote_dp_size_local: int = 0,
request_id: str = "req",
) -> WriteTask:
return WriteTask(
request_id=request_id,
transfer_id=transfer_id,
dst_engine_id="remote-engine",
local_block_ids=[1, 3],
remote_block_ids_hint=None,
layer_name="layer0",
event=None,
remote_notify_port=20000,
remote_ip=remote_ip,
multi_pod_hosts=multi_pod_hosts or [],
remote_dp_size_local=remote_dp_size_local,
)
def test_execute_write_task_ip_is_per_task_under_overwrite():
alloc = {"tA": RemoteAllocInfo(block_ids=None, decode_dp_rank=0)}
worker = _fake_worker(alloc)
worker.multi_pod_hosts = ["10.0.0.2"]
worker.remote_dp_size_local = 1
writer = make_moriio_writer(worker)
task_a = _write_task(
"tA", "10.0.0.1", multi_pod_hosts=["10.0.0.1"], remote_dp_size_local=1
)
writer._execute_write_task(task_a)
assert alloc["tA"].completion_remote_ip == "10.0.0.1"
assert alloc["tA"].completion_remote_ip != "10.0.0.2"
def test_deferred_task_ip_is_per_task_under_overwrite():
alloc = {"tA": RemoteAllocInfo(block_ids=None, decode_dp_rank=0)}
worker = _fake_worker(alloc)
worker.multi_pod_hosts = ["10.0.0.2"]
worker.remote_dp_size_local = 1
writer = make_moriio_writer(worker)
task_a = _write_task(
"tA", "10.0.0.1", multi_pod_hosts=["10.0.0.1"], remote_dp_size_local=1
)
writer._deferred_tasks.append(task_a)
writer._process_deferred_tasks()
assert alloc["tA"].completion_remote_ip == "10.0.0.1"
assert writer._deferred_tasks == []
def test_two_requests_resolve_distinct_hosts():
alloc = {
"tA": RemoteAllocInfo(block_ids=None, decode_dp_rank=0),
"tB": RemoteAllocInfo(block_ids=None, decode_dp_rank=3),
}
worker = _fake_worker(alloc)
writer = make_moriio_writer(worker)
task_a = _write_task(
"tA", "10.0.0.1", multi_pod_hosts=["10.0.0.1"], remote_dp_size_local=1
)
task_b = _write_task(
"tB", "10.0.0.2", multi_pod_hosts=["10.0.0.2"], remote_dp_size_local=1
)
worker.multi_pod_hosts = ["10.0.0.2"]
writer._execute_write_task(task_a)
writer._execute_write_task(task_b)
assert alloc["tA"].completion_remote_ip == "10.0.0.1"
assert alloc["tB"].completion_remote_ip == "10.0.0.2"
def test_finalize_notify_port_uses_pod_local_rank():
base = 20000
info = RemoteAllocInfo(block_ids=None, decode_dp_rank=9)
alloc = {"tA": info}
worker = _fake_worker(alloc, tp_rank=0)
writer = make_moriio_writer(worker)
task = _write_task(
"tA",
"10.0.0.1",
multi_pod_hosts=["10.0.0.1", "10.0.0.2"],
remote_dp_size_local=8,
)
writer._execute_write_task(task)
assert info.completion_remote_ip == "10.0.0.2"
info.writes_expected = 1
info.writes_done = 1
writer._finalize_if_complete("tA", info)
assert worker.moriio_wrapper.sent_notifies
_tid, ip, port, mtype = worker.moriio_wrapper.sent_notifies[-1]
assert mtype == "write_done"
assert ip == "10.0.0.2"
assert port == base + get_port_offset(9 % 8, 0)
def test_finalize_port_ignores_stale_worker_dp_local():
base = 20000
info = RemoteAllocInfo(block_ids=None, decode_dp_rank=9)
alloc = {"tA": info}
worker = _fake_worker(alloc, tp_rank=0, set_remote_dp_size_local=1)
writer = make_moriio_writer(worker)
task = _write_task(
"tA",
"10.0.0.1",
multi_pod_hosts=["10.0.0.1", "10.0.0.2"],
remote_dp_size_local=8,
)
writer._execute_write_task(task)
assert info.completion_notify_port == base + get_port_offset(9 % 8, 0)
info.writes_expected = 1
info.writes_done = 1
writer._finalize_if_complete("tA", info)
_tid, _ip, port, mtype = worker.moriio_wrapper.sent_notifies[-1]
assert mtype == "write_done"
assert port == base + get_port_offset(9 % 8, 0)
assert port != base + get_port_offset(9 % 1, 0)
def test_finalize_single_pod_port_uses_global_rank():
base = 20000
info = RemoteAllocInfo(block_ids=None, decode_dp_rank=3)
alloc = {"tA": info}
worker = _fake_worker(alloc, tp_rank=2)
writer = make_moriio_writer(worker)
task = _write_task("tA", "10.0.0.1", multi_pod_hosts=[], remote_dp_size_local=0)
writer._execute_write_task(task)
assert info.completion_remote_ip == "10.0.0.1"
assert info.completion_notify_port == base + get_port_offset(3, 2)
info.writes_expected = 1
info.writes_done = 1
writer._finalize_if_complete("tA", info)
_tid, _ip, port, _mtype = worker.moriio_wrapper.sent_notifies[-1]
assert port == base + get_port_offset(3, 2)
def test_write_blocks_for_req_forwards_routing_to_schedule_write():
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
captured: dict[str, Any] = {}
def _capture_schedule_write_blocks(**kwargs: Any) -> None:
captured.update(kwargs)
worker.schedule_write_blocks = _capture_schedule_write_blocks # type: ignore[method-assign]
meta = _req_meta(
remote_host="10.0.0.1",
multi_pod_hosts=["10.0.0.1", "10.0.0.2"],
remote_dp_size_local=8,
)
worker._write_blocks_for_req("req-a", meta, "layer0", kv_layer=None)
assert captured["multi_pod_hosts"] == ["10.0.0.1", "10.0.0.2"]
assert captured["remote_dp_size_local"] == 8
assert captured["remote_ip"] == "10.0.0.1"
assert captured["local_block_ids"] == [1]
assert captured["remote_block_ids"] == [2]
assert not hasattr(worker, "multi_pod_hosts")
def test_write_blocks_for_req_falls_back_when_multi_pod_hosts_empty():
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
captured: dict[str, Any] = {}
def _capture_schedule_write_blocks(**kwargs: Any) -> None:
captured.update(kwargs)
worker.schedule_write_blocks = _capture_schedule_write_blocks # type: ignore[method-assign]
meta = _req_meta(
remote_host="10.0.0.9",
multi_pod_hosts=[],
remote_dp_size=4,
remote_dp_size_local=0,
)
worker._write_blocks_for_req("req-b", meta, "layer0", kv_layer=None)
assert captured["multi_pod_hosts"] == ["10.0.0.9"]
assert captured["remote_dp_size_local"] == 4
assert captured["local_block_ids"] == [1]
assert captured["remote_block_ids"] == [2]
+26
View File
@@ -632,3 +632,29 @@ def make_nixl_push_scheduler(
sched.blocks_per_sw = []
return sched
def make_moriio_writer(fake_worker: Any) -> Any:
"""Build a MoRIIOWriter with internals stubbed for unit tests.
Bypasses ``__init__`` and wires only the write/finalize state the tests
touch, including the deferred-task fields used by the routing suite.
"""
import threading
from queue import Queue
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_engine import (
MoRIIOWriter,
)
writer = MoRIIOWriter.__new__(MoRIIOWriter)
writer._worker_ref = lambda: fake_worker
writer._write_task_q = Queue()
writer._write_state_lock = threading.Lock()
writer._scheduled_writes = defaultdict(int)
writer._scheduled_layers = defaultdict(set)
writer._sealed_writes = {}
writer._deferred_tasks = []
writer._defer_timeout = 60.0
writer.ensure_worker_started = lambda: None
return writer