From 92e9c6bf13138da9caa42c815e242affee046c20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Cav=C3=A9?= Date: Thu, 24 Sep 2026 21:52:43 +0200 Subject: [PATCH] [ROCm] Fix misrouting race-condition in multi-decode P/D disagg with mori-io (#51681) Signed-off-by: Vincent Cave Signed-off-by: Shiksha Patel Co-authored-by: Shiksha Patel Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com> Co-authored-by: Andreas Karatzas --- .../unit/test_moriio_kv_layout.py | 31 +- .../unit/test_moriio_write_done_routing.py | 271 ++++++++++++++++++ tests/v1/kv_connector/unit/utils.py | 26 ++ .../kv_connector/v1/moriio/moriio_common.py | 4 +- .../v1/moriio/moriio_connector.py | 31 +- .../kv_connector/v1/moriio/moriio_engine.py | 29 +- 6 files changed, 343 insertions(+), 49 deletions(-) create mode 100644 tests/v1/kv_connector/unit/test_moriio_write_done_routing.py diff --git a/tests/v1/kv_connector/unit/test_moriio_kv_layout.py b/tests/v1/kv_connector/unit/test_moriio_kv_layout.py index 1cb5e271bf..9860a7b7e1 100644 --- a/tests/v1/kv_connector/unit/test_moriio_kv_layout.py +++ b/tests/v1/kv_connector/unit/test_moriio_kv_layout.py @@ -4,8 +4,7 @@ import importlib.util import math import threading -from collections import OrderedDict, defaultdict -from queue import Queue +from collections import OrderedDict from types import SimpleNamespace from typing import Any @@ -19,6 +18,8 @@ from vllm.v1.kv_cache_interface import ( MLAAttentionSpec, ) +from .utils import make_moriio_writer + aiter_available = importlib.util.find_spec("aiter") is not None mori_available = importlib.util.find_spec("mori") is not None @@ -90,18 +91,6 @@ def _remote_meta(num_blocks: int = 16) -> SimpleNamespace: return SimpleNamespace(num_blocks=num_blocks) -def _writer_with_fake_worker(fake_worker: Any) -> Any: - writer = MoRIIOWriter.__new__(MoRIIOWriter) - writer._worker_ref = lambda: fake_worker - writer._write_task_q = Queue() - writer._write_state_lock = threading.Lock() - writer._scheduled_writes = defaultdict(int) - writer._scheduled_layers = defaultdict(set) - writer._sealed_writes = {} - writer.ensure_worker_started = lambda: None - return writer - - def _wrapper_for_messages() -> Any: wrapper = MoRIIOWrapper.__new__(MoRIIOWrapper) wrapper.lock = threading.Lock() @@ -409,7 +398,7 @@ def test_write_scheduler_deduplicates_layers_and_seals_expected_count(): request_info = RemoteAllocInfo(block_ids=[4, 5]) wrapper = _wrapper_for_messages() wrapper.done_remote_allocate_req_dict["xfer"] = request_info - writer = _writer_with_fake_worker(SimpleNamespace(moriio_wrapper=wrapper)) + writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper)) assert writer.schedule_write(_write_task("dense0")) assert not writer.schedule_write(_write_task("dense0")) @@ -459,12 +448,12 @@ def test_write_completion_notifies_once_after_all_sealed_writes_finish(): request_info = RemoteAllocInfo(block_ids=[4, 5], writes_expected=2) request_info.transfer_statuses.extend(["status-a", "status-b"]) request_info.completion_request_id = "req" - request_info.completion_remote_notify_port = 7000 + # Final notify port is resolved in _execute_write_task; finalize reads it + # as-is, so set the already-offset value (base 7000 + tp_rank-2 offset). + request_info.completion_notify_port = 7002 request_info.completion_remote_ip = "127.0.0.1" wrapper.done_remote_allocate_req_dict["xfer"] = request_info - writer = _writer_with_fake_worker( - SimpleNamespace(moriio_wrapper=wrapper, tp_rank=2) - ) + writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper, tp_rank=2)) writer._scheduled_writes["xfer"] = 2 writer._scheduled_layers["xfer"] = {"dense0", "indexer"} writer._sealed_writes["xfer"] = 2 @@ -512,7 +501,7 @@ def test_moriio_wrapper_waits_scoped_statuses_without_global_drain(): def test_write_failure_marks_terminal_and_clears_scheduled_state(): wrapper = _wrapper_for_messages() wrapper.done_remote_allocate_req_dict["xfer"] = RemoteAllocInfo(block_ids=[4, 5]) - writer = _writer_with_fake_worker(SimpleNamespace(moriio_wrapper=wrapper)) + writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper)) writer._scheduled_writes["xfer"] = 2 writer._scheduled_layers["xfer"] = {"dense0", "indexer"} writer._sealed_writes["xfer"] = 2 @@ -530,7 +519,7 @@ def test_write_failure_marks_terminal_and_clears_scheduled_state(): def test_schedule_write_rejects_terminal_transfer_without_recreating_state(): wrapper = _wrapper_for_messages() wrapper.done_remote_allocate_req_dict["xfer"] = RemoteAllocInfo(block_ids=[4, 5]) - writer = _writer_with_fake_worker(SimpleNamespace(moriio_wrapper=wrapper)) + writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper)) writer._scheduled_writes["xfer"] = 1 writer._scheduled_layers["xfer"] = {"dense0"} writer._sealed_writes["xfer"] = 1 diff --git a/tests/v1/kv_connector/unit/test_moriio_write_done_routing.py b/tests/v1/kv_connector/unit/test_moriio_write_done_routing.py new file mode 100644 index 0000000000..adac007d19 --- /dev/null +++ b/tests/v1/kv_connector/unit/test_moriio_write_done_routing.py @@ -0,0 +1,271 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for mori-io write_done routing (#51681).""" + +import threading +from types import SimpleNamespace +from typing import Any + +import pytest + +from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import ( + RemoteAllocInfo, + ReqMeta, + WriteTask, + get_port_offset, +) +from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import ( + MoRIIOConnectorWorker, +) + +from .utils import make_moriio_writer + +pytestmark = pytest.mark.cpu_test + + +def _fake_worker( + alloc: dict[str, Any], + *, + tp_rank: int = 0, + set_remote_dp_size_local: int | None = None, +) -> Any: + wrapper = SimpleNamespace( + done_remote_allocate_req_dict=alloc, + lock=threading.Lock(), + done_req_ids=[], + sent_notifies=[], + ) + wrapper.send_notify = lambda transfer_id, remote_ip, remote_port, message_type: ( + wrapper.sent_notifies.append( + (transfer_id, remote_ip, remote_port, message_type) + ) + ) + wrapper.waiting_for_transfer_complete = lambda statuses: None + wrapper._mark_transfer_terminal_locked = lambda transfer_id: None + wrapper._is_transfer_terminal_locked = lambda transfer_id: False + worker = SimpleNamespace(moriio_wrapper=wrapper, tp_rank=tp_rank) + if set_remote_dp_size_local is not None: + worker.remote_dp_size_local = set_remote_dp_size_local + return worker + + +def _req_meta( + *, + transfer_id: str = "tA", + remote_host: str = "10.0.0.1", + multi_pod_hosts: list[str] | None = None, + remote_dp_size: int = 16, + remote_dp_size_local: int = 8, +) -> ReqMeta: + return ReqMeta( + transfer_id=transfer_id, + local_block_ids=[[1]], + remote_block_ids=[[2]], + remote_host=remote_host, + remote_port=1, + remote_handshake_port=2, + remote_notify_port=20000, + remote_engine_id="remote-engine", + tp_size=1, + remote_dp_size=remote_dp_size, + multi_pod_hosts=list(multi_pod_hosts or []), + remote_dp_size_local=remote_dp_size_local, + ) + + +def _write_task( + transfer_id: str, + remote_ip: str, + *, + multi_pod_hosts: list[str] | None = None, + remote_dp_size_local: int = 0, + request_id: str = "req", +) -> WriteTask: + return WriteTask( + request_id=request_id, + transfer_id=transfer_id, + dst_engine_id="remote-engine", + local_block_ids=[1, 3], + remote_block_ids_hint=None, + layer_name="layer0", + event=None, + remote_notify_port=20000, + remote_ip=remote_ip, + multi_pod_hosts=multi_pod_hosts or [], + remote_dp_size_local=remote_dp_size_local, + ) + + +def test_execute_write_task_ip_is_per_task_under_overwrite(): + alloc = {"tA": RemoteAllocInfo(block_ids=None, decode_dp_rank=0)} + worker = _fake_worker(alloc) + worker.multi_pod_hosts = ["10.0.0.2"] + worker.remote_dp_size_local = 1 + + writer = make_moriio_writer(worker) + task_a = _write_task( + "tA", "10.0.0.1", multi_pod_hosts=["10.0.0.1"], remote_dp_size_local=1 + ) + writer._execute_write_task(task_a) + + assert alloc["tA"].completion_remote_ip == "10.0.0.1" + assert alloc["tA"].completion_remote_ip != "10.0.0.2" + + +def test_deferred_task_ip_is_per_task_under_overwrite(): + alloc = {"tA": RemoteAllocInfo(block_ids=None, decode_dp_rank=0)} + worker = _fake_worker(alloc) + worker.multi_pod_hosts = ["10.0.0.2"] + worker.remote_dp_size_local = 1 + + writer = make_moriio_writer(worker) + task_a = _write_task( + "tA", "10.0.0.1", multi_pod_hosts=["10.0.0.1"], remote_dp_size_local=1 + ) + writer._deferred_tasks.append(task_a) + writer._process_deferred_tasks() + + assert alloc["tA"].completion_remote_ip == "10.0.0.1" + assert writer._deferred_tasks == [] + + +def test_two_requests_resolve_distinct_hosts(): + alloc = { + "tA": RemoteAllocInfo(block_ids=None, decode_dp_rank=0), + "tB": RemoteAllocInfo(block_ids=None, decode_dp_rank=3), + } + worker = _fake_worker(alloc) + writer = make_moriio_writer(worker) + + task_a = _write_task( + "tA", "10.0.0.1", multi_pod_hosts=["10.0.0.1"], remote_dp_size_local=1 + ) + task_b = _write_task( + "tB", "10.0.0.2", multi_pod_hosts=["10.0.0.2"], remote_dp_size_local=1 + ) + worker.multi_pod_hosts = ["10.0.0.2"] + writer._execute_write_task(task_a) + writer._execute_write_task(task_b) + + assert alloc["tA"].completion_remote_ip == "10.0.0.1" + assert alloc["tB"].completion_remote_ip == "10.0.0.2" + + +def test_finalize_notify_port_uses_pod_local_rank(): + base = 20000 + info = RemoteAllocInfo(block_ids=None, decode_dp_rank=9) + alloc = {"tA": info} + worker = _fake_worker(alloc, tp_rank=0) + writer = make_moriio_writer(worker) + + task = _write_task( + "tA", + "10.0.0.1", + multi_pod_hosts=["10.0.0.1", "10.0.0.2"], + remote_dp_size_local=8, + ) + writer._execute_write_task(task) + assert info.completion_remote_ip == "10.0.0.2" + + info.writes_expected = 1 + info.writes_done = 1 + writer._finalize_if_complete("tA", info) + + assert worker.moriio_wrapper.sent_notifies + _tid, ip, port, mtype = worker.moriio_wrapper.sent_notifies[-1] + assert mtype == "write_done" + assert ip == "10.0.0.2" + assert port == base + get_port_offset(9 % 8, 0) + + +def test_finalize_port_ignores_stale_worker_dp_local(): + base = 20000 + info = RemoteAllocInfo(block_ids=None, decode_dp_rank=9) + alloc = {"tA": info} + worker = _fake_worker(alloc, tp_rank=0, set_remote_dp_size_local=1) + writer = make_moriio_writer(worker) + + task = _write_task( + "tA", + "10.0.0.1", + multi_pod_hosts=["10.0.0.1", "10.0.0.2"], + remote_dp_size_local=8, + ) + writer._execute_write_task(task) + assert info.completion_notify_port == base + get_port_offset(9 % 8, 0) + + info.writes_expected = 1 + info.writes_done = 1 + writer._finalize_if_complete("tA", info) + + _tid, _ip, port, mtype = worker.moriio_wrapper.sent_notifies[-1] + assert mtype == "write_done" + assert port == base + get_port_offset(9 % 8, 0) + assert port != base + get_port_offset(9 % 1, 0) + + +def test_finalize_single_pod_port_uses_global_rank(): + base = 20000 + info = RemoteAllocInfo(block_ids=None, decode_dp_rank=3) + alloc = {"tA": info} + worker = _fake_worker(alloc, tp_rank=2) + writer = make_moriio_writer(worker) + + task = _write_task("tA", "10.0.0.1", multi_pod_hosts=[], remote_dp_size_local=0) + writer._execute_write_task(task) + assert info.completion_remote_ip == "10.0.0.1" + assert info.completion_notify_port == base + get_port_offset(3, 2) + + info.writes_expected = 1 + info.writes_done = 1 + writer._finalize_if_complete("tA", info) + + _tid, _ip, port, _mtype = worker.moriio_wrapper.sent_notifies[-1] + assert port == base + get_port_offset(3, 2) + + +def test_write_blocks_for_req_forwards_routing_to_schedule_write(): + worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker) + captured: dict[str, Any] = {} + + def _capture_schedule_write_blocks(**kwargs: Any) -> None: + captured.update(kwargs) + + worker.schedule_write_blocks = _capture_schedule_write_blocks # type: ignore[method-assign] + + meta = _req_meta( + remote_host="10.0.0.1", + multi_pod_hosts=["10.0.0.1", "10.0.0.2"], + remote_dp_size_local=8, + ) + worker._write_blocks_for_req("req-a", meta, "layer0", kv_layer=None) + + assert captured["multi_pod_hosts"] == ["10.0.0.1", "10.0.0.2"] + assert captured["remote_dp_size_local"] == 8 + assert captured["remote_ip"] == "10.0.0.1" + assert captured["local_block_ids"] == [1] + assert captured["remote_block_ids"] == [2] + assert not hasattr(worker, "multi_pod_hosts") + + +def test_write_blocks_for_req_falls_back_when_multi_pod_hosts_empty(): + worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker) + captured: dict[str, Any] = {} + + def _capture_schedule_write_blocks(**kwargs: Any) -> None: + captured.update(kwargs) + + worker.schedule_write_blocks = _capture_schedule_write_blocks # type: ignore[method-assign] + + meta = _req_meta( + remote_host="10.0.0.9", + multi_pod_hosts=[], + remote_dp_size=4, + remote_dp_size_local=0, + ) + worker._write_blocks_for_req("req-b", meta, "layer0", kv_layer=None) + + assert captured["multi_pod_hosts"] == ["10.0.0.9"] + assert captured["remote_dp_size_local"] == 4 + assert captured["local_block_ids"] == [1] + assert captured["remote_block_ids"] == [2] diff --git a/tests/v1/kv_connector/unit/utils.py b/tests/v1/kv_connector/unit/utils.py index 85ada84b0d..dda7c0f386 100644 --- a/tests/v1/kv_connector/unit/utils.py +++ b/tests/v1/kv_connector/unit/utils.py @@ -632,3 +632,29 @@ def make_nixl_push_scheduler( sched.blocks_per_sw = [] return sched + + +def make_moriio_writer(fake_worker: Any) -> Any: + """Build a MoRIIOWriter with internals stubbed for unit tests. + + Bypasses ``__init__`` and wires only the write/finalize state the tests + touch, including the deferred-task fields used by the routing suite. + """ + import threading + from queue import Queue + + from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_engine import ( + MoRIIOWriter, + ) + + writer = MoRIIOWriter.__new__(MoRIIOWriter) + writer._worker_ref = lambda: fake_worker + writer._write_task_q = Queue() + writer._write_state_lock = threading.Lock() + writer._scheduled_writes = defaultdict(int) + writer._scheduled_layers = defaultdict(set) + writer._sealed_writes = {} + writer._deferred_tasks = [] + writer._defer_timeout = 60.0 + writer.ensure_worker_started = lambda: None + return writer diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py index 5f1068562c..ae1eecf6e5 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py @@ -69,6 +69,8 @@ class WriteTask: event: torch.cuda.Event remote_notify_port: int remote_ip: str + multi_pod_hosts: list[str] = field(default_factory=list) + remote_dp_size_local: int = 0 enqueue_time: float = field(default_factory=time.perf_counter) retried: int = 0 @@ -96,7 +98,7 @@ class RemoteAllocInfo: writes_expected: int | None = None decode_dp_rank: int = 0 completion_request_id: str | None = None - completion_remote_notify_port: int | None = None + completion_notify_port: int | None = None completion_remote_ip: str | None = None completion_notified: bool = False transfer_statuses: list[Any] = field(default_factory=list) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py index dce4a98068..0a8965a670 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py @@ -1776,6 +1776,8 @@ class MoRIIOConnectorWorker: kv_layer: torch.Tensor, remote_notify_port: int, remote_ip: str, + multi_pod_hosts: list[str], + remote_dp_size_local: int, ) -> None: """Schedule a block write operation. @@ -1789,6 +1791,8 @@ class MoRIIOConnectorWorker: kv_layer: KV cache tensor remote_notify_port: Port for completion notification remote_ip: IP address of remote node + multi_pod_hosts: List of pod IPs for multi-pod Wide-EP + remote_dp_size_local: Per-pod DP size for multi-pod """ # synchronization to prevent dirty reads between @@ -1810,6 +1814,8 @@ class MoRIIOConnectorWorker: event=event, remote_notify_port=remote_notify_port, remote_ip=remote_ip, + multi_pod_hosts=multi_pod_hosts, + remote_dp_size_local=remote_dp_size_local, ) self._writer.schedule_write(task) @@ -3167,18 +3173,17 @@ class MoRIIOConnectorWorker: ) def _write_blocks_for_req(self, req_id: ReqId, meta: ReqMeta, layer_name, kv_layer): - # Stash multi_pod_hosts + local DP size on the worker so - # MoRIIOEngine._finalize_if_complete (which sees only the WriteTask, - # not ReqMeta) can pick the per-rank pod IP for the completion notify. - # Last-writer-wins is safe: all requests share the same topology. - if meta.multi_pod_hosts: - self.multi_pod_hosts = list(meta.multi_pod_hosts) - else: - self.multi_pod_hosts = [meta.remote_host] - if meta.remote_dp_size_local: - self.remote_dp_size_local = int(meta.remote_dp_size_local) - else: - self.remote_dp_size_local = int(meta.remote_dp_size) + # Compute per-request values to pass through the task (no shared state). + # This avoids race conditions when concurrent requests target different + # decode pods - each WriteTask carries its own routing info. + hosts = ( + list(meta.multi_pod_hosts) if meta.multi_pod_hosts else [meta.remote_host] + ) + dp_local = ( + int(meta.remote_dp_size_local) + if meta.remote_dp_size_local + else int(meta.remote_dp_size) + ) # WRITE does not support HMA, so unwrap blocks into flat lists. # remote_block_ids can itself be empty so need to be careful when unwrapping. local_block_ids = meta.local_block_ids[0] @@ -3193,6 +3198,8 @@ class MoRIIOConnectorWorker: kv_layer=kv_layer, remote_notify_port=meta.remote_notify_port, remote_ip=meta.remote_host, + multi_pod_hosts=hosts, + remote_dp_size_local=dp_local, ) def merge_contiguous_blocks( diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_engine.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_engine.py index a0c6ec1f6d..4ec3f72fb4 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_engine.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_engine.py @@ -314,18 +314,27 @@ class MoRIIOWriter: request_info = self._get_remote_alloc_info(task.transfer_id) with self._write_state_lock: request_info.completion_request_id = task.request_id - request_info.completion_remote_notify_port = task.remote_notify_port # Wide-EP multi-pod: task.remote_ip addresses only the first pod, # so resolve the per-rank host from multi_pod_hosts (falls back to # task.remote_ip for single-pod or on any indexing miss). _remote_ip = task.remote_ip - _hosts = list(getattr(self.worker, "multi_pod_hosts", []) or []) - _dp_local = int(getattr(self.worker, "remote_dp_size_local", 0) or 0) + _hosts = task.multi_pod_hosts + _dp_local = task.remote_dp_size_local if _hosts and _dp_local > 0: _pod_idx = int(request_info.decode_dp_rank) // _dp_local if 0 <= _pod_idx < len(_hosts): _remote_ip = _hosts[_pod_idx] request_info.completion_remote_ip = _remote_ip + # Resolve the final notify port under the lock so finalize only + # reads it. The offset uses the per-pod local rank (% dp_local), + # since each pod binds notify sockets only for its local ranks; + # single-pod is bit-identical (modulus is a no-op). + _decode_dp_rank_for_port = int(request_info.decode_dp_rank) + if _dp_local > 0: + _decode_dp_rank_for_port = _decode_dp_rank_for_port % _dp_local + request_info.completion_notify_port = task.remote_notify_port + ( + get_port_offset(_decode_dp_rank_for_port, self.worker.tp_rank) + ) if task.transfer_id in self._sealed_writes: request_info.writes_expected = self._sealed_writes[task.transfer_id] @@ -461,9 +470,9 @@ class MoRIIOWriter: if request_info.completion_notified: return request_id = request_info.completion_request_id - remote_notify_port = request_info.completion_remote_notify_port + remote_port = request_info.completion_notify_port remote_ip = request_info.completion_remote_ip - if request_id is None or remote_notify_port is None or remote_ip is None: + if request_id is None or remote_port is None or remote_ip is None: return transfer_statuses = list(request_info.transfer_statuses) request_info.transfer_statuses.clear() @@ -472,16 +481,6 @@ class MoRIIOWriter: # Wait for this request's transfers to complete. self.worker.moriio_wrapper.waiting_for_transfer_complete(transfer_statuses) - # The notify port offset must use the per-pod local rank - # (% dp_local), since each pod binds notify sockets only for its local - # ranks. Single-pod is bit-identical (modulus is a no-op). - _dp_local = int(getattr(self.worker, "remote_dp_size_local", 0) or 0) - _decode_dp_rank_for_port = int(request_info.decode_dp_rank) - if _dp_local > 0: - _decode_dp_rank_for_port = _decode_dp_rank_for_port % _dp_local - remote_port = remote_notify_port + get_port_offset( - _decode_dp_rank_for_port, self.worker.tp_rank - ) # Consider using RDMA immediate data in decode side # to eliminate the need for this notification. # Consider including the first gen token from prefill in the notification