[Bugfix][KVConnector] Fix MoRI-IO WRITE-mode requests stranded in WAITING_FOR_REMOTE_KVS (#57705)
Signed-off-by: Akshay Viswanathan <4953343+akshayv@users.noreply.github.com> Co-authored-by: Akshay Viswanathan <4953343+akshayv@users.noreply.github.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Simon Danielsson <70206058+simondanielsson@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Hongxia Yang <62075498+hongxiayang@users.noreply.github.com>
This commit is contained in:
co-authored by
Akshay Viswanathan
Cursor
Simon Danielsson
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Hongxia Yang
parent
44f0ddfd7b
commit
802feb6770
@@ -3,20 +3,30 @@
|
||||
"""Unit tests for mori-io write_done routing (#51681)."""
|
||||
|
||||
import threading
|
||||
import time
|
||||
from collections import OrderedDict
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio import moriio_engine
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import (
|
||||
ROLE,
|
||||
MoRIIOMode,
|
||||
RemoteAllocInfo,
|
||||
ReqMeta,
|
||||
WriteTask,
|
||||
get_port_offset,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import (
|
||||
MoRIIOConnectorScheduler,
|
||||
MoRIIOConnectorWorker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_engine import (
|
||||
MoRIIOWrapper,
|
||||
)
|
||||
|
||||
from .utils import make_moriio_writer
|
||||
|
||||
@@ -79,6 +89,7 @@ def _write_task(
|
||||
*,
|
||||
multi_pod_hosts: list[str] | None = None,
|
||||
remote_dp_size_local: int = 0,
|
||||
remote_dp_size: int = 1,
|
||||
request_id: str = "req",
|
||||
) -> WriteTask:
|
||||
return WriteTask(
|
||||
@@ -93,9 +104,21 @@ def _write_task(
|
||||
remote_ip=remote_ip,
|
||||
multi_pod_hosts=multi_pod_hosts or [],
|
||||
remote_dp_size_local=remote_dp_size_local,
|
||||
remote_dp_size=remote_dp_size,
|
||||
)
|
||||
|
||||
|
||||
def _real_wrapper() -> MoRIIOWrapper:
|
||||
wrapper = MoRIIOWrapper.__new__(MoRIIOWrapper)
|
||||
wrapper.lock = threading.Lock()
|
||||
wrapper.done_req_ids = []
|
||||
wrapper.done_remote_allocate_req_dict = {}
|
||||
wrapper.done_write_cache_req_ids = []
|
||||
wrapper.failed_write_cache_req_ids = []
|
||||
wrapper._terminal_transfer_ids = OrderedDict()
|
||||
return wrapper
|
||||
|
||||
|
||||
def test_execute_write_task_ip_is_per_task_under_overwrite():
|
||||
alloc = {"tA": RemoteAllocInfo(block_ids=None, decode_dp_rank=0)}
|
||||
worker = _fake_worker(alloc)
|
||||
@@ -269,3 +292,84 @@ def test_write_blocks_for_req_falls_back_when_multi_pod_hosts_empty():
|
||||
assert captured["remote_dp_size_local"] == 4
|
||||
assert captured["local_block_ids"] == [1]
|
||||
assert captured["remote_block_ids"] == [2]
|
||||
|
||||
|
||||
def test_deferred_write_timeout_fails_transfer_on_every_decode_rank():
|
||||
wrapper = _real_wrapper()
|
||||
sent: list[tuple[str, int, str]] = []
|
||||
wrapper.send_notify = lambda tid, ip, port, message_type: sent.append(
|
||||
(ip, port, message_type)
|
||||
)
|
||||
writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper, tp_rank=0))
|
||||
task = _write_task(
|
||||
"tA",
|
||||
"10.0.0.1",
|
||||
multi_pod_hosts=["10.0.0.1", "10.0.0.2"],
|
||||
remote_dp_size_local=2,
|
||||
remote_dp_size=4,
|
||||
)
|
||||
task.enqueue_time = time.perf_counter() - 2 * writer._defer_timeout
|
||||
writer._deferred_tasks.append(task)
|
||||
|
||||
writer._process_deferred_tasks()
|
||||
|
||||
assert writer._deferred_tasks == []
|
||||
assert [ack.transfer_id for ack in wrapper.done_req_ids] == ["tA"]
|
||||
# Global DP ranks 0-3, two per pod: the owning rank is unknown.
|
||||
assert sent == [
|
||||
(host, 20000 + get_port_offset(local_rank, 0), "write_failed")
|
||||
for host in ("10.0.0.1", "10.0.0.2")
|
||||
for local_rank in (0, 1)
|
||||
]
|
||||
|
||||
# A late decode allocation must not resurrect a write into freed blocks.
|
||||
with patch.object(moriio_engine, "get_role", return_value=ROLE.PRODUCER):
|
||||
wrapper._handle_remote_blocks_message(
|
||||
{"transfer_id": "tA", "block_notify_list": [7], "decode_rank": 1}
|
||||
)
|
||||
assert wrapper.done_remote_allocate_req_dict == {}
|
||||
|
||||
|
||||
def test_write_scheduler_waits_for_worker_ack_past_defer_deadline():
|
||||
scheduler = SimpleNamespace(
|
||||
is_producer=True,
|
||||
mode=MoRIIOMode.WRITE,
|
||||
_defer_timeout=60.0,
|
||||
_deferred_send_deadlines={"req-1": (time.monotonic() - 1.0, "tA")},
|
||||
_pending_sent_acks={},
|
||||
unmap_request_id=lambda *args, **kwargs: None,
|
||||
)
|
||||
output = SimpleNamespace(finished_sending=set())
|
||||
|
||||
MoRIIOConnectorScheduler.update_connector_output(scheduler, output)
|
||||
assert output.finished_sending is None
|
||||
assert "req-1" in scheduler._deferred_send_deadlines
|
||||
|
||||
output.finished_sending = {"req-1"}
|
||||
MoRIIOConnectorScheduler.update_connector_output(scheduler, output)
|
||||
assert output.finished_sending == {"req-1"}
|
||||
assert scheduler._deferred_send_deadlines == {}
|
||||
|
||||
|
||||
def test_write_failed_is_reported_as_failed_recving_on_owning_rank():
|
||||
wrapper = _real_wrapper()
|
||||
with patch.object(moriio_engine, "get_role", return_value=ROLE.CONSUMER):
|
||||
for tid in ("tA", "tOtherRank"):
|
||||
wrapper._handle_structured_message(
|
||||
{"type": "write_failed", "transfer_id": tid}
|
||||
)
|
||||
worker = SimpleNamespace(
|
||||
mode=MoRIIOMode.WRITE,
|
||||
is_producer=False,
|
||||
moriio_wrapper=wrapper,
|
||||
transfer_id_to_request_id={"tA": "req-1"},
|
||||
_unmatched_write_failures=OrderedDict(),
|
||||
get_finished=lambda: (set(), set()),
|
||||
)
|
||||
|
||||
results = MoRIIOConnectorWorker.get_transfer_results(worker)
|
||||
|
||||
assert results.finished_recving == {"req-1"}
|
||||
assert results.failed_recving == {"req-1"}
|
||||
assert wrapper.pop_finished_write_req_ids() == set()
|
||||
assert list(worker._unmatched_write_failures) == ["tOtherRank"]
|
||||
|
||||
Reference in New Issue
Block a user