llmd+vllm+mori-ep(inter node wide-ep)+mori-io(write) for 2p2d with dp=ep=16 tp=1 (#45043)

Signed-off-by: Shiksha Patel <shikpate@amd.com>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
shikamd123
2026-08-04 00:13:15 +00:00
committed by GitHub
co-authored by Douglas Lehr mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent c810937573
commit f42761204f
8 changed files with 710 additions and 120 deletions
@@ -508,9 +508,15 @@ def test_send_transfer_release_sends_structured_release_message():
scheduler._send_transfer_release("xfer-7", "127.0.0.1", 7000)
payload = sock.send.call_args.args[0]
# WRITE-mode release advertises the consumer (decode) TP size so the prefill
# side counts the right number of ACKs via get_moriio_expected_ack_count,
# mirroring the READ-mode release in _pop_done_transfers (see
# test_read_completion_sends_structured_release_with_consumer_tp_size). The
# fixture's tp_size is 2, so consumer_tp_size == 2.
assert msgspec.msgpack.decode(payload) == {
"type": "release",
"transfer_id": "xfer-7",
"consumer_tp_size": 2,
}
@@ -0,0 +1,92 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for MoRIIO router-authoritative DP-rank routing.
These lock in the redesign that addresses the PR #45043 review feedback
(njhill): the connector must NOT self-derive a DP rank, and the prefill leg
must *return* the rank it actually ran on via ``request_finished`` so the
decode->prefill notify is routed by pure propagation (READ / serial WRITE),
not by an independent hash on each side.
Following test_moriio_unmap.py, we bind the unbound method to a lightweight
stand-in rather than constructing a full scheduler.
"""
from types import SimpleNamespace
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import (
MoRIIOConnectorScheduler,
)
from vllm.v1.request import RequestStatus
_request_finished = MoRIIOConnectorScheduler.request_finished
def _producer(global_dp_rank: int, dp_size: int, dp_size_local: int):
"""Minimal prefill-side (producer) scheduler stand-in for request_finished."""
return SimpleNamespace(
is_producer=True,
_global_dp_rank=global_dp_rank,
engine_id="engine-abc",
host_ip="10.0.0.1",
handshake_port=6300,
side_notify_port=7300,
vllm_config=SimpleNamespace(
parallel_config=SimpleNamespace(
data_parallel_size=dp_size,
data_parallel_size_local=dp_size_local,
tensor_parallel_size=1,
)
),
)
def _req(
request_id: str, params: dict | None, status=RequestStatus.FINISHED_LENGTH_CAPPED
):
return SimpleNamespace(
request_id=request_id,
kv_transfer_params=params,
status=status,
)
def test_request_finished_returns_prefill_rank_for_propagation():
# 2P2D: global rank 11 lives on pod 1 (ranks 8..15), local size 8.
s = _producer(global_dp_rank=11, dp_size=16, dp_size_local=8)
req = _req("r1", {"do_remote_decode": True, "transfer_id": "t1"})
delay_free, params = _request_finished(s, req, block_ids=[])
# The prefill leg returns the rank IT actually ran on, marked authoritative,
# so the router forwards it to decode and the notify propagates (no hashing).
assert params is not None
assert params["remote_dp_rank"] == 11
assert params["remote_dp_rank_override"] is True
assert params["remote_dp_size"] == 16
assert params["remote_dp_size_local"] == 8
assert params["transfer_id"] == "t1"
# Empty block_ids => nothing deferred, blocks freed immediately.
assert delay_free is False
def test_request_finished_rank_tracks_the_actual_rank():
# A different rank must be echoed verbatim -- the value is the producer's
# own rank, never a recomputation of the request id.
s = _producer(global_dp_rank=4, dp_size=8, dp_size_local=8)
req = _req("r2", {"do_remote_decode": True, "transfer_id": "t2"})
_, params = _request_finished(s, req, block_ids=[])
assert params["remote_dp_rank"] == 4
assert params["remote_dp_size_local"] == 8
def test_request_finished_no_kv_params_is_noop():
s = _producer(global_dp_rank=0, dp_size=16, dp_size_local=8)
req = _req("r3", None)
delay_free, params = _request_finished(s, req, block_ids=[])
assert delay_free is False
assert params is None
@@ -296,6 +296,8 @@ def test_read_completion_sends_structured_release_with_consumer_tp_size():
worker._recving_transfers_callback_addr = {
"req": ("127.0.0.1", "7000", "tx-release")
}
# Transfer-timeout reaping state consulted by _pop_done_transfers.
worker._recving_transfers_start = {}
assert worker._pop_done_transfers() == {"tx-release"}
assert worker.moriio_wrapper.sent == [
@@ -0,0 +1,68 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for MoRIIOConnectorScheduler.{map,unmap}_request_id.
vLLM's input_processor mutates a request_id after the connector has stored it
(e.g. ``cmpl-abc-0`` -> ``cmpl-abc-0-956053a4``), so an exact-match unmap can
miss. These tests cover the stable-``transfer_id`` fallback that handles that
case (and the clean exact-match and miss paths).
The map/unmap methods touch only the two id<->id dicts, so we bind them to a
lightweight stand-in rather than constructing a full scheduler.
"""
from types import SimpleNamespace
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import (
MoRIIOConnectorScheduler,
)
_map = MoRIIOConnectorScheduler.map_request_id
_unmap = MoRIIOConnectorScheduler.unmap_request_id
def _sched():
return SimpleNamespace(
transfer_id_to_request_id={},
request_id_to_transfer_id={},
)
def test_exact_match_unmap_clears_both_tables():
s = _sched()
_map(s, "rid-1", "tid-1")
_unmap(s, "rid-1")
assert s.request_id_to_transfer_id == {}
assert s.transfer_id_to_request_id == {}
def test_unmap_via_transfer_id_after_request_id_mutation():
s = _sched()
# Connector stores the ORIGINAL request_id at map time.
_map(s, "cmpl-abc-0", "tid-42")
# input_processor later mutates the rid; exact match would miss, so the
# stable transfer_id is used to find and clear the original entry.
_unmap(s, "cmpl-abc-0-956053a4", transfer_id="tid-42")
assert s.request_id_to_transfer_id == {}
assert s.transfer_id_to_request_id == {}
def test_unmap_miss_is_noop_and_does_not_raise():
s = _sched()
_map(s, "rid-1", "tid-1")
# Unknown rid and unknown/None transfer_id: must not raise, must not touch
# the existing (unrelated) mapping.
_unmap(s, "rid-unknown", transfer_id=None)
_unmap(s, "rid-unknown", transfer_id="tid-unknown")
assert s.request_id_to_transfer_id == {"rid-1": "tid-1"}
assert s.transfer_id_to_request_id == {"tid-1": "rid-1"}
def test_exact_match_preferred_over_transfer_id_fallback():
s = _sched()
_map(s, "rid-1", "tid-1")
# Exact rid is present, so it is removed directly even if a transfer_id is
# also supplied; no stale entries left behind.
_unmap(s, "rid-1", transfer_id="tid-1")
assert s.request_id_to_transfer_id == {}
assert s.transfer_id_to_request_id == {}