llmd+vllm+mori-ep(inter node wide-ep)+mori-io(write) for 2p2d with dp=ep=16 tp=1 (#45043)
Signed-off-by: Shiksha Patel <shikpate@amd.com> Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Douglas Lehr
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
c810937573
commit
f42761204f
@@ -508,9 +508,15 @@ def test_send_transfer_release_sends_structured_release_message():
|
||||
scheduler._send_transfer_release("xfer-7", "127.0.0.1", 7000)
|
||||
|
||||
payload = sock.send.call_args.args[0]
|
||||
# WRITE-mode release advertises the consumer (decode) TP size so the prefill
|
||||
# side counts the right number of ACKs via get_moriio_expected_ack_count,
|
||||
# mirroring the READ-mode release in _pop_done_transfers (see
|
||||
# test_read_completion_sends_structured_release_with_consumer_tp_size). The
|
||||
# fixture's tp_size is 2, so consumer_tp_size == 2.
|
||||
assert msgspec.msgpack.decode(payload) == {
|
||||
"type": "release",
|
||||
"transfer_id": "xfer-7",
|
||||
"consumer_tp_size": 2,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for MoRIIO router-authoritative DP-rank routing.
|
||||
|
||||
These lock in the redesign that addresses the PR #45043 review feedback
|
||||
(njhill): the connector must NOT self-derive a DP rank, and the prefill leg
|
||||
must *return* the rank it actually ran on via ``request_finished`` so the
|
||||
decode->prefill notify is routed by pure propagation (READ / serial WRITE),
|
||||
not by an independent hash on each side.
|
||||
|
||||
Following test_moriio_unmap.py, we bind the unbound method to a lightweight
|
||||
stand-in rather than constructing a full scheduler.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import (
|
||||
MoRIIOConnectorScheduler,
|
||||
)
|
||||
from vllm.v1.request import RequestStatus
|
||||
|
||||
_request_finished = MoRIIOConnectorScheduler.request_finished
|
||||
|
||||
|
||||
def _producer(global_dp_rank: int, dp_size: int, dp_size_local: int):
|
||||
"""Minimal prefill-side (producer) scheduler stand-in for request_finished."""
|
||||
return SimpleNamespace(
|
||||
is_producer=True,
|
||||
_global_dp_rank=global_dp_rank,
|
||||
engine_id="engine-abc",
|
||||
host_ip="10.0.0.1",
|
||||
handshake_port=6300,
|
||||
side_notify_port=7300,
|
||||
vllm_config=SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(
|
||||
data_parallel_size=dp_size,
|
||||
data_parallel_size_local=dp_size_local,
|
||||
tensor_parallel_size=1,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _req(
|
||||
request_id: str, params: dict | None, status=RequestStatus.FINISHED_LENGTH_CAPPED
|
||||
):
|
||||
return SimpleNamespace(
|
||||
request_id=request_id,
|
||||
kv_transfer_params=params,
|
||||
status=status,
|
||||
)
|
||||
|
||||
|
||||
def test_request_finished_returns_prefill_rank_for_propagation():
|
||||
# 2P2D: global rank 11 lives on pod 1 (ranks 8..15), local size 8.
|
||||
s = _producer(global_dp_rank=11, dp_size=16, dp_size_local=8)
|
||||
req = _req("r1", {"do_remote_decode": True, "transfer_id": "t1"})
|
||||
|
||||
delay_free, params = _request_finished(s, req, block_ids=[])
|
||||
|
||||
# The prefill leg returns the rank IT actually ran on, marked authoritative,
|
||||
# so the router forwards it to decode and the notify propagates (no hashing).
|
||||
assert params is not None
|
||||
assert params["remote_dp_rank"] == 11
|
||||
assert params["remote_dp_rank_override"] is True
|
||||
assert params["remote_dp_size"] == 16
|
||||
assert params["remote_dp_size_local"] == 8
|
||||
assert params["transfer_id"] == "t1"
|
||||
# Empty block_ids => nothing deferred, blocks freed immediately.
|
||||
assert delay_free is False
|
||||
|
||||
|
||||
def test_request_finished_rank_tracks_the_actual_rank():
|
||||
# A different rank must be echoed verbatim -- the value is the producer's
|
||||
# own rank, never a recomputation of the request id.
|
||||
s = _producer(global_dp_rank=4, dp_size=8, dp_size_local=8)
|
||||
req = _req("r2", {"do_remote_decode": True, "transfer_id": "t2"})
|
||||
|
||||
_, params = _request_finished(s, req, block_ids=[])
|
||||
|
||||
assert params["remote_dp_rank"] == 4
|
||||
assert params["remote_dp_size_local"] == 8
|
||||
|
||||
|
||||
def test_request_finished_no_kv_params_is_noop():
|
||||
s = _producer(global_dp_rank=0, dp_size=16, dp_size_local=8)
|
||||
req = _req("r3", None)
|
||||
|
||||
delay_free, params = _request_finished(s, req, block_ids=[])
|
||||
|
||||
assert delay_free is False
|
||||
assert params is None
|
||||
@@ -296,6 +296,8 @@ def test_read_completion_sends_structured_release_with_consumer_tp_size():
|
||||
worker._recving_transfers_callback_addr = {
|
||||
"req": ("127.0.0.1", "7000", "tx-release")
|
||||
}
|
||||
# Transfer-timeout reaping state consulted by _pop_done_transfers.
|
||||
worker._recving_transfers_start = {}
|
||||
|
||||
assert worker._pop_done_transfers() == {"tx-release"}
|
||||
assert worker.moriio_wrapper.sent == [
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for MoRIIOConnectorScheduler.{map,unmap}_request_id.
|
||||
|
||||
vLLM's input_processor mutates a request_id after the connector has stored it
|
||||
(e.g. ``cmpl-abc-0`` -> ``cmpl-abc-0-956053a4``), so an exact-match unmap can
|
||||
miss. These tests cover the stable-``transfer_id`` fallback that handles that
|
||||
case (and the clean exact-match and miss paths).
|
||||
|
||||
The map/unmap methods touch only the two id<->id dicts, so we bind them to a
|
||||
lightweight stand-in rather than constructing a full scheduler.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import (
|
||||
MoRIIOConnectorScheduler,
|
||||
)
|
||||
|
||||
_map = MoRIIOConnectorScheduler.map_request_id
|
||||
_unmap = MoRIIOConnectorScheduler.unmap_request_id
|
||||
|
||||
|
||||
def _sched():
|
||||
return SimpleNamespace(
|
||||
transfer_id_to_request_id={},
|
||||
request_id_to_transfer_id={},
|
||||
)
|
||||
|
||||
|
||||
def test_exact_match_unmap_clears_both_tables():
|
||||
s = _sched()
|
||||
_map(s, "rid-1", "tid-1")
|
||||
_unmap(s, "rid-1")
|
||||
assert s.request_id_to_transfer_id == {}
|
||||
assert s.transfer_id_to_request_id == {}
|
||||
|
||||
|
||||
def test_unmap_via_transfer_id_after_request_id_mutation():
|
||||
s = _sched()
|
||||
# Connector stores the ORIGINAL request_id at map time.
|
||||
_map(s, "cmpl-abc-0", "tid-42")
|
||||
# input_processor later mutates the rid; exact match would miss, so the
|
||||
# stable transfer_id is used to find and clear the original entry.
|
||||
_unmap(s, "cmpl-abc-0-956053a4", transfer_id="tid-42")
|
||||
assert s.request_id_to_transfer_id == {}
|
||||
assert s.transfer_id_to_request_id == {}
|
||||
|
||||
|
||||
def test_unmap_miss_is_noop_and_does_not_raise():
|
||||
s = _sched()
|
||||
_map(s, "rid-1", "tid-1")
|
||||
# Unknown rid and unknown/None transfer_id: must not raise, must not touch
|
||||
# the existing (unrelated) mapping.
|
||||
_unmap(s, "rid-unknown", transfer_id=None)
|
||||
_unmap(s, "rid-unknown", transfer_id="tid-unknown")
|
||||
assert s.request_id_to_transfer_id == {"rid-1": "tid-1"}
|
||||
assert s.transfer_id_to_request_id == {"tid-1": "rid-1"}
|
||||
|
||||
|
||||
def test_exact_match_preferred_over_transfer_id_fallback():
|
||||
s = _sched()
|
||||
_map(s, "rid-1", "tid-1")
|
||||
# Exact rid is present, so it is removed directly even if a transfer_id is
|
||||
# also supplied; no stale entries left behind.
|
||||
_unmap(s, "rid-1", transfer_id="tid-1")
|
||||
assert s.request_id_to_transfer_id == {}
|
||||
assert s.transfer_id_to_request_id == {}
|
||||
Reference in New Issue
Block a user