[ROCm][P/D] Support MoRIIO heterogeneous TP fan-in (#46332)
Signed-off-by: Tan Pin Siang <tanpinsiang@gmail.com> Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com> Co-authored-by: Hongxia Yang <hongxia.yang@amd.com> Co-authored-by: Jun Kang Chow <junkangchow@gmail.com> Co-authored-by: Chun Fang <chun.fang@amd.com> Co-authored-by: TianDi101 <ditian12@amd.com> Co-authored-by: functionstackx <47992694+functionstackx@users.noreply.github.com> Co-authored-by: tjtanaa <tunjian.tan@embeddedllm.com>
This commit is contained in:
co-authored by
vllmellm
Hongxia Yang
Jun Kang Chow
Chun Fang
TianDi101
functionstackx
tjtanaa
parent
83fa302ca4
commit
d32575a2d2
@@ -195,7 +195,14 @@ class FakeMoRIIOWrapper:
|
||||
def _handle_completion_message(self, msg: str):
|
||||
pass
|
||||
|
||||
def send_notify(self, req_ids, remote_ip, remote_port, message_type=None):
|
||||
def send_notify(
|
||||
self,
|
||||
req_ids,
|
||||
remote_ip,
|
||||
remote_port,
|
||||
message_type=None,
|
||||
message_fields=None,
|
||||
):
|
||||
pass
|
||||
|
||||
def pop_finished_req_ids(self):
|
||||
|
||||
@@ -36,6 +36,7 @@ msgpack = importlib.import_module("msgpack")
|
||||
|
||||
ROLE = moriio_common.ROLE
|
||||
MoRIIOError = moriio_common.MoRIIOError
|
||||
MoRIIOTransferAck = moriio_common.MoRIIOTransferAck
|
||||
RemoteAllocInfo = moriio_common.RemoteAllocInfo
|
||||
WriteTask = moriio_common.WriteTask
|
||||
set_role = moriio_common.set_role
|
||||
@@ -440,9 +441,16 @@ def test_write_completion_notifies_once_after_all_sealed_writes_finish():
|
||||
def _mark_transfer_terminal_locked(self, transfer_id):
|
||||
self._terminal_transfer_ids[transfer_id] = None
|
||||
|
||||
def send_notify(self, transfer_id, remote_ip, remote_port, message_type=None):
|
||||
def send_notify(
|
||||
self,
|
||||
transfer_id,
|
||||
remote_ip,
|
||||
remote_port,
|
||||
message_type=None,
|
||||
message_fields=None,
|
||||
):
|
||||
self.notifications.append(
|
||||
(transfer_id, remote_ip, remote_port, message_type)
|
||||
(transfer_id, remote_ip, remote_port, message_type, message_fields)
|
||||
)
|
||||
|
||||
wrapper = FakeWrapper()
|
||||
@@ -464,8 +472,8 @@ def test_write_completion_notifies_once_after_all_sealed_writes_finish():
|
||||
writer._mark_write_done("xfer", request_info)
|
||||
writer._finalize_if_complete("xfer", request_info)
|
||||
|
||||
assert wrapper.notifications == [("xfer", "127.0.0.1", 7002, "write_done")]
|
||||
assert wrapper.done_req_ids == ["xfer"]
|
||||
assert wrapper.notifications == [("xfer", "127.0.0.1", 7002, "write_done", None)]
|
||||
assert wrapper.done_req_ids == [MoRIIOTransferAck("xfer")]
|
||||
assert wrapper.done_remote_allocate_req_dict == {}
|
||||
assert wrapper.wait_count == 1
|
||||
assert wrapper.waited_statuses == [["status-a", "status-b"]]
|
||||
@@ -509,7 +517,7 @@ def test_write_failure_marks_terminal_and_clears_scheduled_state():
|
||||
|
||||
writer._mark_request_done("xfer")
|
||||
|
||||
assert wrapper.done_req_ids == ["xfer"]
|
||||
assert wrapper.done_req_ids == [MoRIIOTransferAck("xfer")]
|
||||
assert wrapper.done_remote_allocate_req_dict == {}
|
||||
assert wrapper._is_transfer_terminal_locked("xfer")
|
||||
assert "xfer" not in writer._scheduled_writes
|
||||
@@ -581,9 +589,21 @@ def test_late_remote_blocks_message_is_ignored_after_transfer_done():
|
||||
pytest.param(
|
||||
ROLE.PRODUCER,
|
||||
msgpack.dumps({"type": "release", "transfer_id": "xfer"}),
|
||||
"release",
|
||||
MoRIIOTransferAck("xfer"),
|
||||
id="release",
|
||||
),
|
||||
pytest.param(
|
||||
ROLE.PRODUCER,
|
||||
msgpack.dumps(
|
||||
{
|
||||
"type": "release",
|
||||
"transfer_id": "xfer",
|
||||
"consumer_tp_size": 8,
|
||||
}
|
||||
),
|
||||
MoRIIOTransferAck("xfer", 8),
|
||||
id="release-consumer-tp-size",
|
||||
),
|
||||
pytest.param(None, b"xfer", "plain", id="plain-string"),
|
||||
],
|
||||
)
|
||||
@@ -603,11 +623,11 @@ def test_moriio_wrapper_routes_valid_messages(role, payload, expected):
|
||||
assert request_info.decode_dp_rank == 3
|
||||
elif expected == "write_done":
|
||||
assert wrapper.done_write_cache_req_ids == ["xfer"]
|
||||
elif expected == "release":
|
||||
assert wrapper.done_req_ids == ["xfer"]
|
||||
assert wrapper._is_transfer_terminal_locked("xfer")
|
||||
else:
|
||||
elif expected == "plain":
|
||||
assert completions == ["xfer"]
|
||||
else:
|
||||
assert wrapper.done_req_ids == [expected]
|
||||
assert wrapper._is_transfer_terminal_locked("xfer")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
@@ -0,0 +1,310 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import threading
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import (
|
||||
MoRIIOMode,
|
||||
MoRIIOTransferAck,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import (
|
||||
MoRIIOConnectorWorker,
|
||||
get_moriio_expected_ack_count,
|
||||
get_moriio_remote_tp_rank,
|
||||
resolve_moriio_transfer_ack,
|
||||
validate_moriio_heterogeneous_tp_kv_heads,
|
||||
)
|
||||
|
||||
|
||||
def test_remote_tp_rank_same_tp_maps_to_self():
|
||||
assert [get_moriio_remote_tp_rank(rank, 4, 4) for rank in range(4)] == [
|
||||
0,
|
||||
1,
|
||||
2,
|
||||
3,
|
||||
]
|
||||
|
||||
|
||||
def test_remote_tp_rank_p4_d8_floor_maps_decode_to_prefill():
|
||||
assert [get_moriio_remote_tp_rank(rank, 8, 4) for rank in range(8)] == [
|
||||
0,
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
2,
|
||||
2,
|
||||
3,
|
||||
3,
|
||||
]
|
||||
|
||||
|
||||
def test_remote_tp_rank_p8_d4_maps_to_first_prefill_rank_per_pair():
|
||||
assert [get_moriio_remote_tp_rank(rank, 4, 8) for rank in range(4)] == [
|
||||
0,
|
||||
2,
|
||||
4,
|
||||
6,
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("local_tp_rank", "local_tp_size", "remote_tp_size"),
|
||||
[
|
||||
(0, 6, 4),
|
||||
(0, 4, 6),
|
||||
],
|
||||
)
|
||||
def test_remote_tp_rank_invalid_non_multiple_tp_raises(
|
||||
local_tp_rank: int, local_tp_size: int, remote_tp_size: int
|
||||
):
|
||||
with pytest.raises(ValueError, match="multiple"):
|
||||
get_moriio_remote_tp_rank(local_tp_rank, local_tp_size, remote_tp_size)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("local_tp_size", "remote_tp_size", "total_num_kv_heads"),
|
||||
[
|
||||
(4, 4, 8),
|
||||
(8, 4, 4),
|
||||
(4, 8, 4),
|
||||
],
|
||||
)
|
||||
def test_heterogeneous_tp_head_guard_allows_supported_layouts(
|
||||
local_tp_size: int, remote_tp_size: int, total_num_kv_heads: int
|
||||
):
|
||||
validate_moriio_heterogeneous_tp_kv_heads(
|
||||
local_tp_size,
|
||||
remote_tp_size,
|
||||
total_num_kv_heads,
|
||||
is_mla=False,
|
||||
)
|
||||
|
||||
|
||||
def test_heterogeneous_tp_head_guard_allows_mla_layouts():
|
||||
validate_moriio_heterogeneous_tp_kv_heads(
|
||||
local_tp_size=2,
|
||||
remote_tp_size=4,
|
||||
total_num_kv_heads=4,
|
||||
is_mla=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("local_tp_size", "remote_tp_size", "total_num_kv_heads"),
|
||||
[
|
||||
(4, 2, 4),
|
||||
(2, 4, 4),
|
||||
],
|
||||
)
|
||||
def test_heterogeneous_tp_head_guard_rejects_split_kv_heads(
|
||||
local_tp_size: int, remote_tp_size: int, total_num_kv_heads: int
|
||||
):
|
||||
with pytest.raises(NotImplementedError, match="replicated KV heads"):
|
||||
validate_moriio_heterogeneous_tp_kv_heads(
|
||||
local_tp_size,
|
||||
remote_tp_size,
|
||||
total_num_kv_heads,
|
||||
is_mla=False,
|
||||
)
|
||||
|
||||
|
||||
def test_expected_ack_count_for_homogeneous_or_smaller_consumer_tp_is_one():
|
||||
assert get_moriio_expected_ack_count(4, 4) == 1
|
||||
assert get_moriio_expected_ack_count(8, 4) == 1
|
||||
|
||||
|
||||
def test_expected_ack_count_for_decode_fan_in():
|
||||
assert get_moriio_expected_ack_count(4, 8) == 2
|
||||
|
||||
|
||||
def test_expected_ack_count_rejects_non_multiple_fan_in():
|
||||
with pytest.raises(ValueError, match="multiple"):
|
||||
get_moriio_expected_ack_count(4, 6)
|
||||
|
||||
|
||||
def test_plain_string_ack_is_backward_compatible_single_ack():
|
||||
notification_counts: dict[str, int] = {}
|
||||
completed_transfer_ids: set[str] = set()
|
||||
|
||||
assert (
|
||||
resolve_moriio_transfer_ack(
|
||||
"tx-plain",
|
||||
producer_tp_size=4,
|
||||
live_transfer_ids={"tx-plain"},
|
||||
notification_counts=notification_counts,
|
||||
completed_transfer_ids=completed_transfer_ids,
|
||||
)
|
||||
== "tx-plain"
|
||||
)
|
||||
assert notification_counts == {}
|
||||
assert completed_transfer_ids == {"tx-plain"}
|
||||
|
||||
|
||||
def test_structured_release_ack_waits_for_all_expected_acks():
|
||||
ack = MoRIIOTransferAck("tx-fanin", consumer_tp_size=8)
|
||||
notification_counts: dict[str, int] = {}
|
||||
completed_transfer_ids: set[str] = set()
|
||||
|
||||
assert (
|
||||
resolve_moriio_transfer_ack(
|
||||
ack,
|
||||
producer_tp_size=4,
|
||||
live_transfer_ids={"tx-fanin"},
|
||||
notification_counts=notification_counts,
|
||||
completed_transfer_ids=completed_transfer_ids,
|
||||
)
|
||||
is None
|
||||
)
|
||||
assert notification_counts == {"tx-fanin": 1}
|
||||
assert completed_transfer_ids == set()
|
||||
|
||||
assert (
|
||||
resolve_moriio_transfer_ack(
|
||||
ack,
|
||||
producer_tp_size=4,
|
||||
live_transfer_ids={"tx-fanin"},
|
||||
notification_counts=notification_counts,
|
||||
completed_transfer_ids=completed_transfer_ids,
|
||||
)
|
||||
== "tx-fanin"
|
||||
)
|
||||
assert notification_counts == {}
|
||||
assert completed_transfer_ids == {"tx-fanin"}
|
||||
|
||||
|
||||
def test_duplicate_ack_after_completion_does_not_resolve_twice():
|
||||
ack = MoRIIOTransferAck("tx-dup", consumer_tp_size=8)
|
||||
notification_counts: dict[str, int] = {}
|
||||
completed_transfer_ids: set[str] = set()
|
||||
|
||||
assert (
|
||||
resolve_moriio_transfer_ack(
|
||||
ack,
|
||||
producer_tp_size=4,
|
||||
live_transfer_ids={"tx-dup"},
|
||||
notification_counts=notification_counts,
|
||||
completed_transfer_ids=completed_transfer_ids,
|
||||
)
|
||||
is None
|
||||
)
|
||||
assert (
|
||||
resolve_moriio_transfer_ack(
|
||||
ack,
|
||||
producer_tp_size=4,
|
||||
live_transfer_ids={"tx-dup"},
|
||||
notification_counts=notification_counts,
|
||||
completed_transfer_ids=completed_transfer_ids,
|
||||
)
|
||||
== "tx-dup"
|
||||
)
|
||||
assert (
|
||||
resolve_moriio_transfer_ack(
|
||||
ack,
|
||||
producer_tp_size=4,
|
||||
live_transfer_ids={"tx-dup"},
|
||||
notification_counts=notification_counts,
|
||||
completed_transfer_ids=completed_transfer_ids,
|
||||
)
|
||||
is None
|
||||
)
|
||||
assert notification_counts == {}
|
||||
assert completed_transfer_ids == {"tx-dup"}
|
||||
|
||||
|
||||
def test_ack_for_non_live_transfer_is_ignored():
|
||||
notification_counts: dict[str, int] = {}
|
||||
completed_transfer_ids: set[str] = set()
|
||||
|
||||
assert (
|
||||
resolve_moriio_transfer_ack(
|
||||
MoRIIOTransferAck("tx-stale", consumer_tp_size=8),
|
||||
producer_tp_size=4,
|
||||
live_transfer_ids={"tx-live"},
|
||||
notification_counts=notification_counts,
|
||||
completed_transfer_ids=completed_transfer_ids,
|
||||
)
|
||||
is None
|
||||
)
|
||||
assert notification_counts == {}
|
||||
assert completed_transfer_ids == set()
|
||||
|
||||
|
||||
def test_worker_get_finished_counts_structured_release_fan_in():
|
||||
class FakeWrapper:
|
||||
def __init__(self):
|
||||
self.batches = [
|
||||
[MoRIIOTransferAck("tx-fanin", consumer_tp_size=8)],
|
||||
[MoRIIOTransferAck("tx-fanin", consumer_tp_size=8)],
|
||||
]
|
||||
|
||||
def pop_finished_req_ids(self):
|
||||
return self.batches.pop(0)
|
||||
|
||||
def shutdown(self):
|
||||
pass
|
||||
|
||||
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
|
||||
worker.is_producer = True
|
||||
worker.mode = MoRIIOMode.READ
|
||||
worker.world_size = 4
|
||||
worker.moriio_wrapper = FakeWrapper()
|
||||
worker.transfer_id_to_request_id = {"tx-fanin": "req-fanin"}
|
||||
worker._consumer_notification_counts = {}
|
||||
worker._completed_consumer_notifications = set()
|
||||
|
||||
assert worker.get_finished() == (set(), set())
|
||||
assert worker._consumer_notification_counts == {"tx-fanin": 1}
|
||||
|
||||
assert worker.get_finished() == ({"req-fanin"}, set())
|
||||
assert worker._consumer_notification_counts == {}
|
||||
assert worker._completed_consumer_notifications == {"tx-fanin"}
|
||||
|
||||
|
||||
def test_read_completion_sends_structured_release_with_consumer_tp_size():
|
||||
class DoneStatus:
|
||||
def Succeeded(self):
|
||||
return True
|
||||
|
||||
def Failed(self):
|
||||
return False
|
||||
|
||||
class FakeWrapper:
|
||||
def __init__(self):
|
||||
self.lock = threading.Lock()
|
||||
self.sent = []
|
||||
|
||||
def send_notify(
|
||||
self,
|
||||
transfer_id,
|
||||
host,
|
||||
port,
|
||||
message_type=None,
|
||||
message_fields=None,
|
||||
):
|
||||
self.sent.append((transfer_id, host, port, message_type, message_fields))
|
||||
|
||||
def shutdown(self):
|
||||
pass
|
||||
|
||||
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
|
||||
worker.world_size = 8
|
||||
worker.moriio_wrapper = FakeWrapper()
|
||||
worker._recving_transfers = {"req": [DoneStatus()]}
|
||||
worker._recving_transfers_callback_addr = {
|
||||
"req": ("127.0.0.1", "7000", "tx-release")
|
||||
}
|
||||
|
||||
assert worker._pop_done_transfers() == {"tx-release"}
|
||||
assert worker.moriio_wrapper.sent == [
|
||||
(
|
||||
"tx-release",
|
||||
"127.0.0.1",
|
||||
"7000",
|
||||
"release",
|
||||
{"consumer_tp_size": 8},
|
||||
)
|
||||
]
|
||||
assert worker._recving_transfers == {}
|
||||
assert worker._recving_transfers_callback_addr == {}
|
||||
Reference in New Issue
Block a user