[ROCm][P/D] Support MoRIIO heterogeneous TP fan-in (#46332)
Signed-off-by: Tan Pin Siang <tanpinsiang@gmail.com> Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com> Co-authored-by: Hongxia Yang <hongxia.yang@amd.com> Co-authored-by: Jun Kang Chow <junkangchow@gmail.com> Co-authored-by: Chun Fang <chun.fang@amd.com> Co-authored-by: TianDi101 <ditian12@amd.com> Co-authored-by: functionstackx <47992694+functionstackx@users.noreply.github.com> Co-authored-by: tjtanaa <tunjian.tan@embeddedllm.com>
This commit is contained in:
co-authored by
vllmellm
Hongxia Yang
Jun Kang Chow
Chun Fang
TianDi101
functionstackx
tjtanaa
parent
83fa302ca4
commit
d32575a2d2
@@ -195,7 +195,14 @@ class FakeMoRIIOWrapper:
|
||||
def _handle_completion_message(self, msg: str):
|
||||
pass
|
||||
|
||||
def send_notify(self, req_ids, remote_ip, remote_port, message_type=None):
|
||||
def send_notify(
|
||||
self,
|
||||
req_ids,
|
||||
remote_ip,
|
||||
remote_port,
|
||||
message_type=None,
|
||||
message_fields=None,
|
||||
):
|
||||
pass
|
||||
|
||||
def pop_finished_req_ids(self):
|
||||
|
||||
@@ -36,6 +36,7 @@ msgpack = importlib.import_module("msgpack")
|
||||
|
||||
ROLE = moriio_common.ROLE
|
||||
MoRIIOError = moriio_common.MoRIIOError
|
||||
MoRIIOTransferAck = moriio_common.MoRIIOTransferAck
|
||||
RemoteAllocInfo = moriio_common.RemoteAllocInfo
|
||||
WriteTask = moriio_common.WriteTask
|
||||
set_role = moriio_common.set_role
|
||||
@@ -440,9 +441,16 @@ def test_write_completion_notifies_once_after_all_sealed_writes_finish():
|
||||
def _mark_transfer_terminal_locked(self, transfer_id):
|
||||
self._terminal_transfer_ids[transfer_id] = None
|
||||
|
||||
def send_notify(self, transfer_id, remote_ip, remote_port, message_type=None):
|
||||
def send_notify(
|
||||
self,
|
||||
transfer_id,
|
||||
remote_ip,
|
||||
remote_port,
|
||||
message_type=None,
|
||||
message_fields=None,
|
||||
):
|
||||
self.notifications.append(
|
||||
(transfer_id, remote_ip, remote_port, message_type)
|
||||
(transfer_id, remote_ip, remote_port, message_type, message_fields)
|
||||
)
|
||||
|
||||
wrapper = FakeWrapper()
|
||||
@@ -464,8 +472,8 @@ def test_write_completion_notifies_once_after_all_sealed_writes_finish():
|
||||
writer._mark_write_done("xfer", request_info)
|
||||
writer._finalize_if_complete("xfer", request_info)
|
||||
|
||||
assert wrapper.notifications == [("xfer", "127.0.0.1", 7002, "write_done")]
|
||||
assert wrapper.done_req_ids == ["xfer"]
|
||||
assert wrapper.notifications == [("xfer", "127.0.0.1", 7002, "write_done", None)]
|
||||
assert wrapper.done_req_ids == [MoRIIOTransferAck("xfer")]
|
||||
assert wrapper.done_remote_allocate_req_dict == {}
|
||||
assert wrapper.wait_count == 1
|
||||
assert wrapper.waited_statuses == [["status-a", "status-b"]]
|
||||
@@ -509,7 +517,7 @@ def test_write_failure_marks_terminal_and_clears_scheduled_state():
|
||||
|
||||
writer._mark_request_done("xfer")
|
||||
|
||||
assert wrapper.done_req_ids == ["xfer"]
|
||||
assert wrapper.done_req_ids == [MoRIIOTransferAck("xfer")]
|
||||
assert wrapper.done_remote_allocate_req_dict == {}
|
||||
assert wrapper._is_transfer_terminal_locked("xfer")
|
||||
assert "xfer" not in writer._scheduled_writes
|
||||
@@ -581,9 +589,21 @@ def test_late_remote_blocks_message_is_ignored_after_transfer_done():
|
||||
pytest.param(
|
||||
ROLE.PRODUCER,
|
||||
msgpack.dumps({"type": "release", "transfer_id": "xfer"}),
|
||||
"release",
|
||||
MoRIIOTransferAck("xfer"),
|
||||
id="release",
|
||||
),
|
||||
pytest.param(
|
||||
ROLE.PRODUCER,
|
||||
msgpack.dumps(
|
||||
{
|
||||
"type": "release",
|
||||
"transfer_id": "xfer",
|
||||
"consumer_tp_size": 8,
|
||||
}
|
||||
),
|
||||
MoRIIOTransferAck("xfer", 8),
|
||||
id="release-consumer-tp-size",
|
||||
),
|
||||
pytest.param(None, b"xfer", "plain", id="plain-string"),
|
||||
],
|
||||
)
|
||||
@@ -603,11 +623,11 @@ def test_moriio_wrapper_routes_valid_messages(role, payload, expected):
|
||||
assert request_info.decode_dp_rank == 3
|
||||
elif expected == "write_done":
|
||||
assert wrapper.done_write_cache_req_ids == ["xfer"]
|
||||
elif expected == "release":
|
||||
assert wrapper.done_req_ids == ["xfer"]
|
||||
assert wrapper._is_transfer_terminal_locked("xfer")
|
||||
else:
|
||||
elif expected == "plain":
|
||||
assert completions == ["xfer"]
|
||||
else:
|
||||
assert wrapper.done_req_ids == [expected]
|
||||
assert wrapper._is_transfer_terminal_locked("xfer")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
@@ -0,0 +1,310 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import threading
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import (
|
||||
MoRIIOMode,
|
||||
MoRIIOTransferAck,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import (
|
||||
MoRIIOConnectorWorker,
|
||||
get_moriio_expected_ack_count,
|
||||
get_moriio_remote_tp_rank,
|
||||
resolve_moriio_transfer_ack,
|
||||
validate_moriio_heterogeneous_tp_kv_heads,
|
||||
)
|
||||
|
||||
|
||||
def test_remote_tp_rank_same_tp_maps_to_self():
|
||||
assert [get_moriio_remote_tp_rank(rank, 4, 4) for rank in range(4)] == [
|
||||
0,
|
||||
1,
|
||||
2,
|
||||
3,
|
||||
]
|
||||
|
||||
|
||||
def test_remote_tp_rank_p4_d8_floor_maps_decode_to_prefill():
|
||||
assert [get_moriio_remote_tp_rank(rank, 8, 4) for rank in range(8)] == [
|
||||
0,
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
2,
|
||||
2,
|
||||
3,
|
||||
3,
|
||||
]
|
||||
|
||||
|
||||
def test_remote_tp_rank_p8_d4_maps_to_first_prefill_rank_per_pair():
|
||||
assert [get_moriio_remote_tp_rank(rank, 4, 8) for rank in range(4)] == [
|
||||
0,
|
||||
2,
|
||||
4,
|
||||
6,
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("local_tp_rank", "local_tp_size", "remote_tp_size"),
|
||||
[
|
||||
(0, 6, 4),
|
||||
(0, 4, 6),
|
||||
],
|
||||
)
|
||||
def test_remote_tp_rank_invalid_non_multiple_tp_raises(
|
||||
local_tp_rank: int, local_tp_size: int, remote_tp_size: int
|
||||
):
|
||||
with pytest.raises(ValueError, match="multiple"):
|
||||
get_moriio_remote_tp_rank(local_tp_rank, local_tp_size, remote_tp_size)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("local_tp_size", "remote_tp_size", "total_num_kv_heads"),
|
||||
[
|
||||
(4, 4, 8),
|
||||
(8, 4, 4),
|
||||
(4, 8, 4),
|
||||
],
|
||||
)
|
||||
def test_heterogeneous_tp_head_guard_allows_supported_layouts(
|
||||
local_tp_size: int, remote_tp_size: int, total_num_kv_heads: int
|
||||
):
|
||||
validate_moriio_heterogeneous_tp_kv_heads(
|
||||
local_tp_size,
|
||||
remote_tp_size,
|
||||
total_num_kv_heads,
|
||||
is_mla=False,
|
||||
)
|
||||
|
||||
|
||||
def test_heterogeneous_tp_head_guard_allows_mla_layouts():
|
||||
validate_moriio_heterogeneous_tp_kv_heads(
|
||||
local_tp_size=2,
|
||||
remote_tp_size=4,
|
||||
total_num_kv_heads=4,
|
||||
is_mla=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("local_tp_size", "remote_tp_size", "total_num_kv_heads"),
|
||||
[
|
||||
(4, 2, 4),
|
||||
(2, 4, 4),
|
||||
],
|
||||
)
|
||||
def test_heterogeneous_tp_head_guard_rejects_split_kv_heads(
|
||||
local_tp_size: int, remote_tp_size: int, total_num_kv_heads: int
|
||||
):
|
||||
with pytest.raises(NotImplementedError, match="replicated KV heads"):
|
||||
validate_moriio_heterogeneous_tp_kv_heads(
|
||||
local_tp_size,
|
||||
remote_tp_size,
|
||||
total_num_kv_heads,
|
||||
is_mla=False,
|
||||
)
|
||||
|
||||
|
||||
def test_expected_ack_count_for_homogeneous_or_smaller_consumer_tp_is_one():
|
||||
assert get_moriio_expected_ack_count(4, 4) == 1
|
||||
assert get_moriio_expected_ack_count(8, 4) == 1
|
||||
|
||||
|
||||
def test_expected_ack_count_for_decode_fan_in():
|
||||
assert get_moriio_expected_ack_count(4, 8) == 2
|
||||
|
||||
|
||||
def test_expected_ack_count_rejects_non_multiple_fan_in():
|
||||
with pytest.raises(ValueError, match="multiple"):
|
||||
get_moriio_expected_ack_count(4, 6)
|
||||
|
||||
|
||||
def test_plain_string_ack_is_backward_compatible_single_ack():
|
||||
notification_counts: dict[str, int] = {}
|
||||
completed_transfer_ids: set[str] = set()
|
||||
|
||||
assert (
|
||||
resolve_moriio_transfer_ack(
|
||||
"tx-plain",
|
||||
producer_tp_size=4,
|
||||
live_transfer_ids={"tx-plain"},
|
||||
notification_counts=notification_counts,
|
||||
completed_transfer_ids=completed_transfer_ids,
|
||||
)
|
||||
== "tx-plain"
|
||||
)
|
||||
assert notification_counts == {}
|
||||
assert completed_transfer_ids == {"tx-plain"}
|
||||
|
||||
|
||||
def test_structured_release_ack_waits_for_all_expected_acks():
|
||||
ack = MoRIIOTransferAck("tx-fanin", consumer_tp_size=8)
|
||||
notification_counts: dict[str, int] = {}
|
||||
completed_transfer_ids: set[str] = set()
|
||||
|
||||
assert (
|
||||
resolve_moriio_transfer_ack(
|
||||
ack,
|
||||
producer_tp_size=4,
|
||||
live_transfer_ids={"tx-fanin"},
|
||||
notification_counts=notification_counts,
|
||||
completed_transfer_ids=completed_transfer_ids,
|
||||
)
|
||||
is None
|
||||
)
|
||||
assert notification_counts == {"tx-fanin": 1}
|
||||
assert completed_transfer_ids == set()
|
||||
|
||||
assert (
|
||||
resolve_moriio_transfer_ack(
|
||||
ack,
|
||||
producer_tp_size=4,
|
||||
live_transfer_ids={"tx-fanin"},
|
||||
notification_counts=notification_counts,
|
||||
completed_transfer_ids=completed_transfer_ids,
|
||||
)
|
||||
== "tx-fanin"
|
||||
)
|
||||
assert notification_counts == {}
|
||||
assert completed_transfer_ids == {"tx-fanin"}
|
||||
|
||||
|
||||
def test_duplicate_ack_after_completion_does_not_resolve_twice():
|
||||
ack = MoRIIOTransferAck("tx-dup", consumer_tp_size=8)
|
||||
notification_counts: dict[str, int] = {}
|
||||
completed_transfer_ids: set[str] = set()
|
||||
|
||||
assert (
|
||||
resolve_moriio_transfer_ack(
|
||||
ack,
|
||||
producer_tp_size=4,
|
||||
live_transfer_ids={"tx-dup"},
|
||||
notification_counts=notification_counts,
|
||||
completed_transfer_ids=completed_transfer_ids,
|
||||
)
|
||||
is None
|
||||
)
|
||||
assert (
|
||||
resolve_moriio_transfer_ack(
|
||||
ack,
|
||||
producer_tp_size=4,
|
||||
live_transfer_ids={"tx-dup"},
|
||||
notification_counts=notification_counts,
|
||||
completed_transfer_ids=completed_transfer_ids,
|
||||
)
|
||||
== "tx-dup"
|
||||
)
|
||||
assert (
|
||||
resolve_moriio_transfer_ack(
|
||||
ack,
|
||||
producer_tp_size=4,
|
||||
live_transfer_ids={"tx-dup"},
|
||||
notification_counts=notification_counts,
|
||||
completed_transfer_ids=completed_transfer_ids,
|
||||
)
|
||||
is None
|
||||
)
|
||||
assert notification_counts == {}
|
||||
assert completed_transfer_ids == {"tx-dup"}
|
||||
|
||||
|
||||
def test_ack_for_non_live_transfer_is_ignored():
|
||||
notification_counts: dict[str, int] = {}
|
||||
completed_transfer_ids: set[str] = set()
|
||||
|
||||
assert (
|
||||
resolve_moriio_transfer_ack(
|
||||
MoRIIOTransferAck("tx-stale", consumer_tp_size=8),
|
||||
producer_tp_size=4,
|
||||
live_transfer_ids={"tx-live"},
|
||||
notification_counts=notification_counts,
|
||||
completed_transfer_ids=completed_transfer_ids,
|
||||
)
|
||||
is None
|
||||
)
|
||||
assert notification_counts == {}
|
||||
assert completed_transfer_ids == set()
|
||||
|
||||
|
||||
def test_worker_get_finished_counts_structured_release_fan_in():
|
||||
class FakeWrapper:
|
||||
def __init__(self):
|
||||
self.batches = [
|
||||
[MoRIIOTransferAck("tx-fanin", consumer_tp_size=8)],
|
||||
[MoRIIOTransferAck("tx-fanin", consumer_tp_size=8)],
|
||||
]
|
||||
|
||||
def pop_finished_req_ids(self):
|
||||
return self.batches.pop(0)
|
||||
|
||||
def shutdown(self):
|
||||
pass
|
||||
|
||||
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
|
||||
worker.is_producer = True
|
||||
worker.mode = MoRIIOMode.READ
|
||||
worker.world_size = 4
|
||||
worker.moriio_wrapper = FakeWrapper()
|
||||
worker.transfer_id_to_request_id = {"tx-fanin": "req-fanin"}
|
||||
worker._consumer_notification_counts = {}
|
||||
worker._completed_consumer_notifications = set()
|
||||
|
||||
assert worker.get_finished() == (set(), set())
|
||||
assert worker._consumer_notification_counts == {"tx-fanin": 1}
|
||||
|
||||
assert worker.get_finished() == ({"req-fanin"}, set())
|
||||
assert worker._consumer_notification_counts == {}
|
||||
assert worker._completed_consumer_notifications == {"tx-fanin"}
|
||||
|
||||
|
||||
def test_read_completion_sends_structured_release_with_consumer_tp_size():
|
||||
class DoneStatus:
|
||||
def Succeeded(self):
|
||||
return True
|
||||
|
||||
def Failed(self):
|
||||
return False
|
||||
|
||||
class FakeWrapper:
|
||||
def __init__(self):
|
||||
self.lock = threading.Lock()
|
||||
self.sent = []
|
||||
|
||||
def send_notify(
|
||||
self,
|
||||
transfer_id,
|
||||
host,
|
||||
port,
|
||||
message_type=None,
|
||||
message_fields=None,
|
||||
):
|
||||
self.sent.append((transfer_id, host, port, message_type, message_fields))
|
||||
|
||||
def shutdown(self):
|
||||
pass
|
||||
|
||||
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
|
||||
worker.world_size = 8
|
||||
worker.moriio_wrapper = FakeWrapper()
|
||||
worker._recving_transfers = {"req": [DoneStatus()]}
|
||||
worker._recving_transfers_callback_addr = {
|
||||
"req": ("127.0.0.1", "7000", "tx-release")
|
||||
}
|
||||
|
||||
assert worker._pop_done_transfers() == {"tx-release"}
|
||||
assert worker.moriio_wrapper.sent == [
|
||||
(
|
||||
"tx-release",
|
||||
"127.0.0.1",
|
||||
"7000",
|
||||
"release",
|
||||
{"consumer_tp_size": 8},
|
||||
)
|
||||
]
|
||||
assert worker._recving_transfers == {}
|
||||
assert worker._recving_transfers_callback_addr == {}
|
||||
@@ -6,7 +6,7 @@ import threading
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple
|
||||
|
||||
import msgspec
|
||||
import regex as re
|
||||
@@ -43,6 +43,11 @@ ReqId = str
|
||||
TransferId = str
|
||||
|
||||
|
||||
class MoRIIOTransferAck(NamedTuple):
|
||||
transfer_id: TransferId
|
||||
consumer_tp_size: int = 1
|
||||
|
||||
|
||||
@dataclass
|
||||
class WriteTask:
|
||||
request_id: ReqId
|
||||
|
||||
@@ -6,6 +6,7 @@ import queue
|
||||
import threading
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from collections.abc import Collection
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
@@ -30,6 +31,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import (
|
||||
MoRIIOConnectorMetadata,
|
||||
MoRIIOConstants,
|
||||
MoRIIOMode,
|
||||
MoRIIOTransferAck,
|
||||
ReqId,
|
||||
ReqMeta,
|
||||
TransferId,
|
||||
@@ -98,6 +100,92 @@ def is_moriio_available() -> bool:
|
||||
return MoRIIO_enabled
|
||||
|
||||
|
||||
def get_moriio_remote_tp_rank(
|
||||
local_tp_rank: int, local_tp_size: int, remote_tp_size: int
|
||||
) -> int:
|
||||
if local_tp_size <= 0 or remote_tp_size <= 0:
|
||||
raise ValueError("TP sizes must be positive")
|
||||
if local_tp_rank < 0 or local_tp_rank >= local_tp_size:
|
||||
raise ValueError(
|
||||
f"local_tp_rank {local_tp_rank} must be in [0, {local_tp_size})"
|
||||
)
|
||||
if remote_tp_size == local_tp_size:
|
||||
return local_tp_rank
|
||||
if remote_tp_size > local_tp_size:
|
||||
if remote_tp_size % local_tp_size != 0:
|
||||
raise ValueError(
|
||||
f"remote tp_size {remote_tp_size} must be a multiple of local "
|
||||
f"tp_size {local_tp_size} for heterogeneous-TP P/D"
|
||||
)
|
||||
return local_tp_rank * (remote_tp_size // local_tp_size)
|
||||
if local_tp_size % remote_tp_size != 0:
|
||||
raise ValueError(
|
||||
f"local tp_size {local_tp_size} must be a multiple of remote "
|
||||
f"tp_size {remote_tp_size} for heterogeneous-TP P/D"
|
||||
)
|
||||
return local_tp_rank // (local_tp_size // remote_tp_size)
|
||||
|
||||
|
||||
def validate_moriio_heterogeneous_tp_kv_heads(
|
||||
local_tp_size: int,
|
||||
remote_tp_size: int,
|
||||
total_num_kv_heads: int,
|
||||
is_mla: bool,
|
||||
) -> None:
|
||||
if is_mla or local_tp_size == remote_tp_size:
|
||||
return
|
||||
if local_tp_size <= 0 or remote_tp_size <= 0 or total_num_kv_heads <= 0:
|
||||
raise ValueError("TP sizes and total_num_kv_heads must be positive")
|
||||
if min(local_tp_size, remote_tp_size) >= total_num_kv_heads:
|
||||
return
|
||||
raise NotImplementedError(
|
||||
"MoRIIO heterogeneous TP requires replicated KV heads on both "
|
||||
f"prefill and decode. Got total_num_kv_heads={total_num_kv_heads}, "
|
||||
f"local_tp_size={local_tp_size}, remote_tp_size={remote_tp_size}."
|
||||
)
|
||||
|
||||
|
||||
def get_moriio_expected_ack_count(producer_tp_size: int, consumer_tp_size: int) -> int:
|
||||
if producer_tp_size <= 0 or consumer_tp_size <= 0:
|
||||
raise ValueError("TP sizes must be positive")
|
||||
if consumer_tp_size <= producer_tp_size:
|
||||
return 1
|
||||
if consumer_tp_size % producer_tp_size != 0:
|
||||
raise ValueError(
|
||||
f"consumer tp_size {consumer_tp_size} must be a multiple of "
|
||||
f"producer tp_size {producer_tp_size} for heterogeneous-TP P/D"
|
||||
)
|
||||
return consumer_tp_size // producer_tp_size
|
||||
|
||||
|
||||
def resolve_moriio_transfer_ack(
|
||||
ack: MoRIIOTransferAck | TransferId,
|
||||
producer_tp_size: int,
|
||||
live_transfer_ids: Collection[TransferId],
|
||||
notification_counts: dict[TransferId, int],
|
||||
completed_transfer_ids: set[TransferId],
|
||||
) -> TransferId | None:
|
||||
if isinstance(ack, str):
|
||||
ack = MoRIIOTransferAck(ack)
|
||||
transfer_id = ack.transfer_id
|
||||
if transfer_id not in live_transfer_ids:
|
||||
return None
|
||||
if transfer_id in completed_transfer_ids:
|
||||
return None
|
||||
|
||||
expected_acks = get_moriio_expected_ack_count(
|
||||
producer_tp_size, ack.consumer_tp_size
|
||||
)
|
||||
count = notification_counts.get(transfer_id, 0) + 1
|
||||
if count < expected_acks:
|
||||
notification_counts[transfer_id] = count
|
||||
return None
|
||||
|
||||
notification_counts.pop(transfer_id, None)
|
||||
completed_transfer_ids.add(transfer_id)
|
||||
return transfer_id
|
||||
|
||||
|
||||
class MoRIIOConnector(KVConnectorBase_V1):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -798,6 +886,11 @@ class MoRIIOConnectorWorker:
|
||||
# Completions that arrived before transfer_id_to_request_id was populated.
|
||||
# Retried each step until the mapping is established.
|
||||
self._unmatched_write_completions: set[str] = set()
|
||||
# Producer-side READ-mode ACK fan-in. When decode TP is larger than
|
||||
# prefill TP, multiple decode ranks can read from one prefill rank and
|
||||
# notify the same transfer_id. Blocks are reusable only after all ACKs.
|
||||
self._consumer_notification_counts: dict[TransferId, int] = {}
|
||||
self._completed_consumer_notifications: set[TransferId] = set()
|
||||
|
||||
role = "producer" if self.is_producer else "consumer"
|
||||
engine_suffix = (
|
||||
@@ -1161,7 +1254,9 @@ class MoRIIOConnectorWorker:
|
||||
# a hack to keep us moving. We will switch when moving to etcd
|
||||
# or where we have a single ZMQ socket in the scheduler.
|
||||
|
||||
port_offset = get_port_offset(remote_dp_rank, self.tp_rank)
|
||||
port_offset = get_port_offset(
|
||||
remote_dp_rank, self._remote_tp_rank(remote_tp_size)
|
||||
)
|
||||
path = make_zmq_path("tcp", host, port + port_offset)
|
||||
logger.debug("handshake Querying metadata on path: %s", path)
|
||||
|
||||
@@ -1226,6 +1321,9 @@ class MoRIIOConnectorWorker:
|
||||
|
||||
return {remote_agent_name}
|
||||
|
||||
def _remote_tp_rank(self, remote_tp_size: int) -> int:
|
||||
return get_moriio_remote_tp_rank(self.tp_rank, self.world_size, remote_tp_size)
|
||||
|
||||
def _background_moriio_handshake(
|
||||
self, req_id: ReqId, remote_engine_id: EngineId, meta: ReqMeta
|
||||
):
|
||||
@@ -1435,13 +1533,32 @@ class MoRIIOConnectorWorker:
|
||||
done_sending, done_recving = set(), set()
|
||||
|
||||
if self.is_producer:
|
||||
# pop_finished_req_ids returns transfer_ids (the ZMQ payload sent
|
||||
# by decode via send_notify); map back to req_ids for the scheduler.
|
||||
finished_transfer_ids = self.moriio_wrapper.pop_finished_req_ids()
|
||||
# pop_finished_req_ids returns release ACKs sent by decode. Keep
|
||||
# duplicate ACKs because heterogeneous TP can fan multiple decode
|
||||
# ranks into one prefill rank for the same transfer_id.
|
||||
finished_acks = self.moriio_wrapper.pop_finished_req_ids()
|
||||
resolved_transfer_ids: set[TransferId] = set()
|
||||
for ack in finished_acks:
|
||||
transfer_id = ack if isinstance(ack, str) else ack.transfer_id
|
||||
if transfer_id not in self.transfer_id_to_request_id:
|
||||
logger.warning(
|
||||
"Could not find %s in transfer_id_to_request_id "
|
||||
"lookup table. This could lead to a possible hang.",
|
||||
transfer_id,
|
||||
)
|
||||
continue
|
||||
resolved_transfer_id = resolve_moriio_transfer_ack(
|
||||
ack,
|
||||
producer_tp_size=self.world_size,
|
||||
live_transfer_ids=self.transfer_id_to_request_id.keys(),
|
||||
notification_counts=self._consumer_notification_counts,
|
||||
completed_transfer_ids=(self._completed_consumer_notifications),
|
||||
)
|
||||
if resolved_transfer_id is not None:
|
||||
resolved_transfer_ids.add(resolved_transfer_id)
|
||||
done_sending = {
|
||||
self.transfer_id_to_request_id[xfer_id]
|
||||
for xfer_id in finished_transfer_ids
|
||||
if xfer_id in self.transfer_id_to_request_id
|
||||
for xfer_id in resolved_transfer_ids
|
||||
}
|
||||
else:
|
||||
if self.mode == MoRIIOMode.WRITE:
|
||||
@@ -1486,7 +1603,13 @@ class MoRIIOConnectorWorker:
|
||||
if last.Succeeded():
|
||||
host, port, xfer_id = self._recving_transfers_callback_addr[req_id]
|
||||
done_req_ids.add(xfer_id)
|
||||
self.moriio_wrapper.send_notify(xfer_id, host, port)
|
||||
self.moriio_wrapper.send_notify(
|
||||
xfer_id,
|
||||
host,
|
||||
port,
|
||||
message_type="release",
|
||||
message_fields={"consumer_tp_size": self.world_size},
|
||||
)
|
||||
to_remove.append(req_id)
|
||||
elif last.Failed():
|
||||
logger.error(
|
||||
@@ -1499,7 +1622,13 @@ class MoRIIOConnectorWorker:
|
||||
)
|
||||
host, port, xfer_id = self._recving_transfers_callback_addr[req_id]
|
||||
try:
|
||||
self.moriio_wrapper.send_notify(xfer_id, host, port)
|
||||
self.moriio_wrapper.send_notify(
|
||||
xfer_id,
|
||||
host,
|
||||
port,
|
||||
message_type="release",
|
||||
message_fields={"consumer_tp_size": self.world_size},
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to send error notification for request %s",
|
||||
@@ -1585,6 +1714,15 @@ class MoRIIOConnectorWorker:
|
||||
"""
|
||||
self.transfer_id_to_request_id = metadata.transfer_id_to_request_id
|
||||
if self.is_producer:
|
||||
live_transfer_ids = set(self.transfer_id_to_request_id)
|
||||
self._consumer_notification_counts = {
|
||||
transfer_id: count
|
||||
for transfer_id, count in self._consumer_notification_counts.items()
|
||||
if transfer_id in live_transfer_ids
|
||||
}
|
||||
self._completed_consumer_notifications.intersection_update(
|
||||
live_transfer_ids
|
||||
)
|
||||
self.moriio_wrapper.async_wait_reqid()
|
||||
return
|
||||
if self.mode == MoRIIOMode.WRITE:
|
||||
@@ -1663,6 +1801,7 @@ class MoRIIOConnectorWorker:
|
||||
remote_block_ids=meta.remote_block_ids,
|
||||
remote_host=meta.remote_host,
|
||||
remote_notify_port=meta.remote_notify_port,
|
||||
remote_tp_size=meta.tp_size,
|
||||
)
|
||||
|
||||
def _write_blocks_for_req(self, req_id: ReqId, meta: ReqMeta, layer_name, kv_layer):
|
||||
@@ -1756,6 +1895,7 @@ class MoRIIOConnectorWorker:
|
||||
local_block_ids: list[int],
|
||||
remote_block_ids: list[int],
|
||||
remote_moriio_meta: MoRIIOAgentMetadata,
|
||||
remote_tp_size: int | None = None,
|
||||
) -> tuple[list[int], list[int], list[int]]:
|
||||
"""Compute transfer offsets for block data.
|
||||
|
||||
@@ -1767,6 +1907,14 @@ class MoRIIOConnectorWorker:
|
||||
Returns:
|
||||
Tuple of (local_offsets, remote_offsets, transfer_sizes)
|
||||
"""
|
||||
validate_moriio_heterogeneous_tp_kv_heads(
|
||||
local_tp_size=self.world_size,
|
||||
remote_tp_size=(
|
||||
remote_tp_size if remote_tp_size is not None else self.world_size
|
||||
),
|
||||
total_num_kv_heads=self.model_config.get_total_num_kv_heads(),
|
||||
is_mla=self._is_mla_cache_layer(layer_name),
|
||||
)
|
||||
return compute_block_transfer_offsets(
|
||||
layer_name=layer_name,
|
||||
kv_cache=self.kv_caches[layer_name],
|
||||
@@ -1788,6 +1936,7 @@ class MoRIIOConnectorWorker:
|
||||
transfer_id: str,
|
||||
remote_host: str,
|
||||
remote_notify_port: int,
|
||||
remote_tp_size: int,
|
||||
) -> None:
|
||||
if self.mode == MoRIIOMode.WRITE:
|
||||
return
|
||||
@@ -1800,7 +1949,11 @@ class MoRIIOConnectorWorker:
|
||||
layer_name
|
||||
)
|
||||
offs = self._compute_block_transfer_offsets(
|
||||
layer_name, local_block_ids, remote_block_ids, remote_moriio_meta
|
||||
layer_name,
|
||||
local_block_ids,
|
||||
remote_block_ids,
|
||||
remote_moriio_meta,
|
||||
remote_tp_size=remote_tp_size,
|
||||
)
|
||||
# TODO : apply multi-session batch-read when moriio support it
|
||||
transfer_status = self.moriio_wrapper.read_remote_data(
|
||||
@@ -1810,6 +1963,6 @@ class MoRIIOConnectorWorker:
|
||||
self._recving_transfers[request_id].append(transfer_status)
|
||||
self._recving_transfers_callback_addr[request_id] = (
|
||||
remote_host,
|
||||
str(remote_notify_port + self.tp_rank),
|
||||
str(remote_notify_port + self._remote_tp_rank(remote_tp_size)),
|
||||
transfer_id,
|
||||
)
|
||||
|
||||
@@ -27,6 +27,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import (
|
||||
MoRIIOAgentMetadata,
|
||||
MoRIIOConstants,
|
||||
MoRIIOError,
|
||||
MoRIIOTransferAck,
|
||||
RemoteAllocInfo,
|
||||
TransferError,
|
||||
TransferId,
|
||||
@@ -261,7 +262,7 @@ class MoRIIOWriter:
|
||||
"""Mark a request done so its blocks are freed, even on transfer failure."""
|
||||
wrapper = self.worker.moriio_wrapper
|
||||
with wrapper.lock:
|
||||
wrapper.done_req_ids.append(transfer_id)
|
||||
wrapper.done_req_ids.append(MoRIIOTransferAck(transfer_id))
|
||||
wrapper.done_remote_allocate_req_dict.pop(transfer_id, None)
|
||||
wrapper._mark_transfer_terminal_locked(transfer_id)
|
||||
self._clear_transfer_state(transfer_id)
|
||||
@@ -466,7 +467,9 @@ class MoRIIOWriter:
|
||||
)
|
||||
# mark request as done, then we can free the blocks
|
||||
with self.worker.moriio_wrapper.lock:
|
||||
self.worker.moriio_wrapper.done_req_ids.append(transfer_id)
|
||||
self.worker.moriio_wrapper.done_req_ids.append(
|
||||
MoRIIOTransferAck(transfer_id)
|
||||
)
|
||||
self.worker.moriio_wrapper.done_remote_allocate_req_dict.pop(
|
||||
transfer_id, None
|
||||
)
|
||||
@@ -508,7 +511,7 @@ class MoRIIOWrapper:
|
||||
self.remote_engine_ip: str | None = None
|
||||
self.notify_port: int | None = None
|
||||
self.lock = threading.Lock()
|
||||
self.done_req_ids: list[str] = []
|
||||
self.done_req_ids: list[MoRIIOTransferAck] = []
|
||||
self.done_remote_allocate_req_dict: dict[TransferId, RemoteAllocInfo] = {}
|
||||
self.done_write_cache_req_ids: list[str] = []
|
||||
self._terminal_transfer_ids: OrderedDict[TransferId, None] = OrderedDict()
|
||||
@@ -784,15 +787,20 @@ class MoRIIOWrapper:
|
||||
"Only prefill can get transfer release messages"
|
||||
)
|
||||
transfer_id = data["transfer_id"]
|
||||
consumer_tp_size = int(data.get("consumer_tp_size", 1))
|
||||
if consumer_tp_size <= 0:
|
||||
raise MoRIIOError(
|
||||
f"Invalid consumer_tp_size in release message: {consumer_tp_size}"
|
||||
)
|
||||
with self.lock:
|
||||
self.done_req_ids.append(transfer_id)
|
||||
self.done_req_ids.append(MoRIIOTransferAck(transfer_id, consumer_tp_size))
|
||||
self.done_remote_allocate_req_dict.pop(transfer_id, None)
|
||||
self._mark_transfer_terminal_locked(transfer_id)
|
||||
|
||||
def _handle_completion_message(self, msg: str):
|
||||
with self.lock:
|
||||
if get_role() == ROLE.PRODUCER:
|
||||
self.done_req_ids.append(msg)
|
||||
self.done_req_ids.append(MoRIIOTransferAck(msg))
|
||||
self.done_remote_allocate_req_dict.pop(msg, None)
|
||||
self._mark_transfer_terminal_locked(msg)
|
||||
else:
|
||||
@@ -808,7 +816,12 @@ class MoRIIOWrapper:
|
||||
self._terminal_transfer_ids.popitem(last=False)
|
||||
|
||||
def send_notify(
|
||||
self, req_ids, remote_ip, remote_port, message_type: str | None = None
|
||||
self,
|
||||
req_ids,
|
||||
remote_ip,
|
||||
remote_port,
|
||||
message_type: str | None = None,
|
||||
message_fields: dict[str, Any] | None = None,
|
||||
):
|
||||
if not remote_ip or not remote_port:
|
||||
logger.warning("Missing remote_ip or remote_port for notification")
|
||||
@@ -836,18 +849,21 @@ class MoRIIOWrapper:
|
||||
if message_type is None:
|
||||
sock.send(req_id.encode("utf-8"))
|
||||
else:
|
||||
sock.send(
|
||||
msgpack.dumps({"type": message_type, "transfer_id": req_id})
|
||||
)
|
||||
payload = {"type": message_type, "transfer_id": req_id}
|
||||
if message_fields:
|
||||
payload.update(message_fields)
|
||||
sock.send(msgpack.dumps(payload))
|
||||
except Exception as e:
|
||||
logger.error("Failed to send notification to %s: %s", path, e)
|
||||
self.paths.pop(path, None)
|
||||
raise
|
||||
|
||||
def pop_finished_req_ids(self):
|
||||
# producer invocation: get the set of completed requests at the decode
|
||||
# Producer invocation: return every completion message since the last
|
||||
# call. Do not dedupe: heterogeneous TP can produce multiple release
|
||||
# ACKs for the same transfer_id and the caller must count each one.
|
||||
with self.lock:
|
||||
done_send = set(self.done_req_ids)
|
||||
done_send = list(self.done_req_ids)
|
||||
self.done_req_ids = []
|
||||
return done_send
|
||||
|
||||
|
||||
Reference in New Issue
Block a user