[ROCm][P/D] Support MoRIIO heterogeneous TP fan-in (#46332)

Signed-off-by: Tan Pin Siang <tanpinsiang@gmail.com>
Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com>
Co-authored-by: Hongxia Yang <hongxia.yang@amd.com>
Co-authored-by: Jun Kang Chow <junkangchow@gmail.com>
Co-authored-by: Chun Fang <chun.fang@amd.com>
Co-authored-by: TianDi101 <ditian12@amd.com>
Co-authored-by: functionstackx <47992694+functionstackx@users.noreply.github.com>
Co-authored-by: tjtanaa <tunjian.tan@embeddedllm.com>
This commit is contained in:
Tan Pin Siang
2026-06-23 10:33:23 +00:00
committed by GitHub
co-authored by vllmellm Hongxia Yang Jun Kang Chow Chun Fang TianDi101 functionstackx tjtanaa
parent 83fa302ca4
commit d32575a2d2
6 changed files with 544 additions and 33 deletions
@@ -195,7 +195,14 @@ class FakeMoRIIOWrapper:
def _handle_completion_message(self, msg: str):
pass
def send_notify(self, req_ids, remote_ip, remote_port, message_type=None):
def send_notify(
self,
req_ids,
remote_ip,
remote_port,
message_type=None,
message_fields=None,
):
pass
def pop_finished_req_ids(self):
@@ -36,6 +36,7 @@ msgpack = importlib.import_module("msgpack")
ROLE = moriio_common.ROLE
MoRIIOError = moriio_common.MoRIIOError
MoRIIOTransferAck = moriio_common.MoRIIOTransferAck
RemoteAllocInfo = moriio_common.RemoteAllocInfo
WriteTask = moriio_common.WriteTask
set_role = moriio_common.set_role
@@ -440,9 +441,16 @@ def test_write_completion_notifies_once_after_all_sealed_writes_finish():
def _mark_transfer_terminal_locked(self, transfer_id):
self._terminal_transfer_ids[transfer_id] = None
def send_notify(self, transfer_id, remote_ip, remote_port, message_type=None):
def send_notify(
self,
transfer_id,
remote_ip,
remote_port,
message_type=None,
message_fields=None,
):
self.notifications.append(
(transfer_id, remote_ip, remote_port, message_type)
(transfer_id, remote_ip, remote_port, message_type, message_fields)
)
wrapper = FakeWrapper()
@@ -464,8 +472,8 @@ def test_write_completion_notifies_once_after_all_sealed_writes_finish():
writer._mark_write_done("xfer", request_info)
writer._finalize_if_complete("xfer", request_info)
assert wrapper.notifications == [("xfer", "127.0.0.1", 7002, "write_done")]
assert wrapper.done_req_ids == ["xfer"]
assert wrapper.notifications == [("xfer", "127.0.0.1", 7002, "write_done", None)]
assert wrapper.done_req_ids == [MoRIIOTransferAck("xfer")]
assert wrapper.done_remote_allocate_req_dict == {}
assert wrapper.wait_count == 1
assert wrapper.waited_statuses == [["status-a", "status-b"]]
@@ -509,7 +517,7 @@ def test_write_failure_marks_terminal_and_clears_scheduled_state():
writer._mark_request_done("xfer")
assert wrapper.done_req_ids == ["xfer"]
assert wrapper.done_req_ids == [MoRIIOTransferAck("xfer")]
assert wrapper.done_remote_allocate_req_dict == {}
assert wrapper._is_transfer_terminal_locked("xfer")
assert "xfer" not in writer._scheduled_writes
@@ -581,9 +589,21 @@ def test_late_remote_blocks_message_is_ignored_after_transfer_done():
pytest.param(
ROLE.PRODUCER,
msgpack.dumps({"type": "release", "transfer_id": "xfer"}),
"release",
MoRIIOTransferAck("xfer"),
id="release",
),
pytest.param(
ROLE.PRODUCER,
msgpack.dumps(
{
"type": "release",
"transfer_id": "xfer",
"consumer_tp_size": 8,
}
),
MoRIIOTransferAck("xfer", 8),
id="release-consumer-tp-size",
),
pytest.param(None, b"xfer", "plain", id="plain-string"),
],
)
@@ -603,11 +623,11 @@ def test_moriio_wrapper_routes_valid_messages(role, payload, expected):
assert request_info.decode_dp_rank == 3
elif expected == "write_done":
assert wrapper.done_write_cache_req_ids == ["xfer"]
elif expected == "release":
assert wrapper.done_req_ids == ["xfer"]
assert wrapper._is_transfer_terminal_locked("xfer")
else:
elif expected == "plain":
assert completions == ["xfer"]
else:
assert wrapper.done_req_ids == [expected]
assert wrapper._is_transfer_terminal_locked("xfer")
@pytest.mark.parametrize(
@@ -0,0 +1,310 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import threading
import pytest
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import (
MoRIIOMode,
MoRIIOTransferAck,
)
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import (
MoRIIOConnectorWorker,
get_moriio_expected_ack_count,
get_moriio_remote_tp_rank,
resolve_moriio_transfer_ack,
validate_moriio_heterogeneous_tp_kv_heads,
)
def test_remote_tp_rank_same_tp_maps_to_self():
assert [get_moriio_remote_tp_rank(rank, 4, 4) for rank in range(4)] == [
0,
1,
2,
3,
]
def test_remote_tp_rank_p4_d8_floor_maps_decode_to_prefill():
assert [get_moriio_remote_tp_rank(rank, 8, 4) for rank in range(8)] == [
0,
0,
1,
1,
2,
2,
3,
3,
]
def test_remote_tp_rank_p8_d4_maps_to_first_prefill_rank_per_pair():
assert [get_moriio_remote_tp_rank(rank, 4, 8) for rank in range(4)] == [
0,
2,
4,
6,
]
@pytest.mark.parametrize(
("local_tp_rank", "local_tp_size", "remote_tp_size"),
[
(0, 6, 4),
(0, 4, 6),
],
)
def test_remote_tp_rank_invalid_non_multiple_tp_raises(
local_tp_rank: int, local_tp_size: int, remote_tp_size: int
):
with pytest.raises(ValueError, match="multiple"):
get_moriio_remote_tp_rank(local_tp_rank, local_tp_size, remote_tp_size)
@pytest.mark.parametrize(
("local_tp_size", "remote_tp_size", "total_num_kv_heads"),
[
(4, 4, 8),
(8, 4, 4),
(4, 8, 4),
],
)
def test_heterogeneous_tp_head_guard_allows_supported_layouts(
local_tp_size: int, remote_tp_size: int, total_num_kv_heads: int
):
validate_moriio_heterogeneous_tp_kv_heads(
local_tp_size,
remote_tp_size,
total_num_kv_heads,
is_mla=False,
)
def test_heterogeneous_tp_head_guard_allows_mla_layouts():
validate_moriio_heterogeneous_tp_kv_heads(
local_tp_size=2,
remote_tp_size=4,
total_num_kv_heads=4,
is_mla=True,
)
@pytest.mark.parametrize(
("local_tp_size", "remote_tp_size", "total_num_kv_heads"),
[
(4, 2, 4),
(2, 4, 4),
],
)
def test_heterogeneous_tp_head_guard_rejects_split_kv_heads(
local_tp_size: int, remote_tp_size: int, total_num_kv_heads: int
):
with pytest.raises(NotImplementedError, match="replicated KV heads"):
validate_moriio_heterogeneous_tp_kv_heads(
local_tp_size,
remote_tp_size,
total_num_kv_heads,
is_mla=False,
)
def test_expected_ack_count_for_homogeneous_or_smaller_consumer_tp_is_one():
assert get_moriio_expected_ack_count(4, 4) == 1
assert get_moriio_expected_ack_count(8, 4) == 1
def test_expected_ack_count_for_decode_fan_in():
assert get_moriio_expected_ack_count(4, 8) == 2
def test_expected_ack_count_rejects_non_multiple_fan_in():
with pytest.raises(ValueError, match="multiple"):
get_moriio_expected_ack_count(4, 6)
def test_plain_string_ack_is_backward_compatible_single_ack():
notification_counts: dict[str, int] = {}
completed_transfer_ids: set[str] = set()
assert (
resolve_moriio_transfer_ack(
"tx-plain",
producer_tp_size=4,
live_transfer_ids={"tx-plain"},
notification_counts=notification_counts,
completed_transfer_ids=completed_transfer_ids,
)
== "tx-plain"
)
assert notification_counts == {}
assert completed_transfer_ids == {"tx-plain"}
def test_structured_release_ack_waits_for_all_expected_acks():
ack = MoRIIOTransferAck("tx-fanin", consumer_tp_size=8)
notification_counts: dict[str, int] = {}
completed_transfer_ids: set[str] = set()
assert (
resolve_moriio_transfer_ack(
ack,
producer_tp_size=4,
live_transfer_ids={"tx-fanin"},
notification_counts=notification_counts,
completed_transfer_ids=completed_transfer_ids,
)
is None
)
assert notification_counts == {"tx-fanin": 1}
assert completed_transfer_ids == set()
assert (
resolve_moriio_transfer_ack(
ack,
producer_tp_size=4,
live_transfer_ids={"tx-fanin"},
notification_counts=notification_counts,
completed_transfer_ids=completed_transfer_ids,
)
== "tx-fanin"
)
assert notification_counts == {}
assert completed_transfer_ids == {"tx-fanin"}
def test_duplicate_ack_after_completion_does_not_resolve_twice():
ack = MoRIIOTransferAck("tx-dup", consumer_tp_size=8)
notification_counts: dict[str, int] = {}
completed_transfer_ids: set[str] = set()
assert (
resolve_moriio_transfer_ack(
ack,
producer_tp_size=4,
live_transfer_ids={"tx-dup"},
notification_counts=notification_counts,
completed_transfer_ids=completed_transfer_ids,
)
is None
)
assert (
resolve_moriio_transfer_ack(
ack,
producer_tp_size=4,
live_transfer_ids={"tx-dup"},
notification_counts=notification_counts,
completed_transfer_ids=completed_transfer_ids,
)
== "tx-dup"
)
assert (
resolve_moriio_transfer_ack(
ack,
producer_tp_size=4,
live_transfer_ids={"tx-dup"},
notification_counts=notification_counts,
completed_transfer_ids=completed_transfer_ids,
)
is None
)
assert notification_counts == {}
assert completed_transfer_ids == {"tx-dup"}
def test_ack_for_non_live_transfer_is_ignored():
notification_counts: dict[str, int] = {}
completed_transfer_ids: set[str] = set()
assert (
resolve_moriio_transfer_ack(
MoRIIOTransferAck("tx-stale", consumer_tp_size=8),
producer_tp_size=4,
live_transfer_ids={"tx-live"},
notification_counts=notification_counts,
completed_transfer_ids=completed_transfer_ids,
)
is None
)
assert notification_counts == {}
assert completed_transfer_ids == set()
def test_worker_get_finished_counts_structured_release_fan_in():
class FakeWrapper:
def __init__(self):
self.batches = [
[MoRIIOTransferAck("tx-fanin", consumer_tp_size=8)],
[MoRIIOTransferAck("tx-fanin", consumer_tp_size=8)],
]
def pop_finished_req_ids(self):
return self.batches.pop(0)
def shutdown(self):
pass
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
worker.is_producer = True
worker.mode = MoRIIOMode.READ
worker.world_size = 4
worker.moriio_wrapper = FakeWrapper()
worker.transfer_id_to_request_id = {"tx-fanin": "req-fanin"}
worker._consumer_notification_counts = {}
worker._completed_consumer_notifications = set()
assert worker.get_finished() == (set(), set())
assert worker._consumer_notification_counts == {"tx-fanin": 1}
assert worker.get_finished() == ({"req-fanin"}, set())
assert worker._consumer_notification_counts == {}
assert worker._completed_consumer_notifications == {"tx-fanin"}
def test_read_completion_sends_structured_release_with_consumer_tp_size():
class DoneStatus:
def Succeeded(self):
return True
def Failed(self):
return False
class FakeWrapper:
def __init__(self):
self.lock = threading.Lock()
self.sent = []
def send_notify(
self,
transfer_id,
host,
port,
message_type=None,
message_fields=None,
):
self.sent.append((transfer_id, host, port, message_type, message_fields))
def shutdown(self):
pass
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
worker.world_size = 8
worker.moriio_wrapper = FakeWrapper()
worker._recving_transfers = {"req": [DoneStatus()]}
worker._recving_transfers_callback_addr = {
"req": ("127.0.0.1", "7000", "tx-release")
}
assert worker._pop_done_transfers() == {"tx-release"}
assert worker.moriio_wrapper.sent == [
(
"tx-release",
"127.0.0.1",
"7000",
"release",
{"consumer_tp_size": 8},
)
]
assert worker._recving_transfers == {}
assert worker._recving_transfers_callback_addr == {}
@@ -6,7 +6,7 @@ import threading
import time
from collections.abc import Iterator
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING, Any, NamedTuple
import msgspec
import regex as re
@@ -43,6 +43,11 @@ ReqId = str
TransferId = str
class MoRIIOTransferAck(NamedTuple):
transfer_id: TransferId
consumer_tp_size: int = 1
@dataclass
class WriteTask:
request_id: ReqId
@@ -6,6 +6,7 @@ import queue
import threading
import time
from collections import defaultdict
from collections.abc import Collection
from concurrent.futures import Future, ThreadPoolExecutor
from typing import TYPE_CHECKING, Any
@@ -30,6 +31,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import (
MoRIIOConnectorMetadata,
MoRIIOConstants,
MoRIIOMode,
MoRIIOTransferAck,
ReqId,
ReqMeta,
TransferId,
@@ -98,6 +100,92 @@ def is_moriio_available() -> bool:
return MoRIIO_enabled
def get_moriio_remote_tp_rank(
local_tp_rank: int, local_tp_size: int, remote_tp_size: int
) -> int:
if local_tp_size <= 0 or remote_tp_size <= 0:
raise ValueError("TP sizes must be positive")
if local_tp_rank < 0 or local_tp_rank >= local_tp_size:
raise ValueError(
f"local_tp_rank {local_tp_rank} must be in [0, {local_tp_size})"
)
if remote_tp_size == local_tp_size:
return local_tp_rank
if remote_tp_size > local_tp_size:
if remote_tp_size % local_tp_size != 0:
raise ValueError(
f"remote tp_size {remote_tp_size} must be a multiple of local "
f"tp_size {local_tp_size} for heterogeneous-TP P/D"
)
return local_tp_rank * (remote_tp_size // local_tp_size)
if local_tp_size % remote_tp_size != 0:
raise ValueError(
f"local tp_size {local_tp_size} must be a multiple of remote "
f"tp_size {remote_tp_size} for heterogeneous-TP P/D"
)
return local_tp_rank // (local_tp_size // remote_tp_size)
def validate_moriio_heterogeneous_tp_kv_heads(
local_tp_size: int,
remote_tp_size: int,
total_num_kv_heads: int,
is_mla: bool,
) -> None:
if is_mla or local_tp_size == remote_tp_size:
return
if local_tp_size <= 0 or remote_tp_size <= 0 or total_num_kv_heads <= 0:
raise ValueError("TP sizes and total_num_kv_heads must be positive")
if min(local_tp_size, remote_tp_size) >= total_num_kv_heads:
return
raise NotImplementedError(
"MoRIIO heterogeneous TP requires replicated KV heads on both "
f"prefill and decode. Got total_num_kv_heads={total_num_kv_heads}, "
f"local_tp_size={local_tp_size}, remote_tp_size={remote_tp_size}."
)
def get_moriio_expected_ack_count(producer_tp_size: int, consumer_tp_size: int) -> int:
if producer_tp_size <= 0 or consumer_tp_size <= 0:
raise ValueError("TP sizes must be positive")
if consumer_tp_size <= producer_tp_size:
return 1
if consumer_tp_size % producer_tp_size != 0:
raise ValueError(
f"consumer tp_size {consumer_tp_size} must be a multiple of "
f"producer tp_size {producer_tp_size} for heterogeneous-TP P/D"
)
return consumer_tp_size // producer_tp_size
def resolve_moriio_transfer_ack(
ack: MoRIIOTransferAck | TransferId,
producer_tp_size: int,
live_transfer_ids: Collection[TransferId],
notification_counts: dict[TransferId, int],
completed_transfer_ids: set[TransferId],
) -> TransferId | None:
if isinstance(ack, str):
ack = MoRIIOTransferAck(ack)
transfer_id = ack.transfer_id
if transfer_id not in live_transfer_ids:
return None
if transfer_id in completed_transfer_ids:
return None
expected_acks = get_moriio_expected_ack_count(
producer_tp_size, ack.consumer_tp_size
)
count = notification_counts.get(transfer_id, 0) + 1
if count < expected_acks:
notification_counts[transfer_id] = count
return None
notification_counts.pop(transfer_id, None)
completed_transfer_ids.add(transfer_id)
return transfer_id
class MoRIIOConnector(KVConnectorBase_V1):
def __init__(
self,
@@ -798,6 +886,11 @@ class MoRIIOConnectorWorker:
# Completions that arrived before transfer_id_to_request_id was populated.
# Retried each step until the mapping is established.
self._unmatched_write_completions: set[str] = set()
# Producer-side READ-mode ACK fan-in. When decode TP is larger than
# prefill TP, multiple decode ranks can read from one prefill rank and
# notify the same transfer_id. Blocks are reusable only after all ACKs.
self._consumer_notification_counts: dict[TransferId, int] = {}
self._completed_consumer_notifications: set[TransferId] = set()
role = "producer" if self.is_producer else "consumer"
engine_suffix = (
@@ -1161,7 +1254,9 @@ class MoRIIOConnectorWorker:
# a hack to keep us moving. We will switch when moving to etcd
# or where we have a single ZMQ socket in the scheduler.
port_offset = get_port_offset(remote_dp_rank, self.tp_rank)
port_offset = get_port_offset(
remote_dp_rank, self._remote_tp_rank(remote_tp_size)
)
path = make_zmq_path("tcp", host, port + port_offset)
logger.debug("handshake Querying metadata on path: %s", path)
@@ -1226,6 +1321,9 @@ class MoRIIOConnectorWorker:
return {remote_agent_name}
def _remote_tp_rank(self, remote_tp_size: int) -> int:
return get_moriio_remote_tp_rank(self.tp_rank, self.world_size, remote_tp_size)
def _background_moriio_handshake(
self, req_id: ReqId, remote_engine_id: EngineId, meta: ReqMeta
):
@@ -1435,13 +1533,32 @@ class MoRIIOConnectorWorker:
done_sending, done_recving = set(), set()
if self.is_producer:
# pop_finished_req_ids returns transfer_ids (the ZMQ payload sent
# by decode via send_notify); map back to req_ids for the scheduler.
finished_transfer_ids = self.moriio_wrapper.pop_finished_req_ids()
# pop_finished_req_ids returns release ACKs sent by decode. Keep
# duplicate ACKs because heterogeneous TP can fan multiple decode
# ranks into one prefill rank for the same transfer_id.
finished_acks = self.moriio_wrapper.pop_finished_req_ids()
resolved_transfer_ids: set[TransferId] = set()
for ack in finished_acks:
transfer_id = ack if isinstance(ack, str) else ack.transfer_id
if transfer_id not in self.transfer_id_to_request_id:
logger.warning(
"Could not find %s in transfer_id_to_request_id "
"lookup table. This could lead to a possible hang.",
transfer_id,
)
continue
resolved_transfer_id = resolve_moriio_transfer_ack(
ack,
producer_tp_size=self.world_size,
live_transfer_ids=self.transfer_id_to_request_id.keys(),
notification_counts=self._consumer_notification_counts,
completed_transfer_ids=(self._completed_consumer_notifications),
)
if resolved_transfer_id is not None:
resolved_transfer_ids.add(resolved_transfer_id)
done_sending = {
self.transfer_id_to_request_id[xfer_id]
for xfer_id in finished_transfer_ids
if xfer_id in self.transfer_id_to_request_id
for xfer_id in resolved_transfer_ids
}
else:
if self.mode == MoRIIOMode.WRITE:
@@ -1486,7 +1603,13 @@ class MoRIIOConnectorWorker:
if last.Succeeded():
host, port, xfer_id = self._recving_transfers_callback_addr[req_id]
done_req_ids.add(xfer_id)
self.moriio_wrapper.send_notify(xfer_id, host, port)
self.moriio_wrapper.send_notify(
xfer_id,
host,
port,
message_type="release",
message_fields={"consumer_tp_size": self.world_size},
)
to_remove.append(req_id)
elif last.Failed():
logger.error(
@@ -1499,7 +1622,13 @@ class MoRIIOConnectorWorker:
)
host, port, xfer_id = self._recving_transfers_callback_addr[req_id]
try:
self.moriio_wrapper.send_notify(xfer_id, host, port)
self.moriio_wrapper.send_notify(
xfer_id,
host,
port,
message_type="release",
message_fields={"consumer_tp_size": self.world_size},
)
except Exception:
logger.exception(
"Failed to send error notification for request %s",
@@ -1585,6 +1714,15 @@ class MoRIIOConnectorWorker:
"""
self.transfer_id_to_request_id = metadata.transfer_id_to_request_id
if self.is_producer:
live_transfer_ids = set(self.transfer_id_to_request_id)
self._consumer_notification_counts = {
transfer_id: count
for transfer_id, count in self._consumer_notification_counts.items()
if transfer_id in live_transfer_ids
}
self._completed_consumer_notifications.intersection_update(
live_transfer_ids
)
self.moriio_wrapper.async_wait_reqid()
return
if self.mode == MoRIIOMode.WRITE:
@@ -1663,6 +1801,7 @@ class MoRIIOConnectorWorker:
remote_block_ids=meta.remote_block_ids,
remote_host=meta.remote_host,
remote_notify_port=meta.remote_notify_port,
remote_tp_size=meta.tp_size,
)
def _write_blocks_for_req(self, req_id: ReqId, meta: ReqMeta, layer_name, kv_layer):
@@ -1756,6 +1895,7 @@ class MoRIIOConnectorWorker:
local_block_ids: list[int],
remote_block_ids: list[int],
remote_moriio_meta: MoRIIOAgentMetadata,
remote_tp_size: int | None = None,
) -> tuple[list[int], list[int], list[int]]:
"""Compute transfer offsets for block data.
@@ -1767,6 +1907,14 @@ class MoRIIOConnectorWorker:
Returns:
Tuple of (local_offsets, remote_offsets, transfer_sizes)
"""
validate_moriio_heterogeneous_tp_kv_heads(
local_tp_size=self.world_size,
remote_tp_size=(
remote_tp_size if remote_tp_size is not None else self.world_size
),
total_num_kv_heads=self.model_config.get_total_num_kv_heads(),
is_mla=self._is_mla_cache_layer(layer_name),
)
return compute_block_transfer_offsets(
layer_name=layer_name,
kv_cache=self.kv_caches[layer_name],
@@ -1788,6 +1936,7 @@ class MoRIIOConnectorWorker:
transfer_id: str,
remote_host: str,
remote_notify_port: int,
remote_tp_size: int,
) -> None:
if self.mode == MoRIIOMode.WRITE:
return
@@ -1800,7 +1949,11 @@ class MoRIIOConnectorWorker:
layer_name
)
offs = self._compute_block_transfer_offsets(
layer_name, local_block_ids, remote_block_ids, remote_moriio_meta
layer_name,
local_block_ids,
remote_block_ids,
remote_moriio_meta,
remote_tp_size=remote_tp_size,
)
# TODO : apply multi-session batch-read when moriio support it
transfer_status = self.moriio_wrapper.read_remote_data(
@@ -1810,6 +1963,6 @@ class MoRIIOConnectorWorker:
self._recving_transfers[request_id].append(transfer_status)
self._recving_transfers_callback_addr[request_id] = (
remote_host,
str(remote_notify_port + self.tp_rank),
str(remote_notify_port + self._remote_tp_rank(remote_tp_size)),
transfer_id,
)
@@ -27,6 +27,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import (
MoRIIOAgentMetadata,
MoRIIOConstants,
MoRIIOError,
MoRIIOTransferAck,
RemoteAllocInfo,
TransferError,
TransferId,
@@ -261,7 +262,7 @@ class MoRIIOWriter:
"""Mark a request done so its blocks are freed, even on transfer failure."""
wrapper = self.worker.moriio_wrapper
with wrapper.lock:
wrapper.done_req_ids.append(transfer_id)
wrapper.done_req_ids.append(MoRIIOTransferAck(transfer_id))
wrapper.done_remote_allocate_req_dict.pop(transfer_id, None)
wrapper._mark_transfer_terminal_locked(transfer_id)
self._clear_transfer_state(transfer_id)
@@ -466,7 +467,9 @@ class MoRIIOWriter:
)
# mark request as done, then we can free the blocks
with self.worker.moriio_wrapper.lock:
self.worker.moriio_wrapper.done_req_ids.append(transfer_id)
self.worker.moriio_wrapper.done_req_ids.append(
MoRIIOTransferAck(transfer_id)
)
self.worker.moriio_wrapper.done_remote_allocate_req_dict.pop(
transfer_id, None
)
@@ -508,7 +511,7 @@ class MoRIIOWrapper:
self.remote_engine_ip: str | None = None
self.notify_port: int | None = None
self.lock = threading.Lock()
self.done_req_ids: list[str] = []
self.done_req_ids: list[MoRIIOTransferAck] = []
self.done_remote_allocate_req_dict: dict[TransferId, RemoteAllocInfo] = {}
self.done_write_cache_req_ids: list[str] = []
self._terminal_transfer_ids: OrderedDict[TransferId, None] = OrderedDict()
@@ -784,15 +787,20 @@ class MoRIIOWrapper:
"Only prefill can get transfer release messages"
)
transfer_id = data["transfer_id"]
consumer_tp_size = int(data.get("consumer_tp_size", 1))
if consumer_tp_size <= 0:
raise MoRIIOError(
f"Invalid consumer_tp_size in release message: {consumer_tp_size}"
)
with self.lock:
self.done_req_ids.append(transfer_id)
self.done_req_ids.append(MoRIIOTransferAck(transfer_id, consumer_tp_size))
self.done_remote_allocate_req_dict.pop(transfer_id, None)
self._mark_transfer_terminal_locked(transfer_id)
def _handle_completion_message(self, msg: str):
with self.lock:
if get_role() == ROLE.PRODUCER:
self.done_req_ids.append(msg)
self.done_req_ids.append(MoRIIOTransferAck(msg))
self.done_remote_allocate_req_dict.pop(msg, None)
self._mark_transfer_terminal_locked(msg)
else:
@@ -808,7 +816,12 @@ class MoRIIOWrapper:
self._terminal_transfer_ids.popitem(last=False)
def send_notify(
self, req_ids, remote_ip, remote_port, message_type: str | None = None
self,
req_ids,
remote_ip,
remote_port,
message_type: str | None = None,
message_fields: dict[str, Any] | None = None,
):
if not remote_ip or not remote_port:
logger.warning("Missing remote_ip or remote_port for notification")
@@ -836,18 +849,21 @@ class MoRIIOWrapper:
if message_type is None:
sock.send(req_id.encode("utf-8"))
else:
sock.send(
msgpack.dumps({"type": message_type, "transfer_id": req_id})
)
payload = {"type": message_type, "transfer_id": req_id}
if message_fields:
payload.update(message_fields)
sock.send(msgpack.dumps(payload))
except Exception as e:
logger.error("Failed to send notification to %s: %s", path, e)
self.paths.pop(path, None)
raise
def pop_finished_req_ids(self):
# producer invocation: get the set of completed requests at the decode
# Producer invocation: return every completion message since the last
# call. Do not dedupe: heterogeneous TP can produce multiple release
# ACKs for the same transfer_id and the caller must count each one.
with self.lock:
done_send = set(self.done_req_ids)
done_send = list(self.done_req_ids)
self.done_req_ids = []
return done_send