diff --git a/tests/v1/kv_connector/unit/test_moriio_connector.py b/tests/v1/kv_connector/unit/test_moriio_connector.py index ee296292ea..c236fcc4fd 100644 --- a/tests/v1/kv_connector/unit/test_moriio_connector.py +++ b/tests/v1/kv_connector/unit/test_moriio_connector.py @@ -195,7 +195,14 @@ class FakeMoRIIOWrapper: def _handle_completion_message(self, msg: str): pass - def send_notify(self, req_ids, remote_ip, remote_port, message_type=None): + def send_notify( + self, + req_ids, + remote_ip, + remote_port, + message_type=None, + message_fields=None, + ): pass def pop_finished_req_ids(self): diff --git a/tests/v1/kv_connector/unit/test_moriio_kv_layout.py b/tests/v1/kv_connector/unit/test_moriio_kv_layout.py index 61146ce3c8..49ca683de7 100644 --- a/tests/v1/kv_connector/unit/test_moriio_kv_layout.py +++ b/tests/v1/kv_connector/unit/test_moriio_kv_layout.py @@ -36,6 +36,7 @@ msgpack = importlib.import_module("msgpack") ROLE = moriio_common.ROLE MoRIIOError = moriio_common.MoRIIOError +MoRIIOTransferAck = moriio_common.MoRIIOTransferAck RemoteAllocInfo = moriio_common.RemoteAllocInfo WriteTask = moriio_common.WriteTask set_role = moriio_common.set_role @@ -440,9 +441,16 @@ def test_write_completion_notifies_once_after_all_sealed_writes_finish(): def _mark_transfer_terminal_locked(self, transfer_id): self._terminal_transfer_ids[transfer_id] = None - def send_notify(self, transfer_id, remote_ip, remote_port, message_type=None): + def send_notify( + self, + transfer_id, + remote_ip, + remote_port, + message_type=None, + message_fields=None, + ): self.notifications.append( - (transfer_id, remote_ip, remote_port, message_type) + (transfer_id, remote_ip, remote_port, message_type, message_fields) ) wrapper = FakeWrapper() @@ -464,8 +472,8 @@ def test_write_completion_notifies_once_after_all_sealed_writes_finish(): writer._mark_write_done("xfer", request_info) writer._finalize_if_complete("xfer", request_info) - assert wrapper.notifications == [("xfer", "127.0.0.1", 7002, "write_done")] - assert wrapper.done_req_ids == ["xfer"] + assert wrapper.notifications == [("xfer", "127.0.0.1", 7002, "write_done", None)] + assert wrapper.done_req_ids == [MoRIIOTransferAck("xfer")] assert wrapper.done_remote_allocate_req_dict == {} assert wrapper.wait_count == 1 assert wrapper.waited_statuses == [["status-a", "status-b"]] @@ -509,7 +517,7 @@ def test_write_failure_marks_terminal_and_clears_scheduled_state(): writer._mark_request_done("xfer") - assert wrapper.done_req_ids == ["xfer"] + assert wrapper.done_req_ids == [MoRIIOTransferAck("xfer")] assert wrapper.done_remote_allocate_req_dict == {} assert wrapper._is_transfer_terminal_locked("xfer") assert "xfer" not in writer._scheduled_writes @@ -581,9 +589,21 @@ def test_late_remote_blocks_message_is_ignored_after_transfer_done(): pytest.param( ROLE.PRODUCER, msgpack.dumps({"type": "release", "transfer_id": "xfer"}), - "release", + MoRIIOTransferAck("xfer"), id="release", ), + pytest.param( + ROLE.PRODUCER, + msgpack.dumps( + { + "type": "release", + "transfer_id": "xfer", + "consumer_tp_size": 8, + } + ), + MoRIIOTransferAck("xfer", 8), + id="release-consumer-tp-size", + ), pytest.param(None, b"xfer", "plain", id="plain-string"), ], ) @@ -603,11 +623,11 @@ def test_moriio_wrapper_routes_valid_messages(role, payload, expected): assert request_info.decode_dp_rank == 3 elif expected == "write_done": assert wrapper.done_write_cache_req_ids == ["xfer"] - elif expected == "release": - assert wrapper.done_req_ids == ["xfer"] - assert wrapper._is_transfer_terminal_locked("xfer") - else: + elif expected == "plain": assert completions == ["xfer"] + else: + assert wrapper.done_req_ids == [expected] + assert wrapper._is_transfer_terminal_locked("xfer") @pytest.mark.parametrize( diff --git a/tests/v1/kv_connector/unit/test_moriio_tp_ack.py b/tests/v1/kv_connector/unit/test_moriio_tp_ack.py new file mode 100644 index 0000000000..9abc9b957c --- /dev/null +++ b/tests/v1/kv_connector/unit/test_moriio_tp_ack.py @@ -0,0 +1,310 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import threading + +import pytest + +from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import ( + MoRIIOMode, + MoRIIOTransferAck, +) +from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import ( + MoRIIOConnectorWorker, + get_moriio_expected_ack_count, + get_moriio_remote_tp_rank, + resolve_moriio_transfer_ack, + validate_moriio_heterogeneous_tp_kv_heads, +) + + +def test_remote_tp_rank_same_tp_maps_to_self(): + assert [get_moriio_remote_tp_rank(rank, 4, 4) for rank in range(4)] == [ + 0, + 1, + 2, + 3, + ] + + +def test_remote_tp_rank_p4_d8_floor_maps_decode_to_prefill(): + assert [get_moriio_remote_tp_rank(rank, 8, 4) for rank in range(8)] == [ + 0, + 0, + 1, + 1, + 2, + 2, + 3, + 3, + ] + + +def test_remote_tp_rank_p8_d4_maps_to_first_prefill_rank_per_pair(): + assert [get_moriio_remote_tp_rank(rank, 4, 8) for rank in range(4)] == [ + 0, + 2, + 4, + 6, + ] + + +@pytest.mark.parametrize( + ("local_tp_rank", "local_tp_size", "remote_tp_size"), + [ + (0, 6, 4), + (0, 4, 6), + ], +) +def test_remote_tp_rank_invalid_non_multiple_tp_raises( + local_tp_rank: int, local_tp_size: int, remote_tp_size: int +): + with pytest.raises(ValueError, match="multiple"): + get_moriio_remote_tp_rank(local_tp_rank, local_tp_size, remote_tp_size) + + +@pytest.mark.parametrize( + ("local_tp_size", "remote_tp_size", "total_num_kv_heads"), + [ + (4, 4, 8), + (8, 4, 4), + (4, 8, 4), + ], +) +def test_heterogeneous_tp_head_guard_allows_supported_layouts( + local_tp_size: int, remote_tp_size: int, total_num_kv_heads: int +): + validate_moriio_heterogeneous_tp_kv_heads( + local_tp_size, + remote_tp_size, + total_num_kv_heads, + is_mla=False, + ) + + +def test_heterogeneous_tp_head_guard_allows_mla_layouts(): + validate_moriio_heterogeneous_tp_kv_heads( + local_tp_size=2, + remote_tp_size=4, + total_num_kv_heads=4, + is_mla=True, + ) + + +@pytest.mark.parametrize( + ("local_tp_size", "remote_tp_size", "total_num_kv_heads"), + [ + (4, 2, 4), + (2, 4, 4), + ], +) +def test_heterogeneous_tp_head_guard_rejects_split_kv_heads( + local_tp_size: int, remote_tp_size: int, total_num_kv_heads: int +): + with pytest.raises(NotImplementedError, match="replicated KV heads"): + validate_moriio_heterogeneous_tp_kv_heads( + local_tp_size, + remote_tp_size, + total_num_kv_heads, + is_mla=False, + ) + + +def test_expected_ack_count_for_homogeneous_or_smaller_consumer_tp_is_one(): + assert get_moriio_expected_ack_count(4, 4) == 1 + assert get_moriio_expected_ack_count(8, 4) == 1 + + +def test_expected_ack_count_for_decode_fan_in(): + assert get_moriio_expected_ack_count(4, 8) == 2 + + +def test_expected_ack_count_rejects_non_multiple_fan_in(): + with pytest.raises(ValueError, match="multiple"): + get_moriio_expected_ack_count(4, 6) + + +def test_plain_string_ack_is_backward_compatible_single_ack(): + notification_counts: dict[str, int] = {} + completed_transfer_ids: set[str] = set() + + assert ( + resolve_moriio_transfer_ack( + "tx-plain", + producer_tp_size=4, + live_transfer_ids={"tx-plain"}, + notification_counts=notification_counts, + completed_transfer_ids=completed_transfer_ids, + ) + == "tx-plain" + ) + assert notification_counts == {} + assert completed_transfer_ids == {"tx-plain"} + + +def test_structured_release_ack_waits_for_all_expected_acks(): + ack = MoRIIOTransferAck("tx-fanin", consumer_tp_size=8) + notification_counts: dict[str, int] = {} + completed_transfer_ids: set[str] = set() + + assert ( + resolve_moriio_transfer_ack( + ack, + producer_tp_size=4, + live_transfer_ids={"tx-fanin"}, + notification_counts=notification_counts, + completed_transfer_ids=completed_transfer_ids, + ) + is None + ) + assert notification_counts == {"tx-fanin": 1} + assert completed_transfer_ids == set() + + assert ( + resolve_moriio_transfer_ack( + ack, + producer_tp_size=4, + live_transfer_ids={"tx-fanin"}, + notification_counts=notification_counts, + completed_transfer_ids=completed_transfer_ids, + ) + == "tx-fanin" + ) + assert notification_counts == {} + assert completed_transfer_ids == {"tx-fanin"} + + +def test_duplicate_ack_after_completion_does_not_resolve_twice(): + ack = MoRIIOTransferAck("tx-dup", consumer_tp_size=8) + notification_counts: dict[str, int] = {} + completed_transfer_ids: set[str] = set() + + assert ( + resolve_moriio_transfer_ack( + ack, + producer_tp_size=4, + live_transfer_ids={"tx-dup"}, + notification_counts=notification_counts, + completed_transfer_ids=completed_transfer_ids, + ) + is None + ) + assert ( + resolve_moriio_transfer_ack( + ack, + producer_tp_size=4, + live_transfer_ids={"tx-dup"}, + notification_counts=notification_counts, + completed_transfer_ids=completed_transfer_ids, + ) + == "tx-dup" + ) + assert ( + resolve_moriio_transfer_ack( + ack, + producer_tp_size=4, + live_transfer_ids={"tx-dup"}, + notification_counts=notification_counts, + completed_transfer_ids=completed_transfer_ids, + ) + is None + ) + assert notification_counts == {} + assert completed_transfer_ids == {"tx-dup"} + + +def test_ack_for_non_live_transfer_is_ignored(): + notification_counts: dict[str, int] = {} + completed_transfer_ids: set[str] = set() + + assert ( + resolve_moriio_transfer_ack( + MoRIIOTransferAck("tx-stale", consumer_tp_size=8), + producer_tp_size=4, + live_transfer_ids={"tx-live"}, + notification_counts=notification_counts, + completed_transfer_ids=completed_transfer_ids, + ) + is None + ) + assert notification_counts == {} + assert completed_transfer_ids == set() + + +def test_worker_get_finished_counts_structured_release_fan_in(): + class FakeWrapper: + def __init__(self): + self.batches = [ + [MoRIIOTransferAck("tx-fanin", consumer_tp_size=8)], + [MoRIIOTransferAck("tx-fanin", consumer_tp_size=8)], + ] + + def pop_finished_req_ids(self): + return self.batches.pop(0) + + def shutdown(self): + pass + + worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker) + worker.is_producer = True + worker.mode = MoRIIOMode.READ + worker.world_size = 4 + worker.moriio_wrapper = FakeWrapper() + worker.transfer_id_to_request_id = {"tx-fanin": "req-fanin"} + worker._consumer_notification_counts = {} + worker._completed_consumer_notifications = set() + + assert worker.get_finished() == (set(), set()) + assert worker._consumer_notification_counts == {"tx-fanin": 1} + + assert worker.get_finished() == ({"req-fanin"}, set()) + assert worker._consumer_notification_counts == {} + assert worker._completed_consumer_notifications == {"tx-fanin"} + + +def test_read_completion_sends_structured_release_with_consumer_tp_size(): + class DoneStatus: + def Succeeded(self): + return True + + def Failed(self): + return False + + class FakeWrapper: + def __init__(self): + self.lock = threading.Lock() + self.sent = [] + + def send_notify( + self, + transfer_id, + host, + port, + message_type=None, + message_fields=None, + ): + self.sent.append((transfer_id, host, port, message_type, message_fields)) + + def shutdown(self): + pass + + worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker) + worker.world_size = 8 + worker.moriio_wrapper = FakeWrapper() + worker._recving_transfers = {"req": [DoneStatus()]} + worker._recving_transfers_callback_addr = { + "req": ("127.0.0.1", "7000", "tx-release") + } + + assert worker._pop_done_transfers() == {"tx-release"} + assert worker.moriio_wrapper.sent == [ + ( + "tx-release", + "127.0.0.1", + "7000", + "release", + {"consumer_tp_size": 8}, + ) + ] + assert worker._recving_transfers == {} + assert worker._recving_transfers_callback_addr == {} diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py index 07fae40942..15585123e5 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py @@ -6,7 +6,7 @@ import threading import time from collections.abc import Iterator from dataclasses import dataclass -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, NamedTuple import msgspec import regex as re @@ -43,6 +43,11 @@ ReqId = str TransferId = str +class MoRIIOTransferAck(NamedTuple): + transfer_id: TransferId + consumer_tp_size: int = 1 + + @dataclass class WriteTask: request_id: ReqId diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py index e119ea7b7d..de21a1398e 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py @@ -6,6 +6,7 @@ import queue import threading import time from collections import defaultdict +from collections.abc import Collection from concurrent.futures import Future, ThreadPoolExecutor from typing import TYPE_CHECKING, Any @@ -30,6 +31,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import ( MoRIIOConnectorMetadata, MoRIIOConstants, MoRIIOMode, + MoRIIOTransferAck, ReqId, ReqMeta, TransferId, @@ -98,6 +100,92 @@ def is_moriio_available() -> bool: return MoRIIO_enabled +def get_moriio_remote_tp_rank( + local_tp_rank: int, local_tp_size: int, remote_tp_size: int +) -> int: + if local_tp_size <= 0 or remote_tp_size <= 0: + raise ValueError("TP sizes must be positive") + if local_tp_rank < 0 or local_tp_rank >= local_tp_size: + raise ValueError( + f"local_tp_rank {local_tp_rank} must be in [0, {local_tp_size})" + ) + if remote_tp_size == local_tp_size: + return local_tp_rank + if remote_tp_size > local_tp_size: + if remote_tp_size % local_tp_size != 0: + raise ValueError( + f"remote tp_size {remote_tp_size} must be a multiple of local " + f"tp_size {local_tp_size} for heterogeneous-TP P/D" + ) + return local_tp_rank * (remote_tp_size // local_tp_size) + if local_tp_size % remote_tp_size != 0: + raise ValueError( + f"local tp_size {local_tp_size} must be a multiple of remote " + f"tp_size {remote_tp_size} for heterogeneous-TP P/D" + ) + return local_tp_rank // (local_tp_size // remote_tp_size) + + +def validate_moriio_heterogeneous_tp_kv_heads( + local_tp_size: int, + remote_tp_size: int, + total_num_kv_heads: int, + is_mla: bool, +) -> None: + if is_mla or local_tp_size == remote_tp_size: + return + if local_tp_size <= 0 or remote_tp_size <= 0 or total_num_kv_heads <= 0: + raise ValueError("TP sizes and total_num_kv_heads must be positive") + if min(local_tp_size, remote_tp_size) >= total_num_kv_heads: + return + raise NotImplementedError( + "MoRIIO heterogeneous TP requires replicated KV heads on both " + f"prefill and decode. Got total_num_kv_heads={total_num_kv_heads}, " + f"local_tp_size={local_tp_size}, remote_tp_size={remote_tp_size}." + ) + + +def get_moriio_expected_ack_count(producer_tp_size: int, consumer_tp_size: int) -> int: + if producer_tp_size <= 0 or consumer_tp_size <= 0: + raise ValueError("TP sizes must be positive") + if consumer_tp_size <= producer_tp_size: + return 1 + if consumer_tp_size % producer_tp_size != 0: + raise ValueError( + f"consumer tp_size {consumer_tp_size} must be a multiple of " + f"producer tp_size {producer_tp_size} for heterogeneous-TP P/D" + ) + return consumer_tp_size // producer_tp_size + + +def resolve_moriio_transfer_ack( + ack: MoRIIOTransferAck | TransferId, + producer_tp_size: int, + live_transfer_ids: Collection[TransferId], + notification_counts: dict[TransferId, int], + completed_transfer_ids: set[TransferId], +) -> TransferId | None: + if isinstance(ack, str): + ack = MoRIIOTransferAck(ack) + transfer_id = ack.transfer_id + if transfer_id not in live_transfer_ids: + return None + if transfer_id in completed_transfer_ids: + return None + + expected_acks = get_moriio_expected_ack_count( + producer_tp_size, ack.consumer_tp_size + ) + count = notification_counts.get(transfer_id, 0) + 1 + if count < expected_acks: + notification_counts[transfer_id] = count + return None + + notification_counts.pop(transfer_id, None) + completed_transfer_ids.add(transfer_id) + return transfer_id + + class MoRIIOConnector(KVConnectorBase_V1): def __init__( self, @@ -798,6 +886,11 @@ class MoRIIOConnectorWorker: # Completions that arrived before transfer_id_to_request_id was populated. # Retried each step until the mapping is established. self._unmatched_write_completions: set[str] = set() + # Producer-side READ-mode ACK fan-in. When decode TP is larger than + # prefill TP, multiple decode ranks can read from one prefill rank and + # notify the same transfer_id. Blocks are reusable only after all ACKs. + self._consumer_notification_counts: dict[TransferId, int] = {} + self._completed_consumer_notifications: set[TransferId] = set() role = "producer" if self.is_producer else "consumer" engine_suffix = ( @@ -1161,7 +1254,9 @@ class MoRIIOConnectorWorker: # a hack to keep us moving. We will switch when moving to etcd # or where we have a single ZMQ socket in the scheduler. - port_offset = get_port_offset(remote_dp_rank, self.tp_rank) + port_offset = get_port_offset( + remote_dp_rank, self._remote_tp_rank(remote_tp_size) + ) path = make_zmq_path("tcp", host, port + port_offset) logger.debug("handshake Querying metadata on path: %s", path) @@ -1226,6 +1321,9 @@ class MoRIIOConnectorWorker: return {remote_agent_name} + def _remote_tp_rank(self, remote_tp_size: int) -> int: + return get_moriio_remote_tp_rank(self.tp_rank, self.world_size, remote_tp_size) + def _background_moriio_handshake( self, req_id: ReqId, remote_engine_id: EngineId, meta: ReqMeta ): @@ -1435,13 +1533,32 @@ class MoRIIOConnectorWorker: done_sending, done_recving = set(), set() if self.is_producer: - # pop_finished_req_ids returns transfer_ids (the ZMQ payload sent - # by decode via send_notify); map back to req_ids for the scheduler. - finished_transfer_ids = self.moriio_wrapper.pop_finished_req_ids() + # pop_finished_req_ids returns release ACKs sent by decode. Keep + # duplicate ACKs because heterogeneous TP can fan multiple decode + # ranks into one prefill rank for the same transfer_id. + finished_acks = self.moriio_wrapper.pop_finished_req_ids() + resolved_transfer_ids: set[TransferId] = set() + for ack in finished_acks: + transfer_id = ack if isinstance(ack, str) else ack.transfer_id + if transfer_id not in self.transfer_id_to_request_id: + logger.warning( + "Could not find %s in transfer_id_to_request_id " + "lookup table. This could lead to a possible hang.", + transfer_id, + ) + continue + resolved_transfer_id = resolve_moriio_transfer_ack( + ack, + producer_tp_size=self.world_size, + live_transfer_ids=self.transfer_id_to_request_id.keys(), + notification_counts=self._consumer_notification_counts, + completed_transfer_ids=(self._completed_consumer_notifications), + ) + if resolved_transfer_id is not None: + resolved_transfer_ids.add(resolved_transfer_id) done_sending = { self.transfer_id_to_request_id[xfer_id] - for xfer_id in finished_transfer_ids - if xfer_id in self.transfer_id_to_request_id + for xfer_id in resolved_transfer_ids } else: if self.mode == MoRIIOMode.WRITE: @@ -1486,7 +1603,13 @@ class MoRIIOConnectorWorker: if last.Succeeded(): host, port, xfer_id = self._recving_transfers_callback_addr[req_id] done_req_ids.add(xfer_id) - self.moriio_wrapper.send_notify(xfer_id, host, port) + self.moriio_wrapper.send_notify( + xfer_id, + host, + port, + message_type="release", + message_fields={"consumer_tp_size": self.world_size}, + ) to_remove.append(req_id) elif last.Failed(): logger.error( @@ -1499,7 +1622,13 @@ class MoRIIOConnectorWorker: ) host, port, xfer_id = self._recving_transfers_callback_addr[req_id] try: - self.moriio_wrapper.send_notify(xfer_id, host, port) + self.moriio_wrapper.send_notify( + xfer_id, + host, + port, + message_type="release", + message_fields={"consumer_tp_size": self.world_size}, + ) except Exception: logger.exception( "Failed to send error notification for request %s", @@ -1585,6 +1714,15 @@ class MoRIIOConnectorWorker: """ self.transfer_id_to_request_id = metadata.transfer_id_to_request_id if self.is_producer: + live_transfer_ids = set(self.transfer_id_to_request_id) + self._consumer_notification_counts = { + transfer_id: count + for transfer_id, count in self._consumer_notification_counts.items() + if transfer_id in live_transfer_ids + } + self._completed_consumer_notifications.intersection_update( + live_transfer_ids + ) self.moriio_wrapper.async_wait_reqid() return if self.mode == MoRIIOMode.WRITE: @@ -1663,6 +1801,7 @@ class MoRIIOConnectorWorker: remote_block_ids=meta.remote_block_ids, remote_host=meta.remote_host, remote_notify_port=meta.remote_notify_port, + remote_tp_size=meta.tp_size, ) def _write_blocks_for_req(self, req_id: ReqId, meta: ReqMeta, layer_name, kv_layer): @@ -1756,6 +1895,7 @@ class MoRIIOConnectorWorker: local_block_ids: list[int], remote_block_ids: list[int], remote_moriio_meta: MoRIIOAgentMetadata, + remote_tp_size: int | None = None, ) -> tuple[list[int], list[int], list[int]]: """Compute transfer offsets for block data. @@ -1767,6 +1907,14 @@ class MoRIIOConnectorWorker: Returns: Tuple of (local_offsets, remote_offsets, transfer_sizes) """ + validate_moriio_heterogeneous_tp_kv_heads( + local_tp_size=self.world_size, + remote_tp_size=( + remote_tp_size if remote_tp_size is not None else self.world_size + ), + total_num_kv_heads=self.model_config.get_total_num_kv_heads(), + is_mla=self._is_mla_cache_layer(layer_name), + ) return compute_block_transfer_offsets( layer_name=layer_name, kv_cache=self.kv_caches[layer_name], @@ -1788,6 +1936,7 @@ class MoRIIOConnectorWorker: transfer_id: str, remote_host: str, remote_notify_port: int, + remote_tp_size: int, ) -> None: if self.mode == MoRIIOMode.WRITE: return @@ -1800,7 +1949,11 @@ class MoRIIOConnectorWorker: layer_name ) offs = self._compute_block_transfer_offsets( - layer_name, local_block_ids, remote_block_ids, remote_moriio_meta + layer_name, + local_block_ids, + remote_block_ids, + remote_moriio_meta, + remote_tp_size=remote_tp_size, ) # TODO : apply multi-session batch-read when moriio support it transfer_status = self.moriio_wrapper.read_remote_data( @@ -1810,6 +1963,6 @@ class MoRIIOConnectorWorker: self._recving_transfers[request_id].append(transfer_status) self._recving_transfers_callback_addr[request_id] = ( remote_host, - str(remote_notify_port + self.tp_rank), + str(remote_notify_port + self._remote_tp_rank(remote_tp_size)), transfer_id, ) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_engine.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_engine.py index 3a90151f7e..5814fc8335 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_engine.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_engine.py @@ -27,6 +27,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import ( MoRIIOAgentMetadata, MoRIIOConstants, MoRIIOError, + MoRIIOTransferAck, RemoteAllocInfo, TransferError, TransferId, @@ -261,7 +262,7 @@ class MoRIIOWriter: """Mark a request done so its blocks are freed, even on transfer failure.""" wrapper = self.worker.moriio_wrapper with wrapper.lock: - wrapper.done_req_ids.append(transfer_id) + wrapper.done_req_ids.append(MoRIIOTransferAck(transfer_id)) wrapper.done_remote_allocate_req_dict.pop(transfer_id, None) wrapper._mark_transfer_terminal_locked(transfer_id) self._clear_transfer_state(transfer_id) @@ -466,7 +467,9 @@ class MoRIIOWriter: ) # mark request as done, then we can free the blocks with self.worker.moriio_wrapper.lock: - self.worker.moriio_wrapper.done_req_ids.append(transfer_id) + self.worker.moriio_wrapper.done_req_ids.append( + MoRIIOTransferAck(transfer_id) + ) self.worker.moriio_wrapper.done_remote_allocate_req_dict.pop( transfer_id, None ) @@ -508,7 +511,7 @@ class MoRIIOWrapper: self.remote_engine_ip: str | None = None self.notify_port: int | None = None self.lock = threading.Lock() - self.done_req_ids: list[str] = [] + self.done_req_ids: list[MoRIIOTransferAck] = [] self.done_remote_allocate_req_dict: dict[TransferId, RemoteAllocInfo] = {} self.done_write_cache_req_ids: list[str] = [] self._terminal_transfer_ids: OrderedDict[TransferId, None] = OrderedDict() @@ -784,15 +787,20 @@ class MoRIIOWrapper: "Only prefill can get transfer release messages" ) transfer_id = data["transfer_id"] + consumer_tp_size = int(data.get("consumer_tp_size", 1)) + if consumer_tp_size <= 0: + raise MoRIIOError( + f"Invalid consumer_tp_size in release message: {consumer_tp_size}" + ) with self.lock: - self.done_req_ids.append(transfer_id) + self.done_req_ids.append(MoRIIOTransferAck(transfer_id, consumer_tp_size)) self.done_remote_allocate_req_dict.pop(transfer_id, None) self._mark_transfer_terminal_locked(transfer_id) def _handle_completion_message(self, msg: str): with self.lock: if get_role() == ROLE.PRODUCER: - self.done_req_ids.append(msg) + self.done_req_ids.append(MoRIIOTransferAck(msg)) self.done_remote_allocate_req_dict.pop(msg, None) self._mark_transfer_terminal_locked(msg) else: @@ -808,7 +816,12 @@ class MoRIIOWrapper: self._terminal_transfer_ids.popitem(last=False) def send_notify( - self, req_ids, remote_ip, remote_port, message_type: str | None = None + self, + req_ids, + remote_ip, + remote_port, + message_type: str | None = None, + message_fields: dict[str, Any] | None = None, ): if not remote_ip or not remote_port: logger.warning("Missing remote_ip or remote_port for notification") @@ -836,18 +849,21 @@ class MoRIIOWrapper: if message_type is None: sock.send(req_id.encode("utf-8")) else: - sock.send( - msgpack.dumps({"type": message_type, "transfer_id": req_id}) - ) + payload = {"type": message_type, "transfer_id": req_id} + if message_fields: + payload.update(message_fields) + sock.send(msgpack.dumps(payload)) except Exception as e: logger.error("Failed to send notification to %s: %s", path, e) self.paths.pop(path, None) raise def pop_finished_req_ids(self): - # producer invocation: get the set of completed requests at the decode + # Producer invocation: return every completion message since the last + # call. Do not dedupe: heterogeneous TP can produce multiple release + # ACKs for the same transfer_id and the caller must count each one. with self.lock: - done_send = set(self.done_req_ids) + done_send = list(self.done_req_ids) self.done_req_ids = [] return done_send