llmd+vllm+mori-ep(inter node wide-ep)+mori-io(write) for 2p2d with dp=ep=16 tp=1 (#45043)

Signed-off-by: Shiksha Patel <shikpate@amd.com>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
shikamd123
2026-08-04 00:13:15 +00:00
committed by GitHub
co-authored by Douglas Lehr mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent c810937573
commit f42761204f
8 changed files with 710 additions and 120 deletions
@@ -1,5 +1,20 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#
# Minimal single-node reference proxy for MoRI-IO prefill/decode disaggregation.
# It demonstrates the DP-rank pinning contract the connector expects: pick one
# prefill DP rank per request (flat_interleaved_dp_route) and pin BOTH legs to it
# via the ``X-data-parallel-rank`` header plus ``kv_transfer_params``
# (``remote_dp_rank`` / ``remote_dp_size`` / ``remote_tp_size``).
#
# Scope: single-node only. It intentionally does NOT supply the cross-pod
# Wide-EP (2P2D) peer info -- ``remote_hosts`` / ``multi_pod_hosts``,
# ``data_parallel_size_local``, ``is_request_leader``. In real multi-pod
# deployments that peer info is produced by a production routing sidecar, e.g.:
# * llm-d-router (https://github.com/llm-d/llm-d-router)
# * vLLM's own vllm-router (examples in the vllm-project org)
# * or any NIXL-shaped router that sets the same kv_transfer_params contract.
# Use this file as a protocol reference, not a production router.
import argparse
import asyncio
import copy
@@ -181,7 +196,7 @@ async def send_request_to_prefill(
raise RuntimeError(error_message)
async def start_decode_request(endpoint, req_data, request_id):
async def start_decode_request(endpoint, req_data, request_id, selected_dp_rank=None):
session = aiohttp.ClientSession(
timeout=aiohttp.ClientTimeout(total=6 * 6000 * 6000)
)
@@ -189,6 +204,10 @@ async def start_decode_request(endpoint, req_data, request_id):
"Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}",
"X-Request-Id": request_id,
}
# Pin the decode leg to the same DP rank as its prefill leg so both land on
# the same rank (vLLM reads this header directly; no engine-side routing).
if selected_dp_rank is not None:
headers["X-data-parallel-rank"] = str(selected_dp_rank)
response = await session.post(url=endpoint, json=req_data, headers=headers)
return session, response
@@ -361,7 +380,9 @@ async def handle_request(api: str, request: Request):
decode_request_url = decode_instance_endpoint["request_address"] + api
decode_request_task = asyncio.create_task(
start_decode_request(decode_request_url, req_data, request_id)
start_decode_request(
decode_request_url, req_data, request_id, selected_prefill_dp_rank
)
)
session, decode_response = await decode_request_task
@@ -508,9 +508,15 @@ def test_send_transfer_release_sends_structured_release_message():
scheduler._send_transfer_release("xfer-7", "127.0.0.1", 7000)
payload = sock.send.call_args.args[0]
# WRITE-mode release advertises the consumer (decode) TP size so the prefill
# side counts the right number of ACKs via get_moriio_expected_ack_count,
# mirroring the READ-mode release in _pop_done_transfers (see
# test_read_completion_sends_structured_release_with_consumer_tp_size). The
# fixture's tp_size is 2, so consumer_tp_size == 2.
assert msgspec.msgpack.decode(payload) == {
"type": "release",
"transfer_id": "xfer-7",
"consumer_tp_size": 2,
}
@@ -0,0 +1,92 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for MoRIIO router-authoritative DP-rank routing.
These lock in the redesign that addresses the PR #45043 review feedback
(njhill): the connector must NOT self-derive a DP rank, and the prefill leg
must *return* the rank it actually ran on via ``request_finished`` so the
decode->prefill notify is routed by pure propagation (READ / serial WRITE),
not by an independent hash on each side.
Following test_moriio_unmap.py, we bind the unbound method to a lightweight
stand-in rather than constructing a full scheduler.
"""
from types import SimpleNamespace
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import (
MoRIIOConnectorScheduler,
)
from vllm.v1.request import RequestStatus
_request_finished = MoRIIOConnectorScheduler.request_finished
def _producer(global_dp_rank: int, dp_size: int, dp_size_local: int):
"""Minimal prefill-side (producer) scheduler stand-in for request_finished."""
return SimpleNamespace(
is_producer=True,
_global_dp_rank=global_dp_rank,
engine_id="engine-abc",
host_ip="10.0.0.1",
handshake_port=6300,
side_notify_port=7300,
vllm_config=SimpleNamespace(
parallel_config=SimpleNamespace(
data_parallel_size=dp_size,
data_parallel_size_local=dp_size_local,
tensor_parallel_size=1,
)
),
)
def _req(
request_id: str, params: dict | None, status=RequestStatus.FINISHED_LENGTH_CAPPED
):
return SimpleNamespace(
request_id=request_id,
kv_transfer_params=params,
status=status,
)
def test_request_finished_returns_prefill_rank_for_propagation():
# 2P2D: global rank 11 lives on pod 1 (ranks 8..15), local size 8.
s = _producer(global_dp_rank=11, dp_size=16, dp_size_local=8)
req = _req("r1", {"do_remote_decode": True, "transfer_id": "t1"})
delay_free, params = _request_finished(s, req, block_ids=[])
# The prefill leg returns the rank IT actually ran on, marked authoritative,
# so the router forwards it to decode and the notify propagates (no hashing).
assert params is not None
assert params["remote_dp_rank"] == 11
assert params["remote_dp_rank_override"] is True
assert params["remote_dp_size"] == 16
assert params["remote_dp_size_local"] == 8
assert params["transfer_id"] == "t1"
# Empty block_ids => nothing deferred, blocks freed immediately.
assert delay_free is False
def test_request_finished_rank_tracks_the_actual_rank():
# A different rank must be echoed verbatim -- the value is the producer's
# own rank, never a recomputation of the request id.
s = _producer(global_dp_rank=4, dp_size=8, dp_size_local=8)
req = _req("r2", {"do_remote_decode": True, "transfer_id": "t2"})
_, params = _request_finished(s, req, block_ids=[])
assert params["remote_dp_rank"] == 4
assert params["remote_dp_size_local"] == 8
def test_request_finished_no_kv_params_is_noop():
s = _producer(global_dp_rank=0, dp_size=16, dp_size_local=8)
req = _req("r3", None)
delay_free, params = _request_finished(s, req, block_ids=[])
assert delay_free is False
assert params is None
@@ -296,6 +296,8 @@ def test_read_completion_sends_structured_release_with_consumer_tp_size():
worker._recving_transfers_callback_addr = {
"req": ("127.0.0.1", "7000", "tx-release")
}
# Transfer-timeout reaping state consulted by _pop_done_transfers.
worker._recving_transfers_start = {}
assert worker._pop_done_transfers() == {"tx-release"}
assert worker.moriio_wrapper.sent == [
@@ -0,0 +1,68 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for MoRIIOConnectorScheduler.{map,unmap}_request_id.
vLLM's input_processor mutates a request_id after the connector has stored it
(e.g. ``cmpl-abc-0`` -> ``cmpl-abc-0-956053a4``), so an exact-match unmap can
miss. These tests cover the stable-``transfer_id`` fallback that handles that
case (and the clean exact-match and miss paths).
The map/unmap methods touch only the two id<->id dicts, so we bind them to a
lightweight stand-in rather than constructing a full scheduler.
"""
from types import SimpleNamespace
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import (
MoRIIOConnectorScheduler,
)
_map = MoRIIOConnectorScheduler.map_request_id
_unmap = MoRIIOConnectorScheduler.unmap_request_id
def _sched():
return SimpleNamespace(
transfer_id_to_request_id={},
request_id_to_transfer_id={},
)
def test_exact_match_unmap_clears_both_tables():
s = _sched()
_map(s, "rid-1", "tid-1")
_unmap(s, "rid-1")
assert s.request_id_to_transfer_id == {}
assert s.transfer_id_to_request_id == {}
def test_unmap_via_transfer_id_after_request_id_mutation():
s = _sched()
# Connector stores the ORIGINAL request_id at map time.
_map(s, "cmpl-abc-0", "tid-42")
# input_processor later mutates the rid; exact match would miss, so the
# stable transfer_id is used to find and clear the original entry.
_unmap(s, "cmpl-abc-0-956053a4", transfer_id="tid-42")
assert s.request_id_to_transfer_id == {}
assert s.transfer_id_to_request_id == {}
def test_unmap_miss_is_noop_and_does_not_raise():
s = _sched()
_map(s, "rid-1", "tid-1")
# Unknown rid and unknown/None transfer_id: must not raise, must not touch
# the existing (unrelated) mapping.
_unmap(s, "rid-unknown", transfer_id=None)
_unmap(s, "rid-unknown", transfer_id="tid-unknown")
assert s.request_id_to_transfer_id == {"rid-1": "tid-1"}
assert s.transfer_id_to_request_id == {"tid-1": "rid-1"}
def test_exact_match_preferred_over_transfer_id_fallback():
s = _sched()
_map(s, "rid-1", "tid-1")
# Exact rid is present, so it is removed directly even if a transfer_id is
# also supplied; no stale entries left behind.
_unmap(s, "rid-1", transfer_id="tid-1")
assert s.request_id_to_transfer_id == {}
assert s.transfer_id_to_request_id == {}
@@ -191,6 +191,25 @@ def get_port_offset(dp_rank: int, tp_rank: int, tp_size: int = 1) -> int:
return (dp_rank) * tp_size + tp_rank
def fold_local_rank(global_dp_rank: int, dp_size_local: int) -> int:
"""Fold a global DP rank into its pod-local rank [0, dp_size_local).
``dp_size_local == 0`` is the external-DP sentinel (local size unknown):
return the rank unchanged, since a global DP rank is always < the global
DP size so no folding is needed and the modulo is skipped (never divides
by zero).
"""
return global_dp_rank % dp_size_local if dp_size_local else global_dp_rank
def pod_index(global_dp_rank: int, dp_size_local: int) -> int:
"""Pod index (0-based) a global DP rank lives on for Wide-EP multi-pod.
``dp_size_local == 0`` (external-DP sentinel) collapses to a single pod.
"""
return global_dp_rank // dp_size_local if dp_size_local else 0
def resolve_host_ip(extra_config: dict) -> str:
"""The IP this MoRIIO process advertises for KV transfer.
@@ -280,9 +299,11 @@ class MoRIIOConfig:
kv_transfer_config = vllm_config.kv_transfer_config
extra_config = kv_transfer_config.kv_connector_extra_config
tp_rank = get_tensor_model_parallel_rank()
dp_rank = vllm_config.parallel_config.data_parallel_rank
# Fold the global data_parallel_rank back to [0, dp_size_local) for
# per-node port allocation (handles the external-DP sentinel).
pc = vllm_config.parallel_config
dp_rank = fold_local_rank(pc.data_parallel_rank, pc.data_parallel_size_local)
base_notify_port = int(extra_config["notify_port"])
dp_size = vllm_config.parallel_config.data_parallel_size
tp_size = get_tensor_model_parallel_world_size()
port_offset = get_port_offset(dp_rank, tp_rank)
backend = str(extra_config.get("backend", "rdma")).lower()
@@ -304,15 +325,18 @@ class MoRIIOConfig:
return cls(
local_ip=resolve_host_ip(extra_config),
local_kv_port=get_open_port(),
proxy_ip=extra_config["proxy_ip"],
proxy_ip=extra_config.get("proxy_ip", ""),
local_ping_port=get_open_port(),
proxy_ping_port=int(extra_config["proxy_ping_port"]),
proxy_ping_port=int(extra_config.get("proxy_ping_port", 0)),
http_port=int(extra_config["http_port"]),
handshake_port=int(extra_config["handshake_port"]),
notify_port=base_notify_port + port_offset,
tp_rank=tp_rank,
dp_rank=dp_rank,
dp_size=dp_size,
# Advertise the GLOBAL DP world size to the routing proxy so it can
# address every rank across pods; dp_size_local above is only for
# per-pod port folding, not for routing.
dp_size=pc.data_parallel_size,
tp_size=tp_size,
read_mode=get_moriio_mode(kv_transfer_config) == MoRIIOMode.READ,
qp_per_transfer=int(extra_config.get("qp_per_transfer", 1)),
@@ -391,20 +415,26 @@ def parse_moriio_zmq_address(
return host, handshake_port, notify_port
def get_peer_zmq_from_request_id(request_id: str, is_producer: bool) -> str:
def get_peer_zmq_from_request_id(request_id: str, is_producer: bool) -> str | None:
"""Extract the *peer's* zmq_address from the vLLM router request_id.
The producer (prefill) needs the decode's address; the consumer (decode)
needs the prefill's address.
Returns ``None`` when the request_id does not encode peer info. The
llm-d routing sidecar (``llm-d-inference-scheduler``) does not embed
addresses in ``request_id``; instead it passes ``remote_host``,
``remote_handshake_port`` and ``remote_notify_port`` explicitly in
``kv_transfer_params``. Callers must handle the ``None`` return by
falling back to those fields. See ``add_new_req`` for the canonical
fallback path.
"""
if is_producer:
m = _DECODE_ZMQ_RE.search(request_id)
else:
m = _PREFILL_ZMQ_RE.search(request_id)
if m is None:
raise ValueError(
f"Cannot parse peer zmq_address from request_id: {request_id!r}"
)
return None
return m.group(1)
@@ -426,6 +456,10 @@ class ReqMeta:
# read must target this rank's memory registration; the default 0 preserves
# the symmetric single-DP behaviour.
remote_dp_rank: int = 0
# Multi-pod: list of remote pod IPs indexed by pod_idx.
multi_pod_hosts: list[str] = field(default_factory=list)
# Per-pod DP size; 0 means fallback to remote_dp_size.
remote_dp_size_local: int = 0
class MoRIIOConnectorMetadata(KVConnectorMetadata):
@@ -450,23 +484,76 @@ class MoRIIOConnectorMetadata(KVConnectorMetadata):
kv_transfer_params: dict[str, Any],
write_mode=False,
):
"""Ingest a peer's ``kv_transfer_params`` into a typed ``ReqMeta``.
This is the single place peer info enters the connector. The
``kv_transfer_params`` contract (as produced by the llm-d sidecar /
vLLM router, or echoed by the prefill leg's ``request_finished``):
Required (always):
* ``transfer_id`` -- stable id shared by both legs.
* ``remote_engine_id`` -- peer engine id for the handshake table.
* ``remote_block_ids`` -- peer block ids (may be [] in WRITE mode,
where decode allocates its own blocks).
Peer address -- ONE of the following two must resolve:
* embedded in ``request_id`` (vLLM-router PD id form), OR
* explicit ``remote_host`` + ``remote_handshake_port`` +
``remote_notify_port`` (llm-d sidecar / returnable path).
If neither resolves we raise -- there is no safe default host/port.
Optional (defaulted):
* ``tp_size`` (default 1) -- peer TP size.
* ``remote_dp_size`` (default 1) -- peer GLOBAL DP size.
* ``remote_dp_size_local`` (default = ``remote_dp_size``) -- per-pod
DP size for Wide-EP multi-pod port/host
folding; 0/absent means single-pod.
* ``remote_hosts`` (default [remote_host]) -- per-pod IP list
indexed by ``pod_idx = rank // dp_local``.
Routing keys consumed elsewhere (NOT here): ``remote_dp_rank`` /
``remote_dp_rank_override`` gate the decode->prefill notify target in
MoRIIOConnectorScheduler; they are router-authoritative and never
self-derived (see that class's request routing contract).
"""
transfer_id = kv_transfer_params["transfer_id"]
remote_host = kv_transfer_params.get("remote_host")
remote_handshake_port = kv_transfer_params.get("remote_handshake_port")
remote_notify_port = kv_transfer_params.get("remote_notify_port")
if (
remote_host is None
or remote_handshake_port is None
or remote_notify_port is None
):
# Parse host/ports from the request_id. The router embeds both
# zmq_addresses in PD request IDs, but WRITE decode requests may carry
# a plain request ID and get the remote address via kv_transfer_params.
peer_zmq = get_peer_zmq_from_request_id(request_id, is_producer=write_mode)
# Try request_id embedded address first, fallback to explicit params.
peer_zmq = get_peer_zmq_from_request_id(request_id, is_producer=write_mode)
if peer_zmq is not None:
remote_host, remote_handshake_port, remote_notify_port = (
parse_moriio_zmq_address(peer_zmq)
)
else:
try:
remote_host = kv_transfer_params["remote_host"]
if not remote_host:
raise ValueError(
f"request_id {request_id!r} does not embed a peer "
f"zmq_address and kv_transfer_params['remote_host'] is "
f"empty; cannot route MoRI-IO transfer"
)
remote_handshake_port = int(kv_transfer_params["remote_handshake_port"])
remote_notify_port = int(kv_transfer_params["remote_notify_port"])
except (KeyError, TypeError, ValueError) as e:
raise ValueError(
f"request_id {request_id!r} does not embed a peer "
f"zmq_address and kv_transfer_params is missing one or "
f"more sidecar-fallback keys (need remote_host, "
f"remote_handshake_port, remote_notify_port): {e}"
) from e
# Multi-pod: use multi_pod_hosts list or fallback to single host.
_pod_hosts = kv_transfer_params.get("remote_hosts") or [remote_host]
if not isinstance(_pod_hosts, list):
_pod_hosts = [_pod_hosts]
_pod_hosts = [str(h) for h in _pod_hosts]
_remote_dp_size_local = int(
kv_transfer_params.get(
"remote_dp_size_local",
kv_transfer_params.get("remote_dp_size", 1),
)
)
_req = ReqMeta(
transfer_id=transfer_id,
@@ -488,6 +575,8 @@ class MoRIIOConnectorMetadata(KVConnectorMetadata):
),
remote_dp_size=kv_transfer_params.get("remote_dp_size", 1),
remote_dp_rank=kv_transfer_params.get("remote_dp_rank", 0),
multi_pod_hosts=_pod_hosts,
remote_dp_size_local=_remote_dp_size_local,
)
if write_mode:
self.reqs_to_save[request_id] = _req
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import logging
import math
import os
import queue
import threading
import time
@@ -36,11 +37,13 @@ from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import (
ReqMeta,
TransferId,
WriteTask,
fold_local_rank,
get_moriio_mode,
get_peer_zmq_from_request_id,
get_port_offset,
get_role,
parse_moriio_zmq_address,
pod_index,
resolve_host_ip,
set_role,
zmq_ctx,
@@ -373,13 +376,29 @@ class MoRIIOConnectorScheduler:
"notify_port"
]
self.tp_size = self.vllm_config.parallel_config.tensor_parallel_size
self.dp_rank = self.vllm_config.parallel_config.data_parallel_rank
# Local DP rank (0..dp_local-1) for port arithmetic; fold_local_rank
# handles the external-DP sentinel (dp_local==0) so the arithmetic
# can't ZeroDivisionError at init.
_pc = self.vllm_config.parallel_config
self.dp_rank = fold_local_rank(
_pc.data_parallel_rank, _pc.data_parallel_size_local
)
# Only first-pod ranks originate notify to avoid duplicates. Guarding
# via pod_index keeps _is_kv_master True for the external-DP sentinel
# (single pod) instead of being silently stuck False.
self._is_kv_master = (
pod_index(_pc.data_parallel_rank, _pc.data_parallel_size_local) == 0
)
# Global DP rank for pinned request ownership check.
self._global_dp_rank = self.vllm_config.parallel_config.data_parallel_rank
self.is_producer = self.kv_transfer_config.kv_role == "kv_producer"
# Requests that need to start recv/send.
# New requests are added by update_state_after_alloc in
# the scheduler. Used to make metadata passed to Worker.
self._reqs_need_recv: dict[ReqId, tuple[Request, list[int]]] = {}
self._reqs_need_save: dict[ReqId, tuple[Request, list[int]]] = {}
# Snapshot of kv_transfer_params for chunked prefill recovery.
self._req_kv_params: dict[ReqId, dict] = {}
# For chunked prefill, we perform layer-wise access within the final chunk.
# TODO: Perform transfer at end chunk.
@@ -396,12 +415,15 @@ class MoRIIOConnectorScheduler:
# finished_sending before the deadline, we inject them into
# connector_output.finished_sending so the scheduler frees the blocks to avoid
# hanging indefinitely waiting for a free notification that never comes.
self._deferred_send_deadlines: dict[ReqId, float] = {}
# Value: (deadline, transfer_id) for unmapping after mutation.
self._deferred_send_deadlines: dict[ReqId, tuple[float, TransferId | None]] = {}
self._defer_timeout = float(
self.kv_transfer_config.kv_connector_extra_config.get(
"defer_timeout", MoRIIOConstants.DEFAULT_DEFER_TIMEOUT
)
)
# Buffer for early ACKs that arrive before request_finished.
self._pending_sent_acks: dict[ReqId, float] = {}
self.paths: dict[str, zmq.Socket] = {}
self.transfer_id_to_request_id: dict[TransferId, ReqId] = {}
self.request_id_to_transfer_id: dict[ReqId, TransferId] = {}
@@ -410,23 +432,36 @@ class MoRIIOConnectorScheduler:
self.transfer_id_to_request_id[transfer_id] = request_id
self.request_id_to_transfer_id[request_id] = transfer_id
def unmap_request_id(self, request_id: ReqId):
def unmap_request_id(
self, request_id: ReqId, transfer_id: TransferId | None = None
):
"""Unmap request_id/transfer_id. Uses transfer_id for lookup if
exact request_id match fails (handles input_processor mutation)."""
if request_id in self.request_id_to_transfer_id:
transfer_id = self.request_id_to_transfer_id[request_id]
tid = self.request_id_to_transfer_id[request_id]
del self.request_id_to_transfer_id[request_id]
if transfer_id in self.transfer_id_to_request_id:
del self.transfer_id_to_request_id[transfer_id]
else:
logger.warning(
"transfer id not in transfer_id_to_request_id lookup"
"table. there is likely a bug!"
if tid in self.transfer_id_to_request_id:
del self.transfer_id_to_request_id[tid]
return
# Fallback: use transfer_id to find original request_id.
if transfer_id is not None and transfer_id in self.transfer_id_to_request_id:
original_rid = self.transfer_id_to_request_id[transfer_id]
if original_rid != request_id:
logger.debug(
"MoRI-IO unmap via transfer_id: %r -> %r", request_id, original_rid
)
else:
logger.warning(
"Could not find %s in transfer_id_to_request_id"
"lookup table. This could lead to a possible hang.",
request_id,
)
if original_rid in self.request_id_to_transfer_id:
del self.request_id_to_transfer_id[original_rid]
del self.transfer_id_to_request_id[transfer_id]
return
logger.warning(
"MoRI-IO unmap MISS: rid=%r transfer_id=%r table_size=%d",
request_id,
transfer_id,
len(self.request_id_to_transfer_id),
)
def get_num_new_matched_tokens(
self,
@@ -478,7 +513,12 @@ class MoRIIOConnectorScheduler:
"req_id": req_id,
"transfer_id": transfer_id,
"block_notify_list": block_notify_list or [],
"decode_rank": self.dp_rank,
# GLOBAL decode dp rank: producer derives the per-pod notify offset
# (% dp_local), owning pod index (// dp_local), and write-target
# from it. Sending the LOCAL rank made child-pod consumers look
# like master rank 0 and hang in WAITING_FOR_REMOTE_KVS. Single-pod:
# local == global.
"decode_rank": self._global_dp_rank,
"type": "remote_blocks",
}
serialized_data = msgpack.dumps(data)
@@ -493,8 +533,18 @@ class MoRIIOConnectorScheduler:
)
self.paths[path] = sock
# Advertise the consumer (decode) TP size so prefill counts the right
# number of ACKs (get_moriio_expected_ack_count; see the upstream
# READ-mode release in _pop_done_transfers). Resolves to 1 for our
# homogeneous TP=1 configs; only matters under heterogeneous-TP fan-in.
self.paths[path].send(
msgpack.dumps({"type": "release", "transfer_id": transfer_id})
msgpack.dumps(
{
"type": "release",
"transfer_id": transfer_id,
"consumer_tp_size": self.tp_size,
}
)
)
def _release_write_prefill_blocks(self, request_id: ReqId, params: dict[str, Any]):
@@ -513,6 +563,8 @@ class MoRIIOConnectorScheduler:
if remote_host is None or remote_notify_port is None:
try:
peer_zmq = get_peer_zmq_from_request_id(request_id, is_producer=False)
if peer_zmq is None:
raise ValueError("no peer zmq address for request")
remote_host, _, remote_notify_port = parse_moriio_zmq_address(peer_zmq)
except ValueError:
logger.warning(
@@ -534,15 +586,54 @@ class MoRIIOConnectorScheduler:
num_external_tokens: int,
connector_worker: "MoRIIOConnectorWorker | None" = None,
):
"""Scheduler-side post-allocation hook (decode leg in WRITE mode).
In WRITE mode this fires the decode->prefill "blocks ready" notify
that lets the producer RDMA-Write its KV into the freshly allocated
decode blocks.
DP-rank routing contract (router-authoritative)
------------------------------------------------
Both legs of a disagg pair must agree on a single prefill DP rank,
otherwise the notify lands on a rank that never handshook and the
request hangs until ``VLLM_MORIIO_DEFERRED_TIMEOUT_S``.
The routing authority is the external router/sidecar, NOT the
connector. The llm-d routing sidecar computes ``H = pickDPRank(uuid,
dp_size)`` (see ``dp_rank.go``) once and pins BOTH legs to ``H`` two
ways: the ``X-data-parallel-rank`` dispatch header (which vLLM engine
the leg lands on) and ``kv_transfer_params.remote_dp_rank=H`` (which
prefill rank this notify targets), with ``remote_dp_rank_override=True``.
The connector consumes ``remote_dp_rank`` verbatim and does NOT
self-derive a rank: an independent hash here could disagree with the
router's dispatch pin and misroute the notify. For the returnable paths
(READ / serial WRITE) the prefill leg also echoes back the rank it ran
on via ``request_finished`` so routing is pure propagation.
The owning rank is the only one that originates the notify
(``remote_dp_rank_override`` -> global-rank match; otherwise the
``_is_kv_master`` anti-duplicate gate). See the inline comments
below for the exactly-once-across-pods reasoning.
"""
params = request.kv_transfer_params
if not params:
return
transfer_id = params["transfer_id"]
# LLM-D sidecar compat: the nixlv2 routing sidecar emits NIXL-shaped
# kv_transfer_params without MoRI-IO's transfer_id. Synthesize one
# deterministically from request_id so producer and consumer agree
# without a sidecar wire-protocol change.
transfer_id = params.get("transfer_id") or f"sidecar-{request.request_id}"
params.setdefault("transfer_id", transfer_id)
request_id = request.request_id
self.map_request_id(request_id, transfer_id)
if params.get("do_remote_decode"):
local_block_ids = blocks.get_block_ids()[0]
self._reqs_need_save[request.request_id] = (request, local_block_ids)
# Snapshot params now so chunked-prefill build_connector_meta
# can recover them on the final chunk even if the live
# request.kv_transfer_params has been mutated/cleared.
self._req_kv_params[request.request_id] = dict(params)
if params is not None and params.get("do_remote_prefill"):
if self.mode == MoRIIOMode.READ:
@@ -568,6 +659,9 @@ class MoRIIOConnectorScheduler:
request,
local_block_ids,
)
# Snapshot params for chunked prefill consumption
# in build_connector_meta (see comment above).
self._req_kv_params[request.request_id] = dict(params)
else:
logger.warning(
"Got invalid KVTransferParams: %s. This "
@@ -582,37 +676,113 @@ class MoRIIOConnectorScheduler:
)
remote_dp_rank = request.kv_transfer_params.get("remote_dp_rank", 0)
remote_host = request.kv_transfer_params.get("remote_host")
remote_notify_port = request.kv_transfer_params.get(
"remote_notify_port"
)
if remote_host is None or remote_notify_port is None:
# Effective DP fan-out for the per-pod port/host math below
# (see _remote_dp_rank_for_port / _pod_idx). Capped to the
# per-pod local size when the router advertises it (Wide-EP).
_dp_size = int(request.kv_transfer_params.get("remote_dp_size", 1) or 1)
try:
_dp_local = int(
request.kv_transfer_params.get("remote_dp_size_local", 0) or 0
)
if _dp_local > 0:
_dp_size = min(_dp_size, _dp_local)
except (TypeError, ValueError):
_dp_local = 0
# Rank routing is ROUTER-AUTHORITATIVE: honor the router-pinned
# remote_dp_rank (matched to the X-data-parallel-rank dispatch
# pin). Self-deriving a rank could disagree and notify a rank
# that never served the request. If unpinned in a multi-rank
# deployment, warn instead of guessing.
if (
_dp_size > 1
and "remote_dp_rank" not in request.kv_transfer_params
and "is_request_leader" not in request.kv_transfer_params
):
logger.warning(
"MoRI-IO decode notify: remote_dp_size=%d but the router "
"did not pin remote_dp_rank for request %s; defaulting to "
"rank 0. The router must set remote_dp_rank (and the "
"X-data-parallel-rank dispatch header) so the prefill and "
"decode legs agree on the same rank.",
_dp_size,
request.request_id,
)
# Only the owning rank originates notify (exactly-once).
# Priority: is_request_leader > remote_dp_rank_override > _is_kv_master
_leader_flag = request.kv_transfer_params.get("is_request_leader")
if _leader_flag is not None:
_should_notify = bool(_leader_flag)
elif request.kv_transfer_params.get("remote_dp_rank_override"):
_should_notify = self._global_dp_rank == remote_dp_rank
else:
_should_notify = self._is_kv_master
if _should_notify:
peer_zmq = get_peer_zmq_from_request_id(
request.request_id, is_producer=False
)
remote_host, _, remote_notify_port = parse_moriio_zmq_address(
peer_zmq
)
remote_notify_port = int(remote_notify_port)
if peer_zmq is not None:
remote_host, _, remote_notify_port = parse_moriio_zmq_address(
peer_zmq
)
else:
# Sidecar fallback: use explicit params fields.
params = request.kv_transfer_params or {}
remote_host = params.get("remote_host") or ""
try:
remote_notify_port = int(
params.get("remote_notify_port") or 0
)
except (TypeError, ValueError):
remote_notify_port = 0
if not remote_host or not remote_notify_port:
raise ValueError(
f"request {request.request_id!r}: "
f"request_id has no embedded peer "
f"zmq_address and kv_transfer_params is "
f"missing remote_host / remote_notify_port "
f"(got remote_host={remote_host!r}, "
f"remote_notify_port={remote_notify_port!r})"
)
# num_external_tokens == 0: nothing to push, so don't tell the
# producer to write into these blocks.
block_notify_list = (
blocks.get_block_ids()[0] if num_external_tokens > 0 else []
)
for tp_index in range(self.tp_size):
target_port = remote_notify_port + get_port_offset(
remote_dp_rank, tp_index
# num_external_tokens == 0: nothing to push, so don't tell
# the producer to write into these blocks.
block_notify_list = (
blocks.get_block_ids()[0] if num_external_tokens > 0 else []
)
self.send_notify_block(
req_id=request.request_id,
transfer_id=request.kv_transfer_params["transfer_id"],
block_notify_list=block_notify_list,
host=remote_host,
port=target_port,
# Wide-EP multi-pod: a pod binds notify sockets only for
# its LOCAL ranks, so the port offset must use the per-pod
# local rank (% dp_local), not the global rank. Single-pod
# is bit-identical (modulus is a no-op).
_remote_dp_rank_for_port = fold_local_rank(
remote_dp_rank, _dp_local
)
# The target rank may live on a child pod at a different IP,
# so resolve the per-pod host (pod_idx = global // dp_local).
# Otherwise a notify for child ranks lands on the master
# pod and the request hangs in WAITING_FOR_REMOTE_KVS.
_notify_host = remote_host
_kvp = request.kv_transfer_params or {}
_remote_hosts = _kvp.get("remote_hosts") or []
if _dp_local > 0 and _remote_hosts:
_pod_idx = pod_index(remote_dp_rank, _dp_local)
if 0 <= _pod_idx < len(_remote_hosts):
_notify_host = _remote_hosts[_pod_idx]
for tp_index in range(self.tp_size):
target_port = remote_notify_port + get_port_offset(
_remote_dp_rank_for_port, tp_index
)
self.send_notify_block(
req_id=request.request_id,
transfer_id=request.kv_transfer_params["transfer_id"],
block_notify_list=block_notify_list,
host=_notify_host,
port=target_port,
)
# Only trigger 1 KV transfer per request.
@@ -636,6 +806,12 @@ class MoRIIOConnectorScheduler:
if new_block_ids is not None:
block_ids = new_block_ids[0]
# TODO : hybrid attn, etc
# A non-disagg request (no kv_transfer_params, e.g. smoke
# test) is never registered in _reqs_need_pending_save;
# indexing it unconditionally would KeyError and crash the
# EngineCore. Skip it silently.
if req_id not in self._reqs_need_pending_save:
continue
req, existing_blocks = self._reqs_need_pending_save[req_id]
updated_blocks = list(existing_blocks) + (block_ids)
self._reqs_need_pending_save[req_id] = (req, updated_blocks)
@@ -643,25 +819,30 @@ class MoRIIOConnectorScheduler:
len(self._reqs_need_pending_save[req_id][1]) * self.block_size
>= req.num_prompt_tokens
):
# Final chunk: live kv_transfer_params may be cleared,
# so prefer the snapshot from update_state_after_alloc.
kv_params = self._req_kv_params.pop(
req_id, req.kv_transfer_params or {}
)
meta.add_new_req(
request_id=req_id,
local_block_ids=self._reqs_need_pending_save[req_id][1],
kv_transfer_params=req.kv_transfer_params or {},
kv_transfer_params=kv_params,
write_mode=True,
)
del self._reqs_need_pending_save[req_id]
# Loop through scheduled reqs and convert to ReqMeta.
for req_id, (req, block_ids) in self._reqs_need_recv.items():
assert req.kv_transfer_params is not None
kv_params = self._req_kv_params.get(req_id, req.kv_transfer_params or {})
meta.add_new_req(
request_id=req_id,
local_block_ids=block_ids,
kv_transfer_params=req.kv_transfer_params,
kv_transfer_params=kv_params,
)
for req_id, (req, block_ids) in self._reqs_need_save.items():
assert req.kv_transfer_params is not None
kv_params = self._req_kv_params.get(req_id, req.kv_transfer_params or {})
if req.num_prompt_tokens > len(block_ids) * self.block_size:
# not last chunk prefill
self._reqs_need_pending_save[req_id] = (req, block_ids)
@@ -669,13 +850,22 @@ class MoRIIOConnectorScheduler:
meta.add_new_req(
request_id=req_id,
local_block_ids=block_ids,
kv_transfer_params=req.kv_transfer_params,
kv_transfer_params=kv_params,
write_mode=True,
)
# Clear the list once workers start the transfers
meta.reqs_to_send = self._reqs_need_send
# Reclaim snapshot cache entries that completed this step. Keep
# entries that are still pending (chunked prefill not yet at the
# final chunk) — those will be popped above on the final chunk.
for req_id in self._reqs_need_recv:
self._req_kv_params.pop(req_id, None)
for req_id in self._reqs_need_save:
if req_id not in self._reqs_need_pending_save:
self._req_kv_params.pop(req_id, None)
self._reqs_need_recv.clear()
self._reqs_need_save.clear()
self._reqs_need_send = {}
@@ -702,14 +892,14 @@ class MoRIIOConnectorScheduler:
"""
request_id = request.request_id
params = request.kv_transfer_params
# Consumer: can unmap transfer_id<->request_id immediately since done_recving
# has fired at this point (i.e. KV has been transferred)
# Producer: must keep the mapping until we get notification that blocks can
# be freed, which may be several scheduler steps later.
if not self.is_producer:
self.unmap_request_id(request_id)
params = request.kv_transfer_params
transfer_id = params.get("transfer_id") if params else None
self.unmap_request_id(request_id, transfer_id=transfer_id)
logger.debug(
"MoriioConnector request_finished, request_status=%s, "
"kv_transfer_params=%s",
@@ -754,11 +944,17 @@ class MoRIIOConnectorScheduler:
+ MoRIIOConstants.VLLM_MORI_READ_ABORT_REQUEST_TIMEOUT
)
self._deferred_send_deadlines[request.request_id] = (
time.monotonic() + self._defer_timeout
time.monotonic() + self._defer_timeout,
params.get("transfer_id") if params else None,
)
# Return KV transfer params forwarded verbatim to the decode instance by
# the router.
# the router. remote_dp_rank is the rank THIS prefill leg actually ran
# on: on the returnable paths (READ / serial WRITE) the router forwards
# it to the decode leg, so the decode->prefill notify is routed by pure
# propagation of the producer's real rank -- no hashing, no reliance on
# the router independently pinning the same rank. remote_dp_rank_override
# makes the decode side honor it via the global-rank match gate.
return delay_free_blocks, dict(
do_remote_prefill=True,
do_remote_decode=False,
@@ -767,61 +963,96 @@ class MoRIIOConnectorScheduler:
remote_host=self.host_ip,
remote_handshake_port=self.handshake_port,
remote_notify_port=self.side_notify_port,
remote_dp_rank=self._global_dp_rank,
remote_dp_rank_override=True,
remote_dp_size=self.vllm_config.parallel_config.data_parallel_size,
remote_dp_size_local=(
self.vllm_config.parallel_config.data_parallel_size_local
),
tp_size=self.vllm_config.parallel_config.tensor_parallel_size,
transfer_id=params["transfer_id"],
)
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
"""Free KV blocks from sends that never received a completion signal.
"""Reconcile worker finished_sending ACKs with the producer lifecycle.
Called every scheduler step. When a send is deferred (request_finished
returns True), blocks remain allocated until the worker reports
finished_sending. If that notification is lost (e.g. ibv_post_send
failure), blocks leak permanently. This method injects timed-out
entries into connector_output.finished_sending so the scheduler
frees them via the normal path.
Called every scheduler step (when there is KV connector output),
BEFORE the scheduler's finished_sending free loop. We rewrite
``connector_output.finished_sending`` in place so it contains
exactly the producer requests that are safe for the scheduler to
free this step, letting the scheduler keep the plain upstream
``assert req_id in self.requests`` + ``_free_blocks`` path:
* An ACK whose request is in ``_deferred_send_deadlines``
(request_finished ran with delay_free_blocks=True, so the
scheduler is holding its blocks) is surfaced -> freed now.
* An ACK that arrives BEFORE its request finished is parked in
``_pending_sent_acks`` and released on a later step once the
request enters ``_deferred_send_deadlines``.
* A deferred send whose ACK never arrives is reaped after
``_defer_timeout`` and surfaced, so leaked blocks are freed.
* A parked ACK that never matches a deferral before its own
deadline is a stale duplicate (e.g. a real ACK landing after the
send was already reaped) and is dropped.
Consumers never populate finished_sending (they report
finished_recving), and they unmap in request_finished, so this is a
no-op for them.
"""
# Producer: unmap transfer_id<->request_id for sends that are now (async)
# reported as completed. This unmapping have to be deferred until now
# so get_finished can use it in any scheduler step.
# Consumer: unmapping already done in request_finished
if self.is_producer and connector_output.finished_sending:
for req_id in connector_output.finished_sending:
self.unmap_request_id(req_id)
if not self._deferred_send_deadlines:
if not self.is_producer:
return
# Remove entries the worker already reported as finished_sending, these will be
# freed anyways.
for req_id in connector_output.finished_sending or ():
self._deferred_send_deadlines.pop(req_id, None)
incoming = set(connector_output.finished_sending or ())
now = time.monotonic()
expired_reqs = [
# Surface ACKs whose request is already finished (blocks held for
# delayed free); park the rest until their request finishes.
safe: set[ReqId] = set()
for req_id in incoming:
if req_id in self._deferred_send_deadlines:
safe.add(req_id)
else:
self._pending_sent_acks.setdefault(req_id, now + self._defer_timeout)
# Release previously parked ACKs whose request has since finished.
for req_id in self._pending_sent_acks:
if req_id in self._deferred_send_deadlines:
safe.add(req_id)
# Reap deferred sends whose ACK never arrived (avoid leaking blocks).
expired = [
req_id
for req_id, deadline in self._deferred_send_deadlines.items()
for req_id, (deadline, _) in self._deferred_send_deadlines.items()
if now >= deadline
]
if not expired_reqs:
return
if expired:
safe.update(expired)
logger.warning(
"Reaped %d deferred sends with no finished_sending "
"notification after %.0fs. This indicates lost async KV "
"completion notifications from the KV connector.",
len(expired),
self._defer_timeout,
)
if connector_output.finished_sending is None:
connector_output.finished_sending = set()
# Register the expired requests as finished so the scheduler frees their blocks.
for req_id in expired_reqs:
connector_output.finished_sending.add(req_id)
del self._deferred_send_deadlines[req_id]
if self.is_producer:
self.unmap_request_id(req_id)
logger.warning(
"Reaped %d deferred sends with no finished_sending notification "
"after %.0fs. This indicates lost async KV completion "
"notifications from the KV connector.",
len(expired_reqs),
self._defer_timeout,
)
# Finalize the requests we are surfacing: drop their deferral/park
# bookkeeping and unmap their transfer ids.
for req_id in safe:
deferred_info = self._deferred_send_deadlines.pop(req_id, None)
transfer_id = deferred_info[1] if deferred_info else None
self._pending_sent_acks.pop(req_id, None)
self.unmap_request_id(req_id, transfer_id=transfer_id)
# Drop stale parked ACKs that never matched a deferral in time.
stale = [
req_id
for req_id, deadline in self._pending_sent_acks.items()
if now >= deadline
]
for req_id in stale:
self._pending_sent_acks.pop(req_id, None)
connector_output.finished_sending = safe or None
class MoRIIOConnectorWorker:
@@ -998,6 +1229,10 @@ class MoRIIOConnectorWorker:
self._recving_transfers: defaultdict[ReqId, list] = defaultdict(list)
# Values are (remote_host, remote_notify_port, transfer_id).
self._recving_transfers_callback_addr: dict[ReqId, tuple[str, str, str]] = {}
# Monotonic-clock start times for each in-flight recv transfer.
# Used by _pop_done_transfers to abort transfers whose RDMA
# completion is lost, instead of hanging forever.
self._recving_transfers_start: dict[str, float] = {}
# Track the expiration time of requests that are waiting to be sent.
self._reqs_to_send: dict[ReqId, float] = {}
@@ -1368,6 +1603,11 @@ class MoRIIOConnectorWorker:
port = int(meta.remote_handshake_port)
tp_size = int(meta.tp_size)
remote_dp_size = int(meta.remote_dp_size)
# Wide-EP multi-pod: remote DP ranks span pods at different IPs
# (ranks per pod = dp_local), so resolve the host per cur_dp_rank
# below instead of using a single host for all ranks.
pod_hosts = list(meta.multi_pod_hosts) if meta.multi_pod_hosts else [host]
remote_dp_size_local = int(meta.remote_dp_size_local) or remote_dp_size
def request_ready(_f: Future[Any], entry=(req_id, meta)):
logger.info("MoRIIO handshake done for request %s", req_id)
@@ -1381,8 +1621,21 @@ class MoRIIOConnectorWorker:
for cur_dp_rank in range(remote_dp_size):
dp_engine_id = self.get_engine_name_with_dp(remote_engine_id, cur_dp_rank)
_pod_idx = pod_index(cur_dp_rank, remote_dp_size_local)
if _pod_idx >= len(pod_hosts):
_pod_idx = 0
_per_rank_host = pod_hosts[_pod_idx]
# The handshake port offset must use the per-pod local rank, since
# each pod binds sockets only for its local ranks; dp_engine_id
# keeps the global rank for uniqueness. Single-pod is bit-identical.
_per_rank_local_dp = fold_local_rank(cur_dp_rank, remote_dp_size_local)
future = self._handshake_initiation_executor.submit(
self._moriio_handshake, host, port, tp_size, dp_engine_id, cur_dp_rank
self._moriio_handshake,
_per_rank_host,
port,
tp_size,
dp_engine_id,
_per_rank_local_dp,
)
fut_list.append(future)
@@ -1463,7 +1716,19 @@ class MoRIIOConnectorWorker:
block_shape = first_kv_cache.shape[-block_rank:]
self.num_blocks = first_geometry.num_blocks
self.slot_size_bytes = first_geometry.slot_size_bytes
assert first_geometry.block_size == self.block_size
if first_geometry.block_size != self.block_size:
# DeepSeek-V3 / MLA backends (e.g. FlashMLA) override the
# configured block_size at attention-layer creation time, so
# the KV cache tensor is laid out with a different (usually
# larger) block_size than vllm_config.cache_config.block_size.
# Trust the actual tensor shape and reconcile.
logger.info(
"KV cache block_size=%d differs from config block_size=%d; "
"using actual tensor shape (attention backend override).",
first_geometry.block_size,
self.block_size,
)
self.block_size = first_geometry.block_size
# TODO(tms): self.block_len needs to be per-layer for sliding window,
# hybrid attn, etc
# block size in bytes
@@ -1634,6 +1899,7 @@ class MoRIIOConnectorWorker:
def _pop_done_transfers(self) -> set[str]:
done_req_ids: set[str] = set()
_xfer_timeout = int(os.environ.get("VLLM_MORIIO_TRANSFER_TIMEOUT_S", "120"))
with self.moriio_wrapper.lock:
to_remove = []
for req_id, status_list in self._recving_transfers.items():
@@ -1675,9 +1941,25 @@ class MoRIIOConnectorWorker:
to_remove.append(req_id)
# Do NOT add to done_req_ids: decode KV cache is incomplete.
# The request will expire via the normal request timeout.
elif req_id in self._recving_transfers_start:
# Abort still-in-flight transfers that exceed the
# configured deadline. Otherwise a lost RDMA
# completion would leave the decode worker hung
# indefinitely on this request.
_age = time.monotonic() - self._recving_transfers_start[req_id]
if _age > _xfer_timeout:
logger.error(
"RDMA read TIMED OUT for req %s after %.1fs "
"(VLLM_MORIIO_TRANSFER_TIMEOUT_S=%d)",
req_id,
_age,
_xfer_timeout,
)
to_remove.append(req_id)
for req_id in to_remove:
del self._recving_transfers[req_id]
del self._recving_transfers_callback_addr[req_id]
self._recving_transfers_start.pop(req_id, None)
return done_req_ids
@@ -2043,6 +2325,18 @@ class MoRIIOConnectorWorker:
)
def _write_blocks_for_req(self, req_id: ReqId, meta: ReqMeta, layer_name, kv_layer):
# Stash multi_pod_hosts + local DP size on the worker so
# MoRIIOEngine._finalize_if_complete (which sees only the WriteTask,
# not ReqMeta) can pick the per-rank pod IP for the completion notify.
# Last-writer-wins is safe: all requests share the same topology.
if meta.multi_pod_hosts:
self.multi_pod_hosts = list(meta.multi_pod_hosts)
else:
self.multi_pod_hosts = [meta.remote_host]
if meta.remote_dp_size_local:
self.remote_dp_size_local = int(meta.remote_dp_size_local)
else:
self.remote_dp_size_local = int(meta.remote_dp_size)
self.schedule_write_blocks(
request_id=req_id,
transfer_id=meta.transfer_id,
@@ -2271,6 +2565,7 @@ class MoRIIOConnectorWorker:
_backoff = min(_backoff * 2, 0.05)
with self.moriio_wrapper.lock:
self._recving_transfers[request_id].append(transfer_status)
self._recving_transfers_start.setdefault(request_id, time.monotonic())
self._recving_transfers_callback_addr[request_id] = (
remote_host,
str(
@@ -311,7 +311,17 @@ class MoRIIOWriter:
with self._write_state_lock:
request_info.completion_request_id = task.request_id
request_info.completion_remote_notify_port = task.remote_notify_port
request_info.completion_remote_ip = task.remote_ip
# Wide-EP multi-pod: task.remote_ip addresses only the first pod,
# so resolve the per-rank host from multi_pod_hosts (falls back to
# task.remote_ip for single-pod or on any indexing miss).
_remote_ip = task.remote_ip
_hosts = list(getattr(self.worker, "multi_pod_hosts", []) or [])
_dp_local = int(getattr(self.worker, "remote_dp_size_local", 0) or 0)
if _hosts and _dp_local > 0:
_pod_idx = int(request_info.decode_dp_rank) // _dp_local
if 0 <= _pod_idx < len(_hosts):
_remote_ip = _hosts[_pod_idx]
request_info.completion_remote_ip = _remote_ip
if task.transfer_id in self._sealed_writes:
request_info.writes_expected = self._sealed_writes[task.transfer_id]
@@ -454,8 +464,15 @@ class MoRIIOWriter:
# Wait for this request's transfers to complete.
self.worker.moriio_wrapper.waiting_for_transfer_complete(transfer_statuses)
# The notify port offset must use the per-pod local rank
# (% dp_local), since each pod binds notify sockets only for its local
# ranks. Single-pod is bit-identical (modulus is a no-op).
_dp_local = int(getattr(self.worker, "remote_dp_size_local", 0) or 0)
_decode_dp_rank_for_port = int(request_info.decode_dp_rank)
if _dp_local > 0:
_decode_dp_rank_for_port = _decode_dp_rank_for_port % _dp_local
remote_port = remote_notify_port + get_port_offset(
request_info.decode_dp_rank, self.worker.tp_rank
_decode_dp_rank_for_port, self.worker.tp_rank
)
# Consider using RDMA immediate data in decode side
# to eliminate the need for this notification.