diff --git a/examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py b/examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py index d2d5be7607..8b2c46999e 100644 --- a/examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py +++ b/examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py @@ -1,5 +1,20 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# +# Minimal single-node reference proxy for MoRI-IO prefill/decode disaggregation. +# It demonstrates the DP-rank pinning contract the connector expects: pick one +# prefill DP rank per request (flat_interleaved_dp_route) and pin BOTH legs to it +# via the ``X-data-parallel-rank`` header plus ``kv_transfer_params`` +# (``remote_dp_rank`` / ``remote_dp_size`` / ``remote_tp_size``). +# +# Scope: single-node only. It intentionally does NOT supply the cross-pod +# Wide-EP (2P2D) peer info -- ``remote_hosts`` / ``multi_pod_hosts``, +# ``data_parallel_size_local``, ``is_request_leader``. In real multi-pod +# deployments that peer info is produced by a production routing sidecar, e.g.: +# * llm-d-router (https://github.com/llm-d/llm-d-router) +# * vLLM's own vllm-router (examples in the vllm-project org) +# * or any NIXL-shaped router that sets the same kv_transfer_params contract. +# Use this file as a protocol reference, not a production router. import argparse import asyncio import copy @@ -181,7 +196,7 @@ async def send_request_to_prefill( raise RuntimeError(error_message) -async def start_decode_request(endpoint, req_data, request_id): +async def start_decode_request(endpoint, req_data, request_id, selected_dp_rank=None): session = aiohttp.ClientSession( timeout=aiohttp.ClientTimeout(total=6 * 6000 * 6000) ) @@ -189,6 +204,10 @@ async def start_decode_request(endpoint, req_data, request_id): "Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}", "X-Request-Id": request_id, } + # Pin the decode leg to the same DP rank as its prefill leg so both land on + # the same rank (vLLM reads this header directly; no engine-side routing). + if selected_dp_rank is not None: + headers["X-data-parallel-rank"] = str(selected_dp_rank) response = await session.post(url=endpoint, json=req_data, headers=headers) return session, response @@ -361,7 +380,9 @@ async def handle_request(api: str, request: Request): decode_request_url = decode_instance_endpoint["request_address"] + api decode_request_task = asyncio.create_task( - start_decode_request(decode_request_url, req_data, request_id) + start_decode_request( + decode_request_url, req_data, request_id, selected_prefill_dp_rank + ) ) session, decode_response = await decode_request_task diff --git a/tests/v1/kv_connector/unit/test_moriio_connector.py b/tests/v1/kv_connector/unit/test_moriio_connector.py index c236fcc4fd..ae62313d57 100644 --- a/tests/v1/kv_connector/unit/test_moriio_connector.py +++ b/tests/v1/kv_connector/unit/test_moriio_connector.py @@ -508,9 +508,15 @@ def test_send_transfer_release_sends_structured_release_message(): scheduler._send_transfer_release("xfer-7", "127.0.0.1", 7000) payload = sock.send.call_args.args[0] + # WRITE-mode release advertises the consumer (decode) TP size so the prefill + # side counts the right number of ACKs via get_moriio_expected_ack_count, + # mirroring the READ-mode release in _pop_done_transfers (see + # test_read_completion_sends_structured_release_with_consumer_tp_size). The + # fixture's tp_size is 2, so consumer_tp_size == 2. assert msgspec.msgpack.decode(payload) == { "type": "release", "transfer_id": "xfer-7", + "consumer_tp_size": 2, } diff --git a/tests/v1/kv_connector/unit/test_moriio_routing.py b/tests/v1/kv_connector/unit/test_moriio_routing.py new file mode 100644 index 0000000000..a9a3521beb --- /dev/null +++ b/tests/v1/kv_connector/unit/test_moriio_routing.py @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for MoRIIO router-authoritative DP-rank routing. + +These lock in the redesign that addresses the PR #45043 review feedback +(njhill): the connector must NOT self-derive a DP rank, and the prefill leg +must *return* the rank it actually ran on via ``request_finished`` so the +decode->prefill notify is routed by pure propagation (READ / serial WRITE), +not by an independent hash on each side. + +Following test_moriio_unmap.py, we bind the unbound method to a lightweight +stand-in rather than constructing a full scheduler. +""" + +from types import SimpleNamespace + +from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import ( + MoRIIOConnectorScheduler, +) +from vllm.v1.request import RequestStatus + +_request_finished = MoRIIOConnectorScheduler.request_finished + + +def _producer(global_dp_rank: int, dp_size: int, dp_size_local: int): + """Minimal prefill-side (producer) scheduler stand-in for request_finished.""" + return SimpleNamespace( + is_producer=True, + _global_dp_rank=global_dp_rank, + engine_id="engine-abc", + host_ip="10.0.0.1", + handshake_port=6300, + side_notify_port=7300, + vllm_config=SimpleNamespace( + parallel_config=SimpleNamespace( + data_parallel_size=dp_size, + data_parallel_size_local=dp_size_local, + tensor_parallel_size=1, + ) + ), + ) + + +def _req( + request_id: str, params: dict | None, status=RequestStatus.FINISHED_LENGTH_CAPPED +): + return SimpleNamespace( + request_id=request_id, + kv_transfer_params=params, + status=status, + ) + + +def test_request_finished_returns_prefill_rank_for_propagation(): + # 2P2D: global rank 11 lives on pod 1 (ranks 8..15), local size 8. + s = _producer(global_dp_rank=11, dp_size=16, dp_size_local=8) + req = _req("r1", {"do_remote_decode": True, "transfer_id": "t1"}) + + delay_free, params = _request_finished(s, req, block_ids=[]) + + # The prefill leg returns the rank IT actually ran on, marked authoritative, + # so the router forwards it to decode and the notify propagates (no hashing). + assert params is not None + assert params["remote_dp_rank"] == 11 + assert params["remote_dp_rank_override"] is True + assert params["remote_dp_size"] == 16 + assert params["remote_dp_size_local"] == 8 + assert params["transfer_id"] == "t1" + # Empty block_ids => nothing deferred, blocks freed immediately. + assert delay_free is False + + +def test_request_finished_rank_tracks_the_actual_rank(): + # A different rank must be echoed verbatim -- the value is the producer's + # own rank, never a recomputation of the request id. + s = _producer(global_dp_rank=4, dp_size=8, dp_size_local=8) + req = _req("r2", {"do_remote_decode": True, "transfer_id": "t2"}) + + _, params = _request_finished(s, req, block_ids=[]) + + assert params["remote_dp_rank"] == 4 + assert params["remote_dp_size_local"] == 8 + + +def test_request_finished_no_kv_params_is_noop(): + s = _producer(global_dp_rank=0, dp_size=16, dp_size_local=8) + req = _req("r3", None) + + delay_free, params = _request_finished(s, req, block_ids=[]) + + assert delay_free is False + assert params is None diff --git a/tests/v1/kv_connector/unit/test_moriio_tp_ack.py b/tests/v1/kv_connector/unit/test_moriio_tp_ack.py index 6488135690..35951f41bb 100644 --- a/tests/v1/kv_connector/unit/test_moriio_tp_ack.py +++ b/tests/v1/kv_connector/unit/test_moriio_tp_ack.py @@ -296,6 +296,8 @@ def test_read_completion_sends_structured_release_with_consumer_tp_size(): worker._recving_transfers_callback_addr = { "req": ("127.0.0.1", "7000", "tx-release") } + # Transfer-timeout reaping state consulted by _pop_done_transfers. + worker._recving_transfers_start = {} assert worker._pop_done_transfers() == {"tx-release"} assert worker.moriio_wrapper.sent == [ diff --git a/tests/v1/kv_connector/unit/test_moriio_unmap.py b/tests/v1/kv_connector/unit/test_moriio_unmap.py new file mode 100644 index 0000000000..b4fd4b8813 --- /dev/null +++ b/tests/v1/kv_connector/unit/test_moriio_unmap.py @@ -0,0 +1,68 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for MoRIIOConnectorScheduler.{map,unmap}_request_id. + +vLLM's input_processor mutates a request_id after the connector has stored it +(e.g. ``cmpl-abc-0`` -> ``cmpl-abc-0-956053a4``), so an exact-match unmap can +miss. These tests cover the stable-``transfer_id`` fallback that handles that +case (and the clean exact-match and miss paths). + +The map/unmap methods touch only the two id<->id dicts, so we bind them to a +lightweight stand-in rather than constructing a full scheduler. +""" + +from types import SimpleNamespace + +from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import ( + MoRIIOConnectorScheduler, +) + +_map = MoRIIOConnectorScheduler.map_request_id +_unmap = MoRIIOConnectorScheduler.unmap_request_id + + +def _sched(): + return SimpleNamespace( + transfer_id_to_request_id={}, + request_id_to_transfer_id={}, + ) + + +def test_exact_match_unmap_clears_both_tables(): + s = _sched() + _map(s, "rid-1", "tid-1") + _unmap(s, "rid-1") + assert s.request_id_to_transfer_id == {} + assert s.transfer_id_to_request_id == {} + + +def test_unmap_via_transfer_id_after_request_id_mutation(): + s = _sched() + # Connector stores the ORIGINAL request_id at map time. + _map(s, "cmpl-abc-0", "tid-42") + # input_processor later mutates the rid; exact match would miss, so the + # stable transfer_id is used to find and clear the original entry. + _unmap(s, "cmpl-abc-0-956053a4", transfer_id="tid-42") + assert s.request_id_to_transfer_id == {} + assert s.transfer_id_to_request_id == {} + + +def test_unmap_miss_is_noop_and_does_not_raise(): + s = _sched() + _map(s, "rid-1", "tid-1") + # Unknown rid and unknown/None transfer_id: must not raise, must not touch + # the existing (unrelated) mapping. + _unmap(s, "rid-unknown", transfer_id=None) + _unmap(s, "rid-unknown", transfer_id="tid-unknown") + assert s.request_id_to_transfer_id == {"rid-1": "tid-1"} + assert s.transfer_id_to_request_id == {"tid-1": "rid-1"} + + +def test_exact_match_preferred_over_transfer_id_fallback(): + s = _sched() + _map(s, "rid-1", "tid-1") + # Exact rid is present, so it is removed directly even if a transfer_id is + # also supplied; no stale entries left behind. + _unmap(s, "rid-1", transfer_id="tid-1") + assert s.request_id_to_transfer_id == {} + assert s.transfer_id_to_request_id == {} diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py index 811bedb084..f37868d268 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py @@ -191,6 +191,25 @@ def get_port_offset(dp_rank: int, tp_rank: int, tp_size: int = 1) -> int: return (dp_rank) * tp_size + tp_rank +def fold_local_rank(global_dp_rank: int, dp_size_local: int) -> int: + """Fold a global DP rank into its pod-local rank [0, dp_size_local). + + ``dp_size_local == 0`` is the external-DP sentinel (local size unknown): + return the rank unchanged, since a global DP rank is always < the global + DP size so no folding is needed and the modulo is skipped (never divides + by zero). + """ + return global_dp_rank % dp_size_local if dp_size_local else global_dp_rank + + +def pod_index(global_dp_rank: int, dp_size_local: int) -> int: + """Pod index (0-based) a global DP rank lives on for Wide-EP multi-pod. + + ``dp_size_local == 0`` (external-DP sentinel) collapses to a single pod. + """ + return global_dp_rank // dp_size_local if dp_size_local else 0 + + def resolve_host_ip(extra_config: dict) -> str: """The IP this MoRIIO process advertises for KV transfer. @@ -280,9 +299,11 @@ class MoRIIOConfig: kv_transfer_config = vllm_config.kv_transfer_config extra_config = kv_transfer_config.kv_connector_extra_config tp_rank = get_tensor_model_parallel_rank() - dp_rank = vllm_config.parallel_config.data_parallel_rank + # Fold the global data_parallel_rank back to [0, dp_size_local) for + # per-node port allocation (handles the external-DP sentinel). + pc = vllm_config.parallel_config + dp_rank = fold_local_rank(pc.data_parallel_rank, pc.data_parallel_size_local) base_notify_port = int(extra_config["notify_port"]) - dp_size = vllm_config.parallel_config.data_parallel_size tp_size = get_tensor_model_parallel_world_size() port_offset = get_port_offset(dp_rank, tp_rank) backend = str(extra_config.get("backend", "rdma")).lower() @@ -304,15 +325,18 @@ class MoRIIOConfig: return cls( local_ip=resolve_host_ip(extra_config), local_kv_port=get_open_port(), - proxy_ip=extra_config["proxy_ip"], + proxy_ip=extra_config.get("proxy_ip", ""), local_ping_port=get_open_port(), - proxy_ping_port=int(extra_config["proxy_ping_port"]), + proxy_ping_port=int(extra_config.get("proxy_ping_port", 0)), http_port=int(extra_config["http_port"]), handshake_port=int(extra_config["handshake_port"]), notify_port=base_notify_port + port_offset, tp_rank=tp_rank, dp_rank=dp_rank, - dp_size=dp_size, + # Advertise the GLOBAL DP world size to the routing proxy so it can + # address every rank across pods; dp_size_local above is only for + # per-pod port folding, not for routing. + dp_size=pc.data_parallel_size, tp_size=tp_size, read_mode=get_moriio_mode(kv_transfer_config) == MoRIIOMode.READ, qp_per_transfer=int(extra_config.get("qp_per_transfer", 1)), @@ -391,20 +415,26 @@ def parse_moriio_zmq_address( return host, handshake_port, notify_port -def get_peer_zmq_from_request_id(request_id: str, is_producer: bool) -> str: +def get_peer_zmq_from_request_id(request_id: str, is_producer: bool) -> str | None: """Extract the *peer's* zmq_address from the vLLM router request_id. The producer (prefill) needs the decode's address; the consumer (decode) needs the prefill's address. + + Returns ``None`` when the request_id does not encode peer info. The + llm-d routing sidecar (``llm-d-inference-scheduler``) does not embed + addresses in ``request_id``; instead it passes ``remote_host``, + ``remote_handshake_port`` and ``remote_notify_port`` explicitly in + ``kv_transfer_params``. Callers must handle the ``None`` return by + falling back to those fields. See ``add_new_req`` for the canonical + fallback path. """ if is_producer: m = _DECODE_ZMQ_RE.search(request_id) else: m = _PREFILL_ZMQ_RE.search(request_id) if m is None: - raise ValueError( - f"Cannot parse peer zmq_address from request_id: {request_id!r}" - ) + return None return m.group(1) @@ -426,6 +456,10 @@ class ReqMeta: # read must target this rank's memory registration; the default 0 preserves # the symmetric single-DP behaviour. remote_dp_rank: int = 0 + # Multi-pod: list of remote pod IPs indexed by pod_idx. + multi_pod_hosts: list[str] = field(default_factory=list) + # Per-pod DP size; 0 means fallback to remote_dp_size. + remote_dp_size_local: int = 0 class MoRIIOConnectorMetadata(KVConnectorMetadata): @@ -450,23 +484,76 @@ class MoRIIOConnectorMetadata(KVConnectorMetadata): kv_transfer_params: dict[str, Any], write_mode=False, ): + """Ingest a peer's ``kv_transfer_params`` into a typed ``ReqMeta``. + + This is the single place peer info enters the connector. The + ``kv_transfer_params`` contract (as produced by the llm-d sidecar / + vLLM router, or echoed by the prefill leg's ``request_finished``): + + Required (always): + * ``transfer_id`` -- stable id shared by both legs. + * ``remote_engine_id`` -- peer engine id for the handshake table. + * ``remote_block_ids`` -- peer block ids (may be [] in WRITE mode, + where decode allocates its own blocks). + + Peer address -- ONE of the following two must resolve: + * embedded in ``request_id`` (vLLM-router PD id form), OR + * explicit ``remote_host`` + ``remote_handshake_port`` + + ``remote_notify_port`` (llm-d sidecar / returnable path). + If neither resolves we raise -- there is no safe default host/port. + + Optional (defaulted): + * ``tp_size`` (default 1) -- peer TP size. + * ``remote_dp_size`` (default 1) -- peer GLOBAL DP size. + * ``remote_dp_size_local`` (default = ``remote_dp_size``) -- per-pod + DP size for Wide-EP multi-pod port/host + folding; 0/absent means single-pod. + * ``remote_hosts`` (default [remote_host]) -- per-pod IP list + indexed by ``pod_idx = rank // dp_local``. + + Routing keys consumed elsewhere (NOT here): ``remote_dp_rank`` / + ``remote_dp_rank_override`` gate the decode->prefill notify target in + MoRIIOConnectorScheduler; they are router-authoritative and never + self-derived (see that class's request routing contract). + """ transfer_id = kv_transfer_params["transfer_id"] - remote_host = kv_transfer_params.get("remote_host") - remote_handshake_port = kv_transfer_params.get("remote_handshake_port") - remote_notify_port = kv_transfer_params.get("remote_notify_port") - if ( - remote_host is None - or remote_handshake_port is None - or remote_notify_port is None - ): - # Parse host/ports from the request_id. The router embeds both - # zmq_addresses in PD request IDs, but WRITE decode requests may carry - # a plain request ID and get the remote address via kv_transfer_params. - peer_zmq = get_peer_zmq_from_request_id(request_id, is_producer=write_mode) + # Try request_id embedded address first, fallback to explicit params. + peer_zmq = get_peer_zmq_from_request_id(request_id, is_producer=write_mode) + if peer_zmq is not None: remote_host, remote_handshake_port, remote_notify_port = ( parse_moriio_zmq_address(peer_zmq) ) + else: + try: + remote_host = kv_transfer_params["remote_host"] + if not remote_host: + raise ValueError( + f"request_id {request_id!r} does not embed a peer " + f"zmq_address and kv_transfer_params['remote_host'] is " + f"empty; cannot route MoRI-IO transfer" + ) + remote_handshake_port = int(kv_transfer_params["remote_handshake_port"]) + remote_notify_port = int(kv_transfer_params["remote_notify_port"]) + except (KeyError, TypeError, ValueError) as e: + raise ValueError( + f"request_id {request_id!r} does not embed a peer " + f"zmq_address and kv_transfer_params is missing one or " + f"more sidecar-fallback keys (need remote_host, " + f"remote_handshake_port, remote_notify_port): {e}" + ) from e + + # Multi-pod: use multi_pod_hosts list or fallback to single host. + _pod_hosts = kv_transfer_params.get("remote_hosts") or [remote_host] + if not isinstance(_pod_hosts, list): + _pod_hosts = [_pod_hosts] + _pod_hosts = [str(h) for h in _pod_hosts] + _remote_dp_size_local = int( + kv_transfer_params.get( + "remote_dp_size_local", + kv_transfer_params.get("remote_dp_size", 1), + ) + ) _req = ReqMeta( transfer_id=transfer_id, @@ -488,6 +575,8 @@ class MoRIIOConnectorMetadata(KVConnectorMetadata): ), remote_dp_size=kv_transfer_params.get("remote_dp_size", 1), remote_dp_rank=kv_transfer_params.get("remote_dp_rank", 0), + multi_pod_hosts=_pod_hosts, + remote_dp_size_local=_remote_dp_size_local, ) if write_mode: self.reqs_to_save[request_id] = _req diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py index 2a4c028e15..9b4b6c7bc0 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import logging import math +import os import queue import threading import time @@ -36,11 +37,13 @@ from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import ( ReqMeta, TransferId, WriteTask, + fold_local_rank, get_moriio_mode, get_peer_zmq_from_request_id, get_port_offset, get_role, parse_moriio_zmq_address, + pod_index, resolve_host_ip, set_role, zmq_ctx, @@ -373,13 +376,29 @@ class MoRIIOConnectorScheduler: "notify_port" ] self.tp_size = self.vllm_config.parallel_config.tensor_parallel_size - self.dp_rank = self.vllm_config.parallel_config.data_parallel_rank + # Local DP rank (0..dp_local-1) for port arithmetic; fold_local_rank + # handles the external-DP sentinel (dp_local==0) so the arithmetic + # can't ZeroDivisionError at init. + _pc = self.vllm_config.parallel_config + self.dp_rank = fold_local_rank( + _pc.data_parallel_rank, _pc.data_parallel_size_local + ) + # Only first-pod ranks originate notify to avoid duplicates. Guarding + # via pod_index keeps _is_kv_master True for the external-DP sentinel + # (single pod) instead of being silently stuck False. + self._is_kv_master = ( + pod_index(_pc.data_parallel_rank, _pc.data_parallel_size_local) == 0 + ) + # Global DP rank for pinned request ownership check. + self._global_dp_rank = self.vllm_config.parallel_config.data_parallel_rank self.is_producer = self.kv_transfer_config.kv_role == "kv_producer" # Requests that need to start recv/send. # New requests are added by update_state_after_alloc in # the scheduler. Used to make metadata passed to Worker. self._reqs_need_recv: dict[ReqId, tuple[Request, list[int]]] = {} self._reqs_need_save: dict[ReqId, tuple[Request, list[int]]] = {} + # Snapshot of kv_transfer_params for chunked prefill recovery. + self._req_kv_params: dict[ReqId, dict] = {} # For chunked prefill, we perform layer-wise access within the final chunk. # TODO: Perform transfer at end chunk. @@ -396,12 +415,15 @@ class MoRIIOConnectorScheduler: # finished_sending before the deadline, we inject them into # connector_output.finished_sending so the scheduler frees the blocks to avoid # hanging indefinitely waiting for a free notification that never comes. - self._deferred_send_deadlines: dict[ReqId, float] = {} + # Value: (deadline, transfer_id) for unmapping after mutation. + self._deferred_send_deadlines: dict[ReqId, tuple[float, TransferId | None]] = {} self._defer_timeout = float( self.kv_transfer_config.kv_connector_extra_config.get( "defer_timeout", MoRIIOConstants.DEFAULT_DEFER_TIMEOUT ) ) + # Buffer for early ACKs that arrive before request_finished. + self._pending_sent_acks: dict[ReqId, float] = {} self.paths: dict[str, zmq.Socket] = {} self.transfer_id_to_request_id: dict[TransferId, ReqId] = {} self.request_id_to_transfer_id: dict[ReqId, TransferId] = {} @@ -410,23 +432,36 @@ class MoRIIOConnectorScheduler: self.transfer_id_to_request_id[transfer_id] = request_id self.request_id_to_transfer_id[request_id] = transfer_id - def unmap_request_id(self, request_id: ReqId): + def unmap_request_id( + self, request_id: ReqId, transfer_id: TransferId | None = None + ): + """Unmap request_id/transfer_id. Uses transfer_id for lookup if + exact request_id match fails (handles input_processor mutation).""" if request_id in self.request_id_to_transfer_id: - transfer_id = self.request_id_to_transfer_id[request_id] + tid = self.request_id_to_transfer_id[request_id] del self.request_id_to_transfer_id[request_id] - if transfer_id in self.transfer_id_to_request_id: - del self.transfer_id_to_request_id[transfer_id] - else: - logger.warning( - "transfer id not in transfer_id_to_request_id lookup" - "table. there is likely a bug!" + if tid in self.transfer_id_to_request_id: + del self.transfer_id_to_request_id[tid] + return + + # Fallback: use transfer_id to find original request_id. + if transfer_id is not None and transfer_id in self.transfer_id_to_request_id: + original_rid = self.transfer_id_to_request_id[transfer_id] + if original_rid != request_id: + logger.debug( + "MoRI-IO unmap via transfer_id: %r -> %r", request_id, original_rid ) - else: - logger.warning( - "Could not find %s in transfer_id_to_request_id" - "lookup table. This could lead to a possible hang.", - request_id, - ) + if original_rid in self.request_id_to_transfer_id: + del self.request_id_to_transfer_id[original_rid] + del self.transfer_id_to_request_id[transfer_id] + return + + logger.warning( + "MoRI-IO unmap MISS: rid=%r transfer_id=%r table_size=%d", + request_id, + transfer_id, + len(self.request_id_to_transfer_id), + ) def get_num_new_matched_tokens( self, @@ -478,7 +513,12 @@ class MoRIIOConnectorScheduler: "req_id": req_id, "transfer_id": transfer_id, "block_notify_list": block_notify_list or [], - "decode_rank": self.dp_rank, + # GLOBAL decode dp rank: producer derives the per-pod notify offset + # (% dp_local), owning pod index (// dp_local), and write-target + # from it. Sending the LOCAL rank made child-pod consumers look + # like master rank 0 and hang in WAITING_FOR_REMOTE_KVS. Single-pod: + # local == global. + "decode_rank": self._global_dp_rank, "type": "remote_blocks", } serialized_data = msgpack.dumps(data) @@ -493,8 +533,18 @@ class MoRIIOConnectorScheduler: ) self.paths[path] = sock + # Advertise the consumer (decode) TP size so prefill counts the right + # number of ACKs (get_moriio_expected_ack_count; see the upstream + # READ-mode release in _pop_done_transfers). Resolves to 1 for our + # homogeneous TP=1 configs; only matters under heterogeneous-TP fan-in. self.paths[path].send( - msgpack.dumps({"type": "release", "transfer_id": transfer_id}) + msgpack.dumps( + { + "type": "release", + "transfer_id": transfer_id, + "consumer_tp_size": self.tp_size, + } + ) ) def _release_write_prefill_blocks(self, request_id: ReqId, params: dict[str, Any]): @@ -513,6 +563,8 @@ class MoRIIOConnectorScheduler: if remote_host is None or remote_notify_port is None: try: peer_zmq = get_peer_zmq_from_request_id(request_id, is_producer=False) + if peer_zmq is None: + raise ValueError("no peer zmq address for request") remote_host, _, remote_notify_port = parse_moriio_zmq_address(peer_zmq) except ValueError: logger.warning( @@ -534,15 +586,54 @@ class MoRIIOConnectorScheduler: num_external_tokens: int, connector_worker: "MoRIIOConnectorWorker | None" = None, ): + """Scheduler-side post-allocation hook (decode leg in WRITE mode). + + In WRITE mode this fires the decode->prefill "blocks ready" notify + that lets the producer RDMA-Write its KV into the freshly allocated + decode blocks. + + DP-rank routing contract (router-authoritative) + ------------------------------------------------ + Both legs of a disagg pair must agree on a single prefill DP rank, + otherwise the notify lands on a rank that never handshook and the + request hangs until ``VLLM_MORIIO_DEFERRED_TIMEOUT_S``. + + The routing authority is the external router/sidecar, NOT the + connector. The llm-d routing sidecar computes ``H = pickDPRank(uuid, + dp_size)`` (see ``dp_rank.go``) once and pins BOTH legs to ``H`` two + ways: the ``X-data-parallel-rank`` dispatch header (which vLLM engine + the leg lands on) and ``kv_transfer_params.remote_dp_rank=H`` (which + prefill rank this notify targets), with ``remote_dp_rank_override=True``. + + The connector consumes ``remote_dp_rank`` verbatim and does NOT + self-derive a rank: an independent hash here could disagree with the + router's dispatch pin and misroute the notify. For the returnable paths + (READ / serial WRITE) the prefill leg also echoes back the rank it ran + on via ``request_finished`` so routing is pure propagation. + + The owning rank is the only one that originates the notify + (``remote_dp_rank_override`` -> global-rank match; otherwise the + ``_is_kv_master`` anti-duplicate gate). See the inline comments + below for the exactly-once-across-pods reasoning. + """ params = request.kv_transfer_params if not params: return - transfer_id = params["transfer_id"] + # LLM-D sidecar compat: the nixlv2 routing sidecar emits NIXL-shaped + # kv_transfer_params without MoRI-IO's transfer_id. Synthesize one + # deterministically from request_id so producer and consumer agree + # without a sidecar wire-protocol change. + transfer_id = params.get("transfer_id") or f"sidecar-{request.request_id}" + params.setdefault("transfer_id", transfer_id) request_id = request.request_id self.map_request_id(request_id, transfer_id) if params.get("do_remote_decode"): local_block_ids = blocks.get_block_ids()[0] self._reqs_need_save[request.request_id] = (request, local_block_ids) + # Snapshot params now so chunked-prefill build_connector_meta + # can recover them on the final chunk even if the live + # request.kv_transfer_params has been mutated/cleared. + self._req_kv_params[request.request_id] = dict(params) if params is not None and params.get("do_remote_prefill"): if self.mode == MoRIIOMode.READ: @@ -568,6 +659,9 @@ class MoRIIOConnectorScheduler: request, local_block_ids, ) + # Snapshot params for chunked prefill consumption + # in build_connector_meta (see comment above). + self._req_kv_params[request.request_id] = dict(params) else: logger.warning( "Got invalid KVTransferParams: %s. This " @@ -582,37 +676,113 @@ class MoRIIOConnectorScheduler: ) remote_dp_rank = request.kv_transfer_params.get("remote_dp_rank", 0) - remote_host = request.kv_transfer_params.get("remote_host") - remote_notify_port = request.kv_transfer_params.get( - "remote_notify_port" - ) - if remote_host is None or remote_notify_port is None: + + # Effective DP fan-out for the per-pod port/host math below + # (see _remote_dp_rank_for_port / _pod_idx). Capped to the + # per-pod local size when the router advertises it (Wide-EP). + _dp_size = int(request.kv_transfer_params.get("remote_dp_size", 1) or 1) + try: + _dp_local = int( + request.kv_transfer_params.get("remote_dp_size_local", 0) or 0 + ) + if _dp_local > 0: + _dp_size = min(_dp_size, _dp_local) + except (TypeError, ValueError): + _dp_local = 0 + + # Rank routing is ROUTER-AUTHORITATIVE: honor the router-pinned + # remote_dp_rank (matched to the X-data-parallel-rank dispatch + # pin). Self-deriving a rank could disagree and notify a rank + # that never served the request. If unpinned in a multi-rank + # deployment, warn instead of guessing. + if ( + _dp_size > 1 + and "remote_dp_rank" not in request.kv_transfer_params + and "is_request_leader" not in request.kv_transfer_params + ): + logger.warning( + "MoRI-IO decode notify: remote_dp_size=%d but the router " + "did not pin remote_dp_rank for request %s; defaulting to " + "rank 0. The router must set remote_dp_rank (and the " + "X-data-parallel-rank dispatch header) so the prefill and " + "decode legs agree on the same rank.", + _dp_size, + request.request_id, + ) + + # Only the owning rank originates notify (exactly-once). + # Priority: is_request_leader > remote_dp_rank_override > _is_kv_master + _leader_flag = request.kv_transfer_params.get("is_request_leader") + if _leader_flag is not None: + _should_notify = bool(_leader_flag) + elif request.kv_transfer_params.get("remote_dp_rank_override"): + _should_notify = self._global_dp_rank == remote_dp_rank + else: + _should_notify = self._is_kv_master + if _should_notify: peer_zmq = get_peer_zmq_from_request_id( request.request_id, is_producer=False ) - remote_host, _, remote_notify_port = parse_moriio_zmq_address( - peer_zmq - ) - remote_notify_port = int(remote_notify_port) + if peer_zmq is not None: + remote_host, _, remote_notify_port = parse_moriio_zmq_address( + peer_zmq + ) + else: + # Sidecar fallback: use explicit params fields. + params = request.kv_transfer_params or {} + remote_host = params.get("remote_host") or "" + try: + remote_notify_port = int( + params.get("remote_notify_port") or 0 + ) + except (TypeError, ValueError): + remote_notify_port = 0 + if not remote_host or not remote_notify_port: + raise ValueError( + f"request {request.request_id!r}: " + f"request_id has no embedded peer " + f"zmq_address and kv_transfer_params is " + f"missing remote_host / remote_notify_port " + f"(got remote_host={remote_host!r}, " + f"remote_notify_port={remote_notify_port!r})" + ) - # num_external_tokens == 0: nothing to push, so don't tell the - # producer to write into these blocks. - block_notify_list = ( - blocks.get_block_ids()[0] if num_external_tokens > 0 else [] - ) - - for tp_index in range(self.tp_size): - target_port = remote_notify_port + get_port_offset( - remote_dp_rank, tp_index + # num_external_tokens == 0: nothing to push, so don't tell + # the producer to write into these blocks. + block_notify_list = ( + blocks.get_block_ids()[0] if num_external_tokens > 0 else [] ) - self.send_notify_block( - req_id=request.request_id, - transfer_id=request.kv_transfer_params["transfer_id"], - block_notify_list=block_notify_list, - host=remote_host, - port=target_port, + # Wide-EP multi-pod: a pod binds notify sockets only for + # its LOCAL ranks, so the port offset must use the per-pod + # local rank (% dp_local), not the global rank. Single-pod + # is bit-identical (modulus is a no-op). + _remote_dp_rank_for_port = fold_local_rank( + remote_dp_rank, _dp_local ) + # The target rank may live on a child pod at a different IP, + # so resolve the per-pod host (pod_idx = global // dp_local). + # Otherwise a notify for child ranks lands on the master + # pod and the request hangs in WAITING_FOR_REMOTE_KVS. + _notify_host = remote_host + _kvp = request.kv_transfer_params or {} + _remote_hosts = _kvp.get("remote_hosts") or [] + if _dp_local > 0 and _remote_hosts: + _pod_idx = pod_index(remote_dp_rank, _dp_local) + if 0 <= _pod_idx < len(_remote_hosts): + _notify_host = _remote_hosts[_pod_idx] + for tp_index in range(self.tp_size): + target_port = remote_notify_port + get_port_offset( + _remote_dp_rank_for_port, tp_index + ) + + self.send_notify_block( + req_id=request.request_id, + transfer_id=request.kv_transfer_params["transfer_id"], + block_notify_list=block_notify_list, + host=_notify_host, + port=target_port, + ) # Only trigger 1 KV transfer per request. @@ -636,6 +806,12 @@ class MoRIIOConnectorScheduler: if new_block_ids is not None: block_ids = new_block_ids[0] # TODO : hybrid attn, etc + # A non-disagg request (no kv_transfer_params, e.g. smoke + # test) is never registered in _reqs_need_pending_save; + # indexing it unconditionally would KeyError and crash the + # EngineCore. Skip it silently. + if req_id not in self._reqs_need_pending_save: + continue req, existing_blocks = self._reqs_need_pending_save[req_id] updated_blocks = list(existing_blocks) + (block_ids) self._reqs_need_pending_save[req_id] = (req, updated_blocks) @@ -643,25 +819,30 @@ class MoRIIOConnectorScheduler: len(self._reqs_need_pending_save[req_id][1]) * self.block_size >= req.num_prompt_tokens ): + # Final chunk: live kv_transfer_params may be cleared, + # so prefer the snapshot from update_state_after_alloc. + kv_params = self._req_kv_params.pop( + req_id, req.kv_transfer_params or {} + ) meta.add_new_req( request_id=req_id, local_block_ids=self._reqs_need_pending_save[req_id][1], - kv_transfer_params=req.kv_transfer_params or {}, + kv_transfer_params=kv_params, write_mode=True, ) del self._reqs_need_pending_save[req_id] # Loop through scheduled reqs and convert to ReqMeta. for req_id, (req, block_ids) in self._reqs_need_recv.items(): - assert req.kv_transfer_params is not None + kv_params = self._req_kv_params.get(req_id, req.kv_transfer_params or {}) meta.add_new_req( request_id=req_id, local_block_ids=block_ids, - kv_transfer_params=req.kv_transfer_params, + kv_transfer_params=kv_params, ) for req_id, (req, block_ids) in self._reqs_need_save.items(): - assert req.kv_transfer_params is not None + kv_params = self._req_kv_params.get(req_id, req.kv_transfer_params or {}) if req.num_prompt_tokens > len(block_ids) * self.block_size: # not last chunk prefill self._reqs_need_pending_save[req_id] = (req, block_ids) @@ -669,13 +850,22 @@ class MoRIIOConnectorScheduler: meta.add_new_req( request_id=req_id, local_block_ids=block_ids, - kv_transfer_params=req.kv_transfer_params, + kv_transfer_params=kv_params, write_mode=True, ) # Clear the list once workers start the transfers meta.reqs_to_send = self._reqs_need_send + # Reclaim snapshot cache entries that completed this step. Keep + # entries that are still pending (chunked prefill not yet at the + # final chunk) — those will be popped above on the final chunk. + for req_id in self._reqs_need_recv: + self._req_kv_params.pop(req_id, None) + for req_id in self._reqs_need_save: + if req_id not in self._reqs_need_pending_save: + self._req_kv_params.pop(req_id, None) + self._reqs_need_recv.clear() self._reqs_need_save.clear() self._reqs_need_send = {} @@ -702,14 +892,14 @@ class MoRIIOConnectorScheduler: """ request_id = request.request_id + params = request.kv_transfer_params # Consumer: can unmap transfer_id<->request_id immediately since done_recving # has fired at this point (i.e. KV has been transferred) # Producer: must keep the mapping until we get notification that blocks can # be freed, which may be several scheduler steps later. if not self.is_producer: - self.unmap_request_id(request_id) - - params = request.kv_transfer_params + transfer_id = params.get("transfer_id") if params else None + self.unmap_request_id(request_id, transfer_id=transfer_id) logger.debug( "MoriioConnector request_finished, request_status=%s, " "kv_transfer_params=%s", @@ -754,11 +944,17 @@ class MoRIIOConnectorScheduler: + MoRIIOConstants.VLLM_MORI_READ_ABORT_REQUEST_TIMEOUT ) self._deferred_send_deadlines[request.request_id] = ( - time.monotonic() + self._defer_timeout + time.monotonic() + self._defer_timeout, + params.get("transfer_id") if params else None, ) # Return KV transfer params forwarded verbatim to the decode instance by - # the router. + # the router. remote_dp_rank is the rank THIS prefill leg actually ran + # on: on the returnable paths (READ / serial WRITE) the router forwards + # it to the decode leg, so the decode->prefill notify is routed by pure + # propagation of the producer's real rank -- no hashing, no reliance on + # the router independently pinning the same rank. remote_dp_rank_override + # makes the decode side honor it via the global-rank match gate. return delay_free_blocks, dict( do_remote_prefill=True, do_remote_decode=False, @@ -767,61 +963,96 @@ class MoRIIOConnectorScheduler: remote_host=self.host_ip, remote_handshake_port=self.handshake_port, remote_notify_port=self.side_notify_port, + remote_dp_rank=self._global_dp_rank, + remote_dp_rank_override=True, remote_dp_size=self.vllm_config.parallel_config.data_parallel_size, + remote_dp_size_local=( + self.vllm_config.parallel_config.data_parallel_size_local + ), tp_size=self.vllm_config.parallel_config.tensor_parallel_size, transfer_id=params["transfer_id"], ) def update_connector_output(self, connector_output: KVConnectorOutput) -> None: - """Free KV blocks from sends that never received a completion signal. + """Reconcile worker finished_sending ACKs with the producer lifecycle. - Called every scheduler step. When a send is deferred (request_finished - returns True), blocks remain allocated until the worker reports - finished_sending. If that notification is lost (e.g. ibv_post_send - failure), blocks leak permanently. This method injects timed-out - entries into connector_output.finished_sending so the scheduler - frees them via the normal path. + Called every scheduler step (when there is KV connector output), + BEFORE the scheduler's finished_sending free loop. We rewrite + ``connector_output.finished_sending`` in place so it contains + exactly the producer requests that are safe for the scheduler to + free this step, letting the scheduler keep the plain upstream + ``assert req_id in self.requests`` + ``_free_blocks`` path: + + * An ACK whose request is in ``_deferred_send_deadlines`` + (request_finished ran with delay_free_blocks=True, so the + scheduler is holding its blocks) is surfaced -> freed now. + * An ACK that arrives BEFORE its request finished is parked in + ``_pending_sent_acks`` and released on a later step once the + request enters ``_deferred_send_deadlines``. + * A deferred send whose ACK never arrives is reaped after + ``_defer_timeout`` and surfaced, so leaked blocks are freed. + * A parked ACK that never matches a deferral before its own + deadline is a stale duplicate (e.g. a real ACK landing after the + send was already reaped) and is dropped. + + Consumers never populate finished_sending (they report + finished_recving), and they unmap in request_finished, so this is a + no-op for them. """ - # Producer: unmap transfer_id<->request_id for sends that are now (async) - # reported as completed. This unmapping have to be deferred until now - # so get_finished can use it in any scheduler step. - # Consumer: unmapping already done in request_finished - if self.is_producer and connector_output.finished_sending: - for req_id in connector_output.finished_sending: - self.unmap_request_id(req_id) - - if not self._deferred_send_deadlines: + if not self.is_producer: return - # Remove entries the worker already reported as finished_sending, these will be - # freed anyways. - for req_id in connector_output.finished_sending or (): - self._deferred_send_deadlines.pop(req_id, None) - + incoming = set(connector_output.finished_sending or ()) now = time.monotonic() - expired_reqs = [ + + # Surface ACKs whose request is already finished (blocks held for + # delayed free); park the rest until their request finishes. + safe: set[ReqId] = set() + for req_id in incoming: + if req_id in self._deferred_send_deadlines: + safe.add(req_id) + else: + self._pending_sent_acks.setdefault(req_id, now + self._defer_timeout) + + # Release previously parked ACKs whose request has since finished. + for req_id in self._pending_sent_acks: + if req_id in self._deferred_send_deadlines: + safe.add(req_id) + + # Reap deferred sends whose ACK never arrived (avoid leaking blocks). + expired = [ req_id - for req_id, deadline in self._deferred_send_deadlines.items() + for req_id, (deadline, _) in self._deferred_send_deadlines.items() if now >= deadline ] - if not expired_reqs: - return + if expired: + safe.update(expired) + logger.warning( + "Reaped %d deferred sends with no finished_sending " + "notification after %.0fs. This indicates lost async KV " + "completion notifications from the KV connector.", + len(expired), + self._defer_timeout, + ) - if connector_output.finished_sending is None: - connector_output.finished_sending = set() - # Register the expired requests as finished so the scheduler frees their blocks. - for req_id in expired_reqs: - connector_output.finished_sending.add(req_id) - del self._deferred_send_deadlines[req_id] - if self.is_producer: - self.unmap_request_id(req_id) - logger.warning( - "Reaped %d deferred sends with no finished_sending notification " - "after %.0fs. This indicates lost async KV completion " - "notifications from the KV connector.", - len(expired_reqs), - self._defer_timeout, - ) + # Finalize the requests we are surfacing: drop their deferral/park + # bookkeeping and unmap their transfer ids. + for req_id in safe: + deferred_info = self._deferred_send_deadlines.pop(req_id, None) + transfer_id = deferred_info[1] if deferred_info else None + self._pending_sent_acks.pop(req_id, None) + self.unmap_request_id(req_id, transfer_id=transfer_id) + + # Drop stale parked ACKs that never matched a deferral in time. + stale = [ + req_id + for req_id, deadline in self._pending_sent_acks.items() + if now >= deadline + ] + for req_id in stale: + self._pending_sent_acks.pop(req_id, None) + + connector_output.finished_sending = safe or None class MoRIIOConnectorWorker: @@ -998,6 +1229,10 @@ class MoRIIOConnectorWorker: self._recving_transfers: defaultdict[ReqId, list] = defaultdict(list) # Values are (remote_host, remote_notify_port, transfer_id). self._recving_transfers_callback_addr: dict[ReqId, tuple[str, str, str]] = {} + # Monotonic-clock start times for each in-flight recv transfer. + # Used by _pop_done_transfers to abort transfers whose RDMA + # completion is lost, instead of hanging forever. + self._recving_transfers_start: dict[str, float] = {} # Track the expiration time of requests that are waiting to be sent. self._reqs_to_send: dict[ReqId, float] = {} @@ -1368,6 +1603,11 @@ class MoRIIOConnectorWorker: port = int(meta.remote_handshake_port) tp_size = int(meta.tp_size) remote_dp_size = int(meta.remote_dp_size) + # Wide-EP multi-pod: remote DP ranks span pods at different IPs + # (ranks per pod = dp_local), so resolve the host per cur_dp_rank + # below instead of using a single host for all ranks. + pod_hosts = list(meta.multi_pod_hosts) if meta.multi_pod_hosts else [host] + remote_dp_size_local = int(meta.remote_dp_size_local) or remote_dp_size def request_ready(_f: Future[Any], entry=(req_id, meta)): logger.info("MoRIIO handshake done for request %s", req_id) @@ -1381,8 +1621,21 @@ class MoRIIOConnectorWorker: for cur_dp_rank in range(remote_dp_size): dp_engine_id = self.get_engine_name_with_dp(remote_engine_id, cur_dp_rank) + _pod_idx = pod_index(cur_dp_rank, remote_dp_size_local) + if _pod_idx >= len(pod_hosts): + _pod_idx = 0 + _per_rank_host = pod_hosts[_pod_idx] + # The handshake port offset must use the per-pod local rank, since + # each pod binds sockets only for its local ranks; dp_engine_id + # keeps the global rank for uniqueness. Single-pod is bit-identical. + _per_rank_local_dp = fold_local_rank(cur_dp_rank, remote_dp_size_local) future = self._handshake_initiation_executor.submit( - self._moriio_handshake, host, port, tp_size, dp_engine_id, cur_dp_rank + self._moriio_handshake, + _per_rank_host, + port, + tp_size, + dp_engine_id, + _per_rank_local_dp, ) fut_list.append(future) @@ -1463,7 +1716,19 @@ class MoRIIOConnectorWorker: block_shape = first_kv_cache.shape[-block_rank:] self.num_blocks = first_geometry.num_blocks self.slot_size_bytes = first_geometry.slot_size_bytes - assert first_geometry.block_size == self.block_size + if first_geometry.block_size != self.block_size: + # DeepSeek-V3 / MLA backends (e.g. FlashMLA) override the + # configured block_size at attention-layer creation time, so + # the KV cache tensor is laid out with a different (usually + # larger) block_size than vllm_config.cache_config.block_size. + # Trust the actual tensor shape and reconcile. + logger.info( + "KV cache block_size=%d differs from config block_size=%d; " + "using actual tensor shape (attention backend override).", + first_geometry.block_size, + self.block_size, + ) + self.block_size = first_geometry.block_size # TODO(tms): self.block_len needs to be per-layer for sliding window, # hybrid attn, etc # block size in bytes @@ -1634,6 +1899,7 @@ class MoRIIOConnectorWorker: def _pop_done_transfers(self) -> set[str]: done_req_ids: set[str] = set() + _xfer_timeout = int(os.environ.get("VLLM_MORIIO_TRANSFER_TIMEOUT_S", "120")) with self.moriio_wrapper.lock: to_remove = [] for req_id, status_list in self._recving_transfers.items(): @@ -1675,9 +1941,25 @@ class MoRIIOConnectorWorker: to_remove.append(req_id) # Do NOT add to done_req_ids: decode KV cache is incomplete. # The request will expire via the normal request timeout. + elif req_id in self._recving_transfers_start: + # Abort still-in-flight transfers that exceed the + # configured deadline. Otherwise a lost RDMA + # completion would leave the decode worker hung + # indefinitely on this request. + _age = time.monotonic() - self._recving_transfers_start[req_id] + if _age > _xfer_timeout: + logger.error( + "RDMA read TIMED OUT for req %s after %.1fs " + "(VLLM_MORIIO_TRANSFER_TIMEOUT_S=%d)", + req_id, + _age, + _xfer_timeout, + ) + to_remove.append(req_id) for req_id in to_remove: del self._recving_transfers[req_id] del self._recving_transfers_callback_addr[req_id] + self._recving_transfers_start.pop(req_id, None) return done_req_ids @@ -2043,6 +2325,18 @@ class MoRIIOConnectorWorker: ) def _write_blocks_for_req(self, req_id: ReqId, meta: ReqMeta, layer_name, kv_layer): + # Stash multi_pod_hosts + local DP size on the worker so + # MoRIIOEngine._finalize_if_complete (which sees only the WriteTask, + # not ReqMeta) can pick the per-rank pod IP for the completion notify. + # Last-writer-wins is safe: all requests share the same topology. + if meta.multi_pod_hosts: + self.multi_pod_hosts = list(meta.multi_pod_hosts) + else: + self.multi_pod_hosts = [meta.remote_host] + if meta.remote_dp_size_local: + self.remote_dp_size_local = int(meta.remote_dp_size_local) + else: + self.remote_dp_size_local = int(meta.remote_dp_size) self.schedule_write_blocks( request_id=req_id, transfer_id=meta.transfer_id, @@ -2271,6 +2565,7 @@ class MoRIIOConnectorWorker: _backoff = min(_backoff * 2, 0.05) with self.moriio_wrapper.lock: self._recving_transfers[request_id].append(transfer_status) + self._recving_transfers_start.setdefault(request_id, time.monotonic()) self._recving_transfers_callback_addr[request_id] = ( remote_host, str( diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_engine.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_engine.py index 5814fc8335..faff70278d 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_engine.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_engine.py @@ -311,7 +311,17 @@ class MoRIIOWriter: with self._write_state_lock: request_info.completion_request_id = task.request_id request_info.completion_remote_notify_port = task.remote_notify_port - request_info.completion_remote_ip = task.remote_ip + # Wide-EP multi-pod: task.remote_ip addresses only the first pod, + # so resolve the per-rank host from multi_pod_hosts (falls back to + # task.remote_ip for single-pod or on any indexing miss). + _remote_ip = task.remote_ip + _hosts = list(getattr(self.worker, "multi_pod_hosts", []) or []) + _dp_local = int(getattr(self.worker, "remote_dp_size_local", 0) or 0) + if _hosts and _dp_local > 0: + _pod_idx = int(request_info.decode_dp_rank) // _dp_local + if 0 <= _pod_idx < len(_hosts): + _remote_ip = _hosts[_pod_idx] + request_info.completion_remote_ip = _remote_ip if task.transfer_id in self._sealed_writes: request_info.writes_expected = self._sealed_writes[task.transfer_id] @@ -454,8 +464,15 @@ class MoRIIOWriter: # Wait for this request's transfers to complete. self.worker.moriio_wrapper.waiting_for_transfer_complete(transfer_statuses) + # The notify port offset must use the per-pod local rank + # (% dp_local), since each pod binds notify sockets only for its local + # ranks. Single-pod is bit-identical (modulus is a no-op). + _dp_local = int(getattr(self.worker, "remote_dp_size_local", 0) or 0) + _decode_dp_rank_for_port = int(request_info.decode_dp_rank) + if _dp_local > 0: + _decode_dp_rank_for_port = _decode_dp_rank_for_port % _dp_local remote_port = remote_notify_port + get_port_offset( - request_info.decode_dp_rank, self.worker.tp_rank + _decode_dp_rank_for_port, self.worker.tp_rank ) # Consider using RDMA immediate data in decode side # to eliminate the need for this notification.