llmd+vllm+mori-ep(inter node wide-ep)+mori-io(write) for 2p2d with dp=ep=16 tp=1 (#45043)
Signed-off-by: Shiksha Patel <shikpate@amd.com> Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Douglas Lehr
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
c810937573
commit
f42761204f
@@ -1,5 +1,20 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
#
|
||||
# Minimal single-node reference proxy for MoRI-IO prefill/decode disaggregation.
|
||||
# It demonstrates the DP-rank pinning contract the connector expects: pick one
|
||||
# prefill DP rank per request (flat_interleaved_dp_route) and pin BOTH legs to it
|
||||
# via the ``X-data-parallel-rank`` header plus ``kv_transfer_params``
|
||||
# (``remote_dp_rank`` / ``remote_dp_size`` / ``remote_tp_size``).
|
||||
#
|
||||
# Scope: single-node only. It intentionally does NOT supply the cross-pod
|
||||
# Wide-EP (2P2D) peer info -- ``remote_hosts`` / ``multi_pod_hosts``,
|
||||
# ``data_parallel_size_local``, ``is_request_leader``. In real multi-pod
|
||||
# deployments that peer info is produced by a production routing sidecar, e.g.:
|
||||
# * llm-d-router (https://github.com/llm-d/llm-d-router)
|
||||
# * vLLM's own vllm-router (examples in the vllm-project org)
|
||||
# * or any NIXL-shaped router that sets the same kv_transfer_params contract.
|
||||
# Use this file as a protocol reference, not a production router.
|
||||
import argparse
|
||||
import asyncio
|
||||
import copy
|
||||
@@ -181,7 +196,7 @@ async def send_request_to_prefill(
|
||||
raise RuntimeError(error_message)
|
||||
|
||||
|
||||
async def start_decode_request(endpoint, req_data, request_id):
|
||||
async def start_decode_request(endpoint, req_data, request_id, selected_dp_rank=None):
|
||||
session = aiohttp.ClientSession(
|
||||
timeout=aiohttp.ClientTimeout(total=6 * 6000 * 6000)
|
||||
)
|
||||
@@ -189,6 +204,10 @@ async def start_decode_request(endpoint, req_data, request_id):
|
||||
"Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}",
|
||||
"X-Request-Id": request_id,
|
||||
}
|
||||
# Pin the decode leg to the same DP rank as its prefill leg so both land on
|
||||
# the same rank (vLLM reads this header directly; no engine-side routing).
|
||||
if selected_dp_rank is not None:
|
||||
headers["X-data-parallel-rank"] = str(selected_dp_rank)
|
||||
response = await session.post(url=endpoint, json=req_data, headers=headers)
|
||||
return session, response
|
||||
|
||||
@@ -361,7 +380,9 @@ async def handle_request(api: str, request: Request):
|
||||
|
||||
decode_request_url = decode_instance_endpoint["request_address"] + api
|
||||
decode_request_task = asyncio.create_task(
|
||||
start_decode_request(decode_request_url, req_data, request_id)
|
||||
start_decode_request(
|
||||
decode_request_url, req_data, request_id, selected_prefill_dp_rank
|
||||
)
|
||||
)
|
||||
|
||||
session, decode_response = await decode_request_task
|
||||
|
||||
@@ -508,9 +508,15 @@ def test_send_transfer_release_sends_structured_release_message():
|
||||
scheduler._send_transfer_release("xfer-7", "127.0.0.1", 7000)
|
||||
|
||||
payload = sock.send.call_args.args[0]
|
||||
# WRITE-mode release advertises the consumer (decode) TP size so the prefill
|
||||
# side counts the right number of ACKs via get_moriio_expected_ack_count,
|
||||
# mirroring the READ-mode release in _pop_done_transfers (see
|
||||
# test_read_completion_sends_structured_release_with_consumer_tp_size). The
|
||||
# fixture's tp_size is 2, so consumer_tp_size == 2.
|
||||
assert msgspec.msgpack.decode(payload) == {
|
||||
"type": "release",
|
||||
"transfer_id": "xfer-7",
|
||||
"consumer_tp_size": 2,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for MoRIIO router-authoritative DP-rank routing.
|
||||
|
||||
These lock in the redesign that addresses the PR #45043 review feedback
|
||||
(njhill): the connector must NOT self-derive a DP rank, and the prefill leg
|
||||
must *return* the rank it actually ran on via ``request_finished`` so the
|
||||
decode->prefill notify is routed by pure propagation (READ / serial WRITE),
|
||||
not by an independent hash on each side.
|
||||
|
||||
Following test_moriio_unmap.py, we bind the unbound method to a lightweight
|
||||
stand-in rather than constructing a full scheduler.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import (
|
||||
MoRIIOConnectorScheduler,
|
||||
)
|
||||
from vllm.v1.request import RequestStatus
|
||||
|
||||
_request_finished = MoRIIOConnectorScheduler.request_finished
|
||||
|
||||
|
||||
def _producer(global_dp_rank: int, dp_size: int, dp_size_local: int):
|
||||
"""Minimal prefill-side (producer) scheduler stand-in for request_finished."""
|
||||
return SimpleNamespace(
|
||||
is_producer=True,
|
||||
_global_dp_rank=global_dp_rank,
|
||||
engine_id="engine-abc",
|
||||
host_ip="10.0.0.1",
|
||||
handshake_port=6300,
|
||||
side_notify_port=7300,
|
||||
vllm_config=SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(
|
||||
data_parallel_size=dp_size,
|
||||
data_parallel_size_local=dp_size_local,
|
||||
tensor_parallel_size=1,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _req(
|
||||
request_id: str, params: dict | None, status=RequestStatus.FINISHED_LENGTH_CAPPED
|
||||
):
|
||||
return SimpleNamespace(
|
||||
request_id=request_id,
|
||||
kv_transfer_params=params,
|
||||
status=status,
|
||||
)
|
||||
|
||||
|
||||
def test_request_finished_returns_prefill_rank_for_propagation():
|
||||
# 2P2D: global rank 11 lives on pod 1 (ranks 8..15), local size 8.
|
||||
s = _producer(global_dp_rank=11, dp_size=16, dp_size_local=8)
|
||||
req = _req("r1", {"do_remote_decode": True, "transfer_id": "t1"})
|
||||
|
||||
delay_free, params = _request_finished(s, req, block_ids=[])
|
||||
|
||||
# The prefill leg returns the rank IT actually ran on, marked authoritative,
|
||||
# so the router forwards it to decode and the notify propagates (no hashing).
|
||||
assert params is not None
|
||||
assert params["remote_dp_rank"] == 11
|
||||
assert params["remote_dp_rank_override"] is True
|
||||
assert params["remote_dp_size"] == 16
|
||||
assert params["remote_dp_size_local"] == 8
|
||||
assert params["transfer_id"] == "t1"
|
||||
# Empty block_ids => nothing deferred, blocks freed immediately.
|
||||
assert delay_free is False
|
||||
|
||||
|
||||
def test_request_finished_rank_tracks_the_actual_rank():
|
||||
# A different rank must be echoed verbatim -- the value is the producer's
|
||||
# own rank, never a recomputation of the request id.
|
||||
s = _producer(global_dp_rank=4, dp_size=8, dp_size_local=8)
|
||||
req = _req("r2", {"do_remote_decode": True, "transfer_id": "t2"})
|
||||
|
||||
_, params = _request_finished(s, req, block_ids=[])
|
||||
|
||||
assert params["remote_dp_rank"] == 4
|
||||
assert params["remote_dp_size_local"] == 8
|
||||
|
||||
|
||||
def test_request_finished_no_kv_params_is_noop():
|
||||
s = _producer(global_dp_rank=0, dp_size=16, dp_size_local=8)
|
||||
req = _req("r3", None)
|
||||
|
||||
delay_free, params = _request_finished(s, req, block_ids=[])
|
||||
|
||||
assert delay_free is False
|
||||
assert params is None
|
||||
@@ -296,6 +296,8 @@ def test_read_completion_sends_structured_release_with_consumer_tp_size():
|
||||
worker._recving_transfers_callback_addr = {
|
||||
"req": ("127.0.0.1", "7000", "tx-release")
|
||||
}
|
||||
# Transfer-timeout reaping state consulted by _pop_done_transfers.
|
||||
worker._recving_transfers_start = {}
|
||||
|
||||
assert worker._pop_done_transfers() == {"tx-release"}
|
||||
assert worker.moriio_wrapper.sent == [
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for MoRIIOConnectorScheduler.{map,unmap}_request_id.
|
||||
|
||||
vLLM's input_processor mutates a request_id after the connector has stored it
|
||||
(e.g. ``cmpl-abc-0`` -> ``cmpl-abc-0-956053a4``), so an exact-match unmap can
|
||||
miss. These tests cover the stable-``transfer_id`` fallback that handles that
|
||||
case (and the clean exact-match and miss paths).
|
||||
|
||||
The map/unmap methods touch only the two id<->id dicts, so we bind them to a
|
||||
lightweight stand-in rather than constructing a full scheduler.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import (
|
||||
MoRIIOConnectorScheduler,
|
||||
)
|
||||
|
||||
_map = MoRIIOConnectorScheduler.map_request_id
|
||||
_unmap = MoRIIOConnectorScheduler.unmap_request_id
|
||||
|
||||
|
||||
def _sched():
|
||||
return SimpleNamespace(
|
||||
transfer_id_to_request_id={},
|
||||
request_id_to_transfer_id={},
|
||||
)
|
||||
|
||||
|
||||
def test_exact_match_unmap_clears_both_tables():
|
||||
s = _sched()
|
||||
_map(s, "rid-1", "tid-1")
|
||||
_unmap(s, "rid-1")
|
||||
assert s.request_id_to_transfer_id == {}
|
||||
assert s.transfer_id_to_request_id == {}
|
||||
|
||||
|
||||
def test_unmap_via_transfer_id_after_request_id_mutation():
|
||||
s = _sched()
|
||||
# Connector stores the ORIGINAL request_id at map time.
|
||||
_map(s, "cmpl-abc-0", "tid-42")
|
||||
# input_processor later mutates the rid; exact match would miss, so the
|
||||
# stable transfer_id is used to find and clear the original entry.
|
||||
_unmap(s, "cmpl-abc-0-956053a4", transfer_id="tid-42")
|
||||
assert s.request_id_to_transfer_id == {}
|
||||
assert s.transfer_id_to_request_id == {}
|
||||
|
||||
|
||||
def test_unmap_miss_is_noop_and_does_not_raise():
|
||||
s = _sched()
|
||||
_map(s, "rid-1", "tid-1")
|
||||
# Unknown rid and unknown/None transfer_id: must not raise, must not touch
|
||||
# the existing (unrelated) mapping.
|
||||
_unmap(s, "rid-unknown", transfer_id=None)
|
||||
_unmap(s, "rid-unknown", transfer_id="tid-unknown")
|
||||
assert s.request_id_to_transfer_id == {"rid-1": "tid-1"}
|
||||
assert s.transfer_id_to_request_id == {"tid-1": "rid-1"}
|
||||
|
||||
|
||||
def test_exact_match_preferred_over_transfer_id_fallback():
|
||||
s = _sched()
|
||||
_map(s, "rid-1", "tid-1")
|
||||
# Exact rid is present, so it is removed directly even if a transfer_id is
|
||||
# also supplied; no stale entries left behind.
|
||||
_unmap(s, "rid-1", transfer_id="tid-1")
|
||||
assert s.request_id_to_transfer_id == {}
|
||||
assert s.transfer_id_to_request_id == {}
|
||||
@@ -191,6 +191,25 @@ def get_port_offset(dp_rank: int, tp_rank: int, tp_size: int = 1) -> int:
|
||||
return (dp_rank) * tp_size + tp_rank
|
||||
|
||||
|
||||
def fold_local_rank(global_dp_rank: int, dp_size_local: int) -> int:
|
||||
"""Fold a global DP rank into its pod-local rank [0, dp_size_local).
|
||||
|
||||
``dp_size_local == 0`` is the external-DP sentinel (local size unknown):
|
||||
return the rank unchanged, since a global DP rank is always < the global
|
||||
DP size so no folding is needed and the modulo is skipped (never divides
|
||||
by zero).
|
||||
"""
|
||||
return global_dp_rank % dp_size_local if dp_size_local else global_dp_rank
|
||||
|
||||
|
||||
def pod_index(global_dp_rank: int, dp_size_local: int) -> int:
|
||||
"""Pod index (0-based) a global DP rank lives on for Wide-EP multi-pod.
|
||||
|
||||
``dp_size_local == 0`` (external-DP sentinel) collapses to a single pod.
|
||||
"""
|
||||
return global_dp_rank // dp_size_local if dp_size_local else 0
|
||||
|
||||
|
||||
def resolve_host_ip(extra_config: dict) -> str:
|
||||
"""The IP this MoRIIO process advertises for KV transfer.
|
||||
|
||||
@@ -280,9 +299,11 @@ class MoRIIOConfig:
|
||||
kv_transfer_config = vllm_config.kv_transfer_config
|
||||
extra_config = kv_transfer_config.kv_connector_extra_config
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
dp_rank = vllm_config.parallel_config.data_parallel_rank
|
||||
# Fold the global data_parallel_rank back to [0, dp_size_local) for
|
||||
# per-node port allocation (handles the external-DP sentinel).
|
||||
pc = vllm_config.parallel_config
|
||||
dp_rank = fold_local_rank(pc.data_parallel_rank, pc.data_parallel_size_local)
|
||||
base_notify_port = int(extra_config["notify_port"])
|
||||
dp_size = vllm_config.parallel_config.data_parallel_size
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
port_offset = get_port_offset(dp_rank, tp_rank)
|
||||
backend = str(extra_config.get("backend", "rdma")).lower()
|
||||
@@ -304,15 +325,18 @@ class MoRIIOConfig:
|
||||
return cls(
|
||||
local_ip=resolve_host_ip(extra_config),
|
||||
local_kv_port=get_open_port(),
|
||||
proxy_ip=extra_config["proxy_ip"],
|
||||
proxy_ip=extra_config.get("proxy_ip", ""),
|
||||
local_ping_port=get_open_port(),
|
||||
proxy_ping_port=int(extra_config["proxy_ping_port"]),
|
||||
proxy_ping_port=int(extra_config.get("proxy_ping_port", 0)),
|
||||
http_port=int(extra_config["http_port"]),
|
||||
handshake_port=int(extra_config["handshake_port"]),
|
||||
notify_port=base_notify_port + port_offset,
|
||||
tp_rank=tp_rank,
|
||||
dp_rank=dp_rank,
|
||||
dp_size=dp_size,
|
||||
# Advertise the GLOBAL DP world size to the routing proxy so it can
|
||||
# address every rank across pods; dp_size_local above is only for
|
||||
# per-pod port folding, not for routing.
|
||||
dp_size=pc.data_parallel_size,
|
||||
tp_size=tp_size,
|
||||
read_mode=get_moriio_mode(kv_transfer_config) == MoRIIOMode.READ,
|
||||
qp_per_transfer=int(extra_config.get("qp_per_transfer", 1)),
|
||||
@@ -391,20 +415,26 @@ def parse_moriio_zmq_address(
|
||||
return host, handshake_port, notify_port
|
||||
|
||||
|
||||
def get_peer_zmq_from_request_id(request_id: str, is_producer: bool) -> str:
|
||||
def get_peer_zmq_from_request_id(request_id: str, is_producer: bool) -> str | None:
|
||||
"""Extract the *peer's* zmq_address from the vLLM router request_id.
|
||||
|
||||
The producer (prefill) needs the decode's address; the consumer (decode)
|
||||
needs the prefill's address.
|
||||
|
||||
Returns ``None`` when the request_id does not encode peer info. The
|
||||
llm-d routing sidecar (``llm-d-inference-scheduler``) does not embed
|
||||
addresses in ``request_id``; instead it passes ``remote_host``,
|
||||
``remote_handshake_port`` and ``remote_notify_port`` explicitly in
|
||||
``kv_transfer_params``. Callers must handle the ``None`` return by
|
||||
falling back to those fields. See ``add_new_req`` for the canonical
|
||||
fallback path.
|
||||
"""
|
||||
if is_producer:
|
||||
m = _DECODE_ZMQ_RE.search(request_id)
|
||||
else:
|
||||
m = _PREFILL_ZMQ_RE.search(request_id)
|
||||
if m is None:
|
||||
raise ValueError(
|
||||
f"Cannot parse peer zmq_address from request_id: {request_id!r}"
|
||||
)
|
||||
return None
|
||||
return m.group(1)
|
||||
|
||||
|
||||
@@ -426,6 +456,10 @@ class ReqMeta:
|
||||
# read must target this rank's memory registration; the default 0 preserves
|
||||
# the symmetric single-DP behaviour.
|
||||
remote_dp_rank: int = 0
|
||||
# Multi-pod: list of remote pod IPs indexed by pod_idx.
|
||||
multi_pod_hosts: list[str] = field(default_factory=list)
|
||||
# Per-pod DP size; 0 means fallback to remote_dp_size.
|
||||
remote_dp_size_local: int = 0
|
||||
|
||||
|
||||
class MoRIIOConnectorMetadata(KVConnectorMetadata):
|
||||
@@ -450,23 +484,76 @@ class MoRIIOConnectorMetadata(KVConnectorMetadata):
|
||||
kv_transfer_params: dict[str, Any],
|
||||
write_mode=False,
|
||||
):
|
||||
"""Ingest a peer's ``kv_transfer_params`` into a typed ``ReqMeta``.
|
||||
|
||||
This is the single place peer info enters the connector. The
|
||||
``kv_transfer_params`` contract (as produced by the llm-d sidecar /
|
||||
vLLM router, or echoed by the prefill leg's ``request_finished``):
|
||||
|
||||
Required (always):
|
||||
* ``transfer_id`` -- stable id shared by both legs.
|
||||
* ``remote_engine_id`` -- peer engine id for the handshake table.
|
||||
* ``remote_block_ids`` -- peer block ids (may be [] in WRITE mode,
|
||||
where decode allocates its own blocks).
|
||||
|
||||
Peer address -- ONE of the following two must resolve:
|
||||
* embedded in ``request_id`` (vLLM-router PD id form), OR
|
||||
* explicit ``remote_host`` + ``remote_handshake_port`` +
|
||||
``remote_notify_port`` (llm-d sidecar / returnable path).
|
||||
If neither resolves we raise -- there is no safe default host/port.
|
||||
|
||||
Optional (defaulted):
|
||||
* ``tp_size`` (default 1) -- peer TP size.
|
||||
* ``remote_dp_size`` (default 1) -- peer GLOBAL DP size.
|
||||
* ``remote_dp_size_local`` (default = ``remote_dp_size``) -- per-pod
|
||||
DP size for Wide-EP multi-pod port/host
|
||||
folding; 0/absent means single-pod.
|
||||
* ``remote_hosts`` (default [remote_host]) -- per-pod IP list
|
||||
indexed by ``pod_idx = rank // dp_local``.
|
||||
|
||||
Routing keys consumed elsewhere (NOT here): ``remote_dp_rank`` /
|
||||
``remote_dp_rank_override`` gate the decode->prefill notify target in
|
||||
MoRIIOConnectorScheduler; they are router-authoritative and never
|
||||
self-derived (see that class's request routing contract).
|
||||
"""
|
||||
transfer_id = kv_transfer_params["transfer_id"]
|
||||
|
||||
remote_host = kv_transfer_params.get("remote_host")
|
||||
remote_handshake_port = kv_transfer_params.get("remote_handshake_port")
|
||||
remote_notify_port = kv_transfer_params.get("remote_notify_port")
|
||||
if (
|
||||
remote_host is None
|
||||
or remote_handshake_port is None
|
||||
or remote_notify_port is None
|
||||
):
|
||||
# Parse host/ports from the request_id. The router embeds both
|
||||
# zmq_addresses in PD request IDs, but WRITE decode requests may carry
|
||||
# a plain request ID and get the remote address via kv_transfer_params.
|
||||
peer_zmq = get_peer_zmq_from_request_id(request_id, is_producer=write_mode)
|
||||
# Try request_id embedded address first, fallback to explicit params.
|
||||
peer_zmq = get_peer_zmq_from_request_id(request_id, is_producer=write_mode)
|
||||
if peer_zmq is not None:
|
||||
remote_host, remote_handshake_port, remote_notify_port = (
|
||||
parse_moriio_zmq_address(peer_zmq)
|
||||
)
|
||||
else:
|
||||
try:
|
||||
remote_host = kv_transfer_params["remote_host"]
|
||||
if not remote_host:
|
||||
raise ValueError(
|
||||
f"request_id {request_id!r} does not embed a peer "
|
||||
f"zmq_address and kv_transfer_params['remote_host'] is "
|
||||
f"empty; cannot route MoRI-IO transfer"
|
||||
)
|
||||
remote_handshake_port = int(kv_transfer_params["remote_handshake_port"])
|
||||
remote_notify_port = int(kv_transfer_params["remote_notify_port"])
|
||||
except (KeyError, TypeError, ValueError) as e:
|
||||
raise ValueError(
|
||||
f"request_id {request_id!r} does not embed a peer "
|
||||
f"zmq_address and kv_transfer_params is missing one or "
|
||||
f"more sidecar-fallback keys (need remote_host, "
|
||||
f"remote_handshake_port, remote_notify_port): {e}"
|
||||
) from e
|
||||
|
||||
# Multi-pod: use multi_pod_hosts list or fallback to single host.
|
||||
_pod_hosts = kv_transfer_params.get("remote_hosts") or [remote_host]
|
||||
if not isinstance(_pod_hosts, list):
|
||||
_pod_hosts = [_pod_hosts]
|
||||
_pod_hosts = [str(h) for h in _pod_hosts]
|
||||
_remote_dp_size_local = int(
|
||||
kv_transfer_params.get(
|
||||
"remote_dp_size_local",
|
||||
kv_transfer_params.get("remote_dp_size", 1),
|
||||
)
|
||||
)
|
||||
|
||||
_req = ReqMeta(
|
||||
transfer_id=transfer_id,
|
||||
@@ -488,6 +575,8 @@ class MoRIIOConnectorMetadata(KVConnectorMetadata):
|
||||
),
|
||||
remote_dp_size=kv_transfer_params.get("remote_dp_size", 1),
|
||||
remote_dp_rank=kv_transfer_params.get("remote_dp_rank", 0),
|
||||
multi_pod_hosts=_pod_hosts,
|
||||
remote_dp_size_local=_remote_dp_size_local,
|
||||
)
|
||||
if write_mode:
|
||||
self.reqs_to_save[request_id] = _req
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
@@ -36,11 +37,13 @@ from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import (
|
||||
ReqMeta,
|
||||
TransferId,
|
||||
WriteTask,
|
||||
fold_local_rank,
|
||||
get_moriio_mode,
|
||||
get_peer_zmq_from_request_id,
|
||||
get_port_offset,
|
||||
get_role,
|
||||
parse_moriio_zmq_address,
|
||||
pod_index,
|
||||
resolve_host_ip,
|
||||
set_role,
|
||||
zmq_ctx,
|
||||
@@ -373,13 +376,29 @@ class MoRIIOConnectorScheduler:
|
||||
"notify_port"
|
||||
]
|
||||
self.tp_size = self.vllm_config.parallel_config.tensor_parallel_size
|
||||
self.dp_rank = self.vllm_config.parallel_config.data_parallel_rank
|
||||
# Local DP rank (0..dp_local-1) for port arithmetic; fold_local_rank
|
||||
# handles the external-DP sentinel (dp_local==0) so the arithmetic
|
||||
# can't ZeroDivisionError at init.
|
||||
_pc = self.vllm_config.parallel_config
|
||||
self.dp_rank = fold_local_rank(
|
||||
_pc.data_parallel_rank, _pc.data_parallel_size_local
|
||||
)
|
||||
# Only first-pod ranks originate notify to avoid duplicates. Guarding
|
||||
# via pod_index keeps _is_kv_master True for the external-DP sentinel
|
||||
# (single pod) instead of being silently stuck False.
|
||||
self._is_kv_master = (
|
||||
pod_index(_pc.data_parallel_rank, _pc.data_parallel_size_local) == 0
|
||||
)
|
||||
# Global DP rank for pinned request ownership check.
|
||||
self._global_dp_rank = self.vllm_config.parallel_config.data_parallel_rank
|
||||
self.is_producer = self.kv_transfer_config.kv_role == "kv_producer"
|
||||
# Requests that need to start recv/send.
|
||||
# New requests are added by update_state_after_alloc in
|
||||
# the scheduler. Used to make metadata passed to Worker.
|
||||
self._reqs_need_recv: dict[ReqId, tuple[Request, list[int]]] = {}
|
||||
self._reqs_need_save: dict[ReqId, tuple[Request, list[int]]] = {}
|
||||
# Snapshot of kv_transfer_params for chunked prefill recovery.
|
||||
self._req_kv_params: dict[ReqId, dict] = {}
|
||||
|
||||
# For chunked prefill, we perform layer-wise access within the final chunk.
|
||||
# TODO: Perform transfer at end chunk.
|
||||
@@ -396,12 +415,15 @@ class MoRIIOConnectorScheduler:
|
||||
# finished_sending before the deadline, we inject them into
|
||||
# connector_output.finished_sending so the scheduler frees the blocks to avoid
|
||||
# hanging indefinitely waiting for a free notification that never comes.
|
||||
self._deferred_send_deadlines: dict[ReqId, float] = {}
|
||||
# Value: (deadline, transfer_id) for unmapping after mutation.
|
||||
self._deferred_send_deadlines: dict[ReqId, tuple[float, TransferId | None]] = {}
|
||||
self._defer_timeout = float(
|
||||
self.kv_transfer_config.kv_connector_extra_config.get(
|
||||
"defer_timeout", MoRIIOConstants.DEFAULT_DEFER_TIMEOUT
|
||||
)
|
||||
)
|
||||
# Buffer for early ACKs that arrive before request_finished.
|
||||
self._pending_sent_acks: dict[ReqId, float] = {}
|
||||
self.paths: dict[str, zmq.Socket] = {}
|
||||
self.transfer_id_to_request_id: dict[TransferId, ReqId] = {}
|
||||
self.request_id_to_transfer_id: dict[ReqId, TransferId] = {}
|
||||
@@ -410,23 +432,36 @@ class MoRIIOConnectorScheduler:
|
||||
self.transfer_id_to_request_id[transfer_id] = request_id
|
||||
self.request_id_to_transfer_id[request_id] = transfer_id
|
||||
|
||||
def unmap_request_id(self, request_id: ReqId):
|
||||
def unmap_request_id(
|
||||
self, request_id: ReqId, transfer_id: TransferId | None = None
|
||||
):
|
||||
"""Unmap request_id/transfer_id. Uses transfer_id for lookup if
|
||||
exact request_id match fails (handles input_processor mutation)."""
|
||||
if request_id in self.request_id_to_transfer_id:
|
||||
transfer_id = self.request_id_to_transfer_id[request_id]
|
||||
tid = self.request_id_to_transfer_id[request_id]
|
||||
del self.request_id_to_transfer_id[request_id]
|
||||
if transfer_id in self.transfer_id_to_request_id:
|
||||
del self.transfer_id_to_request_id[transfer_id]
|
||||
else:
|
||||
logger.warning(
|
||||
"transfer id not in transfer_id_to_request_id lookup"
|
||||
"table. there is likely a bug!"
|
||||
if tid in self.transfer_id_to_request_id:
|
||||
del self.transfer_id_to_request_id[tid]
|
||||
return
|
||||
|
||||
# Fallback: use transfer_id to find original request_id.
|
||||
if transfer_id is not None and transfer_id in self.transfer_id_to_request_id:
|
||||
original_rid = self.transfer_id_to_request_id[transfer_id]
|
||||
if original_rid != request_id:
|
||||
logger.debug(
|
||||
"MoRI-IO unmap via transfer_id: %r -> %r", request_id, original_rid
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"Could not find %s in transfer_id_to_request_id"
|
||||
"lookup table. This could lead to a possible hang.",
|
||||
request_id,
|
||||
)
|
||||
if original_rid in self.request_id_to_transfer_id:
|
||||
del self.request_id_to_transfer_id[original_rid]
|
||||
del self.transfer_id_to_request_id[transfer_id]
|
||||
return
|
||||
|
||||
logger.warning(
|
||||
"MoRI-IO unmap MISS: rid=%r transfer_id=%r table_size=%d",
|
||||
request_id,
|
||||
transfer_id,
|
||||
len(self.request_id_to_transfer_id),
|
||||
)
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self,
|
||||
@@ -478,7 +513,12 @@ class MoRIIOConnectorScheduler:
|
||||
"req_id": req_id,
|
||||
"transfer_id": transfer_id,
|
||||
"block_notify_list": block_notify_list or [],
|
||||
"decode_rank": self.dp_rank,
|
||||
# GLOBAL decode dp rank: producer derives the per-pod notify offset
|
||||
# (% dp_local), owning pod index (// dp_local), and write-target
|
||||
# from it. Sending the LOCAL rank made child-pod consumers look
|
||||
# like master rank 0 and hang in WAITING_FOR_REMOTE_KVS. Single-pod:
|
||||
# local == global.
|
||||
"decode_rank": self._global_dp_rank,
|
||||
"type": "remote_blocks",
|
||||
}
|
||||
serialized_data = msgpack.dumps(data)
|
||||
@@ -493,8 +533,18 @@ class MoRIIOConnectorScheduler:
|
||||
)
|
||||
self.paths[path] = sock
|
||||
|
||||
# Advertise the consumer (decode) TP size so prefill counts the right
|
||||
# number of ACKs (get_moriio_expected_ack_count; see the upstream
|
||||
# READ-mode release in _pop_done_transfers). Resolves to 1 for our
|
||||
# homogeneous TP=1 configs; only matters under heterogeneous-TP fan-in.
|
||||
self.paths[path].send(
|
||||
msgpack.dumps({"type": "release", "transfer_id": transfer_id})
|
||||
msgpack.dumps(
|
||||
{
|
||||
"type": "release",
|
||||
"transfer_id": transfer_id,
|
||||
"consumer_tp_size": self.tp_size,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
def _release_write_prefill_blocks(self, request_id: ReqId, params: dict[str, Any]):
|
||||
@@ -513,6 +563,8 @@ class MoRIIOConnectorScheduler:
|
||||
if remote_host is None or remote_notify_port is None:
|
||||
try:
|
||||
peer_zmq = get_peer_zmq_from_request_id(request_id, is_producer=False)
|
||||
if peer_zmq is None:
|
||||
raise ValueError("no peer zmq address for request")
|
||||
remote_host, _, remote_notify_port = parse_moriio_zmq_address(peer_zmq)
|
||||
except ValueError:
|
||||
logger.warning(
|
||||
@@ -534,15 +586,54 @@ class MoRIIOConnectorScheduler:
|
||||
num_external_tokens: int,
|
||||
connector_worker: "MoRIIOConnectorWorker | None" = None,
|
||||
):
|
||||
"""Scheduler-side post-allocation hook (decode leg in WRITE mode).
|
||||
|
||||
In WRITE mode this fires the decode->prefill "blocks ready" notify
|
||||
that lets the producer RDMA-Write its KV into the freshly allocated
|
||||
decode blocks.
|
||||
|
||||
DP-rank routing contract (router-authoritative)
|
||||
------------------------------------------------
|
||||
Both legs of a disagg pair must agree on a single prefill DP rank,
|
||||
otherwise the notify lands on a rank that never handshook and the
|
||||
request hangs until ``VLLM_MORIIO_DEFERRED_TIMEOUT_S``.
|
||||
|
||||
The routing authority is the external router/sidecar, NOT the
|
||||
connector. The llm-d routing sidecar computes ``H = pickDPRank(uuid,
|
||||
dp_size)`` (see ``dp_rank.go``) once and pins BOTH legs to ``H`` two
|
||||
ways: the ``X-data-parallel-rank`` dispatch header (which vLLM engine
|
||||
the leg lands on) and ``kv_transfer_params.remote_dp_rank=H`` (which
|
||||
prefill rank this notify targets), with ``remote_dp_rank_override=True``.
|
||||
|
||||
The connector consumes ``remote_dp_rank`` verbatim and does NOT
|
||||
self-derive a rank: an independent hash here could disagree with the
|
||||
router's dispatch pin and misroute the notify. For the returnable paths
|
||||
(READ / serial WRITE) the prefill leg also echoes back the rank it ran
|
||||
on via ``request_finished`` so routing is pure propagation.
|
||||
|
||||
The owning rank is the only one that originates the notify
|
||||
(``remote_dp_rank_override`` -> global-rank match; otherwise the
|
||||
``_is_kv_master`` anti-duplicate gate). See the inline comments
|
||||
below for the exactly-once-across-pods reasoning.
|
||||
"""
|
||||
params = request.kv_transfer_params
|
||||
if not params:
|
||||
return
|
||||
transfer_id = params["transfer_id"]
|
||||
# LLM-D sidecar compat: the nixlv2 routing sidecar emits NIXL-shaped
|
||||
# kv_transfer_params without MoRI-IO's transfer_id. Synthesize one
|
||||
# deterministically from request_id so producer and consumer agree
|
||||
# without a sidecar wire-protocol change.
|
||||
transfer_id = params.get("transfer_id") or f"sidecar-{request.request_id}"
|
||||
params.setdefault("transfer_id", transfer_id)
|
||||
request_id = request.request_id
|
||||
self.map_request_id(request_id, transfer_id)
|
||||
if params.get("do_remote_decode"):
|
||||
local_block_ids = blocks.get_block_ids()[0]
|
||||
self._reqs_need_save[request.request_id] = (request, local_block_ids)
|
||||
# Snapshot params now so chunked-prefill build_connector_meta
|
||||
# can recover them on the final chunk even if the live
|
||||
# request.kv_transfer_params has been mutated/cleared.
|
||||
self._req_kv_params[request.request_id] = dict(params)
|
||||
|
||||
if params is not None and params.get("do_remote_prefill"):
|
||||
if self.mode == MoRIIOMode.READ:
|
||||
@@ -568,6 +659,9 @@ class MoRIIOConnectorScheduler:
|
||||
request,
|
||||
local_block_ids,
|
||||
)
|
||||
# Snapshot params for chunked prefill consumption
|
||||
# in build_connector_meta (see comment above).
|
||||
self._req_kv_params[request.request_id] = dict(params)
|
||||
else:
|
||||
logger.warning(
|
||||
"Got invalid KVTransferParams: %s. This "
|
||||
@@ -582,37 +676,113 @@ class MoRIIOConnectorScheduler:
|
||||
)
|
||||
|
||||
remote_dp_rank = request.kv_transfer_params.get("remote_dp_rank", 0)
|
||||
remote_host = request.kv_transfer_params.get("remote_host")
|
||||
remote_notify_port = request.kv_transfer_params.get(
|
||||
"remote_notify_port"
|
||||
)
|
||||
if remote_host is None or remote_notify_port is None:
|
||||
|
||||
# Effective DP fan-out for the per-pod port/host math below
|
||||
# (see _remote_dp_rank_for_port / _pod_idx). Capped to the
|
||||
# per-pod local size when the router advertises it (Wide-EP).
|
||||
_dp_size = int(request.kv_transfer_params.get("remote_dp_size", 1) or 1)
|
||||
try:
|
||||
_dp_local = int(
|
||||
request.kv_transfer_params.get("remote_dp_size_local", 0) or 0
|
||||
)
|
||||
if _dp_local > 0:
|
||||
_dp_size = min(_dp_size, _dp_local)
|
||||
except (TypeError, ValueError):
|
||||
_dp_local = 0
|
||||
|
||||
# Rank routing is ROUTER-AUTHORITATIVE: honor the router-pinned
|
||||
# remote_dp_rank (matched to the X-data-parallel-rank dispatch
|
||||
# pin). Self-deriving a rank could disagree and notify a rank
|
||||
# that never served the request. If unpinned in a multi-rank
|
||||
# deployment, warn instead of guessing.
|
||||
if (
|
||||
_dp_size > 1
|
||||
and "remote_dp_rank" not in request.kv_transfer_params
|
||||
and "is_request_leader" not in request.kv_transfer_params
|
||||
):
|
||||
logger.warning(
|
||||
"MoRI-IO decode notify: remote_dp_size=%d but the router "
|
||||
"did not pin remote_dp_rank for request %s; defaulting to "
|
||||
"rank 0. The router must set remote_dp_rank (and the "
|
||||
"X-data-parallel-rank dispatch header) so the prefill and "
|
||||
"decode legs agree on the same rank.",
|
||||
_dp_size,
|
||||
request.request_id,
|
||||
)
|
||||
|
||||
# Only the owning rank originates notify (exactly-once).
|
||||
# Priority: is_request_leader > remote_dp_rank_override > _is_kv_master
|
||||
_leader_flag = request.kv_transfer_params.get("is_request_leader")
|
||||
if _leader_flag is not None:
|
||||
_should_notify = bool(_leader_flag)
|
||||
elif request.kv_transfer_params.get("remote_dp_rank_override"):
|
||||
_should_notify = self._global_dp_rank == remote_dp_rank
|
||||
else:
|
||||
_should_notify = self._is_kv_master
|
||||
if _should_notify:
|
||||
peer_zmq = get_peer_zmq_from_request_id(
|
||||
request.request_id, is_producer=False
|
||||
)
|
||||
remote_host, _, remote_notify_port = parse_moriio_zmq_address(
|
||||
peer_zmq
|
||||
)
|
||||
remote_notify_port = int(remote_notify_port)
|
||||
if peer_zmq is not None:
|
||||
remote_host, _, remote_notify_port = parse_moriio_zmq_address(
|
||||
peer_zmq
|
||||
)
|
||||
else:
|
||||
# Sidecar fallback: use explicit params fields.
|
||||
params = request.kv_transfer_params or {}
|
||||
remote_host = params.get("remote_host") or ""
|
||||
try:
|
||||
remote_notify_port = int(
|
||||
params.get("remote_notify_port") or 0
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
remote_notify_port = 0
|
||||
if not remote_host or not remote_notify_port:
|
||||
raise ValueError(
|
||||
f"request {request.request_id!r}: "
|
||||
f"request_id has no embedded peer "
|
||||
f"zmq_address and kv_transfer_params is "
|
||||
f"missing remote_host / remote_notify_port "
|
||||
f"(got remote_host={remote_host!r}, "
|
||||
f"remote_notify_port={remote_notify_port!r})"
|
||||
)
|
||||
|
||||
# num_external_tokens == 0: nothing to push, so don't tell the
|
||||
# producer to write into these blocks.
|
||||
block_notify_list = (
|
||||
blocks.get_block_ids()[0] if num_external_tokens > 0 else []
|
||||
)
|
||||
|
||||
for tp_index in range(self.tp_size):
|
||||
target_port = remote_notify_port + get_port_offset(
|
||||
remote_dp_rank, tp_index
|
||||
# num_external_tokens == 0: nothing to push, so don't tell
|
||||
# the producer to write into these blocks.
|
||||
block_notify_list = (
|
||||
blocks.get_block_ids()[0] if num_external_tokens > 0 else []
|
||||
)
|
||||
|
||||
self.send_notify_block(
|
||||
req_id=request.request_id,
|
||||
transfer_id=request.kv_transfer_params["transfer_id"],
|
||||
block_notify_list=block_notify_list,
|
||||
host=remote_host,
|
||||
port=target_port,
|
||||
# Wide-EP multi-pod: a pod binds notify sockets only for
|
||||
# its LOCAL ranks, so the port offset must use the per-pod
|
||||
# local rank (% dp_local), not the global rank. Single-pod
|
||||
# is bit-identical (modulus is a no-op).
|
||||
_remote_dp_rank_for_port = fold_local_rank(
|
||||
remote_dp_rank, _dp_local
|
||||
)
|
||||
# The target rank may live on a child pod at a different IP,
|
||||
# so resolve the per-pod host (pod_idx = global // dp_local).
|
||||
# Otherwise a notify for child ranks lands on the master
|
||||
# pod and the request hangs in WAITING_FOR_REMOTE_KVS.
|
||||
_notify_host = remote_host
|
||||
_kvp = request.kv_transfer_params or {}
|
||||
_remote_hosts = _kvp.get("remote_hosts") or []
|
||||
if _dp_local > 0 and _remote_hosts:
|
||||
_pod_idx = pod_index(remote_dp_rank, _dp_local)
|
||||
if 0 <= _pod_idx < len(_remote_hosts):
|
||||
_notify_host = _remote_hosts[_pod_idx]
|
||||
for tp_index in range(self.tp_size):
|
||||
target_port = remote_notify_port + get_port_offset(
|
||||
_remote_dp_rank_for_port, tp_index
|
||||
)
|
||||
|
||||
self.send_notify_block(
|
||||
req_id=request.request_id,
|
||||
transfer_id=request.kv_transfer_params["transfer_id"],
|
||||
block_notify_list=block_notify_list,
|
||||
host=_notify_host,
|
||||
port=target_port,
|
||||
)
|
||||
|
||||
# Only trigger 1 KV transfer per request.
|
||||
|
||||
@@ -636,6 +806,12 @@ class MoRIIOConnectorScheduler:
|
||||
if new_block_ids is not None:
|
||||
block_ids = new_block_ids[0]
|
||||
# TODO : hybrid attn, etc
|
||||
# A non-disagg request (no kv_transfer_params, e.g. smoke
|
||||
# test) is never registered in _reqs_need_pending_save;
|
||||
# indexing it unconditionally would KeyError and crash the
|
||||
# EngineCore. Skip it silently.
|
||||
if req_id not in self._reqs_need_pending_save:
|
||||
continue
|
||||
req, existing_blocks = self._reqs_need_pending_save[req_id]
|
||||
updated_blocks = list(existing_blocks) + (block_ids)
|
||||
self._reqs_need_pending_save[req_id] = (req, updated_blocks)
|
||||
@@ -643,25 +819,30 @@ class MoRIIOConnectorScheduler:
|
||||
len(self._reqs_need_pending_save[req_id][1]) * self.block_size
|
||||
>= req.num_prompt_tokens
|
||||
):
|
||||
# Final chunk: live kv_transfer_params may be cleared,
|
||||
# so prefer the snapshot from update_state_after_alloc.
|
||||
kv_params = self._req_kv_params.pop(
|
||||
req_id, req.kv_transfer_params or {}
|
||||
)
|
||||
meta.add_new_req(
|
||||
request_id=req_id,
|
||||
local_block_ids=self._reqs_need_pending_save[req_id][1],
|
||||
kv_transfer_params=req.kv_transfer_params or {},
|
||||
kv_transfer_params=kv_params,
|
||||
write_mode=True,
|
||||
)
|
||||
del self._reqs_need_pending_save[req_id]
|
||||
|
||||
# Loop through scheduled reqs and convert to ReqMeta.
|
||||
for req_id, (req, block_ids) in self._reqs_need_recv.items():
|
||||
assert req.kv_transfer_params is not None
|
||||
kv_params = self._req_kv_params.get(req_id, req.kv_transfer_params or {})
|
||||
meta.add_new_req(
|
||||
request_id=req_id,
|
||||
local_block_ids=block_ids,
|
||||
kv_transfer_params=req.kv_transfer_params,
|
||||
kv_transfer_params=kv_params,
|
||||
)
|
||||
|
||||
for req_id, (req, block_ids) in self._reqs_need_save.items():
|
||||
assert req.kv_transfer_params is not None
|
||||
kv_params = self._req_kv_params.get(req_id, req.kv_transfer_params or {})
|
||||
if req.num_prompt_tokens > len(block_ids) * self.block_size:
|
||||
# not last chunk prefill
|
||||
self._reqs_need_pending_save[req_id] = (req, block_ids)
|
||||
@@ -669,13 +850,22 @@ class MoRIIOConnectorScheduler:
|
||||
meta.add_new_req(
|
||||
request_id=req_id,
|
||||
local_block_ids=block_ids,
|
||||
kv_transfer_params=req.kv_transfer_params,
|
||||
kv_transfer_params=kv_params,
|
||||
write_mode=True,
|
||||
)
|
||||
# Clear the list once workers start the transfers
|
||||
|
||||
meta.reqs_to_send = self._reqs_need_send
|
||||
|
||||
# Reclaim snapshot cache entries that completed this step. Keep
|
||||
# entries that are still pending (chunked prefill not yet at the
|
||||
# final chunk) — those will be popped above on the final chunk.
|
||||
for req_id in self._reqs_need_recv:
|
||||
self._req_kv_params.pop(req_id, None)
|
||||
for req_id in self._reqs_need_save:
|
||||
if req_id not in self._reqs_need_pending_save:
|
||||
self._req_kv_params.pop(req_id, None)
|
||||
|
||||
self._reqs_need_recv.clear()
|
||||
self._reqs_need_save.clear()
|
||||
self._reqs_need_send = {}
|
||||
@@ -702,14 +892,14 @@ class MoRIIOConnectorScheduler:
|
||||
"""
|
||||
|
||||
request_id = request.request_id
|
||||
params = request.kv_transfer_params
|
||||
# Consumer: can unmap transfer_id<->request_id immediately since done_recving
|
||||
# has fired at this point (i.e. KV has been transferred)
|
||||
# Producer: must keep the mapping until we get notification that blocks can
|
||||
# be freed, which may be several scheduler steps later.
|
||||
if not self.is_producer:
|
||||
self.unmap_request_id(request_id)
|
||||
|
||||
params = request.kv_transfer_params
|
||||
transfer_id = params.get("transfer_id") if params else None
|
||||
self.unmap_request_id(request_id, transfer_id=transfer_id)
|
||||
logger.debug(
|
||||
"MoriioConnector request_finished, request_status=%s, "
|
||||
"kv_transfer_params=%s",
|
||||
@@ -754,11 +944,17 @@ class MoRIIOConnectorScheduler:
|
||||
+ MoRIIOConstants.VLLM_MORI_READ_ABORT_REQUEST_TIMEOUT
|
||||
)
|
||||
self._deferred_send_deadlines[request.request_id] = (
|
||||
time.monotonic() + self._defer_timeout
|
||||
time.monotonic() + self._defer_timeout,
|
||||
params.get("transfer_id") if params else None,
|
||||
)
|
||||
|
||||
# Return KV transfer params forwarded verbatim to the decode instance by
|
||||
# the router.
|
||||
# the router. remote_dp_rank is the rank THIS prefill leg actually ran
|
||||
# on: on the returnable paths (READ / serial WRITE) the router forwards
|
||||
# it to the decode leg, so the decode->prefill notify is routed by pure
|
||||
# propagation of the producer's real rank -- no hashing, no reliance on
|
||||
# the router independently pinning the same rank. remote_dp_rank_override
|
||||
# makes the decode side honor it via the global-rank match gate.
|
||||
return delay_free_blocks, dict(
|
||||
do_remote_prefill=True,
|
||||
do_remote_decode=False,
|
||||
@@ -767,61 +963,96 @@ class MoRIIOConnectorScheduler:
|
||||
remote_host=self.host_ip,
|
||||
remote_handshake_port=self.handshake_port,
|
||||
remote_notify_port=self.side_notify_port,
|
||||
remote_dp_rank=self._global_dp_rank,
|
||||
remote_dp_rank_override=True,
|
||||
remote_dp_size=self.vllm_config.parallel_config.data_parallel_size,
|
||||
remote_dp_size_local=(
|
||||
self.vllm_config.parallel_config.data_parallel_size_local
|
||||
),
|
||||
tp_size=self.vllm_config.parallel_config.tensor_parallel_size,
|
||||
transfer_id=params["transfer_id"],
|
||||
)
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
|
||||
"""Free KV blocks from sends that never received a completion signal.
|
||||
"""Reconcile worker finished_sending ACKs with the producer lifecycle.
|
||||
|
||||
Called every scheduler step. When a send is deferred (request_finished
|
||||
returns True), blocks remain allocated until the worker reports
|
||||
finished_sending. If that notification is lost (e.g. ibv_post_send
|
||||
failure), blocks leak permanently. This method injects timed-out
|
||||
entries into connector_output.finished_sending so the scheduler
|
||||
frees them via the normal path.
|
||||
Called every scheduler step (when there is KV connector output),
|
||||
BEFORE the scheduler's finished_sending free loop. We rewrite
|
||||
``connector_output.finished_sending`` in place so it contains
|
||||
exactly the producer requests that are safe for the scheduler to
|
||||
free this step, letting the scheduler keep the plain upstream
|
||||
``assert req_id in self.requests`` + ``_free_blocks`` path:
|
||||
|
||||
* An ACK whose request is in ``_deferred_send_deadlines``
|
||||
(request_finished ran with delay_free_blocks=True, so the
|
||||
scheduler is holding its blocks) is surfaced -> freed now.
|
||||
* An ACK that arrives BEFORE its request finished is parked in
|
||||
``_pending_sent_acks`` and released on a later step once the
|
||||
request enters ``_deferred_send_deadlines``.
|
||||
* A deferred send whose ACK never arrives is reaped after
|
||||
``_defer_timeout`` and surfaced, so leaked blocks are freed.
|
||||
* A parked ACK that never matches a deferral before its own
|
||||
deadline is a stale duplicate (e.g. a real ACK landing after the
|
||||
send was already reaped) and is dropped.
|
||||
|
||||
Consumers never populate finished_sending (they report
|
||||
finished_recving), and they unmap in request_finished, so this is a
|
||||
no-op for them.
|
||||
"""
|
||||
# Producer: unmap transfer_id<->request_id for sends that are now (async)
|
||||
# reported as completed. This unmapping have to be deferred until now
|
||||
# so get_finished can use it in any scheduler step.
|
||||
# Consumer: unmapping already done in request_finished
|
||||
if self.is_producer and connector_output.finished_sending:
|
||||
for req_id in connector_output.finished_sending:
|
||||
self.unmap_request_id(req_id)
|
||||
|
||||
if not self._deferred_send_deadlines:
|
||||
if not self.is_producer:
|
||||
return
|
||||
|
||||
# Remove entries the worker already reported as finished_sending, these will be
|
||||
# freed anyways.
|
||||
for req_id in connector_output.finished_sending or ():
|
||||
self._deferred_send_deadlines.pop(req_id, None)
|
||||
|
||||
incoming = set(connector_output.finished_sending or ())
|
||||
now = time.monotonic()
|
||||
expired_reqs = [
|
||||
|
||||
# Surface ACKs whose request is already finished (blocks held for
|
||||
# delayed free); park the rest until their request finishes.
|
||||
safe: set[ReqId] = set()
|
||||
for req_id in incoming:
|
||||
if req_id in self._deferred_send_deadlines:
|
||||
safe.add(req_id)
|
||||
else:
|
||||
self._pending_sent_acks.setdefault(req_id, now + self._defer_timeout)
|
||||
|
||||
# Release previously parked ACKs whose request has since finished.
|
||||
for req_id in self._pending_sent_acks:
|
||||
if req_id in self._deferred_send_deadlines:
|
||||
safe.add(req_id)
|
||||
|
||||
# Reap deferred sends whose ACK never arrived (avoid leaking blocks).
|
||||
expired = [
|
||||
req_id
|
||||
for req_id, deadline in self._deferred_send_deadlines.items()
|
||||
for req_id, (deadline, _) in self._deferred_send_deadlines.items()
|
||||
if now >= deadline
|
||||
]
|
||||
if not expired_reqs:
|
||||
return
|
||||
if expired:
|
||||
safe.update(expired)
|
||||
logger.warning(
|
||||
"Reaped %d deferred sends with no finished_sending "
|
||||
"notification after %.0fs. This indicates lost async KV "
|
||||
"completion notifications from the KV connector.",
|
||||
len(expired),
|
||||
self._defer_timeout,
|
||||
)
|
||||
|
||||
if connector_output.finished_sending is None:
|
||||
connector_output.finished_sending = set()
|
||||
# Register the expired requests as finished so the scheduler frees their blocks.
|
||||
for req_id in expired_reqs:
|
||||
connector_output.finished_sending.add(req_id)
|
||||
del self._deferred_send_deadlines[req_id]
|
||||
if self.is_producer:
|
||||
self.unmap_request_id(req_id)
|
||||
logger.warning(
|
||||
"Reaped %d deferred sends with no finished_sending notification "
|
||||
"after %.0fs. This indicates lost async KV completion "
|
||||
"notifications from the KV connector.",
|
||||
len(expired_reqs),
|
||||
self._defer_timeout,
|
||||
)
|
||||
# Finalize the requests we are surfacing: drop their deferral/park
|
||||
# bookkeeping and unmap their transfer ids.
|
||||
for req_id in safe:
|
||||
deferred_info = self._deferred_send_deadlines.pop(req_id, None)
|
||||
transfer_id = deferred_info[1] if deferred_info else None
|
||||
self._pending_sent_acks.pop(req_id, None)
|
||||
self.unmap_request_id(req_id, transfer_id=transfer_id)
|
||||
|
||||
# Drop stale parked ACKs that never matched a deferral in time.
|
||||
stale = [
|
||||
req_id
|
||||
for req_id, deadline in self._pending_sent_acks.items()
|
||||
if now >= deadline
|
||||
]
|
||||
for req_id in stale:
|
||||
self._pending_sent_acks.pop(req_id, None)
|
||||
|
||||
connector_output.finished_sending = safe or None
|
||||
|
||||
|
||||
class MoRIIOConnectorWorker:
|
||||
@@ -998,6 +1229,10 @@ class MoRIIOConnectorWorker:
|
||||
self._recving_transfers: defaultdict[ReqId, list] = defaultdict(list)
|
||||
# Values are (remote_host, remote_notify_port, transfer_id).
|
||||
self._recving_transfers_callback_addr: dict[ReqId, tuple[str, str, str]] = {}
|
||||
# Monotonic-clock start times for each in-flight recv transfer.
|
||||
# Used by _pop_done_transfers to abort transfers whose RDMA
|
||||
# completion is lost, instead of hanging forever.
|
||||
self._recving_transfers_start: dict[str, float] = {}
|
||||
|
||||
# Track the expiration time of requests that are waiting to be sent.
|
||||
self._reqs_to_send: dict[ReqId, float] = {}
|
||||
@@ -1368,6 +1603,11 @@ class MoRIIOConnectorWorker:
|
||||
port = int(meta.remote_handshake_port)
|
||||
tp_size = int(meta.tp_size)
|
||||
remote_dp_size = int(meta.remote_dp_size)
|
||||
# Wide-EP multi-pod: remote DP ranks span pods at different IPs
|
||||
# (ranks per pod = dp_local), so resolve the host per cur_dp_rank
|
||||
# below instead of using a single host for all ranks.
|
||||
pod_hosts = list(meta.multi_pod_hosts) if meta.multi_pod_hosts else [host]
|
||||
remote_dp_size_local = int(meta.remote_dp_size_local) or remote_dp_size
|
||||
|
||||
def request_ready(_f: Future[Any], entry=(req_id, meta)):
|
||||
logger.info("MoRIIO handshake done for request %s", req_id)
|
||||
@@ -1381,8 +1621,21 @@ class MoRIIOConnectorWorker:
|
||||
|
||||
for cur_dp_rank in range(remote_dp_size):
|
||||
dp_engine_id = self.get_engine_name_with_dp(remote_engine_id, cur_dp_rank)
|
||||
_pod_idx = pod_index(cur_dp_rank, remote_dp_size_local)
|
||||
if _pod_idx >= len(pod_hosts):
|
||||
_pod_idx = 0
|
||||
_per_rank_host = pod_hosts[_pod_idx]
|
||||
# The handshake port offset must use the per-pod local rank, since
|
||||
# each pod binds sockets only for its local ranks; dp_engine_id
|
||||
# keeps the global rank for uniqueness. Single-pod is bit-identical.
|
||||
_per_rank_local_dp = fold_local_rank(cur_dp_rank, remote_dp_size_local)
|
||||
future = self._handshake_initiation_executor.submit(
|
||||
self._moriio_handshake, host, port, tp_size, dp_engine_id, cur_dp_rank
|
||||
self._moriio_handshake,
|
||||
_per_rank_host,
|
||||
port,
|
||||
tp_size,
|
||||
dp_engine_id,
|
||||
_per_rank_local_dp,
|
||||
)
|
||||
fut_list.append(future)
|
||||
|
||||
@@ -1463,7 +1716,19 @@ class MoRIIOConnectorWorker:
|
||||
block_shape = first_kv_cache.shape[-block_rank:]
|
||||
self.num_blocks = first_geometry.num_blocks
|
||||
self.slot_size_bytes = first_geometry.slot_size_bytes
|
||||
assert first_geometry.block_size == self.block_size
|
||||
if first_geometry.block_size != self.block_size:
|
||||
# DeepSeek-V3 / MLA backends (e.g. FlashMLA) override the
|
||||
# configured block_size at attention-layer creation time, so
|
||||
# the KV cache tensor is laid out with a different (usually
|
||||
# larger) block_size than vllm_config.cache_config.block_size.
|
||||
# Trust the actual tensor shape and reconcile.
|
||||
logger.info(
|
||||
"KV cache block_size=%d differs from config block_size=%d; "
|
||||
"using actual tensor shape (attention backend override).",
|
||||
first_geometry.block_size,
|
||||
self.block_size,
|
||||
)
|
||||
self.block_size = first_geometry.block_size
|
||||
# TODO(tms): self.block_len needs to be per-layer for sliding window,
|
||||
# hybrid attn, etc
|
||||
# block size in bytes
|
||||
@@ -1634,6 +1899,7 @@ class MoRIIOConnectorWorker:
|
||||
|
||||
def _pop_done_transfers(self) -> set[str]:
|
||||
done_req_ids: set[str] = set()
|
||||
_xfer_timeout = int(os.environ.get("VLLM_MORIIO_TRANSFER_TIMEOUT_S", "120"))
|
||||
with self.moriio_wrapper.lock:
|
||||
to_remove = []
|
||||
for req_id, status_list in self._recving_transfers.items():
|
||||
@@ -1675,9 +1941,25 @@ class MoRIIOConnectorWorker:
|
||||
to_remove.append(req_id)
|
||||
# Do NOT add to done_req_ids: decode KV cache is incomplete.
|
||||
# The request will expire via the normal request timeout.
|
||||
elif req_id in self._recving_transfers_start:
|
||||
# Abort still-in-flight transfers that exceed the
|
||||
# configured deadline. Otherwise a lost RDMA
|
||||
# completion would leave the decode worker hung
|
||||
# indefinitely on this request.
|
||||
_age = time.monotonic() - self._recving_transfers_start[req_id]
|
||||
if _age > _xfer_timeout:
|
||||
logger.error(
|
||||
"RDMA read TIMED OUT for req %s after %.1fs "
|
||||
"(VLLM_MORIIO_TRANSFER_TIMEOUT_S=%d)",
|
||||
req_id,
|
||||
_age,
|
||||
_xfer_timeout,
|
||||
)
|
||||
to_remove.append(req_id)
|
||||
for req_id in to_remove:
|
||||
del self._recving_transfers[req_id]
|
||||
del self._recving_transfers_callback_addr[req_id]
|
||||
self._recving_transfers_start.pop(req_id, None)
|
||||
|
||||
return done_req_ids
|
||||
|
||||
@@ -2043,6 +2325,18 @@ class MoRIIOConnectorWorker:
|
||||
)
|
||||
|
||||
def _write_blocks_for_req(self, req_id: ReqId, meta: ReqMeta, layer_name, kv_layer):
|
||||
# Stash multi_pod_hosts + local DP size on the worker so
|
||||
# MoRIIOEngine._finalize_if_complete (which sees only the WriteTask,
|
||||
# not ReqMeta) can pick the per-rank pod IP for the completion notify.
|
||||
# Last-writer-wins is safe: all requests share the same topology.
|
||||
if meta.multi_pod_hosts:
|
||||
self.multi_pod_hosts = list(meta.multi_pod_hosts)
|
||||
else:
|
||||
self.multi_pod_hosts = [meta.remote_host]
|
||||
if meta.remote_dp_size_local:
|
||||
self.remote_dp_size_local = int(meta.remote_dp_size_local)
|
||||
else:
|
||||
self.remote_dp_size_local = int(meta.remote_dp_size)
|
||||
self.schedule_write_blocks(
|
||||
request_id=req_id,
|
||||
transfer_id=meta.transfer_id,
|
||||
@@ -2271,6 +2565,7 @@ class MoRIIOConnectorWorker:
|
||||
_backoff = min(_backoff * 2, 0.05)
|
||||
with self.moriio_wrapper.lock:
|
||||
self._recving_transfers[request_id].append(transfer_status)
|
||||
self._recving_transfers_start.setdefault(request_id, time.monotonic())
|
||||
self._recving_transfers_callback_addr[request_id] = (
|
||||
remote_host,
|
||||
str(
|
||||
|
||||
@@ -311,7 +311,17 @@ class MoRIIOWriter:
|
||||
with self._write_state_lock:
|
||||
request_info.completion_request_id = task.request_id
|
||||
request_info.completion_remote_notify_port = task.remote_notify_port
|
||||
request_info.completion_remote_ip = task.remote_ip
|
||||
# Wide-EP multi-pod: task.remote_ip addresses only the first pod,
|
||||
# so resolve the per-rank host from multi_pod_hosts (falls back to
|
||||
# task.remote_ip for single-pod or on any indexing miss).
|
||||
_remote_ip = task.remote_ip
|
||||
_hosts = list(getattr(self.worker, "multi_pod_hosts", []) or [])
|
||||
_dp_local = int(getattr(self.worker, "remote_dp_size_local", 0) or 0)
|
||||
if _hosts and _dp_local > 0:
|
||||
_pod_idx = int(request_info.decode_dp_rank) // _dp_local
|
||||
if 0 <= _pod_idx < len(_hosts):
|
||||
_remote_ip = _hosts[_pod_idx]
|
||||
request_info.completion_remote_ip = _remote_ip
|
||||
if task.transfer_id in self._sealed_writes:
|
||||
request_info.writes_expected = self._sealed_writes[task.transfer_id]
|
||||
|
||||
@@ -454,8 +464,15 @@ class MoRIIOWriter:
|
||||
# Wait for this request's transfers to complete.
|
||||
self.worker.moriio_wrapper.waiting_for_transfer_complete(transfer_statuses)
|
||||
|
||||
# The notify port offset must use the per-pod local rank
|
||||
# (% dp_local), since each pod binds notify sockets only for its local
|
||||
# ranks. Single-pod is bit-identical (modulus is a no-op).
|
||||
_dp_local = int(getattr(self.worker, "remote_dp_size_local", 0) or 0)
|
||||
_decode_dp_rank_for_port = int(request_info.decode_dp_rank)
|
||||
if _dp_local > 0:
|
||||
_decode_dp_rank_for_port = _decode_dp_rank_for_port % _dp_local
|
||||
remote_port = remote_notify_port + get_port_offset(
|
||||
request_info.decode_dp_rank, self.worker.tp_rank
|
||||
_decode_dp_rank_for_port, self.worker.tp_rank
|
||||
)
|
||||
# Consider using RDMA immediate data in decode side
|
||||
# to eliminate the need for this notification.
|
||||
|
||||
Reference in New Issue
Block a user