[Bugfix][KV-transfer] MoRIIO: per-layer READ-completion barrier in wait_for_layer_load (#48534)

Signed-off-by: Edwin Lim <edwin.lim@mangoboost.io>
Signed-off-by: Redwan Khan <redwan.khan@mangoboost.io>
Signed-off-by: harishk-mangoboost <harish.kambhampaty@mangoboost.io>
Signed-off-by: limeward <32970461+edwinlim0919@users.noreply.github.com>
Signed-off-by: QinPR <1905873179@qq.com>
Co-authored-by: Jaeyoun Kim <jaeyoun.kim@mangoboost.io>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Redwan Khan <redwan.khan@mangoboost.io>
Co-authored-by: harishk-mangoboost <harish.kambhampaty@mangoboost.io>
Co-authored-by: rkhan055 <redwankhan055@gmail.com>
Co-authored-by: Peiran Qin <66068739+QinPR@users.noreply.github.com>
Co-authored-by: QinPR <1905873179@qq.com>
This commit is contained in:
limeward
2026-08-07 14:23:33 +00:00
committed by GitHub
co-authored by Jaeyoun Kim Claude Opus 4.8 Redwan Khan harishk-mangoboost rkhan055 Peiran Qin QinPR
parent d4ecb75ba2
commit 47228db84c
2 changed files with 86 additions and 13 deletions
@@ -10,6 +10,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import (
MoRIIOTransferAck, MoRIIOTransferAck,
) )
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import ( from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import (
MoRIIOConnector,
MoRIIOConnectorWorker, MoRIIOConnectorWorker,
get_moriio_expected_ack_count, get_moriio_expected_ack_count,
get_moriio_remote_tp_rank, get_moriio_remote_tp_rank,
@@ -292,7 +293,7 @@ def test_read_completion_sends_structured_release_with_consumer_tp_size():
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker) worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
worker.world_size = 8 worker.world_size = 8
worker.moriio_wrapper = FakeWrapper() worker.moriio_wrapper = FakeWrapper()
worker._recving_transfers = {"req": [DoneStatus()]} worker._recving_transfers = {"req": {"layer0": DoneStatus()}}
worker._recving_transfers_callback_addr = { worker._recving_transfers_callback_addr = {
"req": ("127.0.0.1", "7000", "tx-release") "req": ("127.0.0.1", "7000", "tx-release")
} }
@@ -311,3 +312,15 @@ def test_read_completion_sends_structured_release_with_consumer_tp_size():
] ]
assert worker._recving_transfers == {} assert worker._recving_transfers == {}
assert worker._recving_transfers_callback_addr == {} assert worker._recving_transfers_callback_addr == {}
def test_requested_cudagraph_mode_is_never_overridden():
# The configured cudagraph mode is always honored: the barrier fires when
# the operator sets cudagraph_mode=PIECEWISE, and READ mode with full
# graphs only warns instead of silently forcing PIECEWISE.
assert (
MoRIIOConnector.requires_piecewise_for_cudagraph({"read_mode": True}) is False
)
assert (
MoRIIOConnector.requires_piecewise_for_cudagraph({"read_mode": False}) is False
)
@@ -17,7 +17,7 @@ import numpy as np
import torch import torch
import zmq import zmq
from vllm.config import VllmConfig from vllm.config import CUDAGraphMode, VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.base import ( from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1, KVConnectorBase_V1,
KVConnectorMetadata, KVConnectorMetadata,
@@ -65,7 +65,7 @@ from vllm.distributed.parallel_state import (
get_tp_group, get_tp_group,
get_world_group, get_world_group,
) )
from vllm.forward_context import ForwardContext from vllm.forward_context import ForwardContext, get_forward_context
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.utils.network_utils import ( from vllm.utils.network_utils import (
make_zmq_path, make_zmq_path,
@@ -210,6 +210,19 @@ class MoRIIOConnector(KVConnectorBase_V1):
+ str(self.kv_transfer_config.kv_connector_extra_config["handshake_port"]) + str(self.kv_transfer_config.kv_connector_extra_config["handshake_port"])
) )
self.mode = get_moriio_mode(self.kv_transfer_config) self.mode = get_moriio_mode(self.kv_transfer_config)
if (
self.mode == MoRIIOMode.READ
and self.kv_transfer_config.is_kv_consumer
and vllm_config.compilation_config.cudagraph_mode.has_full_cudagraphs()
):
# warn only; kv-read barrier requires PIECEWISE cudagraph mode
logger.warning_once(
"MoRIIO READ mode is running with %s CUDA graphs: per-layer "
"KV-read barrier can't fire inside full graph; accuracy may "
"degrade at high concurrency. Set cudagraph_mode=PIECEWISE "
"in --compilation-config.",
vllm_config.compilation_config.cudagraph_mode.name,
)
if role == KVConnectorRole.SCHEDULER: if role == KVConnectorRole.SCHEDULER:
self.connector_scheduler: MoRIIOConnectorScheduler | None = ( self.connector_scheduler: MoRIIOConnectorScheduler | None = (
MoRIIOConnectorScheduler(vllm_config, self.engine_id) MoRIIOConnectorScheduler(vllm_config, self.engine_id)
@@ -299,7 +312,8 @@ class MoRIIOConnector(KVConnectorBase_V1):
self.connector_worker.start_load_kv(self._connector_metadata) self.connector_worker.start_load_kv(self._connector_metadata)
def wait_for_layer_load(self, layer_name: str) -> None: def wait_for_layer_load(self, layer_name: str) -> None:
pass assert self.connector_worker is not None
self.connector_worker.wait_for_layer_load(layer_name)
def save_kv_layer( def save_kv_layer(
self, self,
@@ -1225,8 +1239,8 @@ class MoRIIOConnectorWorker:
# Map of engine_id -> num_blocks. All ranks in the same deployment will # Map of engine_id -> num_blocks. All ranks in the same deployment will
# have the same number of blocks. # have the same number of blocks.
self.dst_num_blocks: dict[EngineId, int] = {} self.dst_num_blocks: dict[EngineId, int] = {}
# In progress transfers. # In-progress READ transfers: req_id -> {layer_name: status}.
self._recving_transfers: defaultdict[ReqId, list] = defaultdict(list) self._recving_transfers: defaultdict[ReqId, dict] = defaultdict(dict)
# Values are (remote_host, remote_notify_port, transfer_id). # Values are (remote_host, remote_notify_port, transfer_id).
self._recving_transfers_callback_addr: dict[ReqId, tuple[str, str, str]] = {} self._recving_transfers_callback_addr: dict[ReqId, tuple[str, str, str]] = {}
# Monotonic-clock start times for each in-flight recv transfer. # Monotonic-clock start times for each in-flight recv transfer.
@@ -1897,14 +1911,60 @@ class MoRIIOConnectorWorker:
return done_sending, done_recving return done_sending, done_recving
def wait_for_layer_load(self, layer_name: str) -> None:
"""Block until all in-flight READs of this layer have landed.
A host-side blocking wait must not run during full-graph capture.
"""
if self.is_producer or self.mode != MoRIIOMode.READ:
return
if get_forward_context().cudagraph_runtime_mode == CUDAGraphMode.FULL:
return
deadline = time.monotonic() + self.moriio_config.transfer_timeout
while True:
with self.moriio_wrapper.lock:
pending = [
status_by_layer[layer_name]
for status_by_layer in self._recving_transfers.values()
if layer_name in status_by_layer
]
if not pending:
return
still_running = False
for status in pending:
# A failed read is dropped in _pop_done_transfers.
if status.Succeeded() or status.Failed():
continue
still_running = True
if not still_running:
return
if time.monotonic() > deadline:
logger.warning(
"MoRIIO READ barrier timed out for layer %s; proceeding "
"(request dropped via get_finished).",
layer_name,
)
return
time.sleep(0.001)
def _pop_done_transfers(self) -> set[str]: def _pop_done_transfers(self) -> set[str]:
done_req_ids: set[str] = set() done_req_ids: set[str] = set()
_xfer_timeout = int(os.environ.get("VLLM_MORIIO_TRANSFER_TIMEOUT_S", "120")) _xfer_timeout = int(os.environ.get("VLLM_MORIIO_TRANSFER_TIMEOUT_S", "120"))
with self.moriio_wrapper.lock: with self.moriio_wrapper.lock:
to_remove = [] to_remove = []
for req_id, status_list in self._recving_transfers.items(): for req_id, status_by_layer in self._recving_transfers.items():
last = status_list[-1] statuses = list(status_by_layer.values())
if last.Succeeded(): failed_status = next(
(status for status in statuses if status.Failed()), None
)
if statuses and all(status.Succeeded() for status in statuses):
host, port, xfer_id = self._recving_transfers_callback_addr[req_id] host, port, xfer_id = self._recving_transfers_callback_addr[req_id]
done_req_ids.add(xfer_id) done_req_ids.add(xfer_id)
self.moriio_wrapper.send_notify( self.moriio_wrapper.send_notify(
@@ -1915,14 +1975,14 @@ class MoRIIOConnectorWorker:
message_fields={"consumer_tp_size": self.world_size}, message_fields={"consumer_tp_size": self.world_size},
) )
to_remove.append(req_id) to_remove.append(req_id)
elif last.Failed(): elif failed_status is not None:
logger.error( logger.error(
"RDMA transfer failed for request %s: %s (code=%s). " "RDMA transfer failed for request %s: %s (code=%s). "
"Notifying prefill to free blocks; request will be " "Notifying prefill to free blocks; request will be "
"aborted by timeout.", "aborted by timeout.",
req_id, req_id,
last.Message(), failed_status.Message(),
last.Code(), failed_status.Code(),
) )
host, port, xfer_id = self._recving_transfers_callback_addr[req_id] host, port, xfer_id = self._recving_transfers_callback_addr[req_id]
try: try:
@@ -2564,7 +2624,7 @@ class MoRIIOConnectorWorker:
time.sleep(_backoff) time.sleep(_backoff)
_backoff = min(_backoff * 2, 0.05) _backoff = min(_backoff * 2, 0.05)
with self.moriio_wrapper.lock: with self.moriio_wrapper.lock:
self._recving_transfers[request_id].append(transfer_status) self._recving_transfers[request_id][layer_name] = transfer_status
self._recving_transfers_start.setdefault(request_id, time.monotonic()) self._recving_transfers_start.setdefault(request_id, time.monotonic())
self._recving_transfers_callback_addr[request_id] = ( self._recving_transfers_callback_addr[request_id] = (
remote_host, remote_host,