[Bugfix][KV-transfer] MoRIIO: per-layer READ-completion barrier in wait_for_layer_load (#48534)
Signed-off-by: Edwin Lim <edwin.lim@mangoboost.io> Signed-off-by: Redwan Khan <redwan.khan@mangoboost.io> Signed-off-by: harishk-mangoboost <harish.kambhampaty@mangoboost.io> Signed-off-by: limeward <32970461+edwinlim0919@users.noreply.github.com> Signed-off-by: QinPR <1905873179@qq.com> Co-authored-by: Jaeyoun Kim <jaeyoun.kim@mangoboost.io> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Redwan Khan <redwan.khan@mangoboost.io> Co-authored-by: harishk-mangoboost <harish.kambhampaty@mangoboost.io> Co-authored-by: rkhan055 <redwankhan055@gmail.com> Co-authored-by: Peiran Qin <66068739+QinPR@users.noreply.github.com> Co-authored-by: QinPR <1905873179@qq.com>
This commit is contained in:
co-authored by
Jaeyoun Kim
Claude Opus 4.8
Redwan Khan
harishk-mangoboost
rkhan055
Peiran Qin
QinPR
parent
d4ecb75ba2
commit
47228db84c
@@ -17,7 +17,7 @@ import numpy as np
|
||||
import torch
|
||||
import zmq
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config import CUDAGraphMode, VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorBase_V1,
|
||||
KVConnectorMetadata,
|
||||
@@ -65,7 +65,7 @@ from vllm.distributed.parallel_state import (
|
||||
get_tp_group,
|
||||
get_world_group,
|
||||
)
|
||||
from vllm.forward_context import ForwardContext
|
||||
from vllm.forward_context import ForwardContext, get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.network_utils import (
|
||||
make_zmq_path,
|
||||
@@ -210,6 +210,19 @@ class MoRIIOConnector(KVConnectorBase_V1):
|
||||
+ str(self.kv_transfer_config.kv_connector_extra_config["handshake_port"])
|
||||
)
|
||||
self.mode = get_moriio_mode(self.kv_transfer_config)
|
||||
if (
|
||||
self.mode == MoRIIOMode.READ
|
||||
and self.kv_transfer_config.is_kv_consumer
|
||||
and vllm_config.compilation_config.cudagraph_mode.has_full_cudagraphs()
|
||||
):
|
||||
# warn only; kv-read barrier requires PIECEWISE cudagraph mode
|
||||
logger.warning_once(
|
||||
"MoRIIO READ mode is running with %s CUDA graphs: per-layer "
|
||||
"KV-read barrier can't fire inside full graph; accuracy may "
|
||||
"degrade at high concurrency. Set cudagraph_mode=PIECEWISE "
|
||||
"in --compilation-config.",
|
||||
vllm_config.compilation_config.cudagraph_mode.name,
|
||||
)
|
||||
if role == KVConnectorRole.SCHEDULER:
|
||||
self.connector_scheduler: MoRIIOConnectorScheduler | None = (
|
||||
MoRIIOConnectorScheduler(vllm_config, self.engine_id)
|
||||
@@ -299,7 +312,8 @@ class MoRIIOConnector(KVConnectorBase_V1):
|
||||
self.connector_worker.start_load_kv(self._connector_metadata)
|
||||
|
||||
def wait_for_layer_load(self, layer_name: str) -> None:
|
||||
pass
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.wait_for_layer_load(layer_name)
|
||||
|
||||
def save_kv_layer(
|
||||
self,
|
||||
@@ -1225,8 +1239,8 @@ class MoRIIOConnectorWorker:
|
||||
# Map of engine_id -> num_blocks. All ranks in the same deployment will
|
||||
# have the same number of blocks.
|
||||
self.dst_num_blocks: dict[EngineId, int] = {}
|
||||
# In progress transfers.
|
||||
self._recving_transfers: defaultdict[ReqId, list] = defaultdict(list)
|
||||
# In-progress READ transfers: req_id -> {layer_name: status}.
|
||||
self._recving_transfers: defaultdict[ReqId, dict] = defaultdict(dict)
|
||||
# Values are (remote_host, remote_notify_port, transfer_id).
|
||||
self._recving_transfers_callback_addr: dict[ReqId, tuple[str, str, str]] = {}
|
||||
# Monotonic-clock start times for each in-flight recv transfer.
|
||||
@@ -1897,14 +1911,60 @@ class MoRIIOConnectorWorker:
|
||||
|
||||
return done_sending, done_recving
|
||||
|
||||
def wait_for_layer_load(self, layer_name: str) -> None:
|
||||
"""Block until all in-flight READs of this layer have landed.
|
||||
|
||||
A host-side blocking wait must not run during full-graph capture.
|
||||
"""
|
||||
if self.is_producer or self.mode != MoRIIOMode.READ:
|
||||
return
|
||||
|
||||
if get_forward_context().cudagraph_runtime_mode == CUDAGraphMode.FULL:
|
||||
return
|
||||
|
||||
deadline = time.monotonic() + self.moriio_config.transfer_timeout
|
||||
while True:
|
||||
with self.moriio_wrapper.lock:
|
||||
pending = [
|
||||
status_by_layer[layer_name]
|
||||
for status_by_layer in self._recving_transfers.values()
|
||||
if layer_name in status_by_layer
|
||||
]
|
||||
|
||||
if not pending:
|
||||
return
|
||||
|
||||
still_running = False
|
||||
for status in pending:
|
||||
# A failed read is dropped in _pop_done_transfers.
|
||||
if status.Succeeded() or status.Failed():
|
||||
continue
|
||||
still_running = True
|
||||
|
||||
if not still_running:
|
||||
return
|
||||
|
||||
if time.monotonic() > deadline:
|
||||
logger.warning(
|
||||
"MoRIIO READ barrier timed out for layer %s; proceeding "
|
||||
"(request dropped via get_finished).",
|
||||
layer_name,
|
||||
)
|
||||
return
|
||||
|
||||
time.sleep(0.001)
|
||||
|
||||
def _pop_done_transfers(self) -> set[str]:
|
||||
done_req_ids: set[str] = set()
|
||||
_xfer_timeout = int(os.environ.get("VLLM_MORIIO_TRANSFER_TIMEOUT_S", "120"))
|
||||
with self.moriio_wrapper.lock:
|
||||
to_remove = []
|
||||
for req_id, status_list in self._recving_transfers.items():
|
||||
last = status_list[-1]
|
||||
if last.Succeeded():
|
||||
for req_id, status_by_layer in self._recving_transfers.items():
|
||||
statuses = list(status_by_layer.values())
|
||||
failed_status = next(
|
||||
(status for status in statuses if status.Failed()), None
|
||||
)
|
||||
if statuses and all(status.Succeeded() for status in statuses):
|
||||
host, port, xfer_id = self._recving_transfers_callback_addr[req_id]
|
||||
done_req_ids.add(xfer_id)
|
||||
self.moriio_wrapper.send_notify(
|
||||
@@ -1915,14 +1975,14 @@ class MoRIIOConnectorWorker:
|
||||
message_fields={"consumer_tp_size": self.world_size},
|
||||
)
|
||||
to_remove.append(req_id)
|
||||
elif last.Failed():
|
||||
elif failed_status is not None:
|
||||
logger.error(
|
||||
"RDMA transfer failed for request %s: %s (code=%s). "
|
||||
"Notifying prefill to free blocks; request will be "
|
||||
"aborted by timeout.",
|
||||
req_id,
|
||||
last.Message(),
|
||||
last.Code(),
|
||||
failed_status.Message(),
|
||||
failed_status.Code(),
|
||||
)
|
||||
host, port, xfer_id = self._recving_transfers_callback_addr[req_id]
|
||||
try:
|
||||
@@ -2564,7 +2624,7 @@ class MoRIIOConnectorWorker:
|
||||
time.sleep(_backoff)
|
||||
_backoff = min(_backoff * 2, 0.05)
|
||||
with self.moriio_wrapper.lock:
|
||||
self._recving_transfers[request_id].append(transfer_status)
|
||||
self._recving_transfers[request_id][layer_name] = transfer_status
|
||||
self._recving_transfers_start.setdefault(request_id, time.monotonic())
|
||||
self._recving_transfers_callback_addr[request_id] = (
|
||||
remote_host,
|
||||
|
||||
Reference in New Issue
Block a user