[Bugfix][KV-transfer] MoRIIO: per-layer READ-completion barrier in wait_for_layer_load (#48534)
Signed-off-by: Edwin Lim <edwin.lim@mangoboost.io> Signed-off-by: Redwan Khan <redwan.khan@mangoboost.io> Signed-off-by: harishk-mangoboost <harish.kambhampaty@mangoboost.io> Signed-off-by: limeward <32970461+edwinlim0919@users.noreply.github.com> Signed-off-by: QinPR <1905873179@qq.com> Co-authored-by: Jaeyoun Kim <jaeyoun.kim@mangoboost.io> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Redwan Khan <redwan.khan@mangoboost.io> Co-authored-by: harishk-mangoboost <harish.kambhampaty@mangoboost.io> Co-authored-by: rkhan055 <redwankhan055@gmail.com> Co-authored-by: Peiran Qin <66068739+QinPR@users.noreply.github.com> Co-authored-by: QinPR <1905873179@qq.com>
This commit is contained in:
co-authored by
Jaeyoun Kim
Claude Opus 4.8
Redwan Khan
harishk-mangoboost
rkhan055
Peiran Qin
QinPR
parent
d4ecb75ba2
commit
47228db84c
@@ -10,6 +10,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import (
|
|||||||
MoRIIOTransferAck,
|
MoRIIOTransferAck,
|
||||||
)
|
)
|
||||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import (
|
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import (
|
||||||
|
MoRIIOConnector,
|
||||||
MoRIIOConnectorWorker,
|
MoRIIOConnectorWorker,
|
||||||
get_moriio_expected_ack_count,
|
get_moriio_expected_ack_count,
|
||||||
get_moriio_remote_tp_rank,
|
get_moriio_remote_tp_rank,
|
||||||
@@ -292,7 +293,7 @@ def test_read_completion_sends_structured_release_with_consumer_tp_size():
|
|||||||
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
|
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
|
||||||
worker.world_size = 8
|
worker.world_size = 8
|
||||||
worker.moriio_wrapper = FakeWrapper()
|
worker.moriio_wrapper = FakeWrapper()
|
||||||
worker._recving_transfers = {"req": [DoneStatus()]}
|
worker._recving_transfers = {"req": {"layer0": DoneStatus()}}
|
||||||
worker._recving_transfers_callback_addr = {
|
worker._recving_transfers_callback_addr = {
|
||||||
"req": ("127.0.0.1", "7000", "tx-release")
|
"req": ("127.0.0.1", "7000", "tx-release")
|
||||||
}
|
}
|
||||||
@@ -311,3 +312,15 @@ def test_read_completion_sends_structured_release_with_consumer_tp_size():
|
|||||||
]
|
]
|
||||||
assert worker._recving_transfers == {}
|
assert worker._recving_transfers == {}
|
||||||
assert worker._recving_transfers_callback_addr == {}
|
assert worker._recving_transfers_callback_addr == {}
|
||||||
|
|
||||||
|
|
||||||
|
def test_requested_cudagraph_mode_is_never_overridden():
|
||||||
|
# The configured cudagraph mode is always honored: the barrier fires when
|
||||||
|
# the operator sets cudagraph_mode=PIECEWISE, and READ mode with full
|
||||||
|
# graphs only warns instead of silently forcing PIECEWISE.
|
||||||
|
assert (
|
||||||
|
MoRIIOConnector.requires_piecewise_for_cudagraph({"read_mode": True}) is False
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
MoRIIOConnector.requires_piecewise_for_cudagraph({"read_mode": False}) is False
|
||||||
|
)
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ import numpy as np
|
|||||||
import torch
|
import torch
|
||||||
import zmq
|
import zmq
|
||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import CUDAGraphMode, VllmConfig
|
||||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||||
KVConnectorBase_V1,
|
KVConnectorBase_V1,
|
||||||
KVConnectorMetadata,
|
KVConnectorMetadata,
|
||||||
@@ -65,7 +65,7 @@ from vllm.distributed.parallel_state import (
|
|||||||
get_tp_group,
|
get_tp_group,
|
||||||
get_world_group,
|
get_world_group,
|
||||||
)
|
)
|
||||||
from vllm.forward_context import ForwardContext
|
from vllm.forward_context import ForwardContext, get_forward_context
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.utils.network_utils import (
|
from vllm.utils.network_utils import (
|
||||||
make_zmq_path,
|
make_zmq_path,
|
||||||
@@ -210,6 +210,19 @@ class MoRIIOConnector(KVConnectorBase_V1):
|
|||||||
+ str(self.kv_transfer_config.kv_connector_extra_config["handshake_port"])
|
+ str(self.kv_transfer_config.kv_connector_extra_config["handshake_port"])
|
||||||
)
|
)
|
||||||
self.mode = get_moriio_mode(self.kv_transfer_config)
|
self.mode = get_moriio_mode(self.kv_transfer_config)
|
||||||
|
if (
|
||||||
|
self.mode == MoRIIOMode.READ
|
||||||
|
and self.kv_transfer_config.is_kv_consumer
|
||||||
|
and vllm_config.compilation_config.cudagraph_mode.has_full_cudagraphs()
|
||||||
|
):
|
||||||
|
# warn only; kv-read barrier requires PIECEWISE cudagraph mode
|
||||||
|
logger.warning_once(
|
||||||
|
"MoRIIO READ mode is running with %s CUDA graphs: per-layer "
|
||||||
|
"KV-read barrier can't fire inside full graph; accuracy may "
|
||||||
|
"degrade at high concurrency. Set cudagraph_mode=PIECEWISE "
|
||||||
|
"in --compilation-config.",
|
||||||
|
vllm_config.compilation_config.cudagraph_mode.name,
|
||||||
|
)
|
||||||
if role == KVConnectorRole.SCHEDULER:
|
if role == KVConnectorRole.SCHEDULER:
|
||||||
self.connector_scheduler: MoRIIOConnectorScheduler | None = (
|
self.connector_scheduler: MoRIIOConnectorScheduler | None = (
|
||||||
MoRIIOConnectorScheduler(vllm_config, self.engine_id)
|
MoRIIOConnectorScheduler(vllm_config, self.engine_id)
|
||||||
@@ -299,7 +312,8 @@ class MoRIIOConnector(KVConnectorBase_V1):
|
|||||||
self.connector_worker.start_load_kv(self._connector_metadata)
|
self.connector_worker.start_load_kv(self._connector_metadata)
|
||||||
|
|
||||||
def wait_for_layer_load(self, layer_name: str) -> None:
|
def wait_for_layer_load(self, layer_name: str) -> None:
|
||||||
pass
|
assert self.connector_worker is not None
|
||||||
|
self.connector_worker.wait_for_layer_load(layer_name)
|
||||||
|
|
||||||
def save_kv_layer(
|
def save_kv_layer(
|
||||||
self,
|
self,
|
||||||
@@ -1225,8 +1239,8 @@ class MoRIIOConnectorWorker:
|
|||||||
# Map of engine_id -> num_blocks. All ranks in the same deployment will
|
# Map of engine_id -> num_blocks. All ranks in the same deployment will
|
||||||
# have the same number of blocks.
|
# have the same number of blocks.
|
||||||
self.dst_num_blocks: dict[EngineId, int] = {}
|
self.dst_num_blocks: dict[EngineId, int] = {}
|
||||||
# In progress transfers.
|
# In-progress READ transfers: req_id -> {layer_name: status}.
|
||||||
self._recving_transfers: defaultdict[ReqId, list] = defaultdict(list)
|
self._recving_transfers: defaultdict[ReqId, dict] = defaultdict(dict)
|
||||||
# Values are (remote_host, remote_notify_port, transfer_id).
|
# Values are (remote_host, remote_notify_port, transfer_id).
|
||||||
self._recving_transfers_callback_addr: dict[ReqId, tuple[str, str, str]] = {}
|
self._recving_transfers_callback_addr: dict[ReqId, tuple[str, str, str]] = {}
|
||||||
# Monotonic-clock start times for each in-flight recv transfer.
|
# Monotonic-clock start times for each in-flight recv transfer.
|
||||||
@@ -1897,14 +1911,60 @@ class MoRIIOConnectorWorker:
|
|||||||
|
|
||||||
return done_sending, done_recving
|
return done_sending, done_recving
|
||||||
|
|
||||||
|
def wait_for_layer_load(self, layer_name: str) -> None:
|
||||||
|
"""Block until all in-flight READs of this layer have landed.
|
||||||
|
|
||||||
|
A host-side blocking wait must not run during full-graph capture.
|
||||||
|
"""
|
||||||
|
if self.is_producer or self.mode != MoRIIOMode.READ:
|
||||||
|
return
|
||||||
|
|
||||||
|
if get_forward_context().cudagraph_runtime_mode == CUDAGraphMode.FULL:
|
||||||
|
return
|
||||||
|
|
||||||
|
deadline = time.monotonic() + self.moriio_config.transfer_timeout
|
||||||
|
while True:
|
||||||
|
with self.moriio_wrapper.lock:
|
||||||
|
pending = [
|
||||||
|
status_by_layer[layer_name]
|
||||||
|
for status_by_layer in self._recving_transfers.values()
|
||||||
|
if layer_name in status_by_layer
|
||||||
|
]
|
||||||
|
|
||||||
|
if not pending:
|
||||||
|
return
|
||||||
|
|
||||||
|
still_running = False
|
||||||
|
for status in pending:
|
||||||
|
# A failed read is dropped in _pop_done_transfers.
|
||||||
|
if status.Succeeded() or status.Failed():
|
||||||
|
continue
|
||||||
|
still_running = True
|
||||||
|
|
||||||
|
if not still_running:
|
||||||
|
return
|
||||||
|
|
||||||
|
if time.monotonic() > deadline:
|
||||||
|
logger.warning(
|
||||||
|
"MoRIIO READ barrier timed out for layer %s; proceeding "
|
||||||
|
"(request dropped via get_finished).",
|
||||||
|
layer_name,
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
time.sleep(0.001)
|
||||||
|
|
||||||
def _pop_done_transfers(self) -> set[str]:
|
def _pop_done_transfers(self) -> set[str]:
|
||||||
done_req_ids: set[str] = set()
|
done_req_ids: set[str] = set()
|
||||||
_xfer_timeout = int(os.environ.get("VLLM_MORIIO_TRANSFER_TIMEOUT_S", "120"))
|
_xfer_timeout = int(os.environ.get("VLLM_MORIIO_TRANSFER_TIMEOUT_S", "120"))
|
||||||
with self.moriio_wrapper.lock:
|
with self.moriio_wrapper.lock:
|
||||||
to_remove = []
|
to_remove = []
|
||||||
for req_id, status_list in self._recving_transfers.items():
|
for req_id, status_by_layer in self._recving_transfers.items():
|
||||||
last = status_list[-1]
|
statuses = list(status_by_layer.values())
|
||||||
if last.Succeeded():
|
failed_status = next(
|
||||||
|
(status for status in statuses if status.Failed()), None
|
||||||
|
)
|
||||||
|
if statuses and all(status.Succeeded() for status in statuses):
|
||||||
host, port, xfer_id = self._recving_transfers_callback_addr[req_id]
|
host, port, xfer_id = self._recving_transfers_callback_addr[req_id]
|
||||||
done_req_ids.add(xfer_id)
|
done_req_ids.add(xfer_id)
|
||||||
self.moriio_wrapper.send_notify(
|
self.moriio_wrapper.send_notify(
|
||||||
@@ -1915,14 +1975,14 @@ class MoRIIOConnectorWorker:
|
|||||||
message_fields={"consumer_tp_size": self.world_size},
|
message_fields={"consumer_tp_size": self.world_size},
|
||||||
)
|
)
|
||||||
to_remove.append(req_id)
|
to_remove.append(req_id)
|
||||||
elif last.Failed():
|
elif failed_status is not None:
|
||||||
logger.error(
|
logger.error(
|
||||||
"RDMA transfer failed for request %s: %s (code=%s). "
|
"RDMA transfer failed for request %s: %s (code=%s). "
|
||||||
"Notifying prefill to free blocks; request will be "
|
"Notifying prefill to free blocks; request will be "
|
||||||
"aborted by timeout.",
|
"aborted by timeout.",
|
||||||
req_id,
|
req_id,
|
||||||
last.Message(),
|
failed_status.Message(),
|
||||||
last.Code(),
|
failed_status.Code(),
|
||||||
)
|
)
|
||||||
host, port, xfer_id = self._recving_transfers_callback_addr[req_id]
|
host, port, xfer_id = self._recving_transfers_callback_addr[req_id]
|
||||||
try:
|
try:
|
||||||
@@ -2564,7 +2624,7 @@ class MoRIIOConnectorWorker:
|
|||||||
time.sleep(_backoff)
|
time.sleep(_backoff)
|
||||||
_backoff = min(_backoff * 2, 0.05)
|
_backoff = min(_backoff * 2, 0.05)
|
||||||
with self.moriio_wrapper.lock:
|
with self.moriio_wrapper.lock:
|
||||||
self._recving_transfers[request_id].append(transfer_status)
|
self._recving_transfers[request_id][layer_name] = transfer_status
|
||||||
self._recving_transfers_start.setdefault(request_id, time.monotonic())
|
self._recving_transfers_start.setdefault(request_id, time.monotonic())
|
||||||
self._recving_transfers_callback_addr[request_id] = (
|
self._recving_transfers_callback_addr[request_id] = (
|
||||||
remote_host,
|
remote_host,
|
||||||
|
|||||||
Reference in New Issue
Block a user