[Bugfix][MoRIIO] Keep discovery heartbeats running while workers hold the GIL (#59441)

Signed-off-by: whx-sjtu <xiaowang990929@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Hexiang Wang
2026-10-03 14:55:18 +00:00
committed by GitHub
co-authored by Codex
parent e319f86f15
commit f03026a548
4 changed files with 284 additions and 71 deletions
@@ -59,6 +59,9 @@ from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_engine import (
MoRIIOWrapper,
MoRIIOWriter,
)
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_heartbeat import (
MoRIIOHeartbeat,
)
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_layout import (
LayerTransferGeometry,
MambaOffsetTemplate,
@@ -1600,7 +1603,7 @@ class MoRIIOConnectorWorker:
self.moriio_engine = None
self._handle_request_thread = None
self._ping_thread = None
self._heartbeat: MoRIIOHeartbeat | None = None
self._writer = MoRIIOWriter(self)
# Completions that arrived before transfer_id_to_request_id was populated.
# Retried each step until the mapping is established.
@@ -1630,10 +1633,7 @@ class MoRIIOConnectorWorker:
)
if self._rank == 0 and self.moriio_config.proxy_ip:
self._ping_thread = threading.Thread(
target=self._ping, args=(self.zmq_context,), daemon=True
)
self._ping_thread.start()
self._heartbeat = self._start_heartbeat()
logger.info(
"Initializing MoRIIO Engine, engine = %s, role = %s",
@@ -1881,75 +1881,31 @@ class MoRIIOConnectorWorker:
remote_engine_id
]
def _ping(self, zmq_context):
# Use host:port format for http_address (compatible with official router)
http_address = f"{self.request_address}"
# Include host so the router embeds it in the request_id; the connector
# on the other side parses host/ports from there.
zmq_address = (
f"host:{self.local_ip},"
f"handshake:{self.handshake_port},"
f"notify:{self.notify_port}"
def _start_heartbeat(self) -> MoRIIOHeartbeat:
payload = {
"type": "P" if self.is_producer else "D",
"http_address": self.request_address,
"zmq_address": (
f"host:{self.local_ip},handshake:{self.handshake_port},"
f"notify:{self.notify_port}"
),
"dp_size": self.moriio_config.dp_size,
"tp_size": self.moriio_config.tp_size,
"transfer_mode": self.mode.name,
}
return MoRIIOHeartbeat(
f"tcp://{self.proxy_ip}:{self.proxy_ping_port}",
payload,
MoRIIOConstants.PING_INTERVAL,
MoRIIOConstants.MAX_PING_RETRIES,
)
role = "P" if self.is_producer else "D"
retry_count = 0
index = 1
with zmq_context.socket(zmq.DEALER) as sock:
sock.connect(f"tcp://{self.proxy_ip}:{self.proxy_ping_port}")
while True:
try:
data = {
"type": role, # "P" or "D"
"http_address": http_address,
"zmq_address": zmq_address,
# dp_size/tp_size are not used by the official vLLM router
# (routing operates at the http_address level); they are
# consumed only by the toy proxy server.
"dp_size": self.moriio_config.dp_size,
"tp_size": self.moriio_config.tp_size,
# transfer_mode is included so the router can distinguish
# READ (prefill-then-decode, sequential) from WRITE (concurrent)
# scheduling.
"transfer_mode": self.mode.name,
}
sock.send(msgpack.dumps(data))
# logger.debug(f"Successfully sent ping message #{index}")
retry_count = 0
except ConnectionRefusedError:
logger.info(
"Connection refused: %s:%s -> %s:%s",
self.local_ip,
self.local_ping_port,
self.proxy_ip,
self.proxy_ping_port,
)
retry_count += 1
except OSError as e:
logger.info("OS error when sending ping: %s", e)
retry_count += 1
except Exception as e:
logger.info("Unexpected error when sending ping: %s", e)
retry_count += 1
if retry_count >= MoRIIOConstants.MAX_PING_RETRIES:
logger.error(
"Max retries (%s) exceeded. Stopping ping loop.",
MoRIIOConstants.MAX_PING_RETRIES,
)
raise RuntimeError(
f"Ping failed after {retry_count} retries"
) from e
finally:
time.sleep(MoRIIOConstants.PING_INTERVAL)
index += 1
def shutdown(self):
heartbeat = getattr(self, "_heartbeat", None)
if heartbeat is not None:
heartbeat.shutdown()
self._heartbeat = None
if hasattr(self, "moriio_wrapper") and self.moriio_wrapper:
self.moriio_wrapper.shutdown()
@@ -2580,6 +2536,9 @@ class MoRIIOConnectorWorker:
The scheduler process (via the MultiprocExecutor) will use this output
to track which workers are done.
"""
heartbeat = getattr(self, "_heartbeat", None)
if heartbeat is not None:
heartbeat.check_health()
done_sending, done_recving = set(), set()
if self.is_producer:
@@ -0,0 +1,96 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Keep discovery heartbeats independent of the model worker's GIL."""
import json
import logging
import select
import subprocess
import sys
from typing import Any
import msgpack
import zmq
logger = logging.getLogger(__name__)
class MoRIIOHeartbeat:
def __init__(
self,
endpoint: str,
payload: dict[str, Any],
interval: float,
max_retries: int = 100,
):
self._process = subprocess.Popen(
[sys.executable, __file__, endpoint, str(interval), str(max_retries)],
stdin=subprocess.PIPE,
text=True,
)
assert self._process.stdin is not None
try:
self._process.stdin.write(json.dumps(payload) + "\n")
self._process.stdin.flush()
except BaseException:
self.shutdown()
raise
def check_health(self) -> None:
returncode = self._process.poll()
if returncode is not None:
raise RuntimeError(
f"MoRIIO discovery heartbeat exited with code {returncode}"
)
def shutdown(self) -> None:
assert self._process.stdin is not None
try:
if not self._process.stdin.closed:
self._process.stdin.close()
except BrokenPipeError:
pass
finally:
try:
self._process.wait(timeout=5)
except subprocess.TimeoutExpired:
self._process.terminate()
try:
self._process.wait(timeout=1)
except subprocess.TimeoutExpired:
self._process.kill()
self._process.wait(timeout=1)
def main() -> None:
endpoint, interval_arg, max_retries_arg = sys.argv[1:]
initial = sys.stdin.readline()
if not initial:
return
payload = msgpack.dumps(json.loads(initial))
interval = float(interval_arg)
max_retries = int(max_retries_arg)
failures = 0
with zmq.Context() as context, context.socket(zmq.DEALER) as sock:
sock.setsockopt(zmq.LINGER, 0)
sock.setsockopt(zmq.SNDTIMEO, 1000)
sock.connect(endpoint)
while not select.select([sys.stdin], [], [], 0)[0]:
try:
sock.send(payload)
failures = 0
except zmq.Again:
failures = 0
logger.warning("MoRIIO discovery heartbeat send failed; retrying")
except zmq.ZMQError:
failures += 1
logger.exception("MoRIIO discovery heartbeat send failed")
if failures >= max_retries:
raise
# EOF also stops the child when the worker exits without cleanup.
if select.select([sys.stdin], [], [], interval)[0]:
break
if __name__ == "__main__":
main()