[Frontend] Watch frontend processes during engine startup (#43417)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Bugen Zhao
2026-08-06 11:17:24 -07:00
committed by GitHub
co-authored by OpenAI Codex Nick Hill
parent 4f851bef6c
commit 7b9f2dad89
5 changed files with 152 additions and 42 deletions
+64 -26
View File
@@ -5,10 +5,10 @@ import contextlib
import os
import threading
import weakref
from collections.abc import Callable, Iterator
from collections.abc import Callable, Iterator, Sequence
from dataclasses import dataclass
from enum import Enum, auto
from multiprocessing import Process, connection
from multiprocessing import connection
from multiprocessing.process import BaseProcess
from multiprocessing.queues import Queue
from typing import TYPE_CHECKING, cast
@@ -32,7 +32,7 @@ from vllm.utils.system_utils import get_mp_context
from vllm.v1.engine.coordinator import DPCoordinator
from vllm.v1.executor import Executor
from vllm.v1.executor.ray_utils import WORKER_SPECIFIC_ENV_VARS
from vllm.v1.utils import get_engine_client_zmq_addr, shutdown
from vllm.v1.utils import _SubprocessWrapper, get_engine_client_zmq_addr, shutdown
if TYPE_CHECKING:
from ray.util.placement_group import PlacementGroup
@@ -1050,20 +1050,29 @@ def get_engine_zmq_addresses(
)
FrontendProcess = BaseProcess | _SubprocessWrapper
@dataclass
class CoreEngineLaunch:
"""Resources and startup barrier for launched engine processes."""
engine_manager: CoreEngineProcManager | CoreEngineActorManager | None
coordinator: DPCoordinator | None
addresses: EngineZmqAddresses
tensor_queue: Queue | None
# Frontend processes to watch during engine startup; may be assigned by
# the caller before the startup barrier runs on context manager exit.
watched_frontend_processes: Sequence[FrontendProcess] = ()
@contextlib.contextmanager
def launch_core_engines(
vllm_config: VllmConfig,
executor_class: type[Executor],
log_stats: bool,
addresses: EngineZmqAddresses,
) -> Iterator[
tuple[
CoreEngineProcManager | CoreEngineActorManager | None,
DPCoordinator | None,
EngineZmqAddresses,
Queue | None,
]
]:
) -> Iterator[CoreEngineLaunch]:
"""Launch engine and DP coordinator processes as needed."""
parallel_config = vllm_config.parallel_config
@@ -1119,7 +1128,9 @@ def launch_core_engines(
log_stats=log_stats,
)
yield engine_actor_manager, coordinator, addresses, tensor_queue
yield CoreEngineLaunch(
engine_actor_manager, coordinator, addresses, tensor_queue
)
return
if offline_mode:
@@ -1188,30 +1199,27 @@ def launch_core_engines(
else:
local_engine_manager = None
yield local_engine_manager, coordinator, addresses, tensor_queue
# Now wait for engines to start.
launch = CoreEngineLaunch(
local_engine_manager, coordinator, addresses, tensor_queue
)
yield launch
wait_for_engine_startup(
handshake_socket,
addresses,
engines_to_handshake,
parallel_config,
dp_size > 1 and vllm_config.model_config.is_moe,
vllm_config.cache_config,
local_engine_manager,
coordinator.proc if coordinator else None,
launch,
)
def wait_for_engine_startup(
handshake_socket: zmq.Socket,
addresses: EngineZmqAddresses,
core_engines: list[CoreEngine],
parallel_config: ParallelConfig,
coordinated_dp: bool,
cache_config: CacheConfig,
proc_manager: CoreEngineProcManager | None,
coord_process: Process | None,
launch: CoreEngineLaunch,
):
# Wait for engine core process(es) to send ready messages.
local_count = parallel_config.data_parallel_size_local
@@ -1226,11 +1234,21 @@ def wait_for_engine_startup(
and not parallel_config.data_parallel_external_lb
)
if proc_manager is not None:
for sentinel in proc_manager.sentinels():
# 1. Engine processes
if isinstance(launch.engine_manager, CoreEngineProcManager):
for sentinel in launch.engine_manager.sentinels():
poller.register(sentinel, zmq.POLLIN)
# 2. DP Coordinator process, if present
coord_process = launch.coordinator.proc if launch.coordinator else None
if coord_process is not None:
poller.register(coord_process.sentinel, zmq.POLLIN)
# 3. Watched frontend processes, if any
frontend_process_by_fd: dict[int, FrontendProcess] = {}
for proc in launch.watched_frontend_processes:
fd = proc.sentinel if isinstance(proc.sentinel, int) else proc.sentinel.fileno()
frontend_process_by_fd[fd] = proc
poller.register(fd, zmq.POLLIN)
while any(conn_pending) or any(start_pending):
events = poller.poll(STARTUP_POLL_PERIOD_MS)
if not events:
@@ -1246,14 +1264,34 @@ def wait_for_engine_startup(
)
continue
if len(events) > 1 or events[0][0] != handshake_socket:
# One of the local core processes exited.
finished = proc_manager.finished_procs() if proc_manager else {}
# One of the local core, coordinator, or watched frontend processes exited.
if isinstance(launch.engine_manager, CoreEngineProcManager):
finished = launch.engine_manager.finished_procs()
else:
finished = {}
if coord_process is not None and coord_process.exitcode is not None:
finished[coord_process.name] = coord_process.exitcode
failed_frontend_procs = {
proc.name: proc.exitcode
for fd, proc in frontend_process_by_fd.items()
if proc.exitcode is not None
or any(event_fd == fd for event_fd, _ in events)
}
if failed_frontend_procs and not finished:
raise RuntimeError(
"Frontend process failed during engine core initialization. "
"See root cause above. "
f"Failed frontend proc(s): {failed_frontend_procs}"
)
raise RuntimeError(
"Engine core initialization failed. "
"See root cause above. "
f"Failed core proc(s): {finished}"
+ (
f", failed frontend proc(s): {failed_frontend_procs}"
if failed_frontend_procs
else ""
)
)
# Receive HELLO and READY messages from the input socket.
@@ -1293,7 +1331,7 @@ def wait_for_engine_startup(
# Send init message with DP config info.
init_message = msgspec.msgpack.encode(
EngineHandshakeMetadata(
addresses=addresses,
addresses=launch.addresses,
parallel_config={
k: getattr(parallel_config, k)
for k in (