[Frontend] Watch frontend processes during engine startup (#43417)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Bugen Zhao
2026-08-06 11:17:24 -07:00
committed by GitHub
co-authored by OpenAI Codex Nick Hill
parent 4f851bef6c
commit 7b9f2dad89
5 changed files with 152 additions and 42 deletions
@@ -308,12 +308,8 @@ def test_ray_dp_addresses_resolved_before_actor_creation(
executor_class=_DummyExecutor,
log_stats=False,
addresses=addresses,
) as (
engine_manager,
_coordinator,
_addresses_out,
_tensor_queue,
):
) as engine_launch:
engine_manager = engine_launch.engine_manager
assert isinstance(engine_manager, CoreEngineActorManager)
# API-server children bind to the pre-allocated ports.
@@ -0,0 +1,68 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from multiprocessing import connection
from types import SimpleNamespace
import pytest
import zmq
from vllm.v1.engine.utils import (
CoreEngine,
CoreEngineLaunch,
EngineZmqAddresses,
wait_for_engine_startup,
)
pytestmark = pytest.mark.skip_global_cleanup
class _FinishedProcess:
name = "RustFrontend"
def __init__(self, sentinel):
self.sentinel = sentinel
@property
def exitcode(self):
return 1
def test_wait_for_engine_startup_reports_watched_process_exit():
ctx = zmq.Context()
handshake_socket = ctx.socket(zmq.ROUTER)
recv, send = connection.Pipe(duplex=False)
send.close()
parallel_config = SimpleNamespace(
data_parallel_size_local=1,
data_parallel_hybrid_lb=False,
data_parallel_external_lb=False,
)
try:
launch = CoreEngineLaunch(
engine_manager=None,
coordinator=None,
addresses=EngineZmqAddresses(inputs=[], outputs=[]),
tensor_queue=None,
)
launch.watched_frontend_processes = [_FinishedProcess(recv)]
with pytest.raises(RuntimeError) as exc_info:
wait_for_engine_startup(
handshake_socket,
[CoreEngine()],
parallel_config, # type: ignore[arg-type]
coordinated_dp=False,
cache_config=None, # type: ignore[arg-type]
launch=launch,
)
finally:
recv.close()
handshake_socket.close(linger=0)
ctx.term()
assert "Frontend process failed during engine core initialization" in str(
exc_info.value
)
assert "Failed frontend proc(s): {'RustFrontend': 1}" in str(exc_info.value)
+12 -7
View File
@@ -326,12 +326,12 @@ def run_multi_api_server(args: argparse.Namespace):
defer_api_server_ports=not (rust_frontend_path or is_ray_dp),
)
with launch_core_engines(vllm_config, executor_class, log_stats, addresses) as (
local_engine_manager,
coordinator,
addresses,
tensor_queue,
):
with launch_core_engines(
vllm_config, executor_class, log_stats, addresses
) as engine_launch:
local_engine_manager = engine_launch.engine_manager
coordinator = engine_launch.coordinator
addresses = engine_launch.addresses
stats_update_address = (
coordinator.get_stats_publish_address() if coordinator else None
)
@@ -364,7 +364,7 @@ def run_multi_api_server(args: argparse.Namespace):
input_addresses=addresses.inputs,
output_addresses=addresses.outputs,
stats_update_address=stats_update_address,
tensor_queue=tensor_queue,
tensor_queue=engine_launch.tensor_queue,
)
if not is_ray_dp:
@@ -377,6 +377,11 @@ def run_multi_api_server(args: argparse.Namespace):
addresses.inputs = actual_inputs
addresses.outputs = actual_outputs
# Set frontend processes to watch during engine startup.
# If any of these processes exit before the engines are up, the engine startup
# will be aborted with an error.
engine_launch.watched_frontend_processes = api_server_manager.processes
# Wait for API servers.
try:
wait_for_completion_or_failure(
+6 -3
View File
@@ -608,9 +608,12 @@ class MPClient(EngineCoreClient):
with launch_core_engines(
vllm_config, executor_class, log_stats, addresses
) as (engine_manager, coordinator, addresses, tensor_queue):
self.resources.coordinator = coordinator
self.resources.engine_manager = engine_manager
) as engine_launch:
self.resources.coordinator = engine_launch.coordinator
self.resources.engine_manager = engine_launch.engine_manager
coordinator = engine_launch.coordinator
addresses = engine_launch.addresses
tensor_queue = engine_launch.tensor_queue
self.stats_update_address = addresses.frontend_stats_publish_address
if coordinator is not None:
+64 -26
View File
@@ -5,10 +5,10 @@ import contextlib
import os
import threading
import weakref
from collections.abc import Callable, Iterator
from collections.abc import Callable, Iterator, Sequence
from dataclasses import dataclass
from enum import Enum, auto
from multiprocessing import Process, connection
from multiprocessing import connection
from multiprocessing.process import BaseProcess
from multiprocessing.queues import Queue
from typing import TYPE_CHECKING, cast
@@ -32,7 +32,7 @@ from vllm.utils.system_utils import get_mp_context
from vllm.v1.engine.coordinator import DPCoordinator
from vllm.v1.executor import Executor
from vllm.v1.executor.ray_utils import WORKER_SPECIFIC_ENV_VARS
from vllm.v1.utils import get_engine_client_zmq_addr, shutdown
from vllm.v1.utils import _SubprocessWrapper, get_engine_client_zmq_addr, shutdown
if TYPE_CHECKING:
from ray.util.placement_group import PlacementGroup
@@ -1050,20 +1050,29 @@ def get_engine_zmq_addresses(
)
FrontendProcess = BaseProcess | _SubprocessWrapper
@dataclass
class CoreEngineLaunch:
"""Resources and startup barrier for launched engine processes."""
engine_manager: CoreEngineProcManager | CoreEngineActorManager | None
coordinator: DPCoordinator | None
addresses: EngineZmqAddresses
tensor_queue: Queue | None
# Frontend processes to watch during engine startup; may be assigned by
# the caller before the startup barrier runs on context manager exit.
watched_frontend_processes: Sequence[FrontendProcess] = ()
@contextlib.contextmanager
def launch_core_engines(
vllm_config: VllmConfig,
executor_class: type[Executor],
log_stats: bool,
addresses: EngineZmqAddresses,
) -> Iterator[
tuple[
CoreEngineProcManager | CoreEngineActorManager | None,
DPCoordinator | None,
EngineZmqAddresses,
Queue | None,
]
]:
) -> Iterator[CoreEngineLaunch]:
"""Launch engine and DP coordinator processes as needed."""
parallel_config = vllm_config.parallel_config
@@ -1119,7 +1128,9 @@ def launch_core_engines(
log_stats=log_stats,
)
yield engine_actor_manager, coordinator, addresses, tensor_queue
yield CoreEngineLaunch(
engine_actor_manager, coordinator, addresses, tensor_queue
)
return
if offline_mode:
@@ -1188,30 +1199,27 @@ def launch_core_engines(
else:
local_engine_manager = None
yield local_engine_manager, coordinator, addresses, tensor_queue
# Now wait for engines to start.
launch = CoreEngineLaunch(
local_engine_manager, coordinator, addresses, tensor_queue
)
yield launch
wait_for_engine_startup(
handshake_socket,
addresses,
engines_to_handshake,
parallel_config,
dp_size > 1 and vllm_config.model_config.is_moe,
vllm_config.cache_config,
local_engine_manager,
coordinator.proc if coordinator else None,
launch,
)
def wait_for_engine_startup(
handshake_socket: zmq.Socket,
addresses: EngineZmqAddresses,
core_engines: list[CoreEngine],
parallel_config: ParallelConfig,
coordinated_dp: bool,
cache_config: CacheConfig,
proc_manager: CoreEngineProcManager | None,
coord_process: Process | None,
launch: CoreEngineLaunch,
):
# Wait for engine core process(es) to send ready messages.
local_count = parallel_config.data_parallel_size_local
@@ -1226,11 +1234,21 @@ def wait_for_engine_startup(
and not parallel_config.data_parallel_external_lb
)
if proc_manager is not None:
for sentinel in proc_manager.sentinels():
# 1. Engine processes
if isinstance(launch.engine_manager, CoreEngineProcManager):
for sentinel in launch.engine_manager.sentinels():
poller.register(sentinel, zmq.POLLIN)
# 2. DP Coordinator process, if present
coord_process = launch.coordinator.proc if launch.coordinator else None
if coord_process is not None:
poller.register(coord_process.sentinel, zmq.POLLIN)
# 3. Watched frontend processes, if any
frontend_process_by_fd: dict[int, FrontendProcess] = {}
for proc in launch.watched_frontend_processes:
fd = proc.sentinel if isinstance(proc.sentinel, int) else proc.sentinel.fileno()
frontend_process_by_fd[fd] = proc
poller.register(fd, zmq.POLLIN)
while any(conn_pending) or any(start_pending):
events = poller.poll(STARTUP_POLL_PERIOD_MS)
if not events:
@@ -1246,14 +1264,34 @@ def wait_for_engine_startup(
)
continue
if len(events) > 1 or events[0][0] != handshake_socket:
# One of the local core processes exited.
finished = proc_manager.finished_procs() if proc_manager else {}
# One of the local core, coordinator, or watched frontend processes exited.
if isinstance(launch.engine_manager, CoreEngineProcManager):
finished = launch.engine_manager.finished_procs()
else:
finished = {}
if coord_process is not None and coord_process.exitcode is not None:
finished[coord_process.name] = coord_process.exitcode
failed_frontend_procs = {
proc.name: proc.exitcode
for fd, proc in frontend_process_by_fd.items()
if proc.exitcode is not None
or any(event_fd == fd for event_fd, _ in events)
}
if failed_frontend_procs and not finished:
raise RuntimeError(
"Frontend process failed during engine core initialization. "
"See root cause above. "
f"Failed frontend proc(s): {failed_frontend_procs}"
)
raise RuntimeError(
"Engine core initialization failed. "
"See root cause above. "
f"Failed core proc(s): {finished}"
+ (
f", failed frontend proc(s): {failed_frontend_procs}"
if failed_frontend_procs
else ""
)
)
# Receive HELLO and READY messages from the input socket.
@@ -1293,7 +1331,7 @@ def wait_for_engine_startup(
# Send init message with DP config info.
init_message = msgspec.msgpack.encode(
EngineHandshakeMetadata(
addresses=addresses,
addresses=launch.addresses,
parallel_config={
k: getattr(parallel_config, k)
for k in (