[Frontend] Watch frontend processes during engine startup (#43417)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
OpenAI Codex
Nick Hill
parent
4f851bef6c
commit
7b9f2dad89
@@ -308,12 +308,8 @@ def test_ray_dp_addresses_resolved_before_actor_creation(
|
||||
executor_class=_DummyExecutor,
|
||||
log_stats=False,
|
||||
addresses=addresses,
|
||||
) as (
|
||||
engine_manager,
|
||||
_coordinator,
|
||||
_addresses_out,
|
||||
_tensor_queue,
|
||||
):
|
||||
) as engine_launch:
|
||||
engine_manager = engine_launch.engine_manager
|
||||
assert isinstance(engine_manager, CoreEngineActorManager)
|
||||
|
||||
# API-server children bind to the pre-allocated ports.
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from multiprocessing import connection
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import zmq
|
||||
|
||||
from vllm.v1.engine.utils import (
|
||||
CoreEngine,
|
||||
CoreEngineLaunch,
|
||||
EngineZmqAddresses,
|
||||
wait_for_engine_startup,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
|
||||
class _FinishedProcess:
|
||||
name = "RustFrontend"
|
||||
|
||||
def __init__(self, sentinel):
|
||||
self.sentinel = sentinel
|
||||
|
||||
@property
|
||||
def exitcode(self):
|
||||
return 1
|
||||
|
||||
|
||||
def test_wait_for_engine_startup_reports_watched_process_exit():
|
||||
ctx = zmq.Context()
|
||||
handshake_socket = ctx.socket(zmq.ROUTER)
|
||||
recv, send = connection.Pipe(duplex=False)
|
||||
send.close()
|
||||
|
||||
parallel_config = SimpleNamespace(
|
||||
data_parallel_size_local=1,
|
||||
data_parallel_hybrid_lb=False,
|
||||
data_parallel_external_lb=False,
|
||||
)
|
||||
|
||||
try:
|
||||
launch = CoreEngineLaunch(
|
||||
engine_manager=None,
|
||||
coordinator=None,
|
||||
addresses=EngineZmqAddresses(inputs=[], outputs=[]),
|
||||
tensor_queue=None,
|
||||
)
|
||||
launch.watched_frontend_processes = [_FinishedProcess(recv)]
|
||||
with pytest.raises(RuntimeError) as exc_info:
|
||||
wait_for_engine_startup(
|
||||
handshake_socket,
|
||||
[CoreEngine()],
|
||||
parallel_config, # type: ignore[arg-type]
|
||||
coordinated_dp=False,
|
||||
cache_config=None, # type: ignore[arg-type]
|
||||
launch=launch,
|
||||
)
|
||||
finally:
|
||||
recv.close()
|
||||
handshake_socket.close(linger=0)
|
||||
ctx.term()
|
||||
|
||||
assert "Frontend process failed during engine core initialization" in str(
|
||||
exc_info.value
|
||||
)
|
||||
assert "Failed frontend proc(s): {'RustFrontend': 1}" in str(exc_info.value)
|
||||
@@ -326,12 +326,12 @@ def run_multi_api_server(args: argparse.Namespace):
|
||||
defer_api_server_ports=not (rust_frontend_path or is_ray_dp),
|
||||
)
|
||||
|
||||
with launch_core_engines(vllm_config, executor_class, log_stats, addresses) as (
|
||||
local_engine_manager,
|
||||
coordinator,
|
||||
addresses,
|
||||
tensor_queue,
|
||||
):
|
||||
with launch_core_engines(
|
||||
vllm_config, executor_class, log_stats, addresses
|
||||
) as engine_launch:
|
||||
local_engine_manager = engine_launch.engine_manager
|
||||
coordinator = engine_launch.coordinator
|
||||
addresses = engine_launch.addresses
|
||||
stats_update_address = (
|
||||
coordinator.get_stats_publish_address() if coordinator else None
|
||||
)
|
||||
@@ -364,7 +364,7 @@ def run_multi_api_server(args: argparse.Namespace):
|
||||
input_addresses=addresses.inputs,
|
||||
output_addresses=addresses.outputs,
|
||||
stats_update_address=stats_update_address,
|
||||
tensor_queue=tensor_queue,
|
||||
tensor_queue=engine_launch.tensor_queue,
|
||||
)
|
||||
|
||||
if not is_ray_dp:
|
||||
@@ -377,6 +377,11 @@ def run_multi_api_server(args: argparse.Namespace):
|
||||
addresses.inputs = actual_inputs
|
||||
addresses.outputs = actual_outputs
|
||||
|
||||
# Set frontend processes to watch during engine startup.
|
||||
# If any of these processes exit before the engines are up, the engine startup
|
||||
# will be aborted with an error.
|
||||
engine_launch.watched_frontend_processes = api_server_manager.processes
|
||||
|
||||
# Wait for API servers.
|
||||
try:
|
||||
wait_for_completion_or_failure(
|
||||
|
||||
@@ -608,9 +608,12 @@ class MPClient(EngineCoreClient):
|
||||
|
||||
with launch_core_engines(
|
||||
vllm_config, executor_class, log_stats, addresses
|
||||
) as (engine_manager, coordinator, addresses, tensor_queue):
|
||||
self.resources.coordinator = coordinator
|
||||
self.resources.engine_manager = engine_manager
|
||||
) as engine_launch:
|
||||
self.resources.coordinator = engine_launch.coordinator
|
||||
self.resources.engine_manager = engine_launch.engine_manager
|
||||
coordinator = engine_launch.coordinator
|
||||
addresses = engine_launch.addresses
|
||||
tensor_queue = engine_launch.tensor_queue
|
||||
|
||||
self.stats_update_address = addresses.frontend_stats_publish_address
|
||||
if coordinator is not None:
|
||||
|
||||
+64
-26
@@ -5,10 +5,10 @@ import contextlib
|
||||
import os
|
||||
import threading
|
||||
import weakref
|
||||
from collections.abc import Callable, Iterator
|
||||
from collections.abc import Callable, Iterator, Sequence
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum, auto
|
||||
from multiprocessing import Process, connection
|
||||
from multiprocessing import connection
|
||||
from multiprocessing.process import BaseProcess
|
||||
from multiprocessing.queues import Queue
|
||||
from typing import TYPE_CHECKING, cast
|
||||
@@ -32,7 +32,7 @@ from vllm.utils.system_utils import get_mp_context
|
||||
from vllm.v1.engine.coordinator import DPCoordinator
|
||||
from vllm.v1.executor import Executor
|
||||
from vllm.v1.executor.ray_utils import WORKER_SPECIFIC_ENV_VARS
|
||||
from vllm.v1.utils import get_engine_client_zmq_addr, shutdown
|
||||
from vllm.v1.utils import _SubprocessWrapper, get_engine_client_zmq_addr, shutdown
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ray.util.placement_group import PlacementGroup
|
||||
@@ -1050,20 +1050,29 @@ def get_engine_zmq_addresses(
|
||||
)
|
||||
|
||||
|
||||
FrontendProcess = BaseProcess | _SubprocessWrapper
|
||||
|
||||
|
||||
@dataclass
|
||||
class CoreEngineLaunch:
|
||||
"""Resources and startup barrier for launched engine processes."""
|
||||
|
||||
engine_manager: CoreEngineProcManager | CoreEngineActorManager | None
|
||||
coordinator: DPCoordinator | None
|
||||
addresses: EngineZmqAddresses
|
||||
tensor_queue: Queue | None
|
||||
# Frontend processes to watch during engine startup; may be assigned by
|
||||
# the caller before the startup barrier runs on context manager exit.
|
||||
watched_frontend_processes: Sequence[FrontendProcess] = ()
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def launch_core_engines(
|
||||
vllm_config: VllmConfig,
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
addresses: EngineZmqAddresses,
|
||||
) -> Iterator[
|
||||
tuple[
|
||||
CoreEngineProcManager | CoreEngineActorManager | None,
|
||||
DPCoordinator | None,
|
||||
EngineZmqAddresses,
|
||||
Queue | None,
|
||||
]
|
||||
]:
|
||||
) -> Iterator[CoreEngineLaunch]:
|
||||
"""Launch engine and DP coordinator processes as needed."""
|
||||
|
||||
parallel_config = vllm_config.parallel_config
|
||||
@@ -1119,7 +1128,9 @@ def launch_core_engines(
|
||||
log_stats=log_stats,
|
||||
)
|
||||
|
||||
yield engine_actor_manager, coordinator, addresses, tensor_queue
|
||||
yield CoreEngineLaunch(
|
||||
engine_actor_manager, coordinator, addresses, tensor_queue
|
||||
)
|
||||
return
|
||||
|
||||
if offline_mode:
|
||||
@@ -1188,30 +1199,27 @@ def launch_core_engines(
|
||||
else:
|
||||
local_engine_manager = None
|
||||
|
||||
yield local_engine_manager, coordinator, addresses, tensor_queue
|
||||
|
||||
# Now wait for engines to start.
|
||||
launch = CoreEngineLaunch(
|
||||
local_engine_manager, coordinator, addresses, tensor_queue
|
||||
)
|
||||
yield launch
|
||||
wait_for_engine_startup(
|
||||
handshake_socket,
|
||||
addresses,
|
||||
engines_to_handshake,
|
||||
parallel_config,
|
||||
dp_size > 1 and vllm_config.model_config.is_moe,
|
||||
vllm_config.cache_config,
|
||||
local_engine_manager,
|
||||
coordinator.proc if coordinator else None,
|
||||
launch,
|
||||
)
|
||||
|
||||
|
||||
def wait_for_engine_startup(
|
||||
handshake_socket: zmq.Socket,
|
||||
addresses: EngineZmqAddresses,
|
||||
core_engines: list[CoreEngine],
|
||||
parallel_config: ParallelConfig,
|
||||
coordinated_dp: bool,
|
||||
cache_config: CacheConfig,
|
||||
proc_manager: CoreEngineProcManager | None,
|
||||
coord_process: Process | None,
|
||||
launch: CoreEngineLaunch,
|
||||
):
|
||||
# Wait for engine core process(es) to send ready messages.
|
||||
local_count = parallel_config.data_parallel_size_local
|
||||
@@ -1226,11 +1234,21 @@ def wait_for_engine_startup(
|
||||
and not parallel_config.data_parallel_external_lb
|
||||
)
|
||||
|
||||
if proc_manager is not None:
|
||||
for sentinel in proc_manager.sentinels():
|
||||
# 1. Engine processes
|
||||
if isinstance(launch.engine_manager, CoreEngineProcManager):
|
||||
for sentinel in launch.engine_manager.sentinels():
|
||||
poller.register(sentinel, zmq.POLLIN)
|
||||
# 2. DP Coordinator process, if present
|
||||
coord_process = launch.coordinator.proc if launch.coordinator else None
|
||||
if coord_process is not None:
|
||||
poller.register(coord_process.sentinel, zmq.POLLIN)
|
||||
# 3. Watched frontend processes, if any
|
||||
frontend_process_by_fd: dict[int, FrontendProcess] = {}
|
||||
for proc in launch.watched_frontend_processes:
|
||||
fd = proc.sentinel if isinstance(proc.sentinel, int) else proc.sentinel.fileno()
|
||||
frontend_process_by_fd[fd] = proc
|
||||
poller.register(fd, zmq.POLLIN)
|
||||
|
||||
while any(conn_pending) or any(start_pending):
|
||||
events = poller.poll(STARTUP_POLL_PERIOD_MS)
|
||||
if not events:
|
||||
@@ -1246,14 +1264,34 @@ def wait_for_engine_startup(
|
||||
)
|
||||
continue
|
||||
if len(events) > 1 or events[0][0] != handshake_socket:
|
||||
# One of the local core processes exited.
|
||||
finished = proc_manager.finished_procs() if proc_manager else {}
|
||||
# One of the local core, coordinator, or watched frontend processes exited.
|
||||
if isinstance(launch.engine_manager, CoreEngineProcManager):
|
||||
finished = launch.engine_manager.finished_procs()
|
||||
else:
|
||||
finished = {}
|
||||
if coord_process is not None and coord_process.exitcode is not None:
|
||||
finished[coord_process.name] = coord_process.exitcode
|
||||
failed_frontend_procs = {
|
||||
proc.name: proc.exitcode
|
||||
for fd, proc in frontend_process_by_fd.items()
|
||||
if proc.exitcode is not None
|
||||
or any(event_fd == fd for event_fd, _ in events)
|
||||
}
|
||||
if failed_frontend_procs and not finished:
|
||||
raise RuntimeError(
|
||||
"Frontend process failed during engine core initialization. "
|
||||
"See root cause above. "
|
||||
f"Failed frontend proc(s): {failed_frontend_procs}"
|
||||
)
|
||||
raise RuntimeError(
|
||||
"Engine core initialization failed. "
|
||||
"See root cause above. "
|
||||
f"Failed core proc(s): {finished}"
|
||||
+ (
|
||||
f", failed frontend proc(s): {failed_frontend_procs}"
|
||||
if failed_frontend_procs
|
||||
else ""
|
||||
)
|
||||
)
|
||||
|
||||
# Receive HELLO and READY messages from the input socket.
|
||||
@@ -1293,7 +1331,7 @@ def wait_for_engine_startup(
|
||||
# Send init message with DP config info.
|
||||
init_message = msgspec.msgpack.encode(
|
||||
EngineHandshakeMetadata(
|
||||
addresses=addresses,
|
||||
addresses=launch.addresses,
|
||||
parallel_config={
|
||||
k: getattr(parallel_config, k)
|
||||
for k in (
|
||||
|
||||
Reference in New Issue
Block a user