diff --git a/tests/v1/engine/test_core_engine_actor_manager.py b/tests/v1/engine/test_core_engine_actor_manager.py index a7a25ef0d6..69f52e2a7f 100644 --- a/tests/v1/engine/test_core_engine_actor_manager.py +++ b/tests/v1/engine/test_core_engine_actor_manager.py @@ -308,12 +308,8 @@ def test_ray_dp_addresses_resolved_before_actor_creation( executor_class=_DummyExecutor, log_stats=False, addresses=addresses, - ) as ( - engine_manager, - _coordinator, - _addresses_out, - _tensor_queue, - ): + ) as engine_launch: + engine_manager = engine_launch.engine_manager assert isinstance(engine_manager, CoreEngineActorManager) # API-server children bind to the pre-allocated ports. diff --git a/tests/v1/engine/test_startup_watch_processes.py b/tests/v1/engine/test_startup_watch_processes.py new file mode 100644 index 0000000000..b7b4b8a5a6 --- /dev/null +++ b/tests/v1/engine/test_startup_watch_processes.py @@ -0,0 +1,68 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from multiprocessing import connection +from types import SimpleNamespace + +import pytest +import zmq + +from vllm.v1.engine.utils import ( + CoreEngine, + CoreEngineLaunch, + EngineZmqAddresses, + wait_for_engine_startup, +) + +pytestmark = pytest.mark.skip_global_cleanup + + +class _FinishedProcess: + name = "RustFrontend" + + def __init__(self, sentinel): + self.sentinel = sentinel + + @property + def exitcode(self): + return 1 + + +def test_wait_for_engine_startup_reports_watched_process_exit(): + ctx = zmq.Context() + handshake_socket = ctx.socket(zmq.ROUTER) + recv, send = connection.Pipe(duplex=False) + send.close() + + parallel_config = SimpleNamespace( + data_parallel_size_local=1, + data_parallel_hybrid_lb=False, + data_parallel_external_lb=False, + ) + + try: + launch = CoreEngineLaunch( + engine_manager=None, + coordinator=None, + addresses=EngineZmqAddresses(inputs=[], outputs=[]), + tensor_queue=None, + ) + launch.watched_frontend_processes = [_FinishedProcess(recv)] + with pytest.raises(RuntimeError) as exc_info: + wait_for_engine_startup( + handshake_socket, + [CoreEngine()], + parallel_config, # type: ignore[arg-type] + coordinated_dp=False, + cache_config=None, # type: ignore[arg-type] + launch=launch, + ) + finally: + recv.close() + handshake_socket.close(linger=0) + ctx.term() + + assert "Frontend process failed during engine core initialization" in str( + exc_info.value + ) + assert "Failed frontend proc(s): {'RustFrontend': 1}" in str(exc_info.value) diff --git a/vllm/entrypoints/cli/serve.py b/vllm/entrypoints/cli/serve.py index e8f72ab230..f90a4f8c1d 100644 --- a/vllm/entrypoints/cli/serve.py +++ b/vllm/entrypoints/cli/serve.py @@ -326,12 +326,12 @@ def run_multi_api_server(args: argparse.Namespace): defer_api_server_ports=not (rust_frontend_path or is_ray_dp), ) - with launch_core_engines(vllm_config, executor_class, log_stats, addresses) as ( - local_engine_manager, - coordinator, - addresses, - tensor_queue, - ): + with launch_core_engines( + vllm_config, executor_class, log_stats, addresses + ) as engine_launch: + local_engine_manager = engine_launch.engine_manager + coordinator = engine_launch.coordinator + addresses = engine_launch.addresses stats_update_address = ( coordinator.get_stats_publish_address() if coordinator else None ) @@ -364,7 +364,7 @@ def run_multi_api_server(args: argparse.Namespace): input_addresses=addresses.inputs, output_addresses=addresses.outputs, stats_update_address=stats_update_address, - tensor_queue=tensor_queue, + tensor_queue=engine_launch.tensor_queue, ) if not is_ray_dp: @@ -377,6 +377,11 @@ def run_multi_api_server(args: argparse.Namespace): addresses.inputs = actual_inputs addresses.outputs = actual_outputs + # Set frontend processes to watch during engine startup. + # If any of these processes exit before the engines are up, the engine startup + # will be aborted with an error. + engine_launch.watched_frontend_processes = api_server_manager.processes + # Wait for API servers. try: wait_for_completion_or_failure( diff --git a/vllm/v1/engine/core_client.py b/vllm/v1/engine/core_client.py index febaa10ce6..d692a13d65 100644 --- a/vllm/v1/engine/core_client.py +++ b/vllm/v1/engine/core_client.py @@ -608,9 +608,12 @@ class MPClient(EngineCoreClient): with launch_core_engines( vllm_config, executor_class, log_stats, addresses - ) as (engine_manager, coordinator, addresses, tensor_queue): - self.resources.coordinator = coordinator - self.resources.engine_manager = engine_manager + ) as engine_launch: + self.resources.coordinator = engine_launch.coordinator + self.resources.engine_manager = engine_launch.engine_manager + coordinator = engine_launch.coordinator + addresses = engine_launch.addresses + tensor_queue = engine_launch.tensor_queue self.stats_update_address = addresses.frontend_stats_publish_address if coordinator is not None: diff --git a/vllm/v1/engine/utils.py b/vllm/v1/engine/utils.py index af62ed3192..be440e9741 100644 --- a/vllm/v1/engine/utils.py +++ b/vllm/v1/engine/utils.py @@ -5,10 +5,10 @@ import contextlib import os import threading import weakref -from collections.abc import Callable, Iterator +from collections.abc import Callable, Iterator, Sequence from dataclasses import dataclass from enum import Enum, auto -from multiprocessing import Process, connection +from multiprocessing import connection from multiprocessing.process import BaseProcess from multiprocessing.queues import Queue from typing import TYPE_CHECKING, cast @@ -32,7 +32,7 @@ from vllm.utils.system_utils import get_mp_context from vllm.v1.engine.coordinator import DPCoordinator from vllm.v1.executor import Executor from vllm.v1.executor.ray_utils import WORKER_SPECIFIC_ENV_VARS -from vllm.v1.utils import get_engine_client_zmq_addr, shutdown +from vllm.v1.utils import _SubprocessWrapper, get_engine_client_zmq_addr, shutdown if TYPE_CHECKING: from ray.util.placement_group import PlacementGroup @@ -1050,20 +1050,29 @@ def get_engine_zmq_addresses( ) +FrontendProcess = BaseProcess | _SubprocessWrapper + + +@dataclass +class CoreEngineLaunch: + """Resources and startup barrier for launched engine processes.""" + + engine_manager: CoreEngineProcManager | CoreEngineActorManager | None + coordinator: DPCoordinator | None + addresses: EngineZmqAddresses + tensor_queue: Queue | None + # Frontend processes to watch during engine startup; may be assigned by + # the caller before the startup barrier runs on context manager exit. + watched_frontend_processes: Sequence[FrontendProcess] = () + + @contextlib.contextmanager def launch_core_engines( vllm_config: VllmConfig, executor_class: type[Executor], log_stats: bool, addresses: EngineZmqAddresses, -) -> Iterator[ - tuple[ - CoreEngineProcManager | CoreEngineActorManager | None, - DPCoordinator | None, - EngineZmqAddresses, - Queue | None, - ] -]: +) -> Iterator[CoreEngineLaunch]: """Launch engine and DP coordinator processes as needed.""" parallel_config = vllm_config.parallel_config @@ -1119,7 +1128,9 @@ def launch_core_engines( log_stats=log_stats, ) - yield engine_actor_manager, coordinator, addresses, tensor_queue + yield CoreEngineLaunch( + engine_actor_manager, coordinator, addresses, tensor_queue + ) return if offline_mode: @@ -1188,30 +1199,27 @@ def launch_core_engines( else: local_engine_manager = None - yield local_engine_manager, coordinator, addresses, tensor_queue - - # Now wait for engines to start. + launch = CoreEngineLaunch( + local_engine_manager, coordinator, addresses, tensor_queue + ) + yield launch wait_for_engine_startup( handshake_socket, - addresses, engines_to_handshake, parallel_config, dp_size > 1 and vllm_config.model_config.is_moe, vllm_config.cache_config, - local_engine_manager, - coordinator.proc if coordinator else None, + launch, ) def wait_for_engine_startup( handshake_socket: zmq.Socket, - addresses: EngineZmqAddresses, core_engines: list[CoreEngine], parallel_config: ParallelConfig, coordinated_dp: bool, cache_config: CacheConfig, - proc_manager: CoreEngineProcManager | None, - coord_process: Process | None, + launch: CoreEngineLaunch, ): # Wait for engine core process(es) to send ready messages. local_count = parallel_config.data_parallel_size_local @@ -1226,11 +1234,21 @@ def wait_for_engine_startup( and not parallel_config.data_parallel_external_lb ) - if proc_manager is not None: - for sentinel in proc_manager.sentinels(): + # 1. Engine processes + if isinstance(launch.engine_manager, CoreEngineProcManager): + for sentinel in launch.engine_manager.sentinels(): poller.register(sentinel, zmq.POLLIN) + # 2. DP Coordinator process, if present + coord_process = launch.coordinator.proc if launch.coordinator else None if coord_process is not None: poller.register(coord_process.sentinel, zmq.POLLIN) + # 3. Watched frontend processes, if any + frontend_process_by_fd: dict[int, FrontendProcess] = {} + for proc in launch.watched_frontend_processes: + fd = proc.sentinel if isinstance(proc.sentinel, int) else proc.sentinel.fileno() + frontend_process_by_fd[fd] = proc + poller.register(fd, zmq.POLLIN) + while any(conn_pending) or any(start_pending): events = poller.poll(STARTUP_POLL_PERIOD_MS) if not events: @@ -1246,14 +1264,34 @@ def wait_for_engine_startup( ) continue if len(events) > 1 or events[0][0] != handshake_socket: - # One of the local core processes exited. - finished = proc_manager.finished_procs() if proc_manager else {} + # One of the local core, coordinator, or watched frontend processes exited. + if isinstance(launch.engine_manager, CoreEngineProcManager): + finished = launch.engine_manager.finished_procs() + else: + finished = {} if coord_process is not None and coord_process.exitcode is not None: finished[coord_process.name] = coord_process.exitcode + failed_frontend_procs = { + proc.name: proc.exitcode + for fd, proc in frontend_process_by_fd.items() + if proc.exitcode is not None + or any(event_fd == fd for event_fd, _ in events) + } + if failed_frontend_procs and not finished: + raise RuntimeError( + "Frontend process failed during engine core initialization. " + "See root cause above. " + f"Failed frontend proc(s): {failed_frontend_procs}" + ) raise RuntimeError( "Engine core initialization failed. " "See root cause above. " f"Failed core proc(s): {finished}" + + ( + f", failed frontend proc(s): {failed_frontend_procs}" + if failed_frontend_procs + else "" + ) ) # Receive HELLO and READY messages from the input socket. @@ -1293,7 +1331,7 @@ def wait_for_engine_startup( # Send init message with DP config info. init_message = msgspec.msgpack.encode( EngineHandshakeMetadata( - addresses=addresses, + addresses=launch.addresses, parallel_config={ k: getattr(parallel_config, k) for k in (