diff --git a/tests/v1/engine/test_core_engine_actor_manager.py b/tests/v1/engine/test_core_engine_actor_manager.py index 195ddda05a..f60f8c94e7 100644 --- a/tests/v1/engine/test_core_engine_actor_manager.py +++ b/tests/v1/engine/test_core_engine_actor_manager.py @@ -2,6 +2,8 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import os +import socket +import time import uuid from pathlib import Path from types import SimpleNamespace @@ -9,9 +11,17 @@ from typing import Any import pytest import ray +import zmq +from vllm.utils.network_utils import make_zmq_socket, split_zmq_path from vllm.v1.engine.core import EngineCoreActorMixin -from vllm.v1.engine.utils import CoreEngineActorManager, EngineZmqAddresses +from vllm.v1.engine.utils import ( + CoreEngineActorManager, + EngineZmqAddresses, + get_engine_zmq_addresses, + launch_core_engines, +) +from vllm.v1.utils import APIServerProcessManager class _StubEngineCoreActor(EngineCoreActorMixin): @@ -42,6 +52,48 @@ class _StubEngineCoreActor(EngineCoreActorMixin): def get_nixl_side_channel_host(self) -> str | None: return os.environ.get("VLLM_NIXL_SIDE_CHANNEL_HOST") + def get_addresses(self) -> tuple[list[str], list[str]]: + """Return the addresses snapshot the actor was constructed with. + + Used by the Ray-DP regression test to assert that no ``tcp://host:0`` + placeholders were pickled into the actor at ``.remote()`` time. + """ + return list(self.addresses.inputs), list(self.addresses.outputs) + + +# Module-level stub worker for the Ray-DP regression test. Must be importable +# by ``multiprocessing.spawn`` (no closures, no nesting). Mirrors the worker +# in ``tests/entrypoints/test_api_server_process_manager.py``. +def _bind_and_report_worker(listen_address, sock, args, client_config): + """Bind ROUTER/PULL with a kernel-assigned port, report the actual + endpoints back via ``actual_address_pipe``, then exit.""" + ctx = zmq.Context() + try: + in_sock = make_zmq_socket( + ctx, client_config["input_address"], zmq.ROUTER, bind=True + ) + out_sock = make_zmq_socket( + ctx, client_config["output_address"], zmq.PULL, bind=True + ) + try: + pipe = client_config["actual_address_pipe"] + try: + pipe.send( + { + "input_address": in_sock.getsockopt(zmq.LAST_ENDPOINT).decode(), + "output_address": out_sock.getsockopt( + zmq.LAST_ENDPOINT + ).decode(), + } + ) + finally: + pipe.close() + finally: + in_sock.close(linger=0) + out_sock.close(linger=0) + finally: + ctx.term() + class _DummyExecutor: pass @@ -134,3 +186,172 @@ def test_driver_nixl_side_channel_host_does_not_leak_to_engine_core_actor( else: for pg in created_placement_groups: ray.util.remove_placement_group(pg) + + +@pytest.fixture +def ray_context_dp2(): + """Ray context sized for two stub actors (each PG needs ~1 CPU).""" + started_ray = False + if not ray.is_initialized(): + project_root = str(Path(__file__).resolve().parents[3]) + ray.init( + num_cpus=4, + runtime_env={"env_vars": {"PYTHONPATH": project_root}}, + log_to_driver=False, + ) + started_ray = True + + yield + + if started_ray: + ray.shutdown() + + +def _make_vllm_config_ray_dp_multinode() -> SimpleNamespace: + """Minimal vllm_config that drives the Ray-DP multi-API-server path: + ``data_parallel_size != data_parallel_size_local`` forces TCP placeholders + (multi-node fan-out), and ``data_parallel_backend="ray"`` routes + ``launch_core_engines`` through the Ray branch. + """ + return SimpleNamespace( + parallel_config=SimpleNamespace( + data_parallel_size=2, + data_parallel_size_local=1, + data_parallel_rank=0, + data_parallel_rank_local=None, + data_parallel_master_ip="127.0.0.1", + data_parallel_backend="ray", + data_parallel_rpc_port=29550, + local_engines_only=False, + enable_elastic_ep=False, + world_size=1, + ), + model_config=SimpleNamespace(multimodal_config=None, is_moe=False), + cache_config=SimpleNamespace(), + needs_dp_coordinator=False, + kv_transfer_config=None, + # ``_apply_dp_identity_suffix`` reads and rewrites this. + instance_id="vllm-ray-dp-regression-test", + ) + + +@pytest.mark.timeout(120) +@pytest.mark.usefixtures("ray_context_dp2") +def test_ray_dp_addresses_resolved_before_actor_creation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Regression guard for the Ray-DP + multi-API-server hang from PR #42585. + + ``launch_core_engines`` Ray branch pickles ``addresses`` into each engine + actor at ``.remote()`` time, and ``EngineCoreActorMixin._perform_handshakes`` + is a no-op, so the actor uses that pickled snapshot for the rest of its + life. If ``run_multi_api_server`` allocates ``addresses`` as + ``tcp://host:0`` placeholders (its default), the actors hold placeholders + forever and DEALER-connect to port 0 — ZMQ ``connect`` is async and does + not raise, so the failure mode is a deterministic hang. + + The Ray-DP carve-out in ``run_multi_api_server`` forces + ``defer_api_server_ports=False`` when ``data_parallel_backend == "ray"`` + so addresses are pre-allocated in the driver and Ray pickles real ports + into each actor. This test mirrors that call-site logic and asserts the + actors hold real (non-placeholder) endpoints. If the carve-out is + removed without an alternative fix, the test fails. + """ + created_placement_groups: list[Any] = [] + + def create_dp_placement_groups(vllm_config: Any): + pg1 = _make_cpu_placement_group() + pg2 = _make_cpu_placement_group() + created_placement_groups.extend([pg1, pg2]) + return [pg1, pg2], [0, 0] + + monkeypatch.setattr("vllm.v1.engine.core.EngineCoreActor", _StubEngineCoreActor) + monkeypatch.setattr( + CoreEngineActorManager, + "create_dp_placement_groups", + staticmethod(create_dp_placement_groups), + ) + + vllm_config = _make_vllm_config_ray_dp_multinode() + + # Mirror run_multi_api_server's address-allocation logic. The Ray DP + # carve-out forces pre-allocation so the addresses pickled into engine + # actors at .remote() time are real, not ``tcp://host:0``. + is_ray_dp = vllm_config.parallel_config.data_parallel_backend == "ray" + addresses = get_engine_zmq_addresses( + vllm_config, + num_api_servers=2, + defer_api_server_ports=not is_ray_dp, + ) + + sock = socket.socket() + engine_manager: CoreEngineActorManager | None = None + actor_snapshots: list[tuple[list[str], list[str]]] = [] + api_server_manager: APIServerProcessManager | None = None + try: + # Ray actors are spawned here, pickling ``addresses`` into each one. + with launch_core_engines( + vllm_config, + executor_class=_DummyExecutor, + log_stats=False, + addresses=addresses, + num_api_servers=2, + ) as ( + engine_manager, + _coordinator, + _addresses_out, + _tensor_queue, + ): + assert isinstance(engine_manager, CoreEngineActorManager) + + # API-server children bind to the pre-allocated ports. + api_server_manager = APIServerProcessManager( + listen_address="tcp://127.0.0.1:0", + sock=sock, + args="test_args", + num_servers=2, + input_addresses=addresses.inputs, + output_addresses=addresses.outputs, + target_server_fn=_bind_and_report_worker, + ) + + # run_multi_api_server skips ``gather_actual_addresses`` for + # Ray DP (addresses are already real). Mirror that. + if not is_ray_dp: + actual_inputs, actual_outputs = ( + api_server_manager.gather_actual_addresses(timeout=15.0) + ) + addresses.inputs = actual_inputs + addresses.outputs = actual_outputs + + # Snapshot what each Ray actor actually holds. + actors = ( + engine_manager.local_engine_actors + engine_manager.remote_engine_actors + ) + actor_snapshots = ray.get( + [actor.get_addresses.remote() for actor in actors] + ) + finally: + if api_server_manager is not None: + api_server_manager.shutdown() + time.sleep(0.2) + sock.close() + if engine_manager is not None: + engine_manager.shutdown() + else: + for pg in created_placement_groups: + ray.util.remove_placement_group(pg) + + # Every Ray actor must hold real, non-placeholder addresses. + assert actor_snapshots, "expected at least one Ray actor to be created" + for actor_inputs, actor_outputs in actor_snapshots: + for url in actor_inputs + actor_outputs: + scheme, _host, port = split_zmq_path(url) + assert scheme == "tcp", url + assert port and int(port) > 0, ( + f"Ray actor was pickled with placeholder address {url!r}; " + "``run_multi_api_server`` must pre-allocate ports for the " + "Ray DP backend so the actors hold real endpoints by the " + "time they DEALER-connect. See PR #42585 / Ray-DP " + "multi-API-server regression." + ) diff --git a/vllm/entrypoints/cli/serve.py b/vllm/entrypoints/cli/serve.py index cbd3a44724..ea4bf1b62d 100644 --- a/vllm/entrypoints/cli/serve.py +++ b/vllm/entrypoints/cli/serve.py @@ -308,13 +308,16 @@ def run_multi_api_server(args: argparse.Namespace): from vllm.v1.engine.utils import get_engine_zmq_addresses - # Per-API-server ports are picked by the kernel at each child's bind() - # to avoid parent-probe vs child-bind TOCTOU; Rust front-end opts out - # because it has no port-report-back channel. + # Defer port allocation to the child's bind() to avoid TOCTOU, except + # for Rust front-end and Ray DP, which can't see the post-bind rebind + # (CLI-arg subprocess / pickled-into-actor snapshot respectively) and + # so pre-allocate driver-side -- reintroducing the original race only + # there. + is_ray_dp = parallel_config.data_parallel_backend == "ray" addresses = get_engine_zmq_addresses( vllm_config, num_api_servers, - defer_api_server_ports=not rust_frontend_path, + defer_api_server_ports=not (rust_frontend_path or is_ray_dp), ) with launch_core_engines( @@ -348,11 +351,15 @@ def run_multi_api_server(args: argparse.Namespace): tensor_queue=tensor_queue, ) - # Forward each child's bound endpoints to the engine handshake - # (runs on ``with`` exit). - actual_inputs, actual_outputs = api_server_manager.gather_actual_addresses() - addresses.inputs = actual_inputs - addresses.outputs = actual_outputs + if not is_ray_dp: + # Forward each child's bound endpoints to the engine handshake + # (runs on ``with`` exit). Skipped for Ray DP, where addresses + # are pre-allocated above and Ray actors already hold them. + actual_inputs, actual_outputs = ( + api_server_manager.gather_actual_addresses() + ) + addresses.inputs = actual_inputs + addresses.outputs = actual_outputs # Wait for API servers. try: