[Bugfix] Exclude Ray DP from #42585's deferred port allocation (#43864)

Signed-off-by: Vadim Gimpelson <vadim.gimpelson@gmail.com>
This commit is contained in:
Vadim Gimpelson
2026-05-28 15:55:14 +00:00
committed by GitHub
parent c08ebebf30
commit 5d126dd155
2 changed files with 238 additions and 10 deletions
@@ -2,6 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os import os
import socket
import time
import uuid import uuid
from pathlib import Path from pathlib import Path
from types import SimpleNamespace from types import SimpleNamespace
@@ -9,9 +11,17 @@ from typing import Any
import pytest import pytest
import ray import ray
import zmq
from vllm.utils.network_utils import make_zmq_socket, split_zmq_path
from vllm.v1.engine.core import EngineCoreActorMixin from vllm.v1.engine.core import EngineCoreActorMixin
from vllm.v1.engine.utils import CoreEngineActorManager, EngineZmqAddresses from vllm.v1.engine.utils import (
CoreEngineActorManager,
EngineZmqAddresses,
get_engine_zmq_addresses,
launch_core_engines,
)
from vllm.v1.utils import APIServerProcessManager
class _StubEngineCoreActor(EngineCoreActorMixin): class _StubEngineCoreActor(EngineCoreActorMixin):
@@ -42,6 +52,48 @@ class _StubEngineCoreActor(EngineCoreActorMixin):
def get_nixl_side_channel_host(self) -> str | None: def get_nixl_side_channel_host(self) -> str | None:
return os.environ.get("VLLM_NIXL_SIDE_CHANNEL_HOST") return os.environ.get("VLLM_NIXL_SIDE_CHANNEL_HOST")
def get_addresses(self) -> tuple[list[str], list[str]]:
"""Return the addresses snapshot the actor was constructed with.
Used by the Ray-DP regression test to assert that no ``tcp://host:0``
placeholders were pickled into the actor at ``.remote()`` time.
"""
return list(self.addresses.inputs), list(self.addresses.outputs)
# Module-level stub worker for the Ray-DP regression test. Must be importable
# by ``multiprocessing.spawn`` (no closures, no nesting). Mirrors the worker
# in ``tests/entrypoints/test_api_server_process_manager.py``.
def _bind_and_report_worker(listen_address, sock, args, client_config):
"""Bind ROUTER/PULL with a kernel-assigned port, report the actual
endpoints back via ``actual_address_pipe``, then exit."""
ctx = zmq.Context()
try:
in_sock = make_zmq_socket(
ctx, client_config["input_address"], zmq.ROUTER, bind=True
)
out_sock = make_zmq_socket(
ctx, client_config["output_address"], zmq.PULL, bind=True
)
try:
pipe = client_config["actual_address_pipe"]
try:
pipe.send(
{
"input_address": in_sock.getsockopt(zmq.LAST_ENDPOINT).decode(),
"output_address": out_sock.getsockopt(
zmq.LAST_ENDPOINT
).decode(),
}
)
finally:
pipe.close()
finally:
in_sock.close(linger=0)
out_sock.close(linger=0)
finally:
ctx.term()
class _DummyExecutor: class _DummyExecutor:
pass pass
@@ -134,3 +186,172 @@ def test_driver_nixl_side_channel_host_does_not_leak_to_engine_core_actor(
else: else:
for pg in created_placement_groups: for pg in created_placement_groups:
ray.util.remove_placement_group(pg) ray.util.remove_placement_group(pg)
@pytest.fixture
def ray_context_dp2():
"""Ray context sized for two stub actors (each PG needs ~1 CPU)."""
started_ray = False
if not ray.is_initialized():
project_root = str(Path(__file__).resolve().parents[3])
ray.init(
num_cpus=4,
runtime_env={"env_vars": {"PYTHONPATH": project_root}},
log_to_driver=False,
)
started_ray = True
yield
if started_ray:
ray.shutdown()
def _make_vllm_config_ray_dp_multinode() -> SimpleNamespace:
"""Minimal vllm_config that drives the Ray-DP multi-API-server path:
``data_parallel_size != data_parallel_size_local`` forces TCP placeholders
(multi-node fan-out), and ``data_parallel_backend="ray"`` routes
``launch_core_engines`` through the Ray branch.
"""
return SimpleNamespace(
parallel_config=SimpleNamespace(
data_parallel_size=2,
data_parallel_size_local=1,
data_parallel_rank=0,
data_parallel_rank_local=None,
data_parallel_master_ip="127.0.0.1",
data_parallel_backend="ray",
data_parallel_rpc_port=29550,
local_engines_only=False,
enable_elastic_ep=False,
world_size=1,
),
model_config=SimpleNamespace(multimodal_config=None, is_moe=False),
cache_config=SimpleNamespace(),
needs_dp_coordinator=False,
kv_transfer_config=None,
# ``_apply_dp_identity_suffix`` reads and rewrites this.
instance_id="vllm-ray-dp-regression-test",
)
@pytest.mark.timeout(120)
@pytest.mark.usefixtures("ray_context_dp2")
def test_ray_dp_addresses_resolved_before_actor_creation(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Regression guard for the Ray-DP + multi-API-server hang from PR #42585.
``launch_core_engines`` Ray branch pickles ``addresses`` into each engine
actor at ``.remote()`` time, and ``EngineCoreActorMixin._perform_handshakes``
is a no-op, so the actor uses that pickled snapshot for the rest of its
life. If ``run_multi_api_server`` allocates ``addresses`` as
``tcp://host:0`` placeholders (its default), the actors hold placeholders
forever and DEALER-connect to port 0 — ZMQ ``connect`` is async and does
not raise, so the failure mode is a deterministic hang.
The Ray-DP carve-out in ``run_multi_api_server`` forces
``defer_api_server_ports=False`` when ``data_parallel_backend == "ray"``
so addresses are pre-allocated in the driver and Ray pickles real ports
into each actor. This test mirrors that call-site logic and asserts the
actors hold real (non-placeholder) endpoints. If the carve-out is
removed without an alternative fix, the test fails.
"""
created_placement_groups: list[Any] = []
def create_dp_placement_groups(vllm_config: Any):
pg1 = _make_cpu_placement_group()
pg2 = _make_cpu_placement_group()
created_placement_groups.extend([pg1, pg2])
return [pg1, pg2], [0, 0]
monkeypatch.setattr("vllm.v1.engine.core.EngineCoreActor", _StubEngineCoreActor)
monkeypatch.setattr(
CoreEngineActorManager,
"create_dp_placement_groups",
staticmethod(create_dp_placement_groups),
)
vllm_config = _make_vllm_config_ray_dp_multinode()
# Mirror run_multi_api_server's address-allocation logic. The Ray DP
# carve-out forces pre-allocation so the addresses pickled into engine
# actors at .remote() time are real, not ``tcp://host:0``.
is_ray_dp = vllm_config.parallel_config.data_parallel_backend == "ray"
addresses = get_engine_zmq_addresses(
vllm_config,
num_api_servers=2,
defer_api_server_ports=not is_ray_dp,
)
sock = socket.socket()
engine_manager: CoreEngineActorManager | None = None
actor_snapshots: list[tuple[list[str], list[str]]] = []
api_server_manager: APIServerProcessManager | None = None
try:
# Ray actors are spawned here, pickling ``addresses`` into each one.
with launch_core_engines(
vllm_config,
executor_class=_DummyExecutor,
log_stats=False,
addresses=addresses,
num_api_servers=2,
) as (
engine_manager,
_coordinator,
_addresses_out,
_tensor_queue,
):
assert isinstance(engine_manager, CoreEngineActorManager)
# API-server children bind to the pre-allocated ports.
api_server_manager = APIServerProcessManager(
listen_address="tcp://127.0.0.1:0",
sock=sock,
args="test_args",
num_servers=2,
input_addresses=addresses.inputs,
output_addresses=addresses.outputs,
target_server_fn=_bind_and_report_worker,
)
# run_multi_api_server skips ``gather_actual_addresses`` for
# Ray DP (addresses are already real). Mirror that.
if not is_ray_dp:
actual_inputs, actual_outputs = (
api_server_manager.gather_actual_addresses(timeout=15.0)
)
addresses.inputs = actual_inputs
addresses.outputs = actual_outputs
# Snapshot what each Ray actor actually holds.
actors = (
engine_manager.local_engine_actors + engine_manager.remote_engine_actors
)
actor_snapshots = ray.get(
[actor.get_addresses.remote() for actor in actors]
)
finally:
if api_server_manager is not None:
api_server_manager.shutdown()
time.sleep(0.2)
sock.close()
if engine_manager is not None:
engine_manager.shutdown()
else:
for pg in created_placement_groups:
ray.util.remove_placement_group(pg)
# Every Ray actor must hold real, non-placeholder addresses.
assert actor_snapshots, "expected at least one Ray actor to be created"
for actor_inputs, actor_outputs in actor_snapshots:
for url in actor_inputs + actor_outputs:
scheme, _host, port = split_zmq_path(url)
assert scheme == "tcp", url
assert port and int(port) > 0, (
f"Ray actor was pickled with placeholder address {url!r}; "
"``run_multi_api_server`` must pre-allocate ports for the "
"Ray DP backend so the actors hold real endpoints by the "
"time they DEALER-connect. See PR #42585 / Ray-DP "
"multi-API-server regression."
)
+16 -9
View File
@@ -308,13 +308,16 @@ def run_multi_api_server(args: argparse.Namespace):
from vllm.v1.engine.utils import get_engine_zmq_addresses from vllm.v1.engine.utils import get_engine_zmq_addresses
# Per-API-server ports are picked by the kernel at each child's bind() # Defer port allocation to the child's bind() to avoid TOCTOU, except
# to avoid parent-probe vs child-bind TOCTOU; Rust front-end opts out # for Rust front-end and Ray DP, which can't see the post-bind rebind
# because it has no port-report-back channel. # (CLI-arg subprocess / pickled-into-actor snapshot respectively) and
# so pre-allocate driver-side -- reintroducing the original race only
# there.
is_ray_dp = parallel_config.data_parallel_backend == "ray"
addresses = get_engine_zmq_addresses( addresses = get_engine_zmq_addresses(
vllm_config, vllm_config,
num_api_servers, num_api_servers,
defer_api_server_ports=not rust_frontend_path, defer_api_server_ports=not (rust_frontend_path or is_ray_dp),
) )
with launch_core_engines( with launch_core_engines(
@@ -348,11 +351,15 @@ def run_multi_api_server(args: argparse.Namespace):
tensor_queue=tensor_queue, tensor_queue=tensor_queue,
) )
# Forward each child's bound endpoints to the engine handshake if not is_ray_dp:
# (runs on ``with`` exit). # Forward each child's bound endpoints to the engine handshake
actual_inputs, actual_outputs = api_server_manager.gather_actual_addresses() # (runs on ``with`` exit). Skipped for Ray DP, where addresses
addresses.inputs = actual_inputs # are pre-allocated above and Ray actors already hold them.
addresses.outputs = actual_outputs actual_inputs, actual_outputs = (
api_server_manager.gather_actual_addresses()
)
addresses.inputs = actual_inputs
addresses.outputs = actual_outputs
# Wait for API servers. # Wait for API servers.
try: try: