Signed-off-by: Vadim Gimpelson <vadim.gimpelson@gmail.com>
This commit is contained in:
@@ -2,6 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import os
|
||||
import socket
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
@@ -9,9 +11,17 @@ from typing import Any
|
||||
|
||||
import pytest
|
||||
import ray
|
||||
import zmq
|
||||
|
||||
from vllm.utils.network_utils import make_zmq_socket, split_zmq_path
|
||||
from vllm.v1.engine.core import EngineCoreActorMixin
|
||||
from vllm.v1.engine.utils import CoreEngineActorManager, EngineZmqAddresses
|
||||
from vllm.v1.engine.utils import (
|
||||
CoreEngineActorManager,
|
||||
EngineZmqAddresses,
|
||||
get_engine_zmq_addresses,
|
||||
launch_core_engines,
|
||||
)
|
||||
from vllm.v1.utils import APIServerProcessManager
|
||||
|
||||
|
||||
class _StubEngineCoreActor(EngineCoreActorMixin):
|
||||
@@ -42,6 +52,48 @@ class _StubEngineCoreActor(EngineCoreActorMixin):
|
||||
def get_nixl_side_channel_host(self) -> str | None:
|
||||
return os.environ.get("VLLM_NIXL_SIDE_CHANNEL_HOST")
|
||||
|
||||
def get_addresses(self) -> tuple[list[str], list[str]]:
|
||||
"""Return the addresses snapshot the actor was constructed with.
|
||||
|
||||
Used by the Ray-DP regression test to assert that no ``tcp://host:0``
|
||||
placeholders were pickled into the actor at ``.remote()`` time.
|
||||
"""
|
||||
return list(self.addresses.inputs), list(self.addresses.outputs)
|
||||
|
||||
|
||||
# Module-level stub worker for the Ray-DP regression test. Must be importable
|
||||
# by ``multiprocessing.spawn`` (no closures, no nesting). Mirrors the worker
|
||||
# in ``tests/entrypoints/test_api_server_process_manager.py``.
|
||||
def _bind_and_report_worker(listen_address, sock, args, client_config):
|
||||
"""Bind ROUTER/PULL with a kernel-assigned port, report the actual
|
||||
endpoints back via ``actual_address_pipe``, then exit."""
|
||||
ctx = zmq.Context()
|
||||
try:
|
||||
in_sock = make_zmq_socket(
|
||||
ctx, client_config["input_address"], zmq.ROUTER, bind=True
|
||||
)
|
||||
out_sock = make_zmq_socket(
|
||||
ctx, client_config["output_address"], zmq.PULL, bind=True
|
||||
)
|
||||
try:
|
||||
pipe = client_config["actual_address_pipe"]
|
||||
try:
|
||||
pipe.send(
|
||||
{
|
||||
"input_address": in_sock.getsockopt(zmq.LAST_ENDPOINT).decode(),
|
||||
"output_address": out_sock.getsockopt(
|
||||
zmq.LAST_ENDPOINT
|
||||
).decode(),
|
||||
}
|
||||
)
|
||||
finally:
|
||||
pipe.close()
|
||||
finally:
|
||||
in_sock.close(linger=0)
|
||||
out_sock.close(linger=0)
|
||||
finally:
|
||||
ctx.term()
|
||||
|
||||
|
||||
class _DummyExecutor:
|
||||
pass
|
||||
@@ -134,3 +186,172 @@ def test_driver_nixl_side_channel_host_does_not_leak_to_engine_core_actor(
|
||||
else:
|
||||
for pg in created_placement_groups:
|
||||
ray.util.remove_placement_group(pg)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def ray_context_dp2():
|
||||
"""Ray context sized for two stub actors (each PG needs ~1 CPU)."""
|
||||
started_ray = False
|
||||
if not ray.is_initialized():
|
||||
project_root = str(Path(__file__).resolve().parents[3])
|
||||
ray.init(
|
||||
num_cpus=4,
|
||||
runtime_env={"env_vars": {"PYTHONPATH": project_root}},
|
||||
log_to_driver=False,
|
||||
)
|
||||
started_ray = True
|
||||
|
||||
yield
|
||||
|
||||
if started_ray:
|
||||
ray.shutdown()
|
||||
|
||||
|
||||
def _make_vllm_config_ray_dp_multinode() -> SimpleNamespace:
|
||||
"""Minimal vllm_config that drives the Ray-DP multi-API-server path:
|
||||
``data_parallel_size != data_parallel_size_local`` forces TCP placeholders
|
||||
(multi-node fan-out), and ``data_parallel_backend="ray"`` routes
|
||||
``launch_core_engines`` through the Ray branch.
|
||||
"""
|
||||
return SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(
|
||||
data_parallel_size=2,
|
||||
data_parallel_size_local=1,
|
||||
data_parallel_rank=0,
|
||||
data_parallel_rank_local=None,
|
||||
data_parallel_master_ip="127.0.0.1",
|
||||
data_parallel_backend="ray",
|
||||
data_parallel_rpc_port=29550,
|
||||
local_engines_only=False,
|
||||
enable_elastic_ep=False,
|
||||
world_size=1,
|
||||
),
|
||||
model_config=SimpleNamespace(multimodal_config=None, is_moe=False),
|
||||
cache_config=SimpleNamespace(),
|
||||
needs_dp_coordinator=False,
|
||||
kv_transfer_config=None,
|
||||
# ``_apply_dp_identity_suffix`` reads and rewrites this.
|
||||
instance_id="vllm-ray-dp-regression-test",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.timeout(120)
|
||||
@pytest.mark.usefixtures("ray_context_dp2")
|
||||
def test_ray_dp_addresses_resolved_before_actor_creation(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Regression guard for the Ray-DP + multi-API-server hang from PR #42585.
|
||||
|
||||
``launch_core_engines`` Ray branch pickles ``addresses`` into each engine
|
||||
actor at ``.remote()`` time, and ``EngineCoreActorMixin._perform_handshakes``
|
||||
is a no-op, so the actor uses that pickled snapshot for the rest of its
|
||||
life. If ``run_multi_api_server`` allocates ``addresses`` as
|
||||
``tcp://host:0`` placeholders (its default), the actors hold placeholders
|
||||
forever and DEALER-connect to port 0 — ZMQ ``connect`` is async and does
|
||||
not raise, so the failure mode is a deterministic hang.
|
||||
|
||||
The Ray-DP carve-out in ``run_multi_api_server`` forces
|
||||
``defer_api_server_ports=False`` when ``data_parallel_backend == "ray"``
|
||||
so addresses are pre-allocated in the driver and Ray pickles real ports
|
||||
into each actor. This test mirrors that call-site logic and asserts the
|
||||
actors hold real (non-placeholder) endpoints. If the carve-out is
|
||||
removed without an alternative fix, the test fails.
|
||||
"""
|
||||
created_placement_groups: list[Any] = []
|
||||
|
||||
def create_dp_placement_groups(vllm_config: Any):
|
||||
pg1 = _make_cpu_placement_group()
|
||||
pg2 = _make_cpu_placement_group()
|
||||
created_placement_groups.extend([pg1, pg2])
|
||||
return [pg1, pg2], [0, 0]
|
||||
|
||||
monkeypatch.setattr("vllm.v1.engine.core.EngineCoreActor", _StubEngineCoreActor)
|
||||
monkeypatch.setattr(
|
||||
CoreEngineActorManager,
|
||||
"create_dp_placement_groups",
|
||||
staticmethod(create_dp_placement_groups),
|
||||
)
|
||||
|
||||
vllm_config = _make_vllm_config_ray_dp_multinode()
|
||||
|
||||
# Mirror run_multi_api_server's address-allocation logic. The Ray DP
|
||||
# carve-out forces pre-allocation so the addresses pickled into engine
|
||||
# actors at .remote() time are real, not ``tcp://host:0``.
|
||||
is_ray_dp = vllm_config.parallel_config.data_parallel_backend == "ray"
|
||||
addresses = get_engine_zmq_addresses(
|
||||
vllm_config,
|
||||
num_api_servers=2,
|
||||
defer_api_server_ports=not is_ray_dp,
|
||||
)
|
||||
|
||||
sock = socket.socket()
|
||||
engine_manager: CoreEngineActorManager | None = None
|
||||
actor_snapshots: list[tuple[list[str], list[str]]] = []
|
||||
api_server_manager: APIServerProcessManager | None = None
|
||||
try:
|
||||
# Ray actors are spawned here, pickling ``addresses`` into each one.
|
||||
with launch_core_engines(
|
||||
vllm_config,
|
||||
executor_class=_DummyExecutor,
|
||||
log_stats=False,
|
||||
addresses=addresses,
|
||||
num_api_servers=2,
|
||||
) as (
|
||||
engine_manager,
|
||||
_coordinator,
|
||||
_addresses_out,
|
||||
_tensor_queue,
|
||||
):
|
||||
assert isinstance(engine_manager, CoreEngineActorManager)
|
||||
|
||||
# API-server children bind to the pre-allocated ports.
|
||||
api_server_manager = APIServerProcessManager(
|
||||
listen_address="tcp://127.0.0.1:0",
|
||||
sock=sock,
|
||||
args="test_args",
|
||||
num_servers=2,
|
||||
input_addresses=addresses.inputs,
|
||||
output_addresses=addresses.outputs,
|
||||
target_server_fn=_bind_and_report_worker,
|
||||
)
|
||||
|
||||
# run_multi_api_server skips ``gather_actual_addresses`` for
|
||||
# Ray DP (addresses are already real). Mirror that.
|
||||
if not is_ray_dp:
|
||||
actual_inputs, actual_outputs = (
|
||||
api_server_manager.gather_actual_addresses(timeout=15.0)
|
||||
)
|
||||
addresses.inputs = actual_inputs
|
||||
addresses.outputs = actual_outputs
|
||||
|
||||
# Snapshot what each Ray actor actually holds.
|
||||
actors = (
|
||||
engine_manager.local_engine_actors + engine_manager.remote_engine_actors
|
||||
)
|
||||
actor_snapshots = ray.get(
|
||||
[actor.get_addresses.remote() for actor in actors]
|
||||
)
|
||||
finally:
|
||||
if api_server_manager is not None:
|
||||
api_server_manager.shutdown()
|
||||
time.sleep(0.2)
|
||||
sock.close()
|
||||
if engine_manager is not None:
|
||||
engine_manager.shutdown()
|
||||
else:
|
||||
for pg in created_placement_groups:
|
||||
ray.util.remove_placement_group(pg)
|
||||
|
||||
# Every Ray actor must hold real, non-placeholder addresses.
|
||||
assert actor_snapshots, "expected at least one Ray actor to be created"
|
||||
for actor_inputs, actor_outputs in actor_snapshots:
|
||||
for url in actor_inputs + actor_outputs:
|
||||
scheme, _host, port = split_zmq_path(url)
|
||||
assert scheme == "tcp", url
|
||||
assert port and int(port) > 0, (
|
||||
f"Ray actor was pickled with placeholder address {url!r}; "
|
||||
"``run_multi_api_server`` must pre-allocate ports for the "
|
||||
"Ray DP backend so the actors hold real endpoints by the "
|
||||
"time they DEALER-connect. See PR #42585 / Ray-DP "
|
||||
"multi-API-server regression."
|
||||
)
|
||||
|
||||
@@ -308,13 +308,16 @@ def run_multi_api_server(args: argparse.Namespace):
|
||||
|
||||
from vllm.v1.engine.utils import get_engine_zmq_addresses
|
||||
|
||||
# Per-API-server ports are picked by the kernel at each child's bind()
|
||||
# to avoid parent-probe vs child-bind TOCTOU; Rust front-end opts out
|
||||
# because it has no port-report-back channel.
|
||||
# Defer port allocation to the child's bind() to avoid TOCTOU, except
|
||||
# for Rust front-end and Ray DP, which can't see the post-bind rebind
|
||||
# (CLI-arg subprocess / pickled-into-actor snapshot respectively) and
|
||||
# so pre-allocate driver-side -- reintroducing the original race only
|
||||
# there.
|
||||
is_ray_dp = parallel_config.data_parallel_backend == "ray"
|
||||
addresses = get_engine_zmq_addresses(
|
||||
vllm_config,
|
||||
num_api_servers,
|
||||
defer_api_server_ports=not rust_frontend_path,
|
||||
defer_api_server_ports=not (rust_frontend_path or is_ray_dp),
|
||||
)
|
||||
|
||||
with launch_core_engines(
|
||||
@@ -348,11 +351,15 @@ def run_multi_api_server(args: argparse.Namespace):
|
||||
tensor_queue=tensor_queue,
|
||||
)
|
||||
|
||||
# Forward each child's bound endpoints to the engine handshake
|
||||
# (runs on ``with`` exit).
|
||||
actual_inputs, actual_outputs = api_server_manager.gather_actual_addresses()
|
||||
addresses.inputs = actual_inputs
|
||||
addresses.outputs = actual_outputs
|
||||
if not is_ray_dp:
|
||||
# Forward each child's bound endpoints to the engine handshake
|
||||
# (runs on ``with`` exit). Skipped for Ray DP, where addresses
|
||||
# are pre-allocated above and Ray actors already hold them.
|
||||
actual_inputs, actual_outputs = (
|
||||
api_server_manager.gather_actual_addresses()
|
||||
)
|
||||
addresses.inputs = actual_inputs
|
||||
addresses.outputs = actual_outputs
|
||||
|
||||
# Wait for API servers.
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user