[Bugfix] Eliminate frontend ZMQ port TOCTOU with inherited listeners (#54113)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
OpenAI Codex
Claude Opus 5.5
parent
fb2ac82458
commit
f226af0e11
@@ -1,14 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Spawn worker targets kept free of heavy imports.
|
||||
|
||||
``multiprocessing`` with the ``spawn`` start method re-imports the module that
|
||||
defines a process target in the child. Housing these stubs in a stdlib-only
|
||||
module keeps child startup fast and deterministic, instead of paying a multi-
|
||||
second ``import vllm`` before the child can run.
|
||||
"""
|
||||
|
||||
|
||||
def exit_before_report_worker(listen_address, sock, args, client_config=None):
|
||||
"""Exit immediately without touching ``actual_address_pipe``."""
|
||||
return
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import multiprocessing
|
||||
import os
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
@@ -11,14 +12,10 @@ from unittest.mock import patch
|
||||
import pytest
|
||||
import zmq
|
||||
|
||||
from tests.entrypoints.launchers.api_server._api_server_spawn_workers import (
|
||||
exit_before_report_worker,
|
||||
)
|
||||
from vllm.utils.network_utils import make_zmq_socket, split_zmq_path
|
||||
from vllm.utils.network_utils import make_zmq_listener, make_zmq_socket
|
||||
from vllm.v1.engine.admission_control import SharedAdmissionStats
|
||||
from vllm.v1.utils import (
|
||||
APIServerProcessManager,
|
||||
get_engine_client_zmq_addr,
|
||||
wait_for_completion_or_failure,
|
||||
)
|
||||
|
||||
@@ -46,36 +43,30 @@ def update_admission_stats_worker(listen_address, sock, args, client_config):
|
||||
args.results.put(stats.get_num_requests())
|
||||
|
||||
|
||||
# Module-level stub for the gather_actual_addresses test. Must be
|
||||
# importable by `multiprocessing.spawn` (no closures, no nesting).
|
||||
def defer_addresses_stub_worker(listen_address, sock, args, client_config):
|
||||
"""Bind ROUTER/PULL with a kernel-assigned port, report the actual
|
||||
endpoints back via the pipe, then exit."""
|
||||
def _adopt_zmq_listener_worker(listen_address, sock, args, client_config):
|
||||
ctx = zmq.Context()
|
||||
router = make_zmq_socket(
|
||||
ctx,
|
||||
client_config["input_address"],
|
||||
zmq.ROUTER,
|
||||
bind=True,
|
||||
listener=client_config["input_listener"],
|
||||
)
|
||||
pull = make_zmq_socket(
|
||||
ctx,
|
||||
client_config["output_address"],
|
||||
zmq.PULL,
|
||||
bind=True,
|
||||
listener=client_config["output_listener"],
|
||||
)
|
||||
try:
|
||||
in_sock = make_zmq_socket(
|
||||
ctx, client_config["input_address"], zmq.ROUTER, bind=True
|
||||
)
|
||||
out_sock = make_zmq_socket(
|
||||
ctx, client_config["output_address"], zmq.PULL, bind=True
|
||||
)
|
||||
try:
|
||||
pipe = client_config["actual_address_pipe"]
|
||||
try:
|
||||
pipe.send(
|
||||
{
|
||||
"input_address": in_sock.getsockopt(zmq.LAST_ENDPOINT).decode(),
|
||||
"output_address": out_sock.getsockopt(
|
||||
zmq.LAST_ENDPOINT
|
||||
).decode(),
|
||||
}
|
||||
)
|
||||
finally:
|
||||
pipe.close()
|
||||
finally:
|
||||
in_sock.close(linger=0)
|
||||
out_sock.close(linger=0)
|
||||
identity, payload = router.recv_multipart()
|
||||
assert payload == b"input"
|
||||
assert pull.recv() == b"output"
|
||||
router.send_multipart([identity, b"ok"])
|
||||
finally:
|
||||
router.close(linger=0)
|
||||
pull.close(linger=0)
|
||||
ctx.term()
|
||||
|
||||
|
||||
@@ -89,15 +80,11 @@ def api_server_args():
|
||||
"sock": sock,
|
||||
"args": "test_args", # Simple string to avoid pickling issues
|
||||
"num_servers": 3,
|
||||
"input_addresses": [
|
||||
"tcp://127.0.0.1:5001",
|
||||
"tcp://127.0.0.1:5002",
|
||||
"tcp://127.0.0.1:5003",
|
||||
"input_listeners": [
|
||||
make_zmq_listener("tcp://127.0.0.1:0", zmq.ROUTER) for _ in range(3)
|
||||
],
|
||||
"output_addresses": [
|
||||
"tcp://127.0.0.1:6001",
|
||||
"tcp://127.0.0.1:6002",
|
||||
"tcp://127.0.0.1:6003",
|
||||
"output_listeners": [
|
||||
make_zmq_listener("tcp://127.0.0.1:0", zmq.PULL) for _ in range(3)
|
||||
],
|
||||
"stats_update_address": "tcp://127.0.0.1:7000",
|
||||
}
|
||||
@@ -120,6 +107,13 @@ def test_api_server_process_manager_init(api_server_args, with_stats_update):
|
||||
try:
|
||||
# Verify the manager was initialized correctly
|
||||
assert len(manager.processes) == 3
|
||||
assert all(
|
||||
listener.socket.fileno() == -1
|
||||
for listener in (
|
||||
*args["input_listeners"],
|
||||
*args["output_listeners"],
|
||||
)
|
||||
)
|
||||
|
||||
# Verify all processes are running
|
||||
for proc in manager.processes:
|
||||
@@ -145,6 +139,70 @@ def test_api_server_process_manager_init(api_server_args, with_stats_update):
|
||||
assert not proc.is_alive()
|
||||
|
||||
|
||||
def _assert_endpoint_reserved(address: str) -> None:
|
||||
"""A competing bind must fail while the endpoint has a listener owner."""
|
||||
if address.startswith("ipc://"):
|
||||
competitor = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
bind_address: str | tuple[str, int] = address.removeprefix("ipc://")
|
||||
else:
|
||||
competitor = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
bind_address = ("127.0.0.1", int(address.rsplit(":", 1)[1]))
|
||||
with competitor, pytest.raises(OSError, match="Address already in use"):
|
||||
competitor.bind(bind_address)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scheme", ["tcp", "ipc"])
|
||||
def test_api_server_child_adopts_inherited_zmq_listeners(tmp_path, scheme):
|
||||
if scheme == "tcp":
|
||||
input_path = output_path = "tcp://127.0.0.1:0"
|
||||
else:
|
||||
input_path = f"ipc://{tmp_path}/input.sock"
|
||||
output_path = f"ipc://{tmp_path}/output.sock"
|
||||
input_listener = make_zmq_listener(input_path, zmq.ROUTER)
|
||||
output_listener = make_zmq_listener(output_path, zmq.PULL)
|
||||
input_address = input_listener.address
|
||||
output_address = output_listener.address
|
||||
# The supervisor reserves each endpoint as soon as it allocates it.
|
||||
_assert_endpoint_reserved(input_address)
|
||||
_assert_endpoint_reserved(output_address)
|
||||
http_socket = socket.socket()
|
||||
manager = APIServerProcessManager(
|
||||
target_server_fn=_adopt_zmq_listener_worker,
|
||||
listen_address="localhost:8000",
|
||||
sock=http_socket,
|
||||
args="test_args",
|
||||
num_servers=1,
|
||||
input_listeners=[input_listener],
|
||||
output_listeners=[output_listener],
|
||||
)
|
||||
|
||||
ctx = zmq.Context()
|
||||
dealer = make_zmq_socket(
|
||||
ctx, input_address, zmq.DEALER, bind=False, identity=b"test"
|
||||
)
|
||||
push = make_zmq_socket(ctx, output_address, zmq.PUSH, bind=False)
|
||||
dealer.setsockopt(zmq.RCVTIMEO, 30_000)
|
||||
try:
|
||||
assert input_listener.socket.fileno() == -1
|
||||
assert output_listener.socket.fileno() == -1
|
||||
# After the parent closes its copies, the child's inherited descriptors
|
||||
# keep both endpoints reserved; closing must not unlink IPC paths.
|
||||
assert manager.processes[0].is_alive()
|
||||
_assert_endpoint_reserved(input_address)
|
||||
_assert_endpoint_reserved(output_address)
|
||||
dealer.send(b"input")
|
||||
push.send(b"output")
|
||||
assert dealer.recv() == b"ok"
|
||||
manager.processes[0].join(timeout=10)
|
||||
assert manager.processes[0].exitcode == 0
|
||||
finally:
|
||||
manager.shutdown()
|
||||
dealer.close(linger=0)
|
||||
push.close(linger=0)
|
||||
ctx.term()
|
||||
http_socket.close()
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_api_server_processes_share_admission_stats(api_server_args):
|
||||
args = api_server_args.copy()
|
||||
@@ -354,96 +412,7 @@ def test_external_process_monitoring(api_server_args):
|
||||
WORKER_RUNTIME_SECONDS = prev_worker_runtime
|
||||
|
||||
|
||||
@pytest.mark.timeout(60)
|
||||
def test_gather_actual_addresses_end_to_end():
|
||||
"""Each child binds ROUTER/PULL with a kernel-picked port and reports
|
||||
the bound endpoints back via its per-child pipe; the manager surfaces
|
||||
them via :py:meth:`gather_actual_addresses`."""
|
||||
host = "127.0.0.1"
|
||||
num_servers = 4
|
||||
|
||||
placeholder_inputs = [
|
||||
get_engine_client_zmq_addr(local_only=False, host=host)
|
||||
for _ in range(num_servers)
|
||||
]
|
||||
placeholder_outputs = [
|
||||
get_engine_client_zmq_addr(local_only=False, host=host)
|
||||
for _ in range(num_servers)
|
||||
]
|
||||
for addr in placeholder_inputs + placeholder_outputs:
|
||||
assert addr == f"tcp://{host}:0", addr
|
||||
|
||||
sock = socket.socket()
|
||||
manager = APIServerProcessManager(
|
||||
listen_address=f"tcp://{host}:0",
|
||||
sock=sock,
|
||||
args="test_args",
|
||||
num_servers=num_servers,
|
||||
input_addresses=placeholder_inputs,
|
||||
output_addresses=placeholder_outputs,
|
||||
target_server_fn=defer_addresses_stub_worker,
|
||||
)
|
||||
|
||||
try:
|
||||
assert len(manager.processes) == num_servers
|
||||
actual_inputs, actual_outputs = manager.gather_actual_addresses(timeout=15.0)
|
||||
finally:
|
||||
manager.shutdown()
|
||||
time.sleep(0.2)
|
||||
sock.close()
|
||||
|
||||
assert len(actual_inputs) == num_servers
|
||||
assert len(actual_outputs) == num_servers
|
||||
|
||||
for addr in actual_inputs + actual_outputs:
|
||||
scheme, parsed_host, port = split_zmq_path(addr)
|
||||
assert scheme == "tcp", addr
|
||||
assert parsed_host == host, addr
|
||||
assert port and int(port) > 0, addr
|
||||
|
||||
all_addrs = actual_inputs + actual_outputs
|
||||
assert len(set(all_addrs)) == len(all_addrs), all_addrs
|
||||
|
||||
|
||||
@pytest.mark.timeout(30)
|
||||
def test_gather_actual_addresses_child_crash_before_report():
|
||||
"""A child that exits before sending its endpoints must surface a
|
||||
clear ``RuntimeError`` rather than hang or return ``None`` slots."""
|
||||
host = "127.0.0.1"
|
||||
num_servers = 2
|
||||
placeholder_inputs = [
|
||||
get_engine_client_zmq_addr(local_only=False, host=host)
|
||||
for _ in range(num_servers)
|
||||
]
|
||||
placeholder_outputs = [
|
||||
get_engine_client_zmq_addr(local_only=False, host=host)
|
||||
for _ in range(num_servers)
|
||||
]
|
||||
|
||||
sock = socket.socket()
|
||||
manager = APIServerProcessManager(
|
||||
listen_address=f"tcp://{host}:0",
|
||||
sock=sock,
|
||||
args="test_args",
|
||||
num_servers=num_servers,
|
||||
input_addresses=placeholder_inputs,
|
||||
output_addresses=placeholder_outputs,
|
||||
# exit_before_report_worker exits without touching
|
||||
# ``actual_address_pipe`` — simulates a child that dies before
|
||||
# reporting its bound addresses.
|
||||
target_server_fn=exit_before_report_worker,
|
||||
)
|
||||
try:
|
||||
# Sentinel-first vs pipe-EOF-first both produce "reporting".
|
||||
with pytest.raises(RuntimeError, match="reporting"):
|
||||
manager.gather_actual_addresses(timeout=10.0)
|
||||
finally:
|
||||
manager.shutdown()
|
||||
time.sleep(0.2)
|
||||
sock.close()
|
||||
|
||||
|
||||
def test_rust_frontend_launch_log_redacts_credentials(monkeypatch, caplog):
|
||||
def test_rust_frontend_launch_log_redacts_credentials(monkeypatch):
|
||||
"""The Rust frontend command carries every non-default arg as JSON.
|
||||
Credentials must not reach the log line."""
|
||||
import subprocess as subprocess_mod
|
||||
@@ -469,15 +438,17 @@ def test_rust_frontend_launch_log_redacts_credentials(monkeypatch, caplog):
|
||||
return 0
|
||||
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
input_listener = make_zmq_listener("tcp://127.0.0.1:0", zmq.ROUTER)
|
||||
output_listener = make_zmq_listener("tcp://127.0.0.1:0", zmq.PULL)
|
||||
try:
|
||||
monkeypatch.setattr(subprocess_mod, "Popen", lambda *a, **kw: _FakeProc())
|
||||
with caplog.at_level("INFO", logger="vllm.v1.utils"):
|
||||
with patch("vllm.v1.utils.logger.info") as log_info:
|
||||
RustFrontendProcessManager(
|
||||
binary_path="/nonexistent/vllm-rs",
|
||||
sock=sock,
|
||||
args=args,
|
||||
input_address="ipc:///tmp/in",
|
||||
output_address="ipc:///tmp/out",
|
||||
input_listener=input_listener,
|
||||
output_listener=output_listener,
|
||||
engine_start_index=0,
|
||||
engine_count=1,
|
||||
data_parallel_size=1,
|
||||
@@ -485,11 +456,13 @@ def test_rust_frontend_launch_log_redacts_credentials(monkeypatch, caplog):
|
||||
finally:
|
||||
sock.close()
|
||||
|
||||
message = caplog.text
|
||||
log_format, *log_args = log_info.call_args.args
|
||||
message = log_format % tuple(log_args)
|
||||
assert "Launching Rust frontend:" in message
|
||||
assert hf_token not in message
|
||||
assert api_key not in message
|
||||
assert '"hf_token": "***"' in message
|
||||
assert '"api_key": "***"' in message
|
||||
|
||||
|
||||
def test_rust_frontend_uses_config_model_as_model_tag(monkeypatch, caplog, tmp_path):
|
||||
@@ -519,6 +492,8 @@ def test_rust_frontend_uses_config_model_as_model_tag(monkeypatch, caplog, tmp_p
|
||||
return 0
|
||||
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
input_listener = make_zmq_listener("tcp://127.0.0.1:0", zmq.ROUTER)
|
||||
output_listener = make_zmq_listener("tcp://127.0.0.1:0", zmq.PULL)
|
||||
try:
|
||||
monkeypatch.setattr(subprocess_mod, "Popen", lambda *a, **kw: _FakeProc())
|
||||
with caplog.at_level("INFO", logger="vllm.v1.utils"):
|
||||
@@ -526,8 +501,8 @@ def test_rust_frontend_uses_config_model_as_model_tag(monkeypatch, caplog, tmp_p
|
||||
binary_path="/nonexistent/vllm-rs",
|
||||
sock=sock,
|
||||
args=args,
|
||||
input_address="ipc:///tmp/in",
|
||||
output_address="ipc:///tmp/out",
|
||||
input_listener=input_listener,
|
||||
output_listener=output_listener,
|
||||
engine_start_index=0,
|
||||
engine_count=1,
|
||||
data_parallel_size=1,
|
||||
@@ -568,14 +543,18 @@ def test_rust_frontend_inherits_grpc_listener(monkeypatch):
|
||||
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
grpc_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
input_listener = make_zmq_listener("tcp://127.0.0.1:0", zmq.ROUTER)
|
||||
output_listener = make_zmq_listener("tcp://127.0.0.1:0", zmq.PULL)
|
||||
input_fd = input_listener.socket.fileno()
|
||||
output_fd = output_listener.socket.fileno()
|
||||
try:
|
||||
monkeypatch.setattr(subprocess_mod, "Popen", fake_popen)
|
||||
RustFrontendProcessManager(
|
||||
binary_path="/nonexistent/vllm-rs",
|
||||
sock=sock,
|
||||
args=args,
|
||||
input_address="ipc:///tmp/in",
|
||||
output_address="ipc:///tmp/out",
|
||||
input_listener=input_listener,
|
||||
output_listener=output_listener,
|
||||
engine_start_index=0,
|
||||
engine_count=1,
|
||||
data_parallel_size=1,
|
||||
@@ -585,7 +564,44 @@ def test_rust_frontend_inherits_grpc_listener(monkeypatch):
|
||||
grpc_fd = grpc_sock.fileno()
|
||||
flag_index = cmd.index("--grpc-listen-fd")
|
||||
assert cmd[flag_index + 1] == str(grpc_fd)
|
||||
assert list(kwargs["pass_fds"]) == [sock.fileno(), grpc_fd]
|
||||
assert list(kwargs["pass_fds"]) == [
|
||||
sock.fileno(),
|
||||
input_fd,
|
||||
output_fd,
|
||||
grpc_fd,
|
||||
]
|
||||
finally:
|
||||
sock.close()
|
||||
grpc_sock.close()
|
||||
|
||||
|
||||
def test_rust_frontend_spawn_failure_cleans_up_listeners(tmp_path):
|
||||
"""No child owns the listeners when Popen fails, so the parent removes them."""
|
||||
from vllm.entrypoints.launchers.cli_args import make_arg_parser
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
from vllm.v1.utils import RustFrontendProcessManager
|
||||
|
||||
args = make_arg_parser(FlexibleArgumentParser()).parse_args(
|
||||
["--model", "org/model"]
|
||||
)
|
||||
input_listener = make_zmq_listener(f"ipc://{tmp_path}/input.sock", zmq.ROUTER)
|
||||
output_listener = make_zmq_listener(f"ipc://{tmp_path}/output.sock", zmq.PULL)
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
try:
|
||||
with pytest.raises(FileNotFoundError):
|
||||
RustFrontendProcessManager(
|
||||
binary_path=str(tmp_path / "missing-vllm-rs"),
|
||||
sock=sock,
|
||||
args=args,
|
||||
input_listener=input_listener,
|
||||
output_listener=output_listener,
|
||||
engine_start_index=0,
|
||||
engine_count=1,
|
||||
data_parallel_size=1,
|
||||
)
|
||||
finally:
|
||||
sock.close()
|
||||
|
||||
for listener in (input_listener, output_listener):
|
||||
assert listener.socket.fileno() == -1
|
||||
assert not os.path.exists(listener.address.removeprefix("ipc://"))
|
||||
|
||||
@@ -13,6 +13,7 @@ from vllm.utils.network_utils import (
|
||||
get_open_ports_list,
|
||||
get_tcp_uri,
|
||||
join_host_port,
|
||||
make_zmq_listener,
|
||||
make_zmq_path,
|
||||
make_zmq_socket,
|
||||
split_host_port,
|
||||
@@ -205,6 +206,52 @@ def test_make_zmq_socket_ipv6():
|
||||
ctx.term()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scheme", ["tcp", "ipc"])
|
||||
def test_make_zmq_socket_adopts_listener(tmp_path, scheme):
|
||||
path = (
|
||||
"tcp://127.0.0.1:0" if scheme == "tcp" else f"ipc://{tmp_path}/inherited.sock"
|
||||
)
|
||||
listener = make_zmq_listener(path, zmq.ROUTER)
|
||||
ctx = zmq.Context()
|
||||
router = dealer = None
|
||||
try:
|
||||
router = make_zmq_socket(
|
||||
ctx,
|
||||
listener.address,
|
||||
zmq.ROUTER,
|
||||
bind=True,
|
||||
listener=listener.socket,
|
||||
)
|
||||
assert listener.socket.fileno() == -1
|
||||
dealer = make_zmq_socket(
|
||||
ctx, listener.address, zmq.DEALER, bind=False, identity=b"client"
|
||||
)
|
||||
dealer.send(b"ping")
|
||||
assert router.recv_multipart() == [b"client", b"ping"]
|
||||
finally:
|
||||
if router is not None:
|
||||
router.close(linger=0)
|
||||
if dealer is not None:
|
||||
dealer.close(linger=0)
|
||||
ctx.term()
|
||||
listener.cleanup()
|
||||
|
||||
|
||||
def test_make_zmq_listener_failed_ipc_bind_keeps_existing_path(tmp_path):
|
||||
path = tmp_path / "taken.sock"
|
||||
owner = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
owner.bind(str(path))
|
||||
owner.listen()
|
||||
try:
|
||||
with pytest.raises(OSError, match="Address already in use"):
|
||||
make_zmq_listener(f"ipc://{path}", zmq.ROUTER)
|
||||
|
||||
with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client:
|
||||
client.connect(str(path))
|
||||
finally:
|
||||
owner.close()
|
||||
|
||||
|
||||
def test_make_zmq_path():
|
||||
assert make_zmq_path("tcp", "127.0.0.1", 5555) == "tcp://127.0.0.1:5555"
|
||||
assert make_zmq_path("tcp", "::1", 5555) == "tcp://[::1]:5555"
|
||||
|
||||
@@ -23,7 +23,7 @@ from vllm.v1.engine.utils import (
|
||||
EngineZmqAddresses,
|
||||
_dp_nodes_master_first,
|
||||
_node_ip_from_resources,
|
||||
get_engine_zmq_addresses,
|
||||
bind_engine_zmq_listeners,
|
||||
launch_core_engines,
|
||||
)
|
||||
from vllm.v1.utils import APIServerProcessManager
|
||||
@@ -67,35 +67,27 @@ class _StubEngineCoreActor(EngineCoreActorMixin):
|
||||
|
||||
|
||||
# Module-level stub worker for the Ray-DP regression test. Must be importable
|
||||
# by ``multiprocessing.spawn`` (no closures, no nesting). Mirrors the worker
|
||||
# in ``tests/entrypoints/test_api_server_process_manager.py``.
|
||||
def _bind_and_report_worker(listen_address, sock, args, client_config):
|
||||
"""Bind ROUTER/PULL with a kernel-assigned port, report the actual
|
||||
endpoints back via ``actual_address_pipe``, then exit."""
|
||||
# by ``multiprocessing.spawn`` (no closures, no nesting).
|
||||
def _adopt_listener_worker(listen_address, sock, args, client_config):
|
||||
"""Adopt the supervisor-bound ROUTER/PULL listeners, then exit."""
|
||||
ctx = zmq.Context()
|
||||
try:
|
||||
in_sock = make_zmq_socket(
|
||||
ctx, client_config["input_address"], zmq.ROUTER, bind=True
|
||||
ctx,
|
||||
client_config["input_address"],
|
||||
zmq.ROUTER,
|
||||
bind=True,
|
||||
listener=client_config["input_listener"],
|
||||
)
|
||||
out_sock = make_zmq_socket(
|
||||
ctx, client_config["output_address"], zmq.PULL, bind=True
|
||||
ctx,
|
||||
client_config["output_address"],
|
||||
zmq.PULL,
|
||||
bind=True,
|
||||
listener=client_config["output_listener"],
|
||||
)
|
||||
try:
|
||||
pipe = client_config["actual_address_pipe"]
|
||||
try:
|
||||
pipe.send(
|
||||
{
|
||||
"input_address": in_sock.getsockopt(zmq.LAST_ENDPOINT).decode(),
|
||||
"output_address": out_sock.getsockopt(
|
||||
zmq.LAST_ENDPOINT
|
||||
).decode(),
|
||||
}
|
||||
)
|
||||
finally:
|
||||
pipe.close()
|
||||
finally:
|
||||
in_sock.close(linger=0)
|
||||
out_sock.close(linger=0)
|
||||
in_sock.close(linger=0)
|
||||
out_sock.close(linger=0)
|
||||
finally:
|
||||
ctx.term()
|
||||
|
||||
@@ -327,17 +319,9 @@ def test_ray_dp_addresses_resolved_before_actor_creation(
|
||||
``launch_core_engines`` Ray branch pickles ``addresses`` into each engine
|
||||
actor at ``.remote()`` time, and ``EngineCoreActorMixin._perform_handshakes``
|
||||
is a no-op, so the actor uses that pickled snapshot for the rest of its
|
||||
life. If ``run_multi_api_server`` allocates ``addresses`` as
|
||||
``tcp://host:0`` placeholders (its default), the actors hold placeholders
|
||||
forever and DEALER-connect to port 0 — ZMQ ``connect`` is async and does
|
||||
not raise, so the failure mode is a deterministic hang.
|
||||
|
||||
The Ray-DP carve-out in ``run_multi_api_server`` forces
|
||||
``defer_api_server_ports=False`` when ``data_parallel_backend == "ray"``
|
||||
so addresses are pre-allocated in the driver and Ray pickles real ports
|
||||
into each actor. This test mirrors that call-site logic and asserts the
|
||||
actors hold real (non-placeholder) endpoints. If the carve-out is
|
||||
removed without an alternative fix, the test fails.
|
||||
life. The supervisor therefore binds the frontend listeners before actor
|
||||
construction and Ray pickles their resolved endpoints. This test asserts
|
||||
that each actor holds real (non-placeholder) endpoints.
|
||||
"""
|
||||
created_placement_groups: list[Any] = []
|
||||
|
||||
@@ -356,15 +340,10 @@ def test_ray_dp_addresses_resolved_before_actor_creation(
|
||||
|
||||
vllm_config = _make_vllm_config_ray_dp_multinode()
|
||||
|
||||
# Mirror run_multi_api_server's address-allocation logic. The Ray DP
|
||||
# carve-out forces pre-allocation so the addresses pickled into engine
|
||||
# actors at .remote() time are real, not ``tcp://host:0``.
|
||||
is_ray_dp = vllm_config.parallel_config.data_parallel_backend == "ray"
|
||||
addresses = get_engine_zmq_addresses(
|
||||
vllm_config,
|
||||
num_api_servers=2,
|
||||
defer_api_server_ports=not is_ray_dp,
|
||||
)
|
||||
# Mirror run_multi_api_server: bind listeners in the supervisor so the
|
||||
# addresses pickled into Ray actors contain kernel-assigned ports.
|
||||
listeners = bind_engine_zmq_listeners(vllm_config, num_api_servers=2)
|
||||
addresses = listeners.addresses
|
||||
|
||||
sock = socket.socket()
|
||||
engine_manager: CoreEngineActorManager | None = None
|
||||
@@ -381,26 +360,17 @@ def test_ray_dp_addresses_resolved_before_actor_creation(
|
||||
engine_manager = engine_launch.engine_manager
|
||||
assert isinstance(engine_manager, CoreEngineActorManager)
|
||||
|
||||
# API-server children bind to the pre-allocated ports.
|
||||
# API-server children adopt the already-bound listener FDs.
|
||||
api_server_manager = APIServerProcessManager(
|
||||
listen_address="tcp://127.0.0.1:0",
|
||||
sock=sock,
|
||||
args="test_args",
|
||||
num_servers=2,
|
||||
input_addresses=addresses.inputs,
|
||||
output_addresses=addresses.outputs,
|
||||
target_server_fn=_bind_and_report_worker,
|
||||
input_listeners=listeners.inputs,
|
||||
output_listeners=listeners.outputs,
|
||||
target_server_fn=_adopt_listener_worker,
|
||||
)
|
||||
|
||||
# run_multi_api_server skips ``gather_actual_addresses`` for
|
||||
# Ray DP (addresses are already real). Mirror that.
|
||||
if not is_ray_dp:
|
||||
actual_inputs, actual_outputs = (
|
||||
api_server_manager.gather_actual_addresses(timeout=15.0)
|
||||
)
|
||||
addresses.inputs = actual_inputs
|
||||
addresses.outputs = actual_outputs
|
||||
|
||||
# Snapshot what each Ray actor actually holds.
|
||||
actors = (
|
||||
engine_manager.local_engine_actors + engine_manager.remote_engine_actors
|
||||
@@ -427,9 +397,9 @@ def test_ray_dp_addresses_resolved_before_actor_creation(
|
||||
assert scheme == "tcp", url
|
||||
assert port and int(port) > 0, (
|
||||
f"Ray actor was pickled with placeholder address {url!r}; "
|
||||
"``run_multi_api_server`` must pre-allocate ports for the "
|
||||
"Ray DP backend so the actors hold real endpoints by the "
|
||||
"time they DEALER-connect. See PR #42585 / Ray-DP "
|
||||
"``run_multi_api_server`` must bind frontend listeners before "
|
||||
"constructing Ray DP actors so they hold real endpoints when "
|
||||
"they DEALER-connect. See PR #42585 / Ray-DP "
|
||||
"multi-API-server regression."
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user