[Bugfix] Eliminate frontend ZMQ port TOCTOU with inherited listeners (#54113)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
OpenAI Codex
Claude Opus 5.5
parent
fb2ac82458
commit
f226af0e11
@@ -11,7 +11,6 @@ from collections import Counter, defaultdict
|
||||
from collections.abc import Awaitable, Callable, Sequence
|
||||
from concurrent.futures import Future
|
||||
from dataclasses import dataclass
|
||||
from multiprocessing.connection import Connection
|
||||
from multiprocessing.queues import Queue
|
||||
from threading import Thread
|
||||
from typing import Any, TypeAlias, TypeVar
|
||||
@@ -56,7 +55,8 @@ from vllm.v1.engine.tensor_ipc import TensorIpcSender
|
||||
from vllm.v1.engine.utils import (
|
||||
CoreEngineActorManager,
|
||||
CoreEngineProcManager,
|
||||
get_engine_zmq_addresses,
|
||||
EngineZmqAddresses,
|
||||
get_engine_zmq_bind_addresses,
|
||||
launch_core_engines,
|
||||
)
|
||||
from vllm.v1.executor import Executor
|
||||
@@ -634,63 +634,50 @@ class MPClient(EngineCoreClient):
|
||||
self.stats_update_address = client_addresses.get("stats_update_address")
|
||||
# Tensor queues passed via client_addresses for multi-API-server case
|
||||
tensor_queue = client_addresses.get("tensor_queue")
|
||||
input_listener = client_addresses.get("input_listener")
|
||||
output_listener = client_addresses.get("output_listener")
|
||||
self.input_socket = self.resources.input_socket = make_zmq_socket(
|
||||
self.ctx,
|
||||
input_address,
|
||||
zmq.ROUTER,
|
||||
bind=True,
|
||||
router_handover=enable_input_socket_handover,
|
||||
listener=input_listener,
|
||||
)
|
||||
self.resources.output_socket = make_zmq_socket(
|
||||
self.ctx, output_address, zmq.PULL
|
||||
self.ctx,
|
||||
output_address,
|
||||
zmq.PULL,
|
||||
listener=output_listener,
|
||||
)
|
||||
|
||||
# Report bound endpoints back so the parent can forward
|
||||
# them to engines (mirrors the DPCoordinator pattern).
|
||||
actual_address_pipe: Connection | None = client_addresses.get(
|
||||
"actual_address_pipe"
|
||||
)
|
||||
if actual_address_pipe is not None:
|
||||
try:
|
||||
actual_input = self.input_socket.getsockopt(
|
||||
zmq.LAST_ENDPOINT
|
||||
).decode()
|
||||
actual_output = self.resources.output_socket.getsockopt(
|
||||
zmq.LAST_ENDPOINT
|
||||
).decode()
|
||||
actual_address_pipe.send(
|
||||
{
|
||||
"input_address": actual_input,
|
||||
"output_address": actual_output,
|
||||
}
|
||||
)
|
||||
finally:
|
||||
actual_address_pipe.close()
|
||||
# Engines are managed externally: this process does not fork
|
||||
# them, so there is no fork race; start the MM warmup now.
|
||||
self._start_mm_warmup()
|
||||
else:
|
||||
# Engines are managed by this client.
|
||||
addresses = get_engine_zmq_addresses(vllm_config)
|
||||
bind_addresses = get_engine_zmq_bind_addresses(vllm_config)
|
||||
self.input_socket = self.resources.input_socket = make_zmq_socket(
|
||||
self.ctx,
|
||||
addresses.inputs[0],
|
||||
bind_addresses.inputs[0],
|
||||
zmq.ROUTER,
|
||||
bind=True,
|
||||
router_handover=enable_input_socket_handover,
|
||||
)
|
||||
self.resources.output_socket = make_zmq_socket(
|
||||
self.ctx, addresses.outputs[0], zmq.PULL
|
||||
self.ctx, bind_addresses.outputs[0], zmq.PULL
|
||||
)
|
||||
|
||||
# Resolve ``tcp://host:0`` placeholders to bound endpoints
|
||||
# before engines DEALER-connect. No-op for IPC.
|
||||
addresses.inputs[0] = self.input_socket.getsockopt(
|
||||
zmq.LAST_ENDPOINT
|
||||
).decode()
|
||||
addresses.outputs[0] = self.resources.output_socket.getsockopt(
|
||||
zmq.LAST_ENDPOINT
|
||||
).decode()
|
||||
# EngineZmqAddresses contains only endpoints resolved by the
|
||||
# sockets that own the bind operation.
|
||||
addresses = EngineZmqAddresses(
|
||||
inputs=[self.input_socket.getsockopt(zmq.LAST_ENDPOINT).decode()],
|
||||
outputs=[
|
||||
self.resources.output_socket.getsockopt(
|
||||
zmq.LAST_ENDPOINT
|
||||
).decode()
|
||||
],
|
||||
)
|
||||
|
||||
with launch_core_engines(
|
||||
vllm_config, executor_class, log_stats, addresses
|
||||
|
||||
+67
-16
@@ -23,9 +23,10 @@ from vllm.platforms import current_platform
|
||||
from vllm.ray.ray_env import get_env_vars_to_copy
|
||||
from vllm.utils import numa_utils
|
||||
from vllm.utils.network_utils import (
|
||||
get_open_port,
|
||||
ZmqListener,
|
||||
get_open_zmq_ipc_path,
|
||||
get_tcp_uri,
|
||||
make_zmq_listener,
|
||||
zmq_socket_ctx,
|
||||
)
|
||||
from vllm.utils.system_utils import get_mp_context, set_env_var
|
||||
@@ -82,8 +83,18 @@ class CoreEngine:
|
||||
self.state = CoreEngineState.NEW
|
||||
|
||||
|
||||
@dataclass
|
||||
class EngineZmqBindAddresses:
|
||||
"""Frontend endpoints before their owning sockets bind."""
|
||||
|
||||
inputs: list[str]
|
||||
outputs: list[str]
|
||||
|
||||
|
||||
@dataclass
|
||||
class EngineZmqAddresses:
|
||||
"""Resolved frontend endpoints advertised to engine processes."""
|
||||
|
||||
# ZMQ input socket addresses for each front-end client (requests)
|
||||
inputs: list[str]
|
||||
# ZMQ output socket addresses for each front-end client (responses)
|
||||
@@ -98,6 +109,27 @@ class EngineZmqAddresses:
|
||||
frontend_stats_publish_address: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class EngineZmqListeners:
|
||||
inputs: list[ZmqListener]
|
||||
outputs: list[ZmqListener]
|
||||
|
||||
@property
|
||||
def addresses(self) -> EngineZmqAddresses:
|
||||
return EngineZmqAddresses(
|
||||
inputs=[listener.address for listener in self.inputs],
|
||||
outputs=[listener.address for listener in self.outputs],
|
||||
)
|
||||
|
||||
def cleanup(self) -> None:
|
||||
for listener in (*self.inputs, *self.outputs):
|
||||
listener.cleanup()
|
||||
|
||||
def __del__(self) -> None:
|
||||
with contextlib.suppress(Exception):
|
||||
self.cleanup()
|
||||
|
||||
|
||||
@dataclass
|
||||
class EngineHandshakeMetadata:
|
||||
"""Metadata sent to each engine process during startup handshake,
|
||||
@@ -1079,22 +1111,16 @@ class CoreEngineActorManager:
|
||||
ray.util.remove_placement_group(pg)
|
||||
|
||||
|
||||
def get_engine_zmq_addresses(
|
||||
def get_engine_zmq_bind_addresses(
|
||||
vllm_config: VllmConfig,
|
||||
num_api_servers: int = 1,
|
||||
*,
|
||||
defer_api_server_ports: bool = True,
|
||||
) -> EngineZmqAddresses:
|
||||
"""Allocate ZMQ addresses for engine-client communication.
|
||||
) -> EngineZmqBindAddresses:
|
||||
"""Create ZMQ endpoint templates for engine-client communication.
|
||||
|
||||
By default each TCP address is a ``tcp://host:0`` placeholder; the
|
||||
consumer (API-server child or single-process ``MPClient``) binds, then
|
||||
recovers the kernel-assigned port via ``getsockopt(zmq.LAST_ENDPOINT)``
|
||||
and writes it back into ``addresses`` before the engine handshake.
|
||||
|
||||
Set ``defer_api_server_ports=False`` only when the consumer cannot
|
||||
report a bound port back (e.g. the Rust front-end). IPC paths are
|
||||
unaffected."""
|
||||
Each TCP address is a ``tcp://host:0`` placeholder. A single-process
|
||||
``MPClient`` binds its own ZMQ sockets. Cross-process frontends use
|
||||
``bind_engine_zmq_listeners`` so the supervisor binds raw listeners before
|
||||
launching engines and transfers listener ownership to each frontend."""
|
||||
parallel_config = vllm_config.parallel_config
|
||||
local_engine_count = parallel_config.data_parallel_size_local
|
||||
local_start_index = parallel_config.data_parallel_rank_local
|
||||
@@ -1119,14 +1145,39 @@ def get_engine_zmq_addresses(
|
||||
def _addr() -> str:
|
||||
if client_local_only:
|
||||
return get_open_zmq_ipc_path()
|
||||
return get_tcp_uri(host, 0 if defer_api_server_ports else get_open_port())
|
||||
return get_tcp_uri(host, 0)
|
||||
|
||||
return EngineZmqAddresses(
|
||||
return EngineZmqBindAddresses(
|
||||
inputs=[_addr() for _ in range(num_api_servers)],
|
||||
outputs=[_addr() for _ in range(num_api_servers)],
|
||||
)
|
||||
|
||||
|
||||
def bind_engine_zmq_listeners(
|
||||
vllm_config: VllmConfig,
|
||||
num_api_servers: int = 1,
|
||||
) -> EngineZmqListeners:
|
||||
"""Bind frontend transport listeners before launching engines."""
|
||||
bind_addresses = get_engine_zmq_bind_addresses(
|
||||
vllm_config,
|
||||
num_api_servers,
|
||||
)
|
||||
inputs: list[ZmqListener] = []
|
||||
outputs: list[ZmqListener] = []
|
||||
try:
|
||||
inputs.extend(
|
||||
make_zmq_listener(address, zmq.ROUTER) for address in bind_addresses.inputs
|
||||
)
|
||||
outputs.extend(
|
||||
make_zmq_listener(address, zmq.PULL) for address in bind_addresses.outputs
|
||||
)
|
||||
return EngineZmqListeners(inputs=inputs, outputs=outputs)
|
||||
except BaseException:
|
||||
for listener in (*inputs, *outputs):
|
||||
listener.cleanup()
|
||||
raise
|
||||
|
||||
|
||||
FrontendProcess = BaseProcess | _SubprocessWrapper
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user