[Bugfix] Eliminate frontend ZMQ port TOCTOU with inherited listeners (#54113)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Bugen Zhao
2026-10-08 06:25:27 +00:00
committed by GitHub
co-authored by OpenAI Codex Claude Opus 5.5
parent fb2ac82458
commit f226af0e11
18 changed files with 790 additions and 585 deletions
+22 -35
View File
@@ -11,7 +11,6 @@ from collections import Counter, defaultdict
from collections.abc import Awaitable, Callable, Sequence
from concurrent.futures import Future
from dataclasses import dataclass
from multiprocessing.connection import Connection
from multiprocessing.queues import Queue
from threading import Thread
from typing import Any, TypeAlias, TypeVar
@@ -56,7 +55,8 @@ from vllm.v1.engine.tensor_ipc import TensorIpcSender
from vllm.v1.engine.utils import (
CoreEngineActorManager,
CoreEngineProcManager,
get_engine_zmq_addresses,
EngineZmqAddresses,
get_engine_zmq_bind_addresses,
launch_core_engines,
)
from vllm.v1.executor import Executor
@@ -634,63 +634,50 @@ class MPClient(EngineCoreClient):
self.stats_update_address = client_addresses.get("stats_update_address")
# Tensor queues passed via client_addresses for multi-API-server case
tensor_queue = client_addresses.get("tensor_queue")
input_listener = client_addresses.get("input_listener")
output_listener = client_addresses.get("output_listener")
self.input_socket = self.resources.input_socket = make_zmq_socket(
self.ctx,
input_address,
zmq.ROUTER,
bind=True,
router_handover=enable_input_socket_handover,
listener=input_listener,
)
self.resources.output_socket = make_zmq_socket(
self.ctx, output_address, zmq.PULL
self.ctx,
output_address,
zmq.PULL,
listener=output_listener,
)
# Report bound endpoints back so the parent can forward
# them to engines (mirrors the DPCoordinator pattern).
actual_address_pipe: Connection | None = client_addresses.get(
"actual_address_pipe"
)
if actual_address_pipe is not None:
try:
actual_input = self.input_socket.getsockopt(
zmq.LAST_ENDPOINT
).decode()
actual_output = self.resources.output_socket.getsockopt(
zmq.LAST_ENDPOINT
).decode()
actual_address_pipe.send(
{
"input_address": actual_input,
"output_address": actual_output,
}
)
finally:
actual_address_pipe.close()
# Engines are managed externally: this process does not fork
# them, so there is no fork race; start the MM warmup now.
self._start_mm_warmup()
else:
# Engines are managed by this client.
addresses = get_engine_zmq_addresses(vllm_config)
bind_addresses = get_engine_zmq_bind_addresses(vllm_config)
self.input_socket = self.resources.input_socket = make_zmq_socket(
self.ctx,
addresses.inputs[0],
bind_addresses.inputs[0],
zmq.ROUTER,
bind=True,
router_handover=enable_input_socket_handover,
)
self.resources.output_socket = make_zmq_socket(
self.ctx, addresses.outputs[0], zmq.PULL
self.ctx, bind_addresses.outputs[0], zmq.PULL
)
# Resolve ``tcp://host:0`` placeholders to bound endpoints
# before engines DEALER-connect. No-op for IPC.
addresses.inputs[0] = self.input_socket.getsockopt(
zmq.LAST_ENDPOINT
).decode()
addresses.outputs[0] = self.resources.output_socket.getsockopt(
zmq.LAST_ENDPOINT
).decode()
# EngineZmqAddresses contains only endpoints resolved by the
# sockets that own the bind operation.
addresses = EngineZmqAddresses(
inputs=[self.input_socket.getsockopt(zmq.LAST_ENDPOINT).decode()],
outputs=[
self.resources.output_socket.getsockopt(
zmq.LAST_ENDPOINT
).decode()
],
)
with launch_core_engines(
vllm_config, executor_class, log_stats, addresses
+67 -16
View File
@@ -23,9 +23,10 @@ from vllm.platforms import current_platform
from vllm.ray.ray_env import get_env_vars_to_copy
from vllm.utils import numa_utils
from vllm.utils.network_utils import (
get_open_port,
ZmqListener,
get_open_zmq_ipc_path,
get_tcp_uri,
make_zmq_listener,
zmq_socket_ctx,
)
from vllm.utils.system_utils import get_mp_context, set_env_var
@@ -82,8 +83,18 @@ class CoreEngine:
self.state = CoreEngineState.NEW
@dataclass
class EngineZmqBindAddresses:
"""Frontend endpoints before their owning sockets bind."""
inputs: list[str]
outputs: list[str]
@dataclass
class EngineZmqAddresses:
"""Resolved frontend endpoints advertised to engine processes."""
# ZMQ input socket addresses for each front-end client (requests)
inputs: list[str]
# ZMQ output socket addresses for each front-end client (responses)
@@ -98,6 +109,27 @@ class EngineZmqAddresses:
frontend_stats_publish_address: str | None = None
@dataclass
class EngineZmqListeners:
inputs: list[ZmqListener]
outputs: list[ZmqListener]
@property
def addresses(self) -> EngineZmqAddresses:
return EngineZmqAddresses(
inputs=[listener.address for listener in self.inputs],
outputs=[listener.address for listener in self.outputs],
)
def cleanup(self) -> None:
for listener in (*self.inputs, *self.outputs):
listener.cleanup()
def __del__(self) -> None:
with contextlib.suppress(Exception):
self.cleanup()
@dataclass
class EngineHandshakeMetadata:
"""Metadata sent to each engine process during startup handshake,
@@ -1079,22 +1111,16 @@ class CoreEngineActorManager:
ray.util.remove_placement_group(pg)
def get_engine_zmq_addresses(
def get_engine_zmq_bind_addresses(
vllm_config: VllmConfig,
num_api_servers: int = 1,
*,
defer_api_server_ports: bool = True,
) -> EngineZmqAddresses:
"""Allocate ZMQ addresses for engine-client communication.
) -> EngineZmqBindAddresses:
"""Create ZMQ endpoint templates for engine-client communication.
By default each TCP address is a ``tcp://host:0`` placeholder; the
consumer (API-server child or single-process ``MPClient``) binds, then
recovers the kernel-assigned port via ``getsockopt(zmq.LAST_ENDPOINT)``
and writes it back into ``addresses`` before the engine handshake.
Set ``defer_api_server_ports=False`` only when the consumer cannot
report a bound port back (e.g. the Rust front-end). IPC paths are
unaffected."""
Each TCP address is a ``tcp://host:0`` placeholder. A single-process
``MPClient`` binds its own ZMQ sockets. Cross-process frontends use
``bind_engine_zmq_listeners`` so the supervisor binds raw listeners before
launching engines and transfers listener ownership to each frontend."""
parallel_config = vllm_config.parallel_config
local_engine_count = parallel_config.data_parallel_size_local
local_start_index = parallel_config.data_parallel_rank_local
@@ -1119,14 +1145,39 @@ def get_engine_zmq_addresses(
def _addr() -> str:
if client_local_only:
return get_open_zmq_ipc_path()
return get_tcp_uri(host, 0 if defer_api_server_ports else get_open_port())
return get_tcp_uri(host, 0)
return EngineZmqAddresses(
return EngineZmqBindAddresses(
inputs=[_addr() for _ in range(num_api_servers)],
outputs=[_addr() for _ in range(num_api_servers)],
)
def bind_engine_zmq_listeners(
vllm_config: VllmConfig,
num_api_servers: int = 1,
) -> EngineZmqListeners:
"""Bind frontend transport listeners before launching engines."""
bind_addresses = get_engine_zmq_bind_addresses(
vllm_config,
num_api_servers,
)
inputs: list[ZmqListener] = []
outputs: list[ZmqListener] = []
try:
inputs.extend(
make_zmq_listener(address, zmq.ROUTER) for address in bind_addresses.inputs
)
outputs.extend(
make_zmq_listener(address, zmq.PULL) for address in bind_addresses.outputs
)
return EngineZmqListeners(inputs=inputs, outputs=outputs)
except BaseException:
for listener in (*inputs, *outputs):
listener.cleanup()
raise
FrontendProcess = BaseProcess | _SubprocessWrapper