[Bugfix] Eliminate frontend ZMQ port TOCTOU with inherited listeners (#54113)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
OpenAI Codex
Claude Opus 5.5
parent
fb2ac82458
commit
f226af0e11
+92
-11
@@ -11,6 +11,7 @@ from collections.abc import (
|
||||
Iterator,
|
||||
Sequence,
|
||||
)
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
@@ -25,6 +26,23 @@ from vllm.logger import init_logger
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ZmqListener:
|
||||
"""A supervisor-bound listener that a ZMQ socket can adopt."""
|
||||
|
||||
address: str
|
||||
socket: socket.socket
|
||||
|
||||
def close(self) -> None:
|
||||
self.socket.close()
|
||||
|
||||
def cleanup(self) -> None:
|
||||
self.close()
|
||||
if self.address.startswith("ipc://"):
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
os.unlink(self.address.removeprefix("ipc://"))
|
||||
|
||||
|
||||
def close_sockets(sockets: Sequence[zmq.Socket | zmq.asyncio.Socket]):
|
||||
for sock in sockets:
|
||||
if sock is not None:
|
||||
@@ -307,6 +325,27 @@ def make_zmq_path(scheme: str, host: str, port: int | None = None) -> str:
|
||||
|
||||
|
||||
# Adapted from: https://github.com/sgl-project/sglang/blob/v0.4.1/python/sglang/srt/utils.py#L783 # noqa: E501
|
||||
# Apply the existing make_zmq_socket memory policy to every socket owner.
|
||||
_ZMQ_LARGE_BUFFER_SIZE = 512 * 1024**2
|
||||
_ZMQ_LARGE_BUFFER_MIN_TOTAL_MEMORY = 32 * 1024**3
|
||||
_ZMQ_LARGE_BUFFER_MIN_AVAILABLE_MEMORY = 16 * 1024**3
|
||||
|
||||
|
||||
def _get_zmq_socket_buffer_size() -> int:
|
||||
"""Choose the shared libzmq and inherited-listener buffer policy."""
|
||||
mem = psutil.virtual_memory()
|
||||
# For systems with substantial memory (>32GB total, >16GB available):
|
||||
# - Set a large 0.5GB buffer to improve throughput
|
||||
# For systems with less memory:
|
||||
# - Use system default (-1) to avoid excessive memory consumption
|
||||
if (
|
||||
mem.total > _ZMQ_LARGE_BUFFER_MIN_TOTAL_MEMORY
|
||||
and mem.available > _ZMQ_LARGE_BUFFER_MIN_AVAILABLE_MEMORY
|
||||
):
|
||||
return _ZMQ_LARGE_BUFFER_SIZE
|
||||
return -1
|
||||
|
||||
|
||||
def make_zmq_socket(
|
||||
ctx: zmq.asyncio.Context | zmq.Context, # type: ignore[name-defined]
|
||||
path: str,
|
||||
@@ -315,22 +354,19 @@ def make_zmq_socket(
|
||||
identity: bytes | None = None,
|
||||
linger: int | None = None,
|
||||
router_handover: bool = False,
|
||||
listener: socket.socket | None = None,
|
||||
) -> zmq.Socket | zmq.asyncio.Socket: # type: ignore[name-defined]
|
||||
"""Make a ZMQ socket with the proper bind/connect semantics."""
|
||||
mem = psutil.virtual_memory()
|
||||
socket = ctx.socket(socket_type)
|
||||
"""Make a ZMQ socket with the proper bind/connect semantics.
|
||||
|
||||
# Calculate buffer size based on system memory
|
||||
total_mem = mem.total / 1024**3
|
||||
available_mem = mem.available / 1024**3
|
||||
# For systems with substantial memory (>32GB total, >16GB available):
|
||||
# - Set a large 0.5GB buffer to improve throughput
|
||||
# For systems with less memory:
|
||||
# - Use system default (-1) to avoid excessive memory consumption
|
||||
buf_size = int(0.5 * 1024**3) if total_mem > 32 and available_mem > 16 else -1
|
||||
When supplied, ``listener`` is detached and its fd moves to libzmq.
|
||||
"""
|
||||
socket = ctx.socket(socket_type)
|
||||
buf_size = _get_zmq_socket_buffer_size()
|
||||
|
||||
if bind is None:
|
||||
bind = socket_type not in (zmq.PUSH, zmq.SUB, zmq.XSUB)
|
||||
if listener is not None and not bind:
|
||||
raise ValueError("An inherited ZMQ listener requires bind=True")
|
||||
|
||||
if socket_type in (zmq.PULL, zmq.DEALER, zmq.ROUTER):
|
||||
socket.setsockopt(zmq.RCVHWM, 0)
|
||||
@@ -353,6 +389,9 @@ def make_zmq_socket(
|
||||
if socket_type == zmq.XPUB:
|
||||
socket.setsockopt(zmq.XPUB_VERBOSE, True)
|
||||
|
||||
if listener is not None:
|
||||
socket.setsockopt(zmq.USE_FD, listener.detach())
|
||||
|
||||
# Determine if the path is a TCP socket with an IPv6 address.
|
||||
# Enable IPv6 on the zmq socket if so.
|
||||
scheme, host, _ = split_zmq_path(path)
|
||||
@@ -367,6 +406,48 @@ def make_zmq_socket(
|
||||
return socket
|
||||
|
||||
|
||||
def make_zmq_listener(path: str, socket_type: Any) -> ZmqListener:
|
||||
"""Bind and listen on a raw socket for later ZMQ adoption."""
|
||||
scheme, host, port = split_zmq_path(path)
|
||||
if scheme == "tcp":
|
||||
family = socket.AF_INET6 if is_valid_ipv6_address(host) else socket.AF_INET
|
||||
bind_address: str | tuple[str, int] = (host, int(port))
|
||||
elif scheme == "ipc":
|
||||
family = socket.AF_UNIX
|
||||
bind_address = path.removeprefix("ipc://")
|
||||
os.makedirs(os.path.dirname(bind_address), exist_ok=True)
|
||||
else:
|
||||
raise ValueError(f"Cannot inherit a {scheme} ZMQ listener")
|
||||
|
||||
listener = socket.socket(family, socket.SOCK_STREAM)
|
||||
bound = False
|
||||
try:
|
||||
buf_size = _get_zmq_socket_buffer_size()
|
||||
if buf_size >= 0:
|
||||
# Accepted stream sockets inherit these settings from the listener.
|
||||
# ZMQ_USE_FD adopts this pre-created socket, so the policy belongs
|
||||
# here as well as on the eventual libzmq socket.
|
||||
if socket_type in (zmq.PULL, zmq.DEALER, zmq.ROUTER):
|
||||
listener.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, buf_size)
|
||||
if socket_type in (zmq.PUSH, zmq.DEALER, zmq.ROUTER):
|
||||
listener.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, buf_size)
|
||||
|
||||
listener.bind(bind_address)
|
||||
bound = True
|
||||
listener.listen()
|
||||
if scheme == "tcp":
|
||||
path = get_tcp_uri(host, listener.getsockname()[1])
|
||||
return ZmqListener(address=path, socket=listener)
|
||||
except BaseException:
|
||||
listener.close()
|
||||
# A failed bind leaves any existing pathname owned by its listener.
|
||||
if scheme == "ipc" and bound:
|
||||
assert isinstance(bind_address, str)
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
os.unlink(bind_address)
|
||||
raise
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def zmq_socket_ctx(
|
||||
path: str,
|
||||
|
||||
Reference in New Issue
Block a user