[Bugfix] Eliminate frontend ZMQ port TOCTOU with inherited listeners (#54113)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Bugen Zhao
2026-10-08 06:25:27 +00:00
committed by GitHub
co-authored by OpenAI Codex Claude Opus 5.5
parent fb2ac82458
commit f226af0e11
18 changed files with 790 additions and 585 deletions
+104 -129
View File
@@ -29,7 +29,7 @@ from torch.autograd.profiler import record_function
import vllm.envs as envs
from vllm.logger import init_logger
from vllm.usage.usage_lib import UsageContext, is_usage_stats_enabled, usage_message
from vllm.utils.network_utils import get_open_zmq_ipc_path, get_tcp_uri
from vllm.utils.network_utils import ZmqListener, get_open_zmq_ipc_path, get_tcp_uri
from vllm.utils.system_utils import decorate_logs, kill_process_tree, set_process_title
from vllm.utils.torch_utils import PIN_MEMORY
from vllm.v1.core.sched.output import SchedulerOutput
@@ -177,28 +177,22 @@ class APIServerProcessManager:
sock: Any,
args: argparse.Namespace,
num_servers: int,
input_addresses: list[str],
output_addresses: list[str],
input_listeners: list[ZmqListener],
output_listeners: list[ZmqListener],
target_server_fn: Callable | None = None,
stats_update_address: str | None = None,
tensor_queue: Queue | None = None,
):
"""Initialize and start API server worker processes.
``input_addresses``/``output_addresses`` may contain
``tcp://host:0`` placeholders; each child must report the actual
bound endpoint over its ``actual_address_pipe`` in ``client_config``
and the parent collects them via
:py:meth:`gather_actual_addresses`.
Args:
target_server_fn: Override function to call for each API server process
listen_address: Address to listen for client connections
sock: Socket for client connections
args: Command line arguments
num_servers: Number of API server processes to start
input_addresses: Input addresses for each API server
output_addresses: Output addresses for each API server
input_listeners: Input listeners for each API server
output_listeners: Output listeners for each API server
stats_update_address: Optional stats update address
tensor_queue: Optional tensor IPC queue for sharing MM tensors
@@ -209,7 +203,6 @@ class APIServerProcessManager:
spawn_context = multiprocessing.get_context("spawn")
self.processes: list[BaseProcess] = []
self._address_pipes: list[connection.Connection] = []
admission_counters = None
if num_servers > 1 and getattr(args, "max_num_queued_reqs", None) is not None:
@@ -220,126 +213,65 @@ class APIServerProcessManager:
SharedAdmissionStats.num_counters(num_servers),
)
for i, in_addr, out_addr in zip(
range(num_servers), input_addresses, output_addresses
):
client_config: dict[str, Any] = {
"input_address": in_addr,
"output_address": out_addr,
"client_count": num_servers,
"client_index": i,
}
if admission_counters is not None:
client_config["mp_admission_counters"] = admission_counters
if stats_update_address is not None:
client_config["stats_update_address"] = stats_update_address
if tensor_queue is not None:
client_config["tensor_queue"] = tensor_queue
parent_recv, child_send = spawn_context.Pipe(duplex=False)
self._address_pipes.append(parent_recv)
client_config["actual_address_pipe"] = child_send
proc = spawn_context.Process(
target=target_server_fn or run_api_server_worker_proc,
name=f"ApiServer_{i}",
args=(listen_address, sock, args, client_config),
listeners = [*input_listeners, *output_listeners]
self._listeners = listeners
try:
listener_pairs = zip(
range(num_servers), input_listeners, output_listeners, strict=True
)
self.processes.append(proc)
proc.start()
for i, input_listener, output_listener in listener_pairs:
client_config: dict[str, Any] = {
"input_address": input_listener.address,
"output_address": output_listener.address,
"input_listener": input_listener.socket,
"output_listener": output_listener.socket,
"client_count": num_servers,
"client_index": i,
}
if admission_counters is not None:
client_config["mp_admission_counters"] = admission_counters
if stats_update_address is not None:
client_config["stats_update_address"] = stats_update_address
if tensor_queue is not None:
client_config["tensor_queue"] = tensor_queue
# Drop parent's write end so reader sees EOF on child death.
child_send.close()
proc = spawn_context.Process(
target=target_server_fn or run_api_server_worker_proc,
name=f"ApiServer_{i}",
args=(listen_address, sock, args, client_config),
)
self.processes.append(proc)
proc.start()
input_listener.close()
output_listener.close()
except BaseException:
_shutdown_processes_and_cleanup_listeners(self.processes, listeners)
raise
logger.info("Started %d API server processes", len(self.processes))
# Shutdown only the API server processes on garbage collection
# The extra processes are managed by their owners
self._finalizer = weakref.finalize(self, shutdown, self.processes)
def gather_actual_addresses(
self,
timeout: float = envs.VLLM_ENGINE_READY_TIMEOUT_S,
) -> tuple[list[str], list[str]]:
"""Return (inputs, outputs) reported by each child, indexed by
``client_index``. Raises ``RuntimeError`` on timeout or premature
child exit."""
n = len(self._address_pipes)
inputs: list[str | None] = [None] * n
outputs: list[str | None] = [None] * n
pending: dict[connection.Connection, int] = {
pipe: i for i, pipe in enumerate(self._address_pipes)
}
sentinel_to_idx: dict[Any, int] = {
proc.sentinel: i for i, proc in enumerate(self.processes)
}
deadline = time.monotonic() + timeout
try:
while pending:
remaining = deadline - time.monotonic()
if remaining <= 0:
missing = [self.processes[i].name for i in pending.values()]
raise RuntimeError(
f"Timed out after {timeout:.1f}s waiting for "
f"API server(s) to report bound ZMQ addresses: "
f"{missing}"
)
waitables: list[Any] = list(pending.keys()) + list(
sentinel_to_idx.keys()
)
ready = connection.wait(waitables, timeout=remaining)
# Drain pipes before checking sentinels: a child that sent
# its message and then exited can surface both events in
# the same poll, and we must record the success first.
for item in ready:
if isinstance(item, connection.Connection) and item in pending:
idx = pending.pop(item)
try:
msg: dict[str, str] = item.recv()
except EOFError as e:
raise RuntimeError(
f"API server {self.processes[idx].name} "
f"closed its address pipe without "
f"reporting its bound ZMQ addresses"
) from e
inputs[idx] = msg["input_address"]
outputs[idx] = msg["output_address"]
item.close()
for item in ready:
if item in sentinel_to_idx:
idx = sentinel_to_idx.pop(item)
pipe = self._address_pipes[idx]
if pipe in pending:
proc = self.processes[idx]
raise RuntimeError(
f"API server process {proc.name} exited "
f"(code={proc.exitcode}) before reporting "
f"its bound ZMQ addresses"
)
finally:
for pipe in pending:
with contextlib.suppress(Exception):
pipe.close()
return inputs, outputs # type: ignore[return-value]
self._finalizer = weakref.finalize(
self,
_shutdown_processes_and_cleanup_listeners,
self.processes,
listeners,
)
def shutdown(self, timeout: float | None = None) -> None:
"""Shutdown API server processes with configurable timeout"""
for pipe in self._address_pipes:
with contextlib.suppress(Exception):
pipe.close()
self._address_pipes = []
if self._finalizer.detach() is not None:
shutdown(self.processes, timeout=timeout)
_shutdown_processes_and_cleanup_listeners(
self.processes, self._listeners, timeout=timeout
)
class RustFrontendProcessManager:
"""Manages a single Rust frontend subprocess.
Launches the Rust vllm-rs binary in 'frontend' mode, passing the
listening socket fd and ZMQ transport addresses. Provides the same
HTTP and ZMQ listener file descriptors. Provides the same
interface as APIServerProcessManager for process monitoring.
"""
@@ -348,8 +280,8 @@ class RustFrontendProcessManager:
binary_path: str,
sock: Any,
args: argparse.Namespace,
input_address: str,
output_address: str,
input_listener: ZmqListener,
output_listener: ZmqListener,
engine_start_index: int,
engine_count: int,
data_parallel_size: int,
@@ -359,18 +291,22 @@ class RustFrontendProcessManager:
import os
import subprocess
fd = sock.fileno()
os.set_inheritable(fd, True)
listen_fd = sock.fileno()
input_fd = input_listener.socket.fileno()
output_fd = output_listener.socket.fileno()
inherited_fds = [listen_fd, input_fd, output_fd]
for fd in inherited_fds:
os.set_inheritable(fd, True)
cmd = [
binary_path,
"frontend",
"--listen-fd",
str(fd),
"--input-address",
input_address,
"--output-address",
output_address,
str(listen_fd),
"--input-listener-fd",
str(input_fd),
"--output-listener-fd",
str(output_fd),
"--engine-start-index",
str(engine_start_index),
"--engine-count",
@@ -378,12 +314,11 @@ class RustFrontendProcessManager:
"--data-parallel-size",
str(data_parallel_size),
]
pass_fds = [fd]
if grpc_sock is not None:
grpc_fd = grpc_sock.fileno()
os.set_inheritable(grpc_fd, True)
cmd.extend(["--grpc-listen-fd", str(grpc_fd)])
pass_fds.append(grpc_fd)
inherited_fds.append(grpc_fd)
if stats_update_address is not None:
cmd.extend(["--coordinator-address", stats_update_address])
from vllm.entrypoints.serve.utils.api_utils import jsonify_non_default_args
@@ -425,18 +360,34 @@ class RustFrontendProcessManager:
redacted_json = json.dumps(redact_sensitive_args(args_dict), sort_keys=True)
logger.info("Launching Rust frontend: %s", " ".join(cmd[:-1] + [redacted_json]))
self._proc = subprocess.Popen(cmd, pass_fds=pass_fds)
self._listeners = [input_listener, output_listener]
try:
self._proc = subprocess.Popen(cmd, pass_fds=inherited_fds)
except BaseException:
for listener in self._listeners:
listener.cleanup()
raise
else:
input_listener.close()
output_listener.close()
# Create a process wrapper with a sentinel fd for monitoring
self.processes: list[_SubprocessWrapper] = [
_SubprocessWrapper(self._proc, "RustFrontend")
]
self._finalizer = weakref.finalize(self, _shutdown_subprocesses, self.processes)
self._finalizer = weakref.finalize(
self,
_shutdown_subprocesses_and_cleanup_listeners,
self.processes,
self._listeners,
)
def shutdown(self, timeout: float | None = None) -> None:
if self._finalizer.detach() is not None:
_shutdown_subprocesses(self.processes, timeout=timeout)
_shutdown_subprocesses_and_cleanup_listeners(
self.processes, self._listeners, timeout=timeout
)
class _SubprocessWrapper:
@@ -533,6 +484,18 @@ def _shutdown_subprocesses(
logger.debug_once("[shutdown] Subprocess manager: complete")
def _shutdown_subprocesses_and_cleanup_listeners(
procs: list[_SubprocessWrapper],
listeners: list[ZmqListener],
timeout: float | None = None,
) -> None:
try:
_shutdown_subprocesses(procs, timeout=timeout)
finally:
for listener in listeners:
listener.cleanup()
def run_api_server_worker_proc(
listen_address, sock, args, client_config=None, **uvicorn_kwargs
) -> None:
@@ -687,6 +650,18 @@ def shutdown(procs: list[BaseProcess], timeout: float | None = None) -> None:
logger.debug_once("[shutdown] Process manager: complete")
def _shutdown_processes_and_cleanup_listeners(
procs: list[BaseProcess],
listeners: list[ZmqListener],
timeout: float | None = None,
) -> None:
try:
shutdown(procs, timeout=timeout)
finally:
for listener in listeners:
listener.cleanup()
def copy_slice(
from_tensor: torch.Tensor, to_tensor: torch.Tensor, length: int
) -> torch.Tensor: