[Bugfix][V1] Fix TOCTOU race causing intermittent EADDRINUSE on multi-API-server DP startup (#42585)

Signed-off-by: Vadim Gimpelson <vadim.gimpelson@gmail.com>
Signed-off-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
Vadim Gimpelson
2026-05-26 14:06:00 -07:00
committed by GitHub
co-authored by gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
parent d98cbf472b
commit 812e7e7364
7 changed files with 316 additions and 50 deletions
+2 -2
View File
@@ -81,7 +81,7 @@ class AsyncLLM(EngineClient):
start_engine_loop: bool = True,
stat_loggers: list[StatLoggerFactory] | None = None,
aggregate_engine_logging: bool = False,
client_addresses: dict[str, str] | None = None,
client_addresses: dict[str, Any] | None = None,
client_count: int = 1,
client_index: int = 0,
) -> None:
@@ -209,7 +209,7 @@ class AsyncLLM(EngineClient):
enable_log_requests: bool = False,
aggregate_engine_logging: bool = False,
disable_log_stats: bool = False,
client_addresses: dict[str, str] | None = None,
client_addresses: dict[str, Any] | None = None,
client_count: int = 1,
client_index: int = 0,
) -> "AsyncLLM":
+4 -11
View File
@@ -11,7 +11,7 @@ import zmq
from vllm.config import ParallelConfig
from vllm.logger import init_logger
from vllm.utils.network_utils import get_tcp_uri, make_zmq_socket
from vllm.utils.network_utils import make_zmq_socket
from vllm.utils.system_utils import get_mp_context, set_process_title
from vllm.v1.engine import EngineCoreOutputs, EngineCoreRequestType
from vllm.v1.serial_utils import MsgpackDecoder
@@ -91,16 +91,9 @@ class DPCoordinator:
if parallel_config.enable_elastic_ep:
local_only_eng = False
def bind_address(local_only: bool) -> str:
return (
get_engine_client_zmq_addr(local_only=True, host=host)
if local_only
else get_tcp_uri(host, 0)
)
front_publish_address = bind_address(local_only)
back_publish_address = bind_address(local_only_eng)
back_output_address = bind_address(local_only_eng)
front_publish_address = get_engine_client_zmq_addr(local_only, host=host)
back_publish_address = get_engine_client_zmq_addr(local_only_eng, host=host)
back_output_address = get_engine_client_zmq_addr(local_only_eng, host=host)
context = get_mp_context()
parent_zmq_addr_pipe, child_zmq_addr_pipe = context.Pipe(duplex=False)
+38 -6
View File
@@ -11,6 +11,7 @@ from collections import defaultdict, deque
from collections.abc import Awaitable, Callable, Sequence
from concurrent.futures import Future
from dataclasses import dataclass
from multiprocessing.connection import Connection
from multiprocessing.queues import Queue
from threading import Thread
from typing import Any, TypeAlias, TypeVar
@@ -108,7 +109,7 @@ class EngineCoreClient(ABC):
vllm_config: VllmConfig,
executor_class: type[Executor],
log_stats: bool,
client_addresses: dict[str, str] | None = None,
client_addresses: dict[str, Any] | None = None,
client_count: int = 1,
client_index: int = 0,
) -> "AsyncMPClient":
@@ -476,7 +477,7 @@ class MPClient(EngineCoreClient):
vllm_config: VllmConfig,
executor_class: type[Executor],
log_stats: bool,
client_addresses: dict[str, str] | None = None,
client_addresses: dict[str, Any] | None = None,
):
self.vllm_config = vllm_config
@@ -507,7 +508,7 @@ class MPClient(EngineCoreClient):
output_address = client_addresses["output_address"]
self.stats_update_address = client_addresses.get("stats_update_address")
# Tensor queues passed via client_addresses for multi-API-server case
tensor_queue = client_addresses.get("tensor_queue") # type: ignore[assignment]
tensor_queue = client_addresses.get("tensor_queue")
self.input_socket = self.resources.input_socket = make_zmq_socket(
self.ctx,
input_address,
@@ -518,6 +519,28 @@ class MPClient(EngineCoreClient):
self.resources.output_socket = make_zmq_socket(
self.ctx, output_address, zmq.PULL
)
# Report bound endpoints back so the parent can forward
# them to engines (mirrors the DPCoordinator pattern).
actual_address_pipe: Connection | None = client_addresses.get(
"actual_address_pipe"
)
if actual_address_pipe is not None:
try:
actual_input = self.input_socket.getsockopt(
zmq.LAST_ENDPOINT
).decode()
actual_output = self.resources.output_socket.getsockopt(
zmq.LAST_ENDPOINT
).decode()
actual_address_pipe.send(
{
"input_address": actual_input,
"output_address": actual_output,
}
)
finally:
actual_address_pipe.close()
else:
# Engines are managed by this client.
addresses = get_engine_zmq_addresses(vllm_config)
@@ -532,6 +555,15 @@ class MPClient(EngineCoreClient):
self.ctx, addresses.outputs[0], zmq.PULL
)
# Resolve ``tcp://host:0`` placeholders to bound endpoints
# before engines DEALER-connect. No-op for IPC.
addresses.inputs[0] = self.input_socket.getsockopt(
zmq.LAST_ENDPOINT
).decode()
addresses.outputs[0] = self.resources.output_socket.getsockopt(
zmq.LAST_ENDPOINT
).decode()
with launch_core_engines(
vllm_config, executor_class, log_stats, addresses
) as (engine_manager, coordinator, addresses, tensor_queue):
@@ -893,7 +925,7 @@ class AsyncMPClient(MPClient):
vllm_config: VllmConfig,
executor_class: type[Executor],
log_stats: bool,
client_addresses: dict[str, str] | None = None,
client_addresses: dict[str, Any] | None = None,
client_count: int = 1,
client_index: int = 0,
):
@@ -1143,7 +1175,7 @@ class DPAsyncMPClient(AsyncMPClient):
vllm_config: VllmConfig,
executor_class: type[Executor],
log_stats: bool,
client_addresses: dict[str, str] | None = None,
client_addresses: dict[str, Any] | None = None,
client_count: int = 1,
client_index: int = 0,
):
@@ -1323,7 +1355,7 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
vllm_config: VllmConfig,
executor_class: type[Executor],
log_stats: bool,
client_addresses: dict[str, str] | None = None,
client_addresses: dict[str, Any] | None = None,
client_count: int = 1,
client_index: int = 0,
):
+30 -13
View File
@@ -23,7 +23,12 @@ from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.ray.ray_env import get_env_vars_to_copy
from vllm.utils import numa_utils
from vllm.utils.network_utils import get_open_zmq_ipc_path, zmq_socket_ctx
from vllm.utils.network_utils import (
get_open_port,
get_open_zmq_ipc_path,
get_tcp_uri,
zmq_socket_ctx,
)
from vllm.utils.system_utils import get_mp_context
from vllm.v1.engine.coordinator import DPCoordinator
from vllm.v1.executor import Executor
@@ -955,8 +960,19 @@ class CoreEngineActorManager:
def get_engine_zmq_addresses(
vllm_config: VllmConfig,
num_api_servers: int = 1,
*,
defer_api_server_ports: bool = True,
) -> EngineZmqAddresses:
"""Allocate ZMQ addresses for engine-client communication."""
"""Allocate ZMQ addresses for engine-client communication.
By default each TCP address is a ``tcp://host:0`` placeholder; the
consumer (API-server child or single-process ``MPClient``) binds, then
recovers the kernel-assigned port via ``getsockopt(zmq.LAST_ENDPOINT)``
and writes it back into ``addresses`` before the engine handshake.
Set ``defer_api_server_ports=False`` only when the consumer cannot
report a bound port back (e.g. the Rust front-end). IPC paths are
unaffected."""
parallel_config = vllm_config.parallel_config
local_engine_count = parallel_config.data_parallel_size_local
local_start_index = parallel_config.data_parallel_rank_local
@@ -978,15 +994,14 @@ def get_engine_zmq_addresses(
if parallel_config.enable_elastic_ep:
client_local_only = False
def _addr() -> str:
if client_local_only:
return get_open_zmq_ipc_path()
return get_tcp_uri(host, 0 if defer_api_server_ports else get_open_port())
return EngineZmqAddresses(
inputs=[
get_engine_client_zmq_addr(client_local_only, host)
for _ in range(num_api_servers)
],
outputs=[
get_engine_client_zmq_addr(client_local_only, host)
for _ in range(num_api_servers)
],
inputs=[_addr() for _ in range(num_api_servers)],
outputs=[_addr() for _ in range(num_api_servers)],
)
@@ -1095,9 +1110,11 @@ def launch_core_engines(
if parallel_config.enable_elastic_ep:
handshake_local_only = False
handshake_address = get_engine_client_zmq_addr(
handshake_local_only, host, parallel_config.data_parallel_rpc_port
)
# Preserve "port=0 means auto-pick" for the handshake address, which
# is consumed by engines spawned in this process and so cannot defer
# port resolution to bind time.
rpc_port = parallel_config.data_parallel_rpc_port or get_open_port()
handshake_address = get_engine_client_zmq_addr(handshake_local_only, host, rpc_port)
if local_engines_only and dp_rank > 0:
assert not handshake_local_only