[Bugfix][V1] Fix TOCTOU race causing intermittent EADDRINUSE on multi-API-server DP startup (#42585)
Signed-off-by: Vadim Gimpelson <vadim.gimpelson@gmail.com> Signed-off-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
parent
d98cbf472b
commit
812e7e7364
@@ -81,7 +81,7 @@ class AsyncLLM(EngineClient):
|
||||
start_engine_loop: bool = True,
|
||||
stat_loggers: list[StatLoggerFactory] | None = None,
|
||||
aggregate_engine_logging: bool = False,
|
||||
client_addresses: dict[str, str] | None = None,
|
||||
client_addresses: dict[str, Any] | None = None,
|
||||
client_count: int = 1,
|
||||
client_index: int = 0,
|
||||
) -> None:
|
||||
@@ -209,7 +209,7 @@ class AsyncLLM(EngineClient):
|
||||
enable_log_requests: bool = False,
|
||||
aggregate_engine_logging: bool = False,
|
||||
disable_log_stats: bool = False,
|
||||
client_addresses: dict[str, str] | None = None,
|
||||
client_addresses: dict[str, Any] | None = None,
|
||||
client_count: int = 1,
|
||||
client_index: int = 0,
|
||||
) -> "AsyncLLM":
|
||||
|
||||
@@ -11,7 +11,7 @@ import zmq
|
||||
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.network_utils import get_tcp_uri, make_zmq_socket
|
||||
from vllm.utils.network_utils import make_zmq_socket
|
||||
from vllm.utils.system_utils import get_mp_context, set_process_title
|
||||
from vllm.v1.engine import EngineCoreOutputs, EngineCoreRequestType
|
||||
from vllm.v1.serial_utils import MsgpackDecoder
|
||||
@@ -91,16 +91,9 @@ class DPCoordinator:
|
||||
if parallel_config.enable_elastic_ep:
|
||||
local_only_eng = False
|
||||
|
||||
def bind_address(local_only: bool) -> str:
|
||||
return (
|
||||
get_engine_client_zmq_addr(local_only=True, host=host)
|
||||
if local_only
|
||||
else get_tcp_uri(host, 0)
|
||||
)
|
||||
|
||||
front_publish_address = bind_address(local_only)
|
||||
back_publish_address = bind_address(local_only_eng)
|
||||
back_output_address = bind_address(local_only_eng)
|
||||
front_publish_address = get_engine_client_zmq_addr(local_only, host=host)
|
||||
back_publish_address = get_engine_client_zmq_addr(local_only_eng, host=host)
|
||||
back_output_address = get_engine_client_zmq_addr(local_only_eng, host=host)
|
||||
|
||||
context = get_mp_context()
|
||||
parent_zmq_addr_pipe, child_zmq_addr_pipe = context.Pipe(duplex=False)
|
||||
|
||||
@@ -11,6 +11,7 @@ from collections import defaultdict, deque
|
||||
from collections.abc import Awaitable, Callable, Sequence
|
||||
from concurrent.futures import Future
|
||||
from dataclasses import dataclass
|
||||
from multiprocessing.connection import Connection
|
||||
from multiprocessing.queues import Queue
|
||||
from threading import Thread
|
||||
from typing import Any, TypeAlias, TypeVar
|
||||
@@ -108,7 +109,7 @@ class EngineCoreClient(ABC):
|
||||
vllm_config: VllmConfig,
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
client_addresses: dict[str, str] | None = None,
|
||||
client_addresses: dict[str, Any] | None = None,
|
||||
client_count: int = 1,
|
||||
client_index: int = 0,
|
||||
) -> "AsyncMPClient":
|
||||
@@ -476,7 +477,7 @@ class MPClient(EngineCoreClient):
|
||||
vllm_config: VllmConfig,
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
client_addresses: dict[str, str] | None = None,
|
||||
client_addresses: dict[str, Any] | None = None,
|
||||
):
|
||||
self.vllm_config = vllm_config
|
||||
|
||||
@@ -507,7 +508,7 @@ class MPClient(EngineCoreClient):
|
||||
output_address = client_addresses["output_address"]
|
||||
self.stats_update_address = client_addresses.get("stats_update_address")
|
||||
# Tensor queues passed via client_addresses for multi-API-server case
|
||||
tensor_queue = client_addresses.get("tensor_queue") # type: ignore[assignment]
|
||||
tensor_queue = client_addresses.get("tensor_queue")
|
||||
self.input_socket = self.resources.input_socket = make_zmq_socket(
|
||||
self.ctx,
|
||||
input_address,
|
||||
@@ -518,6 +519,28 @@ class MPClient(EngineCoreClient):
|
||||
self.resources.output_socket = make_zmq_socket(
|
||||
self.ctx, output_address, zmq.PULL
|
||||
)
|
||||
|
||||
# Report bound endpoints back so the parent can forward
|
||||
# them to engines (mirrors the DPCoordinator pattern).
|
||||
actual_address_pipe: Connection | None = client_addresses.get(
|
||||
"actual_address_pipe"
|
||||
)
|
||||
if actual_address_pipe is not None:
|
||||
try:
|
||||
actual_input = self.input_socket.getsockopt(
|
||||
zmq.LAST_ENDPOINT
|
||||
).decode()
|
||||
actual_output = self.resources.output_socket.getsockopt(
|
||||
zmq.LAST_ENDPOINT
|
||||
).decode()
|
||||
actual_address_pipe.send(
|
||||
{
|
||||
"input_address": actual_input,
|
||||
"output_address": actual_output,
|
||||
}
|
||||
)
|
||||
finally:
|
||||
actual_address_pipe.close()
|
||||
else:
|
||||
# Engines are managed by this client.
|
||||
addresses = get_engine_zmq_addresses(vllm_config)
|
||||
@@ -532,6 +555,15 @@ class MPClient(EngineCoreClient):
|
||||
self.ctx, addresses.outputs[0], zmq.PULL
|
||||
)
|
||||
|
||||
# Resolve ``tcp://host:0`` placeholders to bound endpoints
|
||||
# before engines DEALER-connect. No-op for IPC.
|
||||
addresses.inputs[0] = self.input_socket.getsockopt(
|
||||
zmq.LAST_ENDPOINT
|
||||
).decode()
|
||||
addresses.outputs[0] = self.resources.output_socket.getsockopt(
|
||||
zmq.LAST_ENDPOINT
|
||||
).decode()
|
||||
|
||||
with launch_core_engines(
|
||||
vllm_config, executor_class, log_stats, addresses
|
||||
) as (engine_manager, coordinator, addresses, tensor_queue):
|
||||
@@ -893,7 +925,7 @@ class AsyncMPClient(MPClient):
|
||||
vllm_config: VllmConfig,
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
client_addresses: dict[str, str] | None = None,
|
||||
client_addresses: dict[str, Any] | None = None,
|
||||
client_count: int = 1,
|
||||
client_index: int = 0,
|
||||
):
|
||||
@@ -1143,7 +1175,7 @@ class DPAsyncMPClient(AsyncMPClient):
|
||||
vllm_config: VllmConfig,
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
client_addresses: dict[str, str] | None = None,
|
||||
client_addresses: dict[str, Any] | None = None,
|
||||
client_count: int = 1,
|
||||
client_index: int = 0,
|
||||
):
|
||||
@@ -1323,7 +1355,7 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
vllm_config: VllmConfig,
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
client_addresses: dict[str, str] | None = None,
|
||||
client_addresses: dict[str, Any] | None = None,
|
||||
client_count: int = 1,
|
||||
client_index: int = 0,
|
||||
):
|
||||
|
||||
+30
-13
@@ -23,7 +23,12 @@ from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.ray.ray_env import get_env_vars_to_copy
|
||||
from vllm.utils import numa_utils
|
||||
from vllm.utils.network_utils import get_open_zmq_ipc_path, zmq_socket_ctx
|
||||
from vllm.utils.network_utils import (
|
||||
get_open_port,
|
||||
get_open_zmq_ipc_path,
|
||||
get_tcp_uri,
|
||||
zmq_socket_ctx,
|
||||
)
|
||||
from vllm.utils.system_utils import get_mp_context
|
||||
from vllm.v1.engine.coordinator import DPCoordinator
|
||||
from vllm.v1.executor import Executor
|
||||
@@ -955,8 +960,19 @@ class CoreEngineActorManager:
|
||||
def get_engine_zmq_addresses(
|
||||
vllm_config: VllmConfig,
|
||||
num_api_servers: int = 1,
|
||||
*,
|
||||
defer_api_server_ports: bool = True,
|
||||
) -> EngineZmqAddresses:
|
||||
"""Allocate ZMQ addresses for engine-client communication."""
|
||||
"""Allocate ZMQ addresses for engine-client communication.
|
||||
|
||||
By default each TCP address is a ``tcp://host:0`` placeholder; the
|
||||
consumer (API-server child or single-process ``MPClient``) binds, then
|
||||
recovers the kernel-assigned port via ``getsockopt(zmq.LAST_ENDPOINT)``
|
||||
and writes it back into ``addresses`` before the engine handshake.
|
||||
|
||||
Set ``defer_api_server_ports=False`` only when the consumer cannot
|
||||
report a bound port back (e.g. the Rust front-end). IPC paths are
|
||||
unaffected."""
|
||||
parallel_config = vllm_config.parallel_config
|
||||
local_engine_count = parallel_config.data_parallel_size_local
|
||||
local_start_index = parallel_config.data_parallel_rank_local
|
||||
@@ -978,15 +994,14 @@ def get_engine_zmq_addresses(
|
||||
if parallel_config.enable_elastic_ep:
|
||||
client_local_only = False
|
||||
|
||||
def _addr() -> str:
|
||||
if client_local_only:
|
||||
return get_open_zmq_ipc_path()
|
||||
return get_tcp_uri(host, 0 if defer_api_server_ports else get_open_port())
|
||||
|
||||
return EngineZmqAddresses(
|
||||
inputs=[
|
||||
get_engine_client_zmq_addr(client_local_only, host)
|
||||
for _ in range(num_api_servers)
|
||||
],
|
||||
outputs=[
|
||||
get_engine_client_zmq_addr(client_local_only, host)
|
||||
for _ in range(num_api_servers)
|
||||
],
|
||||
inputs=[_addr() for _ in range(num_api_servers)],
|
||||
outputs=[_addr() for _ in range(num_api_servers)],
|
||||
)
|
||||
|
||||
|
||||
@@ -1095,9 +1110,11 @@ def launch_core_engines(
|
||||
if parallel_config.enable_elastic_ep:
|
||||
handshake_local_only = False
|
||||
|
||||
handshake_address = get_engine_client_zmq_addr(
|
||||
handshake_local_only, host, parallel_config.data_parallel_rpc_port
|
||||
)
|
||||
# Preserve "port=0 means auto-pick" for the handshake address, which
|
||||
# is consumed by engines spawned in this process and so cannot defer
|
||||
# port resolution to bind time.
|
||||
rpc_port = parallel_config.data_parallel_rpc_port or get_open_port()
|
||||
handshake_address = get_engine_client_zmq_addr(handshake_local_only, host, rpc_port)
|
||||
|
||||
if local_engines_only and dp_rank > 0:
|
||||
assert not handshake_local_only
|
||||
|
||||
Reference in New Issue
Block a user