elastic_ep: Fix stateless group port races (#36330)

Signed-off-by: Itay Alroy <ialroy@nvidia.com>
This commit is contained in:
Itay Alroy
2026-03-18 14:36:18 +00:00
committed by GitHub
parent 99267c23ca
commit de1a86b7de
12 changed files with 221 additions and 222 deletions
+28 -82
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
import socket
from collections.abc import Callable
from typing import TYPE_CHECKING, Any, Literal, overload
@@ -266,33 +267,9 @@ class ParallelConfig:
Set to be private as it's not intended to be configured by users.
"""
_stateless_dp_group_port_list: list[list[int]] = Field(default_factory=list)
"""List of open ports for stateless DP groups when enable_elastic_ep is True.
Set to be private as it's not intended to be configured by users.
It is a list of list[int], with each inner list contains a set of 3 ports
to be used for setting up the stateless CPU/device/TCPStore groups
in StatelessGroupCoordinator. The number of inner lists is equal to
the number of DP groups,
i.e., len(self._stateless_dp_group_port_list) == world_size_across_dp // dp_size,
and len(self._stateless_dp_group_port_list[i]) == 3 for all i.
"""
_stateless_ep_group_port_list: list[list[int]] = Field(default_factory=list)
"""List of open ports for stateless EP groups when enable_elastic_ep is True.
Set to be private as it's not intended to be configured by users.
len(self._stateless_ep_group_port_list) == world_size_across_dp // ep_size,
"""
_stateless_eplb_group_port_list: list[list[int]] = Field(default_factory=list)
"""List of open ports for stateless EPLB groups when enable_elastic_ep is True.
Same topology as EP but separate NCCL communicator to avoid deadlocks.
"""
_stateless_world_group_port_list: list[list[int]] = Field(default_factory=list)
"""List of open ports for stateless world group when enable_elastic_ep is True.
Set to be private as it's not intended to be configured by users.
len(self._stateless_world_group_port_list) == 1,
"""
_coord_store_port: int = 0
"""Port of the coordination TCPStore. Can be set by the API server; workers
connect as clients to exchange self-picked group ports at runtime."""
decode_context_parallel_size: int = 1
"""Number of decode context parallel groups, because the world size does
@@ -465,65 +442,32 @@ class ParallelConfig:
return answer
def allocate_elastic_ep_ports(self) -> None:
"""Allocate all ports for elastic EP (stateless groups + DP master).
def _pick_stateless_dp_port(self) -> tuple[int, socket.socket | None]:
"""Return ``(port, listen_socket)`` for DP group init.
Must be called AFTER ray.init() so that ports claimed by Ray's
idle worker pool are already in use and won't be returned by
get_open_ports_list().
With a coord store, rank 0 binds a socket and publishes the port;
others read it. Without one, pops a pre-allocated port and
returns ``listen_socket=None``.
"""
if not self.enable_elastic_ep:
return
if self._stateless_world_group_port_list:
return
if not self._coord_store_port:
return self.get_next_dp_init_port(), None
num_world_groups = 1
dp_size = self.data_parallel_size
ep_size = self.data_parallel_size * self.world_size_across_dp
num_dp_groups = max(1, self.world_size_across_dp // dp_size)
num_ep_groups = max(1, self.world_size_across_dp // ep_size)
num_eplb_groups = num_ep_groups
total_stateless_ports = (
num_world_groups + num_dp_groups + num_ep_groups + num_eplb_groups
) * 3
num_dp_master_ports = 5
from vllm.distributed.utils import get_cached_tcp_store_client
all_ports = get_open_ports_list(total_stateless_ports + num_dp_master_ports)
store = get_cached_tcp_store_client(
self.data_parallel_master_ip, self._coord_store_port
)
self._data_parallel_master_port_list = all_ports[-num_dp_master_ports:]
self.data_parallel_master_port = self._data_parallel_master_port_list.pop()
all_ports = all_ports[:-num_dp_master_ports]
self._stateless_world_group_port_list = [
all_ports[i : i + 3] for i in range(0, num_world_groups * 3, 3)
]
start_idx = num_world_groups * 3
self._stateless_dp_group_port_list = [
all_ports[i : i + 3]
for i in range(start_idx, start_idx + num_dp_groups * 3, 3)
]
start_idx += num_dp_groups * 3
self._stateless_ep_group_port_list = [
all_ports[i : i + 3]
for i in range(start_idx, start_idx + num_ep_groups * 3, 3)
]
start_idx += num_ep_groups * 3
self._stateless_eplb_group_port_list = [
all_ports[i : i + 3]
for i in range(start_idx, start_idx + num_eplb_groups * 3, 3)
]
def get_next_stateless_world_group_port(self) -> list[int]:
return self._stateless_world_group_port_list.pop()
def get_next_stateless_dp_group_port(self) -> list[int]:
return self._stateless_dp_group_port_list.pop()
def get_next_stateless_ep_group_port(self) -> list[int]:
return self._stateless_ep_group_port_list.pop()
def get_next_stateless_eplb_group_port(self) -> list[int]:
return self._stateless_eplb_group_port_list.pop()
key = "dp_master_port"
if self.data_parallel_rank == 0:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((self.data_parallel_master_ip, 0))
s.listen()
port = s.getsockname()[1]
store.set(key, str(port).encode())
return port, s
else:
return int(store.get(key).decode()), None
@overload
def stateless_init_dp_group(
@@ -553,14 +497,16 @@ class ParallelConfig:
last_exc: Exception | None = None
for _ in range(max_retries):
try:
port, listen_socket = self._pick_stateless_dp_port()
# use gloo since the engine process might not have cuda device
return stateless_init_torch_distributed_process_group(
self.data_parallel_master_ip,
self.get_next_dp_init_port(),
port,
self.data_parallel_rank,
self.data_parallel_size,
backend="gloo",
return_store=return_store,
listen_socket=listen_socket,
)
except DistNetworkError as e:
# We only want to retry when the root cause is EADDRINUSE.