[Bugfix][FT] Pass stateless process-group timeouts explicitly (#58611)

Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Tyler Michael Smith
2026-09-29 03:14:02 +00:00
committed by GitHub
co-authored by Codex
parent cf971b419a
commit 4406874e7c
6 changed files with 39 additions and 14 deletions
+15
View File
@@ -5,6 +5,7 @@ import json
import logging
import os
from dataclasses import MISSING, Field, asdict, dataclass, field
from datetime import timedelta
from pathlib import Path
from types import SimpleNamespace
from typing import cast
@@ -1464,6 +1465,20 @@ def test_all2all_backend_has_portable_default():
assert ParallelConfig().all2all_backend == "allgather_reducescatter"
def test_dp_group_uses_configured_timeout_without_current_config(monkeypatch):
monkeypatch.setattr(vllm_config_module, "_current_vllm_config", None)
config = ParallelConfig(cpu_distributed_timeout_seconds=30)
with (
patch(
"vllm.distributed.utils.rendezvous",
return_value=iter([(torch.distributed.HashStore(), 0, 1)]),
),
patch("vllm.distributed.utils.init_gloo_process_group") as init_group,
):
config.stateless_init_dp_group()
assert init_group.call_args.kwargs["timeout"] == timedelta(seconds=30)
@pytest.mark.parametrize(
"dp_size, across_dp, expected",
[(1, False, 4), (1, True, 4), (2, False, 4), (2, True, 8)],
+7
View File
@@ -4,6 +4,7 @@
import os
import socket
from collections.abc import Callable
from datetime import timedelta
from typing import TYPE_CHECKING, Any, Literal, overload
import regex as re
@@ -613,6 +614,11 @@ class ParallelConfig:
"""
return self.data_parallel_external_lb or self.data_parallel_hybrid_lb
@property
def cpu_distributed_timeout(self) -> timedelta | None:
seconds = self.cpu_distributed_timeout_seconds
return timedelta(seconds=seconds) if seconds is not None else None
def get_next_dp_init_port(self) -> int:
"""We might need to initialize process groups in multiple
processes that is related to data parallelism,
@@ -694,6 +700,7 @@ class ParallelConfig:
backend="gloo",
return_store=return_store,
listen_socket=listen_socket,
timeout=self.cpu_distributed_timeout,
)
except DistNetworkError as e:
# We only want to retry when the root cause is EADDRINUSE.
@@ -17,6 +17,8 @@ from vllm.distributed.parallel_state import (
)
from vllm.distributed.utils import (
StatelessProcessGroup,
get_cpu_distributed_timeout_or_none,
get_distributed_timeout_or_none,
stateless_destroy_torch_distributed_process_group,
stateless_init_torch_distributed_process_group,
)
@@ -104,6 +106,10 @@ class StatelessGroupCoordinator(GroupCoordinator):
backend = str(torch_distributed_backend)
self.backend = backend
cpu_timeout = get_cpu_distributed_timeout_or_none()
device_timeout = (
cpu_timeout if backend == "gloo" else get_distributed_timeout_or_none()
)
for idx, ranks in enumerate(group_ranks):
if self.rank in ranks:
self.ranks = ranks
@@ -130,6 +136,7 @@ class StatelessGroupCoordinator(GroupCoordinator):
backend=backend,
group_name=f"{self.unique_name}_device",
listen_socket=socks[0] if socks else None,
timeout=device_timeout,
)
cpu_group = stateless_init_torch_distributed_process_group(
host=host,
@@ -139,6 +146,7 @@ class StatelessGroupCoordinator(GroupCoordinator):
backend="gloo",
group_name=f"{self.unique_name}_cpu",
listen_socket=socks[1] if socks else None,
timeout=cpu_timeout,
)
tcp_store_group = StatelessProcessGroup.create(
host=host,
+6 -11
View File
@@ -530,8 +530,7 @@ def get_cpu_distributed_timeout_or_none() -> timedelta | None:
vllm_config = get_current_vllm_config_or_none()
if vllm_config is None:
return None
timeout_seconds = vllm_config.parallel_config.cpu_distributed_timeout_seconds
return timedelta(seconds=timeout_seconds) if timeout_seconds is not None else None
return vllm_config.parallel_config.cpu_distributed_timeout
def get_distributed_timeout_or_none() -> timedelta | None:
@@ -582,6 +581,7 @@ def stateless_init_torch_distributed_process_group(
group_name: str | None = None,
return_store: bool = False,
listen_socket: socket.socket | None = None,
timeout: timedelta | None = None,
) -> ProcessGroup | tuple[ProcessGroup, Store]:
"""A replacement for `torch.distributed.init_process_group` that does not
pollute the global state. The created ProcessGroup object can be used for
@@ -617,18 +617,13 @@ def stateless_init_torch_distributed_process_group(
is skipped and a ``TCPStore`` server is created directly using the
pre-bound socket. This is useful for eliminating TOCTOU races
between port allocation and binding.
If *timeout* is None, use PyTorch's default for the backend.
"""
init_method = get_tcp_uri(host, port)
backend = Backend(backend) # it is basically string
timeout = _get_default_timeout(backend)
if backend == "gloo":
gloo_timeout = get_cpu_distributed_timeout_or_none()
if gloo_timeout is not None:
timeout = gloo_timeout
else:
device_timeout = get_distributed_timeout_or_none()
if device_timeout is not None:
timeout = device_timeout
if timeout is None:
timeout = _get_default_timeout(backend)
if listen_socket is not None:
store = create_tcp_store(
@@ -9,7 +9,6 @@ from typing import TYPE_CHECKING
import msgspec
from vllm.config import set_current_vllm_config
from vllm.distributed import stateless_destroy_torch_distributed_process_group
from vllm.distributed.utils import stateless_init_torch_distributed_process_group
from vllm.logger import init_logger
@@ -123,8 +122,7 @@ class EngineCoreSentinel:
engine = self.engine
executor = engine.model_executor
with set_current_vllm_config(engine.vllm_config):
ft_request.params.update(self._reinit_dp_group())
ft_request.params.update(self._reinit_dp_group())
if hasattr(engine, "step_counter"):
engine.step_counter = 0
@@ -165,6 +163,7 @@ class EngineCoreSentinel:
parallel_config.data_parallel_size,
backend="gloo",
return_store=True,
timeout=parallel_config.cpu_distributed_timeout,
)
)
return {"new_stateless_dp_group_ports": worker_ports}
@@ -65,6 +65,7 @@ class WorkerSentinel:
self.dp_rank,
self.dp_size,
backend="gloo",
timeout=self.worker.parallel_config.cpu_distributed_timeout,
)
def _clean_worker_state(self):