diff --git a/tests/test_config.py b/tests/test_config.py index bbb12e3917..5ba3e20c3a 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -5,6 +5,7 @@ import json import logging import os from dataclasses import MISSING, Field, asdict, dataclass, field +from datetime import timedelta from pathlib import Path from types import SimpleNamespace from typing import cast @@ -1464,6 +1465,20 @@ def test_all2all_backend_has_portable_default(): assert ParallelConfig().all2all_backend == "allgather_reducescatter" +def test_dp_group_uses_configured_timeout_without_current_config(monkeypatch): + monkeypatch.setattr(vllm_config_module, "_current_vllm_config", None) + config = ParallelConfig(cpu_distributed_timeout_seconds=30) + with ( + patch( + "vllm.distributed.utils.rendezvous", + return_value=iter([(torch.distributed.HashStore(), 0, 1)]), + ), + patch("vllm.distributed.utils.init_gloo_process_group") as init_group, + ): + config.stateless_init_dp_group() + assert init_group.call_args.kwargs["timeout"] == timedelta(seconds=30) + + @pytest.mark.parametrize( "dp_size, across_dp, expected", [(1, False, 4), (1, True, 4), (2, False, 4), (2, True, 8)], diff --git a/vllm/config/parallel.py b/vllm/config/parallel.py index 3929d3be12..b9136cfbb2 100644 --- a/vllm/config/parallel.py +++ b/vllm/config/parallel.py @@ -4,6 +4,7 @@ import os import socket from collections.abc import Callable +from datetime import timedelta from typing import TYPE_CHECKING, Any, Literal, overload import regex as re @@ -613,6 +614,11 @@ class ParallelConfig: """ return self.data_parallel_external_lb or self.data_parallel_hybrid_lb + @property + def cpu_distributed_timeout(self) -> timedelta | None: + seconds = self.cpu_distributed_timeout_seconds + return timedelta(seconds=seconds) if seconds is not None else None + def get_next_dp_init_port(self) -> int: """We might need to initialize process groups in multiple processes that is related to data parallelism, @@ -694,6 +700,7 @@ class ParallelConfig: backend="gloo", return_store=return_store, listen_socket=listen_socket, + timeout=self.cpu_distributed_timeout, ) except DistNetworkError as e: # We only want to retry when the root cause is EADDRINUSE. diff --git a/vllm/distributed/stateless_coordinator.py b/vllm/distributed/stateless_coordinator.py index 6ce2aa89ac..40957aac3e 100644 --- a/vllm/distributed/stateless_coordinator.py +++ b/vllm/distributed/stateless_coordinator.py @@ -17,6 +17,8 @@ from vllm.distributed.parallel_state import ( ) from vllm.distributed.utils import ( StatelessProcessGroup, + get_cpu_distributed_timeout_or_none, + get_distributed_timeout_or_none, stateless_destroy_torch_distributed_process_group, stateless_init_torch_distributed_process_group, ) @@ -104,6 +106,10 @@ class StatelessGroupCoordinator(GroupCoordinator): backend = str(torch_distributed_backend) self.backend = backend + cpu_timeout = get_cpu_distributed_timeout_or_none() + device_timeout = ( + cpu_timeout if backend == "gloo" else get_distributed_timeout_or_none() + ) for idx, ranks in enumerate(group_ranks): if self.rank in ranks: self.ranks = ranks @@ -130,6 +136,7 @@ class StatelessGroupCoordinator(GroupCoordinator): backend=backend, group_name=f"{self.unique_name}_device", listen_socket=socks[0] if socks else None, + timeout=device_timeout, ) cpu_group = stateless_init_torch_distributed_process_group( host=host, @@ -139,6 +146,7 @@ class StatelessGroupCoordinator(GroupCoordinator): backend="gloo", group_name=f"{self.unique_name}_cpu", listen_socket=socks[1] if socks else None, + timeout=cpu_timeout, ) tcp_store_group = StatelessProcessGroup.create( host=host, diff --git a/vllm/distributed/utils.py b/vllm/distributed/utils.py index b32269ec2b..c780316bb4 100644 --- a/vllm/distributed/utils.py +++ b/vllm/distributed/utils.py @@ -530,8 +530,7 @@ def get_cpu_distributed_timeout_or_none() -> timedelta | None: vllm_config = get_current_vllm_config_or_none() if vllm_config is None: return None - timeout_seconds = vllm_config.parallel_config.cpu_distributed_timeout_seconds - return timedelta(seconds=timeout_seconds) if timeout_seconds is not None else None + return vllm_config.parallel_config.cpu_distributed_timeout def get_distributed_timeout_or_none() -> timedelta | None: @@ -582,6 +581,7 @@ def stateless_init_torch_distributed_process_group( group_name: str | None = None, return_store: bool = False, listen_socket: socket.socket | None = None, + timeout: timedelta | None = None, ) -> ProcessGroup | tuple[ProcessGroup, Store]: """A replacement for `torch.distributed.init_process_group` that does not pollute the global state. The created ProcessGroup object can be used for @@ -617,18 +617,13 @@ def stateless_init_torch_distributed_process_group( is skipped and a ``TCPStore`` server is created directly using the pre-bound socket. This is useful for eliminating TOCTOU races between port allocation and binding. + + If *timeout* is None, use PyTorch's default for the backend. """ init_method = get_tcp_uri(host, port) backend = Backend(backend) # it is basically string - timeout = _get_default_timeout(backend) - if backend == "gloo": - gloo_timeout = get_cpu_distributed_timeout_or_none() - if gloo_timeout is not None: - timeout = gloo_timeout - else: - device_timeout = get_distributed_timeout_or_none() - if device_timeout is not None: - timeout = device_timeout + if timeout is None: + timeout = _get_default_timeout(backend) if listen_socket is not None: store = create_tcp_store( diff --git a/vllm/v1/fault_tolerance/engine_core_sentinel.py b/vllm/v1/fault_tolerance/engine_core_sentinel.py index 1d82dd9f74..3ab493e249 100644 --- a/vllm/v1/fault_tolerance/engine_core_sentinel.py +++ b/vllm/v1/fault_tolerance/engine_core_sentinel.py @@ -9,7 +9,6 @@ from typing import TYPE_CHECKING import msgspec -from vllm.config import set_current_vllm_config from vllm.distributed import stateless_destroy_torch_distributed_process_group from vllm.distributed.utils import stateless_init_torch_distributed_process_group from vllm.logger import init_logger @@ -123,8 +122,7 @@ class EngineCoreSentinel: engine = self.engine executor = engine.model_executor - with set_current_vllm_config(engine.vllm_config): - ft_request.params.update(self._reinit_dp_group()) + ft_request.params.update(self._reinit_dp_group()) if hasattr(engine, "step_counter"): engine.step_counter = 0 @@ -165,6 +163,7 @@ class EngineCoreSentinel: parallel_config.data_parallel_size, backend="gloo", return_store=True, + timeout=parallel_config.cpu_distributed_timeout, ) ) return {"new_stateless_dp_group_ports": worker_ports} diff --git a/vllm/v1/worker/sentinel/gpu_worker_sentinel.py b/vllm/v1/worker/sentinel/gpu_worker_sentinel.py index 80050cf9d3..9d06a40990 100644 --- a/vllm/v1/worker/sentinel/gpu_worker_sentinel.py +++ b/vllm/v1/worker/sentinel/gpu_worker_sentinel.py @@ -65,6 +65,7 @@ class WorkerSentinel: self.dp_rank, self.dp_size, backend="gloo", + timeout=self.worker.parallel_config.cpu_distributed_timeout, ) def _clean_worker_state(self):