[BugFix] Use file:// rendezvous for single-node executors to eliminate startup port races (#50999)

Signed-off-by: Parsa Mazaheri <p_mazaheri@apple.com>
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Parsa Mazaheri <p_mazaheri@apple.com>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-08-09 22:00:04 -04:00
committed by GitHub
co-authored by Parsa Mazaheri aoshen02 Claude
parent 0820125ae9
commit 11ba93f364
6 changed files with 142 additions and 13 deletions
+116
View File
@@ -0,0 +1,116 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import torch
import vllm.config
import vllm.distributed.parallel_state as parallel_state
from vllm.distributed.parallel_state import (
destroy_distributed_environment,
init_distributed_environment,
)
def _file_store_worker(rank, init_method, result_queue):
try:
init_distributed_environment(
world_size=2,
rank=rank,
local_rank=rank,
backend="gloo",
distributed_init_method=init_method,
)
result = torch.tensor(rank)
torch.distributed.all_reduce(result)
result_queue.put(result.item())
finally:
destroy_distributed_environment()
def test_file_store_preserves_raw_path(tmp_path, monkeypatch):
monkeypatch.setenv("VLLM_DISTRIBUTED_USE_SPLIT_GROUP", "0")
store_path = tmp_path / "store#?"
try:
init_distributed_environment(
world_size=1,
rank=0,
local_rank=0,
backend="gloo",
distributed_init_method=f"file://{store_path}",
)
assert store_path.exists()
finally:
destroy_distributed_environment()
def test_file_store_rendezvouses_local_processes(tmp_path, monkeypatch):
monkeypatch.setenv("VLLM_DISTRIBUTED_USE_SPLIT_GROUP", "0")
context = torch.multiprocessing.get_context("spawn")
result_queue = context.SimpleQueue()
torch.multiprocessing.spawn(
_file_store_worker,
args=(f"file://{tmp_path / 'store#?'}", result_queue),
nprocs=2,
join=True,
)
assert sorted(result_queue.get() for _ in range(2)) == [1, 1]
def test_multinode_overrides_file_store_rendezvous(monkeypatch):
monkeypatch.setenv("VLLM_DISTRIBUTED_USE_SPLIT_GROUP", "0")
parallel_config = SimpleNamespace(
distributed_executor_backend="mp",
nnodes=2,
data_parallel_size=1,
enable_elastic_ep=False,
data_parallel_rank=0,
world_size_across_dp=2,
master_addr="192.0.2.1",
master_port=29500,
nnodes_within_dp=2,
)
config = SimpleNamespace(parallel_config=parallel_config)
calls = []
original_world = parallel_state._WORLD
original_node_count = parallel_state._NODE_COUNT
monkeypatch.setattr(vllm.config, "get_current_vllm_config_or_none", lambda: config)
monkeypatch.setattr(torch.distributed, "is_initialized", lambda: False)
monkeypatch.setattr(torch.distributed, "is_backend_available", lambda _: True)
monkeypatch.setattr(torch.distributed, "get_world_size", lambda: 2)
monkeypatch.setattr(
torch.distributed, "init_process_group", lambda **kwargs: calls.append(kwargs)
)
monkeypatch.setattr(
parallel_state,
"init_world_group",
lambda *_: SimpleNamespace(world_size=2, cpu_group=None),
)
try:
init_distributed_environment(
world_size=2,
rank=0,
local_rank=0,
backend="gloo",
distributed_init_method="file:///tmp/vllm_dist_test",
)
finally:
parallel_state._WORLD = original_world
parallel_state._NODE_COUNT = original_node_count
assert calls == [
{
"backend": "gloo",
"init_method": "tcp://192.0.2.1:29500",
"store": None,
"world_size": 2,
"rank": 0,
"timeout": None,
}
]
+11 -2
View File
@@ -1489,6 +1489,7 @@ def _init_process_group_for_split_group(
*,
backend: str,
distributed_init_method: str,
store: Store | None,
world_size: int,
rank: int,
local_rank: int,
@@ -1512,7 +1513,8 @@ def _init_process_group_for_split_group(
device_id = None
torch.distributed.init_process_group(
backend=init_backend,
init_method=distributed_init_method,
init_method=distributed_init_method if store is None else None,
store=store,
world_size=world_size,
rank=rank,
timeout=timeout,
@@ -1654,6 +1656,11 @@ def init_distributed_environment(
"Fallback Gloo backend is not available."
)
backend = "gloo"
store = None
if distributed_init_method.startswith("file://"):
store = torch.distributed.FileStore(
distributed_init_method.removeprefix("file://"), world_size
)
if envs.VLLM_DISTRIBUTED_USE_SPLIT_GROUP:
# split_group needs local_rank early to compute device_id for
# the eager init. local_rank is not available in torch
@@ -1667,6 +1674,7 @@ def init_distributed_environment(
_init_process_group_for_split_group(
backend=backend,
distributed_init_method=distributed_init_method,
store=store,
world_size=world_size,
rank=rank,
local_rank=local_rank,
@@ -1676,7 +1684,8 @@ def init_distributed_environment(
# this backend is used for WORLD
torch.distributed.init_process_group(
backend=backend,
init_method=distributed_init_method,
init_method=distributed_init_method if store is None else None,
store=store,
world_size=world_size,
rank=rank,
timeout=timeout,
+5
View File
@@ -5,6 +5,7 @@ import ipaddress
import os
import socket
import sys
import tempfile
import warnings
from collections.abc import (
Iterator,
@@ -131,6 +132,10 @@ def get_distributed_init_method(ip: str, port: int) -> str:
return get_tcp_uri(ip, port)
def get_file_store_init_method() -> str:
return f"file://{tempfile.gettempdir()}/vllm_dist_{uuid4().hex}"
def get_tcp_uri(ip: str, port: int) -> str:
if is_valid_ipv6_address(ip):
return f"tcp://[{ip}]:{port}"
+2 -7
View File
@@ -46,10 +46,8 @@ from vllm.platforms import current_platform
from vllm.tracing import instrument, maybe_init_worker_tracer
from vllm.utils import numa_utils
from vllm.utils.network_utils import (
get_distributed_init_method,
get_file_store_init_method,
get_ip,
get_loopback_ip,
get_open_port,
)
from vllm.utils.ompmultiprocessing import OMPProcessManager
from vllm.utils.system_utils import (
@@ -129,10 +127,7 @@ class MultiprocExecutor(Executor):
set_multiprocessing_worker_envs(self.local_world_size)
# use the loopback address get_loopback_ip() for communication.
distributed_init_method = get_distributed_init_method(
get_loopback_ip(), get_open_port()
)
distributed_init_method = get_file_store_init_method()
self.rpc_broadcast_mq: MessageQueue | None = None
scheduler_output_handle: Handle | None = None
# Initialize worker and set up message queues for SchedulerOutputs
+2 -2
View File
@@ -12,7 +12,7 @@ import torch.distributed as dist
import vllm.envs as envs
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_distributed_init_method, get_ip, get_open_port
from vllm.utils.network_utils import get_file_store_init_method
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
from vllm.v1.executor.abstract import Executor
from vllm.v1.executor.vllm_net_devices import set_worker_net_device
@@ -70,7 +70,7 @@ class UniProcExecutor(Executor):
def _distributed_args(self) -> tuple[str, int, int]:
"""Return (distributed_init_method, rank, local_rank)."""
distributed_init_method = get_distributed_init_method(get_ip(), get_open_port())
distributed_init_method = get_file_store_init_method()
# set local rank as the device index if specified
device_info = self.vllm_config.device_config.device.__str__().split(":")
local_rank = int(device_info[1]) if len(device_info) > 1 else 0
+6 -2
View File
@@ -157,8 +157,12 @@ class CPUWorker(Worker):
torch.set_num_threads = skip_set_num_threads
# Note: unique identifier for creating allreduce shared memory
os.environ["VLLM_DIST_IDENT"] = self.distributed_init_method.split(":")[-1]
init_method = self.distributed_init_method
os.environ["VLLM_DIST_IDENT"] = (
os.path.basename(init_method.removeprefix("file://"))
if init_method.startswith("file://")
else init_method.split(":")[-1]
)
# Initialize the distributed environment.
init_worker_distributed_environment(
self.vllm_config,