diff --git a/tests/distributed/test_file_store.py b/tests/distributed/test_file_store.py new file mode 100644 index 0000000000..dc840e71bc --- /dev/null +++ b/tests/distributed/test_file_store.py @@ -0,0 +1,116 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from types import SimpleNamespace + +import torch + +import vllm.config +import vllm.distributed.parallel_state as parallel_state +from vllm.distributed.parallel_state import ( + destroy_distributed_environment, + init_distributed_environment, +) + + +def _file_store_worker(rank, init_method, result_queue): + try: + init_distributed_environment( + world_size=2, + rank=rank, + local_rank=rank, + backend="gloo", + distributed_init_method=init_method, + ) + result = torch.tensor(rank) + torch.distributed.all_reduce(result) + result_queue.put(result.item()) + finally: + destroy_distributed_environment() + + +def test_file_store_preserves_raw_path(tmp_path, monkeypatch): + monkeypatch.setenv("VLLM_DISTRIBUTED_USE_SPLIT_GROUP", "0") + store_path = tmp_path / "store#?" + + try: + init_distributed_environment( + world_size=1, + rank=0, + local_rank=0, + backend="gloo", + distributed_init_method=f"file://{store_path}", + ) + assert store_path.exists() + finally: + destroy_distributed_environment() + + +def test_file_store_rendezvouses_local_processes(tmp_path, monkeypatch): + monkeypatch.setenv("VLLM_DISTRIBUTED_USE_SPLIT_GROUP", "0") + context = torch.multiprocessing.get_context("spawn") + result_queue = context.SimpleQueue() + + torch.multiprocessing.spawn( + _file_store_worker, + args=(f"file://{tmp_path / 'store#?'}", result_queue), + nprocs=2, + join=True, + ) + + assert sorted(result_queue.get() for _ in range(2)) == [1, 1] + + +def test_multinode_overrides_file_store_rendezvous(monkeypatch): + monkeypatch.setenv("VLLM_DISTRIBUTED_USE_SPLIT_GROUP", "0") + parallel_config = SimpleNamespace( + distributed_executor_backend="mp", + nnodes=2, + data_parallel_size=1, + enable_elastic_ep=False, + data_parallel_rank=0, + world_size_across_dp=2, + master_addr="192.0.2.1", + master_port=29500, + nnodes_within_dp=2, + ) + config = SimpleNamespace(parallel_config=parallel_config) + calls = [] + original_world = parallel_state._WORLD + original_node_count = parallel_state._NODE_COUNT + + monkeypatch.setattr(vllm.config, "get_current_vllm_config_or_none", lambda: config) + monkeypatch.setattr(torch.distributed, "is_initialized", lambda: False) + monkeypatch.setattr(torch.distributed, "is_backend_available", lambda _: True) + monkeypatch.setattr(torch.distributed, "get_world_size", lambda: 2) + monkeypatch.setattr( + torch.distributed, "init_process_group", lambda **kwargs: calls.append(kwargs) + ) + monkeypatch.setattr( + parallel_state, + "init_world_group", + lambda *_: SimpleNamespace(world_size=2, cpu_group=None), + ) + + try: + init_distributed_environment( + world_size=2, + rank=0, + local_rank=0, + backend="gloo", + distributed_init_method="file:///tmp/vllm_dist_test", + ) + finally: + parallel_state._WORLD = original_world + parallel_state._NODE_COUNT = original_node_count + + assert calls == [ + { + "backend": "gloo", + "init_method": "tcp://192.0.2.1:29500", + "store": None, + "world_size": 2, + "rank": 0, + "timeout": None, + } + ] diff --git a/vllm/distributed/parallel_state.py b/vllm/distributed/parallel_state.py index 149de9150d..128db903d4 100644 --- a/vllm/distributed/parallel_state.py +++ b/vllm/distributed/parallel_state.py @@ -1489,6 +1489,7 @@ def _init_process_group_for_split_group( *, backend: str, distributed_init_method: str, + store: Store | None, world_size: int, rank: int, local_rank: int, @@ -1512,7 +1513,8 @@ def _init_process_group_for_split_group( device_id = None torch.distributed.init_process_group( backend=init_backend, - init_method=distributed_init_method, + init_method=distributed_init_method if store is None else None, + store=store, world_size=world_size, rank=rank, timeout=timeout, @@ -1654,6 +1656,11 @@ def init_distributed_environment( "Fallback Gloo backend is not available." ) backend = "gloo" + store = None + if distributed_init_method.startswith("file://"): + store = torch.distributed.FileStore( + distributed_init_method.removeprefix("file://"), world_size + ) if envs.VLLM_DISTRIBUTED_USE_SPLIT_GROUP: # split_group needs local_rank early to compute device_id for # the eager init. local_rank is not available in torch @@ -1667,6 +1674,7 @@ def init_distributed_environment( _init_process_group_for_split_group( backend=backend, distributed_init_method=distributed_init_method, + store=store, world_size=world_size, rank=rank, local_rank=local_rank, @@ -1676,7 +1684,8 @@ def init_distributed_environment( # this backend is used for WORLD torch.distributed.init_process_group( backend=backend, - init_method=distributed_init_method, + init_method=distributed_init_method if store is None else None, + store=store, world_size=world_size, rank=rank, timeout=timeout, diff --git a/vllm/utils/network_utils.py b/vllm/utils/network_utils.py index 46f3342363..2f4901734b 100644 --- a/vllm/utils/network_utils.py +++ b/vllm/utils/network_utils.py @@ -5,6 +5,7 @@ import ipaddress import os import socket import sys +import tempfile import warnings from collections.abc import ( Iterator, @@ -131,6 +132,10 @@ def get_distributed_init_method(ip: str, port: int) -> str: return get_tcp_uri(ip, port) +def get_file_store_init_method() -> str: + return f"file://{tempfile.gettempdir()}/vllm_dist_{uuid4().hex}" + + def get_tcp_uri(ip: str, port: int) -> str: if is_valid_ipv6_address(ip): return f"tcp://[{ip}]:{port}" diff --git a/vllm/v1/executor/multiproc_executor.py b/vllm/v1/executor/multiproc_executor.py index 276effd282..03067315a9 100644 --- a/vllm/v1/executor/multiproc_executor.py +++ b/vllm/v1/executor/multiproc_executor.py @@ -46,10 +46,8 @@ from vllm.platforms import current_platform from vllm.tracing import instrument, maybe_init_worker_tracer from vllm.utils import numa_utils from vllm.utils.network_utils import ( - get_distributed_init_method, + get_file_store_init_method, get_ip, - get_loopback_ip, - get_open_port, ) from vllm.utils.ompmultiprocessing import OMPProcessManager from vllm.utils.system_utils import ( @@ -129,10 +127,7 @@ class MultiprocExecutor(Executor): set_multiprocessing_worker_envs(self.local_world_size) - # use the loopback address get_loopback_ip() for communication. - distributed_init_method = get_distributed_init_method( - get_loopback_ip(), get_open_port() - ) + distributed_init_method = get_file_store_init_method() self.rpc_broadcast_mq: MessageQueue | None = None scheduler_output_handle: Handle | None = None # Initialize worker and set up message queues for SchedulerOutputs diff --git a/vllm/v1/executor/uniproc_executor.py b/vllm/v1/executor/uniproc_executor.py index 3bac65bf4f..dde563ebc5 100644 --- a/vllm/v1/executor/uniproc_executor.py +++ b/vllm/v1/executor/uniproc_executor.py @@ -12,7 +12,7 @@ import torch.distributed as dist import vllm.envs as envs from vllm.logger import init_logger from vllm.platforms import current_platform -from vllm.utils.network_utils import get_distributed_init_method, get_ip, get_open_port +from vllm.utils.network_utils import get_file_store_init_method from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput from vllm.v1.executor.abstract import Executor from vllm.v1.executor.vllm_net_devices import set_worker_net_device @@ -70,7 +70,7 @@ class UniProcExecutor(Executor): def _distributed_args(self) -> tuple[str, int, int]: """Return (distributed_init_method, rank, local_rank).""" - distributed_init_method = get_distributed_init_method(get_ip(), get_open_port()) + distributed_init_method = get_file_store_init_method() # set local rank as the device index if specified device_info = self.vllm_config.device_config.device.__str__().split(":") local_rank = int(device_info[1]) if len(device_info) > 1 else 0 diff --git a/vllm/v1/worker/cpu_worker.py b/vllm/v1/worker/cpu_worker.py index 0a52ceaa3a..5ece8d514d 100644 --- a/vllm/v1/worker/cpu_worker.py +++ b/vllm/v1/worker/cpu_worker.py @@ -157,8 +157,12 @@ class CPUWorker(Worker): torch.set_num_threads = skip_set_num_threads - # Note: unique identifier for creating allreduce shared memory - os.environ["VLLM_DIST_IDENT"] = self.distributed_init_method.split(":")[-1] + init_method = self.distributed_init_method + os.environ["VLLM_DIST_IDENT"] = ( + os.path.basename(init_method.removeprefix("file://")) + if init_method.startswith("file://") + else init_method.split(":")[-1] + ) # Initialize the distributed environment. init_worker_distributed_environment( self.vllm_config,