From 3fa89225bc5f36ad3ada1edf0ff8c99425d7ff24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stellar=E9=B1=BC?= <2182712990@qq.com> Date: Wed, 12 Aug 2026 10:48:12 +0800 Subject: [PATCH] [CI/Build] Use file rendezvous for local distributed tests (#51652) Signed-off-by: yu-xin-c <2182712990@qq.com> --- .../passes/distributed/test_async_tp.py | 17 ++++++++++------- tests/distributed/test_dcp_a2a.py | 18 +++++++++++++----- tests/utils_/test_network_utils.py | 8 ++++++++ 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/tests/compile/passes/distributed/test_async_tp.py b/tests/compile/passes/distributed/test_async_tp.py index 861464c519..7b547c9720 100644 --- a/tests/compile/passes/distributed/test_async_tp.py +++ b/tests/compile/passes/distributed/test_async_tp.py @@ -29,7 +29,7 @@ from vllm.distributed.parallel_state import ( initialize_model_parallel, ) from vllm.platforms import current_platform -from vllm.utils.network_utils import get_open_port +from vllm.utils.network_utils import get_file_store_init_method from vllm.utils.system_utils import update_environment_variables from vllm.utils.torch_utils import set_random_seed @@ -266,7 +266,7 @@ def test_async_tp_pass_replace( dynamic: bool, ): num_processes = 2 - master_port = str(get_open_port()) + distributed_init_method = get_file_store_init_method() def run_torch_spawn(fn, nprocs): # need to use torch.mp.spawn otherwise will have problems with @@ -281,7 +281,7 @@ def test_async_tp_pass_replace( hidden_size, dtype, dynamic, - master_port, + distributed_init_method, ), nprocs=nprocs, ) @@ -314,7 +314,7 @@ def async_tp_pass_on_test_model( hidden_size: int, dtype: torch.dtype, dynamic: bool, - master_port: str = "0", + distributed_init_method: str, ): set_random_seed(0) @@ -328,13 +328,16 @@ def async_tp_pass_on_test_model( "RANK": str(local_rank), "LOCAL_RANK": str(local_rank), "WORLD_SIZE": str(world_size), - "MASTER_ADDR": "localhost", - "MASTER_PORT": master_port, } ) # initialize distributed - init_distributed_environment() + init_distributed_environment( + world_size=world_size, + rank=local_rank, + distributed_init_method=distributed_init_method, + local_rank=local_rank, + ) # configure vllm config for SequenceParallelismPass vllm_config = VllmConfig() diff --git a/tests/distributed/test_dcp_a2a.py b/tests/distributed/test_dcp_a2a.py index 6dc39a166c..dfd291742c 100644 --- a/tests/distributed/test_dcp_a2a.py +++ b/tests/distributed/test_dcp_a2a.py @@ -17,7 +17,7 @@ import torch.distributed as dist import vllm.envs as envs from vllm.config.parallel import ParallelConfig -from vllm.utils.network_utils import get_open_port +from vllm.utils.network_utils import get_file_store_init_method from vllm.utils.system_utils import update_environment_variables mp.set_start_method("spawn", force=True) @@ -76,7 +76,7 @@ def _assert_packed_a2a_close( def _distributed_run(fn, world_size: int, extra_env: dict[str, str]) -> None: - port = str(get_open_port()) + distributed_init_method = get_file_store_init_method() processes: list[mp.Process] = [] for rank in range(world_size): env = { @@ -84,8 +84,7 @@ def _distributed_run(fn, world_size: int, extra_env: dict[str, str]) -> None: "LOCAL_RANK": str(rank), "WORLD_SIZE": str(world_size), "LOCAL_WORLD_SIZE": str(world_size), - "MASTER_ADDR": "localhost", - "MASTER_PORT": port, + "DISTRIBUTED_INIT_METHOD": distributed_init_method, **extra_env, } process = mp.Process(target=fn, args=(env,)) @@ -483,14 +482,23 @@ class TestPackedA2AKernels: def _distributed_packed_a2a_worker(env: dict[str, str]) -> None: update_environment_variables(env) local_rank = int(env["LOCAL_RANK"]) + world_size = int(env["WORLD_SIZE"]) torch.accelerator.set_device_index(local_rank) if envs.VLLM_DISTRIBUTED_USE_SPLIT_GROUP: dist.init_process_group( backend="cpu:gloo,cuda:nccl", device_id=torch.device(f"cuda:{local_rank}"), + init_method=env["DISTRIBUTED_INIT_METHOD"], + rank=local_rank, + world_size=world_size, ) else: - dist.init_process_group(backend="nccl") + dist.init_process_group( + backend="nccl", + init_method=env["DISTRIBUTED_INIT_METHOD"], + rank=local_rank, + world_size=world_size, + ) use_workspace = env.get("USE_WORKSPACE") == "1" if use_workspace: from vllm.v1.worker.workspace import init_workspace_manager diff --git a/tests/utils_/test_network_utils.py b/tests/utils_/test_network_utils.py index ae63033003..02889c20f6 100644 --- a/tests/utils_/test_network_utils.py +++ b/tests/utils_/test_network_utils.py @@ -8,6 +8,7 @@ import zmq from vllm.utils import network_utils from vllm.utils.network_utils import ( + get_file_store_init_method, get_open_port, get_open_ports_list, get_tcp_uri, @@ -19,6 +20,13 @@ from vllm.utils.network_utils import ( ) +def test_get_file_store_init_method_is_unique(): + init_methods = {get_file_store_init_method() for _ in range(2)} + + assert len(init_methods) == 2 + assert all(method.startswith("file://") for method in init_methods) + + def _call_with_timeout(func, timeout: float = 10.0): """Run func in a daemon thread so a livelock regression fails the test quickly instead of hanging the suite."""