diff --git a/tests/distributed/test_ray_v2_executor.py b/tests/distributed/test_ray_v2_executor.py index b61d422a02..0da64f9523 100644 --- a/tests/distributed/test_ray_v2_executor.py +++ b/tests/distributed/test_ray_v2_executor.py @@ -7,12 +7,17 @@ Validates executor initialization, placement group support, RPC calls, and distributed execution with various TP/PP configurations. """ +import errno import gc +import socket import threading +from types import SimpleNamespace from unittest.mock import patch +from urllib.parse import urlsplit import pytest import ray +from torch.distributed import TCPStore from vllm import LLM from vllm.config import VllmConfig @@ -102,67 +107,30 @@ def assert_executor(executor, tp_size, pp_size): assert handle.node_id is not None -def test_select_tcpstore_port_seeds_disjoint_windows(monkeypatch): - """Co-located DP engines scan distinct, adjacent port windows, so two - engines on a node cannot pick the same TCPStore port.""" - requested = [] +def test_dist_init_store_owns_port(monkeypatch): + """The rank-0 actor keeps the selected TCPStore port bound.""" + worker = ray_executor_v2.RayWorkerProc.__new__(ray_executor_v2.RayWorkerProc) + worker._parallel_config = SimpleNamespace(world_size=1) + monkeypatch.setattr(ray.util, "get_node_ip_address", lambda: "127.0.0.1") - def fake_get_open_port(start_port, max_attempts): - requested.append((start_port, max_attempts)) - return start_port + init_method = worker.create_dist_init_method() + port = urlsplit(init_method).port + assert port is not None and port != 0 - monkeypatch.setattr(ray_executor_v2, "_get_open_port", fake_get_open_port) + with socket.socket() as contender, pytest.raises(OSError) as exc_info: + contender.bind(("127.0.0.1", port)) + assert exc_info.value.errno == errno.EADDRINUSE - ports = [ - RayExecutorV2._select_tcpstore_port(rank, master_port=29500) - for rank in range(4) - ] - - assert requested == [(29600, 32), (29632, 32), (29664, 32), (29696, 32)] - assert len(set(ports)) == 4 - - -def test_select_tcpstore_port_non_dp_uses_random(monkeypatch): - """A non-DP engine has no local rank and uses a random port.""" - monkeypatch.setattr(ray_executor_v2, "get_open_port", lambda: 54321) - assert RayExecutorV2._select_tcpstore_port(None, master_port=29500) == 54321 - - -def test_colocated_tcpstore_ports_differ(): - """Two or more replicas on one host can collide on the same TCPStore port. - Their ports must differ and never be a low system port. - """ - ports = [] - real_select = RayExecutorV2._select_tcpstore_port - - def record(local_dp_rank, master_port): - port = real_select(local_dp_rank, master_port) - ports.append(port) - return port - - with patch.object(RayExecutorV2, "_select_tcpstore_port", staticmethod(record)): - first = RayExecutorV2(vllm_config=create_vllm_config()) - try: - second = RayExecutorV2(vllm_config=create_vllm_config()) - try: - assert len(ports) == 2 - assert min(ports) >= 1024 - assert ports[0] != ports[1] - finally: - second.shutdown() - finally: - first.shutdown() - - -def test_select_tcpstore_port_full_window_uses_random(monkeypatch): - """A fully occupied window falls back to a random port.""" - - def raise_full(start_port, max_attempts): - raise RuntimeError("no open port") - - monkeypatch.setattr(ray_executor_v2, "_get_open_port", raise_full) - monkeypatch.setattr(ray_executor_v2, "get_open_port", lambda: 54321) - assert RayExecutorV2._select_tcpstore_port(0, master_port=29500) == 54321 + reused_store = TCPStore( + "127.0.0.1", + port, + world_size=1, + is_master=True, + wait_for_workers=False, + multi_tenant=True, + ) + worker._dist_init_store.set("key", "value") + assert reused_store.get("key") == b"value" @pytest.mark.parametrize("tp_size, pp_size", [(1, 1), (2, 1), (4, 1), (2, 2)]) diff --git a/vllm/v1/executor/ray_executor_v2.py b/vllm/v1/executor/ray_executor_v2.py index c9772f8f20..e8e5f5107c 100644 --- a/vllm/v1/executor/ray_executor_v2.py +++ b/vllm/v1/executor/ray_executor_v2.py @@ -8,6 +8,8 @@ from collections import defaultdict, deque from dataclasses import dataclass from typing import Any +from torch.distributed import TCPStore + import vllm.envs as envs from vllm.config import VllmConfig from vllm.distributed.device_communicators.shm_broadcast import ( @@ -17,9 +19,7 @@ from vllm.distributed.device_communicators.shm_broadcast import ( from vllm.logger import init_logger from vllm.platforms import current_platform from vllm.utils.network_utils import ( - _get_open_port, get_distributed_init_method, - get_open_port, ) from vllm.v1.executor.multiproc_executor import ( FutureWrapper, @@ -106,22 +106,35 @@ class RayWorkerProc(WorkerProc): self, vllm_config: VllmConfig, rank: int, - distributed_init_method: str, input_shm_handle: Handle, is_driver_worker: bool, is_driver_node: bool = False, ): # Defer WorkerProc.__init__ until GPU IDs are known. self._is_driver_node = is_driver_node + self._parallel_config = vllm_config.parallel_config self._init_kwargs = dict( vllm_config=vllm_config, rank=rank, - distributed_init_method=distributed_init_method, input_shm_handle=input_shm_handle, shared_worker_lock=None, is_driver_worker=is_driver_worker, ) + def create_dist_init_method(self) -> str: + host = ray.util.get_node_ip_address() + store = TCPStore( + host_name=host, + port=0, + world_size=self._parallel_config.world_size, + is_master=True, + wait_for_workers=False, + multi_tenant=True, + ) + # Keep the bound server alive until init_process_group reuses it. + self._dist_init_store = store + return get_distributed_init_method(host, store.port) + def get_node_and_physical_gpu_ids(self) -> tuple[str, list[int]]: """Return (node_id, physical_gpu_ids) assigned to this actor by Ray.""" node_id = ray.get_runtime_context().get_node_id() @@ -140,6 +153,7 @@ class RayWorkerProc(WorkerProc): self, local_rank: int, env_vars: dict[str, str], + distributed_init_method: str, driver_env_vars: dict[str, str] | None = None, assigned_physical_gpu_ids: list[int] | None = None, ) -> None: @@ -163,6 +177,7 @@ class RayWorkerProc(WorkerProc): assigned_physical_gpu_ids ) + self._init_kwargs["distributed_init_method"] = distributed_init_method self.local_rank = local_rank super().__init__( local_rank=local_rank, @@ -260,25 +275,6 @@ class RayExecutorV2(MultiprocExecutor): return {"num_gpus": num_devices} return {"num_gpus": 0, "resources": {device_key: num_devices}} - @staticmethod - def _select_tcpstore_port(local_dp_rank: int | None, master_port: int) -> int: - """Pick the torch.distributed TCPStore port for this engine. - - Co-located DP engines choosing this port with a shared random search - collide intermittently. Seeding by node-local DP rank gives each a - disjoint window. Non-DP engines and full windows fall back to a - random port. - """ - if local_dp_rank is None: - return get_open_port() - # Offset past the DP master port reserved range, one window per rank. - window = 32 - start_port = master_port + 100 + local_dp_rank * window - try: - return _get_open_port(start_port=start_port, max_attempts=window) - except RuntimeError: - return get_open_port() - def _init_executor(self) -> None: """Initialize the RayExecutorV2 executor.""" self._finalizer = weakref.finalize(self, self.shutdown) @@ -324,22 +320,7 @@ class RayExecutorV2(MultiprocExecutor): } ) - # Step 3: Resolve the IP for torch.distributed TCPStore. - # The TCPStore server runs on rank 0's node, so all workers - # must be able to reach this address. - dist_ip = bundle_assignments[0]["node_ip"] - parallel_config = self.vllm_config.parallel_config - local_dp_rank = ( - parallel_config.data_parallel_rank_local - if parallel_config.data_parallel_size > 1 - else None - ) - port = self._select_tcpstore_port( - local_dp_rank, parallel_config.data_parallel_master_port - ) - distributed_init_method = get_distributed_init_method(dist_ip, port) - - # Step 4: Create broadcast MessageQueue. + # Step 3: Create broadcast MessageQueue. # Workers on the driver node use shared memory; the rest use TCP. max_chunk_bytes = envs.VLLM_MQ_MAX_CHUNK_BYTES_MB * 1024 * 1024 n_local = sum(1 for a in bundle_assignments if a["node_id"] == driver_node) @@ -351,9 +332,9 @@ class RayExecutorV2(MultiprocExecutor): ) scheduler_output_handle = self.rpc_broadcast_mq.export_handle() - # Step 5: Spawn RayWorkerProc actors into PG bundles (deferred init). + # Step 4: Spawn RayWorkerProc actors into PG bundles (deferred init). # Workers are created lightweight here; full initialization happens - # in Step 7 after GPU IDs are discovered. + # in Step 6 after GPU IDs are discovered. self.ray_worker_handles: list[RayWorkerHandle] = [] instance_id = self.vllm_config.instance_id @@ -405,7 +386,6 @@ class RayExecutorV2(MultiprocExecutor): .remote( vllm_config=self.vllm_config, rank=bundle["rank"], - distributed_init_method=distributed_init_method, input_shm_handle=scheduler_output_handle, is_driver_worker=is_driver_worker, is_driver_node=is_driver_node, @@ -421,7 +401,7 @@ class RayExecutorV2(MultiprocExecutor): ) self.ray_worker_handles.append(handle) - # Step 6: Discover physical GPU IDs assigned to each worker via Ray + # Step 5: Discover physical GPU IDs assigned to each worker via Ray # runtime context. worker_node_and_physical_gpu_ids = ray.get( [ @@ -440,8 +420,11 @@ class RayExecutorV2(MultiprocExecutor): for node_id, physical_gpu_ids in node_physical_gpu_ids.items(): node_physical_gpu_ids[node_id] = sorted(physical_gpu_ids) - # Step 7: Initialize workers with local logical ranks and the + # Step 6: Initialize workers with local logical ranks and the # logical-to-physical GPU mapping discovered from Ray placement. + distributed_init_method = ray.get( + self.ray_worker_handles[0].actor.create_dist_init_method.remote() + ) init_worker_refs = [] for i, (node_id, _) in enumerate(worker_node_and_physical_gpu_ids): local_rank = node_workers[node_id].index(i) @@ -452,6 +435,7 @@ class RayExecutorV2(MultiprocExecutor): self.ray_worker_handles[i].actor.initialize_worker.remote( local_rank, worker_env_vars, + distributed_init_method, self.driver_env_vars, assigned_physical_gpu_ids=assigned_physical_gpu_ids, ) @@ -465,7 +449,7 @@ class RayExecutorV2(MultiprocExecutor): ) ray.get(init_worker_refs) - # Step 8: Collect response MQ handles + # Step 7: Collect response MQ handles init_results = ray.get( [h.actor.wait_for_init.remote() for h in self.ray_worker_handles] ) @@ -478,12 +462,12 @@ class RayExecutorV2(MultiprocExecutor): MessageQueue.create_from_handle(result["handle"], 0) ) - # Step 9: Start run() before wait_until_ready() to avoid + # Step 8: Start run() before wait_until_ready() to avoid # deadlock — workers send subscriptions inside run(). for handle in self.ray_worker_handles: handle.run() - # Step 10: wait_until_ready() barrier + # Step 9: wait_until_ready() barrier self.rpc_broadcast_mq.wait_until_ready() for response_mq in self.response_mqs: response_mq.wait_until_ready()