diff --git a/tests/distributed/test_ray_v2_executor.py b/tests/distributed/test_ray_v2_executor.py index 398ee30c06..2ad220b507 100644 --- a/tests/distributed/test_ray_v2_executor.py +++ b/tests/distributed/test_ray_v2_executor.py @@ -17,6 +17,7 @@ import ray from vllm import LLM from vllm.config import VllmConfig from vllm.engine.arg_utils import EngineArgs +from vllm.v1.executor import ray_executor_v2 from vllm.v1.executor.ray_executor_v2 import RayExecutorV2 pytestmark = pytest.mark.usefixtures("enable_ray_v2_backend") @@ -95,6 +96,43 @@ def assert_executor(executor, tp_size, pp_size): assert handle.node_id is not None +def test_select_tcpstore_port_seeds_disjoint_windows(monkeypatch): + """Co-located DP engines scan distinct, adjacent port windows, so two + engines on a node cannot pick the same TCPStore port.""" + requested = [] + + def fake_get_open_port(start_port, max_attempts): + requested.append((start_port, max_attempts)) + return start_port + + monkeypatch.setattr(ray_executor_v2, "_get_open_port", fake_get_open_port) + + ports = [ + RayExecutorV2._select_tcpstore_port(rank, master_port=29500) + for rank in range(4) + ] + + assert requested == [(29600, 32), (29632, 32), (29664, 32), (29696, 32)] + assert len(set(ports)) == 4 + + +def test_select_tcpstore_port_non_dp_uses_random(monkeypatch): + """A non-DP engine has no local rank and uses a random port.""" + monkeypatch.setattr(ray_executor_v2, "get_open_port", lambda: 54321) + assert RayExecutorV2._select_tcpstore_port(None, master_port=29500) == 54321 + + +def test_select_tcpstore_port_full_window_uses_random(monkeypatch): + """A fully occupied window falls back to a random port.""" + + def raise_full(start_port, max_attempts): + raise RuntimeError("no open port") + + monkeypatch.setattr(ray_executor_v2, "_get_open_port", raise_full) + monkeypatch.setattr(ray_executor_v2, "get_open_port", lambda: 54321) + assert RayExecutorV2._select_tcpstore_port(0, master_port=29500) == 54321 + + @pytest.mark.parametrize("tp_size, pp_size", [(1, 1), (2, 1), (4, 1), (2, 2)]) def test_ray_v2_executor(tp_size, pp_size): """Validate RayExecutorV2 with various TP/PP configs.""" diff --git a/vllm/v1/executor/ray_executor_v2.py b/vllm/v1/executor/ray_executor_v2.py index d50f06cc62..33c3865157 100644 --- a/vllm/v1/executor/ray_executor_v2.py +++ b/vllm/v1/executor/ray_executor_v2.py @@ -17,6 +17,7 @@ from vllm.distributed.device_communicators.shm_broadcast import ( from vllm.logger import init_logger from vllm.platforms import current_platform from vllm.utils.network_utils import ( + _get_open_port, get_distributed_init_method, get_open_port, ) @@ -259,6 +260,25 @@ class RayExecutorV2(MultiprocExecutor): return {"num_gpus": num_devices} return {"num_gpus": 0, "resources": {device_key: num_devices}} + @staticmethod + def _select_tcpstore_port(local_dp_rank: int | None, master_port: int) -> int: + """Pick the torch.distributed TCPStore port for this engine. + + Co-located DP engines choosing this port with a shared random search + collide intermittently. Seeding by node-local DP rank gives each a + disjoint window. Non-DP engines and full windows fall back to a + random port. + """ + if local_dp_rank is None: + return get_open_port() + # Offset past the DP master port reserved range, one window per rank. + window = 32 + start_port = master_port + 100 + local_dp_rank * window + try: + return _get_open_port(start_port=start_port, max_attempts=window) + except RuntimeError: + return get_open_port() + def _init_executor(self) -> None: """Initialize the RayExecutorV2 executor.""" self._finalizer = weakref.finalize(self, self.shutdown) @@ -308,7 +328,12 @@ class RayExecutorV2(MultiprocExecutor): # The TCPStore server runs on rank 0's node, so all workers # must be able to reach this address. dist_ip = bundle_assignments[0]["node_ip"] - distributed_init_method = get_distributed_init_method(dist_ip, get_open_port()) + parallel_config = self.vllm_config.parallel_config + port = self._select_tcpstore_port( + parallel_config.data_parallel_rank_local, + parallel_config.data_parallel_master_port, + ) + distributed_init_method = get_distributed_init_method(dist_ip, port) # Step 4: Create broadcast MessageQueue. # Workers on the driver node use shared memory; the rest use TCP.