[BugFix] Bind RayExecutorV2 TCPStore before publishing its port (#50969)

Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
aoshen02
2026-08-28 12:27:22 -07:00
committed by GitHub
co-authored by aoshen02 Claude OpenAI Codex Andreas Karatzas
parent 2f1cba799e
commit 67e86d1e6f
2 changed files with 57 additions and 105 deletions
+26 -58
View File
@@ -7,12 +7,17 @@ Validates executor initialization, placement group support, RPC calls,
and distributed execution with various TP/PP configurations.
"""
import errno
import gc
import socket
import threading
from types import SimpleNamespace
from unittest.mock import patch
from urllib.parse import urlsplit
import pytest
import ray
from torch.distributed import TCPStore
from vllm import LLM
from vllm.config import VllmConfig
@@ -102,67 +107,30 @@ def assert_executor(executor, tp_size, pp_size):
assert handle.node_id is not None
def test_select_tcpstore_port_seeds_disjoint_windows(monkeypatch):
"""Co-located DP engines scan distinct, adjacent port windows, so two
engines on a node cannot pick the same TCPStore port."""
requested = []
def test_dist_init_store_owns_port(monkeypatch):
"""The rank-0 actor keeps the selected TCPStore port bound."""
worker = ray_executor_v2.RayWorkerProc.__new__(ray_executor_v2.RayWorkerProc)
worker._parallel_config = SimpleNamespace(world_size=1)
monkeypatch.setattr(ray.util, "get_node_ip_address", lambda: "127.0.0.1")
def fake_get_open_port(start_port, max_attempts):
requested.append((start_port, max_attempts))
return start_port
init_method = worker.create_dist_init_method()
port = urlsplit(init_method).port
assert port is not None and port != 0
monkeypatch.setattr(ray_executor_v2, "_get_open_port", fake_get_open_port)
with socket.socket() as contender, pytest.raises(OSError) as exc_info:
contender.bind(("127.0.0.1", port))
assert exc_info.value.errno == errno.EADDRINUSE
ports = [
RayExecutorV2._select_tcpstore_port(rank, master_port=29500)
for rank in range(4)
]
assert requested == [(29600, 32), (29632, 32), (29664, 32), (29696, 32)]
assert len(set(ports)) == 4
def test_select_tcpstore_port_non_dp_uses_random(monkeypatch):
"""A non-DP engine has no local rank and uses a random port."""
monkeypatch.setattr(ray_executor_v2, "get_open_port", lambda: 54321)
assert RayExecutorV2._select_tcpstore_port(None, master_port=29500) == 54321
def test_colocated_tcpstore_ports_differ():
"""Two or more replicas on one host can collide on the same TCPStore port.
Their ports must differ and never be a low system port.
"""
ports = []
real_select = RayExecutorV2._select_tcpstore_port
def record(local_dp_rank, master_port):
port = real_select(local_dp_rank, master_port)
ports.append(port)
return port
with patch.object(RayExecutorV2, "_select_tcpstore_port", staticmethod(record)):
first = RayExecutorV2(vllm_config=create_vllm_config())
try:
second = RayExecutorV2(vllm_config=create_vllm_config())
try:
assert len(ports) == 2
assert min(ports) >= 1024
assert ports[0] != ports[1]
finally:
second.shutdown()
finally:
first.shutdown()
def test_select_tcpstore_port_full_window_uses_random(monkeypatch):
"""A fully occupied window falls back to a random port."""
def raise_full(start_port, max_attempts):
raise RuntimeError("no open port")
monkeypatch.setattr(ray_executor_v2, "_get_open_port", raise_full)
monkeypatch.setattr(ray_executor_v2, "get_open_port", lambda: 54321)
assert RayExecutorV2._select_tcpstore_port(0, master_port=29500) == 54321
reused_store = TCPStore(
"127.0.0.1",
port,
world_size=1,
is_master=True,
wait_for_workers=False,
multi_tenant=True,
)
worker._dist_init_store.set("key", "value")
assert reused_store.get("key") == b"value"
@pytest.mark.parametrize("tp_size, pp_size", [(1, 1), (2, 1), (4, 1), (2, 2)])
+31 -47
View File
@@ -8,6 +8,8 @@ from collections import defaultdict, deque
from dataclasses import dataclass
from typing import Any
from torch.distributed import TCPStore
import vllm.envs as envs
from vllm.config import VllmConfig
from vllm.distributed.device_communicators.shm_broadcast import (
@@ -17,9 +19,7 @@ from vllm.distributed.device_communicators.shm_broadcast import (
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.network_utils import (
_get_open_port,
get_distributed_init_method,
get_open_port,
)
from vllm.v1.executor.multiproc_executor import (
FutureWrapper,
@@ -106,22 +106,35 @@ class RayWorkerProc(WorkerProc):
self,
vllm_config: VllmConfig,
rank: int,
distributed_init_method: str,
input_shm_handle: Handle,
is_driver_worker: bool,
is_driver_node: bool = False,
):
# Defer WorkerProc.__init__ until GPU IDs are known.
self._is_driver_node = is_driver_node
self._parallel_config = vllm_config.parallel_config
self._init_kwargs = dict(
vllm_config=vllm_config,
rank=rank,
distributed_init_method=distributed_init_method,
input_shm_handle=input_shm_handle,
shared_worker_lock=None,
is_driver_worker=is_driver_worker,
)
def create_dist_init_method(self) -> str:
host = ray.util.get_node_ip_address()
store = TCPStore(
host_name=host,
port=0,
world_size=self._parallel_config.world_size,
is_master=True,
wait_for_workers=False,
multi_tenant=True,
)
# Keep the bound server alive until init_process_group reuses it.
self._dist_init_store = store
return get_distributed_init_method(host, store.port)
def get_node_and_physical_gpu_ids(self) -> tuple[str, list[int]]:
"""Return (node_id, physical_gpu_ids) assigned to this actor by Ray."""
node_id = ray.get_runtime_context().get_node_id()
@@ -140,6 +153,7 @@ class RayWorkerProc(WorkerProc):
self,
local_rank: int,
env_vars: dict[str, str],
distributed_init_method: str,
driver_env_vars: dict[str, str] | None = None,
assigned_physical_gpu_ids: list[int] | None = None,
) -> None:
@@ -163,6 +177,7 @@ class RayWorkerProc(WorkerProc):
assigned_physical_gpu_ids
)
self._init_kwargs["distributed_init_method"] = distributed_init_method
self.local_rank = local_rank
super().__init__(
local_rank=local_rank,
@@ -260,25 +275,6 @@ class RayExecutorV2(MultiprocExecutor):
return {"num_gpus": num_devices}
return {"num_gpus": 0, "resources": {device_key: num_devices}}
@staticmethod
def _select_tcpstore_port(local_dp_rank: int | None, master_port: int) -> int:
"""Pick the torch.distributed TCPStore port for this engine.
Co-located DP engines choosing this port with a shared random search
collide intermittently. Seeding by node-local DP rank gives each a
disjoint window. Non-DP engines and full windows fall back to a
random port.
"""
if local_dp_rank is None:
return get_open_port()
# Offset past the DP master port reserved range, one window per rank.
window = 32
start_port = master_port + 100 + local_dp_rank * window
try:
return _get_open_port(start_port=start_port, max_attempts=window)
except RuntimeError:
return get_open_port()
def _init_executor(self) -> None:
"""Initialize the RayExecutorV2 executor."""
self._finalizer = weakref.finalize(self, self.shutdown)
@@ -324,22 +320,7 @@ class RayExecutorV2(MultiprocExecutor):
}
)
# Step 3: Resolve the IP for torch.distributed TCPStore.
# The TCPStore server runs on rank 0's node, so all workers
# must be able to reach this address.
dist_ip = bundle_assignments[0]["node_ip"]
parallel_config = self.vllm_config.parallel_config
local_dp_rank = (
parallel_config.data_parallel_rank_local
if parallel_config.data_parallel_size > 1
else None
)
port = self._select_tcpstore_port(
local_dp_rank, parallel_config.data_parallel_master_port
)
distributed_init_method = get_distributed_init_method(dist_ip, port)
# Step 4: Create broadcast MessageQueue.
# Step 3: Create broadcast MessageQueue.
# Workers on the driver node use shared memory; the rest use TCP.
max_chunk_bytes = envs.VLLM_MQ_MAX_CHUNK_BYTES_MB * 1024 * 1024
n_local = sum(1 for a in bundle_assignments if a["node_id"] == driver_node)
@@ -351,9 +332,9 @@ class RayExecutorV2(MultiprocExecutor):
)
scheduler_output_handle = self.rpc_broadcast_mq.export_handle()
# Step 5: Spawn RayWorkerProc actors into PG bundles (deferred init).
# Step 4: Spawn RayWorkerProc actors into PG bundles (deferred init).
# Workers are created lightweight here; full initialization happens
# in Step 7 after GPU IDs are discovered.
# in Step 6 after GPU IDs are discovered.
self.ray_worker_handles: list[RayWorkerHandle] = []
instance_id = self.vllm_config.instance_id
@@ -405,7 +386,6 @@ class RayExecutorV2(MultiprocExecutor):
.remote(
vllm_config=self.vllm_config,
rank=bundle["rank"],
distributed_init_method=distributed_init_method,
input_shm_handle=scheduler_output_handle,
is_driver_worker=is_driver_worker,
is_driver_node=is_driver_node,
@@ -421,7 +401,7 @@ class RayExecutorV2(MultiprocExecutor):
)
self.ray_worker_handles.append(handle)
# Step 6: Discover physical GPU IDs assigned to each worker via Ray
# Step 5: Discover physical GPU IDs assigned to each worker via Ray
# runtime context.
worker_node_and_physical_gpu_ids = ray.get(
[
@@ -440,8 +420,11 @@ class RayExecutorV2(MultiprocExecutor):
for node_id, physical_gpu_ids in node_physical_gpu_ids.items():
node_physical_gpu_ids[node_id] = sorted(physical_gpu_ids)
# Step 7: Initialize workers with local logical ranks and the
# Step 6: Initialize workers with local logical ranks and the
# logical-to-physical GPU mapping discovered from Ray placement.
distributed_init_method = ray.get(
self.ray_worker_handles[0].actor.create_dist_init_method.remote()
)
init_worker_refs = []
for i, (node_id, _) in enumerate(worker_node_and_physical_gpu_ids):
local_rank = node_workers[node_id].index(i)
@@ -452,6 +435,7 @@ class RayExecutorV2(MultiprocExecutor):
self.ray_worker_handles[i].actor.initialize_worker.remote(
local_rank,
worker_env_vars,
distributed_init_method,
self.driver_env_vars,
assigned_physical_gpu_ids=assigned_physical_gpu_ids,
)
@@ -465,7 +449,7 @@ class RayExecutorV2(MultiprocExecutor):
)
ray.get(init_worker_refs)
# Step 8: Collect response MQ handles
# Step 7: Collect response MQ handles
init_results = ray.get(
[h.actor.wait_for_init.remote() for h in self.ray_worker_handles]
)
@@ -478,12 +462,12 @@ class RayExecutorV2(MultiprocExecutor):
MessageQueue.create_from_handle(result["handle"], 0)
)
# Step 9: Start run() before wait_until_ready() to avoid
# Step 8: Start run() before wait_until_ready() to avoid
# deadlock — workers send subscriptions inside run().
for handle in self.ray_worker_handles:
handle.run()
# Step 10: wait_until_ready() barrier
# Step 9: wait_until_ready() barrier
self.rpc_broadcast_mq.wait_until_ready()
for response_mq in self.response_mqs:
response_mq.wait_until_ready()