[BugFix] Bind RayExecutorV2 TCPStore before publishing its port (#50969)

Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
aoshen02
2026-08-28 12:27:22 -07:00
committed by GitHub
co-authored by aoshen02 Claude OpenAI Codex Andreas Karatzas
parent 2f1cba799e
commit 67e86d1e6f
2 changed files with 57 additions and 105 deletions
+26 -58
View File
@@ -7,12 +7,17 @@ Validates executor initialization, placement group support, RPC calls,
and distributed execution with various TP/PP configurations.
"""
import errno
import gc
import socket
import threading
from types import SimpleNamespace
from unittest.mock import patch
from urllib.parse import urlsplit
import pytest
import ray
from torch.distributed import TCPStore
from vllm import LLM
from vllm.config import VllmConfig
@@ -102,67 +107,30 @@ def assert_executor(executor, tp_size, pp_size):
assert handle.node_id is not None
def test_select_tcpstore_port_seeds_disjoint_windows(monkeypatch):
"""Co-located DP engines scan distinct, adjacent port windows, so two
engines on a node cannot pick the same TCPStore port."""
requested = []
def test_dist_init_store_owns_port(monkeypatch):
"""The rank-0 actor keeps the selected TCPStore port bound."""
worker = ray_executor_v2.RayWorkerProc.__new__(ray_executor_v2.RayWorkerProc)
worker._parallel_config = SimpleNamespace(world_size=1)
monkeypatch.setattr(ray.util, "get_node_ip_address", lambda: "127.0.0.1")
def fake_get_open_port(start_port, max_attempts):
requested.append((start_port, max_attempts))
return start_port
init_method = worker.create_dist_init_method()
port = urlsplit(init_method).port
assert port is not None and port != 0
monkeypatch.setattr(ray_executor_v2, "_get_open_port", fake_get_open_port)
with socket.socket() as contender, pytest.raises(OSError) as exc_info:
contender.bind(("127.0.0.1", port))
assert exc_info.value.errno == errno.EADDRINUSE
ports = [
RayExecutorV2._select_tcpstore_port(rank, master_port=29500)
for rank in range(4)
]
assert requested == [(29600, 32), (29632, 32), (29664, 32), (29696, 32)]
assert len(set(ports)) == 4
def test_select_tcpstore_port_non_dp_uses_random(monkeypatch):
"""A non-DP engine has no local rank and uses a random port."""
monkeypatch.setattr(ray_executor_v2, "get_open_port", lambda: 54321)
assert RayExecutorV2._select_tcpstore_port(None, master_port=29500) == 54321
def test_colocated_tcpstore_ports_differ():
"""Two or more replicas on one host can collide on the same TCPStore port.
Their ports must differ and never be a low system port.
"""
ports = []
real_select = RayExecutorV2._select_tcpstore_port
def record(local_dp_rank, master_port):
port = real_select(local_dp_rank, master_port)
ports.append(port)
return port
with patch.object(RayExecutorV2, "_select_tcpstore_port", staticmethod(record)):
first = RayExecutorV2(vllm_config=create_vllm_config())
try:
second = RayExecutorV2(vllm_config=create_vllm_config())
try:
assert len(ports) == 2
assert min(ports) >= 1024
assert ports[0] != ports[1]
finally:
second.shutdown()
finally:
first.shutdown()
def test_select_tcpstore_port_full_window_uses_random(monkeypatch):
"""A fully occupied window falls back to a random port."""
def raise_full(start_port, max_attempts):
raise RuntimeError("no open port")
monkeypatch.setattr(ray_executor_v2, "_get_open_port", raise_full)
monkeypatch.setattr(ray_executor_v2, "get_open_port", lambda: 54321)
assert RayExecutorV2._select_tcpstore_port(0, master_port=29500) == 54321
reused_store = TCPStore(
"127.0.0.1",
port,
world_size=1,
is_master=True,
wait_for_workers=False,
multi_tenant=True,
)
worker._dist_init_store.set("key", "value")
assert reused_store.get("key") == b"value"
@pytest.mark.parametrize("tp_size, pp_size", [(1, 1), (2, 1), (4, 1), (2, 2)])