[BugFix] Bind RayExecutorV2 TCPStore before publishing its port (#50969)
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
aoshen02
Claude
OpenAI Codex
Andreas Karatzas
parent
2f1cba799e
commit
67e86d1e6f
@@ -7,12 +7,17 @@ Validates executor initialization, placement group support, RPC calls,
|
||||
and distributed execution with various TP/PP configurations.
|
||||
"""
|
||||
|
||||
import errno
|
||||
import gc
|
||||
import socket
|
||||
import threading
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import pytest
|
||||
import ray
|
||||
from torch.distributed import TCPStore
|
||||
|
||||
from vllm import LLM
|
||||
from vllm.config import VllmConfig
|
||||
@@ -102,67 +107,30 @@ def assert_executor(executor, tp_size, pp_size):
|
||||
assert handle.node_id is not None
|
||||
|
||||
|
||||
def test_select_tcpstore_port_seeds_disjoint_windows(monkeypatch):
|
||||
"""Co-located DP engines scan distinct, adjacent port windows, so two
|
||||
engines on a node cannot pick the same TCPStore port."""
|
||||
requested = []
|
||||
def test_dist_init_store_owns_port(monkeypatch):
|
||||
"""The rank-0 actor keeps the selected TCPStore port bound."""
|
||||
worker = ray_executor_v2.RayWorkerProc.__new__(ray_executor_v2.RayWorkerProc)
|
||||
worker._parallel_config = SimpleNamespace(world_size=1)
|
||||
monkeypatch.setattr(ray.util, "get_node_ip_address", lambda: "127.0.0.1")
|
||||
|
||||
def fake_get_open_port(start_port, max_attempts):
|
||||
requested.append((start_port, max_attempts))
|
||||
return start_port
|
||||
init_method = worker.create_dist_init_method()
|
||||
port = urlsplit(init_method).port
|
||||
assert port is not None and port != 0
|
||||
|
||||
monkeypatch.setattr(ray_executor_v2, "_get_open_port", fake_get_open_port)
|
||||
with socket.socket() as contender, pytest.raises(OSError) as exc_info:
|
||||
contender.bind(("127.0.0.1", port))
|
||||
assert exc_info.value.errno == errno.EADDRINUSE
|
||||
|
||||
ports = [
|
||||
RayExecutorV2._select_tcpstore_port(rank, master_port=29500)
|
||||
for rank in range(4)
|
||||
]
|
||||
|
||||
assert requested == [(29600, 32), (29632, 32), (29664, 32), (29696, 32)]
|
||||
assert len(set(ports)) == 4
|
||||
|
||||
|
||||
def test_select_tcpstore_port_non_dp_uses_random(monkeypatch):
|
||||
"""A non-DP engine has no local rank and uses a random port."""
|
||||
monkeypatch.setattr(ray_executor_v2, "get_open_port", lambda: 54321)
|
||||
assert RayExecutorV2._select_tcpstore_port(None, master_port=29500) == 54321
|
||||
|
||||
|
||||
def test_colocated_tcpstore_ports_differ():
|
||||
"""Two or more replicas on one host can collide on the same TCPStore port.
|
||||
Their ports must differ and never be a low system port.
|
||||
"""
|
||||
ports = []
|
||||
real_select = RayExecutorV2._select_tcpstore_port
|
||||
|
||||
def record(local_dp_rank, master_port):
|
||||
port = real_select(local_dp_rank, master_port)
|
||||
ports.append(port)
|
||||
return port
|
||||
|
||||
with patch.object(RayExecutorV2, "_select_tcpstore_port", staticmethod(record)):
|
||||
first = RayExecutorV2(vllm_config=create_vllm_config())
|
||||
try:
|
||||
second = RayExecutorV2(vllm_config=create_vllm_config())
|
||||
try:
|
||||
assert len(ports) == 2
|
||||
assert min(ports) >= 1024
|
||||
assert ports[0] != ports[1]
|
||||
finally:
|
||||
second.shutdown()
|
||||
finally:
|
||||
first.shutdown()
|
||||
|
||||
|
||||
def test_select_tcpstore_port_full_window_uses_random(monkeypatch):
|
||||
"""A fully occupied window falls back to a random port."""
|
||||
|
||||
def raise_full(start_port, max_attempts):
|
||||
raise RuntimeError("no open port")
|
||||
|
||||
monkeypatch.setattr(ray_executor_v2, "_get_open_port", raise_full)
|
||||
monkeypatch.setattr(ray_executor_v2, "get_open_port", lambda: 54321)
|
||||
assert RayExecutorV2._select_tcpstore_port(0, master_port=29500) == 54321
|
||||
reused_store = TCPStore(
|
||||
"127.0.0.1",
|
||||
port,
|
||||
world_size=1,
|
||||
is_master=True,
|
||||
wait_for_workers=False,
|
||||
multi_tenant=True,
|
||||
)
|
||||
worker._dist_init_store.set("key", "value")
|
||||
assert reused_store.get("key") == b"value"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size, pp_size", [(1, 1), (2, 1), (4, 1), (2, 2)])
|
||||
|
||||
Reference in New Issue
Block a user