[Bugfix] Avoid TCPStore port collision for co-located non-DP Ray engines (#53666)

Signed-off-by: Jeffrey Wang <jeffreywang@anyscale.com>
This commit is contained in:
Jeffrey Wang
2026-08-27 19:58:14 +00:00
committed by GitHub
parent 9db222c6be
commit 79378fe6e8
2 changed files with 32 additions and 2 deletions
+26
View File
@@ -128,6 +128,32 @@ def test_select_tcpstore_port_non_dp_uses_random(monkeypatch):
assert RayExecutorV2._select_tcpstore_port(None, master_port=29500) == 54321
def test_colocated_tcpstore_ports_differ():
"""Two or more replicas on one host can collide on the same TCPStore port.
Their ports must differ and never be a low system port.
"""
ports = []
real_select = RayExecutorV2._select_tcpstore_port
def record(local_dp_rank, master_port):
port = real_select(local_dp_rank, master_port)
ports.append(port)
return port
with patch.object(RayExecutorV2, "_select_tcpstore_port", staticmethod(record)):
first = RayExecutorV2(vllm_config=create_vllm_config())
try:
second = RayExecutorV2(vllm_config=create_vllm_config())
try:
assert len(ports) == 2
assert min(ports) >= 1024
assert ports[0] != ports[1]
finally:
second.shutdown()
finally:
first.shutdown()
def test_select_tcpstore_port_full_window_uses_random(monkeypatch):
"""A fully occupied window falls back to a random port."""
+6 -2
View File
@@ -329,9 +329,13 @@ class RayExecutorV2(MultiprocExecutor):
# must be able to reach this address.
dist_ip = bundle_assignments[0]["node_ip"]
parallel_config = self.vllm_config.parallel_config
local_dp_rank = (
parallel_config.data_parallel_rank_local
if parallel_config.data_parallel_size > 1
else None
)
port = self._select_tcpstore_port(
parallel_config.data_parallel_rank_local,
parallel_config.data_parallel_master_port,
local_dp_rank, parallel_config.data_parallel_master_port
)
distributed_init_method = get_distributed_init_method(dist_ip, port)