[Bugfix] Avoid TCPStore port collision for co-located non-DP Ray engines (#53666)
Signed-off-by: Jeffrey Wang <jeffreywang@anyscale.com>
This commit is contained in:
@@ -128,6 +128,32 @@ def test_select_tcpstore_port_non_dp_uses_random(monkeypatch):
|
||||
assert RayExecutorV2._select_tcpstore_port(None, master_port=29500) == 54321
|
||||
|
||||
|
||||
def test_colocated_tcpstore_ports_differ():
|
||||
"""Two or more replicas on one host can collide on the same TCPStore port.
|
||||
Their ports must differ and never be a low system port.
|
||||
"""
|
||||
ports = []
|
||||
real_select = RayExecutorV2._select_tcpstore_port
|
||||
|
||||
def record(local_dp_rank, master_port):
|
||||
port = real_select(local_dp_rank, master_port)
|
||||
ports.append(port)
|
||||
return port
|
||||
|
||||
with patch.object(RayExecutorV2, "_select_tcpstore_port", staticmethod(record)):
|
||||
first = RayExecutorV2(vllm_config=create_vllm_config())
|
||||
try:
|
||||
second = RayExecutorV2(vllm_config=create_vllm_config())
|
||||
try:
|
||||
assert len(ports) == 2
|
||||
assert min(ports) >= 1024
|
||||
assert ports[0] != ports[1]
|
||||
finally:
|
||||
second.shutdown()
|
||||
finally:
|
||||
first.shutdown()
|
||||
|
||||
|
||||
def test_select_tcpstore_port_full_window_uses_random(monkeypatch):
|
||||
"""A fully occupied window falls back to a random port."""
|
||||
|
||||
|
||||
@@ -329,9 +329,13 @@ class RayExecutorV2(MultiprocExecutor):
|
||||
# must be able to reach this address.
|
||||
dist_ip = bundle_assignments[0]["node_ip"]
|
||||
parallel_config = self.vllm_config.parallel_config
|
||||
local_dp_rank = (
|
||||
parallel_config.data_parallel_rank_local
|
||||
if parallel_config.data_parallel_size > 1
|
||||
else None
|
||||
)
|
||||
port = self._select_tcpstore_port(
|
||||
parallel_config.data_parallel_rank_local,
|
||||
parallel_config.data_parallel_master_port,
|
||||
local_dp_rank, parallel_config.data_parallel_master_port
|
||||
)
|
||||
distributed_init_method = get_distributed_init_method(dist_ip, port)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user