[Bugfix] Seed RayExecutorV2 TCPStore port by DP rank to avoid collisions (#45960)

Signed-off-by: Seiji Eicher <seiji@anyscale.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Seiji Eicher
2026-06-30 07:37:34 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 7a327f0b4f
commit ded6676458
2 changed files with 64 additions and 1 deletions
+26 -1
View File
@@ -17,6 +17,7 @@ from vllm.distributed.device_communicators.shm_broadcast import (
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.network_utils import (
_get_open_port,
get_distributed_init_method,
get_open_port,
)
@@ -259,6 +260,25 @@ class RayExecutorV2(MultiprocExecutor):
return {"num_gpus": num_devices}
return {"num_gpus": 0, "resources": {device_key: num_devices}}
@staticmethod
def _select_tcpstore_port(local_dp_rank: int | None, master_port: int) -> int:
"""Pick the torch.distributed TCPStore port for this engine.
Co-located DP engines choosing this port with a shared random search
collide intermittently. Seeding by node-local DP rank gives each a
disjoint window. Non-DP engines and full windows fall back to a
random port.
"""
if local_dp_rank is None:
return get_open_port()
# Offset past the DP master port reserved range, one window per rank.
window = 32
start_port = master_port + 100 + local_dp_rank * window
try:
return _get_open_port(start_port=start_port, max_attempts=window)
except RuntimeError:
return get_open_port()
def _init_executor(self) -> None:
"""Initialize the RayExecutorV2 executor."""
self._finalizer = weakref.finalize(self, self.shutdown)
@@ -308,7 +328,12 @@ class RayExecutorV2(MultiprocExecutor):
# The TCPStore server runs on rank 0's node, so all workers
# must be able to reach this address.
dist_ip = bundle_assignments[0]["node_ip"]
distributed_init_method = get_distributed_init_method(dist_ip, get_open_port())
parallel_config = self.vllm_config.parallel_config
port = self._select_tcpstore_port(
parallel_config.data_parallel_rank_local,
parallel_config.data_parallel_master_port,
)
distributed_init_method = get_distributed_init_method(dist_ip, port)
# Step 4: Create broadcast MessageQueue.
# Workers on the driver node use shared memory; the rest use TCP.