[BugFix] Pick the DP world-group port at bind time via the coordination store (#51018)
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
aoshen02
Claude
Codex
parent
a4b14e57b6
commit
caff2b44cb
@@ -42,6 +42,7 @@ import torch.distributed
|
||||
import torch.distributed._functional_collectives as funcol
|
||||
import torch.distributed._symmetric_memory
|
||||
from torch.distributed import Backend, ProcessGroup, Store
|
||||
from torch.distributed.constants import default_pg_timeout
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.distributed.device_communicators.base_device_communicator import (
|
||||
@@ -49,6 +50,7 @@ from vllm.distributed.device_communicators.base_device_communicator import (
|
||||
)
|
||||
from vllm.distributed.utils import (
|
||||
StatelessProcessGroup,
|
||||
create_tcp_store,
|
||||
get_cached_tcp_store_client,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
@@ -1829,6 +1831,7 @@ def init_distributed_environment(
|
||||
|
||||
config = get_current_vllm_config_or_none()
|
||||
enable_elastic_ep = config is not None and config.parallel_config.enable_elastic_ep
|
||||
world_pg_store: Store | None = None
|
||||
if (
|
||||
config is not None
|
||||
and config.parallel_config.distributed_executor_backend != "external_launcher"
|
||||
@@ -1852,7 +1855,46 @@ def init_distributed_environment(
|
||||
distributed_init_method = get_distributed_init_method(ip, port)
|
||||
else:
|
||||
ip = parallel_config.data_parallel_master_ip
|
||||
port = parallel_config.get_next_dp_init_port()
|
||||
if (
|
||||
parallel_config._coord_store_port
|
||||
and not torch.distributed.is_initialized()
|
||||
):
|
||||
# Group rank 0 lets the world-group TCPStore bind port 0
|
||||
# and publishes the kernel-assigned port via the
|
||||
# coordination store. A one-shot client of our own: the
|
||||
# cached one is inherited across fork and shared with
|
||||
# sibling workers.
|
||||
coord_store = create_tcp_store(
|
||||
ip,
|
||||
parallel_config._coord_store_port,
|
||||
is_master=False,
|
||||
wait_for_workers=False,
|
||||
)
|
||||
if rank == 0:
|
||||
# Workers can only connect once the port is published.
|
||||
world_pg_store = create_tcp_store(
|
||||
ip,
|
||||
0,
|
||||
world_size=world_size,
|
||||
is_master=True,
|
||||
wait_for_workers=False,
|
||||
timeout=timeout or default_pg_timeout,
|
||||
multi_tenant=True,
|
||||
)
|
||||
port = world_pg_store.port
|
||||
coord_store.set("world_pg_port", str(port).encode())
|
||||
else:
|
||||
port = int(coord_store.get("world_pg_port").decode())
|
||||
world_pg_store = create_tcp_store(
|
||||
ip,
|
||||
port,
|
||||
world_size=world_size,
|
||||
is_master=False,
|
||||
timeout=timeout or default_pg_timeout,
|
||||
multi_tenant=True,
|
||||
)
|
||||
else:
|
||||
port = parallel_config.get_next_dp_init_port()
|
||||
distributed_init_method = get_distributed_init_method(ip, port)
|
||||
logger.debug(
|
||||
"Adjusting world_size=%d rank=%d distributed_init_method=%s for DP",
|
||||
@@ -1882,8 +1924,8 @@ def init_distributed_environment(
|
||||
"Fallback Gloo backend is not available."
|
||||
)
|
||||
backend = "gloo"
|
||||
store = None
|
||||
if distributed_init_method.startswith("file://"):
|
||||
store = world_pg_store
|
||||
if store is None and distributed_init_method.startswith("file://"):
|
||||
store = torch.distributed.FileStore(
|
||||
distributed_init_method.removeprefix("file://"), world_size
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user