[ROCm][Bugfix] Fix elastic EP scaling deadlock (#56610)

Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Matt
2026-09-12 23:09:19 +00:00
committed by GitHub
co-authored by Claude Opus 4.8
parent 7ee8a6dd01
commit 72d4d83157
4 changed files with 106 additions and 38 deletions
+39 -22
View File
@@ -1919,6 +1919,20 @@ def init_distributed_environment(
_INNER_DP_WORLD = _WORLD
def _elastic_join_warmup_ctx() -> AbstractContextManager[Any]:
"""Defer communicator warm-up when joining a live elastic-EP cluster on ROCm.
The existing ranks defer their matching warm-up in create_standby_groups(),
so a joining worker must too or the warm-up collective has no peers. Both
sides warm up at commit.
"""
from vllm.distributed.device_communicators.pynccl import defer_comm_warmup_on_rocm
if envs.VLLM_ELASTIC_EP_SCALE_UP_LAUNCH:
return defer_comm_warmup_on_rocm()
return nullcontext()
def initialize_model_parallel(
tensor_model_parallel_size: int = 1,
pipeline_model_parallel_size: int = 1,
@@ -2101,13 +2115,14 @@ def initialize_model_parallel(
group_ranks = all_ranks.transpose(1, 4).reshape(-1, data_parallel_size).unbind(0)
group_ranks = [x.tolist() for x in group_ranks]
if enable_elastic_ep:
_DP = _init_stateless_group(
group_ranks,
"dp",
parallel_config.data_parallel_master_ip,
backend,
coord_store=coord_store,
)
with _elastic_join_warmup_ctx():
_DP = _init_stateless_group(
group_ranks,
"dp",
parallel_config.data_parallel_master_ip,
backend,
coord_store=coord_store,
)
else:
_DP = init_model_parallel_group(
group_ranks, get_world_group().local_rank, backend, group_name="dp"
@@ -2130,14 +2145,15 @@ def initialize_model_parallel(
group_ranks = [x.tolist() for x in group_ranks]
use_all2all = parallel_config.use_all2all
if enable_elastic_ep:
_EP = _init_stateless_group(
group_ranks,
"ep",
parallel_config.data_parallel_master_ip,
backend,
coord_store=coord_store,
use_all2all=use_all2all,
)
with _elastic_join_warmup_ctx():
_EP = _init_stateless_group(
group_ranks,
"ep",
parallel_config.data_parallel_master_ip,
backend,
coord_store=coord_store,
use_all2all=use_all2all,
)
else:
_EP = init_model_parallel_group(
group_ranks,
@@ -2155,13 +2171,14 @@ def initialize_model_parallel(
assert _EPLB is None, "EPLB group is already initialized"
if config.parallel_config.enable_eplb:
if enable_elastic_ep:
_EPLB = _init_stateless_group(
group_ranks,
"eplb",
parallel_config.data_parallel_master_ip,
backend,
coord_store=coord_store,
)
with _elastic_join_warmup_ctx():
_EPLB = _init_stateless_group(
group_ranks,
"eplb",
parallel_config.data_parallel_master_ip,
backend,
coord_store=coord_store,
)
else:
_EPLB = init_model_parallel_group(
group_ranks,