[ROCm][Bugfix] Fix elastic EP scaling deadlock (#56610)

Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Matt
2026-09-12 23:09:19 +00:00
committed by GitHub
co-authored by Claude Opus 4.8
parent 7ee8a6dd01
commit 72d4d83157
4 changed files with 106 additions and 38 deletions
+13 -2
View File
@@ -50,6 +50,7 @@ from vllm.model_executor.warmup.flashinfer_autotune_cache import (
sync_flashinfer_autotune_cache,
)
from vllm.model_executor.warmup.kernel_warmup import kernel_warmup
from vllm.platforms import current_platform
from vllm.utils import is_moe_layer
from vllm.v1.engine import ReconfigureDistributedRequest, ReconfigureRankType
from vllm.v1.worker.dp_utils import skip_dp_coordination
@@ -283,7 +284,9 @@ class ElasticEPScalingExecutor:
self._prepare_eplb_communicator(get_standby_eplb_group())
if new_dp_size > old_dp_size:
self.transfer_weights(old_dp_size, new_dp_size)
self._warm_target_groups(get_standby_dp_group(), standby_ep_group)
# On ROCm this deadlocks while serving; warm_and_capture warms these at commit.
if not current_platform.is_rocm():
self._warm_target_groups(get_standby_dp_group(), standby_ep_group)
if new_dp_size > old_dp_size and self._can_reuse_fused_moe_kernel():
target_world_group = get_standby_world_group()
assert target_world_group is not None
@@ -577,7 +580,11 @@ class ElasticEPScalingExecutor:
mapping = self.receive_expert_mapping()
self.worker.model_runner.setup_eplb_from_mapping(mapping)
if not self._can_reuse_fused_moe_kernel():
# warm_and_capture's dummy run warms the deferred dp/ep groups.
self.warm_and_capture()
elif current_platform.is_rocm():
# Reuse path skips warm_and_capture; warm the deferred groups here.
self._warm_target_groups(get_dp_group(), get_ep_group())
self._perform_eplb_reshuffle(async_op=True)
if is_existing_worker:
self._start_group_cleanup(retired_groups)
@@ -590,6 +597,9 @@ class ElasticEPScalingExecutor:
retired_groups = self.switch_and_prepare()
if not self._can_reuse_fused_moe_kernel():
self.warm_and_capture()
elif current_platform.is_rocm():
# Reuse path skips warm_and_capture; warm the deferred groups here.
self._warm_target_groups(get_dp_group(), get_ep_group())
self._start_group_cleanup(retired_groups)
def perform_scale_down_eplb_reshuffle(self, new_dp_size: int) -> None:
@@ -646,7 +656,8 @@ class ElasticEPScalingExecutor:
expert_weights=expert_weights,
)
torch.accelerator.synchronize()
self._warm_target_groups(get_dp_group(), get_ep_group())
if not current_platform.is_rocm():
self._warm_target_groups(get_dp_group(), get_ep_group())
if self._can_reuse_fused_moe_kernel():
sync_flashinfer_autotune_cache(self.worker.model_runner, get_world_group())