[ROCm][Bugfix] Fix elastic EP scaling deadlock (#56610)
Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -50,6 +50,7 @@ from vllm.model_executor.warmup.flashinfer_autotune_cache import (
|
||||
sync_flashinfer_autotune_cache,
|
||||
)
|
||||
from vllm.model_executor.warmup.kernel_warmup import kernel_warmup
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils import is_moe_layer
|
||||
from vllm.v1.engine import ReconfigureDistributedRequest, ReconfigureRankType
|
||||
from vllm.v1.worker.dp_utils import skip_dp_coordination
|
||||
@@ -283,7 +284,9 @@ class ElasticEPScalingExecutor:
|
||||
self._prepare_eplb_communicator(get_standby_eplb_group())
|
||||
if new_dp_size > old_dp_size:
|
||||
self.transfer_weights(old_dp_size, new_dp_size)
|
||||
self._warm_target_groups(get_standby_dp_group(), standby_ep_group)
|
||||
# On ROCm this deadlocks while serving; warm_and_capture warms these at commit.
|
||||
if not current_platform.is_rocm():
|
||||
self._warm_target_groups(get_standby_dp_group(), standby_ep_group)
|
||||
if new_dp_size > old_dp_size and self._can_reuse_fused_moe_kernel():
|
||||
target_world_group = get_standby_world_group()
|
||||
assert target_world_group is not None
|
||||
@@ -577,7 +580,11 @@ class ElasticEPScalingExecutor:
|
||||
mapping = self.receive_expert_mapping()
|
||||
self.worker.model_runner.setup_eplb_from_mapping(mapping)
|
||||
if not self._can_reuse_fused_moe_kernel():
|
||||
# warm_and_capture's dummy run warms the deferred dp/ep groups.
|
||||
self.warm_and_capture()
|
||||
elif current_platform.is_rocm():
|
||||
# Reuse path skips warm_and_capture; warm the deferred groups here.
|
||||
self._warm_target_groups(get_dp_group(), get_ep_group())
|
||||
self._perform_eplb_reshuffle(async_op=True)
|
||||
if is_existing_worker:
|
||||
self._start_group_cleanup(retired_groups)
|
||||
@@ -590,6 +597,9 @@ class ElasticEPScalingExecutor:
|
||||
retired_groups = self.switch_and_prepare()
|
||||
if not self._can_reuse_fused_moe_kernel():
|
||||
self.warm_and_capture()
|
||||
elif current_platform.is_rocm():
|
||||
# Reuse path skips warm_and_capture; warm the deferred groups here.
|
||||
self._warm_target_groups(get_dp_group(), get_ep_group())
|
||||
self._start_group_cleanup(retired_groups)
|
||||
|
||||
def perform_scale_down_eplb_reshuffle(self, new_dp_size: int) -> None:
|
||||
@@ -646,7 +656,8 @@ class ElasticEPScalingExecutor:
|
||||
expert_weights=expert_weights,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
self._warm_target_groups(get_dp_group(), get_ep_group())
|
||||
if not current_platform.is_rocm():
|
||||
self._warm_target_groups(get_dp_group(), get_ep_group())
|
||||
if self._can_reuse_fused_moe_kernel():
|
||||
sync_flashinfer_autotune_cache(self.worker.model_runner, get_world_group())
|
||||
|
||||
|
||||
Reference in New Issue
Block a user