[Elastic EP] Reduce eager-mode reconfiguration downtime (#51885)

Signed-off-by: Itay Alroy <ialroy@nvidia.com>
This commit is contained in:
Itay Alroy
2026-08-19 02:13:06 +00:00
committed by GitHub
parent d36cc4254e
commit a2257f95b7
12 changed files with 194 additions and 303 deletions
+108 -54
View File
@@ -29,12 +29,15 @@ from vllm.distributed.elastic_ep.standby_state import (
create_standby_groups,
get_standby_dp_group,
get_standby_ep_group,
get_standby_eplb_group,
pop_standby_groups,
)
from vllm.distributed.eplb.eplb_communicator import create_eplb_communicator
from vllm.distributed.eplb.eplb_communicator import (
EplbCommunicator,
)
from vllm.distributed.parallel_state import (
GroupCoordinator,
_replace_active_groups,
get_eplb_group,
)
from vllm.distributed.stateless_coordinator import StatelessGroupCoordinator
from vllm.logger import init_logger
@@ -149,10 +152,12 @@ class ElasticEPScalingExecutor:
self.worker_ref = weakref.ref(worker)
self.reconfig_request = None
self._staged_moe_quant_methods: dict[nn.Module, FusedMoEMethodBase] = {}
self._prepared_eplb_communicator: EplbCommunicator | None = None
self._async_executor = ThreadPoolExecutor(
max_workers=1, thread_name_prefix="ElasticEPAsync"
)
self._async_future: Future[None] | None = None
self._group_cleanup_future: Future[None] | None = None
@property
def worker(self):
@@ -207,12 +212,40 @@ class ElasticEPScalingExecutor:
self._async_future = None
future.result()
def _destroy_retired_groups(
self, groups: tuple[GroupCoordinator | None, ...]
) -> None:
from vllm.platforms import current_platform
current_platform.set_device(self.worker.device)
for group in groups:
if group is not None:
group.destroy()
def _start_group_cleanup(self, groups: tuple[GroupCoordinator | None, ...]) -> None:
assert self._group_cleanup_future is None
self._group_cleanup_future = self._async_executor.submit(
self._destroy_retired_groups, groups
)
def _wait_for_group_cleanup(self) -> None:
if (future := self._group_cleanup_future) is not None:
self._group_cleanup_future = None
future.result()
def shutdown(self) -> None:
try:
self._wait_for_group_cleanup()
finally:
self._async_executor.shutdown()
def load_model(self) -> None:
self.worker.load_model(load_dummy_weights=True)
def create_standby_groups(
def prepare_reconfiguration(
self, reconfig_request: ReconfigureDistributedRequest, use_all2all: bool
) -> None:
self._wait_for_group_cleanup()
self.reconfig_request = reconfig_request
new_dp_size = reconfig_request.new_data_parallel_size
old_dp_size = get_dp_group().world_size
@@ -227,8 +260,20 @@ class ElasticEPScalingExecutor:
use_all2all=use_all2all,
enable_eplb=parallel_config.enable_eplb,
)
if new_dp_size < old_dp_size:
self.stage_standby_moe_quant_methods()
self.stage_standby_moe_quant_methods()
self._prepare_eplb_communicator(get_standby_eplb_group())
if new_dp_size > old_dp_size:
self.transfer_weights(old_dp_size, new_dp_size)
self._warm_target_groups(get_standby_dp_group(), get_standby_ep_group())
def _prepare_eplb_communicator(self, eplb_group) -> None:
assert eplb_group is not None
model_runner = self.worker.model_runner
eplb_state = model_runner.eplb_state
assert eplb_state is not None
self._prepared_eplb_communicator = eplb_state.create_communicator(
model_runner.model_config, eplb_group
)
def transfer_weights(self, old_dp_size: int, new_dp_size: int) -> None:
standby_dp_group = get_standby_dp_group()
@@ -276,6 +321,15 @@ class ElasticEPScalingExecutor:
)
torch.accelerator.synchronize()
def _warm_target_groups(self, dp_group, ep_group) -> None:
assert dp_group is not None and ep_group is not None
stream = torch.Stream(device=dp_group.device)
with stream:
tensor = torch.zeros(1, dtype=torch.int32, device=dp_group.device)
for group in (dp_group, ep_group):
torch.distributed.all_reduce(tensor, group=group.device_group)
stream.synchronize()
def broadcast_expert_mapping(self) -> None:
standby_dp_group = get_standby_dp_group()
assert standby_dp_group is not None
@@ -362,15 +416,22 @@ class ElasticEPScalingExecutor:
torch.accelerator.empty_cache()
def switch_and_remove(self) -> None:
# Removing ranks skipped preparation, so wait for prior cleanup here.
self._wait_for_group_cleanup()
self._release_cuda_graphs()
_replace_active_groups(world=None, dp=None, ep=None, eplb=None, node_count=None)
retired_groups = _replace_active_groups(
world=None, dp=None, ep=None, eplb=None, node_count=None
)
self._start_group_cleanup(retired_groups)
# Finish collective cleanup before this worker is shut down.
self._wait_for_group_cleanup()
def switch_and_prepare(self) -> None:
def switch_and_prepare(self) -> tuple[GroupCoordinator | None, ...]:
old_dp_size = get_dp_group().world_size
old_ep_size = get_ep_group().world_size
self._release_cuda_graphs()
_replace_active_groups(**pop_standby_groups())
retired_groups = _replace_active_groups(**pop_standby_groups())
parallel_config = self.worker.vllm_config.parallel_config
reconfig_request = self.reconfig_request
@@ -453,7 +514,6 @@ class ElasticEPScalingExecutor:
)
eplb_model_state.expert_load_pass = expanded_expert_load_pass
eplb_model_state.expert_load_window = expanded_expert_load_window
eplb_state.num_valid_physical_experts = old_num_physical_experts
else:
assert pad_size < 0
eplb_model_state.expert_load_pass = eplb_model_state.expert_load_pass[
@@ -462,7 +522,6 @@ class ElasticEPScalingExecutor:
eplb_model_state.expert_load_window = eplb_model_state.expert_load_window[
:, :, :num_physical_experts
]
eplb_state.num_valid_physical_experts = num_physical_experts
model = self.worker.model_runner.get_model()
model.expert_weights = []
@@ -485,18 +544,9 @@ class ElasticEPScalingExecutor:
if getattr(module._quant_method, "wraps_legacy_quant_method", False):
module._replace_quant_method(module._quant_method.old_quant_method)
eplb_model_state.expert_buffer = [
torch.empty_like(w) for w in model.expert_weights[0]
]
assert parallel_config.eplb_config.communicator is not None, (
"EPLB communicator backend must be set by ParallelConfig"
)
eplb_model_state.communicator = create_eplb_communicator(
group_coordinator=get_eplb_group(),
backend=parallel_config.eplb_config.communicator,
expert_weights=model.expert_weights,
expert_buffer=eplb_model_state.expert_buffer,
)
assert self._prepared_eplb_communicator is not None
eplb_state.update_communicator(model_config, self._prepared_eplb_communicator)
self._prepared_eplb_communicator = None
if (
self.worker.vllm_config.compilation_config.mode
@@ -513,8 +563,12 @@ class ElasticEPScalingExecutor:
compilation_counter.stock_torch_compile_count += 1
self.worker.model_runner.model.compile(fullgraph=True, backend=backend)
return retired_groups
def _perform_eplb_reshuffle(
self, rank_mapping: dict[int, int] | None = None
self,
rank_mapping: dict[int, int] | None = None,
async_op: bool = False,
) -> None:
if get_ep_group().rank == 0:
logger.info("[Elastic EP] Starting expert resharding...")
@@ -522,21 +576,19 @@ class ElasticEPScalingExecutor:
eplb_state = self.worker.model_runner.eplb_state
assert eplb_state is not None
model_config = self.worker.model_runner.model_config
eplb_model_state = eplb_state.model_states[model_config.compute_hash()]
is_async_enabled = eplb_state.is_async
eplb_state.is_async = False
run_async = async_op and is_async_enabled
eplb_state.is_async = run_async
if rank_mapping is None:
eplb_state.rearrange()
else:
eplb_state.rearrange(rank_mapping=rank_mapping)
# NOTE(yongji): check whether we need to synchronize here
torch.accelerator.synchronize()
if not run_async:
# Wait for non-blocking expert resharding copies before continuing
# the Elastic EP reconfiguration.
torch.accelerator.synchronize()
# reset expert_rearrangement_step to ensure all ranks are synchronized
eplb_state.expert_rearrangement_step = 0
eplb_state.num_valid_physical_experts = (
eplb_model_state.physical_to_logical_map.shape[1]
)
eplb_state.is_async = is_async_enabled
# Start the async worker thread if it doesn't exist yet (idempotent).
# This is needed for new workers after scale-up: they create EplbState
@@ -544,25 +596,31 @@ class ElasticEPScalingExecutor:
# groups aren't ready yet.
eplb_state.start_async_loop()
if get_ep_group().rank == 0:
logger.info("[Elastic EP] Expert resharding completed")
logger.info(
"[Elastic EP] Expert resharding %s",
"scheduled" if run_async else "completed",
)
def commit_scale_up(self, is_existing_worker: bool) -> None:
if is_existing_worker:
self.broadcast_expert_mapping()
self.switch_and_prepare()
retired_groups = self.switch_and_prepare()
else:
mapping, _, num_valid_experts = self.receive_expert_mapping()
self.worker.model_runner.setup_eplb_from_mapping(mapping, num_valid_experts)
self._perform_eplb_reshuffle()
mapping = self.receive_expert_mapping()
self.worker.model_runner.setup_eplb_from_mapping(mapping)
self.warm_and_capture()
self._perform_eplb_reshuffle(async_op=True)
if is_existing_worker:
self._start_group_cleanup(retired_groups)
def commit_scale_down(self, new_dp_size: int, removing: bool) -> None:
self.perform_scale_down_eplb_reshuffle(new_dp_size)
if removing:
self.switch_and_remove()
else:
self.switch_and_prepare()
retired_groups = self.switch_and_prepare()
self.warm_and_capture()
self._start_group_cleanup(retired_groups)
def perform_scale_down_eplb_reshuffle(self, new_dp_size: int) -> None:
eplb_state = self.worker.model_runner.eplb_state
@@ -578,7 +636,7 @@ class ElasticEPScalingExecutor:
}
self._perform_eplb_reshuffle(rank_mapping=rank_mapping)
def receive_weights(self) -> None:
def prepare_new_worker(self) -> None:
dp_group = get_dp_group()
assert isinstance(dp_group, StatelessGroupCoordinator)
new_dp_size = dp_group.world_size
@@ -618,19 +676,18 @@ class ElasticEPScalingExecutor:
expert_weights=expert_weights,
)
torch.accelerator.synchronize()
self._warm_target_groups(get_dp_group(), get_ep_group())
def receive_expert_mapping(self) -> tuple[torch.Tensor, int, int]:
def receive_expert_mapping(self) -> torch.Tensor:
dp_group = get_dp_group()
assert isinstance(dp_group, StatelessGroupCoordinator)
physical_to_logical, num_local_physical_experts, num_logical_experts = (
broadcast_expert_mapping(
physical_to_logical=None,
num_local_physical_experts=None,
num_logical_experts=None,
dp_group=dp_group,
src_rank=0,
device=self.worker.device,
)
physical_to_logical, num_local_physical_experts, _ = broadcast_expert_mapping(
physical_to_logical=None,
num_local_physical_experts=None,
num_logical_experts=None,
dp_group=dp_group,
src_rank=0,
device=self.worker.device,
)
num_moe_layers = physical_to_logical.shape[0]
new_dp_size = get_dp_group().world_size
@@ -642,13 +699,10 @@ class ElasticEPScalingExecutor:
dtype=physical_to_logical.dtype,
device=physical_to_logical.device,
)
old_num_physical_experts = physical_to_logical.shape[1]
expanded_physical_to_logical[:, :old_num_physical_experts] = physical_to_logical
return (
expanded_physical_to_logical,
num_logical_experts,
old_num_physical_experts,
expanded_physical_to_logical[:, : physical_to_logical.shape[1]] = (
physical_to_logical
)
return expanded_physical_to_logical
def warmup_local_kernels(self) -> None:
with set_current_vllm_config(self.worker.vllm_config):