[Elastic EP] Reduce eager-mode reconfiguration downtime (#51885)
Signed-off-by: Itay Alroy <ialroy@nvidia.com>
This commit is contained in:
@@ -29,12 +29,15 @@ from vllm.distributed.elastic_ep.standby_state import (
|
||||
create_standby_groups,
|
||||
get_standby_dp_group,
|
||||
get_standby_ep_group,
|
||||
get_standby_eplb_group,
|
||||
pop_standby_groups,
|
||||
)
|
||||
from vllm.distributed.eplb.eplb_communicator import create_eplb_communicator
|
||||
from vllm.distributed.eplb.eplb_communicator import (
|
||||
EplbCommunicator,
|
||||
)
|
||||
from vllm.distributed.parallel_state import (
|
||||
GroupCoordinator,
|
||||
_replace_active_groups,
|
||||
get_eplb_group,
|
||||
)
|
||||
from vllm.distributed.stateless_coordinator import StatelessGroupCoordinator
|
||||
from vllm.logger import init_logger
|
||||
@@ -149,10 +152,12 @@ class ElasticEPScalingExecutor:
|
||||
self.worker_ref = weakref.ref(worker)
|
||||
self.reconfig_request = None
|
||||
self._staged_moe_quant_methods: dict[nn.Module, FusedMoEMethodBase] = {}
|
||||
self._prepared_eplb_communicator: EplbCommunicator | None = None
|
||||
self._async_executor = ThreadPoolExecutor(
|
||||
max_workers=1, thread_name_prefix="ElasticEPAsync"
|
||||
)
|
||||
self._async_future: Future[None] | None = None
|
||||
self._group_cleanup_future: Future[None] | None = None
|
||||
|
||||
@property
|
||||
def worker(self):
|
||||
@@ -207,12 +212,40 @@ class ElasticEPScalingExecutor:
|
||||
self._async_future = None
|
||||
future.result()
|
||||
|
||||
def _destroy_retired_groups(
|
||||
self, groups: tuple[GroupCoordinator | None, ...]
|
||||
) -> None:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
current_platform.set_device(self.worker.device)
|
||||
for group in groups:
|
||||
if group is not None:
|
||||
group.destroy()
|
||||
|
||||
def _start_group_cleanup(self, groups: tuple[GroupCoordinator | None, ...]) -> None:
|
||||
assert self._group_cleanup_future is None
|
||||
self._group_cleanup_future = self._async_executor.submit(
|
||||
self._destroy_retired_groups, groups
|
||||
)
|
||||
|
||||
def _wait_for_group_cleanup(self) -> None:
|
||||
if (future := self._group_cleanup_future) is not None:
|
||||
self._group_cleanup_future = None
|
||||
future.result()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
try:
|
||||
self._wait_for_group_cleanup()
|
||||
finally:
|
||||
self._async_executor.shutdown()
|
||||
|
||||
def load_model(self) -> None:
|
||||
self.worker.load_model(load_dummy_weights=True)
|
||||
|
||||
def create_standby_groups(
|
||||
def prepare_reconfiguration(
|
||||
self, reconfig_request: ReconfigureDistributedRequest, use_all2all: bool
|
||||
) -> None:
|
||||
self._wait_for_group_cleanup()
|
||||
self.reconfig_request = reconfig_request
|
||||
new_dp_size = reconfig_request.new_data_parallel_size
|
||||
old_dp_size = get_dp_group().world_size
|
||||
@@ -227,8 +260,20 @@ class ElasticEPScalingExecutor:
|
||||
use_all2all=use_all2all,
|
||||
enable_eplb=parallel_config.enable_eplb,
|
||||
)
|
||||
if new_dp_size < old_dp_size:
|
||||
self.stage_standby_moe_quant_methods()
|
||||
self.stage_standby_moe_quant_methods()
|
||||
self._prepare_eplb_communicator(get_standby_eplb_group())
|
||||
if new_dp_size > old_dp_size:
|
||||
self.transfer_weights(old_dp_size, new_dp_size)
|
||||
self._warm_target_groups(get_standby_dp_group(), get_standby_ep_group())
|
||||
|
||||
def _prepare_eplb_communicator(self, eplb_group) -> None:
|
||||
assert eplb_group is not None
|
||||
model_runner = self.worker.model_runner
|
||||
eplb_state = model_runner.eplb_state
|
||||
assert eplb_state is not None
|
||||
self._prepared_eplb_communicator = eplb_state.create_communicator(
|
||||
model_runner.model_config, eplb_group
|
||||
)
|
||||
|
||||
def transfer_weights(self, old_dp_size: int, new_dp_size: int) -> None:
|
||||
standby_dp_group = get_standby_dp_group()
|
||||
@@ -276,6 +321,15 @@ class ElasticEPScalingExecutor:
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
def _warm_target_groups(self, dp_group, ep_group) -> None:
|
||||
assert dp_group is not None and ep_group is not None
|
||||
stream = torch.Stream(device=dp_group.device)
|
||||
with stream:
|
||||
tensor = torch.zeros(1, dtype=torch.int32, device=dp_group.device)
|
||||
for group in (dp_group, ep_group):
|
||||
torch.distributed.all_reduce(tensor, group=group.device_group)
|
||||
stream.synchronize()
|
||||
|
||||
def broadcast_expert_mapping(self) -> None:
|
||||
standby_dp_group = get_standby_dp_group()
|
||||
assert standby_dp_group is not None
|
||||
@@ -362,15 +416,22 @@ class ElasticEPScalingExecutor:
|
||||
torch.accelerator.empty_cache()
|
||||
|
||||
def switch_and_remove(self) -> None:
|
||||
# Removing ranks skipped preparation, so wait for prior cleanup here.
|
||||
self._wait_for_group_cleanup()
|
||||
self._release_cuda_graphs()
|
||||
_replace_active_groups(world=None, dp=None, ep=None, eplb=None, node_count=None)
|
||||
retired_groups = _replace_active_groups(
|
||||
world=None, dp=None, ep=None, eplb=None, node_count=None
|
||||
)
|
||||
self._start_group_cleanup(retired_groups)
|
||||
# Finish collective cleanup before this worker is shut down.
|
||||
self._wait_for_group_cleanup()
|
||||
|
||||
def switch_and_prepare(self) -> None:
|
||||
def switch_and_prepare(self) -> tuple[GroupCoordinator | None, ...]:
|
||||
old_dp_size = get_dp_group().world_size
|
||||
old_ep_size = get_ep_group().world_size
|
||||
|
||||
self._release_cuda_graphs()
|
||||
_replace_active_groups(**pop_standby_groups())
|
||||
retired_groups = _replace_active_groups(**pop_standby_groups())
|
||||
|
||||
parallel_config = self.worker.vllm_config.parallel_config
|
||||
reconfig_request = self.reconfig_request
|
||||
@@ -453,7 +514,6 @@ class ElasticEPScalingExecutor:
|
||||
)
|
||||
eplb_model_state.expert_load_pass = expanded_expert_load_pass
|
||||
eplb_model_state.expert_load_window = expanded_expert_load_window
|
||||
eplb_state.num_valid_physical_experts = old_num_physical_experts
|
||||
else:
|
||||
assert pad_size < 0
|
||||
eplb_model_state.expert_load_pass = eplb_model_state.expert_load_pass[
|
||||
@@ -462,7 +522,6 @@ class ElasticEPScalingExecutor:
|
||||
eplb_model_state.expert_load_window = eplb_model_state.expert_load_window[
|
||||
:, :, :num_physical_experts
|
||||
]
|
||||
eplb_state.num_valid_physical_experts = num_physical_experts
|
||||
|
||||
model = self.worker.model_runner.get_model()
|
||||
model.expert_weights = []
|
||||
@@ -485,18 +544,9 @@ class ElasticEPScalingExecutor:
|
||||
if getattr(module._quant_method, "wraps_legacy_quant_method", False):
|
||||
module._replace_quant_method(module._quant_method.old_quant_method)
|
||||
|
||||
eplb_model_state.expert_buffer = [
|
||||
torch.empty_like(w) for w in model.expert_weights[0]
|
||||
]
|
||||
assert parallel_config.eplb_config.communicator is not None, (
|
||||
"EPLB communicator backend must be set by ParallelConfig"
|
||||
)
|
||||
eplb_model_state.communicator = create_eplb_communicator(
|
||||
group_coordinator=get_eplb_group(),
|
||||
backend=parallel_config.eplb_config.communicator,
|
||||
expert_weights=model.expert_weights,
|
||||
expert_buffer=eplb_model_state.expert_buffer,
|
||||
)
|
||||
assert self._prepared_eplb_communicator is not None
|
||||
eplb_state.update_communicator(model_config, self._prepared_eplb_communicator)
|
||||
self._prepared_eplb_communicator = None
|
||||
|
||||
if (
|
||||
self.worker.vllm_config.compilation_config.mode
|
||||
@@ -513,8 +563,12 @@ class ElasticEPScalingExecutor:
|
||||
compilation_counter.stock_torch_compile_count += 1
|
||||
self.worker.model_runner.model.compile(fullgraph=True, backend=backend)
|
||||
|
||||
return retired_groups
|
||||
|
||||
def _perform_eplb_reshuffle(
|
||||
self, rank_mapping: dict[int, int] | None = None
|
||||
self,
|
||||
rank_mapping: dict[int, int] | None = None,
|
||||
async_op: bool = False,
|
||||
) -> None:
|
||||
if get_ep_group().rank == 0:
|
||||
logger.info("[Elastic EP] Starting expert resharding...")
|
||||
@@ -522,21 +576,19 @@ class ElasticEPScalingExecutor:
|
||||
eplb_state = self.worker.model_runner.eplb_state
|
||||
assert eplb_state is not None
|
||||
|
||||
model_config = self.worker.model_runner.model_config
|
||||
eplb_model_state = eplb_state.model_states[model_config.compute_hash()]
|
||||
is_async_enabled = eplb_state.is_async
|
||||
eplb_state.is_async = False
|
||||
run_async = async_op and is_async_enabled
|
||||
eplb_state.is_async = run_async
|
||||
if rank_mapping is None:
|
||||
eplb_state.rearrange()
|
||||
else:
|
||||
eplb_state.rearrange(rank_mapping=rank_mapping)
|
||||
# NOTE(yongji): check whether we need to synchronize here
|
||||
torch.accelerator.synchronize()
|
||||
if not run_async:
|
||||
# Wait for non-blocking expert resharding copies before continuing
|
||||
# the Elastic EP reconfiguration.
|
||||
torch.accelerator.synchronize()
|
||||
# reset expert_rearrangement_step to ensure all ranks are synchronized
|
||||
eplb_state.expert_rearrangement_step = 0
|
||||
eplb_state.num_valid_physical_experts = (
|
||||
eplb_model_state.physical_to_logical_map.shape[1]
|
||||
)
|
||||
eplb_state.is_async = is_async_enabled
|
||||
# Start the async worker thread if it doesn't exist yet (idempotent).
|
||||
# This is needed for new workers after scale-up: they create EplbState
|
||||
@@ -544,25 +596,31 @@ class ElasticEPScalingExecutor:
|
||||
# groups aren't ready yet.
|
||||
eplb_state.start_async_loop()
|
||||
if get_ep_group().rank == 0:
|
||||
logger.info("[Elastic EP] Expert resharding completed")
|
||||
logger.info(
|
||||
"[Elastic EP] Expert resharding %s",
|
||||
"scheduled" if run_async else "completed",
|
||||
)
|
||||
|
||||
def commit_scale_up(self, is_existing_worker: bool) -> None:
|
||||
if is_existing_worker:
|
||||
self.broadcast_expert_mapping()
|
||||
self.switch_and_prepare()
|
||||
retired_groups = self.switch_and_prepare()
|
||||
else:
|
||||
mapping, _, num_valid_experts = self.receive_expert_mapping()
|
||||
self.worker.model_runner.setup_eplb_from_mapping(mapping, num_valid_experts)
|
||||
self._perform_eplb_reshuffle()
|
||||
mapping = self.receive_expert_mapping()
|
||||
self.worker.model_runner.setup_eplb_from_mapping(mapping)
|
||||
self.warm_and_capture()
|
||||
self._perform_eplb_reshuffle(async_op=True)
|
||||
if is_existing_worker:
|
||||
self._start_group_cleanup(retired_groups)
|
||||
|
||||
def commit_scale_down(self, new_dp_size: int, removing: bool) -> None:
|
||||
self.perform_scale_down_eplb_reshuffle(new_dp_size)
|
||||
if removing:
|
||||
self.switch_and_remove()
|
||||
else:
|
||||
self.switch_and_prepare()
|
||||
retired_groups = self.switch_and_prepare()
|
||||
self.warm_and_capture()
|
||||
self._start_group_cleanup(retired_groups)
|
||||
|
||||
def perform_scale_down_eplb_reshuffle(self, new_dp_size: int) -> None:
|
||||
eplb_state = self.worker.model_runner.eplb_state
|
||||
@@ -578,7 +636,7 @@ class ElasticEPScalingExecutor:
|
||||
}
|
||||
self._perform_eplb_reshuffle(rank_mapping=rank_mapping)
|
||||
|
||||
def receive_weights(self) -> None:
|
||||
def prepare_new_worker(self) -> None:
|
||||
dp_group = get_dp_group()
|
||||
assert isinstance(dp_group, StatelessGroupCoordinator)
|
||||
new_dp_size = dp_group.world_size
|
||||
@@ -618,19 +676,18 @@ class ElasticEPScalingExecutor:
|
||||
expert_weights=expert_weights,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
self._warm_target_groups(get_dp_group(), get_ep_group())
|
||||
|
||||
def receive_expert_mapping(self) -> tuple[torch.Tensor, int, int]:
|
||||
def receive_expert_mapping(self) -> torch.Tensor:
|
||||
dp_group = get_dp_group()
|
||||
assert isinstance(dp_group, StatelessGroupCoordinator)
|
||||
physical_to_logical, num_local_physical_experts, num_logical_experts = (
|
||||
broadcast_expert_mapping(
|
||||
physical_to_logical=None,
|
||||
num_local_physical_experts=None,
|
||||
num_logical_experts=None,
|
||||
dp_group=dp_group,
|
||||
src_rank=0,
|
||||
device=self.worker.device,
|
||||
)
|
||||
physical_to_logical, num_local_physical_experts, _ = broadcast_expert_mapping(
|
||||
physical_to_logical=None,
|
||||
num_local_physical_experts=None,
|
||||
num_logical_experts=None,
|
||||
dp_group=dp_group,
|
||||
src_rank=0,
|
||||
device=self.worker.device,
|
||||
)
|
||||
num_moe_layers = physical_to_logical.shape[0]
|
||||
new_dp_size = get_dp_group().world_size
|
||||
@@ -642,13 +699,10 @@ class ElasticEPScalingExecutor:
|
||||
dtype=physical_to_logical.dtype,
|
||||
device=physical_to_logical.device,
|
||||
)
|
||||
old_num_physical_experts = physical_to_logical.shape[1]
|
||||
expanded_physical_to_logical[:, :old_num_physical_experts] = physical_to_logical
|
||||
return (
|
||||
expanded_physical_to_logical,
|
||||
num_logical_experts,
|
||||
old_num_physical_experts,
|
||||
expanded_physical_to_logical[:, : physical_to_logical.shape[1]] = (
|
||||
physical_to_logical
|
||||
)
|
||||
return expanded_physical_to_logical
|
||||
|
||||
def warmup_local_kernels(self) -> None:
|
||||
with set_current_vllm_config(self.worker.vllm_config):
|
||||
|
||||
Reference in New Issue
Block a user