[Elastic EP] Async preparation (#47288)
Signed-off-by: Itay Alroy <ialroy@nvidia.com>
This commit is contained in:
@@ -175,6 +175,7 @@ class DeviceCommunicatorBase:
|
||||
unique_name: str = "",
|
||||
global_ranks: list[int] | None = None,
|
||||
global_world_size: int | None = None,
|
||||
use_all2all: bool = False,
|
||||
):
|
||||
self.device = device or torch.device("cpu")
|
||||
self.cpu_group = cpu_group
|
||||
@@ -204,26 +205,15 @@ class DeviceCommunicatorBase:
|
||||
self.global_world_size = dist.get_world_size()
|
||||
self.rank_in_group = dist.get_group_rank(self.cpu_group, self.global_rank)
|
||||
|
||||
use_ep = False
|
||||
all2all_backend = None
|
||||
from vllm.config import get_current_vllm_config_or_none
|
||||
|
||||
config = get_current_vllm_config_or_none()
|
||||
if config is not None:
|
||||
# initialize the all2all manager for DP or sequence-parallel EP.
|
||||
parallel_config = config.parallel_config
|
||||
use_ep = (
|
||||
parallel_config.data_parallel_size > 1
|
||||
or parallel_config.use_sequence_parallel_moe
|
||||
or (
|
||||
parallel_config.enable_expert_parallel
|
||||
and parallel_config.prefill_context_parallel_size > 1
|
||||
)
|
||||
)
|
||||
all2all_backend = parallel_config.all2all_backend
|
||||
all2all_backend = config.parallel_config.all2all_backend
|
||||
|
||||
self.is_ep_communicator = unique_name.split(":")[0] == "ep"
|
||||
self.use_all2all = self.is_ep_communicator and use_ep
|
||||
self.use_all2all = self.is_ep_communicator and use_all2all
|
||||
self.all2all_backend = all2all_backend
|
||||
self.all2all_manager: All2AllManagerBase | None = None
|
||||
|
||||
|
||||
@@ -24,8 +24,11 @@ class CpuCommunicator(DeviceCommunicatorBase):
|
||||
device: torch.device | None = None,
|
||||
device_group: ProcessGroup | None = None,
|
||||
unique_name: str = "",
|
||||
use_all2all: bool = False,
|
||||
):
|
||||
super().__init__(cpu_group, device, device_group, unique_name)
|
||||
super().__init__(
|
||||
cpu_group, device, device_group, unique_name, use_all2all=use_all2all
|
||||
)
|
||||
self.dist_module = torch.distributed
|
||||
|
||||
if (
|
||||
|
||||
@@ -36,6 +36,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
global_ranks: list[int] | None = None,
|
||||
global_world_size: int | None = None,
|
||||
tcp_store_group: StatelessProcessGroup | None = None,
|
||||
use_all2all: bool = False,
|
||||
):
|
||||
super().__init__(
|
||||
cpu_group,
|
||||
@@ -44,6 +45,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
unique_name,
|
||||
global_ranks,
|
||||
global_world_size,
|
||||
use_all2all=use_all2all,
|
||||
)
|
||||
if "tp" not in unique_name:
|
||||
# custom allreduce or torch symm mem can be used only by tp
|
||||
|
||||
@@ -20,8 +20,11 @@ class XpuCommunicator(DeviceCommunicatorBase):
|
||||
device: torch.device | None = None,
|
||||
device_group: ProcessGroup | None = None,
|
||||
unique_name: str = "",
|
||||
use_all2all: bool = False,
|
||||
):
|
||||
super().__init__(cpu_group, device, device_group, unique_name)
|
||||
super().__init__(
|
||||
cpu_group, device, device_group, unique_name, use_all2all=use_all2all
|
||||
)
|
||||
self.ca_comm: None = None
|
||||
if self.use_all2all:
|
||||
if self.all2all_backend in ("naive", "allgather_reducescatter"):
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import copy
|
||||
import gc
|
||||
import weakref
|
||||
from collections.abc import Iterable, Sequence
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from dataclasses import replace
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
@@ -43,6 +43,7 @@ from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig
|
||||
from vllm.model_executor.layers.fused_moe.eep_reconfigure import (
|
||||
make_eep_staged_quant_method,
|
||||
)
|
||||
from vllm.model_executor.warmup.kernel_warmup import kernel_warmup
|
||||
from vllm.utils import is_moe_layer
|
||||
from vllm.v1.engine import ReconfigureDistributedRequest, ReconfigureRankType
|
||||
from vllm.v1.worker.gpu_ubatch_wrapper import UBatchWrapper
|
||||
@@ -145,6 +146,10 @@ class ElasticEPScalingExecutor:
|
||||
self.worker_ref = weakref.ref(worker)
|
||||
self.reconfig_request = None
|
||||
self._staged_moe_quant_methods: dict[nn.Module, FusedMoEMethodBase] = {}
|
||||
self._async_executor = ThreadPoolExecutor(
|
||||
max_workers=1, thread_name_prefix="ElasticEPAsync"
|
||||
)
|
||||
self._async_future: Future[None] | None = None
|
||||
|
||||
@property
|
||||
def worker(self):
|
||||
@@ -159,59 +164,68 @@ class ElasticEPScalingExecutor:
|
||||
raise ValueError(f"Unknown execute method: {execute_method}")
|
||||
return method(*args, **kwargs)
|
||||
|
||||
def _set_eplb_suppressed(self, suppressed: bool) -> None:
|
||||
self.worker.model_runner.eep_eplb_suppressed = suppressed
|
||||
ep_group = get_standby_ep_group() or get_ep_group()
|
||||
if ep_group.rank == 0:
|
||||
logger.info(
|
||||
"[Elastic EP] EPLB %s elastic scaling transition",
|
||||
"disabled during" if suppressed else "re-enabled after",
|
||||
)
|
||||
def start_async(self, execute_method: str, *args, **kwargs) -> str:
|
||||
if self._async_future is not None:
|
||||
raise RuntimeError("Another Elastic EP async method is active")
|
||||
if args and isinstance(args[0], ReconfigureDistributedRequest):
|
||||
self.reconfig_request = args[0]
|
||||
dp_rank = self.worker.vllm_config.parallel_config.data_parallel_rank
|
||||
done_key = f"eep_async/{execute_method}/{dp_rank}/{self.worker.rank}"
|
||||
self._async_future = self._async_executor.submit(
|
||||
self._run_async, execute_method, *args, **kwargs
|
||||
)
|
||||
self._async_future.add_done_callback(lambda _: self._mark_async_done(done_key))
|
||||
return done_key
|
||||
|
||||
def _run_async(self, execute_method: str, *args, **kwargs) -> None:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
self.worker.vllm_config.enable_trace_function_call_for_thread()
|
||||
assert hasattr(self.worker, "device")
|
||||
current_platform.set_device(self.worker.device)
|
||||
with set_current_vllm_config(self.worker.vllm_config):
|
||||
self.execute(execute_method, *args, **kwargs)
|
||||
|
||||
def _mark_async_done(self, done_key: str) -> None:
|
||||
from vllm.distributed.utils import get_cached_tcp_store_client
|
||||
|
||||
assert self.reconfig_request is not None
|
||||
get_cached_tcp_store_client(
|
||||
self.reconfig_request.new_data_parallel_master_ip,
|
||||
self.reconfig_request.coord_store_port,
|
||||
).set(done_key, b"1")
|
||||
|
||||
def clear_async(self) -> None:
|
||||
future = self._async_future
|
||||
if future is None:
|
||||
raise RuntimeError("No Elastic EP async method is active")
|
||||
if not future.done():
|
||||
raise RuntimeError("Elastic EP async method is not done")
|
||||
self._async_future = None
|
||||
future.result()
|
||||
|
||||
def load_model(self) -> None:
|
||||
(
|
||||
expanded_physical_to_logical,
|
||||
num_logical_experts,
|
||||
old_num_physical_experts,
|
||||
) = self.receive_expert_mapping()
|
||||
num_physical_experts = expanded_physical_to_logical.shape[1]
|
||||
self.worker.parallel_config.eplb_config.num_redundant_experts = (
|
||||
num_physical_experts - num_logical_experts
|
||||
)
|
||||
self.worker.load_model(load_dummy_weights=True)
|
||||
self.worker.model_runner.setup_eplb_from_mapping(
|
||||
expanded_physical_to_logical, old_num_physical_experts
|
||||
)
|
||||
self._set_eplb_suppressed(True)
|
||||
|
||||
def create_standby_groups(
|
||||
self, reconfig_request: ReconfigureDistributedRequest
|
||||
self, reconfig_request: ReconfigureDistributedRequest, use_all2all: bool
|
||||
) -> None:
|
||||
self.reconfig_request = reconfig_request
|
||||
new_dp_size = reconfig_request.new_data_parallel_size
|
||||
old_dp_size = get_dp_group().world_size
|
||||
world_size = self.worker.vllm_config.parallel_config.world_size
|
||||
parallel_config = self.worker.vllm_config.parallel_config
|
||||
world_size = parallel_config.world_size
|
||||
new_world_size_across_dp = world_size * new_dp_size
|
||||
updated_config = copy.copy(self.worker.vllm_config)
|
||||
updated_config.parallel_config = copy.deepcopy(
|
||||
self.worker.vllm_config.parallel_config
|
||||
create_standby_groups(
|
||||
new_dp_size=new_dp_size,
|
||||
new_world_size_across_dp=new_world_size_across_dp,
|
||||
master_ip=reconfig_request.new_data_parallel_master_ip,
|
||||
coord_store_port=reconfig_request.coord_store_port,
|
||||
use_all2all=use_all2all,
|
||||
enable_eplb=parallel_config.enable_eplb,
|
||||
)
|
||||
updated_config.parallel_config.data_parallel_size = new_dp_size
|
||||
with set_current_vllm_config(updated_config):
|
||||
create_standby_groups(
|
||||
new_dp_size=new_dp_size,
|
||||
new_world_size_across_dp=new_world_size_across_dp,
|
||||
master_ip=reconfig_request.new_data_parallel_master_ip,
|
||||
coord_store_port=reconfig_request.coord_store_port,
|
||||
enable_eplb=updated_config.parallel_config.enable_eplb,
|
||||
)
|
||||
if new_dp_size > old_dp_size:
|
||||
self._set_eplb_suppressed(True)
|
||||
eplb_state = self.worker.model_runner.eplb_state
|
||||
if eplb_state is not None:
|
||||
eplb_state.drain_async()
|
||||
elif new_dp_size < old_dp_size:
|
||||
self._stage_standby_moe_quant_methods()
|
||||
if new_dp_size < old_dp_size:
|
||||
self.stage_standby_moe_quant_methods()
|
||||
|
||||
def transfer_weights(self, old_dp_size: int, new_dp_size: int) -> None:
|
||||
standby_dp_group = get_standby_dp_group()
|
||||
@@ -265,6 +279,7 @@ class ElasticEPScalingExecutor:
|
||||
model_config = self.worker.model_runner.model_config
|
||||
eplb_state = self.worker.model_runner.eplb_state
|
||||
assert eplb_state is not None
|
||||
eplb_state.drain_async()
|
||||
eplb_model_state = eplb_state.model_states[model_config.compute_hash()]
|
||||
physical_to_logical = eplb_model_state.physical_to_logical_map
|
||||
num_physical_experts = physical_to_logical.shape[1]
|
||||
@@ -278,10 +293,6 @@ class ElasticEPScalingExecutor:
|
||||
src_rank=0,
|
||||
device=self.worker.device,
|
||||
)
|
||||
# New workers enter load_model after receiving the expert mapping.
|
||||
# Stage replacement MoE kernels before returning to the state machine
|
||||
# so existing ranks can participate in collective EP comm creation.
|
||||
self._stage_standby_moe_quant_methods()
|
||||
|
||||
def _make_eep_moe_config(self, module, dp_group, ep_group):
|
||||
parallel_config = self.worker.vllm_config.parallel_config
|
||||
@@ -300,7 +311,7 @@ class ElasticEPScalingExecutor:
|
||||
moe_parallel_config=moe_parallel_config,
|
||||
)
|
||||
|
||||
def _stage_standby_moe_quant_methods(self) -> None:
|
||||
def stage_standby_moe_quant_methods(self) -> None:
|
||||
standby_dp_group = get_standby_dp_group()
|
||||
standby_ep_group = get_standby_ep_group()
|
||||
model = self.worker.model_runner.get_model()
|
||||
@@ -500,26 +511,6 @@ class ElasticEPScalingExecutor:
|
||||
compilation_counter.stock_torch_compile_count += 1
|
||||
self.worker.model_runner.model.compile(fullgraph=True, backend=backend)
|
||||
|
||||
multi_block_table = self.worker.model_runner.input_batch.block_table
|
||||
saved_block_tables: list[tuple[torch.Tensor, torch.Tensor]] = []
|
||||
for bt in multi_block_table.block_tables:
|
||||
saved_block_tables.append(
|
||||
(bt.block_table.gpu.clone(), bt.block_table.cpu.clone())
|
||||
)
|
||||
multi_block_table.clear()
|
||||
|
||||
unlock_workspace()
|
||||
self.worker.compile_or_warm_up_model()
|
||||
lock_workspace()
|
||||
|
||||
for bt, (saved_gpu, saved_cpu) in zip(
|
||||
multi_block_table.block_tables, saved_block_tables
|
||||
):
|
||||
bt.block_table.gpu.copy_(saved_gpu)
|
||||
bt.block_table.cpu.copy_(saved_cpu)
|
||||
if new_dp_size < old_dp_size:
|
||||
self._set_eplb_suppressed(False)
|
||||
|
||||
def _perform_eplb_reshuffle(
|
||||
self, rank_mapping: dict[int, int] | None = None
|
||||
) -> None:
|
||||
@@ -553,12 +544,25 @@ class ElasticEPScalingExecutor:
|
||||
if get_ep_group().rank == 0:
|
||||
logger.info("[Elastic EP] Expert resharding completed")
|
||||
|
||||
def perform_eplb_reshuffle(self) -> None:
|
||||
def commit_scale_up(self, is_existing_worker: bool) -> None:
|
||||
if is_existing_worker:
|
||||
self.broadcast_expert_mapping()
|
||||
self.switch_and_prepare()
|
||||
else:
|
||||
mapping, _, num_valid_experts = self.receive_expert_mapping()
|
||||
self.worker.model_runner.setup_eplb_from_mapping(mapping, num_valid_experts)
|
||||
self._perform_eplb_reshuffle()
|
||||
self._set_eplb_suppressed(False)
|
||||
self.warm_and_capture()
|
||||
|
||||
def commit_scale_down(self, new_dp_size: int, removing: bool) -> None:
|
||||
self.perform_scale_down_eplb_reshuffle(new_dp_size)
|
||||
if removing:
|
||||
self.switch_and_remove()
|
||||
else:
|
||||
self.switch_and_prepare()
|
||||
self.warm_and_capture()
|
||||
|
||||
def perform_scale_down_eplb_reshuffle(self, new_dp_size: int) -> None:
|
||||
self._set_eplb_suppressed(True)
|
||||
eplb_state = self.worker.model_runner.eplb_state
|
||||
if eplb_state is not None:
|
||||
eplb_state.drain_async()
|
||||
@@ -599,12 +603,17 @@ class ElasticEPScalingExecutor:
|
||||
)
|
||||
|
||||
model = self.worker.model_runner.get_model()
|
||||
expert_weights = [
|
||||
module.get_expert_weights()
|
||||
for module in model.modules()
|
||||
if is_moe_layer(module)
|
||||
]
|
||||
batch_transfer_weights(
|
||||
model=model,
|
||||
is_sender=False,
|
||||
peer_rank=sender_rank,
|
||||
dp_group=dp_group,
|
||||
expert_weights=model.expert_weights,
|
||||
expert_weights=expert_weights,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
@@ -643,14 +652,17 @@ class ElasticEPScalingExecutor:
|
||||
with set_current_vllm_config(self.worker.vllm_config):
|
||||
prepare_communication_buffer_for_model(self.worker.model_runner.get_model())
|
||||
|
||||
def rewarm_workspace(self) -> None:
|
||||
def warmup_local_kernels(self) -> None:
|
||||
with set_current_vllm_config(self.worker.vllm_config):
|
||||
kernel_warmup(self.worker, process_local_only=True)
|
||||
|
||||
def warm_and_capture(self) -> None:
|
||||
# Must run on every DP sibling in lockstep: _dummy_run calls
|
||||
# coordinate_batch_across_dp whenever data_parallel_size > 1
|
||||
# (gpu_model_runner.py:3663), which deadlocks if any rank skips it.
|
||||
|
||||
# Save and clear block tables so profile_run/compile_or_warm_up_model
|
||||
# don't write dummy slot mappings into real KV-cache blocks (mirrors
|
||||
# switch_and_prepare's pattern).
|
||||
# Save and clear block tables so the dummy MoE forward doesn't
|
||||
# write dummy slot mappings into real KV-cache blocks.
|
||||
multi_block_table = self.worker.model_runner.input_batch.block_table
|
||||
saved_block_tables: list[tuple[torch.Tensor, torch.Tensor]] = []
|
||||
for bt in multi_block_table.block_tables:
|
||||
@@ -660,19 +672,16 @@ class ElasticEPScalingExecutor:
|
||||
multi_block_table.clear()
|
||||
|
||||
# _ensure_workspace_size allocates a fresh tensor on grow, leaving
|
||||
# captured CUDA graphs with stale data pointers; drop graphs before
|
||||
# re-warm so captures realign with the resized buffer.
|
||||
# any captured CUDA graph with a stale data pointer; drop graphs
|
||||
# before re-warm so captures realign with the resized buffer.
|
||||
self._release_cuda_graphs()
|
||||
unlock_workspace()
|
||||
|
||||
# Grow the MoE workspace at max_num_tokens.
|
||||
# compile_or_warm_up_model alone only exercises cudagraph-capture
|
||||
# sizes (≤64 tokens for this test) and leaves the workspace at
|
||||
# ~10-14 MB; the post-all-to-all per-rank token count under real
|
||||
# post-reshuffle routing needs hundreds of MB. Use _dummy_run
|
||||
# directly (rather than profile_run) with skip_eplb=True so dummy
|
||||
# routing doesn't pollute the just-rebalanced EPLB stats — same
|
||||
# convention compile_or_warm_up_model itself uses.
|
||||
# Grow the MoE workspace at max_num_tokens. compile_or_warm_up_model
|
||||
# alone only exercises cudagraph-capture sizes and can leave the
|
||||
# workspace too small for post-reshuffle routing. Use _dummy_run
|
||||
# directly with skip_eplb=True so dummy routing doesn't pollute the
|
||||
# just-rebalanced EPLB stats.
|
||||
runner = self.worker.model_runner
|
||||
runner._dummy_run(runner.max_num_tokens, is_profile=True, skip_eplb=True)
|
||||
self.worker.compile_or_warm_up_model()
|
||||
|
||||
@@ -1,18 +1,17 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import enum
|
||||
import time
|
||||
import weakref
|
||||
from datetime import timedelta
|
||||
from typing import TYPE_CHECKING, Literal, TypeAlias
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from typing import TYPE_CHECKING, Any, Literal, TypeAlias
|
||||
|
||||
import torch.distributed
|
||||
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.distributed import (
|
||||
sched_yield,
|
||||
stateless_destroy_torch_distributed_process_group,
|
||||
)
|
||||
from vllm.distributed.utils import get_cached_tcp_store_client
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.engine import (
|
||||
EEPNotificationType,
|
||||
@@ -31,35 +30,29 @@ WorkerType = Literal["existing", "new", "removing"]
|
||||
|
||||
|
||||
class ScaleUpExistingEngineState(enum.IntEnum):
|
||||
WAIT_NEW_CORE_ENGINES_INIT = 0
|
||||
CREATE_STANDBY_GROUPS = 1
|
||||
TRANSFER_EXPERT_MAPPING = 2
|
||||
WAIT_NEW_CORE_ENGINES_WEIGHTS_INIT = 3
|
||||
TRANSFER_WEIGHTS = 4
|
||||
SYNC_KV_CACHE_MEMORY_SIZE = 5
|
||||
SWITCH_AND_PREPARE = 6
|
||||
EPLB_RESHUFFLE = 7
|
||||
COMPLETE = 8
|
||||
CREATE_STANDBY_GROUPS = 0
|
||||
STAGE_QUANT_METHODS = 1
|
||||
TRANSFER_WEIGHTS = 2
|
||||
SYNC_KV_CACHE_MEMORY_SIZE = 3
|
||||
COMMIT_SCALE_UP = 4 # Blocks forward passes.
|
||||
COMPLETE = 5
|
||||
|
||||
|
||||
class ScaleUpNewEngineState(enum.IntEnum):
|
||||
PRE_KV_INIT = 0
|
||||
PREPARE = 1
|
||||
EPLB_RESHUFFLE = 2
|
||||
COMPLETE = 3
|
||||
COMPLETE = 2
|
||||
|
||||
|
||||
class ScaleDownRemainingEngineState(enum.IntEnum):
|
||||
PREPARE = 0
|
||||
EPLB_RESHUFFLE = 1
|
||||
SWITCH_AND_PREPARE = 2
|
||||
COMPLETE = 3
|
||||
COMMIT_SCALE_DOWN = 1 # Blocks forward passes.
|
||||
COMPLETE = 2
|
||||
|
||||
|
||||
class ScaleDownRemovingEngineState(enum.IntEnum):
|
||||
PREPARE = 0
|
||||
EPLB_RESHUFFLE = 1
|
||||
COMPLETE = 2
|
||||
COMPLETE = 1
|
||||
|
||||
|
||||
EngineState: TypeAlias = (
|
||||
@@ -70,15 +63,6 @@ EngineState: TypeAlias = (
|
||||
)
|
||||
|
||||
|
||||
class _BarrierTimeoutError(RuntimeError):
|
||||
"""
|
||||
Exception raised for timeout
|
||||
in the first stage of our two-staged
|
||||
TCPStore based barrier to synchronize the
|
||||
execution of all engines in the DP group.
|
||||
"""
|
||||
|
||||
|
||||
class ElasticEPScalingState:
|
||||
def __init__(
|
||||
self,
|
||||
@@ -94,20 +78,24 @@ class ElasticEPScalingState:
|
||||
self.engine_core_ref = weakref.ref(engine_core)
|
||||
self.vllm_config = vllm_config
|
||||
self.old_dp_group = self.engine_core.dp_group if worker_type != "new" else None
|
||||
self.old_dp_store = self.engine_core.dp_store if worker_type != "new" else None
|
||||
self.new_parallel_config: ParallelConfig = new_parallel_config
|
||||
self.new_dp_group = self.engine_core.dp_group if worker_type == "new" else None
|
||||
self.new_dp_store = self.engine_core.dp_store if worker_type == "new" else None
|
||||
self.worker_type = worker_type
|
||||
self.scale_type = scale_type
|
||||
self.reconfig_request = reconfig_request
|
||||
|
||||
self.commit_requested = False
|
||||
self._prepare_executor = ThreadPoolExecutor(
|
||||
max_workers=1, thread_name_prefix="ElasticEPPrepare"
|
||||
)
|
||||
self._prepare_future: Future[Any] | None = None
|
||||
self._new_dp_sync: tuple[object, Any] | None = None
|
||||
self.state: EngineState
|
||||
if scale_type == "scale_up":
|
||||
self.state = (
|
||||
ScaleUpNewEngineState.PRE_KV_INIT
|
||||
if worker_type == "new"
|
||||
else ScaleUpExistingEngineState.WAIT_NEW_CORE_ENGINES_INIT
|
||||
else ScaleUpExistingEngineState.CREATE_STANDBY_GROUPS
|
||||
)
|
||||
else:
|
||||
self.state = (
|
||||
@@ -130,6 +118,31 @@ class ElasticEPScalingState:
|
||||
raise RuntimeError("Engine core has been garbage collected")
|
||||
return engine_core
|
||||
|
||||
def _collective_rpc(self, *args, **kwargs):
|
||||
return self.model_executor.collective_rpc(*args, **kwargs)
|
||||
|
||||
def _execute_async(self, execute_method: str, *args) -> bool:
|
||||
if self._prepare_future is None:
|
||||
done_keys = self._collective_rpc(
|
||||
"elastic_ep_execute",
|
||||
args=("start_async", execute_method, *args),
|
||||
)
|
||||
assert self.reconfig_request is not None
|
||||
coord_store = get_cached_tcp_store_client(
|
||||
self.reconfig_request.new_data_parallel_master_ip,
|
||||
self.reconfig_request.coord_store_port,
|
||||
)
|
||||
self._prepare_future = self._prepare_executor.submit(
|
||||
coord_store.wait, done_keys
|
||||
)
|
||||
if not self._prepare_future.done():
|
||||
return False
|
||||
|
||||
self._prepare_future.result()
|
||||
self._collective_rpc("elastic_ep_execute", args=("clear_async",))
|
||||
self._prepare_future = None
|
||||
return True
|
||||
|
||||
def progress(self) -> bool:
|
||||
if self.scale_type == "scale_up":
|
||||
return (
|
||||
@@ -149,157 +162,43 @@ class ElasticEPScalingState:
|
||||
assert self.progress()
|
||||
assert self.state == ScaleUpNewEngineState.PREPARE
|
||||
|
||||
def _execute_tcp_store_barrier(
|
||||
self, dp_store, group_rank, group_size, barrier_id, timeout=None
|
||||
):
|
||||
arrival_key = f"arrival_{barrier_id}_{group_rank}"
|
||||
dp_store.set(arrival_key, b"1")
|
||||
|
||||
start_time = time.time()
|
||||
processes_arrived: set[int] = set()
|
||||
|
||||
while len(processes_arrived) < group_size:
|
||||
if (
|
||||
timeout is not None
|
||||
and time.time() - start_time > timeout.total_seconds()
|
||||
):
|
||||
raise _BarrierTimeoutError(
|
||||
f"Barrier timed out after {timeout.total_seconds()} seconds"
|
||||
)
|
||||
|
||||
for i in range(group_size):
|
||||
if i in processes_arrived:
|
||||
continue
|
||||
|
||||
key = f"arrival_{barrier_id}_{i}"
|
||||
present = dp_store.check([key])
|
||||
if present:
|
||||
processes_arrived.add(i)
|
||||
|
||||
if len(processes_arrived) < group_size:
|
||||
sched_yield()
|
||||
|
||||
def _staged_barrier(self, use_new_group: bool, barrier_name: str) -> bool:
|
||||
"""
|
||||
Execute a two-staged barrier to synchronize all engines in the DP group.
|
||||
|
||||
Some DP EngineCores may receive the reconfiguration notifications
|
||||
later than others, and already proceed to engine step (model forward)
|
||||
in the busy loop.
|
||||
In this case, EngineCores that already proceed to reconfiguration
|
||||
should skip reconfiguration and execute model forward for one more
|
||||
step, so in the next step, all EngineCores will be synchronized.
|
||||
We use a two-staged barrier to achieve this. The first time each
|
||||
EngineCore executes the barrier, if a timeout is reached before the
|
||||
barrier completes, that means some EngineCores have already entered
|
||||
engine step. The EngineCores that timed out will then proceed to
|
||||
engine step, and will synchronize with the other EngineCores in the
|
||||
next step with a barrier without timeout.
|
||||
"""
|
||||
dp_group = self.new_dp_group if use_new_group else self.old_dp_group
|
||||
dp_store = self.new_dp_store if use_new_group else self.old_dp_store
|
||||
assert dp_group is not None and dp_store is not None
|
||||
|
||||
group_rank = dp_group.rank()
|
||||
group_size = dp_group.size()
|
||||
barrier_id = f"eep_barrier_{barrier_name}"
|
||||
sync_key = f"{barrier_id}_sync"
|
||||
|
||||
# TODO(yongji): figure out appropriate timeout for the barrier
|
||||
timeout = None if dp_store.check([sync_key]) else timedelta(seconds=5)
|
||||
|
||||
try:
|
||||
self._execute_tcp_store_barrier(
|
||||
dp_store, group_rank, group_size, barrier_id, timeout=timeout
|
||||
)
|
||||
torch.distributed.barrier(dp_group)
|
||||
if group_rank == 0:
|
||||
dp_store.delete_key(sync_key)
|
||||
for i in range(group_size):
|
||||
dp_store.delete_key(f"arrival_{barrier_id}_{i}")
|
||||
return True
|
||||
except _BarrierTimeoutError as e:
|
||||
if timeout is None:
|
||||
raise RuntimeError("Unexpected timeout encountered") from e
|
||||
dp_store.compare_set(sync_key, "", b"1")
|
||||
return False
|
||||
|
||||
def _progress_existing_engine(self) -> bool:
|
||||
state = self.state
|
||||
assert self.old_dp_group is not None and self.old_dp_store is not None
|
||||
assert self.old_dp_group is not None
|
||||
|
||||
if state == ScaleUpExistingEngineState.WAIT_NEW_CORE_ENGINES_INIT:
|
||||
return False
|
||||
|
||||
elif state == ScaleUpExistingEngineState.CREATE_STANDBY_GROUPS:
|
||||
# NOTE(yongji): wait for all existing workers to receive the request
|
||||
if (
|
||||
int(self.old_dp_store.get("eep_barrier_engine_count"))
|
||||
< self.old_dp_group.size()
|
||||
):
|
||||
if state == ScaleUpExistingEngineState.CREATE_STANDBY_GROUPS:
|
||||
if not self._create_standby_groups():
|
||||
return False
|
||||
if not self._staged_barrier(
|
||||
use_new_group=False, barrier_name="create_standby_groups"
|
||||
):
|
||||
return False
|
||||
if self.old_dp_group.rank() == 0:
|
||||
self.old_dp_store.delete_key("eep_barrier_engine_count")
|
||||
self._create_standby_groups()
|
||||
self.state = ScaleUpExistingEngineState.TRANSFER_EXPERT_MAPPING
|
||||
self.state = ScaleUpExistingEngineState.STAGE_QUANT_METHODS
|
||||
return True
|
||||
|
||||
elif state == ScaleUpExistingEngineState.TRANSFER_EXPERT_MAPPING:
|
||||
self._transfer_expert_mapping()
|
||||
self.state = ScaleUpExistingEngineState.WAIT_NEW_CORE_ENGINES_WEIGHTS_INIT
|
||||
elif state == ScaleUpExistingEngineState.STAGE_QUANT_METHODS:
|
||||
if not self._execute_async("stage_standby_moe_quant_methods"):
|
||||
return False
|
||||
self.state = ScaleUpExistingEngineState.TRANSFER_WEIGHTS
|
||||
return True
|
||||
|
||||
elif state == ScaleUpExistingEngineState.WAIT_NEW_CORE_ENGINES_WEIGHTS_INIT:
|
||||
return False
|
||||
|
||||
elif state == ScaleUpExistingEngineState.TRANSFER_WEIGHTS:
|
||||
if (
|
||||
int(self.old_dp_store.get("eep_barrier_engine_count"))
|
||||
< self.old_dp_group.size()
|
||||
):
|
||||
if not self._transfer_weights():
|
||||
return False
|
||||
if not self._staged_barrier(
|
||||
use_new_group=False, barrier_name="transfer_weights"
|
||||
):
|
||||
return False
|
||||
if self.old_dp_group.rank() == 0:
|
||||
self.old_dp_store.delete_key("eep_barrier_engine_count")
|
||||
self._transfer_weights()
|
||||
self.state = ScaleUpExistingEngineState.SYNC_KV_CACHE_MEMORY_SIZE
|
||||
return True
|
||||
|
||||
elif state == ScaleUpExistingEngineState.SYNC_KV_CACHE_MEMORY_SIZE:
|
||||
self._sync_kv_cache_memory_size()
|
||||
self.state = ScaleUpExistingEngineState.SWITCH_AND_PREPARE
|
||||
if not self._sync_kv_cache_memory_size():
|
||||
return False
|
||||
self.state = ScaleUpExistingEngineState.COMMIT_SCALE_UP
|
||||
self._mark_ready_for_switch()
|
||||
return True
|
||||
|
||||
elif state == ScaleUpExistingEngineState.SWITCH_AND_PREPARE:
|
||||
self._switch_and_prepare()
|
||||
self.state = ScaleUpExistingEngineState.EPLB_RESHUFFLE
|
||||
assert self.new_dp_store is not None
|
||||
self.new_dp_store.add("eep_barrier_engine_count", 1)
|
||||
return True
|
||||
|
||||
elif state == ScaleUpExistingEngineState.EPLB_RESHUFFLE:
|
||||
assert self.new_dp_group is not None and self.new_dp_store is not None
|
||||
if (
|
||||
int(self.new_dp_store.get("eep_barrier_engine_count"))
|
||||
< self.new_dp_group.size()
|
||||
):
|
||||
elif state == ScaleUpExistingEngineState.COMMIT_SCALE_UP:
|
||||
if not self.commit_requested:
|
||||
return False
|
||||
if not self._staged_barrier(
|
||||
use_new_group=True, barrier_name="eplb_reshuffle"
|
||||
):
|
||||
return False
|
||||
if self.new_dp_group.rank() == 0:
|
||||
self.new_dp_store.delete_key("eep_barrier_engine_count")
|
||||
self._eplb_reshuffle()
|
||||
self._commit_new_dp_group()
|
||||
self._collective_rpc("elastic_ep_execute", args=("commit_scale_up", True))
|
||||
self.state = ScaleUpExistingEngineState.COMPLETE
|
||||
self._update_parallel_config()
|
||||
self._send_reconfigure_finished()
|
||||
return True
|
||||
|
||||
else:
|
||||
@@ -311,22 +210,17 @@ class ElasticEPScalingState:
|
||||
assert self.new_dp_group is not None and self.new_dp_store is not None
|
||||
|
||||
if state == ScaleUpNewEngineState.PRE_KV_INIT:
|
||||
self.engine_core._eep_send_engine_core_notification(
|
||||
EEPNotificationType.NEW_CORE_ENGINES_WEIGHTS_INIT_READY
|
||||
)
|
||||
self.model_executor.collective_rpc(
|
||||
"elastic_ep_execute", args=("receive_weights",)
|
||||
)
|
||||
self._collective_rpc("elastic_ep_execute", args=("receive_weights",))
|
||||
self.engine_core.available_gpu_memory_for_kv_cache = (
|
||||
ParallelConfig.sync_kv_cache_memory_size(self.new_dp_group, -1)
|
||||
)
|
||||
self.model_executor.collective_rpc(
|
||||
"elastic_ep_execute", args=("prepare_new_worker",)
|
||||
)
|
||||
self._collective_rpc("elastic_ep_execute", args=("prepare_new_worker",))
|
||||
self.state = ScaleUpNewEngineState.PREPARE
|
||||
return True
|
||||
|
||||
elif state == ScaleUpNewEngineState.PREPARE:
|
||||
self._collective_rpc("elastic_ep_execute", args=("warmup_local_kernels",))
|
||||
self._mark_ready_for_switch()
|
||||
tensor = torch.tensor([0, 0, 0], dtype=torch.int32, device="cpu")
|
||||
torch.distributed.all_reduce(
|
||||
tensor,
|
||||
@@ -337,22 +231,7 @@ class ElasticEPScalingState:
|
||||
self.engine_core.engines_running = bool(data[0])
|
||||
self.engine_core.current_wave = int(data[1])
|
||||
self.engine_core.step_counter = int(data[2])
|
||||
self.state = ScaleUpNewEngineState.EPLB_RESHUFFLE
|
||||
self.new_dp_store.add("eep_barrier_engine_count", 1)
|
||||
return True
|
||||
|
||||
elif state == ScaleUpNewEngineState.EPLB_RESHUFFLE:
|
||||
if (
|
||||
int(self.new_dp_store.get("eep_barrier_engine_count"))
|
||||
< self.new_dp_group.size()
|
||||
):
|
||||
return False
|
||||
if not self._staged_barrier(
|
||||
use_new_group=True, barrier_name="eplb_reshuffle"
|
||||
):
|
||||
return False
|
||||
assert self.new_dp_group.rank() > 0
|
||||
self._eplb_reshuffle()
|
||||
self._collective_rpc("elastic_ep_execute", args=("commit_scale_up", False))
|
||||
self.state = ScaleUpNewEngineState.COMPLETE
|
||||
return True
|
||||
|
||||
@@ -362,38 +241,23 @@ class ElasticEPScalingState:
|
||||
|
||||
def _progress_remaining_engine(self) -> bool:
|
||||
state = self.state
|
||||
assert self.old_dp_group is not None and self.old_dp_store is not None
|
||||
assert self.old_dp_group is not None
|
||||
|
||||
if state == ScaleDownRemainingEngineState.PREPARE:
|
||||
self.state = ScaleDownRemainingEngineState.EPLB_RESHUFFLE
|
||||
self.old_dp_store.add("eep_barrier_engine_count", 1)
|
||||
return True
|
||||
if self._create_standby_groups():
|
||||
self.state = ScaleDownRemainingEngineState.COMMIT_SCALE_DOWN
|
||||
self._mark_ready_for_switch()
|
||||
return True
|
||||
return False
|
||||
|
||||
elif state == ScaleDownRemainingEngineState.EPLB_RESHUFFLE:
|
||||
if (
|
||||
int(self.old_dp_store.get("eep_barrier_engine_count"))
|
||||
< self.old_dp_group.size()
|
||||
):
|
||||
elif state == ScaleDownRemainingEngineState.COMMIT_SCALE_DOWN:
|
||||
if not self.commit_requested:
|
||||
return False
|
||||
if not self._staged_barrier(
|
||||
use_new_group=False, barrier_name="eplb_reshuffle"
|
||||
):
|
||||
return False
|
||||
if self.old_dp_group.rank() == 0:
|
||||
self.old_dp_store.delete_key("eep_barrier_engine_count")
|
||||
self._eplb_reshuffle_before_scale_down()
|
||||
self.state = ScaleDownRemainingEngineState.SWITCH_AND_PREPARE
|
||||
# NOTE(yongji): currently, after EPLB reshuffle
|
||||
# that redistributes experts to remaining workers, workers
|
||||
# to be removed will immediately initiate shutdown;
|
||||
# existing workers can no longer execute forward steps using
|
||||
# the old setup. In the future, we may keep
|
||||
# the removing workers alive a bit longer,
|
||||
# e.g., to drain in-batch requests.
|
||||
self._create_standby_groups()
|
||||
self._switch_and_prepare()
|
||||
self._commit_scale_down(removing=False)
|
||||
self._commit_new_dp_group()
|
||||
self._update_parallel_config()
|
||||
self.state = ScaleDownRemainingEngineState.COMPLETE
|
||||
self._send_reconfigure_finished()
|
||||
return True
|
||||
|
||||
else:
|
||||
@@ -402,26 +266,11 @@ class ElasticEPScalingState:
|
||||
|
||||
def _progress_removing_engine(self) -> bool:
|
||||
state = self.state
|
||||
assert self.old_dp_group is not None and self.old_dp_store is not None
|
||||
assert self.old_dp_group is not None
|
||||
|
||||
if state == ScaleDownRemovingEngineState.PREPARE:
|
||||
self.state = ScaleDownRemovingEngineState.EPLB_RESHUFFLE
|
||||
self.old_dp_store.add("eep_barrier_engine_count", 1)
|
||||
return True
|
||||
|
||||
if state == ScaleDownRemovingEngineState.EPLB_RESHUFFLE:
|
||||
if (
|
||||
int(self.old_dp_store.get("eep_barrier_engine_count"))
|
||||
< self.old_dp_group.size()
|
||||
):
|
||||
return False
|
||||
if not self._staged_barrier(
|
||||
use_new_group=False, barrier_name="eplb_reshuffle"
|
||||
):
|
||||
return False
|
||||
assert self.old_dp_group.rank() > 0
|
||||
self._eplb_reshuffle_before_scale_down()
|
||||
self._switch_and_remove()
|
||||
self._commit_scale_down(removing=True)
|
||||
self.state = ScaleDownRemovingEngineState.COMPLETE
|
||||
self.engine_core._eep_send_engine_core_notification(
|
||||
EEPNotificationType.SHUTDOWN_COMPLETE
|
||||
@@ -432,22 +281,22 @@ class ElasticEPScalingState:
|
||||
assert self.state == ScaleDownRemovingEngineState.COMPLETE
|
||||
return True
|
||||
|
||||
def handle_notification(self, notification_type: EEPNotificationType):
|
||||
assert self.worker_type != "new"
|
||||
assert self.old_dp_store is not None
|
||||
if (
|
||||
notification_type == EEPNotificationType.NEW_CORE_ENGINES_INIT_READY
|
||||
and self.state == ScaleUpExistingEngineState.WAIT_NEW_CORE_ENGINES_INIT
|
||||
):
|
||||
self.old_dp_store.add("eep_barrier_engine_count", 1)
|
||||
self.state = ScaleUpExistingEngineState.CREATE_STANDBY_GROUPS
|
||||
elif (
|
||||
notification_type == EEPNotificationType.NEW_CORE_ENGINES_WEIGHTS_INIT_READY
|
||||
and self.state
|
||||
== ScaleUpExistingEngineState.WAIT_NEW_CORE_ENGINES_WEIGHTS_INIT
|
||||
):
|
||||
self.old_dp_store.add("eep_barrier_engine_count", 1)
|
||||
self.state = ScaleUpExistingEngineState.TRANSFER_WEIGHTS
|
||||
def is_ready_for_switch(self) -> bool:
|
||||
return self.worker_type == "existing" and (
|
||||
self.state is ScaleUpExistingEngineState.COMMIT_SCALE_UP
|
||||
or self.state is ScaleDownRemainingEngineState.COMMIT_SCALE_DOWN
|
||||
)
|
||||
|
||||
@property
|
||||
def ready_key(self) -> str:
|
||||
return f"eep_ready/{self.engine_core.dp_rank}"
|
||||
|
||||
def _mark_ready_for_switch(self) -> None:
|
||||
parallel_config = self.new_parallel_config
|
||||
get_cached_tcp_store_client(
|
||||
parallel_config.data_parallel_master_ip,
|
||||
parallel_config._coord_store_port,
|
||||
).set(self.ready_key, b"1")
|
||||
|
||||
def is_complete(self) -> bool:
|
||||
if self.scale_type == "scale_up":
|
||||
@@ -462,50 +311,78 @@ class ElasticEPScalingState:
|
||||
else self.state == ScaleDownRemainingEngineState.COMPLETE
|
||||
)
|
||||
|
||||
def _create_standby_groups(self):
|
||||
def _init_new_dp_group(self) -> tuple[Any, Any]:
|
||||
return self.new_parallel_config.stateless_init_dp_group(return_store=True)
|
||||
|
||||
def _ensure_new_dp_group(self) -> bool:
|
||||
if self.new_dp_group is not None:
|
||||
return True
|
||||
|
||||
if self._prepare_future is None:
|
||||
self._prepare_future = self._prepare_executor.submit(
|
||||
self._init_new_dp_group
|
||||
)
|
||||
if not self._prepare_future.done():
|
||||
return False
|
||||
|
||||
self.new_dp_group, self.new_dp_store = self._prepare_future.result()
|
||||
self._prepare_future = None
|
||||
return True
|
||||
|
||||
def _create_standby_groups(self) -> bool:
|
||||
assert self.old_dp_group is not None
|
||||
self.new_dp_group, self.new_dp_store = (
|
||||
self.new_parallel_config.stateless_init_dp_group(return_store=True)
|
||||
)
|
||||
self.model_executor.collective_rpc(
|
||||
"elastic_ep_execute", args=("create_standby_groups", self.reconfig_request)
|
||||
)
|
||||
if not self._ensure_new_dp_group():
|
||||
return False
|
||||
if not self._execute_async(
|
||||
"create_standby_groups",
|
||||
self.reconfig_request,
|
||||
self.new_parallel_config.use_all2all,
|
||||
):
|
||||
return False
|
||||
if self.old_dp_group.rank() == 0:
|
||||
logger.info("[Elastic EP] Created standby communication groups")
|
||||
return True
|
||||
|
||||
def _transfer_weights(self):
|
||||
def _transfer_weights(self) -> bool:
|
||||
assert self.reconfig_request is not None and self.old_dp_group is not None
|
||||
old_dp_size = self.old_dp_group.size()
|
||||
new_dp_size = self.reconfig_request.new_data_parallel_size
|
||||
|
||||
self.model_executor.collective_rpc(
|
||||
"elastic_ep_execute", args=("transfer_weights", old_dp_size, new_dp_size)
|
||||
)
|
||||
if not self._execute_async("transfer_weights", old_dp_size, new_dp_size):
|
||||
return False
|
||||
if self.old_dp_group.rank() == 0:
|
||||
logger.info("[Elastic EP] Transferred weights to new workers")
|
||||
return True
|
||||
|
||||
def _transfer_expert_mapping(self):
|
||||
assert self.old_dp_group is not None
|
||||
self.model_executor.collective_rpc(
|
||||
"elastic_ep_execute", args=("broadcast_expert_mapping",)
|
||||
)
|
||||
if self.old_dp_group.rank() == 0:
|
||||
logger.info("[Elastic EP] Broadcasted expert mapping to new workers")
|
||||
|
||||
def _sync_kv_cache_memory_size(self):
|
||||
def _sync_kv_cache_memory_size(self) -> bool:
|
||||
assert self.engine_core.available_gpu_memory_for_kv_cache > 0
|
||||
assert self.new_dp_group is not None and self.old_dp_group is not None
|
||||
ParallelConfig.sync_kv_cache_memory_size(
|
||||
self.new_dp_group,
|
||||
self.engine_core.available_gpu_memory_for_kv_cache,
|
||||
)
|
||||
|
||||
if self._new_dp_sync is None:
|
||||
tensor = torch.tensor(
|
||||
[self.engine_core.available_gpu_memory_for_kv_cache],
|
||||
dtype=torch.int64,
|
||||
device="cpu",
|
||||
)
|
||||
work = torch.distributed.all_reduce(
|
||||
tensor,
|
||||
op=torch.distributed.ReduceOp.MIN,
|
||||
group=self.new_dp_group,
|
||||
async_op=True,
|
||||
)
|
||||
self._new_dp_sync = (tensor, work)
|
||||
return False
|
||||
|
||||
_, work = self._new_dp_sync
|
||||
if not work.is_completed():
|
||||
return False
|
||||
work.wait()
|
||||
self._new_dp_sync = None
|
||||
if self.old_dp_group.rank() == 0:
|
||||
logger.info("[Elastic EP] Synced KV cache memory size to new workers")
|
||||
return True
|
||||
|
||||
def _switch_and_prepare(self):
|
||||
self.model_executor.collective_rpc(
|
||||
"elastic_ep_execute", args=("switch_and_prepare",)
|
||||
)
|
||||
def _commit_new_dp_group(self):
|
||||
old_dp_group = self.old_dp_group
|
||||
stateless_destroy_torch_distributed_process_group(old_dp_group)
|
||||
assert self.new_dp_group is not None
|
||||
@@ -529,41 +406,28 @@ class ElasticEPScalingState:
|
||||
self.engine_core.current_wave = int(data[1])
|
||||
self.engine_core.step_counter = int(data[2])
|
||||
if new_dp_group.rank() == 0:
|
||||
logger.info("[Elastic EP] Switched to new setup")
|
||||
|
||||
def _send_reconfigure_finished(self):
|
||||
assert self.new_dp_group is not None
|
||||
if self.new_dp_group.rank() == 0:
|
||||
self.engine_core._eep_send_engine_core_notification(
|
||||
EEPNotificationType.RECONFIGURE_FINISHED
|
||||
)
|
||||
logger.info("[Elastic EP] Switched to new setup")
|
||||
|
||||
def _eplb_reshuffle(self):
|
||||
self.model_executor.collective_rpc(
|
||||
"elastic_ep_execute", args=("perform_eplb_reshuffle",)
|
||||
)
|
||||
# Reshuffle changes per-rank token routing; the locked MoE workspace
|
||||
# may now be too small. Rewarm covers both new and existing engines.
|
||||
self.model_executor.collective_rpc(
|
||||
"elastic_ep_execute", args=("rewarm_workspace",)
|
||||
)
|
||||
assert self.new_dp_group is not None
|
||||
if self.new_dp_group.rank() == 0:
|
||||
logger.info("[Elastic EP] EPLB reshuffle completed")
|
||||
|
||||
def _eplb_reshuffle_before_scale_down(self):
|
||||
def _commit_scale_down(self, removing: bool):
|
||||
assert self.reconfig_request is not None and self.old_dp_group is not None
|
||||
self.model_executor.collective_rpc(
|
||||
self._collective_rpc(
|
||||
"elastic_ep_execute",
|
||||
args=(
|
||||
"perform_scale_down_eplb_reshuffle",
|
||||
"commit_scale_down",
|
||||
self.reconfig_request.new_data_parallel_size,
|
||||
removing,
|
||||
),
|
||||
)
|
||||
if self.old_dp_group.rank() == 0:
|
||||
logger.info("[Elastic EP] EPLB reshuffle completed")
|
||||
|
||||
def _switch_and_remove(self):
|
||||
self.model_executor.collective_rpc(
|
||||
"elastic_ep_execute", args=("switch_and_remove",)
|
||||
)
|
||||
|
||||
def _update_parallel_config(self):
|
||||
assert self.reconfig_request is not None
|
||||
reconfig_request = self.reconfig_request
|
||||
|
||||
@@ -39,6 +39,7 @@ def create_standby_groups(
|
||||
new_world_size_across_dp: int,
|
||||
master_ip: str,
|
||||
coord_store_port: int,
|
||||
use_all2all: bool,
|
||||
enable_eplb: bool = True,
|
||||
backend: str | None = None,
|
||||
) -> None:
|
||||
@@ -86,7 +87,7 @@ def create_standby_groups(
|
||||
)
|
||||
standby_ep_ranks = [x.tolist() for x in standby_ep_ranks]
|
||||
_STANDBY_EP = _init_stateless_group(
|
||||
standby_ep_ranks, "ep", master_ip, backend, coord_store=coord_store
|
||||
standby_ep_ranks, "ep", master_ip, backend, coord_store, use_all2all=use_all2all
|
||||
)
|
||||
|
||||
if enable_eplb:
|
||||
|
||||
@@ -414,6 +414,7 @@ class GroupCoordinator:
|
||||
use_device_communicator: bool, # whether to use device communicator
|
||||
use_message_queue_broadcaster: bool = False,
|
||||
group_name: str | None = None,
|
||||
use_all2all: bool = False,
|
||||
):
|
||||
group_name = group_name or "anonymous"
|
||||
self.unique_name = _get_unique_name(group_name)
|
||||
@@ -508,6 +509,7 @@ class GroupCoordinator:
|
||||
device=self.device,
|
||||
device_group=self.device_group,
|
||||
unique_name=self.unique_name,
|
||||
use_all2all=use_all2all,
|
||||
)
|
||||
|
||||
from vllm.distributed.device_communicators.shm_broadcast import MessageQueue
|
||||
@@ -1321,6 +1323,7 @@ def init_model_parallel_group(
|
||||
use_message_queue_broadcaster: bool = False,
|
||||
group_name: str | None = None,
|
||||
use_device_communicator: bool = True,
|
||||
use_all2all: bool = False,
|
||||
) -> GroupCoordinator:
|
||||
return GroupCoordinator(
|
||||
group_ranks=group_ranks,
|
||||
@@ -1329,6 +1332,7 @@ def init_model_parallel_group(
|
||||
use_device_communicator=use_device_communicator,
|
||||
use_message_queue_broadcaster=use_message_queue_broadcaster,
|
||||
group_name=group_name,
|
||||
use_all2all=use_all2all,
|
||||
)
|
||||
|
||||
|
||||
@@ -1339,6 +1343,7 @@ def _init_stateless_group(
|
||||
backend: str,
|
||||
coord_store: Store,
|
||||
use_device_communicator: bool = True,
|
||||
use_all2all: bool = False,
|
||||
) -> "StatelessGroupCoordinator":
|
||||
"""Create a StatelessGroupCoordinator with the given parameters."""
|
||||
from vllm.distributed.stateless_coordinator import StatelessGroupCoordinator
|
||||
@@ -1354,6 +1359,7 @@ def _init_stateless_group(
|
||||
coord_store=coord_store,
|
||||
global_rank=world.rank,
|
||||
global_world_size=world.world_size,
|
||||
use_all2all=use_all2all,
|
||||
)
|
||||
|
||||
|
||||
@@ -1924,6 +1930,7 @@ def initialize_model_parallel(
|
||||
.unbind(0)
|
||||
)
|
||||
group_ranks = [x.tolist() for x in group_ranks]
|
||||
use_all2all = parallel_config.use_all2all
|
||||
if enable_elastic_ep:
|
||||
_EP = _init_stateless_group(
|
||||
group_ranks,
|
||||
@@ -1931,10 +1938,15 @@ def initialize_model_parallel(
|
||||
parallel_config.data_parallel_master_ip,
|
||||
backend,
|
||||
coord_store=coord_store,
|
||||
use_all2all=use_all2all,
|
||||
)
|
||||
else:
|
||||
_EP = init_model_parallel_group(
|
||||
group_ranks, get_world_group().local_rank, backend, group_name="ep"
|
||||
group_ranks,
|
||||
get_world_group().local_rank,
|
||||
backend,
|
||||
group_name="ep",
|
||||
use_all2all=use_all2all,
|
||||
)
|
||||
|
||||
# Create EPLB group with the same ranks as EP if EPLB is enabled.
|
||||
|
||||
@@ -79,6 +79,7 @@ class StatelessGroupCoordinator(GroupCoordinator):
|
||||
host: str = "127.0.0.1",
|
||||
global_rank: int = 0,
|
||||
global_world_size: int = 1,
|
||||
use_all2all: bool = False,
|
||||
):
|
||||
group_name = group_name or "anonymous"
|
||||
self.unique_name = _get_unique_name(group_name)
|
||||
@@ -191,6 +192,7 @@ class StatelessGroupCoordinator(GroupCoordinator):
|
||||
global_ranks=self.ranks,
|
||||
global_world_size=global_world_size,
|
||||
tcp_store_group=self.tcp_store_group,
|
||||
use_all2all=use_all2all,
|
||||
)
|
||||
|
||||
self.mq_broadcaster = None
|
||||
|
||||
Reference in New Issue
Block a user