[Elastic EP] Async preparation (#47288)

Signed-off-by: Itay Alroy <ialroy@nvidia.com>
This commit is contained in:
Itay Alroy
2026-07-28 05:19:13 -07:00
committed by GitHub
parent d2bfc6fe20
commit 35efdf6b34
21 changed files with 701 additions and 677 deletions
@@ -175,6 +175,7 @@ class DeviceCommunicatorBase:
unique_name: str = "",
global_ranks: list[int] | None = None,
global_world_size: int | None = None,
use_all2all: bool = False,
):
self.device = device or torch.device("cpu")
self.cpu_group = cpu_group
@@ -204,26 +205,15 @@ class DeviceCommunicatorBase:
self.global_world_size = dist.get_world_size()
self.rank_in_group = dist.get_group_rank(self.cpu_group, self.global_rank)
use_ep = False
all2all_backend = None
from vllm.config import get_current_vllm_config_or_none
config = get_current_vllm_config_or_none()
if config is not None:
# initialize the all2all manager for DP or sequence-parallel EP.
parallel_config = config.parallel_config
use_ep = (
parallel_config.data_parallel_size > 1
or parallel_config.use_sequence_parallel_moe
or (
parallel_config.enable_expert_parallel
and parallel_config.prefill_context_parallel_size > 1
)
)
all2all_backend = parallel_config.all2all_backend
all2all_backend = config.parallel_config.all2all_backend
self.is_ep_communicator = unique_name.split(":")[0] == "ep"
self.use_all2all = self.is_ep_communicator and use_ep
self.use_all2all = self.is_ep_communicator and use_all2all
self.all2all_backend = all2all_backend
self.all2all_manager: All2AllManagerBase | None = None
@@ -24,8 +24,11 @@ class CpuCommunicator(DeviceCommunicatorBase):
device: torch.device | None = None,
device_group: ProcessGroup | None = None,
unique_name: str = "",
use_all2all: bool = False,
):
super().__init__(cpu_group, device, device_group, unique_name)
super().__init__(
cpu_group, device, device_group, unique_name, use_all2all=use_all2all
)
self.dist_module = torch.distributed
if (
@@ -36,6 +36,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
global_ranks: list[int] | None = None,
global_world_size: int | None = None,
tcp_store_group: StatelessProcessGroup | None = None,
use_all2all: bool = False,
):
super().__init__(
cpu_group,
@@ -44,6 +45,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
unique_name,
global_ranks,
global_world_size,
use_all2all=use_all2all,
)
if "tp" not in unique_name:
# custom allreduce or torch symm mem can be used only by tp
@@ -20,8 +20,11 @@ class XpuCommunicator(DeviceCommunicatorBase):
device: torch.device | None = None,
device_group: ProcessGroup | None = None,
unique_name: str = "",
use_all2all: bool = False,
):
super().__init__(cpu_group, device, device_group, unique_name)
super().__init__(
cpu_group, device, device_group, unique_name, use_all2all=use_all2all
)
self.ca_comm: None = None
if self.use_all2all:
if self.all2all_backend in ("naive", "allgather_reducescatter"):
+95 -86
View File
@@ -1,9 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copy
import gc
import weakref
from collections.abc import Iterable, Sequence
from concurrent.futures import Future, ThreadPoolExecutor
from dataclasses import replace
from typing import TYPE_CHECKING
@@ -43,6 +43,7 @@ from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig
from vllm.model_executor.layers.fused_moe.eep_reconfigure import (
make_eep_staged_quant_method,
)
from vllm.model_executor.warmup.kernel_warmup import kernel_warmup
from vllm.utils import is_moe_layer
from vllm.v1.engine import ReconfigureDistributedRequest, ReconfigureRankType
from vllm.v1.worker.gpu_ubatch_wrapper import UBatchWrapper
@@ -145,6 +146,10 @@ class ElasticEPScalingExecutor:
self.worker_ref = weakref.ref(worker)
self.reconfig_request = None
self._staged_moe_quant_methods: dict[nn.Module, FusedMoEMethodBase] = {}
self._async_executor = ThreadPoolExecutor(
max_workers=1, thread_name_prefix="ElasticEPAsync"
)
self._async_future: Future[None] | None = None
@property
def worker(self):
@@ -159,59 +164,68 @@ class ElasticEPScalingExecutor:
raise ValueError(f"Unknown execute method: {execute_method}")
return method(*args, **kwargs)
def _set_eplb_suppressed(self, suppressed: bool) -> None:
self.worker.model_runner.eep_eplb_suppressed = suppressed
ep_group = get_standby_ep_group() or get_ep_group()
if ep_group.rank == 0:
logger.info(
"[Elastic EP] EPLB %s elastic scaling transition",
"disabled during" if suppressed else "re-enabled after",
)
def start_async(self, execute_method: str, *args, **kwargs) -> str:
if self._async_future is not None:
raise RuntimeError("Another Elastic EP async method is active")
if args and isinstance(args[0], ReconfigureDistributedRequest):
self.reconfig_request = args[0]
dp_rank = self.worker.vllm_config.parallel_config.data_parallel_rank
done_key = f"eep_async/{execute_method}/{dp_rank}/{self.worker.rank}"
self._async_future = self._async_executor.submit(
self._run_async, execute_method, *args, **kwargs
)
self._async_future.add_done_callback(lambda _: self._mark_async_done(done_key))
return done_key
def _run_async(self, execute_method: str, *args, **kwargs) -> None:
from vllm.platforms import current_platform
self.worker.vllm_config.enable_trace_function_call_for_thread()
assert hasattr(self.worker, "device")
current_platform.set_device(self.worker.device)
with set_current_vllm_config(self.worker.vllm_config):
self.execute(execute_method, *args, **kwargs)
def _mark_async_done(self, done_key: str) -> None:
from vllm.distributed.utils import get_cached_tcp_store_client
assert self.reconfig_request is not None
get_cached_tcp_store_client(
self.reconfig_request.new_data_parallel_master_ip,
self.reconfig_request.coord_store_port,
).set(done_key, b"1")
def clear_async(self) -> None:
future = self._async_future
if future is None:
raise RuntimeError("No Elastic EP async method is active")
if not future.done():
raise RuntimeError("Elastic EP async method is not done")
self._async_future = None
future.result()
def load_model(self) -> None:
(
expanded_physical_to_logical,
num_logical_experts,
old_num_physical_experts,
) = self.receive_expert_mapping()
num_physical_experts = expanded_physical_to_logical.shape[1]
self.worker.parallel_config.eplb_config.num_redundant_experts = (
num_physical_experts - num_logical_experts
)
self.worker.load_model(load_dummy_weights=True)
self.worker.model_runner.setup_eplb_from_mapping(
expanded_physical_to_logical, old_num_physical_experts
)
self._set_eplb_suppressed(True)
def create_standby_groups(
self, reconfig_request: ReconfigureDistributedRequest
self, reconfig_request: ReconfigureDistributedRequest, use_all2all: bool
) -> None:
self.reconfig_request = reconfig_request
new_dp_size = reconfig_request.new_data_parallel_size
old_dp_size = get_dp_group().world_size
world_size = self.worker.vllm_config.parallel_config.world_size
parallel_config = self.worker.vllm_config.parallel_config
world_size = parallel_config.world_size
new_world_size_across_dp = world_size * new_dp_size
updated_config = copy.copy(self.worker.vllm_config)
updated_config.parallel_config = copy.deepcopy(
self.worker.vllm_config.parallel_config
create_standby_groups(
new_dp_size=new_dp_size,
new_world_size_across_dp=new_world_size_across_dp,
master_ip=reconfig_request.new_data_parallel_master_ip,
coord_store_port=reconfig_request.coord_store_port,
use_all2all=use_all2all,
enable_eplb=parallel_config.enable_eplb,
)
updated_config.parallel_config.data_parallel_size = new_dp_size
with set_current_vllm_config(updated_config):
create_standby_groups(
new_dp_size=new_dp_size,
new_world_size_across_dp=new_world_size_across_dp,
master_ip=reconfig_request.new_data_parallel_master_ip,
coord_store_port=reconfig_request.coord_store_port,
enable_eplb=updated_config.parallel_config.enable_eplb,
)
if new_dp_size > old_dp_size:
self._set_eplb_suppressed(True)
eplb_state = self.worker.model_runner.eplb_state
if eplb_state is not None:
eplb_state.drain_async()
elif new_dp_size < old_dp_size:
self._stage_standby_moe_quant_methods()
if new_dp_size < old_dp_size:
self.stage_standby_moe_quant_methods()
def transfer_weights(self, old_dp_size: int, new_dp_size: int) -> None:
standby_dp_group = get_standby_dp_group()
@@ -265,6 +279,7 @@ class ElasticEPScalingExecutor:
model_config = self.worker.model_runner.model_config
eplb_state = self.worker.model_runner.eplb_state
assert eplb_state is not None
eplb_state.drain_async()
eplb_model_state = eplb_state.model_states[model_config.compute_hash()]
physical_to_logical = eplb_model_state.physical_to_logical_map
num_physical_experts = physical_to_logical.shape[1]
@@ -278,10 +293,6 @@ class ElasticEPScalingExecutor:
src_rank=0,
device=self.worker.device,
)
# New workers enter load_model after receiving the expert mapping.
# Stage replacement MoE kernels before returning to the state machine
# so existing ranks can participate in collective EP comm creation.
self._stage_standby_moe_quant_methods()
def _make_eep_moe_config(self, module, dp_group, ep_group):
parallel_config = self.worker.vllm_config.parallel_config
@@ -300,7 +311,7 @@ class ElasticEPScalingExecutor:
moe_parallel_config=moe_parallel_config,
)
def _stage_standby_moe_quant_methods(self) -> None:
def stage_standby_moe_quant_methods(self) -> None:
standby_dp_group = get_standby_dp_group()
standby_ep_group = get_standby_ep_group()
model = self.worker.model_runner.get_model()
@@ -500,26 +511,6 @@ class ElasticEPScalingExecutor:
compilation_counter.stock_torch_compile_count += 1
self.worker.model_runner.model.compile(fullgraph=True, backend=backend)
multi_block_table = self.worker.model_runner.input_batch.block_table
saved_block_tables: list[tuple[torch.Tensor, torch.Tensor]] = []
for bt in multi_block_table.block_tables:
saved_block_tables.append(
(bt.block_table.gpu.clone(), bt.block_table.cpu.clone())
)
multi_block_table.clear()
unlock_workspace()
self.worker.compile_or_warm_up_model()
lock_workspace()
for bt, (saved_gpu, saved_cpu) in zip(
multi_block_table.block_tables, saved_block_tables
):
bt.block_table.gpu.copy_(saved_gpu)
bt.block_table.cpu.copy_(saved_cpu)
if new_dp_size < old_dp_size:
self._set_eplb_suppressed(False)
def _perform_eplb_reshuffle(
self, rank_mapping: dict[int, int] | None = None
) -> None:
@@ -553,12 +544,25 @@ class ElasticEPScalingExecutor:
if get_ep_group().rank == 0:
logger.info("[Elastic EP] Expert resharding completed")
def perform_eplb_reshuffle(self) -> None:
def commit_scale_up(self, is_existing_worker: bool) -> None:
if is_existing_worker:
self.broadcast_expert_mapping()
self.switch_and_prepare()
else:
mapping, _, num_valid_experts = self.receive_expert_mapping()
self.worker.model_runner.setup_eplb_from_mapping(mapping, num_valid_experts)
self._perform_eplb_reshuffle()
self._set_eplb_suppressed(False)
self.warm_and_capture()
def commit_scale_down(self, new_dp_size: int, removing: bool) -> None:
self.perform_scale_down_eplb_reshuffle(new_dp_size)
if removing:
self.switch_and_remove()
else:
self.switch_and_prepare()
self.warm_and_capture()
def perform_scale_down_eplb_reshuffle(self, new_dp_size: int) -> None:
self._set_eplb_suppressed(True)
eplb_state = self.worker.model_runner.eplb_state
if eplb_state is not None:
eplb_state.drain_async()
@@ -599,12 +603,17 @@ class ElasticEPScalingExecutor:
)
model = self.worker.model_runner.get_model()
expert_weights = [
module.get_expert_weights()
for module in model.modules()
if is_moe_layer(module)
]
batch_transfer_weights(
model=model,
is_sender=False,
peer_rank=sender_rank,
dp_group=dp_group,
expert_weights=model.expert_weights,
expert_weights=expert_weights,
)
torch.accelerator.synchronize()
@@ -643,14 +652,17 @@ class ElasticEPScalingExecutor:
with set_current_vllm_config(self.worker.vllm_config):
prepare_communication_buffer_for_model(self.worker.model_runner.get_model())
def rewarm_workspace(self) -> None:
def warmup_local_kernels(self) -> None:
with set_current_vllm_config(self.worker.vllm_config):
kernel_warmup(self.worker, process_local_only=True)
def warm_and_capture(self) -> None:
# Must run on every DP sibling in lockstep: _dummy_run calls
# coordinate_batch_across_dp whenever data_parallel_size > 1
# (gpu_model_runner.py:3663), which deadlocks if any rank skips it.
# Save and clear block tables so profile_run/compile_or_warm_up_model
# don't write dummy slot mappings into real KV-cache blocks (mirrors
# switch_and_prepare's pattern).
# Save and clear block tables so the dummy MoE forward doesn't
# write dummy slot mappings into real KV-cache blocks.
multi_block_table = self.worker.model_runner.input_batch.block_table
saved_block_tables: list[tuple[torch.Tensor, torch.Tensor]] = []
for bt in multi_block_table.block_tables:
@@ -660,19 +672,16 @@ class ElasticEPScalingExecutor:
multi_block_table.clear()
# _ensure_workspace_size allocates a fresh tensor on grow, leaving
# captured CUDA graphs with stale data pointers; drop graphs before
# re-warm so captures realign with the resized buffer.
# any captured CUDA graph with a stale data pointer; drop graphs
# before re-warm so captures realign with the resized buffer.
self._release_cuda_graphs()
unlock_workspace()
# Grow the MoE workspace at max_num_tokens.
# compile_or_warm_up_model alone only exercises cudagraph-capture
# sizes (≤64 tokens for this test) and leaves the workspace at
# ~10-14 MB; the post-all-to-all per-rank token count under real
# post-reshuffle routing needs hundreds of MB. Use _dummy_run
# directly (rather than profile_run) with skip_eplb=True so dummy
# routing doesn't pollute the just-rebalanced EPLB stats — same
# convention compile_or_warm_up_model itself uses.
# Grow the MoE workspace at max_num_tokens. compile_or_warm_up_model
# alone only exercises cudagraph-capture sizes and can leave the
# workspace too small for post-reshuffle routing. Use _dummy_run
# directly with skip_eplb=True so dummy routing doesn't pollute the
# just-rebalanced EPLB stats.
runner = self.worker.model_runner
runner._dummy_run(runner.max_num_tokens, is_profile=True, skip_eplb=True)
self.worker.compile_or_warm_up_model()
+162 -298
View File
@@ -1,18 +1,17 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import enum
import time
import weakref
from datetime import timedelta
from typing import TYPE_CHECKING, Literal, TypeAlias
from concurrent.futures import Future, ThreadPoolExecutor
from typing import TYPE_CHECKING, Any, Literal, TypeAlias
import torch.distributed
from vllm.config import ParallelConfig
from vllm.distributed import (
sched_yield,
stateless_destroy_torch_distributed_process_group,
)
from vllm.distributed.utils import get_cached_tcp_store_client
from vllm.logger import init_logger
from vllm.v1.engine import (
EEPNotificationType,
@@ -31,35 +30,29 @@ WorkerType = Literal["existing", "new", "removing"]
class ScaleUpExistingEngineState(enum.IntEnum):
WAIT_NEW_CORE_ENGINES_INIT = 0
CREATE_STANDBY_GROUPS = 1
TRANSFER_EXPERT_MAPPING = 2
WAIT_NEW_CORE_ENGINES_WEIGHTS_INIT = 3
TRANSFER_WEIGHTS = 4
SYNC_KV_CACHE_MEMORY_SIZE = 5
SWITCH_AND_PREPARE = 6
EPLB_RESHUFFLE = 7
COMPLETE = 8
CREATE_STANDBY_GROUPS = 0
STAGE_QUANT_METHODS = 1
TRANSFER_WEIGHTS = 2
SYNC_KV_CACHE_MEMORY_SIZE = 3
COMMIT_SCALE_UP = 4 # Blocks forward passes.
COMPLETE = 5
class ScaleUpNewEngineState(enum.IntEnum):
PRE_KV_INIT = 0
PREPARE = 1
EPLB_RESHUFFLE = 2
COMPLETE = 3
COMPLETE = 2
class ScaleDownRemainingEngineState(enum.IntEnum):
PREPARE = 0
EPLB_RESHUFFLE = 1
SWITCH_AND_PREPARE = 2
COMPLETE = 3
COMMIT_SCALE_DOWN = 1 # Blocks forward passes.
COMPLETE = 2
class ScaleDownRemovingEngineState(enum.IntEnum):
PREPARE = 0
EPLB_RESHUFFLE = 1
COMPLETE = 2
COMPLETE = 1
EngineState: TypeAlias = (
@@ -70,15 +63,6 @@ EngineState: TypeAlias = (
)
class _BarrierTimeoutError(RuntimeError):
"""
Exception raised for timeout
in the first stage of our two-staged
TCPStore based barrier to synchronize the
execution of all engines in the DP group.
"""
class ElasticEPScalingState:
def __init__(
self,
@@ -94,20 +78,24 @@ class ElasticEPScalingState:
self.engine_core_ref = weakref.ref(engine_core)
self.vllm_config = vllm_config
self.old_dp_group = self.engine_core.dp_group if worker_type != "new" else None
self.old_dp_store = self.engine_core.dp_store if worker_type != "new" else None
self.new_parallel_config: ParallelConfig = new_parallel_config
self.new_dp_group = self.engine_core.dp_group if worker_type == "new" else None
self.new_dp_store = self.engine_core.dp_store if worker_type == "new" else None
self.worker_type = worker_type
self.scale_type = scale_type
self.reconfig_request = reconfig_request
self.commit_requested = False
self._prepare_executor = ThreadPoolExecutor(
max_workers=1, thread_name_prefix="ElasticEPPrepare"
)
self._prepare_future: Future[Any] | None = None
self._new_dp_sync: tuple[object, Any] | None = None
self.state: EngineState
if scale_type == "scale_up":
self.state = (
ScaleUpNewEngineState.PRE_KV_INIT
if worker_type == "new"
else ScaleUpExistingEngineState.WAIT_NEW_CORE_ENGINES_INIT
else ScaleUpExistingEngineState.CREATE_STANDBY_GROUPS
)
else:
self.state = (
@@ -130,6 +118,31 @@ class ElasticEPScalingState:
raise RuntimeError("Engine core has been garbage collected")
return engine_core
def _collective_rpc(self, *args, **kwargs):
return self.model_executor.collective_rpc(*args, **kwargs)
def _execute_async(self, execute_method: str, *args) -> bool:
if self._prepare_future is None:
done_keys = self._collective_rpc(
"elastic_ep_execute",
args=("start_async", execute_method, *args),
)
assert self.reconfig_request is not None
coord_store = get_cached_tcp_store_client(
self.reconfig_request.new_data_parallel_master_ip,
self.reconfig_request.coord_store_port,
)
self._prepare_future = self._prepare_executor.submit(
coord_store.wait, done_keys
)
if not self._prepare_future.done():
return False
self._prepare_future.result()
self._collective_rpc("elastic_ep_execute", args=("clear_async",))
self._prepare_future = None
return True
def progress(self) -> bool:
if self.scale_type == "scale_up":
return (
@@ -149,157 +162,43 @@ class ElasticEPScalingState:
assert self.progress()
assert self.state == ScaleUpNewEngineState.PREPARE
def _execute_tcp_store_barrier(
self, dp_store, group_rank, group_size, barrier_id, timeout=None
):
arrival_key = f"arrival_{barrier_id}_{group_rank}"
dp_store.set(arrival_key, b"1")
start_time = time.time()
processes_arrived: set[int] = set()
while len(processes_arrived) < group_size:
if (
timeout is not None
and time.time() - start_time > timeout.total_seconds()
):
raise _BarrierTimeoutError(
f"Barrier timed out after {timeout.total_seconds()} seconds"
)
for i in range(group_size):
if i in processes_arrived:
continue
key = f"arrival_{barrier_id}_{i}"
present = dp_store.check([key])
if present:
processes_arrived.add(i)
if len(processes_arrived) < group_size:
sched_yield()
def _staged_barrier(self, use_new_group: bool, barrier_name: str) -> bool:
"""
Execute a two-staged barrier to synchronize all engines in the DP group.
Some DP EngineCores may receive the reconfiguration notifications
later than others, and already proceed to engine step (model forward)
in the busy loop.
In this case, EngineCores that already proceed to reconfiguration
should skip reconfiguration and execute model forward for one more
step, so in the next step, all EngineCores will be synchronized.
We use a two-staged barrier to achieve this. The first time each
EngineCore executes the barrier, if a timeout is reached before the
barrier completes, that means some EngineCores have already entered
engine step. The EngineCores that timed out will then proceed to
engine step, and will synchronize with the other EngineCores in the
next step with a barrier without timeout.
"""
dp_group = self.new_dp_group if use_new_group else self.old_dp_group
dp_store = self.new_dp_store if use_new_group else self.old_dp_store
assert dp_group is not None and dp_store is not None
group_rank = dp_group.rank()
group_size = dp_group.size()
barrier_id = f"eep_barrier_{barrier_name}"
sync_key = f"{barrier_id}_sync"
# TODO(yongji): figure out appropriate timeout for the barrier
timeout = None if dp_store.check([sync_key]) else timedelta(seconds=5)
try:
self._execute_tcp_store_barrier(
dp_store, group_rank, group_size, barrier_id, timeout=timeout
)
torch.distributed.barrier(dp_group)
if group_rank == 0:
dp_store.delete_key(sync_key)
for i in range(group_size):
dp_store.delete_key(f"arrival_{barrier_id}_{i}")
return True
except _BarrierTimeoutError as e:
if timeout is None:
raise RuntimeError("Unexpected timeout encountered") from e
dp_store.compare_set(sync_key, "", b"1")
return False
def _progress_existing_engine(self) -> bool:
state = self.state
assert self.old_dp_group is not None and self.old_dp_store is not None
assert self.old_dp_group is not None
if state == ScaleUpExistingEngineState.WAIT_NEW_CORE_ENGINES_INIT:
return False
elif state == ScaleUpExistingEngineState.CREATE_STANDBY_GROUPS:
# NOTE(yongji): wait for all existing workers to receive the request
if (
int(self.old_dp_store.get("eep_barrier_engine_count"))
< self.old_dp_group.size()
):
if state == ScaleUpExistingEngineState.CREATE_STANDBY_GROUPS:
if not self._create_standby_groups():
return False
if not self._staged_barrier(
use_new_group=False, barrier_name="create_standby_groups"
):
return False
if self.old_dp_group.rank() == 0:
self.old_dp_store.delete_key("eep_barrier_engine_count")
self._create_standby_groups()
self.state = ScaleUpExistingEngineState.TRANSFER_EXPERT_MAPPING
self.state = ScaleUpExistingEngineState.STAGE_QUANT_METHODS
return True
elif state == ScaleUpExistingEngineState.TRANSFER_EXPERT_MAPPING:
self._transfer_expert_mapping()
self.state = ScaleUpExistingEngineState.WAIT_NEW_CORE_ENGINES_WEIGHTS_INIT
elif state == ScaleUpExistingEngineState.STAGE_QUANT_METHODS:
if not self._execute_async("stage_standby_moe_quant_methods"):
return False
self.state = ScaleUpExistingEngineState.TRANSFER_WEIGHTS
return True
elif state == ScaleUpExistingEngineState.WAIT_NEW_CORE_ENGINES_WEIGHTS_INIT:
return False
elif state == ScaleUpExistingEngineState.TRANSFER_WEIGHTS:
if (
int(self.old_dp_store.get("eep_barrier_engine_count"))
< self.old_dp_group.size()
):
if not self._transfer_weights():
return False
if not self._staged_barrier(
use_new_group=False, barrier_name="transfer_weights"
):
return False
if self.old_dp_group.rank() == 0:
self.old_dp_store.delete_key("eep_barrier_engine_count")
self._transfer_weights()
self.state = ScaleUpExistingEngineState.SYNC_KV_CACHE_MEMORY_SIZE
return True
elif state == ScaleUpExistingEngineState.SYNC_KV_CACHE_MEMORY_SIZE:
self._sync_kv_cache_memory_size()
self.state = ScaleUpExistingEngineState.SWITCH_AND_PREPARE
if not self._sync_kv_cache_memory_size():
return False
self.state = ScaleUpExistingEngineState.COMMIT_SCALE_UP
self._mark_ready_for_switch()
return True
elif state == ScaleUpExistingEngineState.SWITCH_AND_PREPARE:
self._switch_and_prepare()
self.state = ScaleUpExistingEngineState.EPLB_RESHUFFLE
assert self.new_dp_store is not None
self.new_dp_store.add("eep_barrier_engine_count", 1)
return True
elif state == ScaleUpExistingEngineState.EPLB_RESHUFFLE:
assert self.new_dp_group is not None and self.new_dp_store is not None
if (
int(self.new_dp_store.get("eep_barrier_engine_count"))
< self.new_dp_group.size()
):
elif state == ScaleUpExistingEngineState.COMMIT_SCALE_UP:
if not self.commit_requested:
return False
if not self._staged_barrier(
use_new_group=True, barrier_name="eplb_reshuffle"
):
return False
if self.new_dp_group.rank() == 0:
self.new_dp_store.delete_key("eep_barrier_engine_count")
self._eplb_reshuffle()
self._commit_new_dp_group()
self._collective_rpc("elastic_ep_execute", args=("commit_scale_up", True))
self.state = ScaleUpExistingEngineState.COMPLETE
self._update_parallel_config()
self._send_reconfigure_finished()
return True
else:
@@ -311,22 +210,17 @@ class ElasticEPScalingState:
assert self.new_dp_group is not None and self.new_dp_store is not None
if state == ScaleUpNewEngineState.PRE_KV_INIT:
self.engine_core._eep_send_engine_core_notification(
EEPNotificationType.NEW_CORE_ENGINES_WEIGHTS_INIT_READY
)
self.model_executor.collective_rpc(
"elastic_ep_execute", args=("receive_weights",)
)
self._collective_rpc("elastic_ep_execute", args=("receive_weights",))
self.engine_core.available_gpu_memory_for_kv_cache = (
ParallelConfig.sync_kv_cache_memory_size(self.new_dp_group, -1)
)
self.model_executor.collective_rpc(
"elastic_ep_execute", args=("prepare_new_worker",)
)
self._collective_rpc("elastic_ep_execute", args=("prepare_new_worker",))
self.state = ScaleUpNewEngineState.PREPARE
return True
elif state == ScaleUpNewEngineState.PREPARE:
self._collective_rpc("elastic_ep_execute", args=("warmup_local_kernels",))
self._mark_ready_for_switch()
tensor = torch.tensor([0, 0, 0], dtype=torch.int32, device="cpu")
torch.distributed.all_reduce(
tensor,
@@ -337,22 +231,7 @@ class ElasticEPScalingState:
self.engine_core.engines_running = bool(data[0])
self.engine_core.current_wave = int(data[1])
self.engine_core.step_counter = int(data[2])
self.state = ScaleUpNewEngineState.EPLB_RESHUFFLE
self.new_dp_store.add("eep_barrier_engine_count", 1)
return True
elif state == ScaleUpNewEngineState.EPLB_RESHUFFLE:
if (
int(self.new_dp_store.get("eep_barrier_engine_count"))
< self.new_dp_group.size()
):
return False
if not self._staged_barrier(
use_new_group=True, barrier_name="eplb_reshuffle"
):
return False
assert self.new_dp_group.rank() > 0
self._eplb_reshuffle()
self._collective_rpc("elastic_ep_execute", args=("commit_scale_up", False))
self.state = ScaleUpNewEngineState.COMPLETE
return True
@@ -362,38 +241,23 @@ class ElasticEPScalingState:
def _progress_remaining_engine(self) -> bool:
state = self.state
assert self.old_dp_group is not None and self.old_dp_store is not None
assert self.old_dp_group is not None
if state == ScaleDownRemainingEngineState.PREPARE:
self.state = ScaleDownRemainingEngineState.EPLB_RESHUFFLE
self.old_dp_store.add("eep_barrier_engine_count", 1)
return True
if self._create_standby_groups():
self.state = ScaleDownRemainingEngineState.COMMIT_SCALE_DOWN
self._mark_ready_for_switch()
return True
return False
elif state == ScaleDownRemainingEngineState.EPLB_RESHUFFLE:
if (
int(self.old_dp_store.get("eep_barrier_engine_count"))
< self.old_dp_group.size()
):
elif state == ScaleDownRemainingEngineState.COMMIT_SCALE_DOWN:
if not self.commit_requested:
return False
if not self._staged_barrier(
use_new_group=False, barrier_name="eplb_reshuffle"
):
return False
if self.old_dp_group.rank() == 0:
self.old_dp_store.delete_key("eep_barrier_engine_count")
self._eplb_reshuffle_before_scale_down()
self.state = ScaleDownRemainingEngineState.SWITCH_AND_PREPARE
# NOTE(yongji): currently, after EPLB reshuffle
# that redistributes experts to remaining workers, workers
# to be removed will immediately initiate shutdown;
# existing workers can no longer execute forward steps using
# the old setup. In the future, we may keep
# the removing workers alive a bit longer,
# e.g., to drain in-batch requests.
self._create_standby_groups()
self._switch_and_prepare()
self._commit_scale_down(removing=False)
self._commit_new_dp_group()
self._update_parallel_config()
self.state = ScaleDownRemainingEngineState.COMPLETE
self._send_reconfigure_finished()
return True
else:
@@ -402,26 +266,11 @@ class ElasticEPScalingState:
def _progress_removing_engine(self) -> bool:
state = self.state
assert self.old_dp_group is not None and self.old_dp_store is not None
assert self.old_dp_group is not None
if state == ScaleDownRemovingEngineState.PREPARE:
self.state = ScaleDownRemovingEngineState.EPLB_RESHUFFLE
self.old_dp_store.add("eep_barrier_engine_count", 1)
return True
if state == ScaleDownRemovingEngineState.EPLB_RESHUFFLE:
if (
int(self.old_dp_store.get("eep_barrier_engine_count"))
< self.old_dp_group.size()
):
return False
if not self._staged_barrier(
use_new_group=False, barrier_name="eplb_reshuffle"
):
return False
assert self.old_dp_group.rank() > 0
self._eplb_reshuffle_before_scale_down()
self._switch_and_remove()
self._commit_scale_down(removing=True)
self.state = ScaleDownRemovingEngineState.COMPLETE
self.engine_core._eep_send_engine_core_notification(
EEPNotificationType.SHUTDOWN_COMPLETE
@@ -432,22 +281,22 @@ class ElasticEPScalingState:
assert self.state == ScaleDownRemovingEngineState.COMPLETE
return True
def handle_notification(self, notification_type: EEPNotificationType):
assert self.worker_type != "new"
assert self.old_dp_store is not None
if (
notification_type == EEPNotificationType.NEW_CORE_ENGINES_INIT_READY
and self.state == ScaleUpExistingEngineState.WAIT_NEW_CORE_ENGINES_INIT
):
self.old_dp_store.add("eep_barrier_engine_count", 1)
self.state = ScaleUpExistingEngineState.CREATE_STANDBY_GROUPS
elif (
notification_type == EEPNotificationType.NEW_CORE_ENGINES_WEIGHTS_INIT_READY
and self.state
== ScaleUpExistingEngineState.WAIT_NEW_CORE_ENGINES_WEIGHTS_INIT
):
self.old_dp_store.add("eep_barrier_engine_count", 1)
self.state = ScaleUpExistingEngineState.TRANSFER_WEIGHTS
def is_ready_for_switch(self) -> bool:
return self.worker_type == "existing" and (
self.state is ScaleUpExistingEngineState.COMMIT_SCALE_UP
or self.state is ScaleDownRemainingEngineState.COMMIT_SCALE_DOWN
)
@property
def ready_key(self) -> str:
return f"eep_ready/{self.engine_core.dp_rank}"
def _mark_ready_for_switch(self) -> None:
parallel_config = self.new_parallel_config
get_cached_tcp_store_client(
parallel_config.data_parallel_master_ip,
parallel_config._coord_store_port,
).set(self.ready_key, b"1")
def is_complete(self) -> bool:
if self.scale_type == "scale_up":
@@ -462,50 +311,78 @@ class ElasticEPScalingState:
else self.state == ScaleDownRemainingEngineState.COMPLETE
)
def _create_standby_groups(self):
def _init_new_dp_group(self) -> tuple[Any, Any]:
return self.new_parallel_config.stateless_init_dp_group(return_store=True)
def _ensure_new_dp_group(self) -> bool:
if self.new_dp_group is not None:
return True
if self._prepare_future is None:
self._prepare_future = self._prepare_executor.submit(
self._init_new_dp_group
)
if not self._prepare_future.done():
return False
self.new_dp_group, self.new_dp_store = self._prepare_future.result()
self._prepare_future = None
return True
def _create_standby_groups(self) -> bool:
assert self.old_dp_group is not None
self.new_dp_group, self.new_dp_store = (
self.new_parallel_config.stateless_init_dp_group(return_store=True)
)
self.model_executor.collective_rpc(
"elastic_ep_execute", args=("create_standby_groups", self.reconfig_request)
)
if not self._ensure_new_dp_group():
return False
if not self._execute_async(
"create_standby_groups",
self.reconfig_request,
self.new_parallel_config.use_all2all,
):
return False
if self.old_dp_group.rank() == 0:
logger.info("[Elastic EP] Created standby communication groups")
return True
def _transfer_weights(self):
def _transfer_weights(self) -> bool:
assert self.reconfig_request is not None and self.old_dp_group is not None
old_dp_size = self.old_dp_group.size()
new_dp_size = self.reconfig_request.new_data_parallel_size
self.model_executor.collective_rpc(
"elastic_ep_execute", args=("transfer_weights", old_dp_size, new_dp_size)
)
if not self._execute_async("transfer_weights", old_dp_size, new_dp_size):
return False
if self.old_dp_group.rank() == 0:
logger.info("[Elastic EP] Transferred weights to new workers")
return True
def _transfer_expert_mapping(self):
assert self.old_dp_group is not None
self.model_executor.collective_rpc(
"elastic_ep_execute", args=("broadcast_expert_mapping",)
)
if self.old_dp_group.rank() == 0:
logger.info("[Elastic EP] Broadcasted expert mapping to new workers")
def _sync_kv_cache_memory_size(self):
def _sync_kv_cache_memory_size(self) -> bool:
assert self.engine_core.available_gpu_memory_for_kv_cache > 0
assert self.new_dp_group is not None and self.old_dp_group is not None
ParallelConfig.sync_kv_cache_memory_size(
self.new_dp_group,
self.engine_core.available_gpu_memory_for_kv_cache,
)
if self._new_dp_sync is None:
tensor = torch.tensor(
[self.engine_core.available_gpu_memory_for_kv_cache],
dtype=torch.int64,
device="cpu",
)
work = torch.distributed.all_reduce(
tensor,
op=torch.distributed.ReduceOp.MIN,
group=self.new_dp_group,
async_op=True,
)
self._new_dp_sync = (tensor, work)
return False
_, work = self._new_dp_sync
if not work.is_completed():
return False
work.wait()
self._new_dp_sync = None
if self.old_dp_group.rank() == 0:
logger.info("[Elastic EP] Synced KV cache memory size to new workers")
return True
def _switch_and_prepare(self):
self.model_executor.collective_rpc(
"elastic_ep_execute", args=("switch_and_prepare",)
)
def _commit_new_dp_group(self):
old_dp_group = self.old_dp_group
stateless_destroy_torch_distributed_process_group(old_dp_group)
assert self.new_dp_group is not None
@@ -529,41 +406,28 @@ class ElasticEPScalingState:
self.engine_core.current_wave = int(data[1])
self.engine_core.step_counter = int(data[2])
if new_dp_group.rank() == 0:
logger.info("[Elastic EP] Switched to new setup")
def _send_reconfigure_finished(self):
assert self.new_dp_group is not None
if self.new_dp_group.rank() == 0:
self.engine_core._eep_send_engine_core_notification(
EEPNotificationType.RECONFIGURE_FINISHED
)
logger.info("[Elastic EP] Switched to new setup")
def _eplb_reshuffle(self):
self.model_executor.collective_rpc(
"elastic_ep_execute", args=("perform_eplb_reshuffle",)
)
# Reshuffle changes per-rank token routing; the locked MoE workspace
# may now be too small. Rewarm covers both new and existing engines.
self.model_executor.collective_rpc(
"elastic_ep_execute", args=("rewarm_workspace",)
)
assert self.new_dp_group is not None
if self.new_dp_group.rank() == 0:
logger.info("[Elastic EP] EPLB reshuffle completed")
def _eplb_reshuffle_before_scale_down(self):
def _commit_scale_down(self, removing: bool):
assert self.reconfig_request is not None and self.old_dp_group is not None
self.model_executor.collective_rpc(
self._collective_rpc(
"elastic_ep_execute",
args=(
"perform_scale_down_eplb_reshuffle",
"commit_scale_down",
self.reconfig_request.new_data_parallel_size,
removing,
),
)
if self.old_dp_group.rank() == 0:
logger.info("[Elastic EP] EPLB reshuffle completed")
def _switch_and_remove(self):
self.model_executor.collective_rpc(
"elastic_ep_execute", args=("switch_and_remove",)
)
def _update_parallel_config(self):
assert self.reconfig_request is not None
reconfig_request = self.reconfig_request
+2 -1
View File
@@ -39,6 +39,7 @@ def create_standby_groups(
new_world_size_across_dp: int,
master_ip: str,
coord_store_port: int,
use_all2all: bool,
enable_eplb: bool = True,
backend: str | None = None,
) -> None:
@@ -86,7 +87,7 @@ def create_standby_groups(
)
standby_ep_ranks = [x.tolist() for x in standby_ep_ranks]
_STANDBY_EP = _init_stateless_group(
standby_ep_ranks, "ep", master_ip, backend, coord_store=coord_store
standby_ep_ranks, "ep", master_ip, backend, coord_store, use_all2all=use_all2all
)
if enable_eplb:
+13 -1
View File
@@ -414,6 +414,7 @@ class GroupCoordinator:
use_device_communicator: bool, # whether to use device communicator
use_message_queue_broadcaster: bool = False,
group_name: str | None = None,
use_all2all: bool = False,
):
group_name = group_name or "anonymous"
self.unique_name = _get_unique_name(group_name)
@@ -508,6 +509,7 @@ class GroupCoordinator:
device=self.device,
device_group=self.device_group,
unique_name=self.unique_name,
use_all2all=use_all2all,
)
from vllm.distributed.device_communicators.shm_broadcast import MessageQueue
@@ -1321,6 +1323,7 @@ def init_model_parallel_group(
use_message_queue_broadcaster: bool = False,
group_name: str | None = None,
use_device_communicator: bool = True,
use_all2all: bool = False,
) -> GroupCoordinator:
return GroupCoordinator(
group_ranks=group_ranks,
@@ -1329,6 +1332,7 @@ def init_model_parallel_group(
use_device_communicator=use_device_communicator,
use_message_queue_broadcaster=use_message_queue_broadcaster,
group_name=group_name,
use_all2all=use_all2all,
)
@@ -1339,6 +1343,7 @@ def _init_stateless_group(
backend: str,
coord_store: Store,
use_device_communicator: bool = True,
use_all2all: bool = False,
) -> "StatelessGroupCoordinator":
"""Create a StatelessGroupCoordinator with the given parameters."""
from vllm.distributed.stateless_coordinator import StatelessGroupCoordinator
@@ -1354,6 +1359,7 @@ def _init_stateless_group(
coord_store=coord_store,
global_rank=world.rank,
global_world_size=world.world_size,
use_all2all=use_all2all,
)
@@ -1924,6 +1930,7 @@ def initialize_model_parallel(
.unbind(0)
)
group_ranks = [x.tolist() for x in group_ranks]
use_all2all = parallel_config.use_all2all
if enable_elastic_ep:
_EP = _init_stateless_group(
group_ranks,
@@ -1931,10 +1938,15 @@ def initialize_model_parallel(
parallel_config.data_parallel_master_ip,
backend,
coord_store=coord_store,
use_all2all=use_all2all,
)
else:
_EP = init_model_parallel_group(
group_ranks, get_world_group().local_rank, backend, group_name="ep"
group_ranks,
get_world_group().local_rank,
backend,
group_name="ep",
use_all2all=use_all2all,
)
# Create EPLB group with the same ranks as EP if EPLB is enabled.
@@ -79,6 +79,7 @@ class StatelessGroupCoordinator(GroupCoordinator):
host: str = "127.0.0.1",
global_rank: int = 0,
global_world_size: int = 1,
use_all2all: bool = False,
):
group_name = group_name or "anonymous"
self.unique_name = _get_unique_name(group_name)
@@ -191,6 +192,7 @@ class StatelessGroupCoordinator(GroupCoordinator):
global_ranks=self.ranks,
global_world_size=global_world_size,
tcp_store_group=self.tcp_store_group,
use_all2all=use_all2all,
)
self.mq_broadcaster = None