[Elastic EP] Reuse CUDA graphs across reconfiguration (#54985)
This commit is contained in:
@@ -1,6 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import threading
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
@@ -377,6 +379,7 @@ class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
|
||||
@dataclass
|
||||
class _NixlEPBufferState:
|
||||
buffer: Any
|
||||
max_num_ep_ranks: int
|
||||
connected_ep_size: int
|
||||
active_ep_size: int
|
||||
|
||||
@@ -402,7 +405,10 @@ class NixlEPAll2AllManager(All2AllManagerBase):
|
||||
super().__init__(cpu_group, tcp_store_group)
|
||||
self.support_fault_tolerance = True
|
||||
|
||||
self.max_num_ep_ranks = envs.VLLM_NIXL_EP_MAX_NUM_RANKS
|
||||
@property
|
||||
def max_num_ep_ranks(self) -> int:
|
||||
assert NixlEPAll2AllManager._buffer is not None
|
||||
return NixlEPAll2AllManager._buffer.max_num_ep_ranks
|
||||
|
||||
def _init_buffer(
|
||||
self,
|
||||
@@ -412,11 +418,19 @@ class NixlEPAll2AllManager(All2AllManagerBase):
|
||||
) -> None:
|
||||
from nixl_ep import Buffer # type: ignore[import-not-found]
|
||||
|
||||
max_num_global_experts = self.max_num_ep_ranks * num_experts_per_rank
|
||||
parallel_config = get_current_vllm_config().parallel_config
|
||||
max_num_ep_ranks = (
|
||||
self.world_size
|
||||
* parallel_config.elastic_ep_max_dp_size
|
||||
// parallel_config.data_parallel_size
|
||||
if parallel_config.enable_elastic_ep
|
||||
else self.world_size
|
||||
)
|
||||
max_num_global_experts = max_num_ep_ranks * num_experts_per_rank
|
||||
num_rdma_bytes = Buffer.get_rdma_size_hint(
|
||||
num_max_dispatch_tokens_per_rank=max_num_tokens_per_dp_rank,
|
||||
hidden=token_hidden_size,
|
||||
num_ranks=self.max_num_ep_ranks,
|
||||
num_ranks=max_num_ep_ranks,
|
||||
num_experts=max_num_global_experts,
|
||||
)
|
||||
assert NixlEPAll2AllManager._buffer is None, (
|
||||
@@ -427,7 +441,7 @@ class NixlEPAll2AllManager(All2AllManagerBase):
|
||||
tcp_store_group=self.tcp_store_group.store,
|
||||
)
|
||||
buffer.update_memory_buffers(
|
||||
num_ranks=self.max_num_ep_ranks,
|
||||
num_ranks=max_num_ep_ranks,
|
||||
num_experts_per_rank=num_experts_per_rank,
|
||||
num_rdma_bytes=num_rdma_bytes,
|
||||
)
|
||||
@@ -435,6 +449,7 @@ class NixlEPAll2AllManager(All2AllManagerBase):
|
||||
buffer.connect_ranks(ranks_to_connect)
|
||||
NixlEPAll2AllManager._buffer = _NixlEPBufferState(
|
||||
buffer=buffer,
|
||||
max_num_ep_ranks=max_num_ep_ranks,
|
||||
connected_ep_size=self.world_size,
|
||||
active_ep_size=self.world_size,
|
||||
)
|
||||
@@ -474,6 +489,7 @@ class NixlEPAll2AllManager(All2AllManagerBase):
|
||||
|
||||
for rank in range(state.active_ep_size, target_ep_size):
|
||||
state.buffer.update_mask_buffer(rank, mask=False)
|
||||
torch.accelerator.synchronize()
|
||||
state.active_ep_size = target_ep_size
|
||||
|
||||
def _stage_ep_size(self) -> None:
|
||||
@@ -486,8 +502,12 @@ class NixlEPAll2AllManager(All2AllManagerBase):
|
||||
if target_ep_size > state.connected_ep_size:
|
||||
self._connect_to_ep_size(target_ep_size, make_active=False)
|
||||
|
||||
def commit_staged_state(self) -> None:
|
||||
"""Commit staged NIXL EP state to the active communication set."""
|
||||
def stage_ep_size(self) -> None:
|
||||
with NixlEPAll2AllManager._lock:
|
||||
self._stage_ep_size()
|
||||
|
||||
def commit_ep_size(self) -> None:
|
||||
"""Commit the staged EP size to the active communication set."""
|
||||
with NixlEPAll2AllManager._lock:
|
||||
assert NixlEPAll2AllManager._buffer is not None
|
||||
state = NixlEPAll2AllManager._buffer
|
||||
@@ -500,31 +520,31 @@ class NixlEPAll2AllManager(All2AllManagerBase):
|
||||
|
||||
self._unmask_connected_ranks(target_ep_size)
|
||||
|
||||
def _ensure_ep_size(self, *, stage: bool) -> None:
|
||||
if stage:
|
||||
self._stage_ep_size()
|
||||
else:
|
||||
self.commit_staged_state()
|
||||
@contextmanager
|
||||
def mask_remote_ranks(self) -> Iterator[None]:
|
||||
peers = [rank for rank in range(self.world_size) if rank != self.rank]
|
||||
assert NixlEPAll2AllManager._buffer is not None
|
||||
buffer = NixlEPAll2AllManager._buffer.buffer
|
||||
for rank in peers:
|
||||
buffer.update_mask_buffer(rank, mask=True)
|
||||
torch.accelerator.synchronize()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
for rank in peers:
|
||||
buffer.update_mask_buffer(rank, mask=False)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
def get_handle(self, kwargs):
|
||||
with NixlEPAll2AllManager._lock:
|
||||
stage = bool(kwargs.get("stage", False))
|
||||
state = NixlEPAll2AllManager._buffer
|
||||
if state is None:
|
||||
assert not stage, (
|
||||
"NIXL EP staged initialization requires an existing buffer"
|
||||
)
|
||||
max_num_tokens_per_dp_rank = kwargs["max_num_tokens_per_dp_rank"]
|
||||
num_experts_per_rank = (
|
||||
kwargs["num_global_experts"] // kwargs["num_ep_ranks"]
|
||||
)
|
||||
self._init_buffer(
|
||||
max_num_tokens_per_dp_rank=max_num_tokens_per_dp_rank,
|
||||
token_hidden_size=kwargs["token_hidden_size"],
|
||||
num_experts_per_rank=num_experts_per_rank,
|
||||
num_experts_per_rank=kwargs["num_local_experts"],
|
||||
)
|
||||
else:
|
||||
self._ensure_ep_size(stage=stage)
|
||||
|
||||
assert NixlEPAll2AllManager._buffer is not None
|
||||
handle = NixlEPAll2AllManager._buffer.buffer
|
||||
@@ -564,7 +584,7 @@ class NixlEPAll2AllManager(All2AllManagerBase):
|
||||
assert state is not None
|
||||
if NixlEPAll2AllManager._mask is None:
|
||||
NixlEPAll2AllManager._mask = torch.zeros(
|
||||
self.max_num_ep_ranks, device="cuda", dtype=torch.int32
|
||||
state.max_num_ep_ranks, device="cuda", dtype=torch.int32
|
||||
)
|
||||
state.buffer.query_mask_buffer(NixlEPAll2AllManager._mask)
|
||||
return NixlEPAll2AllManager._mask[: state.active_ep_size]
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import threading
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from weakref import WeakValueDictionary
|
||||
|
||||
import torch
|
||||
@@ -157,6 +159,16 @@ class All2AllManagerBase:
|
||||
def destroy(self):
|
||||
pass
|
||||
|
||||
def stage_ep_size(self) -> None:
|
||||
pass
|
||||
|
||||
def commit_ep_size(self) -> None:
|
||||
pass
|
||||
|
||||
@contextmanager
|
||||
def mask_remote_ranks(self) -> Iterator[None]:
|
||||
yield
|
||||
|
||||
|
||||
class DeviceCommunicatorBase:
|
||||
"""
|
||||
|
||||
@@ -2,34 +2,32 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import gc
|
||||
import weakref
|
||||
from collections.abc import Iterable, Sequence
|
||||
from collections.abc import Iterable, Iterator, Sequence
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from dataclasses import replace
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from torch.distributed import P2POp
|
||||
|
||||
from vllm.compilation.counter import compilation_counter
|
||||
from vllm.compilation.cuda_graph import CUDAGraphWrapper
|
||||
from vllm.compilation.wrapper import reset_compile_wrapper
|
||||
from vllm.config import (
|
||||
CompilationMode,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.config import ParallelConfig, set_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
get_dp_group,
|
||||
get_ep_group,
|
||||
get_pcp_group,
|
||||
get_tp_group,
|
||||
get_world_group,
|
||||
)
|
||||
from vllm.distributed.elastic_ep.standby_state import (
|
||||
create_standby_groups,
|
||||
get_standby_dp_group,
|
||||
get_standby_ep_group,
|
||||
get_standby_eplb_group,
|
||||
get_standby_world_group,
|
||||
pop_standby_groups,
|
||||
)
|
||||
from vllm.distributed.eplb.eplb_communicator import (
|
||||
@@ -41,13 +39,20 @@ from vllm.distributed.parallel_state import (
|
||||
)
|
||||
from vllm.distributed.stateless_coordinator import StatelessGroupCoordinator
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
get_ep_all2all_manager,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig
|
||||
from vllm.model_executor.layers.fused_moe.eep_reconfigure import (
|
||||
make_eep_staged_quant_method,
|
||||
)
|
||||
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
|
||||
sync_flashinfer_autotune_cache,
|
||||
)
|
||||
from vllm.model_executor.warmup.kernel_warmup import kernel_warmup
|
||||
from vllm.utils import is_moe_layer
|
||||
from vllm.v1.engine import ReconfigureDistributedRequest, ReconfigureRankType
|
||||
from vllm.v1.worker.dp_utils import skip_dp_coordination
|
||||
from vllm.v1.worker.gpu_ubatch_wrapper import UBatchWrapper
|
||||
from vllm.v1.worker.workspace import lock_workspace, unlock_workspace
|
||||
|
||||
@@ -59,6 +64,15 @@ if TYPE_CHECKING:
|
||||
)
|
||||
|
||||
|
||||
def can_reuse_fused_moe_kernel(parallel_config: ParallelConfig) -> bool:
|
||||
"""Whether fused MoE kernels can be reused across EP rank changes.
|
||||
|
||||
The EP backend must preserve its CUDA graph entries when ranks are connected,
|
||||
disconnected, masked, or unmasked.
|
||||
"""
|
||||
return parallel_config.all2all_backend == "nixl_ep"
|
||||
|
||||
|
||||
def batch_transfer_weights(
|
||||
model: nn.Module,
|
||||
is_sender: bool,
|
||||
@@ -105,31 +119,20 @@ def batch_transfer_weights(
|
||||
|
||||
def broadcast_expert_mapping(
|
||||
physical_to_logical: torch.Tensor | None,
|
||||
num_local_physical_experts: int | None,
|
||||
num_logical_experts: int | None,
|
||||
dp_group: StatelessGroupCoordinator,
|
||||
device: torch.device,
|
||||
src_rank: int = 0,
|
||||
) -> tuple[torch.Tensor, int, int]:
|
||||
) -> torch.Tensor:
|
||||
if dp_group.rank_in_group == src_rank:
|
||||
assert physical_to_logical is not None
|
||||
assert num_local_physical_experts is not None
|
||||
assert num_logical_experts is not None
|
||||
assert physical_to_logical.dtype == torch.int64
|
||||
shape_tensor = torch.tensor(
|
||||
list(physical_to_logical.shape), dtype=torch.int64, device="cpu"
|
||||
)
|
||||
metadata_tensor = torch.tensor(
|
||||
[num_local_physical_experts, num_logical_experts],
|
||||
dtype=torch.int64,
|
||||
device="cpu",
|
||||
)
|
||||
else:
|
||||
shape_tensor = torch.empty(2, dtype=torch.int64, device="cpu")
|
||||
metadata_tensor = torch.empty(2, dtype=torch.int64, device="cpu")
|
||||
|
||||
shape_tensor = dp_group.tcp_store_group.broadcast(shape_tensor, src_rank)
|
||||
metadata_tensor = dp_group.tcp_store_group.broadcast(metadata_tensor, src_rank)
|
||||
|
||||
if dp_group.rank_in_group != src_rank:
|
||||
assert device is not None
|
||||
@@ -140,11 +143,7 @@ def broadcast_expert_mapping(
|
||||
)
|
||||
|
||||
assert physical_to_logical is not None
|
||||
physical_to_logical = dp_group.broadcast(physical_to_logical, src_rank)
|
||||
num_local_physical_experts = int(metadata_tensor[0].item())
|
||||
num_logical_experts = int(metadata_tensor[1].item())
|
||||
|
||||
return physical_to_logical, num_local_physical_experts, num_logical_experts
|
||||
return dp_group.broadcast(physical_to_logical, src_rank)
|
||||
|
||||
|
||||
class ElasticEPScalingExecutor:
|
||||
@@ -166,6 +165,21 @@ class ElasticEPScalingExecutor:
|
||||
raise RuntimeError("Worker has been garbage collected")
|
||||
return worker
|
||||
|
||||
def _can_reuse_fused_moe_kernel(self) -> bool:
|
||||
return can_reuse_fused_moe_kernel(
|
||||
self.worker.vllm_config.parallel_config,
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def _disable_flashinfer_autotune(self) -> Iterator[None]:
|
||||
kernel_config = self.worker.vllm_config.kernel_config
|
||||
enabled = kernel_config.enable_flashinfer_autotune
|
||||
kernel_config.enable_flashinfer_autotune = False
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
kernel_config.enable_flashinfer_autotune = enabled
|
||||
|
||||
def execute(self, execute_method: str, *args, **kwargs):
|
||||
method = getattr(self, execute_method, None)
|
||||
if method is None:
|
||||
@@ -260,11 +274,20 @@ class ElasticEPScalingExecutor:
|
||||
use_all2all=use_all2all,
|
||||
enable_eplb=parallel_config.enable_eplb,
|
||||
)
|
||||
self.stage_standby_moe_quant_methods()
|
||||
standby_ep_group = get_standby_ep_group()
|
||||
assert standby_ep_group is not None
|
||||
all2all_manager = get_ep_all2all_manager(standby_ep_group)
|
||||
all2all_manager.stage_ep_size()
|
||||
if not self._can_reuse_fused_moe_kernel():
|
||||
self.stage_standby_moe_quant_methods(all2all_manager)
|
||||
self._prepare_eplb_communicator(get_standby_eplb_group())
|
||||
if new_dp_size > old_dp_size:
|
||||
self.transfer_weights(old_dp_size, new_dp_size)
|
||||
self._warm_target_groups(get_standby_dp_group(), get_standby_ep_group())
|
||||
self._warm_target_groups(get_standby_dp_group(), standby_ep_group)
|
||||
if new_dp_size > old_dp_size and self._can_reuse_fused_moe_kernel():
|
||||
target_world_group = get_standby_world_group()
|
||||
assert target_world_group is not None
|
||||
sync_flashinfer_autotune_cache(self.worker.model_runner, target_world_group)
|
||||
|
||||
def _prepare_eplb_communicator(self, eplb_group) -> None:
|
||||
assert eplb_group is not None
|
||||
@@ -338,14 +361,9 @@ class ElasticEPScalingExecutor:
|
||||
assert eplb_state is not None
|
||||
eplb_state.drain_async()
|
||||
eplb_model_state = eplb_state.model_states[model_config.compute_hash()]
|
||||
physical_to_logical = eplb_model_state.physical_to_logical_map
|
||||
num_physical_experts = physical_to_logical.shape[1]
|
||||
num_local_physical_experts = num_physical_experts // get_ep_group().world_size
|
||||
num_logical_experts = eplb_model_state.logical_replica_count.shape[1]
|
||||
physical_to_logical = eplb_model_state.physical_to_logical_map_buffer
|
||||
broadcast_expert_mapping(
|
||||
physical_to_logical=physical_to_logical,
|
||||
num_local_physical_experts=num_local_physical_experts,
|
||||
num_logical_experts=num_logical_experts,
|
||||
dp_group=standby_dp_group,
|
||||
src_rank=0,
|
||||
device=self.worker.device,
|
||||
@@ -368,7 +386,7 @@ class ElasticEPScalingExecutor:
|
||||
moe_parallel_config=moe_parallel_config,
|
||||
)
|
||||
|
||||
def stage_standby_moe_quant_methods(self) -> None:
|
||||
def stage_standby_moe_quant_methods(self, all2all_manager) -> None:
|
||||
standby_dp_group = get_standby_dp_group()
|
||||
standby_ep_group = get_standby_ep_group()
|
||||
model = self.worker.model_runner.get_model()
|
||||
@@ -383,6 +401,7 @@ class ElasticEPScalingExecutor:
|
||||
standby_dp_group,
|
||||
standby_ep_group,
|
||||
),
|
||||
all2all_manager=all2all_manager,
|
||||
)
|
||||
if staged_quant_method is not None:
|
||||
self._staged_moe_quant_methods[module] = staged_quant_method
|
||||
@@ -396,7 +415,6 @@ class ElasticEPScalingExecutor:
|
||||
continue
|
||||
assert staged_quant_method.moe_kernel is not None
|
||||
module._replace_quant_method(staged_quant_method)
|
||||
staged_quant_method.moe_kernel.prepare_finalize.on_commit()
|
||||
self._staged_moe_quant_methods.clear()
|
||||
|
||||
def _release_cuda_graphs(self) -> None:
|
||||
@@ -405,7 +423,7 @@ class ElasticEPScalingExecutor:
|
||||
wrapper.concrete_cudagraph_entries = {}
|
||||
|
||||
elif isinstance(self.worker.model_runner.model, UBatchWrapper):
|
||||
raise RuntimeError("DBO is not yet supported in elastic EP")
|
||||
self.worker.model_runner.model.clear_graphs()
|
||||
|
||||
torch.compiler.reset()
|
||||
with set_current_vllm_config(self.worker.vllm_config):
|
||||
@@ -427,10 +445,8 @@ class ElasticEPScalingExecutor:
|
||||
self._wait_for_group_cleanup()
|
||||
|
||||
def switch_and_prepare(self) -> tuple[GroupCoordinator | None, ...]:
|
||||
old_dp_size = get_dp_group().world_size
|
||||
old_ep_size = get_ep_group().world_size
|
||||
|
||||
self._release_cuda_graphs()
|
||||
if not self._can_reuse_fused_moe_kernel():
|
||||
self._release_cuda_graphs()
|
||||
retired_groups = _replace_active_groups(**pop_standby_groups())
|
||||
|
||||
parallel_config = self.worker.vllm_config.parallel_config
|
||||
@@ -487,47 +503,13 @@ class ElasticEPScalingExecutor:
|
||||
parallel_config.eplb_config.num_redundant_experts = (
|
||||
num_physical_experts - num_logical_experts
|
||||
)
|
||||
old_physical_to_logical = eplb_model_state.physical_to_logical_map
|
||||
num_moe_layers = old_physical_to_logical.shape[0]
|
||||
num_local_experts = eplb_model_state.expert_load_pass.shape[1] // old_ep_size
|
||||
if new_dp_size > old_dp_size:
|
||||
expanded_physical_to_logical = torch.full(
|
||||
(num_moe_layers, num_local_experts * new_ep_size),
|
||||
-1,
|
||||
dtype=old_physical_to_logical.dtype,
|
||||
device=old_physical_to_logical.device,
|
||||
)
|
||||
expanded_physical_to_logical[:, : num_local_experts * old_ep_size] = (
|
||||
old_physical_to_logical
|
||||
)
|
||||
eplb_model_state.physical_to_logical_map = expanded_physical_to_logical
|
||||
|
||||
old_num_physical_experts = eplb_model_state.expert_load_pass.shape[1]
|
||||
pad_size = num_physical_experts - old_num_physical_experts
|
||||
if new_dp_size > old_dp_size:
|
||||
assert pad_size > 0
|
||||
expanded_expert_load_pass = F.pad(
|
||||
eplb_model_state.expert_load_pass, (0, pad_size), value=0
|
||||
)
|
||||
expanded_expert_load_window = F.pad(
|
||||
eplb_model_state.expert_load_window, (0, pad_size), value=0
|
||||
)
|
||||
eplb_model_state.expert_load_pass = expanded_expert_load_pass
|
||||
eplb_model_state.expert_load_window = expanded_expert_load_window
|
||||
else:
|
||||
assert pad_size < 0
|
||||
eplb_model_state.expert_load_pass = eplb_model_state.expert_load_pass[
|
||||
:, :num_physical_experts
|
||||
]
|
||||
eplb_model_state.expert_load_window = eplb_model_state.expert_load_window[
|
||||
:, :, :num_physical_experts
|
||||
]
|
||||
eplb_state.reconfigure_physical_expert_slots(model_config, num_physical_experts)
|
||||
|
||||
model = self.worker.model_runner.get_model()
|
||||
model.expert_weights = []
|
||||
with set_current_vllm_config(self.worker.vllm_config):
|
||||
model.set_eplb_state(
|
||||
eplb_model_state.expert_load_pass,
|
||||
eplb_model_state.expert_load_pass_buffer,
|
||||
eplb_model_state.logical_to_physical_map,
|
||||
eplb_model_state.logical_replica_count,
|
||||
)
|
||||
@@ -539,6 +521,7 @@ class ElasticEPScalingExecutor:
|
||||
num_local_physical_experts=num_local_experts,
|
||||
)
|
||||
self._commit_staged_moe_quant_methods()
|
||||
get_ep_all2all_manager().commit_ep_size()
|
||||
# Legacy modular methods need to be recreated for the new EP size.
|
||||
for module in moe_modules:
|
||||
if getattr(module._quant_method, "wraps_legacy_quant_method", False):
|
||||
@@ -548,21 +531,6 @@ class ElasticEPScalingExecutor:
|
||||
eplb_state.update_communicator(model_config, self._prepared_eplb_communicator)
|
||||
self._prepared_eplb_communicator = None
|
||||
|
||||
if (
|
||||
self.worker.vllm_config.compilation_config.mode
|
||||
== CompilationMode.STOCK_TORCH_COMPILE
|
||||
):
|
||||
# NOTE(yongji): when using stock torch.compile,
|
||||
# torch.compile is triggered during GPUModelRunner's load_model()
|
||||
# TODO(yongji):check do we need to re-trigger torch.compile here?
|
||||
# any changes to the tensor shapes in execution should already
|
||||
# be handled internally by torch.compile.
|
||||
backend = self.worker.vllm_config.compilation_config.init_backend(
|
||||
self.worker.vllm_config
|
||||
)
|
||||
compilation_counter.stock_torch_compile_count += 1
|
||||
self.worker.model_runner.model.compile(fullgraph=True, backend=backend)
|
||||
|
||||
return retired_groups
|
||||
|
||||
def _perform_eplb_reshuffle(
|
||||
@@ -608,7 +576,8 @@ class ElasticEPScalingExecutor:
|
||||
else:
|
||||
mapping = self.receive_expert_mapping()
|
||||
self.worker.model_runner.setup_eplb_from_mapping(mapping)
|
||||
self.warm_and_capture()
|
||||
if not self._can_reuse_fused_moe_kernel():
|
||||
self.warm_and_capture()
|
||||
self._perform_eplb_reshuffle(async_op=True)
|
||||
if is_existing_worker:
|
||||
self._start_group_cleanup(retired_groups)
|
||||
@@ -619,7 +588,8 @@ class ElasticEPScalingExecutor:
|
||||
self.switch_and_remove()
|
||||
else:
|
||||
retired_groups = self.switch_and_prepare()
|
||||
self.warm_and_capture()
|
||||
if not self._can_reuse_fused_moe_kernel():
|
||||
self.warm_and_capture()
|
||||
self._start_group_cleanup(retired_groups)
|
||||
|
||||
def perform_scale_down_eplb_reshuffle(self, new_dp_size: int) -> None:
|
||||
@@ -677,42 +647,26 @@ class ElasticEPScalingExecutor:
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
self._warm_target_groups(get_dp_group(), get_ep_group())
|
||||
if self._can_reuse_fused_moe_kernel():
|
||||
sync_flashinfer_autotune_cache(self.worker.model_runner, get_world_group())
|
||||
|
||||
def receive_expert_mapping(self) -> torch.Tensor:
|
||||
dp_group = get_dp_group()
|
||||
assert isinstance(dp_group, StatelessGroupCoordinator)
|
||||
physical_to_logical, num_local_physical_experts, _ = broadcast_expert_mapping(
|
||||
return broadcast_expert_mapping(
|
||||
physical_to_logical=None,
|
||||
num_local_physical_experts=None,
|
||||
num_logical_experts=None,
|
||||
dp_group=dp_group,
|
||||
src_rank=0,
|
||||
device=self.worker.device,
|
||||
)
|
||||
num_moe_layers = physical_to_logical.shape[0]
|
||||
new_dp_size = get_dp_group().world_size
|
||||
tp_size = self.worker.vllm_config.parallel_config.tensor_parallel_size
|
||||
new_ep_size = new_dp_size * tp_size
|
||||
expanded_physical_to_logical = torch.full(
|
||||
(num_moe_layers, num_local_physical_experts * new_ep_size),
|
||||
-1,
|
||||
dtype=physical_to_logical.dtype,
|
||||
device=physical_to_logical.device,
|
||||
)
|
||||
expanded_physical_to_logical[:, : physical_to_logical.shape[1]] = (
|
||||
physical_to_logical
|
||||
)
|
||||
return expanded_physical_to_logical
|
||||
|
||||
def warmup_local_kernels(self) -> None:
|
||||
def warmup_new_worker(self) -> None:
|
||||
with set_current_vllm_config(self.worker.vllm_config):
|
||||
kernel_warmup(self.worker, process_local_only=True)
|
||||
if self._can_reuse_fused_moe_kernel():
|
||||
self.warm_and_capture()
|
||||
|
||||
def warm_and_capture(self) -> None:
|
||||
# Must run on every DP sibling in lockstep: _dummy_run calls
|
||||
# coordinate_batch_across_dp whenever data_parallel_size > 1
|
||||
# (gpu_model_runner.py:3663), which deadlocks if any rank skips it.
|
||||
|
||||
# Save and clear block tables so the dummy MoE forward doesn't
|
||||
# write dummy slot mappings into real KV-cache blocks.
|
||||
multi_block_table = self.worker.model_runner.input_batch.block_table
|
||||
@@ -735,8 +689,15 @@ class ElasticEPScalingExecutor:
|
||||
# directly with skip_eplb=True so dummy routing doesn't pollute the
|
||||
# just-rebalanced EPLB stats.
|
||||
runner = self.worker.model_runner
|
||||
runner._dummy_run(runner.max_num_tokens, is_profile=True, skip_eplb=True)
|
||||
self.worker.compile_or_warm_up_model()
|
||||
all2all_manager = get_ep_all2all_manager()
|
||||
reuse_kernel = self._can_reuse_fused_moe_kernel()
|
||||
with (
|
||||
skip_dp_coordination() if reuse_kernel else nullcontext(),
|
||||
all2all_manager.mask_remote_ranks() if reuse_kernel else nullcontext(),
|
||||
self._disable_flashinfer_autotune() if reuse_kernel else nullcontext(),
|
||||
):
|
||||
runner._dummy_run(runner.max_num_tokens, is_profile=True, skip_eplb=True)
|
||||
self.worker.compile_or_warm_up_model()
|
||||
|
||||
lock_workspace()
|
||||
|
||||
|
||||
@@ -204,7 +204,7 @@ class ElasticEPScalingState:
|
||||
return True
|
||||
|
||||
elif state == ScaleUpNewEngineState.PREPARE:
|
||||
self._collective_rpc("elastic_ep_execute", args=("warmup_local_kernels",))
|
||||
self._collective_rpc("elastic_ep_execute", args=("warmup_new_worker",))
|
||||
self._mark_ready_for_switch()
|
||||
tensor = torch.tensor([0, 0, 0], dtype=torch.int32, device="cpu")
|
||||
torch.distributed.all_reduce(
|
||||
|
||||
@@ -121,6 +121,8 @@ class EplbModelState:
|
||||
[0, 2, 0, 1, 0, 3]]
|
||||
```
|
||||
"""
|
||||
physical_to_logical_map_buffer: torch.Tensor
|
||||
"""Maximum-capacity buffer backing ``physical_to_logical_map``."""
|
||||
logical_to_physical_map: torch.Tensor
|
||||
"""
|
||||
Mapping from logical experts to physical experts.
|
||||
@@ -168,6 +170,8 @@ class EplbModelState:
|
||||
|
||||
Shape: (num_moe_layers, num_physical_experts)
|
||||
"""
|
||||
expert_load_pass_buffer: torch.Tensor
|
||||
"""Maximum-capacity buffer backing ``expert_load_pass``."""
|
||||
expert_load_window: torch.Tensor
|
||||
"""
|
||||
A sliding window of expert load.
|
||||
@@ -373,6 +377,19 @@ class EplbState:
|
||||
physical_to_logical_map_list,
|
||||
device=self.device,
|
||||
)
|
||||
physical_expert_capacity = model.num_physical_experts
|
||||
if self.parallel_config.enable_elastic_ep:
|
||||
physical_expert_capacity = (
|
||||
model.num_local_physical_experts
|
||||
* self.parallel_config.elastic_ep_max_dp_size
|
||||
* get_ep_group().world_size
|
||||
// self.parallel_config.data_parallel_size
|
||||
)
|
||||
physical_to_logical_map = torch.nn.functional.pad(
|
||||
physical_to_logical_map,
|
||||
(0, physical_expert_capacity - model.num_physical_experts),
|
||||
value=-1,
|
||||
)
|
||||
# Assuming 8 GPUs per node, this supports up to
|
||||
# (1023 + 1) / 8 = 128 nodes for now.
|
||||
# TODO(rui): make this configurable
|
||||
@@ -407,6 +424,10 @@ class EplbState:
|
||||
)
|
||||
.contiguous()
|
||||
)
|
||||
physical_to_logical_map_buffer = physical_to_logical_map
|
||||
physical_to_logical_map = physical_to_logical_map_buffer[
|
||||
:, : model.num_physical_experts
|
||||
]
|
||||
logical_to_physical_map = (
|
||||
logical_to_physical_map.unsqueeze(0)
|
||||
.expand(
|
||||
@@ -425,11 +446,12 @@ class EplbState:
|
||||
.contiguous()
|
||||
)
|
||||
|
||||
expert_load_pass = torch.zeros(
|
||||
(model.num_moe_layers, model.num_physical_experts),
|
||||
expert_load_pass_buffer = torch.zeros(
|
||||
(model.num_moe_layers, physical_expert_capacity),
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
expert_load_pass = expert_load_pass_buffer[:, : model.num_physical_experts]
|
||||
self.expert_load_window_size = self.parallel_config.eplb_config.window_size
|
||||
expert_load_window = torch.zeros(
|
||||
(
|
||||
@@ -460,7 +482,7 @@ class EplbState:
|
||||
]
|
||||
|
||||
model.set_eplb_state(
|
||||
expert_load_pass,
|
||||
expert_load_pass_buffer,
|
||||
logical_to_physical_map,
|
||||
logical_replica_count,
|
||||
)
|
||||
@@ -479,9 +501,11 @@ class EplbState:
|
||||
|
||||
model_state = EplbModelState(
|
||||
physical_to_logical_map=physical_to_logical_map,
|
||||
physical_to_logical_map_buffer=physical_to_logical_map_buffer,
|
||||
logical_to_physical_map=logical_to_physical_map,
|
||||
logical_replica_count=logical_replica_count,
|
||||
expert_load_pass=expert_load_pass,
|
||||
expert_load_pass_buffer=expert_load_pass_buffer,
|
||||
expert_load_window=expert_load_window,
|
||||
model_name=model_config.model,
|
||||
model=model,
|
||||
@@ -1075,10 +1099,14 @@ class EplbState:
|
||||
expanded_physical_to_logical: torch.Tensor,
|
||||
) -> None:
|
||||
eplb_model_state = self.model_states[model_config.compute_hash()]
|
||||
eplb_model_state.physical_to_logical_map.copy_(expanded_physical_to_logical)
|
||||
eplb_model_state.physical_to_logical_map_buffer.copy_(
|
||||
expanded_physical_to_logical
|
||||
)
|
||||
eplb_model_state.expert_load_pass.zero_()
|
||||
physical_to_logical_map = eplb_model_state.physical_to_logical_map
|
||||
|
||||
(logical_to_physical_map_cpu, logical_replica_count_cpu) = compute_logical_maps(
|
||||
expanded_physical_to_logical.cpu(),
|
||||
physical_to_logical_map.cpu(),
|
||||
eplb_model_state.model.num_logical_experts,
|
||||
)
|
||||
|
||||
@@ -1097,6 +1125,33 @@ class EplbState:
|
||||
eplb_model_state.logical_to_physical_map.copy_(logical_to_physical_map)
|
||||
eplb_model_state.logical_replica_count.copy_(logical_replica_count)
|
||||
|
||||
def reconfigure_physical_expert_slots(
|
||||
self,
|
||||
model_config: ModelConfig,
|
||||
num_physical_experts: int,
|
||||
) -> None:
|
||||
"""Replace physical_to_logical_map and expert_load_pass with views
|
||||
covering the new active size.
|
||||
"""
|
||||
model_state = self.model_states[model_config.compute_hash()]
|
||||
old_num_physical_experts = model_state.model.num_physical_experts
|
||||
first_slot, last_slot = sorted((old_num_physical_experts, num_physical_experts))
|
||||
physical_to_logical_map_buffer = model_state.physical_to_logical_map_buffer
|
||||
expert_load_pass_buffer = model_state.expert_load_pass_buffer
|
||||
assert last_slot <= physical_to_logical_map_buffer.shape[1]
|
||||
expert_slots = slice(first_slot, last_slot)
|
||||
physical_to_logical_map_buffer[:, expert_slots].fill_(-1)
|
||||
expert_load_pass_buffer[:, expert_slots].zero_()
|
||||
model_state.physical_to_logical_map = physical_to_logical_map_buffer[
|
||||
:, :num_physical_experts
|
||||
]
|
||||
model_state.expert_load_pass = expert_load_pass_buffer[:, :num_physical_experts]
|
||||
|
||||
pad_size = num_physical_experts - model_state.expert_load_window.shape[-1]
|
||||
model_state.expert_load_window = torch.nn.functional.pad(
|
||||
model_state.expert_load_window, (0, pad_size)
|
||||
)
|
||||
|
||||
def create_communicator(
|
||||
self, model_config: ModelConfig, group_coordinator: GroupCoordinator
|
||||
) -> EplbCommunicator:
|
||||
@@ -1333,14 +1388,13 @@ def _commit_eplb_maps(
|
||||
|
||||
if PIN_MEMORY and src.is_cpu:
|
||||
src = src.new_empty(src.shape, pin_memory=True).copy_(src)
|
||||
# Rare Case: When the number of physical experts has changed, discard the old
|
||||
# physical to logical expert map and use the new one. This only happens when the
|
||||
# number of GPUs available to vLLM changes while vLLM is running. Otherwise copy the
|
||||
# new map into the old one.
|
||||
# When the number of physical experts changes, refresh the active map view to
|
||||
# cover the new size before copying it.
|
||||
if src.shape[1] != dst.shape[1]:
|
||||
model_state.physical_to_logical_map = src.to(dst.device, non_blocking=True)
|
||||
else:
|
||||
dst.copy_(src, non_blocking=True)
|
||||
dst = model_state.physical_to_logical_map_buffer[:, : src.shape[1]]
|
||||
assert dst.shape == src.shape
|
||||
model_state.physical_to_logical_map = dst
|
||||
dst.copy_(src, non_blocking=True)
|
||||
|
||||
num_logical_experts = model_state.logical_to_physical_map.shape[1]
|
||||
new_logical, new_replica_count = compute_logical_maps(src, num_logical_experts)
|
||||
|
||||
Reference in New Issue
Block a user