[Elastic EP] Reuse CUDA graphs across reconfiguration (#54985)
This commit is contained in:
@@ -2,34 +2,32 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import gc
|
||||
import weakref
|
||||
from collections.abc import Iterable, Sequence
|
||||
from collections.abc import Iterable, Iterator, Sequence
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from dataclasses import replace
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from torch.distributed import P2POp
|
||||
|
||||
from vllm.compilation.counter import compilation_counter
|
||||
from vllm.compilation.cuda_graph import CUDAGraphWrapper
|
||||
from vllm.compilation.wrapper import reset_compile_wrapper
|
||||
from vllm.config import (
|
||||
CompilationMode,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.config import ParallelConfig, set_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
get_dp_group,
|
||||
get_ep_group,
|
||||
get_pcp_group,
|
||||
get_tp_group,
|
||||
get_world_group,
|
||||
)
|
||||
from vllm.distributed.elastic_ep.standby_state import (
|
||||
create_standby_groups,
|
||||
get_standby_dp_group,
|
||||
get_standby_ep_group,
|
||||
get_standby_eplb_group,
|
||||
get_standby_world_group,
|
||||
pop_standby_groups,
|
||||
)
|
||||
from vllm.distributed.eplb.eplb_communicator import (
|
||||
@@ -41,13 +39,20 @@ from vllm.distributed.parallel_state import (
|
||||
)
|
||||
from vllm.distributed.stateless_coordinator import StatelessGroupCoordinator
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
get_ep_all2all_manager,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig
|
||||
from vllm.model_executor.layers.fused_moe.eep_reconfigure import (
|
||||
make_eep_staged_quant_method,
|
||||
)
|
||||
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
|
||||
sync_flashinfer_autotune_cache,
|
||||
)
|
||||
from vllm.model_executor.warmup.kernel_warmup import kernel_warmup
|
||||
from vllm.utils import is_moe_layer
|
||||
from vllm.v1.engine import ReconfigureDistributedRequest, ReconfigureRankType
|
||||
from vllm.v1.worker.dp_utils import skip_dp_coordination
|
||||
from vllm.v1.worker.gpu_ubatch_wrapper import UBatchWrapper
|
||||
from vllm.v1.worker.workspace import lock_workspace, unlock_workspace
|
||||
|
||||
@@ -59,6 +64,15 @@ if TYPE_CHECKING:
|
||||
)
|
||||
|
||||
|
||||
def can_reuse_fused_moe_kernel(parallel_config: ParallelConfig) -> bool:
|
||||
"""Whether fused MoE kernels can be reused across EP rank changes.
|
||||
|
||||
The EP backend must preserve its CUDA graph entries when ranks are connected,
|
||||
disconnected, masked, or unmasked.
|
||||
"""
|
||||
return parallel_config.all2all_backend == "nixl_ep"
|
||||
|
||||
|
||||
def batch_transfer_weights(
|
||||
model: nn.Module,
|
||||
is_sender: bool,
|
||||
@@ -105,31 +119,20 @@ def batch_transfer_weights(
|
||||
|
||||
def broadcast_expert_mapping(
|
||||
physical_to_logical: torch.Tensor | None,
|
||||
num_local_physical_experts: int | None,
|
||||
num_logical_experts: int | None,
|
||||
dp_group: StatelessGroupCoordinator,
|
||||
device: torch.device,
|
||||
src_rank: int = 0,
|
||||
) -> tuple[torch.Tensor, int, int]:
|
||||
) -> torch.Tensor:
|
||||
if dp_group.rank_in_group == src_rank:
|
||||
assert physical_to_logical is not None
|
||||
assert num_local_physical_experts is not None
|
||||
assert num_logical_experts is not None
|
||||
assert physical_to_logical.dtype == torch.int64
|
||||
shape_tensor = torch.tensor(
|
||||
list(physical_to_logical.shape), dtype=torch.int64, device="cpu"
|
||||
)
|
||||
metadata_tensor = torch.tensor(
|
||||
[num_local_physical_experts, num_logical_experts],
|
||||
dtype=torch.int64,
|
||||
device="cpu",
|
||||
)
|
||||
else:
|
||||
shape_tensor = torch.empty(2, dtype=torch.int64, device="cpu")
|
||||
metadata_tensor = torch.empty(2, dtype=torch.int64, device="cpu")
|
||||
|
||||
shape_tensor = dp_group.tcp_store_group.broadcast(shape_tensor, src_rank)
|
||||
metadata_tensor = dp_group.tcp_store_group.broadcast(metadata_tensor, src_rank)
|
||||
|
||||
if dp_group.rank_in_group != src_rank:
|
||||
assert device is not None
|
||||
@@ -140,11 +143,7 @@ def broadcast_expert_mapping(
|
||||
)
|
||||
|
||||
assert physical_to_logical is not None
|
||||
physical_to_logical = dp_group.broadcast(physical_to_logical, src_rank)
|
||||
num_local_physical_experts = int(metadata_tensor[0].item())
|
||||
num_logical_experts = int(metadata_tensor[1].item())
|
||||
|
||||
return physical_to_logical, num_local_physical_experts, num_logical_experts
|
||||
return dp_group.broadcast(physical_to_logical, src_rank)
|
||||
|
||||
|
||||
class ElasticEPScalingExecutor:
|
||||
@@ -166,6 +165,21 @@ class ElasticEPScalingExecutor:
|
||||
raise RuntimeError("Worker has been garbage collected")
|
||||
return worker
|
||||
|
||||
def _can_reuse_fused_moe_kernel(self) -> bool:
|
||||
return can_reuse_fused_moe_kernel(
|
||||
self.worker.vllm_config.parallel_config,
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def _disable_flashinfer_autotune(self) -> Iterator[None]:
|
||||
kernel_config = self.worker.vllm_config.kernel_config
|
||||
enabled = kernel_config.enable_flashinfer_autotune
|
||||
kernel_config.enable_flashinfer_autotune = False
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
kernel_config.enable_flashinfer_autotune = enabled
|
||||
|
||||
def execute(self, execute_method: str, *args, **kwargs):
|
||||
method = getattr(self, execute_method, None)
|
||||
if method is None:
|
||||
@@ -260,11 +274,20 @@ class ElasticEPScalingExecutor:
|
||||
use_all2all=use_all2all,
|
||||
enable_eplb=parallel_config.enable_eplb,
|
||||
)
|
||||
self.stage_standby_moe_quant_methods()
|
||||
standby_ep_group = get_standby_ep_group()
|
||||
assert standby_ep_group is not None
|
||||
all2all_manager = get_ep_all2all_manager(standby_ep_group)
|
||||
all2all_manager.stage_ep_size()
|
||||
if not self._can_reuse_fused_moe_kernel():
|
||||
self.stage_standby_moe_quant_methods(all2all_manager)
|
||||
self._prepare_eplb_communicator(get_standby_eplb_group())
|
||||
if new_dp_size > old_dp_size:
|
||||
self.transfer_weights(old_dp_size, new_dp_size)
|
||||
self._warm_target_groups(get_standby_dp_group(), get_standby_ep_group())
|
||||
self._warm_target_groups(get_standby_dp_group(), standby_ep_group)
|
||||
if new_dp_size > old_dp_size and self._can_reuse_fused_moe_kernel():
|
||||
target_world_group = get_standby_world_group()
|
||||
assert target_world_group is not None
|
||||
sync_flashinfer_autotune_cache(self.worker.model_runner, target_world_group)
|
||||
|
||||
def _prepare_eplb_communicator(self, eplb_group) -> None:
|
||||
assert eplb_group is not None
|
||||
@@ -338,14 +361,9 @@ class ElasticEPScalingExecutor:
|
||||
assert eplb_state is not None
|
||||
eplb_state.drain_async()
|
||||
eplb_model_state = eplb_state.model_states[model_config.compute_hash()]
|
||||
physical_to_logical = eplb_model_state.physical_to_logical_map
|
||||
num_physical_experts = physical_to_logical.shape[1]
|
||||
num_local_physical_experts = num_physical_experts // get_ep_group().world_size
|
||||
num_logical_experts = eplb_model_state.logical_replica_count.shape[1]
|
||||
physical_to_logical = eplb_model_state.physical_to_logical_map_buffer
|
||||
broadcast_expert_mapping(
|
||||
physical_to_logical=physical_to_logical,
|
||||
num_local_physical_experts=num_local_physical_experts,
|
||||
num_logical_experts=num_logical_experts,
|
||||
dp_group=standby_dp_group,
|
||||
src_rank=0,
|
||||
device=self.worker.device,
|
||||
@@ -368,7 +386,7 @@ class ElasticEPScalingExecutor:
|
||||
moe_parallel_config=moe_parallel_config,
|
||||
)
|
||||
|
||||
def stage_standby_moe_quant_methods(self) -> None:
|
||||
def stage_standby_moe_quant_methods(self, all2all_manager) -> None:
|
||||
standby_dp_group = get_standby_dp_group()
|
||||
standby_ep_group = get_standby_ep_group()
|
||||
model = self.worker.model_runner.get_model()
|
||||
@@ -383,6 +401,7 @@ class ElasticEPScalingExecutor:
|
||||
standby_dp_group,
|
||||
standby_ep_group,
|
||||
),
|
||||
all2all_manager=all2all_manager,
|
||||
)
|
||||
if staged_quant_method is not None:
|
||||
self._staged_moe_quant_methods[module] = staged_quant_method
|
||||
@@ -396,7 +415,6 @@ class ElasticEPScalingExecutor:
|
||||
continue
|
||||
assert staged_quant_method.moe_kernel is not None
|
||||
module._replace_quant_method(staged_quant_method)
|
||||
staged_quant_method.moe_kernel.prepare_finalize.on_commit()
|
||||
self._staged_moe_quant_methods.clear()
|
||||
|
||||
def _release_cuda_graphs(self) -> None:
|
||||
@@ -405,7 +423,7 @@ class ElasticEPScalingExecutor:
|
||||
wrapper.concrete_cudagraph_entries = {}
|
||||
|
||||
elif isinstance(self.worker.model_runner.model, UBatchWrapper):
|
||||
raise RuntimeError("DBO is not yet supported in elastic EP")
|
||||
self.worker.model_runner.model.clear_graphs()
|
||||
|
||||
torch.compiler.reset()
|
||||
with set_current_vllm_config(self.worker.vllm_config):
|
||||
@@ -427,10 +445,8 @@ class ElasticEPScalingExecutor:
|
||||
self._wait_for_group_cleanup()
|
||||
|
||||
def switch_and_prepare(self) -> tuple[GroupCoordinator | None, ...]:
|
||||
old_dp_size = get_dp_group().world_size
|
||||
old_ep_size = get_ep_group().world_size
|
||||
|
||||
self._release_cuda_graphs()
|
||||
if not self._can_reuse_fused_moe_kernel():
|
||||
self._release_cuda_graphs()
|
||||
retired_groups = _replace_active_groups(**pop_standby_groups())
|
||||
|
||||
parallel_config = self.worker.vllm_config.parallel_config
|
||||
@@ -487,47 +503,13 @@ class ElasticEPScalingExecutor:
|
||||
parallel_config.eplb_config.num_redundant_experts = (
|
||||
num_physical_experts - num_logical_experts
|
||||
)
|
||||
old_physical_to_logical = eplb_model_state.physical_to_logical_map
|
||||
num_moe_layers = old_physical_to_logical.shape[0]
|
||||
num_local_experts = eplb_model_state.expert_load_pass.shape[1] // old_ep_size
|
||||
if new_dp_size > old_dp_size:
|
||||
expanded_physical_to_logical = torch.full(
|
||||
(num_moe_layers, num_local_experts * new_ep_size),
|
||||
-1,
|
||||
dtype=old_physical_to_logical.dtype,
|
||||
device=old_physical_to_logical.device,
|
||||
)
|
||||
expanded_physical_to_logical[:, : num_local_experts * old_ep_size] = (
|
||||
old_physical_to_logical
|
||||
)
|
||||
eplb_model_state.physical_to_logical_map = expanded_physical_to_logical
|
||||
|
||||
old_num_physical_experts = eplb_model_state.expert_load_pass.shape[1]
|
||||
pad_size = num_physical_experts - old_num_physical_experts
|
||||
if new_dp_size > old_dp_size:
|
||||
assert pad_size > 0
|
||||
expanded_expert_load_pass = F.pad(
|
||||
eplb_model_state.expert_load_pass, (0, pad_size), value=0
|
||||
)
|
||||
expanded_expert_load_window = F.pad(
|
||||
eplb_model_state.expert_load_window, (0, pad_size), value=0
|
||||
)
|
||||
eplb_model_state.expert_load_pass = expanded_expert_load_pass
|
||||
eplb_model_state.expert_load_window = expanded_expert_load_window
|
||||
else:
|
||||
assert pad_size < 0
|
||||
eplb_model_state.expert_load_pass = eplb_model_state.expert_load_pass[
|
||||
:, :num_physical_experts
|
||||
]
|
||||
eplb_model_state.expert_load_window = eplb_model_state.expert_load_window[
|
||||
:, :, :num_physical_experts
|
||||
]
|
||||
eplb_state.reconfigure_physical_expert_slots(model_config, num_physical_experts)
|
||||
|
||||
model = self.worker.model_runner.get_model()
|
||||
model.expert_weights = []
|
||||
with set_current_vllm_config(self.worker.vllm_config):
|
||||
model.set_eplb_state(
|
||||
eplb_model_state.expert_load_pass,
|
||||
eplb_model_state.expert_load_pass_buffer,
|
||||
eplb_model_state.logical_to_physical_map,
|
||||
eplb_model_state.logical_replica_count,
|
||||
)
|
||||
@@ -539,6 +521,7 @@ class ElasticEPScalingExecutor:
|
||||
num_local_physical_experts=num_local_experts,
|
||||
)
|
||||
self._commit_staged_moe_quant_methods()
|
||||
get_ep_all2all_manager().commit_ep_size()
|
||||
# Legacy modular methods need to be recreated for the new EP size.
|
||||
for module in moe_modules:
|
||||
if getattr(module._quant_method, "wraps_legacy_quant_method", False):
|
||||
@@ -548,21 +531,6 @@ class ElasticEPScalingExecutor:
|
||||
eplb_state.update_communicator(model_config, self._prepared_eplb_communicator)
|
||||
self._prepared_eplb_communicator = None
|
||||
|
||||
if (
|
||||
self.worker.vllm_config.compilation_config.mode
|
||||
== CompilationMode.STOCK_TORCH_COMPILE
|
||||
):
|
||||
# NOTE(yongji): when using stock torch.compile,
|
||||
# torch.compile is triggered during GPUModelRunner's load_model()
|
||||
# TODO(yongji):check do we need to re-trigger torch.compile here?
|
||||
# any changes to the tensor shapes in execution should already
|
||||
# be handled internally by torch.compile.
|
||||
backend = self.worker.vllm_config.compilation_config.init_backend(
|
||||
self.worker.vllm_config
|
||||
)
|
||||
compilation_counter.stock_torch_compile_count += 1
|
||||
self.worker.model_runner.model.compile(fullgraph=True, backend=backend)
|
||||
|
||||
return retired_groups
|
||||
|
||||
def _perform_eplb_reshuffle(
|
||||
@@ -608,7 +576,8 @@ class ElasticEPScalingExecutor:
|
||||
else:
|
||||
mapping = self.receive_expert_mapping()
|
||||
self.worker.model_runner.setup_eplb_from_mapping(mapping)
|
||||
self.warm_and_capture()
|
||||
if not self._can_reuse_fused_moe_kernel():
|
||||
self.warm_and_capture()
|
||||
self._perform_eplb_reshuffle(async_op=True)
|
||||
if is_existing_worker:
|
||||
self._start_group_cleanup(retired_groups)
|
||||
@@ -619,7 +588,8 @@ class ElasticEPScalingExecutor:
|
||||
self.switch_and_remove()
|
||||
else:
|
||||
retired_groups = self.switch_and_prepare()
|
||||
self.warm_and_capture()
|
||||
if not self._can_reuse_fused_moe_kernel():
|
||||
self.warm_and_capture()
|
||||
self._start_group_cleanup(retired_groups)
|
||||
|
||||
def perform_scale_down_eplb_reshuffle(self, new_dp_size: int) -> None:
|
||||
@@ -677,42 +647,26 @@ class ElasticEPScalingExecutor:
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
self._warm_target_groups(get_dp_group(), get_ep_group())
|
||||
if self._can_reuse_fused_moe_kernel():
|
||||
sync_flashinfer_autotune_cache(self.worker.model_runner, get_world_group())
|
||||
|
||||
def receive_expert_mapping(self) -> torch.Tensor:
|
||||
dp_group = get_dp_group()
|
||||
assert isinstance(dp_group, StatelessGroupCoordinator)
|
||||
physical_to_logical, num_local_physical_experts, _ = broadcast_expert_mapping(
|
||||
return broadcast_expert_mapping(
|
||||
physical_to_logical=None,
|
||||
num_local_physical_experts=None,
|
||||
num_logical_experts=None,
|
||||
dp_group=dp_group,
|
||||
src_rank=0,
|
||||
device=self.worker.device,
|
||||
)
|
||||
num_moe_layers = physical_to_logical.shape[0]
|
||||
new_dp_size = get_dp_group().world_size
|
||||
tp_size = self.worker.vllm_config.parallel_config.tensor_parallel_size
|
||||
new_ep_size = new_dp_size * tp_size
|
||||
expanded_physical_to_logical = torch.full(
|
||||
(num_moe_layers, num_local_physical_experts * new_ep_size),
|
||||
-1,
|
||||
dtype=physical_to_logical.dtype,
|
||||
device=physical_to_logical.device,
|
||||
)
|
||||
expanded_physical_to_logical[:, : physical_to_logical.shape[1]] = (
|
||||
physical_to_logical
|
||||
)
|
||||
return expanded_physical_to_logical
|
||||
|
||||
def warmup_local_kernels(self) -> None:
|
||||
def warmup_new_worker(self) -> None:
|
||||
with set_current_vllm_config(self.worker.vllm_config):
|
||||
kernel_warmup(self.worker, process_local_only=True)
|
||||
if self._can_reuse_fused_moe_kernel():
|
||||
self.warm_and_capture()
|
||||
|
||||
def warm_and_capture(self) -> None:
|
||||
# Must run on every DP sibling in lockstep: _dummy_run calls
|
||||
# coordinate_batch_across_dp whenever data_parallel_size > 1
|
||||
# (gpu_model_runner.py:3663), which deadlocks if any rank skips it.
|
||||
|
||||
# Save and clear block tables so the dummy MoE forward doesn't
|
||||
# write dummy slot mappings into real KV-cache blocks.
|
||||
multi_block_table = self.worker.model_runner.input_batch.block_table
|
||||
@@ -735,8 +689,15 @@ class ElasticEPScalingExecutor:
|
||||
# directly with skip_eplb=True so dummy routing doesn't pollute the
|
||||
# just-rebalanced EPLB stats.
|
||||
runner = self.worker.model_runner
|
||||
runner._dummy_run(runner.max_num_tokens, is_profile=True, skip_eplb=True)
|
||||
self.worker.compile_or_warm_up_model()
|
||||
all2all_manager = get_ep_all2all_manager()
|
||||
reuse_kernel = self._can_reuse_fused_moe_kernel()
|
||||
with (
|
||||
skip_dp_coordination() if reuse_kernel else nullcontext(),
|
||||
all2all_manager.mask_remote_ranks() if reuse_kernel else nullcontext(),
|
||||
self._disable_flashinfer_autotune() if reuse_kernel else nullcontext(),
|
||||
):
|
||||
runner._dummy_run(runner.max_num_tokens, is_profile=True, skip_eplb=True)
|
||||
self.worker.compile_or_warm_up_model()
|
||||
|
||||
lock_workspace()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user