elastic_ep: stage/commit MoE quant method on reconfigure (#40881)

Signed-off-by: Itay Alroy <ialroy@nvidia.com>
This commit is contained in:
Itay Alroy
2026-05-22 18:57:26 -04:00
committed by GitHub
parent 8de5cabeb7
commit 6d30655b13
7 changed files with 351 additions and 59 deletions
@@ -49,6 +49,22 @@ if current_platform.is_cuda_alike():
)
def _get_ep_all2all_manager(eep_stage: bool = False) -> Any:
if eep_stage:
from vllm.distributed.elastic_ep.standby_state import get_standby_ep_group
ep_group = get_standby_ep_group()
assert ep_group is not None
device_communicator = ep_group.device_communicator
else:
device_communicator = get_ep_group().device_communicator
assert device_communicator is not None
all2all_manager = device_communicator.all2all_manager
assert all2all_manager is not None
return all2all_manager
def maybe_roundup_layer_hidden_size(
hidden_size: int,
act_dtype: torch.dtype,
@@ -92,6 +108,7 @@ def maybe_make_prepare_finalize(
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
allow_new_interface: bool = False,
use_monolithic: bool = False,
eep_stage: bool = False,
) -> FusedMoEPrepareAndFinalize | None:
# NOTE(rob): we are migrating each quant_method to hold the MK
# in all cases. The allow_new_interface=False flag allow us to fall
@@ -117,21 +134,16 @@ def maybe_make_prepare_finalize(
"Detected DP deployment with no --enable-expert-parallel. "
"Falling back to AllGather+ReduceScatter dispatch/combine."
)
device_communicator = get_ep_group().device_communicator
assert device_communicator is not None
assert device_communicator.all2all_manager is not None
all2all_manager = _get_ep_all2all_manager(eep_stage)
return make_moe_prepare_and_finalize_naive_dp_ep(
is_sequence_parallel=moe.moe_parallel_config.is_sequence_parallel,
num_dispatchers=(device_communicator.all2all_manager.world_size),
num_dispatchers=all2all_manager.world_size,
use_monolithic=use_monolithic,
)
else:
return make_moe_prepare_and_finalize_no_dp_ep(use_monolithic)
device_communicator = get_ep_group().device_communicator
assert device_communicator is not None
all2all_manager = device_communicator.all2all_manager
assert all2all_manager is not None
all2all_manager = _get_ep_all2all_manager(eep_stage)
prepare_finalize: FusedMoEPrepareAndFinalize | None = None
@@ -283,6 +295,7 @@ def maybe_make_prepare_finalize(
num_ep_ranks=all2all_manager.world_size,
num_global_experts=moe.num_experts,
num_local_experts=moe.num_experts // all2all_manager.world_size,
stage=eep_stage,
)
handle = all2all_manager.get_handle(all_to_all_args)
@@ -0,0 +1,123 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from inspect import signature
from typing import TYPE_CHECKING, Any
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.model_executor.layers.fused_moe.all2all_utils import (
maybe_make_prepare_finalize,
)
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
FusedMoEMethodBase,
)
from vllm.model_executor.layers.fused_moe.fused_moe_modular_method import (
FusedMoEModularMethod,
)
from vllm.model_executor.layers.fused_moe.modular_kernel import (
FusedMoEExpertsModular,
FusedMoEPrepareAndFinalizeModular,
)
if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
def _make_eep_experts(
quant_method: FusedMoEMethodBase,
source_experts: FusedMoEExpertsModular,
prepare_finalize: FusedMoEPrepareAndFinalizeModular,
moe_config: FusedMoEConfig,
) -> FusedMoEExpertsModular:
experts_cls = source_experts.__class__
assert quant_method.moe_quant_config is not None
experts_kwargs: dict[str, Any] = {
"moe_config": moe_config,
"quant_config": quant_method.moe_quant_config,
}
if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
max_num_tokens = prepare_finalize.max_num_tokens_per_rank()
assert max_num_tokens is not None
experts_kwargs.update(
max_num_tokens=max_num_tokens,
num_dispatchers=prepare_finalize.num_dispatchers(),
)
# Expert kernels with extra init params need explicit EEP support.
generic_arg_names = set(signature(mk.FusedMoEExperts.__init__).parameters)
ctor_arg_names = set(signature(experts_cls.__init__).parameters)
unsupported_args = ctor_arg_names - generic_arg_names
missing_args = set(experts_kwargs) - ctor_arg_names
if unsupported_args or missing_args:
raise NotImplementedError(
f"{experts_cls.__name__} experts do not support Elastic EP."
)
return experts_cls(**experts_kwargs)
def make_eep_staged_quant_method(
module: "FusedMoE",
moe_config: FusedMoEConfig,
) -> FusedMoEMethodBase | None:
quant_method = module.quant_method
if not quant_method.supports_internal_mk:
return None
if getattr(quant_method, "wraps_legacy_quant_method", False):
return None
old_batched_format = (
module.moe_config.moe_parallel_config.use_batched_activation_format
)
new_batched_format = moe_config.moe_parallel_config.use_batched_activation_format
assert old_batched_format == new_batched_format
moe_kernel = quant_method.moe_kernel
if moe_kernel is None:
return None
if moe_kernel.is_monolithic:
raise NotImplementedError(
"Elastic EP full modular-kernel staging is not supported for "
"monolithic fused MoE kernels."
)
if quant_method.moe_quant_config is None:
raise ValueError(
"Elastic EP full modular-kernel staging requires initialized "
"MoE quant config."
)
prepare_finalize = maybe_make_prepare_finalize(
moe_config,
quant_method.moe_quant_config,
routing_tables=None,
allow_new_interface=True,
use_monolithic=quant_method.is_monolithic,
eep_stage=True,
)
assert prepare_finalize is not None
assert isinstance(prepare_finalize, FusedMoEPrepareAndFinalizeModular)
source_experts = moe_kernel.fused_experts
assert isinstance(source_experts, FusedMoEExpertsModular)
experts = _make_eep_experts(
quant_method,
source_experts,
prepare_finalize,
moe_config,
)
if isinstance(quant_method, FusedMoEModularMethod):
base_quant_method = quant_method.old_quant_method
else:
base_quant_method = quant_method
return FusedMoEModularMethod(
base_quant_method,
mk.FusedMoEKernel(
prepare_finalize,
experts,
inplace=moe_kernel.inplace,
),
)
@@ -31,7 +31,7 @@ class FusedMoEModularMethod(FusedMoEMethodBase, CustomOp):
def __init__(
self, old_quant_method: FusedMoEMethodBase, moe_kernel: FusedMoEKernel
):
super().__init__(old_quant_method.moe)
super().__init__(moe_kernel.moe_config)
self.moe_quant_config = old_quant_method.moe_quant_config
self.moe_kernel = moe_kernel
self.disable_expert_map = getattr(
@@ -42,6 +42,10 @@ class FusedMoEModularMethod(FusedMoEMethodBase, CustomOp):
self.old_quant_method = old_quant_method
logger.debug("Swapping out %s", self.old_quant_method.__class__.__name__)
@property
def wraps_legacy_quant_method(self) -> bool:
return not self.old_quant_method.supports_internal_mk
@staticmethod
def make(
moe_layer: torch.nn.Module,
@@ -246,6 +246,13 @@ class FusedMoEPrepareAndFinalize(ABC):
"""
return False
def on_commit(self) -> None:
"""
Runs after this prepare/finalize has been committed to the active
MoE kernel.
"""
return
# TODO: pass FusedMoEParallelConfig in as ctor parameter?
class FusedMoEPrepareAndFinalizeModular(FusedMoEPrepareAndFinalize):
@@ -1542,6 +1549,12 @@ class FusedMoEKernel:
else:
return False
@property
def inplace(self) -> bool:
if isinstance(self.impl, FusedMoEKernelModularImpl):
return self.impl.inplace
return False
@property
def is_monolithic(self) -> bool:
return isinstance(self.impl, FusedMoEKernelMonolithicImpl)
@@ -1554,6 +1567,10 @@ class FusedMoEKernel:
def fused_experts(self) -> FusedMoEExperts:
return self.impl.fused_experts
@property
def moe_config(self) -> FusedMoEConfig:
return self.fused_experts.moe_config
def supports_lora(self) -> bool:
return self.fused_experts.supports_lora()
@@ -7,6 +7,8 @@ import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm import envs
from vllm.distributed import get_ep_group
from vllm.distributed.device_communicators.all2all import NixlEPAll2AllManager
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
@@ -138,6 +140,17 @@ class NixlEPPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
def max_num_tokens_per_rank(self) -> int | None:
return self.max_tokens_per_rank
def on_commit(self) -> None:
device_communicator = get_ep_group().device_communicator
assert device_communicator is not None
all2all_manager = device_communicator.all2all_manager
assert isinstance(all2all_manager, NixlEPAll2AllManager)
# maybe_make_prepare_finalize(..., eep_stage=True) initializes self.buffer
# with get_handle(..., stage=True), which stages global NIXL state for the
# new config but leaves it inactive while the old config remains active.
# When EEP commit switches to this P/F, this P/F needs to commit that state.
all2all_manager.commit_staged_state()
def topk_indices_dtype(self) -> torch.dtype | None:
return torch.int64