[MoE Refactor] FusedMoE/MoERunner inversion refactor (#41184)

Signed-off-by: Bill Nell <bnell@redhat.com>
This commit is contained in:
bnellnm
2026-06-08 10:42:58 -04:00
committed by GitHub
parent 753e9d55e6
commit dc68bd8c41
90 changed files with 2738 additions and 2031 deletions
+63 -27
View File
@@ -16,6 +16,9 @@
# limitations under the License.
"""Transformers modeling backend mixin for Mixture of Experts (MoE) models."""
from collections.abc import Iterable
from dataclasses import dataclass
from functools import partial
from typing import TYPE_CHECKING, Any
import torch
@@ -27,6 +30,7 @@ from vllm.forward_context import ForwardContext, get_forward_context
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
MoERunner,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.models.interfaces import MixtureOfExperts
@@ -40,33 +44,22 @@ if TYPE_CHECKING:
from vllm.config import VllmConfig
@dataclass
class TransformersMoEState:
topk_ids: torch.Tensor | None = None
is_sequence_parallel: bool = False
# --8<-- [start:transformers_fused_moe]
@PluggableLayer.register("transformers_fused_moe")
class TransformersFusedMoE(FusedMoE):
class TransformersFusedMoE(MoERunner):
"""Custom FusedMoE for the Transformers modeling backend."""
# --8<-- [end:transformers_fused_moe]
def __init__(self, *args, **kwargs):
self._topk_ids: torch.Tensor = None
def custom_routing_function(hidden_states, gating_output, topk, renormalize):
"""Return `topk_weights` from `gating_output` and the
`topk_ids` we stored in the layer earlier."""
topk_weights = gating_output
topk_ids = self._topk_ids
# Handle all gather in expert parallel
if topk_ids.size(0) != hidden_states.size(0):
dp_metadata = get_forward_context().dp_metadata
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
is_sp = self.is_sequence_parallel
dist_group = get_ep_group() if is_sp else get_dp_group()
assert sizes[dist_group.rank_in_group] == topk_ids.shape[0]
(topk_ids,) = dist_group.all_gatherv([topk_ids], 0, sizes)
return topk_weights, topk_ids
kwargs["custom_routing_function"] = custom_routing_function
def __init__(self, *args, moe_state: TransformersMoEState, **kwargs):
super().__init__(*args, **kwargs)
self._moe_state = moe_state
self._moe_state.is_sequence_parallel = self.moe_config.is_sequence_parallel
def forward(
self,
@@ -78,6 +71,8 @@ class TransformersFusedMoE(FusedMoE):
"""In Transformers `experts.forward` will have this signature.
We discard any extra kwargs because we cannot use them here."""
# Note: we need to forward through a custom op so the topk_ids
# can be transferred without interfering with cudagraphs.
return torch.ops.vllm.transformers_moe_forward(
hidden_states,
topk_ids.to(torch.int32),
@@ -85,6 +80,18 @@ class TransformersFusedMoE(FusedMoE):
self.layer_name,
)
def _forward_super(
self,
hidden_states: torch.Tensor,
topk_weights: torch.Tensor,
) -> torch.Tensor:
return super().forward(hidden_states, topk_weights)
def load_weights(
self, weights: Iterable[tuple[str, torch.Tensor]]
) -> Iterable[str]:
return self.routed_experts.load_weights(weights)
def transformers_moe_forward(
hidden_states: torch.Tensor,
@@ -95,11 +102,8 @@ def transformers_moe_forward(
"""Store the `topk_ids` in the layer and call the actual forward."""
forward_context: ForwardContext = get_forward_context()
self = forward_context.no_compile_layers[layer_name]
self._topk_ids = topk_ids
# Clone hidden_states because it will be mutated in-place in FusedMoE
# TODO(bnell): figure out a way to avoid calling runner directly.
# it is a hack that the weight are being passed via logits.
return self.runner.forward(hidden_states.clone(), topk_weights)
self._moe_state.topk_ids = topk_ids
return self._forward_super(hidden_states, topk_weights)
def transformers_moe_forward_fake(
@@ -189,6 +193,7 @@ class MoEMixin(MixtureOfExperts):
ckpt_up_proj_name=up_proj,
num_experts=num_experts,
num_redundant_experts=num_redundant_experts,
routed_experts_prefix="",
)
)
return expert_mapping
@@ -286,8 +291,33 @@ class MoEMixin(MixtureOfExperts):
if "shared_expert" in mlp_param_name:
self.num_shared_experts = 1
break
# Replace experts module with FusedMoE
fused_experts = TransformersFusedMoE(
moe_state = TransformersMoEState()
def custom_routing_function(
hidden_states: torch.Tensor,
gating_output: torch.Tensor,
topk: int,
renormalize: bool,
moe_state: TransformersMoEState,
):
"""Return `topk_weights` from `gating_output` and the
`topk_ids` we stored in the layer earlier."""
topk_weights = gating_output
topk_ids = moe_state.topk_ids
assert topk_ids is not None
# Handle all gather in expert parallel
if topk_ids.size(0) != hidden_states.size(0):
dp_metadata = get_forward_context().dp_metadata
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
is_sp = moe_state.is_sequence_parallel
dist_group = get_ep_group() if is_sp else get_dp_group()
assert sizes[dist_group.rank_in_group] == topk_ids.shape[0]
(topk_ids,) = dist_group.all_gatherv([topk_ids], 0, sizes)
return topk_weights, topk_ids
fused_experts = FusedMoE(
num_experts=num_experts,
top_k=top_k,
hidden_size=hidden_size,
@@ -304,6 +334,12 @@ class MoEMixin(MixtureOfExperts):
num_redundant_experts=num_redundant_experts,
has_bias=has_bias,
expert_mapping=expert_mapping,
custom_routing_function=partial(
custom_routing_function,
moe_state=moe_state,
),
runner_cls=TransformersFusedMoE,
runner_args={"moe_state": moe_state},
)
mlp.experts = fused_experts
log_replacement(qual_name, experts, fused_experts)