[MoE Refactor] FusedMoE/MoERunner inversion refactor (#41184)

Signed-off-by: Bill Nell <bnell@redhat.com>
This commit is contained in:
bnellnm
2026-06-08 10:42:58 -04:00
committed by GitHub
parent 753e9d55e6
commit dc68bd8c41
90 changed files with 2738 additions and 2031 deletions
+2 -1
View File
@@ -20,6 +20,7 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
MoERunner,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -640,7 +641,7 @@ class AfmoeForCausalLM(nn.Module, SupportsPP, SupportsEagle3, SupportsLoRA):
self.num_moe_layers = config.num_hidden_layers - config.num_dense_layers
self.num_expert_groups = config.n_group
self.moe_layers: list[FusedMoE] = []
self.moe_layers: list[MoERunner] = []
example_moe = None
for layer in self.model.layers:
if isinstance(layer, PPMissingLayer):
+12 -9
View File
@@ -16,6 +16,7 @@ from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
RoutedExperts,
)
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -216,7 +217,7 @@ class AriaProjector(nn.Module):
return out
class AriaFusedMoE(FusedMoE):
class AriaRoutedExperts(RoutedExperts):
def weight_loader(
self, param: nn.Parameter, loaded_weight: torch.Tensor, shard_id: str
) -> None:
@@ -225,13 +226,14 @@ class AriaFusedMoE(FusedMoE):
# up weights for each expert.
# Note: Loading expert weights with quantization is not supported
tp_rank = get_tensor_model_parallel_rank()
tp_size = self.moe_config.tp_size
if shard_id == "w13":
# the shape of loaded_weight is
# (num_experts, hidden_size, 2 * moe_intermediate_size)
if self.tp_size > 1:
if tp_size > 1:
up, gate = loaded_weight.chunk(2, dim=-1)
up_current_rank = up.chunk(self.tp_size, dim=-1)[tp_rank]
gate_current_rank = gate.chunk(self.tp_size, dim=-1)[tp_rank]
up_current_rank = up.chunk(tp_size, dim=-1)[tp_rank]
gate_current_rank = gate.chunk(tp_size, dim=-1)[tp_rank]
up_and_gate = torch.cat(
[up_current_rank, gate_current_rank], dim=-1
).transpose(1, 2)
@@ -241,8 +243,8 @@ class AriaFusedMoE(FusedMoE):
elif shard_id == "w2":
# the shape of loaded_weight is
# (num_experts, moe_intermediate_size, hidden_size)
if self.tp_size > 1:
down_current_rank = loaded_weight.chunk(self.tp_size, dim=1)[tp_rank]
if tp_size > 1:
down_current_rank = loaded_weight.chunk(tp_size, dim=1)[tp_rank]
param.data.copy_(down_current_rank.transpose(1, 2))
else:
param.data.copy_(loaded_weight.transpose(1, 2))
@@ -278,7 +280,7 @@ class AriaTextMoELayer(nn.Module):
bias=config.mlp_bias,
)
self.experts = AriaFusedMoE(
self.experts = FusedMoE(
shared_experts=self.shared_experts,
num_experts=config.moe_num_experts,
top_k=config.moe_topk,
@@ -286,6 +288,7 @@ class AriaTextMoELayer(nn.Module):
intermediate_size=config.intermediate_size,
quant_config=quant_config,
prefix=f"{prefix}.experts",
routed_experts_cls=AriaRoutedExperts,
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -351,8 +354,8 @@ class AriaTextModel(LlamaModel, SupportsQuant):
(".qkv_proj", ".v_proj", "v"),
(".gate_up_proj", ".gate_proj", 0),
(".gate_up_proj", ".up_proj", 1),
("experts.w13_weight", "experts.fc1.weight", "w13"),
("experts.w2_weight", "experts.fc2.weight", "w2"),
("experts.routed_experts.w13_weight", "experts.fc1.weight", "w13"),
("experts.routed_experts.w2_weight", "experts.fc2.weight", "w2"),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
@@ -283,7 +283,7 @@ class BailingMoeV25(nn.Module):
self.hidden_size = config.hidden_size
self.quant_config = quant_config
self.num_shared_experts = config.num_shared_experts
self.score_function = getattr(config, "score_function", None)
self.score_function: str | None = getattr(config, "score_function", None)
self.n_group = getattr(config, "n_group", None)
self.topk_group = getattr(config, "topk_group", None)
self.use_grouped_topk = self.n_group is not None and self.topk_group is not None
+17 -19
View File
@@ -17,6 +17,7 @@ from vllm.distributed import (
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
RoutedExperts,
)
from vllm.model_executor.layers.linear import (
QKVParallelLinear,
@@ -73,27 +74,17 @@ class DbrxRouter(nn.Module):
return router_logits
class DbrxExperts(FusedMoE):
class DbrxExperts(RoutedExperts):
def __init__(
self,
*args,
config: DbrxConfig,
quant_config: QuantizationConfig | None = None,
params_dtype: torch.dtype | None = None,
prefix: str = "",
**kwargs,
):
super().__init__(
num_experts=config.ffn_config.moe_num_experts,
top_k=config.ffn_config.moe_top_k,
hidden_size=config.d_model,
intermediate_size=config.ffn_config.ffn_hidden_size,
params_dtype=params_dtype,
renormalize=True,
quant_config=quant_config,
tp_size=get_tensor_model_parallel_world_size(),
prefix=prefix,
)
super().__init__(*args, **kwargs)
self.config = config
self.d_model = config.d_model
self.tp_size = self.moe_config.tp_size
self.intermediate_size = self.config.ffn_config.ffn_hidden_size // self.tp_size
# Define custom weight loader for dbrx model
@@ -168,11 +159,18 @@ class DbrxMoE(nn.Module):
self.router = DbrxRouter(config, self.params_dtype)
self.experts = DbrxExperts(
config=config,
quant_config=quant_config,
self.experts = FusedMoE(
num_experts=config.ffn_config.moe_num_experts,
top_k=config.ffn_config.moe_top_k,
hidden_size=config.d_model,
intermediate_size=config.ffn_config.ffn_hidden_size,
params_dtype=self.params_dtype,
prefix=f"{prefix}.experts",
renormalize=True,
quant_config=quant_config,
tp_size=get_tensor_model_parallel_world_size(),
prefix=prefix,
routed_experts_cls=DbrxExperts,
routed_experts_args={"config": config},
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+2 -1
View File
@@ -44,6 +44,7 @@ from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
MoERunner,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -670,7 +671,7 @@ class Ernie4_5_MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA, MixtureOfExpe
self.num_moe_layers = len(moe_layers_indices)
self.num_expert_groups = 1
self.moe_layers: list[FusedMoE] = []
self.moe_layers: list[MoERunner] = []
example_moe = None
for layer in self.model.layers:
if isinstance(layer, PPMissingLayer):
+8 -2
View File
@@ -697,12 +697,18 @@ class Ernie4_5_VLMoeForCausalLM(nn.Module, SupportsPP):
moe_offset = int(name.split(".")[-3])
vision_expert_start_idx = self.config.moe_num_experts[0]
is_text_expert = moe_offset <= vision_expert_start_idx - 1
routed_experts = (
".routed_experts" if ("w13_" in name or "w2_" in name) else ""
)
if is_text_expert:
name = name.replace(".experts.", ".text_experts.")
name = name.replace(
".experts", f".text_experts{routed_experts}"
)
else:
delta = moe_offset - vision_expert_start_idx
name = name.replace(
f".experts.{moe_offset}",
f".vision_experts.{moe_offset - vision_expert_start_idx}",
f".vision_experts{routed_experts}.{delta}",
)
for mapping in expert_params_mapping:
+4
View File
@@ -1486,6 +1486,10 @@ class Gemma4Model(nn.Module, EagleModelMixin):
continue
if is_pp_missing_parameter(name, self):
continue
# Skip if name doesn't exist in params_dict (e.g., individual
# expert weights that should have been handled above)
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
@@ -33,7 +33,7 @@ from transformers import PretrainedConfig
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import VllmConfig
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
MoERunner,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -215,7 +215,7 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts):
self.num_moe_layers = self.config.num_nextn_predict_layers
self.num_expert_groups = self.config.n_group
self.moe_layers: list[FusedMoE] = []
self.moe_layers: list[MoERunner] = []
self.moe_mlp_layers: list[Glm4MoeLite] = []
example_moe = None
for layer in self.model.layers.values():
+2 -2
View File
@@ -32,7 +32,7 @@ from transformers import PretrainedConfig
from vllm.config import CacheConfig, ParallelConfig, VllmConfig
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
MoERunner,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -201,7 +201,7 @@ class Glm4MoeMTP(nn.Module, Glm4MixtureOfExperts):
self.num_moe_layers = self.config.num_nextn_predict_layers
self.num_expert_groups = self.config.n_group
self.moe_layers: list[FusedMoE] = []
self.moe_layers: list[MoERunner] = []
self.moe_mlp_layers: list[Glm4MoE] = []
example_moe = None
for layer in self.model.layers.values():
+5 -3
View File
@@ -43,6 +43,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
remap_moe_expert_weights,
)
from vllm.model_executor.models.utils import sequence_parallel_chunk
from vllm.platforms import current_platform
@@ -379,7 +380,8 @@ class GptOssModel(nn.Module, EagleModelMixin):
tp_rank_start = tp_rank * per_rank_intermediate_size
tp_rank_end = min((tp_rank + 1) * per_rank_intermediate_size, intermediate_size)
for name, weight in weights:
# Use centralized weight remapping for MoE expert parameters
for name, weight in remap_moe_expert_weights(weights, params_dict):
# Skip layers on other devices.
if is_pp_missing_parameter(name, self):
continue
@@ -580,8 +582,8 @@ class GptOssModel(nn.Module, EagleModelMixin):
Returns:
Weight dtype string (e.g., "mxfp4", "fp8") or None if not available
"""
if hasattr(self.layers[layer_id].mlp.experts.quant_method, "weight_dtype"):
return self.layers[layer_id].mlp.experts.quant_method.weight_dtype
if hasattr(self.layers[layer_id].mlp.experts._quant_method, "weight_dtype"):
return self.layers[layer_id].mlp.experts._quant_method.weight_dtype
return None
intermediate_size = self.config.intermediate_size
@@ -455,6 +455,8 @@ class GraniteMoeHybridModel(nn.Module):
loaded_params.add(n)
def _load_expert(n, p, name, shard_id, expert_id):
if n not in params_dict:
return
param = params_dict[n]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, p, name, shard_id=shard_id, expert_id=expert_id)
@@ -518,14 +520,14 @@ class GraniteMoeHybridModel(nn.Module):
)
w1_param, w3_param = p[e].chunk(2, dim=0)
_load_expert(
n.replace(".input_linear.", ".experts.w13_"),
n.replace(".input_linear.", ".experts.routed_experts.w13_"),
w1_param,
w1_name,
shard_id="w1",
expert_id=e,
)
_load_expert(
n.replace(".input_linear.", ".experts.w13_"),
n.replace(".input_linear.", ".experts.routed_experts.w13_"),
w3_param,
w3_name,
shard_id="w3",
@@ -541,7 +543,7 @@ class GraniteMoeHybridModel(nn.Module):
)
w2_param = p[e]
_load_expert(
n.replace(".output_linear.", ".experts.w2_"),
n.replace(".output_linear.", ".experts.routed_experts.w2_"),
w2_param,
w2_name,
shard_id="w2",
@@ -214,11 +214,11 @@ class GraniteMoeSharedModel(nn.Module):
for e in range(p.size(0)):
w1_name = n.replace(
".block_sparse_moe.input_linear.weight",
f".block_sparse_moe.experts.{e}.w1.weight",
f".block_sparse_moe.experts.routed_experts.{e}.w1.weight",
)
w3_name = n.replace(
".block_sparse_moe.input_linear.weight",
f".block_sparse_moe.experts.{e}.w3.weight",
f".block_sparse_moe.experts.routed_experts.{e}.w3.weight",
)
w1_param, w3_param = p[e].chunk(2, dim=0)
assert w1_name not in new_weights
@@ -229,7 +229,7 @@ class GraniteMoeSharedModel(nn.Module):
for e in range(p.size(0)):
w2_name = n.replace(
".block_sparse_moe.output_linear.weight",
f".block_sparse_moe.experts.{e}.w2.weight",
f".block_sparse_moe.experts.routed_experts.{e}.w2.weight",
)
w2_param = p[e]
assert w2_name not in new_weights
-1
View File
@@ -229,7 +229,6 @@ class NemotronHMoE(nn.Module):
scoring_func="sigmoid",
e_score_correction_bias=self.gate.e_score_correction_bias,
activation=activation_without_mul(config.mlp_hidden_act),
is_act_and_mul=False, # non-gated MoE
enable_eplb=self.enable_eplb,
num_redundant_experts=self.n_redundant_experts,
is_sequence_parallel=self.is_sequence_parallel,
+2 -1
View File
@@ -37,6 +37,7 @@ from vllm.distributed import (
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
MoERunner,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -355,7 +356,7 @@ class SarvamMLAMoE(nn.Module):
routed_scaling_factor=self.routed_scaling_factor,
)
def maybe_get_fused_moe(self) -> FusedMoE:
def maybe_get_fused_moe(self) -> MoERunner:
return self.experts
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+2 -1
View File
@@ -25,6 +25,7 @@ from vllm.model_executor.layers.activation import SiluAndMul, SwigluStepAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
MoERunner,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
@@ -954,7 +955,7 @@ class Step3p5ForCausalLM(nn.Module, SupportsPP, MixtureOfExperts):
) -> None:
for layer_idx, layer in enumerate(self.moe_layers):
experts = layer.experts
assert isinstance(experts, FusedMoE)
assert isinstance(experts, MoERunner)
# Register the expert weights.
self.expert_weights.append(experts.get_expert_weights())
experts.set_eplb_state(
+63 -27
View File
@@ -16,6 +16,9 @@
# limitations under the License.
"""Transformers modeling backend mixin for Mixture of Experts (MoE) models."""
from collections.abc import Iterable
from dataclasses import dataclass
from functools import partial
from typing import TYPE_CHECKING, Any
import torch
@@ -27,6 +30,7 @@ from vllm.forward_context import ForwardContext, get_forward_context
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
MoERunner,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.models.interfaces import MixtureOfExperts
@@ -40,33 +44,22 @@ if TYPE_CHECKING:
from vllm.config import VllmConfig
@dataclass
class TransformersMoEState:
topk_ids: torch.Tensor | None = None
is_sequence_parallel: bool = False
# --8<-- [start:transformers_fused_moe]
@PluggableLayer.register("transformers_fused_moe")
class TransformersFusedMoE(FusedMoE):
class TransformersFusedMoE(MoERunner):
"""Custom FusedMoE for the Transformers modeling backend."""
# --8<-- [end:transformers_fused_moe]
def __init__(self, *args, **kwargs):
self._topk_ids: torch.Tensor = None
def custom_routing_function(hidden_states, gating_output, topk, renormalize):
"""Return `topk_weights` from `gating_output` and the
`topk_ids` we stored in the layer earlier."""
topk_weights = gating_output
topk_ids = self._topk_ids
# Handle all gather in expert parallel
if topk_ids.size(0) != hidden_states.size(0):
dp_metadata = get_forward_context().dp_metadata
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
is_sp = self.is_sequence_parallel
dist_group = get_ep_group() if is_sp else get_dp_group()
assert sizes[dist_group.rank_in_group] == topk_ids.shape[0]
(topk_ids,) = dist_group.all_gatherv([topk_ids], 0, sizes)
return topk_weights, topk_ids
kwargs["custom_routing_function"] = custom_routing_function
def __init__(self, *args, moe_state: TransformersMoEState, **kwargs):
super().__init__(*args, **kwargs)
self._moe_state = moe_state
self._moe_state.is_sequence_parallel = self.moe_config.is_sequence_parallel
def forward(
self,
@@ -78,6 +71,8 @@ class TransformersFusedMoE(FusedMoE):
"""In Transformers `experts.forward` will have this signature.
We discard any extra kwargs because we cannot use them here."""
# Note: we need to forward through a custom op so the topk_ids
# can be transferred without interfering with cudagraphs.
return torch.ops.vllm.transformers_moe_forward(
hidden_states,
topk_ids.to(torch.int32),
@@ -85,6 +80,18 @@ class TransformersFusedMoE(FusedMoE):
self.layer_name,
)
def _forward_super(
self,
hidden_states: torch.Tensor,
topk_weights: torch.Tensor,
) -> torch.Tensor:
return super().forward(hidden_states, topk_weights)
def load_weights(
self, weights: Iterable[tuple[str, torch.Tensor]]
) -> Iterable[str]:
return self.routed_experts.load_weights(weights)
def transformers_moe_forward(
hidden_states: torch.Tensor,
@@ -95,11 +102,8 @@ def transformers_moe_forward(
"""Store the `topk_ids` in the layer and call the actual forward."""
forward_context: ForwardContext = get_forward_context()
self = forward_context.no_compile_layers[layer_name]
self._topk_ids = topk_ids
# Clone hidden_states because it will be mutated in-place in FusedMoE
# TODO(bnell): figure out a way to avoid calling runner directly.
# it is a hack that the weight are being passed via logits.
return self.runner.forward(hidden_states.clone(), topk_weights)
self._moe_state.topk_ids = topk_ids
return self._forward_super(hidden_states, topk_weights)
def transformers_moe_forward_fake(
@@ -189,6 +193,7 @@ class MoEMixin(MixtureOfExperts):
ckpt_up_proj_name=up_proj,
num_experts=num_experts,
num_redundant_experts=num_redundant_experts,
routed_experts_prefix="",
)
)
return expert_mapping
@@ -286,8 +291,33 @@ class MoEMixin(MixtureOfExperts):
if "shared_expert" in mlp_param_name:
self.num_shared_experts = 1
break
# Replace experts module with FusedMoE
fused_experts = TransformersFusedMoE(
moe_state = TransformersMoEState()
def custom_routing_function(
hidden_states: torch.Tensor,
gating_output: torch.Tensor,
topk: int,
renormalize: bool,
moe_state: TransformersMoEState,
):
"""Return `topk_weights` from `gating_output` and the
`topk_ids` we stored in the layer earlier."""
topk_weights = gating_output
topk_ids = moe_state.topk_ids
assert topk_ids is not None
# Handle all gather in expert parallel
if topk_ids.size(0) != hidden_states.size(0):
dp_metadata = get_forward_context().dp_metadata
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
is_sp = moe_state.is_sequence_parallel
dist_group = get_ep_group() if is_sp else get_dp_group()
assert sizes[dist_group.rank_in_group] == topk_ids.shape[0]
(topk_ids,) = dist_group.all_gatherv([topk_ids], 0, sizes)
return topk_weights, topk_ids
fused_experts = FusedMoE(
num_experts=num_experts,
top_k=top_k,
hidden_size=hidden_size,
@@ -304,6 +334,12 @@ class MoEMixin(MixtureOfExperts):
num_redundant_experts=num_redundant_experts,
has_bias=has_bias,
expert_mapping=expert_mapping,
custom_routing_function=partial(
custom_routing_function,
moe_state=moe_state,
),
runner_cls=TransformersFusedMoE,
runner_args={"moe_state": moe_state},
)
mlp.experts = fused_experts
log_replacement(qual_name, experts, fused_experts)