Fix Transformers backend FP8 MoE and remove some boilerplate (#46820)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-06-27 00:16:05 +01:00
committed by GitHub
parent 2ff76a5e85
commit d8eb734d94
29 changed files with 80 additions and 156 deletions
+12 -35
View File
@@ -28,14 +28,9 @@ from vllm.config.utils import getattr_iter
from vllm.distributed import get_dp_group, get_ep_group
from vllm.forward_context import ForwardContext, get_forward_context
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
MoERunner,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe import FusedMoE, MoERunner, RoutedExperts
from vllm.model_executor.models.interfaces import MixtureOfExperts
from vllm.model_executor.models.utils import maybe_prefix
from vllm.platforms import current_platform
from vllm.utils.torch_utils import direct_register_custom_op
from .utils import log_replacement
@@ -52,7 +47,7 @@ class TransformersMoEState:
# --8<-- [start:transformers_fused_moe]
@PluggableLayer.register("transformers_fused_moe")
class TransformersFusedMoE(MoERunner):
class TransformersMoERunner(MoERunner):
"""Custom FusedMoE for the Transformers modeling backend."""
# --8<-- [end:transformers_fused_moe]
@@ -93,7 +88,7 @@ class TransformersFusedMoE(MoERunner):
return self.routed_experts.load_weights(weights)
def transformers_moe_forward(
def _transformers_moe_forward(
hidden_states: torch.Tensor,
topk_ids: torch.Tensor,
topk_weights: torch.Tensor,
@@ -106,7 +101,7 @@ def transformers_moe_forward(
return self._forward_super(hidden_states, topk_weights)
def transformers_moe_forward_fake(
def _transformers_moe_forward_fake(
hidden_states: torch.Tensor,
topk_ids: torch.Tensor,
topk_weights: torch.Tensor,
@@ -117,10 +112,9 @@ def transformers_moe_forward_fake(
direct_register_custom_op(
op_name="transformers_moe_forward",
op_func=transformers_moe_forward,
op_func=_transformers_moe_forward,
mutates_args=["hidden_states"],
fake_impl=transformers_moe_forward_fake,
dispatch_key=current_platform.dispatch_key,
fake_impl=_transformers_moe_forward_fake,
tags=(torch.Tag.needs_fixed_stride_order,),
)
@@ -131,20 +125,6 @@ class MoEMixin(MixtureOfExperts):
# Skip MixtureOfExperts.__init__ and call the next class in MRO
super(MixtureOfExperts, self).__init__(vllm_config=vllm_config, prefix=prefix)
def set_eplb_state(
self,
expert_load_view: torch.Tensor,
logical_to_physical_map: torch.Tensor,
logical_replica_count: torch.Tensor,
):
for moe_layer_idx, mlp_layer in enumerate(self.mlp_moe_layers):
mlp_layer.experts.set_eplb_state(
moe_layer_idx=moe_layer_idx,
expert_load_view=expert_load_view,
logical_to_physical_map=logical_to_physical_map,
logical_replica_count=logical_replica_count,
)
def update_physical_experts_metadata(
self,
num_physical_experts: int,
@@ -154,7 +134,7 @@ class MoEMixin(MixtureOfExperts):
self.num_physical_experts = num_physical_experts
self.num_local_physical_experts = num_local_physical_experts
self.num_redundant_experts = num_physical_experts - self.num_logical_experts
for mlp in self.mlp_moe_layers:
for mlp in self.mlp_layers:
mlp.n_local_physical_experts = num_local_physical_experts
mlp.n_physical_experts = num_physical_experts
mlp.n_redundant_experts = self.num_redundant_experts
@@ -185,7 +165,7 @@ class MoEMixin(MixtureOfExperts):
num_redundant_experts = self.parallel_config.eplb_config.num_redundant_experts
for gate_proj, down_proj, up_proj in ckpt_names:
expert_mapping.extend(
fused_moe_make_expert_params_mapping(
RoutedExperts.make_expert_params_mapping(
self,
ckpt_gate_proj_name=gate_proj,
ckpt_down_proj_name=down_proj,
@@ -248,10 +228,8 @@ class MoEMixin(MixtureOfExperts):
# MixtureOfExperts mixin settings
ep_size = get_ep_group().world_size
self.mlp_moe_layers = [] # Used for MixtureOfExperts methods
self.mlp_layers = [] # Used for MixtureOfExperts methods
self.moe_layers = []
self.expert_weights = []
self.num_moe_layers = 0
self.num_expert_groups = 1 if num_expert_group is None else num_expert_group
self.num_logical_experts = num_experts
self.num_physical_experts = num_experts + num_redundant_experts
@@ -335,19 +313,18 @@ class MoEMixin(MixtureOfExperts):
custom_routing_function,
moe_state=moe_state,
),
runner_cls=TransformersFusedMoE,
runner_cls=TransformersMoERunner,
runner_args={"moe_state": moe_state},
)
mlp.experts = fused_experts
log_replacement(qual_name, experts, fused_experts)
# Update MixtureOfExperts mixin state
self.mlp_moe_layers.append(mlp)
self.mlp_layers.append(mlp)
self.moe_layers.append(fused_experts)
self.expert_weights.append(fused_experts.get_expert_weights())
self.num_moe_layers += 1
else:
_recursive_replace(child_module, prefix=qual_name)
_recursive_replace(self.model, prefix="model")
self.num_moe_layers = len(self.moe_layers)
# Continue with the replacement of layers in Base
super().recursive_replace()