Fix Transformers backend FP8 MoE and remove some boilerplate (#46820)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -28,14 +28,9 @@ from vllm.config.utils import getattr_iter
|
||||
from vllm.distributed import get_dp_group, get_ep_group
|
||||
from vllm.forward_context import ForwardContext, get_forward_context
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
MoERunner,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE, MoERunner, RoutedExperts
|
||||
from vllm.model_executor.models.interfaces import MixtureOfExperts
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
from .utils import log_replacement
|
||||
@@ -52,7 +47,7 @@ class TransformersMoEState:
|
||||
|
||||
# --8<-- [start:transformers_fused_moe]
|
||||
@PluggableLayer.register("transformers_fused_moe")
|
||||
class TransformersFusedMoE(MoERunner):
|
||||
class TransformersMoERunner(MoERunner):
|
||||
"""Custom FusedMoE for the Transformers modeling backend."""
|
||||
|
||||
# --8<-- [end:transformers_fused_moe]
|
||||
@@ -93,7 +88,7 @@ class TransformersFusedMoE(MoERunner):
|
||||
return self.routed_experts.load_weights(weights)
|
||||
|
||||
|
||||
def transformers_moe_forward(
|
||||
def _transformers_moe_forward(
|
||||
hidden_states: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
@@ -106,7 +101,7 @@ def transformers_moe_forward(
|
||||
return self._forward_super(hidden_states, topk_weights)
|
||||
|
||||
|
||||
def transformers_moe_forward_fake(
|
||||
def _transformers_moe_forward_fake(
|
||||
hidden_states: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
@@ -117,10 +112,9 @@ def transformers_moe_forward_fake(
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="transformers_moe_forward",
|
||||
op_func=transformers_moe_forward,
|
||||
op_func=_transformers_moe_forward,
|
||||
mutates_args=["hidden_states"],
|
||||
fake_impl=transformers_moe_forward_fake,
|
||||
dispatch_key=current_platform.dispatch_key,
|
||||
fake_impl=_transformers_moe_forward_fake,
|
||||
tags=(torch.Tag.needs_fixed_stride_order,),
|
||||
)
|
||||
|
||||
@@ -131,20 +125,6 @@ class MoEMixin(MixtureOfExperts):
|
||||
# Skip MixtureOfExperts.__init__ and call the next class in MRO
|
||||
super(MixtureOfExperts, self).__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
|
||||
def set_eplb_state(
|
||||
self,
|
||||
expert_load_view: torch.Tensor,
|
||||
logical_to_physical_map: torch.Tensor,
|
||||
logical_replica_count: torch.Tensor,
|
||||
):
|
||||
for moe_layer_idx, mlp_layer in enumerate(self.mlp_moe_layers):
|
||||
mlp_layer.experts.set_eplb_state(
|
||||
moe_layer_idx=moe_layer_idx,
|
||||
expert_load_view=expert_load_view,
|
||||
logical_to_physical_map=logical_to_physical_map,
|
||||
logical_replica_count=logical_replica_count,
|
||||
)
|
||||
|
||||
def update_physical_experts_metadata(
|
||||
self,
|
||||
num_physical_experts: int,
|
||||
@@ -154,7 +134,7 @@ class MoEMixin(MixtureOfExperts):
|
||||
self.num_physical_experts = num_physical_experts
|
||||
self.num_local_physical_experts = num_local_physical_experts
|
||||
self.num_redundant_experts = num_physical_experts - self.num_logical_experts
|
||||
for mlp in self.mlp_moe_layers:
|
||||
for mlp in self.mlp_layers:
|
||||
mlp.n_local_physical_experts = num_local_physical_experts
|
||||
mlp.n_physical_experts = num_physical_experts
|
||||
mlp.n_redundant_experts = self.num_redundant_experts
|
||||
@@ -185,7 +165,7 @@ class MoEMixin(MixtureOfExperts):
|
||||
num_redundant_experts = self.parallel_config.eplb_config.num_redundant_experts
|
||||
for gate_proj, down_proj, up_proj in ckpt_names:
|
||||
expert_mapping.extend(
|
||||
fused_moe_make_expert_params_mapping(
|
||||
RoutedExperts.make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name=gate_proj,
|
||||
ckpt_down_proj_name=down_proj,
|
||||
@@ -248,10 +228,8 @@ class MoEMixin(MixtureOfExperts):
|
||||
# MixtureOfExperts mixin settings
|
||||
ep_size = get_ep_group().world_size
|
||||
|
||||
self.mlp_moe_layers = [] # Used for MixtureOfExperts methods
|
||||
self.mlp_layers = [] # Used for MixtureOfExperts methods
|
||||
self.moe_layers = []
|
||||
self.expert_weights = []
|
||||
self.num_moe_layers = 0
|
||||
self.num_expert_groups = 1 if num_expert_group is None else num_expert_group
|
||||
self.num_logical_experts = num_experts
|
||||
self.num_physical_experts = num_experts + num_redundant_experts
|
||||
@@ -335,19 +313,18 @@ class MoEMixin(MixtureOfExperts):
|
||||
custom_routing_function,
|
||||
moe_state=moe_state,
|
||||
),
|
||||
runner_cls=TransformersFusedMoE,
|
||||
runner_cls=TransformersMoERunner,
|
||||
runner_args={"moe_state": moe_state},
|
||||
)
|
||||
mlp.experts = fused_experts
|
||||
log_replacement(qual_name, experts, fused_experts)
|
||||
# Update MixtureOfExperts mixin state
|
||||
self.mlp_moe_layers.append(mlp)
|
||||
self.mlp_layers.append(mlp)
|
||||
self.moe_layers.append(fused_experts)
|
||||
self.expert_weights.append(fused_experts.get_expert_weights())
|
||||
self.num_moe_layers += 1
|
||||
else:
|
||||
_recursive_replace(child_module, prefix=qual_name)
|
||||
|
||||
_recursive_replace(self.model, prefix="model")
|
||||
self.num_moe_layers = len(self.moe_layers)
|
||||
# Continue with the replacement of layers in Base
|
||||
super().recursive_replace()
|
||||
|
||||
Reference in New Issue
Block a user