[MoE Refactor] FusedMoE/MoERunner inversion refactor (#41184)
Signed-off-by: Bill Nell <bnell@redhat.com>
This commit is contained in:
@@ -16,6 +16,9 @@
|
||||
# limitations under the License.
|
||||
"""Transformers modeling backend mixin for Mixture of Experts (MoE) models."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
@@ -27,6 +30,7 @@ from vllm.forward_context import ForwardContext, get_forward_context
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
MoERunner,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import MixtureOfExperts
|
||||
@@ -40,33 +44,22 @@ if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
|
||||
@dataclass
|
||||
class TransformersMoEState:
|
||||
topk_ids: torch.Tensor | None = None
|
||||
is_sequence_parallel: bool = False
|
||||
|
||||
|
||||
# --8<-- [start:transformers_fused_moe]
|
||||
@PluggableLayer.register("transformers_fused_moe")
|
||||
class TransformersFusedMoE(FusedMoE):
|
||||
class TransformersFusedMoE(MoERunner):
|
||||
"""Custom FusedMoE for the Transformers modeling backend."""
|
||||
|
||||
# --8<-- [end:transformers_fused_moe]
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self._topk_ids: torch.Tensor = None
|
||||
|
||||
def custom_routing_function(hidden_states, gating_output, topk, renormalize):
|
||||
"""Return `topk_weights` from `gating_output` and the
|
||||
`topk_ids` we stored in the layer earlier."""
|
||||
topk_weights = gating_output
|
||||
topk_ids = self._topk_ids
|
||||
# Handle all gather in expert parallel
|
||||
if topk_ids.size(0) != hidden_states.size(0):
|
||||
dp_metadata = get_forward_context().dp_metadata
|
||||
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
|
||||
is_sp = self.is_sequence_parallel
|
||||
dist_group = get_ep_group() if is_sp else get_dp_group()
|
||||
assert sizes[dist_group.rank_in_group] == topk_ids.shape[0]
|
||||
(topk_ids,) = dist_group.all_gatherv([topk_ids], 0, sizes)
|
||||
return topk_weights, topk_ids
|
||||
|
||||
kwargs["custom_routing_function"] = custom_routing_function
|
||||
def __init__(self, *args, moe_state: TransformersMoEState, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self._moe_state = moe_state
|
||||
self._moe_state.is_sequence_parallel = self.moe_config.is_sequence_parallel
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -78,6 +71,8 @@ class TransformersFusedMoE(FusedMoE):
|
||||
"""In Transformers `experts.forward` will have this signature.
|
||||
|
||||
We discard any extra kwargs because we cannot use them here."""
|
||||
# Note: we need to forward through a custom op so the topk_ids
|
||||
# can be transferred without interfering with cudagraphs.
|
||||
return torch.ops.vllm.transformers_moe_forward(
|
||||
hidden_states,
|
||||
topk_ids.to(torch.int32),
|
||||
@@ -85,6 +80,18 @@ class TransformersFusedMoE(FusedMoE):
|
||||
self.layer_name,
|
||||
)
|
||||
|
||||
def _forward_super(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return super().forward(hidden_states, topk_weights)
|
||||
|
||||
def load_weights(
|
||||
self, weights: Iterable[tuple[str, torch.Tensor]]
|
||||
) -> Iterable[str]:
|
||||
return self.routed_experts.load_weights(weights)
|
||||
|
||||
|
||||
def transformers_moe_forward(
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -95,11 +102,8 @@ def transformers_moe_forward(
|
||||
"""Store the `topk_ids` in the layer and call the actual forward."""
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
self = forward_context.no_compile_layers[layer_name]
|
||||
self._topk_ids = topk_ids
|
||||
# Clone hidden_states because it will be mutated in-place in FusedMoE
|
||||
# TODO(bnell): figure out a way to avoid calling runner directly.
|
||||
# it is a hack that the weight are being passed via logits.
|
||||
return self.runner.forward(hidden_states.clone(), topk_weights)
|
||||
self._moe_state.topk_ids = topk_ids
|
||||
return self._forward_super(hidden_states, topk_weights)
|
||||
|
||||
|
||||
def transformers_moe_forward_fake(
|
||||
@@ -189,6 +193,7 @@ class MoEMixin(MixtureOfExperts):
|
||||
ckpt_up_proj_name=up_proj,
|
||||
num_experts=num_experts,
|
||||
num_redundant_experts=num_redundant_experts,
|
||||
routed_experts_prefix="",
|
||||
)
|
||||
)
|
||||
return expert_mapping
|
||||
@@ -286,8 +291,33 @@ class MoEMixin(MixtureOfExperts):
|
||||
if "shared_expert" in mlp_param_name:
|
||||
self.num_shared_experts = 1
|
||||
break
|
||||
|
||||
# Replace experts module with FusedMoE
|
||||
fused_experts = TransformersFusedMoE(
|
||||
moe_state = TransformersMoEState()
|
||||
|
||||
def custom_routing_function(
|
||||
hidden_states: torch.Tensor,
|
||||
gating_output: torch.Tensor,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
moe_state: TransformersMoEState,
|
||||
):
|
||||
"""Return `topk_weights` from `gating_output` and the
|
||||
`topk_ids` we stored in the layer earlier."""
|
||||
topk_weights = gating_output
|
||||
topk_ids = moe_state.topk_ids
|
||||
assert topk_ids is not None
|
||||
# Handle all gather in expert parallel
|
||||
if topk_ids.size(0) != hidden_states.size(0):
|
||||
dp_metadata = get_forward_context().dp_metadata
|
||||
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
|
||||
is_sp = moe_state.is_sequence_parallel
|
||||
dist_group = get_ep_group() if is_sp else get_dp_group()
|
||||
assert sizes[dist_group.rank_in_group] == topk_ids.shape[0]
|
||||
(topk_ids,) = dist_group.all_gatherv([topk_ids], 0, sizes)
|
||||
return topk_weights, topk_ids
|
||||
|
||||
fused_experts = FusedMoE(
|
||||
num_experts=num_experts,
|
||||
top_k=top_k,
|
||||
hidden_size=hidden_size,
|
||||
@@ -304,6 +334,12 @@ class MoEMixin(MixtureOfExperts):
|
||||
num_redundant_experts=num_redundant_experts,
|
||||
has_bias=has_bias,
|
||||
expert_mapping=expert_mapping,
|
||||
custom_routing_function=partial(
|
||||
custom_routing_function,
|
||||
moe_state=moe_state,
|
||||
),
|
||||
runner_cls=TransformersFusedMoE,
|
||||
runner_args={"moe_state": moe_state},
|
||||
)
|
||||
mlp.experts = fused_experts
|
||||
log_replacement(qual_name, experts, fused_experts)
|
||||
|
||||
Reference in New Issue
Block a user