[MoE Refactor] FusedMoE/MoERunner inversion refactor (#41184)
Signed-off-by: Bill Nell <bnell@redhat.com>
This commit is contained in:
@@ -20,6 +20,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
MoERunner,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -640,7 +641,7 @@ class AfmoeForCausalLM(nn.Module, SupportsPP, SupportsEagle3, SupportsLoRA):
|
||||
self.num_moe_layers = config.num_hidden_layers - config.num_dense_layers
|
||||
self.num_expert_groups = config.n_group
|
||||
|
||||
self.moe_layers: list[FusedMoE] = []
|
||||
self.moe_layers: list[MoERunner] = []
|
||||
example_moe = None
|
||||
for layer in self.model.layers:
|
||||
if isinstance(layer, PPMissingLayer):
|
||||
|
||||
@@ -16,6 +16,7 @@ from vllm.inputs import MultiModalDataDict
|
||||
from vllm.model_executor.layers.activation import get_act_fn
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
RoutedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
@@ -216,7 +217,7 @@ class AriaProjector(nn.Module):
|
||||
return out
|
||||
|
||||
|
||||
class AriaFusedMoE(FusedMoE):
|
||||
class AriaRoutedExperts(RoutedExperts):
|
||||
def weight_loader(
|
||||
self, param: nn.Parameter, loaded_weight: torch.Tensor, shard_id: str
|
||||
) -> None:
|
||||
@@ -225,13 +226,14 @@ class AriaFusedMoE(FusedMoE):
|
||||
# up weights for each expert.
|
||||
# Note: Loading expert weights with quantization is not supported
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
tp_size = self.moe_config.tp_size
|
||||
if shard_id == "w13":
|
||||
# the shape of loaded_weight is
|
||||
# (num_experts, hidden_size, 2 * moe_intermediate_size)
|
||||
if self.tp_size > 1:
|
||||
if tp_size > 1:
|
||||
up, gate = loaded_weight.chunk(2, dim=-1)
|
||||
up_current_rank = up.chunk(self.tp_size, dim=-1)[tp_rank]
|
||||
gate_current_rank = gate.chunk(self.tp_size, dim=-1)[tp_rank]
|
||||
up_current_rank = up.chunk(tp_size, dim=-1)[tp_rank]
|
||||
gate_current_rank = gate.chunk(tp_size, dim=-1)[tp_rank]
|
||||
up_and_gate = torch.cat(
|
||||
[up_current_rank, gate_current_rank], dim=-1
|
||||
).transpose(1, 2)
|
||||
@@ -241,8 +243,8 @@ class AriaFusedMoE(FusedMoE):
|
||||
elif shard_id == "w2":
|
||||
# the shape of loaded_weight is
|
||||
# (num_experts, moe_intermediate_size, hidden_size)
|
||||
if self.tp_size > 1:
|
||||
down_current_rank = loaded_weight.chunk(self.tp_size, dim=1)[tp_rank]
|
||||
if tp_size > 1:
|
||||
down_current_rank = loaded_weight.chunk(tp_size, dim=1)[tp_rank]
|
||||
param.data.copy_(down_current_rank.transpose(1, 2))
|
||||
else:
|
||||
param.data.copy_(loaded_weight.transpose(1, 2))
|
||||
@@ -278,7 +280,7 @@ class AriaTextMoELayer(nn.Module):
|
||||
bias=config.mlp_bias,
|
||||
)
|
||||
|
||||
self.experts = AriaFusedMoE(
|
||||
self.experts = FusedMoE(
|
||||
shared_experts=self.shared_experts,
|
||||
num_experts=config.moe_num_experts,
|
||||
top_k=config.moe_topk,
|
||||
@@ -286,6 +288,7 @@ class AriaTextMoELayer(nn.Module):
|
||||
intermediate_size=config.intermediate_size,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.experts",
|
||||
routed_experts_cls=AriaRoutedExperts,
|
||||
)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
@@ -351,8 +354,8 @@ class AriaTextModel(LlamaModel, SupportsQuant):
|
||||
(".qkv_proj", ".v_proj", "v"),
|
||||
(".gate_up_proj", ".gate_proj", 0),
|
||||
(".gate_up_proj", ".up_proj", 1),
|
||||
("experts.w13_weight", "experts.fc1.weight", "w13"),
|
||||
("experts.w2_weight", "experts.fc2.weight", "w2"),
|
||||
("experts.routed_experts.w13_weight", "experts.fc1.weight", "w13"),
|
||||
("experts.routed_experts.w2_weight", "experts.fc2.weight", "w2"),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
@@ -283,7 +283,7 @@ class BailingMoeV25(nn.Module):
|
||||
self.hidden_size = config.hidden_size
|
||||
self.quant_config = quant_config
|
||||
self.num_shared_experts = config.num_shared_experts
|
||||
self.score_function = getattr(config, "score_function", None)
|
||||
self.score_function: str | None = getattr(config, "score_function", None)
|
||||
self.n_group = getattr(config, "n_group", None)
|
||||
self.topk_group = getattr(config, "topk_group", None)
|
||||
self.use_grouped_topk = self.n_group is not None and self.topk_group is not None
|
||||
|
||||
@@ -17,6 +17,7 @@ from vllm.distributed import (
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
RoutedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
QKVParallelLinear,
|
||||
@@ -73,27 +74,17 @@ class DbrxRouter(nn.Module):
|
||||
return router_logits
|
||||
|
||||
|
||||
class DbrxExperts(FusedMoE):
|
||||
class DbrxExperts(RoutedExperts):
|
||||
def __init__(
|
||||
self,
|
||||
*args,
|
||||
config: DbrxConfig,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
params_dtype: torch.dtype | None = None,
|
||||
prefix: str = "",
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(
|
||||
num_experts=config.ffn_config.moe_num_experts,
|
||||
top_k=config.ffn_config.moe_top_k,
|
||||
hidden_size=config.d_model,
|
||||
intermediate_size=config.ffn_config.ffn_hidden_size,
|
||||
params_dtype=params_dtype,
|
||||
renormalize=True,
|
||||
quant_config=quant_config,
|
||||
tp_size=get_tensor_model_parallel_world_size(),
|
||||
prefix=prefix,
|
||||
)
|
||||
super().__init__(*args, **kwargs)
|
||||
self.config = config
|
||||
self.d_model = config.d_model
|
||||
self.tp_size = self.moe_config.tp_size
|
||||
self.intermediate_size = self.config.ffn_config.ffn_hidden_size // self.tp_size
|
||||
|
||||
# Define custom weight loader for dbrx model
|
||||
@@ -168,11 +159,18 @@ class DbrxMoE(nn.Module):
|
||||
|
||||
self.router = DbrxRouter(config, self.params_dtype)
|
||||
|
||||
self.experts = DbrxExperts(
|
||||
config=config,
|
||||
quant_config=quant_config,
|
||||
self.experts = FusedMoE(
|
||||
num_experts=config.ffn_config.moe_num_experts,
|
||||
top_k=config.ffn_config.moe_top_k,
|
||||
hidden_size=config.d_model,
|
||||
intermediate_size=config.ffn_config.ffn_hidden_size,
|
||||
params_dtype=self.params_dtype,
|
||||
prefix=f"{prefix}.experts",
|
||||
renormalize=True,
|
||||
quant_config=quant_config,
|
||||
tp_size=get_tensor_model_parallel_world_size(),
|
||||
prefix=prefix,
|
||||
routed_experts_cls=DbrxExperts,
|
||||
routed_experts_args={"config": config},
|
||||
)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
@@ -44,6 +44,7 @@ from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
MoERunner,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -670,7 +671,7 @@ class Ernie4_5_MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA, MixtureOfExpe
|
||||
self.num_moe_layers = len(moe_layers_indices)
|
||||
self.num_expert_groups = 1
|
||||
|
||||
self.moe_layers: list[FusedMoE] = []
|
||||
self.moe_layers: list[MoERunner] = []
|
||||
example_moe = None
|
||||
for layer in self.model.layers:
|
||||
if isinstance(layer, PPMissingLayer):
|
||||
|
||||
@@ -697,12 +697,18 @@ class Ernie4_5_VLMoeForCausalLM(nn.Module, SupportsPP):
|
||||
moe_offset = int(name.split(".")[-3])
|
||||
vision_expert_start_idx = self.config.moe_num_experts[0]
|
||||
is_text_expert = moe_offset <= vision_expert_start_idx - 1
|
||||
routed_experts = (
|
||||
".routed_experts" if ("w13_" in name or "w2_" in name) else ""
|
||||
)
|
||||
if is_text_expert:
|
||||
name = name.replace(".experts.", ".text_experts.")
|
||||
name = name.replace(
|
||||
".experts", f".text_experts{routed_experts}"
|
||||
)
|
||||
else:
|
||||
delta = moe_offset - vision_expert_start_idx
|
||||
name = name.replace(
|
||||
f".experts.{moe_offset}",
|
||||
f".vision_experts.{moe_offset - vision_expert_start_idx}",
|
||||
f".vision_experts{routed_experts}.{delta}",
|
||||
)
|
||||
|
||||
for mapping in expert_params_mapping:
|
||||
|
||||
@@ -1486,6 +1486,10 @@ class Gemma4Model(nn.Module, EagleModelMixin):
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
# Skip if name doesn't exist in params_dict (e.g., individual
|
||||
# expert weights that should have been handled above)
|
||||
if name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
|
||||
@@ -33,7 +33,7 @@ from transformers import PretrainedConfig
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
MoERunner,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -215,7 +215,7 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts):
|
||||
self.num_moe_layers = self.config.num_nextn_predict_layers
|
||||
self.num_expert_groups = self.config.n_group
|
||||
|
||||
self.moe_layers: list[FusedMoE] = []
|
||||
self.moe_layers: list[MoERunner] = []
|
||||
self.moe_mlp_layers: list[Glm4MoeLite] = []
|
||||
example_moe = None
|
||||
for layer in self.model.layers.values():
|
||||
|
||||
@@ -32,7 +32,7 @@ from transformers import PretrainedConfig
|
||||
|
||||
from vllm.config import CacheConfig, ParallelConfig, VllmConfig
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
MoERunner,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -201,7 +201,7 @@ class Glm4MoeMTP(nn.Module, Glm4MixtureOfExperts):
|
||||
self.num_moe_layers = self.config.num_nextn_predict_layers
|
||||
self.num_expert_groups = self.config.n_group
|
||||
|
||||
self.moe_layers: list[FusedMoE] = []
|
||||
self.moe_layers: list[MoERunner] = []
|
||||
self.moe_mlp_layers: list[Glm4MoE] = []
|
||||
example_moe = None
|
||||
for layer in self.model.layers.values():
|
||||
|
||||
@@ -43,6 +43,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
remap_moe_expert_weights,
|
||||
)
|
||||
from vllm.model_executor.models.utils import sequence_parallel_chunk
|
||||
from vllm.platforms import current_platform
|
||||
@@ -379,7 +380,8 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
||||
tp_rank_start = tp_rank * per_rank_intermediate_size
|
||||
tp_rank_end = min((tp_rank + 1) * per_rank_intermediate_size, intermediate_size)
|
||||
|
||||
for name, weight in weights:
|
||||
# Use centralized weight remapping for MoE expert parameters
|
||||
for name, weight in remap_moe_expert_weights(weights, params_dict):
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
@@ -580,8 +582,8 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
||||
Returns:
|
||||
Weight dtype string (e.g., "mxfp4", "fp8") or None if not available
|
||||
"""
|
||||
if hasattr(self.layers[layer_id].mlp.experts.quant_method, "weight_dtype"):
|
||||
return self.layers[layer_id].mlp.experts.quant_method.weight_dtype
|
||||
if hasattr(self.layers[layer_id].mlp.experts._quant_method, "weight_dtype"):
|
||||
return self.layers[layer_id].mlp.experts._quant_method.weight_dtype
|
||||
return None
|
||||
|
||||
intermediate_size = self.config.intermediate_size
|
||||
|
||||
@@ -455,6 +455,8 @@ class GraniteMoeHybridModel(nn.Module):
|
||||
loaded_params.add(n)
|
||||
|
||||
def _load_expert(n, p, name, shard_id, expert_id):
|
||||
if n not in params_dict:
|
||||
return
|
||||
param = params_dict[n]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, p, name, shard_id=shard_id, expert_id=expert_id)
|
||||
@@ -518,14 +520,14 @@ class GraniteMoeHybridModel(nn.Module):
|
||||
)
|
||||
w1_param, w3_param = p[e].chunk(2, dim=0)
|
||||
_load_expert(
|
||||
n.replace(".input_linear.", ".experts.w13_"),
|
||||
n.replace(".input_linear.", ".experts.routed_experts.w13_"),
|
||||
w1_param,
|
||||
w1_name,
|
||||
shard_id="w1",
|
||||
expert_id=e,
|
||||
)
|
||||
_load_expert(
|
||||
n.replace(".input_linear.", ".experts.w13_"),
|
||||
n.replace(".input_linear.", ".experts.routed_experts.w13_"),
|
||||
w3_param,
|
||||
w3_name,
|
||||
shard_id="w3",
|
||||
@@ -541,7 +543,7 @@ class GraniteMoeHybridModel(nn.Module):
|
||||
)
|
||||
w2_param = p[e]
|
||||
_load_expert(
|
||||
n.replace(".output_linear.", ".experts.w2_"),
|
||||
n.replace(".output_linear.", ".experts.routed_experts.w2_"),
|
||||
w2_param,
|
||||
w2_name,
|
||||
shard_id="w2",
|
||||
|
||||
@@ -214,11 +214,11 @@ class GraniteMoeSharedModel(nn.Module):
|
||||
for e in range(p.size(0)):
|
||||
w1_name = n.replace(
|
||||
".block_sparse_moe.input_linear.weight",
|
||||
f".block_sparse_moe.experts.{e}.w1.weight",
|
||||
f".block_sparse_moe.experts.routed_experts.{e}.w1.weight",
|
||||
)
|
||||
w3_name = n.replace(
|
||||
".block_sparse_moe.input_linear.weight",
|
||||
f".block_sparse_moe.experts.{e}.w3.weight",
|
||||
f".block_sparse_moe.experts.routed_experts.{e}.w3.weight",
|
||||
)
|
||||
w1_param, w3_param = p[e].chunk(2, dim=0)
|
||||
assert w1_name not in new_weights
|
||||
@@ -229,7 +229,7 @@ class GraniteMoeSharedModel(nn.Module):
|
||||
for e in range(p.size(0)):
|
||||
w2_name = n.replace(
|
||||
".block_sparse_moe.output_linear.weight",
|
||||
f".block_sparse_moe.experts.{e}.w2.weight",
|
||||
f".block_sparse_moe.experts.routed_experts.{e}.w2.weight",
|
||||
)
|
||||
w2_param = p[e]
|
||||
assert w2_name not in new_weights
|
||||
|
||||
@@ -229,7 +229,6 @@ class NemotronHMoE(nn.Module):
|
||||
scoring_func="sigmoid",
|
||||
e_score_correction_bias=self.gate.e_score_correction_bias,
|
||||
activation=activation_without_mul(config.mlp_hidden_act),
|
||||
is_act_and_mul=False, # non-gated MoE
|
||||
enable_eplb=self.enable_eplb,
|
||||
num_redundant_experts=self.n_redundant_experts,
|
||||
is_sequence_parallel=self.is_sequence_parallel,
|
||||
|
||||
@@ -37,6 +37,7 @@ from vllm.distributed import (
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
MoERunner,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -355,7 +356,7 @@ class SarvamMLAMoE(nn.Module):
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
)
|
||||
|
||||
def maybe_get_fused_moe(self) -> FusedMoE:
|
||||
def maybe_get_fused_moe(self) -> MoERunner:
|
||||
return self.experts
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
@@ -25,6 +25,7 @@ from vllm.model_executor.layers.activation import SiluAndMul, SwigluStepAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
MoERunner,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
|
||||
@@ -954,7 +955,7 @@ class Step3p5ForCausalLM(nn.Module, SupportsPP, MixtureOfExperts):
|
||||
) -> None:
|
||||
for layer_idx, layer in enumerate(self.moe_layers):
|
||||
experts = layer.experts
|
||||
assert isinstance(experts, FusedMoE)
|
||||
assert isinstance(experts, MoERunner)
|
||||
# Register the expert weights.
|
||||
self.expert_weights.append(experts.get_expert_weights())
|
||||
experts.set_eplb_state(
|
||||
|
||||
@@ -16,6 +16,9 @@
|
||||
# limitations under the License.
|
||||
"""Transformers modeling backend mixin for Mixture of Experts (MoE) models."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
@@ -27,6 +30,7 @@ from vllm.forward_context import ForwardContext, get_forward_context
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
MoERunner,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import MixtureOfExperts
|
||||
@@ -40,33 +44,22 @@ if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
|
||||
@dataclass
|
||||
class TransformersMoEState:
|
||||
topk_ids: torch.Tensor | None = None
|
||||
is_sequence_parallel: bool = False
|
||||
|
||||
|
||||
# --8<-- [start:transformers_fused_moe]
|
||||
@PluggableLayer.register("transformers_fused_moe")
|
||||
class TransformersFusedMoE(FusedMoE):
|
||||
class TransformersFusedMoE(MoERunner):
|
||||
"""Custom FusedMoE for the Transformers modeling backend."""
|
||||
|
||||
# --8<-- [end:transformers_fused_moe]
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self._topk_ids: torch.Tensor = None
|
||||
|
||||
def custom_routing_function(hidden_states, gating_output, topk, renormalize):
|
||||
"""Return `topk_weights` from `gating_output` and the
|
||||
`topk_ids` we stored in the layer earlier."""
|
||||
topk_weights = gating_output
|
||||
topk_ids = self._topk_ids
|
||||
# Handle all gather in expert parallel
|
||||
if topk_ids.size(0) != hidden_states.size(0):
|
||||
dp_metadata = get_forward_context().dp_metadata
|
||||
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
|
||||
is_sp = self.is_sequence_parallel
|
||||
dist_group = get_ep_group() if is_sp else get_dp_group()
|
||||
assert sizes[dist_group.rank_in_group] == topk_ids.shape[0]
|
||||
(topk_ids,) = dist_group.all_gatherv([topk_ids], 0, sizes)
|
||||
return topk_weights, topk_ids
|
||||
|
||||
kwargs["custom_routing_function"] = custom_routing_function
|
||||
def __init__(self, *args, moe_state: TransformersMoEState, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self._moe_state = moe_state
|
||||
self._moe_state.is_sequence_parallel = self.moe_config.is_sequence_parallel
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -78,6 +71,8 @@ class TransformersFusedMoE(FusedMoE):
|
||||
"""In Transformers `experts.forward` will have this signature.
|
||||
|
||||
We discard any extra kwargs because we cannot use them here."""
|
||||
# Note: we need to forward through a custom op so the topk_ids
|
||||
# can be transferred without interfering with cudagraphs.
|
||||
return torch.ops.vllm.transformers_moe_forward(
|
||||
hidden_states,
|
||||
topk_ids.to(torch.int32),
|
||||
@@ -85,6 +80,18 @@ class TransformersFusedMoE(FusedMoE):
|
||||
self.layer_name,
|
||||
)
|
||||
|
||||
def _forward_super(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return super().forward(hidden_states, topk_weights)
|
||||
|
||||
def load_weights(
|
||||
self, weights: Iterable[tuple[str, torch.Tensor]]
|
||||
) -> Iterable[str]:
|
||||
return self.routed_experts.load_weights(weights)
|
||||
|
||||
|
||||
def transformers_moe_forward(
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -95,11 +102,8 @@ def transformers_moe_forward(
|
||||
"""Store the `topk_ids` in the layer and call the actual forward."""
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
self = forward_context.no_compile_layers[layer_name]
|
||||
self._topk_ids = topk_ids
|
||||
# Clone hidden_states because it will be mutated in-place in FusedMoE
|
||||
# TODO(bnell): figure out a way to avoid calling runner directly.
|
||||
# it is a hack that the weight are being passed via logits.
|
||||
return self.runner.forward(hidden_states.clone(), topk_weights)
|
||||
self._moe_state.topk_ids = topk_ids
|
||||
return self._forward_super(hidden_states, topk_weights)
|
||||
|
||||
|
||||
def transformers_moe_forward_fake(
|
||||
@@ -189,6 +193,7 @@ class MoEMixin(MixtureOfExperts):
|
||||
ckpt_up_proj_name=up_proj,
|
||||
num_experts=num_experts,
|
||||
num_redundant_experts=num_redundant_experts,
|
||||
routed_experts_prefix="",
|
||||
)
|
||||
)
|
||||
return expert_mapping
|
||||
@@ -286,8 +291,33 @@ class MoEMixin(MixtureOfExperts):
|
||||
if "shared_expert" in mlp_param_name:
|
||||
self.num_shared_experts = 1
|
||||
break
|
||||
|
||||
# Replace experts module with FusedMoE
|
||||
fused_experts = TransformersFusedMoE(
|
||||
moe_state = TransformersMoEState()
|
||||
|
||||
def custom_routing_function(
|
||||
hidden_states: torch.Tensor,
|
||||
gating_output: torch.Tensor,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
moe_state: TransformersMoEState,
|
||||
):
|
||||
"""Return `topk_weights` from `gating_output` and the
|
||||
`topk_ids` we stored in the layer earlier."""
|
||||
topk_weights = gating_output
|
||||
topk_ids = moe_state.topk_ids
|
||||
assert topk_ids is not None
|
||||
# Handle all gather in expert parallel
|
||||
if topk_ids.size(0) != hidden_states.size(0):
|
||||
dp_metadata = get_forward_context().dp_metadata
|
||||
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
|
||||
is_sp = moe_state.is_sequence_parallel
|
||||
dist_group = get_ep_group() if is_sp else get_dp_group()
|
||||
assert sizes[dist_group.rank_in_group] == topk_ids.shape[0]
|
||||
(topk_ids,) = dist_group.all_gatherv([topk_ids], 0, sizes)
|
||||
return topk_weights, topk_ids
|
||||
|
||||
fused_experts = FusedMoE(
|
||||
num_experts=num_experts,
|
||||
top_k=top_k,
|
||||
hidden_size=hidden_size,
|
||||
@@ -304,6 +334,12 @@ class MoEMixin(MixtureOfExperts):
|
||||
num_redundant_experts=num_redundant_experts,
|
||||
has_bias=has_bias,
|
||||
expert_mapping=expert_mapping,
|
||||
custom_routing_function=partial(
|
||||
custom_routing_function,
|
||||
moe_state=moe_state,
|
||||
),
|
||||
runner_cls=TransformersFusedMoE,
|
||||
runner_args={"moe_state": moe_state},
|
||||
)
|
||||
mlp.experts = fused_experts
|
||||
log_replacement(qual_name, experts, fused_experts)
|
||||
|
||||
Reference in New Issue
Block a user