[MoE Refactor] Rename FusedMoE to FusedMoEFactory (#44941)
Signed-off-by: Bill Nell <bnell@redhat.com> Signed-off-by: bnellnm <49004751+bnellnm@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -43,7 +43,7 @@ from vllm.distributed import (
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -168,7 +168,7 @@ class AXK1MoE(nn.Module):
|
||||
prefix=f"{prefix}.shared_experts",
|
||||
)
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
gate=self.gate,
|
||||
num_experts=config.n_routed_experts,
|
||||
|
||||
@@ -18,7 +18,7 @@ from vllm.distributed import (
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
MoERunner,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -122,8 +122,8 @@ class AfmoeMoE(nn.Module):
|
||||
prefix=f"{prefix}.shared_experts",
|
||||
)
|
||||
|
||||
# Routed experts using FusedMoE
|
||||
self.experts = FusedMoE(
|
||||
# Routed experts using FusedMoEFactory
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
num_experts=config.num_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
|
||||
@@ -13,7 +13,9 @@ from vllm.config import VllmConfig
|
||||
from vllm.config.multimodal import BaseDummyOptions
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.model_executor.layers.activation import get_act_fn
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEFactory,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
@@ -222,7 +224,7 @@ class AriaTextMoELayer(nn.Module):
|
||||
bias=config.mlp_bias,
|
||||
)
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
num_experts=config.moe_num_experts,
|
||||
top_k=config.moe_topk,
|
||||
|
||||
@@ -41,7 +41,7 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -289,7 +289,7 @@ class BailingMoE(nn.Module):
|
||||
else:
|
||||
self.shared_experts = None
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
num_experts=self.num_experts,
|
||||
top_k=self.top_k,
|
||||
|
||||
@@ -18,7 +18,7 @@ from vllm.distributed import (
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -329,8 +329,8 @@ class BailingMoeV25(nn.Module):
|
||||
else:
|
||||
self.shared_experts = None
|
||||
|
||||
# Routed experts using FusedMoE
|
||||
self.experts = FusedMoE(
|
||||
# Routed experts using FusedMoEFactory
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
num_experts=self.num_experts,
|
||||
top_k=self.top_k,
|
||||
@@ -602,7 +602,7 @@ class BailingMoeV25Model(nn.Module):
|
||||
(".gate_up_proj", ".up_proj", 1),
|
||||
]
|
||||
|
||||
# Expert parameter mappings from FusedMoE
|
||||
# Expert parameter mappings from FusedMoEFactory
|
||||
expert_mappings = list(self.get_expert_mapping())
|
||||
|
||||
def load_param(name: str, tensor: torch.Tensor, shard_id=None) -> bool:
|
||||
|
||||
@@ -320,7 +320,7 @@ class NomicMoE(nn.Module):
|
||||
# router_logits: (num_tokens, n_experts)
|
||||
router_logits, _ = self.router(hidden_states)
|
||||
# FIXME(Isotr0py): This implementation is too tricky,
|
||||
# we should use FusedMoE instead in the future
|
||||
# we should use FusedMoEFactory instead in the future
|
||||
# after supporting ungated activation for it.
|
||||
topk_weights, topk_ids, _ = fused_topk(
|
||||
hidden_states, router_logits, self.top_k, renormalize=False
|
||||
|
||||
@@ -16,7 +16,7 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
@@ -300,7 +300,7 @@ class Cohere2Moe(nn.Module):
|
||||
self.shared_experts = None
|
||||
self.shared_expert_combination_strategy = None
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=config.num_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
hidden_size=config.hidden_size,
|
||||
@@ -318,7 +318,7 @@ class Cohere2Moe(nn.Module):
|
||||
orig_shape = hidden_states.shape
|
||||
hidden_states = hidden_states.view(-1, self.hidden_size)
|
||||
router_logits, _ = self.gate(hidden_states)
|
||||
# FusedMoE handles shared expert overlap internally and returns
|
||||
# FusedMoEFactory handles shared expert overlap internally and returns
|
||||
# shared_output + routed_output when shared_experts is set.
|
||||
final_hidden_states = self.experts(hidden_states, router_logits)
|
||||
if self.shared_expert_combination_strategy == "average":
|
||||
|
||||
@@ -16,7 +16,7 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
RoutedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
@@ -159,7 +159,7 @@ class DbrxMoE(nn.Module):
|
||||
|
||||
self.router = DbrxRouter(config, self.params_dtype)
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=config.ffn_config.moe_num_experts,
|
||||
top_k=config.ffn_config.moe_top_k,
|
||||
hidden_size=config.d_model,
|
||||
|
||||
@@ -50,7 +50,7 @@ from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention, RSWAAttention
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
GateLinear,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
@@ -361,7 +361,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
prefix=f"{prefix}.shared_experts",
|
||||
)
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
gate=self.gate,
|
||||
num_experts=config.n_routed_experts,
|
||||
|
||||
@@ -41,7 +41,7 @@ from vllm.distributed import (
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE, MoERunner
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory, MoERunner
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -183,7 +183,7 @@ class Ernie4_5_MoeMoE(nn.Module):
|
||||
else:
|
||||
self.shared_experts = None
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
num_experts=config.moe_num_experts,
|
||||
top_k=config.moe_k,
|
||||
|
||||
@@ -37,7 +37,7 @@ from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -260,7 +260,7 @@ class Ernie4_5_VLMoeMoE(nn.Module):
|
||||
prefix=f"{prefix}.text_experts_gate",
|
||||
)
|
||||
|
||||
self.text_experts = FusedMoE(
|
||||
self.text_experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
num_experts=config.moe_num_experts[0],
|
||||
top_k=config.moe_k,
|
||||
@@ -297,7 +297,7 @@ class Ernie4_5_VLMoeMoE(nn.Module):
|
||||
prefix=f"{prefix}.vision_experts_gate",
|
||||
)
|
||||
|
||||
self.vision_experts = FusedMoE(
|
||||
self.vision_experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
num_experts=config.moe_num_experts[1],
|
||||
top_k=config.moe_k,
|
||||
|
||||
@@ -29,9 +29,7 @@ from vllm.distributed import (
|
||||
get_pp_group,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
@@ -126,7 +124,7 @@ class ExaoneMoe(nn.Module):
|
||||
else:
|
||||
self.shared_experts = None
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
gate=self.gate,
|
||||
num_experts=self.n_routed_experts,
|
||||
|
||||
@@ -21,7 +21,7 @@ from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
@@ -73,7 +73,7 @@ class FlexOlmoMoE(nn.Module):
|
||||
prefix=f"{prefix}.gate",
|
||||
)
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=hf_config.num_experts,
|
||||
top_k=hf_config.num_experts_per_tok,
|
||||
hidden_size=hf_config.hidden_size,
|
||||
|
||||
@@ -38,7 +38,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import get_act_and_mul_fn
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
GateLinear,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
@@ -305,14 +305,14 @@ class Gemma4Router(nn.Module):
|
||||
|
||||
|
||||
class Gemma4MoE(nn.Module):
|
||||
"""Mixture of Experts for Gemma4 using vLLM's FusedMoE.
|
||||
"""Mixture of Experts for Gemma4 using vLLM's MoERunner.
|
||||
|
||||
Wraps FusedMoE with custom routing. The router projection is
|
||||
Wraps MoERunner with custom routing. The router projection is
|
||||
external (Gemma4Router) — this class only handles expert dispatch.
|
||||
|
||||
Gemma4 routing: softmax over ALL experts → top-k → renormalize.
|
||||
per_expert_scale is folded into routing weights for mathematical
|
||||
correctness with FusedMoE's fused kernel.
|
||||
correctness with MoERunner's fused kernel.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -326,11 +326,11 @@ class Gemma4MoE(nn.Module):
|
||||
self.num_experts = config.num_experts
|
||||
|
||||
# Per-expert output scale folded into routing weights so that
|
||||
# FusedMoE's fused kernel computes: Σ_e (expert_e * w_e * scale_e)
|
||||
# MoERunner's fused kernel computes: Σ_e (expert_e * w_e * scale_e)
|
||||
self.per_expert_scale = nn.Parameter(torch.ones(config.num_experts))
|
||||
|
||||
# Gemma4 routing: softmax over ALL experts → top-k → renormalize.
|
||||
# FusedMoE's built-in fused_topk scopes softmax differently, so
|
||||
# MoERunner's built-in fused_topk scopes softmax differently, so
|
||||
# a custom routing function is needed for numerical correctness.
|
||||
# NOTE: self.per_expert_scale is read at call time (not captured into
|
||||
# a local) so that torch.func.functional_call parameter substitution
|
||||
@@ -350,8 +350,8 @@ class Gemma4MoE(nn.Module):
|
||||
gating_output, topk, self.per_expert_scale
|
||||
)
|
||||
|
||||
# FusedMoE experts with custom Gemma4 routing
|
||||
self.experts = FusedMoE(
|
||||
# MoERunner experts with custom Gemma4 routing
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=config.num_experts,
|
||||
top_k=config.top_k_experts,
|
||||
hidden_size=config.hidden_size,
|
||||
@@ -1382,10 +1382,10 @@ class Gemma4Model(nn.Module, EagleModelMixin):
|
||||
# MoE expert weight mapping: checkpoint can have either:
|
||||
# 1. 3D packed tensors (exploded in _weight_iterator to per-expert 2D)
|
||||
# 2. Already per-expert 2D weights (if quantized)
|
||||
# Map to FusedMoE parameters:
|
||||
# moe.experts.{id}.gate_proj → FusedMoE w1 (shard of w13)
|
||||
# moe.experts.{id}.up_proj → FusedMoE w3 (shard of w13)
|
||||
# moe.experts.{id}.down_proj → FusedMoE w2
|
||||
# Map to MoERunner parameters:
|
||||
# moe.experts.{id}.gate_proj → MoERunner w1 (shard of w13)
|
||||
# moe.experts.{id}.up_proj → MoERunner w3 (shard of w13)
|
||||
# moe.experts.{id}.down_proj → MoERunner w2
|
||||
num_experts = getattr(self.config, "num_experts", None) or 0
|
||||
# Strategy A: dot-separated suffix
|
||||
# (standard AWQ/GPTQ e.g. .qweight, .scales, .weight)
|
||||
@@ -1475,7 +1475,7 @@ class Gemma4Model(nn.Module, EagleModelMixin):
|
||||
continue
|
||||
param = params_dict[moe_name]
|
||||
# Expert weights are already in the correct
|
||||
# orientation for FusedMoE after _weight_iterator:
|
||||
# orientation for MoERunner after _weight_iterator:
|
||||
# gate/up: [I, H] → w1/w3 expects [I, H]
|
||||
# down: [H, I] → w2 expects [H, I]
|
||||
# Scales and other quantization params may be 1D or scalar.
|
||||
@@ -1668,19 +1668,19 @@ class Gemma4ForCausalLM(
|
||||
|
||||
# MoE expert weights: checkpoint stores as 3D packed
|
||||
# tensors. Explode into per-expert 2D weights for
|
||||
# FusedMoE weight_loader.
|
||||
# MoERunner weight_loader.
|
||||
#
|
||||
# Checkpoint format:
|
||||
# moe.gate_up_proj: [E, 2*I, H] (fused gate + up)
|
||||
# moe.down_proj: [E, H, I]
|
||||
#
|
||||
# FusedMoE expects per-expert:
|
||||
# MoERunner expects per-expert:
|
||||
# w1 (gate): [I, H] — first half of gate_up
|
||||
# w3 (up): [I, H] — second half of gate_up
|
||||
# w2 (down): [H, I] — as-is from checkpoint
|
||||
#
|
||||
# No transpose needed: checkpoint orientation already
|
||||
# matches FusedMoE's expected layout.
|
||||
# matches MoERunner's expected layout.
|
||||
if "moe.gate_up_proj" in name and weight.dim() == 3:
|
||||
num_experts = weight.size(0)
|
||||
intermediate_size = weight.size(1) // 2
|
||||
|
||||
@@ -42,7 +42,9 @@ from vllm.distributed import (
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEFactory,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -164,7 +166,7 @@ class Glm4MoE(nn.Module):
|
||||
)
|
||||
|
||||
# AITER fused shared-expert (FSE) gate; mirrors the deepseek_v2.py
|
||||
# pattern (see Glm4MoE / FusedMoE wiring there).
|
||||
# pattern (see Glm4MoE / MoERunner wiring there).
|
||||
self.is_rocm_aiter_moe_enabled = rocm_aiter_ops.is_fused_moe_enabled()
|
||||
self.is_fusion_moe_shared_experts_enabled = (
|
||||
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
@@ -183,7 +185,7 @@ class Glm4MoE(nn.Module):
|
||||
prefix=f"{prefix}.shared_experts",
|
||||
)
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
num_experts=config.n_routed_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
|
||||
@@ -21,7 +21,7 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig
|
||||
@@ -211,7 +211,7 @@ class MLPBlock(torch.nn.Module):
|
||||
return_bias=False,
|
||||
)
|
||||
assert config.intermediate_size % self.world_size == 0
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=config.num_local_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
hidden_size=config.hidden_size,
|
||||
@@ -1010,7 +1010,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
||||
tp_rank_end = min((tp_rank + 1) * per_rank_intermediate_size, intermediate_size)
|
||||
|
||||
# Use centralized weight remapping for MoE expert parameters.
|
||||
# The FusedMoE refactor moved expert params under
|
||||
# The MoERunner refactor moved expert params under
|
||||
# `mlp.experts.routed_experts.*`; this remaps checkpoint names so
|
||||
# MoE weight/bias keys resolve against params_dict.
|
||||
for name, weight in remap_moe_expert_weights(weights, params_dict):
|
||||
|
||||
@@ -40,7 +40,7 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -101,7 +101,7 @@ class GraniteMoeMoE(nn.Module):
|
||||
prefix=f"{prefix}.gate",
|
||||
)
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=num_experts,
|
||||
top_k=top_k,
|
||||
hidden_size=hidden_size,
|
||||
|
||||
@@ -43,7 +43,7 @@ from vllm.distributed import (
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -441,7 +441,7 @@ class HunYuanSparseMoeBlock(nn.Module):
|
||||
else:
|
||||
self.shared_mlp = None
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_mlp,
|
||||
num_experts=self.n_routed_experts,
|
||||
top_k=top_k,
|
||||
|
||||
@@ -44,7 +44,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
GateLinear,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
@@ -184,7 +184,7 @@ class HYV3MoEFused(nn.Module):
|
||||
scoring_func = "sigmoid"
|
||||
e_score_correction_bias = self.expert_bias
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=self.n_routed_experts,
|
||||
top_k=top_k,
|
||||
hidden_size=config.hidden_size,
|
||||
|
||||
@@ -42,7 +42,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
@@ -173,7 +173,7 @@ class InternS1ProMoeSparseMoeBlock(nn.Module):
|
||||
# For custom routing function
|
||||
self.n_groups = getattr(config, "router_n_groups", -1)
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=self.n_routed_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
hidden_size=config.hidden_size,
|
||||
|
||||
@@ -14,7 +14,9 @@ from vllm.config import CacheConfig, ModelConfig, VllmConfig
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEFactory,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
QKVParallelLinear,
|
||||
@@ -81,7 +83,7 @@ class JambaMoE(nn.Module):
|
||||
prefix=f"{prefix}.router",
|
||||
)
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
self.num_total_experts,
|
||||
self.top_k,
|
||||
self.hidden_size,
|
||||
|
||||
@@ -19,7 +19,7 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -174,7 +174,7 @@ class LagunaMoE(nn.Module):
|
||||
prefix=f"{prefix}.gate",
|
||||
)
|
||||
|
||||
# Shared expert (optional) - passed to FusedMoE for overlap optimization
|
||||
# Shared expert (optional) - passed to FusedMoEFactory for overlap optimization
|
||||
self.shared_expert: LagunaMLP | None
|
||||
if config.shared_expert_intermediate_size > 0:
|
||||
self.shared_expert = LagunaMLP(
|
||||
@@ -191,20 +191,22 @@ class LagunaMoE(nn.Module):
|
||||
# Auxiliary-loss-free load-balancing bias (arXiv:2408.15664). The
|
||||
# checkpoint stores one [num_experts] tensor per MoE layer at
|
||||
# `mlp.experts.e_score_correction_bias`; registering it as a Parameter
|
||||
# on the FusedMoE lets the weight loader pick it up and the router
|
||||
# on the MoERunner lets the weight loader pick it up and the router
|
||||
# add it during top-k selection. The fused top-k bias router requires
|
||||
# float32 regardless of model dtype.
|
||||
e_score_correction_bias = torch.nn.Parameter(
|
||||
torch.zeros(config.num_experts, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
# FusedMoE with SIGMOID routing. Passing `shared_experts=` lets the
|
||||
# MoERunner with SIGMOID routing. Passing `shared_experts=` lets the
|
||||
# layer overlap the shared-expert compute with the all2all dispatch.
|
||||
# `apply_routed_scale_to_output=True` makes FusedMoE handle the
|
||||
# `apply_routed_scale_to_output=True` makes MoERunner handle the
|
||||
# FusedMoEFactory with SIGMOID routing. Passing `shared_experts=` lets the
|
||||
# layer overlap the shared-expert compute with the all2all dispatch.
|
||||
# `apply_routed_scale_to_output=True` makes FusedMoEFactory handle the
|
||||
# routed_scaling_factor, shared+routed combine, and TP all-reduce
|
||||
# internally, so forward() just returns the final hidden states.
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_expert,
|
||||
num_experts=config.num_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
|
||||
@@ -15,7 +15,9 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEFactory,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -143,7 +145,7 @@ class Lfm2MoeSparseMoeBlock(nn.Module):
|
||||
else:
|
||||
self.gate.e_score_correction_bias = None
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=self.n_routed_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
hidden_size=config.hidden_size,
|
||||
|
||||
@@ -37,7 +37,7 @@ from vllm.model_executor.layers.attention import (
|
||||
ChunkedLocalAttention,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -131,7 +131,7 @@ class Llama4MoE(nn.Module):
|
||||
self.n_physical_experts = self.n_local_experts + self.n_redundant_experts
|
||||
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_expert,
|
||||
num_experts=config.num_local_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
|
||||
@@ -47,7 +47,7 @@ from vllm.distributed import get_pp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -297,7 +297,7 @@ class LongcatMoe(nn.Module):
|
||||
)
|
||||
|
||||
assert config.zero_expert_type is not None
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
zero_expert_type=config.zero_expert_type,
|
||||
e_score_correction_bias=self.router.e_score_correction_bias,
|
||||
num_experts=num_experts,
|
||||
@@ -317,7 +317,7 @@ class LongcatMoe(nn.Module):
|
||||
num_tokens, hidden_dim = hidden_states.shape
|
||||
hidden_states = hidden_states.view(-1, hidden_dim)
|
||||
|
||||
# Align to FusedMoE padded hidden size to avoid dim mismatch
|
||||
# Align to MoERunner padded hidden size to avoid dim mismatch
|
||||
padded_hidden = self.experts.moe_config.hidden_dim
|
||||
if hidden_dim < padded_hidden:
|
||||
hidden_states_padded = torch.nn.functional.pad(
|
||||
@@ -333,7 +333,7 @@ class LongcatMoe(nn.Module):
|
||||
hidden_states_padded.to(self.router_params_dtype)
|
||||
)
|
||||
|
||||
# FusedMoE handles routing memoization and zero expert computation
|
||||
# MoERunner handles routing memoization and zero expert computation
|
||||
# internally. Pass full router_logits (including zero experts) so that
|
||||
# zero experts can be properly identified in routing.
|
||||
final_hidden_states = self.experts(
|
||||
|
||||
@@ -24,7 +24,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -171,7 +171,7 @@ class MiMoV2MoE(nn.Module):
|
||||
torch.empty(config.n_routed_experts, dtype=self.gate_dtype)
|
||||
)
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=self.n_routed_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
hidden_size=config.hidden_size,
|
||||
|
||||
@@ -39,9 +39,7 @@ from vllm.distributed import (
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
@@ -95,7 +93,7 @@ class MiniMaxM2MoE(nn.Module):
|
||||
else:
|
||||
self.e_score_correction_bias = None
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=config.num_local_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
scoring_func=config.scoring_func,
|
||||
|
||||
@@ -39,9 +39,7 @@ from vllm.distributed import (
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
QKVParallelLinear,
|
||||
@@ -123,7 +121,7 @@ class MixtralMoE(nn.Module):
|
||||
prefix=f"{prefix}.gate",
|
||||
)
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=num_experts,
|
||||
top_k=top_k,
|
||||
hidden_size=hidden_size,
|
||||
|
||||
@@ -33,7 +33,7 @@ from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.model_executor.layers.activation import ReLUSquaredActivation
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
GateLinear,
|
||||
activation_without_mul,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
@@ -209,7 +209,7 @@ class NemotronHMoE(nn.Module):
|
||||
self.fc1_latent_proj = None
|
||||
self.fc2_latent_proj = None
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
num_experts=config.n_routed_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
|
||||
@@ -32,9 +32,7 @@ from vllm.distributed import (
|
||||
from vllm.distributed.utils import split_tensor_along_last_dim
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
QKVParallelLinear,
|
||||
@@ -94,7 +92,7 @@ class OlmoeMoE(nn.Module):
|
||||
prefix=f"{prefix}.gate",
|
||||
)
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=num_experts,
|
||||
top_k=top_k,
|
||||
hidden_size=hidden_size,
|
||||
|
||||
@@ -43,7 +43,7 @@ from vllm.model_executor.layers.attention import (
|
||||
Attention,
|
||||
StaticSinkAttention,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -195,7 +195,7 @@ class OpenPanguMoE(nn.Module):
|
||||
else:
|
||||
self.shared_experts = None
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
num_experts=config.n_routed_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
|
||||
@@ -32,7 +32,9 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEFactory,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -325,7 +327,7 @@ class Param2MoEMoEBlock(nn.Module):
|
||||
else:
|
||||
self.shared_experts = None # type: ignore[assignment]
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
num_experts=self.num_experts,
|
||||
top_k=self.top_k,
|
||||
@@ -342,7 +344,7 @@ class Param2MoEMoEBlock(nn.Module):
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
)
|
||||
|
||||
def maybe_get_fused_moe(self) -> FusedMoE:
|
||||
def maybe_get_fused_moe(self) -> FusedMoEFactory:
|
||||
return self.experts
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
@@ -35,9 +35,7 @@ from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.linear import (
|
||||
QKVParallelLinear,
|
||||
ReplicatedLinear,
|
||||
@@ -273,7 +271,7 @@ class PhiMoE(nn.Module):
|
||||
prefix=f"{prefix}.gate",
|
||||
)
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=num_experts,
|
||||
top_k=top_k,
|
||||
hidden_size=hidden_size,
|
||||
|
||||
@@ -40,9 +40,7 @@ from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -165,7 +163,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
|
||||
else:
|
||||
self.shared_expert = None
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_expert,
|
||||
num_experts=config.num_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
|
||||
@@ -42,7 +42,7 @@ from vllm.distributed import (
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -201,7 +201,7 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
|
||||
self.shared_expert_gate = None
|
||||
self.shared_expert = None
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_expert,
|
||||
gate=self.gate,
|
||||
num_experts=self.n_routed_experts,
|
||||
@@ -228,13 +228,13 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
|
||||
hidden_states = sequence_parallel_chunk(hidden_states)
|
||||
|
||||
if self.experts.is_internal_router:
|
||||
# In this case, the gate/router runs inside the FusedMoE class
|
||||
# In this case, the gate/router runs inside the MoERunner class
|
||||
final_hidden_states = self.experts(
|
||||
hidden_states=hidden_states, router_logits=hidden_states
|
||||
)
|
||||
else:
|
||||
# Actually this will be dead code, since we always pass gate into
|
||||
# FusedMoE in the current implementation. But we keep this code
|
||||
# MoERunner in the current implementation. But we keep this code
|
||||
# here for clarity and future flexibility.
|
||||
router_logits, _ = self.gate(hidden_states)
|
||||
final_hidden_states = self.experts(
|
||||
|
||||
@@ -20,9 +20,7 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.fused_qk_norm_rope import fused_qk_rmsnorm_rope_gate
|
||||
from vllm.model_executor.layers.layernorm import (
|
||||
GemmaRMSNorm as Qwen3NextRMSNorm,
|
||||
@@ -175,7 +173,7 @@ class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
prefix=f"{prefix}.shared_expert",
|
||||
)
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_expert,
|
||||
gate=self.gate,
|
||||
num_experts=self.n_routed_experts,
|
||||
@@ -208,7 +206,7 @@ class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
hidden_states = sequence_parallel_chunk(hidden_states)
|
||||
|
||||
if self.experts.is_internal_router:
|
||||
# In this case, the gate/router runs inside the FusedMoE class
|
||||
# In this case, the gate/router runs inside the MoERunner class
|
||||
final_hidden_states = self.experts(
|
||||
hidden_states=hidden_states, router_logits=hidden_states
|
||||
)
|
||||
|
||||
@@ -35,7 +35,7 @@ from vllm.distributed import (
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE, MoERunner
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory, MoERunner
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -334,7 +334,7 @@ class SarvamMLAMoE(nn.Module):
|
||||
else:
|
||||
self.shared_experts = None
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_experts,
|
||||
num_experts=self.num_experts,
|
||||
top_k=self.top_k,
|
||||
|
||||
@@ -19,7 +19,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
@@ -68,7 +68,7 @@ class FusedMoEBlock(nn.Module):
|
||||
f"the number of experts {config.moe_num_experts}."
|
||||
)
|
||||
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=config.moe_num_experts,
|
||||
top_k=config.moe_top_k,
|
||||
hidden_size=config.hidden_size,
|
||||
|
||||
@@ -24,7 +24,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul, SwigluStepAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEFactory,
|
||||
MoERunner,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
@@ -376,7 +376,7 @@ class FusedMoEBlock(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.share_expert",
|
||||
)
|
||||
self.experts = FusedMoE(
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.share_expert,
|
||||
gate=self.gate,
|
||||
num_experts=config.moe_num_experts,
|
||||
@@ -404,7 +404,7 @@ class FusedMoEBlock(nn.Module):
|
||||
hidden_states=hidden_states, router_logits=hidden_states
|
||||
)
|
||||
else:
|
||||
# TODO(bnell): this gate could be moved into the FusedMoE?
|
||||
# TODO(bnell): this gate could be moved into the MoERunner?
|
||||
router_logits, _ = self.gate(hidden_states)
|
||||
final_hidden_states = self.experts(
|
||||
hidden_states=hidden_states, router_logits=router_logits
|
||||
|
||||
@@ -29,7 +29,11 @@ from vllm.distributed import get_dp_group, get_ep_group
|
||||
from vllm.forward_context import ForwardContext, get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE, MoERunner, RoutedExperts
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEFactory,
|
||||
MoERunner,
|
||||
RoutedExperts,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import MixtureOfExperts
|
||||
from vllm.model_executor.models.transformers.fuser import get_fuser
|
||||
from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser
|
||||
@@ -53,7 +57,7 @@ class TransformersMoEState:
|
||||
# --8<-- [start:transformers_fused_moe]
|
||||
@PluggableLayer.register("transformers_fused_moe")
|
||||
class TransformersMoERunner(MoERunner):
|
||||
"""Custom FusedMoE for the Transformers modeling backend."""
|
||||
"""Custom MoERunner for the Transformers modeling backend."""
|
||||
|
||||
# --8<-- [end:transformers_fused_moe]
|
||||
def __init__(self, *args, moe_state: TransformersMoEState, **kwargs):
|
||||
@@ -297,7 +301,7 @@ class MoEMixin(MixtureOfExperts):
|
||||
)
|
||||
if fuser is not None and not reaches_fp16_trick:
|
||||
# MoE block forward is fully replaced.
|
||||
# gate/router and shared expert (if any) runs in FusedMoE.
|
||||
# gate/router and shared expert (if any) runs in MoERunner.
|
||||
shared_experts = fuser.shared_experts(moe_block, prefix)
|
||||
# Store shared experts for later down projection adjustment
|
||||
if shared_experts is not None:
|
||||
@@ -331,7 +335,7 @@ class MoEMixin(MixtureOfExperts):
|
||||
if fuser.shared_name:
|
||||
routed += " + shared experts"
|
||||
logger.info_once(
|
||||
"Fused: %s (%s) -> FusedMoE (internal routing)",
|
||||
"Fused: %s (%s) -> MoERunner (internal routing)",
|
||||
routed,
|
||||
moe_block_cls,
|
||||
)
|
||||
@@ -373,10 +377,10 @@ class MoEMixin(MixtureOfExperts):
|
||||
runner_args={"moe_state": moe_state},
|
||||
)
|
||||
logger.info_once(
|
||||
"Fused: experts (%s) -> FusedMoE (external routing)",
|
||||
"Fused: experts (%s) -> MoERunner (external routing)",
|
||||
experts_cls,
|
||||
)
|
||||
fused_experts = FusedMoE(**kwargs)
|
||||
fused_experts = FusedMoEFactory(**kwargs)
|
||||
moe_block.experts = fused_experts
|
||||
log_replacement(qual_name, experts, fused_experts)
|
||||
# Update MixtureOfExperts mixin state
|
||||
|
||||
Reference in New Issue
Block a user