[MoE] Use GateLinear for all MoE models (#58234)

Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
Jee Jee Li
2026-09-23 05:40:13 +00:00
committed by GitHub
parent e975732152
commit de6674b002
34 changed files with 221 additions and 225 deletions
+2 -3
View File
@@ -44,6 +44,7 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.utils import (
@@ -128,11 +129,9 @@ class AXK1MoE(nn.Module):
"Only silu is supported for now."
)
self.gate = ReplicatedLinear(
self.gate = GateLinear(
config.hidden_size,
config.n_routed_experts,
bias=False,
quant_config=None,
prefix=f"{prefix}.gate",
)
if config.topk_method == "noaux_tc":
+6 -4
View File
@@ -19,6 +19,7 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
MoERunner,
)
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -82,11 +83,12 @@ class AfmoeMoE(nn.Module):
)
# Router gate
self.gate = nn.Linear(
self.gate = GateLinear(
config.hidden_size,
config.num_experts,
bias=False,
dtype=torch.float32,
out_dtype=torch.float32,
params_dtype=torch.float32,
prefix=f"{prefix}.gate",
)
self.expert_bias = nn.Parameter(
torch.empty(config.num_experts, dtype=torch.float32)
@@ -140,7 +142,7 @@ class AfmoeMoE(nn.Module):
num_tokens, hidden_dim = hidden_states.shape
hidden_states = hidden_states.view(-1, hidden_dim)
router_logits = self.gate(hidden_states.to(dtype=torch.float32))
router_logits, _ = self.gate(hidden_states)
final_hidden_states = self.experts(
hidden_states=hidden_states, router_logits=router_logits
+9 -5
View File
@@ -41,7 +41,10 @@ from vllm.distributed import (
)
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
@@ -243,11 +246,12 @@ class BailingMoE(nn.Module):
else:
self.router_dtype = torch.bfloat16
self.gate = nn.Linear(
self.gate = GateLinear(
self.hidden_size,
self.num_experts,
bias=False,
dtype=self.router_dtype,
out_dtype=self.router_dtype,
params_dtype=self.router_dtype,
prefix=f"{prefix}.gate",
)
if getattr(config, "moe_router_enable_expert_bias", False):
@@ -312,7 +316,7 @@ class BailingMoE(nn.Module):
hidden_states = hidden_states.view(-1, hidden_size)
# router_logits: (num_tokens, n_experts)
router_logits = self.gate(hidden_states.to(self.router_dtype))
router_logits, _ = self.gate(hidden_states)
router_logits = router_logits.to(hidden_states.dtype)
final_hidden_states = self.experts(
@@ -5,7 +5,6 @@ from collections.abc import Callable, Iterable
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers.configuration_utils import PretrainedConfig
from vllm.compilation.decorators import support_torch_compile
@@ -19,6 +18,7 @@ from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -229,37 +229,6 @@ class BailingMoeV25MLAAttention(nn.Module):
return self.mla_attn(positions, hidden_states)
class BailingMoEGate(nn.Module):
def __init__(
self,
config: PretrainedConfig,
params_dtype: torch.dtype | None = None,
prefix: str = "",
):
super().__init__()
if params_dtype is None:
params_dtype = torch.get_default_dtype()
self.params_dtype = params_dtype
self.weight = nn.Parameter(
torch.empty(
(config.num_experts, config.hidden_size),
dtype=self.params_dtype,
),
)
if getattr(config, "moe_router_enable_expert_bias", False):
self.expert_bias = nn.Parameter(
torch.empty((config.num_experts,), dtype=torch.float32),
)
else:
self.expert_bias = None
def forward(self, hidden_states):
logits = F.linear(hidden_states.to(self.weight.dtype), self.weight, None).to(
hidden_states.dtype
)
return logits
class BailingMoeV25(nn.Module):
"""Bailing MoE v2.5 - standalone implementation for linear attention model."""
@@ -296,11 +265,19 @@ class BailingMoeV25(nn.Module):
self.router_dtype = torch.bfloat16
# Gate for routing
self.gate = BailingMoEGate(
config=config,
self.gate = GateLinear(
self.hidden_size,
self.num_experts,
out_dtype=self.router_dtype,
params_dtype=self.router_dtype,
prefix=f"{prefix}.gate",
)
if getattr(config, "moe_router_enable_expert_bias", False):
self.gate.expert_bias = nn.Parameter(
torch.empty((self.num_experts,), dtype=torch.float32),
)
else:
self.gate.expert_bias = None
correction_bias = (
self.gate.expert_bias if self.gate.expert_bias is not None else None
)
@@ -357,7 +334,7 @@ class BailingMoeV25(nn.Module):
hidden_states = hidden_states.contiguous().view(-1, hidden_size)
# router_logits: (num_tokens, n_experts)
router_logits = self.gate(hidden_states.to(self.router_dtype))
router_logits, _ = self.gate(hidden_states)
router_logits = router_logits.to(hidden_states.dtype)
final_hidden_states = self.experts(
+12 -25
View File
@@ -16,7 +16,6 @@ from typing import TypeGuard
import regex as re
import torch
import torch.nn as nn
import torch.nn.functional as F
from einops import rearrange
from transformers.configuration_utils import PretrainedConfig
@@ -32,6 +31,7 @@ from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.activation import SiluAndMul, SwigluStepAndMul
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -1118,28 +1118,6 @@ class BailingMoeV3KimiDeltaAttention(PluggableLayer, MambaBase):
core_attn_out[0, :num_actual_tokens] = out[0, :num_actual_tokens]
class BailingMoeV3Gate(nn.Module):
def __init__(
self,
config: PretrainedConfig,
params_dtype: torch.dtype | None = None,
) -> None:
super().__init__()
if params_dtype is None:
params_dtype = torch.float32
self.weight = nn.Parameter(
torch.empty((config.num_experts, config.hidden_size), dtype=params_dtype)
)
self.expert_bias = nn.Parameter(
torch.empty((config.num_experts,), dtype=torch.float32)
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
return F.linear(hidden_states.to(self.weight.dtype), self.weight).to(
hidden_states.dtype
)
class BailingMoeV3MoE(nn.Module):
def __init__(
self,
@@ -1153,7 +1131,16 @@ class BailingMoeV3MoE(nn.Module):
self.top_k = config.num_experts_per_tok
self.hidden_size = config.hidden_size
self.routed_scaling_factor = getattr(config, "routed_scaling_factor", 1.0)
self.gate = BailingMoeV3Gate(config)
self.gate = GateLinear(
config.hidden_size,
config.num_experts,
out_dtype=torch.float32,
params_dtype=torch.float32,
prefix=f"{prefix}.gate",
)
self.gate.expert_bias = nn.Parameter(
torch.empty((config.num_experts,), dtype=torch.float32)
)
self.expert_swiglu_limit = _get_layer_swiglu_limit(
getattr(config, "expert_swiglu_limit_list", None), layer_id
)
@@ -1196,7 +1183,7 @@ class BailingMoeV3MoE(nn.Module):
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
num_tokens, hidden_size = hidden_states.shape
hidden_states = hidden_states.contiguous().view(-1, hidden_size)
router_logits = self.gate(hidden_states.to(torch.float32))
router_logits, _ = self.gate(hidden_states)
hidden_states = self.experts(
hidden_states=hidden_states, router_logits=router_logits
)
+6 -4
View File
@@ -18,7 +18,11 @@ from vllm.model_executor.layers.activation import get_act_and_mul_fn, get_act_fn
from vllm.model_executor.layers.attention import (
EncoderOnlyAttention,
)
from vllm.model_executor.layers.fused_moe import activation_without_mul, fused_topk
from vllm.model_executor.layers.fused_moe import (
GateLinear,
activation_without_mul,
fused_topk,
)
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
MergedColumnParallelLinear,
@@ -254,9 +258,7 @@ class NomicMoE(nn.Module):
params_dtype = torch.get_default_dtype()
self.params_dtype = params_dtype
self.router = ReplicatedLinear(
self.hidden_size, self.num_total_experts, bias=False
)
self.router = GateLinear(self.hidden_size, self.num_total_experts)
self.w1 = nn.Parameter(
torch.empty(
self.num_total_experts,
+5 -5
View File
@@ -16,11 +16,13 @@ from vllm.distributed import (
)
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
)
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
QKVParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -275,12 +277,10 @@ class Cohere2Moe(nn.Module):
else:
self.custom_routing_function = None
self.gate = ReplicatedLinear(
self.gate = GateLinear(
config.hidden_size,
config.num_experts,
bias=False,
params_dtype=params_dtype,
quant_config=None,
prefix=f"{prefix}.gate",
)
+2 -4
View File
@@ -18,11 +18,11 @@ from vllm.distributed import (
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
RoutedExperts,
)
from vllm.model_executor.layers.linear import (
QKVParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -62,12 +62,10 @@ class DbrxRouter(nn.Module):
self.tp_size = get_tensor_model_parallel_world_size()
self.num_total_experts = config.ffn_config.moe_num_experts
self.d_model = config.d_model
self.layer = ReplicatedLinear(
self.layer = GateLinear(
self.d_model,
self.num_total_experts,
bias=False,
params_dtype=params_dtype,
quant_config=None,
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+8 -6
View File
@@ -41,12 +41,15 @@ from vllm.distributed import (
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import FusedMoEFactory, MoERunner
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
MoERunner,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
QKVParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -150,12 +153,11 @@ class Ernie4_5_MoeMoE(nn.Module):
f"the number of experts {config.moe_num_experts}."
)
self.gate = ReplicatedLinear(
self.gate = GateLinear(
config.hidden_size,
config.moe_num_experts,
bias=False,
out_dtype=torch.float32,
params_dtype=torch.float32,
quant_config=None,
prefix=f"{prefix}.gate",
)
@@ -199,7 +201,7 @@ class Ernie4_5_MoeMoE(nn.Module):
hidden_dim = hidden_states.shape[-1]
hidden_states = hidden_states.view(-1, hidden_dim)
router_logits, _ = self.gate(hidden_states.to(dtype=torch.float32))
router_logits, _ = self.gate(hidden_states)
final_hidden_states = self.experts(
hidden_states=hidden_states, router_logits=router_logits
+5 -5
View File
@@ -30,11 +30,13 @@ from vllm.distributed import (
get_tensor_model_parallel_world_size,
)
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
QKVParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -84,11 +86,9 @@ class ExaoneMoe(nn.Module):
f"the number of experts {config.num_experts}."
)
self.gate = ReplicatedLinear(
self.gate = GateLinear(
config.hidden_size,
config.num_experts,
bias=False,
quant_config=None,
prefix=f"{prefix}.gate",
)
+6 -7
View File
@@ -43,6 +43,7 @@ from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
@@ -137,14 +138,12 @@ class Glm4MoE(nn.Module):
f"Unsupported activation: {config.hidden_act}. "
"Only silu is supported for now."
)
# NOTE In the transformers implementation, the gate isn't an nn.Linear,
# so we cannot use ReplicatedLinear here.
# See: https://github.com/huggingface/transformers/blob/v4.55.1/src/transformers/models/glm4_moe/modeling_glm4_moe.py#L260
self.gate = nn.Linear(
self.gate = GateLinear(
config.hidden_size,
config.n_routed_experts,
bias=False,
dtype=torch.float32,
out_dtype=torch.float32,
params_dtype=torch.float32,
prefix=f"{prefix}.gate",
)
self.gate.e_score_correction_bias = nn.Parameter(
torch.empty(config.n_routed_experts, dtype=torch.float32)
@@ -212,7 +211,7 @@ class Glm4MoE(nn.Module):
hidden_states = hidden_states.view(-1, hidden_dim)
# router_logits: (num_tokens, n_experts)
router_logits = self.gate(hidden_states.to(dtype=torch.float32))
router_logits, _ = self.gate(hidden_states)
final_hidden_states = self.experts(
hidden_states=hidden_states, router_logits=router_logits
+2 -3
View File
@@ -22,6 +22,7 @@ from vllm.distributed import (
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig
@@ -29,7 +30,6 @@ from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
QKVParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -377,11 +377,10 @@ class MLPBlock(torch.nn.Module):
self.hidden_size = config.hidden_size
self.experts_per_token = config.num_experts_per_tok
self.world_size = dist.get_world_size() if dist.is_initialized() else 1
self.router = ReplicatedLinear(
self.router = GateLinear(
config.hidden_size,
config.num_local_experts,
bias=True,
quant_config=None,
prefix=f"{prefix}.router",
return_bias=False,
)
+5 -5
View File
@@ -39,11 +39,13 @@ from vllm.distributed import (
tensor_model_parallel_all_gather,
)
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
QKVParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -94,12 +96,10 @@ class GraniteMoeMoE(nn.Module):
self.is_sequence_parallel = is_sequence_parallel
# Gate always runs at half / full precision for now.
self.gate = ReplicatedLinear(
self.gate = GateLinear(
hidden_size,
num_experts,
bias=False,
params_dtype=params_dtype,
quant_config=None,
prefix=f"{prefix}.gate",
)
+2 -3
View File
@@ -43,12 +43,12 @@ from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
QKVParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -181,10 +181,9 @@ class InternS1ProMoeSparseMoeBlock(nn.Module):
custom_routing_function=self._custom_routing_function,
)
self.gate = ReplicatedLinear(
self.gate = GateLinear(
config.hidden_size,
config.num_experts,
bias=False,
prefix=f"{prefix}.gate",
)
@@ -17,7 +17,10 @@ from vllm.distributed import (
tensor_model_parallel_all_gather,
tensor_model_parallel_reduce_scatter,
)
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
)
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import (
@@ -93,11 +96,9 @@ class InternS2MobiusMetaMoeBlock(nn.Module):
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
self.gate = ReplicatedLinear(
self.gate = GateLinear(
config.hidden_size,
config.num_experts,
bias=False,
quant_config=None,
prefix=f"{prefix}.gate",
)
self.experts = FusedMoEFactory(
+2 -4
View File
@@ -16,11 +16,11 @@ from vllm.distributed.parallel_state import get_pp_group
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
QKVParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -74,11 +74,9 @@ class JambaMoE(nn.Module):
self.intermediate_size = config.intermediate_size
if self.num_total_experts > 1:
self.router = ReplicatedLinear(
self.router = GateLinear(
self.hidden_size,
self.num_total_experts,
bias=False,
quant_config=None,
params_dtype=params_dtype,
prefix=f"{prefix}.router",
)
+2 -1
View File
@@ -49,6 +49,7 @@ from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.config import (
@@ -398,7 +399,7 @@ class K2HorizonSparseMoeBlock(nn.Module):
self.physical_expert_start + self.n_local_physical_experts
)
self.gate = ReplicatedLinear(
self.gate = GateLinear(
config.hidden_size,
config.num_experts,
bias=config.moe_gate_bias,
+5 -5
View File
@@ -19,12 +19,14 @@ from vllm.distributed import (
)
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
QKVParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -171,11 +173,9 @@ class LagunaMoE(nn.Module):
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
# Router gate
self.gate = ReplicatedLinear(
self.gate = GateLinear(
config.hidden_size,
config.num_experts,
bias=False,
quant_config=None,
prefix=f"{prefix}.gate",
)
+2 -3
View File
@@ -17,12 +17,12 @@ from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
QKVParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -125,10 +125,9 @@ class Lfm2MoeSparseMoeBlock(nn.Module):
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
self.gate = ReplicatedLinear(
self.gate = GateLinear(
config.hidden_size,
config.num_experts,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.gate",
)
+2 -4
View File
@@ -38,12 +38,12 @@ from vllm.model_executor.layers.attention import (
)
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
QKVParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -98,11 +98,9 @@ class Llama4MoE(nn.Module):
self.ep_size = self.ep_group.size()
intermediate_size_moe = config.intermediate_size
self.router = ReplicatedLinear(
self.router = GateLinear(
config.hidden_size,
config.num_local_experts,
bias=False,
quant_config=None,
prefix=f"{prefix}.router",
)
+4 -6
View File
@@ -48,12 +48,12 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -254,12 +254,12 @@ class LongcatRouter(nn.Module):
else config.num_experts[0]
)
self.n_routed_experts = self.n_routed_experts + zero_expert_num
self.classifier = ReplicatedLinear(
self.classifier = GateLinear(
config.hidden_size,
self.n_routed_experts,
bias=config.router_bias,
out_dtype=router_params_dtype,
params_dtype=router_params_dtype,
quant_config=None,
prefix=f"{prefix}.classifier",
)
self.e_score_correction_bias = nn.Parameter(
@@ -331,9 +331,7 @@ class LongcatMoe(nn.Module):
else:
hidden_states_padded = hidden_states
router_logits_full = self.router(
hidden_states_padded.to(self.router_params_dtype)
)
router_logits_full = self.router(hidden_states_padded)
# MoERunner handles routing memoization and zero expert computation
# internally. Pass full router_logits (including zero experts) so that
+6 -5
View File
@@ -43,12 +43,15 @@ from vllm.distributed import (
)
from vllm.model_executor.layers.activation import FatreluAndMul, SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import fused_experts, fused_topk
from vllm.model_executor.layers.fused_moe import (
GateLinear,
fused_experts,
fused_topk,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
QKVParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -109,12 +112,10 @@ class MiniCPMMoE(nn.Module):
params_dtype = torch.get_default_dtype()
self.params_dtype = params_dtype
self.gate = ReplicatedLinear(
self.gate = GateLinear(
self.hidden_size,
self.num_total_experts,
bias=False,
params_dtype=self.params_dtype,
quant_config=None,
prefix=f"{prefix}.gate",
)
+5 -5
View File
@@ -39,11 +39,13 @@ from vllm.distributed import (
get_tensor_model_parallel_world_size,
)
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
QKVParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -106,12 +108,10 @@ class MixtralMoE(nn.Module):
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
# Gate always runs at half / full precision for now.
self.gate = ReplicatedLinear(
self.gate = GateLinear(
hidden_size,
num_experts,
bias=False,
params_dtype=params_dtype,
quant_config=None,
prefix=f"{prefix}.gate",
)
+6 -4
View File
@@ -28,12 +28,15 @@ from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.attention.mm_encoder_attention import (
MMEncoderAttention,
)
from vllm.model_executor.layers.fused_moe import MoEActivation, fused_experts
from vllm.model_executor.layers.fused_moe import (
GateLinear,
MoEActivation,
fused_experts,
)
from vllm.model_executor.layers.fused_moe.config import biased_moe_quant_config
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
QKVParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -453,11 +456,10 @@ class Moondream3TextMoE(nn.Module):
self.num_local_experts = self.experts_per_rank
# Router (gate) - use ReplicatedLinear for compatibility
self.gate = ReplicatedLinear(
self.gate = GateLinear(
hidden_size,
num_experts,
bias=True,
quant_config=None,
prefix=f"{prefix}.gate",
)
+5 -5
View File
@@ -32,11 +32,13 @@ from vllm.distributed import (
from vllm.distributed.utils import split_tensor_along_last_dim
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
QKVParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -84,11 +86,9 @@ class OlmoeMoE(nn.Module):
self.hidden_size = hidden_size
# Gate always runs at half / full precision for now.
self.gate = ReplicatedLinear(
self.gate = GateLinear(
hidden_size,
num_experts,
bias=False,
quant_config=None,
prefix=f"{prefix}.gate",
)
+5 -4
View File
@@ -43,7 +43,10 @@ from vllm.model_executor.layers.attention import (
Attention,
StaticSinkAttention,
)
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
@@ -149,11 +152,9 @@ class OpenPanguMoE(nn.Module):
self.is_sequence_parallel = parallel_config.use_sequence_parallel_moe
check_ffn_act_fn(config.hidden_act)
self.gate = ReplicatedLinear(
self.gate = GateLinear(
config.hidden_size,
config.n_routed_experts,
bias=False,
quant_config=None,
prefix=f"{prefix}.gate",
)
if (
+10 -12
View File
@@ -22,7 +22,6 @@ from collections.abc import Iterable, Iterator
from itertools import islice
import torch
import torch.nn.functional as F
from torch import nn
from vllm.config import CacheConfig, VllmConfig
@@ -34,6 +33,7 @@ from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
MoERunner,
)
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -289,10 +289,12 @@ class Param2MoEMoEBlock(nn.Module):
self.norm_expert_prob: bool = getattr(config, "norm_topk_prob", True)
self.score_function: str = getattr(config, "score_function", "sigmoid")
self.gate = nn.Linear(
self.gate = GateLinear(
self.hidden_size,
self.num_experts,
bias=False,
out_dtype=torch.float32,
params_dtype=torch.float32,
prefix=f"{prefix}.gate",
)
if getattr(config, "moe_router_enable_expert_bias", True):
@@ -350,15 +352,11 @@ class Param2MoEMoEBlock(nn.Module):
num_tokens, hidden_dim = hidden_states.shape
hidden_states = hidden_states.view(-1, hidden_dim)
# Router: both input and weight must be float32 for numerical
# stability (mirrors the original Param2MoEGate behaviour).
# The gate nn.Linear weight lives in the model dtype (bfloat16),
# so we must cast both explicitly via F.linear instead of calling
# self.gate() which would hit a dtype mismatch.
router_logits = F.linear(
hidden_states.float(),
self.gate.weight.float(),
).to(hidden_states.dtype)
# Router logits are accumulated in float32 for numerical stability
# (mirrors the original Param2MoEGate behaviour) and handed to the
# experts in the activation dtype.
router_logits, _ = self.gate(hidden_states)
router_logits = router_logits.to(hidden_states.dtype)
expert_output = self.experts(
hidden_states=hidden_states,
+5 -5
View File
@@ -35,10 +35,12 @@ from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
)
from vllm.model_executor.layers.linear import (
QKVParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -262,12 +264,10 @@ class PhiMoE(nn.Module):
self.hidden_size = hidden_size
# Gate always runs at half / full precision for now.
self.gate = ReplicatedLinear(
self.gate = GateLinear(
hidden_size,
num_experts,
bias=False,
params_dtype=params_dtype,
quant_config=None,
prefix=f"{prefix}.gate",
)
+5 -4
View File
@@ -40,7 +40,10 @@ from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
@@ -136,11 +139,9 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
f"the number of experts {config.num_experts}."
)
self.gate = ReplicatedLinear(
self.gate = GateLinear(
config.hidden_size,
config.num_experts,
bias=False,
quant_config=None,
prefix=f"{prefix}.gate",
)
+5 -3
View File
@@ -42,7 +42,10 @@ from vllm.distributed import (
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
@@ -164,10 +167,9 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
self.gate = ReplicatedLinear(
self.gate = GateLinear(
config.hidden_size,
config.num_experts,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.gate",
)
+5 -4
View File
@@ -18,7 +18,10 @@ from vllm.distributed import (
tensor_model_parallel_reduce_scatter,
)
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
resolve_layer_fused_shared_expert,
@@ -180,11 +183,9 @@ class Qwen3NextSparseMoeBlock(nn.Module):
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
self.gate = ReplicatedLinear(
self.gate = GateLinear(
config.hidden_size,
config.num_experts,
bias=False,
quant_config=None,
prefix=f"{prefix}.gate",
)
+2 -3
View File
@@ -20,6 +20,7 @@ from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
@@ -77,11 +78,9 @@ class FusedMoEBlock(nn.Module):
quant_config=quant_config,
prefix=f"{prefix}.experts",
)
self.gate = ReplicatedLinear(
self.gate = GateLinear(
config.hidden_size,
config.moe_num_experts,
bias=False,
quant_config=None,
prefix=f"{prefix}.gate",
)
+5 -17
View File
@@ -8,7 +8,6 @@ from typing import Any
import torch
from torch import nn
from torch.nn.parameter import Parameter
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
@@ -25,6 +24,7 @@ from vllm.model_executor.layers.activation import SiluAndMul, SwigluStepAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
MoERunner,
fused_moe_make_expert_params_mapping,
)
@@ -33,7 +33,6 @@ from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
MergedColumnParallelLinear,
QKVParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -68,17 +67,6 @@ from .utils import (
logger = init_logger(__name__)
class FP32ReplicatedLinear(ReplicatedLinear):
"""Use FP32 for higher precision."""
def forward(
self,
x: torch.Tensor,
) -> torch.Tensor | tuple[torch.Tensor, Parameter | None]:
assert self.params_dtype == torch.float32
return super().forward(x.to(torch.float32))
class Step3p5MLP(nn.Module):
def __init__(
self,
@@ -326,12 +314,12 @@ class FusedMoEBlock(nn.Module):
f"the number of experts {config.moe_num_experts}."
)
self.gate = FP32ReplicatedLinear(
# Router logits are accumulated in FP32 for higher precision.
self.gate = GateLinear(
config.hidden_size,
config.moe_num_experts,
bias=False,
quant_config=None,
params_dtype=torch.float32, # Use FP32 for higher precision.
out_dtype=torch.float32,
params_dtype=torch.float32,
prefix=f"{prefix}.gate",
)
self.use_moe_router_bias = config.use_moe_router_bias