[MoE] Use GateLinear for all MoE models (#58234)

Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
Jee Jee Li
2026-09-23 05:40:13 +00:00
committed by GitHub
parent e975732152
commit de6674b002
34 changed files with 221 additions and 225 deletions
+10 -12
View File
@@ -22,7 +22,6 @@ from collections.abc import Iterable, Iterator
from itertools import islice
import torch
import torch.nn.functional as F
from torch import nn
from vllm.config import CacheConfig, VllmConfig
@@ -34,6 +33,7 @@ from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
MoERunner,
)
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -289,10 +289,12 @@ class Param2MoEMoEBlock(nn.Module):
self.norm_expert_prob: bool = getattr(config, "norm_topk_prob", True)
self.score_function: str = getattr(config, "score_function", "sigmoid")
self.gate = nn.Linear(
self.gate = GateLinear(
self.hidden_size,
self.num_experts,
bias=False,
out_dtype=torch.float32,
params_dtype=torch.float32,
prefix=f"{prefix}.gate",
)
if getattr(config, "moe_router_enable_expert_bias", True):
@@ -350,15 +352,11 @@ class Param2MoEMoEBlock(nn.Module):
num_tokens, hidden_dim = hidden_states.shape
hidden_states = hidden_states.view(-1, hidden_dim)
# Router: both input and weight must be float32 for numerical
# stability (mirrors the original Param2MoEGate behaviour).
# The gate nn.Linear weight lives in the model dtype (bfloat16),
# so we must cast both explicitly via F.linear instead of calling
# self.gate() which would hit a dtype mismatch.
router_logits = F.linear(
hidden_states.float(),
self.gate.weight.float(),
).to(hidden_states.dtype)
# Router logits are accumulated in float32 for numerical stability
# (mirrors the original Param2MoEGate behaviour) and handed to the
# experts in the activation dtype.
router_logits, _ = self.gate(hidden_states)
router_logits = router_logits.to(hidden_states.dtype)
expert_output = self.experts(
hidden_states=hidden_states,