[MoE] Use GateLinear for all MoE models (#58234)

Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
Jee Jee Li
2026-09-23 05:40:13 +00:00
committed by GitHub
parent e975732152
commit de6674b002
34 changed files with 221 additions and 225 deletions
+5 -17
View File
@@ -8,7 +8,6 @@ from typing import Any
import torch
from torch import nn
from torch.nn.parameter import Parameter
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
@@ -25,6 +24,7 @@ from vllm.model_executor.layers.activation import SiluAndMul, SwigluStepAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
MoERunner,
fused_moe_make_expert_params_mapping,
)
@@ -33,7 +33,6 @@ from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
MergedColumnParallelLinear,
QKVParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -68,17 +67,6 @@ from .utils import (
logger = init_logger(__name__)
class FP32ReplicatedLinear(ReplicatedLinear):
"""Use FP32 for higher precision."""
def forward(
self,
x: torch.Tensor,
) -> torch.Tensor | tuple[torch.Tensor, Parameter | None]:
assert self.params_dtype == torch.float32
return super().forward(x.to(torch.float32))
class Step3p5MLP(nn.Module):
def __init__(
self,
@@ -326,12 +314,12 @@ class FusedMoEBlock(nn.Module):
f"the number of experts {config.moe_num_experts}."
)
self.gate = FP32ReplicatedLinear(
# Router logits are accumulated in FP32 for higher precision.
self.gate = GateLinear(
config.hidden_size,
config.moe_num_experts,
bias=False,
quant_config=None,
params_dtype=torch.float32, # Use FP32 for higher precision.
out_dtype=torch.float32,
params_dtype=torch.float32,
prefix=f"{prefix}.gate",
)
self.use_moe_router_bias = config.use_moe_router_bias