[MoE] Use GateLinear for all MoE models (#58234)

Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
Jee Jee Li
2026-09-23 05:40:13 +00:00
committed by GitHub
parent e975732152
commit de6674b002
34 changed files with 221 additions and 225 deletions
+6 -4
View File
@@ -18,7 +18,11 @@ from vllm.model_executor.layers.activation import get_act_and_mul_fn, get_act_fn
from vllm.model_executor.layers.attention import (
EncoderOnlyAttention,
)
from vllm.model_executor.layers.fused_moe import activation_without_mul, fused_topk
from vllm.model_executor.layers.fused_moe import (
GateLinear,
activation_without_mul,
fused_topk,
)
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
MergedColumnParallelLinear,
@@ -254,9 +258,7 @@ class NomicMoE(nn.Module):
params_dtype = torch.get_default_dtype()
self.params_dtype = params_dtype
self.router = ReplicatedLinear(
self.hidden_size, self.num_total_experts, bias=False
)
self.router = GateLinear(self.hidden_size, self.num_total_experts)
self.w1 = nn.Parameter(
torch.empty(
self.num_total_experts,