[MoE] Use GateLinear for all MoE models (#58234)
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
@@ -22,7 +22,6 @@ from collections.abc import Iterable, Iterator
|
||||
from itertools import islice
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
@@ -34,6 +33,7 @@ from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEFactory,
|
||||
GateLinear,
|
||||
MoERunner,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -289,10 +289,12 @@ class Param2MoEMoEBlock(nn.Module):
|
||||
self.norm_expert_prob: bool = getattr(config, "norm_topk_prob", True)
|
||||
self.score_function: str = getattr(config, "score_function", "sigmoid")
|
||||
|
||||
self.gate = nn.Linear(
|
||||
self.gate = GateLinear(
|
||||
self.hidden_size,
|
||||
self.num_experts,
|
||||
bias=False,
|
||||
out_dtype=torch.float32,
|
||||
params_dtype=torch.float32,
|
||||
prefix=f"{prefix}.gate",
|
||||
)
|
||||
|
||||
if getattr(config, "moe_router_enable_expert_bias", True):
|
||||
@@ -350,15 +352,11 @@ class Param2MoEMoEBlock(nn.Module):
|
||||
num_tokens, hidden_dim = hidden_states.shape
|
||||
hidden_states = hidden_states.view(-1, hidden_dim)
|
||||
|
||||
# Router: both input and weight must be float32 for numerical
|
||||
# stability (mirrors the original Param2MoEGate behaviour).
|
||||
# The gate nn.Linear weight lives in the model dtype (bfloat16),
|
||||
# so we must cast both explicitly via F.linear instead of calling
|
||||
# self.gate() which would hit a dtype mismatch.
|
||||
router_logits = F.linear(
|
||||
hidden_states.float(),
|
||||
self.gate.weight.float(),
|
||||
).to(hidden_states.dtype)
|
||||
# Router logits are accumulated in float32 for numerical stability
|
||||
# (mirrors the original Param2MoEGate behaviour) and handed to the
|
||||
# experts in the activation dtype.
|
||||
router_logits, _ = self.gate(hidden_states)
|
||||
router_logits = router_logits.to(hidden_states.dtype)
|
||||
|
||||
expert_output = self.experts(
|
||||
hidden_states=hidden_states,
|
||||
|
||||
Reference in New Issue
Block a user