[MoE] Use GateLinear for all MoE models (#58234)
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
@@ -8,7 +8,6 @@ from typing import Any
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
@@ -25,6 +24,7 @@ from vllm.model_executor.layers.activation import SiluAndMul, SwigluStepAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEFactory,
|
||||
GateLinear,
|
||||
MoERunner,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
@@ -33,7 +33,6 @@ from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
@@ -68,17 +67,6 @@ from .utils import (
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class FP32ReplicatedLinear(ReplicatedLinear):
|
||||
"""Use FP32 for higher precision."""
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, Parameter | None]:
|
||||
assert self.params_dtype == torch.float32
|
||||
return super().forward(x.to(torch.float32))
|
||||
|
||||
|
||||
class Step3p5MLP(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -326,12 +314,12 @@ class FusedMoEBlock(nn.Module):
|
||||
f"the number of experts {config.moe_num_experts}."
|
||||
)
|
||||
|
||||
self.gate = FP32ReplicatedLinear(
|
||||
# Router logits are accumulated in FP32 for higher precision.
|
||||
self.gate = GateLinear(
|
||||
config.hidden_size,
|
||||
config.moe_num_experts,
|
||||
bias=False,
|
||||
quant_config=None,
|
||||
params_dtype=torch.float32, # Use FP32 for higher precision.
|
||||
out_dtype=torch.float32,
|
||||
params_dtype=torch.float32,
|
||||
prefix=f"{prefix}.gate",
|
||||
)
|
||||
self.use_moe_router_bias = config.use_moe_router_bias
|
||||
|
||||
Reference in New Issue
Block a user