[MoE] Use GateLinear for all MoE models (#58234)
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
@@ -16,11 +16,11 @@ from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEFactory,
|
||||
GateLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
QKVParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
@@ -74,11 +74,9 @@ class JambaMoE(nn.Module):
|
||||
self.intermediate_size = config.intermediate_size
|
||||
|
||||
if self.num_total_experts > 1:
|
||||
self.router = ReplicatedLinear(
|
||||
self.router = GateLinear(
|
||||
self.hidden_size,
|
||||
self.num_total_experts,
|
||||
bias=False,
|
||||
quant_config=None,
|
||||
params_dtype=params_dtype,
|
||||
prefix=f"{prefix}.router",
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user