diff --git a/vllm/model_executor/layers/fused_moe/runner/moe_runner.py b/vllm/model_executor/layers/fused_moe/runner/moe_runner.py index 8d3bc79428..870d04c0c8 100644 --- a/vllm/model_executor/layers/fused_moe/runner/moe_runner.py +++ b/vllm/model_executor/layers/fused_moe/runner/moe_runner.py @@ -45,6 +45,7 @@ from vllm.model_executor.layers.fused_moe.runner.shared_experts import ( SharedExperts, SharedExpertsOrder, ) +from vllm.model_executor.layers.utils import dispatch_unquantized_gemm from vllm.platforms import current_platform from vllm.utils.torch_utils import ( _USE_LAYERNAME, @@ -273,7 +274,7 @@ class MoERunner(MoERunnerInterface): # When both gates are present and FSE is enabled, fuse their # weight matrices into [num_experts + num_shared, hidden] so one - # F.linear produces combined logits. The topk kernel can then + # GEMM produces combined logits. The topk kernel can then # apply routing softmax and shared expert activation (sigmoid) # in a single launch. self._fse_fuse_gate = gate is not None and shared_expert_gate is not None @@ -898,7 +899,9 @@ class MoERunner(MoERunnerInterface): if self.gate is not None: if self._fse_fuse_gate: self._maybe_fuse_gate_weights() - router_logits = F.linear(hidden_states, self._combined_gate_weight) + router_logits = dispatch_unquantized_gemm()( + self, hidden_states, self._combined_gate_weight, None + ) else: router_logits, _ = self.gate(hidden_states)