[ROCm][Perf] Route the fused shared-expert gate GEMM through the platform dispatcher (#54185)
Signed-off-by: Mikko Tukiainen <Mikko.Tukiainen@amd.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
fba33e39cb
commit
17322375a2
@@ -45,6 +45,7 @@ from vllm.model_executor.layers.fused_moe.runner.shared_experts import (
|
||||
SharedExperts,
|
||||
SharedExpertsOrder,
|
||||
)
|
||||
from vllm.model_executor.layers.utils import dispatch_unquantized_gemm
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import (
|
||||
_USE_LAYERNAME,
|
||||
@@ -273,7 +274,7 @@ class MoERunner(MoERunnerInterface):
|
||||
|
||||
# When both gates are present and FSE is enabled, fuse their
|
||||
# weight matrices into [num_experts + num_shared, hidden] so one
|
||||
# F.linear produces combined logits. The topk kernel can then
|
||||
# GEMM produces combined logits. The topk kernel can then
|
||||
# apply routing softmax and shared expert activation (sigmoid)
|
||||
# in a single launch.
|
||||
self._fse_fuse_gate = gate is not None and shared_expert_gate is not None
|
||||
@@ -898,7 +899,9 @@ class MoERunner(MoERunnerInterface):
|
||||
if self.gate is not None:
|
||||
if self._fse_fuse_gate:
|
||||
self._maybe_fuse_gate_weights()
|
||||
router_logits = F.linear(hidden_states, self._combined_gate_weight)
|
||||
router_logits = dispatch_unquantized_gemm()(
|
||||
self, hidden_states, self._combined_gate_weight, None
|
||||
)
|
||||
else:
|
||||
router_logits, _ = self.gate(hidden_states)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user