[ROCm][Perf] Add Fused Shared Expert (FSE) support for Qwen3-Next (#39280)

Signed-off-by: nholmber <nholmber@users.noreply.github.com>
Signed-off-by: Tres Popp <tres.popp@amd.com>
Signed-off-by: Doug Lehr <douglehr@amd.com>
Co-authored-by: nholmber <nholmber@users.noreply.github.com>
Co-authored-by: Tres <tpopp@users.noreply.github.com>
Co-authored-by: Tres Popp <tres.popp@amd.com>
Co-authored-by: Doug Lehr <douglehr@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
Nico Holmberg
2026-05-08 15:38:00 -04:00
committed by GitHub
co-authored by nholmber Tres Tres Popp Doug Lehr Cursor Douglas Lehr
parent 44e6b44a21
commit 2c6b59b807
8 changed files with 359 additions and 10 deletions
@@ -223,6 +223,7 @@ class MoERunner(MoERunnerInterface):
shared_experts: torch.nn.Module | None,
quant_method: FusedMoEMethodBase,
enable_dbo: bool,
shared_expert_gate: torch.nn.Module | None = None,
routed_output_transform: torch.nn.Module | None = None,
routed_scaling_factor: float = 1.0,
):
@@ -233,9 +234,18 @@ class MoERunner(MoERunnerInterface):
self.routed_output_transform = routed_output_transform
self.routed_scaling_factor = routed_scaling_factor
self.gate = gate
self.shared_expert_gate = shared_expert_gate
self._quant_method = quant_method
self.enable_dbo = enable_dbo
# When both gates are present and FSE is enabled, fuse their
# weight matrices into [num_experts + num_shared, hidden] so one
# F.linear produces combined logits. The topk kernel can then
# apply routing softmax and shared expert activation (sigmoid)
# in a single launch.
self._fse_fuse_gate = gate is not None and shared_expert_gate is not None
self._combined_gate_weight: torch.Tensor | None = None
self._shared_experts: SharedExperts | None = None
if shared_experts is not None:
self._shared_experts = SharedExperts(
@@ -278,6 +288,20 @@ class MoERunner(MoERunnerInterface):
self._shared_experts._quant_method = quant_method
self._quant_method = quant_method
def _maybe_fuse_gate_weights(self):
"""Fuse router and shared expert gate weights on first call.
Cannot be done at __init__ because gate weights are loaded after
module construction (via weight_loader). Called once from
_forward_impl before the first forward pass.
"""
if self._combined_gate_weight is None:
assert self.gate is not None and self.shared_expert_gate is not None
self._combined_gate_weight = torch.cat(
[self.gate.weight, self.shared_expert_gate.weight],
dim=0,
)
def is_internal_router(self) -> bool:
return self.gate is not None
@@ -752,7 +776,11 @@ class MoERunner(MoERunnerInterface):
# so it can run overlapped with the
# NOTE: in future PR, MoE runner will always hold the gate.
if self.gate is not None:
router_logits, _ = self.gate(hidden_states)
if self._fse_fuse_gate:
self._maybe_fuse_gate_weights()
router_logits = F.linear(hidden_states, self._combined_gate_weight)
else:
router_logits, _ = self.gate(hidden_states)
with self._sequence_parallel_context():
# TODO(bnell): parts of the dispatch/combine steps will go away once