[ROCm][Perf] Add Fused Shared Expert (FSE) support for Qwen3-Next (#39280)
Signed-off-by: nholmber <nholmber@users.noreply.github.com> Signed-off-by: Tres Popp <tres.popp@amd.com> Signed-off-by: Doug Lehr <douglehr@amd.com> Co-authored-by: nholmber <nholmber@users.noreply.github.com> Co-authored-by: Tres <tpopp@users.noreply.github.com> Co-authored-by: Tres Popp <tres.popp@amd.com> Co-authored-by: Doug Lehr <douglehr@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
co-authored by
nholmber
Tres
Tres Popp
Doug Lehr
Cursor
Douglas Lehr
parent
44e6b44a21
commit
2c6b59b807
@@ -223,6 +223,7 @@ class MoERunner(MoERunnerInterface):
|
||||
shared_experts: torch.nn.Module | None,
|
||||
quant_method: FusedMoEMethodBase,
|
||||
enable_dbo: bool,
|
||||
shared_expert_gate: torch.nn.Module | None = None,
|
||||
routed_output_transform: torch.nn.Module | None = None,
|
||||
routed_scaling_factor: float = 1.0,
|
||||
):
|
||||
@@ -233,9 +234,18 @@ class MoERunner(MoERunnerInterface):
|
||||
self.routed_output_transform = routed_output_transform
|
||||
self.routed_scaling_factor = routed_scaling_factor
|
||||
self.gate = gate
|
||||
self.shared_expert_gate = shared_expert_gate
|
||||
self._quant_method = quant_method
|
||||
self.enable_dbo = enable_dbo
|
||||
|
||||
# When both gates are present and FSE is enabled, fuse their
|
||||
# weight matrices into [num_experts + num_shared, hidden] so one
|
||||
# F.linear produces combined logits. The topk kernel can then
|
||||
# apply routing softmax and shared expert activation (sigmoid)
|
||||
# in a single launch.
|
||||
self._fse_fuse_gate = gate is not None and shared_expert_gate is not None
|
||||
self._combined_gate_weight: torch.Tensor | None = None
|
||||
|
||||
self._shared_experts: SharedExperts | None = None
|
||||
if shared_experts is not None:
|
||||
self._shared_experts = SharedExperts(
|
||||
@@ -278,6 +288,20 @@ class MoERunner(MoERunnerInterface):
|
||||
self._shared_experts._quant_method = quant_method
|
||||
self._quant_method = quant_method
|
||||
|
||||
def _maybe_fuse_gate_weights(self):
|
||||
"""Fuse router and shared expert gate weights on first call.
|
||||
|
||||
Cannot be done at __init__ because gate weights are loaded after
|
||||
module construction (via weight_loader). Called once from
|
||||
_forward_impl before the first forward pass.
|
||||
"""
|
||||
if self._combined_gate_weight is None:
|
||||
assert self.gate is not None and self.shared_expert_gate is not None
|
||||
self._combined_gate_weight = torch.cat(
|
||||
[self.gate.weight, self.shared_expert_gate.weight],
|
||||
dim=0,
|
||||
)
|
||||
|
||||
def is_internal_router(self) -> bool:
|
||||
return self.gate is not None
|
||||
|
||||
@@ -752,7 +776,11 @@ class MoERunner(MoERunnerInterface):
|
||||
# so it can run overlapped with the
|
||||
# NOTE: in future PR, MoE runner will always hold the gate.
|
||||
if self.gate is not None:
|
||||
router_logits, _ = self.gate(hidden_states)
|
||||
if self._fse_fuse_gate:
|
||||
self._maybe_fuse_gate_weights()
|
||||
router_logits = F.linear(hidden_states, self._combined_gate_weight)
|
||||
else:
|
||||
router_logits, _ = self.gate(hidden_states)
|
||||
|
||||
with self._sequence_parallel_context():
|
||||
# TODO(bnell): parts of the dispatch/combine steps will go away once
|
||||
|
||||
Reference in New Issue
Block a user