[ROCm][Perf] Add Fused Shared Expert (FSE) support for Qwen3-Next (#39280)

Signed-off-by: nholmber <nholmber@users.noreply.github.com>
Signed-off-by: Tres Popp <tres.popp@amd.com>
Signed-off-by: Doug Lehr <douglehr@amd.com>
Co-authored-by: nholmber <nholmber@users.noreply.github.com>
Co-authored-by: Tres <tpopp@users.noreply.github.com>
Co-authored-by: Tres Popp <tres.popp@amd.com>
Co-authored-by: Doug Lehr <douglehr@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
Nico Holmberg
2026-05-08 15:38:00 -04:00
committed by GitHub
co-authored by nholmber Tres Tres Popp Doug Lehr Cursor Douglas Lehr
parent 44e6b44a21
commit 2c6b59b807
8 changed files with 359 additions and 10 deletions
+26 -4
View File
@@ -8,6 +8,7 @@ from itertools import islice
import torch
from torch import nn
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.decorators import support_torch_compile
from vllm.config import (
CacheConfig,
@@ -135,7 +136,12 @@ class Qwen3NextSparseMoeBlock(nn.Module):
prefix=f"{prefix}.shared_expert_gate",
)
if config.shared_expert_intermediate_size > 0:
if (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
or config.shared_expert_intermediate_size <= 0
):
self.shared_expert = None
else:
self.shared_expert = Qwen3NextMLP(
hidden_size=config.hidden_size,
intermediate_size=config.shared_expert_intermediate_size,
@@ -146,8 +152,6 @@ class Qwen3NextSparseMoeBlock(nn.Module):
is_sequence_parallel=self.is_sequence_parallel,
prefix=f"{prefix}.shared_expert",
)
else:
self.shared_expert = None
self.experts = FusedMoE(
shared_experts=self.shared_expert,
@@ -162,6 +166,10 @@ class Qwen3NextSparseMoeBlock(nn.Module):
enable_eplb=self.enable_eplb,
num_redundant_experts=self.n_redundant_experts,
is_sequence_parallel=self.is_sequence_parallel,
n_shared_experts=1 if self.shared_expert is None else None,
shared_expert_gate=self.shared_expert_gate
if self.shared_expert is None
else None,
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -536,12 +544,15 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
# Params for weights, fp8 weight scales, fp8 activation scales
# (param_name, weight_name, expert_id, shard_id)
num_experts = getattr(self.config, "num_experts", 0)
if rocm_aiter_ops.is_fusion_moe_shared_experts_enabled():
num_experts += 1
return fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=getattr(self.config, "num_experts", 0),
num_experts=num_experts,
num_redundant_experts=self.num_redundant_experts,
)
@@ -558,6 +569,10 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
expert_params_mapping = self.get_expert_mapping()
is_fse = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
num_routed = getattr(self.config, "num_experts", 0)
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name:
continue
@@ -571,6 +586,13 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
if name is None:
continue
# FSE: remap shared_expert weights to the fused expert slot
if is_fse and "mlp.shared_expert." in name:
name = name.replace(
"mlp.shared_expert.",
f"mlp.experts.{num_routed}.",
)
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
+14 -1
View File
@@ -7,6 +7,7 @@ from collections.abc import Iterable
import torch
from torch import nn
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
from vllm.distributed.parallel_state import get_pp_group
@@ -147,13 +148,18 @@ class Qwen3NextMultiTokenPredictor(nn.Module):
# Params for weights, fp8 weight scales, fp8 activation scales
# (param_name, weight_name, expert_id, shard_id)
is_fse = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
num_experts = self.config.num_experts
if is_fse:
num_experts += 1
expert_params_mapping = fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=self.config.num_experts,
num_experts=num_experts,
)
num_routed = self.config.num_experts
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
@@ -161,6 +167,13 @@ class Qwen3NextMultiTokenPredictor(nn.Module):
if "rotary_emb.inv_freq" in name:
continue
# FSE: remap shared_expert weights to the fused expert slot
if is_fse and "mlp.shared_expert." in name:
name = name.replace(
"mlp.shared_expert.",
f"mlp.experts.{num_routed}.",
)
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue