[ROCm][Perf] Add Fused Shared Expert (FSE) support for Qwen3-Next (#39280)
Signed-off-by: nholmber <nholmber@users.noreply.github.com> Signed-off-by: Tres Popp <tres.popp@amd.com> Signed-off-by: Doug Lehr <douglehr@amd.com> Co-authored-by: nholmber <nholmber@users.noreply.github.com> Co-authored-by: Tres <tpopp@users.noreply.github.com> Co-authored-by: Tres Popp <tres.popp@amd.com> Co-authored-by: Doug Lehr <douglehr@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
co-authored by
nholmber
Tres
Tres Popp
Doug Lehr
Cursor
Douglas Lehr
parent
44e6b44a21
commit
2c6b59b807
@@ -8,6 +8,7 @@ from itertools import islice
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import (
|
||||
CacheConfig,
|
||||
@@ -135,7 +136,12 @@ class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
prefix=f"{prefix}.shared_expert_gate",
|
||||
)
|
||||
|
||||
if config.shared_expert_intermediate_size > 0:
|
||||
if (
|
||||
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
or config.shared_expert_intermediate_size <= 0
|
||||
):
|
||||
self.shared_expert = None
|
||||
else:
|
||||
self.shared_expert = Qwen3NextMLP(
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.shared_expert_intermediate_size,
|
||||
@@ -146,8 +152,6 @@ class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
is_sequence_parallel=self.is_sequence_parallel,
|
||||
prefix=f"{prefix}.shared_expert",
|
||||
)
|
||||
else:
|
||||
self.shared_expert = None
|
||||
|
||||
self.experts = FusedMoE(
|
||||
shared_experts=self.shared_expert,
|
||||
@@ -162,6 +166,10 @@ class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
enable_eplb=self.enable_eplb,
|
||||
num_redundant_experts=self.n_redundant_experts,
|
||||
is_sequence_parallel=self.is_sequence_parallel,
|
||||
n_shared_experts=1 if self.shared_expert is None else None,
|
||||
shared_expert_gate=self.shared_expert_gate
|
||||
if self.shared_expert is None
|
||||
else None,
|
||||
)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
@@ -536,12 +544,15 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
num_experts = getattr(self.config, "num_experts", 0)
|
||||
if rocm_aiter_ops.is_fusion_moe_shared_experts_enabled():
|
||||
num_experts += 1
|
||||
return fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=getattr(self.config, "num_experts", 0),
|
||||
num_experts=num_experts,
|
||||
num_redundant_experts=self.num_redundant_experts,
|
||||
)
|
||||
|
||||
@@ -558,6 +569,10 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
expert_params_mapping = self.get_expert_mapping()
|
||||
|
||||
is_fse = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
num_routed = getattr(self.config, "num_experts", 0)
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
@@ -571,6 +586,13 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
|
||||
if name is None:
|
||||
continue
|
||||
|
||||
# FSE: remap shared_expert weights to the fused expert slot
|
||||
if is_fse and "mlp.shared_expert." in name:
|
||||
name = name.replace(
|
||||
"mlp.shared_expert.",
|
||||
f"mlp.experts.{num_routed}.",
|
||||
)
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
|
||||
@@ -7,6 +7,7 @@ from collections.abc import Iterable
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
@@ -147,13 +148,18 @@ class Qwen3NextMultiTokenPredictor(nn.Module):
|
||||
|
||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
is_fse = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
num_experts = self.config.num_experts
|
||||
if is_fse:
|
||||
num_experts += 1
|
||||
expert_params_mapping = fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=self.config.num_experts,
|
||||
num_experts=num_experts,
|
||||
)
|
||||
num_routed = self.config.num_experts
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
@@ -161,6 +167,13 @@ class Qwen3NextMultiTokenPredictor(nn.Module):
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
|
||||
# FSE: remap shared_expert weights to the fused expert slot
|
||||
if is_fse and "mlp.shared_expert." in name:
|
||||
name = name.replace(
|
||||
"mlp.shared_expert.",
|
||||
f"mlp.experts.{num_routed}.",
|
||||
)
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
|
||||
Reference in New Issue
Block a user