[ROCm][Perf] Add Fused Shared Expert (FSE) support for Qwen3-Next (#39280)

Signed-off-by: nholmber <nholmber@users.noreply.github.com>
Signed-off-by: Tres Popp <tres.popp@amd.com>
Signed-off-by: Doug Lehr <douglehr@amd.com>
Co-authored-by: nholmber <nholmber@users.noreply.github.com>
Co-authored-by: Tres <tpopp@users.noreply.github.com>
Co-authored-by: Tres Popp <tres.popp@amd.com>
Co-authored-by: Doug Lehr <douglehr@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
Nico Holmberg
2026-05-08 15:38:00 -04:00
committed by GitHub
co-authored by nholmber Tres Tres Popp Doug Lehr Cursor Douglas Lehr
parent 44e6b44a21
commit 2c6b59b807
8 changed files with 359 additions and 10 deletions
+69 -2
View File
@@ -261,11 +261,19 @@ def _rocm_aiter_topk_softmax_impl(
token_expert_indices: torch.Tensor,
gating_output: torch.Tensor,
renormalize: bool,
num_shared_experts: int = 0,
shared_expert_scoring_func: str = "",
) -> None:
from aiter import topk_softmax
topk_softmax(
topk_weights, topk_indices, token_expert_indices, gating_output, renormalize
topk_weights,
topk_indices,
token_expert_indices,
gating_output,
renormalize,
num_shared_experts,
shared_expert_scoring_func,
)
@@ -275,6 +283,8 @@ def _rocm_aiter_topk_softmax_fake(
token_expert_indices: torch.Tensor,
gating_output: torch.Tensor,
renormalize: bool,
num_shared_experts: int = 0,
shared_expert_scoring_func: str = "",
) -> None:
pass
@@ -1206,6 +1216,9 @@ class rocm_aiter_ops:
_MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
# TODO: Consolidate under _LINEAR_ENABLED
_TRITON_UNQUANT_GEMM = envs.VLLM_ROCM_USE_AITER_TRITON_GEMM
# Lazily probed: whether aiter.topk_softmax supports the
# num_shared_experts / shared_expert_scoring_func args (7-arg form).
_TOPK_SOFTMAX_FUSED_SIGMOID: bool | None = None
_ALL_REDUCE_MAX_SIZE: int = 8192 * 1024 * 8 * 2
_CUSTOM_ALL_REDUCE: AiterCustomAllreduceProto | None = None
@@ -1322,6 +1335,52 @@ class rocm_aiter_ops:
def is_fusion_moe_shared_experts_enabled(cls) -> bool:
return cls.is_fused_moe_enabled() and cls._MOE_SHARED_EXPERTS_ENABLED
@classmethod
@if_aiter_supported
def topk_softmax_supports_fused_sigmoid(cls) -> bool:
"""Check if topk_softmax supports fused shared expert activation."""
if cls._TOPK_SOFTMAX_FUSED_SIGMOID is None:
try:
import inspect
from aiter import topk_softmax
params = inspect.signature(topk_softmax).parameters
if "num_shared_experts" in params:
cls._TOPK_SOFTMAX_FUSED_SIGMOID = True
else:
# @compile_ops wrapper loses the original signature.
# Fall back to the torch custom op schema.
import torch
schema = getattr(
getattr(torch.ops.aiter, "topk_softmax", None), "default", None
)
schema_str = str(getattr(schema, "_schema", ""))
cls._TOPK_SOFTMAX_FUSED_SIGMOID = "num_shared_experts" in schema_str
except (ImportError, ValueError):
cls._TOPK_SOFTMAX_FUSED_SIGMOID = False
return cls._TOPK_SOFTMAX_FUSED_SIGMOID
@classmethod
@if_aiter_supported
def fuse_sigmoid_in_kernel(cls, aiter_topK_meta_data: object) -> bool:
"""Whether fused shared-expert sigmoid in the topk kernel is usable.
Combines the cached static capability checks (FSE enabled, fused-moe
enabled, topk_softmax supports fused sigmoid) with the runtime
readiness check (topK meta-data buffer initialized).
``aiter_topK_meta_data`` is accepted as a parameter rather than
imported internally so callers cannot hit initialization-order
issues where the module-level global has not been set yet.
"""
return (
cls.is_fusion_moe_shared_experts_enabled()
and cls.topk_softmax_supports_fused_sigmoid()
and aiter_topK_meta_data is not None
)
@classmethod
@if_aiter_supported
def is_mla_enabled(cls) -> bool:
@@ -1795,9 +1854,17 @@ class rocm_aiter_ops:
token_expert_indices: torch.Tensor,
gating_output: torch.Tensor,
renormalize: bool,
num_shared_experts: int = 0,
shared_expert_scoring_func: str = "",
) -> tuple[torch.Tensor, ...]:
torch.ops.vllm.rocm_aiter_topk_softmax(
topk_weights, topk_indices, token_expert_indices, gating_output, renormalize
topk_weights,
topk_indices,
token_expert_indices,
gating_output,
renormalize,
num_shared_experts,
shared_expert_scoring_func,
)
return topk_weights, topk_indices
@@ -285,6 +285,7 @@ class FusedMoE(PluggableLayer):
router_logits_dtype: torch.dtype | None = None,
gate: torch.nn.Module | None = None,
shared_experts: torch.nn.Module | None = None,
shared_expert_gate: torch.nn.Module | None = None,
routed_input_transform: torch.nn.Module | None = None,
routed_output_transform: torch.nn.Module | None = None,
apply_routed_scale_to_output: bool = False,
@@ -370,6 +371,8 @@ class FusedMoE(PluggableLayer):
if n_shared_experts is not None and self.aiter_fmoe_shared_expert_enabled
else 0
)
self.shared_expert_gate = shared_expert_gate
if (
not self.aiter_fmoe_shared_expert_enabled
and self.num_fused_shared_experts != 0
@@ -608,6 +611,7 @@ class FusedMoE(PluggableLayer):
router=self.router,
gate=gate,
shared_experts=shared_experts,
shared_expert_gate=self.shared_expert_gate,
quant_method=self.quant_method,
enable_dbo=self.vllm_config.parallel_config.enable_dbo,
routed_input_transform=routed_input_transform,
@@ -109,6 +109,55 @@ def init_aiter_topK_meta_data(
aiter_topK_meta_data = (total_topk_weights, total_topk_ids)
def inject_shared_expert_weights(
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
topk: int,
num_fused_shared_experts: int,
shared_expert_weights: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Merge routed topk results with the shared expert buffer and inject
dynamic per-token shared expert gate values for AITER fusion.
For routers that already return the combined buffer (e.g. GroupedTopKRouter
via rocm_aiter_grouped_topk), only the dynamic weight injection is needed.
For routers that return only routed slots (e.g. FusedTopKRouter), this also
copies the routed results into the pre-allocated combined buffer.
"""
if num_fused_shared_experts == 0:
return topk_weights, topk_ids
assert aiter_topK_meta_data is not None, (
"aiter_topK_meta_data is not initialized but "
"num_fused_shared_experts > 0. Ensure init_aiter_topK_meta_data "
"is called before routing."
)
total_topk_weights, total_topk_ids = aiter_topK_meta_data
token = topk_weights.shape[0]
assert total_topk_weights.shape[0] >= token, (
f"AITER topK meta data supports {total_topk_weights.shape[0]} "
f"tokens, but got {token} tokens."
)
total_topk_weights_slice = total_topk_weights[:token]
total_topk_ids_slice = total_topk_ids[:token]
if topk_weights.shape[1] == topk:
total_topk_weights_slice[:, :topk] = topk_weights
total_topk_ids_slice[:, :topk] = topk_ids
topk_weights = total_topk_weights_slice
topk_ids = total_topk_ids_slice
if shared_expert_weights is not None:
topk_weights[:, topk : topk + num_fused_shared_experts] = shared_expert_weights[
:token
]
return topk_weights, topk_ids
def rocm_aiter_grouped_topk(
hidden_states: torch.Tensor,
gating_output: torch.Tensor,
@@ -0,0 +1,143 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
import torch
from vllm._aiter_ops import rocm_aiter_ops
from vllm.distributed.eplb.eplb_state import EplbLayerState
from vllm.model_executor.layers.fused_moe.config import (
RoutingMethodType,
get_routing_method_type,
)
from vllm.model_executor.layers.fused_moe.router.base_router import BaseRouter
from vllm.model_executor.layers.fused_moe.router.fused_topk_router import (
dispatch_topk_softmax_func,
)
class AiterSharedRoutedFusedMoERouter(BaseRouter):
"""
ROCm AITER router for models with fused shared experts (e.g. Qwen3-MoE).
When the AITER topk_softmax kernel supports sigmoid fusion, the routing
softmax and shared-expert sigmoid are computed in a single kernel launch.
Otherwise the shared-expert weights are injected into the pre-allocated
AITER buffer via a fallback path.
Only instantiated when rocm_aiter fused-MoE is active and
num_fused_shared_experts > 0.
"""
def __init__(
self,
top_k: int,
global_num_experts: int,
eplb_state: EplbLayerState,
num_fused_shared_experts: int,
scoring_func: str = "softmax",
renormalize: bool = True,
enable_eplb: bool = False,
indices_type_getter: Callable[[], torch.dtype | None] | None = None,
):
super().__init__(
top_k=top_k,
global_num_experts=global_num_experts,
eplb_state=eplb_state,
enable_eplb=enable_eplb,
indices_type_getter=indices_type_getter,
)
self.renormalize = renormalize
self.scoring_func = scoring_func
self.num_fused_shared_experts = num_fused_shared_experts
@property
def routing_method_type(self) -> RoutingMethodType:
return get_routing_method_type(
scoring_func=self.scoring_func,
top_k=self.top_k,
renormalize=self.renormalize,
num_expert_group=None,
has_e_score_bias=False,
)
def _compute_routing(
self,
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
indices_type: torch.dtype | None,
*,
input_ids: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
assert hidden_states.size(0) == router_logits.size(0), (
"Number of tokens mismatch"
)
from vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe import (
aiter_topK_meta_data,
)
M = hidden_states.size(0)
topk = self.top_k
num_fse = self.num_fused_shared_experts
token_expert_indices = torch.empty(
M, topk, dtype=torch.int32, device=hidden_states.device
)
if rocm_aiter_ops.fuse_sigmoid_in_kernel(aiter_topK_meta_data):
total_topk_weights, total_topk_ids = aiter_topK_meta_data # type: ignore[misc]
total_topk_weights_slice = total_topk_weights[:M]
topk_ids_slice = total_topk_ids[:M, :topk]
topk_func = dispatch_topk_softmax_func(use_rocm_aiter=True)
topk_func(
total_topk_weights_slice,
topk_ids_slice,
token_expert_indices,
router_logits,
self.renormalize,
num_fse,
"sigmoid",
)
return total_topk_weights_slice, total_topk_ids[:M]
routing_logits = router_logits[:, :-num_fse]
shared_logits = router_logits[:, -num_fse:]
topk_weights = torch.empty(
M, topk, dtype=torch.float32, device=hidden_states.device
)
topk_ids = torch.empty(
M,
topk,
dtype=torch.int32 if indices_type is None else indices_type,
device=hidden_states.device,
)
topk_func = dispatch_topk_softmax_func(
use_rocm_aiter=rocm_aiter_ops.is_fused_moe_enabled()
)
topk_weights, topk_ids = topk_func(
topk_weights,
topk_ids,
token_expert_indices,
routing_logits,
self.renormalize,
)
if aiter_topK_meta_data is not None:
from vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe import (
inject_shared_expert_weights,
)
shared_weights = torch.sigmoid(shared_logits)
topk_weights, topk_ids = inject_shared_expert_weights(
topk_weights,
topk_ids,
topk=topk,
num_fused_shared_experts=num_fse,
shared_expert_weights=shared_weights,
)
return topk_weights, topk_ids
@@ -5,8 +5,14 @@ from collections.abc import Callable
import torch
import vllm.envs as envs
from vllm._aiter_ops import rocm_aiter_ops
from vllm.distributed.eplb.eplb_state import EplbLayerState
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
from vllm.model_executor.layers.fused_moe.config import (
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.router.aiter_shared_routed_fused_moe_router import ( # noqa: E501
AiterSharedRoutedFusedMoERouter,
)
from vllm.model_executor.layers.fused_moe.router.custom_routing_router import (
CustomRoutingRouter,
)
@@ -67,7 +73,8 @@ def create_fused_moe_router(
3. GroupedTopKRouter - if use_grouped_topk is True
4. CustomRoutingRouter - if custom_routing_function is not None
5. FusedTopKBiasRouter - if e_score_correction_bias is not None
6. FusedTopKRouter - default fallback
6. AiterSharedRoutedFusedMoERouter - if num_fused_shared_experts > 0
7. FusedTopKRouter - default fallback
Common arguments:
top_k: Number of experts to select per token
@@ -199,6 +206,22 @@ def create_fused_moe_router(
hash_indices_table=hash_indices_table,
)
if (
num_fused_shared_experts > 0
and scoring_func == "softmax"
and rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
):
return AiterSharedRoutedFusedMoERouter(
top_k=top_k,
global_num_experts=global_num_experts,
eplb_state=eplb_state,
num_fused_shared_experts=num_fused_shared_experts,
renormalize=renormalize,
scoring_func=scoring_func,
enable_eplb=enable_eplb,
indices_type_getter=indices_type_getter,
)
return FusedTopKRouter(
top_k=top_k,
global_num_experts=global_num_experts,
@@ -223,6 +223,7 @@ class MoERunner(MoERunnerInterface):
shared_experts: torch.nn.Module | None,
quant_method: FusedMoEMethodBase,
enable_dbo: bool,
shared_expert_gate: torch.nn.Module | None = None,
routed_output_transform: torch.nn.Module | None = None,
routed_scaling_factor: float = 1.0,
):
@@ -233,9 +234,18 @@ class MoERunner(MoERunnerInterface):
self.routed_output_transform = routed_output_transform
self.routed_scaling_factor = routed_scaling_factor
self.gate = gate
self.shared_expert_gate = shared_expert_gate
self._quant_method = quant_method
self.enable_dbo = enable_dbo
# When both gates are present and FSE is enabled, fuse their
# weight matrices into [num_experts + num_shared, hidden] so one
# F.linear produces combined logits. The topk kernel can then
# apply routing softmax and shared expert activation (sigmoid)
# in a single launch.
self._fse_fuse_gate = gate is not None and shared_expert_gate is not None
self._combined_gate_weight: torch.Tensor | None = None
self._shared_experts: SharedExperts | None = None
if shared_experts is not None:
self._shared_experts = SharedExperts(
@@ -278,6 +288,20 @@ class MoERunner(MoERunnerInterface):
self._shared_experts._quant_method = quant_method
self._quant_method = quant_method
def _maybe_fuse_gate_weights(self):
"""Fuse router and shared expert gate weights on first call.
Cannot be done at __init__ because gate weights are loaded after
module construction (via weight_loader). Called once from
_forward_impl before the first forward pass.
"""
if self._combined_gate_weight is None:
assert self.gate is not None and self.shared_expert_gate is not None
self._combined_gate_weight = torch.cat(
[self.gate.weight, self.shared_expert_gate.weight],
dim=0,
)
def is_internal_router(self) -> bool:
return self.gate is not None
@@ -752,7 +776,11 @@ class MoERunner(MoERunnerInterface):
# so it can run overlapped with the
# NOTE: in future PR, MoE runner will always hold the gate.
if self.gate is not None:
router_logits, _ = self.gate(hidden_states)
if self._fse_fuse_gate:
self._maybe_fuse_gate_weights()
router_logits = F.linear(hidden_states, self._combined_gate_weight)
else:
router_logits, _ = self.gate(hidden_states)
with self._sequence_parallel_context():
# TODO(bnell): parts of the dispatch/combine steps will go away once
+26 -4
View File
@@ -8,6 +8,7 @@ from itertools import islice
import torch
from torch import nn
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.decorators import support_torch_compile
from vllm.config import (
CacheConfig,
@@ -135,7 +136,12 @@ class Qwen3NextSparseMoeBlock(nn.Module):
prefix=f"{prefix}.shared_expert_gate",
)
if config.shared_expert_intermediate_size > 0:
if (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
or config.shared_expert_intermediate_size <= 0
):
self.shared_expert = None
else:
self.shared_expert = Qwen3NextMLP(
hidden_size=config.hidden_size,
intermediate_size=config.shared_expert_intermediate_size,
@@ -146,8 +152,6 @@ class Qwen3NextSparseMoeBlock(nn.Module):
is_sequence_parallel=self.is_sequence_parallel,
prefix=f"{prefix}.shared_expert",
)
else:
self.shared_expert = None
self.experts = FusedMoE(
shared_experts=self.shared_expert,
@@ -162,6 +166,10 @@ class Qwen3NextSparseMoeBlock(nn.Module):
enable_eplb=self.enable_eplb,
num_redundant_experts=self.n_redundant_experts,
is_sequence_parallel=self.is_sequence_parallel,
n_shared_experts=1 if self.shared_expert is None else None,
shared_expert_gate=self.shared_expert_gate
if self.shared_expert is None
else None,
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -536,12 +544,15 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
# Params for weights, fp8 weight scales, fp8 activation scales
# (param_name, weight_name, expert_id, shard_id)
num_experts = getattr(self.config, "num_experts", 0)
if rocm_aiter_ops.is_fusion_moe_shared_experts_enabled():
num_experts += 1
return fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=getattr(self.config, "num_experts", 0),
num_experts=num_experts,
num_redundant_experts=self.num_redundant_experts,
)
@@ -558,6 +569,10 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
expert_params_mapping = self.get_expert_mapping()
is_fse = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
num_routed = getattr(self.config, "num_experts", 0)
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name:
continue
@@ -571,6 +586,13 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
if name is None:
continue
# FSE: remap shared_expert weights to the fused expert slot
if is_fse and "mlp.shared_expert." in name:
name = name.replace(
"mlp.shared_expert.",
f"mlp.experts.{num_routed}.",
)
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
+14 -1
View File
@@ -7,6 +7,7 @@ from collections.abc import Iterable
import torch
from torch import nn
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
from vllm.distributed.parallel_state import get_pp_group
@@ -147,13 +148,18 @@ class Qwen3NextMultiTokenPredictor(nn.Module):
# Params for weights, fp8 weight scales, fp8 activation scales
# (param_name, weight_name, expert_id, shard_id)
is_fse = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
num_experts = self.config.num_experts
if is_fse:
num_experts += 1
expert_params_mapping = fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=self.config.num_experts,
num_experts=num_experts,
)
num_routed = self.config.num_experts
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
@@ -161,6 +167,13 @@ class Qwen3NextMultiTokenPredictor(nn.Module):
if "rotary_emb.inv_freq" in name:
continue
# FSE: remap shared_expert weights to the fused expert slot
if is_fse and "mlp.shared_expert." in name:
name = name.replace(
"mlp.shared_expert.",
f"mlp.experts.{num_routed}.",
)
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue