From 2c6b59b80771ac3bfa1c789abe3cb27c379bf3a1 Mon Sep 17 00:00:00 2001 From: Nico Holmberg Date: Fri, 8 May 2026 12:38:00 -0700 Subject: [PATCH] [ROCm][Perf] Add Fused Shared Expert (FSE) support for Qwen3-Next (#39280) Signed-off-by: nholmber Signed-off-by: Tres Popp Signed-off-by: Doug Lehr Co-authored-by: nholmber Co-authored-by: Tres Co-authored-by: Tres Popp Co-authored-by: Doug Lehr Co-authored-by: Cursor Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com> --- vllm/_aiter_ops.py | 71 ++++++++- vllm/model_executor/layers/fused_moe/layer.py | 4 + .../layers/fused_moe/rocm_aiter_fused_moe.py | 49 ++++++ .../aiter_shared_routed_fused_moe_router.py | 143 ++++++++++++++++++ .../layers/fused_moe/router/router_factory.py | 27 +++- .../layers/fused_moe/runner/moe_runner.py | 30 +++- vllm/model_executor/models/qwen3_next.py | 30 +++- vllm/model_executor/models/qwen3_next_mtp.py | 15 +- 8 files changed, 359 insertions(+), 10 deletions(-) create mode 100644 vllm/model_executor/layers/fused_moe/router/aiter_shared_routed_fused_moe_router.py diff --git a/vllm/_aiter_ops.py b/vllm/_aiter_ops.py index 8459b20fae..e628c97835 100644 --- a/vllm/_aiter_ops.py +++ b/vllm/_aiter_ops.py @@ -261,11 +261,19 @@ def _rocm_aiter_topk_softmax_impl( token_expert_indices: torch.Tensor, gating_output: torch.Tensor, renormalize: bool, + num_shared_experts: int = 0, + shared_expert_scoring_func: str = "", ) -> None: from aiter import topk_softmax topk_softmax( - topk_weights, topk_indices, token_expert_indices, gating_output, renormalize + topk_weights, + topk_indices, + token_expert_indices, + gating_output, + renormalize, + num_shared_experts, + shared_expert_scoring_func, ) @@ -275,6 +283,8 @@ def _rocm_aiter_topk_softmax_fake( token_expert_indices: torch.Tensor, gating_output: torch.Tensor, renormalize: bool, + num_shared_experts: int = 0, + shared_expert_scoring_func: str = "", ) -> None: pass @@ -1206,6 +1216,9 @@ class rocm_aiter_ops: _MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS # TODO: Consolidate under _LINEAR_ENABLED _TRITON_UNQUANT_GEMM = envs.VLLM_ROCM_USE_AITER_TRITON_GEMM + # Lazily probed: whether aiter.topk_softmax supports the + # num_shared_experts / shared_expert_scoring_func args (7-arg form). + _TOPK_SOFTMAX_FUSED_SIGMOID: bool | None = None _ALL_REDUCE_MAX_SIZE: int = 8192 * 1024 * 8 * 2 _CUSTOM_ALL_REDUCE: AiterCustomAllreduceProto | None = None @@ -1322,6 +1335,52 @@ class rocm_aiter_ops: def is_fusion_moe_shared_experts_enabled(cls) -> bool: return cls.is_fused_moe_enabled() and cls._MOE_SHARED_EXPERTS_ENABLED + @classmethod + @if_aiter_supported + def topk_softmax_supports_fused_sigmoid(cls) -> bool: + """Check if topk_softmax supports fused shared expert activation.""" + if cls._TOPK_SOFTMAX_FUSED_SIGMOID is None: + try: + import inspect + + from aiter import topk_softmax + + params = inspect.signature(topk_softmax).parameters + if "num_shared_experts" in params: + cls._TOPK_SOFTMAX_FUSED_SIGMOID = True + else: + # @compile_ops wrapper loses the original signature. + # Fall back to the torch custom op schema. + import torch + + schema = getattr( + getattr(torch.ops.aiter, "topk_softmax", None), "default", None + ) + schema_str = str(getattr(schema, "_schema", "")) + cls._TOPK_SOFTMAX_FUSED_SIGMOID = "num_shared_experts" in schema_str + except (ImportError, ValueError): + cls._TOPK_SOFTMAX_FUSED_SIGMOID = False + return cls._TOPK_SOFTMAX_FUSED_SIGMOID + + @classmethod + @if_aiter_supported + def fuse_sigmoid_in_kernel(cls, aiter_topK_meta_data: object) -> bool: + """Whether fused shared-expert sigmoid in the topk kernel is usable. + + Combines the cached static capability checks (FSE enabled, fused-moe + enabled, topk_softmax supports fused sigmoid) with the runtime + readiness check (topK meta-data buffer initialized). + + ``aiter_topK_meta_data`` is accepted as a parameter rather than + imported internally so callers cannot hit initialization-order + issues where the module-level global has not been set yet. + """ + return ( + cls.is_fusion_moe_shared_experts_enabled() + and cls.topk_softmax_supports_fused_sigmoid() + and aiter_topK_meta_data is not None + ) + @classmethod @if_aiter_supported def is_mla_enabled(cls) -> bool: @@ -1795,9 +1854,17 @@ class rocm_aiter_ops: token_expert_indices: torch.Tensor, gating_output: torch.Tensor, renormalize: bool, + num_shared_experts: int = 0, + shared_expert_scoring_func: str = "", ) -> tuple[torch.Tensor, ...]: torch.ops.vllm.rocm_aiter_topk_softmax( - topk_weights, topk_indices, token_expert_indices, gating_output, renormalize + topk_weights, + topk_indices, + token_expert_indices, + gating_output, + renormalize, + num_shared_experts, + shared_expert_scoring_func, ) return topk_weights, topk_indices diff --git a/vllm/model_executor/layers/fused_moe/layer.py b/vllm/model_executor/layers/fused_moe/layer.py index 49586a6750..acb601eff5 100644 --- a/vllm/model_executor/layers/fused_moe/layer.py +++ b/vllm/model_executor/layers/fused_moe/layer.py @@ -285,6 +285,7 @@ class FusedMoE(PluggableLayer): router_logits_dtype: torch.dtype | None = None, gate: torch.nn.Module | None = None, shared_experts: torch.nn.Module | None = None, + shared_expert_gate: torch.nn.Module | None = None, routed_input_transform: torch.nn.Module | None = None, routed_output_transform: torch.nn.Module | None = None, apply_routed_scale_to_output: bool = False, @@ -370,6 +371,8 @@ class FusedMoE(PluggableLayer): if n_shared_experts is not None and self.aiter_fmoe_shared_expert_enabled else 0 ) + self.shared_expert_gate = shared_expert_gate + if ( not self.aiter_fmoe_shared_expert_enabled and self.num_fused_shared_experts != 0 @@ -608,6 +611,7 @@ class FusedMoE(PluggableLayer): router=self.router, gate=gate, shared_experts=shared_experts, + shared_expert_gate=self.shared_expert_gate, quant_method=self.quant_method, enable_dbo=self.vllm_config.parallel_config.enable_dbo, routed_input_transform=routed_input_transform, diff --git a/vllm/model_executor/layers/fused_moe/rocm_aiter_fused_moe.py b/vllm/model_executor/layers/fused_moe/rocm_aiter_fused_moe.py index 830309496d..dd16f6bc24 100644 --- a/vllm/model_executor/layers/fused_moe/rocm_aiter_fused_moe.py +++ b/vllm/model_executor/layers/fused_moe/rocm_aiter_fused_moe.py @@ -109,6 +109,55 @@ def init_aiter_topK_meta_data( aiter_topK_meta_data = (total_topk_weights, total_topk_ids) +def inject_shared_expert_weights( + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + topk: int, + num_fused_shared_experts: int, + shared_expert_weights: torch.Tensor | None = None, +) -> tuple[torch.Tensor, torch.Tensor]: + """Merge routed topk results with the shared expert buffer and inject + dynamic per-token shared expert gate values for AITER fusion. + + For routers that already return the combined buffer (e.g. GroupedTopKRouter + via rocm_aiter_grouped_topk), only the dynamic weight injection is needed. + For routers that return only routed slots (e.g. FusedTopKRouter), this also + copies the routed results into the pre-allocated combined buffer. + """ + if num_fused_shared_experts == 0: + return topk_weights, topk_ids + + assert aiter_topK_meta_data is not None, ( + "aiter_topK_meta_data is not initialized but " + "num_fused_shared_experts > 0. Ensure init_aiter_topK_meta_data " + "is called before routing." + ) + + total_topk_weights, total_topk_ids = aiter_topK_meta_data + token = topk_weights.shape[0] + + assert total_topk_weights.shape[0] >= token, ( + f"AITER topK meta data supports {total_topk_weights.shape[0]} " + f"tokens, but got {token} tokens." + ) + + total_topk_weights_slice = total_topk_weights[:token] + total_topk_ids_slice = total_topk_ids[:token] + + if topk_weights.shape[1] == topk: + total_topk_weights_slice[:, :topk] = topk_weights + total_topk_ids_slice[:, :topk] = topk_ids + topk_weights = total_topk_weights_slice + topk_ids = total_topk_ids_slice + + if shared_expert_weights is not None: + topk_weights[:, topk : topk + num_fused_shared_experts] = shared_expert_weights[ + :token + ] + + return topk_weights, topk_ids + + def rocm_aiter_grouped_topk( hidden_states: torch.Tensor, gating_output: torch.Tensor, diff --git a/vllm/model_executor/layers/fused_moe/router/aiter_shared_routed_fused_moe_router.py b/vllm/model_executor/layers/fused_moe/router/aiter_shared_routed_fused_moe_router.py new file mode 100644 index 0000000000..9d92f570df --- /dev/null +++ b/vllm/model_executor/layers/fused_moe/router/aiter_shared_routed_fused_moe_router.py @@ -0,0 +1,143 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from collections.abc import Callable + +import torch + +from vllm._aiter_ops import rocm_aiter_ops +from vllm.distributed.eplb.eplb_state import EplbLayerState +from vllm.model_executor.layers.fused_moe.config import ( + RoutingMethodType, + get_routing_method_type, +) +from vllm.model_executor.layers.fused_moe.router.base_router import BaseRouter +from vllm.model_executor.layers.fused_moe.router.fused_topk_router import ( + dispatch_topk_softmax_func, +) + + +class AiterSharedRoutedFusedMoERouter(BaseRouter): + """ + ROCm AITER router for models with fused shared experts (e.g. Qwen3-MoE). + + When the AITER topk_softmax kernel supports sigmoid fusion, the routing + softmax and shared-expert sigmoid are computed in a single kernel launch. + Otherwise the shared-expert weights are injected into the pre-allocated + AITER buffer via a fallback path. + + Only instantiated when rocm_aiter fused-MoE is active and + num_fused_shared_experts > 0. + """ + + def __init__( + self, + top_k: int, + global_num_experts: int, + eplb_state: EplbLayerState, + num_fused_shared_experts: int, + scoring_func: str = "softmax", + renormalize: bool = True, + enable_eplb: bool = False, + indices_type_getter: Callable[[], torch.dtype | None] | None = None, + ): + super().__init__( + top_k=top_k, + global_num_experts=global_num_experts, + eplb_state=eplb_state, + enable_eplb=enable_eplb, + indices_type_getter=indices_type_getter, + ) + self.renormalize = renormalize + self.scoring_func = scoring_func + self.num_fused_shared_experts = num_fused_shared_experts + + @property + def routing_method_type(self) -> RoutingMethodType: + return get_routing_method_type( + scoring_func=self.scoring_func, + top_k=self.top_k, + renormalize=self.renormalize, + num_expert_group=None, + has_e_score_bias=False, + ) + + def _compute_routing( + self, + hidden_states: torch.Tensor, + router_logits: torch.Tensor, + indices_type: torch.dtype | None, + *, + input_ids: torch.Tensor | None = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + assert hidden_states.size(0) == router_logits.size(0), ( + "Number of tokens mismatch" + ) + + from vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe import ( + aiter_topK_meta_data, + ) + + M = hidden_states.size(0) + topk = self.top_k + num_fse = self.num_fused_shared_experts + + token_expert_indices = torch.empty( + M, topk, dtype=torch.int32, device=hidden_states.device + ) + + if rocm_aiter_ops.fuse_sigmoid_in_kernel(aiter_topK_meta_data): + total_topk_weights, total_topk_ids = aiter_topK_meta_data # type: ignore[misc] + total_topk_weights_slice = total_topk_weights[:M] + topk_ids_slice = total_topk_ids[:M, :topk] + + topk_func = dispatch_topk_softmax_func(use_rocm_aiter=True) + topk_func( + total_topk_weights_slice, + topk_ids_slice, + token_expert_indices, + router_logits, + self.renormalize, + num_fse, + "sigmoid", + ) + return total_topk_weights_slice, total_topk_ids[:M] + + routing_logits = router_logits[:, :-num_fse] + shared_logits = router_logits[:, -num_fse:] + + topk_weights = torch.empty( + M, topk, dtype=torch.float32, device=hidden_states.device + ) + topk_ids = torch.empty( + M, + topk, + dtype=torch.int32 if indices_type is None else indices_type, + device=hidden_states.device, + ) + + topk_func = dispatch_topk_softmax_func( + use_rocm_aiter=rocm_aiter_ops.is_fused_moe_enabled() + ) + topk_weights, topk_ids = topk_func( + topk_weights, + topk_ids, + token_expert_indices, + routing_logits, + self.renormalize, + ) + + if aiter_topK_meta_data is not None: + from vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe import ( + inject_shared_expert_weights, + ) + + shared_weights = torch.sigmoid(shared_logits) + topk_weights, topk_ids = inject_shared_expert_weights( + topk_weights, + topk_ids, + topk=topk, + num_fused_shared_experts=num_fse, + shared_expert_weights=shared_weights, + ) + + return topk_weights, topk_ids diff --git a/vllm/model_executor/layers/fused_moe/router/router_factory.py b/vllm/model_executor/layers/fused_moe/router/router_factory.py index da7896de61..2f2c4d39c4 100644 --- a/vllm/model_executor/layers/fused_moe/router/router_factory.py +++ b/vllm/model_executor/layers/fused_moe/router/router_factory.py @@ -5,8 +5,14 @@ from collections.abc import Callable import torch import vllm.envs as envs +from vllm._aiter_ops import rocm_aiter_ops from vllm.distributed.eplb.eplb_state import EplbLayerState -from vllm.model_executor.layers.fused_moe.config import RoutingMethodType +from vllm.model_executor.layers.fused_moe.config import ( + RoutingMethodType, +) +from vllm.model_executor.layers.fused_moe.router.aiter_shared_routed_fused_moe_router import ( # noqa: E501 + AiterSharedRoutedFusedMoERouter, +) from vllm.model_executor.layers.fused_moe.router.custom_routing_router import ( CustomRoutingRouter, ) @@ -67,7 +73,8 @@ def create_fused_moe_router( 3. GroupedTopKRouter - if use_grouped_topk is True 4. CustomRoutingRouter - if custom_routing_function is not None 5. FusedTopKBiasRouter - if e_score_correction_bias is not None - 6. FusedTopKRouter - default fallback + 6. AiterSharedRoutedFusedMoERouter - if num_fused_shared_experts > 0 + 7. FusedTopKRouter - default fallback Common arguments: top_k: Number of experts to select per token @@ -199,6 +206,22 @@ def create_fused_moe_router( hash_indices_table=hash_indices_table, ) + if ( + num_fused_shared_experts > 0 + and scoring_func == "softmax" + and rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() + ): + return AiterSharedRoutedFusedMoERouter( + top_k=top_k, + global_num_experts=global_num_experts, + eplb_state=eplb_state, + num_fused_shared_experts=num_fused_shared_experts, + renormalize=renormalize, + scoring_func=scoring_func, + enable_eplb=enable_eplb, + indices_type_getter=indices_type_getter, + ) + return FusedTopKRouter( top_k=top_k, global_num_experts=global_num_experts, diff --git a/vllm/model_executor/layers/fused_moe/runner/moe_runner.py b/vllm/model_executor/layers/fused_moe/runner/moe_runner.py index 3be5f91b34..37b15ed406 100644 --- a/vllm/model_executor/layers/fused_moe/runner/moe_runner.py +++ b/vllm/model_executor/layers/fused_moe/runner/moe_runner.py @@ -223,6 +223,7 @@ class MoERunner(MoERunnerInterface): shared_experts: torch.nn.Module | None, quant_method: FusedMoEMethodBase, enable_dbo: bool, + shared_expert_gate: torch.nn.Module | None = None, routed_output_transform: torch.nn.Module | None = None, routed_scaling_factor: float = 1.0, ): @@ -233,9 +234,18 @@ class MoERunner(MoERunnerInterface): self.routed_output_transform = routed_output_transform self.routed_scaling_factor = routed_scaling_factor self.gate = gate + self.shared_expert_gate = shared_expert_gate self._quant_method = quant_method self.enable_dbo = enable_dbo + # When both gates are present and FSE is enabled, fuse their + # weight matrices into [num_experts + num_shared, hidden] so one + # F.linear produces combined logits. The topk kernel can then + # apply routing softmax and shared expert activation (sigmoid) + # in a single launch. + self._fse_fuse_gate = gate is not None and shared_expert_gate is not None + self._combined_gate_weight: torch.Tensor | None = None + self._shared_experts: SharedExperts | None = None if shared_experts is not None: self._shared_experts = SharedExperts( @@ -278,6 +288,20 @@ class MoERunner(MoERunnerInterface): self._shared_experts._quant_method = quant_method self._quant_method = quant_method + def _maybe_fuse_gate_weights(self): + """Fuse router and shared expert gate weights on first call. + + Cannot be done at __init__ because gate weights are loaded after + module construction (via weight_loader). Called once from + _forward_impl before the first forward pass. + """ + if self._combined_gate_weight is None: + assert self.gate is not None and self.shared_expert_gate is not None + self._combined_gate_weight = torch.cat( + [self.gate.weight, self.shared_expert_gate.weight], + dim=0, + ) + def is_internal_router(self) -> bool: return self.gate is not None @@ -752,7 +776,11 @@ class MoERunner(MoERunnerInterface): # so it can run overlapped with the # NOTE: in future PR, MoE runner will always hold the gate. if self.gate is not None: - router_logits, _ = self.gate(hidden_states) + if self._fse_fuse_gate: + self._maybe_fuse_gate_weights() + router_logits = F.linear(hidden_states, self._combined_gate_weight) + else: + router_logits, _ = self.gate(hidden_states) with self._sequence_parallel_context(): # TODO(bnell): parts of the dispatch/combine steps will go away once diff --git a/vllm/model_executor/models/qwen3_next.py b/vllm/model_executor/models/qwen3_next.py index 96d7e9c713..f4c7167890 100644 --- a/vllm/model_executor/models/qwen3_next.py +++ b/vllm/model_executor/models/qwen3_next.py @@ -8,6 +8,7 @@ from itertools import islice import torch from torch import nn +from vllm._aiter_ops import rocm_aiter_ops from vllm.compilation.decorators import support_torch_compile from vllm.config import ( CacheConfig, @@ -135,7 +136,12 @@ class Qwen3NextSparseMoeBlock(nn.Module): prefix=f"{prefix}.shared_expert_gate", ) - if config.shared_expert_intermediate_size > 0: + if ( + rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() + or config.shared_expert_intermediate_size <= 0 + ): + self.shared_expert = None + else: self.shared_expert = Qwen3NextMLP( hidden_size=config.hidden_size, intermediate_size=config.shared_expert_intermediate_size, @@ -146,8 +152,6 @@ class Qwen3NextSparseMoeBlock(nn.Module): is_sequence_parallel=self.is_sequence_parallel, prefix=f"{prefix}.shared_expert", ) - else: - self.shared_expert = None self.experts = FusedMoE( shared_experts=self.shared_expert, @@ -162,6 +166,10 @@ class Qwen3NextSparseMoeBlock(nn.Module): enable_eplb=self.enable_eplb, num_redundant_experts=self.n_redundant_experts, is_sequence_parallel=self.is_sequence_parallel, + n_shared_experts=1 if self.shared_expert is None else None, + shared_expert_gate=self.shared_expert_gate + if self.shared_expert is None + else None, ) def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: @@ -536,12 +544,15 @@ class Qwen3NextModel(nn.Module, EagleModelMixin): def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) + num_experts = getattr(self.config, "num_experts", 0) + if rocm_aiter_ops.is_fusion_moe_shared_experts_enabled(): + num_experts += 1 return fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", ckpt_up_proj_name="up_proj", - num_experts=getattr(self.config, "num_experts", 0), + num_experts=num_experts, num_redundant_experts=self.num_redundant_experts, ) @@ -558,6 +569,10 @@ class Qwen3NextModel(nn.Module, EagleModelMixin): params_dict = dict(self.named_parameters()) loaded_params: set[str] = set() expert_params_mapping = self.get_expert_mapping() + + is_fse = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() + num_routed = getattr(self.config, "num_experts", 0) + for name, loaded_weight in weights: if "rotary_emb.inv_freq" in name: continue @@ -571,6 +586,13 @@ class Qwen3NextModel(nn.Module, EagleModelMixin): if name is None: continue + # FSE: remap shared_expert weights to the fused expert slot + if is_fse and "mlp.shared_expert." in name: + name = name.replace( + "mlp.shared_expert.", + f"mlp.experts.{num_routed}.", + ) + for param_name, weight_name, shard_id in stacked_params_mapping: if weight_name not in name: continue diff --git a/vllm/model_executor/models/qwen3_next_mtp.py b/vllm/model_executor/models/qwen3_next_mtp.py index 2f411c48a6..4d8ff951c0 100644 --- a/vllm/model_executor/models/qwen3_next_mtp.py +++ b/vllm/model_executor/models/qwen3_next_mtp.py @@ -7,6 +7,7 @@ from collections.abc import Iterable import torch from torch import nn +from vllm._aiter_ops import rocm_aiter_ops from vllm.compilation.decorators import support_torch_compile from vllm.config import VllmConfig from vllm.distributed.parallel_state import get_pp_group @@ -147,13 +148,18 @@ class Qwen3NextMultiTokenPredictor(nn.Module): # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) + is_fse = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() + num_experts = self.config.num_experts + if is_fse: + num_experts += 1 expert_params_mapping = fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", ckpt_up_proj_name="up_proj", - num_experts=self.config.num_experts, + num_experts=num_experts, ) + num_routed = self.config.num_experts params_dict = dict(self.named_parameters()) loaded_params: set[str] = set() @@ -161,6 +167,13 @@ class Qwen3NextMultiTokenPredictor(nn.Module): if "rotary_emb.inv_freq" in name: continue + # FSE: remap shared_expert weights to the fused expert slot + if is_fse and "mlp.shared_expert." in name: + name = name.replace( + "mlp.shared_expert.", + f"mlp.experts.{num_routed}.", + ) + for param_name, weight_name, shard_id in stacked_params_mapping: if weight_name not in name: continue