[ROCm] [MoE] [Perf] Shared-expert fusion for bias-routed MoE; enable on MiniMax-M3 mxfp8 model (#46545)

Signed-off-by: Hongxia Yang <hongxia.yang@amd.com>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Hongxia Yang
2026-06-26 07:05:20 -07:00
committed by GitHub
co-authored by Claude Opus 4.8
parent 8921c4be88
commit c2507fb293
5 changed files with 110 additions and 20 deletions
@@ -227,10 +227,16 @@ def fused_moe_mxfp8_native(
tiles = _mxfp8_moe_tiles(T)
block_m = tiles["block_m"]
# Bin by the actual number of expert weight rows. With fused shared experts
# the weight tensor has more rows than ``global_num_experts`` (the routed
# count), and their ids fall outside [0, global_num_experts); binning by the
# routed count would treat them as invalid. Under EP (expert_map set) the
# tensor holds only local experts, so keep the global count for remapping.
num_align_experts = w13.shape[0] if expert_map is None else global_num_experts
sorted_ids, expert_ids, num_post = moe_align_block_size(
topk_ids,
block_m,
global_num_experts,
num_align_experts,
expert_map,
ignore_invalid_experts=expert_map is not None,
)
+27 -16
View File
@@ -6,6 +6,7 @@ from typing import Any
import torch
import vllm.envs as envs
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import ParallelConfig, get_current_vllm_config
from vllm.distributed import (
@@ -77,24 +78,20 @@ def determine_expert_counts(
) -> tuple[int, int, int]:
global_num_experts = num_experts + num_redundant_experts
logical_num_experts = num_experts
# ROCm aiter shared experts fusion
# AITER only supports gated activations (silu/gelu), so disable it
# for non-gated MoE (is_act_and_mul=False)
# rocm_aiter_fmoe_enabled = rocm_aiter_ops.is_fused_moe_enabled() and is_act_and_mul
aiter_fmoe_shared_expert_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() and is_act_and_mul
)
# Shared-expert fusion: append the shared expert(s) as routed-expert slots
# so they run in the same grouped GEMM. Gated by
# VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: either the native aiter fused-MoE
# path (env + master switch, via is_fusion_moe_shared_experts_enabled) or the
# backend-neutral router-append path (env alone, independent of the master
# switch; e.g. the MM3 triton/flydsl mxfp8 MoE). Gated activations only.
fuse_shared_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
or envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
) and is_act_and_mul
num_fused_shared_experts = (
n_shared_experts
if n_shared_experts is not None and aiter_fmoe_shared_expert_enabled
else 0
n_shared_experts if n_shared_experts is not None and fuse_shared_enabled else 0
)
if not aiter_fmoe_shared_expert_enabled and num_fused_shared_experts != 0:
raise ValueError(
"n_shared_experts is only supported on ROCm aiter when "
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled"
)
return global_num_experts, logical_num_experts, num_fused_shared_experts
@@ -188,7 +185,8 @@ def FusedMoE(
has_bias: Whether expert layers have bias terms
is_sequence_parallel: Whether sequence parallelism is enabled
expert_mapping: Expert parameter mapping for weight loading
n_shared_experts: Number of shared experts (ROCm aiter only)
n_shared_experts: Number of shared experts to fuse into the routed
grouped GEMM (ROCm; requires aiter FSE or the router-append path)
router_logits_dtype: Data type for router logits buffers
gate: Pre-configured gate module
shared_experts: Pre-configured shared experts module
@@ -289,6 +287,19 @@ def FusedMoE(
else 1.0,
e_score_correction_bias=e_score_correction_bias,
num_fused_shared_experts=num_fused_shared_experts,
# Fused shared-expert slot weight. With apply_routed_scale_to_output
# the runner scales the combined output by routed_scaling_factor, so
# the shared slot weight must be 1/routed_scaling_factor for its net
# contribution to be 1.0 (matching the un-scaled separate-MLP add).
shared_expert_weight=(
(1.0 / routed_scaling_factor)
if (
apply_routed_scale_to_output
and num_fused_shared_experts > 0
and routed_scaling_factor
)
else 1.0
),
zero_expert_type=zero_expert_type,
num_logical_experts=logical_num_experts,
hash_indices_table=hash_indices_table,
@@ -334,6 +334,8 @@ class FusedTopKBiasRouter(BaseRouter):
*,
scoring_func: str = "sigmoid",
hash_indices_table: torch.Tensor | None = None,
num_fused_shared_experts: int = 0,
shared_expert_weight: float = 1.0,
):
super().__init__(
top_k=top_k,
@@ -346,6 +348,11 @@ class FusedTopKBiasRouter(BaseRouter):
self.routed_scaling_factor = routed_scaling_factor
self.scoring_func = scoring_func
self._hash_indices_table = hash_indices_table
# Fused shared experts: append constant slots (ids immediately after
# the routed experts, [global, global+n)) routed to by every token at
# ``shared_expert_weight``, AFTER the routed top-k is renormalized.
self.num_fused_shared_experts = num_fused_shared_experts
self.shared_expert_weight = shared_expert_weight
@property
def routing_method_type(self) -> RoutingMethodType:
@@ -382,4 +389,23 @@ class FusedTopKBiasRouter(BaseRouter):
routed_scaling_factor=self.routed_scaling_factor,
)
if self.num_fused_shared_experts > 0:
m = topk_ids.shape[0]
n = self.num_fused_shared_experts
# global_num_experts counts only the routed experts; the fused
# shared experts occupy the slots immediately after them, i.e. ids
# [global_num_experts, global_num_experts + n).
base = self.global_num_experts
shared_ids = torch.arange(
base, base + n, dtype=topk_ids.dtype, device=topk_ids.device
).expand(m, n)
shared_w = torch.full(
(m, n),
self.shared_expert_weight,
dtype=topk_weights.dtype,
device=topk_weights.device,
)
topk_ids = torch.cat([topk_ids, shared_ids], dim=-1)
topk_weights = torch.cat([topk_weights, shared_w], dim=-1)
return topk_weights, topk_ids
@@ -47,6 +47,7 @@ def create_fused_moe_router(
topk_group: int | None = None,
scoring_func: str = "softmax",
num_fused_shared_experts: int = 0,
shared_expert_weight: float = 1.0,
# grouped topk + fused topk bias parameters
routed_scaling_factor: float = 1.0,
e_score_correction_bias: torch.Tensor | None = None,
@@ -188,6 +189,8 @@ def create_fused_moe_router(
routed_scaling_factor=routed_scaling_factor,
scoring_func=scoring_func,
hash_indices_table=hash_indices_table,
num_fused_shared_experts=num_fused_shared_experts,
shared_expert_weight=shared_expert_weight,
)
if (
+47 -3
View File
@@ -23,6 +23,7 @@ import torch
from torch import nn
from transformers import PretrainedConfig
import vllm.envs as envs
from vllm import _custom_ops as ops
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import (
@@ -95,6 +96,7 @@ from vllm.models.minimax_m3.common.sparse_attention import (
)
from vllm.models.minimax_m3.common.vision_tower import MiniMaxVLVisionModel
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.utils.torch_utils import kv_cache_dtype_str_to_dtype
from vllm.v1.kv_cache_interface import (
@@ -104,6 +106,23 @@ from vllm.v1.kv_cache_interface import (
)
def _fuse_shared_experts_enabled(config: PretrainedConfig) -> bool:
"""Whether to fuse the shared expert into the routed grouped MoE.
ROCm only. Opt-in via ``VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS`` (the
router-append fusion runs on the triton/flydsl mxfp8 MoE independent of the
aiter master switch); requires a shared expert and is disabled under expert
parallelism (the shared slot is appended to the routed top-k, which the EP
expert-map path does not handle).
"""
return bool(
current_platform.is_rocm()
and getattr(config, "n_shared_experts", None)
and envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
and not get_current_vllm_config().parallel_config.enable_expert_parallel
)
def _sparse_attention_layer_ids(config: PretrainedConfig) -> set[int]:
"""Layer ids whose attention runs the extra sparse "index" branch."""
cfg = getattr(config, "sparse_attention_config", None)
@@ -294,8 +313,13 @@ class MiniMaxM3MoE(nn.Module):
prefix=f"{prefix}.gate",
)
# Fuse the shared expert into the routed grouped GEMM when opted in via
# VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: it becomes routed-expert slot
# ``num_local_experts``, reached by every token, eliminating the
# separate dense-MLP launches. Not supported under expert parallelism.
self.fuse_shared_experts = _fuse_shared_experts_enabled(config)
self.shared_experts: MiniMaxM3MLP | None = None
if self.n_shared_experts:
if self.n_shared_experts and not self.fuse_shared_experts:
self.shared_experts = MiniMaxM3MLP(
config=config,
intermediate_size=config.intermediate_size * self.n_shared_experts,
@@ -321,6 +345,9 @@ class MiniMaxM3MoE(nn.Module):
apply_routed_scale_to_output=True,
router_logits_dtype=self.gate.out_dtype,
shared_experts=self.shared_experts,
n_shared_experts=(
self.n_shared_experts if self.fuse_shared_experts else None
),
quant_config=quant_config,
prefix=f"{prefix}.experts",
)
@@ -864,13 +891,18 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin):
return hidden_states
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
# Checkpoint experts use w1=gate, w2=down, w3=up.
# Checkpoint experts use w1=gate, w2=down, w3=up. When fusing the shared
# expert, include the appended slot (id == num_local_experts).
n_shared = getattr(self.config, "n_shared_experts", 0) or 0
num_experts = self.config.num_local_experts + (
n_shared if _fuse_shared_experts_enabled(self.config) else 0
)
return fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="w1",
ckpt_down_proj_name="w2",
ckpt_up_proj_name="w3",
num_experts=self.config.num_local_experts,
num_experts=num_experts,
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
@@ -897,6 +929,7 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin):
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
_fuse_shared = _fuse_shared_experts_enabled(self.config)
for name, loaded_weight in weights:
# The MTP module is not modeled yet.
if "mtp." in name:
@@ -907,6 +940,17 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin):
if "weight_scale_inv" in name:
name = name.replace("weight_scale_inv", "weight_scale")
# Shared-expert fusion: redirect the checkpoint shared expert into
# routed-expert slot ``num_local_experts`` (gate->w1, up->w3,
# down->w2) so it loads via the routed expert loader. Runs before the
# stacked/dense mappings so shared_experts.gate_proj/up_proj are not
# captured by the dense gate_up_proj mapping.
if _fuse_shared and ".shared_experts." in name:
sid = self.config.num_local_experts
name = name.replace(".shared_experts.gate_proj.", f".experts.{sid}.w1.")
name = name.replace(".shared_experts.up_proj.", f".experts.{sid}.w3.")
name = name.replace(".shared_experts.down_proj.", f".experts.{sid}.w2.")
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue