[ROCm][Perf] Fused shared expert for Minimax M3 (#46474)

Signed-off-by: Fangzhou-Ai <fangzhouai@gmail.com>
Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
Co-authored-by: tjtanaa <tunjian.tan@embeddedllm.com>
This commit is contained in:
Fangzhou Ai
2026-06-27 12:34:17 +00:00
committed by GitHub
co-authored by Claude Opus 4.8 tjtanaa
parent 867fd5e8ed
commit 51a99565c3
+46 -13
View File
@@ -23,8 +23,9 @@ import torch
from torch import nn
from transformers import PretrainedConfig
import vllm.envs as envs
from vllm import _custom_ops as ops
from vllm import envs
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import (
CacheConfig,
@@ -96,7 +97,6 @@ from vllm.models.minimax_m3.common.sparse_attention import (
)
from vllm.models.minimax_m3.common.vision_tower import MiniMaxVLVisionModel
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.utils.torch_utils import kv_cache_dtype_str_to_dtype
from vllm.v1.kv_cache_interface import (
@@ -107,14 +107,15 @@ from vllm.v1.kv_cache_interface import (
def _fuse_shared_experts_enabled(config: PretrainedConfig) -> bool:
"""Whether to fuse the shared expert into the routed grouped MoE.
"""Whether to fuse the shared expert with routed experts.
ROCm only. Opt-in via ``VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS`` (the
router-append fusion runs on the triton/flydsl mxfp8 MoE independent of the
aiter master switch); requires a shared expert and is disabled under expert
parallelism (the shared slot is appended to the routed top-k, which the EP
expert-map path does not handle).
router-append fusion runs on both aiter and non-aiter MoE);
it is disabled under expert parallelism (the shared slot is appended to
the routed top-k, which the EP expert-mapping path does not handle).
"""
from vllm.platforms import current_platform
return bool(
current_platform.is_rocm()
and getattr(config, "n_shared_experts", None)
@@ -268,6 +269,24 @@ class MiniMaxM3MLP(nn.Module):
return x
def _aiter_moe_fused_shared_experts_enabled(config: PretrainedConfig) -> bool:
"""Whether the fused shared expert routes through aiter's grouped top-k MoE.
A strict sub-case of :func:`_fuse_shared_experts_enabled`: shared-expert
fusion must already be opted in (``VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS``)
and allowed (not under expert parallelism). When additionally on gfx950 with
an active aiter MoE backend, the shared expert is appended inside aiter's
biased grouped top-k kernel (``num_fused_shared_experts``) instead of the
vLLM router's torch concat. Otherwise FSE still runs via the vLLM top-k bias
router.
"""
if not _fuse_shared_experts_enabled(config):
return False
from vllm.platforms.rocm import on_gfx950
return on_gfx950() and rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
class MiniMaxM3MoE(nn.Module):
"""Sigmoid-routed MoE block with a routing-bias correction and a shared
expert."""
@@ -313,11 +332,14 @@ class MiniMaxM3MoE(nn.Module):
prefix=f"{prefix}.gate",
)
# Fuse the shared expert into the routed grouped GEMM when opted in via
# VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: it becomes routed-expert slot
# ``num_local_experts``, reached by every token, eliminating the
# separate dense-MLP launches. Not supported under expert parallelism.
# Shared-expert fusion (opt-in via VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS,
# off under expert parallelism) folds the shared expert into the routed
# MoE call as the last expert slot, so we don't build a separate module.
# On gfx950 with aiter MoE the append is fused inside aiter's grouped
# top-k kernel; otherwise it goes through the vLLM top-k bias router.
self.fuse_shared_experts = _fuse_shared_experts_enabled(config)
self.use_aiter_moe_fse = _aiter_moe_fused_shared_experts_enabled(config)
self.shared_experts: MiniMaxM3MLP | None = None
if self.n_shared_experts and not self.fuse_shared_experts:
self.shared_experts = MiniMaxM3MLP(
@@ -328,6 +350,13 @@ class MiniMaxM3MoE(nn.Module):
prefix=f"{prefix}.shared_experts",
)
# The aiter MoE fused path goes through aiter's biased grouped top-k
# (GroupedTopKRouter, as in DeepSeek-V4): M3 is not group-routed, so a
# trivial single group (num_expert_group=topk_group=1) reduces to plain
# top-k while applying the sigmoid + bias correction and appending the
# always-on shared expert; aiter applies the routed scaling internally.
# Every other path (vLLM top-k bias router, or no fusion) applies the
# routed scaling to the MoE output here.
self.experts = FusedMoE(
num_experts=config.num_local_experts,
top_k=config.num_experts_per_tok,
@@ -337,12 +366,15 @@ class MiniMaxM3MoE(nn.Module):
scoring_func=config.scoring_func,
e_score_correction_bias=self.e_score_correction_bias,
renormalize=True,
use_grouped_topk=self.use_aiter_moe_fse,
num_expert_group=1 if self.use_aiter_moe_fse else None,
topk_group=1 if self.use_aiter_moe_fse else None,
activation="swigluoai_uninterleave",
swiglu_limit=config.swiglu_limit,
swiglu_alpha=config.swiglu_alpha,
swiglu_beta=config.swiglu_beta,
routed_scaling_factor=self.routed_scaling_factor,
apply_routed_scale_to_output=True,
apply_routed_scale_to_output=not self.use_aiter_moe_fse,
router_logits_dtype=self.gate.out_dtype,
shared_experts=self.shared_experts,
n_shared_experts=(
@@ -927,9 +959,10 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin):
# (param_name, weight_name, expert_id, shard_id)
expert_params_mapping = self.get_expert_mapping()
_fuse_shared = _fuse_shared_experts_enabled(self.config)
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
_fuse_shared = _fuse_shared_experts_enabled(self.config)
for name, loaded_weight in weights:
# The MTP module is not modeled yet.
if "mtp." in name: