[ROCm][Perf] Fused shared expert for Minimax M3 (#46474)
Signed-off-by: Fangzhou-Ai <fangzhouai@gmail.com> Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com> Co-authored-by: tjtanaa <tunjian.tan@embeddedllm.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
tjtanaa
parent
867fd5e8ed
commit
51a99565c3
@@ -23,8 +23,9 @@ import torch
|
||||
from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm import envs
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import (
|
||||
CacheConfig,
|
||||
@@ -96,7 +97,6 @@ from vllm.models.minimax_m3.common.sparse_attention import (
|
||||
)
|
||||
from vllm.models.minimax_m3.common.vision_tower import MiniMaxVLVisionModel
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.torch_utils import kv_cache_dtype_str_to_dtype
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
@@ -107,14 +107,15 @@ from vllm.v1.kv_cache_interface import (
|
||||
|
||||
|
||||
def _fuse_shared_experts_enabled(config: PretrainedConfig) -> bool:
|
||||
"""Whether to fuse the shared expert into the routed grouped MoE.
|
||||
"""Whether to fuse the shared expert with routed experts.
|
||||
|
||||
ROCm only. Opt-in via ``VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS`` (the
|
||||
router-append fusion runs on the triton/flydsl mxfp8 MoE independent of the
|
||||
aiter master switch); requires a shared expert and is disabled under expert
|
||||
parallelism (the shared slot is appended to the routed top-k, which the EP
|
||||
expert-map path does not handle).
|
||||
router-append fusion runs on both aiter and non-aiter MoE);
|
||||
it is disabled under expert parallelism (the shared slot is appended to
|
||||
the routed top-k, which the EP expert-mapping path does not handle).
|
||||
"""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
return bool(
|
||||
current_platform.is_rocm()
|
||||
and getattr(config, "n_shared_experts", None)
|
||||
@@ -268,6 +269,24 @@ class MiniMaxM3MLP(nn.Module):
|
||||
return x
|
||||
|
||||
|
||||
def _aiter_moe_fused_shared_experts_enabled(config: PretrainedConfig) -> bool:
|
||||
"""Whether the fused shared expert routes through aiter's grouped top-k MoE.
|
||||
|
||||
A strict sub-case of :func:`_fuse_shared_experts_enabled`: shared-expert
|
||||
fusion must already be opted in (``VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS``)
|
||||
and allowed (not under expert parallelism). When additionally on gfx950 with
|
||||
an active aiter MoE backend, the shared expert is appended inside aiter's
|
||||
biased grouped top-k kernel (``num_fused_shared_experts``) instead of the
|
||||
vLLM router's torch concat. Otherwise FSE still runs via the vLLM top-k bias
|
||||
router.
|
||||
"""
|
||||
if not _fuse_shared_experts_enabled(config):
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
return on_gfx950() and rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
|
||||
|
||||
class MiniMaxM3MoE(nn.Module):
|
||||
"""Sigmoid-routed MoE block with a routing-bias correction and a shared
|
||||
expert."""
|
||||
@@ -313,11 +332,14 @@ class MiniMaxM3MoE(nn.Module):
|
||||
prefix=f"{prefix}.gate",
|
||||
)
|
||||
|
||||
# Fuse the shared expert into the routed grouped GEMM when opted in via
|
||||
# VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: it becomes routed-expert slot
|
||||
# ``num_local_experts``, reached by every token, eliminating the
|
||||
# separate dense-MLP launches. Not supported under expert parallelism.
|
||||
# Shared-expert fusion (opt-in via VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS,
|
||||
# off under expert parallelism) folds the shared expert into the routed
|
||||
# MoE call as the last expert slot, so we don't build a separate module.
|
||||
# On gfx950 with aiter MoE the append is fused inside aiter's grouped
|
||||
# top-k kernel; otherwise it goes through the vLLM top-k bias router.
|
||||
self.fuse_shared_experts = _fuse_shared_experts_enabled(config)
|
||||
self.use_aiter_moe_fse = _aiter_moe_fused_shared_experts_enabled(config)
|
||||
|
||||
self.shared_experts: MiniMaxM3MLP | None = None
|
||||
if self.n_shared_experts and not self.fuse_shared_experts:
|
||||
self.shared_experts = MiniMaxM3MLP(
|
||||
@@ -328,6 +350,13 @@ class MiniMaxM3MoE(nn.Module):
|
||||
prefix=f"{prefix}.shared_experts",
|
||||
)
|
||||
|
||||
# The aiter MoE fused path goes through aiter's biased grouped top-k
|
||||
# (GroupedTopKRouter, as in DeepSeek-V4): M3 is not group-routed, so a
|
||||
# trivial single group (num_expert_group=topk_group=1) reduces to plain
|
||||
# top-k while applying the sigmoid + bias correction and appending the
|
||||
# always-on shared expert; aiter applies the routed scaling internally.
|
||||
# Every other path (vLLM top-k bias router, or no fusion) applies the
|
||||
# routed scaling to the MoE output here.
|
||||
self.experts = FusedMoE(
|
||||
num_experts=config.num_local_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
@@ -337,12 +366,15 @@ class MiniMaxM3MoE(nn.Module):
|
||||
scoring_func=config.scoring_func,
|
||||
e_score_correction_bias=self.e_score_correction_bias,
|
||||
renormalize=True,
|
||||
use_grouped_topk=self.use_aiter_moe_fse,
|
||||
num_expert_group=1 if self.use_aiter_moe_fse else None,
|
||||
topk_group=1 if self.use_aiter_moe_fse else None,
|
||||
activation="swigluoai_uninterleave",
|
||||
swiglu_limit=config.swiglu_limit,
|
||||
swiglu_alpha=config.swiglu_alpha,
|
||||
swiglu_beta=config.swiglu_beta,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
apply_routed_scale_to_output=True,
|
||||
apply_routed_scale_to_output=not self.use_aiter_moe_fse,
|
||||
router_logits_dtype=self.gate.out_dtype,
|
||||
shared_experts=self.shared_experts,
|
||||
n_shared_experts=(
|
||||
@@ -927,9 +959,10 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin):
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
expert_params_mapping = self.get_expert_mapping()
|
||||
|
||||
_fuse_shared = _fuse_shared_experts_enabled(self.config)
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
_fuse_shared = _fuse_shared_experts_enabled(self.config)
|
||||
for name, loaded_weight in weights:
|
||||
# The MTP module is not modeled yet.
|
||||
if "mtp." in name:
|
||||
|
||||
Reference in New Issue
Block a user