[ROCm] [MoE] [Perf] Shared-expert fusion for bias-routed MoE; enable on MiniMax-M3 mxfp8 model (#46545)
Signed-off-by: Hongxia Yang <hongxia.yang@amd.com> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
8921c4be88
commit
c2507fb293
@@ -227,10 +227,16 @@ def fused_moe_mxfp8_native(
|
||||
|
||||
tiles = _mxfp8_moe_tiles(T)
|
||||
block_m = tiles["block_m"]
|
||||
# Bin by the actual number of expert weight rows. With fused shared experts
|
||||
# the weight tensor has more rows than ``global_num_experts`` (the routed
|
||||
# count), and their ids fall outside [0, global_num_experts); binning by the
|
||||
# routed count would treat them as invalid. Under EP (expert_map set) the
|
||||
# tensor holds only local experts, so keep the global count for remapping.
|
||||
num_align_experts = w13.shape[0] if expert_map is None else global_num_experts
|
||||
sorted_ids, expert_ids, num_post = moe_align_block_size(
|
||||
topk_ids,
|
||||
block_m,
|
||||
global_num_experts,
|
||||
num_align_experts,
|
||||
expert_map,
|
||||
ignore_invalid_experts=expert_map is not None,
|
||||
)
|
||||
|
||||
@@ -6,6 +6,7 @@ from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import ParallelConfig, get_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
@@ -77,24 +78,20 @@ def determine_expert_counts(
|
||||
) -> tuple[int, int, int]:
|
||||
global_num_experts = num_experts + num_redundant_experts
|
||||
logical_num_experts = num_experts
|
||||
# ROCm aiter shared experts fusion
|
||||
# AITER only supports gated activations (silu/gelu), so disable it
|
||||
# for non-gated MoE (is_act_and_mul=False)
|
||||
# rocm_aiter_fmoe_enabled = rocm_aiter_ops.is_fused_moe_enabled() and is_act_and_mul
|
||||
aiter_fmoe_shared_expert_enabled = (
|
||||
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() and is_act_and_mul
|
||||
)
|
||||
# Shared-expert fusion: append the shared expert(s) as routed-expert slots
|
||||
# so they run in the same grouped GEMM. Gated by
|
||||
# VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: either the native aiter fused-MoE
|
||||
# path (env + master switch, via is_fusion_moe_shared_experts_enabled) or the
|
||||
# backend-neutral router-append path (env alone, independent of the master
|
||||
# switch; e.g. the MM3 triton/flydsl mxfp8 MoE). Gated activations only.
|
||||
fuse_shared_enabled = (
|
||||
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
or envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
|
||||
) and is_act_and_mul
|
||||
|
||||
num_fused_shared_experts = (
|
||||
n_shared_experts
|
||||
if n_shared_experts is not None and aiter_fmoe_shared_expert_enabled
|
||||
else 0
|
||||
n_shared_experts if n_shared_experts is not None and fuse_shared_enabled else 0
|
||||
)
|
||||
if not aiter_fmoe_shared_expert_enabled and num_fused_shared_experts != 0:
|
||||
raise ValueError(
|
||||
"n_shared_experts is only supported on ROCm aiter when "
|
||||
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled"
|
||||
)
|
||||
|
||||
return global_num_experts, logical_num_experts, num_fused_shared_experts
|
||||
|
||||
@@ -188,7 +185,8 @@ def FusedMoE(
|
||||
has_bias: Whether expert layers have bias terms
|
||||
is_sequence_parallel: Whether sequence parallelism is enabled
|
||||
expert_mapping: Expert parameter mapping for weight loading
|
||||
n_shared_experts: Number of shared experts (ROCm aiter only)
|
||||
n_shared_experts: Number of shared experts to fuse into the routed
|
||||
grouped GEMM (ROCm; requires aiter FSE or the router-append path)
|
||||
router_logits_dtype: Data type for router logits buffers
|
||||
gate: Pre-configured gate module
|
||||
shared_experts: Pre-configured shared experts module
|
||||
@@ -289,6 +287,19 @@ def FusedMoE(
|
||||
else 1.0,
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
num_fused_shared_experts=num_fused_shared_experts,
|
||||
# Fused shared-expert slot weight. With apply_routed_scale_to_output
|
||||
# the runner scales the combined output by routed_scaling_factor, so
|
||||
# the shared slot weight must be 1/routed_scaling_factor for its net
|
||||
# contribution to be 1.0 (matching the un-scaled separate-MLP add).
|
||||
shared_expert_weight=(
|
||||
(1.0 / routed_scaling_factor)
|
||||
if (
|
||||
apply_routed_scale_to_output
|
||||
and num_fused_shared_experts > 0
|
||||
and routed_scaling_factor
|
||||
)
|
||||
else 1.0
|
||||
),
|
||||
zero_expert_type=zero_expert_type,
|
||||
num_logical_experts=logical_num_experts,
|
||||
hash_indices_table=hash_indices_table,
|
||||
|
||||
@@ -334,6 +334,8 @@ class FusedTopKBiasRouter(BaseRouter):
|
||||
*,
|
||||
scoring_func: str = "sigmoid",
|
||||
hash_indices_table: torch.Tensor | None = None,
|
||||
num_fused_shared_experts: int = 0,
|
||||
shared_expert_weight: float = 1.0,
|
||||
):
|
||||
super().__init__(
|
||||
top_k=top_k,
|
||||
@@ -346,6 +348,11 @@ class FusedTopKBiasRouter(BaseRouter):
|
||||
self.routed_scaling_factor = routed_scaling_factor
|
||||
self.scoring_func = scoring_func
|
||||
self._hash_indices_table = hash_indices_table
|
||||
# Fused shared experts: append constant slots (ids immediately after
|
||||
# the routed experts, [global, global+n)) routed to by every token at
|
||||
# ``shared_expert_weight``, AFTER the routed top-k is renormalized.
|
||||
self.num_fused_shared_experts = num_fused_shared_experts
|
||||
self.shared_expert_weight = shared_expert_weight
|
||||
|
||||
@property
|
||||
def routing_method_type(self) -> RoutingMethodType:
|
||||
@@ -382,4 +389,23 @@ class FusedTopKBiasRouter(BaseRouter):
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
)
|
||||
|
||||
if self.num_fused_shared_experts > 0:
|
||||
m = topk_ids.shape[0]
|
||||
n = self.num_fused_shared_experts
|
||||
# global_num_experts counts only the routed experts; the fused
|
||||
# shared experts occupy the slots immediately after them, i.e. ids
|
||||
# [global_num_experts, global_num_experts + n).
|
||||
base = self.global_num_experts
|
||||
shared_ids = torch.arange(
|
||||
base, base + n, dtype=topk_ids.dtype, device=topk_ids.device
|
||||
).expand(m, n)
|
||||
shared_w = torch.full(
|
||||
(m, n),
|
||||
self.shared_expert_weight,
|
||||
dtype=topk_weights.dtype,
|
||||
device=topk_weights.device,
|
||||
)
|
||||
topk_ids = torch.cat([topk_ids, shared_ids], dim=-1)
|
||||
topk_weights = torch.cat([topk_weights, shared_w], dim=-1)
|
||||
|
||||
return topk_weights, topk_ids
|
||||
|
||||
@@ -47,6 +47,7 @@ def create_fused_moe_router(
|
||||
topk_group: int | None = None,
|
||||
scoring_func: str = "softmax",
|
||||
num_fused_shared_experts: int = 0,
|
||||
shared_expert_weight: float = 1.0,
|
||||
# grouped topk + fused topk bias parameters
|
||||
routed_scaling_factor: float = 1.0,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
@@ -188,6 +189,8 @@ def create_fused_moe_router(
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
scoring_func=scoring_func,
|
||||
hash_indices_table=hash_indices_table,
|
||||
num_fused_shared_experts=num_fused_shared_experts,
|
||||
shared_expert_weight=shared_expert_weight,
|
||||
)
|
||||
|
||||
if (
|
||||
|
||||
Reference in New Issue
Block a user