[ROCm][Perf] Use flydsl moe with Minimax-M3 mxfp8 weights on gfx950 and implemented moe-backend selection (#46184)

Signed-off-by: Hongxia Yang <hongxia.yang@amd.com>
Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: Tan Pin Siang <tanpinsiang@gmail.com>
This commit is contained in:
Hongxia Yang
2026-06-27 10:22:57 +00:00
committed by GitHub
co-authored by Claude Opus 4.8 TJian Andreas Karatzas Tan Pin Siang
parent 9fd00ee006
commit 867fd5e8ed
5 changed files with 374 additions and 4 deletions
@@ -0,0 +1,166 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MXFP8 (1x32 block, E8M0) MoE via AITER's FlyDSL two-stage grouped GEMM
(gfx950); alternative to ``Mxfp8NativeTritonExperts``. Routes through
``aiter.fused_moe`` (per_1x32, gate_mode=INTERLEAVE); weights are preshuffled in
``convert_to_fp8_moe_kernel_format``.
"""
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.experts.mxfp8_emulation_moe import (
Mxfp8TritonExpertsBase,
)
from vllm.platforms import current_platform
logger = init_logger(__name__)
def is_aiter_mxfp8_moe_available() -> bool:
"""True when the FlyDSL MXFP8 MoE can run here: gfx950, the ``flydsl``
package is importable, AND the installed aiter carries the mxfp8 FlyDSL
2-stage support from ROCm/aiter#3811.
``flydsl`` and ``aiter`` are separate packages, so ``is_flydsl_available()``
(flydsl pkg + arch) is necessary but not sufficient: an older aiter without
#3811 still ships the flydsl pkg and the ``aiter.ops.flydsl`` module but a
broken/missing ``per_1x32 + fp8`` 2-stage path. Without this extra gate a
nightly lacking #3811 would wrongly select FlyDSL instead of falling back to
the native Triton dot_scaled path. #3811 added no probe-able public symbol,
so detect the ``minimax_m3_mxfp8`` tuned config it shipped. Every check fails
closed (returns False -> triton dot_scaled), which is always safe."""
if not (current_platform.is_rocm() and current_platform.supports_mx()):
return False
try:
import os
import aiter
from aiter.ops.flydsl.utils import is_flydsl_available
if not is_flydsl_available():
return False
return os.path.exists(
os.path.join(
os.path.dirname(aiter.__file__),
"configs",
"model_configs",
"minimax_m3_mxfp8_tuned_fmoe.csv",
)
)
except Exception:
return False
class AiterMxfp8Experts(Mxfp8TritonExpertsBase):
"""MXFP8 MoE through AITER's FlyDSL two-stage grouped GEMM (gfx950)."""
@property
def quant_dtype(self) -> torch.dtype | str | None:
return self.quant_config.quant_dtype
@property
def block_shape(self) -> list[int] | None:
return self.quant_config.block_shape
@property
def expects_unquantized_inputs(self) -> bool:
# aiter.fused_moe MXFP8-quantizes the activations internally.
return True
@staticmethod
def _supports_current_device() -> bool:
# Device capability only (gfx950 / MX-capable ROCm). The flydsl package
# check lives in is_supported_config so a missing package is reported
# distinctly from an unsupported device.
return current_platform.is_rocm() and current_platform.supports_mx()
@staticmethod
def _supports_parallel_config(moe_parallel_config) -> bool:
# Both TP (expert_map=None) and EP are supported: apply() forwards the
# expert_map as aiter's ``expert_mask`` (the per-rank local-expert
# selection), mirroring the native rocm_aiter_moe path.
return True
@staticmethod
def is_supported_config(
cls, moe_config, weight_key, activation_key, activation_format
):
is_supported, reason = super().is_supported_config(
cls, moe_config, weight_key, activation_key, activation_format
)
# _supports_current_device() only gates on the device; surface a clear
# reason when the device is fine but the flydsl package is missing.
if is_supported and not is_aiter_mxfp8_moe_available():
return False, (
"kernel requires the aiter flydsl package, which is not installed"
)
return is_supported, reason
def apply(
self,
output: torch.Tensor,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
activation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
a2_scale: torch.Tensor | None,
workspace13: torch.Tensor,
workspace2: torch.Tensor,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
):
from aiter import ActivationType, QuantType
from aiter.ops.flydsl.moe_common import GateMode
from vllm._aiter_ops import rocm_aiter_ops
# Re-tag the preshuffled weights: replace_parameter drops the
# is_shuffled flag, without which aiter picks a broken CK kernel.
w1.is_shuffled = True
w2.is_shuffled = True
limit = self.quant_config.gemm1_clamp_limit
swiglu_limit = 0.0 if limit is None else float(limit)
# Under EP, aiter expects ``expert_mask`` as a 0/1 *local-expert* mask
# over global ids with a trailing fake-expert sentinel slot
# (shape ``[global_num_experts + 1]``), NOT vLLM's expert_map (a
# global->local index map with -1 for non-local). Convert it; aiter
# derives the global->local compaction from the mask itself. ``None``
# under pure TP.
if expert_map is not None:
local_mask = (expert_map >= 0).to(torch.int32)
expert_mask = torch.cat([local_mask, local_mask.new_zeros(1)])
else:
expert_mask = None
# Route through the graph-safe ``rocm_aiter_fused_moe`` custom op so the
# call is captured under HIP graphs / torch.compile (a direct
# ``aiter.fused_moe`` is opaque to the dispatcher). aiter requires FP32
# routing weights / INT32 ids.
out = rocm_aiter_ops.fused_moe(
hidden_states,
w1,
w2,
topk_weights.to(torch.float32),
topk_ids.to(torch.int32),
expert_mask=expert_mask,
activation_method=ActivationType.Swiglu.value,
quant_method=QuantType.per_1x32.value,
doweight_stage1=apply_router_weight_on_input,
w1_scale=self.w1_scale_val,
w2_scale=self.w2_scale_val,
a1_scale=None,
a2_scale=None,
gate_mode=GateMode.INTERLEAVE.value,
swiglu_limit=swiglu_limit,
output_dtype=output.dtype,
)
output.copy_(out.to(output.dtype))
@@ -59,7 +59,9 @@ class Fp8MoeBackend(Enum):
# MXFP8 MoE via a Triton ``dot_scaled`` kernel that lowers to CDNA4
# (gfx950) native MX matrix-core ops. Weights stay in MXFP8 (no load-time
# format conversion); the FP8 values + E8M0 scales are consumed directly.
NATIVE_MXFP8 = "NATIVE_MXFP8"
TRITON_MXFP8 = "TRITON_MXFP8"
# MXFP8 MoE via AITER (FlyDSL two-stage grouped GEMM) on gfx950.
AITER_MXFP8 = "AITER_MXFP8"
def _get_priority_backends(
@@ -423,6 +425,10 @@ def convert_to_fp8_moe_kernel_format(
)
elif fp8_backend == Fp8MoeBackend.AITER:
w13, w2 = rocm_aiter_ops.shuffle_weights(w13, w2)
elif fp8_backend == Fp8MoeBackend.AITER_MXFP8:
w13, w2, w13_scale, w2_scale = rocm_aiter_ops.shuffle_mxfp8_moe_weights(
w13, w2, w13_scale, w2_scale
)
elif fp8_backend == Fp8MoeBackend.MARLIN:
weight_block_size = getattr(layer, "weight_block_size", None)
if weight_block_size == [1, 32]:
@@ -484,7 +490,7 @@ def convert_to_fp8_moe_kernel_format(
# EMULATION dequantizes weights at runtime; NATIVE_MXFP8 consumes
# the MXFP8 weights as-is — neither needs a load-time layout change.
Fp8MoeBackend.EMULATION,
Fp8MoeBackend.NATIVE_MXFP8,
Fp8MoeBackend.TRITON_MXFP8,
]:
raise ValueError(f"Unsupported FP8 MoE backend: {fp8_backend.value}")
@@ -21,6 +21,10 @@ _SUPPORTED_BACKENDS = (
Fp8MoeBackend.DEEPGEMM,
Fp8MoeBackend.MARLIN,
Fp8MoeBackend.XPU,
# AITER FlyDSL (gfx950): auto-picked by select_mxfp8_moe_backend when
# is_supported_config passes (gfx950 + flydsl installed + not EP). On other
# devices / no flydsl / EP it is skipped and native is used.
Fp8MoeBackend.AITER_MXFP8,
)
_BACKEND_NAME_MAP: dict[str, Fp8MoeBackend] = {
@@ -28,6 +32,8 @@ _BACKEND_NAME_MAP: dict[str, Fp8MoeBackend] = {
"deep_gemm": Fp8MoeBackend.DEEPGEMM,
"marlin": Fp8MoeBackend.MARLIN,
"xpu": Fp8MoeBackend.XPU,
"aiter": Fp8MoeBackend.AITER_MXFP8,
"triton": Fp8MoeBackend.TRITON_MXFP8,
}
@@ -46,6 +52,27 @@ def _mxfp8_backend_to_kernel_cls(
)
return [DeepGemmExperts]
if backend == Fp8MoeBackend.AITER_MXFP8:
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp8_moe import (
AiterMxfp8Experts,
)
return [AiterMxfp8Experts]
if backend == Fp8MoeBackend.TRITON_MXFP8:
# Explicit ``--moe-backend triton``: the Triton mxfp8 path, i.e.
# dot_scaled on MX-capable HW (gfx950) and BF16 emulation otherwise.
# Mirrors the ROCm auto-fallback in ``_select_rocm_mxfp8_backend``.
if current_platform.supports_mx():
from vllm.model_executor.layers.fused_moe.experts.mxfp8_native_moe import (
Mxfp8NativeTritonExperts,
)
return [Mxfp8NativeTritonExperts]
from vllm.model_executor.layers.fused_moe.experts.mxfp8_emulation_moe import (
Mxfp8EmulationTritonExperts,
)
return [Mxfp8EmulationTritonExperts]
return backend_to_kernel_cls(backend)
@@ -77,7 +104,13 @@ def _select_kernel_cls(
def _select_rocm_mxfp8_backend() -> tuple[Fp8MoeBackend, type[mk.FusedMoEExperts]]:
"""ROCm fallback when vendor MXFP8 backends are unavailable."""
"""ROCm fallback when no auto-selected MXFP8 backend is available.
The aiter FlyDSL backend (``AITER_MXFP8``) is auto-picked earlier by
``select_mxfp8_moe_backend`` via ``_SUPPORTED_BACKENDS`` when usable, or
explicitly via ``--moe-backend aiter``; this fallback handles the rest
(native dot_scaled on gfx950, else BF16 emulation).
"""
if current_platform.supports_mx():
from vllm.model_executor.layers.fused_moe.experts.mxfp8_native_moe import (
@@ -85,7 +118,7 @@ def _select_rocm_mxfp8_backend() -> tuple[Fp8MoeBackend, type[mk.FusedMoEExperts
)
logger.info_once("Using native CDNA4 (gfx950) MXFP8 dot_scaled MoE backend.")
return Fp8MoeBackend.NATIVE_MXFP8, Mxfp8NativeTritonExperts
return Fp8MoeBackend.TRITON_MXFP8, Mxfp8NativeTritonExperts
from vllm.model_executor.layers.fused_moe.experts.mxfp8_emulation_moe import (
Mxfp8EmulationTritonExperts,