[ROCm][Perf] Use flydsl moe with Minimax-M3 mxfp8 weights on gfx950 and implemented moe-backend selection (#46184)

Signed-off-by: Hongxia Yang <hongxia.yang@amd.com>
Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: Tan Pin Siang <tanpinsiang@gmail.com>
This commit is contained in:
Hongxia Yang
2026-06-27 10:22:57 +00:00
committed by GitHub
co-authored by Claude Opus 4.8 TJian Andreas Karatzas Tan Pin Siang
parent 9fd00ee006
commit 867fd5e8ed
5 changed files with 374 additions and 4 deletions
@@ -0,0 +1,135 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MXFP8 MoE backend selection for the AITER FlyDSL kernel (gfx950).
GPU-free: mocks the platform (gfx950) and the ``flydsl`` package check, then
exercises the oracle so the FlyDSL backend is auto-picked when usable (including
under expert parallelism, since apply() forwards the expert_map as aiter's
expert_mask) and skipped (native fallback) when the device/package is missing.
"""
import dataclasses
from unittest.mock import patch
import pytest
from vllm.platforms import current_platform
if not current_platform.is_rocm():
pytest.skip("This test can only run on ROCm.", allow_module_level=True)
from tests.kernels.moe.utils import make_dummy_moe_config # noqa: E402
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp8_moe import ( # noqa: E402
AiterMxfp8Experts,
)
from vllm.model_executor.layers.fused_moe.modular_kernel import ( # noqa: E402
FusedMoEActivationFormat,
)
from vllm.model_executor.layers.fused_moe.oracle.fp8 import ( # noqa: E402
Fp8MoeBackend,
)
from vllm.model_executor.layers.fused_moe.oracle.mxfp8 import ( # noqa: E402
_BACKEND_NAME_MAP,
_SUPPORTED_BACKENDS,
_mxfp8_backend_to_kernel_cls,
_select_kernel_cls,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import ( # noqa: E402
kMxfp8Dynamic,
kMxfp8Static,
)
_AITER_MOD = "vllm.model_executor.layers.fused_moe.experts.aiter_mxfp8_moe"
def _config(ep_size: int = 1):
cfg = make_dummy_moe_config(num_experts=128, experts_per_token=4, hidden_dim=6144)
if ep_size != 1:
cfg = dataclasses.replace(
cfg,
moe_parallel_config=dataclasses.replace(
cfg.moe_parallel_config, ep_size=ep_size, use_ep=True
),
)
return cfg
def _gfx950():
"""Patch the platform so the device gate (gfx950 / MX) passes off-ROCm."""
return patch.multiple(
f"{_AITER_MOD}.current_platform",
is_rocm=lambda: True,
supports_mx=lambda: True,
)
def _flydsl_installed(present: bool):
return patch(f"{_AITER_MOD}.is_aiter_mxfp8_moe_available", return_value=present)
def test_aiter_mxfp8_registered():
"""The FlyDSL backend is auto-selectable and reachable via --moe-backend aiter."""
assert Fp8MoeBackend.AITER_MXFP8 in _SUPPORTED_BACKENDS
assert _BACKEND_NAME_MAP["aiter"] is Fp8MoeBackend.AITER_MXFP8
assert _mxfp8_backend_to_kernel_cls(Fp8MoeBackend.AITER_MXFP8) == [
AiterMxfp8Experts
]
def test_triton_selectable():
assert _BACKEND_NAME_MAP["triton"] is Fp8MoeBackend.TRITON_MXFP8
# Not auto-selected (only reachable explicitly), so FlyDSL still wins auto.
assert Fp8MoeBackend.TRITON_MXFP8 not in _SUPPORTED_BACKENDS
@pytest.mark.parametrize("ep_size", [1, 2])
def test_ep_supported(ep_size):
"""FlyDSL accepts both TP and EP: apply() forwards expert_map as expert_mask."""
assert (
AiterMxfp8Experts._supports_parallel_config(
_config(ep_size).moe_parallel_config
)
is True
)
@pytest.mark.parametrize(
"present,ep_size,supported,reason_substr",
[
(True, 1, True, None), # gfx950 + flydsl + TP -> selectable
(True, 2, True, None), # gfx950 + flydsl + EP -> selectable (expert_mask)
(False, 1, False, "flydsl package"), # package missing -> clear reason
],
)
def test_is_supported_config(present, ep_size, supported, reason_substr):
with _gfx950(), _flydsl_installed(present):
ok, reason = AiterMxfp8Experts.is_supported_config(
AiterMxfp8Experts,
_config(ep_size),
kMxfp8Static,
kMxfp8Dynamic,
FusedMoEActivationFormat.Standard,
)
assert ok is supported
if reason_substr is not None:
assert reason_substr in reason
def test_explicit_moe_backend_aiter():
"""--moe-backend aiter: returns FlyDSL when usable (TP or EP), else a clear
ValueError when the flydsl package is missing."""
with _gfx950(), _flydsl_installed(True):
assert (
_select_kernel_cls(Fp8MoeBackend.AITER_MXFP8, _config(1))
is AiterMxfp8Experts
)
assert (
_select_kernel_cls(Fp8MoeBackend.AITER_MXFP8, _config(2))
is AiterMxfp8Experts
)
with (
_gfx950(),
_flydsl_installed(False),
pytest.raises(ValueError, match="flydsl package"),
):
_select_kernel_cls(Fp8MoeBackend.AITER_MXFP8, _config(1))
+30
View File
@@ -171,6 +171,7 @@ def _rocm_aiter_fused_moe_impl(
bias1: torch.Tensor | None = None,
bias2: torch.Tensor | None = None,
moe_sorting_dispatch_policy: int = 0,
swiglu_limit: float = 0.0,
) -> torch.Tensor:
from aiter import ActivationType, QuantType
from aiter.fused_moe import fused_moe
@@ -203,6 +204,7 @@ def _rocm_aiter_fused_moe_impl(
bias1=bias1,
bias2=bias2,
moe_sorting_dispatch_policy=moe_sorting_dispatch_policy,
swiglu_limit=swiglu_limit,
**extra_kwargs,
)
@@ -229,6 +231,7 @@ def _rocm_aiter_fused_moe_fake(
bias1: torch.Tensor | None = None,
bias2: torch.Tensor | None = None,
moe_sorting_dispatch_policy: int = 0,
swiglu_limit: float = 0.0,
) -> torch.Tensor:
if output_dtype is not None:
return torch.empty_like(hidden_states, dtype=output_dtype)
@@ -2201,6 +2204,7 @@ class rocm_aiter_ops:
bias1: torch.Tensor | None = None,
bias2: torch.Tensor | None = None,
moe_sorting_dispatch_policy: int = 0,
swiglu_limit: float = 0.0,
) -> torch.Tensor:
return torch.ops.vllm.rocm_aiter_fused_moe(
hidden_states,
@@ -2224,6 +2228,7 @@ class rocm_aiter_ops:
bias1,
bias2,
moe_sorting_dispatch_policy,
swiglu_limit,
)
@staticmethod
@@ -2678,6 +2683,31 @@ class rocm_aiter_ops:
return tuple(shuffle_weight(tensor, layout=layout) for tensor in tensors)
@staticmethod
def shuffle_mxfp8_moe_weights(
w13: torch.Tensor,
w2: torch.Tensor,
w13_scale: torch.Tensor,
w2_scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Preshuffle MXFP8 MoE weights + E8M0 scales into AITER's FlyDSL layout:
gate/up-interleaved weights, interleaved scale for w13 (gate/up), plain
scale for w2 (the interleaved variant is gate/up-only and misaligns w2).
"""
from aiter.ops.shuffle import shuffle_scale, shuffle_weight
num_experts = w13.shape[0]
w13 = shuffle_weight(w13, is_guinterleave=True, gate_up=True)
w2 = shuffle_weight(w2, is_guinterleave=True, gate_up=False)
w13_scale = shuffle_scale(
w13_scale.reshape(-1, w13_scale.shape[-1]),
num_experts,
is_guinterleave=True,
gate_up=True,
)
w2_scale = shuffle_scale(w2_scale.reshape(-1, w2_scale.shape[-1]))
return w13, w2, w13_scale, w2_scale
@staticmethod
def flash_attn_varlen_func(
q: torch.Tensor,
@@ -0,0 +1,166 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MXFP8 (1x32 block, E8M0) MoE via AITER's FlyDSL two-stage grouped GEMM
(gfx950); alternative to ``Mxfp8NativeTritonExperts``. Routes through
``aiter.fused_moe`` (per_1x32, gate_mode=INTERLEAVE); weights are preshuffled in
``convert_to_fp8_moe_kernel_format``.
"""
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.experts.mxfp8_emulation_moe import (
Mxfp8TritonExpertsBase,
)
from vllm.platforms import current_platform
logger = init_logger(__name__)
def is_aiter_mxfp8_moe_available() -> bool:
"""True when the FlyDSL MXFP8 MoE can run here: gfx950, the ``flydsl``
package is importable, AND the installed aiter carries the mxfp8 FlyDSL
2-stage support from ROCm/aiter#3811.
``flydsl`` and ``aiter`` are separate packages, so ``is_flydsl_available()``
(flydsl pkg + arch) is necessary but not sufficient: an older aiter without
#3811 still ships the flydsl pkg and the ``aiter.ops.flydsl`` module but a
broken/missing ``per_1x32 + fp8`` 2-stage path. Without this extra gate a
nightly lacking #3811 would wrongly select FlyDSL instead of falling back to
the native Triton dot_scaled path. #3811 added no probe-able public symbol,
so detect the ``minimax_m3_mxfp8`` tuned config it shipped. Every check fails
closed (returns False -> triton dot_scaled), which is always safe."""
if not (current_platform.is_rocm() and current_platform.supports_mx()):
return False
try:
import os
import aiter
from aiter.ops.flydsl.utils import is_flydsl_available
if not is_flydsl_available():
return False
return os.path.exists(
os.path.join(
os.path.dirname(aiter.__file__),
"configs",
"model_configs",
"minimax_m3_mxfp8_tuned_fmoe.csv",
)
)
except Exception:
return False
class AiterMxfp8Experts(Mxfp8TritonExpertsBase):
"""MXFP8 MoE through AITER's FlyDSL two-stage grouped GEMM (gfx950)."""
@property
def quant_dtype(self) -> torch.dtype | str | None:
return self.quant_config.quant_dtype
@property
def block_shape(self) -> list[int] | None:
return self.quant_config.block_shape
@property
def expects_unquantized_inputs(self) -> bool:
# aiter.fused_moe MXFP8-quantizes the activations internally.
return True
@staticmethod
def _supports_current_device() -> bool:
# Device capability only (gfx950 / MX-capable ROCm). The flydsl package
# check lives in is_supported_config so a missing package is reported
# distinctly from an unsupported device.
return current_platform.is_rocm() and current_platform.supports_mx()
@staticmethod
def _supports_parallel_config(moe_parallel_config) -> bool:
# Both TP (expert_map=None) and EP are supported: apply() forwards the
# expert_map as aiter's ``expert_mask`` (the per-rank local-expert
# selection), mirroring the native rocm_aiter_moe path.
return True
@staticmethod
def is_supported_config(
cls, moe_config, weight_key, activation_key, activation_format
):
is_supported, reason = super().is_supported_config(
cls, moe_config, weight_key, activation_key, activation_format
)
# _supports_current_device() only gates on the device; surface a clear
# reason when the device is fine but the flydsl package is missing.
if is_supported and not is_aiter_mxfp8_moe_available():
return False, (
"kernel requires the aiter flydsl package, which is not installed"
)
return is_supported, reason
def apply(
self,
output: torch.Tensor,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
activation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
a2_scale: torch.Tensor | None,
workspace13: torch.Tensor,
workspace2: torch.Tensor,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
):
from aiter import ActivationType, QuantType
from aiter.ops.flydsl.moe_common import GateMode
from vllm._aiter_ops import rocm_aiter_ops
# Re-tag the preshuffled weights: replace_parameter drops the
# is_shuffled flag, without which aiter picks a broken CK kernel.
w1.is_shuffled = True
w2.is_shuffled = True
limit = self.quant_config.gemm1_clamp_limit
swiglu_limit = 0.0 if limit is None else float(limit)
# Under EP, aiter expects ``expert_mask`` as a 0/1 *local-expert* mask
# over global ids with a trailing fake-expert sentinel slot
# (shape ``[global_num_experts + 1]``), NOT vLLM's expert_map (a
# global->local index map with -1 for non-local). Convert it; aiter
# derives the global->local compaction from the mask itself. ``None``
# under pure TP.
if expert_map is not None:
local_mask = (expert_map >= 0).to(torch.int32)
expert_mask = torch.cat([local_mask, local_mask.new_zeros(1)])
else:
expert_mask = None
# Route through the graph-safe ``rocm_aiter_fused_moe`` custom op so the
# call is captured under HIP graphs / torch.compile (a direct
# ``aiter.fused_moe`` is opaque to the dispatcher). aiter requires FP32
# routing weights / INT32 ids.
out = rocm_aiter_ops.fused_moe(
hidden_states,
w1,
w2,
topk_weights.to(torch.float32),
topk_ids.to(torch.int32),
expert_mask=expert_mask,
activation_method=ActivationType.Swiglu.value,
quant_method=QuantType.per_1x32.value,
doweight_stage1=apply_router_weight_on_input,
w1_scale=self.w1_scale_val,
w2_scale=self.w2_scale_val,
a1_scale=None,
a2_scale=None,
gate_mode=GateMode.INTERLEAVE.value,
swiglu_limit=swiglu_limit,
output_dtype=output.dtype,
)
output.copy_(out.to(output.dtype))
@@ -59,7 +59,9 @@ class Fp8MoeBackend(Enum):
# MXFP8 MoE via a Triton ``dot_scaled`` kernel that lowers to CDNA4
# (gfx950) native MX matrix-core ops. Weights stay in MXFP8 (no load-time
# format conversion); the FP8 values + E8M0 scales are consumed directly.
NATIVE_MXFP8 = "NATIVE_MXFP8"
TRITON_MXFP8 = "TRITON_MXFP8"
# MXFP8 MoE via AITER (FlyDSL two-stage grouped GEMM) on gfx950.
AITER_MXFP8 = "AITER_MXFP8"
def _get_priority_backends(
@@ -423,6 +425,10 @@ def convert_to_fp8_moe_kernel_format(
)
elif fp8_backend == Fp8MoeBackend.AITER:
w13, w2 = rocm_aiter_ops.shuffle_weights(w13, w2)
elif fp8_backend == Fp8MoeBackend.AITER_MXFP8:
w13, w2, w13_scale, w2_scale = rocm_aiter_ops.shuffle_mxfp8_moe_weights(
w13, w2, w13_scale, w2_scale
)
elif fp8_backend == Fp8MoeBackend.MARLIN:
weight_block_size = getattr(layer, "weight_block_size", None)
if weight_block_size == [1, 32]:
@@ -484,7 +490,7 @@ def convert_to_fp8_moe_kernel_format(
# EMULATION dequantizes weights at runtime; NATIVE_MXFP8 consumes
# the MXFP8 weights as-is — neither needs a load-time layout change.
Fp8MoeBackend.EMULATION,
Fp8MoeBackend.NATIVE_MXFP8,
Fp8MoeBackend.TRITON_MXFP8,
]:
raise ValueError(f"Unsupported FP8 MoE backend: {fp8_backend.value}")
@@ -21,6 +21,10 @@ _SUPPORTED_BACKENDS = (
Fp8MoeBackend.DEEPGEMM,
Fp8MoeBackend.MARLIN,
Fp8MoeBackend.XPU,
# AITER FlyDSL (gfx950): auto-picked by select_mxfp8_moe_backend when
# is_supported_config passes (gfx950 + flydsl installed + not EP). On other
# devices / no flydsl / EP it is skipped and native is used.
Fp8MoeBackend.AITER_MXFP8,
)
_BACKEND_NAME_MAP: dict[str, Fp8MoeBackend] = {
@@ -28,6 +32,8 @@ _BACKEND_NAME_MAP: dict[str, Fp8MoeBackend] = {
"deep_gemm": Fp8MoeBackend.DEEPGEMM,
"marlin": Fp8MoeBackend.MARLIN,
"xpu": Fp8MoeBackend.XPU,
"aiter": Fp8MoeBackend.AITER_MXFP8,
"triton": Fp8MoeBackend.TRITON_MXFP8,
}
@@ -46,6 +52,27 @@ def _mxfp8_backend_to_kernel_cls(
)
return [DeepGemmExperts]
if backend == Fp8MoeBackend.AITER_MXFP8:
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp8_moe import (
AiterMxfp8Experts,
)
return [AiterMxfp8Experts]
if backend == Fp8MoeBackend.TRITON_MXFP8:
# Explicit ``--moe-backend triton``: the Triton mxfp8 path, i.e.
# dot_scaled on MX-capable HW (gfx950) and BF16 emulation otherwise.
# Mirrors the ROCm auto-fallback in ``_select_rocm_mxfp8_backend``.
if current_platform.supports_mx():
from vllm.model_executor.layers.fused_moe.experts.mxfp8_native_moe import (
Mxfp8NativeTritonExperts,
)
return [Mxfp8NativeTritonExperts]
from vllm.model_executor.layers.fused_moe.experts.mxfp8_emulation_moe import (
Mxfp8EmulationTritonExperts,
)
return [Mxfp8EmulationTritonExperts]
return backend_to_kernel_cls(backend)
@@ -77,7 +104,13 @@ def _select_kernel_cls(
def _select_rocm_mxfp8_backend() -> tuple[Fp8MoeBackend, type[mk.FusedMoEExperts]]:
"""ROCm fallback when vendor MXFP8 backends are unavailable."""
"""ROCm fallback when no auto-selected MXFP8 backend is available.
The aiter FlyDSL backend (``AITER_MXFP8``) is auto-picked earlier by
``select_mxfp8_moe_backend`` via ``_SUPPORTED_BACKENDS`` when usable, or
explicitly via ``--moe-backend aiter``; this fallback handles the rest
(native dot_scaled on gfx950, else BF16 emulation).
"""
if current_platform.supports_mx():
from vllm.model_executor.layers.fused_moe.experts.mxfp8_native_moe import (
@@ -85,7 +118,7 @@ def _select_rocm_mxfp8_backend() -> tuple[Fp8MoeBackend, type[mk.FusedMoEExperts
)
logger.info_once("Using native CDNA4 (gfx950) MXFP8 dot_scaled MoE backend.")
return Fp8MoeBackend.NATIVE_MXFP8, Mxfp8NativeTritonExperts
return Fp8MoeBackend.TRITON_MXFP8, Mxfp8NativeTritonExperts
from vllm.model_executor.layers.fused_moe.experts.mxfp8_emulation_moe import (
Mxfp8EmulationTritonExperts,