From 867fd5e8ed6b0bfcf84b24f82658c9fb698a6d35 Mon Sep 17 00:00:00 2001 From: Hongxia Yang <62075498+hongxiayang@users.noreply.github.com> Date: Sat, 27 Jun 2026 06:22:57 -0400 Subject: [PATCH] [ROCm][Perf] Use flydsl moe with Minimax-M3 mxfp8 weights on gfx950 and implemented moe-backend selection (#46184) Signed-off-by: Hongxia Yang Signed-off-by: tjtanaa Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: TJian Co-authored-by: Andreas Karatzas Co-authored-by: Tan Pin Siang --- .../moe/test_mxfp8_aiter_backend_selection.py | 135 ++++++++++++++ vllm/_aiter_ops.py | 30 ++++ .../fused_moe/experts/aiter_mxfp8_moe.py | 166 ++++++++++++++++++ .../layers/fused_moe/oracle/fp8.py | 10 +- .../layers/fused_moe/oracle/mxfp8.py | 37 +++- 5 files changed, 374 insertions(+), 4 deletions(-) create mode 100644 tests/kernels/moe/test_mxfp8_aiter_backend_selection.py create mode 100644 vllm/model_executor/layers/fused_moe/experts/aiter_mxfp8_moe.py diff --git a/tests/kernels/moe/test_mxfp8_aiter_backend_selection.py b/tests/kernels/moe/test_mxfp8_aiter_backend_selection.py new file mode 100644 index 0000000000..7c2fdbabe2 --- /dev/null +++ b/tests/kernels/moe/test_mxfp8_aiter_backend_selection.py @@ -0,0 +1,135 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""MXFP8 MoE backend selection for the AITER FlyDSL kernel (gfx950). + +GPU-free: mocks the platform (gfx950) and the ``flydsl`` package check, then +exercises the oracle so the FlyDSL backend is auto-picked when usable (including +under expert parallelism, since apply() forwards the expert_map as aiter's +expert_mask) and skipped (native fallback) when the device/package is missing. +""" + +import dataclasses +from unittest.mock import patch + +import pytest + +from vllm.platforms import current_platform + +if not current_platform.is_rocm(): + pytest.skip("This test can only run on ROCm.", allow_module_level=True) + +from tests.kernels.moe.utils import make_dummy_moe_config # noqa: E402 +from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp8_moe import ( # noqa: E402 + AiterMxfp8Experts, +) +from vllm.model_executor.layers.fused_moe.modular_kernel import ( # noqa: E402 + FusedMoEActivationFormat, +) +from vllm.model_executor.layers.fused_moe.oracle.fp8 import ( # noqa: E402 + Fp8MoeBackend, +) +from vllm.model_executor.layers.fused_moe.oracle.mxfp8 import ( # noqa: E402 + _BACKEND_NAME_MAP, + _SUPPORTED_BACKENDS, + _mxfp8_backend_to_kernel_cls, + _select_kernel_cls, +) +from vllm.model_executor.layers.quantization.utils.quant_utils import ( # noqa: E402 + kMxfp8Dynamic, + kMxfp8Static, +) + +_AITER_MOD = "vllm.model_executor.layers.fused_moe.experts.aiter_mxfp8_moe" + + +def _config(ep_size: int = 1): + cfg = make_dummy_moe_config(num_experts=128, experts_per_token=4, hidden_dim=6144) + if ep_size != 1: + cfg = dataclasses.replace( + cfg, + moe_parallel_config=dataclasses.replace( + cfg.moe_parallel_config, ep_size=ep_size, use_ep=True + ), + ) + return cfg + + +def _gfx950(): + """Patch the platform so the device gate (gfx950 / MX) passes off-ROCm.""" + return patch.multiple( + f"{_AITER_MOD}.current_platform", + is_rocm=lambda: True, + supports_mx=lambda: True, + ) + + +def _flydsl_installed(present: bool): + return patch(f"{_AITER_MOD}.is_aiter_mxfp8_moe_available", return_value=present) + + +def test_aiter_mxfp8_registered(): + """The FlyDSL backend is auto-selectable and reachable via --moe-backend aiter.""" + assert Fp8MoeBackend.AITER_MXFP8 in _SUPPORTED_BACKENDS + assert _BACKEND_NAME_MAP["aiter"] is Fp8MoeBackend.AITER_MXFP8 + assert _mxfp8_backend_to_kernel_cls(Fp8MoeBackend.AITER_MXFP8) == [ + AiterMxfp8Experts + ] + + +def test_triton_selectable(): + assert _BACKEND_NAME_MAP["triton"] is Fp8MoeBackend.TRITON_MXFP8 + # Not auto-selected (only reachable explicitly), so FlyDSL still wins auto. + assert Fp8MoeBackend.TRITON_MXFP8 not in _SUPPORTED_BACKENDS + + +@pytest.mark.parametrize("ep_size", [1, 2]) +def test_ep_supported(ep_size): + """FlyDSL accepts both TP and EP: apply() forwards expert_map as expert_mask.""" + assert ( + AiterMxfp8Experts._supports_parallel_config( + _config(ep_size).moe_parallel_config + ) + is True + ) + + +@pytest.mark.parametrize( + "present,ep_size,supported,reason_substr", + [ + (True, 1, True, None), # gfx950 + flydsl + TP -> selectable + (True, 2, True, None), # gfx950 + flydsl + EP -> selectable (expert_mask) + (False, 1, False, "flydsl package"), # package missing -> clear reason + ], +) +def test_is_supported_config(present, ep_size, supported, reason_substr): + with _gfx950(), _flydsl_installed(present): + ok, reason = AiterMxfp8Experts.is_supported_config( + AiterMxfp8Experts, + _config(ep_size), + kMxfp8Static, + kMxfp8Dynamic, + FusedMoEActivationFormat.Standard, + ) + assert ok is supported + if reason_substr is not None: + assert reason_substr in reason + + +def test_explicit_moe_backend_aiter(): + """--moe-backend aiter: returns FlyDSL when usable (TP or EP), else a clear + ValueError when the flydsl package is missing.""" + with _gfx950(), _flydsl_installed(True): + assert ( + _select_kernel_cls(Fp8MoeBackend.AITER_MXFP8, _config(1)) + is AiterMxfp8Experts + ) + assert ( + _select_kernel_cls(Fp8MoeBackend.AITER_MXFP8, _config(2)) + is AiterMxfp8Experts + ) + with ( + _gfx950(), + _flydsl_installed(False), + pytest.raises(ValueError, match="flydsl package"), + ): + _select_kernel_cls(Fp8MoeBackend.AITER_MXFP8, _config(1)) diff --git a/vllm/_aiter_ops.py b/vllm/_aiter_ops.py index 95a5361032..4a8b4209d8 100644 --- a/vllm/_aiter_ops.py +++ b/vllm/_aiter_ops.py @@ -171,6 +171,7 @@ def _rocm_aiter_fused_moe_impl( bias1: torch.Tensor | None = None, bias2: torch.Tensor | None = None, moe_sorting_dispatch_policy: int = 0, + swiglu_limit: float = 0.0, ) -> torch.Tensor: from aiter import ActivationType, QuantType from aiter.fused_moe import fused_moe @@ -203,6 +204,7 @@ def _rocm_aiter_fused_moe_impl( bias1=bias1, bias2=bias2, moe_sorting_dispatch_policy=moe_sorting_dispatch_policy, + swiglu_limit=swiglu_limit, **extra_kwargs, ) @@ -229,6 +231,7 @@ def _rocm_aiter_fused_moe_fake( bias1: torch.Tensor | None = None, bias2: torch.Tensor | None = None, moe_sorting_dispatch_policy: int = 0, + swiglu_limit: float = 0.0, ) -> torch.Tensor: if output_dtype is not None: return torch.empty_like(hidden_states, dtype=output_dtype) @@ -2201,6 +2204,7 @@ class rocm_aiter_ops: bias1: torch.Tensor | None = None, bias2: torch.Tensor | None = None, moe_sorting_dispatch_policy: int = 0, + swiglu_limit: float = 0.0, ) -> torch.Tensor: return torch.ops.vllm.rocm_aiter_fused_moe( hidden_states, @@ -2224,6 +2228,7 @@ class rocm_aiter_ops: bias1, bias2, moe_sorting_dispatch_policy, + swiglu_limit, ) @staticmethod @@ -2678,6 +2683,31 @@ class rocm_aiter_ops: return tuple(shuffle_weight(tensor, layout=layout) for tensor in tensors) + @staticmethod + def shuffle_mxfp8_moe_weights( + w13: torch.Tensor, + w2: torch.Tensor, + w13_scale: torch.Tensor, + w2_scale: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Preshuffle MXFP8 MoE weights + E8M0 scales into AITER's FlyDSL layout: + gate/up-interleaved weights, interleaved scale for w13 (gate/up), plain + scale for w2 (the interleaved variant is gate/up-only and misaligns w2). + """ + from aiter.ops.shuffle import shuffle_scale, shuffle_weight + + num_experts = w13.shape[0] + w13 = shuffle_weight(w13, is_guinterleave=True, gate_up=True) + w2 = shuffle_weight(w2, is_guinterleave=True, gate_up=False) + w13_scale = shuffle_scale( + w13_scale.reshape(-1, w13_scale.shape[-1]), + num_experts, + is_guinterleave=True, + gate_up=True, + ) + w2_scale = shuffle_scale(w2_scale.reshape(-1, w2_scale.shape[-1])) + return w13, w2, w13_scale, w2_scale + @staticmethod def flash_attn_varlen_func( q: torch.Tensor, diff --git a/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp8_moe.py b/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp8_moe.py new file mode 100644 index 0000000000..3cbab0a0d5 --- /dev/null +++ b/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp8_moe.py @@ -0,0 +1,166 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""MXFP8 (1x32 block, E8M0) MoE via AITER's FlyDSL two-stage grouped GEMM +(gfx950); alternative to ``Mxfp8NativeTritonExperts``. Routes through +``aiter.fused_moe`` (per_1x32, gate_mode=INTERLEAVE); weights are preshuffled in +``convert_to_fp8_moe_kernel_format``. +""" + +import torch + +import vllm.model_executor.layers.fused_moe.modular_kernel as mk +from vllm.logger import init_logger +from vllm.model_executor.layers.fused_moe.experts.mxfp8_emulation_moe import ( + Mxfp8TritonExpertsBase, +) +from vllm.platforms import current_platform + +logger = init_logger(__name__) + + +def is_aiter_mxfp8_moe_available() -> bool: + """True when the FlyDSL MXFP8 MoE can run here: gfx950, the ``flydsl`` + package is importable, AND the installed aiter carries the mxfp8 FlyDSL + 2-stage support from ROCm/aiter#3811. + + ``flydsl`` and ``aiter`` are separate packages, so ``is_flydsl_available()`` + (flydsl pkg + arch) is necessary but not sufficient: an older aiter without + #3811 still ships the flydsl pkg and the ``aiter.ops.flydsl`` module but a + broken/missing ``per_1x32 + fp8`` 2-stage path. Without this extra gate a + nightly lacking #3811 would wrongly select FlyDSL instead of falling back to + the native Triton dot_scaled path. #3811 added no probe-able public symbol, + so detect the ``minimax_m3_mxfp8`` tuned config it shipped. Every check fails + closed (returns False -> triton dot_scaled), which is always safe.""" + if not (current_platform.is_rocm() and current_platform.supports_mx()): + return False + try: + import os + + import aiter + from aiter.ops.flydsl.utils import is_flydsl_available + + if not is_flydsl_available(): + return False + return os.path.exists( + os.path.join( + os.path.dirname(aiter.__file__), + "configs", + "model_configs", + "minimax_m3_mxfp8_tuned_fmoe.csv", + ) + ) + except Exception: + return False + + +class AiterMxfp8Experts(Mxfp8TritonExpertsBase): + """MXFP8 MoE through AITER's FlyDSL two-stage grouped GEMM (gfx950).""" + + @property + def quant_dtype(self) -> torch.dtype | str | None: + return self.quant_config.quant_dtype + + @property + def block_shape(self) -> list[int] | None: + return self.quant_config.block_shape + + @property + def expects_unquantized_inputs(self) -> bool: + # aiter.fused_moe MXFP8-quantizes the activations internally. + return True + + @staticmethod + def _supports_current_device() -> bool: + # Device capability only (gfx950 / MX-capable ROCm). The flydsl package + # check lives in is_supported_config so a missing package is reported + # distinctly from an unsupported device. + return current_platform.is_rocm() and current_platform.supports_mx() + + @staticmethod + def _supports_parallel_config(moe_parallel_config) -> bool: + # Both TP (expert_map=None) and EP are supported: apply() forwards the + # expert_map as aiter's ``expert_mask`` (the per-rank local-expert + # selection), mirroring the native rocm_aiter_moe path. + return True + + @staticmethod + def is_supported_config( + cls, moe_config, weight_key, activation_key, activation_format + ): + is_supported, reason = super().is_supported_config( + cls, moe_config, weight_key, activation_key, activation_format + ) + # _supports_current_device() only gates on the device; surface a clear + # reason when the device is fine but the flydsl package is missing. + if is_supported and not is_aiter_mxfp8_moe_available(): + return False, ( + "kernel requires the aiter flydsl package, which is not installed" + ) + return is_supported, reason + + def apply( + self, + output: torch.Tensor, + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + activation, + global_num_experts: int, + expert_map: torch.Tensor | None, + a1q_scale: torch.Tensor | None, + a2_scale: torch.Tensor | None, + workspace13: torch.Tensor, + workspace2: torch.Tensor, + expert_tokens_meta: mk.ExpertTokensMetadata | None, + apply_router_weight_on_input: bool, + ): + from aiter import ActivationType, QuantType + from aiter.ops.flydsl.moe_common import GateMode + + from vllm._aiter_ops import rocm_aiter_ops + + # Re-tag the preshuffled weights: replace_parameter drops the + # is_shuffled flag, without which aiter picks a broken CK kernel. + w1.is_shuffled = True + w2.is_shuffled = True + + limit = self.quant_config.gemm1_clamp_limit + swiglu_limit = 0.0 if limit is None else float(limit) + + # Under EP, aiter expects ``expert_mask`` as a 0/1 *local-expert* mask + # over global ids with a trailing fake-expert sentinel slot + # (shape ``[global_num_experts + 1]``), NOT vLLM's expert_map (a + # global->local index map with -1 for non-local). Convert it; aiter + # derives the global->local compaction from the mask itself. ``None`` + # under pure TP. + if expert_map is not None: + local_mask = (expert_map >= 0).to(torch.int32) + expert_mask = torch.cat([local_mask, local_mask.new_zeros(1)]) + else: + expert_mask = None + + # Route through the graph-safe ``rocm_aiter_fused_moe`` custom op so the + # call is captured under HIP graphs / torch.compile (a direct + # ``aiter.fused_moe`` is opaque to the dispatcher). aiter requires FP32 + # routing weights / INT32 ids. + out = rocm_aiter_ops.fused_moe( + hidden_states, + w1, + w2, + topk_weights.to(torch.float32), + topk_ids.to(torch.int32), + expert_mask=expert_mask, + activation_method=ActivationType.Swiglu.value, + quant_method=QuantType.per_1x32.value, + doweight_stage1=apply_router_weight_on_input, + w1_scale=self.w1_scale_val, + w2_scale=self.w2_scale_val, + a1_scale=None, + a2_scale=None, + gate_mode=GateMode.INTERLEAVE.value, + swiglu_limit=swiglu_limit, + output_dtype=output.dtype, + ) + output.copy_(out.to(output.dtype)) diff --git a/vllm/model_executor/layers/fused_moe/oracle/fp8.py b/vllm/model_executor/layers/fused_moe/oracle/fp8.py index 9f930f1d58..862f292009 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/fp8.py +++ b/vllm/model_executor/layers/fused_moe/oracle/fp8.py @@ -59,7 +59,9 @@ class Fp8MoeBackend(Enum): # MXFP8 MoE via a Triton ``dot_scaled`` kernel that lowers to CDNA4 # (gfx950) native MX matrix-core ops. Weights stay in MXFP8 (no load-time # format conversion); the FP8 values + E8M0 scales are consumed directly. - NATIVE_MXFP8 = "NATIVE_MXFP8" + TRITON_MXFP8 = "TRITON_MXFP8" + # MXFP8 MoE via AITER (FlyDSL two-stage grouped GEMM) on gfx950. + AITER_MXFP8 = "AITER_MXFP8" def _get_priority_backends( @@ -423,6 +425,10 @@ def convert_to_fp8_moe_kernel_format( ) elif fp8_backend == Fp8MoeBackend.AITER: w13, w2 = rocm_aiter_ops.shuffle_weights(w13, w2) + elif fp8_backend == Fp8MoeBackend.AITER_MXFP8: + w13, w2, w13_scale, w2_scale = rocm_aiter_ops.shuffle_mxfp8_moe_weights( + w13, w2, w13_scale, w2_scale + ) elif fp8_backend == Fp8MoeBackend.MARLIN: weight_block_size = getattr(layer, "weight_block_size", None) if weight_block_size == [1, 32]: @@ -484,7 +490,7 @@ def convert_to_fp8_moe_kernel_format( # EMULATION dequantizes weights at runtime; NATIVE_MXFP8 consumes # the MXFP8 weights as-is — neither needs a load-time layout change. Fp8MoeBackend.EMULATION, - Fp8MoeBackend.NATIVE_MXFP8, + Fp8MoeBackend.TRITON_MXFP8, ]: raise ValueError(f"Unsupported FP8 MoE backend: {fp8_backend.value}") diff --git a/vllm/model_executor/layers/fused_moe/oracle/mxfp8.py b/vllm/model_executor/layers/fused_moe/oracle/mxfp8.py index d0d7c76481..06b622a6c4 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/mxfp8.py +++ b/vllm/model_executor/layers/fused_moe/oracle/mxfp8.py @@ -21,6 +21,10 @@ _SUPPORTED_BACKENDS = ( Fp8MoeBackend.DEEPGEMM, Fp8MoeBackend.MARLIN, Fp8MoeBackend.XPU, + # AITER FlyDSL (gfx950): auto-picked by select_mxfp8_moe_backend when + # is_supported_config passes (gfx950 + flydsl installed + not EP). On other + # devices / no flydsl / EP it is skipped and native is used. + Fp8MoeBackend.AITER_MXFP8, ) _BACKEND_NAME_MAP: dict[str, Fp8MoeBackend] = { @@ -28,6 +32,8 @@ _BACKEND_NAME_MAP: dict[str, Fp8MoeBackend] = { "deep_gemm": Fp8MoeBackend.DEEPGEMM, "marlin": Fp8MoeBackend.MARLIN, "xpu": Fp8MoeBackend.XPU, + "aiter": Fp8MoeBackend.AITER_MXFP8, + "triton": Fp8MoeBackend.TRITON_MXFP8, } @@ -46,6 +52,27 @@ def _mxfp8_backend_to_kernel_cls( ) return [DeepGemmExperts] + if backend == Fp8MoeBackend.AITER_MXFP8: + from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp8_moe import ( + AiterMxfp8Experts, + ) + + return [AiterMxfp8Experts] + if backend == Fp8MoeBackend.TRITON_MXFP8: + # Explicit ``--moe-backend triton``: the Triton mxfp8 path, i.e. + # dot_scaled on MX-capable HW (gfx950) and BF16 emulation otherwise. + # Mirrors the ROCm auto-fallback in ``_select_rocm_mxfp8_backend``. + if current_platform.supports_mx(): + from vllm.model_executor.layers.fused_moe.experts.mxfp8_native_moe import ( + Mxfp8NativeTritonExperts, + ) + + return [Mxfp8NativeTritonExperts] + from vllm.model_executor.layers.fused_moe.experts.mxfp8_emulation_moe import ( + Mxfp8EmulationTritonExperts, + ) + + return [Mxfp8EmulationTritonExperts] return backend_to_kernel_cls(backend) @@ -77,7 +104,13 @@ def _select_kernel_cls( def _select_rocm_mxfp8_backend() -> tuple[Fp8MoeBackend, type[mk.FusedMoEExperts]]: - """ROCm fallback when vendor MXFP8 backends are unavailable.""" + """ROCm fallback when no auto-selected MXFP8 backend is available. + + The aiter FlyDSL backend (``AITER_MXFP8``) is auto-picked earlier by + ``select_mxfp8_moe_backend`` via ``_SUPPORTED_BACKENDS`` when usable, or + explicitly via ``--moe-backend aiter``; this fallback handles the rest + (native dot_scaled on gfx950, else BF16 emulation). + """ if current_platform.supports_mx(): from vllm.model_executor.layers.fused_moe.experts.mxfp8_native_moe import ( @@ -85,7 +118,7 @@ def _select_rocm_mxfp8_backend() -> tuple[Fp8MoeBackend, type[mk.FusedMoEExperts ) logger.info_once("Using native CDNA4 (gfx950) MXFP8 dot_scaled MoE backend.") - return Fp8MoeBackend.NATIVE_MXFP8, Mxfp8NativeTritonExperts + return Fp8MoeBackend.TRITON_MXFP8, Mxfp8NativeTritonExperts from vllm.model_executor.layers.fused_moe.experts.mxfp8_emulation_moe import ( Mxfp8EmulationTritonExperts,