[MXFP8][ROCm] Fix MXFP8 MoE backend selection (#49747)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
This commit is contained in:
fxmarty-amd
2026-07-29 07:57:20 +08:00
committed by GitHub
parent e7f6a39db8
commit 5369f7b7b8
6 changed files with 244 additions and 61 deletions
@@ -6,10 +6,13 @@
``convert_to_fp8_moe_kernel_format``.
"""
import math
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.experts.mxfp8_emulation_moe import (
Mxfp8TritonExpertsBase,
)
@@ -17,6 +20,9 @@ from vllm.platforms import current_platform
logger = init_logger(__name__)
_AITER_SWIGLU_ALPHA = 1.702
_AITER_SWIGLU_BETA = 1.0
def is_aiter_mxfp8_moe_available() -> bool:
"""True when the FlyDSL MXFP8 MoE can run here: gfx950, the ``flydsl``
@@ -93,6 +99,27 @@ class AiterMxfp8Experts(Mxfp8TritonExpertsBase):
return False, (
"kernel requires the aiter flydsl package, which is not installed"
)
if (
is_supported
and moe_config.activation != MoEActivation.SWIGLUOAI_UNINTERLEAVE
):
return False, (
"kernel hardcodes SwiGLU-OAI activation and requires "
f"activation={MoEActivation.SWIGLUOAI_UNINTERLEAVE.value}; "
f"got activation={moe_config.activation.value}"
)
if is_supported and (
moe_config.swiglu_alpha is None
or not math.isclose(float(moe_config.swiglu_alpha), _AITER_SWIGLU_ALPHA)
or moe_config.swiglu_beta is None
or not math.isclose(float(moe_config.swiglu_beta), _AITER_SWIGLU_BETA)
):
return False, (
"kernel hardcodes SwiGLU-OAI with "
f"alpha={_AITER_SWIGLU_ALPHA} and beta={_AITER_SWIGLU_BETA}; "
f"got swiglu_alpha={moe_config.swiglu_alpha} and "
f"swiglu_beta={moe_config.swiglu_beta}"
)
return is_supported, reason
def apply(
@@ -107,17 +107,13 @@ class Mxfp8EmulationTritonExperts(Mxfp8TritonExpertsBase):
limit = self.quant_config.gemm1_clamp_limit
if limit is None:
raise ValueError("SWIGLUOAI_UNINTERLEAVE requires gemm1_clamp_limit")
alpha = self.quant_config.gemm1_alpha
alpha = 1.702 if alpha is None else float(alpha)
beta = self.quant_config.gemm1_beta
beta = 1.0 if beta is None else float(beta)
apply_moe_activation(
activation,
output,
input,
clamp_limit=float(limit),
alpha=alpha,
beta=beta,
alpha=self.gemm1_alpha,
beta=self.gemm1_beta,
)
return
super().activation(activation, output, input)
@@ -362,10 +362,7 @@ class Mxfp8NativeTritonExperts(Mxfp8TritonExpertsBase):
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
):
alpha = self.quant_config.gemm1_alpha
alpha = 1.702 if alpha is None else float(alpha)
beta = self.quant_config.gemm1_beta
beta = 1.0 if beta is None else float(beta)
# `self.gemm1_alpha` and `self.gemm1_beta`` are set by `TritonExperts.__init__`.
limit = self.quant_config.gemm1_clamp_limit
limit = None if limit is None else float(limit)
out = fused_moe_mxfp8_native(
@@ -376,8 +373,8 @@ class Mxfp8NativeTritonExperts(Mxfp8TritonExpertsBase):
self.w2_scale_val,
topk_weights,
topk_ids,
alpha=alpha,
beta=beta,
alpha=self.gemm1_alpha,
beta=self.gemm1_beta,
limit=limit,
global_num_experts=global_num_experts,
expert_map=expert_map,
@@ -12,10 +12,10 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kMxfp8Dynamic,
kMxfp8Static,
)
from vllm.platforms import current_platform
logger = init_logger(__name__)
# Ordered by priority.
_SUPPORTED_BACKENDS = (
Fp8MoeBackend.FLASHINFER_TRTLLM,
Fp8MoeBackend.DEEPGEMM,
@@ -26,6 +26,8 @@ _SUPPORTED_BACKENDS = (
# devices / no flydsl / EP it is skipped and native is used.
Fp8MoeBackend.AITER_MXFP8,
Fp8MoeBackend.HUMMING,
Fp8MoeBackend.TRITON_MXFP8,
Fp8MoeBackend.EMULATION,
)
_BACKEND_NAME_MAP: dict[str, Fp8MoeBackend] = {
@@ -61,15 +63,12 @@ def _mxfp8_backend_to_kernel_cls(
return [AiterMxfp8Experts]
if backend == Fp8MoeBackend.TRITON_MXFP8:
# Explicit ``--moe-backend triton``: the Triton mxfp8 path, i.e.
# dot_scaled on MX-capable HW (gfx950) and BF16 emulation otherwise.
# Mirrors the ROCm auto-fallback in ``_select_rocm_mxfp8_backend``.
if current_platform.supports_mx():
from vllm.model_executor.layers.fused_moe.experts.mxfp8_native_moe import (
Mxfp8NativeTritonExperts,
)
from vllm.model_executor.layers.fused_moe.experts.mxfp8_native_moe import (
Mxfp8NativeTritonExperts,
)
return [Mxfp8NativeTritonExperts]
return [Mxfp8NativeTritonExperts]
if backend == Fp8MoeBackend.EMULATION:
from vllm.model_executor.layers.fused_moe.experts.mxfp8_emulation_moe import (
Mxfp8EmulationTritonExperts,
)
@@ -105,35 +104,6 @@ def _select_kernel_cls(
)
def _select_rocm_mxfp8_backend() -> tuple[Fp8MoeBackend, type[mk.FusedMoEExperts]]:
"""ROCm fallback when no auto-selected MXFP8 backend is available.
The aiter FlyDSL backend (``AITER_MXFP8``) is auto-picked earlier by
``select_mxfp8_moe_backend`` via ``_SUPPORTED_BACKENDS`` when usable, or
explicitly via ``--moe-backend aiter``; this fallback handles the rest
(native dot_scaled on gfx950, else BF16 emulation).
"""
if current_platform.supports_mx():
from vllm.model_executor.layers.fused_moe.experts.mxfp8_native_moe import (
Mxfp8NativeTritonExperts,
)
logger.info_once("Using native CDNA4 (gfx950) MXFP8 dot_scaled MoE backend.")
return Fp8MoeBackend.TRITON_MXFP8, Mxfp8NativeTritonExperts
from vllm.model_executor.layers.fused_moe.experts.mxfp8_emulation_moe import (
Mxfp8EmulationTritonExperts,
)
logger.info_once(
"No native MXFP8 MoE backend available on this device; "
"MXFP8 weights will be dequantized to BF16 once at load time and the "
"MoE will run in BF16 (no per-step dequant)."
)
return Fp8MoeBackend.EMULATION, Mxfp8EmulationTritonExperts
def select_mxfp8_moe_backend(
config: FusedMoEConfig,
) -> tuple[Fp8MoeBackend, type[mk.FusedMoEExperts]]:
@@ -167,8 +137,5 @@ def select_mxfp8_moe_backend(
logger.info_once("Using '%s' MxFp8 MoE backend.", backend.value)
return backend, experts_cls
# simplify the logic for rocm, refactor later when more backends are supported
if current_platform.is_rocm():
return _select_rocm_mxfp8_backend()
# TODO: add debug log with reason.
raise ValueError("No MXFP8 MoE backends available.")