[ROCm][Perf] Use flydsl moe with Minimax-M3 mxfp8 weights on gfx950 and implemented moe-backend selection (#46184)
Signed-off-by: Hongxia Yang <hongxia.yang@amd.com> Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: TJian <tunjian.tan@embeddedllm.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: Tan Pin Siang <tanpinsiang@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
TJian
Andreas Karatzas
Tan Pin Siang
parent
9fd00ee006
commit
867fd5e8ed
@@ -0,0 +1,166 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MXFP8 (1x32 block, E8M0) MoE via AITER's FlyDSL two-stage grouped GEMM
|
||||
(gfx950); alternative to ``Mxfp8NativeTritonExperts``. Routes through
|
||||
``aiter.fused_moe`` (per_1x32, gate_mode=INTERLEAVE); weights are preshuffled in
|
||||
``convert_to_fp8_moe_kernel_format``.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.experts.mxfp8_emulation_moe import (
|
||||
Mxfp8TritonExpertsBase,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def is_aiter_mxfp8_moe_available() -> bool:
|
||||
"""True when the FlyDSL MXFP8 MoE can run here: gfx950, the ``flydsl``
|
||||
package is importable, AND the installed aiter carries the mxfp8 FlyDSL
|
||||
2-stage support from ROCm/aiter#3811.
|
||||
|
||||
``flydsl`` and ``aiter`` are separate packages, so ``is_flydsl_available()``
|
||||
(flydsl pkg + arch) is necessary but not sufficient: an older aiter without
|
||||
#3811 still ships the flydsl pkg and the ``aiter.ops.flydsl`` module but a
|
||||
broken/missing ``per_1x32 + fp8`` 2-stage path. Without this extra gate a
|
||||
nightly lacking #3811 would wrongly select FlyDSL instead of falling back to
|
||||
the native Triton dot_scaled path. #3811 added no probe-able public symbol,
|
||||
so detect the ``minimax_m3_mxfp8`` tuned config it shipped. Every check fails
|
||||
closed (returns False -> triton dot_scaled), which is always safe."""
|
||||
if not (current_platform.is_rocm() and current_platform.supports_mx()):
|
||||
return False
|
||||
try:
|
||||
import os
|
||||
|
||||
import aiter
|
||||
from aiter.ops.flydsl.utils import is_flydsl_available
|
||||
|
||||
if not is_flydsl_available():
|
||||
return False
|
||||
return os.path.exists(
|
||||
os.path.join(
|
||||
os.path.dirname(aiter.__file__),
|
||||
"configs",
|
||||
"model_configs",
|
||||
"minimax_m3_mxfp8_tuned_fmoe.csv",
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class AiterMxfp8Experts(Mxfp8TritonExpertsBase):
|
||||
"""MXFP8 MoE through AITER's FlyDSL two-stage grouped GEMM (gfx950)."""
|
||||
|
||||
@property
|
||||
def quant_dtype(self) -> torch.dtype | str | None:
|
||||
return self.quant_config.quant_dtype
|
||||
|
||||
@property
|
||||
def block_shape(self) -> list[int] | None:
|
||||
return self.quant_config.block_shape
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
# aiter.fused_moe MXFP8-quantizes the activations internally.
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
# Device capability only (gfx950 / MX-capable ROCm). The flydsl package
|
||||
# check lives in is_supported_config so a missing package is reported
|
||||
# distinctly from an unsupported device.
|
||||
return current_platform.is_rocm() and current_platform.supports_mx()
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(moe_parallel_config) -> bool:
|
||||
# Both TP (expert_map=None) and EP are supported: apply() forwards the
|
||||
# expert_map as aiter's ``expert_mask`` (the per-rank local-expert
|
||||
# selection), mirroring the native rocm_aiter_moe path.
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def is_supported_config(
|
||||
cls, moe_config, weight_key, activation_key, activation_format
|
||||
):
|
||||
is_supported, reason = super().is_supported_config(
|
||||
cls, moe_config, weight_key, activation_key, activation_format
|
||||
)
|
||||
# _supports_current_device() only gates on the device; surface a clear
|
||||
# reason when the device is fine but the flydsl package is missing.
|
||||
if is_supported and not is_aiter_mxfp8_moe_available():
|
||||
return False, (
|
||||
"kernel requires the aiter flydsl package, which is not installed"
|
||||
)
|
||||
return is_supported, reason
|
||||
|
||||
def apply(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
activation,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
a2_scale: torch.Tensor | None,
|
||||
workspace13: torch.Tensor,
|
||||
workspace2: torch.Tensor,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
):
|
||||
from aiter import ActivationType, QuantType
|
||||
from aiter.ops.flydsl.moe_common import GateMode
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
# Re-tag the preshuffled weights: replace_parameter drops the
|
||||
# is_shuffled flag, without which aiter picks a broken CK kernel.
|
||||
w1.is_shuffled = True
|
||||
w2.is_shuffled = True
|
||||
|
||||
limit = self.quant_config.gemm1_clamp_limit
|
||||
swiglu_limit = 0.0 if limit is None else float(limit)
|
||||
|
||||
# Under EP, aiter expects ``expert_mask`` as a 0/1 *local-expert* mask
|
||||
# over global ids with a trailing fake-expert sentinel slot
|
||||
# (shape ``[global_num_experts + 1]``), NOT vLLM's expert_map (a
|
||||
# global->local index map with -1 for non-local). Convert it; aiter
|
||||
# derives the global->local compaction from the mask itself. ``None``
|
||||
# under pure TP.
|
||||
if expert_map is not None:
|
||||
local_mask = (expert_map >= 0).to(torch.int32)
|
||||
expert_mask = torch.cat([local_mask, local_mask.new_zeros(1)])
|
||||
else:
|
||||
expert_mask = None
|
||||
|
||||
# Route through the graph-safe ``rocm_aiter_fused_moe`` custom op so the
|
||||
# call is captured under HIP graphs / torch.compile (a direct
|
||||
# ``aiter.fused_moe`` is opaque to the dispatcher). aiter requires FP32
|
||||
# routing weights / INT32 ids.
|
||||
out = rocm_aiter_ops.fused_moe(
|
||||
hidden_states,
|
||||
w1,
|
||||
w2,
|
||||
topk_weights.to(torch.float32),
|
||||
topk_ids.to(torch.int32),
|
||||
expert_mask=expert_mask,
|
||||
activation_method=ActivationType.Swiglu.value,
|
||||
quant_method=QuantType.per_1x32.value,
|
||||
doweight_stage1=apply_router_weight_on_input,
|
||||
w1_scale=self.w1_scale_val,
|
||||
w2_scale=self.w2_scale_val,
|
||||
a1_scale=None,
|
||||
a2_scale=None,
|
||||
gate_mode=GateMode.INTERLEAVE.value,
|
||||
swiglu_limit=swiglu_limit,
|
||||
output_dtype=output.dtype,
|
||||
)
|
||||
output.copy_(out.to(output.dtype))
|
||||
@@ -59,7 +59,9 @@ class Fp8MoeBackend(Enum):
|
||||
# MXFP8 MoE via a Triton ``dot_scaled`` kernel that lowers to CDNA4
|
||||
# (gfx950) native MX matrix-core ops. Weights stay in MXFP8 (no load-time
|
||||
# format conversion); the FP8 values + E8M0 scales are consumed directly.
|
||||
NATIVE_MXFP8 = "NATIVE_MXFP8"
|
||||
TRITON_MXFP8 = "TRITON_MXFP8"
|
||||
# MXFP8 MoE via AITER (FlyDSL two-stage grouped GEMM) on gfx950.
|
||||
AITER_MXFP8 = "AITER_MXFP8"
|
||||
|
||||
|
||||
def _get_priority_backends(
|
||||
@@ -423,6 +425,10 @@ def convert_to_fp8_moe_kernel_format(
|
||||
)
|
||||
elif fp8_backend == Fp8MoeBackend.AITER:
|
||||
w13, w2 = rocm_aiter_ops.shuffle_weights(w13, w2)
|
||||
elif fp8_backend == Fp8MoeBackend.AITER_MXFP8:
|
||||
w13, w2, w13_scale, w2_scale = rocm_aiter_ops.shuffle_mxfp8_moe_weights(
|
||||
w13, w2, w13_scale, w2_scale
|
||||
)
|
||||
elif fp8_backend == Fp8MoeBackend.MARLIN:
|
||||
weight_block_size = getattr(layer, "weight_block_size", None)
|
||||
if weight_block_size == [1, 32]:
|
||||
@@ -484,7 +490,7 @@ def convert_to_fp8_moe_kernel_format(
|
||||
# EMULATION dequantizes weights at runtime; NATIVE_MXFP8 consumes
|
||||
# the MXFP8 weights as-is — neither needs a load-time layout change.
|
||||
Fp8MoeBackend.EMULATION,
|
||||
Fp8MoeBackend.NATIVE_MXFP8,
|
||||
Fp8MoeBackend.TRITON_MXFP8,
|
||||
]:
|
||||
raise ValueError(f"Unsupported FP8 MoE backend: {fp8_backend.value}")
|
||||
|
||||
|
||||
@@ -21,6 +21,10 @@ _SUPPORTED_BACKENDS = (
|
||||
Fp8MoeBackend.DEEPGEMM,
|
||||
Fp8MoeBackend.MARLIN,
|
||||
Fp8MoeBackend.XPU,
|
||||
# AITER FlyDSL (gfx950): auto-picked by select_mxfp8_moe_backend when
|
||||
# is_supported_config passes (gfx950 + flydsl installed + not EP). On other
|
||||
# devices / no flydsl / EP it is skipped and native is used.
|
||||
Fp8MoeBackend.AITER_MXFP8,
|
||||
)
|
||||
|
||||
_BACKEND_NAME_MAP: dict[str, Fp8MoeBackend] = {
|
||||
@@ -28,6 +32,8 @@ _BACKEND_NAME_MAP: dict[str, Fp8MoeBackend] = {
|
||||
"deep_gemm": Fp8MoeBackend.DEEPGEMM,
|
||||
"marlin": Fp8MoeBackend.MARLIN,
|
||||
"xpu": Fp8MoeBackend.XPU,
|
||||
"aiter": Fp8MoeBackend.AITER_MXFP8,
|
||||
"triton": Fp8MoeBackend.TRITON_MXFP8,
|
||||
}
|
||||
|
||||
|
||||
@@ -46,6 +52,27 @@ def _mxfp8_backend_to_kernel_cls(
|
||||
)
|
||||
|
||||
return [DeepGemmExperts]
|
||||
if backend == Fp8MoeBackend.AITER_MXFP8:
|
||||
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp8_moe import (
|
||||
AiterMxfp8Experts,
|
||||
)
|
||||
|
||||
return [AiterMxfp8Experts]
|
||||
if backend == Fp8MoeBackend.TRITON_MXFP8:
|
||||
# Explicit ``--moe-backend triton``: the Triton mxfp8 path, i.e.
|
||||
# dot_scaled on MX-capable HW (gfx950) and BF16 emulation otherwise.
|
||||
# Mirrors the ROCm auto-fallback in ``_select_rocm_mxfp8_backend``.
|
||||
if current_platform.supports_mx():
|
||||
from vllm.model_executor.layers.fused_moe.experts.mxfp8_native_moe import (
|
||||
Mxfp8NativeTritonExperts,
|
||||
)
|
||||
|
||||
return [Mxfp8NativeTritonExperts]
|
||||
from vllm.model_executor.layers.fused_moe.experts.mxfp8_emulation_moe import (
|
||||
Mxfp8EmulationTritonExperts,
|
||||
)
|
||||
|
||||
return [Mxfp8EmulationTritonExperts]
|
||||
return backend_to_kernel_cls(backend)
|
||||
|
||||
|
||||
@@ -77,7 +104,13 @@ def _select_kernel_cls(
|
||||
|
||||
|
||||
def _select_rocm_mxfp8_backend() -> tuple[Fp8MoeBackend, type[mk.FusedMoEExperts]]:
|
||||
"""ROCm fallback when vendor MXFP8 backends are unavailable."""
|
||||
"""ROCm fallback when no auto-selected MXFP8 backend is available.
|
||||
|
||||
The aiter FlyDSL backend (``AITER_MXFP8``) is auto-picked earlier by
|
||||
``select_mxfp8_moe_backend`` via ``_SUPPORTED_BACKENDS`` when usable, or
|
||||
explicitly via ``--moe-backend aiter``; this fallback handles the rest
|
||||
(native dot_scaled on gfx950, else BF16 emulation).
|
||||
"""
|
||||
|
||||
if current_platform.supports_mx():
|
||||
from vllm.model_executor.layers.fused_moe.experts.mxfp8_native_moe import (
|
||||
@@ -85,7 +118,7 @@ def _select_rocm_mxfp8_backend() -> tuple[Fp8MoeBackend, type[mk.FusedMoEExperts
|
||||
)
|
||||
|
||||
logger.info_once("Using native CDNA4 (gfx950) MXFP8 dot_scaled MoE backend.")
|
||||
return Fp8MoeBackend.NATIVE_MXFP8, Mxfp8NativeTritonExperts
|
||||
return Fp8MoeBackend.TRITON_MXFP8, Mxfp8NativeTritonExperts
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.experts.mxfp8_emulation_moe import (
|
||||
Mxfp8EmulationTritonExperts,
|
||||
|
||||
Reference in New Issue
Block a user