[ROCm][Perf] Use flydsl moe with Minimax-M3 mxfp8 weights on gfx950 and implemented moe-backend selection (#46184)
Signed-off-by: Hongxia Yang <hongxia.yang@amd.com> Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: TJian <tunjian.tan@embeddedllm.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: Tan Pin Siang <tanpinsiang@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
TJian
Andreas Karatzas
Tan Pin Siang
parent
9fd00ee006
commit
867fd5e8ed
@@ -0,0 +1,135 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MXFP8 MoE backend selection for the AITER FlyDSL kernel (gfx950).
|
||||
|
||||
GPU-free: mocks the platform (gfx950) and the ``flydsl`` package check, then
|
||||
exercises the oracle so the FlyDSL backend is auto-picked when usable (including
|
||||
under expert parallelism, since apply() forwards the expert_map as aiter's
|
||||
expert_mask) and skipped (native fallback) when the device/package is missing.
|
||||
"""
|
||||
|
||||
import dataclasses
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
pytest.skip("This test can only run on ROCm.", allow_module_level=True)
|
||||
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config # noqa: E402
|
||||
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp8_moe import ( # noqa: E402
|
||||
AiterMxfp8Experts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.modular_kernel import ( # noqa: E402
|
||||
FusedMoEActivationFormat,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import ( # noqa: E402
|
||||
Fp8MoeBackend,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp8 import ( # noqa: E402
|
||||
_BACKEND_NAME_MAP,
|
||||
_SUPPORTED_BACKENDS,
|
||||
_mxfp8_backend_to_kernel_cls,
|
||||
_select_kernel_cls,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import ( # noqa: E402
|
||||
kMxfp8Dynamic,
|
||||
kMxfp8Static,
|
||||
)
|
||||
|
||||
_AITER_MOD = "vllm.model_executor.layers.fused_moe.experts.aiter_mxfp8_moe"
|
||||
|
||||
|
||||
def _config(ep_size: int = 1):
|
||||
cfg = make_dummy_moe_config(num_experts=128, experts_per_token=4, hidden_dim=6144)
|
||||
if ep_size != 1:
|
||||
cfg = dataclasses.replace(
|
||||
cfg,
|
||||
moe_parallel_config=dataclasses.replace(
|
||||
cfg.moe_parallel_config, ep_size=ep_size, use_ep=True
|
||||
),
|
||||
)
|
||||
return cfg
|
||||
|
||||
|
||||
def _gfx950():
|
||||
"""Patch the platform so the device gate (gfx950 / MX) passes off-ROCm."""
|
||||
return patch.multiple(
|
||||
f"{_AITER_MOD}.current_platform",
|
||||
is_rocm=lambda: True,
|
||||
supports_mx=lambda: True,
|
||||
)
|
||||
|
||||
|
||||
def _flydsl_installed(present: bool):
|
||||
return patch(f"{_AITER_MOD}.is_aiter_mxfp8_moe_available", return_value=present)
|
||||
|
||||
|
||||
def test_aiter_mxfp8_registered():
|
||||
"""The FlyDSL backend is auto-selectable and reachable via --moe-backend aiter."""
|
||||
assert Fp8MoeBackend.AITER_MXFP8 in _SUPPORTED_BACKENDS
|
||||
assert _BACKEND_NAME_MAP["aiter"] is Fp8MoeBackend.AITER_MXFP8
|
||||
assert _mxfp8_backend_to_kernel_cls(Fp8MoeBackend.AITER_MXFP8) == [
|
||||
AiterMxfp8Experts
|
||||
]
|
||||
|
||||
|
||||
def test_triton_selectable():
|
||||
assert _BACKEND_NAME_MAP["triton"] is Fp8MoeBackend.TRITON_MXFP8
|
||||
# Not auto-selected (only reachable explicitly), so FlyDSL still wins auto.
|
||||
assert Fp8MoeBackend.TRITON_MXFP8 not in _SUPPORTED_BACKENDS
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ep_size", [1, 2])
|
||||
def test_ep_supported(ep_size):
|
||||
"""FlyDSL accepts both TP and EP: apply() forwards expert_map as expert_mask."""
|
||||
assert (
|
||||
AiterMxfp8Experts._supports_parallel_config(
|
||||
_config(ep_size).moe_parallel_config
|
||||
)
|
||||
is True
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"present,ep_size,supported,reason_substr",
|
||||
[
|
||||
(True, 1, True, None), # gfx950 + flydsl + TP -> selectable
|
||||
(True, 2, True, None), # gfx950 + flydsl + EP -> selectable (expert_mask)
|
||||
(False, 1, False, "flydsl package"), # package missing -> clear reason
|
||||
],
|
||||
)
|
||||
def test_is_supported_config(present, ep_size, supported, reason_substr):
|
||||
with _gfx950(), _flydsl_installed(present):
|
||||
ok, reason = AiterMxfp8Experts.is_supported_config(
|
||||
AiterMxfp8Experts,
|
||||
_config(ep_size),
|
||||
kMxfp8Static,
|
||||
kMxfp8Dynamic,
|
||||
FusedMoEActivationFormat.Standard,
|
||||
)
|
||||
assert ok is supported
|
||||
if reason_substr is not None:
|
||||
assert reason_substr in reason
|
||||
|
||||
|
||||
def test_explicit_moe_backend_aiter():
|
||||
"""--moe-backend aiter: returns FlyDSL when usable (TP or EP), else a clear
|
||||
ValueError when the flydsl package is missing."""
|
||||
with _gfx950(), _flydsl_installed(True):
|
||||
assert (
|
||||
_select_kernel_cls(Fp8MoeBackend.AITER_MXFP8, _config(1))
|
||||
is AiterMxfp8Experts
|
||||
)
|
||||
assert (
|
||||
_select_kernel_cls(Fp8MoeBackend.AITER_MXFP8, _config(2))
|
||||
is AiterMxfp8Experts
|
||||
)
|
||||
with (
|
||||
_gfx950(),
|
||||
_flydsl_installed(False),
|
||||
pytest.raises(ValueError, match="flydsl package"),
|
||||
):
|
||||
_select_kernel_cls(Fp8MoeBackend.AITER_MXFP8, _config(1))
|
||||
@@ -171,6 +171,7 @@ def _rocm_aiter_fused_moe_impl(
|
||||
bias1: torch.Tensor | None = None,
|
||||
bias2: torch.Tensor | None = None,
|
||||
moe_sorting_dispatch_policy: int = 0,
|
||||
swiglu_limit: float = 0.0,
|
||||
) -> torch.Tensor:
|
||||
from aiter import ActivationType, QuantType
|
||||
from aiter.fused_moe import fused_moe
|
||||
@@ -203,6 +204,7 @@ def _rocm_aiter_fused_moe_impl(
|
||||
bias1=bias1,
|
||||
bias2=bias2,
|
||||
moe_sorting_dispatch_policy=moe_sorting_dispatch_policy,
|
||||
swiglu_limit=swiglu_limit,
|
||||
**extra_kwargs,
|
||||
)
|
||||
|
||||
@@ -229,6 +231,7 @@ def _rocm_aiter_fused_moe_fake(
|
||||
bias1: torch.Tensor | None = None,
|
||||
bias2: torch.Tensor | None = None,
|
||||
moe_sorting_dispatch_policy: int = 0,
|
||||
swiglu_limit: float = 0.0,
|
||||
) -> torch.Tensor:
|
||||
if output_dtype is not None:
|
||||
return torch.empty_like(hidden_states, dtype=output_dtype)
|
||||
@@ -2201,6 +2204,7 @@ class rocm_aiter_ops:
|
||||
bias1: torch.Tensor | None = None,
|
||||
bias2: torch.Tensor | None = None,
|
||||
moe_sorting_dispatch_policy: int = 0,
|
||||
swiglu_limit: float = 0.0,
|
||||
) -> torch.Tensor:
|
||||
return torch.ops.vllm.rocm_aiter_fused_moe(
|
||||
hidden_states,
|
||||
@@ -2224,6 +2228,7 @@ class rocm_aiter_ops:
|
||||
bias1,
|
||||
bias2,
|
||||
moe_sorting_dispatch_policy,
|
||||
swiglu_limit,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -2678,6 +2683,31 @@ class rocm_aiter_ops:
|
||||
|
||||
return tuple(shuffle_weight(tensor, layout=layout) for tensor in tensors)
|
||||
|
||||
@staticmethod
|
||||
def shuffle_mxfp8_moe_weights(
|
||||
w13: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
w2_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Preshuffle MXFP8 MoE weights + E8M0 scales into AITER's FlyDSL layout:
|
||||
gate/up-interleaved weights, interleaved scale for w13 (gate/up), plain
|
||||
scale for w2 (the interleaved variant is gate/up-only and misaligns w2).
|
||||
"""
|
||||
from aiter.ops.shuffle import shuffle_scale, shuffle_weight
|
||||
|
||||
num_experts = w13.shape[0]
|
||||
w13 = shuffle_weight(w13, is_guinterleave=True, gate_up=True)
|
||||
w2 = shuffle_weight(w2, is_guinterleave=True, gate_up=False)
|
||||
w13_scale = shuffle_scale(
|
||||
w13_scale.reshape(-1, w13_scale.shape[-1]),
|
||||
num_experts,
|
||||
is_guinterleave=True,
|
||||
gate_up=True,
|
||||
)
|
||||
w2_scale = shuffle_scale(w2_scale.reshape(-1, w2_scale.shape[-1]))
|
||||
return w13, w2, w13_scale, w2_scale
|
||||
|
||||
@staticmethod
|
||||
def flash_attn_varlen_func(
|
||||
q: torch.Tensor,
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MXFP8 (1x32 block, E8M0) MoE via AITER's FlyDSL two-stage grouped GEMM
|
||||
(gfx950); alternative to ``Mxfp8NativeTritonExperts``. Routes through
|
||||
``aiter.fused_moe`` (per_1x32, gate_mode=INTERLEAVE); weights are preshuffled in
|
||||
``convert_to_fp8_moe_kernel_format``.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.experts.mxfp8_emulation_moe import (
|
||||
Mxfp8TritonExpertsBase,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def is_aiter_mxfp8_moe_available() -> bool:
|
||||
"""True when the FlyDSL MXFP8 MoE can run here: gfx950, the ``flydsl``
|
||||
package is importable, AND the installed aiter carries the mxfp8 FlyDSL
|
||||
2-stage support from ROCm/aiter#3811.
|
||||
|
||||
``flydsl`` and ``aiter`` are separate packages, so ``is_flydsl_available()``
|
||||
(flydsl pkg + arch) is necessary but not sufficient: an older aiter without
|
||||
#3811 still ships the flydsl pkg and the ``aiter.ops.flydsl`` module but a
|
||||
broken/missing ``per_1x32 + fp8`` 2-stage path. Without this extra gate a
|
||||
nightly lacking #3811 would wrongly select FlyDSL instead of falling back to
|
||||
the native Triton dot_scaled path. #3811 added no probe-able public symbol,
|
||||
so detect the ``minimax_m3_mxfp8`` tuned config it shipped. Every check fails
|
||||
closed (returns False -> triton dot_scaled), which is always safe."""
|
||||
if not (current_platform.is_rocm() and current_platform.supports_mx()):
|
||||
return False
|
||||
try:
|
||||
import os
|
||||
|
||||
import aiter
|
||||
from aiter.ops.flydsl.utils import is_flydsl_available
|
||||
|
||||
if not is_flydsl_available():
|
||||
return False
|
||||
return os.path.exists(
|
||||
os.path.join(
|
||||
os.path.dirname(aiter.__file__),
|
||||
"configs",
|
||||
"model_configs",
|
||||
"minimax_m3_mxfp8_tuned_fmoe.csv",
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class AiterMxfp8Experts(Mxfp8TritonExpertsBase):
|
||||
"""MXFP8 MoE through AITER's FlyDSL two-stage grouped GEMM (gfx950)."""
|
||||
|
||||
@property
|
||||
def quant_dtype(self) -> torch.dtype | str | None:
|
||||
return self.quant_config.quant_dtype
|
||||
|
||||
@property
|
||||
def block_shape(self) -> list[int] | None:
|
||||
return self.quant_config.block_shape
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
# aiter.fused_moe MXFP8-quantizes the activations internally.
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
# Device capability only (gfx950 / MX-capable ROCm). The flydsl package
|
||||
# check lives in is_supported_config so a missing package is reported
|
||||
# distinctly from an unsupported device.
|
||||
return current_platform.is_rocm() and current_platform.supports_mx()
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(moe_parallel_config) -> bool:
|
||||
# Both TP (expert_map=None) and EP are supported: apply() forwards the
|
||||
# expert_map as aiter's ``expert_mask`` (the per-rank local-expert
|
||||
# selection), mirroring the native rocm_aiter_moe path.
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def is_supported_config(
|
||||
cls, moe_config, weight_key, activation_key, activation_format
|
||||
):
|
||||
is_supported, reason = super().is_supported_config(
|
||||
cls, moe_config, weight_key, activation_key, activation_format
|
||||
)
|
||||
# _supports_current_device() only gates on the device; surface a clear
|
||||
# reason when the device is fine but the flydsl package is missing.
|
||||
if is_supported and not is_aiter_mxfp8_moe_available():
|
||||
return False, (
|
||||
"kernel requires the aiter flydsl package, which is not installed"
|
||||
)
|
||||
return is_supported, reason
|
||||
|
||||
def apply(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
activation,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
a2_scale: torch.Tensor | None,
|
||||
workspace13: torch.Tensor,
|
||||
workspace2: torch.Tensor,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
):
|
||||
from aiter import ActivationType, QuantType
|
||||
from aiter.ops.flydsl.moe_common import GateMode
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
# Re-tag the preshuffled weights: replace_parameter drops the
|
||||
# is_shuffled flag, without which aiter picks a broken CK kernel.
|
||||
w1.is_shuffled = True
|
||||
w2.is_shuffled = True
|
||||
|
||||
limit = self.quant_config.gemm1_clamp_limit
|
||||
swiglu_limit = 0.0 if limit is None else float(limit)
|
||||
|
||||
# Under EP, aiter expects ``expert_mask`` as a 0/1 *local-expert* mask
|
||||
# over global ids with a trailing fake-expert sentinel slot
|
||||
# (shape ``[global_num_experts + 1]``), NOT vLLM's expert_map (a
|
||||
# global->local index map with -1 for non-local). Convert it; aiter
|
||||
# derives the global->local compaction from the mask itself. ``None``
|
||||
# under pure TP.
|
||||
if expert_map is not None:
|
||||
local_mask = (expert_map >= 0).to(torch.int32)
|
||||
expert_mask = torch.cat([local_mask, local_mask.new_zeros(1)])
|
||||
else:
|
||||
expert_mask = None
|
||||
|
||||
# Route through the graph-safe ``rocm_aiter_fused_moe`` custom op so the
|
||||
# call is captured under HIP graphs / torch.compile (a direct
|
||||
# ``aiter.fused_moe`` is opaque to the dispatcher). aiter requires FP32
|
||||
# routing weights / INT32 ids.
|
||||
out = rocm_aiter_ops.fused_moe(
|
||||
hidden_states,
|
||||
w1,
|
||||
w2,
|
||||
topk_weights.to(torch.float32),
|
||||
topk_ids.to(torch.int32),
|
||||
expert_mask=expert_mask,
|
||||
activation_method=ActivationType.Swiglu.value,
|
||||
quant_method=QuantType.per_1x32.value,
|
||||
doweight_stage1=apply_router_weight_on_input,
|
||||
w1_scale=self.w1_scale_val,
|
||||
w2_scale=self.w2_scale_val,
|
||||
a1_scale=None,
|
||||
a2_scale=None,
|
||||
gate_mode=GateMode.INTERLEAVE.value,
|
||||
swiglu_limit=swiglu_limit,
|
||||
output_dtype=output.dtype,
|
||||
)
|
||||
output.copy_(out.to(output.dtype))
|
||||
@@ -59,7 +59,9 @@ class Fp8MoeBackend(Enum):
|
||||
# MXFP8 MoE via a Triton ``dot_scaled`` kernel that lowers to CDNA4
|
||||
# (gfx950) native MX matrix-core ops. Weights stay in MXFP8 (no load-time
|
||||
# format conversion); the FP8 values + E8M0 scales are consumed directly.
|
||||
NATIVE_MXFP8 = "NATIVE_MXFP8"
|
||||
TRITON_MXFP8 = "TRITON_MXFP8"
|
||||
# MXFP8 MoE via AITER (FlyDSL two-stage grouped GEMM) on gfx950.
|
||||
AITER_MXFP8 = "AITER_MXFP8"
|
||||
|
||||
|
||||
def _get_priority_backends(
|
||||
@@ -423,6 +425,10 @@ def convert_to_fp8_moe_kernel_format(
|
||||
)
|
||||
elif fp8_backend == Fp8MoeBackend.AITER:
|
||||
w13, w2 = rocm_aiter_ops.shuffle_weights(w13, w2)
|
||||
elif fp8_backend == Fp8MoeBackend.AITER_MXFP8:
|
||||
w13, w2, w13_scale, w2_scale = rocm_aiter_ops.shuffle_mxfp8_moe_weights(
|
||||
w13, w2, w13_scale, w2_scale
|
||||
)
|
||||
elif fp8_backend == Fp8MoeBackend.MARLIN:
|
||||
weight_block_size = getattr(layer, "weight_block_size", None)
|
||||
if weight_block_size == [1, 32]:
|
||||
@@ -484,7 +490,7 @@ def convert_to_fp8_moe_kernel_format(
|
||||
# EMULATION dequantizes weights at runtime; NATIVE_MXFP8 consumes
|
||||
# the MXFP8 weights as-is — neither needs a load-time layout change.
|
||||
Fp8MoeBackend.EMULATION,
|
||||
Fp8MoeBackend.NATIVE_MXFP8,
|
||||
Fp8MoeBackend.TRITON_MXFP8,
|
||||
]:
|
||||
raise ValueError(f"Unsupported FP8 MoE backend: {fp8_backend.value}")
|
||||
|
||||
|
||||
@@ -21,6 +21,10 @@ _SUPPORTED_BACKENDS = (
|
||||
Fp8MoeBackend.DEEPGEMM,
|
||||
Fp8MoeBackend.MARLIN,
|
||||
Fp8MoeBackend.XPU,
|
||||
# AITER FlyDSL (gfx950): auto-picked by select_mxfp8_moe_backend when
|
||||
# is_supported_config passes (gfx950 + flydsl installed + not EP). On other
|
||||
# devices / no flydsl / EP it is skipped and native is used.
|
||||
Fp8MoeBackend.AITER_MXFP8,
|
||||
)
|
||||
|
||||
_BACKEND_NAME_MAP: dict[str, Fp8MoeBackend] = {
|
||||
@@ -28,6 +32,8 @@ _BACKEND_NAME_MAP: dict[str, Fp8MoeBackend] = {
|
||||
"deep_gemm": Fp8MoeBackend.DEEPGEMM,
|
||||
"marlin": Fp8MoeBackend.MARLIN,
|
||||
"xpu": Fp8MoeBackend.XPU,
|
||||
"aiter": Fp8MoeBackend.AITER_MXFP8,
|
||||
"triton": Fp8MoeBackend.TRITON_MXFP8,
|
||||
}
|
||||
|
||||
|
||||
@@ -46,6 +52,27 @@ def _mxfp8_backend_to_kernel_cls(
|
||||
)
|
||||
|
||||
return [DeepGemmExperts]
|
||||
if backend == Fp8MoeBackend.AITER_MXFP8:
|
||||
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp8_moe import (
|
||||
AiterMxfp8Experts,
|
||||
)
|
||||
|
||||
return [AiterMxfp8Experts]
|
||||
if backend == Fp8MoeBackend.TRITON_MXFP8:
|
||||
# Explicit ``--moe-backend triton``: the Triton mxfp8 path, i.e.
|
||||
# dot_scaled on MX-capable HW (gfx950) and BF16 emulation otherwise.
|
||||
# Mirrors the ROCm auto-fallback in ``_select_rocm_mxfp8_backend``.
|
||||
if current_platform.supports_mx():
|
||||
from vllm.model_executor.layers.fused_moe.experts.mxfp8_native_moe import (
|
||||
Mxfp8NativeTritonExperts,
|
||||
)
|
||||
|
||||
return [Mxfp8NativeTritonExperts]
|
||||
from vllm.model_executor.layers.fused_moe.experts.mxfp8_emulation_moe import (
|
||||
Mxfp8EmulationTritonExperts,
|
||||
)
|
||||
|
||||
return [Mxfp8EmulationTritonExperts]
|
||||
return backend_to_kernel_cls(backend)
|
||||
|
||||
|
||||
@@ -77,7 +104,13 @@ def _select_kernel_cls(
|
||||
|
||||
|
||||
def _select_rocm_mxfp8_backend() -> tuple[Fp8MoeBackend, type[mk.FusedMoEExperts]]:
|
||||
"""ROCm fallback when vendor MXFP8 backends are unavailable."""
|
||||
"""ROCm fallback when no auto-selected MXFP8 backend is available.
|
||||
|
||||
The aiter FlyDSL backend (``AITER_MXFP8``) is auto-picked earlier by
|
||||
``select_mxfp8_moe_backend`` via ``_SUPPORTED_BACKENDS`` when usable, or
|
||||
explicitly via ``--moe-backend aiter``; this fallback handles the rest
|
||||
(native dot_scaled on gfx950, else BF16 emulation).
|
||||
"""
|
||||
|
||||
if current_platform.supports_mx():
|
||||
from vllm.model_executor.layers.fused_moe.experts.mxfp8_native_moe import (
|
||||
@@ -85,7 +118,7 @@ def _select_rocm_mxfp8_backend() -> tuple[Fp8MoeBackend, type[mk.FusedMoEExperts
|
||||
)
|
||||
|
||||
logger.info_once("Using native CDNA4 (gfx950) MXFP8 dot_scaled MoE backend.")
|
||||
return Fp8MoeBackend.NATIVE_MXFP8, Mxfp8NativeTritonExperts
|
||||
return Fp8MoeBackend.TRITON_MXFP8, Mxfp8NativeTritonExperts
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.experts.mxfp8_emulation_moe import (
|
||||
Mxfp8EmulationTritonExperts,
|
||||
|
||||
Reference in New Issue
Block a user