[MoE] MXFP4 oracle: add CUTLASS W4A4 backend and BF16 activation fallback (#60079)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Misha Goin
2026-10-08 11:39:09 -07:00
committed by GitHub
co-authored by Claude
parent d8fa463e2b
commit 30c7405890
3 changed files with 214 additions and 14 deletions
+84
View File
@@ -37,9 +37,12 @@ from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
run_cutlass_moe_fp4,
run_cutlass_moe_fp8,
run_cutlass_moe_mxfp4,
swizzle_mxfp4_scales,
)
from vllm.model_executor.layers.fused_moe.oracle import mxfp4 as mxfp4_oracle
from vllm.model_executor.layers.fused_moe.oracle import nvfp4 as nvfp4_oracle
from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Dynamic
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
@@ -294,6 +297,87 @@ def test_cutlass_fp4_moe_padded_routes_do_not_change_valid_output(quantization:
torch.testing.assert_close(actual[1:], torch.zeros_like(actual[1:]))
@torch.inference_mode()
def test_cutlass_mxfp4_oracle_matches_direct_kernel(monkeypatch, workspace_init):
"""The MXFP4 oracle selects CUTLASS for W4A4 and swizzles checkpoint
scales into the layout the kernel expects."""
if not CutlassExpertsMxfp4._supports_current_device():
pytest.skip("CUTLASS mxfp4 MoE is not supported on this GPU")
monkeypatch.setattr(mxfp4_oracle, "_user_moe_activation_override", lambda: None)
set_random_seed(7)
e, m, n, k, topk = 8, 33, 256, 512, 2
device = torch.device("cuda")
w1 = torch.randint(0, 256, (e, 2 * n, k // 2), device=device, dtype=torch.uint8)
w2 = torch.randint(0, 256, (e, k, n // 2), device=device, dtype=torch.uint8)
w1_scale = torch.randint(118, 124, (e, 2 * n, k // 32), device=device).byte()
w2_scale = torch.randint(118, 124, (e, k, n // 32), device=device).byte()
a = torch.randn((m, k), device=device, dtype=torch.bfloat16)
score = torch.randn((m, e), device=device, dtype=torch.bfloat16)
topk_weights, topk_ids, _ = fused_topk(a, score, topk, renormalize=False)
def swizzle(scale: torch.Tensor) -> torch.Tensor:
rows, cols = scale.shape[1:]
return torch.stack(
[swizzle_mxfp4_scales(s, rows, cols * 32).view(rows, cols) for s in scale]
)
expected = torch.empty_like(a)
run_cutlass_moe_mxfp4(
output=expected,
a=a,
w1_fp4=w1,
w1_blockscale=swizzle(w1_scale),
w2_fp4=w2,
w2_blockscale=swizzle(w2_scale),
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=MoEActivation.SILU,
workspace13=torch.empty((m * topk, max(2 * n, k)), device=device).to(a),
workspace2=torch.empty((m * topk, n), device=device).to(a),
m=m,
n=n,
k=k,
e=e,
device=device,
apply_router_weight_on_input=False,
)
moe_config = make_dummy_moe_config(
num_experts=e, experts_per_token=topk, hidden_dim=k, intermediate_size=n
)
with set_current_vllm_config(vllm_config):
backend, experts_cls = mxfp4_oracle.select_mxfp4_moe_backend(
moe_config, activation_key=kMxfp4Dynamic
)
assert backend == mxfp4_oracle.Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4
w13, w2, w13_scale, w2_scale, _, _ = (
mxfp4_oracle.convert_weight_to_mxfp4_moe_kernel_format(
backend, torch.nn.Module(), w1, w2, w1_scale, w2_scale
)
)
kernel = mxfp4_oracle.make_mxfp4_moe_kernel(
mxfp4_oracle.make_mxfp4_moe_quant_config(backend, w13_scale, w2_scale),
moe_config,
experts_cls,
backend,
)
actual = kernel.apply(
a,
w13,
w2,
topk_weights,
topk_ids,
activation=MoEActivation.SILU,
global_num_experts=e,
expert_map=None,
apply_router_weight_on_input=False,
)
torch.testing.assert_close(actual, expected, atol=0, rtol=0)
@dataclasses.dataclass
class MOETensors:
a: torch.Tensor
+51 -1
View File
@@ -7,7 +7,11 @@ from dataclasses import dataclass, replace
import pytest
import torch
from tests.kernels.moe.utils import check_accuracy, check_deferred_moe_finalize
from tests.kernels.moe.utils import (
check_accuracy,
check_deferred_moe_finalize,
make_dummy_moe_config,
)
from vllm._aiter_ops import (
is_aiter_found,
is_aiter_found_and_supported,
@@ -2009,6 +2013,52 @@ def test_select_mxfp4_moe_backend_raises_with_unsupported_reasons(
mxfp4_oracle.select_mxfp4_moe_backend(moe_config)
@pytest.mark.parametrize(
"moe_backend,w4a4,supported,expected",
[
# The checkpoint's activation wins over BF16 backends listed first.
("auto", True, {"CUTLASS_MXFP4_MXFP4", "MARLIN"}, "CUTLASS_MXFP4_MXFP4"),
# No W4A4 kernel: run at BF16 rather than another quantized activation.
("auto", True, {"FLASHINFER_TRTLLM_MXFP4_MXFP8", "MARLIN"}, "MARLIN"),
("marlin", True, {"MARLIN"}, "MARLIN"),
# A BF16 checkpoint never gets its activations quantized.
("auto", False, {"CUTLASS_MXFP4_MXFP4", "MARLIN"}, "MARLIN"),
],
)
def test_select_mxfp4_moe_backend_respects_checkpoint_activation(
monkeypatch: pytest.MonkeyPatch,
moe_backend: str,
w4a4: bool,
supported: set[str],
expected: str,
):
"""Activations run at the checkpoint's precision when a backend supports
it and fall back to BF16 otherwise, never to a lower precision."""
import vllm.model_executor.layers.fused_moe.oracle.mxfp4 as mxfp4_oracle
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kMxfp4Dynamic,
)
def backend_to_kernel_cls(backend):
class Experts:
@staticmethod
def is_supported_config(*args):
return backend.name in supported, None
return [Experts]
monkeypatch.setattr(mxfp4_oracle, "backend_to_kernel_cls", backend_to_kernel_cls)
monkeypatch.setattr(mxfp4_oracle, "_user_moe_activation_override", lambda: None)
moe_config = make_dummy_moe_config()
moe_config.moe_backend = moe_backend
backend, _ = mxfp4_oracle.select_mxfp4_moe_backend(
moe_config, activation_key=kMxfp4Dynamic if w4a4 else None
)
assert backend.name == expected
@pytest.mark.parametrize("backend_name", ["AITER_TRITON_MXFP4_BF16", "AITER_MXFP4_FP8"])
@pytest.mark.parametrize("use_ep", [False, True])
def test_aiter_mxfp4_monolithic_rejects_expert_parallel(
@@ -114,6 +114,18 @@ def _pack_deepgemm_mxfp4_scales(
)
def _swizzle_cutlass_mxfp4_scales(scale: torch.Tensor) -> torch.Tensor:
"""Swizzle [E, N, K // 32] block scales into the CUTLASS tiled layout."""
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
swizzle_mxfp4_scales,
)
n, scale_k = scale.shape[1:]
return torch.stack(
[swizzle_mxfp4_scales(s, n, scale_k * 32).reshape(n, scale_k) for s in scale]
)
class Mxfp4MoeBackend(Enum):
NONE = "None"
# b12x backends
@@ -127,6 +139,8 @@ class Mxfp4MoeBackend(Enum):
# FlashInfer CUTLASS backends
FLASHINFER_CUTLASS_MXFP4_MXFP8 = "FLASHINFER_CUTLASS_MXFP4_MXFP8"
FLASHINFER_CUTLASS_MXFP4_BF16 = "FLASHINFER_CUTLASS_MXFP4_BF16"
# vLLM CUTLASS backend
CUTLASS_MXFP4_MXFP4 = "CUTLASS_MXFP4_MXFP4"
# Marlin
BATCHED_MARLIN = "BATCHED_MARLIN"
MARLIN = "MARLIN"
@@ -218,6 +232,13 @@ def backend_to_kernel_cls(
return [FlashInferExperts]
elif backend == Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4:
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
CutlassExpertsMxfp4,
)
return [CutlassExpertsMxfp4]
elif backend == Mxfp4MoeBackend.TRITON:
from vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe import ( # noqa: E501
OAITritonExperts,
@@ -378,6 +399,7 @@ def _get_priority_backends_for_gpt_oss() -> list[Mxfp4MoeBackend]:
Mxfp4MoeBackend.TRITON,
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4,
# TRITON_UNFUSED has bug with MTP support
# TODO re-enable after kernel is fixed
# TRITON_UNFUSED
@@ -440,7 +462,10 @@ def _backend_activation_key(backend: Mxfp4MoeBackend) -> QuantKey | None:
return kMxfp8Dynamic
if backend == Mxfp4MoeBackend.AITER_MXFP4_FP8:
return kFp8StaticTensorSym
if backend == Mxfp4MoeBackend.AITER_MXFP4_MXFP4:
if backend in (
Mxfp4MoeBackend.AITER_MXFP4_MXFP4,
Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4,
):
return kMxfp4Dynamic
return None # BF16 activation
@@ -474,6 +499,22 @@ def _make_log_backend(backend: Mxfp4MoeBackend) -> str:
return f"Using '{backend.value}' Mxfp4 MoE backend."
def _warn_on_bf16_fallback(
backend: Mxfp4MoeBackend, requested_activation_key: QuantKey | None
) -> None:
if (
requested_activation_key is not None
and backend != Mxfp4MoeBackend.EMULATION
and _backend_activation_key(backend) is None
):
logger.warning_once(
"No Mxfp4 MoE backend supports activation=%s for the deployment "
"configuration. Falling back to '%s' with BF16 activations.",
requested_activation_key,
backend.value,
)
def _make_log_unsupported(backend: Mxfp4MoeBackend, reason: str | None) -> str:
base = (
f"Mxfp4 MoE backend '{backend.value}' does not support the "
@@ -505,17 +546,19 @@ def _filter_by_activation(
backends: list[Mxfp4MoeBackend],
requested_activation_key: QuantKey | None,
) -> list[Mxfp4MoeBackend]:
"""Pick variants matching ``requested_activation_key``; without one,
prefer BF16 if the list has any, else keep the list as-is so explicit
non-BF16 picks (e.g. the ``_afp8`` aliases) still land."""
"""Pick variants matching ``requested_activation_key``, followed by BF16
variants as a higher-precision fallback; without one, prefer BF16 if the
list has any, else keep the list as-is so explicit non-BF16 picks (e.g.
the ``_afp8`` aliases) still land."""
bf16 = [b for b in backends if _backend_activation_key(b) is None]
if requested_activation_key is not None:
return [
matching = [
b
for b in backends
if _backend_activation_key(b) == requested_activation_key
or b == Mxfp4MoeBackend.EMULATION
]
bf16 = [b for b in backends if _backend_activation_key(b) is None]
return matching + [b for b in bf16 if b not in matching]
return bf16 if bf16 else backends
@@ -576,8 +619,9 @@ def select_mxfp4_moe_backend(
Args:
config: MoE configuration
activation_key: Optional activation quantization key. If provided,
overrides the default activation key for backend selection.
activation_key: Optional activation quantization key declared by
the checkpoint. Backends that match it are preferred, with BF16
activation backends as a fallback.
Use kFp8StaticTensorSym for W4A8 scheme.
Note: Shape-specific fallbacks may still occur at runtime.
@@ -614,7 +658,7 @@ def select_mxfp4_moe_backend(
else _backend_activation_key(requested_backend)
)
try:
return _return_or_raise(
selected = _return_or_raise(
requested_backend,
config,
kMxfp4Static,
@@ -623,6 +667,9 @@ def select_mxfp4_moe_backend(
)
except ValueError as e:
last_error = e
else:
_warn_on_bf16_fallback(requested_backend, requested_activation_key)
return selected
assert last_error is not None
raise last_error
@@ -648,10 +695,10 @@ def select_mxfp4_moe_backend(
unsupported_reasons = []
for backend in AVAILABLE_BACKENDS:
# Use requested_activation_key if provided, otherwise use backend default
# EMULATION takes the requested key, other backends use their own
act_key = (
requested_activation_key
if requested_activation_key is not None
if backend == Mxfp4MoeBackend.EMULATION
else _backend_activation_key(backend)
)
for k_cls in backend_to_kernel_cls(backend):
@@ -660,6 +707,7 @@ def select_mxfp4_moe_backend(
)
if supported:
logger.info_once(_make_log_backend(backend))
_warn_on_bf16_fallback(backend, requested_activation_key)
return backend, k_cls
else:
logger.debug_once(_make_log_unsupported(backend, reason))
@@ -789,6 +837,10 @@ def mxfp4_round_up_hidden_size_and_intermediate_size(
# DeepGEMM requires M/N/K alignment
intermediate_size = round_up(intermediate_size, 128)
hidden_size = round_up(hidden_size, 128)
elif backend == Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4:
# CUTLASS scale tiles cover 128 rows by 128 elements
intermediate_size = round_up(intermediate_size, 128)
hidden_size = round_up(hidden_size, 128)
elif backend in (Mxfp4MoeBackend.MARLIN, Mxfp4MoeBackend.BATCHED_MARLIN):
intermediate_size = round_up(intermediate_size, 128)
if current_platform.is_xpu():
@@ -1433,7 +1485,8 @@ def convert_weight_to_mxfp4_moe_kernel_format(
]:
"""Convert loaded weights into backend-specific kernel format.
Supports DeepGEMM, FlashInfer, TRTLLM MXFP8, Triton and Marlin backends.
Supports CUTLASS, DeepGEMM, FlashInfer, TRTLLM MXFP8, Triton and Marlin
backends.
"""
is_gfx1250 = False
if current_platform.is_rocm():
@@ -1451,6 +1504,16 @@ def convert_weight_to_mxfp4_moe_kernel_format(
w2_bias,
)
if mxfp4_backend == Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4:
return (
w13_weight.data,
w2_weight.data,
_swizzle_cutlass_mxfp4_scales(w13_weight_scale.data),
_swizzle_cutlass_mxfp4_scales(w2_weight_scale.data),
w13_bias,
w2_bias,
)
if mxfp4_backend == Mxfp4MoeBackend.DEEPGEMM_MXFP4:
w13_weight_scale, w2_weight_scale = _pack_deepgemm_mxfp4_scales(
w13_weight,
@@ -2005,7 +2068,10 @@ def make_mxfp4_moe_quant_config(
block_shape=None,
gemm1_clamp_limit=swiglu_limit,
)
elif mxfp4_backend == Mxfp4MoeBackend.AITER_MXFP4_MXFP4:
elif mxfp4_backend in (
Mxfp4MoeBackend.AITER_MXFP4_MXFP4,
Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4,
):
return ocp_mx_moe_quant_config(
quant_dtype="mxfp4",
w1_bias=w1_bias,