[MoE] MXFP4 oracle: add CUTLASS W4A4 backend and BF16 activation fallback (#60079)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -37,9 +37,12 @@ from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
|
||||
run_cutlass_moe_fp4,
|
||||
run_cutlass_moe_fp8,
|
||||
run_cutlass_moe_mxfp4,
|
||||
swizzle_mxfp4_scales,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle import mxfp4 as mxfp4_oracle
|
||||
from vllm.model_executor.layers.fused_moe.oracle import nvfp4 as nvfp4_oracle
|
||||
from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Dynamic
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
@@ -294,6 +297,87 @@ def test_cutlass_fp4_moe_padded_routes_do_not_change_valid_output(quantization:
|
||||
torch.testing.assert_close(actual[1:], torch.zeros_like(actual[1:]))
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_cutlass_mxfp4_oracle_matches_direct_kernel(monkeypatch, workspace_init):
|
||||
"""The MXFP4 oracle selects CUTLASS for W4A4 and swizzles checkpoint
|
||||
scales into the layout the kernel expects."""
|
||||
if not CutlassExpertsMxfp4._supports_current_device():
|
||||
pytest.skip("CUTLASS mxfp4 MoE is not supported on this GPU")
|
||||
monkeypatch.setattr(mxfp4_oracle, "_user_moe_activation_override", lambda: None)
|
||||
|
||||
set_random_seed(7)
|
||||
e, m, n, k, topk = 8, 33, 256, 512, 2
|
||||
device = torch.device("cuda")
|
||||
w1 = torch.randint(0, 256, (e, 2 * n, k // 2), device=device, dtype=torch.uint8)
|
||||
w2 = torch.randint(0, 256, (e, k, n // 2), device=device, dtype=torch.uint8)
|
||||
w1_scale = torch.randint(118, 124, (e, 2 * n, k // 32), device=device).byte()
|
||||
w2_scale = torch.randint(118, 124, (e, k, n // 32), device=device).byte()
|
||||
a = torch.randn((m, k), device=device, dtype=torch.bfloat16)
|
||||
score = torch.randn((m, e), device=device, dtype=torch.bfloat16)
|
||||
topk_weights, topk_ids, _ = fused_topk(a, score, topk, renormalize=False)
|
||||
|
||||
def swizzle(scale: torch.Tensor) -> torch.Tensor:
|
||||
rows, cols = scale.shape[1:]
|
||||
return torch.stack(
|
||||
[swizzle_mxfp4_scales(s, rows, cols * 32).view(rows, cols) for s in scale]
|
||||
)
|
||||
|
||||
expected = torch.empty_like(a)
|
||||
run_cutlass_moe_mxfp4(
|
||||
output=expected,
|
||||
a=a,
|
||||
w1_fp4=w1,
|
||||
w1_blockscale=swizzle(w1_scale),
|
||||
w2_fp4=w2,
|
||||
w2_blockscale=swizzle(w2_scale),
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=MoEActivation.SILU,
|
||||
workspace13=torch.empty((m * topk, max(2 * n, k)), device=device).to(a),
|
||||
workspace2=torch.empty((m * topk, n), device=device).to(a),
|
||||
m=m,
|
||||
n=n,
|
||||
k=k,
|
||||
e=e,
|
||||
device=device,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
moe_config = make_dummy_moe_config(
|
||||
num_experts=e, experts_per_token=topk, hidden_dim=k, intermediate_size=n
|
||||
)
|
||||
with set_current_vllm_config(vllm_config):
|
||||
backend, experts_cls = mxfp4_oracle.select_mxfp4_moe_backend(
|
||||
moe_config, activation_key=kMxfp4Dynamic
|
||||
)
|
||||
assert backend == mxfp4_oracle.Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4
|
||||
|
||||
w13, w2, w13_scale, w2_scale, _, _ = (
|
||||
mxfp4_oracle.convert_weight_to_mxfp4_moe_kernel_format(
|
||||
backend, torch.nn.Module(), w1, w2, w1_scale, w2_scale
|
||||
)
|
||||
)
|
||||
kernel = mxfp4_oracle.make_mxfp4_moe_kernel(
|
||||
mxfp4_oracle.make_mxfp4_moe_quant_config(backend, w13_scale, w2_scale),
|
||||
moe_config,
|
||||
experts_cls,
|
||||
backend,
|
||||
)
|
||||
actual = kernel.apply(
|
||||
a,
|
||||
w13,
|
||||
w2,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
activation=MoEActivation.SILU,
|
||||
global_num_experts=e,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, atol=0, rtol=0)
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class MOETensors:
|
||||
a: torch.Tensor
|
||||
|
||||
@@ -7,7 +7,11 @@ from dataclasses import dataclass, replace
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.moe.utils import check_accuracy, check_deferred_moe_finalize
|
||||
from tests.kernels.moe.utils import (
|
||||
check_accuracy,
|
||||
check_deferred_moe_finalize,
|
||||
make_dummy_moe_config,
|
||||
)
|
||||
from vllm._aiter_ops import (
|
||||
is_aiter_found,
|
||||
is_aiter_found_and_supported,
|
||||
@@ -2009,6 +2013,52 @@ def test_select_mxfp4_moe_backend_raises_with_unsupported_reasons(
|
||||
mxfp4_oracle.select_mxfp4_moe_backend(moe_config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"moe_backend,w4a4,supported,expected",
|
||||
[
|
||||
# The checkpoint's activation wins over BF16 backends listed first.
|
||||
("auto", True, {"CUTLASS_MXFP4_MXFP4", "MARLIN"}, "CUTLASS_MXFP4_MXFP4"),
|
||||
# No W4A4 kernel: run at BF16 rather than another quantized activation.
|
||||
("auto", True, {"FLASHINFER_TRTLLM_MXFP4_MXFP8", "MARLIN"}, "MARLIN"),
|
||||
("marlin", True, {"MARLIN"}, "MARLIN"),
|
||||
# A BF16 checkpoint never gets its activations quantized.
|
||||
("auto", False, {"CUTLASS_MXFP4_MXFP4", "MARLIN"}, "MARLIN"),
|
||||
],
|
||||
)
|
||||
def test_select_mxfp4_moe_backend_respects_checkpoint_activation(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
moe_backend: str,
|
||||
w4a4: bool,
|
||||
supported: set[str],
|
||||
expected: str,
|
||||
):
|
||||
"""Activations run at the checkpoint's precision when a backend supports
|
||||
it and fall back to BF16 otherwise, never to a lower precision."""
|
||||
import vllm.model_executor.layers.fused_moe.oracle.mxfp4 as mxfp4_oracle
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kMxfp4Dynamic,
|
||||
)
|
||||
|
||||
def backend_to_kernel_cls(backend):
|
||||
class Experts:
|
||||
@staticmethod
|
||||
def is_supported_config(*args):
|
||||
return backend.name in supported, None
|
||||
|
||||
return [Experts]
|
||||
|
||||
monkeypatch.setattr(mxfp4_oracle, "backend_to_kernel_cls", backend_to_kernel_cls)
|
||||
monkeypatch.setattr(mxfp4_oracle, "_user_moe_activation_override", lambda: None)
|
||||
moe_config = make_dummy_moe_config()
|
||||
moe_config.moe_backend = moe_backend
|
||||
|
||||
backend, _ = mxfp4_oracle.select_mxfp4_moe_backend(
|
||||
moe_config, activation_key=kMxfp4Dynamic if w4a4 else None
|
||||
)
|
||||
|
||||
assert backend.name == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend_name", ["AITER_TRITON_MXFP4_BF16", "AITER_MXFP4_FP8"])
|
||||
@pytest.mark.parametrize("use_ep", [False, True])
|
||||
def test_aiter_mxfp4_monolithic_rejects_expert_parallel(
|
||||
|
||||
@@ -114,6 +114,18 @@ def _pack_deepgemm_mxfp4_scales(
|
||||
)
|
||||
|
||||
|
||||
def _swizzle_cutlass_mxfp4_scales(scale: torch.Tensor) -> torch.Tensor:
|
||||
"""Swizzle [E, N, K // 32] block scales into the CUTLASS tiled layout."""
|
||||
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
|
||||
swizzle_mxfp4_scales,
|
||||
)
|
||||
|
||||
n, scale_k = scale.shape[1:]
|
||||
return torch.stack(
|
||||
[swizzle_mxfp4_scales(s, n, scale_k * 32).reshape(n, scale_k) for s in scale]
|
||||
)
|
||||
|
||||
|
||||
class Mxfp4MoeBackend(Enum):
|
||||
NONE = "None"
|
||||
# b12x backends
|
||||
@@ -127,6 +139,8 @@ class Mxfp4MoeBackend(Enum):
|
||||
# FlashInfer CUTLASS backends
|
||||
FLASHINFER_CUTLASS_MXFP4_MXFP8 = "FLASHINFER_CUTLASS_MXFP4_MXFP8"
|
||||
FLASHINFER_CUTLASS_MXFP4_BF16 = "FLASHINFER_CUTLASS_MXFP4_BF16"
|
||||
# vLLM CUTLASS backend
|
||||
CUTLASS_MXFP4_MXFP4 = "CUTLASS_MXFP4_MXFP4"
|
||||
# Marlin
|
||||
BATCHED_MARLIN = "BATCHED_MARLIN"
|
||||
MARLIN = "MARLIN"
|
||||
@@ -218,6 +232,13 @@ def backend_to_kernel_cls(
|
||||
|
||||
return [FlashInferExperts]
|
||||
|
||||
elif backend == Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4:
|
||||
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
|
||||
CutlassExpertsMxfp4,
|
||||
)
|
||||
|
||||
return [CutlassExpertsMxfp4]
|
||||
|
||||
elif backend == Mxfp4MoeBackend.TRITON:
|
||||
from vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe import ( # noqa: E501
|
||||
OAITritonExperts,
|
||||
@@ -378,6 +399,7 @@ def _get_priority_backends_for_gpt_oss() -> list[Mxfp4MoeBackend]:
|
||||
Mxfp4MoeBackend.TRITON,
|
||||
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
|
||||
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
|
||||
Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4,
|
||||
# TRITON_UNFUSED has bug with MTP support
|
||||
# TODO re-enable after kernel is fixed
|
||||
# TRITON_UNFUSED
|
||||
@@ -440,7 +462,10 @@ def _backend_activation_key(backend: Mxfp4MoeBackend) -> QuantKey | None:
|
||||
return kMxfp8Dynamic
|
||||
if backend == Mxfp4MoeBackend.AITER_MXFP4_FP8:
|
||||
return kFp8StaticTensorSym
|
||||
if backend == Mxfp4MoeBackend.AITER_MXFP4_MXFP4:
|
||||
if backend in (
|
||||
Mxfp4MoeBackend.AITER_MXFP4_MXFP4,
|
||||
Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4,
|
||||
):
|
||||
return kMxfp4Dynamic
|
||||
return None # BF16 activation
|
||||
|
||||
@@ -474,6 +499,22 @@ def _make_log_backend(backend: Mxfp4MoeBackend) -> str:
|
||||
return f"Using '{backend.value}' Mxfp4 MoE backend."
|
||||
|
||||
|
||||
def _warn_on_bf16_fallback(
|
||||
backend: Mxfp4MoeBackend, requested_activation_key: QuantKey | None
|
||||
) -> None:
|
||||
if (
|
||||
requested_activation_key is not None
|
||||
and backend != Mxfp4MoeBackend.EMULATION
|
||||
and _backend_activation_key(backend) is None
|
||||
):
|
||||
logger.warning_once(
|
||||
"No Mxfp4 MoE backend supports activation=%s for the deployment "
|
||||
"configuration. Falling back to '%s' with BF16 activations.",
|
||||
requested_activation_key,
|
||||
backend.value,
|
||||
)
|
||||
|
||||
|
||||
def _make_log_unsupported(backend: Mxfp4MoeBackend, reason: str | None) -> str:
|
||||
base = (
|
||||
f"Mxfp4 MoE backend '{backend.value}' does not support the "
|
||||
@@ -505,17 +546,19 @@ def _filter_by_activation(
|
||||
backends: list[Mxfp4MoeBackend],
|
||||
requested_activation_key: QuantKey | None,
|
||||
) -> list[Mxfp4MoeBackend]:
|
||||
"""Pick variants matching ``requested_activation_key``; without one,
|
||||
prefer BF16 if the list has any, else keep the list as-is so explicit
|
||||
non-BF16 picks (e.g. the ``_afp8`` aliases) still land."""
|
||||
"""Pick variants matching ``requested_activation_key``, followed by BF16
|
||||
variants as a higher-precision fallback; without one, prefer BF16 if the
|
||||
list has any, else keep the list as-is so explicit non-BF16 picks (e.g.
|
||||
the ``_afp8`` aliases) still land."""
|
||||
bf16 = [b for b in backends if _backend_activation_key(b) is None]
|
||||
if requested_activation_key is not None:
|
||||
return [
|
||||
matching = [
|
||||
b
|
||||
for b in backends
|
||||
if _backend_activation_key(b) == requested_activation_key
|
||||
or b == Mxfp4MoeBackend.EMULATION
|
||||
]
|
||||
bf16 = [b for b in backends if _backend_activation_key(b) is None]
|
||||
return matching + [b for b in bf16 if b not in matching]
|
||||
return bf16 if bf16 else backends
|
||||
|
||||
|
||||
@@ -576,8 +619,9 @@ def select_mxfp4_moe_backend(
|
||||
|
||||
Args:
|
||||
config: MoE configuration
|
||||
activation_key: Optional activation quantization key. If provided,
|
||||
overrides the default activation key for backend selection.
|
||||
activation_key: Optional activation quantization key declared by
|
||||
the checkpoint. Backends that match it are preferred, with BF16
|
||||
activation backends as a fallback.
|
||||
Use kFp8StaticTensorSym for W4A8 scheme.
|
||||
|
||||
Note: Shape-specific fallbacks may still occur at runtime.
|
||||
@@ -614,7 +658,7 @@ def select_mxfp4_moe_backend(
|
||||
else _backend_activation_key(requested_backend)
|
||||
)
|
||||
try:
|
||||
return _return_or_raise(
|
||||
selected = _return_or_raise(
|
||||
requested_backend,
|
||||
config,
|
||||
kMxfp4Static,
|
||||
@@ -623,6 +667,9 @@ def select_mxfp4_moe_backend(
|
||||
)
|
||||
except ValueError as e:
|
||||
last_error = e
|
||||
else:
|
||||
_warn_on_bf16_fallback(requested_backend, requested_activation_key)
|
||||
return selected
|
||||
assert last_error is not None
|
||||
raise last_error
|
||||
|
||||
@@ -648,10 +695,10 @@ def select_mxfp4_moe_backend(
|
||||
|
||||
unsupported_reasons = []
|
||||
for backend in AVAILABLE_BACKENDS:
|
||||
# Use requested_activation_key if provided, otherwise use backend default
|
||||
# EMULATION takes the requested key, other backends use their own
|
||||
act_key = (
|
||||
requested_activation_key
|
||||
if requested_activation_key is not None
|
||||
if backend == Mxfp4MoeBackend.EMULATION
|
||||
else _backend_activation_key(backend)
|
||||
)
|
||||
for k_cls in backend_to_kernel_cls(backend):
|
||||
@@ -660,6 +707,7 @@ def select_mxfp4_moe_backend(
|
||||
)
|
||||
if supported:
|
||||
logger.info_once(_make_log_backend(backend))
|
||||
_warn_on_bf16_fallback(backend, requested_activation_key)
|
||||
return backend, k_cls
|
||||
else:
|
||||
logger.debug_once(_make_log_unsupported(backend, reason))
|
||||
@@ -789,6 +837,10 @@ def mxfp4_round_up_hidden_size_and_intermediate_size(
|
||||
# DeepGEMM requires M/N/K alignment
|
||||
intermediate_size = round_up(intermediate_size, 128)
|
||||
hidden_size = round_up(hidden_size, 128)
|
||||
elif backend == Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4:
|
||||
# CUTLASS scale tiles cover 128 rows by 128 elements
|
||||
intermediate_size = round_up(intermediate_size, 128)
|
||||
hidden_size = round_up(hidden_size, 128)
|
||||
elif backend in (Mxfp4MoeBackend.MARLIN, Mxfp4MoeBackend.BATCHED_MARLIN):
|
||||
intermediate_size = round_up(intermediate_size, 128)
|
||||
if current_platform.is_xpu():
|
||||
@@ -1433,7 +1485,8 @@ def convert_weight_to_mxfp4_moe_kernel_format(
|
||||
]:
|
||||
"""Convert loaded weights into backend-specific kernel format.
|
||||
|
||||
Supports DeepGEMM, FlashInfer, TRTLLM MXFP8, Triton and Marlin backends.
|
||||
Supports CUTLASS, DeepGEMM, FlashInfer, TRTLLM MXFP8, Triton and Marlin
|
||||
backends.
|
||||
"""
|
||||
is_gfx1250 = False
|
||||
if current_platform.is_rocm():
|
||||
@@ -1451,6 +1504,16 @@ def convert_weight_to_mxfp4_moe_kernel_format(
|
||||
w2_bias,
|
||||
)
|
||||
|
||||
if mxfp4_backend == Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4:
|
||||
return (
|
||||
w13_weight.data,
|
||||
w2_weight.data,
|
||||
_swizzle_cutlass_mxfp4_scales(w13_weight_scale.data),
|
||||
_swizzle_cutlass_mxfp4_scales(w2_weight_scale.data),
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
)
|
||||
|
||||
if mxfp4_backend == Mxfp4MoeBackend.DEEPGEMM_MXFP4:
|
||||
w13_weight_scale, w2_weight_scale = _pack_deepgemm_mxfp4_scales(
|
||||
w13_weight,
|
||||
@@ -2005,7 +2068,10 @@ def make_mxfp4_moe_quant_config(
|
||||
block_shape=None,
|
||||
gemm1_clamp_limit=swiglu_limit,
|
||||
)
|
||||
elif mxfp4_backend == Mxfp4MoeBackend.AITER_MXFP4_MXFP4:
|
||||
elif mxfp4_backend in (
|
||||
Mxfp4MoeBackend.AITER_MXFP4_MXFP4,
|
||||
Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4,
|
||||
):
|
||||
return ocp_mx_moe_quant_config(
|
||||
quant_dtype="mxfp4",
|
||||
w1_bias=w1_bias,
|
||||
|
||||
Reference in New Issue
Block a user