diff --git a/tests/kernels/moe/test_cutlass_moe.py b/tests/kernels/moe/test_cutlass_moe.py index 1910796623..050e1bc104 100644 --- a/tests/kernels/moe/test_cutlass_moe.py +++ b/tests/kernels/moe/test_cutlass_moe.py @@ -37,9 +37,12 @@ from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import ( run_cutlass_moe_fp4, run_cutlass_moe_fp8, run_cutlass_moe_mxfp4, + swizzle_mxfp4_scales, ) +from vllm.model_executor.layers.fused_moe.oracle import mxfp4 as mxfp4_oracle from vllm.model_executor.layers.fused_moe.oracle import nvfp4 as nvfp4_oracle from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input +from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Dynamic from vllm.platforms import current_platform from vllm.utils.torch_utils import set_random_seed @@ -294,6 +297,87 @@ def test_cutlass_fp4_moe_padded_routes_do_not_change_valid_output(quantization: torch.testing.assert_close(actual[1:], torch.zeros_like(actual[1:])) +@torch.inference_mode() +def test_cutlass_mxfp4_oracle_matches_direct_kernel(monkeypatch, workspace_init): + """The MXFP4 oracle selects CUTLASS for W4A4 and swizzles checkpoint + scales into the layout the kernel expects.""" + if not CutlassExpertsMxfp4._supports_current_device(): + pytest.skip("CUTLASS mxfp4 MoE is not supported on this GPU") + monkeypatch.setattr(mxfp4_oracle, "_user_moe_activation_override", lambda: None) + + set_random_seed(7) + e, m, n, k, topk = 8, 33, 256, 512, 2 + device = torch.device("cuda") + w1 = torch.randint(0, 256, (e, 2 * n, k // 2), device=device, dtype=torch.uint8) + w2 = torch.randint(0, 256, (e, k, n // 2), device=device, dtype=torch.uint8) + w1_scale = torch.randint(118, 124, (e, 2 * n, k // 32), device=device).byte() + w2_scale = torch.randint(118, 124, (e, k, n // 32), device=device).byte() + a = torch.randn((m, k), device=device, dtype=torch.bfloat16) + score = torch.randn((m, e), device=device, dtype=torch.bfloat16) + topk_weights, topk_ids, _ = fused_topk(a, score, topk, renormalize=False) + + def swizzle(scale: torch.Tensor) -> torch.Tensor: + rows, cols = scale.shape[1:] + return torch.stack( + [swizzle_mxfp4_scales(s, rows, cols * 32).view(rows, cols) for s in scale] + ) + + expected = torch.empty_like(a) + run_cutlass_moe_mxfp4( + output=expected, + a=a, + w1_fp4=w1, + w1_blockscale=swizzle(w1_scale), + w2_fp4=w2, + w2_blockscale=swizzle(w2_scale), + topk_weights=topk_weights, + topk_ids=topk_ids, + activation=MoEActivation.SILU, + workspace13=torch.empty((m * topk, max(2 * n, k)), device=device).to(a), + workspace2=torch.empty((m * topk, n), device=device).to(a), + m=m, + n=n, + k=k, + e=e, + device=device, + apply_router_weight_on_input=False, + ) + + moe_config = make_dummy_moe_config( + num_experts=e, experts_per_token=topk, hidden_dim=k, intermediate_size=n + ) + with set_current_vllm_config(vllm_config): + backend, experts_cls = mxfp4_oracle.select_mxfp4_moe_backend( + moe_config, activation_key=kMxfp4Dynamic + ) + assert backend == mxfp4_oracle.Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4 + + w13, w2, w13_scale, w2_scale, _, _ = ( + mxfp4_oracle.convert_weight_to_mxfp4_moe_kernel_format( + backend, torch.nn.Module(), w1, w2, w1_scale, w2_scale + ) + ) + kernel = mxfp4_oracle.make_mxfp4_moe_kernel( + mxfp4_oracle.make_mxfp4_moe_quant_config(backend, w13_scale, w2_scale), + moe_config, + experts_cls, + backend, + ) + actual = kernel.apply( + a, + w13, + w2, + topk_weights, + topk_ids, + activation=MoEActivation.SILU, + global_num_experts=e, + expert_map=None, + apply_router_weight_on_input=False, + ) + + torch.testing.assert_close(actual, expected, atol=0, rtol=0) + + @dataclasses.dataclass class MOETensors: a: torch.Tensor diff --git a/tests/kernels/moe/test_ocp_mx_moe.py b/tests/kernels/moe/test_ocp_mx_moe.py index 5a7053a8b1..f20927565e 100644 --- a/tests/kernels/moe/test_ocp_mx_moe.py +++ b/tests/kernels/moe/test_ocp_mx_moe.py @@ -7,7 +7,11 @@ from dataclasses import dataclass, replace import pytest import torch -from tests.kernels.moe.utils import check_accuracy, check_deferred_moe_finalize +from tests.kernels.moe.utils import ( + check_accuracy, + check_deferred_moe_finalize, + make_dummy_moe_config, +) from vllm._aiter_ops import ( is_aiter_found, is_aiter_found_and_supported, @@ -2009,6 +2013,52 @@ def test_select_mxfp4_moe_backend_raises_with_unsupported_reasons( mxfp4_oracle.select_mxfp4_moe_backend(moe_config) +@pytest.mark.parametrize( + "moe_backend,w4a4,supported,expected", + [ + # The checkpoint's activation wins over BF16 backends listed first. + ("auto", True, {"CUTLASS_MXFP4_MXFP4", "MARLIN"}, "CUTLASS_MXFP4_MXFP4"), + # No W4A4 kernel: run at BF16 rather than another quantized activation. + ("auto", True, {"FLASHINFER_TRTLLM_MXFP4_MXFP8", "MARLIN"}, "MARLIN"), + ("marlin", True, {"MARLIN"}, "MARLIN"), + # A BF16 checkpoint never gets its activations quantized. + ("auto", False, {"CUTLASS_MXFP4_MXFP4", "MARLIN"}, "MARLIN"), + ], +) +def test_select_mxfp4_moe_backend_respects_checkpoint_activation( + monkeypatch: pytest.MonkeyPatch, + moe_backend: str, + w4a4: bool, + supported: set[str], + expected: str, +): + """Activations run at the checkpoint's precision when a backend supports + it and fall back to BF16 otherwise, never to a lower precision.""" + import vllm.model_executor.layers.fused_moe.oracle.mxfp4 as mxfp4_oracle + from vllm.model_executor.layers.quantization.utils.quant_utils import ( + kMxfp4Dynamic, + ) + + def backend_to_kernel_cls(backend): + class Experts: + @staticmethod + def is_supported_config(*args): + return backend.name in supported, None + + return [Experts] + + monkeypatch.setattr(mxfp4_oracle, "backend_to_kernel_cls", backend_to_kernel_cls) + monkeypatch.setattr(mxfp4_oracle, "_user_moe_activation_override", lambda: None) + moe_config = make_dummy_moe_config() + moe_config.moe_backend = moe_backend + + backend, _ = mxfp4_oracle.select_mxfp4_moe_backend( + moe_config, activation_key=kMxfp4Dynamic if w4a4 else None + ) + + assert backend.name == expected + + @pytest.mark.parametrize("backend_name", ["AITER_TRITON_MXFP4_BF16", "AITER_MXFP4_FP8"]) @pytest.mark.parametrize("use_ep", [False, True]) def test_aiter_mxfp4_monolithic_rejects_expert_parallel( diff --git a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py index 5463853cab..f5fccc0343 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py @@ -114,6 +114,18 @@ def _pack_deepgemm_mxfp4_scales( ) +def _swizzle_cutlass_mxfp4_scales(scale: torch.Tensor) -> torch.Tensor: + """Swizzle [E, N, K // 32] block scales into the CUTLASS tiled layout.""" + from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import ( + swizzle_mxfp4_scales, + ) + + n, scale_k = scale.shape[1:] + return torch.stack( + [swizzle_mxfp4_scales(s, n, scale_k * 32).reshape(n, scale_k) for s in scale] + ) + + class Mxfp4MoeBackend(Enum): NONE = "None" # b12x backends @@ -127,6 +139,8 @@ class Mxfp4MoeBackend(Enum): # FlashInfer CUTLASS backends FLASHINFER_CUTLASS_MXFP4_MXFP8 = "FLASHINFER_CUTLASS_MXFP4_MXFP8" FLASHINFER_CUTLASS_MXFP4_BF16 = "FLASHINFER_CUTLASS_MXFP4_BF16" + # vLLM CUTLASS backend + CUTLASS_MXFP4_MXFP4 = "CUTLASS_MXFP4_MXFP4" # Marlin BATCHED_MARLIN = "BATCHED_MARLIN" MARLIN = "MARLIN" @@ -218,6 +232,13 @@ def backend_to_kernel_cls( return [FlashInferExperts] + elif backend == Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4: + from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import ( + CutlassExpertsMxfp4, + ) + + return [CutlassExpertsMxfp4] + elif backend == Mxfp4MoeBackend.TRITON: from vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe import ( # noqa: E501 OAITritonExperts, @@ -378,6 +399,7 @@ def _get_priority_backends_for_gpt_oss() -> list[Mxfp4MoeBackend]: Mxfp4MoeBackend.TRITON, Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16, Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8, + Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4, # TRITON_UNFUSED has bug with MTP support # TODO re-enable after kernel is fixed # TRITON_UNFUSED @@ -440,7 +462,10 @@ def _backend_activation_key(backend: Mxfp4MoeBackend) -> QuantKey | None: return kMxfp8Dynamic if backend == Mxfp4MoeBackend.AITER_MXFP4_FP8: return kFp8StaticTensorSym - if backend == Mxfp4MoeBackend.AITER_MXFP4_MXFP4: + if backend in ( + Mxfp4MoeBackend.AITER_MXFP4_MXFP4, + Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4, + ): return kMxfp4Dynamic return None # BF16 activation @@ -474,6 +499,22 @@ def _make_log_backend(backend: Mxfp4MoeBackend) -> str: return f"Using '{backend.value}' Mxfp4 MoE backend." +def _warn_on_bf16_fallback( + backend: Mxfp4MoeBackend, requested_activation_key: QuantKey | None +) -> None: + if ( + requested_activation_key is not None + and backend != Mxfp4MoeBackend.EMULATION + and _backend_activation_key(backend) is None + ): + logger.warning_once( + "No Mxfp4 MoE backend supports activation=%s for the deployment " + "configuration. Falling back to '%s' with BF16 activations.", + requested_activation_key, + backend.value, + ) + + def _make_log_unsupported(backend: Mxfp4MoeBackend, reason: str | None) -> str: base = ( f"Mxfp4 MoE backend '{backend.value}' does not support the " @@ -505,17 +546,19 @@ def _filter_by_activation( backends: list[Mxfp4MoeBackend], requested_activation_key: QuantKey | None, ) -> list[Mxfp4MoeBackend]: - """Pick variants matching ``requested_activation_key``; without one, - prefer BF16 if the list has any, else keep the list as-is so explicit - non-BF16 picks (e.g. the ``_afp8`` aliases) still land.""" + """Pick variants matching ``requested_activation_key``, followed by BF16 + variants as a higher-precision fallback; without one, prefer BF16 if the + list has any, else keep the list as-is so explicit non-BF16 picks (e.g. + the ``_afp8`` aliases) still land.""" + bf16 = [b for b in backends if _backend_activation_key(b) is None] if requested_activation_key is not None: - return [ + matching = [ b for b in backends if _backend_activation_key(b) == requested_activation_key or b == Mxfp4MoeBackend.EMULATION ] - bf16 = [b for b in backends if _backend_activation_key(b) is None] + return matching + [b for b in bf16 if b not in matching] return bf16 if bf16 else backends @@ -576,8 +619,9 @@ def select_mxfp4_moe_backend( Args: config: MoE configuration - activation_key: Optional activation quantization key. If provided, - overrides the default activation key for backend selection. + activation_key: Optional activation quantization key declared by + the checkpoint. Backends that match it are preferred, with BF16 + activation backends as a fallback. Use kFp8StaticTensorSym for W4A8 scheme. Note: Shape-specific fallbacks may still occur at runtime. @@ -614,7 +658,7 @@ def select_mxfp4_moe_backend( else _backend_activation_key(requested_backend) ) try: - return _return_or_raise( + selected = _return_or_raise( requested_backend, config, kMxfp4Static, @@ -623,6 +667,9 @@ def select_mxfp4_moe_backend( ) except ValueError as e: last_error = e + else: + _warn_on_bf16_fallback(requested_backend, requested_activation_key) + return selected assert last_error is not None raise last_error @@ -648,10 +695,10 @@ def select_mxfp4_moe_backend( unsupported_reasons = [] for backend in AVAILABLE_BACKENDS: - # Use requested_activation_key if provided, otherwise use backend default + # EMULATION takes the requested key, other backends use their own act_key = ( requested_activation_key - if requested_activation_key is not None + if backend == Mxfp4MoeBackend.EMULATION else _backend_activation_key(backend) ) for k_cls in backend_to_kernel_cls(backend): @@ -660,6 +707,7 @@ def select_mxfp4_moe_backend( ) if supported: logger.info_once(_make_log_backend(backend)) + _warn_on_bf16_fallback(backend, requested_activation_key) return backend, k_cls else: logger.debug_once(_make_log_unsupported(backend, reason)) @@ -789,6 +837,10 @@ def mxfp4_round_up_hidden_size_and_intermediate_size( # DeepGEMM requires M/N/K alignment intermediate_size = round_up(intermediate_size, 128) hidden_size = round_up(hidden_size, 128) + elif backend == Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4: + # CUTLASS scale tiles cover 128 rows by 128 elements + intermediate_size = round_up(intermediate_size, 128) + hidden_size = round_up(hidden_size, 128) elif backend in (Mxfp4MoeBackend.MARLIN, Mxfp4MoeBackend.BATCHED_MARLIN): intermediate_size = round_up(intermediate_size, 128) if current_platform.is_xpu(): @@ -1433,7 +1485,8 @@ def convert_weight_to_mxfp4_moe_kernel_format( ]: """Convert loaded weights into backend-specific kernel format. - Supports DeepGEMM, FlashInfer, TRTLLM MXFP8, Triton and Marlin backends. + Supports CUTLASS, DeepGEMM, FlashInfer, TRTLLM MXFP8, Triton and Marlin + backends. """ is_gfx1250 = False if current_platform.is_rocm(): @@ -1451,6 +1504,16 @@ def convert_weight_to_mxfp4_moe_kernel_format( w2_bias, ) + if mxfp4_backend == Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4: + return ( + w13_weight.data, + w2_weight.data, + _swizzle_cutlass_mxfp4_scales(w13_weight_scale.data), + _swizzle_cutlass_mxfp4_scales(w2_weight_scale.data), + w13_bias, + w2_bias, + ) + if mxfp4_backend == Mxfp4MoeBackend.DEEPGEMM_MXFP4: w13_weight_scale, w2_weight_scale = _pack_deepgemm_mxfp4_scales( w13_weight, @@ -2005,7 +2068,10 @@ def make_mxfp4_moe_quant_config( block_shape=None, gemm1_clamp_limit=swiglu_limit, ) - elif mxfp4_backend == Mxfp4MoeBackend.AITER_MXFP4_MXFP4: + elif mxfp4_backend in ( + Mxfp4MoeBackend.AITER_MXFP4_MXFP4, + Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4, + ): return ocp_mx_moe_quant_config( quant_dtype="mxfp4", w1_bias=w1_bias,