diff --git a/tests/kernels/moe/test_gpt_oss_triton_kernels.py b/tests/kernels/moe/test_gpt_oss_triton_kernels.py index c1d83d8349..406e525bdf 100644 --- a/tests/kernels/moe/test_gpt_oss_triton_kernels.py +++ b/tests/kernels/moe/test_gpt_oss_triton_kernels.py @@ -32,8 +32,11 @@ from triton_kernels.testing import assert_close from vllm.model_executor.layers.fused_moe.config import mxfp4_w4a16_moe_quant_config from vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe import ( + _sm12x_block_m, triton_kernel_moe_forward, ) +from vllm.model_executor.layers.fused_moe.oracle import mxfp4 as mxfp4_oracle +from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import Mxfp4MoeBackend from vllm.model_executor.layers.quantization.utils.mxfp4_utils import mx_scale_kwargs from vllm.utils.math_utils import round_up @@ -314,9 +317,13 @@ def test_equiv(num_token, a_dtype, w_dtype, tp, workspace_init): pc2, ) = init_compute_data(M, K, N, E, a_dtype, w_dtype, num_warps=8) - if current_platform.is_device_capability_family(100): + if current_platform.is_device_capability_family( + 100 + ) or current_platform.is_device_capability_family(120): + # Mirrors `_swizzle_mxfp4`; SM12x also needs epilogue_subtile=1. constraints = { "is_persistent": True, + "epilogue_subtile": 1, } opt_flags.update_opt_flags_constraints(constraints) @@ -356,6 +363,38 @@ def test_equiv(num_token, a_dtype, w_dtype, tp, workspace_init): assert_close(ref=out_ref, tri=out_triton_monolithic, maxtol=0.025, rmstol=0.005) +@pytest.mark.parametrize( + ("num_rows", "num_experts", "expected"), + [ + (4, 32, 16), # bs=1 decode: below the 16-row floor + (32 * 20, 32, 32), # 20 tokens/expert rounds up to 32 + (32 * 33, 32, 32), # heuristic would pick 64; capped for 99KB smem + (12000, 32, 32), # prefill: heuristic would pick 128 + ], +) +def test_sm12x_block_m(num_rows, num_experts, expected): + assert _sm12x_block_m(num_rows, num_experts) == expected + + +@pytest.mark.parametrize("sm12x", [True, False]) +def test_gpt_oss_priority_prefers_marlin_on_sm12x( + monkeypatch: pytest.MonkeyPatch, sm12x: bool +): + """SM12x runs the OAI Triton kernels at small tiles only, so MARLIN must + stay the default there while TRITON remains selectable explicitly.""" + monkeypatch.setattr(mxfp4_oracle.current_platform, "is_cuda", lambda: True) + monkeypatch.setattr( + mxfp4_oracle.current_platform, + "is_device_capability_family", + lambda family: sm12x and family == 120, + ) + order = mxfp4_oracle._get_priority_backends_for_gpt_oss() + triton_first = order.index(Mxfp4MoeBackend.TRITON) < order.index( + Mxfp4MoeBackend.MARLIN + ) + assert triton_first is not sm12x + + def test_unit_shuffle(): N = ModelConfig.intermediate_size K = ModelConfig.hidden_size diff --git a/vllm/model_executor/layers/fused_moe/experts/gpt_oss_triton_kernels_moe.py b/vllm/model_executor/layers/fused_moe/experts/gpt_oss_triton_kernels_moe.py index d23edaa01a..e3b6995d69 100644 --- a/vllm/model_executor/layers/fused_moe/experts/gpt_oss_triton_kernels_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/gpt_oss_triton_kernels_moe.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import functools + import torch import vllm.model_executor.layers.fused_moe.modular_kernel as mk @@ -41,11 +43,11 @@ def _triton_kernel_moe_supports_current_device() -> bool: p = current_platform if p.is_cuda(): cap = p.get_device_capability() - # Keep the original `(9, 0) <= cap < (11, 0)` window on - # CUDA (covers Hopper SM90 and Blackwell SM100, excludes - # SM120) — this PR is ROCm-scoped and the broader CUDA - # range was not validated. - return cap is not None and (9, 0) <= (cap.major, cap.minor) < (11, 0) + # Hopper SM90, datacenter Blackwell SM100/103 and consumer Blackwell + # SM120/121. SM12x needs the opt-flag constraints applied by + # `_swizzle_mxfp4` and `_constrain_sm12x_block_m` to fit its 99KB of + # shared memory; it is still ordered after MARLIN by the selector. + return cap is not None and (9, 0) <= (cap.major, cap.minor) < (13, 0) if p.is_rocm(): from vllm.platforms.rocm import on_gfx1x, on_gfx9 @@ -57,6 +59,36 @@ def _triton_kernel_moe_supports_current_device() -> bool: return False +@functools.cache +def _is_sm12x() -> bool: + return current_platform.is_cuda() and current_platform.is_device_capability_family( + 120 + ) + + +# Consumer Blackwell has 99KB of opt-in shared memory: with block_m >= 64 the +# persistent mxfp4 kernel fits a single pipeline stage and runs 2-3x slower +# than with 32, but triton_kernels' block_m heuristic still tops out at 128. +_SM12X_MAX_BLOCK_M = 32 + + +def _sm12x_block_m(num_rows: int, num_experts: int) -> int: + """triton_kernels' tokens-per-expert block_m heuristic, capped at 32.""" + tokens_per_expt = max(1, num_rows // num_experts) + return max(16, min(triton.next_power_of_2(tokens_per_expt), _SM12X_MAX_BLOCK_M)) + + +def _constrain_sm12x_block_m(num_rows: int, num_experts: int) -> None: + """Pin block_m for the following matmul_ogs calls on SM12x.""" + if not _is_sm12x(): + return + import triton_kernels.matmul_ogs_details.opt_flags as opt_flags + + opt_flags.update_opt_flags_constraints( + {"block_m": _sm12x_block_m(num_rows, num_experts)} + ) + + def _patch_make_bitmatrix_metadata() -> None: """Monkey-patch make_bitmatrix_metadata to support non-power-of-2 top_k. @@ -784,6 +816,7 @@ def triton_kernel_fused_experts( ) ) gammas = routing_data.gate_scal if routing_data else None + _constrain_sm12x_block_m(M * topk, E) matmul_ogs( hidden_states, @@ -1403,6 +1436,7 @@ class UnfusedOAITritonExperts(LoRAExpertsMixin, BaseOAITritonExperts): intermediate_cache2 = _resize_cache(workspace13, (M * topk, activation_out_dim)) gammas = routing_data.gate_scal if routing_data else None + _constrain_sm12x_block_m(M * topk, E) matmul_ogs( hidden_states, diff --git a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py index a3c65dc4a5..691d849228 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py @@ -386,6 +386,15 @@ def _get_priority_backends_for_gpt_oss() -> list[Mxfp4MoeBackend]: Mxfp4MoeBackend.CPU, Mxfp4MoeBackend.EMULATION, ] + if current_platform.is_cuda() and current_platform.is_device_capability_family(120): + # Consumer Blackwell: the OAI Triton kernels run but their 99KB shared + # memory budget caps them at small tiles, so MARLIN prefills ~10% + # faster at equal decode throughput. Keep TRITON as an opt-in. + _AVAILABLE_BACKENDS.remove(Mxfp4MoeBackend.TRITON) + _AVAILABLE_BACKENDS.insert( + _AVAILABLE_BACKENDS.index(Mxfp4MoeBackend.BATCHED_MARLIN) + 1, + Mxfp4MoeBackend.TRITON, + ) return _AVAILABLE_BACKENDS diff --git a/vllm/model_executor/layers/quantization/utils/mxfp4_utils.py b/vllm/model_executor/layers/quantization/utils/mxfp4_utils.py index 8cc626f386..45562b12b1 100644 --- a/vllm/model_executor/layers/quantization/utils/mxfp4_utils.py +++ b/vllm/model_executor/layers/quantization/utils/mxfp4_utils.py @@ -120,7 +120,14 @@ def _swizzle_mxfp4(quant_tensor, scale, num_warps=8): K = scale.shape[-1] pad_k = -K % 4 scale = torch.nn.functional.pad(scale, (0, pad_k)) - elif current_platform.is_device_capability_family(100): + elif current_platform.is_device_capability_family( + 100 + ) or current_platform.is_device_capability_family(120): + # Native MXFP (cap >= 10.0) requires the persistent kernel. On + # SM12x, epilogue_subtile=1 also keeps the heuristic away from a + # triton_kernels 3.5.1 codegen bug (SWAP_XW with a subtiled + # epilogue); block_m is pinned per call in + # gpt_oss_triton_kernels_moe to fit the 99KB shared memory. constraints = { "is_persistent": True, "epilogue_subtile": 1,