[Kernel] Enable OAI Triton MXFP4 MoE on SM12x (#58877)

Signed-off-by: lazymio <mio@lazym.io>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: Shengqi Chen <harry-chen@outlook.com>
This commit is contained in:
lazymio
2026-10-06 01:49:13 -04:00
committed by GitHub
co-authored by Claude Fable 5.1 Shengqi Chen
parent b3082d3044
commit 40c86ae1ce
4 changed files with 96 additions and 7 deletions
@@ -32,8 +32,11 @@ from triton_kernels.testing import assert_close
from vllm.model_executor.layers.fused_moe.config import mxfp4_w4a16_moe_quant_config
from vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe import (
_sm12x_block_m,
triton_kernel_moe_forward,
)
from vllm.model_executor.layers.fused_moe.oracle import mxfp4 as mxfp4_oracle
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import Mxfp4MoeBackend
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import mx_scale_kwargs
from vllm.utils.math_utils import round_up
@@ -314,9 +317,13 @@ def test_equiv(num_token, a_dtype, w_dtype, tp, workspace_init):
pc2,
) = init_compute_data(M, K, N, E, a_dtype, w_dtype, num_warps=8)
if current_platform.is_device_capability_family(100):
if current_platform.is_device_capability_family(
100
) or current_platform.is_device_capability_family(120):
# Mirrors `_swizzle_mxfp4`; SM12x also needs epilogue_subtile=1.
constraints = {
"is_persistent": True,
"epilogue_subtile": 1,
}
opt_flags.update_opt_flags_constraints(constraints)
@@ -356,6 +363,38 @@ def test_equiv(num_token, a_dtype, w_dtype, tp, workspace_init):
assert_close(ref=out_ref, tri=out_triton_monolithic, maxtol=0.025, rmstol=0.005)
@pytest.mark.parametrize(
("num_rows", "num_experts", "expected"),
[
(4, 32, 16), # bs=1 decode: below the 16-row floor
(32 * 20, 32, 32), # 20 tokens/expert rounds up to 32
(32 * 33, 32, 32), # heuristic would pick 64; capped for 99KB smem
(12000, 32, 32), # prefill: heuristic would pick 128
],
)
def test_sm12x_block_m(num_rows, num_experts, expected):
assert _sm12x_block_m(num_rows, num_experts) == expected
@pytest.mark.parametrize("sm12x", [True, False])
def test_gpt_oss_priority_prefers_marlin_on_sm12x(
monkeypatch: pytest.MonkeyPatch, sm12x: bool
):
"""SM12x runs the OAI Triton kernels at small tiles only, so MARLIN must
stay the default there while TRITON remains selectable explicitly."""
monkeypatch.setattr(mxfp4_oracle.current_platform, "is_cuda", lambda: True)
monkeypatch.setattr(
mxfp4_oracle.current_platform,
"is_device_capability_family",
lambda family: sm12x and family == 120,
)
order = mxfp4_oracle._get_priority_backends_for_gpt_oss()
triton_first = order.index(Mxfp4MoeBackend.TRITON) < order.index(
Mxfp4MoeBackend.MARLIN
)
assert triton_first is not sm12x
def test_unit_shuffle():
N = ModelConfig.intermediate_size
K = ModelConfig.hidden_size
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import functools
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
@@ -41,11 +43,11 @@ def _triton_kernel_moe_supports_current_device() -> bool:
p = current_platform
if p.is_cuda():
cap = p.get_device_capability()
# Keep the original `(9, 0) <= cap < (11, 0)` window on
# CUDA (covers Hopper SM90 and Blackwell SM100, excludes
# SM120) — this PR is ROCm-scoped and the broader CUDA
# range was not validated.
return cap is not None and (9, 0) <= (cap.major, cap.minor) < (11, 0)
# Hopper SM90, datacenter Blackwell SM100/103 and consumer Blackwell
# SM120/121. SM12x needs the opt-flag constraints applied by
# `_swizzle_mxfp4` and `_constrain_sm12x_block_m` to fit its 99KB of
# shared memory; it is still ordered after MARLIN by the selector.
return cap is not None and (9, 0) <= (cap.major, cap.minor) < (13, 0)
if p.is_rocm():
from vllm.platforms.rocm import on_gfx1x, on_gfx9
@@ -57,6 +59,36 @@ def _triton_kernel_moe_supports_current_device() -> bool:
return False
@functools.cache
def _is_sm12x() -> bool:
return current_platform.is_cuda() and current_platform.is_device_capability_family(
120
)
# Consumer Blackwell has 99KB of opt-in shared memory: with block_m >= 64 the
# persistent mxfp4 kernel fits a single pipeline stage and runs 2-3x slower
# than with 32, but triton_kernels' block_m heuristic still tops out at 128.
_SM12X_MAX_BLOCK_M = 32
def _sm12x_block_m(num_rows: int, num_experts: int) -> int:
"""triton_kernels' tokens-per-expert block_m heuristic, capped at 32."""
tokens_per_expt = max(1, num_rows // num_experts)
return max(16, min(triton.next_power_of_2(tokens_per_expt), _SM12X_MAX_BLOCK_M))
def _constrain_sm12x_block_m(num_rows: int, num_experts: int) -> None:
"""Pin block_m for the following matmul_ogs calls on SM12x."""
if not _is_sm12x():
return
import triton_kernels.matmul_ogs_details.opt_flags as opt_flags
opt_flags.update_opt_flags_constraints(
{"block_m": _sm12x_block_m(num_rows, num_experts)}
)
def _patch_make_bitmatrix_metadata() -> None:
"""Monkey-patch make_bitmatrix_metadata to support non-power-of-2 top_k.
@@ -784,6 +816,7 @@ def triton_kernel_fused_experts(
)
)
gammas = routing_data.gate_scal if routing_data else None
_constrain_sm12x_block_m(M * topk, E)
matmul_ogs(
hidden_states,
@@ -1403,6 +1436,7 @@ class UnfusedOAITritonExperts(LoRAExpertsMixin, BaseOAITritonExperts):
intermediate_cache2 = _resize_cache(workspace13, (M * topk, activation_out_dim))
gammas = routing_data.gate_scal if routing_data else None
_constrain_sm12x_block_m(M * topk, E)
matmul_ogs(
hidden_states,
@@ -386,6 +386,15 @@ def _get_priority_backends_for_gpt_oss() -> list[Mxfp4MoeBackend]:
Mxfp4MoeBackend.CPU,
Mxfp4MoeBackend.EMULATION,
]
if current_platform.is_cuda() and current_platform.is_device_capability_family(120):
# Consumer Blackwell: the OAI Triton kernels run but their 99KB shared
# memory budget caps them at small tiles, so MARLIN prefills ~10%
# faster at equal decode throughput. Keep TRITON as an opt-in.
_AVAILABLE_BACKENDS.remove(Mxfp4MoeBackend.TRITON)
_AVAILABLE_BACKENDS.insert(
_AVAILABLE_BACKENDS.index(Mxfp4MoeBackend.BATCHED_MARLIN) + 1,
Mxfp4MoeBackend.TRITON,
)
return _AVAILABLE_BACKENDS
@@ -120,7 +120,14 @@ def _swizzle_mxfp4(quant_tensor, scale, num_warps=8):
K = scale.shape[-1]
pad_k = -K % 4
scale = torch.nn.functional.pad(scale, (0, pad_k))
elif current_platform.is_device_capability_family(100):
elif current_platform.is_device_capability_family(
100
) or current_platform.is_device_capability_family(120):
# Native MXFP (cap >= 10.0) requires the persistent kernel. On
# SM12x, epilogue_subtile=1 also keeps the heuristic away from a
# triton_kernels 3.5.1 codegen bug (SWAP_XW with a subtiled
# epilogue); block_m is pinned per call in
# gpt_oss_triton_kernels_moe to fit the 99KB shared memory.
constraints = {
"is_persistent": True,
"epilogue_subtile": 1,