[Kernel] Enable OAI Triton MXFP4 MoE on SM12x (#58877)
Signed-off-by: lazymio <mio@lazym.io> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: Shengqi Chen <harry-chen@outlook.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
Shengqi Chen
parent
b3082d3044
commit
40c86ae1ce
@@ -32,8 +32,11 @@ from triton_kernels.testing import assert_close
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.config import mxfp4_w4a16_moe_quant_config
|
||||
from vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe import (
|
||||
_sm12x_block_m,
|
||||
triton_kernel_moe_forward,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle import mxfp4 as mxfp4_oracle
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import Mxfp4MoeBackend
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import mx_scale_kwargs
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
@@ -314,9 +317,13 @@ def test_equiv(num_token, a_dtype, w_dtype, tp, workspace_init):
|
||||
pc2,
|
||||
) = init_compute_data(M, K, N, E, a_dtype, w_dtype, num_warps=8)
|
||||
|
||||
if current_platform.is_device_capability_family(100):
|
||||
if current_platform.is_device_capability_family(
|
||||
100
|
||||
) or current_platform.is_device_capability_family(120):
|
||||
# Mirrors `_swizzle_mxfp4`; SM12x also needs epilogue_subtile=1.
|
||||
constraints = {
|
||||
"is_persistent": True,
|
||||
"epilogue_subtile": 1,
|
||||
}
|
||||
opt_flags.update_opt_flags_constraints(constraints)
|
||||
|
||||
@@ -356,6 +363,38 @@ def test_equiv(num_token, a_dtype, w_dtype, tp, workspace_init):
|
||||
assert_close(ref=out_ref, tri=out_triton_monolithic, maxtol=0.025, rmstol=0.005)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_rows", "num_experts", "expected"),
|
||||
[
|
||||
(4, 32, 16), # bs=1 decode: below the 16-row floor
|
||||
(32 * 20, 32, 32), # 20 tokens/expert rounds up to 32
|
||||
(32 * 33, 32, 32), # heuristic would pick 64; capped for 99KB smem
|
||||
(12000, 32, 32), # prefill: heuristic would pick 128
|
||||
],
|
||||
)
|
||||
def test_sm12x_block_m(num_rows, num_experts, expected):
|
||||
assert _sm12x_block_m(num_rows, num_experts) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sm12x", [True, False])
|
||||
def test_gpt_oss_priority_prefers_marlin_on_sm12x(
|
||||
monkeypatch: pytest.MonkeyPatch, sm12x: bool
|
||||
):
|
||||
"""SM12x runs the OAI Triton kernels at small tiles only, so MARLIN must
|
||||
stay the default there while TRITON remains selectable explicitly."""
|
||||
monkeypatch.setattr(mxfp4_oracle.current_platform, "is_cuda", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
mxfp4_oracle.current_platform,
|
||||
"is_device_capability_family",
|
||||
lambda family: sm12x and family == 120,
|
||||
)
|
||||
order = mxfp4_oracle._get_priority_backends_for_gpt_oss()
|
||||
triton_first = order.index(Mxfp4MoeBackend.TRITON) < order.index(
|
||||
Mxfp4MoeBackend.MARLIN
|
||||
)
|
||||
assert triton_first is not sm12x
|
||||
|
||||
|
||||
def test_unit_shuffle():
|
||||
N = ModelConfig.intermediate_size
|
||||
K = ModelConfig.hidden_size
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import functools
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
@@ -41,11 +43,11 @@ def _triton_kernel_moe_supports_current_device() -> bool:
|
||||
p = current_platform
|
||||
if p.is_cuda():
|
||||
cap = p.get_device_capability()
|
||||
# Keep the original `(9, 0) <= cap < (11, 0)` window on
|
||||
# CUDA (covers Hopper SM90 and Blackwell SM100, excludes
|
||||
# SM120) — this PR is ROCm-scoped and the broader CUDA
|
||||
# range was not validated.
|
||||
return cap is not None and (9, 0) <= (cap.major, cap.minor) < (11, 0)
|
||||
# Hopper SM90, datacenter Blackwell SM100/103 and consumer Blackwell
|
||||
# SM120/121. SM12x needs the opt-flag constraints applied by
|
||||
# `_swizzle_mxfp4` and `_constrain_sm12x_block_m` to fit its 99KB of
|
||||
# shared memory; it is still ordered after MARLIN by the selector.
|
||||
return cap is not None and (9, 0) <= (cap.major, cap.minor) < (13, 0)
|
||||
if p.is_rocm():
|
||||
from vllm.platforms.rocm import on_gfx1x, on_gfx9
|
||||
|
||||
@@ -57,6 +59,36 @@ def _triton_kernel_moe_supports_current_device() -> bool:
|
||||
return False
|
||||
|
||||
|
||||
@functools.cache
|
||||
def _is_sm12x() -> bool:
|
||||
return current_platform.is_cuda() and current_platform.is_device_capability_family(
|
||||
120
|
||||
)
|
||||
|
||||
|
||||
# Consumer Blackwell has 99KB of opt-in shared memory: with block_m >= 64 the
|
||||
# persistent mxfp4 kernel fits a single pipeline stage and runs 2-3x slower
|
||||
# than with 32, but triton_kernels' block_m heuristic still tops out at 128.
|
||||
_SM12X_MAX_BLOCK_M = 32
|
||||
|
||||
|
||||
def _sm12x_block_m(num_rows: int, num_experts: int) -> int:
|
||||
"""triton_kernels' tokens-per-expert block_m heuristic, capped at 32."""
|
||||
tokens_per_expt = max(1, num_rows // num_experts)
|
||||
return max(16, min(triton.next_power_of_2(tokens_per_expt), _SM12X_MAX_BLOCK_M))
|
||||
|
||||
|
||||
def _constrain_sm12x_block_m(num_rows: int, num_experts: int) -> None:
|
||||
"""Pin block_m for the following matmul_ogs calls on SM12x."""
|
||||
if not _is_sm12x():
|
||||
return
|
||||
import triton_kernels.matmul_ogs_details.opt_flags as opt_flags
|
||||
|
||||
opt_flags.update_opt_flags_constraints(
|
||||
{"block_m": _sm12x_block_m(num_rows, num_experts)}
|
||||
)
|
||||
|
||||
|
||||
def _patch_make_bitmatrix_metadata() -> None:
|
||||
"""Monkey-patch make_bitmatrix_metadata to support non-power-of-2 top_k.
|
||||
|
||||
@@ -784,6 +816,7 @@ def triton_kernel_fused_experts(
|
||||
)
|
||||
)
|
||||
gammas = routing_data.gate_scal if routing_data else None
|
||||
_constrain_sm12x_block_m(M * topk, E)
|
||||
|
||||
matmul_ogs(
|
||||
hidden_states,
|
||||
@@ -1403,6 +1436,7 @@ class UnfusedOAITritonExperts(LoRAExpertsMixin, BaseOAITritonExperts):
|
||||
intermediate_cache2 = _resize_cache(workspace13, (M * topk, activation_out_dim))
|
||||
|
||||
gammas = routing_data.gate_scal if routing_data else None
|
||||
_constrain_sm12x_block_m(M * topk, E)
|
||||
|
||||
matmul_ogs(
|
||||
hidden_states,
|
||||
|
||||
@@ -386,6 +386,15 @@ def _get_priority_backends_for_gpt_oss() -> list[Mxfp4MoeBackend]:
|
||||
Mxfp4MoeBackend.CPU,
|
||||
Mxfp4MoeBackend.EMULATION,
|
||||
]
|
||||
if current_platform.is_cuda() and current_platform.is_device_capability_family(120):
|
||||
# Consumer Blackwell: the OAI Triton kernels run but their 99KB shared
|
||||
# memory budget caps them at small tiles, so MARLIN prefills ~10%
|
||||
# faster at equal decode throughput. Keep TRITON as an opt-in.
|
||||
_AVAILABLE_BACKENDS.remove(Mxfp4MoeBackend.TRITON)
|
||||
_AVAILABLE_BACKENDS.insert(
|
||||
_AVAILABLE_BACKENDS.index(Mxfp4MoeBackend.BATCHED_MARLIN) + 1,
|
||||
Mxfp4MoeBackend.TRITON,
|
||||
)
|
||||
return _AVAILABLE_BACKENDS
|
||||
|
||||
|
||||
|
||||
@@ -120,7 +120,14 @@ def _swizzle_mxfp4(quant_tensor, scale, num_warps=8):
|
||||
K = scale.shape[-1]
|
||||
pad_k = -K % 4
|
||||
scale = torch.nn.functional.pad(scale, (0, pad_k))
|
||||
elif current_platform.is_device_capability_family(100):
|
||||
elif current_platform.is_device_capability_family(
|
||||
100
|
||||
) or current_platform.is_device_capability_family(120):
|
||||
# Native MXFP (cap >= 10.0) requires the persistent kernel. On
|
||||
# SM12x, epilogue_subtile=1 also keeps the heuristic away from a
|
||||
# triton_kernels 3.5.1 codegen bug (SWAP_XW with a subtiled
|
||||
# epilogue); block_m is pinned per call in
|
||||
# gpt_oss_triton_kernels_moe to fit the 99KB shared memory.
|
||||
constraints = {
|
||||
"is_persistent": True,
|
||||
"epilogue_subtile": 1,
|
||||
|
||||
Reference in New Issue
Block a user