[Perf] Improve BF16x3 router GEMM accuracy and make it default on sm100 (#55899)

Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
Thien Tran
2026-09-09 07:33:12 -07:00
committed by GitHub
co-authored by OpenAI Codex Jee Jee Li
parent 3116c5d06b
commit cc4210f671
6 changed files with 314 additions and 88 deletions
@@ -6,6 +6,10 @@ Computes ``X @ W.T`` for BF16 ``X`` with shape ``[N, K]`` and FP32 router
weights ``W`` with shape ``[M, K]`` by decomposing each FP32 weight value into
three BF16 residual terms inside the kernel, then accumulating the three BF16
MMA results into FP32 TMEM output.
The TMEM accumulation chain is bounded to ``num_tmem_acc`` K-tiles: at chunk
boundaries the epilogue warps drain the main-term accumulator into a register
master accumulator and the MMA warp resets it.
"""
from functools import cache
@@ -22,6 +26,7 @@ from quack.compile_utils import make_fake_tensor
from vllm.cute_utils import _tcgen05, simple_tma_copy
from vllm.triton_utils import tl, triton
from vllm.utils import math_utils
from vllm.utils.torch_utils import direct_register_custom_op
__all__ = ["bf16x3_router_gemm"]
@@ -82,6 +87,7 @@ class Sm100BF16x3RouterGemm:
self.cta_tile = (BN, 128, 64)
self.num_stages = 2
self.num_warps = 10
self.num_tmem_acc = 8
@cute.jit
def _make_tma(self, tensor: cute.Tensor, BM: int, BK: int):
@@ -129,6 +135,7 @@ class Sm100BF16x3RouterGemm:
BN, BM, BK = self.cta_tile
num_stages = self.num_stages
num_tmem_acc = self.num_tmem_acc
N, K = X_tma.tma_tensor.shape
M, _ = W_tma.tma_tensor.shape
@@ -151,7 +158,8 @@ class Sm100BF16x3RouterGemm:
tma_full_mbar = smem.allocate_array(Int64, num_stages)
tma_empty_mbar = smem.allocate_array(Int64, num_stages)
w_full_mbar = smem.allocate_array(Int64, num_stages)
mma_mbar = smem.allocate_array(Int64, 1)
acc_full_mbar = smem.allocate_array(Int64, 1)
acc_empty_mbar = smem.allocate_array(Int64, 1)
taddr = smem.allocate(Int32, 4)
BAR_TMEM_ALLOC = 1
@@ -171,7 +179,8 @@ class Sm100BF16x3RouterGemm:
cute.arch.mbarrier_init(tma_full_mbar + i, 1)
cute.arch.mbarrier_init(tma_empty_mbar + i, 1)
cute.arch.mbarrier_init(w_full_mbar + i, 128)
cute.arch.mbarrier_init(mma_mbar, 1)
cute.arch.mbarrier_init(acc_full_mbar, 1)
cute.arch.mbarrier_init(acc_empty_mbar, 128)
cute.arch.mbarrier_init_fence()
elif warp_id == 1:
cpasync.prefetch_descriptor(X_tma.atom)
@@ -213,40 +222,52 @@ class Sm100BF16x3RouterGemm:
elif warp_id == 8:
# MMA warp
stage_id = 0
parity = 0
tma_stage_id = 0
tma_parity = 0
tmem_acc_count = 0
tmem_parity = 1
idesc = _tcgen05.make_bf16_idesc(BM, BN)
sdesc = _tcgen05.make_sdesc_128B_swizzle(0)
for tile_k in cutlass.range(bid_k, k_tiles, split_k, unroll=1):
cute.arch.mbarrier_wait(tma_full_mbar + stage_id, parity)
cute.arch.mbarrier_wait(w_full_mbar + stage_id, parity)
if tmem_acc_count == 0:
cute.arch.mbarrier_wait(acc_empty_mbar, tmem_parity)
cute.arch.mbarrier_wait(tma_full_mbar + tma_stage_id, tma_parity)
cute.arch.mbarrier_wait(w_full_mbar + tma_stage_id, tma_parity)
_tcgen05.fence_after_thread_sync()
w_tmem = w_tmem_base + stage_id * (BK // 2 * 3)
x_desc = sdesc | (sX[None, None, stage_id].iterator.toint() >> 4)
w_tmem = w_tmem_base + tma_stage_id * (BK // 2 * 3)
x_desc = sdesc | (sX[None, None, tma_stage_id].iterator.toint() >> 4)
for k in cutlass.range_constexpr(BK // 16):
enable_d = (tile_k > bid_k) or (k > 0)
enable_d_main = (tmem_acc_count > 0) or (k > 0)
enable_d_res = (tile_k > bid_k) or (k > 0)
_tcgen05.mma_ts_f16(
acc_main, w_tmem + k * 8, x_desc, idesc, enable_d
acc_main, w_tmem + k * 8, x_desc, idesc, enable_d_main
)
_tcgen05.mma_ts_f16(
acc_res, w_tmem + 32 + k * 8, x_desc, idesc, enable_d
acc_res, w_tmem + 32 + k * 8, x_desc, idesc, enable_d_res
)
_tcgen05.mma_ts_f16(
acc_res, w_tmem + 64 + k * 8, x_desc, idesc, True
)
x_desc += 32 >> 4
_tcgen05.commit(tma_empty_mbar + stage_id)
_tcgen05.commit(tma_empty_mbar + tma_stage_id)
stage_id = (stage_id + 1) % self.num_stages
if stage_id == 0:
parity ^= 1
tma_stage_id = (tma_stage_id + 1) % num_stages
if tma_stage_id == 0:
tma_parity ^= 1
_tcgen05.commit(mma_mbar)
tmem_acc_count = (tmem_acc_count + 1) % num_tmem_acc
if tmem_acc_count == 0:
_tcgen05.commit(acc_full_mbar)
tmem_parity ^= 1
if tmem_acc_count != 0:
_tcgen05.commit(acc_full_mbar)
elif warp_id >= 4:
# prep warps: decompose FP32 W into 3xBF16
@@ -303,39 +324,93 @@ class Sm100BF16x3RouterGemm:
_tcgen05.alloc(taddr)
cute.arch.barrier(barrier_id=BAR_TMEM_ALLOC, number_of_threads=128)
if warp_id == 0:
cute.arch.mbarrier_wait(mma_mbar, 0)
cute.arch.barrier(barrier_id=BAR_EPI, number_of_threads=128)
_tcgen05.fence_after_thread_sync()
cute.arch.griddepcontrol_launch_dependents()
tiles_local = cute.ceil_div(k_tiles - bid_k, split_k)
num_chunks = cute.ceil_div(tiles_local, num_tmem_acc)
WIDTH = 8
for i in cutlass.range_constexpr(BN // WIDTH):
tcol = i * WIDTH
main_regs = cute.make_rmem_tensor(WIDTH, Float32)
res_regs = cute.make_rmem_tensor(WIDTH, Float32)
main_regs.store(_tcgen05.ld(warp_id * 32, tcol, "32x32b", WIDTH))
res_regs.store(_tcgen05.ld(warp_id * 32, BN + tcol, "32x32b", WIDTH))
_tcgen05.wait_ld()
# CuteDSL will codegen add.f32x2
for j in cutlass.range(WIDTH, vectorize=True):
main_regs[j] += res_regs[j]
main_regs = cute.make_rmem_tensor(WIDTH, Float32)
res_regs = cute.make_rmem_tensor(WIDTH, Float32)
if num_chunks == 1:
# single chunk
cute.arch.mbarrier_wait(acc_full_mbar, 0)
_tcgen05.fence_after_thread_sync()
w_row_idx = bid_m * BM + tid
for j in cutlass.range_constexpr(WIDTH):
x_row_idx = bid_n * BN + i * WIDTH + j
if x_row_idx < N and w_row_idx < M:
out[bid_k, x_row_idx, w_row_idx] = main_regs[j]
for i in cutlass.range_constexpr(BN // WIDTH):
tcol = i * WIDTH
main_regs.store(_tcgen05.ld(warp_id * 32, tcol, "32x32b", WIDTH))
res_regs.store(
_tcgen05.ld(warp_id * 32, BN + tcol, "32x32b", WIDTH)
)
_tcgen05.wait_ld()
# CuteDSL will codegen add.f32x2
for j in cutlass.range(WIDTH, vectorize=True):
main_regs[j] += res_regs[j]
for j in cutlass.range_constexpr(WIDTH):
x_row_idx = bid_n * BN + i * WIDTH + j
if x_row_idx < N and w_row_idx < M:
out[bid_k, x_row_idx, w_row_idx] = main_regs[j]
else:
# multiple chunks
master_acc = cute.make_rmem_tensor(BN, Float32)
# chunk 0: pure TMEM load, no accumulation.
cute.arch.mbarrier_wait(acc_full_mbar, 0)
_tcgen05.fence_after_thread_sync()
master_acc.store(_tcgen05.ld(warp_id * 32, 0, "32x32b", BN))
_tcgen05.wait_ld()
_tcgen05.fence_before_thread_sync()
cute.arch.mbarrier_arrive(acc_empty_mbar)
# accumulate main tmem acc
for chunk in cutlass.range(1, num_chunks, unroll=1):
cute.arch.mbarrier_wait(acc_full_mbar, chunk & 1)
_tcgen05.fence_after_thread_sync()
for i in cutlass.range_constexpr(BN // WIDTH):
tcol = i * WIDTH
main_regs.store(
_tcgen05.ld(warp_id * 32, tcol, "32x32b", WIDTH)
)
_tcgen05.wait_ld()
# CuteDSL refuses to compile
# master_acc[i * WIDTH + j] += main_regs[j]
# when vectorize=True
master_view = cute.local_tile(master_acc, (WIDTH,), (i,))
for j in cutlass.range(WIDTH, vectorize=True):
master_view[j] += main_regs[j]
_tcgen05.fence_before_thread_sync()
cute.arch.mbarrier_arrive(acc_empty_mbar)
# fold in the residual accumulator
for i in cutlass.range_constexpr(BN // WIDTH):
tcol = i * WIDTH
res_regs.store(
_tcgen05.ld(warp_id * 32, BN + tcol, "32x32b", WIDTH)
)
_tcgen05.wait_ld()
master_view = cute.local_tile(master_acc, (WIDTH,), (i,))
for j in cutlass.range(WIDTH, vectorize=True):
master_view[j] += res_regs[j]
w_row_idx = bid_m * BM + tid
for j in cutlass.range_constexpr(WIDTH):
x_row_idx = bid_n * BN + i * WIDTH + j
if x_row_idx < N and w_row_idx < M:
out[bid_k, x_row_idx, w_row_idx] = master_acc[i * WIDTH + j]
cute.arch.griddepcontrol_launch_dependents()
cute.arch.barrier(barrier_id=BAR_EPI, number_of_threads=128)
if warp_id == 0:
_tcgen05.dealloc()
@cache
@staticmethod
def compile(BN: int = 128, K: int = 6144):
def compile(K: int, BN: int = 128):
N = cute.sym_int()
M = cute.sym_int()
SPLIT_K = cute.sym_int()
@@ -392,16 +467,19 @@ def _splitk_reduce_kernel(
)
def _splitk_reduce_config(split_k: int) -> tuple[int, int, int]:
block_s = triton.next_power_of_2(split_k)
if block_s >= 64:
return 1, 32, block_s
if block_s >= 8:
return 1, 256, block_s
return min(16, 32 // block_s), 32, block_s
def splitk_reduce_triton(partials: torch.Tensor, out: torch.Tensor):
split_k, N, M = partials.shape
block_s = 1 << (split_k - 1).bit_length()
split_stride = partials.stride(0)
if block_s >= 64:
BN, BM = 1, 32
elif block_s >= 8:
BN, BM = 1, 256
else:
BN, BM = min(16, 32 // block_s), 32
BN, BM, block_s = _splitk_reduce_config(split_k)
grid = (triton.cdiv(N, BN), triton.cdiv(M, BM))
_splitk_reduce_kernel[grid](
partials,
@@ -419,27 +497,124 @@ def splitk_reduce_triton(partials: torch.Tensor, out: torch.Tensor):
)
def bf16x3_router_gemm(X: torch.Tensor, W: torch.Tensor) -> torch.Tensor:
"""Return ``X @ W.T`` using the SM100 BF16x3 router GEMM kernel."""
N, K = X.shape
M, _ = W.shape
num_sms = torch.cuda.get_device_properties(X.device).multi_processor_count
# Accuracy-constrained (BN, split_k) configs from GB300 sweeps.
_TILE_CONFIG_OVERRIDES = {
(2816, 256): { # Hunyuan-V4
64: (16, 16),
128: (32, 16),
256: (64, 16),
512: (128, 16),
},
(3072, 256): { # MiniMax-M2
64: (16, 16),
128: (32, 16),
256: (64, 16),
512: (128, 16),
},
(4096, 192): { # Hunyuan-V3
64: (32, 32),
128: (64, 32),
256: (128, 32),
512: (128, 19),
},
(6144, 128): { # MiniMax-M3
64: (32, 64),
128: (64, 64),
256: (64, 32),
512: (128, 32),
},
}
def _pick_tile_config(N: int, K: int, M: int, num_sms: int) -> tuple[int, int]:
"""Return (BN, split_k): tuned table mid-range, generic rule otherwise."""
k_tiles = math_utils.cdiv(K, 64)
if 32 < N <= 512:
token_bucket = triton.next_power_of_2(N)
overrides = _TILE_CONFIG_OVERRIDES.get((K, M))
if overrides is not None:
BN, split_k = overrides[token_bucket]
return BN, min(split_k, k_tiles)
# next power of 2 within 8 and 128
BN = triton.next_power_of_2(N)
BN = min(max(BN, 8), 128)
BM = 128
BK = 64
k_tiles = math_utils.cdiv(K, BK)
grid_m = math_utils.cdiv(M, BM)
grid_m = math_utils.cdiv(M, 128)
grid_n = math_utils.cdiv(N, BN)
base_ctas = grid_m * grid_n
split_k = min(k_tiles, max(1, num_sms // base_ctas))
return BN, split_k
def warmup_bf16x3_router_gemm(
K: int,
M: int,
min_num_tokens: int,
max_num_tokens: int,
) -> tuple[int, ...]:
"""Compile every BF16x3 router GEMM configuration reachable at runtime."""
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
TritonWarmupTensor,
triton_scalar_specialization_rep,
)
device = torch.accelerator.current_device_index()
num_sms = torch.cuda.get_device_properties(device).multi_processor_count
gemm_configs: set[int] = set()
reduce_configs: set[tuple[int, int, int, int, int, int]] = set()
for N in range(min_num_tokens, max_num_tokens + 1):
BN, split_k = _pick_tile_config(N, K, M, num_sms)
gemm_configs.add(BN)
if split_k == 1:
continue
reduce_BN, reduce_BM, block_s = _splitk_reduce_config(split_k)
reduce_configs.add(
(
triton_scalar_specialization_rep(N),
triton_scalar_specialization_rep(N * M),
triton_scalar_specialization_rep(split_k),
reduce_BN,
reduce_BM,
block_s,
)
)
configs = tuple(sorted(gemm_configs))
for BN in configs:
Sm100BF16x3RouterGemm.compile(K, BN)
for N, split_stride, split_k, BN, BM, block_s in sorted(reduce_configs):
partials = TritonWarmupTensor(torch.float32)
out = TritonWarmupTensor(torch.float32)
_splitk_reduce_kernel.warmup(
partials,
out,
N,
M,
split_stride,
split_k,
BN=BN,
BM=BM,
BS=block_s,
USE_PDL=True,
num_warps=4,
launch_pdl=True,
grid=(1, 1),
)
return configs
def _bf16x3_router_gemm(X: torch.Tensor, W: torch.Tensor) -> torch.Tensor:
"""Return ``X @ W.T`` using the SM100 BF16x3 router GEMM kernel."""
N, K = X.shape
M, _ = W.shape
num_sms = torch.cuda.get_device_properties(X.device).multi_processor_count
BN, split_k = _pick_tile_config(N, K, M, num_sms)
partials = X.new_empty(split_k, N, M, dtype=torch.float32)
Sm100BF16x3RouterGemm.compile(BN, K)(X, W, partials, split_k)
Sm100BF16x3RouterGemm.compile(K, BN)(X, W, partials, split_k)
if split_k == 1:
return partials.squeeze(0)
@@ -447,3 +622,21 @@ def bf16x3_router_gemm(X: torch.Tensor, W: torch.Tensor) -> torch.Tensor:
out = X.new_empty(N, M, dtype=torch.float32)
splitk_reduce_triton(partials, out)
return out
def _bf16x3_router_gemm_fake(
X: torch.Tensor,
W: torch.Tensor,
) -> torch.Tensor:
return X.new_empty((X.shape[0], W.shape[0]), dtype=torch.float32)
direct_register_custom_op(
op_name="bf16x3_router_gemm",
op_func=_bf16x3_router_gemm,
fake_impl=_bf16x3_router_gemm_fake,
)
def bf16x3_router_gemm(X: torch.Tensor, W: torch.Tensor) -> torch.Tensor:
return torch.ops.vllm.bf16x3_router_gemm(X, W)
@@ -4,15 +4,11 @@ import torch
from torch.nn.parameter import Parameter
import vllm._custom_ops as ops
from vllm.config import get_current_vllm_config_or_none
from vllm.logger import init_logger
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.platforms import current_platform
from vllm.utils.torch_utils import direct_register_custom_op
logger = init_logger(__name__)
@PluggableLayer.register("gate_linear")
class GateLinear(ReplicatedLinear):
@@ -22,7 +18,7 @@ class GateLinear(ReplicatedLinear):
K divisible by 8)
2. fp32 specialized kernel (SM90+ or gfx950, bf16/fp32 in, fp32 out,
M<=32, model-specific shapes)
3. experimental bf16x3 CuteDSL kernel (opt-in, SM100, bf16 in, fp32 weight)
3. bf16x3 CuteDSL kernel (SM100, bf16 in, fp32 weight)
4. cuBLAS bf16×bf16→fp32 (SM90+ + bf16 weight + fp32 out_dtype)
5. F.linear via ReplicatedLinear (ultimate fallback)
@@ -85,11 +81,6 @@ class GateLinear(ReplicatedLinear):
self.allow_specialized_router_gemm = can_use_specialized_kernels
# fp32 specialized kernel eligibility (exact dims, fp32 weight)
vllm_config = get_current_vllm_config_or_none()
enable_bf16x3_router_gemm = (
vllm_config is not None
and vllm_config.kernel_config.enable_bf16x3_router_gemm
)
self.allow_fp32_router_gemm = (
not bias
and self.weight.dtype == torch.float32
@@ -108,10 +99,7 @@ class GateLinear(ReplicatedLinear):
and current_platform.is_cuda()
and is_blackwell
and input_size % 8 == 0
and enable_bf16x3_router_gemm
)
if self.allow_bf16x3_router_gemm:
logger.info_once("Enabled experimental SM100 BF16x3 router GEMM.")
# Fused bf16 x bf16 -> fp32 GEMM eligibility. torch.mm's out_dtype
# epilogue folds the fp32 cast into the GEMM, removing the standalone
@@ -198,7 +186,7 @@ class GateLinear(ReplicatedLinear):
)
return output, None
# Tier 3: experimental bf16x3 CuteDSL kernel for fp32 router weights
# Tier 3: bf16x3 CuteDSL kernel for fp32 router weights
if self.allow_bf16x3_router_gemm and x.dtype == torch.bfloat16:
from vllm.model_executor.layers.fused_moe.router.bf16x3_router_gemm_cutedsl import ( # noqa: E501
bf16x3_router_gemm,
@@ -251,10 +239,10 @@ def fp32_router_gemm_dispatch_impl(
if allow_bf16x3_router_gemm and x.dtype == torch.bfloat16:
from vllm.model_executor.layers.fused_moe.router.bf16x3_router_gemm_cutedsl import ( # noqa: E501
bf16x3_router_gemm,
_bf16x3_router_gemm,
)
return bf16x3_router_gemm(x, weight)
return _bf16x3_router_gemm(x, weight)
return torch.nn.functional.linear(x.float(), weight)