From cc4210f6719e3285fdfe4da862ae04c8dbb77ab1 Mon Sep 17 00:00:00 2001 From: Thien Tran Date: Wed, 9 Sep 2026 22:33:12 +0800 Subject: [PATCH] [Perf] Improve BF16x3 router GEMM accuracy and make it default on sm100 (#55899) Signed-off-by: Thien Tran Co-authored-by: OpenAI Codex Co-authored-by: Jee Jee Li --- .../test_bf16x3_router_gemm_cutedsl.py | 18 +- vllm/config/kernel.py | 3 - vllm/engine/arg_utils.py | 7 - .../router/bf16x3_router_gemm_cutedsl.py | 305 ++++++++++++++---- .../layers/fused_moe/router/gate_linear.py | 20 +- vllm/model_executor/warmup/kernel_warmup.py | 49 ++- 6 files changed, 314 insertions(+), 88 deletions(-) diff --git a/tests/kernels/test_bf16x3_router_gemm_cutedsl.py b/tests/kernels/test_bf16x3_router_gemm_cutedsl.py index 3c17165cad..4dee321523 100644 --- a/tests/kernels/test_bf16x3_router_gemm_cutedsl.py +++ b/tests/kernels/test_bf16x3_router_gemm_cutedsl.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests for the experimental SM100 BF16x3 router GEMM.""" +"""Tests for the SM100 BF16x3 router GEMM.""" import pytest import torch @@ -20,7 +20,16 @@ def _requires_sm100_cutedsl(): @pytest.mark.parametrize( ("num_tokens", "hidden_dim", "num_experts"), - [(48, 6144, 128), (96, 3072, 256), (129, 3072, 17)], + [ + (48, 6144, 128), + (96, 3072, 256), + (129, 3072, 17), + # long-K cases exercise the multi-accumulation path (the split-K + # heuristic leaves chains of 15 and 32 K-tiles here, above the + # kernel's num_tmem_acc bound) + (1024, 8192, 256), + (2048, 8192, 256), + ], ) def test_bf16x3_router_gemm_matches_reference( num_tokens: int, hidden_dim: int, num_experts: int @@ -36,8 +45,9 @@ def test_bf16x3_router_gemm_matches_reference( # Match the observed router weight scale w *= 0.053 out = bf16x3_router_gemm(x, w) - ref = torch.nn.functional.linear(x.float(), w) + # FP64 reference: the FP32 reference itself drifts by ~5e-6 at N=2048 + ref = torch.nn.functional.linear(x.double(), w.double()) assert out.shape == (num_tokens, num_experts) assert out.dtype == torch.float32 - assert torch.mean(torch.abs(out - ref)).item() < 5e-6 + assert torch.mean(torch.abs(out.double() - ref)).item() < 5e-6 diff --git a/vllm/config/kernel.py b/vllm/config/kernel.py index 3e597b01fc..1312cbb241 100644 --- a/vllm/config/kernel.py +++ b/vllm/config/kernel.py @@ -230,9 +230,6 @@ class KernelConfig: enable_jit_warmup: bool = True """If True, run JIT compile warmup during kernel warmup.""" - enable_bf16x3_router_gemm: bool = False - """If True, use the experimental SM100 BF16x3 CuteDSL router GEMM.""" - moe_backend: MoEBackend = "auto" """Backend for MoE expert computation kernels. Available options: diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index acdbf63b7f..2fed4f3b6a 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -695,7 +695,6 @@ class EngineArgs: enable_flashinfer_autotune: bool = get_field( KernelConfig, "enable_flashinfer_autotune" ) - enable_bf16x3_router_gemm: bool | None = None worker_cls: str = ParallelConfig.worker_cls worker_extension_cls: str = ParallelConfig.worker_extension_cls @@ -1655,10 +1654,6 @@ class EngineArgs: "--enable-flashinfer-autotune", **kernel_kwargs["enable_flashinfer_autotune"], ) - kernel_group.add_argument( - "--enable-bf16x3-router-gemm", - **kernel_kwargs["enable_bf16x3_router_gemm"], - ) moe_backend_kwargs = kernel_kwargs["moe_backend"] moe_backend_kwargs["type"] = lambda s: s.lower().replace("-", "_") kernel_group.add_argument("--moe-backend", **moe_backend_kwargs) @@ -2519,8 +2514,6 @@ class EngineArgs: "are mutually exclusive" ) kernel_config.enable_flashinfer_autotune = self.enable_flashinfer_autotune - if self.enable_bf16x3_router_gemm is not None: - kernel_config.enable_bf16x3_router_gemm = self.enable_bf16x3_router_gemm if self.moe_backend != "auto": kernel_config.moe_backend = self.moe_backend if self.linear_backend != "auto": diff --git a/vllm/model_executor/layers/fused_moe/router/bf16x3_router_gemm_cutedsl.py b/vllm/model_executor/layers/fused_moe/router/bf16x3_router_gemm_cutedsl.py index 2860c64bb3..2190eea157 100644 --- a/vllm/model_executor/layers/fused_moe/router/bf16x3_router_gemm_cutedsl.py +++ b/vllm/model_executor/layers/fused_moe/router/bf16x3_router_gemm_cutedsl.py @@ -6,6 +6,10 @@ Computes ``X @ W.T`` for BF16 ``X`` with shape ``[N, K]`` and FP32 router weights ``W`` with shape ``[M, K]`` by decomposing each FP32 weight value into three BF16 residual terms inside the kernel, then accumulating the three BF16 MMA results into FP32 TMEM output. + +The TMEM accumulation chain is bounded to ``num_tmem_acc`` K-tiles: at chunk +boundaries the epilogue warps drain the main-term accumulator into a register +master accumulator and the MMA warp resets it. """ from functools import cache @@ -22,6 +26,7 @@ from quack.compile_utils import make_fake_tensor from vllm.cute_utils import _tcgen05, simple_tma_copy from vllm.triton_utils import tl, triton from vllm.utils import math_utils +from vllm.utils.torch_utils import direct_register_custom_op __all__ = ["bf16x3_router_gemm"] @@ -82,6 +87,7 @@ class Sm100BF16x3RouterGemm: self.cta_tile = (BN, 128, 64) self.num_stages = 2 self.num_warps = 10 + self.num_tmem_acc = 8 @cute.jit def _make_tma(self, tensor: cute.Tensor, BM: int, BK: int): @@ -129,6 +135,7 @@ class Sm100BF16x3RouterGemm: BN, BM, BK = self.cta_tile num_stages = self.num_stages + num_tmem_acc = self.num_tmem_acc N, K = X_tma.tma_tensor.shape M, _ = W_tma.tma_tensor.shape @@ -151,7 +158,8 @@ class Sm100BF16x3RouterGemm: tma_full_mbar = smem.allocate_array(Int64, num_stages) tma_empty_mbar = smem.allocate_array(Int64, num_stages) w_full_mbar = smem.allocate_array(Int64, num_stages) - mma_mbar = smem.allocate_array(Int64, 1) + acc_full_mbar = smem.allocate_array(Int64, 1) + acc_empty_mbar = smem.allocate_array(Int64, 1) taddr = smem.allocate(Int32, 4) BAR_TMEM_ALLOC = 1 @@ -171,7 +179,8 @@ class Sm100BF16x3RouterGemm: cute.arch.mbarrier_init(tma_full_mbar + i, 1) cute.arch.mbarrier_init(tma_empty_mbar + i, 1) cute.arch.mbarrier_init(w_full_mbar + i, 128) - cute.arch.mbarrier_init(mma_mbar, 1) + cute.arch.mbarrier_init(acc_full_mbar, 1) + cute.arch.mbarrier_init(acc_empty_mbar, 128) cute.arch.mbarrier_init_fence() elif warp_id == 1: cpasync.prefetch_descriptor(X_tma.atom) @@ -213,40 +222,52 @@ class Sm100BF16x3RouterGemm: elif warp_id == 8: # MMA warp - stage_id = 0 - parity = 0 + tma_stage_id = 0 + tma_parity = 0 + + tmem_acc_count = 0 + tmem_parity = 1 idesc = _tcgen05.make_bf16_idesc(BM, BN) sdesc = _tcgen05.make_sdesc_128B_swizzle(0) for tile_k in cutlass.range(bid_k, k_tiles, split_k, unroll=1): - cute.arch.mbarrier_wait(tma_full_mbar + stage_id, parity) - cute.arch.mbarrier_wait(w_full_mbar + stage_id, parity) + if tmem_acc_count == 0: + cute.arch.mbarrier_wait(acc_empty_mbar, tmem_parity) + cute.arch.mbarrier_wait(tma_full_mbar + tma_stage_id, tma_parity) + cute.arch.mbarrier_wait(w_full_mbar + tma_stage_id, tma_parity) _tcgen05.fence_after_thread_sync() - w_tmem = w_tmem_base + stage_id * (BK // 2 * 3) - x_desc = sdesc | (sX[None, None, stage_id].iterator.toint() >> 4) + w_tmem = w_tmem_base + tma_stage_id * (BK // 2 * 3) + x_desc = sdesc | (sX[None, None, tma_stage_id].iterator.toint() >> 4) for k in cutlass.range_constexpr(BK // 16): - enable_d = (tile_k > bid_k) or (k > 0) + enable_d_main = (tmem_acc_count > 0) or (k > 0) + enable_d_res = (tile_k > bid_k) or (k > 0) _tcgen05.mma_ts_f16( - acc_main, w_tmem + k * 8, x_desc, idesc, enable_d + acc_main, w_tmem + k * 8, x_desc, idesc, enable_d_main ) _tcgen05.mma_ts_f16( - acc_res, w_tmem + 32 + k * 8, x_desc, idesc, enable_d + acc_res, w_tmem + 32 + k * 8, x_desc, idesc, enable_d_res ) _tcgen05.mma_ts_f16( acc_res, w_tmem + 64 + k * 8, x_desc, idesc, True ) x_desc += 32 >> 4 - _tcgen05.commit(tma_empty_mbar + stage_id) + _tcgen05.commit(tma_empty_mbar + tma_stage_id) - stage_id = (stage_id + 1) % self.num_stages - if stage_id == 0: - parity ^= 1 + tma_stage_id = (tma_stage_id + 1) % num_stages + if tma_stage_id == 0: + tma_parity ^= 1 - _tcgen05.commit(mma_mbar) + tmem_acc_count = (tmem_acc_count + 1) % num_tmem_acc + if tmem_acc_count == 0: + _tcgen05.commit(acc_full_mbar) + tmem_parity ^= 1 + + if tmem_acc_count != 0: + _tcgen05.commit(acc_full_mbar) elif warp_id >= 4: # prep warps: decompose FP32 W into 3xBF16 @@ -303,39 +324,93 @@ class Sm100BF16x3RouterGemm: _tcgen05.alloc(taddr) cute.arch.barrier(barrier_id=BAR_TMEM_ALLOC, number_of_threads=128) - if warp_id == 0: - cute.arch.mbarrier_wait(mma_mbar, 0) - cute.arch.barrier(barrier_id=BAR_EPI, number_of_threads=128) - _tcgen05.fence_after_thread_sync() - - cute.arch.griddepcontrol_launch_dependents() + tiles_local = cute.ceil_div(k_tiles - bid_k, split_k) + num_chunks = cute.ceil_div(tiles_local, num_tmem_acc) WIDTH = 8 - for i in cutlass.range_constexpr(BN // WIDTH): - tcol = i * WIDTH - main_regs = cute.make_rmem_tensor(WIDTH, Float32) - res_regs = cute.make_rmem_tensor(WIDTH, Float32) - main_regs.store(_tcgen05.ld(warp_id * 32, tcol, "32x32b", WIDTH)) - res_regs.store(_tcgen05.ld(warp_id * 32, BN + tcol, "32x32b", WIDTH)) - _tcgen05.wait_ld() - - # CuteDSL will codegen add.f32x2 - for j in cutlass.range(WIDTH, vectorize=True): - main_regs[j] += res_regs[j] + main_regs = cute.make_rmem_tensor(WIDTH, Float32) + res_regs = cute.make_rmem_tensor(WIDTH, Float32) + if num_chunks == 1: + # single chunk + cute.arch.mbarrier_wait(acc_full_mbar, 0) + _tcgen05.fence_after_thread_sync() w_row_idx = bid_m * BM + tid - for j in cutlass.range_constexpr(WIDTH): - x_row_idx = bid_n * BN + i * WIDTH + j - if x_row_idx < N and w_row_idx < M: - out[bid_k, x_row_idx, w_row_idx] = main_regs[j] + for i in cutlass.range_constexpr(BN // WIDTH): + tcol = i * WIDTH + main_regs.store(_tcgen05.ld(warp_id * 32, tcol, "32x32b", WIDTH)) + res_regs.store( + _tcgen05.ld(warp_id * 32, BN + tcol, "32x32b", WIDTH) + ) + _tcgen05.wait_ld() + # CuteDSL will codegen add.f32x2 + for j in cutlass.range(WIDTH, vectorize=True): + main_regs[j] += res_regs[j] + + for j in cutlass.range_constexpr(WIDTH): + x_row_idx = bid_n * BN + i * WIDTH + j + if x_row_idx < N and w_row_idx < M: + out[bid_k, x_row_idx, w_row_idx] = main_regs[j] + + else: + # multiple chunks + master_acc = cute.make_rmem_tensor(BN, Float32) + + # chunk 0: pure TMEM load, no accumulation. + cute.arch.mbarrier_wait(acc_full_mbar, 0) + _tcgen05.fence_after_thread_sync() + master_acc.store(_tcgen05.ld(warp_id * 32, 0, "32x32b", BN)) + _tcgen05.wait_ld() + _tcgen05.fence_before_thread_sync() + cute.arch.mbarrier_arrive(acc_empty_mbar) + + # accumulate main tmem acc + for chunk in cutlass.range(1, num_chunks, unroll=1): + cute.arch.mbarrier_wait(acc_full_mbar, chunk & 1) + _tcgen05.fence_after_thread_sync() + + for i in cutlass.range_constexpr(BN // WIDTH): + tcol = i * WIDTH + main_regs.store( + _tcgen05.ld(warp_id * 32, tcol, "32x32b", WIDTH) + ) + _tcgen05.wait_ld() + # CuteDSL refuses to compile + # master_acc[i * WIDTH + j] += main_regs[j] + # when vectorize=True + master_view = cute.local_tile(master_acc, (WIDTH,), (i,)) + for j in cutlass.range(WIDTH, vectorize=True): + master_view[j] += main_regs[j] + + _tcgen05.fence_before_thread_sync() + cute.arch.mbarrier_arrive(acc_empty_mbar) + + # fold in the residual accumulator + for i in cutlass.range_constexpr(BN // WIDTH): + tcol = i * WIDTH + res_regs.store( + _tcgen05.ld(warp_id * 32, BN + tcol, "32x32b", WIDTH) + ) + _tcgen05.wait_ld() + master_view = cute.local_tile(master_acc, (WIDTH,), (i,)) + for j in cutlass.range(WIDTH, vectorize=True): + master_view[j] += res_regs[j] + + w_row_idx = bid_m * BM + tid + for j in cutlass.range_constexpr(WIDTH): + x_row_idx = bid_n * BN + i * WIDTH + j + if x_row_idx < N and w_row_idx < M: + out[bid_k, x_row_idx, w_row_idx] = master_acc[i * WIDTH + j] + + cute.arch.griddepcontrol_launch_dependents() cute.arch.barrier(barrier_id=BAR_EPI, number_of_threads=128) if warp_id == 0: _tcgen05.dealloc() @cache @staticmethod - def compile(BN: int = 128, K: int = 6144): + def compile(K: int, BN: int = 128): N = cute.sym_int() M = cute.sym_int() SPLIT_K = cute.sym_int() @@ -392,16 +467,19 @@ def _splitk_reduce_kernel( ) +def _splitk_reduce_config(split_k: int) -> tuple[int, int, int]: + block_s = triton.next_power_of_2(split_k) + if block_s >= 64: + return 1, 32, block_s + if block_s >= 8: + return 1, 256, block_s + return min(16, 32 // block_s), 32, block_s + + def splitk_reduce_triton(partials: torch.Tensor, out: torch.Tensor): split_k, N, M = partials.shape - block_s = 1 << (split_k - 1).bit_length() split_stride = partials.stride(0) - if block_s >= 64: - BN, BM = 1, 32 - elif block_s >= 8: - BN, BM = 1, 256 - else: - BN, BM = min(16, 32 // block_s), 32 + BN, BM, block_s = _splitk_reduce_config(split_k) grid = (triton.cdiv(N, BN), triton.cdiv(M, BM)) _splitk_reduce_kernel[grid]( partials, @@ -419,27 +497,124 @@ def splitk_reduce_triton(partials: torch.Tensor, out: torch.Tensor): ) -def bf16x3_router_gemm(X: torch.Tensor, W: torch.Tensor) -> torch.Tensor: - """Return ``X @ W.T`` using the SM100 BF16x3 router GEMM kernel.""" - N, K = X.shape - M, _ = W.shape - num_sms = torch.cuda.get_device_properties(X.device).multi_processor_count +# Accuracy-constrained (BN, split_k) configs from GB300 sweeps. +_TILE_CONFIG_OVERRIDES = { + (2816, 256): { # Hunyuan-V4 + 64: (16, 16), + 128: (32, 16), + 256: (64, 16), + 512: (128, 16), + }, + (3072, 256): { # MiniMax-M2 + 64: (16, 16), + 128: (32, 16), + 256: (64, 16), + 512: (128, 16), + }, + (4096, 192): { # Hunyuan-V3 + 64: (32, 32), + 128: (64, 32), + 256: (128, 32), + 512: (128, 19), + }, + (6144, 128): { # MiniMax-M3 + 64: (32, 64), + 128: (64, 64), + 256: (64, 32), + 512: (128, 32), + }, +} + + +def _pick_tile_config(N: int, K: int, M: int, num_sms: int) -> tuple[int, int]: + """Return (BN, split_k): tuned table mid-range, generic rule otherwise.""" + k_tiles = math_utils.cdiv(K, 64) + + if 32 < N <= 512: + token_bucket = triton.next_power_of_2(N) + overrides = _TILE_CONFIG_OVERRIDES.get((K, M)) + if overrides is not None: + BN, split_k = overrides[token_bucket] + return BN, min(split_k, k_tiles) # next power of 2 within 8 and 128 BN = triton.next_power_of_2(N) BN = min(max(BN, 8), 128) - BM = 128 - BK = 64 - k_tiles = math_utils.cdiv(K, BK) - grid_m = math_utils.cdiv(M, BM) + grid_m = math_utils.cdiv(M, 128) grid_n = math_utils.cdiv(N, BN) - base_ctas = grid_m * grid_n split_k = min(k_tiles, max(1, num_sms // base_ctas)) + return BN, split_k + + +def warmup_bf16x3_router_gemm( + K: int, + M: int, + min_num_tokens: int, + max_num_tokens: int, +) -> tuple[int, ...]: + """Compile every BF16x3 router GEMM configuration reachable at runtime.""" + from vllm.model_executor.warmup.jit_warmup_triton_helper import ( + TritonWarmupTensor, + triton_scalar_specialization_rep, + ) + + device = torch.accelerator.current_device_index() + num_sms = torch.cuda.get_device_properties(device).multi_processor_count + gemm_configs: set[int] = set() + reduce_configs: set[tuple[int, int, int, int, int, int]] = set() + for N in range(min_num_tokens, max_num_tokens + 1): + BN, split_k = _pick_tile_config(N, K, M, num_sms) + gemm_configs.add(BN) + if split_k == 1: + continue + reduce_BN, reduce_BM, block_s = _splitk_reduce_config(split_k) + reduce_configs.add( + ( + triton_scalar_specialization_rep(N), + triton_scalar_specialization_rep(N * M), + triton_scalar_specialization_rep(split_k), + reduce_BN, + reduce_BM, + block_s, + ) + ) + + configs = tuple(sorted(gemm_configs)) + for BN in configs: + Sm100BF16x3RouterGemm.compile(K, BN) + for N, split_stride, split_k, BN, BM, block_s in sorted(reduce_configs): + partials = TritonWarmupTensor(torch.float32) + out = TritonWarmupTensor(torch.float32) + _splitk_reduce_kernel.warmup( + partials, + out, + N, + M, + split_stride, + split_k, + BN=BN, + BM=BM, + BS=block_s, + USE_PDL=True, + num_warps=4, + launch_pdl=True, + grid=(1, 1), + ) + return configs + + +def _bf16x3_router_gemm(X: torch.Tensor, W: torch.Tensor) -> torch.Tensor: + """Return ``X @ W.T`` using the SM100 BF16x3 router GEMM kernel.""" + N, K = X.shape + M, _ = W.shape + num_sms = torch.cuda.get_device_properties(X.device).multi_processor_count + + BN, split_k = _pick_tile_config(N, K, M, num_sms) partials = X.new_empty(split_k, N, M, dtype=torch.float32) - Sm100BF16x3RouterGemm.compile(BN, K)(X, W, partials, split_k) + Sm100BF16x3RouterGemm.compile(K, BN)(X, W, partials, split_k) if split_k == 1: return partials.squeeze(0) @@ -447,3 +622,21 @@ def bf16x3_router_gemm(X: torch.Tensor, W: torch.Tensor) -> torch.Tensor: out = X.new_empty(N, M, dtype=torch.float32) splitk_reduce_triton(partials, out) return out + + +def _bf16x3_router_gemm_fake( + X: torch.Tensor, + W: torch.Tensor, +) -> torch.Tensor: + return X.new_empty((X.shape[0], W.shape[0]), dtype=torch.float32) + + +direct_register_custom_op( + op_name="bf16x3_router_gemm", + op_func=_bf16x3_router_gemm, + fake_impl=_bf16x3_router_gemm_fake, +) + + +def bf16x3_router_gemm(X: torch.Tensor, W: torch.Tensor) -> torch.Tensor: + return torch.ops.vllm.bf16x3_router_gemm(X, W) diff --git a/vllm/model_executor/layers/fused_moe/router/gate_linear.py b/vllm/model_executor/layers/fused_moe/router/gate_linear.py index 0f45aaf6fb..1f086b4488 100644 --- a/vllm/model_executor/layers/fused_moe/router/gate_linear.py +++ b/vllm/model_executor/layers/fused_moe/router/gate_linear.py @@ -4,15 +4,11 @@ import torch from torch.nn.parameter import Parameter import vllm._custom_ops as ops -from vllm.config import get_current_vllm_config_or_none -from vllm.logger import init_logger from vllm.model_executor.custom_op import PluggableLayer from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.platforms import current_platform from vllm.utils.torch_utils import direct_register_custom_op -logger = init_logger(__name__) - @PluggableLayer.register("gate_linear") class GateLinear(ReplicatedLinear): @@ -22,7 +18,7 @@ class GateLinear(ReplicatedLinear): K divisible by 8) 2. fp32 specialized kernel (SM90+ or gfx950, bf16/fp32 in, fp32 out, M<=32, model-specific shapes) - 3. experimental bf16x3 CuteDSL kernel (opt-in, SM100, bf16 in, fp32 weight) + 3. bf16x3 CuteDSL kernel (SM100, bf16 in, fp32 weight) 4. cuBLAS bf16×bf16→fp32 (SM90+ + bf16 weight + fp32 out_dtype) 5. F.linear via ReplicatedLinear (ultimate fallback) @@ -85,11 +81,6 @@ class GateLinear(ReplicatedLinear): self.allow_specialized_router_gemm = can_use_specialized_kernels # fp32 specialized kernel eligibility (exact dims, fp32 weight) - vllm_config = get_current_vllm_config_or_none() - enable_bf16x3_router_gemm = ( - vllm_config is not None - and vllm_config.kernel_config.enable_bf16x3_router_gemm - ) self.allow_fp32_router_gemm = ( not bias and self.weight.dtype == torch.float32 @@ -108,10 +99,7 @@ class GateLinear(ReplicatedLinear): and current_platform.is_cuda() and is_blackwell and input_size % 8 == 0 - and enable_bf16x3_router_gemm ) - if self.allow_bf16x3_router_gemm: - logger.info_once("Enabled experimental SM100 BF16x3 router GEMM.") # Fused bf16 x bf16 -> fp32 GEMM eligibility. torch.mm's out_dtype # epilogue folds the fp32 cast into the GEMM, removing the standalone @@ -198,7 +186,7 @@ class GateLinear(ReplicatedLinear): ) return output, None - # Tier 3: experimental bf16x3 CuteDSL kernel for fp32 router weights + # Tier 3: bf16x3 CuteDSL kernel for fp32 router weights if self.allow_bf16x3_router_gemm and x.dtype == torch.bfloat16: from vllm.model_executor.layers.fused_moe.router.bf16x3_router_gemm_cutedsl import ( # noqa: E501 bf16x3_router_gemm, @@ -251,10 +239,10 @@ def fp32_router_gemm_dispatch_impl( if allow_bf16x3_router_gemm and x.dtype == torch.bfloat16: from vllm.model_executor.layers.fused_moe.router.bf16x3_router_gemm_cutedsl import ( # noqa: E501 - bf16x3_router_gemm, + _bf16x3_router_gemm, ) - return bf16x3_router_gemm(x, weight) + return _bf16x3_router_gemm(x, weight) return torch.nn.functional.linear(x.float(), weight) diff --git a/vllm/model_executor/warmup/kernel_warmup.py b/vllm/model_executor/warmup/kernel_warmup.py index 7158e23456..c9c740f368 100644 --- a/vllm/model_executor/warmup/kernel_warmup.py +++ b/vllm/model_executor/warmup/kernel_warmup.py @@ -101,6 +101,44 @@ def _warmup_ll_bf16_router_gemm(model: torch.nn.Module) -> None: ) +def _warmup_bf16x3_router_gemm( + model: torch.nn.Module, + max_num_tokens: int, +) -> None: + from vllm.model_executor.layers.fused_moe.router.bf16x3_router_gemm_cutedsl import ( # noqa: E501 + warmup_bf16x3_router_gemm, + ) + from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear + + gate = next( + ( + module + for module in model.modules() + if isinstance(module, GateLinear) and module.allow_bf16x3_router_gemm + ), + None, + ) + if gate is None: + logger.debug_once( + "Skipping BF16x3 router GEMM warmup: no eligible GateLinear found." + ) + return + + min_num_tokens = gate.FP32_MAX_TOKENS + 1 if gate.allow_fp32_router_gemm else 1 + logger.info_once( + "Warming up BF16x3 router GEMM for K=%d, M=%d.", + gate.input_size, + gate.output_size, + ) + configs = warmup_bf16x3_router_gemm( + gate.input_size, + gate.output_size, + min_num_tokens, + max_num_tokens, + ) + logger.info_once("Warmed up BF16x3 router GEMM configs: %s.", configs) + + def _warmup_kimi_k3_gemm_rs_ar() -> None: # Kimi-K3 model construction imports this module only when GEMM-RS/AR is # enabled and initializes its singleton before kernel_warmup runs. Avoid @@ -131,7 +169,8 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False): if zeroer is not None: zeroer.warmup(worker.model_runner.kv_cache_config.num_blocks) - if worker.vllm_config.kernel_config.enable_jit_warmup: + enable_jit_warmup = worker.vllm_config.kernel_config.enable_jit_warmup + if enable_jit_warmup: logger.info("JIT kernel warmup starting.") jit_warmup_start = time.perf_counter() try: @@ -164,11 +203,17 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False): ) # Run next so input-prep kernels JIT against pristine runner state. - if worker.vllm_config.kernel_config.enable_jit_warmup: + if enable_jit_warmup: kimi_k3_triton_warmup(worker) spec_decode_rejection_warmup(worker) qwen4_exp_qsa_triton_warmup(worker) + if enable_jit_warmup and current_platform.is_device_capability_family(100): + _warmup_bf16x3_router_gemm( + worker.get_model(), + worker.scheduler_config.max_num_batched_tokens, + ) + if current_platform.has_device_capability(90): _warmup_ll_bf16_router_gemm(worker.get_model())