diff --git a/tests/kernels/test_bf16x3_router_gemm_cutedsl.py b/tests/kernels/test_bf16x3_router_gemm_cutedsl.py new file mode 100644 index 0000000000..3c17165cad --- /dev/null +++ b/tests/kernels/test_bf16x3_router_gemm_cutedsl.py @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for the experimental SM100 BF16x3 router GEMM.""" + +import pytest +import torch + +from vllm.utils.import_utils import has_cutedsl + + +def _requires_sm100_cutedsl(): + if not torch.cuda.is_available(): + pytest.skip("CUDA not available") + major, _ = torch.cuda.get_device_capability() + if major != 10: + pytest.skip("bf16x3 router GEMM requires SM100-class GPU") + if not has_cutedsl(): + pytest.skip("cutedsl (cutlass) not installed") + + +@pytest.mark.parametrize( + ("num_tokens", "hidden_dim", "num_experts"), + [(48, 6144, 128), (96, 3072, 256), (129, 3072, 17)], +) +def test_bf16x3_router_gemm_matches_reference( + num_tokens: int, hidden_dim: int, num_experts: int +): + _requires_sm100_cutedsl() + from vllm.model_executor.layers.fused_moe.router.bf16x3_router_gemm_cutedsl import ( # noqa: E501 + bf16x3_router_gemm, + ) + + torch.manual_seed(42) + x = torch.randn(num_tokens, hidden_dim, dtype=torch.bfloat16, device="cuda") + w = torch.randn(num_experts, hidden_dim, dtype=torch.float32, device="cuda") + # Match the observed router weight scale + w *= 0.053 + out = bf16x3_router_gemm(x, w) + ref = torch.nn.functional.linear(x.float(), w) + + assert out.shape == (num_tokens, num_experts) + assert out.dtype == torch.float32 + assert torch.mean(torch.abs(out - ref)).item() < 5e-6 diff --git a/vllm/config/kernel.py b/vllm/config/kernel.py index aff94fe03d..f13562f59c 100644 --- a/vllm/config/kernel.py +++ b/vllm/config/kernel.py @@ -178,6 +178,9 @@ class KernelConfig: enable_cutedsl_warmup: bool = True """If True, run CuTeDSL compile warmup during kernel warmup.""" + enable_bf16x3_router_gemm: bool = False + """If True, use the experimental SM100 BF16x3 CuteDSL router GEMM.""" + moe_backend: MoEBackend = "auto" """Backend for MoE expert computation kernels. Available options: diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 07877b0812..366f6eca68 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -665,6 +665,7 @@ class EngineArgs: enable_flashinfer_autotune: bool = get_field( KernelConfig, "enable_flashinfer_autotune" ) + enable_bf16x3_router_gemm: bool | None = None worker_cls: str = ParallelConfig.worker_cls worker_extension_cls: str = ParallelConfig.worker_extension_cls @@ -1508,6 +1509,10 @@ class EngineArgs: "--enable-flashinfer-autotune", **kernel_kwargs["enable_flashinfer_autotune"], ) + kernel_group.add_argument( + "--enable-bf16x3-router-gemm", + **kernel_kwargs["enable_bf16x3_router_gemm"], + ) moe_backend_kwargs = kernel_kwargs["moe_backend"] moe_backend_kwargs["type"] = lambda s: s.lower().replace("-", "_") kernel_group.add_argument("--moe-backend", **moe_backend_kwargs) @@ -2286,6 +2291,8 @@ class EngineArgs: "are mutually exclusive" ) kernel_config.enable_flashinfer_autotune = self.enable_flashinfer_autotune + if self.enable_bf16x3_router_gemm is not None: + kernel_config.enable_bf16x3_router_gemm = self.enable_bf16x3_router_gemm if self.moe_backend != "auto": kernel_config.moe_backend = self.moe_backend if self.linear_backend != "auto": diff --git a/vllm/model_executor/layers/fused_moe/router/bf16x3_router_gemm_cutedsl.py b/vllm/model_executor/layers/fused_moe/router/bf16x3_router_gemm_cutedsl.py new file mode 100644 index 0000000000..2860c64bb3 --- /dev/null +++ b/vllm/model_executor/layers/fused_moe/router/bf16x3_router_gemm_cutedsl.py @@ -0,0 +1,449 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""CuteDSL BF16x3 router GEMM. + +Computes ``X @ W.T`` for BF16 ``X`` with shape ``[N, K]`` and FP32 router +weights ``W`` with shape ``[M, K]`` by decomposing each FP32 weight value into +three BF16 residual terms inside the kernel, then accumulating the three BF16 +MMA results into FP32 TMEM output. +""" + +from functools import cache + +import cutlass +import torch +from cuda.bindings.driver import CUstream +from cutlass import BFloat16, Float32, Int32, Int64, Uint32, cute +from cutlass._mlir.dialects import llvm +from cutlass.cute.nvgpu import cpasync +from cutlass.cutlass_dsl import T, dsl_user_op +from quack.compile_utils import make_fake_tensor + +from vllm.cute_utils import _tcgen05, simple_tma_copy +from vllm.triton_utils import tl, triton +from vllm.utils import math_utils + +__all__ = ["bf16x3_router_gemm"] + + +@dsl_user_op +def _decompose_fp32x2_to_3xbf16x2( + w0: Float32, + w1: Float32, + *, + loc=None, + ip=None, +) -> tuple[Uint32, Uint32, Uint32]: + # this PTX snippets does the following + # out0 = BF16(in); res = in - FP32(out0) + # out1 = BF16(res); res = res - FP32(out1) + # out2 = BF16(res) + # + # for normal FP32, this decomposition is exact + # i.e. in = FP32(out0) + FP32(out1) + FP32(out2) + # + out = llvm.inline_asm( + llvm.StructType.get_literal([T.i32(), T.i32(), T.i32()]), + [w0.ir_value(loc=loc, ip=ip), w1.ir_value(loc=loc, ip=ip)], + "{\n\t" + ".reg .b32 r1_lo, r1_hi, r2_lo, r2_hi;\n\t" + ".reg .b32 w0_lo, w0_hi, w1_lo, w1_hi;\n\t" + ".reg .b64 a_pair, w0_pair, w1_pair, r1_pair, r2_pair;\n\t" + "cvt.rn.bf16x2.f32 $0, $4, $3;\n\t" + "shl.b32 w0_lo, $0, 16;\n\t" + "and.b32 w0_hi, $0, 0xffff0000;\n\t" + "mov.b64 a_pair, {$3, $4};\n\t" + "mov.b64 w0_pair, {w0_lo, w0_hi};\n\t" + "sub.rn.f32x2 r1_pair, a_pair, w0_pair;\n\t" + "mov.b64 {r1_lo, r1_hi}, r1_pair;\n\t" + "cvt.rn.bf16x2.f32 $1, r1_hi, r1_lo;\n\t" + "shl.b32 w1_lo, $1, 16;\n\t" + "and.b32 w1_hi, $1, 0xffff0000;\n\t" + "mov.b64 w1_pair, {w1_lo, w1_hi};\n\t" + "sub.rn.f32x2 r2_pair, r1_pair, w1_pair;\n\t" + "mov.b64 {r2_lo, r2_hi}, r2_pair;\n\t" + "cvt.rn.bf16x2.f32 $2, r2_hi, r2_lo;\n\t" + "}\n", + "=r,=r,=r,f,f", + has_side_effects=False, + is_align_stack=False, + loc=loc, + ip=ip, + ) + return ( + Uint32(llvm.extractvalue(T.i32(), out, [0], loc=loc, ip=ip)), + Uint32(llvm.extractvalue(T.i32(), out, [1], loc=loc, ip=ip)), + Uint32(llvm.extractvalue(T.i32(), out, [2], loc=loc, ip=ip)), + ) + + +class Sm100BF16x3RouterGemm: + def __init__(self, BN: int = 128) -> None: + self.cta_tile = (BN, 128, 64) + self.num_stages = 2 + self.num_warps = 10 + + @cute.jit + def _make_tma(self, tensor: cute.Tensor, BM: int, BK: int): + op = cpasync.CopyBulkTensorTileG2SOp() + swizzle_128B = cute.make_swizzle(3, 4, 3) + elems = 128 * 8 // tensor.element_type.width # 128B + slayout = cute.make_layout( + (BM, (elems, BK // elems), self.num_stages), + stride=(elems, (1, BM * elems), BM * BK), + ) + slayout = cute.make_composed_layout(swizzle_128B, 0, slayout) + return cpasync.make_tiled_tma_atom(op, tensor, slayout, (BM, BK)) + + @cute.jit + def __call__( + self, + X: cute.Tensor, + W: cute.Tensor, + out: cute.Tensor, + split_k: Int32, + stream: CUstream, + ): + BN, BM, BK = self.cta_tile + W_tma = self._make_tma(W, BM, BK) + X_tma = self._make_tma(X, BN, BK) + + grid_m = cute.ceil_div(W.shape[0], BM) + grid_n = cute.ceil_div(X.shape[0], BN) + + self.kernel(X_tma, W_tma, out).launch( + grid=(grid_m, grid_n, split_k), + block=(self.num_warps * 32, 1, 1), + stream=stream, + use_pdl=True, + ) + + @cute.kernel + def kernel(self, X_tma: cpasync.TmaInfo, W_tma: cpasync.TmaInfo, out: cute.Tensor): + tid, _, _ = cute.arch.thread_idx() + bid_m, bid_n, bid_k = cute.arch.block_idx() + _, _, split_k = cute.arch.grid_dim() + + warp_id = cute.arch.make_warp_uniform(cute.arch.warp_idx()) + lane_id = cute.arch.lane_idx() + + BN, BM, BK = self.cta_tile + num_stages = self.num_stages + + N, K = X_tma.tma_tensor.shape + M, _ = W_tma.tma_tensor.shape + k_tiles = cute.ceil_div(K, BK) + + smem = cutlass.utils.SmemAllocator() + sX = smem.allocate_tensor( + BFloat16, + X_tma.smem_layout.outer, + byte_alignment=128, + swizzle=X_tma.smem_layout.inner, + ) + sW = smem.allocate_tensor( + Float32, + W_tma.smem_layout.outer, + byte_alignment=128, + swizzle=W_tma.smem_layout.inner, + ) + + tma_full_mbar = smem.allocate_array(Int64, num_stages) + tma_empty_mbar = smem.allocate_array(Int64, num_stages) + w_full_mbar = smem.allocate_array(Int64, num_stages) + mma_mbar = smem.allocate_array(Int64, 1) + taddr = smem.allocate(Int32, 4) + + BAR_TMEM_ALLOC = 1 + BAR_PREP = 2 + BAR_EPI = 3 + + # tmem "allocation" + # acc_main is for the 1st BF16 term. acc_res is for the 2nd and 3rd + # BF16 terms, which are much smaller than the first term. + acc_main = 0 + acc_res = BN + w_tmem_base = BN * 2 + + if warp_id == 0: + with cute.arch.elect_one(): + for i in cutlass.range_constexpr(num_stages): + cute.arch.mbarrier_init(tma_full_mbar + i, 1) + cute.arch.mbarrier_init(tma_empty_mbar + i, 1) + cute.arch.mbarrier_init(w_full_mbar + i, 128) + cute.arch.mbarrier_init(mma_mbar, 1) + cute.arch.mbarrier_init_fence() + elif warp_id == 1: + cpasync.prefetch_descriptor(X_tma.atom) + cpasync.prefetch_descriptor(W_tma.atom) + cute.arch.sync_threads() + cute.arch.griddepcontrol_wait() + + if warp_id == 9: + # TMA warp + stage_id = 0 + parity = 1 + + # (BM, BK, K/BK) + gW_tiles = cute.local_tile(W_tma.tma_tensor, (BM, BK), (bid_m, None)) + gX_tiles = cute.local_tile(X_tma.tma_tensor, (BN, BK), (bid_n, None)) + + for tile_k in cutlass.range(bid_k, k_tiles, split_k, unroll=1): + cute.arch.mbarrier_wait(tma_empty_mbar + stage_id, parity) + mbar = tma_full_mbar + stage_id + with cute.arch.elect_one(): + stage_bytes = BN * BK * 2 + BM * BK * 4 + cute.arch.mbarrier_arrive_and_expect_tx(mbar, stage_bytes) + simple_tma_copy( + W_tma.atom, + gW_tiles[None, None, tile_k], + sW[None, None, stage_id], + mbar, + ) + simple_tma_copy( + X_tma.atom, + gX_tiles[None, None, tile_k], + sX[None, None, stage_id], + mbar, + ) + + stage_id = (stage_id + 1) % num_stages + if stage_id == 0: + parity ^= 1 + + elif warp_id == 8: + # MMA warp + stage_id = 0 + parity = 0 + + idesc = _tcgen05.make_bf16_idesc(BM, BN) + sdesc = _tcgen05.make_sdesc_128B_swizzle(0) + + for tile_k in cutlass.range(bid_k, k_tiles, split_k, unroll=1): + cute.arch.mbarrier_wait(tma_full_mbar + stage_id, parity) + cute.arch.mbarrier_wait(w_full_mbar + stage_id, parity) + _tcgen05.fence_after_thread_sync() + + w_tmem = w_tmem_base + stage_id * (BK // 2 * 3) + x_desc = sdesc | (sX[None, None, stage_id].iterator.toint() >> 4) + + for k in cutlass.range_constexpr(BK // 16): + enable_d = (tile_k > bid_k) or (k > 0) + _tcgen05.mma_ts_f16( + acc_main, w_tmem + k * 8, x_desc, idesc, enable_d + ) + _tcgen05.mma_ts_f16( + acc_res, w_tmem + 32 + k * 8, x_desc, idesc, enable_d + ) + _tcgen05.mma_ts_f16( + acc_res, w_tmem + 64 + k * 8, x_desc, idesc, True + ) + x_desc += 32 >> 4 + + _tcgen05.commit(tma_empty_mbar + stage_id) + + stage_id = (stage_id + 1) % self.num_stages + if stage_id == 0: + parity ^= 1 + + _tcgen05.commit(mma_mbar) + + elif warp_id >= 4: + # prep warps: decompose FP32 W into 3xBF16 + warp_id_ = warp_id % 4 + + stage_id = 0 + parity = 0 + + # ld.shared.v4.f32 + op = cute.nvgpu.CopyUniversalOp() + cp_atom = cute.make_copy_atom(op, Float32, num_bits_per_copy=128) + + # sW_view: ((4, 1), (BK/4, num_stages)) + row = warp_id_ * 32 + lane_id + sW_view = cute.zipped_divide(sW[row, None, None], (4, 1)) + + for _ in cutlass.range(bid_k, k_tiles, split_k, unroll=1): + if warp_id_ == 0: + cute.arch.mbarrier_wait(tma_full_mbar + stage_id, parity) + cute.arch.barrier(barrier_id=BAR_PREP, number_of_threads=128) + + row = warp_id_ * 32 + lane_id + w_tmem = w_tmem_base + stage_id * (BK // 2 * 3) + for kblock in cutlass.range_constexpr(BK // 4): + w0 = cute.make_rmem_tensor(2, Uint32) + w1 = cute.make_rmem_tensor(2, Uint32) + w2 = cute.make_rmem_tensor(2, Uint32) + + w_tmp = cute.make_rmem_tensor(4, Float32) + cute.copy(cp_atom, sW_view[None, (kblock, stage_id)], w_tmp) + w0[0], w1[0], w2[0] = _decompose_fp32x2_to_3xbf16x2( + w_tmp[0], w_tmp[1] + ) + w0[1], w1[1], w2[1] = _decompose_fp32x2_to_3xbf16x2( + w_tmp[2], w_tmp[3] + ) + + tcol = kblock * 2 + _tcgen05.st(warp_id_ * 32, w_tmem + 0 + tcol, "32x32b", 2, w0) + _tcgen05.st(warp_id_ * 32, w_tmem + 32 + tcol, "32x32b", 2, w1) + _tcgen05.st(warp_id_ * 32, w_tmem + 64 + tcol, "32x32b", 2, w2) + + _tcgen05.wait_st() + _tcgen05.fence_before_thread_sync() + cute.arch.mbarrier_arrive(w_full_mbar + stage_id) + + stage_id = (stage_id + 1) % self.num_stages + if stage_id == 0: + parity ^= 1 + + else: + # epilogue warps + if warp_id == 0: + _tcgen05.alloc(taddr) + cute.arch.barrier(barrier_id=BAR_TMEM_ALLOC, number_of_threads=128) + + if warp_id == 0: + cute.arch.mbarrier_wait(mma_mbar, 0) + cute.arch.barrier(barrier_id=BAR_EPI, number_of_threads=128) + _tcgen05.fence_after_thread_sync() + + cute.arch.griddepcontrol_launch_dependents() + + WIDTH = 8 + for i in cutlass.range_constexpr(BN // WIDTH): + tcol = i * WIDTH + main_regs = cute.make_rmem_tensor(WIDTH, Float32) + res_regs = cute.make_rmem_tensor(WIDTH, Float32) + main_regs.store(_tcgen05.ld(warp_id * 32, tcol, "32x32b", WIDTH)) + res_regs.store(_tcgen05.ld(warp_id * 32, BN + tcol, "32x32b", WIDTH)) + _tcgen05.wait_ld() + + # CuteDSL will codegen add.f32x2 + for j in cutlass.range(WIDTH, vectorize=True): + main_regs[j] += res_regs[j] + + w_row_idx = bid_m * BM + tid + for j in cutlass.range_constexpr(WIDTH): + x_row_idx = bid_n * BN + i * WIDTH + j + if x_row_idx < N and w_row_idx < M: + out[bid_k, x_row_idx, w_row_idx] = main_regs[j] + + cute.arch.barrier(barrier_id=BAR_EPI, number_of_threads=128) + if warp_id == 0: + _tcgen05.dealloc() + + @cache + @staticmethod + def compile(BN: int = 128, K: int = 6144): + N = cute.sym_int() + M = cute.sym_int() + SPLIT_K = cute.sym_int() + X = make_fake_tensor(BFloat16, (N, K), divisibility=8) + W = make_fake_tensor(Float32, (M, K), divisibility=4) + out = make_fake_tensor(Float32, (SPLIT_K, N, M), divisibility=1) + stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True) + kernel = Sm100BF16x3RouterGemm(BN) + return cute.compile( + kernel, X, W, out, Int32(1), stream, options="--enable-tvm-ffi" + ) + + +@triton.jit +def _splitk_reduce_kernel( + partials, + out, + N, + M: tl.constexpr, + split_stride, + k_splits, + BN: tl.constexpr, + BM: tl.constexpr, + BS: tl.constexpr, + USE_PDL: tl.constexpr, +): + pid_n = tl.program_id(0) + pid_m = tl.program_id(1) + offs_n = pid_n * BN + tl.arange(0, BN) + offs_m = pid_m * BM + tl.arange(0, BM) + offs_s = tl.arange(0, BS) + + if USE_PDL: + tl.extra.cuda.gdc_wait() + tl.extra.cuda.gdc_launch_dependents() + + vals = tl.load( + partials + + offs_s[:, None, None] * split_stride + + offs_n[None, :, None] * M + + offs_m[None, None, :], + mask=( + (offs_s[:, None, None] < k_splits) + & (offs_n[None, :, None] < N) + & (offs_m[None, None, :] < M) + ), + other=0.0, + ) + acc = tl.sum(vals, axis=0) + tl.store( + out + offs_n[:, None] * M + offs_m[None, :], + acc, + mask=(offs_n[:, None] < N) & (offs_m[None, :] < M), + ) + + +def splitk_reduce_triton(partials: torch.Tensor, out: torch.Tensor): + split_k, N, M = partials.shape + block_s = 1 << (split_k - 1).bit_length() + split_stride = partials.stride(0) + if block_s >= 64: + BN, BM = 1, 32 + elif block_s >= 8: + BN, BM = 1, 256 + else: + BN, BM = min(16, 32 // block_s), 32 + grid = (triton.cdiv(N, BN), triton.cdiv(M, BM)) + _splitk_reduce_kernel[grid]( + partials, + out, + N, + M, + split_stride, + split_k, + BN=BN, + BM=BM, + BS=block_s, + USE_PDL=True, + num_warps=4, + launch_pdl=True, + ) + + +def bf16x3_router_gemm(X: torch.Tensor, W: torch.Tensor) -> torch.Tensor: + """Return ``X @ W.T`` using the SM100 BF16x3 router GEMM kernel.""" + N, K = X.shape + M, _ = W.shape + num_sms = torch.cuda.get_device_properties(X.device).multi_processor_count + + # next power of 2 within 8 and 128 + BN = triton.next_power_of_2(N) + BN = min(max(BN, 8), 128) + + BM = 128 + BK = 64 + k_tiles = math_utils.cdiv(K, BK) + grid_m = math_utils.cdiv(M, BM) + grid_n = math_utils.cdiv(N, BN) + + base_ctas = grid_m * grid_n + split_k = min(k_tiles, max(1, num_sms // base_ctas)) + + partials = X.new_empty(split_k, N, M, dtype=torch.float32) + Sm100BF16x3RouterGemm.compile(BN, K)(X, W, partials, split_k) + + if split_k == 1: + return partials.squeeze(0) + + out = X.new_empty(N, M, dtype=torch.float32) + splitk_reduce_triton(partials, out) + return out diff --git a/vllm/model_executor/layers/fused_moe/router/gate_linear.py b/vllm/model_executor/layers/fused_moe/router/gate_linear.py index e55cd5f4f2..39c4f4d815 100644 --- a/vllm/model_executor/layers/fused_moe/router/gate_linear.py +++ b/vllm/model_executor/layers/fused_moe/router/gate_linear.py @@ -4,11 +4,15 @@ import torch from torch.nn.parameter import Parameter import vllm._custom_ops as ops +from vllm.config import get_current_vllm_config_or_none +from vllm.logger import init_logger from vllm.model_executor.custom_op import PluggableLayer from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.platforms import current_platform from vllm.utils.torch_utils import direct_register_custom_op +logger = init_logger(__name__) + @PluggableLayer.register("gate_linear") class GateLinear(ReplicatedLinear): @@ -19,8 +23,9 @@ class GateLinear(ReplicatedLinear): 2. DSV3 specialized kernel (SM90+, M<=16, H=7168 E=256/384, H=6144 E=256) 3. fp32 specialized kernel (SM90+, bf16/fp32 in, fp32 out, M<=32, (H, E) in {(3072, 256), (6144, 128), (6144, 256)}) - 4. cuBLAS bf16×bf16→fp32 (SM90+ + bf16 weight + fp32 out_dtype) - 5. F.linear via ReplicatedLinear (ultimate fallback) + 4. experimental bf16x3 CuteDSL kernel (opt-in, SM100, bf16 in, fp32 weight) + 5. cuBLAS bf16×bf16→fp32 (SM90+ + bf16 weight + fp32 out_dtype) + 6. F.linear via ReplicatedLinear (ultimate fallback) The ``out_dtype`` attribute is mutable and can be set after init (e.g. when the required dtype depends on the expert quantization @@ -86,6 +91,11 @@ class GateLinear(ReplicatedLinear): self._dsv3_max_batch = 16 if is_hopper else 8 # fp32 specialized kernel eligibility (SM90+, exact dims, fp32 weight) + vllm_config = get_current_vllm_config_or_none() + enable_bf16x3_router_gemm = ( + vllm_config is not None + and vllm_config.kernel_config.enable_bf16x3_router_gemm + ) self.allow_fp32_router_gemm = ( not bias and self.weight.dtype == torch.float32 @@ -93,6 +103,16 @@ class GateLinear(ReplicatedLinear): and (is_hopper or is_blackwell) and (input_size, output_size) in self.FP32_SUPPORTED_SHAPES ) + self.allow_bf16x3_router_gemm = ( + not bias + and self.weight.dtype == torch.float32 + and current_platform.is_cuda() + and is_blackwell + and input_size % 8 == 0 + and enable_bf16x3_router_gemm + ) + if self.allow_bf16x3_router_gemm: + logger.info_once("Enabled experimental SM100 BF16x3 router GEMM.") # cuBLAS bf16→fp32 eligibility self.allow_cublas_router_gemm = ( @@ -173,15 +193,26 @@ class GateLinear(ReplicatedLinear): torch.float32, torch.bfloat16, ): - output = torch.ops.vllm.fp32_router_gemm_dispatch(x, self.weight) + output = torch.ops.vllm.fp32_router_gemm_dispatch( + x, self.weight, self.allow_bf16x3_router_gemm + ) return output, None - # Tier 4: cuBLAS bf16→fp32 + # Tier 4: experimental bf16x3 CuteDSL kernel for fp32 router weights + if self.allow_bf16x3_router_gemm and x.dtype == torch.bfloat16: + from vllm.model_executor.layers.fused_moe.router.bf16x3_router_gemm_cutedsl import ( # noqa: E501 + bf16x3_router_gemm, + ) + + output = bf16x3_router_gemm(x, self.weight) + return output, None + + # Tier 5: cuBLAS bf16→fp32 if self.allow_cublas_router_gemm and x.dtype == torch.bfloat16: output = torch.mm(x, self.weight.T, out_dtype=torch.float32) return output, None - # Tier 5: F.linear (ReplicatedLinear) + # Tier 6: F.linear (ReplicatedLinear) if self.out_dtype is not None and x.dtype != self.weight.dtype: x = x.to(self.weight.dtype) output, output_bias = super().forward(x) @@ -194,22 +225,34 @@ _FP32_ROUTER_GEMM_MAX_TOKENS = GateLinear.FP32_MAX_TOKENS def fp32_router_gemm_dispatch_impl( - x: torch.Tensor, weight: torch.Tensor + x: torch.Tensor, + weight: torch.Tensor, + allow_bf16x3_router_gemm: bool, ) -> torch.Tensor: """ Dynamically run fp32 specialized gemm if num_tokens <= FP32_MAX_TOKENS, - otherwise fall back to F.linear. + otherwise optionally run the experimental BF16x3 kernel for medium/large + SM100 router batches, then fall back to F.linear. This must be wrapped in a custom op because our torch.compile integration does not support runtime dispatching on num_tokens. """ if x.shape[0] <= _FP32_ROUTER_GEMM_MAX_TOKENS: return ops.fp32_router_gemm(x, weight) - else: - return torch.nn.functional.linear(x.float(), weight) + + if allow_bf16x3_router_gemm and x.dtype == torch.bfloat16: + from vllm.model_executor.layers.fused_moe.router.bf16x3_router_gemm_cutedsl import ( # noqa: E501 + bf16x3_router_gemm, + ) + + return bf16x3_router_gemm(x, weight) + + return torch.nn.functional.linear(x.float(), weight) def fp32_router_gemm_dispatch_fake( - x: torch.Tensor, weight: torch.Tensor + x: torch.Tensor, + weight: torch.Tensor, + allow_bf16x3_router_gemm: bool, ) -> torch.Tensor: return x.new_empty((x.shape[0], weight.shape[0]), dtype=torch.float32)