BF16x3 router GEMM (#47973)
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
@@ -0,0 +1,43 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the experimental SM100 BF16x3 router GEMM."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.utils.import_utils import has_cutedsl
|
||||
|
||||
|
||||
def _requires_sm100_cutedsl():
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA not available")
|
||||
major, _ = torch.cuda.get_device_capability()
|
||||
if major != 10:
|
||||
pytest.skip("bf16x3 router GEMM requires SM100-class GPU")
|
||||
if not has_cutedsl():
|
||||
pytest.skip("cutedsl (cutlass) not installed")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_tokens", "hidden_dim", "num_experts"),
|
||||
[(48, 6144, 128), (96, 3072, 256), (129, 3072, 17)],
|
||||
)
|
||||
def test_bf16x3_router_gemm_matches_reference(
|
||||
num_tokens: int, hidden_dim: int, num_experts: int
|
||||
):
|
||||
_requires_sm100_cutedsl()
|
||||
from vllm.model_executor.layers.fused_moe.router.bf16x3_router_gemm_cutedsl import ( # noqa: E501
|
||||
bf16x3_router_gemm,
|
||||
)
|
||||
|
||||
torch.manual_seed(42)
|
||||
x = torch.randn(num_tokens, hidden_dim, dtype=torch.bfloat16, device="cuda")
|
||||
w = torch.randn(num_experts, hidden_dim, dtype=torch.float32, device="cuda")
|
||||
# Match the observed router weight scale
|
||||
w *= 0.053
|
||||
out = bf16x3_router_gemm(x, w)
|
||||
ref = torch.nn.functional.linear(x.float(), w)
|
||||
|
||||
assert out.shape == (num_tokens, num_experts)
|
||||
assert out.dtype == torch.float32
|
||||
assert torch.mean(torch.abs(out - ref)).item() < 5e-6
|
||||
@@ -178,6 +178,9 @@ class KernelConfig:
|
||||
enable_cutedsl_warmup: bool = True
|
||||
"""If True, run CuTeDSL compile warmup during kernel warmup."""
|
||||
|
||||
enable_bf16x3_router_gemm: bool = False
|
||||
"""If True, use the experimental SM100 BF16x3 CuteDSL router GEMM."""
|
||||
|
||||
moe_backend: MoEBackend = "auto"
|
||||
"""Backend for MoE expert computation kernels. Available options:
|
||||
|
||||
|
||||
@@ -665,6 +665,7 @@ class EngineArgs:
|
||||
enable_flashinfer_autotune: bool = get_field(
|
||||
KernelConfig, "enable_flashinfer_autotune"
|
||||
)
|
||||
enable_bf16x3_router_gemm: bool | None = None
|
||||
worker_cls: str = ParallelConfig.worker_cls
|
||||
worker_extension_cls: str = ParallelConfig.worker_extension_cls
|
||||
|
||||
@@ -1508,6 +1509,10 @@ class EngineArgs:
|
||||
"--enable-flashinfer-autotune",
|
||||
**kernel_kwargs["enable_flashinfer_autotune"],
|
||||
)
|
||||
kernel_group.add_argument(
|
||||
"--enable-bf16x3-router-gemm",
|
||||
**kernel_kwargs["enable_bf16x3_router_gemm"],
|
||||
)
|
||||
moe_backend_kwargs = kernel_kwargs["moe_backend"]
|
||||
moe_backend_kwargs["type"] = lambda s: s.lower().replace("-", "_")
|
||||
kernel_group.add_argument("--moe-backend", **moe_backend_kwargs)
|
||||
@@ -2286,6 +2291,8 @@ class EngineArgs:
|
||||
"are mutually exclusive"
|
||||
)
|
||||
kernel_config.enable_flashinfer_autotune = self.enable_flashinfer_autotune
|
||||
if self.enable_bf16x3_router_gemm is not None:
|
||||
kernel_config.enable_bf16x3_router_gemm = self.enable_bf16x3_router_gemm
|
||||
if self.moe_backend != "auto":
|
||||
kernel_config.moe_backend = self.moe_backend
|
||||
if self.linear_backend != "auto":
|
||||
|
||||
@@ -0,0 +1,449 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CuteDSL BF16x3 router GEMM.
|
||||
|
||||
Computes ``X @ W.T`` for BF16 ``X`` with shape ``[N, K]`` and FP32 router
|
||||
weights ``W`` with shape ``[M, K]`` by decomposing each FP32 weight value into
|
||||
three BF16 residual terms inside the kernel, then accumulating the three BF16
|
||||
MMA results into FP32 TMEM output.
|
||||
"""
|
||||
|
||||
from functools import cache
|
||||
|
||||
import cutlass
|
||||
import torch
|
||||
from cuda.bindings.driver import CUstream
|
||||
from cutlass import BFloat16, Float32, Int32, Int64, Uint32, cute
|
||||
from cutlass._mlir.dialects import llvm
|
||||
from cutlass.cute.nvgpu import cpasync
|
||||
from cutlass.cutlass_dsl import T, dsl_user_op
|
||||
from quack.compile_utils import make_fake_tensor
|
||||
|
||||
from vllm.cute_utils import _tcgen05, simple_tma_copy
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils import math_utils
|
||||
|
||||
__all__ = ["bf16x3_router_gemm"]
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def _decompose_fp32x2_to_3xbf16x2(
|
||||
w0: Float32,
|
||||
w1: Float32,
|
||||
*,
|
||||
loc=None,
|
||||
ip=None,
|
||||
) -> tuple[Uint32, Uint32, Uint32]:
|
||||
# this PTX snippets does the following
|
||||
# out0 = BF16(in); res = in - FP32(out0)
|
||||
# out1 = BF16(res); res = res - FP32(out1)
|
||||
# out2 = BF16(res)
|
||||
#
|
||||
# for normal FP32, this decomposition is exact
|
||||
# i.e. in = FP32(out0) + FP32(out1) + FP32(out2)
|
||||
#
|
||||
out = llvm.inline_asm(
|
||||
llvm.StructType.get_literal([T.i32(), T.i32(), T.i32()]),
|
||||
[w0.ir_value(loc=loc, ip=ip), w1.ir_value(loc=loc, ip=ip)],
|
||||
"{\n\t"
|
||||
".reg .b32 r1_lo, r1_hi, r2_lo, r2_hi;\n\t"
|
||||
".reg .b32 w0_lo, w0_hi, w1_lo, w1_hi;\n\t"
|
||||
".reg .b64 a_pair, w0_pair, w1_pair, r1_pair, r2_pair;\n\t"
|
||||
"cvt.rn.bf16x2.f32 $0, $4, $3;\n\t"
|
||||
"shl.b32 w0_lo, $0, 16;\n\t"
|
||||
"and.b32 w0_hi, $0, 0xffff0000;\n\t"
|
||||
"mov.b64 a_pair, {$3, $4};\n\t"
|
||||
"mov.b64 w0_pair, {w0_lo, w0_hi};\n\t"
|
||||
"sub.rn.f32x2 r1_pair, a_pair, w0_pair;\n\t"
|
||||
"mov.b64 {r1_lo, r1_hi}, r1_pair;\n\t"
|
||||
"cvt.rn.bf16x2.f32 $1, r1_hi, r1_lo;\n\t"
|
||||
"shl.b32 w1_lo, $1, 16;\n\t"
|
||||
"and.b32 w1_hi, $1, 0xffff0000;\n\t"
|
||||
"mov.b64 w1_pair, {w1_lo, w1_hi};\n\t"
|
||||
"sub.rn.f32x2 r2_pair, r1_pair, w1_pair;\n\t"
|
||||
"mov.b64 {r2_lo, r2_hi}, r2_pair;\n\t"
|
||||
"cvt.rn.bf16x2.f32 $2, r2_hi, r2_lo;\n\t"
|
||||
"}\n",
|
||||
"=r,=r,=r,f,f",
|
||||
has_side_effects=False,
|
||||
is_align_stack=False,
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
return (
|
||||
Uint32(llvm.extractvalue(T.i32(), out, [0], loc=loc, ip=ip)),
|
||||
Uint32(llvm.extractvalue(T.i32(), out, [1], loc=loc, ip=ip)),
|
||||
Uint32(llvm.extractvalue(T.i32(), out, [2], loc=loc, ip=ip)),
|
||||
)
|
||||
|
||||
|
||||
class Sm100BF16x3RouterGemm:
|
||||
def __init__(self, BN: int = 128) -> None:
|
||||
self.cta_tile = (BN, 128, 64)
|
||||
self.num_stages = 2
|
||||
self.num_warps = 10
|
||||
|
||||
@cute.jit
|
||||
def _make_tma(self, tensor: cute.Tensor, BM: int, BK: int):
|
||||
op = cpasync.CopyBulkTensorTileG2SOp()
|
||||
swizzle_128B = cute.make_swizzle(3, 4, 3)
|
||||
elems = 128 * 8 // tensor.element_type.width # 128B
|
||||
slayout = cute.make_layout(
|
||||
(BM, (elems, BK // elems), self.num_stages),
|
||||
stride=(elems, (1, BM * elems), BM * BK),
|
||||
)
|
||||
slayout = cute.make_composed_layout(swizzle_128B, 0, slayout)
|
||||
return cpasync.make_tiled_tma_atom(op, tensor, slayout, (BM, BK))
|
||||
|
||||
@cute.jit
|
||||
def __call__(
|
||||
self,
|
||||
X: cute.Tensor,
|
||||
W: cute.Tensor,
|
||||
out: cute.Tensor,
|
||||
split_k: Int32,
|
||||
stream: CUstream,
|
||||
):
|
||||
BN, BM, BK = self.cta_tile
|
||||
W_tma = self._make_tma(W, BM, BK)
|
||||
X_tma = self._make_tma(X, BN, BK)
|
||||
|
||||
grid_m = cute.ceil_div(W.shape[0], BM)
|
||||
grid_n = cute.ceil_div(X.shape[0], BN)
|
||||
|
||||
self.kernel(X_tma, W_tma, out).launch(
|
||||
grid=(grid_m, grid_n, split_k),
|
||||
block=(self.num_warps * 32, 1, 1),
|
||||
stream=stream,
|
||||
use_pdl=True,
|
||||
)
|
||||
|
||||
@cute.kernel
|
||||
def kernel(self, X_tma: cpasync.TmaInfo, W_tma: cpasync.TmaInfo, out: cute.Tensor):
|
||||
tid, _, _ = cute.arch.thread_idx()
|
||||
bid_m, bid_n, bid_k = cute.arch.block_idx()
|
||||
_, _, split_k = cute.arch.grid_dim()
|
||||
|
||||
warp_id = cute.arch.make_warp_uniform(cute.arch.warp_idx())
|
||||
lane_id = cute.arch.lane_idx()
|
||||
|
||||
BN, BM, BK = self.cta_tile
|
||||
num_stages = self.num_stages
|
||||
|
||||
N, K = X_tma.tma_tensor.shape
|
||||
M, _ = W_tma.tma_tensor.shape
|
||||
k_tiles = cute.ceil_div(K, BK)
|
||||
|
||||
smem = cutlass.utils.SmemAllocator()
|
||||
sX = smem.allocate_tensor(
|
||||
BFloat16,
|
||||
X_tma.smem_layout.outer,
|
||||
byte_alignment=128,
|
||||
swizzle=X_tma.smem_layout.inner,
|
||||
)
|
||||
sW = smem.allocate_tensor(
|
||||
Float32,
|
||||
W_tma.smem_layout.outer,
|
||||
byte_alignment=128,
|
||||
swizzle=W_tma.smem_layout.inner,
|
||||
)
|
||||
|
||||
tma_full_mbar = smem.allocate_array(Int64, num_stages)
|
||||
tma_empty_mbar = smem.allocate_array(Int64, num_stages)
|
||||
w_full_mbar = smem.allocate_array(Int64, num_stages)
|
||||
mma_mbar = smem.allocate_array(Int64, 1)
|
||||
taddr = smem.allocate(Int32, 4)
|
||||
|
||||
BAR_TMEM_ALLOC = 1
|
||||
BAR_PREP = 2
|
||||
BAR_EPI = 3
|
||||
|
||||
# tmem "allocation"
|
||||
# acc_main is for the 1st BF16 term. acc_res is for the 2nd and 3rd
|
||||
# BF16 terms, which are much smaller than the first term.
|
||||
acc_main = 0
|
||||
acc_res = BN
|
||||
w_tmem_base = BN * 2
|
||||
|
||||
if warp_id == 0:
|
||||
with cute.arch.elect_one():
|
||||
for i in cutlass.range_constexpr(num_stages):
|
||||
cute.arch.mbarrier_init(tma_full_mbar + i, 1)
|
||||
cute.arch.mbarrier_init(tma_empty_mbar + i, 1)
|
||||
cute.arch.mbarrier_init(w_full_mbar + i, 128)
|
||||
cute.arch.mbarrier_init(mma_mbar, 1)
|
||||
cute.arch.mbarrier_init_fence()
|
||||
elif warp_id == 1:
|
||||
cpasync.prefetch_descriptor(X_tma.atom)
|
||||
cpasync.prefetch_descriptor(W_tma.atom)
|
||||
cute.arch.sync_threads()
|
||||
cute.arch.griddepcontrol_wait()
|
||||
|
||||
if warp_id == 9:
|
||||
# TMA warp
|
||||
stage_id = 0
|
||||
parity = 1
|
||||
|
||||
# (BM, BK, K/BK)
|
||||
gW_tiles = cute.local_tile(W_tma.tma_tensor, (BM, BK), (bid_m, None))
|
||||
gX_tiles = cute.local_tile(X_tma.tma_tensor, (BN, BK), (bid_n, None))
|
||||
|
||||
for tile_k in cutlass.range(bid_k, k_tiles, split_k, unroll=1):
|
||||
cute.arch.mbarrier_wait(tma_empty_mbar + stage_id, parity)
|
||||
mbar = tma_full_mbar + stage_id
|
||||
with cute.arch.elect_one():
|
||||
stage_bytes = BN * BK * 2 + BM * BK * 4
|
||||
cute.arch.mbarrier_arrive_and_expect_tx(mbar, stage_bytes)
|
||||
simple_tma_copy(
|
||||
W_tma.atom,
|
||||
gW_tiles[None, None, tile_k],
|
||||
sW[None, None, stage_id],
|
||||
mbar,
|
||||
)
|
||||
simple_tma_copy(
|
||||
X_tma.atom,
|
||||
gX_tiles[None, None, tile_k],
|
||||
sX[None, None, stage_id],
|
||||
mbar,
|
||||
)
|
||||
|
||||
stage_id = (stage_id + 1) % num_stages
|
||||
if stage_id == 0:
|
||||
parity ^= 1
|
||||
|
||||
elif warp_id == 8:
|
||||
# MMA warp
|
||||
stage_id = 0
|
||||
parity = 0
|
||||
|
||||
idesc = _tcgen05.make_bf16_idesc(BM, BN)
|
||||
sdesc = _tcgen05.make_sdesc_128B_swizzle(0)
|
||||
|
||||
for tile_k in cutlass.range(bid_k, k_tiles, split_k, unroll=1):
|
||||
cute.arch.mbarrier_wait(tma_full_mbar + stage_id, parity)
|
||||
cute.arch.mbarrier_wait(w_full_mbar + stage_id, parity)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
|
||||
w_tmem = w_tmem_base + stage_id * (BK // 2 * 3)
|
||||
x_desc = sdesc | (sX[None, None, stage_id].iterator.toint() >> 4)
|
||||
|
||||
for k in cutlass.range_constexpr(BK // 16):
|
||||
enable_d = (tile_k > bid_k) or (k > 0)
|
||||
_tcgen05.mma_ts_f16(
|
||||
acc_main, w_tmem + k * 8, x_desc, idesc, enable_d
|
||||
)
|
||||
_tcgen05.mma_ts_f16(
|
||||
acc_res, w_tmem + 32 + k * 8, x_desc, idesc, enable_d
|
||||
)
|
||||
_tcgen05.mma_ts_f16(
|
||||
acc_res, w_tmem + 64 + k * 8, x_desc, idesc, True
|
||||
)
|
||||
x_desc += 32 >> 4
|
||||
|
||||
_tcgen05.commit(tma_empty_mbar + stage_id)
|
||||
|
||||
stage_id = (stage_id + 1) % self.num_stages
|
||||
if stage_id == 0:
|
||||
parity ^= 1
|
||||
|
||||
_tcgen05.commit(mma_mbar)
|
||||
|
||||
elif warp_id >= 4:
|
||||
# prep warps: decompose FP32 W into 3xBF16
|
||||
warp_id_ = warp_id % 4
|
||||
|
||||
stage_id = 0
|
||||
parity = 0
|
||||
|
||||
# ld.shared.v4.f32
|
||||
op = cute.nvgpu.CopyUniversalOp()
|
||||
cp_atom = cute.make_copy_atom(op, Float32, num_bits_per_copy=128)
|
||||
|
||||
# sW_view: ((4, 1), (BK/4, num_stages))
|
||||
row = warp_id_ * 32 + lane_id
|
||||
sW_view = cute.zipped_divide(sW[row, None, None], (4, 1))
|
||||
|
||||
for _ in cutlass.range(bid_k, k_tiles, split_k, unroll=1):
|
||||
if warp_id_ == 0:
|
||||
cute.arch.mbarrier_wait(tma_full_mbar + stage_id, parity)
|
||||
cute.arch.barrier(barrier_id=BAR_PREP, number_of_threads=128)
|
||||
|
||||
row = warp_id_ * 32 + lane_id
|
||||
w_tmem = w_tmem_base + stage_id * (BK // 2 * 3)
|
||||
for kblock in cutlass.range_constexpr(BK // 4):
|
||||
w0 = cute.make_rmem_tensor(2, Uint32)
|
||||
w1 = cute.make_rmem_tensor(2, Uint32)
|
||||
w2 = cute.make_rmem_tensor(2, Uint32)
|
||||
|
||||
w_tmp = cute.make_rmem_tensor(4, Float32)
|
||||
cute.copy(cp_atom, sW_view[None, (kblock, stage_id)], w_tmp)
|
||||
w0[0], w1[0], w2[0] = _decompose_fp32x2_to_3xbf16x2(
|
||||
w_tmp[0], w_tmp[1]
|
||||
)
|
||||
w0[1], w1[1], w2[1] = _decompose_fp32x2_to_3xbf16x2(
|
||||
w_tmp[2], w_tmp[3]
|
||||
)
|
||||
|
||||
tcol = kblock * 2
|
||||
_tcgen05.st(warp_id_ * 32, w_tmem + 0 + tcol, "32x32b", 2, w0)
|
||||
_tcgen05.st(warp_id_ * 32, w_tmem + 32 + tcol, "32x32b", 2, w1)
|
||||
_tcgen05.st(warp_id_ * 32, w_tmem + 64 + tcol, "32x32b", 2, w2)
|
||||
|
||||
_tcgen05.wait_st()
|
||||
_tcgen05.fence_before_thread_sync()
|
||||
cute.arch.mbarrier_arrive(w_full_mbar + stage_id)
|
||||
|
||||
stage_id = (stage_id + 1) % self.num_stages
|
||||
if stage_id == 0:
|
||||
parity ^= 1
|
||||
|
||||
else:
|
||||
# epilogue warps
|
||||
if warp_id == 0:
|
||||
_tcgen05.alloc(taddr)
|
||||
cute.arch.barrier(barrier_id=BAR_TMEM_ALLOC, number_of_threads=128)
|
||||
|
||||
if warp_id == 0:
|
||||
cute.arch.mbarrier_wait(mma_mbar, 0)
|
||||
cute.arch.barrier(barrier_id=BAR_EPI, number_of_threads=128)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
|
||||
cute.arch.griddepcontrol_launch_dependents()
|
||||
|
||||
WIDTH = 8
|
||||
for i in cutlass.range_constexpr(BN // WIDTH):
|
||||
tcol = i * WIDTH
|
||||
main_regs = cute.make_rmem_tensor(WIDTH, Float32)
|
||||
res_regs = cute.make_rmem_tensor(WIDTH, Float32)
|
||||
main_regs.store(_tcgen05.ld(warp_id * 32, tcol, "32x32b", WIDTH))
|
||||
res_regs.store(_tcgen05.ld(warp_id * 32, BN + tcol, "32x32b", WIDTH))
|
||||
_tcgen05.wait_ld()
|
||||
|
||||
# CuteDSL will codegen add.f32x2
|
||||
for j in cutlass.range(WIDTH, vectorize=True):
|
||||
main_regs[j] += res_regs[j]
|
||||
|
||||
w_row_idx = bid_m * BM + tid
|
||||
for j in cutlass.range_constexpr(WIDTH):
|
||||
x_row_idx = bid_n * BN + i * WIDTH + j
|
||||
if x_row_idx < N and w_row_idx < M:
|
||||
out[bid_k, x_row_idx, w_row_idx] = main_regs[j]
|
||||
|
||||
cute.arch.barrier(barrier_id=BAR_EPI, number_of_threads=128)
|
||||
if warp_id == 0:
|
||||
_tcgen05.dealloc()
|
||||
|
||||
@cache
|
||||
@staticmethod
|
||||
def compile(BN: int = 128, K: int = 6144):
|
||||
N = cute.sym_int()
|
||||
M = cute.sym_int()
|
||||
SPLIT_K = cute.sym_int()
|
||||
X = make_fake_tensor(BFloat16, (N, K), divisibility=8)
|
||||
W = make_fake_tensor(Float32, (M, K), divisibility=4)
|
||||
out = make_fake_tensor(Float32, (SPLIT_K, N, M), divisibility=1)
|
||||
stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True)
|
||||
kernel = Sm100BF16x3RouterGemm(BN)
|
||||
return cute.compile(
|
||||
kernel, X, W, out, Int32(1), stream, options="--enable-tvm-ffi"
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _splitk_reduce_kernel(
|
||||
partials,
|
||||
out,
|
||||
N,
|
||||
M: tl.constexpr,
|
||||
split_stride,
|
||||
k_splits,
|
||||
BN: tl.constexpr,
|
||||
BM: tl.constexpr,
|
||||
BS: tl.constexpr,
|
||||
USE_PDL: tl.constexpr,
|
||||
):
|
||||
pid_n = tl.program_id(0)
|
||||
pid_m = tl.program_id(1)
|
||||
offs_n = pid_n * BN + tl.arange(0, BN)
|
||||
offs_m = pid_m * BM + tl.arange(0, BM)
|
||||
offs_s = tl.arange(0, BS)
|
||||
|
||||
if USE_PDL:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
vals = tl.load(
|
||||
partials
|
||||
+ offs_s[:, None, None] * split_stride
|
||||
+ offs_n[None, :, None] * M
|
||||
+ offs_m[None, None, :],
|
||||
mask=(
|
||||
(offs_s[:, None, None] < k_splits)
|
||||
& (offs_n[None, :, None] < N)
|
||||
& (offs_m[None, None, :] < M)
|
||||
),
|
||||
other=0.0,
|
||||
)
|
||||
acc = tl.sum(vals, axis=0)
|
||||
tl.store(
|
||||
out + offs_n[:, None] * M + offs_m[None, :],
|
||||
acc,
|
||||
mask=(offs_n[:, None] < N) & (offs_m[None, :] < M),
|
||||
)
|
||||
|
||||
|
||||
def splitk_reduce_triton(partials: torch.Tensor, out: torch.Tensor):
|
||||
split_k, N, M = partials.shape
|
||||
block_s = 1 << (split_k - 1).bit_length()
|
||||
split_stride = partials.stride(0)
|
||||
if block_s >= 64:
|
||||
BN, BM = 1, 32
|
||||
elif block_s >= 8:
|
||||
BN, BM = 1, 256
|
||||
else:
|
||||
BN, BM = min(16, 32 // block_s), 32
|
||||
grid = (triton.cdiv(N, BN), triton.cdiv(M, BM))
|
||||
_splitk_reduce_kernel[grid](
|
||||
partials,
|
||||
out,
|
||||
N,
|
||||
M,
|
||||
split_stride,
|
||||
split_k,
|
||||
BN=BN,
|
||||
BM=BM,
|
||||
BS=block_s,
|
||||
USE_PDL=True,
|
||||
num_warps=4,
|
||||
launch_pdl=True,
|
||||
)
|
||||
|
||||
|
||||
def bf16x3_router_gemm(X: torch.Tensor, W: torch.Tensor) -> torch.Tensor:
|
||||
"""Return ``X @ W.T`` using the SM100 BF16x3 router GEMM kernel."""
|
||||
N, K = X.shape
|
||||
M, _ = W.shape
|
||||
num_sms = torch.cuda.get_device_properties(X.device).multi_processor_count
|
||||
|
||||
# next power of 2 within 8 and 128
|
||||
BN = triton.next_power_of_2(N)
|
||||
BN = min(max(BN, 8), 128)
|
||||
|
||||
BM = 128
|
||||
BK = 64
|
||||
k_tiles = math_utils.cdiv(K, BK)
|
||||
grid_m = math_utils.cdiv(M, BM)
|
||||
grid_n = math_utils.cdiv(N, BN)
|
||||
|
||||
base_ctas = grid_m * grid_n
|
||||
split_k = min(k_tiles, max(1, num_sms // base_ctas))
|
||||
|
||||
partials = X.new_empty(split_k, N, M, dtype=torch.float32)
|
||||
Sm100BF16x3RouterGemm.compile(BN, K)(X, W, partials, split_k)
|
||||
|
||||
if split_k == 1:
|
||||
return partials.squeeze(0)
|
||||
|
||||
out = X.new_empty(N, M, dtype=torch.float32)
|
||||
splitk_reduce_triton(partials, out)
|
||||
return out
|
||||
@@ -4,11 +4,15 @@ import torch
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
from vllm.config import get_current_vllm_config_or_none
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@PluggableLayer.register("gate_linear")
|
||||
class GateLinear(ReplicatedLinear):
|
||||
@@ -19,8 +23,9 @@ class GateLinear(ReplicatedLinear):
|
||||
2. DSV3 specialized kernel (SM90+, M<=16, H=7168 E=256/384, H=6144 E=256)
|
||||
3. fp32 specialized kernel (SM90+, bf16/fp32 in, fp32 out, M<=32,
|
||||
(H, E) in {(3072, 256), (6144, 128), (6144, 256)})
|
||||
4. cuBLAS bf16×bf16→fp32 (SM90+ + bf16 weight + fp32 out_dtype)
|
||||
5. F.linear via ReplicatedLinear (ultimate fallback)
|
||||
4. experimental bf16x3 CuteDSL kernel (opt-in, SM100, bf16 in, fp32 weight)
|
||||
5. cuBLAS bf16×bf16→fp32 (SM90+ + bf16 weight + fp32 out_dtype)
|
||||
6. F.linear via ReplicatedLinear (ultimate fallback)
|
||||
|
||||
The ``out_dtype`` attribute is mutable and can be set after init
|
||||
(e.g. when the required dtype depends on the expert quantization
|
||||
@@ -86,6 +91,11 @@ class GateLinear(ReplicatedLinear):
|
||||
self._dsv3_max_batch = 16 if is_hopper else 8
|
||||
|
||||
# fp32 specialized kernel eligibility (SM90+, exact dims, fp32 weight)
|
||||
vllm_config = get_current_vllm_config_or_none()
|
||||
enable_bf16x3_router_gemm = (
|
||||
vllm_config is not None
|
||||
and vllm_config.kernel_config.enable_bf16x3_router_gemm
|
||||
)
|
||||
self.allow_fp32_router_gemm = (
|
||||
not bias
|
||||
and self.weight.dtype == torch.float32
|
||||
@@ -93,6 +103,16 @@ class GateLinear(ReplicatedLinear):
|
||||
and (is_hopper or is_blackwell)
|
||||
and (input_size, output_size) in self.FP32_SUPPORTED_SHAPES
|
||||
)
|
||||
self.allow_bf16x3_router_gemm = (
|
||||
not bias
|
||||
and self.weight.dtype == torch.float32
|
||||
and current_platform.is_cuda()
|
||||
and is_blackwell
|
||||
and input_size % 8 == 0
|
||||
and enable_bf16x3_router_gemm
|
||||
)
|
||||
if self.allow_bf16x3_router_gemm:
|
||||
logger.info_once("Enabled experimental SM100 BF16x3 router GEMM.")
|
||||
|
||||
# cuBLAS bf16→fp32 eligibility
|
||||
self.allow_cublas_router_gemm = (
|
||||
@@ -173,15 +193,26 @@ class GateLinear(ReplicatedLinear):
|
||||
torch.float32,
|
||||
torch.bfloat16,
|
||||
):
|
||||
output = torch.ops.vllm.fp32_router_gemm_dispatch(x, self.weight)
|
||||
output = torch.ops.vllm.fp32_router_gemm_dispatch(
|
||||
x, self.weight, self.allow_bf16x3_router_gemm
|
||||
)
|
||||
return output, None
|
||||
|
||||
# Tier 4: cuBLAS bf16→fp32
|
||||
# Tier 4: experimental bf16x3 CuteDSL kernel for fp32 router weights
|
||||
if self.allow_bf16x3_router_gemm and x.dtype == torch.bfloat16:
|
||||
from vllm.model_executor.layers.fused_moe.router.bf16x3_router_gemm_cutedsl import ( # noqa: E501
|
||||
bf16x3_router_gemm,
|
||||
)
|
||||
|
||||
output = bf16x3_router_gemm(x, self.weight)
|
||||
return output, None
|
||||
|
||||
# Tier 5: cuBLAS bf16→fp32
|
||||
if self.allow_cublas_router_gemm and x.dtype == torch.bfloat16:
|
||||
output = torch.mm(x, self.weight.T, out_dtype=torch.float32)
|
||||
return output, None
|
||||
|
||||
# Tier 5: F.linear (ReplicatedLinear)
|
||||
# Tier 6: F.linear (ReplicatedLinear)
|
||||
if self.out_dtype is not None and x.dtype != self.weight.dtype:
|
||||
x = x.to(self.weight.dtype)
|
||||
output, output_bias = super().forward(x)
|
||||
@@ -194,22 +225,34 @@ _FP32_ROUTER_GEMM_MAX_TOKENS = GateLinear.FP32_MAX_TOKENS
|
||||
|
||||
|
||||
def fp32_router_gemm_dispatch_impl(
|
||||
x: torch.Tensor, weight: torch.Tensor
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
allow_bf16x3_router_gemm: bool,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Dynamically run fp32 specialized gemm if num_tokens <= FP32_MAX_TOKENS,
|
||||
otherwise fall back to F.linear.
|
||||
otherwise optionally run the experimental BF16x3 kernel for medium/large
|
||||
SM100 router batches, then fall back to F.linear.
|
||||
This must be wrapped in a custom op because our torch.compile integration
|
||||
does not support runtime dispatching on num_tokens.
|
||||
"""
|
||||
if x.shape[0] <= _FP32_ROUTER_GEMM_MAX_TOKENS:
|
||||
return ops.fp32_router_gemm(x, weight)
|
||||
else:
|
||||
return torch.nn.functional.linear(x.float(), weight)
|
||||
|
||||
if allow_bf16x3_router_gemm and x.dtype == torch.bfloat16:
|
||||
from vllm.model_executor.layers.fused_moe.router.bf16x3_router_gemm_cutedsl import ( # noqa: E501
|
||||
bf16x3_router_gemm,
|
||||
)
|
||||
|
||||
return bf16x3_router_gemm(x, weight)
|
||||
|
||||
return torch.nn.functional.linear(x.float(), weight)
|
||||
|
||||
|
||||
def fp32_router_gemm_dispatch_fake(
|
||||
x: torch.Tensor, weight: torch.Tensor
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
allow_bf16x3_router_gemm: bool,
|
||||
) -> torch.Tensor:
|
||||
return x.new_empty((x.shape[0], weight.shape[0]), dtype=torch.float32)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user