[Perf] Improve BF16x3 router GEMM accuracy and make it default on sm100 (#55899)
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
co-authored by
OpenAI Codex
Jee Jee Li
parent
3116c5d06b
commit
cc4210f671
@@ -1,6 +1,6 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
"""Tests for the experimental SM100 BF16x3 router GEMM."""
|
"""Tests for the SM100 BF16x3 router GEMM."""
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import torch
|
import torch
|
||||||
@@ -20,7 +20,16 @@ def _requires_sm100_cutedsl():
|
|||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
("num_tokens", "hidden_dim", "num_experts"),
|
("num_tokens", "hidden_dim", "num_experts"),
|
||||||
[(48, 6144, 128), (96, 3072, 256), (129, 3072, 17)],
|
[
|
||||||
|
(48, 6144, 128),
|
||||||
|
(96, 3072, 256),
|
||||||
|
(129, 3072, 17),
|
||||||
|
# long-K cases exercise the multi-accumulation path (the split-K
|
||||||
|
# heuristic leaves chains of 15 and 32 K-tiles here, above the
|
||||||
|
# kernel's num_tmem_acc bound)
|
||||||
|
(1024, 8192, 256),
|
||||||
|
(2048, 8192, 256),
|
||||||
|
],
|
||||||
)
|
)
|
||||||
def test_bf16x3_router_gemm_matches_reference(
|
def test_bf16x3_router_gemm_matches_reference(
|
||||||
num_tokens: int, hidden_dim: int, num_experts: int
|
num_tokens: int, hidden_dim: int, num_experts: int
|
||||||
@@ -36,8 +45,9 @@ def test_bf16x3_router_gemm_matches_reference(
|
|||||||
# Match the observed router weight scale
|
# Match the observed router weight scale
|
||||||
w *= 0.053
|
w *= 0.053
|
||||||
out = bf16x3_router_gemm(x, w)
|
out = bf16x3_router_gemm(x, w)
|
||||||
ref = torch.nn.functional.linear(x.float(), w)
|
# FP64 reference: the FP32 reference itself drifts by ~5e-6 at N=2048
|
||||||
|
ref = torch.nn.functional.linear(x.double(), w.double())
|
||||||
|
|
||||||
assert out.shape == (num_tokens, num_experts)
|
assert out.shape == (num_tokens, num_experts)
|
||||||
assert out.dtype == torch.float32
|
assert out.dtype == torch.float32
|
||||||
assert torch.mean(torch.abs(out - ref)).item() < 5e-6
|
assert torch.mean(torch.abs(out.double() - ref)).item() < 5e-6
|
||||||
|
|||||||
@@ -230,9 +230,6 @@ class KernelConfig:
|
|||||||
enable_jit_warmup: bool = True
|
enable_jit_warmup: bool = True
|
||||||
"""If True, run JIT compile warmup during kernel warmup."""
|
"""If True, run JIT compile warmup during kernel warmup."""
|
||||||
|
|
||||||
enable_bf16x3_router_gemm: bool = False
|
|
||||||
"""If True, use the experimental SM100 BF16x3 CuteDSL router GEMM."""
|
|
||||||
|
|
||||||
moe_backend: MoEBackend = "auto"
|
moe_backend: MoEBackend = "auto"
|
||||||
"""Backend for MoE expert computation kernels. Available options:
|
"""Backend for MoE expert computation kernels. Available options:
|
||||||
|
|
||||||
|
|||||||
@@ -695,7 +695,6 @@ class EngineArgs:
|
|||||||
enable_flashinfer_autotune: bool = get_field(
|
enable_flashinfer_autotune: bool = get_field(
|
||||||
KernelConfig, "enable_flashinfer_autotune"
|
KernelConfig, "enable_flashinfer_autotune"
|
||||||
)
|
)
|
||||||
enable_bf16x3_router_gemm: bool | None = None
|
|
||||||
worker_cls: str = ParallelConfig.worker_cls
|
worker_cls: str = ParallelConfig.worker_cls
|
||||||
worker_extension_cls: str = ParallelConfig.worker_extension_cls
|
worker_extension_cls: str = ParallelConfig.worker_extension_cls
|
||||||
|
|
||||||
@@ -1655,10 +1654,6 @@ class EngineArgs:
|
|||||||
"--enable-flashinfer-autotune",
|
"--enable-flashinfer-autotune",
|
||||||
**kernel_kwargs["enable_flashinfer_autotune"],
|
**kernel_kwargs["enable_flashinfer_autotune"],
|
||||||
)
|
)
|
||||||
kernel_group.add_argument(
|
|
||||||
"--enable-bf16x3-router-gemm",
|
|
||||||
**kernel_kwargs["enable_bf16x3_router_gemm"],
|
|
||||||
)
|
|
||||||
moe_backend_kwargs = kernel_kwargs["moe_backend"]
|
moe_backend_kwargs = kernel_kwargs["moe_backend"]
|
||||||
moe_backend_kwargs["type"] = lambda s: s.lower().replace("-", "_")
|
moe_backend_kwargs["type"] = lambda s: s.lower().replace("-", "_")
|
||||||
kernel_group.add_argument("--moe-backend", **moe_backend_kwargs)
|
kernel_group.add_argument("--moe-backend", **moe_backend_kwargs)
|
||||||
@@ -2519,8 +2514,6 @@ class EngineArgs:
|
|||||||
"are mutually exclusive"
|
"are mutually exclusive"
|
||||||
)
|
)
|
||||||
kernel_config.enable_flashinfer_autotune = self.enable_flashinfer_autotune
|
kernel_config.enable_flashinfer_autotune = self.enable_flashinfer_autotune
|
||||||
if self.enable_bf16x3_router_gemm is not None:
|
|
||||||
kernel_config.enable_bf16x3_router_gemm = self.enable_bf16x3_router_gemm
|
|
||||||
if self.moe_backend != "auto":
|
if self.moe_backend != "auto":
|
||||||
kernel_config.moe_backend = self.moe_backend
|
kernel_config.moe_backend = self.moe_backend
|
||||||
if self.linear_backend != "auto":
|
if self.linear_backend != "auto":
|
||||||
|
|||||||
@@ -6,6 +6,10 @@ Computes ``X @ W.T`` for BF16 ``X`` with shape ``[N, K]`` and FP32 router
|
|||||||
weights ``W`` with shape ``[M, K]`` by decomposing each FP32 weight value into
|
weights ``W`` with shape ``[M, K]`` by decomposing each FP32 weight value into
|
||||||
three BF16 residual terms inside the kernel, then accumulating the three BF16
|
three BF16 residual terms inside the kernel, then accumulating the three BF16
|
||||||
MMA results into FP32 TMEM output.
|
MMA results into FP32 TMEM output.
|
||||||
|
|
||||||
|
The TMEM accumulation chain is bounded to ``num_tmem_acc`` K-tiles: at chunk
|
||||||
|
boundaries the epilogue warps drain the main-term accumulator into a register
|
||||||
|
master accumulator and the MMA warp resets it.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from functools import cache
|
from functools import cache
|
||||||
@@ -22,6 +26,7 @@ from quack.compile_utils import make_fake_tensor
|
|||||||
from vllm.cute_utils import _tcgen05, simple_tma_copy
|
from vllm.cute_utils import _tcgen05, simple_tma_copy
|
||||||
from vllm.triton_utils import tl, triton
|
from vllm.triton_utils import tl, triton
|
||||||
from vllm.utils import math_utils
|
from vllm.utils import math_utils
|
||||||
|
from vllm.utils.torch_utils import direct_register_custom_op
|
||||||
|
|
||||||
__all__ = ["bf16x3_router_gemm"]
|
__all__ = ["bf16x3_router_gemm"]
|
||||||
|
|
||||||
@@ -82,6 +87,7 @@ class Sm100BF16x3RouterGemm:
|
|||||||
self.cta_tile = (BN, 128, 64)
|
self.cta_tile = (BN, 128, 64)
|
||||||
self.num_stages = 2
|
self.num_stages = 2
|
||||||
self.num_warps = 10
|
self.num_warps = 10
|
||||||
|
self.num_tmem_acc = 8
|
||||||
|
|
||||||
@cute.jit
|
@cute.jit
|
||||||
def _make_tma(self, tensor: cute.Tensor, BM: int, BK: int):
|
def _make_tma(self, tensor: cute.Tensor, BM: int, BK: int):
|
||||||
@@ -129,6 +135,7 @@ class Sm100BF16x3RouterGemm:
|
|||||||
|
|
||||||
BN, BM, BK = self.cta_tile
|
BN, BM, BK = self.cta_tile
|
||||||
num_stages = self.num_stages
|
num_stages = self.num_stages
|
||||||
|
num_tmem_acc = self.num_tmem_acc
|
||||||
|
|
||||||
N, K = X_tma.tma_tensor.shape
|
N, K = X_tma.tma_tensor.shape
|
||||||
M, _ = W_tma.tma_tensor.shape
|
M, _ = W_tma.tma_tensor.shape
|
||||||
@@ -151,7 +158,8 @@ class Sm100BF16x3RouterGemm:
|
|||||||
tma_full_mbar = smem.allocate_array(Int64, num_stages)
|
tma_full_mbar = smem.allocate_array(Int64, num_stages)
|
||||||
tma_empty_mbar = smem.allocate_array(Int64, num_stages)
|
tma_empty_mbar = smem.allocate_array(Int64, num_stages)
|
||||||
w_full_mbar = smem.allocate_array(Int64, num_stages)
|
w_full_mbar = smem.allocate_array(Int64, num_stages)
|
||||||
mma_mbar = smem.allocate_array(Int64, 1)
|
acc_full_mbar = smem.allocate_array(Int64, 1)
|
||||||
|
acc_empty_mbar = smem.allocate_array(Int64, 1)
|
||||||
taddr = smem.allocate(Int32, 4)
|
taddr = smem.allocate(Int32, 4)
|
||||||
|
|
||||||
BAR_TMEM_ALLOC = 1
|
BAR_TMEM_ALLOC = 1
|
||||||
@@ -171,7 +179,8 @@ class Sm100BF16x3RouterGemm:
|
|||||||
cute.arch.mbarrier_init(tma_full_mbar + i, 1)
|
cute.arch.mbarrier_init(tma_full_mbar + i, 1)
|
||||||
cute.arch.mbarrier_init(tma_empty_mbar + i, 1)
|
cute.arch.mbarrier_init(tma_empty_mbar + i, 1)
|
||||||
cute.arch.mbarrier_init(w_full_mbar + i, 128)
|
cute.arch.mbarrier_init(w_full_mbar + i, 128)
|
||||||
cute.arch.mbarrier_init(mma_mbar, 1)
|
cute.arch.mbarrier_init(acc_full_mbar, 1)
|
||||||
|
cute.arch.mbarrier_init(acc_empty_mbar, 128)
|
||||||
cute.arch.mbarrier_init_fence()
|
cute.arch.mbarrier_init_fence()
|
||||||
elif warp_id == 1:
|
elif warp_id == 1:
|
||||||
cpasync.prefetch_descriptor(X_tma.atom)
|
cpasync.prefetch_descriptor(X_tma.atom)
|
||||||
@@ -213,40 +222,52 @@ class Sm100BF16x3RouterGemm:
|
|||||||
|
|
||||||
elif warp_id == 8:
|
elif warp_id == 8:
|
||||||
# MMA warp
|
# MMA warp
|
||||||
stage_id = 0
|
tma_stage_id = 0
|
||||||
parity = 0
|
tma_parity = 0
|
||||||
|
|
||||||
|
tmem_acc_count = 0
|
||||||
|
tmem_parity = 1
|
||||||
|
|
||||||
idesc = _tcgen05.make_bf16_idesc(BM, BN)
|
idesc = _tcgen05.make_bf16_idesc(BM, BN)
|
||||||
sdesc = _tcgen05.make_sdesc_128B_swizzle(0)
|
sdesc = _tcgen05.make_sdesc_128B_swizzle(0)
|
||||||
|
|
||||||
for tile_k in cutlass.range(bid_k, k_tiles, split_k, unroll=1):
|
for tile_k in cutlass.range(bid_k, k_tiles, split_k, unroll=1):
|
||||||
cute.arch.mbarrier_wait(tma_full_mbar + stage_id, parity)
|
if tmem_acc_count == 0:
|
||||||
cute.arch.mbarrier_wait(w_full_mbar + stage_id, parity)
|
cute.arch.mbarrier_wait(acc_empty_mbar, tmem_parity)
|
||||||
|
cute.arch.mbarrier_wait(tma_full_mbar + tma_stage_id, tma_parity)
|
||||||
|
cute.arch.mbarrier_wait(w_full_mbar + tma_stage_id, tma_parity)
|
||||||
_tcgen05.fence_after_thread_sync()
|
_tcgen05.fence_after_thread_sync()
|
||||||
|
|
||||||
w_tmem = w_tmem_base + stage_id * (BK // 2 * 3)
|
w_tmem = w_tmem_base + tma_stage_id * (BK // 2 * 3)
|
||||||
x_desc = sdesc | (sX[None, None, stage_id].iterator.toint() >> 4)
|
x_desc = sdesc | (sX[None, None, tma_stage_id].iterator.toint() >> 4)
|
||||||
|
|
||||||
for k in cutlass.range_constexpr(BK // 16):
|
for k in cutlass.range_constexpr(BK // 16):
|
||||||
enable_d = (tile_k > bid_k) or (k > 0)
|
enable_d_main = (tmem_acc_count > 0) or (k > 0)
|
||||||
|
enable_d_res = (tile_k > bid_k) or (k > 0)
|
||||||
_tcgen05.mma_ts_f16(
|
_tcgen05.mma_ts_f16(
|
||||||
acc_main, w_tmem + k * 8, x_desc, idesc, enable_d
|
acc_main, w_tmem + k * 8, x_desc, idesc, enable_d_main
|
||||||
)
|
)
|
||||||
_tcgen05.mma_ts_f16(
|
_tcgen05.mma_ts_f16(
|
||||||
acc_res, w_tmem + 32 + k * 8, x_desc, idesc, enable_d
|
acc_res, w_tmem + 32 + k * 8, x_desc, idesc, enable_d_res
|
||||||
)
|
)
|
||||||
_tcgen05.mma_ts_f16(
|
_tcgen05.mma_ts_f16(
|
||||||
acc_res, w_tmem + 64 + k * 8, x_desc, idesc, True
|
acc_res, w_tmem + 64 + k * 8, x_desc, idesc, True
|
||||||
)
|
)
|
||||||
x_desc += 32 >> 4
|
x_desc += 32 >> 4
|
||||||
|
|
||||||
_tcgen05.commit(tma_empty_mbar + stage_id)
|
_tcgen05.commit(tma_empty_mbar + tma_stage_id)
|
||||||
|
|
||||||
stage_id = (stage_id + 1) % self.num_stages
|
tma_stage_id = (tma_stage_id + 1) % num_stages
|
||||||
if stage_id == 0:
|
if tma_stage_id == 0:
|
||||||
parity ^= 1
|
tma_parity ^= 1
|
||||||
|
|
||||||
_tcgen05.commit(mma_mbar)
|
tmem_acc_count = (tmem_acc_count + 1) % num_tmem_acc
|
||||||
|
if tmem_acc_count == 0:
|
||||||
|
_tcgen05.commit(acc_full_mbar)
|
||||||
|
tmem_parity ^= 1
|
||||||
|
|
||||||
|
if tmem_acc_count != 0:
|
||||||
|
_tcgen05.commit(acc_full_mbar)
|
||||||
|
|
||||||
elif warp_id >= 4:
|
elif warp_id >= 4:
|
||||||
# prep warps: decompose FP32 W into 3xBF16
|
# prep warps: decompose FP32 W into 3xBF16
|
||||||
@@ -303,39 +324,93 @@ class Sm100BF16x3RouterGemm:
|
|||||||
_tcgen05.alloc(taddr)
|
_tcgen05.alloc(taddr)
|
||||||
cute.arch.barrier(barrier_id=BAR_TMEM_ALLOC, number_of_threads=128)
|
cute.arch.barrier(barrier_id=BAR_TMEM_ALLOC, number_of_threads=128)
|
||||||
|
|
||||||
if warp_id == 0:
|
tiles_local = cute.ceil_div(k_tiles - bid_k, split_k)
|
||||||
cute.arch.mbarrier_wait(mma_mbar, 0)
|
num_chunks = cute.ceil_div(tiles_local, num_tmem_acc)
|
||||||
cute.arch.barrier(barrier_id=BAR_EPI, number_of_threads=128)
|
|
||||||
_tcgen05.fence_after_thread_sync()
|
|
||||||
|
|
||||||
cute.arch.griddepcontrol_launch_dependents()
|
|
||||||
|
|
||||||
WIDTH = 8
|
WIDTH = 8
|
||||||
for i in cutlass.range_constexpr(BN // WIDTH):
|
main_regs = cute.make_rmem_tensor(WIDTH, Float32)
|
||||||
tcol = i * WIDTH
|
res_regs = cute.make_rmem_tensor(WIDTH, Float32)
|
||||||
main_regs = cute.make_rmem_tensor(WIDTH, Float32)
|
|
||||||
res_regs = cute.make_rmem_tensor(WIDTH, Float32)
|
|
||||||
main_regs.store(_tcgen05.ld(warp_id * 32, tcol, "32x32b", WIDTH))
|
|
||||||
res_regs.store(_tcgen05.ld(warp_id * 32, BN + tcol, "32x32b", WIDTH))
|
|
||||||
_tcgen05.wait_ld()
|
|
||||||
|
|
||||||
# CuteDSL will codegen add.f32x2
|
|
||||||
for j in cutlass.range(WIDTH, vectorize=True):
|
|
||||||
main_regs[j] += res_regs[j]
|
|
||||||
|
|
||||||
|
if num_chunks == 1:
|
||||||
|
# single chunk
|
||||||
|
cute.arch.mbarrier_wait(acc_full_mbar, 0)
|
||||||
|
_tcgen05.fence_after_thread_sync()
|
||||||
w_row_idx = bid_m * BM + tid
|
w_row_idx = bid_m * BM + tid
|
||||||
for j in cutlass.range_constexpr(WIDTH):
|
for i in cutlass.range_constexpr(BN // WIDTH):
|
||||||
x_row_idx = bid_n * BN + i * WIDTH + j
|
tcol = i * WIDTH
|
||||||
if x_row_idx < N and w_row_idx < M:
|
main_regs.store(_tcgen05.ld(warp_id * 32, tcol, "32x32b", WIDTH))
|
||||||
out[bid_k, x_row_idx, w_row_idx] = main_regs[j]
|
res_regs.store(
|
||||||
|
_tcgen05.ld(warp_id * 32, BN + tcol, "32x32b", WIDTH)
|
||||||
|
)
|
||||||
|
_tcgen05.wait_ld()
|
||||||
|
|
||||||
|
# CuteDSL will codegen add.f32x2
|
||||||
|
for j in cutlass.range(WIDTH, vectorize=True):
|
||||||
|
main_regs[j] += res_regs[j]
|
||||||
|
|
||||||
|
for j in cutlass.range_constexpr(WIDTH):
|
||||||
|
x_row_idx = bid_n * BN + i * WIDTH + j
|
||||||
|
if x_row_idx < N and w_row_idx < M:
|
||||||
|
out[bid_k, x_row_idx, w_row_idx] = main_regs[j]
|
||||||
|
|
||||||
|
else:
|
||||||
|
# multiple chunks
|
||||||
|
master_acc = cute.make_rmem_tensor(BN, Float32)
|
||||||
|
|
||||||
|
# chunk 0: pure TMEM load, no accumulation.
|
||||||
|
cute.arch.mbarrier_wait(acc_full_mbar, 0)
|
||||||
|
_tcgen05.fence_after_thread_sync()
|
||||||
|
master_acc.store(_tcgen05.ld(warp_id * 32, 0, "32x32b", BN))
|
||||||
|
_tcgen05.wait_ld()
|
||||||
|
_tcgen05.fence_before_thread_sync()
|
||||||
|
cute.arch.mbarrier_arrive(acc_empty_mbar)
|
||||||
|
|
||||||
|
# accumulate main tmem acc
|
||||||
|
for chunk in cutlass.range(1, num_chunks, unroll=1):
|
||||||
|
cute.arch.mbarrier_wait(acc_full_mbar, chunk & 1)
|
||||||
|
_tcgen05.fence_after_thread_sync()
|
||||||
|
|
||||||
|
for i in cutlass.range_constexpr(BN // WIDTH):
|
||||||
|
tcol = i * WIDTH
|
||||||
|
main_regs.store(
|
||||||
|
_tcgen05.ld(warp_id * 32, tcol, "32x32b", WIDTH)
|
||||||
|
)
|
||||||
|
_tcgen05.wait_ld()
|
||||||
|
# CuteDSL refuses to compile
|
||||||
|
# master_acc[i * WIDTH + j] += main_regs[j]
|
||||||
|
# when vectorize=True
|
||||||
|
master_view = cute.local_tile(master_acc, (WIDTH,), (i,))
|
||||||
|
for j in cutlass.range(WIDTH, vectorize=True):
|
||||||
|
master_view[j] += main_regs[j]
|
||||||
|
|
||||||
|
_tcgen05.fence_before_thread_sync()
|
||||||
|
cute.arch.mbarrier_arrive(acc_empty_mbar)
|
||||||
|
|
||||||
|
# fold in the residual accumulator
|
||||||
|
for i in cutlass.range_constexpr(BN // WIDTH):
|
||||||
|
tcol = i * WIDTH
|
||||||
|
res_regs.store(
|
||||||
|
_tcgen05.ld(warp_id * 32, BN + tcol, "32x32b", WIDTH)
|
||||||
|
)
|
||||||
|
_tcgen05.wait_ld()
|
||||||
|
master_view = cute.local_tile(master_acc, (WIDTH,), (i,))
|
||||||
|
for j in cutlass.range(WIDTH, vectorize=True):
|
||||||
|
master_view[j] += res_regs[j]
|
||||||
|
|
||||||
|
w_row_idx = bid_m * BM + tid
|
||||||
|
for j in cutlass.range_constexpr(WIDTH):
|
||||||
|
x_row_idx = bid_n * BN + i * WIDTH + j
|
||||||
|
if x_row_idx < N and w_row_idx < M:
|
||||||
|
out[bid_k, x_row_idx, w_row_idx] = master_acc[i * WIDTH + j]
|
||||||
|
|
||||||
|
cute.arch.griddepcontrol_launch_dependents()
|
||||||
cute.arch.barrier(barrier_id=BAR_EPI, number_of_threads=128)
|
cute.arch.barrier(barrier_id=BAR_EPI, number_of_threads=128)
|
||||||
if warp_id == 0:
|
if warp_id == 0:
|
||||||
_tcgen05.dealloc()
|
_tcgen05.dealloc()
|
||||||
|
|
||||||
@cache
|
@cache
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def compile(BN: int = 128, K: int = 6144):
|
def compile(K: int, BN: int = 128):
|
||||||
N = cute.sym_int()
|
N = cute.sym_int()
|
||||||
M = cute.sym_int()
|
M = cute.sym_int()
|
||||||
SPLIT_K = cute.sym_int()
|
SPLIT_K = cute.sym_int()
|
||||||
@@ -392,16 +467,19 @@ def _splitk_reduce_kernel(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _splitk_reduce_config(split_k: int) -> tuple[int, int, int]:
|
||||||
|
block_s = triton.next_power_of_2(split_k)
|
||||||
|
if block_s >= 64:
|
||||||
|
return 1, 32, block_s
|
||||||
|
if block_s >= 8:
|
||||||
|
return 1, 256, block_s
|
||||||
|
return min(16, 32 // block_s), 32, block_s
|
||||||
|
|
||||||
|
|
||||||
def splitk_reduce_triton(partials: torch.Tensor, out: torch.Tensor):
|
def splitk_reduce_triton(partials: torch.Tensor, out: torch.Tensor):
|
||||||
split_k, N, M = partials.shape
|
split_k, N, M = partials.shape
|
||||||
block_s = 1 << (split_k - 1).bit_length()
|
|
||||||
split_stride = partials.stride(0)
|
split_stride = partials.stride(0)
|
||||||
if block_s >= 64:
|
BN, BM, block_s = _splitk_reduce_config(split_k)
|
||||||
BN, BM = 1, 32
|
|
||||||
elif block_s >= 8:
|
|
||||||
BN, BM = 1, 256
|
|
||||||
else:
|
|
||||||
BN, BM = min(16, 32 // block_s), 32
|
|
||||||
grid = (triton.cdiv(N, BN), triton.cdiv(M, BM))
|
grid = (triton.cdiv(N, BN), triton.cdiv(M, BM))
|
||||||
_splitk_reduce_kernel[grid](
|
_splitk_reduce_kernel[grid](
|
||||||
partials,
|
partials,
|
||||||
@@ -419,27 +497,124 @@ def splitk_reduce_triton(partials: torch.Tensor, out: torch.Tensor):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def bf16x3_router_gemm(X: torch.Tensor, W: torch.Tensor) -> torch.Tensor:
|
# Accuracy-constrained (BN, split_k) configs from GB300 sweeps.
|
||||||
"""Return ``X @ W.T`` using the SM100 BF16x3 router GEMM kernel."""
|
_TILE_CONFIG_OVERRIDES = {
|
||||||
N, K = X.shape
|
(2816, 256): { # Hunyuan-V4
|
||||||
M, _ = W.shape
|
64: (16, 16),
|
||||||
num_sms = torch.cuda.get_device_properties(X.device).multi_processor_count
|
128: (32, 16),
|
||||||
|
256: (64, 16),
|
||||||
|
512: (128, 16),
|
||||||
|
},
|
||||||
|
(3072, 256): { # MiniMax-M2
|
||||||
|
64: (16, 16),
|
||||||
|
128: (32, 16),
|
||||||
|
256: (64, 16),
|
||||||
|
512: (128, 16),
|
||||||
|
},
|
||||||
|
(4096, 192): { # Hunyuan-V3
|
||||||
|
64: (32, 32),
|
||||||
|
128: (64, 32),
|
||||||
|
256: (128, 32),
|
||||||
|
512: (128, 19),
|
||||||
|
},
|
||||||
|
(6144, 128): { # MiniMax-M3
|
||||||
|
64: (32, 64),
|
||||||
|
128: (64, 64),
|
||||||
|
256: (64, 32),
|
||||||
|
512: (128, 32),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _pick_tile_config(N: int, K: int, M: int, num_sms: int) -> tuple[int, int]:
|
||||||
|
"""Return (BN, split_k): tuned table mid-range, generic rule otherwise."""
|
||||||
|
k_tiles = math_utils.cdiv(K, 64)
|
||||||
|
|
||||||
|
if 32 < N <= 512:
|
||||||
|
token_bucket = triton.next_power_of_2(N)
|
||||||
|
overrides = _TILE_CONFIG_OVERRIDES.get((K, M))
|
||||||
|
if overrides is not None:
|
||||||
|
BN, split_k = overrides[token_bucket]
|
||||||
|
return BN, min(split_k, k_tiles)
|
||||||
|
|
||||||
# next power of 2 within 8 and 128
|
# next power of 2 within 8 and 128
|
||||||
BN = triton.next_power_of_2(N)
|
BN = triton.next_power_of_2(N)
|
||||||
BN = min(max(BN, 8), 128)
|
BN = min(max(BN, 8), 128)
|
||||||
|
|
||||||
BM = 128
|
grid_m = math_utils.cdiv(M, 128)
|
||||||
BK = 64
|
|
||||||
k_tiles = math_utils.cdiv(K, BK)
|
|
||||||
grid_m = math_utils.cdiv(M, BM)
|
|
||||||
grid_n = math_utils.cdiv(N, BN)
|
grid_n = math_utils.cdiv(N, BN)
|
||||||
|
|
||||||
base_ctas = grid_m * grid_n
|
base_ctas = grid_m * grid_n
|
||||||
split_k = min(k_tiles, max(1, num_sms // base_ctas))
|
split_k = min(k_tiles, max(1, num_sms // base_ctas))
|
||||||
|
return BN, split_k
|
||||||
|
|
||||||
|
|
||||||
|
def warmup_bf16x3_router_gemm(
|
||||||
|
K: int,
|
||||||
|
M: int,
|
||||||
|
min_num_tokens: int,
|
||||||
|
max_num_tokens: int,
|
||||||
|
) -> tuple[int, ...]:
|
||||||
|
"""Compile every BF16x3 router GEMM configuration reachable at runtime."""
|
||||||
|
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||||
|
TritonWarmupTensor,
|
||||||
|
triton_scalar_specialization_rep,
|
||||||
|
)
|
||||||
|
|
||||||
|
device = torch.accelerator.current_device_index()
|
||||||
|
num_sms = torch.cuda.get_device_properties(device).multi_processor_count
|
||||||
|
gemm_configs: set[int] = set()
|
||||||
|
reduce_configs: set[tuple[int, int, int, int, int, int]] = set()
|
||||||
|
for N in range(min_num_tokens, max_num_tokens + 1):
|
||||||
|
BN, split_k = _pick_tile_config(N, K, M, num_sms)
|
||||||
|
gemm_configs.add(BN)
|
||||||
|
if split_k == 1:
|
||||||
|
continue
|
||||||
|
reduce_BN, reduce_BM, block_s = _splitk_reduce_config(split_k)
|
||||||
|
reduce_configs.add(
|
||||||
|
(
|
||||||
|
triton_scalar_specialization_rep(N),
|
||||||
|
triton_scalar_specialization_rep(N * M),
|
||||||
|
triton_scalar_specialization_rep(split_k),
|
||||||
|
reduce_BN,
|
||||||
|
reduce_BM,
|
||||||
|
block_s,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
configs = tuple(sorted(gemm_configs))
|
||||||
|
for BN in configs:
|
||||||
|
Sm100BF16x3RouterGemm.compile(K, BN)
|
||||||
|
for N, split_stride, split_k, BN, BM, block_s in sorted(reduce_configs):
|
||||||
|
partials = TritonWarmupTensor(torch.float32)
|
||||||
|
out = TritonWarmupTensor(torch.float32)
|
||||||
|
_splitk_reduce_kernel.warmup(
|
||||||
|
partials,
|
||||||
|
out,
|
||||||
|
N,
|
||||||
|
M,
|
||||||
|
split_stride,
|
||||||
|
split_k,
|
||||||
|
BN=BN,
|
||||||
|
BM=BM,
|
||||||
|
BS=block_s,
|
||||||
|
USE_PDL=True,
|
||||||
|
num_warps=4,
|
||||||
|
launch_pdl=True,
|
||||||
|
grid=(1, 1),
|
||||||
|
)
|
||||||
|
return configs
|
||||||
|
|
||||||
|
|
||||||
|
def _bf16x3_router_gemm(X: torch.Tensor, W: torch.Tensor) -> torch.Tensor:
|
||||||
|
"""Return ``X @ W.T`` using the SM100 BF16x3 router GEMM kernel."""
|
||||||
|
N, K = X.shape
|
||||||
|
M, _ = W.shape
|
||||||
|
num_sms = torch.cuda.get_device_properties(X.device).multi_processor_count
|
||||||
|
|
||||||
|
BN, split_k = _pick_tile_config(N, K, M, num_sms)
|
||||||
|
|
||||||
partials = X.new_empty(split_k, N, M, dtype=torch.float32)
|
partials = X.new_empty(split_k, N, M, dtype=torch.float32)
|
||||||
Sm100BF16x3RouterGemm.compile(BN, K)(X, W, partials, split_k)
|
Sm100BF16x3RouterGemm.compile(K, BN)(X, W, partials, split_k)
|
||||||
|
|
||||||
if split_k == 1:
|
if split_k == 1:
|
||||||
return partials.squeeze(0)
|
return partials.squeeze(0)
|
||||||
@@ -447,3 +622,21 @@ def bf16x3_router_gemm(X: torch.Tensor, W: torch.Tensor) -> torch.Tensor:
|
|||||||
out = X.new_empty(N, M, dtype=torch.float32)
|
out = X.new_empty(N, M, dtype=torch.float32)
|
||||||
splitk_reduce_triton(partials, out)
|
splitk_reduce_triton(partials, out)
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _bf16x3_router_gemm_fake(
|
||||||
|
X: torch.Tensor,
|
||||||
|
W: torch.Tensor,
|
||||||
|
) -> torch.Tensor:
|
||||||
|
return X.new_empty((X.shape[0], W.shape[0]), dtype=torch.float32)
|
||||||
|
|
||||||
|
|
||||||
|
direct_register_custom_op(
|
||||||
|
op_name="bf16x3_router_gemm",
|
||||||
|
op_func=_bf16x3_router_gemm,
|
||||||
|
fake_impl=_bf16x3_router_gemm_fake,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def bf16x3_router_gemm(X: torch.Tensor, W: torch.Tensor) -> torch.Tensor:
|
||||||
|
return torch.ops.vllm.bf16x3_router_gemm(X, W)
|
||||||
|
|||||||
@@ -4,15 +4,11 @@ import torch
|
|||||||
from torch.nn.parameter import Parameter
|
from torch.nn.parameter import Parameter
|
||||||
|
|
||||||
import vllm._custom_ops as ops
|
import vllm._custom_ops as ops
|
||||||
from vllm.config import get_current_vllm_config_or_none
|
|
||||||
from vllm.logger import init_logger
|
|
||||||
from vllm.model_executor.custom_op import PluggableLayer
|
from vllm.model_executor.custom_op import PluggableLayer
|
||||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
from vllm.utils.torch_utils import direct_register_custom_op
|
from vllm.utils.torch_utils import direct_register_custom_op
|
||||||
|
|
||||||
logger = init_logger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
@PluggableLayer.register("gate_linear")
|
@PluggableLayer.register("gate_linear")
|
||||||
class GateLinear(ReplicatedLinear):
|
class GateLinear(ReplicatedLinear):
|
||||||
@@ -22,7 +18,7 @@ class GateLinear(ReplicatedLinear):
|
|||||||
K divisible by 8)
|
K divisible by 8)
|
||||||
2. fp32 specialized kernel (SM90+ or gfx950, bf16/fp32 in, fp32 out,
|
2. fp32 specialized kernel (SM90+ or gfx950, bf16/fp32 in, fp32 out,
|
||||||
M<=32, model-specific shapes)
|
M<=32, model-specific shapes)
|
||||||
3. experimental bf16x3 CuteDSL kernel (opt-in, SM100, bf16 in, fp32 weight)
|
3. bf16x3 CuteDSL kernel (SM100, bf16 in, fp32 weight)
|
||||||
4. cuBLAS bf16×bf16→fp32 (SM90+ + bf16 weight + fp32 out_dtype)
|
4. cuBLAS bf16×bf16→fp32 (SM90+ + bf16 weight + fp32 out_dtype)
|
||||||
5. F.linear via ReplicatedLinear (ultimate fallback)
|
5. F.linear via ReplicatedLinear (ultimate fallback)
|
||||||
|
|
||||||
@@ -85,11 +81,6 @@ class GateLinear(ReplicatedLinear):
|
|||||||
self.allow_specialized_router_gemm = can_use_specialized_kernels
|
self.allow_specialized_router_gemm = can_use_specialized_kernels
|
||||||
|
|
||||||
# fp32 specialized kernel eligibility (exact dims, fp32 weight)
|
# fp32 specialized kernel eligibility (exact dims, fp32 weight)
|
||||||
vllm_config = get_current_vllm_config_or_none()
|
|
||||||
enable_bf16x3_router_gemm = (
|
|
||||||
vllm_config is not None
|
|
||||||
and vllm_config.kernel_config.enable_bf16x3_router_gemm
|
|
||||||
)
|
|
||||||
self.allow_fp32_router_gemm = (
|
self.allow_fp32_router_gemm = (
|
||||||
not bias
|
not bias
|
||||||
and self.weight.dtype == torch.float32
|
and self.weight.dtype == torch.float32
|
||||||
@@ -108,10 +99,7 @@ class GateLinear(ReplicatedLinear):
|
|||||||
and current_platform.is_cuda()
|
and current_platform.is_cuda()
|
||||||
and is_blackwell
|
and is_blackwell
|
||||||
and input_size % 8 == 0
|
and input_size % 8 == 0
|
||||||
and enable_bf16x3_router_gemm
|
|
||||||
)
|
)
|
||||||
if self.allow_bf16x3_router_gemm:
|
|
||||||
logger.info_once("Enabled experimental SM100 BF16x3 router GEMM.")
|
|
||||||
|
|
||||||
# Fused bf16 x bf16 -> fp32 GEMM eligibility. torch.mm's out_dtype
|
# Fused bf16 x bf16 -> fp32 GEMM eligibility. torch.mm's out_dtype
|
||||||
# epilogue folds the fp32 cast into the GEMM, removing the standalone
|
# epilogue folds the fp32 cast into the GEMM, removing the standalone
|
||||||
@@ -198,7 +186,7 @@ class GateLinear(ReplicatedLinear):
|
|||||||
)
|
)
|
||||||
return output, None
|
return output, None
|
||||||
|
|
||||||
# Tier 3: experimental bf16x3 CuteDSL kernel for fp32 router weights
|
# Tier 3: bf16x3 CuteDSL kernel for fp32 router weights
|
||||||
if self.allow_bf16x3_router_gemm and x.dtype == torch.bfloat16:
|
if self.allow_bf16x3_router_gemm and x.dtype == torch.bfloat16:
|
||||||
from vllm.model_executor.layers.fused_moe.router.bf16x3_router_gemm_cutedsl import ( # noqa: E501
|
from vllm.model_executor.layers.fused_moe.router.bf16x3_router_gemm_cutedsl import ( # noqa: E501
|
||||||
bf16x3_router_gemm,
|
bf16x3_router_gemm,
|
||||||
@@ -251,10 +239,10 @@ def fp32_router_gemm_dispatch_impl(
|
|||||||
|
|
||||||
if allow_bf16x3_router_gemm and x.dtype == torch.bfloat16:
|
if allow_bf16x3_router_gemm and x.dtype == torch.bfloat16:
|
||||||
from vllm.model_executor.layers.fused_moe.router.bf16x3_router_gemm_cutedsl import ( # noqa: E501
|
from vllm.model_executor.layers.fused_moe.router.bf16x3_router_gemm_cutedsl import ( # noqa: E501
|
||||||
bf16x3_router_gemm,
|
_bf16x3_router_gemm,
|
||||||
)
|
)
|
||||||
|
|
||||||
return bf16x3_router_gemm(x, weight)
|
return _bf16x3_router_gemm(x, weight)
|
||||||
|
|
||||||
return torch.nn.functional.linear(x.float(), weight)
|
return torch.nn.functional.linear(x.float(), weight)
|
||||||
|
|
||||||
|
|||||||
@@ -101,6 +101,44 @@ def _warmup_ll_bf16_router_gemm(model: torch.nn.Module) -> None:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _warmup_bf16x3_router_gemm(
|
||||||
|
model: torch.nn.Module,
|
||||||
|
max_num_tokens: int,
|
||||||
|
) -> None:
|
||||||
|
from vllm.model_executor.layers.fused_moe.router.bf16x3_router_gemm_cutedsl import ( # noqa: E501
|
||||||
|
warmup_bf16x3_router_gemm,
|
||||||
|
)
|
||||||
|
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
|
||||||
|
|
||||||
|
gate = next(
|
||||||
|
(
|
||||||
|
module
|
||||||
|
for module in model.modules()
|
||||||
|
if isinstance(module, GateLinear) and module.allow_bf16x3_router_gemm
|
||||||
|
),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
if gate is None:
|
||||||
|
logger.debug_once(
|
||||||
|
"Skipping BF16x3 router GEMM warmup: no eligible GateLinear found."
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
min_num_tokens = gate.FP32_MAX_TOKENS + 1 if gate.allow_fp32_router_gemm else 1
|
||||||
|
logger.info_once(
|
||||||
|
"Warming up BF16x3 router GEMM for K=%d, M=%d.",
|
||||||
|
gate.input_size,
|
||||||
|
gate.output_size,
|
||||||
|
)
|
||||||
|
configs = warmup_bf16x3_router_gemm(
|
||||||
|
gate.input_size,
|
||||||
|
gate.output_size,
|
||||||
|
min_num_tokens,
|
||||||
|
max_num_tokens,
|
||||||
|
)
|
||||||
|
logger.info_once("Warmed up BF16x3 router GEMM configs: %s.", configs)
|
||||||
|
|
||||||
|
|
||||||
def _warmup_kimi_k3_gemm_rs_ar() -> None:
|
def _warmup_kimi_k3_gemm_rs_ar() -> None:
|
||||||
# Kimi-K3 model construction imports this module only when GEMM-RS/AR is
|
# Kimi-K3 model construction imports this module only when GEMM-RS/AR is
|
||||||
# enabled and initializes its singleton before kernel_warmup runs. Avoid
|
# enabled and initializes its singleton before kernel_warmup runs. Avoid
|
||||||
@@ -131,7 +169,8 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
|
|||||||
if zeroer is not None:
|
if zeroer is not None:
|
||||||
zeroer.warmup(worker.model_runner.kv_cache_config.num_blocks)
|
zeroer.warmup(worker.model_runner.kv_cache_config.num_blocks)
|
||||||
|
|
||||||
if worker.vllm_config.kernel_config.enable_jit_warmup:
|
enable_jit_warmup = worker.vllm_config.kernel_config.enable_jit_warmup
|
||||||
|
if enable_jit_warmup:
|
||||||
logger.info("JIT kernel warmup starting.")
|
logger.info("JIT kernel warmup starting.")
|
||||||
jit_warmup_start = time.perf_counter()
|
jit_warmup_start = time.perf_counter()
|
||||||
try:
|
try:
|
||||||
@@ -164,11 +203,17 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Run next so input-prep kernels JIT against pristine runner state.
|
# Run next so input-prep kernels JIT against pristine runner state.
|
||||||
if worker.vllm_config.kernel_config.enable_jit_warmup:
|
if enable_jit_warmup:
|
||||||
kimi_k3_triton_warmup(worker)
|
kimi_k3_triton_warmup(worker)
|
||||||
spec_decode_rejection_warmup(worker)
|
spec_decode_rejection_warmup(worker)
|
||||||
qwen4_exp_qsa_triton_warmup(worker)
|
qwen4_exp_qsa_triton_warmup(worker)
|
||||||
|
|
||||||
|
if enable_jit_warmup and current_platform.is_device_capability_family(100):
|
||||||
|
_warmup_bf16x3_router_gemm(
|
||||||
|
worker.get_model(),
|
||||||
|
worker.scheduler_config.max_num_batched_tokens,
|
||||||
|
)
|
||||||
|
|
||||||
if current_platform.has_device_capability(90):
|
if current_platform.has_device_capability(90):
|
||||||
_warmup_ll_bf16_router_gemm(worker.get_model())
|
_warmup_ll_bf16_router_gemm(worker.get_model())
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user