[Perf] Improve BF16x3 router GEMM accuracy and make it default on sm100 (#55899)
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
co-authored by
OpenAI Codex
Jee Jee Li
parent
3116c5d06b
commit
cc4210f671
@@ -6,6 +6,10 @@ Computes ``X @ W.T`` for BF16 ``X`` with shape ``[N, K]`` and FP32 router
|
||||
weights ``W`` with shape ``[M, K]`` by decomposing each FP32 weight value into
|
||||
three BF16 residual terms inside the kernel, then accumulating the three BF16
|
||||
MMA results into FP32 TMEM output.
|
||||
|
||||
The TMEM accumulation chain is bounded to ``num_tmem_acc`` K-tiles: at chunk
|
||||
boundaries the epilogue warps drain the main-term accumulator into a register
|
||||
master accumulator and the MMA warp resets it.
|
||||
"""
|
||||
|
||||
from functools import cache
|
||||
@@ -22,6 +26,7 @@ from quack.compile_utils import make_fake_tensor
|
||||
from vllm.cute_utils import _tcgen05, simple_tma_copy
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils import math_utils
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
__all__ = ["bf16x3_router_gemm"]
|
||||
|
||||
@@ -82,6 +87,7 @@ class Sm100BF16x3RouterGemm:
|
||||
self.cta_tile = (BN, 128, 64)
|
||||
self.num_stages = 2
|
||||
self.num_warps = 10
|
||||
self.num_tmem_acc = 8
|
||||
|
||||
@cute.jit
|
||||
def _make_tma(self, tensor: cute.Tensor, BM: int, BK: int):
|
||||
@@ -129,6 +135,7 @@ class Sm100BF16x3RouterGemm:
|
||||
|
||||
BN, BM, BK = self.cta_tile
|
||||
num_stages = self.num_stages
|
||||
num_tmem_acc = self.num_tmem_acc
|
||||
|
||||
N, K = X_tma.tma_tensor.shape
|
||||
M, _ = W_tma.tma_tensor.shape
|
||||
@@ -151,7 +158,8 @@ class Sm100BF16x3RouterGemm:
|
||||
tma_full_mbar = smem.allocate_array(Int64, num_stages)
|
||||
tma_empty_mbar = smem.allocate_array(Int64, num_stages)
|
||||
w_full_mbar = smem.allocate_array(Int64, num_stages)
|
||||
mma_mbar = smem.allocate_array(Int64, 1)
|
||||
acc_full_mbar = smem.allocate_array(Int64, 1)
|
||||
acc_empty_mbar = smem.allocate_array(Int64, 1)
|
||||
taddr = smem.allocate(Int32, 4)
|
||||
|
||||
BAR_TMEM_ALLOC = 1
|
||||
@@ -171,7 +179,8 @@ class Sm100BF16x3RouterGemm:
|
||||
cute.arch.mbarrier_init(tma_full_mbar + i, 1)
|
||||
cute.arch.mbarrier_init(tma_empty_mbar + i, 1)
|
||||
cute.arch.mbarrier_init(w_full_mbar + i, 128)
|
||||
cute.arch.mbarrier_init(mma_mbar, 1)
|
||||
cute.arch.mbarrier_init(acc_full_mbar, 1)
|
||||
cute.arch.mbarrier_init(acc_empty_mbar, 128)
|
||||
cute.arch.mbarrier_init_fence()
|
||||
elif warp_id == 1:
|
||||
cpasync.prefetch_descriptor(X_tma.atom)
|
||||
@@ -213,40 +222,52 @@ class Sm100BF16x3RouterGemm:
|
||||
|
||||
elif warp_id == 8:
|
||||
# MMA warp
|
||||
stage_id = 0
|
||||
parity = 0
|
||||
tma_stage_id = 0
|
||||
tma_parity = 0
|
||||
|
||||
tmem_acc_count = 0
|
||||
tmem_parity = 1
|
||||
|
||||
idesc = _tcgen05.make_bf16_idesc(BM, BN)
|
||||
sdesc = _tcgen05.make_sdesc_128B_swizzle(0)
|
||||
|
||||
for tile_k in cutlass.range(bid_k, k_tiles, split_k, unroll=1):
|
||||
cute.arch.mbarrier_wait(tma_full_mbar + stage_id, parity)
|
||||
cute.arch.mbarrier_wait(w_full_mbar + stage_id, parity)
|
||||
if tmem_acc_count == 0:
|
||||
cute.arch.mbarrier_wait(acc_empty_mbar, tmem_parity)
|
||||
cute.arch.mbarrier_wait(tma_full_mbar + tma_stage_id, tma_parity)
|
||||
cute.arch.mbarrier_wait(w_full_mbar + tma_stage_id, tma_parity)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
|
||||
w_tmem = w_tmem_base + stage_id * (BK // 2 * 3)
|
||||
x_desc = sdesc | (sX[None, None, stage_id].iterator.toint() >> 4)
|
||||
w_tmem = w_tmem_base + tma_stage_id * (BK // 2 * 3)
|
||||
x_desc = sdesc | (sX[None, None, tma_stage_id].iterator.toint() >> 4)
|
||||
|
||||
for k in cutlass.range_constexpr(BK // 16):
|
||||
enable_d = (tile_k > bid_k) or (k > 0)
|
||||
enable_d_main = (tmem_acc_count > 0) or (k > 0)
|
||||
enable_d_res = (tile_k > bid_k) or (k > 0)
|
||||
_tcgen05.mma_ts_f16(
|
||||
acc_main, w_tmem + k * 8, x_desc, idesc, enable_d
|
||||
acc_main, w_tmem + k * 8, x_desc, idesc, enable_d_main
|
||||
)
|
||||
_tcgen05.mma_ts_f16(
|
||||
acc_res, w_tmem + 32 + k * 8, x_desc, idesc, enable_d
|
||||
acc_res, w_tmem + 32 + k * 8, x_desc, idesc, enable_d_res
|
||||
)
|
||||
_tcgen05.mma_ts_f16(
|
||||
acc_res, w_tmem + 64 + k * 8, x_desc, idesc, True
|
||||
)
|
||||
x_desc += 32 >> 4
|
||||
|
||||
_tcgen05.commit(tma_empty_mbar + stage_id)
|
||||
_tcgen05.commit(tma_empty_mbar + tma_stage_id)
|
||||
|
||||
stage_id = (stage_id + 1) % self.num_stages
|
||||
if stage_id == 0:
|
||||
parity ^= 1
|
||||
tma_stage_id = (tma_stage_id + 1) % num_stages
|
||||
if tma_stage_id == 0:
|
||||
tma_parity ^= 1
|
||||
|
||||
_tcgen05.commit(mma_mbar)
|
||||
tmem_acc_count = (tmem_acc_count + 1) % num_tmem_acc
|
||||
if tmem_acc_count == 0:
|
||||
_tcgen05.commit(acc_full_mbar)
|
||||
tmem_parity ^= 1
|
||||
|
||||
if tmem_acc_count != 0:
|
||||
_tcgen05.commit(acc_full_mbar)
|
||||
|
||||
elif warp_id >= 4:
|
||||
# prep warps: decompose FP32 W into 3xBF16
|
||||
@@ -303,39 +324,93 @@ class Sm100BF16x3RouterGemm:
|
||||
_tcgen05.alloc(taddr)
|
||||
cute.arch.barrier(barrier_id=BAR_TMEM_ALLOC, number_of_threads=128)
|
||||
|
||||
if warp_id == 0:
|
||||
cute.arch.mbarrier_wait(mma_mbar, 0)
|
||||
cute.arch.barrier(barrier_id=BAR_EPI, number_of_threads=128)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
|
||||
cute.arch.griddepcontrol_launch_dependents()
|
||||
tiles_local = cute.ceil_div(k_tiles - bid_k, split_k)
|
||||
num_chunks = cute.ceil_div(tiles_local, num_tmem_acc)
|
||||
|
||||
WIDTH = 8
|
||||
for i in cutlass.range_constexpr(BN // WIDTH):
|
||||
tcol = i * WIDTH
|
||||
main_regs = cute.make_rmem_tensor(WIDTH, Float32)
|
||||
res_regs = cute.make_rmem_tensor(WIDTH, Float32)
|
||||
main_regs.store(_tcgen05.ld(warp_id * 32, tcol, "32x32b", WIDTH))
|
||||
res_regs.store(_tcgen05.ld(warp_id * 32, BN + tcol, "32x32b", WIDTH))
|
||||
_tcgen05.wait_ld()
|
||||
|
||||
# CuteDSL will codegen add.f32x2
|
||||
for j in cutlass.range(WIDTH, vectorize=True):
|
||||
main_regs[j] += res_regs[j]
|
||||
main_regs = cute.make_rmem_tensor(WIDTH, Float32)
|
||||
res_regs = cute.make_rmem_tensor(WIDTH, Float32)
|
||||
|
||||
if num_chunks == 1:
|
||||
# single chunk
|
||||
cute.arch.mbarrier_wait(acc_full_mbar, 0)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
w_row_idx = bid_m * BM + tid
|
||||
for j in cutlass.range_constexpr(WIDTH):
|
||||
x_row_idx = bid_n * BN + i * WIDTH + j
|
||||
if x_row_idx < N and w_row_idx < M:
|
||||
out[bid_k, x_row_idx, w_row_idx] = main_regs[j]
|
||||
for i in cutlass.range_constexpr(BN // WIDTH):
|
||||
tcol = i * WIDTH
|
||||
main_regs.store(_tcgen05.ld(warp_id * 32, tcol, "32x32b", WIDTH))
|
||||
res_regs.store(
|
||||
_tcgen05.ld(warp_id * 32, BN + tcol, "32x32b", WIDTH)
|
||||
)
|
||||
_tcgen05.wait_ld()
|
||||
|
||||
# CuteDSL will codegen add.f32x2
|
||||
for j in cutlass.range(WIDTH, vectorize=True):
|
||||
main_regs[j] += res_regs[j]
|
||||
|
||||
for j in cutlass.range_constexpr(WIDTH):
|
||||
x_row_idx = bid_n * BN + i * WIDTH + j
|
||||
if x_row_idx < N and w_row_idx < M:
|
||||
out[bid_k, x_row_idx, w_row_idx] = main_regs[j]
|
||||
|
||||
else:
|
||||
# multiple chunks
|
||||
master_acc = cute.make_rmem_tensor(BN, Float32)
|
||||
|
||||
# chunk 0: pure TMEM load, no accumulation.
|
||||
cute.arch.mbarrier_wait(acc_full_mbar, 0)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
master_acc.store(_tcgen05.ld(warp_id * 32, 0, "32x32b", BN))
|
||||
_tcgen05.wait_ld()
|
||||
_tcgen05.fence_before_thread_sync()
|
||||
cute.arch.mbarrier_arrive(acc_empty_mbar)
|
||||
|
||||
# accumulate main tmem acc
|
||||
for chunk in cutlass.range(1, num_chunks, unroll=1):
|
||||
cute.arch.mbarrier_wait(acc_full_mbar, chunk & 1)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
|
||||
for i in cutlass.range_constexpr(BN // WIDTH):
|
||||
tcol = i * WIDTH
|
||||
main_regs.store(
|
||||
_tcgen05.ld(warp_id * 32, tcol, "32x32b", WIDTH)
|
||||
)
|
||||
_tcgen05.wait_ld()
|
||||
# CuteDSL refuses to compile
|
||||
# master_acc[i * WIDTH + j] += main_regs[j]
|
||||
# when vectorize=True
|
||||
master_view = cute.local_tile(master_acc, (WIDTH,), (i,))
|
||||
for j in cutlass.range(WIDTH, vectorize=True):
|
||||
master_view[j] += main_regs[j]
|
||||
|
||||
_tcgen05.fence_before_thread_sync()
|
||||
cute.arch.mbarrier_arrive(acc_empty_mbar)
|
||||
|
||||
# fold in the residual accumulator
|
||||
for i in cutlass.range_constexpr(BN // WIDTH):
|
||||
tcol = i * WIDTH
|
||||
res_regs.store(
|
||||
_tcgen05.ld(warp_id * 32, BN + tcol, "32x32b", WIDTH)
|
||||
)
|
||||
_tcgen05.wait_ld()
|
||||
master_view = cute.local_tile(master_acc, (WIDTH,), (i,))
|
||||
for j in cutlass.range(WIDTH, vectorize=True):
|
||||
master_view[j] += res_regs[j]
|
||||
|
||||
w_row_idx = bid_m * BM + tid
|
||||
for j in cutlass.range_constexpr(WIDTH):
|
||||
x_row_idx = bid_n * BN + i * WIDTH + j
|
||||
if x_row_idx < N and w_row_idx < M:
|
||||
out[bid_k, x_row_idx, w_row_idx] = master_acc[i * WIDTH + j]
|
||||
|
||||
cute.arch.griddepcontrol_launch_dependents()
|
||||
cute.arch.barrier(barrier_id=BAR_EPI, number_of_threads=128)
|
||||
if warp_id == 0:
|
||||
_tcgen05.dealloc()
|
||||
|
||||
@cache
|
||||
@staticmethod
|
||||
def compile(BN: int = 128, K: int = 6144):
|
||||
def compile(K: int, BN: int = 128):
|
||||
N = cute.sym_int()
|
||||
M = cute.sym_int()
|
||||
SPLIT_K = cute.sym_int()
|
||||
@@ -392,16 +467,19 @@ def _splitk_reduce_kernel(
|
||||
)
|
||||
|
||||
|
||||
def _splitk_reduce_config(split_k: int) -> tuple[int, int, int]:
|
||||
block_s = triton.next_power_of_2(split_k)
|
||||
if block_s >= 64:
|
||||
return 1, 32, block_s
|
||||
if block_s >= 8:
|
||||
return 1, 256, block_s
|
||||
return min(16, 32 // block_s), 32, block_s
|
||||
|
||||
|
||||
def splitk_reduce_triton(partials: torch.Tensor, out: torch.Tensor):
|
||||
split_k, N, M = partials.shape
|
||||
block_s = 1 << (split_k - 1).bit_length()
|
||||
split_stride = partials.stride(0)
|
||||
if block_s >= 64:
|
||||
BN, BM = 1, 32
|
||||
elif block_s >= 8:
|
||||
BN, BM = 1, 256
|
||||
else:
|
||||
BN, BM = min(16, 32 // block_s), 32
|
||||
BN, BM, block_s = _splitk_reduce_config(split_k)
|
||||
grid = (triton.cdiv(N, BN), triton.cdiv(M, BM))
|
||||
_splitk_reduce_kernel[grid](
|
||||
partials,
|
||||
@@ -419,27 +497,124 @@ def splitk_reduce_triton(partials: torch.Tensor, out: torch.Tensor):
|
||||
)
|
||||
|
||||
|
||||
def bf16x3_router_gemm(X: torch.Tensor, W: torch.Tensor) -> torch.Tensor:
|
||||
"""Return ``X @ W.T`` using the SM100 BF16x3 router GEMM kernel."""
|
||||
N, K = X.shape
|
||||
M, _ = W.shape
|
||||
num_sms = torch.cuda.get_device_properties(X.device).multi_processor_count
|
||||
# Accuracy-constrained (BN, split_k) configs from GB300 sweeps.
|
||||
_TILE_CONFIG_OVERRIDES = {
|
||||
(2816, 256): { # Hunyuan-V4
|
||||
64: (16, 16),
|
||||
128: (32, 16),
|
||||
256: (64, 16),
|
||||
512: (128, 16),
|
||||
},
|
||||
(3072, 256): { # MiniMax-M2
|
||||
64: (16, 16),
|
||||
128: (32, 16),
|
||||
256: (64, 16),
|
||||
512: (128, 16),
|
||||
},
|
||||
(4096, 192): { # Hunyuan-V3
|
||||
64: (32, 32),
|
||||
128: (64, 32),
|
||||
256: (128, 32),
|
||||
512: (128, 19),
|
||||
},
|
||||
(6144, 128): { # MiniMax-M3
|
||||
64: (32, 64),
|
||||
128: (64, 64),
|
||||
256: (64, 32),
|
||||
512: (128, 32),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _pick_tile_config(N: int, K: int, M: int, num_sms: int) -> tuple[int, int]:
|
||||
"""Return (BN, split_k): tuned table mid-range, generic rule otherwise."""
|
||||
k_tiles = math_utils.cdiv(K, 64)
|
||||
|
||||
if 32 < N <= 512:
|
||||
token_bucket = triton.next_power_of_2(N)
|
||||
overrides = _TILE_CONFIG_OVERRIDES.get((K, M))
|
||||
if overrides is not None:
|
||||
BN, split_k = overrides[token_bucket]
|
||||
return BN, min(split_k, k_tiles)
|
||||
|
||||
# next power of 2 within 8 and 128
|
||||
BN = triton.next_power_of_2(N)
|
||||
BN = min(max(BN, 8), 128)
|
||||
|
||||
BM = 128
|
||||
BK = 64
|
||||
k_tiles = math_utils.cdiv(K, BK)
|
||||
grid_m = math_utils.cdiv(M, BM)
|
||||
grid_m = math_utils.cdiv(M, 128)
|
||||
grid_n = math_utils.cdiv(N, BN)
|
||||
|
||||
base_ctas = grid_m * grid_n
|
||||
split_k = min(k_tiles, max(1, num_sms // base_ctas))
|
||||
return BN, split_k
|
||||
|
||||
|
||||
def warmup_bf16x3_router_gemm(
|
||||
K: int,
|
||||
M: int,
|
||||
min_num_tokens: int,
|
||||
max_num_tokens: int,
|
||||
) -> tuple[int, ...]:
|
||||
"""Compile every BF16x3 router GEMM configuration reachable at runtime."""
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
TritonWarmupTensor,
|
||||
triton_scalar_specialization_rep,
|
||||
)
|
||||
|
||||
device = torch.accelerator.current_device_index()
|
||||
num_sms = torch.cuda.get_device_properties(device).multi_processor_count
|
||||
gemm_configs: set[int] = set()
|
||||
reduce_configs: set[tuple[int, int, int, int, int, int]] = set()
|
||||
for N in range(min_num_tokens, max_num_tokens + 1):
|
||||
BN, split_k = _pick_tile_config(N, K, M, num_sms)
|
||||
gemm_configs.add(BN)
|
||||
if split_k == 1:
|
||||
continue
|
||||
reduce_BN, reduce_BM, block_s = _splitk_reduce_config(split_k)
|
||||
reduce_configs.add(
|
||||
(
|
||||
triton_scalar_specialization_rep(N),
|
||||
triton_scalar_specialization_rep(N * M),
|
||||
triton_scalar_specialization_rep(split_k),
|
||||
reduce_BN,
|
||||
reduce_BM,
|
||||
block_s,
|
||||
)
|
||||
)
|
||||
|
||||
configs = tuple(sorted(gemm_configs))
|
||||
for BN in configs:
|
||||
Sm100BF16x3RouterGemm.compile(K, BN)
|
||||
for N, split_stride, split_k, BN, BM, block_s in sorted(reduce_configs):
|
||||
partials = TritonWarmupTensor(torch.float32)
|
||||
out = TritonWarmupTensor(torch.float32)
|
||||
_splitk_reduce_kernel.warmup(
|
||||
partials,
|
||||
out,
|
||||
N,
|
||||
M,
|
||||
split_stride,
|
||||
split_k,
|
||||
BN=BN,
|
||||
BM=BM,
|
||||
BS=block_s,
|
||||
USE_PDL=True,
|
||||
num_warps=4,
|
||||
launch_pdl=True,
|
||||
grid=(1, 1),
|
||||
)
|
||||
return configs
|
||||
|
||||
|
||||
def _bf16x3_router_gemm(X: torch.Tensor, W: torch.Tensor) -> torch.Tensor:
|
||||
"""Return ``X @ W.T`` using the SM100 BF16x3 router GEMM kernel."""
|
||||
N, K = X.shape
|
||||
M, _ = W.shape
|
||||
num_sms = torch.cuda.get_device_properties(X.device).multi_processor_count
|
||||
|
||||
BN, split_k = _pick_tile_config(N, K, M, num_sms)
|
||||
|
||||
partials = X.new_empty(split_k, N, M, dtype=torch.float32)
|
||||
Sm100BF16x3RouterGemm.compile(BN, K)(X, W, partials, split_k)
|
||||
Sm100BF16x3RouterGemm.compile(K, BN)(X, W, partials, split_k)
|
||||
|
||||
if split_k == 1:
|
||||
return partials.squeeze(0)
|
||||
@@ -447,3 +622,21 @@ def bf16x3_router_gemm(X: torch.Tensor, W: torch.Tensor) -> torch.Tensor:
|
||||
out = X.new_empty(N, M, dtype=torch.float32)
|
||||
splitk_reduce_triton(partials, out)
|
||||
return out
|
||||
|
||||
|
||||
def _bf16x3_router_gemm_fake(
|
||||
X: torch.Tensor,
|
||||
W: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return X.new_empty((X.shape[0], W.shape[0]), dtype=torch.float32)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="bf16x3_router_gemm",
|
||||
op_func=_bf16x3_router_gemm,
|
||||
fake_impl=_bf16x3_router_gemm_fake,
|
||||
)
|
||||
|
||||
|
||||
def bf16x3_router_gemm(X: torch.Tensor, W: torch.Tensor) -> torch.Tensor:
|
||||
return torch.ops.vllm.bf16x3_router_gemm(X, W)
|
||||
|
||||
@@ -4,15 +4,11 @@ import torch
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
from vllm.config import get_current_vllm_config_or_none
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@PluggableLayer.register("gate_linear")
|
||||
class GateLinear(ReplicatedLinear):
|
||||
@@ -22,7 +18,7 @@ class GateLinear(ReplicatedLinear):
|
||||
K divisible by 8)
|
||||
2. fp32 specialized kernel (SM90+ or gfx950, bf16/fp32 in, fp32 out,
|
||||
M<=32, model-specific shapes)
|
||||
3. experimental bf16x3 CuteDSL kernel (opt-in, SM100, bf16 in, fp32 weight)
|
||||
3. bf16x3 CuteDSL kernel (SM100, bf16 in, fp32 weight)
|
||||
4. cuBLAS bf16×bf16→fp32 (SM90+ + bf16 weight + fp32 out_dtype)
|
||||
5. F.linear via ReplicatedLinear (ultimate fallback)
|
||||
|
||||
@@ -85,11 +81,6 @@ class GateLinear(ReplicatedLinear):
|
||||
self.allow_specialized_router_gemm = can_use_specialized_kernels
|
||||
|
||||
# fp32 specialized kernel eligibility (exact dims, fp32 weight)
|
||||
vllm_config = get_current_vllm_config_or_none()
|
||||
enable_bf16x3_router_gemm = (
|
||||
vllm_config is not None
|
||||
and vllm_config.kernel_config.enable_bf16x3_router_gemm
|
||||
)
|
||||
self.allow_fp32_router_gemm = (
|
||||
not bias
|
||||
and self.weight.dtype == torch.float32
|
||||
@@ -108,10 +99,7 @@ class GateLinear(ReplicatedLinear):
|
||||
and current_platform.is_cuda()
|
||||
and is_blackwell
|
||||
and input_size % 8 == 0
|
||||
and enable_bf16x3_router_gemm
|
||||
)
|
||||
if self.allow_bf16x3_router_gemm:
|
||||
logger.info_once("Enabled experimental SM100 BF16x3 router GEMM.")
|
||||
|
||||
# Fused bf16 x bf16 -> fp32 GEMM eligibility. torch.mm's out_dtype
|
||||
# epilogue folds the fp32 cast into the GEMM, removing the standalone
|
||||
@@ -198,7 +186,7 @@ class GateLinear(ReplicatedLinear):
|
||||
)
|
||||
return output, None
|
||||
|
||||
# Tier 3: experimental bf16x3 CuteDSL kernel for fp32 router weights
|
||||
# Tier 3: bf16x3 CuteDSL kernel for fp32 router weights
|
||||
if self.allow_bf16x3_router_gemm and x.dtype == torch.bfloat16:
|
||||
from vllm.model_executor.layers.fused_moe.router.bf16x3_router_gemm_cutedsl import ( # noqa: E501
|
||||
bf16x3_router_gemm,
|
||||
@@ -251,10 +239,10 @@ def fp32_router_gemm_dispatch_impl(
|
||||
|
||||
if allow_bf16x3_router_gemm and x.dtype == torch.bfloat16:
|
||||
from vllm.model_executor.layers.fused_moe.router.bf16x3_router_gemm_cutedsl import ( # noqa: E501
|
||||
bf16x3_router_gemm,
|
||||
_bf16x3_router_gemm,
|
||||
)
|
||||
|
||||
return bf16x3_router_gemm(x, weight)
|
||||
return _bf16x3_router_gemm(x, weight)
|
||||
|
||||
return torch.nn.functional.linear(x.float(), weight)
|
||||
|
||||
|
||||
@@ -101,6 +101,44 @@ def _warmup_ll_bf16_router_gemm(model: torch.nn.Module) -> None:
|
||||
)
|
||||
|
||||
|
||||
def _warmup_bf16x3_router_gemm(
|
||||
model: torch.nn.Module,
|
||||
max_num_tokens: int,
|
||||
) -> None:
|
||||
from vllm.model_executor.layers.fused_moe.router.bf16x3_router_gemm_cutedsl import ( # noqa: E501
|
||||
warmup_bf16x3_router_gemm,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
|
||||
|
||||
gate = next(
|
||||
(
|
||||
module
|
||||
for module in model.modules()
|
||||
if isinstance(module, GateLinear) and module.allow_bf16x3_router_gemm
|
||||
),
|
||||
None,
|
||||
)
|
||||
if gate is None:
|
||||
logger.debug_once(
|
||||
"Skipping BF16x3 router GEMM warmup: no eligible GateLinear found."
|
||||
)
|
||||
return
|
||||
|
||||
min_num_tokens = gate.FP32_MAX_TOKENS + 1 if gate.allow_fp32_router_gemm else 1
|
||||
logger.info_once(
|
||||
"Warming up BF16x3 router GEMM for K=%d, M=%d.",
|
||||
gate.input_size,
|
||||
gate.output_size,
|
||||
)
|
||||
configs = warmup_bf16x3_router_gemm(
|
||||
gate.input_size,
|
||||
gate.output_size,
|
||||
min_num_tokens,
|
||||
max_num_tokens,
|
||||
)
|
||||
logger.info_once("Warmed up BF16x3 router GEMM configs: %s.", configs)
|
||||
|
||||
|
||||
def _warmup_kimi_k3_gemm_rs_ar() -> None:
|
||||
# Kimi-K3 model construction imports this module only when GEMM-RS/AR is
|
||||
# enabled and initializes its singleton before kernel_warmup runs. Avoid
|
||||
@@ -131,7 +169,8 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
|
||||
if zeroer is not None:
|
||||
zeroer.warmup(worker.model_runner.kv_cache_config.num_blocks)
|
||||
|
||||
if worker.vllm_config.kernel_config.enable_jit_warmup:
|
||||
enable_jit_warmup = worker.vllm_config.kernel_config.enable_jit_warmup
|
||||
if enable_jit_warmup:
|
||||
logger.info("JIT kernel warmup starting.")
|
||||
jit_warmup_start = time.perf_counter()
|
||||
try:
|
||||
@@ -164,11 +203,17 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
|
||||
)
|
||||
|
||||
# Run next so input-prep kernels JIT against pristine runner state.
|
||||
if worker.vllm_config.kernel_config.enable_jit_warmup:
|
||||
if enable_jit_warmup:
|
||||
kimi_k3_triton_warmup(worker)
|
||||
spec_decode_rejection_warmup(worker)
|
||||
qwen4_exp_qsa_triton_warmup(worker)
|
||||
|
||||
if enable_jit_warmup and current_platform.is_device_capability_family(100):
|
||||
_warmup_bf16x3_router_gemm(
|
||||
worker.get_model(),
|
||||
worker.scheduler_config.max_num_batched_tokens,
|
||||
)
|
||||
|
||||
if current_platform.has_device_capability(90):
|
||||
_warmup_ll_bf16_router_gemm(worker.get_model())
|
||||
|
||||
|
||||
Reference in New Issue
Block a user