[Kernel] Add router GEMM kernels for (5120, 256) and BF16 output to ll_bf16_gemm (#62042)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Lain <fusiyuan2000@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: vllm-agent <claw@inferact.ai>
This commit is contained in:
Woosuk Kwon
2026-10-16 15:39:46 -07:00
committed by GitHub
co-authored by Lain Codex Claude Opus 5.5 vllm-agent
parent 153a17166e
commit 596e530675
12 changed files with 258 additions and 64 deletions
@@ -356,10 +356,12 @@ class LLBf16Dotprod:
if tidx == 0:
for m in cutlass.range_constexpr(M):
partials = sm[m, None].load()
gC[m, n_idx] = partials.reduce(
cute.ReductionOp.ADD,
init_val=cutlass.Float32(0.0),
reduction_profile=0,
)
gC[m, n_idx] = cutlass.Float32(
partials.reduce(
cute.ReductionOp.ADD,
init_val=cutlass.Float32(0.0),
reduction_profile=0,
)
).to(gC.element_type)
if const_expr(self.use_pdl):
cute.arch.griddepcontrol_launch_dependents()
@@ -47,7 +47,7 @@ class LLBf16SplitK:
:param ab_dtype: Element type for A and B operands.
:param acc_dtype: Accumulator type used by MMA and reductions.
:param out_dtype: Output element type. The public wrapper uses fp32.
:param out_dtype: Output element type (fp32 or bf16).
:param tile_n: CTA tile size in N. M is fixed to 16 for router batches.
:type tile_n: int
:param tile_k: K tile size staged through shared memory.
@@ -580,6 +580,6 @@ class LLBf16SplitK:
reduction_profile=0,
)
)
gC[local_coord] = acc
gC[local_coord] = cutlass.Float32(acc).to(gC.element_type)
cute.arch.sync_threads()
@@ -76,8 +76,8 @@ def make_fake_gemm_tensors(
a_dtype: torch.dtype,
b_dtype: torch.dtype,
divisibility: int,
output_dtype: torch.dtype = torch.float32,
):
from cutlass import Float32
from quack.compile_utils import make_fake_tensor
hidden_states = make_fake_tensor(
@@ -86,7 +86,7 @@ def make_fake_gemm_tensors(
router_weight = make_fake_tensor(
cutlass_dtype(b_dtype), (N, K), divisibility=divisibility
)
output = make_fake_tensor(Float32, (M, N), divisibility=1)
output = make_fake_tensor(cutlass_dtype(output_dtype), (M, N), divisibility=1)
return hidden_states, router_weight, output
@@ -97,6 +97,7 @@ def validate_common_gemm_inputs(
*,
op_name: str,
k_multiple: int | None = None,
output_dtypes: tuple[torch.dtype, ...] = (torch.float32,),
) -> None:
if hidden_states.dim() != 2 or router_weight.dim() != 2:
raise ValueError("hidden_states and router_weight must be 2D tensors")
@@ -106,8 +107,11 @@ def validate_common_gemm_inputs(
raise ValueError(
"hidden_states and router_weight must be on the same CUDA device"
)
if output_dtype != torch.float32:
raise ValueError(f"{op_name} only supports output_dtype=torch.float32")
if output_dtype not in output_dtypes:
raise ValueError(
f"{op_name} only supports output_dtype="
+ " or ".join(str(dtype) for dtype in output_dtypes)
)
if hidden_states.shape[1] != router_weight.shape[1]:
raise ValueError(
"hidden_states and router_weight must have matching K dimensions"
@@ -47,6 +47,8 @@ _SM100F_TUNED_SPLITK_CONFIGS: dict[tuple[int, int], dict[int, tuple[int, int]]]
**{M: (8, 5) for M in (5, 8)},
9: (8, 2),
},
(5120, 1792): {M: (2, 4) for M in range(5, 17)},
(5120, 3584): {M: (2, 4) for M in range(5, 17)},
(7168, 256): {14: (8, 2)},
(6144, 256): {M: (8, 2) for M in (9, 12, 16)},
(7168, 384): {M: (7, 5) for M in (13, 16)},
@@ -101,39 +103,52 @@ class LLBf16Gemm:
bs: int = 0
split_k: int = 0
num_stages: int = 0
output_dtype: torch.dtype = torch.float32
def __init__(self, *, prefetch_pdl_weights: bool = False) -> None:
self._prefetch_pdl_weights = prefetch_pdl_weights
# Dot-prod: keyed on (M, K, bs), because M and K are Constexpr.
self._compiled_cache: dict[tuple[int, int, int], Any] = {}
# Split-K: keyed on (split_k, num_stages), fully shape-dynamic.
self._splitk_cache: dict[tuple[int, int], Any] = {}
# M and K are compile-time constants for dot-product.
self._compiled_cache: dict[tuple[int, int, int, torch.dtype], Any] = {}
# Split-K is shape-dynamic; both caches distinguish output dtype.
self._splitk_cache: dict[tuple[int, int, torch.dtype], Any] = {}
def dispatch(self, *, M: int, K: int, N: int) -> CompileKey:
def dispatch(
self, *, M: int, K: int, N: int, output_dtype: torch.dtype = torch.float32
) -> CompileKey:
tuned_bs, tuned_splitk = _arch_tuned_configs()
if M <= _DEFAULT_DOTPROD_MAX_M or K < 2048:
bs = tuned_bs.get((K, N), {}).get(M, _DEFAULT_DOTPROD_BS)
return self.CompileKey(backend="dotprod", M=M, K=K, bs=bs)
return self.CompileKey(
backend="dotprod", M=M, K=K, bs=bs, output_dtype=output_dtype
)
split_k, num_stages = tuned_splitk.get((K, N), {}).get(
M, _DEFAULT_SPLITK_CONFIG
)
return self.CompileKey(backend="splitk", split_k=split_k, num_stages=num_stages)
return self.CompileKey(
backend="splitk",
split_k=split_k,
num_stages=num_stages,
output_dtype=output_dtype,
)
def get_warmup_keys(
self,
*,
shapes: Iterable[tuple[int, int]],
m_values: Iterable[int],
output_dtype: torch.dtype = torch.float32,
) -> list[CompileKey]:
return list(
dict.fromkeys(
self.dispatch(M=M, K=K, N=N) for K, N in shapes for M in m_values
self.dispatch(M=M, K=K, N=N, output_dtype=output_dtype)
for K, N in shapes
for M in m_values
)
)
@staticmethod
def _fake_gemm_tensors(*, M, K, N, divisibility: int):
def _fake_gemm_tensors(*, M, K, N, divisibility: int, output_dtype: torch.dtype):
return make_fake_gemm_tensors(
M=M,
K=K,
@@ -141,10 +156,13 @@ class LLBf16Gemm:
a_dtype=torch.bfloat16,
b_dtype=torch.bfloat16,
divisibility=divisibility,
output_dtype=output_dtype,
)
def _compile_splitk(self, compile_key: CompileKey) -> None:
cute, _ = cute_context()
from cutlass import BFloat16, Float32
from ._ll_bf16_splitk import LLBf16SplitK
hidden_states, router_weight, output = self._fake_gemm_tensors(
@@ -152,8 +170,12 @@ class LLBf16Gemm:
K=cute.sym_int(),
N=cute.sym_int(),
divisibility=8,
output_dtype=compile_key.output_dtype,
)
gemm = LLBf16SplitK(
out_dtype=BFloat16
if compile_key.output_dtype == torch.bfloat16
else Float32,
tile_n=16,
tile_k=256,
num_stages=compile_key.num_stages,
@@ -169,7 +191,9 @@ class LLBf16Gemm:
current_cuda_stream(),
options="--enable-tvm-ffi",
)
self._splitk_cache[(compile_key.split_k, compile_key.num_stages)] = compiled
self._splitk_cache[
(compile_key.split_k, compile_key.num_stages, compile_key.output_dtype)
] = compiled
logger.debug(
"Compiled ll_bf16_splitk: sk=%d ns=%d",
compile_key.split_k,
@@ -187,6 +211,7 @@ class LLBf16Gemm:
K=compile_key.K,
N=N,
divisibility=stride_divisibility,
output_dtype=compile_key.output_dtype,
)
gemm = LLBf16Dotprod(
k=compile_key.K,
@@ -205,7 +230,9 @@ class LLBf16Gemm:
current_cuda_stream(),
options="--enable-tvm-ffi --ptxas-options -maxrregcount=64",
)
self._compiled_cache[(compile_key.M, compile_key.K, compile_key.bs)] = compiled
self._compiled_cache[
(compile_key.M, compile_key.K, compile_key.bs, compile_key.output_dtype)
] = compiled
logger.debug(
"Compiled ll_bf16_dotprod: M=%d, K=%d, bs=%d",
compile_key.M,
@@ -215,12 +242,21 @@ class LLBf16Gemm:
def compile(self, compile_key: CompileKey) -> None:
if compile_key.backend == "splitk":
splitk_cache_key = (compile_key.split_k, compile_key.num_stages)
splitk_cache_key = (
compile_key.split_k,
compile_key.num_stages,
compile_key.output_dtype,
)
if splitk_cache_key not in self._splitk_cache:
self._compile_splitk(compile_key)
return
dotprod_cache_key = (compile_key.M, compile_key.K, compile_key.bs)
dotprod_cache_key = (
compile_key.M,
compile_key.K,
compile_key.bs,
compile_key.output_dtype,
)
if dotprod_cache_key not in self._compiled_cache:
self._compile_dotprod(compile_key)
@@ -229,8 +265,11 @@ class LLBf16Gemm:
*,
shapes: Iterable[tuple[int, int]],
m_values: Iterable[int],
output_dtype: torch.dtype = torch.float32,
) -> None:
for compile_key in self.get_warmup_keys(shapes=shapes, m_values=m_values):
for compile_key in self.get_warmup_keys(
shapes=shapes, m_values=m_values, output_dtype=output_dtype
):
self.compile(compile_key)
@staticmethod
@@ -251,6 +290,7 @@ class LLBf16Gemm:
output_dtype,
op_name="ll_bf16_gemm",
k_multiple=8,
output_dtypes=(torch.float32, torch.bfloat16),
)
def __call__(
@@ -258,19 +298,28 @@ class LLBf16Gemm:
hidden_states: torch.Tensor, # [M, K] bf16
router_weight: torch.Tensor, # [N, K] bf16
output_dtype: torch.dtype = torch.float32,
) -> torch.Tensor: # [M, N] fp32
) -> torch.Tensor: # [M, N]
self._validate_inputs(hidden_states, router_weight, output_dtype)
M, K = hidden_states.shape
N = router_weight.shape[0]
compile_key = self.dispatch(M=M, K=K, N=N)
compile_key = self.dispatch(M=M, K=K, N=N, output_dtype=output_dtype)
if compile_key.backend == "splitk":
splitk_cache_key = (compile_key.split_k, compile_key.num_stages)
splitk_cache_key = (
compile_key.split_k,
compile_key.num_stages,
compile_key.output_dtype,
)
if splitk_cache_key not in self._splitk_cache:
self.compile(compile_key)
kernel = self._splitk_cache[splitk_cache_key]
else:
dotprod_cache_key = (compile_key.M, compile_key.K, compile_key.bs)
dotprod_cache_key = (
compile_key.M,
compile_key.K,
compile_key.bs,
compile_key.output_dtype,
)
if dotprod_cache_key not in self._compiled_cache:
self.compile(compile_key)
kernel = self._compiled_cache[dotprod_cache_key]
@@ -517,6 +517,7 @@ _TILE_CONFIG_OVERRIDES = {
256: (128, 32),
512: (128, 19),
},
(5120, 256): {64: (64, 64)}, # GB200
(6144, 128): { # MiniMax-M3
64: (32, 64),
128: (64, 64),
@@ -533,7 +534,7 @@ def _pick_tile_config(N: int, K: int, M: int, num_sms: int) -> tuple[int, int]:
if 32 < N <= 512:
token_bucket = triton.next_power_of_2(N)
overrides = _TILE_CONFIG_OVERRIDES.get((K, M))
if overrides is not None:
if overrides is not None and token_bucket in overrides:
BN, split_k = overrides[token_bucket]
return BN, min(split_k, k_tiles)
@@ -609,6 +610,8 @@ def _bf16x3_router_gemm(X: torch.Tensor, W: torch.Tensor) -> torch.Tensor:
"""Return ``X @ W.T`` using the SM100 BF16x3 router GEMM kernel."""
N, K = X.shape
M, _ = W.shape
if N == 0:
return X.new_empty((N, M), dtype=torch.float32)
num_sms = torch.cuda.get_device_properties(X.device).multi_processor_count
BN, split_k = _pick_tile_config(N, K, M, num_sms)
@@ -24,8 +24,8 @@ class GateLinear(ReplicatedLinear):
1. cuteDSL ll_bf16_gemm (SM90+, M<=16, bf16 in, fp32 out,
any shape w/ K divisible by 8)
2. fp32 specialized kernel (bf16/fp32 in, fp32 out, M<=32; SM90+
model-specific shapes, gfx950 any shape)
2. fp32 specialized kernel (bf16/fp32 in, fp32 out; SM90+
model-specific shapes and batch limits, gfx950 any shape with M<=32)
3. cuteDSL ll_fp32w_gemm (SM90+, M<=32, bf16/fp32 in, fp32 out,
any shape)
4. bf16x3 CuteDSL kernel (SM100, bf16 in, fp32 weight)
@@ -46,9 +46,8 @@ class GateLinear(ReplicatedLinear):
split-K from the token count and skip the batch-invariant linear path.
"""
# (hidden_size, num_experts) pairs with an instantiated fp32 kernel:
# (3072, 256) -> MiniMax-M2/M2.5, (6144, 128) -> MiniMax-M3
FP32_SUPPORTED_SHAPES = {(3072, 256), (6144, 128)}
# (hidden_size, num_experts) pairs with an instantiated fp32 kernel.
FP32_SUPPORTED_SHAPES = {(3072, 256), (6144, 128), (5120, 256)}
FP32_MAX_TOKENS = 32
# Largest batch for which tier 1 beats the cuBLAS epilogue. Subclasses
# whose router shape stays ahead further up the range may raise it.
@@ -454,9 +453,9 @@ def fp32_router_gemm_dispatch_impl(
weight: torch.Tensor,
allow_bf16x3_router_gemm: bool,
) -> torch.Tensor:
"""Dynamically run fp32 specialized gemm if num_tokens <= FP32_MAX_TOKENS,
otherwise optionally run the experimental BF16x3 kernel for medium/large
SM100 router batches, then fall back to F.linear.
"""Run the FP32 kernel for small batches, with shape-specific limits.
Larger SM100 BF16 batches use BF16x3; other inputs fall back to F.linear.
This must be wrapped in a custom op because our torch.compile integration
does not support runtime dispatching on num_tokens.
"""
@@ -473,7 +472,15 @@ def fp32_router_gemm_dispatch_impl(
if weight.dtype == torch.bfloat16:
return torch.mm(x, weight.T, out_dtype=torch.float32)
return torch.nn.functional.linear(x.float(), weight)
if x.shape[0] <= _FP32_ROUTER_GEMM_MAX_TOKENS:
max_tokens = _FP32_ROUTER_GEMM_MAX_TOKENS
if (
allow_bf16x3_router_gemm
and x.dtype == torch.bfloat16
and weight.shape == (256, 5120)
):
# GB200: four token groups win through 48 for multiples of eight.
max_tokens = 48 if x.shape[0] % 8 == 0 else 16
if x.shape[0] <= max_tokens:
return ops.fp32_router_gemm(x, weight)
if allow_bf16x3_router_gemm and x.dtype == torch.bfloat16: