[Kernel] Add router GEMM kernels for (5120, 256) and BF16 output to ll_bf16_gemm (#62042)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Lain <fusiyuan2000@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: vllm-agent <claw@inferact.ai>
This commit is contained in:
Woosuk Kwon
2026-10-16 15:39:46 -07:00
committed by GitHub
co-authored by Lain Codex Claude Opus 5.5 vllm-agent
parent 153a17166e
commit 596e530675
12 changed files with 258 additions and 64 deletions
+49 -5
View File
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
//
// Router GEMM: activation(T) x weight(fp32) -> fp32, M<=32, for the
// Router GEMM: activation(T) x weight(fp32) -> fp32, M<=48, for the
// supported (E, H) pairs listed at the bottom of this file.
// Supports bf16 or fp32 activation; weight is always fp32.
@@ -214,8 +214,26 @@ void invokeFp32RouterGemm(float* output, InputT const* mat_a,
// production-fidelity harness (CUDA-graph replay, per-layer cold weights).
// Only enabled on the Blackwell family where it was validated; Hopper and
// other shapes / fp32 activation keep the legacy geometry.
if constexpr (std::is_same_v<InputT, __nv_bfloat16> && kNumExperts == 128 &&
kHiddenDim == 6144) {
if constexpr (std::is_same_v<InputT, __nv_bfloat16> && kNumExperts == 256 &&
kHiddenDim == 5120) {
if (!isBlackwellFamily()) {
launchFp32RouterGemm<InputT, 128, 1, kNumTokens, kNumExperts, kHiddenDim>(
output, mat_a, mat_b, stream);
return;
}
if constexpr (kNumTokens > 16 && kNumTokens % 8 == 0) {
constexpr int block_size = kNumTokens == 24 ? 160 : 128;
launchFp32RouterGemm<InputT, block_size, 2, kNumTokens, kNumExperts,
kHiddenDim, 4>(output, mat_a, mat_b, stream);
} else {
constexpr int block_size = kNumTokens == 1 ? 64 : 128;
constexpr int token_groups = kNumTokens % 2 == 0 ? 2 : 1;
launchFp32RouterGemm<InputT, block_size, 1, kNumTokens, kNumExperts,
kHiddenDim, token_groups>(output, mat_a, mat_b,
stream);
}
} else if constexpr (std::is_same_v<InputT, __nv_bfloat16> &&
kNumExperts == 128 && kHiddenDim == 6144) {
// MiniMax-M3. Legacy 128/1 only fills 128 blocks and pays the same
// accumulator register cliffs; B300 sweep:
// even M in [6, 10] : BS=384, EPB=1, 2 token groups (1.26-1.43x)
@@ -268,8 +286,8 @@ void invokeFp32RouterGemm(float* output, InputT const* mat_a,
}
// ---------------------------------------------------------------------------
// Explicit instantiations: M=1..32, for both input types, for the supported
// (E, H) pairs: (256, 3072) [MiniMax-M2/M2.5] and (128, 6144) [MiniMax-M3].
// Explicit instantiations for both input types.
// Supported (E, H) pairs: (256, 3072), (128, 6144), and (256, 5120).
// ---------------------------------------------------------------------------
#define INSTANTIATE(T, M, E, H) \
@@ -315,5 +333,31 @@ INSTANTIATE_ALL(__nv_bfloat16, 256, 3072)
INSTANTIATE_ALL(float, 128, 6144)
INSTANTIATE_ALL(__nv_bfloat16, 128, 6144)
INSTANTIATE_ALL(float, 256, 5120)
INSTANTIATE_ALL(__nv_bfloat16, 256, 5120)
#define INSTANTIATE_LARGE(T, E, H) \
INSTANTIATE(T, 33, E, H) \
INSTANTIATE(T, 34, E, H) \
INSTANTIATE(T, 35, E, H) \
INSTANTIATE(T, 36, E, H) \
INSTANTIATE(T, 37, E, H) \
INSTANTIATE(T, 38, E, H) \
INSTANTIATE(T, 39, E, H) \
INSTANTIATE(T, 40, E, H) \
INSTANTIATE(T, 41, E, H) \
INSTANTIATE(T, 42, E, H) \
INSTANTIATE(T, 43, E, H) \
INSTANTIATE(T, 44, E, H) \
INSTANTIATE(T, 45, E, H) \
INSTANTIATE(T, 46, E, H) \
INSTANTIATE(T, 47, E, H) \
INSTANTIATE(T, 48, E, H)
INSTANTIATE_LARGE(float, 256, 5120)
INSTANTIATE_LARGE(__nv_bfloat16, 256, 5120)
#undef INSTANTIATE_LARGE
#undef INSTANTIATE_ALL
#undef INSTANTIATE
+12 -7
View File
@@ -25,10 +25,11 @@ inline int getSMVersion() {
static constexpr int FP32_MAX_TOKENS = 32;
// Supported (hidden_dim, num_experts) pairs (must match the instantiations in
// fp32_router_gemm.cu): (3072, 256) for MiniMax-M2/M2.5, (6144, 128) for M3.
// fp32_router_gemm.cu).
static inline bool fp32_router_gemm_supported(int hidden_dim, int num_experts) {
return (hidden_dim == 3072 && num_experts == 256) ||
(hidden_dim == 6144 && num_experts == 128);
(hidden_dim == 6144 && num_experts == 128) ||
(hidden_dim == 5120 && num_experts == 256);
}
// Forward declarations — 4 template params must match fp32_router_gemm.cu
@@ -60,8 +61,7 @@ struct Fp32LoopUnroller<InputT, kNumExperts, kHiddenDim, kEnd, kEnd> {
invokeFp32RouterGemm<InputT, kEnd, kNumExperts, kHiddenDim>(
output, mat_a, mat_b, stream);
} else {
throw std::invalid_argument(
"fp32_router_gemm: num_tokens must be in [1, 32]");
throw std::invalid_argument("fp32_router_gemm: unsupported num_tokens");
}
}
};
@@ -77,6 +77,9 @@ void dispatchFp32RouterGemm(int num_experts, int hidden_dim, int num_tokens,
} else if (num_experts == 128 && hidden_dim == 6144) {
Fp32LoopUnroller<InputT, 128, 6144, 1, FP32_MAX_TOKENS>::unroll(
num_tokens, output, mat_a, mat_b, stream);
} else if (num_experts == 256 && hidden_dim == 5120) {
Fp32LoopUnroller<InputT, 256, 5120, 1, 48>::unroll(num_tokens, output,
mat_a, mat_b, stream);
} else {
throw std::invalid_argument(
"fp32_router_gemm: unsupported (hidden_dim, num_experts) pair");
@@ -111,9 +114,11 @@ void fp32_router_gemm(
STD_TORCH_CHECK(
fp32_router_gemm_supported(hidden_dim, num_experts),
"fp32_router_gemm: supported (hidden_dim, num_experts) pairs are "
"(3072, 256) and (6144, 128)");
STD_TORCH_CHECK(num_tokens <= FP32_MAX_TOKENS,
"fp32_router_gemm: num_tokens must be in [0, 32]");
"(3072, 256), (6144, 128), and (5120, 256)");
const int max_tokens =
hidden_dim == 5120 && num_experts == 256 ? 48 : FP32_MAX_TOKENS;
STD_TORCH_CHECK(num_tokens <= max_tokens,
"fp32_router_gemm: num_tokens exceeds supported range");
STD_TORCH_CHECK(
mat_a.scalar_type() == torch::headeronly::ScalarType::Float ||
mat_a.scalar_type() == torch::headeronly::ScalarType::BFloat16,