[Kernel] Add router GEMM kernels for (5120, 256) and BF16 output to ll_bf16_gemm (#62042)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Lain <fusiyuan2000@gmail.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: vllm-agent <claw@inferact.ai>
This commit is contained in:
co-authored by
Lain
Codex
Claude Opus 5.5
vllm-agent
parent
153a17166e
commit
596e530675
@@ -1,7 +1,7 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
//
|
||||
// Router GEMM: activation(T) x weight(fp32) -> fp32, M<=32, for the
|
||||
// Router GEMM: activation(T) x weight(fp32) -> fp32, M<=48, for the
|
||||
// supported (E, H) pairs listed at the bottom of this file.
|
||||
// Supports bf16 or fp32 activation; weight is always fp32.
|
||||
|
||||
@@ -214,8 +214,26 @@ void invokeFp32RouterGemm(float* output, InputT const* mat_a,
|
||||
// production-fidelity harness (CUDA-graph replay, per-layer cold weights).
|
||||
// Only enabled on the Blackwell family where it was validated; Hopper and
|
||||
// other shapes / fp32 activation keep the legacy geometry.
|
||||
if constexpr (std::is_same_v<InputT, __nv_bfloat16> && kNumExperts == 128 &&
|
||||
kHiddenDim == 6144) {
|
||||
if constexpr (std::is_same_v<InputT, __nv_bfloat16> && kNumExperts == 256 &&
|
||||
kHiddenDim == 5120) {
|
||||
if (!isBlackwellFamily()) {
|
||||
launchFp32RouterGemm<InputT, 128, 1, kNumTokens, kNumExperts, kHiddenDim>(
|
||||
output, mat_a, mat_b, stream);
|
||||
return;
|
||||
}
|
||||
if constexpr (kNumTokens > 16 && kNumTokens % 8 == 0) {
|
||||
constexpr int block_size = kNumTokens == 24 ? 160 : 128;
|
||||
launchFp32RouterGemm<InputT, block_size, 2, kNumTokens, kNumExperts,
|
||||
kHiddenDim, 4>(output, mat_a, mat_b, stream);
|
||||
} else {
|
||||
constexpr int block_size = kNumTokens == 1 ? 64 : 128;
|
||||
constexpr int token_groups = kNumTokens % 2 == 0 ? 2 : 1;
|
||||
launchFp32RouterGemm<InputT, block_size, 1, kNumTokens, kNumExperts,
|
||||
kHiddenDim, token_groups>(output, mat_a, mat_b,
|
||||
stream);
|
||||
}
|
||||
} else if constexpr (std::is_same_v<InputT, __nv_bfloat16> &&
|
||||
kNumExperts == 128 && kHiddenDim == 6144) {
|
||||
// MiniMax-M3. Legacy 128/1 only fills 128 blocks and pays the same
|
||||
// accumulator register cliffs; B300 sweep:
|
||||
// even M in [6, 10] : BS=384, EPB=1, 2 token groups (1.26-1.43x)
|
||||
@@ -268,8 +286,8 @@ void invokeFp32RouterGemm(float* output, InputT const* mat_a,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Explicit instantiations: M=1..32, for both input types, for the supported
|
||||
// (E, H) pairs: (256, 3072) [MiniMax-M2/M2.5] and (128, 6144) [MiniMax-M3].
|
||||
// Explicit instantiations for both input types.
|
||||
// Supported (E, H) pairs: (256, 3072), (128, 6144), and (256, 5120).
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#define INSTANTIATE(T, M, E, H) \
|
||||
@@ -315,5 +333,31 @@ INSTANTIATE_ALL(__nv_bfloat16, 256, 3072)
|
||||
INSTANTIATE_ALL(float, 128, 6144)
|
||||
INSTANTIATE_ALL(__nv_bfloat16, 128, 6144)
|
||||
|
||||
INSTANTIATE_ALL(float, 256, 5120)
|
||||
INSTANTIATE_ALL(__nv_bfloat16, 256, 5120)
|
||||
|
||||
#define INSTANTIATE_LARGE(T, E, H) \
|
||||
INSTANTIATE(T, 33, E, H) \
|
||||
INSTANTIATE(T, 34, E, H) \
|
||||
INSTANTIATE(T, 35, E, H) \
|
||||
INSTANTIATE(T, 36, E, H) \
|
||||
INSTANTIATE(T, 37, E, H) \
|
||||
INSTANTIATE(T, 38, E, H) \
|
||||
INSTANTIATE(T, 39, E, H) \
|
||||
INSTANTIATE(T, 40, E, H) \
|
||||
INSTANTIATE(T, 41, E, H) \
|
||||
INSTANTIATE(T, 42, E, H) \
|
||||
INSTANTIATE(T, 43, E, H) \
|
||||
INSTANTIATE(T, 44, E, H) \
|
||||
INSTANTIATE(T, 45, E, H) \
|
||||
INSTANTIATE(T, 46, E, H) \
|
||||
INSTANTIATE(T, 47, E, H) \
|
||||
INSTANTIATE(T, 48, E, H)
|
||||
|
||||
INSTANTIATE_LARGE(float, 256, 5120)
|
||||
INSTANTIATE_LARGE(__nv_bfloat16, 256, 5120)
|
||||
|
||||
#undef INSTANTIATE_LARGE
|
||||
|
||||
#undef INSTANTIATE_ALL
|
||||
#undef INSTANTIATE
|
||||
|
||||
@@ -25,10 +25,11 @@ inline int getSMVersion() {
|
||||
static constexpr int FP32_MAX_TOKENS = 32;
|
||||
|
||||
// Supported (hidden_dim, num_experts) pairs (must match the instantiations in
|
||||
// fp32_router_gemm.cu): (3072, 256) for MiniMax-M2/M2.5, (6144, 128) for M3.
|
||||
// fp32_router_gemm.cu).
|
||||
static inline bool fp32_router_gemm_supported(int hidden_dim, int num_experts) {
|
||||
return (hidden_dim == 3072 && num_experts == 256) ||
|
||||
(hidden_dim == 6144 && num_experts == 128);
|
||||
(hidden_dim == 6144 && num_experts == 128) ||
|
||||
(hidden_dim == 5120 && num_experts == 256);
|
||||
}
|
||||
|
||||
// Forward declarations — 4 template params must match fp32_router_gemm.cu
|
||||
@@ -60,8 +61,7 @@ struct Fp32LoopUnroller<InputT, kNumExperts, kHiddenDim, kEnd, kEnd> {
|
||||
invokeFp32RouterGemm<InputT, kEnd, kNumExperts, kHiddenDim>(
|
||||
output, mat_a, mat_b, stream);
|
||||
} else {
|
||||
throw std::invalid_argument(
|
||||
"fp32_router_gemm: num_tokens must be in [1, 32]");
|
||||
throw std::invalid_argument("fp32_router_gemm: unsupported num_tokens");
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -77,6 +77,9 @@ void dispatchFp32RouterGemm(int num_experts, int hidden_dim, int num_tokens,
|
||||
} else if (num_experts == 128 && hidden_dim == 6144) {
|
||||
Fp32LoopUnroller<InputT, 128, 6144, 1, FP32_MAX_TOKENS>::unroll(
|
||||
num_tokens, output, mat_a, mat_b, stream);
|
||||
} else if (num_experts == 256 && hidden_dim == 5120) {
|
||||
Fp32LoopUnroller<InputT, 256, 5120, 1, 48>::unroll(num_tokens, output,
|
||||
mat_a, mat_b, stream);
|
||||
} else {
|
||||
throw std::invalid_argument(
|
||||
"fp32_router_gemm: unsupported (hidden_dim, num_experts) pair");
|
||||
@@ -111,9 +114,11 @@ void fp32_router_gemm(
|
||||
STD_TORCH_CHECK(
|
||||
fp32_router_gemm_supported(hidden_dim, num_experts),
|
||||
"fp32_router_gemm: supported (hidden_dim, num_experts) pairs are "
|
||||
"(3072, 256) and (6144, 128)");
|
||||
STD_TORCH_CHECK(num_tokens <= FP32_MAX_TOKENS,
|
||||
"fp32_router_gemm: num_tokens must be in [0, 32]");
|
||||
"(3072, 256), (6144, 128), and (5120, 256)");
|
||||
const int max_tokens =
|
||||
hidden_dim == 5120 && num_experts == 256 ? 48 : FP32_MAX_TOKENS;
|
||||
STD_TORCH_CHECK(num_tokens <= max_tokens,
|
||||
"fp32_router_gemm: num_tokens exceeds supported range");
|
||||
STD_TORCH_CHECK(
|
||||
mat_a.scalar_type() == torch::headeronly::ScalarType::Float ||
|
||||
mat_a.scalar_type() == torch::headeronly::ScalarType::BFloat16,
|
||||
|
||||
Reference in New Issue
Block a user