[Kernel] GLM5 Router GEMM (#46385)
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
@@ -286,3 +286,52 @@ template void invokeRouterGemmBf16Output<__nv_bfloat16, 15, 384, 7168>(
|
||||
|
||||
template void invokeRouterGemmBf16Output<__nv_bfloat16, 16, 384, 7168>(
|
||||
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t);
|
||||
|
||||
// Template instantiations for GLM-5 (DEFAULT_NUM_EXPERTS, hidden_dim=6144)
|
||||
template void invokeRouterGemmBf16Output<__nv_bfloat16, 1, 256, 6144>(
|
||||
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t);
|
||||
|
||||
template void invokeRouterGemmBf16Output<__nv_bfloat16, 2, 256, 6144>(
|
||||
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t);
|
||||
|
||||
template void invokeRouterGemmBf16Output<__nv_bfloat16, 3, 256, 6144>(
|
||||
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t);
|
||||
|
||||
template void invokeRouterGemmBf16Output<__nv_bfloat16, 4, 256, 6144>(
|
||||
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t);
|
||||
|
||||
template void invokeRouterGemmBf16Output<__nv_bfloat16, 5, 256, 6144>(
|
||||
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t);
|
||||
|
||||
template void invokeRouterGemmBf16Output<__nv_bfloat16, 6, 256, 6144>(
|
||||
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t);
|
||||
|
||||
template void invokeRouterGemmBf16Output<__nv_bfloat16, 7, 256, 6144>(
|
||||
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t);
|
||||
|
||||
template void invokeRouterGemmBf16Output<__nv_bfloat16, 8, 256, 6144>(
|
||||
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t);
|
||||
|
||||
template void invokeRouterGemmBf16Output<__nv_bfloat16, 9, 256, 6144>(
|
||||
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t);
|
||||
|
||||
template void invokeRouterGemmBf16Output<__nv_bfloat16, 10, 256, 6144>(
|
||||
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t);
|
||||
|
||||
template void invokeRouterGemmBf16Output<__nv_bfloat16, 11, 256, 6144>(
|
||||
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t);
|
||||
|
||||
template void invokeRouterGemmBf16Output<__nv_bfloat16, 12, 256, 6144>(
|
||||
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t);
|
||||
|
||||
template void invokeRouterGemmBf16Output<__nv_bfloat16, 13, 256, 6144>(
|
||||
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t);
|
||||
|
||||
template void invokeRouterGemmBf16Output<__nv_bfloat16, 14, 256, 6144>(
|
||||
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t);
|
||||
|
||||
template void invokeRouterGemmBf16Output<__nv_bfloat16, 15, 256, 6144>(
|
||||
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t);
|
||||
|
||||
template void invokeRouterGemmBf16Output<__nv_bfloat16, 16, 256, 6144>(
|
||||
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t);
|
||||
|
||||
@@ -41,6 +41,7 @@ inline int getSMVersion() {
|
||||
static constexpr int DEFAULT_NUM_EXPERTS = 256;
|
||||
static constexpr int KIMI_K2_NUM_EXPERTS = 384;
|
||||
static constexpr int DEFAULT_HIDDEN_DIM = 7168;
|
||||
static constexpr int GLM_5_HIDDEN_DIM = 6144;
|
||||
|
||||
template <typename T, int kNumTokens, int kNumExperts, int kHiddenDim>
|
||||
void invokeRouterGemmFloatOutput(float* output, T const* mat_a, T const* mat_b,
|
||||
@@ -121,14 +122,21 @@ void dsv3_router_gemm(
|
||||
|
||||
STD_TORCH_CHECK(mat_a.size(1) == mat_b.size(1),
|
||||
"mat_a and mat_b must have the same hidden_dim");
|
||||
STD_TORCH_CHECK(hidden_dim == DEFAULT_HIDDEN_DIM,
|
||||
"Expected hidden_dim=", DEFAULT_HIDDEN_DIM,
|
||||
", but got hidden_dim=", hidden_dim);
|
||||
STD_TORCH_CHECK(
|
||||
hidden_dim == DEFAULT_HIDDEN_DIM || hidden_dim == GLM_5_HIDDEN_DIM,
|
||||
"Expected hidden_dim=", DEFAULT_HIDDEN_DIM,
|
||||
" or hidden_dim=", GLM_5_HIDDEN_DIM, ", but got hidden_dim=", hidden_dim);
|
||||
STD_TORCH_CHECK(
|
||||
num_experts == DEFAULT_NUM_EXPERTS || num_experts == KIMI_K2_NUM_EXPERTS,
|
||||
"Expected num_experts=", DEFAULT_NUM_EXPERTS,
|
||||
" or num_experts=", KIMI_K2_NUM_EXPERTS,
|
||||
", but got num_experts=", num_experts);
|
||||
// KIMI_K2_NUM_EXPERTS is only instantiated for the default hidden_dim.
|
||||
STD_TORCH_CHECK(
|
||||
hidden_dim == DEFAULT_HIDDEN_DIM || num_experts == DEFAULT_NUM_EXPERTS,
|
||||
"hidden_dim=", GLM_5_HIDDEN_DIM,
|
||||
" only supports num_experts=", DEFAULT_NUM_EXPERTS,
|
||||
", but got num_experts=", num_experts);
|
||||
STD_TORCH_CHECK(num_tokens >= 1 && num_tokens <= 16,
|
||||
"currently num_tokens must be less than or equal to 16 for "
|
||||
"router_gemm");
|
||||
@@ -148,35 +156,49 @@ void dsv3_router_gemm(
|
||||
|
||||
const cudaStream_t stream = get_current_cuda_stream(mat_a.get_device_index());
|
||||
|
||||
__nv_bfloat16 const* a_ptr =
|
||||
reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr());
|
||||
__nv_bfloat16 const* b_ptr =
|
||||
reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr());
|
||||
|
||||
if (output.scalar_type() == torch::headeronly::ScalarType::Float) {
|
||||
if (num_experts == DEFAULT_NUM_EXPERTS) {
|
||||
LoopUnroller<1, 16, DEFAULT_NUM_EXPERTS, DEFAULT_HIDDEN_DIM>::
|
||||
unroll_float_output(
|
||||
num_tokens, reinterpret_cast<float*>(output.mutable_data_ptr()),
|
||||
reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()),
|
||||
reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), stream);
|
||||
} else if (num_experts == KIMI_K2_NUM_EXPERTS) {
|
||||
LoopUnroller<1, 16, KIMI_K2_NUM_EXPERTS, DEFAULT_HIDDEN_DIM>::
|
||||
unroll_float_output(
|
||||
num_tokens, reinterpret_cast<float*>(output.mutable_data_ptr()),
|
||||
reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()),
|
||||
reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), stream);
|
||||
float* out_ptr = reinterpret_cast<float*>(output.mutable_data_ptr());
|
||||
if (hidden_dim == DEFAULT_HIDDEN_DIM) {
|
||||
if (num_experts == DEFAULT_NUM_EXPERTS) {
|
||||
LoopUnroller<1, 16, DEFAULT_NUM_EXPERTS,
|
||||
DEFAULT_HIDDEN_DIM>::unroll_float_output(num_tokens,
|
||||
out_ptr, a_ptr,
|
||||
b_ptr, stream);
|
||||
} else {
|
||||
LoopUnroller<1, 16, KIMI_K2_NUM_EXPERTS,
|
||||
DEFAULT_HIDDEN_DIM>::unroll_float_output(num_tokens,
|
||||
out_ptr, a_ptr,
|
||||
b_ptr, stream);
|
||||
}
|
||||
} else { // GLM_5_HIDDEN_DIM
|
||||
LoopUnroller<1, 16, DEFAULT_NUM_EXPERTS,
|
||||
GLM_5_HIDDEN_DIM>::unroll_float_output(num_tokens, out_ptr,
|
||||
a_ptr, b_ptr, stream);
|
||||
}
|
||||
} else if (output.scalar_type() == torch::headeronly::ScalarType::BFloat16) {
|
||||
if (num_experts == DEFAULT_NUM_EXPERTS) {
|
||||
LoopUnroller<1, 16, DEFAULT_NUM_EXPERTS, DEFAULT_HIDDEN_DIM>::
|
||||
unroll_bf16_output(
|
||||
num_tokens,
|
||||
reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr()),
|
||||
reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()),
|
||||
reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), stream);
|
||||
} else if (num_experts == KIMI_K2_NUM_EXPERTS) {
|
||||
LoopUnroller<1, 16, KIMI_K2_NUM_EXPERTS, DEFAULT_HIDDEN_DIM>::
|
||||
unroll_bf16_output(
|
||||
num_tokens,
|
||||
reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr()),
|
||||
reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()),
|
||||
reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), stream);
|
||||
__nv_bfloat16* out_ptr =
|
||||
reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr());
|
||||
if (hidden_dim == DEFAULT_HIDDEN_DIM) {
|
||||
if (num_experts == DEFAULT_NUM_EXPERTS) {
|
||||
LoopUnroller<1, 16, DEFAULT_NUM_EXPERTS,
|
||||
DEFAULT_HIDDEN_DIM>::unroll_bf16_output(num_tokens,
|
||||
out_ptr, a_ptr,
|
||||
b_ptr, stream);
|
||||
} else {
|
||||
LoopUnroller<1, 16, KIMI_K2_NUM_EXPERTS,
|
||||
DEFAULT_HIDDEN_DIM>::unroll_bf16_output(num_tokens,
|
||||
out_ptr, a_ptr,
|
||||
b_ptr, stream);
|
||||
}
|
||||
} else { // GLM_5_HIDDEN_DIM
|
||||
LoopUnroller<1, 16, DEFAULT_NUM_EXPERTS,
|
||||
GLM_5_HIDDEN_DIM>::unroll_bf16_output(num_tokens, out_ptr,
|
||||
a_ptr, b_ptr, stream);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -286,3 +286,52 @@ template void invokeRouterGemmFloatOutput<__nv_bfloat16, 15, 384, 7168>(
|
||||
|
||||
template void invokeRouterGemmFloatOutput<__nv_bfloat16, 16, 384, 7168>(
|
||||
float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t);
|
||||
|
||||
// Template instantiations for GLM-5 (DEFAULT_NUM_EXPERTS, hidden_dim=6144)
|
||||
template void invokeRouterGemmFloatOutput<__nv_bfloat16, 1, 256, 6144>(
|
||||
float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t);
|
||||
|
||||
template void invokeRouterGemmFloatOutput<__nv_bfloat16, 2, 256, 6144>(
|
||||
float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t);
|
||||
|
||||
template void invokeRouterGemmFloatOutput<__nv_bfloat16, 3, 256, 6144>(
|
||||
float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t);
|
||||
|
||||
template void invokeRouterGemmFloatOutput<__nv_bfloat16, 4, 256, 6144>(
|
||||
float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t);
|
||||
|
||||
template void invokeRouterGemmFloatOutput<__nv_bfloat16, 5, 256, 6144>(
|
||||
float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t);
|
||||
|
||||
template void invokeRouterGemmFloatOutput<__nv_bfloat16, 6, 256, 6144>(
|
||||
float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t);
|
||||
|
||||
template void invokeRouterGemmFloatOutput<__nv_bfloat16, 7, 256, 6144>(
|
||||
float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t);
|
||||
|
||||
template void invokeRouterGemmFloatOutput<__nv_bfloat16, 8, 256, 6144>(
|
||||
float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t);
|
||||
|
||||
template void invokeRouterGemmFloatOutput<__nv_bfloat16, 9, 256, 6144>(
|
||||
float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t);
|
||||
|
||||
template void invokeRouterGemmFloatOutput<__nv_bfloat16, 10, 256, 6144>(
|
||||
float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t);
|
||||
|
||||
template void invokeRouterGemmFloatOutput<__nv_bfloat16, 11, 256, 6144>(
|
||||
float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t);
|
||||
|
||||
template void invokeRouterGemmFloatOutput<__nv_bfloat16, 12, 256, 6144>(
|
||||
float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t);
|
||||
|
||||
template void invokeRouterGemmFloatOutput<__nv_bfloat16, 13, 256, 6144>(
|
||||
float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t);
|
||||
|
||||
template void invokeRouterGemmFloatOutput<__nv_bfloat16, 14, 256, 6144>(
|
||||
float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t);
|
||||
|
||||
template void invokeRouterGemmFloatOutput<__nv_bfloat16, 15, 256, 6144>(
|
||||
float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t);
|
||||
|
||||
template void invokeRouterGemmFloatOutput<__nv_bfloat16, 16, 256, 6144>(
|
||||
float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t);
|
||||
|
||||
@@ -14,7 +14,7 @@ from vllm.utils.torch_utils import direct_register_custom_op
|
||||
class GateLinear(ReplicatedLinear):
|
||||
"""MoE gate linear layer with multi-tier GEMM dispatch:
|
||||
|
||||
1. DSV3 specialized kernel (SM90+, fp32 out, M<=16, H=7168, E=256/384)
|
||||
1. DSV3 specialized kernel (SM90+, M<=16, H=7168 E=256/384, H=6144 E=256)
|
||||
2. fp32 specialized kernel (SM90+, bf16/fp32 in, fp32 out,
|
||||
M<=32, H=3072, E=256)
|
||||
3. cuBLAS bf16×bf16→fp32 (SM90+ + bf16 weight + fp32 out_dtype)
|
||||
@@ -25,9 +25,14 @@ class GateLinear(ReplicatedLinear):
|
||||
method which is only known later).
|
||||
"""
|
||||
|
||||
# Dimensions supported by the DSV3 specialized kernel
|
||||
# Dimensions supported by the DSV3 specialized kernel.
|
||||
# Valid (hidden_size, num_experts) combinations:
|
||||
# (7168, 256) -> DeepSeek-V3, (7168, 384) -> Kimi-K2,
|
||||
# (6144, 256) -> GLM-5
|
||||
DSV3_SUPPORTED_NUM_EXPERTS = [256, 384]
|
||||
DSV3_SUPPORTED_HIDDEN_SIZES = [7168]
|
||||
DSV3_SUPPORTED_HIDDEN_SIZES = [7168, 6144]
|
||||
# num_experts=384 is only instantiated for hidden_size=7168.
|
||||
DSV3_UNSUPPORTED_SHAPES = {(6144, 384)}
|
||||
|
||||
# (hidden_size, num_experts) pairs with an instantiated fp32 kernel:
|
||||
# (3072, 256) -> MiniMax-M2/M2.5, (6144, 128) -> MiniMax-M3
|
||||
@@ -71,6 +76,7 @@ class GateLinear(ReplicatedLinear):
|
||||
self.allow_specialized_router_gemm
|
||||
and output_size in self.DSV3_SUPPORTED_NUM_EXPERTS
|
||||
and input_size in self.DSV3_SUPPORTED_HIDDEN_SIZES
|
||||
and (input_size, output_size) not in self.DSV3_UNSUPPORTED_SHAPES
|
||||
)
|
||||
# See https://github.com/vllm-project/vllm/pull/44217
|
||||
# for more details.
|
||||
|
||||
Reference in New Issue
Block a user