diff --git a/csrc/libtorch_stable/moe/dsv3_router_gemm_bf16_out.cu b/csrc/libtorch_stable/moe/dsv3_router_gemm_bf16_out.cu index 776c92678d..bee4e00a8d 100644 --- a/csrc/libtorch_stable/moe/dsv3_router_gemm_bf16_out.cu +++ b/csrc/libtorch_stable/moe/dsv3_router_gemm_bf16_out.cu @@ -286,3 +286,52 @@ template void invokeRouterGemmBf16Output<__nv_bfloat16, 15, 384, 7168>( template void invokeRouterGemmBf16Output<__nv_bfloat16, 16, 384, 7168>( __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +// Template instantiations for GLM-5 (DEFAULT_NUM_EXPERTS, hidden_dim=6144) +template void invokeRouterGemmBf16Output<__nv_bfloat16, 1, 256, 6144>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 2, 256, 6144>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 3, 256, 6144>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 4, 256, 6144>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 5, 256, 6144>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 6, 256, 6144>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 7, 256, 6144>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 8, 256, 6144>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 9, 256, 6144>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 10, 256, 6144>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 11, 256, 6144>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 12, 256, 6144>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 13, 256, 6144>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 14, 256, 6144>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 15, 256, 6144>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 16, 256, 6144>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); diff --git a/csrc/libtorch_stable/moe/dsv3_router_gemm_entry.cu b/csrc/libtorch_stable/moe/dsv3_router_gemm_entry.cu index 53a64fa8c1..4e06cd3b9a 100644 --- a/csrc/libtorch_stable/moe/dsv3_router_gemm_entry.cu +++ b/csrc/libtorch_stable/moe/dsv3_router_gemm_entry.cu @@ -41,6 +41,7 @@ inline int getSMVersion() { static constexpr int DEFAULT_NUM_EXPERTS = 256; static constexpr int KIMI_K2_NUM_EXPERTS = 384; static constexpr int DEFAULT_HIDDEN_DIM = 7168; +static constexpr int GLM_5_HIDDEN_DIM = 6144; template void invokeRouterGemmFloatOutput(float* output, T const* mat_a, T const* mat_b, @@ -121,14 +122,21 @@ void dsv3_router_gemm( STD_TORCH_CHECK(mat_a.size(1) == mat_b.size(1), "mat_a and mat_b must have the same hidden_dim"); - STD_TORCH_CHECK(hidden_dim == DEFAULT_HIDDEN_DIM, - "Expected hidden_dim=", DEFAULT_HIDDEN_DIM, - ", but got hidden_dim=", hidden_dim); + STD_TORCH_CHECK( + hidden_dim == DEFAULT_HIDDEN_DIM || hidden_dim == GLM_5_HIDDEN_DIM, + "Expected hidden_dim=", DEFAULT_HIDDEN_DIM, + " or hidden_dim=", GLM_5_HIDDEN_DIM, ", but got hidden_dim=", hidden_dim); STD_TORCH_CHECK( num_experts == DEFAULT_NUM_EXPERTS || num_experts == KIMI_K2_NUM_EXPERTS, "Expected num_experts=", DEFAULT_NUM_EXPERTS, " or num_experts=", KIMI_K2_NUM_EXPERTS, ", but got num_experts=", num_experts); + // KIMI_K2_NUM_EXPERTS is only instantiated for the default hidden_dim. + STD_TORCH_CHECK( + hidden_dim == DEFAULT_HIDDEN_DIM || num_experts == DEFAULT_NUM_EXPERTS, + "hidden_dim=", GLM_5_HIDDEN_DIM, + " only supports num_experts=", DEFAULT_NUM_EXPERTS, + ", but got num_experts=", num_experts); STD_TORCH_CHECK(num_tokens >= 1 && num_tokens <= 16, "currently num_tokens must be less than or equal to 16 for " "router_gemm"); @@ -148,35 +156,49 @@ void dsv3_router_gemm( const cudaStream_t stream = get_current_cuda_stream(mat_a.get_device_index()); + __nv_bfloat16 const* a_ptr = + reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()); + __nv_bfloat16 const* b_ptr = + reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()); + if (output.scalar_type() == torch::headeronly::ScalarType::Float) { - if (num_experts == DEFAULT_NUM_EXPERTS) { - LoopUnroller<1, 16, DEFAULT_NUM_EXPERTS, DEFAULT_HIDDEN_DIM>:: - unroll_float_output( - num_tokens, reinterpret_cast(output.mutable_data_ptr()), - reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()), - reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), stream); - } else if (num_experts == KIMI_K2_NUM_EXPERTS) { - LoopUnroller<1, 16, KIMI_K2_NUM_EXPERTS, DEFAULT_HIDDEN_DIM>:: - unroll_float_output( - num_tokens, reinterpret_cast(output.mutable_data_ptr()), - reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()), - reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), stream); + float* out_ptr = reinterpret_cast(output.mutable_data_ptr()); + if (hidden_dim == DEFAULT_HIDDEN_DIM) { + if (num_experts == DEFAULT_NUM_EXPERTS) { + LoopUnroller<1, 16, DEFAULT_NUM_EXPERTS, + DEFAULT_HIDDEN_DIM>::unroll_float_output(num_tokens, + out_ptr, a_ptr, + b_ptr, stream); + } else { + LoopUnroller<1, 16, KIMI_K2_NUM_EXPERTS, + DEFAULT_HIDDEN_DIM>::unroll_float_output(num_tokens, + out_ptr, a_ptr, + b_ptr, stream); + } + } else { // GLM_5_HIDDEN_DIM + LoopUnroller<1, 16, DEFAULT_NUM_EXPERTS, + GLM_5_HIDDEN_DIM>::unroll_float_output(num_tokens, out_ptr, + a_ptr, b_ptr, stream); } } else if (output.scalar_type() == torch::headeronly::ScalarType::BFloat16) { - if (num_experts == DEFAULT_NUM_EXPERTS) { - LoopUnroller<1, 16, DEFAULT_NUM_EXPERTS, DEFAULT_HIDDEN_DIM>:: - unroll_bf16_output( - num_tokens, - reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr()), - reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()), - reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), stream); - } else if (num_experts == KIMI_K2_NUM_EXPERTS) { - LoopUnroller<1, 16, KIMI_K2_NUM_EXPERTS, DEFAULT_HIDDEN_DIM>:: - unroll_bf16_output( - num_tokens, - reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr()), - reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()), - reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), stream); + __nv_bfloat16* out_ptr = + reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr()); + if (hidden_dim == DEFAULT_HIDDEN_DIM) { + if (num_experts == DEFAULT_NUM_EXPERTS) { + LoopUnroller<1, 16, DEFAULT_NUM_EXPERTS, + DEFAULT_HIDDEN_DIM>::unroll_bf16_output(num_tokens, + out_ptr, a_ptr, + b_ptr, stream); + } else { + LoopUnroller<1, 16, KIMI_K2_NUM_EXPERTS, + DEFAULT_HIDDEN_DIM>::unroll_bf16_output(num_tokens, + out_ptr, a_ptr, + b_ptr, stream); + } + } else { // GLM_5_HIDDEN_DIM + LoopUnroller<1, 16, DEFAULT_NUM_EXPERTS, + GLM_5_HIDDEN_DIM>::unroll_bf16_output(num_tokens, out_ptr, + a_ptr, b_ptr, stream); } } } diff --git a/csrc/libtorch_stable/moe/dsv3_router_gemm_float_out.cu b/csrc/libtorch_stable/moe/dsv3_router_gemm_float_out.cu index 113ad27638..fe940d5433 100644 --- a/csrc/libtorch_stable/moe/dsv3_router_gemm_float_out.cu +++ b/csrc/libtorch_stable/moe/dsv3_router_gemm_float_out.cu @@ -286,3 +286,52 @@ template void invokeRouterGemmFloatOutput<__nv_bfloat16, 15, 384, 7168>( template void invokeRouterGemmFloatOutput<__nv_bfloat16, 16, 384, 7168>( float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +// Template instantiations for GLM-5 (DEFAULT_NUM_EXPERTS, hidden_dim=6144) +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 1, 256, 6144>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 2, 256, 6144>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 3, 256, 6144>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 4, 256, 6144>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 5, 256, 6144>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 6, 256, 6144>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 7, 256, 6144>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 8, 256, 6144>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 9, 256, 6144>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 10, 256, 6144>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 11, 256, 6144>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 12, 256, 6144>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 13, 256, 6144>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 14, 256, 6144>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 15, 256, 6144>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 16, 256, 6144>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); diff --git a/vllm/model_executor/layers/fused_moe/router/gate_linear.py b/vllm/model_executor/layers/fused_moe/router/gate_linear.py index f230b4d579..63e40ab9b5 100644 --- a/vllm/model_executor/layers/fused_moe/router/gate_linear.py +++ b/vllm/model_executor/layers/fused_moe/router/gate_linear.py @@ -14,7 +14,7 @@ from vllm.utils.torch_utils import direct_register_custom_op class GateLinear(ReplicatedLinear): """MoE gate linear layer with multi-tier GEMM dispatch: - 1. DSV3 specialized kernel (SM90+, fp32 out, M<=16, H=7168, E=256/384) + 1. DSV3 specialized kernel (SM90+, M<=16, H=7168 E=256/384, H=6144 E=256) 2. fp32 specialized kernel (SM90+, bf16/fp32 in, fp32 out, M<=32, H=3072, E=256) 3. cuBLAS bf16×bf16→fp32 (SM90+ + bf16 weight + fp32 out_dtype) @@ -25,9 +25,14 @@ class GateLinear(ReplicatedLinear): method which is only known later). """ - # Dimensions supported by the DSV3 specialized kernel + # Dimensions supported by the DSV3 specialized kernel. + # Valid (hidden_size, num_experts) combinations: + # (7168, 256) -> DeepSeek-V3, (7168, 384) -> Kimi-K2, + # (6144, 256) -> GLM-5 DSV3_SUPPORTED_NUM_EXPERTS = [256, 384] - DSV3_SUPPORTED_HIDDEN_SIZES = [7168] + DSV3_SUPPORTED_HIDDEN_SIZES = [7168, 6144] + # num_experts=384 is only instantiated for hidden_size=7168. + DSV3_UNSUPPORTED_SHAPES = {(6144, 384)} # (hidden_size, num_experts) pairs with an instantiated fp32 kernel: # (3072, 256) -> MiniMax-M2/M2.5, (6144, 128) -> MiniMax-M3 @@ -71,6 +76,7 @@ class GateLinear(ReplicatedLinear): self.allow_specialized_router_gemm and output_size in self.DSV3_SUPPORTED_NUM_EXPERTS and input_size in self.DSV3_SUPPORTED_HIDDEN_SIZES + and (input_size, output_size) not in self.DSV3_UNSUPPORTED_SHAPES ) # See https://github.com/vllm-project/vllm/pull/44217 # for more details.