From 88a860d7545aad69661daad7a1c2b04f59c76144 Mon Sep 17 00:00:00 2001 From: Yuwen Zhou Date: Mon, 18 May 2026 18:04:45 +0800 Subject: [PATCH] [CPU] Add MXFP4 W4A16 MoE support (#41922) Signed-off-by: yuwenzho Signed-off-by: Yuwen Zhou --- .buildkite/hardware_tests/cpu.yaml | 6 +- csrc/cpu/sgl-kernels/common.h | 7 + csrc/cpu/sgl-kernels/gemm.h | 58 +- csrc/cpu/sgl-kernels/gemm_fp8.cpp | 86 ++- csrc/cpu/sgl-kernels/moe.cpp | 87 ++- csrc/cpu/sgl-kernels/moe.h | 106 ++++ csrc/cpu/sgl-kernels/moe_fp8.cpp | 146 +++-- csrc/cpu/sgl-kernels/vec.h | 17 +- csrc/cpu/torch_bindings.cpp | 14 +- tests/kernels/moe/test_cpu_fp8_fused_moe.py | 256 --------- tests/kernels/moe/test_cpu_quant_fused_moe.py | 500 ++++++++++++++++++ tests/quantization/test_cpu_wna16.py | 1 + vllm/_custom_ops.py | 15 +- .../layers/fused_moe/cpu_fused_moe.py | 4 + .../layers/fused_moe/experts/cpu_moe.py | 169 +++++- .../layers/fused_moe/oracle/mxfp4.py | 49 ++ 16 files changed, 1172 insertions(+), 349 deletions(-) delete mode 100644 tests/kernels/moe/test_cpu_fp8_fused_moe.py create mode 100644 tests/kernels/moe/test_cpu_quant_fused_moe.py diff --git a/.buildkite/hardware_tests/cpu.yaml b/.buildkite/hardware_tests/cpu.yaml index 83a2a20888..6189d5d61e 100644 --- a/.buildkite/hardware_tests/cpu.yaml +++ b/.buildkite/hardware_tests/cpu.yaml @@ -12,7 +12,7 @@ steps: - vllm/_custom_ops.py - tests/kernels/attention/test_cpu_attn.py - tests/kernels/moe/test_cpu_fused_moe.py - - tests/kernels/moe/test_cpu_fp8_fused_moe.py + - tests/kernels/moe/test_cpu_quant_fused_moe.py - tests/kernels/test_onednn.py - tests/kernels/test_awq_int4_to_int8.py - tests/kernels/quantization/test_cpu_fp8_scaled_mm.py @@ -21,7 +21,7 @@ steps: bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 30m " pytest -x -v -s tests/kernels/attention/test_cpu_attn.py pytest -x -v -s tests/kernels/moe/test_cpu_fused_moe.py - pytest -x -v -s tests/kernels/moe/test_cpu_fp8_fused_moe.py + pytest -x -v -s tests/kernels/moe/test_cpu_quant_fused_moe.py pytest -x -v -s tests/kernels/test_onednn.py pytest -x -v -s tests/kernels/test_awq_int4_to_int8.py pytest -x -v -s tests/kernels/quantization/test_cpu_fp8_scaled_mm.py" @@ -70,7 +70,7 @@ steps: - tests/quantization/test_cpu_wna16.py commands: - | - bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 20m " + bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 30m " pytest -x -v -s tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_logprobs pytest -x -v -s tests/quantization/test_cpu_wna16.py" diff --git a/csrc/cpu/sgl-kernels/common.h b/csrc/cpu/sgl-kernels/common.h index 425d5d131f..d501d2adb8 100644 --- a/csrc/cpu/sgl-kernels/common.h +++ b/csrc/cpu/sgl-kernels/common.h @@ -348,6 +348,13 @@ inline int get_cache_blocks(int chunk_size) { return std::min(MAX_CACHE_BLOCK_SIZE, cache_block_size); } +template <> +inline int get_cache_blocks(int chunk_size) { + // mxfp4 uses bf16 as accumulate type + int cache_block_size = get_cache_blocks(chunk_size); + return std::min(MAX_CACHE_BLOCK_SIZE, cache_block_size); +} + // 2d sequential loop in range : [mb0, mb1), [nb0, nb1) template inline void loop_2d(int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1, int64_t chunk_size, const func_t& f) { diff --git a/csrc/cpu/sgl-kernels/gemm.h b/csrc/cpu/sgl-kernels/gemm.h index b43be94ffd..f3fb37a5f6 100644 --- a/csrc/cpu/sgl-kernels/gemm.h +++ b/csrc/cpu/sgl-kernels/gemm.h @@ -72,7 +72,17 @@ inline int64_t get_row_size(int64_t K, bool use_int8_w8a8) { return use_int8_w8a8 ? K + sizeof(int32_t) : K; } -enum class CPUQuantMethod : int64_t { BF16 = 0, INT8_W8A8 = 1, FP8_W8A16 = 2, INT4_W4A8 = 3 }; +enum class CPUAcTMethod : int { silu_and_mul = 0, swiglu = 1 }; + +constexpr bool operator==(CPUAcTMethod a, int b) { + return static_cast(a) == b; +} + +constexpr bool operator==(int a, CPUAcTMethod b) { + return a == static_cast(b); +} + +enum class CPUQuantMethod : int64_t { BF16 = 0, INT8_W8A8 = 1, FP8_W8A16 = 2, INT4_W4A8 = 3, MXFP4 = 4 }; constexpr bool operator==(CPUQuantMethod a, int64_t b) { return static_cast(a) == b; @@ -99,6 +109,9 @@ inline int64_t get_4bit_block_k_size(int64_t group_size) { // pack weight to vnni format at::Tensor convert_weight_packed(at::Tensor& weight); +// pack scale to blocked format for mxfp4 +at::Tensor convert_scale_packed(at::Tensor& scale); + // pack weight to vnni format for int4 std::tuple convert_weight_packed_scale_zp(at::Tensor qweight, at::Tensor qzeros, at::Tensor scales); @@ -129,9 +142,9 @@ void fused_experts_int8_kernel_impl( int64_t topk, int64_t num_tokens_post_pad); -// moe implementations for fp8 w8a16 -template -void fused_experts_fp8_kernel_impl( +// moe implementations for fp8 w8a16 and mxfp4 +template +void fused_experts_fp_kernel_impl( scalar_t* __restrict__ output, scalar_t* __restrict__ ic0, scalar_t* __restrict__ ic1, @@ -140,10 +153,12 @@ void fused_experts_fp8_kernel_impl( scalar_t* __restrict__ B_tmp, float* __restrict__ C_tmp, const scalar_t* __restrict__ input, - const at::Float8_e4m3fn* __restrict__ packed_w1, - const at::Float8_e4m3fn* __restrict__ packed_w2, - const float* __restrict__ w1s, - const float* __restrict__ w2s, + const packed_t* __restrict__ packed_w1, + const packed_t* __restrict__ packed_w2, + const float* __restrict__ w1_bias, + const float* __restrict__ w2_bias, + const param_t* __restrict__ w1s, + const param_t* __restrict__ w2s, int64_t block_size_N, int64_t block_size_K, const float* __restrict__ topk_weights, @@ -155,7 +170,11 @@ void fused_experts_fp8_kernel_impl( int64_t K, int64_t E, int64_t topk, - int64_t num_tokens_post_pad); + int64_t num_tokens_post_pad, + float alpha, + float limit, + CPUAcTMethod act_func, + bool with_bias); // shared expert implementation for int8 w8a8 template @@ -266,6 +285,7 @@ void tinygemm_kernel( scalar_t* __restrict__ C, scalar_t* __restrict__ Btmp, float* __restrict__ Ctmp, + const float* __restrict__ Bbias, const float* __restrict__ scale, int64_t M, int64_t N, @@ -294,6 +314,26 @@ void tinygemm_kernel( int64_t ldc, bool brg); +// mxfp4 +template +void tinygemm_kernel( + const scalar_t* __restrict__ A, + const uint8_t* __restrict__ B, + scalar_t* __restrict__ C, + scalar_t* __restrict__ Btmp, + float* __restrict__ Ctmp, + const float* __restrict__ Bbias, + const uint8_t* __restrict__ scale, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc, + bool brg, + int64_t block_size_K, + bool do_unpack = true); + template void tinygemm_kernel( scalar_t* C, diff --git a/csrc/cpu/sgl-kernels/gemm_fp8.cpp b/csrc/cpu/sgl-kernels/gemm_fp8.cpp index dfc3044685..b47eb9256e 100644 --- a/csrc/cpu/sgl-kernels/gemm_fp8.cpp +++ b/csrc/cpu/sgl-kernels/gemm_fp8.cpp @@ -67,6 +67,23 @@ inline void copy_mul_stub(scalar_t* __restrict__ out, const float* __restrict__ } } +template <> +inline void +copy_add_stub(float* __restrict__ out, const float* __restrict__ input, const float* __restrict__ bias, int64_t size) { + using fVec = at::vec::Vectorized; + constexpr int kVecSize = fVec::size(); + + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= size - kVecSize; d += kVecSize) { + fVec data = fVec::loadu(input + d) + fVec::loadu(bias + d); + data.store(out + d); + } + for (; d < size; ++d) { + out[d] = input[d] + bias[d]; + } +} + inline void unpack_B( at::BFloat16* __restrict__ Btmp, const at::Float8_e4m3fn* __restrict__ packed_B, @@ -343,7 +360,6 @@ struct tinygemm_kernel_nn{}(storec); } }; - template struct tinygemm_kernel_nn2 { static inline void apply( @@ -918,6 +934,7 @@ void tinygemm_kernel( scalar_t* __restrict__ C, scalar_t* __restrict__ Btmp, float* __restrict__ Ctmp, + const float* __restrict__ Bbias, const float* __restrict__ scale, int64_t M, int64_t N, @@ -928,6 +945,11 @@ void tinygemm_kernel( bool brg, int64_t block_size_K, bool do_unpack) { + if (Bbias != nullptr) { + tinygemm_kernel( + A, B, C, Btmp, Ctmp, scale, Bbias, M, N, K, lda, ldb, ldc, brg, block_size_K, do_unpack); + return; + } tinygemm_kernel( A, B, C, Btmp, Ctmp, scale, nullptr, M, N, K, lda, ldb, ldc, brg, block_size_K, do_unpack); } @@ -949,7 +971,6 @@ void tinygemm_kernel( bool brg) { tinygemm_kernel2(A, B, C, Btmp, Ctmp, scale, M, N, K, lda, ldb, ldc, brg); } - template void tinygemm_kernel( const scalar_t* __restrict__ A, @@ -957,6 +978,7 @@ void tinygemm_kernel( scalar_t* __restrict__ C, scalar_t* __restrict__ Btmp, float* __restrict__ Ctmp, + const float* __restrict__ Bbias, const uint8_t* __restrict__ scale, int64_t M, int64_t N, @@ -967,10 +989,68 @@ void tinygemm_kernel( bool brg, int64_t block_size_K, bool do_unpack) { + if (Bbias != nullptr) { + tinygemm_kernel( + A, B, C, Btmp, Ctmp, scale, Bbias, M, N, K, lda, ldb, ldc, brg, block_size_K, do_unpack); + return; + } tinygemm_kernel( A, B, C, Btmp, Ctmp, scale, nullptr, M, N, K, lda, ldb, ldc, brg, block_size_K, do_unpack); } +// tinygemm interface +template +void tinygemm_kernel( + const scalar_t* __restrict__ A, + const at::Float8_e4m3fn* __restrict__ B, + float* __restrict__ C, + scalar_t* __restrict__ Btmp, + const float* __restrict__ Bbias, + const float* __restrict__ scale, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc, + bool brg, + int64_t block_size_K, + bool do_unpack) { + if (Bbias != nullptr) { + tinygemm_kernel( + A, B, C, Btmp, scale, Bbias, M, N, K, lda, ldb, ldc, brg, block_size_K, do_unpack); + return; + } + tinygemm_kernel( + A, B, C, Btmp, scale, nullptr, M, N, K, lda, ldb, ldc, brg, block_size_K, do_unpack); +} + +template +void tinygemm_kernel( + const scalar_t* __restrict__ A, + const uint8_t* __restrict__ B, + float* __restrict__ C, + scalar_t* __restrict__ Btmp, + const float* __restrict__ Bbias, + const uint8_t* __restrict__ scale, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc, + bool brg, + int64_t block_size_K, + bool do_unpack) { + if (Bbias != nullptr) { + tinygemm_kernel( + A, B, C, Btmp, scale, Bbias, M, N, K, lda, ldb, ldc, brg, block_size_K, do_unpack); + return; + } + tinygemm_kernel( + A, B, C, Btmp, scale, nullptr, M, N, K, lda, ldb, ldc, brg, block_size_K, do_unpack); +} + #define INSTANTIATE_TINYGEMM_TEMPLATE(TYPE_A, TYPE_B, TYPE_S) \ template void tinygemm_kernel( \ const TYPE_A* __restrict__ A, \ @@ -978,6 +1058,7 @@ void tinygemm_kernel( TYPE_A* __restrict__ C, \ TYPE_A* __restrict__ Btmp, \ float* __restrict__ Ctmp, \ + const float* __restrict__ Bbias, \ const TYPE_S* __restrict__ scale, \ int64_t M, \ int64_t N, \ @@ -1020,7 +1101,6 @@ inline const float* get_bias_data(const std::optional& bias, int64_t } return nullptr; } - // FP8 and MXFP4 WoQ uses the same pattern: // Btmp : [T, BLOCK_N * K] // Ctmp : [T, BLOCK_M * BLOCK_N] diff --git a/csrc/cpu/sgl-kernels/moe.cpp b/csrc/cpu/sgl-kernels/moe.cpp index c248f678be..06f9f7c373 100644 --- a/csrc/cpu/sgl-kernels/moe.cpp +++ b/csrc/cpu/sgl-kernels/moe.cpp @@ -812,6 +812,7 @@ void shared_expert_kernel_impl( static inline void check_moe_scales( bool use_int8_w8a8, bool use_fp8_w8a16, + bool use_mxfp4, const std::optional& w1_scale, const std::optional& w2_scale, const std::optional> block_size) { @@ -825,6 +826,12 @@ static inline void check_moe_scales( TORCH_CHECK(block_size.has_value(), "missing block_size for fp8 w8a16."); TORCH_CHECK(block_size.value().size() == 2, "expect block_size.size() to be 2."); } + if (use_mxfp4) { + TORCH_CHECK(w1_scale.has_value(), "missing w1_scale for mxfp4."); + TORCH_CHECK(w2_scale.has_value(), "missing w2_scale for mxfp4."); + TORCH_CHECK(w1_scale.value().scalar_type() == at::kByte, "expect w1_scale to be uint8."); + TORCH_CHECK(w2_scale.value().scalar_type() == at::kByte, "expect w2_scale to be uint8."); + } } #define CHECK_MOE_SCALES_FP8(DIM0, DIM1) \ @@ -839,8 +846,8 @@ static inline void check_moe_scales( TORCH_CHECK(w2s.size(DIM1) == div_up(N, block_size_K)) // hidden_states: [M, K] -// w1: [E, 2N, K] -// w2: [E, K, N] +// w1: [E, 2N, K] or [E, 2N, K / 2] for uint8 +// w2: [E, K, N] or [E, K, N / 2] for uint8 // topk_weights: [M, topk] // topk_ids: [M, topk] (int32_t) // @@ -858,6 +865,10 @@ at::Tensor fused_experts_cpu( const std::optional& w1_zero, const std::optional& w2_zero, const std::optional> block_size, + const std::optional& w1_bias, + const std::optional& w2_bias, + const std::optional& alpha, + const std::optional& limit, bool is_vnni) { auto packed_w1 = is_vnni ? w1 : convert_weight_packed(w1); auto packed_w2 = is_vnni ? w2 : convert_weight_packed(w2); @@ -897,8 +908,12 @@ at::Tensor fused_experts_cpu( int64_t topk = topk_weights_.size(1); // we use int32_t compensation for int8 w8a8 - int64_t packed_K = get_row_size(K, moe_comp_method == CPUQuantMethod::INT8_W8A8); - int64_t packed_N = get_row_size(N, moe_comp_method == CPUQuantMethod::INT8_W8A8); + int64_t packed_K = moe_comp_method == CPUQuantMethod::MXFP4 + ? get_row_size(K) + : get_row_size(K, moe_comp_method == CPUQuantMethod::INT8_W8A8); + int64_t packed_N = moe_comp_method == CPUQuantMethod::MXFP4 + ? get_row_size(N) + : get_row_size(N, moe_comp_method == CPUQuantMethod::INT8_W8A8); // check weight shapes CHECK_EQ(w2.size(0), E); @@ -911,6 +926,7 @@ at::Tensor fused_experts_cpu( check_moe_scales( moe_comp_method == CPUQuantMethod::INT8_W8A8, moe_comp_method == CPUQuantMethod::FP8_W8A16, + moe_comp_method == CPUQuantMethod::MXFP4, w1_scale, w2_scale, block_size); @@ -965,7 +981,7 @@ at::Tensor fused_experts_cpu( // 5. Aq_tmp : [M, K] or [M * topk, N] // 6. As_tmp : [M * topk] // - // for fp8 w8a16: + // for fp8 w8a16 and mxfp4: // 7. intermediate_cache0 : [M * topk, 2N] // 8. B_tmp : [T, MAX_CACHE_BLOCK_SIZE, BLOCK_N, std::max(K, N)] // @@ -978,7 +994,7 @@ at::Tensor fused_experts_cpu( if (moe_comp_method == CPUQuantMethod::INT8_W8A8) { buffer_size_nbytes += std::max(M * K, M * topk * N) + M * topk * sizeof(float); } - if (moe_comp_method == CPUQuantMethod::FP8_W8A16) { + if (moe_comp_method == CPUQuantMethod::FP8_W8A16 || moe_comp_method == CPUQuantMethod::MXFP4) { buffer_size_nbytes += M * topk * 2 * N * 2 + num_threads * MAX_CACHE_BLOCK_SIZE * BLOCK_N * std::max(K, N) * 2; } if (moe_comp_method == CPUQuantMethod::INT4_W4A8) { @@ -1031,9 +1047,11 @@ at::Tensor fused_experts_cpu( float* __restrict__ C_tmp = (float*)((void*)(A_tmp + num_threads * BLOCK_M * K)); scalar_t* __restrict__ intermediate_cache0 = (scalar_t*)((void*)(C_tmp + num_threads * 2 * BLOCK_M * BLOCK_N)); scalar_t* __restrict__ B_tmp = (scalar_t*)((void*)(intermediate_cache0 + M * topk * 2 * N)); + bool with_bias = w1_bias.has_value(); + auto act_func = alpha.has_value() && limit.has_value() ? CPUAcTMethod::swiglu : CPUAcTMethod::silu_and_mul; CHECK_MOE_SCALES_FP8(1, 2); - fused_experts_fp8_kernel_impl( + fused_experts_fp_kernel_impl( out_hidden_states.data_ptr(), intermediate_cache0, intermediate_cache1, @@ -1044,6 +1062,8 @@ at::Tensor fused_experts_cpu( hidden_states.data_ptr(), packed_w1.data_ptr(), packed_w2.data_ptr(), + with_bias ? w1_bias.value().data_ptr() : nullptr, + with_bias ? w2_bias.value().data_ptr() : nullptr, w1s.data_ptr(), w2s.data_ptr(), block_size_N, @@ -1057,7 +1077,56 @@ at::Tensor fused_experts_cpu( K, E, topk, - num_tokens_post_pad); + num_tokens_post_pad, + alpha.has_value() ? float(alpha.value()) : 0, + limit.has_value() ? float(limit.value()) : 0, + act_func, + with_bias); + } else if (moe_comp_method == CPUQuantMethod::MXFP4) { + scalar_t* __restrict__ A_tmp = (scalar_t*)((void*)(intermediate_cache2 + M * topk * K)); + float* __restrict__ C_tmp = (float*)((void*)(A_tmp + num_threads * BLOCK_M * K)); + scalar_t* __restrict__ intermediate_cache0 = (scalar_t*)((void*)(C_tmp + num_threads * 2 * BLOCK_M * BLOCK_N)); + scalar_t* __restrict__ B_tmp = (scalar_t*)((void*)(intermediate_cache0 + M * topk * 2 * N)); + bool with_bias = w1_bias.has_value(); + auto act_func = alpha.has_value() && limit.has_value() ? CPUAcTMethod::swiglu : CPUAcTMethod::silu_and_mul; + + // mxfp4 supports only group size of 32 (2^5) + constexpr int64_t group_size = 32; + auto w1s = w1_scale.value(); + auto w2s = w2_scale.value(); + TORCH_CHECK(w1s.numel() == E * 2 * N * K / group_size, "w1_scale size mismatch"); + TORCH_CHECK(w2s.numel() == E * K * N / group_size, "w2_scale size mismatch"); + fused_experts_fp_kernel_impl( + out_hidden_states.data_ptr(), + intermediate_cache0, + intermediate_cache1, + intermediate_cache2, + A_tmp, + B_tmp, + C_tmp, + hidden_states.data_ptr(), + packed_w1.data_ptr(), + packed_w2.data_ptr(), + with_bias ? w1_bias.value().data_ptr() : nullptr, + with_bias ? w2_bias.value().data_ptr() : nullptr, + w1s.data_ptr(), + w2s.data_ptr(), + /*block_size_N*/ 1, + /*block_size_K*/ group_size, + topk_weights_.data_ptr(), + sorted_ids, + expert_ids, + offsets, + M, + N, + K, + E, + topk, + num_tokens_post_pad, + alpha.has_value() ? float(alpha.value()) : 0, + limit.has_value() ? float(limit.value()) : 0, + act_func, + with_bias); } else if (moe_comp_method == CPUQuantMethod::INT4_W4A8) { uint8_t* __restrict__ A_tmp = (uint8_t*)((void*)(intermediate_cache2 + M * topk * K)); float* __restrict__ C_tmp = (float*)((void*)(A_tmp + num_threads * BLOCK_M * K)); @@ -1185,7 +1254,7 @@ at::Tensor shared_expert_cpu( CHECK_EQ(packed_w2.size(1), packed_N); // check scales - check_moe_scales(use_int8_w8a8, use_fp8_w8a16, w1_scale, w2_scale, block_size); + check_moe_scales(use_int8_w8a8, use_fp8_w8a16, false, w1_scale, w2_scale, block_size); at::Tensor out_hidden_states = inplace ? hidden_states : at::empty_like(hidden_states); diff --git a/csrc/cpu/sgl-kernels/moe.h b/csrc/cpu/sgl-kernels/moe.h index d80ed50cf9..b6d2e9e7f6 100644 --- a/csrc/cpu/sgl-kernels/moe.h +++ b/csrc/cpu/sgl-kernels/moe.h @@ -176,3 +176,109 @@ inline void silu_and_mul_stub( out_vec.store(out + d); } } + +template +inline void copy_mul_stub(scalar_t* __restrict__ out, const float* __restrict__ input, float weight, int64_t size) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + const fVec weight_vec = fVec(weight); + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= size - kVecSize; d += kVecSize) { + fVec data0 = fVec::loadu(input + d) * weight_vec; + fVec data1 = fVec::loadu(input + d + fVec::size()) * weight_vec; + bVec out_vec = convert_from_float_ext(data0, data1); + out_vec.store(out + d); + } + for (; d < size; ++d) { + out[d] = static_cast(input[d] * weight); + } +} + +// input = input + input2 +inline void add_bias_stub(float* __restrict__ input, const float* __restrict__ input2, int64_t size) { + using fVec = at::vec::Vectorized; + constexpr int kVecSize = fVec::size(); + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= size - kVecSize; d += kVecSize) { + fVec x_fvec = fVec::loadu(input + d); + fVec y_fvec = fVec::loadu(input2 + d); + x_fvec = x_fvec + y_fvec; + x_fvec.store(input + d); + } + for (; d < size; ++d) { + input[d] = input[d] + input2[d]; + } +} + +template +inline void copy_mul_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ input, float weight, int64_t size) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + const fVec weight_vec = fVec(weight); + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= size - kVecSize; d += kVecSize) { + bVec x = bVec::loadu(input + d); + fVec x0, x1; + std::tie(x0, x1) = at::vec::convert_to_float(x); + x0 = x0 * weight_vec; + x1 = x1 * weight_vec; + bVec out_vec = convert_from_float_ext(x0, x1); + out_vec.store(out + d); + } + for (; d < size; ++d) { + out[d] = static_cast(input[d] * weight); + } +} + +template +inline void clamp_sigmoid_and_mul_stub( + scalar_t* __restrict__ out, + const scalar_t* __restrict__ input, + int64_t size, + const float alpha, + const float limit) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + const fVec one = fVec(1.f); + const fVec zero = fVec(0.f); + const fVec limit_v = fVec(limit); + const fVec nlimit_v = fVec(-limit); + const fVec alpha_v = fVec(alpha); + + // no remainder +#pragma GCC unroll 4 + for (int64_t d = 0; d < size; d += bVec::size()) { + bVec x = bVec::loadu(input + d); + fVec x0_, y0_; + std::tie(x0_, y0_) = at::vec::convert_to_float(x); + float tmp_buffer[fVec::size() * 2]; // 32 + float tmp_glu[fVec::size()]; // 16 + float tmp_linear[fVec::size()]; // 16 + x0_.store(tmp_buffer); + y0_.store(tmp_buffer + fVec::size()); + // interleaved: x[2i] = glu, x[2i+1] = linear + for (int j = 0; j < fVec::size(); ++j) { + // x0 [0,2,..30] + tmp_glu[j] = tmp_buffer[j * 2]; + // y0 [1,3,...31] + tmp_linear[j] = tmp_buffer[j * 2 + 1]; + } + fVec x0 = fVec::loadu(tmp_glu); + fVec y0 = fVec::loadu(tmp_linear); + + // clamp + x0 = at::vec::minimum(x0, limit_v); + y0 = at::vec::minimum(limit_v, at::vec::maximum(nlimit_v, y0)); + // x * sigmoid(x * alpha) + x0 = x0 / (one + (x0 * alpha_v).neg().exp_u20()); + // (y + 1) * x + y0 = y0 + one; + x0 = x0 * y0; + convert_from_float_and_store(out + d / 2, x0); + } +} diff --git a/csrc/cpu/sgl-kernels/moe_fp8.cpp b/csrc/cpu/sgl-kernels/moe_fp8.cpp index f93630ae26..7b33a65855 100644 --- a/csrc/cpu/sgl-kernels/moe_fp8.cpp +++ b/csrc/cpu/sgl-kernels/moe_fp8.cpp @@ -7,8 +7,8 @@ #include "gemm.h" #include "moe.h" -template -void fused_experts_fp8_kernel_impl( +template +void fused_experts_fp_kernel_impl( scalar_t* __restrict__ output, scalar_t* __restrict__ ic0, scalar_t* __restrict__ ic1, @@ -17,10 +17,12 @@ void fused_experts_fp8_kernel_impl( scalar_t* __restrict__ B_tmp, float* __restrict__ C_tmp, const scalar_t* __restrict__ input, - const at::Float8_e4m3fn* __restrict__ packed_w1, - const at::Float8_e4m3fn* __restrict__ packed_w2, - const float* __restrict__ w1s, - const float* __restrict__ w2s, + const packed_t* __restrict__ packed_w1, + const packed_t* __restrict__ packed_w2, + const float* __restrict__ w1_bias, + const float* __restrict__ w2_bias, + const param_t* __restrict__ w1s, + const param_t* __restrict__ w2s, int64_t block_size_N, int64_t block_size_K, const float* __restrict__ topk_weights, @@ -32,7 +34,11 @@ void fused_experts_fp8_kernel_impl( int64_t K, int64_t E, int64_t topk, - int64_t num_tokens_post_pad) { + int64_t num_tokens_post_pad, + float alpha, + float limit, + CPUAcTMethod act_func, + bool with_bias) { constexpr int64_t BLOCK_M = block_size_m(); constexpr int64_t BLOCK_N = block_size_n(); @@ -42,12 +48,20 @@ void fused_experts_fp8_kernel_impl( int64_t scale_size_N = div_up(2 * N, block_size_N); int64_t scale_size_K = div_up(K, block_size_K); int64_t blocks_n_per_group = block_size_N / BLOCK_N; + std::function scale_offset_per_block; + if constexpr (is_mxfp4) { + scale_offset_per_block = [&](int64_t a) { return a * BLOCK_N; }; + } else { + scale_offset_per_block = [&](int64_t a) { return a / blocks_n_per_group; }; + } - const int64_t stride_e = 2 * N * K; - const int64_t stride_n = K; + const int64_t packed_K = get_row_size(K); + + const int64_t stride_e = 2 * N * packed_K; + const int64_t stride_n = packed_K; int64_t avg_M = std::max(int64_t(1), M * topk / E); - const bool use_brgemm = can_use_brgemm(avg_M); + const bool use_brgemm = can_use_brgemm(avg_M); int64_t B_tmp_size_per_thread = MAX_CACHE_BLOCK_SIZE * BLOCK_N * std::max(K, N); @@ -57,14 +71,15 @@ void fused_experts_fp8_kernel_impl( int tid = get_thread_num(); scalar_t* __restrict__ A = A_tmp + tid * BLOCK_M * K; - loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * K, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * K, [&](int64_t mb, int64_t nb, int64_t nb_offset) { int64_t n_size = std::min(2 * N - nb * BLOCK_N, BLOCK_N); // B shape [K, n_size] in vnni format int32_t expert_id = expert_ids[mb]; - const at::Float8_e4m3fn* __restrict__ B = packed_w1 + expert_id * stride_e + nb * BLOCK_N * stride_n; - const float* __restrict__ Bs = - w1s + expert_id * scale_size_N * scale_size_K + (nb / blocks_n_per_group) * scale_size_K; + const packed_t* __restrict__ B = packed_w1 + expert_id * stride_e + nb * BLOCK_N * stride_n; + const param_t* __restrict__ Bs = + w1s + expert_id * scale_size_N * scale_size_K + scale_offset_per_block(nb) * scale_size_K; + const float* __restrict__ B_bias = with_bias ? w1_bias + expert_id * 2 * N + nb * BLOCK_N : nullptr; // do unpacking for the first row or a new expert int32_t pre_expert_id = mb == 0 ? -1 : expert_ids[mb - 1]; @@ -88,6 +103,7 @@ void fused_experts_fp8_kernel_impl( /* C */ ic0 + offset * 2 * N + nb * BLOCK_N, /* Btmp */ B_tmp + tid * B_tmp_size_per_thread + nb_offset * BLOCK_N * K, /* Ctmp */ C_tmp + tid * 2 * BLOCK_M * BLOCK_N, + /* Bbias */ B_bias, /* scale */ Bs, /* M */ m_size, /* N */ n_size, @@ -106,12 +122,20 @@ void fused_experts_fp8_kernel_impl( }); // stage 1.5: intermediate_cache1 = silu(intermediate_cache0) - at::parallel_for(0, M * topk, 0, [&](int64_t begin, int64_t end) { - for (int64_t m = begin; m < end; ++m) { - silu_and_mul_stub(ic1 + m * N, ic0 + m * 2 * N, ic0 + m * 2 * N + N, N); - } - }); - + if (act_func == CPUAcTMethod::silu_and_mul) { + at::parallel_for(0, M * topk, 0, [&](int64_t begin, int64_t end) { + for (int64_t m = begin; m < end; ++m) { + silu_and_mul_stub(ic1 + m * N, ic0 + m * 2 * N, ic0 + m * 2 * N + N, N); + } + }); + } else if (act_func == CPUAcTMethod::swiglu) { + at::parallel_for(0, M * topk, 0, [&](int64_t begin, int64_t end) { + for (int64_t m = begin; m < end; ++m) { + clamp_sigmoid_and_mul_stub(ic1 + m * N, ic0 + m * 2 * N, N, alpha, limit); + clamp_sigmoid_and_mul_stub(ic1 + m * N + N / 2, ic0 + m * 2 * N + N, N, alpha, limit); + } + }); + } // stage 2: intermediate_cache2 = intermediate_cache1 @ w2 // w2 : [E, K, N] as [E, OC, IC] const int64_t OC = K; // rename K as OC @@ -120,15 +144,16 @@ void fused_experts_fp8_kernel_impl( const int64_t NB2 = div_up(OC, BLOCK_N); scale_size_N = div_up(K, block_size_N); scale_size_K = div_up(N, block_size_K); - const int64_t stride_e2 = OC * IC; - const int64_t stride_oc = IC; + const int64_t packed_IC = get_row_size(IC); + const int64_t stride_e2 = OC * packed_IC; + const int64_t stride_oc = packed_IC; // parallel on [MB2, NB2] parallel_2d(MB2, NB2, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { int tid = get_thread_num(); alignas(64) scalar_t C[BLOCK_M * BLOCK_K]; - loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * IC, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * IC, [&](int64_t mb, int64_t nb, int64_t nb_offset) { int64_t m_size = offsets[mb + 1] - offsets[mb]; int64_t n_size = std::min(OC - nb * BLOCK_N, BLOCK_N); @@ -139,9 +164,10 @@ void fused_experts_fp8_kernel_impl( // B shape [IC, n_size] in vnni format int32_t expert_id = expert_ids[mb]; - const at::Float8_e4m3fn* __restrict__ B = packed_w2 + expert_id * stride_e2 + nb * BLOCK_N * stride_oc; - const float* __restrict__ Bs = - w2s + expert_id * scale_size_N * scale_size_K + (nb / blocks_n_per_group) * scale_size_K; + const packed_t* __restrict__ B = packed_w2 + expert_id * stride_e2 + nb * BLOCK_N * stride_oc; + const param_t* __restrict__ Bs = + w2s + expert_id * scale_size_N * scale_size_K + scale_offset_per_block(nb) * scale_size_K; + const float* __restrict__ B_bias = with_bias ? w2_bias + expert_id * OC + nb * BLOCK_N : nullptr; // do unpacking for the first row or a new expert int32_t pre_expert_id = mb == 0 ? -1 : expert_ids[mb - 1]; @@ -153,6 +179,7 @@ void fused_experts_fp8_kernel_impl( /* C */ C, /* Btmp */ B_tmp + tid * B_tmp_size_per_thread + nb_offset * BLOCK_N * IC, /* Ctmp */ C_tmp + tid * 2 * BLOCK_M * BLOCK_N, + /* Bbias */ B_bias, /* scale */ Bs, /* M */ m_size, /* N */ n_size, @@ -177,7 +204,6 @@ void fused_experts_fp8_kernel_impl( at::native::cpublas::brgemm_release(); } }); - // stage 3: out = intermediate_cache2.sum(dim=1) // from [M, topk, K] to [M, K] at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) { @@ -187,35 +213,43 @@ void fused_experts_fp8_kernel_impl( }); } -#define INSTANTIATE_MOE_FP8_TEMPLATE(TYPE) \ - template void fused_experts_fp8_kernel_impl( \ - TYPE* __restrict__ output, \ - TYPE* __restrict__ ic0, \ - TYPE* __restrict__ ic1, \ - TYPE* __restrict__ ic2, \ - TYPE* __restrict__ A_tmp, \ - TYPE* __restrict__ B_tmp, \ - float* __restrict__ C_tmp, \ - const TYPE* __restrict__ input, \ - const at::Float8_e4m3fn* __restrict__ packed_w1, \ - const at::Float8_e4m3fn* __restrict__ packed_w2, \ - const float* __restrict__ w1s, \ - const float* __restrict__ w2s, \ - int64_t block_size_N, \ - int64_t block_size_K, \ - const float* __restrict__ topk_weights, \ - const int32_t* __restrict__ sorted_ids, \ - const int32_t* __restrict__ expert_ids, \ - const int32_t* __restrict__ offsets, \ - int64_t M, \ - int64_t N, \ - int64_t K, \ - int64_t E, \ - int64_t topk, \ - int64_t num_tokens_post_pad) +#define INSTANTIATE_MOE_FP_TEMPLATE(TYPE1, TYPE2, TYPE3, IS_MXFP4) \ + template void fused_experts_fp_kernel_impl( \ + TYPE1* __restrict__ output, \ + TYPE1* __restrict__ ic0, \ + TYPE1* __restrict__ ic1, \ + TYPE1* __restrict__ ic2, \ + TYPE1* __restrict__ A_tmp, \ + TYPE1* __restrict__ B_tmp, \ + float* __restrict__ C_tmp, \ + const TYPE1* __restrict__ input, \ + const TYPE2* __restrict__ packed_w1, \ + const TYPE2* __restrict__ packed_w2, \ + const float* __restrict__ w1_bias, \ + const float* __restrict__ w2_bias, \ + const TYPE3* __restrict__ w1s, \ + const TYPE3* __restrict__ w2s, \ + int64_t block_size_N, \ + int64_t block_size_K, \ + const float* __restrict__ topk_weights, \ + const int32_t* __restrict__ sorted_ids, \ + const int32_t* __restrict__ expert_ids, \ + const int32_t* __restrict__ offsets, \ + int64_t M, \ + int64_t N, \ + int64_t K, \ + int64_t E, \ + int64_t topk, \ + int64_t num_tokens_post_pad, \ + float alpha, \ + float limit, \ + CPUAcTMethod act_func, \ + bool with_bias) -INSTANTIATE_MOE_FP8_TEMPLATE(at::BFloat16); -INSTANTIATE_MOE_FP8_TEMPLATE(at::Half); +INSTANTIATE_MOE_FP_TEMPLATE(at::BFloat16, at::Float8_e4m3fn, float, false); +INSTANTIATE_MOE_FP_TEMPLATE(at::Half, at::Float8_e4m3fn, float, false); +INSTANTIATE_MOE_FP_TEMPLATE(at::BFloat16, uint8_t, uint8_t, true); +INSTANTIATE_MOE_FP_TEMPLATE(at::Half, uint8_t, uint8_t, true); template void shared_expert_fp8_kernel_impl( @@ -266,6 +300,7 @@ void shared_expert_fp8_kernel_impl( /* C */ ic0 + mb * BLOCK_M * 2 * N + nb * BLOCK_N, /* Btmp */ B_tmp + tid * B_tmp_size_per_thread + nb_offset * BLOCK_N * K, /* Ctmp */ C_tmp + tid * 2 * BLOCK_M * BLOCK_N, + /* Bbias */ nullptr, /* scale */ w1s + (nb / blocks_n_per_group) * scale_size_K, /* M */ m_size, /* N */ n_size, @@ -317,6 +352,7 @@ void shared_expert_fp8_kernel_impl( /* C */ C, /* Btmp */ B_tmp + tid * B_tmp_size_per_thread + nb_offset * BLOCK_N * IC, /* Ctmp */ C_tmp + tid * 2 * BLOCK_M * BLOCK_N, + /* Bbias */ nullptr, /* scale */ w2s + (nb / blocks_n_per_group) * scale_size_K, /* M */ m_size, /* N */ n_size, diff --git a/csrc/cpu/sgl-kernels/vec.h b/csrc/cpu/sgl-kernels/vec.h index c045a8cc7d..52b5ff7bed 100644 --- a/csrc/cpu/sgl-kernels/vec.h +++ b/csrc/cpu/sgl-kernels/vec.h @@ -11,7 +11,7 @@ #include #include - +#include namespace { using namespace at::vec; @@ -21,6 +21,15 @@ inline Vectorized convert_from_float_ext(const Vectorized& a, c return at::vec::convert_from_float(a, b); } +template +inline void convert_from_float_and_store(scalar_t* out, const Vectorized& a) { + float out_buffer[at::vec::Vectorized::size()]; + a.store(out_buffer); + for (int i = 0; i < 16; i++) { + out[i] = (scalar_t)out_buffer[i]; + } +} + // allow f16, bf16 template , int> = 1> inline std::tuple, Vectorized> load_float_vec2(const scalar_t* __restrict__ data) { @@ -50,6 +59,11 @@ convert_from_float_ext(const Vectorized& a, const Vectorize return (__m512i)(_mm512_cvtne2ps_pbh(__m512(b), __m512(a))); } +template <> +inline void convert_from_float_and_store(at::BFloat16* out, const Vectorized& a) { + _mm256_storeu_si256((__m256i*)out, (__m256i)(_mm512_cvtneps_pbh(__m512(a)))); +} + #define CVT_BF16_TO_FP32(a) _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(a), 16)) #define CVT_FP16_TO_FP32(a) _mm512_cvtph_ps(a) @@ -125,7 +139,6 @@ inline __m512bh CVT_FP8_TO_BF16(__m256i a) { return cvt_e4m3_bf16_intrinsic_with_denorm(a); #endif } - // faster version of float8_e4m3fn conversion to bfloat16 // // we mapped cuda implementation from below link and vectorized with avx512: diff --git a/csrc/cpu/torch_bindings.cpp b/csrc/cpu/torch_bindings.cpp index bd4559d6b1..35350cf247 100644 --- a/csrc/cpu/torch_bindings.cpp +++ b/csrc/cpu/torch_bindings.cpp @@ -64,6 +64,8 @@ at::Tensor weight_packed_linear(at::Tensor& mat1, at::Tensor& mat2, at::Tensor convert_weight_packed(at::Tensor& weight); +at::Tensor convert_scale_packed(at::Tensor& scale); + at::Tensor fused_experts_cpu( at::Tensor& hidden_states, at::Tensor& w1, at::Tensor& w2, at::Tensor& topk_weights, at::Tensor& topk_ids, bool inplace, @@ -71,7 +73,11 @@ at::Tensor fused_experts_cpu( const std::optional& w2_scale, const std::optional& w1_zero, const std::optional& w2_zero, - const std::optional> block_size, bool is_vnni); + const std::optional> block_size, + const std::optional& w1_bias, + const std::optional& w2_bias, + const std::optional& alpha, const std::optional& limit, + bool is_vnni); at::Tensor int8_scaled_mm_with_quant(at::Tensor& mat1, at::Tensor& mat2, at::Tensor& scales2, @@ -404,11 +410,15 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) { ops.impl("weight_packed_linear", torch::kCPU, &weight_packed_linear); ops.def("convert_weight_packed(Tensor! weight) -> Tensor"); ops.impl("convert_weight_packed", torch::kCPU, &convert_weight_packed); + ops.def("convert_scale_packed(Tensor! scale) -> Tensor"); + ops.impl("convert_scale_packed", torch::kCPU, &convert_scale_packed); ops.def( "fused_experts_cpu(Tensor hidden_states, Tensor w1, Tensor w2, Tensor " "topk_weights, Tensor topk_ids, bool " "inplace, int moe_comp_method, Tensor? w1_scale, Tensor? w2_scale, " - "Tensor? w1_zero, Tensor? w2_zero, int[]? block_size, bool is_vnni) -> " + "Tensor? w1_zero, Tensor? w2_zero, int[]? block_size, " + "Tensor? w1_bias, Tensor? w2_bias, float? alpha, float? limit, " + "bool is_vnni) -> " "Tensor"); ops.impl("fused_experts_cpu", torch::kCPU, &fused_experts_cpu); ops.def( diff --git a/tests/kernels/moe/test_cpu_fp8_fused_moe.py b/tests/kernels/moe/test_cpu_fp8_fused_moe.py deleted file mode 100644 index dbc6cd198c..0000000000 --- a/tests/kernels/moe/test_cpu_fp8_fused_moe.py +++ /dev/null @@ -1,256 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests for CPU FP8 W8A16 block-scaled fused MoE kernel.""" - -import math -import sys - -import pytest -import torch -import torch.nn.functional as F - -from vllm.platforms import current_platform -from vllm.utils.torch_utils import set_random_seed - -if not current_platform.is_cpu(): - pytest.skip("skipping CPU-only tests", allow_module_level=True) - -import vllm._custom_ops as ops # noqa: E402 - -if not hasattr(torch.ops._C, "fused_experts_cpu"): - pytest.skip("fused_experts_cpu op not available", allow_module_level=True) - - -BLOCK_SIZE = [128, 128] # [block_n, block_k] - -# FP8 weight generation parameters -_FP8_INFO = torch.finfo(torch.float8_e4m3fn) -FP8_SCALE = _FP8_INFO.max # 448.0 -FACTOR_FOR_SCALE = 1e-3 - -# Tolerance for FP8 W8A16 -FP8_W8A16_ATOL = 1e-2 -FP8_W8A16_RTOL = 1e-2 - - -def _silu_and_mul(x: torch.Tensor) -> torch.Tensor: - d = x.shape[-1] // 2 - return F.silu(x[..., :d]) * x[..., d:] - - -def _block_dequant_weight( - weight: torch.Tensor, - scales: torch.Tensor, - block_size: list[int], -) -> torch.Tensor: - """Block-dequantize FP8 weight [E, N, K] → float [E, N, K]. - - Each (block_n × block_k) tile is multiplied by its per-block scale. - """ - E, N, K = weight.shape - block_n, block_k = block_size - pad_N = (block_n - N % block_n) % block_n - pad_K = (block_k - K % block_k) % block_k - - if pad_N > 0 or pad_K > 0: - weight = F.pad(weight, (0, pad_K, 0, pad_N)) - - n_tiles = math.ceil(N / block_n) - k_tiles = math.ceil(K / block_k) - - weight_block = ( - weight.view(E, n_tiles, block_n, k_tiles, block_k) - .permute(0, 1, 3, 2, 4) - .float() - .contiguous() - ) - weight_scaled = ( - (weight_block * scales.view(E, n_tiles, k_tiles, 1, 1)) - .permute(0, 1, 3, 2, 4) - .contiguous() - ) - if pad_N > 0 or pad_K > 0: - weight_scaled = weight_scaled.view(E, N + pad_N, K + pad_K) - weight_scaled = weight_scaled[..., :N, :K].contiguous() - else: - weight_scaled = weight_scaled.view(E, N, K) - return weight_scaled - - -def ref_w8a16_block_fp8_moe( - a: torch.Tensor, - w1: torch.Tensor, - w2: torch.Tensor, - w1_s: torch.Tensor, - w2_s: torch.Tensor, - topk_weight: torch.Tensor, - topk_ids: torch.Tensor, - block_size: list[int], -) -> torch.Tensor: - """Reference FP8 W8A16 block-scaled fused MoE in pure torch. - - Steps: - 1. Block-dequant FP8 weights → float - 2. For each expert: matmul → SiLU+Mul → matmul - 3. Weighted sum across top-k experts - """ - B, D = a.shape - topk = topk_ids.size(1) - - w1_dq = _block_dequant_weight(w1, w1_s, block_size) - w2_dq = _block_dequant_weight(w2, w2_s, block_size) - - a_exp = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D).float() - out = torch.zeros(B * topk, w2_dq.shape[1], dtype=torch.float32) - - topk_weight_flat = topk_weight.view(-1) - topk_ids_flat = topk_ids.view(-1) - - for i in range(w1_dq.shape[0]): - mask = topk_ids_flat == i - if mask.sum(): - ic0 = torch.matmul(a_exp[mask], w1_dq[i].transpose(0, 1)) - ic1 = _silu_and_mul(ic0) - out[mask] = torch.matmul(ic1, w2_dq[i].transpose(0, 1)) - - return ( - (out.view(B, -1, w2_dq.shape[1]) * topk_weight_flat.view(B, -1, 1)) - .sum(dim=1) - .to(a.dtype) - ) - - -def _make_fp8_moe_weights( - E: int, - N: int, - K: int, - block_size: list[int], -) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: - """Generate random FP8 MoE weights with random block scales. - - Weight generation follows SGLang: ``randn * FP8_SCALE`` → clamp → cast. - Scales are small random values (``FACTOR_FOR_SCALE``), independent of - the actual weight magnitudes — this is sufficient to test the kernel's - block-dequant + matmul correctness. - - Returns: (w1, w2, w1_s, w2_s) - """ - block_n, block_k = block_size - - w1 = ( - (torch.randn(E, 2 * N, K) * FP8_SCALE) - .clamp(min=-FP8_SCALE, max=FP8_SCALE) - .to(torch.float8_e4m3fn) - ) - w2 = ( - (torch.randn(E, K, N) * FP8_SCALE) - .clamp(min=-FP8_SCALE, max=FP8_SCALE) - .to(torch.float8_e4m3fn) - ) - - w1_s = ( - torch.randn(E, math.ceil(2 * N / block_n), math.ceil(K / block_k)) - * FACTOR_FOR_SCALE - ) - w2_s = ( - torch.randn(E, math.ceil(K / block_n), math.ceil(N / block_k)) - * FACTOR_FOR_SCALE - ) - return w1, w2, w1_s, w2_s - - -def _prepack_experts(w: torch.Tensor) -> torch.Tensor: - """VNNI-prepack each expert's weight via ``convert_weight_packed``.""" - return torch.stack( - [torch.ops._C.convert_weight_packed(w[e]) for e in range(w.shape[0])] - ) - - -NUM_TOKENS = [1, 2, 64, 121] -# (M, intermediate_size N, hidden_size K, num_experts E, topk) -MoE_CONFIGS = [ - (256, 512, 8, 2), - (256, 512, 8, 4), - (512, 256, 8, 2), - (512, 256, 8, 4), - (512, 512, 8, 2), - (512, 512, 8, 4), - (768, 2048, 8, 2), - (768, 2048, 8, 4), - (768, 2048, 128, 8), -] -SEEDS = [0] - - -@pytest.mark.parametrize("M", NUM_TOKENS) -@pytest.mark.parametrize("N,K,E,topk", MoE_CONFIGS) -@pytest.mark.parametrize("seed", SEEDS) -def test_w8a16_block_fp8_cpu_fused_moe(M, N, K, E, topk, seed): - """Test fused_experts_cpu FP8 W8A16 against dequantised torch reference.""" - set_random_seed(seed) - - a = torch.randn(M, K, dtype=torch.bfloat16) / math.sqrt(K) - w1, w2, w1_s, w2_s = _make_fp8_moe_weights(E, N, K, BLOCK_SIZE) - - score = torch.randn(M, E, dtype=torch.bfloat16) - score = torch.softmax(score, dim=-1, dtype=torch.float32) - topk_weight, topk_ids = torch.topk(score, topk) - topk_ids = topk_ids.to(torch.int32) - - ref_out = ref_w8a16_block_fp8_moe( - a, - w1, - w2, - w1_s, - w2_s, - topk_weight, - topk_ids, - BLOCK_SIZE, - ) - - pw1, pw2 = _prepack_experts(w1), _prepack_experts(w2) - - # Test inplace=False against reference - out = ops.fused_experts_cpu( - a.clone(), - pw1, - pw2, - topk_weight, - topk_ids, - False, - ops.CPUQuantMethod.FP8_W8A16, - w1_s, - w2_s, - None, - None, - BLOCK_SIZE, - True, - ) - torch.testing.assert_close( - ref_out.bfloat16(), - out, - atol=FP8_W8A16_ATOL, - rtol=FP8_W8A16_RTOL, - ) - - # Test inplace=True produces identical output - out_inplace = ops.fused_experts_cpu( - a.clone(), - pw1, - pw2, - topk_weight, - topk_ids, - True, - ops.CPUQuantMethod.FP8_W8A16, - w1_s, - w2_s, - None, - None, - BLOCK_SIZE, - True, - ) - torch.testing.assert_close(out_inplace, out, atol=0, rtol=0) - - -if __name__ == "__main__": - sys.exit(pytest.main([__file__, "-v"])) diff --git a/tests/kernels/moe/test_cpu_quant_fused_moe.py b/tests/kernels/moe/test_cpu_quant_fused_moe.py new file mode 100644 index 0000000000..f8967b1992 --- /dev/null +++ b/tests/kernels/moe/test_cpu_quant_fused_moe.py @@ -0,0 +1,500 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for CPU quantized fused MoE kernels (FP8 W8A16 and MXFP4 W4A16).""" + +import math +import sys + +import pytest +import torch +import torch.nn.functional as F + +from vllm.platforms import current_platform +from vllm.utils.torch_utils import set_random_seed + +if not current_platform.is_cpu(): + pytest.skip("skipping CPU-only tests", allow_module_level=True) + +import vllm._custom_ops as ops # noqa: E402 + +if not hasattr(torch.ops._C, "fused_experts_cpu"): + pytest.skip("fused_experts_cpu op not available", allow_module_level=True) + + +def _silu_and_mul(x: torch.Tensor) -> torch.Tensor: + d = x.shape[-1] // 2 + return F.silu(x[..., :d]) * x[..., d:] + + +def _prepack_experts(w: torch.Tensor) -> torch.Tensor: + """VNNI-prepack expert weights via ``convert_weight_packed``.""" + return torch.ops._C.convert_weight_packed(w) + + +# FP8 W8A16 block-scaled fused MoE + +BLOCK_SIZE = [128, 128] # [block_n, block_k] + +_FP8_INFO = torch.finfo(torch.float8_e4m3fn) +FP8_SCALE = _FP8_INFO.max # 448.0 +FACTOR_FOR_SCALE = 1e-3 + + +def _block_dequant_weight( + weight: torch.Tensor, + scales: torch.Tensor, + block_size: list[int], +) -> torch.Tensor: + """Block-dequantize FP8 weight [E, N, K] -> float [E, N, K].""" + E, N, K = weight.shape + block_n, block_k = block_size + pad_N = (block_n - N % block_n) % block_n + pad_K = (block_k - K % block_k) % block_k + + if pad_N > 0 or pad_K > 0: + weight = F.pad(weight, (0, pad_K, 0, pad_N)) + + n_tiles = math.ceil(N / block_n) + k_tiles = math.ceil(K / block_k) + + weight_block = ( + weight.view(E, n_tiles, block_n, k_tiles, block_k) + .permute(0, 1, 3, 2, 4) + .float() + .contiguous() + ) + weight_scaled = ( + (weight_block * scales.view(E, n_tiles, k_tiles, 1, 1)) + .permute(0, 1, 3, 2, 4) + .contiguous() + ) + if pad_N > 0 or pad_K > 0: + weight_scaled = weight_scaled.view(E, N + pad_N, K + pad_K) + weight_scaled = weight_scaled[..., :N, :K].contiguous() + else: + weight_scaled = weight_scaled.view(E, N, K) + return weight_scaled + + +def ref_w8a16_block_fp8_moe( + a: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + w1_s: torch.Tensor, + w2_s: torch.Tensor, + topk_weight: torch.Tensor, + topk_ids: torch.Tensor, + block_size: list[int], +) -> torch.Tensor: + """Reference FP8 W8A16 block-scaled fused MoE in pure torch.""" + B, D = a.shape + topk = topk_ids.size(1) + + w1_dq = _block_dequant_weight(w1, w1_s, block_size) + w2_dq = _block_dequant_weight(w2, w2_s, block_size) + + a_exp = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D).float() + out = torch.zeros(B * topk, w2_dq.shape[1], dtype=torch.float32) + + topk_weight_flat = topk_weight.view(-1) + topk_ids_flat = topk_ids.view(-1) + + for i in range(w1_dq.shape[0]): + mask = topk_ids_flat == i + if mask.sum(): + ic0 = torch.matmul(a_exp[mask], w1_dq[i].transpose(0, 1)) + ic1 = _silu_and_mul(ic0) + out[mask] = torch.matmul(ic1, w2_dq[i].transpose(0, 1)) + + return ( + (out.view(B, -1, w2_dq.shape[1]) * topk_weight_flat.view(B, -1, 1)) + .sum(dim=1) + .to(a.dtype) + ) + + +def _make_fp8_moe_weights( + E: int, + N: int, + K: int, + block_size: list[int], +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Generate random FP8 MoE weights with random block scales.""" + block_n, block_k = block_size + + w1 = ( + (torch.randn(E, 2 * N, K) * FP8_SCALE) + .clamp(min=-FP8_SCALE, max=FP8_SCALE) + .to(torch.float8_e4m3fn) + ) + w2 = ( + (torch.randn(E, K, N) * FP8_SCALE) + .clamp(min=-FP8_SCALE, max=FP8_SCALE) + .to(torch.float8_e4m3fn) + ) + + w1_s = ( + torch.randn(E, math.ceil(2 * N / block_n), math.ceil(K / block_k)) + * FACTOR_FOR_SCALE + ) + w2_s = ( + torch.randn(E, math.ceil(K / block_n), math.ceil(N / block_k)) + * FACTOR_FOR_SCALE + ) + return w1, w2, w1_s, w2_s + + +FP8_NUM_TOKENS = [1, 2, 64, 121] +FP8_MOE_CONFIGS = [ + (256, 512, 8, 2), + (256, 512, 8, 4), + (512, 256, 8, 2), + (512, 256, 8, 4), + (512, 512, 8, 2), + (512, 512, 8, 4), + (768, 2048, 8, 2), + (768, 2048, 8, 4), + (768, 2048, 128, 8), +] + + +@pytest.mark.parametrize("M", FP8_NUM_TOKENS) +@pytest.mark.parametrize("N,K,E,topk", FP8_MOE_CONFIGS) +@pytest.mark.parametrize("seed", [0]) +def test_w8a16_block_fp8_cpu_fused_moe(M, N, K, E, topk, seed): + """Test fused_experts_cpu FP8 W8A16 against dequantised torch reference.""" + set_random_seed(seed) + + a = torch.randn(M, K, dtype=torch.bfloat16) / math.sqrt(K) + w1, w2, w1_s, w2_s = _make_fp8_moe_weights(E, N, K, BLOCK_SIZE) + + score = torch.randn(M, E, dtype=torch.bfloat16) + score = torch.softmax(score, dim=-1, dtype=torch.float32) + topk_weight, topk_ids = torch.topk(score, topk) + topk_ids = topk_ids.to(torch.int32) + + ref_out = ref_w8a16_block_fp8_moe( + a, w1, w2, w1_s, w2_s, topk_weight, topk_ids, BLOCK_SIZE + ) + + pw1, pw2 = _prepack_experts(w1), _prepack_experts(w2) + + # Test inplace=False against reference + out = ops.fused_experts_cpu( + a.clone(), + pw1, + pw2, + topk_weight, + topk_ids, + False, + ops.CPUQuantMethod.FP8_W8A16, + w1_s, + w2_s, + None, + None, + BLOCK_SIZE, + is_vnni=True, + ) + torch.testing.assert_close(ref_out.bfloat16(), out, atol=1e-2, rtol=1e-2) + + # Test inplace=True produces identical output + out_inplace = ops.fused_experts_cpu( + a.clone(), + pw1, + pw2, + topk_weight, + topk_ids, + True, + ops.CPUQuantMethod.FP8_W8A16, + w1_s, + w2_s, + None, + None, + BLOCK_SIZE, + is_vnni=True, + ) + torch.testing.assert_close(out_inplace, out, atol=0, rtol=0) + + +# MXFP4 W4A16 fused MoE + + +class MXFP4QuantizeUtil: + """MXFP4 quantization utility.""" + + E2M1_max = 6.0 + E2M1_values = [0, 0.5, 1, 1.5, 2, 3, 4, 6] + E2M1_bounds = torch.tensor([0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5]) + block_size = 32 + + @classmethod + def quantize(cls, input: torch.Tensor) -> tuple: + """Quantize BF16 tensor to MXFP4 packed uint8 format.""" + + def cast_fp4(x): + sign = torch.sign(x) + sign_bit = (2 - sign) // 2 + ord_ = torch.sum( + (x.abs().unsqueeze(-1) - cls.E2M1_bounds.to(x.device)) > 0, dim=-1 + ) + fp4_val = (sign_bit * 0b1000 + ord_).to(torch.uint8) + return fp4_val + + def fuse_uint4_to_uint8(x): + left_side = x[..., 0::2] + right_side = x[..., 1::2] + new_data = right_side.clone() << 4 + new_data[..., : left_side.shape[-1]] += left_side + return new_data + + original_shape = input.shape + input = input.view(-1, cls.block_size) + input_amax = input.abs().max(dim=-1, keepdim=True).values + descale = input_amax / cls.E2M1_max + min_value = torch.tensor(-127.0, device=descale.device) + e8m0_scale = torch.ceil(torch.maximum(torch.log2(descale), min_value)) + + input = (input / torch.exp2(e8m0_scale)).view(original_shape) + input_q = cast_fp4(input) + input_q = fuse_uint4_to_uint8(input_q) + e8m0_scale = (e8m0_scale + 127).to(torch.uint8) + return input_q, e8m0_scale + + @classmethod + def dequantize(cls, quantized_data, dtype: torch.dtype, scale): + """Dequantize MXFP4 packed tensor back to float.""" + + def unfuse_uint8_to_uint4(x): + left_side = x & 0x0F + right_side = (x >> 4) & 0x0F + shape = list(x.shape) + shape[-1] = shape[-1] * 2 + result = torch.zeros(shape, dtype=torch.uint8, device=x.device) + result[..., 0::2] = left_side + result[..., 1::2] = right_side + return result + + e8m0_scale = scale + x_unfused = unfuse_uint8_to_uint4(quantized_data) + sign = 1 - 2 * ((x_unfused & 0b1000) >> 3).to(torch.float32) + magnitude = (x_unfused & 0b0111).to(torch.long) + values = torch.tensor(cls.E2M1_values, device=quantized_data.device) + original_shape = magnitude.shape + x_float = values[magnitude.reshape(-1)].reshape(original_shape) + x_float = sign.float() * x_float + x_float = x_float.reshape(-1, cls.block_size) + scale_factor = torch.exp2(e8m0_scale.float() - 127) + scale_factor = scale_factor.reshape(-1, 1) + x_float = x_float * scale_factor + return x_float.reshape(original_shape).to(dtype) + + +def _swiglu(x: torch.Tensor, alpha: float, limit: float) -> torch.Tensor: + """SwigLU activation used in GPT-OSS. + Input is interleaved: [gate_0, up_0, gate_1, up_1, ...] in last dim. + """ + gate = x[..., 0::2] + up = x[..., 1::2] + gate_clamped = torch.clamp(gate, max=limit) + up_clamped = torch.clamp(up, min=-limit, max=limit) + return gate_clamped * torch.sigmoid(alpha * gate_clamped) * (up_clamped + 1) + + +def ref_mxfp4_fused_moe( + a: torch.Tensor, + w1_dq: torch.Tensor, + w2_dq: torch.Tensor, + topk_weight: torch.Tensor, + topk_ids: torch.Tensor, + topk: int, +) -> torch.Tensor: + """Reference MXFP4 fused MoE with SiLU activation.""" + B, D = a.shape + a_f = a.float() + out = torch.zeros(B * topk, w2_dq.shape[1], dtype=torch.float32) + topk_ids_flat = topk_ids.view(-1) + + for i in range(w1_dq.shape[0]): + mask = topk_ids_flat == i + if mask.sum() == 0: + continue + token_indices = torch.where(mask)[0] + source_indices = token_indices // topk + ic0 = torch.matmul(a_f[source_indices], w1_dq[i].float().T) + ic1 = _silu_and_mul(ic0) + out[mask] = torch.matmul(ic1, w2_dq[i].float().T) + + return (out.view(B, topk, -1) * topk_weight.unsqueeze(-1)).sum(dim=1).to(a.dtype) + + +def ref_mxfp4_fused_moe_gptoss( + a: torch.Tensor, + w1_dq: torch.Tensor, + w2_dq: torch.Tensor, + w1_bias: torch.Tensor, + w2_bias: torch.Tensor, + topk_weight: torch.Tensor, + topk_ids: torch.Tensor, + alpha: float, + limit: float, +) -> torch.Tensor: + """Reference MXFP4 fused MoE with SwigLU+bias (GPT-OSS style).""" + B, D = a.shape + topk = topk_ids.shape[1] + a_f = a.float() + E = w1_dq.shape[0] + out = torch.zeros(B * topk, w2_dq.shape[1], dtype=torch.float32) + topk_ids_flat = topk_ids.view(-1) + + for i in range(E): + mask = topk_ids_flat == i + if mask.sum() == 0: + continue + token_indices = torch.where(mask)[0] + source_indices = token_indices // topk + ic0 = torch.matmul(a_f[source_indices], w1_dq[i].float().T) + ic0 = ic0 + w1_bias[i].float() + ic1 = _swiglu(ic0, alpha, limit) + ic2 = torch.matmul(ic1, w2_dq[i].float().T) + ic2 = ic2 + w2_bias[i].float() + out[mask] = ic2 + + return (out.view(B, topk, -1) * topk_weight.unsqueeze(-1)).sum(dim=1).to(a.dtype) + + +def _prepack_mxfp4_experts( + w: torch.Tensor, w_scale: torch.Tensor +) -> tuple[torch.Tensor, torch.Tensor]: + """VNNI-prepack MXFP4 weights and repack scales.""" + packed_w = torch.ops._C.convert_weight_packed(w) + packed_s = torch.ops._C.convert_scale_packed(w_scale) + return packed_w, packed_s + + +MXFP4_NUM_TOKENS = [1, 2, 32, 121] +MXFP4_MOE_CONFIGS = [ + (128, 128, 4, 2), + (256, 256, 8, 4), + (352, 256, 8, 4), + (512, 320, 8, 4), +] + + +@pytest.mark.parametrize("M", MXFP4_NUM_TOKENS) +@pytest.mark.parametrize("N,K,E,topk", MXFP4_MOE_CONFIGS) +@pytest.mark.parametrize("seed", [0]) +def test_mxfp4_cpu_fused_moe(M, N, K, E, topk, seed): + """Test fused_experts_mxfp4_cpu against dequantized torch reference.""" + set_random_seed(seed) + dtype = torch.bfloat16 + + a = torch.randn(M, K, dtype=dtype) / 10 + + # Generate and quantize weights + w1_bf16 = torch.randn(E, 2 * N, K, dtype=dtype) / 10 + w1q, w1s = MXFP4QuantizeUtil.quantize(w1_bf16) + w1s = w1s.reshape(E, 2 * N, K // 32) + w1dq = MXFP4QuantizeUtil.dequantize(w1q, dtype, w1s) + + w2_bf16 = torch.randn(E, K, N, dtype=dtype) / 10 + w2q, w2s = MXFP4QuantizeUtil.quantize(w2_bf16) + w2s = w2s.reshape(E, K, N // 32) + w2dq = MXFP4QuantizeUtil.dequantize(w2q, dtype, w2s) + + # Routing + score = torch.randn(M, E, dtype=dtype) + score = torch.softmax(score, dim=-1, dtype=torch.float32) + topk_weight, topk_ids = torch.topk(score, topk) + topk_ids = topk_ids.to(torch.int32) + + # Reference + ref_out = ref_mxfp4_fused_moe(a, w1dq, w2dq, topk_weight, topk_ids, topk) + + # Pack weights for kernel + pw1, pw1s = _prepack_mxfp4_experts(w1q, w1s) + pw2, pw2s = _prepack_mxfp4_experts(w2q, w2s) + + # Kernel + out = ops.fused_experts_cpu( + a.clone(), + pw1, + pw2, + topk_weight, + topk_ids, + False, # inplace + ops.CPUQuantMethod.MXFP4, + pw1s, # w1_scale + pw2s, # w2_scale + None, # w1_zero + None, # w2_zero + None, # block_size + ) + + torch.testing.assert_close(ref_out.bfloat16(), out, atol=1e-2, rtol=1e-2) + + +@pytest.mark.parametrize("M", [1, 32]) +@pytest.mark.parametrize("N,K,E,topk", [(128, 128, 4, 2), (64, 64, 4, 2)]) +@pytest.mark.parametrize("seed", [0]) +def test_mxfp4_cpu_fused_moe_bias_swiglu(M, N, K, E, topk, seed): + """Test fused_experts_mxfp4_cpu with bias and SwigLU activation (GPT-OSS).""" + set_random_seed(seed) + dtype = torch.bfloat16 + alpha = 1.702 + limit = 7.0 + + a = torch.randn(M, K, dtype=dtype) / 10 + + # Generate and quantize weights + w1_bf16 = torch.randn(E, 2 * N, K, dtype=dtype) / 10 + w1q, w1s = MXFP4QuantizeUtil.quantize(w1_bf16) + w1s = w1s.reshape(E, 2 * N, K // 32) + w1dq = MXFP4QuantizeUtil.dequantize(w1q, dtype, w1s) + w1_b = torch.randn(E, 2 * N, dtype=torch.float32) / 10 + + w2_bf16 = torch.randn(E, K, N, dtype=dtype) / 10 + w2q, w2s = MXFP4QuantizeUtil.quantize(w2_bf16) + w2s = w2s.reshape(E, K, N // 32) + w2dq = MXFP4QuantizeUtil.dequantize(w2q, dtype, w2s) + w2_b = torch.randn(E, K, dtype=torch.float32) / 10 + + # Routing + score = torch.randn(M, E, dtype=dtype) + score = torch.softmax(score, dim=-1, dtype=torch.float32) + topk_weight, topk_ids = torch.topk(score, topk) + topk_ids = topk_ids.to(torch.int32) + + # Reference + ref_out = ref_mxfp4_fused_moe_gptoss( + a, w1dq, w2dq, w1_b, w2_b, topk_weight, topk_ids, alpha, limit + ) + + # Pack weights for kernel + pw1, pw1s = _prepack_mxfp4_experts(w1q, w1s) + pw2, pw2s = _prepack_mxfp4_experts(w2q, w2s) + + # Kernel + out = ops.fused_experts_cpu( + a.clone(), + pw1, + pw2, + topk_weight, + topk_ids, + False, # inplace + ops.CPUQuantMethod.MXFP4, + pw1s, # w1_scale + pw2s, # w2_scale + None, # w1_zero + None, # w2_zero + None, # block_size + w1_bias=w1_b, + w2_bias=w2_b, + alpha=alpha, + limit=limit, + ) + + torch.testing.assert_close(ref_out.bfloat16(), out, atol=1e-2, rtol=1e-2) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/tests/quantization/test_cpu_wna16.py b/tests/quantization/test_cpu_wna16.py index 1fbe222019..5414d7571a 100644 --- a/tests/quantization/test_cpu_wna16.py +++ b/tests/quantization/test_cpu_wna16.py @@ -15,6 +15,7 @@ MODELS = [ "OPEA/Qwen2.5-0.5B-Instruct-int4-sym-inc", "Qwen/Qwen3-0.6B-FP8", # FP8 W8A16 block-quantized linear "Qwen/Qwen3-30B-A3B-FP8", # FP8 W8A16 block-quantized MoE + "openai/gpt-oss-20b", # MXFP4 W4A16 ] DTYPE = ["bfloat16"] diff --git a/vllm/_custom_ops.py b/vllm/_custom_ops.py index bda8fbf392..2346cb40a0 100644 --- a/vllm/_custom_ops.py +++ b/vllm/_custom_ops.py @@ -3195,6 +3195,7 @@ class CPUQuantMethod(IntEnum): INT8_W8A8 = 1 FP8_W8A16 = 2 INT4_W4A8 = 3 + MXFP4 = 4 if hasattr(torch.ops._C, "fused_experts_cpu"): @@ -3213,6 +3214,10 @@ if hasattr(torch.ops._C, "fused_experts_cpu"): w1_zero: torch.Tensor | None, w2_zero: torch.Tensor | None, block_size: list[int] | None, + w1_bias: torch.Tensor | None, + w2_bias: torch.Tensor | None, + alpha: float | None, + limit: float | None, is_vnni: bool, ) -> torch.Tensor: return torch.empty_like(hidden_states) @@ -3231,7 +3236,11 @@ def fused_experts_cpu( w1_zero: torch.Tensor | None, w2_zero: torch.Tensor | None, block_size: list[int] | None, - is_vnni: bool, + w1_bias: torch.Tensor | None = None, + w2_bias: torch.Tensor | None = None, + alpha: float | None = None, + limit: float | None = None, + is_vnni: bool = True, ) -> torch.Tensor: return torch.ops._C.fused_experts_cpu( hidden_states, @@ -3246,6 +3255,10 @@ def fused_experts_cpu( w1_zero, w2_zero, block_size, + w1_bias, + w2_bias, + alpha, + limit, is_vnni, ) diff --git a/vllm/model_executor/layers/fused_moe/cpu_fused_moe.py b/vllm/model_executor/layers/fused_moe/cpu_fused_moe.py index c26090a56b..9192b6a9b7 100644 --- a/vllm/model_executor/layers/fused_moe/cpu_fused_moe.py +++ b/vllm/model_executor/layers/fused_moe/cpu_fused_moe.py @@ -213,6 +213,10 @@ class SGLFusedMOE: None, # w1_zero None, # w2_zero None, # block_size + None, # w1_bias + None, # w2_bias + None, # alpha + None, # limit True, # is_vnni ) diff --git a/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py b/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py index 3b3d5cdfae..54b264ef77 100644 --- a/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""CPU FP8 W8A16 block-quantized fused MoE experts.""" +"""CPU FP8 W8A16 and MXFP4 W4A16 fused MoE experts.""" import torch @@ -17,6 +17,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( QuantKey, kFp8Dynamic128Sym, kFp8Static128BlockSym, + kMxfp4Static, ) from vllm.platforms import current_platform @@ -26,14 +27,8 @@ def prepare_fp8_moe_layer_for_cpu( w2: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor]: """VNNI-prepack FP8 MoE weights for CPU kernel.""" - num_experts = w13.size(0) - packed_w13_list = [] - packed_w2_list = [] - for i in range(num_experts): - packed_w13_list.append(torch.ops._C.convert_weight_packed(w13[i])) - packed_w2_list.append(torch.ops._C.convert_weight_packed(w2[i])) - packed_w13 = torch.stack(packed_w13_list) - packed_w2 = torch.stack(packed_w2_list) + packed_w13 = torch.ops._C.convert_weight_packed(w13) + packed_w2 = torch.ops._C.convert_weight_packed(w2) return packed_w13, packed_w2 @@ -171,5 +166,161 @@ class CPUExpertsFp8(mk.FusedMoEExpertsMonolithic): None, # w1_zero None, # w2_zero block_shape, # block_size + None, # w1_bias + None, # w2_bias + None, # alpha + None, # limit + True, # is_vnni + ) + + +def prepare_mxfp4_moe_layer_for_cpu( + w13: torch.Tensor, + w2: torch.Tensor, + w13_scale: torch.Tensor, + w2_scale: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """VNNI-prepack MXFP4 MoE weights and repack scales for CPU AMX kernel.""" + packed_w13 = torch.ops._C.convert_weight_packed(w13) + packed_w2 = torch.ops._C.convert_weight_packed(w2) + packed_w13_scale = torch.ops._C.convert_scale_packed(w13_scale) + packed_w2_scale = torch.ops._C.convert_scale_packed(w2_scale) + return packed_w13, packed_w2, packed_w13_scale, packed_w2_scale + + +class CPUExpertsMxfp4(mk.FusedMoEExpertsMonolithic): + """CPU MXFP4 W4A16 monolithic MoE experts.""" + + def __init__( + self, + moe_config: FusedMoEConfig, + quant_config: FusedMoEQuantConfig, + ): + super().__init__( + moe_config, + quant_config, + ) + + @property + def expects_unquantized_inputs(self) -> bool: + return True + + @staticmethod + def activation_format() -> mk.FusedMoEActivationFormat: + return mk.FusedMoEActivationFormat.Standard + + @staticmethod + def _supports_current_device() -> bool: + return current_platform.is_cpu() + + @staticmethod + def _supports_no_act_and_mul() -> bool: + return False + + @staticmethod + def _supports_activation(activation: MoEActivation) -> bool: + return activation in (MoEActivation.SILU, MoEActivation.SWIGLUOAI) + + @staticmethod + def _supports_parallel_config( + moe_parallel_config: FusedMoEParallelConfig, + ) -> bool: + return True + + @staticmethod + def _supports_quant_scheme( + weight_key: QuantKey | None, + activation_key: QuantKey | None, + ) -> bool: + SUPPORTED_W_A = [ + (kMxfp4Static, None), + ] + return (weight_key, activation_key) in SUPPORTED_W_A + + @staticmethod + def _supports_routing_method( + routing_method: RoutingMethodType, + weight_key: QuantKey | None, + activation_key: QuantKey | None, + ) -> bool: + return routing_method in [ + RoutingMethodType.Default, + RoutingMethodType.Renormalize, + RoutingMethodType.RenormalizeNaive, + ] + + @staticmethod + def _supports_router_logits_dtype( + router_logits_dtype: torch.dtype | None, + routing_method: RoutingMethodType, + ) -> bool: + return True + + def supports_expert_map(self) -> bool: + return False + + def apply( + self, + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + router_logits: torch.Tensor, + activation: MoEActivation, + global_num_experts: int, + expert_map: torch.Tensor | None, + a1q_scale: torch.Tensor | None, + apply_router_weight_on_input: bool, + # grouped topk + fused topk bias parameters + num_expert_group: int | None = None, + e_score_correction_bias: torch.Tensor | None = None, + routed_scaling_factor: float | None = None, + topk_group: int | None = None, + ) -> torch.Tensor: + from vllm.model_executor.layers.fused_moe.cpu_fused_moe import ( + select_experts, + ) + + topk_weights, topk_ids = select_experts( + hidden_states=hidden_states, + router_logits=router_logits, + use_grouped_topk=num_expert_group is not None, + top_k=self.moe_config.experts_per_token, + renormalize=self.moe_config.routing_method + in ( + RoutingMethodType.Renormalize, + RoutingMethodType.RenormalizeNaive, + ), + topk_group=topk_group, + num_expert_group=num_expert_group, + scoring_func="softmax", + routed_scaling_factor=( + routed_scaling_factor if routed_scaling_factor is not None else 1.0 + ), + e_score_correction_bias=e_score_correction_bias, + ) + + # Get bias and swiglu params from quant config + w1_bias = self.quant_config.w1_bias + w2_bias = self.quant_config.w2_bias + alpha = getattr(self.quant_config, "gemm1_alpha", None) + limit = getattr(self.quant_config, "gemm1_clamp_limit", None) + + return fused_experts_cpu( + hidden_states, + w1, + w2, + topk_weights, + topk_ids, + False, # inplace + CPUQuantMethod.MXFP4, # moe_comp_method + self.w1_scale, # w1_scale + self.w2_scale, # w2_scale + None, # w1_zero + None, # w2_zero + None, # block_size + w1_bias, + w2_bias, + alpha, + limit, True, # is_vnni ) diff --git a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py index cbc38409c3..111a6f2b16 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py @@ -79,6 +79,8 @@ class Mxfp4MoeBackend(Enum): TRITON_UNFUSED = "TRITON_UNFUSED" # XPU XPU = "XPU" + # CPU + CPU = "CPU" # Emulation EMULATION = "EMULATION" # Humming @@ -198,6 +200,11 @@ def backend_to_kernel_cls( return [XPUExpertsMXFp4] + elif backend == Mxfp4MoeBackend.CPU: + from vllm.model_executor.layers.fused_moe.experts.cpu_moe import CPUExpertsMxfp4 + + return [CPUExpertsMxfp4] + elif backend == Mxfp4MoeBackend.EMULATION: from vllm.model_executor.layers.fused_moe.experts.ocp_mx_emulation_moe import ( OCP_MXQuantizationEmulationTritonExperts, @@ -237,6 +244,7 @@ def map_mxfp4_backend(runner_backend: MoEBackend) -> list[Mxfp4MoeBackend]: ], "aiter_mxfp4_fp8": [Mxfp4MoeBackend.AITER_MXFP4_FP8], "xpu": [Mxfp4MoeBackend.XPU], + "cpu": [Mxfp4MoeBackend.CPU], "emulation": [Mxfp4MoeBackend.EMULATION], } if backends := mapping.get(runner_backend): @@ -535,6 +543,17 @@ def select_mxfp4_moe_backend( activation_format, ) + if current_platform.is_cpu(): + backend = Mxfp4MoeBackend.CPU + logger.info_once(_make_log_backend(backend)) + return _return_or_raise( + Mxfp4MoeBackend.CPU, + config, + kMxfp4Static, + None, + activation_format, + ) + if current_platform.is_cuda() or current_platform.is_rocm(): raise NotImplementedError( "No MXFP4 MoE backend supports the deployment configuration." @@ -639,6 +658,10 @@ def mxfp4_round_up_hidden_size_and_intermediate_size( elif current_platform.is_rocm(): intermediate_size = round_up(intermediate_size, 256) hidden_size = round_up(hidden_size, 256) + elif backend == Mxfp4MoeBackend.CPU: + # CPU AMX kernel uses BLOCK_N=32, align to 32 + intermediate_size = round_up(intermediate_size, 32) + hidden_size = round_up(hidden_size, 32) else: intermediate_size = round_up(intermediate_size, 64) return hidden_size, intermediate_size @@ -1098,6 +1121,31 @@ def convert_gpt_oss_weight_to_mxfp4_moe_kernel_format( w13_bias, w2_bias, ) + elif mxfp4_backend == Mxfp4MoeBackend.CPU: + from vllm.model_executor.layers.fused_moe.experts.cpu_moe import ( + prepare_mxfp4_moe_layer_for_cpu, + ) + + packed_w13, packed_w2, packed_w13_scale, packed_w2_scale = ( + prepare_mxfp4_moe_layer_for_cpu( + w13_weight.data, + w2_weight.data, + w13_weight_scale.data, + w2_weight_scale.data, + ) + ) + if w13_bias is not None: + w13_bias = w13_bias.data.to(torch.float32) + if w2_bias is not None: + w2_bias = w2_bias.data.to(torch.float32) + return ( + packed_w13, + packed_w2, + packed_w13_scale, + packed_w2_scale, + w13_bias, + w2_bias, + ) elif mxfp4_backend == Mxfp4MoeBackend.EMULATION: # No additional transformation needed for emulation backend, # weights are dequantized on the fly in the experts class. @@ -1501,6 +1549,7 @@ def make_mxfp4_moe_quant_config( Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16, Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16, Mxfp4MoeBackend.AITER_MXFP4_BF16, + Mxfp4MoeBackend.CPU, ): return mxfp4_w4a16_moe_quant_config( w1_bias=w1_bias,