From 061043eacac0473769d86a1babb9adeb8402e833 Mon Sep 17 00:00:00 2001 From: Fadi Arafeh <115173828+fadara01@users.noreply.github.com> Date: Wed, 24 Jun 2026 15:14:35 +0100 Subject: [PATCH] [CPU][Perf] Accelerate unquantized MoE for AArch64 (#46353) Signed-off-by: Fadi Arafeh --- .../scripts/hardware_ci/run-cpu-test-arm.sh | 4 +- .../kernels/cpu/benchmark_cpu_fused_moe.py | 15 +- cmake/cpu_extension.cmake | 1 + csrc/cpu/cpu_fused_moe.cpp | 90 +++- csrc/cpu/cpu_types_arm.hpp | 31 +- csrc/cpu/cpu_types_scalar.hpp | 7 + csrc/cpu/cpu_types_x86.hpp | 9 + csrc/cpu/micro_gemm/cpu_micro_gemm_amx.hpp | 2 + csrc/cpu/micro_gemm/cpu_micro_gemm_impl.hpp | 3 + csrc/cpu/micro_gemm/cpu_micro_gemm_neon.hpp | 503 ++++++++++++++++++ csrc/cpu/micro_gemm/cpu_micro_gemm_vec.hpp | 2 + csrc/cpu/torch_bindings.cpp | 4 +- csrc/cpu/utils.hpp | 5 +- tests/kernels/moe/test_cpu_fused_moe.py | 16 +- .../layers/fused_moe/cpu_fused_moe.py | 12 + 15 files changed, 672 insertions(+), 32 deletions(-) create mode 100644 csrc/cpu/micro_gemm/cpu_micro_gemm_neon.hpp diff --git a/.buildkite/scripts/hardware_ci/run-cpu-test-arm.sh b/.buildkite/scripts/hardware_ci/run-cpu-test-arm.sh index 35513727f1..252eeeef8c 100755 --- a/.buildkite/scripts/hardware_ci/run-cpu-test-arm.sh +++ b/.buildkite/scripts/hardware_ci/run-cpu-test-arm.sh @@ -8,7 +8,7 @@ set -ex CORE_RANGE=${CORE_RANGE:-0-31} OMP_CORE_RANGE=${OMP_CORE_RANGE:-0-31} -export CMAKE_BUILD_PARALLEL_LEVEL=16 +export CMAKE_BUILD_PARALLEL_LEVEL=32 # Setup cleanup remove_docker_container() { @@ -37,7 +37,7 @@ function cpu_tests() { pytest -x -v -s tests/kernels/test_onednn.py pytest -x -v -s tests/kernels/attention/test_cpu_attn.py pytest -x -v -s tests/kernels/core/test_cpu_activation.py - pytest -x -v -s tests/kernels/moe/test_moe.py -k test_cpu_fused_moe_basic + pytest -x -v -s tests/kernels/moe/test_cpu_fused_moe.py pytest -x -v -s tests/kernels/mamba/cpu/test_cpu_gdn_ops.py" # skip tests requiring model downloads if HF_TOKEN is not set diff --git a/benchmarks/kernels/cpu/benchmark_cpu_fused_moe.py b/benchmarks/kernels/cpu/benchmark_cpu_fused_moe.py index aff443083a..f5a5ed1dc5 100644 --- a/benchmarks/kernels/cpu/benchmark_cpu_fused_moe.py +++ b/benchmarks/kernels/cpu/benchmark_cpu_fused_moe.py @@ -7,6 +7,7 @@ import time import numpy as np import torch +from vllm.platforms import CpuArchEnum, current_platform from vllm.utils.argparse_utils import FlexibleArgumentParser from vllm.utils.torch_utils import set_random_seed @@ -14,17 +15,15 @@ from vllm.utils.torch_utils import set_random_seed try: from vllm._custom_ops import cpu_fused_moe, cpu_prepack_moe_weight except (ImportError, AttributeError) as e: - print("ERROR: CPU fused MoE operations are not available on this platform.") - print("This benchmark requires x86 CPU with proper vLLM CPU extensions compiled.") - print( - "The cpu_fused_moe kernel is typically available on Linux x86_64 " - "with AVX2/AVX512." - ) print(f"Import error: {e}") sys.exit(1) # ISA selection following test_cpu_fused_moe.py pattern -ISA_CHOICES = ["amx", "vec"] if torch.cpu._is_amx_tile_supported() else ["vec"] +ISA_CHOICES = ["vec"] +if torch.cpu._is_amx_tile_supported(): + ISA_CHOICES.append("amx") +if current_platform.get_cpu_architecture() == CpuArchEnum.ARM: + ISA_CHOICES.append("neon") @torch.inference_mode() @@ -145,7 +144,7 @@ if __name__ == "__main__": "--isa", type=str, choices=ISA_CHOICES, - default=ISA_CHOICES[0], + default="vec", help=f"ISA to use (available: {ISA_CHOICES})", ) parser.add_argument("--seed", type=int, default=0) diff --git a/cmake/cpu_extension.cmake b/cmake/cpu_extension.cmake index b39112d24c..386f9e30c7 100644 --- a/cmake/cpu_extension.cmake +++ b/cmake/cpu_extension.cmake @@ -423,6 +423,7 @@ if (ASIMD_FOUND AND NOT APPLE_SILICON_FOUND) set(VLLM_EXT_SRC "csrc/cpu/shm.cpp" "csrc/cpu/activation_lut_bf16.cpp" + "csrc/cpu/cpu_fused_moe.cpp" ${VLLM_EXT_SRC}) endif() diff --git a/csrc/cpu/cpu_fused_moe.cpp b/csrc/cpu/cpu_fused_moe.cpp index c0d92bde77..35c23df97b 100644 --- a/csrc/cpu/cpu_fused_moe.cpp +++ b/csrc/cpu/cpu_fused_moe.cpp @@ -1,5 +1,3 @@ -#include - #include "cpu/cpu_types.hpp" #include "cpu/utils.hpp" #include "cpu/micro_gemm/cpu_micro_gemm_vec.hpp" @@ -16,6 +14,18 @@ #define AMX_DISPATCH(...) case cpu_utils::ISA::AMX: #endif +#if defined(ARM_BF16_SUPPORT) + #include "cpu/micro_gemm/cpu_micro_gemm_neon.hpp" + #define NEON_DISPATCH(...) \ + case cpu_utils::ISA::NEON: { \ + using gemm_t = \ + cpu_micro_gemm::MicroGemm; \ + return __VA_ARGS__(); \ + } +#else + #define NEON_DISPATCH(...) case cpu_utils::ISA::NEON: +#endif + #define CPU_ISA_DISPATCH_IMPL(ISA_TYPE, ...) \ [&] { \ switch (ISA_TYPE) { \ @@ -25,6 +35,7 @@ cpu_micro_gemm::MicroGemm; \ return __VA_ARGS__(); \ } \ + NEON_DISPATCH(__VA_ARGS__) \ default: { \ TORCH_CHECK(false, "Invalid CPU ISA type."); \ } \ @@ -59,10 +70,12 @@ void swigluoai_and_mul(float* __restrict__ input, scalar_t* __restrict__ output, const int32_t input_stride, const int32_t output_stride) { using scalar_vec_t = typename cpu_utils::VecTypeTrait::vec_t; +#if !defined(__aarch64__) // For GPT-OSS interleaved gate-up weights alignas(64) static int32_t index[16] = {0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30}; vec_op::INT32Vec16 index_vec(index); +#endif vec_op::FP32Vec16 gate_up_max_vec(7.0); vec_op::FP32Vec16 up_min_vec(-7.0); vec_op::FP32Vec16 alpha_vec(1.702); @@ -72,8 +85,15 @@ void swigluoai_and_mul(float* __restrict__ input, scalar_t* __restrict__ output, for (int32_t m = 0; m < m_size; ++m) { for (int32_t n = 0; n < n_size; n += 32) { + // Note: AdvSIMD does not support gather loads +#if defined(__aarch64__) + vec_op::FP32Vec16 gate_vec(vec_op::uninit); + vec_op::FP32Vec16 up_vec(vec_op::uninit); + vec_op::FP32Vec16::load_even_odd(input + n, gate_vec, up_vec); +#else vec_op::FP32Vec16 gate_vec(input + n, index_vec); vec_op::FP32Vec16 up_vec(input + n + 1, index_vec); +#endif gate_vec = gate_vec.min(gate_up_max_vec); up_vec = up_vec.clamp(up_min_vec, gate_up_max_vec); auto sigmoid_vec = one_vec / (one_vec + fast_exp(-gate_vec * alpha_vec)); @@ -174,7 +194,7 @@ void gelu_tanh_and_mul(float* __restrict__ input, scalar_t* __restrict__ output, auto inner_vec = w1_vec * (gate_vec + w3_vec * gate_pow3_vec); // Note: can't use fast_exp form because diffusiongemma will generate // wrong results - vec_op::FP32Vec16 tanh_vec(Sleef_tanhf16_u10(inner_vec.reg)); + auto tanh_vec = inner_vec.tanh(); auto gelu_tanh = gate_vec * w2_vec * (one_vec + tanh_vec); auto gated_output_fp32 = up_vec * gelu_tanh; scalar_vec_t gated_output = scalar_vec_t(gated_output_fp32); @@ -240,6 +260,7 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input, constexpr int32_t gemm_n_tile_size = gemm_t::NSize; constexpr int32_t gemm_m_tile_size = gemm_t::MaxMSize; constexpr int32_t min_w13_n_tile_size = 2 * gemm_n_tile_size; + constexpr bool pack_a = gemm_t::PackA; static_assert(gemm_n_tile_size % 16 == 0); TORCH_CHECK_EQ(output_size_13 % min_w13_n_tile_size, 0); @@ -266,12 +287,18 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input, const int32_t w2_input_tile_size = cpu_utils::round_up<64>( gemm_m_tile_size * input_size_2 * sizeof(scalar_t)); + // use w2 input buffer only when we need to pack input + const int32_t w2_input_buffer_size = + pack_a ? cpu_utils::round_up<64>(gemm_m_tile_size * input_size_2 * + sizeof(scalar_t)) + : 0; const int32_t w2_n_tile_size = [&]() { const int64_t cache_size = cpu_utils::get_available_l2_size(); - // input tile + weight + // input tile + optional packed input + weight const int32_t n_size_cache_limit = - (cache_size - w2_input_tile_size) / (input_size_2 * sizeof(scalar_t)); + (cache_size - (pack_a ? w2_input_buffer_size : w2_input_tile_size)) / + (input_size_2 * sizeof(scalar_t)); const int32_t n_size_thread_limit = output_size_2 / std::max(1, thread_num / topk_num); const int32_t n_size = cpu_utils::round_down( @@ -324,6 +351,9 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input, const int32_t w13_output_buffer_offset = w13_thread_buffer_offset; w13_thread_buffer_offset += w13_output_buffer_size; + const int32_t w2_input_buffer_offset = w13_thread_buffer_offset; + w13_thread_buffer_offset += w2_input_buffer_size; + // Weighted sum thread buffer const int32_t ws_output_buffer_size = cpu_utils::round_up<64>(output_size_2 * sizeof(float)); @@ -403,7 +433,8 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input, gemm_t gemm; const int32_t input_size_13_bytes = input_size_13 * sizeof(scalar_t); - const int32_t w13_n_group_stride = 16 * input_size_13; + const int32_t w13_n_group_stride = + gemm_t::WeightOCGroupSize * input_size_13; const int32_t w13_n_tile_stride = gemm_n_tile_size * input_size_13; for (;;) { @@ -466,8 +497,23 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input, token_idx += gemm_m_tile_size) { const int32_t actual_token_num = std::min(gemm_m_tile_size, curr_token_num - token_idx); - // copy inputs - { + + scalar_t* __restrict__ curr_w13_gemm_input_buffer = nullptr; + if constexpr (pack_a) { + // copy and pack inputs + curr_w13_gemm_input_buffer = w13_input_buffer; + const scalar_t* w13_input_rows[gemm_m_tile_size]; + for (int32_t i = 0; i < actual_token_num; ++i) { + w13_input_rows[i] = + input + curr_expand_token_id_buffer[i] * input_size_13; + } + gemm_t::pack_input_from_rows(w13_input_rows, + curr_w13_gemm_input_buffer, + actual_token_num, input_size_13); + curr_expand_token_id_buffer += actual_token_num; + } else { + // copy inputs + curr_w13_gemm_input_buffer = curr_w13_input_buffer; scalar_t* __restrict__ curr_w13_input_buffer_iter = curr_w13_input_buffer; for (int32_t i = 0; i < actual_token_num; ++i) { @@ -499,14 +545,12 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input, scalar_t* __restrict__ w13_weight_ptr_1_iter = w13_weight_ptr_1; scalar_t* __restrict__ w13_bias_ptr_0_iter = w13_bias_ptr_0; scalar_t* __restrict__ w13_bias_ptr_1_iter = w13_bias_ptr_1; - scalar_t* __restrict__ curr_w13_input_buffer_iter = - curr_w13_input_buffer; float* __restrict__ w13_output_buffer_0_iter = w13_output_buffer; float* __restrict__ w13_output_buffer_1_iter = w13_output_buffer + actual_n_tile_size / 2; for (int32_t i = 0; i < actual_n_tile_size; i += min_w13_n_tile_size) { - gemm.gemm(curr_w13_input_buffer_iter, w13_weight_ptr_0_iter, + gemm.gemm(curr_w13_gemm_input_buffer, w13_weight_ptr_0_iter, w13_output_buffer_0_iter, actual_token_num, input_size_13, input_size_13, w13_n_group_stride, actual_n_tile_size, false); @@ -519,7 +563,7 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input, w13_bias_ptr_0_iter += gemm_n_tile_size; } - gemm.gemm(curr_w13_input_buffer_iter, w13_weight_ptr_1_iter, + gemm.gemm(curr_w13_gemm_input_buffer, w13_weight_ptr_1_iter, w13_output_buffer_1_iter, actual_token_num, input_size_13, input_size_13, w13_n_group_stride, actual_n_tile_size, false); @@ -572,7 +616,8 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input, gemm_t gemm; const int32_t w2_n_tile_stride = gemm_n_tile_size * input_size_2; - const int32_t w2_n_group_stride = 16 * input_size_2; + const int32_t w2_n_group_stride = + gemm_t::WeightOCGroupSize * input_size_2; for (;;) { int32_t task_id = counter_ptr->acquire_counter(); @@ -611,13 +656,30 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input, token_idx += gemm_m_tile_size) { const int32_t actual_token_num = std::min(gemm_m_tile_size, curr_token_num - token_idx); + scalar_t* __restrict__ curr_w2_gemm_input_buffer = + curr_w13_gemm_output_buffer; + if constexpr (pack_a) { + uint8_t* __restrict__ thread_buffer = + thread_buffer_start + thread_id * w13_thread_buffer_offset; + scalar_t* __restrict__ w2_input_buffer = + reinterpret_cast(thread_buffer + + w2_input_buffer_offset); + curr_w2_gemm_input_buffer = w2_input_buffer; + const scalar_t* w2_input_rows[gemm_m_tile_size]; + for (int32_t i = 0; i < actual_token_num; ++i) { + w2_input_rows[i] = curr_w13_gemm_output_buffer + i * input_size_2; + } + gemm_t::pack_input_from_rows(w2_input_rows, + curr_w2_gemm_input_buffer, + actual_token_num, input_size_2); + } scalar_t* __restrict__ w2_weight_ptr_iter = w2_weight_ptr; scalar_t* __restrict__ w2_bias_ptr_iter = w2_bias_ptr; float* __restrict__ curr_w2_gemm_output_buffer_iter = curr_w2_gemm_output_buffer; for (int32_t i = 0; i < actual_n_tile_size; i += gemm_n_tile_size) { - gemm.gemm(curr_w13_gemm_output_buffer, w2_weight_ptr_iter, + gemm.gemm(curr_w2_gemm_input_buffer, w2_weight_ptr_iter, curr_w2_gemm_output_buffer_iter, actual_token_num, input_size_2, input_size_2, w2_n_group_stride, output_size_2, false); diff --git a/csrc/cpu/cpu_types_arm.hpp b/csrc/cpu/cpu_types_arm.hpp index b408731f40..fc987f706a 100644 --- a/csrc/cpu/cpu_types_arm.hpp +++ b/csrc/cpu/cpu_types_arm.hpp @@ -497,6 +497,26 @@ struct FP32Vec16 : public VectorizedRegWrapper { reg.val[3] = Vectorized(vcvt_f32_f16(vget_high_f16(v.reg.val[1]))); }; + static FORCE_INLINE void load_even_odd(const float* ptr, FP32Vec16& even, + FP32Vec16& odd) noexcept { + const float32x4x2_t x01 = vuzpq_f32(vld1q_f32(ptr), vld1q_f32(ptr + 4)); + const float32x4x2_t x23 = + vuzpq_f32(vld1q_f32(ptr + 8), vld1q_f32(ptr + 12)); + const float32x4x2_t x45 = + vuzpq_f32(vld1q_f32(ptr + 16), vld1q_f32(ptr + 20)); + const float32x4x2_t x67 = + vuzpq_f32(vld1q_f32(ptr + 24), vld1q_f32(ptr + 28)); + + even.reg.val[0] = VectorizedT(x01.val[0]); + even.reg.val[1] = VectorizedT(x23.val[0]); + even.reg.val[2] = VectorizedT(x45.val[0]); + even.reg.val[3] = VectorizedT(x67.val[0]); + odd.reg.val[0] = VectorizedT(x01.val[1]); + odd.reg.val[1] = VectorizedT(x23.val[1]); + odd.reg.val[2] = VectorizedT(x45.val[1]); + odd.reg.val[3] = VectorizedT(x67.val[1]); + } + FORCE_INLINE FP32Vec16 operator+(const FP32Vec16& b) const noexcept { FP32Vec16 r(uninit); r.reg.val[0] = reg.val[0] + b.reg.val[0]; @@ -515,6 +535,15 @@ struct FP32Vec16 : public VectorizedRegWrapper { return r; } + FORCE_INLINE FP32Vec16 operator-() const noexcept { + FP32Vec16 r(uninit); + r.reg.val[0] = reg.val[0].neg(); + r.reg.val[1] = reg.val[1].neg(); + r.reg.val[2] = reg.val[2].neg(); + r.reg.val[3] = reg.val[3].neg(); + return r; + } + FORCE_INLINE FP32Vec16 operator*(const FP32Vec16& b) const noexcept { FP32Vec16 r(uninit); r.reg.val[0] = reg.val[0] * b.reg.val[0]; @@ -933,4 +962,4 @@ inline void storeFP32(float v, c10::BFloat16* ptr) { inline void prefetch(const void* addr) { __builtin_prefetch(addr, 0, 1); }; -}; // namespace vec_op \ No newline at end of file +}; // namespace vec_op diff --git a/csrc/cpu/cpu_types_scalar.hpp b/csrc/cpu/cpu_types_scalar.hpp index d1c2fc8593..94b5179b17 100644 --- a/csrc/cpu/cpu_types_scalar.hpp +++ b/csrc/cpu/cpu_types_scalar.hpp @@ -363,6 +363,13 @@ struct FP32Vec16 : public Vec { return FP32Vec16(ret); } + FP32Vec16 tanh() const { + f32x16_t ret; + unroll_loop( + [&ret, this](int i) { ret.val[i] = std::tanh(reg.val[i]); }); + return FP32Vec16(ret); + } + float reduce_sum() const { float result = 0.0f; unroll_loop( diff --git a/csrc/cpu/cpu_types_x86.hpp b/csrc/cpu/cpu_types_x86.hpp index 396b9b7e04..d2a72ce9cc 100644 --- a/csrc/cpu/cpu_types_x86.hpp +++ b/csrc/cpu/cpu_types_x86.hpp @@ -3,6 +3,7 @@ #define CPU_TYPES_X86_HPP #include +#include #include #ifndef __AVX2__ @@ -592,6 +593,8 @@ struct FP32Vec16 : public Vec { FP32Vec16 abs() const { return FP32Vec16(_mm512_abs_ps(reg)); } + FP32Vec16 tanh() const { return FP32Vec16(Sleef_tanhf16_u10(reg)); } + float reduce_sum() const { return _mm512_reduce_add_ps(reg); } float reduce_max() const { return _mm512_reduce_max_ps(reg); } @@ -789,6 +792,12 @@ struct FP32Vec16 : public Vec { _mm256_andnot_ps(sign_mask, reg_high)); } + FP32Vec16 tanh() const { + FP32Vec8 low(reg_low); + FP32Vec8 high(reg_high); + return FP32Vec16(low.tanh().reg, high.tanh().reg); + } + FP32Vec16 min(const FP32Vec16& b) const { return FP32Vec16(_mm256_min_ps(reg_low, b.reg_low), _mm256_min_ps(reg_high, b.reg_high)); diff --git a/csrc/cpu/micro_gemm/cpu_micro_gemm_amx.hpp b/csrc/cpu/micro_gemm/cpu_micro_gemm_amx.hpp index 357c7cf1d7..99e7c4a1d5 100644 --- a/csrc/cpu/micro_gemm/cpu_micro_gemm_amx.hpp +++ b/csrc/cpu/micro_gemm/cpu_micro_gemm_amx.hpp @@ -213,6 +213,8 @@ class MicroGemm { public: static constexpr int32_t MaxMSize = 32; static constexpr int32_t NSize = 32; + static constexpr int32_t WeightOCGroupSize = 16; + static constexpr bool PackA = false; public: MicroGemm() : curr_m_(-1) { diff --git a/csrc/cpu/micro_gemm/cpu_micro_gemm_impl.hpp b/csrc/cpu/micro_gemm/cpu_micro_gemm_impl.hpp index 23e78a681b..f0471f7147 100644 --- a/csrc/cpu/micro_gemm/cpu_micro_gemm_impl.hpp +++ b/csrc/cpu/micro_gemm/cpu_micro_gemm_impl.hpp @@ -21,6 +21,9 @@ class MicroGemm { public: static constexpr int32_t MaxMSize = 16; static constexpr int32_t NSize = 16; + static constexpr int32_t WeightOCGroupSize = 16; + // callers must pack A matrix before GEMM + static constexpr bool PackA = false; public: void gemm(DEFINE_CPU_MICRO_GEMM_PARAMS) { diff --git a/csrc/cpu/micro_gemm/cpu_micro_gemm_neon.hpp b/csrc/cpu/micro_gemm/cpu_micro_gemm_neon.hpp new file mode 100644 index 0000000000..7d4898852b --- /dev/null +++ b/csrc/cpu/micro_gemm/cpu_micro_gemm_neon.hpp @@ -0,0 +1,503 @@ +#ifndef CPU_MICRO_GEMM_NEON_HPP +#define CPU_MICRO_GEMM_NEON_HPP + +#include +#include + +#include "cpu/micro_gemm/cpu_micro_gemm_impl.hpp" + +#include +#include + +namespace cpu_micro_gemm { + +namespace { + +constexpr int32_t K = 4; +constexpr int32_t Cols = 2; +constexpr int32_t TileSize = K * Cols; +constexpr int32_t Mr = 8; +constexpr int32_t Nr = 8; +constexpr int32_t Nr_gemv = 16; + +// a = [a0, a1, a2, a3], b = [b0, b1, b2, b3] -> [a0, a1, b0, b1] +FORCE_INLINE float32x4_t zip1_f32x4(const float32x4_t a, const float32x4_t b) { + return vreinterpretq_f32_f64( + vzip1q_f64(vreinterpretq_f64_f32(a), vreinterpretq_f64_f32(b))); +} + +// a = [a0, a1, a2, a3], b = [b0, b1, b2, b3] -> [a2, a3, b2, b3] +FORCE_INLINE float32x4_t zip2_f32x4(const float32x4_t a, const float32x4_t b) { + return vreinterpretq_f32_f64( + vzip2q_f64(vreinterpretq_f64_f32(a), vreinterpretq_f64_f32(b))); +} + +FORCE_INLINE void init_acc_rowpair(float32x4_t& acc01, float32x4_t& acc23, + float32x4_t& acc45, float32x4_t& acc67, + const float* __restrict__ c_ptr, + const int64_t ldc, const int32_t m_rows, + const bool accum_c) { + if (!accum_c || m_rows == 0) { + acc01 = vdupq_n_f32(0.0f); + acc23 = vdupq_n_f32(0.0f); + acc45 = vdupq_n_f32(0.0f); + acc67 = vdupq_n_f32(0.0f); + return; + } + + const float32x4_t row0_0123 = vld1q_f32(c_ptr); + const float32x4_t row0_4567 = vld1q_f32(c_ptr + 4); + const float32x4_t row1_0123 = + (m_rows == 2) ? vld1q_f32(c_ptr + ldc) : vdupq_n_f32(0.0f); + const float32x4_t row1_4567 = + (m_rows == 2) ? vld1q_f32(c_ptr + ldc + 4) : vdupq_n_f32(0.0f); + + acc01 = zip1_f32x4(row0_0123, row1_0123); + acc23 = zip2_f32x4(row0_0123, row1_0123); + acc45 = zip1_f32x4(row0_4567, row1_4567); + acc67 = zip2_f32x4(row0_4567, row1_4567); +} + +FORCE_INLINE void store_acc_rowpair(const float32x4_t acc01, + const float32x4_t acc23, + const float32x4_t acc45, + const float32x4_t acc67, + float* __restrict__ c_ptr, + const int64_t ldc, const int32_t m_rows) { + if (m_rows == 0) { + return; + } + + vst1q_f32(c_ptr, zip1_f32x4(acc01, acc23)); + vst1q_f32(c_ptr + 4, zip1_f32x4(acc45, acc67)); + + if (m_rows == 2) { + vst1q_f32(c_ptr + ldc, zip2_f32x4(acc01, acc23)); + vst1q_f32(c_ptr + ldc + 4, zip2_f32x4(acc45, acc67)); + } +} + +FORCE_INLINE void gemm_micro_bfmmla_8x8_packed_a( + const bfloat16_t* __restrict__ a_packed, + const bfloat16_t* __restrict__ b_packed, float* __restrict__ c_ptr, + const int32_t m, const int32_t k_size, const int64_t ldc, + const bool accum_c) { + float32x4_t acc0101, acc0123, acc0145, acc0167; + float32x4_t acc2301, acc2323, acc2345, acc2367; + float32x4_t acc4501, acc4523, acc4545, acc4567; + float32x4_t acc6701, acc6723, acc6745, acc6767; + + init_acc_rowpair(acc0101, acc0123, acc0145, acc0167, c_ptr, ldc, + std::min(2, m), accum_c); + init_acc_rowpair(acc2301, acc2323, acc2345, acc2367, c_ptr + 2 * ldc, ldc, + std::min(2, std::max(0, m - 2)), accum_c); + init_acc_rowpair(acc4501, acc4523, acc4545, acc4567, c_ptr + 4 * ldc, ldc, + std::min(2, std::max(0, m - 4)), accum_c); + init_acc_rowpair(acc6701, acc6723, acc6745, acc6767, c_ptr + 6 * ldc, ldc, + std::min(2, std::max(0, m - 6)), accum_c); + + const bfloat16_t* __restrict__ a_tile = a_packed; + const bfloat16_t* __restrict__ b_tile = b_packed; + +#pragma GCC unroll 8 + for (int32_t k_idx = 0; k_idx < k_size; k_idx += K) { + const bfloat16x8_t a_tile01 = vld1q_bf16(a_tile); + const bfloat16x8_t a_tile23 = vld1q_bf16(a_tile + TileSize); + const bfloat16x8_t a_tile45 = vld1q_bf16(a_tile + 2 * TileSize); + const bfloat16x8_t a_tile67 = vld1q_bf16(a_tile + 3 * TileSize); + + const bfloat16x8_t b_tile01 = vld1q_bf16(b_tile); + const bfloat16x8_t b_tile23 = vld1q_bf16(b_tile + TileSize); + const bfloat16x8_t b_tile45 = vld1q_bf16(b_tile + 2 * TileSize); + const bfloat16x8_t b_tile67 = vld1q_bf16(b_tile + 3 * TileSize); + + acc0101 = vbfmmlaq_f32(acc0101, a_tile01, b_tile01); + acc2301 = vbfmmlaq_f32(acc2301, a_tile23, b_tile01); + acc4501 = vbfmmlaq_f32(acc4501, a_tile45, b_tile01); + acc6701 = vbfmmlaq_f32(acc6701, a_tile67, b_tile01); + + acc0123 = vbfmmlaq_f32(acc0123, a_tile01, b_tile23); + acc2323 = vbfmmlaq_f32(acc2323, a_tile23, b_tile23); + acc4523 = vbfmmlaq_f32(acc4523, a_tile45, b_tile23); + acc6723 = vbfmmlaq_f32(acc6723, a_tile67, b_tile23); + + acc0145 = vbfmmlaq_f32(acc0145, a_tile01, b_tile45); + acc2345 = vbfmmlaq_f32(acc2345, a_tile23, b_tile45); + acc4545 = vbfmmlaq_f32(acc4545, a_tile45, b_tile45); + acc6745 = vbfmmlaq_f32(acc6745, a_tile67, b_tile45); + + acc0167 = vbfmmlaq_f32(acc0167, a_tile01, b_tile67); + acc2367 = vbfmmlaq_f32(acc2367, a_tile23, b_tile67); + acc4567 = vbfmmlaq_f32(acc4567, a_tile45, b_tile67); + acc6767 = vbfmmlaq_f32(acc6767, a_tile67, b_tile67); + + a_tile += 4 * TileSize; + b_tile += Nr * K; + } + + store_acc_rowpair(acc0101, acc0123, acc0145, acc0167, c_ptr, ldc, + std::min(2, m)); + store_acc_rowpair(acc2301, acc2323, acc2345, acc2367, c_ptr + 2 * ldc, ldc, + std::min(2, std::max(0, m - 2))); + store_acc_rowpair(acc4501, acc4523, acc4545, acc4567, c_ptr + 4 * ldc, ldc, + std::min(2, std::max(0, m - 4))); + store_acc_rowpair(acc6701, acc6723, acc6745, acc6767, c_ptr + 6 * ldc, ldc, + std::min(2, std::max(0, m - 6))); +} + +FORCE_INLINE void gemm_micro_bfmmla_4x16_packed_a( + const bfloat16_t* __restrict__ a_packed, + const bfloat16_t* __restrict__ b_packed, float* __restrict__ c_ptr, + const int32_t m, const int32_t k_size, const int64_t b_n_group_stride, + const int64_t ldc, const bool accum_c) { + const int32_t m_rows_01 = std::min(2, m); + const int32_t m_rows_23 = std::min(2, std::max(0, m - 2)); + + float32x4_t acc0101, acc0123, acc0145, acc0167; + float32x4_t acc2301, acc2323, acc2345, acc2367; + float32x4_t acc0189, acc011011, acc011213, acc011415; + float32x4_t acc2389, acc231011, acc231213, acc231415; + + init_acc_rowpair(acc0101, acc0123, acc0145, acc0167, c_ptr, ldc, m_rows_01, + accum_c); + init_acc_rowpair(acc2301, acc2323, acc2345, acc2367, c_ptr + 2 * ldc, ldc, + m_rows_23, accum_c); + init_acc_rowpair(acc0189, acc011011, acc011213, acc011415, c_ptr + 8, ldc, + m_rows_01, accum_c); + init_acc_rowpair(acc2389, acc231011, acc231213, acc231415, + c_ptr + 2 * ldc + 8, ldc, m_rows_23, accum_c); + + const bfloat16_t* __restrict__ a_tile = a_packed; + const bfloat16_t* __restrict__ b_tile0 = b_packed; + const bfloat16_t* __restrict__ b_tile1 = b_packed + b_n_group_stride; + +#pragma GCC unroll 8 + for (int32_t k_idx = 0; k_idx < k_size; k_idx += K) { + const bfloat16x8_t a_tile01 = vld1q_bf16(a_tile); + const bfloat16x8_t a_tile23 = vld1q_bf16(a_tile + TileSize); + const bfloat16x8_t b_tile01 = vld1q_bf16(b_tile0); + const bfloat16x8_t b_tile23 = vld1q_bf16(b_tile0 + TileSize); + const bfloat16x8_t b_tile45 = vld1q_bf16(b_tile0 + 2 * TileSize); + const bfloat16x8_t b_tile67 = vld1q_bf16(b_tile0 + 3 * TileSize); + const bfloat16x8_t b_tile89 = vld1q_bf16(b_tile1); + const bfloat16x8_t b_tile1011 = vld1q_bf16(b_tile1 + TileSize); + const bfloat16x8_t b_tile1213 = vld1q_bf16(b_tile1 + 2 * TileSize); + const bfloat16x8_t b_tile1415 = vld1q_bf16(b_tile1 + 3 * TileSize); + + acc0101 = vbfmmlaq_f32(acc0101, a_tile01, b_tile01); + acc2301 = vbfmmlaq_f32(acc2301, a_tile23, b_tile01); + acc0123 = vbfmmlaq_f32(acc0123, a_tile01, b_tile23); + acc2323 = vbfmmlaq_f32(acc2323, a_tile23, b_tile23); + + acc0145 = vbfmmlaq_f32(acc0145, a_tile01, b_tile45); + acc2345 = vbfmmlaq_f32(acc2345, a_tile23, b_tile45); + acc0167 = vbfmmlaq_f32(acc0167, a_tile01, b_tile67); + acc2367 = vbfmmlaq_f32(acc2367, a_tile23, b_tile67); + + acc0189 = vbfmmlaq_f32(acc0189, a_tile01, b_tile89); + acc2389 = vbfmmlaq_f32(acc2389, a_tile23, b_tile89); + acc011011 = vbfmmlaq_f32(acc011011, a_tile01, b_tile1011); + acc231011 = vbfmmlaq_f32(acc231011, a_tile23, b_tile1011); + + acc011213 = vbfmmlaq_f32(acc011213, a_tile01, b_tile1213); + acc231213 = vbfmmlaq_f32(acc231213, a_tile23, b_tile1213); + acc011415 = vbfmmlaq_f32(acc011415, a_tile01, b_tile1415); + acc231415 = vbfmmlaq_f32(acc231415, a_tile23, b_tile1415); + + a_tile += 2 * TileSize; + b_tile0 += Nr * K; + b_tile1 += Nr * K; + } + + store_acc_rowpair(acc0101, acc0123, acc0145, acc0167, c_ptr, ldc, m_rows_01); + store_acc_rowpair(acc2301, acc2323, acc2345, acc2367, c_ptr + 2 * ldc, ldc, + m_rows_23); + store_acc_rowpair(acc0189, acc011011, acc011213, acc011415, c_ptr + 8, ldc, + m_rows_01); + store_acc_rowpair(acc2389, acc231011, acc231213, acc231415, + c_ptr + 2 * ldc + 8, ldc, m_rows_23); +} + +} // namespace + +template +class MicroGemm { + public: + static constexpr int32_t MaxMSize = 8; + static constexpr int32_t NSize = 32; + static constexpr int32_t WeightOCGroupSize = Nr; + static constexpr bool PackA = false; + + public: + void gemm(DEFINE_CPU_MICRO_GEMM_PARAMS) { + TORCH_CHECK(false, "NEON BFMMLA MicroGemm only supports bfloat16."); + } + + static void pack_weight(const scalar_t* __restrict__ /*weight*/, + scalar_t* __restrict__ /*packed_weight*/, + const int32_t /*output_size*/, + const int32_t /*input_size*/) { + TORCH_CHECK(false, "NEON BFMMLA MicroGemm only supports bfloat16."); + } +}; + +template <> +class MicroGemm { + public: + using scalar_t = c10::BFloat16; + + static constexpr int32_t MaxMSize = 8; + static constexpr int32_t NSize = 32; + static constexpr int32_t WeightOCGroupSize = Nr; + static constexpr bool PackA = true; + + public: + // physical layout [ + // M / 8; Mr is 8 + // K / 4; K for bfmmla is 4 + // 4, ; 4 row-pairs for each 8 rows + // 2, ; row-pair is 2 rows + // 4 ; 4 elements per row + // ] + + static void pack_input_from_rows(const scalar_t* const* __restrict__ rows, + scalar_t* __restrict__ a_packed, + const int32_t m, const int32_t k) { + TORCH_CHECK(m > 0 && m <= MaxMSize); + TORCH_CHECK_EQ(k % K, 0); + + auto* __restrict__ out = reinterpret_cast(a_packed); + const bfloat16x8_t zero_q = vdupq_n_bf16(bfloat16_t{}); + const bfloat16x4_t zero = vget_low_bf16(zero_q); + + for (int32_t row_base = 0; row_base < m; row_base += Mr) { + const int32_t actual_m = std::min(Mr, m - row_base); + const bfloat16_t* __restrict__ row[Mr]; + for (int32_t i = 0; i < actual_m; ++i) { + row[i] = reinterpret_cast(rows[row_base + i]); + } + + if (actual_m == 8) { + int32_t k_idx = 0; + for (; k_idx + 8 <= k; k_idx += 8) { + bfloat16_t* __restrict__ block0 = out; + bfloat16_t* __restrict__ block1 = out + 4 * TileSize; + + bfloat16x8_t a0 = vld1q_bf16(row[0] + k_idx); + bfloat16x8_t a1 = vld1q_bf16(row[1] + k_idx); + vst1q_bf16(block0, + vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1))); + vst1q_bf16(block1, + vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1))); + + a0 = vld1q_bf16(row[2] + k_idx); + a1 = vld1q_bf16(row[3] + k_idx); + vst1q_bf16(block0 + TileSize, + vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1))); + vst1q_bf16(block1 + TileSize, + vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1))); + + a0 = vld1q_bf16(row[4] + k_idx); + a1 = vld1q_bf16(row[5] + k_idx); + vst1q_bf16(block0 + 2 * TileSize, + vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1))); + vst1q_bf16(block1 + 2 * TileSize, + vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1))); + + a0 = vld1q_bf16(row[6] + k_idx); + a1 = vld1q_bf16(row[7] + k_idx); + vst1q_bf16(block0 + 3 * TileSize, + vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1))); + vst1q_bf16(block1 + 3 * TileSize, + vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1))); + + out += 8 * TileSize; + } + + for (; k_idx < k; k_idx += K) { + bfloat16x4_t a0 = vld1_bf16(row[0] + k_idx); + bfloat16x4_t a1 = vld1_bf16(row[1] + k_idx); + vst1q_bf16(out, vcombine_bf16(a0, a1)); + + a0 = vld1_bf16(row[2] + k_idx); + a1 = vld1_bf16(row[3] + k_idx); + vst1q_bf16(out + TileSize, vcombine_bf16(a0, a1)); + + a0 = vld1_bf16(row[4] + k_idx); + a1 = vld1_bf16(row[5] + k_idx); + vst1q_bf16(out + 2 * TileSize, vcombine_bf16(a0, a1)); + + a0 = vld1_bf16(row[6] + k_idx); + a1 = vld1_bf16(row[7] + k_idx); + vst1q_bf16(out + 3 * TileSize, vcombine_bf16(a0, a1)); + + out += 4 * TileSize; + } + continue; + } + + if (actual_m == 4) { + int32_t k_idx = 0; + for (; k_idx + 8 <= k; k_idx += 8) { + bfloat16_t* __restrict__ block0 = out; + bfloat16_t* __restrict__ block1 = out + 2 * TileSize; + + bfloat16x8_t a0 = vld1q_bf16(row[0] + k_idx); + bfloat16x8_t a1 = vld1q_bf16(row[1] + k_idx); + vst1q_bf16(block0, + vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1))); + vst1q_bf16(block1, + vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1))); + + a0 = vld1q_bf16(row[2] + k_idx); + a1 = vld1q_bf16(row[3] + k_idx); + vst1q_bf16(block0 + TileSize, + vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1))); + vst1q_bf16(block1 + TileSize, + vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1))); + + out += 4 * TileSize; + } + + for (; k_idx < k; k_idx += K) { + bfloat16x4_t a0 = vld1_bf16(row[0] + k_idx); + bfloat16x4_t a1 = vld1_bf16(row[1] + k_idx); + vst1q_bf16(out, vcombine_bf16(a0, a1)); + + a0 = vld1_bf16(row[2] + k_idx); + a1 = vld1_bf16(row[3] + k_idx); + vst1q_bf16(out + TileSize, vcombine_bf16(a0, a1)); + + out += 2 * TileSize; + } + continue; + } + + const int32_t row_pair_count = (actual_m <= 4) ? 2 : Mr / 2; + + int32_t k_idx = 0; + for (; k_idx + 8 <= k; k_idx += 8) { + bfloat16_t* __restrict__ block0 = out; + bfloat16_t* __restrict__ block1 = out + row_pair_count * TileSize; + + bfloat16x8_t a0 = vld1q_bf16(row[0] + k_idx); + bfloat16x8_t a1 = (actual_m > 1) ? vld1q_bf16(row[1] + k_idx) : zero_q; + vst1q_bf16(block0, vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1))); + vst1q_bf16(block1, + vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1))); + + a0 = (actual_m > 2) ? vld1q_bf16(row[2] + k_idx) : zero_q; + a1 = (actual_m > 3) ? vld1q_bf16(row[3] + k_idx) : zero_q; + vst1q_bf16(block0 + TileSize, + vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1))); + vst1q_bf16(block1 + TileSize, + vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1))); + + if (actual_m > 4) { + a0 = vld1q_bf16(row[4] + k_idx); + a1 = (actual_m > 5) ? vld1q_bf16(row[5] + k_idx) : zero_q; + vst1q_bf16(block0 + 2 * TileSize, + vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1))); + vst1q_bf16(block1 + 2 * TileSize, + vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1))); + + a0 = (actual_m > 6) ? vld1q_bf16(row[6] + k_idx) : zero_q; + a1 = (actual_m > 7) ? vld1q_bf16(row[7] + k_idx) : zero_q; + vst1q_bf16(block0 + 3 * TileSize, + vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1))); + vst1q_bf16(block1 + 3 * TileSize, + vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1))); + } + + out += 2 * row_pair_count * TileSize; + } + + for (; k_idx < k; k_idx += K) { + bfloat16x4_t a0 = vld1_bf16(row[0] + k_idx); + bfloat16x4_t a1 = (actual_m > 1) ? vld1_bf16(row[1] + k_idx) : zero; + vst1q_bf16(out, vcombine_bf16(a0, a1)); + + a0 = (actual_m > 2) ? vld1_bf16(row[2] + k_idx) : zero; + a1 = (actual_m > 3) ? vld1_bf16(row[3] + k_idx) : zero; + vst1q_bf16(out + TileSize, vcombine_bf16(a0, a1)); + + if (actual_m > 4) { + a0 = vld1_bf16(row[4] + k_idx); + a1 = (actual_m > 5) ? vld1_bf16(row[5] + k_idx) : zero; + vst1q_bf16(out + 2 * TileSize, vcombine_bf16(a0, a1)); + + a0 = (actual_m > 6) ? vld1_bf16(row[6] + k_idx) : zero; + a1 = (actual_m > 7) ? vld1_bf16(row[7] + k_idx) : zero; + vst1q_bf16(out + 3 * TileSize, vcombine_bf16(a0, a1)); + } + out += row_pair_count * TileSize; + } + } + } + + void gemm(DEFINE_CPU_MICRO_GEMM_PARAMS) { + (void)lda; // A is packed, so lda is not needed + TORCH_CHECK_EQ(k % K, 0); + + for (int32_t n_idx = 0; n_idx < NSize; n_idx += Nr_gemv) { + const bfloat16_t* __restrict__ b_panel = + reinterpret_cast(b_ptr) + n_idx * k; + + for (int32_t row_base = 0; row_base < m; row_base += Mr) { + const int32_t panel_m = std::min(Mr, m - row_base); + const bfloat16_t* __restrict__ a_panel = + reinterpret_cast(a_ptr) + row_base * k; + float* __restrict__ c_panel = c_ptr + row_base * ldc + n_idx; + + if (panel_m <= 4) { + gemm_micro_bfmmla_4x16_packed_a(a_panel, b_panel, c_panel, panel_m, k, + b_n_group_stride, ldc, accum_c); + } else { + gemm_micro_bfmmla_8x8_packed_a(a_panel, b_panel, c_panel, panel_m, k, + ldc, accum_c); + gemm_micro_bfmmla_8x8_packed_a(a_panel, b_panel + b_n_group_stride, + c_panel + Nr, panel_m, k, ldc, + accum_c); + } + } + } + } + + // physical layout [ + // N / 8; Nr is 8 + // K / 4; K for bfmmla is 4 + // 4, ; 4 col-pairs for each 8 cols + // 2, ; col-pair is 2 cols + // 4 ; 4 elements per col + // ] + static void pack_weight(const c10::BFloat16* __restrict__ weight, + c10::BFloat16* __restrict__ packed_weight, + const int32_t output_size, const int32_t input_size) { + TORCH_CHECK_EQ(output_size % NSize, 0); + TORCH_CHECK_EQ(input_size % K, 0); + + for (int32_t o_idx = 0; o_idx < output_size; o_idx += Nr) { + c10::BFloat16* __restrict__ dst = packed_weight + o_idx * input_size; + for (int32_t k_idx = 0; k_idx < input_size; k_idx += K) { + for (int32_t pair_idx = 0; pair_idx < Nr; pair_idx += Cols) { + const c10::BFloat16* __restrict__ row0 = + weight + (o_idx + pair_idx) * input_size; + const c10::BFloat16* __restrict__ row1 = row0 + input_size; + dst[0] = row0[k_idx + 0]; + dst[1] = row0[k_idx + 1]; + dst[2] = row0[k_idx + 2]; + dst[3] = row0[k_idx + 3]; + dst[4] = row1[k_idx + 0]; + dst[5] = row1[k_idx + 1]; + dst[6] = row1[k_idx + 2]; + dst[7] = row1[k_idx + 3]; + dst += TileSize; + } + } + } + } +}; + +} // namespace cpu_micro_gemm + +#endif diff --git a/csrc/cpu/micro_gemm/cpu_micro_gemm_vec.hpp b/csrc/cpu/micro_gemm/cpu_micro_gemm_vec.hpp index 1c605a2851..ad7d4be113 100644 --- a/csrc/cpu/micro_gemm/cpu_micro_gemm_vec.hpp +++ b/csrc/cpu/micro_gemm/cpu_micro_gemm_vec.hpp @@ -104,6 +104,8 @@ class MicroGemm { public: static constexpr int32_t MaxMSize = 8; static constexpr int32_t NSize = 32; + static constexpr int32_t WeightOCGroupSize = 16; + static constexpr bool PackA = false; public: void gemm(DEFINE_CPU_MICRO_GEMM_PARAMS) { diff --git a/csrc/cpu/torch_bindings.cpp b/csrc/cpu/torch_bindings.cpp index 0204f266b8..9cef2d0d53 100644 --- a/csrc/cpu/torch_bindings.cpp +++ b/csrc/cpu/torch_bindings.cpp @@ -538,7 +538,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) { #endif // fused moe -#if defined(__AVX512F__) +#if defined(__AVX512F__) || (defined(ARM_BF16_SUPPORT)) ops.def( "prepack_moe_weight(Tensor weight, Tensor(a1!) packed_weight, str isa) " "-> ()"); @@ -549,7 +549,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) { "bool skip_weighted, " "str act, str isa) -> ()"); ops.impl("cpu_fused_moe", torch::kCPU, &cpu_fused_moe); -#endif +#endif // #if defined(__AVX512F__) || (defined(ARM_BF16_SUPPORT)) ops.def( "mla_decode_kvcache(" " Tensor! out, Tensor query, Tensor kv_cache," diff --git a/csrc/cpu/utils.hpp b/csrc/cpu/utils.hpp index dedf520134..ec10a0f352 100644 --- a/csrc/cpu/utils.hpp +++ b/csrc/cpu/utils.hpp @@ -2,13 +2,14 @@ #define UTILS_HPP #include +#include #include #include #include "cpu/cpu_types.hpp" namespace cpu_utils { -enum class ISA { AMX, VEC, RVV }; +enum class ISA { AMX, VEC, RVV, NEON }; inline ISA get_isa(const std::string& isa) { if (isa == "amx") { @@ -17,6 +18,8 @@ inline ISA get_isa(const std::string& isa) { return ISA::VEC; } else if (isa == "rvv") { return ISA::RVV; + } else if (isa == "neon") { + return ISA::NEON; } else { TORCH_CHECK(false, "Invalid isa type: " + isa); } diff --git a/tests/kernels/moe/test_cpu_fused_moe.py b/tests/kernels/moe/test_cpu_fused_moe.py index ca25b8c2e9..41ae9be517 100644 --- a/tests/kernels/moe/test_cpu_fused_moe.py +++ b/tests/kernels/moe/test_cpu_fused_moe.py @@ -5,10 +5,13 @@ import pytest import torch from tests.kernels.allclose_default import get_default_atol, get_default_rtol -from vllm._custom_ops import cpu_fused_moe, cpu_prepack_moe_weight +from vllm._custom_ops import ( + cpu_fused_moe, + cpu_prepack_moe_weight, +) from vllm.model_executor.layers.fused_moe.activation import MoEActivation from vllm.model_executor.layers.fused_moe.cpu_fused_moe import _CPU_MOE_ACT_FN -from vllm.platforms import current_platform +from vllm.platforms import CpuArchEnum, current_platform from vllm.utils.torch_utils import set_random_seed if not current_platform.is_cpu(): @@ -26,8 +29,13 @@ ACT = [ MoEActivation.GELU, MoEActivation.GELU_TANH, ] -USE_BIAS = [True, False] -ISA = ["amx", "vec"] if torch.cpu._is_amx_tile_supported() else ["vec"] +USE_BIAS = [False, True] +ISA = ["vec"] +if current_platform.get_cpu_architecture() == CpuArchEnum.ARM: + ISA.append("neon") +if torch.cpu._is_amx_tile_supported(): + ISA.append("amx") + DTYPE = [torch.bfloat16] diff --git a/vllm/model_executor/layers/fused_moe/cpu_fused_moe.py b/vllm/model_executor/layers/fused_moe/cpu_fused_moe.py index d49270122a..868d26e749 100644 --- a/vllm/model_executor/layers/fused_moe/cpu_fused_moe.py +++ b/vllm/model_executor/layers/fused_moe/cpu_fused_moe.py @@ -16,6 +16,7 @@ from vllm._custom_ops import ( from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.fused_moe.activation import MoEActivation from vllm.model_executor.layers.quantization.utils.layer_utils import replace_parameter +from vllm.platforms import CpuArchEnum, current_platform from vllm.utils.torch_utils import direct_register_custom_op _CPU_MOE_LAYER_CACHE = {} @@ -312,6 +313,17 @@ class CPUFusedMOE: if supports_amx: return False, "none" + supports_neon = current_platform.get_cpu_architecture() == CpuArchEnum.ARM + if supports_neon: + if ( + dtype == torch.bfloat16 + and w13_input_size % 4 == 0 + and w2_input_size % 4 == 0 + ): + return True, "neon" + else: + return False, "none" + return True, "vec" def init_moe_grouped_gemm(