[CPU][Perf] Accelerate unquantized MoE for AArch64 (#46353)

Signed-off-by: Fadi Arafeh <fadi.arafeh@arm.com>
This commit is contained in:
Fadi Arafeh
2026-06-24 14:14:35 +00:00
committed by GitHub
parent 93ec645878
commit 061043eaca
15 changed files with 672 additions and 32 deletions
@@ -8,7 +8,7 @@ set -ex
CORE_RANGE=${CORE_RANGE:-0-31}
OMP_CORE_RANGE=${OMP_CORE_RANGE:-0-31}
export CMAKE_BUILD_PARALLEL_LEVEL=16
export CMAKE_BUILD_PARALLEL_LEVEL=32
# Setup cleanup
remove_docker_container() {
@@ -37,7 +37,7 @@ function cpu_tests() {
pytest -x -v -s tests/kernels/test_onednn.py
pytest -x -v -s tests/kernels/attention/test_cpu_attn.py
pytest -x -v -s tests/kernels/core/test_cpu_activation.py
pytest -x -v -s tests/kernels/moe/test_moe.py -k test_cpu_fused_moe_basic
pytest -x -v -s tests/kernels/moe/test_cpu_fused_moe.py
pytest -x -v -s tests/kernels/mamba/cpu/test_cpu_gdn_ops.py"
# skip tests requiring model downloads if HF_TOKEN is not set
@@ -7,6 +7,7 @@ import time
import numpy as np
import torch
from vllm.platforms import CpuArchEnum, current_platform
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.utils.torch_utils import set_random_seed
@@ -14,17 +15,15 @@ from vllm.utils.torch_utils import set_random_seed
try:
from vllm._custom_ops import cpu_fused_moe, cpu_prepack_moe_weight
except (ImportError, AttributeError) as e:
print("ERROR: CPU fused MoE operations are not available on this platform.")
print("This benchmark requires x86 CPU with proper vLLM CPU extensions compiled.")
print(
"The cpu_fused_moe kernel is typically available on Linux x86_64 "
"with AVX2/AVX512."
)
print(f"Import error: {e}")
sys.exit(1)
# ISA selection following test_cpu_fused_moe.py pattern
ISA_CHOICES = ["amx", "vec"] if torch.cpu._is_amx_tile_supported() else ["vec"]
ISA_CHOICES = ["vec"]
if torch.cpu._is_amx_tile_supported():
ISA_CHOICES.append("amx")
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
ISA_CHOICES.append("neon")
@torch.inference_mode()
@@ -145,7 +144,7 @@ if __name__ == "__main__":
"--isa",
type=str,
choices=ISA_CHOICES,
default=ISA_CHOICES[0],
default="vec",
help=f"ISA to use (available: {ISA_CHOICES})",
)
parser.add_argument("--seed", type=int, default=0)
+1
View File
@@ -423,6 +423,7 @@ if (ASIMD_FOUND AND NOT APPLE_SILICON_FOUND)
set(VLLM_EXT_SRC
"csrc/cpu/shm.cpp"
"csrc/cpu/activation_lut_bf16.cpp"
"csrc/cpu/cpu_fused_moe.cpp"
${VLLM_EXT_SRC})
endif()
+76 -14
View File
@@ -1,5 +1,3 @@
#include <sleef.h>
#include "cpu/cpu_types.hpp"
#include "cpu/utils.hpp"
#include "cpu/micro_gemm/cpu_micro_gemm_vec.hpp"
@@ -16,6 +14,18 @@
#define AMX_DISPATCH(...) case cpu_utils::ISA::AMX:
#endif
#if defined(ARM_BF16_SUPPORT)
#include "cpu/micro_gemm/cpu_micro_gemm_neon.hpp"
#define NEON_DISPATCH(...) \
case cpu_utils::ISA::NEON: { \
using gemm_t = \
cpu_micro_gemm::MicroGemm<cpu_utils::ISA::NEON, scalar_t>; \
return __VA_ARGS__(); \
}
#else
#define NEON_DISPATCH(...) case cpu_utils::ISA::NEON:
#endif
#define CPU_ISA_DISPATCH_IMPL(ISA_TYPE, ...) \
[&] { \
switch (ISA_TYPE) { \
@@ -25,6 +35,7 @@
cpu_micro_gemm::MicroGemm<cpu_utils::ISA::VEC, scalar_t>; \
return __VA_ARGS__(); \
} \
NEON_DISPATCH(__VA_ARGS__) \
default: { \
TORCH_CHECK(false, "Invalid CPU ISA type."); \
} \
@@ -59,10 +70,12 @@ void swigluoai_and_mul(float* __restrict__ input, scalar_t* __restrict__ output,
const int32_t input_stride,
const int32_t output_stride) {
using scalar_vec_t = typename cpu_utils::VecTypeTrait<scalar_t>::vec_t;
#if !defined(__aarch64__)
// For GPT-OSS interleaved gate-up weights
alignas(64) static int32_t index[16] = {0, 2, 4, 6, 8, 10, 12, 14,
16, 18, 20, 22, 24, 26, 28, 30};
vec_op::INT32Vec16 index_vec(index);
#endif
vec_op::FP32Vec16 gate_up_max_vec(7.0);
vec_op::FP32Vec16 up_min_vec(-7.0);
vec_op::FP32Vec16 alpha_vec(1.702);
@@ -72,8 +85,15 @@ void swigluoai_and_mul(float* __restrict__ input, scalar_t* __restrict__ output,
for (int32_t m = 0; m < m_size; ++m) {
for (int32_t n = 0; n < n_size; n += 32) {
// Note: AdvSIMD does not support gather loads
#if defined(__aarch64__)
vec_op::FP32Vec16 gate_vec(vec_op::uninit);
vec_op::FP32Vec16 up_vec(vec_op::uninit);
vec_op::FP32Vec16::load_even_odd(input + n, gate_vec, up_vec);
#else
vec_op::FP32Vec16 gate_vec(input + n, index_vec);
vec_op::FP32Vec16 up_vec(input + n + 1, index_vec);
#endif
gate_vec = gate_vec.min(gate_up_max_vec);
up_vec = up_vec.clamp(up_min_vec, gate_up_max_vec);
auto sigmoid_vec = one_vec / (one_vec + fast_exp(-gate_vec * alpha_vec));
@@ -174,7 +194,7 @@ void gelu_tanh_and_mul(float* __restrict__ input, scalar_t* __restrict__ output,
auto inner_vec = w1_vec * (gate_vec + w3_vec * gate_pow3_vec);
// Note: can't use fast_exp form because diffusiongemma will generate
// wrong results
vec_op::FP32Vec16 tanh_vec(Sleef_tanhf16_u10(inner_vec.reg));
auto tanh_vec = inner_vec.tanh();
auto gelu_tanh = gate_vec * w2_vec * (one_vec + tanh_vec);
auto gated_output_fp32 = up_vec * gelu_tanh;
scalar_vec_t gated_output = scalar_vec_t(gated_output_fp32);
@@ -240,6 +260,7 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
constexpr int32_t gemm_n_tile_size = gemm_t::NSize;
constexpr int32_t gemm_m_tile_size = gemm_t::MaxMSize;
constexpr int32_t min_w13_n_tile_size = 2 * gemm_n_tile_size;
constexpr bool pack_a = gemm_t::PackA;
static_assert(gemm_n_tile_size % 16 == 0);
TORCH_CHECK_EQ(output_size_13 % min_w13_n_tile_size, 0);
@@ -266,12 +287,18 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
const int32_t w2_input_tile_size = cpu_utils::round_up<64>(
gemm_m_tile_size * input_size_2 * sizeof(scalar_t));
// use w2 input buffer only when we need to pack input
const int32_t w2_input_buffer_size =
pack_a ? cpu_utils::round_up<64>(gemm_m_tile_size * input_size_2 *
sizeof(scalar_t))
: 0;
const int32_t w2_n_tile_size = [&]() {
const int64_t cache_size = cpu_utils::get_available_l2_size();
// input tile + weight
// input tile + optional packed input + weight
const int32_t n_size_cache_limit =
(cache_size - w2_input_tile_size) / (input_size_2 * sizeof(scalar_t));
(cache_size - (pack_a ? w2_input_buffer_size : w2_input_tile_size)) /
(input_size_2 * sizeof(scalar_t));
const int32_t n_size_thread_limit =
output_size_2 / std::max(1, thread_num / topk_num);
const int32_t n_size = cpu_utils::round_down<gemm_n_tile_size>(
@@ -324,6 +351,9 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
const int32_t w13_output_buffer_offset = w13_thread_buffer_offset;
w13_thread_buffer_offset += w13_output_buffer_size;
const int32_t w2_input_buffer_offset = w13_thread_buffer_offset;
w13_thread_buffer_offset += w2_input_buffer_size;
// Weighted sum thread buffer
const int32_t ws_output_buffer_size =
cpu_utils::round_up<64>(output_size_2 * sizeof(float));
@@ -403,7 +433,8 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
gemm_t gemm;
const int32_t input_size_13_bytes = input_size_13 * sizeof(scalar_t);
const int32_t w13_n_group_stride = 16 * input_size_13;
const int32_t w13_n_group_stride =
gemm_t::WeightOCGroupSize * input_size_13;
const int32_t w13_n_tile_stride = gemm_n_tile_size * input_size_13;
for (;;) {
@@ -466,8 +497,23 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
token_idx += gemm_m_tile_size) {
const int32_t actual_token_num =
std::min(gemm_m_tile_size, curr_token_num - token_idx);
// copy inputs
{
scalar_t* __restrict__ curr_w13_gemm_input_buffer = nullptr;
if constexpr (pack_a) {
// copy and pack inputs
curr_w13_gemm_input_buffer = w13_input_buffer;
const scalar_t* w13_input_rows[gemm_m_tile_size];
for (int32_t i = 0; i < actual_token_num; ++i) {
w13_input_rows[i] =
input + curr_expand_token_id_buffer[i] * input_size_13;
}
gemm_t::pack_input_from_rows(w13_input_rows,
curr_w13_gemm_input_buffer,
actual_token_num, input_size_13);
curr_expand_token_id_buffer += actual_token_num;
} else {
// copy inputs
curr_w13_gemm_input_buffer = curr_w13_input_buffer;
scalar_t* __restrict__ curr_w13_input_buffer_iter =
curr_w13_input_buffer;
for (int32_t i = 0; i < actual_token_num; ++i) {
@@ -499,14 +545,12 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
scalar_t* __restrict__ w13_weight_ptr_1_iter = w13_weight_ptr_1;
scalar_t* __restrict__ w13_bias_ptr_0_iter = w13_bias_ptr_0;
scalar_t* __restrict__ w13_bias_ptr_1_iter = w13_bias_ptr_1;
scalar_t* __restrict__ curr_w13_input_buffer_iter =
curr_w13_input_buffer;
float* __restrict__ w13_output_buffer_0_iter = w13_output_buffer;
float* __restrict__ w13_output_buffer_1_iter =
w13_output_buffer + actual_n_tile_size / 2;
for (int32_t i = 0; i < actual_n_tile_size;
i += min_w13_n_tile_size) {
gemm.gemm(curr_w13_input_buffer_iter, w13_weight_ptr_0_iter,
gemm.gemm(curr_w13_gemm_input_buffer, w13_weight_ptr_0_iter,
w13_output_buffer_0_iter, actual_token_num,
input_size_13, input_size_13, w13_n_group_stride,
actual_n_tile_size, false);
@@ -519,7 +563,7 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
w13_bias_ptr_0_iter += gemm_n_tile_size;
}
gemm.gemm(curr_w13_input_buffer_iter, w13_weight_ptr_1_iter,
gemm.gemm(curr_w13_gemm_input_buffer, w13_weight_ptr_1_iter,
w13_output_buffer_1_iter, actual_token_num,
input_size_13, input_size_13, w13_n_group_stride,
actual_n_tile_size, false);
@@ -572,7 +616,8 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
gemm_t gemm;
const int32_t w2_n_tile_stride = gemm_n_tile_size * input_size_2;
const int32_t w2_n_group_stride = 16 * input_size_2;
const int32_t w2_n_group_stride =
gemm_t::WeightOCGroupSize * input_size_2;
for (;;) {
int32_t task_id = counter_ptr->acquire_counter();
@@ -611,13 +656,30 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
token_idx += gemm_m_tile_size) {
const int32_t actual_token_num =
std::min(gemm_m_tile_size, curr_token_num - token_idx);
scalar_t* __restrict__ curr_w2_gemm_input_buffer =
curr_w13_gemm_output_buffer;
if constexpr (pack_a) {
uint8_t* __restrict__ thread_buffer =
thread_buffer_start + thread_id * w13_thread_buffer_offset;
scalar_t* __restrict__ w2_input_buffer =
reinterpret_cast<scalar_t*>(thread_buffer +
w2_input_buffer_offset);
curr_w2_gemm_input_buffer = w2_input_buffer;
const scalar_t* w2_input_rows[gemm_m_tile_size];
for (int32_t i = 0; i < actual_token_num; ++i) {
w2_input_rows[i] = curr_w13_gemm_output_buffer + i * input_size_2;
}
gemm_t::pack_input_from_rows(w2_input_rows,
curr_w2_gemm_input_buffer,
actual_token_num, input_size_2);
}
scalar_t* __restrict__ w2_weight_ptr_iter = w2_weight_ptr;
scalar_t* __restrict__ w2_bias_ptr_iter = w2_bias_ptr;
float* __restrict__ curr_w2_gemm_output_buffer_iter =
curr_w2_gemm_output_buffer;
for (int32_t i = 0; i < actual_n_tile_size; i += gemm_n_tile_size) {
gemm.gemm(curr_w13_gemm_output_buffer, w2_weight_ptr_iter,
gemm.gemm(curr_w2_gemm_input_buffer, w2_weight_ptr_iter,
curr_w2_gemm_output_buffer_iter, actual_token_num,
input_size_2, input_size_2, w2_n_group_stride,
output_size_2, false);
+30 -1
View File
@@ -497,6 +497,26 @@ struct FP32Vec16 : public VectorizedRegWrapper<FP32Vec16, 4, float> {
reg.val[3] = Vectorized<float>(vcvt_f32_f16(vget_high_f16(v.reg.val[1])));
};
static FORCE_INLINE void load_even_odd(const float* ptr, FP32Vec16& even,
FP32Vec16& odd) noexcept {
const float32x4x2_t x01 = vuzpq_f32(vld1q_f32(ptr), vld1q_f32(ptr + 4));
const float32x4x2_t x23 =
vuzpq_f32(vld1q_f32(ptr + 8), vld1q_f32(ptr + 12));
const float32x4x2_t x45 =
vuzpq_f32(vld1q_f32(ptr + 16), vld1q_f32(ptr + 20));
const float32x4x2_t x67 =
vuzpq_f32(vld1q_f32(ptr + 24), vld1q_f32(ptr + 28));
even.reg.val[0] = VectorizedT(x01.val[0]);
even.reg.val[1] = VectorizedT(x23.val[0]);
even.reg.val[2] = VectorizedT(x45.val[0]);
even.reg.val[3] = VectorizedT(x67.val[0]);
odd.reg.val[0] = VectorizedT(x01.val[1]);
odd.reg.val[1] = VectorizedT(x23.val[1]);
odd.reg.val[2] = VectorizedT(x45.val[1]);
odd.reg.val[3] = VectorizedT(x67.val[1]);
}
FORCE_INLINE FP32Vec16 operator+(const FP32Vec16& b) const noexcept {
FP32Vec16 r(uninit);
r.reg.val[0] = reg.val[0] + b.reg.val[0];
@@ -515,6 +535,15 @@ struct FP32Vec16 : public VectorizedRegWrapper<FP32Vec16, 4, float> {
return r;
}
FORCE_INLINE FP32Vec16 operator-() const noexcept {
FP32Vec16 r(uninit);
r.reg.val[0] = reg.val[0].neg();
r.reg.val[1] = reg.val[1].neg();
r.reg.val[2] = reg.val[2].neg();
r.reg.val[3] = reg.val[3].neg();
return r;
}
FORCE_INLINE FP32Vec16 operator*(const FP32Vec16& b) const noexcept {
FP32Vec16 r(uninit);
r.reg.val[0] = reg.val[0] * b.reg.val[0];
@@ -933,4 +962,4 @@ inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16* ptr) {
inline void prefetch(const void* addr) { __builtin_prefetch(addr, 0, 1); };
}; // namespace vec_op
}; // namespace vec_op
+7
View File
@@ -363,6 +363,13 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
return FP32Vec16(ret);
}
FP32Vec16 tanh() const {
f32x16_t ret;
unroll_loop<int, VEC_ELEM_NUM>(
[&ret, this](int i) { ret.val[i] = std::tanh(reg.val[i]); });
return FP32Vec16(ret);
}
float reduce_sum() const {
float result = 0.0f;
unroll_loop<int, VEC_ELEM_NUM>(
+9
View File
@@ -3,6 +3,7 @@
#define CPU_TYPES_X86_HPP
#include <immintrin.h>
#include <sleef.h>
#include <torch/all.h>
#ifndef __AVX2__
@@ -592,6 +593,8 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
FP32Vec16 abs() const { return FP32Vec16(_mm512_abs_ps(reg)); }
FP32Vec16 tanh() const { return FP32Vec16(Sleef_tanhf16_u10(reg)); }
float reduce_sum() const { return _mm512_reduce_add_ps(reg); }
float reduce_max() const { return _mm512_reduce_max_ps(reg); }
@@ -789,6 +792,12 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
_mm256_andnot_ps(sign_mask, reg_high));
}
FP32Vec16 tanh() const {
FP32Vec8 low(reg_low);
FP32Vec8 high(reg_high);
return FP32Vec16(low.tanh().reg, high.tanh().reg);
}
FP32Vec16 min(const FP32Vec16& b) const {
return FP32Vec16(_mm256_min_ps(reg_low, b.reg_low),
_mm256_min_ps(reg_high, b.reg_high));
@@ -213,6 +213,8 @@ class MicroGemm<cpu_utils::ISA::AMX, scalar_t> {
public:
static constexpr int32_t MaxMSize = 32;
static constexpr int32_t NSize = 32;
static constexpr int32_t WeightOCGroupSize = 16;
static constexpr bool PackA = false;
public:
MicroGemm() : curr_m_(-1) {
@@ -21,6 +21,9 @@ class MicroGemm {
public:
static constexpr int32_t MaxMSize = 16;
static constexpr int32_t NSize = 16;
static constexpr int32_t WeightOCGroupSize = 16;
// callers must pack A matrix before GEMM
static constexpr bool PackA = false;
public:
void gemm(DEFINE_CPU_MICRO_GEMM_PARAMS) {
+503
View File
@@ -0,0 +1,503 @@
#ifndef CPU_MICRO_GEMM_NEON_HPP
#define CPU_MICRO_GEMM_NEON_HPP
#include <algorithm>
#include <cstdint>
#include "cpu/micro_gemm/cpu_micro_gemm_impl.hpp"
#include <arm_bf16.h>
#include <arm_neon.h>
namespace cpu_micro_gemm {
namespace {
constexpr int32_t K = 4;
constexpr int32_t Cols = 2;
constexpr int32_t TileSize = K * Cols;
constexpr int32_t Mr = 8;
constexpr int32_t Nr = 8;
constexpr int32_t Nr_gemv = 16;
// a = [a0, a1, a2, a3], b = [b0, b1, b2, b3] -> [a0, a1, b0, b1]
FORCE_INLINE float32x4_t zip1_f32x4(const float32x4_t a, const float32x4_t b) {
return vreinterpretq_f32_f64(
vzip1q_f64(vreinterpretq_f64_f32(a), vreinterpretq_f64_f32(b)));
}
// a = [a0, a1, a2, a3], b = [b0, b1, b2, b3] -> [a2, a3, b2, b3]
FORCE_INLINE float32x4_t zip2_f32x4(const float32x4_t a, const float32x4_t b) {
return vreinterpretq_f32_f64(
vzip2q_f64(vreinterpretq_f64_f32(a), vreinterpretq_f64_f32(b)));
}
FORCE_INLINE void init_acc_rowpair(float32x4_t& acc01, float32x4_t& acc23,
float32x4_t& acc45, float32x4_t& acc67,
const float* __restrict__ c_ptr,
const int64_t ldc, const int32_t m_rows,
const bool accum_c) {
if (!accum_c || m_rows == 0) {
acc01 = vdupq_n_f32(0.0f);
acc23 = vdupq_n_f32(0.0f);
acc45 = vdupq_n_f32(0.0f);
acc67 = vdupq_n_f32(0.0f);
return;
}
const float32x4_t row0_0123 = vld1q_f32(c_ptr);
const float32x4_t row0_4567 = vld1q_f32(c_ptr + 4);
const float32x4_t row1_0123 =
(m_rows == 2) ? vld1q_f32(c_ptr + ldc) : vdupq_n_f32(0.0f);
const float32x4_t row1_4567 =
(m_rows == 2) ? vld1q_f32(c_ptr + ldc + 4) : vdupq_n_f32(0.0f);
acc01 = zip1_f32x4(row0_0123, row1_0123);
acc23 = zip2_f32x4(row0_0123, row1_0123);
acc45 = zip1_f32x4(row0_4567, row1_4567);
acc67 = zip2_f32x4(row0_4567, row1_4567);
}
FORCE_INLINE void store_acc_rowpair(const float32x4_t acc01,
const float32x4_t acc23,
const float32x4_t acc45,
const float32x4_t acc67,
float* __restrict__ c_ptr,
const int64_t ldc, const int32_t m_rows) {
if (m_rows == 0) {
return;
}
vst1q_f32(c_ptr, zip1_f32x4(acc01, acc23));
vst1q_f32(c_ptr + 4, zip1_f32x4(acc45, acc67));
if (m_rows == 2) {
vst1q_f32(c_ptr + ldc, zip2_f32x4(acc01, acc23));
vst1q_f32(c_ptr + ldc + 4, zip2_f32x4(acc45, acc67));
}
}
FORCE_INLINE void gemm_micro_bfmmla_8x8_packed_a(
const bfloat16_t* __restrict__ a_packed,
const bfloat16_t* __restrict__ b_packed, float* __restrict__ c_ptr,
const int32_t m, const int32_t k_size, const int64_t ldc,
const bool accum_c) {
float32x4_t acc0101, acc0123, acc0145, acc0167;
float32x4_t acc2301, acc2323, acc2345, acc2367;
float32x4_t acc4501, acc4523, acc4545, acc4567;
float32x4_t acc6701, acc6723, acc6745, acc6767;
init_acc_rowpair(acc0101, acc0123, acc0145, acc0167, c_ptr, ldc,
std::min(2, m), accum_c);
init_acc_rowpair(acc2301, acc2323, acc2345, acc2367, c_ptr + 2 * ldc, ldc,
std::min(2, std::max(0, m - 2)), accum_c);
init_acc_rowpair(acc4501, acc4523, acc4545, acc4567, c_ptr + 4 * ldc, ldc,
std::min(2, std::max(0, m - 4)), accum_c);
init_acc_rowpair(acc6701, acc6723, acc6745, acc6767, c_ptr + 6 * ldc, ldc,
std::min(2, std::max(0, m - 6)), accum_c);
const bfloat16_t* __restrict__ a_tile = a_packed;
const bfloat16_t* __restrict__ b_tile = b_packed;
#pragma GCC unroll 8
for (int32_t k_idx = 0; k_idx < k_size; k_idx += K) {
const bfloat16x8_t a_tile01 = vld1q_bf16(a_tile);
const bfloat16x8_t a_tile23 = vld1q_bf16(a_tile + TileSize);
const bfloat16x8_t a_tile45 = vld1q_bf16(a_tile + 2 * TileSize);
const bfloat16x8_t a_tile67 = vld1q_bf16(a_tile + 3 * TileSize);
const bfloat16x8_t b_tile01 = vld1q_bf16(b_tile);
const bfloat16x8_t b_tile23 = vld1q_bf16(b_tile + TileSize);
const bfloat16x8_t b_tile45 = vld1q_bf16(b_tile + 2 * TileSize);
const bfloat16x8_t b_tile67 = vld1q_bf16(b_tile + 3 * TileSize);
acc0101 = vbfmmlaq_f32(acc0101, a_tile01, b_tile01);
acc2301 = vbfmmlaq_f32(acc2301, a_tile23, b_tile01);
acc4501 = vbfmmlaq_f32(acc4501, a_tile45, b_tile01);
acc6701 = vbfmmlaq_f32(acc6701, a_tile67, b_tile01);
acc0123 = vbfmmlaq_f32(acc0123, a_tile01, b_tile23);
acc2323 = vbfmmlaq_f32(acc2323, a_tile23, b_tile23);
acc4523 = vbfmmlaq_f32(acc4523, a_tile45, b_tile23);
acc6723 = vbfmmlaq_f32(acc6723, a_tile67, b_tile23);
acc0145 = vbfmmlaq_f32(acc0145, a_tile01, b_tile45);
acc2345 = vbfmmlaq_f32(acc2345, a_tile23, b_tile45);
acc4545 = vbfmmlaq_f32(acc4545, a_tile45, b_tile45);
acc6745 = vbfmmlaq_f32(acc6745, a_tile67, b_tile45);
acc0167 = vbfmmlaq_f32(acc0167, a_tile01, b_tile67);
acc2367 = vbfmmlaq_f32(acc2367, a_tile23, b_tile67);
acc4567 = vbfmmlaq_f32(acc4567, a_tile45, b_tile67);
acc6767 = vbfmmlaq_f32(acc6767, a_tile67, b_tile67);
a_tile += 4 * TileSize;
b_tile += Nr * K;
}
store_acc_rowpair(acc0101, acc0123, acc0145, acc0167, c_ptr, ldc,
std::min(2, m));
store_acc_rowpair(acc2301, acc2323, acc2345, acc2367, c_ptr + 2 * ldc, ldc,
std::min(2, std::max(0, m - 2)));
store_acc_rowpair(acc4501, acc4523, acc4545, acc4567, c_ptr + 4 * ldc, ldc,
std::min(2, std::max(0, m - 4)));
store_acc_rowpair(acc6701, acc6723, acc6745, acc6767, c_ptr + 6 * ldc, ldc,
std::min(2, std::max(0, m - 6)));
}
FORCE_INLINE void gemm_micro_bfmmla_4x16_packed_a(
const bfloat16_t* __restrict__ a_packed,
const bfloat16_t* __restrict__ b_packed, float* __restrict__ c_ptr,
const int32_t m, const int32_t k_size, const int64_t b_n_group_stride,
const int64_t ldc, const bool accum_c) {
const int32_t m_rows_01 = std::min(2, m);
const int32_t m_rows_23 = std::min(2, std::max(0, m - 2));
float32x4_t acc0101, acc0123, acc0145, acc0167;
float32x4_t acc2301, acc2323, acc2345, acc2367;
float32x4_t acc0189, acc011011, acc011213, acc011415;
float32x4_t acc2389, acc231011, acc231213, acc231415;
init_acc_rowpair(acc0101, acc0123, acc0145, acc0167, c_ptr, ldc, m_rows_01,
accum_c);
init_acc_rowpair(acc2301, acc2323, acc2345, acc2367, c_ptr + 2 * ldc, ldc,
m_rows_23, accum_c);
init_acc_rowpair(acc0189, acc011011, acc011213, acc011415, c_ptr + 8, ldc,
m_rows_01, accum_c);
init_acc_rowpair(acc2389, acc231011, acc231213, acc231415,
c_ptr + 2 * ldc + 8, ldc, m_rows_23, accum_c);
const bfloat16_t* __restrict__ a_tile = a_packed;
const bfloat16_t* __restrict__ b_tile0 = b_packed;
const bfloat16_t* __restrict__ b_tile1 = b_packed + b_n_group_stride;
#pragma GCC unroll 8
for (int32_t k_idx = 0; k_idx < k_size; k_idx += K) {
const bfloat16x8_t a_tile01 = vld1q_bf16(a_tile);
const bfloat16x8_t a_tile23 = vld1q_bf16(a_tile + TileSize);
const bfloat16x8_t b_tile01 = vld1q_bf16(b_tile0);
const bfloat16x8_t b_tile23 = vld1q_bf16(b_tile0 + TileSize);
const bfloat16x8_t b_tile45 = vld1q_bf16(b_tile0 + 2 * TileSize);
const bfloat16x8_t b_tile67 = vld1q_bf16(b_tile0 + 3 * TileSize);
const bfloat16x8_t b_tile89 = vld1q_bf16(b_tile1);
const bfloat16x8_t b_tile1011 = vld1q_bf16(b_tile1 + TileSize);
const bfloat16x8_t b_tile1213 = vld1q_bf16(b_tile1 + 2 * TileSize);
const bfloat16x8_t b_tile1415 = vld1q_bf16(b_tile1 + 3 * TileSize);
acc0101 = vbfmmlaq_f32(acc0101, a_tile01, b_tile01);
acc2301 = vbfmmlaq_f32(acc2301, a_tile23, b_tile01);
acc0123 = vbfmmlaq_f32(acc0123, a_tile01, b_tile23);
acc2323 = vbfmmlaq_f32(acc2323, a_tile23, b_tile23);
acc0145 = vbfmmlaq_f32(acc0145, a_tile01, b_tile45);
acc2345 = vbfmmlaq_f32(acc2345, a_tile23, b_tile45);
acc0167 = vbfmmlaq_f32(acc0167, a_tile01, b_tile67);
acc2367 = vbfmmlaq_f32(acc2367, a_tile23, b_tile67);
acc0189 = vbfmmlaq_f32(acc0189, a_tile01, b_tile89);
acc2389 = vbfmmlaq_f32(acc2389, a_tile23, b_tile89);
acc011011 = vbfmmlaq_f32(acc011011, a_tile01, b_tile1011);
acc231011 = vbfmmlaq_f32(acc231011, a_tile23, b_tile1011);
acc011213 = vbfmmlaq_f32(acc011213, a_tile01, b_tile1213);
acc231213 = vbfmmlaq_f32(acc231213, a_tile23, b_tile1213);
acc011415 = vbfmmlaq_f32(acc011415, a_tile01, b_tile1415);
acc231415 = vbfmmlaq_f32(acc231415, a_tile23, b_tile1415);
a_tile += 2 * TileSize;
b_tile0 += Nr * K;
b_tile1 += Nr * K;
}
store_acc_rowpair(acc0101, acc0123, acc0145, acc0167, c_ptr, ldc, m_rows_01);
store_acc_rowpair(acc2301, acc2323, acc2345, acc2367, c_ptr + 2 * ldc, ldc,
m_rows_23);
store_acc_rowpair(acc0189, acc011011, acc011213, acc011415, c_ptr + 8, ldc,
m_rows_01);
store_acc_rowpair(acc2389, acc231011, acc231213, acc231415,
c_ptr + 2 * ldc + 8, ldc, m_rows_23);
}
} // namespace
template <typename scalar_t>
class MicroGemm<cpu_utils::ISA::NEON, scalar_t> {
public:
static constexpr int32_t MaxMSize = 8;
static constexpr int32_t NSize = 32;
static constexpr int32_t WeightOCGroupSize = Nr;
static constexpr bool PackA = false;
public:
void gemm(DEFINE_CPU_MICRO_GEMM_PARAMS) {
TORCH_CHECK(false, "NEON BFMMLA MicroGemm only supports bfloat16.");
}
static void pack_weight(const scalar_t* __restrict__ /*weight*/,
scalar_t* __restrict__ /*packed_weight*/,
const int32_t /*output_size*/,
const int32_t /*input_size*/) {
TORCH_CHECK(false, "NEON BFMMLA MicroGemm only supports bfloat16.");
}
};
template <>
class MicroGemm<cpu_utils::ISA::NEON, c10::BFloat16> {
public:
using scalar_t = c10::BFloat16;
static constexpr int32_t MaxMSize = 8;
static constexpr int32_t NSize = 32;
static constexpr int32_t WeightOCGroupSize = Nr;
static constexpr bool PackA = true;
public:
// physical layout [
// M / 8; Mr is 8
// K / 4; K for bfmmla is 4
// 4, ; 4 row-pairs for each 8 rows
// 2, ; row-pair is 2 rows
// 4 ; 4 elements per row
// ]
static void pack_input_from_rows(const scalar_t* const* __restrict__ rows,
scalar_t* __restrict__ a_packed,
const int32_t m, const int32_t k) {
TORCH_CHECK(m > 0 && m <= MaxMSize);
TORCH_CHECK_EQ(k % K, 0);
auto* __restrict__ out = reinterpret_cast<bfloat16_t*>(a_packed);
const bfloat16x8_t zero_q = vdupq_n_bf16(bfloat16_t{});
const bfloat16x4_t zero = vget_low_bf16(zero_q);
for (int32_t row_base = 0; row_base < m; row_base += Mr) {
const int32_t actual_m = std::min(Mr, m - row_base);
const bfloat16_t* __restrict__ row[Mr];
for (int32_t i = 0; i < actual_m; ++i) {
row[i] = reinterpret_cast<const bfloat16_t*>(rows[row_base + i]);
}
if (actual_m == 8) {
int32_t k_idx = 0;
for (; k_idx + 8 <= k; k_idx += 8) {
bfloat16_t* __restrict__ block0 = out;
bfloat16_t* __restrict__ block1 = out + 4 * TileSize;
bfloat16x8_t a0 = vld1q_bf16(row[0] + k_idx);
bfloat16x8_t a1 = vld1q_bf16(row[1] + k_idx);
vst1q_bf16(block0,
vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
vst1q_bf16(block1,
vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
a0 = vld1q_bf16(row[2] + k_idx);
a1 = vld1q_bf16(row[3] + k_idx);
vst1q_bf16(block0 + TileSize,
vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
vst1q_bf16(block1 + TileSize,
vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
a0 = vld1q_bf16(row[4] + k_idx);
a1 = vld1q_bf16(row[5] + k_idx);
vst1q_bf16(block0 + 2 * TileSize,
vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
vst1q_bf16(block1 + 2 * TileSize,
vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
a0 = vld1q_bf16(row[6] + k_idx);
a1 = vld1q_bf16(row[7] + k_idx);
vst1q_bf16(block0 + 3 * TileSize,
vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
vst1q_bf16(block1 + 3 * TileSize,
vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
out += 8 * TileSize;
}
for (; k_idx < k; k_idx += K) {
bfloat16x4_t a0 = vld1_bf16(row[0] + k_idx);
bfloat16x4_t a1 = vld1_bf16(row[1] + k_idx);
vst1q_bf16(out, vcombine_bf16(a0, a1));
a0 = vld1_bf16(row[2] + k_idx);
a1 = vld1_bf16(row[3] + k_idx);
vst1q_bf16(out + TileSize, vcombine_bf16(a0, a1));
a0 = vld1_bf16(row[4] + k_idx);
a1 = vld1_bf16(row[5] + k_idx);
vst1q_bf16(out + 2 * TileSize, vcombine_bf16(a0, a1));
a0 = vld1_bf16(row[6] + k_idx);
a1 = vld1_bf16(row[7] + k_idx);
vst1q_bf16(out + 3 * TileSize, vcombine_bf16(a0, a1));
out += 4 * TileSize;
}
continue;
}
if (actual_m == 4) {
int32_t k_idx = 0;
for (; k_idx + 8 <= k; k_idx += 8) {
bfloat16_t* __restrict__ block0 = out;
bfloat16_t* __restrict__ block1 = out + 2 * TileSize;
bfloat16x8_t a0 = vld1q_bf16(row[0] + k_idx);
bfloat16x8_t a1 = vld1q_bf16(row[1] + k_idx);
vst1q_bf16(block0,
vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
vst1q_bf16(block1,
vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
a0 = vld1q_bf16(row[2] + k_idx);
a1 = vld1q_bf16(row[3] + k_idx);
vst1q_bf16(block0 + TileSize,
vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
vst1q_bf16(block1 + TileSize,
vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
out += 4 * TileSize;
}
for (; k_idx < k; k_idx += K) {
bfloat16x4_t a0 = vld1_bf16(row[0] + k_idx);
bfloat16x4_t a1 = vld1_bf16(row[1] + k_idx);
vst1q_bf16(out, vcombine_bf16(a0, a1));
a0 = vld1_bf16(row[2] + k_idx);
a1 = vld1_bf16(row[3] + k_idx);
vst1q_bf16(out + TileSize, vcombine_bf16(a0, a1));
out += 2 * TileSize;
}
continue;
}
const int32_t row_pair_count = (actual_m <= 4) ? 2 : Mr / 2;
int32_t k_idx = 0;
for (; k_idx + 8 <= k; k_idx += 8) {
bfloat16_t* __restrict__ block0 = out;
bfloat16_t* __restrict__ block1 = out + row_pair_count * TileSize;
bfloat16x8_t a0 = vld1q_bf16(row[0] + k_idx);
bfloat16x8_t a1 = (actual_m > 1) ? vld1q_bf16(row[1] + k_idx) : zero_q;
vst1q_bf16(block0, vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
vst1q_bf16(block1,
vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
a0 = (actual_m > 2) ? vld1q_bf16(row[2] + k_idx) : zero_q;
a1 = (actual_m > 3) ? vld1q_bf16(row[3] + k_idx) : zero_q;
vst1q_bf16(block0 + TileSize,
vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
vst1q_bf16(block1 + TileSize,
vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
if (actual_m > 4) {
a0 = vld1q_bf16(row[4] + k_idx);
a1 = (actual_m > 5) ? vld1q_bf16(row[5] + k_idx) : zero_q;
vst1q_bf16(block0 + 2 * TileSize,
vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
vst1q_bf16(block1 + 2 * TileSize,
vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
a0 = (actual_m > 6) ? vld1q_bf16(row[6] + k_idx) : zero_q;
a1 = (actual_m > 7) ? vld1q_bf16(row[7] + k_idx) : zero_q;
vst1q_bf16(block0 + 3 * TileSize,
vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
vst1q_bf16(block1 + 3 * TileSize,
vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
}
out += 2 * row_pair_count * TileSize;
}
for (; k_idx < k; k_idx += K) {
bfloat16x4_t a0 = vld1_bf16(row[0] + k_idx);
bfloat16x4_t a1 = (actual_m > 1) ? vld1_bf16(row[1] + k_idx) : zero;
vst1q_bf16(out, vcombine_bf16(a0, a1));
a0 = (actual_m > 2) ? vld1_bf16(row[2] + k_idx) : zero;
a1 = (actual_m > 3) ? vld1_bf16(row[3] + k_idx) : zero;
vst1q_bf16(out + TileSize, vcombine_bf16(a0, a1));
if (actual_m > 4) {
a0 = vld1_bf16(row[4] + k_idx);
a1 = (actual_m > 5) ? vld1_bf16(row[5] + k_idx) : zero;
vst1q_bf16(out + 2 * TileSize, vcombine_bf16(a0, a1));
a0 = (actual_m > 6) ? vld1_bf16(row[6] + k_idx) : zero;
a1 = (actual_m > 7) ? vld1_bf16(row[7] + k_idx) : zero;
vst1q_bf16(out + 3 * TileSize, vcombine_bf16(a0, a1));
}
out += row_pair_count * TileSize;
}
}
}
void gemm(DEFINE_CPU_MICRO_GEMM_PARAMS) {
(void)lda; // A is packed, so lda is not needed
TORCH_CHECK_EQ(k % K, 0);
for (int32_t n_idx = 0; n_idx < NSize; n_idx += Nr_gemv) {
const bfloat16_t* __restrict__ b_panel =
reinterpret_cast<const bfloat16_t*>(b_ptr) + n_idx * k;
for (int32_t row_base = 0; row_base < m; row_base += Mr) {
const int32_t panel_m = std::min(Mr, m - row_base);
const bfloat16_t* __restrict__ a_panel =
reinterpret_cast<const bfloat16_t*>(a_ptr) + row_base * k;
float* __restrict__ c_panel = c_ptr + row_base * ldc + n_idx;
if (panel_m <= 4) {
gemm_micro_bfmmla_4x16_packed_a(a_panel, b_panel, c_panel, panel_m, k,
b_n_group_stride, ldc, accum_c);
} else {
gemm_micro_bfmmla_8x8_packed_a(a_panel, b_panel, c_panel, panel_m, k,
ldc, accum_c);
gemm_micro_bfmmla_8x8_packed_a(a_panel, b_panel + b_n_group_stride,
c_panel + Nr, panel_m, k, ldc,
accum_c);
}
}
}
}
// physical layout [
// N / 8; Nr is 8
// K / 4; K for bfmmla is 4
// 4, ; 4 col-pairs for each 8 cols
// 2, ; col-pair is 2 cols
// 4 ; 4 elements per col
// ]
static void pack_weight(const c10::BFloat16* __restrict__ weight,
c10::BFloat16* __restrict__ packed_weight,
const int32_t output_size, const int32_t input_size) {
TORCH_CHECK_EQ(output_size % NSize, 0);
TORCH_CHECK_EQ(input_size % K, 0);
for (int32_t o_idx = 0; o_idx < output_size; o_idx += Nr) {
c10::BFloat16* __restrict__ dst = packed_weight + o_idx * input_size;
for (int32_t k_idx = 0; k_idx < input_size; k_idx += K) {
for (int32_t pair_idx = 0; pair_idx < Nr; pair_idx += Cols) {
const c10::BFloat16* __restrict__ row0 =
weight + (o_idx + pair_idx) * input_size;
const c10::BFloat16* __restrict__ row1 = row0 + input_size;
dst[0] = row0[k_idx + 0];
dst[1] = row0[k_idx + 1];
dst[2] = row0[k_idx + 2];
dst[3] = row0[k_idx + 3];
dst[4] = row1[k_idx + 0];
dst[5] = row1[k_idx + 1];
dst[6] = row1[k_idx + 2];
dst[7] = row1[k_idx + 3];
dst += TileSize;
}
}
}
}
};
} // namespace cpu_micro_gemm
#endif
@@ -104,6 +104,8 @@ class MicroGemm<cpu_utils::ISA::VEC, scalar_t> {
public:
static constexpr int32_t MaxMSize = 8;
static constexpr int32_t NSize = 32;
static constexpr int32_t WeightOCGroupSize = 16;
static constexpr bool PackA = false;
public:
void gemm(DEFINE_CPU_MICRO_GEMM_PARAMS) {
+2 -2
View File
@@ -538,7 +538,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
#endif
// fused moe
#if defined(__AVX512F__)
#if defined(__AVX512F__) || (defined(ARM_BF16_SUPPORT))
ops.def(
"prepack_moe_weight(Tensor weight, Tensor(a1!) packed_weight, str isa) "
"-> ()");
@@ -549,7 +549,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
"bool skip_weighted, "
"str act, str isa) -> ()");
ops.impl("cpu_fused_moe", torch::kCPU, &cpu_fused_moe);
#endif
#endif // #if defined(__AVX512F__) || (defined(ARM_BF16_SUPPORT))
ops.def(
"mla_decode_kvcache("
" Tensor! out, Tensor query, Tensor kv_cache,"
+4 -1
View File
@@ -2,13 +2,14 @@
#define UTILS_HPP
#include <atomic>
#include <string>
#include <unistd.h>
#include <ATen/cpu/Utils.h>
#include "cpu/cpu_types.hpp"
namespace cpu_utils {
enum class ISA { AMX, VEC, RVV };
enum class ISA { AMX, VEC, RVV, NEON };
inline ISA get_isa(const std::string& isa) {
if (isa == "amx") {
@@ -17,6 +18,8 @@ inline ISA get_isa(const std::string& isa) {
return ISA::VEC;
} else if (isa == "rvv") {
return ISA::RVV;
} else if (isa == "neon") {
return ISA::NEON;
} else {
TORCH_CHECK(false, "Invalid isa type: " + isa);
}
+12 -4
View File
@@ -5,10 +5,13 @@ import pytest
import torch
from tests.kernels.allclose_default import get_default_atol, get_default_rtol
from vllm._custom_ops import cpu_fused_moe, cpu_prepack_moe_weight
from vllm._custom_ops import (
cpu_fused_moe,
cpu_prepack_moe_weight,
)
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.cpu_fused_moe import _CPU_MOE_ACT_FN
from vllm.platforms import current_platform
from vllm.platforms import CpuArchEnum, current_platform
from vllm.utils.torch_utils import set_random_seed
if not current_platform.is_cpu():
@@ -26,8 +29,13 @@ ACT = [
MoEActivation.GELU,
MoEActivation.GELU_TANH,
]
USE_BIAS = [True, False]
ISA = ["amx", "vec"] if torch.cpu._is_amx_tile_supported() else ["vec"]
USE_BIAS = [False, True]
ISA = ["vec"]
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
ISA.append("neon")
if torch.cpu._is_amx_tile_supported():
ISA.append("amx")
DTYPE = [torch.bfloat16]
@@ -16,6 +16,7 @@ from vllm._custom_ops import (
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.quantization.utils.layer_utils import replace_parameter
from vllm.platforms import CpuArchEnum, current_platform
from vllm.utils.torch_utils import direct_register_custom_op
_CPU_MOE_LAYER_CACHE = {}
@@ -312,6 +313,17 @@ class CPUFusedMOE:
if supports_amx:
return False, "none"
supports_neon = current_platform.get_cpu_architecture() == CpuArchEnum.ARM
if supports_neon:
if (
dtype == torch.bfloat16
and w13_input_size % 4 == 0
and w2_input_size % 4 == 0
):
return True, "neon"
else:
return False, "none"
return True, "vec"
def init_moe_grouped_gemm(