[CPU][Perf] Accelerate unquantized MoE for AArch64 (#46353)
Signed-off-by: Fadi Arafeh <fadi.arafeh@arm.com>
This commit is contained in:
@@ -8,7 +8,7 @@ set -ex
|
||||
CORE_RANGE=${CORE_RANGE:-0-31}
|
||||
OMP_CORE_RANGE=${OMP_CORE_RANGE:-0-31}
|
||||
|
||||
export CMAKE_BUILD_PARALLEL_LEVEL=16
|
||||
export CMAKE_BUILD_PARALLEL_LEVEL=32
|
||||
|
||||
# Setup cleanup
|
||||
remove_docker_container() {
|
||||
@@ -37,7 +37,7 @@ function cpu_tests() {
|
||||
pytest -x -v -s tests/kernels/test_onednn.py
|
||||
pytest -x -v -s tests/kernels/attention/test_cpu_attn.py
|
||||
pytest -x -v -s tests/kernels/core/test_cpu_activation.py
|
||||
pytest -x -v -s tests/kernels/moe/test_moe.py -k test_cpu_fused_moe_basic
|
||||
pytest -x -v -s tests/kernels/moe/test_cpu_fused_moe.py
|
||||
pytest -x -v -s tests/kernels/mamba/cpu/test_cpu_gdn_ops.py"
|
||||
|
||||
# skip tests requiring model downloads if HF_TOKEN is not set
|
||||
|
||||
@@ -7,6 +7,7 @@ import time
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
@@ -14,17 +15,15 @@ from vllm.utils.torch_utils import set_random_seed
|
||||
try:
|
||||
from vllm._custom_ops import cpu_fused_moe, cpu_prepack_moe_weight
|
||||
except (ImportError, AttributeError) as e:
|
||||
print("ERROR: CPU fused MoE operations are not available on this platform.")
|
||||
print("This benchmark requires x86 CPU with proper vLLM CPU extensions compiled.")
|
||||
print(
|
||||
"The cpu_fused_moe kernel is typically available on Linux x86_64 "
|
||||
"with AVX2/AVX512."
|
||||
)
|
||||
print(f"Import error: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
# ISA selection following test_cpu_fused_moe.py pattern
|
||||
ISA_CHOICES = ["amx", "vec"] if torch.cpu._is_amx_tile_supported() else ["vec"]
|
||||
ISA_CHOICES = ["vec"]
|
||||
if torch.cpu._is_amx_tile_supported():
|
||||
ISA_CHOICES.append("amx")
|
||||
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
|
||||
ISA_CHOICES.append("neon")
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
@@ -145,7 +144,7 @@ if __name__ == "__main__":
|
||||
"--isa",
|
||||
type=str,
|
||||
choices=ISA_CHOICES,
|
||||
default=ISA_CHOICES[0],
|
||||
default="vec",
|
||||
help=f"ISA to use (available: {ISA_CHOICES})",
|
||||
)
|
||||
parser.add_argument("--seed", type=int, default=0)
|
||||
|
||||
@@ -423,6 +423,7 @@ if (ASIMD_FOUND AND NOT APPLE_SILICON_FOUND)
|
||||
set(VLLM_EXT_SRC
|
||||
"csrc/cpu/shm.cpp"
|
||||
"csrc/cpu/activation_lut_bf16.cpp"
|
||||
"csrc/cpu/cpu_fused_moe.cpp"
|
||||
${VLLM_EXT_SRC})
|
||||
endif()
|
||||
|
||||
|
||||
+76
-14
@@ -1,5 +1,3 @@
|
||||
#include <sleef.h>
|
||||
|
||||
#include "cpu/cpu_types.hpp"
|
||||
#include "cpu/utils.hpp"
|
||||
#include "cpu/micro_gemm/cpu_micro_gemm_vec.hpp"
|
||||
@@ -16,6 +14,18 @@
|
||||
#define AMX_DISPATCH(...) case cpu_utils::ISA::AMX:
|
||||
#endif
|
||||
|
||||
#if defined(ARM_BF16_SUPPORT)
|
||||
#include "cpu/micro_gemm/cpu_micro_gemm_neon.hpp"
|
||||
#define NEON_DISPATCH(...) \
|
||||
case cpu_utils::ISA::NEON: { \
|
||||
using gemm_t = \
|
||||
cpu_micro_gemm::MicroGemm<cpu_utils::ISA::NEON, scalar_t>; \
|
||||
return __VA_ARGS__(); \
|
||||
}
|
||||
#else
|
||||
#define NEON_DISPATCH(...) case cpu_utils::ISA::NEON:
|
||||
#endif
|
||||
|
||||
#define CPU_ISA_DISPATCH_IMPL(ISA_TYPE, ...) \
|
||||
[&] { \
|
||||
switch (ISA_TYPE) { \
|
||||
@@ -25,6 +35,7 @@
|
||||
cpu_micro_gemm::MicroGemm<cpu_utils::ISA::VEC, scalar_t>; \
|
||||
return __VA_ARGS__(); \
|
||||
} \
|
||||
NEON_DISPATCH(__VA_ARGS__) \
|
||||
default: { \
|
||||
TORCH_CHECK(false, "Invalid CPU ISA type."); \
|
||||
} \
|
||||
@@ -59,10 +70,12 @@ void swigluoai_and_mul(float* __restrict__ input, scalar_t* __restrict__ output,
|
||||
const int32_t input_stride,
|
||||
const int32_t output_stride) {
|
||||
using scalar_vec_t = typename cpu_utils::VecTypeTrait<scalar_t>::vec_t;
|
||||
#if !defined(__aarch64__)
|
||||
// For GPT-OSS interleaved gate-up weights
|
||||
alignas(64) static int32_t index[16] = {0, 2, 4, 6, 8, 10, 12, 14,
|
||||
16, 18, 20, 22, 24, 26, 28, 30};
|
||||
vec_op::INT32Vec16 index_vec(index);
|
||||
#endif
|
||||
vec_op::FP32Vec16 gate_up_max_vec(7.0);
|
||||
vec_op::FP32Vec16 up_min_vec(-7.0);
|
||||
vec_op::FP32Vec16 alpha_vec(1.702);
|
||||
@@ -72,8 +85,15 @@ void swigluoai_and_mul(float* __restrict__ input, scalar_t* __restrict__ output,
|
||||
|
||||
for (int32_t m = 0; m < m_size; ++m) {
|
||||
for (int32_t n = 0; n < n_size; n += 32) {
|
||||
// Note: AdvSIMD does not support gather loads
|
||||
#if defined(__aarch64__)
|
||||
vec_op::FP32Vec16 gate_vec(vec_op::uninit);
|
||||
vec_op::FP32Vec16 up_vec(vec_op::uninit);
|
||||
vec_op::FP32Vec16::load_even_odd(input + n, gate_vec, up_vec);
|
||||
#else
|
||||
vec_op::FP32Vec16 gate_vec(input + n, index_vec);
|
||||
vec_op::FP32Vec16 up_vec(input + n + 1, index_vec);
|
||||
#endif
|
||||
gate_vec = gate_vec.min(gate_up_max_vec);
|
||||
up_vec = up_vec.clamp(up_min_vec, gate_up_max_vec);
|
||||
auto sigmoid_vec = one_vec / (one_vec + fast_exp(-gate_vec * alpha_vec));
|
||||
@@ -174,7 +194,7 @@ void gelu_tanh_and_mul(float* __restrict__ input, scalar_t* __restrict__ output,
|
||||
auto inner_vec = w1_vec * (gate_vec + w3_vec * gate_pow3_vec);
|
||||
// Note: can't use fast_exp form because diffusiongemma will generate
|
||||
// wrong results
|
||||
vec_op::FP32Vec16 tanh_vec(Sleef_tanhf16_u10(inner_vec.reg));
|
||||
auto tanh_vec = inner_vec.tanh();
|
||||
auto gelu_tanh = gate_vec * w2_vec * (one_vec + tanh_vec);
|
||||
auto gated_output_fp32 = up_vec * gelu_tanh;
|
||||
scalar_vec_t gated_output = scalar_vec_t(gated_output_fp32);
|
||||
@@ -240,6 +260,7 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
|
||||
constexpr int32_t gemm_n_tile_size = gemm_t::NSize;
|
||||
constexpr int32_t gemm_m_tile_size = gemm_t::MaxMSize;
|
||||
constexpr int32_t min_w13_n_tile_size = 2 * gemm_n_tile_size;
|
||||
constexpr bool pack_a = gemm_t::PackA;
|
||||
static_assert(gemm_n_tile_size % 16 == 0);
|
||||
|
||||
TORCH_CHECK_EQ(output_size_13 % min_w13_n_tile_size, 0);
|
||||
@@ -266,12 +287,18 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
|
||||
|
||||
const int32_t w2_input_tile_size = cpu_utils::round_up<64>(
|
||||
gemm_m_tile_size * input_size_2 * sizeof(scalar_t));
|
||||
// use w2 input buffer only when we need to pack input
|
||||
const int32_t w2_input_buffer_size =
|
||||
pack_a ? cpu_utils::round_up<64>(gemm_m_tile_size * input_size_2 *
|
||||
sizeof(scalar_t))
|
||||
: 0;
|
||||
|
||||
const int32_t w2_n_tile_size = [&]() {
|
||||
const int64_t cache_size = cpu_utils::get_available_l2_size();
|
||||
// input tile + weight
|
||||
// input tile + optional packed input + weight
|
||||
const int32_t n_size_cache_limit =
|
||||
(cache_size - w2_input_tile_size) / (input_size_2 * sizeof(scalar_t));
|
||||
(cache_size - (pack_a ? w2_input_buffer_size : w2_input_tile_size)) /
|
||||
(input_size_2 * sizeof(scalar_t));
|
||||
const int32_t n_size_thread_limit =
|
||||
output_size_2 / std::max(1, thread_num / topk_num);
|
||||
const int32_t n_size = cpu_utils::round_down<gemm_n_tile_size>(
|
||||
@@ -324,6 +351,9 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
|
||||
const int32_t w13_output_buffer_offset = w13_thread_buffer_offset;
|
||||
w13_thread_buffer_offset += w13_output_buffer_size;
|
||||
|
||||
const int32_t w2_input_buffer_offset = w13_thread_buffer_offset;
|
||||
w13_thread_buffer_offset += w2_input_buffer_size;
|
||||
|
||||
// Weighted sum thread buffer
|
||||
const int32_t ws_output_buffer_size =
|
||||
cpu_utils::round_up<64>(output_size_2 * sizeof(float));
|
||||
@@ -403,7 +433,8 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
|
||||
gemm_t gemm;
|
||||
|
||||
const int32_t input_size_13_bytes = input_size_13 * sizeof(scalar_t);
|
||||
const int32_t w13_n_group_stride = 16 * input_size_13;
|
||||
const int32_t w13_n_group_stride =
|
||||
gemm_t::WeightOCGroupSize * input_size_13;
|
||||
const int32_t w13_n_tile_stride = gemm_n_tile_size * input_size_13;
|
||||
|
||||
for (;;) {
|
||||
@@ -466,8 +497,23 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
|
||||
token_idx += gemm_m_tile_size) {
|
||||
const int32_t actual_token_num =
|
||||
std::min(gemm_m_tile_size, curr_token_num - token_idx);
|
||||
// copy inputs
|
||||
{
|
||||
|
||||
scalar_t* __restrict__ curr_w13_gemm_input_buffer = nullptr;
|
||||
if constexpr (pack_a) {
|
||||
// copy and pack inputs
|
||||
curr_w13_gemm_input_buffer = w13_input_buffer;
|
||||
const scalar_t* w13_input_rows[gemm_m_tile_size];
|
||||
for (int32_t i = 0; i < actual_token_num; ++i) {
|
||||
w13_input_rows[i] =
|
||||
input + curr_expand_token_id_buffer[i] * input_size_13;
|
||||
}
|
||||
gemm_t::pack_input_from_rows(w13_input_rows,
|
||||
curr_w13_gemm_input_buffer,
|
||||
actual_token_num, input_size_13);
|
||||
curr_expand_token_id_buffer += actual_token_num;
|
||||
} else {
|
||||
// copy inputs
|
||||
curr_w13_gemm_input_buffer = curr_w13_input_buffer;
|
||||
scalar_t* __restrict__ curr_w13_input_buffer_iter =
|
||||
curr_w13_input_buffer;
|
||||
for (int32_t i = 0; i < actual_token_num; ++i) {
|
||||
@@ -499,14 +545,12 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
|
||||
scalar_t* __restrict__ w13_weight_ptr_1_iter = w13_weight_ptr_1;
|
||||
scalar_t* __restrict__ w13_bias_ptr_0_iter = w13_bias_ptr_0;
|
||||
scalar_t* __restrict__ w13_bias_ptr_1_iter = w13_bias_ptr_1;
|
||||
scalar_t* __restrict__ curr_w13_input_buffer_iter =
|
||||
curr_w13_input_buffer;
|
||||
float* __restrict__ w13_output_buffer_0_iter = w13_output_buffer;
|
||||
float* __restrict__ w13_output_buffer_1_iter =
|
||||
w13_output_buffer + actual_n_tile_size / 2;
|
||||
for (int32_t i = 0; i < actual_n_tile_size;
|
||||
i += min_w13_n_tile_size) {
|
||||
gemm.gemm(curr_w13_input_buffer_iter, w13_weight_ptr_0_iter,
|
||||
gemm.gemm(curr_w13_gemm_input_buffer, w13_weight_ptr_0_iter,
|
||||
w13_output_buffer_0_iter, actual_token_num,
|
||||
input_size_13, input_size_13, w13_n_group_stride,
|
||||
actual_n_tile_size, false);
|
||||
@@ -519,7 +563,7 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
|
||||
w13_bias_ptr_0_iter += gemm_n_tile_size;
|
||||
}
|
||||
|
||||
gemm.gemm(curr_w13_input_buffer_iter, w13_weight_ptr_1_iter,
|
||||
gemm.gemm(curr_w13_gemm_input_buffer, w13_weight_ptr_1_iter,
|
||||
w13_output_buffer_1_iter, actual_token_num,
|
||||
input_size_13, input_size_13, w13_n_group_stride,
|
||||
actual_n_tile_size, false);
|
||||
@@ -572,7 +616,8 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
|
||||
gemm_t gemm;
|
||||
|
||||
const int32_t w2_n_tile_stride = gemm_n_tile_size * input_size_2;
|
||||
const int32_t w2_n_group_stride = 16 * input_size_2;
|
||||
const int32_t w2_n_group_stride =
|
||||
gemm_t::WeightOCGroupSize * input_size_2;
|
||||
|
||||
for (;;) {
|
||||
int32_t task_id = counter_ptr->acquire_counter();
|
||||
@@ -611,13 +656,30 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
|
||||
token_idx += gemm_m_tile_size) {
|
||||
const int32_t actual_token_num =
|
||||
std::min(gemm_m_tile_size, curr_token_num - token_idx);
|
||||
scalar_t* __restrict__ curr_w2_gemm_input_buffer =
|
||||
curr_w13_gemm_output_buffer;
|
||||
if constexpr (pack_a) {
|
||||
uint8_t* __restrict__ thread_buffer =
|
||||
thread_buffer_start + thread_id * w13_thread_buffer_offset;
|
||||
scalar_t* __restrict__ w2_input_buffer =
|
||||
reinterpret_cast<scalar_t*>(thread_buffer +
|
||||
w2_input_buffer_offset);
|
||||
curr_w2_gemm_input_buffer = w2_input_buffer;
|
||||
const scalar_t* w2_input_rows[gemm_m_tile_size];
|
||||
for (int32_t i = 0; i < actual_token_num; ++i) {
|
||||
w2_input_rows[i] = curr_w13_gemm_output_buffer + i * input_size_2;
|
||||
}
|
||||
gemm_t::pack_input_from_rows(w2_input_rows,
|
||||
curr_w2_gemm_input_buffer,
|
||||
actual_token_num, input_size_2);
|
||||
}
|
||||
|
||||
scalar_t* __restrict__ w2_weight_ptr_iter = w2_weight_ptr;
|
||||
scalar_t* __restrict__ w2_bias_ptr_iter = w2_bias_ptr;
|
||||
float* __restrict__ curr_w2_gemm_output_buffer_iter =
|
||||
curr_w2_gemm_output_buffer;
|
||||
for (int32_t i = 0; i < actual_n_tile_size; i += gemm_n_tile_size) {
|
||||
gemm.gemm(curr_w13_gemm_output_buffer, w2_weight_ptr_iter,
|
||||
gemm.gemm(curr_w2_gemm_input_buffer, w2_weight_ptr_iter,
|
||||
curr_w2_gemm_output_buffer_iter, actual_token_num,
|
||||
input_size_2, input_size_2, w2_n_group_stride,
|
||||
output_size_2, false);
|
||||
|
||||
@@ -497,6 +497,26 @@ struct FP32Vec16 : public VectorizedRegWrapper<FP32Vec16, 4, float> {
|
||||
reg.val[3] = Vectorized<float>(vcvt_f32_f16(vget_high_f16(v.reg.val[1])));
|
||||
};
|
||||
|
||||
static FORCE_INLINE void load_even_odd(const float* ptr, FP32Vec16& even,
|
||||
FP32Vec16& odd) noexcept {
|
||||
const float32x4x2_t x01 = vuzpq_f32(vld1q_f32(ptr), vld1q_f32(ptr + 4));
|
||||
const float32x4x2_t x23 =
|
||||
vuzpq_f32(vld1q_f32(ptr + 8), vld1q_f32(ptr + 12));
|
||||
const float32x4x2_t x45 =
|
||||
vuzpq_f32(vld1q_f32(ptr + 16), vld1q_f32(ptr + 20));
|
||||
const float32x4x2_t x67 =
|
||||
vuzpq_f32(vld1q_f32(ptr + 24), vld1q_f32(ptr + 28));
|
||||
|
||||
even.reg.val[0] = VectorizedT(x01.val[0]);
|
||||
even.reg.val[1] = VectorizedT(x23.val[0]);
|
||||
even.reg.val[2] = VectorizedT(x45.val[0]);
|
||||
even.reg.val[3] = VectorizedT(x67.val[0]);
|
||||
odd.reg.val[0] = VectorizedT(x01.val[1]);
|
||||
odd.reg.val[1] = VectorizedT(x23.val[1]);
|
||||
odd.reg.val[2] = VectorizedT(x45.val[1]);
|
||||
odd.reg.val[3] = VectorizedT(x67.val[1]);
|
||||
}
|
||||
|
||||
FORCE_INLINE FP32Vec16 operator+(const FP32Vec16& b) const noexcept {
|
||||
FP32Vec16 r(uninit);
|
||||
r.reg.val[0] = reg.val[0] + b.reg.val[0];
|
||||
@@ -515,6 +535,15 @@ struct FP32Vec16 : public VectorizedRegWrapper<FP32Vec16, 4, float> {
|
||||
return r;
|
||||
}
|
||||
|
||||
FORCE_INLINE FP32Vec16 operator-() const noexcept {
|
||||
FP32Vec16 r(uninit);
|
||||
r.reg.val[0] = reg.val[0].neg();
|
||||
r.reg.val[1] = reg.val[1].neg();
|
||||
r.reg.val[2] = reg.val[2].neg();
|
||||
r.reg.val[3] = reg.val[3].neg();
|
||||
return r;
|
||||
}
|
||||
|
||||
FORCE_INLINE FP32Vec16 operator*(const FP32Vec16& b) const noexcept {
|
||||
FP32Vec16 r(uninit);
|
||||
r.reg.val[0] = reg.val[0] * b.reg.val[0];
|
||||
@@ -933,4 +962,4 @@ inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16* ptr) {
|
||||
|
||||
inline void prefetch(const void* addr) { __builtin_prefetch(addr, 0, 1); };
|
||||
|
||||
}; // namespace vec_op
|
||||
}; // namespace vec_op
|
||||
|
||||
@@ -363,6 +363,13 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
|
||||
return FP32Vec16(ret);
|
||||
}
|
||||
|
||||
FP32Vec16 tanh() const {
|
||||
f32x16_t ret;
|
||||
unroll_loop<int, VEC_ELEM_NUM>(
|
||||
[&ret, this](int i) { ret.val[i] = std::tanh(reg.val[i]); });
|
||||
return FP32Vec16(ret);
|
||||
}
|
||||
|
||||
float reduce_sum() const {
|
||||
float result = 0.0f;
|
||||
unroll_loop<int, VEC_ELEM_NUM>(
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
#define CPU_TYPES_X86_HPP
|
||||
|
||||
#include <immintrin.h>
|
||||
#include <sleef.h>
|
||||
#include <torch/all.h>
|
||||
|
||||
#ifndef __AVX2__
|
||||
@@ -592,6 +593,8 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
|
||||
|
||||
FP32Vec16 abs() const { return FP32Vec16(_mm512_abs_ps(reg)); }
|
||||
|
||||
FP32Vec16 tanh() const { return FP32Vec16(Sleef_tanhf16_u10(reg)); }
|
||||
|
||||
float reduce_sum() const { return _mm512_reduce_add_ps(reg); }
|
||||
|
||||
float reduce_max() const { return _mm512_reduce_max_ps(reg); }
|
||||
@@ -789,6 +792,12 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
|
||||
_mm256_andnot_ps(sign_mask, reg_high));
|
||||
}
|
||||
|
||||
FP32Vec16 tanh() const {
|
||||
FP32Vec8 low(reg_low);
|
||||
FP32Vec8 high(reg_high);
|
||||
return FP32Vec16(low.tanh().reg, high.tanh().reg);
|
||||
}
|
||||
|
||||
FP32Vec16 min(const FP32Vec16& b) const {
|
||||
return FP32Vec16(_mm256_min_ps(reg_low, b.reg_low),
|
||||
_mm256_min_ps(reg_high, b.reg_high));
|
||||
|
||||
@@ -213,6 +213,8 @@ class MicroGemm<cpu_utils::ISA::AMX, scalar_t> {
|
||||
public:
|
||||
static constexpr int32_t MaxMSize = 32;
|
||||
static constexpr int32_t NSize = 32;
|
||||
static constexpr int32_t WeightOCGroupSize = 16;
|
||||
static constexpr bool PackA = false;
|
||||
|
||||
public:
|
||||
MicroGemm() : curr_m_(-1) {
|
||||
|
||||
@@ -21,6 +21,9 @@ class MicroGemm {
|
||||
public:
|
||||
static constexpr int32_t MaxMSize = 16;
|
||||
static constexpr int32_t NSize = 16;
|
||||
static constexpr int32_t WeightOCGroupSize = 16;
|
||||
// callers must pack A matrix before GEMM
|
||||
static constexpr bool PackA = false;
|
||||
|
||||
public:
|
||||
void gemm(DEFINE_CPU_MICRO_GEMM_PARAMS) {
|
||||
|
||||
@@ -0,0 +1,503 @@
|
||||
#ifndef CPU_MICRO_GEMM_NEON_HPP
|
||||
#define CPU_MICRO_GEMM_NEON_HPP
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
|
||||
#include "cpu/micro_gemm/cpu_micro_gemm_impl.hpp"
|
||||
|
||||
#include <arm_bf16.h>
|
||||
#include <arm_neon.h>
|
||||
|
||||
namespace cpu_micro_gemm {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr int32_t K = 4;
|
||||
constexpr int32_t Cols = 2;
|
||||
constexpr int32_t TileSize = K * Cols;
|
||||
constexpr int32_t Mr = 8;
|
||||
constexpr int32_t Nr = 8;
|
||||
constexpr int32_t Nr_gemv = 16;
|
||||
|
||||
// a = [a0, a1, a2, a3], b = [b0, b1, b2, b3] -> [a0, a1, b0, b1]
|
||||
FORCE_INLINE float32x4_t zip1_f32x4(const float32x4_t a, const float32x4_t b) {
|
||||
return vreinterpretq_f32_f64(
|
||||
vzip1q_f64(vreinterpretq_f64_f32(a), vreinterpretq_f64_f32(b)));
|
||||
}
|
||||
|
||||
// a = [a0, a1, a2, a3], b = [b0, b1, b2, b3] -> [a2, a3, b2, b3]
|
||||
FORCE_INLINE float32x4_t zip2_f32x4(const float32x4_t a, const float32x4_t b) {
|
||||
return vreinterpretq_f32_f64(
|
||||
vzip2q_f64(vreinterpretq_f64_f32(a), vreinterpretq_f64_f32(b)));
|
||||
}
|
||||
|
||||
FORCE_INLINE void init_acc_rowpair(float32x4_t& acc01, float32x4_t& acc23,
|
||||
float32x4_t& acc45, float32x4_t& acc67,
|
||||
const float* __restrict__ c_ptr,
|
||||
const int64_t ldc, const int32_t m_rows,
|
||||
const bool accum_c) {
|
||||
if (!accum_c || m_rows == 0) {
|
||||
acc01 = vdupq_n_f32(0.0f);
|
||||
acc23 = vdupq_n_f32(0.0f);
|
||||
acc45 = vdupq_n_f32(0.0f);
|
||||
acc67 = vdupq_n_f32(0.0f);
|
||||
return;
|
||||
}
|
||||
|
||||
const float32x4_t row0_0123 = vld1q_f32(c_ptr);
|
||||
const float32x4_t row0_4567 = vld1q_f32(c_ptr + 4);
|
||||
const float32x4_t row1_0123 =
|
||||
(m_rows == 2) ? vld1q_f32(c_ptr + ldc) : vdupq_n_f32(0.0f);
|
||||
const float32x4_t row1_4567 =
|
||||
(m_rows == 2) ? vld1q_f32(c_ptr + ldc + 4) : vdupq_n_f32(0.0f);
|
||||
|
||||
acc01 = zip1_f32x4(row0_0123, row1_0123);
|
||||
acc23 = zip2_f32x4(row0_0123, row1_0123);
|
||||
acc45 = zip1_f32x4(row0_4567, row1_4567);
|
||||
acc67 = zip2_f32x4(row0_4567, row1_4567);
|
||||
}
|
||||
|
||||
FORCE_INLINE void store_acc_rowpair(const float32x4_t acc01,
|
||||
const float32x4_t acc23,
|
||||
const float32x4_t acc45,
|
||||
const float32x4_t acc67,
|
||||
float* __restrict__ c_ptr,
|
||||
const int64_t ldc, const int32_t m_rows) {
|
||||
if (m_rows == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
vst1q_f32(c_ptr, zip1_f32x4(acc01, acc23));
|
||||
vst1q_f32(c_ptr + 4, zip1_f32x4(acc45, acc67));
|
||||
|
||||
if (m_rows == 2) {
|
||||
vst1q_f32(c_ptr + ldc, zip2_f32x4(acc01, acc23));
|
||||
vst1q_f32(c_ptr + ldc + 4, zip2_f32x4(acc45, acc67));
|
||||
}
|
||||
}
|
||||
|
||||
FORCE_INLINE void gemm_micro_bfmmla_8x8_packed_a(
|
||||
const bfloat16_t* __restrict__ a_packed,
|
||||
const bfloat16_t* __restrict__ b_packed, float* __restrict__ c_ptr,
|
||||
const int32_t m, const int32_t k_size, const int64_t ldc,
|
||||
const bool accum_c) {
|
||||
float32x4_t acc0101, acc0123, acc0145, acc0167;
|
||||
float32x4_t acc2301, acc2323, acc2345, acc2367;
|
||||
float32x4_t acc4501, acc4523, acc4545, acc4567;
|
||||
float32x4_t acc6701, acc6723, acc6745, acc6767;
|
||||
|
||||
init_acc_rowpair(acc0101, acc0123, acc0145, acc0167, c_ptr, ldc,
|
||||
std::min(2, m), accum_c);
|
||||
init_acc_rowpair(acc2301, acc2323, acc2345, acc2367, c_ptr + 2 * ldc, ldc,
|
||||
std::min(2, std::max(0, m - 2)), accum_c);
|
||||
init_acc_rowpair(acc4501, acc4523, acc4545, acc4567, c_ptr + 4 * ldc, ldc,
|
||||
std::min(2, std::max(0, m - 4)), accum_c);
|
||||
init_acc_rowpair(acc6701, acc6723, acc6745, acc6767, c_ptr + 6 * ldc, ldc,
|
||||
std::min(2, std::max(0, m - 6)), accum_c);
|
||||
|
||||
const bfloat16_t* __restrict__ a_tile = a_packed;
|
||||
const bfloat16_t* __restrict__ b_tile = b_packed;
|
||||
|
||||
#pragma GCC unroll 8
|
||||
for (int32_t k_idx = 0; k_idx < k_size; k_idx += K) {
|
||||
const bfloat16x8_t a_tile01 = vld1q_bf16(a_tile);
|
||||
const bfloat16x8_t a_tile23 = vld1q_bf16(a_tile + TileSize);
|
||||
const bfloat16x8_t a_tile45 = vld1q_bf16(a_tile + 2 * TileSize);
|
||||
const bfloat16x8_t a_tile67 = vld1q_bf16(a_tile + 3 * TileSize);
|
||||
|
||||
const bfloat16x8_t b_tile01 = vld1q_bf16(b_tile);
|
||||
const bfloat16x8_t b_tile23 = vld1q_bf16(b_tile + TileSize);
|
||||
const bfloat16x8_t b_tile45 = vld1q_bf16(b_tile + 2 * TileSize);
|
||||
const bfloat16x8_t b_tile67 = vld1q_bf16(b_tile + 3 * TileSize);
|
||||
|
||||
acc0101 = vbfmmlaq_f32(acc0101, a_tile01, b_tile01);
|
||||
acc2301 = vbfmmlaq_f32(acc2301, a_tile23, b_tile01);
|
||||
acc4501 = vbfmmlaq_f32(acc4501, a_tile45, b_tile01);
|
||||
acc6701 = vbfmmlaq_f32(acc6701, a_tile67, b_tile01);
|
||||
|
||||
acc0123 = vbfmmlaq_f32(acc0123, a_tile01, b_tile23);
|
||||
acc2323 = vbfmmlaq_f32(acc2323, a_tile23, b_tile23);
|
||||
acc4523 = vbfmmlaq_f32(acc4523, a_tile45, b_tile23);
|
||||
acc6723 = vbfmmlaq_f32(acc6723, a_tile67, b_tile23);
|
||||
|
||||
acc0145 = vbfmmlaq_f32(acc0145, a_tile01, b_tile45);
|
||||
acc2345 = vbfmmlaq_f32(acc2345, a_tile23, b_tile45);
|
||||
acc4545 = vbfmmlaq_f32(acc4545, a_tile45, b_tile45);
|
||||
acc6745 = vbfmmlaq_f32(acc6745, a_tile67, b_tile45);
|
||||
|
||||
acc0167 = vbfmmlaq_f32(acc0167, a_tile01, b_tile67);
|
||||
acc2367 = vbfmmlaq_f32(acc2367, a_tile23, b_tile67);
|
||||
acc4567 = vbfmmlaq_f32(acc4567, a_tile45, b_tile67);
|
||||
acc6767 = vbfmmlaq_f32(acc6767, a_tile67, b_tile67);
|
||||
|
||||
a_tile += 4 * TileSize;
|
||||
b_tile += Nr * K;
|
||||
}
|
||||
|
||||
store_acc_rowpair(acc0101, acc0123, acc0145, acc0167, c_ptr, ldc,
|
||||
std::min(2, m));
|
||||
store_acc_rowpair(acc2301, acc2323, acc2345, acc2367, c_ptr + 2 * ldc, ldc,
|
||||
std::min(2, std::max(0, m - 2)));
|
||||
store_acc_rowpair(acc4501, acc4523, acc4545, acc4567, c_ptr + 4 * ldc, ldc,
|
||||
std::min(2, std::max(0, m - 4)));
|
||||
store_acc_rowpair(acc6701, acc6723, acc6745, acc6767, c_ptr + 6 * ldc, ldc,
|
||||
std::min(2, std::max(0, m - 6)));
|
||||
}
|
||||
|
||||
FORCE_INLINE void gemm_micro_bfmmla_4x16_packed_a(
|
||||
const bfloat16_t* __restrict__ a_packed,
|
||||
const bfloat16_t* __restrict__ b_packed, float* __restrict__ c_ptr,
|
||||
const int32_t m, const int32_t k_size, const int64_t b_n_group_stride,
|
||||
const int64_t ldc, const bool accum_c) {
|
||||
const int32_t m_rows_01 = std::min(2, m);
|
||||
const int32_t m_rows_23 = std::min(2, std::max(0, m - 2));
|
||||
|
||||
float32x4_t acc0101, acc0123, acc0145, acc0167;
|
||||
float32x4_t acc2301, acc2323, acc2345, acc2367;
|
||||
float32x4_t acc0189, acc011011, acc011213, acc011415;
|
||||
float32x4_t acc2389, acc231011, acc231213, acc231415;
|
||||
|
||||
init_acc_rowpair(acc0101, acc0123, acc0145, acc0167, c_ptr, ldc, m_rows_01,
|
||||
accum_c);
|
||||
init_acc_rowpair(acc2301, acc2323, acc2345, acc2367, c_ptr + 2 * ldc, ldc,
|
||||
m_rows_23, accum_c);
|
||||
init_acc_rowpair(acc0189, acc011011, acc011213, acc011415, c_ptr + 8, ldc,
|
||||
m_rows_01, accum_c);
|
||||
init_acc_rowpair(acc2389, acc231011, acc231213, acc231415,
|
||||
c_ptr + 2 * ldc + 8, ldc, m_rows_23, accum_c);
|
||||
|
||||
const bfloat16_t* __restrict__ a_tile = a_packed;
|
||||
const bfloat16_t* __restrict__ b_tile0 = b_packed;
|
||||
const bfloat16_t* __restrict__ b_tile1 = b_packed + b_n_group_stride;
|
||||
|
||||
#pragma GCC unroll 8
|
||||
for (int32_t k_idx = 0; k_idx < k_size; k_idx += K) {
|
||||
const bfloat16x8_t a_tile01 = vld1q_bf16(a_tile);
|
||||
const bfloat16x8_t a_tile23 = vld1q_bf16(a_tile + TileSize);
|
||||
const bfloat16x8_t b_tile01 = vld1q_bf16(b_tile0);
|
||||
const bfloat16x8_t b_tile23 = vld1q_bf16(b_tile0 + TileSize);
|
||||
const bfloat16x8_t b_tile45 = vld1q_bf16(b_tile0 + 2 * TileSize);
|
||||
const bfloat16x8_t b_tile67 = vld1q_bf16(b_tile0 + 3 * TileSize);
|
||||
const bfloat16x8_t b_tile89 = vld1q_bf16(b_tile1);
|
||||
const bfloat16x8_t b_tile1011 = vld1q_bf16(b_tile1 + TileSize);
|
||||
const bfloat16x8_t b_tile1213 = vld1q_bf16(b_tile1 + 2 * TileSize);
|
||||
const bfloat16x8_t b_tile1415 = vld1q_bf16(b_tile1 + 3 * TileSize);
|
||||
|
||||
acc0101 = vbfmmlaq_f32(acc0101, a_tile01, b_tile01);
|
||||
acc2301 = vbfmmlaq_f32(acc2301, a_tile23, b_tile01);
|
||||
acc0123 = vbfmmlaq_f32(acc0123, a_tile01, b_tile23);
|
||||
acc2323 = vbfmmlaq_f32(acc2323, a_tile23, b_tile23);
|
||||
|
||||
acc0145 = vbfmmlaq_f32(acc0145, a_tile01, b_tile45);
|
||||
acc2345 = vbfmmlaq_f32(acc2345, a_tile23, b_tile45);
|
||||
acc0167 = vbfmmlaq_f32(acc0167, a_tile01, b_tile67);
|
||||
acc2367 = vbfmmlaq_f32(acc2367, a_tile23, b_tile67);
|
||||
|
||||
acc0189 = vbfmmlaq_f32(acc0189, a_tile01, b_tile89);
|
||||
acc2389 = vbfmmlaq_f32(acc2389, a_tile23, b_tile89);
|
||||
acc011011 = vbfmmlaq_f32(acc011011, a_tile01, b_tile1011);
|
||||
acc231011 = vbfmmlaq_f32(acc231011, a_tile23, b_tile1011);
|
||||
|
||||
acc011213 = vbfmmlaq_f32(acc011213, a_tile01, b_tile1213);
|
||||
acc231213 = vbfmmlaq_f32(acc231213, a_tile23, b_tile1213);
|
||||
acc011415 = vbfmmlaq_f32(acc011415, a_tile01, b_tile1415);
|
||||
acc231415 = vbfmmlaq_f32(acc231415, a_tile23, b_tile1415);
|
||||
|
||||
a_tile += 2 * TileSize;
|
||||
b_tile0 += Nr * K;
|
||||
b_tile1 += Nr * K;
|
||||
}
|
||||
|
||||
store_acc_rowpair(acc0101, acc0123, acc0145, acc0167, c_ptr, ldc, m_rows_01);
|
||||
store_acc_rowpair(acc2301, acc2323, acc2345, acc2367, c_ptr + 2 * ldc, ldc,
|
||||
m_rows_23);
|
||||
store_acc_rowpair(acc0189, acc011011, acc011213, acc011415, c_ptr + 8, ldc,
|
||||
m_rows_01);
|
||||
store_acc_rowpair(acc2389, acc231011, acc231213, acc231415,
|
||||
c_ptr + 2 * ldc + 8, ldc, m_rows_23);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
template <typename scalar_t>
|
||||
class MicroGemm<cpu_utils::ISA::NEON, scalar_t> {
|
||||
public:
|
||||
static constexpr int32_t MaxMSize = 8;
|
||||
static constexpr int32_t NSize = 32;
|
||||
static constexpr int32_t WeightOCGroupSize = Nr;
|
||||
static constexpr bool PackA = false;
|
||||
|
||||
public:
|
||||
void gemm(DEFINE_CPU_MICRO_GEMM_PARAMS) {
|
||||
TORCH_CHECK(false, "NEON BFMMLA MicroGemm only supports bfloat16.");
|
||||
}
|
||||
|
||||
static void pack_weight(const scalar_t* __restrict__ /*weight*/,
|
||||
scalar_t* __restrict__ /*packed_weight*/,
|
||||
const int32_t /*output_size*/,
|
||||
const int32_t /*input_size*/) {
|
||||
TORCH_CHECK(false, "NEON BFMMLA MicroGemm only supports bfloat16.");
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
class MicroGemm<cpu_utils::ISA::NEON, c10::BFloat16> {
|
||||
public:
|
||||
using scalar_t = c10::BFloat16;
|
||||
|
||||
static constexpr int32_t MaxMSize = 8;
|
||||
static constexpr int32_t NSize = 32;
|
||||
static constexpr int32_t WeightOCGroupSize = Nr;
|
||||
static constexpr bool PackA = true;
|
||||
|
||||
public:
|
||||
// physical layout [
|
||||
// M / 8; Mr is 8
|
||||
// K / 4; K for bfmmla is 4
|
||||
// 4, ; 4 row-pairs for each 8 rows
|
||||
// 2, ; row-pair is 2 rows
|
||||
// 4 ; 4 elements per row
|
||||
// ]
|
||||
|
||||
static void pack_input_from_rows(const scalar_t* const* __restrict__ rows,
|
||||
scalar_t* __restrict__ a_packed,
|
||||
const int32_t m, const int32_t k) {
|
||||
TORCH_CHECK(m > 0 && m <= MaxMSize);
|
||||
TORCH_CHECK_EQ(k % K, 0);
|
||||
|
||||
auto* __restrict__ out = reinterpret_cast<bfloat16_t*>(a_packed);
|
||||
const bfloat16x8_t zero_q = vdupq_n_bf16(bfloat16_t{});
|
||||
const bfloat16x4_t zero = vget_low_bf16(zero_q);
|
||||
|
||||
for (int32_t row_base = 0; row_base < m; row_base += Mr) {
|
||||
const int32_t actual_m = std::min(Mr, m - row_base);
|
||||
const bfloat16_t* __restrict__ row[Mr];
|
||||
for (int32_t i = 0; i < actual_m; ++i) {
|
||||
row[i] = reinterpret_cast<const bfloat16_t*>(rows[row_base + i]);
|
||||
}
|
||||
|
||||
if (actual_m == 8) {
|
||||
int32_t k_idx = 0;
|
||||
for (; k_idx + 8 <= k; k_idx += 8) {
|
||||
bfloat16_t* __restrict__ block0 = out;
|
||||
bfloat16_t* __restrict__ block1 = out + 4 * TileSize;
|
||||
|
||||
bfloat16x8_t a0 = vld1q_bf16(row[0] + k_idx);
|
||||
bfloat16x8_t a1 = vld1q_bf16(row[1] + k_idx);
|
||||
vst1q_bf16(block0,
|
||||
vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
|
||||
vst1q_bf16(block1,
|
||||
vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
|
||||
|
||||
a0 = vld1q_bf16(row[2] + k_idx);
|
||||
a1 = vld1q_bf16(row[3] + k_idx);
|
||||
vst1q_bf16(block0 + TileSize,
|
||||
vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
|
||||
vst1q_bf16(block1 + TileSize,
|
||||
vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
|
||||
|
||||
a0 = vld1q_bf16(row[4] + k_idx);
|
||||
a1 = vld1q_bf16(row[5] + k_idx);
|
||||
vst1q_bf16(block0 + 2 * TileSize,
|
||||
vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
|
||||
vst1q_bf16(block1 + 2 * TileSize,
|
||||
vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
|
||||
|
||||
a0 = vld1q_bf16(row[6] + k_idx);
|
||||
a1 = vld1q_bf16(row[7] + k_idx);
|
||||
vst1q_bf16(block0 + 3 * TileSize,
|
||||
vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
|
||||
vst1q_bf16(block1 + 3 * TileSize,
|
||||
vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
|
||||
|
||||
out += 8 * TileSize;
|
||||
}
|
||||
|
||||
for (; k_idx < k; k_idx += K) {
|
||||
bfloat16x4_t a0 = vld1_bf16(row[0] + k_idx);
|
||||
bfloat16x4_t a1 = vld1_bf16(row[1] + k_idx);
|
||||
vst1q_bf16(out, vcombine_bf16(a0, a1));
|
||||
|
||||
a0 = vld1_bf16(row[2] + k_idx);
|
||||
a1 = vld1_bf16(row[3] + k_idx);
|
||||
vst1q_bf16(out + TileSize, vcombine_bf16(a0, a1));
|
||||
|
||||
a0 = vld1_bf16(row[4] + k_idx);
|
||||
a1 = vld1_bf16(row[5] + k_idx);
|
||||
vst1q_bf16(out + 2 * TileSize, vcombine_bf16(a0, a1));
|
||||
|
||||
a0 = vld1_bf16(row[6] + k_idx);
|
||||
a1 = vld1_bf16(row[7] + k_idx);
|
||||
vst1q_bf16(out + 3 * TileSize, vcombine_bf16(a0, a1));
|
||||
|
||||
out += 4 * TileSize;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (actual_m == 4) {
|
||||
int32_t k_idx = 0;
|
||||
for (; k_idx + 8 <= k; k_idx += 8) {
|
||||
bfloat16_t* __restrict__ block0 = out;
|
||||
bfloat16_t* __restrict__ block1 = out + 2 * TileSize;
|
||||
|
||||
bfloat16x8_t a0 = vld1q_bf16(row[0] + k_idx);
|
||||
bfloat16x8_t a1 = vld1q_bf16(row[1] + k_idx);
|
||||
vst1q_bf16(block0,
|
||||
vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
|
||||
vst1q_bf16(block1,
|
||||
vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
|
||||
|
||||
a0 = vld1q_bf16(row[2] + k_idx);
|
||||
a1 = vld1q_bf16(row[3] + k_idx);
|
||||
vst1q_bf16(block0 + TileSize,
|
||||
vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
|
||||
vst1q_bf16(block1 + TileSize,
|
||||
vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
|
||||
|
||||
out += 4 * TileSize;
|
||||
}
|
||||
|
||||
for (; k_idx < k; k_idx += K) {
|
||||
bfloat16x4_t a0 = vld1_bf16(row[0] + k_idx);
|
||||
bfloat16x4_t a1 = vld1_bf16(row[1] + k_idx);
|
||||
vst1q_bf16(out, vcombine_bf16(a0, a1));
|
||||
|
||||
a0 = vld1_bf16(row[2] + k_idx);
|
||||
a1 = vld1_bf16(row[3] + k_idx);
|
||||
vst1q_bf16(out + TileSize, vcombine_bf16(a0, a1));
|
||||
|
||||
out += 2 * TileSize;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const int32_t row_pair_count = (actual_m <= 4) ? 2 : Mr / 2;
|
||||
|
||||
int32_t k_idx = 0;
|
||||
for (; k_idx + 8 <= k; k_idx += 8) {
|
||||
bfloat16_t* __restrict__ block0 = out;
|
||||
bfloat16_t* __restrict__ block1 = out + row_pair_count * TileSize;
|
||||
|
||||
bfloat16x8_t a0 = vld1q_bf16(row[0] + k_idx);
|
||||
bfloat16x8_t a1 = (actual_m > 1) ? vld1q_bf16(row[1] + k_idx) : zero_q;
|
||||
vst1q_bf16(block0, vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
|
||||
vst1q_bf16(block1,
|
||||
vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
|
||||
|
||||
a0 = (actual_m > 2) ? vld1q_bf16(row[2] + k_idx) : zero_q;
|
||||
a1 = (actual_m > 3) ? vld1q_bf16(row[3] + k_idx) : zero_q;
|
||||
vst1q_bf16(block0 + TileSize,
|
||||
vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
|
||||
vst1q_bf16(block1 + TileSize,
|
||||
vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
|
||||
|
||||
if (actual_m > 4) {
|
||||
a0 = vld1q_bf16(row[4] + k_idx);
|
||||
a1 = (actual_m > 5) ? vld1q_bf16(row[5] + k_idx) : zero_q;
|
||||
vst1q_bf16(block0 + 2 * TileSize,
|
||||
vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
|
||||
vst1q_bf16(block1 + 2 * TileSize,
|
||||
vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
|
||||
|
||||
a0 = (actual_m > 6) ? vld1q_bf16(row[6] + k_idx) : zero_q;
|
||||
a1 = (actual_m > 7) ? vld1q_bf16(row[7] + k_idx) : zero_q;
|
||||
vst1q_bf16(block0 + 3 * TileSize,
|
||||
vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
|
||||
vst1q_bf16(block1 + 3 * TileSize,
|
||||
vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
|
||||
}
|
||||
|
||||
out += 2 * row_pair_count * TileSize;
|
||||
}
|
||||
|
||||
for (; k_idx < k; k_idx += K) {
|
||||
bfloat16x4_t a0 = vld1_bf16(row[0] + k_idx);
|
||||
bfloat16x4_t a1 = (actual_m > 1) ? vld1_bf16(row[1] + k_idx) : zero;
|
||||
vst1q_bf16(out, vcombine_bf16(a0, a1));
|
||||
|
||||
a0 = (actual_m > 2) ? vld1_bf16(row[2] + k_idx) : zero;
|
||||
a1 = (actual_m > 3) ? vld1_bf16(row[3] + k_idx) : zero;
|
||||
vst1q_bf16(out + TileSize, vcombine_bf16(a0, a1));
|
||||
|
||||
if (actual_m > 4) {
|
||||
a0 = vld1_bf16(row[4] + k_idx);
|
||||
a1 = (actual_m > 5) ? vld1_bf16(row[5] + k_idx) : zero;
|
||||
vst1q_bf16(out + 2 * TileSize, vcombine_bf16(a0, a1));
|
||||
|
||||
a0 = (actual_m > 6) ? vld1_bf16(row[6] + k_idx) : zero;
|
||||
a1 = (actual_m > 7) ? vld1_bf16(row[7] + k_idx) : zero;
|
||||
vst1q_bf16(out + 3 * TileSize, vcombine_bf16(a0, a1));
|
||||
}
|
||||
out += row_pair_count * TileSize;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void gemm(DEFINE_CPU_MICRO_GEMM_PARAMS) {
|
||||
(void)lda; // A is packed, so lda is not needed
|
||||
TORCH_CHECK_EQ(k % K, 0);
|
||||
|
||||
for (int32_t n_idx = 0; n_idx < NSize; n_idx += Nr_gemv) {
|
||||
const bfloat16_t* __restrict__ b_panel =
|
||||
reinterpret_cast<const bfloat16_t*>(b_ptr) + n_idx * k;
|
||||
|
||||
for (int32_t row_base = 0; row_base < m; row_base += Mr) {
|
||||
const int32_t panel_m = std::min(Mr, m - row_base);
|
||||
const bfloat16_t* __restrict__ a_panel =
|
||||
reinterpret_cast<const bfloat16_t*>(a_ptr) + row_base * k;
|
||||
float* __restrict__ c_panel = c_ptr + row_base * ldc + n_idx;
|
||||
|
||||
if (panel_m <= 4) {
|
||||
gemm_micro_bfmmla_4x16_packed_a(a_panel, b_panel, c_panel, panel_m, k,
|
||||
b_n_group_stride, ldc, accum_c);
|
||||
} else {
|
||||
gemm_micro_bfmmla_8x8_packed_a(a_panel, b_panel, c_panel, panel_m, k,
|
||||
ldc, accum_c);
|
||||
gemm_micro_bfmmla_8x8_packed_a(a_panel, b_panel + b_n_group_stride,
|
||||
c_panel + Nr, panel_m, k, ldc,
|
||||
accum_c);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// physical layout [
|
||||
// N / 8; Nr is 8
|
||||
// K / 4; K for bfmmla is 4
|
||||
// 4, ; 4 col-pairs for each 8 cols
|
||||
// 2, ; col-pair is 2 cols
|
||||
// 4 ; 4 elements per col
|
||||
// ]
|
||||
static void pack_weight(const c10::BFloat16* __restrict__ weight,
|
||||
c10::BFloat16* __restrict__ packed_weight,
|
||||
const int32_t output_size, const int32_t input_size) {
|
||||
TORCH_CHECK_EQ(output_size % NSize, 0);
|
||||
TORCH_CHECK_EQ(input_size % K, 0);
|
||||
|
||||
for (int32_t o_idx = 0; o_idx < output_size; o_idx += Nr) {
|
||||
c10::BFloat16* __restrict__ dst = packed_weight + o_idx * input_size;
|
||||
for (int32_t k_idx = 0; k_idx < input_size; k_idx += K) {
|
||||
for (int32_t pair_idx = 0; pair_idx < Nr; pair_idx += Cols) {
|
||||
const c10::BFloat16* __restrict__ row0 =
|
||||
weight + (o_idx + pair_idx) * input_size;
|
||||
const c10::BFloat16* __restrict__ row1 = row0 + input_size;
|
||||
dst[0] = row0[k_idx + 0];
|
||||
dst[1] = row0[k_idx + 1];
|
||||
dst[2] = row0[k_idx + 2];
|
||||
dst[3] = row0[k_idx + 3];
|
||||
dst[4] = row1[k_idx + 0];
|
||||
dst[5] = row1[k_idx + 1];
|
||||
dst[6] = row1[k_idx + 2];
|
||||
dst[7] = row1[k_idx + 3];
|
||||
dst += TileSize;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace cpu_micro_gemm
|
||||
|
||||
#endif
|
||||
@@ -104,6 +104,8 @@ class MicroGemm<cpu_utils::ISA::VEC, scalar_t> {
|
||||
public:
|
||||
static constexpr int32_t MaxMSize = 8;
|
||||
static constexpr int32_t NSize = 32;
|
||||
static constexpr int32_t WeightOCGroupSize = 16;
|
||||
static constexpr bool PackA = false;
|
||||
|
||||
public:
|
||||
void gemm(DEFINE_CPU_MICRO_GEMM_PARAMS) {
|
||||
|
||||
@@ -538,7 +538,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
|
||||
#endif
|
||||
|
||||
// fused moe
|
||||
#if defined(__AVX512F__)
|
||||
#if defined(__AVX512F__) || (defined(ARM_BF16_SUPPORT))
|
||||
ops.def(
|
||||
"prepack_moe_weight(Tensor weight, Tensor(a1!) packed_weight, str isa) "
|
||||
"-> ()");
|
||||
@@ -549,7 +549,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
|
||||
"bool skip_weighted, "
|
||||
"str act, str isa) -> ()");
|
||||
ops.impl("cpu_fused_moe", torch::kCPU, &cpu_fused_moe);
|
||||
#endif
|
||||
#endif // #if defined(__AVX512F__) || (defined(ARM_BF16_SUPPORT))
|
||||
ops.def(
|
||||
"mla_decode_kvcache("
|
||||
" Tensor! out, Tensor query, Tensor kv_cache,"
|
||||
|
||||
+4
-1
@@ -2,13 +2,14 @@
|
||||
#define UTILS_HPP
|
||||
|
||||
#include <atomic>
|
||||
#include <string>
|
||||
#include <unistd.h>
|
||||
#include <ATen/cpu/Utils.h>
|
||||
|
||||
#include "cpu/cpu_types.hpp"
|
||||
|
||||
namespace cpu_utils {
|
||||
enum class ISA { AMX, VEC, RVV };
|
||||
enum class ISA { AMX, VEC, RVV, NEON };
|
||||
|
||||
inline ISA get_isa(const std::string& isa) {
|
||||
if (isa == "amx") {
|
||||
@@ -17,6 +18,8 @@ inline ISA get_isa(const std::string& isa) {
|
||||
return ISA::VEC;
|
||||
} else if (isa == "rvv") {
|
||||
return ISA::RVV;
|
||||
} else if (isa == "neon") {
|
||||
return ISA::NEON;
|
||||
} else {
|
||||
TORCH_CHECK(false, "Invalid isa type: " + isa);
|
||||
}
|
||||
|
||||
@@ -5,10 +5,13 @@ import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.allclose_default import get_default_atol, get_default_rtol
|
||||
from vllm._custom_ops import cpu_fused_moe, cpu_prepack_moe_weight
|
||||
from vllm._custom_ops import (
|
||||
cpu_fused_moe,
|
||||
cpu_prepack_moe_weight,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.cpu_fused_moe import _CPU_MOE_ACT_FN
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
if not current_platform.is_cpu():
|
||||
@@ -26,8 +29,13 @@ ACT = [
|
||||
MoEActivation.GELU,
|
||||
MoEActivation.GELU_TANH,
|
||||
]
|
||||
USE_BIAS = [True, False]
|
||||
ISA = ["amx", "vec"] if torch.cpu._is_amx_tile_supported() else ["vec"]
|
||||
USE_BIAS = [False, True]
|
||||
ISA = ["vec"]
|
||||
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
|
||||
ISA.append("neon")
|
||||
if torch.cpu._is_amx_tile_supported():
|
||||
ISA.append("amx")
|
||||
|
||||
DTYPE = [torch.bfloat16]
|
||||
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ from vllm._custom_ops import (
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.quantization.utils.layer_utils import replace_parameter
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
_CPU_MOE_LAYER_CACHE = {}
|
||||
@@ -312,6 +313,17 @@ class CPUFusedMOE:
|
||||
if supports_amx:
|
||||
return False, "none"
|
||||
|
||||
supports_neon = current_platform.get_cpu_architecture() == CpuArchEnum.ARM
|
||||
if supports_neon:
|
||||
if (
|
||||
dtype == torch.bfloat16
|
||||
and w13_input_size % 4 == 0
|
||||
and w2_input_size % 4 == 0
|
||||
):
|
||||
return True, "neon"
|
||||
else:
|
||||
return False, "none"
|
||||
|
||||
return True, "vec"
|
||||
|
||||
def init_moe_grouped_gemm(
|
||||
|
||||
Reference in New Issue
Block a user