[CPU][Perf] INT8 Fused MoE Kernel for Arm CPUs (#48637)
Signed-off-by: Fadi Arafeh <fadi.arafeh@arm.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
co-authored by
Li, Jiang <jiang1.li@intel.com>
parent
f19ee27e39
commit
c314af1abf
@@ -207,6 +207,20 @@ void cpu_fused_moe(torch::Tensor& output, const torch::Tensor& input,
|
||||
const torch::Tensor& topk_id, const bool skip_weighted,
|
||||
const std::string& act, const std::string& isa);
|
||||
|
||||
void prepack_moe_weight_int8(const torch::Tensor& weight,
|
||||
torch::Tensor& packed_weight,
|
||||
const std::string& isa);
|
||||
|
||||
void cpu_fused_moe_int8(torch::Tensor& output, const torch::Tensor& input,
|
||||
const torch::Tensor& w13, const torch::Tensor& w2,
|
||||
const torch::Tensor& w13_scale,
|
||||
const torch::Tensor& w2_scale,
|
||||
const std::optional<torch::Tensor>& w13_bias,
|
||||
const std::optional<torch::Tensor>& w2_bias,
|
||||
const torch::Tensor& topk_weights,
|
||||
const torch::Tensor& topk_id, const bool skip_weighted,
|
||||
const std::string& act, const std::string& isa);
|
||||
|
||||
void compute_slot_mapping_kernel_impl(const torch::Tensor query_start_loc,
|
||||
const torch::Tensor positions,
|
||||
const torch::Tensor block_table,
|
||||
@@ -596,8 +610,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
|
||||
#endif
|
||||
|
||||
// fused moe
|
||||
#if defined(__AVX512F__) || \
|
||||
(defined(__aarch64__) && !defined(__APPLE__) && defined(ARM_BF16_SUPPORT))
|
||||
#if defined(__AVX512F__) || (defined(ARM_BF16_SUPPORT) && !defined(__APPLE__))
|
||||
ops.def(
|
||||
"prepack_moe_weight(Tensor weight, Tensor(a1!) packed_weight, str isa) "
|
||||
"-> ()");
|
||||
@@ -608,7 +621,22 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
|
||||
"bool skip_weighted, "
|
||||
"str act, str isa) -> ()");
|
||||
ops.impl("cpu_fused_moe", torch::kCPU, &cpu_fused_moe);
|
||||
#endif
|
||||
#endif // #if defined(__AVX512F__) || (defined(ARM_BF16_SUPPORT) &&
|
||||
// !defined(__APPLE__))
|
||||
#if defined(ARM_I8MM_SUPPORT) && defined(ARM_BF16_SUPPORT) && \
|
||||
!defined(__APPLE__)
|
||||
ops.def(
|
||||
"prepack_moe_weight_int8(Tensor weight, Tensor(a1!) packed_weight, "
|
||||
"str isa) -> ()");
|
||||
ops.impl("prepack_moe_weight_int8", torch::kCPU, &prepack_moe_weight_int8);
|
||||
ops.def(
|
||||
"cpu_fused_moe_int8(Tensor(a0!) output, Tensor input, Tensor w13, "
|
||||
"Tensor w2, Tensor w13_scale, Tensor w2_scale, Tensor? w13_bias, "
|
||||
"Tensor? w2_bias, Tensor topk_weights, Tensor topk_id, bool "
|
||||
"skip_weighted, str act, str isa) -> ()");
|
||||
ops.impl("cpu_fused_moe_int8", torch::kCPU, &cpu_fused_moe_int8);
|
||||
#endif // #if defined(ARM_I8MM_SUPPORT) && defined(ARM_BF16_SUPPORT) &&
|
||||
// !defined(__APPLE__)
|
||||
ops.def(
|
||||
"mla_decode_kvcache("
|
||||
" Tensor! out, Tensor query, Tensor kv_cache,"
|
||||
|
||||
Reference in New Issue
Block a user