[CPU][Perf] INT8 Fused MoE Kernel for Arm CPUs (#48637)
Signed-off-by: Fadi Arafeh <fadi.arafeh@arm.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
co-authored by
Li, Jiang <jiang1.li@intel.com>
parent
f19ee27e39
commit
c314af1abf
@@ -15,6 +15,7 @@ endif()
|
||||
#
|
||||
set(ENABLE_X86_ISA $ENV{VLLM_CPU_X86})
|
||||
set(ENABLE_ARM_BF16 $ENV{VLLM_CPU_ARM_BF16})
|
||||
set(ENABLE_ARM_I8MM $ENV{VLLM_CPU_ARM_I8MM})
|
||||
set(ENABLE_RVV_BF16 $ENV{VLLM_CPU_RVV_BF16})
|
||||
|
||||
include_directories("${CMAKE_SOURCE_DIR}/csrc")
|
||||
@@ -96,12 +97,14 @@ if (MACOSX_FOUND AND CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64")
|
||||
set(ENABLE_NUMA OFF)
|
||||
check_sysctl(hw.optional.neon ASIMD_FOUND)
|
||||
check_sysctl(hw.optional.arm.FEAT_BF16 ARM_BF16_FOUND)
|
||||
check_sysctl(hw.optional.arm.FEAT_I8MM ARM_I8MM_FOUND)
|
||||
else()
|
||||
find_isa(${CPUINFO} "Power11" POWER11_FOUND)
|
||||
find_isa(${CPUINFO} "POWER10" POWER10_FOUND)
|
||||
find_isa(${CPUINFO} "POWER9" POWER9_FOUND)
|
||||
find_isa(${CPUINFO} "asimd" ASIMD_FOUND) # Check for ARM NEON support
|
||||
find_isa(${CPUINFO} "bf16" ARM_BF16_FOUND) # Check for ARM BF16 support
|
||||
find_isa(${CPUINFO} "i8mm" ARM_I8MM_FOUND) # Check for ARM I8MM support
|
||||
find_isa(${CPUINFO} "S390" S390_FOUND)
|
||||
find_isa(${CPUINFO} "zvfhmin" RVV_FP16_FOUND) # Check for RISC-V Vector FP16 support
|
||||
find_isa(${CPUINFO} "zvfbfmin" RVV_BF16_FOUND) # Check for RISC-V Vector BF16 support
|
||||
@@ -111,6 +114,11 @@ else()
|
||||
set(ARM_BF16_FOUND ON)
|
||||
message(STATUS "ARM BF16 support enabled via VLLM_CPU_ARM_BF16 environment variable")
|
||||
endif()
|
||||
if (ENABLE_ARM_I8MM)
|
||||
set(ARM_I8MM_FOUND ON)
|
||||
message(STATUS
|
||||
"ARM I8MM support enabled via VLLM_CPU_ARM_I8MM environment variable")
|
||||
endif()
|
||||
# Some kernels (e.g. Bianbu on Spacemit X100) do not report zvfbfmin
|
||||
# in /proc/cpuinfo despite hardware support. VLLM_CPU_RVV_BF16=1
|
||||
# overrides the detection result.
|
||||
@@ -166,6 +174,11 @@ elseif (ASIMD_FOUND)
|
||||
message(WARNING "BF16 functionality is not available")
|
||||
set(MARCH_FLAGS "-march=armv8.2-a+dotprod+fp16")
|
||||
endif()
|
||||
if(ARM_I8MM_FOUND)
|
||||
message(STATUS "I8MM extension detected")
|
||||
string(APPEND MARCH_FLAGS "+i8mm")
|
||||
add_compile_definitions(ARM_I8MM_SUPPORT)
|
||||
endif()
|
||||
list(APPEND CXX_COMPILE_FLAGS ${MARCH_FLAGS})
|
||||
elseif (S390_FOUND)
|
||||
message(STATUS "S390 detected")
|
||||
@@ -447,8 +460,13 @@ if (ASIMD_FOUND AND NOT APPLE_SILICON_FOUND)
|
||||
"csrc/cpu/shm.cpp"
|
||||
"csrc/cpu/activation_lut_bf16.cpp"
|
||||
"csrc/cpu/cpu_tanhf_neon.hpp"
|
||||
"csrc/cpu/cpu_fused_moe.cpp"
|
||||
${VLLM_EXT_SRC})
|
||||
if (ARM_BF16_FOUND)
|
||||
set(VLLM_EXT_SRC "csrc/cpu/cpu_fused_moe.cpp" ${VLLM_EXT_SRC})
|
||||
if (ARM_I8MM_FOUND)
|
||||
set(VLLM_EXT_SRC "csrc/cpu/cpu_fused_moe_int8.cpp" ${VLLM_EXT_SRC})
|
||||
endif()
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if (POWER9_FOUND OR POWER10_FOUND OR POWER11_FOUND)
|
||||
|
||||
+5
-187
@@ -1,5 +1,6 @@
|
||||
#include "cpu/cpu_types.hpp"
|
||||
#include "cpu/utils.hpp"
|
||||
#include "cpu/cpu_fused_moe_activations.hpp"
|
||||
#include "cpu/micro_gemm/cpu_micro_gemm_vec.hpp"
|
||||
#include "cpu/cpu_arch_macros.h"
|
||||
|
||||
@@ -43,193 +44,9 @@
|
||||
}()
|
||||
|
||||
namespace {
|
||||
enum class FusedMOEAct {
|
||||
SiluAndMul,
|
||||
SwigluOAIAndMul,
|
||||
GeluAndMul,
|
||||
GeluTanhAndMul,
|
||||
};
|
||||
|
||||
FusedMOEAct get_act_type(const std::string& act) {
|
||||
if (act == "silu") {
|
||||
return FusedMOEAct::SiluAndMul;
|
||||
} else if (act == "swigluoai") {
|
||||
return FusedMOEAct::SwigluOAIAndMul;
|
||||
} else if (act == "gelu") {
|
||||
return FusedMOEAct::GeluAndMul;
|
||||
} else if (act == "gelu_tanh") {
|
||||
return FusedMOEAct::GeluTanhAndMul;
|
||||
} else {
|
||||
TORCH_CHECK(false, "Invalid act type: " + act);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
void swigluoai_and_mul(float* __restrict__ input, scalar_t* __restrict__ output,
|
||||
const int32_t m_size, const int32_t n_size,
|
||||
const int32_t input_stride,
|
||||
const int32_t output_stride) {
|
||||
using scalar_vec_t = typename cpu_utils::VecTypeTrait<scalar_t>::vec_t;
|
||||
#if !defined(__aarch64__)
|
||||
// For GPT-OSS interleaved gate-up weights
|
||||
alignas(64) static int32_t index[16] = {0, 2, 4, 6, 8, 10, 12, 14,
|
||||
16, 18, 20, 22, 24, 26, 28, 30};
|
||||
vec_op::INT32Vec16 index_vec(index);
|
||||
#endif
|
||||
vec_op::FP32Vec16 gate_up_max_vec(7.0);
|
||||
vec_op::FP32Vec16 up_min_vec(-7.0);
|
||||
vec_op::FP32Vec16 alpha_vec(1.702);
|
||||
vec_op::FP32Vec16 one_vec(1.0);
|
||||
|
||||
DEFINE_FAST_EXP
|
||||
|
||||
for (int32_t m = 0; m < m_size; ++m) {
|
||||
for (int32_t n = 0; n < n_size; n += 32) {
|
||||
// Note: AdvSIMD does not support gather loads
|
||||
#if defined(__aarch64__)
|
||||
vec_op::FP32Vec16 gate_vec(vec_op::uninit);
|
||||
vec_op::FP32Vec16 up_vec(vec_op::uninit);
|
||||
vec_op::FP32Vec16::load_even_odd(input + n, gate_vec, up_vec);
|
||||
#else
|
||||
vec_op::FP32Vec16 gate_vec(input + n, index_vec);
|
||||
vec_op::FP32Vec16 up_vec(input + n + 1, index_vec);
|
||||
#endif
|
||||
gate_vec = gate_vec.min(gate_up_max_vec);
|
||||
up_vec = up_vec.clamp(up_min_vec, gate_up_max_vec);
|
||||
auto sigmoid_vec = one_vec / (one_vec + fast_exp(-gate_vec * alpha_vec));
|
||||
auto glu = gate_vec * sigmoid_vec;
|
||||
auto gated_output_fp32 = (one_vec + up_vec) * glu;
|
||||
scalar_vec_t gated_output = scalar_vec_t(gated_output_fp32);
|
||||
gated_output.save(output + n / 2);
|
||||
}
|
||||
input += input_stride;
|
||||
output += output_stride;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
void silu_and_mul(float* __restrict__ input, scalar_t* __restrict__ output,
|
||||
const int32_t m_size, const int32_t n_size,
|
||||
const int32_t input_stride, const int32_t output_stride) {
|
||||
using scalar_vec_t = typename cpu_utils::VecTypeTrait<scalar_t>::vec_t;
|
||||
const int32_t dim = n_size / 2;
|
||||
float* __restrict__ gate = input;
|
||||
float* __restrict__ up = input + dim;
|
||||
vec_op::FP32Vec16 one_vec(1.0);
|
||||
|
||||
DEFINE_FAST_EXP
|
||||
|
||||
for (int32_t m = 0; m < m_size; ++m) {
|
||||
for (int32_t n = 0; n < dim; n += 16) {
|
||||
vec_op::FP32Vec16 gate_vec(gate + n);
|
||||
vec_op::FP32Vec16 up_vec(up + n);
|
||||
auto sigmoid_vec = one_vec / (one_vec + fast_exp(-gate_vec));
|
||||
auto silu = gate_vec * sigmoid_vec;
|
||||
auto gated_output_fp32 = up_vec * silu;
|
||||
scalar_vec_t gated_output = scalar_vec_t(gated_output_fp32);
|
||||
gated_output.save(output + n);
|
||||
}
|
||||
gate += input_stride;
|
||||
up += input_stride;
|
||||
output += output_stride;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
void gelu_and_mul(float* __restrict__ input, scalar_t* __restrict__ output,
|
||||
const int32_t m_size, const int32_t n_size,
|
||||
const int32_t input_stride, const int32_t output_stride) {
|
||||
using scalar_vec_t = typename cpu_utils::VecTypeTrait<scalar_t>::vec_t;
|
||||
const int32_t dim = n_size / 2;
|
||||
float* __restrict__ gate = input;
|
||||
float* __restrict__ up = input + dim;
|
||||
vec_op::FP32Vec16 one_vec(1.0);
|
||||
vec_op::FP32Vec16 w1_vec(M_SQRT1_2);
|
||||
vec_op::FP32Vec16 w2_vec(0.5);
|
||||
alignas(64) float temp[16];
|
||||
|
||||
DEFINE_FAST_EXP
|
||||
|
||||
for (int32_t m = 0; m < m_size; ++m) {
|
||||
for (int32_t n = 0; n < dim; n += 16) {
|
||||
vec_op::FP32Vec16 gate_vec(gate + n);
|
||||
vec_op::FP32Vec16 up_vec(up + n);
|
||||
auto er_input_vec = gate_vec * w1_vec;
|
||||
|
||||
er_input_vec.save(temp);
|
||||
for (int32_t i = 0; i < 16; ++i) {
|
||||
temp[i] = std::erf(temp[i]);
|
||||
}
|
||||
vec_op::FP32Vec16 er_vec(temp);
|
||||
auto gelu = gate_vec * w2_vec * (one_vec + er_vec);
|
||||
auto gated_output_fp32 = up_vec * gelu;
|
||||
scalar_vec_t gated_output = scalar_vec_t(gated_output_fp32);
|
||||
gated_output.save(output + n);
|
||||
}
|
||||
gate += input_stride;
|
||||
up += input_stride;
|
||||
output += output_stride;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
void gelu_tanh_and_mul(float* __restrict__ input, scalar_t* __restrict__ output,
|
||||
const int32_t m_size, const int32_t n_size,
|
||||
const int32_t input_stride,
|
||||
const int32_t output_stride) {
|
||||
using scalar_vec_t = typename cpu_utils::VecTypeTrait<scalar_t>::vec_t;
|
||||
const int32_t dim = n_size / 2;
|
||||
float* __restrict__ gate = input;
|
||||
float* __restrict__ up = input + dim;
|
||||
vec_op::FP32Vec16 one_vec(1.0);
|
||||
vec_op::FP32Vec16 w1_vec(0.7978845608028654);
|
||||
vec_op::FP32Vec16 w2_vec(0.5);
|
||||
vec_op::FP32Vec16 w3_vec(0.044715);
|
||||
|
||||
for (int32_t m = 0; m < m_size; ++m) {
|
||||
for (int32_t n = 0; n < dim; n += 16) {
|
||||
vec_op::FP32Vec16 gate_vec(gate + n);
|
||||
vec_op::FP32Vec16 up_vec(up + n);
|
||||
auto gate_pow3_vec = gate_vec * gate_vec * gate_vec;
|
||||
auto inner_vec = w1_vec * (gate_vec + w3_vec * gate_pow3_vec);
|
||||
// Note: can't use fast_exp form because diffusiongemma will generate
|
||||
// wrong results
|
||||
auto tanh_vec = inner_vec.tanh();
|
||||
auto gelu_tanh = gate_vec * w2_vec * (one_vec + tanh_vec);
|
||||
auto gated_output_fp32 = up_vec * gelu_tanh;
|
||||
scalar_vec_t gated_output = scalar_vec_t(gated_output_fp32);
|
||||
gated_output.save(output + n);
|
||||
}
|
||||
gate += input_stride;
|
||||
up += input_stride;
|
||||
output += output_stride;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
FORCE_INLINE void apply_gated_act(const FusedMOEAct act,
|
||||
float* __restrict__ input,
|
||||
scalar_t* __restrict__ output,
|
||||
const int32_t m, const int32_t n,
|
||||
const int32_t input_stride,
|
||||
const int32_t output_stride) {
|
||||
switch (act) {
|
||||
case FusedMOEAct::SwigluOAIAndMul:
|
||||
swigluoai_and_mul(input, output, m, n, input_stride, output_stride);
|
||||
return;
|
||||
case FusedMOEAct::SiluAndMul:
|
||||
silu_and_mul(input, output, m, n, input_stride, output_stride);
|
||||
return;
|
||||
case FusedMOEAct::GeluAndMul:
|
||||
gelu_and_mul(input, output, m, n, input_stride, output_stride);
|
||||
return;
|
||||
case FusedMOEAct::GeluTanhAndMul:
|
||||
gelu_tanh_and_mul(input, output, m, n, input_stride, output_stride);
|
||||
return;
|
||||
default:
|
||||
TORCH_CHECK(false, "Unsupported act type.");
|
||||
}
|
||||
}
|
||||
using cpu_fused_moe_utils::apply_gated_act;
|
||||
using cpu_fused_moe_utils::FusedMOEAct;
|
||||
|
||||
template <typename scalar_t, typename gemm_t>
|
||||
void prepack_moe_weight_impl(scalar_t* __restrict__ weight_ptr,
|
||||
@@ -817,6 +634,7 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
void prepack_moe_weight(
|
||||
@@ -864,7 +682,7 @@ void cpu_fused_moe(
|
||||
const int32_t input_size_2 = w2.size(2);
|
||||
const int32_t output_size_2 = w2.size(1);
|
||||
const int32_t topk_num = topk_id.size(1);
|
||||
const FusedMOEAct act_type = get_act_type(act);
|
||||
const FusedMOEAct act_type = cpu_fused_moe_utils::get_act_type(act);
|
||||
cpu_utils::ISA isa_type = cpu_utils::get_isa(isa);
|
||||
TORCH_CHECK(!skip_weighted || topk_num == 1,
|
||||
"skip_weighted is only supported for topk=1 on CPU");
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
#ifndef CPU_FUSED_MOE_ACTIVATIONS_HPP
|
||||
#define CPU_FUSED_MOE_ACTIVATIONS_HPP
|
||||
|
||||
#include <cmath>
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
|
||||
#include "cpu/cpu_arch_macros.h"
|
||||
#include "cpu/utils.hpp"
|
||||
|
||||
namespace cpu_fused_moe_utils {
|
||||
enum class FusedMOEAct {
|
||||
SiluAndMul,
|
||||
SwigluOAIAndMul,
|
||||
GeluAndMul,
|
||||
GeluTanhAndMul,
|
||||
};
|
||||
|
||||
inline FusedMOEAct get_act_type(const std::string& act) {
|
||||
if (act == "silu") {
|
||||
return FusedMOEAct::SiluAndMul;
|
||||
} else if (act == "swigluoai") {
|
||||
return FusedMOEAct::SwigluOAIAndMul;
|
||||
} else if (act == "gelu") {
|
||||
return FusedMOEAct::GeluAndMul;
|
||||
} else if (act == "gelu_tanh") {
|
||||
return FusedMOEAct::GeluTanhAndMul;
|
||||
} else {
|
||||
TORCH_CHECK(false, "Invalid act type: " + act);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
void swigluoai_and_mul(float* __restrict__ input, scalar_t* __restrict__ output,
|
||||
const int32_t m_size, const int32_t n_size,
|
||||
const int32_t input_stride,
|
||||
const int32_t output_stride) {
|
||||
using scalar_vec_t = typename cpu_utils::VecTypeTrait<scalar_t>::vec_t;
|
||||
#if !defined(__aarch64__)
|
||||
// For GPT-OSS interleaved gate-up weights
|
||||
alignas(64) static int32_t index[16] = {0, 2, 4, 6, 8, 10, 12, 14,
|
||||
16, 18, 20, 22, 24, 26, 28, 30};
|
||||
vec_op::INT32Vec16 index_vec(index);
|
||||
#endif
|
||||
vec_op::FP32Vec16 gate_up_max_vec(7.0);
|
||||
vec_op::FP32Vec16 up_min_vec(-7.0);
|
||||
vec_op::FP32Vec16 alpha_vec(1.702);
|
||||
vec_op::FP32Vec16 one_vec(1.0);
|
||||
|
||||
DEFINE_FAST_EXP
|
||||
|
||||
for (int32_t m = 0; m < m_size; ++m) {
|
||||
for (int32_t n = 0; n < n_size; n += 32) {
|
||||
// Note: AdvSIMD does not support gather loads
|
||||
#if defined(__aarch64__)
|
||||
vec_op::FP32Vec16 gate_vec(vec_op::uninit);
|
||||
vec_op::FP32Vec16 up_vec(vec_op::uninit);
|
||||
vec_op::FP32Vec16::load_even_odd(input + n, gate_vec, up_vec);
|
||||
#else
|
||||
vec_op::FP32Vec16 gate_vec(input + n, index_vec);
|
||||
vec_op::FP32Vec16 up_vec(input + n + 1, index_vec);
|
||||
#endif
|
||||
gate_vec = gate_vec.min(gate_up_max_vec);
|
||||
up_vec = up_vec.clamp(up_min_vec, gate_up_max_vec);
|
||||
auto sigmoid_vec = one_vec / (one_vec + fast_exp(-gate_vec * alpha_vec));
|
||||
auto glu = gate_vec * sigmoid_vec;
|
||||
auto gated_output_fp32 = (one_vec + up_vec) * glu;
|
||||
scalar_vec_t gated_output = scalar_vec_t(gated_output_fp32);
|
||||
gated_output.save(output + n / 2);
|
||||
}
|
||||
input += input_stride;
|
||||
output += output_stride;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
void silu_and_mul(float* __restrict__ input, scalar_t* __restrict__ output,
|
||||
const int32_t m_size, const int32_t n_size,
|
||||
const int32_t input_stride, const int32_t output_stride) {
|
||||
using scalar_vec_t = typename cpu_utils::VecTypeTrait<scalar_t>::vec_t;
|
||||
const int32_t dim = n_size / 2;
|
||||
float* __restrict__ gate = input;
|
||||
float* __restrict__ up = input + dim;
|
||||
vec_op::FP32Vec16 one_vec(1.0);
|
||||
|
||||
DEFINE_FAST_EXP
|
||||
|
||||
for (int32_t m = 0; m < m_size; ++m) {
|
||||
for (int32_t n = 0; n < dim; n += 16) {
|
||||
vec_op::FP32Vec16 gate_vec(gate + n);
|
||||
vec_op::FP32Vec16 up_vec(up + n);
|
||||
auto sigmoid_vec = one_vec / (one_vec + fast_exp(-gate_vec));
|
||||
auto silu = gate_vec * sigmoid_vec;
|
||||
auto gated_output_fp32 = up_vec * silu;
|
||||
scalar_vec_t gated_output = scalar_vec_t(gated_output_fp32);
|
||||
gated_output.save(output + n);
|
||||
}
|
||||
gate += input_stride;
|
||||
up += input_stride;
|
||||
output += output_stride;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
void gelu_and_mul(float* __restrict__ input, scalar_t* __restrict__ output,
|
||||
const int32_t m_size, const int32_t n_size,
|
||||
const int32_t input_stride, const int32_t output_stride) {
|
||||
using scalar_vec_t = typename cpu_utils::VecTypeTrait<scalar_t>::vec_t;
|
||||
const int32_t dim = n_size / 2;
|
||||
float* __restrict__ gate = input;
|
||||
float* __restrict__ up = input + dim;
|
||||
vec_op::FP32Vec16 one_vec(1.0);
|
||||
vec_op::FP32Vec16 w1_vec(M_SQRT1_2);
|
||||
vec_op::FP32Vec16 w2_vec(0.5);
|
||||
alignas(64) float temp[16];
|
||||
|
||||
DEFINE_FAST_EXP
|
||||
|
||||
for (int32_t m = 0; m < m_size; ++m) {
|
||||
for (int32_t n = 0; n < dim; n += 16) {
|
||||
vec_op::FP32Vec16 gate_vec(gate + n);
|
||||
vec_op::FP32Vec16 up_vec(up + n);
|
||||
auto er_input_vec = gate_vec * w1_vec;
|
||||
|
||||
er_input_vec.save(temp);
|
||||
for (int32_t i = 0; i < 16; ++i) {
|
||||
temp[i] = std::erf(temp[i]);
|
||||
}
|
||||
vec_op::FP32Vec16 er_vec(temp);
|
||||
auto gelu = gate_vec * w2_vec * (one_vec + er_vec);
|
||||
auto gated_output_fp32 = up_vec * gelu;
|
||||
scalar_vec_t gated_output = scalar_vec_t(gated_output_fp32);
|
||||
gated_output.save(output + n);
|
||||
}
|
||||
gate += input_stride;
|
||||
up += input_stride;
|
||||
output += output_stride;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
void gelu_tanh_and_mul(float* __restrict__ input, scalar_t* __restrict__ output,
|
||||
const int32_t m_size, const int32_t n_size,
|
||||
const int32_t input_stride,
|
||||
const int32_t output_stride) {
|
||||
using scalar_vec_t = typename cpu_utils::VecTypeTrait<scalar_t>::vec_t;
|
||||
const int32_t dim = n_size / 2;
|
||||
float* __restrict__ gate = input;
|
||||
float* __restrict__ up = input + dim;
|
||||
vec_op::FP32Vec16 one_vec(1.0);
|
||||
vec_op::FP32Vec16 w1_vec(0.7978845608028654);
|
||||
vec_op::FP32Vec16 w2_vec(0.5);
|
||||
vec_op::FP32Vec16 w3_vec(0.044715);
|
||||
|
||||
for (int32_t m = 0; m < m_size; ++m) {
|
||||
for (int32_t n = 0; n < dim; n += 16) {
|
||||
vec_op::FP32Vec16 gate_vec(gate + n);
|
||||
vec_op::FP32Vec16 up_vec(up + n);
|
||||
auto gate_pow3_vec = gate_vec * gate_vec * gate_vec;
|
||||
auto inner_vec = w1_vec * (gate_vec + w3_vec * gate_pow3_vec);
|
||||
// Note: can't use fast_exp form because diffusiongemma will generate
|
||||
// wrong results
|
||||
auto tanh_vec = inner_vec.tanh();
|
||||
auto gelu_tanh = gate_vec * w2_vec * (one_vec + tanh_vec);
|
||||
auto gated_output_fp32 = up_vec * gelu_tanh;
|
||||
scalar_vec_t gated_output = scalar_vec_t(gated_output_fp32);
|
||||
gated_output.save(output + n);
|
||||
}
|
||||
gate += input_stride;
|
||||
up += input_stride;
|
||||
output += output_stride;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
FORCE_INLINE void apply_gated_act(const FusedMOEAct act,
|
||||
float* __restrict__ input,
|
||||
scalar_t* __restrict__ output,
|
||||
const int32_t m, const int32_t n,
|
||||
const int32_t input_stride,
|
||||
const int32_t output_stride) {
|
||||
switch (act) {
|
||||
case FusedMOEAct::SwigluOAIAndMul:
|
||||
swigluoai_and_mul(input, output, m, n, input_stride, output_stride);
|
||||
return;
|
||||
case FusedMOEAct::SiluAndMul:
|
||||
silu_and_mul(input, output, m, n, input_stride, output_stride);
|
||||
return;
|
||||
case FusedMOEAct::GeluAndMul:
|
||||
gelu_and_mul(input, output, m, n, input_stride, output_stride);
|
||||
return;
|
||||
case FusedMOEAct::GeluTanhAndMul:
|
||||
gelu_tanh_and_mul(input, output, m, n, input_stride, output_stride);
|
||||
return;
|
||||
default:
|
||||
TORCH_CHECK(false, "Unsupported act type.");
|
||||
}
|
||||
}
|
||||
} // namespace cpu_fused_moe_utils
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,647 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
#include "cpu/cpu_arch_macros.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
|
||||
#include "cpu/cpu_fused_moe_activations.hpp"
|
||||
#include "cpu/cpu_types.hpp"
|
||||
#include "cpu/micro_gemm/cpu_micro_gemm_impl.hpp"
|
||||
#include "cpu/utils.hpp"
|
||||
|
||||
#if defined(ARM_I8MM_SUPPORT) && defined(ARM_BF16_SUPPORT)
|
||||
#include "cpu/micro_gemm/cpu_micro_gemm_int8_neon.hpp"
|
||||
#define NEON_DISPATCH(SCALAR_TYPE, ...) \
|
||||
case cpu_utils::ISA::NEON: { \
|
||||
using gemm_t = \
|
||||
cpu_micro_gemm::MicroGemmINT8<cpu_utils::ISA::NEON, SCALAR_TYPE>; \
|
||||
return __VA_ARGS__(); \
|
||||
}
|
||||
#else
|
||||
#define NEON_DISPATCH(SCALAR_TYPE, ...) case cpu_utils::ISA::NEON:
|
||||
#endif
|
||||
|
||||
#define CPU_INT8_ISA_DISPATCH_IMPL(ISA_TYPE, SCALAR_TYPE, ...) \
|
||||
[&] { \
|
||||
switch (ISA_TYPE) { \
|
||||
NEON_DISPATCH(SCALAR_TYPE, __VA_ARGS__) \
|
||||
default: { \
|
||||
TORCH_CHECK(false, "Invalid CPU ISA type."); \
|
||||
} \
|
||||
} \
|
||||
}()
|
||||
|
||||
namespace {
|
||||
using cpu_fused_moe_utils::apply_gated_act;
|
||||
using cpu_fused_moe_utils::FusedMOEAct;
|
||||
|
||||
template <typename gemm_t>
|
||||
void prepack_moe_weight_int8_impl(const int8_t* __restrict__ weight_ptr,
|
||||
int8_t* __restrict__ packed_weight_ptr,
|
||||
const int32_t expert_num,
|
||||
const int32_t output_size,
|
||||
const int32_t input_size,
|
||||
const int64_t expert_stride) {
|
||||
#pragma omp parallel for
|
||||
for (int32_t e_idx = 0; e_idx < expert_num; ++e_idx) {
|
||||
gemm_t::pack_weight(weight_ptr + expert_stride * e_idx,
|
||||
packed_weight_ptr + expert_stride * e_idx, output_size,
|
||||
input_size);
|
||||
}
|
||||
}
|
||||
|
||||
// INT8 MoE kernel, based on the original BF16 kernel in cpu_fused_moe.cpp
|
||||
template <typename scalar_t, typename gemm_t>
|
||||
void fused_moe_int8_impl(
|
||||
scalar_t* __restrict__ output, const scalar_t* __restrict__ input,
|
||||
const int8_t* __restrict__ w13, const int8_t* __restrict__ w2,
|
||||
const float* __restrict__ w13_scales, const float* __restrict__ w2_scales,
|
||||
scalar_t* __restrict__ w13_bias, scalar_t* __restrict__ w2_bias,
|
||||
const float* __restrict__ topk_weights, const int32_t* __restrict__ topk_id,
|
||||
const FusedMOEAct act_type, const int32_t token_num,
|
||||
const int32_t expert_num, const int32_t topk_num,
|
||||
const int32_t input_size_13, const int32_t output_size_13,
|
||||
const int32_t input_size_2, const int32_t output_size_2,
|
||||
const bool skip_weighted) {
|
||||
using scalar_vec_t = typename cpu_utils::VecTypeTrait<scalar_t>::vec_t;
|
||||
constexpr int32_t gemm_n_tile_size = gemm_t::NSize;
|
||||
constexpr int32_t gemm_m_tile_size = gemm_t::MaxMSize;
|
||||
constexpr int32_t min_w13_n_tile_size = 2 * gemm_n_tile_size;
|
||||
|
||||
TORCH_CHECK_EQ(input_size_13 % gemm_t::K, 0);
|
||||
TORCH_CHECK_EQ(input_size_2 % gemm_t::K, 0);
|
||||
TORCH_CHECK_EQ(output_size_13 % min_w13_n_tile_size, 0);
|
||||
TORCH_CHECK_EQ(output_size_2 % gemm_n_tile_size, 0);
|
||||
TORCH_CHECK_EQ(output_size_13 / 2, input_size_2);
|
||||
|
||||
const int32_t thread_num = cpu_utils::get_max_threads();
|
||||
const int32_t w13_input_buffer_size = cpu_utils::round_up<64>(
|
||||
gemm_m_tile_size * input_size_13 * sizeof(int8_t));
|
||||
const int32_t w2_input_buffer_size =
|
||||
cpu_utils::round_up<64>(gemm_m_tile_size * input_size_2 * sizeof(int8_t));
|
||||
|
||||
const int32_t w13_n_tile_size = [&]() {
|
||||
const int64_t cache_size = cpu_utils::get_available_l2_size();
|
||||
const int32_t n_size_cache_limit =
|
||||
(cache_size - w13_input_buffer_size) /
|
||||
(gemm_m_tile_size * sizeof(float) + input_size_13 * sizeof(int8_t));
|
||||
const int32_t n_size_thread_limit =
|
||||
output_size_13 / std::max(1, thread_num / topk_num);
|
||||
const int32_t n_size = cpu_utils::round_down<min_w13_n_tile_size>(
|
||||
std::min(n_size_cache_limit, n_size_thread_limit));
|
||||
return std::max(n_size, min_w13_n_tile_size);
|
||||
}();
|
||||
|
||||
const int32_t w2_n_tile_size = [&]() {
|
||||
const int64_t cache_size = cpu_utils::get_available_l2_size();
|
||||
const int32_t n_size_cache_limit =
|
||||
(cache_size - w2_input_buffer_size) / (input_size_2 * sizeof(int8_t));
|
||||
const int32_t n_size_thread_limit =
|
||||
output_size_2 / std::max(1, thread_num / topk_num);
|
||||
const int32_t n_size = cpu_utils::round_down<gemm_n_tile_size>(
|
||||
std::min(n_size_cache_limit, n_size_thread_limit));
|
||||
return std::max(n_size, gemm_n_tile_size);
|
||||
}();
|
||||
|
||||
int32_t common_buffer_offset = 0;
|
||||
const int32_t token_num_per_group_buffer_offset = common_buffer_offset;
|
||||
common_buffer_offset += cpu_utils::round_up<64>(expert_num * sizeof(int32_t));
|
||||
const int32_t cu_token_num_per_group_buffer_offset = common_buffer_offset;
|
||||
common_buffer_offset +=
|
||||
cpu_utils::round_up<64>((expert_num + 1) * sizeof(int32_t));
|
||||
const int32_t expanded_token_num = token_num * topk_num;
|
||||
const int32_t expand_token_id_buffer_offset = common_buffer_offset;
|
||||
common_buffer_offset +=
|
||||
cpu_utils::round_up<64>(expanded_token_num * sizeof(int32_t));
|
||||
const int32_t expand_token_id_index_buffer_offset = common_buffer_offset;
|
||||
common_buffer_offset +=
|
||||
cpu_utils::round_up<64>(expanded_token_num * sizeof(int32_t));
|
||||
const int32_t input_quant_buffer_offset = common_buffer_offset;
|
||||
common_buffer_offset +=
|
||||
cpu_utils::round_up<64>(token_num * input_size_13 * sizeof(int8_t));
|
||||
const int32_t input_scale_buffer_offset = common_buffer_offset;
|
||||
common_buffer_offset += cpu_utils::round_up<64>(token_num * sizeof(float));
|
||||
const int32_t w13_gemm_output_buffer_offset = common_buffer_offset;
|
||||
common_buffer_offset += cpu_utils::round_up<64>(
|
||||
expanded_token_num * input_size_2 * sizeof(scalar_t));
|
||||
const int32_t w13_output_scale_buffer_offset = common_buffer_offset;
|
||||
common_buffer_offset +=
|
||||
cpu_utils::round_up<64>(expanded_token_num * sizeof(float));
|
||||
const int32_t w2_gemm_output_buffer_offset = common_buffer_offset;
|
||||
common_buffer_offset += cpu_utils::round_up<64>(
|
||||
expanded_token_num * output_size_2 * sizeof(float));
|
||||
|
||||
int32_t gemm_thread_buffer_offset = 0;
|
||||
const int32_t gemm_input_buffer_offset = gemm_thread_buffer_offset;
|
||||
gemm_thread_buffer_offset +=
|
||||
std::max(w13_input_buffer_size, w2_input_buffer_size);
|
||||
const int32_t gemm_output_buffer_offset = gemm_thread_buffer_offset;
|
||||
gemm_thread_buffer_offset += cpu_utils::round_up<64>(
|
||||
gemm_m_tile_size * std::max(w13_n_tile_size, w2_n_tile_size) *
|
||||
sizeof(int32_t));
|
||||
|
||||
const int32_t ws_output_buffer_offset = 0;
|
||||
const int32_t ws_thread_buffer_size =
|
||||
cpu_utils::round_up<64>(output_size_2 * sizeof(float));
|
||||
const int32_t thread_buffer_size =
|
||||
std::max(gemm_thread_buffer_offset, ws_thread_buffer_size);
|
||||
const int32_t buffer_size =
|
||||
common_buffer_offset + thread_buffer_size * thread_num;
|
||||
cpu_utils::ScratchPadManager::get_scratchpad_manager()->realloc(buffer_size);
|
||||
uint8_t* common_buffer_start =
|
||||
cpu_utils::ScratchPadManager::get_scratchpad_manager()
|
||||
->get_data<uint8_t>();
|
||||
uint8_t* thread_buffer_start = common_buffer_start + common_buffer_offset;
|
||||
|
||||
int32_t* __restrict__ token_num_per_group_buffer = reinterpret_cast<int32_t*>(
|
||||
common_buffer_start + token_num_per_group_buffer_offset);
|
||||
int32_t* __restrict__ cu_token_num_per_group_buffer =
|
||||
reinterpret_cast<int32_t*>(common_buffer_start +
|
||||
cu_token_num_per_group_buffer_offset);
|
||||
int32_t* __restrict__ expand_token_id_buffer = reinterpret_cast<int32_t*>(
|
||||
common_buffer_start + expand_token_id_buffer_offset);
|
||||
int32_t* __restrict__ expand_token_id_index_buffer =
|
||||
reinterpret_cast<int32_t*>(common_buffer_start +
|
||||
expand_token_id_index_buffer_offset);
|
||||
int8_t* __restrict__ input_quant_buffer = reinterpret_cast<int8_t*>(
|
||||
common_buffer_start + input_quant_buffer_offset);
|
||||
float* __restrict__ input_scale_buffer =
|
||||
reinterpret_cast<float*>(common_buffer_start + input_scale_buffer_offset);
|
||||
|
||||
std::memset(token_num_per_group_buffer, 0, expert_num * sizeof(int32_t));
|
||||
for (int32_t i = 0; i < expanded_token_num; ++i) {
|
||||
++token_num_per_group_buffer[topk_id[i]];
|
||||
}
|
||||
|
||||
int32_t token_num_sum = 0;
|
||||
cu_token_num_per_group_buffer[0] = 0;
|
||||
int32_t* token_index_buffer = cu_token_num_per_group_buffer + 1;
|
||||
for (int32_t i = 0; i < expert_num; ++i) {
|
||||
token_index_buffer[i] = token_num_sum;
|
||||
token_num_sum += token_num_per_group_buffer[i];
|
||||
}
|
||||
|
||||
for (int32_t i = 0; i < token_num; ++i) {
|
||||
const int32_t* curr_topk_id = topk_id + i * topk_num;
|
||||
int32_t* curr_index_buffer = expand_token_id_index_buffer + i * topk_num;
|
||||
for (int32_t j = 0; j < topk_num; ++j) {
|
||||
const int32_t curr_expert_id = curr_topk_id[j];
|
||||
const int32_t curr_index = token_index_buffer[curr_expert_id]++;
|
||||
expand_token_id_buffer[curr_index] = i;
|
||||
curr_index_buffer[j] = curr_index;
|
||||
}
|
||||
}
|
||||
|
||||
// quantize inputs
|
||||
#pragma omp parallel for
|
||||
for (int32_t token_idx = 0; token_idx < token_num; ++token_idx) {
|
||||
gemm_t::quantize_row(input + token_idx * input_size_13,
|
||||
input_quant_buffer + token_idx * input_size_13,
|
||||
input_scale_buffer[token_idx], input_size_13);
|
||||
}
|
||||
|
||||
{
|
||||
alignas(64) cpu_utils::Counter counter;
|
||||
cpu_utils::Counter* counter_ptr = &counter;
|
||||
|
||||
// w13 GEMM + act
|
||||
#pragma omp parallel for schedule(static, 1)
|
||||
for (int32_t thread_id = 0; thread_id < thread_num; ++thread_id) {
|
||||
const int32_t task_num_per_expert =
|
||||
(output_size_13 + w13_n_tile_size - 1) / w13_n_tile_size;
|
||||
const int32_t task_num = task_num_per_expert * expert_num;
|
||||
uint8_t* __restrict__ thread_buffer =
|
||||
thread_buffer_start + thread_id * thread_buffer_size;
|
||||
int8_t* __restrict__ gemm_input_buffer =
|
||||
reinterpret_cast<int8_t*>(thread_buffer + gemm_input_buffer_offset);
|
||||
float* __restrict__ gemm_output_buffer =
|
||||
reinterpret_cast<float*>(thread_buffer + gemm_output_buffer_offset);
|
||||
auto* __restrict__ w13_gemm_output_buffer = reinterpret_cast<scalar_t*>(
|
||||
common_buffer_start + w13_gemm_output_buffer_offset);
|
||||
gemm_t gemm;
|
||||
|
||||
const int32_t w13_n_group_stride =
|
||||
gemm_t::WeightOCGroupSize * input_size_13;
|
||||
const int32_t w13_n_tile_stride = gemm_n_tile_size * input_size_13;
|
||||
|
||||
for (;;) {
|
||||
const int32_t task_id = counter_ptr->acquire_counter();
|
||||
if (task_id >= task_num) {
|
||||
break;
|
||||
}
|
||||
const int32_t curr_expert_id = task_id / task_num_per_expert;
|
||||
const int32_t curr_output_group_id = task_id % task_num_per_expert;
|
||||
const int32_t curr_token_num =
|
||||
token_num_per_group_buffer[curr_expert_id];
|
||||
if (curr_token_num == 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const int32_t actual_n_tile_size =
|
||||
std::min(w13_n_tile_size,
|
||||
output_size_13 - curr_output_group_id * w13_n_tile_size);
|
||||
const int32_t* __restrict__ curr_expand_token_id_buffer =
|
||||
expand_token_id_buffer +
|
||||
cu_token_num_per_group_buffer[curr_expert_id];
|
||||
scalar_t* __restrict__ curr_w13_gemm_output_buffer =
|
||||
w13_gemm_output_buffer +
|
||||
cu_token_num_per_group_buffer[curr_expert_id] * input_size_2 +
|
||||
curr_output_group_id * w13_n_tile_size / 2;
|
||||
|
||||
const int8_t* w13_weight_ptr_0 = nullptr;
|
||||
const int8_t* w13_weight_ptr_1 = nullptr;
|
||||
const float* w13_scale_ptr_0 = nullptr;
|
||||
const float* w13_scale_ptr_1 = nullptr;
|
||||
scalar_t* w13_bias_ptr_0 = nullptr;
|
||||
scalar_t* w13_bias_ptr_1 = nullptr;
|
||||
if (act_type == FusedMOEAct::SwigluOAIAndMul) {
|
||||
const int32_t output_offset = curr_output_group_id * w13_n_tile_size;
|
||||
w13_weight_ptr_0 = w13 +
|
||||
curr_expert_id * input_size_13 * output_size_13 +
|
||||
output_offset * input_size_13;
|
||||
w13_weight_ptr_1 =
|
||||
w13_weight_ptr_0 + actual_n_tile_size / 2 * input_size_13;
|
||||
w13_scale_ptr_0 =
|
||||
w13_scales + curr_expert_id * output_size_13 + output_offset;
|
||||
w13_scale_ptr_1 = w13_scale_ptr_0 + actual_n_tile_size / 2;
|
||||
if (w13_bias != nullptr) {
|
||||
w13_bias_ptr_0 =
|
||||
w13_bias + curr_expert_id * output_size_13 + output_offset;
|
||||
w13_bias_ptr_1 = w13_bias_ptr_0 + actual_n_tile_size / 2;
|
||||
}
|
||||
} else {
|
||||
const int32_t output_offset =
|
||||
curr_output_group_id * (w13_n_tile_size / 2);
|
||||
w13_weight_ptr_0 = w13 +
|
||||
curr_expert_id * input_size_13 * output_size_13 +
|
||||
output_offset * input_size_13;
|
||||
w13_weight_ptr_1 =
|
||||
w13_weight_ptr_0 + output_size_13 / 2 * input_size_13;
|
||||
w13_scale_ptr_0 =
|
||||
w13_scales + curr_expert_id * output_size_13 + output_offset;
|
||||
w13_scale_ptr_1 = w13_scale_ptr_0 + output_size_13 / 2;
|
||||
if (w13_bias != nullptr) {
|
||||
w13_bias_ptr_0 =
|
||||
w13_bias + curr_expert_id * output_size_13 + output_offset;
|
||||
w13_bias_ptr_1 = w13_bias_ptr_0 + output_size_13 / 2;
|
||||
}
|
||||
}
|
||||
|
||||
for (int32_t token_idx = 0; token_idx < curr_token_num;
|
||||
token_idx += gemm_m_tile_size) {
|
||||
const int32_t actual_token_num =
|
||||
std::min(gemm_m_tile_size, curr_token_num - token_idx);
|
||||
const int8_t* input_rows[gemm_m_tile_size];
|
||||
alignas(64) float input_scales[gemm_m_tile_size];
|
||||
// gather and pack
|
||||
for (int32_t i = 0; i < actual_token_num; ++i) {
|
||||
const int32_t curr_token_id = curr_expand_token_id_buffer[i];
|
||||
input_rows[i] = input_quant_buffer + curr_token_id * input_size_13;
|
||||
input_scales[i] = input_scale_buffer[curr_token_id];
|
||||
}
|
||||
gemm_t::pack_input_from_rows(input_rows, gemm_input_buffer,
|
||||
actual_token_num, input_size_13);
|
||||
curr_expand_token_id_buffer += actual_token_num;
|
||||
|
||||
const int8_t* w13_weight_ptr_0_iter = w13_weight_ptr_0;
|
||||
const int8_t* w13_weight_ptr_1_iter = w13_weight_ptr_1;
|
||||
const float* w13_scale_ptr_0_iter = w13_scale_ptr_0;
|
||||
const float* w13_scale_ptr_1_iter = w13_scale_ptr_1;
|
||||
scalar_t* w13_bias_ptr_0_iter = w13_bias_ptr_0;
|
||||
scalar_t* w13_bias_ptr_1_iter = w13_bias_ptr_1;
|
||||
float* w13_output_buffer_0_iter = gemm_output_buffer;
|
||||
float* w13_output_buffer_1_iter =
|
||||
gemm_output_buffer + actual_n_tile_size / 2;
|
||||
|
||||
for (int32_t i = 0; i < actual_n_tile_size;
|
||||
i += min_w13_n_tile_size) {
|
||||
auto* output_0_int32 =
|
||||
reinterpret_cast<int32_t*>(w13_output_buffer_0_iter);
|
||||
gemm.gemm(gemm_input_buffer, w13_weight_ptr_0_iter, output_0_int32,
|
||||
actual_token_num, input_size_13, w13_n_group_stride,
|
||||
actual_n_tile_size);
|
||||
gemm_t::dequantize_tile(output_0_int32, w13_output_buffer_0_iter,
|
||||
input_scales, w13_scale_ptr_0_iter,
|
||||
actual_token_num, gemm_n_tile_size,
|
||||
actual_n_tile_size);
|
||||
if (w13_bias != nullptr) {
|
||||
cpu_micro_gemm::add_bias_epilogue<gemm_n_tile_size>(
|
||||
w13_output_buffer_0_iter, w13_output_buffer_0_iter,
|
||||
w13_bias_ptr_0_iter, actual_token_num, actual_n_tile_size,
|
||||
actual_n_tile_size);
|
||||
w13_bias_ptr_0_iter += gemm_n_tile_size;
|
||||
}
|
||||
|
||||
auto* output_1_int32 =
|
||||
reinterpret_cast<int32_t*>(w13_output_buffer_1_iter);
|
||||
gemm.gemm(gemm_input_buffer, w13_weight_ptr_1_iter, output_1_int32,
|
||||
actual_token_num, input_size_13, w13_n_group_stride,
|
||||
actual_n_tile_size);
|
||||
gemm_t::dequantize_tile(output_1_int32, w13_output_buffer_1_iter,
|
||||
input_scales, w13_scale_ptr_1_iter,
|
||||
actual_token_num, gemm_n_tile_size,
|
||||
actual_n_tile_size);
|
||||
if (w13_bias != nullptr) {
|
||||
cpu_micro_gemm::add_bias_epilogue<gemm_n_tile_size>(
|
||||
w13_output_buffer_1_iter, w13_output_buffer_1_iter,
|
||||
w13_bias_ptr_1_iter, actual_token_num, actual_n_tile_size,
|
||||
actual_n_tile_size);
|
||||
w13_bias_ptr_1_iter += gemm_n_tile_size;
|
||||
}
|
||||
|
||||
w13_weight_ptr_0_iter += w13_n_tile_stride;
|
||||
w13_weight_ptr_1_iter += w13_n_tile_stride;
|
||||
w13_scale_ptr_0_iter += gemm_n_tile_size;
|
||||
w13_scale_ptr_1_iter += gemm_n_tile_size;
|
||||
w13_output_buffer_0_iter += gemm_n_tile_size;
|
||||
w13_output_buffer_1_iter += gemm_n_tile_size;
|
||||
}
|
||||
|
||||
apply_gated_act(act_type, gemm_output_buffer,
|
||||
curr_w13_gemm_output_buffer, actual_token_num,
|
||||
actual_n_tile_size, actual_n_tile_size, input_size_2);
|
||||
curr_w13_gemm_output_buffer += gemm_m_tile_size * input_size_2;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
auto* __restrict__ w13_gemm_output_buffer = reinterpret_cast<scalar_t*>(
|
||||
common_buffer_start + w13_gemm_output_buffer_offset);
|
||||
float* __restrict__ w13_output_scale_buffer = reinterpret_cast<float*>(
|
||||
common_buffer_start + w13_output_scale_buffer_offset);
|
||||
|
||||
// quantize w2 inputs - in place
|
||||
#pragma omp parallel for
|
||||
for (int32_t token_idx = 0; token_idx < expanded_token_num; ++token_idx) {
|
||||
scalar_t* input_row = w13_gemm_output_buffer + token_idx * input_size_2;
|
||||
int8_t* output_row = reinterpret_cast<int8_t*>(input_row);
|
||||
gemm_t::quantize_row(input_row, output_row,
|
||||
w13_output_scale_buffer[token_idx], input_size_2);
|
||||
}
|
||||
|
||||
{
|
||||
alignas(64) cpu_utils::Counter counter;
|
||||
cpu_utils::Counter* counter_ptr = &counter;
|
||||
|
||||
// w2 gemm
|
||||
#pragma omp parallel for schedule(static, 1)
|
||||
for (int32_t thread_id = 0; thread_id < thread_num; ++thread_id) {
|
||||
const int32_t task_num_per_expert =
|
||||
(output_size_2 + w2_n_tile_size - 1) / w2_n_tile_size;
|
||||
const int32_t task_num = task_num_per_expert * expert_num;
|
||||
uint8_t* __restrict__ thread_buffer =
|
||||
thread_buffer_start + thread_id * thread_buffer_size;
|
||||
int8_t* __restrict__ gemm_input_buffer =
|
||||
reinterpret_cast<int8_t*>(thread_buffer + gemm_input_buffer_offset);
|
||||
float* __restrict__ gemm_output_buffer =
|
||||
reinterpret_cast<float*>(thread_buffer + gemm_output_buffer_offset);
|
||||
float* __restrict__ w2_gemm_output_buffer = reinterpret_cast<float*>(
|
||||
common_buffer_start + w2_gemm_output_buffer_offset);
|
||||
gemm_t gemm;
|
||||
|
||||
const int32_t w2_n_group_stride =
|
||||
gemm_t::WeightOCGroupSize * input_size_2;
|
||||
const int32_t w2_n_tile_stride = gemm_n_tile_size * input_size_2;
|
||||
|
||||
for (;;) {
|
||||
const int32_t task_id = counter_ptr->acquire_counter();
|
||||
if (task_id >= task_num) {
|
||||
break;
|
||||
}
|
||||
const int32_t curr_expert_id = task_id / task_num_per_expert;
|
||||
const int32_t curr_output_group_id = task_id % task_num_per_expert;
|
||||
const int32_t curr_token_num =
|
||||
token_num_per_group_buffer[curr_expert_id];
|
||||
if (curr_token_num == 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const int32_t actual_n_tile_size =
|
||||
std::min(w2_n_tile_size,
|
||||
output_size_2 - curr_output_group_id * w2_n_tile_size);
|
||||
scalar_t* __restrict__ curr_w13_gemm_output_buffer =
|
||||
w13_gemm_output_buffer +
|
||||
cu_token_num_per_group_buffer[curr_expert_id] * input_size_2;
|
||||
float* __restrict__ curr_w13_output_scale_buffer =
|
||||
w13_output_scale_buffer +
|
||||
cu_token_num_per_group_buffer[curr_expert_id];
|
||||
float* __restrict__ curr_w2_gemm_output_buffer =
|
||||
w2_gemm_output_buffer +
|
||||
cu_token_num_per_group_buffer[curr_expert_id] * output_size_2 +
|
||||
curr_output_group_id * w2_n_tile_size;
|
||||
const int8_t* __restrict__ w2_weight_ptr =
|
||||
w2 + curr_expert_id * output_size_2 * input_size_2 +
|
||||
curr_output_group_id * w2_n_tile_size * input_size_2;
|
||||
const float* __restrict__ w2_scale_ptr =
|
||||
w2_scales + curr_expert_id * output_size_2 +
|
||||
curr_output_group_id * w2_n_tile_size;
|
||||
scalar_t* w2_bias_ptr = nullptr;
|
||||
if (w2_bias != nullptr) {
|
||||
w2_bias_ptr = w2_bias + curr_expert_id * output_size_2 +
|
||||
curr_output_group_id * w2_n_tile_size;
|
||||
}
|
||||
|
||||
for (int32_t token_idx = 0; token_idx < curr_token_num;
|
||||
token_idx += gemm_m_tile_size) {
|
||||
const int32_t actual_token_num =
|
||||
std::min(gemm_m_tile_size, curr_token_num - token_idx);
|
||||
const int8_t* input_rows[gemm_m_tile_size];
|
||||
alignas(64) float input_scales[gemm_m_tile_size];
|
||||
for (int32_t i = 0; i < actual_token_num; ++i) {
|
||||
input_rows[i] = reinterpret_cast<const int8_t*>(
|
||||
curr_w13_gemm_output_buffer + i * input_size_2);
|
||||
input_scales[i] = curr_w13_output_scale_buffer[i];
|
||||
}
|
||||
gemm_t::pack_input_from_rows(input_rows, gemm_input_buffer,
|
||||
actual_token_num, input_size_2);
|
||||
|
||||
const int8_t* w2_weight_ptr_iter = w2_weight_ptr;
|
||||
const float* w2_scale_ptr_iter = w2_scale_ptr;
|
||||
scalar_t* w2_bias_ptr_iter = w2_bias_ptr;
|
||||
float* curr_w2_gemm_output_buffer_iter = curr_w2_gemm_output_buffer;
|
||||
for (int32_t i = 0; i < actual_n_tile_size; i += gemm_n_tile_size) {
|
||||
auto* output_int32 = reinterpret_cast<int32_t*>(gemm_output_buffer);
|
||||
gemm.gemm(gemm_input_buffer, w2_weight_ptr_iter, output_int32,
|
||||
actual_token_num, input_size_2, w2_n_group_stride,
|
||||
gemm_n_tile_size);
|
||||
gemm_t::dequantize_tile(output_int32, gemm_output_buffer,
|
||||
input_scales, w2_scale_ptr_iter,
|
||||
actual_token_num, gemm_n_tile_size,
|
||||
gemm_n_tile_size);
|
||||
if (w2_bias != nullptr) {
|
||||
cpu_micro_gemm::add_bias_epilogue<gemm_n_tile_size>(
|
||||
gemm_output_buffer, gemm_output_buffer, w2_bias_ptr_iter,
|
||||
actual_token_num, gemm_n_tile_size, gemm_n_tile_size);
|
||||
w2_bias_ptr_iter += gemm_n_tile_size;
|
||||
}
|
||||
for (int32_t m_idx = 0; m_idx < actual_token_num; ++m_idx) {
|
||||
std::memcpy(
|
||||
curr_w2_gemm_output_buffer_iter + m_idx * output_size_2,
|
||||
gemm_output_buffer + m_idx * gemm_n_tile_size,
|
||||
gemm_n_tile_size * sizeof(float));
|
||||
}
|
||||
|
||||
w2_weight_ptr_iter += w2_n_tile_stride;
|
||||
w2_scale_ptr_iter += gemm_n_tile_size;
|
||||
curr_w2_gemm_output_buffer_iter += gemm_n_tile_size;
|
||||
}
|
||||
|
||||
curr_w13_gemm_output_buffer += gemm_m_tile_size * input_size_2;
|
||||
curr_w13_output_scale_buffer += gemm_m_tile_size;
|
||||
curr_w2_gemm_output_buffer += gemm_m_tile_size * output_size_2;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
alignas(64) cpu_utils::Counter counter;
|
||||
cpu_utils::Counter* counter_ptr = &counter;
|
||||
|
||||
#pragma omp parallel for schedule(static, 1)
|
||||
for (int32_t thread_id = 0; thread_id < thread_num; ++thread_id) {
|
||||
uint8_t* __restrict__ thread_buffer =
|
||||
thread_buffer_start + thread_id * thread_buffer_size;
|
||||
float* __restrict__ ws_output_buffer =
|
||||
reinterpret_cast<float*>(thread_buffer + ws_output_buffer_offset);
|
||||
float* __restrict__ w2_gemm_output_buffer = reinterpret_cast<float*>(
|
||||
common_buffer_start + w2_gemm_output_buffer_offset);
|
||||
|
||||
for (;;) {
|
||||
const int32_t token_id = counter_ptr->acquire_counter();
|
||||
if (token_id >= token_num) {
|
||||
break;
|
||||
}
|
||||
int32_t* __restrict__ curr_expand_token_id_index_buffer =
|
||||
expand_token_id_index_buffer + token_id * topk_num;
|
||||
const float* __restrict__ curr_weight =
|
||||
topk_weights + token_id * topk_num;
|
||||
const float first_weight = skip_weighted ? 1.0f : curr_weight[0];
|
||||
scalar_t* __restrict__ curr_output_buffer =
|
||||
output + token_id * output_size_2;
|
||||
|
||||
if (topk_num > 1) {
|
||||
int32_t w2_output_idx = curr_expand_token_id_index_buffer[0];
|
||||
float* w2_output_iter =
|
||||
w2_gemm_output_buffer + w2_output_idx * output_size_2;
|
||||
float* ws_output_buffer_iter = ws_output_buffer;
|
||||
vec_op::FP32Vec16 weight_vec(first_weight);
|
||||
for (int32_t i = 0; i < output_size_2; i += 16) {
|
||||
vec_op::FP32Vec16 vec(w2_output_iter);
|
||||
(vec * weight_vec).save(ws_output_buffer_iter);
|
||||
w2_output_iter += 16;
|
||||
ws_output_buffer_iter += 16;
|
||||
}
|
||||
|
||||
for (int32_t idx = 1; idx < topk_num - 1; ++idx) {
|
||||
w2_output_idx = curr_expand_token_id_index_buffer[idx];
|
||||
w2_output_iter =
|
||||
w2_gemm_output_buffer + w2_output_idx * output_size_2;
|
||||
ws_output_buffer_iter = ws_output_buffer;
|
||||
weight_vec = vec_op::FP32Vec16(curr_weight[idx]);
|
||||
for (int32_t i = 0; i < output_size_2; i += 16) {
|
||||
vec_op::FP32Vec16 vec(w2_output_iter);
|
||||
vec_op::FP32Vec16 sum(ws_output_buffer_iter);
|
||||
(sum + vec * weight_vec).save(ws_output_buffer_iter);
|
||||
w2_output_iter += 16;
|
||||
ws_output_buffer_iter += 16;
|
||||
}
|
||||
}
|
||||
|
||||
const int32_t last_idx = topk_num - 1;
|
||||
w2_output_idx = curr_expand_token_id_index_buffer[last_idx];
|
||||
w2_output_iter =
|
||||
w2_gemm_output_buffer + w2_output_idx * output_size_2;
|
||||
ws_output_buffer_iter = ws_output_buffer;
|
||||
scalar_t* curr_output_buffer_iter = curr_output_buffer;
|
||||
weight_vec = vec_op::FP32Vec16(curr_weight[last_idx]);
|
||||
for (int32_t i = 0; i < output_size_2; i += 16) {
|
||||
vec_op::FP32Vec16 vec(w2_output_iter);
|
||||
vec_op::FP32Vec16 sum(ws_output_buffer_iter);
|
||||
scalar_vec_t(sum + vec * weight_vec).save(curr_output_buffer_iter);
|
||||
w2_output_iter += 16;
|
||||
ws_output_buffer_iter += 16;
|
||||
curr_output_buffer_iter += 16;
|
||||
}
|
||||
} else {
|
||||
const int32_t w2_output_idx = curr_expand_token_id_index_buffer[0];
|
||||
float* w2_output_iter =
|
||||
w2_gemm_output_buffer + w2_output_idx * output_size_2;
|
||||
scalar_t* curr_output_buffer_iter = curr_output_buffer;
|
||||
vec_op::FP32Vec16 weight_vec(first_weight);
|
||||
for (int32_t i = 0; i < output_size_2; i += 16) {
|
||||
vec_op::FP32Vec16 vec(w2_output_iter);
|
||||
scalar_vec_t(vec * weight_vec).save(curr_output_buffer_iter);
|
||||
w2_output_iter += 16;
|
||||
curr_output_buffer_iter += 16;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} // namespace
|
||||
|
||||
void prepack_moe_weight_int8(
|
||||
const torch::Tensor& weight, // [expert_num, output_size, input_size]
|
||||
torch::Tensor& packed_weight, const std::string& isa) {
|
||||
TORCH_CHECK(weight.is_contiguous());
|
||||
const int32_t expert_num = weight.size(0);
|
||||
const int32_t output_size = weight.size(1);
|
||||
const int32_t input_size = weight.size(2);
|
||||
const int64_t expert_stride = weight.stride(0);
|
||||
const cpu_utils::ISA isa_type = cpu_utils::get_isa(isa);
|
||||
TORCH_CHECK_EQ(output_size % 32, 0);
|
||||
|
||||
CPU_INT8_ISA_DISPATCH_IMPL(isa_type, c10::BFloat16, [&]() {
|
||||
TORCH_CHECK_EQ(input_size % gemm_t::K, 0);
|
||||
prepack_moe_weight_int8_impl<gemm_t>(
|
||||
weight.data_ptr<int8_t>(), packed_weight.data_ptr<int8_t>(), expert_num,
|
||||
output_size, input_size, expert_stride);
|
||||
});
|
||||
}
|
||||
|
||||
void cpu_fused_moe_int8(torch::Tensor& output, const torch::Tensor& input,
|
||||
const torch::Tensor& w13, const torch::Tensor& w2,
|
||||
const torch::Tensor& w13_scale,
|
||||
const torch::Tensor& w2_scale,
|
||||
const std::optional<torch::Tensor>& w13_bias,
|
||||
const std::optional<torch::Tensor>& w2_bias,
|
||||
const torch::Tensor& topk_weights,
|
||||
const torch::Tensor& topk_id, const bool skip_weighted,
|
||||
const std::string& act, const std::string& isa) {
|
||||
const int32_t token_num = input.size(0);
|
||||
const int32_t input_size_13 = input.size(1);
|
||||
const int64_t input_stride = input.stride(0);
|
||||
TORCH_CHECK_EQ(input_stride, input_size_13);
|
||||
const int32_t expert_num = w13.size(0);
|
||||
const int32_t output_size_13 = w13.size(1);
|
||||
const int32_t input_size_2 = w2.size(2);
|
||||
const int32_t output_size_2 = w2.size(1);
|
||||
const int32_t topk_num = topk_id.size(1);
|
||||
const FusedMOEAct act_type = cpu_fused_moe_utils::get_act_type(act);
|
||||
const cpu_utils::ISA isa_type = cpu_utils::get_isa(isa);
|
||||
TORCH_CHECK(!skip_weighted || topk_num == 1,
|
||||
"skip_weighted is only supported for topk=1 on CPU");
|
||||
|
||||
VLLM_DISPATCH_FLOATING_TYPES(
|
||||
input.scalar_type(), "cpu_fused_moe_int8", [&]() {
|
||||
CPU_INT8_ISA_DISPATCH_IMPL(isa_type, scalar_t, [&]() {
|
||||
fused_moe_int8_impl<scalar_t, gemm_t>(
|
||||
output.data_ptr<scalar_t>(), input.data_ptr<scalar_t>(),
|
||||
w13.data_ptr<int8_t>(), w2.data_ptr<int8_t>(),
|
||||
w13_scale.data_ptr<float>(), w2_scale.data_ptr<float>(),
|
||||
w13_bias.has_value() ? w13_bias->data_ptr<scalar_t>() : nullptr,
|
||||
w2_bias.has_value() ? w2_bias->data_ptr<scalar_t>() : nullptr,
|
||||
topk_weights.data_ptr<float>(), topk_id.data_ptr<int32_t>(),
|
||||
act_type, token_num, expert_num, topk_num, input_size_13,
|
||||
output_size_13, input_size_2, output_size_2, skip_weighted);
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -31,6 +31,9 @@ class MicroGemm {
|
||||
}
|
||||
};
|
||||
|
||||
template <cpu_utils::ISA isa, typename scalar_t>
|
||||
class MicroGemmINT8;
|
||||
|
||||
template <int32_t n_size, typename scalar_t>
|
||||
FORCE_INLINE void default_epilogue(float* __restrict__ c_ptr,
|
||||
scalar_t* __restrict__ d_ptr,
|
||||
|
||||
@@ -0,0 +1,424 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
#ifndef CPU_MICRO_GEMM_INT8_NEON_HPP
|
||||
#define CPU_MICRO_GEMM_INT8_NEON_HPP
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
|
||||
#include "cpu/micro_gemm/cpu_micro_gemm_impl.hpp"
|
||||
|
||||
#include <arm_bf16.h>
|
||||
#include <arm_neon.h>
|
||||
#include <c10/util/BFloat16.h>
|
||||
#include <c10/util/Exception.h>
|
||||
#include <c10/util/Half.h>
|
||||
|
||||
namespace cpu_micro_gemm {
|
||||
|
||||
namespace neon_smmla {
|
||||
|
||||
constexpr int32_t K = 8;
|
||||
constexpr int32_t Cols = 2;
|
||||
constexpr int32_t TileSize = K * Cols;
|
||||
|
||||
FORCE_INLINE float32x4x2_t load_as_f32(const float* input) {
|
||||
float32x4x2_t result;
|
||||
result.val[0] = vld1q_f32(input);
|
||||
result.val[1] = vld1q_f32(input + 4);
|
||||
return result;
|
||||
}
|
||||
|
||||
FORCE_INLINE float32x4x2_t load_as_f32(const c10::Half* input) {
|
||||
const auto input_vec = vld1q_f16(reinterpret_cast<const float16_t*>(input));
|
||||
float32x4x2_t result;
|
||||
result.val[0] = vcvt_f32_f16(vget_low_f16(input_vec));
|
||||
result.val[1] = vcvt_f32_f16(vget_high_f16(input_vec));
|
||||
return result;
|
||||
}
|
||||
|
||||
FORCE_INLINE float32x4x2_t load_as_f32(const c10::BFloat16* input) {
|
||||
const auto input_vec = vld1q_bf16(reinterpret_cast<const bfloat16_t*>(input));
|
||||
float32x4x2_t result;
|
||||
result.val[0] = vcvt_f32_bf16(vget_low_bf16(input_vec));
|
||||
result.val[1] = vcvt_f32_bf16(vget_high_bf16(input_vec));
|
||||
return result;
|
||||
}
|
||||
|
||||
FORCE_INLINE void store_acc_rowpair(const int32x4_t acc01,
|
||||
const int32x4_t acc23,
|
||||
const int32x4_t acc45,
|
||||
const int32x4_t acc67,
|
||||
int32_t* __restrict__ c_ptr,
|
||||
const int64_t ldc, const int32_t m_rows) {
|
||||
if (m_rows == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
vst1q_s32(c_ptr, vcombine_s32(vget_low_s32(acc01), vget_low_s32(acc23)));
|
||||
vst1q_s32(c_ptr + 4, vcombine_s32(vget_low_s32(acc45), vget_low_s32(acc67)));
|
||||
|
||||
if (m_rows == 2) {
|
||||
vst1q_s32(c_ptr + ldc,
|
||||
vcombine_s32(vget_high_s32(acc01), vget_high_s32(acc23)));
|
||||
vst1q_s32(c_ptr + ldc + 4,
|
||||
vcombine_s32(vget_high_s32(acc45), vget_high_s32(acc67)));
|
||||
}
|
||||
}
|
||||
|
||||
FORCE_INLINE void gemm_micro_smmla_8x8_packed_a(
|
||||
const int8_t* __restrict__ a_packed, const int8_t* __restrict__ b_packed,
|
||||
int32_t* __restrict__ c_ptr, const int32_t m, const int32_t k_size,
|
||||
const int64_t ldc) {
|
||||
const int32x4_t zero = vdupq_n_s32(0);
|
||||
int32x4_t acc0101 = zero, acc0123 = zero, acc0145 = zero, acc0167 = zero;
|
||||
int32x4_t acc2301 = zero, acc2323 = zero, acc2345 = zero, acc2367 = zero;
|
||||
int32x4_t acc4501 = zero, acc4523 = zero, acc4545 = zero, acc4567 = zero;
|
||||
int32x4_t acc6701 = zero, acc6723 = zero, acc6745 = zero, acc6767 = zero;
|
||||
|
||||
const int8_t* __restrict__ a_tile = a_packed;
|
||||
const int8_t* __restrict__ b_tile = b_packed;
|
||||
|
||||
#pragma GCC unroll 8
|
||||
for (int32_t k_idx = 0; k_idx < k_size; k_idx += K) {
|
||||
const int8x16_t a_tile01 = vld1q_s8(a_tile);
|
||||
const int8x16_t a_tile23 = vld1q_s8(a_tile + TileSize);
|
||||
const int8x16_t a_tile45 = vld1q_s8(a_tile + 2 * TileSize);
|
||||
const int8x16_t a_tile67 = vld1q_s8(a_tile + 3 * TileSize);
|
||||
|
||||
const int8x16_t b_tile01 = vld1q_s8(b_tile);
|
||||
const int8x16_t b_tile23 = vld1q_s8(b_tile + TileSize);
|
||||
const int8x16_t b_tile45 = vld1q_s8(b_tile + 2 * TileSize);
|
||||
const int8x16_t b_tile67 = vld1q_s8(b_tile + 3 * TileSize);
|
||||
|
||||
acc0101 = vmmlaq_s32(acc0101, a_tile01, b_tile01);
|
||||
acc2301 = vmmlaq_s32(acc2301, a_tile23, b_tile01);
|
||||
acc4501 = vmmlaq_s32(acc4501, a_tile45, b_tile01);
|
||||
acc6701 = vmmlaq_s32(acc6701, a_tile67, b_tile01);
|
||||
|
||||
acc0123 = vmmlaq_s32(acc0123, a_tile01, b_tile23);
|
||||
acc2323 = vmmlaq_s32(acc2323, a_tile23, b_tile23);
|
||||
acc4523 = vmmlaq_s32(acc4523, a_tile45, b_tile23);
|
||||
acc6723 = vmmlaq_s32(acc6723, a_tile67, b_tile23);
|
||||
|
||||
acc0145 = vmmlaq_s32(acc0145, a_tile01, b_tile45);
|
||||
acc2345 = vmmlaq_s32(acc2345, a_tile23, b_tile45);
|
||||
acc4545 = vmmlaq_s32(acc4545, a_tile45, b_tile45);
|
||||
acc6745 = vmmlaq_s32(acc6745, a_tile67, b_tile45);
|
||||
|
||||
acc0167 = vmmlaq_s32(acc0167, a_tile01, b_tile67);
|
||||
acc2367 = vmmlaq_s32(acc2367, a_tile23, b_tile67);
|
||||
acc4567 = vmmlaq_s32(acc4567, a_tile45, b_tile67);
|
||||
acc6767 = vmmlaq_s32(acc6767, a_tile67, b_tile67);
|
||||
|
||||
a_tile += 4 * TileSize;
|
||||
b_tile += 4 * TileSize;
|
||||
}
|
||||
|
||||
store_acc_rowpair(acc0101, acc0123, acc0145, acc0167, c_ptr, ldc,
|
||||
std::min(2, m));
|
||||
store_acc_rowpair(acc2301, acc2323, acc2345, acc2367, c_ptr + 2 * ldc, ldc,
|
||||
std::min(2, std::max(0, m - 2)));
|
||||
store_acc_rowpair(acc4501, acc4523, acc4545, acc4567, c_ptr + 4 * ldc, ldc,
|
||||
std::min(2, std::max(0, m - 4)));
|
||||
store_acc_rowpair(acc6701, acc6723, acc6745, acc6767, c_ptr + 6 * ldc, ldc,
|
||||
std::min(2, std::max(0, m - 6)));
|
||||
}
|
||||
|
||||
FORCE_INLINE void gemm_micro_smmla_4x16_packed_a(
|
||||
const int8_t* __restrict__ a_packed, const int8_t* __restrict__ b_packed,
|
||||
int32_t* __restrict__ c_ptr, const int32_t m, const int32_t k_size,
|
||||
const int64_t b_n_group_stride, const int64_t ldc) {
|
||||
const int32_t m_rows_01 = std::min(2, m);
|
||||
const int32_t m_rows_23 = std::min(2, std::max(0, m - 2));
|
||||
const int32x4_t zero = vdupq_n_s32(0);
|
||||
|
||||
int32x4_t acc0101 = zero, acc0123 = zero, acc0145 = zero, acc0167 = zero;
|
||||
int32x4_t acc2301 = zero, acc2323 = zero, acc2345 = zero, acc2367 = zero;
|
||||
int32x4_t acc0189 = zero, acc011011 = zero, acc011213 = zero,
|
||||
acc011415 = zero;
|
||||
int32x4_t acc2389 = zero, acc231011 = zero, acc231213 = zero,
|
||||
acc231415 = zero;
|
||||
|
||||
const int8_t* __restrict__ a_tile = a_packed;
|
||||
// note: b packs 8 panels contiguously, so we need 2 b_tile ptrs
|
||||
// for the 4x16 microkernel
|
||||
const int8_t* __restrict__ b_tile0 = b_packed;
|
||||
const int8_t* __restrict__ b_tile1 = b_packed + b_n_group_stride;
|
||||
|
||||
#pragma GCC unroll 8
|
||||
for (int32_t k_idx = 0; k_idx < k_size; k_idx += K) {
|
||||
const int8x16_t a_tile01 = vld1q_s8(a_tile);
|
||||
const int8x16_t a_tile23 = vld1q_s8(a_tile + TileSize);
|
||||
const int8x16_t b_tile01 = vld1q_s8(b_tile0);
|
||||
const int8x16_t b_tile23 = vld1q_s8(b_tile0 + TileSize);
|
||||
const int8x16_t b_tile45 = vld1q_s8(b_tile0 + 2 * TileSize);
|
||||
const int8x16_t b_tile67 = vld1q_s8(b_tile0 + 3 * TileSize);
|
||||
const int8x16_t b_tile89 = vld1q_s8(b_tile1);
|
||||
const int8x16_t b_tile1011 = vld1q_s8(b_tile1 + TileSize);
|
||||
const int8x16_t b_tile1213 = vld1q_s8(b_tile1 + 2 * TileSize);
|
||||
const int8x16_t b_tile1415 = vld1q_s8(b_tile1 + 3 * TileSize);
|
||||
|
||||
acc0101 = vmmlaq_s32(acc0101, a_tile01, b_tile01);
|
||||
acc2301 = vmmlaq_s32(acc2301, a_tile23, b_tile01);
|
||||
acc0123 = vmmlaq_s32(acc0123, a_tile01, b_tile23);
|
||||
acc2323 = vmmlaq_s32(acc2323, a_tile23, b_tile23);
|
||||
|
||||
acc0145 = vmmlaq_s32(acc0145, a_tile01, b_tile45);
|
||||
acc2345 = vmmlaq_s32(acc2345, a_tile23, b_tile45);
|
||||
acc0167 = vmmlaq_s32(acc0167, a_tile01, b_tile67);
|
||||
acc2367 = vmmlaq_s32(acc2367, a_tile23, b_tile67);
|
||||
|
||||
acc0189 = vmmlaq_s32(acc0189, a_tile01, b_tile89);
|
||||
acc2389 = vmmlaq_s32(acc2389, a_tile23, b_tile89);
|
||||
acc011011 = vmmlaq_s32(acc011011, a_tile01, b_tile1011);
|
||||
acc231011 = vmmlaq_s32(acc231011, a_tile23, b_tile1011);
|
||||
|
||||
acc011213 = vmmlaq_s32(acc011213, a_tile01, b_tile1213);
|
||||
acc231213 = vmmlaq_s32(acc231213, a_tile23, b_tile1213);
|
||||
acc011415 = vmmlaq_s32(acc011415, a_tile01, b_tile1415);
|
||||
acc231415 = vmmlaq_s32(acc231415, a_tile23, b_tile1415);
|
||||
|
||||
a_tile += 2 * TileSize;
|
||||
b_tile0 += 4 * TileSize;
|
||||
b_tile1 += 4 * TileSize;
|
||||
}
|
||||
|
||||
// rows 0-1, columns 0-7
|
||||
store_acc_rowpair(acc0101, acc0123, acc0145, acc0167, c_ptr, ldc, m_rows_01);
|
||||
// rows 0-1, columns 8-15
|
||||
store_acc_rowpair(acc0189, acc011011, acc011213, acc011415, c_ptr + 8, ldc,
|
||||
m_rows_01);
|
||||
// rows 2-3, columns 0-7
|
||||
store_acc_rowpair(acc2301, acc2323, acc2345, acc2367, c_ptr + 2 * ldc, ldc,
|
||||
m_rows_23);
|
||||
// rows 2-3, columns 8-15
|
||||
store_acc_rowpair(acc2389, acc231011, acc231213, acc231415,
|
||||
c_ptr + 2 * ldc + 8, ldc, m_rows_23);
|
||||
}
|
||||
|
||||
} // namespace neon_smmla
|
||||
|
||||
template <typename scalar_t>
|
||||
class MicroGemmINT8<cpu_utils::ISA::NEON, scalar_t> {
|
||||
public:
|
||||
static constexpr int32_t K = neon_smmla::K;
|
||||
static constexpr int32_t Mr = 8;
|
||||
static constexpr int32_t Nr = 8;
|
||||
static constexpr int32_t NrGemv = 16;
|
||||
static constexpr int32_t MaxMSize = 8;
|
||||
static constexpr int32_t NSize = 32;
|
||||
static constexpr int32_t WeightOCGroupSize = Nr;
|
||||
static_assert(MaxMSize % Mr == 0);
|
||||
|
||||
static FORCE_INLINE void quantize_row(const scalar_t* input, int8_t* output,
|
||||
float& scale, const int32_t size) {
|
||||
TORCH_CHECK_EQ(size % K, 0);
|
||||
float32x4_t max_vec = vdupq_n_f32(0.0f);
|
||||
|
||||
for (int32_t i = 0; i < size; i += K) {
|
||||
const float32x4x2_t input_vec = neon_smmla::load_as_f32(input + i);
|
||||
max_vec = vmaxq_f32(max_vec, vabsq_f32(input_vec.val[0]));
|
||||
max_vec = vmaxq_f32(max_vec, vabsq_f32(input_vec.val[1]));
|
||||
}
|
||||
|
||||
const float abs_max = std::max(vmaxvq_f32(max_vec), 1.0e-7f);
|
||||
scale = abs_max / 127.0f;
|
||||
const float32x4_t inv_scale_vec = vdupq_n_f32(127.0f / abs_max);
|
||||
|
||||
for (int32_t i = 0; i < size; i += K) {
|
||||
const float32x4x2_t input_vec = neon_smmla::load_as_f32(input + i);
|
||||
const int32x4_t output_low =
|
||||
vcvtnq_s32_f32(vmulq_f32(input_vec.val[0], inv_scale_vec));
|
||||
const int32x4_t output_high =
|
||||
vcvtnq_s32_f32(vmulq_f32(input_vec.val[1], inv_scale_vec));
|
||||
const int16x8_t output_s16 =
|
||||
vcombine_s16(vqmovn_s32(output_low), vqmovn_s32(output_high));
|
||||
vst1_s8(output + i, vqmovn_s16(output_s16));
|
||||
}
|
||||
}
|
||||
|
||||
// with current code, fusing this into the gemm micro kernel didn't move the
|
||||
// needle
|
||||
static FORCE_INLINE void dequantize_tile(
|
||||
int32_t* input, float* output, const float* __restrict__ input_scales,
|
||||
const float* __restrict__ weight_scales, const int32_t m, const int32_t n,
|
||||
const int32_t stride) {
|
||||
TORCH_CHECK_EQ(n % 4, 0);
|
||||
for (int32_t m_idx = 0; m_idx < m; ++m_idx) {
|
||||
const float32x4_t input_scale_vec = vdupq_n_f32(input_scales[m_idx]);
|
||||
for (int32_t n_idx = 0; n_idx < n; n_idx += 4) {
|
||||
const int32x4_t input_vec = vld1q_s32(input + m_idx * stride + n_idx);
|
||||
const float32x4_t weight_scale_vec = vld1q_f32(weight_scales + n_idx);
|
||||
const float32x4_t output_vec =
|
||||
vmulq_f32(vcvtq_f32_s32(input_vec),
|
||||
vmulq_f32(input_scale_vec, weight_scale_vec));
|
||||
vst1q_f32(output + m_idx * stride + n_idx, output_vec);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// physical layout [
|
||||
// M / (8 or 4); Mr is 8 or 4
|
||||
// K / 8; K for smmla is 8
|
||||
// 4, ; 4 row-pairs for each 8 rows
|
||||
// 2, ; row-pair is 2 rows
|
||||
// 4 ; 4 elements per row
|
||||
// ]
|
||||
static void pack_input_from_rows(const int8_t* const* __restrict__ rows,
|
||||
int8_t* __restrict__ a_packed,
|
||||
const int32_t m, const int32_t k) {
|
||||
TORCH_CHECK(m > 0 && m <= MaxMSize);
|
||||
TORCH_CHECK(k % K == 0);
|
||||
const int8x8_t zero = vdup_n_s8(0);
|
||||
|
||||
for (int32_t row_base = 0; row_base < m; row_base += Mr) {
|
||||
const int32_t panel_m = std::min(Mr, m - row_base);
|
||||
const int8_t* const* panel_rows = rows + row_base;
|
||||
int8_t* __restrict__ out = a_packed + row_base * k;
|
||||
|
||||
// fast path for full 8-row panels (fast path for 4-row panels didn't move
|
||||
// the needle)
|
||||
if (panel_m == Mr) {
|
||||
const int8_t* __restrict__ row0 = panel_rows[0];
|
||||
const int8_t* __restrict__ row1 = panel_rows[1];
|
||||
const int8_t* __restrict__ row2 = panel_rows[2];
|
||||
const int8_t* __restrict__ row3 = panel_rows[3];
|
||||
const int8_t* __restrict__ row4 = panel_rows[4];
|
||||
const int8_t* __restrict__ row5 = panel_rows[5];
|
||||
const int8_t* __restrict__ row6 = panel_rows[6];
|
||||
const int8_t* __restrict__ row7 = panel_rows[7];
|
||||
int32_t k_idx = 0;
|
||||
for (; k_idx + 2 * K <= k; k_idx += 2 * K) {
|
||||
int8_t* __restrict__ block0 = out;
|
||||
int8_t* __restrict__ block1 = out + 4 * neon_smmla::TileSize;
|
||||
|
||||
int8x16_t a0 = vld1q_s8(row0 + k_idx);
|
||||
int8x16_t a1 = vld1q_s8(row1 + k_idx);
|
||||
vst1q_s8(block0, vcombine_s8(vget_low_s8(a0), vget_low_s8(a1)));
|
||||
vst1q_s8(block1, vcombine_s8(vget_high_s8(a0), vget_high_s8(a1)));
|
||||
|
||||
a0 = vld1q_s8(row2 + k_idx);
|
||||
a1 = vld1q_s8(row3 + k_idx);
|
||||
vst1q_s8(block0 + neon_smmla::TileSize,
|
||||
vcombine_s8(vget_low_s8(a0), vget_low_s8(a1)));
|
||||
vst1q_s8(block1 + neon_smmla::TileSize,
|
||||
vcombine_s8(vget_high_s8(a0), vget_high_s8(a1)));
|
||||
|
||||
a0 = vld1q_s8(row4 + k_idx);
|
||||
a1 = vld1q_s8(row5 + k_idx);
|
||||
vst1q_s8(block0 + 2 * neon_smmla::TileSize,
|
||||
vcombine_s8(vget_low_s8(a0), vget_low_s8(a1)));
|
||||
vst1q_s8(block1 + 2 * neon_smmla::TileSize,
|
||||
vcombine_s8(vget_high_s8(a0), vget_high_s8(a1)));
|
||||
|
||||
a0 = vld1q_s8(row6 + k_idx);
|
||||
a1 = vld1q_s8(row7 + k_idx);
|
||||
vst1q_s8(block0 + 3 * neon_smmla::TileSize,
|
||||
vcombine_s8(vget_low_s8(a0), vget_low_s8(a1)));
|
||||
vst1q_s8(block1 + 3 * neon_smmla::TileSize,
|
||||
vcombine_s8(vget_high_s8(a0), vget_high_s8(a1)));
|
||||
|
||||
out += 8 * neon_smmla::TileSize;
|
||||
}
|
||||
|
||||
for (; k_idx < k; k_idx += K) {
|
||||
int8x8_t a0 = vld1_s8(row0 + k_idx);
|
||||
int8x8_t a1 = vld1_s8(row1 + k_idx);
|
||||
vst1q_s8(out, vcombine_s8(a0, a1));
|
||||
|
||||
a0 = vld1_s8(row2 + k_idx);
|
||||
a1 = vld1_s8(row3 + k_idx);
|
||||
vst1q_s8(out + neon_smmla::TileSize, vcombine_s8(a0, a1));
|
||||
|
||||
a0 = vld1_s8(row4 + k_idx);
|
||||
a1 = vld1_s8(row5 + k_idx);
|
||||
vst1q_s8(out + 2 * neon_smmla::TileSize, vcombine_s8(a0, a1));
|
||||
|
||||
a0 = vld1_s8(row6 + k_idx);
|
||||
a1 = vld1_s8(row7 + k_idx);
|
||||
vst1q_s8(out + 3 * neon_smmla::TileSize, vcombine_s8(a0, a1));
|
||||
|
||||
out += 4 * neon_smmla::TileSize;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const int32_t row_pairs = (panel_m <= 4) ? 2 : Mr / 2;
|
||||
for (int32_t k_idx = 0; k_idx < k; k_idx += K) {
|
||||
for (int32_t pair_idx = 0; pair_idx < row_pairs; ++pair_idx) {
|
||||
const int32_t row_idx = pair_idx * 2;
|
||||
const int8x8_t row0 =
|
||||
(row_idx < panel_m) ? vld1_s8(panel_rows[row_idx] + k_idx) : zero;
|
||||
const int8x8_t row1 = (row_idx + 1 < panel_m)
|
||||
? vld1_s8(panel_rows[row_idx + 1] + k_idx)
|
||||
: zero;
|
||||
vst1q_s8(out, vcombine_s8(row0, row1));
|
||||
out += neon_smmla::TileSize;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// physical layout [
|
||||
// N / 8; Nr is 8
|
||||
// K / 8; K for smmla is 8
|
||||
// 4, ; 4 col-pairs for each 8 cols
|
||||
// 2, ; col-pair is 2 cols
|
||||
// 4 ; 4 elements per col
|
||||
// ]
|
||||
static void pack_weight(const int8_t* __restrict__ weight,
|
||||
int8_t* __restrict__ packed_weight,
|
||||
const int32_t output_size, const int32_t input_size) {
|
||||
TORCH_CHECK(output_size % NSize == 0);
|
||||
TORCH_CHECK(input_size % K == 0);
|
||||
|
||||
for (int32_t o_idx = 0; o_idx < output_size; o_idx += Nr) {
|
||||
int8_t* __restrict__ dst = packed_weight + o_idx * input_size;
|
||||
for (int32_t k_idx = 0; k_idx < input_size; k_idx += K) {
|
||||
for (int32_t pair_idx = 0; pair_idx < Nr;
|
||||
pair_idx += neon_smmla::Cols) {
|
||||
const int8_t* __restrict__ row0 =
|
||||
weight + (o_idx + pair_idx) * input_size + k_idx;
|
||||
const int8_t* __restrict__ row1 = row0 + input_size;
|
||||
vst1q_s8(dst, vcombine_s8(vld1_s8(row0), vld1_s8(row1)));
|
||||
dst += neon_smmla::TileSize;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void gemm(const int8_t* __restrict__ a_packed,
|
||||
const int8_t* __restrict__ b_packed, int32_t* __restrict__ c,
|
||||
const int32_t m, const int32_t k, const int64_t b_n_group_stride,
|
||||
const int64_t ldc) const {
|
||||
TORCH_CHECK(m > 0 && m <= MaxMSize);
|
||||
TORCH_CHECK(k % K == 0);
|
||||
|
||||
for (int32_t n_idx = 0; n_idx < NSize; n_idx += NrGemv) {
|
||||
const int8_t* __restrict__ b_panel = b_packed + n_idx * k;
|
||||
|
||||
for (int32_t row_base = 0; row_base < m; row_base += Mr) {
|
||||
const int32_t panel_m = std::min(Mr, m - row_base);
|
||||
const int8_t* __restrict__ a_panel = a_packed + row_base * k;
|
||||
int32_t* __restrict__ c_panel = c + row_base * ldc + n_idx;
|
||||
|
||||
if (panel_m <= 4) {
|
||||
neon_smmla::gemm_micro_smmla_4x16_packed_a(
|
||||
a_panel, b_panel, c_panel, panel_m, k, b_n_group_stride, ldc);
|
||||
} else {
|
||||
neon_smmla::gemm_micro_smmla_8x8_packed_a(a_panel, b_panel, c_panel,
|
||||
panel_m, k, ldc);
|
||||
neon_smmla::gemm_micro_smmla_8x8_packed_a(
|
||||
a_panel, b_panel + b_n_group_stride, c_panel + Nr, panel_m, k,
|
||||
ldc);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace cpu_micro_gemm
|
||||
|
||||
#endif
|
||||
@@ -1,3 +1,6 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
#ifndef CPU_MICRO_GEMM_NEON_HPP
|
||||
#define CPU_MICRO_GEMM_NEON_HPP
|
||||
|
||||
@@ -16,9 +19,6 @@ namespace {
|
||||
constexpr int32_t K = 4;
|
||||
constexpr int32_t Cols = 2;
|
||||
constexpr int32_t TileSize = K * Cols;
|
||||
constexpr int32_t Mr = 8;
|
||||
constexpr int32_t Nr = 8;
|
||||
constexpr int32_t Nr_gemv = 16;
|
||||
|
||||
// a = [a0, a1, a2, a3], b = [b0, b1, b2, b3] -> [a0, a1, b0, b1]
|
||||
FORCE_INLINE float32x4_t zip1_f32x4(const float32x4_t a, const float32x4_t b) {
|
||||
@@ -132,7 +132,7 @@ FORCE_INLINE void gemm_micro_bfmmla_8x8_packed_a(
|
||||
acc6767 = vbfmmlaq_f32(acc6767, a_tile67, b_tile67);
|
||||
|
||||
a_tile += 4 * TileSize;
|
||||
b_tile += Nr * K;
|
||||
b_tile += 4 * TileSize;
|
||||
}
|
||||
|
||||
store_acc_rowpair(acc0101, acc0123, acc0145, acc0167, c_ptr, ldc,
|
||||
@@ -205,8 +205,8 @@ FORCE_INLINE void gemm_micro_bfmmla_4x16_packed_a(
|
||||
acc231415 = vbfmmlaq_f32(acc231415, a_tile23, b_tile1415);
|
||||
|
||||
a_tile += 2 * TileSize;
|
||||
b_tile0 += Nr * K;
|
||||
b_tile1 += Nr * K;
|
||||
b_tile0 += 4 * TileSize;
|
||||
b_tile1 += 4 * TileSize;
|
||||
}
|
||||
|
||||
store_acc_rowpair(acc0101, acc0123, acc0145, acc0167, c_ptr, ldc, m_rows_01);
|
||||
@@ -223,6 +223,9 @@ FORCE_INLINE void gemm_micro_bfmmla_4x16_packed_a(
|
||||
template <typename scalar_t>
|
||||
class MicroGemm<cpu_utils::ISA::NEON, scalar_t> {
|
||||
public:
|
||||
static constexpr int32_t Mr = 8;
|
||||
static constexpr int32_t Nr = 8;
|
||||
static constexpr int32_t NrGemv = 16;
|
||||
static constexpr int32_t MaxMSize = 8;
|
||||
static constexpr int32_t NSize = 32;
|
||||
static constexpr int32_t WeightOCGroupSize = Nr;
|
||||
@@ -246,6 +249,9 @@ class MicroGemm<cpu_utils::ISA::NEON, c10::BFloat16> {
|
||||
public:
|
||||
using scalar_t = c10::BFloat16;
|
||||
|
||||
static constexpr int32_t Mr = 8;
|
||||
static constexpr int32_t Nr = 8;
|
||||
static constexpr int32_t NrGemv = 16;
|
||||
static constexpr int32_t MaxMSize = 8;
|
||||
static constexpr int32_t NSize = 32;
|
||||
static constexpr int32_t WeightOCGroupSize = Nr;
|
||||
@@ -253,7 +259,7 @@ class MicroGemm<cpu_utils::ISA::NEON, c10::BFloat16> {
|
||||
|
||||
public:
|
||||
// physical layout [
|
||||
// M / 8; Mr is 8
|
||||
// M / (8 or 4); Mr is 8 or 4
|
||||
// K / 4; K for bfmmla is 4
|
||||
// 4, ; 4 row-pairs for each 8 rows
|
||||
// 2, ; row-pair is 2 rows
|
||||
@@ -439,7 +445,7 @@ class MicroGemm<cpu_utils::ISA::NEON, c10::BFloat16> {
|
||||
(void)lda; // A is packed, so lda is not needed
|
||||
TORCH_CHECK_EQ(k % K, 0);
|
||||
|
||||
for (int32_t n_idx = 0; n_idx < NSize; n_idx += Nr_gemv) {
|
||||
for (int32_t n_idx = 0; n_idx < NSize; n_idx += NrGemv) {
|
||||
const bfloat16_t* __restrict__ b_panel =
|
||||
reinterpret_cast<const bfloat16_t*>(b_ptr) + n_idx * k;
|
||||
|
||||
|
||||
@@ -207,6 +207,20 @@ void cpu_fused_moe(torch::Tensor& output, const torch::Tensor& input,
|
||||
const torch::Tensor& topk_id, const bool skip_weighted,
|
||||
const std::string& act, const std::string& isa);
|
||||
|
||||
void prepack_moe_weight_int8(const torch::Tensor& weight,
|
||||
torch::Tensor& packed_weight,
|
||||
const std::string& isa);
|
||||
|
||||
void cpu_fused_moe_int8(torch::Tensor& output, const torch::Tensor& input,
|
||||
const torch::Tensor& w13, const torch::Tensor& w2,
|
||||
const torch::Tensor& w13_scale,
|
||||
const torch::Tensor& w2_scale,
|
||||
const std::optional<torch::Tensor>& w13_bias,
|
||||
const std::optional<torch::Tensor>& w2_bias,
|
||||
const torch::Tensor& topk_weights,
|
||||
const torch::Tensor& topk_id, const bool skip_weighted,
|
||||
const std::string& act, const std::string& isa);
|
||||
|
||||
void compute_slot_mapping_kernel_impl(const torch::Tensor query_start_loc,
|
||||
const torch::Tensor positions,
|
||||
const torch::Tensor block_table,
|
||||
@@ -596,8 +610,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
|
||||
#endif
|
||||
|
||||
// fused moe
|
||||
#if defined(__AVX512F__) || \
|
||||
(defined(__aarch64__) && !defined(__APPLE__) && defined(ARM_BF16_SUPPORT))
|
||||
#if defined(__AVX512F__) || (defined(ARM_BF16_SUPPORT) && !defined(__APPLE__))
|
||||
ops.def(
|
||||
"prepack_moe_weight(Tensor weight, Tensor(a1!) packed_weight, str isa) "
|
||||
"-> ()");
|
||||
@@ -608,7 +621,22 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
|
||||
"bool skip_weighted, "
|
||||
"str act, str isa) -> ()");
|
||||
ops.impl("cpu_fused_moe", torch::kCPU, &cpu_fused_moe);
|
||||
#endif
|
||||
#endif // #if defined(__AVX512F__) || (defined(ARM_BF16_SUPPORT) &&
|
||||
// !defined(__APPLE__))
|
||||
#if defined(ARM_I8MM_SUPPORT) && defined(ARM_BF16_SUPPORT) && \
|
||||
!defined(__APPLE__)
|
||||
ops.def(
|
||||
"prepack_moe_weight_int8(Tensor weight, Tensor(a1!) packed_weight, "
|
||||
"str isa) -> ()");
|
||||
ops.impl("prepack_moe_weight_int8", torch::kCPU, &prepack_moe_weight_int8);
|
||||
ops.def(
|
||||
"cpu_fused_moe_int8(Tensor(a0!) output, Tensor input, Tensor w13, "
|
||||
"Tensor w2, Tensor w13_scale, Tensor w2_scale, Tensor? w13_bias, "
|
||||
"Tensor? w2_bias, Tensor topk_weights, Tensor topk_id, bool "
|
||||
"skip_weighted, str act, str isa) -> ()");
|
||||
ops.impl("cpu_fused_moe_int8", torch::kCPU, &cpu_fused_moe_int8);
|
||||
#endif // #if defined(ARM_I8MM_SUPPORT) && defined(ARM_BF16_SUPPORT) &&
|
||||
// !defined(__APPLE__)
|
||||
ops.def(
|
||||
"mla_decode_kvcache("
|
||||
" Tensor! out, Tensor query, Tensor kv_cache,"
|
||||
|
||||
@@ -7,10 +7,14 @@ import torch
|
||||
from tests.kernels.allclose_default import get_default_atol, get_default_rtol
|
||||
from vllm._custom_ops import (
|
||||
cpu_fused_moe,
|
||||
cpu_fused_moe_int8,
|
||||
cpu_prepack_moe_weight,
|
||||
cpu_prepack_moe_weight_int8,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.cpu_fused_moe import _CPU_MOE_ACT_FN
|
||||
from vllm.model_executor.layers.fused_moe.cpu_fused_moe import (
|
||||
_CPU_MOE_ACT_FN,
|
||||
)
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
@@ -101,6 +105,90 @@ def ref_fused_moe(
|
||||
return final_out
|
||||
|
||||
|
||||
def quantize_per_channel(
|
||||
weight: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Symmetrically quantize each weight output channel."""
|
||||
weight_fp32 = weight.float()
|
||||
scale = weight_fp32.abs().amax(dim=-1).clamp_min(1e-12) / 127.0
|
||||
quantized = (
|
||||
(weight_fp32 / scale.unsqueeze(-1)).round().clamp(-127, 127).to(torch.int8)
|
||||
)
|
||||
return quantized, scale
|
||||
|
||||
|
||||
def quantize_per_token(
|
||||
input: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Symmetrically quantize each input token."""
|
||||
input_fp32 = input.float()
|
||||
scale = input_fp32.abs().amax(dim=-1, keepdim=True).clamp_min(1e-7) / 127.0
|
||||
quantized = (input_fp32 / scale).round().clamp(-127, 127).to(torch.int8)
|
||||
return quantized, scale
|
||||
|
||||
|
||||
def ref_fused_moe_int8(
|
||||
input: torch.Tensor,
|
||||
w13: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
w2_scale: torch.Tensor,
|
||||
w13_bias: torch.Tensor | None,
|
||||
w2_bias: torch.Tensor | None,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
) -> torch.Tensor:
|
||||
"""Reference the two dynamically quantized INT8 GEMMs."""
|
||||
input_int8, input_scale = quantize_per_token(input)
|
||||
expert_num = w13.size(0)
|
||||
|
||||
counts = topk_ids.new_zeros((topk_ids.size(0), expert_num))
|
||||
counts.scatter_(1, topk_ids.to(torch.int64), 1)
|
||||
tokens_per_expert = counts.sum(dim=0).tolist()
|
||||
sorted_route_ids = topk_ids.view(-1).argsort()
|
||||
sorted_token_ids = sorted_route_ids // topk_ids.size(1)
|
||||
|
||||
outputs = []
|
||||
start_idx = 0
|
||||
for expert_idx, token_count in enumerate(tokens_per_expert):
|
||||
end_idx = start_idx + token_count
|
||||
if token_count == 0:
|
||||
continue
|
||||
|
||||
token_ids = sorted_token_ids[start_idx:end_idx]
|
||||
gate_up = torch.matmul(
|
||||
input_int8[token_ids].float(),
|
||||
w13[expert_idx].float().T,
|
||||
)
|
||||
gate_up *= input_scale[token_ids] * w13_scale[expert_idx]
|
||||
if w13_bias is not None:
|
||||
gate_up += w13_bias[expert_idx].float()
|
||||
|
||||
intermediate = _CPU_MOE_ACT_FN[activation](gate_up).to(input.dtype)
|
||||
intermediate_int8, intermediate_scale = quantize_per_token(intermediate)
|
||||
output = torch.matmul(
|
||||
intermediate_int8.float(),
|
||||
w2[expert_idx].float().T,
|
||||
)
|
||||
output *= intermediate_scale * w2_scale[expert_idx]
|
||||
if w2_bias is not None:
|
||||
output += w2_bias[expert_idx].float()
|
||||
|
||||
outputs.append(output)
|
||||
start_idx = end_idx
|
||||
|
||||
sorted_output = torch.cat(outputs, dim=0)
|
||||
routed_output = torch.empty_like(sorted_output)
|
||||
routed_output[sorted_route_ids] = sorted_output
|
||||
return (
|
||||
routed_output.view(*topk_ids.shape, -1)
|
||||
.mul_(topk_weights.unsqueeze(-1))
|
||||
.sum(dim=1)
|
||||
.to(input.dtype)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("batch_size", BATCH_SIZE)
|
||||
@pytest.mark.parametrize("expert_num", EXPERT_NUM)
|
||||
@pytest.mark.parametrize("hidden_size", HIDDEN_DIM)
|
||||
@@ -176,3 +264,85 @@ def test_cpu_fused_moe(
|
||||
torch.testing.assert_close(output, ref_output, atol=atol, rtol=rtol),
|
||||
f"{torch.max(torch.abs(output - ref_output))}",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
current_platform.get_cpu_architecture() != CpuArchEnum.ARM,
|
||||
reason="Requires Arm CPU",
|
||||
)
|
||||
@pytest.mark.parametrize("batch_size", BATCH_SIZE)
|
||||
@pytest.mark.parametrize("expert_num", EXPERT_NUM)
|
||||
@pytest.mark.parametrize("hidden_size", HIDDEN_DIM)
|
||||
@pytest.mark.parametrize("intermediate_size", INTERMEDIATE_DIM)
|
||||
@pytest.mark.parametrize("use_bias", USE_BIAS)
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("act", ACT)
|
||||
@pytest.mark.parametrize("isa", ["neon"])
|
||||
def test_cpu_fused_moe_int8(
|
||||
batch_size: int,
|
||||
expert_num: int,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
use_bias: bool,
|
||||
dtype: torch.dtype,
|
||||
act: MoEActivation,
|
||||
isa: str,
|
||||
):
|
||||
set_random_seed(0)
|
||||
topk_num = max(expert_num // 2, 1)
|
||||
up_dim = 2 * intermediate_size
|
||||
|
||||
input = torch.randn((batch_size, hidden_size), dtype=dtype) / (
|
||||
0.5 * hidden_size**0.5
|
||||
)
|
||||
w13_fp = torch.randn((expert_num, up_dim, hidden_size), dtype=dtype) / (
|
||||
0.5 * hidden_size**0.5
|
||||
)
|
||||
w2_fp = torch.randn((expert_num, hidden_size, intermediate_size), dtype=dtype) / (
|
||||
0.5 * intermediate_size**0.5
|
||||
)
|
||||
w13, w13_scale = quantize_per_channel(w13_fp)
|
||||
w2, w2_scale = quantize_per_channel(w2_fp)
|
||||
|
||||
w13_bias = None
|
||||
w2_bias = None
|
||||
if use_bias:
|
||||
w13_bias = torch.randn((expert_num, up_dim), dtype=dtype) / (0.5 * up_dim**0.5)
|
||||
w2_bias = torch.randn((expert_num, hidden_size), dtype=dtype) / (
|
||||
0.5 * hidden_size**0.5
|
||||
)
|
||||
|
||||
router_logits = torch.randn((batch_size, expert_num), dtype=dtype)
|
||||
score = torch.softmax(router_logits, dim=-1, dtype=torch.float32)
|
||||
topk_weights, topk_ids = torch.topk(score, topk_num)
|
||||
topk_ids = topk_ids.to(torch.int32)
|
||||
|
||||
ref_output = ref_fused_moe_int8(
|
||||
input,
|
||||
w13,
|
||||
w2,
|
||||
w13_scale,
|
||||
w2_scale,
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
act,
|
||||
)
|
||||
packed_w13 = cpu_prepack_moe_weight_int8(w13, isa)
|
||||
packed_w2 = cpu_prepack_moe_weight_int8(w2, isa)
|
||||
output = cpu_fused_moe_int8(
|
||||
input,
|
||||
packed_w13,
|
||||
packed_w2,
|
||||
w13_scale,
|
||||
w2_scale,
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
act.value,
|
||||
isa,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(output, ref_output, atol=2e-2, rtol=2e-2)
|
||||
|
||||
@@ -3787,6 +3787,15 @@ def cpu_prepack_moe_weight(
|
||||
return output
|
||||
|
||||
|
||||
def cpu_prepack_moe_weight_int8(
|
||||
weight: torch.Tensor,
|
||||
isa: str,
|
||||
) -> torch.Tensor:
|
||||
output = torch.empty_like(weight)
|
||||
torch.ops._C.prepack_moe_weight_int8(weight, output, isa)
|
||||
return output
|
||||
|
||||
|
||||
def cpu_fused_moe(
|
||||
input: torch.Tensor,
|
||||
w13: torch.Tensor,
|
||||
@@ -3816,6 +3825,39 @@ def cpu_fused_moe(
|
||||
return output
|
||||
|
||||
|
||||
def cpu_fused_moe_int8(
|
||||
input: torch.Tensor,
|
||||
w13: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
w2_scale: torch.Tensor,
|
||||
w13_bias: torch.Tensor | None,
|
||||
w2_bias: torch.Tensor | None,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
act: str,
|
||||
isa: str,
|
||||
skip_weighted: bool = False,
|
||||
) -> torch.Tensor:
|
||||
output = torch.empty_like(input)
|
||||
torch.ops._C.cpu_fused_moe_int8(
|
||||
output,
|
||||
input,
|
||||
w13,
|
||||
w2,
|
||||
w13_scale,
|
||||
w2_scale,
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
skip_weighted,
|
||||
act,
|
||||
isa,
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
if hasattr(torch.ops._qutlass_C, "matmul_mxf4_bf16_tn"):
|
||||
|
||||
@register_fake("_qutlass_C::matmul_mxf4_bf16_tn")
|
||||
|
||||
@@ -9,6 +9,8 @@ from vllm._custom_ops import (
|
||||
CPUQuantAlgo,
|
||||
CPUQuantMethod,
|
||||
convert_weight_packed_scale_zp,
|
||||
cpu_fused_moe_int8,
|
||||
cpu_prepack_moe_weight_int8,
|
||||
fused_experts_cpu,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
@@ -27,7 +29,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kInt8StaticChannelSym,
|
||||
kMxfp4Static,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
|
||||
# ===========================================================================
|
||||
# FP8 W8A16 MoE
|
||||
@@ -556,7 +558,14 @@ def prepare_int8_moe_layer_for_cpu(
|
||||
w13: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""VNNI-prepack INT8 MoE weights for CPU kernel."""
|
||||
"""Prepack INT8 MoE weights for the current CPU architecture."""
|
||||
# SMMLA packing for AArch64
|
||||
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
|
||||
return (
|
||||
cpu_prepack_moe_weight_int8(w13, "neon"),
|
||||
cpu_prepack_moe_weight_int8(w2, "neon"),
|
||||
)
|
||||
# VNNI packing for x86
|
||||
packed_w13 = torch.ops._C.convert_weight_packed(w13)
|
||||
packed_w2 = torch.ops._C.convert_weight_packed(w2)
|
||||
return packed_w13, packed_w2
|
||||
@@ -586,7 +595,10 @@ class CPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
return current_platform.is_cpu()
|
||||
return (
|
||||
current_platform.is_cpu()
|
||||
and current_platform.get_cpu_architecture() == CpuArchEnum.X86
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _supports_no_act_and_mul() -> bool:
|
||||
@@ -701,3 +713,167 @@ class CPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
|
||||
None, # limit
|
||||
True, # is_vnni
|
||||
)
|
||||
|
||||
|
||||
class ArmCPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
|
||||
"""Arm INT8 MoE with per-token activation and channelwise weight quantization."""
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
return mk.FusedMoEActivationFormat.Standard
|
||||
|
||||
@staticmethod
|
||||
def is_supported_config(
|
||||
cls: type[mk.FusedMoEExperts],
|
||||
moe_config: FusedMoEConfig,
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
activation_format: mk.FusedMoEActivationFormat,
|
||||
) -> tuple[bool, str | None]:
|
||||
supported, reason = mk.FusedMoEExperts.is_supported_config(
|
||||
cls,
|
||||
moe_config,
|
||||
weight_key,
|
||||
activation_key,
|
||||
activation_format,
|
||||
)
|
||||
if not supported:
|
||||
return supported, reason
|
||||
if moe_config.in_dtype not in (
|
||||
torch.float32,
|
||||
torch.float16,
|
||||
torch.bfloat16,
|
||||
):
|
||||
return False, "kernel requires float32, float16, or bfloat16 activations"
|
||||
if moe_config.hidden_dim % 32 != 0:
|
||||
return False, "kernel requires hidden dim divisible by 32"
|
||||
if moe_config.intermediate_size_per_partition % 32 != 0:
|
||||
return False, "kernel requires intermediate dim divisible by 32"
|
||||
return True, None
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
return (
|
||||
current_platform.is_cpu()
|
||||
and current_platform.get_cpu_architecture() == CpuArchEnum.ARM
|
||||
and hasattr(torch.ops._C, "cpu_fused_moe_int8")
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _supports_no_act_and_mul() -> bool:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
return activation in (
|
||||
MoEActivation.SILU,
|
||||
MoEActivation.SWIGLUOAI,
|
||||
MoEActivation.GELU,
|
||||
MoEActivation.GELU_TANH,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(
|
||||
moe_parallel_config: FusedMoEParallelConfig,
|
||||
) -> bool:
|
||||
return not moe_parallel_config.use_ep
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
return (weight_key, activation_key) == (
|
||||
kInt8StaticChannelSym,
|
||||
kInt8DynamicTokenSym,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _supports_routing_method(
|
||||
routing_method: RoutingMethodType,
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
return routing_method in [
|
||||
RoutingMethodType.Default,
|
||||
RoutingMethodType.Renormalize,
|
||||
RoutingMethodType.RenormalizeNaive,
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _supports_router_logits_dtype(
|
||||
router_logits_dtype: torch.dtype | None,
|
||||
routing_method: RoutingMethodType,
|
||||
) -> bool:
|
||||
return True
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
|
||||
w13, w2 = prepare_int8_moe_layer_for_cpu(layer.w13_weight, layer.w2_weight)
|
||||
replace_parameter(layer, "w13_weight", w13)
|
||||
replace_parameter(layer, "w2_weight", w2)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
num_expert_group: int | None = None,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.model_executor.layers.fused_moe.cpu_fused_moe import (
|
||||
select_experts,
|
||||
)
|
||||
|
||||
topk_weights, topk_ids = select_experts(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
use_grouped_topk=num_expert_group is not None,
|
||||
top_k=self.moe_config.experts_per_token,
|
||||
renormalize=self.moe_config.routing_method
|
||||
in (
|
||||
RoutingMethodType.Renormalize,
|
||||
RoutingMethodType.RenormalizeNaive,
|
||||
),
|
||||
topk_group=topk_group,
|
||||
num_expert_group=num_expert_group,
|
||||
scoring_func="softmax",
|
||||
routed_scaling_factor=(
|
||||
routed_scaling_factor if routed_scaling_factor is not None else 1.0
|
||||
),
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
)
|
||||
|
||||
if apply_router_weight_on_input:
|
||||
assert topk_ids.size(1) == 1
|
||||
hidden_states.mul_(topk_weights.to(hidden_states.dtype))
|
||||
|
||||
assert self.w1_scale is not None
|
||||
assert self.w2_scale is not None
|
||||
return cpu_fused_moe_int8(
|
||||
hidden_states,
|
||||
w1,
|
||||
w2,
|
||||
self.w1_scale,
|
||||
self.w2_scale,
|
||||
self.w1_bias,
|
||||
self.w2_bias,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
activation.value,
|
||||
"neon",
|
||||
skip_weighted=apply_router_weight_on_input,
|
||||
)
|
||||
|
||||
@@ -24,7 +24,6 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kInt8StaticChannelSym,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -41,20 +40,12 @@ def _get_priority_backends(
|
||||
"""
|
||||
Get available backends in priority order based on platform and config.
|
||||
"""
|
||||
_AVAILABLE_BACKENDS = [
|
||||
return [
|
||||
Int8MoeBackend.TRITON,
|
||||
Int8MoeBackend.HUMMING,
|
||||
Int8MoeBackend.CPU,
|
||||
]
|
||||
|
||||
def _move_to_front(backends: list[Int8MoeBackend], backend: Int8MoeBackend) -> None:
|
||||
backends.insert(0, backends.pop(backends.index(backend)))
|
||||
|
||||
if current_platform.is_cpu():
|
||||
_move_to_front(_AVAILABLE_BACKENDS, Int8MoeBackend.CPU)
|
||||
|
||||
return _AVAILABLE_BACKENDS
|
||||
|
||||
|
||||
def backend_to_kernel_cls(
|
||||
backend: Int8MoeBackend,
|
||||
@@ -78,14 +69,13 @@ def backend_to_kernel_cls(
|
||||
HummingGroupedExperts,
|
||||
HummingIndexedExperts,
|
||||
]
|
||||
|
||||
elif backend == Int8MoeBackend.CPU:
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
ArmCPUExpertsInt8,
|
||||
CPUExpertsInt8,
|
||||
)
|
||||
|
||||
return [CPUExpertsInt8]
|
||||
|
||||
return [ArmCPUExpertsInt8, CPUExpertsInt8]
|
||||
else:
|
||||
raise ValueError(f"Unknown Int8 MoE backend: {backend.value}")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user