diff --git a/cmake/cpu_extension.cmake b/cmake/cpu_extension.cmake index 4e7837df0d..64df94e947 100644 --- a/cmake/cpu_extension.cmake +++ b/cmake/cpu_extension.cmake @@ -15,6 +15,7 @@ endif() # set(ENABLE_X86_ISA $ENV{VLLM_CPU_X86}) set(ENABLE_ARM_BF16 $ENV{VLLM_CPU_ARM_BF16}) +set(ENABLE_ARM_I8MM $ENV{VLLM_CPU_ARM_I8MM}) set(ENABLE_RVV_BF16 $ENV{VLLM_CPU_RVV_BF16}) include_directories("${CMAKE_SOURCE_DIR}/csrc") @@ -96,12 +97,14 @@ if (MACOSX_FOUND AND CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64") set(ENABLE_NUMA OFF) check_sysctl(hw.optional.neon ASIMD_FOUND) check_sysctl(hw.optional.arm.FEAT_BF16 ARM_BF16_FOUND) + check_sysctl(hw.optional.arm.FEAT_I8MM ARM_I8MM_FOUND) else() find_isa(${CPUINFO} "Power11" POWER11_FOUND) find_isa(${CPUINFO} "POWER10" POWER10_FOUND) find_isa(${CPUINFO} "POWER9" POWER9_FOUND) find_isa(${CPUINFO} "asimd" ASIMD_FOUND) # Check for ARM NEON support find_isa(${CPUINFO} "bf16" ARM_BF16_FOUND) # Check for ARM BF16 support + find_isa(${CPUINFO} "i8mm" ARM_I8MM_FOUND) # Check for ARM I8MM support find_isa(${CPUINFO} "S390" S390_FOUND) find_isa(${CPUINFO} "zvfhmin" RVV_FP16_FOUND) # Check for RISC-V Vector FP16 support find_isa(${CPUINFO} "zvfbfmin" RVV_BF16_FOUND) # Check for RISC-V Vector BF16 support @@ -111,6 +114,11 @@ else() set(ARM_BF16_FOUND ON) message(STATUS "ARM BF16 support enabled via VLLM_CPU_ARM_BF16 environment variable") endif() + if (ENABLE_ARM_I8MM) + set(ARM_I8MM_FOUND ON) + message(STATUS + "ARM I8MM support enabled via VLLM_CPU_ARM_I8MM environment variable") + endif() # Some kernels (e.g. Bianbu on Spacemit X100) do not report zvfbfmin # in /proc/cpuinfo despite hardware support. VLLM_CPU_RVV_BF16=1 # overrides the detection result. @@ -166,6 +174,11 @@ elseif (ASIMD_FOUND) message(WARNING "BF16 functionality is not available") set(MARCH_FLAGS "-march=armv8.2-a+dotprod+fp16") endif() + if(ARM_I8MM_FOUND) + message(STATUS "I8MM extension detected") + string(APPEND MARCH_FLAGS "+i8mm") + add_compile_definitions(ARM_I8MM_SUPPORT) + endif() list(APPEND CXX_COMPILE_FLAGS ${MARCH_FLAGS}) elseif (S390_FOUND) message(STATUS "S390 detected") @@ -447,8 +460,13 @@ if (ASIMD_FOUND AND NOT APPLE_SILICON_FOUND) "csrc/cpu/shm.cpp" "csrc/cpu/activation_lut_bf16.cpp" "csrc/cpu/cpu_tanhf_neon.hpp" - "csrc/cpu/cpu_fused_moe.cpp" ${VLLM_EXT_SRC}) + if (ARM_BF16_FOUND) + set(VLLM_EXT_SRC "csrc/cpu/cpu_fused_moe.cpp" ${VLLM_EXT_SRC}) + if (ARM_I8MM_FOUND) + set(VLLM_EXT_SRC "csrc/cpu/cpu_fused_moe_int8.cpp" ${VLLM_EXT_SRC}) + endif() + endif() endif() if (POWER9_FOUND OR POWER10_FOUND OR POWER11_FOUND) diff --git a/csrc/cpu/cpu_fused_moe.cpp b/csrc/cpu/cpu_fused_moe.cpp index 07b0aaf868..b7bbef4c2d 100644 --- a/csrc/cpu/cpu_fused_moe.cpp +++ b/csrc/cpu/cpu_fused_moe.cpp @@ -1,5 +1,6 @@ #include "cpu/cpu_types.hpp" #include "cpu/utils.hpp" +#include "cpu/cpu_fused_moe_activations.hpp" #include "cpu/micro_gemm/cpu_micro_gemm_vec.hpp" #include "cpu/cpu_arch_macros.h" @@ -43,193 +44,9 @@ }() namespace { -enum class FusedMOEAct { - SiluAndMul, - SwigluOAIAndMul, - GeluAndMul, - GeluTanhAndMul, -}; -FusedMOEAct get_act_type(const std::string& act) { - if (act == "silu") { - return FusedMOEAct::SiluAndMul; - } else if (act == "swigluoai") { - return FusedMOEAct::SwigluOAIAndMul; - } else if (act == "gelu") { - return FusedMOEAct::GeluAndMul; - } else if (act == "gelu_tanh") { - return FusedMOEAct::GeluTanhAndMul; - } else { - TORCH_CHECK(false, "Invalid act type: " + act); - } -} - -template -void swigluoai_and_mul(float* __restrict__ input, scalar_t* __restrict__ output, - const int32_t m_size, const int32_t n_size, - const int32_t input_stride, - const int32_t output_stride) { - using scalar_vec_t = typename cpu_utils::VecTypeTrait::vec_t; -#if !defined(__aarch64__) - // For GPT-OSS interleaved gate-up weights - alignas(64) static int32_t index[16] = {0, 2, 4, 6, 8, 10, 12, 14, - 16, 18, 20, 22, 24, 26, 28, 30}; - vec_op::INT32Vec16 index_vec(index); -#endif - vec_op::FP32Vec16 gate_up_max_vec(7.0); - vec_op::FP32Vec16 up_min_vec(-7.0); - vec_op::FP32Vec16 alpha_vec(1.702); - vec_op::FP32Vec16 one_vec(1.0); - - DEFINE_FAST_EXP - - for (int32_t m = 0; m < m_size; ++m) { - for (int32_t n = 0; n < n_size; n += 32) { - // Note: AdvSIMD does not support gather loads -#if defined(__aarch64__) - vec_op::FP32Vec16 gate_vec(vec_op::uninit); - vec_op::FP32Vec16 up_vec(vec_op::uninit); - vec_op::FP32Vec16::load_even_odd(input + n, gate_vec, up_vec); -#else - vec_op::FP32Vec16 gate_vec(input + n, index_vec); - vec_op::FP32Vec16 up_vec(input + n + 1, index_vec); -#endif - gate_vec = gate_vec.min(gate_up_max_vec); - up_vec = up_vec.clamp(up_min_vec, gate_up_max_vec); - auto sigmoid_vec = one_vec / (one_vec + fast_exp(-gate_vec * alpha_vec)); - auto glu = gate_vec * sigmoid_vec; - auto gated_output_fp32 = (one_vec + up_vec) * glu; - scalar_vec_t gated_output = scalar_vec_t(gated_output_fp32); - gated_output.save(output + n / 2); - } - input += input_stride; - output += output_stride; - } -} - -template -void silu_and_mul(float* __restrict__ input, scalar_t* __restrict__ output, - const int32_t m_size, const int32_t n_size, - const int32_t input_stride, const int32_t output_stride) { - using scalar_vec_t = typename cpu_utils::VecTypeTrait::vec_t; - const int32_t dim = n_size / 2; - float* __restrict__ gate = input; - float* __restrict__ up = input + dim; - vec_op::FP32Vec16 one_vec(1.0); - - DEFINE_FAST_EXP - - for (int32_t m = 0; m < m_size; ++m) { - for (int32_t n = 0; n < dim; n += 16) { - vec_op::FP32Vec16 gate_vec(gate + n); - vec_op::FP32Vec16 up_vec(up + n); - auto sigmoid_vec = one_vec / (one_vec + fast_exp(-gate_vec)); - auto silu = gate_vec * sigmoid_vec; - auto gated_output_fp32 = up_vec * silu; - scalar_vec_t gated_output = scalar_vec_t(gated_output_fp32); - gated_output.save(output + n); - } - gate += input_stride; - up += input_stride; - output += output_stride; - } -} - -template -void gelu_and_mul(float* __restrict__ input, scalar_t* __restrict__ output, - const int32_t m_size, const int32_t n_size, - const int32_t input_stride, const int32_t output_stride) { - using scalar_vec_t = typename cpu_utils::VecTypeTrait::vec_t; - const int32_t dim = n_size / 2; - float* __restrict__ gate = input; - float* __restrict__ up = input + dim; - vec_op::FP32Vec16 one_vec(1.0); - vec_op::FP32Vec16 w1_vec(M_SQRT1_2); - vec_op::FP32Vec16 w2_vec(0.5); - alignas(64) float temp[16]; - - DEFINE_FAST_EXP - - for (int32_t m = 0; m < m_size; ++m) { - for (int32_t n = 0; n < dim; n += 16) { - vec_op::FP32Vec16 gate_vec(gate + n); - vec_op::FP32Vec16 up_vec(up + n); - auto er_input_vec = gate_vec * w1_vec; - - er_input_vec.save(temp); - for (int32_t i = 0; i < 16; ++i) { - temp[i] = std::erf(temp[i]); - } - vec_op::FP32Vec16 er_vec(temp); - auto gelu = gate_vec * w2_vec * (one_vec + er_vec); - auto gated_output_fp32 = up_vec * gelu; - scalar_vec_t gated_output = scalar_vec_t(gated_output_fp32); - gated_output.save(output + n); - } - gate += input_stride; - up += input_stride; - output += output_stride; - } -} - -template -void gelu_tanh_and_mul(float* __restrict__ input, scalar_t* __restrict__ output, - const int32_t m_size, const int32_t n_size, - const int32_t input_stride, - const int32_t output_stride) { - using scalar_vec_t = typename cpu_utils::VecTypeTrait::vec_t; - const int32_t dim = n_size / 2; - float* __restrict__ gate = input; - float* __restrict__ up = input + dim; - vec_op::FP32Vec16 one_vec(1.0); - vec_op::FP32Vec16 w1_vec(0.7978845608028654); - vec_op::FP32Vec16 w2_vec(0.5); - vec_op::FP32Vec16 w3_vec(0.044715); - - for (int32_t m = 0; m < m_size; ++m) { - for (int32_t n = 0; n < dim; n += 16) { - vec_op::FP32Vec16 gate_vec(gate + n); - vec_op::FP32Vec16 up_vec(up + n); - auto gate_pow3_vec = gate_vec * gate_vec * gate_vec; - auto inner_vec = w1_vec * (gate_vec + w3_vec * gate_pow3_vec); - // Note: can't use fast_exp form because diffusiongemma will generate - // wrong results - auto tanh_vec = inner_vec.tanh(); - auto gelu_tanh = gate_vec * w2_vec * (one_vec + tanh_vec); - auto gated_output_fp32 = up_vec * gelu_tanh; - scalar_vec_t gated_output = scalar_vec_t(gated_output_fp32); - gated_output.save(output + n); - } - gate += input_stride; - up += input_stride; - output += output_stride; - } -} - -template -FORCE_INLINE void apply_gated_act(const FusedMOEAct act, - float* __restrict__ input, - scalar_t* __restrict__ output, - const int32_t m, const int32_t n, - const int32_t input_stride, - const int32_t output_stride) { - switch (act) { - case FusedMOEAct::SwigluOAIAndMul: - swigluoai_and_mul(input, output, m, n, input_stride, output_stride); - return; - case FusedMOEAct::SiluAndMul: - silu_and_mul(input, output, m, n, input_stride, output_stride); - return; - case FusedMOEAct::GeluAndMul: - gelu_and_mul(input, output, m, n, input_stride, output_stride); - return; - case FusedMOEAct::GeluTanhAndMul: - gelu_tanh_and_mul(input, output, m, n, input_stride, output_stride); - return; - default: - TORCH_CHECK(false, "Unsupported act type."); - } -} +using cpu_fused_moe_utils::apply_gated_act; +using cpu_fused_moe_utils::FusedMOEAct; template void prepack_moe_weight_impl(scalar_t* __restrict__ weight_ptr, @@ -817,6 +634,7 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input, } } } + } // namespace void prepack_moe_weight( @@ -864,7 +682,7 @@ void cpu_fused_moe( const int32_t input_size_2 = w2.size(2); const int32_t output_size_2 = w2.size(1); const int32_t topk_num = topk_id.size(1); - const FusedMOEAct act_type = get_act_type(act); + const FusedMOEAct act_type = cpu_fused_moe_utils::get_act_type(act); cpu_utils::ISA isa_type = cpu_utils::get_isa(isa); TORCH_CHECK(!skip_weighted || topk_num == 1, "skip_weighted is only supported for topk=1 on CPU"); diff --git a/csrc/cpu/cpu_fused_moe_activations.hpp b/csrc/cpu/cpu_fused_moe_activations.hpp new file mode 100644 index 0000000000..31e8cb009b --- /dev/null +++ b/csrc/cpu/cpu_fused_moe_activations.hpp @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +#ifndef CPU_FUSED_MOE_ACTIVATIONS_HPP +#define CPU_FUSED_MOE_ACTIVATIONS_HPP + +#include +#include +#include + +#include "cpu/cpu_arch_macros.h" +#include "cpu/utils.hpp" + +namespace cpu_fused_moe_utils { +enum class FusedMOEAct { + SiluAndMul, + SwigluOAIAndMul, + GeluAndMul, + GeluTanhAndMul, +}; + +inline FusedMOEAct get_act_type(const std::string& act) { + if (act == "silu") { + return FusedMOEAct::SiluAndMul; + } else if (act == "swigluoai") { + return FusedMOEAct::SwigluOAIAndMul; + } else if (act == "gelu") { + return FusedMOEAct::GeluAndMul; + } else if (act == "gelu_tanh") { + return FusedMOEAct::GeluTanhAndMul; + } else { + TORCH_CHECK(false, "Invalid act type: " + act); + } +} + +template +void swigluoai_and_mul(float* __restrict__ input, scalar_t* __restrict__ output, + const int32_t m_size, const int32_t n_size, + const int32_t input_stride, + const int32_t output_stride) { + using scalar_vec_t = typename cpu_utils::VecTypeTrait::vec_t; +#if !defined(__aarch64__) + // For GPT-OSS interleaved gate-up weights + alignas(64) static int32_t index[16] = {0, 2, 4, 6, 8, 10, 12, 14, + 16, 18, 20, 22, 24, 26, 28, 30}; + vec_op::INT32Vec16 index_vec(index); +#endif + vec_op::FP32Vec16 gate_up_max_vec(7.0); + vec_op::FP32Vec16 up_min_vec(-7.0); + vec_op::FP32Vec16 alpha_vec(1.702); + vec_op::FP32Vec16 one_vec(1.0); + + DEFINE_FAST_EXP + + for (int32_t m = 0; m < m_size; ++m) { + for (int32_t n = 0; n < n_size; n += 32) { + // Note: AdvSIMD does not support gather loads +#if defined(__aarch64__) + vec_op::FP32Vec16 gate_vec(vec_op::uninit); + vec_op::FP32Vec16 up_vec(vec_op::uninit); + vec_op::FP32Vec16::load_even_odd(input + n, gate_vec, up_vec); +#else + vec_op::FP32Vec16 gate_vec(input + n, index_vec); + vec_op::FP32Vec16 up_vec(input + n + 1, index_vec); +#endif + gate_vec = gate_vec.min(gate_up_max_vec); + up_vec = up_vec.clamp(up_min_vec, gate_up_max_vec); + auto sigmoid_vec = one_vec / (one_vec + fast_exp(-gate_vec * alpha_vec)); + auto glu = gate_vec * sigmoid_vec; + auto gated_output_fp32 = (one_vec + up_vec) * glu; + scalar_vec_t gated_output = scalar_vec_t(gated_output_fp32); + gated_output.save(output + n / 2); + } + input += input_stride; + output += output_stride; + } +} + +template +void silu_and_mul(float* __restrict__ input, scalar_t* __restrict__ output, + const int32_t m_size, const int32_t n_size, + const int32_t input_stride, const int32_t output_stride) { + using scalar_vec_t = typename cpu_utils::VecTypeTrait::vec_t; + const int32_t dim = n_size / 2; + float* __restrict__ gate = input; + float* __restrict__ up = input + dim; + vec_op::FP32Vec16 one_vec(1.0); + + DEFINE_FAST_EXP + + for (int32_t m = 0; m < m_size; ++m) { + for (int32_t n = 0; n < dim; n += 16) { + vec_op::FP32Vec16 gate_vec(gate + n); + vec_op::FP32Vec16 up_vec(up + n); + auto sigmoid_vec = one_vec / (one_vec + fast_exp(-gate_vec)); + auto silu = gate_vec * sigmoid_vec; + auto gated_output_fp32 = up_vec * silu; + scalar_vec_t gated_output = scalar_vec_t(gated_output_fp32); + gated_output.save(output + n); + } + gate += input_stride; + up += input_stride; + output += output_stride; + } +} + +template +void gelu_and_mul(float* __restrict__ input, scalar_t* __restrict__ output, + const int32_t m_size, const int32_t n_size, + const int32_t input_stride, const int32_t output_stride) { + using scalar_vec_t = typename cpu_utils::VecTypeTrait::vec_t; + const int32_t dim = n_size / 2; + float* __restrict__ gate = input; + float* __restrict__ up = input + dim; + vec_op::FP32Vec16 one_vec(1.0); + vec_op::FP32Vec16 w1_vec(M_SQRT1_2); + vec_op::FP32Vec16 w2_vec(0.5); + alignas(64) float temp[16]; + + DEFINE_FAST_EXP + + for (int32_t m = 0; m < m_size; ++m) { + for (int32_t n = 0; n < dim; n += 16) { + vec_op::FP32Vec16 gate_vec(gate + n); + vec_op::FP32Vec16 up_vec(up + n); + auto er_input_vec = gate_vec * w1_vec; + + er_input_vec.save(temp); + for (int32_t i = 0; i < 16; ++i) { + temp[i] = std::erf(temp[i]); + } + vec_op::FP32Vec16 er_vec(temp); + auto gelu = gate_vec * w2_vec * (one_vec + er_vec); + auto gated_output_fp32 = up_vec * gelu; + scalar_vec_t gated_output = scalar_vec_t(gated_output_fp32); + gated_output.save(output + n); + } + gate += input_stride; + up += input_stride; + output += output_stride; + } +} + +template +void gelu_tanh_and_mul(float* __restrict__ input, scalar_t* __restrict__ output, + const int32_t m_size, const int32_t n_size, + const int32_t input_stride, + const int32_t output_stride) { + using scalar_vec_t = typename cpu_utils::VecTypeTrait::vec_t; + const int32_t dim = n_size / 2; + float* __restrict__ gate = input; + float* __restrict__ up = input + dim; + vec_op::FP32Vec16 one_vec(1.0); + vec_op::FP32Vec16 w1_vec(0.7978845608028654); + vec_op::FP32Vec16 w2_vec(0.5); + vec_op::FP32Vec16 w3_vec(0.044715); + + for (int32_t m = 0; m < m_size; ++m) { + for (int32_t n = 0; n < dim; n += 16) { + vec_op::FP32Vec16 gate_vec(gate + n); + vec_op::FP32Vec16 up_vec(up + n); + auto gate_pow3_vec = gate_vec * gate_vec * gate_vec; + auto inner_vec = w1_vec * (gate_vec + w3_vec * gate_pow3_vec); + // Note: can't use fast_exp form because diffusiongemma will generate + // wrong results + auto tanh_vec = inner_vec.tanh(); + auto gelu_tanh = gate_vec * w2_vec * (one_vec + tanh_vec); + auto gated_output_fp32 = up_vec * gelu_tanh; + scalar_vec_t gated_output = scalar_vec_t(gated_output_fp32); + gated_output.save(output + n); + } + gate += input_stride; + up += input_stride; + output += output_stride; + } +} + +template +FORCE_INLINE void apply_gated_act(const FusedMOEAct act, + float* __restrict__ input, + scalar_t* __restrict__ output, + const int32_t m, const int32_t n, + const int32_t input_stride, + const int32_t output_stride) { + switch (act) { + case FusedMOEAct::SwigluOAIAndMul: + swigluoai_and_mul(input, output, m, n, input_stride, output_stride); + return; + case FusedMOEAct::SiluAndMul: + silu_and_mul(input, output, m, n, input_stride, output_stride); + return; + case FusedMOEAct::GeluAndMul: + gelu_and_mul(input, output, m, n, input_stride, output_stride); + return; + case FusedMOEAct::GeluTanhAndMul: + gelu_tanh_and_mul(input, output, m, n, input_stride, output_stride); + return; + default: + TORCH_CHECK(false, "Unsupported act type."); + } +} +} // namespace cpu_fused_moe_utils + +#endif diff --git a/csrc/cpu/cpu_fused_moe_int8.cpp b/csrc/cpu/cpu_fused_moe_int8.cpp new file mode 100644 index 0000000000..649317b478 --- /dev/null +++ b/csrc/cpu/cpu_fused_moe_int8.cpp @@ -0,0 +1,647 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +#include "cpu/cpu_arch_macros.h" + +#include +#include +#include +#include +#include + +#include "cpu/cpu_fused_moe_activations.hpp" +#include "cpu/cpu_types.hpp" +#include "cpu/micro_gemm/cpu_micro_gemm_impl.hpp" +#include "cpu/utils.hpp" + +#if defined(ARM_I8MM_SUPPORT) && defined(ARM_BF16_SUPPORT) + #include "cpu/micro_gemm/cpu_micro_gemm_int8_neon.hpp" + #define NEON_DISPATCH(SCALAR_TYPE, ...) \ + case cpu_utils::ISA::NEON: { \ + using gemm_t = \ + cpu_micro_gemm::MicroGemmINT8; \ + return __VA_ARGS__(); \ + } +#else + #define NEON_DISPATCH(SCALAR_TYPE, ...) case cpu_utils::ISA::NEON: +#endif + +#define CPU_INT8_ISA_DISPATCH_IMPL(ISA_TYPE, SCALAR_TYPE, ...) \ + [&] { \ + switch (ISA_TYPE) { \ + NEON_DISPATCH(SCALAR_TYPE, __VA_ARGS__) \ + default: { \ + TORCH_CHECK(false, "Invalid CPU ISA type."); \ + } \ + } \ + }() + +namespace { +using cpu_fused_moe_utils::apply_gated_act; +using cpu_fused_moe_utils::FusedMOEAct; + +template +void prepack_moe_weight_int8_impl(const int8_t* __restrict__ weight_ptr, + int8_t* __restrict__ packed_weight_ptr, + const int32_t expert_num, + const int32_t output_size, + const int32_t input_size, + const int64_t expert_stride) { +#pragma omp parallel for + for (int32_t e_idx = 0; e_idx < expert_num; ++e_idx) { + gemm_t::pack_weight(weight_ptr + expert_stride * e_idx, + packed_weight_ptr + expert_stride * e_idx, output_size, + input_size); + } +} + +// INT8 MoE kernel, based on the original BF16 kernel in cpu_fused_moe.cpp +template +void fused_moe_int8_impl( + scalar_t* __restrict__ output, const scalar_t* __restrict__ input, + const int8_t* __restrict__ w13, const int8_t* __restrict__ w2, + const float* __restrict__ w13_scales, const float* __restrict__ w2_scales, + scalar_t* __restrict__ w13_bias, scalar_t* __restrict__ w2_bias, + const float* __restrict__ topk_weights, const int32_t* __restrict__ topk_id, + const FusedMOEAct act_type, const int32_t token_num, + const int32_t expert_num, const int32_t topk_num, + const int32_t input_size_13, const int32_t output_size_13, + const int32_t input_size_2, const int32_t output_size_2, + const bool skip_weighted) { + using scalar_vec_t = typename cpu_utils::VecTypeTrait::vec_t; + constexpr int32_t gemm_n_tile_size = gemm_t::NSize; + constexpr int32_t gemm_m_tile_size = gemm_t::MaxMSize; + constexpr int32_t min_w13_n_tile_size = 2 * gemm_n_tile_size; + + TORCH_CHECK_EQ(input_size_13 % gemm_t::K, 0); + TORCH_CHECK_EQ(input_size_2 % gemm_t::K, 0); + TORCH_CHECK_EQ(output_size_13 % min_w13_n_tile_size, 0); + TORCH_CHECK_EQ(output_size_2 % gemm_n_tile_size, 0); + TORCH_CHECK_EQ(output_size_13 / 2, input_size_2); + + const int32_t thread_num = cpu_utils::get_max_threads(); + const int32_t w13_input_buffer_size = cpu_utils::round_up<64>( + gemm_m_tile_size * input_size_13 * sizeof(int8_t)); + const int32_t w2_input_buffer_size = + cpu_utils::round_up<64>(gemm_m_tile_size * input_size_2 * sizeof(int8_t)); + + const int32_t w13_n_tile_size = [&]() { + const int64_t cache_size = cpu_utils::get_available_l2_size(); + const int32_t n_size_cache_limit = + (cache_size - w13_input_buffer_size) / + (gemm_m_tile_size * sizeof(float) + input_size_13 * sizeof(int8_t)); + const int32_t n_size_thread_limit = + output_size_13 / std::max(1, thread_num / topk_num); + const int32_t n_size = cpu_utils::round_down( + std::min(n_size_cache_limit, n_size_thread_limit)); + return std::max(n_size, min_w13_n_tile_size); + }(); + + const int32_t w2_n_tile_size = [&]() { + const int64_t cache_size = cpu_utils::get_available_l2_size(); + const int32_t n_size_cache_limit = + (cache_size - w2_input_buffer_size) / (input_size_2 * sizeof(int8_t)); + const int32_t n_size_thread_limit = + output_size_2 / std::max(1, thread_num / topk_num); + const int32_t n_size = cpu_utils::round_down( + std::min(n_size_cache_limit, n_size_thread_limit)); + return std::max(n_size, gemm_n_tile_size); + }(); + + int32_t common_buffer_offset = 0; + const int32_t token_num_per_group_buffer_offset = common_buffer_offset; + common_buffer_offset += cpu_utils::round_up<64>(expert_num * sizeof(int32_t)); + const int32_t cu_token_num_per_group_buffer_offset = common_buffer_offset; + common_buffer_offset += + cpu_utils::round_up<64>((expert_num + 1) * sizeof(int32_t)); + const int32_t expanded_token_num = token_num * topk_num; + const int32_t expand_token_id_buffer_offset = common_buffer_offset; + common_buffer_offset += + cpu_utils::round_up<64>(expanded_token_num * sizeof(int32_t)); + const int32_t expand_token_id_index_buffer_offset = common_buffer_offset; + common_buffer_offset += + cpu_utils::round_up<64>(expanded_token_num * sizeof(int32_t)); + const int32_t input_quant_buffer_offset = common_buffer_offset; + common_buffer_offset += + cpu_utils::round_up<64>(token_num * input_size_13 * sizeof(int8_t)); + const int32_t input_scale_buffer_offset = common_buffer_offset; + common_buffer_offset += cpu_utils::round_up<64>(token_num * sizeof(float)); + const int32_t w13_gemm_output_buffer_offset = common_buffer_offset; + common_buffer_offset += cpu_utils::round_up<64>( + expanded_token_num * input_size_2 * sizeof(scalar_t)); + const int32_t w13_output_scale_buffer_offset = common_buffer_offset; + common_buffer_offset += + cpu_utils::round_up<64>(expanded_token_num * sizeof(float)); + const int32_t w2_gemm_output_buffer_offset = common_buffer_offset; + common_buffer_offset += cpu_utils::round_up<64>( + expanded_token_num * output_size_2 * sizeof(float)); + + int32_t gemm_thread_buffer_offset = 0; + const int32_t gemm_input_buffer_offset = gemm_thread_buffer_offset; + gemm_thread_buffer_offset += + std::max(w13_input_buffer_size, w2_input_buffer_size); + const int32_t gemm_output_buffer_offset = gemm_thread_buffer_offset; + gemm_thread_buffer_offset += cpu_utils::round_up<64>( + gemm_m_tile_size * std::max(w13_n_tile_size, w2_n_tile_size) * + sizeof(int32_t)); + + const int32_t ws_output_buffer_offset = 0; + const int32_t ws_thread_buffer_size = + cpu_utils::round_up<64>(output_size_2 * sizeof(float)); + const int32_t thread_buffer_size = + std::max(gemm_thread_buffer_offset, ws_thread_buffer_size); + const int32_t buffer_size = + common_buffer_offset + thread_buffer_size * thread_num; + cpu_utils::ScratchPadManager::get_scratchpad_manager()->realloc(buffer_size); + uint8_t* common_buffer_start = + cpu_utils::ScratchPadManager::get_scratchpad_manager() + ->get_data(); + uint8_t* thread_buffer_start = common_buffer_start + common_buffer_offset; + + int32_t* __restrict__ token_num_per_group_buffer = reinterpret_cast( + common_buffer_start + token_num_per_group_buffer_offset); + int32_t* __restrict__ cu_token_num_per_group_buffer = + reinterpret_cast(common_buffer_start + + cu_token_num_per_group_buffer_offset); + int32_t* __restrict__ expand_token_id_buffer = reinterpret_cast( + common_buffer_start + expand_token_id_buffer_offset); + int32_t* __restrict__ expand_token_id_index_buffer = + reinterpret_cast(common_buffer_start + + expand_token_id_index_buffer_offset); + int8_t* __restrict__ input_quant_buffer = reinterpret_cast( + common_buffer_start + input_quant_buffer_offset); + float* __restrict__ input_scale_buffer = + reinterpret_cast(common_buffer_start + input_scale_buffer_offset); + + std::memset(token_num_per_group_buffer, 0, expert_num * sizeof(int32_t)); + for (int32_t i = 0; i < expanded_token_num; ++i) { + ++token_num_per_group_buffer[topk_id[i]]; + } + + int32_t token_num_sum = 0; + cu_token_num_per_group_buffer[0] = 0; + int32_t* token_index_buffer = cu_token_num_per_group_buffer + 1; + for (int32_t i = 0; i < expert_num; ++i) { + token_index_buffer[i] = token_num_sum; + token_num_sum += token_num_per_group_buffer[i]; + } + + for (int32_t i = 0; i < token_num; ++i) { + const int32_t* curr_topk_id = topk_id + i * topk_num; + int32_t* curr_index_buffer = expand_token_id_index_buffer + i * topk_num; + for (int32_t j = 0; j < topk_num; ++j) { + const int32_t curr_expert_id = curr_topk_id[j]; + const int32_t curr_index = token_index_buffer[curr_expert_id]++; + expand_token_id_buffer[curr_index] = i; + curr_index_buffer[j] = curr_index; + } + } + +// quantize inputs +#pragma omp parallel for + for (int32_t token_idx = 0; token_idx < token_num; ++token_idx) { + gemm_t::quantize_row(input + token_idx * input_size_13, + input_quant_buffer + token_idx * input_size_13, + input_scale_buffer[token_idx], input_size_13); + } + + { + alignas(64) cpu_utils::Counter counter; + cpu_utils::Counter* counter_ptr = &counter; + +// w13 GEMM + act +#pragma omp parallel for schedule(static, 1) + for (int32_t thread_id = 0; thread_id < thread_num; ++thread_id) { + const int32_t task_num_per_expert = + (output_size_13 + w13_n_tile_size - 1) / w13_n_tile_size; + const int32_t task_num = task_num_per_expert * expert_num; + uint8_t* __restrict__ thread_buffer = + thread_buffer_start + thread_id * thread_buffer_size; + int8_t* __restrict__ gemm_input_buffer = + reinterpret_cast(thread_buffer + gemm_input_buffer_offset); + float* __restrict__ gemm_output_buffer = + reinterpret_cast(thread_buffer + gemm_output_buffer_offset); + auto* __restrict__ w13_gemm_output_buffer = reinterpret_cast( + common_buffer_start + w13_gemm_output_buffer_offset); + gemm_t gemm; + + const int32_t w13_n_group_stride = + gemm_t::WeightOCGroupSize * input_size_13; + const int32_t w13_n_tile_stride = gemm_n_tile_size * input_size_13; + + for (;;) { + const int32_t task_id = counter_ptr->acquire_counter(); + if (task_id >= task_num) { + break; + } + const int32_t curr_expert_id = task_id / task_num_per_expert; + const int32_t curr_output_group_id = task_id % task_num_per_expert; + const int32_t curr_token_num = + token_num_per_group_buffer[curr_expert_id]; + if (curr_token_num == 0) { + continue; + } + + const int32_t actual_n_tile_size = + std::min(w13_n_tile_size, + output_size_13 - curr_output_group_id * w13_n_tile_size); + const int32_t* __restrict__ curr_expand_token_id_buffer = + expand_token_id_buffer + + cu_token_num_per_group_buffer[curr_expert_id]; + scalar_t* __restrict__ curr_w13_gemm_output_buffer = + w13_gemm_output_buffer + + cu_token_num_per_group_buffer[curr_expert_id] * input_size_2 + + curr_output_group_id * w13_n_tile_size / 2; + + const int8_t* w13_weight_ptr_0 = nullptr; + const int8_t* w13_weight_ptr_1 = nullptr; + const float* w13_scale_ptr_0 = nullptr; + const float* w13_scale_ptr_1 = nullptr; + scalar_t* w13_bias_ptr_0 = nullptr; + scalar_t* w13_bias_ptr_1 = nullptr; + if (act_type == FusedMOEAct::SwigluOAIAndMul) { + const int32_t output_offset = curr_output_group_id * w13_n_tile_size; + w13_weight_ptr_0 = w13 + + curr_expert_id * input_size_13 * output_size_13 + + output_offset * input_size_13; + w13_weight_ptr_1 = + w13_weight_ptr_0 + actual_n_tile_size / 2 * input_size_13; + w13_scale_ptr_0 = + w13_scales + curr_expert_id * output_size_13 + output_offset; + w13_scale_ptr_1 = w13_scale_ptr_0 + actual_n_tile_size / 2; + if (w13_bias != nullptr) { + w13_bias_ptr_0 = + w13_bias + curr_expert_id * output_size_13 + output_offset; + w13_bias_ptr_1 = w13_bias_ptr_0 + actual_n_tile_size / 2; + } + } else { + const int32_t output_offset = + curr_output_group_id * (w13_n_tile_size / 2); + w13_weight_ptr_0 = w13 + + curr_expert_id * input_size_13 * output_size_13 + + output_offset * input_size_13; + w13_weight_ptr_1 = + w13_weight_ptr_0 + output_size_13 / 2 * input_size_13; + w13_scale_ptr_0 = + w13_scales + curr_expert_id * output_size_13 + output_offset; + w13_scale_ptr_1 = w13_scale_ptr_0 + output_size_13 / 2; + if (w13_bias != nullptr) { + w13_bias_ptr_0 = + w13_bias + curr_expert_id * output_size_13 + output_offset; + w13_bias_ptr_1 = w13_bias_ptr_0 + output_size_13 / 2; + } + } + + for (int32_t token_idx = 0; token_idx < curr_token_num; + token_idx += gemm_m_tile_size) { + const int32_t actual_token_num = + std::min(gemm_m_tile_size, curr_token_num - token_idx); + const int8_t* input_rows[gemm_m_tile_size]; + alignas(64) float input_scales[gemm_m_tile_size]; + // gather and pack + for (int32_t i = 0; i < actual_token_num; ++i) { + const int32_t curr_token_id = curr_expand_token_id_buffer[i]; + input_rows[i] = input_quant_buffer + curr_token_id * input_size_13; + input_scales[i] = input_scale_buffer[curr_token_id]; + } + gemm_t::pack_input_from_rows(input_rows, gemm_input_buffer, + actual_token_num, input_size_13); + curr_expand_token_id_buffer += actual_token_num; + + const int8_t* w13_weight_ptr_0_iter = w13_weight_ptr_0; + const int8_t* w13_weight_ptr_1_iter = w13_weight_ptr_1; + const float* w13_scale_ptr_0_iter = w13_scale_ptr_0; + const float* w13_scale_ptr_1_iter = w13_scale_ptr_1; + scalar_t* w13_bias_ptr_0_iter = w13_bias_ptr_0; + scalar_t* w13_bias_ptr_1_iter = w13_bias_ptr_1; + float* w13_output_buffer_0_iter = gemm_output_buffer; + float* w13_output_buffer_1_iter = + gemm_output_buffer + actual_n_tile_size / 2; + + for (int32_t i = 0; i < actual_n_tile_size; + i += min_w13_n_tile_size) { + auto* output_0_int32 = + reinterpret_cast(w13_output_buffer_0_iter); + gemm.gemm(gemm_input_buffer, w13_weight_ptr_0_iter, output_0_int32, + actual_token_num, input_size_13, w13_n_group_stride, + actual_n_tile_size); + gemm_t::dequantize_tile(output_0_int32, w13_output_buffer_0_iter, + input_scales, w13_scale_ptr_0_iter, + actual_token_num, gemm_n_tile_size, + actual_n_tile_size); + if (w13_bias != nullptr) { + cpu_micro_gemm::add_bias_epilogue( + w13_output_buffer_0_iter, w13_output_buffer_0_iter, + w13_bias_ptr_0_iter, actual_token_num, actual_n_tile_size, + actual_n_tile_size); + w13_bias_ptr_0_iter += gemm_n_tile_size; + } + + auto* output_1_int32 = + reinterpret_cast(w13_output_buffer_1_iter); + gemm.gemm(gemm_input_buffer, w13_weight_ptr_1_iter, output_1_int32, + actual_token_num, input_size_13, w13_n_group_stride, + actual_n_tile_size); + gemm_t::dequantize_tile(output_1_int32, w13_output_buffer_1_iter, + input_scales, w13_scale_ptr_1_iter, + actual_token_num, gemm_n_tile_size, + actual_n_tile_size); + if (w13_bias != nullptr) { + cpu_micro_gemm::add_bias_epilogue( + w13_output_buffer_1_iter, w13_output_buffer_1_iter, + w13_bias_ptr_1_iter, actual_token_num, actual_n_tile_size, + actual_n_tile_size); + w13_bias_ptr_1_iter += gemm_n_tile_size; + } + + w13_weight_ptr_0_iter += w13_n_tile_stride; + w13_weight_ptr_1_iter += w13_n_tile_stride; + w13_scale_ptr_0_iter += gemm_n_tile_size; + w13_scale_ptr_1_iter += gemm_n_tile_size; + w13_output_buffer_0_iter += gemm_n_tile_size; + w13_output_buffer_1_iter += gemm_n_tile_size; + } + + apply_gated_act(act_type, gemm_output_buffer, + curr_w13_gemm_output_buffer, actual_token_num, + actual_n_tile_size, actual_n_tile_size, input_size_2); + curr_w13_gemm_output_buffer += gemm_m_tile_size * input_size_2; + } + } + } + } + + auto* __restrict__ w13_gemm_output_buffer = reinterpret_cast( + common_buffer_start + w13_gemm_output_buffer_offset); + float* __restrict__ w13_output_scale_buffer = reinterpret_cast( + common_buffer_start + w13_output_scale_buffer_offset); + +// quantize w2 inputs - in place +#pragma omp parallel for + for (int32_t token_idx = 0; token_idx < expanded_token_num; ++token_idx) { + scalar_t* input_row = w13_gemm_output_buffer + token_idx * input_size_2; + int8_t* output_row = reinterpret_cast(input_row); + gemm_t::quantize_row(input_row, output_row, + w13_output_scale_buffer[token_idx], input_size_2); + } + + { + alignas(64) cpu_utils::Counter counter; + cpu_utils::Counter* counter_ptr = &counter; + +// w2 gemm +#pragma omp parallel for schedule(static, 1) + for (int32_t thread_id = 0; thread_id < thread_num; ++thread_id) { + const int32_t task_num_per_expert = + (output_size_2 + w2_n_tile_size - 1) / w2_n_tile_size; + const int32_t task_num = task_num_per_expert * expert_num; + uint8_t* __restrict__ thread_buffer = + thread_buffer_start + thread_id * thread_buffer_size; + int8_t* __restrict__ gemm_input_buffer = + reinterpret_cast(thread_buffer + gemm_input_buffer_offset); + float* __restrict__ gemm_output_buffer = + reinterpret_cast(thread_buffer + gemm_output_buffer_offset); + float* __restrict__ w2_gemm_output_buffer = reinterpret_cast( + common_buffer_start + w2_gemm_output_buffer_offset); + gemm_t gemm; + + const int32_t w2_n_group_stride = + gemm_t::WeightOCGroupSize * input_size_2; + const int32_t w2_n_tile_stride = gemm_n_tile_size * input_size_2; + + for (;;) { + const int32_t task_id = counter_ptr->acquire_counter(); + if (task_id >= task_num) { + break; + } + const int32_t curr_expert_id = task_id / task_num_per_expert; + const int32_t curr_output_group_id = task_id % task_num_per_expert; + const int32_t curr_token_num = + token_num_per_group_buffer[curr_expert_id]; + if (curr_token_num == 0) { + continue; + } + + const int32_t actual_n_tile_size = + std::min(w2_n_tile_size, + output_size_2 - curr_output_group_id * w2_n_tile_size); + scalar_t* __restrict__ curr_w13_gemm_output_buffer = + w13_gemm_output_buffer + + cu_token_num_per_group_buffer[curr_expert_id] * input_size_2; + float* __restrict__ curr_w13_output_scale_buffer = + w13_output_scale_buffer + + cu_token_num_per_group_buffer[curr_expert_id]; + float* __restrict__ curr_w2_gemm_output_buffer = + w2_gemm_output_buffer + + cu_token_num_per_group_buffer[curr_expert_id] * output_size_2 + + curr_output_group_id * w2_n_tile_size; + const int8_t* __restrict__ w2_weight_ptr = + w2 + curr_expert_id * output_size_2 * input_size_2 + + curr_output_group_id * w2_n_tile_size * input_size_2; + const float* __restrict__ w2_scale_ptr = + w2_scales + curr_expert_id * output_size_2 + + curr_output_group_id * w2_n_tile_size; + scalar_t* w2_bias_ptr = nullptr; + if (w2_bias != nullptr) { + w2_bias_ptr = w2_bias + curr_expert_id * output_size_2 + + curr_output_group_id * w2_n_tile_size; + } + + for (int32_t token_idx = 0; token_idx < curr_token_num; + token_idx += gemm_m_tile_size) { + const int32_t actual_token_num = + std::min(gemm_m_tile_size, curr_token_num - token_idx); + const int8_t* input_rows[gemm_m_tile_size]; + alignas(64) float input_scales[gemm_m_tile_size]; + for (int32_t i = 0; i < actual_token_num; ++i) { + input_rows[i] = reinterpret_cast( + curr_w13_gemm_output_buffer + i * input_size_2); + input_scales[i] = curr_w13_output_scale_buffer[i]; + } + gemm_t::pack_input_from_rows(input_rows, gemm_input_buffer, + actual_token_num, input_size_2); + + const int8_t* w2_weight_ptr_iter = w2_weight_ptr; + const float* w2_scale_ptr_iter = w2_scale_ptr; + scalar_t* w2_bias_ptr_iter = w2_bias_ptr; + float* curr_w2_gemm_output_buffer_iter = curr_w2_gemm_output_buffer; + for (int32_t i = 0; i < actual_n_tile_size; i += gemm_n_tile_size) { + auto* output_int32 = reinterpret_cast(gemm_output_buffer); + gemm.gemm(gemm_input_buffer, w2_weight_ptr_iter, output_int32, + actual_token_num, input_size_2, w2_n_group_stride, + gemm_n_tile_size); + gemm_t::dequantize_tile(output_int32, gemm_output_buffer, + input_scales, w2_scale_ptr_iter, + actual_token_num, gemm_n_tile_size, + gemm_n_tile_size); + if (w2_bias != nullptr) { + cpu_micro_gemm::add_bias_epilogue( + gemm_output_buffer, gemm_output_buffer, w2_bias_ptr_iter, + actual_token_num, gemm_n_tile_size, gemm_n_tile_size); + w2_bias_ptr_iter += gemm_n_tile_size; + } + for (int32_t m_idx = 0; m_idx < actual_token_num; ++m_idx) { + std::memcpy( + curr_w2_gemm_output_buffer_iter + m_idx * output_size_2, + gemm_output_buffer + m_idx * gemm_n_tile_size, + gemm_n_tile_size * sizeof(float)); + } + + w2_weight_ptr_iter += w2_n_tile_stride; + w2_scale_ptr_iter += gemm_n_tile_size; + curr_w2_gemm_output_buffer_iter += gemm_n_tile_size; + } + + curr_w13_gemm_output_buffer += gemm_m_tile_size * input_size_2; + curr_w13_output_scale_buffer += gemm_m_tile_size; + curr_w2_gemm_output_buffer += gemm_m_tile_size * output_size_2; + } + } + } + } + + { + alignas(64) cpu_utils::Counter counter; + cpu_utils::Counter* counter_ptr = &counter; + +#pragma omp parallel for schedule(static, 1) + for (int32_t thread_id = 0; thread_id < thread_num; ++thread_id) { + uint8_t* __restrict__ thread_buffer = + thread_buffer_start + thread_id * thread_buffer_size; + float* __restrict__ ws_output_buffer = + reinterpret_cast(thread_buffer + ws_output_buffer_offset); + float* __restrict__ w2_gemm_output_buffer = reinterpret_cast( + common_buffer_start + w2_gemm_output_buffer_offset); + + for (;;) { + const int32_t token_id = counter_ptr->acquire_counter(); + if (token_id >= token_num) { + break; + } + int32_t* __restrict__ curr_expand_token_id_index_buffer = + expand_token_id_index_buffer + token_id * topk_num; + const float* __restrict__ curr_weight = + topk_weights + token_id * topk_num; + const float first_weight = skip_weighted ? 1.0f : curr_weight[0]; + scalar_t* __restrict__ curr_output_buffer = + output + token_id * output_size_2; + + if (topk_num > 1) { + int32_t w2_output_idx = curr_expand_token_id_index_buffer[0]; + float* w2_output_iter = + w2_gemm_output_buffer + w2_output_idx * output_size_2; + float* ws_output_buffer_iter = ws_output_buffer; + vec_op::FP32Vec16 weight_vec(first_weight); + for (int32_t i = 0; i < output_size_2; i += 16) { + vec_op::FP32Vec16 vec(w2_output_iter); + (vec * weight_vec).save(ws_output_buffer_iter); + w2_output_iter += 16; + ws_output_buffer_iter += 16; + } + + for (int32_t idx = 1; idx < topk_num - 1; ++idx) { + w2_output_idx = curr_expand_token_id_index_buffer[idx]; + w2_output_iter = + w2_gemm_output_buffer + w2_output_idx * output_size_2; + ws_output_buffer_iter = ws_output_buffer; + weight_vec = vec_op::FP32Vec16(curr_weight[idx]); + for (int32_t i = 0; i < output_size_2; i += 16) { + vec_op::FP32Vec16 vec(w2_output_iter); + vec_op::FP32Vec16 sum(ws_output_buffer_iter); + (sum + vec * weight_vec).save(ws_output_buffer_iter); + w2_output_iter += 16; + ws_output_buffer_iter += 16; + } + } + + const int32_t last_idx = topk_num - 1; + w2_output_idx = curr_expand_token_id_index_buffer[last_idx]; + w2_output_iter = + w2_gemm_output_buffer + w2_output_idx * output_size_2; + ws_output_buffer_iter = ws_output_buffer; + scalar_t* curr_output_buffer_iter = curr_output_buffer; + weight_vec = vec_op::FP32Vec16(curr_weight[last_idx]); + for (int32_t i = 0; i < output_size_2; i += 16) { + vec_op::FP32Vec16 vec(w2_output_iter); + vec_op::FP32Vec16 sum(ws_output_buffer_iter); + scalar_vec_t(sum + vec * weight_vec).save(curr_output_buffer_iter); + w2_output_iter += 16; + ws_output_buffer_iter += 16; + curr_output_buffer_iter += 16; + } + } else { + const int32_t w2_output_idx = curr_expand_token_id_index_buffer[0]; + float* w2_output_iter = + w2_gemm_output_buffer + w2_output_idx * output_size_2; + scalar_t* curr_output_buffer_iter = curr_output_buffer; + vec_op::FP32Vec16 weight_vec(first_weight); + for (int32_t i = 0; i < output_size_2; i += 16) { + vec_op::FP32Vec16 vec(w2_output_iter); + scalar_vec_t(vec * weight_vec).save(curr_output_buffer_iter); + w2_output_iter += 16; + curr_output_buffer_iter += 16; + } + } + } + } + } +} +} // namespace + +void prepack_moe_weight_int8( + const torch::Tensor& weight, // [expert_num, output_size, input_size] + torch::Tensor& packed_weight, const std::string& isa) { + TORCH_CHECK(weight.is_contiguous()); + const int32_t expert_num = weight.size(0); + const int32_t output_size = weight.size(1); + const int32_t input_size = weight.size(2); + const int64_t expert_stride = weight.stride(0); + const cpu_utils::ISA isa_type = cpu_utils::get_isa(isa); + TORCH_CHECK_EQ(output_size % 32, 0); + + CPU_INT8_ISA_DISPATCH_IMPL(isa_type, c10::BFloat16, [&]() { + TORCH_CHECK_EQ(input_size % gemm_t::K, 0); + prepack_moe_weight_int8_impl( + weight.data_ptr(), packed_weight.data_ptr(), expert_num, + output_size, input_size, expert_stride); + }); +} + +void cpu_fused_moe_int8(torch::Tensor& output, const torch::Tensor& input, + const torch::Tensor& w13, const torch::Tensor& w2, + const torch::Tensor& w13_scale, + const torch::Tensor& w2_scale, + const std::optional& w13_bias, + const std::optional& w2_bias, + const torch::Tensor& topk_weights, + const torch::Tensor& topk_id, const bool skip_weighted, + const std::string& act, const std::string& isa) { + const int32_t token_num = input.size(0); + const int32_t input_size_13 = input.size(1); + const int64_t input_stride = input.stride(0); + TORCH_CHECK_EQ(input_stride, input_size_13); + const int32_t expert_num = w13.size(0); + const int32_t output_size_13 = w13.size(1); + const int32_t input_size_2 = w2.size(2); + const int32_t output_size_2 = w2.size(1); + const int32_t topk_num = topk_id.size(1); + const FusedMOEAct act_type = cpu_fused_moe_utils::get_act_type(act); + const cpu_utils::ISA isa_type = cpu_utils::get_isa(isa); + TORCH_CHECK(!skip_weighted || topk_num == 1, + "skip_weighted is only supported for topk=1 on CPU"); + + VLLM_DISPATCH_FLOATING_TYPES( + input.scalar_type(), "cpu_fused_moe_int8", [&]() { + CPU_INT8_ISA_DISPATCH_IMPL(isa_type, scalar_t, [&]() { + fused_moe_int8_impl( + output.data_ptr(), input.data_ptr(), + w13.data_ptr(), w2.data_ptr(), + w13_scale.data_ptr(), w2_scale.data_ptr(), + w13_bias.has_value() ? w13_bias->data_ptr() : nullptr, + w2_bias.has_value() ? w2_bias->data_ptr() : nullptr, + topk_weights.data_ptr(), topk_id.data_ptr(), + act_type, token_num, expert_num, topk_num, input_size_13, + output_size_13, input_size_2, output_size_2, skip_weighted); + }); + }); +} diff --git a/csrc/cpu/micro_gemm/cpu_micro_gemm_impl.hpp b/csrc/cpu/micro_gemm/cpu_micro_gemm_impl.hpp index f0471f7147..75505176ee 100644 --- a/csrc/cpu/micro_gemm/cpu_micro_gemm_impl.hpp +++ b/csrc/cpu/micro_gemm/cpu_micro_gemm_impl.hpp @@ -31,6 +31,9 @@ class MicroGemm { } }; +template +class MicroGemmINT8; + template FORCE_INLINE void default_epilogue(float* __restrict__ c_ptr, scalar_t* __restrict__ d_ptr, diff --git a/csrc/cpu/micro_gemm/cpu_micro_gemm_int8_neon.hpp b/csrc/cpu/micro_gemm/cpu_micro_gemm_int8_neon.hpp new file mode 100644 index 0000000000..2e194fd390 --- /dev/null +++ b/csrc/cpu/micro_gemm/cpu_micro_gemm_int8_neon.hpp @@ -0,0 +1,424 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +#ifndef CPU_MICRO_GEMM_INT8_NEON_HPP +#define CPU_MICRO_GEMM_INT8_NEON_HPP + +#include +#include + +#include "cpu/micro_gemm/cpu_micro_gemm_impl.hpp" + +#include +#include +#include +#include +#include + +namespace cpu_micro_gemm { + +namespace neon_smmla { + +constexpr int32_t K = 8; +constexpr int32_t Cols = 2; +constexpr int32_t TileSize = K * Cols; + +FORCE_INLINE float32x4x2_t load_as_f32(const float* input) { + float32x4x2_t result; + result.val[0] = vld1q_f32(input); + result.val[1] = vld1q_f32(input + 4); + return result; +} + +FORCE_INLINE float32x4x2_t load_as_f32(const c10::Half* input) { + const auto input_vec = vld1q_f16(reinterpret_cast(input)); + float32x4x2_t result; + result.val[0] = vcvt_f32_f16(vget_low_f16(input_vec)); + result.val[1] = vcvt_f32_f16(vget_high_f16(input_vec)); + return result; +} + +FORCE_INLINE float32x4x2_t load_as_f32(const c10::BFloat16* input) { + const auto input_vec = vld1q_bf16(reinterpret_cast(input)); + float32x4x2_t result; + result.val[0] = vcvt_f32_bf16(vget_low_bf16(input_vec)); + result.val[1] = vcvt_f32_bf16(vget_high_bf16(input_vec)); + return result; +} + +FORCE_INLINE void store_acc_rowpair(const int32x4_t acc01, + const int32x4_t acc23, + const int32x4_t acc45, + const int32x4_t acc67, + int32_t* __restrict__ c_ptr, + const int64_t ldc, const int32_t m_rows) { + if (m_rows == 0) { + return; + } + + vst1q_s32(c_ptr, vcombine_s32(vget_low_s32(acc01), vget_low_s32(acc23))); + vst1q_s32(c_ptr + 4, vcombine_s32(vget_low_s32(acc45), vget_low_s32(acc67))); + + if (m_rows == 2) { + vst1q_s32(c_ptr + ldc, + vcombine_s32(vget_high_s32(acc01), vget_high_s32(acc23))); + vst1q_s32(c_ptr + ldc + 4, + vcombine_s32(vget_high_s32(acc45), vget_high_s32(acc67))); + } +} + +FORCE_INLINE void gemm_micro_smmla_8x8_packed_a( + const int8_t* __restrict__ a_packed, const int8_t* __restrict__ b_packed, + int32_t* __restrict__ c_ptr, const int32_t m, const int32_t k_size, + const int64_t ldc) { + const int32x4_t zero = vdupq_n_s32(0); + int32x4_t acc0101 = zero, acc0123 = zero, acc0145 = zero, acc0167 = zero; + int32x4_t acc2301 = zero, acc2323 = zero, acc2345 = zero, acc2367 = zero; + int32x4_t acc4501 = zero, acc4523 = zero, acc4545 = zero, acc4567 = zero; + int32x4_t acc6701 = zero, acc6723 = zero, acc6745 = zero, acc6767 = zero; + + const int8_t* __restrict__ a_tile = a_packed; + const int8_t* __restrict__ b_tile = b_packed; + +#pragma GCC unroll 8 + for (int32_t k_idx = 0; k_idx < k_size; k_idx += K) { + const int8x16_t a_tile01 = vld1q_s8(a_tile); + const int8x16_t a_tile23 = vld1q_s8(a_tile + TileSize); + const int8x16_t a_tile45 = vld1q_s8(a_tile + 2 * TileSize); + const int8x16_t a_tile67 = vld1q_s8(a_tile + 3 * TileSize); + + const int8x16_t b_tile01 = vld1q_s8(b_tile); + const int8x16_t b_tile23 = vld1q_s8(b_tile + TileSize); + const int8x16_t b_tile45 = vld1q_s8(b_tile + 2 * TileSize); + const int8x16_t b_tile67 = vld1q_s8(b_tile + 3 * TileSize); + + acc0101 = vmmlaq_s32(acc0101, a_tile01, b_tile01); + acc2301 = vmmlaq_s32(acc2301, a_tile23, b_tile01); + acc4501 = vmmlaq_s32(acc4501, a_tile45, b_tile01); + acc6701 = vmmlaq_s32(acc6701, a_tile67, b_tile01); + + acc0123 = vmmlaq_s32(acc0123, a_tile01, b_tile23); + acc2323 = vmmlaq_s32(acc2323, a_tile23, b_tile23); + acc4523 = vmmlaq_s32(acc4523, a_tile45, b_tile23); + acc6723 = vmmlaq_s32(acc6723, a_tile67, b_tile23); + + acc0145 = vmmlaq_s32(acc0145, a_tile01, b_tile45); + acc2345 = vmmlaq_s32(acc2345, a_tile23, b_tile45); + acc4545 = vmmlaq_s32(acc4545, a_tile45, b_tile45); + acc6745 = vmmlaq_s32(acc6745, a_tile67, b_tile45); + + acc0167 = vmmlaq_s32(acc0167, a_tile01, b_tile67); + acc2367 = vmmlaq_s32(acc2367, a_tile23, b_tile67); + acc4567 = vmmlaq_s32(acc4567, a_tile45, b_tile67); + acc6767 = vmmlaq_s32(acc6767, a_tile67, b_tile67); + + a_tile += 4 * TileSize; + b_tile += 4 * TileSize; + } + + store_acc_rowpair(acc0101, acc0123, acc0145, acc0167, c_ptr, ldc, + std::min(2, m)); + store_acc_rowpair(acc2301, acc2323, acc2345, acc2367, c_ptr + 2 * ldc, ldc, + std::min(2, std::max(0, m - 2))); + store_acc_rowpair(acc4501, acc4523, acc4545, acc4567, c_ptr + 4 * ldc, ldc, + std::min(2, std::max(0, m - 4))); + store_acc_rowpair(acc6701, acc6723, acc6745, acc6767, c_ptr + 6 * ldc, ldc, + std::min(2, std::max(0, m - 6))); +} + +FORCE_INLINE void gemm_micro_smmla_4x16_packed_a( + const int8_t* __restrict__ a_packed, const int8_t* __restrict__ b_packed, + int32_t* __restrict__ c_ptr, const int32_t m, const int32_t k_size, + const int64_t b_n_group_stride, const int64_t ldc) { + const int32_t m_rows_01 = std::min(2, m); + const int32_t m_rows_23 = std::min(2, std::max(0, m - 2)); + const int32x4_t zero = vdupq_n_s32(0); + + int32x4_t acc0101 = zero, acc0123 = zero, acc0145 = zero, acc0167 = zero; + int32x4_t acc2301 = zero, acc2323 = zero, acc2345 = zero, acc2367 = zero; + int32x4_t acc0189 = zero, acc011011 = zero, acc011213 = zero, + acc011415 = zero; + int32x4_t acc2389 = zero, acc231011 = zero, acc231213 = zero, + acc231415 = zero; + + const int8_t* __restrict__ a_tile = a_packed; + // note: b packs 8 panels contiguously, so we need 2 b_tile ptrs + // for the 4x16 microkernel + const int8_t* __restrict__ b_tile0 = b_packed; + const int8_t* __restrict__ b_tile1 = b_packed + b_n_group_stride; + +#pragma GCC unroll 8 + for (int32_t k_idx = 0; k_idx < k_size; k_idx += K) { + const int8x16_t a_tile01 = vld1q_s8(a_tile); + const int8x16_t a_tile23 = vld1q_s8(a_tile + TileSize); + const int8x16_t b_tile01 = vld1q_s8(b_tile0); + const int8x16_t b_tile23 = vld1q_s8(b_tile0 + TileSize); + const int8x16_t b_tile45 = vld1q_s8(b_tile0 + 2 * TileSize); + const int8x16_t b_tile67 = vld1q_s8(b_tile0 + 3 * TileSize); + const int8x16_t b_tile89 = vld1q_s8(b_tile1); + const int8x16_t b_tile1011 = vld1q_s8(b_tile1 + TileSize); + const int8x16_t b_tile1213 = vld1q_s8(b_tile1 + 2 * TileSize); + const int8x16_t b_tile1415 = vld1q_s8(b_tile1 + 3 * TileSize); + + acc0101 = vmmlaq_s32(acc0101, a_tile01, b_tile01); + acc2301 = vmmlaq_s32(acc2301, a_tile23, b_tile01); + acc0123 = vmmlaq_s32(acc0123, a_tile01, b_tile23); + acc2323 = vmmlaq_s32(acc2323, a_tile23, b_tile23); + + acc0145 = vmmlaq_s32(acc0145, a_tile01, b_tile45); + acc2345 = vmmlaq_s32(acc2345, a_tile23, b_tile45); + acc0167 = vmmlaq_s32(acc0167, a_tile01, b_tile67); + acc2367 = vmmlaq_s32(acc2367, a_tile23, b_tile67); + + acc0189 = vmmlaq_s32(acc0189, a_tile01, b_tile89); + acc2389 = vmmlaq_s32(acc2389, a_tile23, b_tile89); + acc011011 = vmmlaq_s32(acc011011, a_tile01, b_tile1011); + acc231011 = vmmlaq_s32(acc231011, a_tile23, b_tile1011); + + acc011213 = vmmlaq_s32(acc011213, a_tile01, b_tile1213); + acc231213 = vmmlaq_s32(acc231213, a_tile23, b_tile1213); + acc011415 = vmmlaq_s32(acc011415, a_tile01, b_tile1415); + acc231415 = vmmlaq_s32(acc231415, a_tile23, b_tile1415); + + a_tile += 2 * TileSize; + b_tile0 += 4 * TileSize; + b_tile1 += 4 * TileSize; + } + + // rows 0-1, columns 0-7 + store_acc_rowpair(acc0101, acc0123, acc0145, acc0167, c_ptr, ldc, m_rows_01); + // rows 0-1, columns 8-15 + store_acc_rowpair(acc0189, acc011011, acc011213, acc011415, c_ptr + 8, ldc, + m_rows_01); + // rows 2-3, columns 0-7 + store_acc_rowpair(acc2301, acc2323, acc2345, acc2367, c_ptr + 2 * ldc, ldc, + m_rows_23); + // rows 2-3, columns 8-15 + store_acc_rowpair(acc2389, acc231011, acc231213, acc231415, + c_ptr + 2 * ldc + 8, ldc, m_rows_23); +} + +} // namespace neon_smmla + +template +class MicroGemmINT8 { + public: + static constexpr int32_t K = neon_smmla::K; + static constexpr int32_t Mr = 8; + static constexpr int32_t Nr = 8; + static constexpr int32_t NrGemv = 16; + static constexpr int32_t MaxMSize = 8; + static constexpr int32_t NSize = 32; + static constexpr int32_t WeightOCGroupSize = Nr; + static_assert(MaxMSize % Mr == 0); + + static FORCE_INLINE void quantize_row(const scalar_t* input, int8_t* output, + float& scale, const int32_t size) { + TORCH_CHECK_EQ(size % K, 0); + float32x4_t max_vec = vdupq_n_f32(0.0f); + + for (int32_t i = 0; i < size; i += K) { + const float32x4x2_t input_vec = neon_smmla::load_as_f32(input + i); + max_vec = vmaxq_f32(max_vec, vabsq_f32(input_vec.val[0])); + max_vec = vmaxq_f32(max_vec, vabsq_f32(input_vec.val[1])); + } + + const float abs_max = std::max(vmaxvq_f32(max_vec), 1.0e-7f); + scale = abs_max / 127.0f; + const float32x4_t inv_scale_vec = vdupq_n_f32(127.0f / abs_max); + + for (int32_t i = 0; i < size; i += K) { + const float32x4x2_t input_vec = neon_smmla::load_as_f32(input + i); + const int32x4_t output_low = + vcvtnq_s32_f32(vmulq_f32(input_vec.val[0], inv_scale_vec)); + const int32x4_t output_high = + vcvtnq_s32_f32(vmulq_f32(input_vec.val[1], inv_scale_vec)); + const int16x8_t output_s16 = + vcombine_s16(vqmovn_s32(output_low), vqmovn_s32(output_high)); + vst1_s8(output + i, vqmovn_s16(output_s16)); + } + } + + // with current code, fusing this into the gemm micro kernel didn't move the + // needle + static FORCE_INLINE void dequantize_tile( + int32_t* input, float* output, const float* __restrict__ input_scales, + const float* __restrict__ weight_scales, const int32_t m, const int32_t n, + const int32_t stride) { + TORCH_CHECK_EQ(n % 4, 0); + for (int32_t m_idx = 0; m_idx < m; ++m_idx) { + const float32x4_t input_scale_vec = vdupq_n_f32(input_scales[m_idx]); + for (int32_t n_idx = 0; n_idx < n; n_idx += 4) { + const int32x4_t input_vec = vld1q_s32(input + m_idx * stride + n_idx); + const float32x4_t weight_scale_vec = vld1q_f32(weight_scales + n_idx); + const float32x4_t output_vec = + vmulq_f32(vcvtq_f32_s32(input_vec), + vmulq_f32(input_scale_vec, weight_scale_vec)); + vst1q_f32(output + m_idx * stride + n_idx, output_vec); + } + } + } + + // physical layout [ + // M / (8 or 4); Mr is 8 or 4 + // K / 8; K for smmla is 8 + // 4, ; 4 row-pairs for each 8 rows + // 2, ; row-pair is 2 rows + // 4 ; 4 elements per row + // ] + static void pack_input_from_rows(const int8_t* const* __restrict__ rows, + int8_t* __restrict__ a_packed, + const int32_t m, const int32_t k) { + TORCH_CHECK(m > 0 && m <= MaxMSize); + TORCH_CHECK(k % K == 0); + const int8x8_t zero = vdup_n_s8(0); + + for (int32_t row_base = 0; row_base < m; row_base += Mr) { + const int32_t panel_m = std::min(Mr, m - row_base); + const int8_t* const* panel_rows = rows + row_base; + int8_t* __restrict__ out = a_packed + row_base * k; + + // fast path for full 8-row panels (fast path for 4-row panels didn't move + // the needle) + if (panel_m == Mr) { + const int8_t* __restrict__ row0 = panel_rows[0]; + const int8_t* __restrict__ row1 = panel_rows[1]; + const int8_t* __restrict__ row2 = panel_rows[2]; + const int8_t* __restrict__ row3 = panel_rows[3]; + const int8_t* __restrict__ row4 = panel_rows[4]; + const int8_t* __restrict__ row5 = panel_rows[5]; + const int8_t* __restrict__ row6 = panel_rows[6]; + const int8_t* __restrict__ row7 = panel_rows[7]; + int32_t k_idx = 0; + for (; k_idx + 2 * K <= k; k_idx += 2 * K) { + int8_t* __restrict__ block0 = out; + int8_t* __restrict__ block1 = out + 4 * neon_smmla::TileSize; + + int8x16_t a0 = vld1q_s8(row0 + k_idx); + int8x16_t a1 = vld1q_s8(row1 + k_idx); + vst1q_s8(block0, vcombine_s8(vget_low_s8(a0), vget_low_s8(a1))); + vst1q_s8(block1, vcombine_s8(vget_high_s8(a0), vget_high_s8(a1))); + + a0 = vld1q_s8(row2 + k_idx); + a1 = vld1q_s8(row3 + k_idx); + vst1q_s8(block0 + neon_smmla::TileSize, + vcombine_s8(vget_low_s8(a0), vget_low_s8(a1))); + vst1q_s8(block1 + neon_smmla::TileSize, + vcombine_s8(vget_high_s8(a0), vget_high_s8(a1))); + + a0 = vld1q_s8(row4 + k_idx); + a1 = vld1q_s8(row5 + k_idx); + vst1q_s8(block0 + 2 * neon_smmla::TileSize, + vcombine_s8(vget_low_s8(a0), vget_low_s8(a1))); + vst1q_s8(block1 + 2 * neon_smmla::TileSize, + vcombine_s8(vget_high_s8(a0), vget_high_s8(a1))); + + a0 = vld1q_s8(row6 + k_idx); + a1 = vld1q_s8(row7 + k_idx); + vst1q_s8(block0 + 3 * neon_smmla::TileSize, + vcombine_s8(vget_low_s8(a0), vget_low_s8(a1))); + vst1q_s8(block1 + 3 * neon_smmla::TileSize, + vcombine_s8(vget_high_s8(a0), vget_high_s8(a1))); + + out += 8 * neon_smmla::TileSize; + } + + for (; k_idx < k; k_idx += K) { + int8x8_t a0 = vld1_s8(row0 + k_idx); + int8x8_t a1 = vld1_s8(row1 + k_idx); + vst1q_s8(out, vcombine_s8(a0, a1)); + + a0 = vld1_s8(row2 + k_idx); + a1 = vld1_s8(row3 + k_idx); + vst1q_s8(out + neon_smmla::TileSize, vcombine_s8(a0, a1)); + + a0 = vld1_s8(row4 + k_idx); + a1 = vld1_s8(row5 + k_idx); + vst1q_s8(out + 2 * neon_smmla::TileSize, vcombine_s8(a0, a1)); + + a0 = vld1_s8(row6 + k_idx); + a1 = vld1_s8(row7 + k_idx); + vst1q_s8(out + 3 * neon_smmla::TileSize, vcombine_s8(a0, a1)); + + out += 4 * neon_smmla::TileSize; + } + continue; + } + + const int32_t row_pairs = (panel_m <= 4) ? 2 : Mr / 2; + for (int32_t k_idx = 0; k_idx < k; k_idx += K) { + for (int32_t pair_idx = 0; pair_idx < row_pairs; ++pair_idx) { + const int32_t row_idx = pair_idx * 2; + const int8x8_t row0 = + (row_idx < panel_m) ? vld1_s8(panel_rows[row_idx] + k_idx) : zero; + const int8x8_t row1 = (row_idx + 1 < panel_m) + ? vld1_s8(panel_rows[row_idx + 1] + k_idx) + : zero; + vst1q_s8(out, vcombine_s8(row0, row1)); + out += neon_smmla::TileSize; + } + } + } + } + + // physical layout [ + // N / 8; Nr is 8 + // K / 8; K for smmla is 8 + // 4, ; 4 col-pairs for each 8 cols + // 2, ; col-pair is 2 cols + // 4 ; 4 elements per col + // ] + static void pack_weight(const int8_t* __restrict__ weight, + int8_t* __restrict__ packed_weight, + const int32_t output_size, const int32_t input_size) { + TORCH_CHECK(output_size % NSize == 0); + TORCH_CHECK(input_size % K == 0); + + for (int32_t o_idx = 0; o_idx < output_size; o_idx += Nr) { + int8_t* __restrict__ dst = packed_weight + o_idx * input_size; + for (int32_t k_idx = 0; k_idx < input_size; k_idx += K) { + for (int32_t pair_idx = 0; pair_idx < Nr; + pair_idx += neon_smmla::Cols) { + const int8_t* __restrict__ row0 = + weight + (o_idx + pair_idx) * input_size + k_idx; + const int8_t* __restrict__ row1 = row0 + input_size; + vst1q_s8(dst, vcombine_s8(vld1_s8(row0), vld1_s8(row1))); + dst += neon_smmla::TileSize; + } + } + } + } + + void gemm(const int8_t* __restrict__ a_packed, + const int8_t* __restrict__ b_packed, int32_t* __restrict__ c, + const int32_t m, const int32_t k, const int64_t b_n_group_stride, + const int64_t ldc) const { + TORCH_CHECK(m > 0 && m <= MaxMSize); + TORCH_CHECK(k % K == 0); + + for (int32_t n_idx = 0; n_idx < NSize; n_idx += NrGemv) { + const int8_t* __restrict__ b_panel = b_packed + n_idx * k; + + for (int32_t row_base = 0; row_base < m; row_base += Mr) { + const int32_t panel_m = std::min(Mr, m - row_base); + const int8_t* __restrict__ a_panel = a_packed + row_base * k; + int32_t* __restrict__ c_panel = c + row_base * ldc + n_idx; + + if (panel_m <= 4) { + neon_smmla::gemm_micro_smmla_4x16_packed_a( + a_panel, b_panel, c_panel, panel_m, k, b_n_group_stride, ldc); + } else { + neon_smmla::gemm_micro_smmla_8x8_packed_a(a_panel, b_panel, c_panel, + panel_m, k, ldc); + neon_smmla::gemm_micro_smmla_8x8_packed_a( + a_panel, b_panel + b_n_group_stride, c_panel + Nr, panel_m, k, + ldc); + } + } + } + } +}; + +} // namespace cpu_micro_gemm + +#endif diff --git a/csrc/cpu/micro_gemm/cpu_micro_gemm_neon.hpp b/csrc/cpu/micro_gemm/cpu_micro_gemm_neon.hpp index 7d4898852b..b38337956f 100644 --- a/csrc/cpu/micro_gemm/cpu_micro_gemm_neon.hpp +++ b/csrc/cpu/micro_gemm/cpu_micro_gemm_neon.hpp @@ -1,3 +1,6 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + #ifndef CPU_MICRO_GEMM_NEON_HPP #define CPU_MICRO_GEMM_NEON_HPP @@ -16,9 +19,6 @@ namespace { constexpr int32_t K = 4; constexpr int32_t Cols = 2; constexpr int32_t TileSize = K * Cols; -constexpr int32_t Mr = 8; -constexpr int32_t Nr = 8; -constexpr int32_t Nr_gemv = 16; // a = [a0, a1, a2, a3], b = [b0, b1, b2, b3] -> [a0, a1, b0, b1] FORCE_INLINE float32x4_t zip1_f32x4(const float32x4_t a, const float32x4_t b) { @@ -132,7 +132,7 @@ FORCE_INLINE void gemm_micro_bfmmla_8x8_packed_a( acc6767 = vbfmmlaq_f32(acc6767, a_tile67, b_tile67); a_tile += 4 * TileSize; - b_tile += Nr * K; + b_tile += 4 * TileSize; } store_acc_rowpair(acc0101, acc0123, acc0145, acc0167, c_ptr, ldc, @@ -205,8 +205,8 @@ FORCE_INLINE void gemm_micro_bfmmla_4x16_packed_a( acc231415 = vbfmmlaq_f32(acc231415, a_tile23, b_tile1415); a_tile += 2 * TileSize; - b_tile0 += Nr * K; - b_tile1 += Nr * K; + b_tile0 += 4 * TileSize; + b_tile1 += 4 * TileSize; } store_acc_rowpair(acc0101, acc0123, acc0145, acc0167, c_ptr, ldc, m_rows_01); @@ -223,6 +223,9 @@ FORCE_INLINE void gemm_micro_bfmmla_4x16_packed_a( template class MicroGemm { public: + static constexpr int32_t Mr = 8; + static constexpr int32_t Nr = 8; + static constexpr int32_t NrGemv = 16; static constexpr int32_t MaxMSize = 8; static constexpr int32_t NSize = 32; static constexpr int32_t WeightOCGroupSize = Nr; @@ -246,6 +249,9 @@ class MicroGemm { public: using scalar_t = c10::BFloat16; + static constexpr int32_t Mr = 8; + static constexpr int32_t Nr = 8; + static constexpr int32_t NrGemv = 16; static constexpr int32_t MaxMSize = 8; static constexpr int32_t NSize = 32; static constexpr int32_t WeightOCGroupSize = Nr; @@ -253,7 +259,7 @@ class MicroGemm { public: // physical layout [ - // M / 8; Mr is 8 + // M / (8 or 4); Mr is 8 or 4 // K / 4; K for bfmmla is 4 // 4, ; 4 row-pairs for each 8 rows // 2, ; row-pair is 2 rows @@ -439,7 +445,7 @@ class MicroGemm { (void)lda; // A is packed, so lda is not needed TORCH_CHECK_EQ(k % K, 0); - for (int32_t n_idx = 0; n_idx < NSize; n_idx += Nr_gemv) { + for (int32_t n_idx = 0; n_idx < NSize; n_idx += NrGemv) { const bfloat16_t* __restrict__ b_panel = reinterpret_cast(b_ptr) + n_idx * k; diff --git a/csrc/cpu/torch_bindings.cpp b/csrc/cpu/torch_bindings.cpp index 8b7d924dac..d5876202f1 100644 --- a/csrc/cpu/torch_bindings.cpp +++ b/csrc/cpu/torch_bindings.cpp @@ -207,6 +207,20 @@ void cpu_fused_moe(torch::Tensor& output, const torch::Tensor& input, const torch::Tensor& topk_id, const bool skip_weighted, const std::string& act, const std::string& isa); +void prepack_moe_weight_int8(const torch::Tensor& weight, + torch::Tensor& packed_weight, + const std::string& isa); + +void cpu_fused_moe_int8(torch::Tensor& output, const torch::Tensor& input, + const torch::Tensor& w13, const torch::Tensor& w2, + const torch::Tensor& w13_scale, + const torch::Tensor& w2_scale, + const std::optional& w13_bias, + const std::optional& w2_bias, + const torch::Tensor& topk_weights, + const torch::Tensor& topk_id, const bool skip_weighted, + const std::string& act, const std::string& isa); + void compute_slot_mapping_kernel_impl(const torch::Tensor query_start_loc, const torch::Tensor positions, const torch::Tensor block_table, @@ -596,8 +610,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) { #endif // fused moe -#if defined(__AVX512F__) || \ - (defined(__aarch64__) && !defined(__APPLE__) && defined(ARM_BF16_SUPPORT)) +#if defined(__AVX512F__) || (defined(ARM_BF16_SUPPORT) && !defined(__APPLE__)) ops.def( "prepack_moe_weight(Tensor weight, Tensor(a1!) packed_weight, str isa) " "-> ()"); @@ -608,7 +621,22 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) { "bool skip_weighted, " "str act, str isa) -> ()"); ops.impl("cpu_fused_moe", torch::kCPU, &cpu_fused_moe); -#endif +#endif // #if defined(__AVX512F__) || (defined(ARM_BF16_SUPPORT) && + // !defined(__APPLE__)) +#if defined(ARM_I8MM_SUPPORT) && defined(ARM_BF16_SUPPORT) && \ + !defined(__APPLE__) + ops.def( + "prepack_moe_weight_int8(Tensor weight, Tensor(a1!) packed_weight, " + "str isa) -> ()"); + ops.impl("prepack_moe_weight_int8", torch::kCPU, &prepack_moe_weight_int8); + ops.def( + "cpu_fused_moe_int8(Tensor(a0!) output, Tensor input, Tensor w13, " + "Tensor w2, Tensor w13_scale, Tensor w2_scale, Tensor? w13_bias, " + "Tensor? w2_bias, Tensor topk_weights, Tensor topk_id, bool " + "skip_weighted, str act, str isa) -> ()"); + ops.impl("cpu_fused_moe_int8", torch::kCPU, &cpu_fused_moe_int8); +#endif // #if defined(ARM_I8MM_SUPPORT) && defined(ARM_BF16_SUPPORT) && + // !defined(__APPLE__) ops.def( "mla_decode_kvcache(" " Tensor! out, Tensor query, Tensor kv_cache," diff --git a/tests/kernels/moe/test_cpu_fused_moe.py b/tests/kernels/moe/test_cpu_fused_moe.py index 41ae9be517..75e34e6766 100644 --- a/tests/kernels/moe/test_cpu_fused_moe.py +++ b/tests/kernels/moe/test_cpu_fused_moe.py @@ -7,10 +7,14 @@ import torch from tests.kernels.allclose_default import get_default_atol, get_default_rtol from vllm._custom_ops import ( cpu_fused_moe, + cpu_fused_moe_int8, cpu_prepack_moe_weight, + cpu_prepack_moe_weight_int8, ) from vllm.model_executor.layers.fused_moe.activation import MoEActivation -from vllm.model_executor.layers.fused_moe.cpu_fused_moe import _CPU_MOE_ACT_FN +from vllm.model_executor.layers.fused_moe.cpu_fused_moe import ( + _CPU_MOE_ACT_FN, +) from vllm.platforms import CpuArchEnum, current_platform from vllm.utils.torch_utils import set_random_seed @@ -101,6 +105,90 @@ def ref_fused_moe( return final_out +def quantize_per_channel( + weight: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor]: + """Symmetrically quantize each weight output channel.""" + weight_fp32 = weight.float() + scale = weight_fp32.abs().amax(dim=-1).clamp_min(1e-12) / 127.0 + quantized = ( + (weight_fp32 / scale.unsqueeze(-1)).round().clamp(-127, 127).to(torch.int8) + ) + return quantized, scale + + +def quantize_per_token( + input: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor]: + """Symmetrically quantize each input token.""" + input_fp32 = input.float() + scale = input_fp32.abs().amax(dim=-1, keepdim=True).clamp_min(1e-7) / 127.0 + quantized = (input_fp32 / scale).round().clamp(-127, 127).to(torch.int8) + return quantized, scale + + +def ref_fused_moe_int8( + input: torch.Tensor, + w13: torch.Tensor, + w2: torch.Tensor, + w13_scale: torch.Tensor, + w2_scale: torch.Tensor, + w13_bias: torch.Tensor | None, + w2_bias: torch.Tensor | None, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + activation: MoEActivation, +) -> torch.Tensor: + """Reference the two dynamically quantized INT8 GEMMs.""" + input_int8, input_scale = quantize_per_token(input) + expert_num = w13.size(0) + + counts = topk_ids.new_zeros((topk_ids.size(0), expert_num)) + counts.scatter_(1, topk_ids.to(torch.int64), 1) + tokens_per_expert = counts.sum(dim=0).tolist() + sorted_route_ids = topk_ids.view(-1).argsort() + sorted_token_ids = sorted_route_ids // topk_ids.size(1) + + outputs = [] + start_idx = 0 + for expert_idx, token_count in enumerate(tokens_per_expert): + end_idx = start_idx + token_count + if token_count == 0: + continue + + token_ids = sorted_token_ids[start_idx:end_idx] + gate_up = torch.matmul( + input_int8[token_ids].float(), + w13[expert_idx].float().T, + ) + gate_up *= input_scale[token_ids] * w13_scale[expert_idx] + if w13_bias is not None: + gate_up += w13_bias[expert_idx].float() + + intermediate = _CPU_MOE_ACT_FN[activation](gate_up).to(input.dtype) + intermediate_int8, intermediate_scale = quantize_per_token(intermediate) + output = torch.matmul( + intermediate_int8.float(), + w2[expert_idx].float().T, + ) + output *= intermediate_scale * w2_scale[expert_idx] + if w2_bias is not None: + output += w2_bias[expert_idx].float() + + outputs.append(output) + start_idx = end_idx + + sorted_output = torch.cat(outputs, dim=0) + routed_output = torch.empty_like(sorted_output) + routed_output[sorted_route_ids] = sorted_output + return ( + routed_output.view(*topk_ids.shape, -1) + .mul_(topk_weights.unsqueeze(-1)) + .sum(dim=1) + .to(input.dtype) + ) + + @pytest.mark.parametrize("batch_size", BATCH_SIZE) @pytest.mark.parametrize("expert_num", EXPERT_NUM) @pytest.mark.parametrize("hidden_size", HIDDEN_DIM) @@ -176,3 +264,85 @@ def test_cpu_fused_moe( torch.testing.assert_close(output, ref_output, atol=atol, rtol=rtol), f"{torch.max(torch.abs(output - ref_output))}", ) + + +@pytest.mark.skipif( + current_platform.get_cpu_architecture() != CpuArchEnum.ARM, + reason="Requires Arm CPU", +) +@pytest.mark.parametrize("batch_size", BATCH_SIZE) +@pytest.mark.parametrize("expert_num", EXPERT_NUM) +@pytest.mark.parametrize("hidden_size", HIDDEN_DIM) +@pytest.mark.parametrize("intermediate_size", INTERMEDIATE_DIM) +@pytest.mark.parametrize("use_bias", USE_BIAS) +@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("act", ACT) +@pytest.mark.parametrize("isa", ["neon"]) +def test_cpu_fused_moe_int8( + batch_size: int, + expert_num: int, + hidden_size: int, + intermediate_size: int, + use_bias: bool, + dtype: torch.dtype, + act: MoEActivation, + isa: str, +): + set_random_seed(0) + topk_num = max(expert_num // 2, 1) + up_dim = 2 * intermediate_size + + input = torch.randn((batch_size, hidden_size), dtype=dtype) / ( + 0.5 * hidden_size**0.5 + ) + w13_fp = torch.randn((expert_num, up_dim, hidden_size), dtype=dtype) / ( + 0.5 * hidden_size**0.5 + ) + w2_fp = torch.randn((expert_num, hidden_size, intermediate_size), dtype=dtype) / ( + 0.5 * intermediate_size**0.5 + ) + w13, w13_scale = quantize_per_channel(w13_fp) + w2, w2_scale = quantize_per_channel(w2_fp) + + w13_bias = None + w2_bias = None + if use_bias: + w13_bias = torch.randn((expert_num, up_dim), dtype=dtype) / (0.5 * up_dim**0.5) + w2_bias = torch.randn((expert_num, hidden_size), dtype=dtype) / ( + 0.5 * hidden_size**0.5 + ) + + router_logits = torch.randn((batch_size, expert_num), dtype=dtype) + score = torch.softmax(router_logits, dim=-1, dtype=torch.float32) + topk_weights, topk_ids = torch.topk(score, topk_num) + topk_ids = topk_ids.to(torch.int32) + + ref_output = ref_fused_moe_int8( + input, + w13, + w2, + w13_scale, + w2_scale, + w13_bias, + w2_bias, + topk_weights, + topk_ids, + act, + ) + packed_w13 = cpu_prepack_moe_weight_int8(w13, isa) + packed_w2 = cpu_prepack_moe_weight_int8(w2, isa) + output = cpu_fused_moe_int8( + input, + packed_w13, + packed_w2, + w13_scale, + w2_scale, + w13_bias, + w2_bias, + topk_weights, + topk_ids, + act.value, + isa, + ) + + torch.testing.assert_close(output, ref_output, atol=2e-2, rtol=2e-2) diff --git a/vllm/_custom_ops.py b/vllm/_custom_ops.py index aa75c50a51..82044c87c1 100644 --- a/vllm/_custom_ops.py +++ b/vllm/_custom_ops.py @@ -3787,6 +3787,15 @@ def cpu_prepack_moe_weight( return output +def cpu_prepack_moe_weight_int8( + weight: torch.Tensor, + isa: str, +) -> torch.Tensor: + output = torch.empty_like(weight) + torch.ops._C.prepack_moe_weight_int8(weight, output, isa) + return output + + def cpu_fused_moe( input: torch.Tensor, w13: torch.Tensor, @@ -3816,6 +3825,39 @@ def cpu_fused_moe( return output +def cpu_fused_moe_int8( + input: torch.Tensor, + w13: torch.Tensor, + w2: torch.Tensor, + w13_scale: torch.Tensor, + w2_scale: torch.Tensor, + w13_bias: torch.Tensor | None, + w2_bias: torch.Tensor | None, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + act: str, + isa: str, + skip_weighted: bool = False, +) -> torch.Tensor: + output = torch.empty_like(input) + torch.ops._C.cpu_fused_moe_int8( + output, + input, + w13, + w2, + w13_scale, + w2_scale, + w13_bias, + w2_bias, + topk_weights, + topk_ids, + skip_weighted, + act, + isa, + ) + return output + + if hasattr(torch.ops._qutlass_C, "matmul_mxf4_bf16_tn"): @register_fake("_qutlass_C::matmul_mxf4_bf16_tn") diff --git a/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py b/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py index 3ed1734cb9..bf6b00377f 100644 --- a/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py @@ -9,6 +9,8 @@ from vllm._custom_ops import ( CPUQuantAlgo, CPUQuantMethod, convert_weight_packed_scale_zp, + cpu_fused_moe_int8, + cpu_prepack_moe_weight_int8, fused_experts_cpu, ) from vllm.model_executor.layers.fused_moe.activation import MoEActivation @@ -27,7 +29,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( kInt8StaticChannelSym, kMxfp4Static, ) -from vllm.platforms import current_platform +from vllm.platforms import CpuArchEnum, current_platform # =========================================================================== # FP8 W8A16 MoE @@ -556,7 +558,14 @@ def prepare_int8_moe_layer_for_cpu( w13: torch.Tensor, w2: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor]: - """VNNI-prepack INT8 MoE weights for CPU kernel.""" + """Prepack INT8 MoE weights for the current CPU architecture.""" + # SMMLA packing for AArch64 + if current_platform.get_cpu_architecture() == CpuArchEnum.ARM: + return ( + cpu_prepack_moe_weight_int8(w13, "neon"), + cpu_prepack_moe_weight_int8(w2, "neon"), + ) + # VNNI packing for x86 packed_w13 = torch.ops._C.convert_weight_packed(w13) packed_w2 = torch.ops._C.convert_weight_packed(w2) return packed_w13, packed_w2 @@ -586,7 +595,10 @@ class CPUExpertsInt8(mk.FusedMoEExpertsMonolithic): @staticmethod def _supports_current_device() -> bool: - return current_platform.is_cpu() + return ( + current_platform.is_cpu() + and current_platform.get_cpu_architecture() == CpuArchEnum.X86 + ) @staticmethod def _supports_no_act_and_mul() -> bool: @@ -701,3 +713,167 @@ class CPUExpertsInt8(mk.FusedMoEExpertsMonolithic): None, # limit True, # is_vnni ) + + +class ArmCPUExpertsInt8(mk.FusedMoEExpertsMonolithic): + """Arm INT8 MoE with per-token activation and channelwise weight quantization.""" + + @property + def expects_unquantized_inputs(self) -> bool: + return True + + @staticmethod + def activation_format() -> mk.FusedMoEActivationFormat: + return mk.FusedMoEActivationFormat.Standard + + @staticmethod + def is_supported_config( + cls: type[mk.FusedMoEExperts], + moe_config: FusedMoEConfig, + weight_key: QuantKey | None, + activation_key: QuantKey | None, + activation_format: mk.FusedMoEActivationFormat, + ) -> tuple[bool, str | None]: + supported, reason = mk.FusedMoEExperts.is_supported_config( + cls, + moe_config, + weight_key, + activation_key, + activation_format, + ) + if not supported: + return supported, reason + if moe_config.in_dtype not in ( + torch.float32, + torch.float16, + torch.bfloat16, + ): + return False, "kernel requires float32, float16, or bfloat16 activations" + if moe_config.hidden_dim % 32 != 0: + return False, "kernel requires hidden dim divisible by 32" + if moe_config.intermediate_size_per_partition % 32 != 0: + return False, "kernel requires intermediate dim divisible by 32" + return True, None + + @staticmethod + def _supports_current_device() -> bool: + return ( + current_platform.is_cpu() + and current_platform.get_cpu_architecture() == CpuArchEnum.ARM + and hasattr(torch.ops._C, "cpu_fused_moe_int8") + ) + + @staticmethod + def _supports_no_act_and_mul() -> bool: + return False + + @staticmethod + def _supports_activation(activation: MoEActivation) -> bool: + return activation in ( + MoEActivation.SILU, + MoEActivation.SWIGLUOAI, + MoEActivation.GELU, + MoEActivation.GELU_TANH, + ) + + @staticmethod + def _supports_parallel_config( + moe_parallel_config: FusedMoEParallelConfig, + ) -> bool: + return not moe_parallel_config.use_ep + + @staticmethod + def _supports_quant_scheme( + weight_key: QuantKey | None, + activation_key: QuantKey | None, + ) -> bool: + return (weight_key, activation_key) == ( + kInt8StaticChannelSym, + kInt8DynamicTokenSym, + ) + + @staticmethod + def _supports_routing_method( + routing_method: RoutingMethodType, + weight_key: QuantKey | None, + activation_key: QuantKey | None, + ) -> bool: + return routing_method in [ + RoutingMethodType.Default, + RoutingMethodType.Renormalize, + RoutingMethodType.RenormalizeNaive, + ] + + @staticmethod + def _supports_router_logits_dtype( + router_logits_dtype: torch.dtype | None, + routing_method: RoutingMethodType, + ) -> bool: + return True + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + from vllm.model_executor.utils import replace_parameter + + w13, w2 = prepare_int8_moe_layer_for_cpu(layer.w13_weight, layer.w2_weight) + replace_parameter(layer, "w13_weight", w13) + replace_parameter(layer, "w2_weight", w2) + + def apply( + self, + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + router_logits: torch.Tensor, + activation: MoEActivation, + global_num_experts: int, + expert_map: torch.Tensor | None, + a1q_scale: torch.Tensor | None, + apply_router_weight_on_input: bool, + num_expert_group: int | None = None, + e_score_correction_bias: torch.Tensor | None = None, + routed_scaling_factor: float | None = None, + topk_group: int | None = None, + ) -> torch.Tensor: + from vllm.model_executor.layers.fused_moe.cpu_fused_moe import ( + select_experts, + ) + + topk_weights, topk_ids = select_experts( + hidden_states=hidden_states, + router_logits=router_logits, + use_grouped_topk=num_expert_group is not None, + top_k=self.moe_config.experts_per_token, + renormalize=self.moe_config.routing_method + in ( + RoutingMethodType.Renormalize, + RoutingMethodType.RenormalizeNaive, + ), + topk_group=topk_group, + num_expert_group=num_expert_group, + scoring_func="softmax", + routed_scaling_factor=( + routed_scaling_factor if routed_scaling_factor is not None else 1.0 + ), + e_score_correction_bias=e_score_correction_bias, + ) + + if apply_router_weight_on_input: + assert topk_ids.size(1) == 1 + hidden_states.mul_(topk_weights.to(hidden_states.dtype)) + + assert self.w1_scale is not None + assert self.w2_scale is not None + return cpu_fused_moe_int8( + hidden_states, + w1, + w2, + self.w1_scale, + self.w2_scale, + self.w1_bias, + self.w2_bias, + topk_weights, + topk_ids, + activation.value, + "neon", + skip_weighted=apply_router_weight_on_input, + ) diff --git a/vllm/model_executor/layers/fused_moe/oracle/int8.py b/vllm/model_executor/layers/fused_moe/oracle/int8.py index 5a2b4c3a75..e2fb3e9114 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/int8.py +++ b/vllm/model_executor/layers/fused_moe/oracle/int8.py @@ -24,7 +24,6 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( kInt8StaticChannelSym, ) from vllm.model_executor.utils import replace_parameter -from vllm.platforms import current_platform logger = init_logger(__name__) @@ -41,20 +40,12 @@ def _get_priority_backends( """ Get available backends in priority order based on platform and config. """ - _AVAILABLE_BACKENDS = [ + return [ Int8MoeBackend.TRITON, Int8MoeBackend.HUMMING, Int8MoeBackend.CPU, ] - def _move_to_front(backends: list[Int8MoeBackend], backend: Int8MoeBackend) -> None: - backends.insert(0, backends.pop(backends.index(backend))) - - if current_platform.is_cpu(): - _move_to_front(_AVAILABLE_BACKENDS, Int8MoeBackend.CPU) - - return _AVAILABLE_BACKENDS - def backend_to_kernel_cls( backend: Int8MoeBackend, @@ -78,14 +69,13 @@ def backend_to_kernel_cls( HummingGroupedExperts, HummingIndexedExperts, ] - elif backend == Int8MoeBackend.CPU: from vllm.model_executor.layers.fused_moe.experts.cpu_moe import ( + ArmCPUExpertsInt8, CPUExpertsInt8, ) - return [CPUExpertsInt8] - + return [ArmCPUExpertsInt8, CPUExpertsInt8] else: raise ValueError(f"Unknown Int8 MoE backend: {backend.value}")