diff --git a/.buildkite/hardware_tests/cpu.yaml b/.buildkite/hardware_tests/cpu.yaml index 1f6c6e61c9..92b0196ec2 100644 --- a/.buildkite/hardware_tests/cpu.yaml +++ b/.buildkite/hardware_tests/cpu.yaml @@ -65,6 +65,7 @@ steps: - vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py - vllm/model_executor/layers/quantization/kernels/scaled_mm/cpu.py - vllm/model_executor/layers/quantization/kernels/mixed_precision/cpu.py + - vllm/model_executor/layers/fused_moe/experts/cpu_moe.py - tests/quantization/test_compressed_tensors.py - tests/quantization/test_cpu_wna16.py commands: diff --git a/cmake/cpu_extension.cmake b/cmake/cpu_extension.cmake index 650dbf3651..d27a5ea93d 100644 --- a/cmake/cpu_extension.cmake +++ b/cmake/cpu_extension.cmake @@ -409,12 +409,15 @@ endif() if (ENABLE_X86_ISA) set(VLLM_EXT_SRC_SGL + "csrc/cpu/sgl-kernels/fla.cpp" + "csrc/cpu/sgl-kernels/conv.cpp" "csrc/cpu/sgl-kernels/gemm.cpp" "csrc/cpu/sgl-kernels/gemm_int8.cpp" "csrc/cpu/sgl-kernels/gemm_fp8.cpp" "csrc/cpu/sgl-kernels/gemm_int4.cpp" "csrc/cpu/sgl-kernels/moe.cpp" "csrc/cpu/sgl-kernels/moe_int8.cpp" + "csrc/cpu/sgl-kernels/moe_int4.cpp" "csrc/cpu/sgl-kernels/moe_fp8.cpp") set(VLLM_EXT_SRC_AVX512 diff --git a/csrc/cpu/sgl-kernels/common.h b/csrc/cpu/sgl-kernels/common.h index 31be725fab..425d5d131f 100644 --- a/csrc/cpu/sgl-kernels/common.h +++ b/csrc/cpu/sgl-kernels/common.h @@ -1,13 +1,12 @@ // Adapted from // https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu +// clang-format off + #pragma once #include #include -#include - -// clang-format off #if defined(_OPENMP) #include @@ -16,40 +15,157 @@ namespace { // dispatch bool -#define AT_DISPATCH_BOOL(BOOL_V, BOOL_NAME, ...) \ - [&] { \ - if (BOOL_V) { \ - constexpr bool BOOL_NAME = true; \ - return __VA_ARGS__(); \ - } else { \ - constexpr bool BOOL_NAME = false; \ - return __VA_ARGS__(); \ - } \ +#define AT_DISPATCH_BOOL(BOOL_V, BOOL_NAME, ...) \ + [&] { \ + if (BOOL_V) { \ + constexpr bool BOOL_NAME = true; \ + return __VA_ARGS__(); \ + } else { \ + constexpr bool BOOL_NAME = false; \ + return __VA_ARGS__(); \ + } \ }() -// dispatch: bfloat16, float16, int8_t, fp8_e4m3 -#define CPU_DISPATCH_PACKED_TYPES(TYPE, ...) \ - [&] { \ - switch (TYPE) { \ - case at::ScalarType::BFloat16 : { \ - using packed_t = at::BFloat16; \ - return __VA_ARGS__(); \ - } \ - case at::ScalarType::Half: { \ - using packed_t = at::Half; \ - return __VA_ARGS__(); \ - } \ - case at::ScalarType::Char : { \ - using packed_t = int8_t; \ - return __VA_ARGS__(); \ - } \ - case at::ScalarType::Float8_e4m3fn : { \ - using packed_t = at::Float8_e4m3fn; \ - return __VA_ARGS__(); \ - } \ - default: \ - TORCH_CHECK(false, "Unsupported floating data type.\n"); \ - } \ +#define AT_DISPATCH_BOOL2(BOOL_V1, BOOL_NAME1, BOOL_V2, BOOL_NAME2, ...) \ + [&] { \ + if (BOOL_V1) { \ + constexpr bool BOOL_NAME1 = true; \ + if (BOOL_V2) { \ + constexpr bool BOOL_NAME2 = true; \ + return __VA_ARGS__(); \ + } else { \ + constexpr bool BOOL_NAME2 = false; \ + return __VA_ARGS__(); \ + } \ + } else { \ + constexpr bool BOOL_NAME1 = false; \ + if (BOOL_V2) { \ + constexpr bool BOOL_NAME2 = true; \ + return __VA_ARGS__(); \ + } else { \ + constexpr bool BOOL_NAME2 = false; \ + return __VA_ARGS__(); \ + } \ + } \ + }() + +// dispatch: bfloat16, float16, int8_t, fp8_e4m3, uint8_t(mxfp4/int4) +#define CPU_DISPATCH_PACKED_TYPES(TYPE, ...) \ + [&] { \ + switch (TYPE) { \ + case at::ScalarType::BFloat16: { \ + using packed_t = at::BFloat16; \ + return __VA_ARGS__(); \ + } \ + case at::ScalarType::Half: { \ + using packed_t = at::Half; \ + return __VA_ARGS__(); \ + } \ + case at::ScalarType::Char: { \ + using packed_t = int8_t; \ + return __VA_ARGS__(); \ + } \ + case at::ScalarType::Float8_e4m3fn: { \ + using packed_t = at::Float8_e4m3fn; \ + return __VA_ARGS__(); \ + } \ + case at::ScalarType::Byte: { \ + using packed_t = uint8_t; \ + return __VA_ARGS__(); \ + } \ + default: \ + TORCH_CHECK(false, "Unsupported floating data type.\n"); \ + } \ + }() + +// Helper MICRO for CPU_DISPATCH_FLOATING_TYPES_EXT: +// TYPE1: the primary dtype (input, output, weight); +// TYPE2: defined as PARAM_T input +#define CPU_DISPATCH_TYPE1_WITH_PARAM(TYPE1, PARAM_T, ...) \ + switch (TYPE1) { \ + case at::ScalarType::BFloat16: { \ + using scalar_t = at::BFloat16; \ + using param_t = PARAM_T; \ + return __VA_ARGS__(); \ + } \ + case at::ScalarType::Half: { \ + using scalar_t = at::Half; \ + using param_t = PARAM_T; \ + return __VA_ARGS__(); \ + } \ + case at::ScalarType::Float: { \ + using scalar_t = float; \ + using param_t = PARAM_T; \ + return __VA_ARGS__(); \ + } \ + default: \ + TORCH_CHECK(false, "Unsupported floating data type."); \ + } + +// Helper MICRO for CPU_DISPATCH_REDUCED_FLOATING_TYPES_EXT: +// TYPE1: the primary dtype (input, output, weight); +// TYPE2: defined as PARAM_T input +#define CPU_DISPATCH_TYPE1_WITH_PARAM_REDUCED(TYPE1, PARAM_T, ...) \ + switch (TYPE1) { \ + case at::ScalarType::BFloat16: { \ + using scalar_t = at::BFloat16; \ + using param_t = PARAM_T; \ + return __VA_ARGS__(); \ + } \ + case at::ScalarType::Half: { \ + using scalar_t = at::Half; \ + using param_t = PARAM_T; \ + return __VA_ARGS__(); \ + } \ + default: \ + TORCH_CHECK(false, "Unsupported floating data type."); \ + } + +// Helper MICRO for CPU_DISPATCH_REDUCED_FLOATING_TYPES_EXT: +// TYPE1: the dtype both for scalar_t and param_t +#define CPU_DISPATCH_TYPE1_WITH_SAME_PARAM_REDUCED(TYPE1, ...) \ + switch (TYPE1) { \ + case at::ScalarType::BFloat16: { \ + using scalar_t = at::BFloat16; \ + using param_t = at::BFloat16; \ + return __VA_ARGS__(); \ + } \ + case at::ScalarType::Half: { \ + using scalar_t = at::Half; \ + using param_t = at::Half; \ + return __VA_ARGS__(); \ + } \ + default: \ + TORCH_CHECK(false, "Unsupported reduced floating data type."); \ + } + +// dispatch with mixed dtypes (TYPE1, TYPE2): +// TYPE1: the primary dtype (input, output, weight); +// TYPE2: the secondary dtype (bias, etc.). +#define CPU_DISPATCH_FLOATING_TYPES_EXT(TYPE1, TYPE2, ...) \ + [&] { \ + if (TYPE2 == at::kFloat) { \ + CPU_DISPATCH_TYPE1_WITH_PARAM(TYPE1, float, __VA_ARGS__) \ + } else if (TYPE2 == at::ScalarType::BFloat16) { \ + CPU_DISPATCH_TYPE1_WITH_PARAM(TYPE1, at::BFloat16, __VA_ARGS__) \ + } else if (TYPE2 == at::ScalarType::Half) { \ + CPU_DISPATCH_TYPE1_WITH_PARAM(TYPE1, at::Half, __VA_ARGS__) \ + } else { \ + TORCH_CHECK(false, "Unsupported floating data type."); \ + } \ + }() + +// dispatch with mixed dtypes (reduced one, no float for TYPE1) (TYPE1, TYPE2): +// TYPE1: the primary dtype (input, output, weight); +// TYPE2: the secondary dtype (bias, etc.). +#define CPU_DISPATCH_REDUCED_FLOATING_TYPES_EXT(TYPE1, TYPE2, ...) \ + [&] { \ + if (TYPE2 == at::kFloat) { \ + CPU_DISPATCH_TYPE1_WITH_PARAM_REDUCED(TYPE1, float, __VA_ARGS__) \ + } else { \ + TORCH_CHECK(TYPE1 == TYPE2); \ + CPU_DISPATCH_TYPE1_WITH_SAME_PARAM_REDUCED(TYPE1, __VA_ARGS__) \ + } \ }() #define UNUSED(x) (void)(x) @@ -70,13 +186,51 @@ namespace { #define CHECK_DIM(d, x) TORCH_CHECK(x.dim() == d, #x " must be a " #d "D tensor") #define CHECK_EQ(a, b) TORCH_CHECK((a) == (b), "CHECK_EQ(" #a ", " #b ") failed. ", a, " vs ", b) +#define CHECK_GT(a, b) TORCH_CHECK((a) > (b), "CHECK_GT(" #a ", " #b ") failed. ", a, " vs ", b) +#define CHECK_GE(a, b) TORCH_CHECK((a) >= (b), "CHECK_GE(" #a ", " #b ") failed. ", a, " vs ", b) -// parallel routines +template +static inline void CHECK_INPUT_SHAPE_DTYPE(const at::Tensor& tensor, const at::IntArrayRef sizes, at::ScalarType st) { + TORCH_CHECK(tensor.sizes() == sizes, "Input tensor shape mismatch: expected ", sizes, ", got ", tensor.sizes()); + TORCH_CHECK(tensor.scalar_type() == st, "Input tensor dtype mismatch"); + if constexpr (is_only_lastdim_contiguous) { + CHECK_LAST_DIM_CONTIGUOUS_INPUT(tensor); + } else { + CHECK_INPUT(tensor); + } +} + +// [NB] Parallel Routines +// +// * at::parallel_for - applies for most of generic use cases, this will be compiled +// against openmp in default torch release. +// +// * parallel_for - same function as above, can choose payload partition scheme in +// balance211. +// +// * parallel_2d - parallel for 2 dimensions, used in GEMM, etc. +// this one will do payload balance across 2 dimensions. +// + +// grain size for each thread constexpr int GRAIN_SIZE = 1024; template ::value, int>::type = 0> -inline T div_up(T x, T y) { return (x + y - 1) / y; } +inline T div_up(T x, T y) { + return (x + y - 1) / y; +} +// you can only use at::get_thread_num() with at::parallel_for() +// as it is lazy initialized, otherwise it will always return 0. +inline int get_thread_num() { +#if defined(_OPENMP) + return omp_get_thread_num(); +#else + return 0; +#endif +} + +// balance payload across each thread template inline void balance211(T n, T nth, T ith, T& n_start, T& n_end) { #if 0 @@ -94,10 +248,10 @@ inline void balance211(T n, T nth, T ith, T& n_start, T& n_end) { } n_end += n_start; #else - // pytorch aten partition pattern - T n_my = div_up(n, nth); - n_start = ith * n_my; - n_end = std::min(n_start + n_my, n); + // pytorch aten partition pattern + T n_my = div_up(n, nth); + n_start = ith * n_my; + n_end = std::min(n_start + n_my, n); #endif } @@ -105,23 +259,15 @@ template inline void parallel_for(int n, const func_t& f) { #if defined(_OPENMP) #pragma omp parallel -{ + { int nth = omp_get_num_threads(); int ith = omp_get_thread_num(); int tbegin, tend; balance211(n, nth, ith, tbegin, tend); f(tbegin, tend); -} + } #else - f(0, n); -#endif -} - -inline int get_thread_num() { -#if defined(_OPENMP) - return omp_get_thread_num(); -#else - return 0; + f(0, n); #endif } @@ -137,7 +283,6 @@ int inline adjust_num_threads(int m) { template inline void parallel_2d(int m, int n, const func_t& f) { - // make sure we have even num_threads int nth = adjust_num_threads(m); @@ -165,31 +310,59 @@ inline void parallel_2d(int m, int n, const func_t& f) { #if defined(_OPENMP) #pragma omp parallel num_threads(nth) -{ - int ith = omp_get_thread_num(); - int ith_m = ith / nth_n; - int ith_n = ith % nth_n; + { + int ith = omp_get_thread_num(); + int ith_m = ith / nth_n; + int ith_n = ith % nth_n; - int thread_block_m = div_up(m, nth_m); - int thread_block_n = div_up(n, nth_n); + int thread_block_m = div_up(m, nth_m); + int thread_block_n = div_up(n, nth_n); - int begin_m = ith_m * thread_block_m; - int end_m = std::min(m, begin_m + thread_block_m); - int begin_n = ith_n * thread_block_n; - int end_n = std::min(n, begin_n + thread_block_n); + int begin_m = ith_m * thread_block_m; + int end_m = std::min(m, begin_m + thread_block_m); + int begin_n = ith_n * thread_block_n; + int end_n = std::min(n, begin_n + thread_block_n); - f(begin_m, end_m, begin_n, end_n); -} + f(begin_m, end_m, begin_n, end_n); + } #else f(0, m, 0, n); #endif } +// limit max cache blocks +// when we need to do pre-unpack for weights, e.g. fp8 +#define MAX_CACHE_BLOCK_SIZE 4 + template -int get_cache_blocks(int BLOCK_SIZE, int K) { +inline int get_cache_blocks(int chunk_size) { // L2 2MB and ratio of 50% const int L2_size = 2048 * 1024 >> 1; - return std::max(1, int(L2_size / (BLOCK_SIZE * K * sizeof(T)))); + return std::max(1, int(L2_size / (chunk_size * sizeof(T)))); +} + +template <> +inline int get_cache_blocks(int chunk_size) { + // fp8 uses bf16 as accumulate type + int cache_block_size = get_cache_blocks(chunk_size); + return std::min(MAX_CACHE_BLOCK_SIZE, cache_block_size); +} + +// 2d sequential loop in range : [mb0, mb1), [nb0, nb1) +template +inline void loop_2d(int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1, int64_t chunk_size, const func_t& f) { + // get number of blocks for L2 in most inner loop + int64_t cache_blocks_nb = get_cache_blocks(chunk_size); + + // loop order: [NB / cache_blocks_nb, MB, cache_blocks_nb] + // TODO: implement reverse order of [MB / cache_blocks_mb, NB, cache_blocks_mb] + for (int64_t nbb = nb0; nbb < nb1; nbb += cache_blocks_nb) { + for (int64_t mb = mb0; mb < mb1; ++mb) { + for (int64_t nb = nbb; nb < std::min(nbb + cache_blocks_nb, nb1); ++nb) { + f(mb, nb, nb - nbb); + } + } + } } // data indexing for dimension collapse @@ -243,4 +416,10 @@ struct Unroll<1> { } }; -} // anonymous namespace +// conditional data ptr for optional tensor +template +inline T* conditional_data_ptr(const std::optional& opt) { + return opt.has_value() ? opt.value().data_ptr() : nullptr; +} + +} // anonymous namespace diff --git a/csrc/cpu/sgl-kernels/conv.cpp b/csrc/cpu/sgl-kernels/conv.cpp new file mode 100644 index 0000000000..10ed1f85fb --- /dev/null +++ b/csrc/cpu/sgl-kernels/conv.cpp @@ -0,0 +1,709 @@ +// Adapted from +// https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu + +// clang-format off + +#include "common.h" +#include "gemm.h" +#include "vec.h" + +namespace { + +template +inline void copy_stub(scalar_t* __restrict__ y, const scalar_t* __restrict__ x, int64_t size) { + using Vec = at::vec::Vectorized; + const bool is_padding = (x == nullptr); + for (int64_t d = 0; d < size; d += Vec::size()) { + Vec data_vec = is_padding ? Vec(0.f) : Vec::loadu(x + d); + data_vec.store(y + d); + } +} + +// no remainder +template +void inline update_conv_state( + scalar_t* __restrict__ conv_states, + const scalar_t* __restrict__ input, + int64_t width, + int64_t dim, + int64_t seqlen, + bool has_initial_states) { + // width for `conv_states` + int64_t width1 = width - 1; + int64_t w = 0; + for (; w < width1 - seqlen; ++w) { + scalar_t* y = conv_states + w * dim; + const scalar_t* x = has_initial_states ? conv_states + (w + seqlen) * dim : nullptr; + copy_stub(y, x, dim); + } + for (; w < width1; ++w) { + scalar_t* y = conv_states + w * dim; + const scalar_t* x = input + (w + seqlen - width1) * dim; + copy_stub(y, x, dim); + } +} + +// A : [M, BLOCK_N] +// B : [BLOCK_N, K], prepacked as [K/2, BLOCK_N, 2] +// C : [M, BLOCK_N] +// bias : [BLOCK_N] +// +// lda : leading dimension of `input` and `out` +// +template +struct tinygemm_kernel { + static inline void apply( + const scalar_t* __restrict__ A, + const scalar_t* __restrict__ B, + scalar_t* __restrict__ C, + const scalar_t* __restrict__ bias, + const scalar_t* __restrict__ conv_states, + bool has_initial_state, + int64_t M, + int64_t lda, + bool is_first_token) { + TORCH_CHECK(false, "tinygemm_kernel_nn: scalar path not implemented!"); + } +}; + +#if defined(CPU_CAPABILITY_AVX512) +template +struct tinygemm_kernel { + static inline void apply( + const at::BFloat16* __restrict__ A, + const at::BFloat16* __restrict__ B, + at::BFloat16* __restrict__ C, + const at::BFloat16* __restrict__ bias, + const at::BFloat16* __restrict__ conv_states, + bool has_initial_state, + int64_t M, + int64_t lda, + bool is_first_token) { + assert(K == 4); + constexpr int ROWS = K; + constexpr int COLS = BLOCK_N / block_size_n(); + + // leading dimension size for b for next block [K/2, 32, 2] + constexpr int ldb = block_size_n() * K; + + __m512bh va[ROWS * COLS]; + __m512bh vb[ROWS * COLS]; + __m512 vc[COLS * 2]; + + // k: {-3, -2, -1} -> {0, 1, 2} + auto set_conv_states = [&](int k, int col) -> __m512i { + return has_initial_state ? _mm512_loadu_si512(conv_states + (k + K - 1) * lda + col * 32) + : _mm512_setzero_si512(); + }; + +#define MM512_LOAD_A(idx) \ + ((idx) < 0 && is_first_token) ? (__m512bh)(set_conv_states((idx), col)) \ + : (__m512bh)(_mm512_loadu_si512(A + (idx) * lda + col * 32)) + +#define MM512_PACK_A(ap, bp, a, b) \ + do { \ + __m512i r0 = (__m512i)(a); \ + __m512i r1 = (__m512i)(b); \ + __m512i d0 = _mm512_unpacklo_epi16(r0, r1); \ + __m512i d1 = _mm512_unpackhi_epi16(r0, r1); \ + r0 = _mm512_shuffle_i32x4(d0, d1, 0x88); \ + r1 = _mm512_shuffle_i32x4(d0, d1, 0xdd); \ + (ap) = (__m512bh)_mm512_shuffle_i32x4(r0, r1, 0x88); \ + (bp) = (__m512bh)_mm512_shuffle_i32x4(r0, r1, 0xdd); \ + } while (0) + + // step 0 : preload a at time step [-3][-2][-1] + auto preloada = [&](auto i) { + constexpr int col = i; + int64_t m = 0; + va[1 * COLS + col] = MM512_LOAD_A(m - 3); + va[2 * COLS + col] = MM512_LOAD_A(m - 2); + va[3 * COLS + col] = MM512_LOAD_A(m - 1); + }; + Unroll{}(preloada); + + auto loada = [&](auto i, int64_t m) { + constexpr int col = i; + // update previous time step + va[0 * COLS + col] = va[1 * COLS + col]; + va[1 * COLS + col] = va[2 * COLS + col]; + va[2 * COLS + col] = va[3 * COLS + col]; + // load current time step + va[3 * COLS + col] = MM512_LOAD_A(m); + }; + + // step 1 : load weight for just once + auto loadb = [&](auto i) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + vb[row * COLS + col] = (__m512bh)(_mm512_loadu_si512(B + col * ldb + row * 32)); + }; + Unroll{}(loadb); + + // [NB] accumulates 4x32 bfloat16 blocks + // + // +------------+------------+ + // | col0 | col1 | + // +------------+------------+ + // | va0 va1 | va0 va1 | + // | va2 va3 | va2 va3 | + // +------------+------------+ + // | vc0 vc1 | vc0 vc1 | + // +------------+------------+ + // + // * va and vb shares the same memory layout + // * block_n 32 with 4 rows equals to 4 registers + // * 37 uops with avx512bf16 v.s. 57 uops with avx512f + // + auto compute = [&](auto i) { + constexpr int col = i; + + // init accumulators + if constexpr (has_bias) { + __m512i b16 = _mm512_loadu_si512(reinterpret_cast(bias + col * 32)); + vc[col * 2 + 0] = CVT_BF16_TO_FP32(_mm512_extracti32x8_epi32(b16, 0)); + vc[col * 2 + 1] = CVT_BF16_TO_FP32(_mm512_extracti32x8_epi32(b16, 1)); + } else { + vc[col * 2 + 0] = _mm512_set1_ps(0.f); + vc[col * 2 + 1] = _mm512_set1_ps(0.f); + } + + // convert to vnni2 format + __m512bh va0, va1, va2, va3; + MM512_PACK_A(va0, va1, va[0 * COLS + col], va[1 * COLS + col]); + MM512_PACK_A(va2, va3, va[2 * COLS + col], va[3 * COLS + col]); + + // accumulate + vc[col * 2 + 0] = _mm512_dpbf16_ps(vc[col * 2 + 0], va0, vb[0 * COLS + col]); + vc[col * 2 + 0] = _mm512_dpbf16_ps(vc[col * 2 + 0], va2, vb[2 * COLS + col]); + vc[col * 2 + 1] = _mm512_dpbf16_ps(vc[col * 2 + 1], va1, vb[1 * COLS + col]); + vc[col * 2 + 1] = _mm512_dpbf16_ps(vc[col * 2 + 1], va3, vb[3 * COLS + col]); + }; + + using fVec = at::vec::Vectorized; + using bVec = at::vec::Vectorized; + const fVec one = fVec(1.f); + auto storec = [&](auto i, int64_t m) { + constexpr int col = i; + fVec x0 = fVec(vc[col * 2 + 0]); + fVec x1 = fVec(vc[col * 2 + 1]); + if constexpr (has_silu) { + x0 = x0 / (one + x0.neg().exp_u20()); + x1 = x1 / (one + x1.neg().exp_u20()); + } + bVec out_vec = convert_from_float_ext(x0, x1); + out_vec.store(C + m * lda + col * 32); + }; + + for (int64_t m = 0; m < M; ++m) { + // step 3.a : load a at current time step + Unroll{}(loada, m); + // step 3.b : accumulate for window size (4) + Unroll{}(compute); + // step 3.c : store c at current time step + Unroll{}(storec, m); + } + } +}; +#endif + +#define LAUNCH_TINYGEMM_KERNEL(K, NB_SIZE) \ + tinygemm_kernel::apply( \ + input + bs * seqlen * dim + mb_start * dim + nb_start, \ + weight + nb_start * width, \ + out + bs * seqlen * dim + mb_start * dim + nb_start, \ + has_bias ? bias + nb_start : nullptr, \ + has_conv_states ? conv_states + conv_state_index * (K - 1) * dim + nb_start : nullptr, \ + has_initial_states_value, \ + mb_size, \ + dim, \ + mb_start == 0); + +template +void causal_conv1d_fwd_kernel_impl( + scalar_t* __restrict__ out, + const scalar_t* __restrict__ input, + const scalar_t* __restrict__ weight, + const scalar_t* __restrict__ bias, + scalar_t* __restrict__ conv_states, + const int32_t* __restrict__ conv_indices, + const bool* __restrict__ has_initial_state, + bool silu_activation, + int64_t batch, + int64_t dim, + int64_t seqlen, + int64_t width, + int64_t num_seq_blocks) { + // handle 32 x 64 per block + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n() * 2; + const int64_t NB = div_up(dim, BLOCK_N); + + const int64_t num_blocks_per_seq = div_up(seqlen, BLOCK_M); + const bool has_conv_states = conv_states != nullptr; + const bool has_conv_indices = conv_indices != nullptr; + + // parallel on [batch, seq, NB] + AT_DISPATCH_BOOL2(bias != nullptr, has_bias, silu_activation, has_silu, [&] { + at::parallel_for(0, num_seq_blocks * NB, 0, [&](int64_t begin, int64_t end) { + int64_t mb{0}, nb{0}; + data_index_init(begin, mb, num_seq_blocks, nb, NB); + + for (int64_t i = begin; i < end; ++i) { + int64_t bs = mb / num_blocks_per_seq; + + int64_t mb_start = (mb % num_blocks_per_seq) * BLOCK_M; + int64_t mb_size = std::min(seqlen - mb_start, BLOCK_M); + int64_t nb_start = nb * BLOCK_N; + int64_t nb_size = std::min(dim - nb_start, BLOCK_N); + + const bool has_initial_states_value = has_conv_states ? has_initial_state[bs] : false; + int32_t conv_state_index = has_conv_indices ? conv_indices[bs] : bs; + + switch (width << 4 | nb_size >> 4) { + case 0x42: + LAUNCH_TINYGEMM_KERNEL(4, 32); + break; + case 0x44: + LAUNCH_TINYGEMM_KERNEL(4, 64); + break; + default: + TORCH_CHECK(false, "Unexpected block size, ", width, " x ", nb_size); + } + + // move to the next index + data_index_step(mb, num_seq_blocks, nb, NB); + } + }); + }); + + // update conv_states if necessary + if (has_conv_states) { + at::parallel_for(0, batch, 0, [&](int64_t begin, int64_t end) { + for (int64_t bs = begin; bs < end; ++bs) { + update_conv_state( + conv_states + bs * (width - 1) * dim, input + bs * seqlen * dim, width, dim, seqlen, has_initial_state[bs]); + } + }); + } +} + +#define LAUNCH_TINYGEMM_VARLEN_KERNEL(K, NB_SIZE) \ + tinygemm_kernel::apply( \ + input + batch_offset * dim + mb_start * dim + nb_start, \ + weight + nb_start * width, \ + out + batch_offset * dim + mb_start * dim + nb_start, \ + has_bias ? bias + nb_start : nullptr, \ + nullptr, \ + false, \ + mb_size, \ + dim, \ + mb_start == 0); + +// TODO: add `has_initial_state` support for varlen kernel +template +void causal_conv1d_fwd_varlen_kernel_impl( + scalar_t* __restrict__ out, + const scalar_t* __restrict__ input, + const scalar_t* __restrict__ weight, + const scalar_t* __restrict__ bias, + scalar_t* __restrict__ conv_states, + const int32_t* __restrict__ query_start_loc, + const int32_t* __restrict__ conv_indices, + const bool* __restrict__ has_initial_state, + const int32_t* __restrict__ block_indices, + bool silu_activation, + int64_t batch, + int64_t dim, + int64_t width, + int64_t num_seq_blocks) { + // handle 32 x 64 per block + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n() * 2; + const int64_t NB = div_up(dim, BLOCK_N); + + const bool has_conv_states = conv_states != nullptr; + const bool has_conv_indices = conv_indices != nullptr; + + // parallel on [batch, seq, NB] + AT_DISPATCH_BOOL2(bias != nullptr, has_bias, silu_activation, has_silu, [&] { + at::parallel_for(0, num_seq_blocks * NB, 0, [&](int64_t begin, int64_t end) { + int64_t mb{0}, nb{0}; + data_index_init(begin, mb, num_seq_blocks, nb, NB); + + for (int64_t i = begin; i < end; ++i) { + int32_t bs = block_indices[mb * 2 + 0]; + int32_t batch_offset = query_start_loc[bs]; + int32_t seqlen = query_start_loc[bs + 1] - query_start_loc[bs]; + + int64_t mb_start = block_indices[mb * 2 + 1] * BLOCK_M; + int64_t mb_size = std::min(seqlen - mb_start, BLOCK_M); + int64_t nb_start = nb * BLOCK_N; + int64_t nb_size = std::min(dim - nb_start, BLOCK_N); + + switch (width << 4 | nb_size >> 4) { + case 0x42: + LAUNCH_TINYGEMM_VARLEN_KERNEL(4, 32); + break; + case 0x44: + LAUNCH_TINYGEMM_VARLEN_KERNEL(4, 64); + break; + default: + TORCH_CHECK(false, "Unexpected block size, ", width, " x ", nb_size); + } + + // move to the next index + data_index_step(mb, num_seq_blocks, nb, NB); + } + }); + }); + + // update conv_states if necessary + if (has_conv_states) { + at::parallel_for(0, batch, 0, [&](int64_t begin, int64_t end) { + for (int64_t bs = begin; bs < end; ++bs) { + int32_t conv_state_index = has_conv_indices ? conv_indices[bs] : bs; + int32_t seqlen = query_start_loc[bs + 1] - query_start_loc[bs]; + int32_t batch_offset = query_start_loc[bs]; + update_conv_state( + conv_states + conv_state_index * (width - 1) * dim, + input + batch_offset * dim, + width, + dim, + seqlen, + /* has_initial_state */ false); + } + }); + } +} + +template +void causal_conv1d_update_kernel_impl( + scalar_t* __restrict__ out, + const scalar_t* __restrict__ input, + scalar_t* __restrict__ conv_states, + const scalar_t* __restrict__ weight, + const scalar_t* __restrict__ bias, + const int32_t* __restrict__ conv_indices, + bool silu_activation, + int64_t batch, + int64_t dim, + int64_t seqlen, + int64_t width) { + // handle 32 x 64 per block + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n() * 2; + const int64_t NB = div_up(dim, BLOCK_N); + + const bool has_conv_states = conv_states != nullptr; + const bool has_conv_indices = conv_indices != nullptr; + + // parallel on [batch, NB] + AT_DISPATCH_BOOL2(bias != nullptr, has_bias, silu_activation, has_silu, [&] { + at::parallel_for(0, batch * NB, 0, [&](int64_t begin, int64_t end) { + int64_t bs{0}, nb{0}; + data_index_init(begin, bs, batch, nb, NB); + + for (int64_t i = begin; i < end; ++i) { + int64_t mb_start = 0; + int64_t mb_size = 1; + int64_t nb_start = nb * BLOCK_N; + int64_t nb_size = std::min(dim - nb_start, BLOCK_N); + + const bool has_initial_states_value = true; + int32_t conv_state_index = has_conv_indices ? conv_indices[bs] : bs; + + switch (width << 4 | nb_size >> 4) { + case 0x42: + LAUNCH_TINYGEMM_KERNEL(4, 32); + break; + case 0x44: + LAUNCH_TINYGEMM_KERNEL(4, 64); + break; + default: + TORCH_CHECK(false, "Unexpected block size, ", width, " x ", nb_size); + } + + // move to the next index + data_index_step(bs, batch, nb, NB); + } + }); + }); + +#define CONV_STATE_INDEXR(w) conv_states + conv_state_index*(width - 1) * dim + (w) * dim + + // update conv_states + at::parallel_for(0, batch, 0, [&](int64_t begin, int64_t end) { + for (int64_t bs = begin; bs < end; ++bs) { + // update old states, range [1, width - 1) + int32_t conv_state_index = has_conv_indices ? conv_indices[bs] : bs; + for (int64_t w = 1; w < width - 1; ++w) { + std::memcpy(CONV_STATE_INDEXR(w - 1), CONV_STATE_INDEXR(w), dim * sizeof(scalar_t)); + } + // copy new states + std::memcpy(CONV_STATE_INDEXR(width - 2), input + bs * dim, dim * sizeof(scalar_t)); + } + }); +} + +} // anonymous namespace + +// from [dim, width] or [N, K] +// to [N/BLOCK_N, K/2, BLOCK_N, 2] +at::Tensor causal_conv1d_weight_pack(const at::Tensor& weight) { + CHECK_INPUT(weight); + + int64_t dim = weight.size(0); + int64_t width = weight.size(1); + constexpr int64_t BLOCK_N = block_size_n(); + TORCH_CHECK(width == 4, "causal_conv1d_weight_pack: support only width of 4"); + TORCH_CHECK(dim % BLOCK_N == 0, "causal_conv1d_weight_pack: invalid dim size ", dim); + + const int64_t N = dim, K2 = width >> 1; + const int64_t NB = div_up(N, BLOCK_N); + + auto packed_weight = at::empty_like(weight); + AT_DISPATCH_REDUCED_FLOATING_TYPES(weight.scalar_type(), "causal_conv1d_fwd_kernel_impl", [&] { + // cast to float32 as vnni size is 2 + const float* w_data = reinterpret_cast(weight.data_ptr()); + float* packed_data = reinterpret_cast(packed_weight.data_ptr()); + + at::parallel_for(0, NB * K2 * BLOCK_N, 0, [&](int64_t begin, int64_t end) { + int64_t nb{0}, k2{0}, n{0}; + data_index_init(begin, nb, NB, k2, K2, n, BLOCK_N); + + // TODO: optimize this if we need to online prepacking. + for (int64_t i = begin; i < end; ++i) { + packed_data[i] = w_data[nb * BLOCK_N * K2 + n * K2 + k2]; + + // move to the next index + data_index_step(nb, NB, k2, K2, n, BLOCK_N); + } + }); + }); + return packed_weight; +} + +#define CHECK_OPTIONAL_SHAPE_DTYPE(OPT, SIZE, DTYPE) \ + if (OPT.has_value()) { \ + const auto tensor = OPT.value(); \ + CHECK_CONTIGUOUS(tensor); \ + CHECK_EQ(tensor.size(0), SIZE); \ + CHECK_EQ(tensor.scalar_type(), DTYPE); \ + } + +template +int64_t get_block_count(const std::optional& offsets, int64_t batch, int64_t seqlen) { + if (offsets.has_value()) { + const int32_t* offsets_data = offsets.value().data_ptr(); + int32_t num_seq_blocks = 0; + for (int64_t row = 0; row < batch; ++row) { + num_seq_blocks += div_up(offsets_data[row + 1] - offsets_data[row], BLOCK_M); + } + return num_seq_blocks; + } + return batch * div_up(seqlen, int64_t(BLOCK_M)); +} + +template +at::Tensor get_block_indices(const std::optional& offsets, int64_t num_seq_blocks) { + if (!offsets.has_value()) { + return at::Tensor(); + } + + const at::Tensor& offsets_ = offsets.value(); + at::Tensor indices = at::empty({num_seq_blocks, 2}, offsets_.options()); + + int64_t batch = offsets_.size(0) - 1; + + const int32_t* offsets_data = offsets_.data_ptr(); + int32_t* indices_data = indices.data_ptr(); + + int64_t idx = 0; + for (int32_t row = 0; row < batch; ++row) { + int32_t blocks = div_up(offsets_data[row + 1] - offsets_data[row], BLOCK_M); + + for (int32_t col = 0; col < blocks; ++col) { + indices_data[idx * 2 + 0] = row; + indices_data[idx * 2 + 1] = col; + idx++; + } + } + return indices; +} + +// API aligned with GPUs +// +// x: (batch, dim, seqlen) or (dim, cu_seq_len) for varlen +// weight: (dim, width) +// bias: (dim,) +// query_start_loc: (batch + 1) int32 +// cache_indices: (batch) int32 +// has_initial_state: (batch) bool +// conv_states: (..., dim, width - 1) itype +// activation: either None or "silu" or "swish" +// pad_slot_id: int +// +at::Tensor causal_conv1d_fwd_cpu( + const at::Tensor& x, + const at::Tensor& weight, + const std::optional& bias, + const std::optional& conv_states, + const std::optional& query_start_loc, + const std::optional& conv_state_indices, + const std::optional& has_initial_state, + bool silu_activation, + int64_t pad_slot_id, + bool is_vnni) { + CHECK_CONTIGUOUS(weight); + auto packed_w = is_vnni ? weight : causal_conv1d_weight_pack(weight); + + const bool is_var_seqlen = query_start_loc.has_value(); + const int64_t input_ndim = is_var_seqlen ? 2 : 3; + TORCH_CHECK(x.dim() == input_ndim, "causal_conv1d_fwd_cpu: expect x to be ", input_ndim, "D tensor."); + TORCH_CHECK(x.stride(-2) == 1 && x.stride(-1) == x.size(-2), "causal_conv1d_fwd_cpu: expect x to be transposed."); + + const int64_t batch = is_var_seqlen ? query_start_loc.value().size(0) - 1 : x.size(0); + const int64_t dim = x.size(-2); + const int64_t seqlen = x.size(-1); + const int64_t width = weight.size(-1); + + const auto scalar_type = x.scalar_type(); + CHECK_EQ(weight.scalar_type(), scalar_type); + CHECK_OPTIONAL_SHAPE_DTYPE(bias, dim, scalar_type); + CHECK_OPTIONAL_SHAPE_DTYPE(query_start_loc, batch + 1, at::kInt); + CHECK_OPTIONAL_SHAPE_DTYPE(conv_state_indices, batch, at::kInt); + CHECK_OPTIONAL_SHAPE_DTYPE(has_initial_state, batch, at::kBool); + + if (conv_states.has_value()) { + auto& conv_states_val = conv_states.value(); + int64_t padded_batch = conv_states_val.size(0); + CHECK_EQ(conv_states_val.scalar_type(), scalar_type); + CHECK_GE(padded_batch, batch); + CHECK_EQ(conv_states_val.size(1), dim); + CHECK_EQ(conv_states_val.size(2), width - 1); + + // adjust `conv_states` to be contiguous on `dim` + // should happen only once + if (conv_states_val.stride(-2) != 1) { + auto conv_states_copy = conv_states_val.clone(); + conv_states_val.as_strided_({padded_batch, dim, width - 1}, {(width - 1) * dim, 1, dim}); + conv_states_val.copy_(conv_states_copy); + } + } + + // block size for sequence blocks, 32 + constexpr int64_t BLOCK_M = block_size_m(); + + // total number of sequence blocks + int64_t num_seq_blocks = get_block_count(query_start_loc, batch, seqlen); + + at::Tensor out = at::empty_like(x); + AT_DISPATCH_REDUCED_FLOATING_TYPES(scalar_type, "causal_conv1d_fwd_kernel_impl", [&] { + if (is_var_seqlen) { + // record seq blocks in Coordinate format, aka [num_seq_blocks, 2] + at::Tensor block_indices = get_block_indices(query_start_loc, num_seq_blocks); + + causal_conv1d_fwd_varlen_kernel_impl( + out.data_ptr(), + x.data_ptr(), + packed_w.data_ptr(), + conditional_data_ptr(bias), + conditional_data_ptr(conv_states), + conditional_data_ptr(query_start_loc), + conditional_data_ptr(conv_state_indices), + conditional_data_ptr(has_initial_state), + block_indices.data_ptr(), + silu_activation, + batch, + dim, + width, + num_seq_blocks); + } else { + causal_conv1d_fwd_kernel_impl( + out.data_ptr(), + x.data_ptr(), + packed_w.data_ptr(), + conditional_data_ptr(bias), + conditional_data_ptr(conv_states), + conditional_data_ptr(conv_state_indices), + conditional_data_ptr(has_initial_state), + silu_activation, + batch, + dim, + seqlen, + width, + num_seq_blocks); + } + }); + return out; +} + +// API aligned with GPUs +// +// x: (batch, dim) or (batch, dim, seqlen) +// conv_state: (..., dim, state_len), where state_len >= width - 1 +// weight: (dim, width) +// bias: (dim,) +// cache_seqlens: (batch,), dtype int32. +// conv_state_indices: (batch,), dtype int32 +// pad_slot_id: int +// out: (batch, dim) or (batch, dim, seqlen) +// +at::Tensor causal_conv1d_update_cpu( + const at::Tensor& x, + const at::Tensor& conv_states, + const at::Tensor& weight, + const std::optional& bias, + bool silu_activation, + const std::optional& cache_seqlens, + const std::optional& conv_state_indices, + int64_t pad_slot_id, + bool is_vnni) { + CHECK_CONTIGUOUS(x); + CHECK_CONTIGUOUS(weight); + auto packed_w = is_vnni ? weight : causal_conv1d_weight_pack(weight); + + // TODO: add multi-token prediction support + TORCH_CHECK(x.dim() == 2, "causal_conv1d_update_cpu: expect x to be 2D tensor."); + TORCH_CHECK(!cache_seqlens.has_value(), "causal_conv1d_update_cpu: don't support cache_seqlens."); + + int64_t batch = x.size(0); + int64_t dim = x.size(1); + int64_t seqlen = 1; + int64_t width = weight.size(-1); + + const auto scalar_type = x.scalar_type(); + CHECK_EQ(weight.scalar_type(), scalar_type); + CHECK_OPTIONAL_SHAPE_DTYPE(bias, dim, scalar_type); + CHECK_OPTIONAL_SHAPE_DTYPE(conv_state_indices, batch, at::kInt); + + CHECK_EQ(conv_states.scalar_type(), scalar_type); + CHECK_EQ(conv_states.size(1), dim); + CHECK_EQ(conv_states.size(2), width - 1); + + // adjust `conv_states` to be contiguous on `dim` + if (conv_states.stride(-2) != 1) { + int64_t num_cache_lines = conv_states.size(0); + auto conv_states_copy = conv_states.clone(); + conv_states.as_strided_({num_cache_lines, dim, width - 1}, {(width - 1) * dim, 1, dim}); + conv_states.copy_(conv_states_copy); + } + + at::Tensor out = at::empty_like(x); + AT_DISPATCH_REDUCED_FLOATING_TYPES(scalar_type, "causal_conv1d_update_kernel_impl", [&] { + causal_conv1d_update_kernel_impl( + out.data_ptr(), + x.data_ptr(), + conv_states.data_ptr(), + packed_w.data_ptr(), + conditional_data_ptr(bias), + conditional_data_ptr(conv_state_indices), + silu_activation, + batch, + dim, + seqlen, + width); + }); + return out; +} diff --git a/csrc/cpu/sgl-kernels/fla.cpp b/csrc/cpu/sgl-kernels/fla.cpp new file mode 100644 index 0000000000..b78e9863bf --- /dev/null +++ b/csrc/cpu/sgl-kernels/fla.cpp @@ -0,0 +1,1394 @@ +// Adapted from +// https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu + +// clang-format off + +#include "common.h" +#include "gemm.h" +#include "vec.h" +#include "vec_pack.h" + +namespace { +// For this cpu kernel, we have some innovations aside from the existing gpu kernels: +// 1) Use less parallel loops, i.e. 4 including l2_norm. +// 2) Fuse part of l2_norm with the rest of the computation. + +#define THREAD_BUFFER_ALLOC(dst, base_ptr, offset, type, size) \ + type* dst = reinterpret_cast((base_ptr) + (offset)); \ + offset += (size); + +template +inline void fill_stub(scalar_t* __restrict__ out, float val, int size) { + using Vec = at::vec::Vectorized; + constexpr int kVecSize = Vec::size(); + const Vec data_vec = Vec(static_cast(val)); + int d = 0; +#pragma GCC unroll 4 + for (; d <= size - kVecSize; d += kVecSize) { + data_vec.store(out + d); + } + if (size - d > 0) { + data_vec.store(out + d, size - d); + } +} + +template +void chunk_gated_delta_rule_kernel_impl( + scalar_t* __restrict__ out, // [B, T, HV, EV] + float* __restrict__ final_state_data, // [N, HV, EK, EV] + const scalar_t* __restrict__ q_orig, // [B, T, HK, EK] + const scalar_t* __restrict__ k_orig, // [B, T, HK, EK] + const scalar_t* __restrict__ v_orig, // [B, T, HV, EV] + const float* __restrict__ g_orig, // [B, T, HV] FP32 + const scalar_t* __restrict__ b_orig, // [B, T, HV] + const int32_t* __restrict__ cu_seqlens_ptr, // [N + 1] INT32 + float* __restrict__ buff, + scalar_t* __restrict__ reduced_buff, + scalar_t* __restrict__ thread_buff, + const int32_t* __restrict__ chunk_offsets_ptr, + const int32_t* __restrict__ chunk_indices_ptr, + bool use_qk_l2norm_in_kernel, + const int64_t& batch_size, + const int64_t& global_seq_len, + const int64_t& qk_num_head, + const int64_t& v_num_head, + const int64_t& qk_head_size, + const int64_t& v_head_size, + const int64_t& qStrideH, + const int64_t& qStrideT, + const int64_t& kStrideH, + const int64_t& kStrideT, + const int64_t& vStrideH, + const int64_t& vStrideT, + const int64_t& oStrideH, + const int64_t& oStrideT, + const int64_t& global_total_seq_length, + const int64_t& global_num_chunk, + const int64_t& buff_size_16bit_per_thread, + double eps = 1e-5) { + int64_t gStrideH = 1; + int64_t gStrideT = v_num_head; + int64_t bStrideH = 1; + int64_t bStrideT = v_num_head; + int64_t final_state_StrideN = v_num_head * qk_head_size * v_head_size; + int64_t final_state_StrideH = qk_head_size * v_head_size; + int64_t final_state_StrideE = v_head_size; + int64_t head_group = v_num_head / qk_num_head; + float scale = 1.0 / std::sqrt(qk_head_size); + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int64_t VecSize = bVec::size(); + constexpr int64_t fVecSize = fVec::size(); + + // Data pointers + float* g_pad = buff; + float* core_attn_out = g_pad + v_num_head * global_total_seq_length; + float* decay_mask = core_attn_out + batch_size * v_num_head * global_total_seq_length * v_head_size; + float* v_beta_attn = decay_mask + v_num_head * global_total_seq_length * chunk_size; + + scalar_t* q_pad = reduced_buff; + scalar_t* k_pad = q_pad + qk_num_head * global_total_seq_length * qk_head_size; + scalar_t* v_pad = k_pad + qk_num_head * global_total_seq_length * qk_head_size; + scalar_t* k_beta = v_pad + v_num_head * global_total_seq_length * v_head_size; + scalar_t* v_beta = k_beta + v_num_head * global_total_seq_length * qk_head_size; + scalar_t* k_cumdecay_reduced = v_beta + v_num_head * global_total_seq_length * v_head_size; + scalar_t* q_norm_sum = k_cumdecay_reduced + v_num_head * global_total_seq_length * qk_head_size; + scalar_t* k_norm_sum = q_norm_sum + qk_num_head * global_seq_len; + + if (use_qk_l2norm_in_kernel) { + at::parallel_for(0, qk_num_head * global_seq_len, 0, [&](int64_t begin, int64_t end) { + int64_t h_qk = 0, l = 0; + data_index_init(begin, h_qk, qk_num_head, l, global_seq_len); + for (int64_t i = begin; i < end; ++i) { + auto q_norm_sum_ptr = q_norm_sum + h_qk * global_seq_len + l; + auto k_norm_sum_ptr = k_norm_sum + h_qk * global_seq_len + l; + float sum_q = float(0); + float sum_k = float(0); + fVec sum_q_fvec = fVec(float(0)); + fVec sum_k_fvec = fVec(float(0)); + int64_t q_offset = l * qStrideT + h_qk * qStrideH; + int64_t k_offset = l * qStrideT + h_qk * qStrideH; + int64_t d; + for (d = 0; d <= qk_head_size - VecSize; d += VecSize) { + bVec q_bvec = bVec::loadu(q_orig + q_offset + d); + fVec q_fvec0, q_fvec1; + std::tie(q_fvec0, q_fvec1) = at::vec::convert_to_float(q_bvec); + sum_q_fvec += q_fvec0 * q_fvec0; + sum_q_fvec += q_fvec1 * q_fvec1; + bVec k_bvec = bVec::loadu(k_orig + k_offset + d); + fVec k_fvec0, k_fvec1; + std::tie(k_fvec0, k_fvec1) = at::vec::convert_to_float(k_bvec); + sum_k_fvec += k_fvec0 * k_fvec0; + sum_k_fvec += k_fvec1 * k_fvec1; + } + sum_q += vec_reduce_sum(sum_q_fvec); + sum_k += vec_reduce_sum(sum_k_fvec); + q_norm_sum_ptr[0] = static_cast(float(1) / std::sqrt(sum_q + eps)); + k_norm_sum_ptr[0] = static_cast(float(1) / std::sqrt(sum_k + eps)); + data_index_step(h_qk, qk_num_head, l, global_seq_len); + } + }); + } + + // query = query * scale + // k_beta = key * beta.unsqueeze(-1) + // v_beta = value * beta.unsqueeze(-1) + // Padding for q/k/v/beta + at::parallel_for(0, qk_num_head * global_num_chunk, 1, [&](int64_t begin, int64_t end) { + int ompIdx = at::get_thread_num(); + int64_t h_qk = 0, c = 0; + data_index_init(begin, h_qk, qk_num_head, c, global_num_chunk); + for ([[maybe_unused]] auto z : c10::irange(begin, end)) { + int64_t ib = chunk_indices_ptr[c * 2]; // idx_batch + int64_t ic = chunk_indices_ptr[c * 2 + 1]; // idx_chunk + int64_t l_orig = cu_seqlens_ptr[ib] + ic * chunk_size; + int64_t l = c * chunk_size; + bool is_tail = (c + 1 == chunk_offsets_ptr[ib + 1]); + int64_t seq_len = cu_seqlens_ptr[ib + 1] - cu_seqlens_ptr[ib]; + int64_t real_chunk_size = is_tail ? seq_len - ic * chunk_size : chunk_size; + auto q_orig_ptr = q_orig + h_qk * qStrideH + l_orig * qStrideT; + auto k_orig_ptr = k_orig + h_qk * kStrideH + l_orig * kStrideT; + auto v_orig_ptr = v_orig + l_orig * vStrideT; + auto b_orig_ptr = b_orig + l_orig * bStrideT; + auto q_pad_ptr = q_pad + h_qk * global_total_seq_length * qk_head_size + l * qk_head_size; + auto k_pad_ptr = k_pad + h_qk * global_total_seq_length * qk_head_size + l * qk_head_size; + auto v_pad_ptr = v_pad + l * v_head_size; + auto k_beta_ptr = k_beta + l * qk_head_size; + auto v_beta_ptr = v_beta + l * v_head_size; + + for (int64_t j = 0; j < real_chunk_size; j++) { + auto curr_q_orig = q_orig_ptr + j * qStrideT; + auto curr_k_orig = k_orig_ptr + j * kStrideT; + auto curr_q_pad = q_pad_ptr + j * qk_head_size; + auto curr_k_pad = k_pad_ptr + j * qk_head_size; + auto q_scale = + use_qk_l2norm_in_kernel ? *(q_norm_sum + h_qk * global_seq_len + l_orig + j) : static_cast(1); + auto k_scale = + use_qk_l2norm_in_kernel ? *(k_norm_sum + h_qk * global_seq_len + l_orig + j) : static_cast(1); + auto q_scale_vec = bVec(q_scale); + auto k_scale_vec = bVec(k_scale); + int64_t i = 0; + scalar_t scale_reduced = static_cast(scale); + auto vec_scale_reduced = bVec(scale_reduced); + for (; i < fVecSize * (qk_head_size / fVecSize); i += fVecSize) { + auto tmp0 = bVec::loadu(curr_q_orig + i, fVecSize); + auto tmp1 = tmp0 * q_scale_vec * vec_scale_reduced; + tmp1.store(curr_q_pad + i, fVecSize); + auto tmp3 = bVec::loadu(curr_k_orig + i, fVecSize); + auto tmp4 = tmp3 * k_scale_vec; + tmp4.store(curr_k_pad + i, fVecSize); + } + + for (auto hi = 0; hi < head_group; hi++) { + int64_t h = h_qk * head_group + hi; + auto curr_v_orig = v_orig_ptr + h * vStrideH + j * vStrideT; + auto curr_b_orig = b_orig_ptr + h * bStrideH + j * bStrideT; + scalar_t b_orig_val_reduced = *(curr_b_orig); + auto curr_v_pad = v_pad_ptr + h * global_total_seq_length * v_head_size + j * v_head_size; + auto curr_k_beta = k_beta_ptr + h * global_total_seq_length * qk_head_size + j * qk_head_size; + auto curr_v_beta = v_beta_ptr + h * global_total_seq_length * v_head_size + j * v_head_size; + + // query = query * scale + // k_beta = key * beta.unsqueeze(-1) + int64_t i = 0; + auto vec_b_reduced = bVec(b_orig_val_reduced); + for (; i < fVecSize * (qk_head_size / fVecSize); i += fVecSize) { + auto tmp0 = bVec::loadu(curr_k_orig + i, fVecSize); + auto tmp2 = tmp0 * k_scale_vec * vec_b_reduced; + tmp2.store(curr_k_beta + i, fVecSize); + } + // v_beta = value * beta.unsqueeze(-1) + i = 0; + for (; i < VecSize * (v_head_size / VecSize); i += VecSize) { + auto tmp3 = bVec::loadu(curr_v_orig + i); + tmp3.store(curr_v_pad + i); + auto tmp5 = tmp3 * vec_b_reduced; + tmp5.store(curr_v_beta + i); + } + } + } + + for (int64_t j = real_chunk_size; j < chunk_size; j++) { + auto curr_q_pad = q_pad_ptr + j * qk_head_size; + auto curr_k_pad = k_pad_ptr + j * qk_head_size; + int64_t i = 0; + auto vec_zero = bVec(0.0); + for (; i < VecSize * (qk_head_size / VecSize); i += VecSize) { + vec_zero.store(curr_q_pad + i); + vec_zero.store(curr_k_pad + i); + } + for (auto hi = 0; hi < head_group; hi++) { + int64_t h = h_qk * head_group + hi; + auto curr_v_pad = v_pad_ptr + h * global_total_seq_length * v_head_size + j * v_head_size; + auto curr_k_beta = k_beta_ptr + h * global_total_seq_length * qk_head_size + j * qk_head_size; + auto curr_v_beta = v_beta_ptr + h * global_total_seq_length * v_head_size + j * v_head_size; + int64_t i = 0; + for (; i < VecSize * (qk_head_size / VecSize); i += VecSize) { + vec_zero.store(curr_k_beta + i); + } + i = 0; + for (; i < VecSize * (v_head_size / VecSize); i += VecSize) { + vec_zero.store(curr_v_pad + i); + vec_zero.store(curr_v_beta + i); + } + } + } + // Move to the next query + data_index_step(h_qk, qk_num_head, c, global_num_chunk); + } + }); + + at::parallel_for(0, v_num_head * global_num_chunk, 1, [&](int64_t begin, int64_t end) { + int64_t h = 0, c = 0; + data_index_init(begin, h, v_num_head, c, global_num_chunk); + int ompIdx = at::get_thread_num(); + int64_t offset = 0; + scalar_t* thread_buff_ptr = thread_buff + ompIdx * buff_size_16bit_per_thread; + THREAD_BUFFER_ALLOC(k_transpose, thread_buff_ptr, offset, scalar_t, qk_head_size * chunk_size); + THREAD_BUFFER_ALLOC(v_pack, thread_buff_ptr, offset, scalar_t, chunk_size * v_head_size); + THREAD_BUFFER_ALLOC(k_beta_g, thread_buff_ptr, offset, scalar_t, chunk_size * qk_head_size); + THREAD_BUFFER_ALLOC(k_beta_g_pack, thread_buff_ptr, offset, scalar_t, chunk_size * qk_head_size); + THREAD_BUFFER_ALLOC(curr_attn, thread_buff_ptr, offset, float, chunk_size* chunk_size * 2); + THREAD_BUFFER_ALLOC(curr_attn_reduced, thread_buff_ptr, offset, scalar_t, chunk_size * chunk_size); + THREAD_BUFFER_ALLOC(k_cumdecay, thread_buff_ptr, offset, float, chunk_size* qk_head_size * 2); + THREAD_BUFFER_ALLOC(row, thread_buff_ptr, offset, float, chunk_size * 2); + THREAD_BUFFER_ALLOC(updated, thread_buff_ptr, offset, float, chunk_size * 2); + for ([[maybe_unused]] auto z : c10::irange(begin, end)) { + int64_t ib = chunk_indices_ptr[c * 2]; // idx_batch + int64_t ic = chunk_indices_ptr[c * 2 + 1]; // idx_chunk + int64_t l_orig = cu_seqlens_ptr[ib] + ic * chunk_size; + int64_t seq_len = cu_seqlens_ptr[ib + 1] - cu_seqlens_ptr[ib]; + int64_t h_qk = h / head_group; + auto curr_g_orig = g_orig + h * gStrideH + l_orig * gStrideT; + auto curr_g_pad = g_pad + h * global_total_seq_length + c * chunk_size; + auto curr_decay_mask = decay_mask + h * global_total_seq_length * chunk_size + c * chunk_size * chunk_size; + auto curr_k_pad = k_pad + h_qk * global_total_seq_length * qk_head_size + c * chunk_size * qk_head_size; + auto curr_k_beta = k_beta + h * global_total_seq_length * qk_head_size + c * chunk_size * qk_head_size; + auto curr_k_cumdecay_reduced = + k_cumdecay_reduced + h * global_total_seq_length * qk_head_size + c * chunk_size * qk_head_size; + auto curr_v_beta = v_beta + h * global_total_seq_length * v_head_size + c * chunk_size * v_head_size; + auto curr_value = v_beta_attn + h * global_total_seq_length * v_head_size + c * chunk_size * v_head_size; + + float acc_val = 0; + for (int64_t i = 0; i < chunk_size; i++) { + // Padding for g + // g = g.cumsum(dim=-1) + // g: [B, HV, num_chunk, chunk_size] + if (ic * chunk_size + i < seq_len) { + acc_val += curr_g_orig[i * gStrideT]; + } + curr_g_pad[i] = acc_val; + // decay_mask = ((g.unsqueeze(-1) - g.unsqueeze(-2)).tril().exp().float()).tril() + // decay_mask: [B, HV, num_chunk, chunk_size, chunk_size] + float curr_g_pad_i = static_cast(curr_g_pad[i]); + auto vec_curr_g_pad_i = fVec(curr_g_pad_i); + int64_t j = 0; + int64_t len = i + 1; + for (; j < fVecSize * (len / fVecSize); j += fVecSize) { + auto tmp0 = fVec::loadu(curr_g_pad + j); + auto tmp1 = vec_curr_g_pad_i - tmp0; + auto tmp2 = tmp1.exp_u20(); + tmp2.store(curr_decay_mask + i * chunk_size + j); + } + if (j < len) { + auto tmp0 = fVec::loadu(curr_g_pad + j, len - j); + auto tmp1 = vec_curr_g_pad_i - tmp0; + auto tmp2 = tmp1.exp_u20(); + tmp2.store(curr_decay_mask + i * chunk_size + j, len - j); + } + } + + // attn = k_beta @ key.transpose(-1, -2) + // attn: [B, HV, num_chunk, chunk_size, chunk_size] + // transpose and pack for key + pack_vnni( + /* dst */ k_transpose, + /* src */ curr_k_pad, + /* N */ chunk_size, + /* K */ qk_head_size, + /* ld_src */ qk_head_size, + /* ld_dst */ chunk_size); + // k_beta @ key.transpose(-1, -2) + at::native::cpublas::brgemm( + /* M */ chunk_size, + /* N */ chunk_size, + /* K */ qk_head_size, + /* lda */ qk_head_size, + /* ldb */ chunk_size, + /* ldc */ chunk_size, + /* add_C */ false, + /* A */ curr_k_beta, + /* B */ k_transpose, + /* C */ curr_attn); + // attn = attn * decay_mask + for (int64_t m = 0; m < chunk_size; m++) { + at::vec::map2( + [](fVec x, fVec y) { return fVec(0) - x * y; }, + curr_attn + m * chunk_size, + curr_attn + m * chunk_size, + curr_decay_mask + m * chunk_size, + chunk_size); + } + + // chunk decay + // attn: [B, HV, num_chunk, chunk_size, chunk_size] + // mask = torch.triu(torch.ones(chunk_size, chunk_size, dtype=torch.bool, device=query.device), diagonal=0) + // attn = -attn.masked_fill(mask, 0) + // attn[..., i, :i] = row + (row.unsqueeze(-1) * sub).sum(-2) [B, HV, num_chunk, i] + // attn = attn + torch.eye(chunk_size, dtype=attn.dtype, device=attn.device) + // attn = -attn.masked_fill(mask, 0) + for (int i = 0; i < chunk_size; i++) { + const auto vec_zero = fVec(0); + int64_t len = chunk_size - i; + int64_t front = len % fVecSize; + int64_t j = i; + // first masked vec for alignment + if (front > 0) { + vec_zero.store(curr_attn + i * chunk_size + j, front); + j += front; + } + for (; j < fVecSize * (chunk_size / fVecSize); j += fVecSize) { + vec_zero.store(curr_attn + i * chunk_size + j); + } + } + for (int i = 1; i < chunk_size; i++) { + // row = attn[..., i, :i] [B, HK, num_chunk, i] + int64_t j = 0; + int64_t len = i; + for (; j < fVecSize * (len / fVecSize); j += fVecSize) { + auto tmp0 = fVec::loadu(curr_attn + i * chunk_size + j); + tmp0.store(row + j); + } + if (j < len) { + auto tmp0 = fVec::loadu(curr_attn + i * chunk_size + j, len - j); + tmp0.store(row + j, len - j); + } + // (row.unsqueeze(-1) * sub).sum(-2) + fill_stub(updated, 0, i); + for (int k = 0; k < i; k++) { + float row_k = row[k]; + auto vec_row_k = fVec(row_k); + int64_t j = 0; + int64_t len = i; + for (; j < fVecSize * (len / fVecSize); j += fVecSize) { + auto tmp0 = fVec::loadu(curr_attn + k * chunk_size + j); + auto tmp1 = vec_row_k * tmp0; + auto tmp2 = fVec::loadu(updated + j); + auto tmp3 = tmp1 + tmp2; + tmp3.store(updated + j); + } + if (j < len) { + auto tmp0 = fVec::loadu(curr_attn + k * chunk_size + j, len - j); + auto tmp1 = vec_row_k * tmp0; + auto tmp2 = fVec::loadu(updated + j); + auto tmp3 = tmp1 + tmp2; + tmp3.store(updated + j, len - j); + } + } + // attn[..., i, :i] = row + sum(...) + j = 0; + len = i; + for (; j < fVecSize * (len / fVecSize); j += fVecSize) { + auto tmp0 = fVec::loadu(row + j); + auto tmp1 = fVec::loadu(updated + j); + auto tmp2 = tmp0 + tmp1; + tmp2.store(curr_attn + i * chunk_size + j); + } + if (j < len) { + auto tmp0 = fVec::loadu(row + j, len - j); + auto tmp1 = fVec::loadu(updated + j, len - j); + auto tmp2 = tmp0 + tmp1; + tmp2.store(curr_attn + i * chunk_size + j, len - j); + } + } + for (int i = 0; i < chunk_size; i++) { + curr_attn[i * chunk_size + i] += 1.0f; + at::vec::map( + [](fVec x) { return x; }, curr_attn_reduced + i * chunk_size, curr_attn + i * chunk_size, chunk_size); + } + + // v_beta_attn = attn @ v_beta + // k_cumdecay = attn @ (k_beta * g.exp().unsqueeze(-1)) + // v_beta_attn: [B, HV, num_chunk, chunk_size, EV] + // k_beta_g = k_beta * g: [B, HV, num_chunk, chunk_size, EK] + // k_cumdecay: [B, HV, num_chunk, chunk_size, EK] + // pack for value + pack_vnni2( + /* dst */ v_pack, + /* src */ curr_v_beta, + /* N */ chunk_size, + /* K */ v_head_size, + /* ld_src */ v_head_size, + /* ld_dst */ v_head_size); + // value = attn @ v_beta + at::native::cpublas::brgemm( + /* M */ chunk_size, + /* N */ v_head_size, + /* K */ chunk_size, + /* lda */ chunk_size, + /* ldb */ v_head_size, + /* ldc */ v_head_size, + /* add_C */ false, + /* A */ curr_attn_reduced, + /* B */ v_pack, + /* C */ curr_value); + // k_beta_g = k_beta * g.exp().unsqueeze(-1) + for (int64_t j = 0; j < chunk_size; j++) { + int64_t i = 0; + float g_exp = std::exp(curr_g_pad[j]); + scalar_t g_exp_reduced = static_cast(g_exp); + auto vec_g_exp_reduced = bVec(g_exp_reduced); + for (; i < VecSize * (qk_head_size / VecSize); i += VecSize) { + auto tmp0 = bVec::loadu(curr_k_beta + j * qk_head_size + i); + auto tmp1 = tmp0 * vec_g_exp_reduced; + tmp1.store(k_beta_g + j * qk_head_size + i); + } + } + // pack for k_beta_g + pack_vnni2( + /* dst */ k_beta_g_pack, + /* src */ k_beta_g, + /* N */ chunk_size, + /* K */ qk_head_size, + /* ld_src */ qk_head_size, + /* ld_dst */ qk_head_size); + // k_cumdecay = attn @ k_beta_g + at::native::cpublas::brgemm( + /* M */ chunk_size, + /* N */ qk_head_size, + /* K */ chunk_size, + /* lda */ chunk_size, + /* ldb */ qk_head_size, + /* ldc */ qk_head_size, + /* add_C */ false, + /* A */ curr_attn_reduced, + /* B */ k_beta_g_pack, + /* C */ k_cumdecay); + for (int i = 0; i < chunk_size; i++) { + at::vec::map( + [](fVec x) { return x; }, + curr_k_cumdecay_reduced + i * qk_head_size, + k_cumdecay + i * qk_head_size, + qk_head_size); + } + + // Move to the next query + data_index_step(h, v_num_head, c, global_num_chunk); + } + }); + + // for each chunk + at::parallel_for(0, batch_size * v_num_head, 1, [&](int64_t begin, int64_t end) { + int64_t b = 0, h = 0; + data_index_init(begin, b, batch_size, h, v_num_head); + int ompIdx = at::get_thread_num(); + int64_t offset = + /* k_transpose */ qk_head_size * chunk_size + + /* v_pack */ chunk_size * v_head_size + + /* k_beta_g */ chunk_size * qk_head_size + + /* k_beta_g_pack */ chunk_size * qk_head_size + + /* attn */ chunk_size * chunk_size * 2 + + /* attn_reduced */ chunk_size * chunk_size + + /* k_cumdecay */ chunk_size * qk_head_size * 2 + + /* row */ chunk_size * 2 + + /* updated */ chunk_size * 2; + scalar_t* thread_buff_ptr = thread_buff + ompIdx * buff_size_16bit_per_thread; + THREAD_BUFFER_ALLOC( + curr_last_recurrent_state_reduced, thread_buff_ptr, offset, scalar_t, qk_head_size * v_head_size); + THREAD_BUFFER_ALLOC( + curr_last_recurrent_state_pack_reduced, thread_buff_ptr, offset, scalar_t, qk_head_size * v_head_size); + THREAD_BUFFER_ALLOC(k_transpose_i, thread_buff_ptr, offset, scalar_t, qk_head_size * chunk_size); + THREAD_BUFFER_ALLOC(attn_i, thread_buff_ptr, offset, float, chunk_size* chunk_size * 2); + THREAD_BUFFER_ALLOC(attn_i_reduced, thread_buff_ptr, offset, scalar_t, chunk_size * chunk_size); + THREAD_BUFFER_ALLOC(v_prime, thread_buff_ptr, offset, float, chunk_size* v_head_size * 2); + THREAD_BUFFER_ALLOC(v_prime_reduced, thread_buff_ptr, offset, scalar_t, chunk_size * v_head_size); + THREAD_BUFFER_ALLOC(v_prime_pack_reduced, thread_buff_ptr, offset, scalar_t, chunk_size * v_head_size); + THREAD_BUFFER_ALLOC(qg, thread_buff_ptr, offset, scalar_t, chunk_size * qk_head_size); + THREAD_BUFFER_ALLOC(attn_inter, thread_buff_ptr, offset, float, chunk_size* v_head_size * 2); + THREAD_BUFFER_ALLOC(kg, thread_buff_ptr, offset, scalar_t, chunk_size * qk_head_size); + THREAD_BUFFER_ALLOC(kg_transpose, thread_buff_ptr, offset, scalar_t, qk_head_size * chunk_size); + THREAD_BUFFER_ALLOC(kgv, thread_buff_ptr, offset, float, qk_head_size* v_head_size * 2); + + for ([[maybe_unused]] auto z : c10::irange(begin, end)) { + int64_t start_q = cu_seqlens_ptr[b]; + int64_t seq_len = cu_seqlens_ptr[b + 1] - start_q; + int64_t num_chunk = chunk_offsets_ptr[b + 1] - chunk_offsets_ptr[b]; + int64_t chunk_offset = chunk_offsets_ptr[b]; + int64_t len_offset = chunk_offset * chunk_size; + + int64_t h_qk = h / head_group; + auto out_ptr = out + start_q * oStrideT; + auto curr_q = q_pad + len_offset * qk_head_size + + h_qk * global_total_seq_length * qk_head_size; // [num_chunk, chunk_size, EK] + auto curr_k = k_pad + len_offset * qk_head_size + + h_qk * global_total_seq_length * qk_head_size; // [num_chunk, chunk_size, EK] + auto curr_v = v_beta_attn + h * global_total_seq_length * v_head_size; // [num_chunk, chunk_size, EV] + auto curr_decay_mask = + decay_mask + h * global_total_seq_length * chunk_size; // [num_chunk, chunk_size, chunk_size] + auto curr_k_cumdecay_reduced = + k_cumdecay_reduced + h * global_total_seq_length * qk_head_size; // [num_chunk, chunk_size, EK] + auto curr_last_recurrent_state = + final_state_data + b * final_state_StrideN + h * final_state_StrideH; // [EK, EV] + auto curr_g_pad = g_pad + len_offset + h * global_total_seq_length; // [num_chunk, chunk_size] + auto curr_core_attn_out = core_attn_out + len_offset * v_head_size + + h * global_total_seq_length * v_head_size; // [num_chunk, chunk_size, EV] + for (int64_t c = 0; c < num_chunk; c++) { + for (int i = 0; i < qk_head_size; i++) { + at::vec::map( + [](fVec x) { return x; }, + curr_last_recurrent_state_reduced + i * v_head_size, + curr_last_recurrent_state + i * v_head_size, + v_head_size); + } + auto q_i = curr_q + c * chunk_size * qk_head_size; // [chunk_size, EK] + auto k_i = curr_k + c * chunk_size * qk_head_size; // [chunk_size, EK] + auto v_i = curr_v + (chunk_offset + c) * chunk_size * v_head_size; // [chunk_size, EV] + auto decay_mask_i = curr_decay_mask + (chunk_offset + c) * chunk_size * chunk_size; // [chunk_size, chunk_size] + auto k_cumdecay_i_reduced = + curr_k_cumdecay_reduced + (chunk_offset + c) * chunk_size * qk_head_size; // [chunk_size, EK] + auto g_pad_i = curr_g_pad + c * chunk_size; // [chunk_size] + auto core_attn_out_i = curr_core_attn_out + c * chunk_size * v_head_size; // [chunk_size, EV] + + // attn_i = (q_i @ k_i.transpose(-1, -2) * decay_mask[:, :, i]).masked_fill_(mask, 0) + // k_transpose_i = k_i.transpose(-1, -2) + pack_vnni( + /* dst */ k_transpose_i, + /* src */ k_i, + /* N */ chunk_size, + /* K */ qk_head_size, + /* ld_src */ qk_head_size, + /* ld_dst */ chunk_size); + // attn_i = q_i @ k_transpose_i + at::native::cpublas::brgemm( + /* M */ chunk_size, + /* N */ chunk_size, + /* K */ qk_head_size, + /* lda */ qk_head_size, + /* ldb */ chunk_size, + /* ldc */ chunk_size, + /* add_C */ false, + /* A */ q_i, + /* B */ k_transpose_i, + /* C */ attn_i); + // attn_i = attn_i * decay_mask_i + for (int64_t m = 0; m < chunk_size; m++) { + auto attn_i_m = attn_i + m * chunk_size; + auto attn_i_reduced_m = attn_i_reduced + m * chunk_size; + auto decay_mask_i_m = decay_mask_i + m * chunk_size; + int64_t n = 0; + for (; n < fVecSize * (chunk_size / fVecSize); n += fVecSize) { + auto tmp0 = fVec::loadu(attn_i_m + n); + auto tmp1 = fVec::loadu(decay_mask_i_m + n); + auto tmp2 = tmp0 * tmp1; + auto tmp3 = at::vec::convert(tmp2); + tmp3.store(attn_i_reduced_m + n, fVecSize); + } + if (n < chunk_size) { + auto tmp0 = fVec::loadu(attn_i_m + n, chunk_size - n); + auto tmp1 = fVec::loadu(decay_mask_i_m + n, chunk_size - n); + auto tmp2 = tmp0 * tmp1; + auto tmp3 = at::vec::convert(tmp2); + tmp3.store(attn_i_reduced_m + n, chunk_size - n); + } + } + // mask = torch.triu(torch.ones(chunk_size, chunk_size, dtype=torch.bool, device=query.device), diagonal=1) + // attn_i = attn_i.masked_fill_(mask, 0) + for (int i = 0; i < chunk_size - 1; i++) { + const auto vec_zero = bVec(0); + int64_t len = chunk_size - i - 1; + int64_t front = len % VecSize; + int64_t j = i + 1; + // first masked vec for alignment + if (front > 0) { + vec_zero.store(attn_i_reduced + i * chunk_size + j, front); + j += front; + } + for (; j < VecSize * (chunk_size / VecSize); j += VecSize) { + vec_zero.store(attn_i_reduced + i * chunk_size + j); + } + } + + // pack for curr_last_recurrent_state + pack_vnni2( + /* dst */ curr_last_recurrent_state_pack_reduced, + /* src */ curr_last_recurrent_state_reduced, + /* N */ qk_head_size, + /* K */ v_head_size, + /* ld_src */ v_head_size, + /* ld_dst */ v_head_size); + + // v_prime = k_cumdecay_i @ curr_last_recurrent_state: [chunk_size, EV] + // k_cumdecay_i: [chunk_size, EK] + // curr_last_recurrent_state: [EK, EV] + at::native::cpublas::brgemm( + /* M */ chunk_size, + /* N */ v_head_size, + /* K */ qk_head_size, + /* lda */ qk_head_size, + /* ldb */ v_head_size, + /* ldc */ v_head_size, + /* add_C */ false, + /* A */ k_cumdecay_i_reduced, + /* B */ curr_last_recurrent_state_pack_reduced, + /* C */ v_prime); + + // v_new = v_prime = v_i - v_prime + // v_i: [chunk_size, EV] + for (int64_t m = 0; m < chunk_size; m++) { + int64_t i = 0; + for (; i < fVecSize * (v_head_size / fVecSize); i += fVecSize) { + auto tmp0 = fVec::loadu(v_i + m * v_head_size + i); + auto tmp1 = fVec::loadu(v_prime + m * v_head_size + i); + auto tmp2 = tmp0 - tmp1; + auto tmp3 = at::vec::convert(tmp2); + tmp3.store(v_prime_reduced + m * v_head_size + i, fVecSize); + } + } + + // attn_inter = (q_i * g[:, :, i, :, None].exp()) @ last_recurrent_state + // qg = q_i * g[:, :, i, :, None].exp(): [chunk_size, EK] + // q_i: [chunk_size, EK] + // g[:, :, i, :, None]: [chunk_size, 1] + for (int64_t m = 0; m < chunk_size; m++) { + auto g_pad_i_m = g_pad_i + m; + auto g_exp = std::exp(*g_pad_i_m); + int64_t i = 0; + scalar_t g_exp_reduced = static_cast(g_exp); + auto vec_g_exp_reduced = bVec(g_exp_reduced); + for (; i < VecSize * (qk_head_size / VecSize); i += VecSize) { + auto tmp0 = bVec::loadu(q_i + m * qk_head_size + i); + auto tmp2 = tmp0 * vec_g_exp_reduced; + tmp2.store(qg + m * qk_head_size + i); + } + } + // attn_inter = qg @ curr_last_recurrent_state: [chunk_size, EV] + // curr_last_recurrent_state: [EK, EV] + at::native::cpublas::brgemm( + /* M */ chunk_size, + /* N */ v_head_size, + /* K */ qk_head_size, + /* lda */ qk_head_size, + /* ldb */ v_head_size, + /* ldc */ v_head_size, + /* add_C */ false, + /* A */ qg, + /* B */ curr_last_recurrent_state_pack_reduced, + /* C */ attn_inter); + + // core_attn_out[:, :, i] = attn_inter + attn_i @ v_new + // pack for v_prime + pack_vnni2( + /* dst */ v_prime_pack_reduced, + /* src */ v_prime_reduced, + /* N */ chunk_size, + /* K */ v_head_size, + /* ld_src */ v_head_size, + /* ld_dst */ v_head_size); + // attn_inter = attn_inter + attn_i @ v_new: [chunk_size, EV] + // attn_i: [chunk_size, chunk_size] + // v_new: [chunk_size, EV] + at::native::cpublas::brgemm( + /* M */ chunk_size, + /* N */ v_head_size, + /* K */ chunk_size, + /* lda */ chunk_size, + /* ldb */ v_head_size, + /* ldc */ v_head_size, + /* add_C */ true, + /* A */ attn_i_reduced, + /* B */ v_prime_pack_reduced, + /* C */ attn_inter); + + // core_attn_out[:, :, i] = attn_inter + for (int64_t m = 0; m < chunk_size; m++) { + at::vec::map( + [](fVec x) { return x; }, core_attn_out_i + m * v_head_size, attn_inter + m * v_head_size, v_head_size); + } + + // last_recurrent_state = ( + // last_recurrent_state * g[:, :, i, -1, None, None].exp() + // + (k_i * (g[:, :, i, -1, None] - g[:, :, i]).exp()[..., None]).transpose(-1, -2) @ v_new + // ) + // 1) last_recurrent_state * g[:, :, i, -1, None, None].exp() + // curr_last_recurrent_state: [EK, EV] + // g[:, :, i, -1, None, None]: [1, 1] + // last_recurrent_state * g[:, :, i, -1, None, None].exp(): [EK, EV] + auto g_pad_i_last = g_pad_i + chunk_size - 1; + auto g_exp_last = std::exp(g_pad_i_last[0]); + for (int64_t m = 0; m < qk_head_size; m++) { + int64_t i = 0; + auto vec_g_exp_last = fVec(g_exp_last); + for (; i < fVecSize * (v_head_size / fVecSize); i += fVecSize) { + auto tmp0 = bVec::loadu(curr_last_recurrent_state_reduced + m * v_head_size + i); + auto tmp1 = at::vec::convert(tmp0); + auto tmp2 = tmp1 * vec_g_exp_last; + tmp2.store(curr_last_recurrent_state + m * v_head_size + i); + } + if (i < v_head_size) { + auto tmp0 = bVec::loadu(curr_last_recurrent_state_reduced + m * v_head_size + i, v_head_size - i); + auto tmp1 = at::vec::convert(tmp0); + auto tmp2 = tmp1 * vec_g_exp_last; + tmp2.store(curr_last_recurrent_state + m * v_head_size + i, v_head_size - i); + } + } + // 2) (k_i * (g[:, :, i, -1, None] - g[:, :, i]).exp()[..., None]).transpose(-1, -2) @ v_new + // k_i: [chunk_size, EK] + // g[:, :, i, -1, None]: [1] + // g[:, :, i]: [chunk_size] + // (g[:, :, i, -1, None] - g[:, :, i]).exp()[..., None]: [chunk_size, 1] + // kg = k_i * (g[:, :, i, -1, None] - g[:, :, i]).exp()[..., None]: [chunk_size, EK] + // (k_i * (g[:, :, i, -1, None] - g[:, :, i]).exp()[..., None]).transpose(-1, -2): [EK, chunk_size] + // v_new: [chunk_size, EV] + // (k_i * (g[:, :, i, -1, None] - g[:, :, i]).exp()[..., None]).transpose(-1, -2) @ v_new: [EK, EV] + // kg = k_i * (g[:, :, i, -1, None] - g[:, :, i]).exp()[..., None] + for (int64_t m = 0; m < chunk_size; m++) { + auto g_exp = std::exp((g_pad_i_last[0] - g_pad_i[m])); + int64_t i = 0; + scalar_t g_exp_reduced = static_cast(g_exp); + auto vec_g_exp_reduced = bVec(g_exp_reduced); + for (; i < VecSize * (qk_head_size / VecSize); i += VecSize) { + auto tmp0 = bVec::loadu(k_i + m * qk_head_size + i); + auto tmp2 = tmp0 * vec_g_exp_reduced; + tmp2.store(kg + m * qk_head_size + i); + } + } + // kg.transpose(-1, -2): [EK, chunk_size] + at::native::utils::transpose( + /* M */ chunk_size, + /* N */ qk_head_size, + /* src */ kg, + /* ld_src */ qk_head_size, + /* dst */ kg_transpose, + /* ld_dst */ chunk_size); + // kgv = kg.transpose(-1, -2) @ v_new + // v_new: [chunk_size, EV] + at::native::cpublas::brgemm( + /* M */ qk_head_size, + /* N */ v_head_size, + /* K */ chunk_size, + /* lda */ chunk_size, + /* ldb */ v_head_size, + /* ldc */ v_head_size, + /* add_C */ false, + /* A */ kg_transpose, + /* B */ v_prime_pack_reduced, + /* C */ kgv); + // last_recurrent_state = 1) + 2) + for (int64_t m = 0; m < qk_head_size; m++) { + at::vec::map2( + [](fVec x, fVec y) { return x + y; }, + curr_last_recurrent_state + m * v_head_size, + curr_last_recurrent_state + m * v_head_size, + kgv + m * v_head_size, + v_head_size); + } + } + + // core_attn_out -> output + // output: [B, T, HV, EV] + // core_attn_out: [B, HV, padded_T, EV] + auto curr_out = out_ptr + h * oStrideH; + for (int64_t m = 0; m < seq_len; m++) { + at::vec::map( + [](fVec x) { return x; }, curr_out + m * oStrideT, curr_core_attn_out + m * v_head_size, v_head_size); + } + + // Move to the next query + data_index_step(b, batch_size, h, v_num_head); + } + }); +} + +inline float softplus(float x, double threshold = 20.0) { + if (x > threshold) + return x; + else if (x < -threshold) + return std::exp(x); + else + return std::log1p(std::exp(x)); +} + +inline at::vec::Vectorized softplus(const at::vec::Vectorized& x, double threshold = 20.0) { + using Vec = at::vec::Vectorized; + Vec mask_hi = x > Vec(threshold); + Vec mask_lo = x < Vec(-threshold); + + Vec expx = x.exp_u20(); + Vec log1pex = (expx + Vec(1.0f)).log(); + + return Vec::blendv(Vec::blendv(log1pex, expx, mask_lo), x, mask_hi); +} + +template +void fused_sigmoid_gating_delta_rule_update_kernel_impl( + const scalar_t* __restrict__ q_ptr, + const scalar_t* __restrict__ k_ptr, + const scalar_t* __restrict__ v_ptr, + const param_t* __restrict__ A_log_ptr, + const scalar_t* __restrict__ a_ptr, + const scalar_t* __restrict__ dt_bias_ptr, + const scalar_t* __restrict__ b_ptr, + const int32_t* __restrict__ indices_ptr, + float* __restrict__ state_ptr, + scalar_t* __restrict__ o_ptr, + float* __restrict__ qk_scale_buf, + int64_t seq_len, + int64_t batch_size, + int64_t num_heads, + int64_t head_dim, + int64_t v_num_heads, + int64_t v_head_dim, + int64_t q_strideB, + int64_t q_strideS, + int64_t q_strideH, + int64_t k_strideB, + int64_t k_strideS, + int64_t k_strideH, + int64_t v_strideB, + int64_t v_strideS, + int64_t v_strideH, + bool use_qk_l2norm_in_kernel, + double softplus_threshold) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + + constexpr int64_t VecSize = bVec::size(); + constexpr int64_t fVecSize = fVec::size(); + int64_t group_size = v_num_heads / num_heads; + double scale = 1 / std::sqrt(head_dim); + fVec scale_vec = fVec(scale); + if (use_qk_l2norm_in_kernel) { + float eps = 1e-5; + at::parallel_for(0, batch_size * seq_len * num_heads, 0, [&](int64_t begin, int64_t end) { + int64_t bi{0}, si{0}, ni{0}; + data_index_init(begin, bi, batch_size, si, seq_len, ni, num_heads); + for (int64_t i = begin; i < end; ++i) { + float sum_q = float(0); + float sum_k = float(0); + fVec sum_q_fvec = fVec(float(0)); + fVec sum_k_fvec = fVec(float(0)); + int64_t q_offset = bi * q_strideB + si * q_strideS + ni * q_strideH; + int64_t k_offset = bi * k_strideB + si * k_strideS + ni * k_strideH; + int64_t q_scale_offset = bi * seq_len * num_heads + si * num_heads + ni; + int64_t k_scale_offset = q_scale_offset + batch_size * seq_len * num_heads; + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= head_dim - VecSize; d += VecSize) { + bVec q_bvec = bVec::loadu(q_ptr + q_offset + d); + fVec q_fvec0, q_fvec1; + std::tie(q_fvec0, q_fvec1) = at::vec::convert_to_float(q_bvec); + sum_q_fvec += q_fvec0 * q_fvec0; + sum_q_fvec += q_fvec1 * q_fvec1; + bVec k_bvec = bVec::loadu(k_ptr + k_offset + d); + fVec k_fvec0, k_fvec1; + std::tie(k_fvec0, k_fvec1) = at::vec::convert_to_float(k_bvec); + sum_k_fvec += k_fvec0 * k_fvec0; + sum_k_fvec += k_fvec1 * k_fvec1; + } +#pragma GCC unroll 4 + for (; d < head_dim; ++d) { + float q_val = static_cast(q_ptr[q_offset + d]); + sum_q += q_val * q_val; + float k_val = static_cast(k_ptr[k_offset + d]); + sum_k += k_val * k_val; + } + + sum_q += vec_reduce_sum(sum_q_fvec); + sum_k += vec_reduce_sum(sum_k_fvec); + qk_scale_buf[q_scale_offset] = float(1) / std::sqrt(sum_q + eps); + qk_scale_buf[k_scale_offset] = float(1) / std::sqrt(sum_k + eps); + + data_index_step(bi, batch_size, si, seq_len, ni, num_heads); + } + }); + } + at::parallel_for(0, batch_size * seq_len * v_num_heads, 0, [&](int64_t begin, int64_t end) { + int64_t bi{0}, si{0}, ni{0}; + data_index_init(begin, bi, batch_size, si, seq_len, ni, v_num_heads); + for (int64_t i = begin; i < end; ++i) { + int64_t cache_index = indices_ptr[bi]; + int64_t state_offset = (cache_index * v_num_heads + ni) * head_dim * v_head_dim; + float g_val = -std::exp(float(A_log_ptr[ni])) * + softplus(float(a_ptr[bi * v_num_heads + ni]) + float(dt_bias_ptr[ni]), softplus_threshold); + float g_val_exp = std::exp(g_val); + fVec g_val_exp_vec = fVec(g_val_exp); + int64_t q_offset = si * q_strideS + bi * q_strideB + (ni / group_size) * q_strideH; + int64_t k_offset = si * k_strideS + bi * k_strideB + (ni / group_size) * k_strideH; + int64_t q_scale_offset = bi * seq_len * num_heads + si * num_heads + (ni / group_size); + int64_t k_scale_offset = q_scale_offset + batch_size * seq_len * num_heads; + float q_scale = use_qk_l2norm_in_kernel ? qk_scale_buf[q_scale_offset] : 1.0f; + float k_scale = use_qk_l2norm_in_kernel ? qk_scale_buf[k_scale_offset] : 1.0f; + int64_t v_offset = si * v_strideS + bi * v_strideB + ni * v_strideH; + int64_t o_offset = ((bi * seq_len + si) * v_num_heads + ni) * v_head_dim; + float beta_val = 1 / (1 + std::exp(-b_ptr[ni])); + fVec beta_vec = fVec(beta_val); + int64_t dvi = 0; + for (; dvi <= v_head_dim - VecSize; dvi += VecSize) { + fVec kv_mem_vec0 = fVec(float(0)); + fVec kv_mem_vec1 = fVec(float(0)); + for (int di = 0; di < head_dim; ++di) { + fVec k_val_vec = fVec(k_ptr[k_offset + di] * k_scale); + fVec state_vec0 = fVec::loadu(state_ptr + state_offset + di * v_head_dim + dvi); + fVec state_vec1 = fVec::loadu(state_ptr + state_offset + di * v_head_dim + dvi + fVecSize); + kv_mem_vec0 = kv_mem_vec0 + state_vec0 * g_val_exp_vec * k_val_vec; + kv_mem_vec1 = kv_mem_vec1 + state_vec1 * g_val_exp_vec * k_val_vec; + } + bVec v_bvec = bVec::loadu(v_ptr + v_offset + dvi); + fVec v_vec0, v_vec1; + std::tie(v_vec0, v_vec1) = at::vec::convert_to_float(v_bvec); + fVec dt_vec0 = (v_vec0 - kv_mem_vec0) * beta_vec; + fVec dt_vec1 = (v_vec1 - kv_mem_vec1) * beta_vec; + fVec o_vec0 = fVec(float(0)); + fVec o_vec1 = fVec(float(0)); + for (int di = 0; di < head_dim; ++di) { + fVec q_vec = fVec(q_ptr[q_offset + di] * q_scale); + fVec k_vec = fVec(k_ptr[k_offset + di] * k_scale); + fVec state_vec0 = fVec::loadu(state_ptr + state_offset + di * v_head_dim + dvi); + fVec state_vec1 = fVec::loadu(state_ptr + state_offset + di * v_head_dim + dvi + fVecSize); + state_vec0 = state_vec0 * g_val_exp_vec + k_vec * dt_vec0; + state_vec1 = state_vec1 * g_val_exp_vec + k_vec * dt_vec1; + o_vec0 = o_vec0 + state_vec0 * q_vec * scale_vec; + o_vec1 = o_vec1 + state_vec1 * q_vec * scale_vec; + state_vec0.store(state_ptr + state_offset + di * v_head_dim + dvi); + state_vec1.store(state_ptr + state_offset + di * v_head_dim + dvi + fVecSize); + } + bVec o_vec = at::vec::convert_from_float(o_vec0, o_vec1); + o_vec.store(o_ptr + o_offset + dvi); + } + for (; dvi < v_head_dim; ++dvi) { + float kv_mem_val = 0; + for (int di = 0; di < head_dim; ++di) { + float k_val = k_ptr[k_offset + di] * k_scale; + state_ptr[state_offset + di * v_head_dim + dvi] *= g_val_exp; + kv_mem_val += state_ptr[state_offset + di * v_head_dim + dvi] * k_val; + } + float v_val = v_ptr[v_offset + dvi]; + float dt_val = (v_val - kv_mem_val) * beta_val; + float o_val = 0; + for (int di = 0; di < head_dim; ++di) { + float q_val = q_ptr[q_offset + di] * q_scale; + float k_val = k_ptr[k_offset + di] * k_scale; + state_ptr[state_offset + di * v_head_dim + dvi] += k_val * dt_val; + o_val += state_ptr[state_offset + di * v_head_dim + dvi] * q_val * scale; + } + o_ptr[o_offset + dvi] = o_val; + } + data_index_step(bi, batch_size, si, seq_len, ni, v_num_heads); + } + }); +} + +template +void fused_gdn_gating_kernel_impl( + float* __restrict__ A_log, + const scalar_t* __restrict__ a, + const scalar_t* __restrict__ b, + const scalar_t* __restrict__ dt_bias, + float* __restrict__ out, + scalar_t* __restrict__ beta, + int64_t batch, + int64_t num_heads) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int vec_size = bVec::size(); + constexpr int fvec_size = fVec::size(); + const fVec neg_one(-1.0f); + const fVec one(1.0f); + at::parallel_for(0, batch, 0, [&](int64_t begin, int64_t end) { + for (int64_t i = begin; i < end; ++i) { + int64_t j = 0; + for (; j < num_heads - (num_heads % vec_size); j += vec_size) { + fVec A_log_vec0 = fVec::loadu(A_log + j); + fVec A_log_vec1 = fVec::loadu(A_log + j + fvec_size); + bVec dt_bias_vec = bVec::loadu(dt_bias + j); + bVec a_bvec = bVec::loadu(a + i * num_heads + j); + bVec b_bvec = bVec::loadu(b + i * num_heads + j); + fVec a0, a1, dt_bias_vec0, dt_bias_vec1, b0, b1; + std::tie(a0, a1) = at::vec::convert_to_float(a_bvec); + std::tie(b0, b1) = at::vec::convert_to_float(b_bvec); + std::tie(dt_bias_vec0, dt_bias_vec1) = at::vec::convert_to_float(dt_bias_vec); + + fVec g0 = neg_one * A_log_vec0.exp_u20() * softplus(a0 + dt_bias_vec0); + fVec g1 = neg_one * A_log_vec1.exp_u20() * softplus(a1 + dt_bias_vec1); + fVec beta0 = one / (one + (neg_one * b0).exp_u20()); + fVec beta1 = one / (one + (neg_one * b1).exp_u20()); + + g0.store(out + i * num_heads + j); + g1.store(out + i * num_heads + j + fvec_size); + bVec beta_vec = at::vec::convert_from_float(beta0, beta1); + beta_vec.store(beta + i * num_heads + j); + } + for (; j < num_heads; ++j) { + out[i * num_heads + j] = -std::exp(A_log[j]) * softplus(float(a[i * num_heads + j]) + float(dt_bias[j])); + beta[i * num_heads + j] = 1 / (1 + std::exp(-b[i * num_heads + j])); + } + } + }); +} + +template +void fused_gdn_gating_kernel_impl( + scalar_t* __restrict__ A_log, + const scalar_t* __restrict__ a, + const scalar_t* __restrict__ b, + const scalar_t* __restrict__ dt_bias, + float* __restrict__ out, + scalar_t* __restrict__ beta, + int64_t batch, + int64_t num_heads) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int vec_size = bVec::size(); + constexpr int fvec_size = fVec::size(); + const fVec neg_one(-1.0f); + const fVec one(1.0f); + at::parallel_for(0, batch, 0, [&](int64_t begin, int64_t end) { + for (int64_t i = begin; i < end; ++i) { + int64_t j = 0; + for (; j < num_heads - (num_heads % vec_size); j += vec_size) { + bVec A_log_bvec = bVec::loadu(A_log + j); + fVec A_log_vec0, A_log_vec1; + std::tie(A_log_vec0, A_log_vec1) = at::vec::convert_to_float(A_log_bvec); + bVec dt_bias_vec = bVec::loadu(dt_bias + j); + bVec a_bvec = bVec::loadu(a + i * num_heads + j); + bVec b_bvec = bVec::loadu(b + i * num_heads + j); + fVec a0, a1, dt_bias_vec0, dt_bias_vec1, b0, b1; + std::tie(a0, a1) = at::vec::convert_to_float(a_bvec); + std::tie(b0, b1) = at::vec::convert_to_float(b_bvec); + std::tie(dt_bias_vec0, dt_bias_vec1) = at::vec::convert_to_float(dt_bias_vec); + + fVec g0 = neg_one * A_log_vec0.exp_u20() * softplus(a0 + dt_bias_vec0); + fVec g1 = neg_one * A_log_vec1.exp_u20() * softplus(a1 + dt_bias_vec1); + fVec beta0 = one / (one + (neg_one * b0).exp_u20()); + fVec beta1 = one / (one + (neg_one * b1).exp_u20()); + + g0.store(out + i * num_heads + j); + g1.store(out + i * num_heads + j + fvec_size); + bVec beta_vec = at::vec::convert_from_float(beta0, beta1); + beta_vec.store(beta + i * num_heads + j); + } + for (; j < num_heads; ++j) { + out[i * num_heads + j] = -std::exp(float(A_log[j])) * softplus(float(a[i * num_heads + j]) + float(dt_bias[j])); + beta[i * num_heads + j] = 1 / (1 + std::exp(-b[i * num_heads + j])); + } + } + }); +} + +} // anonymous namespace + +template +inline void +CHECK_INPUT_SHAPE_DTYPE(const at::Tensor& tensor, const int64_t& dim, const at::IntArrayRef& sizes, at::ScalarType st) { + TORCH_CHECK(tensor.sizes() == sizes, "Input tensor shape mismatch: expected ", sizes, ", got ", tensor.sizes()); + TORCH_CHECK(tensor.dtype() == st, "Input tensor dtype mismatch"); + CHECK_DIM(dim, tensor); + if (is_last_dim_contiguous) { + CHECK_LAST_DIM_CONTIGUOUS_INPUT(tensor); + } else { + CHECK_CONTIGUOUS(tensor); + } +} + +// query: [B, T, HK, EK] +// key: [B, T, HK, EK] +// value: [B, T, HV, EV] +// g: [B, T, HV] FP32 +// beta: [B, T, HV] +// initial_state: [N, HV, EK, EV] FP32 +// output_final_state: bool +// cu_seqlens: [N + 1] INT32 +// head_first: bool +// use_qk_l2norm_in_kernel: bool +std::tuple chunk_gated_delta_rule_cpu( + const at::Tensor& query, + const at::Tensor& key, + const at::Tensor& value, + const at::Tensor& g, + const at::Tensor& beta, + const at::Tensor& initial_state, + bool output_final_state, + const at::Tensor& cu_seqlens, + bool head_first, + bool use_qk_l2norm_in_kernel, + double eps = 1e-5) { + TORCH_CHECK(head_first == false, "chunk_gated_delta_rule_cpu does not support head first"); + int64_t B = query.size(0); + int64_t global_seq_len = query.size(1); + int64_t qk_num_head = query.size(2); + int64_t qk_head_size = query.size(3); + int64_t v_num_head = value.size(2); + int64_t v_head_size = value.size(3); + int64_t batch_size = initial_state.size(0); + CHECK_EQ(B, 1); + TORCH_CHECK(v_num_head % qk_num_head == 0, "expect v_num_head multiple of qk_num_head."); + TORCH_CHECK(qk_head_size % 32 == 0, "expect qk_head_size to be multiples of 32."); + TORCH_CHECK(v_head_size % 32 == 0, "expect v_head_size to be multiples of 32."); + CHECK_INPUT_SHAPE_DTYPE(query, 4, {B, global_seq_len, qk_num_head, qk_head_size}, at::kBFloat16); + CHECK_INPUT_SHAPE_DTYPE(key, 4, {B, global_seq_len, qk_num_head, qk_head_size}, at::kBFloat16); + CHECK_INPUT_SHAPE_DTYPE(value, 4, {B, global_seq_len, v_num_head, v_head_size}, at::kBFloat16); + CHECK_INPUT_SHAPE_DTYPE(g, 3, {B, global_seq_len, v_num_head}, at::kFloat); + CHECK_INPUT_SHAPE_DTYPE(beta, 3, {B, global_seq_len, v_num_head}, at::kBFloat16); + CHECK_INPUT_SHAPE_DTYPE(cu_seqlens, 1, {batch_size + 1}, at::kInt); + CHECK_INPUT_SHAPE_DTYPE(initial_state, 4, {batch_size, v_num_head, qk_head_size, v_head_size}, at::kFloat); + + at::Tensor output = at::empty_like(value, value.options()); // [B, T, HV, EV] + at::Tensor final_state = initial_state.to(at::kFloat); // [N, HV, EK, EV] + + // Strides + int64_t qStrideH = query.stride(2); + int64_t qStrideT = query.stride(1); + int64_t kStrideH = key.stride(2); + int64_t kStrideT = key.stride(1); + int64_t vStrideH = value.stride(2); + int64_t vStrideT = value.stride(1); + int64_t oStrideH = output.stride(2); + int64_t oStrideT = output.stride(1); + + constexpr int64_t chunk_size = 64; + // Deduce the global chunks + // e.g. cu_seqlens: [0, 5, 13, 16], chunk_size = 4 + // chunk_offsets: [0, 2, 4, 5] + // chunk_indices (batch_id, local_chunk_id): [[0, 0], [0, 1], [1, 0], [1, 1], [2, 0]] + at::Tensor chunk_offsets = at::empty(batch_size + 1, cu_seqlens.options()); + auto chunk_offsets_ptr = chunk_offsets.data_ptr(); + chunk_offsets_ptr[0] = 0; + int32_t* cu_seqlens_ptr = cu_seqlens.data_ptr(); + int64_t s = 0; + int64_t e = 0; + int64_t s_pad = 0; + int64_t e_pad = 0; + for (int64_t b = 0; b < batch_size; b++) { + e = cu_seqlens_ptr[b + 1]; + int64_t seq_len = e - s; + int64_t pad_size = (chunk_size - seq_len % chunk_size) % chunk_size; + int64_t total_seq_length = seq_len + pad_size; + e_pad = s_pad + total_seq_length; + chunk_offsets[b + 1] = e_pad / chunk_size; + s = e; + s_pad = e_pad; + } + int64_t global_total_seq_length = e_pad; + int64_t global_num_chunk = chunk_offsets_ptr[batch_size]; + at::Tensor chunk_indices = at::empty(global_num_chunk * 2, cu_seqlens.options()); + auto chunk_indices_ptr = chunk_indices.data_ptr(); + int64_t curr_c = 0; + for (int64_t b = 0; b < batch_size; b++) { + int64_t batch_chunk_num = chunk_offsets_ptr[b + 1] - chunk_offsets_ptr[b]; + for (int64_t c = 0; c < batch_chunk_num; c++) { + chunk_indices_ptr[curr_c * 2] = b; + chunk_indices_ptr[curr_c * 2 + 1] = c; + curr_c += 1; + } + } + + // Allocate buffer + int64_t buff_size = v_num_head * global_total_seq_length // g_pad_data + + batch_size * v_num_head * global_total_seq_length * v_head_size // core_attn + + v_num_head * global_total_seq_length * chunk_size // decay_mask + + v_num_head * global_total_seq_length * v_head_size; // v_beta_attn + at::Tensor buff_data = at::empty({buff_size}, query.options().dtype(at::kFloat)); + int64_t reduced_buff_size = qk_num_head * global_total_seq_length * qk_head_size // q_pad_data + + qk_num_head * global_total_seq_length * qk_head_size // k_pad_data + + v_num_head * global_total_seq_length * v_head_size // v_pad_data + + v_num_head * global_total_seq_length * qk_head_size // k_beta_data + + v_num_head * global_total_seq_length * v_head_size // v_beta_data + + v_num_head * global_total_seq_length * qk_head_size // k_cumdecay_reduced + + qk_num_head * global_seq_len // q_norm_sum + + qk_num_head * global_seq_len; // k_norm_sum + at::Tensor reduced_buff_data = at::empty({reduced_buff_size}, query.options()); + int64_t num_thread = at::get_num_threads(); + int64_t buff_size_16bit_per_thread = + /* k_transpose */ qk_head_size * chunk_size + + /* v_pack */ chunk_size * v_head_size + + /* k_beta_g */ chunk_size * qk_head_size + + /* k_beta_g_pack */ chunk_size * qk_head_size + + /* attn */ chunk_size * chunk_size * 2 + + /* attn_reduced */ chunk_size * chunk_size + + /* k_cumdecay */ chunk_size * qk_head_size * 2 + + /* row */ chunk_size * 2 + + /* updated */ chunk_size * 2 + + /* curr_last_recurrent_state_reduced */ qk_head_size * v_head_size + + /* curr_last_recurrent_state_pack_reduced */ qk_head_size * v_head_size + + /* k_transpose_i */ qk_head_size * chunk_size + + /* attn_i */ chunk_size * chunk_size * 2 + + /* attn_i_reduced */ chunk_size * chunk_size + + /* v_prime */ chunk_size * v_head_size * 2 + + /* v_prime_reduced */ chunk_size * v_head_size + + /* v_prime_pack_reduced */ chunk_size * v_head_size + + /* qg */ chunk_size * qk_head_size + + /* attn_inter */ chunk_size * v_head_size * 2 + + /* kg */ chunk_size * qk_head_size + + /* kg_transpose */ qk_head_size * chunk_size + + /* kgv */ qk_head_size * v_head_size * 2; + at::Tensor thread_buff_data = at::empty({num_thread, buff_size_16bit_per_thread}, query.options()); + + AT_DISPATCH_REDUCED_FLOATING_TYPES(query.scalar_type(), "chunk_gated_delta_rule_kernel", [&] { + chunk_gated_delta_rule_kernel_impl( + output.data_ptr(), + final_state.data_ptr(), + query.data_ptr(), + key.data_ptr(), + value.data_ptr(), + g.data_ptr(), + beta.data_ptr(), + cu_seqlens_ptr, + buff_data.data_ptr(), + reduced_buff_data.data_ptr(), + thread_buff_data.data_ptr(), + chunk_offsets_ptr, + chunk_indices_ptr, + use_qk_l2norm_in_kernel, + batch_size, + global_seq_len, + qk_num_head, + v_num_head, + qk_head_size, + v_head_size, + qStrideH, + qStrideT, + kStrideH, + kStrideT, + vStrideH, + vStrideT, + oStrideH, + oStrideT, + global_total_seq_length, + global_num_chunk, + buff_size_16bit_per_thread, + eps); + }); + return std::make_tuple(std::move(output), std::move(final_state)); +} + +// A_log: [v_num_heads] +// dt_bias: [v_num_heads] +// query: [seq_len, batch_size, num_heads, head_dim] +// key: [seq_len, batch_size, num_heads, head_dim] +// value: [seq_len, batch_size, v_num_heads, v_head_dim] +// a: [batch_size, v_num_heads] +// b: [batch_size, v_num_heads] +// initial_state_source:[num_tokens, v_num_heads, head_dim, v_head_dim] +// initial_state_indices: [batch_size] +// cu_seqlens: [batch_size + 1] +at::Tensor fused_sigmoid_gating_delta_rule_update_cpu( + const at::Tensor& A_log, + const at::Tensor& dt_bias, + const at::Tensor& q, + const at::Tensor& k, + const at::Tensor& v, + const at::Tensor& a, + const at::Tensor& b, + at::Tensor& initial_state_source, + const at::Tensor& initial_state_indices, + const at::Tensor& cu_seqlens, + bool use_qk_l2norm_in_kernel, + double softplus_beta = 1.0, + double softplus_threshold = 20.0) { + CHECK_DIM(4, q); + CHECK_DIM(4, v); + CHECK_LAST_DIM_CONTIGUOUS_INPUT(q); + int64_t seq_len = q.size(0); + int64_t batch_size = q.size(1); + int64_t num_heads = q.size(2); + int64_t head_dim = q.size(3); + int64_t v_num_heads = v.size(2); + int64_t v_head_dim = v.size(3); + CHECK_INPUT_SHAPE_DTYPE(k, {seq_len, batch_size, num_heads, head_dim}, q.scalar_type()); + CHECK_INPUT_SHAPE_DTYPE(v, {seq_len, batch_size, v_num_heads, v_head_dim}, q.scalar_type()); + CHECK_INPUT_SHAPE_DTYPE(a, {batch_size, v_num_heads}, q.scalar_type()); + CHECK_INPUT_SHAPE_DTYPE(dt_bias, {v_num_heads}, q.scalar_type()); + CHECK_INPUT_SHAPE_DTYPE(b, {batch_size, v_num_heads}, q.scalar_type()); + CHECK_INPUT_SHAPE_DTYPE(initial_state_indices, {batch_size}, at::kInt); + CHECK_INPUT_SHAPE_DTYPE(cu_seqlens, {batch_size + 1}, at::kInt); + CHECK_INPUT_SHAPE_DTYPE( + initial_state_source, {initial_state_source.size(0), v_num_heads, head_dim, v_head_dim}, at::kFloat); + CHECK(initial_state_source.size(0) >= batch_size); + CHECK_EQ(v_num_heads % num_heads, 0); + TORCH_CHECK( + A_log.sizes() == at::IntArrayRef({v_num_heads}), + "Input tensor shape mismatch: expected ", + at::IntArrayRef({v_num_heads}), + ", got ", + A_log.sizes()); + + int64_t q_strideB = q.stride(1); + int64_t q_strideS = q.stride(0); + int64_t q_strideH = q.stride(2); + int64_t k_strideB = k.stride(1); + int64_t k_strideS = k.stride(0); + int64_t k_strideH = k.stride(2); + int64_t v_strideB = v.stride(1); + int64_t v_strideS = v.stride(0); + int64_t v_strideH = v.stride(2); + at::Tensor core_attn_out = at::empty({batch_size, seq_len, v_num_heads, v_head_dim}, q.options()); + at::Tensor qk_scale_buf = at::empty({2 * batch_size, seq_len, num_heads}, at::kFloat); + + CPU_DISPATCH_REDUCED_FLOATING_TYPES_EXT( + q.scalar_type(), A_log.scalar_type(), "fused_sigmoid_gating_delta_rule_update_kernel_impl", [&] { + fused_sigmoid_gating_delta_rule_update_kernel_impl( + q.data_ptr(), + k.data_ptr(), + v.data_ptr(), + A_log.data_ptr(), + a.data_ptr(), + dt_bias.data_ptr(), + b.data_ptr(), + initial_state_indices.data_ptr(), + initial_state_source.data_ptr(), + core_attn_out.data_ptr(), + qk_scale_buf.data_ptr(), + seq_len, + batch_size, + num_heads, + head_dim, + v_num_heads, + v_head_dim, + q_strideB, + q_strideS, + q_strideH, + k_strideB, + k_strideS, + k_strideH, + v_strideB, + v_strideS, + v_strideH, + use_qk_l2norm_in_kernel, + softplus_threshold); + }); + return core_attn_out; +} + +// A_log: [num_v_heads] +// a: [batch, num_v_heads] +// b: [batch, num_v_heads] +// dt_bias: [num_v_heads] +// -A_log.float().exp() * F.softplus(a.float() + dt_bias) +std::tuple +fused_gdn_gating_cpu(const at::Tensor& A_log, const at::Tensor& a, const at::Tensor& b, const at::Tensor& dt_bias) { + CHECK_DIM(1, A_log); + CHECK_DIM(2, a); + CHECK_DIM(2, b); + CHECK_DIM(1, dt_bias); + CHECK_CONTIGUOUS(a); + CHECK_EQ(A_log.size(0), a.size(1)); + CHECK_EQ(A_log.size(0), dt_bias.size(0)); + int batch = a.size(0); + int num_heads = a.size(1); + CHECK_EQ(b.size(0), batch); + CHECK_EQ(b.size(1), num_heads); + at::Tensor out = at::empty({1, batch, num_heads}, a.options().dtype(at::kFloat)); + at::Tensor beta = at::empty({1, batch, num_heads}, b.options()); + CPU_DISPATCH_REDUCED_FLOATING_TYPES_EXT(a.scalar_type(), A_log.scalar_type(), "fused_gdn_gating_kernel", [&] { + fused_gdn_gating_kernel_impl( + A_log.data_ptr(), + a.data_ptr(), + b.data_ptr(), + dt_bias.data_ptr(), + out.data_ptr(), + beta.data_ptr(), + batch, + num_heads); + }); + return std::make_tuple(out, beta); +} diff --git a/csrc/cpu/sgl-kernels/gemm.cpp b/csrc/cpu/sgl-kernels/gemm.cpp index 65c56943c5..38e6d9f4ce 100644 --- a/csrc/cpu/sgl-kernels/gemm.cpp +++ b/csrc/cpu/sgl-kernels/gemm.cpp @@ -1,11 +1,12 @@ // Adapted from // https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu -#include "common.h" -#include "vec.h" +// clang-format off + #include "gemm.h" -// clang-format off +#include "common.h" +#include "vec.h" namespace { @@ -26,13 +27,13 @@ inline void s8s8_compensation(int8_t* __restrict__ packed, int K) { const __m512i off = _mm512_set1_epi8(static_cast(0x80)); for (int k = 0; k < K / 4; ++k) { for (int col = 0; col < COLS; ++col) { - __m512i vb = _mm512_loadu_si512((const __m512i *)(packed + k * BLOCK_N * 4 + col * 64)); + __m512i vb = _mm512_loadu_si512((const __m512i*)(packed + k * BLOCK_N * 4 + col * 64)); vcomp[col] = _mm512_dpbusd_epi32(vcomp[col], off, vb); } } for (int col = 0; col < COLS; ++col) { - _mm512_storeu_si512((__m512i *)(packed + offset + col * 64), vcomp[col]); + _mm512_storeu_si512((__m512i*)(packed + offset + col * 64), vcomp[col]); } #else TORCH_CHECK(false, "s8s8_compensation not implemented!"); @@ -69,6 +70,43 @@ inline void pack_vnni(int8_t* __restrict__ packed, const int8_t* __restr s8s8_compensation(packed, K); } +// uint8_t: mxfp4 or int4 +// pack to vnni2 format as they are computed with bfloat16 +// +// from [N, K'/2, 2] to [K'/2, N, 2], view 2x int4 as unit8: +// from [N, K ] to [K, N ] where K = K'/2 +// +template <> +inline void pack_vnni(uint8_t* __restrict__ packed, const uint8_t* __restrict__ weight, int N, int K) { + constexpr int BLOCK_N = block_size_n(); + + uint8_t unpacked[2 * BLOCK_N]; + + // 32-way pack (align with BLOCK_N), faster for avx512 unpacking + // + // for a range of (64): + // {0, 1, 2, ..., 63} + // + // original format: + // { 1|0, 3|2, ..., 63|62} + // + // packed format: + // {32|0, 31|1, ..., 63|31} + // + for (int k = 0; k < K; ++k) { + // unpack first + for (int n = 0; n < N; ++n) { + uint8_t value = weight[n * K + k]; + unpacked[n * 2 + 0] = value & 0xF; // lower 4 bits + unpacked[n * 2 + 1] = value >> 4; // higher 4 bits + } + // re-pack to 32-way + for (int n = 0; n < N; ++n) { + packed[k * N + n] = (unpacked[n + BLOCK_N] << 4) | unpacked[n]; + } + } +} + template inline void copy_stub(scalar_t* __restrict__ out, const float* __restrict__ input, int64_t size) { using bVec = at::vec::Vectorized; @@ -76,7 +114,7 @@ inline void copy_stub(scalar_t* __restrict__ out, const float* __restrict__ inpu constexpr int kVecSize = bVec::size(); int64_t d; - #pragma GCC unroll 4 +#pragma GCC unroll 4 for (d = 0; d <= size - kVecSize; d += kVecSize) { fVec data0 = fVec::loadu(input + d); fVec data1 = fVec::loadu(input + d + fVec::size()); @@ -89,13 +127,34 @@ inline void copy_stub(scalar_t* __restrict__ out, const float* __restrict__ inpu } template -inline void copy_add_stub(scalar_t* __restrict__ out, const float* __restrict__ input, const float* __restrict__ bias, int64_t size) { +inline void copy_stub(float* __restrict__ out, const scalar_t* __restrict__ input, int64_t size) { using bVec = at::vec::Vectorized; using fVec = at::vec::Vectorized; constexpr int kVecSize = bVec::size(); int64_t d; - #pragma GCC unroll 4 +#pragma GCC unroll 4 + for (d = 0; d <= size - kVecSize; d += kVecSize) { + fVec data0, data1; + bVec b_vec = bVec::loadu(input + d); + std::tie(data0, data1) = at::vec::convert_to_float(b_vec); + data0.store(out + d); + data1.store(out + d + fVec::size()); + } + for (; d < size; ++d) { + out[d] = static_cast(input[d]); + } +} + +template +inline void copy_add_stub( + scalar_t* __restrict__ out, const float* __restrict__ input, const float* __restrict__ bias, int64_t size) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + + int64_t d; +#pragma GCC unroll 4 for (d = 0; d <= size - kVecSize; d += kVecSize) { fVec data0 = fVec::loadu(input + d) + fVec::loadu(bias + d); fVec data1 = fVec::loadu(input + d + fVec::size()) + fVec::loadu(bias + d + fVec::size()); @@ -107,11 +166,51 @@ inline void copy_add_stub(scalar_t* __restrict__ out, const float* __restrict__ } } +template +inline void scalar_sigmoid_and_mul( + scalar_t* __restrict__ out, + const float* __restrict__ input, + const float* __restrict__ bias, + const scalar_t* __restrict__ mul, + int SIZE) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + // scalar sigmoid + const fVec one = fVec(1.f); + fVec X; + if constexpr (has_bias) { + assert(bias != nullptr); + X = fVec(input[0] + bias[0]); + } else { + X = fVec(input[0]); + } + X = one / (one + X.neg().exp_u20()); + + // vec mul + constexpr int kVecSize = bVec::size(); + for (int d = 0; d < SIZE; d += kVecSize) { + bVec m_bvec = bVec::loadu(mul + d); + fVec m_fvec0, m_fvec1; + std::tie(m_fvec0, m_fvec1) = at::vec::convert_to_float(m_bvec); + m_fvec0 = m_fvec0 * X; + m_fvec1 = m_fvec1 * X; + + bVec out_vec = convert_from_float_ext(m_fvec0, m_fvec1); + out_vec.store(out + d); + } +} + template struct tinygemm_kernel_nn { static inline void apply( - const scalar_t* __restrict__ A, const scalar_t* __restrict__ B, scalar_t* __restrict__ C, - const float* __restrict__ bias, int64_t K, int64_t lda, int64_t ldb, int64_t ldc) { + const scalar_t* __restrict__ A, + const scalar_t* __restrict__ B, + scalar_t* __restrict__ C, + const float* __restrict__ bias, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { TORCH_CHECK(false, "tinygemm_kernel_nn: scalar path not implemented!"); } }; @@ -120,9 +219,14 @@ struct tinygemm_kernel_nn { template struct tinygemm_kernel_nn { static inline void apply( - const at::BFloat16* __restrict__ A, const at::BFloat16* __restrict__ B, at::BFloat16* __restrict__ C, - const float* __restrict__ bias, int64_t K, int64_t lda, int64_t ldb, int64_t ldc) { - + const at::BFloat16* __restrict__ A, + const at::BFloat16* __restrict__ B, + at::BFloat16* __restrict__ C, + const float* __restrict__ bias, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { constexpr int ROWS = BLOCK_M; constexpr int COLS = BLOCK_N / 16; @@ -145,7 +249,7 @@ struct tinygemm_kernel_nn { const int64_t K2 = K >> 1; const int64_t lda2 = lda >> 1; - const int64_t ldb2 = ldb; // ldb * 2 >> 1; + const int64_t ldb2 = ldb; // ldb * 2 >> 1; const float* a_ptr = reinterpret_cast(A); const float* b_ptr = reinterpret_cast(B); @@ -180,9 +284,7 @@ struct tinygemm_kernel_nn { (__m512i)(_mm512_cvtne2ps_pbh(vc[row * COLS + col + 1], vc[row * COLS + col]))); } } else { - _mm256_storeu_si256( - reinterpret_cast<__m256i*>(C + row * ldc + col * 16), - (__m256i)(_mm512_cvtneps_pbh(vc[i]))); + _mm256_storeu_si256(reinterpret_cast<__m256i*>(C + row * ldc + col * 16), (__m256i)(_mm512_cvtneps_pbh(vc[i]))); } }; Unroll{}(storec); @@ -190,22 +292,33 @@ struct tinygemm_kernel_nn { }; #endif -#define LAUNCH_TINYGEMM_KERNEL_NN(MB_SIZE, NB_SIZE) \ - tinygemm_kernel_nn::apply( \ - A + mb_start * lda, B + nb_start * 2, C + mb_start * ldc + nb_start, \ - has_bias ? bias + nb_start : nullptr, K, lda, ldb, ldc); +#define LAUNCH_TINYGEMM_KERNEL_NN(MB_SIZE, NB_SIZE) \ + tinygemm_kernel_nn::apply( \ + A + mb_start * lda, \ + B + nb_start * 2, \ + C + mb_start * ldc + nb_start, \ + has_bias ? bias + nb_start : nullptr, \ + K, \ + lda, \ + ldb, \ + ldc); template struct brgemm { static inline void apply( - const scalar_t* __restrict__ A, const scalar_t* __restrict__ B, scalar_t* __restrict__ C, - float* __restrict__ Ctmp, const float* __restrict__ bias, - int64_t M, int64_t N, int64_t K, int64_t lda, int64_t ldb, int64_t ldc) { - + const scalar_t* __restrict__ A, + const scalar_t* __restrict__ B, + scalar_t* __restrict__ C, + float* __restrict__ Ctmp, + const float* __restrict__ bias, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { constexpr int BLOCK_N = block_size_n(); - at::native::cpublas::brgemm( - M, N, K, lda, ldb, BLOCK_N, /* add_C */false, - A, B, Ctmp); + at::native::cpublas::brgemm(M, N, K, lda, ldb, BLOCK_N, /* add_C */ false, A, B, Ctmp); // copy from Ctmp to C for (int64_t m = 0; m < M; ++m) { @@ -216,6 +329,21 @@ struct brgemm { } } } + static inline void apply( + const float* __restrict__ A, + const float* __restrict__ B, + scalar_t* __restrict__ C, + float* __restrict__ Ctmp, + const float* __restrict__ bias, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { + constexpr int BLOCK_N = block_size_n(); + at::native::cpublas::brgemm(M, N, K, lda, ldb, BLOCK_N, /* add_C */ false, A, B, Ctmp); + } }; template @@ -232,15 +360,12 @@ void tinygemm_kernel( int64_t ldb, int64_t ldc, bool brg) { - if (brg) { - brgemm::apply( - A, B, C, Ctmp, bias, - M, N, K, lda, ldb, ldc); + brgemm::apply(A, B, C, Ctmp, bias, M, N, K, lda, ldb, ldc); return; } - // pattern: 1-4-16 + // pattern: 1-4-16, N = 16, 32, 48, 64 constexpr int64_t BLOCK_M = 4; constexpr int64_t BLOCK_N = 64; const int64_t MB = div_up(M, BLOCK_M); @@ -252,25 +377,88 @@ void tinygemm_kernel( int64_t nb_start = nb * BLOCK_N; int64_t nb_size = std::min(BLOCK_N, N - nb_start); - switch(mb_size << 4 | nb_size >> 4) { + switch (mb_size << 4 | nb_size >> 4) { // mb_size = 1 - case 0x12: LAUNCH_TINYGEMM_KERNEL_NN(1, 32); break; - case 0x14: LAUNCH_TINYGEMM_KERNEL_NN(1, 64); break; + case 0x11: + LAUNCH_TINYGEMM_KERNEL_NN(1, 16); + break; + case 0x12: + LAUNCH_TINYGEMM_KERNEL_NN(1, 32); + break; + case 0x13: + LAUNCH_TINYGEMM_KERNEL_NN(1, 48); + break; + case 0x14: + LAUNCH_TINYGEMM_KERNEL_NN(1, 64); + break; // mb_size = 2 - case 0x22: LAUNCH_TINYGEMM_KERNEL_NN(2, 32); break; - case 0x24: LAUNCH_TINYGEMM_KERNEL_NN(2, 64); break; + case 0x21: + LAUNCH_TINYGEMM_KERNEL_NN(2, 16); + break; + case 0x22: + LAUNCH_TINYGEMM_KERNEL_NN(2, 32); + break; + case 0x23: + LAUNCH_TINYGEMM_KERNEL_NN(2, 48); + break; + case 0x24: + LAUNCH_TINYGEMM_KERNEL_NN(2, 64); + break; // mb_size = 3 - case 0x32: LAUNCH_TINYGEMM_KERNEL_NN(3, 32); break; - case 0x34: LAUNCH_TINYGEMM_KERNEL_NN(3, 64); break; + case 0x31: + LAUNCH_TINYGEMM_KERNEL_NN(3, 16); + break; + case 0x32: + LAUNCH_TINYGEMM_KERNEL_NN(3, 32); + break; + case 0x33: + LAUNCH_TINYGEMM_KERNEL_NN(3, 48); + break; + case 0x34: + LAUNCH_TINYGEMM_KERNEL_NN(3, 64); + break; // mb_size = 4 - case 0x42: LAUNCH_TINYGEMM_KERNEL_NN(4, 32); break; - case 0x44: LAUNCH_TINYGEMM_KERNEL_NN(4, 64); break; - default: TORCH_CHECK(false, "Unexpected block size, ", mb_size, "x", nb_size); + case 0x41: + LAUNCH_TINYGEMM_KERNEL_NN(4, 16); + break; + case 0x42: + LAUNCH_TINYGEMM_KERNEL_NN(4, 32); + break; + case 0x43: + LAUNCH_TINYGEMM_KERNEL_NN(4, 48); + break; + case 0x44: + LAUNCH_TINYGEMM_KERNEL_NN(4, 64); + break; + default: + TORCH_CHECK(false, "Unexpected block size, ", mb_size, " x ", nb_size); } } } } +template +void tinygemm_kernel( + const float* __restrict__ A, + const float* __restrict__ B, + scalar_t* __restrict__ C, + float* __restrict__ Ctmp, + const float* __restrict__ bias, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc, + bool brg) { + TORCH_CHECK(brg, "Expected to use fp32 brgemm for small N GEMM"); + if (brg) { + brgemm::apply(A, B, C, Ctmp, bias, M, N, K, lda, ldb, ldc); + return; + } + // TODO : add intrinsic path +} + template void weight_packed_linear_kernel_impl( scalar_t* __restrict__ out, @@ -282,29 +470,20 @@ void weight_packed_linear_kernel_impl( int64_t K, int64_t mat1_strideM, int64_t out_strideM) { - constexpr int64_t BLOCK_M = block_size_m(); constexpr int64_t BLOCK_N = block_size_n(); const int64_t MB = div_up(M, BLOCK_M); const int64_t NB = div_up(N, BLOCK_N); - // use avx512-bf16 when a) M is small; b) dtype is bfloat16, otherwise use amx - const bool use_brgemm = (M > 4) || (!std::is_same_v); - - // l2 cache block for n - int64_t cache_blocks_nb = get_cache_blocks(BLOCK_N, K); + const bool use_brgemm = can_use_brgemm(M); // parallel on [MB, NB] AT_DISPATCH_BOOL(bias != nullptr, has_bias, [&] { - parallel_2d(MB, NB, [&](int64_t begin_mb, int64_t end_mb, int64_t begin_nb, int64_t end_nb) { - + parallel_2d(MB, NB, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { // for brgemm, use float32 for accumulate alignas(64) float Ctmp[BLOCK_M * BLOCK_N]; - for (int64_t nbb = begin_nb; nbb < end_nb; nbb += cache_blocks_nb) { - for (int64_t mb = begin_mb; mb < end_mb; ++mb) { - for (int64_t nb = nbb; nb < std::min(nbb + cache_blocks_nb, end_nb); ++nb) { - + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * K, [&](int64_t mb, int64_t nb, int64_t nb_offset) { int64_t mb_start = mb * BLOCK_M; int64_t mb_size = std::min(M - mb_start, BLOCK_M); int64_t nb_start = nb * BLOCK_N; @@ -323,7 +502,7 @@ void weight_packed_linear_kernel_impl( /* ldb */ nb_size, /* ldc */ out_strideM, /* brg */ use_brgemm); - }}} + }); if (use_brgemm) { at::native::cpublas::brgemm_release(); @@ -332,20 +511,113 @@ void weight_packed_linear_kernel_impl( }); } -} // anonymous namespace +template +void weight_packed_linear_kernel_impl( + scalar_t* __restrict__ out, + const scalar_t* __restrict__ mat1, + const float* __restrict__ mat2, + const float* __restrict__ bias, + const scalar_t* __restrict__ post_mul_mat, + int64_t M, + int64_t N, + int64_t K, + int64_t mat1_strideM, + int64_t out_strideM) { + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n(); + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N); + + const bool use_brgemm = true; // TODO: add intrinsic path + // parallel on [MB, NB] + AT_DISPATCH_BOOL(bias != nullptr, has_bias, [&] { + parallel_2d(MB, NB, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + // for brgemm, use float32 for accumulate + alignas(64) float Atmp[BLOCK_M * K]; + alignas(64) float Ctmp[BLOCK_M * BLOCK_N]; + + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * K, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + int64_t mb_start = mb * BLOCK_M; + int64_t mb_size = std::min(M - mb_start, BLOCK_M); + int64_t nb_start = nb * BLOCK_N; + int64_t nb_size = std::min(N - nb_start, BLOCK_N); + for (int64_t m = 0; m < mb_size; ++m) { + copy_stub(Atmp + m * K, mat1 + mb_start * mat1_strideM + m * K, K); + } + tinygemm_kernel( + /* A */ Atmp, + /* B */ mat2 + nb_start * K /* nb * BLOCK_N * K */, + /* C */ out + mb_start * out_strideM + nb_start, + /* Ctmp*/ Ctmp, + /* bias*/ bias + nb_start, + /* M */ mb_size, + /* N */ nb_size, + /* K */ K, + /* lda */ mat1_strideM, + /* ldb */ nb_size, + /* ldc */ out_strideM, + /* brg */ use_brgemm); + + if (post_mul_mat != nullptr) { + for (int64_t m = 0; m < mb_size; ++m) { + scalar_sigmoid_and_mul( + out + mb_start * out_strideM + nb_start + m * out_strideM, + Ctmp + m * BLOCK_N, + bias + nb_start, + post_mul_mat + mb_start * out_strideM + m * out_strideM, + out_strideM); + } + } else { + for (int64_t m = 0; m < mb_size; ++m) { + if constexpr (has_bias) { + copy_add_stub( + out + mb_start * out_strideM + nb_start + m * out_strideM, Ctmp + m * BLOCK_N, bias + nb_start, N); + } else { + copy_stub(out + mb_start * out_strideM + nb_start + m * out_strideM, Ctmp + m * BLOCK_N, N); + } + } + } + }); + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); + }); +} + +} // anonymous namespace // tinygemm interface template -void tinygemm_kernel(const scalar_t* __restrict__ A, const scalar_t* __restrict__ B, scalar_t* __restrict__ C, - float* __restrict__ Ctmp, int64_t M, int64_t N, int64_t K, int64_t lda, int64_t ldb, int64_t ldc, bool brg) { +void tinygemm_kernel( + const scalar_t* __restrict__ A, + const scalar_t* __restrict__ B, + scalar_t* __restrict__ C, + float* __restrict__ Ctmp, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc, + bool brg) { tinygemm_kernel(A, B, C, Ctmp, nullptr, M, N, K, lda, ldb, ldc, brg); } -#define INSTANTIATE_TINYGEMM_TEMPLATE(TYPE) \ - template void tinygemm_kernel( \ - const TYPE* __restrict__ A, const TYPE* __restrict__ B, TYPE* __restrict__ C, \ - float* __restrict__ Ctmp, int64_t M, int64_t N, int64_t K, int64_t lda, \ - int64_t ldb, int64_t ldc, bool brg) +#define INSTANTIATE_TINYGEMM_TEMPLATE(TYPE) \ + template void tinygemm_kernel( \ + const TYPE* __restrict__ A, \ + const TYPE* __restrict__ B, \ + TYPE* __restrict__ C, \ + float* __restrict__ Ctmp, \ + int64_t M, \ + int64_t N, \ + int64_t K, \ + int64_t lda, \ + int64_t ldb, \ + int64_t ldc, \ + bool brg) INSTANTIATE_TINYGEMM_TEMPLATE(at::BFloat16); INSTANTIATE_TINYGEMM_TEMPLATE(at::Half); @@ -359,14 +631,23 @@ at::Tensor convert_weight_packed(at::Tensor& weight) { const int64_t ndim = weight.ndimension(); TORCH_CHECK(ndim == 2 || ndim == 3, "expect weight to be 2d or 3d, got ", ndim, "d tensor."); + + if (ndim == 2 && weight.size(0) < TILE_N) { + // for 2D weight and small OC shape, we use fma linear path, which needs transpose not pack + return weight.to(at::kFloat).t().contiguous(); + } + const auto st = weight.scalar_type(); const int64_t E = ndim == 3 ? weight.size(0) : 1; const int64_t OC = ndim == 3 ? weight.size(1) : weight.size(0); const int64_t IC = ndim == 3 ? weight.size(2) : weight.size(1); + // mxfp4 or int4 are packed with uint8 + const int64_t actual_IC = st == at::kByte ? IC * 2 : IC; + // we handle 2 TILE_N at a time. TORCH_CHECK(OC % TILE_N == 0, "invalid weight out features ", OC); - TORCH_CHECK(IC % TILE_K == 0, "invalid weight input features ", IC); + TORCH_CHECK(actual_IC % TILE_K == 0, "invalid weight input features ", actual_IC); constexpr int64_t BLOCK_N = block_size_n(); const int64_t NB = div_up(OC, BLOCK_N); @@ -375,12 +656,14 @@ at::Tensor convert_weight_packed(at::Tensor& weight) { auto packed_weight = at::empty({}, weight.options()); const int64_t stride = OC * IC; - TORCH_CHECK(st == at::kBFloat16 || st == at::kHalf || st == at::kChar || st == at::kFloat8_e4m3fn, - "expect weight to be bfloat16, float16, int8 or fp8_e4m3."); + // Note: for `kByte` (uint8), it represents either `mxfp4` or `int4`. + TORCH_CHECK( + st == at::kBFloat16 || st == at::kHalf || st == at::kChar || st == at::kFloat8_e4m3fn || st == at::kByte, + "expect weight to be bfloat16, float16, int8, fp8_e4m3 or uint8(mxfp4 or int4)."); CPU_DISPATCH_PACKED_TYPES(st, [&] { // adjust most inner dimension size - const int packed_row_size = get_row_size(IC); + const int packed_row_size = get_row_size(actual_IC); auto sizes = weight.sizes().vec(); sizes[ndim - 1] = packed_row_size; packed_weight.resize_(sizes); @@ -399,10 +682,7 @@ at::Tensor convert_weight_packed(at::Tensor& weight) { int64_t n = nb * BLOCK_N; int64_t n_size = std::min(BLOCK_N, OC - n); pack_vnni( - packed_data + e * OC * packed_row_size + n * packed_row_size, - w_data + e * stride + n * IC, - n_size, - IC); + packed_data + e * OC * packed_row_size + n * packed_row_size, w_data + e * stride + n * IC, n_size, IC); // move to the next index data_index_step(e, E, nb, NB); @@ -412,33 +692,71 @@ at::Tensor convert_weight_packed(at::Tensor& weight) { return packed_weight; } +at::Tensor convert_scale_packed(at::Tensor& scale) { + CHECK_INPUT(scale); + + const int64_t ndim = scale.ndimension(); + TORCH_CHECK(ndim == 2 || ndim == 3, "expect scale to be 2d or 3d, got ", ndim, "d tensor."); + const auto st = scale.scalar_type(); + const int64_t E = ndim == 3 ? scale.size(0) : 1; + const int64_t N = ndim == 3 ? scale.size(1) : scale.size(0); + // number of groups, e.g. K/32 + const int64_t G = ndim == 3 ? scale.size(2) : scale.size(1); + + constexpr int64_t BLOCK_N = block_size_n(); + TORCH_CHECK(N % BLOCK_N == 0, "invalid weight out features ", N); + const int64_t NB = N / BLOCK_N; + + auto packed_scale = at::empty_like(scale); + TORCH_CHECK(st == at::kByte, "expect scale to be uint8."); + + const uint8_t* s_data = scale.data_ptr(); + uint8_t* packed_data = packed_scale.data_ptr(); + + // parallel on src {E, NB, BLOCK_N, G}, dst {E, NB, G, BLOCK_N} + at::parallel_for(0, E * NB * BLOCK_N * G, 0, [&](int64_t begin, int64_t end) { + int64_t e{0}, nb{0}, n{0}, g{0}; + data_index_init(begin, e, E, nb, NB, n, BLOCK_N, g, G); + + for (int64_t i = begin; i < end; ++i) { + packed_data[e * N * G + nb * G * BLOCK_N + g * BLOCK_N + n] = s_data[i]; + // move to the next index + data_index_step(e, E, nb, NB, n, BLOCK_N, g, G); + } + }); + return packed_scale; +} + // mat1 : [M, K] -// mat2 : [N, K] +// mat2 : [N, K] ([K, N] if use_fma_gemm) // bias : [N] // out : [M, N] // -at::Tensor weight_packed_linear(at::Tensor& mat1, at::Tensor& mat2, - const std::optional& bias, bool is_vnni) { - RECORD_FUNCTION( - "sgl-kernel::weight_packed_linear", std::vector({mat1, mat2, bias})); - +at::Tensor +weight_packed_linear(at::Tensor& mat1, at::Tensor& mat2, const std::optional& bias, bool is_vnni) { auto packed_w = is_vnni ? mat2 : convert_weight_packed(mat2); + bool use_fma_gemm = false; + if (packed_w.scalar_type() == at::kFloat) { + use_fma_gemm = true; + } + + int64_t M = mat1.size(0); + int64_t K = mat1.size(1); + int64_t N = use_fma_gemm ? mat2.size(1) : mat2.size(0); CHECK_LAST_DIM_CONTIGUOUS_INPUT(mat1); CHECK_INPUT(mat2); - - int64_t M = mat1.size(0); - int64_t N = mat2.size(0); - int64_t K = mat2.size(1); - CHECK_EQ(mat1.size(1), K); CHECK_DIM(2, mat1); CHECK_DIM(2, mat2); + if (!use_fma_gemm) { + CHECK_EQ(mat1.size(1), K); + } + auto dispatch_type = mat1.scalar_type(); auto out = at::empty({M, N}, mat1.options()); - // strides - int64_t mat1_strideM = mat1.stride(0); int64_t out_strideM = out.stride(0); + int64_t mat1_strideM = mat1.stride(0); const bool has_bias = bias.has_value(); const float* bias_data = nullptr; @@ -447,12 +765,83 @@ at::Tensor weight_packed_linear(at::Tensor& mat1, at::Tensor& mat2, bias_data = bias.value().data_ptr(); } - AT_DISPATCH_REDUCED_FLOATING_TYPES(mat1.scalar_type(), "weight_packed_linear_kernel_impl", [&] { + AT_DISPATCH_REDUCED_FLOATING_TYPES(dispatch_type, "weight_packed_linear_kernel_impl", [&] { + if (use_fma_gemm) { + weight_packed_linear_kernel_impl( + out.data_ptr(), + mat1.data_ptr(), + packed_w.data_ptr(), + bias_data, + nullptr, + M, + N, + K, + mat1_strideM, + out_strideM); + } else { + weight_packed_linear_kernel_impl( + out.data_ptr(), + mat1.data_ptr(), + packed_w.data_ptr(), + bias_data, + M, + N, + K, + mat1_strideM, + out_strideM); + } + }); + + return out; +} + +// mat1 : [M, K] +// mat2 : [K, 1] +// post_mul_mat : [M, K] +// bias : [N] +// out : [M, N] +// +at::Tensor fused_linear_sigmoid_mul( + at::Tensor& mat1, + at::Tensor& mat2, + const std::optional& bias, + bool is_vnni, + const at::Tensor& post_mul_mat) { + auto packed_w = is_vnni ? mat2 : convert_weight_packed(mat2); + TORCH_CHECK(packed_w.scalar_type() == at::kFloat, "fused_linear_sigmoid_mul requires packed float weight") + + int64_t M = mat1.size(0); + int64_t K = mat1.size(1); + int64_t N = mat2.size(1); + + CHECK_LAST_DIM_CONTIGUOUS_INPUT(mat1); + CHECK_INPUT(mat2); + CHECK_DIM(2, mat1); + CHECK_DIM(2, mat2); + + int64_t out_strideM = post_mul_mat.size(1); + int64_t mat1_strideM = mat1.stride(0); + auto dispatch_type = mat1.scalar_type(); + auto out = at::empty({M, out_strideM}, mat1.options()); + + TORCH_CHECK( + N == 1 && out_strideM % 32 == 0, + "post_mul_mat tensor size(1) should be 32 dividable, and the mat2 OC=1 (Mx1 as linear output shape)") + + const bool has_bias = bias.has_value(); + const float* bias_data = nullptr; + if (has_bias) { + CHECK_EQ(bias.value().size(0), N); + bias_data = bias.value().data_ptr(); + } + + AT_DISPATCH_REDUCED_FLOATING_TYPES(dispatch_type, "fused_linear_sigmoid_mul", [&] { weight_packed_linear_kernel_impl( out.data_ptr(), mat1.data_ptr(), - packed_w.data_ptr(), + packed_w.data_ptr(), bias_data, + post_mul_mat.data_ptr(), M, N, K, diff --git a/csrc/cpu/sgl-kernels/gemm.h b/csrc/cpu/sgl-kernels/gemm.h index aa78c8807f..b43be94ffd 100644 --- a/csrc/cpu/sgl-kernels/gemm.h +++ b/csrc/cpu/sgl-kernels/gemm.h @@ -1,29 +1,55 @@ -#pragma once - -#include +// Adapted from +// https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu // clang-format off +#pragma once +#include + +#include "common.h" + // amx-bf16 #define TILE_M 16 #define TILE_N 16 #define TILE_K 32 // block size for AMX gemm -constexpr int block_size_m() { return 2 * TILE_M; } -constexpr int block_size_n() { return 2 * TILE_N; } +constexpr int block_size_m() { + return 2 * TILE_M; +} +constexpr int block_size_n() { + return 2 * TILE_N; +} // define threshold using brgemm (intel AMX) -template inline bool can_use_brgemm(int M); -template <> inline bool can_use_brgemm(int M) { return M > 4; } -template <> inline bool can_use_brgemm(int M) { return true; } -template <> inline bool can_use_brgemm(int M) { return M > 4; } -template <> inline bool can_use_brgemm(int M) { return M > 4; } -template <> inline bool can_use_brgemm(int M) { return M > 4; } -template <> inline bool can_use_brgemm(int M) { return M > 4; } +template +inline bool can_use_brgemm(int M); +template <> +inline bool can_use_brgemm(int M) { + return M > 4; +} +template <> +inline bool can_use_brgemm(int M) { + return true; +} +// this requires PyTorch 2.7 or above +template <> +inline bool can_use_brgemm(int M) { + return M > 4; +} + +template <> +inline bool can_use_brgemm(int M) { + return M > 4; +} + +template <> +inline bool can_use_brgemm(int M) { + return M > 4; +} // work around compiler internal error -#define BLOCK_K 128 // 4 * TILE_K +#define BLOCK_K 128 // 4 * TILE_K // adjust leading dimension size for K template @@ -36,18 +62,44 @@ inline int64_t get_row_size(int64_t K) { return K + sizeof(int32_t); } +// uint8: mxfp4 or int4 +template <> +inline int64_t get_row_size(int64_t K) { + return K >> 1; +} + inline int64_t get_row_size(int64_t K, bool use_int8_w8a8) { return use_int8_w8a8 ? K + sizeof(int32_t) : K; } +enum class CPUQuantMethod : int64_t { BF16 = 0, INT8_W8A8 = 1, FP8_W8A16 = 2, INT4_W4A8 = 3 }; + +constexpr bool operator==(CPUQuantMethod a, int64_t b) { + return static_cast(a) == b; +} + +constexpr bool operator==(int64_t a, CPUQuantMethod b) { + return a == static_cast(b); +} + +enum class CPUQuantAlgo : int64_t { AWQ = 0, GPTQ = 1 }; + +constexpr bool operator==(CPUQuantAlgo a, int64_t b) { + return static_cast(a) == b; +} + +constexpr bool operator==(int64_t a, CPUQuantAlgo b) { + return a == static_cast(b); +} + inline int64_t get_4bit_block_k_size(int64_t group_size) { return group_size > 128 ? 128 : group_size; } -// pack weight into vnni format +// pack weight to vnni format at::Tensor convert_weight_packed(at::Tensor& weight); -// pack weight to vnni format for int4 (adapted from sglang) +// pack weight to vnni format for int4 std::tuple convert_weight_packed_scale_zp(at::Tensor qweight, at::Tensor qzeros, at::Tensor scales); @@ -105,35 +157,6 @@ void fused_experts_fp8_kernel_impl( int64_t topk, int64_t num_tokens_post_pad); -// moe implementations for int4 w4a16 -template -void fused_experts_int4_w4a16_kernel_impl( - scalar_t* __restrict__ output, - scalar_t* __restrict__ ic0, - scalar_t* __restrict__ ic1, - scalar_t* __restrict__ ic2, - scalar_t* __restrict__ A_tmp, - scalar_t* __restrict__ B_tmp, - float* __restrict__ C_tmp, - const scalar_t* __restrict__ input, - const at::quint4x2* __restrict__ packed_w1, - const at::quint4x2* __restrict__ packed_w2, - const uint8_t* __restrict__ w1z, - const uint8_t* __restrict__ w2z, - const scalar_t* __restrict__ w1s, - const scalar_t* __restrict__ w2s, - int group_size, - const float* __restrict__ topk_weights, - const int32_t* __restrict__ sorted_ids, - const int32_t* __restrict__ expert_ids, - const int32_t* __restrict__ offsets, - int64_t M, - int64_t N, - int64_t K, - int64_t E, - int64_t topk, - int64_t num_tokens_post_pad); - // shared expert implementation for int8 w8a8 template void shared_expert_int8_kernel_impl( @@ -153,6 +176,37 @@ void shared_expert_int8_kernel_impl( int64_t N, int64_t K); +template +void fused_experts_int4_w4a8_kernel_impl( + scalar_t* __restrict__ output, + scalar_t* __restrict__ ic0, + scalar_t* __restrict__ ic1, + scalar_t* __restrict__ ic2, + uint8_t* __restrict__ A_tmp, + uint8_t* __restrict__ Aq_tmp, + float* __restrict__ As_tmp, + int32_t* __restrict__ Azp_tmp, + float* __restrict__ C_tmp, + int8_t* __restrict__ dqB_tmp, + const scalar_t* __restrict__ input, + const uint8_t* __restrict__ packed_w1, + const uint8_t* __restrict__ packed_w2, + const int8_t* __restrict__ w1z, + const int8_t* __restrict__ w2z, + const float* __restrict__ w1s, + const float* __restrict__ w2s, + int group_size, + const float* __restrict__ topk_weights, + const int32_t* __restrict__ sorted_ids, + const int32_t* __restrict__ expert_ids, + const int32_t* __restrict__ offsets, + int64_t M, + int64_t N, + int64_t K, + int64_t E, + int64_t topk, + int64_t num_tokens_post_pad); + template void shared_expert_fp8_kernel_impl( scalar_t* __restrict__ output, @@ -204,6 +258,7 @@ void tinygemm_kernel( int64_t ldc, bool brg); +// block quantization template void tinygemm_kernel( const scalar_t* __restrict__ A, @@ -219,33 +274,26 @@ void tinygemm_kernel( int64_t ldb, int64_t ldc, bool brg, - int64_t block_size_K); + int64_t block_size_K, + bool do_unpack = true); +// per tensor quantization template void tinygemm_kernel( const scalar_t* __restrict__ A, - const at::quint4x2* __restrict__ B, + const at::Float8_e4m3fn* __restrict__ B, scalar_t* __restrict__ C, - const uint8_t* __restrict__ Bz, - const scalar_t* __restrict__ Bs, scalar_t* __restrict__ Btmp, float* __restrict__ Ctmp, + float scale, int64_t M, int64_t N, int64_t K, - int group_size, int64_t lda, int64_t ldb, int64_t ldc, - int64_t strideBz, - int64_t strideBs, bool brg); -// int4 scaled GEMM (adapted from sglang) -at::Tensor int4_scaled_mm_cpu( - at::Tensor& x, at::Tensor& w, at::Tensor& w_zeros, at::Tensor& w_scales, std::optional bias); - -// int4 tinygemm kernel interface(adapted from sglang) template void tinygemm_kernel( scalar_t* C, @@ -266,34 +314,21 @@ void tinygemm_kernel( bool store_out, bool use_brgemm); -// TODO: debug print, remove me later -inline void print_16x32i(const __m512i x) { - int32_t a[16]; - _mm512_storeu_si512((__m512i *)a, x); - - for (int i = 0; i < 16; i++){ - std::cout << a[i] << " "; - } - std::cout << std::endl; -} - -inline void print_16x32(const __m512 x) { - float a[16]; - _mm512_storeu_ps((__m512 *)a, x); - - for (int i = 0; i < 16; i++){ - std::cout << a[i] << " "; - } - std::cout << std::endl; -} - - -inline void print_32x8u(const __m256i x) { - uint8_t a[32]; - _mm256_storeu_si256((__m256i *)a, x); - - for (int i = 0; i < 32; ++i) { - std::cout << int32_t(a[i]) << " "; - } - std::cout << std::endl; -} +// mxfp4 +template +void tinygemm_kernel( + const scalar_t* __restrict__ A, + const uint8_t* __restrict__ B, + scalar_t* __restrict__ C, + scalar_t* __restrict__ Btmp, + float* __restrict__ Ctmp, + const uint8_t* __restrict__ scale, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc, + bool brg, + int64_t block_size_K, + bool do_unpack = true); diff --git a/csrc/cpu/sgl-kernels/gemm_fp8.cpp b/csrc/cpu/sgl-kernels/gemm_fp8.cpp index 487a73d094..dfc3044685 100644 --- a/csrc/cpu/sgl-kernels/gemm_fp8.cpp +++ b/csrc/cpu/sgl-kernels/gemm_fp8.cpp @@ -1,14 +1,11 @@ // Adapted from // https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu -#include "common.h" -#include "vec.h" -#include "gemm.h" - // clang-format off -// we use 4x32 for BLOCK_M -#define BLOCK_SIZE_M_SCALE 4 +#include "common.h" +#include "gemm.h" +#include "vec.h" namespace { @@ -19,7 +16,7 @@ inline void copy_stub(scalar_t* __restrict__ out, const float* __restrict__ inpu constexpr int kVecSize = bVec::size(); int64_t d; - #pragma GCC unroll 4 +#pragma GCC unroll 4 for (d = 0; d <= size - kVecSize; d += kVecSize) { fVec data0 = fVec::loadu(input + d); fVec data1 = fVec::loadu(input + d + fVec::size()); @@ -32,13 +29,14 @@ inline void copy_stub(scalar_t* __restrict__ out, const float* __restrict__ inpu } template -inline void copy_add_stub(scalar_t* __restrict__ out, const float* __restrict__ input, const float* __restrict__ bias, int64_t size) { +inline void copy_add_stub( + scalar_t* __restrict__ out, const float* __restrict__ input, const float* __restrict__ bias, int64_t size) { using bVec = at::vec::Vectorized; using fVec = at::vec::Vectorized; constexpr int kVecSize = bVec::size(); int64_t d; - #pragma GCC unroll 4 +#pragma GCC unroll 4 for (d = 0; d <= size - kVecSize; d += kVecSize) { fVec data0 = fVec::loadu(input + d) + fVec::loadu(bias + d); fVec data1 = fVec::loadu(input + d + fVec::size()) + fVec::loadu(bias + d + fVec::size()); @@ -49,21 +47,41 @@ inline void copy_add_stub(scalar_t* __restrict__ out, const float* __restrict__ out[d] = static_cast(input[d] + bias[d]); } } +template +inline void copy_mul_stub(scalar_t* __restrict__ out, const float* __restrict__ input, int size, float scale) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + const fVec vscale = fVec(scale); + + int d; +#pragma GCC unroll 4 + for (d = 0; d <= size - kVecSize; d += kVecSize) { + fVec data0 = fVec::loadu(input + d) * vscale; + fVec data1 = fVec::loadu(input + d + fVec::size()) * vscale; + bVec out_vec = convert_from_float_ext(data0, data1); + out_vec.store(out + d); + } + for (; d < size; ++d) { + out[d] = static_cast(input[d] * scale); + } +} inline void unpack_B( at::BFloat16* __restrict__ Btmp, const at::Float8_e4m3fn* __restrict__ packed_B, - int N, - int K, - int ldb, - int ldb_tmp, + int64_t N, + int64_t K, + int64_t ldb, + int64_t ldb_tmp, float scale) { #if defined(CPU_CAPABILITY_AVX512) // [K/2, N, 2] - const int K2 = K >> 1; - const int ldb2 = ldb; // ldb * 2 >> 1; + const int64_t K2 = K >> 1; + const int64_t ldb2 = ldb; // ldb * 2 >> 1; const uint16_t* b_ptr = reinterpret_cast(packed_B); - const __m512 vd = _mm512_set1_ps(scale); + const __m512 vexp = _mm512_castsi512_ps(_mm512_set1_epi32(kFP8_BIAS)); + const __m512 vd = _mm512_mul_ps(_mm512_set1_ps(scale), vexp); constexpr int BLOCK_N = block_size_n(); static_assert(BLOCK_N == 32); @@ -72,7 +90,7 @@ inline void unpack_B( constexpr int PREFETCH_SIZE_K = 64; #pragma GCC unroll 4 - for (int k = 0; k < K2; ++k) { + for (int64_t k = 0; k < K2; ++k) { __m512i b8 = _mm512_loadu_si512(b_ptr + k * ldb2); if constexpr (PREFETCH_SIZE_K > 0) { _mm_prefetch(b_ptr + (k + PREFETCH_SIZE_K) * ldb2, _MM_HINT_T0); @@ -81,8 +99,8 @@ inline void unpack_B( __m256i b8_0 = _mm512_extracti32x8_epi32(b8, 0); __m256i b8_1 = _mm512_extracti32x8_epi32(b8, 1); - __m512bh bf16_0 = CVT_FP8_TO_BF16(b8_0); - __m512bh bf16_1 = CVT_FP8_TO_BF16(b8_1); + __m512bh bf16_0 = CVT_FP8_TO_BF16_EXT(b8_0); + __m512bh bf16_1 = CVT_FP8_TO_BF16_EXT(b8_1); // Apply scale __m512 f0_lo = CVT_BF16_TO_FP32(_mm512_extracti32x8_epi32((__m512i)bf16_0, 0)); @@ -106,26 +124,139 @@ inline void unpack_B( #endif } -template +inline void unpack_B( + at::BFloat16* __restrict__ Btmp, + const at::Float8_e4m3fn* __restrict__ packed_B, + int N, + int K, + int ldb, + int ldb_tmp) { +#if defined(CPU_CAPABILITY_AVX512) + // [K/2, N, 2] + const int K2 = K >> 1; + const int ldb2 = ldb; // ldb * 2 >> 1; + const uint16_t* b_ptr = reinterpret_cast(packed_B); + + // prefetch distance + constexpr int PREFETCH_SIZE_K = 64; +#pragma GCC unroll 4 + for (int k = 0; k < K2; ++k) { + __m512i b8 = _mm512_loadu_si512(b_ptr + k * ldb2); + if constexpr (PREFETCH_SIZE_K > 0) { + _mm_prefetch(b_ptr + (k + PREFETCH_SIZE_K) * ldb2, _MM_HINT_T0); + } + + __m256i b8_0 = _mm512_extracti32x8_epi32(b8, 0); + __m256i b8_1 = _mm512_extracti32x8_epi32(b8, 1); + + __m512bh bf16_0 = CVT_FP8_TO_BF16(b8_0); + __m512bh bf16_1 = CVT_FP8_TO_BF16(b8_1); + _mm512_storeu_si512(Btmp + k * ldb_tmp * 2 + 0, (__m512i)bf16_0); + _mm512_storeu_si512(Btmp + k * ldb_tmp * 2 + 32, (__m512i)bf16_1); + } +#else + TORCH_CHECK(false, "unpack_B: scalar path not implemented!"); +#endif +} + +// mxfp4 +inline void unpack_B( + at::BFloat16* __restrict__ Btmp, + const uint8_t* __restrict__ packed_B, + int64_t N, + int64_t K, + int64_t ldb, + int64_t ldb_tmp, + const uint8_t* __restrict__ scale) { +#if defined(CPU_CAPABILITY_AVX512) + // [K/2, N, 2] + const int64_t K2 = K >> 1; + const int64_t ldb2 = ldb; // ldb * 2 >> 1; + const uint8_t* b_ptr = reinterpret_cast(packed_B); // 2 * 4 bit = 8 bit + + constexpr int BLOCK_N = block_size_n(); + static_assert(BLOCK_N == 32); + + // prefetch distance + constexpr int PREFETCH_SIZE_K = 64; + + // exponent bias 127 + const __m512i off = _mm512_set1_epi16(0x7F); + + // load 32 bytes only once for each block + __m256i s8 = _mm256_loadu_si256(reinterpret_cast(scale)); + __m512i s16 = _mm512_slli_epi16(_mm512_sub_epi16(_mm512_cvtepu8_epi16(s8), off), 0x7); + + // holds Nx2(64) scales, interleaved as 2 belongs to K dimension + // e.g. vs0: { s0, s0, s1, s1, ..., s15, s15} + // vs1: {s16, s16, s17, s17, ..., s31, s31} + auto [vscale0, vscale1] = transpose_2x32_16bit(s16, s16); + +#pragma GCC unroll 4 + for (int64_t k = 0; k < K2; ++k) { + __m256i b4 = _mm256_loadu_si256(reinterpret_cast(b_ptr + k * ldb2)); + if constexpr (PREFETCH_SIZE_K > 0) { + _mm_prefetch(b_ptr + (k + PREFETCH_SIZE_K) * ldb2, _MM_HINT_T0); + } + auto [vb0, vb1] = CVT_MXFP4_TO_BF16(b4, vscale0, vscale1); + + _mm512_storeu_si512(Btmp + k * ldb_tmp * 2 + 0, (__m512i)vb0); + _mm512_storeu_si512(Btmp + k * ldb_tmp * 2 + 32, (__m512i)vb1); + } +#else + TORCH_CHECK(false, "unpack_B: scalar path not implemented!"); +#endif +} + +template struct tinygemm_kernel_nn { static inline void apply( - const scalar_t* __restrict__ A, const packed_t* __restrict__ B, scalar_t* __restrict__ C, - const float* __restrict__ bias, const float* __restrict__ scale, int K, int lda, int ldb, int ldc, int64_t block_size_K) { + const scalar_t* __restrict__ A, + const packed_t* __restrict__ B, + scalar_t* __restrict__ C, + const float* __restrict__ bias, + const param_t* __restrict__ scale, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc, + int64_t block_size_K) { TORCH_CHECK(false, "tinygemm_kernel_nn: scalar path not implemented!"); } }; +template +struct tinygemm_kernel_nn2 { + static inline void apply( + const scalar_t* __restrict__ A, + const at::Float8_e4m3fn* __restrict__ B, + scalar_t* __restrict__ C, + float scale, + int K, + int lda, + int ldb, + int ldc) { + TORCH_CHECK(false, "tinygemm_kernel_nn: scalar path not implemented!"); + } +}; #if defined(CPU_CAPABILITY_AVX512) template -struct tinygemm_kernel_nn { +struct tinygemm_kernel_nn { static inline void apply( - const at::BFloat16* __restrict__ A, const at::Float8_e4m3fn* __restrict__ B, at::BFloat16* __restrict__ C, - const float* __restrict__ bias, const float* __restrict__ scale, int K, int lda, int ldb, int ldc, int64_t block_size_K) { - + const at::BFloat16* __restrict__ A, + const at::Float8_e4m3fn* __restrict__ B, + at::BFloat16* __restrict__ C, + const float* __restrict__ bias, + const float* __restrict__ scale, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc, + int64_t block_size_K) { constexpr int ROWS = BLOCK_M; constexpr int COLS = BLOCK_N / 16; - const int KB = div_up(K, BLOCK_K); + const int64_t KB = div_up(K, (int64_t)BLOCK_K); // prefetch distance constexpr int PREFETCH_SIZE_K = 64; @@ -139,6 +270,8 @@ struct tinygemm_kernel_nn{}(loadc); - const int lda2 = lda >> 1; - const int ldb2 = ldb; // ldb * 2 >> 1; + const int64_t lda2 = lda >> 1; + const int64_t ldb2 = ldb; // ldb * 2 >> 1; const float* a_ptr = reinterpret_cast(A); const uint16_t* b_ptr = reinterpret_cast(B); @@ -170,34 +303,201 @@ struct tinygemm_kernel_nn 0) { _mm_prefetch(b_ptr + (k + PREFETCH_SIZE_K) * ldb2 + col * 16, _MM_HINT_T0); } - vb[col + 0] = CVT_FP8_TO_BF16(_mm512_extracti32x8_epi32(b8, 0)); - vb[col + 1] = CVT_FP8_TO_BF16(_mm512_extracti32x8_epi32(b8, 1)); + vb[col + 0] = CVT_FP8_TO_BF16_EXT(_mm512_extracti32x8_epi32(b8, 0)); + vb[col + 1] = CVT_FP8_TO_BF16_EXT(_mm512_extracti32x8_epi32(b8, 1)); } } vsum[i] = _mm512_dpbf16_ps(vsum[i], va, vb[col]); }; - constexpr int BLOCK_K2 = BLOCK_K >> 1; - for (int kb = 0; kb < KB; ++kb) { - int kb_start = kb * BLOCK_K2; - int kb_end = std::min(K, kb_start + BLOCK_K2); + constexpr int64_t BLOCK_K2 = BLOCK_K >> 1; + for (int64_t kb = 0; kb < KB; ++kb) { + int64_t kb_start = kb * BLOCK_K2; + int64_t kb_end = std::min(K >> 1, kb_start + BLOCK_K2); // 1. load scale vector vscale = _mm512_set1_ps(scale[kb]); + vscale = _mm512_mul_ps(vscale, vexp); if constexpr (PREFETCH_SIZE_KB > 0) { _mm_prefetch(scale + kb + PREFETCH_SIZE_KB, _MM_HINT_T0); } // 2. zero vsum for each block - Unroll{}([&](auto i) { - vsum[i] = _mm512_setzero_ps(); - }); + Unroll{}([&](auto i) { vsum[i] = _mm512_setzero_ps(); }); // 3. accumulate across each block for (int k = kb_start; k < kb_end; ++k) { Unroll{}(compute, k); } // 4. apply scale - Unroll{}([&](auto i) { - vc[i] = _mm512_fmadd_ps(vsum[i], vscale, vc[i]); - }); + Unroll{}([&](auto i) { vc[i] = _mm512_fmadd_ps(vsum[i], vscale, vc[i]); }); + } + + auto storec = [&](auto i) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + // for COLS = 2,4 use 512bit store + if constexpr (col % 2 == 0) { + _mm512_storeu_si512( + reinterpret_cast<__m512i*>((C + row * ldc + col * 16)), + (__m512i)(_mm512_cvtne2ps_pbh(vc[row * COLS + col + 1], vc[row * COLS + col]))); + } + }; + Unroll{}(storec); + } +}; + +template +struct tinygemm_kernel_nn2 { + static inline void apply( + const at::BFloat16* __restrict__ A, + const at::Float8_e4m3fn* __restrict__ B, + at::BFloat16* __restrict__ C, + float scale, + int K, + int lda, + int ldb, + int ldc) { + constexpr int ROWS = BLOCK_M; + constexpr int COLS = BLOCK_N / 16; + + // prefetch distance + constexpr int PREFETCH_SIZE_K = 64; + + __m512bh va; + __m512bh vb[COLS]; + __m512 vc[ROWS * COLS]; + + const __m512 vscale = _mm512_set1_ps(scale); + + auto loadc = [&](auto i) { vc[i] = _mm512_setzero_ps(); }; + Unroll{}(loadc); + + const int K2 = K >> 1; + const int lda2 = lda >> 1; + const int ldb2 = ldb; // ldb * 2 >> 1; + const float* a_ptr = reinterpret_cast(A); + const uint16_t* b_ptr = reinterpret_cast(B); + + auto compute = [&](auto i, int k) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + + if constexpr (col == 0) { + va = (__m512bh)(_mm512_set1_ps(a_ptr[row * lda2 + k])); + } + if constexpr (row == 0) { + if constexpr (col % 2 == 0) { + __m512i b8 = _mm512_loadu_si512(b_ptr + k * ldb2 + col * 16); + if constexpr (PREFETCH_SIZE_K > 0) { + _mm_prefetch(b_ptr + (k + PREFETCH_SIZE_K) * ldb2 + col * 16, _MM_HINT_T0); + } + vb[col + 0] = CVT_FP8_TO_BF16(_mm512_extracti32x8_epi32(b8, 0)); + vb[col + 1] = CVT_FP8_TO_BF16(_mm512_extracti32x8_epi32(b8, 1)); + } + } + vc[i] = _mm512_dpbf16_ps(vc[i], va, vb[col]); + }; + for (int k = 0; k < K2; ++k) { + Unroll{}(compute, k); + } + + auto storec = [&](auto i) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + // for COLS = 2, 4 use 512bit store + if constexpr (col % 2 == 0) { + __m512 vc0 = _mm512_mul_ps(vc[row * COLS + col + 0], vscale); + __m512 vc1 = _mm512_mul_ps(vc[row * COLS + col + 1], vscale); + _mm512_storeu_si512( + reinterpret_cast<__m512i*>((C + row * ldc + col * 16)), (__m512i)(_mm512_cvtne2ps_pbh(vc1, vc0))); + } + }; + Unroll{}(storec); + } +}; + +template +struct tinygemm_kernel_nn { + static inline void apply( + const at::BFloat16* __restrict__ A, + const uint8_t* __restrict__ B, + at::BFloat16* __restrict__ C, + const float* __restrict__ bias, + const uint8_t* __restrict__ scale, + int K, + int lda, + int ldb, + int ldc, + int64_t block_size_K) { + // mxfp4 supports only group size of 32 + // expect weight packed in 32-way, vnni2 format Nx2(64) + assert(block_size_K == 32); + assert(BLOCK_N == 32); + + constexpr int ROWS = BLOCK_M; + constexpr int COLS = BLOCK_N / 16; + + // prefetch distance + constexpr int PREFETCH_SIZE_K = 64; + constexpr int PREFETCH_SIZE_KB = 1; + + __m512bh va; + __m512bh vb[COLS]; + __m512 vc[ROWS * COLS]; + + // holds Nx2(64) scales, interleaved as 2 belongs to K dimension + // e.g. vs0: { s0, s0, s1, s1, ..., s15, s15} + // vs1: {s16, s16, s17, s17, ..., s31, s31} + __m512i vscale[COLS]; + + // exponent bias 127 + const __m512i off = _mm512_set1_epi16(0x7F); + + auto loadc = [&](auto i) { + constexpr int col = i % COLS; + if constexpr (has_bias) { + vc[i] = _mm512_loadu_ps(bias + col * 16); + } else { + vc[i] = _mm512_setzero_ps(); + } + }; + Unroll{}(loadc); + + const int64_t K2 = K >> 1; + const int64_t lda2 = lda >> 1; + const int64_t ldb2 = ldb; // ldb * 2 >> 1; + const float* a_ptr = reinterpret_cast(A); + const uint8_t* b_ptr = reinterpret_cast(B); + + auto compute = [&](auto i, int k) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + + if constexpr (col == 0) { + va = (__m512bh)(_mm512_set1_ps(a_ptr[row * lda2 + k])); + if constexpr (PREFETCH_SIZE_K > 0) { + _mm_prefetch(a_ptr + row * lda2 + k + PREFETCH_SIZE_K, _MM_HINT_T0); + } + } + if constexpr (row == 0) { + // load 32 * 2 (64) int4 at a time + if constexpr (col % 2 == 0) { + __m256i b4 = _mm256_loadu_si256(reinterpret_cast(b_ptr + k * ldb2 + col * 16)); + if constexpr (PREFETCH_SIZE_K > 0) { + _mm_prefetch(b_ptr + (k + PREFETCH_SIZE_K) * ldb2 + col * 16, _MM_HINT_T0); + } + std::tie(vb[col + 0], vb[col + 1]) = CVT_MXFP4_TO_BF16(b4, vscale[col + 0], vscale[col + 1]); + } + } + vc[i] = _mm512_dpbf16_ps(vc[i], va, vb[col]); + }; + + for (int64_t k = 0; k < K2; ++k) { + // update scales every 16x2 K + if ((k & 15) == 0) { + __m256i s8 = _mm256_loadu_si256(reinterpret_cast(scale + (k >> 4) * 32)); + __m512i s16 = _mm512_slli_epi16(_mm512_sub_epi16(_mm512_cvtepu8_epi16(s8), off), 0x7); + std::tie(vscale[0], vscale[1]) = transpose_2x32_16bit(s16, s16); + } + Unroll{}(compute, k); } auto storec = [&](auto i) { @@ -215,12 +515,24 @@ struct tinygemm_kernel_nn::apply( \ - A + mb_start * lda, B + nb_start * 2, C + mb_start * ldc + nb_start, \ - has_bias ? bias + nb_start : nullptr, scale, K, lda, ldb, ldc, block_size_K); +#define LAUNCH_TINYGEMM_KERNEL_NN(MB_SIZE, NB_SIZE) \ + tinygemm_kernel_nn::apply( \ + A + mb_start * lda, \ + B + nb_start * 2, \ + C + mb_start * ldc + nb_start, \ + has_bias ? bias + nb_start : nullptr, \ + scale, \ + K, \ + lda, \ + ldb, \ + ldc, \ + block_size_K); -template +#define LAUNCH_TINYGEMM_KERNEL_NN2(MB_SIZE, NB_SIZE) \ + tinygemm_kernel_nn2::apply( \ + A + mb_start * lda, B + nb_start * 2, C + mb_start * ldc + nb_start, scale, K, lda, ldb, ldc); + +template struct brgemm { static inline void apply( const scalar_t* __restrict__ A, @@ -229,19 +541,22 @@ struct brgemm { scalar_t* __restrict__ Btmp, float* __restrict__ Ctmp, const float* __restrict__ bias, - const float* __restrict__ scale, + const param_t* __restrict__ scale, int M, int N, int K, int lda, int ldb, - int ldc) { + int ldc, + bool do_unpack = true) { TORCH_CHECK(false, "struct brgemm: primary template not implemented!"); } }; +template +struct brgemm2 {}; template -struct brgemm { +struct brgemm { static inline void apply( const at::BFloat16* __restrict__ A, const at::Float8_e4m3fn* __restrict__ B, @@ -255,22 +570,23 @@ struct brgemm { int K, int lda, int ldb, - int ldc) { - + int ldc, + bool do_unpack = true) { constexpr int BLOCK_N = block_size_n(); // [K, BLOCK_N] -> [K / 2, BLOCK_N * 2] const int ldb_tmp = BLOCK_N; - for (int k = 0; k < K; k += BLOCK_K) { - int kb_size = std::min(BLOCK_K, K - k); + if (do_unpack) { + for (int k = 0; k < K; k += BLOCK_K) { + int kb_size = std::min(BLOCK_K, K - k); - int idx = k >> 7; // k / BLOCK_K where BLOCK_K = 128 - unpack_B(Btmp + k * ldb_tmp, B + k * ldb, N, kb_size, ldb, ldb_tmp, scale[idx]); + int idx = k >> 7; // k / BLOCK_K where BLOCK_K = 128 + unpack_B(Btmp + k * ldb_tmp, B + k * ldb, N, kb_size, ldb, ldb_tmp, scale[idx]); + } } - at::native::cpublas::brgemm( - M, N, K, lda, ldb_tmp, BLOCK_N, /* add_C */ false, A, Btmp, Ctmp); + at::native::cpublas::brgemm(M, N, K, lda, ldb_tmp, BLOCK_N, /* add_C */ false, A, Btmp, Ctmp); // copy from Ctmp to C for (int m = 0; m < M; ++m) { @@ -283,14 +599,92 @@ struct brgemm { } }; -template +template <> +struct brgemm2 { + static inline void apply( + const at::BFloat16* __restrict__ A, + const at::Float8_e4m3fn* __restrict__ B, + at::BFloat16* __restrict__ C, + at::BFloat16* __restrict__ Btmp, + float* __restrict__ Ctmp, + float scale, + int M, + int N, + int K, + int lda, + int ldb, + int ldc) { + constexpr int BLOCK_N = block_size_n(); + + // [BLOCK_K, BLOCK_N] -> [BLOCK_K / 2, BLOCK_N * 2] + const int ldb_tmp = block_size_n(); + + // accumulate across K per BLOCK_K + for (int k = 0; k < K; k += BLOCK_K) { + int kb_size = std::min(BLOCK_K, K - k); + unpack_B(Btmp, B + k * ldb, N, kb_size, ldb, ldb_tmp); + + const bool add_C = (k != 0); + at::native::cpublas::brgemm(M, N, kb_size, lda, ldb_tmp, BLOCK_N, add_C, A + k, Btmp, Ctmp); + } + + // copy from Ctmp to C and mul scale + for (int m = 0; m < M; ++m) { + copy_mul_stub(C + m * ldc, Ctmp + m * BLOCK_N, N, scale); + } + } +}; + +template +struct brgemm { + static inline void apply( + const at::BFloat16* __restrict__ A, + const uint8_t* __restrict__ B, + at::BFloat16* __restrict__ C, + at::BFloat16* __restrict__ Btmp, + float* __restrict__ Ctmp, + const float* __restrict__ bias, + const uint8_t* __restrict__ scale, + int M, + int N, + int K, + int lda, + int ldb, + int ldc, + bool do_unpack = true) { + constexpr int BLOCK_N = block_size_n(); + + // [K, BLOCK_N] -> [K / 2, BLOCK_N * 2] + const int ldb_tmp = BLOCK_N; + + if (do_unpack) { + // group size 32 for mxfp4 + for (int k = 0; k < K; k += 32) { + unpack_B(Btmp + k * ldb_tmp, B + k * (ldb >> 1), N, 32, ldb, ldb_tmp, scale + (k >> 5) * BLOCK_N); + } + } + + at::native::cpublas::brgemm(M, N, K, lda, ldb_tmp, BLOCK_N, /* add_C */ false, A, Btmp, Ctmp); + + // copy from Ctmp to C + for (int m = 0; m < M; ++m) { + if constexpr (has_bias) { + copy_add_stub(C + m * ldc, Ctmp + m * BLOCK_N, bias, N); + } else { + copy_stub(C + m * ldc, Ctmp + m * BLOCK_N, N); + } + } + } +}; + +template void tinygemm_kernel( const scalar_t* __restrict__ A, - const at::Float8_e4m3fn* __restrict__ B, + const packed_t* __restrict__ B, scalar_t* __restrict__ C, scalar_t* __restrict__ Btmp, float* __restrict__ Ctmp, - const float* __restrict__ scale, + const param_t* __restrict__ scale, const float* __restrict__ bias, int64_t M, int64_t N, @@ -299,11 +693,11 @@ void tinygemm_kernel( int64_t ldb, int64_t ldc, bool brg, - int64_t block_size_K) { - + int64_t block_size_K, + bool do_unpack = true) { if (brg) { - brgemm::apply( - A, B, C, Btmp, Ctmp, bias, scale, M, N, K, lda, ldb, ldc); + brgemm::apply( + A, B, C, Btmp, Ctmp, bias, scale, M, N, K, lda, ldb, ldc, do_unpack); return; } @@ -319,23 +713,136 @@ void tinygemm_kernel( int64_t nb_start = nb * BLOCK_N; int64_t nb_size = std::min(BLOCK_N, N - nb_start); - switch(mb_size << 4 | nb_size >> 4) { - case 0x12: LAUNCH_TINYGEMM_KERNEL_NN(1, 32); break; - case 0x22: LAUNCH_TINYGEMM_KERNEL_NN(2, 32); break; - case 0x32: LAUNCH_TINYGEMM_KERNEL_NN(3, 32); break; - case 0x42: LAUNCH_TINYGEMM_KERNEL_NN(4, 32); break; - default: TORCH_CHECK(false, "Unexpected block size, ", mb_size, "x", nb_size); + switch (mb_size << 4 | nb_size >> 4) { + case 0x12: + LAUNCH_TINYGEMM_KERNEL_NN(1, 32); + break; + case 0x22: + LAUNCH_TINYGEMM_KERNEL_NN(2, 32); + break; + case 0x32: + LAUNCH_TINYGEMM_KERNEL_NN(3, 32); + break; + case 0x42: + LAUNCH_TINYGEMM_KERNEL_NN(4, 32); + break; + default: + TORCH_CHECK(false, "Unexpected block size, ", mb_size, "x", "nb_size"); } } } } template -void fp8_scaled_mm_kernel_impl( +void tinygemm_kernel2( + const scalar_t* __restrict__ A, + const at::Float8_e4m3fn* __restrict__ B, + scalar_t* __restrict__ C, + scalar_t* __restrict__ Btmp, + float* __restrict__ Ctmp, + float scale, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc, + bool brg) { + if (brg) { + brgemm2::apply(A, B, C, Btmp, Ctmp, scale, M, N, K, lda, ldb, ldc); + return; + } + + // pattern: 1-8-8 + if (M == 1) { + constexpr int64_t BLOCK_N = 128; + const int64_t NB = div_up(N, BLOCK_N); + int64_t mb_start = 0; + + for (int64_t nb = 0; nb < NB; ++nb) { + int64_t nb_start = nb * BLOCK_N; + int64_t nb_size = std::min(BLOCK_N, N - nb_start); + + switch (nb_size >> 4) { + case 2: + LAUNCH_TINYGEMM_KERNEL_NN2(1, 32); + break; + case 4: + LAUNCH_TINYGEMM_KERNEL_NN2(1, 64); + break; + case 6: + LAUNCH_TINYGEMM_KERNEL_NN2(1, 96); + break; + case 8: + LAUNCH_TINYGEMM_KERNEL_NN2(1, 128); + break; + default: + TORCH_CHECK(false, "Unexpected block size, 1x", "nb_size"); + } + } + return; + } + + // pattern: 1-4-16 + constexpr int64_t BLOCK_M = 4; + constexpr int64_t BLOCK_N = 64; + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N); + for (int64_t mb = 0; mb < MB; ++mb) { + int64_t mb_start = mb * BLOCK_M; + int64_t mb_size = std::min(BLOCK_M, M - mb_start); + for (int64_t nb = 0; nb < NB; ++nb) { + int64_t nb_start = nb * BLOCK_N; + int64_t nb_size = std::min(BLOCK_N, N - nb_start); + + switch (mb_size << 4 | nb_size >> 4) { + // mb_size = 1 + case 0x12: + LAUNCH_TINYGEMM_KERNEL_NN2(1, 32); + break; + case 0x14: + LAUNCH_TINYGEMM_KERNEL_NN2(1, 64); + break; + // mb_size = 2 + case 0x22: + LAUNCH_TINYGEMM_KERNEL_NN2(2, 32); + break; + case 0x24: + LAUNCH_TINYGEMM_KERNEL_NN2(2, 64); + break; + // mb_size = 3 + case 0x32: + LAUNCH_TINYGEMM_KERNEL_NN2(3, 32); + break; + case 0x34: + LAUNCH_TINYGEMM_KERNEL_NN2(3, 64); + break; + // mb_size = 4 + case 0x42: + LAUNCH_TINYGEMM_KERNEL_NN2(4, 32); + break; + case 0x44: + LAUNCH_TINYGEMM_KERNEL_NN2(4, 64); + break; + default: + TORCH_CHECK(false, "Unexpected block size, ", mb_size, "x", "nb_size"); + } + } + } +} + +// NB: fp8/fp4 scaled mm kernel implementation +// +// scalar_t packed_t param_t +// FP8 BF16 FP8 FP32 +// MXFP4 BF16 U8 U8 +// +template +void fp_scaled_mm_kernel_impl( scalar_t* __restrict__ out, const scalar_t* __restrict__ mat1, - const at::Float8_e4m3fn* __restrict__ mat2, - const float* __restrict__ scales2, + const packed_t* __restrict__ mat2, + const param_t* __restrict__ scales2, const float* __restrict__ bias, scalar_t* __restrict__ buffer, int64_t M, @@ -345,42 +852,41 @@ void fp8_scaled_mm_kernel_impl( int64_t out_strideM, int64_t block_size_N, int64_t block_size_K, - int64_t buffer_size_per_thread) { - - constexpr int64_t BLOCK_M = block_size_m() * BLOCK_SIZE_M_SCALE; + int64_t buffer_size_per_thread, + const func_t& scale_offset_per_block) { + constexpr int64_t BLOCK_M = block_size_m(); constexpr int64_t BLOCK_N = block_size_n(); const int64_t MB = div_up(M, BLOCK_M); const int64_t NB = div_up(N, BLOCK_N); - const int64_t scale_size_K = div_up(K, block_size_K); - const int64_t blocks_n_per_group = block_size_N / BLOCK_N; + const bool use_brgemm = can_use_brgemm(M); - const bool use_brgemm = can_use_brgemm(M); + // use K/2 for mxfp4 and K for fp8 + const int64_t packed_K = get_row_size(K); // parallel on [MB, NB] AT_DISPATCH_BOOL(bias != nullptr, has_bias, [&] { - at::parallel_for(0, MB * NB, 0, [&](int64_t begin, int64_t end) { - int64_t mb{0}, nb{0}; - data_index_init(begin, mb, MB, nb, NB); - - int tid = at::get_thread_num(); + parallel_2d(MB, NB, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + int tid = get_thread_num(); scalar_t* __restrict__ Btmp = buffer + tid * buffer_size_per_thread; - float* __restrict__ Ctmp = (float*)((void*)(Btmp + BLOCK_N * K)); + float* __restrict__ Ctmp = (float*)((void*)(Btmp + MAX_CACHE_BLOCK_SIZE * BLOCK_N * K)); - for (int64_t i = begin; i < end; ++i) { - UNUSED(i); - const float* scale_ptr = scales2 + (nb / blocks_n_per_group) * scale_size_K; + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * K, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + const param_t* scale_ptr = scales2 + scale_offset_per_block(nb); int64_t mb_start = mb * BLOCK_M; int64_t mb_size = std::min(M - mb_start, BLOCK_M); int64_t nb_start = nb * BLOCK_N; int64_t nb_size = std::min(N - nb_start, BLOCK_N); - tinygemm_kernel( + // only do unpacking for the first row + bool do_unpack = (mb == mb0); + + tinygemm_kernel( /* A */ mat1 + mb_start * mat1_strideM, - /* B */ mat2 + nb_start * K, // nb * BLOCK_N * K + /* B */ mat2 + nb_start * packed_K, // nb * BLOCK_N * K /* C */ out + mb_start * out_strideM + nb_start, - /* Btmp */ Btmp, + /* Btmp */ Btmp + nb_offset * BLOCK_N * K, /* Ctmp */ Ctmp, /* scale */ scale_ptr, /* bias */ bias + nb_start, @@ -391,11 +897,9 @@ void fp8_scaled_mm_kernel_impl( /* ldb */ nb_size, /* ldc */ out_strideM, /* brg */ use_brgemm, - /* block_size_K */ block_size_K); - - // move to the next index - data_index_step(mb, MB, nb, NB); - } + /* block_size_K */ block_size_K, + /* do_unpack */ do_unpack); + }); if (use_brgemm) { at::native::cpublas::brgemm_release(); @@ -404,7 +908,7 @@ void fp8_scaled_mm_kernel_impl( }); } -} // anonymous namespace +} // anonymous namespace // tinygemm interface template @@ -422,42 +926,126 @@ void tinygemm_kernel( int64_t ldb, int64_t ldc, bool brg, - int64_t block_size_K) { - tinygemm_kernel(A, B, C, Btmp, Ctmp, scale, nullptr, M, N, K, lda, ldb, ldc, brg, block_size_K); + int64_t block_size_K, + bool do_unpack) { + tinygemm_kernel( + A, B, C, Btmp, Ctmp, scale, nullptr, M, N, K, lda, ldb, ldc, brg, block_size_K, do_unpack); } -#define INSTANTIATE_TINYGEMM_TEMPLATE(TYPE) \ +template +void tinygemm_kernel( + const scalar_t* __restrict__ A, + const at::Float8_e4m3fn* __restrict__ B, + scalar_t* __restrict__ C, + scalar_t* __restrict__ Btmp, + float* __restrict__ Ctmp, + float scale, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc, + bool brg) { + tinygemm_kernel2(A, B, C, Btmp, Ctmp, scale, M, N, K, lda, ldb, ldc, brg); +} + +template +void tinygemm_kernel( + const scalar_t* __restrict__ A, + const uint8_t* __restrict__ B, + scalar_t* __restrict__ C, + scalar_t* __restrict__ Btmp, + float* __restrict__ Ctmp, + const uint8_t* __restrict__ scale, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc, + bool brg, + int64_t block_size_K, + bool do_unpack) { + tinygemm_kernel( + A, B, C, Btmp, Ctmp, scale, nullptr, M, N, K, lda, ldb, ldc, brg, block_size_K, do_unpack); +} + +#define INSTANTIATE_TINYGEMM_TEMPLATE(TYPE_A, TYPE_B, TYPE_S) \ + template void tinygemm_kernel( \ + const TYPE_A* __restrict__ A, \ + const TYPE_B* __restrict__ B, \ + TYPE_A* __restrict__ C, \ + TYPE_A* __restrict__ Btmp, \ + float* __restrict__ Ctmp, \ + const TYPE_S* __restrict__ scale, \ + int64_t M, \ + int64_t N, \ + int64_t K, \ + int64_t lda, \ + int64_t ldb, \ + int64_t ldc, \ + bool brg, \ + int64_t block_size_K, \ + bool do_unpack) + +INSTANTIATE_TINYGEMM_TEMPLATE(at::BFloat16, at::Float8_e4m3fn, float); +INSTANTIATE_TINYGEMM_TEMPLATE(at::Half, at::Float8_e4m3fn, float); +INSTANTIATE_TINYGEMM_TEMPLATE(at::BFloat16, uint8_t, uint8_t); +INSTANTIATE_TINYGEMM_TEMPLATE(at::Half, uint8_t, uint8_t); + +#define INSTANTIATE_TINYGEMM_TEMPLATE2(TYPE) \ template void tinygemm_kernel( \ const TYPE* __restrict__ A, \ const at::Float8_e4m3fn* __restrict__ B, \ TYPE* __restrict__ C, \ TYPE* __restrict__ Btmp, \ float* __restrict__ Ctmp, \ - const float* __restrict__ scale, \ + float scale, \ int64_t M, \ int64_t N, \ int64_t K, \ int64_t lda, \ int64_t ldb, \ int64_t ldc, \ - bool brg, \ - int64_t block_size_K) + bool brg) -INSTANTIATE_TINYGEMM_TEMPLATE(at::BFloat16); -INSTANTIATE_TINYGEMM_TEMPLATE(at::Half); +INSTANTIATE_TINYGEMM_TEMPLATE2(at::BFloat16); -at::Tensor fp8_scaled_mm_cpu(at::Tensor& mat1, at::Tensor& mat2, at::Tensor& scales2, - std::vector block_size, const std::optional& bias, - at::ScalarType out_dtype, bool is_vnni) { - RECORD_FUNCTION("sgl-kernel::fp8_scaled_mm_cpu", std::vector({mat1, mat2, scales2, block_size, bias})); +inline const float* get_bias_data(const std::optional& bias, int64_t N) { + if (bias.has_value()) { + const auto& bias_ref = bias.value(); + CHECK_EQ(bias_ref.size(0), N); + return bias_ref.data_ptr(); + } + return nullptr; +} +// FP8 and MXFP4 WoQ uses the same pattern: +// Btmp : [T, BLOCK_N * K] +// Ctmp : [T, BLOCK_M * BLOCK_N] +inline at::Tensor alloc_thread_buffer(const at::TensorOptions& options, int64_t K) { + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n(); + int num_threads = at::get_num_threads(); + int64_t size_per_thread = MAX_CACHE_BLOCK_SIZE * BLOCK_N * K + BLOCK_M * BLOCK_N * 2; + return at::empty({num_threads, size_per_thread}, options); +} + +at::Tensor fp8_scaled_mm_cpu( + at::Tensor& mat1, + at::Tensor& mat2, + at::Tensor& scales2, + std::vector block_size, + const std::optional& bias, + at::ScalarType out_dtype, + bool is_vnni) { auto packed_w = is_vnni ? mat2 : convert_weight_packed(mat2); CHECK_LAST_DIM_CONTIGUOUS_INPUT(mat1); CHECK_INPUT(mat2); CHECK_INPUT(scales2); - TORCH_CHECK(scales2.scalar_type() == at::kFloat, - "fp8_scaled_mm_cpu: expect scales2 to be float32."); + TORCH_CHECK(scales2.scalar_type() == at::kFloat, "fp8_scaled_mm_cpu: expect scales2 to be float32."); int64_t M = mat1.size(0); int64_t N = mat2.size(0); @@ -467,13 +1055,10 @@ at::Tensor fp8_scaled_mm_cpu(at::Tensor& mat1, at::Tensor& mat2, at::Tensor& sca CHECK_DIM(2, mat1); CHECK_DIM(2, mat2); - TORCH_CHECK(block_size.size() == 2, - "fp8_scaled_mm_cpu: expect block_size.size() to be 2."); - + TORCH_CHECK(block_size.size() == 2, "fp8_scaled_mm_cpu: expect block_size.size() to be 2."); int64_t block_size_N = block_size[0]; int64_t block_size_K = block_size[1]; - constexpr int64_t BLOCK_M = block_size_m() * BLOCK_SIZE_M_SCALE; constexpr int64_t BLOCK_N = block_size_n(); TORCH_CHECK(block_size_N % BLOCK_N == 0, "fp8_scaled_mm_cpu: expect block_size_N to be multiples of BLOCK_N"); TORCH_CHECK(block_size_K == BLOCK_K, "fp8_scaled_mm_cpu: expect block_size_K equals to BLOCK_K"); @@ -481,49 +1066,94 @@ at::Tensor fp8_scaled_mm_cpu(at::Tensor& mat1, at::Tensor& mat2, at::Tensor& sca CHECK_EQ(scales2.size(1), div_up(K, block_size_K)); const auto st = mat1.scalar_type(); - TORCH_CHECK(st == at::kBFloat16 || st == at::kHalf, - "fp8_scaled_mm_cpu: expect A to be bfloat16 or half."); - TORCH_CHECK(st == out_dtype, - "fp8_scaled_mm_cpu: expect A has same dtype with out_dtype."); - TORCH_CHECK(mat2.scalar_type() == at::kFloat8_e4m3fn, - "fp8_scaled_mm_cpu: expect mat2 to be fp8_e4m3."); - TORCH_CHECK(scales2.scalar_type() == at::kFloat, - "fp8_scaled_mm_cpu: expect scales to be float32."); + TORCH_CHECK(st == at::kBFloat16 || st == at::kHalf, "fp8_scaled_mm_cpu: expect A to be bfloat16 or half."); + TORCH_CHECK(st == out_dtype, "fp8_scaled_mm_cpu: expect A has same dtype with out_dtype."); + TORCH_CHECK(mat2.scalar_type() == at::kFloat8_e4m3fn, "fp8_scaled_mm_cpu: expect mat2 to be fp8_e4m3."); + TORCH_CHECK(scales2.scalar_type() == at::kFloat, "fp8_scaled_mm_cpu: expect scales to be float32."); auto out = at::empty({M, N}, mat1.options().dtype(out_dtype)); - // strides - int64_t mat1_strideM = mat1.stride(0); - int64_t out_strideM = out.stride(0); - - const bool has_bias = bias.has_value(); - const float* bias_data = nullptr; - if (has_bias) { - CHECK_EQ(bias.value().size(0), N); - bias_data = bias.value().data_ptr(); - } - - // Btmp : [T, BLOCK_N * K] - // Ctmp : [T, BLOCK_M * BLOCK_N] - int num_threads = at::get_num_threads(); - int64_t size_per_thread = BLOCK_N * K + BLOCK_M * BLOCK_N * 2; - auto buffer = at::empty({num_threads, size_per_thread}, mat1.options()); + auto buffer = alloc_thread_buffer(mat1.options(), K); AT_DISPATCH_REDUCED_FLOATING_TYPES(out_dtype, "fp8_scaled_mm_kernel_impl", [&] { - fp8_scaled_mm_kernel_impl( + // used for lambda computing scale offset for each block + // fp8 block gemm sale shape: [N/128, K/128] + // for each block: [1, K/128] + const int64_t scale_size_K = div_up(K, block_size_K); + const int64_t blocks_n_per_group = block_size_N / BLOCK_N; + + fp_scaled_mm_kernel_impl( out.data_ptr(), mat1.data_ptr(), packed_w.data_ptr(), scales2.data_ptr(), - bias_data, + get_bias_data(bias, N), buffer.data_ptr(), M, N, K, - mat1_strideM, - out_strideM, + mat1.stride(0), + out.stride(0), block_size_N, block_size_K, - size_per_thread); + buffer.size(-1), + [&](int64_t nb) { return (nb / blocks_n_per_group) * scale_size_K; }); + }); + + return out; +} + +// mat1 : [M, K] bfloat16 +// mat2 : [N, K / 2] uint8, actual layout: [N / BLOCK_N, K / 2, BLOCK_N, 2] +// scales2: [N, K / G], actual layout: [N / BLOCK_N, K / G, BLOCK_N] +at::Tensor mxfp4_scaled_mm_cpu( + at::Tensor& mat1, at::Tensor& mat2, at::Tensor& scales2, const std::optional& bias, bool is_vnni) { + auto packed_w = is_vnni ? mat2 : convert_weight_packed(mat2); + + CHECK_INPUT(mat1); + CHECK_INPUT(mat2); + CHECK_INPUT(scales2); + + int64_t M = mat1.size(0); + int64_t N = mat2.size(0); + int64_t K = mat2.size(1) * 2; + + // mxfp4 supports only group size of 32 (2^5) + constexpr int64_t group_size = 32; + constexpr int64_t BLOCK_N = block_size_n(); + + CHECK_EQ(mat1.size(1), K); + CHECK_EQ(scales2.numel(), N * K >> 5); + + const auto st = mat1.scalar_type(); + TORCH_CHECK(st == at::kBFloat16 || st == at::kHalf, "mxfp4_scaled_mm_cpu: expect A to be bfloat16 or half."); + TORCH_CHECK(mat2.scalar_type() == at::kByte, "mxfp4_scaled_mm_cpu: expect mat2 to be uint8."); + TORCH_CHECK(scales2.scalar_type() == at::kByte, "mxfp4_scaled_mm_cpu: expect scales to be uint8."); + auto out = at::empty({M, N}, mat1.options()); + + auto buffer = alloc_thread_buffer(mat1.options(), K); + + AT_DISPATCH_REDUCED_FLOATING_TYPES(st, "mxfp4_scaled_mm_kernel_impl", [&] { + // used for lambda computing scale offset for each block + // mxfp4 block gemm sale shape: [N/BLOCK_N, K/32, BLOCK_N] + // for each block: [K/32, BLOCK_N] + const int64_t s_strideN = (K >> 5) * BLOCK_N; + + fp_scaled_mm_kernel_impl( + out.data_ptr(), + mat1.data_ptr(), + packed_w.data_ptr(), + scales2.data_ptr(), + get_bias_data(bias, N), + buffer.data_ptr(), + M, + N, + K, + mat1.stride(0), + out.stride(0), + /* block_size_N */ 1, + /* block_size_K */ group_size, + buffer.size(-1), + [&](int64_t nb) { return nb * s_strideN; }); }); return out; diff --git a/csrc/cpu/sgl-kernels/gemm_int4.cpp b/csrc/cpu/sgl-kernels/gemm_int4.cpp index 4a04c50660..5b66b2a5ae 100644 --- a/csrc/cpu/sgl-kernels/gemm_int4.cpp +++ b/csrc/cpu/sgl-kernels/gemm_int4.cpp @@ -1,10 +1,10 @@ -// SPDX-License-Identifier: Apache-2.0 -// Adapted from sgl-project/sglang -// https://github.com/sgl-project/sglang/pull/8226 +// Adapted from +// https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu -#include +// clang-format off + +#include -#include "common.h" #include "gemm.h" #include "vec.h" @@ -24,19 +24,22 @@ struct ActDtype { using type = uint8_t; }; +#if defined(CPU_CAPABILITY_AVX512) struct alignas(32) m256i_wrapper { __m256i data; }; -#if defined(CPU_CAPABILITY_AVX512) -inline std::array load_zps_4vnni( - const int8_t* __restrict__ zps) { - __m256i vzps_low = _mm256_set1_epi64x(*reinterpret_cast(zps)); - __m256i vzps_high = - _mm256_set1_epi64x(*reinterpret_cast(zps + 8)); +inline std::array load_zps_4vnni(const int8_t* __restrict__ zps) { + // broadcast 01234567 to + // 01234567012345670123456701234567 + __m256i vzps_low = _mm256_set1_epi64x(*reinterpret_cast(zps)); + __m256i vzps_high = _mm256_set1_epi64x(*reinterpret_cast(zps + 8)); + // shuffle from + // 01234567012345670123456701234567 + // to + // 00001111222233334444555566667777 __m256i shuffle_mask = - _mm256_set_epi8(7, 7, 7, 7, 6, 6, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 3, 3, 3, - 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0); + _mm256_set_epi8(7, 7, 7, 7, 6, 6, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0); vzps_low = _mm256_shuffle_epi8(vzps_low, shuffle_mask); vzps_high = _mm256_shuffle_epi8(vzps_high, shuffle_mask); m256i_wrapper vzps_low_wp, vzps_high_wp; @@ -45,8 +48,7 @@ inline std::array load_zps_4vnni( return {vzps_low_wp, vzps_high_wp}; } -inline std::array load_uint4_as_int8( - const uint8_t* __restrict__ qB) { +inline std::array load_uint4_as_int8(const uint8_t* __restrict__ qB) { __m256i packed = _mm256_loadu_si256(reinterpret_cast(qB)); const __m256i low_mask = _mm256_set1_epi8(0x0f); __m256i high = _mm256_srli_epi16(packed, 4); @@ -58,37 +60,42 @@ inline std::array load_uint4_as_int8( return {low_wp, high_wp}; } -template -void _dequant_weight_zp_only(const uint8_t* __restrict__ B, int8_t* dqB, - const int8_t* __restrict__ qzeros, int64_t K) { - #pragma GCC unroll 2 +template +void _dequant_weight_zp_only(const uint8_t* __restrict__ B, int8_t* dqB, const int8_t* __restrict__ qzeros, int64_t K) { + // unpack weight int8 -> two int4 + // subtract zero point + // B shape = [K, ldb] = [K, N / 2], actual shape = [K / 4, N / 2, 4] + // dqB shape = [K, N], actual shape = [K / 4, N, 4] +#pragma GCC unroll 2 for (int n = 0; n < N; n += 16) { auto [zps_low_wp, zps_high_wp] = load_zps_4vnni(&qzeros[n]); auto zps_low = zps_low_wp.data; auto zps_high = zps_high_wp.data; for (int k = 0; k < K; k += 4) { - auto [vb_low_wp, vb_high_wp] = - load_uint4_as_int8(B + ldb * k + n / 2 * 4); + auto [vb_low_wp, vb_high_wp] = load_uint4_as_int8(B + ldb * k + n / 2 * 4); auto vb_low = vb_low_wp.data; auto vb_high = vb_high_wp.data; vb_high = _mm256_sub_epi8(vb_high, zps_high); vb_low = _mm256_sub_epi8(vb_low, zps_low); - _mm256_storeu_si256(reinterpret_cast<__m256i_u*>(dqB + N * k + n * 4), - vb_low); - _mm256_storeu_si256( - reinterpret_cast<__m256i_u*>(dqB + N * k + (n + 8) * 4), vb_high); + // store vb to B + _mm256_storeu_si256(reinterpret_cast<__m256i_u*>(dqB + N * k + n * 4), vb_low); + _mm256_storeu_si256(reinterpret_cast<__m256i_u*>(dqB + N * k + (n + 8) * 4), vb_high); } } } -template -void _dequant_and_store(float* __restrict__ output, - const int32_t* __restrict__ input, - const float* __restrict__ scale_a, - const int32_t* __restrict__ zp_a, - const float* __restrict__ scale_b, - const int32_t* __restrict__ comp_b, int M, int ldi, - int ldo, int ldsa = 1) { +template +void _dequant_and_store( + float* __restrict__ output, + const int32_t* __restrict__ input, + const float* __restrict__ scale_a, + const int32_t* __restrict__ zp_a, + const float* __restrict__ scale_b, + const int32_t* __restrict__ comp_b, + int M, + int ldi, + int ldo, + int ldsa = 1) { for (int m = 0; m < M; ++m) { float a_scale = *(scale_a + m * ldsa); __m512 va_scale = _mm512_set1_ps(a_scale); @@ -99,7 +106,7 @@ void _dequant_and_store(float* __restrict__ output, va_zp = _mm512_set1_epi32(a_zp); } int n = 0; - #pragma GCC unroll 2 +#pragma GCC unroll 2 for (; n < N; n += 16) { __m512i vc = _mm512_loadu_si512(input + m * ldi + n); if constexpr (!sym_quant_act) { @@ -122,8 +129,7 @@ void _dequant_and_store(float* __restrict__ output, if constexpr (sym_quant_act) { dq_val = (float)input[m * ldi + n] * a_scale * scale_b[n]; } else { - dq_val = (float)(input[m * ldi + n] - a_zp * comp_b[n]) * a_scale * - scale_b[n]; + dq_val = (float)(input[m * ldi + n] - a_zp * comp_b[n]) * a_scale * scale_b[n]; } if constexpr (accum) { output[m * ldo + n] += dq_val; @@ -135,9 +141,10 @@ void _dequant_and_store(float* __restrict__ output, } #else -template -void _dequant_weight_zp_only(const uint8_t* B, int8_t* dqB, - const int8_t* qzeros, int64_t K) { +template +void _dequant_weight_zp_only(const uint8_t* B, int8_t* dqB, const int8_t* qzeros, int64_t K) { + // B shape = [K, N / 2] + // dqB shape = [K, N] for (int k = 0; k < K; ++k) { for (int n = 0; n < N / 2; ++n) { int32_t b = (int32_t)B[k * ldb + n]; @@ -158,20 +165,31 @@ inline __m512i combine_m256i(std::array two_256) { return combine_m256i(two_256[0].data, two_256[1].data); } +// negate elements in a according to b's sign static inline __m512i _mm512_sign_epi8(__m512i a, __m512i b) { __m512i zero = _mm512_setzero_si512(); __mmask64 blt0 = _mm512_movepi8_mask(b); return _mm512_mask_sub_epi8(a, blt0, zero, a); } -template -void _dequant_gemm_accum_small_M(float* __restrict__ C, const uint8_t* A, - const float* scales_a, const int32_t* qzeros_a, - const uint8_t* B, const float* scales_b, - const int8_t* qzeros_b, int64_t K, int64_t lda, - int64_t ldc) { +template +void _dequant_gemm_accum_small_M( + float* __restrict__ C, + const uint8_t* A, + const float* scales_a, + const int32_t* qzeros_a, + const uint8_t* B, + const float* scales_b, + const int8_t* qzeros_b, + int64_t K, + int64_t lda, + int64_t ldc) { + // if sym_quant_act is true, A pointer type is passed in as uint8_t* but actually int8_t*. + constexpr int COLS = N / 16; - __m512i ones = _mm512_set1_epi8(1); + // Computing compensation is faster than loading it for small M + // because it's memory bound. + __m512i ones = _mm512_set1_epi8(1); // used for computing compensation __m512i va; __m512i vb[COLS]; __m512i vc[M * COLS]; @@ -179,6 +197,7 @@ void _dequant_gemm_accum_small_M(float* __restrict__ C, const uint8_t* A, __m512i vzps[COLS]; __m512i vcompensate[COLS]; + // Load scales and zps Unroll{}([&](auto i) { vscales[i] = _mm512_loadu_ps(scales_b + i * 16); vzps[i] = combine_m256i(load_zps_4vnni(qzeros_b + i * 16)); @@ -214,25 +233,25 @@ void _dequant_gemm_accum_small_M(float* __restrict__ C, const uint8_t* A, } }; + // Accumulate along k constexpr const int unroll = 4; int k = 0; for (; k < K / 4 / unroll; k++) { - Unroll{}( - [&](auto i) { Unroll{}(compute, 4 * (k * unroll + i)); }); + Unroll{}([&](auto i) { Unroll{}(compute, 4 * (k * unroll + i)); }); } k *= 4 * unroll; for (; k < K; k += 4) { Unroll{}(compute, k); } + // Store to C auto store = [&](auto i) { constexpr const int row = i / COLS; constexpr const int col = i % COLS; + // compute (qC - compensate * zp_a) * scale_a * scale_b __m512 vc_float; if constexpr (!sym_quant_act) { - vc[i] = _mm512_sub_epi32( - vc[i], _mm512_mullo_epi32(vcompensate[col], - _mm512_set1_epi32(*(qzeros_a + row)))); + vc[i] = _mm512_sub_epi32(vc[i], _mm512_mullo_epi32(vcompensate[col], _mm512_set1_epi32(*(qzeros_a + row)))); } vc_float = _mm512_cvtepi32_ps(vc[i]); vc_float = _mm512_mul_ps(vc_float, _mm512_set1_ps(*(scales_a + row))); @@ -245,17 +264,28 @@ void _dequant_gemm_accum_small_M(float* __restrict__ C, const uint8_t* A, Unroll{}(store); } - #define CALL_DEQUANT_GEMM_ACCUM_SMALL_M(M) \ - _dequant_gemm_accum_small_M( \ - C, A, scales_a, qzeros_a, B, scales_b, qzeros_b, K, lda, ldc); +#define CALL_DEQUANT_GEMM_ACCUM_SMALL_M(M) \ + _dequant_gemm_accum_small_M(C, A, scales_a, qzeros_a, B, scales_b, qzeros_b, K, lda, ldc); #endif -template -void _dequant_gemm_accum(float* C, const uint8_t* A, const float* scales_a, - const int32_t* qzeros_a, const uint8_t* B, - const float* scales_b, const int8_t* qzeros_b, - const int32_t* compensation, int8_t* dqB, int64_t M, - int64_t K, int64_t lda, int64_t ldc, bool use_brgemm) { +template +void _dequant_gemm_accum( + float* C, + const uint8_t* A, + const float* scales_a, + const int32_t* qzeros_a, + const uint8_t* B, + const float* scales_b, + const int8_t* qzeros_b, + const int32_t* compensation, + int8_t* dqB, + int64_t M, + int64_t K, + int64_t lda, + int64_t ldc, + bool use_brgemm) { + // Compute GEMM int8 * int8 -> int32 + // dequant result to float by applying scales/qzeros #if defined(CPU_CAPABILITY_AVX512) if (!use_brgemm) { switch (M) { @@ -282,14 +312,12 @@ void _dequant_gemm_accum(float* C, const uint8_t* A, const float* scales_a, Tin* A_ptr = (Tin*)A; if (use_brgemm) { int32_t C_i32[M * N]; - at::native::cpublas::brgemm(M, N, K, lda, N /*ldb*/, N /*ldc*/, - false /* add_C */, A_ptr, dqB, C_i32, - true /* is_vnni */); + at::native::cpublas::brgemm( + M, N, K, lda, N /*ldb*/, N /*ldc*/, false /* add_C */, A_ptr, dqB, C_i32, true /* is_vnni */); _mm_prefetch(B + N * K / 2, _MM_HINT_T0); _mm_prefetch(A + K, _MM_HINT_T0); - _dequant_and_store(C, C_i32, scales_a, qzeros_a, - scales_b, compensation, M, - N /*ldi*/, ldc, 1 /*ldsa*/); + _dequant_and_store( + C, C_i32, scales_a, qzeros_a, scales_b, compensation, M, N /*ldi*/, ldc, 1 /*ldsa*/); } else #endif { @@ -297,13 +325,13 @@ void _dequant_gemm_accum(float* C, const uint8_t* A, const float* scales_a, } } -template +template inline void copy_bias(const float* bias_ptr, float* y_buf, int64_t m) { if (bias_ptr) { for (int i = 0; i < m; ++i) { int j = 0; #if defined(CPU_CAPABILITY_AVX512) - #pragma GCC unroll 2 +#pragma GCC unroll 2 for (; j < N; j += 16) { __m512 bias_vec = _mm512_loadu_ps(bias_ptr + j); _mm512_storeu_ps(y_buf + i * N + j, bias_vec); @@ -313,11 +341,11 @@ inline void copy_bias(const float* bias_ptr, float* y_buf, int64_t m) { y_buf[i * N + j] = bias_ptr[j]; } } - } else { + } else { // initialize to zero for (int i = 0; i < m; ++i) { int j = 0; #if defined(CPU_CAPABILITY_AVX512) - #pragma GCC unroll 2 +#pragma GCC unroll 2 for (; j < N; j += 16) { __m512 zero_vec = _mm512_setzero_ps(); _mm512_storeu_ps(y_buf + i * N + j, zero_vec); @@ -330,14 +358,13 @@ inline void copy_bias(const float* bias_ptr, float* y_buf, int64_t m) { } } -template -inline void store_out(const float* y_buf, out_dtype* c_ptr, int64_t m, - int64_t lda) { +template +inline void store_out(const float* y_buf, out_dtype* c_ptr, int64_t m, /* int64_t n, */ int64_t lda) { for (int i = 0; i < m; ++i) { int j = 0; if constexpr (std::is_same::value) { #if defined(CPU_CAPABILITY_AVX512) - #pragma GCC unroll 2 +#pragma GCC unroll 2 for (; j < N; j += 16) { __m512 y_vec = _mm512_loadu_ps(y_buf + i * N + j); _mm512_storeu_ps(c_ptr + i * lda + j, y_vec); @@ -348,12 +375,11 @@ inline void store_out(const float* y_buf, out_dtype* c_ptr, int64_t m, } } else if constexpr (std::is_same::value) { #if defined(CPU_CAPABILITY_AVX512) - #pragma GCC unroll 2 +#pragma GCC unroll 2 for (; j < N; j += 16) { __m512 y_vec = _mm512_loadu_ps(y_buf + i * N + j); __m256i y_bf16_vec = at::vec::cvtfp32_bf16(y_vec); - _mm256_storeu_si256(reinterpret_cast<__m256i*>(c_ptr + i * lda + j), - y_bf16_vec); + _mm256_storeu_si256(reinterpret_cast<__m256i*>(c_ptr + i * lda + j), y_bf16_vec); } #endif for (; j < N; ++j) { @@ -361,12 +387,11 @@ inline void store_out(const float* y_buf, out_dtype* c_ptr, int64_t m, } } else if constexpr (std::is_same::value) { #if defined(CPU_CAPABILITY_AVX512) - #pragma GCC unroll 2 +#pragma GCC unroll 2 for (; j < N; j += 16) { __m512 y_vec = _mm512_loadu_ps(y_buf + i * N + j); __m256i y_fp16_vec = at::vec::cvtfp32_fp16(y_vec); - _mm256_storeu_si256(reinterpret_cast<__m256i*>(c_ptr + i * lda + j), - y_fp16_vec); + _mm256_storeu_si256(reinterpret_cast<__m256i*>(c_ptr + i * lda + j), y_fp16_vec); } #endif for (; j < N; ++j) { @@ -392,16 +417,25 @@ void fill_val_stub(int32_t* __restrict__ output, int32_t value, int64_t size) { } } -template +template void _da8w4_linear_impl( - act_dtype* __restrict__ input, const float* __restrict__ input_scales, + act_dtype* __restrict__ input, + const float* __restrict__ input_scales, const int32_t* __restrict__ input_qzeros, - const uint8_t* __restrict__ weight, const float* __restrict__ weight_scales, - const int8_t* __restrict__ weight_qzeros, const float* __restrict__ bias, - out_dtype* __restrict__ output, float* __restrict__ output_temp, - int8_t* __restrict__ dequant_weight_temp, int64_t M, int64_t N, int64_t K, + const uint8_t* __restrict__ weight, + const float* __restrict__ weight_scales, + const int8_t* __restrict__ weight_qzeros, + const float* __restrict__ bias, + out_dtype* __restrict__ output, + float* __restrict__ output_temp, + int8_t* __restrict__ dequant_weight_temp, + int64_t M, + int64_t N, + int64_t K, int64_t num_groups) { - const bool use_brgemm = can_use_brgemm(M); + // weight + compensation shape = [Nc, Kc, BLOCK_N * _block_k / 2 + BLOCK_N*sizeof(int32_t)] + // scales/qzeros shape = [Nc, G, BLOCK_N] + const bool use_brgemm = can_use_brgemm(M); int64_t block_m = [&]() -> long { if (M <= 48) { return M; @@ -432,30 +466,24 @@ void _da8w4_linear_impl( int64_t mc_end = parallel_on_M ? mc + 1 : Mc; for (int mci = mc; mci < mc_end; ++mci) { - int64_t m_size = - mci * block_m + block_m > M ? M - mci * block_m : block_m; + int64_t m_size = mci * block_m + block_m > M ? M - mci * block_m : block_m; + // copy bias to y_buf if bias is not None auto bias_data = bias ? bias + nc * BLOCK_N : nullptr; copy_bias(bias_data, C_tmp, m_size); for (int kci = 0; kci < Kc; ++kci) { int32_t* compensation_ptr = sym_quant_act ? nullptr - : (int32_t*)(void*)(weight + - (nc * Kc + kci) * - (BLOCK_N * - (_block_k / 2 + sizeof(int32_t))) + - _block_k * BLOCK_N / 2); - _dequant_gemm_accum( + : (int32_t*)(void*)(weight + (nc * Kc + kci) * (BLOCK_N * (_block_k / 2 + sizeof(int32_t))) + + _block_k * BLOCK_N / 2) /*Bcomp*/; + _dequant_gemm_accum( /*C*/ C_tmp, /*A*/ (uint8_t*)input + mci * block_m * K + kci * _block_k, /*scales_a*/ input_scales + mci * block_m, /*qzeros_a*/ input_qzeros + mci * block_m, - /*B*/ weight + (nc * Kc + kci) * - (BLOCK_N * (_block_k / 2 + sizeof(int32_t))), - /*scales_b*/ weight_scales + nc * BLOCK_N * num_groups + - kci / block_per_group * BLOCK_N, - /*qzeros_b*/ weight_qzeros + nc * BLOCK_N * num_groups + - kci / block_per_group * BLOCK_N, + /*B*/ weight + (nc * Kc + kci) * (BLOCK_N * (_block_k / 2 + sizeof(int32_t))), + /*scales_b*/ weight_scales + nc * BLOCK_N * num_groups + kci / block_per_group * BLOCK_N, + /*qzeros_b*/ weight_qzeros + nc * BLOCK_N * num_groups + kci / block_per_group * BLOCK_N, /*Bcomp*/ compensation_ptr, /*dqB_tmp*/ dqB_tmp, /*M*/ m_size, @@ -464,8 +492,8 @@ void _da8w4_linear_impl( /*ldc*/ BLOCK_N, /*use_brgemm*/ use_brgemm); } - store_out(C_tmp, output + mci * block_m * N + nc * BLOCK_N, - m_size, N /*lda*/); + // store y_buf to output with dtype conversion + store_out(C_tmp, output + mci * block_m * N + nc * BLOCK_N, m_size, N /*lda*/); } } if (use_brgemm) { @@ -476,15 +504,16 @@ void _da8w4_linear_impl( } // anonymous namespace -std::tuple -convert_int4_weight_packed_with_compensation(const at::Tensor& weight, - const at::Tensor& scales, - const at::Tensor& qzeros) { - TORCH_CHECK(weight.dim() == 2, - "DA8W4 CPU: Weight should be a 2D tensor for packing"); - TORCH_CHECK( - weight.size(1) % 2 == 0, - "DA8W4 CPU: Weight should have even number of columns for packing"); +/* +return: packed_weight, packed_scales, packed_qzeros +*/ +std::tuple convert_int4_weight_packed_with_compensation( + const at::Tensor& weight, const at::Tensor& scales, const at::Tensor& qzeros) { + // weight shape = [N, K] + // scales shape = [N, G] + // qzeros shape = [N, G] + TORCH_CHECK(weight.dim() == 2, "DA8W4 CPU: Weight should be a 2D tensor for packing"); + TORCH_CHECK(weight.size(1) % 2 == 0, "DA8W4 CPU: Weight should have even number of columns for packing"); auto new_scales = scales; auto new_qzeros = qzeros; @@ -505,20 +534,21 @@ convert_int4_weight_packed_with_compensation(const at::Tensor& weight, int64_t Nc = N / block_n; int64_t Kc = K / _block_k; + // Reorder weight to [N/block_n, K/_block_k, _block_k, block_n] + // Reorder scales/qzeros to [N/block_n, G, block_n] + // weight + compensation shape = [Nc, Kc, block_n * _block_k / 2 + block_n*sizeof(int32_t)] + // scales/qzeros shape = [Nc, G, block_n] auto weight_view = weight.view({Nc, block_n, Kc, _block_k}); at::Tensor weight_reordered = weight_view.permute({0, 2, 3, 1}).contiguous(); at::Tensor blocked_weight; - at::Tensor blocked_scales = - new_scales.view({Nc, block_n, G}).permute({0, 2, 1}).contiguous(); - at::Tensor blocked_qzeros = - new_qzeros.view({Nc, block_n, G}).permute({0, 2, 1}).contiguous(); - auto weight_sub_qzero = weight.view({Nc, block_n, G, -1}).to(at::kInt) - - new_qzeros.view({Nc, block_n, G, -1}); + at::Tensor blocked_scales = new_scales.view({Nc, block_n, G}).permute({0, 2, 1}).contiguous(); + at::Tensor blocked_qzeros = new_qzeros.view({Nc, block_n, G}).permute({0, 2, 1}).contiguous(); + // Compensation = Σ(k)(W[k][n] - ZP[n]) for each block. + auto weight_sub_qzero = weight.view({Nc, block_n, G, -1}).to(at::kInt) - new_qzeros.view({Nc, block_n, G, -1}); weight_sub_qzero = weight_sub_qzero.view({Nc, block_n, Kc, _block_k}); at::Tensor compensation = weight_sub_qzero.sum(-1); compensation = compensation.permute({0, 2, 1}).contiguous().to(at::kInt); - int64_t buffer_size_nbytes = - _block_k * block_n / 2 + block_n * sizeof(int32_t); + int64_t buffer_size_nbytes = _block_k * block_n / 2 + block_n * sizeof(int32_t); blocked_weight = at::empty({Nc, Kc, buffer_size_nbytes}, weight.options()); auto weight_ptr = weight_reordered.data_ptr(); @@ -528,24 +558,25 @@ convert_int4_weight_packed_with_compensation(const at::Tensor& weight, at::parallel_for(0, num_blocks, 1, [&](int64_t begin, int64_t end) { for (const auto i : c10::irange(begin, end)) { auto in_ptr = weight_ptr + i * _block_k * block_n; - auto out_ptr = - blocked_weight_ptr + i * block_n * (_block_k / 2 + sizeof(int32_t)); + auto out_ptr = blocked_weight_ptr + i * block_n * (_block_k / 2 + sizeof(int32_t)); int32_t* comp_in_prt = compensation_ptr + i * block_n; - int32_t* comp_out_prt = - (int32_t*)(void*)(blocked_weight_ptr + - i * block_n * (_block_k / 2 + sizeof(int32_t)) + - _block_k * block_n / 2); + int32_t* comp_out_prt = (int32_t*)(void*)(blocked_weight_ptr + i * block_n * (_block_k / 2 + sizeof(int32_t)) + + _block_k * block_n / 2); + // Reorder weight block to VNNI4 and pack two lanes along N + // N=16 viewed as two lanes: a0, ...a7, b0, ...b7 + // pack two lanes: [a0, b0], ..., [a7, b7] + // plain shape = [_block_k, block_n] + // packed shape = [_block_k / 4, block_n / 2, 4] viewed as [_block_k, block_n / 2] constexpr int n_group_size = 8; constexpr int vnni_size = 4; - constexpr int n_group = block_n / n_group_size; + constexpr int n_group = block_n / n_group_size; // 4 for (int nb = 0; nb < n_group; nb += 2) { for (int k = 0; k < _block_k; k += vnni_size) { for (int ni = 0; ni < n_group_size; ++ni) { for (int ki = 0; ki < vnni_size; ++ki) { int src_idx_1 = nb * n_group_size + ni + (k + ki) * block_n; int src_idx_2 = (nb + 1) * n_group_size + ni + (k + ki) * block_n; - int dst_idx = (nb / 2 * n_group_size + ni) * vnni_size + - k * block_n / 2 + ki; + int dst_idx = (nb / 2 * n_group_size + ni) * vnni_size + k * block_n / 2 + ki; uint8_t src_1 = *(in_ptr + src_idx_1); uint8_t src_2 = *(in_ptr + src_idx_2); uint8_t dst = (src_1 & 0x0f) | ((src_2 & 0x0f) << 4); @@ -554,39 +585,128 @@ convert_int4_weight_packed_with_compensation(const at::Tensor& weight, } } } + // compensation [block_n] for (int nb = 0; nb < block_n; nb++) { *(comp_out_prt + nb) = *(comp_in_prt + nb); } } }); - return std::make_tuple(std::move(blocked_weight), std::move(blocked_scales), - std::move(blocked_qzeros)); + return std::make_tuple(std::move(blocked_weight), std::move(blocked_scales), std::move(blocked_qzeros)); } -std::tuple autoawq_to_int4pack(at::Tensor qweight, - at::Tensor qzeros) { - auto bitshifts = at::tensor({0, 4, 1, 5, 2, 6, 3, 7}, at::kInt) * 4; - auto qweight_unsq = qweight.unsqueeze(-1); - auto unpacked = at::bitwise_right_shift(qweight_unsq, bitshifts) & 0xF; - auto qweight_final = unpacked.flatten(-2).transpose(-1, -2).to(at::kByte); +std::tuple unpack_4bit_to_32bit_signed(const at::Tensor& qweight, const at::Tensor& qzeros) { + TORCH_CHECK(qweight.scalar_type() == at::kInt, "qweight must be int32"); + TORCH_CHECK(qzeros.scalar_type() == at::kInt, "qzeros must be int32"); + const auto W0 = qweight.size(0); + const auto W1 = qweight.size(1); + const auto Z0 = qzeros.size(0); + const auto Z1 = qzeros.size(1); - auto qzeros_unsq = qzeros.unsqueeze(-1); - auto qzeros_unpacked = at::bitwise_right_shift(qzeros_unsq, bitshifts) & 0xF; - auto qzeros_final = qzeros_unpacked.flatten(-2).to(at::kByte); + // unpacked_weights: (W0 * 8, W1), int8 + auto unpacked_weights = at::zeros({W0 * 8, W1}, at::TensorOptions().dtype(at::kChar)); + // unpacked_zeros: (Z0, Z1 * 8), int8 + auto unpacked_zeros = at::zeros({Z0, Z1 * 8}, at::TensorOptions().dtype(at::kChar)); - return std::make_tuple(qweight_final, qzeros_final); + const int32_t* qw_ptr = qweight.data_ptr(); + const int32_t* qz_ptr = qzeros.data_ptr(); + int8_t* uw_ptr = unpacked_weights.data_ptr(); + int8_t* uz_ptr = unpacked_zeros.data_ptr(); + + // ---- unpack qweight ---- + for (int64_t row = 0; row < W0 * 8; ++row) { + const int i = row & 7; // row % 8 + const int src_row = row >> 3; // row // 8 + const int shift = 4 * i; + for (int64_t col = 0; col < W1; ++col) { + int32_t v = qw_ptr[src_row * W1 + col]; + uw_ptr[row * W1 + col] = static_cast((v >> shift) & 0xF); + } + } + // ---- unpack qzeros ---- + for (int64_t col = 0; col < Z1 * 8; ++col) { + const int i = col & 7; + const int src_col = col >> 3; + const int shift = 4 * i; + + for (int64_t row = 0; row < Z0; ++row) { + int32_t v = qz_ptr[row * Z1 + src_col]; + uz_ptr[row * (Z1 * 8) + col] = static_cast((v >> shift) & 0xF); + } + } + + return std::make_tuple(unpacked_weights, unpacked_zeros + 1); +} + +std::tuple +autogptq_to_int4pack(const at::Tensor& qweight_tensor, const at::Tensor& qzeros_tensor) { + TORCH_CHECK(qweight_tensor.scalar_type() == at::kInt, "qweight_tensor must be int32"); + TORCH_CHECK(qzeros_tensor.scalar_type() == at::kInt, "qzeros_tensor must be int32"); + TORCH_CHECK(qweight_tensor.is_cpu(), "CPU only implementation"); + if (qweight_tensor.dim() == 3) { + const int64_t B = qweight_tensor.size(0); + std::vector qweight_list; + std::vector qzeros_list; + qweight_list.reserve(B); + qzeros_list.reserve(B); + for (int64_t i = 0; i < B; ++i) { + auto outputs = unpack_4bit_to_32bit_signed(qweight_tensor[i], qzeros_tensor[i]); + at::Tensor unpacked_qweight = std::get<0>(outputs); + at::Tensor unpacked_qzeros = std::get<1>(outputs); + qweight_list.push_back(unpacked_qweight.transpose(0, 1).contiguous().to(at::kByte)); + qzeros_list.push_back(unpacked_qzeros.contiguous().to(at::kByte)); + } + return std::make_tuple(at::stack(qweight_list).detach(), at::stack(qzeros_list).detach()); + } + auto outputs = unpack_4bit_to_32bit_signed(qweight_tensor, qzeros_tensor); + at::Tensor unpacked_qweight = std::get<0>(outputs); + at::Tensor unpacked_qzeros = std::get<1>(outputs); + at::Tensor return_qweight = unpacked_qweight.transpose(0, 1).contiguous().to(at::kByte); + at::Tensor return_qzeros = unpacked_qzeros.contiguous().to(at::kByte); + return std::make_tuple(return_qweight, return_qzeros); +} + +std::tuple int4pack(at::Tensor qweight, at::Tensor qzeros, int64_t quant_method_4bit) { + if (quant_method_4bit == CPUQuantAlgo::AWQ) { + // autoawq unpacking + qweight = qweight.contiguous(); + qzeros = qzeros.contiguous(); + // bitshifts: [0, 4, 1, 5, 2, 6, 3, 7] * 4 + auto bitshifts = at::tensor({0, 4, 1, 5, 2, 6, 3, 7}, at::kInt) * 4; + auto qweight_unsq = qweight.unsqueeze(-1); // [..., K, N/8, 1] + auto unpacked = (at::bitwise_right_shift(qweight_unsq, bitshifts) & 0xF).contiguous(); + auto qweight_final = unpacked.flatten(-2).transpose(-1, -2).to(at::kByte).clone(); + auto qzeros_unsq = qzeros.unsqueeze(-1); + auto qzeros_unpacked = (at::bitwise_right_shift(qzeros_unsq, bitshifts) & 0xF).contiguous(); + auto qzeros_final = qzeros_unpacked.flatten(-2).to(at::kByte).clone(); + return std::make_tuple(qweight_final, qzeros_final); + } else if (quant_method_4bit == CPUQuantAlgo::GPTQ) { + // autogptq unpacking + auto outputs = autogptq_to_int4pack(qweight, qzeros); + at::Tensor unpacked_qweight = std::get<0>(outputs); + at::Tensor unpacked_qzeros = std::get<1>(outputs); + return std::make_tuple(unpacked_qweight, unpacked_qzeros); + } else { + TORCH_CHECK(false, "CPU int4 pack only support AWQ or GPTQ..."); + } } std::tuple convert_weight_packed_scale_zp( - at::Tensor qweight, at::Tensor qzeros, at::Tensor scales) { - auto res = autoawq_to_int4pack(qweight, qzeros); - auto _qweight = std::get<0>(res); - auto _qzeros = std::get<1>(res); + at::Tensor qweight, // awq: (*, K, N / 8) || gptq: (*, K / 8, N) , int32 + at::Tensor qzeros, // awq: (*, K / group_size, N / 8) || gptq: (*, K / group_size, N / 8) , int32 + at::Tensor scales, // awq: (*, K / group_size, N) || gptq: (*, K / group_size, N) , bfloat16 + int64_t quant_method_4bit) { + at::Tensor _qweight; + at::Tensor _qzeros; + + auto res = int4pack(qweight, qzeros, quant_method_4bit); + _qweight = std::get<0>(res); + _qzeros = std::get<1>(res); + auto _scales = scales; - _qzeros = _qzeros.transpose(-2, -1).contiguous(); + _qzeros = _qzeros.transpose(-2, -1).contiguous(); // .T _scales = _scales.transpose(-2, -1).contiguous(); - if (_qweight.dim() == 3) { + if (_qweight.dim() == 3) { // Dim=3 for MOE packing, TODO: refine a unified loop int64_t E = _qweight.size(0); int64_t K = _qweight.size(2); int64_t G = _scales.size(2); @@ -595,17 +715,12 @@ std::tuple convert_weight_packed_scale_zp( int64_t block_n = block_size_n(); int64_t Nc = _qweight.size(1) / block_n; int64_t Kc = K / _block_k; - int64_t buffer_size_nbytes = - _block_k * block_n / 2 + block_n * sizeof(int32_t); - auto blocked_weight = - at::empty({E, Nc, Kc, buffer_size_nbytes}, _qweight.options()); - auto blocked_scales = - at::empty({E, Nc, G, block_n}, _scales.options()).to(at::kFloat); - auto blocked_qzeros = - at::empty({E, Nc, G, block_n}, _qzeros.options()).to(at::kChar); + int64_t buffer_size_nbytes = _block_k * block_n / 2 + block_n * sizeof(int32_t); + auto blocked_weight = at::empty({E, Nc, Kc, buffer_size_nbytes}, _qweight.options()); + auto blocked_scales = at::empty({E, Nc, G, block_n}, _scales.options()).to(at::kFloat); + auto blocked_qzeros = at::empty({E, Nc, G, block_n}, _qzeros.options()).to(at::kChar); for (int i = 0; i < _qweight.size(0); i++) { - auto res_ = convert_int4_weight_packed_with_compensation( - _qweight[i], _scales[i], _qzeros[i]); + auto res_ = convert_int4_weight_packed_with_compensation(_qweight[i], _scales[i], _qzeros[i]); blocked_weight[i] = std::get<0>(res_); blocked_scales[i] = std::get<1>(res_); blocked_qzeros[i] = std::get<2>(res_); @@ -614,8 +729,7 @@ std::tuple convert_weight_packed_scale_zp( _scales = blocked_scales; _qzeros = blocked_qzeros; } else { - auto res_ = convert_int4_weight_packed_with_compensation(_qweight, _scales, - _qzeros); + auto res_ = convert_int4_weight_packed_with_compensation(_qweight, _scales, _qzeros); _qweight = std::get<0>(res_); _scales = std::get<1>(res_); _qzeros = std::get<2>(res_); @@ -624,89 +738,93 @@ std::tuple convert_weight_packed_scale_zp( return std::make_tuple(_qweight, _qzeros, _scales); } -at::Tensor int4_scaled_mm_cpu_with_quant(const at::Tensor& input, - const at::Tensor& weight, - const at::Tensor& weight_scales, - const at::Tensor& weight_qzeros, - const std::optional& bias, - at::ScalarType output_dtype) { - RECORD_FUNCTION("vllm::int4_scaled_mm_cpu_with_quant", - std::vector({input, weight})); - +at::Tensor int4_scaled_mm_cpu_with_quant( + const at::Tensor& input, + const at::Tensor& weight, + const at::Tensor& weight_scales, + const at::Tensor& weight_qzeros, + const std::optional& bias, + at::ScalarType output_dtype) { int64_t M_a = input.size(0); int64_t K_a = input.size(1); int64_t lda = input.stride(0); const auto st = input.scalar_type(); TORCH_CHECK( - st == at::kBFloat16 || st == at::kHalf, - "int4_scaled_mm_cpu_with_quant: expect A to be bfloat16 or half."); + st == at::kBFloat16 || st == at::kHalf, "int4_scaled_mm_cpu_with_quant: expect A to be bfloat16 or half."); - constexpr bool sym_quant_act = false; + constexpr bool sym_quant_act = false; // TODO: add sym quant path using Tin = typename ActDtype::type; - int64_t act_buffer_size = - M_a * K_a + M_a * sizeof(float) + M_a * sizeof(int32_t); - auto act_buffer = - at::empty({act_buffer_size}, input.options().dtype(at::kByte)); + int64_t act_buffer_size = /* act quant */ M_a * K_a + + /* act scale */ M_a * sizeof(float) + + /* act zp */ M_a * sizeof(int32_t); + auto act_buffer = at::empty({act_buffer_size}, input.options().dtype(at::kByte)); + // asym path, activation quants into uint8_t auto Aq_data = act_buffer.data_ptr(); auto As_data = reinterpret_cast(Aq_data + M_a * K_a); auto Azp_data = reinterpret_cast(As_data + M_a); - fill_val_stub(Azp_data, 128, M_a); + fill_val_stub(Azp_data, 128, M_a); // sym_a s8s8 is unified to u8s8 with compensation (128) auto out_sizes = input.sizes().vec(); int64_t N = weight_scales.size(0) * weight_scales.size(-1); out_sizes.back() = N; auto output = at::empty(out_sizes, input.options()); + // weight + compensation shape = [Nc, Kc, BLOCK_N * _block_k / 2 + BLOCK_N*sizeof(int32_t)] + // scales/qzeros shape = [Nc, G, BLOCK_N] int64_t Nc = weight.size(0); int64_t Kc = weight.size(1); int64_t _block_k = K_a / Kc; TORCH_CHECK(N == Nc * BLOCK_N, "DA8W4: weight and input shapes mismatch"); + // scales/qzeros shape = [Nc, G, BLOCK_N] int64_t num_groups = weight_scales.size(1); const uint8_t* b_ptr = weight.data_ptr(); const float* b_scales_ptr = weight_scales.data_ptr(); const int8_t* b_qzeros_ptr = weight_qzeros.data_ptr(); - const float* bias_ptr = - bias.has_value() ? bias.value().data_ptr() : nullptr; + const float* bias_ptr = bias.has_value() ? bias.value().data_ptr() : nullptr; int num_threads = at::get_num_threads(); - int64_t temp_buffer_size = num_threads * BLOCK_M * BLOCK_N * sizeof(float) + - num_threads * _block_k * BLOCK_N; - auto c_temp_buffer = - at::empty({temp_buffer_size}, input.options().dtype(at::kChar)); + int64_t temp_buffer_size = /* output temp */ num_threads * BLOCK_M * BLOCK_N * sizeof(float) + + /* weight dequant temp */ num_threads * _block_k * BLOCK_N; + auto c_temp_buffer = at::empty({temp_buffer_size}, input.options().dtype(at::kChar)); float* c_temp_ptr = (float*)((void*)(c_temp_buffer.data_ptr())); - int8_t* dqB_temp_ptr = - (int8_t*)((void*)(c_temp_ptr + num_threads * BLOCK_M * BLOCK_N)); + int8_t* dqB_temp_ptr = (int8_t*)((void*)(c_temp_ptr + num_threads * BLOCK_M * BLOCK_N)); -#define LAUNCH_DA8W4_LINEAR_WITH_QUANT_IMPL(sym_quant_act) \ - AT_DISPATCH_FLOATING_TYPES_AND2( \ - at::ScalarType::BFloat16, at::ScalarType::Half, output_dtype, \ - "int4_scaled_mm_cpu", [&] { \ - const scalar_t* __restrict__ A_data = input.data_ptr(); \ - scalar_t* __restrict__ c_ptr = output.data_ptr(); \ - at::parallel_for(0, M_a, 0, [&](int64_t begin, int64_t end) { \ - for (int64_t m = begin; m < end; ++m) { \ - quantize_row_int8(Aq_data + m * K_a, As_data[m], \ - A_data + m * lda, K_a); \ - } \ - }); \ - _da8w4_linear_impl( \ - Aq_data, As_data, Azp_data, b_ptr, b_scales_ptr, b_qzeros_ptr, \ - bias_ptr, c_ptr, c_temp_ptr, dqB_temp_ptr, M_a, N, K_a, \ - num_groups); \ +#define LAUNCH_DA8W4_LINEAR_WITH_QUANT_IMPL(sym_quant_act) \ + AT_DISPATCH_FLOATING_TYPES_AND2( \ + at::ScalarType::BFloat16, at::ScalarType::Half, output_dtype, "int4_scaled_mm_cpu_with_quant", [&] { \ + const scalar_t* __restrict__ A_data = input.data_ptr(); \ + scalar_t* __restrict__ c_ptr = output.data_ptr(); \ + at::parallel_for(0, M_a, 0, [&](int64_t begin, int64_t end) { \ + for (int64_t m = begin; m < end; ++m) { \ + quantize_row_int8(Aq_data + m * K_a, As_data[m], A_data + m * lda, K_a); \ + } \ + }); \ + _da8w4_linear_impl( \ + Aq_data, \ + As_data, \ + Azp_data, \ + b_ptr, \ + b_scales_ptr, \ + b_qzeros_ptr, \ + bias_ptr, \ + c_ptr, \ + c_temp_ptr, \ + dqB_temp_ptr, \ + M_a, \ + N, \ + K_a, \ + num_groups); \ }); LAUNCH_DA8W4_LINEAR_WITH_QUANT_IMPL(sym_quant_act); return output; } - -namespace { - template -inline void copy_stub(scalar_t* __restrict__ out, - const float* __restrict__ input, int64_t size) { +inline void copy_stub(scalar_t* __restrict__ out, const float* __restrict__ input, int64_t size) { using Vec = at::vec::Vectorized; using fVec = at::vec::Vectorized; +// no remainder #pragma GCC unroll 4 for (int64_t d = 0; d < size; d += Vec::size()) { fVec x0 = fVec::loadu(input + d); @@ -716,40 +834,61 @@ inline void copy_stub(scalar_t* __restrict__ out, } } -} // anonymous namespace - template -void tinygemm_kernel(scalar_t* C, float* C_temp, const uint8_t* A, - const float* scales_a, const int32_t* qzeros_a, - const uint8_t* B, const float* scales_b, - const int8_t* qzeros_b, const int32_t* compensation, - int8_t* dqB_tmp, int64_t M, int64_t K, int64_t lda, - int64_t ldc_f, int64_t ldc_s, bool store_out, - bool use_brgemm) { - _dequant_gemm_accum( - C_temp, A, scales_a, qzeros_a, B, scales_b, qzeros_b, compensation, - dqB_tmp, M, K, lda, ldc_f, use_brgemm); +void tinygemm_kernel( + scalar_t* C, + float* C_temp, + const uint8_t* A, + const float* scales_a, + const int32_t* qzeros_a, + const uint8_t* B, + const float* scales_b, + const int8_t* qzeros_b, + const int32_t* compensation, + int8_t* dqB_tmp, + int64_t M, + int64_t K, + int64_t lda, + int64_t ldc_f, + int64_t ldc_s, + bool store_out, + bool use_brgemm) { + // TODO: add sym quant act, now only asym + _dequant_gemm_accum( + C_temp, A, scales_a, qzeros_a, B, scales_b, qzeros_b, compensation, dqB_tmp, M, K, lda, ldc_f, use_brgemm); if (store_out) { + // copy from Ctmp to C for (int64_t m = 0; m < M; ++m) { copy_stub(C + m * ldc_s, C_temp + m * ldc_f, BLOCK_N); } } } -#define INSTANTIATE_TINYGEMM_TEMPLATE(TYPE) \ - template void tinygemm_kernel( \ - TYPE * C, float* C_temp, const uint8_t* A, const float* scales_a, \ - const int32_t* qzeros_a, const uint8_t* B, const float* scales_b, \ - const int8_t* qzeros_b, const int32_t* compensation, int8_t* dqB_tmp, \ - int64_t M, int64_t K, int64_t lda, int64_t ldc_f, int64_t ldc_s, \ - bool store_out, bool use_brgemm) +#define INSTANTIATE_TINYGEMM_TEMPLATE(TYPE) \ + template void tinygemm_kernel( \ + TYPE * C, \ + float* C_temp, \ + const uint8_t* A, \ + const float* scales_a, \ + const int32_t* qzeros_a, \ + const uint8_t* B, \ + const float* scales_b, \ + const int8_t* qzeros_b, \ + const int32_t* compensation, \ + int8_t* dqB_tmp, \ + int64_t M, \ + int64_t K, \ + int64_t lda, \ + int64_t ldc_f, \ + int64_t ldc_s, \ + bool store_out, \ + bool use_brgemm) INSTANTIATE_TINYGEMM_TEMPLATE(at::BFloat16); INSTANTIATE_TINYGEMM_TEMPLATE(at::Half); -at::Tensor int4_scaled_mm_cpu(at::Tensor& x, at::Tensor& w, at::Tensor& w_zeros, - at::Tensor& w_scales, - std::optional bias) { - return int4_scaled_mm_cpu_with_quant(x, w, w_scales, w_zeros, bias, - x.scalar_type()); +// int4 gemm dispatch api register +at::Tensor int4_scaled_mm_cpu( + at::Tensor& x, at::Tensor& w, at::Tensor& w_zeros, at::Tensor& w_scales, std::optional bias) { + return int4_scaled_mm_cpu_with_quant(x, w, w_scales, w_zeros, bias, x.scalar_type()); } diff --git a/csrc/cpu/sgl-kernels/gemm_int8.cpp b/csrc/cpu/sgl-kernels/gemm_int8.cpp index 4d6560cceb..c9b79de801 100644 --- a/csrc/cpu/sgl-kernels/gemm_int8.cpp +++ b/csrc/cpu/sgl-kernels/gemm_int8.cpp @@ -1,20 +1,83 @@ // Adapted from // https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu -#include "common.h" -#include "vec.h" -#include "gemm.h" - // clang-format off +#include "common.h" +#include "gemm.h" +#include "vec.h" + namespace { +template +struct scale_C { + static inline void apply( + scalar_t* __restrict__ C, + const int32_t* __restrict__ Ctmp, + const int32_t* __restrict__ Bcomp, + const float* __restrict__ bias, + float As, + const float* __restrict__ Bs) { + TORCH_CHECK(false, "scale_C: scalar path not implemented!"); + } +}; + +#if defined(CPU_CAPABILITY_AVX512) +template +struct scale_C { + static inline void apply( + at::BFloat16* __restrict__ C, + const int32_t* __restrict__ Ctmp, + const int32_t* __restrict__ Bcomp, + const float* __restrict__ bias, + float As, + const float* __restrict__ Bs) { + constexpr int COLS = BLOCK_N / 16; + static_assert(COLS % 2 == 0); + + __m512 vc[COLS]; + __m512 vd0 = _mm512_set1_ps(As); + + auto compute = [&](auto col) { + __m512 vd1 = _mm512_loadu_ps(Bs + col * 16); + __m512i vcomp = _mm512_loadu_si512(Bcomp + col * 16); + __m512i vc32 = _mm512_loadu_si512(Ctmp + col * 16); + vc[col] = _mm512_cvtepi32_ps(_mm512_sub_epi32(vc32, vcomp)); + if constexpr (has_bias) { + __m512 vbias = _mm512_loadu_ps(bias + col * 16); + vc[col] = _mm512_fmadd_ps(_mm512_mul_ps(vc[col], vd0), vd1, vbias); + } else { + vc[col] = _mm512_mul_ps(_mm512_mul_ps(vc[col], vd0), vd1); + } + }; + Unroll{}(compute); + + auto storec = [&](auto col) { + // for COLS = 2, 4 use 512bit store + if constexpr (col % 2 == 0) { + _mm512_storeu_si512( + reinterpret_cast<__m512i*>((C + col * 16)), (__m512i)(_mm512_cvtne2ps_pbh(vc[col + 1], vc[col + 0]))); + } + }; + Unroll{}(storec); + } +}; +#endif + template struct tinygemm_kernel_nn { static inline void apply( - const uint8_t* __restrict__ A, const int8_t* __restrict__ B, scalar_t* __restrict__ C, - const float* __restrict__ As, const float* __restrict__ Bs, const int32_t* __restrict__ Bcomp, - const float* __restrict__ bias, int64_t K, int64_t lda, int64_t ldb, int64_t ldc) { + const uint8_t* __restrict__ A, + const int8_t* __restrict__ B, + scalar_t* __restrict__ C, + const float* __restrict__ As, + const float* __restrict__ Bs, + const int32_t* __restrict__ Bcomp, + const float* __restrict__ bias, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { TORCH_CHECK(false, "tinygemm_kernel_nn: scalar path not implemented!"); } }; @@ -23,10 +86,17 @@ struct tinygemm_kernel_nn { template struct tinygemm_kernel_nn { static inline void apply( - const uint8_t* __restrict__ A, const int8_t* __restrict__ B, at::BFloat16* __restrict__ C, - const float* __restrict__ As, const float* __restrict__ Bs, const int32_t* __restrict__ Bcomp, - const float* __restrict__ bias, int64_t K, int64_t lda, int64_t ldb, int64_t ldc) { - + const uint8_t* __restrict__ A, + const int8_t* __restrict__ B, + at::BFloat16* __restrict__ C, + const float* __restrict__ As, + const float* __restrict__ Bs, + const int32_t* __restrict__ Bcomp, + const float* __restrict__ bias, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { constexpr int ROWS = BLOCK_M; constexpr int COLS = BLOCK_N / 16; static_assert(COLS % 2 == 0); @@ -38,10 +108,10 @@ struct tinygemm_kernel_nn { __m512i vb[COLS]; __m512i vc[ROWS * COLS]; __m512i vcomp[COLS]; - __m512 vd0; - __m512 vd1[COLS]; + __m512 vd0; + __m512 vd1[COLS]; - // oops! 4x4 spills but luckily we use 4x2 + // oops! 4x4 spills but we use 4x2 __m512 vbias[COLS]; // [NOTE]: s8s8 igemm compensation in avx512-vnni @@ -54,14 +124,12 @@ struct tinygemm_kernel_nn { // 1) 128 * b is pre-computed when packing B to vnni formats // 2) a + 128 is fused when dynamically quantize A // - auto loadc = [&](auto i) { - vc[i] = _mm512_set1_epi32(0); - }; + auto loadc = [&](auto i) { vc[i] = _mm512_set1_epi32(0); }; Unroll{}(loadc); const int64_t K4 = K >> 2; const int64_t lda4 = lda >> 2; - const int64_t ldb4 = ldb; // ldb * 4 >> 2; + const int64_t ldb4 = ldb; // ldb * 4 >> 2; const int32_t* a_ptr = reinterpret_cast(A); const int32_t* b_ptr = reinterpret_cast(B); @@ -89,7 +157,7 @@ struct tinygemm_kernel_nn { constexpr int col = i % COLS; // load a scale - if constexpr(col == 0) { + if constexpr (col == 0) { vd0 = _mm512_set1_ps(As[row]); } // load b scale and vcomp per 2 vectors @@ -120,8 +188,7 @@ struct tinygemm_kernel_nn { } _mm512_storeu_si512( - reinterpret_cast<__m512i*>((C + row * ldc + col * 16)), - (__m512i)(_mm512_cvtne2ps_pbh(vc1, vc0))); + reinterpret_cast<__m512i*>((C + row * ldc + col * 16)), (__m512i)(_mm512_cvtne2ps_pbh(vc1, vc0))); } }; Unroll{}(storec); @@ -129,11 +196,19 @@ struct tinygemm_kernel_nn { }; #endif -#define LAUNCH_TINYGEMM_KERNEL_NN(MB_SIZE, NB_SIZE) \ - tinygemm_kernel_nn::apply( \ - A + mb_start * lda, B + nb_start * 4, C + mb_start * ldc + nb_start, \ - As + mb_start, Bs + nb_start, Bcomp + nb_start, \ - has_bias ? bias + nb_start : nullptr, K, lda, ldb, ldc); +#define LAUNCH_TINYGEMM_KERNEL_NN(MB_SIZE, NB_SIZE) \ + tinygemm_kernel_nn::apply( \ + A + mb_start * lda, \ + B + nb_start * 4, \ + C + mb_start * ldc + nb_start, \ + As + mb_start, \ + Bs + nb_start, \ + Bcomp + nb_start, \ + has_bias ? bias + nb_start : nullptr, \ + K, \ + lda, \ + ldb, \ + ldc); template void tinygemm_kernel( @@ -151,10 +226,20 @@ void tinygemm_kernel( int64_t ldb, int64_t ldc, bool brg) { - // B compensation const int32_t* Bcomp = reinterpret_cast(B + block_size_n() * K); + if (brg) { + constexpr int BLOCK_N = block_size_n(); + at::native::cpublas::brgemm(M, N, K, lda, ldb, BLOCK_N, /* add_C */ false, A, B, Ctmp); + + // apply compensation and scale + for (int64_t m = 0; m < M; ++m) { + scale_C::apply(C + m * ldc, Ctmp + m * BLOCK_N, Bcomp, bias, As[m], Bs); + } + return; + } + // pattern: 1-4-16 constexpr int64_t BLOCK_M = 4; constexpr int64_t BLOCK_N = 64; @@ -167,26 +252,43 @@ void tinygemm_kernel( int64_t nb_start = nb * BLOCK_N; int64_t nb_size = std::min(BLOCK_N, N - nb_start); - switch(mb_size << 4 | nb_size >> 4) { + switch (mb_size << 4 | nb_size >> 4) { // mb_size = 1 - case 0x12: LAUNCH_TINYGEMM_KERNEL_NN(1, 32); break; - case 0x14: LAUNCH_TINYGEMM_KERNEL_NN(1, 64); break; + case 0x12: + LAUNCH_TINYGEMM_KERNEL_NN(1, 32); + break; + case 0x14: + LAUNCH_TINYGEMM_KERNEL_NN(1, 64); + break; // mb_size = 2 - case 0x22: LAUNCH_TINYGEMM_KERNEL_NN(2, 32); break; - case 0x24: LAUNCH_TINYGEMM_KERNEL_NN(2, 64); break; + case 0x22: + LAUNCH_TINYGEMM_KERNEL_NN(2, 32); + break; + case 0x24: + LAUNCH_TINYGEMM_KERNEL_NN(2, 64); + break; // mb_size = 3 - case 0x32: LAUNCH_TINYGEMM_KERNEL_NN(3, 32); break; - case 0x34: LAUNCH_TINYGEMM_KERNEL_NN(3, 64); break; + case 0x32: + LAUNCH_TINYGEMM_KERNEL_NN(3, 32); + break; + case 0x34: + LAUNCH_TINYGEMM_KERNEL_NN(3, 64); + break; // mb_size = 4 - case 0x42: LAUNCH_TINYGEMM_KERNEL_NN(4, 32); break; - case 0x44: LAUNCH_TINYGEMM_KERNEL_NN(4, 64); break; - default: TORCH_CHECK(false, "Unexpected block size, ", mb_size, "x", nb_size); + case 0x42: + LAUNCH_TINYGEMM_KERNEL_NN(4, 32); + break; + case 0x44: + LAUNCH_TINYGEMM_KERNEL_NN(4, 64); + break; + default: + TORCH_CHECK(false, "Unexpected block size, ", mb_size, "x", "nb_size"); } } } } -template +template void int8_scaled_mm_kernel_impl( scalar_t* __restrict__ out, const uint8_t* __restrict__ mat1, @@ -197,28 +299,22 @@ void int8_scaled_mm_kernel_impl( int64_t M, int64_t N, int64_t K) { - constexpr int64_t BLOCK_M = block_size_m(); constexpr int64_t BLOCK_N = block_size_n(); const int64_t MB = div_up(M, BLOCK_M); const int64_t NB = div_up(N, BLOCK_N); - // TODO: brgemm u8s8 depends on PyTorch 2.7 release. - const bool use_brgemm = false; + const bool use_brgemm = can_use_brgemm(M); // K + 4 after compensation const int64_t packed_row_size = get_row_size(K); AT_DISPATCH_BOOL(bias != nullptr, has_bias, [&] { - at::parallel_for(0, MB * NB, 0, [&](int64_t begin, int64_t end) { - int64_t mb{0}, nb{0}; - data_index_init(begin, mb, MB, nb, NB); - + parallel_2d(MB, NB, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { // for brgemm, use int32_t for accumulate alignas(64) int32_t Ctmp[BLOCK_M * BLOCK_N]; - for (int i = begin; i < end; ++i) { - UNUSED(i); + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * K, [&](int64_t mb, int64_t nb, int64_t nb_offset) { int mb_start = mb * BLOCK_M; int mb_size = std::min(M - mb_start, BLOCK_M); int nb_start = nb * BLOCK_N; @@ -239,10 +335,7 @@ void int8_scaled_mm_kernel_impl( /* ldb */ nb_size, /* ldc */ N, /* brg */ use_brgemm); - - // move to the next index - data_index_step(mb, MB, nb, NB); - } + }); if (use_brgemm) { at::native::cpublas::brgemm_release(); @@ -251,28 +344,47 @@ void int8_scaled_mm_kernel_impl( }); } -} // anonymous namespace +} // anonymous namespace // tinygemm interface template -void tinygemm_kernel(const uint8_t* __restrict__ A, const int8_t* __restrict__ B, scalar_t* __restrict__ C, - int32_t* __restrict__ Ctmp, const float* __restrict__ As, const float* __restrict__ Bs, - int64_t M, int64_t N, int64_t K, int64_t lda, int64_t ldb, int64_t ldc, bool brg) { +void tinygemm_kernel( + const uint8_t* __restrict__ A, + const int8_t* __restrict__ B, + scalar_t* __restrict__ C, + int32_t* __restrict__ Ctmp, + const float* __restrict__ As, + const float* __restrict__ Bs, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc, + bool brg) { tinygemm_kernel(A, B, C, Ctmp, As, Bs, nullptr, M, N, K, lda, ldb, ldc, brg); } -#define INSTANTIATE_TINYGEMM_TEMPLATE(TYPE) \ - template void tinygemm_kernel( \ - const uint8_t* __restrict__ A, const int8_t* __restrict__ B, TYPE* __restrict__ C, \ - int32_t* __restrict__ Ctmp, const float* __restrict__ As, const float* __restrict__ Bs, \ - int64_t M, int64_t N, int64_t K, int64_t lda, int64_t ldb, int64_t ldc, bool brg) +#define INSTANTIATE_TINYGEMM_TEMPLATE(TYPE) \ + template void tinygemm_kernel( \ + const uint8_t* __restrict__ A, \ + const int8_t* __restrict__ B, \ + TYPE* __restrict__ C, \ + int32_t* __restrict__ Ctmp, \ + const float* __restrict__ As, \ + const float* __restrict__ Bs, \ + int64_t M, \ + int64_t N, \ + int64_t K, \ + int64_t lda, \ + int64_t ldb, \ + int64_t ldc, \ + bool brg) INSTANTIATE_TINYGEMM_TEMPLATE(at::BFloat16); INSTANTIATE_TINYGEMM_TEMPLATE(at::Half); std::tuple per_token_quant_int8_cpu(at::Tensor& A) { - RECORD_FUNCTION("sgl-kernel::per_token_quant_int8_cpu", std::vector({A})); - CHECK_LAST_DIM_CONTIGUOUS_INPUT(A); CHECK_DIM(2, A); @@ -281,8 +393,7 @@ std::tuple per_token_quant_int8_cpu(at::Tensor& A) { int64_t lda = A.stride(0); const auto st = A.scalar_type(); - TORCH_CHECK(st == at::kBFloat16 || st == at::kHalf, - "per_token_quant_int8: expect A to be bfloat16 or half."); + TORCH_CHECK(st == at::kBFloat16 || st == at::kHalf, "per_token_quant_int8: expect A to be bfloat16 or half."); auto Aq = at::empty({M, K}, A.options().dtype(at::kByte)); auto As = at::empty({M}, A.options().dtype(at::kFloat)); @@ -292,13 +403,9 @@ std::tuple per_token_quant_int8_cpu(at::Tensor& A) { float* __restrict__ As_data = As.data_ptr(); const scalar_t* __restrict__ A_data = A.data_ptr(); - at::parallel_for(0, M, 0, [&] (int64_t begin, int64_t end) { + at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) { for (int64_t m = begin; m < end; ++m) { - quantize_row_int8( - Aq_data + m * K, - As_data[m], - A_data + m * lda, - K); + quantize_row_int8(Aq_data + m * K, As_data[m], A_data + m * lda, K); } }); }); @@ -315,11 +422,14 @@ std::tuple per_token_quant_int8_cpu(at::Tensor& A) { // bias : [N] // out : [M, N] // -at::Tensor int8_scaled_mm_cpu(at::Tensor& mat1, at::Tensor& mat2, - at::Tensor& scales1, at::Tensor& scales2, - std::optional& bias, at::ScalarType out_dtype, bool is_vnni) { - RECORD_FUNCTION("sgl-kernel::int8_scaled_mm_cpu", std::vector({mat1, mat2, scales1, scales2, bias})); - +at::Tensor int8_scaled_mm_cpu( + at::Tensor& mat1, + at::Tensor& mat2, + at::Tensor& scales1, + at::Tensor& scales2, + const std::optional& bias, + at::ScalarType out_dtype, + bool is_vnni) { auto packed_w = is_vnni ? mat2 : convert_weight_packed(mat2); CHECK_INPUT(mat1); @@ -340,7 +450,8 @@ at::Tensor int8_scaled_mm_cpu(at::Tensor& mat1, at::Tensor& mat2, TORCH_CHECK(mat1.scalar_type() == at::kByte, "int8_scaled_mm: expect mat1 to be uint8."); TORCH_CHECK(mat2.scalar_type() == at::kChar, "int8_scaled_mm: expect mat2 to be int8."); - TORCH_CHECK(scales1.scalar_type() == at::kFloat && scales2.scalar_type() == at::kFloat, + TORCH_CHECK( + scales1.scalar_type() == at::kFloat && scales2.scalar_type() == at::kFloat, "int8_scaled_mm: expect scales to be float32."); auto out = at::empty({M, N}, mat1.options().dtype(out_dtype)); @@ -368,10 +479,13 @@ at::Tensor int8_scaled_mm_cpu(at::Tensor& mat1, at::Tensor& mat2, } // fused `per_token_quant_int8_cpu` and `int8_scaled_mm_cpu` -at::Tensor int8_scaled_mm_with_quant(at::Tensor& mat1, at::Tensor& mat2, at::Tensor& scales2, - const std::optional& bias, at::ScalarType out_dtype, bool is_vnni) { - RECORD_FUNCTION("sgl-kernel::int8_scaled_mm_cpu", std::vector({mat1, mat2, scales2, bias})); - +at::Tensor int8_scaled_mm_with_quant( + at::Tensor& mat1, + at::Tensor& mat2, + at::Tensor& scales2, + const std::optional& bias, + at::ScalarType out_dtype, + bool is_vnni) { auto packed_w = is_vnni ? mat2 : convert_weight_packed(mat2); CHECK_LAST_DIM_CONTIGUOUS_INPUT(mat1); @@ -390,14 +504,10 @@ at::Tensor int8_scaled_mm_with_quant(at::Tensor& mat1, at::Tensor& mat2, at::Ten CHECK_EQ(scales2.numel(), N); const auto st = mat1.scalar_type(); - TORCH_CHECK(st == at::kBFloat16 || st == at::kHalf, - "int8_scaled_mm_with_quant: expect A to be bfloat16 or half."); - TORCH_CHECK(st == out_dtype, - "int8_scaled_mm_with_quant: expect A has same dtype with out_dtype."); - TORCH_CHECK(mat2.scalar_type() == at::kChar, - "int8_scaled_mm_with_quant: expect mat2 to be int8."); - TORCH_CHECK(scales2.scalar_type() == at::kFloat, - "int8_scaled_mm_with_quant: expect scales to be float32."); + TORCH_CHECK(st == at::kBFloat16 || st == at::kHalf, "int8_scaled_mm_with_quant: expect A to be bfloat16 or half."); + TORCH_CHECK(st == out_dtype, "int8_scaled_mm_with_quant: expect A has same dtype with out_dtype."); + TORCH_CHECK(mat2.scalar_type() == at::kChar, "int8_scaled_mm_with_quant: expect mat2 to be int8."); + TORCH_CHECK(scales2.scalar_type() == at::kFloat, "int8_scaled_mm_with_quant: expect scales to be float32."); const int64_t buffer_size = M * K + M * sizeof(float); auto buffer = at::empty({buffer_size}, mat1.options().dtype(at::kByte)); @@ -415,13 +525,9 @@ at::Tensor int8_scaled_mm_with_quant(at::Tensor& mat1, at::Tensor& mat2, at::Ten float* __restrict__ As_data = (float*)((void*)(Aq_data + M * K)); const scalar_t* __restrict__ A_data = mat1.data_ptr(); - at::parallel_for(0, M, 0, [&] (int64_t begin, int64_t end) { + at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) { for (int64_t m = begin; m < end; ++m) { - quantize_row_int8( - Aq_data + m * K, - As_data[m], - A_data + m * lda, - K); + quantize_row_int8(Aq_data + m * K, As_data[m], A_data + m * lda, K); } }); diff --git a/csrc/cpu/sgl-kernels/moe.cpp b/csrc/cpu/sgl-kernels/moe.cpp index c01bfd376d..c248f678be 100644 --- a/csrc/cpu/sgl-kernels/moe.cpp +++ b/csrc/cpu/sgl-kernels/moe.cpp @@ -1,12 +1,13 @@ // Adapted from // https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu -#include "common.h" -#include "vec.h" -#include "gemm.h" - // clang-format off +#include "moe.h" + +#include "common.h" +#include "gemm.h" + namespace { // [NOTE]: Fused MoE kernel with AMX @@ -30,109 +31,6 @@ namespace { // 3. abstract at::native::cpublas::brgemm with WoQ gemm (M = 1 & M != 1) // -template -inline void fill_stub(scalar_t* __restrict__ out, scalar_t val, int64_t size) { - using Vec = at::vec::Vectorized; - const Vec data_vec(val); - at::vec::map([data_vec](Vec out) { return out = data_vec; }, out, out, size); -} - -template -inline void copy_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ input, int64_t size) { - using Vec = at::vec::Vectorized; - // no remainder - #pragma GCC unroll 4 - for (int64_t d = 0; d < size; d += Vec::size()) { - Vec data = Vec::loadu(input + d); - data.store(out + d); - } -} - -template -inline void copy_mul_stub(scalar_t* __restrict__ out, const float* __restrict__ input, float weight, int64_t size) { - using bVec = at::vec::Vectorized; - using fVec = at::vec::Vectorized; - constexpr int kVecSize = bVec::size(); - const fVec weight_vec = fVec(weight); - int64_t d; - #pragma GCC unroll 4 - for (d = 0; d <= size - kVecSize; d += kVecSize) { - fVec data0 = fVec::loadu(input + d) * weight_vec; - fVec data1 = fVec::loadu(input + d + fVec::size()) * weight_vec; - bVec out_vec = convert_from_float_ext(data0, data1); - out_vec.store(out + d); - } - for (; d < size; ++d) { - out[d] = static_cast(input[d] * weight); - } -} - -// acc from [topk, K] to [K] -template -inline void sum_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ input, int64_t topk, int64_t K) { - using bVec = at::vec::Vectorized; - using fVec = at::vec::Vectorized; - constexpr int kVecSize = bVec::size(); - if (topk == 1) { - // do copy for topk = 1 - copy_stub(out, input, K); - } else { - // do sum for topk != 1 - int64_t d; - #pragma GCC unroll 4 - for (d = 0; d <= K - kVecSize; d += kVecSize) { - fVec sum_fvec0 = fVec(0.f); - fVec sum_fvec1 = fVec(0.f); - for (int t = 0; t < topk; ++t) { - bVec x_bvec = bVec::loadu(input + t * K + d); - fVec x_fvec0, x_fvec1; - std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec); - - sum_fvec0 += x_fvec0; - sum_fvec1 += x_fvec1; - } - bVec out_bvec = convert_from_float_ext(sum_fvec0, sum_fvec1); - out_bvec.store(out + d); - } - for (; d < K; ++d) { - float sum_val = 0.f; - for (int t = 0; t < topk; ++t) { - sum_val += static_cast(input[t * K + d]); - } - out[d] = static_cast(sum_val); - } - } -} - -// out = input + input2 * scale -template -inline void add_mul_stub(scalar_t* __restrict__ out, const float* __restrict__ input, - const scalar_t* __restrict__ input2, float scale, int64_t size) { - - using bVec = at::vec::Vectorized; - using fVec = at::vec::Vectorized; - constexpr int kVecSize = bVec::size(); - const fVec s_vec = fVec(scale); - int64_t d; - #pragma GCC unroll 4 - for (d = 0; d <= size - kVecSize; d += kVecSize) { - fVec x0 = fVec::loadu(input + d); - fVec x1 = fVec::loadu(input + d + fVec::size()); - - bVec y_bvec = bVec::loadu(input2 + d); - fVec y0, y1; - std::tie(y0, y1) = at::vec::convert_to_float(y_bvec); - - x0 = x0 + y0 * s_vec; - x1 = x1 + y1 * s_vec; - bVec out_vec = convert_from_float_ext(x0, x1); - out_vec.store(out + d); - } - for (; d < size; ++d) { - out[d] = static_cast(input[d] + float(input2[d]) * scale); - } -} - template int moe_align_block_size( int32_t* __restrict__ sorted_ids, @@ -144,8 +42,7 @@ int moe_align_block_size( int num_experts, int numel, int num_threads) { - - #define T_INDEX(tt) total_cnts + (tt) * num_experts +#define T_INDEX(tt) total_cnts + (tt) * num_experts // accumulate count of expert ids locally at::parallel_for(0, numel, 0, [&](int begin, int end) { @@ -160,8 +57,7 @@ int moe_align_block_size( using iVec = at::vec::Vectorized; for (int t = 0; t < num_threads; ++t) { at::vec::map2( - [](iVec x, iVec y) { return x + y; }, - T_INDEX(t + 1), T_INDEX(t + 1), T_INDEX(t), num_experts); + [](iVec x, iVec y) { return x + y; }, T_INDEX(t + 1), T_INDEX(t + 1), T_INDEX(t), num_experts); } // the last row holds sums of each experts @@ -201,7 +97,9 @@ int moe_align_block_size( // padding value for sorted_ids: numel auto sorted_id_size = [=](const int32_t* sorted_ids_ptr) { for (int d = 0; d < BLOCK_M; ++d) { - if (sorted_ids_ptr[d] == numel) { return d; } + if (sorted_ids_ptr[d] == numel) { + return d; + } } return BLOCK_M; }; @@ -215,7 +113,7 @@ int moe_align_block_size( offsets[mb + 1] = sorted_id_size(sorted_ids + mb * BLOCK_M); } }); - // TODO: do we need to vectorize this ? + // TODO: do we need to vecterize this ? for (int mb = 0; mb < num_token_blocks; ++mb) { offsets[mb + 1] += offsets[mb]; } @@ -236,7 +134,6 @@ inline void silu_and_mul( const float* __restrict__ input1, // y: y0, y1 int64_t m_size, int64_t N) { - using bVec = at::vec::Vectorized; using fVec = at::vec::Vectorized; @@ -269,8 +166,14 @@ inline void silu_and_mul( template struct tinygemm_kernel_nn2 { static inline void apply( - const scalar_t* __restrict__ A, const scalar_t* __restrict__ B0, const scalar_t* __restrict__ B1, - scalar_t* __restrict__ C, int64_t K, int64_t lda, int64_t ldb, int64_t ldc) { + const scalar_t* __restrict__ A, + const scalar_t* __restrict__ B0, + const scalar_t* __restrict__ B1, + scalar_t* __restrict__ C, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { TORCH_CHECK(false, "tinygemm_kernel_nn: scalar path not implemented!"); } }; @@ -279,9 +182,14 @@ struct tinygemm_kernel_nn2 { template struct tinygemm_kernel_nn2 { static inline void apply( - const at::BFloat16* __restrict__ A, const at::BFloat16* __restrict__ B0, const at::BFloat16* __restrict__ B1, - at::BFloat16* __restrict__ C, int64_t K, int64_t lda, int64_t ldb, int64_t ldc) { - + const at::BFloat16* __restrict__ A, + const at::BFloat16* __restrict__ B0, + const at::BFloat16* __restrict__ B1, + at::BFloat16* __restrict__ C, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { constexpr int ROWS = BLOCK_M; constexpr int COLS = BLOCK_N / 16; @@ -304,7 +212,7 @@ struct tinygemm_kernel_nn2 { const int64_t K2 = K >> 1; const int64_t lda2 = lda >> 1; - const int64_t ldb2 = ldb; // ldb * 2 >> 1; + const int64_t ldb2 = ldb; // ldb * 2 >> 1; const float* a_ptr = reinterpret_cast(A); const float* b0_ptr = reinterpret_cast(B0); const float* b1_ptr = reinterpret_cast(B1); @@ -352,17 +260,16 @@ struct tinygemm_kernel_nn2 { _mm512_storeu_si512( reinterpret_cast<__m512i*>((C + row * ldc + col * 16)), (__m512i)(_mm512_cvtne2ps_pbh(__m512(x1), __m512(x0)))); - } + } }; Unroll{}(storec); } }; #endif -#define LAUNCH_TINYGEMM_KERNEL_NN(MB_SIZE, NB_SIZE) \ - tinygemm_kernel_nn2::apply( \ - A + mb_start * lda, B0 + nb_start * 2, B1 + nb_start * 2, \ - C + mb_start * ldc + nb_start, K, lda, ldb, ldc); +#define LAUNCH_TINYGEMM_KERNEL_NN(MB_SIZE, NB_SIZE) \ + tinygemm_kernel_nn2::apply( \ + A + mb_start * lda, B0 + nb_start * 2, B1 + nb_start * 2, C + mb_start * ldc + nb_start, K, lda, ldb, ldc); template void tinygemm_kernel( @@ -376,7 +283,6 @@ void tinygemm_kernel( int64_t lda, int64_t ldb, int64_t ldc) { - // pattern: 1-(2+2)-(8+8) constexpr int64_t BLOCK_M = 4; constexpr int64_t BLOCK_N = 32; @@ -389,16 +295,25 @@ void tinygemm_kernel( int64_t nb_start = nb * BLOCK_N; int64_t nb_size = std::min(BLOCK_N, N - nb_start); - switch(mb_size << 4 | nb_size >> 4) { + switch (mb_size << 4 | nb_size >> 4) { // mb_size = 1 - case 0x12: LAUNCH_TINYGEMM_KERNEL_NN(1, 32); break; + case 0x12: + LAUNCH_TINYGEMM_KERNEL_NN(1, 32); + break; // mb_size = 2 - case 0x22: LAUNCH_TINYGEMM_KERNEL_NN(2, 32); break; + case 0x22: + LAUNCH_TINYGEMM_KERNEL_NN(2, 32); + break; // mb_size = 3 - case 0x32: LAUNCH_TINYGEMM_KERNEL_NN(3, 32); break; + case 0x32: + LAUNCH_TINYGEMM_KERNEL_NN(3, 32); + break; // mb_size = 4 - case 0x42: LAUNCH_TINYGEMM_KERNEL_NN(4, 32); break; - default: TORCH_CHECK(false, "Unexpected block size, ", mb_size, "x", nb_size); + case 0x42: + LAUNCH_TINYGEMM_KERNEL_NN(4, 32); + break; + default: + TORCH_CHECK(false, "Unexpected block size, ", mb_size, "x", "nb_size"); } } } @@ -407,8 +322,13 @@ void tinygemm_kernel( template struct tinygemm_kernel_nn { static inline void apply( - const scalar_t* __restrict__ A, const scalar_t* __restrict__ B, float* __restrict__ C, - int64_t K, int64_t lda, int64_t ldb, int64_t ldc) { + const scalar_t* __restrict__ A, + const scalar_t* __restrict__ B, + float* __restrict__ C, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { TORCH_CHECK(false, "tinygemm_kernel_nn: scalar path not implemented!"); } }; @@ -417,9 +337,13 @@ struct tinygemm_kernel_nn { template struct tinygemm_kernel_nn { static inline void apply( - const at::BFloat16* __restrict__ A, const at::BFloat16* __restrict__ B, float* __restrict__ C, - int64_t K, int64_t lda, int64_t ldb, int64_t ldc) { - + const at::BFloat16* __restrict__ A, + const at::BFloat16* __restrict__ B, + float* __restrict__ C, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { constexpr int ROWS = BLOCK_M; constexpr int COLS = BLOCK_N / 16; @@ -432,14 +356,12 @@ struct tinygemm_kernel_nn { __m512bh vb[COLS]; __m512 vc[ROWS * COLS]; - auto loadc = [&](auto i) { - vc[i] = _mm512_set1_ps(0.f); - }; + auto loadc = [&](auto i) { vc[i] = _mm512_set1_ps(0.f); }; Unroll{}(loadc); const int64_t K2 = K >> 1; const int64_t lda2 = lda >> 1; - const int64_t ldb2 = ldb; // ldb * 2 >> 1; + const int64_t ldb2 = ldb; // ldb * 2 >> 1; const float* a_ptr = reinterpret_cast(A); const float* b_ptr = reinterpret_cast(B); @@ -466,17 +388,15 @@ struct tinygemm_kernel_nn { constexpr int row = i / COLS; constexpr int col = i % COLS; _mm512_storeu_ps(reinterpret_cast<__m512*>(C + row * ldc + col * 16), vc[i]); - }; Unroll{}(storec); } }; #endif -#define LAUNCH_TINYGEMM_KERNEL_NN2(MB_SIZE, NB_SIZE) \ - tinygemm_kernel_nn::apply( \ - A + mb_start * lda, B + nb_start * 2, C + mb_start * ldc + nb_start, \ - K, lda, ldb, ldc); +#define LAUNCH_TINYGEMM_KERNEL_NN2(MB_SIZE, NB_SIZE) \ + tinygemm_kernel_nn::apply( \ + A + mb_start * lda, B + nb_start * 2, C + mb_start * ldc + nb_start, K, lda, ldb, ldc); template void tinygemm_kernel( @@ -489,7 +409,6 @@ void tinygemm_kernel( int64_t lda, int64_t ldb, int64_t ldc) { - // pattern: 1-2-8 constexpr int64_t BLOCK_M = 4; constexpr int64_t BLOCK_N = 32; @@ -502,16 +421,25 @@ void tinygemm_kernel( int64_t nb_start = nb * BLOCK_N; int64_t nb_size = std::min(BLOCK_N, N - nb_start); - switch(mb_size << 4 | nb_size >> 4) { + switch (mb_size << 4 | nb_size >> 4) { // mb_size = 1 - case 0x12: LAUNCH_TINYGEMM_KERNEL_NN2(1, 32); break; + case 0x12: + LAUNCH_TINYGEMM_KERNEL_NN2(1, 32); + break; // mb_size = 2 - case 0x22: LAUNCH_TINYGEMM_KERNEL_NN2(2, 32); break; + case 0x22: + LAUNCH_TINYGEMM_KERNEL_NN2(2, 32); + break; // mb_size = 3 - case 0x32: LAUNCH_TINYGEMM_KERNEL_NN2(3, 32); break; + case 0x32: + LAUNCH_TINYGEMM_KERNEL_NN2(3, 32); + break; // mb_size = 4 - case 0x42: LAUNCH_TINYGEMM_KERNEL_NN2(4, 32); break; - default: TORCH_CHECK(false, "Unexpected block size, ", mb_size, "x", nb_size); + case 0x42: + LAUNCH_TINYGEMM_KERNEL_NN2(4, 32); + break; + default: + TORCH_CHECK(false, "Unexpected block size, ", mb_size, "x", "nb_size"); } } } @@ -537,7 +465,6 @@ void fused_experts_kernel_impl( int64_t E, int64_t topk, int64_t num_tokens_post_pad) { - // handle 2 tiles per block constexpr int64_t BLOCK_M = block_size_m(); constexpr int64_t BLOCK_N = block_size_n(); @@ -552,39 +479,36 @@ void fused_experts_kernel_impl( const int64_t stride_e = 2 * N * K; const int64_t stride_n = K; + int64_t avg_M = std::max(int64_t(1), M * topk / E); + const bool use_brgemm = can_use_brgemm(avg_M); + // here we only parallel on half of 2N to fuse silu_and_mul with gemm - at::parallel_for(0, MB * NB, 0, [&](int64_t begin, int64_t end) { + parallel_2d(MB, NB, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { // get local pointers - int tid = at::get_thread_num(); + int tid = get_thread_num(); scalar_t* __restrict__ A = A_tmp + tid * BLOCK_M * K; float* __restrict__ C0 = C_tmp + tid * 2 * BLOCK_M * BLOCK_N; float* __restrict__ C1 = C0 + BLOCK_M * BLOCK_N; - bool is_brgemm_used = false; - - for (int64_t i = begin; i < end; ++i) { - int64_t mb = i / NB; - int64_t nb = i % NB; - - // nb0 from top half and nb1 from bottom half - int64_t nb0 = nb, nb1 = nb + NB; - int64_t n_size = std::min(N - nb0 * BLOCK_N, BLOCK_N); + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * K * 2, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + // nb_upper from top half and nb_lower from bottom half + int64_t nb_upper = nb, nb_lower = nb + NB; + int64_t n_size = std::min(N - nb * BLOCK_N, BLOCK_N); // B shape [K, n_size] in vnni format int32_t expert_id = expert_ids[mb]; - const scalar_t* __restrict__ B0 = packed_w1 + expert_id * stride_e + nb0 * BLOCK_N * stride_n; - const scalar_t* __restrict__ B1 = packed_w1 + expert_id * stride_e + nb1 * BLOCK_N * stride_n; + const scalar_t* __restrict__ B0 = packed_w1 + expert_id * stride_e + nb_upper * BLOCK_N * stride_n; + const scalar_t* __restrict__ B1 = packed_w1 + expert_id * stride_e + nb_lower * BLOCK_N * stride_n; - // 1.a load A - const int32_t* A_ids = sorted_ids + mb * BLOCK_M; int64_t m_size = offsets[mb + 1] - offsets[mb]; - const bool use_brgemm = can_use_brgemm(m_size); - is_brgemm_used = is_brgemm_used || use_brgemm; - - for (int64_t m = 0; m < m_size; ++m) { - int32_t index = A_ids[m] / topk; - copy_stub(A + m * K, input + index * K, K); + if (nb_offset == 0) { + // 1.a load A + const int32_t* A_ids = sorted_ids + mb * BLOCK_M; + for (int64_t m = 0; m < m_size; ++m) { + int32_t index = A_ids[m] / topk; + copy_stub(A + m * K, input + index * K, K); + } } if (use_brgemm) { @@ -616,12 +540,7 @@ void fused_experts_kernel_impl( // 1.d silu and mul const int64_t offset = offsets[mb]; - silu_and_mul( - ic1 + offset * N + nb * BLOCK_N, - C0, - C1, - m_size, - N); + silu_and_mul(ic1 + offset * N + nb * BLOCK_N, C0, C1, m_size, N); } else { // fused 1.bcd: silu_and_mul(A @ B0, A @ B1) const int64_t offset = offsets[mb]; @@ -637,9 +556,9 @@ void fused_experts_kernel_impl( /* ldb */ n_size, /* ldc */ N); } - } + }); - if (is_brgemm_used) { + if (use_brgemm) { at::native::cpublas::brgemm_release(); } }); @@ -654,24 +573,16 @@ void fused_experts_kernel_impl( const int64_t stride_oc = IC; // parallel on [MB2, NB2] - at::parallel_for(0, MB2 * NB2, 0, [&](int64_t begin, int64_t end) { + parallel_2d(MB2, NB2, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { // get local pointers - int tid = at::get_thread_num(); + int tid = get_thread_num(); // we won't be using C1 for gemm2 float* __restrict__ C = C_tmp + tid * 2 * BLOCK_M * BLOCK_N; - bool is_brgemm_used = false; - - for (int64_t i = begin; i < end; ++i) { - int64_t mb = i / NB2; - int64_t nb = i % NB2; - + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * IC, [&](int64_t mb, int64_t nb, int64_t nb_offset) { int64_t m_size = offsets[mb + 1] - offsets[mb]; int64_t n_size = std::min(OC - nb * BLOCK_N, BLOCK_N); - const bool use_brgemm = can_use_brgemm(m_size); - is_brgemm_used = is_brgemm_used || use_brgemm; - // A ptr from ic1 of [M * topk, N] in sorted order // so as to avoid copy A to tmp buffer again const scalar_t* __restrict__ A = ic1 + offsets[mb] * N; @@ -714,9 +625,9 @@ void fused_experts_kernel_impl( float weight = topk_weights[index]; copy_mul_stub(ic2 + index * K + nb * BLOCK_N, C + m * BLOCK_N, weight, n_size); } - } + }); - if (is_brgemm_used) { + if (use_brgemm) { at::native::cpublas::brgemm_release(); } }); @@ -743,7 +654,6 @@ void shared_expert_kernel_impl( int64_t M, int64_t N, int64_t K) { - // handle 2 tiles per block constexpr int64_t BLOCK_M = block_size_m(); constexpr int64_t BLOCK_N = block_size_n(); @@ -755,36 +665,29 @@ void shared_expert_kernel_impl( TORCH_CHECK(N % BLOCK_N == 0, "Fixme when N is not multiples of ", BLOCK_N); const int64_t stride_n = K; + const bool use_brgemm = can_use_brgemm(M); + + const bool apply_scaling_factor = fused_experts_out != nullptr; + // here we only parallel on half of 2N to fuse silu_and_mul with gemm - at::parallel_for(0, MB * NB, 0, [&](int64_t begin, int64_t end) { + parallel_2d(MB, NB, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { // get local pointers - int tid = at::get_thread_num(); + int tid = get_thread_num(); float* __restrict__ C0 = C_tmp + tid * 2 * BLOCK_M * BLOCK_N; float* __restrict__ C1 = C0 + BLOCK_M * BLOCK_N; - bool is_brgemm_used = false; - - for (int64_t i = begin; i < end; ++i) { - int64_t mb = i / NB; - int64_t nb = i % NB; - - // nb0 from top half and nb1 from bottom half - int64_t nb0 = nb, nb1 = nb + NB; - int64_t n_size = std::min(N - nb0 * BLOCK_N, BLOCK_N); + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * K * 2, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + // nb_upper from top half and nb_lower from bottom half + int64_t nb_upper = nb, nb_lower = nb + NB; + int64_t n_size = std::min(N - nb * BLOCK_N, BLOCK_N); int64_t m_size = std::min(M - mb * BLOCK_M, BLOCK_M); - //int64_t mb_start = mb * BLOCK_M; - //int64_t mb_size = std::min(M - mb_start, BLOCK_M); - // A shape [m_size, K] const scalar_t* A = input + mb * BLOCK_M * K; // B shape [K, n_size] in vnni format - const scalar_t* __restrict__ B0 = packed_w1 + nb0 * BLOCK_N * stride_n; - const scalar_t* __restrict__ B1 = packed_w1 + nb1 * BLOCK_N * stride_n; - - const bool use_brgemm = can_use_brgemm(m_size); - is_brgemm_used = is_brgemm_used || use_brgemm; + const scalar_t* __restrict__ B0 = packed_w1 + nb_upper * BLOCK_N * stride_n; + const scalar_t* __restrict__ B1 = packed_w1 + nb_lower * BLOCK_N * stride_n; if (use_brgemm) { // 1.b gemm: C0 = A @ B0 @@ -814,12 +717,7 @@ void shared_expert_kernel_impl( /* C */ C1); // 1.d silu and mul - silu_and_mul( - ic1 + mb * BLOCK_M * N + nb * BLOCK_N, - C0, - C1, - m_size, - N); + silu_and_mul(ic1 + mb * BLOCK_M * N + nb * BLOCK_N, C0, C1, m_size, N); } else { // fused 1.bcd: silu_and_mul(A @ B0, A @ B1) tinygemm_kernel( @@ -834,9 +732,9 @@ void shared_expert_kernel_impl( /* ldb */ n_size, /* ldc */ N); } - } + }); - if (is_brgemm_used) { + if (use_brgemm) { at::native::cpublas::brgemm_release(); } }); @@ -850,24 +748,16 @@ void shared_expert_kernel_impl( const int64_t stride_oc = IC; // parallel on [MB2, NB2] - at::parallel_for(0, MB2 * NB2, 0, [&](int64_t begin, int64_t end) { + parallel_2d(MB2, NB2, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { // get local pointers - int tid = at::get_thread_num(); + int tid = get_thread_num(); // we won't be using C1 for gemm2 float* __restrict__ C = C_tmp + tid * 2 * BLOCK_M * BLOCK_N; - bool is_brgemm_used = false; - - for (int64_t i = begin; i < end; ++i) { - int64_t mb = i / NB2; - int64_t nb = i % NB2; - + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * IC, [&](int64_t mb, int64_t nb, int64_t nb_offset) { int64_t m_size = std::min(M - mb * BLOCK_M, BLOCK_M); int64_t n_size = std::min(OC - nb * BLOCK_N, BLOCK_N); - const bool use_brgemm = can_use_brgemm(m_size); - is_brgemm_used = is_brgemm_used || use_brgemm; - // A shape [m_size, IC] const scalar_t* __restrict__ A = ic1 + mb * BLOCK_M * N; @@ -902,19 +792,21 @@ void shared_expert_kernel_impl( // 2.b copy from C to output and add fused_experts_out scalar_t* __restrict__ out = output + mb * BLOCK_M * K + nb * BLOCK_N; - const scalar_t* __restrict__ fused_out = fused_experts_out + mb * BLOCK_M * K + nb * BLOCK_N; + const scalar_t* __restrict__ fused_out = + apply_scaling_factor ? fused_experts_out + mb * BLOCK_M * K + nb * BLOCK_N : nullptr; for (int64_t m = 0; m < m_size; ++m) { - add_mul_stub(out + m * K, C + m * BLOCK_N, fused_out + m * K, routed_scaling_factor, n_size); + const scalar_t* __restrict__ fused_out_row = apply_scaling_factor ? (fused_out + m * K) : nullptr; + add_mul_stub(out + m * K, C + m * BLOCK_N, fused_out_row, routed_scaling_factor, n_size); } - } + }); - if (is_brgemm_used) { + if (use_brgemm) { at::native::cpublas::brgemm_release(); } }); } -} // anonymous namespace +} // anonymous namespace // common checks static inline void check_moe_scales( @@ -922,14 +814,10 @@ static inline void check_moe_scales( bool use_fp8_w8a16, const std::optional& w1_scale, const std::optional& w2_scale, - const std::optional> block_size, - const std::optional& a1_scale, - const std::optional& a2_scale) { + const std::optional> block_size) { if (use_int8_w8a8) { TORCH_CHECK(w1_scale.has_value(), "missing w1_scale for int8 w8a8."); TORCH_CHECK(w2_scale.has_value(), "missing w2_scale for int8 w8a8."); - TORCH_CHECK(!a1_scale.has_value(), "static quantization for activation not supported."); - TORCH_CHECK(!a2_scale.has_value(), "static quantization for activation not supported."); } if (use_fp8_w8a16) { TORCH_CHECK(w1_scale.has_value(), "missing w1_scale for fp8 w8a16."); @@ -939,16 +827,16 @@ static inline void check_moe_scales( } } -#define CHECK_MOE_SCALES_FP8(DIM0, DIM1) \ - auto w1s = w1_scale.value(); \ - auto w2s = w2_scale.value(); \ - auto block_size_val = block_size.value(); \ - int64_t block_size_N = block_size_val[0]; \ - int64_t block_size_K = block_size_val[1]; \ - TORCH_CHECK(w1s.size(DIM0) == 2 * N / block_size_N); \ - TORCH_CHECK(w1s.size(DIM1) == K / block_size_K); \ - TORCH_CHECK(w2s.size(DIM0) == K / block_size_N); \ - TORCH_CHECK(w2s.size(DIM1) == N / block_size_K) +#define CHECK_MOE_SCALES_FP8(DIM0, DIM1) \ + auto w1s = w1_scale.value(); \ + auto w2s = w2_scale.value(); \ + auto block_size_val = block_size.value(); \ + int64_t block_size_N = block_size_val[0]; \ + int64_t block_size_K = block_size_val[1]; \ + TORCH_CHECK(w1s.size(DIM0) == div_up(2 * N, block_size_N)); \ + TORCH_CHECK(w1s.size(DIM1) == div_up(K, block_size_K)); \ + TORCH_CHECK(w2s.size(DIM0) == div_up(K, block_size_N)); \ + TORCH_CHECK(w2s.size(DIM1) == div_up(N, block_size_K)) // hidden_states: [M, K] // w1: [E, 2N, K] @@ -956,6 +844,7 @@ static inline void check_moe_scales( // topk_weights: [M, topk] // topk_ids: [M, topk] (int32_t) // + at::Tensor fused_experts_cpu( at::Tensor& hidden_states, at::Tensor& w1, @@ -963,16 +852,13 @@ at::Tensor fused_experts_cpu( at::Tensor& topk_weights, at::Tensor& topk_ids, bool inplace, - bool use_int8_w8a8, - bool use_fp8_w8a16, + int64_t moe_comp_method, const std::optional& w1_scale, const std::optional& w2_scale, + const std::optional& w1_zero, + const std::optional& w2_zero, const std::optional> block_size, - const std::optional& a1_scale, - const std::optional& a2_scale, bool is_vnni) { - RECORD_FUNCTION("sgl-kernel::fused_experts_cpu", std::vector({hidden_states, w1, w2, topk_weights, topk_ids})); - auto packed_w1 = is_vnni ? w1 : convert_weight_packed(w1); auto packed_w2 = is_vnni ? w2 : convert_weight_packed(w2); @@ -985,32 +871,49 @@ at::Tensor fused_experts_cpu( CHECK_INPUT(w2); CHECK_EQ(topk_weights.sizes(), topk_ids.sizes()); CHECK_DIM(2, hidden_states); - CHECK_DIM(3, w1); - CHECK_DIM(3, w2); + if (moe_comp_method == CPUQuantMethod::INT4_W4A8 && is_vnni) { + CHECK_DIM(4, w1); + CHECK_DIM(4, w2); + } else { + CHECK_DIM(3, w1); + CHECK_DIM(3, w2); + } CHECK_DIM(2, topk_weights); CHECK_DIM(2, topk_ids); CHECK_EQ(topk_ids.scalar_type(), at::kInt); - CHECK_EQ(topk_weights.scalar_type(), at::kFloat); + + // TODO: support topk_weights to be bf16 or fp16 in the kernel. + // The topk_weights of llama4 is computed via Llama4MoE:custom_routing_function and is bf16/fp16 + // while the kernel currently only supports it to be float32 + auto topk_weights_ = topk_weights.to(at::kFloat); + CHECK_EQ(topk_weights_.scalar_type(), at::kFloat); int64_t M = hidden_states.size(0); int64_t K = hidden_states.size(1); - int64_t N = w1.size(1) / 2; + int64_t N = moe_comp_method == CPUQuantMethod::INT4_W4A8 ? w1_scale.value().size(1) * w1_scale.value().size(3) / 2 + : w1.size(1) / 2; int64_t E = w1.size(0); - int64_t topk = topk_weights.size(1); + int64_t topk = topk_weights_.size(1); // we use int32_t compensation for int8 w8a8 - int64_t packed_K = get_row_size(K, use_int8_w8a8); - int64_t packed_N = get_row_size(N, use_int8_w8a8); + int64_t packed_K = get_row_size(K, moe_comp_method == CPUQuantMethod::INT8_W8A8); + int64_t packed_N = get_row_size(N, moe_comp_method == CPUQuantMethod::INT8_W8A8); // check weight shapes CHECK_EQ(w2.size(0), E); - CHECK_EQ(w2.size(1), K); - CHECK_EQ(packed_w1.size(2), packed_K); - CHECK_EQ(packed_w2.size(2), packed_N); - + if (!(moe_comp_method == CPUQuantMethod::INT4_W4A8)) { + CHECK_EQ(w2.size(1), K); + CHECK_EQ(packed_w1.size(2), packed_K / (moe_comp_method == CPUQuantMethod::INT4_W4A8 ? 2 : 1)); + CHECK_EQ(packed_w2.size(2), packed_N / (moe_comp_method == CPUQuantMethod::INT4_W4A8 ? 2 : 1)); + } // check scales - check_moe_scales(use_int8_w8a8, use_fp8_w8a16, w1_scale, w2_scale, block_size, a1_scale, a2_scale); + check_moe_scales( + moe_comp_method == CPUQuantMethod::INT8_W8A8, + moe_comp_method == CPUQuantMethod::FP8_W8A16, + w1_scale, + w2_scale, + block_size); at::Tensor out_hidden_states = inplace ? hidden_states : at::empty_like(hidden_states); @@ -1031,8 +934,8 @@ at::Tensor fused_experts_cpu( int32_t* __restrict__ sorted_ids = buffer.data_ptr(); int32_t* __restrict__ expert_ids = sorted_ids + max_num_tokens_padded; int32_t* __restrict__ total_cnts = expert_ids + max_num_blocks; - int32_t* __restrict__ cumsums = total_cnts + (num_threads + 1) * E; - int32_t* __restrict__ offsets = cumsums + (E + 1); + int32_t* __restrict__ cumsums = total_cnts + (num_threads + 1) * E; + int32_t* __restrict__ offsets = cumsums + (E + 1); // init sorted_ids with `numel` as the padding number // init expert_ids with `num_experts` @@ -1064,26 +967,31 @@ at::Tensor fused_experts_cpu( // // for fp8 w8a16: // 7. intermediate_cache0 : [M * topk, 2N] - // 8. B_tmp : [T, BLOCK_N, std::max(K, N)] + // 8. B_tmp : [T, MAX_CACHE_BLOCK_SIZE, BLOCK_N, std::max(K, N)] // - int64_t buffer_size_nbytes = M * topk * N * 2 + M * topk * K * 2 + - num_threads * BLOCK_M * K * (use_int8_w8a8 ? 1 : 2) + + int64_t buffer_size_nbytes = + M * topk * N * 2 + M * topk * K * 2 + + num_threads * BLOCK_M * K * + (moe_comp_method == CPUQuantMethod::INT8_W8A8 | moe_comp_method == CPUQuantMethod::INT4_W4A8 ? 1 : 2) + num_threads * 2 * BLOCK_M * BLOCK_N * sizeof(float); - if (use_int8_w8a8) { + if (moe_comp_method == CPUQuantMethod::INT8_W8A8) { buffer_size_nbytes += std::max(M * K, M * topk * N) + M * topk * sizeof(float); } - if (use_fp8_w8a16) { - buffer_size_nbytes += M * topk * 2 * N * 2 + num_threads * BLOCK_N * std::max(K, N) * 2; + if (moe_comp_method == CPUQuantMethod::FP8_W8A16) { + buffer_size_nbytes += M * topk * 2 * N * 2 + num_threads * MAX_CACHE_BLOCK_SIZE * BLOCK_N * std::max(K, N) * 2; + } + if (moe_comp_method == CPUQuantMethod::INT4_W4A8) { + buffer_size_nbytes += M * topk * 2 * N * 2 + std::max(M * K, M * topk * N) + M * topk * sizeof(float) + + num_threads * 2 * get_4bit_block_k_size(K / w1_scale.value().size(2)) * BLOCK_N; } - auto buffer2 = at::empty({buffer_size_nbytes}, hidden_states.options().dtype(at::kChar)); AT_DISPATCH_REDUCED_FLOATING_TYPES(st, "fused_experts_kernel_impl", [&] { scalar_t* __restrict__ intermediate_cache1 = (scalar_t*)((void*)(buffer2.data_ptr())); scalar_t* __restrict__ intermediate_cache2 = intermediate_cache1 + M * topk * N; - if (use_int8_w8a8) { + if (moe_comp_method == CPUQuantMethod::INT8_W8A8) { uint8_t* __restrict__ A_tmp = (uint8_t*)((void*)(intermediate_cache2 + M * topk * K)); float* __restrict__ C_tmp = (float*)((void*)(A_tmp + num_threads * BLOCK_M * K)); uint8_t* __restrict__ Aq_tmp = (uint8_t*)((void*)(C_tmp + num_threads * 2 * BLOCK_M * BLOCK_N)); @@ -1107,7 +1015,7 @@ at::Tensor fused_experts_cpu( packed_w2.data_ptr(), w1s.data_ptr(), w2s.data_ptr(), - topk_weights.data_ptr(), + topk_weights_.data_ptr(), sorted_ids, expert_ids, offsets, @@ -1117,7 +1025,7 @@ at::Tensor fused_experts_cpu( E, topk, num_tokens_post_pad); - } else if (use_fp8_w8a16) { + } else if (moe_comp_method == CPUQuantMethod::FP8_W8A16) { // here we just ignore C_tmp as it is not used scalar_t* __restrict__ A_tmp = (scalar_t*)((void*)(intermediate_cache2 + M * topk * K)); float* __restrict__ C_tmp = (float*)((void*)(A_tmp + num_threads * BLOCK_M * K)); @@ -1140,6 +1048,48 @@ at::Tensor fused_experts_cpu( w2s.data_ptr(), block_size_N, block_size_K, + topk_weights_.data_ptr(), + sorted_ids, + expert_ids, + offsets, + M, + N, + K, + E, + topk, + num_tokens_post_pad); + } else if (moe_comp_method == CPUQuantMethod::INT4_W4A8) { + uint8_t* __restrict__ A_tmp = (uint8_t*)((void*)(intermediate_cache2 + M * topk * K)); + float* __restrict__ C_tmp = (float*)((void*)(A_tmp + num_threads * BLOCK_M * K)); + scalar_t* __restrict__ intermediate_cache0 = (scalar_t*)((void*)(C_tmp + num_threads * 2 * BLOCK_M * BLOCK_N)); + uint8_t* __restrict__ Aq_tmp = (uint8_t*)((void*)(intermediate_cache0 + M * topk * 2 * N)); + float* __restrict__ As_tmp = (float*)((void*)(Aq_tmp + std::max(M * K, M * topk * N))); + int8_t* __restrict__ dqB_tmp = (int8_t*)((void*)(As_tmp + M * topk)); + + // weight + compensation shape = [Nc, Kc, block_n * block_k / 2 + block_n*sizeof(int32_t)] + // scales/qzeros shape = [E, Nc, G, block_n] + int64_t num_groups = w1_scale.value().size(2); + const int group_size = K / num_groups; + // TODO: check scales and zeros + fused_experts_int4_w4a8_kernel_impl( + out_hidden_states.data_ptr(), + intermediate_cache0, + intermediate_cache1, + intermediate_cache2, + A_tmp, + Aq_tmp, + As_tmp, + nullptr, + C_tmp, + dqB_tmp, + hidden_states.data_ptr(), + packed_w1.data_ptr(), + packed_w2.data_ptr(), + w1_zero.value().data_ptr(), + w2_zero.value().data_ptr(), + w1_scale.value().data_ptr(), + w2_scale.value().data_ptr(), + group_size, topk_weights.data_ptr(), sorted_ids, expert_ids, @@ -1163,7 +1113,7 @@ at::Tensor fused_experts_cpu( hidden_states.data_ptr(), packed_w1.data_ptr(), packed_w2.data_ptr(), - topk_weights.data_ptr(), + topk_weights_.data_ptr(), sorted_ids, expert_ids, offsets, @@ -1188,34 +1138,37 @@ at::Tensor shared_expert_cpu( at::Tensor& hidden_states, at::Tensor& w1, at::Tensor& w2, - at::Tensor& fused_experts_out, - double routed_scaling_factor, + const std::optional& fused_experts_out, + const std::optional routed_scaling_factor, bool inplace, bool use_int8_w8a8, bool use_fp8_w8a16, - std::optional& w1_scale, - std::optional& w2_scale, - std::optional> block_size, - std::optional& a1_scale, - std::optional& a2_scale, + const std::optional& w1_scale, + const std::optional& w2_scale, + const std::optional> block_size, bool is_vnni) { - RECORD_FUNCTION("sgl-kernel::shared_expert_cpu", std::vector({hidden_states, w1, w2})); - auto packed_w1 = is_vnni ? w1 : convert_weight_packed(w1); auto packed_w2 = is_vnni ? w2 : convert_weight_packed(w2); constexpr int64_t BLOCK_M = block_size_m(); constexpr int64_t BLOCK_N = block_size_n(); + double routed_scaling_factor_value = 0; + if (routed_scaling_factor.has_value()) { + TORCH_CHECK(fused_experts_out.has_value(), "shared_expert_cpu: expect fused_experts_out."); + const auto fused_experts_out_tensor = fused_experts_out.value(); + routed_scaling_factor_value = routed_scaling_factor.value(); + CHECK_INPUT(fused_experts_out_tensor); + CHECK_EQ(hidden_states.sizes(), fused_experts_out_tensor.sizes()); + } + const auto st = hidden_states.scalar_type(); CHECK_INPUT(hidden_states); - CHECK_INPUT(fused_experts_out); CHECK_INPUT(w1); CHECK_INPUT(w2); CHECK_DIM(2, hidden_states); CHECK_DIM(2, w1); CHECK_DIM(2, w2); - CHECK_EQ(hidden_states.sizes(), fused_experts_out.sizes()); CHECK_EQ(hidden_states.scalar_type(), st); int64_t M = hidden_states.size(0); @@ -1232,7 +1185,7 @@ at::Tensor shared_expert_cpu( CHECK_EQ(packed_w2.size(1), packed_N); // check scales - check_moe_scales(use_int8_w8a8, use_fp8_w8a16, w1_scale, w2_scale, block_size, a1_scale, a2_scale); + check_moe_scales(use_int8_w8a8, use_fp8_w8a16, w1_scale, w2_scale, block_size); at::Tensor out_hidden_states = inplace ? hidden_states : at::empty_like(hidden_states); @@ -1246,7 +1199,7 @@ at::Tensor shared_expert_cpu( // // for fp8 w8a16: // 5. intermediate_cache0 : [M, 2N] - // 6. B_tmp: [T, BLOCK_M, max(K, N)] + // 6. B_tmp: [T, MAX_CACHE_BLOCK_SIZE, BLOCK_M, max(K, N)] // int num_threads = at::get_num_threads(); int64_t buffer_size_nbytes = M * N * 2 + num_threads * 2 * BLOCK_M * BLOCK_N * sizeof(float); @@ -1255,7 +1208,7 @@ at::Tensor shared_expert_cpu( buffer_size_nbytes += std::max(M * K, M * N) + M * sizeof(float); } if (use_fp8_w8a16) { - buffer_size_nbytes += M * 2 * N * 2 + num_threads * BLOCK_M * std::max(K, N) * 2; + buffer_size_nbytes += M * 2 * N * 2 + num_threads * MAX_CACHE_BLOCK_SIZE * BLOCK_M * std::max(K, N) * 2; } auto buffer = at::empty({buffer_size_nbytes}, hidden_states.options().dtype(at::kChar)); @@ -1283,8 +1236,8 @@ at::Tensor shared_expert_cpu( packed_w2.data_ptr(), w1s.data_ptr(), w2s.data_ptr(), - fused_experts_out.data_ptr(), - routed_scaling_factor, + conditional_data_ptr(fused_experts_out), + routed_scaling_factor_value, M, N, K); @@ -1306,8 +1259,8 @@ at::Tensor shared_expert_cpu( w2s.data_ptr(), block_size_N, block_size_K, - fused_experts_out.data_ptr(), - routed_scaling_factor, + conditional_data_ptr(fused_experts_out), + routed_scaling_factor_value, M, N, K); @@ -1319,8 +1272,8 @@ at::Tensor shared_expert_cpu( hidden_states.data_ptr(), packed_w1.data_ptr(), packed_w2.data_ptr(), - fused_experts_out.data_ptr(), - routed_scaling_factor, + conditional_data_ptr(fused_experts_out), + routed_scaling_factor_value, M, N, K); diff --git a/csrc/cpu/sgl-kernels/moe.h b/csrc/cpu/sgl-kernels/moe.h new file mode 100644 index 0000000000..d80ed50cf9 --- /dev/null +++ b/csrc/cpu/sgl-kernels/moe.h @@ -0,0 +1,178 @@ +// Adapted from +// https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu + +// clang-format off + +#pragma once +#include "vec.h" + +template +inline void fill_stub(scalar_t* __restrict__ out, scalar_t val, int64_t size) { + using Vec = at::vec::Vectorized; + const Vec data_vec(val); + at::vec::map([data_vec](Vec out) { return out = data_vec; }, out, out, size); +} + +template +inline void copy_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ input, int64_t size) { + using Vec = at::vec::Vectorized; + constexpr int kVecSize = Vec::size(); + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= size - kVecSize; d += kVecSize) { + Vec data = Vec::loadu(input + d); + data.store(out + d); + } + for (; d < size; ++d) { + out[d] = input[d]; + } +} + +template +inline void copy_stub(scalar_t* __restrict__ out, const float* __restrict__ input, int64_t size) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= size - kVecSize; d += kVecSize) { + auto [x0, x1] = load_float_vec2(input + d); + bVec out_vec = convert_from_float_ext(x0, x1); + out_vec.store(out + d); + } + for (; d < size; ++d) { + out[d] = static_cast(input[d]); + } +} + +template <> +inline void copy_stub(uint8_t* __restrict__ out, const uint8_t* __restrict__ input, int64_t size) { + // size might be 64x + 32 + std::memcpy(out, input, size * sizeof(uint8_t)); +} + +template +inline void copy_mul_stub(scalar_t* __restrict__ out, const input_t* __restrict__ input, float weight, int64_t size) { + static_assert( + std::is_same_v || std::is_same_v, + "copy_mul_stub only supports input_t == float or input_t == scalar_t"); + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + const fVec weight_vec = fVec(weight); + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= size - kVecSize; d += kVecSize) { + auto [x0, x1] = load_float_vec2(input + d); + x0 = x0 * weight_vec; + x1 = x1 * weight_vec; + bVec out_vec = convert_from_float_ext(x0, x1); + out_vec.store(out + d); + } + for (; d < size; ++d) { + out[d] = static_cast(input[d] * weight); + } +} + +// acc from [topk, K] to [K] +template +inline void sum_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ input, int64_t topk, int64_t K) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + if (topk == 1) { + // do copy for topk = 1 + copy_stub(out, input, K); + } else { + // do sum for topk != 1 + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= K - kVecSize; d += kVecSize) { + fVec sum_fvec0 = fVec(0.f); + fVec sum_fvec1 = fVec(0.f); + for (int t = 0; t < topk; ++t) { + bVec x_bvec = bVec::loadu(input + t * K + d); + fVec x_fvec0, x_fvec1; + std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec); + + sum_fvec0 += x_fvec0; + sum_fvec1 += x_fvec1; + } + bVec out_bvec = convert_from_float_ext(sum_fvec0, sum_fvec1); + out_bvec.store(out + d); + } + for (; d < K; ++d) { + float sum_val = 0.f; + for (int t = 0; t < topk; ++t) { + sum_val += static_cast(input[t * K + d]); + } + out[d] = static_cast(sum_val); + } + } +} + +// out = input + input2 * scale +template +inline void add_mul_stub( + scalar_t* __restrict__ out, + const input_t* __restrict__ input, + const scalar_t* __restrict__ input2, + float scale, + int64_t size) { + static_assert( + std::is_same_v || std::is_same_v, + "add_mul_stub only supports input_t == float or input_t == scalar_t"); + + // out = input (without scale factor) + if (input2 == nullptr) { + copy_stub(out, input, size); + return; + } + + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + const fVec s_vec = fVec(scale); + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= size - kVecSize; d += kVecSize) { + auto [x0, x1] = load_float_vec2(input + d); + + bVec y_bvec = bVec::loadu(input2 + d); + fVec y0, y1; + std::tie(y0, y1) = at::vec::convert_to_float(y_bvec); + + x0 = x0 + y0 * s_vec; + x1 = x1 + y1 * s_vec; + bVec out_vec = convert_from_float_ext(x0, x1); + out_vec.store(out + d); + } + for (; d < size; ++d) { + out[d] = static_cast(input[d] + float(input2[d]) * scale); + } +} + +template +inline void silu_and_mul_stub( + scalar_t* __restrict__ out, const scalar_t* __restrict__ input, const scalar_t* __restrict__ input2, int64_t size) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + const fVec one = fVec(1.f); + + // no remainder +#pragma GCC unroll 4 + for (int64_t d = 0; d < size; d += bVec::size()) { + bVec x = bVec::loadu(input + d); + fVec x0, x1; + std::tie(x0, x1) = at::vec::convert_to_float(x); + bVec y = bVec::loadu(input2 + d); + fVec y0, y1; + std::tie(y0, y1) = at::vec::convert_to_float(y); + x0 = x0 / (one + x0.neg().exp_u20()); + x1 = x1 / (one + x1.neg().exp_u20()); + x0 = x0 * y0; + x1 = x1 * y1; + bVec out_vec = convert_from_float_ext(x0, x1); + out_vec.store(out + d); + } +} diff --git a/csrc/cpu/sgl-kernels/moe_fp8.cpp b/csrc/cpu/sgl-kernels/moe_fp8.cpp index 84a6af2677..f93630ae26 100644 --- a/csrc/cpu/sgl-kernels/moe_fp8.cpp +++ b/csrc/cpu/sgl-kernels/moe_fp8.cpp @@ -1,147 +1,11 @@ // Adapted from // https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu -#include "common.h" -#include "gemm.h" -#include "vec.h" - // clang-format off -namespace { - -template -inline void copy_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ input, int64_t size) { - using Vec = at::vec::Vectorized; - // no remainder - #pragma GCC unroll 4 - for (int64_t d = 0; d < size; d += Vec::size()) { - Vec data = Vec::loadu(input + d); - data.store(out + d); - } -} - -template -inline void copy_mul_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ input, float weight, int64_t size) { - using bVec = at::vec::Vectorized; - using fVec = at::vec::Vectorized; - constexpr int kVecSize = bVec::size(); - const fVec weight_vec = fVec(weight); - int64_t d; - #pragma GCC unroll 4 - for (d = 0; d <= size - kVecSize; d += kVecSize) { - bVec x = bVec::loadu(input + d); - fVec x0, x1; - std::tie(x0, x1) = at::vec::convert_to_float(x); - x0 = x0 * weight_vec; - x1 = x1 * weight_vec; - bVec out_vec = convert_from_float_ext(x0, x1); - out_vec.store(out + d); - } - for (; d < size; ++d) { - out[d] = static_cast(input[d] * weight); - } -} - -// acc from [topk, K] to [K] -template -inline void sum_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ input, int64_t topk, int64_t K) { - using bVec = at::vec::Vectorized; - using fVec = at::vec::Vectorized; - constexpr int kVecSize = bVec::size(); - if (topk == 1) { - // do copy for topk = 1 - copy_stub(out, input, K); - } else { - // do sum for topk != 1 - int64_t d; - #pragma GCC unroll 4 - for (d = 0; d <= K - kVecSize; d += kVecSize) { - fVec sum_fvec0 = fVec(0.f); - fVec sum_fvec1 = fVec(0.f); - for (int t = 0; t < topk; ++t) { - bVec x_bvec = bVec::loadu(input + t * K + d); - fVec x_fvec0, x_fvec1; - std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec); - - sum_fvec0 += x_fvec0; - sum_fvec1 += x_fvec1; - } - bVec out_bvec = convert_from_float_ext(sum_fvec0, sum_fvec1); - out_bvec.store(out + d); - } - for (; d < K; ++d) { - float sum_val = 0.f; - for (int t = 0; t < topk; ++t) { - sum_val += static_cast(input[t * K + d]); - } - out[d] = static_cast(sum_val); - } - } -} - -// out = input + input2 * scale -template -inline void add_mul_stub( - scalar_t* __restrict__ out, - const scalar_t* __restrict__ input, - const scalar_t* __restrict__ input2, - float scale, - int64_t size) { - using bVec = at::vec::Vectorized; - using fVec = at::vec::Vectorized; - constexpr int kVecSize = bVec::size(); - const fVec s_vec = fVec(scale); - - int64_t d; -#pragma GCC unroll 4 - for (d = 0; d <= size - kVecSize; d += kVecSize) { - bVec x_bvec = bVec::loadu(input + d); - fVec x0, x1; - std::tie(x0, x1) = at::vec::convert_to_float(x_bvec); - - bVec y_bvec = bVec::loadu(input2 + d); - fVec y0, y1; - std::tie(y0, y1) = at::vec::convert_to_float(y_bvec); - - x0 = x0 + y0 * s_vec; - x1 = x1 + y1 * s_vec; - bVec out_vec = convert_from_float_ext(x0, x1); - out_vec.store(out + d); - } - for (; d < size; ++d) { - out[d] = static_cast(input[d] + float(input2[d]) * scale); - } -} - -template -inline void silu_and_mul_stub( - scalar_t* __restrict__ out, - const scalar_t* __restrict__ input, - const scalar_t* __restrict__ input2, - int64_t size) { - using bVec = at::vec::Vectorized; - using fVec = at::vec::Vectorized; - const fVec one = fVec(1.f); - - // no remainder -#pragma GCC unroll 4 - for (int64_t d = 0; d < size; d += bVec::size()) { - bVec x = bVec::loadu(input + d); - fVec x0, x1; - std::tie(x0, x1) = at::vec::convert_to_float(x); - bVec y = bVec::loadu(input2 + d); - fVec y0, y1; - std::tie(y0, y1) = at::vec::convert_to_float(y); - x0 = x0 / (one + x0.neg().exp_u20()); - x1 = x1 / (one + x1.neg().exp_u20()); - x0 = x0 * y0; - x1 = x1 * y1; - bVec out_vec = convert_from_float_ext(x0, x1); - out_vec.store(out + d); - } -} - -} // anonymous namespace +#include "common.h" +#include "gemm.h" +#include "moe.h" template void fused_experts_fp8_kernel_impl( @@ -169,7 +33,6 @@ void fused_experts_fp8_kernel_impl( int64_t E, int64_t topk, int64_t num_tokens_post_pad) { - constexpr int64_t BLOCK_M = block_size_m(); constexpr int64_t BLOCK_N = block_size_n(); @@ -183,35 +46,39 @@ void fused_experts_fp8_kernel_impl( const int64_t stride_e = 2 * N * K; const int64_t stride_n = K; + int64_t avg_M = std::max(int64_t(1), M * topk / E); + const bool use_brgemm = can_use_brgemm(avg_M); + + int64_t B_tmp_size_per_thread = MAX_CACHE_BLOCK_SIZE * BLOCK_N * std::max(K, N); + // here we only parallel on half of 2N to fuse silu_and_mul with gemm - at::parallel_for(0, MB * NB, 0, [&](int64_t begin, int64_t end) { + parallel_2d(MB, NB, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { // get local pointers - int tid = at::get_thread_num(); + int tid = get_thread_num(); scalar_t* __restrict__ A = A_tmp + tid * BLOCK_M * K; - bool is_brgemm_used = false; - - for (int64_t i = begin; i < end; ++i) { - int64_t mb = i / NB; - int64_t nb = i % NB; - + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * K, [&](int64_t mb, int64_t nb, int64_t nb_offset) { int64_t n_size = std::min(2 * N - nb * BLOCK_N, BLOCK_N); // B shape [K, n_size] in vnni format int32_t expert_id = expert_ids[mb]; const at::Float8_e4m3fn* __restrict__ B = packed_w1 + expert_id * stride_e + nb * BLOCK_N * stride_n; - const float* __restrict__ Bs = w1s + expert_id * scale_size_N * scale_size_K + (nb / blocks_n_per_group) * scale_size_K; + const float* __restrict__ Bs = + w1s + expert_id * scale_size_N * scale_size_K + (nb / blocks_n_per_group) * scale_size_K; + + // do unpacking for the first row or a new expert + int32_t pre_expert_id = mb == 0 ? -1 : expert_ids[mb - 1]; + bool do_unpack = (mb == mb0) || (expert_id != pre_expert_id); - // 1.a load A - const int32_t* A_ids = sorted_ids + mb * BLOCK_M; int64_t m_size = offsets[mb + 1] - offsets[mb]; - const bool use_brgemm = can_use_brgemm(m_size); - is_brgemm_used = is_brgemm_used || use_brgemm; - - for (int64_t m = 0; m < m_size; ++m) { - int32_t index = A_ids[m] / topk; - copy_stub(A + m * K, input + index * K, K); + if (nb_offset == 0) { + // 1.a load A + const int32_t* A_ids = sorted_ids + mb * BLOCK_M; + for (int64_t m = 0; m < m_size; ++m) { + int32_t index = A_ids[m] / topk; + copy_stub(A + m * K, input + index * K, K); + } } const int64_t offset = offsets[mb]; @@ -219,7 +86,7 @@ void fused_experts_fp8_kernel_impl( /* A */ A, /* B */ B, /* C */ ic0 + offset * 2 * N + nb * BLOCK_N, - /* Btmp */ B_tmp + tid * BLOCK_N * std::max(K, N), + /* Btmp */ B_tmp + tid * B_tmp_size_per_thread + nb_offset * BLOCK_N * K, /* Ctmp */ C_tmp + tid * 2 * BLOCK_M * BLOCK_N, /* scale */ Bs, /* M */ m_size, @@ -229,10 +96,11 @@ void fused_experts_fp8_kernel_impl( /* ldb */ n_size, /* ldc */ 2 * N, /* brg */ use_brgemm, - /* block_size_K */ block_size_K); - } + /* block_size_K */ block_size_K, + /* do_unpack */ do_unpack); + }); - if (is_brgemm_used) { + if (use_brgemm) { at::native::cpublas::brgemm_release(); } }); @@ -240,11 +108,7 @@ void fused_experts_fp8_kernel_impl( // stage 1.5: intermediate_cache1 = silu(intermediate_cache0) at::parallel_for(0, M * topk, 0, [&](int64_t begin, int64_t end) { for (int64_t m = begin; m < end; ++m) { - silu_and_mul_stub( - ic1 + m * N, - ic0 + m * 2 * N, - ic0 + m * 2 * N + N, - N); + silu_and_mul_stub(ic1 + m * N, ic0 + m * 2 * N, ic0 + m * 2 * N + N, N); } }); @@ -260,22 +124,14 @@ void fused_experts_fp8_kernel_impl( const int64_t stride_oc = IC; // parallel on [MB2, NB2] - at::parallel_for(0, MB2 * NB2, 0, [&](int64_t begin, int64_t end) { - int tid = at::get_thread_num(); + parallel_2d(MB2, NB2, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + int tid = get_thread_num(); alignas(64) scalar_t C[BLOCK_M * BLOCK_K]; - bool is_brgemm_used = false; - - for (int64_t i = begin; i < end; ++i) { - int64_t mb = i / NB2; - int64_t nb = i % NB2; - + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * IC, [&](int64_t mb, int64_t nb, int64_t nb_offset) { int64_t m_size = offsets[mb + 1] - offsets[mb]; int64_t n_size = std::min(OC - nb * BLOCK_N, BLOCK_N); - const bool use_brgemm = can_use_brgemm(m_size); - is_brgemm_used = is_brgemm_used || use_brgemm; - // A ptr from ic1 of [M * topk, N] in sorted order // so as to avoid copy A to tmp buffer again const scalar_t* __restrict__ A = ic1 + offsets[mb] * N; @@ -284,13 +140,18 @@ void fused_experts_fp8_kernel_impl( // B shape [IC, n_size] in vnni format int32_t expert_id = expert_ids[mb]; const at::Float8_e4m3fn* __restrict__ B = packed_w2 + expert_id * stride_e2 + nb * BLOCK_N * stride_oc; - const float* __restrict__ Bs = w2s + expert_id * scale_size_N * scale_size_K + (nb / blocks_n_per_group) * scale_size_K; + const float* __restrict__ Bs = + w2s + expert_id * scale_size_N * scale_size_K + (nb / blocks_n_per_group) * scale_size_K; + + // do unpacking for the first row or a new expert + int32_t pre_expert_id = mb == 0 ? -1 : expert_ids[mb - 1]; + bool do_unpack = (mb == mb0) || (expert_id != pre_expert_id); tinygemm_kernel( /* A */ A, /* B */ B, /* C */ C, - /* Btmp */ B_tmp + tid * BLOCK_N * std::max(K, N), + /* Btmp */ B_tmp + tid * B_tmp_size_per_thread + nb_offset * BLOCK_N * IC, /* Ctmp */ C_tmp + tid * 2 * BLOCK_M * BLOCK_N, /* scale */ Bs, /* M */ m_size, @@ -300,7 +161,8 @@ void fused_experts_fp8_kernel_impl( /* ldb */ n_size, /* ldc */ BLOCK_N, /* brg */ use_brgemm, - /* block_size_K */ block_size_K); + /* block_size_K */ block_size_K, + /* do_unpack */ do_unpack); // 2.b copy from C to ic2 in original order // and also mul topk_weights in float32 @@ -309,9 +171,9 @@ void fused_experts_fp8_kernel_impl( float weight = topk_weights[index]; copy_mul_stub(ic2 + index * K + nb * BLOCK_N, C + m * BLOCK_N, weight, n_size); } - } + }); - if (is_brgemm_used) { + if (use_brgemm) { at::native::cpublas::brgemm_release(); } }); @@ -374,7 +236,6 @@ void shared_expert_fp8_kernel_impl( int64_t M, int64_t N, int64_t K) { - constexpr int64_t BLOCK_M = block_size_m(); constexpr int64_t BLOCK_N = block_size_n(); @@ -385,21 +246,25 @@ void shared_expert_fp8_kernel_impl( int64_t blocks_n_per_group = block_size_N / BLOCK_N; const bool use_brgemm = can_use_brgemm(M); + const bool apply_scaling_factor = fused_experts_out != nullptr; - at::parallel_for(0, MB * NB, 0, [&](int64_t begin, int64_t end) { - int tid = at::get_thread_num(); + int64_t B_tmp_size_per_thread = MAX_CACHE_BLOCK_SIZE * BLOCK_N * std::max(K, N); - for (int64_t i = begin; i < end; ++i) { - int64_t mb = i / NB; - int64_t nb = i % NB; + parallel_2d(MB, NB, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + int tid = get_thread_num(); + + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * K, [&](int64_t mb, int64_t nb, int64_t nb_offset) { int64_t m_size = std::min(M - mb * BLOCK_M, BLOCK_M); int64_t n_size = std::min(2 * N - nb * BLOCK_N, BLOCK_N); + // do unpacking for the first row + bool do_unpack = (mb == mb0); + tinygemm_kernel( /* A */ input + mb * BLOCK_M * K, /* B */ packed_w1 + nb * BLOCK_N * K, /* C */ ic0 + mb * BLOCK_M * 2 * N + nb * BLOCK_N, - /* Btmp */ B_tmp + tid * BLOCK_N * std::max(K, N), + /* Btmp */ B_tmp + tid * B_tmp_size_per_thread + nb_offset * BLOCK_N * K, /* Ctmp */ C_tmp + tid * 2 * BLOCK_M * BLOCK_N, /* scale */ w1s + (nb / blocks_n_per_group) * scale_size_K, /* M */ m_size, @@ -409,8 +274,9 @@ void shared_expert_fp8_kernel_impl( /* ldb */ n_size, /* ldc */ 2 * N, /* brg */ use_brgemm, - /* block_size_K */ block_size_K); - } + /* block_size_K */ block_size_K, + /* do_unpack */ do_unpack); + }); if (use_brgemm) { at::native::cpublas::brgemm_release(); @@ -420,11 +286,7 @@ void shared_expert_fp8_kernel_impl( // stage 1.5: intermediate_cache1 = silu(intermediate_cache0) at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) { for (int64_t m = begin; m < end; ++m) { - silu_and_mul_stub( - ic1 + m * N, - ic0 + m * 2 * N, - ic0 + m * 2 * N + N, - N); + silu_and_mul_stub(ic1 + m * N, ic0 + m * 2 * N, ic0 + m * 2 * N + N, N); } }); @@ -437,22 +299,23 @@ void shared_expert_fp8_kernel_impl( scale_size_K = div_up(N, block_size_K); // parallel on [MB2, NB2] - at::parallel_for(0, MB2 * NB2, 0, [&](int64_t begin, int64_t end) { - int tid = at::get_thread_num(); + parallel_2d(MB2, NB2, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + int tid = get_thread_num(); alignas(64) scalar_t C[BLOCK_M * BLOCK_K]; - for (int64_t i = begin; i < end; ++i) { - int64_t mb = i / NB2; - int64_t nb = i % NB2; + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * IC, [&](int64_t mb, int64_t nb, int64_t nb_offset) { int64_t m_size = std::min(M - mb * BLOCK_M, BLOCK_M); int64_t n_size = std::min(OC - nb * BLOCK_N, BLOCK_N); + // do unpacking for the first row + bool do_unpack = (mb == mb0); + // 2.a gemm: C = A @ B tinygemm_kernel( /* A */ ic1 + mb * BLOCK_M * N, /* B */ packed_w2 + nb * BLOCK_N * N, /* C */ C, - /* Btmp */ B_tmp + tid * BLOCK_N * std::max(K, N), + /* Btmp */ B_tmp + tid * B_tmp_size_per_thread + nb_offset * BLOCK_N * IC, /* Ctmp */ C_tmp + tid * 2 * BLOCK_M * BLOCK_N, /* scale */ w2s + (nb / blocks_n_per_group) * scale_size_K, /* M */ m_size, @@ -462,15 +325,18 @@ void shared_expert_fp8_kernel_impl( /* ldb */ n_size, /* ldc */ BLOCK_N, /* brg */ use_brgemm, - /* block_size_K */ block_size_K); + /* block_size_K */ block_size_K, + /* do_unpack */ do_unpack); // 2.b copy from C to output and add fused_experts_out scalar_t* __restrict__ out = output + mb * BLOCK_M * K + nb * BLOCK_N; - const scalar_t* __restrict__ fused_out = fused_experts_out + mb * BLOCK_M * K + nb * BLOCK_N; + const scalar_t* __restrict__ fused_out = + apply_scaling_factor ? fused_experts_out + mb * BLOCK_M * K + nb * BLOCK_N : nullptr; for (int64_t m = 0; m < m_size; ++m) { - add_mul_stub(out + m * K, C + m * BLOCK_N, fused_out + m * K, routed_scaling_factor, n_size); + const scalar_t* __restrict__ fused_out_row = apply_scaling_factor ? (fused_out + m * K) : nullptr; + add_mul_stub(out + m * K, C + m * BLOCK_N, fused_out_row, routed_scaling_factor, n_size); } - } + }); }); if (use_brgemm) { diff --git a/csrc/cpu/sgl-kernels/moe_int4.cpp b/csrc/cpu/sgl-kernels/moe_int4.cpp new file mode 100644 index 0000000000..c97784847a --- /dev/null +++ b/csrc/cpu/sgl-kernels/moe_int4.cpp @@ -0,0 +1,323 @@ +// Adapted from +// https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu + +// clang-format off + +#include "common.h" +#include "gemm.h" +#include "moe.h" + +template +inline void copy_bias(const float* bias_ptr, float* y_buf, int64_t m, int64_t ldn) { + using Vec = at::vec::Vectorized; + constexpr int kVecSize = Vec::size(); + static_assert(N % kVecSize == 0, "copy_bias requires N to be a multiple of Vectorized::size()"); + const bool has_bias = bias_ptr != nullptr; + const Vec zero_vec(0.f); + for (int i = 0; i < m; ++i) { +#pragma GCC unroll 2 + for (int j = 0; j < N; j += kVecSize) { + Vec vec = has_bias ? Vec::loadu(bias_ptr + j) : zero_vec; + vec.store(y_buf + i * ldn + j); + } + } +} + +template +void fused_experts_int4_w4a8_kernel_impl( + scalar_t* __restrict__ output, + scalar_t* __restrict__ ic0, + scalar_t* __restrict__ ic1, + scalar_t* __restrict__ ic2, + uint8_t* __restrict__ A_tmp, + uint8_t* __restrict__ Aq_tmp, + float* __restrict__ As_tmp, + int32_t* __restrict__ Azp_tmp, + float* __restrict__ C_tmp, + int8_t* __restrict__ dqB_tmp, + const scalar_t* __restrict__ input, + const uint8_t* __restrict__ packed_w1, + const uint8_t* __restrict__ packed_w2, + const int8_t* __restrict__ w1z, + const int8_t* __restrict__ w2z, + const float* __restrict__ w1s, + const float* __restrict__ w2s, + int group_size, + const float* __restrict__ topk_weights, + const int32_t* __restrict__ sorted_ids, + const int32_t* __restrict__ expert_ids, + const int32_t* __restrict__ offsets, + int64_t M, + int64_t N, + int64_t K, + int64_t E, + int64_t topk, + int64_t num_tokens_post_pad) { + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n(); + int num_threads = at::get_num_threads(); + // int64_t buffer_size_nbytes = M * topk * N * 2 + // M * topk * K * 2 + + // num_threads * BLOCK_M * K + + // num_threads * 2 * BLOCK_M * BLOCK_N * sizeof(float) + + // M * topk * 2 * N * 2 + + // max(M * K, M * topk * N) + + // M * topk * sizeof(float); + + // intermediate_cache1 (scalar_t): START + M * topk * N + // intermediate_cache2 (scalar_t): + M * topk * K + // A_tmp (uint8_t): + num_threads * BLOCK_M * K + // C_tmp (float): + num_threads * 2 * BLOCK_M * BLOCK_N + // intermediate_cache0 (scalar_t): + M * topk * 2 * N + // Aq_tmp (uint8_t): + max(M * K, M * topk * N) + // As_tmp (float): + M * topk + // dqB_tmp (int8_t) + num_threads * _block_k * BlOCK_N + + // stage 0: quantize input to uint8, [M, K] + at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) { + for (int64_t m = begin; m < end; ++m) { + quantize_row_int8(Aq_tmp + m * K, As_tmp[m], input + m * K, K); + } + }); + int64_t _block_k = get_4bit_block_k_size(group_size); + auto Azp = at::ones({M * topk}).to(at::kInt).mul(128); + auto Azp_ptr = Azp.data_ptr(); + // stage 1: intermediate_cache0 = hidden_states @ w1 + const int64_t MB = div_up(num_tokens_post_pad, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N); + + int64_t block_per_group = group_size / _block_k; + int64_t Kc = K / _block_k; + int64_t num_groups = K / group_size; + + const int64_t stride_e = 2 * NB * Kc * (BLOCK_N * (_block_k / 2 + sizeof(int32_t))); + const bool sym_quant_act = false; + // weight + compensation shape = [E, Nc, Kc, block_n * _block_k / 2 + block_n*sizeof(int32_t)] + // scales/qzeros shape = [E, Nc, G, block_n] + + // here we only parallel on half of 2N to fuse silu_and_mul with gemm + at::parallel_for(0, MB * NB, 0, [&](int64_t begin, int64_t end) { + // get local pointers + int tid = at::get_thread_num(); + int8_t* dqB_tmp1 = dqB_tmp + tid * 2 * _block_k * BLOCK_N; + int8_t* dqB_tmp2 = dqB_tmp1 + _block_k * BLOCK_N; + alignas(64) float As[BLOCK_M]; + uint8_t* __restrict__ A = A_tmp + tid * BLOCK_M * K; + float* __restrict__ C0 = C_tmp + tid * 2 * BLOCK_M * BLOCK_N; + float* __restrict__ C1 = C0 + BLOCK_M * BLOCK_N; + bool is_brgemm_used = false; + for (int64_t i = begin; i < end; ++i) { + int64_t mb = i / NB; + int64_t nb = i % NB; + int64_t nb1 = nb + NB; + int64_t n_size = std::min(N - nb * BLOCK_N, BLOCK_N); + // B shape [K, n_size] in vnni format + int32_t expert_id = expert_ids[mb]; + const uint8_t* __restrict__ B = packed_w1 + expert_id * stride_e; + // Bz and Bs: [E, K/gs, 2N] + const int8_t* __restrict__ Bz = w1z + expert_id * (num_groups) * (2 * N); + const float* __restrict__ Bs = w1s + expert_id * (num_groups) * (2 * N); + + // 1.a load A + const int32_t* A_ids = sorted_ids + mb * BLOCK_M; + int64_t m_size = offsets[mb + 1] - offsets[mb]; + const bool use_brgemm = can_use_brgemm(m_size); + is_brgemm_used = is_brgemm_used || use_brgemm; + // copy to A [BLOCK_M, K] + for (int64_t m = 0; m < m_size; ++m) { + int32_t index = A_ids[m] / topk; + copy_stub(A + m * K, Aq_tmp + index * K, K); + As[m] = As_tmp[index]; + } + const int64_t offset = offsets[mb]; + copy_bias(nullptr, C0, m_size, BLOCK_N); + copy_bias(nullptr, C1, m_size, BLOCK_N); + for (int kci = 0; kci < Kc; ++kci) { + int32_t* compensation_ptr = + sym_quant_act ? nullptr + : (int32_t*)(void*)(B + (nb * Kc + kci) * (BLOCK_N * (_block_k / 2 + sizeof(int32_t))) + + _block_k * BLOCK_N / 2) /*Bcomp*/; + tinygemm_kernel( + ic0 + offset * 2 * N + nb * BLOCK_N, + C0, + A + kci * _block_k, + As, + Azp_ptr, + B + (nb * Kc + kci) * (BLOCK_N * (_block_k / 2 + sizeof(int32_t))) /*B*/, + Bs + nb * BLOCK_N * num_groups + kci / block_per_group * BLOCK_N /*scales_b*/, + Bz + nb * BLOCK_N * num_groups + kci / block_per_group * BLOCK_N /*qzeros_b*/, + compensation_ptr, + dqB_tmp1, + m_size, + _block_k, + K, + BLOCK_N, + 2 * N, + kci == Kc - 1, + use_brgemm); + } + + for (int kci = 0; kci < Kc; ++kci) { + int32_t* compensation_ptr = + sym_quant_act ? nullptr + : (int32_t*)(void*)(B + (nb1 * Kc + kci) * (BLOCK_N * (_block_k / 2 + sizeof(int32_t))) + + _block_k * BLOCK_N / 2) /*Bcomp*/; + tinygemm_kernel( + ic0 + offset * 2 * N + nb1 * BLOCK_N, + C1, + A + kci * _block_k, + As, + Azp_ptr, + B + (nb1 * Kc + kci) * (BLOCK_N * (_block_k / 2 + sizeof(int32_t))) /*B*/, + Bs + nb1 * BLOCK_N * num_groups + kci / block_per_group * BLOCK_N /*scales_b*/, + Bz + nb1 * BLOCK_N * num_groups + kci / block_per_group * BLOCK_N /*qzeros_b*/, + compensation_ptr, + dqB_tmp2, + m_size, + _block_k, + K, + BLOCK_N, + 2 * N, + kci == Kc - 1, + use_brgemm); + } + } + + if (is_brgemm_used) { + at::native::cpublas::brgemm_release(); + } + }); + + // stage 1.5: intermediate_cache1 = silu(intermediate_cache0) + at::parallel_for(0, M * topk, 0, [&](int64_t begin, int64_t end) { + for (int64_t m = begin; m < end; ++m) { + silu_and_mul_stub(ic1 + m * N, ic0 + m * 2 * N, ic0 + m * 2 * N + N, N); + } + }); + + // stage 1.5: quantize ic1 to uint8, [M * topk, N] + at::parallel_for(0, M * topk, 0, [&](int64_t begin, int64_t end) { + for (int64_t m = begin; m < end; ++m) { + quantize_row_int8(Aq_tmp + m * N, As_tmp[m], ic1 + m * N, N); + } + }); + // stage 2: intermediate_cache2 = intermediate_cache1 @ w2 + // w2 : [E, K, N] as [E, OC, IC] + const int64_t OC = K; // rename K as OC + const int64_t IC = N; // rename N as IC + const int64_t MB2 = MB; + const int64_t NB2 = div_up(OC, BLOCK_N); + const int64_t stride_oc = IC; + num_groups = IC / group_size; + Kc = IC / _block_k; + const int64_t stride_e2 = NB2 * Kc * (BLOCK_N * (_block_k / 2 + sizeof(int32_t))); + // parallel on [MB2, NB2] + at::parallel_for(0, MB2 * NB2, 0, [&](int64_t begin, int64_t end) { + int tid = at::get_thread_num(); + int8_t* dqB_tmp1 = dqB_tmp + tid * 2 * _block_k * BLOCK_N; + float* __restrict__ C2 = C_tmp + tid * 2 * BLOCK_M * BLOCK_N; + bool is_brgemm_used = false; + for (int64_t i = begin; i < end; ++i) { + int64_t mb = i / NB2; + int64_t nb = i % NB2; + + int64_t m_size = offsets[mb + 1] - offsets[mb]; + int64_t n_size = std::min(OC - nb * BLOCK_N, BLOCK_N); + const bool use_brgemm = can_use_brgemm(m_size); + is_brgemm_used = is_brgemm_used || use_brgemm; + const int32_t* A_ids = sorted_ids + mb * BLOCK_M; + + // B shape [IC, n_size] in vnni format + int32_t expert_id = expert_ids[mb]; + const uint8_t* __restrict__ B = packed_w2 + expert_id * stride_e2; + + // Bz and Bs: [E, IC/gs, OC] + const int8_t* __restrict__ Bz = w2z + expert_id * (num_groups)*OC; + const float* __restrict__ Bs = w2s + expert_id * (num_groups)*OC; + + // A ptr from ic1 of [M * topk, N] in sorted order + // so as to avoid copy A to tmp buffer again + const uint8_t* __restrict__ A = Aq_tmp + offsets[mb] * IC; + const float* __restrict__ As = As_tmp + offsets[mb]; + copy_bias(nullptr, C2, m_size, BLOCK_N); + for (int kci = 0; kci < Kc; ++kci) { + int32_t* compensation_ptr = + sym_quant_act ? nullptr + : (int32_t*)(void*)(B + (nb * Kc + kci) * (BLOCK_N * (_block_k / 2 + sizeof(int32_t))) + + _block_k * BLOCK_N / 2) /*Bcomp*/; + tinygemm_kernel( + nullptr, /*store_out is false*/ + C2, + A + kci * _block_k, + As, + Azp_ptr, + B + (nb * Kc + kci) * (BLOCK_N * (_block_k / 2 + sizeof(int32_t))), + Bs + nb * BLOCK_N * num_groups + kci / block_per_group * BLOCK_N /*scales_b*/, + Bz + nb * BLOCK_N * num_groups + kci / block_per_group * BLOCK_N /*zeros_b*/, + compensation_ptr, + dqB_tmp1, + m_size, + _block_k, + IC, + BLOCK_N, + BLOCK_N, + false, + use_brgemm); + } + + // 2.b copy from C to ic2 in original order + // and also mul topk_weights in float32 + for (int64_t m = 0; m < m_size; ++m) { + int32_t index = A_ids[m]; + float weight = topk_weights[index]; + copy_mul_stub(ic2 + index * K + nb * BLOCK_N, C2 + m * BLOCK_N, weight, n_size); + } + } + + if (is_brgemm_used) { + at::native::cpublas::brgemm_release(); + } + }); + + // stage 3: out = intermediate_cache2.sum(dim=1) + // from [M, topk, K] to [M, K] + at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) { + for (int64_t m = begin; m < end; ++m) { + sum_stub(output + m * K, ic2 + m * topk * K, topk, K); + } + }); +} + +#define INSTANTIATE_MOE_INT4_W4A8_TEMPLATE(TYPE) \ + template void fused_experts_int4_w4a8_kernel_impl( \ + TYPE* __restrict__ output, \ + TYPE* __restrict__ ic0, \ + TYPE* __restrict__ ic1, \ + TYPE* __restrict__ ic2, \ + uint8_t* __restrict__ A_tmp, \ + uint8_t* __restrict__ Aq_tmp, \ + float* __restrict__ As_tmp, \ + int32_t* __restrict__ Azp_tmp, \ + float* __restrict__ C_tmp, \ + int8_t* __restrict__ dqB_tmp, \ + const TYPE* __restrict__ input, \ + const uint8_t* __restrict__ packed_w1, \ + const uint8_t* __restrict__ packed_w2, \ + const int8_t* __restrict__ w1z, \ + const int8_t* __restrict__ w2z, \ + const float* __restrict__ w1s, \ + const float* __restrict__ w2s, \ + int group_size, \ + const float* __restrict__ topk_weights, \ + const int32_t* __restrict__ sorted_ids, \ + const int32_t* __restrict__ expert_ids, \ + const int32_t* __restrict__ offsets, \ + int64_t M, \ + int64_t N, \ + int64_t K, \ + int64_t E, \ + int64_t topk, \ + int64_t num_tokens_post_pad) + +INSTANTIATE_MOE_INT4_W4A8_TEMPLATE(at::BFloat16); +INSTANTIATE_MOE_INT4_W4A8_TEMPLATE(at::Half); diff --git a/csrc/cpu/sgl-kernels/moe_int8.cpp b/csrc/cpu/sgl-kernels/moe_int8.cpp index e28b4fc4ee..3bdd5892d0 100644 --- a/csrc/cpu/sgl-kernels/moe_int8.cpp +++ b/csrc/cpu/sgl-kernels/moe_int8.cpp @@ -1,124 +1,145 @@ // Adapted from // https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu -#include "common.h" -#include "vec.h" -#include "gemm.h" - // clang-format off +#include "common.h" +#include "gemm.h" +#include "moe.h" + namespace { -template -inline void copy_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ input, int64_t size) { - using Vec = at::vec::Vectorized; - // no remainder - #pragma GCC unroll 4 - for (int64_t d = 0; d < size; d += Vec::size()) { - Vec data = Vec::loadu(input + d); - data.store(out + d); - } -} +template +inline void silu_and_mul( + scalar_t* __restrict__ C, + const int32_t* __restrict__ C0, // x: x0, x1 + const int32_t* __restrict__ C1, // y: y0, y1 + const float* __restrict__ As, + const float* __restrict__ Bs0, + const float* __restrict__ Bs1, + const int32_t* __restrict__ Bcomp0, + const int32_t* __restrict__ Bcomp1, + int64_t m_size, + int64_t N) { +#if defined(CPU_CAPABILITY_AVX512) + constexpr int COLS = BLOCK_N / 16; + static_assert(COLS % 2 == 0); -template <> -inline void copy_stub(uint8_t* __restrict__ out, const uint8_t* __restrict__ input, int64_t size) { - // size might be 64x + 32 - std::memcpy(out, input, size * sizeof(uint8_t)); -} + __m512 vc0[COLS]; + __m512 vc1[COLS]; + __m512i vcomp0[COLS]; + __m512i vcomp1[COLS]; + __m512 vas; + __m512 vbs0[COLS]; + __m512 vbs1[COLS]; + + auto load_scale_and_comp = [&](auto col) { + vcomp0[col] = _mm512_loadu_si512(Bcomp0 + col * 16); + vcomp1[col] = _mm512_loadu_si512(Bcomp1 + col * 16); + vbs0[col] = _mm512_loadu_ps(Bs0 + col * 16); + vbs1[col] = _mm512_loadu_ps(Bs1 + col * 16); + }; + Unroll{}(load_scale_and_comp); + + auto scalec = [&](auto col, int64_t m) { + // update As + vas = _mm512_set1_ps(As[m]); + // C = As * (C - Bcomp) * Bs + __m512i vc32_0 = _mm512_loadu_si512(C0 + m * BLOCK_N + col * 16); + __m512i vc32_1 = _mm512_loadu_si512(C1 + m * BLOCK_N + col * 16); + vc0[col] = _mm512_cvtepi32_ps(_mm512_sub_epi32(vc32_0, vcomp0[col])); + vc1[col] = _mm512_cvtepi32_ps(_mm512_sub_epi32(vc32_1, vcomp1[col])); + vc0[col] = _mm512_mul_ps(_mm512_mul_ps(vc0[col], vas), vbs0[col]); + vc1[col] = _mm512_mul_ps(_mm512_mul_ps(vc1[col], vas), vbs1[col]); + }; -template -inline void copy_mul_stub(scalar_t* __restrict__ out, const float* __restrict__ input, float weight, int64_t size) { using bVec = at::vec::Vectorized; using fVec = at::vec::Vectorized; - constexpr int kVecSize = bVec::size(); - const fVec weight_vec = fVec(weight); - int64_t d; - #pragma GCC unroll 4 - for (d = 0; d <= size - kVecSize; d += kVecSize) { - fVec data0 = fVec::loadu(input + d) * weight_vec; - fVec data1 = fVec::loadu(input + d + fVec::size()) * weight_vec; - bVec out_vec = convert_from_float_ext(data0, data1); - out_vec.store(out + d); - } - for (; d < size; ++d) { - out[d] = static_cast(input[d] * weight); - } -} + const fVec one = fVec(1.f); + auto silu_and_mul = [&](auto col) { + fVec x = fVec(vc0[col]); + fVec y = fVec(vc1[col]); + x = x / (one + x.neg().exp_u20()); + vc0[col] = x * y; + }; -// acc from [topk, K] to [K] -template -inline void sum_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ input, int64_t topk, int64_t K) { - using bVec = at::vec::Vectorized; - using fVec = at::vec::Vectorized; - constexpr int kVecSize = bVec::size(); - if (topk == 1) { - // do copy for topk = 1 - copy_stub(out, input, K); - } else { - // do sum for topk != 1 - int64_t d; - #pragma GCC unroll 4 - for (d = 0; d <= K - kVecSize; d += kVecSize) { - fVec sum_fvec0 = fVec(0.f); - fVec sum_fvec1 = fVec(0.f); - for (int t = 0; t < topk; ++t) { - bVec x_bvec = bVec::loadu(input + t * K + d); - fVec x_fvec0, x_fvec1; - std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec); - - sum_fvec0 += x_fvec0; - sum_fvec1 += x_fvec1; - } - bVec out_bvec = convert_from_float_ext(sum_fvec0, sum_fvec1); - out_bvec.store(out + d); - } - for (; d < K; ++d) { - float sum_val = 0.f; - for (int t = 0; t < topk; ++t) { - sum_val += static_cast(input[t * K + d]); - } - out[d] = static_cast(sum_val); + auto storec = [&](auto col, int64_t m) { + if constexpr (col % 2 == 0) { + fVec x0 = fVec(vc0[col + 0]); + fVec x1 = fVec(vc0[col + 1]); + bVec out_vec = convert_from_float_ext(x0, x1); + out_vec.store(C + m * N + col * 16); } + }; + + for (int64_t m = 0; m < m_size; ++m) { + Unroll{}(scalec, m); + Unroll{}(silu_and_mul); + Unroll{}(storec, m); } +#else + TORCH_CHECK(false, "silu_and_mul: scalar path not implemented!"); +#endif } -// out = input + input2 * scale -template -inline void add_mul_stub(scalar_t* __restrict__ out, const float* __restrict__ input, - const scalar_t* __restrict__ input2, float scale, int64_t size) { +template +inline void scale_C( + float* __restrict__ C, + const int32_t* __restrict__ Ctmp, + const float* __restrict__ As, + const float* __restrict__ Bs, + const int32_t* __restrict__ Bcomp, + int64_t m_size) { +#if defined(CPU_CAPABILITY_AVX512) + constexpr int COLS = BLOCK_N / 16; + static_assert(COLS % 2 == 0); - using bVec = at::vec::Vectorized; - using fVec = at::vec::Vectorized; - constexpr int kVecSize = bVec::size(); - const fVec s_vec = fVec(scale); - int64_t d; - #pragma GCC unroll 4 - for (d = 0; d <= size - kVecSize; d += kVecSize) { - fVec x0 = fVec::loadu(input + d); - fVec x1 = fVec::loadu(input + d + fVec::size()); + __m512 vc[COLS]; + __m512i vcomp[COLS]; + __m512 vas; + __m512 vbs[COLS]; - bVec y_bvec = bVec::loadu(input2 + d); - fVec y0, y1; - std::tie(y0, y1) = at::vec::convert_to_float(y_bvec); + auto load_scale_and_comp = [&](auto col) { + vcomp[col] = _mm512_loadu_si512(Bcomp + col * 16); + vbs[col] = _mm512_loadu_ps(Bs + col * 16); + }; + Unroll{}(load_scale_and_comp); - x0 = x0 + y0 * s_vec; - x1 = x1 + y1 * s_vec; - bVec out_vec = convert_from_float_ext(x0, x1); - out_vec.store(out + d); - } - for (; d < size; ++d) { - out[d] = static_cast(input[d] + float(input2[d]) * scale); + auto scalec = [&](auto col, int64_t m) { + // update As + vas = _mm512_set1_ps(As[m]); + // C = As * (C - Bcomp) * Bs + __m512i vc32 = _mm512_loadu_si512(Ctmp + m * BLOCK_N + col * 16); + vc[col] = _mm512_cvtepi32_ps(_mm512_sub_epi32(vc32, vcomp[col])); + vc[col] = _mm512_mul_ps(_mm512_mul_ps(vc[col], vas), vbs[col]); + _mm512_storeu_ps(C + m * BLOCK_N + col * 16, vc[col]); + }; + + for (int64_t m = 0; m < m_size; ++m) { + Unroll{}(scalec, m); } +#else + TORCH_CHECK(false, "scale_C: scalar path not implemented!"); +#endif } /// gemm for w13 template struct tinygemm_kernel_vnni { static inline void apply( - const uint8_t* __restrict__ A, const int8_t* __restrict__ B0, const int8_t* __restrict__ B1, scalar_t* __restrict__ C, - const float* __restrict__ As, const float* __restrict__ Bs0, const float* __restrict__ Bs1, - const int32_t* __restrict__ Bcomp0, const int32_t* __restrict__ Bcomp1, - int64_t K, int64_t lda, int64_t ldb, int64_t ldc) { + const uint8_t* __restrict__ A, + const int8_t* __restrict__ B0, + const int8_t* __restrict__ B1, + scalar_t* __restrict__ C, + const float* __restrict__ As, + const float* __restrict__ Bs0, + const float* __restrict__ Bs1, + const int32_t* __restrict__ Bcomp0, + const int32_t* __restrict__ Bcomp1, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { TORCH_CHECK(false, "tinygemm_kernel_nn: scalar path not implemented!"); } }; @@ -127,11 +148,19 @@ struct tinygemm_kernel_vnni { template struct tinygemm_kernel_vnni { static inline void apply( - const uint8_t* __restrict__ A, const int8_t* __restrict__ B0, const int8_t* __restrict__ B1, at::BFloat16* __restrict__ C, - const float* __restrict__ As, const float* __restrict__ Bs0, const float* __restrict__ Bs1, - const int32_t* __restrict__ Bcomp0, const int32_t* __restrict__ Bcomp1, - int64_t K, int64_t lda, int64_t ldb, int64_t ldc) { - + const uint8_t* __restrict__ A, + const int8_t* __restrict__ B0, + const int8_t* __restrict__ B1, + at::BFloat16* __restrict__ C, + const float* __restrict__ As, + const float* __restrict__ Bs0, + const float* __restrict__ Bs1, + const int32_t* __restrict__ Bcomp0, + const int32_t* __restrict__ Bcomp1, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { constexpr int ROWS = BLOCK_M; constexpr int COLS = BLOCK_N / 16; static_assert(COLS % 2 == 0); @@ -143,9 +172,9 @@ struct tinygemm_kernel_vnni { __m512i vc1[ROWS * COLS]; __m512i vcomp0[COLS]; __m512i vcomp1[COLS]; - __m512 was; - __m512 vbs0[COLS]; - __m512 vbs1[COLS]; + __m512 vas; + __m512 vbs0[COLS]; + __m512 vbs1[COLS]; auto loadc = [&](auto i) { vc0[i] = _mm512_set1_epi32(0); @@ -155,7 +184,7 @@ struct tinygemm_kernel_vnni { const int64_t K4 = K >> 2; const int64_t lda4 = lda >> 2; - const int64_t ldb4 = ldb; // ldb * 4 >> 2; + const int64_t ldb4 = ldb; // ldb * 4 >> 2; const int32_t* a_ptr = reinterpret_cast(A); const int32_t* b0_ptr = reinterpret_cast(B0); const int32_t* b1_ptr = reinterpret_cast(B1); @@ -183,8 +212,8 @@ struct tinygemm_kernel_vnni { constexpr int col = i % COLS; // load a scale - if constexpr(col == 0) { - was = _mm512_set1_ps(As[row]); + if constexpr (col == 0) { + vas = _mm512_set1_ps(As[row]); } // load b scale and vcomp if constexpr (row == 0) { @@ -195,8 +224,8 @@ struct tinygemm_kernel_vnni { } __m512 c0 = _mm512_cvtepi32_ps(_mm512_sub_epi32(vc0[i], vcomp0[col])); __m512 c1 = _mm512_cvtepi32_ps(_mm512_sub_epi32(vc1[i], vcomp1[col])); - vc0[i] = _mm512_castps_si512(_mm512_mul_ps(_mm512_mul_ps(c0, was), vbs0[col])); - vc1[i] = _mm512_castps_si512(_mm512_mul_ps(_mm512_mul_ps(c1, was), vbs1[col])); + vc0[i] = _mm512_castps_si512(_mm512_mul_ps(_mm512_mul_ps(c0, vas), vbs0[col])); + vc1[i] = _mm512_castps_si512(_mm512_mul_ps(_mm512_mul_ps(c1, vas), vbs1[col])); }; Unroll{}(scalec); @@ -221,19 +250,28 @@ struct tinygemm_kernel_vnni { _mm512_storeu_si512( reinterpret_cast<__m512i*>((C + row * ldc + col * 16)), (__m512i)(_mm512_cvtne2ps_pbh(__m512(x1), __m512(x0)))); - } + } }; Unroll{}(storec); } }; #endif -#define LAUNCH_TINYGEMM_KERNEL_VNNI(MB_SIZE, NB_SIZE) \ - tinygemm_kernel_vnni::apply( \ - A + mb_start * lda, B0 + nb_start * 4, B1 + nb_start * 4, \ - C + mb_start * ldc + nb_start, As + mb_start, \ - Bs0 + nb_start, Bs1 + nb_start, Bcomp0 + nb_start, Bcomp1 + nb_start,\ - K, lda, ldb, ldc); +#define LAUNCH_TINYGEMM_KERNEL_VNNI(MB_SIZE, NB_SIZE) \ + tinygemm_kernel_vnni::apply( \ + A + mb_start * lda, \ + B0 + nb_start * 4, \ + B1 + nb_start * 4, \ + C + mb_start * ldc + nb_start, \ + As + mb_start, \ + Bs0 + nb_start, \ + Bs1 + nb_start, \ + Bcomp0 + nb_start, \ + Bcomp1 + nb_start, \ + K, \ + lda, \ + ldb, \ + ldc); template void tinygemm_kernel( @@ -250,7 +288,6 @@ void tinygemm_kernel( int64_t lda, int64_t ldb, int64_t ldc) { - const int32_t* Bcomp0 = reinterpret_cast(B0 + block_size_n() * K); const int32_t* Bcomp1 = reinterpret_cast(B1 + block_size_n() * K); @@ -266,12 +303,21 @@ void tinygemm_kernel( int64_t nb_start = nb * BLOCK_N; int64_t nb_size = std::min(BLOCK_N, N - nb_start); - switch(mb_size << 4 | nb_size >> 4) { - case 0x12: LAUNCH_TINYGEMM_KERNEL_VNNI(1, 32); break; - case 0x22: LAUNCH_TINYGEMM_KERNEL_VNNI(2, 32); break; - case 0x32: LAUNCH_TINYGEMM_KERNEL_VNNI(3, 32); break; - case 0x42: LAUNCH_TINYGEMM_KERNEL_VNNI(4, 32); break; - default: TORCH_CHECK(false, "Unexpected block size, ", mb_size, "x", nb_size); + switch (mb_size << 4 | nb_size >> 4) { + case 0x12: + LAUNCH_TINYGEMM_KERNEL_VNNI(1, 32); + break; + case 0x22: + LAUNCH_TINYGEMM_KERNEL_VNNI(2, 32); + break; + case 0x32: + LAUNCH_TINYGEMM_KERNEL_VNNI(3, 32); + break; + case 0x42: + LAUNCH_TINYGEMM_KERNEL_VNNI(4, 32); + break; + default: + TORCH_CHECK(false, "Unexpected block size, ", mb_size, "x", "nb_size"); } } } @@ -281,9 +327,16 @@ void tinygemm_kernel( template struct tinygemm_kernel_vnni2 { static inline void apply( - const uint8_t* __restrict__ A, const int8_t* __restrict__ B, float* __restrict__ C, - const float* __restrict__ As, const float* __restrict__ Bs, const int32_t* __restrict__ Bcomp, - int64_t K, int64_t lda, int64_t ldb, int64_t ldc) { + const uint8_t* __restrict__ A, + const int8_t* __restrict__ B, + float* __restrict__ C, + const float* __restrict__ As, + const float* __restrict__ Bs, + const int32_t* __restrict__ Bcomp, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { TORCH_CHECK(false, "tinygemm_kernel_nn: scalar path not implemented!"); } }; @@ -292,10 +345,16 @@ struct tinygemm_kernel_vnni2 { template struct tinygemm_kernel_vnni2 { static inline void apply( - const uint8_t* __restrict__ A, const int8_t* __restrict__ B, float* __restrict__ C, - const float* __restrict__ As, const float* __restrict__ Bs, const int32_t* __restrict__ Bcomp, - int64_t K, int64_t lda, int64_t ldb, int64_t ldc) { - + const uint8_t* __restrict__ A, + const int8_t* __restrict__ B, + float* __restrict__ C, + const float* __restrict__ As, + const float* __restrict__ Bs, + const int32_t* __restrict__ Bcomp, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { constexpr int ROWS = BLOCK_M; constexpr int COLS = BLOCK_N / 16; static_assert(COLS % 2 == 0); @@ -304,17 +363,15 @@ struct tinygemm_kernel_vnni2 { __m512i vb[COLS]; __m512i vc[ROWS * COLS]; __m512i vcomp[COLS]; - __m512 was; - __m512 vbs[COLS]; + __m512 vas; + __m512 vbs[COLS]; - auto loadc = [&](auto i) { - vc[i] = _mm512_set1_epi32(0); - }; + auto loadc = [&](auto i) { vc[i] = _mm512_set1_epi32(0); }; Unroll{}(loadc); const int64_t K4 = K >> 2; const int64_t lda4 = lda >> 2; - const int64_t ldb4 = ldb; // ldb * 4 >> 2; + const int64_t ldb4 = ldb; // ldb * 4 >> 2; const int32_t* a_ptr = reinterpret_cast(A); const int32_t* b_ptr = reinterpret_cast(B); @@ -339,8 +396,8 @@ struct tinygemm_kernel_vnni2 { constexpr int col = i % COLS; // load a scale - if constexpr(col == 0) { - was = _mm512_set1_ps(As[row]); + if constexpr (col == 0) { + vas = _mm512_set1_ps(As[row]); } // load b scale and vcomp per 2 vectors // also load bias if any @@ -353,7 +410,7 @@ struct tinygemm_kernel_vnni2 { } } __m512 x = _mm512_cvtepi32_ps(_mm512_sub_epi32(vc[i], vcomp[col])); - x = _mm512_mul_ps(_mm512_mul_ps(x, was), vbs[col]); + x = _mm512_mul_ps(_mm512_mul_ps(x, vas), vbs[col]); _mm512_storeu_ps(reinterpret_cast<__m512*>(C + row * ldc + col * 16), x); }; Unroll{}(storec); @@ -361,11 +418,18 @@ struct tinygemm_kernel_vnni2 { }; #endif -#define LAUNCH_TINYGEMM_KERNEL_VNNI2(MB_SIZE, NB_SIZE) \ - tinygemm_kernel_vnni2::apply( \ - A + mb_start * lda, B + nb_start * 4, C + mb_start * ldc + nb_start, \ - As + mb_start, Bs + nb_start, Bcomp + nb_start, \ - K, lda, ldb, ldc); +#define LAUNCH_TINYGEMM_KERNEL_VNNI2(MB_SIZE, NB_SIZE) \ + tinygemm_kernel_vnni2::apply( \ + A + mb_start * lda, \ + B + nb_start * 4, \ + C + mb_start * ldc + nb_start, \ + As + mb_start, \ + Bs + nb_start, \ + Bcomp + nb_start, \ + K, \ + lda, \ + ldb, \ + ldc); template void tinygemm_kernel( @@ -380,7 +444,6 @@ void tinygemm_kernel( int64_t lda, int64_t ldb, int64_t ldc) { - // B compensation const int32_t* Bcomp = reinterpret_cast(B + block_size_n() * K); @@ -396,18 +459,27 @@ void tinygemm_kernel( int64_t nb_start = nb * BLOCK_N; int64_t nb_size = std::min(BLOCK_N, N - nb_start); - switch(mb_size << 4 | nb_size >> 4) { - case 0x12: LAUNCH_TINYGEMM_KERNEL_VNNI2(1, 32); break; - case 0x22: LAUNCH_TINYGEMM_KERNEL_VNNI2(2, 32); break; - case 0x32: LAUNCH_TINYGEMM_KERNEL_VNNI2(3, 32); break; - case 0x42: LAUNCH_TINYGEMM_KERNEL_VNNI2(4, 32); break; - default: TORCH_CHECK(false, "Unexpected block size, ", mb_size, "x", nb_size); + switch (mb_size << 4 | nb_size >> 4) { + case 0x12: + LAUNCH_TINYGEMM_KERNEL_VNNI2(1, 32); + break; + case 0x22: + LAUNCH_TINYGEMM_KERNEL_VNNI2(2, 32); + break; + case 0x32: + LAUNCH_TINYGEMM_KERNEL_VNNI2(3, 32); + break; + case 0x42: + LAUNCH_TINYGEMM_KERNEL_VNNI2(4, 32); + break; + default: + TORCH_CHECK(false, "Unexpected block size, ", mb_size, "x", "nb_size"); } } } } -} // anonymous namespace +} // anonymous namespace template void fused_experts_int8_kernel_impl( @@ -433,7 +505,6 @@ void fused_experts_int8_kernel_impl( int64_t E, int64_t topk, int64_t num_tokens_post_pad) { - // handle 2 tiles per block constexpr int64_t BLOCK_M = block_size_m(); constexpr int64_t BLOCK_N = block_size_n(); @@ -441,11 +512,7 @@ void fused_experts_int8_kernel_impl( // stage 0: quantize input to uint8, [M, K] at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) { for (int64_t m = begin; m < end; ++m) { - quantize_row_int8( - Aq_tmp + m * K, - As_tmp[m], - input + m * K, - K); + quantize_row_int8(Aq_tmp + m * K, As_tmp[m], input + m * K, K); } }); @@ -462,66 +529,107 @@ void fused_experts_int8_kernel_impl( const int64_t stride_e = 2 * N * packed_K; const int64_t stride_n = packed_K; + + int64_t avg_M = std::max(int64_t(1), M * topk / E); + const bool use_brgemm = can_use_brgemm(avg_M); + // here we only parallel on half of 2N to fuse silu_and_mul with gemm - at::parallel_for(0, MB * NB, 0, [&](int64_t begin, int64_t end) { + parallel_2d(MB, NB, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { // get local pointers - int tid = at::get_thread_num(); + int tid = get_thread_num(); uint8_t* __restrict__ A = A_tmp + tid * BLOCK_M * K; + int32_t* __restrict__ C0 = reinterpret_cast(C_tmp) + tid * 2 * BLOCK_M * BLOCK_N; + int32_t* __restrict__ C1 = C0 + BLOCK_M * BLOCK_N; alignas(64) float As[BLOCK_M]; - for (int64_t i = begin; i < end; ++i) { - int64_t mb = i / NB; - int64_t nb = i % NB; - - // nb0 from top half and nb1 from bottom half - int64_t nb0 = nb, nb1 = nb + NB; - int64_t n_size = std::min(N - nb0 * BLOCK_N, BLOCK_N); + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * K * 2, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + // nb_upper from top half and nb_lower from bottom half + int64_t nb_upper = nb, nb_lower = nb + NB; + int64_t n_size = std::min(N - nb * BLOCK_N, BLOCK_N); // B shape [K, n_size] in vnni format int32_t expert_id = expert_ids[mb]; - const int8_t* __restrict__ B0 = packed_w1 + expert_id * stride_e + nb0 * BLOCK_N * stride_n; - const int8_t* __restrict__ B1 = packed_w1 + expert_id * stride_e + nb1 * BLOCK_N * stride_n; - const float* __restrict__ Bs0 = w1s + expert_id * 2 * N + nb0 * BLOCK_N; - const float* __restrict__ Bs1 = w1s + expert_id * 2 * N + nb1 * BLOCK_N; + const int8_t* __restrict__ B0 = packed_w1 + expert_id * stride_e + nb_upper * BLOCK_N * stride_n; + const int8_t* __restrict__ B1 = packed_w1 + expert_id * stride_e + nb_lower * BLOCK_N * stride_n; + const float* __restrict__ Bs0 = w1s + expert_id * 2 * N + nb_upper * BLOCK_N; + const float* __restrict__ Bs1 = w1s + expert_id * 2 * N + nb_lower * BLOCK_N; - // 1.a load A - const int32_t* A_ids = sorted_ids + mb * BLOCK_M; int64_t m_size = offsets[mb + 1] - offsets[mb]; - for (int64_t m = 0; m < m_size; ++m) { - int32_t index = A_ids[m] / topk; - copy_stub(A + m * K, Aq_tmp + index * K, K); - As[m] = As_tmp[index]; + if (nb_offset == 0) { + // 1.a load A + const int32_t* A_ids = sorted_ids + mb * BLOCK_M; + for (int64_t m = 0; m < m_size; ++m) { + int32_t index = A_ids[m] / topk; + copy_stub(A + m * K, Aq_tmp + index * K, K); + As[m] = As_tmp[index]; + } } - // fused 1.b: silu_and_mul(A @ B0, A @ B1) - const int64_t offset = offsets[mb]; - tinygemm_kernel( - /* A */ A, - /* B0 */ B0, - /* B1 */ B1, - /* C */ ic1 + offset * N + nb * BLOCK_N, - /* As */ As, - /* Bs0 */ Bs0, - /* Bs1 */ Bs1, - /* M */ m_size, - /* N */ n_size, - /* K */ K, - /* lda */ K, - /* ldb */ n_size, - /* ldc */ N); + if (use_brgemm) { + // 1.b gemm: C0 = A @ B0 + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ n_size, + /* K */ K, + /* lda */ K, + /* ldb */ n_size, + /* ldc */ BLOCK_N, + /* add_C */ false, + /* A */ A, + /* B */ B0, + /* C */ C0); + + // 1.c gemm: C1 = A @ B1 + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ n_size, + /* K */ K, + /* lda */ K, + /* ldb */ n_size, + /* ldc */ BLOCK_N, + /* add_C */ false, + /* A */ A, + /* B */ B1, + /* C */ C1); + + const int32_t* Bcomp0 = reinterpret_cast(B0 + block_size_n() * K); + const int32_t* Bcomp1 = reinterpret_cast(B1 + block_size_n() * K); + + // 1.d silu and mul + const int64_t offset = offsets[mb]; + silu_and_mul( + ic1 + offset * N + nb * BLOCK_N, C0, C1, As, Bs0, Bs1, Bcomp0, Bcomp1, m_size, N); + } else { + // fused 1.bcd: silu_and_mul(A @ B0, A @ B1) + const int64_t offset = offsets[mb]; + tinygemm_kernel( + /* A */ A, + /* B0 */ B0, + /* B1 */ B1, + /* C */ ic1 + offset * N + nb * BLOCK_N, + /* As */ As, + /* Bs0 */ Bs0, + /* Bs1 */ Bs1, + /* M */ m_size, + /* N */ n_size, + /* K */ K, + /* lda */ K, + /* ldb */ n_size, + /* ldc */ N); + } + }); + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); } }); // stage 1.5: quantize ic1 to uint8, [M * topk, N] at::parallel_for(0, M * topk, 0, [&](int64_t begin, int64_t end) { for (int64_t m = begin; m < end; ++m) { - quantize_row_int8( - Aq_tmp + m * N, - As_tmp[m], - ic1 + m * N, - N); + quantize_row_int8(Aq_tmp + m * N, As_tmp[m], ic1 + m * N, N); } }); @@ -535,16 +643,13 @@ void fused_experts_int8_kernel_impl( const int64_t stride_oc = packed_N; // parallel on [MB2, NB2] - at::parallel_for(0, MB2 * NB2, 0, [&](int64_t begin, int64_t end) { + parallel_2d(MB2, NB2, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { // get local pointers - int tid = at::get_thread_num(); - // we won't be using C1 for gemm2 + int tid = get_thread_num(); float* __restrict__ C = C_tmp + tid * 2 * BLOCK_M * BLOCK_N; + int32_t* __restrict__ C32 = reinterpret_cast(C + BLOCK_M * BLOCK_N); - for (int64_t i = begin; i < end; ++i) { - int64_t mb = i / NB2; - int64_t nb = i % NB2; - + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * IC, [&](int64_t mb, int64_t nb, int64_t nb_offset) { int64_t m_size = offsets[mb + 1] - offsets[mb]; int64_t n_size = std::min(OC - nb * BLOCK_N, BLOCK_N); @@ -560,18 +665,36 @@ void fused_experts_int8_kernel_impl( const float* __restrict__ Bs = w2s + expert_id * K + nb * BLOCK_N; // 2.a gemm: C = A @ B - tinygemm_kernel( - /* A */ A, - /* B */ B, - /* C */ C, - /* As */ As, - /* Bs */ Bs, - /* M */ m_size, - /* N */ n_size, - /* K */ IC, - /* lda */ IC, - /* ldb */ n_size, - /* ldc */ BLOCK_N); + if (use_brgemm) { + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ n_size, + /* K */ IC, + /* lda */ IC, + /* ldb */ n_size, + /* ldc */ BLOCK_N, + /* add_C */ false, + /* A */ A, + /* B */ B, + /* C */ C32); + + // apply scales + const int32_t* Bcomp = reinterpret_cast(B + block_size_n() * IC); + scale_C(C, C32, As, Bs, Bcomp, m_size); + } else { + tinygemm_kernel( + /* A */ A, + /* B */ B, + /* C */ C, + /* As */ As, + /* Bs */ Bs, + /* M */ m_size, + /* N */ n_size, + /* K */ IC, + /* lda */ IC, + /* ldb */ n_size, + /* ldc */ BLOCK_N); + } // 2.b copy from C to ic2 in original order // and also mul topk_weights in float32 @@ -580,6 +703,10 @@ void fused_experts_int8_kernel_impl( float weight = topk_weights[index]; copy_mul_stub(ic2 + index * K + nb * BLOCK_N, C + m * BLOCK_N, weight, n_size); } + }); + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); } }); @@ -592,17 +719,30 @@ void fused_experts_int8_kernel_impl( }); } -#define INSTANTIATE_MOE_INT8_TEMPLATE(TYPE) \ - template void fused_experts_int8_kernel_impl ( \ - TYPE* __restrict__ output, TYPE* __restrict__ ic1, \ - TYPE* __restrict__ ic2, uint8_t* __restrict__ A_tmp, \ - float* __restrict__ C_tmp, uint8_t* __restrict__ Aq_tmp, \ - float* __restrict__ As_tmp, const TYPE* __restrict__ input, \ - const int8_t* __restrict__ packed_w1, const int8_t* __restrict__ packed_w2, \ - const float* __restrict__ w1s, const float* __restrict__ w2s, \ - const float* __restrict__ topk_weights, const int32_t* __restrict__ sorted_ids, \ - const int32_t* __restrict__ expert_ids, const int32_t* __restrict__ offsets, \ - int64_t M, int64_t N, int64_t K, int64_t E, int64_t topk, int64_t num_tokens_post_pad) +#define INSTANTIATE_MOE_INT8_TEMPLATE(TYPE) \ + template void fused_experts_int8_kernel_impl( \ + TYPE* __restrict__ output, \ + TYPE* __restrict__ ic1, \ + TYPE* __restrict__ ic2, \ + uint8_t* __restrict__ A_tmp, \ + float* __restrict__ C_tmp, \ + uint8_t* __restrict__ Aq_tmp, \ + float* __restrict__ As_tmp, \ + const TYPE* __restrict__ input, \ + const int8_t* __restrict__ packed_w1, \ + const int8_t* __restrict__ packed_w2, \ + const float* __restrict__ w1s, \ + const float* __restrict__ w2s, \ + const float* __restrict__ topk_weights, \ + const int32_t* __restrict__ sorted_ids, \ + const int32_t* __restrict__ expert_ids, \ + const int32_t* __restrict__ offsets, \ + int64_t M, \ + int64_t N, \ + int64_t K, \ + int64_t E, \ + int64_t topk, \ + int64_t num_tokens_post_pad) INSTANTIATE_MOE_INT8_TEMPLATE(at::BFloat16); INSTANTIATE_MOE_INT8_TEMPLATE(at::Half); @@ -624,7 +764,6 @@ void shared_expert_int8_kernel_impl( int64_t M, int64_t N, int64_t K) { - // handle 2 tiles per block constexpr int64_t BLOCK_M = block_size_m(); constexpr int64_t BLOCK_N = block_size_n(); @@ -632,15 +771,11 @@ void shared_expert_int8_kernel_impl( // stage 0: quantize input to uint8, [M, K] at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) { for (int64_t m = begin; m < end; ++m) { - quantize_row_int8( - Aq_tmp + m * K, - As_tmp[m], - input + m * K, - K); + quantize_row_int8(Aq_tmp + m * K, As_tmp[m], input + m * K, K); } }); - // stage 1: intermediate_cache1 = silu(hidden_states @ w1) + // stage 1: intermediate_cache1 = silu(hidden_states @ w1) const int64_t MB = div_up(M, BLOCK_M); const int64_t NB = div_up(N, BLOCK_N); @@ -651,15 +786,20 @@ void shared_expert_int8_kernel_impl( const int64_t packed_N = get_row_size(N); const int64_t stride_n = packed_K; - // here we only parallel on half of 2N to fuse silu_and_mul with gemm - at::parallel_for(0, MB * NB, 0, [&](int64_t begin, int64_t end) { - for (int64_t i = begin; i < end; ++i) { - int64_t mb = i / NB; - int64_t nb = i % NB; + const bool use_brgemm = can_use_brgemm(M); + const bool apply_scaling_factor = fused_experts_out != nullptr; - // nb0 from top half and nb1 from bottom half - int64_t nb0 = nb, nb1 = nb + NB; - int64_t n_size = std::min(N - nb0 * BLOCK_N, BLOCK_N); + // here we only parallel on half of 2N to fuse silu_and_mul with gemm + parallel_2d(MB, NB, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + // get local pointers + int tid = get_thread_num(); + int32_t* __restrict__ C0 = reinterpret_cast(C_tmp) + tid * 2 * BLOCK_M * BLOCK_N; + int32_t* __restrict__ C1 = C0 + BLOCK_M * BLOCK_N; + + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * K * 2, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + // nb_upper from top half and nb_lower from bottom half + int64_t nb_upper = nb, nb_lower = nb + NB; + int64_t n_size = std::min(N - nb * BLOCK_N, BLOCK_N); int64_t m_size = std::min(M - mb * BLOCK_M, BLOCK_M); // A shape [m_size, K] @@ -667,37 +807,72 @@ void shared_expert_int8_kernel_impl( const float* As = As_tmp + mb * BLOCK_M; // B shape [K, n_size] in vnni format - const int8_t* __restrict__ B0 = packed_w1 + nb0 * BLOCK_N * stride_n; - const int8_t* __restrict__ B1 = packed_w1 + nb1 * BLOCK_N * stride_n; - const float* __restrict__ Bs0 = w1s + nb0 * BLOCK_N; - const float* __restrict__ Bs1 = w1s + nb1 * BLOCK_N; + const int8_t* __restrict__ B0 = packed_w1 + nb_upper * BLOCK_N * stride_n; + const int8_t* __restrict__ B1 = packed_w1 + nb_lower * BLOCK_N * stride_n; + const float* __restrict__ Bs0 = w1s + nb_upper * BLOCK_N; + const float* __restrict__ Bs1 = w1s + nb_lower * BLOCK_N; - // fused 1.b: silu_and_mul(A @ B0, A @ B1) - tinygemm_kernel( - /* A */ A, - /* B0 */ B0, - /* B1 */ B1, - /* C */ ic1 + mb * BLOCK_M * N + nb * BLOCK_N, - /* As */ As, - /* Bs0 */ Bs0, - /* Bs1 */ Bs1, - /* M */ m_size, - /* N */ n_size, - /* K */ K, - /* lda */ K, - /* ldb */ n_size, - /* ldc */ N); + if (use_brgemm) { + // 1.b gemm: C0 = A @ B0 + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ n_size, + /* K */ K, + /* lda */ K, + /* ldb */ n_size, + /* ldc */ BLOCK_N, + /* add_C */ false, + /* A */ A, + /* B */ B0, + /* C */ C0); + + // 1.c gemm: C1 = A @ B1 + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ n_size, + /* K */ K, + /* lda */ K, + /* ldb */ n_size, + /* ldc */ BLOCK_N, + /* add_C */ false, + /* A */ A, + /* B */ B1, + /* C */ C1); + + const int32_t* Bcomp0 = reinterpret_cast(B0 + block_size_n() * K); + const int32_t* Bcomp1 = reinterpret_cast(B1 + block_size_n() * K); + + // 1.d silu and mul + silu_and_mul( + ic1 + mb * BLOCK_M * N + nb * BLOCK_N, C0, C1, As, Bs0, Bs1, Bcomp0, Bcomp1, m_size, N); + } else { + // fused 1.bcd: silu_and_mul(A @ B0, A @ B1) + tinygemm_kernel( + /* A */ A, + /* B0 */ B0, + /* B1 */ B1, + /* C */ ic1 + mb * BLOCK_M * N + nb * BLOCK_N, + /* As */ As, + /* Bs0 */ Bs0, + /* Bs1 */ Bs1, + /* M */ m_size, + /* N */ n_size, + /* K */ K, + /* lda */ K, + /* ldb */ n_size, + /* ldc */ N); + } + }); + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); } }); // stage 1.5: quantize ic1 to uint8, [M * topk, N] at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) { for (int64_t m = begin; m < end; ++m) { - quantize_row_int8( - Aq_tmp + m * N, - As_tmp[m], - ic1 + m * N, - N); + quantize_row_int8(Aq_tmp + m * N, As_tmp[m], ic1 + m * N, N); } }); @@ -710,16 +885,13 @@ void shared_expert_int8_kernel_impl( const int64_t stride_oc = packed_N; // parallel on [MB2, NB2] - at::parallel_for(0, MB2 * NB2, 0, [&](int64_t begin, int64_t end) { + parallel_2d(MB2, NB2, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { // get local pointers - int tid = at::get_thread_num(); - // we won't be using C1 for gemm2 + int tid = get_thread_num(); float* __restrict__ C = C_tmp + tid * 2 * BLOCK_M * BLOCK_N; + int32_t* __restrict__ C32 = reinterpret_cast(C + BLOCK_M * BLOCK_N); - for (int64_t i = begin; i < end; ++i) { - int64_t mb = i / NB2; - int64_t nb = i % NB2; - + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * IC, [&](int64_t mb, int64_t nb, int64_t nb_offset) { int64_t m_size = std::min(M - mb * BLOCK_M, BLOCK_M); int64_t n_size = std::min(OC - nb * BLOCK_N, BLOCK_N); @@ -731,39 +903,71 @@ void shared_expert_int8_kernel_impl( const int8_t* __restrict__ B = packed_w2 + nb * BLOCK_N * stride_oc; const float* __restrict__ Bs = w2s + nb * BLOCK_N; - // 2.a gemm: C = A @ B - tinygemm_kernel( - /* A */ A, - /* B */ B, - /* C */ C, - /* As */ As, - /* Bs */ Bs, - /* M */ m_size, - /* N */ n_size, - /* K */ IC, - /* lda */ IC, - /* ldb */ n_size, - /* ldc */ BLOCK_N); + if (use_brgemm) { + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ n_size, + /* K */ IC, + /* lda */ IC, + /* ldb */ n_size, + /* ldc */ BLOCK_N, + /* add_C */ false, + /* A */ A, + /* B */ B, + /* C */ C32); + + // apply scales + const int32_t* Bcomp = reinterpret_cast(B + block_size_n() * IC); + scale_C(C, C32, As, Bs, Bcomp, m_size); + } else { + // 2.a gemm: C = A @ B + tinygemm_kernel( + /* A */ A, + /* B */ B, + /* C */ C, + /* As */ As, + /* Bs */ Bs, + /* M */ m_size, + /* N */ n_size, + /* K */ IC, + /* lda */ IC, + /* ldb */ n_size, + /* ldc */ BLOCK_N); + } // 2.b copy from C to output and add fused_experts_out scalar_t* __restrict__ out = output + mb * BLOCK_M * K + nb * BLOCK_N; - const scalar_t* __restrict__ fused_out = fused_experts_out + mb * BLOCK_M * K + nb * BLOCK_N; + const scalar_t* __restrict__ fused_out = + apply_scaling_factor ? fused_experts_out + mb * BLOCK_M * K + nb * BLOCK_N : nullptr; for (int64_t m = 0; m < m_size; ++m) { - add_mul_stub(out + m * K, C + m * BLOCK_N, fused_out + m * K, routed_scaling_factor, n_size); + const scalar_t* __restrict__ fused_out_row = apply_scaling_factor ? (fused_out + m * K) : nullptr; + add_mul_stub(out + m * K, C + m * BLOCK_N, fused_out_row, routed_scaling_factor, n_size); } + }); + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); } }); } -#define INSTANTIATE_SHARED_EXPERT_INT8_TEMPLATE(TYPE) \ - template void shared_expert_int8_kernel_impl ( \ - TYPE* __restrict__ output, TYPE* __restrict__ ic1, \ - float* __restrict__ C_tmp, uint8_t* __restrict__ Aq_tmp, \ - float* __restrict__ As_tmp, const TYPE* __restrict__ input, \ - const int8_t* __restrict__ packed_w1, const int8_t* __restrict__ packed_w2, \ - const float* __restrict__ w1s, const float* __restrict__ w2s, \ - const TYPE* __restrict__ fused_experts_out, float routed_scaling_factor, \ - int64_t M, int64_t N, int64_t K) +#define INSTANTIATE_SHARED_EXPERT_INT8_TEMPLATE(TYPE) \ + template void shared_expert_int8_kernel_impl( \ + TYPE* __restrict__ output, \ + TYPE* __restrict__ ic1, \ + float* __restrict__ C_tmp, \ + uint8_t* __restrict__ Aq_tmp, \ + float* __restrict__ As_tmp, \ + const TYPE* __restrict__ input, \ + const int8_t* __restrict__ packed_w1, \ + const int8_t* __restrict__ packed_w2, \ + const float* __restrict__ w1s, \ + const float* __restrict__ w2s, \ + const TYPE* __restrict__ fused_experts_out, \ + float routed_scaling_factor, \ + int64_t M, \ + int64_t N, \ + int64_t K) INSTANTIATE_SHARED_EXPERT_INT8_TEMPLATE(at::BFloat16); INSTANTIATE_SHARED_EXPERT_INT8_TEMPLATE(at::Half); diff --git a/csrc/cpu/sgl-kernels/vec.h b/csrc/cpu/sgl-kernels/vec.h index 160845c9b1..c045a8cc7d 100644 --- a/csrc/cpu/sgl-kernels/vec.h +++ b/csrc/cpu/sgl-kernels/vec.h @@ -1,10 +1,10 @@ // Adapted from // https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu -#pragma once - // clang-format off +#pragma once + #if defined(__AVX512F__) && defined(__AVX512BF16__) && defined(__AMX_BF16__) #define CPU_CAPABILITY_AVX512 #endif @@ -16,38 +16,51 @@ namespace { using namespace at::vec; -template , int> = 0> +template , int> = 0> inline Vectorized convert_from_float_ext(const Vectorized& a, const Vectorized& b) { return at::vec::convert_from_float(a, b); } +// allow f16, bf16 +template , int> = 1> +inline std::tuple, Vectorized> load_float_vec2(const scalar_t* __restrict__ data) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + bVec x_vec = bVec::loadu(data); + fVec x0, x1; + std::tie(x0, x1) = at::vec::convert_to_float(x_vec); + return std::make_tuple(x0, x1); +} + +// allow f32 +inline std::tuple, Vectorized> load_float_vec2(const float* __restrict__ data) { + using fVec = at::vec::Vectorized; + fVec x0 = fVec::loadu(data); + fVec x1 = fVec::loadu(data + fVec::size()); + return std::make_tuple(x0, x1); +} + #if defined(CPU_CAPABILITY_AVX512) // `at::vec::convert_from_float<>` from PyTorch doesn't have avx512-bf16 intrinsics // use native instruction for bfloat16->float32 conversion template <> -inline Vectorized convert_from_float_ext(const Vectorized& a, const Vectorized& b) { +inline Vectorized +convert_from_float_ext(const Vectorized& a, const Vectorized& b) { return (__m512i)(_mm512_cvtne2ps_pbh(__m512(b), __m512(a))); } -#define CVT_BF16_TO_FP32(a) \ - _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(a), 16)) +#define CVT_BF16_TO_FP32(a) _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(a), 16)) -#define CVT_FP16_TO_FP32(a) \ - _mm512_cvtps_ph(a, (_MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC)) +#define CVT_FP16_TO_FP32(a) _mm512_cvtph_ps(a) // this doesn't handle NaN. inline __m512bh cvt_e4m3_bf16_intrinsic_no_nan(__m256i fp8_vec) { const __m512i x = _mm512_cvtepu8_epi16(fp8_vec); - - const __m512i mant = _mm512_slli_epi16(_mm512_and_si512(x, _mm512_set1_epi16(0x07)), 4); - const __m512i raw_exp = _mm512_srli_epi16(_mm512_and_si512(x, _mm512_set1_epi16(0x78)), 3); - const __m512i exp = _mm512_slli_epi16(_mm512_add_epi16(raw_exp, _mm512_set1_epi16(120)), 7); - const __m512i nonsign = _mm512_or_si512(exp, mant); - - const __m512i sign = _mm512_slli_epi16(_mm512_and_si512(x, _mm512_set1_epi16(0x80)), 8); - const __m512i combined = _mm512_or_si512(nonsign, sign); + __m512i combined = _mm512_add_epi16(x, _mm512_set1_epi16(0x0780)); + combined = _mm512_slli_epi16(combined, 4); + combined = _mm512_and_si512(combined, _mm512_set1_epi16(0x87f0)); + combined = _mm512_add_epi16(combined, _mm512_set1_epi16(0x3c00)); const __mmask32 is_nonzero = _mm512_cmpneq_epi16_mask(x, _mm512_setzero_si512()); return (__m512bh)_mm512_maskz_mov_epi16(is_nonzero, combined); @@ -113,10 +126,77 @@ inline __m512bh CVT_FP8_TO_BF16(__m256i a) { #endif } +// faster version of float8_e4m3fn conversion to bfloat16 +// +// we mapped cuda implementation from below link and vectorized with avx512: +// https://github.com/thu-pacman/chitu/blob/1ed2078ec26581ebdca05b7306d4385f86edaa7c/csrc/cuda/marlin/marlin_gemm/dequant.h#L387 +// +inline __attribute__((always_inline)) __m512bh CVT_FP8_TO_BF16_EXT(__m256i a) { + const __m512i mask0 = _mm512_set1_epi16(0x80); // sign bit + const __m512i mask1 = _mm512_set1_epi16(0x7F); // exponent and mantissa + const __m512i mask2 = _mm512_set1_epi16(0x4000); + + __m512i x = _mm512_cvtepu8_epi16(a); + __m512i vsign = _mm512_and_si512(x, mask0); + vsign = _mm512_slli_epi16(vsign, 8); + + __m512i vexp_and_mant = _mm512_and_si512(x, mask1); + vexp_and_mant = _mm512_slli_epi16(vexp_and_mant, 4); + + // _MM_TERNLOG_A | _MM_TERNLOG_B | _MM_TERNLOG_C: 0b11111110 + return (__m512bh)(_mm512_ternarylogic_epi32(vsign, mask2, vexp_and_mant, 0b11111110)); +} + +// bias for conversion of fp8 to bf16 1/256 in float32 +#define kFP8_BIAS 0x3b800000 + +// remove warning: ignoring attributes on template argument ‘__m512bh’ [-Wignored-attributes] +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wignored-attributes" + +#define MXFP4_VALUES \ + -6.0f, -4.0f, -3.0f, -2.0f, -1.5f, -1.0f, -0.5f, -0.0f, 6.0f, 4.0f, 3.0f, 2.0f, 1.5f, 1.0f, 0.5f, 0.0f + +// convert 64 mxfp4 to 2x bf16 vectors, expect input 32-way packing +inline std::tuple<__m512bh, __m512bh> cvt_mxfp4_e2m1_bf16_intrinsic_lut(__m256i a, __m512i s0, __m512i s1) { + // LUT + const __m512 values = _mm512_set_ps(MXFP4_VALUES); + const __m512i lut = (__m512i)(_mm512_cvtne2ps_pbh(values, values)); + + const __m512i abs_mask = _mm512_set1_epi16(0x7FFF); + const __m512i zero = _mm512_setzero_si512(); + + // expand values to 16-bit integers + __m512i x0 = _mm512_cvtepu8_epi16(a); + __m512i x1 = _mm512_srli_epi32(x0, 4); + + // LUT to convert mxfp4 values to bf16 + x0 = _mm512_permutexvar_epi16(x0, lut); + x1 = _mm512_permutexvar_epi16(x1, lut); + + // check for zeros + __mmask32 mask0 = _mm512_cmp_epi16_mask(_mm512_and_si512(x0, abs_mask), zero, _MM_CMPINT_EQ); + __mmask32 mask1 = _mm512_cmp_epi16_mask(_mm512_and_si512(x1, abs_mask), zero, _MM_CMPINT_EQ); + + // emulate bf16 mul with scale factor + x0 = _mm512_add_epi16(x0, s0); + x1 = _mm512_add_epi16(x1, s1); + + // blend with zero + x0 = _mm512_mask_blend_epi16(mask0, x0, zero); + x1 = _mm512_mask_blend_epi16(mask1, x1, zero); + + return std::make_tuple(__m512bh(x0), __m512bh(x1)); +} + +#define CVT_MXFP4_TO_BF16(a, s0, s1) cvt_mxfp4_e2m1_bf16_intrinsic_lut(a, s0, s1) + +#pragma GCC diagnostic pop + #endif // vector to scalar reduction -#if defined(CPU_CAPABILITY_AVX512) && 0 +#if defined(CPU_CAPABILITY_AVX512) inline float vec_reduce_sum(const Vectorized& a) { return _mm512_reduce_add_ps(__m512(a)); } @@ -136,10 +216,9 @@ inline float vec_reduce_max(const Vectorized& a) { // https://github.com/InternLM/lmdeploy/blob/086481ed84b59bee3b8e4274e5fc69620040c048/lmdeploy/pytorch/kernels/cuda/w8a8_triton_kernels.py#L282 template -inline void quantize_row_int8(uint8_t* __restrict__ Aq, float& As, - const scalar_t* __restrict__ A, int64_t K, float eps = 1e-7) { - - float amax = 0.f; // absolute max +inline void +quantize_row_int8(uint8_t* __restrict__ Aq, float& As, const scalar_t* __restrict__ A, int64_t K, float eps = 1e-7) { + float amax = 0.f; // absolute max for (int64_t k = 0; k < K; ++k) { const float val = static_cast(A[k]); amax = std::max(amax, std::abs(val)); @@ -158,9 +237,8 @@ inline void quantize_row_int8(uint8_t* __restrict__ Aq, float& As, #if defined(CPU_CAPABILITY_AVX512) template <> -inline void quantize_row_int8(uint8_t* __restrict__ Aq, float& As, - const at::BFloat16* __restrict__ A, int64_t K, float eps) { - +inline void quantize_row_int8( + uint8_t* __restrict__ Aq, float& As, const at::BFloat16* __restrict__ A, int64_t K, float eps) { const __m512 signBit = _mm512_set1_ps(-0.0f); const __m512i off = _mm512_set1_epi32(128); @@ -200,7 +278,7 @@ inline void quantize_row_int8(uint8_t* __restrict__ Aq, float& As, // transpose utils // taken from my PR in ggml: https://github.com/ggml-org/llama.cpp/pull/8998 #if defined(CPU_CAPABILITY_AVX512) -inline void transpose_16x16_32bit(__m512i * v) { +inline void transpose_16x16_32bit(__m512i* v) { __m512i v1[16]; v1[0] = _mm512_unpacklo_epi32(v[0], v[1]); v1[1] = _mm512_unpackhi_epi32(v[0], v[1]); @@ -293,16 +371,56 @@ inline std::tuple<__m512i, __m512i> transpose_2x32_16bit(__m512i r0, __m512i r1) } #pragma GCC diagnostic pop +inline __attribute__((always_inline)) __m512 _mm512_fexp_u20_ps(const __m512 values) { + const __m512 vec_c0 = _mm512_set1_ps(0.00010703434948458272f); + const __m512 vec_c1 = _mm512_set1_ps(0.30354260500649682f); + const __m512 vec_c2 = _mm512_set1_ps(-0.22433836478672356); + const __m512 vec_c3 = _mm512_set1_ps(-0.079204240219773236); + + const __m512 vec_exp_log2ef = _mm512_castsi512_ps(_mm512_set1_epi32(0x3fb8aa3b)); // log2(e) + + const __m512 vec_a = _mm512_set1_ps(std::pow(2, 23) / std::log2(2)); + const __m512 vec_b = _mm512_set1_ps(std::pow(2, 23) * 127.f); + + const __m512 vec_ln_flt_min = _mm512_castsi512_ps(_mm512_set1_epi32(0xc2aeac50)); + const __m512 vec_ln_flt_max = _mm512_castsi512_ps(_mm512_set1_epi32(0x42b17218)); + __m512i vec_infinity = _mm512_set1_epi32(0x7F800000); + __m512i vec_zero = _mm512_setzero_epi32(); + + // Fast Exponential Computation on SIMD Architectures + // A. Cristiano I. Malossi, Yves Ineichen, Costas Bekas, and Alessandro + // Curioni exp(x) = 2**(x * log2(e)) + // = 2**xi * 2**xf - TIPS we are using the EEEE floating point + // representation with identification to the exponent and the + // mentissa + // 2**xf will be approximated to a polynomial of degree 3 computed with + // Horner method + // mask for the boundary condition + auto min_mask = _mm512_cmp_ps_mask(values, vec_ln_flt_min, _CMP_LT_OS); + auto max_mask = _mm512_cmp_ps_mask(values, vec_ln_flt_max, _CMP_GT_OS); + + // transformation with log2(e) + auto vec_src = _mm512_mul_ps(values, vec_exp_log2ef); + auto vec_fractional = _mm512_sub_ps(vec_src, _mm512_floor_ps(vec_src)); + + // compute polynomial using Horner Scheme, for superscalar processor + auto vec_res = _mm512_fmadd_ps(vec_fractional, vec_c3, vec_c2); + vec_res = _mm512_fmadd_ps(vec_fractional, vec_res, vec_c1); + vec_res = _mm512_fmadd_ps(vec_fractional, vec_res, vec_c0); + + vec_src = _mm512_sub_ps(vec_src, vec_res); + // the tips is here, headache in perspective + auto tmp = _mm512_fmadd_ps(vec_a, vec_src, vec_b); + // headache bis - we loose precision with the cast but it "fits", but ok + // after f32 -> f16 later + __m512i casted_integer = _mm512_cvttps_epi32(tmp); + // boundary condition, lower than the min -> 0 + casted_integer = _mm512_mask_mov_epi32(casted_integer, min_mask, vec_zero); + // boundary condition, larger than the max -> +oo + casted_integer = _mm512_mask_mov_epi32(casted_integer, max_mask, vec_infinity); + // final interpretation to float + return _mm512_castsi512_ps(casted_integer); +} #endif -// TODO: debug print, remove me later -template -void print_array(scalar_t* ptr, int size) { - for (int d = 0; d < size; ++d) { - if (d % 16 == 0) { std::cout << std::endl; } - std::cout << ptr[d] << " "; - } - std::cout << std::endl; -} - -} // anonymous namespace +} // anonymous namespace diff --git a/csrc/cpu/sgl-kernels/vec_pack.h b/csrc/cpu/sgl-kernels/vec_pack.h new file mode 100644 index 0000000000..d245d2a407 --- /dev/null +++ b/csrc/cpu/sgl-kernels/vec_pack.h @@ -0,0 +1,299 @@ +// Adapted from +// https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu + +// clang-format off + +// To use the transpose functions +#include + +#include "vec.h" + +namespace { + +using namespace at::vec; + +template +inline index_t get_index(index_t* ind, int i) { + return (ind == nullptr) ? (index_t)i : ind[i]; +} + +#if defined(CPU_CAPABILITY_AVX512) +// key: from [N, 32] to [32/2, N, 2] +template +inline void pack_vnni_Nx32( + scalar_t* __restrict__ dst, + const scalar_t* __restrict__ src, + const index_t* __restrict__ ind, + int N, + int ld_src, + int ld_dst) { + __m512i vinputs[16]; + + int n = 0; + for (; n < N; ++n) { + index_t index = get_index(ind, n); + vinputs[n] = _mm512_loadu_si512(src + index * ld_src); + } + // padding with zero to avoid uninitialized vectors + for (; n < 16; ++n) { + vinputs[n] = _mm512_set1_epi32(0); + } + + // pack key + transpose_16x16_32bit(vinputs); + + const __mmask16 vmask = (1 << N) - 1; + for (int k = 0; k < 16; ++k) { + _mm512_mask_storeu_epi32(dst + k * ld_dst * 2, vmask, vinputs[k]); + } +} + +template +inline void pack_vnni_N_remainder( + scalar_t* __restrict__ dst, + const scalar_t* __restrict__ src, + const index_t* __restrict__ ind, + int N, + int K, + int ld_src, + int ld_dst) { + __m512i vinputs[16]; + + int K2 = K >> 1; + const __mmask16 vmask = (1 << K2) - 1; + + int n = 0; + for (; n < N; ++n) { + index_t index = get_index(ind, n); + vinputs[n] = _mm512_maskz_loadu_epi32(vmask, src + index * ld_src); + } + // padding with zero to avoid uninitialized vectors + for (; n < 16; ++n) { + vinputs[n] = _mm512_set1_epi32(0); + } + + // pack key + transpose_16x16_32bit(vinputs); + + const __mmask16 vmask2 = (1 << N) - 1; + for (int k = 0; k < K2; ++k) { + _mm512_mask_storeu_epi32(dst + k * ld_dst * 2, vmask2, vinputs[k]); + } +} + +// value: from [K, 32] to [K/2, 32, 2] +template +inline void pack_vnni_Kx32( + scalar_t* __restrict__ dst, + const scalar_t* __restrict__ src, + const index_t* __restrict__ ind, + int K, + int ld_src, + int ld_dst) { + __m512i vinputs[2]; + + int k = 0; + for (; k < K; ++k) { + index_t index = get_index(ind, k); + vinputs[k] = _mm512_loadu_si512(src + index * ld_src); + } + // padding with zero to avoid uninitialized vectors + for (; k < 2; ++k) { + vinputs[k] = _mm512_set1_epi32(0); + } + + // pack value + __m512i d0, d1; + std::tie(d0, d1) = transpose_2x32_16bit(vinputs[0], vinputs[1]); + _mm512_storeu_si512(dst + 0 * ld_dst * 2, d0); + _mm512_storeu_si512(dst + 0 * ld_dst * 2 + 32, d1); +} + +template +inline void pack_vnni_K_remainder( + scalar_t* __restrict__ dst, + const scalar_t* __restrict__ src, + const index_t* __restrict__ ind, + int K, + int N, + int ld_src, + int ld_dst) { + __m512i vinputs[2]; + + const __mmask32 vmask = (1 << N) - 1; + + int k = 0; + for (; k < K; ++k) { + index_t index = get_index(ind, k); + vinputs[k] = _mm512_maskz_loadu_epi16(vmask, src + index * ld_src); + } + // padding with zero to avoid uninitialized vectors + for (; k < 2; ++k) { + vinputs[k] = _mm512_set1_epi32(0); + } + + // pack value + __m512i d0, d1; + std::tie(d0, d1) = transpose_2x32_16bit(vinputs[0], vinputs[1]); + + if (N <= 16) { + // 2N * 16bits: N * 32bits + const __mmask16 vmask2 = (1 << N) - 1; + _mm512_mask_storeu_epi32(dst + 0 * ld_dst * 2, vmask2, d0); + } else { + // 2(N-16) * 16bits: (N-16) * 32bits + const __mmask16 vmask2 = (1 << (N - 16)) - 1; + _mm512_storeu_epi32(dst + 0 * ld_dst * 2, d0); + _mm512_mask_storeu_epi32(dst + 0 * ld_dst * 2 + 32, vmask2, d1); + } +} +#endif + +// convert to vnni format +// from [N, K/2, 2] to [K/2, N, 2] for bfloat16 and float16 +template +void pack_vnni( + scalar_t* __restrict__ dst, + const scalar_t* __restrict__ src, + const index_t* __restrict__ ind, + int N, + int K, + int ld_src, + int ld_dst) { +#if defined(CPU_CAPABILITY_AVX512) + const int NB = div_up(N, 16); + const int KB = K / 32; + const int K_remainder = K - KB * 32; + + for (int nb = 0; nb < NB; ++nb) { + int nb_size = std::min(N - nb * 16, 16); + for (int kb = 0; kb < KB; ++kb) { + // handle 16x512bits each block + pack_vnni_Nx32( + /* dst */ dst + ((kb * 32) >> 1) * ld_dst * 2 + nb * 16 * 2, + /* src */ src + kb * 32 + (is_indexed ? 0 : nb * 16 * ld_src), + /* ind */ is_indexed ? ind + nb * 16 : nullptr, + /* N */ nb_size, + /* ld_src */ ld_src, + /* ld_dst */ ld_dst); + } + if (K_remainder > 0) { + pack_vnni_N_remainder( + /* dst */ dst + ((KB * 32) >> 1) * ld_dst * 2 + nb * 16 * 2, + /* src */ src + KB * 32 + (is_indexed ? 0 : nb * 16 * ld_src), + /* ind */ is_indexed ? ind + nb * 16 : nullptr, + /* N */ nb_size, + /* K */ K_remainder, + /* ld_src */ ld_src, + /* ld_dst */ ld_dst); + } + } +#else + for (int n = 0; n < N; ++n) { + index_t index = get_index(ind, n); + for (int k = 0; k < K / 2; ++k) { + for (int d = 0; d < 2; ++d) { + dst[k * ld_dst * 2 + n * 2 + d] = src[index * ld_src + k * 2 + d]; + } + } + } +#endif +} + +template +void pack_vnni(scalar_t* __restrict__ dst, const scalar_t* __restrict__ src, int N, int K, int ld_src, int ld_dst) { + pack_vnni(dst, src, nullptr, N, K, ld_src, ld_dst); +} + +template +void pack_vnni( + scalar_t* __restrict__ dst, + const scalar_t* __restrict__ src, + const index_t* __restrict__ ind, + int N, + int K, + int ld_src, + int ld_dst) { + assert(ind != nullptr); + pack_vnni(dst, src, ind, N, K, ld_src, ld_dst); +} + +// convert to vnni format +// from [K/2, 2, N] to [K/2, N, 2] for bfloat16 and float16 +template +void pack_vnni2( + scalar_t* __restrict__ dst, + const scalar_t* __restrict__ src, + const index_t* __restrict__ ind, + int K, + int N, + int ld_src, + int ld_dst) { +#if defined(CPU_CAPABILITY_AVX512) + const int KB = div_up(K, 2); + const int NB = N / 32; + const int N_remainder = N - NB * 32; + + for (int kb = 0; kb < KB; ++kb) { + int kb_size = std::min(K - kb * 2, 2); + for (int nb = 0; nb < NB; ++nb) { + // handle 2x512bits each block + pack_vnni_Kx32( + /* dst */ dst + ((kb * 2) >> 1) * ld_dst * 2 + nb * 32 * 2, + /* src */ src + (is_indexed ? 0 : kb * 2 * ld_src) + nb * 32, + /* ind */ is_indexed ? ind + kb * 2 : nullptr, + /* K */ kb_size, + /* ld_src */ ld_src, + /* ld_dst */ ld_dst); + } + if (N_remainder > 0) { + pack_vnni_K_remainder( + /* dst */ dst + ((kb * 2) >> 1) * ld_dst * 2 + NB * 32 * 2, + /* src */ src + (is_indexed ? 0 : kb * 2 * ld_src) + NB * 32, + /* ind */ is_indexed ? ind + kb * 2 : nullptr, + /* K */ kb_size, + /* N */ N_remainder, + /* ld_src */ ld_src, + /* ld_dst */ ld_dst); + } + } +#else + int k = 0; + for (; k < (K >> 1) * 2; k += 2) { + index_t index0 = get_index(ind, k + 0); + index_t index1 = get_index(ind, k + 1); + for (int n = 0; n < N; ++n) { + dst[(k >> 1) * ld_dst * 2 + n * 2 + 0] = src[index0 * ld_src + n]; + dst[(k >> 1) * ld_dst * 2 + n * 2 + 1] = src[index1 * ld_src + n]; + } + } + if (K % 2 != 0) { + index_t index = get_index(ind, K - 1); + for (int n = 0; n < N; ++n) { + dst[(K >> 1) * ld_dst * 2 + n * 2 + 0] = src[index * ld_src + n]; + dst[(K >> 1) * ld_dst * 2 + n * 2 + 1] = 0; + } + k += 2; + } +#endif +} + +template +void pack_vnni2(scalar_t* __restrict__ dst, const scalar_t* __restrict__ src, int K, int N, int ld_src, int ld_dst) { + pack_vnni2(dst, src, nullptr, K, N, ld_src, ld_dst); +} + +template +void pack_vnni2( + scalar_t* __restrict__ dst, + const scalar_t* __restrict__ src, + const index_t* __restrict__ ind, + int K, + int N, + int ld_src, + int ld_dst) { + assert(ind != nullptr); + pack_vnni2(dst, src, ind, K, N, ld_src, ld_dst); +} + +} // anonymous namespace diff --git a/csrc/cpu/torch_bindings.cpp b/csrc/cpu/torch_bindings.cpp index c1f04a0527..9d3c354558 100644 --- a/csrc/cpu/torch_bindings.cpp +++ b/csrc/cpu/torch_bindings.cpp @@ -56,6 +56,8 @@ void shm_send_tensor_list(int64_t handle, std::vector shm_recv_tensor_list(int64_t handle, int64_t src); +// SGL CPU kernels + at::Tensor weight_packed_linear(at::Tensor& mat1, at::Tensor& mat2, const std::optional& bias, bool is_vnni); @@ -65,12 +67,11 @@ at::Tensor convert_weight_packed(at::Tensor& weight); at::Tensor fused_experts_cpu( at::Tensor& hidden_states, at::Tensor& w1, at::Tensor& w2, at::Tensor& topk_weights, at::Tensor& topk_ids, bool inplace, - bool use_int8_w8a8, bool use_fp8_w8a16, - const std::optional& w1_scale, + int64_t moe_comp_method, const std::optional& w1_scale, const std::optional& w2_scale, - const std::optional> block_size, - const std::optional& a1_scale, - const std::optional& a2_scale, bool is_vnni); + const std::optional& w1_zero, + const std::optional& w2_zero, + const std::optional> block_size, bool is_vnni); at::Tensor int8_scaled_mm_with_quant(at::Tensor& mat1, at::Tensor& mat2, at::Tensor& scales2, @@ -86,7 +87,12 @@ at::Tensor fp8_scaled_mm_cpu(at::Tensor& mat1, at::Tensor& mat2, // Adapted from sglang: INT4 W4A8 kernels std::tuple convert_weight_packed_scale_zp( - at::Tensor qweight, at::Tensor qzeros, at::Tensor scales); + at::Tensor qweight, // awq: (*, K, N / 8) || gptq: (*, K / 8, N) , int32 + at::Tensor qzeros, // awq: (*, K / group_size, N / 8) || gptq: (*, K / + // group_size, N / 8) , int32 + at::Tensor scales, // awq: (*, K / group_size, N) || gptq: (*, K / + // group_size, N) , bfloat16 + int64_t quant_method_4bit); at::Tensor int4_scaled_mm_cpu(at::Tensor& x, at::Tensor& w, at::Tensor& w_zeros, at::Tensor& w_scales, @@ -360,10 +366,10 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) { ops.def("convert_weight_packed(Tensor! weight) -> Tensor"); ops.impl("convert_weight_packed", torch::kCPU, &convert_weight_packed); ops.def( - "fused_experts_cpu(Tensor! hidden_states, Tensor w1, Tensor w2, Tensor " - "topk_weights, Tensor topk_ids, bool inplace, bool use_int8_w8a8, bool " - "use_fp8_w8a16, Tensor? w1_scale, Tensor? w2_scale, SymInt[]? " - "block_size, Tensor? a1_scale, Tensor? a2_scale, bool is_vnni) -> " + "fused_experts_cpu(Tensor hidden_states, Tensor w1, Tensor w2, Tensor " + "topk_weights, Tensor topk_ids, bool " + "inplace, int moe_comp_method, Tensor? w1_scale, Tensor? w2_scale, " + "Tensor? w1_zero, Tensor? w2_zero, int[]? block_size, bool is_vnni) -> " "Tensor"); ops.impl("fused_experts_cpu", torch::kCPU, &fused_experts_cpu); ops.def( @@ -374,8 +380,9 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) { // Adapted from sglang: INT4 W4A8 kernels ops.def( - "convert_weight_packed_scale_zp(Tensor qweight, Tensor qzeros, " - "Tensor scales) -> (Tensor, Tensor, Tensor)"); + "convert_weight_packed_scale_zp(Tensor weight, Tensor qzeros, Tensor " + "scales, int quant_method_4bit) -> (Tensor, " + "Tensor, Tensor)"); ops.impl("convert_weight_packed_scale_zp", torch::kCPU, &convert_weight_packed_scale_zp); diff --git a/pyproject.toml b/pyproject.toml index b8d1446325..6a0ae2fea6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -125,7 +125,7 @@ extend-exclude = ["tests/models/fixtures/*", "tests/prompts/*", "tests/tokenizer "examples/pooling/token_embed/*", "tests/models/language/pooling/*", "vllm/third_party/*", "vllm/entrypoints/serve/instrumentator/static/*", "tests/entrypoints/openai/speech_to_text/test_transcription_validation.py", "docs/governance/process.md", "docs/assets/contributing/vllm_bench_serve_timeline.html", - "tests/v1/engine/test_fast_incdec_prefix_err.py", ".git/*"] + "tests/v1/engine/test_fast_incdec_prefix_err.py", ".git/*", "csrc/cpu/sgl-kernels/*"] ignore-hidden = false [tool.typos.default] diff --git a/tests/kernels/moe/test_cpu_fp8_fused_moe.py b/tests/kernels/moe/test_cpu_fp8_fused_moe.py index 695a0d8d46..dbc6cd198c 100644 --- a/tests/kernels/moe/test_cpu_fp8_fused_moe.py +++ b/tests/kernels/moe/test_cpu_fp8_fused_moe.py @@ -218,13 +218,12 @@ def test_w8a16_block_fp8_cpu_fused_moe(M, N, K, E, topk, seed): topk_weight, topk_ids, False, - False, - True, + ops.CPUQuantMethod.FP8_W8A16, w1_s, w2_s, + None, + None, BLOCK_SIZE, - None, - None, True, ) torch.testing.assert_close( @@ -242,13 +241,12 @@ def test_w8a16_block_fp8_cpu_fused_moe(M, N, K, E, topk, seed): topk_weight, topk_ids, True, - False, - True, + ops.CPUQuantMethod.FP8_W8A16, w1_s, w2_s, + None, + None, BLOCK_SIZE, - None, - None, True, ) torch.testing.assert_close(out_inplace, out, atol=0, rtol=0) diff --git a/tests/kernels/test_awq_int4_to_int8.py b/tests/kernels/test_awq_int4_to_int8.py index 829c0d2277..f94214785d 100644 --- a/tests/kernels/test_awq_int4_to_int8.py +++ b/tests/kernels/test_awq_int4_to_int8.py @@ -20,7 +20,11 @@ import numpy as np import pytest import torch -from vllm._custom_ops import _supports_cpu_w4a8_int8 +from vllm._custom_ops import ( + CPUQuantAlgo, + convert_weight_packed_scale_zp, + int4_scaled_mm_cpu, +) from vllm.model_executor.layers.quantization.utils.quant_utils import ( pack_cols, ) @@ -29,11 +33,6 @@ from vllm.platforms import current_platform if not current_platform.is_cpu(): pytest.skip("skipping CPU-only tests", allow_module_level=True) -requires_cpu_w4a8_int8 = pytest.mark.skipif( - not _supports_cpu_w4a8_int8, - reason="Requires vLLM CPU build with SGLang INT4 W4A8 kernels", -) - def make_awq_checkpoint_data(K, N, group_size, seed=42): """Create synthetic AWQ checkpoint data in packed int32 format. @@ -87,7 +86,6 @@ def make_awq_checkpoint_data(K, N, group_size, seed=42): class TestConvertWeightPackedScaleZp: """Tests for convert_weight_packed_scale_zp weightpacking.""" - @requires_cpu_w4a8_int8 @pytest.mark.parametrize( "K,N,group_size", [ @@ -102,8 +100,11 @@ class TestConvertWeightPackedScaleZp: K, N, group_size ) - blocked_w, blocked_zp, blocked_s = torch.ops._C.convert_weight_packed_scale_zp( - packed_qweight, packed_qzeros, scales + blocked_w, blocked_zp, blocked_s = convert_weight_packed_scale_zp( + packed_qweight, + packed_qzeros, + scales, + CPUQuantAlgo.AWQ, ) block_n = 32 @@ -129,7 +130,6 @@ class TestConvertWeightPackedScaleZp: class TestInt4ScaledMmCpu: """Tests for int4_scaled_mm_cpu GEMM kernel.""" - @requires_cpu_w4a8_int8 @pytest.mark.parametrize( "M,K,N,group_size", [ @@ -146,12 +146,15 @@ class TestInt4ScaledMmCpu: make_awq_checkpoint_data(K, N, group_size) ) - blocked_w, blocked_zp, blocked_s = torch.ops._C.convert_weight_packed_scale_zp( - packed_qweight, packed_qzeros, scales + blocked_w, blocked_zp, blocked_s = convert_weight_packed_scale_zp( + packed_qweight, + packed_qzeros, + scales, + CPUQuantAlgo.AWQ, ) x = torch.randn(M, K, dtype=torch.bfloat16) - out = torch.ops._C.int4_scaled_mm_cpu(x, blocked_w, blocked_zp, blocked_s, None) + out = int4_scaled_mm_cpu(x, blocked_w, blocked_zp, blocked_s, None) ref_out = torch.mm(x.float(), float_ref) @@ -169,7 +172,6 @@ class TestInt4ScaledMmCpu: ) print(f" [PASS] INT4 GEMM correct: M={M}, K={K}, N={N}") - @requires_cpu_w4a8_int8 @pytest.mark.parametrize("M", [1, 8, 32]) def test_gemm_with_bias(self, M): """INT4 W4A8 GEMM with bias should match reference.""" @@ -178,14 +180,17 @@ class TestInt4ScaledMmCpu: make_awq_checkpoint_data(K, N, group_size) ) - blocked_w, blocked_zp, blocked_s = torch.ops._C.convert_weight_packed_scale_zp( - packed_qweight, packed_qzeros, scales + blocked_w, blocked_zp, blocked_s = convert_weight_packed_scale_zp( + packed_qweight, + packed_qzeros, + scales, + CPUQuantAlgo.AWQ, ) bias = torch.randn(N, dtype=torch.float32) x = torch.randn(M, K, dtype=torch.bfloat16) - out = torch.ops._C.int4_scaled_mm_cpu(x, blocked_w, blocked_zp, blocked_s, bias) + out = int4_scaled_mm_cpu(x, blocked_w, blocked_zp, blocked_s, bias) ref_out = torch.mm(x.float(), float_ref) + bias abs_diff = (out.float() - ref_out).abs() @@ -197,7 +202,6 @@ class TestInt4ScaledMmCpu: ) print(f" [PASS] INT4 GEMM with bias: M={M}") - @requires_cpu_w4a8_int8 def test_gemm_3d_input(self): """apply() reshapes 3D input [B, S, K] -> [B*S, K] -> back to 3D.""" K, N, group_size = 256, 128, 128 @@ -205,17 +209,18 @@ class TestInt4ScaledMmCpu: make_awq_checkpoint_data(K, N, group_size) ) - blocked_w, blocked_zp, blocked_s = torch.ops._C.convert_weight_packed_scale_zp( - packed_qweight, packed_qzeros, scales + blocked_w, blocked_zp, blocked_s = convert_weight_packed_scale_zp( + packed_qweight, + packed_qzeros, + scales, + CPUQuantAlgo.AWQ, ) B, S = 2, 8 x_3d = torch.randn(B, S, K, dtype=torch.bfloat16) x_2d = x_3d.reshape(-1, K) - out_2d = torch.ops._C.int4_scaled_mm_cpu( - x_2d, blocked_w, blocked_zp, blocked_s, None - ) + out_2d = int4_scaled_mm_cpu(x_2d, blocked_w, blocked_zp, blocked_s, None) out_3d = out_2d.reshape(B, S, N) ref_out = torch.mm(x_2d.float(), float_ref).reshape(B, S, N) @@ -229,7 +234,6 @@ class TestInt4ScaledMmCpu: assert mean_rel < 0.05, f"Mean relative error {mean_rel:.4f} for 3D exceeds 5%" print(f" [PASS] 3D input [{B},{S},{K}] -> output [{B},{S},{N}]") - @requires_cpu_w4a8_int8 def test_gemm_fp16_input(self): """INT4 GEMM should also work with fp16 input.""" K, N, group_size, M = 256, 256, 128, 8 @@ -237,12 +241,15 @@ class TestInt4ScaledMmCpu: make_awq_checkpoint_data(K, N, group_size) ) - blocked_w, blocked_zp, blocked_s = torch.ops._C.convert_weight_packed_scale_zp( - packed_qweight, packed_qzeros, scales + blocked_w, blocked_zp, blocked_s = convert_weight_packed_scale_zp( + packed_qweight, + packed_qzeros, + scales, + CPUQuantAlgo.AWQ, ) x = torch.randn(M, K, dtype=torch.float16) - out = torch.ops._C.int4_scaled_mm_cpu(x, blocked_w, blocked_zp, blocked_s, None) + out = int4_scaled_mm_cpu(x, blocked_w, blocked_zp, blocked_s, None) ref_out = torch.mm(x.float(), float_ref) abs_diff = (out.float() - ref_out).abs() diff --git a/vllm/_custom_ops.py b/vllm/_custom_ops.py index 9f3f033b48..564276adaa 100644 --- a/vllm/_custom_ops.py +++ b/vllm/_custom_ops.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from enum import IntEnum from typing import TYPE_CHECKING, Literal import torch @@ -3150,6 +3151,13 @@ if hasattr(torch.ops._C, "weight_packed_linear"): ) +class CPUQuantMethod(IntEnum): + UNQUANT = 0 + INT8_W8A8 = 1 + FP8_W8A16 = 2 + INT4_W4A8 = 3 + + if hasattr(torch.ops._C, "fused_experts_cpu"): @register_fake("_C::fused_experts_cpu") @@ -3160,13 +3168,12 @@ if hasattr(torch.ops._C, "fused_experts_cpu"): topk_weights: torch.Tensor, topk_ids: torch.Tensor, inplace: bool, - use_int8_w8a8: bool, - use_fp8_w8a16: bool, + moe_comp_method: CPUQuantMethod, w1_scale: torch.Tensor | None, w2_scale: torch.Tensor | None, + w1_zero: torch.Tensor | None, + w2_zero: torch.Tensor | None, block_size: list[int] | None, - a1_scale: torch.Tensor | None, - a2_scale: torch.Tensor | None, is_vnni: bool, ) -> torch.Tensor: return torch.empty_like(hidden_states) @@ -3179,13 +3186,12 @@ def fused_experts_cpu( topk_weights: torch.Tensor, topk_ids: torch.Tensor, inplace: bool, - use_int8_w8a8: bool, - use_fp8_w8a16: bool, + moe_comp_method: CPUQuantMethod, w1_scale: torch.Tensor | None, w2_scale: torch.Tensor | None, + w1_zero: torch.Tensor | None, + w2_zero: torch.Tensor | None, block_size: list[int] | None, - a1_scale: torch.Tensor | None, - a2_scale: torch.Tensor | None, is_vnni: bool, ) -> torch.Tensor: return torch.ops._C.fused_experts_cpu( @@ -3195,13 +3201,12 @@ def fused_experts_cpu( topk_weights, topk_ids, inplace, - use_int8_w8a8, - use_fp8_w8a16, + moe_comp_method, w1_scale, w2_scale, + w1_zero, + w2_zero, block_size, - a1_scale, - a2_scale, is_vnni, ) @@ -3222,6 +3227,11 @@ if hasattr(torch.ops._C, "int8_scaled_mm_with_quant"): return torch.empty((M, N), dtype=out_dtype) +class CPUQuantAlgo(IntEnum): + AWQ = 0 + GPTQ = 1 + + if hasattr(torch.ops._C, "convert_weight_packed_scale_zp"): @register_fake("_C::convert_weight_packed_scale_zp") @@ -3229,6 +3239,7 @@ if hasattr(torch.ops._C, "convert_weight_packed_scale_zp"): qweight: torch.Tensor, qzeros: torch.Tensor, scales: torch.Tensor, + quant_method_4bit: CPUQuantAlgo, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: return ( torch.empty_like(qweight), @@ -3237,6 +3248,20 @@ if hasattr(torch.ops._C, "convert_weight_packed_scale_zp"): ) +def convert_weight_packed_scale_zp( + qweight: torch.Tensor, + qzeros: torch.Tensor, + scales: torch.Tensor, + quant_method_4bit: CPUQuantAlgo, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + return torch.ops._C.convert_weight_packed_scale_zp( + qweight, + qzeros, + scales, + quant_method_4bit, + ) + + if hasattr(torch.ops._C, "int4_scaled_mm_cpu"): @register_fake("_C::int4_scaled_mm_cpu") @@ -3251,7 +3276,26 @@ if hasattr(torch.ops._C, "int4_scaled_mm_cpu"): return torch.empty((x.size(0), N), dtype=x.dtype, device=x.device) -_supports_cpu_w4a8_int8 = bool(hasattr(torch.ops._C, "convert_weight_packed_scale_zp")) +def int4_scaled_mm_cpu( + x: torch.Tensor, + w: torch.Tensor, + w_zeros: torch.Tensor, + w_scales: torch.Tensor, + bias: torch.Tensor | None, +) -> torch.Tensor: + x_shape = x.shape + x_2d = x.reshape(-1, x_shape[-1]) if len(x_shape) > 2 else x + + out = torch.ops._C.int4_scaled_mm_cpu( + x_2d, + w, + w_zeros, + w_scales, + bias, + ) + out = out.reshape(x_shape[:-1] + (out.size(-1),)) if len(x_shape) > 2 else out + return out + if hasattr(torch.ops._C, "fp8_scaled_mm_cpu"): diff --git a/vllm/model_executor/layers/fused_moe/cpu_fused_moe.py b/vllm/model_executor/layers/fused_moe/cpu_fused_moe.py index 985f33e100..c26090a56b 100644 --- a/vllm/model_executor/layers/fused_moe/cpu_fused_moe.py +++ b/vllm/model_executor/layers/fused_moe/cpu_fused_moe.py @@ -7,7 +7,12 @@ import torch from torch.nn import functional as F from vllm import _custom_ops as ops -from vllm._custom_ops import cpu_fused_moe, cpu_prepack_moe_weight +from vllm._custom_ops import ( + CPUQuantMethod, + cpu_fused_moe, + cpu_prepack_moe_weight, + fused_experts_cpu, +) from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.fused_moe.activation import MoEActivation from vllm.model_executor.layers.quantization.utils.layer_utils import replace_parameter @@ -195,23 +200,21 @@ class SGLFusedMOE: e_score_correction_bias=e_score_correction_bias, ) - torch.ops._C.fused_experts_cpu( + return fused_experts_cpu( x, layer.w13_weight, layer.w2_weight, topk_weights, topk_ids, - True, - False, - False, - None, - None, - None, - None, - None, - True, + False, # inplace + CPUQuantMethod.UNQUANT, # moe_comp_method + None, # w1_scale + None, # w2_scale + None, # w1_zero + None, # w2_zero + None, # block_size + True, # is_vnni ) - return x class CPUFusedMOE: diff --git a/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py b/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py index 273b5356f6..3b3d5cdfae 100644 --- a/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py @@ -5,6 +5,7 @@ import torch import vllm.model_executor.layers.fused_moe.modular_kernel as mk +from vllm._custom_ops import CPUQuantMethod, fused_experts_cpu from vllm.model_executor.layers.fused_moe.activation import MoEActivation from vllm.model_executor.layers.fused_moe.config import ( FusedMoEConfig, @@ -157,19 +158,18 @@ class CPUExpertsFp8(mk.FusedMoEExpertsMonolithic): ) ) - return torch.ops._C.fused_experts_cpu( + return fused_experts_cpu( hidden_states, w1, w2, topk_weights, topk_ids, False, # inplace - False, # use_int8_w8a8 - True, # use_fp8_w8a16 + CPUQuantMethod.FP8_W8A16, # moe_comp_method self.w1_scale, # w1_scale self.w2_scale, # w2_scale + None, # w1_zero + None, # w2_zero block_shape, # block_size - None, # a1_scale (W8A16: no activation quantization) - None, # a2_scale True, # is_vnni ) diff --git a/vllm/model_executor/layers/quantization/cpu_wna16.py b/vllm/model_executor/layers/quantization/cpu_wna16.py index aea1067ff2..6853013af8 100644 --- a/vllm/model_executor/layers/quantization/cpu_wna16.py +++ b/vllm/model_executor/layers/quantization/cpu_wna16.py @@ -287,8 +287,11 @@ class CPUAWQLinearMethod(LinearMethodBase): packed_weight = layer.qweight.data packed_zeros = layer.qzeros.data scales = layer.scales.data - blocked_w, blocked_zp, blocked_s = torch.ops._C.convert_weight_packed_scale_zp( - packed_weight, packed_zeros, scales + blocked_w, blocked_zp, blocked_s = ops.convert_weight_packed_scale_zp( + packed_weight, + packed_zeros, + scales, + ops.CPUQuantAlgo.AWQ, ) layer.packed_weight = blocked_w @@ -334,18 +337,13 @@ class CPUAWQLinearMethod(LinearMethodBase): bias: torch.Tensor | None = None, ) -> torch.Tensor: """SGLang INT4 W4A8 GEMM path.""" - x_shape = x.shape - x_2d = x.reshape(-1, x_shape[-1]) if len(x_shape) > 2 else x - - out = torch.ops._C.int4_scaled_mm_cpu( - x_2d, + return ops.int4_scaled_mm_cpu( + x, layer.packed_weight, layer.packed_qzeros, layer.packed_scales, bias, ) - out = out.reshape(x_shape[:-1] + (out.size(-1),)) if len(x_shape) > 2 else out - return out def _get_isa_hint(dtype: torch.dtype) -> str: