[CPU] Bump up to the latest CPU kernels (#41924)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
@@ -65,6 +65,7 @@ steps:
|
||||
- vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py
|
||||
- vllm/model_executor/layers/quantization/kernels/scaled_mm/cpu.py
|
||||
- vllm/model_executor/layers/quantization/kernels/mixed_precision/cpu.py
|
||||
- vllm/model_executor/layers/fused_moe/experts/cpu_moe.py
|
||||
- tests/quantization/test_compressed_tensors.py
|
||||
- tests/quantization/test_cpu_wna16.py
|
||||
commands:
|
||||
|
||||
@@ -409,12 +409,15 @@ endif()
|
||||
|
||||
if (ENABLE_X86_ISA)
|
||||
set(VLLM_EXT_SRC_SGL
|
||||
"csrc/cpu/sgl-kernels/fla.cpp"
|
||||
"csrc/cpu/sgl-kernels/conv.cpp"
|
||||
"csrc/cpu/sgl-kernels/gemm.cpp"
|
||||
"csrc/cpu/sgl-kernels/gemm_int8.cpp"
|
||||
"csrc/cpu/sgl-kernels/gemm_fp8.cpp"
|
||||
"csrc/cpu/sgl-kernels/gemm_int4.cpp"
|
||||
"csrc/cpu/sgl-kernels/moe.cpp"
|
||||
"csrc/cpu/sgl-kernels/moe_int8.cpp"
|
||||
"csrc/cpu/sgl-kernels/moe_int4.cpp"
|
||||
"csrc/cpu/sgl-kernels/moe_fp8.cpp")
|
||||
|
||||
set(VLLM_EXT_SRC_AVX512
|
||||
|
||||
+247
-68
@@ -1,13 +1,12 @@
|
||||
// Adapted from
|
||||
// https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu
|
||||
|
||||
// clang-format off
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <ATen/ATen.h>
|
||||
#include <ATen/Parallel.h>
|
||||
#include <ATen/record_function.h>
|
||||
|
||||
// clang-format off
|
||||
|
||||
#if defined(_OPENMP)
|
||||
#include <omp.h>
|
||||
@@ -16,40 +15,157 @@
|
||||
namespace {
|
||||
|
||||
// dispatch bool
|
||||
#define AT_DISPATCH_BOOL(BOOL_V, BOOL_NAME, ...) \
|
||||
[&] { \
|
||||
if (BOOL_V) { \
|
||||
constexpr bool BOOL_NAME = true; \
|
||||
return __VA_ARGS__(); \
|
||||
} else { \
|
||||
constexpr bool BOOL_NAME = false; \
|
||||
return __VA_ARGS__(); \
|
||||
} \
|
||||
#define AT_DISPATCH_BOOL(BOOL_V, BOOL_NAME, ...) \
|
||||
[&] { \
|
||||
if (BOOL_V) { \
|
||||
constexpr bool BOOL_NAME = true; \
|
||||
return __VA_ARGS__(); \
|
||||
} else { \
|
||||
constexpr bool BOOL_NAME = false; \
|
||||
return __VA_ARGS__(); \
|
||||
} \
|
||||
}()
|
||||
|
||||
// dispatch: bfloat16, float16, int8_t, fp8_e4m3
|
||||
#define CPU_DISPATCH_PACKED_TYPES(TYPE, ...) \
|
||||
[&] { \
|
||||
switch (TYPE) { \
|
||||
case at::ScalarType::BFloat16 : { \
|
||||
using packed_t = at::BFloat16; \
|
||||
return __VA_ARGS__(); \
|
||||
} \
|
||||
case at::ScalarType::Half: { \
|
||||
using packed_t = at::Half; \
|
||||
return __VA_ARGS__(); \
|
||||
} \
|
||||
case at::ScalarType::Char : { \
|
||||
using packed_t = int8_t; \
|
||||
return __VA_ARGS__(); \
|
||||
} \
|
||||
case at::ScalarType::Float8_e4m3fn : { \
|
||||
using packed_t = at::Float8_e4m3fn; \
|
||||
return __VA_ARGS__(); \
|
||||
} \
|
||||
default: \
|
||||
TORCH_CHECK(false, "Unsupported floating data type.\n"); \
|
||||
} \
|
||||
#define AT_DISPATCH_BOOL2(BOOL_V1, BOOL_NAME1, BOOL_V2, BOOL_NAME2, ...) \
|
||||
[&] { \
|
||||
if (BOOL_V1) { \
|
||||
constexpr bool BOOL_NAME1 = true; \
|
||||
if (BOOL_V2) { \
|
||||
constexpr bool BOOL_NAME2 = true; \
|
||||
return __VA_ARGS__(); \
|
||||
} else { \
|
||||
constexpr bool BOOL_NAME2 = false; \
|
||||
return __VA_ARGS__(); \
|
||||
} \
|
||||
} else { \
|
||||
constexpr bool BOOL_NAME1 = false; \
|
||||
if (BOOL_V2) { \
|
||||
constexpr bool BOOL_NAME2 = true; \
|
||||
return __VA_ARGS__(); \
|
||||
} else { \
|
||||
constexpr bool BOOL_NAME2 = false; \
|
||||
return __VA_ARGS__(); \
|
||||
} \
|
||||
} \
|
||||
}()
|
||||
|
||||
// dispatch: bfloat16, float16, int8_t, fp8_e4m3, uint8_t(mxfp4/int4)
|
||||
#define CPU_DISPATCH_PACKED_TYPES(TYPE, ...) \
|
||||
[&] { \
|
||||
switch (TYPE) { \
|
||||
case at::ScalarType::BFloat16: { \
|
||||
using packed_t = at::BFloat16; \
|
||||
return __VA_ARGS__(); \
|
||||
} \
|
||||
case at::ScalarType::Half: { \
|
||||
using packed_t = at::Half; \
|
||||
return __VA_ARGS__(); \
|
||||
} \
|
||||
case at::ScalarType::Char: { \
|
||||
using packed_t = int8_t; \
|
||||
return __VA_ARGS__(); \
|
||||
} \
|
||||
case at::ScalarType::Float8_e4m3fn: { \
|
||||
using packed_t = at::Float8_e4m3fn; \
|
||||
return __VA_ARGS__(); \
|
||||
} \
|
||||
case at::ScalarType::Byte: { \
|
||||
using packed_t = uint8_t; \
|
||||
return __VA_ARGS__(); \
|
||||
} \
|
||||
default: \
|
||||
TORCH_CHECK(false, "Unsupported floating data type.\n"); \
|
||||
} \
|
||||
}()
|
||||
|
||||
// Helper MICRO for CPU_DISPATCH_FLOATING_TYPES_EXT:
|
||||
// TYPE1: the primary dtype (input, output, weight);
|
||||
// TYPE2: defined as PARAM_T input
|
||||
#define CPU_DISPATCH_TYPE1_WITH_PARAM(TYPE1, PARAM_T, ...) \
|
||||
switch (TYPE1) { \
|
||||
case at::ScalarType::BFloat16: { \
|
||||
using scalar_t = at::BFloat16; \
|
||||
using param_t = PARAM_T; \
|
||||
return __VA_ARGS__(); \
|
||||
} \
|
||||
case at::ScalarType::Half: { \
|
||||
using scalar_t = at::Half; \
|
||||
using param_t = PARAM_T; \
|
||||
return __VA_ARGS__(); \
|
||||
} \
|
||||
case at::ScalarType::Float: { \
|
||||
using scalar_t = float; \
|
||||
using param_t = PARAM_T; \
|
||||
return __VA_ARGS__(); \
|
||||
} \
|
||||
default: \
|
||||
TORCH_CHECK(false, "Unsupported floating data type."); \
|
||||
}
|
||||
|
||||
// Helper MICRO for CPU_DISPATCH_REDUCED_FLOATING_TYPES_EXT:
|
||||
// TYPE1: the primary dtype (input, output, weight);
|
||||
// TYPE2: defined as PARAM_T input
|
||||
#define CPU_DISPATCH_TYPE1_WITH_PARAM_REDUCED(TYPE1, PARAM_T, ...) \
|
||||
switch (TYPE1) { \
|
||||
case at::ScalarType::BFloat16: { \
|
||||
using scalar_t = at::BFloat16; \
|
||||
using param_t = PARAM_T; \
|
||||
return __VA_ARGS__(); \
|
||||
} \
|
||||
case at::ScalarType::Half: { \
|
||||
using scalar_t = at::Half; \
|
||||
using param_t = PARAM_T; \
|
||||
return __VA_ARGS__(); \
|
||||
} \
|
||||
default: \
|
||||
TORCH_CHECK(false, "Unsupported floating data type."); \
|
||||
}
|
||||
|
||||
// Helper MICRO for CPU_DISPATCH_REDUCED_FLOATING_TYPES_EXT:
|
||||
// TYPE1: the dtype both for scalar_t and param_t
|
||||
#define CPU_DISPATCH_TYPE1_WITH_SAME_PARAM_REDUCED(TYPE1, ...) \
|
||||
switch (TYPE1) { \
|
||||
case at::ScalarType::BFloat16: { \
|
||||
using scalar_t = at::BFloat16; \
|
||||
using param_t = at::BFloat16; \
|
||||
return __VA_ARGS__(); \
|
||||
} \
|
||||
case at::ScalarType::Half: { \
|
||||
using scalar_t = at::Half; \
|
||||
using param_t = at::Half; \
|
||||
return __VA_ARGS__(); \
|
||||
} \
|
||||
default: \
|
||||
TORCH_CHECK(false, "Unsupported reduced floating data type."); \
|
||||
}
|
||||
|
||||
// dispatch with mixed dtypes (TYPE1, TYPE2):
|
||||
// TYPE1: the primary dtype (input, output, weight);
|
||||
// TYPE2: the secondary dtype (bias, etc.).
|
||||
#define CPU_DISPATCH_FLOATING_TYPES_EXT(TYPE1, TYPE2, ...) \
|
||||
[&] { \
|
||||
if (TYPE2 == at::kFloat) { \
|
||||
CPU_DISPATCH_TYPE1_WITH_PARAM(TYPE1, float, __VA_ARGS__) \
|
||||
} else if (TYPE2 == at::ScalarType::BFloat16) { \
|
||||
CPU_DISPATCH_TYPE1_WITH_PARAM(TYPE1, at::BFloat16, __VA_ARGS__) \
|
||||
} else if (TYPE2 == at::ScalarType::Half) { \
|
||||
CPU_DISPATCH_TYPE1_WITH_PARAM(TYPE1, at::Half, __VA_ARGS__) \
|
||||
} else { \
|
||||
TORCH_CHECK(false, "Unsupported floating data type."); \
|
||||
} \
|
||||
}()
|
||||
|
||||
// dispatch with mixed dtypes (reduced one, no float for TYPE1) (TYPE1, TYPE2):
|
||||
// TYPE1: the primary dtype (input, output, weight);
|
||||
// TYPE2: the secondary dtype (bias, etc.).
|
||||
#define CPU_DISPATCH_REDUCED_FLOATING_TYPES_EXT(TYPE1, TYPE2, ...) \
|
||||
[&] { \
|
||||
if (TYPE2 == at::kFloat) { \
|
||||
CPU_DISPATCH_TYPE1_WITH_PARAM_REDUCED(TYPE1, float, __VA_ARGS__) \
|
||||
} else { \
|
||||
TORCH_CHECK(TYPE1 == TYPE2); \
|
||||
CPU_DISPATCH_TYPE1_WITH_SAME_PARAM_REDUCED(TYPE1, __VA_ARGS__) \
|
||||
} \
|
||||
}()
|
||||
|
||||
#define UNUSED(x) (void)(x)
|
||||
@@ -70,13 +186,51 @@ namespace {
|
||||
#define CHECK_DIM(d, x) TORCH_CHECK(x.dim() == d, #x " must be a " #d "D tensor")
|
||||
|
||||
#define CHECK_EQ(a, b) TORCH_CHECK((a) == (b), "CHECK_EQ(" #a ", " #b ") failed. ", a, " vs ", b)
|
||||
#define CHECK_GT(a, b) TORCH_CHECK((a) > (b), "CHECK_GT(" #a ", " #b ") failed. ", a, " vs ", b)
|
||||
#define CHECK_GE(a, b) TORCH_CHECK((a) >= (b), "CHECK_GE(" #a ", " #b ") failed. ", a, " vs ", b)
|
||||
|
||||
// parallel routines
|
||||
template <bool is_only_lastdim_contiguous>
|
||||
static inline void CHECK_INPUT_SHAPE_DTYPE(const at::Tensor& tensor, const at::IntArrayRef sizes, at::ScalarType st) {
|
||||
TORCH_CHECK(tensor.sizes() == sizes, "Input tensor shape mismatch: expected ", sizes, ", got ", tensor.sizes());
|
||||
TORCH_CHECK(tensor.scalar_type() == st, "Input tensor dtype mismatch");
|
||||
if constexpr (is_only_lastdim_contiguous) {
|
||||
CHECK_LAST_DIM_CONTIGUOUS_INPUT(tensor);
|
||||
} else {
|
||||
CHECK_INPUT(tensor);
|
||||
}
|
||||
}
|
||||
|
||||
// [NB] Parallel Routines
|
||||
//
|
||||
// * at::parallel_for - applies for most of generic use cases, this will be compiled
|
||||
// against openmp in default torch release.
|
||||
//
|
||||
// * parallel_for - same function as above, can choose payload partition scheme in
|
||||
// balance211.
|
||||
//
|
||||
// * parallel_2d - parallel for 2 dimensions, used in GEMM, etc.
|
||||
// this one will do payload balance across 2 dimensions.
|
||||
//
|
||||
|
||||
// grain size for each thread
|
||||
constexpr int GRAIN_SIZE = 1024;
|
||||
|
||||
template <typename T, typename std::enable_if<std::is_integral<T>::value, int>::type = 0>
|
||||
inline T div_up(T x, T y) { return (x + y - 1) / y; }
|
||||
inline T div_up(T x, T y) {
|
||||
return (x + y - 1) / y;
|
||||
}
|
||||
|
||||
// you can only use at::get_thread_num() with at::parallel_for()
|
||||
// as it is lazy initialized, otherwise it will always return 0.
|
||||
inline int get_thread_num() {
|
||||
#if defined(_OPENMP)
|
||||
return omp_get_thread_num();
|
||||
#else
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
// balance payload across each thread
|
||||
template <typename T>
|
||||
inline void balance211(T n, T nth, T ith, T& n_start, T& n_end) {
|
||||
#if 0
|
||||
@@ -94,10 +248,10 @@ inline void balance211(T n, T nth, T ith, T& n_start, T& n_end) {
|
||||
}
|
||||
n_end += n_start;
|
||||
#else
|
||||
// pytorch aten partition pattern
|
||||
T n_my = div_up(n, nth);
|
||||
n_start = ith * n_my;
|
||||
n_end = std::min(n_start + n_my, n);
|
||||
// pytorch aten partition pattern
|
||||
T n_my = div_up(n, nth);
|
||||
n_start = ith * n_my;
|
||||
n_end = std::min(n_start + n_my, n);
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -105,23 +259,15 @@ template <typename func_t>
|
||||
inline void parallel_for(int n, const func_t& f) {
|
||||
#if defined(_OPENMP)
|
||||
#pragma omp parallel
|
||||
{
|
||||
{
|
||||
int nth = omp_get_num_threads();
|
||||
int ith = omp_get_thread_num();
|
||||
int tbegin, tend;
|
||||
balance211(n, nth, ith, tbegin, tend);
|
||||
f(tbegin, tend);
|
||||
}
|
||||
}
|
||||
#else
|
||||
f(0, n);
|
||||
#endif
|
||||
}
|
||||
|
||||
inline int get_thread_num() {
|
||||
#if defined(_OPENMP)
|
||||
return omp_get_thread_num();
|
||||
#else
|
||||
return 0;
|
||||
f(0, n);
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -137,7 +283,6 @@ int inline adjust_num_threads(int m) {
|
||||
|
||||
template <typename func_t>
|
||||
inline void parallel_2d(int m, int n, const func_t& f) {
|
||||
|
||||
// make sure we have even num_threads
|
||||
int nth = adjust_num_threads(m);
|
||||
|
||||
@@ -165,31 +310,59 @@ inline void parallel_2d(int m, int n, const func_t& f) {
|
||||
|
||||
#if defined(_OPENMP)
|
||||
#pragma omp parallel num_threads(nth)
|
||||
{
|
||||
int ith = omp_get_thread_num();
|
||||
int ith_m = ith / nth_n;
|
||||
int ith_n = ith % nth_n;
|
||||
{
|
||||
int ith = omp_get_thread_num();
|
||||
int ith_m = ith / nth_n;
|
||||
int ith_n = ith % nth_n;
|
||||
|
||||
int thread_block_m = div_up(m, nth_m);
|
||||
int thread_block_n = div_up(n, nth_n);
|
||||
int thread_block_m = div_up(m, nth_m);
|
||||
int thread_block_n = div_up(n, nth_n);
|
||||
|
||||
int begin_m = ith_m * thread_block_m;
|
||||
int end_m = std::min(m, begin_m + thread_block_m);
|
||||
int begin_n = ith_n * thread_block_n;
|
||||
int end_n = std::min(n, begin_n + thread_block_n);
|
||||
int begin_m = ith_m * thread_block_m;
|
||||
int end_m = std::min(m, begin_m + thread_block_m);
|
||||
int begin_n = ith_n * thread_block_n;
|
||||
int end_n = std::min(n, begin_n + thread_block_n);
|
||||
|
||||
f(begin_m, end_m, begin_n, end_n);
|
||||
}
|
||||
f(begin_m, end_m, begin_n, end_n);
|
||||
}
|
||||
#else
|
||||
f(0, m, 0, n);
|
||||
#endif
|
||||
}
|
||||
|
||||
// limit max cache blocks
|
||||
// when we need to do pre-unpack for weights, e.g. fp8
|
||||
#define MAX_CACHE_BLOCK_SIZE 4
|
||||
|
||||
template <typename T>
|
||||
int get_cache_blocks(int BLOCK_SIZE, int K) {
|
||||
inline int get_cache_blocks(int chunk_size) {
|
||||
// L2 2MB and ratio of 50%
|
||||
const int L2_size = 2048 * 1024 >> 1;
|
||||
return std::max(1, int(L2_size / (BLOCK_SIZE * K * sizeof(T))));
|
||||
return std::max(1, int(L2_size / (chunk_size * sizeof(T))));
|
||||
}
|
||||
|
||||
template <>
|
||||
inline int get_cache_blocks<at::Float8_e4m3fn>(int chunk_size) {
|
||||
// fp8 uses bf16 as accumulate type
|
||||
int cache_block_size = get_cache_blocks<at::BFloat16>(chunk_size);
|
||||
return std::min(MAX_CACHE_BLOCK_SIZE, cache_block_size);
|
||||
}
|
||||
|
||||
// 2d sequential loop in range : [mb0, mb1), [nb0, nb1)
|
||||
template <typename T, typename func_t>
|
||||
inline void loop_2d(int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1, int64_t chunk_size, const func_t& f) {
|
||||
// get number of blocks for L2 in most inner loop
|
||||
int64_t cache_blocks_nb = get_cache_blocks<T>(chunk_size);
|
||||
|
||||
// loop order: [NB / cache_blocks_nb, MB, cache_blocks_nb]
|
||||
// TODO: implement reverse order of [MB / cache_blocks_mb, NB, cache_blocks_mb]
|
||||
for (int64_t nbb = nb0; nbb < nb1; nbb += cache_blocks_nb) {
|
||||
for (int64_t mb = mb0; mb < mb1; ++mb) {
|
||||
for (int64_t nb = nbb; nb < std::min(nbb + cache_blocks_nb, nb1); ++nb) {
|
||||
f(mb, nb, nb - nbb);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// data indexing for dimension collapse
|
||||
@@ -243,4 +416,10 @@ struct Unroll<1> {
|
||||
}
|
||||
};
|
||||
|
||||
} // anonymous namespace
|
||||
// conditional data ptr for optional tensor
|
||||
template <typename T>
|
||||
inline T* conditional_data_ptr(const std::optional<at::Tensor>& opt) {
|
||||
return opt.has_value() ? opt.value().data_ptr<T>() : nullptr;
|
||||
}
|
||||
|
||||
} // anonymous namespace
|
||||
|
||||
@@ -0,0 +1,709 @@
|
||||
// Adapted from
|
||||
// https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu
|
||||
|
||||
// clang-format off
|
||||
|
||||
#include "common.h"
|
||||
#include "gemm.h"
|
||||
#include "vec.h"
|
||||
|
||||
namespace {
|
||||
|
||||
template <typename scalar_t>
|
||||
inline void copy_stub(scalar_t* __restrict__ y, const scalar_t* __restrict__ x, int64_t size) {
|
||||
using Vec = at::vec::Vectorized<scalar_t>;
|
||||
const bool is_padding = (x == nullptr);
|
||||
for (int64_t d = 0; d < size; d += Vec::size()) {
|
||||
Vec data_vec = is_padding ? Vec(0.f) : Vec::loadu(x + d);
|
||||
data_vec.store(y + d);
|
||||
}
|
||||
}
|
||||
|
||||
// no remainder
|
||||
template <typename scalar_t>
|
||||
void inline update_conv_state(
|
||||
scalar_t* __restrict__ conv_states,
|
||||
const scalar_t* __restrict__ input,
|
||||
int64_t width,
|
||||
int64_t dim,
|
||||
int64_t seqlen,
|
||||
bool has_initial_states) {
|
||||
// width for `conv_states`
|
||||
int64_t width1 = width - 1;
|
||||
int64_t w = 0;
|
||||
for (; w < width1 - seqlen; ++w) {
|
||||
scalar_t* y = conv_states + w * dim;
|
||||
const scalar_t* x = has_initial_states ? conv_states + (w + seqlen) * dim : nullptr;
|
||||
copy_stub(y, x, dim);
|
||||
}
|
||||
for (; w < width1; ++w) {
|
||||
scalar_t* y = conv_states + w * dim;
|
||||
const scalar_t* x = input + (w + seqlen - width1) * dim;
|
||||
copy_stub(y, x, dim);
|
||||
}
|
||||
}
|
||||
|
||||
// A : [M, BLOCK_N]
|
||||
// B : [BLOCK_N, K], prepacked as [K/2, BLOCK_N, 2]
|
||||
// C : [M, BLOCK_N]
|
||||
// bias : [BLOCK_N]
|
||||
//
|
||||
// lda : leading dimension of `input` and `out`
|
||||
//
|
||||
template <typename scalar_t, int K, int BLOCK_N, bool has_bias, bool has_silu>
|
||||
struct tinygemm_kernel {
|
||||
static inline void apply(
|
||||
const scalar_t* __restrict__ A,
|
||||
const scalar_t* __restrict__ B,
|
||||
scalar_t* __restrict__ C,
|
||||
const scalar_t* __restrict__ bias,
|
||||
const scalar_t* __restrict__ conv_states,
|
||||
bool has_initial_state,
|
||||
int64_t M,
|
||||
int64_t lda,
|
||||
bool is_first_token) {
|
||||
TORCH_CHECK(false, "tinygemm_kernel_nn: scalar path not implemented!");
|
||||
}
|
||||
};
|
||||
|
||||
#if defined(CPU_CAPABILITY_AVX512)
|
||||
template <int K, int BLOCK_N, bool has_bias, bool has_silu>
|
||||
struct tinygemm_kernel<at::BFloat16, K, BLOCK_N, has_bias, has_silu> {
|
||||
static inline void apply(
|
||||
const at::BFloat16* __restrict__ A,
|
||||
const at::BFloat16* __restrict__ B,
|
||||
at::BFloat16* __restrict__ C,
|
||||
const at::BFloat16* __restrict__ bias,
|
||||
const at::BFloat16* __restrict__ conv_states,
|
||||
bool has_initial_state,
|
||||
int64_t M,
|
||||
int64_t lda,
|
||||
bool is_first_token) {
|
||||
assert(K == 4);
|
||||
constexpr int ROWS = K;
|
||||
constexpr int COLS = BLOCK_N / block_size_n();
|
||||
|
||||
// leading dimension size for b for next block [K/2, 32, 2]
|
||||
constexpr int ldb = block_size_n() * K;
|
||||
|
||||
__m512bh va[ROWS * COLS];
|
||||
__m512bh vb[ROWS * COLS];
|
||||
__m512 vc[COLS * 2];
|
||||
|
||||
// k: {-3, -2, -1} -> {0, 1, 2}
|
||||
auto set_conv_states = [&](int k, int col) -> __m512i {
|
||||
return has_initial_state ? _mm512_loadu_si512(conv_states + (k + K - 1) * lda + col * 32)
|
||||
: _mm512_setzero_si512();
|
||||
};
|
||||
|
||||
#define MM512_LOAD_A(idx) \
|
||||
((idx) < 0 && is_first_token) ? (__m512bh)(set_conv_states((idx), col)) \
|
||||
: (__m512bh)(_mm512_loadu_si512(A + (idx) * lda + col * 32))
|
||||
|
||||
#define MM512_PACK_A(ap, bp, a, b) \
|
||||
do { \
|
||||
__m512i r0 = (__m512i)(a); \
|
||||
__m512i r1 = (__m512i)(b); \
|
||||
__m512i d0 = _mm512_unpacklo_epi16(r0, r1); \
|
||||
__m512i d1 = _mm512_unpackhi_epi16(r0, r1); \
|
||||
r0 = _mm512_shuffle_i32x4(d0, d1, 0x88); \
|
||||
r1 = _mm512_shuffle_i32x4(d0, d1, 0xdd); \
|
||||
(ap) = (__m512bh)_mm512_shuffle_i32x4(r0, r1, 0x88); \
|
||||
(bp) = (__m512bh)_mm512_shuffle_i32x4(r0, r1, 0xdd); \
|
||||
} while (0)
|
||||
|
||||
// step 0 : preload a at time step [-3][-2][-1]
|
||||
auto preloada = [&](auto i) {
|
||||
constexpr int col = i;
|
||||
int64_t m = 0;
|
||||
va[1 * COLS + col] = MM512_LOAD_A(m - 3);
|
||||
va[2 * COLS + col] = MM512_LOAD_A(m - 2);
|
||||
va[3 * COLS + col] = MM512_LOAD_A(m - 1);
|
||||
};
|
||||
Unroll<COLS>{}(preloada);
|
||||
|
||||
auto loada = [&](auto i, int64_t m) {
|
||||
constexpr int col = i;
|
||||
// update previous time step
|
||||
va[0 * COLS + col] = va[1 * COLS + col];
|
||||
va[1 * COLS + col] = va[2 * COLS + col];
|
||||
va[2 * COLS + col] = va[3 * COLS + col];
|
||||
// load current time step
|
||||
va[3 * COLS + col] = MM512_LOAD_A(m);
|
||||
};
|
||||
|
||||
// step 1 : load weight for just once
|
||||
auto loadb = [&](auto i) {
|
||||
constexpr int row = i / COLS;
|
||||
constexpr int col = i % COLS;
|
||||
vb[row * COLS + col] = (__m512bh)(_mm512_loadu_si512(B + col * ldb + row * 32));
|
||||
};
|
||||
Unroll<ROWS * COLS>{}(loadb);
|
||||
|
||||
// [NB] accumulates 4x32 bfloat16 blocks
|
||||
//
|
||||
// +------------+------------+
|
||||
// | col0 | col1 |
|
||||
// +------------+------------+
|
||||
// | va0 va1 | va0 va1 |
|
||||
// | va2 va3 | va2 va3 |
|
||||
// +------------+------------+
|
||||
// | vc0 vc1 | vc0 vc1 |
|
||||
// +------------+------------+
|
||||
//
|
||||
// * va and vb shares the same memory layout
|
||||
// * block_n 32 with 4 rows equals to 4 registers
|
||||
// * 37 uops with avx512bf16 v.s. 57 uops with avx512f
|
||||
//
|
||||
auto compute = [&](auto i) {
|
||||
constexpr int col = i;
|
||||
|
||||
// init accumulators
|
||||
if constexpr (has_bias) {
|
||||
__m512i b16 = _mm512_loadu_si512(reinterpret_cast<const __m512i*>(bias + col * 32));
|
||||
vc[col * 2 + 0] = CVT_BF16_TO_FP32(_mm512_extracti32x8_epi32(b16, 0));
|
||||
vc[col * 2 + 1] = CVT_BF16_TO_FP32(_mm512_extracti32x8_epi32(b16, 1));
|
||||
} else {
|
||||
vc[col * 2 + 0] = _mm512_set1_ps(0.f);
|
||||
vc[col * 2 + 1] = _mm512_set1_ps(0.f);
|
||||
}
|
||||
|
||||
// convert to vnni2 format
|
||||
__m512bh va0, va1, va2, va3;
|
||||
MM512_PACK_A(va0, va1, va[0 * COLS + col], va[1 * COLS + col]);
|
||||
MM512_PACK_A(va2, va3, va[2 * COLS + col], va[3 * COLS + col]);
|
||||
|
||||
// accumulate
|
||||
vc[col * 2 + 0] = _mm512_dpbf16_ps(vc[col * 2 + 0], va0, vb[0 * COLS + col]);
|
||||
vc[col * 2 + 0] = _mm512_dpbf16_ps(vc[col * 2 + 0], va2, vb[2 * COLS + col]);
|
||||
vc[col * 2 + 1] = _mm512_dpbf16_ps(vc[col * 2 + 1], va1, vb[1 * COLS + col]);
|
||||
vc[col * 2 + 1] = _mm512_dpbf16_ps(vc[col * 2 + 1], va3, vb[3 * COLS + col]);
|
||||
};
|
||||
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
using bVec = at::vec::Vectorized<at::BFloat16>;
|
||||
const fVec one = fVec(1.f);
|
||||
auto storec = [&](auto i, int64_t m) {
|
||||
constexpr int col = i;
|
||||
fVec x0 = fVec(vc[col * 2 + 0]);
|
||||
fVec x1 = fVec(vc[col * 2 + 1]);
|
||||
if constexpr (has_silu) {
|
||||
x0 = x0 / (one + x0.neg().exp_u20());
|
||||
x1 = x1 / (one + x1.neg().exp_u20());
|
||||
}
|
||||
bVec out_vec = convert_from_float_ext<at::BFloat16>(x0, x1);
|
||||
out_vec.store(C + m * lda + col * 32);
|
||||
};
|
||||
|
||||
for (int64_t m = 0; m < M; ++m) {
|
||||
// step 3.a : load a at current time step
|
||||
Unroll<COLS>{}(loada, m);
|
||||
// step 3.b : accumulate for window size (4)
|
||||
Unroll<COLS>{}(compute);
|
||||
// step 3.c : store c at current time step
|
||||
Unroll<COLS>{}(storec, m);
|
||||
}
|
||||
}
|
||||
};
|
||||
#endif
|
||||
|
||||
#define LAUNCH_TINYGEMM_KERNEL(K, NB_SIZE) \
|
||||
tinygemm_kernel<scalar_t, K, NB_SIZE, has_bias, has_silu>::apply( \
|
||||
input + bs * seqlen * dim + mb_start * dim + nb_start, \
|
||||
weight + nb_start * width, \
|
||||
out + bs * seqlen * dim + mb_start * dim + nb_start, \
|
||||
has_bias ? bias + nb_start : nullptr, \
|
||||
has_conv_states ? conv_states + conv_state_index * (K - 1) * dim + nb_start : nullptr, \
|
||||
has_initial_states_value, \
|
||||
mb_size, \
|
||||
dim, \
|
||||
mb_start == 0);
|
||||
|
||||
template <typename scalar_t>
|
||||
void causal_conv1d_fwd_kernel_impl(
|
||||
scalar_t* __restrict__ out,
|
||||
const scalar_t* __restrict__ input,
|
||||
const scalar_t* __restrict__ weight,
|
||||
const scalar_t* __restrict__ bias,
|
||||
scalar_t* __restrict__ conv_states,
|
||||
const int32_t* __restrict__ conv_indices,
|
||||
const bool* __restrict__ has_initial_state,
|
||||
bool silu_activation,
|
||||
int64_t batch,
|
||||
int64_t dim,
|
||||
int64_t seqlen,
|
||||
int64_t width,
|
||||
int64_t num_seq_blocks) {
|
||||
// handle 32 x 64 per block
|
||||
constexpr int64_t BLOCK_M = block_size_m();
|
||||
constexpr int64_t BLOCK_N = block_size_n() * 2;
|
||||
const int64_t NB = div_up(dim, BLOCK_N);
|
||||
|
||||
const int64_t num_blocks_per_seq = div_up(seqlen, BLOCK_M);
|
||||
const bool has_conv_states = conv_states != nullptr;
|
||||
const bool has_conv_indices = conv_indices != nullptr;
|
||||
|
||||
// parallel on [batch, seq, NB]
|
||||
AT_DISPATCH_BOOL2(bias != nullptr, has_bias, silu_activation, has_silu, [&] {
|
||||
at::parallel_for(0, num_seq_blocks * NB, 0, [&](int64_t begin, int64_t end) {
|
||||
int64_t mb{0}, nb{0};
|
||||
data_index_init(begin, mb, num_seq_blocks, nb, NB);
|
||||
|
||||
for (int64_t i = begin; i < end; ++i) {
|
||||
int64_t bs = mb / num_blocks_per_seq;
|
||||
|
||||
int64_t mb_start = (mb % num_blocks_per_seq) * BLOCK_M;
|
||||
int64_t mb_size = std::min(seqlen - mb_start, BLOCK_M);
|
||||
int64_t nb_start = nb * BLOCK_N;
|
||||
int64_t nb_size = std::min(dim - nb_start, BLOCK_N);
|
||||
|
||||
const bool has_initial_states_value = has_conv_states ? has_initial_state[bs] : false;
|
||||
int32_t conv_state_index = has_conv_indices ? conv_indices[bs] : bs;
|
||||
|
||||
switch (width << 4 | nb_size >> 4) {
|
||||
case 0x42:
|
||||
LAUNCH_TINYGEMM_KERNEL(4, 32);
|
||||
break;
|
||||
case 0x44:
|
||||
LAUNCH_TINYGEMM_KERNEL(4, 64);
|
||||
break;
|
||||
default:
|
||||
TORCH_CHECK(false, "Unexpected block size, ", width, " x ", nb_size);
|
||||
}
|
||||
|
||||
// move to the next index
|
||||
data_index_step(mb, num_seq_blocks, nb, NB);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// update conv_states if necessary
|
||||
if (has_conv_states) {
|
||||
at::parallel_for(0, batch, 0, [&](int64_t begin, int64_t end) {
|
||||
for (int64_t bs = begin; bs < end; ++bs) {
|
||||
update_conv_state(
|
||||
conv_states + bs * (width - 1) * dim, input + bs * seqlen * dim, width, dim, seqlen, has_initial_state[bs]);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#define LAUNCH_TINYGEMM_VARLEN_KERNEL(K, NB_SIZE) \
|
||||
tinygemm_kernel<scalar_t, K, NB_SIZE, has_bias, has_silu>::apply( \
|
||||
input + batch_offset * dim + mb_start * dim + nb_start, \
|
||||
weight + nb_start * width, \
|
||||
out + batch_offset * dim + mb_start * dim + nb_start, \
|
||||
has_bias ? bias + nb_start : nullptr, \
|
||||
nullptr, \
|
||||
false, \
|
||||
mb_size, \
|
||||
dim, \
|
||||
mb_start == 0);
|
||||
|
||||
// TODO: add `has_initial_state` support for varlen kernel
|
||||
template <typename scalar_t>
|
||||
void causal_conv1d_fwd_varlen_kernel_impl(
|
||||
scalar_t* __restrict__ out,
|
||||
const scalar_t* __restrict__ input,
|
||||
const scalar_t* __restrict__ weight,
|
||||
const scalar_t* __restrict__ bias,
|
||||
scalar_t* __restrict__ conv_states,
|
||||
const int32_t* __restrict__ query_start_loc,
|
||||
const int32_t* __restrict__ conv_indices,
|
||||
const bool* __restrict__ has_initial_state,
|
||||
const int32_t* __restrict__ block_indices,
|
||||
bool silu_activation,
|
||||
int64_t batch,
|
||||
int64_t dim,
|
||||
int64_t width,
|
||||
int64_t num_seq_blocks) {
|
||||
// handle 32 x 64 per block
|
||||
constexpr int64_t BLOCK_M = block_size_m();
|
||||
constexpr int64_t BLOCK_N = block_size_n() * 2;
|
||||
const int64_t NB = div_up(dim, BLOCK_N);
|
||||
|
||||
const bool has_conv_states = conv_states != nullptr;
|
||||
const bool has_conv_indices = conv_indices != nullptr;
|
||||
|
||||
// parallel on [batch, seq, NB]
|
||||
AT_DISPATCH_BOOL2(bias != nullptr, has_bias, silu_activation, has_silu, [&] {
|
||||
at::parallel_for(0, num_seq_blocks * NB, 0, [&](int64_t begin, int64_t end) {
|
||||
int64_t mb{0}, nb{0};
|
||||
data_index_init(begin, mb, num_seq_blocks, nb, NB);
|
||||
|
||||
for (int64_t i = begin; i < end; ++i) {
|
||||
int32_t bs = block_indices[mb * 2 + 0];
|
||||
int32_t batch_offset = query_start_loc[bs];
|
||||
int32_t seqlen = query_start_loc[bs + 1] - query_start_loc[bs];
|
||||
|
||||
int64_t mb_start = block_indices[mb * 2 + 1] * BLOCK_M;
|
||||
int64_t mb_size = std::min(seqlen - mb_start, BLOCK_M);
|
||||
int64_t nb_start = nb * BLOCK_N;
|
||||
int64_t nb_size = std::min(dim - nb_start, BLOCK_N);
|
||||
|
||||
switch (width << 4 | nb_size >> 4) {
|
||||
case 0x42:
|
||||
LAUNCH_TINYGEMM_VARLEN_KERNEL(4, 32);
|
||||
break;
|
||||
case 0x44:
|
||||
LAUNCH_TINYGEMM_VARLEN_KERNEL(4, 64);
|
||||
break;
|
||||
default:
|
||||
TORCH_CHECK(false, "Unexpected block size, ", width, " x ", nb_size);
|
||||
}
|
||||
|
||||
// move to the next index
|
||||
data_index_step(mb, num_seq_blocks, nb, NB);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// update conv_states if necessary
|
||||
if (has_conv_states) {
|
||||
at::parallel_for(0, batch, 0, [&](int64_t begin, int64_t end) {
|
||||
for (int64_t bs = begin; bs < end; ++bs) {
|
||||
int32_t conv_state_index = has_conv_indices ? conv_indices[bs] : bs;
|
||||
int32_t seqlen = query_start_loc[bs + 1] - query_start_loc[bs];
|
||||
int32_t batch_offset = query_start_loc[bs];
|
||||
update_conv_state(
|
||||
conv_states + conv_state_index * (width - 1) * dim,
|
||||
input + batch_offset * dim,
|
||||
width,
|
||||
dim,
|
||||
seqlen,
|
||||
/* has_initial_state */ false);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
void causal_conv1d_update_kernel_impl(
|
||||
scalar_t* __restrict__ out,
|
||||
const scalar_t* __restrict__ input,
|
||||
scalar_t* __restrict__ conv_states,
|
||||
const scalar_t* __restrict__ weight,
|
||||
const scalar_t* __restrict__ bias,
|
||||
const int32_t* __restrict__ conv_indices,
|
||||
bool silu_activation,
|
||||
int64_t batch,
|
||||
int64_t dim,
|
||||
int64_t seqlen,
|
||||
int64_t width) {
|
||||
// handle 32 x 64 per block
|
||||
constexpr int64_t BLOCK_M = block_size_m();
|
||||
constexpr int64_t BLOCK_N = block_size_n() * 2;
|
||||
const int64_t NB = div_up(dim, BLOCK_N);
|
||||
|
||||
const bool has_conv_states = conv_states != nullptr;
|
||||
const bool has_conv_indices = conv_indices != nullptr;
|
||||
|
||||
// parallel on [batch, NB]
|
||||
AT_DISPATCH_BOOL2(bias != nullptr, has_bias, silu_activation, has_silu, [&] {
|
||||
at::parallel_for(0, batch * NB, 0, [&](int64_t begin, int64_t end) {
|
||||
int64_t bs{0}, nb{0};
|
||||
data_index_init(begin, bs, batch, nb, NB);
|
||||
|
||||
for (int64_t i = begin; i < end; ++i) {
|
||||
int64_t mb_start = 0;
|
||||
int64_t mb_size = 1;
|
||||
int64_t nb_start = nb * BLOCK_N;
|
||||
int64_t nb_size = std::min(dim - nb_start, BLOCK_N);
|
||||
|
||||
const bool has_initial_states_value = true;
|
||||
int32_t conv_state_index = has_conv_indices ? conv_indices[bs] : bs;
|
||||
|
||||
switch (width << 4 | nb_size >> 4) {
|
||||
case 0x42:
|
||||
LAUNCH_TINYGEMM_KERNEL(4, 32);
|
||||
break;
|
||||
case 0x44:
|
||||
LAUNCH_TINYGEMM_KERNEL(4, 64);
|
||||
break;
|
||||
default:
|
||||
TORCH_CHECK(false, "Unexpected block size, ", width, " x ", nb_size);
|
||||
}
|
||||
|
||||
// move to the next index
|
||||
data_index_step(bs, batch, nb, NB);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
#define CONV_STATE_INDEXR(w) conv_states + conv_state_index*(width - 1) * dim + (w) * dim
|
||||
|
||||
// update conv_states
|
||||
at::parallel_for(0, batch, 0, [&](int64_t begin, int64_t end) {
|
||||
for (int64_t bs = begin; bs < end; ++bs) {
|
||||
// update old states, range [1, width - 1)
|
||||
int32_t conv_state_index = has_conv_indices ? conv_indices[bs] : bs;
|
||||
for (int64_t w = 1; w < width - 1; ++w) {
|
||||
std::memcpy(CONV_STATE_INDEXR(w - 1), CONV_STATE_INDEXR(w), dim * sizeof(scalar_t));
|
||||
}
|
||||
// copy new states
|
||||
std::memcpy(CONV_STATE_INDEXR(width - 2), input + bs * dim, dim * sizeof(scalar_t));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
} // anonymous namespace
|
||||
|
||||
// from [dim, width] or [N, K]
|
||||
// to [N/BLOCK_N, K/2, BLOCK_N, 2]
|
||||
at::Tensor causal_conv1d_weight_pack(const at::Tensor& weight) {
|
||||
CHECK_INPUT(weight);
|
||||
|
||||
int64_t dim = weight.size(0);
|
||||
int64_t width = weight.size(1);
|
||||
constexpr int64_t BLOCK_N = block_size_n();
|
||||
TORCH_CHECK(width == 4, "causal_conv1d_weight_pack: support only width of 4");
|
||||
TORCH_CHECK(dim % BLOCK_N == 0, "causal_conv1d_weight_pack: invalid dim size ", dim);
|
||||
|
||||
const int64_t N = dim, K2 = width >> 1;
|
||||
const int64_t NB = div_up(N, BLOCK_N);
|
||||
|
||||
auto packed_weight = at::empty_like(weight);
|
||||
AT_DISPATCH_REDUCED_FLOATING_TYPES(weight.scalar_type(), "causal_conv1d_fwd_kernel_impl", [&] {
|
||||
// cast to float32 as vnni size is 2
|
||||
const float* w_data = reinterpret_cast<float*>(weight.data_ptr<scalar_t>());
|
||||
float* packed_data = reinterpret_cast<float*>(packed_weight.data_ptr<scalar_t>());
|
||||
|
||||
at::parallel_for(0, NB * K2 * BLOCK_N, 0, [&](int64_t begin, int64_t end) {
|
||||
int64_t nb{0}, k2{0}, n{0};
|
||||
data_index_init(begin, nb, NB, k2, K2, n, BLOCK_N);
|
||||
|
||||
// TODO: optimize this if we need to online prepacking.
|
||||
for (int64_t i = begin; i < end; ++i) {
|
||||
packed_data[i] = w_data[nb * BLOCK_N * K2 + n * K2 + k2];
|
||||
|
||||
// move to the next index
|
||||
data_index_step(nb, NB, k2, K2, n, BLOCK_N);
|
||||
}
|
||||
});
|
||||
});
|
||||
return packed_weight;
|
||||
}
|
||||
|
||||
#define CHECK_OPTIONAL_SHAPE_DTYPE(OPT, SIZE, DTYPE) \
|
||||
if (OPT.has_value()) { \
|
||||
const auto tensor = OPT.value(); \
|
||||
CHECK_CONTIGUOUS(tensor); \
|
||||
CHECK_EQ(tensor.size(0), SIZE); \
|
||||
CHECK_EQ(tensor.scalar_type(), DTYPE); \
|
||||
}
|
||||
|
||||
template <int BLOCK_M>
|
||||
int64_t get_block_count(const std::optional<at::Tensor>& offsets, int64_t batch, int64_t seqlen) {
|
||||
if (offsets.has_value()) {
|
||||
const int32_t* offsets_data = offsets.value().data_ptr<int32_t>();
|
||||
int32_t num_seq_blocks = 0;
|
||||
for (int64_t row = 0; row < batch; ++row) {
|
||||
num_seq_blocks += div_up(offsets_data[row + 1] - offsets_data[row], BLOCK_M);
|
||||
}
|
||||
return num_seq_blocks;
|
||||
}
|
||||
return batch * div_up(seqlen, int64_t(BLOCK_M));
|
||||
}
|
||||
|
||||
template <int BLOCK_M>
|
||||
at::Tensor get_block_indices(const std::optional<at::Tensor>& offsets, int64_t num_seq_blocks) {
|
||||
if (!offsets.has_value()) {
|
||||
return at::Tensor();
|
||||
}
|
||||
|
||||
const at::Tensor& offsets_ = offsets.value();
|
||||
at::Tensor indices = at::empty({num_seq_blocks, 2}, offsets_.options());
|
||||
|
||||
int64_t batch = offsets_.size(0) - 1;
|
||||
|
||||
const int32_t* offsets_data = offsets_.data_ptr<int32_t>();
|
||||
int32_t* indices_data = indices.data_ptr<int32_t>();
|
||||
|
||||
int64_t idx = 0;
|
||||
for (int32_t row = 0; row < batch; ++row) {
|
||||
int32_t blocks = div_up(offsets_data[row + 1] - offsets_data[row], BLOCK_M);
|
||||
|
||||
for (int32_t col = 0; col < blocks; ++col) {
|
||||
indices_data[idx * 2 + 0] = row;
|
||||
indices_data[idx * 2 + 1] = col;
|
||||
idx++;
|
||||
}
|
||||
}
|
||||
return indices;
|
||||
}
|
||||
|
||||
// API aligned with GPUs
|
||||
//
|
||||
// x: (batch, dim, seqlen) or (dim, cu_seq_len) for varlen
|
||||
// weight: (dim, width)
|
||||
// bias: (dim,)
|
||||
// query_start_loc: (batch + 1) int32
|
||||
// cache_indices: (batch) int32
|
||||
// has_initial_state: (batch) bool
|
||||
// conv_states: (..., dim, width - 1) itype
|
||||
// activation: either None or "silu" or "swish"
|
||||
// pad_slot_id: int
|
||||
//
|
||||
at::Tensor causal_conv1d_fwd_cpu(
|
||||
const at::Tensor& x,
|
||||
const at::Tensor& weight,
|
||||
const std::optional<at::Tensor>& bias,
|
||||
const std::optional<at::Tensor>& conv_states,
|
||||
const std::optional<at::Tensor>& query_start_loc,
|
||||
const std::optional<at::Tensor>& conv_state_indices,
|
||||
const std::optional<at::Tensor>& has_initial_state,
|
||||
bool silu_activation,
|
||||
int64_t pad_slot_id,
|
||||
bool is_vnni) {
|
||||
CHECK_CONTIGUOUS(weight);
|
||||
auto packed_w = is_vnni ? weight : causal_conv1d_weight_pack(weight);
|
||||
|
||||
const bool is_var_seqlen = query_start_loc.has_value();
|
||||
const int64_t input_ndim = is_var_seqlen ? 2 : 3;
|
||||
TORCH_CHECK(x.dim() == input_ndim, "causal_conv1d_fwd_cpu: expect x to be ", input_ndim, "D tensor.");
|
||||
TORCH_CHECK(x.stride(-2) == 1 && x.stride(-1) == x.size(-2), "causal_conv1d_fwd_cpu: expect x to be transposed.");
|
||||
|
||||
const int64_t batch = is_var_seqlen ? query_start_loc.value().size(0) - 1 : x.size(0);
|
||||
const int64_t dim = x.size(-2);
|
||||
const int64_t seqlen = x.size(-1);
|
||||
const int64_t width = weight.size(-1);
|
||||
|
||||
const auto scalar_type = x.scalar_type();
|
||||
CHECK_EQ(weight.scalar_type(), scalar_type);
|
||||
CHECK_OPTIONAL_SHAPE_DTYPE(bias, dim, scalar_type);
|
||||
CHECK_OPTIONAL_SHAPE_DTYPE(query_start_loc, batch + 1, at::kInt);
|
||||
CHECK_OPTIONAL_SHAPE_DTYPE(conv_state_indices, batch, at::kInt);
|
||||
CHECK_OPTIONAL_SHAPE_DTYPE(has_initial_state, batch, at::kBool);
|
||||
|
||||
if (conv_states.has_value()) {
|
||||
auto& conv_states_val = conv_states.value();
|
||||
int64_t padded_batch = conv_states_val.size(0);
|
||||
CHECK_EQ(conv_states_val.scalar_type(), scalar_type);
|
||||
CHECK_GE(padded_batch, batch);
|
||||
CHECK_EQ(conv_states_val.size(1), dim);
|
||||
CHECK_EQ(conv_states_val.size(2), width - 1);
|
||||
|
||||
// adjust `conv_states` to be contiguous on `dim`
|
||||
// should happen only once
|
||||
if (conv_states_val.stride(-2) != 1) {
|
||||
auto conv_states_copy = conv_states_val.clone();
|
||||
conv_states_val.as_strided_({padded_batch, dim, width - 1}, {(width - 1) * dim, 1, dim});
|
||||
conv_states_val.copy_(conv_states_copy);
|
||||
}
|
||||
}
|
||||
|
||||
// block size for sequence blocks, 32
|
||||
constexpr int64_t BLOCK_M = block_size_m();
|
||||
|
||||
// total number of sequence blocks
|
||||
int64_t num_seq_blocks = get_block_count<BLOCK_M>(query_start_loc, batch, seqlen);
|
||||
|
||||
at::Tensor out = at::empty_like(x);
|
||||
AT_DISPATCH_REDUCED_FLOATING_TYPES(scalar_type, "causal_conv1d_fwd_kernel_impl", [&] {
|
||||
if (is_var_seqlen) {
|
||||
// record seq blocks in Coordinate format, aka [num_seq_blocks, 2]
|
||||
at::Tensor block_indices = get_block_indices<BLOCK_M>(query_start_loc, num_seq_blocks);
|
||||
|
||||
causal_conv1d_fwd_varlen_kernel_impl(
|
||||
out.data_ptr<scalar_t>(),
|
||||
x.data_ptr<scalar_t>(),
|
||||
packed_w.data_ptr<scalar_t>(),
|
||||
conditional_data_ptr<scalar_t>(bias),
|
||||
conditional_data_ptr<scalar_t>(conv_states),
|
||||
conditional_data_ptr<int32_t>(query_start_loc),
|
||||
conditional_data_ptr<int32_t>(conv_state_indices),
|
||||
conditional_data_ptr<bool>(has_initial_state),
|
||||
block_indices.data_ptr<int32_t>(),
|
||||
silu_activation,
|
||||
batch,
|
||||
dim,
|
||||
width,
|
||||
num_seq_blocks);
|
||||
} else {
|
||||
causal_conv1d_fwd_kernel_impl<scalar_t>(
|
||||
out.data_ptr<scalar_t>(),
|
||||
x.data_ptr<scalar_t>(),
|
||||
packed_w.data_ptr<scalar_t>(),
|
||||
conditional_data_ptr<scalar_t>(bias),
|
||||
conditional_data_ptr<scalar_t>(conv_states),
|
||||
conditional_data_ptr<int32_t>(conv_state_indices),
|
||||
conditional_data_ptr<bool>(has_initial_state),
|
||||
silu_activation,
|
||||
batch,
|
||||
dim,
|
||||
seqlen,
|
||||
width,
|
||||
num_seq_blocks);
|
||||
}
|
||||
});
|
||||
return out;
|
||||
}
|
||||
|
||||
// API aligned with GPUs
|
||||
//
|
||||
// x: (batch, dim) or (batch, dim, seqlen)
|
||||
// conv_state: (..., dim, state_len), where state_len >= width - 1
|
||||
// weight: (dim, width)
|
||||
// bias: (dim,)
|
||||
// cache_seqlens: (batch,), dtype int32.
|
||||
// conv_state_indices: (batch,), dtype int32
|
||||
// pad_slot_id: int
|
||||
// out: (batch, dim) or (batch, dim, seqlen)
|
||||
//
|
||||
at::Tensor causal_conv1d_update_cpu(
|
||||
const at::Tensor& x,
|
||||
const at::Tensor& conv_states,
|
||||
const at::Tensor& weight,
|
||||
const std::optional<at::Tensor>& bias,
|
||||
bool silu_activation,
|
||||
const std::optional<at::Tensor>& cache_seqlens,
|
||||
const std::optional<at::Tensor>& conv_state_indices,
|
||||
int64_t pad_slot_id,
|
||||
bool is_vnni) {
|
||||
CHECK_CONTIGUOUS(x);
|
||||
CHECK_CONTIGUOUS(weight);
|
||||
auto packed_w = is_vnni ? weight : causal_conv1d_weight_pack(weight);
|
||||
|
||||
// TODO: add multi-token prediction support
|
||||
TORCH_CHECK(x.dim() == 2, "causal_conv1d_update_cpu: expect x to be 2D tensor.");
|
||||
TORCH_CHECK(!cache_seqlens.has_value(), "causal_conv1d_update_cpu: don't support cache_seqlens.");
|
||||
|
||||
int64_t batch = x.size(0);
|
||||
int64_t dim = x.size(1);
|
||||
int64_t seqlen = 1;
|
||||
int64_t width = weight.size(-1);
|
||||
|
||||
const auto scalar_type = x.scalar_type();
|
||||
CHECK_EQ(weight.scalar_type(), scalar_type);
|
||||
CHECK_OPTIONAL_SHAPE_DTYPE(bias, dim, scalar_type);
|
||||
CHECK_OPTIONAL_SHAPE_DTYPE(conv_state_indices, batch, at::kInt);
|
||||
|
||||
CHECK_EQ(conv_states.scalar_type(), scalar_type);
|
||||
CHECK_EQ(conv_states.size(1), dim);
|
||||
CHECK_EQ(conv_states.size(2), width - 1);
|
||||
|
||||
// adjust `conv_states` to be contiguous on `dim`
|
||||
if (conv_states.stride(-2) != 1) {
|
||||
int64_t num_cache_lines = conv_states.size(0);
|
||||
auto conv_states_copy = conv_states.clone();
|
||||
conv_states.as_strided_({num_cache_lines, dim, width - 1}, {(width - 1) * dim, 1, dim});
|
||||
conv_states.copy_(conv_states_copy);
|
||||
}
|
||||
|
||||
at::Tensor out = at::empty_like(x);
|
||||
AT_DISPATCH_REDUCED_FLOATING_TYPES(scalar_type, "causal_conv1d_update_kernel_impl", [&] {
|
||||
causal_conv1d_update_kernel_impl<scalar_t>(
|
||||
out.data_ptr<scalar_t>(),
|
||||
x.data_ptr<scalar_t>(),
|
||||
conv_states.data_ptr<scalar_t>(),
|
||||
packed_w.data_ptr<scalar_t>(),
|
||||
conditional_data_ptr<scalar_t>(bias),
|
||||
conditional_data_ptr<int32_t>(conv_state_indices),
|
||||
silu_activation,
|
||||
batch,
|
||||
dim,
|
||||
seqlen,
|
||||
width);
|
||||
});
|
||||
return out;
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
+476
-87
@@ -1,11 +1,12 @@
|
||||
// Adapted from
|
||||
// https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu
|
||||
|
||||
#include "common.h"
|
||||
#include "vec.h"
|
||||
// clang-format off
|
||||
|
||||
#include "gemm.h"
|
||||
|
||||
// clang-format off
|
||||
#include "common.h"
|
||||
#include "vec.h"
|
||||
|
||||
namespace {
|
||||
|
||||
@@ -26,13 +27,13 @@ inline void s8s8_compensation(int8_t* __restrict__ packed, int K) {
|
||||
const __m512i off = _mm512_set1_epi8(static_cast<char>(0x80));
|
||||
for (int k = 0; k < K / 4; ++k) {
|
||||
for (int col = 0; col < COLS; ++col) {
|
||||
__m512i vb = _mm512_loadu_si512((const __m512i *)(packed + k * BLOCK_N * 4 + col * 64));
|
||||
__m512i vb = _mm512_loadu_si512((const __m512i*)(packed + k * BLOCK_N * 4 + col * 64));
|
||||
vcomp[col] = _mm512_dpbusd_epi32(vcomp[col], off, vb);
|
||||
}
|
||||
}
|
||||
|
||||
for (int col = 0; col < COLS; ++col) {
|
||||
_mm512_storeu_si512((__m512i *)(packed + offset + col * 64), vcomp[col]);
|
||||
_mm512_storeu_si512((__m512i*)(packed + offset + col * 64), vcomp[col]);
|
||||
}
|
||||
#else
|
||||
TORCH_CHECK(false, "s8s8_compensation not implemented!");
|
||||
@@ -69,6 +70,43 @@ inline void pack_vnni<int8_t>(int8_t* __restrict__ packed, const int8_t* __restr
|
||||
s8s8_compensation<BLOCK_N>(packed, K);
|
||||
}
|
||||
|
||||
// uint8_t: mxfp4 or int4
|
||||
// pack to vnni2 format as they are computed with bfloat16
|
||||
//
|
||||
// from [N, K'/2, 2] to [K'/2, N, 2], view 2x int4 as unit8:
|
||||
// from [N, K ] to [K, N ] where K = K'/2
|
||||
//
|
||||
template <>
|
||||
inline void pack_vnni<uint8_t>(uint8_t* __restrict__ packed, const uint8_t* __restrict__ weight, int N, int K) {
|
||||
constexpr int BLOCK_N = block_size_n();
|
||||
|
||||
uint8_t unpacked[2 * BLOCK_N];
|
||||
|
||||
// 32-way pack (align with BLOCK_N), faster for avx512 unpacking
|
||||
//
|
||||
// for a range of (64):
|
||||
// {0, 1, 2, ..., 63}
|
||||
//
|
||||
// original format:
|
||||
// { 1|0, 3|2, ..., 63|62}
|
||||
//
|
||||
// packed format:
|
||||
// {32|0, 31|1, ..., 63|31}
|
||||
//
|
||||
for (int k = 0; k < K; ++k) {
|
||||
// unpack first
|
||||
for (int n = 0; n < N; ++n) {
|
||||
uint8_t value = weight[n * K + k];
|
||||
unpacked[n * 2 + 0] = value & 0xF; // lower 4 bits
|
||||
unpacked[n * 2 + 1] = value >> 4; // higher 4 bits
|
||||
}
|
||||
// re-pack to 32-way
|
||||
for (int n = 0; n < N; ++n) {
|
||||
packed[k * N + n] = (unpacked[n + BLOCK_N] << 4) | unpacked[n];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
inline void copy_stub(scalar_t* __restrict__ out, const float* __restrict__ input, int64_t size) {
|
||||
using bVec = at::vec::Vectorized<scalar_t>;
|
||||
@@ -76,7 +114,7 @@ inline void copy_stub(scalar_t* __restrict__ out, const float* __restrict__ inpu
|
||||
constexpr int kVecSize = bVec::size();
|
||||
|
||||
int64_t d;
|
||||
#pragma GCC unroll 4
|
||||
#pragma GCC unroll 4
|
||||
for (d = 0; d <= size - kVecSize; d += kVecSize) {
|
||||
fVec data0 = fVec::loadu(input + d);
|
||||
fVec data1 = fVec::loadu(input + d + fVec::size());
|
||||
@@ -89,13 +127,34 @@ inline void copy_stub(scalar_t* __restrict__ out, const float* __restrict__ inpu
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
inline void copy_add_stub(scalar_t* __restrict__ out, const float* __restrict__ input, const float* __restrict__ bias, int64_t size) {
|
||||
inline void copy_stub(float* __restrict__ out, const scalar_t* __restrict__ input, int64_t size) {
|
||||
using bVec = at::vec::Vectorized<scalar_t>;
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
constexpr int kVecSize = bVec::size();
|
||||
|
||||
int64_t d;
|
||||
#pragma GCC unroll 4
|
||||
#pragma GCC unroll 4
|
||||
for (d = 0; d <= size - kVecSize; d += kVecSize) {
|
||||
fVec data0, data1;
|
||||
bVec b_vec = bVec::loadu(input + d);
|
||||
std::tie(data0, data1) = at::vec::convert_to_float(b_vec);
|
||||
data0.store(out + d);
|
||||
data1.store(out + d + fVec::size());
|
||||
}
|
||||
for (; d < size; ++d) {
|
||||
out[d] = static_cast<float>(input[d]);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
inline void copy_add_stub(
|
||||
scalar_t* __restrict__ out, const float* __restrict__ input, const float* __restrict__ bias, int64_t size) {
|
||||
using bVec = at::vec::Vectorized<scalar_t>;
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
constexpr int kVecSize = bVec::size();
|
||||
|
||||
int64_t d;
|
||||
#pragma GCC unroll 4
|
||||
for (d = 0; d <= size - kVecSize; d += kVecSize) {
|
||||
fVec data0 = fVec::loadu(input + d) + fVec::loadu(bias + d);
|
||||
fVec data1 = fVec::loadu(input + d + fVec::size()) + fVec::loadu(bias + d + fVec::size());
|
||||
@@ -107,11 +166,51 @@ inline void copy_add_stub(scalar_t* __restrict__ out, const float* __restrict__
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t, bool has_bias>
|
||||
inline void scalar_sigmoid_and_mul(
|
||||
scalar_t* __restrict__ out,
|
||||
const float* __restrict__ input,
|
||||
const float* __restrict__ bias,
|
||||
const scalar_t* __restrict__ mul,
|
||||
int SIZE) {
|
||||
using bVec = at::vec::Vectorized<scalar_t>;
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
// scalar sigmoid
|
||||
const fVec one = fVec(1.f);
|
||||
fVec X;
|
||||
if constexpr (has_bias) {
|
||||
assert(bias != nullptr);
|
||||
X = fVec(input[0] + bias[0]);
|
||||
} else {
|
||||
X = fVec(input[0]);
|
||||
}
|
||||
X = one / (one + X.neg().exp_u20());
|
||||
|
||||
// vec mul
|
||||
constexpr int kVecSize = bVec::size();
|
||||
for (int d = 0; d < SIZE; d += kVecSize) {
|
||||
bVec m_bvec = bVec::loadu(mul + d);
|
||||
fVec m_fvec0, m_fvec1;
|
||||
std::tie(m_fvec0, m_fvec1) = at::vec::convert_to_float(m_bvec);
|
||||
m_fvec0 = m_fvec0 * X;
|
||||
m_fvec1 = m_fvec1 * X;
|
||||
|
||||
bVec out_vec = convert_from_float_ext<scalar_t>(m_fvec0, m_fvec1);
|
||||
out_vec.store(out + d);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t, bool has_bias, int BLOCK_M, int BLOCK_N>
|
||||
struct tinygemm_kernel_nn {
|
||||
static inline void apply(
|
||||
const scalar_t* __restrict__ A, const scalar_t* __restrict__ B, scalar_t* __restrict__ C,
|
||||
const float* __restrict__ bias, int64_t K, int64_t lda, int64_t ldb, int64_t ldc) {
|
||||
const scalar_t* __restrict__ A,
|
||||
const scalar_t* __restrict__ B,
|
||||
scalar_t* __restrict__ C,
|
||||
const float* __restrict__ bias,
|
||||
int64_t K,
|
||||
int64_t lda,
|
||||
int64_t ldb,
|
||||
int64_t ldc) {
|
||||
TORCH_CHECK(false, "tinygemm_kernel_nn: scalar path not implemented!");
|
||||
}
|
||||
};
|
||||
@@ -120,9 +219,14 @@ struct tinygemm_kernel_nn {
|
||||
template <bool has_bias, int BLOCK_M, int BLOCK_N>
|
||||
struct tinygemm_kernel_nn<at::BFloat16, has_bias, BLOCK_M, BLOCK_N> {
|
||||
static inline void apply(
|
||||
const at::BFloat16* __restrict__ A, const at::BFloat16* __restrict__ B, at::BFloat16* __restrict__ C,
|
||||
const float* __restrict__ bias, int64_t K, int64_t lda, int64_t ldb, int64_t ldc) {
|
||||
|
||||
const at::BFloat16* __restrict__ A,
|
||||
const at::BFloat16* __restrict__ B,
|
||||
at::BFloat16* __restrict__ C,
|
||||
const float* __restrict__ bias,
|
||||
int64_t K,
|
||||
int64_t lda,
|
||||
int64_t ldb,
|
||||
int64_t ldc) {
|
||||
constexpr int ROWS = BLOCK_M;
|
||||
constexpr int COLS = BLOCK_N / 16;
|
||||
|
||||
@@ -145,7 +249,7 @@ struct tinygemm_kernel_nn<at::BFloat16, has_bias, BLOCK_M, BLOCK_N> {
|
||||
|
||||
const int64_t K2 = K >> 1;
|
||||
const int64_t lda2 = lda >> 1;
|
||||
const int64_t ldb2 = ldb; // ldb * 2 >> 1;
|
||||
const int64_t ldb2 = ldb; // ldb * 2 >> 1;
|
||||
const float* a_ptr = reinterpret_cast<const float*>(A);
|
||||
const float* b_ptr = reinterpret_cast<const float*>(B);
|
||||
|
||||
@@ -180,9 +284,7 @@ struct tinygemm_kernel_nn<at::BFloat16, has_bias, BLOCK_M, BLOCK_N> {
|
||||
(__m512i)(_mm512_cvtne2ps_pbh(vc[row * COLS + col + 1], vc[row * COLS + col])));
|
||||
}
|
||||
} else {
|
||||
_mm256_storeu_si256(
|
||||
reinterpret_cast<__m256i*>(C + row * ldc + col * 16),
|
||||
(__m256i)(_mm512_cvtneps_pbh(vc[i])));
|
||||
_mm256_storeu_si256(reinterpret_cast<__m256i*>(C + row * ldc + col * 16), (__m256i)(_mm512_cvtneps_pbh(vc[i])));
|
||||
}
|
||||
};
|
||||
Unroll<ROWS * COLS>{}(storec);
|
||||
@@ -190,22 +292,33 @@ struct tinygemm_kernel_nn<at::BFloat16, has_bias, BLOCK_M, BLOCK_N> {
|
||||
};
|
||||
#endif
|
||||
|
||||
#define LAUNCH_TINYGEMM_KERNEL_NN(MB_SIZE, NB_SIZE) \
|
||||
tinygemm_kernel_nn<scalar_t, has_bias, MB_SIZE, NB_SIZE>::apply( \
|
||||
A + mb_start * lda, B + nb_start * 2, C + mb_start * ldc + nb_start, \
|
||||
has_bias ? bias + nb_start : nullptr, K, lda, ldb, ldc);
|
||||
#define LAUNCH_TINYGEMM_KERNEL_NN(MB_SIZE, NB_SIZE) \
|
||||
tinygemm_kernel_nn<scalar_t, has_bias, MB_SIZE, NB_SIZE>::apply( \
|
||||
A + mb_start * lda, \
|
||||
B + nb_start * 2, \
|
||||
C + mb_start * ldc + nb_start, \
|
||||
has_bias ? bias + nb_start : nullptr, \
|
||||
K, \
|
||||
lda, \
|
||||
ldb, \
|
||||
ldc);
|
||||
|
||||
template <typename scalar_t, bool has_bias>
|
||||
struct brgemm {
|
||||
static inline void apply(
|
||||
const scalar_t* __restrict__ A, const scalar_t* __restrict__ B, scalar_t* __restrict__ C,
|
||||
float* __restrict__ Ctmp, const float* __restrict__ bias,
|
||||
int64_t M, int64_t N, int64_t K, int64_t lda, int64_t ldb, int64_t ldc) {
|
||||
|
||||
const scalar_t* __restrict__ A,
|
||||
const scalar_t* __restrict__ B,
|
||||
scalar_t* __restrict__ C,
|
||||
float* __restrict__ Ctmp,
|
||||
const float* __restrict__ bias,
|
||||
int64_t M,
|
||||
int64_t N,
|
||||
int64_t K,
|
||||
int64_t lda,
|
||||
int64_t ldb,
|
||||
int64_t ldc) {
|
||||
constexpr int BLOCK_N = block_size_n();
|
||||
at::native::cpublas::brgemm(
|
||||
M, N, K, lda, ldb, BLOCK_N, /* add_C */false,
|
||||
A, B, Ctmp);
|
||||
at::native::cpublas::brgemm(M, N, K, lda, ldb, BLOCK_N, /* add_C */ false, A, B, Ctmp);
|
||||
|
||||
// copy from Ctmp to C
|
||||
for (int64_t m = 0; m < M; ++m) {
|
||||
@@ -216,6 +329,21 @@ struct brgemm {
|
||||
}
|
||||
}
|
||||
}
|
||||
static inline void apply(
|
||||
const float* __restrict__ A,
|
||||
const float* __restrict__ B,
|
||||
scalar_t* __restrict__ C,
|
||||
float* __restrict__ Ctmp,
|
||||
const float* __restrict__ bias,
|
||||
int64_t M,
|
||||
int64_t N,
|
||||
int64_t K,
|
||||
int64_t lda,
|
||||
int64_t ldb,
|
||||
int64_t ldc) {
|
||||
constexpr int BLOCK_N = block_size_n();
|
||||
at::native::cpublas::brgemm(M, N, K, lda, ldb, BLOCK_N, /* add_C */ false, A, B, Ctmp);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename scalar_t, bool has_bias>
|
||||
@@ -232,15 +360,12 @@ void tinygemm_kernel(
|
||||
int64_t ldb,
|
||||
int64_t ldc,
|
||||
bool brg) {
|
||||
|
||||
if (brg) {
|
||||
brgemm<scalar_t, has_bias>::apply(
|
||||
A, B, C, Ctmp, bias,
|
||||
M, N, K, lda, ldb, ldc);
|
||||
brgemm<scalar_t, has_bias>::apply(A, B, C, Ctmp, bias, M, N, K, lda, ldb, ldc);
|
||||
return;
|
||||
}
|
||||
|
||||
// pattern: 1-4-16
|
||||
// pattern: 1-4-16, N = 16, 32, 48, 64
|
||||
constexpr int64_t BLOCK_M = 4;
|
||||
constexpr int64_t BLOCK_N = 64;
|
||||
const int64_t MB = div_up(M, BLOCK_M);
|
||||
@@ -252,25 +377,88 @@ void tinygemm_kernel(
|
||||
int64_t nb_start = nb * BLOCK_N;
|
||||
int64_t nb_size = std::min(BLOCK_N, N - nb_start);
|
||||
|
||||
switch(mb_size << 4 | nb_size >> 4) {
|
||||
switch (mb_size << 4 | nb_size >> 4) {
|
||||
// mb_size = 1
|
||||
case 0x12: LAUNCH_TINYGEMM_KERNEL_NN(1, 32); break;
|
||||
case 0x14: LAUNCH_TINYGEMM_KERNEL_NN(1, 64); break;
|
||||
case 0x11:
|
||||
LAUNCH_TINYGEMM_KERNEL_NN(1, 16);
|
||||
break;
|
||||
case 0x12:
|
||||
LAUNCH_TINYGEMM_KERNEL_NN(1, 32);
|
||||
break;
|
||||
case 0x13:
|
||||
LAUNCH_TINYGEMM_KERNEL_NN(1, 48);
|
||||
break;
|
||||
case 0x14:
|
||||
LAUNCH_TINYGEMM_KERNEL_NN(1, 64);
|
||||
break;
|
||||
// mb_size = 2
|
||||
case 0x22: LAUNCH_TINYGEMM_KERNEL_NN(2, 32); break;
|
||||
case 0x24: LAUNCH_TINYGEMM_KERNEL_NN(2, 64); break;
|
||||
case 0x21:
|
||||
LAUNCH_TINYGEMM_KERNEL_NN(2, 16);
|
||||
break;
|
||||
case 0x22:
|
||||
LAUNCH_TINYGEMM_KERNEL_NN(2, 32);
|
||||
break;
|
||||
case 0x23:
|
||||
LAUNCH_TINYGEMM_KERNEL_NN(2, 48);
|
||||
break;
|
||||
case 0x24:
|
||||
LAUNCH_TINYGEMM_KERNEL_NN(2, 64);
|
||||
break;
|
||||
// mb_size = 3
|
||||
case 0x32: LAUNCH_TINYGEMM_KERNEL_NN(3, 32); break;
|
||||
case 0x34: LAUNCH_TINYGEMM_KERNEL_NN(3, 64); break;
|
||||
case 0x31:
|
||||
LAUNCH_TINYGEMM_KERNEL_NN(3, 16);
|
||||
break;
|
||||
case 0x32:
|
||||
LAUNCH_TINYGEMM_KERNEL_NN(3, 32);
|
||||
break;
|
||||
case 0x33:
|
||||
LAUNCH_TINYGEMM_KERNEL_NN(3, 48);
|
||||
break;
|
||||
case 0x34:
|
||||
LAUNCH_TINYGEMM_KERNEL_NN(3, 64);
|
||||
break;
|
||||
// mb_size = 4
|
||||
case 0x42: LAUNCH_TINYGEMM_KERNEL_NN(4, 32); break;
|
||||
case 0x44: LAUNCH_TINYGEMM_KERNEL_NN(4, 64); break;
|
||||
default: TORCH_CHECK(false, "Unexpected block size, ", mb_size, "x", nb_size);
|
||||
case 0x41:
|
||||
LAUNCH_TINYGEMM_KERNEL_NN(4, 16);
|
||||
break;
|
||||
case 0x42:
|
||||
LAUNCH_TINYGEMM_KERNEL_NN(4, 32);
|
||||
break;
|
||||
case 0x43:
|
||||
LAUNCH_TINYGEMM_KERNEL_NN(4, 48);
|
||||
break;
|
||||
case 0x44:
|
||||
LAUNCH_TINYGEMM_KERNEL_NN(4, 64);
|
||||
break;
|
||||
default:
|
||||
TORCH_CHECK(false, "Unexpected block size, ", mb_size, " x ", nb_size);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t, bool has_bias>
|
||||
void tinygemm_kernel(
|
||||
const float* __restrict__ A,
|
||||
const float* __restrict__ B,
|
||||
scalar_t* __restrict__ C,
|
||||
float* __restrict__ Ctmp,
|
||||
const float* __restrict__ bias,
|
||||
int64_t M,
|
||||
int64_t N,
|
||||
int64_t K,
|
||||
int64_t lda,
|
||||
int64_t ldb,
|
||||
int64_t ldc,
|
||||
bool brg) {
|
||||
TORCH_CHECK(brg, "Expected to use fp32 brgemm for small N GEMM");
|
||||
if (brg) {
|
||||
brgemm<scalar_t, has_bias>::apply(A, B, C, Ctmp, bias, M, N, K, lda, ldb, ldc);
|
||||
return;
|
||||
}
|
||||
// TODO : add intrinsic path
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
void weight_packed_linear_kernel_impl(
|
||||
scalar_t* __restrict__ out,
|
||||
@@ -282,29 +470,20 @@ void weight_packed_linear_kernel_impl(
|
||||
int64_t K,
|
||||
int64_t mat1_strideM,
|
||||
int64_t out_strideM) {
|
||||
|
||||
constexpr int64_t BLOCK_M = block_size_m();
|
||||
constexpr int64_t BLOCK_N = block_size_n();
|
||||
const int64_t MB = div_up(M, BLOCK_M);
|
||||
const int64_t NB = div_up(N, BLOCK_N);
|
||||
|
||||
// use avx512-bf16 when a) M is small; b) dtype is bfloat16, otherwise use amx
|
||||
const bool use_brgemm = (M > 4) || (!std::is_same_v<scalar_t, at::BFloat16>);
|
||||
|
||||
// l2 cache block for n
|
||||
int64_t cache_blocks_nb = get_cache_blocks<scalar_t>(BLOCK_N, K);
|
||||
const bool use_brgemm = can_use_brgemm<scalar_t>(M);
|
||||
|
||||
// parallel on [MB, NB]
|
||||
AT_DISPATCH_BOOL(bias != nullptr, has_bias, [&] {
|
||||
parallel_2d(MB, NB, [&](int64_t begin_mb, int64_t end_mb, int64_t begin_nb, int64_t end_nb) {
|
||||
|
||||
parallel_2d(MB, NB, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) {
|
||||
// for brgemm, use float32 for accumulate
|
||||
alignas(64) float Ctmp[BLOCK_M * BLOCK_N];
|
||||
|
||||
for (int64_t nbb = begin_nb; nbb < end_nb; nbb += cache_blocks_nb) {
|
||||
for (int64_t mb = begin_mb; mb < end_mb; ++mb) {
|
||||
for (int64_t nb = nbb; nb < std::min(nbb + cache_blocks_nb, end_nb); ++nb) {
|
||||
|
||||
loop_2d<scalar_t>(mb0, mb1, nb0, nb1, BLOCK_N * K, [&](int64_t mb, int64_t nb, int64_t nb_offset) {
|
||||
int64_t mb_start = mb * BLOCK_M;
|
||||
int64_t mb_size = std::min(M - mb_start, BLOCK_M);
|
||||
int64_t nb_start = nb * BLOCK_N;
|
||||
@@ -323,7 +502,7 @@ void weight_packed_linear_kernel_impl(
|
||||
/* ldb */ nb_size,
|
||||
/* ldc */ out_strideM,
|
||||
/* brg */ use_brgemm);
|
||||
}}}
|
||||
});
|
||||
|
||||
if (use_brgemm) {
|
||||
at::native::cpublas::brgemm_release();
|
||||
@@ -332,20 +511,113 @@ void weight_packed_linear_kernel_impl(
|
||||
});
|
||||
}
|
||||
|
||||
} // anonymous namespace
|
||||
template <typename scalar_t>
|
||||
void weight_packed_linear_kernel_impl(
|
||||
scalar_t* __restrict__ out,
|
||||
const scalar_t* __restrict__ mat1,
|
||||
const float* __restrict__ mat2,
|
||||
const float* __restrict__ bias,
|
||||
const scalar_t* __restrict__ post_mul_mat,
|
||||
int64_t M,
|
||||
int64_t N,
|
||||
int64_t K,
|
||||
int64_t mat1_strideM,
|
||||
int64_t out_strideM) {
|
||||
constexpr int64_t BLOCK_M = block_size_m();
|
||||
constexpr int64_t BLOCK_N = block_size_n();
|
||||
const int64_t MB = div_up(M, BLOCK_M);
|
||||
const int64_t NB = div_up(N, BLOCK_N);
|
||||
|
||||
const bool use_brgemm = true; // TODO: add intrinsic path
|
||||
// parallel on [MB, NB]
|
||||
AT_DISPATCH_BOOL(bias != nullptr, has_bias, [&] {
|
||||
parallel_2d(MB, NB, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) {
|
||||
// for brgemm, use float32 for accumulate
|
||||
alignas(64) float Atmp[BLOCK_M * K];
|
||||
alignas(64) float Ctmp[BLOCK_M * BLOCK_N];
|
||||
|
||||
loop_2d<float>(mb0, mb1, nb0, nb1, BLOCK_N * K, [&](int64_t mb, int64_t nb, int64_t nb_offset) {
|
||||
int64_t mb_start = mb * BLOCK_M;
|
||||
int64_t mb_size = std::min(M - mb_start, BLOCK_M);
|
||||
int64_t nb_start = nb * BLOCK_N;
|
||||
int64_t nb_size = std::min(N - nb_start, BLOCK_N);
|
||||
for (int64_t m = 0; m < mb_size; ++m) {
|
||||
copy_stub<scalar_t>(Atmp + m * K, mat1 + mb_start * mat1_strideM + m * K, K);
|
||||
}
|
||||
tinygemm_kernel<scalar_t, has_bias>(
|
||||
/* A */ Atmp,
|
||||
/* B */ mat2 + nb_start * K /* nb * BLOCK_N * K */,
|
||||
/* C */ out + mb_start * out_strideM + nb_start,
|
||||
/* Ctmp*/ Ctmp,
|
||||
/* bias*/ bias + nb_start,
|
||||
/* M */ mb_size,
|
||||
/* N */ nb_size,
|
||||
/* K */ K,
|
||||
/* lda */ mat1_strideM,
|
||||
/* ldb */ nb_size,
|
||||
/* ldc */ out_strideM,
|
||||
/* brg */ use_brgemm);
|
||||
|
||||
if (post_mul_mat != nullptr) {
|
||||
for (int64_t m = 0; m < mb_size; ++m) {
|
||||
scalar_sigmoid_and_mul<scalar_t, has_bias>(
|
||||
out + mb_start * out_strideM + nb_start + m * out_strideM,
|
||||
Ctmp + m * BLOCK_N,
|
||||
bias + nb_start,
|
||||
post_mul_mat + mb_start * out_strideM + m * out_strideM,
|
||||
out_strideM);
|
||||
}
|
||||
} else {
|
||||
for (int64_t m = 0; m < mb_size; ++m) {
|
||||
if constexpr (has_bias) {
|
||||
copy_add_stub(
|
||||
out + mb_start * out_strideM + nb_start + m * out_strideM, Ctmp + m * BLOCK_N, bias + nb_start, N);
|
||||
} else {
|
||||
copy_stub(out + mb_start * out_strideM + nb_start + m * out_strideM, Ctmp + m * BLOCK_N, N);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if (use_brgemm) {
|
||||
at::native::cpublas::brgemm_release();
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
} // anonymous namespace
|
||||
|
||||
// tinygemm interface
|
||||
template <typename scalar_t>
|
||||
void tinygemm_kernel(const scalar_t* __restrict__ A, const scalar_t* __restrict__ B, scalar_t* __restrict__ C,
|
||||
float* __restrict__ Ctmp, int64_t M, int64_t N, int64_t K, int64_t lda, int64_t ldb, int64_t ldc, bool brg) {
|
||||
void tinygemm_kernel(
|
||||
const scalar_t* __restrict__ A,
|
||||
const scalar_t* __restrict__ B,
|
||||
scalar_t* __restrict__ C,
|
||||
float* __restrict__ Ctmp,
|
||||
int64_t M,
|
||||
int64_t N,
|
||||
int64_t K,
|
||||
int64_t lda,
|
||||
int64_t ldb,
|
||||
int64_t ldc,
|
||||
bool brg) {
|
||||
tinygemm_kernel<scalar_t, false>(A, B, C, Ctmp, nullptr, M, N, K, lda, ldb, ldc, brg);
|
||||
}
|
||||
|
||||
#define INSTANTIATE_TINYGEMM_TEMPLATE(TYPE) \
|
||||
template void tinygemm_kernel<TYPE>( \
|
||||
const TYPE* __restrict__ A, const TYPE* __restrict__ B, TYPE* __restrict__ C, \
|
||||
float* __restrict__ Ctmp, int64_t M, int64_t N, int64_t K, int64_t lda, \
|
||||
int64_t ldb, int64_t ldc, bool brg)
|
||||
#define INSTANTIATE_TINYGEMM_TEMPLATE(TYPE) \
|
||||
template void tinygemm_kernel<TYPE>( \
|
||||
const TYPE* __restrict__ A, \
|
||||
const TYPE* __restrict__ B, \
|
||||
TYPE* __restrict__ C, \
|
||||
float* __restrict__ Ctmp, \
|
||||
int64_t M, \
|
||||
int64_t N, \
|
||||
int64_t K, \
|
||||
int64_t lda, \
|
||||
int64_t ldb, \
|
||||
int64_t ldc, \
|
||||
bool brg)
|
||||
|
||||
INSTANTIATE_TINYGEMM_TEMPLATE(at::BFloat16);
|
||||
INSTANTIATE_TINYGEMM_TEMPLATE(at::Half);
|
||||
@@ -359,14 +631,23 @@ at::Tensor convert_weight_packed(at::Tensor& weight) {
|
||||
|
||||
const int64_t ndim = weight.ndimension();
|
||||
TORCH_CHECK(ndim == 2 || ndim == 3, "expect weight to be 2d or 3d, got ", ndim, "d tensor.");
|
||||
|
||||
if (ndim == 2 && weight.size(0) < TILE_N) {
|
||||
// for 2D weight and small OC shape, we use fma linear path, which needs transpose not pack
|
||||
return weight.to(at::kFloat).t().contiguous();
|
||||
}
|
||||
|
||||
const auto st = weight.scalar_type();
|
||||
const int64_t E = ndim == 3 ? weight.size(0) : 1;
|
||||
const int64_t OC = ndim == 3 ? weight.size(1) : weight.size(0);
|
||||
const int64_t IC = ndim == 3 ? weight.size(2) : weight.size(1);
|
||||
|
||||
// mxfp4 or int4 are packed with uint8
|
||||
const int64_t actual_IC = st == at::kByte ? IC * 2 : IC;
|
||||
|
||||
// we handle 2 TILE_N at a time.
|
||||
TORCH_CHECK(OC % TILE_N == 0, "invalid weight out features ", OC);
|
||||
TORCH_CHECK(IC % TILE_K == 0, "invalid weight input features ", IC);
|
||||
TORCH_CHECK(actual_IC % TILE_K == 0, "invalid weight input features ", actual_IC);
|
||||
|
||||
constexpr int64_t BLOCK_N = block_size_n();
|
||||
const int64_t NB = div_up(OC, BLOCK_N);
|
||||
@@ -375,12 +656,14 @@ at::Tensor convert_weight_packed(at::Tensor& weight) {
|
||||
auto packed_weight = at::empty({}, weight.options());
|
||||
const int64_t stride = OC * IC;
|
||||
|
||||
TORCH_CHECK(st == at::kBFloat16 || st == at::kHalf || st == at::kChar || st == at::kFloat8_e4m3fn,
|
||||
"expect weight to be bfloat16, float16, int8 or fp8_e4m3.");
|
||||
// Note: for `kByte` (uint8), it represents either `mxfp4` or `int4`.
|
||||
TORCH_CHECK(
|
||||
st == at::kBFloat16 || st == at::kHalf || st == at::kChar || st == at::kFloat8_e4m3fn || st == at::kByte,
|
||||
"expect weight to be bfloat16, float16, int8, fp8_e4m3 or uint8(mxfp4 or int4).");
|
||||
|
||||
CPU_DISPATCH_PACKED_TYPES(st, [&] {
|
||||
// adjust most inner dimension size
|
||||
const int packed_row_size = get_row_size<packed_t>(IC);
|
||||
const int packed_row_size = get_row_size<packed_t>(actual_IC);
|
||||
auto sizes = weight.sizes().vec();
|
||||
sizes[ndim - 1] = packed_row_size;
|
||||
packed_weight.resize_(sizes);
|
||||
@@ -399,10 +682,7 @@ at::Tensor convert_weight_packed(at::Tensor& weight) {
|
||||
int64_t n = nb * BLOCK_N;
|
||||
int64_t n_size = std::min(BLOCK_N, OC - n);
|
||||
pack_vnni<packed_t>(
|
||||
packed_data + e * OC * packed_row_size + n * packed_row_size,
|
||||
w_data + e * stride + n * IC,
|
||||
n_size,
|
||||
IC);
|
||||
packed_data + e * OC * packed_row_size + n * packed_row_size, w_data + e * stride + n * IC, n_size, IC);
|
||||
|
||||
// move to the next index
|
||||
data_index_step(e, E, nb, NB);
|
||||
@@ -412,33 +692,71 @@ at::Tensor convert_weight_packed(at::Tensor& weight) {
|
||||
return packed_weight;
|
||||
}
|
||||
|
||||
at::Tensor convert_scale_packed(at::Tensor& scale) {
|
||||
CHECK_INPUT(scale);
|
||||
|
||||
const int64_t ndim = scale.ndimension();
|
||||
TORCH_CHECK(ndim == 2 || ndim == 3, "expect scale to be 2d or 3d, got ", ndim, "d tensor.");
|
||||
const auto st = scale.scalar_type();
|
||||
const int64_t E = ndim == 3 ? scale.size(0) : 1;
|
||||
const int64_t N = ndim == 3 ? scale.size(1) : scale.size(0);
|
||||
// number of groups, e.g. K/32
|
||||
const int64_t G = ndim == 3 ? scale.size(2) : scale.size(1);
|
||||
|
||||
constexpr int64_t BLOCK_N = block_size_n();
|
||||
TORCH_CHECK(N % BLOCK_N == 0, "invalid weight out features ", N);
|
||||
const int64_t NB = N / BLOCK_N;
|
||||
|
||||
auto packed_scale = at::empty_like(scale);
|
||||
TORCH_CHECK(st == at::kByte, "expect scale to be uint8.");
|
||||
|
||||
const uint8_t* s_data = scale.data_ptr<uint8_t>();
|
||||
uint8_t* packed_data = packed_scale.data_ptr<uint8_t>();
|
||||
|
||||
// parallel on src {E, NB, BLOCK_N, G}, dst {E, NB, G, BLOCK_N}
|
||||
at::parallel_for(0, E * NB * BLOCK_N * G, 0, [&](int64_t begin, int64_t end) {
|
||||
int64_t e{0}, nb{0}, n{0}, g{0};
|
||||
data_index_init(begin, e, E, nb, NB, n, BLOCK_N, g, G);
|
||||
|
||||
for (int64_t i = begin; i < end; ++i) {
|
||||
packed_data[e * N * G + nb * G * BLOCK_N + g * BLOCK_N + n] = s_data[i];
|
||||
// move to the next index
|
||||
data_index_step(e, E, nb, NB, n, BLOCK_N, g, G);
|
||||
}
|
||||
});
|
||||
return packed_scale;
|
||||
}
|
||||
|
||||
// mat1 : [M, K]
|
||||
// mat2 : [N, K]
|
||||
// mat2 : [N, K] ([K, N] if use_fma_gemm)
|
||||
// bias : [N]
|
||||
// out : [M, N]
|
||||
//
|
||||
at::Tensor weight_packed_linear(at::Tensor& mat1, at::Tensor& mat2,
|
||||
const std::optional<at::Tensor>& bias, bool is_vnni) {
|
||||
RECORD_FUNCTION(
|
||||
"sgl-kernel::weight_packed_linear", std::vector<c10::IValue>({mat1, mat2, bias}));
|
||||
|
||||
at::Tensor
|
||||
weight_packed_linear(at::Tensor& mat1, at::Tensor& mat2, const std::optional<at::Tensor>& bias, bool is_vnni) {
|
||||
auto packed_w = is_vnni ? mat2 : convert_weight_packed(mat2);
|
||||
bool use_fma_gemm = false;
|
||||
if (packed_w.scalar_type() == at::kFloat) {
|
||||
use_fma_gemm = true;
|
||||
}
|
||||
|
||||
int64_t M = mat1.size(0);
|
||||
int64_t K = mat1.size(1);
|
||||
int64_t N = use_fma_gemm ? mat2.size(1) : mat2.size(0);
|
||||
|
||||
CHECK_LAST_DIM_CONTIGUOUS_INPUT(mat1);
|
||||
CHECK_INPUT(mat2);
|
||||
|
||||
int64_t M = mat1.size(0);
|
||||
int64_t N = mat2.size(0);
|
||||
int64_t K = mat2.size(1);
|
||||
CHECK_EQ(mat1.size(1), K);
|
||||
CHECK_DIM(2, mat1);
|
||||
CHECK_DIM(2, mat2);
|
||||
if (!use_fma_gemm) {
|
||||
CHECK_EQ(mat1.size(1), K);
|
||||
}
|
||||
|
||||
auto dispatch_type = mat1.scalar_type();
|
||||
auto out = at::empty({M, N}, mat1.options());
|
||||
|
||||
// strides
|
||||
int64_t mat1_strideM = mat1.stride(0);
|
||||
int64_t out_strideM = out.stride(0);
|
||||
int64_t mat1_strideM = mat1.stride(0);
|
||||
|
||||
const bool has_bias = bias.has_value();
|
||||
const float* bias_data = nullptr;
|
||||
@@ -447,12 +765,83 @@ at::Tensor weight_packed_linear(at::Tensor& mat1, at::Tensor& mat2,
|
||||
bias_data = bias.value().data_ptr<float>();
|
||||
}
|
||||
|
||||
AT_DISPATCH_REDUCED_FLOATING_TYPES(mat1.scalar_type(), "weight_packed_linear_kernel_impl", [&] {
|
||||
AT_DISPATCH_REDUCED_FLOATING_TYPES(dispatch_type, "weight_packed_linear_kernel_impl", [&] {
|
||||
if (use_fma_gemm) {
|
||||
weight_packed_linear_kernel_impl<scalar_t>(
|
||||
out.data_ptr<scalar_t>(),
|
||||
mat1.data_ptr<scalar_t>(),
|
||||
packed_w.data_ptr<float>(),
|
||||
bias_data,
|
||||
nullptr,
|
||||
M,
|
||||
N,
|
||||
K,
|
||||
mat1_strideM,
|
||||
out_strideM);
|
||||
} else {
|
||||
weight_packed_linear_kernel_impl<scalar_t>(
|
||||
out.data_ptr<scalar_t>(),
|
||||
mat1.data_ptr<scalar_t>(),
|
||||
packed_w.data_ptr<scalar_t>(),
|
||||
bias_data,
|
||||
M,
|
||||
N,
|
||||
K,
|
||||
mat1_strideM,
|
||||
out_strideM);
|
||||
}
|
||||
});
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
// mat1 : [M, K]
|
||||
// mat2 : [K, 1]
|
||||
// post_mul_mat : [M, K]
|
||||
// bias : [N]
|
||||
// out : [M, N]
|
||||
//
|
||||
at::Tensor fused_linear_sigmoid_mul(
|
||||
at::Tensor& mat1,
|
||||
at::Tensor& mat2,
|
||||
const std::optional<at::Tensor>& bias,
|
||||
bool is_vnni,
|
||||
const at::Tensor& post_mul_mat) {
|
||||
auto packed_w = is_vnni ? mat2 : convert_weight_packed(mat2);
|
||||
TORCH_CHECK(packed_w.scalar_type() == at::kFloat, "fused_linear_sigmoid_mul requires packed float weight")
|
||||
|
||||
int64_t M = mat1.size(0);
|
||||
int64_t K = mat1.size(1);
|
||||
int64_t N = mat2.size(1);
|
||||
|
||||
CHECK_LAST_DIM_CONTIGUOUS_INPUT(mat1);
|
||||
CHECK_INPUT(mat2);
|
||||
CHECK_DIM(2, mat1);
|
||||
CHECK_DIM(2, mat2);
|
||||
|
||||
int64_t out_strideM = post_mul_mat.size(1);
|
||||
int64_t mat1_strideM = mat1.stride(0);
|
||||
auto dispatch_type = mat1.scalar_type();
|
||||
auto out = at::empty({M, out_strideM}, mat1.options());
|
||||
|
||||
TORCH_CHECK(
|
||||
N == 1 && out_strideM % 32 == 0,
|
||||
"post_mul_mat tensor size(1) should be 32 dividable, and the mat2 OC=1 (Mx1 as linear output shape)")
|
||||
|
||||
const bool has_bias = bias.has_value();
|
||||
const float* bias_data = nullptr;
|
||||
if (has_bias) {
|
||||
CHECK_EQ(bias.value().size(0), N);
|
||||
bias_data = bias.value().data_ptr<float>();
|
||||
}
|
||||
|
||||
AT_DISPATCH_REDUCED_FLOATING_TYPES(dispatch_type, "fused_linear_sigmoid_mul", [&] {
|
||||
weight_packed_linear_kernel_impl<scalar_t>(
|
||||
out.data_ptr<scalar_t>(),
|
||||
mat1.data_ptr<scalar_t>(),
|
||||
packed_w.data_ptr<scalar_t>(),
|
||||
packed_w.data_ptr<float>(),
|
||||
bias_data,
|
||||
post_mul_mat.data_ptr<scalar_t>(),
|
||||
M,
|
||||
N,
|
||||
K,
|
||||
|
||||
+122
-87
@@ -1,29 +1,55 @@
|
||||
#pragma once
|
||||
|
||||
#include <ATen/native/CPUBlas.h>
|
||||
// Adapted from
|
||||
// https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu
|
||||
|
||||
// clang-format off
|
||||
|
||||
#pragma once
|
||||
#include <ATen/native/CPUBlas.h>
|
||||
|
||||
#include "common.h"
|
||||
|
||||
// amx-bf16
|
||||
#define TILE_M 16
|
||||
#define TILE_N 16
|
||||
#define TILE_K 32
|
||||
|
||||
// block size for AMX gemm
|
||||
constexpr int block_size_m() { return 2 * TILE_M; }
|
||||
constexpr int block_size_n() { return 2 * TILE_N; }
|
||||
constexpr int block_size_m() {
|
||||
return 2 * TILE_M;
|
||||
}
|
||||
constexpr int block_size_n() {
|
||||
return 2 * TILE_N;
|
||||
}
|
||||
|
||||
// define threshold using brgemm (intel AMX)
|
||||
template <typename T> inline bool can_use_brgemm(int M);
|
||||
template <> inline bool can_use_brgemm<at::BFloat16>(int M) { return M > 4; }
|
||||
template <> inline bool can_use_brgemm<at::Half>(int M) { return true; }
|
||||
template <> inline bool can_use_brgemm<int8_t>(int M) { return M > 4; }
|
||||
template <> inline bool can_use_brgemm<uint8_t>(int M) { return M > 4; }
|
||||
template <> inline bool can_use_brgemm<at::Float8_e4m3fn>(int M) { return M > 4; }
|
||||
template <> inline bool can_use_brgemm<at::quint4x2>(int M) { return M > 4; }
|
||||
template <typename T>
|
||||
inline bool can_use_brgemm(int M);
|
||||
template <>
|
||||
inline bool can_use_brgemm<at::BFloat16>(int M) {
|
||||
return M > 4;
|
||||
}
|
||||
template <>
|
||||
inline bool can_use_brgemm<at::Half>(int M) {
|
||||
return true;
|
||||
}
|
||||
// this requires PyTorch 2.7 or above
|
||||
template <>
|
||||
inline bool can_use_brgemm<int8_t>(int M) {
|
||||
return M > 4;
|
||||
}
|
||||
|
||||
template <>
|
||||
inline bool can_use_brgemm<uint8_t>(int M) {
|
||||
return M > 4;
|
||||
}
|
||||
|
||||
template <>
|
||||
inline bool can_use_brgemm<at::Float8_e4m3fn>(int M) {
|
||||
return M > 4;
|
||||
}
|
||||
|
||||
// work around compiler internal error
|
||||
#define BLOCK_K 128 // 4 * TILE_K
|
||||
#define BLOCK_K 128 // 4 * TILE_K
|
||||
|
||||
// adjust leading dimension size for K
|
||||
template <typename T>
|
||||
@@ -36,18 +62,44 @@ inline int64_t get_row_size<int8_t>(int64_t K) {
|
||||
return K + sizeof(int32_t);
|
||||
}
|
||||
|
||||
// uint8: mxfp4 or int4
|
||||
template <>
|
||||
inline int64_t get_row_size<uint8_t>(int64_t K) {
|
||||
return K >> 1;
|
||||
}
|
||||
|
||||
inline int64_t get_row_size(int64_t K, bool use_int8_w8a8) {
|
||||
return use_int8_w8a8 ? K + sizeof(int32_t) : K;
|
||||
}
|
||||
|
||||
enum class CPUQuantMethod : int64_t { BF16 = 0, INT8_W8A8 = 1, FP8_W8A16 = 2, INT4_W4A8 = 3 };
|
||||
|
||||
constexpr bool operator==(CPUQuantMethod a, int64_t b) {
|
||||
return static_cast<int64_t>(a) == b;
|
||||
}
|
||||
|
||||
constexpr bool operator==(int64_t a, CPUQuantMethod b) {
|
||||
return a == static_cast<int64_t>(b);
|
||||
}
|
||||
|
||||
enum class CPUQuantAlgo : int64_t { AWQ = 0, GPTQ = 1 };
|
||||
|
||||
constexpr bool operator==(CPUQuantAlgo a, int64_t b) {
|
||||
return static_cast<int64_t>(a) == b;
|
||||
}
|
||||
|
||||
constexpr bool operator==(int64_t a, CPUQuantAlgo b) {
|
||||
return a == static_cast<int64_t>(b);
|
||||
}
|
||||
|
||||
inline int64_t get_4bit_block_k_size(int64_t group_size) {
|
||||
return group_size > 128 ? 128 : group_size;
|
||||
}
|
||||
|
||||
// pack weight into vnni format
|
||||
// pack weight to vnni format
|
||||
at::Tensor convert_weight_packed(at::Tensor& weight);
|
||||
|
||||
// pack weight to vnni format for int4 (adapted from sglang)
|
||||
// pack weight to vnni format for int4
|
||||
std::tuple<at::Tensor, at::Tensor, at::Tensor>
|
||||
convert_weight_packed_scale_zp(at::Tensor qweight, at::Tensor qzeros, at::Tensor scales);
|
||||
|
||||
@@ -105,35 +157,6 @@ void fused_experts_fp8_kernel_impl(
|
||||
int64_t topk,
|
||||
int64_t num_tokens_post_pad);
|
||||
|
||||
// moe implementations for int4 w4a16
|
||||
template <typename scalar_t>
|
||||
void fused_experts_int4_w4a16_kernel_impl(
|
||||
scalar_t* __restrict__ output,
|
||||
scalar_t* __restrict__ ic0,
|
||||
scalar_t* __restrict__ ic1,
|
||||
scalar_t* __restrict__ ic2,
|
||||
scalar_t* __restrict__ A_tmp,
|
||||
scalar_t* __restrict__ B_tmp,
|
||||
float* __restrict__ C_tmp,
|
||||
const scalar_t* __restrict__ input,
|
||||
const at::quint4x2* __restrict__ packed_w1,
|
||||
const at::quint4x2* __restrict__ packed_w2,
|
||||
const uint8_t* __restrict__ w1z,
|
||||
const uint8_t* __restrict__ w2z,
|
||||
const scalar_t* __restrict__ w1s,
|
||||
const scalar_t* __restrict__ w2s,
|
||||
int group_size,
|
||||
const float* __restrict__ topk_weights,
|
||||
const int32_t* __restrict__ sorted_ids,
|
||||
const int32_t* __restrict__ expert_ids,
|
||||
const int32_t* __restrict__ offsets,
|
||||
int64_t M,
|
||||
int64_t N,
|
||||
int64_t K,
|
||||
int64_t E,
|
||||
int64_t topk,
|
||||
int64_t num_tokens_post_pad);
|
||||
|
||||
// shared expert implementation for int8 w8a8
|
||||
template <typename scalar_t>
|
||||
void shared_expert_int8_kernel_impl(
|
||||
@@ -153,6 +176,37 @@ void shared_expert_int8_kernel_impl(
|
||||
int64_t N,
|
||||
int64_t K);
|
||||
|
||||
template <typename scalar_t>
|
||||
void fused_experts_int4_w4a8_kernel_impl(
|
||||
scalar_t* __restrict__ output,
|
||||
scalar_t* __restrict__ ic0,
|
||||
scalar_t* __restrict__ ic1,
|
||||
scalar_t* __restrict__ ic2,
|
||||
uint8_t* __restrict__ A_tmp,
|
||||
uint8_t* __restrict__ Aq_tmp,
|
||||
float* __restrict__ As_tmp,
|
||||
int32_t* __restrict__ Azp_tmp,
|
||||
float* __restrict__ C_tmp,
|
||||
int8_t* __restrict__ dqB_tmp,
|
||||
const scalar_t* __restrict__ input,
|
||||
const uint8_t* __restrict__ packed_w1,
|
||||
const uint8_t* __restrict__ packed_w2,
|
||||
const int8_t* __restrict__ w1z,
|
||||
const int8_t* __restrict__ w2z,
|
||||
const float* __restrict__ w1s,
|
||||
const float* __restrict__ w2s,
|
||||
int group_size,
|
||||
const float* __restrict__ topk_weights,
|
||||
const int32_t* __restrict__ sorted_ids,
|
||||
const int32_t* __restrict__ expert_ids,
|
||||
const int32_t* __restrict__ offsets,
|
||||
int64_t M,
|
||||
int64_t N,
|
||||
int64_t K,
|
||||
int64_t E,
|
||||
int64_t topk,
|
||||
int64_t num_tokens_post_pad);
|
||||
|
||||
template <typename scalar_t>
|
||||
void shared_expert_fp8_kernel_impl(
|
||||
scalar_t* __restrict__ output,
|
||||
@@ -204,6 +258,7 @@ void tinygemm_kernel(
|
||||
int64_t ldc,
|
||||
bool brg);
|
||||
|
||||
// block quantization
|
||||
template <typename scalar_t>
|
||||
void tinygemm_kernel(
|
||||
const scalar_t* __restrict__ A,
|
||||
@@ -219,33 +274,26 @@ void tinygemm_kernel(
|
||||
int64_t ldb,
|
||||
int64_t ldc,
|
||||
bool brg,
|
||||
int64_t block_size_K);
|
||||
int64_t block_size_K,
|
||||
bool do_unpack = true);
|
||||
|
||||
// per tensor quantization
|
||||
template <typename scalar_t>
|
||||
void tinygemm_kernel(
|
||||
const scalar_t* __restrict__ A,
|
||||
const at::quint4x2* __restrict__ B,
|
||||
const at::Float8_e4m3fn* __restrict__ B,
|
||||
scalar_t* __restrict__ C,
|
||||
const uint8_t* __restrict__ Bz,
|
||||
const scalar_t* __restrict__ Bs,
|
||||
scalar_t* __restrict__ Btmp,
|
||||
float* __restrict__ Ctmp,
|
||||
float scale,
|
||||
int64_t M,
|
||||
int64_t N,
|
||||
int64_t K,
|
||||
int group_size,
|
||||
int64_t lda,
|
||||
int64_t ldb,
|
||||
int64_t ldc,
|
||||
int64_t strideBz,
|
||||
int64_t strideBs,
|
||||
bool brg);
|
||||
|
||||
// int4 scaled GEMM (adapted from sglang)
|
||||
at::Tensor int4_scaled_mm_cpu(
|
||||
at::Tensor& x, at::Tensor& w, at::Tensor& w_zeros, at::Tensor& w_scales, std::optional<at::Tensor> bias);
|
||||
|
||||
// int4 tinygemm kernel interface(adapted from sglang)
|
||||
template <typename scalar_t>
|
||||
void tinygemm_kernel(
|
||||
scalar_t* C,
|
||||
@@ -266,34 +314,21 @@ void tinygemm_kernel(
|
||||
bool store_out,
|
||||
bool use_brgemm);
|
||||
|
||||
// TODO: debug print, remove me later
|
||||
inline void print_16x32i(const __m512i x) {
|
||||
int32_t a[16];
|
||||
_mm512_storeu_si512((__m512i *)a, x);
|
||||
|
||||
for (int i = 0; i < 16; i++){
|
||||
std::cout << a[i] << " ";
|
||||
}
|
||||
std::cout << std::endl;
|
||||
}
|
||||
|
||||
inline void print_16x32(const __m512 x) {
|
||||
float a[16];
|
||||
_mm512_storeu_ps((__m512 *)a, x);
|
||||
|
||||
for (int i = 0; i < 16; i++){
|
||||
std::cout << a[i] << " ";
|
||||
}
|
||||
std::cout << std::endl;
|
||||
}
|
||||
|
||||
|
||||
inline void print_32x8u(const __m256i x) {
|
||||
uint8_t a[32];
|
||||
_mm256_storeu_si256((__m256i *)a, x);
|
||||
|
||||
for (int i = 0; i < 32; ++i) {
|
||||
std::cout << int32_t(a[i]) << " ";
|
||||
}
|
||||
std::cout << std::endl;
|
||||
}
|
||||
// mxfp4
|
||||
template <typename scalar_t>
|
||||
void tinygemm_kernel(
|
||||
const scalar_t* __restrict__ A,
|
||||
const uint8_t* __restrict__ B,
|
||||
scalar_t* __restrict__ C,
|
||||
scalar_t* __restrict__ Btmp,
|
||||
float* __restrict__ Ctmp,
|
||||
const uint8_t* __restrict__ scale,
|
||||
int64_t M,
|
||||
int64_t N,
|
||||
int64_t K,
|
||||
int64_t lda,
|
||||
int64_t ldb,
|
||||
int64_t ldc,
|
||||
bool brg,
|
||||
int64_t block_size_K,
|
||||
bool do_unpack = true);
|
||||
|
||||
+774
-144
File diff suppressed because it is too large
Load Diff
+368
-229
@@ -1,10 +1,10 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// Adapted from sgl-project/sglang
|
||||
// https://github.com/sgl-project/sglang/pull/8226
|
||||
// Adapted from
|
||||
// https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu
|
||||
|
||||
#include <ATen/ATen.h>
|
||||
// clang-format off
|
||||
|
||||
#include <torch/all.h>
|
||||
|
||||
#include "common.h"
|
||||
#include "gemm.h"
|
||||
#include "vec.h"
|
||||
|
||||
@@ -24,19 +24,22 @@ struct ActDtype<false> {
|
||||
using type = uint8_t;
|
||||
};
|
||||
|
||||
#if defined(CPU_CAPABILITY_AVX512)
|
||||
struct alignas(32) m256i_wrapper {
|
||||
__m256i data;
|
||||
};
|
||||
|
||||
#if defined(CPU_CAPABILITY_AVX512)
|
||||
inline std::array<m256i_wrapper, 2> load_zps_4vnni(
|
||||
const int8_t* __restrict__ zps) {
|
||||
__m256i vzps_low = _mm256_set1_epi64x(*reinterpret_cast<const int64_t*>(zps));
|
||||
__m256i vzps_high =
|
||||
_mm256_set1_epi64x(*reinterpret_cast<const int64_t*>(zps + 8));
|
||||
inline std::array<m256i_wrapper, 2> load_zps_4vnni(const int8_t* __restrict__ zps) {
|
||||
// broadcast 01234567 to
|
||||
// 01234567012345670123456701234567
|
||||
__m256i vzps_low = _mm256_set1_epi64x(*reinterpret_cast<const long*>(zps));
|
||||
__m256i vzps_high = _mm256_set1_epi64x(*reinterpret_cast<const long*>(zps + 8));
|
||||
// shuffle from
|
||||
// 01234567012345670123456701234567
|
||||
// to
|
||||
// 00001111222233334444555566667777
|
||||
__m256i shuffle_mask =
|
||||
_mm256_set_epi8(7, 7, 7, 7, 6, 6, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 3, 3, 3,
|
||||
3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);
|
||||
_mm256_set_epi8(7, 7, 7, 7, 6, 6, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);
|
||||
vzps_low = _mm256_shuffle_epi8(vzps_low, shuffle_mask);
|
||||
vzps_high = _mm256_shuffle_epi8(vzps_high, shuffle_mask);
|
||||
m256i_wrapper vzps_low_wp, vzps_high_wp;
|
||||
@@ -45,8 +48,7 @@ inline std::array<m256i_wrapper, 2> load_zps_4vnni(
|
||||
return {vzps_low_wp, vzps_high_wp};
|
||||
}
|
||||
|
||||
inline std::array<m256i_wrapper, 2> load_uint4_as_int8(
|
||||
const uint8_t* __restrict__ qB) {
|
||||
inline std::array<m256i_wrapper, 2> load_uint4_as_int8(const uint8_t* __restrict__ qB) {
|
||||
__m256i packed = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(qB));
|
||||
const __m256i low_mask = _mm256_set1_epi8(0x0f);
|
||||
__m256i high = _mm256_srli_epi16(packed, 4);
|
||||
@@ -58,37 +60,42 @@ inline std::array<m256i_wrapper, 2> load_uint4_as_int8(
|
||||
return {low_wp, high_wp};
|
||||
}
|
||||
|
||||
template <int N, int ldb>
|
||||
void _dequant_weight_zp_only(const uint8_t* __restrict__ B, int8_t* dqB,
|
||||
const int8_t* __restrict__ qzeros, int64_t K) {
|
||||
#pragma GCC unroll 2
|
||||
template <int64_t N, int64_t ldb>
|
||||
void _dequant_weight_zp_only(const uint8_t* __restrict__ B, int8_t* dqB, const int8_t* __restrict__ qzeros, int64_t K) {
|
||||
// unpack weight int8 -> two int4
|
||||
// subtract zero point
|
||||
// B shape = [K, ldb] = [K, N / 2], actual shape = [K / 4, N / 2, 4]
|
||||
// dqB shape = [K, N], actual shape = [K / 4, N, 4]
|
||||
#pragma GCC unroll 2
|
||||
for (int n = 0; n < N; n += 16) {
|
||||
auto [zps_low_wp, zps_high_wp] = load_zps_4vnni(&qzeros[n]);
|
||||
auto zps_low = zps_low_wp.data;
|
||||
auto zps_high = zps_high_wp.data;
|
||||
for (int k = 0; k < K; k += 4) {
|
||||
auto [vb_low_wp, vb_high_wp] =
|
||||
load_uint4_as_int8(B + ldb * k + n / 2 * 4);
|
||||
auto [vb_low_wp, vb_high_wp] = load_uint4_as_int8(B + ldb * k + n / 2 * 4);
|
||||
auto vb_low = vb_low_wp.data;
|
||||
auto vb_high = vb_high_wp.data;
|
||||
vb_high = _mm256_sub_epi8(vb_high, zps_high);
|
||||
vb_low = _mm256_sub_epi8(vb_low, zps_low);
|
||||
_mm256_storeu_si256(reinterpret_cast<__m256i_u*>(dqB + N * k + n * 4),
|
||||
vb_low);
|
||||
_mm256_storeu_si256(
|
||||
reinterpret_cast<__m256i_u*>(dqB + N * k + (n + 8) * 4), vb_high);
|
||||
// store vb to B
|
||||
_mm256_storeu_si256(reinterpret_cast<__m256i_u*>(dqB + N * k + n * 4), vb_low);
|
||||
_mm256_storeu_si256(reinterpret_cast<__m256i_u*>(dqB + N * k + (n + 8) * 4), vb_high);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <bool sym_quant_act, int N, bool accum>
|
||||
void _dequant_and_store(float* __restrict__ output,
|
||||
const int32_t* __restrict__ input,
|
||||
const float* __restrict__ scale_a,
|
||||
const int32_t* __restrict__ zp_a,
|
||||
const float* __restrict__ scale_b,
|
||||
const int32_t* __restrict__ comp_b, int M, int ldi,
|
||||
int ldo, int ldsa = 1) {
|
||||
template <bool accum, int64_t N, bool sym_quant_act>
|
||||
void _dequant_and_store(
|
||||
float* __restrict__ output,
|
||||
const int32_t* __restrict__ input,
|
||||
const float* __restrict__ scale_a,
|
||||
const int32_t* __restrict__ zp_a,
|
||||
const float* __restrict__ scale_b,
|
||||
const int32_t* __restrict__ comp_b,
|
||||
int M,
|
||||
int ldi,
|
||||
int ldo,
|
||||
int ldsa = 1) {
|
||||
for (int m = 0; m < M; ++m) {
|
||||
float a_scale = *(scale_a + m * ldsa);
|
||||
__m512 va_scale = _mm512_set1_ps(a_scale);
|
||||
@@ -99,7 +106,7 @@ void _dequant_and_store(float* __restrict__ output,
|
||||
va_zp = _mm512_set1_epi32(a_zp);
|
||||
}
|
||||
int n = 0;
|
||||
#pragma GCC unroll 2
|
||||
#pragma GCC unroll 2
|
||||
for (; n < N; n += 16) {
|
||||
__m512i vc = _mm512_loadu_si512(input + m * ldi + n);
|
||||
if constexpr (!sym_quant_act) {
|
||||
@@ -122,8 +129,7 @@ void _dequant_and_store(float* __restrict__ output,
|
||||
if constexpr (sym_quant_act) {
|
||||
dq_val = (float)input[m * ldi + n] * a_scale * scale_b[n];
|
||||
} else {
|
||||
dq_val = (float)(input[m * ldi + n] - a_zp * comp_b[n]) * a_scale *
|
||||
scale_b[n];
|
||||
dq_val = (float)(input[m * ldi + n] - a_zp * comp_b[n]) * a_scale * scale_b[n];
|
||||
}
|
||||
if constexpr (accum) {
|
||||
output[m * ldo + n] += dq_val;
|
||||
@@ -135,9 +141,10 @@ void _dequant_and_store(float* __restrict__ output,
|
||||
}
|
||||
|
||||
#else
|
||||
template <int N, int ldb>
|
||||
void _dequant_weight_zp_only(const uint8_t* B, int8_t* dqB,
|
||||
const int8_t* qzeros, int64_t K) {
|
||||
template <int64_t N, int64_t ldb>
|
||||
void _dequant_weight_zp_only(const uint8_t* B, int8_t* dqB, const int8_t* qzeros, int64_t K) {
|
||||
// B shape = [K, N / 2]
|
||||
// dqB shape = [K, N]
|
||||
for (int k = 0; k < K; ++k) {
|
||||
for (int n = 0; n < N / 2; ++n) {
|
||||
int32_t b = (int32_t)B[k * ldb + n];
|
||||
@@ -158,20 +165,31 @@ inline __m512i combine_m256i(std::array<m256i_wrapper, 2> two_256) {
|
||||
return combine_m256i(two_256[0].data, two_256[1].data);
|
||||
}
|
||||
|
||||
// negate elements in a according to b's sign
|
||||
static inline __m512i _mm512_sign_epi8(__m512i a, __m512i b) {
|
||||
__m512i zero = _mm512_setzero_si512();
|
||||
__mmask64 blt0 = _mm512_movepi8_mask(b);
|
||||
return _mm512_mask_sub_epi8(a, blt0, zero, a);
|
||||
}
|
||||
|
||||
template <bool sym_quant_act, int M, int N, int ldb>
|
||||
void _dequant_gemm_accum_small_M(float* __restrict__ C, const uint8_t* A,
|
||||
const float* scales_a, const int32_t* qzeros_a,
|
||||
const uint8_t* B, const float* scales_b,
|
||||
const int8_t* qzeros_b, int64_t K, int64_t lda,
|
||||
int64_t ldc) {
|
||||
template <int64_t M, int64_t N, int64_t ldb, bool sym_quant_act>
|
||||
void _dequant_gemm_accum_small_M(
|
||||
float* __restrict__ C,
|
||||
const uint8_t* A,
|
||||
const float* scales_a,
|
||||
const int32_t* qzeros_a,
|
||||
const uint8_t* B,
|
||||
const float* scales_b,
|
||||
const int8_t* qzeros_b,
|
||||
int64_t K,
|
||||
int64_t lda,
|
||||
int64_t ldc) {
|
||||
// if sym_quant_act is true, A pointer type is passed in as uint8_t* but actually int8_t*.
|
||||
|
||||
constexpr int COLS = N / 16;
|
||||
__m512i ones = _mm512_set1_epi8(1);
|
||||
// Computing compensation is faster than loading it for small M
|
||||
// because it's memory bound.
|
||||
__m512i ones = _mm512_set1_epi8(1); // used for computing compensation
|
||||
__m512i va;
|
||||
__m512i vb[COLS];
|
||||
__m512i vc[M * COLS];
|
||||
@@ -179,6 +197,7 @@ void _dequant_gemm_accum_small_M(float* __restrict__ C, const uint8_t* A,
|
||||
__m512i vzps[COLS];
|
||||
__m512i vcompensate[COLS];
|
||||
|
||||
// Load scales and zps
|
||||
Unroll<COLS>{}([&](auto i) {
|
||||
vscales[i] = _mm512_loadu_ps(scales_b + i * 16);
|
||||
vzps[i] = combine_m256i(load_zps_4vnni(qzeros_b + i * 16));
|
||||
@@ -214,25 +233,25 @@ void _dequant_gemm_accum_small_M(float* __restrict__ C, const uint8_t* A,
|
||||
}
|
||||
};
|
||||
|
||||
// Accumulate along k
|
||||
constexpr const int unroll = 4;
|
||||
int k = 0;
|
||||
for (; k < K / 4 / unroll; k++) {
|
||||
Unroll<unroll>{}(
|
||||
[&](auto i) { Unroll<M * COLS>{}(compute, 4 * (k * unroll + i)); });
|
||||
Unroll<unroll>{}([&](auto i) { Unroll<M * COLS>{}(compute, 4 * (k * unroll + i)); });
|
||||
}
|
||||
k *= 4 * unroll;
|
||||
for (; k < K; k += 4) {
|
||||
Unroll<M * COLS>{}(compute, k);
|
||||
}
|
||||
|
||||
// Store to C
|
||||
auto store = [&](auto i) {
|
||||
constexpr const int row = i / COLS;
|
||||
constexpr const int col = i % COLS;
|
||||
// compute (qC - compensate * zp_a) * scale_a * scale_b
|
||||
__m512 vc_float;
|
||||
if constexpr (!sym_quant_act) {
|
||||
vc[i] = _mm512_sub_epi32(
|
||||
vc[i], _mm512_mullo_epi32(vcompensate[col],
|
||||
_mm512_set1_epi32(*(qzeros_a + row))));
|
||||
vc[i] = _mm512_sub_epi32(vc[i], _mm512_mullo_epi32(vcompensate[col], _mm512_set1_epi32(*(qzeros_a + row))));
|
||||
}
|
||||
vc_float = _mm512_cvtepi32_ps(vc[i]);
|
||||
vc_float = _mm512_mul_ps(vc_float, _mm512_set1_ps(*(scales_a + row)));
|
||||
@@ -245,17 +264,28 @@ void _dequant_gemm_accum_small_M(float* __restrict__ C, const uint8_t* A,
|
||||
Unroll<M * COLS>{}(store);
|
||||
}
|
||||
|
||||
#define CALL_DEQUANT_GEMM_ACCUM_SMALL_M(M) \
|
||||
_dequant_gemm_accum_small_M<sym_quant_act, M, N, ldb>( \
|
||||
C, A, scales_a, qzeros_a, B, scales_b, qzeros_b, K, lda, ldc);
|
||||
#define CALL_DEQUANT_GEMM_ACCUM_SMALL_M(M) \
|
||||
_dequant_gemm_accum_small_M<M, N, ldb, sym_quant_act>(C, A, scales_a, qzeros_a, B, scales_b, qzeros_b, K, lda, ldc);
|
||||
#endif
|
||||
|
||||
template <bool sym_quant_act, int N, int ldb>
|
||||
void _dequant_gemm_accum(float* C, const uint8_t* A, const float* scales_a,
|
||||
const int32_t* qzeros_a, const uint8_t* B,
|
||||
const float* scales_b, const int8_t* qzeros_b,
|
||||
const int32_t* compensation, int8_t* dqB, int64_t M,
|
||||
int64_t K, int64_t lda, int64_t ldc, bool use_brgemm) {
|
||||
template <int64_t N, int64_t ldb, bool sym_quant_act>
|
||||
void _dequant_gemm_accum(
|
||||
float* C,
|
||||
const uint8_t* A,
|
||||
const float* scales_a,
|
||||
const int32_t* qzeros_a,
|
||||
const uint8_t* B,
|
||||
const float* scales_b,
|
||||
const int8_t* qzeros_b,
|
||||
const int32_t* compensation,
|
||||
int8_t* dqB,
|
||||
int64_t M,
|
||||
int64_t K,
|
||||
int64_t lda,
|
||||
int64_t ldc,
|
||||
bool use_brgemm) {
|
||||
// Compute GEMM int8 * int8 -> int32
|
||||
// dequant result to float by applying scales/qzeros
|
||||
#if defined(CPU_CAPABILITY_AVX512)
|
||||
if (!use_brgemm) {
|
||||
switch (M) {
|
||||
@@ -282,14 +312,12 @@ void _dequant_gemm_accum(float* C, const uint8_t* A, const float* scales_a,
|
||||
Tin* A_ptr = (Tin*)A;
|
||||
if (use_brgemm) {
|
||||
int32_t C_i32[M * N];
|
||||
at::native::cpublas::brgemm(M, N, K, lda, N /*ldb*/, N /*ldc*/,
|
||||
false /* add_C */, A_ptr, dqB, C_i32,
|
||||
true /* is_vnni */);
|
||||
at::native::cpublas::brgemm(
|
||||
M, N, K, lda, N /*ldb*/, N /*ldc*/, false /* add_C */, A_ptr, dqB, C_i32, true /* is_vnni */);
|
||||
_mm_prefetch(B + N * K / 2, _MM_HINT_T0);
|
||||
_mm_prefetch(A + K, _MM_HINT_T0);
|
||||
_dequant_and_store<sym_quant_act, N, true>(C, C_i32, scales_a, qzeros_a,
|
||||
scales_b, compensation, M,
|
||||
N /*ldi*/, ldc, 1 /*ldsa*/);
|
||||
_dequant_and_store<true, N, sym_quant_act>(
|
||||
C, C_i32, scales_a, qzeros_a, scales_b, compensation, M, N /*ldi*/, ldc, 1 /*ldsa*/);
|
||||
} else
|
||||
#endif
|
||||
{
|
||||
@@ -297,13 +325,13 @@ void _dequant_gemm_accum(float* C, const uint8_t* A, const float* scales_a,
|
||||
}
|
||||
}
|
||||
|
||||
template <int N>
|
||||
template <int64_t N>
|
||||
inline void copy_bias(const float* bias_ptr, float* y_buf, int64_t m) {
|
||||
if (bias_ptr) {
|
||||
for (int i = 0; i < m; ++i) {
|
||||
int j = 0;
|
||||
#if defined(CPU_CAPABILITY_AVX512)
|
||||
#pragma GCC unroll 2
|
||||
#pragma GCC unroll 2
|
||||
for (; j < N; j += 16) {
|
||||
__m512 bias_vec = _mm512_loadu_ps(bias_ptr + j);
|
||||
_mm512_storeu_ps(y_buf + i * N + j, bias_vec);
|
||||
@@ -313,11 +341,11 @@ inline void copy_bias(const float* bias_ptr, float* y_buf, int64_t m) {
|
||||
y_buf[i * N + j] = bias_ptr[j];
|
||||
}
|
||||
}
|
||||
} else {
|
||||
} else { // initialize to zero
|
||||
for (int i = 0; i < m; ++i) {
|
||||
int j = 0;
|
||||
#if defined(CPU_CAPABILITY_AVX512)
|
||||
#pragma GCC unroll 2
|
||||
#pragma GCC unroll 2
|
||||
for (; j < N; j += 16) {
|
||||
__m512 zero_vec = _mm512_setzero_ps();
|
||||
_mm512_storeu_ps(y_buf + i * N + j, zero_vec);
|
||||
@@ -330,14 +358,13 @@ inline void copy_bias(const float* bias_ptr, float* y_buf, int64_t m) {
|
||||
}
|
||||
}
|
||||
|
||||
template <int N, typename out_dtype>
|
||||
inline void store_out(const float* y_buf, out_dtype* c_ptr, int64_t m,
|
||||
int64_t lda) {
|
||||
template <typename out_dtype, int64_t N>
|
||||
inline void store_out(const float* y_buf, out_dtype* c_ptr, int64_t m, /* int64_t n, */ int64_t lda) {
|
||||
for (int i = 0; i < m; ++i) {
|
||||
int j = 0;
|
||||
if constexpr (std::is_same<out_dtype, float>::value) {
|
||||
#if defined(CPU_CAPABILITY_AVX512)
|
||||
#pragma GCC unroll 2
|
||||
#pragma GCC unroll 2
|
||||
for (; j < N; j += 16) {
|
||||
__m512 y_vec = _mm512_loadu_ps(y_buf + i * N + j);
|
||||
_mm512_storeu_ps(c_ptr + i * lda + j, y_vec);
|
||||
@@ -348,12 +375,11 @@ inline void store_out(const float* y_buf, out_dtype* c_ptr, int64_t m,
|
||||
}
|
||||
} else if constexpr (std::is_same<out_dtype, at::BFloat16>::value) {
|
||||
#if defined(CPU_CAPABILITY_AVX512)
|
||||
#pragma GCC unroll 2
|
||||
#pragma GCC unroll 2
|
||||
for (; j < N; j += 16) {
|
||||
__m512 y_vec = _mm512_loadu_ps(y_buf + i * N + j);
|
||||
__m256i y_bf16_vec = at::vec::cvtfp32_bf16(y_vec);
|
||||
_mm256_storeu_si256(reinterpret_cast<__m256i*>(c_ptr + i * lda + j),
|
||||
y_bf16_vec);
|
||||
_mm256_storeu_si256(reinterpret_cast<__m256i*>(c_ptr + i * lda + j), y_bf16_vec);
|
||||
}
|
||||
#endif
|
||||
for (; j < N; ++j) {
|
||||
@@ -361,12 +387,11 @@ inline void store_out(const float* y_buf, out_dtype* c_ptr, int64_t m,
|
||||
}
|
||||
} else if constexpr (std::is_same<out_dtype, at::Half>::value) {
|
||||
#if defined(CPU_CAPABILITY_AVX512)
|
||||
#pragma GCC unroll 2
|
||||
#pragma GCC unroll 2
|
||||
for (; j < N; j += 16) {
|
||||
__m512 y_vec = _mm512_loadu_ps(y_buf + i * N + j);
|
||||
__m256i y_fp16_vec = at::vec::cvtfp32_fp16(y_vec);
|
||||
_mm256_storeu_si256(reinterpret_cast<__m256i*>(c_ptr + i * lda + j),
|
||||
y_fp16_vec);
|
||||
_mm256_storeu_si256(reinterpret_cast<__m256i*>(c_ptr + i * lda + j), y_fp16_vec);
|
||||
}
|
||||
#endif
|
||||
for (; j < N; ++j) {
|
||||
@@ -392,16 +417,25 @@ void fill_val_stub(int32_t* __restrict__ output, int32_t value, int64_t size) {
|
||||
}
|
||||
}
|
||||
|
||||
template <bool sym_quant_act, typename act_dtype, typename out_dtype>
|
||||
template <typename act_dtype, typename out_dtype, bool sym_quant_act>
|
||||
void _da8w4_linear_impl(
|
||||
act_dtype* __restrict__ input, const float* __restrict__ input_scales,
|
||||
act_dtype* __restrict__ input,
|
||||
const float* __restrict__ input_scales,
|
||||
const int32_t* __restrict__ input_qzeros,
|
||||
const uint8_t* __restrict__ weight, const float* __restrict__ weight_scales,
|
||||
const int8_t* __restrict__ weight_qzeros, const float* __restrict__ bias,
|
||||
out_dtype* __restrict__ output, float* __restrict__ output_temp,
|
||||
int8_t* __restrict__ dequant_weight_temp, int64_t M, int64_t N, int64_t K,
|
||||
const uint8_t* __restrict__ weight,
|
||||
const float* __restrict__ weight_scales,
|
||||
const int8_t* __restrict__ weight_qzeros,
|
||||
const float* __restrict__ bias,
|
||||
out_dtype* __restrict__ output,
|
||||
float* __restrict__ output_temp,
|
||||
int8_t* __restrict__ dequant_weight_temp,
|
||||
int64_t M,
|
||||
int64_t N,
|
||||
int64_t K,
|
||||
int64_t num_groups) {
|
||||
const bool use_brgemm = can_use_brgemm<act_dtype>(M);
|
||||
// weight + compensation shape = [Nc, Kc, BLOCK_N * _block_k / 2 + BLOCK_N*sizeof(int32_t)]
|
||||
// scales/qzeros shape = [Nc, G, BLOCK_N]
|
||||
const bool use_brgemm = can_use_brgemm<int8_t>(M);
|
||||
int64_t block_m = [&]() -> long {
|
||||
if (M <= 48) {
|
||||
return M;
|
||||
@@ -432,30 +466,24 @@ void _da8w4_linear_impl(
|
||||
int64_t mc_end = parallel_on_M ? mc + 1 : Mc;
|
||||
|
||||
for (int mci = mc; mci < mc_end; ++mci) {
|
||||
int64_t m_size =
|
||||
mci * block_m + block_m > M ? M - mci * block_m : block_m;
|
||||
int64_t m_size = mci * block_m + block_m > M ? M - mci * block_m : block_m;
|
||||
// copy bias to y_buf if bias is not None
|
||||
auto bias_data = bias ? bias + nc * BLOCK_N : nullptr;
|
||||
copy_bias<BLOCK_N>(bias_data, C_tmp, m_size);
|
||||
for (int kci = 0; kci < Kc; ++kci) {
|
||||
int32_t* compensation_ptr =
|
||||
sym_quant_act
|
||||
? nullptr
|
||||
: (int32_t*)(void*)(weight +
|
||||
(nc * Kc + kci) *
|
||||
(BLOCK_N *
|
||||
(_block_k / 2 + sizeof(int32_t))) +
|
||||
_block_k * BLOCK_N / 2);
|
||||
_dequant_gemm_accum<sym_quant_act, BLOCK_N, BLOCK_N / 2>(
|
||||
: (int32_t*)(void*)(weight + (nc * Kc + kci) * (BLOCK_N * (_block_k / 2 + sizeof(int32_t))) +
|
||||
_block_k * BLOCK_N / 2) /*Bcomp*/;
|
||||
_dequant_gemm_accum<BLOCK_N, BLOCK_N / 2, sym_quant_act>(
|
||||
/*C*/ C_tmp,
|
||||
/*A*/ (uint8_t*)input + mci * block_m * K + kci * _block_k,
|
||||
/*scales_a*/ input_scales + mci * block_m,
|
||||
/*qzeros_a*/ input_qzeros + mci * block_m,
|
||||
/*B*/ weight + (nc * Kc + kci) *
|
||||
(BLOCK_N * (_block_k / 2 + sizeof(int32_t))),
|
||||
/*scales_b*/ weight_scales + nc * BLOCK_N * num_groups +
|
||||
kci / block_per_group * BLOCK_N,
|
||||
/*qzeros_b*/ weight_qzeros + nc * BLOCK_N * num_groups +
|
||||
kci / block_per_group * BLOCK_N,
|
||||
/*B*/ weight + (nc * Kc + kci) * (BLOCK_N * (_block_k / 2 + sizeof(int32_t))),
|
||||
/*scales_b*/ weight_scales + nc * BLOCK_N * num_groups + kci / block_per_group * BLOCK_N,
|
||||
/*qzeros_b*/ weight_qzeros + nc * BLOCK_N * num_groups + kci / block_per_group * BLOCK_N,
|
||||
/*Bcomp*/ compensation_ptr,
|
||||
/*dqB_tmp*/ dqB_tmp,
|
||||
/*M*/ m_size,
|
||||
@@ -464,8 +492,8 @@ void _da8w4_linear_impl(
|
||||
/*ldc*/ BLOCK_N,
|
||||
/*use_brgemm*/ use_brgemm);
|
||||
}
|
||||
store_out<BLOCK_N>(C_tmp, output + mci * block_m * N + nc * BLOCK_N,
|
||||
m_size, N /*lda*/);
|
||||
// store y_buf to output with dtype conversion
|
||||
store_out<out_dtype, BLOCK_N>(C_tmp, output + mci * block_m * N + nc * BLOCK_N, m_size, N /*lda*/);
|
||||
}
|
||||
}
|
||||
if (use_brgemm) {
|
||||
@@ -476,15 +504,16 @@ void _da8w4_linear_impl(
|
||||
|
||||
} // anonymous namespace
|
||||
|
||||
std::tuple<at::Tensor, at::Tensor, at::Tensor>
|
||||
convert_int4_weight_packed_with_compensation(const at::Tensor& weight,
|
||||
const at::Tensor& scales,
|
||||
const at::Tensor& qzeros) {
|
||||
TORCH_CHECK(weight.dim() == 2,
|
||||
"DA8W4 CPU: Weight should be a 2D tensor for packing");
|
||||
TORCH_CHECK(
|
||||
weight.size(1) % 2 == 0,
|
||||
"DA8W4 CPU: Weight should have even number of columns for packing");
|
||||
/*
|
||||
return: packed_weight, packed_scales, packed_qzeros
|
||||
*/
|
||||
std::tuple<at::Tensor, at::Tensor, at::Tensor> convert_int4_weight_packed_with_compensation(
|
||||
const at::Tensor& weight, const at::Tensor& scales, const at::Tensor& qzeros) {
|
||||
// weight shape = [N, K]
|
||||
// scales shape = [N, G]
|
||||
// qzeros shape = [N, G]
|
||||
TORCH_CHECK(weight.dim() == 2, "DA8W4 CPU: Weight should be a 2D tensor for packing");
|
||||
TORCH_CHECK(weight.size(1) % 2 == 0, "DA8W4 CPU: Weight should have even number of columns for packing");
|
||||
|
||||
auto new_scales = scales;
|
||||
auto new_qzeros = qzeros;
|
||||
@@ -505,20 +534,21 @@ convert_int4_weight_packed_with_compensation(const at::Tensor& weight,
|
||||
int64_t Nc = N / block_n;
|
||||
int64_t Kc = K / _block_k;
|
||||
|
||||
// Reorder weight to [N/block_n, K/_block_k, _block_k, block_n]
|
||||
// Reorder scales/qzeros to [N/block_n, G, block_n]
|
||||
// weight + compensation shape = [Nc, Kc, block_n * _block_k / 2 + block_n*sizeof(int32_t)]
|
||||
// scales/qzeros shape = [Nc, G, block_n]
|
||||
auto weight_view = weight.view({Nc, block_n, Kc, _block_k});
|
||||
at::Tensor weight_reordered = weight_view.permute({0, 2, 3, 1}).contiguous();
|
||||
at::Tensor blocked_weight;
|
||||
at::Tensor blocked_scales =
|
||||
new_scales.view({Nc, block_n, G}).permute({0, 2, 1}).contiguous();
|
||||
at::Tensor blocked_qzeros =
|
||||
new_qzeros.view({Nc, block_n, G}).permute({0, 2, 1}).contiguous();
|
||||
auto weight_sub_qzero = weight.view({Nc, block_n, G, -1}).to(at::kInt) -
|
||||
new_qzeros.view({Nc, block_n, G, -1});
|
||||
at::Tensor blocked_scales = new_scales.view({Nc, block_n, G}).permute({0, 2, 1}).contiguous();
|
||||
at::Tensor blocked_qzeros = new_qzeros.view({Nc, block_n, G}).permute({0, 2, 1}).contiguous();
|
||||
// Compensation = Σ(k)(W[k][n] - ZP[n]) for each block.
|
||||
auto weight_sub_qzero = weight.view({Nc, block_n, G, -1}).to(at::kInt) - new_qzeros.view({Nc, block_n, G, -1});
|
||||
weight_sub_qzero = weight_sub_qzero.view({Nc, block_n, Kc, _block_k});
|
||||
at::Tensor compensation = weight_sub_qzero.sum(-1);
|
||||
compensation = compensation.permute({0, 2, 1}).contiguous().to(at::kInt);
|
||||
int64_t buffer_size_nbytes =
|
||||
_block_k * block_n / 2 + block_n * sizeof(int32_t);
|
||||
int64_t buffer_size_nbytes = _block_k * block_n / 2 + block_n * sizeof(int32_t);
|
||||
blocked_weight = at::empty({Nc, Kc, buffer_size_nbytes}, weight.options());
|
||||
|
||||
auto weight_ptr = weight_reordered.data_ptr<uint8_t>();
|
||||
@@ -528,24 +558,25 @@ convert_int4_weight_packed_with_compensation(const at::Tensor& weight,
|
||||
at::parallel_for(0, num_blocks, 1, [&](int64_t begin, int64_t end) {
|
||||
for (const auto i : c10::irange(begin, end)) {
|
||||
auto in_ptr = weight_ptr + i * _block_k * block_n;
|
||||
auto out_ptr =
|
||||
blocked_weight_ptr + i * block_n * (_block_k / 2 + sizeof(int32_t));
|
||||
auto out_ptr = blocked_weight_ptr + i * block_n * (_block_k / 2 + sizeof(int32_t));
|
||||
int32_t* comp_in_prt = compensation_ptr + i * block_n;
|
||||
int32_t* comp_out_prt =
|
||||
(int32_t*)(void*)(blocked_weight_ptr +
|
||||
i * block_n * (_block_k / 2 + sizeof(int32_t)) +
|
||||
_block_k * block_n / 2);
|
||||
int32_t* comp_out_prt = (int32_t*)(void*)(blocked_weight_ptr + i * block_n * (_block_k / 2 + sizeof(int32_t)) +
|
||||
_block_k * block_n / 2);
|
||||
// Reorder weight block to VNNI4 and pack two lanes along N
|
||||
// N=16 viewed as two lanes: a0, ...a7, b0, ...b7
|
||||
// pack two lanes: [a0, b0], ..., [a7, b7]
|
||||
// plain shape = [_block_k, block_n]
|
||||
// packed shape = [_block_k / 4, block_n / 2, 4] viewed as [_block_k, block_n / 2]
|
||||
constexpr int n_group_size = 8;
|
||||
constexpr int vnni_size = 4;
|
||||
constexpr int n_group = block_n / n_group_size;
|
||||
constexpr int n_group = block_n / n_group_size; // 4
|
||||
for (int nb = 0; nb < n_group; nb += 2) {
|
||||
for (int k = 0; k < _block_k; k += vnni_size) {
|
||||
for (int ni = 0; ni < n_group_size; ++ni) {
|
||||
for (int ki = 0; ki < vnni_size; ++ki) {
|
||||
int src_idx_1 = nb * n_group_size + ni + (k + ki) * block_n;
|
||||
int src_idx_2 = (nb + 1) * n_group_size + ni + (k + ki) * block_n;
|
||||
int dst_idx = (nb / 2 * n_group_size + ni) * vnni_size +
|
||||
k * block_n / 2 + ki;
|
||||
int dst_idx = (nb / 2 * n_group_size + ni) * vnni_size + k * block_n / 2 + ki;
|
||||
uint8_t src_1 = *(in_ptr + src_idx_1);
|
||||
uint8_t src_2 = *(in_ptr + src_idx_2);
|
||||
uint8_t dst = (src_1 & 0x0f) | ((src_2 & 0x0f) << 4);
|
||||
@@ -554,39 +585,128 @@ convert_int4_weight_packed_with_compensation(const at::Tensor& weight,
|
||||
}
|
||||
}
|
||||
}
|
||||
// compensation [block_n]
|
||||
for (int nb = 0; nb < block_n; nb++) {
|
||||
*(comp_out_prt + nb) = *(comp_in_prt + nb);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return std::make_tuple(std::move(blocked_weight), std::move(blocked_scales),
|
||||
std::move(blocked_qzeros));
|
||||
return std::make_tuple(std::move(blocked_weight), std::move(blocked_scales), std::move(blocked_qzeros));
|
||||
}
|
||||
|
||||
std::tuple<at::Tensor, at::Tensor> autoawq_to_int4pack(at::Tensor qweight,
|
||||
at::Tensor qzeros) {
|
||||
auto bitshifts = at::tensor({0, 4, 1, 5, 2, 6, 3, 7}, at::kInt) * 4;
|
||||
auto qweight_unsq = qweight.unsqueeze(-1);
|
||||
auto unpacked = at::bitwise_right_shift(qweight_unsq, bitshifts) & 0xF;
|
||||
auto qweight_final = unpacked.flatten(-2).transpose(-1, -2).to(at::kByte);
|
||||
std::tuple<at::Tensor, at::Tensor> unpack_4bit_to_32bit_signed(const at::Tensor& qweight, const at::Tensor& qzeros) {
|
||||
TORCH_CHECK(qweight.scalar_type() == at::kInt, "qweight must be int32");
|
||||
TORCH_CHECK(qzeros.scalar_type() == at::kInt, "qzeros must be int32");
|
||||
const auto W0 = qweight.size(0);
|
||||
const auto W1 = qweight.size(1);
|
||||
const auto Z0 = qzeros.size(0);
|
||||
const auto Z1 = qzeros.size(1);
|
||||
|
||||
auto qzeros_unsq = qzeros.unsqueeze(-1);
|
||||
auto qzeros_unpacked = at::bitwise_right_shift(qzeros_unsq, bitshifts) & 0xF;
|
||||
auto qzeros_final = qzeros_unpacked.flatten(-2).to(at::kByte);
|
||||
// unpacked_weights: (W0 * 8, W1), int8
|
||||
auto unpacked_weights = at::zeros({W0 * 8, W1}, at::TensorOptions().dtype(at::kChar));
|
||||
// unpacked_zeros: (Z0, Z1 * 8), int8
|
||||
auto unpacked_zeros = at::zeros({Z0, Z1 * 8}, at::TensorOptions().dtype(at::kChar));
|
||||
|
||||
return std::make_tuple(qweight_final, qzeros_final);
|
||||
const int32_t* qw_ptr = qweight.data_ptr<int32_t>();
|
||||
const int32_t* qz_ptr = qzeros.data_ptr<int32_t>();
|
||||
int8_t* uw_ptr = unpacked_weights.data_ptr<int8_t>();
|
||||
int8_t* uz_ptr = unpacked_zeros.data_ptr<int8_t>();
|
||||
|
||||
// ---- unpack qweight ----
|
||||
for (int64_t row = 0; row < W0 * 8; ++row) {
|
||||
const int i = row & 7; // row % 8
|
||||
const int src_row = row >> 3; // row // 8
|
||||
const int shift = 4 * i;
|
||||
for (int64_t col = 0; col < W1; ++col) {
|
||||
int32_t v = qw_ptr[src_row * W1 + col];
|
||||
uw_ptr[row * W1 + col] = static_cast<int8_t>((v >> shift) & 0xF);
|
||||
}
|
||||
}
|
||||
// ---- unpack qzeros ----
|
||||
for (int64_t col = 0; col < Z1 * 8; ++col) {
|
||||
const int i = col & 7;
|
||||
const int src_col = col >> 3;
|
||||
const int shift = 4 * i;
|
||||
|
||||
for (int64_t row = 0; row < Z0; ++row) {
|
||||
int32_t v = qz_ptr[row * Z1 + src_col];
|
||||
uz_ptr[row * (Z1 * 8) + col] = static_cast<int8_t>((v >> shift) & 0xF);
|
||||
}
|
||||
}
|
||||
|
||||
return std::make_tuple(unpacked_weights, unpacked_zeros + 1);
|
||||
}
|
||||
|
||||
std::tuple<at::Tensor, at::Tensor>
|
||||
autogptq_to_int4pack(const at::Tensor& qweight_tensor, const at::Tensor& qzeros_tensor) {
|
||||
TORCH_CHECK(qweight_tensor.scalar_type() == at::kInt, "qweight_tensor must be int32");
|
||||
TORCH_CHECK(qzeros_tensor.scalar_type() == at::kInt, "qzeros_tensor must be int32");
|
||||
TORCH_CHECK(qweight_tensor.is_cpu(), "CPU only implementation");
|
||||
if (qweight_tensor.dim() == 3) {
|
||||
const int64_t B = qweight_tensor.size(0);
|
||||
std::vector<at::Tensor> qweight_list;
|
||||
std::vector<at::Tensor> qzeros_list;
|
||||
qweight_list.reserve(B);
|
||||
qzeros_list.reserve(B);
|
||||
for (int64_t i = 0; i < B; ++i) {
|
||||
auto outputs = unpack_4bit_to_32bit_signed(qweight_tensor[i], qzeros_tensor[i]);
|
||||
at::Tensor unpacked_qweight = std::get<0>(outputs);
|
||||
at::Tensor unpacked_qzeros = std::get<1>(outputs);
|
||||
qweight_list.push_back(unpacked_qweight.transpose(0, 1).contiguous().to(at::kByte));
|
||||
qzeros_list.push_back(unpacked_qzeros.contiguous().to(at::kByte));
|
||||
}
|
||||
return std::make_tuple(at::stack(qweight_list).detach(), at::stack(qzeros_list).detach());
|
||||
}
|
||||
auto outputs = unpack_4bit_to_32bit_signed(qweight_tensor, qzeros_tensor);
|
||||
at::Tensor unpacked_qweight = std::get<0>(outputs);
|
||||
at::Tensor unpacked_qzeros = std::get<1>(outputs);
|
||||
at::Tensor return_qweight = unpacked_qweight.transpose(0, 1).contiguous().to(at::kByte);
|
||||
at::Tensor return_qzeros = unpacked_qzeros.contiguous().to(at::kByte);
|
||||
return std::make_tuple(return_qweight, return_qzeros);
|
||||
}
|
||||
|
||||
std::tuple<at::Tensor, at::Tensor> int4pack(at::Tensor qweight, at::Tensor qzeros, int64_t quant_method_4bit) {
|
||||
if (quant_method_4bit == CPUQuantAlgo::AWQ) {
|
||||
// autoawq unpacking
|
||||
qweight = qweight.contiguous();
|
||||
qzeros = qzeros.contiguous();
|
||||
// bitshifts: [0, 4, 1, 5, 2, 6, 3, 7] * 4
|
||||
auto bitshifts = at::tensor({0, 4, 1, 5, 2, 6, 3, 7}, at::kInt) * 4;
|
||||
auto qweight_unsq = qweight.unsqueeze(-1); // [..., K, N/8, 1]
|
||||
auto unpacked = (at::bitwise_right_shift(qweight_unsq, bitshifts) & 0xF).contiguous();
|
||||
auto qweight_final = unpacked.flatten(-2).transpose(-1, -2).to(at::kByte).clone();
|
||||
auto qzeros_unsq = qzeros.unsqueeze(-1);
|
||||
auto qzeros_unpacked = (at::bitwise_right_shift(qzeros_unsq, bitshifts) & 0xF).contiguous();
|
||||
auto qzeros_final = qzeros_unpacked.flatten(-2).to(at::kByte).clone();
|
||||
return std::make_tuple(qweight_final, qzeros_final);
|
||||
} else if (quant_method_4bit == CPUQuantAlgo::GPTQ) {
|
||||
// autogptq unpacking
|
||||
auto outputs = autogptq_to_int4pack(qweight, qzeros);
|
||||
at::Tensor unpacked_qweight = std::get<0>(outputs);
|
||||
at::Tensor unpacked_qzeros = std::get<1>(outputs);
|
||||
return std::make_tuple(unpacked_qweight, unpacked_qzeros);
|
||||
} else {
|
||||
TORCH_CHECK(false, "CPU int4 pack only support AWQ or GPTQ...");
|
||||
}
|
||||
}
|
||||
|
||||
std::tuple<at::Tensor, at::Tensor, at::Tensor> convert_weight_packed_scale_zp(
|
||||
at::Tensor qweight, at::Tensor qzeros, at::Tensor scales) {
|
||||
auto res = autoawq_to_int4pack(qweight, qzeros);
|
||||
auto _qweight = std::get<0>(res);
|
||||
auto _qzeros = std::get<1>(res);
|
||||
at::Tensor qweight, // awq: (*, K, N / 8) || gptq: (*, K / 8, N) , int32
|
||||
at::Tensor qzeros, // awq: (*, K / group_size, N / 8) || gptq: (*, K / group_size, N / 8) , int32
|
||||
at::Tensor scales, // awq: (*, K / group_size, N) || gptq: (*, K / group_size, N) , bfloat16
|
||||
int64_t quant_method_4bit) {
|
||||
at::Tensor _qweight;
|
||||
at::Tensor _qzeros;
|
||||
|
||||
auto res = int4pack(qweight, qzeros, quant_method_4bit);
|
||||
_qweight = std::get<0>(res);
|
||||
_qzeros = std::get<1>(res);
|
||||
|
||||
auto _scales = scales;
|
||||
_qzeros = _qzeros.transpose(-2, -1).contiguous();
|
||||
_qzeros = _qzeros.transpose(-2, -1).contiguous(); // .T
|
||||
_scales = _scales.transpose(-2, -1).contiguous();
|
||||
if (_qweight.dim() == 3) {
|
||||
if (_qweight.dim() == 3) { // Dim=3 for MOE packing, TODO: refine a unified loop
|
||||
int64_t E = _qweight.size(0);
|
||||
int64_t K = _qweight.size(2);
|
||||
int64_t G = _scales.size(2);
|
||||
@@ -595,17 +715,12 @@ std::tuple<at::Tensor, at::Tensor, at::Tensor> convert_weight_packed_scale_zp(
|
||||
int64_t block_n = block_size_n();
|
||||
int64_t Nc = _qweight.size(1) / block_n;
|
||||
int64_t Kc = K / _block_k;
|
||||
int64_t buffer_size_nbytes =
|
||||
_block_k * block_n / 2 + block_n * sizeof(int32_t);
|
||||
auto blocked_weight =
|
||||
at::empty({E, Nc, Kc, buffer_size_nbytes}, _qweight.options());
|
||||
auto blocked_scales =
|
||||
at::empty({E, Nc, G, block_n}, _scales.options()).to(at::kFloat);
|
||||
auto blocked_qzeros =
|
||||
at::empty({E, Nc, G, block_n}, _qzeros.options()).to(at::kChar);
|
||||
int64_t buffer_size_nbytes = _block_k * block_n / 2 + block_n * sizeof(int32_t);
|
||||
auto blocked_weight = at::empty({E, Nc, Kc, buffer_size_nbytes}, _qweight.options());
|
||||
auto blocked_scales = at::empty({E, Nc, G, block_n}, _scales.options()).to(at::kFloat);
|
||||
auto blocked_qzeros = at::empty({E, Nc, G, block_n}, _qzeros.options()).to(at::kChar);
|
||||
for (int i = 0; i < _qweight.size(0); i++) {
|
||||
auto res_ = convert_int4_weight_packed_with_compensation(
|
||||
_qweight[i], _scales[i], _qzeros[i]);
|
||||
auto res_ = convert_int4_weight_packed_with_compensation(_qweight[i], _scales[i], _qzeros[i]);
|
||||
blocked_weight[i] = std::get<0>(res_);
|
||||
blocked_scales[i] = std::get<1>(res_);
|
||||
blocked_qzeros[i] = std::get<2>(res_);
|
||||
@@ -614,8 +729,7 @@ std::tuple<at::Tensor, at::Tensor, at::Tensor> convert_weight_packed_scale_zp(
|
||||
_scales = blocked_scales;
|
||||
_qzeros = blocked_qzeros;
|
||||
} else {
|
||||
auto res_ = convert_int4_weight_packed_with_compensation(_qweight, _scales,
|
||||
_qzeros);
|
||||
auto res_ = convert_int4_weight_packed_with_compensation(_qweight, _scales, _qzeros);
|
||||
_qweight = std::get<0>(res_);
|
||||
_scales = std::get<1>(res_);
|
||||
_qzeros = std::get<2>(res_);
|
||||
@@ -624,89 +738,93 @@ std::tuple<at::Tensor, at::Tensor, at::Tensor> convert_weight_packed_scale_zp(
|
||||
return std::make_tuple(_qweight, _qzeros, _scales);
|
||||
}
|
||||
|
||||
at::Tensor int4_scaled_mm_cpu_with_quant(const at::Tensor& input,
|
||||
const at::Tensor& weight,
|
||||
const at::Tensor& weight_scales,
|
||||
const at::Tensor& weight_qzeros,
|
||||
const std::optional<at::Tensor>& bias,
|
||||
at::ScalarType output_dtype) {
|
||||
RECORD_FUNCTION("vllm::int4_scaled_mm_cpu_with_quant",
|
||||
std::vector<c10::IValue>({input, weight}));
|
||||
|
||||
at::Tensor int4_scaled_mm_cpu_with_quant(
|
||||
const at::Tensor& input,
|
||||
const at::Tensor& weight,
|
||||
const at::Tensor& weight_scales,
|
||||
const at::Tensor& weight_qzeros,
|
||||
const std::optional<at::Tensor>& bias,
|
||||
at::ScalarType output_dtype) {
|
||||
int64_t M_a = input.size(0);
|
||||
int64_t K_a = input.size(1);
|
||||
int64_t lda = input.stride(0);
|
||||
|
||||
const auto st = input.scalar_type();
|
||||
TORCH_CHECK(
|
||||
st == at::kBFloat16 || st == at::kHalf,
|
||||
"int4_scaled_mm_cpu_with_quant: expect A to be bfloat16 or half.");
|
||||
st == at::kBFloat16 || st == at::kHalf, "int4_scaled_mm_cpu_with_quant: expect A to be bfloat16 or half.");
|
||||
|
||||
constexpr bool sym_quant_act = false;
|
||||
constexpr bool sym_quant_act = false; // TODO: add sym quant path
|
||||
using Tin = typename ActDtype<sym_quant_act>::type;
|
||||
int64_t act_buffer_size =
|
||||
M_a * K_a + M_a * sizeof(float) + M_a * sizeof(int32_t);
|
||||
auto act_buffer =
|
||||
at::empty({act_buffer_size}, input.options().dtype(at::kByte));
|
||||
int64_t act_buffer_size = /* act quant */ M_a * K_a +
|
||||
/* act scale */ M_a * sizeof(float) +
|
||||
/* act zp */ M_a * sizeof(int32_t);
|
||||
auto act_buffer = at::empty({act_buffer_size}, input.options().dtype(at::kByte));
|
||||
// asym path, activation quants into uint8_t
|
||||
auto Aq_data = act_buffer.data_ptr<uint8_t>();
|
||||
auto As_data = reinterpret_cast<float*>(Aq_data + M_a * K_a);
|
||||
auto Azp_data = reinterpret_cast<int32_t*>(As_data + M_a);
|
||||
fill_val_stub(Azp_data, 128, M_a);
|
||||
fill_val_stub(Azp_data, 128, M_a); // sym_a s8s8 is unified to u8s8 with compensation (128)
|
||||
|
||||
auto out_sizes = input.sizes().vec();
|
||||
int64_t N = weight_scales.size(0) * weight_scales.size(-1);
|
||||
out_sizes.back() = N;
|
||||
auto output = at::empty(out_sizes, input.options());
|
||||
// weight + compensation shape = [Nc, Kc, BLOCK_N * _block_k / 2 + BLOCK_N*sizeof(int32_t)]
|
||||
// scales/qzeros shape = [Nc, G, BLOCK_N]
|
||||
int64_t Nc = weight.size(0);
|
||||
int64_t Kc = weight.size(1);
|
||||
int64_t _block_k = K_a / Kc;
|
||||
TORCH_CHECK(N == Nc * BLOCK_N, "DA8W4: weight and input shapes mismatch");
|
||||
// scales/qzeros shape = [Nc, G, BLOCK_N]
|
||||
int64_t num_groups = weight_scales.size(1);
|
||||
|
||||
const uint8_t* b_ptr = weight.data_ptr<uint8_t>();
|
||||
const float* b_scales_ptr = weight_scales.data_ptr<float>();
|
||||
const int8_t* b_qzeros_ptr = weight_qzeros.data_ptr<int8_t>();
|
||||
const float* bias_ptr =
|
||||
bias.has_value() ? bias.value().data_ptr<float>() : nullptr;
|
||||
const float* bias_ptr = bias.has_value() ? bias.value().data_ptr<float>() : nullptr;
|
||||
int num_threads = at::get_num_threads();
|
||||
int64_t temp_buffer_size = num_threads * BLOCK_M * BLOCK_N * sizeof(float) +
|
||||
num_threads * _block_k * BLOCK_N;
|
||||
auto c_temp_buffer =
|
||||
at::empty({temp_buffer_size}, input.options().dtype(at::kChar));
|
||||
int64_t temp_buffer_size = /* output temp */ num_threads * BLOCK_M * BLOCK_N * sizeof(float) +
|
||||
/* weight dequant temp */ num_threads * _block_k * BLOCK_N;
|
||||
auto c_temp_buffer = at::empty({temp_buffer_size}, input.options().dtype(at::kChar));
|
||||
float* c_temp_ptr = (float*)((void*)(c_temp_buffer.data_ptr<int8_t>()));
|
||||
int8_t* dqB_temp_ptr =
|
||||
(int8_t*)((void*)(c_temp_ptr + num_threads * BLOCK_M * BLOCK_N));
|
||||
int8_t* dqB_temp_ptr = (int8_t*)((void*)(c_temp_ptr + num_threads * BLOCK_M * BLOCK_N));
|
||||
|
||||
#define LAUNCH_DA8W4_LINEAR_WITH_QUANT_IMPL(sym_quant_act) \
|
||||
AT_DISPATCH_FLOATING_TYPES_AND2( \
|
||||
at::ScalarType::BFloat16, at::ScalarType::Half, output_dtype, \
|
||||
"int4_scaled_mm_cpu", [&] { \
|
||||
const scalar_t* __restrict__ A_data = input.data_ptr<scalar_t>(); \
|
||||
scalar_t* __restrict__ c_ptr = output.data_ptr<scalar_t>(); \
|
||||
at::parallel_for(0, M_a, 0, [&](int64_t begin, int64_t end) { \
|
||||
for (int64_t m = begin; m < end; ++m) { \
|
||||
quantize_row_int8<scalar_t>(Aq_data + m * K_a, As_data[m], \
|
||||
A_data + m * lda, K_a); \
|
||||
} \
|
||||
}); \
|
||||
_da8w4_linear_impl<sym_quant_act, Tin, scalar_t>( \
|
||||
Aq_data, As_data, Azp_data, b_ptr, b_scales_ptr, b_qzeros_ptr, \
|
||||
bias_ptr, c_ptr, c_temp_ptr, dqB_temp_ptr, M_a, N, K_a, \
|
||||
num_groups); \
|
||||
#define LAUNCH_DA8W4_LINEAR_WITH_QUANT_IMPL(sym_quant_act) \
|
||||
AT_DISPATCH_FLOATING_TYPES_AND2( \
|
||||
at::ScalarType::BFloat16, at::ScalarType::Half, output_dtype, "int4_scaled_mm_cpu_with_quant", [&] { \
|
||||
const scalar_t* __restrict__ A_data = input.data_ptr<scalar_t>(); \
|
||||
scalar_t* __restrict__ c_ptr = output.data_ptr<scalar_t>(); \
|
||||
at::parallel_for(0, M_a, 0, [&](int64_t begin, int64_t end) { \
|
||||
for (int64_t m = begin; m < end; ++m) { \
|
||||
quantize_row_int8<scalar_t>(Aq_data + m * K_a, As_data[m], A_data + m * lda, K_a); \
|
||||
} \
|
||||
}); \
|
||||
_da8w4_linear_impl<Tin, scalar_t, sym_quant_act>( \
|
||||
Aq_data, \
|
||||
As_data, \
|
||||
Azp_data, \
|
||||
b_ptr, \
|
||||
b_scales_ptr, \
|
||||
b_qzeros_ptr, \
|
||||
bias_ptr, \
|
||||
c_ptr, \
|
||||
c_temp_ptr, \
|
||||
dqB_temp_ptr, \
|
||||
M_a, \
|
||||
N, \
|
||||
K_a, \
|
||||
num_groups); \
|
||||
});
|
||||
|
||||
LAUNCH_DA8W4_LINEAR_WITH_QUANT_IMPL(sym_quant_act);
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
||||
template <typename scalar_t>
|
||||
inline void copy_stub(scalar_t* __restrict__ out,
|
||||
const float* __restrict__ input, int64_t size) {
|
||||
inline void copy_stub(scalar_t* __restrict__ out, const float* __restrict__ input, int64_t size) {
|
||||
using Vec = at::vec::Vectorized<scalar_t>;
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
// no remainder
|
||||
#pragma GCC unroll 4
|
||||
for (int64_t d = 0; d < size; d += Vec::size()) {
|
||||
fVec x0 = fVec::loadu(input + d);
|
||||
@@ -716,40 +834,61 @@ inline void copy_stub(scalar_t* __restrict__ out,
|
||||
}
|
||||
}
|
||||
|
||||
} // anonymous namespace
|
||||
|
||||
template <typename scalar_t>
|
||||
void tinygemm_kernel(scalar_t* C, float* C_temp, const uint8_t* A,
|
||||
const float* scales_a, const int32_t* qzeros_a,
|
||||
const uint8_t* B, const float* scales_b,
|
||||
const int8_t* qzeros_b, const int32_t* compensation,
|
||||
int8_t* dqB_tmp, int64_t M, int64_t K, int64_t lda,
|
||||
int64_t ldc_f, int64_t ldc_s, bool store_out,
|
||||
bool use_brgemm) {
|
||||
_dequant_gemm_accum<false, BLOCK_N, BLOCK_N / 2>(
|
||||
C_temp, A, scales_a, qzeros_a, B, scales_b, qzeros_b, compensation,
|
||||
dqB_tmp, M, K, lda, ldc_f, use_brgemm);
|
||||
void tinygemm_kernel(
|
||||
scalar_t* C,
|
||||
float* C_temp,
|
||||
const uint8_t* A,
|
||||
const float* scales_a,
|
||||
const int32_t* qzeros_a,
|
||||
const uint8_t* B,
|
||||
const float* scales_b,
|
||||
const int8_t* qzeros_b,
|
||||
const int32_t* compensation,
|
||||
int8_t* dqB_tmp,
|
||||
int64_t M,
|
||||
int64_t K,
|
||||
int64_t lda,
|
||||
int64_t ldc_f,
|
||||
int64_t ldc_s,
|
||||
bool store_out,
|
||||
bool use_brgemm) {
|
||||
// TODO: add sym quant act, now only asym
|
||||
_dequant_gemm_accum<BLOCK_N, BLOCK_N / 2, false>(
|
||||
C_temp, A, scales_a, qzeros_a, B, scales_b, qzeros_b, compensation, dqB_tmp, M, K, lda, ldc_f, use_brgemm);
|
||||
if (store_out) {
|
||||
// copy from Ctmp to C
|
||||
for (int64_t m = 0; m < M; ++m) {
|
||||
copy_stub<scalar_t>(C + m * ldc_s, C_temp + m * ldc_f, BLOCK_N);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#define INSTANTIATE_TINYGEMM_TEMPLATE(TYPE) \
|
||||
template void tinygemm_kernel<TYPE>( \
|
||||
TYPE * C, float* C_temp, const uint8_t* A, const float* scales_a, \
|
||||
const int32_t* qzeros_a, const uint8_t* B, const float* scales_b, \
|
||||
const int8_t* qzeros_b, const int32_t* compensation, int8_t* dqB_tmp, \
|
||||
int64_t M, int64_t K, int64_t lda, int64_t ldc_f, int64_t ldc_s, \
|
||||
bool store_out, bool use_brgemm)
|
||||
#define INSTANTIATE_TINYGEMM_TEMPLATE(TYPE) \
|
||||
template void tinygemm_kernel<TYPE>( \
|
||||
TYPE * C, \
|
||||
float* C_temp, \
|
||||
const uint8_t* A, \
|
||||
const float* scales_a, \
|
||||
const int32_t* qzeros_a, \
|
||||
const uint8_t* B, \
|
||||
const float* scales_b, \
|
||||
const int8_t* qzeros_b, \
|
||||
const int32_t* compensation, \
|
||||
int8_t* dqB_tmp, \
|
||||
int64_t M, \
|
||||
int64_t K, \
|
||||
int64_t lda, \
|
||||
int64_t ldc_f, \
|
||||
int64_t ldc_s, \
|
||||
bool store_out, \
|
||||
bool use_brgemm)
|
||||
|
||||
INSTANTIATE_TINYGEMM_TEMPLATE(at::BFloat16);
|
||||
INSTANTIATE_TINYGEMM_TEMPLATE(at::Half);
|
||||
|
||||
at::Tensor int4_scaled_mm_cpu(at::Tensor& x, at::Tensor& w, at::Tensor& w_zeros,
|
||||
at::Tensor& w_scales,
|
||||
std::optional<at::Tensor> bias) {
|
||||
return int4_scaled_mm_cpu_with_quant(x, w, w_scales, w_zeros, bias,
|
||||
x.scalar_type());
|
||||
// int4 gemm dispatch api register
|
||||
at::Tensor int4_scaled_mm_cpu(
|
||||
at::Tensor& x, at::Tensor& w, at::Tensor& w_zeros, at::Tensor& w_scales, std::optional<at::Tensor> bias) {
|
||||
return int4_scaled_mm_cpu_with_quant(x, w, w_scales, w_zeros, bias, x.scalar_type());
|
||||
}
|
||||
|
||||
@@ -1,20 +1,83 @@
|
||||
// Adapted from
|
||||
// https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu
|
||||
|
||||
#include "common.h"
|
||||
#include "vec.h"
|
||||
#include "gemm.h"
|
||||
|
||||
// clang-format off
|
||||
|
||||
#include "common.h"
|
||||
#include "gemm.h"
|
||||
#include "vec.h"
|
||||
|
||||
namespace {
|
||||
|
||||
template <typename scalar_t, bool has_bias, int BLOCK_N>
|
||||
struct scale_C {
|
||||
static inline void apply(
|
||||
scalar_t* __restrict__ C,
|
||||
const int32_t* __restrict__ Ctmp,
|
||||
const int32_t* __restrict__ Bcomp,
|
||||
const float* __restrict__ bias,
|
||||
float As,
|
||||
const float* __restrict__ Bs) {
|
||||
TORCH_CHECK(false, "scale_C: scalar path not implemented!");
|
||||
}
|
||||
};
|
||||
|
||||
#if defined(CPU_CAPABILITY_AVX512)
|
||||
template <bool has_bias, int BLOCK_N>
|
||||
struct scale_C<at::BFloat16, has_bias, BLOCK_N> {
|
||||
static inline void apply(
|
||||
at::BFloat16* __restrict__ C,
|
||||
const int32_t* __restrict__ Ctmp,
|
||||
const int32_t* __restrict__ Bcomp,
|
||||
const float* __restrict__ bias,
|
||||
float As,
|
||||
const float* __restrict__ Bs) {
|
||||
constexpr int COLS = BLOCK_N / 16;
|
||||
static_assert(COLS % 2 == 0);
|
||||
|
||||
__m512 vc[COLS];
|
||||
__m512 vd0 = _mm512_set1_ps(As);
|
||||
|
||||
auto compute = [&](auto col) {
|
||||
__m512 vd1 = _mm512_loadu_ps(Bs + col * 16);
|
||||
__m512i vcomp = _mm512_loadu_si512(Bcomp + col * 16);
|
||||
__m512i vc32 = _mm512_loadu_si512(Ctmp + col * 16);
|
||||
vc[col] = _mm512_cvtepi32_ps(_mm512_sub_epi32(vc32, vcomp));
|
||||
if constexpr (has_bias) {
|
||||
__m512 vbias = _mm512_loadu_ps(bias + col * 16);
|
||||
vc[col] = _mm512_fmadd_ps(_mm512_mul_ps(vc[col], vd0), vd1, vbias);
|
||||
} else {
|
||||
vc[col] = _mm512_mul_ps(_mm512_mul_ps(vc[col], vd0), vd1);
|
||||
}
|
||||
};
|
||||
Unroll<COLS>{}(compute);
|
||||
|
||||
auto storec = [&](auto col) {
|
||||
// for COLS = 2, 4 use 512bit store
|
||||
if constexpr (col % 2 == 0) {
|
||||
_mm512_storeu_si512(
|
||||
reinterpret_cast<__m512i*>((C + col * 16)), (__m512i)(_mm512_cvtne2ps_pbh(vc[col + 1], vc[col + 0])));
|
||||
}
|
||||
};
|
||||
Unroll<COLS>{}(storec);
|
||||
}
|
||||
};
|
||||
#endif
|
||||
|
||||
template <typename scalar_t, bool has_bias, int BLOCK_M, int BLOCK_N>
|
||||
struct tinygemm_kernel_nn {
|
||||
static inline void apply(
|
||||
const uint8_t* __restrict__ A, const int8_t* __restrict__ B, scalar_t* __restrict__ C,
|
||||
const float* __restrict__ As, const float* __restrict__ Bs, const int32_t* __restrict__ Bcomp,
|
||||
const float* __restrict__ bias, int64_t K, int64_t lda, int64_t ldb, int64_t ldc) {
|
||||
const uint8_t* __restrict__ A,
|
||||
const int8_t* __restrict__ B,
|
||||
scalar_t* __restrict__ C,
|
||||
const float* __restrict__ As,
|
||||
const float* __restrict__ Bs,
|
||||
const int32_t* __restrict__ Bcomp,
|
||||
const float* __restrict__ bias,
|
||||
int64_t K,
|
||||
int64_t lda,
|
||||
int64_t ldb,
|
||||
int64_t ldc) {
|
||||
TORCH_CHECK(false, "tinygemm_kernel_nn: scalar path not implemented!");
|
||||
}
|
||||
};
|
||||
@@ -23,10 +86,17 @@ struct tinygemm_kernel_nn {
|
||||
template <bool has_bias, int BLOCK_M, int BLOCK_N>
|
||||
struct tinygemm_kernel_nn<at::BFloat16, has_bias, BLOCK_M, BLOCK_N> {
|
||||
static inline void apply(
|
||||
const uint8_t* __restrict__ A, const int8_t* __restrict__ B, at::BFloat16* __restrict__ C,
|
||||
const float* __restrict__ As, const float* __restrict__ Bs, const int32_t* __restrict__ Bcomp,
|
||||
const float* __restrict__ bias, int64_t K, int64_t lda, int64_t ldb, int64_t ldc) {
|
||||
|
||||
const uint8_t* __restrict__ A,
|
||||
const int8_t* __restrict__ B,
|
||||
at::BFloat16* __restrict__ C,
|
||||
const float* __restrict__ As,
|
||||
const float* __restrict__ Bs,
|
||||
const int32_t* __restrict__ Bcomp,
|
||||
const float* __restrict__ bias,
|
||||
int64_t K,
|
||||
int64_t lda,
|
||||
int64_t ldb,
|
||||
int64_t ldc) {
|
||||
constexpr int ROWS = BLOCK_M;
|
||||
constexpr int COLS = BLOCK_N / 16;
|
||||
static_assert(COLS % 2 == 0);
|
||||
@@ -38,10 +108,10 @@ struct tinygemm_kernel_nn<at::BFloat16, has_bias, BLOCK_M, BLOCK_N> {
|
||||
__m512i vb[COLS];
|
||||
__m512i vc[ROWS * COLS];
|
||||
__m512i vcomp[COLS];
|
||||
__m512 vd0;
|
||||
__m512 vd1[COLS];
|
||||
__m512 vd0;
|
||||
__m512 vd1[COLS];
|
||||
|
||||
// oops! 4x4 spills but luckily we use 4x2
|
||||
// oops! 4x4 spills but we use 4x2
|
||||
__m512 vbias[COLS];
|
||||
|
||||
// [NOTE]: s8s8 igemm compensation in avx512-vnni
|
||||
@@ -54,14 +124,12 @@ struct tinygemm_kernel_nn<at::BFloat16, has_bias, BLOCK_M, BLOCK_N> {
|
||||
// 1) 128 * b is pre-computed when packing B to vnni formats
|
||||
// 2) a + 128 is fused when dynamically quantize A
|
||||
//
|
||||
auto loadc = [&](auto i) {
|
||||
vc[i] = _mm512_set1_epi32(0);
|
||||
};
|
||||
auto loadc = [&](auto i) { vc[i] = _mm512_set1_epi32(0); };
|
||||
Unroll<ROWS * COLS>{}(loadc);
|
||||
|
||||
const int64_t K4 = K >> 2;
|
||||
const int64_t lda4 = lda >> 2;
|
||||
const int64_t ldb4 = ldb; // ldb * 4 >> 2;
|
||||
const int64_t ldb4 = ldb; // ldb * 4 >> 2;
|
||||
const int32_t* a_ptr = reinterpret_cast<const int32_t*>(A);
|
||||
const int32_t* b_ptr = reinterpret_cast<const int32_t*>(B);
|
||||
|
||||
@@ -89,7 +157,7 @@ struct tinygemm_kernel_nn<at::BFloat16, has_bias, BLOCK_M, BLOCK_N> {
|
||||
constexpr int col = i % COLS;
|
||||
|
||||
// load a scale
|
||||
if constexpr(col == 0) {
|
||||
if constexpr (col == 0) {
|
||||
vd0 = _mm512_set1_ps(As[row]);
|
||||
}
|
||||
// load b scale and vcomp per 2 vectors
|
||||
@@ -120,8 +188,7 @@ struct tinygemm_kernel_nn<at::BFloat16, has_bias, BLOCK_M, BLOCK_N> {
|
||||
}
|
||||
|
||||
_mm512_storeu_si512(
|
||||
reinterpret_cast<__m512i*>((C + row * ldc + col * 16)),
|
||||
(__m512i)(_mm512_cvtne2ps_pbh(vc1, vc0)));
|
||||
reinterpret_cast<__m512i*>((C + row * ldc + col * 16)), (__m512i)(_mm512_cvtne2ps_pbh(vc1, vc0)));
|
||||
}
|
||||
};
|
||||
Unroll<ROWS * COLS>{}(storec);
|
||||
@@ -129,11 +196,19 @@ struct tinygemm_kernel_nn<at::BFloat16, has_bias, BLOCK_M, BLOCK_N> {
|
||||
};
|
||||
#endif
|
||||
|
||||
#define LAUNCH_TINYGEMM_KERNEL_NN(MB_SIZE, NB_SIZE) \
|
||||
tinygemm_kernel_nn<scalar_t, has_bias, MB_SIZE, NB_SIZE>::apply( \
|
||||
A + mb_start * lda, B + nb_start * 4, C + mb_start * ldc + nb_start, \
|
||||
As + mb_start, Bs + nb_start, Bcomp + nb_start, \
|
||||
has_bias ? bias + nb_start : nullptr, K, lda, ldb, ldc);
|
||||
#define LAUNCH_TINYGEMM_KERNEL_NN(MB_SIZE, NB_SIZE) \
|
||||
tinygemm_kernel_nn<scalar_t, has_bias, MB_SIZE, NB_SIZE>::apply( \
|
||||
A + mb_start * lda, \
|
||||
B + nb_start * 4, \
|
||||
C + mb_start * ldc + nb_start, \
|
||||
As + mb_start, \
|
||||
Bs + nb_start, \
|
||||
Bcomp + nb_start, \
|
||||
has_bias ? bias + nb_start : nullptr, \
|
||||
K, \
|
||||
lda, \
|
||||
ldb, \
|
||||
ldc);
|
||||
|
||||
template <typename scalar_t, bool has_bias>
|
||||
void tinygemm_kernel(
|
||||
@@ -151,10 +226,20 @@ void tinygemm_kernel(
|
||||
int64_t ldb,
|
||||
int64_t ldc,
|
||||
bool brg) {
|
||||
|
||||
// B compensation
|
||||
const int32_t* Bcomp = reinterpret_cast<const int32_t*>(B + block_size_n() * K);
|
||||
|
||||
if (brg) {
|
||||
constexpr int BLOCK_N = block_size_n();
|
||||
at::native::cpublas::brgemm(M, N, K, lda, ldb, BLOCK_N, /* add_C */ false, A, B, Ctmp);
|
||||
|
||||
// apply compensation and scale
|
||||
for (int64_t m = 0; m < M; ++m) {
|
||||
scale_C<scalar_t, has_bias, BLOCK_N>::apply(C + m * ldc, Ctmp + m * BLOCK_N, Bcomp, bias, As[m], Bs);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// pattern: 1-4-16
|
||||
constexpr int64_t BLOCK_M = 4;
|
||||
constexpr int64_t BLOCK_N = 64;
|
||||
@@ -167,26 +252,43 @@ void tinygemm_kernel(
|
||||
int64_t nb_start = nb * BLOCK_N;
|
||||
int64_t nb_size = std::min(BLOCK_N, N - nb_start);
|
||||
|
||||
switch(mb_size << 4 | nb_size >> 4) {
|
||||
switch (mb_size << 4 | nb_size >> 4) {
|
||||
// mb_size = 1
|
||||
case 0x12: LAUNCH_TINYGEMM_KERNEL_NN(1, 32); break;
|
||||
case 0x14: LAUNCH_TINYGEMM_KERNEL_NN(1, 64); break;
|
||||
case 0x12:
|
||||
LAUNCH_TINYGEMM_KERNEL_NN(1, 32);
|
||||
break;
|
||||
case 0x14:
|
||||
LAUNCH_TINYGEMM_KERNEL_NN(1, 64);
|
||||
break;
|
||||
// mb_size = 2
|
||||
case 0x22: LAUNCH_TINYGEMM_KERNEL_NN(2, 32); break;
|
||||
case 0x24: LAUNCH_TINYGEMM_KERNEL_NN(2, 64); break;
|
||||
case 0x22:
|
||||
LAUNCH_TINYGEMM_KERNEL_NN(2, 32);
|
||||
break;
|
||||
case 0x24:
|
||||
LAUNCH_TINYGEMM_KERNEL_NN(2, 64);
|
||||
break;
|
||||
// mb_size = 3
|
||||
case 0x32: LAUNCH_TINYGEMM_KERNEL_NN(3, 32); break;
|
||||
case 0x34: LAUNCH_TINYGEMM_KERNEL_NN(3, 64); break;
|
||||
case 0x32:
|
||||
LAUNCH_TINYGEMM_KERNEL_NN(3, 32);
|
||||
break;
|
||||
case 0x34:
|
||||
LAUNCH_TINYGEMM_KERNEL_NN(3, 64);
|
||||
break;
|
||||
// mb_size = 4
|
||||
case 0x42: LAUNCH_TINYGEMM_KERNEL_NN(4, 32); break;
|
||||
case 0x44: LAUNCH_TINYGEMM_KERNEL_NN(4, 64); break;
|
||||
default: TORCH_CHECK(false, "Unexpected block size, ", mb_size, "x", nb_size);
|
||||
case 0x42:
|
||||
LAUNCH_TINYGEMM_KERNEL_NN(4, 32);
|
||||
break;
|
||||
case 0x44:
|
||||
LAUNCH_TINYGEMM_KERNEL_NN(4, 64);
|
||||
break;
|
||||
default:
|
||||
TORCH_CHECK(false, "Unexpected block size, ", mb_size, "x", "nb_size");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template<typename scalar_t>
|
||||
template <typename scalar_t>
|
||||
void int8_scaled_mm_kernel_impl(
|
||||
scalar_t* __restrict__ out,
|
||||
const uint8_t* __restrict__ mat1,
|
||||
@@ -197,28 +299,22 @@ void int8_scaled_mm_kernel_impl(
|
||||
int64_t M,
|
||||
int64_t N,
|
||||
int64_t K) {
|
||||
|
||||
constexpr int64_t BLOCK_M = block_size_m();
|
||||
constexpr int64_t BLOCK_N = block_size_n();
|
||||
const int64_t MB = div_up(M, BLOCK_M);
|
||||
const int64_t NB = div_up(N, BLOCK_N);
|
||||
|
||||
// TODO: brgemm u8s8 depends on PyTorch 2.7 release.
|
||||
const bool use_brgemm = false;
|
||||
const bool use_brgemm = can_use_brgemm<int8_t>(M);
|
||||
|
||||
// K + 4 after compensation
|
||||
const int64_t packed_row_size = get_row_size<int8_t>(K);
|
||||
|
||||
AT_DISPATCH_BOOL(bias != nullptr, has_bias, [&] {
|
||||
at::parallel_for(0, MB * NB, 0, [&](int64_t begin, int64_t end) {
|
||||
int64_t mb{0}, nb{0};
|
||||
data_index_init(begin, mb, MB, nb, NB);
|
||||
|
||||
parallel_2d(MB, NB, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) {
|
||||
// for brgemm, use int32_t for accumulate
|
||||
alignas(64) int32_t Ctmp[BLOCK_M * BLOCK_N];
|
||||
|
||||
for (int i = begin; i < end; ++i) {
|
||||
UNUSED(i);
|
||||
loop_2d<int8_t>(mb0, mb1, nb0, nb1, BLOCK_N * K, [&](int64_t mb, int64_t nb, int64_t nb_offset) {
|
||||
int mb_start = mb * BLOCK_M;
|
||||
int mb_size = std::min(M - mb_start, BLOCK_M);
|
||||
int nb_start = nb * BLOCK_N;
|
||||
@@ -239,10 +335,7 @@ void int8_scaled_mm_kernel_impl(
|
||||
/* ldb */ nb_size,
|
||||
/* ldc */ N,
|
||||
/* brg */ use_brgemm);
|
||||
|
||||
// move to the next index
|
||||
data_index_step(mb, MB, nb, NB);
|
||||
}
|
||||
});
|
||||
|
||||
if (use_brgemm) {
|
||||
at::native::cpublas::brgemm_release();
|
||||
@@ -251,28 +344,47 @@ void int8_scaled_mm_kernel_impl(
|
||||
});
|
||||
}
|
||||
|
||||
} // anonymous namespace
|
||||
} // anonymous namespace
|
||||
|
||||
// tinygemm interface
|
||||
template <typename scalar_t>
|
||||
void tinygemm_kernel(const uint8_t* __restrict__ A, const int8_t* __restrict__ B, scalar_t* __restrict__ C,
|
||||
int32_t* __restrict__ Ctmp, const float* __restrict__ As, const float* __restrict__ Bs,
|
||||
int64_t M, int64_t N, int64_t K, int64_t lda, int64_t ldb, int64_t ldc, bool brg) {
|
||||
void tinygemm_kernel(
|
||||
const uint8_t* __restrict__ A,
|
||||
const int8_t* __restrict__ B,
|
||||
scalar_t* __restrict__ C,
|
||||
int32_t* __restrict__ Ctmp,
|
||||
const float* __restrict__ As,
|
||||
const float* __restrict__ Bs,
|
||||
int64_t M,
|
||||
int64_t N,
|
||||
int64_t K,
|
||||
int64_t lda,
|
||||
int64_t ldb,
|
||||
int64_t ldc,
|
||||
bool brg) {
|
||||
tinygemm_kernel<scalar_t, false>(A, B, C, Ctmp, As, Bs, nullptr, M, N, K, lda, ldb, ldc, brg);
|
||||
}
|
||||
|
||||
#define INSTANTIATE_TINYGEMM_TEMPLATE(TYPE) \
|
||||
template void tinygemm_kernel<TYPE>( \
|
||||
const uint8_t* __restrict__ A, const int8_t* __restrict__ B, TYPE* __restrict__ C, \
|
||||
int32_t* __restrict__ Ctmp, const float* __restrict__ As, const float* __restrict__ Bs, \
|
||||
int64_t M, int64_t N, int64_t K, int64_t lda, int64_t ldb, int64_t ldc, bool brg)
|
||||
#define INSTANTIATE_TINYGEMM_TEMPLATE(TYPE) \
|
||||
template void tinygemm_kernel<TYPE>( \
|
||||
const uint8_t* __restrict__ A, \
|
||||
const int8_t* __restrict__ B, \
|
||||
TYPE* __restrict__ C, \
|
||||
int32_t* __restrict__ Ctmp, \
|
||||
const float* __restrict__ As, \
|
||||
const float* __restrict__ Bs, \
|
||||
int64_t M, \
|
||||
int64_t N, \
|
||||
int64_t K, \
|
||||
int64_t lda, \
|
||||
int64_t ldb, \
|
||||
int64_t ldc, \
|
||||
bool brg)
|
||||
|
||||
INSTANTIATE_TINYGEMM_TEMPLATE(at::BFloat16);
|
||||
INSTANTIATE_TINYGEMM_TEMPLATE(at::Half);
|
||||
|
||||
std::tuple<at::Tensor, at::Tensor> per_token_quant_int8_cpu(at::Tensor& A) {
|
||||
RECORD_FUNCTION("sgl-kernel::per_token_quant_int8_cpu", std::vector<c10::IValue>({A}));
|
||||
|
||||
CHECK_LAST_DIM_CONTIGUOUS_INPUT(A);
|
||||
CHECK_DIM(2, A);
|
||||
|
||||
@@ -281,8 +393,7 @@ std::tuple<at::Tensor, at::Tensor> per_token_quant_int8_cpu(at::Tensor& A) {
|
||||
int64_t lda = A.stride(0);
|
||||
|
||||
const auto st = A.scalar_type();
|
||||
TORCH_CHECK(st == at::kBFloat16 || st == at::kHalf,
|
||||
"per_token_quant_int8: expect A to be bfloat16 or half.");
|
||||
TORCH_CHECK(st == at::kBFloat16 || st == at::kHalf, "per_token_quant_int8: expect A to be bfloat16 or half.");
|
||||
|
||||
auto Aq = at::empty({M, K}, A.options().dtype(at::kByte));
|
||||
auto As = at::empty({M}, A.options().dtype(at::kFloat));
|
||||
@@ -292,13 +403,9 @@ std::tuple<at::Tensor, at::Tensor> per_token_quant_int8_cpu(at::Tensor& A) {
|
||||
float* __restrict__ As_data = As.data_ptr<float>();
|
||||
const scalar_t* __restrict__ A_data = A.data_ptr<scalar_t>();
|
||||
|
||||
at::parallel_for(0, M, 0, [&] (int64_t begin, int64_t end) {
|
||||
at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) {
|
||||
for (int64_t m = begin; m < end; ++m) {
|
||||
quantize_row_int8<scalar_t>(
|
||||
Aq_data + m * K,
|
||||
As_data[m],
|
||||
A_data + m * lda,
|
||||
K);
|
||||
quantize_row_int8<scalar_t>(Aq_data + m * K, As_data[m], A_data + m * lda, K);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -315,11 +422,14 @@ std::tuple<at::Tensor, at::Tensor> per_token_quant_int8_cpu(at::Tensor& A) {
|
||||
// bias : [N]
|
||||
// out : [M, N]
|
||||
//
|
||||
at::Tensor int8_scaled_mm_cpu(at::Tensor& mat1, at::Tensor& mat2,
|
||||
at::Tensor& scales1, at::Tensor& scales2,
|
||||
std::optional<at::Tensor>& bias, at::ScalarType out_dtype, bool is_vnni) {
|
||||
RECORD_FUNCTION("sgl-kernel::int8_scaled_mm_cpu", std::vector<c10::IValue>({mat1, mat2, scales1, scales2, bias}));
|
||||
|
||||
at::Tensor int8_scaled_mm_cpu(
|
||||
at::Tensor& mat1,
|
||||
at::Tensor& mat2,
|
||||
at::Tensor& scales1,
|
||||
at::Tensor& scales2,
|
||||
const std::optional<at::Tensor>& bias,
|
||||
at::ScalarType out_dtype,
|
||||
bool is_vnni) {
|
||||
auto packed_w = is_vnni ? mat2 : convert_weight_packed(mat2);
|
||||
|
||||
CHECK_INPUT(mat1);
|
||||
@@ -340,7 +450,8 @@ at::Tensor int8_scaled_mm_cpu(at::Tensor& mat1, at::Tensor& mat2,
|
||||
|
||||
TORCH_CHECK(mat1.scalar_type() == at::kByte, "int8_scaled_mm: expect mat1 to be uint8.");
|
||||
TORCH_CHECK(mat2.scalar_type() == at::kChar, "int8_scaled_mm: expect mat2 to be int8.");
|
||||
TORCH_CHECK(scales1.scalar_type() == at::kFloat && scales2.scalar_type() == at::kFloat,
|
||||
TORCH_CHECK(
|
||||
scales1.scalar_type() == at::kFloat && scales2.scalar_type() == at::kFloat,
|
||||
"int8_scaled_mm: expect scales to be float32.");
|
||||
|
||||
auto out = at::empty({M, N}, mat1.options().dtype(out_dtype));
|
||||
@@ -368,10 +479,13 @@ at::Tensor int8_scaled_mm_cpu(at::Tensor& mat1, at::Tensor& mat2,
|
||||
}
|
||||
|
||||
// fused `per_token_quant_int8_cpu` and `int8_scaled_mm_cpu`
|
||||
at::Tensor int8_scaled_mm_with_quant(at::Tensor& mat1, at::Tensor& mat2, at::Tensor& scales2,
|
||||
const std::optional<at::Tensor>& bias, at::ScalarType out_dtype, bool is_vnni) {
|
||||
RECORD_FUNCTION("sgl-kernel::int8_scaled_mm_cpu", std::vector<c10::IValue>({mat1, mat2, scales2, bias}));
|
||||
|
||||
at::Tensor int8_scaled_mm_with_quant(
|
||||
at::Tensor& mat1,
|
||||
at::Tensor& mat2,
|
||||
at::Tensor& scales2,
|
||||
const std::optional<at::Tensor>& bias,
|
||||
at::ScalarType out_dtype,
|
||||
bool is_vnni) {
|
||||
auto packed_w = is_vnni ? mat2 : convert_weight_packed(mat2);
|
||||
|
||||
CHECK_LAST_DIM_CONTIGUOUS_INPUT(mat1);
|
||||
@@ -390,14 +504,10 @@ at::Tensor int8_scaled_mm_with_quant(at::Tensor& mat1, at::Tensor& mat2, at::Ten
|
||||
CHECK_EQ(scales2.numel(), N);
|
||||
|
||||
const auto st = mat1.scalar_type();
|
||||
TORCH_CHECK(st == at::kBFloat16 || st == at::kHalf,
|
||||
"int8_scaled_mm_with_quant: expect A to be bfloat16 or half.");
|
||||
TORCH_CHECK(st == out_dtype,
|
||||
"int8_scaled_mm_with_quant: expect A has same dtype with out_dtype.");
|
||||
TORCH_CHECK(mat2.scalar_type() == at::kChar,
|
||||
"int8_scaled_mm_with_quant: expect mat2 to be int8.");
|
||||
TORCH_CHECK(scales2.scalar_type() == at::kFloat,
|
||||
"int8_scaled_mm_with_quant: expect scales to be float32.");
|
||||
TORCH_CHECK(st == at::kBFloat16 || st == at::kHalf, "int8_scaled_mm_with_quant: expect A to be bfloat16 or half.");
|
||||
TORCH_CHECK(st == out_dtype, "int8_scaled_mm_with_quant: expect A has same dtype with out_dtype.");
|
||||
TORCH_CHECK(mat2.scalar_type() == at::kChar, "int8_scaled_mm_with_quant: expect mat2 to be int8.");
|
||||
TORCH_CHECK(scales2.scalar_type() == at::kFloat, "int8_scaled_mm_with_quant: expect scales to be float32.");
|
||||
|
||||
const int64_t buffer_size = M * K + M * sizeof(float);
|
||||
auto buffer = at::empty({buffer_size}, mat1.options().dtype(at::kByte));
|
||||
@@ -415,13 +525,9 @@ at::Tensor int8_scaled_mm_with_quant(at::Tensor& mat1, at::Tensor& mat2, at::Ten
|
||||
float* __restrict__ As_data = (float*)((void*)(Aq_data + M * K));
|
||||
const scalar_t* __restrict__ A_data = mat1.data_ptr<scalar_t>();
|
||||
|
||||
at::parallel_for(0, M, 0, [&] (int64_t begin, int64_t end) {
|
||||
at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) {
|
||||
for (int64_t m = begin; m < end; ++m) {
|
||||
quantize_row_int8<scalar_t>(
|
||||
Aq_data + m * K,
|
||||
As_data[m],
|
||||
A_data + m * lda,
|
||||
K);
|
||||
quantize_row_int8<scalar_t>(Aq_data + m * K, As_data[m], A_data + m * lda, K);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
+259
-306
@@ -1,12 +1,13 @@
|
||||
// Adapted from
|
||||
// https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu
|
||||
|
||||
#include "common.h"
|
||||
#include "vec.h"
|
||||
#include "gemm.h"
|
||||
|
||||
// clang-format off
|
||||
|
||||
#include "moe.h"
|
||||
|
||||
#include "common.h"
|
||||
#include "gemm.h"
|
||||
|
||||
namespace {
|
||||
|
||||
// [NOTE]: Fused MoE kernel with AMX
|
||||
@@ -30,109 +31,6 @@ namespace {
|
||||
// 3. abstract at::native::cpublas::brgemm with WoQ gemm (M = 1 & M != 1)
|
||||
//
|
||||
|
||||
template <typename scalar_t>
|
||||
inline void fill_stub(scalar_t* __restrict__ out, scalar_t val, int64_t size) {
|
||||
using Vec = at::vec::Vectorized<scalar_t>;
|
||||
const Vec data_vec(val);
|
||||
at::vec::map<scalar_t>([data_vec](Vec out) { return out = data_vec; }, out, out, size);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
inline void copy_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ input, int64_t size) {
|
||||
using Vec = at::vec::Vectorized<scalar_t>;
|
||||
// no remainder
|
||||
#pragma GCC unroll 4
|
||||
for (int64_t d = 0; d < size; d += Vec::size()) {
|
||||
Vec data = Vec::loadu(input + d);
|
||||
data.store(out + d);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
inline void copy_mul_stub(scalar_t* __restrict__ out, const float* __restrict__ input, float weight, int64_t size) {
|
||||
using bVec = at::vec::Vectorized<scalar_t>;
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
constexpr int kVecSize = bVec::size();
|
||||
const fVec weight_vec = fVec(weight);
|
||||
int64_t d;
|
||||
#pragma GCC unroll 4
|
||||
for (d = 0; d <= size - kVecSize; d += kVecSize) {
|
||||
fVec data0 = fVec::loadu(input + d) * weight_vec;
|
||||
fVec data1 = fVec::loadu(input + d + fVec::size()) * weight_vec;
|
||||
bVec out_vec = convert_from_float_ext<scalar_t>(data0, data1);
|
||||
out_vec.store(out + d);
|
||||
}
|
||||
for (; d < size; ++d) {
|
||||
out[d] = static_cast<scalar_t>(input[d] * weight);
|
||||
}
|
||||
}
|
||||
|
||||
// acc from [topk, K] to [K]
|
||||
template <typename scalar_t>
|
||||
inline void sum_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ input, int64_t topk, int64_t K) {
|
||||
using bVec = at::vec::Vectorized<scalar_t>;
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
constexpr int kVecSize = bVec::size();
|
||||
if (topk == 1) {
|
||||
// do copy for topk = 1
|
||||
copy_stub(out, input, K);
|
||||
} else {
|
||||
// do sum for topk != 1
|
||||
int64_t d;
|
||||
#pragma GCC unroll 4
|
||||
for (d = 0; d <= K - kVecSize; d += kVecSize) {
|
||||
fVec sum_fvec0 = fVec(0.f);
|
||||
fVec sum_fvec1 = fVec(0.f);
|
||||
for (int t = 0; t < topk; ++t) {
|
||||
bVec x_bvec = bVec::loadu(input + t * K + d);
|
||||
fVec x_fvec0, x_fvec1;
|
||||
std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec);
|
||||
|
||||
sum_fvec0 += x_fvec0;
|
||||
sum_fvec1 += x_fvec1;
|
||||
}
|
||||
bVec out_bvec = convert_from_float_ext<scalar_t>(sum_fvec0, sum_fvec1);
|
||||
out_bvec.store(out + d);
|
||||
}
|
||||
for (; d < K; ++d) {
|
||||
float sum_val = 0.f;
|
||||
for (int t = 0; t < topk; ++t) {
|
||||
sum_val += static_cast<float>(input[t * K + d]);
|
||||
}
|
||||
out[d] = static_cast<scalar_t>(sum_val);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// out = input + input2 * scale
|
||||
template <typename scalar_t>
|
||||
inline void add_mul_stub(scalar_t* __restrict__ out, const float* __restrict__ input,
|
||||
const scalar_t* __restrict__ input2, float scale, int64_t size) {
|
||||
|
||||
using bVec = at::vec::Vectorized<scalar_t>;
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
constexpr int kVecSize = bVec::size();
|
||||
const fVec s_vec = fVec(scale);
|
||||
int64_t d;
|
||||
#pragma GCC unroll 4
|
||||
for (d = 0; d <= size - kVecSize; d += kVecSize) {
|
||||
fVec x0 = fVec::loadu(input + d);
|
||||
fVec x1 = fVec::loadu(input + d + fVec::size());
|
||||
|
||||
bVec y_bvec = bVec::loadu(input2 + d);
|
||||
fVec y0, y1;
|
||||
std::tie(y0, y1) = at::vec::convert_to_float(y_bvec);
|
||||
|
||||
x0 = x0 + y0 * s_vec;
|
||||
x1 = x1 + y1 * s_vec;
|
||||
bVec out_vec = convert_from_float_ext<scalar_t>(x0, x1);
|
||||
out_vec.store(out + d);
|
||||
}
|
||||
for (; d < size; ++d) {
|
||||
out[d] = static_cast<scalar_t>(input[d] + float(input2[d]) * scale);
|
||||
}
|
||||
}
|
||||
|
||||
template <int BLOCK_M>
|
||||
int moe_align_block_size(
|
||||
int32_t* __restrict__ sorted_ids,
|
||||
@@ -144,8 +42,7 @@ int moe_align_block_size(
|
||||
int num_experts,
|
||||
int numel,
|
||||
int num_threads) {
|
||||
|
||||
#define T_INDEX(tt) total_cnts + (tt) * num_experts
|
||||
#define T_INDEX(tt) total_cnts + (tt) * num_experts
|
||||
|
||||
// accumulate count of expert ids locally
|
||||
at::parallel_for(0, numel, 0, [&](int begin, int end) {
|
||||
@@ -160,8 +57,7 @@ int moe_align_block_size(
|
||||
using iVec = at::vec::Vectorized<int32_t>;
|
||||
for (int t = 0; t < num_threads; ++t) {
|
||||
at::vec::map2<int32_t>(
|
||||
[](iVec x, iVec y) { return x + y; },
|
||||
T_INDEX(t + 1), T_INDEX(t + 1), T_INDEX(t), num_experts);
|
||||
[](iVec x, iVec y) { return x + y; }, T_INDEX(t + 1), T_INDEX(t + 1), T_INDEX(t), num_experts);
|
||||
}
|
||||
|
||||
// the last row holds sums of each experts
|
||||
@@ -201,7 +97,9 @@ int moe_align_block_size(
|
||||
// padding value for sorted_ids: numel
|
||||
auto sorted_id_size = [=](const int32_t* sorted_ids_ptr) {
|
||||
for (int d = 0; d < BLOCK_M; ++d) {
|
||||
if (sorted_ids_ptr[d] == numel) { return d; }
|
||||
if (sorted_ids_ptr[d] == numel) {
|
||||
return d;
|
||||
}
|
||||
}
|
||||
return BLOCK_M;
|
||||
};
|
||||
@@ -215,7 +113,7 @@ int moe_align_block_size(
|
||||
offsets[mb + 1] = sorted_id_size(sorted_ids + mb * BLOCK_M);
|
||||
}
|
||||
});
|
||||
// TODO: do we need to vectorize this ?
|
||||
// TODO: do we need to vecterize this ?
|
||||
for (int mb = 0; mb < num_token_blocks; ++mb) {
|
||||
offsets[mb + 1] += offsets[mb];
|
||||
}
|
||||
@@ -236,7 +134,6 @@ inline void silu_and_mul(
|
||||
const float* __restrict__ input1, // y: y0, y1
|
||||
int64_t m_size,
|
||||
int64_t N) {
|
||||
|
||||
using bVec = at::vec::Vectorized<scalar_t>;
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
|
||||
@@ -269,8 +166,14 @@ inline void silu_and_mul(
|
||||
template <typename scalar_t, int BLOCK_M, int BLOCK_N>
|
||||
struct tinygemm_kernel_nn2 {
|
||||
static inline void apply(
|
||||
const scalar_t* __restrict__ A, const scalar_t* __restrict__ B0, const scalar_t* __restrict__ B1,
|
||||
scalar_t* __restrict__ C, int64_t K, int64_t lda, int64_t ldb, int64_t ldc) {
|
||||
const scalar_t* __restrict__ A,
|
||||
const scalar_t* __restrict__ B0,
|
||||
const scalar_t* __restrict__ B1,
|
||||
scalar_t* __restrict__ C,
|
||||
int64_t K,
|
||||
int64_t lda,
|
||||
int64_t ldb,
|
||||
int64_t ldc) {
|
||||
TORCH_CHECK(false, "tinygemm_kernel_nn: scalar path not implemented!");
|
||||
}
|
||||
};
|
||||
@@ -279,9 +182,14 @@ struct tinygemm_kernel_nn2 {
|
||||
template <int BLOCK_M, int BLOCK_N>
|
||||
struct tinygemm_kernel_nn2<at::BFloat16, BLOCK_M, BLOCK_N> {
|
||||
static inline void apply(
|
||||
const at::BFloat16* __restrict__ A, const at::BFloat16* __restrict__ B0, const at::BFloat16* __restrict__ B1,
|
||||
at::BFloat16* __restrict__ C, int64_t K, int64_t lda, int64_t ldb, int64_t ldc) {
|
||||
|
||||
const at::BFloat16* __restrict__ A,
|
||||
const at::BFloat16* __restrict__ B0,
|
||||
const at::BFloat16* __restrict__ B1,
|
||||
at::BFloat16* __restrict__ C,
|
||||
int64_t K,
|
||||
int64_t lda,
|
||||
int64_t ldb,
|
||||
int64_t ldc) {
|
||||
constexpr int ROWS = BLOCK_M;
|
||||
constexpr int COLS = BLOCK_N / 16;
|
||||
|
||||
@@ -304,7 +212,7 @@ struct tinygemm_kernel_nn2<at::BFloat16, BLOCK_M, BLOCK_N> {
|
||||
|
||||
const int64_t K2 = K >> 1;
|
||||
const int64_t lda2 = lda >> 1;
|
||||
const int64_t ldb2 = ldb; // ldb * 2 >> 1;
|
||||
const int64_t ldb2 = ldb; // ldb * 2 >> 1;
|
||||
const float* a_ptr = reinterpret_cast<const float*>(A);
|
||||
const float* b0_ptr = reinterpret_cast<const float*>(B0);
|
||||
const float* b1_ptr = reinterpret_cast<const float*>(B1);
|
||||
@@ -352,17 +260,16 @@ struct tinygemm_kernel_nn2<at::BFloat16, BLOCK_M, BLOCK_N> {
|
||||
_mm512_storeu_si512(
|
||||
reinterpret_cast<__m512i*>((C + row * ldc + col * 16)),
|
||||
(__m512i)(_mm512_cvtne2ps_pbh(__m512(x1), __m512(x0))));
|
||||
}
|
||||
}
|
||||
};
|
||||
Unroll<ROWS * COLS>{}(storec);
|
||||
}
|
||||
};
|
||||
#endif
|
||||
|
||||
#define LAUNCH_TINYGEMM_KERNEL_NN(MB_SIZE, NB_SIZE) \
|
||||
tinygemm_kernel_nn2<scalar_t, MB_SIZE, NB_SIZE>::apply( \
|
||||
A + mb_start * lda, B0 + nb_start * 2, B1 + nb_start * 2, \
|
||||
C + mb_start * ldc + nb_start, K, lda, ldb, ldc);
|
||||
#define LAUNCH_TINYGEMM_KERNEL_NN(MB_SIZE, NB_SIZE) \
|
||||
tinygemm_kernel_nn2<scalar_t, MB_SIZE, NB_SIZE>::apply( \
|
||||
A + mb_start * lda, B0 + nb_start * 2, B1 + nb_start * 2, C + mb_start * ldc + nb_start, K, lda, ldb, ldc);
|
||||
|
||||
template <typename scalar_t>
|
||||
void tinygemm_kernel(
|
||||
@@ -376,7 +283,6 @@ void tinygemm_kernel(
|
||||
int64_t lda,
|
||||
int64_t ldb,
|
||||
int64_t ldc) {
|
||||
|
||||
// pattern: 1-(2+2)-(8+8)
|
||||
constexpr int64_t BLOCK_M = 4;
|
||||
constexpr int64_t BLOCK_N = 32;
|
||||
@@ -389,16 +295,25 @@ void tinygemm_kernel(
|
||||
int64_t nb_start = nb * BLOCK_N;
|
||||
int64_t nb_size = std::min(BLOCK_N, N - nb_start);
|
||||
|
||||
switch(mb_size << 4 | nb_size >> 4) {
|
||||
switch (mb_size << 4 | nb_size >> 4) {
|
||||
// mb_size = 1
|
||||
case 0x12: LAUNCH_TINYGEMM_KERNEL_NN(1, 32); break;
|
||||
case 0x12:
|
||||
LAUNCH_TINYGEMM_KERNEL_NN(1, 32);
|
||||
break;
|
||||
// mb_size = 2
|
||||
case 0x22: LAUNCH_TINYGEMM_KERNEL_NN(2, 32); break;
|
||||
case 0x22:
|
||||
LAUNCH_TINYGEMM_KERNEL_NN(2, 32);
|
||||
break;
|
||||
// mb_size = 3
|
||||
case 0x32: LAUNCH_TINYGEMM_KERNEL_NN(3, 32); break;
|
||||
case 0x32:
|
||||
LAUNCH_TINYGEMM_KERNEL_NN(3, 32);
|
||||
break;
|
||||
// mb_size = 4
|
||||
case 0x42: LAUNCH_TINYGEMM_KERNEL_NN(4, 32); break;
|
||||
default: TORCH_CHECK(false, "Unexpected block size, ", mb_size, "x", nb_size);
|
||||
case 0x42:
|
||||
LAUNCH_TINYGEMM_KERNEL_NN(4, 32);
|
||||
break;
|
||||
default:
|
||||
TORCH_CHECK(false, "Unexpected block size, ", mb_size, "x", "nb_size");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -407,8 +322,13 @@ void tinygemm_kernel(
|
||||
template <typename scalar_t, int BLOCK_M, int BLOCK_N>
|
||||
struct tinygemm_kernel_nn {
|
||||
static inline void apply(
|
||||
const scalar_t* __restrict__ A, const scalar_t* __restrict__ B, float* __restrict__ C,
|
||||
int64_t K, int64_t lda, int64_t ldb, int64_t ldc) {
|
||||
const scalar_t* __restrict__ A,
|
||||
const scalar_t* __restrict__ B,
|
||||
float* __restrict__ C,
|
||||
int64_t K,
|
||||
int64_t lda,
|
||||
int64_t ldb,
|
||||
int64_t ldc) {
|
||||
TORCH_CHECK(false, "tinygemm_kernel_nn: scalar path not implemented!");
|
||||
}
|
||||
};
|
||||
@@ -417,9 +337,13 @@ struct tinygemm_kernel_nn {
|
||||
template <int BLOCK_M, int BLOCK_N>
|
||||
struct tinygemm_kernel_nn<at::BFloat16, BLOCK_M, BLOCK_N> {
|
||||
static inline void apply(
|
||||
const at::BFloat16* __restrict__ A, const at::BFloat16* __restrict__ B, float* __restrict__ C,
|
||||
int64_t K, int64_t lda, int64_t ldb, int64_t ldc) {
|
||||
|
||||
const at::BFloat16* __restrict__ A,
|
||||
const at::BFloat16* __restrict__ B,
|
||||
float* __restrict__ C,
|
||||
int64_t K,
|
||||
int64_t lda,
|
||||
int64_t ldb,
|
||||
int64_t ldc) {
|
||||
constexpr int ROWS = BLOCK_M;
|
||||
constexpr int COLS = BLOCK_N / 16;
|
||||
|
||||
@@ -432,14 +356,12 @@ struct tinygemm_kernel_nn<at::BFloat16, BLOCK_M, BLOCK_N> {
|
||||
__m512bh vb[COLS];
|
||||
__m512 vc[ROWS * COLS];
|
||||
|
||||
auto loadc = [&](auto i) {
|
||||
vc[i] = _mm512_set1_ps(0.f);
|
||||
};
|
||||
auto loadc = [&](auto i) { vc[i] = _mm512_set1_ps(0.f); };
|
||||
Unroll<ROWS * COLS>{}(loadc);
|
||||
|
||||
const int64_t K2 = K >> 1;
|
||||
const int64_t lda2 = lda >> 1;
|
||||
const int64_t ldb2 = ldb; // ldb * 2 >> 1;
|
||||
const int64_t ldb2 = ldb; // ldb * 2 >> 1;
|
||||
const float* a_ptr = reinterpret_cast<const float*>(A);
|
||||
const float* b_ptr = reinterpret_cast<const float*>(B);
|
||||
|
||||
@@ -466,17 +388,15 @@ struct tinygemm_kernel_nn<at::BFloat16, BLOCK_M, BLOCK_N> {
|
||||
constexpr int row = i / COLS;
|
||||
constexpr int col = i % COLS;
|
||||
_mm512_storeu_ps(reinterpret_cast<__m512*>(C + row * ldc + col * 16), vc[i]);
|
||||
|
||||
};
|
||||
Unroll<ROWS * COLS>{}(storec);
|
||||
}
|
||||
};
|
||||
#endif
|
||||
|
||||
#define LAUNCH_TINYGEMM_KERNEL_NN2(MB_SIZE, NB_SIZE) \
|
||||
tinygemm_kernel_nn<scalar_t, MB_SIZE, NB_SIZE>::apply( \
|
||||
A + mb_start * lda, B + nb_start * 2, C + mb_start * ldc + nb_start, \
|
||||
K, lda, ldb, ldc);
|
||||
#define LAUNCH_TINYGEMM_KERNEL_NN2(MB_SIZE, NB_SIZE) \
|
||||
tinygemm_kernel_nn<scalar_t, MB_SIZE, NB_SIZE>::apply( \
|
||||
A + mb_start * lda, B + nb_start * 2, C + mb_start * ldc + nb_start, K, lda, ldb, ldc);
|
||||
|
||||
template <typename scalar_t>
|
||||
void tinygemm_kernel(
|
||||
@@ -489,7 +409,6 @@ void tinygemm_kernel(
|
||||
int64_t lda,
|
||||
int64_t ldb,
|
||||
int64_t ldc) {
|
||||
|
||||
// pattern: 1-2-8
|
||||
constexpr int64_t BLOCK_M = 4;
|
||||
constexpr int64_t BLOCK_N = 32;
|
||||
@@ -502,16 +421,25 @@ void tinygemm_kernel(
|
||||
int64_t nb_start = nb * BLOCK_N;
|
||||
int64_t nb_size = std::min(BLOCK_N, N - nb_start);
|
||||
|
||||
switch(mb_size << 4 | nb_size >> 4) {
|
||||
switch (mb_size << 4 | nb_size >> 4) {
|
||||
// mb_size = 1
|
||||
case 0x12: LAUNCH_TINYGEMM_KERNEL_NN2(1, 32); break;
|
||||
case 0x12:
|
||||
LAUNCH_TINYGEMM_KERNEL_NN2(1, 32);
|
||||
break;
|
||||
// mb_size = 2
|
||||
case 0x22: LAUNCH_TINYGEMM_KERNEL_NN2(2, 32); break;
|
||||
case 0x22:
|
||||
LAUNCH_TINYGEMM_KERNEL_NN2(2, 32);
|
||||
break;
|
||||
// mb_size = 3
|
||||
case 0x32: LAUNCH_TINYGEMM_KERNEL_NN2(3, 32); break;
|
||||
case 0x32:
|
||||
LAUNCH_TINYGEMM_KERNEL_NN2(3, 32);
|
||||
break;
|
||||
// mb_size = 4
|
||||
case 0x42: LAUNCH_TINYGEMM_KERNEL_NN2(4, 32); break;
|
||||
default: TORCH_CHECK(false, "Unexpected block size, ", mb_size, "x", nb_size);
|
||||
case 0x42:
|
||||
LAUNCH_TINYGEMM_KERNEL_NN2(4, 32);
|
||||
break;
|
||||
default:
|
||||
TORCH_CHECK(false, "Unexpected block size, ", mb_size, "x", "nb_size");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -537,7 +465,6 @@ void fused_experts_kernel_impl(
|
||||
int64_t E,
|
||||
int64_t topk,
|
||||
int64_t num_tokens_post_pad) {
|
||||
|
||||
// handle 2 tiles per block
|
||||
constexpr int64_t BLOCK_M = block_size_m();
|
||||
constexpr int64_t BLOCK_N = block_size_n();
|
||||
@@ -552,39 +479,36 @@ void fused_experts_kernel_impl(
|
||||
const int64_t stride_e = 2 * N * K;
|
||||
const int64_t stride_n = K;
|
||||
|
||||
int64_t avg_M = std::max(int64_t(1), M * topk / E);
|
||||
const bool use_brgemm = can_use_brgemm<scalar_t>(avg_M);
|
||||
|
||||
// here we only parallel on half of 2N to fuse silu_and_mul with gemm
|
||||
at::parallel_for(0, MB * NB, 0, [&](int64_t begin, int64_t end) {
|
||||
parallel_2d(MB, NB, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) {
|
||||
// get local pointers
|
||||
int tid = at::get_thread_num();
|
||||
int tid = get_thread_num();
|
||||
scalar_t* __restrict__ A = A_tmp + tid * BLOCK_M * K;
|
||||
float* __restrict__ C0 = C_tmp + tid * 2 * BLOCK_M * BLOCK_N;
|
||||
float* __restrict__ C1 = C0 + BLOCK_M * BLOCK_N;
|
||||
|
||||
bool is_brgemm_used = false;
|
||||
|
||||
for (int64_t i = begin; i < end; ++i) {
|
||||
int64_t mb = i / NB;
|
||||
int64_t nb = i % NB;
|
||||
|
||||
// nb0 from top half and nb1 from bottom half
|
||||
int64_t nb0 = nb, nb1 = nb + NB;
|
||||
int64_t n_size = std::min(N - nb0 * BLOCK_N, BLOCK_N);
|
||||
loop_2d<scalar_t>(mb0, mb1, nb0, nb1, BLOCK_N * K * 2, [&](int64_t mb, int64_t nb, int64_t nb_offset) {
|
||||
// nb_upper from top half and nb_lower from bottom half
|
||||
int64_t nb_upper = nb, nb_lower = nb + NB;
|
||||
int64_t n_size = std::min(N - nb * BLOCK_N, BLOCK_N);
|
||||
|
||||
// B shape [K, n_size] in vnni format
|
||||
int32_t expert_id = expert_ids[mb];
|
||||
const scalar_t* __restrict__ B0 = packed_w1 + expert_id * stride_e + nb0 * BLOCK_N * stride_n;
|
||||
const scalar_t* __restrict__ B1 = packed_w1 + expert_id * stride_e + nb1 * BLOCK_N * stride_n;
|
||||
const scalar_t* __restrict__ B0 = packed_w1 + expert_id * stride_e + nb_upper * BLOCK_N * stride_n;
|
||||
const scalar_t* __restrict__ B1 = packed_w1 + expert_id * stride_e + nb_lower * BLOCK_N * stride_n;
|
||||
|
||||
// 1.a load A
|
||||
const int32_t* A_ids = sorted_ids + mb * BLOCK_M;
|
||||
int64_t m_size = offsets[mb + 1] - offsets[mb];
|
||||
|
||||
const bool use_brgemm = can_use_brgemm<scalar_t>(m_size);
|
||||
is_brgemm_used = is_brgemm_used || use_brgemm;
|
||||
|
||||
for (int64_t m = 0; m < m_size; ++m) {
|
||||
int32_t index = A_ids[m] / topk;
|
||||
copy_stub(A + m * K, input + index * K, K);
|
||||
if (nb_offset == 0) {
|
||||
// 1.a load A
|
||||
const int32_t* A_ids = sorted_ids + mb * BLOCK_M;
|
||||
for (int64_t m = 0; m < m_size; ++m) {
|
||||
int32_t index = A_ids[m] / topk;
|
||||
copy_stub(A + m * K, input + index * K, K);
|
||||
}
|
||||
}
|
||||
|
||||
if (use_brgemm) {
|
||||
@@ -616,12 +540,7 @@ void fused_experts_kernel_impl(
|
||||
|
||||
// 1.d silu and mul
|
||||
const int64_t offset = offsets[mb];
|
||||
silu_and_mul<scalar_t, BLOCK_N>(
|
||||
ic1 + offset * N + nb * BLOCK_N,
|
||||
C0,
|
||||
C1,
|
||||
m_size,
|
||||
N);
|
||||
silu_and_mul<scalar_t, BLOCK_N>(ic1 + offset * N + nb * BLOCK_N, C0, C1, m_size, N);
|
||||
} else {
|
||||
// fused 1.bcd: silu_and_mul(A @ B0, A @ B1)
|
||||
const int64_t offset = offsets[mb];
|
||||
@@ -637,9 +556,9 @@ void fused_experts_kernel_impl(
|
||||
/* ldb */ n_size,
|
||||
/* ldc */ N);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if (is_brgemm_used) {
|
||||
if (use_brgemm) {
|
||||
at::native::cpublas::brgemm_release();
|
||||
}
|
||||
});
|
||||
@@ -654,24 +573,16 @@ void fused_experts_kernel_impl(
|
||||
const int64_t stride_oc = IC;
|
||||
|
||||
// parallel on [MB2, NB2]
|
||||
at::parallel_for(0, MB2 * NB2, 0, [&](int64_t begin, int64_t end) {
|
||||
parallel_2d(MB2, NB2, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) {
|
||||
// get local pointers
|
||||
int tid = at::get_thread_num();
|
||||
int tid = get_thread_num();
|
||||
// we won't be using C1 for gemm2
|
||||
float* __restrict__ C = C_tmp + tid * 2 * BLOCK_M * BLOCK_N;
|
||||
|
||||
bool is_brgemm_used = false;
|
||||
|
||||
for (int64_t i = begin; i < end; ++i) {
|
||||
int64_t mb = i / NB2;
|
||||
int64_t nb = i % NB2;
|
||||
|
||||
loop_2d<scalar_t>(mb0, mb1, nb0, nb1, BLOCK_N * IC, [&](int64_t mb, int64_t nb, int64_t nb_offset) {
|
||||
int64_t m_size = offsets[mb + 1] - offsets[mb];
|
||||
int64_t n_size = std::min(OC - nb * BLOCK_N, BLOCK_N);
|
||||
|
||||
const bool use_brgemm = can_use_brgemm<scalar_t>(m_size);
|
||||
is_brgemm_used = is_brgemm_used || use_brgemm;
|
||||
|
||||
// A ptr from ic1 of [M * topk, N] in sorted order
|
||||
// so as to avoid copy A to tmp buffer again
|
||||
const scalar_t* __restrict__ A = ic1 + offsets[mb] * N;
|
||||
@@ -714,9 +625,9 @@ void fused_experts_kernel_impl(
|
||||
float weight = topk_weights[index];
|
||||
copy_mul_stub(ic2 + index * K + nb * BLOCK_N, C + m * BLOCK_N, weight, n_size);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if (is_brgemm_used) {
|
||||
if (use_brgemm) {
|
||||
at::native::cpublas::brgemm_release();
|
||||
}
|
||||
});
|
||||
@@ -743,7 +654,6 @@ void shared_expert_kernel_impl(
|
||||
int64_t M,
|
||||
int64_t N,
|
||||
int64_t K) {
|
||||
|
||||
// handle 2 tiles per block
|
||||
constexpr int64_t BLOCK_M = block_size_m();
|
||||
constexpr int64_t BLOCK_N = block_size_n();
|
||||
@@ -755,36 +665,29 @@ void shared_expert_kernel_impl(
|
||||
TORCH_CHECK(N % BLOCK_N == 0, "Fixme when N is not multiples of ", BLOCK_N);
|
||||
const int64_t stride_n = K;
|
||||
|
||||
const bool use_brgemm = can_use_brgemm<scalar_t>(M);
|
||||
|
||||
const bool apply_scaling_factor = fused_experts_out != nullptr;
|
||||
|
||||
// here we only parallel on half of 2N to fuse silu_and_mul with gemm
|
||||
at::parallel_for(0, MB * NB, 0, [&](int64_t begin, int64_t end) {
|
||||
parallel_2d(MB, NB, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) {
|
||||
// get local pointers
|
||||
int tid = at::get_thread_num();
|
||||
int tid = get_thread_num();
|
||||
float* __restrict__ C0 = C_tmp + tid * 2 * BLOCK_M * BLOCK_N;
|
||||
float* __restrict__ C1 = C0 + BLOCK_M * BLOCK_N;
|
||||
|
||||
bool is_brgemm_used = false;
|
||||
|
||||
for (int64_t i = begin; i < end; ++i) {
|
||||
int64_t mb = i / NB;
|
||||
int64_t nb = i % NB;
|
||||
|
||||
// nb0 from top half and nb1 from bottom half
|
||||
int64_t nb0 = nb, nb1 = nb + NB;
|
||||
int64_t n_size = std::min(N - nb0 * BLOCK_N, BLOCK_N);
|
||||
loop_2d<scalar_t>(mb0, mb1, nb0, nb1, BLOCK_N * K * 2, [&](int64_t mb, int64_t nb, int64_t nb_offset) {
|
||||
// nb_upper from top half and nb_lower from bottom half
|
||||
int64_t nb_upper = nb, nb_lower = nb + NB;
|
||||
int64_t n_size = std::min(N - nb * BLOCK_N, BLOCK_N);
|
||||
int64_t m_size = std::min(M - mb * BLOCK_M, BLOCK_M);
|
||||
|
||||
//int64_t mb_start = mb * BLOCK_M;
|
||||
//int64_t mb_size = std::min(M - mb_start, BLOCK_M);
|
||||
|
||||
// A shape [m_size, K]
|
||||
const scalar_t* A = input + mb * BLOCK_M * K;
|
||||
|
||||
// B shape [K, n_size] in vnni format
|
||||
const scalar_t* __restrict__ B0 = packed_w1 + nb0 * BLOCK_N * stride_n;
|
||||
const scalar_t* __restrict__ B1 = packed_w1 + nb1 * BLOCK_N * stride_n;
|
||||
|
||||
const bool use_brgemm = can_use_brgemm<scalar_t>(m_size);
|
||||
is_brgemm_used = is_brgemm_used || use_brgemm;
|
||||
const scalar_t* __restrict__ B0 = packed_w1 + nb_upper * BLOCK_N * stride_n;
|
||||
const scalar_t* __restrict__ B1 = packed_w1 + nb_lower * BLOCK_N * stride_n;
|
||||
|
||||
if (use_brgemm) {
|
||||
// 1.b gemm: C0 = A @ B0
|
||||
@@ -814,12 +717,7 @@ void shared_expert_kernel_impl(
|
||||
/* C */ C1);
|
||||
|
||||
// 1.d silu and mul
|
||||
silu_and_mul<scalar_t, BLOCK_N>(
|
||||
ic1 + mb * BLOCK_M * N + nb * BLOCK_N,
|
||||
C0,
|
||||
C1,
|
||||
m_size,
|
||||
N);
|
||||
silu_and_mul<scalar_t, BLOCK_N>(ic1 + mb * BLOCK_M * N + nb * BLOCK_N, C0, C1, m_size, N);
|
||||
} else {
|
||||
// fused 1.bcd: silu_and_mul(A @ B0, A @ B1)
|
||||
tinygemm_kernel(
|
||||
@@ -834,9 +732,9 @@ void shared_expert_kernel_impl(
|
||||
/* ldb */ n_size,
|
||||
/* ldc */ N);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if (is_brgemm_used) {
|
||||
if (use_brgemm) {
|
||||
at::native::cpublas::brgemm_release();
|
||||
}
|
||||
});
|
||||
@@ -850,24 +748,16 @@ void shared_expert_kernel_impl(
|
||||
const int64_t stride_oc = IC;
|
||||
|
||||
// parallel on [MB2, NB2]
|
||||
at::parallel_for(0, MB2 * NB2, 0, [&](int64_t begin, int64_t end) {
|
||||
parallel_2d(MB2, NB2, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) {
|
||||
// get local pointers
|
||||
int tid = at::get_thread_num();
|
||||
int tid = get_thread_num();
|
||||
// we won't be using C1 for gemm2
|
||||
float* __restrict__ C = C_tmp + tid * 2 * BLOCK_M * BLOCK_N;
|
||||
|
||||
bool is_brgemm_used = false;
|
||||
|
||||
for (int64_t i = begin; i < end; ++i) {
|
||||
int64_t mb = i / NB2;
|
||||
int64_t nb = i % NB2;
|
||||
|
||||
loop_2d<scalar_t>(mb0, mb1, nb0, nb1, BLOCK_N * IC, [&](int64_t mb, int64_t nb, int64_t nb_offset) {
|
||||
int64_t m_size = std::min(M - mb * BLOCK_M, BLOCK_M);
|
||||
int64_t n_size = std::min(OC - nb * BLOCK_N, BLOCK_N);
|
||||
|
||||
const bool use_brgemm = can_use_brgemm<scalar_t>(m_size);
|
||||
is_brgemm_used = is_brgemm_used || use_brgemm;
|
||||
|
||||
// A shape [m_size, IC]
|
||||
const scalar_t* __restrict__ A = ic1 + mb * BLOCK_M * N;
|
||||
|
||||
@@ -902,19 +792,21 @@ void shared_expert_kernel_impl(
|
||||
|
||||
// 2.b copy from C to output and add fused_experts_out
|
||||
scalar_t* __restrict__ out = output + mb * BLOCK_M * K + nb * BLOCK_N;
|
||||
const scalar_t* __restrict__ fused_out = fused_experts_out + mb * BLOCK_M * K + nb * BLOCK_N;
|
||||
const scalar_t* __restrict__ fused_out =
|
||||
apply_scaling_factor ? fused_experts_out + mb * BLOCK_M * K + nb * BLOCK_N : nullptr;
|
||||
for (int64_t m = 0; m < m_size; ++m) {
|
||||
add_mul_stub(out + m * K, C + m * BLOCK_N, fused_out + m * K, routed_scaling_factor, n_size);
|
||||
const scalar_t* __restrict__ fused_out_row = apply_scaling_factor ? (fused_out + m * K) : nullptr;
|
||||
add_mul_stub(out + m * K, C + m * BLOCK_N, fused_out_row, routed_scaling_factor, n_size);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if (is_brgemm_used) {
|
||||
if (use_brgemm) {
|
||||
at::native::cpublas::brgemm_release();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
} // anonymous namespace
|
||||
} // anonymous namespace
|
||||
|
||||
// common checks
|
||||
static inline void check_moe_scales(
|
||||
@@ -922,14 +814,10 @@ static inline void check_moe_scales(
|
||||
bool use_fp8_w8a16,
|
||||
const std::optional<at::Tensor>& w1_scale,
|
||||
const std::optional<at::Tensor>& w2_scale,
|
||||
const std::optional<std::vector<int64_t>> block_size,
|
||||
const std::optional<at::Tensor>& a1_scale,
|
||||
const std::optional<at::Tensor>& a2_scale) {
|
||||
const std::optional<std::vector<int64_t>> block_size) {
|
||||
if (use_int8_w8a8) {
|
||||
TORCH_CHECK(w1_scale.has_value(), "missing w1_scale for int8 w8a8.");
|
||||
TORCH_CHECK(w2_scale.has_value(), "missing w2_scale for int8 w8a8.");
|
||||
TORCH_CHECK(!a1_scale.has_value(), "static quantization for activation not supported.");
|
||||
TORCH_CHECK(!a2_scale.has_value(), "static quantization for activation not supported.");
|
||||
}
|
||||
if (use_fp8_w8a16) {
|
||||
TORCH_CHECK(w1_scale.has_value(), "missing w1_scale for fp8 w8a16.");
|
||||
@@ -939,16 +827,16 @@ static inline void check_moe_scales(
|
||||
}
|
||||
}
|
||||
|
||||
#define CHECK_MOE_SCALES_FP8(DIM0, DIM1) \
|
||||
auto w1s = w1_scale.value(); \
|
||||
auto w2s = w2_scale.value(); \
|
||||
auto block_size_val = block_size.value(); \
|
||||
int64_t block_size_N = block_size_val[0]; \
|
||||
int64_t block_size_K = block_size_val[1]; \
|
||||
TORCH_CHECK(w1s.size(DIM0) == 2 * N / block_size_N); \
|
||||
TORCH_CHECK(w1s.size(DIM1) == K / block_size_K); \
|
||||
TORCH_CHECK(w2s.size(DIM0) == K / block_size_N); \
|
||||
TORCH_CHECK(w2s.size(DIM1) == N / block_size_K)
|
||||
#define CHECK_MOE_SCALES_FP8(DIM0, DIM1) \
|
||||
auto w1s = w1_scale.value(); \
|
||||
auto w2s = w2_scale.value(); \
|
||||
auto block_size_val = block_size.value(); \
|
||||
int64_t block_size_N = block_size_val[0]; \
|
||||
int64_t block_size_K = block_size_val[1]; \
|
||||
TORCH_CHECK(w1s.size(DIM0) == div_up(2 * N, block_size_N)); \
|
||||
TORCH_CHECK(w1s.size(DIM1) == div_up(K, block_size_K)); \
|
||||
TORCH_CHECK(w2s.size(DIM0) == div_up(K, block_size_N)); \
|
||||
TORCH_CHECK(w2s.size(DIM1) == div_up(N, block_size_K))
|
||||
|
||||
// hidden_states: [M, K]
|
||||
// w1: [E, 2N, K]
|
||||
@@ -956,6 +844,7 @@ static inline void check_moe_scales(
|
||||
// topk_weights: [M, topk]
|
||||
// topk_ids: [M, topk] (int32_t)
|
||||
//
|
||||
|
||||
at::Tensor fused_experts_cpu(
|
||||
at::Tensor& hidden_states,
|
||||
at::Tensor& w1,
|
||||
@@ -963,16 +852,13 @@ at::Tensor fused_experts_cpu(
|
||||
at::Tensor& topk_weights,
|
||||
at::Tensor& topk_ids,
|
||||
bool inplace,
|
||||
bool use_int8_w8a8,
|
||||
bool use_fp8_w8a16,
|
||||
int64_t moe_comp_method,
|
||||
const std::optional<at::Tensor>& w1_scale,
|
||||
const std::optional<at::Tensor>& w2_scale,
|
||||
const std::optional<at::Tensor>& w1_zero,
|
||||
const std::optional<at::Tensor>& w2_zero,
|
||||
const std::optional<std::vector<int64_t>> block_size,
|
||||
const std::optional<at::Tensor>& a1_scale,
|
||||
const std::optional<at::Tensor>& a2_scale,
|
||||
bool is_vnni) {
|
||||
RECORD_FUNCTION("sgl-kernel::fused_experts_cpu", std::vector<c10::IValue>({hidden_states, w1, w2, topk_weights, topk_ids}));
|
||||
|
||||
auto packed_w1 = is_vnni ? w1 : convert_weight_packed(w1);
|
||||
auto packed_w2 = is_vnni ? w2 : convert_weight_packed(w2);
|
||||
|
||||
@@ -985,32 +871,49 @@ at::Tensor fused_experts_cpu(
|
||||
CHECK_INPUT(w2);
|
||||
CHECK_EQ(topk_weights.sizes(), topk_ids.sizes());
|
||||
CHECK_DIM(2, hidden_states);
|
||||
CHECK_DIM(3, w1);
|
||||
CHECK_DIM(3, w2);
|
||||
if (moe_comp_method == CPUQuantMethod::INT4_W4A8 && is_vnni) {
|
||||
CHECK_DIM(4, w1);
|
||||
CHECK_DIM(4, w2);
|
||||
} else {
|
||||
CHECK_DIM(3, w1);
|
||||
CHECK_DIM(3, w2);
|
||||
}
|
||||
CHECK_DIM(2, topk_weights);
|
||||
CHECK_DIM(2, topk_ids);
|
||||
|
||||
CHECK_EQ(topk_ids.scalar_type(), at::kInt);
|
||||
CHECK_EQ(topk_weights.scalar_type(), at::kFloat);
|
||||
|
||||
// TODO: support topk_weights to be bf16 or fp16 in the kernel.
|
||||
// The topk_weights of llama4 is computed via Llama4MoE:custom_routing_function and is bf16/fp16
|
||||
// while the kernel currently only supports it to be float32
|
||||
auto topk_weights_ = topk_weights.to(at::kFloat);
|
||||
CHECK_EQ(topk_weights_.scalar_type(), at::kFloat);
|
||||
|
||||
int64_t M = hidden_states.size(0);
|
||||
int64_t K = hidden_states.size(1);
|
||||
int64_t N = w1.size(1) / 2;
|
||||
int64_t N = moe_comp_method == CPUQuantMethod::INT4_W4A8 ? w1_scale.value().size(1) * w1_scale.value().size(3) / 2
|
||||
: w1.size(1) / 2;
|
||||
int64_t E = w1.size(0);
|
||||
int64_t topk = topk_weights.size(1);
|
||||
int64_t topk = topk_weights_.size(1);
|
||||
|
||||
// we use int32_t compensation for int8 w8a8
|
||||
int64_t packed_K = get_row_size(K, use_int8_w8a8);
|
||||
int64_t packed_N = get_row_size(N, use_int8_w8a8);
|
||||
int64_t packed_K = get_row_size(K, moe_comp_method == CPUQuantMethod::INT8_W8A8);
|
||||
int64_t packed_N = get_row_size(N, moe_comp_method == CPUQuantMethod::INT8_W8A8);
|
||||
|
||||
// check weight shapes
|
||||
CHECK_EQ(w2.size(0), E);
|
||||
CHECK_EQ(w2.size(1), K);
|
||||
CHECK_EQ(packed_w1.size(2), packed_K);
|
||||
CHECK_EQ(packed_w2.size(2), packed_N);
|
||||
|
||||
if (!(moe_comp_method == CPUQuantMethod::INT4_W4A8)) {
|
||||
CHECK_EQ(w2.size(1), K);
|
||||
CHECK_EQ(packed_w1.size(2), packed_K / (moe_comp_method == CPUQuantMethod::INT4_W4A8 ? 2 : 1));
|
||||
CHECK_EQ(packed_w2.size(2), packed_N / (moe_comp_method == CPUQuantMethod::INT4_W4A8 ? 2 : 1));
|
||||
}
|
||||
// check scales
|
||||
check_moe_scales(use_int8_w8a8, use_fp8_w8a16, w1_scale, w2_scale, block_size, a1_scale, a2_scale);
|
||||
check_moe_scales(
|
||||
moe_comp_method == CPUQuantMethod::INT8_W8A8,
|
||||
moe_comp_method == CPUQuantMethod::FP8_W8A16,
|
||||
w1_scale,
|
||||
w2_scale,
|
||||
block_size);
|
||||
|
||||
at::Tensor out_hidden_states = inplace ? hidden_states : at::empty_like(hidden_states);
|
||||
|
||||
@@ -1031,8 +934,8 @@ at::Tensor fused_experts_cpu(
|
||||
int32_t* __restrict__ sorted_ids = buffer.data_ptr<int32_t>();
|
||||
int32_t* __restrict__ expert_ids = sorted_ids + max_num_tokens_padded;
|
||||
int32_t* __restrict__ total_cnts = expert_ids + max_num_blocks;
|
||||
int32_t* __restrict__ cumsums = total_cnts + (num_threads + 1) * E;
|
||||
int32_t* __restrict__ offsets = cumsums + (E + 1);
|
||||
int32_t* __restrict__ cumsums = total_cnts + (num_threads + 1) * E;
|
||||
int32_t* __restrict__ offsets = cumsums + (E + 1);
|
||||
|
||||
// init sorted_ids with `numel` as the padding number
|
||||
// init expert_ids with `num_experts`
|
||||
@@ -1064,26 +967,31 @@ at::Tensor fused_experts_cpu(
|
||||
//
|
||||
// for fp8 w8a16:
|
||||
// 7. intermediate_cache0 : [M * topk, 2N]
|
||||
// 8. B_tmp : [T, BLOCK_N, std::max(K, N)]
|
||||
// 8. B_tmp : [T, MAX_CACHE_BLOCK_SIZE, BLOCK_N, std::max(K, N)]
|
||||
//
|
||||
int64_t buffer_size_nbytes = M * topk * N * 2 + M * topk * K * 2 +
|
||||
num_threads * BLOCK_M * K * (use_int8_w8a8 ? 1 : 2) +
|
||||
int64_t buffer_size_nbytes =
|
||||
M * topk * N * 2 + M * topk * K * 2 +
|
||||
num_threads * BLOCK_M * K *
|
||||
(moe_comp_method == CPUQuantMethod::INT8_W8A8 | moe_comp_method == CPUQuantMethod::INT4_W4A8 ? 1 : 2) +
|
||||
num_threads * 2 * BLOCK_M * BLOCK_N * sizeof(float);
|
||||
|
||||
if (use_int8_w8a8) {
|
||||
if (moe_comp_method == CPUQuantMethod::INT8_W8A8) {
|
||||
buffer_size_nbytes += std::max(M * K, M * topk * N) + M * topk * sizeof(float);
|
||||
}
|
||||
if (use_fp8_w8a16) {
|
||||
buffer_size_nbytes += M * topk * 2 * N * 2 + num_threads * BLOCK_N * std::max(K, N) * 2;
|
||||
if (moe_comp_method == CPUQuantMethod::FP8_W8A16) {
|
||||
buffer_size_nbytes += M * topk * 2 * N * 2 + num_threads * MAX_CACHE_BLOCK_SIZE * BLOCK_N * std::max(K, N) * 2;
|
||||
}
|
||||
if (moe_comp_method == CPUQuantMethod::INT4_W4A8) {
|
||||
buffer_size_nbytes += M * topk * 2 * N * 2 + std::max(M * K, M * topk * N) + M * topk * sizeof(float) +
|
||||
num_threads * 2 * get_4bit_block_k_size(K / w1_scale.value().size(2)) * BLOCK_N;
|
||||
}
|
||||
|
||||
auto buffer2 = at::empty({buffer_size_nbytes}, hidden_states.options().dtype(at::kChar));
|
||||
|
||||
AT_DISPATCH_REDUCED_FLOATING_TYPES(st, "fused_experts_kernel_impl", [&] {
|
||||
scalar_t* __restrict__ intermediate_cache1 = (scalar_t*)((void*)(buffer2.data_ptr<int8_t>()));
|
||||
scalar_t* __restrict__ intermediate_cache2 = intermediate_cache1 + M * topk * N;
|
||||
|
||||
if (use_int8_w8a8) {
|
||||
if (moe_comp_method == CPUQuantMethod::INT8_W8A8) {
|
||||
uint8_t* __restrict__ A_tmp = (uint8_t*)((void*)(intermediate_cache2 + M * topk * K));
|
||||
float* __restrict__ C_tmp = (float*)((void*)(A_tmp + num_threads * BLOCK_M * K));
|
||||
uint8_t* __restrict__ Aq_tmp = (uint8_t*)((void*)(C_tmp + num_threads * 2 * BLOCK_M * BLOCK_N));
|
||||
@@ -1107,7 +1015,7 @@ at::Tensor fused_experts_cpu(
|
||||
packed_w2.data_ptr<int8_t>(),
|
||||
w1s.data_ptr<float>(),
|
||||
w2s.data_ptr<float>(),
|
||||
topk_weights.data_ptr<float>(),
|
||||
topk_weights_.data_ptr<float>(),
|
||||
sorted_ids,
|
||||
expert_ids,
|
||||
offsets,
|
||||
@@ -1117,7 +1025,7 @@ at::Tensor fused_experts_cpu(
|
||||
E,
|
||||
topk,
|
||||
num_tokens_post_pad);
|
||||
} else if (use_fp8_w8a16) {
|
||||
} else if (moe_comp_method == CPUQuantMethod::FP8_W8A16) {
|
||||
// here we just ignore C_tmp as it is not used
|
||||
scalar_t* __restrict__ A_tmp = (scalar_t*)((void*)(intermediate_cache2 + M * topk * K));
|
||||
float* __restrict__ C_tmp = (float*)((void*)(A_tmp + num_threads * BLOCK_M * K));
|
||||
@@ -1140,6 +1048,48 @@ at::Tensor fused_experts_cpu(
|
||||
w2s.data_ptr<float>(),
|
||||
block_size_N,
|
||||
block_size_K,
|
||||
topk_weights_.data_ptr<float>(),
|
||||
sorted_ids,
|
||||
expert_ids,
|
||||
offsets,
|
||||
M,
|
||||
N,
|
||||
K,
|
||||
E,
|
||||
topk,
|
||||
num_tokens_post_pad);
|
||||
} else if (moe_comp_method == CPUQuantMethod::INT4_W4A8) {
|
||||
uint8_t* __restrict__ A_tmp = (uint8_t*)((void*)(intermediate_cache2 + M * topk * K));
|
||||
float* __restrict__ C_tmp = (float*)((void*)(A_tmp + num_threads * BLOCK_M * K));
|
||||
scalar_t* __restrict__ intermediate_cache0 = (scalar_t*)((void*)(C_tmp + num_threads * 2 * BLOCK_M * BLOCK_N));
|
||||
uint8_t* __restrict__ Aq_tmp = (uint8_t*)((void*)(intermediate_cache0 + M * topk * 2 * N));
|
||||
float* __restrict__ As_tmp = (float*)((void*)(Aq_tmp + std::max(M * K, M * topk * N)));
|
||||
int8_t* __restrict__ dqB_tmp = (int8_t*)((void*)(As_tmp + M * topk));
|
||||
|
||||
// weight + compensation shape = [Nc, Kc, block_n * block_k / 2 + block_n*sizeof(int32_t)]
|
||||
// scales/qzeros shape = [E, Nc, G, block_n]
|
||||
int64_t num_groups = w1_scale.value().size(2);
|
||||
const int group_size = K / num_groups;
|
||||
// TODO: check scales and zeros
|
||||
fused_experts_int4_w4a8_kernel_impl<scalar_t>(
|
||||
out_hidden_states.data_ptr<scalar_t>(),
|
||||
intermediate_cache0,
|
||||
intermediate_cache1,
|
||||
intermediate_cache2,
|
||||
A_tmp,
|
||||
Aq_tmp,
|
||||
As_tmp,
|
||||
nullptr,
|
||||
C_tmp,
|
||||
dqB_tmp,
|
||||
hidden_states.data_ptr<scalar_t>(),
|
||||
packed_w1.data_ptr<uint8_t>(),
|
||||
packed_w2.data_ptr<uint8_t>(),
|
||||
w1_zero.value().data_ptr<int8_t>(),
|
||||
w2_zero.value().data_ptr<int8_t>(),
|
||||
w1_scale.value().data_ptr<float>(),
|
||||
w2_scale.value().data_ptr<float>(),
|
||||
group_size,
|
||||
topk_weights.data_ptr<float>(),
|
||||
sorted_ids,
|
||||
expert_ids,
|
||||
@@ -1163,7 +1113,7 @@ at::Tensor fused_experts_cpu(
|
||||
hidden_states.data_ptr<scalar_t>(),
|
||||
packed_w1.data_ptr<scalar_t>(),
|
||||
packed_w2.data_ptr<scalar_t>(),
|
||||
topk_weights.data_ptr<float>(),
|
||||
topk_weights_.data_ptr<float>(),
|
||||
sorted_ids,
|
||||
expert_ids,
|
||||
offsets,
|
||||
@@ -1188,34 +1138,37 @@ at::Tensor shared_expert_cpu(
|
||||
at::Tensor& hidden_states,
|
||||
at::Tensor& w1,
|
||||
at::Tensor& w2,
|
||||
at::Tensor& fused_experts_out,
|
||||
double routed_scaling_factor,
|
||||
const std::optional<at::Tensor>& fused_experts_out,
|
||||
const std::optional<double> routed_scaling_factor,
|
||||
bool inplace,
|
||||
bool use_int8_w8a8,
|
||||
bool use_fp8_w8a16,
|
||||
std::optional<at::Tensor>& w1_scale,
|
||||
std::optional<at::Tensor>& w2_scale,
|
||||
std::optional<std::vector<int64_t>> block_size,
|
||||
std::optional<at::Tensor>& a1_scale,
|
||||
std::optional<at::Tensor>& a2_scale,
|
||||
const std::optional<at::Tensor>& w1_scale,
|
||||
const std::optional<at::Tensor>& w2_scale,
|
||||
const std::optional<std::vector<int64_t>> block_size,
|
||||
bool is_vnni) {
|
||||
RECORD_FUNCTION("sgl-kernel::shared_expert_cpu", std::vector<c10::IValue>({hidden_states, w1, w2}));
|
||||
|
||||
auto packed_w1 = is_vnni ? w1 : convert_weight_packed(w1);
|
||||
auto packed_w2 = is_vnni ? w2 : convert_weight_packed(w2);
|
||||
|
||||
constexpr int64_t BLOCK_M = block_size_m();
|
||||
constexpr int64_t BLOCK_N = block_size_n();
|
||||
|
||||
double routed_scaling_factor_value = 0;
|
||||
if (routed_scaling_factor.has_value()) {
|
||||
TORCH_CHECK(fused_experts_out.has_value(), "shared_expert_cpu: expect fused_experts_out.");
|
||||
const auto fused_experts_out_tensor = fused_experts_out.value();
|
||||
routed_scaling_factor_value = routed_scaling_factor.value();
|
||||
CHECK_INPUT(fused_experts_out_tensor);
|
||||
CHECK_EQ(hidden_states.sizes(), fused_experts_out_tensor.sizes());
|
||||
}
|
||||
|
||||
const auto st = hidden_states.scalar_type();
|
||||
CHECK_INPUT(hidden_states);
|
||||
CHECK_INPUT(fused_experts_out);
|
||||
CHECK_INPUT(w1);
|
||||
CHECK_INPUT(w2);
|
||||
CHECK_DIM(2, hidden_states);
|
||||
CHECK_DIM(2, w1);
|
||||
CHECK_DIM(2, w2);
|
||||
CHECK_EQ(hidden_states.sizes(), fused_experts_out.sizes());
|
||||
CHECK_EQ(hidden_states.scalar_type(), st);
|
||||
|
||||
int64_t M = hidden_states.size(0);
|
||||
@@ -1232,7 +1185,7 @@ at::Tensor shared_expert_cpu(
|
||||
CHECK_EQ(packed_w2.size(1), packed_N);
|
||||
|
||||
// check scales
|
||||
check_moe_scales(use_int8_w8a8, use_fp8_w8a16, w1_scale, w2_scale, block_size, a1_scale, a2_scale);
|
||||
check_moe_scales(use_int8_w8a8, use_fp8_w8a16, w1_scale, w2_scale, block_size);
|
||||
|
||||
at::Tensor out_hidden_states = inplace ? hidden_states : at::empty_like(hidden_states);
|
||||
|
||||
@@ -1246,7 +1199,7 @@ at::Tensor shared_expert_cpu(
|
||||
//
|
||||
// for fp8 w8a16:
|
||||
// 5. intermediate_cache0 : [M, 2N]
|
||||
// 6. B_tmp: [T, BLOCK_M, max(K, N)]
|
||||
// 6. B_tmp: [T, MAX_CACHE_BLOCK_SIZE, BLOCK_M, max(K, N)]
|
||||
//
|
||||
int num_threads = at::get_num_threads();
|
||||
int64_t buffer_size_nbytes = M * N * 2 + num_threads * 2 * BLOCK_M * BLOCK_N * sizeof(float);
|
||||
@@ -1255,7 +1208,7 @@ at::Tensor shared_expert_cpu(
|
||||
buffer_size_nbytes += std::max(M * K, M * N) + M * sizeof(float);
|
||||
}
|
||||
if (use_fp8_w8a16) {
|
||||
buffer_size_nbytes += M * 2 * N * 2 + num_threads * BLOCK_M * std::max(K, N) * 2;
|
||||
buffer_size_nbytes += M * 2 * N * 2 + num_threads * MAX_CACHE_BLOCK_SIZE * BLOCK_M * std::max(K, N) * 2;
|
||||
}
|
||||
|
||||
auto buffer = at::empty({buffer_size_nbytes}, hidden_states.options().dtype(at::kChar));
|
||||
@@ -1283,8 +1236,8 @@ at::Tensor shared_expert_cpu(
|
||||
packed_w2.data_ptr<int8_t>(),
|
||||
w1s.data_ptr<float>(),
|
||||
w2s.data_ptr<float>(),
|
||||
fused_experts_out.data_ptr<scalar_t>(),
|
||||
routed_scaling_factor,
|
||||
conditional_data_ptr<scalar_t>(fused_experts_out),
|
||||
routed_scaling_factor_value,
|
||||
M,
|
||||
N,
|
||||
K);
|
||||
@@ -1306,8 +1259,8 @@ at::Tensor shared_expert_cpu(
|
||||
w2s.data_ptr<float>(),
|
||||
block_size_N,
|
||||
block_size_K,
|
||||
fused_experts_out.data_ptr<scalar_t>(),
|
||||
routed_scaling_factor,
|
||||
conditional_data_ptr<scalar_t>(fused_experts_out),
|
||||
routed_scaling_factor_value,
|
||||
M,
|
||||
N,
|
||||
K);
|
||||
@@ -1319,8 +1272,8 @@ at::Tensor shared_expert_cpu(
|
||||
hidden_states.data_ptr<scalar_t>(),
|
||||
packed_w1.data_ptr<scalar_t>(),
|
||||
packed_w2.data_ptr<scalar_t>(),
|
||||
fused_experts_out.data_ptr<scalar_t>(),
|
||||
routed_scaling_factor,
|
||||
conditional_data_ptr<scalar_t>(fused_experts_out),
|
||||
routed_scaling_factor_value,
|
||||
M,
|
||||
N,
|
||||
K);
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
// Adapted from
|
||||
// https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu
|
||||
|
||||
// clang-format off
|
||||
|
||||
#pragma once
|
||||
#include "vec.h"
|
||||
|
||||
template <typename scalar_t>
|
||||
inline void fill_stub(scalar_t* __restrict__ out, scalar_t val, int64_t size) {
|
||||
using Vec = at::vec::Vectorized<scalar_t>;
|
||||
const Vec data_vec(val);
|
||||
at::vec::map<scalar_t>([data_vec](Vec out) { return out = data_vec; }, out, out, size);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
inline void copy_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ input, int64_t size) {
|
||||
using Vec = at::vec::Vectorized<scalar_t>;
|
||||
constexpr int kVecSize = Vec::size();
|
||||
int64_t d;
|
||||
#pragma GCC unroll 4
|
||||
for (d = 0; d <= size - kVecSize; d += kVecSize) {
|
||||
Vec data = Vec::loadu(input + d);
|
||||
data.store(out + d);
|
||||
}
|
||||
for (; d < size; ++d) {
|
||||
out[d] = input[d];
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
inline void copy_stub(scalar_t* __restrict__ out, const float* __restrict__ input, int64_t size) {
|
||||
using bVec = at::vec::Vectorized<scalar_t>;
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
constexpr int kVecSize = bVec::size();
|
||||
int64_t d;
|
||||
#pragma GCC unroll 4
|
||||
for (d = 0; d <= size - kVecSize; d += kVecSize) {
|
||||
auto [x0, x1] = load_float_vec2(input + d);
|
||||
bVec out_vec = convert_from_float_ext<scalar_t>(x0, x1);
|
||||
out_vec.store(out + d);
|
||||
}
|
||||
for (; d < size; ++d) {
|
||||
out[d] = static_cast<scalar_t>(input[d]);
|
||||
}
|
||||
}
|
||||
|
||||
template <>
|
||||
inline void copy_stub<uint8_t>(uint8_t* __restrict__ out, const uint8_t* __restrict__ input, int64_t size) {
|
||||
// size might be 64x + 32
|
||||
std::memcpy(out, input, size * sizeof(uint8_t));
|
||||
}
|
||||
|
||||
template <typename scalar_t, typename input_t>
|
||||
inline void copy_mul_stub(scalar_t* __restrict__ out, const input_t* __restrict__ input, float weight, int64_t size) {
|
||||
static_assert(
|
||||
std::is_same_v<input_t, float> || std::is_same_v<input_t, scalar_t>,
|
||||
"copy_mul_stub only supports input_t == float or input_t == scalar_t");
|
||||
using bVec = at::vec::Vectorized<scalar_t>;
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
constexpr int kVecSize = bVec::size();
|
||||
const fVec weight_vec = fVec(weight);
|
||||
int64_t d;
|
||||
#pragma GCC unroll 4
|
||||
for (d = 0; d <= size - kVecSize; d += kVecSize) {
|
||||
auto [x0, x1] = load_float_vec2(input + d);
|
||||
x0 = x0 * weight_vec;
|
||||
x1 = x1 * weight_vec;
|
||||
bVec out_vec = convert_from_float_ext<scalar_t>(x0, x1);
|
||||
out_vec.store(out + d);
|
||||
}
|
||||
for (; d < size; ++d) {
|
||||
out[d] = static_cast<scalar_t>(input[d] * weight);
|
||||
}
|
||||
}
|
||||
|
||||
// acc from [topk, K] to [K]
|
||||
template <typename scalar_t>
|
||||
inline void sum_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ input, int64_t topk, int64_t K) {
|
||||
using bVec = at::vec::Vectorized<scalar_t>;
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
constexpr int kVecSize = bVec::size();
|
||||
if (topk == 1) {
|
||||
// do copy for topk = 1
|
||||
copy_stub(out, input, K);
|
||||
} else {
|
||||
// do sum for topk != 1
|
||||
int64_t d;
|
||||
#pragma GCC unroll 4
|
||||
for (d = 0; d <= K - kVecSize; d += kVecSize) {
|
||||
fVec sum_fvec0 = fVec(0.f);
|
||||
fVec sum_fvec1 = fVec(0.f);
|
||||
for (int t = 0; t < topk; ++t) {
|
||||
bVec x_bvec = bVec::loadu(input + t * K + d);
|
||||
fVec x_fvec0, x_fvec1;
|
||||
std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec);
|
||||
|
||||
sum_fvec0 += x_fvec0;
|
||||
sum_fvec1 += x_fvec1;
|
||||
}
|
||||
bVec out_bvec = convert_from_float_ext<scalar_t>(sum_fvec0, sum_fvec1);
|
||||
out_bvec.store(out + d);
|
||||
}
|
||||
for (; d < K; ++d) {
|
||||
float sum_val = 0.f;
|
||||
for (int t = 0; t < topk; ++t) {
|
||||
sum_val += static_cast<float>(input[t * K + d]);
|
||||
}
|
||||
out[d] = static_cast<scalar_t>(sum_val);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// out = input + input2 * scale
|
||||
template <typename scalar_t, typename input_t>
|
||||
inline void add_mul_stub(
|
||||
scalar_t* __restrict__ out,
|
||||
const input_t* __restrict__ input,
|
||||
const scalar_t* __restrict__ input2,
|
||||
float scale,
|
||||
int64_t size) {
|
||||
static_assert(
|
||||
std::is_same_v<input_t, float> || std::is_same_v<input_t, scalar_t>,
|
||||
"add_mul_stub only supports input_t == float or input_t == scalar_t");
|
||||
|
||||
// out = input (without scale factor)
|
||||
if (input2 == nullptr) {
|
||||
copy_stub(out, input, size);
|
||||
return;
|
||||
}
|
||||
|
||||
using bVec = at::vec::Vectorized<scalar_t>;
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
constexpr int kVecSize = bVec::size();
|
||||
const fVec s_vec = fVec(scale);
|
||||
int64_t d;
|
||||
#pragma GCC unroll 4
|
||||
for (d = 0; d <= size - kVecSize; d += kVecSize) {
|
||||
auto [x0, x1] = load_float_vec2(input + d);
|
||||
|
||||
bVec y_bvec = bVec::loadu(input2 + d);
|
||||
fVec y0, y1;
|
||||
std::tie(y0, y1) = at::vec::convert_to_float(y_bvec);
|
||||
|
||||
x0 = x0 + y0 * s_vec;
|
||||
x1 = x1 + y1 * s_vec;
|
||||
bVec out_vec = convert_from_float_ext<scalar_t>(x0, x1);
|
||||
out_vec.store(out + d);
|
||||
}
|
||||
for (; d < size; ++d) {
|
||||
out[d] = static_cast<scalar_t>(input[d] + float(input2[d]) * scale);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
inline void silu_and_mul_stub(
|
||||
scalar_t* __restrict__ out, const scalar_t* __restrict__ input, const scalar_t* __restrict__ input2, int64_t size) {
|
||||
using bVec = at::vec::Vectorized<scalar_t>;
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
const fVec one = fVec(1.f);
|
||||
|
||||
// no remainder
|
||||
#pragma GCC unroll 4
|
||||
for (int64_t d = 0; d < size; d += bVec::size()) {
|
||||
bVec x = bVec::loadu(input + d);
|
||||
fVec x0, x1;
|
||||
std::tie(x0, x1) = at::vec::convert_to_float(x);
|
||||
bVec y = bVec::loadu(input2 + d);
|
||||
fVec y0, y1;
|
||||
std::tie(y0, y1) = at::vec::convert_to_float(y);
|
||||
x0 = x0 / (one + x0.neg().exp_u20());
|
||||
x1 = x1 / (one + x1.neg().exp_u20());
|
||||
x0 = x0 * y0;
|
||||
x1 = x1 * y1;
|
||||
bVec out_vec = convert_from_float_ext<scalar_t>(x0, x1);
|
||||
out_vec.store(out + d);
|
||||
}
|
||||
}
|
||||
@@ -1,147 +1,11 @@
|
||||
// Adapted from
|
||||
// https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu
|
||||
|
||||
#include "common.h"
|
||||
#include "gemm.h"
|
||||
#include "vec.h"
|
||||
|
||||
// clang-format off
|
||||
|
||||
namespace {
|
||||
|
||||
template <typename scalar_t>
|
||||
inline void copy_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ input, int64_t size) {
|
||||
using Vec = at::vec::Vectorized<scalar_t>;
|
||||
// no remainder
|
||||
#pragma GCC unroll 4
|
||||
for (int64_t d = 0; d < size; d += Vec::size()) {
|
||||
Vec data = Vec::loadu(input + d);
|
||||
data.store(out + d);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
inline void copy_mul_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ input, float weight, int64_t size) {
|
||||
using bVec = at::vec::Vectorized<scalar_t>;
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
constexpr int kVecSize = bVec::size();
|
||||
const fVec weight_vec = fVec(weight);
|
||||
int64_t d;
|
||||
#pragma GCC unroll 4
|
||||
for (d = 0; d <= size - kVecSize; d += kVecSize) {
|
||||
bVec x = bVec::loadu(input + d);
|
||||
fVec x0, x1;
|
||||
std::tie(x0, x1) = at::vec::convert_to_float(x);
|
||||
x0 = x0 * weight_vec;
|
||||
x1 = x1 * weight_vec;
|
||||
bVec out_vec = convert_from_float_ext<scalar_t>(x0, x1);
|
||||
out_vec.store(out + d);
|
||||
}
|
||||
for (; d < size; ++d) {
|
||||
out[d] = static_cast<scalar_t>(input[d] * weight);
|
||||
}
|
||||
}
|
||||
|
||||
// acc from [topk, K] to [K]
|
||||
template <typename scalar_t>
|
||||
inline void sum_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ input, int64_t topk, int64_t K) {
|
||||
using bVec = at::vec::Vectorized<scalar_t>;
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
constexpr int kVecSize = bVec::size();
|
||||
if (topk == 1) {
|
||||
// do copy for topk = 1
|
||||
copy_stub(out, input, K);
|
||||
} else {
|
||||
// do sum for topk != 1
|
||||
int64_t d;
|
||||
#pragma GCC unroll 4
|
||||
for (d = 0; d <= K - kVecSize; d += kVecSize) {
|
||||
fVec sum_fvec0 = fVec(0.f);
|
||||
fVec sum_fvec1 = fVec(0.f);
|
||||
for (int t = 0; t < topk; ++t) {
|
||||
bVec x_bvec = bVec::loadu(input + t * K + d);
|
||||
fVec x_fvec0, x_fvec1;
|
||||
std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec);
|
||||
|
||||
sum_fvec0 += x_fvec0;
|
||||
sum_fvec1 += x_fvec1;
|
||||
}
|
||||
bVec out_bvec = convert_from_float_ext<scalar_t>(sum_fvec0, sum_fvec1);
|
||||
out_bvec.store(out + d);
|
||||
}
|
||||
for (; d < K; ++d) {
|
||||
float sum_val = 0.f;
|
||||
for (int t = 0; t < topk; ++t) {
|
||||
sum_val += static_cast<float>(input[t * K + d]);
|
||||
}
|
||||
out[d] = static_cast<scalar_t>(sum_val);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// out = input + input2 * scale
|
||||
template <typename scalar_t>
|
||||
inline void add_mul_stub(
|
||||
scalar_t* __restrict__ out,
|
||||
const scalar_t* __restrict__ input,
|
||||
const scalar_t* __restrict__ input2,
|
||||
float scale,
|
||||
int64_t size) {
|
||||
using bVec = at::vec::Vectorized<scalar_t>;
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
constexpr int kVecSize = bVec::size();
|
||||
const fVec s_vec = fVec(scale);
|
||||
|
||||
int64_t d;
|
||||
#pragma GCC unroll 4
|
||||
for (d = 0; d <= size - kVecSize; d += kVecSize) {
|
||||
bVec x_bvec = bVec::loadu(input + d);
|
||||
fVec x0, x1;
|
||||
std::tie(x0, x1) = at::vec::convert_to_float(x_bvec);
|
||||
|
||||
bVec y_bvec = bVec::loadu(input2 + d);
|
||||
fVec y0, y1;
|
||||
std::tie(y0, y1) = at::vec::convert_to_float(y_bvec);
|
||||
|
||||
x0 = x0 + y0 * s_vec;
|
||||
x1 = x1 + y1 * s_vec;
|
||||
bVec out_vec = convert_from_float_ext<scalar_t>(x0, x1);
|
||||
out_vec.store(out + d);
|
||||
}
|
||||
for (; d < size; ++d) {
|
||||
out[d] = static_cast<scalar_t>(input[d] + float(input2[d]) * scale);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
inline void silu_and_mul_stub(
|
||||
scalar_t* __restrict__ out,
|
||||
const scalar_t* __restrict__ input,
|
||||
const scalar_t* __restrict__ input2,
|
||||
int64_t size) {
|
||||
using bVec = at::vec::Vectorized<scalar_t>;
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
const fVec one = fVec(1.f);
|
||||
|
||||
// no remainder
|
||||
#pragma GCC unroll 4
|
||||
for (int64_t d = 0; d < size; d += bVec::size()) {
|
||||
bVec x = bVec::loadu(input + d);
|
||||
fVec x0, x1;
|
||||
std::tie(x0, x1) = at::vec::convert_to_float(x);
|
||||
bVec y = bVec::loadu(input2 + d);
|
||||
fVec y0, y1;
|
||||
std::tie(y0, y1) = at::vec::convert_to_float(y);
|
||||
x0 = x0 / (one + x0.neg().exp_u20());
|
||||
x1 = x1 / (one + x1.neg().exp_u20());
|
||||
x0 = x0 * y0;
|
||||
x1 = x1 * y1;
|
||||
bVec out_vec = convert_from_float_ext<scalar_t>(x0, x1);
|
||||
out_vec.store(out + d);
|
||||
}
|
||||
}
|
||||
|
||||
} // anonymous namespace
|
||||
#include "common.h"
|
||||
#include "gemm.h"
|
||||
#include "moe.h"
|
||||
|
||||
template <typename scalar_t>
|
||||
void fused_experts_fp8_kernel_impl(
|
||||
@@ -169,7 +33,6 @@ void fused_experts_fp8_kernel_impl(
|
||||
int64_t E,
|
||||
int64_t topk,
|
||||
int64_t num_tokens_post_pad) {
|
||||
|
||||
constexpr int64_t BLOCK_M = block_size_m();
|
||||
constexpr int64_t BLOCK_N = block_size_n();
|
||||
|
||||
@@ -183,35 +46,39 @@ void fused_experts_fp8_kernel_impl(
|
||||
const int64_t stride_e = 2 * N * K;
|
||||
const int64_t stride_n = K;
|
||||
|
||||
int64_t avg_M = std::max(int64_t(1), M * topk / E);
|
||||
const bool use_brgemm = can_use_brgemm<at::Float8_e4m3fn>(avg_M);
|
||||
|
||||
int64_t B_tmp_size_per_thread = MAX_CACHE_BLOCK_SIZE * BLOCK_N * std::max(K, N);
|
||||
|
||||
// here we only parallel on half of 2N to fuse silu_and_mul with gemm
|
||||
at::parallel_for(0, MB * NB, 0, [&](int64_t begin, int64_t end) {
|
||||
parallel_2d(MB, NB, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) {
|
||||
// get local pointers
|
||||
int tid = at::get_thread_num();
|
||||
int tid = get_thread_num();
|
||||
scalar_t* __restrict__ A = A_tmp + tid * BLOCK_M * K;
|
||||
|
||||
bool is_brgemm_used = false;
|
||||
|
||||
for (int64_t i = begin; i < end; ++i) {
|
||||
int64_t mb = i / NB;
|
||||
int64_t nb = i % NB;
|
||||
|
||||
loop_2d<at::Float8_e4m3fn>(mb0, mb1, nb0, nb1, BLOCK_N * K, [&](int64_t mb, int64_t nb, int64_t nb_offset) {
|
||||
int64_t n_size = std::min(2 * N - nb * BLOCK_N, BLOCK_N);
|
||||
|
||||
// B shape [K, n_size] in vnni format
|
||||
int32_t expert_id = expert_ids[mb];
|
||||
const at::Float8_e4m3fn* __restrict__ B = packed_w1 + expert_id * stride_e + nb * BLOCK_N * stride_n;
|
||||
const float* __restrict__ Bs = w1s + expert_id * scale_size_N * scale_size_K + (nb / blocks_n_per_group) * scale_size_K;
|
||||
const float* __restrict__ Bs =
|
||||
w1s + expert_id * scale_size_N * scale_size_K + (nb / blocks_n_per_group) * scale_size_K;
|
||||
|
||||
// do unpacking for the first row or a new expert
|
||||
int32_t pre_expert_id = mb == 0 ? -1 : expert_ids[mb - 1];
|
||||
bool do_unpack = (mb == mb0) || (expert_id != pre_expert_id);
|
||||
|
||||
// 1.a load A
|
||||
const int32_t* A_ids = sorted_ids + mb * BLOCK_M;
|
||||
int64_t m_size = offsets[mb + 1] - offsets[mb];
|
||||
|
||||
const bool use_brgemm = can_use_brgemm<at::Float8_e4m3fn>(m_size);
|
||||
is_brgemm_used = is_brgemm_used || use_brgemm;
|
||||
|
||||
for (int64_t m = 0; m < m_size; ++m) {
|
||||
int32_t index = A_ids[m] / topk;
|
||||
copy_stub(A + m * K, input + index * K, K);
|
||||
if (nb_offset == 0) {
|
||||
// 1.a load A
|
||||
const int32_t* A_ids = sorted_ids + mb * BLOCK_M;
|
||||
for (int64_t m = 0; m < m_size; ++m) {
|
||||
int32_t index = A_ids[m] / topk;
|
||||
copy_stub(A + m * K, input + index * K, K);
|
||||
}
|
||||
}
|
||||
|
||||
const int64_t offset = offsets[mb];
|
||||
@@ -219,7 +86,7 @@ void fused_experts_fp8_kernel_impl(
|
||||
/* A */ A,
|
||||
/* B */ B,
|
||||
/* C */ ic0 + offset * 2 * N + nb * BLOCK_N,
|
||||
/* Btmp */ B_tmp + tid * BLOCK_N * std::max(K, N),
|
||||
/* Btmp */ B_tmp + tid * B_tmp_size_per_thread + nb_offset * BLOCK_N * K,
|
||||
/* Ctmp */ C_tmp + tid * 2 * BLOCK_M * BLOCK_N,
|
||||
/* scale */ Bs,
|
||||
/* M */ m_size,
|
||||
@@ -229,10 +96,11 @@ void fused_experts_fp8_kernel_impl(
|
||||
/* ldb */ n_size,
|
||||
/* ldc */ 2 * N,
|
||||
/* brg */ use_brgemm,
|
||||
/* block_size_K */ block_size_K);
|
||||
}
|
||||
/* block_size_K */ block_size_K,
|
||||
/* do_unpack */ do_unpack);
|
||||
});
|
||||
|
||||
if (is_brgemm_used) {
|
||||
if (use_brgemm) {
|
||||
at::native::cpublas::brgemm_release();
|
||||
}
|
||||
});
|
||||
@@ -240,11 +108,7 @@ void fused_experts_fp8_kernel_impl(
|
||||
// stage 1.5: intermediate_cache1 = silu(intermediate_cache0)
|
||||
at::parallel_for(0, M * topk, 0, [&](int64_t begin, int64_t end) {
|
||||
for (int64_t m = begin; m < end; ++m) {
|
||||
silu_and_mul_stub(
|
||||
ic1 + m * N,
|
||||
ic0 + m * 2 * N,
|
||||
ic0 + m * 2 * N + N,
|
||||
N);
|
||||
silu_and_mul_stub(ic1 + m * N, ic0 + m * 2 * N, ic0 + m * 2 * N + N, N);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -260,22 +124,14 @@ void fused_experts_fp8_kernel_impl(
|
||||
const int64_t stride_oc = IC;
|
||||
|
||||
// parallel on [MB2, NB2]
|
||||
at::parallel_for(0, MB2 * NB2, 0, [&](int64_t begin, int64_t end) {
|
||||
int tid = at::get_thread_num();
|
||||
parallel_2d(MB2, NB2, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) {
|
||||
int tid = get_thread_num();
|
||||
alignas(64) scalar_t C[BLOCK_M * BLOCK_K];
|
||||
|
||||
bool is_brgemm_used = false;
|
||||
|
||||
for (int64_t i = begin; i < end; ++i) {
|
||||
int64_t mb = i / NB2;
|
||||
int64_t nb = i % NB2;
|
||||
|
||||
loop_2d<at::Float8_e4m3fn>(mb0, mb1, nb0, nb1, BLOCK_N * IC, [&](int64_t mb, int64_t nb, int64_t nb_offset) {
|
||||
int64_t m_size = offsets[mb + 1] - offsets[mb];
|
||||
int64_t n_size = std::min(OC - nb * BLOCK_N, BLOCK_N);
|
||||
|
||||
const bool use_brgemm = can_use_brgemm<at::Float8_e4m3fn>(m_size);
|
||||
is_brgemm_used = is_brgemm_used || use_brgemm;
|
||||
|
||||
// A ptr from ic1 of [M * topk, N] in sorted order
|
||||
// so as to avoid copy A to tmp buffer again
|
||||
const scalar_t* __restrict__ A = ic1 + offsets[mb] * N;
|
||||
@@ -284,13 +140,18 @@ void fused_experts_fp8_kernel_impl(
|
||||
// B shape [IC, n_size] in vnni format
|
||||
int32_t expert_id = expert_ids[mb];
|
||||
const at::Float8_e4m3fn* __restrict__ B = packed_w2 + expert_id * stride_e2 + nb * BLOCK_N * stride_oc;
|
||||
const float* __restrict__ Bs = w2s + expert_id * scale_size_N * scale_size_K + (nb / blocks_n_per_group) * scale_size_K;
|
||||
const float* __restrict__ Bs =
|
||||
w2s + expert_id * scale_size_N * scale_size_K + (nb / blocks_n_per_group) * scale_size_K;
|
||||
|
||||
// do unpacking for the first row or a new expert
|
||||
int32_t pre_expert_id = mb == 0 ? -1 : expert_ids[mb - 1];
|
||||
bool do_unpack = (mb == mb0) || (expert_id != pre_expert_id);
|
||||
|
||||
tinygemm_kernel<scalar_t>(
|
||||
/* A */ A,
|
||||
/* B */ B,
|
||||
/* C */ C,
|
||||
/* Btmp */ B_tmp + tid * BLOCK_N * std::max(K, N),
|
||||
/* Btmp */ B_tmp + tid * B_tmp_size_per_thread + nb_offset * BLOCK_N * IC,
|
||||
/* Ctmp */ C_tmp + tid * 2 * BLOCK_M * BLOCK_N,
|
||||
/* scale */ Bs,
|
||||
/* M */ m_size,
|
||||
@@ -300,7 +161,8 @@ void fused_experts_fp8_kernel_impl(
|
||||
/* ldb */ n_size,
|
||||
/* ldc */ BLOCK_N,
|
||||
/* brg */ use_brgemm,
|
||||
/* block_size_K */ block_size_K);
|
||||
/* block_size_K */ block_size_K,
|
||||
/* do_unpack */ do_unpack);
|
||||
|
||||
// 2.b copy from C to ic2 in original order
|
||||
// and also mul topk_weights in float32
|
||||
@@ -309,9 +171,9 @@ void fused_experts_fp8_kernel_impl(
|
||||
float weight = topk_weights[index];
|
||||
copy_mul_stub(ic2 + index * K + nb * BLOCK_N, C + m * BLOCK_N, weight, n_size);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if (is_brgemm_used) {
|
||||
if (use_brgemm) {
|
||||
at::native::cpublas::brgemm_release();
|
||||
}
|
||||
});
|
||||
@@ -374,7 +236,6 @@ void shared_expert_fp8_kernel_impl(
|
||||
int64_t M,
|
||||
int64_t N,
|
||||
int64_t K) {
|
||||
|
||||
constexpr int64_t BLOCK_M = block_size_m();
|
||||
constexpr int64_t BLOCK_N = block_size_n();
|
||||
|
||||
@@ -385,21 +246,25 @@ void shared_expert_fp8_kernel_impl(
|
||||
int64_t blocks_n_per_group = block_size_N / BLOCK_N;
|
||||
|
||||
const bool use_brgemm = can_use_brgemm<at::Float8_e4m3fn>(M);
|
||||
const bool apply_scaling_factor = fused_experts_out != nullptr;
|
||||
|
||||
at::parallel_for(0, MB * NB, 0, [&](int64_t begin, int64_t end) {
|
||||
int tid = at::get_thread_num();
|
||||
int64_t B_tmp_size_per_thread = MAX_CACHE_BLOCK_SIZE * BLOCK_N * std::max(K, N);
|
||||
|
||||
for (int64_t i = begin; i < end; ++i) {
|
||||
int64_t mb = i / NB;
|
||||
int64_t nb = i % NB;
|
||||
parallel_2d(MB, NB, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) {
|
||||
int tid = get_thread_num();
|
||||
|
||||
loop_2d<at::Float8_e4m3fn>(mb0, mb1, nb0, nb1, BLOCK_N * K, [&](int64_t mb, int64_t nb, int64_t nb_offset) {
|
||||
int64_t m_size = std::min(M - mb * BLOCK_M, BLOCK_M);
|
||||
int64_t n_size = std::min(2 * N - nb * BLOCK_N, BLOCK_N);
|
||||
|
||||
// do unpacking for the first row
|
||||
bool do_unpack = (mb == mb0);
|
||||
|
||||
tinygemm_kernel<scalar_t>(
|
||||
/* A */ input + mb * BLOCK_M * K,
|
||||
/* B */ packed_w1 + nb * BLOCK_N * K,
|
||||
/* C */ ic0 + mb * BLOCK_M * 2 * N + nb * BLOCK_N,
|
||||
/* Btmp */ B_tmp + tid * BLOCK_N * std::max(K, N),
|
||||
/* Btmp */ B_tmp + tid * B_tmp_size_per_thread + nb_offset * BLOCK_N * K,
|
||||
/* Ctmp */ C_tmp + tid * 2 * BLOCK_M * BLOCK_N,
|
||||
/* scale */ w1s + (nb / blocks_n_per_group) * scale_size_K,
|
||||
/* M */ m_size,
|
||||
@@ -409,8 +274,9 @@ void shared_expert_fp8_kernel_impl(
|
||||
/* ldb */ n_size,
|
||||
/* ldc */ 2 * N,
|
||||
/* brg */ use_brgemm,
|
||||
/* block_size_K */ block_size_K);
|
||||
}
|
||||
/* block_size_K */ block_size_K,
|
||||
/* do_unpack */ do_unpack);
|
||||
});
|
||||
|
||||
if (use_brgemm) {
|
||||
at::native::cpublas::brgemm_release();
|
||||
@@ -420,11 +286,7 @@ void shared_expert_fp8_kernel_impl(
|
||||
// stage 1.5: intermediate_cache1 = silu(intermediate_cache0)
|
||||
at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) {
|
||||
for (int64_t m = begin; m < end; ++m) {
|
||||
silu_and_mul_stub(
|
||||
ic1 + m * N,
|
||||
ic0 + m * 2 * N,
|
||||
ic0 + m * 2 * N + N,
|
||||
N);
|
||||
silu_and_mul_stub(ic1 + m * N, ic0 + m * 2 * N, ic0 + m * 2 * N + N, N);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -437,22 +299,23 @@ void shared_expert_fp8_kernel_impl(
|
||||
scale_size_K = div_up(N, block_size_K);
|
||||
|
||||
// parallel on [MB2, NB2]
|
||||
at::parallel_for(0, MB2 * NB2, 0, [&](int64_t begin, int64_t end) {
|
||||
int tid = at::get_thread_num();
|
||||
parallel_2d(MB2, NB2, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) {
|
||||
int tid = get_thread_num();
|
||||
alignas(64) scalar_t C[BLOCK_M * BLOCK_K];
|
||||
|
||||
for (int64_t i = begin; i < end; ++i) {
|
||||
int64_t mb = i / NB2;
|
||||
int64_t nb = i % NB2;
|
||||
loop_2d<at::Float8_e4m3fn>(mb0, mb1, nb0, nb1, BLOCK_N * IC, [&](int64_t mb, int64_t nb, int64_t nb_offset) {
|
||||
int64_t m_size = std::min(M - mb * BLOCK_M, BLOCK_M);
|
||||
int64_t n_size = std::min(OC - nb * BLOCK_N, BLOCK_N);
|
||||
|
||||
// do unpacking for the first row
|
||||
bool do_unpack = (mb == mb0);
|
||||
|
||||
// 2.a gemm: C = A @ B
|
||||
tinygemm_kernel<scalar_t>(
|
||||
/* A */ ic1 + mb * BLOCK_M * N,
|
||||
/* B */ packed_w2 + nb * BLOCK_N * N,
|
||||
/* C */ C,
|
||||
/* Btmp */ B_tmp + tid * BLOCK_N * std::max(K, N),
|
||||
/* Btmp */ B_tmp + tid * B_tmp_size_per_thread + nb_offset * BLOCK_N * IC,
|
||||
/* Ctmp */ C_tmp + tid * 2 * BLOCK_M * BLOCK_N,
|
||||
/* scale */ w2s + (nb / blocks_n_per_group) * scale_size_K,
|
||||
/* M */ m_size,
|
||||
@@ -462,15 +325,18 @@ void shared_expert_fp8_kernel_impl(
|
||||
/* ldb */ n_size,
|
||||
/* ldc */ BLOCK_N,
|
||||
/* brg */ use_brgemm,
|
||||
/* block_size_K */ block_size_K);
|
||||
/* block_size_K */ block_size_K,
|
||||
/* do_unpack */ do_unpack);
|
||||
|
||||
// 2.b copy from C to output and add fused_experts_out
|
||||
scalar_t* __restrict__ out = output + mb * BLOCK_M * K + nb * BLOCK_N;
|
||||
const scalar_t* __restrict__ fused_out = fused_experts_out + mb * BLOCK_M * K + nb * BLOCK_N;
|
||||
const scalar_t* __restrict__ fused_out =
|
||||
apply_scaling_factor ? fused_experts_out + mb * BLOCK_M * K + nb * BLOCK_N : nullptr;
|
||||
for (int64_t m = 0; m < m_size; ++m) {
|
||||
add_mul_stub(out + m * K, C + m * BLOCK_N, fused_out + m * K, routed_scaling_factor, n_size);
|
||||
const scalar_t* __restrict__ fused_out_row = apply_scaling_factor ? (fused_out + m * K) : nullptr;
|
||||
add_mul_stub(out + m * K, C + m * BLOCK_N, fused_out_row, routed_scaling_factor, n_size);
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
if (use_brgemm) {
|
||||
|
||||
@@ -0,0 +1,323 @@
|
||||
// Adapted from
|
||||
// https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu
|
||||
|
||||
// clang-format off
|
||||
|
||||
#include "common.h"
|
||||
#include "gemm.h"
|
||||
#include "moe.h"
|
||||
|
||||
template <int64_t N>
|
||||
inline void copy_bias(const float* bias_ptr, float* y_buf, int64_t m, int64_t ldn) {
|
||||
using Vec = at::vec::Vectorized<float>;
|
||||
constexpr int kVecSize = Vec::size();
|
||||
static_assert(N % kVecSize == 0, "copy_bias requires N to be a multiple of Vectorized<float>::size()");
|
||||
const bool has_bias = bias_ptr != nullptr;
|
||||
const Vec zero_vec(0.f);
|
||||
for (int i = 0; i < m; ++i) {
|
||||
#pragma GCC unroll 2
|
||||
for (int j = 0; j < N; j += kVecSize) {
|
||||
Vec vec = has_bias ? Vec::loadu(bias_ptr + j) : zero_vec;
|
||||
vec.store(y_buf + i * ldn + j);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
void fused_experts_int4_w4a8_kernel_impl(
|
||||
scalar_t* __restrict__ output,
|
||||
scalar_t* __restrict__ ic0,
|
||||
scalar_t* __restrict__ ic1,
|
||||
scalar_t* __restrict__ ic2,
|
||||
uint8_t* __restrict__ A_tmp,
|
||||
uint8_t* __restrict__ Aq_tmp,
|
||||
float* __restrict__ As_tmp,
|
||||
int32_t* __restrict__ Azp_tmp,
|
||||
float* __restrict__ C_tmp,
|
||||
int8_t* __restrict__ dqB_tmp,
|
||||
const scalar_t* __restrict__ input,
|
||||
const uint8_t* __restrict__ packed_w1,
|
||||
const uint8_t* __restrict__ packed_w2,
|
||||
const int8_t* __restrict__ w1z,
|
||||
const int8_t* __restrict__ w2z,
|
||||
const float* __restrict__ w1s,
|
||||
const float* __restrict__ w2s,
|
||||
int group_size,
|
||||
const float* __restrict__ topk_weights,
|
||||
const int32_t* __restrict__ sorted_ids,
|
||||
const int32_t* __restrict__ expert_ids,
|
||||
const int32_t* __restrict__ offsets,
|
||||
int64_t M,
|
||||
int64_t N,
|
||||
int64_t K,
|
||||
int64_t E,
|
||||
int64_t topk,
|
||||
int64_t num_tokens_post_pad) {
|
||||
constexpr int64_t BLOCK_M = block_size_m();
|
||||
constexpr int64_t BLOCK_N = block_size_n();
|
||||
int num_threads = at::get_num_threads();
|
||||
// int64_t buffer_size_nbytes = M * topk * N * 2
|
||||
// M * topk * K * 2 +
|
||||
// num_threads * BLOCK_M * K +
|
||||
// num_threads * 2 * BLOCK_M * BLOCK_N * sizeof(float) +
|
||||
// M * topk * 2 * N * 2 +
|
||||
// max(M * K, M * topk * N) +
|
||||
// M * topk * sizeof(float);
|
||||
|
||||
// intermediate_cache1 (scalar_t): START + M * topk * N
|
||||
// intermediate_cache2 (scalar_t): + M * topk * K
|
||||
// A_tmp (uint8_t): + num_threads * BLOCK_M * K
|
||||
// C_tmp (float): + num_threads * 2 * BLOCK_M * BLOCK_N
|
||||
// intermediate_cache0 (scalar_t): + M * topk * 2 * N
|
||||
// Aq_tmp (uint8_t): + max(M * K, M * topk * N)
|
||||
// As_tmp (float): + M * topk
|
||||
// dqB_tmp (int8_t) + num_threads * _block_k * BlOCK_N
|
||||
|
||||
// stage 0: quantize input to uint8, [M, K]
|
||||
at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) {
|
||||
for (int64_t m = begin; m < end; ++m) {
|
||||
quantize_row_int8<scalar_t>(Aq_tmp + m * K, As_tmp[m], input + m * K, K);
|
||||
}
|
||||
});
|
||||
int64_t _block_k = get_4bit_block_k_size(group_size);
|
||||
auto Azp = at::ones({M * topk}).to(at::kInt).mul(128);
|
||||
auto Azp_ptr = Azp.data_ptr<int32_t>();
|
||||
// stage 1: intermediate_cache0 = hidden_states @ w1
|
||||
const int64_t MB = div_up(num_tokens_post_pad, BLOCK_M);
|
||||
const int64_t NB = div_up(N, BLOCK_N);
|
||||
|
||||
int64_t block_per_group = group_size / _block_k;
|
||||
int64_t Kc = K / _block_k;
|
||||
int64_t num_groups = K / group_size;
|
||||
|
||||
const int64_t stride_e = 2 * NB * Kc * (BLOCK_N * (_block_k / 2 + sizeof(int32_t)));
|
||||
const bool sym_quant_act = false;
|
||||
// weight + compensation shape = [E, Nc, Kc, block_n * _block_k / 2 + block_n*sizeof(int32_t)]
|
||||
// scales/qzeros shape = [E, Nc, G, block_n]
|
||||
|
||||
// here we only parallel on half of 2N to fuse silu_and_mul with gemm
|
||||
at::parallel_for(0, MB * NB, 0, [&](int64_t begin, int64_t end) {
|
||||
// get local pointers
|
||||
int tid = at::get_thread_num();
|
||||
int8_t* dqB_tmp1 = dqB_tmp + tid * 2 * _block_k * BLOCK_N;
|
||||
int8_t* dqB_tmp2 = dqB_tmp1 + _block_k * BLOCK_N;
|
||||
alignas(64) float As[BLOCK_M];
|
||||
uint8_t* __restrict__ A = A_tmp + tid * BLOCK_M * K;
|
||||
float* __restrict__ C0 = C_tmp + tid * 2 * BLOCK_M * BLOCK_N;
|
||||
float* __restrict__ C1 = C0 + BLOCK_M * BLOCK_N;
|
||||
bool is_brgemm_used = false;
|
||||
for (int64_t i = begin; i < end; ++i) {
|
||||
int64_t mb = i / NB;
|
||||
int64_t nb = i % NB;
|
||||
int64_t nb1 = nb + NB;
|
||||
int64_t n_size = std::min(N - nb * BLOCK_N, BLOCK_N);
|
||||
// B shape [K, n_size] in vnni format
|
||||
int32_t expert_id = expert_ids[mb];
|
||||
const uint8_t* __restrict__ B = packed_w1 + expert_id * stride_e;
|
||||
// Bz and Bs: [E, K/gs, 2N]
|
||||
const int8_t* __restrict__ Bz = w1z + expert_id * (num_groups) * (2 * N);
|
||||
const float* __restrict__ Bs = w1s + expert_id * (num_groups) * (2 * N);
|
||||
|
||||
// 1.a load A
|
||||
const int32_t* A_ids = sorted_ids + mb * BLOCK_M;
|
||||
int64_t m_size = offsets[mb + 1] - offsets[mb];
|
||||
const bool use_brgemm = can_use_brgemm<int8_t>(m_size);
|
||||
is_brgemm_used = is_brgemm_used || use_brgemm;
|
||||
// copy to A [BLOCK_M, K]
|
||||
for (int64_t m = 0; m < m_size; ++m) {
|
||||
int32_t index = A_ids[m] / topk;
|
||||
copy_stub(A + m * K, Aq_tmp + index * K, K);
|
||||
As[m] = As_tmp[index];
|
||||
}
|
||||
const int64_t offset = offsets[mb];
|
||||
copy_bias<BLOCK_N>(nullptr, C0, m_size, BLOCK_N);
|
||||
copy_bias<BLOCK_N>(nullptr, C1, m_size, BLOCK_N);
|
||||
for (int kci = 0; kci < Kc; ++kci) {
|
||||
int32_t* compensation_ptr =
|
||||
sym_quant_act ? nullptr
|
||||
: (int32_t*)(void*)(B + (nb * Kc + kci) * (BLOCK_N * (_block_k / 2 + sizeof(int32_t))) +
|
||||
_block_k * BLOCK_N / 2) /*Bcomp*/;
|
||||
tinygemm_kernel<scalar_t>(
|
||||
ic0 + offset * 2 * N + nb * BLOCK_N,
|
||||
C0,
|
||||
A + kci * _block_k,
|
||||
As,
|
||||
Azp_ptr,
|
||||
B + (nb * Kc + kci) * (BLOCK_N * (_block_k / 2 + sizeof(int32_t))) /*B*/,
|
||||
Bs + nb * BLOCK_N * num_groups + kci / block_per_group * BLOCK_N /*scales_b*/,
|
||||
Bz + nb * BLOCK_N * num_groups + kci / block_per_group * BLOCK_N /*qzeros_b*/,
|
||||
compensation_ptr,
|
||||
dqB_tmp1,
|
||||
m_size,
|
||||
_block_k,
|
||||
K,
|
||||
BLOCK_N,
|
||||
2 * N,
|
||||
kci == Kc - 1,
|
||||
use_brgemm);
|
||||
}
|
||||
|
||||
for (int kci = 0; kci < Kc; ++kci) {
|
||||
int32_t* compensation_ptr =
|
||||
sym_quant_act ? nullptr
|
||||
: (int32_t*)(void*)(B + (nb1 * Kc + kci) * (BLOCK_N * (_block_k / 2 + sizeof(int32_t))) +
|
||||
_block_k * BLOCK_N / 2) /*Bcomp*/;
|
||||
tinygemm_kernel<scalar_t>(
|
||||
ic0 + offset * 2 * N + nb1 * BLOCK_N,
|
||||
C1,
|
||||
A + kci * _block_k,
|
||||
As,
|
||||
Azp_ptr,
|
||||
B + (nb1 * Kc + kci) * (BLOCK_N * (_block_k / 2 + sizeof(int32_t))) /*B*/,
|
||||
Bs + nb1 * BLOCK_N * num_groups + kci / block_per_group * BLOCK_N /*scales_b*/,
|
||||
Bz + nb1 * BLOCK_N * num_groups + kci / block_per_group * BLOCK_N /*qzeros_b*/,
|
||||
compensation_ptr,
|
||||
dqB_tmp2,
|
||||
m_size,
|
||||
_block_k,
|
||||
K,
|
||||
BLOCK_N,
|
||||
2 * N,
|
||||
kci == Kc - 1,
|
||||
use_brgemm);
|
||||
}
|
||||
}
|
||||
|
||||
if (is_brgemm_used) {
|
||||
at::native::cpublas::brgemm_release();
|
||||
}
|
||||
});
|
||||
|
||||
// stage 1.5: intermediate_cache1 = silu(intermediate_cache0)
|
||||
at::parallel_for(0, M * topk, 0, [&](int64_t begin, int64_t end) {
|
||||
for (int64_t m = begin; m < end; ++m) {
|
||||
silu_and_mul_stub(ic1 + m * N, ic0 + m * 2 * N, ic0 + m * 2 * N + N, N);
|
||||
}
|
||||
});
|
||||
|
||||
// stage 1.5: quantize ic1 to uint8, [M * topk, N]
|
||||
at::parallel_for(0, M * topk, 0, [&](int64_t begin, int64_t end) {
|
||||
for (int64_t m = begin; m < end; ++m) {
|
||||
quantize_row_int8<scalar_t>(Aq_tmp + m * N, As_tmp[m], ic1 + m * N, N);
|
||||
}
|
||||
});
|
||||
// stage 2: intermediate_cache2 = intermediate_cache1 @ w2
|
||||
// w2 : [E, K, N] as [E, OC, IC]
|
||||
const int64_t OC = K; // rename K as OC
|
||||
const int64_t IC = N; // rename N as IC
|
||||
const int64_t MB2 = MB;
|
||||
const int64_t NB2 = div_up(OC, BLOCK_N);
|
||||
const int64_t stride_oc = IC;
|
||||
num_groups = IC / group_size;
|
||||
Kc = IC / _block_k;
|
||||
const int64_t stride_e2 = NB2 * Kc * (BLOCK_N * (_block_k / 2 + sizeof(int32_t)));
|
||||
// parallel on [MB2, NB2]
|
||||
at::parallel_for(0, MB2 * NB2, 0, [&](int64_t begin, int64_t end) {
|
||||
int tid = at::get_thread_num();
|
||||
int8_t* dqB_tmp1 = dqB_tmp + tid * 2 * _block_k * BLOCK_N;
|
||||
float* __restrict__ C2 = C_tmp + tid * 2 * BLOCK_M * BLOCK_N;
|
||||
bool is_brgemm_used = false;
|
||||
for (int64_t i = begin; i < end; ++i) {
|
||||
int64_t mb = i / NB2;
|
||||
int64_t nb = i % NB2;
|
||||
|
||||
int64_t m_size = offsets[mb + 1] - offsets[mb];
|
||||
int64_t n_size = std::min(OC - nb * BLOCK_N, BLOCK_N);
|
||||
const bool use_brgemm = can_use_brgemm<int8_t>(m_size);
|
||||
is_brgemm_used = is_brgemm_used || use_brgemm;
|
||||
const int32_t* A_ids = sorted_ids + mb * BLOCK_M;
|
||||
|
||||
// B shape [IC, n_size] in vnni format
|
||||
int32_t expert_id = expert_ids[mb];
|
||||
const uint8_t* __restrict__ B = packed_w2 + expert_id * stride_e2;
|
||||
|
||||
// Bz and Bs: [E, IC/gs, OC]
|
||||
const int8_t* __restrict__ Bz = w2z + expert_id * (num_groups)*OC;
|
||||
const float* __restrict__ Bs = w2s + expert_id * (num_groups)*OC;
|
||||
|
||||
// A ptr from ic1 of [M * topk, N] in sorted order
|
||||
// so as to avoid copy A to tmp buffer again
|
||||
const uint8_t* __restrict__ A = Aq_tmp + offsets[mb] * IC;
|
||||
const float* __restrict__ As = As_tmp + offsets[mb];
|
||||
copy_bias<BLOCK_N>(nullptr, C2, m_size, BLOCK_N);
|
||||
for (int kci = 0; kci < Kc; ++kci) {
|
||||
int32_t* compensation_ptr =
|
||||
sym_quant_act ? nullptr
|
||||
: (int32_t*)(void*)(B + (nb * Kc + kci) * (BLOCK_N * (_block_k / 2 + sizeof(int32_t))) +
|
||||
_block_k * BLOCK_N / 2) /*Bcomp*/;
|
||||
tinygemm_kernel<scalar_t>(
|
||||
nullptr, /*store_out is false*/
|
||||
C2,
|
||||
A + kci * _block_k,
|
||||
As,
|
||||
Azp_ptr,
|
||||
B + (nb * Kc + kci) * (BLOCK_N * (_block_k / 2 + sizeof(int32_t))),
|
||||
Bs + nb * BLOCK_N * num_groups + kci / block_per_group * BLOCK_N /*scales_b*/,
|
||||
Bz + nb * BLOCK_N * num_groups + kci / block_per_group * BLOCK_N /*zeros_b*/,
|
||||
compensation_ptr,
|
||||
dqB_tmp1,
|
||||
m_size,
|
||||
_block_k,
|
||||
IC,
|
||||
BLOCK_N,
|
||||
BLOCK_N,
|
||||
false,
|
||||
use_brgemm);
|
||||
}
|
||||
|
||||
// 2.b copy from C to ic2 in original order
|
||||
// and also mul topk_weights in float32
|
||||
for (int64_t m = 0; m < m_size; ++m) {
|
||||
int32_t index = A_ids[m];
|
||||
float weight = topk_weights[index];
|
||||
copy_mul_stub(ic2 + index * K + nb * BLOCK_N, C2 + m * BLOCK_N, weight, n_size);
|
||||
}
|
||||
}
|
||||
|
||||
if (is_brgemm_used) {
|
||||
at::native::cpublas::brgemm_release();
|
||||
}
|
||||
});
|
||||
|
||||
// stage 3: out = intermediate_cache2.sum(dim=1)
|
||||
// from [M, topk, K] to [M, K]
|
||||
at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) {
|
||||
for (int64_t m = begin; m < end; ++m) {
|
||||
sum_stub(output + m * K, ic2 + m * topk * K, topk, K);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#define INSTANTIATE_MOE_INT4_W4A8_TEMPLATE(TYPE) \
|
||||
template void fused_experts_int4_w4a8_kernel_impl<TYPE>( \
|
||||
TYPE* __restrict__ output, \
|
||||
TYPE* __restrict__ ic0, \
|
||||
TYPE* __restrict__ ic1, \
|
||||
TYPE* __restrict__ ic2, \
|
||||
uint8_t* __restrict__ A_tmp, \
|
||||
uint8_t* __restrict__ Aq_tmp, \
|
||||
float* __restrict__ As_tmp, \
|
||||
int32_t* __restrict__ Azp_tmp, \
|
||||
float* __restrict__ C_tmp, \
|
||||
int8_t* __restrict__ dqB_tmp, \
|
||||
const TYPE* __restrict__ input, \
|
||||
const uint8_t* __restrict__ packed_w1, \
|
||||
const uint8_t* __restrict__ packed_w2, \
|
||||
const int8_t* __restrict__ w1z, \
|
||||
const int8_t* __restrict__ w2z, \
|
||||
const float* __restrict__ w1s, \
|
||||
const float* __restrict__ w2s, \
|
||||
int group_size, \
|
||||
const float* __restrict__ topk_weights, \
|
||||
const int32_t* __restrict__ sorted_ids, \
|
||||
const int32_t* __restrict__ expert_ids, \
|
||||
const int32_t* __restrict__ offsets, \
|
||||
int64_t M, \
|
||||
int64_t N, \
|
||||
int64_t K, \
|
||||
int64_t E, \
|
||||
int64_t topk, \
|
||||
int64_t num_tokens_post_pad)
|
||||
|
||||
INSTANTIATE_MOE_INT4_W4A8_TEMPLATE(at::BFloat16);
|
||||
INSTANTIATE_MOE_INT4_W4A8_TEMPLATE(at::Half);
|
||||
+501
-297
File diff suppressed because it is too large
Load Diff
+155
-37
@@ -1,10 +1,10 @@
|
||||
// Adapted from
|
||||
// https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu
|
||||
|
||||
#pragma once
|
||||
|
||||
// clang-format off
|
||||
|
||||
#pragma once
|
||||
|
||||
#if defined(__AVX512F__) && defined(__AVX512BF16__) && defined(__AMX_BF16__)
|
||||
#define CPU_CAPABILITY_AVX512
|
||||
#endif
|
||||
@@ -16,38 +16,51 @@ namespace {
|
||||
|
||||
using namespace at::vec;
|
||||
|
||||
template <typename scalar_t,
|
||||
typename std::enable_if_t<is_reduced_floating_point_v<scalar_t>, int> = 0>
|
||||
template <typename scalar_t, typename std::enable_if_t<is_reduced_floating_point_v<scalar_t>, int> = 0>
|
||||
inline Vectorized<scalar_t> convert_from_float_ext(const Vectorized<float>& a, const Vectorized<float>& b) {
|
||||
return at::vec::convert_from_float<scalar_t>(a, b);
|
||||
}
|
||||
|
||||
// allow f16, bf16
|
||||
template <typename scalar_t, typename std::enable_if_t<is_reduced_floating_point_v<scalar_t>, int> = 1>
|
||||
inline std::tuple<Vectorized<float>, Vectorized<float>> load_float_vec2(const scalar_t* __restrict__ data) {
|
||||
using bVec = at::vec::Vectorized<scalar_t>;
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
bVec x_vec = bVec::loadu(data);
|
||||
fVec x0, x1;
|
||||
std::tie(x0, x1) = at::vec::convert_to_float(x_vec);
|
||||
return std::make_tuple(x0, x1);
|
||||
}
|
||||
|
||||
// allow f32
|
||||
inline std::tuple<Vectorized<float>, Vectorized<float>> load_float_vec2(const float* __restrict__ data) {
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
fVec x0 = fVec::loadu(data);
|
||||
fVec x1 = fVec::loadu(data + fVec::size());
|
||||
return std::make_tuple(x0, x1);
|
||||
}
|
||||
|
||||
#if defined(CPU_CAPABILITY_AVX512)
|
||||
|
||||
// `at::vec::convert_from_float<>` from PyTorch doesn't have avx512-bf16 intrinsics
|
||||
// use native instruction for bfloat16->float32 conversion
|
||||
template <>
|
||||
inline Vectorized<at::BFloat16> convert_from_float_ext<at::BFloat16>(const Vectorized<float>& a, const Vectorized<float>& b) {
|
||||
inline Vectorized<at::BFloat16>
|
||||
convert_from_float_ext<at::BFloat16>(const Vectorized<float>& a, const Vectorized<float>& b) {
|
||||
return (__m512i)(_mm512_cvtne2ps_pbh(__m512(b), __m512(a)));
|
||||
}
|
||||
|
||||
#define CVT_BF16_TO_FP32(a) \
|
||||
_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(a), 16))
|
||||
#define CVT_BF16_TO_FP32(a) _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(a), 16))
|
||||
|
||||
#define CVT_FP16_TO_FP32(a) \
|
||||
_mm512_cvtps_ph(a, (_MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC))
|
||||
#define CVT_FP16_TO_FP32(a) _mm512_cvtph_ps(a)
|
||||
|
||||
// this doesn't handle NaN.
|
||||
inline __m512bh cvt_e4m3_bf16_intrinsic_no_nan(__m256i fp8_vec) {
|
||||
const __m512i x = _mm512_cvtepu8_epi16(fp8_vec);
|
||||
|
||||
const __m512i mant = _mm512_slli_epi16(_mm512_and_si512(x, _mm512_set1_epi16(0x07)), 4);
|
||||
const __m512i raw_exp = _mm512_srli_epi16(_mm512_and_si512(x, _mm512_set1_epi16(0x78)), 3);
|
||||
const __m512i exp = _mm512_slli_epi16(_mm512_add_epi16(raw_exp, _mm512_set1_epi16(120)), 7);
|
||||
const __m512i nonsign = _mm512_or_si512(exp, mant);
|
||||
|
||||
const __m512i sign = _mm512_slli_epi16(_mm512_and_si512(x, _mm512_set1_epi16(0x80)), 8);
|
||||
const __m512i combined = _mm512_or_si512(nonsign, sign);
|
||||
__m512i combined = _mm512_add_epi16(x, _mm512_set1_epi16(0x0780));
|
||||
combined = _mm512_slli_epi16(combined, 4);
|
||||
combined = _mm512_and_si512(combined, _mm512_set1_epi16(0x87f0));
|
||||
combined = _mm512_add_epi16(combined, _mm512_set1_epi16(0x3c00));
|
||||
|
||||
const __mmask32 is_nonzero = _mm512_cmpneq_epi16_mask(x, _mm512_setzero_si512());
|
||||
return (__m512bh)_mm512_maskz_mov_epi16(is_nonzero, combined);
|
||||
@@ -113,10 +126,77 @@ inline __m512bh CVT_FP8_TO_BF16(__m256i a) {
|
||||
#endif
|
||||
}
|
||||
|
||||
// faster version of float8_e4m3fn conversion to bfloat16
|
||||
//
|
||||
// we mapped cuda implementation from below link and vectorized with avx512:
|
||||
// https://github.com/thu-pacman/chitu/blob/1ed2078ec26581ebdca05b7306d4385f86edaa7c/csrc/cuda/marlin/marlin_gemm/dequant.h#L387
|
||||
//
|
||||
inline __attribute__((always_inline)) __m512bh CVT_FP8_TO_BF16_EXT(__m256i a) {
|
||||
const __m512i mask0 = _mm512_set1_epi16(0x80); // sign bit
|
||||
const __m512i mask1 = _mm512_set1_epi16(0x7F); // exponent and mantissa
|
||||
const __m512i mask2 = _mm512_set1_epi16(0x4000);
|
||||
|
||||
__m512i x = _mm512_cvtepu8_epi16(a);
|
||||
__m512i vsign = _mm512_and_si512(x, mask0);
|
||||
vsign = _mm512_slli_epi16(vsign, 8);
|
||||
|
||||
__m512i vexp_and_mant = _mm512_and_si512(x, mask1);
|
||||
vexp_and_mant = _mm512_slli_epi16(vexp_and_mant, 4);
|
||||
|
||||
// _MM_TERNLOG_A | _MM_TERNLOG_B | _MM_TERNLOG_C: 0b11111110
|
||||
return (__m512bh)(_mm512_ternarylogic_epi32(vsign, mask2, vexp_and_mant, 0b11111110));
|
||||
}
|
||||
|
||||
// bias for conversion of fp8 to bf16 1/256 in float32
|
||||
#define kFP8_BIAS 0x3b800000
|
||||
|
||||
// remove warning: ignoring attributes on template argument ‘__m512bh’ [-Wignored-attributes]
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wignored-attributes"
|
||||
|
||||
#define MXFP4_VALUES \
|
||||
-6.0f, -4.0f, -3.0f, -2.0f, -1.5f, -1.0f, -0.5f, -0.0f, 6.0f, 4.0f, 3.0f, 2.0f, 1.5f, 1.0f, 0.5f, 0.0f
|
||||
|
||||
// convert 64 mxfp4 to 2x bf16 vectors, expect input 32-way packing
|
||||
inline std::tuple<__m512bh, __m512bh> cvt_mxfp4_e2m1_bf16_intrinsic_lut(__m256i a, __m512i s0, __m512i s1) {
|
||||
// LUT
|
||||
const __m512 values = _mm512_set_ps(MXFP4_VALUES);
|
||||
const __m512i lut = (__m512i)(_mm512_cvtne2ps_pbh(values, values));
|
||||
|
||||
const __m512i abs_mask = _mm512_set1_epi16(0x7FFF);
|
||||
const __m512i zero = _mm512_setzero_si512();
|
||||
|
||||
// expand values to 16-bit integers
|
||||
__m512i x0 = _mm512_cvtepu8_epi16(a);
|
||||
__m512i x1 = _mm512_srli_epi32(x0, 4);
|
||||
|
||||
// LUT to convert mxfp4 values to bf16
|
||||
x0 = _mm512_permutexvar_epi16(x0, lut);
|
||||
x1 = _mm512_permutexvar_epi16(x1, lut);
|
||||
|
||||
// check for zeros
|
||||
__mmask32 mask0 = _mm512_cmp_epi16_mask(_mm512_and_si512(x0, abs_mask), zero, _MM_CMPINT_EQ);
|
||||
__mmask32 mask1 = _mm512_cmp_epi16_mask(_mm512_and_si512(x1, abs_mask), zero, _MM_CMPINT_EQ);
|
||||
|
||||
// emulate bf16 mul with scale factor
|
||||
x0 = _mm512_add_epi16(x0, s0);
|
||||
x1 = _mm512_add_epi16(x1, s1);
|
||||
|
||||
// blend with zero
|
||||
x0 = _mm512_mask_blend_epi16(mask0, x0, zero);
|
||||
x1 = _mm512_mask_blend_epi16(mask1, x1, zero);
|
||||
|
||||
return std::make_tuple(__m512bh(x0), __m512bh(x1));
|
||||
}
|
||||
|
||||
#define CVT_MXFP4_TO_BF16(a, s0, s1) cvt_mxfp4_e2m1_bf16_intrinsic_lut(a, s0, s1)
|
||||
|
||||
#pragma GCC diagnostic pop
|
||||
|
||||
#endif
|
||||
|
||||
// vector to scalar reduction
|
||||
#if defined(CPU_CAPABILITY_AVX512) && 0
|
||||
#if defined(CPU_CAPABILITY_AVX512)
|
||||
inline float vec_reduce_sum(const Vectorized<float>& a) {
|
||||
return _mm512_reduce_add_ps(__m512(a));
|
||||
}
|
||||
@@ -136,10 +216,9 @@ inline float vec_reduce_max(const Vectorized<float>& a) {
|
||||
|
||||
// https://github.com/InternLM/lmdeploy/blob/086481ed84b59bee3b8e4274e5fc69620040c048/lmdeploy/pytorch/kernels/cuda/w8a8_triton_kernels.py#L282
|
||||
template <typename scalar_t>
|
||||
inline void quantize_row_int8(uint8_t* __restrict__ Aq, float& As,
|
||||
const scalar_t* __restrict__ A, int64_t K, float eps = 1e-7) {
|
||||
|
||||
float amax = 0.f; // absolute max
|
||||
inline void
|
||||
quantize_row_int8(uint8_t* __restrict__ Aq, float& As, const scalar_t* __restrict__ A, int64_t K, float eps = 1e-7) {
|
||||
float amax = 0.f; // absolute max
|
||||
for (int64_t k = 0; k < K; ++k) {
|
||||
const float val = static_cast<float>(A[k]);
|
||||
amax = std::max(amax, std::abs(val));
|
||||
@@ -158,9 +237,8 @@ inline void quantize_row_int8(uint8_t* __restrict__ Aq, float& As,
|
||||
|
||||
#if defined(CPU_CAPABILITY_AVX512)
|
||||
template <>
|
||||
inline void quantize_row_int8<at::BFloat16>(uint8_t* __restrict__ Aq, float& As,
|
||||
const at::BFloat16* __restrict__ A, int64_t K, float eps) {
|
||||
|
||||
inline void quantize_row_int8<at::BFloat16>(
|
||||
uint8_t* __restrict__ Aq, float& As, const at::BFloat16* __restrict__ A, int64_t K, float eps) {
|
||||
const __m512 signBit = _mm512_set1_ps(-0.0f);
|
||||
const __m512i off = _mm512_set1_epi32(128);
|
||||
|
||||
@@ -200,7 +278,7 @@ inline void quantize_row_int8<at::BFloat16>(uint8_t* __restrict__ Aq, float& As,
|
||||
// transpose utils
|
||||
// taken from my PR in ggml: https://github.com/ggml-org/llama.cpp/pull/8998
|
||||
#if defined(CPU_CAPABILITY_AVX512)
|
||||
inline void transpose_16x16_32bit(__m512i * v) {
|
||||
inline void transpose_16x16_32bit(__m512i* v) {
|
||||
__m512i v1[16];
|
||||
v1[0] = _mm512_unpacklo_epi32(v[0], v[1]);
|
||||
v1[1] = _mm512_unpackhi_epi32(v[0], v[1]);
|
||||
@@ -293,16 +371,56 @@ inline std::tuple<__m512i, __m512i> transpose_2x32_16bit(__m512i r0, __m512i r1)
|
||||
}
|
||||
#pragma GCC diagnostic pop
|
||||
|
||||
inline __attribute__((always_inline)) __m512 _mm512_fexp_u20_ps(const __m512 values) {
|
||||
const __m512 vec_c0 = _mm512_set1_ps(0.00010703434948458272f);
|
||||
const __m512 vec_c1 = _mm512_set1_ps(0.30354260500649682f);
|
||||
const __m512 vec_c2 = _mm512_set1_ps(-0.22433836478672356);
|
||||
const __m512 vec_c3 = _mm512_set1_ps(-0.079204240219773236);
|
||||
|
||||
const __m512 vec_exp_log2ef = _mm512_castsi512_ps(_mm512_set1_epi32(0x3fb8aa3b)); // log2(e)
|
||||
|
||||
const __m512 vec_a = _mm512_set1_ps(std::pow(2, 23) / std::log2(2));
|
||||
const __m512 vec_b = _mm512_set1_ps(std::pow(2, 23) * 127.f);
|
||||
|
||||
const __m512 vec_ln_flt_min = _mm512_castsi512_ps(_mm512_set1_epi32(0xc2aeac50));
|
||||
const __m512 vec_ln_flt_max = _mm512_castsi512_ps(_mm512_set1_epi32(0x42b17218));
|
||||
__m512i vec_infinity = _mm512_set1_epi32(0x7F800000);
|
||||
__m512i vec_zero = _mm512_setzero_epi32();
|
||||
|
||||
// Fast Exponential Computation on SIMD Architectures
|
||||
// A. Cristiano I. Malossi, Yves Ineichen, Costas Bekas, and Alessandro
|
||||
// Curioni exp(x) = 2**(x * log2(e))
|
||||
// = 2**xi * 2**xf - TIPS we are using the EEEE floating point
|
||||
// representation with identification to the exponent and the
|
||||
// mentissa
|
||||
// 2**xf will be approximated to a polynomial of degree 3 computed with
|
||||
// Horner method
|
||||
// mask for the boundary condition
|
||||
auto min_mask = _mm512_cmp_ps_mask(values, vec_ln_flt_min, _CMP_LT_OS);
|
||||
auto max_mask = _mm512_cmp_ps_mask(values, vec_ln_flt_max, _CMP_GT_OS);
|
||||
|
||||
// transformation with log2(e)
|
||||
auto vec_src = _mm512_mul_ps(values, vec_exp_log2ef);
|
||||
auto vec_fractional = _mm512_sub_ps(vec_src, _mm512_floor_ps(vec_src));
|
||||
|
||||
// compute polynomial using Horner Scheme, for superscalar processor
|
||||
auto vec_res = _mm512_fmadd_ps(vec_fractional, vec_c3, vec_c2);
|
||||
vec_res = _mm512_fmadd_ps(vec_fractional, vec_res, vec_c1);
|
||||
vec_res = _mm512_fmadd_ps(vec_fractional, vec_res, vec_c0);
|
||||
|
||||
vec_src = _mm512_sub_ps(vec_src, vec_res);
|
||||
// the tips is here, headache in perspective
|
||||
auto tmp = _mm512_fmadd_ps(vec_a, vec_src, vec_b);
|
||||
// headache bis - we loose precision with the cast but it "fits", but ok
|
||||
// after f32 -> f16 later
|
||||
__m512i casted_integer = _mm512_cvttps_epi32(tmp);
|
||||
// boundary condition, lower than the min -> 0
|
||||
casted_integer = _mm512_mask_mov_epi32(casted_integer, min_mask, vec_zero);
|
||||
// boundary condition, larger than the max -> +oo
|
||||
casted_integer = _mm512_mask_mov_epi32(casted_integer, max_mask, vec_infinity);
|
||||
// final interpretation to float
|
||||
return _mm512_castsi512_ps(casted_integer);
|
||||
}
|
||||
#endif
|
||||
|
||||
// TODO: debug print, remove me later
|
||||
template<typename scalar_t>
|
||||
void print_array(scalar_t* ptr, int size) {
|
||||
for (int d = 0; d < size; ++d) {
|
||||
if (d % 16 == 0) { std::cout << std::endl; }
|
||||
std::cout << ptr[d] << " ";
|
||||
}
|
||||
std::cout << std::endl;
|
||||
}
|
||||
|
||||
} // anonymous namespace
|
||||
} // anonymous namespace
|
||||
|
||||
@@ -0,0 +1,299 @@
|
||||
// Adapted from
|
||||
// https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu
|
||||
|
||||
// clang-format off
|
||||
|
||||
// To use the transpose functions
|
||||
#include <ATen/native/cpu/utils.h>
|
||||
|
||||
#include "vec.h"
|
||||
|
||||
namespace {
|
||||
|
||||
using namespace at::vec;
|
||||
|
||||
template <typename index_t>
|
||||
inline index_t get_index(index_t* ind, int i) {
|
||||
return (ind == nullptr) ? (index_t)i : ind[i];
|
||||
}
|
||||
|
||||
#if defined(CPU_CAPABILITY_AVX512)
|
||||
// key: from [N, 32] to [32/2, N, 2]
|
||||
template <typename scalar_t, typename index_t>
|
||||
inline void pack_vnni_Nx32(
|
||||
scalar_t* __restrict__ dst,
|
||||
const scalar_t* __restrict__ src,
|
||||
const index_t* __restrict__ ind,
|
||||
int N,
|
||||
int ld_src,
|
||||
int ld_dst) {
|
||||
__m512i vinputs[16];
|
||||
|
||||
int n = 0;
|
||||
for (; n < N; ++n) {
|
||||
index_t index = get_index(ind, n);
|
||||
vinputs[n] = _mm512_loadu_si512(src + index * ld_src);
|
||||
}
|
||||
// padding with zero to avoid uninitialized vectors
|
||||
for (; n < 16; ++n) {
|
||||
vinputs[n] = _mm512_set1_epi32(0);
|
||||
}
|
||||
|
||||
// pack key
|
||||
transpose_16x16_32bit(vinputs);
|
||||
|
||||
const __mmask16 vmask = (1 << N) - 1;
|
||||
for (int k = 0; k < 16; ++k) {
|
||||
_mm512_mask_storeu_epi32(dst + k * ld_dst * 2, vmask, vinputs[k]);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t, typename index_t>
|
||||
inline void pack_vnni_N_remainder(
|
||||
scalar_t* __restrict__ dst,
|
||||
const scalar_t* __restrict__ src,
|
||||
const index_t* __restrict__ ind,
|
||||
int N,
|
||||
int K,
|
||||
int ld_src,
|
||||
int ld_dst) {
|
||||
__m512i vinputs[16];
|
||||
|
||||
int K2 = K >> 1;
|
||||
const __mmask16 vmask = (1 << K2) - 1;
|
||||
|
||||
int n = 0;
|
||||
for (; n < N; ++n) {
|
||||
index_t index = get_index(ind, n);
|
||||
vinputs[n] = _mm512_maskz_loadu_epi32(vmask, src + index * ld_src);
|
||||
}
|
||||
// padding with zero to avoid uninitialized vectors
|
||||
for (; n < 16; ++n) {
|
||||
vinputs[n] = _mm512_set1_epi32(0);
|
||||
}
|
||||
|
||||
// pack key
|
||||
transpose_16x16_32bit(vinputs);
|
||||
|
||||
const __mmask16 vmask2 = (1 << N) - 1;
|
||||
for (int k = 0; k < K2; ++k) {
|
||||
_mm512_mask_storeu_epi32(dst + k * ld_dst * 2, vmask2, vinputs[k]);
|
||||
}
|
||||
}
|
||||
|
||||
// value: from [K, 32] to [K/2, 32, 2]
|
||||
template <typename scalar_t, typename index_t>
|
||||
inline void pack_vnni_Kx32(
|
||||
scalar_t* __restrict__ dst,
|
||||
const scalar_t* __restrict__ src,
|
||||
const index_t* __restrict__ ind,
|
||||
int K,
|
||||
int ld_src,
|
||||
int ld_dst) {
|
||||
__m512i vinputs[2];
|
||||
|
||||
int k = 0;
|
||||
for (; k < K; ++k) {
|
||||
index_t index = get_index(ind, k);
|
||||
vinputs[k] = _mm512_loadu_si512(src + index * ld_src);
|
||||
}
|
||||
// padding with zero to avoid uninitialized vectors
|
||||
for (; k < 2; ++k) {
|
||||
vinputs[k] = _mm512_set1_epi32(0);
|
||||
}
|
||||
|
||||
// pack value
|
||||
__m512i d0, d1;
|
||||
std::tie(d0, d1) = transpose_2x32_16bit(vinputs[0], vinputs[1]);
|
||||
_mm512_storeu_si512(dst + 0 * ld_dst * 2, d0);
|
||||
_mm512_storeu_si512(dst + 0 * ld_dst * 2 + 32, d1);
|
||||
}
|
||||
|
||||
template <typename scalar_t, typename index_t>
|
||||
inline void pack_vnni_K_remainder(
|
||||
scalar_t* __restrict__ dst,
|
||||
const scalar_t* __restrict__ src,
|
||||
const index_t* __restrict__ ind,
|
||||
int K,
|
||||
int N,
|
||||
int ld_src,
|
||||
int ld_dst) {
|
||||
__m512i vinputs[2];
|
||||
|
||||
const __mmask32 vmask = (1 << N) - 1;
|
||||
|
||||
int k = 0;
|
||||
for (; k < K; ++k) {
|
||||
index_t index = get_index(ind, k);
|
||||
vinputs[k] = _mm512_maskz_loadu_epi16(vmask, src + index * ld_src);
|
||||
}
|
||||
// padding with zero to avoid uninitialized vectors
|
||||
for (; k < 2; ++k) {
|
||||
vinputs[k] = _mm512_set1_epi32(0);
|
||||
}
|
||||
|
||||
// pack value
|
||||
__m512i d0, d1;
|
||||
std::tie(d0, d1) = transpose_2x32_16bit(vinputs[0], vinputs[1]);
|
||||
|
||||
if (N <= 16) {
|
||||
// 2N * 16bits: N * 32bits
|
||||
const __mmask16 vmask2 = (1 << N) - 1;
|
||||
_mm512_mask_storeu_epi32(dst + 0 * ld_dst * 2, vmask2, d0);
|
||||
} else {
|
||||
// 2(N-16) * 16bits: (N-16) * 32bits
|
||||
const __mmask16 vmask2 = (1 << (N - 16)) - 1;
|
||||
_mm512_storeu_epi32(dst + 0 * ld_dst * 2, d0);
|
||||
_mm512_mask_storeu_epi32(dst + 0 * ld_dst * 2 + 32, vmask2, d1);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
// convert to vnni format
|
||||
// from [N, K/2, 2] to [K/2, N, 2] for bfloat16 and float16
|
||||
template <typename scalar_t, typename index_t, bool is_indexed>
|
||||
void pack_vnni(
|
||||
scalar_t* __restrict__ dst,
|
||||
const scalar_t* __restrict__ src,
|
||||
const index_t* __restrict__ ind,
|
||||
int N,
|
||||
int K,
|
||||
int ld_src,
|
||||
int ld_dst) {
|
||||
#if defined(CPU_CAPABILITY_AVX512)
|
||||
const int NB = div_up(N, 16);
|
||||
const int KB = K / 32;
|
||||
const int K_remainder = K - KB * 32;
|
||||
|
||||
for (int nb = 0; nb < NB; ++nb) {
|
||||
int nb_size = std::min(N - nb * 16, 16);
|
||||
for (int kb = 0; kb < KB; ++kb) {
|
||||
// handle 16x512bits each block
|
||||
pack_vnni_Nx32<scalar_t, index_t>(
|
||||
/* dst */ dst + ((kb * 32) >> 1) * ld_dst * 2 + nb * 16 * 2,
|
||||
/* src */ src + kb * 32 + (is_indexed ? 0 : nb * 16 * ld_src),
|
||||
/* ind */ is_indexed ? ind + nb * 16 : nullptr,
|
||||
/* N */ nb_size,
|
||||
/* ld_src */ ld_src,
|
||||
/* ld_dst */ ld_dst);
|
||||
}
|
||||
if (K_remainder > 0) {
|
||||
pack_vnni_N_remainder<scalar_t, index_t>(
|
||||
/* dst */ dst + ((KB * 32) >> 1) * ld_dst * 2 + nb * 16 * 2,
|
||||
/* src */ src + KB * 32 + (is_indexed ? 0 : nb * 16 * ld_src),
|
||||
/* ind */ is_indexed ? ind + nb * 16 : nullptr,
|
||||
/* N */ nb_size,
|
||||
/* K */ K_remainder,
|
||||
/* ld_src */ ld_src,
|
||||
/* ld_dst */ ld_dst);
|
||||
}
|
||||
}
|
||||
#else
|
||||
for (int n = 0; n < N; ++n) {
|
||||
index_t index = get_index(ind, n);
|
||||
for (int k = 0; k < K / 2; ++k) {
|
||||
for (int d = 0; d < 2; ++d) {
|
||||
dst[k * ld_dst * 2 + n * 2 + d] = src[index * ld_src + k * 2 + d];
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
void pack_vnni(scalar_t* __restrict__ dst, const scalar_t* __restrict__ src, int N, int K, int ld_src, int ld_dst) {
|
||||
pack_vnni<scalar_t, int32_t, false>(dst, src, nullptr, N, K, ld_src, ld_dst);
|
||||
}
|
||||
|
||||
template <typename scalar_t, typename index_t>
|
||||
void pack_vnni(
|
||||
scalar_t* __restrict__ dst,
|
||||
const scalar_t* __restrict__ src,
|
||||
const index_t* __restrict__ ind,
|
||||
int N,
|
||||
int K,
|
||||
int ld_src,
|
||||
int ld_dst) {
|
||||
assert(ind != nullptr);
|
||||
pack_vnni<scalar_t, index_t, true>(dst, src, ind, N, K, ld_src, ld_dst);
|
||||
}
|
||||
|
||||
// convert to vnni format
|
||||
// from [K/2, 2, N] to [K/2, N, 2] for bfloat16 and float16
|
||||
template <typename scalar_t, typename index_t, bool is_indexed>
|
||||
void pack_vnni2(
|
||||
scalar_t* __restrict__ dst,
|
||||
const scalar_t* __restrict__ src,
|
||||
const index_t* __restrict__ ind,
|
||||
int K,
|
||||
int N,
|
||||
int ld_src,
|
||||
int ld_dst) {
|
||||
#if defined(CPU_CAPABILITY_AVX512)
|
||||
const int KB = div_up(K, 2);
|
||||
const int NB = N / 32;
|
||||
const int N_remainder = N - NB * 32;
|
||||
|
||||
for (int kb = 0; kb < KB; ++kb) {
|
||||
int kb_size = std::min(K - kb * 2, 2);
|
||||
for (int nb = 0; nb < NB; ++nb) {
|
||||
// handle 2x512bits each block
|
||||
pack_vnni_Kx32<scalar_t, index_t>(
|
||||
/* dst */ dst + ((kb * 2) >> 1) * ld_dst * 2 + nb * 32 * 2,
|
||||
/* src */ src + (is_indexed ? 0 : kb * 2 * ld_src) + nb * 32,
|
||||
/* ind */ is_indexed ? ind + kb * 2 : nullptr,
|
||||
/* K */ kb_size,
|
||||
/* ld_src */ ld_src,
|
||||
/* ld_dst */ ld_dst);
|
||||
}
|
||||
if (N_remainder > 0) {
|
||||
pack_vnni_K_remainder(
|
||||
/* dst */ dst + ((kb * 2) >> 1) * ld_dst * 2 + NB * 32 * 2,
|
||||
/* src */ src + (is_indexed ? 0 : kb * 2 * ld_src) + NB * 32,
|
||||
/* ind */ is_indexed ? ind + kb * 2 : nullptr,
|
||||
/* K */ kb_size,
|
||||
/* N */ N_remainder,
|
||||
/* ld_src */ ld_src,
|
||||
/* ld_dst */ ld_dst);
|
||||
}
|
||||
}
|
||||
#else
|
||||
int k = 0;
|
||||
for (; k < (K >> 1) * 2; k += 2) {
|
||||
index_t index0 = get_index(ind, k + 0);
|
||||
index_t index1 = get_index(ind, k + 1);
|
||||
for (int n = 0; n < N; ++n) {
|
||||
dst[(k >> 1) * ld_dst * 2 + n * 2 + 0] = src[index0 * ld_src + n];
|
||||
dst[(k >> 1) * ld_dst * 2 + n * 2 + 1] = src[index1 * ld_src + n];
|
||||
}
|
||||
}
|
||||
if (K % 2 != 0) {
|
||||
index_t index = get_index(ind, K - 1);
|
||||
for (int n = 0; n < N; ++n) {
|
||||
dst[(K >> 1) * ld_dst * 2 + n * 2 + 0] = src[index * ld_src + n];
|
||||
dst[(K >> 1) * ld_dst * 2 + n * 2 + 1] = 0;
|
||||
}
|
||||
k += 2;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
void pack_vnni2(scalar_t* __restrict__ dst, const scalar_t* __restrict__ src, int K, int N, int ld_src, int ld_dst) {
|
||||
pack_vnni2<scalar_t, int32_t, false>(dst, src, nullptr, K, N, ld_src, ld_dst);
|
||||
}
|
||||
|
||||
template <typename scalar_t, typename index_t>
|
||||
void pack_vnni2(
|
||||
scalar_t* __restrict__ dst,
|
||||
const scalar_t* __restrict__ src,
|
||||
const index_t* __restrict__ ind,
|
||||
int K,
|
||||
int N,
|
||||
int ld_src,
|
||||
int ld_dst) {
|
||||
assert(ind != nullptr);
|
||||
pack_vnni2<scalar_t, index_t, true>(dst, src, ind, K, N, ld_src, ld_dst);
|
||||
}
|
||||
|
||||
} // anonymous namespace
|
||||
+19
-12
@@ -56,6 +56,8 @@ void shm_send_tensor_list(int64_t handle,
|
||||
|
||||
std::vector<torch::Tensor> shm_recv_tensor_list(int64_t handle, int64_t src);
|
||||
|
||||
// SGL CPU kernels
|
||||
|
||||
at::Tensor weight_packed_linear(at::Tensor& mat1, at::Tensor& mat2,
|
||||
const std::optional<at::Tensor>& bias,
|
||||
bool is_vnni);
|
||||
@@ -65,12 +67,11 @@ at::Tensor convert_weight_packed(at::Tensor& weight);
|
||||
at::Tensor fused_experts_cpu(
|
||||
at::Tensor& hidden_states, at::Tensor& w1, at::Tensor& w2,
|
||||
at::Tensor& topk_weights, at::Tensor& topk_ids, bool inplace,
|
||||
bool use_int8_w8a8, bool use_fp8_w8a16,
|
||||
const std::optional<at::Tensor>& w1_scale,
|
||||
int64_t moe_comp_method, const std::optional<at::Tensor>& w1_scale,
|
||||
const std::optional<at::Tensor>& w2_scale,
|
||||
const std::optional<std::vector<int64_t>> block_size,
|
||||
const std::optional<at::Tensor>& a1_scale,
|
||||
const std::optional<at::Tensor>& a2_scale, bool is_vnni);
|
||||
const std::optional<at::Tensor>& w1_zero,
|
||||
const std::optional<at::Tensor>& w2_zero,
|
||||
const std::optional<std::vector<int64_t>> block_size, bool is_vnni);
|
||||
|
||||
at::Tensor int8_scaled_mm_with_quant(at::Tensor& mat1, at::Tensor& mat2,
|
||||
at::Tensor& scales2,
|
||||
@@ -86,7 +87,12 @@ at::Tensor fp8_scaled_mm_cpu(at::Tensor& mat1, at::Tensor& mat2,
|
||||
|
||||
// Adapted from sglang: INT4 W4A8 kernels
|
||||
std::tuple<at::Tensor, at::Tensor, at::Tensor> convert_weight_packed_scale_zp(
|
||||
at::Tensor qweight, at::Tensor qzeros, at::Tensor scales);
|
||||
at::Tensor qweight, // awq: (*, K, N / 8) || gptq: (*, K / 8, N) , int32
|
||||
at::Tensor qzeros, // awq: (*, K / group_size, N / 8) || gptq: (*, K /
|
||||
// group_size, N / 8) , int32
|
||||
at::Tensor scales, // awq: (*, K / group_size, N) || gptq: (*, K /
|
||||
// group_size, N) , bfloat16
|
||||
int64_t quant_method_4bit);
|
||||
|
||||
at::Tensor int4_scaled_mm_cpu(at::Tensor& x, at::Tensor& w, at::Tensor& w_zeros,
|
||||
at::Tensor& w_scales,
|
||||
@@ -360,10 +366,10 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
|
||||
ops.def("convert_weight_packed(Tensor! weight) -> Tensor");
|
||||
ops.impl("convert_weight_packed", torch::kCPU, &convert_weight_packed);
|
||||
ops.def(
|
||||
"fused_experts_cpu(Tensor! hidden_states, Tensor w1, Tensor w2, Tensor "
|
||||
"topk_weights, Tensor topk_ids, bool inplace, bool use_int8_w8a8, bool "
|
||||
"use_fp8_w8a16, Tensor? w1_scale, Tensor? w2_scale, SymInt[]? "
|
||||
"block_size, Tensor? a1_scale, Tensor? a2_scale, bool is_vnni) -> "
|
||||
"fused_experts_cpu(Tensor hidden_states, Tensor w1, Tensor w2, Tensor "
|
||||
"topk_weights, Tensor topk_ids, bool "
|
||||
"inplace, int moe_comp_method, Tensor? w1_scale, Tensor? w2_scale, "
|
||||
"Tensor? w1_zero, Tensor? w2_zero, int[]? block_size, bool is_vnni) -> "
|
||||
"Tensor");
|
||||
ops.impl("fused_experts_cpu", torch::kCPU, &fused_experts_cpu);
|
||||
ops.def(
|
||||
@@ -374,8 +380,9 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
|
||||
|
||||
// Adapted from sglang: INT4 W4A8 kernels
|
||||
ops.def(
|
||||
"convert_weight_packed_scale_zp(Tensor qweight, Tensor qzeros, "
|
||||
"Tensor scales) -> (Tensor, Tensor, Tensor)");
|
||||
"convert_weight_packed_scale_zp(Tensor weight, Tensor qzeros, Tensor "
|
||||
"scales, int quant_method_4bit) -> (Tensor, "
|
||||
"Tensor, Tensor)");
|
||||
ops.impl("convert_weight_packed_scale_zp", torch::kCPU,
|
||||
&convert_weight_packed_scale_zp);
|
||||
|
||||
|
||||
+1
-1
@@ -125,7 +125,7 @@ extend-exclude = ["tests/models/fixtures/*", "tests/prompts/*", "tests/tokenizer
|
||||
"examples/pooling/token_embed/*", "tests/models/language/pooling/*",
|
||||
"vllm/third_party/*", "vllm/entrypoints/serve/instrumentator/static/*", "tests/entrypoints/openai/speech_to_text/test_transcription_validation.py",
|
||||
"docs/governance/process.md", "docs/assets/contributing/vllm_bench_serve_timeline.html",
|
||||
"tests/v1/engine/test_fast_incdec_prefix_err.py", ".git/*"]
|
||||
"tests/v1/engine/test_fast_incdec_prefix_err.py", ".git/*", "csrc/cpu/sgl-kernels/*"]
|
||||
ignore-hidden = false
|
||||
|
||||
[tool.typos.default]
|
||||
|
||||
@@ -218,13 +218,12 @@ def test_w8a16_block_fp8_cpu_fused_moe(M, N, K, E, topk, seed):
|
||||
topk_weight,
|
||||
topk_ids,
|
||||
False,
|
||||
False,
|
||||
True,
|
||||
ops.CPUQuantMethod.FP8_W8A16,
|
||||
w1_s,
|
||||
w2_s,
|
||||
None,
|
||||
None,
|
||||
BLOCK_SIZE,
|
||||
None,
|
||||
None,
|
||||
True,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
@@ -242,13 +241,12 @@ def test_w8a16_block_fp8_cpu_fused_moe(M, N, K, E, topk, seed):
|
||||
topk_weight,
|
||||
topk_ids,
|
||||
True,
|
||||
False,
|
||||
True,
|
||||
ops.CPUQuantMethod.FP8_W8A16,
|
||||
w1_s,
|
||||
w2_s,
|
||||
None,
|
||||
None,
|
||||
BLOCK_SIZE,
|
||||
None,
|
||||
None,
|
||||
True,
|
||||
)
|
||||
torch.testing.assert_close(out_inplace, out, atol=0, rtol=0)
|
||||
|
||||
@@ -20,7 +20,11 @@ import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm._custom_ops import _supports_cpu_w4a8_int8
|
||||
from vllm._custom_ops import (
|
||||
CPUQuantAlgo,
|
||||
convert_weight_packed_scale_zp,
|
||||
int4_scaled_mm_cpu,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
pack_cols,
|
||||
)
|
||||
@@ -29,11 +33,6 @@ from vllm.platforms import current_platform
|
||||
if not current_platform.is_cpu():
|
||||
pytest.skip("skipping CPU-only tests", allow_module_level=True)
|
||||
|
||||
requires_cpu_w4a8_int8 = pytest.mark.skipif(
|
||||
not _supports_cpu_w4a8_int8,
|
||||
reason="Requires vLLM CPU build with SGLang INT4 W4A8 kernels",
|
||||
)
|
||||
|
||||
|
||||
def make_awq_checkpoint_data(K, N, group_size, seed=42):
|
||||
"""Create synthetic AWQ checkpoint data in packed int32 format.
|
||||
@@ -87,7 +86,6 @@ def make_awq_checkpoint_data(K, N, group_size, seed=42):
|
||||
class TestConvertWeightPackedScaleZp:
|
||||
"""Tests for convert_weight_packed_scale_zp weightpacking."""
|
||||
|
||||
@requires_cpu_w4a8_int8
|
||||
@pytest.mark.parametrize(
|
||||
"K,N,group_size",
|
||||
[
|
||||
@@ -102,8 +100,11 @@ class TestConvertWeightPackedScaleZp:
|
||||
K, N, group_size
|
||||
)
|
||||
|
||||
blocked_w, blocked_zp, blocked_s = torch.ops._C.convert_weight_packed_scale_zp(
|
||||
packed_qweight, packed_qzeros, scales
|
||||
blocked_w, blocked_zp, blocked_s = convert_weight_packed_scale_zp(
|
||||
packed_qweight,
|
||||
packed_qzeros,
|
||||
scales,
|
||||
CPUQuantAlgo.AWQ,
|
||||
)
|
||||
|
||||
block_n = 32
|
||||
@@ -129,7 +130,6 @@ class TestConvertWeightPackedScaleZp:
|
||||
class TestInt4ScaledMmCpu:
|
||||
"""Tests for int4_scaled_mm_cpu GEMM kernel."""
|
||||
|
||||
@requires_cpu_w4a8_int8
|
||||
@pytest.mark.parametrize(
|
||||
"M,K,N,group_size",
|
||||
[
|
||||
@@ -146,12 +146,15 @@ class TestInt4ScaledMmCpu:
|
||||
make_awq_checkpoint_data(K, N, group_size)
|
||||
)
|
||||
|
||||
blocked_w, blocked_zp, blocked_s = torch.ops._C.convert_weight_packed_scale_zp(
|
||||
packed_qweight, packed_qzeros, scales
|
||||
blocked_w, blocked_zp, blocked_s = convert_weight_packed_scale_zp(
|
||||
packed_qweight,
|
||||
packed_qzeros,
|
||||
scales,
|
||||
CPUQuantAlgo.AWQ,
|
||||
)
|
||||
|
||||
x = torch.randn(M, K, dtype=torch.bfloat16)
|
||||
out = torch.ops._C.int4_scaled_mm_cpu(x, blocked_w, blocked_zp, blocked_s, None)
|
||||
out = int4_scaled_mm_cpu(x, blocked_w, blocked_zp, blocked_s, None)
|
||||
|
||||
ref_out = torch.mm(x.float(), float_ref)
|
||||
|
||||
@@ -169,7 +172,6 @@ class TestInt4ScaledMmCpu:
|
||||
)
|
||||
print(f" [PASS] INT4 GEMM correct: M={M}, K={K}, N={N}")
|
||||
|
||||
@requires_cpu_w4a8_int8
|
||||
@pytest.mark.parametrize("M", [1, 8, 32])
|
||||
def test_gemm_with_bias(self, M):
|
||||
"""INT4 W4A8 GEMM with bias should match reference."""
|
||||
@@ -178,14 +180,17 @@ class TestInt4ScaledMmCpu:
|
||||
make_awq_checkpoint_data(K, N, group_size)
|
||||
)
|
||||
|
||||
blocked_w, blocked_zp, blocked_s = torch.ops._C.convert_weight_packed_scale_zp(
|
||||
packed_qweight, packed_qzeros, scales
|
||||
blocked_w, blocked_zp, blocked_s = convert_weight_packed_scale_zp(
|
||||
packed_qweight,
|
||||
packed_qzeros,
|
||||
scales,
|
||||
CPUQuantAlgo.AWQ,
|
||||
)
|
||||
|
||||
bias = torch.randn(N, dtype=torch.float32)
|
||||
x = torch.randn(M, K, dtype=torch.bfloat16)
|
||||
|
||||
out = torch.ops._C.int4_scaled_mm_cpu(x, blocked_w, blocked_zp, blocked_s, bias)
|
||||
out = int4_scaled_mm_cpu(x, blocked_w, blocked_zp, blocked_s, bias)
|
||||
|
||||
ref_out = torch.mm(x.float(), float_ref) + bias
|
||||
abs_diff = (out.float() - ref_out).abs()
|
||||
@@ -197,7 +202,6 @@ class TestInt4ScaledMmCpu:
|
||||
)
|
||||
print(f" [PASS] INT4 GEMM with bias: M={M}")
|
||||
|
||||
@requires_cpu_w4a8_int8
|
||||
def test_gemm_3d_input(self):
|
||||
"""apply() reshapes 3D input [B, S, K] -> [B*S, K] -> back to 3D."""
|
||||
K, N, group_size = 256, 128, 128
|
||||
@@ -205,17 +209,18 @@ class TestInt4ScaledMmCpu:
|
||||
make_awq_checkpoint_data(K, N, group_size)
|
||||
)
|
||||
|
||||
blocked_w, blocked_zp, blocked_s = torch.ops._C.convert_weight_packed_scale_zp(
|
||||
packed_qweight, packed_qzeros, scales
|
||||
blocked_w, blocked_zp, blocked_s = convert_weight_packed_scale_zp(
|
||||
packed_qweight,
|
||||
packed_qzeros,
|
||||
scales,
|
||||
CPUQuantAlgo.AWQ,
|
||||
)
|
||||
|
||||
B, S = 2, 8
|
||||
x_3d = torch.randn(B, S, K, dtype=torch.bfloat16)
|
||||
x_2d = x_3d.reshape(-1, K)
|
||||
|
||||
out_2d = torch.ops._C.int4_scaled_mm_cpu(
|
||||
x_2d, blocked_w, blocked_zp, blocked_s, None
|
||||
)
|
||||
out_2d = int4_scaled_mm_cpu(x_2d, blocked_w, blocked_zp, blocked_s, None)
|
||||
out_3d = out_2d.reshape(B, S, N)
|
||||
|
||||
ref_out = torch.mm(x_2d.float(), float_ref).reshape(B, S, N)
|
||||
@@ -229,7 +234,6 @@ class TestInt4ScaledMmCpu:
|
||||
assert mean_rel < 0.05, f"Mean relative error {mean_rel:.4f} for 3D exceeds 5%"
|
||||
print(f" [PASS] 3D input [{B},{S},{K}] -> output [{B},{S},{N}]")
|
||||
|
||||
@requires_cpu_w4a8_int8
|
||||
def test_gemm_fp16_input(self):
|
||||
"""INT4 GEMM should also work with fp16 input."""
|
||||
K, N, group_size, M = 256, 256, 128, 8
|
||||
@@ -237,12 +241,15 @@ class TestInt4ScaledMmCpu:
|
||||
make_awq_checkpoint_data(K, N, group_size)
|
||||
)
|
||||
|
||||
blocked_w, blocked_zp, blocked_s = torch.ops._C.convert_weight_packed_scale_zp(
|
||||
packed_qweight, packed_qzeros, scales
|
||||
blocked_w, blocked_zp, blocked_s = convert_weight_packed_scale_zp(
|
||||
packed_qweight,
|
||||
packed_qzeros,
|
||||
scales,
|
||||
CPUQuantAlgo.AWQ,
|
||||
)
|
||||
|
||||
x = torch.randn(M, K, dtype=torch.float16)
|
||||
out = torch.ops._C.int4_scaled_mm_cpu(x, blocked_w, blocked_zp, blocked_s, None)
|
||||
out = int4_scaled_mm_cpu(x, blocked_w, blocked_zp, blocked_s, None)
|
||||
|
||||
ref_out = torch.mm(x.float(), float_ref)
|
||||
abs_diff = (out.float() - ref_out).abs()
|
||||
|
||||
+57
-13
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from enum import IntEnum
|
||||
from typing import TYPE_CHECKING, Literal
|
||||
|
||||
import torch
|
||||
@@ -3150,6 +3151,13 @@ if hasattr(torch.ops._C, "weight_packed_linear"):
|
||||
)
|
||||
|
||||
|
||||
class CPUQuantMethod(IntEnum):
|
||||
UNQUANT = 0
|
||||
INT8_W8A8 = 1
|
||||
FP8_W8A16 = 2
|
||||
INT4_W4A8 = 3
|
||||
|
||||
|
||||
if hasattr(torch.ops._C, "fused_experts_cpu"):
|
||||
|
||||
@register_fake("_C::fused_experts_cpu")
|
||||
@@ -3160,13 +3168,12 @@ if hasattr(torch.ops._C, "fused_experts_cpu"):
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
inplace: bool,
|
||||
use_int8_w8a8: bool,
|
||||
use_fp8_w8a16: bool,
|
||||
moe_comp_method: CPUQuantMethod,
|
||||
w1_scale: torch.Tensor | None,
|
||||
w2_scale: torch.Tensor | None,
|
||||
w1_zero: torch.Tensor | None,
|
||||
w2_zero: torch.Tensor | None,
|
||||
block_size: list[int] | None,
|
||||
a1_scale: torch.Tensor | None,
|
||||
a2_scale: torch.Tensor | None,
|
||||
is_vnni: bool,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty_like(hidden_states)
|
||||
@@ -3179,13 +3186,12 @@ def fused_experts_cpu(
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
inplace: bool,
|
||||
use_int8_w8a8: bool,
|
||||
use_fp8_w8a16: bool,
|
||||
moe_comp_method: CPUQuantMethod,
|
||||
w1_scale: torch.Tensor | None,
|
||||
w2_scale: torch.Tensor | None,
|
||||
w1_zero: torch.Tensor | None,
|
||||
w2_zero: torch.Tensor | None,
|
||||
block_size: list[int] | None,
|
||||
a1_scale: torch.Tensor | None,
|
||||
a2_scale: torch.Tensor | None,
|
||||
is_vnni: bool,
|
||||
) -> torch.Tensor:
|
||||
return torch.ops._C.fused_experts_cpu(
|
||||
@@ -3195,13 +3201,12 @@ def fused_experts_cpu(
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
inplace,
|
||||
use_int8_w8a8,
|
||||
use_fp8_w8a16,
|
||||
moe_comp_method,
|
||||
w1_scale,
|
||||
w2_scale,
|
||||
w1_zero,
|
||||
w2_zero,
|
||||
block_size,
|
||||
a1_scale,
|
||||
a2_scale,
|
||||
is_vnni,
|
||||
)
|
||||
|
||||
@@ -3222,6 +3227,11 @@ if hasattr(torch.ops._C, "int8_scaled_mm_with_quant"):
|
||||
return torch.empty((M, N), dtype=out_dtype)
|
||||
|
||||
|
||||
class CPUQuantAlgo(IntEnum):
|
||||
AWQ = 0
|
||||
GPTQ = 1
|
||||
|
||||
|
||||
if hasattr(torch.ops._C, "convert_weight_packed_scale_zp"):
|
||||
|
||||
@register_fake("_C::convert_weight_packed_scale_zp")
|
||||
@@ -3229,6 +3239,7 @@ if hasattr(torch.ops._C, "convert_weight_packed_scale_zp"):
|
||||
qweight: torch.Tensor,
|
||||
qzeros: torch.Tensor,
|
||||
scales: torch.Tensor,
|
||||
quant_method_4bit: CPUQuantAlgo,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
return (
|
||||
torch.empty_like(qweight),
|
||||
@@ -3237,6 +3248,20 @@ if hasattr(torch.ops._C, "convert_weight_packed_scale_zp"):
|
||||
)
|
||||
|
||||
|
||||
def convert_weight_packed_scale_zp(
|
||||
qweight: torch.Tensor,
|
||||
qzeros: torch.Tensor,
|
||||
scales: torch.Tensor,
|
||||
quant_method_4bit: CPUQuantAlgo,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
return torch.ops._C.convert_weight_packed_scale_zp(
|
||||
qweight,
|
||||
qzeros,
|
||||
scales,
|
||||
quant_method_4bit,
|
||||
)
|
||||
|
||||
|
||||
if hasattr(torch.ops._C, "int4_scaled_mm_cpu"):
|
||||
|
||||
@register_fake("_C::int4_scaled_mm_cpu")
|
||||
@@ -3251,7 +3276,26 @@ if hasattr(torch.ops._C, "int4_scaled_mm_cpu"):
|
||||
return torch.empty((x.size(0), N), dtype=x.dtype, device=x.device)
|
||||
|
||||
|
||||
_supports_cpu_w4a8_int8 = bool(hasattr(torch.ops._C, "convert_weight_packed_scale_zp"))
|
||||
def int4_scaled_mm_cpu(
|
||||
x: torch.Tensor,
|
||||
w: torch.Tensor,
|
||||
w_zeros: torch.Tensor,
|
||||
w_scales: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
x_shape = x.shape
|
||||
x_2d = x.reshape(-1, x_shape[-1]) if len(x_shape) > 2 else x
|
||||
|
||||
out = torch.ops._C.int4_scaled_mm_cpu(
|
||||
x_2d,
|
||||
w,
|
||||
w_zeros,
|
||||
w_scales,
|
||||
bias,
|
||||
)
|
||||
out = out.reshape(x_shape[:-1] + (out.size(-1),)) if len(x_shape) > 2 else out
|
||||
return out
|
||||
|
||||
|
||||
if hasattr(torch.ops._C, "fp8_scaled_mm_cpu"):
|
||||
|
||||
|
||||
@@ -7,7 +7,12 @@ import torch
|
||||
from torch.nn import functional as F
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm._custom_ops import cpu_fused_moe, cpu_prepack_moe_weight
|
||||
from vllm._custom_ops import (
|
||||
CPUQuantMethod,
|
||||
cpu_fused_moe,
|
||||
cpu_prepack_moe_weight,
|
||||
fused_experts_cpu,
|
||||
)
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.quantization.utils.layer_utils import replace_parameter
|
||||
@@ -195,23 +200,21 @@ class SGLFusedMOE:
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
)
|
||||
|
||||
torch.ops._C.fused_experts_cpu(
|
||||
return fused_experts_cpu(
|
||||
x,
|
||||
layer.w13_weight,
|
||||
layer.w2_weight,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
True,
|
||||
False,
|
||||
False,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
True,
|
||||
False, # inplace
|
||||
CPUQuantMethod.UNQUANT, # moe_comp_method
|
||||
None, # w1_scale
|
||||
None, # w2_scale
|
||||
None, # w1_zero
|
||||
None, # w2_zero
|
||||
None, # block_size
|
||||
True, # is_vnni
|
||||
)
|
||||
return x
|
||||
|
||||
|
||||
class CPUFusedMOE:
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm._custom_ops import CPUQuantMethod, fused_experts_cpu
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
@@ -157,19 +158,18 @@ class CPUExpertsFp8(mk.FusedMoEExpertsMonolithic):
|
||||
)
|
||||
)
|
||||
|
||||
return torch.ops._C.fused_experts_cpu(
|
||||
return fused_experts_cpu(
|
||||
hidden_states,
|
||||
w1,
|
||||
w2,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
False, # inplace
|
||||
False, # use_int8_w8a8
|
||||
True, # use_fp8_w8a16
|
||||
CPUQuantMethod.FP8_W8A16, # moe_comp_method
|
||||
self.w1_scale, # w1_scale
|
||||
self.w2_scale, # w2_scale
|
||||
None, # w1_zero
|
||||
None, # w2_zero
|
||||
block_shape, # block_size
|
||||
None, # a1_scale (W8A16: no activation quantization)
|
||||
None, # a2_scale
|
||||
True, # is_vnni
|
||||
)
|
||||
|
||||
@@ -287,8 +287,11 @@ class CPUAWQLinearMethod(LinearMethodBase):
|
||||
packed_weight = layer.qweight.data
|
||||
packed_zeros = layer.qzeros.data
|
||||
scales = layer.scales.data
|
||||
blocked_w, blocked_zp, blocked_s = torch.ops._C.convert_weight_packed_scale_zp(
|
||||
packed_weight, packed_zeros, scales
|
||||
blocked_w, blocked_zp, blocked_s = ops.convert_weight_packed_scale_zp(
|
||||
packed_weight,
|
||||
packed_zeros,
|
||||
scales,
|
||||
ops.CPUQuantAlgo.AWQ,
|
||||
)
|
||||
|
||||
layer.packed_weight = blocked_w
|
||||
@@ -334,18 +337,13 @@ class CPUAWQLinearMethod(LinearMethodBase):
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""SGLang INT4 W4A8 GEMM path."""
|
||||
x_shape = x.shape
|
||||
x_2d = x.reshape(-1, x_shape[-1]) if len(x_shape) > 2 else x
|
||||
|
||||
out = torch.ops._C.int4_scaled_mm_cpu(
|
||||
x_2d,
|
||||
return ops.int4_scaled_mm_cpu(
|
||||
x,
|
||||
layer.packed_weight,
|
||||
layer.packed_qzeros,
|
||||
layer.packed_scales,
|
||||
bias,
|
||||
)
|
||||
out = out.reshape(x_shape[:-1] + (out.size(-1),)) if len(x_shape) > 2 else out
|
||||
return out
|
||||
|
||||
|
||||
def _get_isa_hint(dtype: torch.dtype) -> str:
|
||||
|
||||
Reference in New Issue
Block a user