[CPU] Add CPU FP8 W8A8 linear/MoE support (#49942)
Signed-off-by: yuwenzho <yuwen.zhou@intel.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
co-authored by
Li, Jiang <jiang1.li@intel.com>
parent
6df2b1a8c6
commit
b346479065
@@ -12,11 +12,45 @@
|
||||
#include <ATen/ATen.h>
|
||||
#include <ATen/Dispatch.h>
|
||||
#include <ATen/Parallel.h>
|
||||
#include <cstring>
|
||||
|
||||
#if defined(_OPENMP)
|
||||
#include <omp.h>
|
||||
#endif
|
||||
|
||||
// Adapted from sglang: dispatch utilities for FP8 W8A8 kernels
|
||||
#define AT_DISPATCH_OUT_TYPES(TYPE, NAME, ...) \
|
||||
AT_DISPATCH_SWITCH( \
|
||||
TYPE, \
|
||||
NAME, \
|
||||
AT_PRIVATE_CASE_TYPE_USING_HINT(at::ScalarType::Float, out_t, __VA_ARGS__) \
|
||||
AT_PRIVATE_CASE_TYPE_USING_HINT(at::ScalarType::BFloat16, out_t, __VA_ARGS__) \
|
||||
AT_PRIVATE_CASE_TYPE_USING_HINT(at::ScalarType::Half, out_t, __VA_ARGS__))
|
||||
|
||||
#define AT_DISPATCH_CASE_ENUM_NO_RETURN(VALUE, TYPE, HINT, ...) \
|
||||
case VALUE: { \
|
||||
constexpr TYPE HINT = VALUE; \
|
||||
__VA_ARGS__; \
|
||||
break; \
|
||||
}
|
||||
|
||||
#define AT_DISPATCH_BOOL_NO_RETURN(VALUE, NAME, HINT, ...) \
|
||||
[&]() { \
|
||||
switch (VALUE) { \
|
||||
AT_DISPATCH_CASE_ENUM_NO_RETURN(true, bool, HINT, __VA_ARGS__) \
|
||||
AT_DISPATCH_CASE_ENUM_NO_RETURN(false, bool, HINT, __VA_ARGS__) \
|
||||
} \
|
||||
}()
|
||||
|
||||
#define AT_DISPATCH_QUANT_MODE_NO_RETURN(MODE, NAME, HINT, ...) \
|
||||
[&]() { \
|
||||
switch (MODE) { \
|
||||
AT_DISPATCH_CASE_ENUM_NO_RETURN(PER_TENSOR, int, HINT, __VA_ARGS__) \
|
||||
AT_DISPATCH_CASE_ENUM_NO_RETURN(PER_ROW, int, HINT, __VA_ARGS__) \
|
||||
AT_DISPATCH_CASE_ENUM_NO_RETURN(PER_GROUP, int, HINT, __VA_ARGS__) \
|
||||
} \
|
||||
}()
|
||||
|
||||
namespace {
|
||||
|
||||
// dispatch bool
|
||||
@@ -472,4 +506,53 @@ inline T* conditional_data_ptr(const std::optional<at::Tensor>& opt) {
|
||||
return opt.has_value() ? opt.value().data_ptr<T>() : nullptr;
|
||||
}
|
||||
|
||||
// Adapted from sglang: M blocking utilities for FP8 W8A8 kernels
|
||||
inline int64_t get_m_block(int64_t M) {
|
||||
if (M <= 48) {
|
||||
return M;
|
||||
} else if (M < 64) {
|
||||
return 32;
|
||||
} else if (M < 96) {
|
||||
return 64;
|
||||
} else {
|
||||
return 128;
|
||||
}
|
||||
}
|
||||
|
||||
inline std::tuple<bool, int64_t, int64_t, int64_t> get_m_blocking(int64_t M) {
|
||||
bool parallel_on_M = M > 128;
|
||||
int64_t block_m = get_m_block(M);
|
||||
int64_t Mc = (M + block_m - 1) / block_m;
|
||||
int64_t Mc_parallel = parallel_on_M ? Mc : 1;
|
||||
return std::make_tuple(parallel_on_M, block_m, Mc, Mc_parallel);
|
||||
}
|
||||
|
||||
#if defined(CPU_CAPABILITY_AVX512)
|
||||
template <typename T>
|
||||
void zero_buffer(T* data, int64_t size) {
|
||||
const int32_t vec_size = at::vec::Vectorized<T>::size();
|
||||
auto zero_vec = at::vec::Vectorized<T>(0);
|
||||
int64_t d = 0;
|
||||
for (; d < size - (size % vec_size); d += vec_size) {
|
||||
zero_vec.store(data + d);
|
||||
}
|
||||
if (d < size) {
|
||||
zero_vec.store(data + d, size - d);
|
||||
}
|
||||
}
|
||||
#else
|
||||
template <typename T>
|
||||
void zero_buffer(T* data, int64_t size) {
|
||||
memset(data, 0, sizeof(T) * size);
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(__x86_64__)
|
||||
bool avx10_2_available() {
|
||||
// __builtin_cpu_supports returns the masked feature bit rather than 0/1.
|
||||
static const bool available = __builtin_cpu_supports("avx10.2");
|
||||
return available;
|
||||
}
|
||||
#endif
|
||||
|
||||
} // anonymous namespace
|
||||
|
||||
Reference in New Issue
Block a user