[CPU] Add MXFP4 W4A16 MoE support (#41922)
Signed-off-by: yuwenzho <yuwen.zhou@intel.com> Signed-off-by: Yuwen Zhou <yuwen.zhou@intel.com>
This commit is contained in:
@@ -67,6 +67,23 @@ inline void copy_mul_stub(scalar_t* __restrict__ out, const float* __restrict__
|
||||
}
|
||||
}
|
||||
|
||||
template <>
|
||||
inline void
|
||||
copy_add_stub(float* __restrict__ out, const float* __restrict__ input, const float* __restrict__ bias, int64_t size) {
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
constexpr int kVecSize = fVec::size();
|
||||
|
||||
int64_t d;
|
||||
#pragma GCC unroll 4
|
||||
for (d = 0; d <= size - kVecSize; d += kVecSize) {
|
||||
fVec data = fVec::loadu(input + d) + fVec::loadu(bias + d);
|
||||
data.store(out + d);
|
||||
}
|
||||
for (; d < size; ++d) {
|
||||
out[d] = input[d] + bias[d];
|
||||
}
|
||||
}
|
||||
|
||||
inline void unpack_B(
|
||||
at::BFloat16* __restrict__ Btmp,
|
||||
const at::Float8_e4m3fn* __restrict__ packed_B,
|
||||
@@ -343,7 +360,6 @@ struct tinygemm_kernel_nn<at::BFloat16, at::Float8_e4m3fn, float, has_bias, BLOC
|
||||
Unroll<ROWS * COLS>{}(storec);
|
||||
}
|
||||
};
|
||||
|
||||
template <int BLOCK_M, int BLOCK_N>
|
||||
struct tinygemm_kernel_nn2<at::BFloat16, BLOCK_M, BLOCK_N> {
|
||||
static inline void apply(
|
||||
@@ -918,6 +934,7 @@ void tinygemm_kernel(
|
||||
scalar_t* __restrict__ C,
|
||||
scalar_t* __restrict__ Btmp,
|
||||
float* __restrict__ Ctmp,
|
||||
const float* __restrict__ Bbias,
|
||||
const float* __restrict__ scale,
|
||||
int64_t M,
|
||||
int64_t N,
|
||||
@@ -928,6 +945,11 @@ void tinygemm_kernel(
|
||||
bool brg,
|
||||
int64_t block_size_K,
|
||||
bool do_unpack) {
|
||||
if (Bbias != nullptr) {
|
||||
tinygemm_kernel<scalar_t, at::Float8_e4m3fn, float, true>(
|
||||
A, B, C, Btmp, Ctmp, scale, Bbias, M, N, K, lda, ldb, ldc, brg, block_size_K, do_unpack);
|
||||
return;
|
||||
}
|
||||
tinygemm_kernel<scalar_t, at::Float8_e4m3fn, float, false>(
|
||||
A, B, C, Btmp, Ctmp, scale, nullptr, M, N, K, lda, ldb, ldc, brg, block_size_K, do_unpack);
|
||||
}
|
||||
@@ -949,7 +971,6 @@ void tinygemm_kernel(
|
||||
bool brg) {
|
||||
tinygemm_kernel2<scalar_t>(A, B, C, Btmp, Ctmp, scale, M, N, K, lda, ldb, ldc, brg);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
void tinygemm_kernel(
|
||||
const scalar_t* __restrict__ A,
|
||||
@@ -957,6 +978,7 @@ void tinygemm_kernel(
|
||||
scalar_t* __restrict__ C,
|
||||
scalar_t* __restrict__ Btmp,
|
||||
float* __restrict__ Ctmp,
|
||||
const float* __restrict__ Bbias,
|
||||
const uint8_t* __restrict__ scale,
|
||||
int64_t M,
|
||||
int64_t N,
|
||||
@@ -967,10 +989,68 @@ void tinygemm_kernel(
|
||||
bool brg,
|
||||
int64_t block_size_K,
|
||||
bool do_unpack) {
|
||||
if (Bbias != nullptr) {
|
||||
tinygemm_kernel<scalar_t, uint8_t, uint8_t, true>(
|
||||
A, B, C, Btmp, Ctmp, scale, Bbias, M, N, K, lda, ldb, ldc, brg, block_size_K, do_unpack);
|
||||
return;
|
||||
}
|
||||
tinygemm_kernel<scalar_t, uint8_t, uint8_t, false>(
|
||||
A, B, C, Btmp, Ctmp, scale, nullptr, M, N, K, lda, ldb, ldc, brg, block_size_K, do_unpack);
|
||||
}
|
||||
|
||||
// tinygemm interface
|
||||
template <typename scalar_t>
|
||||
void tinygemm_kernel(
|
||||
const scalar_t* __restrict__ A,
|
||||
const at::Float8_e4m3fn* __restrict__ B,
|
||||
float* __restrict__ C,
|
||||
scalar_t* __restrict__ Btmp,
|
||||
const float* __restrict__ Bbias,
|
||||
const float* __restrict__ scale,
|
||||
int64_t M,
|
||||
int64_t N,
|
||||
int64_t K,
|
||||
int64_t lda,
|
||||
int64_t ldb,
|
||||
int64_t ldc,
|
||||
bool brg,
|
||||
int64_t block_size_K,
|
||||
bool do_unpack) {
|
||||
if (Bbias != nullptr) {
|
||||
tinygemm_kernel<scalar_t, at::Float8_e4m3fn, float, true>(
|
||||
A, B, C, Btmp, scale, Bbias, M, N, K, lda, ldb, ldc, brg, block_size_K, do_unpack);
|
||||
return;
|
||||
}
|
||||
tinygemm_kernel<scalar_t, at::Float8_e4m3fn, float, false>(
|
||||
A, B, C, Btmp, scale, nullptr, M, N, K, lda, ldb, ldc, brg, block_size_K, do_unpack);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
void tinygemm_kernel(
|
||||
const scalar_t* __restrict__ A,
|
||||
const uint8_t* __restrict__ B,
|
||||
float* __restrict__ C,
|
||||
scalar_t* __restrict__ Btmp,
|
||||
const float* __restrict__ Bbias,
|
||||
const uint8_t* __restrict__ scale,
|
||||
int64_t M,
|
||||
int64_t N,
|
||||
int64_t K,
|
||||
int64_t lda,
|
||||
int64_t ldb,
|
||||
int64_t ldc,
|
||||
bool brg,
|
||||
int64_t block_size_K,
|
||||
bool do_unpack) {
|
||||
if (Bbias != nullptr) {
|
||||
tinygemm_kernel<scalar_t, uint8_t, uint8_t, true>(
|
||||
A, B, C, Btmp, scale, Bbias, M, N, K, lda, ldb, ldc, brg, block_size_K, do_unpack);
|
||||
return;
|
||||
}
|
||||
tinygemm_kernel<scalar_t, uint8_t, uint8_t, false>(
|
||||
A, B, C, Btmp, scale, nullptr, M, N, K, lda, ldb, ldc, brg, block_size_K, do_unpack);
|
||||
}
|
||||
|
||||
#define INSTANTIATE_TINYGEMM_TEMPLATE(TYPE_A, TYPE_B, TYPE_S) \
|
||||
template void tinygemm_kernel<TYPE_A>( \
|
||||
const TYPE_A* __restrict__ A, \
|
||||
@@ -978,6 +1058,7 @@ void tinygemm_kernel(
|
||||
TYPE_A* __restrict__ C, \
|
||||
TYPE_A* __restrict__ Btmp, \
|
||||
float* __restrict__ Ctmp, \
|
||||
const float* __restrict__ Bbias, \
|
||||
const TYPE_S* __restrict__ scale, \
|
||||
int64_t M, \
|
||||
int64_t N, \
|
||||
@@ -1020,7 +1101,6 @@ inline const float* get_bias_data(const std::optional<at::Tensor>& bias, int64_t
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// FP8 and MXFP4 WoQ uses the same pattern:
|
||||
// Btmp : [T, BLOCK_N * K]
|
||||
// Ctmp : [T, BLOCK_M * BLOCK_N]
|
||||
|
||||
Reference in New Issue
Block a user