[CPU] Add MXFP4 W4A16 MoE support (#41922)

Signed-off-by: yuwenzho <yuwen.zhou@intel.com>
Signed-off-by: Yuwen Zhou <yuwen.zhou@intel.com>
This commit is contained in:
Yuwen Zhou
2026-05-18 03:04:45 -07:00
committed by GitHub
parent cac81b6eda
commit 88a860d754
16 changed files with 1172 additions and 349 deletions
+83 -3
View File
@@ -67,6 +67,23 @@ inline void copy_mul_stub(scalar_t* __restrict__ out, const float* __restrict__
}
}
template <>
inline void
copy_add_stub(float* __restrict__ out, const float* __restrict__ input, const float* __restrict__ bias, int64_t size) {
using fVec = at::vec::Vectorized<float>;
constexpr int kVecSize = fVec::size();
int64_t d;
#pragma GCC unroll 4
for (d = 0; d <= size - kVecSize; d += kVecSize) {
fVec data = fVec::loadu(input + d) + fVec::loadu(bias + d);
data.store(out + d);
}
for (; d < size; ++d) {
out[d] = input[d] + bias[d];
}
}
inline void unpack_B(
at::BFloat16* __restrict__ Btmp,
const at::Float8_e4m3fn* __restrict__ packed_B,
@@ -343,7 +360,6 @@ struct tinygemm_kernel_nn<at::BFloat16, at::Float8_e4m3fn, float, has_bias, BLOC
Unroll<ROWS * COLS>{}(storec);
}
};
template <int BLOCK_M, int BLOCK_N>
struct tinygemm_kernel_nn2<at::BFloat16, BLOCK_M, BLOCK_N> {
static inline void apply(
@@ -918,6 +934,7 @@ void tinygemm_kernel(
scalar_t* __restrict__ C,
scalar_t* __restrict__ Btmp,
float* __restrict__ Ctmp,
const float* __restrict__ Bbias,
const float* __restrict__ scale,
int64_t M,
int64_t N,
@@ -928,6 +945,11 @@ void tinygemm_kernel(
bool brg,
int64_t block_size_K,
bool do_unpack) {
if (Bbias != nullptr) {
tinygemm_kernel<scalar_t, at::Float8_e4m3fn, float, true>(
A, B, C, Btmp, Ctmp, scale, Bbias, M, N, K, lda, ldb, ldc, brg, block_size_K, do_unpack);
return;
}
tinygemm_kernel<scalar_t, at::Float8_e4m3fn, float, false>(
A, B, C, Btmp, Ctmp, scale, nullptr, M, N, K, lda, ldb, ldc, brg, block_size_K, do_unpack);
}
@@ -949,7 +971,6 @@ void tinygemm_kernel(
bool brg) {
tinygemm_kernel2<scalar_t>(A, B, C, Btmp, Ctmp, scale, M, N, K, lda, ldb, ldc, brg);
}
template <typename scalar_t>
void tinygemm_kernel(
const scalar_t* __restrict__ A,
@@ -957,6 +978,7 @@ void tinygemm_kernel(
scalar_t* __restrict__ C,
scalar_t* __restrict__ Btmp,
float* __restrict__ Ctmp,
const float* __restrict__ Bbias,
const uint8_t* __restrict__ scale,
int64_t M,
int64_t N,
@@ -967,10 +989,68 @@ void tinygemm_kernel(
bool brg,
int64_t block_size_K,
bool do_unpack) {
if (Bbias != nullptr) {
tinygemm_kernel<scalar_t, uint8_t, uint8_t, true>(
A, B, C, Btmp, Ctmp, scale, Bbias, M, N, K, lda, ldb, ldc, brg, block_size_K, do_unpack);
return;
}
tinygemm_kernel<scalar_t, uint8_t, uint8_t, false>(
A, B, C, Btmp, Ctmp, scale, nullptr, M, N, K, lda, ldb, ldc, brg, block_size_K, do_unpack);
}
// tinygemm interface
template <typename scalar_t>
void tinygemm_kernel(
const scalar_t* __restrict__ A,
const at::Float8_e4m3fn* __restrict__ B,
float* __restrict__ C,
scalar_t* __restrict__ Btmp,
const float* __restrict__ Bbias,
const float* __restrict__ scale,
int64_t M,
int64_t N,
int64_t K,
int64_t lda,
int64_t ldb,
int64_t ldc,
bool brg,
int64_t block_size_K,
bool do_unpack) {
if (Bbias != nullptr) {
tinygemm_kernel<scalar_t, at::Float8_e4m3fn, float, true>(
A, B, C, Btmp, scale, Bbias, M, N, K, lda, ldb, ldc, brg, block_size_K, do_unpack);
return;
}
tinygemm_kernel<scalar_t, at::Float8_e4m3fn, float, false>(
A, B, C, Btmp, scale, nullptr, M, N, K, lda, ldb, ldc, brg, block_size_K, do_unpack);
}
template <typename scalar_t>
void tinygemm_kernel(
const scalar_t* __restrict__ A,
const uint8_t* __restrict__ B,
float* __restrict__ C,
scalar_t* __restrict__ Btmp,
const float* __restrict__ Bbias,
const uint8_t* __restrict__ scale,
int64_t M,
int64_t N,
int64_t K,
int64_t lda,
int64_t ldb,
int64_t ldc,
bool brg,
int64_t block_size_K,
bool do_unpack) {
if (Bbias != nullptr) {
tinygemm_kernel<scalar_t, uint8_t, uint8_t, true>(
A, B, C, Btmp, scale, Bbias, M, N, K, lda, ldb, ldc, brg, block_size_K, do_unpack);
return;
}
tinygemm_kernel<scalar_t, uint8_t, uint8_t, false>(
A, B, C, Btmp, scale, nullptr, M, N, K, lda, ldb, ldc, brg, block_size_K, do_unpack);
}
#define INSTANTIATE_TINYGEMM_TEMPLATE(TYPE_A, TYPE_B, TYPE_S) \
template void tinygemm_kernel<TYPE_A>( \
const TYPE_A* __restrict__ A, \
@@ -978,6 +1058,7 @@ void tinygemm_kernel(
TYPE_A* __restrict__ C, \
TYPE_A* __restrict__ Btmp, \
float* __restrict__ Ctmp, \
const float* __restrict__ Bbias, \
const TYPE_S* __restrict__ scale, \
int64_t M, \
int64_t N, \
@@ -1020,7 +1101,6 @@ inline const float* get_bias_data(const std::optional<at::Tensor>& bias, int64_t
}
return nullptr;
}
// FP8 and MXFP4 WoQ uses the same pattern:
// Btmp : [T, BLOCK_N * K]
// Ctmp : [T, BLOCK_M * BLOCK_N]