[CPU] Bump up CPU kernels to latest version (#50387)
Signed-off-by: jiang1.li <jiang1.li@intel.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
+124
-100
@@ -123,46 +123,6 @@ int moe_align_block_size(
|
||||
return num_tokens_post_pad;
|
||||
}
|
||||
|
||||
// silu : shape leading dimension
|
||||
// input0 [m_size, BLOCK_N] BLOCK_N
|
||||
// input1 [m_size, BLOCK_N] BLOCK_N
|
||||
// output [M * topk, N] N
|
||||
template <typename scalar_t, int BLOCK_N>
|
||||
inline void silu_and_mul(
|
||||
scalar_t* __restrict__ output,
|
||||
const float* __restrict__ input0, // x: x0, x1
|
||||
const float* __restrict__ input1, // y: y0, y1
|
||||
int64_t m_size,
|
||||
int64_t N) {
|
||||
using bVec = at::vec::Vectorized<scalar_t>;
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
|
||||
const fVec one = fVec(1.f);
|
||||
|
||||
// no remainder
|
||||
for (int64_t m = 0; m < m_size; ++m) {
|
||||
scalar_t* __restrict__ out = output + m * N;
|
||||
const float* __restrict__ x = input0 + m * BLOCK_N;
|
||||
const float* __restrict__ y = input1 + m * BLOCK_N;
|
||||
|
||||
for (int64_t d = 0; d < BLOCK_N; d += bVec::size()) {
|
||||
fVec x0 = fVec::loadu(x + d);
|
||||
fVec x1 = fVec::loadu(x + d + fVec::size());
|
||||
fVec y0 = fVec::loadu(y + d);
|
||||
fVec y1 = fVec::loadu(y + d + fVec::size());
|
||||
// silu
|
||||
x0 = x0 / (one + x0.neg().exp_u20());
|
||||
x1 = x1 / (one + x1.neg().exp_u20());
|
||||
// mul
|
||||
x0 = x0 * y0;
|
||||
x1 = x1 * y1;
|
||||
// convert
|
||||
bVec out_vec = convert_from_float_ext<scalar_t>(x0, x1);
|
||||
out_vec.store(out + d);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t, int BLOCK_M, int BLOCK_N>
|
||||
struct tinygemm_kernel_nn2 {
|
||||
static inline void apply(
|
||||
@@ -239,23 +199,17 @@ struct tinygemm_kernel_nn2<at::BFloat16, BLOCK_M, BLOCK_N> {
|
||||
Unroll<ROWS * COLS>{}(compute, k);
|
||||
}
|
||||
|
||||
using Vec = at::vec::Vectorized<float>;
|
||||
const Vec one = Vec(1.f);
|
||||
auto storec = [&](auto i) {
|
||||
constexpr int row = i / COLS;
|
||||
constexpr int col = i % COLS;
|
||||
// for COLS = 2, 4 use 512bit store
|
||||
if constexpr (col % 2 == 0) {
|
||||
Vec x0 = vc0[row * COLS + col + 0];
|
||||
Vec x1 = vc0[row * COLS + col + 1];
|
||||
Vec y0 = vc1[row * COLS + col + 0];
|
||||
Vec y1 = vc1[row * COLS + col + 1];
|
||||
// silu
|
||||
x0 = x0 / (one + x0.neg().exp_u20());
|
||||
x1 = x1 / (one + x1.neg().exp_u20());
|
||||
// mul
|
||||
x0 = x0 * y0;
|
||||
x1 = x1 * y1;
|
||||
__m512 x0 = vc0[row * COLS + col + 0];
|
||||
__m512 x1 = vc0[row * COLS + col + 1];
|
||||
__m512 y0 = vc1[row * COLS + col + 0];
|
||||
__m512 y1 = vc1[row * COLS + col + 1];
|
||||
x0 = _mm512_mul_ps(_mm512_rcp14_silu_ps(x0), y0);
|
||||
x1 = _mm512_mul_ps(_mm512_rcp14_silu_ps(x1), y1);
|
||||
|
||||
_mm512_storeu_si512(
|
||||
reinterpret_cast<__m512i*>((C + row * ldc + col * 16)),
|
||||
@@ -455,6 +409,8 @@ void fused_experts_kernel_impl(
|
||||
const scalar_t* __restrict__ input,
|
||||
const scalar_t* __restrict__ packed_w1,
|
||||
const scalar_t* __restrict__ packed_w2,
|
||||
const float* __restrict__ w1_bias,
|
||||
const float* __restrict__ w2_bias,
|
||||
const float* __restrict__ topk_weights,
|
||||
const int32_t* __restrict__ sorted_ids,
|
||||
const int32_t* __restrict__ expert_ids,
|
||||
@@ -464,7 +420,11 @@ void fused_experts_kernel_impl(
|
||||
int64_t K,
|
||||
int64_t E,
|
||||
int64_t topk,
|
||||
int64_t num_tokens_post_pad) {
|
||||
int64_t num_tokens_post_pad,
|
||||
float alpha,
|
||||
float limit,
|
||||
CPUActMethod act_func,
|
||||
bool with_bias) {
|
||||
// handle 2 tiles per block
|
||||
constexpr int64_t BLOCK_M = block_size_m();
|
||||
constexpr int64_t BLOCK_N = block_size_n();
|
||||
@@ -499,6 +459,8 @@ void fused_experts_kernel_impl(
|
||||
int32_t expert_id = expert_ids[mb];
|
||||
const scalar_t* __restrict__ B0 = packed_w1 + expert_id * stride_e + nb_upper * BLOCK_N * stride_n;
|
||||
const scalar_t* __restrict__ B1 = packed_w1 + expert_id * stride_e + nb_lower * BLOCK_N * stride_n;
|
||||
const float* __restrict__ B0_bias = w1_bias + expert_id * 2 * N + nb_upper * BLOCK_N;
|
||||
const float* __restrict__ B1_bias = w1_bias + expert_id * 2 * N + nb_lower * BLOCK_N;
|
||||
|
||||
int64_t m_size = offsets[mb + 1] - offsets[mb];
|
||||
|
||||
@@ -538,23 +500,62 @@ void fused_experts_kernel_impl(
|
||||
/* B */ B1,
|
||||
/* C */ C1);
|
||||
|
||||
// 1.d silu and mul
|
||||
const int64_t offset = offsets[mb];
|
||||
silu_and_mul<scalar_t, BLOCK_N>(ic1 + offset * N + nb * BLOCK_N, C0, C1, m_size, N);
|
||||
} else {
|
||||
// fused 1.bcd: silu_and_mul(A @ B0, A @ B1)
|
||||
const int64_t offset = offsets[mb];
|
||||
tinygemm_kernel(
|
||||
/* A */ A,
|
||||
/* B0 */ B0,
|
||||
/* B1 */ B1,
|
||||
/* C */ ic1 + offset * N + nb * BLOCK_N,
|
||||
/* M */ m_size,
|
||||
/* N */ n_size,
|
||||
/* K */ K,
|
||||
/* lda */ K,
|
||||
/* ldb */ n_size,
|
||||
/* ldc */ N);
|
||||
if (act_func == CPUActMethod::swiglu) {
|
||||
tinygemm_kernel(
|
||||
/* A */ A,
|
||||
/* B */ B0,
|
||||
/* C */ C0,
|
||||
/* M */ m_size,
|
||||
/* N */ n_size,
|
||||
/* K */ K,
|
||||
/* lda */ K,
|
||||
/* ldb */ n_size,
|
||||
/* ldc */ BLOCK_N);
|
||||
tinygemm_kernel(
|
||||
/* A */ A,
|
||||
/* B */ B1,
|
||||
/* C */ C1,
|
||||
/* M */ m_size,
|
||||
/* N */ n_size,
|
||||
/* K */ K,
|
||||
/* lda */ K,
|
||||
/* ldb */ n_size,
|
||||
/* ldc */ BLOCK_N);
|
||||
} else {
|
||||
// fused 1.bcd: silu_and_mul(A @ B0, A @ B1)
|
||||
tinygemm_kernel(
|
||||
/* A */ A,
|
||||
/* B0 */ B0,
|
||||
/* B1 */ B1,
|
||||
/* C */ ic1 + offset * N + nb * BLOCK_N,
|
||||
/* M */ m_size,
|
||||
/* N */ n_size,
|
||||
/* K */ K,
|
||||
/* lda */ K,
|
||||
/* ldb */ n_size,
|
||||
/* ldc */ N);
|
||||
}
|
||||
}
|
||||
if (with_bias) {
|
||||
for (int64_t m = 0; m < m_size; ++m) {
|
||||
add_bias_stub(C0 + m * BLOCK_N, B0_bias, n_size);
|
||||
add_bias_stub(C1 + m * BLOCK_N, B1_bias, n_size);
|
||||
}
|
||||
}
|
||||
// 1.d silu and mul
|
||||
const int64_t offset = offsets[mb];
|
||||
if (act_func == CPUActMethod::silu_and_mul && use_brgemm) {
|
||||
for (int64_t m = 0; m < m_size; ++m) {
|
||||
silu_and_mul_stub(ic1 + (offset + m) * N + nb * BLOCK_N, C0 + m * BLOCK_N, C1 + m * BLOCK_N, BLOCK_N);
|
||||
}
|
||||
} else if (act_func == CPUActMethod::swiglu) {
|
||||
for (int64_t m = 0; m < m_size; ++m) {
|
||||
scalar_t* __restrict__ ic1_row = ic1 + (offset + m) * N;
|
||||
clamp_sigmoid_and_mul_stub(ic1_row + nb * BLOCK_N / 2, C0 + m * BLOCK_N, BLOCK_N / 2, alpha, limit);
|
||||
clamp_sigmoid_and_mul_stub(ic1_row + N / 2 + nb * BLOCK_N / 2, C1 + m * BLOCK_N, BLOCK_N / 2, alpha, limit);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
@@ -591,6 +592,7 @@ void fused_experts_kernel_impl(
|
||||
// B shape [IC, n_size] in vnni format
|
||||
int32_t expert_id = expert_ids[mb];
|
||||
const scalar_t* __restrict__ B = packed_w2 + expert_id * stride_e2 + nb * BLOCK_N * stride_oc;
|
||||
const float* __restrict__ B_bias = w2_bias + expert_id * OC + nb * BLOCK_N;
|
||||
|
||||
// 2.a gemm: C = A @ B
|
||||
if (use_brgemm) {
|
||||
@@ -618,6 +620,11 @@ void fused_experts_kernel_impl(
|
||||
/* ldc */ BLOCK_N);
|
||||
}
|
||||
|
||||
if (with_bias) {
|
||||
for (int64_t m = 0; m < m_size; ++m) {
|
||||
add_bias_stub(C + m * BLOCK_N, B_bias, n_size);
|
||||
}
|
||||
}
|
||||
// 2.b copy from C to ic2 in original order
|
||||
// and also mul topk_weights in float32
|
||||
for (int64_t m = 0; m < m_size; ++m) {
|
||||
@@ -717,7 +724,9 @@ void shared_expert_kernel_impl(
|
||||
/* C */ C1);
|
||||
|
||||
// 1.d silu and mul
|
||||
silu_and_mul<scalar_t, BLOCK_N>(ic1 + mb * BLOCK_M * N + nb * BLOCK_N, C0, C1, m_size, N);
|
||||
for (int64_t m = 0; m < m_size; ++m) {
|
||||
silu_and_mul_stub(ic1 + (mb * BLOCK_M + m) * N + nb * BLOCK_N, C0 + m * BLOCK_N, C1 + m * BLOCK_N, BLOCK_N);
|
||||
}
|
||||
} else {
|
||||
// fused 1.bcd: silu_and_mul(A @ B0, A @ B1)
|
||||
tinygemm_kernel(
|
||||
@@ -809,24 +818,20 @@ void shared_expert_kernel_impl(
|
||||
} // anonymous namespace
|
||||
|
||||
// common checks
|
||||
template <CPUQuantMethod quant>
|
||||
static inline void check_moe_scales(
|
||||
bool use_int8_w8a8,
|
||||
bool use_fp8_w8a16,
|
||||
bool use_mxfp4,
|
||||
const std::optional<at::Tensor>& w1_scale,
|
||||
const std::optional<at::Tensor>& w2_scale,
|
||||
const std::optional<std::vector<int64_t>> block_size) {
|
||||
if (use_int8_w8a8) {
|
||||
if constexpr (quant == CPUQuantMethod::INT8_W8A8) {
|
||||
TORCH_CHECK(w1_scale.has_value(), "missing w1_scale for int8 w8a8.");
|
||||
TORCH_CHECK(w2_scale.has_value(), "missing w2_scale for int8 w8a8.");
|
||||
}
|
||||
if (use_fp8_w8a16) {
|
||||
} else if constexpr (quant == CPUQuantMethod::FP8_W8A16) {
|
||||
TORCH_CHECK(w1_scale.has_value(), "missing w1_scale for fp8 w8a16.");
|
||||
TORCH_CHECK(w2_scale.has_value(), "missing w2_scale for fp8 w8a16.");
|
||||
TORCH_CHECK(block_size.has_value(), "missing block_size for fp8 w8a16.");
|
||||
TORCH_CHECK(block_size.value().size() == 2, "expect block_size.size() to be 2.");
|
||||
}
|
||||
if (use_mxfp4) {
|
||||
} else if constexpr (quant == CPUQuantMethod::MXFP4) {
|
||||
TORCH_CHECK(w1_scale.has_value(), "missing w1_scale for mxfp4.");
|
||||
TORCH_CHECK(w2_scale.has_value(), "missing w2_scale for mxfp4.");
|
||||
TORCH_CHECK(w1_scale.value().scalar_type() == at::kByte, "expect w1_scale to be uint8.");
|
||||
@@ -834,6 +839,20 @@ static inline void check_moe_scales(
|
||||
}
|
||||
}
|
||||
|
||||
static inline void check_moe_scales(
|
||||
int64_t moe_comp_method,
|
||||
const std::optional<at::Tensor>& w1_scale,
|
||||
const std::optional<at::Tensor>& w2_scale,
|
||||
const std::optional<std::vector<int64_t>> block_size) {
|
||||
if (moe_comp_method == CPUQuantMethod::INT8_W8A8) {
|
||||
check_moe_scales<CPUQuantMethod::INT8_W8A8>(w1_scale, w2_scale, block_size);
|
||||
} else if (moe_comp_method == CPUQuantMethod::FP8_W8A16) {
|
||||
check_moe_scales<CPUQuantMethod::FP8_W8A16>(w1_scale, w2_scale, block_size);
|
||||
} else if (moe_comp_method == CPUQuantMethod::MXFP4) {
|
||||
check_moe_scales<CPUQuantMethod::MXFP4>(w1_scale, w2_scale, block_size);
|
||||
}
|
||||
}
|
||||
|
||||
#define CHECK_MOE_SCALES_FP8(DIM0, DIM1) \
|
||||
auto w1s = w1_scale.value(); \
|
||||
auto w2s = w2_scale.value(); \
|
||||
@@ -877,10 +896,14 @@ at::Tensor fused_experts_cpu(
|
||||
constexpr int64_t BLOCK_N = block_size_n();
|
||||
|
||||
const auto st = hidden_states.scalar_type();
|
||||
// TODO: fused_topk_torch_native (CPU fallback for models like MiniMax)
|
||||
// returns int64 topk_ids; fused_experts_cpu requires int32. Remove the typecast after topk kernel is provided
|
||||
auto topk_ids_ = topk_ids.scalar_type() == at::kInt ? topk_ids : topk_ids.to(at::kInt);
|
||||
|
||||
CHECK_INPUT(hidden_states);
|
||||
CHECK_INPUT(w1);
|
||||
CHECK_INPUT(w2);
|
||||
CHECK_EQ(topk_weights.sizes(), topk_ids.sizes());
|
||||
CHECK_EQ(topk_weights.sizes(), topk_ids_.sizes());
|
||||
CHECK_DIM(2, hidden_states);
|
||||
if (moe_comp_method == CPUQuantMethod::INT4_W4A8 && is_vnni) {
|
||||
CHECK_DIM(4, w1);
|
||||
@@ -890,9 +913,8 @@ at::Tensor fused_experts_cpu(
|
||||
CHECK_DIM(3, w2);
|
||||
}
|
||||
CHECK_DIM(2, topk_weights);
|
||||
CHECK_DIM(2, topk_ids);
|
||||
|
||||
CHECK_EQ(topk_ids.scalar_type(), at::kInt);
|
||||
CHECK_DIM(2, topk_ids_);
|
||||
CHECK_EQ(topk_ids_.scalar_type(), at::kInt);
|
||||
|
||||
// TODO: support topk_weights to be bf16 or fp16 in the kernel.
|
||||
// The topk_weights of llama4 is computed via Llama4MoE:custom_routing_function and is bf16/fp16
|
||||
@@ -908,12 +930,8 @@ at::Tensor fused_experts_cpu(
|
||||
int64_t topk = topk_weights_.size(1);
|
||||
|
||||
// we use int32_t compensation for int8 w8a8
|
||||
int64_t packed_K = moe_comp_method == CPUQuantMethod::MXFP4
|
||||
? get_row_size<uint8_t>(K)
|
||||
: get_row_size(K, moe_comp_method == CPUQuantMethod::INT8_W8A8);
|
||||
int64_t packed_N = moe_comp_method == CPUQuantMethod::MXFP4
|
||||
? get_row_size<uint8_t>(N)
|
||||
: get_row_size(N, moe_comp_method == CPUQuantMethod::INT8_W8A8);
|
||||
int64_t packed_K = get_row_size(static_cast<CPUQuantMethod>(moe_comp_method), K);
|
||||
int64_t packed_N = get_row_size(static_cast<CPUQuantMethod>(moe_comp_method), N);
|
||||
|
||||
// check weight shapes
|
||||
CHECK_EQ(w2.size(0), E);
|
||||
@@ -923,13 +941,7 @@ at::Tensor fused_experts_cpu(
|
||||
CHECK_EQ(packed_w2.size(2), packed_N / (moe_comp_method == CPUQuantMethod::INT4_W4A8 ? 2 : 1));
|
||||
}
|
||||
// check scales
|
||||
check_moe_scales(
|
||||
moe_comp_method == CPUQuantMethod::INT8_W8A8,
|
||||
moe_comp_method == CPUQuantMethod::FP8_W8A16,
|
||||
moe_comp_method == CPUQuantMethod::MXFP4,
|
||||
w1_scale,
|
||||
w2_scale,
|
||||
block_size);
|
||||
check_moe_scales(moe_comp_method, w1_scale, w2_scale, block_size);
|
||||
|
||||
at::Tensor out_hidden_states = inplace ? hidden_states : at::empty_like(hidden_states);
|
||||
|
||||
@@ -945,7 +957,7 @@ at::Tensor fused_experts_cpu(
|
||||
int64_t max_num_blocks = div_up(max_num_tokens_padded, BLOCK_M);
|
||||
auto buffer = at::empty(
|
||||
{max_num_tokens_padded + max_num_blocks + (num_threads + 1) * E + (E + 1) + (max_num_blocks + 1)},
|
||||
topk_ids.options());
|
||||
topk_ids_.options());
|
||||
|
||||
int32_t* __restrict__ sorted_ids = buffer.data_ptr<int32_t>();
|
||||
int32_t* __restrict__ expert_ids = sorted_ids + max_num_tokens_padded;
|
||||
@@ -969,7 +981,7 @@ at::Tensor fused_experts_cpu(
|
||||
|
||||
// align experts index
|
||||
int64_t num_tokens_post_pad = moe_align_block_size<BLOCK_M>(
|
||||
sorted_ids, expert_ids, topk_ids.data_ptr<int32_t>(), total_cnts, cumsums, offsets, E, numel, num_threads);
|
||||
sorted_ids, expert_ids, topk_ids_.data_ptr<int32_t>(), total_cnts, cumsums, offsets, E, numel, num_threads);
|
||||
|
||||
// unlike triton kernel, we fuse silu with gemm1 so only need 2 intermediate_caches:
|
||||
// 1. intermediate_cache1 : [M * topk, N]
|
||||
@@ -1048,7 +1060,7 @@ at::Tensor fused_experts_cpu(
|
||||
scalar_t* __restrict__ intermediate_cache0 = (scalar_t*)((void*)(C_tmp + num_threads * 2 * BLOCK_M * BLOCK_N));
|
||||
scalar_t* __restrict__ B_tmp = (scalar_t*)((void*)(intermediate_cache0 + M * topk * 2 * N));
|
||||
bool with_bias = w1_bias.has_value();
|
||||
auto act_func = alpha.has_value() && limit.has_value() ? CPUAcTMethod::swiglu : CPUAcTMethod::silu_and_mul;
|
||||
auto act_func = alpha.has_value() && limit.has_value() ? CPUActMethod::swiglu : CPUActMethod::silu_and_mul;
|
||||
|
||||
CHECK_MOE_SCALES_FP8(1, 2);
|
||||
fused_experts_fp_kernel_impl<scalar_t, at::Float8_e4m3fn, float, false>(
|
||||
@@ -1088,7 +1100,7 @@ at::Tensor fused_experts_cpu(
|
||||
scalar_t* __restrict__ intermediate_cache0 = (scalar_t*)((void*)(C_tmp + num_threads * 2 * BLOCK_M * BLOCK_N));
|
||||
scalar_t* __restrict__ B_tmp = (scalar_t*)((void*)(intermediate_cache0 + M * topk * 2 * N));
|
||||
bool with_bias = w1_bias.has_value();
|
||||
auto act_func = alpha.has_value() && limit.has_value() ? CPUAcTMethod::swiglu : CPUAcTMethod::silu_and_mul;
|
||||
auto act_func = alpha.has_value() && limit.has_value() ? CPUActMethod::swiglu : CPUActMethod::silu_and_mul;
|
||||
|
||||
// mxfp4 supports only group size of 32 (2^5)
|
||||
constexpr int64_t group_size = 32;
|
||||
@@ -1172,6 +1184,8 @@ at::Tensor fused_experts_cpu(
|
||||
} else {
|
||||
scalar_t* __restrict__ A_tmp = intermediate_cache2 + M * topk * K;
|
||||
float* __restrict__ C_tmp = (float*)((void*)(A_tmp + num_threads * BLOCK_M * K));
|
||||
bool with_bias = w1_bias.has_value();
|
||||
auto act_func = alpha.has_value() && limit.has_value() ? CPUActMethod::swiglu : CPUActMethod::silu_and_mul;
|
||||
|
||||
fused_experts_kernel_impl<scalar_t>(
|
||||
out_hidden_states.data_ptr<scalar_t>(),
|
||||
@@ -1182,6 +1196,8 @@ at::Tensor fused_experts_cpu(
|
||||
hidden_states.data_ptr<scalar_t>(),
|
||||
packed_w1.data_ptr<scalar_t>(),
|
||||
packed_w2.data_ptr<scalar_t>(),
|
||||
with_bias ? w1_bias.value().data_ptr<float>() : nullptr,
|
||||
with_bias ? w2_bias.value().data_ptr<float>() : nullptr,
|
||||
topk_weights_.data_ptr<float>(),
|
||||
sorted_ids,
|
||||
expert_ids,
|
||||
@@ -1191,7 +1207,11 @@ at::Tensor fused_experts_cpu(
|
||||
K,
|
||||
E,
|
||||
topk,
|
||||
num_tokens_post_pad);
|
||||
num_tokens_post_pad,
|
||||
alpha.has_value() ? float(alpha.value()) : 0,
|
||||
limit.has_value() ? float(limit.value()) : 0,
|
||||
act_func,
|
||||
with_bias);
|
||||
}
|
||||
});
|
||||
return out_hidden_states;
|
||||
@@ -1254,7 +1274,11 @@ at::Tensor shared_expert_cpu(
|
||||
CHECK_EQ(packed_w2.size(1), packed_N);
|
||||
|
||||
// check scales
|
||||
check_moe_scales(use_int8_w8a8, use_fp8_w8a16, false, w1_scale, w2_scale, block_size);
|
||||
if (use_int8_w8a8) {
|
||||
check_moe_scales<CPUQuantMethod::INT8_W8A8>(w1_scale, w2_scale, block_size);
|
||||
} else if (use_fp8_w8a16) {
|
||||
check_moe_scales<CPUQuantMethod::FP8_W8A16>(w1_scale, w2_scale, block_size);
|
||||
}
|
||||
|
||||
at::Tensor out_hidden_states = inplace ? hidden_states : at::empty_like(hidden_states);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user