[CPU] Bump up CPU kernels to latest version (#50387)

Signed-off-by: jiang1.li <jiang1.li@intel.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Li, Jiang
2026-07-30 19:18:43 +08:00
committed by GitHub
co-authored by Claude
parent aeeb36b1f1
commit 072a472782
15 changed files with 2181 additions and 1290 deletions
+13
View File
@@ -1,11 +1,16 @@
// Adapted from
// https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu
//
// Synced from
// https://github.com/sgl-project/sglang/tree/7c248dde7fe1f3b5100966f8143f97a9932c22a4/sgl-kernel/csrc/cpu
// Sync date: 2026-07-29
// clang-format off
#pragma once
#include <ATen/ATen.h>
#include <ATen/Dispatch.h>
#include <ATen/Parallel.h>
#if defined(_OPENMP)
@@ -49,6 +54,14 @@ namespace {
} \
}()
// Half + BFloat16, plus one extra scalar type
#define AT_DISPATCH_CASE_REDUCED_FLOATING_TYPES_AND(SCALARTYPE, ...) \
AT_DISPATCH_CASE_REDUCED_FLOATING_TYPES(__VA_ARGS__) \
AT_DISPATCH_CASE(SCALARTYPE, __VA_ARGS__)
#define AT_DISPATCH_REDUCED_FLOATING_TYPES_AND(SCALARTYPE, TYPE, NAME, ...) \
AT_DISPATCH_SWITCH(TYPE, NAME, AT_DISPATCH_CASE_REDUCED_FLOATING_TYPES_AND(SCALARTYPE, __VA_ARGS__))
// dispatch: bfloat16, float16, int8_t, fp8_e4m3, uint8_t(mxfp4/int4)
#define CPU_DISPATCH_PACKED_TYPES(TYPE, ...) \
[&] { \
+2 -3
View File
@@ -182,14 +182,13 @@ struct tinygemm_kernel<at::BFloat16, K, BLOCK_N, has_bias, has_silu> {
using fVec = at::vec::Vectorized<float>;
using bVec = at::vec::Vectorized<at::BFloat16>;
const fVec one = fVec(1.f);
auto storec = [&](auto i, int64_t m) {
constexpr int col = i;
fVec x0 = fVec(vc[col * 2 + 0]);
fVec x1 = fVec(vc[col * 2 + 1]);
if constexpr (has_silu) {
x0 = x0 / (one + x0.neg().exp_u20());
x1 = x1 / (one + x1.neg().exp_u20());
x0 = fast_silu(x0);
x1 = fast_silu(x1);
}
bVec out_vec = convert_from_float_ext<at::BFloat16>(x0, x1);
out_vec.store(C + m * lda + col * 32);
+1699 -1003
View File
File diff suppressed because it is too large Load Diff
+22 -28
View File
@@ -116,8 +116,7 @@ inline void copy_stub(scalar_t* __restrict__ out, const float* __restrict__ inpu
int64_t d;
#pragma GCC unroll 4
for (d = 0; d <= size - kVecSize; d += kVecSize) {
fVec data0 = fVec::loadu(input + d);
fVec data1 = fVec::loadu(input + d + fVec::size());
auto [data0, data1] = load_float_vec2(input + d);
bVec out_vec = convert_from_float_ext<scalar_t>(data0, data1);
out_vec.store(out + d);
}
@@ -135,9 +134,7 @@ inline void copy_stub(float* __restrict__ out, const scalar_t* __restrict__ inpu
int64_t d;
#pragma GCC unroll 4
for (d = 0; d <= size - kVecSize; d += kVecSize) {
fVec data0, data1;
bVec b_vec = bVec::loadu(input + d);
std::tie(data0, data1) = at::vec::convert_to_float(b_vec);
auto [data0, data1] = load_float_vec2(input + d);
data0.store(out + d);
data1.store(out + d + fVec::size());
}
@@ -156,9 +153,9 @@ inline void copy_add_stub(
int64_t d;
#pragma GCC unroll 4
for (d = 0; d <= size - kVecSize; d += kVecSize) {
fVec data0 = fVec::loadu(input + d) + fVec::loadu(bias + d);
fVec data1 = fVec::loadu(input + d + fVec::size()) + fVec::loadu(bias + d + fVec::size());
bVec out_vec = convert_from_float_ext<scalar_t>(data0, data1);
auto [data0, data1] = load_float_vec2(input + d);
auto [bias0, bias1] = load_float_vec2(bias + d);
bVec out_vec = convert_from_float_ext<scalar_t>(data0 + bias0, data1 + bias1);
out_vec.store(out + d);
}
for (; d < size; ++d) {
@@ -176,7 +173,6 @@ inline void scalar_sigmoid_and_mul(
using bVec = at::vec::Vectorized<scalar_t>;
using fVec = at::vec::Vectorized<float>;
// scalar sigmoid
const fVec one = fVec(1.f);
fVec X;
if constexpr (has_bias) {
assert(bias != nullptr);
@@ -184,18 +180,13 @@ inline void scalar_sigmoid_and_mul(
} else {
X = fVec(input[0]);
}
X = one / (one + X.neg().exp_u20());
X = fast_sigmoid(X);
// vec mul
constexpr int kVecSize = bVec::size();
for (int d = 0; d < SIZE; d += kVecSize) {
bVec m_bvec = bVec::loadu(mul + d);
fVec m_fvec0, m_fvec1;
std::tie(m_fvec0, m_fvec1) = at::vec::convert_to_float(m_bvec);
m_fvec0 = m_fvec0 * X;
m_fvec1 = m_fvec1 * X;
bVec out_vec = convert_from_float_ext<scalar_t>(m_fvec0, m_fvec1);
auto [m_fvec0, m_fvec1] = load_float_vec2(mul + d);
bVec out_vec = convert_from_float_ext<scalar_t>(m_fvec0 * X, m_fvec1 * X);
out_vec.store(out + d);
}
}
@@ -727,10 +718,10 @@ at::Tensor convert_scale_packed(at::Tensor& scale) {
return packed_scale;
}
// mat1 : [M, K]
// mat1 : [*, K]
// mat2 : [N, K] ([K, N] if use_fma_gemm)
// bias : [N]
// out : [M, N]
// out : [*, N]
//
at::Tensor
weight_packed_linear(at::Tensor& mat1, at::Tensor& mat2, const std::optional<at::Tensor>& bias, bool is_vnni) {
@@ -740,23 +731,25 @@ weight_packed_linear(at::Tensor& mat1, at::Tensor& mat2, const std::optional<at:
use_fma_gemm = true;
}
int64_t M = mat1.size(0);
int64_t K = mat1.size(1);
int64_t N = use_fma_gemm ? mat2.size(1) : mat2.size(0);
CHECK_LAST_DIM_CONTIGUOUS_INPUT(mat1);
CHECK_INPUT(mat2);
CHECK_DIM(2, mat1);
const int64_t ndim = mat1.ndimension();
auto input_sizes = mat1.sizes().vec();
int64_t N = use_fma_gemm ? mat2.size(1) : mat2.size(0);
int64_t K = use_fma_gemm ? mat1.size(1) : mat2.size(1);
int64_t M = use_fma_gemm ? mat1.size(0) : mat1.numel() / K;
CHECK_DIM(2, mat2);
if (!use_fma_gemm) {
CHECK_EQ(mat1.size(1), K);
if (use_fma_gemm) {
CHECK_DIM(2, mat1);
} else {
CHECK_EQ(mat1.size(ndim - 1), K);
}
auto dispatch_type = mat1.scalar_type();
auto out = at::empty({M, N}, mat1.options());
// strides
int64_t out_strideM = out.stride(0);
int64_t mat1_strideM = mat1.stride(0);
int64_t mat1_strideM = mat1.stride(-2);
const bool has_bias = bias.has_value();
const float* bias_data = nullptr;
@@ -792,7 +785,8 @@ weight_packed_linear(at::Tensor& mat1, at::Tensor& mat2, const std::optional<at:
}
});
return out;
input_sizes[ndim - 1] = N;
return out.view(input_sizes);
}
// mat1 : [M, K]
+17 -10
View File
@@ -83,15 +83,11 @@ inline int64_t get_row_size(int64_t K, bool use_int8_w8a8) {
return use_int8_w8a8 ? K + sizeof(int32_t) : K;
}
enum class CPUAcTMethod : int { silu_and_mul = 0, swiglu = 1 };
constexpr bool operator==(CPUAcTMethod a, int b) {
return static_cast<int>(a) == b;
}
constexpr bool operator==(int a, CPUAcTMethod b) {
return a == static_cast<int>(b);
}
enum class CPUActMethod : int {
silu_and_mul = 0,
swiglu = 1,
gelu_and_mul = 2,
};
enum class CPUQuantMethod : int64_t { BF16 = 0, INT8_W8A8 = 1, FP8_W8A16 = 2, INT4_W4A8 = 3, MXFP4 = 4 };
@@ -113,6 +109,17 @@ constexpr bool operator==(int64_t a, CPUQuantAlgo b) {
return a == static_cast<int64_t>(b);
}
inline int64_t get_row_size(CPUQuantMethod quant, int64_t K) {
switch (quant) {
case CPUQuantMethod::INT8_W8A8:
return K + sizeof(int32_t);
case CPUQuantMethod::MXFP4:
return K >> 1;
default:
return K;
}
}
inline int64_t get_4bit_block_k_size(int64_t group_size) {
return group_size > 128 ? 128 : group_size;
}
@@ -184,7 +191,7 @@ void fused_experts_fp_kernel_impl(
int64_t num_tokens_post_pad,
float alpha,
float limit,
CPUAcTMethod act_func,
CPUActMethod act_func,
bool with_bias);
// shared expert implementation for int8 w8a8
+6 -8
View File
@@ -18,8 +18,7 @@ inline void copy_stub(scalar_t* __restrict__ out, const float* __restrict__ inpu
int64_t d;
#pragma GCC unroll 4
for (d = 0; d <= size - kVecSize; d += kVecSize) {
fVec data0 = fVec::loadu(input + d);
fVec data1 = fVec::loadu(input + d + fVec::size());
auto [data0, data1] = load_float_vec2(input + d);
bVec out_vec = convert_from_float_ext<scalar_t>(data0, data1);
out_vec.store(out + d);
}
@@ -38,9 +37,9 @@ inline void copy_add_stub(
int64_t d;
#pragma GCC unroll 4
for (d = 0; d <= size - kVecSize; d += kVecSize) {
fVec data0 = fVec::loadu(input + d) + fVec::loadu(bias + d);
fVec data1 = fVec::loadu(input + d + fVec::size()) + fVec::loadu(bias + d + fVec::size());
bVec out_vec = convert_from_float_ext<scalar_t>(data0, data1);
auto [data0, data1] = load_float_vec2(input + d);
auto [bias0, bias1] = load_float_vec2(bias + d);
bVec out_vec = convert_from_float_ext<scalar_t>(data0 + bias0, data1 + bias1);
out_vec.store(out + d);
}
for (; d < size; ++d) {
@@ -57,9 +56,8 @@ inline void copy_mul_stub(scalar_t* __restrict__ out, const float* __restrict__
int d;
#pragma GCC unroll 4
for (d = 0; d <= size - kVecSize; d += kVecSize) {
fVec data0 = fVec::loadu(input + d) * vscale;
fVec data1 = fVec::loadu(input + d + fVec::size()) * vscale;
bVec out_vec = convert_from_float_ext<scalar_t>(data0, data1);
auto [data0, data1] = load_float_vec2(input + d);
bVec out_vec = convert_from_float_ext<scalar_t>(data0 * vscale, data1 * vscale);
out_vec.store(out + d);
}
for (; d < size; ++d) {
+124 -100
View File
@@ -123,46 +123,6 @@ int moe_align_block_size(
return num_tokens_post_pad;
}
// silu : shape leading dimension
// input0 [m_size, BLOCK_N] BLOCK_N
// input1 [m_size, BLOCK_N] BLOCK_N
// output [M * topk, N] N
template <typename scalar_t, int BLOCK_N>
inline void silu_and_mul(
scalar_t* __restrict__ output,
const float* __restrict__ input0, // x: x0, x1
const float* __restrict__ input1, // y: y0, y1
int64_t m_size,
int64_t N) {
using bVec = at::vec::Vectorized<scalar_t>;
using fVec = at::vec::Vectorized<float>;
const fVec one = fVec(1.f);
// no remainder
for (int64_t m = 0; m < m_size; ++m) {
scalar_t* __restrict__ out = output + m * N;
const float* __restrict__ x = input0 + m * BLOCK_N;
const float* __restrict__ y = input1 + m * BLOCK_N;
for (int64_t d = 0; d < BLOCK_N; d += bVec::size()) {
fVec x0 = fVec::loadu(x + d);
fVec x1 = fVec::loadu(x + d + fVec::size());
fVec y0 = fVec::loadu(y + d);
fVec y1 = fVec::loadu(y + d + fVec::size());
// silu
x0 = x0 / (one + x0.neg().exp_u20());
x1 = x1 / (one + x1.neg().exp_u20());
// mul
x0 = x0 * y0;
x1 = x1 * y1;
// convert
bVec out_vec = convert_from_float_ext<scalar_t>(x0, x1);
out_vec.store(out + d);
}
}
}
template <typename scalar_t, int BLOCK_M, int BLOCK_N>
struct tinygemm_kernel_nn2 {
static inline void apply(
@@ -239,23 +199,17 @@ struct tinygemm_kernel_nn2<at::BFloat16, BLOCK_M, BLOCK_N> {
Unroll<ROWS * COLS>{}(compute, k);
}
using Vec = at::vec::Vectorized<float>;
const Vec one = Vec(1.f);
auto storec = [&](auto i) {
constexpr int row = i / COLS;
constexpr int col = i % COLS;
// for COLS = 2, 4 use 512bit store
if constexpr (col % 2 == 0) {
Vec x0 = vc0[row * COLS + col + 0];
Vec x1 = vc0[row * COLS + col + 1];
Vec y0 = vc1[row * COLS + col + 0];
Vec y1 = vc1[row * COLS + col + 1];
// silu
x0 = x0 / (one + x0.neg().exp_u20());
x1 = x1 / (one + x1.neg().exp_u20());
// mul
x0 = x0 * y0;
x1 = x1 * y1;
__m512 x0 = vc0[row * COLS + col + 0];
__m512 x1 = vc0[row * COLS + col + 1];
__m512 y0 = vc1[row * COLS + col + 0];
__m512 y1 = vc1[row * COLS + col + 1];
x0 = _mm512_mul_ps(_mm512_rcp14_silu_ps(x0), y0);
x1 = _mm512_mul_ps(_mm512_rcp14_silu_ps(x1), y1);
_mm512_storeu_si512(
reinterpret_cast<__m512i*>((C + row * ldc + col * 16)),
@@ -455,6 +409,8 @@ void fused_experts_kernel_impl(
const scalar_t* __restrict__ input,
const scalar_t* __restrict__ packed_w1,
const scalar_t* __restrict__ packed_w2,
const float* __restrict__ w1_bias,
const float* __restrict__ w2_bias,
const float* __restrict__ topk_weights,
const int32_t* __restrict__ sorted_ids,
const int32_t* __restrict__ expert_ids,
@@ -464,7 +420,11 @@ void fused_experts_kernel_impl(
int64_t K,
int64_t E,
int64_t topk,
int64_t num_tokens_post_pad) {
int64_t num_tokens_post_pad,
float alpha,
float limit,
CPUActMethod act_func,
bool with_bias) {
// handle 2 tiles per block
constexpr int64_t BLOCK_M = block_size_m();
constexpr int64_t BLOCK_N = block_size_n();
@@ -499,6 +459,8 @@ void fused_experts_kernel_impl(
int32_t expert_id = expert_ids[mb];
const scalar_t* __restrict__ B0 = packed_w1 + expert_id * stride_e + nb_upper * BLOCK_N * stride_n;
const scalar_t* __restrict__ B1 = packed_w1 + expert_id * stride_e + nb_lower * BLOCK_N * stride_n;
const float* __restrict__ B0_bias = w1_bias + expert_id * 2 * N + nb_upper * BLOCK_N;
const float* __restrict__ B1_bias = w1_bias + expert_id * 2 * N + nb_lower * BLOCK_N;
int64_t m_size = offsets[mb + 1] - offsets[mb];
@@ -538,23 +500,62 @@ void fused_experts_kernel_impl(
/* B */ B1,
/* C */ C1);
// 1.d silu and mul
const int64_t offset = offsets[mb];
silu_and_mul<scalar_t, BLOCK_N>(ic1 + offset * N + nb * BLOCK_N, C0, C1, m_size, N);
} else {
// fused 1.bcd: silu_and_mul(A @ B0, A @ B1)
const int64_t offset = offsets[mb];
tinygemm_kernel(
/* A */ A,
/* B0 */ B0,
/* B1 */ B1,
/* C */ ic1 + offset * N + nb * BLOCK_N,
/* M */ m_size,
/* N */ n_size,
/* K */ K,
/* lda */ K,
/* ldb */ n_size,
/* ldc */ N);
if (act_func == CPUActMethod::swiglu) {
tinygemm_kernel(
/* A */ A,
/* B */ B0,
/* C */ C0,
/* M */ m_size,
/* N */ n_size,
/* K */ K,
/* lda */ K,
/* ldb */ n_size,
/* ldc */ BLOCK_N);
tinygemm_kernel(
/* A */ A,
/* B */ B1,
/* C */ C1,
/* M */ m_size,
/* N */ n_size,
/* K */ K,
/* lda */ K,
/* ldb */ n_size,
/* ldc */ BLOCK_N);
} else {
// fused 1.bcd: silu_and_mul(A @ B0, A @ B1)
tinygemm_kernel(
/* A */ A,
/* B0 */ B0,
/* B1 */ B1,
/* C */ ic1 + offset * N + nb * BLOCK_N,
/* M */ m_size,
/* N */ n_size,
/* K */ K,
/* lda */ K,
/* ldb */ n_size,
/* ldc */ N);
}
}
if (with_bias) {
for (int64_t m = 0; m < m_size; ++m) {
add_bias_stub(C0 + m * BLOCK_N, B0_bias, n_size);
add_bias_stub(C1 + m * BLOCK_N, B1_bias, n_size);
}
}
// 1.d silu and mul
const int64_t offset = offsets[mb];
if (act_func == CPUActMethod::silu_and_mul && use_brgemm) {
for (int64_t m = 0; m < m_size; ++m) {
silu_and_mul_stub(ic1 + (offset + m) * N + nb * BLOCK_N, C0 + m * BLOCK_N, C1 + m * BLOCK_N, BLOCK_N);
}
} else if (act_func == CPUActMethod::swiglu) {
for (int64_t m = 0; m < m_size; ++m) {
scalar_t* __restrict__ ic1_row = ic1 + (offset + m) * N;
clamp_sigmoid_and_mul_stub(ic1_row + nb * BLOCK_N / 2, C0 + m * BLOCK_N, BLOCK_N / 2, alpha, limit);
clamp_sigmoid_and_mul_stub(ic1_row + N / 2 + nb * BLOCK_N / 2, C1 + m * BLOCK_N, BLOCK_N / 2, alpha, limit);
}
}
});
@@ -591,6 +592,7 @@ void fused_experts_kernel_impl(
// B shape [IC, n_size] in vnni format
int32_t expert_id = expert_ids[mb];
const scalar_t* __restrict__ B = packed_w2 + expert_id * stride_e2 + nb * BLOCK_N * stride_oc;
const float* __restrict__ B_bias = w2_bias + expert_id * OC + nb * BLOCK_N;
// 2.a gemm: C = A @ B
if (use_brgemm) {
@@ -618,6 +620,11 @@ void fused_experts_kernel_impl(
/* ldc */ BLOCK_N);
}
if (with_bias) {
for (int64_t m = 0; m < m_size; ++m) {
add_bias_stub(C + m * BLOCK_N, B_bias, n_size);
}
}
// 2.b copy from C to ic2 in original order
// and also mul topk_weights in float32
for (int64_t m = 0; m < m_size; ++m) {
@@ -717,7 +724,9 @@ void shared_expert_kernel_impl(
/* C */ C1);
// 1.d silu and mul
silu_and_mul<scalar_t, BLOCK_N>(ic1 + mb * BLOCK_M * N + nb * BLOCK_N, C0, C1, m_size, N);
for (int64_t m = 0; m < m_size; ++m) {
silu_and_mul_stub(ic1 + (mb * BLOCK_M + m) * N + nb * BLOCK_N, C0 + m * BLOCK_N, C1 + m * BLOCK_N, BLOCK_N);
}
} else {
// fused 1.bcd: silu_and_mul(A @ B0, A @ B1)
tinygemm_kernel(
@@ -809,24 +818,20 @@ void shared_expert_kernel_impl(
} // anonymous namespace
// common checks
template <CPUQuantMethod quant>
static inline void check_moe_scales(
bool use_int8_w8a8,
bool use_fp8_w8a16,
bool use_mxfp4,
const std::optional<at::Tensor>& w1_scale,
const std::optional<at::Tensor>& w2_scale,
const std::optional<std::vector<int64_t>> block_size) {
if (use_int8_w8a8) {
if constexpr (quant == CPUQuantMethod::INT8_W8A8) {
TORCH_CHECK(w1_scale.has_value(), "missing w1_scale for int8 w8a8.");
TORCH_CHECK(w2_scale.has_value(), "missing w2_scale for int8 w8a8.");
}
if (use_fp8_w8a16) {
} else if constexpr (quant == CPUQuantMethod::FP8_W8A16) {
TORCH_CHECK(w1_scale.has_value(), "missing w1_scale for fp8 w8a16.");
TORCH_CHECK(w2_scale.has_value(), "missing w2_scale for fp8 w8a16.");
TORCH_CHECK(block_size.has_value(), "missing block_size for fp8 w8a16.");
TORCH_CHECK(block_size.value().size() == 2, "expect block_size.size() to be 2.");
}
if (use_mxfp4) {
} else if constexpr (quant == CPUQuantMethod::MXFP4) {
TORCH_CHECK(w1_scale.has_value(), "missing w1_scale for mxfp4.");
TORCH_CHECK(w2_scale.has_value(), "missing w2_scale for mxfp4.");
TORCH_CHECK(w1_scale.value().scalar_type() == at::kByte, "expect w1_scale to be uint8.");
@@ -834,6 +839,20 @@ static inline void check_moe_scales(
}
}
static inline void check_moe_scales(
int64_t moe_comp_method,
const std::optional<at::Tensor>& w1_scale,
const std::optional<at::Tensor>& w2_scale,
const std::optional<std::vector<int64_t>> block_size) {
if (moe_comp_method == CPUQuantMethod::INT8_W8A8) {
check_moe_scales<CPUQuantMethod::INT8_W8A8>(w1_scale, w2_scale, block_size);
} else if (moe_comp_method == CPUQuantMethod::FP8_W8A16) {
check_moe_scales<CPUQuantMethod::FP8_W8A16>(w1_scale, w2_scale, block_size);
} else if (moe_comp_method == CPUQuantMethod::MXFP4) {
check_moe_scales<CPUQuantMethod::MXFP4>(w1_scale, w2_scale, block_size);
}
}
#define CHECK_MOE_SCALES_FP8(DIM0, DIM1) \
auto w1s = w1_scale.value(); \
auto w2s = w2_scale.value(); \
@@ -877,10 +896,14 @@ at::Tensor fused_experts_cpu(
constexpr int64_t BLOCK_N = block_size_n();
const auto st = hidden_states.scalar_type();
// TODO: fused_topk_torch_native (CPU fallback for models like MiniMax)
// returns int64 topk_ids; fused_experts_cpu requires int32. Remove the typecast after topk kernel is provided
auto topk_ids_ = topk_ids.scalar_type() == at::kInt ? topk_ids : topk_ids.to(at::kInt);
CHECK_INPUT(hidden_states);
CHECK_INPUT(w1);
CHECK_INPUT(w2);
CHECK_EQ(topk_weights.sizes(), topk_ids.sizes());
CHECK_EQ(topk_weights.sizes(), topk_ids_.sizes());
CHECK_DIM(2, hidden_states);
if (moe_comp_method == CPUQuantMethod::INT4_W4A8 && is_vnni) {
CHECK_DIM(4, w1);
@@ -890,9 +913,8 @@ at::Tensor fused_experts_cpu(
CHECK_DIM(3, w2);
}
CHECK_DIM(2, topk_weights);
CHECK_DIM(2, topk_ids);
CHECK_EQ(topk_ids.scalar_type(), at::kInt);
CHECK_DIM(2, topk_ids_);
CHECK_EQ(topk_ids_.scalar_type(), at::kInt);
// TODO: support topk_weights to be bf16 or fp16 in the kernel.
// The topk_weights of llama4 is computed via Llama4MoE:custom_routing_function and is bf16/fp16
@@ -908,12 +930,8 @@ at::Tensor fused_experts_cpu(
int64_t topk = topk_weights_.size(1);
// we use int32_t compensation for int8 w8a8
int64_t packed_K = moe_comp_method == CPUQuantMethod::MXFP4
? get_row_size<uint8_t>(K)
: get_row_size(K, moe_comp_method == CPUQuantMethod::INT8_W8A8);
int64_t packed_N = moe_comp_method == CPUQuantMethod::MXFP4
? get_row_size<uint8_t>(N)
: get_row_size(N, moe_comp_method == CPUQuantMethod::INT8_W8A8);
int64_t packed_K = get_row_size(static_cast<CPUQuantMethod>(moe_comp_method), K);
int64_t packed_N = get_row_size(static_cast<CPUQuantMethod>(moe_comp_method), N);
// check weight shapes
CHECK_EQ(w2.size(0), E);
@@ -923,13 +941,7 @@ at::Tensor fused_experts_cpu(
CHECK_EQ(packed_w2.size(2), packed_N / (moe_comp_method == CPUQuantMethod::INT4_W4A8 ? 2 : 1));
}
// check scales
check_moe_scales(
moe_comp_method == CPUQuantMethod::INT8_W8A8,
moe_comp_method == CPUQuantMethod::FP8_W8A16,
moe_comp_method == CPUQuantMethod::MXFP4,
w1_scale,
w2_scale,
block_size);
check_moe_scales(moe_comp_method, w1_scale, w2_scale, block_size);
at::Tensor out_hidden_states = inplace ? hidden_states : at::empty_like(hidden_states);
@@ -945,7 +957,7 @@ at::Tensor fused_experts_cpu(
int64_t max_num_blocks = div_up(max_num_tokens_padded, BLOCK_M);
auto buffer = at::empty(
{max_num_tokens_padded + max_num_blocks + (num_threads + 1) * E + (E + 1) + (max_num_blocks + 1)},
topk_ids.options());
topk_ids_.options());
int32_t* __restrict__ sorted_ids = buffer.data_ptr<int32_t>();
int32_t* __restrict__ expert_ids = sorted_ids + max_num_tokens_padded;
@@ -969,7 +981,7 @@ at::Tensor fused_experts_cpu(
// align experts index
int64_t num_tokens_post_pad = moe_align_block_size<BLOCK_M>(
sorted_ids, expert_ids, topk_ids.data_ptr<int32_t>(), total_cnts, cumsums, offsets, E, numel, num_threads);
sorted_ids, expert_ids, topk_ids_.data_ptr<int32_t>(), total_cnts, cumsums, offsets, E, numel, num_threads);
// unlike triton kernel, we fuse silu with gemm1 so only need 2 intermediate_caches:
// 1. intermediate_cache1 : [M * topk, N]
@@ -1048,7 +1060,7 @@ at::Tensor fused_experts_cpu(
scalar_t* __restrict__ intermediate_cache0 = (scalar_t*)((void*)(C_tmp + num_threads * 2 * BLOCK_M * BLOCK_N));
scalar_t* __restrict__ B_tmp = (scalar_t*)((void*)(intermediate_cache0 + M * topk * 2 * N));
bool with_bias = w1_bias.has_value();
auto act_func = alpha.has_value() && limit.has_value() ? CPUAcTMethod::swiglu : CPUAcTMethod::silu_and_mul;
auto act_func = alpha.has_value() && limit.has_value() ? CPUActMethod::swiglu : CPUActMethod::silu_and_mul;
CHECK_MOE_SCALES_FP8(1, 2);
fused_experts_fp_kernel_impl<scalar_t, at::Float8_e4m3fn, float, false>(
@@ -1088,7 +1100,7 @@ at::Tensor fused_experts_cpu(
scalar_t* __restrict__ intermediate_cache0 = (scalar_t*)((void*)(C_tmp + num_threads * 2 * BLOCK_M * BLOCK_N));
scalar_t* __restrict__ B_tmp = (scalar_t*)((void*)(intermediate_cache0 + M * topk * 2 * N));
bool with_bias = w1_bias.has_value();
auto act_func = alpha.has_value() && limit.has_value() ? CPUAcTMethod::swiglu : CPUAcTMethod::silu_and_mul;
auto act_func = alpha.has_value() && limit.has_value() ? CPUActMethod::swiglu : CPUActMethod::silu_and_mul;
// mxfp4 supports only group size of 32 (2^5)
constexpr int64_t group_size = 32;
@@ -1172,6 +1184,8 @@ at::Tensor fused_experts_cpu(
} else {
scalar_t* __restrict__ A_tmp = intermediate_cache2 + M * topk * K;
float* __restrict__ C_tmp = (float*)((void*)(A_tmp + num_threads * BLOCK_M * K));
bool with_bias = w1_bias.has_value();
auto act_func = alpha.has_value() && limit.has_value() ? CPUActMethod::swiglu : CPUActMethod::silu_and_mul;
fused_experts_kernel_impl<scalar_t>(
out_hidden_states.data_ptr<scalar_t>(),
@@ -1182,6 +1196,8 @@ at::Tensor fused_experts_cpu(
hidden_states.data_ptr<scalar_t>(),
packed_w1.data_ptr<scalar_t>(),
packed_w2.data_ptr<scalar_t>(),
with_bias ? w1_bias.value().data_ptr<float>() : nullptr,
with_bias ? w2_bias.value().data_ptr<float>() : nullptr,
topk_weights_.data_ptr<float>(),
sorted_ids,
expert_ids,
@@ -1191,7 +1207,11 @@ at::Tensor fused_experts_cpu(
K,
E,
topk,
num_tokens_post_pad);
num_tokens_post_pad,
alpha.has_value() ? float(alpha.value()) : 0,
limit.has_value() ? float(limit.value()) : 0,
act_func,
with_bias);
}
});
return out_hidden_states;
@@ -1254,7 +1274,11 @@ at::Tensor shared_expert_cpu(
CHECK_EQ(packed_w2.size(1), packed_N);
// check scales
check_moe_scales(use_int8_w8a8, use_fp8_w8a16, false, w1_scale, w2_scale, block_size);
if (use_int8_w8a8) {
check_moe_scales<CPUQuantMethod::INT8_W8A8>(w1_scale, w2_scale, block_size);
} else if (use_fp8_w8a16) {
check_moe_scales<CPUQuantMethod::FP8_W8A16>(w1_scale, w2_scale, block_size);
}
at::Tensor out_hidden_states = inplace ? hidden_states : at::empty_like(hidden_states);
+41 -82
View File
@@ -64,9 +64,7 @@ inline void copy_mul_stub(scalar_t* __restrict__ out, const input_t* __restrict_
#pragma GCC unroll 4
for (d = 0; d <= size - kVecSize; d += kVecSize) {
auto [x0, x1] = load_float_vec2(input + d);
x0 = x0 * weight_vec;
x1 = x1 * weight_vec;
bVec out_vec = convert_from_float_ext<scalar_t>(x0, x1);
bVec out_vec = convert_from_float_ext<scalar_t>(x0 * weight_vec, x1 * weight_vec);
out_vec.store(out + d);
}
for (; d < size; ++d) {
@@ -91,10 +89,7 @@ inline void sum_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ in
fVec sum_fvec0 = fVec(0.f);
fVec sum_fvec1 = fVec(0.f);
for (int t = 0; t < topk; ++t) {
bVec x_bvec = bVec::loadu(input + t * K + d);
fVec x_fvec0, x_fvec1;
std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec);
auto [x_fvec0, x_fvec1] = load_float_vec2(input + t * K + d);
sum_fvec0 += x_fvec0;
sum_fvec1 += x_fvec1;
}
@@ -137,11 +132,7 @@ inline void add_mul_stub(
#pragma GCC unroll 4
for (d = 0; d <= size - kVecSize; d += kVecSize) {
auto [x0, x1] = load_float_vec2(input + d);
bVec y_bvec = bVec::loadu(input2 + d);
fVec y0, y1;
std::tie(y0, y1) = at::vec::convert_to_float(y_bvec);
auto [y0, y1] = load_float_vec2(input2 + d);
x0 = x0 + y0 * s_vec;
x1 = x1 + y1 * s_vec;
bVec out_vec = convert_from_float_ext<scalar_t>(x0, x1);
@@ -152,31 +143,52 @@ inline void add_mul_stub(
}
}
template <typename scalar_t>
template <typename scalar_t, typename input_t>
inline void silu_and_mul_stub(
scalar_t* __restrict__ out, const scalar_t* __restrict__ input, const scalar_t* __restrict__ input2, int64_t size) {
scalar_t* __restrict__ out, const input_t* __restrict__ input, const input_t* __restrict__ input2, int64_t size) {
static_assert(
std::is_same_v<input_t, float> || std::is_same_v<input_t, scalar_t>,
"silu_and_mul_stub only supports input_t == float or input_t == scalar_t");
using bVec = at::vec::Vectorized<scalar_t>;
using fVec = at::vec::Vectorized<float>;
const fVec one = fVec(1.f);
// no remainder
#pragma GCC unroll 4
for (int64_t d = 0; d < size; d += bVec::size()) {
bVec x = bVec::loadu(input + d);
fVec x0, x1;
std::tie(x0, x1) = at::vec::convert_to_float(x);
bVec y = bVec::loadu(input2 + d);
fVec y0, y1;
std::tie(y0, y1) = at::vec::convert_to_float(y);
x0 = x0 / (one + x0.neg().exp_u20());
x1 = x1 / (one + x1.neg().exp_u20());
x0 = x0 * y0;
x1 = x1 * y1;
auto [x0, x1] = load_float_vec2(input + d);
auto [y0, y1] = load_float_vec2(input2 + d);
x0 = fast_silu(x0) * y0;
x1 = fast_silu(x1) * y1;
bVec out_vec = convert_from_float_ext<scalar_t>(x0, x1);
out_vec.store(out + d);
}
}
template <typename scalar_t, typename input_t>
inline void clamp_sigmoid_and_mul_stub(
scalar_t* __restrict__ out, const input_t* __restrict__ input, int64_t size, const float alpha, const float limit) {
static_assert(
std::is_same_v<input_t, float> || std::is_same_v<input_t, scalar_t>,
"clamp_sigmoid_and_mul_stub only supports input_t == float or input_t == scalar_t");
using bVec = at::vec::Vectorized<scalar_t>;
using fVec = at::vec::Vectorized<float>;
const fVec one = fVec(1.f);
const fVec limit_v = fVec(limit);
const fVec nlimit_v = fVec(-limit);
const fVec alpha_v = fVec(alpha);
#pragma GCC unroll 4
for (int64_t d = 0; d < 2 * size; d += bVec::size()) {
auto [x0_, y0_] = load_float_vec2(input + d);
auto [x0, y0] = at::vec::deinterleave2<float>(x0_, y0_);
x0 = at::vec::minimum(x0, limit_v);
y0 = at::vec::minimum(limit_v, at::vec::maximum(nlimit_v, y0));
x0 = fast_sigmoid_glu(x0, alpha_v) * (y0 + one);
store_from_float_ext(out + d / 2, x0);
}
}
template <typename scalar_t>
inline void copy_mul_stub(scalar_t* __restrict__ out, const float* __restrict__ input, float weight, int64_t size) {
using bVec = at::vec::Vectorized<scalar_t>;
@@ -186,9 +198,8 @@ inline void copy_mul_stub(scalar_t* __restrict__ out, const float* __restrict__
int64_t d;
#pragma GCC unroll 4
for (d = 0; d <= size - kVecSize; d += kVecSize) {
fVec data0 = fVec::loadu(input + d) * weight_vec;
fVec data1 = fVec::loadu(input + d + fVec::size()) * weight_vec;
bVec out_vec = convert_from_float_ext<scalar_t>(data0, data1);
auto [x0, x1] = load_float_vec2(input + d);
bVec out_vec = convert_from_float_ext<scalar_t>(x0 * weight_vec, x1 * weight_vec);
out_vec.store(out + d);
}
for (; d < size; ++d) {
@@ -222,63 +233,11 @@ inline void copy_mul_stub(scalar_t* __restrict__ out, const scalar_t* __restrict
int64_t d;
#pragma GCC unroll 4
for (d = 0; d <= size - kVecSize; d += kVecSize) {
bVec x = bVec::loadu(input + d);
fVec x0, x1;
std::tie(x0, x1) = at::vec::convert_to_float(x);
x0 = x0 * weight_vec;
x1 = x1 * weight_vec;
bVec out_vec = convert_from_float_ext<scalar_t>(x0, x1);
auto [x0, x1] = load_float_vec2(input + d);
bVec out_vec = convert_from_float_ext<scalar_t>(x0 * weight_vec, x1 * weight_vec);
out_vec.store(out + d);
}
for (; d < size; ++d) {
out[d] = static_cast<scalar_t>(input[d] * weight);
}
}
template <typename scalar_t>
inline void clamp_sigmoid_and_mul_stub(
scalar_t* __restrict__ out,
const scalar_t* __restrict__ input,
int64_t size,
const float alpha,
const float limit) {
using bVec = at::vec::Vectorized<scalar_t>;
using fVec = at::vec::Vectorized<float>;
const fVec one = fVec(1.f);
const fVec zero = fVec(0.f);
const fVec limit_v = fVec(limit);
const fVec nlimit_v = fVec(-limit);
const fVec alpha_v = fVec(alpha);
// no remainder
#pragma GCC unroll 4
for (int64_t d = 0; d < size; d += bVec::size()) {
bVec x = bVec::loadu(input + d);
fVec x0_, y0_;
std::tie(x0_, y0_) = at::vec::convert_to_float(x);
float tmp_buffer[fVec::size() * 2]; // 32
float tmp_glu[fVec::size()]; // 16
float tmp_linear[fVec::size()]; // 16
x0_.store(tmp_buffer);
y0_.store(tmp_buffer + fVec::size());
// interleaved: x[2i] = glu, x[2i+1] = linear
for (int j = 0; j < fVec::size(); ++j) {
// x0 [0,2,..30]
tmp_glu[j] = tmp_buffer[j * 2];
// y0 [1,3,...31]
tmp_linear[j] = tmp_buffer[j * 2 + 1];
}
fVec x0 = fVec::loadu(tmp_glu);
fVec y0 = fVec::loadu(tmp_linear);
// clamp
x0 = at::vec::minimum(x0, limit_v);
y0 = at::vec::minimum(limit_v, at::vec::maximum(nlimit_v, y0));
// x * sigmoid(x * alpha)
x0 = x0 / (one + (x0 * alpha_v).neg().exp_u20());
// (y + 1) * x
y0 = y0 + one;
x0 = x0 * y0;
convert_from_float_and_store<scalar_t>(out + d / 2, x0);
}
}
+6 -6
View File
@@ -37,7 +37,7 @@ void fused_experts_fp_kernel_impl(
int64_t num_tokens_post_pad,
float alpha,
float limit,
CPUAcTMethod act_func,
CPUActMethod act_func,
bool with_bias) {
constexpr int64_t BLOCK_M = block_size_m();
constexpr int64_t BLOCK_N = block_size_n();
@@ -122,17 +122,17 @@ void fused_experts_fp_kernel_impl(
});
// stage 1.5: intermediate_cache1 = silu(intermediate_cache0)
if (act_func == CPUAcTMethod::silu_and_mul) {
if (act_func == CPUActMethod::silu_and_mul) {
at::parallel_for(0, M * topk, 0, [&](int64_t begin, int64_t end) {
for (int64_t m = begin; m < end; ++m) {
silu_and_mul_stub(ic1 + m * N, ic0 + m * 2 * N, ic0 + m * 2 * N + N, N);
}
});
} else if (act_func == CPUAcTMethod::swiglu) {
} else if (act_func == CPUActMethod::swiglu) {
at::parallel_for(0, M * topk, 0, [&](int64_t begin, int64_t end) {
for (int64_t m = begin; m < end; ++m) {
clamp_sigmoid_and_mul_stub(ic1 + m * N, ic0 + m * 2 * N, N, alpha, limit);
clamp_sigmoid_and_mul_stub(ic1 + m * N + N / 2, ic0 + m * 2 * N + N, N, alpha, limit);
clamp_sigmoid_and_mul_stub(ic1 + m * N, ic0 + m * 2 * N, N / 2, alpha, limit);
clamp_sigmoid_and_mul_stub(ic1 + m * N + N / 2, ic0 + m * 2 * N + N, N / 2, alpha, limit);
}
});
}
@@ -243,7 +243,7 @@ void fused_experts_fp_kernel_impl(
int64_t num_tokens_post_pad, \
float alpha, \
float limit, \
CPUAcTMethod act_func, \
CPUActMethod act_func, \
bool with_bias)
INSTANTIATE_MOE_FP_TEMPLATE(at::BFloat16, at::Float8_e4m3fn, float, false);
+11 -19
View File
@@ -53,16 +53,14 @@ inline void silu_and_mul(
vc1[col] = _mm512_mul_ps(_mm512_mul_ps(vc1[col], vas), vbs1[col]);
};
using bVec = at::vec::Vectorized<scalar_t>;
using fVec = at::vec::Vectorized<float>;
const fVec one = fVec(1.f);
auto silu_and_mul = [&](auto col) {
fVec x = fVec(vc0[col]);
fVec y = fVec(vc1[col]);
x = x / (one + x.neg().exp_u20());
vc0[col] = x * y;
__m512 x = vc0[col];
__m512 y = vc1[col];
vc0[col] = _mm512_mul_ps(_mm512_rcp14_silu_ps(x), y);
};
using bVec = at::vec::Vectorized<scalar_t>;
using fVec = at::vec::Vectorized<float>;
auto storec = [&](auto col, int64_t m) {
if constexpr (col % 2 == 0) {
fVec x0 = fVec(vc0[col + 0]);
@@ -229,23 +227,17 @@ struct tinygemm_kernel_vnni<at::BFloat16, BLOCK_M, BLOCK_N> {
};
Unroll<ROWS * COLS>{}(scalec);
using Vec = at::vec::Vectorized<float>;
const Vec one = Vec(1.f);
auto storec = [&](auto i) {
constexpr int row = i / COLS;
constexpr int col = i % COLS;
// for COLS = 2, 4 use 512bit store
if constexpr (col % 2 == 0) {
Vec x0 = _mm512_castsi512_ps(vc0[row * COLS + col + 0]);
Vec x1 = _mm512_castsi512_ps(vc0[row * COLS + col + 1]);
Vec y0 = _mm512_castsi512_ps(vc1[row * COLS + col + 0]);
Vec y1 = _mm512_castsi512_ps(vc1[row * COLS + col + 1]);
// silu
x0 = x0 / (one + x0.neg().exp_u20());
x1 = x1 / (one + x1.neg().exp_u20());
// mul
x0 = x0 * y0;
x1 = x1 * y1;
__m512 x0 = _mm512_castsi512_ps(vc0[row * COLS + col + 0]);
__m512 x1 = _mm512_castsi512_ps(vc0[row * COLS + col + 1]);
__m512 y0 = _mm512_castsi512_ps(vc1[row * COLS + col + 0]);
__m512 y1 = _mm512_castsi512_ps(vc1[row * COLS + col + 1]);
x0 = _mm512_mul_ps(_mm512_rcp14_silu_ps(x0), y0);
x1 = _mm512_mul_ps(_mm512_rcp14_silu_ps(x1), y1);
_mm512_storeu_si512(
reinterpret_cast<__m512i*>((C + row * ldc + col * 16)),
+195 -6
View File
@@ -32,11 +32,11 @@ inline Vectorized<scalar_t> convert_from_float_ext(const Vectorized<float>& a, c
}
template <typename scalar_t>
inline void convert_from_float_and_store(scalar_t* out, const Vectorized<float>& a) {
float out_buffer[at::vec::Vectorized<float>::size()];
inline void store_from_float_ext(scalar_t* out, const Vectorized<float>& a) {
float out_buffer[Vectorized<float>::size()];
a.store(out_buffer);
for (int i = 0; i < 16; i++) {
out[i] = (scalar_t)out_buffer[i];
for (int i = 0; i < Vectorized<float>::size(); ++i) {
out[i] = static_cast<scalar_t>(out_buffer[i]);
}
}
@@ -59,6 +59,17 @@ inline std::tuple<Vectorized<float>, Vectorized<float>> load_float_vec2(const fl
return std::make_tuple(x0, x1);
}
template <typename scalar_t, typename std::enable_if_t<is_reduced_floating_point_v<scalar_t>, int> = 1>
inline at::vec::Vectorized<float> load_float_vec(const scalar_t* __restrict__ data) {
at::vec::Vectorized<float> out;
if constexpr (std::is_same_v<scalar_t, at::BFloat16>) {
at::vec::load_fp32_from_bf16(data, out);
} else {
at::vec::load_fp32_from_fp16(data, out);
}
return out;
}
#if defined(CPU_CAPABILITY_AVX512)
// `at::vec::convert_from_float<>` from PyTorch doesn't have avx512-bf16 intrinsics
@@ -70,8 +81,14 @@ convert_from_float_ext<at::BFloat16>(const Vectorized<float>& a, const Vectorize
}
template <>
inline void convert_from_float_and_store<at::BFloat16>(at::BFloat16* out, const Vectorized<float>& a) {
_mm256_storeu_si256((__m256i*)out, (__m256i)(_mm512_cvtneps_pbh(__m512(a))));
inline void store_from_float_ext<at::BFloat16>(at::BFloat16* out, const Vectorized<float>& a) {
_mm256_storeu_si256(reinterpret_cast<__m256i*>(out), (__m256i)(_mm512_cvtneps_pbh(__m512(a))));
}
template <>
inline void store_from_float_ext<at::Half>(at::Half* out, const Vectorized<float>& a) {
_mm256_storeu_si256(
reinterpret_cast<__m256i*>(out), _mm512_cvtps_ph(__m512(a), _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC));
}
#define CVT_BF16_TO_FP32(a) _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(a), 16))
@@ -301,6 +318,77 @@ inline void quantize_row_int8<at::BFloat16>(
// transpose utils
// taken from my PR in ggml: https://github.com/ggml-org/llama.cpp/pull/8998
#if defined(CPU_CAPABILITY_AVX512)
inline void transpose_16x16_16bit(__m256i* v) {
__m256i v1[16];
v1[0] = _mm256_unpacklo_epi16(v[0], v[1]);
v1[1] = _mm256_unpackhi_epi16(v[0], v[1]);
v1[2] = _mm256_unpacklo_epi16(v[2], v[3]);
v1[3] = _mm256_unpackhi_epi16(v[2], v[3]);
v1[4] = _mm256_unpacklo_epi16(v[4], v[5]);
v1[5] = _mm256_unpackhi_epi16(v[4], v[5]);
v1[6] = _mm256_unpacklo_epi16(v[6], v[7]);
v1[7] = _mm256_unpackhi_epi16(v[6], v[7]);
v1[8] = _mm256_unpacklo_epi16(v[8], v[9]);
v1[9] = _mm256_unpackhi_epi16(v[8], v[9]);
v1[10] = _mm256_unpacklo_epi16(v[10], v[11]);
v1[11] = _mm256_unpackhi_epi16(v[10], v[11]);
v1[12] = _mm256_unpacklo_epi16(v[12], v[13]);
v1[13] = _mm256_unpackhi_epi16(v[12], v[13]);
v1[14] = _mm256_unpacklo_epi16(v[14], v[15]);
v1[15] = _mm256_unpackhi_epi16(v[14], v[15]);
v[0] = _mm256_unpacklo_epi32(v1[0], v1[2]);
v[1] = _mm256_unpackhi_epi32(v1[0], v1[2]);
v[2] = _mm256_unpacklo_epi32(v1[1], v1[3]);
v[3] = _mm256_unpackhi_epi32(v1[1], v1[3]);
v[4] = _mm256_unpacklo_epi32(v1[4], v1[6]);
v[5] = _mm256_unpackhi_epi32(v1[4], v1[6]);
v[6] = _mm256_unpacklo_epi32(v1[5], v1[7]);
v[7] = _mm256_unpackhi_epi32(v1[5], v1[7]);
v[8] = _mm256_unpacklo_epi32(v1[8], v1[10]);
v[9] = _mm256_unpackhi_epi32(v1[8], v1[10]);
v[10] = _mm256_unpacklo_epi32(v1[9], v1[11]);
v[11] = _mm256_unpackhi_epi32(v1[9], v1[11]);
v[12] = _mm256_unpacklo_epi32(v1[12], v1[14]);
v[13] = _mm256_unpackhi_epi32(v1[12], v1[14]);
v[14] = _mm256_unpacklo_epi32(v1[13], v1[15]);
v[15] = _mm256_unpackhi_epi32(v1[13], v1[15]);
v1[0] = _mm256_unpacklo_epi64(v[0], v[4]);
v1[1] = _mm256_unpackhi_epi64(v[0], v[4]);
v1[2] = _mm256_unpacklo_epi64(v[1], v[5]);
v1[3] = _mm256_unpackhi_epi64(v[1], v[5]);
v1[4] = _mm256_unpacklo_epi64(v[2], v[6]);
v1[5] = _mm256_unpackhi_epi64(v[2], v[6]);
v1[6] = _mm256_unpacklo_epi64(v[3], v[7]);
v1[7] = _mm256_unpackhi_epi64(v[3], v[7]);
v1[8] = _mm256_unpacklo_epi64(v[8], v[12]);
v1[9] = _mm256_unpackhi_epi64(v[8], v[12]);
v1[10] = _mm256_unpacklo_epi64(v[9], v[13]);
v1[11] = _mm256_unpackhi_epi64(v[9], v[13]);
v1[12] = _mm256_unpacklo_epi64(v[10], v[14]);
v1[13] = _mm256_unpackhi_epi64(v[10], v[14]);
v1[14] = _mm256_unpacklo_epi64(v[11], v[15]);
v1[15] = _mm256_unpackhi_epi64(v[11], v[15]);
v[0] = _mm256_permute2x128_si256(v1[0], v1[8], 0x20);
v[1] = _mm256_permute2x128_si256(v1[1], v1[9], 0x20);
v[2] = _mm256_permute2x128_si256(v1[2], v1[10], 0x20);
v[3] = _mm256_permute2x128_si256(v1[3], v1[11], 0x20);
v[4] = _mm256_permute2x128_si256(v1[4], v1[12], 0x20);
v[5] = _mm256_permute2x128_si256(v1[5], v1[13], 0x20);
v[6] = _mm256_permute2x128_si256(v1[6], v1[14], 0x20);
v[7] = _mm256_permute2x128_si256(v1[7], v1[15], 0x20);
v[8] = _mm256_permute2x128_si256(v1[0], v1[8], 0x31);
v[9] = _mm256_permute2x128_si256(v1[1], v1[9], 0x31);
v[10] = _mm256_permute2x128_si256(v1[2], v1[10], 0x31);
v[11] = _mm256_permute2x128_si256(v1[3], v1[11], 0x31);
v[12] = _mm256_permute2x128_si256(v1[4], v1[12], 0x31);
v[13] = _mm256_permute2x128_si256(v1[5], v1[13], 0x31);
v[14] = _mm256_permute2x128_si256(v1[6], v1[14], 0x31);
v[15] = _mm256_permute2x128_si256(v1[7], v1[15], 0x31);
}
inline void transpose_16x16_32bit(__m512i* v) {
__m512i v1[16];
v1[0] = _mm512_unpacklo_epi32(v[0], v[1]);
@@ -394,6 +482,62 @@ inline std::tuple<__m512i, __m512i> transpose_2x32_16bit(__m512i r0, __m512i r1)
}
#pragma GCC diagnostic pop
// Note: mapped from aten exp_u20
inline __attribute__((always_inline)) __m512 _mm512_exp_u20_ps(const __m512 values) {
const __m512 vec_factorial_1 = _mm512_set1_ps(0.999999701f);
const __m512 vec_factorial_2 = _mm512_set1_ps(0.499991506f);
const __m512 vec_factorial_3 = _mm512_set1_ps(0.166676521f);
const __m512 vec_factorial_4 = _mm512_set1_ps(0.0418978221f);
const __m512 vec_factorial_5 = _mm512_set1_ps(0.00828929059f);
const __m512 vec_exp_log2ef = _mm512_castsi512_ps(_mm512_set1_epi32(0x3fb8aa3b)); // log2(e)
const __m512 vec_half = _mm512_set1_ps(0.5f);
const __m512 vec_one = _mm512_set1_ps(1.f);
const __m512 vec_zero = _mm512_set1_ps(0.f);
const __m512 vec_two = _mm512_set1_ps(2.f);
const __m512 vec_ln2f = _mm512_castsi512_ps(_mm512_set1_epi32(0x3f317218));
const __m512 vec_ln_flt_min = _mm512_castsi512_ps(_mm512_set1_epi32(0xc2aeac50));
const __m512 vec_ln_flt_max = _mm512_castsi512_ps(_mm512_set1_epi32(0x42b17218));
const __m512i vec_127 = _mm512_set1_epi32(0x0000007f);
const int n_mantissa_bits = 23;
// exp(x) =
// = exp(n * ln(2) + r) // divide x by ln(2) and get quot and rem
// = 2^n * exp(r) // simplify the exp(n*ln(2)) expression
auto less_ln_flt_min_mask = _mm512_cmp_ps_mask(values, vec_ln_flt_min, 1 /*_CMP_LT_OS*/);
auto vec_src = _mm512_min_ps(values, vec_ln_flt_max);
vec_src = _mm512_max_ps(vec_src, vec_ln_flt_min);
// fx = floorf(x * log2ef + 0.5)
auto vec_fx = _mm512_fmadd_ps(vec_src, vec_exp_log2ef, vec_half);
auto vec_fx_i = _mm512_cvt_roundps_epi32(vec_fx, _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC);
vec_fx = _mm512_cvtepi32_ps(vec_fx_i);
// x = x - fx * ln2
auto vec_exp_poly = _mm512_fnmadd_ps(vec_fx, vec_ln2f, vec_src);
// compute polynomial
auto vec_res = _mm512_fmadd_ps(vec_exp_poly, vec_factorial_5, vec_factorial_4);
vec_res = _mm512_fmadd_ps(vec_exp_poly, vec_res, vec_factorial_3);
vec_res = _mm512_fmadd_ps(vec_exp_poly, vec_res, vec_factorial_2);
vec_res = _mm512_fmadd_ps(vec_exp_poly, vec_res, vec_factorial_1);
vec_res = _mm512_fmadd_ps(vec_exp_poly, vec_res, vec_one);
// compute 2^(n-1)
auto vec_exp_number = _mm512_sub_ps(vec_fx, vec_one);
auto vec_exp_number_i = _mm512_cvtps_epi32(vec_exp_number);
auto vec_two_pow_n_i = _mm512_add_epi32(vec_exp_number_i, vec_127);
vec_two_pow_n_i = _mm512_slli_epi32(vec_two_pow_n_i, n_mantissa_bits);
auto vec_two_pow_n = _mm512_castsi512_ps(vec_two_pow_n_i);
vec_two_pow_n = _mm512_mask_blend_ps(less_ln_flt_min_mask, vec_two_pow_n, vec_zero);
// y = y * 2^n
vec_res = _mm512_mul_ps(vec_res, vec_two_pow_n);
vec_res = _mm512_mul_ps(vec_res, vec_two);
return vec_res;
}
// Note: mapped from aten fexp_u20
inline __attribute__((always_inline)) __m512 _mm512_fexp_u20_ps(const __m512 values) {
const __m512 vec_c0 = _mm512_set1_ps(0.00010703434948458272f);
const __m512 vec_c1 = _mm512_set1_ps(0.30354260500649682f);
@@ -444,6 +588,51 @@ inline __attribute__((always_inline)) __m512 _mm512_fexp_u20_ps(const __m512 val
// final interpretation to float
return _mm512_castsi512_ps(casted_integer);
}
// sigmoid(x) = 1 / (1 + exp(-x)); avoid vdivps via rcp14
inline __attribute__((always_inline)) __m512 _mm512_rcp14_sigmoid_ps(__m512 x) {
__m512 minus_x = _mm512_xor_ps(_mm512_set1_ps(-0.f), x);
__m512 denom = _mm512_add_ps(_mm512_exp_u20_ps(minus_x), _mm512_set1_ps(1.f));
return _mm512_rcp14_ps(denom);
}
// SiLU(x) = x * sigmoid(x)
inline __attribute__((always_inline)) __m512 _mm512_rcp14_silu_ps(__m512 x) {
return _mm512_mul_ps(x, _mm512_rcp14_sigmoid_ps(x));
}
// x * sigmoid(x * alpha) for clamped SwiGLU
inline __attribute__((always_inline)) __m512 _mm512_rcp14_sigmoid_glu_ps(__m512 x, __m512 alpha) {
__m512 xa = _mm512_mul_ps(x, alpha);
return _mm512_mul_ps(x, _mm512_rcp14_sigmoid_ps(xa));
}
#endif
inline at::vec::Vectorized<float> fast_sigmoid(const at::vec::Vectorized<float>& x) {
#if defined(CPU_CAPABILITY_AVX512)
return at::vec::Vectorized<float>(_mm512_rcp14_sigmoid_ps(x));
#else
const auto one = at::vec::Vectorized<float>(1.f);
return one / (one + x.neg().exp_u20());
#endif
}
inline at::vec::Vectorized<float> fast_silu(const at::vec::Vectorized<float>& x) {
#if defined(CPU_CAPABILITY_AVX512)
return at::vec::Vectorized<float>(_mm512_rcp14_silu_ps(x));
#else
return x * fast_sigmoid(x);
#endif
}
inline at::vec::Vectorized<float>
fast_sigmoid_glu(const at::vec::Vectorized<float>& x, const at::vec::Vectorized<float>& alpha) {
#if defined(CPU_CAPABILITY_AVX512)
return at::vec::Vectorized<float>(_mm512_rcp14_sigmoid_glu_ps(x, alpha));
#else
return x * fast_sigmoid(x * alpha);
#endif
}
} // anonymous namespace
+3 -2
View File
@@ -110,7 +110,7 @@ std::tuple<at::Tensor, at::Tensor> chunk_gated_delta_rule_cpu(
const at::Tensor& g, const at::Tensor& beta,
const at::Tensor& initial_state, bool output_final_state,
const at::Tensor& cu_seqlens, bool head_first, bool use_qk_l2norm_in_kernel,
double eps = 1e-5);
const at::Tensor& initial_state_indices, double eps = 1e-5);
at::Tensor fused_sigmoid_gating_delta_rule_update_cpu(
const at::Tensor& A_log, const at::Tensor& dt_bias, const at::Tensor& q,
@@ -546,7 +546,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
"Tensor g, Tensor beta, "
"Tensor initial_state, bool output_final_state, Tensor cu_seqlens, bool "
"head_first, "
"bool use_qk_l2norm_in_kernel, float eps=1e-5) -> (Tensor, Tensor)");
"bool use_qk_l2norm_in_kernel, Tensor initial_state_indices, float "
"eps=1e-5) -> (Tensor, Tensor)");
ops.impl("chunk_gated_delta_rule_cpu", torch::kCPU,
&chunk_gated_delta_rule_cpu);
ops.def(
+13 -2
View File
@@ -26,6 +26,13 @@ HEAD_DIMS = [
(32, 32),
(64, 32),
]
# chunk_gated_delta_rule_cpu (the chunked-prefill kernel) only supports
# head_dim == head_dim_v in {64, 128}; the decode-path update kernel above
# has no such restriction and keeps using the wider HEAD_DIMS list.
CHUNK_HEAD_DIMS = [
(64, 64),
(128, 128),
]
CHUNK_SIZE = 64
CONV_DIM = 128
CONV_KERNEL = 4
@@ -258,7 +265,7 @@ def test_fused_sigmoid_gating_delta_rule_update_cpu(
# prefill path
@pytest.mark.parametrize("seq_lens", PREFILL_SEQ_LENS)
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@pytest.mark.parametrize("head_dims", HEAD_DIMS)
@pytest.mark.parametrize("head_dims", CHUNK_HEAD_DIMS)
@torch.inference_mode()
def test_chunk_gated_delta_rule_cpu(
seq_lens: list[int],
@@ -307,6 +314,7 @@ def test_chunk_gated_delta_rule_cpu(
cu_seqlens=cu_seqlens,
head_first=False,
use_qk_l2norm_in_kernel=True,
initial_state_indices=torch.arange(len(seq_lens), dtype=torch.int32),
)
torch.testing.assert_close(out, out_ref, atol=1e-2, rtol=1e-2)
@@ -331,7 +339,7 @@ TWO_CALL_SPLITS = [
@pytest.mark.parametrize("total_tokens, split", TWO_CALL_SPLITS)
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@pytest.mark.parametrize("head_dims", HEAD_DIMS)
@pytest.mark.parametrize("head_dims", CHUNK_HEAD_DIMS)
@torch.inference_mode()
def test_chunk_gated_delta_rule_cpu_two_call_split(
total_tokens: int,
@@ -370,6 +378,7 @@ def test_chunk_gated_delta_rule_cpu_two_call_split(
cu_seqlens=torch.tensor([0, total_tokens], dtype=torch.int32),
head_first=False,
use_qk_l2norm_in_kernel=True,
initial_state_indices=torch.zeros(1, dtype=torch.int32),
)
# Call 1: tokens [0:split], no initial state, capture final state.
@@ -384,6 +393,7 @@ def test_chunk_gated_delta_rule_cpu_two_call_split(
cu_seqlens=torch.tensor([0, split], dtype=torch.int32),
head_first=False,
use_qk_l2norm_in_kernel=True,
initial_state_indices=torch.zeros(1, dtype=torch.int32),
)
# Call 2: tokens [split:T] seeded with call 1's final state and a cu_seqlens
# rebased to start at 0, as cpu_gdn_attention_core continues a prefill chunk.
@@ -399,6 +409,7 @@ def test_chunk_gated_delta_rule_cpu_two_call_split(
cu_seqlens=torch.tensor([0, tail], dtype=torch.int32),
head_first=False,
use_qk_l2norm_in_kernel=True,
initial_state_indices=torch.zeros(1, dtype=torch.int32),
)
out_split = torch.cat([out1, out2], dim=1)
+2
View File
@@ -3491,6 +3491,7 @@ def chunk_gated_delta_rule_cpu(
cu_seqlens: torch.Tensor,
head_first: bool,
use_qk_l2norm_in_kernel: bool,
initial_state_indices: torch.Tensor,
eps: float = 1e-5,
) -> tuple[torch.Tensor, torch.Tensor]:
return torch.ops._C.chunk_gated_delta_rule_cpu(
@@ -3504,6 +3505,7 @@ def chunk_gated_delta_rule_cpu(
cu_seqlens,
head_first,
use_qk_l2norm_in_kernel,
initial_state_indices,
eps,
)
@@ -236,22 +236,24 @@ def _cpu_gdn_attention_nonspec(
A_log=layer.A_log, a=prefill_a, b=prefill_b, dt_bias=layer.dt_bias
)
initial_state = ssm_state[prefill_state_indices]
initial_state[~prefill_has_initial_state, ...] = 0
attn_out, last_recurrent_state = ops.chunk_gated_delta_rule_cpu(
# zero pool slots for sequences without a prior state; the kernel
# gathers/mutates ssm_state in place via prefill_state_indices, so no
# separate scatter-back is needed after the call.
no_initial_state = prefill_state_indices[~prefill_has_initial_state]
if no_initial_state.numel() > 0:
ssm_state[no_initial_state] = 0
attn_out, _ = ops.chunk_gated_delta_rule_cpu(
query=query,
key=key,
value=value,
g=g,
beta=beta,
initial_state=initial_state,
initial_state=ssm_state,
output_final_state=True,
cu_seqlens=prefill_query_start_loc,
head_first=False,
use_qk_l2norm_in_kernel=True,
)
ssm_state[prefill_state_indices] = last_recurrent_state.to(
ssm_state.dtype, copy=False
initial_state_indices=prefill_state_indices,
)
core_attn_out[prefill_token_start:prefill_token_end] = attn_out.squeeze(0)
@@ -616,22 +618,24 @@ def _spec_aware_nonspec(
g, beta = ops.fused_gdn_gating_cpu(
A_log=layer.A_log, a=prefill_a, b=prefill_b, dt_bias=layer.dt_bias
)
initial_state = ssm_state[prefill_state_indices]
initial_state[~prefill_has_initial_state, ...] = 0
attn_out, last_recurrent_state = ops.chunk_gated_delta_rule_cpu(
# zero pool slots for sequences without a prior state; the kernel
# gathers/mutates ssm_state in place via prefill_state_indices, so no
# separate scatter-back is needed after the call.
no_initial_state = prefill_state_indices[~prefill_has_initial_state]
if no_initial_state.numel() > 0:
ssm_state[no_initial_state] = 0
attn_out, _ = ops.chunk_gated_delta_rule_cpu(
query=query,
key=key,
value=value,
g=g,
beta=beta,
initial_state=initial_state,
initial_state=ssm_state,
output_final_state=True,
cu_seqlens=prefill_query_start_loc,
head_first=False,
use_qk_l2norm_in_kernel=True,
)
ssm_state[prefill_state_indices] = last_recurrent_state.to(
ssm_state.dtype, copy=False
initial_state_indices=prefill_state_indices,
)
core_attn_out[prefill_token_start:prefill_token_end] = attn_out.squeeze(0)
@@ -690,22 +694,24 @@ def _spec_aware_nonspec_subset(
g, beta = ops.fused_gdn_gating_cpu(
A_log=layer.A_log, a=a, b=b, dt_bias=layer.dt_bias
)
initial_state = ssm_state[prefill_state_indices]
initial_state[~has_initial_state, ...] = 0
attn_out, last_recurrent_state = ops.chunk_gated_delta_rule_cpu(
# zero pool slots for sequences without a prior state; the kernel
# gathers/mutates ssm_state in place via prefill_state_indices, so no
# separate scatter-back is needed after the call.
no_initial_state = prefill_state_indices[~has_initial_state]
if no_initial_state.numel() > 0:
ssm_state[no_initial_state] = 0
attn_out, _ = ops.chunk_gated_delta_rule_cpu(
query=query,
key=key,
value=value,
g=g,
beta=beta,
initial_state=initial_state,
initial_state=ssm_state,
output_final_state=True,
cu_seqlens=prefill_qsl,
head_first=False,
use_qk_l2norm_in_kernel=True,
)
ssm_state[prefill_state_indices] = last_recurrent_state.to(
ssm_state.dtype, copy=False
initial_state_indices=prefill_state_indices,
)
return attn_out.squeeze(0)