[CPU] Bump up CPU kernels to latest version (#50387)
Signed-off-by: jiang1.li <jiang1.li@intel.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -1,11 +1,16 @@
|
||||
// Adapted from
|
||||
// https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu
|
||||
//
|
||||
// Synced from
|
||||
// https://github.com/sgl-project/sglang/tree/7c248dde7fe1f3b5100966f8143f97a9932c22a4/sgl-kernel/csrc/cpu
|
||||
// Sync date: 2026-07-29
|
||||
|
||||
// clang-format off
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <ATen/ATen.h>
|
||||
#include <ATen/Dispatch.h>
|
||||
#include <ATen/Parallel.h>
|
||||
|
||||
#if defined(_OPENMP)
|
||||
@@ -49,6 +54,14 @@ namespace {
|
||||
} \
|
||||
}()
|
||||
|
||||
// Half + BFloat16, plus one extra scalar type
|
||||
#define AT_DISPATCH_CASE_REDUCED_FLOATING_TYPES_AND(SCALARTYPE, ...) \
|
||||
AT_DISPATCH_CASE_REDUCED_FLOATING_TYPES(__VA_ARGS__) \
|
||||
AT_DISPATCH_CASE(SCALARTYPE, __VA_ARGS__)
|
||||
|
||||
#define AT_DISPATCH_REDUCED_FLOATING_TYPES_AND(SCALARTYPE, TYPE, NAME, ...) \
|
||||
AT_DISPATCH_SWITCH(TYPE, NAME, AT_DISPATCH_CASE_REDUCED_FLOATING_TYPES_AND(SCALARTYPE, __VA_ARGS__))
|
||||
|
||||
// dispatch: bfloat16, float16, int8_t, fp8_e4m3, uint8_t(mxfp4/int4)
|
||||
#define CPU_DISPATCH_PACKED_TYPES(TYPE, ...) \
|
||||
[&] { \
|
||||
|
||||
@@ -182,14 +182,13 @@ struct tinygemm_kernel<at::BFloat16, K, BLOCK_N, has_bias, has_silu> {
|
||||
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
using bVec = at::vec::Vectorized<at::BFloat16>;
|
||||
const fVec one = fVec(1.f);
|
||||
auto storec = [&](auto i, int64_t m) {
|
||||
constexpr int col = i;
|
||||
fVec x0 = fVec(vc[col * 2 + 0]);
|
||||
fVec x1 = fVec(vc[col * 2 + 1]);
|
||||
if constexpr (has_silu) {
|
||||
x0 = x0 / (one + x0.neg().exp_u20());
|
||||
x1 = x1 / (one + x1.neg().exp_u20());
|
||||
x0 = fast_silu(x0);
|
||||
x1 = fast_silu(x1);
|
||||
}
|
||||
bVec out_vec = convert_from_float_ext<at::BFloat16>(x0, x1);
|
||||
out_vec.store(C + m * lda + col * 32);
|
||||
|
||||
+1699
-1003
File diff suppressed because it is too large
Load Diff
@@ -116,8 +116,7 @@ inline void copy_stub(scalar_t* __restrict__ out, const float* __restrict__ inpu
|
||||
int64_t d;
|
||||
#pragma GCC unroll 4
|
||||
for (d = 0; d <= size - kVecSize; d += kVecSize) {
|
||||
fVec data0 = fVec::loadu(input + d);
|
||||
fVec data1 = fVec::loadu(input + d + fVec::size());
|
||||
auto [data0, data1] = load_float_vec2(input + d);
|
||||
bVec out_vec = convert_from_float_ext<scalar_t>(data0, data1);
|
||||
out_vec.store(out + d);
|
||||
}
|
||||
@@ -135,9 +134,7 @@ inline void copy_stub(float* __restrict__ out, const scalar_t* __restrict__ inpu
|
||||
int64_t d;
|
||||
#pragma GCC unroll 4
|
||||
for (d = 0; d <= size - kVecSize; d += kVecSize) {
|
||||
fVec data0, data1;
|
||||
bVec b_vec = bVec::loadu(input + d);
|
||||
std::tie(data0, data1) = at::vec::convert_to_float(b_vec);
|
||||
auto [data0, data1] = load_float_vec2(input + d);
|
||||
data0.store(out + d);
|
||||
data1.store(out + d + fVec::size());
|
||||
}
|
||||
@@ -156,9 +153,9 @@ inline void copy_add_stub(
|
||||
int64_t d;
|
||||
#pragma GCC unroll 4
|
||||
for (d = 0; d <= size - kVecSize; d += kVecSize) {
|
||||
fVec data0 = fVec::loadu(input + d) + fVec::loadu(bias + d);
|
||||
fVec data1 = fVec::loadu(input + d + fVec::size()) + fVec::loadu(bias + d + fVec::size());
|
||||
bVec out_vec = convert_from_float_ext<scalar_t>(data0, data1);
|
||||
auto [data0, data1] = load_float_vec2(input + d);
|
||||
auto [bias0, bias1] = load_float_vec2(bias + d);
|
||||
bVec out_vec = convert_from_float_ext<scalar_t>(data0 + bias0, data1 + bias1);
|
||||
out_vec.store(out + d);
|
||||
}
|
||||
for (; d < size; ++d) {
|
||||
@@ -176,7 +173,6 @@ inline void scalar_sigmoid_and_mul(
|
||||
using bVec = at::vec::Vectorized<scalar_t>;
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
// scalar sigmoid
|
||||
const fVec one = fVec(1.f);
|
||||
fVec X;
|
||||
if constexpr (has_bias) {
|
||||
assert(bias != nullptr);
|
||||
@@ -184,18 +180,13 @@ inline void scalar_sigmoid_and_mul(
|
||||
} else {
|
||||
X = fVec(input[0]);
|
||||
}
|
||||
X = one / (one + X.neg().exp_u20());
|
||||
X = fast_sigmoid(X);
|
||||
|
||||
// vec mul
|
||||
constexpr int kVecSize = bVec::size();
|
||||
for (int d = 0; d < SIZE; d += kVecSize) {
|
||||
bVec m_bvec = bVec::loadu(mul + d);
|
||||
fVec m_fvec0, m_fvec1;
|
||||
std::tie(m_fvec0, m_fvec1) = at::vec::convert_to_float(m_bvec);
|
||||
m_fvec0 = m_fvec0 * X;
|
||||
m_fvec1 = m_fvec1 * X;
|
||||
|
||||
bVec out_vec = convert_from_float_ext<scalar_t>(m_fvec0, m_fvec1);
|
||||
auto [m_fvec0, m_fvec1] = load_float_vec2(mul + d);
|
||||
bVec out_vec = convert_from_float_ext<scalar_t>(m_fvec0 * X, m_fvec1 * X);
|
||||
out_vec.store(out + d);
|
||||
}
|
||||
}
|
||||
@@ -727,10 +718,10 @@ at::Tensor convert_scale_packed(at::Tensor& scale) {
|
||||
return packed_scale;
|
||||
}
|
||||
|
||||
// mat1 : [M, K]
|
||||
// mat1 : [*, K]
|
||||
// mat2 : [N, K] ([K, N] if use_fma_gemm)
|
||||
// bias : [N]
|
||||
// out : [M, N]
|
||||
// out : [*, N]
|
||||
//
|
||||
at::Tensor
|
||||
weight_packed_linear(at::Tensor& mat1, at::Tensor& mat2, const std::optional<at::Tensor>& bias, bool is_vnni) {
|
||||
@@ -740,23 +731,25 @@ weight_packed_linear(at::Tensor& mat1, at::Tensor& mat2, const std::optional<at:
|
||||
use_fma_gemm = true;
|
||||
}
|
||||
|
||||
int64_t M = mat1.size(0);
|
||||
int64_t K = mat1.size(1);
|
||||
int64_t N = use_fma_gemm ? mat2.size(1) : mat2.size(0);
|
||||
|
||||
CHECK_LAST_DIM_CONTIGUOUS_INPUT(mat1);
|
||||
CHECK_INPUT(mat2);
|
||||
CHECK_DIM(2, mat1);
|
||||
const int64_t ndim = mat1.ndimension();
|
||||
auto input_sizes = mat1.sizes().vec();
|
||||
int64_t N = use_fma_gemm ? mat2.size(1) : mat2.size(0);
|
||||
int64_t K = use_fma_gemm ? mat1.size(1) : mat2.size(1);
|
||||
int64_t M = use_fma_gemm ? mat1.size(0) : mat1.numel() / K;
|
||||
CHECK_DIM(2, mat2);
|
||||
if (!use_fma_gemm) {
|
||||
CHECK_EQ(mat1.size(1), K);
|
||||
if (use_fma_gemm) {
|
||||
CHECK_DIM(2, mat1);
|
||||
} else {
|
||||
CHECK_EQ(mat1.size(ndim - 1), K);
|
||||
}
|
||||
|
||||
auto dispatch_type = mat1.scalar_type();
|
||||
auto out = at::empty({M, N}, mat1.options());
|
||||
// strides
|
||||
int64_t out_strideM = out.stride(0);
|
||||
int64_t mat1_strideM = mat1.stride(0);
|
||||
int64_t mat1_strideM = mat1.stride(-2);
|
||||
|
||||
const bool has_bias = bias.has_value();
|
||||
const float* bias_data = nullptr;
|
||||
@@ -792,7 +785,8 @@ weight_packed_linear(at::Tensor& mat1, at::Tensor& mat2, const std::optional<at:
|
||||
}
|
||||
});
|
||||
|
||||
return out;
|
||||
input_sizes[ndim - 1] = N;
|
||||
return out.view(input_sizes);
|
||||
}
|
||||
|
||||
// mat1 : [M, K]
|
||||
|
||||
+17
-10
@@ -83,15 +83,11 @@ inline int64_t get_row_size(int64_t K, bool use_int8_w8a8) {
|
||||
return use_int8_w8a8 ? K + sizeof(int32_t) : K;
|
||||
}
|
||||
|
||||
enum class CPUAcTMethod : int { silu_and_mul = 0, swiglu = 1 };
|
||||
|
||||
constexpr bool operator==(CPUAcTMethod a, int b) {
|
||||
return static_cast<int>(a) == b;
|
||||
}
|
||||
|
||||
constexpr bool operator==(int a, CPUAcTMethod b) {
|
||||
return a == static_cast<int>(b);
|
||||
}
|
||||
enum class CPUActMethod : int {
|
||||
silu_and_mul = 0,
|
||||
swiglu = 1,
|
||||
gelu_and_mul = 2,
|
||||
};
|
||||
|
||||
enum class CPUQuantMethod : int64_t { BF16 = 0, INT8_W8A8 = 1, FP8_W8A16 = 2, INT4_W4A8 = 3, MXFP4 = 4 };
|
||||
|
||||
@@ -113,6 +109,17 @@ constexpr bool operator==(int64_t a, CPUQuantAlgo b) {
|
||||
return a == static_cast<int64_t>(b);
|
||||
}
|
||||
|
||||
inline int64_t get_row_size(CPUQuantMethod quant, int64_t K) {
|
||||
switch (quant) {
|
||||
case CPUQuantMethod::INT8_W8A8:
|
||||
return K + sizeof(int32_t);
|
||||
case CPUQuantMethod::MXFP4:
|
||||
return K >> 1;
|
||||
default:
|
||||
return K;
|
||||
}
|
||||
}
|
||||
|
||||
inline int64_t get_4bit_block_k_size(int64_t group_size) {
|
||||
return group_size > 128 ? 128 : group_size;
|
||||
}
|
||||
@@ -184,7 +191,7 @@ void fused_experts_fp_kernel_impl(
|
||||
int64_t num_tokens_post_pad,
|
||||
float alpha,
|
||||
float limit,
|
||||
CPUAcTMethod act_func,
|
||||
CPUActMethod act_func,
|
||||
bool with_bias);
|
||||
|
||||
// shared expert implementation for int8 w8a8
|
||||
|
||||
@@ -18,8 +18,7 @@ inline void copy_stub(scalar_t* __restrict__ out, const float* __restrict__ inpu
|
||||
int64_t d;
|
||||
#pragma GCC unroll 4
|
||||
for (d = 0; d <= size - kVecSize; d += kVecSize) {
|
||||
fVec data0 = fVec::loadu(input + d);
|
||||
fVec data1 = fVec::loadu(input + d + fVec::size());
|
||||
auto [data0, data1] = load_float_vec2(input + d);
|
||||
bVec out_vec = convert_from_float_ext<scalar_t>(data0, data1);
|
||||
out_vec.store(out + d);
|
||||
}
|
||||
@@ -38,9 +37,9 @@ inline void copy_add_stub(
|
||||
int64_t d;
|
||||
#pragma GCC unroll 4
|
||||
for (d = 0; d <= size - kVecSize; d += kVecSize) {
|
||||
fVec data0 = fVec::loadu(input + d) + fVec::loadu(bias + d);
|
||||
fVec data1 = fVec::loadu(input + d + fVec::size()) + fVec::loadu(bias + d + fVec::size());
|
||||
bVec out_vec = convert_from_float_ext<scalar_t>(data0, data1);
|
||||
auto [data0, data1] = load_float_vec2(input + d);
|
||||
auto [bias0, bias1] = load_float_vec2(bias + d);
|
||||
bVec out_vec = convert_from_float_ext<scalar_t>(data0 + bias0, data1 + bias1);
|
||||
out_vec.store(out + d);
|
||||
}
|
||||
for (; d < size; ++d) {
|
||||
@@ -57,9 +56,8 @@ inline void copy_mul_stub(scalar_t* __restrict__ out, const float* __restrict__
|
||||
int d;
|
||||
#pragma GCC unroll 4
|
||||
for (d = 0; d <= size - kVecSize; d += kVecSize) {
|
||||
fVec data0 = fVec::loadu(input + d) * vscale;
|
||||
fVec data1 = fVec::loadu(input + d + fVec::size()) * vscale;
|
||||
bVec out_vec = convert_from_float_ext<scalar_t>(data0, data1);
|
||||
auto [data0, data1] = load_float_vec2(input + d);
|
||||
bVec out_vec = convert_from_float_ext<scalar_t>(data0 * vscale, data1 * vscale);
|
||||
out_vec.store(out + d);
|
||||
}
|
||||
for (; d < size; ++d) {
|
||||
|
||||
+124
-100
@@ -123,46 +123,6 @@ int moe_align_block_size(
|
||||
return num_tokens_post_pad;
|
||||
}
|
||||
|
||||
// silu : shape leading dimension
|
||||
// input0 [m_size, BLOCK_N] BLOCK_N
|
||||
// input1 [m_size, BLOCK_N] BLOCK_N
|
||||
// output [M * topk, N] N
|
||||
template <typename scalar_t, int BLOCK_N>
|
||||
inline void silu_and_mul(
|
||||
scalar_t* __restrict__ output,
|
||||
const float* __restrict__ input0, // x: x0, x1
|
||||
const float* __restrict__ input1, // y: y0, y1
|
||||
int64_t m_size,
|
||||
int64_t N) {
|
||||
using bVec = at::vec::Vectorized<scalar_t>;
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
|
||||
const fVec one = fVec(1.f);
|
||||
|
||||
// no remainder
|
||||
for (int64_t m = 0; m < m_size; ++m) {
|
||||
scalar_t* __restrict__ out = output + m * N;
|
||||
const float* __restrict__ x = input0 + m * BLOCK_N;
|
||||
const float* __restrict__ y = input1 + m * BLOCK_N;
|
||||
|
||||
for (int64_t d = 0; d < BLOCK_N; d += bVec::size()) {
|
||||
fVec x0 = fVec::loadu(x + d);
|
||||
fVec x1 = fVec::loadu(x + d + fVec::size());
|
||||
fVec y0 = fVec::loadu(y + d);
|
||||
fVec y1 = fVec::loadu(y + d + fVec::size());
|
||||
// silu
|
||||
x0 = x0 / (one + x0.neg().exp_u20());
|
||||
x1 = x1 / (one + x1.neg().exp_u20());
|
||||
// mul
|
||||
x0 = x0 * y0;
|
||||
x1 = x1 * y1;
|
||||
// convert
|
||||
bVec out_vec = convert_from_float_ext<scalar_t>(x0, x1);
|
||||
out_vec.store(out + d);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t, int BLOCK_M, int BLOCK_N>
|
||||
struct tinygemm_kernel_nn2 {
|
||||
static inline void apply(
|
||||
@@ -239,23 +199,17 @@ struct tinygemm_kernel_nn2<at::BFloat16, BLOCK_M, BLOCK_N> {
|
||||
Unroll<ROWS * COLS>{}(compute, k);
|
||||
}
|
||||
|
||||
using Vec = at::vec::Vectorized<float>;
|
||||
const Vec one = Vec(1.f);
|
||||
auto storec = [&](auto i) {
|
||||
constexpr int row = i / COLS;
|
||||
constexpr int col = i % COLS;
|
||||
// for COLS = 2, 4 use 512bit store
|
||||
if constexpr (col % 2 == 0) {
|
||||
Vec x0 = vc0[row * COLS + col + 0];
|
||||
Vec x1 = vc0[row * COLS + col + 1];
|
||||
Vec y0 = vc1[row * COLS + col + 0];
|
||||
Vec y1 = vc1[row * COLS + col + 1];
|
||||
// silu
|
||||
x0 = x0 / (one + x0.neg().exp_u20());
|
||||
x1 = x1 / (one + x1.neg().exp_u20());
|
||||
// mul
|
||||
x0 = x0 * y0;
|
||||
x1 = x1 * y1;
|
||||
__m512 x0 = vc0[row * COLS + col + 0];
|
||||
__m512 x1 = vc0[row * COLS + col + 1];
|
||||
__m512 y0 = vc1[row * COLS + col + 0];
|
||||
__m512 y1 = vc1[row * COLS + col + 1];
|
||||
x0 = _mm512_mul_ps(_mm512_rcp14_silu_ps(x0), y0);
|
||||
x1 = _mm512_mul_ps(_mm512_rcp14_silu_ps(x1), y1);
|
||||
|
||||
_mm512_storeu_si512(
|
||||
reinterpret_cast<__m512i*>((C + row * ldc + col * 16)),
|
||||
@@ -455,6 +409,8 @@ void fused_experts_kernel_impl(
|
||||
const scalar_t* __restrict__ input,
|
||||
const scalar_t* __restrict__ packed_w1,
|
||||
const scalar_t* __restrict__ packed_w2,
|
||||
const float* __restrict__ w1_bias,
|
||||
const float* __restrict__ w2_bias,
|
||||
const float* __restrict__ topk_weights,
|
||||
const int32_t* __restrict__ sorted_ids,
|
||||
const int32_t* __restrict__ expert_ids,
|
||||
@@ -464,7 +420,11 @@ void fused_experts_kernel_impl(
|
||||
int64_t K,
|
||||
int64_t E,
|
||||
int64_t topk,
|
||||
int64_t num_tokens_post_pad) {
|
||||
int64_t num_tokens_post_pad,
|
||||
float alpha,
|
||||
float limit,
|
||||
CPUActMethod act_func,
|
||||
bool with_bias) {
|
||||
// handle 2 tiles per block
|
||||
constexpr int64_t BLOCK_M = block_size_m();
|
||||
constexpr int64_t BLOCK_N = block_size_n();
|
||||
@@ -499,6 +459,8 @@ void fused_experts_kernel_impl(
|
||||
int32_t expert_id = expert_ids[mb];
|
||||
const scalar_t* __restrict__ B0 = packed_w1 + expert_id * stride_e + nb_upper * BLOCK_N * stride_n;
|
||||
const scalar_t* __restrict__ B1 = packed_w1 + expert_id * stride_e + nb_lower * BLOCK_N * stride_n;
|
||||
const float* __restrict__ B0_bias = w1_bias + expert_id * 2 * N + nb_upper * BLOCK_N;
|
||||
const float* __restrict__ B1_bias = w1_bias + expert_id * 2 * N + nb_lower * BLOCK_N;
|
||||
|
||||
int64_t m_size = offsets[mb + 1] - offsets[mb];
|
||||
|
||||
@@ -538,23 +500,62 @@ void fused_experts_kernel_impl(
|
||||
/* B */ B1,
|
||||
/* C */ C1);
|
||||
|
||||
// 1.d silu and mul
|
||||
const int64_t offset = offsets[mb];
|
||||
silu_and_mul<scalar_t, BLOCK_N>(ic1 + offset * N + nb * BLOCK_N, C0, C1, m_size, N);
|
||||
} else {
|
||||
// fused 1.bcd: silu_and_mul(A @ B0, A @ B1)
|
||||
const int64_t offset = offsets[mb];
|
||||
tinygemm_kernel(
|
||||
/* A */ A,
|
||||
/* B0 */ B0,
|
||||
/* B1 */ B1,
|
||||
/* C */ ic1 + offset * N + nb * BLOCK_N,
|
||||
/* M */ m_size,
|
||||
/* N */ n_size,
|
||||
/* K */ K,
|
||||
/* lda */ K,
|
||||
/* ldb */ n_size,
|
||||
/* ldc */ N);
|
||||
if (act_func == CPUActMethod::swiglu) {
|
||||
tinygemm_kernel(
|
||||
/* A */ A,
|
||||
/* B */ B0,
|
||||
/* C */ C0,
|
||||
/* M */ m_size,
|
||||
/* N */ n_size,
|
||||
/* K */ K,
|
||||
/* lda */ K,
|
||||
/* ldb */ n_size,
|
||||
/* ldc */ BLOCK_N);
|
||||
tinygemm_kernel(
|
||||
/* A */ A,
|
||||
/* B */ B1,
|
||||
/* C */ C1,
|
||||
/* M */ m_size,
|
||||
/* N */ n_size,
|
||||
/* K */ K,
|
||||
/* lda */ K,
|
||||
/* ldb */ n_size,
|
||||
/* ldc */ BLOCK_N);
|
||||
} else {
|
||||
// fused 1.bcd: silu_and_mul(A @ B0, A @ B1)
|
||||
tinygemm_kernel(
|
||||
/* A */ A,
|
||||
/* B0 */ B0,
|
||||
/* B1 */ B1,
|
||||
/* C */ ic1 + offset * N + nb * BLOCK_N,
|
||||
/* M */ m_size,
|
||||
/* N */ n_size,
|
||||
/* K */ K,
|
||||
/* lda */ K,
|
||||
/* ldb */ n_size,
|
||||
/* ldc */ N);
|
||||
}
|
||||
}
|
||||
if (with_bias) {
|
||||
for (int64_t m = 0; m < m_size; ++m) {
|
||||
add_bias_stub(C0 + m * BLOCK_N, B0_bias, n_size);
|
||||
add_bias_stub(C1 + m * BLOCK_N, B1_bias, n_size);
|
||||
}
|
||||
}
|
||||
// 1.d silu and mul
|
||||
const int64_t offset = offsets[mb];
|
||||
if (act_func == CPUActMethod::silu_and_mul && use_brgemm) {
|
||||
for (int64_t m = 0; m < m_size; ++m) {
|
||||
silu_and_mul_stub(ic1 + (offset + m) * N + nb * BLOCK_N, C0 + m * BLOCK_N, C1 + m * BLOCK_N, BLOCK_N);
|
||||
}
|
||||
} else if (act_func == CPUActMethod::swiglu) {
|
||||
for (int64_t m = 0; m < m_size; ++m) {
|
||||
scalar_t* __restrict__ ic1_row = ic1 + (offset + m) * N;
|
||||
clamp_sigmoid_and_mul_stub(ic1_row + nb * BLOCK_N / 2, C0 + m * BLOCK_N, BLOCK_N / 2, alpha, limit);
|
||||
clamp_sigmoid_and_mul_stub(ic1_row + N / 2 + nb * BLOCK_N / 2, C1 + m * BLOCK_N, BLOCK_N / 2, alpha, limit);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
@@ -591,6 +592,7 @@ void fused_experts_kernel_impl(
|
||||
// B shape [IC, n_size] in vnni format
|
||||
int32_t expert_id = expert_ids[mb];
|
||||
const scalar_t* __restrict__ B = packed_w2 + expert_id * stride_e2 + nb * BLOCK_N * stride_oc;
|
||||
const float* __restrict__ B_bias = w2_bias + expert_id * OC + nb * BLOCK_N;
|
||||
|
||||
// 2.a gemm: C = A @ B
|
||||
if (use_brgemm) {
|
||||
@@ -618,6 +620,11 @@ void fused_experts_kernel_impl(
|
||||
/* ldc */ BLOCK_N);
|
||||
}
|
||||
|
||||
if (with_bias) {
|
||||
for (int64_t m = 0; m < m_size; ++m) {
|
||||
add_bias_stub(C + m * BLOCK_N, B_bias, n_size);
|
||||
}
|
||||
}
|
||||
// 2.b copy from C to ic2 in original order
|
||||
// and also mul topk_weights in float32
|
||||
for (int64_t m = 0; m < m_size; ++m) {
|
||||
@@ -717,7 +724,9 @@ void shared_expert_kernel_impl(
|
||||
/* C */ C1);
|
||||
|
||||
// 1.d silu and mul
|
||||
silu_and_mul<scalar_t, BLOCK_N>(ic1 + mb * BLOCK_M * N + nb * BLOCK_N, C0, C1, m_size, N);
|
||||
for (int64_t m = 0; m < m_size; ++m) {
|
||||
silu_and_mul_stub(ic1 + (mb * BLOCK_M + m) * N + nb * BLOCK_N, C0 + m * BLOCK_N, C1 + m * BLOCK_N, BLOCK_N);
|
||||
}
|
||||
} else {
|
||||
// fused 1.bcd: silu_and_mul(A @ B0, A @ B1)
|
||||
tinygemm_kernel(
|
||||
@@ -809,24 +818,20 @@ void shared_expert_kernel_impl(
|
||||
} // anonymous namespace
|
||||
|
||||
// common checks
|
||||
template <CPUQuantMethod quant>
|
||||
static inline void check_moe_scales(
|
||||
bool use_int8_w8a8,
|
||||
bool use_fp8_w8a16,
|
||||
bool use_mxfp4,
|
||||
const std::optional<at::Tensor>& w1_scale,
|
||||
const std::optional<at::Tensor>& w2_scale,
|
||||
const std::optional<std::vector<int64_t>> block_size) {
|
||||
if (use_int8_w8a8) {
|
||||
if constexpr (quant == CPUQuantMethod::INT8_W8A8) {
|
||||
TORCH_CHECK(w1_scale.has_value(), "missing w1_scale for int8 w8a8.");
|
||||
TORCH_CHECK(w2_scale.has_value(), "missing w2_scale for int8 w8a8.");
|
||||
}
|
||||
if (use_fp8_w8a16) {
|
||||
} else if constexpr (quant == CPUQuantMethod::FP8_W8A16) {
|
||||
TORCH_CHECK(w1_scale.has_value(), "missing w1_scale for fp8 w8a16.");
|
||||
TORCH_CHECK(w2_scale.has_value(), "missing w2_scale for fp8 w8a16.");
|
||||
TORCH_CHECK(block_size.has_value(), "missing block_size for fp8 w8a16.");
|
||||
TORCH_CHECK(block_size.value().size() == 2, "expect block_size.size() to be 2.");
|
||||
}
|
||||
if (use_mxfp4) {
|
||||
} else if constexpr (quant == CPUQuantMethod::MXFP4) {
|
||||
TORCH_CHECK(w1_scale.has_value(), "missing w1_scale for mxfp4.");
|
||||
TORCH_CHECK(w2_scale.has_value(), "missing w2_scale for mxfp4.");
|
||||
TORCH_CHECK(w1_scale.value().scalar_type() == at::kByte, "expect w1_scale to be uint8.");
|
||||
@@ -834,6 +839,20 @@ static inline void check_moe_scales(
|
||||
}
|
||||
}
|
||||
|
||||
static inline void check_moe_scales(
|
||||
int64_t moe_comp_method,
|
||||
const std::optional<at::Tensor>& w1_scale,
|
||||
const std::optional<at::Tensor>& w2_scale,
|
||||
const std::optional<std::vector<int64_t>> block_size) {
|
||||
if (moe_comp_method == CPUQuantMethod::INT8_W8A8) {
|
||||
check_moe_scales<CPUQuantMethod::INT8_W8A8>(w1_scale, w2_scale, block_size);
|
||||
} else if (moe_comp_method == CPUQuantMethod::FP8_W8A16) {
|
||||
check_moe_scales<CPUQuantMethod::FP8_W8A16>(w1_scale, w2_scale, block_size);
|
||||
} else if (moe_comp_method == CPUQuantMethod::MXFP4) {
|
||||
check_moe_scales<CPUQuantMethod::MXFP4>(w1_scale, w2_scale, block_size);
|
||||
}
|
||||
}
|
||||
|
||||
#define CHECK_MOE_SCALES_FP8(DIM0, DIM1) \
|
||||
auto w1s = w1_scale.value(); \
|
||||
auto w2s = w2_scale.value(); \
|
||||
@@ -877,10 +896,14 @@ at::Tensor fused_experts_cpu(
|
||||
constexpr int64_t BLOCK_N = block_size_n();
|
||||
|
||||
const auto st = hidden_states.scalar_type();
|
||||
// TODO: fused_topk_torch_native (CPU fallback for models like MiniMax)
|
||||
// returns int64 topk_ids; fused_experts_cpu requires int32. Remove the typecast after topk kernel is provided
|
||||
auto topk_ids_ = topk_ids.scalar_type() == at::kInt ? topk_ids : topk_ids.to(at::kInt);
|
||||
|
||||
CHECK_INPUT(hidden_states);
|
||||
CHECK_INPUT(w1);
|
||||
CHECK_INPUT(w2);
|
||||
CHECK_EQ(topk_weights.sizes(), topk_ids.sizes());
|
||||
CHECK_EQ(topk_weights.sizes(), topk_ids_.sizes());
|
||||
CHECK_DIM(2, hidden_states);
|
||||
if (moe_comp_method == CPUQuantMethod::INT4_W4A8 && is_vnni) {
|
||||
CHECK_DIM(4, w1);
|
||||
@@ -890,9 +913,8 @@ at::Tensor fused_experts_cpu(
|
||||
CHECK_DIM(3, w2);
|
||||
}
|
||||
CHECK_DIM(2, topk_weights);
|
||||
CHECK_DIM(2, topk_ids);
|
||||
|
||||
CHECK_EQ(topk_ids.scalar_type(), at::kInt);
|
||||
CHECK_DIM(2, topk_ids_);
|
||||
CHECK_EQ(topk_ids_.scalar_type(), at::kInt);
|
||||
|
||||
// TODO: support topk_weights to be bf16 or fp16 in the kernel.
|
||||
// The topk_weights of llama4 is computed via Llama4MoE:custom_routing_function and is bf16/fp16
|
||||
@@ -908,12 +930,8 @@ at::Tensor fused_experts_cpu(
|
||||
int64_t topk = topk_weights_.size(1);
|
||||
|
||||
// we use int32_t compensation for int8 w8a8
|
||||
int64_t packed_K = moe_comp_method == CPUQuantMethod::MXFP4
|
||||
? get_row_size<uint8_t>(K)
|
||||
: get_row_size(K, moe_comp_method == CPUQuantMethod::INT8_W8A8);
|
||||
int64_t packed_N = moe_comp_method == CPUQuantMethod::MXFP4
|
||||
? get_row_size<uint8_t>(N)
|
||||
: get_row_size(N, moe_comp_method == CPUQuantMethod::INT8_W8A8);
|
||||
int64_t packed_K = get_row_size(static_cast<CPUQuantMethod>(moe_comp_method), K);
|
||||
int64_t packed_N = get_row_size(static_cast<CPUQuantMethod>(moe_comp_method), N);
|
||||
|
||||
// check weight shapes
|
||||
CHECK_EQ(w2.size(0), E);
|
||||
@@ -923,13 +941,7 @@ at::Tensor fused_experts_cpu(
|
||||
CHECK_EQ(packed_w2.size(2), packed_N / (moe_comp_method == CPUQuantMethod::INT4_W4A8 ? 2 : 1));
|
||||
}
|
||||
// check scales
|
||||
check_moe_scales(
|
||||
moe_comp_method == CPUQuantMethod::INT8_W8A8,
|
||||
moe_comp_method == CPUQuantMethod::FP8_W8A16,
|
||||
moe_comp_method == CPUQuantMethod::MXFP4,
|
||||
w1_scale,
|
||||
w2_scale,
|
||||
block_size);
|
||||
check_moe_scales(moe_comp_method, w1_scale, w2_scale, block_size);
|
||||
|
||||
at::Tensor out_hidden_states = inplace ? hidden_states : at::empty_like(hidden_states);
|
||||
|
||||
@@ -945,7 +957,7 @@ at::Tensor fused_experts_cpu(
|
||||
int64_t max_num_blocks = div_up(max_num_tokens_padded, BLOCK_M);
|
||||
auto buffer = at::empty(
|
||||
{max_num_tokens_padded + max_num_blocks + (num_threads + 1) * E + (E + 1) + (max_num_blocks + 1)},
|
||||
topk_ids.options());
|
||||
topk_ids_.options());
|
||||
|
||||
int32_t* __restrict__ sorted_ids = buffer.data_ptr<int32_t>();
|
||||
int32_t* __restrict__ expert_ids = sorted_ids + max_num_tokens_padded;
|
||||
@@ -969,7 +981,7 @@ at::Tensor fused_experts_cpu(
|
||||
|
||||
// align experts index
|
||||
int64_t num_tokens_post_pad = moe_align_block_size<BLOCK_M>(
|
||||
sorted_ids, expert_ids, topk_ids.data_ptr<int32_t>(), total_cnts, cumsums, offsets, E, numel, num_threads);
|
||||
sorted_ids, expert_ids, topk_ids_.data_ptr<int32_t>(), total_cnts, cumsums, offsets, E, numel, num_threads);
|
||||
|
||||
// unlike triton kernel, we fuse silu with gemm1 so only need 2 intermediate_caches:
|
||||
// 1. intermediate_cache1 : [M * topk, N]
|
||||
@@ -1048,7 +1060,7 @@ at::Tensor fused_experts_cpu(
|
||||
scalar_t* __restrict__ intermediate_cache0 = (scalar_t*)((void*)(C_tmp + num_threads * 2 * BLOCK_M * BLOCK_N));
|
||||
scalar_t* __restrict__ B_tmp = (scalar_t*)((void*)(intermediate_cache0 + M * topk * 2 * N));
|
||||
bool with_bias = w1_bias.has_value();
|
||||
auto act_func = alpha.has_value() && limit.has_value() ? CPUAcTMethod::swiglu : CPUAcTMethod::silu_and_mul;
|
||||
auto act_func = alpha.has_value() && limit.has_value() ? CPUActMethod::swiglu : CPUActMethod::silu_and_mul;
|
||||
|
||||
CHECK_MOE_SCALES_FP8(1, 2);
|
||||
fused_experts_fp_kernel_impl<scalar_t, at::Float8_e4m3fn, float, false>(
|
||||
@@ -1088,7 +1100,7 @@ at::Tensor fused_experts_cpu(
|
||||
scalar_t* __restrict__ intermediate_cache0 = (scalar_t*)((void*)(C_tmp + num_threads * 2 * BLOCK_M * BLOCK_N));
|
||||
scalar_t* __restrict__ B_tmp = (scalar_t*)((void*)(intermediate_cache0 + M * topk * 2 * N));
|
||||
bool with_bias = w1_bias.has_value();
|
||||
auto act_func = alpha.has_value() && limit.has_value() ? CPUAcTMethod::swiglu : CPUAcTMethod::silu_and_mul;
|
||||
auto act_func = alpha.has_value() && limit.has_value() ? CPUActMethod::swiglu : CPUActMethod::silu_and_mul;
|
||||
|
||||
// mxfp4 supports only group size of 32 (2^5)
|
||||
constexpr int64_t group_size = 32;
|
||||
@@ -1172,6 +1184,8 @@ at::Tensor fused_experts_cpu(
|
||||
} else {
|
||||
scalar_t* __restrict__ A_tmp = intermediate_cache2 + M * topk * K;
|
||||
float* __restrict__ C_tmp = (float*)((void*)(A_tmp + num_threads * BLOCK_M * K));
|
||||
bool with_bias = w1_bias.has_value();
|
||||
auto act_func = alpha.has_value() && limit.has_value() ? CPUActMethod::swiglu : CPUActMethod::silu_and_mul;
|
||||
|
||||
fused_experts_kernel_impl<scalar_t>(
|
||||
out_hidden_states.data_ptr<scalar_t>(),
|
||||
@@ -1182,6 +1196,8 @@ at::Tensor fused_experts_cpu(
|
||||
hidden_states.data_ptr<scalar_t>(),
|
||||
packed_w1.data_ptr<scalar_t>(),
|
||||
packed_w2.data_ptr<scalar_t>(),
|
||||
with_bias ? w1_bias.value().data_ptr<float>() : nullptr,
|
||||
with_bias ? w2_bias.value().data_ptr<float>() : nullptr,
|
||||
topk_weights_.data_ptr<float>(),
|
||||
sorted_ids,
|
||||
expert_ids,
|
||||
@@ -1191,7 +1207,11 @@ at::Tensor fused_experts_cpu(
|
||||
K,
|
||||
E,
|
||||
topk,
|
||||
num_tokens_post_pad);
|
||||
num_tokens_post_pad,
|
||||
alpha.has_value() ? float(alpha.value()) : 0,
|
||||
limit.has_value() ? float(limit.value()) : 0,
|
||||
act_func,
|
||||
with_bias);
|
||||
}
|
||||
});
|
||||
return out_hidden_states;
|
||||
@@ -1254,7 +1274,11 @@ at::Tensor shared_expert_cpu(
|
||||
CHECK_EQ(packed_w2.size(1), packed_N);
|
||||
|
||||
// check scales
|
||||
check_moe_scales(use_int8_w8a8, use_fp8_w8a16, false, w1_scale, w2_scale, block_size);
|
||||
if (use_int8_w8a8) {
|
||||
check_moe_scales<CPUQuantMethod::INT8_W8A8>(w1_scale, w2_scale, block_size);
|
||||
} else if (use_fp8_w8a16) {
|
||||
check_moe_scales<CPUQuantMethod::FP8_W8A16>(w1_scale, w2_scale, block_size);
|
||||
}
|
||||
|
||||
at::Tensor out_hidden_states = inplace ? hidden_states : at::empty_like(hidden_states);
|
||||
|
||||
|
||||
+41
-82
@@ -64,9 +64,7 @@ inline void copy_mul_stub(scalar_t* __restrict__ out, const input_t* __restrict_
|
||||
#pragma GCC unroll 4
|
||||
for (d = 0; d <= size - kVecSize; d += kVecSize) {
|
||||
auto [x0, x1] = load_float_vec2(input + d);
|
||||
x0 = x0 * weight_vec;
|
||||
x1 = x1 * weight_vec;
|
||||
bVec out_vec = convert_from_float_ext<scalar_t>(x0, x1);
|
||||
bVec out_vec = convert_from_float_ext<scalar_t>(x0 * weight_vec, x1 * weight_vec);
|
||||
out_vec.store(out + d);
|
||||
}
|
||||
for (; d < size; ++d) {
|
||||
@@ -91,10 +89,7 @@ inline void sum_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ in
|
||||
fVec sum_fvec0 = fVec(0.f);
|
||||
fVec sum_fvec1 = fVec(0.f);
|
||||
for (int t = 0; t < topk; ++t) {
|
||||
bVec x_bvec = bVec::loadu(input + t * K + d);
|
||||
fVec x_fvec0, x_fvec1;
|
||||
std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec);
|
||||
|
||||
auto [x_fvec0, x_fvec1] = load_float_vec2(input + t * K + d);
|
||||
sum_fvec0 += x_fvec0;
|
||||
sum_fvec1 += x_fvec1;
|
||||
}
|
||||
@@ -137,11 +132,7 @@ inline void add_mul_stub(
|
||||
#pragma GCC unroll 4
|
||||
for (d = 0; d <= size - kVecSize; d += kVecSize) {
|
||||
auto [x0, x1] = load_float_vec2(input + d);
|
||||
|
||||
bVec y_bvec = bVec::loadu(input2 + d);
|
||||
fVec y0, y1;
|
||||
std::tie(y0, y1) = at::vec::convert_to_float(y_bvec);
|
||||
|
||||
auto [y0, y1] = load_float_vec2(input2 + d);
|
||||
x0 = x0 + y0 * s_vec;
|
||||
x1 = x1 + y1 * s_vec;
|
||||
bVec out_vec = convert_from_float_ext<scalar_t>(x0, x1);
|
||||
@@ -152,31 +143,52 @@ inline void add_mul_stub(
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
template <typename scalar_t, typename input_t>
|
||||
inline void silu_and_mul_stub(
|
||||
scalar_t* __restrict__ out, const scalar_t* __restrict__ input, const scalar_t* __restrict__ input2, int64_t size) {
|
||||
scalar_t* __restrict__ out, const input_t* __restrict__ input, const input_t* __restrict__ input2, int64_t size) {
|
||||
static_assert(
|
||||
std::is_same_v<input_t, float> || std::is_same_v<input_t, scalar_t>,
|
||||
"silu_and_mul_stub only supports input_t == float or input_t == scalar_t");
|
||||
using bVec = at::vec::Vectorized<scalar_t>;
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
const fVec one = fVec(1.f);
|
||||
|
||||
// no remainder
|
||||
#pragma GCC unroll 4
|
||||
for (int64_t d = 0; d < size; d += bVec::size()) {
|
||||
bVec x = bVec::loadu(input + d);
|
||||
fVec x0, x1;
|
||||
std::tie(x0, x1) = at::vec::convert_to_float(x);
|
||||
bVec y = bVec::loadu(input2 + d);
|
||||
fVec y0, y1;
|
||||
std::tie(y0, y1) = at::vec::convert_to_float(y);
|
||||
x0 = x0 / (one + x0.neg().exp_u20());
|
||||
x1 = x1 / (one + x1.neg().exp_u20());
|
||||
x0 = x0 * y0;
|
||||
x1 = x1 * y1;
|
||||
auto [x0, x1] = load_float_vec2(input + d);
|
||||
auto [y0, y1] = load_float_vec2(input2 + d);
|
||||
x0 = fast_silu(x0) * y0;
|
||||
x1 = fast_silu(x1) * y1;
|
||||
bVec out_vec = convert_from_float_ext<scalar_t>(x0, x1);
|
||||
out_vec.store(out + d);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t, typename input_t>
|
||||
inline void clamp_sigmoid_and_mul_stub(
|
||||
scalar_t* __restrict__ out, const input_t* __restrict__ input, int64_t size, const float alpha, const float limit) {
|
||||
static_assert(
|
||||
std::is_same_v<input_t, float> || std::is_same_v<input_t, scalar_t>,
|
||||
"clamp_sigmoid_and_mul_stub only supports input_t == float or input_t == scalar_t");
|
||||
using bVec = at::vec::Vectorized<scalar_t>;
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
const fVec one = fVec(1.f);
|
||||
const fVec limit_v = fVec(limit);
|
||||
const fVec nlimit_v = fVec(-limit);
|
||||
const fVec alpha_v = fVec(alpha);
|
||||
|
||||
#pragma GCC unroll 4
|
||||
for (int64_t d = 0; d < 2 * size; d += bVec::size()) {
|
||||
auto [x0_, y0_] = load_float_vec2(input + d);
|
||||
auto [x0, y0] = at::vec::deinterleave2<float>(x0_, y0_);
|
||||
|
||||
x0 = at::vec::minimum(x0, limit_v);
|
||||
y0 = at::vec::minimum(limit_v, at::vec::maximum(nlimit_v, y0));
|
||||
x0 = fast_sigmoid_glu(x0, alpha_v) * (y0 + one);
|
||||
store_from_float_ext(out + d / 2, x0);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
inline void copy_mul_stub(scalar_t* __restrict__ out, const float* __restrict__ input, float weight, int64_t size) {
|
||||
using bVec = at::vec::Vectorized<scalar_t>;
|
||||
@@ -186,9 +198,8 @@ inline void copy_mul_stub(scalar_t* __restrict__ out, const float* __restrict__
|
||||
int64_t d;
|
||||
#pragma GCC unroll 4
|
||||
for (d = 0; d <= size - kVecSize; d += kVecSize) {
|
||||
fVec data0 = fVec::loadu(input + d) * weight_vec;
|
||||
fVec data1 = fVec::loadu(input + d + fVec::size()) * weight_vec;
|
||||
bVec out_vec = convert_from_float_ext<scalar_t>(data0, data1);
|
||||
auto [x0, x1] = load_float_vec2(input + d);
|
||||
bVec out_vec = convert_from_float_ext<scalar_t>(x0 * weight_vec, x1 * weight_vec);
|
||||
out_vec.store(out + d);
|
||||
}
|
||||
for (; d < size; ++d) {
|
||||
@@ -222,63 +233,11 @@ inline void copy_mul_stub(scalar_t* __restrict__ out, const scalar_t* __restrict
|
||||
int64_t d;
|
||||
#pragma GCC unroll 4
|
||||
for (d = 0; d <= size - kVecSize; d += kVecSize) {
|
||||
bVec x = bVec::loadu(input + d);
|
||||
fVec x0, x1;
|
||||
std::tie(x0, x1) = at::vec::convert_to_float(x);
|
||||
x0 = x0 * weight_vec;
|
||||
x1 = x1 * weight_vec;
|
||||
bVec out_vec = convert_from_float_ext<scalar_t>(x0, x1);
|
||||
auto [x0, x1] = load_float_vec2(input + d);
|
||||
bVec out_vec = convert_from_float_ext<scalar_t>(x0 * weight_vec, x1 * weight_vec);
|
||||
out_vec.store(out + d);
|
||||
}
|
||||
for (; d < size; ++d) {
|
||||
out[d] = static_cast<scalar_t>(input[d] * weight);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
inline void clamp_sigmoid_and_mul_stub(
|
||||
scalar_t* __restrict__ out,
|
||||
const scalar_t* __restrict__ input,
|
||||
int64_t size,
|
||||
const float alpha,
|
||||
const float limit) {
|
||||
using bVec = at::vec::Vectorized<scalar_t>;
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
const fVec one = fVec(1.f);
|
||||
const fVec zero = fVec(0.f);
|
||||
const fVec limit_v = fVec(limit);
|
||||
const fVec nlimit_v = fVec(-limit);
|
||||
const fVec alpha_v = fVec(alpha);
|
||||
|
||||
// no remainder
|
||||
#pragma GCC unroll 4
|
||||
for (int64_t d = 0; d < size; d += bVec::size()) {
|
||||
bVec x = bVec::loadu(input + d);
|
||||
fVec x0_, y0_;
|
||||
std::tie(x0_, y0_) = at::vec::convert_to_float(x);
|
||||
float tmp_buffer[fVec::size() * 2]; // 32
|
||||
float tmp_glu[fVec::size()]; // 16
|
||||
float tmp_linear[fVec::size()]; // 16
|
||||
x0_.store(tmp_buffer);
|
||||
y0_.store(tmp_buffer + fVec::size());
|
||||
// interleaved: x[2i] = glu, x[2i+1] = linear
|
||||
for (int j = 0; j < fVec::size(); ++j) {
|
||||
// x0 [0,2,..30]
|
||||
tmp_glu[j] = tmp_buffer[j * 2];
|
||||
// y0 [1,3,...31]
|
||||
tmp_linear[j] = tmp_buffer[j * 2 + 1];
|
||||
}
|
||||
fVec x0 = fVec::loadu(tmp_glu);
|
||||
fVec y0 = fVec::loadu(tmp_linear);
|
||||
|
||||
// clamp
|
||||
x0 = at::vec::minimum(x0, limit_v);
|
||||
y0 = at::vec::minimum(limit_v, at::vec::maximum(nlimit_v, y0));
|
||||
// x * sigmoid(x * alpha)
|
||||
x0 = x0 / (one + (x0 * alpha_v).neg().exp_u20());
|
||||
// (y + 1) * x
|
||||
y0 = y0 + one;
|
||||
x0 = x0 * y0;
|
||||
convert_from_float_and_store<scalar_t>(out + d / 2, x0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -37,7 +37,7 @@ void fused_experts_fp_kernel_impl(
|
||||
int64_t num_tokens_post_pad,
|
||||
float alpha,
|
||||
float limit,
|
||||
CPUAcTMethod act_func,
|
||||
CPUActMethod act_func,
|
||||
bool with_bias) {
|
||||
constexpr int64_t BLOCK_M = block_size_m();
|
||||
constexpr int64_t BLOCK_N = block_size_n();
|
||||
@@ -122,17 +122,17 @@ void fused_experts_fp_kernel_impl(
|
||||
});
|
||||
|
||||
// stage 1.5: intermediate_cache1 = silu(intermediate_cache0)
|
||||
if (act_func == CPUAcTMethod::silu_and_mul) {
|
||||
if (act_func == CPUActMethod::silu_and_mul) {
|
||||
at::parallel_for(0, M * topk, 0, [&](int64_t begin, int64_t end) {
|
||||
for (int64_t m = begin; m < end; ++m) {
|
||||
silu_and_mul_stub(ic1 + m * N, ic0 + m * 2 * N, ic0 + m * 2 * N + N, N);
|
||||
}
|
||||
});
|
||||
} else if (act_func == CPUAcTMethod::swiglu) {
|
||||
} else if (act_func == CPUActMethod::swiglu) {
|
||||
at::parallel_for(0, M * topk, 0, [&](int64_t begin, int64_t end) {
|
||||
for (int64_t m = begin; m < end; ++m) {
|
||||
clamp_sigmoid_and_mul_stub(ic1 + m * N, ic0 + m * 2 * N, N, alpha, limit);
|
||||
clamp_sigmoid_and_mul_stub(ic1 + m * N + N / 2, ic0 + m * 2 * N + N, N, alpha, limit);
|
||||
clamp_sigmoid_and_mul_stub(ic1 + m * N, ic0 + m * 2 * N, N / 2, alpha, limit);
|
||||
clamp_sigmoid_and_mul_stub(ic1 + m * N + N / 2, ic0 + m * 2 * N + N, N / 2, alpha, limit);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -243,7 +243,7 @@ void fused_experts_fp_kernel_impl(
|
||||
int64_t num_tokens_post_pad, \
|
||||
float alpha, \
|
||||
float limit, \
|
||||
CPUAcTMethod act_func, \
|
||||
CPUActMethod act_func, \
|
||||
bool with_bias)
|
||||
|
||||
INSTANTIATE_MOE_FP_TEMPLATE(at::BFloat16, at::Float8_e4m3fn, float, false);
|
||||
|
||||
@@ -53,16 +53,14 @@ inline void silu_and_mul(
|
||||
vc1[col] = _mm512_mul_ps(_mm512_mul_ps(vc1[col], vas), vbs1[col]);
|
||||
};
|
||||
|
||||
using bVec = at::vec::Vectorized<scalar_t>;
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
const fVec one = fVec(1.f);
|
||||
auto silu_and_mul = [&](auto col) {
|
||||
fVec x = fVec(vc0[col]);
|
||||
fVec y = fVec(vc1[col]);
|
||||
x = x / (one + x.neg().exp_u20());
|
||||
vc0[col] = x * y;
|
||||
__m512 x = vc0[col];
|
||||
__m512 y = vc1[col];
|
||||
vc0[col] = _mm512_mul_ps(_mm512_rcp14_silu_ps(x), y);
|
||||
};
|
||||
|
||||
using bVec = at::vec::Vectorized<scalar_t>;
|
||||
using fVec = at::vec::Vectorized<float>;
|
||||
auto storec = [&](auto col, int64_t m) {
|
||||
if constexpr (col % 2 == 0) {
|
||||
fVec x0 = fVec(vc0[col + 0]);
|
||||
@@ -229,23 +227,17 @@ struct tinygemm_kernel_vnni<at::BFloat16, BLOCK_M, BLOCK_N> {
|
||||
};
|
||||
Unroll<ROWS * COLS>{}(scalec);
|
||||
|
||||
using Vec = at::vec::Vectorized<float>;
|
||||
const Vec one = Vec(1.f);
|
||||
auto storec = [&](auto i) {
|
||||
constexpr int row = i / COLS;
|
||||
constexpr int col = i % COLS;
|
||||
// for COLS = 2, 4 use 512bit store
|
||||
if constexpr (col % 2 == 0) {
|
||||
Vec x0 = _mm512_castsi512_ps(vc0[row * COLS + col + 0]);
|
||||
Vec x1 = _mm512_castsi512_ps(vc0[row * COLS + col + 1]);
|
||||
Vec y0 = _mm512_castsi512_ps(vc1[row * COLS + col + 0]);
|
||||
Vec y1 = _mm512_castsi512_ps(vc1[row * COLS + col + 1]);
|
||||
// silu
|
||||
x0 = x0 / (one + x0.neg().exp_u20());
|
||||
x1 = x1 / (one + x1.neg().exp_u20());
|
||||
// mul
|
||||
x0 = x0 * y0;
|
||||
x1 = x1 * y1;
|
||||
__m512 x0 = _mm512_castsi512_ps(vc0[row * COLS + col + 0]);
|
||||
__m512 x1 = _mm512_castsi512_ps(vc0[row * COLS + col + 1]);
|
||||
__m512 y0 = _mm512_castsi512_ps(vc1[row * COLS + col + 0]);
|
||||
__m512 y1 = _mm512_castsi512_ps(vc1[row * COLS + col + 1]);
|
||||
x0 = _mm512_mul_ps(_mm512_rcp14_silu_ps(x0), y0);
|
||||
x1 = _mm512_mul_ps(_mm512_rcp14_silu_ps(x1), y1);
|
||||
|
||||
_mm512_storeu_si512(
|
||||
reinterpret_cast<__m512i*>((C + row * ldc + col * 16)),
|
||||
|
||||
+195
-6
@@ -32,11 +32,11 @@ inline Vectorized<scalar_t> convert_from_float_ext(const Vectorized<float>& a, c
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
inline void convert_from_float_and_store(scalar_t* out, const Vectorized<float>& a) {
|
||||
float out_buffer[at::vec::Vectorized<float>::size()];
|
||||
inline void store_from_float_ext(scalar_t* out, const Vectorized<float>& a) {
|
||||
float out_buffer[Vectorized<float>::size()];
|
||||
a.store(out_buffer);
|
||||
for (int i = 0; i < 16; i++) {
|
||||
out[i] = (scalar_t)out_buffer[i];
|
||||
for (int i = 0; i < Vectorized<float>::size(); ++i) {
|
||||
out[i] = static_cast<scalar_t>(out_buffer[i]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -59,6 +59,17 @@ inline std::tuple<Vectorized<float>, Vectorized<float>> load_float_vec2(const fl
|
||||
return std::make_tuple(x0, x1);
|
||||
}
|
||||
|
||||
template <typename scalar_t, typename std::enable_if_t<is_reduced_floating_point_v<scalar_t>, int> = 1>
|
||||
inline at::vec::Vectorized<float> load_float_vec(const scalar_t* __restrict__ data) {
|
||||
at::vec::Vectorized<float> out;
|
||||
if constexpr (std::is_same_v<scalar_t, at::BFloat16>) {
|
||||
at::vec::load_fp32_from_bf16(data, out);
|
||||
} else {
|
||||
at::vec::load_fp32_from_fp16(data, out);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
#if defined(CPU_CAPABILITY_AVX512)
|
||||
|
||||
// `at::vec::convert_from_float<>` from PyTorch doesn't have avx512-bf16 intrinsics
|
||||
@@ -70,8 +81,14 @@ convert_from_float_ext<at::BFloat16>(const Vectorized<float>& a, const Vectorize
|
||||
}
|
||||
|
||||
template <>
|
||||
inline void convert_from_float_and_store<at::BFloat16>(at::BFloat16* out, const Vectorized<float>& a) {
|
||||
_mm256_storeu_si256((__m256i*)out, (__m256i)(_mm512_cvtneps_pbh(__m512(a))));
|
||||
inline void store_from_float_ext<at::BFloat16>(at::BFloat16* out, const Vectorized<float>& a) {
|
||||
_mm256_storeu_si256(reinterpret_cast<__m256i*>(out), (__m256i)(_mm512_cvtneps_pbh(__m512(a))));
|
||||
}
|
||||
|
||||
template <>
|
||||
inline void store_from_float_ext<at::Half>(at::Half* out, const Vectorized<float>& a) {
|
||||
_mm256_storeu_si256(
|
||||
reinterpret_cast<__m256i*>(out), _mm512_cvtps_ph(__m512(a), _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC));
|
||||
}
|
||||
|
||||
#define CVT_BF16_TO_FP32(a) _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(a), 16))
|
||||
@@ -301,6 +318,77 @@ inline void quantize_row_int8<at::BFloat16>(
|
||||
// transpose utils
|
||||
// taken from my PR in ggml: https://github.com/ggml-org/llama.cpp/pull/8998
|
||||
#if defined(CPU_CAPABILITY_AVX512)
|
||||
inline void transpose_16x16_16bit(__m256i* v) {
|
||||
__m256i v1[16];
|
||||
v1[0] = _mm256_unpacklo_epi16(v[0], v[1]);
|
||||
v1[1] = _mm256_unpackhi_epi16(v[0], v[1]);
|
||||
v1[2] = _mm256_unpacklo_epi16(v[2], v[3]);
|
||||
v1[3] = _mm256_unpackhi_epi16(v[2], v[3]);
|
||||
v1[4] = _mm256_unpacklo_epi16(v[4], v[5]);
|
||||
v1[5] = _mm256_unpackhi_epi16(v[4], v[5]);
|
||||
v1[6] = _mm256_unpacklo_epi16(v[6], v[7]);
|
||||
v1[7] = _mm256_unpackhi_epi16(v[6], v[7]);
|
||||
v1[8] = _mm256_unpacklo_epi16(v[8], v[9]);
|
||||
v1[9] = _mm256_unpackhi_epi16(v[8], v[9]);
|
||||
v1[10] = _mm256_unpacklo_epi16(v[10], v[11]);
|
||||
v1[11] = _mm256_unpackhi_epi16(v[10], v[11]);
|
||||
v1[12] = _mm256_unpacklo_epi16(v[12], v[13]);
|
||||
v1[13] = _mm256_unpackhi_epi16(v[12], v[13]);
|
||||
v1[14] = _mm256_unpacklo_epi16(v[14], v[15]);
|
||||
v1[15] = _mm256_unpackhi_epi16(v[14], v[15]);
|
||||
|
||||
v[0] = _mm256_unpacklo_epi32(v1[0], v1[2]);
|
||||
v[1] = _mm256_unpackhi_epi32(v1[0], v1[2]);
|
||||
v[2] = _mm256_unpacklo_epi32(v1[1], v1[3]);
|
||||
v[3] = _mm256_unpackhi_epi32(v1[1], v1[3]);
|
||||
v[4] = _mm256_unpacklo_epi32(v1[4], v1[6]);
|
||||
v[5] = _mm256_unpackhi_epi32(v1[4], v1[6]);
|
||||
v[6] = _mm256_unpacklo_epi32(v1[5], v1[7]);
|
||||
v[7] = _mm256_unpackhi_epi32(v1[5], v1[7]);
|
||||
v[8] = _mm256_unpacklo_epi32(v1[8], v1[10]);
|
||||
v[9] = _mm256_unpackhi_epi32(v1[8], v1[10]);
|
||||
v[10] = _mm256_unpacklo_epi32(v1[9], v1[11]);
|
||||
v[11] = _mm256_unpackhi_epi32(v1[9], v1[11]);
|
||||
v[12] = _mm256_unpacklo_epi32(v1[12], v1[14]);
|
||||
v[13] = _mm256_unpackhi_epi32(v1[12], v1[14]);
|
||||
v[14] = _mm256_unpacklo_epi32(v1[13], v1[15]);
|
||||
v[15] = _mm256_unpackhi_epi32(v1[13], v1[15]);
|
||||
|
||||
v1[0] = _mm256_unpacklo_epi64(v[0], v[4]);
|
||||
v1[1] = _mm256_unpackhi_epi64(v[0], v[4]);
|
||||
v1[2] = _mm256_unpacklo_epi64(v[1], v[5]);
|
||||
v1[3] = _mm256_unpackhi_epi64(v[1], v[5]);
|
||||
v1[4] = _mm256_unpacklo_epi64(v[2], v[6]);
|
||||
v1[5] = _mm256_unpackhi_epi64(v[2], v[6]);
|
||||
v1[6] = _mm256_unpacklo_epi64(v[3], v[7]);
|
||||
v1[7] = _mm256_unpackhi_epi64(v[3], v[7]);
|
||||
v1[8] = _mm256_unpacklo_epi64(v[8], v[12]);
|
||||
v1[9] = _mm256_unpackhi_epi64(v[8], v[12]);
|
||||
v1[10] = _mm256_unpacklo_epi64(v[9], v[13]);
|
||||
v1[11] = _mm256_unpackhi_epi64(v[9], v[13]);
|
||||
v1[12] = _mm256_unpacklo_epi64(v[10], v[14]);
|
||||
v1[13] = _mm256_unpackhi_epi64(v[10], v[14]);
|
||||
v1[14] = _mm256_unpacklo_epi64(v[11], v[15]);
|
||||
v1[15] = _mm256_unpackhi_epi64(v[11], v[15]);
|
||||
|
||||
v[0] = _mm256_permute2x128_si256(v1[0], v1[8], 0x20);
|
||||
v[1] = _mm256_permute2x128_si256(v1[1], v1[9], 0x20);
|
||||
v[2] = _mm256_permute2x128_si256(v1[2], v1[10], 0x20);
|
||||
v[3] = _mm256_permute2x128_si256(v1[3], v1[11], 0x20);
|
||||
v[4] = _mm256_permute2x128_si256(v1[4], v1[12], 0x20);
|
||||
v[5] = _mm256_permute2x128_si256(v1[5], v1[13], 0x20);
|
||||
v[6] = _mm256_permute2x128_si256(v1[6], v1[14], 0x20);
|
||||
v[7] = _mm256_permute2x128_si256(v1[7], v1[15], 0x20);
|
||||
v[8] = _mm256_permute2x128_si256(v1[0], v1[8], 0x31);
|
||||
v[9] = _mm256_permute2x128_si256(v1[1], v1[9], 0x31);
|
||||
v[10] = _mm256_permute2x128_si256(v1[2], v1[10], 0x31);
|
||||
v[11] = _mm256_permute2x128_si256(v1[3], v1[11], 0x31);
|
||||
v[12] = _mm256_permute2x128_si256(v1[4], v1[12], 0x31);
|
||||
v[13] = _mm256_permute2x128_si256(v1[5], v1[13], 0x31);
|
||||
v[14] = _mm256_permute2x128_si256(v1[6], v1[14], 0x31);
|
||||
v[15] = _mm256_permute2x128_si256(v1[7], v1[15], 0x31);
|
||||
}
|
||||
|
||||
inline void transpose_16x16_32bit(__m512i* v) {
|
||||
__m512i v1[16];
|
||||
v1[0] = _mm512_unpacklo_epi32(v[0], v[1]);
|
||||
@@ -394,6 +482,62 @@ inline std::tuple<__m512i, __m512i> transpose_2x32_16bit(__m512i r0, __m512i r1)
|
||||
}
|
||||
#pragma GCC diagnostic pop
|
||||
|
||||
// Note: mapped from aten exp_u20
|
||||
inline __attribute__((always_inline)) __m512 _mm512_exp_u20_ps(const __m512 values) {
|
||||
const __m512 vec_factorial_1 = _mm512_set1_ps(0.999999701f);
|
||||
const __m512 vec_factorial_2 = _mm512_set1_ps(0.499991506f);
|
||||
const __m512 vec_factorial_3 = _mm512_set1_ps(0.166676521f);
|
||||
const __m512 vec_factorial_4 = _mm512_set1_ps(0.0418978221f);
|
||||
const __m512 vec_factorial_5 = _mm512_set1_ps(0.00828929059f);
|
||||
const __m512 vec_exp_log2ef = _mm512_castsi512_ps(_mm512_set1_epi32(0x3fb8aa3b)); // log2(e)
|
||||
const __m512 vec_half = _mm512_set1_ps(0.5f);
|
||||
const __m512 vec_one = _mm512_set1_ps(1.f);
|
||||
const __m512 vec_zero = _mm512_set1_ps(0.f);
|
||||
const __m512 vec_two = _mm512_set1_ps(2.f);
|
||||
const __m512 vec_ln2f = _mm512_castsi512_ps(_mm512_set1_epi32(0x3f317218));
|
||||
const __m512 vec_ln_flt_min = _mm512_castsi512_ps(_mm512_set1_epi32(0xc2aeac50));
|
||||
const __m512 vec_ln_flt_max = _mm512_castsi512_ps(_mm512_set1_epi32(0x42b17218));
|
||||
const __m512i vec_127 = _mm512_set1_epi32(0x0000007f);
|
||||
const int n_mantissa_bits = 23;
|
||||
|
||||
// exp(x) =
|
||||
// = exp(n * ln(2) + r) // divide x by ln(2) and get quot and rem
|
||||
// = 2^n * exp(r) // simplify the exp(n*ln(2)) expression
|
||||
|
||||
auto less_ln_flt_min_mask = _mm512_cmp_ps_mask(values, vec_ln_flt_min, 1 /*_CMP_LT_OS*/);
|
||||
auto vec_src = _mm512_min_ps(values, vec_ln_flt_max);
|
||||
vec_src = _mm512_max_ps(vec_src, vec_ln_flt_min);
|
||||
|
||||
// fx = floorf(x * log2ef + 0.5)
|
||||
auto vec_fx = _mm512_fmadd_ps(vec_src, vec_exp_log2ef, vec_half);
|
||||
auto vec_fx_i = _mm512_cvt_roundps_epi32(vec_fx, _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC);
|
||||
vec_fx = _mm512_cvtepi32_ps(vec_fx_i);
|
||||
|
||||
// x = x - fx * ln2
|
||||
auto vec_exp_poly = _mm512_fnmadd_ps(vec_fx, vec_ln2f, vec_src);
|
||||
|
||||
// compute polynomial
|
||||
auto vec_res = _mm512_fmadd_ps(vec_exp_poly, vec_factorial_5, vec_factorial_4);
|
||||
vec_res = _mm512_fmadd_ps(vec_exp_poly, vec_res, vec_factorial_3);
|
||||
vec_res = _mm512_fmadd_ps(vec_exp_poly, vec_res, vec_factorial_2);
|
||||
vec_res = _mm512_fmadd_ps(vec_exp_poly, vec_res, vec_factorial_1);
|
||||
vec_res = _mm512_fmadd_ps(vec_exp_poly, vec_res, vec_one);
|
||||
|
||||
// compute 2^(n-1)
|
||||
auto vec_exp_number = _mm512_sub_ps(vec_fx, vec_one);
|
||||
auto vec_exp_number_i = _mm512_cvtps_epi32(vec_exp_number);
|
||||
auto vec_two_pow_n_i = _mm512_add_epi32(vec_exp_number_i, vec_127);
|
||||
vec_two_pow_n_i = _mm512_slli_epi32(vec_two_pow_n_i, n_mantissa_bits);
|
||||
auto vec_two_pow_n = _mm512_castsi512_ps(vec_two_pow_n_i);
|
||||
vec_two_pow_n = _mm512_mask_blend_ps(less_ln_flt_min_mask, vec_two_pow_n, vec_zero);
|
||||
|
||||
// y = y * 2^n
|
||||
vec_res = _mm512_mul_ps(vec_res, vec_two_pow_n);
|
||||
vec_res = _mm512_mul_ps(vec_res, vec_two);
|
||||
return vec_res;
|
||||
}
|
||||
|
||||
// Note: mapped from aten fexp_u20
|
||||
inline __attribute__((always_inline)) __m512 _mm512_fexp_u20_ps(const __m512 values) {
|
||||
const __m512 vec_c0 = _mm512_set1_ps(0.00010703434948458272f);
|
||||
const __m512 vec_c1 = _mm512_set1_ps(0.30354260500649682f);
|
||||
@@ -444,6 +588,51 @@ inline __attribute__((always_inline)) __m512 _mm512_fexp_u20_ps(const __m512 val
|
||||
// final interpretation to float
|
||||
return _mm512_castsi512_ps(casted_integer);
|
||||
}
|
||||
|
||||
// sigmoid(x) = 1 / (1 + exp(-x)); avoid vdivps via rcp14
|
||||
inline __attribute__((always_inline)) __m512 _mm512_rcp14_sigmoid_ps(__m512 x) {
|
||||
__m512 minus_x = _mm512_xor_ps(_mm512_set1_ps(-0.f), x);
|
||||
__m512 denom = _mm512_add_ps(_mm512_exp_u20_ps(minus_x), _mm512_set1_ps(1.f));
|
||||
return _mm512_rcp14_ps(denom);
|
||||
}
|
||||
|
||||
// SiLU(x) = x * sigmoid(x)
|
||||
inline __attribute__((always_inline)) __m512 _mm512_rcp14_silu_ps(__m512 x) {
|
||||
return _mm512_mul_ps(x, _mm512_rcp14_sigmoid_ps(x));
|
||||
}
|
||||
|
||||
// x * sigmoid(x * alpha) for clamped SwiGLU
|
||||
inline __attribute__((always_inline)) __m512 _mm512_rcp14_sigmoid_glu_ps(__m512 x, __m512 alpha) {
|
||||
__m512 xa = _mm512_mul_ps(x, alpha);
|
||||
return _mm512_mul_ps(x, _mm512_rcp14_sigmoid_ps(xa));
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
inline at::vec::Vectorized<float> fast_sigmoid(const at::vec::Vectorized<float>& x) {
|
||||
#if defined(CPU_CAPABILITY_AVX512)
|
||||
return at::vec::Vectorized<float>(_mm512_rcp14_sigmoid_ps(x));
|
||||
#else
|
||||
const auto one = at::vec::Vectorized<float>(1.f);
|
||||
return one / (one + x.neg().exp_u20());
|
||||
#endif
|
||||
}
|
||||
|
||||
inline at::vec::Vectorized<float> fast_silu(const at::vec::Vectorized<float>& x) {
|
||||
#if defined(CPU_CAPABILITY_AVX512)
|
||||
return at::vec::Vectorized<float>(_mm512_rcp14_silu_ps(x));
|
||||
#else
|
||||
return x * fast_sigmoid(x);
|
||||
#endif
|
||||
}
|
||||
|
||||
inline at::vec::Vectorized<float>
|
||||
fast_sigmoid_glu(const at::vec::Vectorized<float>& x, const at::vec::Vectorized<float>& alpha) {
|
||||
#if defined(CPU_CAPABILITY_AVX512)
|
||||
return at::vec::Vectorized<float>(_mm512_rcp14_sigmoid_glu_ps(x, alpha));
|
||||
#else
|
||||
return x * fast_sigmoid(x * alpha);
|
||||
#endif
|
||||
}
|
||||
|
||||
} // anonymous namespace
|
||||
|
||||
@@ -110,7 +110,7 @@ std::tuple<at::Tensor, at::Tensor> chunk_gated_delta_rule_cpu(
|
||||
const at::Tensor& g, const at::Tensor& beta,
|
||||
const at::Tensor& initial_state, bool output_final_state,
|
||||
const at::Tensor& cu_seqlens, bool head_first, bool use_qk_l2norm_in_kernel,
|
||||
double eps = 1e-5);
|
||||
const at::Tensor& initial_state_indices, double eps = 1e-5);
|
||||
|
||||
at::Tensor fused_sigmoid_gating_delta_rule_update_cpu(
|
||||
const at::Tensor& A_log, const at::Tensor& dt_bias, const at::Tensor& q,
|
||||
@@ -546,7 +546,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
|
||||
"Tensor g, Tensor beta, "
|
||||
"Tensor initial_state, bool output_final_state, Tensor cu_seqlens, bool "
|
||||
"head_first, "
|
||||
"bool use_qk_l2norm_in_kernel, float eps=1e-5) -> (Tensor, Tensor)");
|
||||
"bool use_qk_l2norm_in_kernel, Tensor initial_state_indices, float "
|
||||
"eps=1e-5) -> (Tensor, Tensor)");
|
||||
ops.impl("chunk_gated_delta_rule_cpu", torch::kCPU,
|
||||
&chunk_gated_delta_rule_cpu);
|
||||
ops.def(
|
||||
|
||||
@@ -26,6 +26,13 @@ HEAD_DIMS = [
|
||||
(32, 32),
|
||||
(64, 32),
|
||||
]
|
||||
# chunk_gated_delta_rule_cpu (the chunked-prefill kernel) only supports
|
||||
# head_dim == head_dim_v in {64, 128}; the decode-path update kernel above
|
||||
# has no such restriction and keeps using the wider HEAD_DIMS list.
|
||||
CHUNK_HEAD_DIMS = [
|
||||
(64, 64),
|
||||
(128, 128),
|
||||
]
|
||||
CHUNK_SIZE = 64
|
||||
CONV_DIM = 128
|
||||
CONV_KERNEL = 4
|
||||
@@ -258,7 +265,7 @@ def test_fused_sigmoid_gating_delta_rule_update_cpu(
|
||||
# prefill path
|
||||
@pytest.mark.parametrize("seq_lens", PREFILL_SEQ_LENS)
|
||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||
@pytest.mark.parametrize("head_dims", HEAD_DIMS)
|
||||
@pytest.mark.parametrize("head_dims", CHUNK_HEAD_DIMS)
|
||||
@torch.inference_mode()
|
||||
def test_chunk_gated_delta_rule_cpu(
|
||||
seq_lens: list[int],
|
||||
@@ -307,6 +314,7 @@ def test_chunk_gated_delta_rule_cpu(
|
||||
cu_seqlens=cu_seqlens,
|
||||
head_first=False,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
initial_state_indices=torch.arange(len(seq_lens), dtype=torch.int32),
|
||||
)
|
||||
|
||||
torch.testing.assert_close(out, out_ref, atol=1e-2, rtol=1e-2)
|
||||
@@ -331,7 +339,7 @@ TWO_CALL_SPLITS = [
|
||||
|
||||
@pytest.mark.parametrize("total_tokens, split", TWO_CALL_SPLITS)
|
||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||
@pytest.mark.parametrize("head_dims", HEAD_DIMS)
|
||||
@pytest.mark.parametrize("head_dims", CHUNK_HEAD_DIMS)
|
||||
@torch.inference_mode()
|
||||
def test_chunk_gated_delta_rule_cpu_two_call_split(
|
||||
total_tokens: int,
|
||||
@@ -370,6 +378,7 @@ def test_chunk_gated_delta_rule_cpu_two_call_split(
|
||||
cu_seqlens=torch.tensor([0, total_tokens], dtype=torch.int32),
|
||||
head_first=False,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
initial_state_indices=torch.zeros(1, dtype=torch.int32),
|
||||
)
|
||||
|
||||
# Call 1: tokens [0:split], no initial state, capture final state.
|
||||
@@ -384,6 +393,7 @@ def test_chunk_gated_delta_rule_cpu_two_call_split(
|
||||
cu_seqlens=torch.tensor([0, split], dtype=torch.int32),
|
||||
head_first=False,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
initial_state_indices=torch.zeros(1, dtype=torch.int32),
|
||||
)
|
||||
# Call 2: tokens [split:T] seeded with call 1's final state and a cu_seqlens
|
||||
# rebased to start at 0, as cpu_gdn_attention_core continues a prefill chunk.
|
||||
@@ -399,6 +409,7 @@ def test_chunk_gated_delta_rule_cpu_two_call_split(
|
||||
cu_seqlens=torch.tensor([0, tail], dtype=torch.int32),
|
||||
head_first=False,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
initial_state_indices=torch.zeros(1, dtype=torch.int32),
|
||||
)
|
||||
|
||||
out_split = torch.cat([out1, out2], dim=1)
|
||||
|
||||
@@ -3491,6 +3491,7 @@ def chunk_gated_delta_rule_cpu(
|
||||
cu_seqlens: torch.Tensor,
|
||||
head_first: bool,
|
||||
use_qk_l2norm_in_kernel: bool,
|
||||
initial_state_indices: torch.Tensor,
|
||||
eps: float = 1e-5,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return torch.ops._C.chunk_gated_delta_rule_cpu(
|
||||
@@ -3504,6 +3505,7 @@ def chunk_gated_delta_rule_cpu(
|
||||
cu_seqlens,
|
||||
head_first,
|
||||
use_qk_l2norm_in_kernel,
|
||||
initial_state_indices,
|
||||
eps,
|
||||
)
|
||||
|
||||
|
||||
@@ -236,22 +236,24 @@ def _cpu_gdn_attention_nonspec(
|
||||
A_log=layer.A_log, a=prefill_a, b=prefill_b, dt_bias=layer.dt_bias
|
||||
)
|
||||
|
||||
initial_state = ssm_state[prefill_state_indices]
|
||||
initial_state[~prefill_has_initial_state, ...] = 0
|
||||
attn_out, last_recurrent_state = ops.chunk_gated_delta_rule_cpu(
|
||||
# zero pool slots for sequences without a prior state; the kernel
|
||||
# gathers/mutates ssm_state in place via prefill_state_indices, so no
|
||||
# separate scatter-back is needed after the call.
|
||||
no_initial_state = prefill_state_indices[~prefill_has_initial_state]
|
||||
if no_initial_state.numel() > 0:
|
||||
ssm_state[no_initial_state] = 0
|
||||
attn_out, _ = ops.chunk_gated_delta_rule_cpu(
|
||||
query=query,
|
||||
key=key,
|
||||
value=value,
|
||||
g=g,
|
||||
beta=beta,
|
||||
initial_state=initial_state,
|
||||
initial_state=ssm_state,
|
||||
output_final_state=True,
|
||||
cu_seqlens=prefill_query_start_loc,
|
||||
head_first=False,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
)
|
||||
ssm_state[prefill_state_indices] = last_recurrent_state.to(
|
||||
ssm_state.dtype, copy=False
|
||||
initial_state_indices=prefill_state_indices,
|
||||
)
|
||||
core_attn_out[prefill_token_start:prefill_token_end] = attn_out.squeeze(0)
|
||||
|
||||
@@ -616,22 +618,24 @@ def _spec_aware_nonspec(
|
||||
g, beta = ops.fused_gdn_gating_cpu(
|
||||
A_log=layer.A_log, a=prefill_a, b=prefill_b, dt_bias=layer.dt_bias
|
||||
)
|
||||
initial_state = ssm_state[prefill_state_indices]
|
||||
initial_state[~prefill_has_initial_state, ...] = 0
|
||||
attn_out, last_recurrent_state = ops.chunk_gated_delta_rule_cpu(
|
||||
# zero pool slots for sequences without a prior state; the kernel
|
||||
# gathers/mutates ssm_state in place via prefill_state_indices, so no
|
||||
# separate scatter-back is needed after the call.
|
||||
no_initial_state = prefill_state_indices[~prefill_has_initial_state]
|
||||
if no_initial_state.numel() > 0:
|
||||
ssm_state[no_initial_state] = 0
|
||||
attn_out, _ = ops.chunk_gated_delta_rule_cpu(
|
||||
query=query,
|
||||
key=key,
|
||||
value=value,
|
||||
g=g,
|
||||
beta=beta,
|
||||
initial_state=initial_state,
|
||||
initial_state=ssm_state,
|
||||
output_final_state=True,
|
||||
cu_seqlens=prefill_query_start_loc,
|
||||
head_first=False,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
)
|
||||
ssm_state[prefill_state_indices] = last_recurrent_state.to(
|
||||
ssm_state.dtype, copy=False
|
||||
initial_state_indices=prefill_state_indices,
|
||||
)
|
||||
core_attn_out[prefill_token_start:prefill_token_end] = attn_out.squeeze(0)
|
||||
|
||||
@@ -690,22 +694,24 @@ def _spec_aware_nonspec_subset(
|
||||
g, beta = ops.fused_gdn_gating_cpu(
|
||||
A_log=layer.A_log, a=a, b=b, dt_bias=layer.dt_bias
|
||||
)
|
||||
initial_state = ssm_state[prefill_state_indices]
|
||||
initial_state[~has_initial_state, ...] = 0
|
||||
attn_out, last_recurrent_state = ops.chunk_gated_delta_rule_cpu(
|
||||
# zero pool slots for sequences without a prior state; the kernel
|
||||
# gathers/mutates ssm_state in place via prefill_state_indices, so no
|
||||
# separate scatter-back is needed after the call.
|
||||
no_initial_state = prefill_state_indices[~has_initial_state]
|
||||
if no_initial_state.numel() > 0:
|
||||
ssm_state[no_initial_state] = 0
|
||||
attn_out, _ = ops.chunk_gated_delta_rule_cpu(
|
||||
query=query,
|
||||
key=key,
|
||||
value=value,
|
||||
g=g,
|
||||
beta=beta,
|
||||
initial_state=initial_state,
|
||||
initial_state=ssm_state,
|
||||
output_final_state=True,
|
||||
cu_seqlens=prefill_qsl,
|
||||
head_first=False,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
)
|
||||
ssm_state[prefill_state_indices] = last_recurrent_state.to(
|
||||
ssm_state.dtype, copy=False
|
||||
initial_state_indices=prefill_state_indices,
|
||||
)
|
||||
return attn_out.squeeze(0)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user