[CPU] Add fused GDN support for AMX CPU platform (#42707)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
@@ -213,7 +213,7 @@ struct tinygemm_kernel<at::BFloat16, K, BLOCK_N, has_bias, has_silu> {
|
||||
weight + nb_start * width, \
|
||||
out + bs * seqlen * dim + mb_start * dim + nb_start, \
|
||||
has_bias ? bias + nb_start : nullptr, \
|
||||
has_conv_states ? conv_states + conv_state_index * (K - 1) * dim + nb_start : nullptr, \
|
||||
has_conv_states ? conv_states + conv_state_index * conv_state_slot_stride + nb_start : nullptr, \
|
||||
has_initial_states_value, \
|
||||
mb_size, \
|
||||
dim, \
|
||||
@@ -233,7 +233,8 @@ void causal_conv1d_fwd_kernel_impl(
|
||||
int64_t dim,
|
||||
int64_t seqlen,
|
||||
int64_t width,
|
||||
int64_t num_seq_blocks) {
|
||||
int64_t num_seq_blocks,
|
||||
int64_t conv_state_slot_stride) {
|
||||
// handle 32 x 64 per block
|
||||
constexpr int64_t BLOCK_M = block_size_m();
|
||||
constexpr int64_t BLOCK_N = block_size_n() * 2;
|
||||
@@ -282,7 +283,7 @@ void causal_conv1d_fwd_kernel_impl(
|
||||
at::parallel_for(0, batch, 0, [&](int64_t begin, int64_t end) {
|
||||
for (int64_t bs = begin; bs < end; ++bs) {
|
||||
update_conv_state(
|
||||
conv_states + bs * (width - 1) * dim, input + bs * seqlen * dim, width, dim, seqlen, has_initial_state[bs]);
|
||||
conv_states + bs * conv_state_slot_stride, input + bs * seqlen * dim, width, dim, seqlen, has_initial_state[bs]);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -316,7 +317,8 @@ void causal_conv1d_fwd_varlen_kernel_impl(
|
||||
int64_t batch,
|
||||
int64_t dim,
|
||||
int64_t width,
|
||||
int64_t num_seq_blocks) {
|
||||
int64_t num_seq_blocks,
|
||||
int64_t conv_state_slot_stride) {
|
||||
// handle 32 x 64 per block
|
||||
constexpr int64_t BLOCK_M = block_size_m();
|
||||
constexpr int64_t BLOCK_N = block_size_n() * 2;
|
||||
@@ -366,7 +368,7 @@ void causal_conv1d_fwd_varlen_kernel_impl(
|
||||
int32_t seqlen = query_start_loc[bs + 1] - query_start_loc[bs];
|
||||
int32_t batch_offset = query_start_loc[bs];
|
||||
update_conv_state(
|
||||
conv_states + conv_state_index * (width - 1) * dim,
|
||||
conv_states + conv_state_index * conv_state_slot_stride,
|
||||
input + batch_offset * dim,
|
||||
width,
|
||||
dim,
|
||||
@@ -389,7 +391,8 @@ void causal_conv1d_update_kernel_impl(
|
||||
int64_t batch,
|
||||
int64_t dim,
|
||||
int64_t seqlen,
|
||||
int64_t width) {
|
||||
int64_t width,
|
||||
int64_t conv_state_slot_stride) {
|
||||
// handle 32 x 64 per block
|
||||
constexpr int64_t BLOCK_M = block_size_m();
|
||||
constexpr int64_t BLOCK_N = block_size_n() * 2;
|
||||
@@ -430,7 +433,7 @@ void causal_conv1d_update_kernel_impl(
|
||||
});
|
||||
});
|
||||
|
||||
#define CONV_STATE_INDEXR(w) conv_states + conv_state_index*(width - 1) * dim + (w) * dim
|
||||
#define CONV_STATE_INDEXR(w) conv_states + conv_state_index*conv_state_slot_stride + (w) * dim
|
||||
|
||||
// update conv_states
|
||||
at::parallel_for(0, batch, 0, [&](int64_t begin, int64_t end) {
|
||||
@@ -592,6 +595,9 @@ at::Tensor causal_conv1d_fwd_cpu(
|
||||
}
|
||||
}
|
||||
|
||||
// IMPORTANT: To make the kernal compatible with vLLM KV cache layout
|
||||
int64_t conv_state_slot_stride = conv_states->stride(0);
|
||||
|
||||
// block size for sequence blocks, 32
|
||||
constexpr int64_t BLOCK_M = block_size_m();
|
||||
|
||||
@@ -618,7 +624,8 @@ at::Tensor causal_conv1d_fwd_cpu(
|
||||
batch,
|
||||
dim,
|
||||
width,
|
||||
num_seq_blocks);
|
||||
num_seq_blocks,
|
||||
conv_state_slot_stride);
|
||||
} else {
|
||||
causal_conv1d_fwd_kernel_impl<scalar_t>(
|
||||
out.data_ptr<scalar_t>(),
|
||||
@@ -633,7 +640,8 @@ at::Tensor causal_conv1d_fwd_cpu(
|
||||
dim,
|
||||
seqlen,
|
||||
width,
|
||||
num_seq_blocks);
|
||||
num_seq_blocks,
|
||||
conv_state_slot_stride);
|
||||
}
|
||||
});
|
||||
return out;
|
||||
@@ -690,6 +698,8 @@ at::Tensor causal_conv1d_update_cpu(
|
||||
conv_states.copy_(conv_states_copy);
|
||||
}
|
||||
|
||||
// IMPORTANT: To make the kernal compatible with vLLM KV cache layout
|
||||
int64_t conv_state_slot_stride = conv_states.stride(0);
|
||||
at::Tensor out = at::empty_like(x);
|
||||
AT_DISPATCH_REDUCED_FLOATING_TYPES(scalar_type, "causal_conv1d_update_kernel_impl", [&] {
|
||||
causal_conv1d_update_kernel_impl<scalar_t>(
|
||||
@@ -703,7 +713,8 @@ at::Tensor causal_conv1d_update_cpu(
|
||||
batch,
|
||||
dim,
|
||||
seqlen,
|
||||
width);
|
||||
width,
|
||||
conv_state_slot_stride);
|
||||
});
|
||||
return out;
|
||||
}
|
||||
|
||||
@@ -847,6 +847,7 @@ void fused_sigmoid_gating_delta_rule_update_kernel_impl(
|
||||
int64_t v_strideB,
|
||||
int64_t v_strideS,
|
||||
int64_t v_strideH,
|
||||
int64_t state_slot_stride,
|
||||
bool use_qk_l2norm_in_kernel,
|
||||
double softplus_threshold) {
|
||||
using bVec = at::vec::Vectorized<scalar_t>;
|
||||
@@ -907,7 +908,7 @@ void fused_sigmoid_gating_delta_rule_update_kernel_impl(
|
||||
data_index_init(begin, bi, batch_size, si, seq_len, ni, v_num_heads);
|
||||
for (int64_t i = begin; i < end; ++i) {
|
||||
int64_t cache_index = indices_ptr[bi];
|
||||
int64_t state_offset = (cache_index * v_num_heads + ni) * head_dim * v_head_dim;
|
||||
int64_t state_offset = cache_index * state_slot_stride + ni * head_dim * v_head_dim;
|
||||
float g_val = -std::exp(float(A_log_ptr[ni])) *
|
||||
softplus(float(a_ptr[bi * v_num_heads + ni]) + float(dt_bias_ptr[ni]), softplus_threshold);
|
||||
float g_val_exp = std::exp(g_val);
|
||||
@@ -1321,6 +1322,8 @@ at::Tensor fused_sigmoid_gating_delta_rule_update_cpu(
|
||||
int64_t v_strideB = v.stride(1);
|
||||
int64_t v_strideS = v.stride(0);
|
||||
int64_t v_strideH = v.stride(2);
|
||||
// IMPORTANT: To make the kernal compatible with vLLM KV cache layout
|
||||
int64_t state_slot_stride = initial_state_source.stride(0);
|
||||
at::Tensor core_attn_out = at::empty({batch_size, seq_len, v_num_heads, v_head_dim}, q.options());
|
||||
at::Tensor qk_scale_buf = at::empty({2 * batch_size, seq_len, num_heads}, at::kFloat);
|
||||
|
||||
@@ -1353,6 +1356,7 @@ at::Tensor fused_sigmoid_gating_delta_rule_update_cpu(
|
||||
v_strideB,
|
||||
v_strideS,
|
||||
v_strideH,
|
||||
state_slot_stride,
|
||||
use_qk_l2norm_in_kernel,
|
||||
softplus_threshold);
|
||||
});
|
||||
|
||||
@@ -98,6 +98,45 @@ at::Tensor int4_scaled_mm_cpu(at::Tensor& x, at::Tensor& w, at::Tensor& w_zeros,
|
||||
at::Tensor& w_scales,
|
||||
std::optional<at::Tensor> bias);
|
||||
|
||||
// Adapted from sglang: GDN
|
||||
std::tuple<at::Tensor, at::Tensor> chunk_gated_delta_rule_cpu(
|
||||
const at::Tensor& query, const at::Tensor& key, const at::Tensor& value,
|
||||
const at::Tensor& g, const at::Tensor& beta,
|
||||
const at::Tensor& initial_state, bool output_final_state,
|
||||
const at::Tensor& cu_seqlens, bool head_first, bool use_qk_l2norm_in_kernel,
|
||||
double eps = 1e-5);
|
||||
|
||||
at::Tensor fused_sigmoid_gating_delta_rule_update_cpu(
|
||||
const at::Tensor& A_log, const at::Tensor& dt_bias, const at::Tensor& q,
|
||||
const at::Tensor& k, const at::Tensor& v, const at::Tensor& a,
|
||||
const at::Tensor& b, at::Tensor& initial_state_source,
|
||||
const at::Tensor& initial_state_indices, const at::Tensor& cu_seqlens,
|
||||
bool use_qk_l2norm_in_kernel, double softplus_beta = 1.0,
|
||||
double softplus_threshold = 20.0);
|
||||
|
||||
std::tuple<at::Tensor, at::Tensor> fused_gdn_gating_cpu(
|
||||
const at::Tensor& A_log, const at::Tensor& a, const at::Tensor& b,
|
||||
const at::Tensor& dt_bias);
|
||||
|
||||
// Adapted from sglang: casual_conv1d kernels
|
||||
at::Tensor causal_conv1d_weight_pack(const at::Tensor& weight);
|
||||
|
||||
at::Tensor causal_conv1d_fwd_cpu(
|
||||
const at::Tensor& x, const at::Tensor& weight,
|
||||
const std::optional<at::Tensor>& bias,
|
||||
const std::optional<at::Tensor>& conv_states,
|
||||
const std::optional<at::Tensor>& query_start_loc,
|
||||
const std::optional<at::Tensor>& cache_indices,
|
||||
const std::optional<at::Tensor>& has_initial_state, bool silu_activation,
|
||||
int64_t pad_slot_id, bool is_vnni);
|
||||
|
||||
at::Tensor causal_conv1d_update_cpu(
|
||||
const at::Tensor& x, const at::Tensor& conv_states,
|
||||
const at::Tensor& weight, const std::optional<at::Tensor>& bias,
|
||||
bool silu_activation, const std::optional<at::Tensor>& cache_seqlens,
|
||||
const std::optional<at::Tensor>& conv_state_indices, int64_t pad_slot_id,
|
||||
bool is_vnni);
|
||||
|
||||
void activation_lut_bf16(torch::Tensor& out, torch::Tensor& input,
|
||||
const std::string& activation);
|
||||
|
||||
@@ -397,6 +436,47 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
|
||||
"scales2, SymInt[] block_size, Tensor? bias, ScalarType out_dtype, "
|
||||
"bool is_vnni) -> Tensor");
|
||||
ops.impl("fp8_scaled_mm_cpu", torch::kCPU, &fp8_scaled_mm_cpu);
|
||||
|
||||
// Adapted from sglang: GDN kernels
|
||||
ops.def(
|
||||
"chunk_gated_delta_rule_cpu(Tensor query, Tensor key, Tensor value, "
|
||||
"Tensor g, Tensor beta, "
|
||||
"Tensor initial_state, bool output_final_state, Tensor cu_seqlens, bool "
|
||||
"head_first, "
|
||||
"bool use_qk_l2norm_in_kernel, float eps=1e-5) -> (Tensor, Tensor)");
|
||||
ops.impl("chunk_gated_delta_rule_cpu", torch::kCPU,
|
||||
&chunk_gated_delta_rule_cpu);
|
||||
ops.def(
|
||||
"fused_sigmoid_gating_delta_rule_update_cpu(Tensor A_log, Tensor "
|
||||
"dt_bias, Tensor q, Tensor k, Tensor v, Tensor "
|
||||
"a, Tensor b, Tensor(a!) initial_state_source, Tensor "
|
||||
"initial_state_indices, Tensor cu_seqlens, bool "
|
||||
"use_qk_l2norm_in_kernel, float softplus_beta=1.0, float "
|
||||
"softplus_threshold=20.0) -> Tensor");
|
||||
ops.impl("fused_sigmoid_gating_delta_rule_update_cpu", torch::kCPU,
|
||||
&fused_sigmoid_gating_delta_rule_update_cpu);
|
||||
ops.def(
|
||||
"fused_gdn_gating_cpu(Tensor A_log, Tensor a, Tensor b, Tensor dt_bias) "
|
||||
"-> (Tensor, Tensor)");
|
||||
ops.impl("fused_gdn_gating_cpu", torch::kCPU, &fused_gdn_gating_cpu);
|
||||
|
||||
// Adapted from sglang: casual_conv1d kernels
|
||||
ops.def("causal_conv1d_weight_pack(Tensor weight) -> Tensor");
|
||||
ops.impl("causal_conv1d_weight_pack", torch::kCPU,
|
||||
&causal_conv1d_weight_pack);
|
||||
ops.def(
|
||||
"causal_conv1d_fwd_cpu(Tensor x, Tensor weight, Tensor? bias, Tensor? "
|
||||
"conv_states, Tensor? query_start_loc,"
|
||||
"Tensor? cache_indices, Tensor? has_initial_state, bool silu_activation, "
|
||||
"int pad_slot_id, bool is_vnni) -> "
|
||||
"Tensor");
|
||||
ops.impl("causal_conv1d_fwd_cpu", torch::kCPU, &causal_conv1d_fwd_cpu);
|
||||
ops.def(
|
||||
"causal_conv1d_update_cpu(Tensor x, Tensor(a!) conv_states, Tensor "
|
||||
"weight, Tensor? bias, bool silu_activation,"
|
||||
"Tensor? cache_seqlens, Tensor? conv_state_indices, int pad_slot_id, "
|
||||
"bool is_vnni) -> Tensor");
|
||||
ops.impl("causal_conv1d_update_cpu", torch::kCPU, &causal_conv1d_update_cpu);
|
||||
#endif
|
||||
|
||||
// CPU attention kernels
|
||||
|
||||
Reference in New Issue
Block a user