[CPU] Refine CPU attention frontend (#45391)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
@@ -822,8 +822,8 @@ struct AttentionInput {
|
||||
logits_buffer_t *__restrict__ logits_buffer, \
|
||||
float *__restrict__ partial_q_buffer, float *__restrict__ max_buffer, \
|
||||
float *__restrict__ sum_buffer, int32_t *__restrict__ block_table, \
|
||||
const int32_t kv_tile_start_pos, const int32_t kv_tile_end_pos, \
|
||||
const int32_t kv_tile_token_num, \
|
||||
const int32_t kv_end_pos, const int32_t kv_tile_start_pos, \
|
||||
const int32_t kv_tile_end_pos, const int32_t kv_tile_token_num, \
|
||||
const int64_t kv_cache_num_blocks_stride, const int32_t q_head_num, \
|
||||
const int32_t q_token_num, const int32_t q_tile_start_pos, \
|
||||
const int32_t q_heads_per_kv, const int32_t block_size, \
|
||||
@@ -834,7 +834,7 @@ struct AttentionInput {
|
||||
|
||||
#define CPU_ATTENTION_PARAMS \
|
||||
q_heads_buffer, k_head_cache_ptr, v_head_cache_ptr, logits_buffer, \
|
||||
partial_q_buffer, max_buffer, sum_buffer, block_table, \
|
||||
partial_q_buffer, max_buffer, sum_buffer, block_table, kv_end_pos, \
|
||||
kv_tile_start_pos, kv_tile_end_pos, kv_tile_token_num, \
|
||||
kv_cache_num_blocks_stride, q_head_num, q_token_num, q_tile_start_pos, \
|
||||
q_heads_per_kv, block_size, left_window_size, right_window_size, scale, \
|
||||
@@ -917,6 +917,7 @@ class AttentionMainLoop {
|
||||
// - max_buffer: [MaxQHeadNumPerIteration, 1], store max logits
|
||||
// - sum_buffer: [MaxQHeadNumPerIteration, 1], store sum of exp
|
||||
// - block_table
|
||||
// - kv_end_pos: un-aligned end position of KV cache
|
||||
// - kv_tile_start_pos: start position of KV cache, aligned to
|
||||
// BlockSizeAlignment
|
||||
// - kv_tile_end_pos: end position of KV cache, aligned to
|
||||
@@ -1043,7 +1044,7 @@ class AttentionMainLoop {
|
||||
}
|
||||
|
||||
apply_mask(logits_buffer, kv_tile_token_num, q_tile_start_pos,
|
||||
kv_tile_start_pos, kv_tile_end_pos, q_token_num,
|
||||
kv_end_pos, kv_tile_start_pos, kv_tile_end_pos, q_token_num,
|
||||
q_heads_per_kv, left_window_size, right_window_size);
|
||||
|
||||
// if (debug_info){
|
||||
@@ -1126,7 +1127,7 @@ class AttentionMainLoop {
|
||||
|
||||
void apply_mask(logits_buffer_t* __restrict__ logits_buffer,
|
||||
const int64_t logits_buffer_stride,
|
||||
const int32_t q_tile_start_pos,
|
||||
const int32_t q_tile_start_pos, const int32_t kv_end_pos,
|
||||
const int32_t kv_tile_start_pos,
|
||||
const int32_t kv_tile_end_pos, const int32_t q_token_num,
|
||||
const int32_t q_heads_per_kv,
|
||||
@@ -1154,7 +1155,7 @@ class AttentionMainLoop {
|
||||
std::max(kv_tile_start_pos,
|
||||
curr_token_pos + sliding_window_right + 1));
|
||||
}
|
||||
return pos;
|
||||
return std::min(pos, kv_end_pos);
|
||||
}();
|
||||
|
||||
int32_t left_invalid_token_num = left_kv_pos - kv_tile_start_pos;
|
||||
@@ -1789,7 +1790,7 @@ class AttentionMainLoop {
|
||||
attn_impl.template execute_attention<Attention>(
|
||||
curr_q_heads_buffer, curr_k_cache, curr_v_cache,
|
||||
logits_buffer, curr_partial_q_buffer, curr_max_buffer,
|
||||
curr_sum_buffer, curr_block_table,
|
||||
curr_sum_buffer, curr_block_table, kv_end_pos,
|
||||
aligned_actual_kv_tile_pos_left,
|
||||
aligned_actual_kv_tile_pos_right, actual_kv_token_num,
|
||||
kv_cache_block_num_stride, q_tile_head_num,
|
||||
|
||||
@@ -11,7 +11,7 @@ import os
|
||||
HEAD_DIMS_32 = [32, 64, 96, 128, 160, 192, 224, 256, 512]
|
||||
|
||||
# Head dimensions divisible by 16 but not 32 (VEC16 only)
|
||||
HEAD_DIMS_16 = [80, 112]
|
||||
HEAD_DIMS_16 = [48, 80, 112]
|
||||
|
||||
# ISA types
|
||||
ISA_TYPES = {
|
||||
|
||||
Reference in New Issue
Block a user