[CPU] Support Gemma Diffusion (#45690)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
+62
-27
@@ -388,13 +388,13 @@ class AttentionScheduler {
|
||||
int32_t head_dim;
|
||||
int32_t* query_start_loc;
|
||||
int32_t* seq_lens;
|
||||
int32_t left_sliding_window_size;
|
||||
int32_t right_sliding_window_size;
|
||||
bool casual;
|
||||
int32_t sliding_window_size;
|
||||
bool causal;
|
||||
cpu_attention::ISA isa;
|
||||
int32_t max_num_q_per_iter; // max Q head num can be hold in registers
|
||||
int32_t kv_block_alignment; // context length alignment requirement
|
||||
bool enable_kv_split;
|
||||
bool* dynamic_causal;
|
||||
};
|
||||
|
||||
static constexpr int32_t MaxQTileIterNum = 128;
|
||||
@@ -403,7 +403,8 @@ class AttentionScheduler {
|
||||
: available_cache_size_(cpu_utils::get_available_l2_size()) {}
|
||||
|
||||
torch::Tensor schedule(const ScheduleInput& input) const {
|
||||
const bool casual = input.casual;
|
||||
const bool causal = input.causal;
|
||||
const bool is_dynamic_causal = input.dynamic_causal != nullptr;
|
||||
const int32_t thread_num = omp_get_max_threads();
|
||||
const int64_t cache_size = cpu_utils::get_available_l2_size();
|
||||
const int32_t max_num_q_per_iter = input.max_num_q_per_iter;
|
||||
@@ -434,8 +435,7 @@ class AttentionScheduler {
|
||||
const int32_t default_tile_token_num = default_tile_size / q_head_per_kv;
|
||||
const int32_t split_kv_q_token_num_threshold =
|
||||
input.enable_kv_split ? 1 : 0;
|
||||
const int32_t left_sliding_window_size = input.left_sliding_window_size;
|
||||
const int32_t right_sliding_window_size = input.right_sliding_window_size;
|
||||
const int32_t sliding_window_size = input.sliding_window_size;
|
||||
TORCH_CHECK_LE(split_kv_q_token_num_threshold * q_head_per_kv, 16);
|
||||
|
||||
// get total kv len
|
||||
@@ -444,7 +444,9 @@ class AttentionScheduler {
|
||||
const int32_t seq_len = input.seq_lens[req_id];
|
||||
const int32_t q_token_num =
|
||||
input.query_start_loc[req_id + 1] - input.query_start_loc[req_id];
|
||||
const int32_t q_start_pos = (casual ? (seq_len - q_token_num) : 0);
|
||||
const bool req_causal =
|
||||
is_dynamic_causal ? input.dynamic_causal[req_id] : causal;
|
||||
const int32_t q_start_pos = seq_len - q_token_num;
|
||||
const int32_t kv_start_pos = 0;
|
||||
const int32_t kv_end_pos = seq_len;
|
||||
|
||||
@@ -456,7 +458,7 @@ class AttentionScheduler {
|
||||
const int32_t q_tile_pos_right = q_tile_pos_left + q_tile_token_num;
|
||||
const auto [kv_tile_pos_left, kv_tile_pos_right] = calcu_kv_tile_pos(
|
||||
kv_start_pos, kv_end_pos, q_tile_pos_left, q_tile_pos_right,
|
||||
left_sliding_window_size, right_sliding_window_size);
|
||||
sliding_window_size, req_causal);
|
||||
const auto [aligned_kv_tile_pos_left, aligned_kv_tile_pos_right] =
|
||||
align_kv_tile_pos(kv_tile_pos_left, kv_tile_pos_right,
|
||||
kv_len_alignment);
|
||||
@@ -484,7 +486,9 @@ class AttentionScheduler {
|
||||
const int32_t seq_len = input.seq_lens[req_id];
|
||||
const int32_t q_token_num =
|
||||
input.query_start_loc[req_id + 1] - input.query_start_loc[req_id];
|
||||
const int32_t q_start_pos = (casual ? (seq_len - q_token_num) : 0);
|
||||
const bool req_causal =
|
||||
is_dynamic_causal ? input.dynamic_causal[req_id] : causal;
|
||||
const int32_t q_start_pos = seq_len - q_token_num;
|
||||
const int32_t kv_start_pos = 0;
|
||||
const int32_t kv_end_pos = seq_len;
|
||||
int32_t local_split_id = 0;
|
||||
@@ -498,7 +502,7 @@ class AttentionScheduler {
|
||||
const int32_t q_tile_pos_right = q_tile_pos_left + q_tile_token_num;
|
||||
const auto [kv_tile_pos_left, kv_tile_pos_right] = calcu_kv_tile_pos(
|
||||
kv_start_pos, kv_end_pos, q_tile_pos_left, q_tile_pos_right,
|
||||
left_sliding_window_size, right_sliding_window_size);
|
||||
sliding_window_size, req_causal);
|
||||
const auto [aligned_kv_tile_pos_left, aligned_kv_tile_pos_right] =
|
||||
align_kv_tile_pos(kv_tile_pos_left, kv_tile_pos_right,
|
||||
kv_len_alignment);
|
||||
@@ -708,15 +712,41 @@ class AttentionScheduler {
|
||||
return metadata_tensor;
|
||||
}
|
||||
|
||||
FORCE_INLINE static std::pair<int32_t, int32_t> calcu_sliding_window_size(
|
||||
int32_t window_size, bool causal) {
|
||||
int32_t left_sliding_window_size, right_sliding_window_size;
|
||||
if (window_size != -1) {
|
||||
left_sliding_window_size = window_size - 1;
|
||||
if (causal) {
|
||||
right_sliding_window_size = 0;
|
||||
} else {
|
||||
right_sliding_window_size = window_size - 1;
|
||||
}
|
||||
} else {
|
||||
left_sliding_window_size = -1;
|
||||
if (causal) {
|
||||
right_sliding_window_size = 0;
|
||||
} else {
|
||||
right_sliding_window_size = -1;
|
||||
}
|
||||
}
|
||||
|
||||
return {left_sliding_window_size, right_sliding_window_size};
|
||||
}
|
||||
|
||||
FORCE_INLINE static std::pair<int32_t, int32_t> calcu_kv_tile_pos(
|
||||
int32_t kv_left_pos, int32_t kv_right_pos, int32_t q_left_pos,
|
||||
int32_t q_right_pos, int32_t sliding_window_left,
|
||||
int32_t sliding_window_right) {
|
||||
if (sliding_window_left != -1) {
|
||||
kv_left_pos = std::max(kv_left_pos, q_left_pos - sliding_window_left);
|
||||
int32_t q_right_pos, int32_t window_size, bool causal) {
|
||||
auto [left_sliding_window_size, right_sliding_window_size] =
|
||||
calcu_sliding_window_size(window_size, causal);
|
||||
|
||||
if (left_sliding_window_size != -1) {
|
||||
kv_left_pos =
|
||||
std::max(kv_left_pos, q_left_pos - left_sliding_window_size);
|
||||
}
|
||||
if (sliding_window_right != -1) {
|
||||
kv_right_pos = std::min(kv_right_pos, q_right_pos + sliding_window_right);
|
||||
if (right_sliding_window_size != -1) {
|
||||
kv_right_pos =
|
||||
std::min(kv_right_pos, q_right_pos + right_sliding_window_size);
|
||||
}
|
||||
return {kv_left_pos, kv_right_pos};
|
||||
}
|
||||
@@ -805,10 +835,10 @@ struct AttentionInput {
|
||||
int32_t* block_table;
|
||||
float* alibi_slopes;
|
||||
c10::BFloat16* s_aux;
|
||||
bool* dynamic_causal;
|
||||
float scale;
|
||||
bool causal;
|
||||
int32_t sliding_window_left;
|
||||
int32_t sliding_window_right;
|
||||
int32_t sliding_window_size;
|
||||
float softcap;
|
||||
// FP8 KV cache scales (used by FP8 attention implementations)
|
||||
float k_scale_fp8 = 1.0f;
|
||||
@@ -1442,15 +1472,16 @@ class AttentionMainLoop {
|
||||
const int64_t q_head_num_stride = input->query_num_heads_stride;
|
||||
const int64_t kv_cache_head_num_stride = input->cache_num_kv_heads_stride;
|
||||
const int64_t kv_cache_block_num_stride = input->cache_num_blocks_stride;
|
||||
const int32_t sliding_window_left = input->sliding_window_left;
|
||||
const int32_t sliding_window_right = input->sliding_window_right;
|
||||
const int32_t sliding_window_size = input->sliding_window_size;
|
||||
const int32_t block_size = input->block_size;
|
||||
const float scale = input->scale;
|
||||
const float softcap_scale = input->softcap;
|
||||
const float* alibi_slopes = input->alibi_slopes;
|
||||
const c10::BFloat16* s_aux = input->s_aux;
|
||||
const bool* dynamic_causal = input->dynamic_causal;
|
||||
const bool is_dynamic_causal = dynamic_causal != nullptr;
|
||||
|
||||
const bool casual = input->causal;
|
||||
const bool causal = input->causal;
|
||||
int32_t* const block_table = input->block_table;
|
||||
const int64_t block_table_stride = input->blt_num_tokens_stride;
|
||||
|
||||
@@ -1533,6 +1564,11 @@ class AttentionMainLoop {
|
||||
&curr_workitem_groups[workitem_group_idx];
|
||||
|
||||
const int32_t current_group_idx = current_workitem_group->req_id;
|
||||
const int32_t current_group_causal =
|
||||
is_dynamic_causal ? dynamic_causal[current_group_idx] : causal;
|
||||
auto [sliding_window_left, sliding_window_right] =
|
||||
AttentionScheduler::calcu_sliding_window_size(
|
||||
sliding_window_size, current_group_causal);
|
||||
const int32_t kv_start_pos =
|
||||
current_workitem_group->kv_split_pos_start;
|
||||
const int32_t kv_end_pos = current_workitem_group->kv_split_pos_end;
|
||||
@@ -1560,8 +1596,7 @@ class AttentionMainLoop {
|
||||
const int32_t q_end = input->query_start_loc[current_group_idx + 1];
|
||||
const int32_t q_start = input->query_start_loc[current_group_idx];
|
||||
const int32_t seq_len = input->seq_lens[current_group_idx];
|
||||
const int32_t q_start_pos =
|
||||
(casual ? seq_len - (q_end - q_start) : 0);
|
||||
const int32_t q_start_pos = seq_len - (q_end - q_start);
|
||||
const int32_t block_num = (seq_len + block_size - 1) / block_size;
|
||||
// Only apply sink for the first KV split
|
||||
bool use_sink = (s_aux != nullptr &&
|
||||
@@ -1611,8 +1646,8 @@ class AttentionMainLoop {
|
||||
const auto [kv_tile_start_pos, kv_tile_end_pos] =
|
||||
AttentionScheduler::calcu_kv_tile_pos(
|
||||
kv_start_pos, kv_end_pos, q_tile_start_pos,
|
||||
q_tile_end_pos, sliding_window_left,
|
||||
sliding_window_right);
|
||||
q_tile_end_pos, sliding_window_size,
|
||||
current_group_causal);
|
||||
const auto [rounded_kv_tile_start_pos, rounded_kv_tile_end_pos] =
|
||||
AttentionScheduler::align_kv_tile_pos(
|
||||
kv_tile_start_pos, kv_tile_end_pos, blocksize_alignment);
|
||||
@@ -1725,8 +1760,8 @@ class AttentionMainLoop {
|
||||
actual_kv_tile_pos_right] =
|
||||
AttentionScheduler::calcu_kv_tile_pos(
|
||||
kv_tile_pos_left, kv_tile_pos_right, q_tile_pos_left,
|
||||
q_tile_pos_right, sliding_window_left,
|
||||
sliding_window_right);
|
||||
q_tile_pos_right, sliding_window_size,
|
||||
current_group_causal);
|
||||
const int32_t q_iter_idx =
|
||||
q_head_tile_token_offset / curr_max_q_token_num_per_iter;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user