[CPU] Support Gemma Diffusion (#45690)

Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
Li, Jiang
2026-06-16 14:39:56 +08:00
committed by GitHub
parent 8bf374955f
commit a7fdfeef72
7 changed files with 213 additions and 100 deletions
+62 -27
View File
@@ -388,13 +388,13 @@ class AttentionScheduler {
int32_t head_dim;
int32_t* query_start_loc;
int32_t* seq_lens;
int32_t left_sliding_window_size;
int32_t right_sliding_window_size;
bool casual;
int32_t sliding_window_size;
bool causal;
cpu_attention::ISA isa;
int32_t max_num_q_per_iter; // max Q head num can be hold in registers
int32_t kv_block_alignment; // context length alignment requirement
bool enable_kv_split;
bool* dynamic_causal;
};
static constexpr int32_t MaxQTileIterNum = 128;
@@ -403,7 +403,8 @@ class AttentionScheduler {
: available_cache_size_(cpu_utils::get_available_l2_size()) {}
torch::Tensor schedule(const ScheduleInput& input) const {
const bool casual = input.casual;
const bool causal = input.causal;
const bool is_dynamic_causal = input.dynamic_causal != nullptr;
const int32_t thread_num = omp_get_max_threads();
const int64_t cache_size = cpu_utils::get_available_l2_size();
const int32_t max_num_q_per_iter = input.max_num_q_per_iter;
@@ -434,8 +435,7 @@ class AttentionScheduler {
const int32_t default_tile_token_num = default_tile_size / q_head_per_kv;
const int32_t split_kv_q_token_num_threshold =
input.enable_kv_split ? 1 : 0;
const int32_t left_sliding_window_size = input.left_sliding_window_size;
const int32_t right_sliding_window_size = input.right_sliding_window_size;
const int32_t sliding_window_size = input.sliding_window_size;
TORCH_CHECK_LE(split_kv_q_token_num_threshold * q_head_per_kv, 16);
// get total kv len
@@ -444,7 +444,9 @@ class AttentionScheduler {
const int32_t seq_len = input.seq_lens[req_id];
const int32_t q_token_num =
input.query_start_loc[req_id + 1] - input.query_start_loc[req_id];
const int32_t q_start_pos = (casual ? (seq_len - q_token_num) : 0);
const bool req_causal =
is_dynamic_causal ? input.dynamic_causal[req_id] : causal;
const int32_t q_start_pos = seq_len - q_token_num;
const int32_t kv_start_pos = 0;
const int32_t kv_end_pos = seq_len;
@@ -456,7 +458,7 @@ class AttentionScheduler {
const int32_t q_tile_pos_right = q_tile_pos_left + q_tile_token_num;
const auto [kv_tile_pos_left, kv_tile_pos_right] = calcu_kv_tile_pos(
kv_start_pos, kv_end_pos, q_tile_pos_left, q_tile_pos_right,
left_sliding_window_size, right_sliding_window_size);
sliding_window_size, req_causal);
const auto [aligned_kv_tile_pos_left, aligned_kv_tile_pos_right] =
align_kv_tile_pos(kv_tile_pos_left, kv_tile_pos_right,
kv_len_alignment);
@@ -484,7 +486,9 @@ class AttentionScheduler {
const int32_t seq_len = input.seq_lens[req_id];
const int32_t q_token_num =
input.query_start_loc[req_id + 1] - input.query_start_loc[req_id];
const int32_t q_start_pos = (casual ? (seq_len - q_token_num) : 0);
const bool req_causal =
is_dynamic_causal ? input.dynamic_causal[req_id] : causal;
const int32_t q_start_pos = seq_len - q_token_num;
const int32_t kv_start_pos = 0;
const int32_t kv_end_pos = seq_len;
int32_t local_split_id = 0;
@@ -498,7 +502,7 @@ class AttentionScheduler {
const int32_t q_tile_pos_right = q_tile_pos_left + q_tile_token_num;
const auto [kv_tile_pos_left, kv_tile_pos_right] = calcu_kv_tile_pos(
kv_start_pos, kv_end_pos, q_tile_pos_left, q_tile_pos_right,
left_sliding_window_size, right_sliding_window_size);
sliding_window_size, req_causal);
const auto [aligned_kv_tile_pos_left, aligned_kv_tile_pos_right] =
align_kv_tile_pos(kv_tile_pos_left, kv_tile_pos_right,
kv_len_alignment);
@@ -708,15 +712,41 @@ class AttentionScheduler {
return metadata_tensor;
}
FORCE_INLINE static std::pair<int32_t, int32_t> calcu_sliding_window_size(
int32_t window_size, bool causal) {
int32_t left_sliding_window_size, right_sliding_window_size;
if (window_size != -1) {
left_sliding_window_size = window_size - 1;
if (causal) {
right_sliding_window_size = 0;
} else {
right_sliding_window_size = window_size - 1;
}
} else {
left_sliding_window_size = -1;
if (causal) {
right_sliding_window_size = 0;
} else {
right_sliding_window_size = -1;
}
}
return {left_sliding_window_size, right_sliding_window_size};
}
FORCE_INLINE static std::pair<int32_t, int32_t> calcu_kv_tile_pos(
int32_t kv_left_pos, int32_t kv_right_pos, int32_t q_left_pos,
int32_t q_right_pos, int32_t sliding_window_left,
int32_t sliding_window_right) {
if (sliding_window_left != -1) {
kv_left_pos = std::max(kv_left_pos, q_left_pos - sliding_window_left);
int32_t q_right_pos, int32_t window_size, bool causal) {
auto [left_sliding_window_size, right_sliding_window_size] =
calcu_sliding_window_size(window_size, causal);
if (left_sliding_window_size != -1) {
kv_left_pos =
std::max(kv_left_pos, q_left_pos - left_sliding_window_size);
}
if (sliding_window_right != -1) {
kv_right_pos = std::min(kv_right_pos, q_right_pos + sliding_window_right);
if (right_sliding_window_size != -1) {
kv_right_pos =
std::min(kv_right_pos, q_right_pos + right_sliding_window_size);
}
return {kv_left_pos, kv_right_pos};
}
@@ -805,10 +835,10 @@ struct AttentionInput {
int32_t* block_table;
float* alibi_slopes;
c10::BFloat16* s_aux;
bool* dynamic_causal;
float scale;
bool causal;
int32_t sliding_window_left;
int32_t sliding_window_right;
int32_t sliding_window_size;
float softcap;
// FP8 KV cache scales (used by FP8 attention implementations)
float k_scale_fp8 = 1.0f;
@@ -1442,15 +1472,16 @@ class AttentionMainLoop {
const int64_t q_head_num_stride = input->query_num_heads_stride;
const int64_t kv_cache_head_num_stride = input->cache_num_kv_heads_stride;
const int64_t kv_cache_block_num_stride = input->cache_num_blocks_stride;
const int32_t sliding_window_left = input->sliding_window_left;
const int32_t sliding_window_right = input->sliding_window_right;
const int32_t sliding_window_size = input->sliding_window_size;
const int32_t block_size = input->block_size;
const float scale = input->scale;
const float softcap_scale = input->softcap;
const float* alibi_slopes = input->alibi_slopes;
const c10::BFloat16* s_aux = input->s_aux;
const bool* dynamic_causal = input->dynamic_causal;
const bool is_dynamic_causal = dynamic_causal != nullptr;
const bool casual = input->causal;
const bool causal = input->causal;
int32_t* const block_table = input->block_table;
const int64_t block_table_stride = input->blt_num_tokens_stride;
@@ -1533,6 +1564,11 @@ class AttentionMainLoop {
&curr_workitem_groups[workitem_group_idx];
const int32_t current_group_idx = current_workitem_group->req_id;
const int32_t current_group_causal =
is_dynamic_causal ? dynamic_causal[current_group_idx] : causal;
auto [sliding_window_left, sliding_window_right] =
AttentionScheduler::calcu_sliding_window_size(
sliding_window_size, current_group_causal);
const int32_t kv_start_pos =
current_workitem_group->kv_split_pos_start;
const int32_t kv_end_pos = current_workitem_group->kv_split_pos_end;
@@ -1560,8 +1596,7 @@ class AttentionMainLoop {
const int32_t q_end = input->query_start_loc[current_group_idx + 1];
const int32_t q_start = input->query_start_loc[current_group_idx];
const int32_t seq_len = input->seq_lens[current_group_idx];
const int32_t q_start_pos =
(casual ? seq_len - (q_end - q_start) : 0);
const int32_t q_start_pos = seq_len - (q_end - q_start);
const int32_t block_num = (seq_len + block_size - 1) / block_size;
// Only apply sink for the first KV split
bool use_sink = (s_aux != nullptr &&
@@ -1611,8 +1646,8 @@ class AttentionMainLoop {
const auto [kv_tile_start_pos, kv_tile_end_pos] =
AttentionScheduler::calcu_kv_tile_pos(
kv_start_pos, kv_end_pos, q_tile_start_pos,
q_tile_end_pos, sliding_window_left,
sliding_window_right);
q_tile_end_pos, sliding_window_size,
current_group_causal);
const auto [rounded_kv_tile_start_pos, rounded_kv_tile_end_pos] =
AttentionScheduler::align_kv_tile_pos(
kv_tile_start_pos, kv_tile_end_pos, blocksize_alignment);
@@ -1725,8 +1760,8 @@ class AttentionMainLoop {
actual_kv_tile_pos_right] =
AttentionScheduler::calcu_kv_tile_pos(
kv_tile_pos_left, kv_tile_pos_right, q_tile_pos_left,
q_tile_pos_right, sliding_window_left,
sliding_window_right);
q_tile_pos_right, sliding_window_size,
current_group_causal);
const int32_t q_iter_idx =
q_head_tile_token_offset / curr_max_q_token_num_per_iter;