[Bugfix][DiffusionGemma] Read the causal mask as int32 and compile the sample step (#59992)
Signed-off-by: ojeda-e <eojeda@league.com>
This commit is contained in:
@@ -78,8 +78,9 @@ torch::Tensor get_scheduler_metadata(
|
||||
input.causal = causal;
|
||||
input.isa = isa;
|
||||
input.enable_kv_split = enable_kv_split;
|
||||
input.dynamic_causal =
|
||||
dynamic_causal.has_value() ? dynamic_causal->data_ptr<bool>() : nullptr;
|
||||
input.dynamic_causal = dynamic_causal.has_value()
|
||||
? dynamic_causal->data_ptr<int32_t>()
|
||||
: nullptr;
|
||||
|
||||
const int64_t kv_cache_idx =
|
||||
static_cast<int64_t>(parse_fp8_kv_dtype(kv_cache_dtype));
|
||||
@@ -256,8 +257,9 @@ void cpu_attention_with_kv_cache(
|
||||
input.s_aux = nullptr;
|
||||
input.s_aux_is_bf16 = false;
|
||||
}
|
||||
input.dynamic_causal =
|
||||
dynamic_causal.has_value() ? dynamic_causal->data_ptr<bool>() : nullptr;
|
||||
input.dynamic_causal = dynamic_causal.has_value()
|
||||
? dynamic_causal->data_ptr<int32_t>()
|
||||
: nullptr;
|
||||
input.scale = scale;
|
||||
input.causal = causal;
|
||||
input.sliding_window_size = sliding_window;
|
||||
|
||||
@@ -397,7 +397,7 @@ class AttentionScheduler {
|
||||
int32_t max_num_q_per_iter; // max Q head num can be hold in registers
|
||||
int32_t kv_block_alignment; // context length alignment requirement
|
||||
bool enable_kv_split;
|
||||
bool* dynamic_causal;
|
||||
int32_t* dynamic_causal;
|
||||
};
|
||||
|
||||
static constexpr int32_t MaxQTileIterNum = 128;
|
||||
@@ -450,7 +450,7 @@ class AttentionScheduler {
|
||||
const int32_t q_token_num =
|
||||
input.query_start_loc[req_id + 1] - input.query_start_loc[req_id];
|
||||
const bool req_causal =
|
||||
is_dynamic_causal ? input.dynamic_causal[req_id] : causal;
|
||||
is_dynamic_causal ? input.dynamic_causal[req_id] != 0 : causal;
|
||||
const int32_t q_start_pos = seq_len - q_token_num;
|
||||
const int32_t kv_start_pos = 0;
|
||||
const int32_t kv_end_pos = seq_len;
|
||||
@@ -492,7 +492,7 @@ class AttentionScheduler {
|
||||
const int32_t q_token_num =
|
||||
input.query_start_loc[req_id + 1] - input.query_start_loc[req_id];
|
||||
const bool req_causal =
|
||||
is_dynamic_causal ? input.dynamic_causal[req_id] : causal;
|
||||
is_dynamic_causal ? input.dynamic_causal[req_id] != 0 : causal;
|
||||
const int32_t q_start_pos = seq_len - q_token_num;
|
||||
const int32_t kv_start_pos = 0;
|
||||
const int32_t kv_end_pos = seq_len;
|
||||
@@ -884,7 +884,7 @@ struct AttentionInput {
|
||||
// provided as fp32 and executed in full float precision.
|
||||
const void* s_aux;
|
||||
bool s_aux_is_bf16;
|
||||
bool* dynamic_causal;
|
||||
int32_t* dynamic_causal;
|
||||
float scale;
|
||||
bool causal;
|
||||
int32_t sliding_window_size;
|
||||
@@ -1571,7 +1571,7 @@ class AttentionMainLoop {
|
||||
const float* alibi_slopes = input->alibi_slopes;
|
||||
const void* s_aux = input->s_aux;
|
||||
const bool s_aux_is_bf16 = input->s_aux_is_bf16;
|
||||
const bool* dynamic_causal = input->dynamic_causal;
|
||||
const int32_t* dynamic_causal = input->dynamic_causal;
|
||||
const bool is_dynamic_causal = dynamic_causal != nullptr;
|
||||
|
||||
const bool causal = input->causal;
|
||||
@@ -1658,7 +1658,8 @@ class AttentionMainLoop {
|
||||
|
||||
const int32_t current_group_idx = current_workitem_group->req_id;
|
||||
const int32_t current_group_causal =
|
||||
is_dynamic_causal ? dynamic_causal[current_group_idx] : causal;
|
||||
is_dynamic_causal ? dynamic_causal[current_group_idx] != 0
|
||||
: causal;
|
||||
auto [sliding_window_left, sliding_window_right] =
|
||||
AttentionScheduler::calcu_sliding_window_size(
|
||||
sliding_window_size, current_group_causal);
|
||||
|
||||
Reference in New Issue
Block a user