[CPU] Adds support for fp32 attention sinks (#56252)

Signed-off-by: Ankit Jaiswal <ankit.jaiswal@amd.com>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
Ankit-Jaiswal-AMD
2026-09-23 08:55:09 -05:00
committed by GitHub
co-authored by Douglas Lehr
parent 9ed81581a8
commit 88afb9dcac
4 changed files with 95 additions and 13 deletions
+13 -1
View File
@@ -243,7 +243,19 @@ void cpu_attention_with_kv_cache(
input.block_table = block_table.data_ptr<int32_t>();
input.alibi_slopes =
alibi_slopes.has_value() ? alibi_slopes->data_ptr<float>() : nullptr;
input.s_aux = s_aux.has_value() ? s_aux->data_ptr<c10::BFloat16>() : nullptr;
// Attention sinks may be bf16 (native path) or fp32. Anything that is not
// bf16 must be provided as fp32 and is executed in full float precision.
if (s_aux.has_value()) {
TORCH_CHECK(s_aux->scalar_type() == at::ScalarType::BFloat16 ||
s_aux->scalar_type() == at::ScalarType::Float,
"cpu_attention_with_kv_cache: s_aux (attention sinks) dtype ",
s_aux->scalar_type(), " must be bfloat16 or float32");
input.s_aux = s_aux->data_ptr();
input.s_aux_is_bf16 = s_aux->scalar_type() == at::ScalarType::BFloat16;
} else {
input.s_aux = nullptr;
input.s_aux_is_bf16 = false;
}
input.dynamic_causal =
dynamic_causal.has_value() ? dynamic_causal->data_ptr<bool>() : nullptr;
input.scale = scale;
+24 -9
View File
@@ -879,7 +879,11 @@ struct AttentionInput {
int32_t* seq_lens;
int32_t* block_table;
float* alibi_slopes;
c10::BFloat16* s_aux;
// Attention sinks pointer. May reference bf16 or fp32 data depending on
// `s_aux_is_bf16`. bf16 sinks keep the native bf16 path; anything else is
// provided as fp32 and executed in full float precision.
const void* s_aux;
bool s_aux_is_bf16;
bool* dynamic_causal;
float scale;
bool causal;
@@ -1565,7 +1569,8 @@ class AttentionMainLoop {
const float scale = input->scale;
const float softcap_scale = input->softcap;
const float* alibi_slopes = input->alibi_slopes;
const c10::BFloat16* s_aux = input->s_aux;
const void* s_aux = input->s_aux;
const bool s_aux_is_bf16 = input->s_aux_is_bf16;
const bool* dynamic_causal = input->dynamic_causal;
const bool is_dynamic_causal = dynamic_causal != nullptr;
@@ -1773,9 +1778,6 @@ class AttentionMainLoop {
const float* curr_alibi_slopes =
(alibi_slopes != nullptr ? alibi_slopes + q_head_start_idx
: nullptr);
const c10::BFloat16* curr_s_aux =
(s_aux != nullptr ? s_aux + q_head_start_idx : nullptr);
// copy the Q tile to q_buffer, the logical layout of q_buffer is
// [actual_q_token_num, curr_q_heads_per_kv, head_dim]
{
@@ -1787,10 +1789,23 @@ class AttentionMainLoop {
if (use_sink) {
alignas(64) float s_aux_fp32[16];
// All other platforms have BF16Vec16 available
vec_op::BF16Vec16 vec_bf16(curr_s_aux);
vec_op::FP32Vec16 vec_fp32(vec_bf16);
vec_fp32.save(s_aux_fp32);
// Sinks may be provided as bf16 or fp32. Either way they are
// loaded into fp32: bf16 sinks are up-converted (native
// BF16Vec16 path, available on all platforms), while fp32
// sinks are loaded directly and executed in full precision.
if (s_aux_is_bf16) {
const c10::BFloat16* curr_s_aux =
static_cast<const c10::BFloat16*>(s_aux) +
q_head_start_idx;
vec_op::BF16Vec16 vec_bf16(curr_s_aux);
vec_op::FP32Vec16 vec_fp32(vec_bf16);
vec_fp32.save(s_aux_fp32);
} else {
const float* curr_s_aux =
static_cast<const float*>(s_aux) + q_head_start_idx;
vec_op::FP32Vec16 vec_fp32(curr_s_aux);
vec_fp32.save(s_aux_fp32);
}
float* __restrict__ curr_sum_buffer = sum_buffer;
float* __restrict__ curr_max_buffer = max_buffer;