[CPU] Adds support for fp32 attention sinks (#56252)

Signed-off-by: Ankit Jaiswal <ankit.jaiswal@amd.com>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
Ankit-Jaiswal-AMD
2026-09-23 08:55:09 -05:00
committed by GitHub
co-authored by Douglas Lehr
parent 9ed81581a8
commit 88afb9dcac
4 changed files with 95 additions and 13 deletions
+13 -1
View File
@@ -243,7 +243,19 @@ void cpu_attention_with_kv_cache(
input.block_table = block_table.data_ptr<int32_t>();
input.alibi_slopes =
alibi_slopes.has_value() ? alibi_slopes->data_ptr<float>() : nullptr;
input.s_aux = s_aux.has_value() ? s_aux->data_ptr<c10::BFloat16>() : nullptr;
// Attention sinks may be bf16 (native path) or fp32. Anything that is not
// bf16 must be provided as fp32 and is executed in full float precision.
if (s_aux.has_value()) {
TORCH_CHECK(s_aux->scalar_type() == at::ScalarType::BFloat16 ||
s_aux->scalar_type() == at::ScalarType::Float,
"cpu_attention_with_kv_cache: s_aux (attention sinks) dtype ",
s_aux->scalar_type(), " must be bfloat16 or float32");
input.s_aux = s_aux->data_ptr();
input.s_aux_is_bf16 = s_aux->scalar_type() == at::ScalarType::BFloat16;
} else {
input.s_aux = nullptr;
input.s_aux_is_bf16 = false;
}
input.dynamic_causal =
dynamic_causal.has_value() ? dynamic_causal->data_ptr<bool>() : nullptr;
input.scale = scale;