[Feat] CPU fp8 attn for AMX/AVX-512 (#39445)

Signed-off-by: Li, Tianmu <tianmu.li@intel.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
Tianmu Li
2026-04-29 20:43:21 +08:00
committed by GitHub
co-authored by Claude Li, Jiang <jiang1.li@intel.com>
parent 9d8ad5b408
commit 22524f7a92
19 changed files with 1067 additions and 283 deletions
+11 -5
View File
@@ -101,7 +101,9 @@ void cpu_attn_reshape_and_cache(const torch::Tensor& key,
torch::Tensor& key_cache,
torch::Tensor& value_cache,
const torch::Tensor& slot_mapping,
const std::string& isa);
const std::string& isa, const double k_scale,
const double v_scale,
const std::string& kv_cache_dtype);
void cpu_attention_with_kv_cache(
const torch::Tensor& query, const torch::Tensor& key_cache,
@@ -112,7 +114,8 @@ void cpu_attention_with_kv_cache(
const int64_t sliding_window_left, const int64_t sliding_window_right,
const torch::Tensor& block_table, const double softcap,
const torch::Tensor& scheduler_metadata,
const std::optional<torch::Tensor>& s_aux);
const std::optional<torch::Tensor>& s_aux, const double k_scale,
const double v_scale, const std::string& kv_cache_dtype);
// Note: just for avoiding importing errors
void placeholder_op() { TORCH_CHECK(false, "Unimplemented"); }
@@ -384,15 +387,18 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
&get_scheduler_metadata);
ops.def(
"cpu_attn_reshape_and_cache(Tensor key, Tensor value, Tensor(a2!) "
"key_cache, Tensor(a3!) value_cache, Tensor slot_mapping, str "
"isa) -> ()",
"key_cache, Tensor(a3!) value_cache, Tensor slot_mapping, str isa, "
"float k_scale=1.0, float v_scale=1.0, str kv_cache_dtype=\"auto\") -> "
"()",
&cpu_attn_reshape_and_cache);
ops.def(
"cpu_attention_with_kv_cache(Tensor query, Tensor key_cache, Tensor "
"value_cache, Tensor(a3!) output, Tensor query_start_loc, Tensor "
"seq_lens, float scale, bool causal, Tensor? alibi_slopes, SymInt "
"sliding_window_left, SymInt sliding_window_right, Tensor block_table, "
"float softcap, Tensor scheduler_metadata, Tensor? s_aux) -> ()",
"float softcap, Tensor scheduler_metadata, Tensor? s_aux, "
"float k_scale=1.0, float v_scale=1.0, str kv_cache_dtype=\"auto\") -> "
"()",
&cpu_attention_with_kv_cache);
// placeholders