[CPU] [Feat] Add native AMX-FP8 attention impl for Diamond Rapids (#49410)

Signed-off-by: zhejiangxiaomai <zhenhui.zhao@intel.com>
This commit is contained in:
zhao, zhenhui
2026-09-06 11:06:56 +08:00
committed by GitHub
parent 039ea82660
commit 1c344ed41e
10 changed files with 1211 additions and 61 deletions
+54 -36
View File
@@ -12,7 +12,7 @@
#include "cpu/utils.hpp"
namespace cpu_attention {
enum class ISA { AMX, VEC, VEC16, NEON, VXE, RVV, VSX };
enum class ISA { AMX, VEC, VEC16, NEON, VXE, RVV, VSX, AMX_FP8 };
// Mirrors csrc/attention/dtype_fp8.cuh Fp8KVCacheDataType exactly.
enum class Fp8KVCacheDataType {
@@ -152,6 +152,9 @@ struct AttentionMetadata {
case ISA::AMX:
ss << "AMX, ";
break;
case ISA::AMX_FP8:
ss << "AMX_FP8, ";
break;
case ISA::VEC:
ss << "VEC, ";
break;
@@ -885,6 +888,8 @@ struct AttentionInput {
// FP8 KV cache scales (used by FP8 attention implementations)
float k_scale_fp8 = 1.0f;
float v_scale_fp8 = 1.0f;
// FP8 query scale (used by AMX_FP8 which quantizes query inside the kernel)
float q_scale_fp8 = 1.0f;
};
#define DEFINE_CPU_ATTENTION_PARAMS \
@@ -1092,11 +1097,6 @@ class AttentionMainLoop {
// process logits
{
// if (debug_info){
// print_logits("raw logits", logits_buffer, q_head_num,
// kv_tile_token_num, kv_tile_token_num);
// }
if (softcap_scale != 0.0f) {
apply_softcap(logits_buffer, kv_tile_token_num, q_head_num,
kv_tile_token_num, softcap_scale);
@@ -1127,30 +1127,33 @@ class AttentionMainLoop {
apply_softmax(logits_buffer, partial_q_buffer, max_buffer, sum_buffer,
kv_tile_token_num, q_head_num, kv_tile_token_num,
is_first_iter, use_sink);
// if (debug_info){
// print_logits("softmax logits",
// reinterpret_cast<prob_buffer_t*>(logits_buffer), q_head_num,
// kv_tile_token_num, kv_tile_token_num * sizeof(logits_buffer_t) /
// sizeof(prob_buffer_t));
// print_logits("new_max", max_buffer, 1, q_head_num, q_head_num);
// print_logits("new_sum", sum_buffer, 1, q_head_num, q_head_num);
// }
}
// compute P@V
{
constexpr bool prequantize_probabilities = []() {
if constexpr (requires { tile_gemm_t::prequantize_probabilities; }) {
return tile_gemm_t::prequantize_probabilities;
}
return false;
}();
int32_t curr_group_offset =
start_block_group_offset * v_cache_token_group_stride;
int32_t curr_group_num_in_block =
token_group_num_per_block - start_block_group_offset;
int32_t remaining_group_num = token_group_num;
int32_t head_dim_group_num = head_dim / headdim_alignment;
prob_buffer_t* curr_prob_buffer =
reinterpret_cast<prob_buffer_t*>(logits_buffer);
int64_t prob_buffer_stride =
kv_tile_token_num *
(sizeof(logits_buffer_t) / sizeof(prob_buffer_t));
using pv_prob_buffer_t = std::conditional_t<prequantize_probabilities,
uint8_t, prob_buffer_t>;
pv_prob_buffer_t* curr_prob_buffer =
reinterpret_cast<pv_prob_buffer_t*>(logits_buffer);
const int64_t prob_buffer_stride =
prequantize_probabilities
? kv_tile_token_num
: kv_tile_token_num *
(sizeof(logits_buffer_t) / sizeof(prob_buffer_t));
constexpr int64_t prob_buffer_elem_size =
prequantize_probabilities ? sizeof(uint8_t) : sizeof(prob_buffer_t);
partial_output_buffer_t* curr_partial_q_buffer = partial_q_buffer;
bool accum_c = !is_first_iter;
for (int32_t block_idx = start_block_idx; block_idx < end_block_idx;
@@ -1184,15 +1187,13 @@ class AttentionMainLoop {
remaining_group_num -= curr_group_num_in_block;
curr_group_offset = 0;
curr_group_num_in_block = token_group_num_per_block;
curr_prob_buffer += curr_token_num;
curr_prob_buffer = reinterpret_cast<pv_prob_buffer_t*>(
reinterpret_cast<uint8_t*>(curr_prob_buffer) +
curr_token_num * prob_buffer_elem_size);
curr_partial_q_buffer = partial_q_buffer;
accum_c = true;
}
}
// if (debug_info) {
// print_logits("output", partial_q_buffer, q_head_num, head_dim,
// head_dim);
// }
}
void apply_mask(logits_buffer_t* __restrict__ logits_buffer,
@@ -1262,9 +1263,19 @@ class AttentionMainLoop {
#endif
using prob_buffer_vec_t = typename VecTypeTrait<prob_buffer_t>::vec_t;
constexpr bool prequantize_probabilities = []() {
if constexpr (requires { tile_gemm_t::prequantize_probabilities; }) {
return tile_gemm_t::prequantize_probabilities;
}
return false;
}();
using softmax_prob_buffer_t =
std::conditional_t<prequantize_probabilities, uint8_t, prob_buffer_t>;
static_assert(sizeof(prob_buffer_t) <= sizeof(logits_buffer_t));
logits_buffer_t* __restrict__ curr_logits_buffer = logits_buffer;
softmax_prob_buffer_t* __restrict__ curr_prob_buffer =
reinterpret_cast<softmax_prob_buffer_t*>(logits_buffer);
float* __restrict__ curr_partial_q_buffer = partial_q_buffer;
const int32_t vec_num = kv_tile_token_num / 16;
const int32_t head_vec_num = head_dim / 16;
@@ -1302,8 +1313,8 @@ class AttentionMainLoop {
{
logits_buffer_t* __restrict__ curr_logits_buffer_iter =
curr_logits_buffer;
prob_buffer_t* __restrict__ curr_prob_buffer_iter =
reinterpret_cast<prob_buffer_t*>(curr_logits_buffer);
softmax_prob_buffer_t* __restrict__ curr_prob_buffer_iter =
curr_prob_buffer;
for (int32_t j = 0; j < vec_num; ++j) {
vec_op::FP32Vec16 vec(curr_logits_buffer_iter);
vec = vec - max_vec;
@@ -1320,8 +1331,12 @@ class AttentionMainLoop {
vec = fast_exp(vec);
}
prob_buffer_vec_t output_vec(vec);
output_vec.save(curr_prob_buffer_iter);
if constexpr (prequantize_probabilities) {
vec.save(curr_logits_buffer_iter);
} else {
prob_buffer_vec_t output_vec(vec);
output_vec.save(curr_prob_buffer_iter);
}
#else
vec.save(curr_logits_buffer_iter);
for (int32_t k = 0; k < 16; ++k) {
@@ -1335,6 +1350,10 @@ class AttentionMainLoop {
curr_logits_buffer_iter += 16;
curr_prob_buffer_iter += 16;
}
if constexpr (prequantize_probabilities) {
tile_gemm_t::quantize_probability_row(
curr_logits_buffer, curr_prob_buffer, kv_tile_token_num);
}
}
float new_sum_val = sum_vec.reduce_sum();
@@ -1366,6 +1385,12 @@ class AttentionMainLoop {
sum_buffer[i] = new_sum_val;
curr_logits_buffer += logits_buffer_stride;
if constexpr (prequantize_probabilities) {
curr_prob_buffer += kv_tile_token_num;
} else {
curr_prob_buffer =
reinterpret_cast<softmax_prob_buffer_t*>(curr_logits_buffer);
}
curr_partial_q_buffer += head_dim;
}
}
@@ -1840,13 +1865,6 @@ class AttentionMainLoop {
float* curr_sum_buffer = sum_buffer + q_tile_head_offset;
bool debug_info = false;
// bool debug_info = (
// q_head_start_idx == 4 &&
// (q_token_start_idx + q_head_tile_token_offset) <=
// 4
// && (q_token_start_idx + q_head_tile_token_offset +
// q_tile_token_num) > 4
// );
// if (debug_info) {
// std::printf("\tq_iter_idx: %d, q_token_start: %d,"
// "q_token_end: %d, q_token_num: %d, q_head_num: %d,"