[CPU] [Feat] Add native AMX-FP8 attention impl for Diamond Rapids (#49410)

Signed-off-by: zhejiangxiaomai <zhenhui.zhao@intel.com>
This commit is contained in:
zhao, zhenhui
2026-09-06 11:06:56 +08:00
committed by GitHub
parent 039ea82660
commit 1c344ed41e
10 changed files with 1211 additions and 61 deletions
+30 -2
View File
@@ -1,5 +1,17 @@
#include "cpu_attn_dispatch_generated.h"
#if defined(__x86_64__) || defined(_M_X64)
#include <cpuid.h>
// Runtime check for AMX-FP8 (Diamond Rapids): CPUID leaf 7, subleaf 1, EAX[21].
static bool runtime_has_amx_fp8() {
unsigned int eax = 0, ebx = 0, ecx = 0, edx = 0;
if (__get_cpuid_count(7, 1, &eax, &ebx, &ecx, &edx)) {
return (eax >> 21) & 1u;
}
return false;
}
#endif // x86_64
// Maps kv_cache_dtype string to Fp8KVCacheDataType enum.
// "auto" -> kAuto(0); "fp8"/"fp8_e4m3" -> kFp8E4M3; "fp8_e5m2" -> kFp8E5M2.
static inline cpu_attention::Fp8KVCacheDataType parse_fp8_kv_dtype(
@@ -18,6 +30,16 @@ bool cpu_attn_has_isa(const std::string& isa) {
return true;
#else
return false;
#endif
}
if (isa == "amx_fp8") {
#ifdef CPU_CAPABILITY_AMXFP8
// Guard with runtime CPUID check: the binary may be compiled with
// -mamx-fp8 (CPU_CAPABILITY_AMXFP8 defined) but run on a CPU that
// has AMX-BF16 without AMX-FP8 (e.g. Sapphire Rapids, Emerald Rapids).
return runtime_has_amx_fp8();
#else
return false;
#endif
}
return false;
@@ -35,6 +57,8 @@ torch::Tensor get_scheduler_metadata(
cpu_attention::ISA isa;
if (isa_hint == "amx") {
isa = cpu_attention::ISA::AMX;
} else if (isa_hint == "amx_fp8") {
isa = cpu_attention::ISA::AMX_FP8;
} else if (isa_hint == "vec") {
isa = cpu_attention::ISA::VEC;
} else if (isa_hint == "vec16") {
@@ -130,6 +154,8 @@ void cpu_attn_reshape_and_cache(
cpu_attention::ISA isa_tag = [&]() {
if (isa == "amx") {
return cpu_attention::ISA::AMX;
} else if (isa == "amx_fp8") {
return cpu_attention::ISA::AMX_FP8;
} else if (isa == "vec") {
return cpu_attention::ISA::VEC;
} else if (isa == "vec16") {
@@ -149,8 +175,9 @@ void cpu_attn_reshape_and_cache(
if (is_fp8) {
TORCH_CHECK(isa_tag == cpu_attention::ISA::AMX ||
isa_tag == cpu_attention::ISA::AMX_FP8 ||
isa_tag == cpu_attention::ISA::VEC,
"FP8 KV cache is only supported on x86 (AMX/VEC) ISA");
"FP8 KV cache is only supported on x86 (AMX_FP8/AMX/VEC) ISA");
}
VLLM_DISPATCH_FLOATING_TYPES(
@@ -237,8 +264,9 @@ void cpu_attention_with_kv_cache(
input.k_scale_fp8 = static_cast<float>(k_scale);
input.v_scale_fp8 = static_cast<float>(v_scale);
TORCH_CHECK(input.metadata->isa == cpu_attention::ISA::AMX ||
input.metadata->isa == cpu_attention::ISA::AMX_FP8 ||
input.metadata->isa == cpu_attention::ISA::VEC,
"FP8 KV cache is only supported on x86 (AMX/VEC) ISA");
"FP8 KV cache is only supported on x86 (AMX_FP8/AMX/VEC) ISA");
}
VLLM_DISPATCH_FLOATING_TYPES(