[CPU] [Feat] Add native AMX-FP8 attention impl for Diamond Rapids (#49410)
Signed-off-by: zhejiangxiaomai <zhenhui.zhao@intel.com>
This commit is contained in:
+30
-2
@@ -1,5 +1,17 @@
|
||||
#include "cpu_attn_dispatch_generated.h"
|
||||
|
||||
#if defined(__x86_64__) || defined(_M_X64)
|
||||
#include <cpuid.h>
|
||||
// Runtime check for AMX-FP8 (Diamond Rapids): CPUID leaf 7, subleaf 1, EAX[21].
|
||||
static bool runtime_has_amx_fp8() {
|
||||
unsigned int eax = 0, ebx = 0, ecx = 0, edx = 0;
|
||||
if (__get_cpuid_count(7, 1, &eax, &ebx, &ecx, &edx)) {
|
||||
return (eax >> 21) & 1u;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
#endif // x86_64
|
||||
|
||||
// Maps kv_cache_dtype string to Fp8KVCacheDataType enum.
|
||||
// "auto" -> kAuto(0); "fp8"/"fp8_e4m3" -> kFp8E4M3; "fp8_e5m2" -> kFp8E5M2.
|
||||
static inline cpu_attention::Fp8KVCacheDataType parse_fp8_kv_dtype(
|
||||
@@ -18,6 +30,16 @@ bool cpu_attn_has_isa(const std::string& isa) {
|
||||
return true;
|
||||
#else
|
||||
return false;
|
||||
#endif
|
||||
}
|
||||
if (isa == "amx_fp8") {
|
||||
#ifdef CPU_CAPABILITY_AMXFP8
|
||||
// Guard with runtime CPUID check: the binary may be compiled with
|
||||
// -mamx-fp8 (CPU_CAPABILITY_AMXFP8 defined) but run on a CPU that
|
||||
// has AMX-BF16 without AMX-FP8 (e.g. Sapphire Rapids, Emerald Rapids).
|
||||
return runtime_has_amx_fp8();
|
||||
#else
|
||||
return false;
|
||||
#endif
|
||||
}
|
||||
return false;
|
||||
@@ -35,6 +57,8 @@ torch::Tensor get_scheduler_metadata(
|
||||
cpu_attention::ISA isa;
|
||||
if (isa_hint == "amx") {
|
||||
isa = cpu_attention::ISA::AMX;
|
||||
} else if (isa_hint == "amx_fp8") {
|
||||
isa = cpu_attention::ISA::AMX_FP8;
|
||||
} else if (isa_hint == "vec") {
|
||||
isa = cpu_attention::ISA::VEC;
|
||||
} else if (isa_hint == "vec16") {
|
||||
@@ -130,6 +154,8 @@ void cpu_attn_reshape_and_cache(
|
||||
cpu_attention::ISA isa_tag = [&]() {
|
||||
if (isa == "amx") {
|
||||
return cpu_attention::ISA::AMX;
|
||||
} else if (isa == "amx_fp8") {
|
||||
return cpu_attention::ISA::AMX_FP8;
|
||||
} else if (isa == "vec") {
|
||||
return cpu_attention::ISA::VEC;
|
||||
} else if (isa == "vec16") {
|
||||
@@ -149,8 +175,9 @@ void cpu_attn_reshape_and_cache(
|
||||
|
||||
if (is_fp8) {
|
||||
TORCH_CHECK(isa_tag == cpu_attention::ISA::AMX ||
|
||||
isa_tag == cpu_attention::ISA::AMX_FP8 ||
|
||||
isa_tag == cpu_attention::ISA::VEC,
|
||||
"FP8 KV cache is only supported on x86 (AMX/VEC) ISA");
|
||||
"FP8 KV cache is only supported on x86 (AMX_FP8/AMX/VEC) ISA");
|
||||
}
|
||||
|
||||
VLLM_DISPATCH_FLOATING_TYPES(
|
||||
@@ -237,8 +264,9 @@ void cpu_attention_with_kv_cache(
|
||||
input.k_scale_fp8 = static_cast<float>(k_scale);
|
||||
input.v_scale_fp8 = static_cast<float>(v_scale);
|
||||
TORCH_CHECK(input.metadata->isa == cpu_attention::ISA::AMX ||
|
||||
input.metadata->isa == cpu_attention::ISA::AMX_FP8 ||
|
||||
input.metadata->isa == cpu_attention::ISA::VEC,
|
||||
"FP8 KV cache is only supported on x86 (AMX/VEC) ISA");
|
||||
"FP8 KV cache is only supported on x86 (AMX_FP8/AMX/VEC) ISA");
|
||||
}
|
||||
|
||||
VLLM_DISPATCH_FLOATING_TYPES(
|
||||
|
||||
Reference in New Issue
Block a user