[CPU] [Feat] Add native AMX-FP8 attention impl for Diamond Rapids (#49410)

Signed-off-by: zhejiangxiaomai <zhenhui.zhao@intel.com>
This commit is contained in:
zhao, zhenhui
2026-09-06 11:06:56 +08:00
committed by GitHub
parent 039ea82660
commit 1c344ed41e
10 changed files with 1211 additions and 61 deletions
+30 -9
View File
@@ -22,6 +22,7 @@ ISA_TYPES = {
"VXE": 4,
"RVV": 5,
"VSX": 6,
"AMX_FP8": 7,
}
# KV cache index: 0 = auto (same as scalar_t), 1 = fp8_e4m3, 2 = fp8_e5m2
@@ -39,13 +40,13 @@ KV_CACHE_CPP_TYPES = {
}
# ISAs supported for head_dims divisible by 32
ISA_FOR_32 = ["AMX", "NEON", "VEC", "VEC16", "VXE", "RVV", "VSX"]
ISA_FOR_32 = ["AMX", "AMX_FP8", "NEON", "VEC", "VEC16", "VXE", "RVV", "VSX"]
# ISAs supported for head_dims divisible by 16 only
ISA_FOR_16 = ["VEC16"]
# ISAs that support FP8 KV cache (x86 AVX2/AVX-512 required)
ISA_FOR_FP8 = ["AMX", "VEC"]
ISA_FOR_FP8 = ["AMX_FP8", "AMX", "VEC"]
def encode_params(head_dim: int, isa_type: str, kv_cache: str = "auto") -> int:
@@ -64,12 +65,21 @@ def _make_case(
encoded = encode_params(head_dim, isa, kv_cache)
actual_isa = isa_override if isa_override else isa
cpp_type = KV_CACHE_CPP_TYPES[kv_cache]
attn_impl = (
f"cpu_attention::AttentionImpl<"
f"cpu_attention::ISA::{actual_isa}, \\\n"
f" "
native_impl = (
f"cpu_attention::AttentionImpl<cpu_attention::ISA::{actual_isa}, "
f"scalar_t, head_dim, {cpp_type}>"
)
if actual_isa == "AMX_FP8":
fallback_impl = (
f"cpu_attention::AttentionImpl<cpu_attention::ISA::AMX, "
f"scalar_t, head_dim, {cpp_type}>"
)
attn_impl = (
"std::conditional_t<std::is_same_v<scalar_t, c10::BFloat16>, "
f"{native_impl}, {fallback_impl}>"
)
else:
attn_impl = native_impl
comment = (
f"head_dim={head_dim}, isa={isa}"
if kv_cache == "auto"
@@ -98,7 +108,8 @@ def generate_cases_for_isa_group(isa_list: list[str], include_fp8: bool = False)
for isa in isa_list:
if isa not in ISA_FOR_32:
continue
cases.append(_make_case(head_dim, isa, "auto"))
isa_override = "AMX" if isa == "AMX_FP8" else None
cases.append(_make_case(head_dim, isa, "auto", isa_override))
# Non-FP8 cases for head_dims divisible by 16 only
for head_dim in HEAD_DIMS_16:
@@ -112,7 +123,8 @@ def generate_cases_for_isa_group(isa_list: list[str], include_fp8: bool = False)
for isa in isa_list:
if isa not in ISA_FOR_FP8:
continue
cases.append(_make_case(head_dim, isa, fp8_type))
isa_override = "AMX" if isa == "AMX_FP8" and head_dim % 64 else None
cases.append(_make_case(head_dim, isa, fp8_type, isa_override))
return "\n".join(cases)
@@ -138,6 +150,10 @@ def generate_header_file() -> str:
#include "cpu_attn_vec.hpp"
#include "cpu_attn_vec16.hpp"
#ifdef CPU_CAPABILITY_AMXFP8
#include "cpu_attn_amx_fp8.hpp"
#endif
#ifdef CPU_CAPABILITY_AMXBF16
#include "cpu_attn_amx.hpp"
#endif
@@ -206,7 +222,12 @@ def generate_header_file() -> str:
)
header += _macro_block(
"#if defined(CPU_CAPABILITY_AMXBF16)",
"#if defined(CPU_CAPABILITY_AMXFP8)",
["AMX_FP8", "AMX", "VEC", "VEC16"],
fp8=True,
)
header += _macro_block(
"#elif defined(CPU_CAPABILITY_AMXBF16)",
["AMX", "VEC", "VEC16"],
fp8=True,
)