[CPU] [Feat] Add native AMX-FP8 attention impl for Diamond Rapids (#49410)
Signed-off-by: zhejiangxiaomai <zhenhui.zhao@intel.com>
This commit is contained in:
@@ -22,6 +22,7 @@ ISA_TYPES = {
|
||||
"VXE": 4,
|
||||
"RVV": 5,
|
||||
"VSX": 6,
|
||||
"AMX_FP8": 7,
|
||||
}
|
||||
|
||||
# KV cache index: 0 = auto (same as scalar_t), 1 = fp8_e4m3, 2 = fp8_e5m2
|
||||
@@ -39,13 +40,13 @@ KV_CACHE_CPP_TYPES = {
|
||||
}
|
||||
|
||||
# ISAs supported for head_dims divisible by 32
|
||||
ISA_FOR_32 = ["AMX", "NEON", "VEC", "VEC16", "VXE", "RVV", "VSX"]
|
||||
ISA_FOR_32 = ["AMX", "AMX_FP8", "NEON", "VEC", "VEC16", "VXE", "RVV", "VSX"]
|
||||
|
||||
# ISAs supported for head_dims divisible by 16 only
|
||||
ISA_FOR_16 = ["VEC16"]
|
||||
|
||||
# ISAs that support FP8 KV cache (x86 AVX2/AVX-512 required)
|
||||
ISA_FOR_FP8 = ["AMX", "VEC"]
|
||||
ISA_FOR_FP8 = ["AMX_FP8", "AMX", "VEC"]
|
||||
|
||||
|
||||
def encode_params(head_dim: int, isa_type: str, kv_cache: str = "auto") -> int:
|
||||
@@ -64,12 +65,21 @@ def _make_case(
|
||||
encoded = encode_params(head_dim, isa, kv_cache)
|
||||
actual_isa = isa_override if isa_override else isa
|
||||
cpp_type = KV_CACHE_CPP_TYPES[kv_cache]
|
||||
attn_impl = (
|
||||
f"cpu_attention::AttentionImpl<"
|
||||
f"cpu_attention::ISA::{actual_isa}, \\\n"
|
||||
f" "
|
||||
native_impl = (
|
||||
f"cpu_attention::AttentionImpl<cpu_attention::ISA::{actual_isa}, "
|
||||
f"scalar_t, head_dim, {cpp_type}>"
|
||||
)
|
||||
if actual_isa == "AMX_FP8":
|
||||
fallback_impl = (
|
||||
f"cpu_attention::AttentionImpl<cpu_attention::ISA::AMX, "
|
||||
f"scalar_t, head_dim, {cpp_type}>"
|
||||
)
|
||||
attn_impl = (
|
||||
"std::conditional_t<std::is_same_v<scalar_t, c10::BFloat16>, "
|
||||
f"{native_impl}, {fallback_impl}>"
|
||||
)
|
||||
else:
|
||||
attn_impl = native_impl
|
||||
comment = (
|
||||
f"head_dim={head_dim}, isa={isa}"
|
||||
if kv_cache == "auto"
|
||||
@@ -98,7 +108,8 @@ def generate_cases_for_isa_group(isa_list: list[str], include_fp8: bool = False)
|
||||
for isa in isa_list:
|
||||
if isa not in ISA_FOR_32:
|
||||
continue
|
||||
cases.append(_make_case(head_dim, isa, "auto"))
|
||||
isa_override = "AMX" if isa == "AMX_FP8" else None
|
||||
cases.append(_make_case(head_dim, isa, "auto", isa_override))
|
||||
|
||||
# Non-FP8 cases for head_dims divisible by 16 only
|
||||
for head_dim in HEAD_DIMS_16:
|
||||
@@ -112,7 +123,8 @@ def generate_cases_for_isa_group(isa_list: list[str], include_fp8: bool = False)
|
||||
for isa in isa_list:
|
||||
if isa not in ISA_FOR_FP8:
|
||||
continue
|
||||
cases.append(_make_case(head_dim, isa, fp8_type))
|
||||
isa_override = "AMX" if isa == "AMX_FP8" and head_dim % 64 else None
|
||||
cases.append(_make_case(head_dim, isa, fp8_type, isa_override))
|
||||
|
||||
return "\n".join(cases)
|
||||
|
||||
@@ -138,6 +150,10 @@ def generate_header_file() -> str:
|
||||
#include "cpu_attn_vec.hpp"
|
||||
#include "cpu_attn_vec16.hpp"
|
||||
|
||||
#ifdef CPU_CAPABILITY_AMXFP8
|
||||
#include "cpu_attn_amx_fp8.hpp"
|
||||
#endif
|
||||
|
||||
#ifdef CPU_CAPABILITY_AMXBF16
|
||||
#include "cpu_attn_amx.hpp"
|
||||
#endif
|
||||
@@ -206,7 +222,12 @@ def generate_header_file() -> str:
|
||||
)
|
||||
|
||||
header += _macro_block(
|
||||
"#if defined(CPU_CAPABILITY_AMXBF16)",
|
||||
"#if defined(CPU_CAPABILITY_AMXFP8)",
|
||||
["AMX_FP8", "AMX", "VEC", "VEC16"],
|
||||
fp8=True,
|
||||
)
|
||||
header += _macro_block(
|
||||
"#elif defined(CPU_CAPABILITY_AMXBF16)",
|
||||
["AMX", "VEC", "VEC16"],
|
||||
fp8=True,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user