[Feat] CPU fp8 attn for AMX/AVX-512 (#39445)
Signed-off-by: Li, Tianmu <tianmu.li@intel.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
co-authored by
Claude
Li, Jiang <jiang1.li@intel.com>
parent
9d8ad5b408
commit
22524f7a92
+78
-25
@@ -1,5 +1,16 @@
|
||||
#include "cpu_attn_dispatch_generated.h"
|
||||
|
||||
// Maps kv_cache_dtype string to Fp8KVCacheDataType enum.
|
||||
// "auto" -> kAuto(0); "fp8"/"fp8_e4m3" -> kFp8E4M3; "fp8_e5m2" -> kFp8E5M2.
|
||||
static inline cpu_attention::Fp8KVCacheDataType parse_fp8_kv_dtype(
|
||||
const std::string& kv_cache_dtype) {
|
||||
if (kv_cache_dtype == "fp8_e5m2")
|
||||
return cpu_attention::Fp8KVCacheDataType::kFp8E5M2;
|
||||
if (kv_cache_dtype == "fp8_e4m3" || kv_cache_dtype == "fp8")
|
||||
return cpu_attention::Fp8KVCacheDataType::kFp8E4M3;
|
||||
return cpu_attention::Fp8KVCacheDataType::kAuto;
|
||||
}
|
||||
|
||||
torch::Tensor get_scheduler_metadata(
|
||||
const int64_t num_req, const int64_t num_heads_q,
|
||||
const int64_t num_heads_kv, const int64_t head_dim,
|
||||
@@ -49,7 +60,7 @@ torch::Tensor get_scheduler_metadata(
|
||||
input.enable_kv_split = enable_kv_split;
|
||||
|
||||
VLLM_DISPATCH_FLOATING_TYPES(dtype, "get_scheduler_metadata", [&]() {
|
||||
CPU_ATTN_DISPATCH(head_dim, isa, [&]() {
|
||||
CPU_ATTN_DISPATCH(head_dim, isa, 0, [&]() {
|
||||
input.elem_size = sizeof(scalar_t);
|
||||
input.q_buffer_elem_size = sizeof(attn_impl::q_buffer_t);
|
||||
input.logits_buffer_elem_size = sizeof(attn_impl::logits_buffer_t);
|
||||
@@ -72,7 +83,9 @@ void cpu_attn_reshape_and_cache(
|
||||
key_cache, // [num_blocks, num_kv_heads, block_size, head_size]
|
||||
torch::Tensor&
|
||||
value_cache, // [num_blocks, num_kv_heads, block_size, head_size]
|
||||
const torch::Tensor& slot_mapping, const std::string& isa) {
|
||||
const torch::Tensor& slot_mapping, const std::string& isa,
|
||||
const double k_scale = 1.0, const double v_scale = 1.0,
|
||||
const std::string& kv_cache_dtype = "auto") {
|
||||
TORCH_CHECK_EQ(key.dim(), 3);
|
||||
TORCH_CHECK_EQ(value.dim(), 3);
|
||||
TORCH_CHECK_EQ(key_cache.dim(), 4);
|
||||
@@ -80,18 +93,30 @@ void cpu_attn_reshape_and_cache(
|
||||
TORCH_CHECK_EQ(key.stride(2), 1);
|
||||
TORCH_CHECK_EQ(value.stride(2), 1);
|
||||
|
||||
const int64_t kv_cache_idx =
|
||||
static_cast<int64_t>(parse_fp8_kv_dtype(kv_cache_dtype));
|
||||
const bool is_fp8 = (kv_cache_idx != 0);
|
||||
|
||||
if (is_fp8) {
|
||||
TORCH_CHECK(key_cache.scalar_type() == at::ScalarType::Byte,
|
||||
"key_cache must be uint8 for FP8 path");
|
||||
TORCH_CHECK(value_cache.scalar_type() == at::ScalarType::Byte,
|
||||
"value_cache must be uint8 for FP8 path");
|
||||
TORCH_CHECK(k_scale > 0, "k_scale must be positive for FP8 path");
|
||||
TORCH_CHECK(v_scale > 0, "v_scale must be positive for FP8 path");
|
||||
}
|
||||
|
||||
const float k_inv = is_fp8 ? 1.0f / static_cast<float>(k_scale) : 0.0f;
|
||||
const float v_inv = is_fp8 ? 1.0f / static_cast<float>(v_scale) : 0.0f;
|
||||
|
||||
const int64_t token_num = key.size(0);
|
||||
const int64_t key_token_num_stride = key.stride(0);
|
||||
const int64_t value_token_num_stride = value.stride(0);
|
||||
const int64_t head_num = value.size(1);
|
||||
const int64_t key_head_num_stride = key.stride(1);
|
||||
const int64_t value_head_num_stride = value.stride(1);
|
||||
const int64_t head_num = key.size(1);
|
||||
const int64_t head_dim = key.size(2);
|
||||
const int64_t num_blocks = key_cache.size(0);
|
||||
const int64_t num_blocks_stride = key_cache.stride(0);
|
||||
const int64_t cache_head_num_stride = key_cache.stride(1);
|
||||
const int64_t block_size = key_cache.size(2);
|
||||
const int64_t block_size_stride = key_cache.stride(2);
|
||||
const int64_t head_dim = key.size(-1);
|
||||
|
||||
cpu_attention::ISA isa_tag = [&]() {
|
||||
if (isa == "amx") {
|
||||
@@ -109,16 +134,24 @@ void cpu_attn_reshape_and_cache(
|
||||
}
|
||||
}();
|
||||
|
||||
if (is_fp8) {
|
||||
TORCH_CHECK(isa_tag == cpu_attention::ISA::AMX ||
|
||||
isa_tag == cpu_attention::ISA::VEC,
|
||||
"FP8 KV cache is only supported on x86 (AMX/VEC) ISA");
|
||||
}
|
||||
|
||||
VLLM_DISPATCH_FLOATING_TYPES(
|
||||
key.scalar_type(), "cpu_attn_reshape_and_cache", [&]() {
|
||||
CPU_ATTN_DISPATCH(head_dim, isa_tag, [&]() {
|
||||
CPU_ATTN_DISPATCH(head_dim, isa_tag, kv_cache_idx, [&]() {
|
||||
using kv_t = typename attn_impl::kv_cache_t;
|
||||
attn_impl::reshape_and_cache(
|
||||
key.data_ptr<scalar_t>(), value.data_ptr<scalar_t>(),
|
||||
key_cache.data_ptr<scalar_t>(), value_cache.data_ptr<scalar_t>(),
|
||||
slot_mapping.data_ptr<int64_t>(), token_num, key_token_num_stride,
|
||||
value_token_num_stride, head_num, key_head_num_stride,
|
||||
value_head_num_stride, num_blocks, num_blocks_stride,
|
||||
cache_head_num_stride, block_size, block_size_stride);
|
||||
reinterpret_cast<kv_t*>(key_cache.data_ptr()),
|
||||
reinterpret_cast<kv_t*>(value_cache.data_ptr()),
|
||||
slot_mapping.data_ptr<int64_t>(), token_num, key.stride(0),
|
||||
value.stride(0), head_num, key.stride(1), value.stride(1),
|
||||
num_blocks, num_blocks_stride, cache_head_num_stride, block_size,
|
||||
block_size_stride, k_inv, v_inv);
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -137,13 +170,26 @@ void cpu_attention_with_kv_cache(
|
||||
const int64_t sliding_window_left, const int64_t sliding_window_right,
|
||||
const torch::Tensor& block_table, // [num_tokens, max_block_num]
|
||||
const double softcap, const torch::Tensor& scheduler_metadata,
|
||||
const std::optional<torch::Tensor>& s_aux // [num_heads]
|
||||
) {
|
||||
const std::optional<torch::Tensor>& s_aux, // [num_heads]
|
||||
const double k_scale = 1.0, const double v_scale = 1.0,
|
||||
const std::string& kv_cache_dtype = "auto") {
|
||||
TORCH_CHECK_EQ(query.dim(), 3);
|
||||
TORCH_CHECK_EQ(query.stride(2), 1);
|
||||
TORCH_CHECK_EQ(key_cache.dim(), 4);
|
||||
TORCH_CHECK_EQ(value_cache.dim(), 4);
|
||||
|
||||
const int64_t kv_cache_idx =
|
||||
static_cast<int64_t>(parse_fp8_kv_dtype(kv_cache_dtype));
|
||||
const bool is_fp8 = (kv_cache_idx != 0);
|
||||
if (is_fp8) {
|
||||
TORCH_CHECK(key_cache.scalar_type() == at::ScalarType::Byte,
|
||||
"key_cache must be uint8 for FP8 path");
|
||||
TORCH_CHECK(value_cache.scalar_type() == at::ScalarType::Byte,
|
||||
"value_cache must be uint8 for FP8 path");
|
||||
TORCH_CHECK(k_scale > 0, "k_scale must be positive for FP8 path");
|
||||
TORCH_CHECK(v_scale > 0, "v_scale must be positive for FP8 path");
|
||||
}
|
||||
|
||||
cpu_attention::AttentionInput input;
|
||||
input.metadata = reinterpret_cast<cpu_attention::AttentionMetadata*>(
|
||||
scheduler_metadata.data_ptr());
|
||||
@@ -165,25 +211,32 @@ void cpu_attention_with_kv_cache(
|
||||
input.block_table = block_table.data_ptr<int32_t>();
|
||||
input.alibi_slopes =
|
||||
alibi_slopes.has_value() ? alibi_slopes->data_ptr<float>() : nullptr;
|
||||
// For now sink must be bf16
|
||||
input.s_aux = s_aux.has_value() ? s_aux->data_ptr<c10::BFloat16>() : nullptr;
|
||||
input.scale = scale;
|
||||
input.causal = causal;
|
||||
input.sliding_window_left = sliding_window_left;
|
||||
input.sliding_window_right = sliding_window_right;
|
||||
if (input.causal) {
|
||||
// to make boundary calculation easier
|
||||
input.sliding_window_right = 0;
|
||||
}
|
||||
float softcap_fp32 = softcap;
|
||||
input.softcap = softcap_fp32;
|
||||
input.softcap = static_cast<float>(softcap);
|
||||
|
||||
if (is_fp8) {
|
||||
input.k_scale_fp8 = static_cast<float>(k_scale);
|
||||
input.v_scale_fp8 = static_cast<float>(v_scale);
|
||||
TORCH_CHECK(input.metadata->isa == cpu_attention::ISA::AMX ||
|
||||
input.metadata->isa == cpu_attention::ISA::VEC,
|
||||
"FP8 KV cache is only supported on x86 (AMX/VEC) ISA");
|
||||
}
|
||||
|
||||
VLLM_DISPATCH_FLOATING_TYPES(
|
||||
query.scalar_type(), "cpu_attention_with_kv_cache", [&]() {
|
||||
CPU_ATTN_DISPATCH(query.size(2), input.metadata->isa, [&]() {
|
||||
TORCH_CHECK_EQ(input.block_size % attn_impl::BlockSizeAlignment, 0);
|
||||
cpu_attention::AttentionMainLoop<attn_impl> mainloop;
|
||||
mainloop(&input);
|
||||
});
|
||||
CPU_ATTN_DISPATCH(
|
||||
query.size(2), input.metadata->isa, kv_cache_idx, [&]() {
|
||||
TORCH_CHECK_EQ(input.block_size % attn_impl::BlockSizeAlignment,
|
||||
0);
|
||||
cpu_attention::AttentionMainLoop<attn_impl> mainloop;
|
||||
mainloop(&input);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user