[CPU] Fix FP8 attention scratchpad sizing (#50194)

Signed-off-by: Li, Tianmu <tianmu.li@intel.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
Tianmu Li
2026-07-29 15:34:37 +08:00
committed by GitHub
co-authored by Codex Li, Jiang <jiang1.li@intel.com>
parent 100d655a23
commit 65a1a16594
5 changed files with 35 additions and 7 deletions
+6 -3
View File
@@ -30,7 +30,8 @@ torch::Tensor get_scheduler_metadata(
const torch::Tensor& query_start_loc, const bool causal,
const int64_t window_size, const std::string& isa_hint,
const bool enable_kv_split,
const std::optional<torch::Tensor>& dynamic_causal) {
const std::optional<torch::Tensor>& dynamic_causal,
const std::string& kv_cache_dtype) {
cpu_attention::ISA isa;
if (isa_hint == "amx") {
isa = cpu_attention::ISA::AMX;
@@ -65,9 +66,11 @@ torch::Tensor get_scheduler_metadata(
input.dynamic_causal =
dynamic_causal.has_value() ? dynamic_causal->data_ptr<bool>() : nullptr;
const int64_t kv_cache_idx =
static_cast<int64_t>(parse_fp8_kv_dtype(kv_cache_dtype));
VLLM_DISPATCH_FLOATING_TYPES(dtype, "get_scheduler_metadata", [&]() {
CPU_ATTN_DISPATCH(head_dim, isa, 0, [&]() {
input.elem_size = sizeof(scalar_t);
CPU_ATTN_DISPATCH(head_dim, isa, kv_cache_idx, [&]() {
input.elem_size = sizeof(attn_impl::kv_cache_t);
input.q_buffer_elem_size = sizeof(attn_impl::q_buffer_t);
input.logits_buffer_elem_size = sizeof(attn_impl::logits_buffer_t);
input.output_buffer_elem_size =