[CPU] Fix FP8 attention scratchpad sizing (#50194)
Signed-off-by: Li, Tianmu <tianmu.li@intel.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
co-authored by
Codex
Li, Jiang <jiang1.li@intel.com>
parent
100d655a23
commit
65a1a16594
@@ -3814,6 +3814,7 @@ def cpu_attn_get_scheduler_metadata(
|
||||
isa: str,
|
||||
enable_kv_split: bool,
|
||||
dynamic_causal: torch.Tensor | None = None,
|
||||
kv_cache_dtype: str = "auto",
|
||||
) -> torch.Tensor:
|
||||
scheduler_metadata = torch.ops._C.get_scheduler_metadata(
|
||||
num_reqs,
|
||||
@@ -3828,6 +3829,7 @@ def cpu_attn_get_scheduler_metadata(
|
||||
isa,
|
||||
enable_kv_split,
|
||||
dynamic_causal,
|
||||
kv_cache_dtype,
|
||||
)
|
||||
return scheduler_metadata
|
||||
|
||||
|
||||
@@ -155,12 +155,12 @@ class CPUAttentionMetadataBuilder(AttentionMetadataBuilder[CPUAttentionMetadata]
|
||||
if self.window_size is None:
|
||||
self.window_size = -1
|
||||
self.block_size = vllm_config.cache_config.block_size
|
||||
kv_cache_dtype_str = vllm_config.cache_config.cache_dtype
|
||||
self.kv_cache_dtype = vllm_config.cache_config.cache_dtype
|
||||
self.isa = _get_attn_isa(
|
||||
self.dtype,
|
||||
self.block_size,
|
||||
self.head_dim,
|
||||
kv_cache_dtype_str,
|
||||
self.kv_cache_dtype,
|
||||
)
|
||||
self.is_cross_attention = isinstance(kv_cache_spec, CrossAttentionSpec)
|
||||
self.is_encoder_only_attention = isinstance(
|
||||
@@ -234,6 +234,7 @@ class CPUAttentionMetadataBuilder(AttentionMetadataBuilder[CPUAttentionMetadata]
|
||||
isa=self.isa,
|
||||
enable_kv_split=envs.VLLM_CPU_ATTN_SPLIT_KV,
|
||||
dynamic_causal=dynamic_casual,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
)
|
||||
|
||||
attn_metadata = CPUAttentionMetadata(
|
||||
|
||||
Reference in New Issue
Block a user