[CPU] Specify required KV cache layout for CPU attention backend (#42740)
Signed-off-by: Tony Lin <tony.lin@intel.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
co-authored by
Li, Jiang <jiang1.li@intel.com>
parent
c38bed4248
commit
965d076148
@@ -25,6 +25,7 @@ from vllm.v1.attention.backend import (
|
||||
MultipleOf,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
KVCacheLayoutType,
|
||||
split_decodes_and_prefills,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec, CrossAttentionSpec
|
||||
@@ -94,6 +95,10 @@ class CPUAttentionBackend(AttentionBackend):
|
||||
) -> tuple[int, ...]:
|
||||
return 2, num_blocks, num_kv_heads, block_size, head_size
|
||||
|
||||
@classmethod
|
||||
def get_required_kv_cache_layout(cls) -> "KVCacheLayoutType | None":
|
||||
return "HND"
|
||||
|
||||
@staticmethod
|
||||
def use_cascade_attention(*args, **kwargs) -> bool:
|
||||
return False
|
||||
|
||||
Reference in New Issue
Block a user