[Feat] CPU fp8 attn for AMX/AVX-512 (#39445)
Signed-off-by: Li, Tianmu <tianmu.li@intel.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
co-authored by
Claude
Li, Jiang <jiang1.li@intel.com>
parent
9d8ad5b408
commit
22524f7a92
+4
-15
@@ -16,7 +16,6 @@ from vllm.utils.cpu_resource_utils import (
|
||||
get_memory_node_info,
|
||||
)
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
from .interface import CpuArchEnum, Platform, PlatformEnum
|
||||
@@ -134,20 +133,6 @@ class CpuPlatform(Platform):
|
||||
scheduler_config = vllm_config.scheduler_config
|
||||
# async scheduling is not required on CPU
|
||||
scheduler_config.async_scheduling = False
|
||||
if (
|
||||
scheduler_config.enable_chunked_prefill
|
||||
or cache_config.enable_prefix_caching
|
||||
) and is_quantized_kv_cache(cache_config.cache_dtype):
|
||||
raise RuntimeError(
|
||||
"Chunked-prefill and prefix-cache on the CPU "
|
||||
"backend is not compatible with FP8 KV cache."
|
||||
)
|
||||
|
||||
if is_quantized_kv_cache(cache_config.cache_dtype):
|
||||
logger.warning(
|
||||
"CPU backend doesn't support KV cache quantization fallback to auto."
|
||||
)
|
||||
cache_config.cache_dtype = "auto"
|
||||
|
||||
parallel_config = vllm_config.parallel_config
|
||||
# OMP requires the MP executor to function correctly, UniProc is not
|
||||
@@ -458,6 +443,10 @@ class CpuPlatform(Platform):
|
||||
block_offsets.reshape(1, block_size)
|
||||
+ indices.reshape(num_blocks, 1) * block_size
|
||||
).flatten()
|
||||
if key_cache.dtype == torch.uint8:
|
||||
raise NotImplementedError(
|
||||
"FP8 KV cache is not yet supported with KV transfer on CPU"
|
||||
)
|
||||
cpu_attn_reshape_and_cache(
|
||||
key,
|
||||
value,
|
||||
|
||||
Reference in New Issue
Block a user