Support FP8 KVCache on XPU (#37731)
Signed-off-by: Xinyu Chen <xinyu1.chen@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
co-authored by
Kunshang Ji
parent
0f7be0f2f7
commit
639402f5a2
@@ -35,6 +35,7 @@ steps:
|
||||
python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager -tp 2 --distributed-executor-backend mp &&
|
||||
python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --attention-backend=TRITON_ATTN &&
|
||||
python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8 &&
|
||||
python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --kv-cache-dtype fp8 &&
|
||||
python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --block-size 64 --enforce-eager --max-model-len 8192 &&
|
||||
python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 &&
|
||||
python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 --enable-expert-parallel'
|
||||
|
||||
@@ -258,6 +258,9 @@ class xpu_ops:
|
||||
# alibi_slopes = alibi_slopes,
|
||||
# softcap=softcap,
|
||||
return_softmax_lse=return_softmax_lse,
|
||||
q_descale=q_descale,
|
||||
k_descale=k_descale,
|
||||
v_descale=v_descale,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -166,12 +166,18 @@ def is_fa_version_supported(fa_version: int) -> bool:
|
||||
|
||||
|
||||
def flash_attn_supports_fp8() -> bool:
|
||||
if current_platform.is_xpu():
|
||||
return True
|
||||
return (
|
||||
get_flash_attn_version() == 3
|
||||
and current_platform.is_device_capability_family(90)
|
||||
)
|
||||
|
||||
|
||||
def flash_attn_supports_quant_query_input() -> bool:
|
||||
return not current_platform.is_xpu()
|
||||
|
||||
|
||||
def flash_attn_supports_sinks() -> bool:
|
||||
if current_platform.is_xpu():
|
||||
return True
|
||||
|
||||
@@ -20,6 +20,7 @@ from vllm.v1.attention.backend import (
|
||||
)
|
||||
from vllm.v1.attention.backends.fa_utils import (
|
||||
flash_attn_supports_fp8,
|
||||
flash_attn_supports_quant_query_input,
|
||||
get_flash_attn_version,
|
||||
is_fa_version_supported,
|
||||
is_flash_attn_varlen_func_available,
|
||||
@@ -656,7 +657,7 @@ class FlashAttentionImpl(AttentionImpl):
|
||||
"heads in the layer"
|
||||
)
|
||||
|
||||
self.supports_quant_query_input = True
|
||||
self.supports_quant_query_input = flash_attn_supports_quant_query_input()
|
||||
|
||||
vllm_config = get_current_vllm_config_or_none()
|
||||
dcp_a2a = (
|
||||
@@ -757,7 +758,11 @@ class FlashAttentionImpl(AttentionImpl):
|
||||
|
||||
descale_shape = (cu_seqlens_q.shape[0] - 1, self.num_kv_heads)
|
||||
|
||||
q_descale = layer._q_scale.expand(descale_shape)
|
||||
q_descale = (
|
||||
layer._q_scale.expand(descale_shape)
|
||||
if self.supports_quant_query_input
|
||||
else None
|
||||
)
|
||||
k_descale = layer._k_scale.expand(descale_shape)
|
||||
v_descale = layer._v_scale.expand(descale_shape)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user