From 639402f5a24a9176f1049f1f9dd72dd2a19e8e15 Mon Sep 17 00:00:00 2001 From: Xinyu Chen Date: Sun, 12 Apr 2026 11:53:32 +0800 Subject: [PATCH] Support FP8 KVCache on XPU (#37731) Signed-off-by: Xinyu Chen Co-authored-by: Kunshang Ji --- .buildkite/intel_jobs/test-intel.yaml | 1 + vllm/_xpu_ops.py | 3 +++ vllm/v1/attention/backends/fa_utils.py | 6 ++++++ vllm/v1/attention/backends/flash_attn.py | 9 +++++++-- 4 files changed, 17 insertions(+), 2 deletions(-) diff --git a/.buildkite/intel_jobs/test-intel.yaml b/.buildkite/intel_jobs/test-intel.yaml index a05712fd97..c59be69950 100644 --- a/.buildkite/intel_jobs/test-intel.yaml +++ b/.buildkite/intel_jobs/test-intel.yaml @@ -35,6 +35,7 @@ steps: python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager -tp 2 --distributed-executor-backend mp && python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --attention-backend=TRITON_ATTN && python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8 && + python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --kv-cache-dtype fp8 && python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --block-size 64 --enforce-eager --max-model-len 8192 && python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 && python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 --enable-expert-parallel' diff --git a/vllm/_xpu_ops.py b/vllm/_xpu_ops.py index 646322163e..e4d2d8b68e 100644 --- a/vllm/_xpu_ops.py +++ b/vllm/_xpu_ops.py @@ -258,6 +258,9 @@ class xpu_ops: # alibi_slopes = alibi_slopes, # softcap=softcap, return_softmax_lse=return_softmax_lse, + q_descale=q_descale, + k_descale=k_descale, + v_descale=v_descale, ) @staticmethod diff --git a/vllm/v1/attention/backends/fa_utils.py b/vllm/v1/attention/backends/fa_utils.py index 97fc35e70c..db8cafeb74 100644 --- a/vllm/v1/attention/backends/fa_utils.py +++ b/vllm/v1/attention/backends/fa_utils.py @@ -166,12 +166,18 @@ def is_fa_version_supported(fa_version: int) -> bool: def flash_attn_supports_fp8() -> bool: + if current_platform.is_xpu(): + return True return ( get_flash_attn_version() == 3 and current_platform.is_device_capability_family(90) ) +def flash_attn_supports_quant_query_input() -> bool: + return not current_platform.is_xpu() + + def flash_attn_supports_sinks() -> bool: if current_platform.is_xpu(): return True diff --git a/vllm/v1/attention/backends/flash_attn.py b/vllm/v1/attention/backends/flash_attn.py index b9ccd4fce1..0e926a99b8 100755 --- a/vllm/v1/attention/backends/flash_attn.py +++ b/vllm/v1/attention/backends/flash_attn.py @@ -20,6 +20,7 @@ from vllm.v1.attention.backend import ( ) from vllm.v1.attention.backends.fa_utils import ( flash_attn_supports_fp8, + flash_attn_supports_quant_query_input, get_flash_attn_version, is_fa_version_supported, is_flash_attn_varlen_func_available, @@ -656,7 +657,7 @@ class FlashAttentionImpl(AttentionImpl): "heads in the layer" ) - self.supports_quant_query_input = True + self.supports_quant_query_input = flash_attn_supports_quant_query_input() vllm_config = get_current_vllm_config_or_none() dcp_a2a = ( @@ -757,7 +758,11 @@ class FlashAttentionImpl(AttentionImpl): descale_shape = (cu_seqlens_q.shape[0] - 1, self.num_kv_heads) - q_descale = layer._q_scale.expand(descale_shape) + q_descale = ( + layer._q_scale.expand(descale_shape) + if self.supports_quant_query_input + else None + ) k_descale = layer._k_scale.expand(descale_shape) v_descale = layer._v_scale.expand(descale_shape)