[FlashAttn] Fix supports_kv_cache_dtype() accepting unhandled fp8 kv-cache dtype variants (#42685)

Signed-off-by: Lanze Liu <lanzetech@gmail.com>
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
Lanze Liu
2026-05-15 15:34:59 -04:00
committed by GitHub
co-authored by Matthew Bonanni
parent 4d67d3bde2
commit b2c58ee942
6 changed files with 54 additions and 52 deletions
@@ -514,3 +514,35 @@ def test_non_causal_autoselect_backend():
kv_cache_dtype=None,
)
assert backend.supports_non_causal()
@pytest.mark.parametrize(
"kv_cache_dtype",
[
"fp8_e5m2",
"fp8_ds_mla",
"fp8_inc",
"nvfp4",
"fp8_per_token_head",
"int8_per_token_head",
],
)
def test_flash_attn_rejects_unhandled_kv_cache_dtypes(kv_cache_dtype: str):
"""FlashAttentionBackend must not claim support for kv_cache dtypes
that it cannot handle."""
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
assert not FlashAttentionBackend.supports_kv_cache_dtype(kv_cache_dtype)
@pytest.mark.parametrize("kv_cache_dtype", ["fp8", "fp8_e4m3"])
def test_flash_attn_accepts_handled_fp8_variants(
kv_cache_dtype: str, monkeypatch: pytest.MonkeyPatch
):
"""FlashAttentionBackend must accept the two fp8 dtypes it can actually
handle: 'fp8' (alias for fp8_e4m3fn) and 'fp8_e4m3'."""
import vllm.v1.attention.backends.flash_attn as fa_mod
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
monkeypatch.setattr(fa_mod.current_platform, "is_xpu", lambda: True)
assert FlashAttentionBackend.supports_kv_cache_dtype(kv_cache_dtype)
+8 -2
View File
@@ -9,8 +9,8 @@ Note: these tests will only pass on L4 GPU.
import pytest
from tests.quantization.utils import is_quant_method_supported
from vllm.v1.attention.backends.fa_utils import flash_attn_supports_fp8
from vllm.platforms import current_platform
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
from ..utils import check_logprobs_close
@@ -68,7 +68,13 @@ def test_models(
if kv_cache_dtype == "fp8_e5m2" and current_platform.is_rocm():
pytest.skip(f"{kv_cache_dtype} is currently not supported on ROCm/HIP.")
if not flash_attn_supports_fp8():
if not (
current_platform.is_xpu()
or (
get_flash_attn_version() == 3
and current_platform.is_device_capability_family(90)
)
):
pytest.skip(
f"{kv_cache_dtype} is not supported on this GPU type with {backend} attention."
)
@@ -878,7 +878,7 @@ def parse_flash_attn_features() -> dict[str, dict[str, Any]]:
return {}
# Analyze the functions to determine FA3/FA4-specific features
fa3_supports_fp8 = False
fa3_supports_fp8 = True
fa3_supports_sinks = False
fa4_supports_sinks = False
fa3_compute_cap: str | None = None
@@ -888,18 +888,6 @@ def parse_flash_attn_features() -> dict[str, dict[str, Any]]:
if not isinstance(node, ast.FunctionDef):
continue
# Check flash_attn_supports_fp8 - looks for `get_flash_attn_version() == 3`
if node.name == "flash_attn_supports_fp8":
for n in ast.walk(node):
if (
isinstance(n, ast.Compare)
and isinstance(n.left, ast.Call)
and isinstance(n.left.func, ast.Name)
and n.left.func.id == "get_flash_attn_version"
):
fa3_supports_fp8 = True
break
# Check flash_attn_supports_sinks - looks for `fa_version == 3/4`
# or `get_flash_attn_version() == 3/4` (also accepts `in (3, 4)`)
if node.name == "flash_attn_supports_sinks":
-9
View File
@@ -191,15 +191,6 @@ def is_fa_version_supported(fa_version: int) -> bool:
return False
def flash_attn_supports_fp8() -> bool:
if current_platform.is_xpu():
return True
return (
get_flash_attn_version() == 3
and current_platform.is_device_capability_family(90)
)
def flash_attn_supports_quant_query_input() -> bool:
return not current_platform.is_xpu()
+10 -23
View File
@@ -22,7 +22,6 @@ from vllm.v1.attention.backend import (
MultipleOf,
)
from vllm.v1.attention.backends.fa_utils import (
flash_attn_supports_fp8,
flash_attn_supports_quant_query_input,
get_flash_attn_version,
is_fa_version_supported,
@@ -170,13 +169,6 @@ class FlashAttentionBackend(AttentionBackend):
raise ValueError(f"Unknown cache layout format {cache_layout}.")
return stride_order
@staticmethod
def get_fp8_dtype_for_flashattn(kv_cache_dtype: str) -> torch.dtype:
if kv_cache_dtype in ("fp8", "fp8_e4m3"):
return torch.float8_e4m3fn
else:
raise ValueError(f"Unrecognized FP8 dtype: {kv_cache_dtype}")
@classmethod
def supports_head_size(cls, head_size: int) -> bool:
if head_size % 8 != 0:
@@ -191,8 +183,13 @@ class FlashAttentionBackend(AttentionBackend):
def supports_kv_cache_dtype(cls, kv_cache_dtype: CacheDType | None) -> bool:
if kv_cache_dtype is None:
return True
if is_quantized_kv_cache(kv_cache_dtype):
return flash_attn_supports_fp8()
if kv_cache_dtype in ("fp8", "fp8_e4m3"):
if current_platform.is_xpu():
return True
return (
get_flash_attn_version() == 3
and current_platform.is_device_capability_family(90)
)
return kv_cache_dtype in ["auto", "float16", "bfloat16"]
@classmethod
@@ -450,9 +447,7 @@ class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetad
):
cache_dtype = self.cache_config.cache_dtype
if is_quantized_kv_cache(cache_dtype):
qkv_dtype = FlashAttentionBackend.get_fp8_dtype_for_flashattn(
cache_dtype
)
qkv_dtype = current_platform.fp8_dtype()
else:
qkv_dtype = self.kv_cache_dtype
if aot_schedule:
@@ -645,11 +640,6 @@ class FlashAttentionImpl(AttentionImpl):
# Cache the batch invariant result for use in forward passes
self.batch_invariant_enabled = envs.VLLM_BATCH_INVARIANT
if is_quantized_kv_cache(self.kv_cache_dtype) and not flash_attn_supports_fp8():
raise NotImplementedError(
"FlashAttention does not support fp8 kv-cache on this device."
)
self.sinks = sinks
if self.sinks is not None:
assert flash_attn_supports_sinks(), (
@@ -762,11 +752,8 @@ class FlashAttentionImpl(AttentionImpl):
if is_quantized_kv_cache(self.kv_cache_dtype):
# queries are quantized in the attention layer
dtype = FlashAttentionBackend.get_fp8_dtype_for_flashattn(
self.kv_cache_dtype
)
key_cache = key_cache.view(dtype)
value_cache = value_cache.view(dtype)
key_cache = key_cache.view(current_platform.fp8_dtype())
value_cache = value_cache.view(current_platform.fp8_dtype())
if not attn_metadata.use_cascade:
cu_seqlens_q = attn_metadata.query_start_loc
@@ -5,6 +5,7 @@
import torch
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.torch_utils import (
canonicalize_singleton_dim_strides,
is_quantized_kv_cache,
@@ -230,11 +231,8 @@ class FlashAttentionDiffKVImpl(FlashAttentionImpl):
if is_quantized_kv_cache(self.kv_cache_dtype):
# queries are quantized in the attention layer
dtype = FlashAttentionBackend.get_fp8_dtype_for_flashattn(
self.kv_cache_dtype
)
key_cache = key_cache.view(dtype)
value_cache = value_cache.view(dtype)
key_cache = key_cache.view(current_platform.fp8_dtype())
value_cache = value_cache.view(current_platform.fp8_dtype())
if not attn_metadata.use_cascade:
cu_seqlens_q = attn_metadata.query_start_loc