diff --git a/tests/kernels/attention/test_attention_selector.py b/tests/kernels/attention/test_attention_selector.py index 95561b500b..db4dcc8a63 100644 --- a/tests/kernels/attention/test_attention_selector.py +++ b/tests/kernels/attention/test_attention_selector.py @@ -514,3 +514,35 @@ def test_non_causal_autoselect_backend(): kv_cache_dtype=None, ) assert backend.supports_non_causal() + + +@pytest.mark.parametrize( + "kv_cache_dtype", + [ + "fp8_e5m2", + "fp8_ds_mla", + "fp8_inc", + "nvfp4", + "fp8_per_token_head", + "int8_per_token_head", + ], +) +def test_flash_attn_rejects_unhandled_kv_cache_dtypes(kv_cache_dtype: str): + """FlashAttentionBackend must not claim support for kv_cache dtypes + that it cannot handle.""" + from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend + + assert not FlashAttentionBackend.supports_kv_cache_dtype(kv_cache_dtype) + + +@pytest.mark.parametrize("kv_cache_dtype", ["fp8", "fp8_e4m3"]) +def test_flash_attn_accepts_handled_fp8_variants( + kv_cache_dtype: str, monkeypatch: pytest.MonkeyPatch +): + """FlashAttentionBackend must accept the two fp8 dtypes it can actually + handle: 'fp8' (alias for fp8_e4m3fn) and 'fp8_e4m3'.""" + import vllm.v1.attention.backends.flash_attn as fa_mod + from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend + + monkeypatch.setattr(fa_mod.current_platform, "is_xpu", lambda: True) + assert FlashAttentionBackend.supports_kv_cache_dtype(kv_cache_dtype) diff --git a/tests/models/quantization/test_fp8.py b/tests/models/quantization/test_fp8.py index 9be5fd3302..5f3c654761 100644 --- a/tests/models/quantization/test_fp8.py +++ b/tests/models/quantization/test_fp8.py @@ -9,8 +9,8 @@ Note: these tests will only pass on L4 GPU. import pytest from tests.quantization.utils import is_quant_method_supported -from vllm.v1.attention.backends.fa_utils import flash_attn_supports_fp8 from vllm.platforms import current_platform +from vllm.v1.attention.backends.fa_utils import get_flash_attn_version from ..utils import check_logprobs_close @@ -68,7 +68,13 @@ def test_models( if kv_cache_dtype == "fp8_e5m2" and current_platform.is_rocm(): pytest.skip(f"{kv_cache_dtype} is currently not supported on ROCm/HIP.") - if not flash_attn_supports_fp8(): + if not ( + current_platform.is_xpu() + or ( + get_flash_attn_version() == 3 + and current_platform.is_device_capability_family(90) + ) + ): pytest.skip( f"{kv_cache_dtype} is not supported on this GPU type with {backend} attention." ) diff --git a/tools/pre_commit/generate_attention_backend_docs.py b/tools/pre_commit/generate_attention_backend_docs.py index c0503fd697..abbbdce09b 100644 --- a/tools/pre_commit/generate_attention_backend_docs.py +++ b/tools/pre_commit/generate_attention_backend_docs.py @@ -878,7 +878,7 @@ def parse_flash_attn_features() -> dict[str, dict[str, Any]]: return {} # Analyze the functions to determine FA3/FA4-specific features - fa3_supports_fp8 = False + fa3_supports_fp8 = True fa3_supports_sinks = False fa4_supports_sinks = False fa3_compute_cap: str | None = None @@ -888,18 +888,6 @@ def parse_flash_attn_features() -> dict[str, dict[str, Any]]: if not isinstance(node, ast.FunctionDef): continue - # Check flash_attn_supports_fp8 - looks for `get_flash_attn_version() == 3` - if node.name == "flash_attn_supports_fp8": - for n in ast.walk(node): - if ( - isinstance(n, ast.Compare) - and isinstance(n.left, ast.Call) - and isinstance(n.left.func, ast.Name) - and n.left.func.id == "get_flash_attn_version" - ): - fa3_supports_fp8 = True - break - # Check flash_attn_supports_sinks - looks for `fa_version == 3/4` # or `get_flash_attn_version() == 3/4` (also accepts `in (3, 4)`) if node.name == "flash_attn_supports_sinks": diff --git a/vllm/v1/attention/backends/fa_utils.py b/vllm/v1/attention/backends/fa_utils.py index 1e74e4c48e..0d6a3d298b 100644 --- a/vllm/v1/attention/backends/fa_utils.py +++ b/vllm/v1/attention/backends/fa_utils.py @@ -191,15 +191,6 @@ def is_fa_version_supported(fa_version: int) -> bool: return False -def flash_attn_supports_fp8() -> bool: - if current_platform.is_xpu(): - return True - return ( - get_flash_attn_version() == 3 - and current_platform.is_device_capability_family(90) - ) - - def flash_attn_supports_quant_query_input() -> bool: return not current_platform.is_xpu() diff --git a/vllm/v1/attention/backends/flash_attn.py b/vllm/v1/attention/backends/flash_attn.py index 4b8b86d864..cfb52dee3b 100755 --- a/vllm/v1/attention/backends/flash_attn.py +++ b/vllm/v1/attention/backends/flash_attn.py @@ -22,7 +22,6 @@ from vllm.v1.attention.backend import ( MultipleOf, ) from vllm.v1.attention.backends.fa_utils import ( - flash_attn_supports_fp8, flash_attn_supports_quant_query_input, get_flash_attn_version, is_fa_version_supported, @@ -170,13 +169,6 @@ class FlashAttentionBackend(AttentionBackend): raise ValueError(f"Unknown cache layout format {cache_layout}.") return stride_order - @staticmethod - def get_fp8_dtype_for_flashattn(kv_cache_dtype: str) -> torch.dtype: - if kv_cache_dtype in ("fp8", "fp8_e4m3"): - return torch.float8_e4m3fn - else: - raise ValueError(f"Unrecognized FP8 dtype: {kv_cache_dtype}") - @classmethod def supports_head_size(cls, head_size: int) -> bool: if head_size % 8 != 0: @@ -191,8 +183,13 @@ class FlashAttentionBackend(AttentionBackend): def supports_kv_cache_dtype(cls, kv_cache_dtype: CacheDType | None) -> bool: if kv_cache_dtype is None: return True - if is_quantized_kv_cache(kv_cache_dtype): - return flash_attn_supports_fp8() + if kv_cache_dtype in ("fp8", "fp8_e4m3"): + if current_platform.is_xpu(): + return True + return ( + get_flash_attn_version() == 3 + and current_platform.is_device_capability_family(90) + ) return kv_cache_dtype in ["auto", "float16", "bfloat16"] @classmethod @@ -450,9 +447,7 @@ class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetad ): cache_dtype = self.cache_config.cache_dtype if is_quantized_kv_cache(cache_dtype): - qkv_dtype = FlashAttentionBackend.get_fp8_dtype_for_flashattn( - cache_dtype - ) + qkv_dtype = current_platform.fp8_dtype() else: qkv_dtype = self.kv_cache_dtype if aot_schedule: @@ -645,11 +640,6 @@ class FlashAttentionImpl(AttentionImpl): # Cache the batch invariant result for use in forward passes self.batch_invariant_enabled = envs.VLLM_BATCH_INVARIANT - if is_quantized_kv_cache(self.kv_cache_dtype) and not flash_attn_supports_fp8(): - raise NotImplementedError( - "FlashAttention does not support fp8 kv-cache on this device." - ) - self.sinks = sinks if self.sinks is not None: assert flash_attn_supports_sinks(), ( @@ -762,11 +752,8 @@ class FlashAttentionImpl(AttentionImpl): if is_quantized_kv_cache(self.kv_cache_dtype): # queries are quantized in the attention layer - dtype = FlashAttentionBackend.get_fp8_dtype_for_flashattn( - self.kv_cache_dtype - ) - key_cache = key_cache.view(dtype) - value_cache = value_cache.view(dtype) + key_cache = key_cache.view(current_platform.fp8_dtype()) + value_cache = value_cache.view(current_platform.fp8_dtype()) if not attn_metadata.use_cascade: cu_seqlens_q = attn_metadata.query_start_loc diff --git a/vllm/v1/attention/backends/flash_attn_diffkv.py b/vllm/v1/attention/backends/flash_attn_diffkv.py index 82a9f07a4e..e788b0e349 100644 --- a/vllm/v1/attention/backends/flash_attn_diffkv.py +++ b/vllm/v1/attention/backends/flash_attn_diffkv.py @@ -5,6 +5,7 @@ import torch from vllm.logger import init_logger +from vllm.platforms import current_platform from vllm.utils.torch_utils import ( canonicalize_singleton_dim_strides, is_quantized_kv_cache, @@ -230,11 +231,8 @@ class FlashAttentionDiffKVImpl(FlashAttentionImpl): if is_quantized_kv_cache(self.kv_cache_dtype): # queries are quantized in the attention layer - dtype = FlashAttentionBackend.get_fp8_dtype_for_flashattn( - self.kv_cache_dtype - ) - key_cache = key_cache.view(dtype) - value_cache = value_cache.view(dtype) + key_cache = key_cache.view(current_platform.fp8_dtype()) + value_cache = value_cache.view(current_platform.fp8_dtype()) if not attn_metadata.use_cascade: cu_seqlens_q = attn_metadata.query_start_loc