[FlashAttn] Fix supports_kv_cache_dtype() accepting unhandled fp8 kv-cache dtype variants (#42685)
Signed-off-by: Lanze Liu <lanzetech@gmail.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
co-authored by
Matthew Bonanni
parent
4d67d3bde2
commit
b2c58ee942
@@ -514,3 +514,35 @@ def test_non_causal_autoselect_backend():
|
||||
kv_cache_dtype=None,
|
||||
)
|
||||
assert backend.supports_non_causal()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"kv_cache_dtype",
|
||||
[
|
||||
"fp8_e5m2",
|
||||
"fp8_ds_mla",
|
||||
"fp8_inc",
|
||||
"nvfp4",
|
||||
"fp8_per_token_head",
|
||||
"int8_per_token_head",
|
||||
],
|
||||
)
|
||||
def test_flash_attn_rejects_unhandled_kv_cache_dtypes(kv_cache_dtype: str):
|
||||
"""FlashAttentionBackend must not claim support for kv_cache dtypes
|
||||
that it cannot handle."""
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
|
||||
|
||||
assert not FlashAttentionBackend.supports_kv_cache_dtype(kv_cache_dtype)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["fp8", "fp8_e4m3"])
|
||||
def test_flash_attn_accepts_handled_fp8_variants(
|
||||
kv_cache_dtype: str, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
"""FlashAttentionBackend must accept the two fp8 dtypes it can actually
|
||||
handle: 'fp8' (alias for fp8_e4m3fn) and 'fp8_e4m3'."""
|
||||
import vllm.v1.attention.backends.flash_attn as fa_mod
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
|
||||
|
||||
monkeypatch.setattr(fa_mod.current_platform, "is_xpu", lambda: True)
|
||||
assert FlashAttentionBackend.supports_kv_cache_dtype(kv_cache_dtype)
|
||||
|
||||
@@ -9,8 +9,8 @@ Note: these tests will only pass on L4 GPU.
|
||||
import pytest
|
||||
|
||||
from tests.quantization.utils import is_quant_method_supported
|
||||
from vllm.v1.attention.backends.fa_utils import flash_attn_supports_fp8
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
|
||||
from ..utils import check_logprobs_close
|
||||
|
||||
|
||||
@@ -68,7 +68,13 @@ def test_models(
|
||||
if kv_cache_dtype == "fp8_e5m2" and current_platform.is_rocm():
|
||||
pytest.skip(f"{kv_cache_dtype} is currently not supported on ROCm/HIP.")
|
||||
|
||||
if not flash_attn_supports_fp8():
|
||||
if not (
|
||||
current_platform.is_xpu()
|
||||
or (
|
||||
get_flash_attn_version() == 3
|
||||
and current_platform.is_device_capability_family(90)
|
||||
)
|
||||
):
|
||||
pytest.skip(
|
||||
f"{kv_cache_dtype} is not supported on this GPU type with {backend} attention."
|
||||
)
|
||||
|
||||
@@ -878,7 +878,7 @@ def parse_flash_attn_features() -> dict[str, dict[str, Any]]:
|
||||
return {}
|
||||
|
||||
# Analyze the functions to determine FA3/FA4-specific features
|
||||
fa3_supports_fp8 = False
|
||||
fa3_supports_fp8 = True
|
||||
fa3_supports_sinks = False
|
||||
fa4_supports_sinks = False
|
||||
fa3_compute_cap: str | None = None
|
||||
@@ -888,18 +888,6 @@ def parse_flash_attn_features() -> dict[str, dict[str, Any]]:
|
||||
if not isinstance(node, ast.FunctionDef):
|
||||
continue
|
||||
|
||||
# Check flash_attn_supports_fp8 - looks for `get_flash_attn_version() == 3`
|
||||
if node.name == "flash_attn_supports_fp8":
|
||||
for n in ast.walk(node):
|
||||
if (
|
||||
isinstance(n, ast.Compare)
|
||||
and isinstance(n.left, ast.Call)
|
||||
and isinstance(n.left.func, ast.Name)
|
||||
and n.left.func.id == "get_flash_attn_version"
|
||||
):
|
||||
fa3_supports_fp8 = True
|
||||
break
|
||||
|
||||
# Check flash_attn_supports_sinks - looks for `fa_version == 3/4`
|
||||
# or `get_flash_attn_version() == 3/4` (also accepts `in (3, 4)`)
|
||||
if node.name == "flash_attn_supports_sinks":
|
||||
|
||||
@@ -191,15 +191,6 @@ def is_fa_version_supported(fa_version: int) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def flash_attn_supports_fp8() -> bool:
|
||||
if current_platform.is_xpu():
|
||||
return True
|
||||
return (
|
||||
get_flash_attn_version() == 3
|
||||
and current_platform.is_device_capability_family(90)
|
||||
)
|
||||
|
||||
|
||||
def flash_attn_supports_quant_query_input() -> bool:
|
||||
return not current_platform.is_xpu()
|
||||
|
||||
|
||||
@@ -22,7 +22,6 @@ from vllm.v1.attention.backend import (
|
||||
MultipleOf,
|
||||
)
|
||||
from vllm.v1.attention.backends.fa_utils import (
|
||||
flash_attn_supports_fp8,
|
||||
flash_attn_supports_quant_query_input,
|
||||
get_flash_attn_version,
|
||||
is_fa_version_supported,
|
||||
@@ -170,13 +169,6 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
raise ValueError(f"Unknown cache layout format {cache_layout}.")
|
||||
return stride_order
|
||||
|
||||
@staticmethod
|
||||
def get_fp8_dtype_for_flashattn(kv_cache_dtype: str) -> torch.dtype:
|
||||
if kv_cache_dtype in ("fp8", "fp8_e4m3"):
|
||||
return torch.float8_e4m3fn
|
||||
else:
|
||||
raise ValueError(f"Unrecognized FP8 dtype: {kv_cache_dtype}")
|
||||
|
||||
@classmethod
|
||||
def supports_head_size(cls, head_size: int) -> bool:
|
||||
if head_size % 8 != 0:
|
||||
@@ -191,8 +183,13 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
def supports_kv_cache_dtype(cls, kv_cache_dtype: CacheDType | None) -> bool:
|
||||
if kv_cache_dtype is None:
|
||||
return True
|
||||
if is_quantized_kv_cache(kv_cache_dtype):
|
||||
return flash_attn_supports_fp8()
|
||||
if kv_cache_dtype in ("fp8", "fp8_e4m3"):
|
||||
if current_platform.is_xpu():
|
||||
return True
|
||||
return (
|
||||
get_flash_attn_version() == 3
|
||||
and current_platform.is_device_capability_family(90)
|
||||
)
|
||||
return kv_cache_dtype in ["auto", "float16", "bfloat16"]
|
||||
|
||||
@classmethod
|
||||
@@ -450,9 +447,7 @@ class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetad
|
||||
):
|
||||
cache_dtype = self.cache_config.cache_dtype
|
||||
if is_quantized_kv_cache(cache_dtype):
|
||||
qkv_dtype = FlashAttentionBackend.get_fp8_dtype_for_flashattn(
|
||||
cache_dtype
|
||||
)
|
||||
qkv_dtype = current_platform.fp8_dtype()
|
||||
else:
|
||||
qkv_dtype = self.kv_cache_dtype
|
||||
if aot_schedule:
|
||||
@@ -645,11 +640,6 @@ class FlashAttentionImpl(AttentionImpl):
|
||||
# Cache the batch invariant result for use in forward passes
|
||||
self.batch_invariant_enabled = envs.VLLM_BATCH_INVARIANT
|
||||
|
||||
if is_quantized_kv_cache(self.kv_cache_dtype) and not flash_attn_supports_fp8():
|
||||
raise NotImplementedError(
|
||||
"FlashAttention does not support fp8 kv-cache on this device."
|
||||
)
|
||||
|
||||
self.sinks = sinks
|
||||
if self.sinks is not None:
|
||||
assert flash_attn_supports_sinks(), (
|
||||
@@ -762,11 +752,8 @@ class FlashAttentionImpl(AttentionImpl):
|
||||
|
||||
if is_quantized_kv_cache(self.kv_cache_dtype):
|
||||
# queries are quantized in the attention layer
|
||||
dtype = FlashAttentionBackend.get_fp8_dtype_for_flashattn(
|
||||
self.kv_cache_dtype
|
||||
)
|
||||
key_cache = key_cache.view(dtype)
|
||||
value_cache = value_cache.view(dtype)
|
||||
key_cache = key_cache.view(current_platform.fp8_dtype())
|
||||
value_cache = value_cache.view(current_platform.fp8_dtype())
|
||||
|
||||
if not attn_metadata.use_cascade:
|
||||
cu_seqlens_q = attn_metadata.query_start_loc
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import (
|
||||
canonicalize_singleton_dim_strides,
|
||||
is_quantized_kv_cache,
|
||||
@@ -230,11 +231,8 @@ class FlashAttentionDiffKVImpl(FlashAttentionImpl):
|
||||
|
||||
if is_quantized_kv_cache(self.kv_cache_dtype):
|
||||
# queries are quantized in the attention layer
|
||||
dtype = FlashAttentionBackend.get_fp8_dtype_for_flashattn(
|
||||
self.kv_cache_dtype
|
||||
)
|
||||
key_cache = key_cache.view(dtype)
|
||||
value_cache = value_cache.view(dtype)
|
||||
key_cache = key_cache.view(current_platform.fp8_dtype())
|
||||
value_cache = value_cache.view(current_platform.fp8_dtype())
|
||||
|
||||
if not attn_metadata.use_cascade:
|
||||
cu_seqlens_q = attn_metadata.query_start_loc
|
||||
|
||||
Reference in New Issue
Block a user