[Attention] FlashAttention 4 SM100 FP8 kv cache support (#42569)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Matthew Bonanni
2026-07-20 10:53:27 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent b23bd73f54
commit bd091079cb
8 changed files with 194 additions and 33 deletions
+23
View File
@@ -273,6 +273,29 @@ def is_fa_version_supported(fa_version: int) -> bool:
return False
def flash_attn_supports_kv_cache_dtype(
kv_cache_dtype: str = "fp8_e4m3",
*,
requires_alibi: bool = False,
head_size: int | None = None,
head_size_v: int | None = None,
has_sinks: bool = False,
) -> bool:
if kv_cache_dtype == "fp8_e5m2":
return False
if current_platform.is_xpu():
return True
fa_version = get_flash_attn_version(
requires_alibi=requires_alibi,
head_size=head_size,
head_size_v=head_size_v,
has_sinks=has_sinks,
)
return (fa_version == 3 and current_platform.is_device_capability_family(90)) or (
fa_version == 4 and current_platform.is_device_capability_family(100)
)
def flash_attn_supports_quant_query_input() -> bool:
return not current_platform.is_xpu()
+34 -8
View File
@@ -22,6 +22,7 @@ from vllm.v1.attention.backend import (
MultipleOf,
)
from vllm.v1.attention.backends.fa_utils import (
flash_attn_supports_kv_cache_dtype,
flash_attn_supports_quant_query_input,
get_flash_attn_version,
is_fa_version_supported,
@@ -74,6 +75,8 @@ class FlashAttentionBackend(AttentionBackend):
"auto",
"float16",
"bfloat16",
"fp8",
"fp8_e4m3",
]
@staticmethod
@@ -178,14 +181,11 @@ class FlashAttentionBackend(AttentionBackend):
def supports_kv_cache_dtype(cls, kv_cache_dtype: CacheDType | None) -> bool:
if kv_cache_dtype is None:
return True
if kv_cache_dtype in ("fp8", "fp8_e4m3"):
if current_platform.is_xpu():
return True
return (
get_flash_attn_version() == 3
and current_platform.is_device_capability_family(90)
)
return kv_cache_dtype in ["auto", "float16", "bfloat16"]
if kv_cache_dtype not in cls.supported_kv_cache_dtypes:
return False
if is_quantized_kv_cache(kv_cache_dtype):
return flash_attn_supports_kv_cache_dtype(kv_cache_dtype)
return True
@classmethod
def supports_mm_prefix(cls) -> bool:
@@ -216,6 +216,17 @@ class FlashAttentionBackend(AttentionBackend):
) -> str | None:
if has_sink and device_capability < DeviceCapability(9, 0):
return "sink not supported on compute capability < 9.0"
if (
kv_cache_dtype is not None
and is_quantized_kv_cache(kv_cache_dtype)
and not flash_attn_supports_kv_cache_dtype(
kv_cache_dtype,
head_size=head_size,
head_size_v=head_size,
has_sinks=has_sink,
)
):
return "FP8 KV cache requires FA3 on SM90 or FA4 on SM100"
if (
use_mm_prefix
and get_flash_attn_version(head_size=head_size, has_sinks=has_sink) != 4
@@ -772,6 +783,7 @@ class FlashAttentionImpl(AttentionImpl):
self.vllm_flash_attn_version = get_flash_attn_version(
requires_alibi=alibi_slopes is not None,
head_size=head_size,
has_sinks=sinks is not None,
)
logger.info_once(
"Using FlashAttention version %s",
@@ -780,6 +792,20 @@ class FlashAttentionImpl(AttentionImpl):
# Cache the batch invariant result for use in forward passes
self.batch_invariant_enabled = envs.VLLM_BATCH_INVARIANT
if is_quantized_kv_cache(
self.kv_cache_dtype
) and not flash_attn_supports_kv_cache_dtype(
self.kv_cache_dtype,
requires_alibi=alibi_slopes is not None,
head_size=head_size,
head_size_v=head_size,
has_sinks=sinks is not None,
):
raise NotImplementedError(
f"FlashAttention does not support {self.kv_cache_dtype}"
" kv-cache on this device."
)
self.sinks = sinks
if self.sinks is not None:
assert flash_attn_supports_sinks(), (