[Attention] FlashAttention 4 SM100 FP8 kv cache support (#42569)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
b23bd73f54
commit
bd091079cb
@@ -273,6 +273,29 @@ def is_fa_version_supported(fa_version: int) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def flash_attn_supports_kv_cache_dtype(
|
||||
kv_cache_dtype: str = "fp8_e4m3",
|
||||
*,
|
||||
requires_alibi: bool = False,
|
||||
head_size: int | None = None,
|
||||
head_size_v: int | None = None,
|
||||
has_sinks: bool = False,
|
||||
) -> bool:
|
||||
if kv_cache_dtype == "fp8_e5m2":
|
||||
return False
|
||||
if current_platform.is_xpu():
|
||||
return True
|
||||
fa_version = get_flash_attn_version(
|
||||
requires_alibi=requires_alibi,
|
||||
head_size=head_size,
|
||||
head_size_v=head_size_v,
|
||||
has_sinks=has_sinks,
|
||||
)
|
||||
return (fa_version == 3 and current_platform.is_device_capability_family(90)) or (
|
||||
fa_version == 4 and current_platform.is_device_capability_family(100)
|
||||
)
|
||||
|
||||
|
||||
def flash_attn_supports_quant_query_input() -> bool:
|
||||
return not current_platform.is_xpu()
|
||||
|
||||
|
||||
@@ -22,6 +22,7 @@ from vllm.v1.attention.backend import (
|
||||
MultipleOf,
|
||||
)
|
||||
from vllm.v1.attention.backends.fa_utils import (
|
||||
flash_attn_supports_kv_cache_dtype,
|
||||
flash_attn_supports_quant_query_input,
|
||||
get_flash_attn_version,
|
||||
is_fa_version_supported,
|
||||
@@ -74,6 +75,8 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
"auto",
|
||||
"float16",
|
||||
"bfloat16",
|
||||
"fp8",
|
||||
"fp8_e4m3",
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
@@ -178,14 +181,11 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
def supports_kv_cache_dtype(cls, kv_cache_dtype: CacheDType | None) -> bool:
|
||||
if kv_cache_dtype is None:
|
||||
return True
|
||||
if kv_cache_dtype in ("fp8", "fp8_e4m3"):
|
||||
if current_platform.is_xpu():
|
||||
return True
|
||||
return (
|
||||
get_flash_attn_version() == 3
|
||||
and current_platform.is_device_capability_family(90)
|
||||
)
|
||||
return kv_cache_dtype in ["auto", "float16", "bfloat16"]
|
||||
if kv_cache_dtype not in cls.supported_kv_cache_dtypes:
|
||||
return False
|
||||
if is_quantized_kv_cache(kv_cache_dtype):
|
||||
return flash_attn_supports_kv_cache_dtype(kv_cache_dtype)
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def supports_mm_prefix(cls) -> bool:
|
||||
@@ -216,6 +216,17 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
) -> str | None:
|
||||
if has_sink and device_capability < DeviceCapability(9, 0):
|
||||
return "sink not supported on compute capability < 9.0"
|
||||
if (
|
||||
kv_cache_dtype is not None
|
||||
and is_quantized_kv_cache(kv_cache_dtype)
|
||||
and not flash_attn_supports_kv_cache_dtype(
|
||||
kv_cache_dtype,
|
||||
head_size=head_size,
|
||||
head_size_v=head_size,
|
||||
has_sinks=has_sink,
|
||||
)
|
||||
):
|
||||
return "FP8 KV cache requires FA3 on SM90 or FA4 on SM100"
|
||||
if (
|
||||
use_mm_prefix
|
||||
and get_flash_attn_version(head_size=head_size, has_sinks=has_sink) != 4
|
||||
@@ -772,6 +783,7 @@ class FlashAttentionImpl(AttentionImpl):
|
||||
self.vllm_flash_attn_version = get_flash_attn_version(
|
||||
requires_alibi=alibi_slopes is not None,
|
||||
head_size=head_size,
|
||||
has_sinks=sinks is not None,
|
||||
)
|
||||
logger.info_once(
|
||||
"Using FlashAttention version %s",
|
||||
@@ -780,6 +792,20 @@ class FlashAttentionImpl(AttentionImpl):
|
||||
# Cache the batch invariant result for use in forward passes
|
||||
self.batch_invariant_enabled = envs.VLLM_BATCH_INVARIANT
|
||||
|
||||
if is_quantized_kv_cache(
|
||||
self.kv_cache_dtype
|
||||
) and not flash_attn_supports_kv_cache_dtype(
|
||||
self.kv_cache_dtype,
|
||||
requires_alibi=alibi_slopes is not None,
|
||||
head_size=head_size,
|
||||
head_size_v=head_size,
|
||||
has_sinks=sinks is not None,
|
||||
):
|
||||
raise NotImplementedError(
|
||||
f"FlashAttention does not support {self.kv_cache_dtype}"
|
||||
" kv-cache on this device."
|
||||
)
|
||||
|
||||
self.sinks = sinks
|
||||
if self.sinks is not None:
|
||||
assert flash_attn_supports_sinks(), (
|
||||
|
||||
Reference in New Issue
Block a user