[Attention] FlashAttention 4 SM100 FP8 kv cache support (#42569)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
b23bd73f54
commit
bd091079cb
@@ -1083,7 +1083,8 @@ def parse_flash_attn_features() -> dict[str, dict[str, Any]]:
|
||||
return {}
|
||||
|
||||
# Analyze the functions to determine FA3/FA4-specific features
|
||||
fa3_supports_fp8 = True
|
||||
fa3_supports_fp8 = False
|
||||
fa4_supports_fp8 = False
|
||||
fa3_supports_sinks = False
|
||||
fa4_supports_sinks = False
|
||||
fa3_compute_cap: str | None = None
|
||||
@@ -1093,6 +1094,44 @@ def parse_flash_attn_features() -> dict[str, dict[str, Any]]:
|
||||
if not isinstance(node, ast.FunctionDef):
|
||||
continue
|
||||
|
||||
# Check flash_attn_supports_kv_cache_dtype for fp8 support per FA version.
|
||||
# Accept both equality checks and membership checks such as `in (3, 4)`.
|
||||
if node.name == "flash_attn_supports_kv_cache_dtype":
|
||||
for n in ast.walk(node):
|
||||
if not (
|
||||
isinstance(n, ast.Compare)
|
||||
and len(n.ops) == 1
|
||||
and len(n.comparators) == 1
|
||||
):
|
||||
continue
|
||||
is_version_compare = (
|
||||
isinstance(n.left, ast.Name) and n.left.id == "fa_version"
|
||||
) or (
|
||||
isinstance(n.left, ast.Call)
|
||||
and isinstance(n.left.func, ast.Name)
|
||||
and n.left.func.id == "get_flash_attn_version"
|
||||
)
|
||||
if not is_version_compare:
|
||||
continue
|
||||
|
||||
versions: list[Any] = []
|
||||
comparator = n.comparators[0]
|
||||
if isinstance(n.ops[0], ast.Eq) and isinstance(
|
||||
comparator, ast.Constant
|
||||
):
|
||||
versions = [comparator.value]
|
||||
elif isinstance(n.ops[0], ast.In) and isinstance(
|
||||
comparator, (ast.Tuple, ast.List, ast.Set)
|
||||
):
|
||||
versions = [
|
||||
elt.value
|
||||
for elt in comparator.elts
|
||||
if isinstance(elt, ast.Constant)
|
||||
]
|
||||
|
||||
fa3_supports_fp8 |= 3 in versions
|
||||
fa4_supports_fp8 |= 4 in versions
|
||||
|
||||
# Check flash_attn_supports_sinks - looks for `fa_version == 3/4`
|
||||
# or `get_flash_attn_version() == 3/4` (also accepts `in (3, 4)`)
|
||||
if node.name == "flash_attn_supports_sinks":
|
||||
@@ -1199,7 +1238,7 @@ def parse_flash_attn_features() -> dict[str, dict[str, Any]]:
|
||||
},
|
||||
"fa4": {
|
||||
"compute_capability": fa4_compute_cap,
|
||||
"supports_fp8": False,
|
||||
"supports_fp8": fa4_supports_fp8,
|
||||
"supports_sink": fa4_supports_sinks,
|
||||
},
|
||||
}
|
||||
@@ -1255,6 +1294,22 @@ def parse_flashinfer_trtllm_features() -> dict[str, dict[str, Any]]:
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _apply_fp8_support(kv_cache_dtypes: str, supports_fp8: bool) -> str:
|
||||
"""Add or remove fp8 dtypes from a comma-separated kv_cache_dtypes string.
|
||||
|
||||
The base FLASH_ATTN backend lists fp8 dtypes in ``supported_kv_cache_dtypes``,
|
||||
but actual support varies by FA version (e.g. FA2 has no fp8 support), so each
|
||||
variant's row is normalized to match its own capability.
|
||||
"""
|
||||
fp8_dtypes = {"fp8", "fp8_e4m3", "fp8_e5m2"}
|
||||
dtypes = kv_cache_dtypes.split(", ")
|
||||
base_dtypes = [dtype for dtype in dtypes if dtype not in fp8_dtypes]
|
||||
if supports_fp8:
|
||||
advertised_fp8_dtypes = [dtype for dtype in dtypes if dtype in fp8_dtypes]
|
||||
return ", ".join(base_dtypes + advertised_fp8_dtypes)
|
||||
return ", ".join(base_dtypes)
|
||||
|
||||
|
||||
def _expand_flash_attn_variants(
|
||||
all_backends: list[dict[str, Any]],
|
||||
fa_features: dict[str, dict[str, Any]],
|
||||
@@ -1275,6 +1330,9 @@ def _expand_flash_attn_variants(
|
||||
fa2["_sort_key"] = "FLASH_ATTN"
|
||||
fa2["_sort_order"] = 0
|
||||
fa2["supports_sink"] = fa_features["fa2"]["supports_sink"]
|
||||
fa2["kv_cache_dtypes"] = _apply_fp8_support(
|
||||
backend["kv_cache_dtypes"], fa_features["fa2"]["supports_fp8"]
|
||||
)
|
||||
|
||||
# Create FA3 entry (uses parsed compute_capability from fa_utils)
|
||||
fa3 = backend.copy()
|
||||
@@ -1284,11 +1342,9 @@ def _expand_flash_attn_variants(
|
||||
if fa_features["fa3"]["compute_capability"]:
|
||||
fa3["compute_capability"] = fa_features["fa3"]["compute_capability"]
|
||||
fa3["supports_sink"] = fa_features["fa3"]["supports_sink"]
|
||||
if fa_features["fa3"]["supports_fp8"]:
|
||||
base_dtypes = backend["kv_cache_dtypes"].split(", ")
|
||||
fp8_dtypes = ["fp8", "fp8_e4m3", "fp8_e5m2"]
|
||||
new_dtypes = [d for d in fp8_dtypes if d not in base_dtypes]
|
||||
fa3["kv_cache_dtypes"] = ", ".join(base_dtypes + new_dtypes)
|
||||
fa3["kv_cache_dtypes"] = _apply_fp8_support(
|
||||
backend["kv_cache_dtypes"], fa_features["fa3"]["supports_fp8"]
|
||||
)
|
||||
|
||||
expanded.append(fa2)
|
||||
expanded.append(fa3)
|
||||
@@ -1302,6 +1358,9 @@ def _expand_flash_attn_variants(
|
||||
if fa_features["fa4"].get("compute_capability"):
|
||||
fa4["compute_capability"] = fa_features["fa4"]["compute_capability"]
|
||||
fa4["supports_sink"] = fa_features["fa4"]["supports_sink"]
|
||||
fa4["kv_cache_dtypes"] = _apply_fp8_support(
|
||||
backend["kv_cache_dtypes"], fa_features["fa4"]["supports_fp8"]
|
||||
)
|
||||
expanded.append(fa4)
|
||||
|
||||
return expanded
|
||||
|
||||
Reference in New Issue
Block a user