[Attention] FlashAttention 4 SM100 FP8 kv cache support (#42569)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Matthew Bonanni
2026-07-20 10:53:27 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent b23bd73f54
commit bd091079cb
8 changed files with 194 additions and 33 deletions
@@ -1083,7 +1083,8 @@ def parse_flash_attn_features() -> dict[str, dict[str, Any]]:
return {}
# Analyze the functions to determine FA3/FA4-specific features
fa3_supports_fp8 = True
fa3_supports_fp8 = False
fa4_supports_fp8 = False
fa3_supports_sinks = False
fa4_supports_sinks = False
fa3_compute_cap: str | None = None
@@ -1093,6 +1094,44 @@ def parse_flash_attn_features() -> dict[str, dict[str, Any]]:
if not isinstance(node, ast.FunctionDef):
continue
# Check flash_attn_supports_kv_cache_dtype for fp8 support per FA version.
# Accept both equality checks and membership checks such as `in (3, 4)`.
if node.name == "flash_attn_supports_kv_cache_dtype":
for n in ast.walk(node):
if not (
isinstance(n, ast.Compare)
and len(n.ops) == 1
and len(n.comparators) == 1
):
continue
is_version_compare = (
isinstance(n.left, ast.Name) and n.left.id == "fa_version"
) or (
isinstance(n.left, ast.Call)
and isinstance(n.left.func, ast.Name)
and n.left.func.id == "get_flash_attn_version"
)
if not is_version_compare:
continue
versions: list[Any] = []
comparator = n.comparators[0]
if isinstance(n.ops[0], ast.Eq) and isinstance(
comparator, ast.Constant
):
versions = [comparator.value]
elif isinstance(n.ops[0], ast.In) and isinstance(
comparator, (ast.Tuple, ast.List, ast.Set)
):
versions = [
elt.value
for elt in comparator.elts
if isinstance(elt, ast.Constant)
]
fa3_supports_fp8 |= 3 in versions
fa4_supports_fp8 |= 4 in versions
# Check flash_attn_supports_sinks - looks for `fa_version == 3/4`
# or `get_flash_attn_version() == 3/4` (also accepts `in (3, 4)`)
if node.name == "flash_attn_supports_sinks":
@@ -1199,7 +1238,7 @@ def parse_flash_attn_features() -> dict[str, dict[str, Any]]:
},
"fa4": {
"compute_capability": fa4_compute_cap,
"supports_fp8": False,
"supports_fp8": fa4_supports_fp8,
"supports_sink": fa4_supports_sinks,
},
}
@@ -1255,6 +1294,22 @@ def parse_flashinfer_trtllm_features() -> dict[str, dict[str, Any]]:
# ---------------------------------------------------------------------------
def _apply_fp8_support(kv_cache_dtypes: str, supports_fp8: bool) -> str:
"""Add or remove fp8 dtypes from a comma-separated kv_cache_dtypes string.
The base FLASH_ATTN backend lists fp8 dtypes in ``supported_kv_cache_dtypes``,
but actual support varies by FA version (e.g. FA2 has no fp8 support), so each
variant's row is normalized to match its own capability.
"""
fp8_dtypes = {"fp8", "fp8_e4m3", "fp8_e5m2"}
dtypes = kv_cache_dtypes.split(", ")
base_dtypes = [dtype for dtype in dtypes if dtype not in fp8_dtypes]
if supports_fp8:
advertised_fp8_dtypes = [dtype for dtype in dtypes if dtype in fp8_dtypes]
return ", ".join(base_dtypes + advertised_fp8_dtypes)
return ", ".join(base_dtypes)
def _expand_flash_attn_variants(
all_backends: list[dict[str, Any]],
fa_features: dict[str, dict[str, Any]],
@@ -1275,6 +1330,9 @@ def _expand_flash_attn_variants(
fa2["_sort_key"] = "FLASH_ATTN"
fa2["_sort_order"] = 0
fa2["supports_sink"] = fa_features["fa2"]["supports_sink"]
fa2["kv_cache_dtypes"] = _apply_fp8_support(
backend["kv_cache_dtypes"], fa_features["fa2"]["supports_fp8"]
)
# Create FA3 entry (uses parsed compute_capability from fa_utils)
fa3 = backend.copy()
@@ -1284,11 +1342,9 @@ def _expand_flash_attn_variants(
if fa_features["fa3"]["compute_capability"]:
fa3["compute_capability"] = fa_features["fa3"]["compute_capability"]
fa3["supports_sink"] = fa_features["fa3"]["supports_sink"]
if fa_features["fa3"]["supports_fp8"]:
base_dtypes = backend["kv_cache_dtypes"].split(", ")
fp8_dtypes = ["fp8", "fp8_e4m3", "fp8_e5m2"]
new_dtypes = [d for d in fp8_dtypes if d not in base_dtypes]
fa3["kv_cache_dtypes"] = ", ".join(base_dtypes + new_dtypes)
fa3["kv_cache_dtypes"] = _apply_fp8_support(
backend["kv_cache_dtypes"], fa_features["fa3"]["supports_fp8"]
)
expanded.append(fa2)
expanded.append(fa3)
@@ -1302,6 +1358,9 @@ def _expand_flash_attn_variants(
if fa_features["fa4"].get("compute_capability"):
fa4["compute_capability"] = fa_features["fa4"]["compute_capability"]
fa4["supports_sink"] = fa_features["fa4"]["supports_sink"]
fa4["kv_cache_dtypes"] = _apply_fp8_support(
backend["kv_cache_dtypes"], fa_features["fa4"]["supports_fp8"]
)
expanded.append(fa4)
return expanded