[SM100] Hdim 256 optimized (#52980)

Signed-off-by: Simon Veitner <sveitner@redhat.com>
This commit is contained in:
simon-veitner-redhat
2026-08-25 20:28:40 -07:00
committed by GitHub
parent f6130145c8
commit 7156c63bef
7 changed files with 423 additions and 22 deletions
+79 -14
View File
@@ -6,9 +6,13 @@ from typing import Any
import vllm.envs as envs
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.torch_utils import is_quantized_kv_cache
logger = init_logger(__name__)
# The dedicated hd256 kernel requires one 128-token page per tile.
FA4_HD256_PAGE_SIZE = 128
# Track whether upstream flash-attn is available on ROCm.
# Set during module initialization and never modified afterwards.
# This module-level flag avoids repeated import attempts and ensures
@@ -72,6 +76,9 @@ def get_flash_attn_version(
head_size: int | None = None,
head_size_v: int | None = None,
has_sinks: bool = False,
requires_softcap: bool = False,
kv_cache_block_size: int | None = None,
supports_fa4_hd256: bool = False,
) -> int | None:
if current_platform.is_xpu():
return 2
@@ -168,26 +175,31 @@ def get_flash_attn_version(
)
fa_version = 2
# TODO: Restore the `requires_local_attention` restriction when FA4
# head-dim 256 is re-enabled.
if fa_version == 4 and device_capability.major >= 10 and head_size == 256:
logger.warning_once(
"FA4 on Blackwell is temporarily disabled for head_size=256, "
"defaulting to FA version 2."
)
fa_version = 2
if fa_version == 4 and uses_fa4_hd256_kernel(head_size, head_size_v):
if not supports_fa4_hd256:
fa_version = 2
elif (
reason := _fa4_hd256_fallback_reason(
has_sinks, requires_softcap, kv_cache_block_size, vllm_config
)
) is not None:
logger.warning_once(
"FA4's Blackwell head_size=256 kernel does not support %s, "
"defaulting to FA version 2.",
reason,
)
fa_version = 2
# FA4 on SM100 (Blackwell) has TMEM capacity limits that restrict
# supported head dimensions to ≤128. The 192/128 MLA prefill case is
# supported; 256 is temporarily disabled until upstream supports the
# required features. Development of symmetric 192, 384, and 512 support
# is tracked in https://github.com/Dao-AILab/flash-attention/issues/2456
# FA4 head dimensions on Blackwell are limited by TMEM capacity.
if (
fa_version == 4
and device_capability.major >= 10
and head_size is not None
and head_size > 128
and not (head_size == 192 and head_size_v == 128)
and not (
(head_size == 256 and head_size_v in (None, 256))
or (head_size == 192 and head_size_v == 128)
)
):
logger.warning_once(
"FA4 on Blackwell does not support head_size=%d due to TMEM "
@@ -209,6 +221,53 @@ def get_flash_attn_version(
return None
def uses_fa4_hd256_kernel(
head_size: int | None, head_size_v: int | None = None
) -> bool:
"""Return whether FA4 uses its dedicated hd256 kernel."""
if head_size != 256:
return False
if head_size_v is not None and head_size_v != 256:
return False
capability = current_platform.get_device_capability()
return capability is not None and capability.major in (10, 11)
def _fa4_hd256_fallback_reason(
has_sinks: bool,
requires_softcap: bool,
kv_cache_block_size: int | None,
vllm_config: Any,
) -> str | None:
model_config = vllm_config.model_config if vllm_config is not None else None
cache_config = vllm_config.cache_config if vllm_config is not None else None
if has_sinks:
return "attention sinks"
if requires_softcap or (
# Keep model-level and per-layer version selection consistent.
model_config is not None
and getattr(model_config.hf_text_config, "attn_logit_softcapping", None)
):
return "logits soft capping"
if cache_config is not None and is_quantized_kv_cache(cache_config.cache_dtype):
return f"quantized KV cache dtype {cache_config.cache_dtype}"
if kv_cache_block_size is not None and kv_cache_block_size % FA4_HD256_PAGE_SIZE:
# Larger blocks are split into 128-token kernel pages.
return f"a KV cache block size of {kv_cache_block_size}"
if model_config is not None:
if model_config.is_mm_prefix_lm:
return "mm_prefix bidirectional attention"
if model_config.rswa_window is not None:
return "R-SWA"
if (
vllm_config is not None
and vllm_config.parallel_config.decode_context_parallel_size > 1
):
# DCP reuses an unsliced block table with a non-page-aligned length.
return "decode context parallelism"
return None
def is_fa_version_supported(fa_version: int) -> bool:
try:
from vllm.vllm_flash_attn.flash_attn_interface import (
@@ -227,6 +286,9 @@ def flash_attn_supports_kv_cache_dtype(
head_size: int | None = None,
head_size_v: int | None = None,
has_sinks: bool = False,
requires_softcap: bool = False,
kv_cache_block_size: int | None = None,
supports_fa4_hd256: bool = False,
) -> bool:
if kv_cache_dtype == "fp8_e5m2":
return False
@@ -237,6 +299,9 @@ def flash_attn_supports_kv_cache_dtype(
head_size=head_size,
head_size_v=head_size_v,
has_sinks=has_sinks,
requires_softcap=requires_softcap,
kv_cache_block_size=kv_cache_block_size,
supports_fa4_hd256=supports_fa4_hd256,
)
return (fa_version == 3 and current_platform.is_device_capability_family(90)) or (
fa_version == 4 and current_platform.is_device_capability_family(100)