[SM100] Hdim 256 optimized (#52980)
Signed-off-by: Simon Veitner <sveitner@redhat.com>
This commit is contained in:
@@ -6,9 +6,13 @@ from typing import Any
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# The dedicated hd256 kernel requires one 128-token page per tile.
|
||||
FA4_HD256_PAGE_SIZE = 128
|
||||
|
||||
# Track whether upstream flash-attn is available on ROCm.
|
||||
# Set during module initialization and never modified afterwards.
|
||||
# This module-level flag avoids repeated import attempts and ensures
|
||||
@@ -72,6 +76,9 @@ def get_flash_attn_version(
|
||||
head_size: int | None = None,
|
||||
head_size_v: int | None = None,
|
||||
has_sinks: bool = False,
|
||||
requires_softcap: bool = False,
|
||||
kv_cache_block_size: int | None = None,
|
||||
supports_fa4_hd256: bool = False,
|
||||
) -> int | None:
|
||||
if current_platform.is_xpu():
|
||||
return 2
|
||||
@@ -168,26 +175,31 @@ def get_flash_attn_version(
|
||||
)
|
||||
fa_version = 2
|
||||
|
||||
# TODO: Restore the `requires_local_attention` restriction when FA4
|
||||
# head-dim 256 is re-enabled.
|
||||
if fa_version == 4 and device_capability.major >= 10 and head_size == 256:
|
||||
logger.warning_once(
|
||||
"FA4 on Blackwell is temporarily disabled for head_size=256, "
|
||||
"defaulting to FA version 2."
|
||||
)
|
||||
fa_version = 2
|
||||
if fa_version == 4 and uses_fa4_hd256_kernel(head_size, head_size_v):
|
||||
if not supports_fa4_hd256:
|
||||
fa_version = 2
|
||||
elif (
|
||||
reason := _fa4_hd256_fallback_reason(
|
||||
has_sinks, requires_softcap, kv_cache_block_size, vllm_config
|
||||
)
|
||||
) is not None:
|
||||
logger.warning_once(
|
||||
"FA4's Blackwell head_size=256 kernel does not support %s, "
|
||||
"defaulting to FA version 2.",
|
||||
reason,
|
||||
)
|
||||
fa_version = 2
|
||||
|
||||
# FA4 on SM100 (Blackwell) has TMEM capacity limits that restrict
|
||||
# supported head dimensions to ≤128. The 192/128 MLA prefill case is
|
||||
# supported; 256 is temporarily disabled until upstream supports the
|
||||
# required features. Development of symmetric 192, 384, and 512 support
|
||||
# is tracked in https://github.com/Dao-AILab/flash-attention/issues/2456
|
||||
# FA4 head dimensions on Blackwell are limited by TMEM capacity.
|
||||
if (
|
||||
fa_version == 4
|
||||
and device_capability.major >= 10
|
||||
and head_size is not None
|
||||
and head_size > 128
|
||||
and not (head_size == 192 and head_size_v == 128)
|
||||
and not (
|
||||
(head_size == 256 and head_size_v in (None, 256))
|
||||
or (head_size == 192 and head_size_v == 128)
|
||||
)
|
||||
):
|
||||
logger.warning_once(
|
||||
"FA4 on Blackwell does not support head_size=%d due to TMEM "
|
||||
@@ -209,6 +221,53 @@ def get_flash_attn_version(
|
||||
return None
|
||||
|
||||
|
||||
def uses_fa4_hd256_kernel(
|
||||
head_size: int | None, head_size_v: int | None = None
|
||||
) -> bool:
|
||||
"""Return whether FA4 uses its dedicated hd256 kernel."""
|
||||
if head_size != 256:
|
||||
return False
|
||||
if head_size_v is not None and head_size_v != 256:
|
||||
return False
|
||||
capability = current_platform.get_device_capability()
|
||||
return capability is not None and capability.major in (10, 11)
|
||||
|
||||
|
||||
def _fa4_hd256_fallback_reason(
|
||||
has_sinks: bool,
|
||||
requires_softcap: bool,
|
||||
kv_cache_block_size: int | None,
|
||||
vllm_config: Any,
|
||||
) -> str | None:
|
||||
model_config = vllm_config.model_config if vllm_config is not None else None
|
||||
cache_config = vllm_config.cache_config if vllm_config is not None else None
|
||||
if has_sinks:
|
||||
return "attention sinks"
|
||||
if requires_softcap or (
|
||||
# Keep model-level and per-layer version selection consistent.
|
||||
model_config is not None
|
||||
and getattr(model_config.hf_text_config, "attn_logit_softcapping", None)
|
||||
):
|
||||
return "logits soft capping"
|
||||
if cache_config is not None and is_quantized_kv_cache(cache_config.cache_dtype):
|
||||
return f"quantized KV cache dtype {cache_config.cache_dtype}"
|
||||
if kv_cache_block_size is not None and kv_cache_block_size % FA4_HD256_PAGE_SIZE:
|
||||
# Larger blocks are split into 128-token kernel pages.
|
||||
return f"a KV cache block size of {kv_cache_block_size}"
|
||||
if model_config is not None:
|
||||
if model_config.is_mm_prefix_lm:
|
||||
return "mm_prefix bidirectional attention"
|
||||
if model_config.rswa_window is not None:
|
||||
return "R-SWA"
|
||||
if (
|
||||
vllm_config is not None
|
||||
and vllm_config.parallel_config.decode_context_parallel_size > 1
|
||||
):
|
||||
# DCP reuses an unsliced block table with a non-page-aligned length.
|
||||
return "decode context parallelism"
|
||||
return None
|
||||
|
||||
|
||||
def is_fa_version_supported(fa_version: int) -> bool:
|
||||
try:
|
||||
from vllm.vllm_flash_attn.flash_attn_interface import (
|
||||
@@ -227,6 +286,9 @@ def flash_attn_supports_kv_cache_dtype(
|
||||
head_size: int | None = None,
|
||||
head_size_v: int | None = None,
|
||||
has_sinks: bool = False,
|
||||
requires_softcap: bool = False,
|
||||
kv_cache_block_size: int | None = None,
|
||||
supports_fa4_hd256: bool = False,
|
||||
) -> bool:
|
||||
if kv_cache_dtype == "fp8_e5m2":
|
||||
return False
|
||||
@@ -237,6 +299,9 @@ def flash_attn_supports_kv_cache_dtype(
|
||||
head_size=head_size,
|
||||
head_size_v=head_size_v,
|
||||
has_sinks=has_sinks,
|
||||
requires_softcap=requires_softcap,
|
||||
kv_cache_block_size=kv_cache_block_size,
|
||||
supports_fa4_hd256=supports_fa4_hd256,
|
||||
)
|
||||
return (fa_version == 3 and current_platform.is_device_capability_family(90)) or (
|
||||
fa_version == 4 and current_platform.is_device_capability_family(100)
|
||||
|
||||
Reference in New Issue
Block a user