[Bugfix][Attention] Temporarily disable FA4 head-dim 256 (#52050)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
co-authored by
Wentao Ye
parent
7d7b6f26f4
commit
1f0e0bf612
@@ -223,7 +223,10 @@ def _run_multimodal_text_query_image_docs_test(
|
||||
max_model_len=4096,
|
||||
enforce_eager=True,
|
||||
gpu_memory_utilization=GPU_MEMORY_UTILIZATION,
|
||||
attention_backend="FLASH_ATTN",
|
||||
kernel_config={"enable_flashinfer_autotune": False},
|
||||
) as vllm_model:
|
||||
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
scores = vllm_model.llm.score(query, image_docs)
|
||||
|
||||
assert len(scores) == 2
|
||||
@@ -297,9 +300,11 @@ def _run_multimodal_image_query_text_docs_test(
|
||||
@pytest.mark.parametrize("dtype", [DTYPE])
|
||||
def test_colpali_multimodal_text_query_image_docs(
|
||||
vllm_runner,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
_run_multimodal_text_query_image_docs_test(vllm_runner, model, dtype=dtype)
|
||||
|
||||
|
||||
|
||||
@@ -72,7 +72,6 @@ def get_flash_attn_version(
|
||||
head_size: int | None = None,
|
||||
head_size_v: int | None = None,
|
||||
has_sinks: bool = False,
|
||||
requires_local_attention: bool = False,
|
||||
) -> int | None:
|
||||
if current_platform.is_xpu():
|
||||
return 2
|
||||
@@ -169,28 +168,26 @@ def get_flash_attn_version(
|
||||
)
|
||||
fa_version = 2
|
||||
|
||||
if (
|
||||
fa_version == 4
|
||||
and device_capability.major >= 10
|
||||
and head_size == 256
|
||||
and requires_local_attention
|
||||
):
|
||||
# TODO: Restore the `requires_local_attention` restriction when FA4
|
||||
# head-dim 256 is re-enabled.
|
||||
if fa_version == 4 and device_capability.major >= 10 and head_size == 256:
|
||||
logger.warning_once(
|
||||
"FA4 on Blackwell does not support local attention with "
|
||||
"head_size=256, defaulting to FA version 2."
|
||||
"FA4 on Blackwell is temporarily disabled for head_size=256, "
|
||||
"defaulting to FA version 2."
|
||||
)
|
||||
fa_version = 2
|
||||
|
||||
# FA4 on SM100 (Blackwell) has TMEM capacity limits that restrict
|
||||
# supported head dimensions to ≤128, with exceptions for 256 and 192/128 (MLA
|
||||
# prefill). Development of symmetric 192, 384, and 512 support is being tracked
|
||||
# in https://github.com/Dao-AILab/flash-attention/issues/2456
|
||||
# supported head dimensions to ≤128. The 192/128 MLA prefill case is
|
||||
# supported; 256 is temporarily disabled until upstream supports the
|
||||
# required features. Development of symmetric 192, 384, and 512 support
|
||||
# is tracked in https://github.com/Dao-AILab/flash-attention/issues/2456
|
||||
if (
|
||||
fa_version == 4
|
||||
and device_capability.major >= 10
|
||||
and head_size is not None
|
||||
and head_size > 128
|
||||
and not (head_size == 256 or (head_size == 192 and head_size_v == 128))
|
||||
and not (head_size == 192 and head_size_v == 128)
|
||||
):
|
||||
logger.warning_once(
|
||||
"FA4 on Blackwell does not support head_size=%d due to TMEM "
|
||||
|
||||
@@ -905,7 +905,6 @@ class FlashAttentionImpl(AttentionImpl):
|
||||
self.attn_type = attn_type
|
||||
self.vllm_flash_attn_version = get_flash_attn_version(
|
||||
requires_alibi=alibi_slopes is not None,
|
||||
requires_local_attention=sliding_window is not None,
|
||||
head_size=head_size,
|
||||
has_sinks=sinks is not None,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user