[ROCm][Perf][MLA] Add AITER FlashAttention MLA prefill backend (ROCM_AITER_FA) (#45033)
Signed-off-by: Xavier Aguilar <xavier.aguilarfruto@amd.com> Signed-off-by: Xavier Aguilar <Xavier.AguilarFruto@amd.com> Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
This commit is contained in:
@@ -133,3 +133,20 @@ def test_clear_override():
|
||||
def test_unknown_backend_name_raises():
|
||||
with pytest.raises(ValueError, match="Unknown MLA prefill backend"):
|
||||
MLAPrefillBackendEnum["NONEXISTENT"]
|
||||
|
||||
|
||||
def test_rocm_aiter_fa_registered():
|
||||
"""ROCM_AITER_FA is a known backend pointing at the AITER FA class."""
|
||||
assert "ROCM_AITER_FA" in MLAPrefillBackendEnum.__members__
|
||||
|
||||
path = MLAPrefillBackendEnum.ROCM_AITER_FA.get_path()
|
||||
assert path == (
|
||||
"vllm.v1.attention.backends.mla.prefill.aiter_flash_attn."
|
||||
"AiterFlashAttnPrefillBackend"
|
||||
)
|
||||
|
||||
backend_cls = MLAPrefillBackendEnum.ROCM_AITER_FA.get_class()
|
||||
assert backend_cls.get_name() == "ROCM_AITER_FA"
|
||||
# The AITER FA path is the fp16/bf16 generic-varlen prefill path.
|
||||
assert backend_cls.supports_dtype(torch.bfloat16)
|
||||
assert backend_cls.supports_dtype(torch.float16)
|
||||
|
||||
@@ -14,6 +14,7 @@ from vllm.v1.attention.backends.mla.prefill.registry import MLAPrefillBackendEnu
|
||||
from vllm.v1.attention.backends.mla.prefill.selector import (
|
||||
MLAPrefillSelectorConfig,
|
||||
_auto_select_mla_prefill_backend,
|
||||
_get_mla_prefill_backend_priorities,
|
||||
get_mla_prefill_backend,
|
||||
)
|
||||
|
||||
@@ -166,6 +167,7 @@ class TestAutoSelectMLAPrefillBackend:
|
||||
return
|
||||
|
||||
with (
|
||||
patch("vllm.platforms.current_platform") as mock_platform,
|
||||
patch.object(
|
||||
MLAPrefillBackendEnum.FLASH_ATTN,
|
||||
"get_class",
|
||||
@@ -173,6 +175,8 @@ class TestAutoSelectMLAPrefillBackend:
|
||||
),
|
||||
patch.object(trtllm_cls, "validate_configuration", return_value=[]),
|
||||
):
|
||||
# Force the non-ROCm priority on the Blackwell.
|
||||
mock_platform.is_rocm.return_value = False
|
||||
backend = _auto_select_mla_prefill_backend(
|
||||
capability,
|
||||
selector_config,
|
||||
@@ -272,6 +276,121 @@ class TestBackendValidation:
|
||||
assert invalid_reasons == []
|
||||
|
||||
|
||||
class TestROCmAiterFAPrefillSelection:
|
||||
"""Tests for the ROCm AITER FlashAttention MLA prefill backend."""
|
||||
|
||||
def test_rocm_priorities_prefer_aiter_fa(self):
|
||||
"""On ROCm, ROCM_AITER_FA is tried first, FLASH_ATTN as fallback."""
|
||||
with patch("vllm.platforms.current_platform") as mock_platform:
|
||||
mock_platform.is_rocm.return_value = True
|
||||
priorities = _get_mla_prefill_backend_priorities(
|
||||
DeviceCapability(major=9, minor=5)
|
||||
)
|
||||
|
||||
assert priorities == [
|
||||
MLAPrefillBackendEnum.ROCM_AITER_FA,
|
||||
MLAPrefillBackendEnum.FLASH_ATTN,
|
||||
]
|
||||
|
||||
def test_supported_dtypes_are_fp16_bf16_only(self):
|
||||
from vllm.v1.attention.backends.mla.prefill.aiter_flash_attn import (
|
||||
AiterFlashAttnPrefillBackend,
|
||||
)
|
||||
|
||||
assert AiterFlashAttnPrefillBackend.supports_dtype(torch.bfloat16)
|
||||
assert AiterFlashAttnPrefillBackend.supports_dtype(torch.float16)
|
||||
# FP8 is served by the separate AITER ASM backend, not this one.
|
||||
assert not AiterFlashAttnPrefillBackend.supports_dtype(torch.float8_e4m3fn)
|
||||
|
||||
def test_supports_compute_capability_on_rocm(self):
|
||||
from vllm.v1.attention.backends.mla.prefill import aiter_flash_attn as mod
|
||||
|
||||
# Gating is decided by on_mi3xx(), not by capability
|
||||
capability = MagicMock()
|
||||
|
||||
with patch.object(mod.current_platform, "is_rocm", return_value=False):
|
||||
assert not mod.AiterFlashAttnPrefillBackend.supports_compute_capability(
|
||||
capability
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(mod.current_platform, "is_rocm", return_value=True),
|
||||
patch("vllm.platforms.rocm.on_mi3xx", return_value=False),
|
||||
):
|
||||
assert not mod.AiterFlashAttnPrefillBackend.supports_compute_capability(
|
||||
capability
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(mod.current_platform, "is_rocm", return_value=True),
|
||||
patch("vllm.platforms.rocm.on_mi3xx", return_value=True),
|
||||
):
|
||||
assert mod.AiterFlashAttnPrefillBackend.supports_compute_capability(
|
||||
capability
|
||||
)
|
||||
|
||||
def test_is_available_delegates_to_rocm_aiter_ops(self):
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.v1.attention.backends.mla.prefill import aiter_flash_attn as mod
|
||||
|
||||
with patch.object(rocm_aiter_ops, "is_enabled", return_value=False):
|
||||
assert not mod.AiterFlashAttnPrefillBackend.is_available()
|
||||
|
||||
with patch.object(rocm_aiter_ops, "is_enabled", return_value=True):
|
||||
assert mod.AiterFlashAttnPrefillBackend.is_available()
|
||||
|
||||
def test_auto_select_prefers_aiter_fa_on_rocm(self):
|
||||
from vllm.v1.attention.backends.mla.prefill.aiter_flash_attn import (
|
||||
AiterFlashAttnPrefillBackend,
|
||||
)
|
||||
|
||||
# gfx gating is simulated via the mocked validate_configuration,
|
||||
# not the capability.
|
||||
capability = MagicMock()
|
||||
selector_config = MLAPrefillSelectorConfig(dtype=torch.bfloat16)
|
||||
|
||||
with (
|
||||
patch("vllm.platforms.current_platform") as mock_platform,
|
||||
patch.object(
|
||||
AiterFlashAttnPrefillBackend,
|
||||
"validate_configuration",
|
||||
return_value=[],
|
||||
),
|
||||
):
|
||||
mock_platform.is_rocm.return_value = True
|
||||
backend = _auto_select_mla_prefill_backend(capability, selector_config)
|
||||
assert backend.get_name() == "ROCM_AITER_FA"
|
||||
|
||||
def test_auto_select_falls_back_to_flash_attn_when_aiter_invalid(self):
|
||||
from vllm.v1.attention.backends.mla.prefill.aiter_flash_attn import (
|
||||
AiterFlashAttnPrefillBackend,
|
||||
)
|
||||
|
||||
try:
|
||||
flash_attn_cls = MLAPrefillBackendEnum.FLASH_ATTN.get_class()
|
||||
except ImportError:
|
||||
pytest.skip("FLASH_ATTN backend not available")
|
||||
return
|
||||
|
||||
# the fallback is forced by the mocked validate_configuration,
|
||||
# not the capability.
|
||||
capability = MagicMock()
|
||||
selector_config = MLAPrefillSelectorConfig(dtype=torch.bfloat16)
|
||||
|
||||
with (
|
||||
patch("vllm.platforms.current_platform") as mock_platform,
|
||||
patch.object(
|
||||
AiterFlashAttnPrefillBackend,
|
||||
"validate_configuration",
|
||||
return_value=["compute capability not supported"],
|
||||
),
|
||||
patch.object(flash_attn_cls, "validate_configuration", return_value=[]),
|
||||
):
|
||||
mock_platform.is_rocm.return_value = True
|
||||
backend = _auto_select_mla_prefill_backend(capability, selector_config)
|
||||
assert backend.get_name() == "FLASH_ATTN"
|
||||
|
||||
|
||||
class TestMLAPrefillBackendParsing:
|
||||
"""Tests for string-based mla_prefill_backend parsing from CLI args."""
|
||||
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""AITER FlashAttention backend for MLA prefill (ROCm).
|
||||
|
||||
This backend calls ``aiter.flash_attn_varlen_func`` directly, which natively
|
||||
supports different q/k and v head dims (qk headdim 192, v headdim 128) without
|
||||
padding V, and dispatches to the fast ``aiter::fmha_fwd_`` kernel on
|
||||
gfx942/gfx950 (fp16/bf16).
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.mla.prefill.base import MLAPrefillBackend
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
|
||||
|
||||
class AiterFlashAttnPrefillBackend(MLAPrefillBackend):
|
||||
"""AITER FlashAttention backend for MLA prefill"""
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "ROCM_AITER_FA"
|
||||
|
||||
@classmethod
|
||||
def supports_compute_capability(cls, device_capability: "DeviceCapability") -> bool:
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm.platforms.rocm import on_mi3xx
|
||||
|
||||
return on_mi3xx()
|
||||
|
||||
@classmethod
|
||||
def is_available(cls) -> bool:
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
return rocm_aiter_ops.is_enabled()
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_heads: int,
|
||||
scale: float,
|
||||
kv_lora_rank: int,
|
||||
qk_nope_head_dim: int,
|
||||
qk_rope_head_dim: int,
|
||||
v_head_dim: int,
|
||||
vllm_config: "VllmConfig",
|
||||
) -> None:
|
||||
super().__init__(
|
||||
num_heads=num_heads,
|
||||
scale=scale,
|
||||
kv_lora_rank=kv_lora_rank,
|
||||
qk_nope_head_dim=qk_nope_head_dim,
|
||||
qk_rope_head_dim=qk_rope_head_dim,
|
||||
v_head_dim=v_head_dim,
|
||||
vllm_config=vllm_config,
|
||||
)
|
||||
|
||||
from aiter import flash_attn_varlen_func
|
||||
|
||||
self.flash_attn_varlen_func = flash_attn_varlen_func
|
||||
|
||||
def run_prefill_new_tokens(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
return_softmax_lse: bool,
|
||||
out: torch.Tensor | None = None,
|
||||
output_scale: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
assert output_scale is None, (
|
||||
"AiterFlashAttnPrefillBackend does not support fused quantized output."
|
||||
)
|
||||
result = self.flash_attn_varlen_func(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
cu_seqlens_q=self._prefill_metadata.query_start_loc,
|
||||
cu_seqlens_k=self._prefill_metadata.query_start_loc,
|
||||
max_seqlen_q=self._prefill_metadata.max_query_len,
|
||||
max_seqlen_k=self._prefill_metadata.max_query_len,
|
||||
softmax_scale=self.scale,
|
||||
causal=True,
|
||||
return_lse=return_softmax_lse,
|
||||
out=out,
|
||||
)
|
||||
|
||||
# aiter returns the bare output tensor when return_lse is False, and
|
||||
# (out, softmax_lse) when it is True.
|
||||
if return_softmax_lse:
|
||||
return result[0], result[1]
|
||||
return result
|
||||
|
||||
def run_prefill_context_chunk(
|
||||
self,
|
||||
chunk_idx: int,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert self._prefill_metadata.chunked_context is not None
|
||||
chunked = self._prefill_metadata.chunked_context
|
||||
out, lse = self.flash_attn_varlen_func(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
cu_seqlens_q=self._prefill_metadata.query_start_loc,
|
||||
cu_seqlens_k=chunked.cu_seq_lens[chunk_idx],
|
||||
max_seqlen_q=self._prefill_metadata.max_query_len,
|
||||
max_seqlen_k=chunked.max_seq_lens[chunk_idx],
|
||||
softmax_scale=self.scale,
|
||||
causal=False,
|
||||
return_lse=True,
|
||||
)
|
||||
return out, lse
|
||||
@@ -48,6 +48,10 @@ class MLAPrefillBackendEnum(Enum, metaclass=_MLAPrefillBackendEnumMeta):
|
||||
"vllm.v1.attention.backends.mla.prefill.tokenspeed_mla."
|
||||
"TokenspeedMLAPrefillBackend"
|
||||
)
|
||||
ROCM_AITER_FA = (
|
||||
"vllm.v1.attention.backends.mla.prefill.aiter_flash_attn."
|
||||
"AiterFlashAttnPrefillBackend"
|
||||
)
|
||||
# Placeholder for third-party/custom backends - must be registered before use
|
||||
# set to None to avoid alias with other backend, whose value is an empty string
|
||||
CUSTOM = None
|
||||
|
||||
@@ -56,6 +56,14 @@ def _get_mla_prefill_backend_priorities(
|
||||
Returns:
|
||||
List of backends in priority order (highest priority first).
|
||||
"""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_rocm():
|
||||
return [
|
||||
MLAPrefillBackendEnum.ROCM_AITER_FA,
|
||||
MLAPrefillBackendEnum.FLASH_ATTN,
|
||||
]
|
||||
|
||||
if device_capability.major == 10: # Blackwell
|
||||
return [
|
||||
MLAPrefillBackendEnum.FLASH_ATTN,
|
||||
|
||||
Reference in New Issue
Block a user