diff --git a/tests/v1/attention/test_mla_prefill_registry.py b/tests/v1/attention/test_mla_prefill_registry.py index 668c17c3f5..dfa3a029ce 100644 --- a/tests/v1/attention/test_mla_prefill_registry.py +++ b/tests/v1/attention/test_mla_prefill_registry.py @@ -133,3 +133,20 @@ def test_clear_override(): def test_unknown_backend_name_raises(): with pytest.raises(ValueError, match="Unknown MLA prefill backend"): MLAPrefillBackendEnum["NONEXISTENT"] + + +def test_rocm_aiter_fa_registered(): + """ROCM_AITER_FA is a known backend pointing at the AITER FA class.""" + assert "ROCM_AITER_FA" in MLAPrefillBackendEnum.__members__ + + path = MLAPrefillBackendEnum.ROCM_AITER_FA.get_path() + assert path == ( + "vllm.v1.attention.backends.mla.prefill.aiter_flash_attn." + "AiterFlashAttnPrefillBackend" + ) + + backend_cls = MLAPrefillBackendEnum.ROCM_AITER_FA.get_class() + assert backend_cls.get_name() == "ROCM_AITER_FA" + # The AITER FA path is the fp16/bf16 generic-varlen prefill path. + assert backend_cls.supports_dtype(torch.bfloat16) + assert backend_cls.supports_dtype(torch.float16) diff --git a/tests/v1/attention/test_mla_prefill_selector.py b/tests/v1/attention/test_mla_prefill_selector.py index 54e68e03f2..c893203246 100644 --- a/tests/v1/attention/test_mla_prefill_selector.py +++ b/tests/v1/attention/test_mla_prefill_selector.py @@ -14,6 +14,7 @@ from vllm.v1.attention.backends.mla.prefill.registry import MLAPrefillBackendEnu from vllm.v1.attention.backends.mla.prefill.selector import ( MLAPrefillSelectorConfig, _auto_select_mla_prefill_backend, + _get_mla_prefill_backend_priorities, get_mla_prefill_backend, ) @@ -166,6 +167,7 @@ class TestAutoSelectMLAPrefillBackend: return with ( + patch("vllm.platforms.current_platform") as mock_platform, patch.object( MLAPrefillBackendEnum.FLASH_ATTN, "get_class", @@ -173,6 +175,8 @@ class TestAutoSelectMLAPrefillBackend: ), patch.object(trtllm_cls, "validate_configuration", return_value=[]), ): + # Force the non-ROCm priority on the Blackwell. + mock_platform.is_rocm.return_value = False backend = _auto_select_mla_prefill_backend( capability, selector_config, @@ -272,6 +276,121 @@ class TestBackendValidation: assert invalid_reasons == [] +class TestROCmAiterFAPrefillSelection: + """Tests for the ROCm AITER FlashAttention MLA prefill backend.""" + + def test_rocm_priorities_prefer_aiter_fa(self): + """On ROCm, ROCM_AITER_FA is tried first, FLASH_ATTN as fallback.""" + with patch("vllm.platforms.current_platform") as mock_platform: + mock_platform.is_rocm.return_value = True + priorities = _get_mla_prefill_backend_priorities( + DeviceCapability(major=9, minor=5) + ) + + assert priorities == [ + MLAPrefillBackendEnum.ROCM_AITER_FA, + MLAPrefillBackendEnum.FLASH_ATTN, + ] + + def test_supported_dtypes_are_fp16_bf16_only(self): + from vllm.v1.attention.backends.mla.prefill.aiter_flash_attn import ( + AiterFlashAttnPrefillBackend, + ) + + assert AiterFlashAttnPrefillBackend.supports_dtype(torch.bfloat16) + assert AiterFlashAttnPrefillBackend.supports_dtype(torch.float16) + # FP8 is served by the separate AITER ASM backend, not this one. + assert not AiterFlashAttnPrefillBackend.supports_dtype(torch.float8_e4m3fn) + + def test_supports_compute_capability_on_rocm(self): + from vllm.v1.attention.backends.mla.prefill import aiter_flash_attn as mod + + # Gating is decided by on_mi3xx(), not by capability + capability = MagicMock() + + with patch.object(mod.current_platform, "is_rocm", return_value=False): + assert not mod.AiterFlashAttnPrefillBackend.supports_compute_capability( + capability + ) + + with ( + patch.object(mod.current_platform, "is_rocm", return_value=True), + patch("vllm.platforms.rocm.on_mi3xx", return_value=False), + ): + assert not mod.AiterFlashAttnPrefillBackend.supports_compute_capability( + capability + ) + + with ( + patch.object(mod.current_platform, "is_rocm", return_value=True), + patch("vllm.platforms.rocm.on_mi3xx", return_value=True), + ): + assert mod.AiterFlashAttnPrefillBackend.supports_compute_capability( + capability + ) + + def test_is_available_delegates_to_rocm_aiter_ops(self): + from vllm._aiter_ops import rocm_aiter_ops + from vllm.v1.attention.backends.mla.prefill import aiter_flash_attn as mod + + with patch.object(rocm_aiter_ops, "is_enabled", return_value=False): + assert not mod.AiterFlashAttnPrefillBackend.is_available() + + with patch.object(rocm_aiter_ops, "is_enabled", return_value=True): + assert mod.AiterFlashAttnPrefillBackend.is_available() + + def test_auto_select_prefers_aiter_fa_on_rocm(self): + from vllm.v1.attention.backends.mla.prefill.aiter_flash_attn import ( + AiterFlashAttnPrefillBackend, + ) + + # gfx gating is simulated via the mocked validate_configuration, + # not the capability. + capability = MagicMock() + selector_config = MLAPrefillSelectorConfig(dtype=torch.bfloat16) + + with ( + patch("vllm.platforms.current_platform") as mock_platform, + patch.object( + AiterFlashAttnPrefillBackend, + "validate_configuration", + return_value=[], + ), + ): + mock_platform.is_rocm.return_value = True + backend = _auto_select_mla_prefill_backend(capability, selector_config) + assert backend.get_name() == "ROCM_AITER_FA" + + def test_auto_select_falls_back_to_flash_attn_when_aiter_invalid(self): + from vllm.v1.attention.backends.mla.prefill.aiter_flash_attn import ( + AiterFlashAttnPrefillBackend, + ) + + try: + flash_attn_cls = MLAPrefillBackendEnum.FLASH_ATTN.get_class() + except ImportError: + pytest.skip("FLASH_ATTN backend not available") + return + + # the fallback is forced by the mocked validate_configuration, + # not the capability. + capability = MagicMock() + selector_config = MLAPrefillSelectorConfig(dtype=torch.bfloat16) + + with ( + patch("vllm.platforms.current_platform") as mock_platform, + patch.object( + AiterFlashAttnPrefillBackend, + "validate_configuration", + return_value=["compute capability not supported"], + ), + patch.object(flash_attn_cls, "validate_configuration", return_value=[]), + ): + mock_platform.is_rocm.return_value = True + backend = _auto_select_mla_prefill_backend(capability, selector_config) + assert backend.get_name() == "FLASH_ATTN" + + class TestMLAPrefillBackendParsing: """Tests for string-based mla_prefill_backend parsing from CLI args.""" diff --git a/vllm/v1/attention/backends/mla/prefill/aiter_flash_attn.py b/vllm/v1/attention/backends/mla/prefill/aiter_flash_attn.py new file mode 100644 index 0000000000..130fcf394b --- /dev/null +++ b/vllm/v1/attention/backends/mla/prefill/aiter_flash_attn.py @@ -0,0 +1,121 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""AITER FlashAttention backend for MLA prefill (ROCm). + +This backend calls ``aiter.flash_attn_varlen_func`` directly, which natively +supports different q/k and v head dims (qk headdim 192, v headdim 128) without +padding V, and dispatches to the fast ``aiter::fmha_fwd_`` kernel on +gfx942/gfx950 (fp16/bf16). +""" + +from typing import TYPE_CHECKING + +import torch + +from vllm.platforms import current_platform +from vllm.v1.attention.backends.mla.prefill.base import MLAPrefillBackend + +if TYPE_CHECKING: + from vllm.config import VllmConfig + from vllm.platforms.interface import DeviceCapability + + +class AiterFlashAttnPrefillBackend(MLAPrefillBackend): + """AITER FlashAttention backend for MLA prefill""" + + @staticmethod + def get_name() -> str: + return "ROCM_AITER_FA" + + @classmethod + def supports_compute_capability(cls, device_capability: "DeviceCapability") -> bool: + if not current_platform.is_rocm(): + return False + from vllm.platforms.rocm import on_mi3xx + + return on_mi3xx() + + @classmethod + def is_available(cls) -> bool: + from vllm._aiter_ops import rocm_aiter_ops + + return rocm_aiter_ops.is_enabled() + + def __init__( + self, + num_heads: int, + scale: float, + kv_lora_rank: int, + qk_nope_head_dim: int, + qk_rope_head_dim: int, + v_head_dim: int, + vllm_config: "VllmConfig", + ) -> None: + super().__init__( + num_heads=num_heads, + scale=scale, + kv_lora_rank=kv_lora_rank, + qk_nope_head_dim=qk_nope_head_dim, + qk_rope_head_dim=qk_rope_head_dim, + v_head_dim=v_head_dim, + vllm_config=vllm_config, + ) + + from aiter import flash_attn_varlen_func + + self.flash_attn_varlen_func = flash_attn_varlen_func + + def run_prefill_new_tokens( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + return_softmax_lse: bool, + out: torch.Tensor | None = None, + output_scale: torch.Tensor | None = None, + ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + assert output_scale is None, ( + "AiterFlashAttnPrefillBackend does not support fused quantized output." + ) + result = self.flash_attn_varlen_func( + q=q, + k=k, + v=v, + cu_seqlens_q=self._prefill_metadata.query_start_loc, + cu_seqlens_k=self._prefill_metadata.query_start_loc, + max_seqlen_q=self._prefill_metadata.max_query_len, + max_seqlen_k=self._prefill_metadata.max_query_len, + softmax_scale=self.scale, + causal=True, + return_lse=return_softmax_lse, + out=out, + ) + + # aiter returns the bare output tensor when return_lse is False, and + # (out, softmax_lse) when it is True. + if return_softmax_lse: + return result[0], result[1] + return result + + def run_prefill_context_chunk( + self, + chunk_idx: int, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + assert self._prefill_metadata.chunked_context is not None + chunked = self._prefill_metadata.chunked_context + out, lse = self.flash_attn_varlen_func( + q=q, + k=k, + v=v, + cu_seqlens_q=self._prefill_metadata.query_start_loc, + cu_seqlens_k=chunked.cu_seq_lens[chunk_idx], + max_seqlen_q=self._prefill_metadata.max_query_len, + max_seqlen_k=chunked.max_seq_lens[chunk_idx], + softmax_scale=self.scale, + causal=False, + return_lse=True, + ) + return out, lse diff --git a/vllm/v1/attention/backends/mla/prefill/registry.py b/vllm/v1/attention/backends/mla/prefill/registry.py index 9c83ea1b13..0d818a084b 100644 --- a/vllm/v1/attention/backends/mla/prefill/registry.py +++ b/vllm/v1/attention/backends/mla/prefill/registry.py @@ -48,6 +48,10 @@ class MLAPrefillBackendEnum(Enum, metaclass=_MLAPrefillBackendEnumMeta): "vllm.v1.attention.backends.mla.prefill.tokenspeed_mla." "TokenspeedMLAPrefillBackend" ) + ROCM_AITER_FA = ( + "vllm.v1.attention.backends.mla.prefill.aiter_flash_attn." + "AiterFlashAttnPrefillBackend" + ) # Placeholder for third-party/custom backends - must be registered before use # set to None to avoid alias with other backend, whose value is an empty string CUSTOM = None diff --git a/vllm/v1/attention/backends/mla/prefill/selector.py b/vllm/v1/attention/backends/mla/prefill/selector.py index e100c098ac..a38b274dfc 100644 --- a/vllm/v1/attention/backends/mla/prefill/selector.py +++ b/vllm/v1/attention/backends/mla/prefill/selector.py @@ -56,6 +56,14 @@ def _get_mla_prefill_backend_priorities( Returns: List of backends in priority order (highest priority first). """ + from vllm.platforms import current_platform + + if current_platform.is_rocm(): + return [ + MLAPrefillBackendEnum.ROCM_AITER_FA, + MLAPrefillBackendEnum.FLASH_ATTN, + ] + if device_capability.major == 10: # Blackwell return [ MLAPrefillBackendEnum.FLASH_ATTN,