diff --git a/benchmarks/kernels/benchmark_vit_aiter_fp8_attn.py b/benchmarks/kernels/benchmark_vit_aiter_fp8_attn.py new file mode 100644 index 0000000000..6bb504b7f0 --- /dev/null +++ b/benchmarks/kernels/benchmark_vit_aiter_fp8_attn.py @@ -0,0 +1,124 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Benchmark complete AITER BF16 and FP8 ViT attention calls on ROCm. + +The FP8 timing includes Q/K/V quantization and attention. Dynamic scales are +calibrated once per input before timing so the measured path matches serving +with a static scale file. + +Example: + python benchmarks/kernels/benchmark_vit_aiter_fp8_attn.py \ + --seq-lens 2304 4096 8192 16384 --head-dim 72 +""" + +from types import SimpleNamespace + +import torch + +from vllm.config import VllmConfig, set_current_vllm_config +from vllm.config.multimodal import MultiModalConfig +from vllm.model_executor.layers.attention.mm_encoder_attention import ( + MMEncoderAttention, +) +from vllm.platforms import current_platform +from vllm.triton_utils import triton +from vllm.utils.argparse_utils import FlexibleArgumentParser +from vllm.v1.attention.backends.registry import AttentionBackendEnum + + +def make_attention(num_heads: int, head_dim: int, fp8: bool) -> MMEncoderAttention: + mm_config = MultiModalConfig( + mm_encoder_attn_backend=AttentionBackendEnum.ROCM_AITER_FA, + mm_encoder_attn_dtype="fp8" if fp8 else None, + ) + vllm_config = VllmConfig() + vllm_config.model_config = SimpleNamespace(multimodal_config=mm_config) + with set_current_vllm_config(vllm_config): + return MMEncoderAttention(num_heads, head_dim).to("cuda") + + +def bench( + seq_lens: list[int], + num_heads: int, + head_dim: int, + warmup_ms: int, + repeat_ms: int, +) -> None: + if not current_platform.is_rocm(): + raise RuntimeError("This benchmark requires ROCm and AITER.") + + old_dtype = torch.get_default_dtype() + torch.set_default_dtype(torch.bfloat16) + try: + fp8_attention = make_attention(num_heads, head_dim, fp8=True) + bf16_attention = make_attention(num_heads, head_dim, fp8=False) + finally: + torch.set_default_dtype(old_dtype) + + print( + f"{'seq_len':>8} {'BF16 ms':>12} {'FP8 ms':>12} " + f"{'speedup':>10} {'FP8/BF16':>12}" + ) + print("-" * 60) + for seq_len in seq_lens: + torch.manual_seed(0) + qkv = torch.randn( + 1, + seq_len, + 3, + num_heads, + head_dim, + device="cuda", + dtype=torch.bfloat16, + ) + q, k, v = qkv.unbind(dim=2) + cu_seqlens = torch.tensor([0, seq_len], device="cuda", dtype=torch.int32) + max_seqlen = torch.tensor(seq_len, device="cuda", dtype=torch.int32) + + # Calibrate per-tensor scales once, then benchmark the static-scale path. + fp8_attention._fp8_dynamic_scale = True + fp8_attention._forward_aiter_fp8(q, k, v, cu_seqlens, max_seqlen) + fp8_attention._fp8_dynamic_scale = False + + bf16_ms = triton.testing.do_bench( + lambda q=q, k=k, v=v, cu=cu_seqlens, ms=max_seqlen: ( + bf16_attention._forward_fa(q, k, v, cu, ms) + ), + warmup=warmup_ms, + rep=repeat_ms, + ) + fp8_ms = triton.testing.do_bench( + lambda q=q, k=k, v=v, cu=cu_seqlens, ms=max_seqlen: ( + fp8_attention._forward_aiter_fp8(q, k, v, cu, ms) + ), + warmup=warmup_ms, + rep=repeat_ms, + ) + speedup = bf16_ms / fp8_ms + ratio = fp8_ms / bf16_ms + print( + f"{seq_len:>8} {bf16_ms:>12.3f} {fp8_ms:>12.3f} " + f"{speedup:>9.2f}x {ratio:>12.3f}" + ) + + +if __name__ == "__main__": + parser = FlexibleArgumentParser( + description="Benchmark AITER BF16 vs FP8 ViT attention." + ) + parser.add_argument( + "--seq-lens", type=int, nargs="+", default=[2304, 4096, 8192, 16384] + ) + parser.add_argument("--num-heads", type=int, default=16) + parser.add_argument("--head-dim", type=int, default=72) + parser.add_argument("--warmup-ms", type=int, default=100) + parser.add_argument("--repeat-ms", type=int, default=500) + args = parser.parse_args() + bench( + args.seq_lens, + args.num_heads, + args.head_dim, + args.warmup_ms, + args.repeat_ms, + ) diff --git a/docs/features/quantization/fp8_vit_attn.md b/docs/features/quantization/fp8_vit_attn.md index bf628cd8a7..18f42caa90 100644 --- a/docs/features/quantization/fp8_vit_attn.md +++ b/docs/features/quantization/fp8_vit_attn.md @@ -3,9 +3,9 @@ For visual understanding workloads with large images (e.g. QHD, 4K) and relatively short text prompts/generation, the ViT encoder attention can become a significant bottleneck, especially when the text model is quantized (e.g. NVFP4). vLLM -supports optional FP8 quantization for the ViT encoder attention via the -FlashInfer cuDNN backend. Q/K/V are quantized on-the-fly to FP8 before the -cuDNN attention call. +supports optional FP8 quantization for ViT encoder attention via FlashInfer +cuDNN on NVIDIA GPUs and AITER on AMD GPUs. Q/K/V are quantized on-the-fly to +FP8 before the attention call. !!! note - Currently supports Qwen3-VL family models only (`qwen3_vl`, `qwen3_vl_moe`, @@ -15,20 +15,29 @@ cuDNN attention call. requests. Smaller images may see no speedup due to quantization overhead (3 quantization kernel launches + un-padding). - FP8 tensor-core speedup is more pronounced on GB300 than GB200. + - On ROCm, packed variable-length image and video batches are supported by + AITER's native varlen FP8 attention. ## Requirements -- FlashInfer cuDNN backend with cuDNN >= 9.17.1. +- NVIDIA: FlashInfer cuDNN backend with cuDNN >= 9.17.1. +- AMD: AITER with `flash_attn_varlen_fp8_pertensor_func` support on gfx942 + (MI300 series) or gfx950 (MI350 series). ## Usage -Enable FP8 ViT attention by passing `--mm-encoder-attn-dtype fp8` together -with `--mm-encoder-attn-backend FLASHINFER`: +Enable FP8 ViT attention by passing `--mm-encoder-attn-dtype fp8` and selecting +the backend for the current platform: ```bash vllm serve $MODEL \ --mm-encoder-attn-backend FLASHINFER \ --mm-encoder-attn-dtype fp8 + +# AMD ROCm +vllm serve $MODEL \ + --mm-encoder-attn-backend ROCM_AITER_FA \ + --mm-encoder-attn-dtype fp8 ``` By default (no scale file), **dynamic scaling** is used: a 16-entry circular @@ -45,7 +54,7 @@ reuse them to avoid the dynamic overhead: # Step 1: calibrate and save scales (runs dynamic scaling for 16 passes, # then dumps the learned scales to JSON). vllm bench mm-processor \ - --model $MODEL --mm-encoder-attn-backend FLASHINFER \ + --model $MODEL --mm-encoder-attn-backend $MM_ATTN_BACKEND \ --mm-encoder-attn-dtype fp8 \ --mm-encoder-fp8-scale-save-path /path/to/scales.json \ --dataset-name hf --dataset-path lmarena-ai/VisionArena-Chat \ @@ -53,7 +62,7 @@ vllm bench mm-processor \ # Step 2: serve with static scales (no dynamic overhead). vllm serve $MODEL \ - --mm-encoder-attn-backend FLASHINFER \ + --mm-encoder-attn-backend $MM_ATTN_BACKEND \ --mm-encoder-attn-dtype fp8 \ --mm-encoder-fp8-scale-path /path/to/scales.json ``` @@ -94,6 +103,16 @@ Keys `q_scale` / `k_scale` / `v_scale` are accepted as aliases. Crossover is around FullHD with 3 images/request. At QHD and above, FP8 wins. +**Complete AITER attention call on MI300X** (BF16 input, 16 heads, +head_dim=72; FP8 includes Q/K/V quantization): + +| Sequence length | AITER BF16 | AITER FP8 | Speedup | +| --------------- | ---------- | --------- | ------- | +| 2304 | 0.467 ms | 0.337 ms | **1.38x** | +| 4096 | 0.812 ms | 0.764 ms | **1.06x** | +| 8192 | 2.555 ms | 2.364 ms | **1.08x** | +| 16384 | 9.769 ms | 8.655 ms | **1.13x** | + ## Accuracy ChartQA, Qwen3-VL-8B-Instruct, 500 samples. FP8 static uses scales calibrated diff --git a/tests/kernels/attention/test_mha_attn.py b/tests/kernels/attention/test_mha_attn.py index 858d9504a1..d73acfc0ee 100644 --- a/tests/kernels/attention/test_mha_attn.py +++ b/tests/kernels/attention/test_mha_attn.py @@ -20,7 +20,7 @@ from vllm.platforms import current_platform from vllm.platforms.cpu import CpuPlatform from vllm.platforms.cuda import CudaPlatform from vllm.platforms.interface import DeviceCapability -from vllm.platforms.rocm import RocmPlatform +from vllm.platforms.rocm import RocmPlatform, on_mi3xx from vllm.utils.torch_utils import set_default_torch_dtype, set_random_seed from vllm.v1.attention.backends.registry import AttentionBackendEnum from vllm.v1.attention.selector import _cached_get_attn_backend @@ -346,3 +346,137 @@ def test_mha_attn_varlen_forward_flashinfer( torch.testing.assert_close(output, ref_output, atol=1e-2, rtol=1e-2) finally: vllm_config.model_config = old_model_config + + +@pytest.mark.skipif( + not current_platform.is_rocm() or not on_mi3xx(), + reason="AITER FP8 attention requires MI300/MI350", +) +@pytest.mark.parametrize("var_seq_len", [[128, 193], [256, 384, 511]]) +@pytest.mark.parametrize("head_size", [64, 72, 80]) +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half]) +def test_mha_attn_varlen_forward_aiter_fp8( + default_vllm_config, + var_seq_len: list[int], + head_size: int, + dtype: torch.dtype, +): + """Compare packed AITER FP8 ViT attention with a BF16/FP16 reference.""" + aiter = pytest.importorskip("aiter") + if not hasattr(aiter, "flash_attn_varlen_fp8_pertensor_func"): + pytest.skip("Installed AITER does not provide varlen FP8 attention") + + num_heads = 4 + set_random_seed(0) + torch.set_default_device("cuda") + torch.set_default_dtype(dtype) + + vllm_config = get_current_vllm_config() + old_model_config = getattr(vllm_config, "model_config", None) + minimal_model_config = type( + "MinimalModelConfig", + (), + { + "multimodal_config": MultiModalConfig( + mm_encoder_attn_backend=AttentionBackendEnum.ROCM_AITER_FA, + mm_encoder_attn_dtype="fp8", + ), + }, + )() + vllm_config.model_config = minimal_model_config + try: + total_len = sum(var_seq_len) + # Keep Q/K/V as non-contiguous views, matching interleaved ViT QKV. + qkv = torch.randn(1, total_len, 3, num_heads, head_size) + q, k, v = qkv.unbind(dim=2) + cu_seqlens = torch.tensor( + [0] + list(itertools.accumulate(var_seq_len)), dtype=torch.int32 + ) + max_seqlen = torch.tensor(max(var_seq_len), dtype=torch.int32) + scale = 1.0 / head_size**0.5 + + attn = MMEncoderAttention(num_heads, head_size, scale=scale) + assert attn.attn_backend == AttentionBackendEnum.ROCM_AITER_FA + assert attn.fp8_enabled + output = attn( + q, + k, + v, + cu_seqlens=cu_seqlens, + max_seqlen=max_seqlen, + ) + + ref_output = torch.cat( + [ + ref_attention(q_i, k_i, v_i, scale=scale) + for q_i, k_i, v_i in zip( + torch.split(q, var_seq_len, dim=1), + torch.split(k, var_seq_len, dim=1), + torch.split(v, var_seq_len, dim=1), + ) + ], + dim=1, + ) + diff = (output.float() - ref_output.float()).abs() + cosine = torch.nn.functional.cosine_similarity( + output.float().flatten(), + ref_output.float().flatten(), + dim=0, + ) + + assert output.shape == ref_output.shape + assert output.dtype == dtype + assert cosine.item() >= 0.99 + assert diff.mean().item() <= 0.03 + assert diff.max().item() <= 0.30 + finally: + vllm_config.model_config = old_model_config + + +@pytest.mark.skipif( + not current_platform.is_rocm(), reason="AITER FP8 attention requires ROCm" +) +def test_mha_attn_aiter_fp8_rejects_unsupported_arch(default_vllm_config): + vllm_config = get_current_vllm_config() + old_model_config = getattr(vllm_config, "model_config", None) + vllm_config.model_config = type( + "MinimalModelConfig", + (), + { + "multimodal_config": MultiModalConfig( + mm_encoder_attn_backend=AttentionBackendEnum.ROCM_AITER_FA, + mm_encoder_attn_dtype="fp8", + ), + }, + )() + try: + with ( + patch("vllm.platforms.rocm.on_mi3xx", return_value=False), + pytest.raises(ValueError, match="gfx942 or gfx950"), + ): + MMEncoderAttention(4, 64) + finally: + vllm_config.model_config = old_model_config + + +@pytest.mark.skipif( + not current_platform.is_rocm(), reason="AITER FP8 attention requires ROCm" +) +def test_mha_attn_fp8_rejects_wrong_backend(default_vllm_config): + vllm_config = get_current_vllm_config() + old_model_config = getattr(vllm_config, "model_config", None) + vllm_config.model_config = type( + "MinimalModelConfig", + (), + { + "multimodal_config": MultiModalConfig( + mm_encoder_attn_backend=AttentionBackendEnum.FLASH_ATTN, + mm_encoder_attn_dtype="fp8", + ), + }, + )() + try: + with pytest.raises(ValueError, match="requires either"): + MMEncoderAttention(4, 64) + finally: + vllm_config.model_config = old_model_config diff --git a/tests/kernels/core/test_vit_fp8_scaling.py b/tests/kernels/core/test_vit_fp8_scaling.py index f3b2dbfa56..bd0685af70 100644 --- a/tests/kernels/core/test_vit_fp8_scaling.py +++ b/tests/kernels/core/test_vit_fp8_scaling.py @@ -14,6 +14,7 @@ from vllm.model_executor.layers.attention.mm_encoder_attention import ( _FP8_AMAX_HISTORY_LEN, _FP8_MAX, ) +from vllm.platforms import current_platform from vllm.utils.flashinfer import ( is_flashinfer_cudnn_fp8_prefill_attn_supported, ) @@ -29,8 +30,17 @@ NUM_HEADS = 16 HEAD_DIM = 72 +def _is_mi3xx() -> bool: + if not current_platform.is_rocm(): + return False + + from vllm.platforms.rocm import on_mi3xx + + return on_mi3xx() + + @contextlib.contextmanager -def _build_attention(mm_config): +def _build_attention(mm_config, attn_backend=None): """Yield an MMEncoderAttention with the given multimodal config. The VllmConfig context stays active while the test runs so that @@ -44,7 +54,13 @@ def _build_attention(mm_config): ) from vllm.v1.attention.backends.registry import AttentionBackendEnum - if not is_flashinfer_cudnn_fp8_prefill_attn_supported(): + if attn_backend is None: + attn_backend = AttentionBackendEnum.FLASHINFER + + if ( + attn_backend == AttentionBackendEnum.FLASHINFER + and not is_flashinfer_cudnn_fp8_prefill_attn_supported() + ): yield None return @@ -56,7 +72,7 @@ def _build_attention(mm_config): patch( "vllm.model_executor.layers.attention.mm_encoder_attention" ".get_vit_attn_backend", - return_value=AttentionBackendEnum.FLASHINFER, + return_value=attn_backend, ), ): attn = MMEncoderAttention( @@ -169,6 +185,31 @@ def test_static_scales_loaded(_make_static_attention) -> None: assert not hasattr(attn, "_fp8_q_amax") +@pytest.mark.skipif( + not _is_mi3xx(), + reason="AITER FP8 attention requires MI300/MI350", +) +def test_aiter_static_scales_loaded(tmp_path) -> None: + """Verify AITER reuses the existing static FP8 scale loading path.""" + from vllm.config.multimodal import MultiModalConfig + from vllm.v1.attention.backends.registry import AttentionBackendEnum + + scale_file = tmp_path / "aiter_scales.json" + scale_file.write_text(json.dumps({LAYER_0: {"q": 224.0, "k": 198.0, "v": 210.0}})) + mm_config = MultiModalConfig( + mm_encoder_attn_dtype="fp8", + mm_encoder_fp8_scale_path=str(scale_file), + ) + + with _build_attention(mm_config, AttentionBackendEnum.ROCM_AITER_FA) as attn: + assert attn is not None + assert attn.fp8_enabled + assert not attn._fp8_dynamic_scale + assert attn._fp8_q_scale.item() == 224.0 + assert attn._fp8_k_scale.item() == 198.0 + assert attn._fp8_v_scale.item() == 210.0 + + def test_static_scales_missing_layer(tmp_path) -> None: """Verify error when requested layer is not in the scale file.""" from vllm.config import VllmConfig, set_current_vllm_config diff --git a/vllm/_aiter_ops.py b/vllm/_aiter_ops.py index e4b08e90ac..9ab6c81dce 100644 --- a/vllm/_aiter_ops.py +++ b/vllm/_aiter_ops.py @@ -1404,6 +1404,65 @@ def _triton_rotary_embedding_fake( return +def _rocm_aiter_fp8_attn_impl( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + q_descale: torch.Tensor, + k_descale: torch.Tensor, + v_descale: torch.Tensor, + batch_size: int, + output_dtype: torch.dtype, + scale: float | None = None, + cu_seqlens: torch.Tensor | None = None, + max_seqlen: torch.Tensor | None = None, +) -> torch.Tensor: + """Run AITER FP8 attention for fixed or packed inputs.""" + from aiter import flash_attn_varlen_fp8_pertensor_func + + q_len = q.size(1) + if cu_seqlens is None: + cu_seqlens = torch.arange( + 0, (batch_size + 1) * q_len, step=q_len, dtype=torch.int32, device=q.device + ) + max_seqlen_value = q_len if max_seqlen is None else max_seqlen.item() + + q, k, v = (x.flatten(0, 1) for x in (q, k, v)) + output = flash_attn_varlen_fp8_pertensor_func( + q, + k, + v, + q_descale=q_descale, + k_descale=k_descale, + v_descale=v_descale, + cu_seqlens_q=cu_seqlens, + cu_seqlens_k=cu_seqlens, + max_seqlen_q=max_seqlen_value, + max_seqlen_k=max_seqlen_value, + causal=False, + softmax_scale=scale, + ) + return output.to(output_dtype).reshape(batch_size, q_len, *output.shape[1:]) + + +def _rocm_aiter_fp8_attn_fake( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + q_descale: torch.Tensor, + k_descale: torch.Tensor, + v_descale: torch.Tensor, + batch_size: int, + output_dtype: torch.dtype, + scale: float | None = None, + cu_seqlens: torch.Tensor | None = None, + max_seqlen: torch.Tensor | None = None, +) -> torch.Tensor: + return torch.empty( + (*q.shape[:-1], v.shape[-1]), device=q.device, dtype=output_dtype + ) + + # Global flag to ensure ops are registered only once _OPS_REGISTERED = False @@ -1959,6 +2018,14 @@ class rocm_aiter_ops: dispatch_key=current_platform.dispatch_key, ) + direct_register_custom_op( + op_name="aiter_fp8_attn_wrapper", + op_func=_rocm_aiter_fp8_attn_impl, + mutates_args=[], + fake_impl=_rocm_aiter_fp8_attn_fake, + dispatch_key=current_platform.dispatch_key, + ) + direct_register_custom_op( op_name="rocm_aiter_gemm_a8wfp4", op_func=_rocm_aiter_gemm_a8wfp4_impl, @@ -2830,6 +2897,34 @@ class rocm_aiter_ops: sink_ptr=sink_ptr, ) + @staticmethod + def fp8_attn_wrapper( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + q_descale: torch.Tensor, + k_descale: torch.Tensor, + v_descale: torch.Tensor, + batch_size: int, + output_dtype: torch.dtype, + scale: float | None = None, + cu_seqlens: torch.Tensor | None = None, + max_seqlen: torch.Tensor | None = None, + ) -> torch.Tensor: + return torch.ops.vllm.aiter_fp8_attn_wrapper( + q, + k, + v, + q_descale, + k_descale, + v_descale, + batch_size, + output_dtype, + scale, + cu_seqlens, + max_seqlen, + ) + @staticmethod def pa_fwd_asm( Q: torch.Tensor, diff --git a/vllm/model_executor/layers/attention/mm_encoder_attention.py b/vllm/model_executor/layers/attention/mm_encoder_attention.py index bb1c995aeb..85dbd6331b 100644 --- a/vllm/model_executor/layers/attention/mm_encoder_attention.py +++ b/vllm/model_executor/layers/attention/mm_encoder_attention.py @@ -24,6 +24,7 @@ from vllm.model_executor.models.vision import ( get_multimodal_config, get_vit_attn_backend, ) +from vllm.platforms import current_platform from vllm.utils.flashinfer import ( is_flashinfer_cudnn_fp8_prefill_attn_supported, ) @@ -32,6 +33,7 @@ from vllm.utils.torch_utils import async_tensor_h2d from vllm.v1.attention.backends.fa_utils import get_flash_attn_version from vllm.v1.attention.backends.registry import AttentionBackendEnum from vllm.v1.attention.ops.vit_attn_wrappers import ( + vit_aiter_fp8_attn_wrapper, vit_flash_attn_wrapper, vit_flashinfer_wrapper, vit_torch_sdpa_wrapper, @@ -380,8 +382,8 @@ class MMEncoderAttention(CustomOp): No-op if FP8 is not requested. Raises ``ValueError`` if FP8 is requested but the platform does not support it. """ - # Populate defaults so ``_forward_flashinfer`` can - # check ``self.fp8_enabled`` and others without AttributeError. + # Populate defaults so backend forward methods can check + # ``self.fp8_enabled`` and related state without AttributeError. self.fp8_enabled = False self._fp8_dynamic_scale = False self.fp8_quant: QuantFP8 | None = None @@ -393,13 +395,38 @@ class MMEncoderAttention(CustomOp): if mm_cfg is None or mm_cfg.mm_encoder_attn_dtype != "fp8": return - # FP8 path - if not is_flashinfer_cudnn_fp8_prefill_attn_supported(): + if self.attn_backend == AttentionBackendEnum.ROCM_AITER_FA: + if not current_platform.is_rocm(): + raise ValueError("AITER FP8 ViT attention requires ROCm.") + + from vllm.platforms.rocm import on_mi3xx + + if not on_mi3xx(): + raise ValueError( + "AITER FP8 ViT attention requires an MI300-series or " + "MI350-series GPU (gfx942 or gfx950)." + ) + try: + from aiter import flash_attn_varlen_fp8_pertensor_func # noqa: F401 + except ImportError as exc: + raise ValueError( + "mm_encoder_attn_dtype='fp8' with ROCM_AITER_FA requires " + "an AITER build that provides " + "flash_attn_varlen_fp8_pertensor_func." + ) from exc + elif self.attn_backend == AttentionBackendEnum.FLASHINFER: + if not is_flashinfer_cudnn_fp8_prefill_attn_supported(): + raise ValueError( + "mm_encoder_attn_dtype='fp8' requires the FlashInfer " + "cuDNN backend with cuDNN >= 9.17.1 on Blackwell (SM 100) " + "or newer. cuDNN's FP8 SDPA path with bf16/fp16 output is " + "not available on Hopper (H100/H200) or earlier." + ) + else: raise ValueError( - "mm_encoder_attn_dtype='fp8' requires the FlashInfer " - "cuDNN backend with cuDNN >= 9.17.1 on Blackwell (SM 100) " - "or newer. cuDNN's FP8 SDPA path with bf16/fp16 output is " - "not available on Hopper (H100/H200) or earlier." + "mm_encoder_attn_dtype='fp8' requires either the " + "ROCM_AITER_FA backend on ROCm or the FlashInfer cuDNN " + f"backend on CUDA, got {self.attn_backend}." ) self.fp8_enabled = True @@ -642,6 +669,37 @@ class MMEncoderAttention(CustomOp): buffer_wrapped, ) + def _quantize_qkv_fp8( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + assert self.fp8_quant is not None + + if self._fp8_dynamic_scale: + self._record_amax_and_update_scales(query, key, value) + + query = quantize_fp8_maybe_pad_head_dim( + query, + self._fp8_q_scale, + skip_scale=self.skip_scale_q, + fp8_quant=self.fp8_quant, + ) + key = quantize_fp8_maybe_pad_head_dim( + key, + self._fp8_k_scale, + skip_scale=self.skip_scale_k, + fp8_quant=self.fp8_quant, + ) + value = quantize_fp8_maybe_pad_head_dim( + value, + self._fp8_v_scale, + skip_scale=self.skip_scale_v, + fp8_quant=self.fp8_quant, + ) + return query, key, value + def _forward_flashinfer( self, query: torch.Tensor, @@ -653,29 +711,7 @@ class MMEncoderAttention(CustomOp): | None = None, # Only used for FlashInfer CuDNN backend ) -> torch.Tensor: if self.fp8_enabled: - assert self.fp8_quant is not None - - if self._fp8_dynamic_scale: - self._record_amax_and_update_scales(query, key, value) - - query = quantize_fp8_maybe_pad_head_dim( - query, - self._fp8_q_scale, - skip_scale=self.skip_scale_q, - fp8_quant=self.fp8_quant, - ) - key = quantize_fp8_maybe_pad_head_dim( - key, - self._fp8_k_scale, - skip_scale=self.skip_scale_k, - fp8_quant=self.fp8_quant, - ) - value = quantize_fp8_maybe_pad_head_dim( - value, - self._fp8_v_scale, - skip_scale=self.skip_scale_v, - fp8_quant=self.fp8_quant, - ) + query, key, value = self._quantize_qkv_fp8(query, key, value) output = vit_flashinfer_wrapper( q=query, @@ -697,6 +733,44 @@ class MMEncoderAttention(CustomOp): return output + def _forward_aiter_fp8( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + cu_seqlens: torch.Tensor | None = None, + max_seqlen: torch.Tensor | None = None, + ) -> torch.Tensor: + assert (cu_seqlens is not None and max_seqlen is not None) or ( + cu_seqlens is None and max_seqlen is None + ), "cu_seqlens and max_seqlen should be both set or both None." + + bsz, q_len = query.size()[:2] + kv_len = key.size(1) + is_reshaped = query.dim() != 4 + query, key, value = self.view_qkv_to_4d(query, key, value, bsz, q_len, kv_len) + + query, key, value = self._quantize_qkv_fp8(query, key, value) + + output = vit_aiter_fp8_attn_wrapper( + q=query, + k=key, + v=value, + q_descale=self._fp8_q_scale, + k_descale=self._fp8_k_scale, + v_descale=self._fp8_v_scale, + batch_size=bsz, + output_dtype=self.dtype, + scale=self.scale, + cu_seqlens=cu_seqlens, + max_seqlen=max_seqlen, + ) + if output.shape[-1] != self.head_size: + output = output[..., : self.head_size].contiguous() + if is_reshaped: + output = output.reshape(bsz, q_len, -1) + return output + def forward_native( self, query: torch.Tensor, @@ -719,7 +793,9 @@ class MMEncoderAttention(CustomOp): sequence_lengths: torch.Tensor | None = None, # Only used for FlashInfer CuDNN backend ) -> torch.Tensor: - if self.is_flash_attn_backend: + if self.fp8_enabled and self.attn_backend == AttentionBackendEnum.ROCM_AITER_FA: + return self._forward_aiter_fp8(query, key, value, cu_seqlens, max_seqlen) + elif self.is_flash_attn_backend: return self._forward_fa(query, key, value, cu_seqlens, max_seqlen) elif self.attn_backend == AttentionBackendEnum.TRITON_ATTN: return self._forward_triton(query, key, value, cu_seqlens, max_seqlen) diff --git a/vllm/platforms/rocm.py b/vllm/platforms/rocm.py index 4b182aeb88..a5ca055f74 100644 --- a/vllm/platforms/rocm.py +++ b/vllm/platforms/rocm.py @@ -657,7 +657,7 @@ class RocmPlatform(Platform): from vllm._aiter_ops import rocm_aiter_ops - if rocm_aiter_ops.is_enabled() and on_gfx9(): + if rocm_aiter_ops.is_mha_enabled() and on_gfx9(): logger.info_once("Using AITER Flash Attention backend for ViT model.") return AttentionBackendEnum.ROCM_AITER_FA diff --git a/vllm/v1/attention/ops/vit_attn_wrappers.py b/vllm/v1/attention/ops/vit_attn_wrappers.py index 5bbcc3386e..86a0b6ee9d 100644 --- a/vllm/v1/attention/ops/vit_attn_wrappers.py +++ b/vllm/v1/attention/ops/vit_attn_wrappers.py @@ -18,6 +18,7 @@ import einops import torch import torch.nn.functional as F +from vllm._aiter_ops import rocm_aiter_ops from vllm.platforms import current_platform from vllm.utils.torch_utils import direct_register_custom_op @@ -114,6 +115,34 @@ def vit_flash_attn_wrapper( ) +def vit_aiter_fp8_attn_wrapper( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + q_descale: torch.Tensor, + k_descale: torch.Tensor, + v_descale: torch.Tensor, + batch_size: int, + output_dtype: torch.dtype, + scale: float | None = None, + cu_seqlens: torch.Tensor | None = None, + max_seqlen: torch.Tensor | None = None, +) -> torch.Tensor: + return rocm_aiter_ops.fp8_attn_wrapper( + q, + k, + v, + q_descale, + k_descale, + v_descale, + batch_size, + output_dtype, + scale, + cu_seqlens, + max_seqlen, + ) + + def triton_attn_wrapper( q: torch.Tensor, k: torch.Tensor,