[ROCm] Add AITER FP8 ViT encoder attention (#49937)

Signed-off-by: Liuyinfeng01 <yinfeliu@amd.com>
This commit is contained in:
yinfengLiu
2026-07-30 06:22:10 +00:00
committed by GitHub
parent 0028fc8d8d
commit f1e8fd27ae
8 changed files with 563 additions and 45 deletions
@@ -0,0 +1,124 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Benchmark complete AITER BF16 and FP8 ViT attention calls on ROCm.
The FP8 timing includes Q/K/V quantization and attention. Dynamic scales are
calibrated once per input before timing so the measured path matches serving
with a static scale file.
Example:
python benchmarks/kernels/benchmark_vit_aiter_fp8_attn.py \
--seq-lens 2304 4096 8192 16384 --head-dim 72
"""
from types import SimpleNamespace
import torch
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.config.multimodal import MultiModalConfig
from vllm.model_executor.layers.attention.mm_encoder_attention import (
MMEncoderAttention,
)
from vllm.platforms import current_platform
from vllm.triton_utils import triton
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.v1.attention.backends.registry import AttentionBackendEnum
def make_attention(num_heads: int, head_dim: int, fp8: bool) -> MMEncoderAttention:
mm_config = MultiModalConfig(
mm_encoder_attn_backend=AttentionBackendEnum.ROCM_AITER_FA,
mm_encoder_attn_dtype="fp8" if fp8 else None,
)
vllm_config = VllmConfig()
vllm_config.model_config = SimpleNamespace(multimodal_config=mm_config)
with set_current_vllm_config(vllm_config):
return MMEncoderAttention(num_heads, head_dim).to("cuda")
def bench(
seq_lens: list[int],
num_heads: int,
head_dim: int,
warmup_ms: int,
repeat_ms: int,
) -> None:
if not current_platform.is_rocm():
raise RuntimeError("This benchmark requires ROCm and AITER.")
old_dtype = torch.get_default_dtype()
torch.set_default_dtype(torch.bfloat16)
try:
fp8_attention = make_attention(num_heads, head_dim, fp8=True)
bf16_attention = make_attention(num_heads, head_dim, fp8=False)
finally:
torch.set_default_dtype(old_dtype)
print(
f"{'seq_len':>8} {'BF16 ms':>12} {'FP8 ms':>12} "
f"{'speedup':>10} {'FP8/BF16':>12}"
)
print("-" * 60)
for seq_len in seq_lens:
torch.manual_seed(0)
qkv = torch.randn(
1,
seq_len,
3,
num_heads,
head_dim,
device="cuda",
dtype=torch.bfloat16,
)
q, k, v = qkv.unbind(dim=2)
cu_seqlens = torch.tensor([0, seq_len], device="cuda", dtype=torch.int32)
max_seqlen = torch.tensor(seq_len, device="cuda", dtype=torch.int32)
# Calibrate per-tensor scales once, then benchmark the static-scale path.
fp8_attention._fp8_dynamic_scale = True
fp8_attention._forward_aiter_fp8(q, k, v, cu_seqlens, max_seqlen)
fp8_attention._fp8_dynamic_scale = False
bf16_ms = triton.testing.do_bench(
lambda q=q, k=k, v=v, cu=cu_seqlens, ms=max_seqlen: (
bf16_attention._forward_fa(q, k, v, cu, ms)
),
warmup=warmup_ms,
rep=repeat_ms,
)
fp8_ms = triton.testing.do_bench(
lambda q=q, k=k, v=v, cu=cu_seqlens, ms=max_seqlen: (
fp8_attention._forward_aiter_fp8(q, k, v, cu, ms)
),
warmup=warmup_ms,
rep=repeat_ms,
)
speedup = bf16_ms / fp8_ms
ratio = fp8_ms / bf16_ms
print(
f"{seq_len:>8} {bf16_ms:>12.3f} {fp8_ms:>12.3f} "
f"{speedup:>9.2f}x {ratio:>12.3f}"
)
if __name__ == "__main__":
parser = FlexibleArgumentParser(
description="Benchmark AITER BF16 vs FP8 ViT attention."
)
parser.add_argument(
"--seq-lens", type=int, nargs="+", default=[2304, 4096, 8192, 16384]
)
parser.add_argument("--num-heads", type=int, default=16)
parser.add_argument("--head-dim", type=int, default=72)
parser.add_argument("--warmup-ms", type=int, default=100)
parser.add_argument("--repeat-ms", type=int, default=500)
args = parser.parse_args()
bench(
args.seq_lens,
args.num_heads,
args.head_dim,
args.warmup_ms,
args.repeat_ms,
)
+27 -8
View File
@@ -3,9 +3,9 @@
For visual understanding workloads with large images (e.g. QHD, 4K) and relatively
short text prompts/generation, the ViT encoder attention can become a significant
bottleneck, especially when the text model is quantized (e.g. NVFP4). vLLM
supports optional FP8 quantization for the ViT encoder attention via the
FlashInfer cuDNN backend. Q/K/V are quantized on-the-fly to FP8 before the
cuDNN attention call.
supports optional FP8 quantization for ViT encoder attention via FlashInfer
cuDNN on NVIDIA GPUs and AITER on AMD GPUs. Q/K/V are quantized on-the-fly to
FP8 before the attention call.
!!! note
- Currently supports Qwen3-VL family models only (`qwen3_vl`, `qwen3_vl_moe`,
@@ -15,20 +15,29 @@ cuDNN attention call.
requests. Smaller images may see no speedup due to quantization overhead
(3 quantization kernel launches + un-padding).
- FP8 tensor-core speedup is more pronounced on GB300 than GB200.
- On ROCm, packed variable-length image and video batches are supported by
AITER's native varlen FP8 attention.
## Requirements
- FlashInfer cuDNN backend with cuDNN >= 9.17.1.
- NVIDIA: FlashInfer cuDNN backend with cuDNN >= 9.17.1.
- AMD: AITER with `flash_attn_varlen_fp8_pertensor_func` support on gfx942
(MI300 series) or gfx950 (MI350 series).
## Usage
Enable FP8 ViT attention by passing `--mm-encoder-attn-dtype fp8` together
with `--mm-encoder-attn-backend FLASHINFER`:
Enable FP8 ViT attention by passing `--mm-encoder-attn-dtype fp8` and selecting
the backend for the current platform:
```bash
vllm serve $MODEL \
--mm-encoder-attn-backend FLASHINFER \
--mm-encoder-attn-dtype fp8
# AMD ROCm
vllm serve $MODEL \
--mm-encoder-attn-backend ROCM_AITER_FA \
--mm-encoder-attn-dtype fp8
```
By default (no scale file), **dynamic scaling** is used: a 16-entry circular
@@ -45,7 +54,7 @@ reuse them to avoid the dynamic overhead:
# Step 1: calibrate and save scales (runs dynamic scaling for 16 passes,
# then dumps the learned scales to JSON).
vllm bench mm-processor \
--model $MODEL --mm-encoder-attn-backend FLASHINFER \
--model $MODEL --mm-encoder-attn-backend $MM_ATTN_BACKEND \
--mm-encoder-attn-dtype fp8 \
--mm-encoder-fp8-scale-save-path /path/to/scales.json \
--dataset-name hf --dataset-path lmarena-ai/VisionArena-Chat \
@@ -53,7 +62,7 @@ vllm bench mm-processor \
# Step 2: serve with static scales (no dynamic overhead).
vllm serve $MODEL \
--mm-encoder-attn-backend FLASHINFER \
--mm-encoder-attn-backend $MM_ATTN_BACKEND \
--mm-encoder-attn-dtype fp8 \
--mm-encoder-fp8-scale-path /path/to/scales.json
```
@@ -94,6 +103,16 @@ Keys `q_scale` / `k_scale` / `v_scale` are accepted as aliases.
Crossover is around FullHD with 3 images/request. At QHD and above, FP8 wins.
**Complete AITER attention call on MI300X** (BF16 input, 16 heads,
head_dim=72; FP8 includes Q/K/V quantization):
| Sequence length | AITER BF16 | AITER FP8 | Speedup |
| --------------- | ---------- | --------- | ------- |
| 2304 | 0.467 ms | 0.337 ms | **1.38x** |
| 4096 | 0.812 ms | 0.764 ms | **1.06x** |
| 8192 | 2.555 ms | 2.364 ms | **1.08x** |
| 16384 | 9.769 ms | 8.655 ms | **1.13x** |
## Accuracy
ChartQA, Qwen3-VL-8B-Instruct, 500 samples. FP8 static uses scales calibrated
+135 -1
View File
@@ -20,7 +20,7 @@ from vllm.platforms import current_platform
from vllm.platforms.cpu import CpuPlatform
from vllm.platforms.cuda import CudaPlatform
from vllm.platforms.interface import DeviceCapability
from vllm.platforms.rocm import RocmPlatform
from vllm.platforms.rocm import RocmPlatform, on_mi3xx
from vllm.utils.torch_utils import set_default_torch_dtype, set_random_seed
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.attention.selector import _cached_get_attn_backend
@@ -346,3 +346,137 @@ def test_mha_attn_varlen_forward_flashinfer(
torch.testing.assert_close(output, ref_output, atol=1e-2, rtol=1e-2)
finally:
vllm_config.model_config = old_model_config
@pytest.mark.skipif(
not current_platform.is_rocm() or not on_mi3xx(),
reason="AITER FP8 attention requires MI300/MI350",
)
@pytest.mark.parametrize("var_seq_len", [[128, 193], [256, 384, 511]])
@pytest.mark.parametrize("head_size", [64, 72, 80])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half])
def test_mha_attn_varlen_forward_aiter_fp8(
default_vllm_config,
var_seq_len: list[int],
head_size: int,
dtype: torch.dtype,
):
"""Compare packed AITER FP8 ViT attention with a BF16/FP16 reference."""
aiter = pytest.importorskip("aiter")
if not hasattr(aiter, "flash_attn_varlen_fp8_pertensor_func"):
pytest.skip("Installed AITER does not provide varlen FP8 attention")
num_heads = 4
set_random_seed(0)
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
vllm_config = get_current_vllm_config()
old_model_config = getattr(vllm_config, "model_config", None)
minimal_model_config = type(
"MinimalModelConfig",
(),
{
"multimodal_config": MultiModalConfig(
mm_encoder_attn_backend=AttentionBackendEnum.ROCM_AITER_FA,
mm_encoder_attn_dtype="fp8",
),
},
)()
vllm_config.model_config = minimal_model_config
try:
total_len = sum(var_seq_len)
# Keep Q/K/V as non-contiguous views, matching interleaved ViT QKV.
qkv = torch.randn(1, total_len, 3, num_heads, head_size)
q, k, v = qkv.unbind(dim=2)
cu_seqlens = torch.tensor(
[0] + list(itertools.accumulate(var_seq_len)), dtype=torch.int32
)
max_seqlen = torch.tensor(max(var_seq_len), dtype=torch.int32)
scale = 1.0 / head_size**0.5
attn = MMEncoderAttention(num_heads, head_size, scale=scale)
assert attn.attn_backend == AttentionBackendEnum.ROCM_AITER_FA
assert attn.fp8_enabled
output = attn(
q,
k,
v,
cu_seqlens=cu_seqlens,
max_seqlen=max_seqlen,
)
ref_output = torch.cat(
[
ref_attention(q_i, k_i, v_i, scale=scale)
for q_i, k_i, v_i in zip(
torch.split(q, var_seq_len, dim=1),
torch.split(k, var_seq_len, dim=1),
torch.split(v, var_seq_len, dim=1),
)
],
dim=1,
)
diff = (output.float() - ref_output.float()).abs()
cosine = torch.nn.functional.cosine_similarity(
output.float().flatten(),
ref_output.float().flatten(),
dim=0,
)
assert output.shape == ref_output.shape
assert output.dtype == dtype
assert cosine.item() >= 0.99
assert diff.mean().item() <= 0.03
assert diff.max().item() <= 0.30
finally:
vllm_config.model_config = old_model_config
@pytest.mark.skipif(
not current_platform.is_rocm(), reason="AITER FP8 attention requires ROCm"
)
def test_mha_attn_aiter_fp8_rejects_unsupported_arch(default_vllm_config):
vllm_config = get_current_vllm_config()
old_model_config = getattr(vllm_config, "model_config", None)
vllm_config.model_config = type(
"MinimalModelConfig",
(),
{
"multimodal_config": MultiModalConfig(
mm_encoder_attn_backend=AttentionBackendEnum.ROCM_AITER_FA,
mm_encoder_attn_dtype="fp8",
),
},
)()
try:
with (
patch("vllm.platforms.rocm.on_mi3xx", return_value=False),
pytest.raises(ValueError, match="gfx942 or gfx950"),
):
MMEncoderAttention(4, 64)
finally:
vllm_config.model_config = old_model_config
@pytest.mark.skipif(
not current_platform.is_rocm(), reason="AITER FP8 attention requires ROCm"
)
def test_mha_attn_fp8_rejects_wrong_backend(default_vllm_config):
vllm_config = get_current_vllm_config()
old_model_config = getattr(vllm_config, "model_config", None)
vllm_config.model_config = type(
"MinimalModelConfig",
(),
{
"multimodal_config": MultiModalConfig(
mm_encoder_attn_backend=AttentionBackendEnum.FLASH_ATTN,
mm_encoder_attn_dtype="fp8",
),
},
)()
try:
with pytest.raises(ValueError, match="requires either"):
MMEncoderAttention(4, 64)
finally:
vllm_config.model_config = old_model_config
+44 -3
View File
@@ -14,6 +14,7 @@ from vllm.model_executor.layers.attention.mm_encoder_attention import (
_FP8_AMAX_HISTORY_LEN,
_FP8_MAX,
)
from vllm.platforms import current_platform
from vllm.utils.flashinfer import (
is_flashinfer_cudnn_fp8_prefill_attn_supported,
)
@@ -29,8 +30,17 @@ NUM_HEADS = 16
HEAD_DIM = 72
def _is_mi3xx() -> bool:
if not current_platform.is_rocm():
return False
from vllm.platforms.rocm import on_mi3xx
return on_mi3xx()
@contextlib.contextmanager
def _build_attention(mm_config):
def _build_attention(mm_config, attn_backend=None):
"""Yield an MMEncoderAttention with the given multimodal config.
The VllmConfig context stays active while the test runs so that
@@ -44,7 +54,13 @@ def _build_attention(mm_config):
)
from vllm.v1.attention.backends.registry import AttentionBackendEnum
if not is_flashinfer_cudnn_fp8_prefill_attn_supported():
if attn_backend is None:
attn_backend = AttentionBackendEnum.FLASHINFER
if (
attn_backend == AttentionBackendEnum.FLASHINFER
and not is_flashinfer_cudnn_fp8_prefill_attn_supported()
):
yield None
return
@@ -56,7 +72,7 @@ def _build_attention(mm_config):
patch(
"vllm.model_executor.layers.attention.mm_encoder_attention"
".get_vit_attn_backend",
return_value=AttentionBackendEnum.FLASHINFER,
return_value=attn_backend,
),
):
attn = MMEncoderAttention(
@@ -169,6 +185,31 @@ def test_static_scales_loaded(_make_static_attention) -> None:
assert not hasattr(attn, "_fp8_q_amax")
@pytest.mark.skipif(
not _is_mi3xx(),
reason="AITER FP8 attention requires MI300/MI350",
)
def test_aiter_static_scales_loaded(tmp_path) -> None:
"""Verify AITER reuses the existing static FP8 scale loading path."""
from vllm.config.multimodal import MultiModalConfig
from vllm.v1.attention.backends.registry import AttentionBackendEnum
scale_file = tmp_path / "aiter_scales.json"
scale_file.write_text(json.dumps({LAYER_0: {"q": 224.0, "k": 198.0, "v": 210.0}}))
mm_config = MultiModalConfig(
mm_encoder_attn_dtype="fp8",
mm_encoder_fp8_scale_path=str(scale_file),
)
with _build_attention(mm_config, AttentionBackendEnum.ROCM_AITER_FA) as attn:
assert attn is not None
assert attn.fp8_enabled
assert not attn._fp8_dynamic_scale
assert attn._fp8_q_scale.item() == 224.0
assert attn._fp8_k_scale.item() == 198.0
assert attn._fp8_v_scale.item() == 210.0
def test_static_scales_missing_layer(tmp_path) -> None:
"""Verify error when requested layer is not in the scale file."""
from vllm.config import VllmConfig, set_current_vllm_config
+95
View File
@@ -1404,6 +1404,65 @@ def _triton_rotary_embedding_fake(
return
def _rocm_aiter_fp8_attn_impl(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
q_descale: torch.Tensor,
k_descale: torch.Tensor,
v_descale: torch.Tensor,
batch_size: int,
output_dtype: torch.dtype,
scale: float | None = None,
cu_seqlens: torch.Tensor | None = None,
max_seqlen: torch.Tensor | None = None,
) -> torch.Tensor:
"""Run AITER FP8 attention for fixed or packed inputs."""
from aiter import flash_attn_varlen_fp8_pertensor_func
q_len = q.size(1)
if cu_seqlens is None:
cu_seqlens = torch.arange(
0, (batch_size + 1) * q_len, step=q_len, dtype=torch.int32, device=q.device
)
max_seqlen_value = q_len if max_seqlen is None else max_seqlen.item()
q, k, v = (x.flatten(0, 1) for x in (q, k, v))
output = flash_attn_varlen_fp8_pertensor_func(
q,
k,
v,
q_descale=q_descale,
k_descale=k_descale,
v_descale=v_descale,
cu_seqlens_q=cu_seqlens,
cu_seqlens_k=cu_seqlens,
max_seqlen_q=max_seqlen_value,
max_seqlen_k=max_seqlen_value,
causal=False,
softmax_scale=scale,
)
return output.to(output_dtype).reshape(batch_size, q_len, *output.shape[1:])
def _rocm_aiter_fp8_attn_fake(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
q_descale: torch.Tensor,
k_descale: torch.Tensor,
v_descale: torch.Tensor,
batch_size: int,
output_dtype: torch.dtype,
scale: float | None = None,
cu_seqlens: torch.Tensor | None = None,
max_seqlen: torch.Tensor | None = None,
) -> torch.Tensor:
return torch.empty(
(*q.shape[:-1], v.shape[-1]), device=q.device, dtype=output_dtype
)
# Global flag to ensure ops are registered only once
_OPS_REGISTERED = False
@@ -1959,6 +2018,14 @@ class rocm_aiter_ops:
dispatch_key=current_platform.dispatch_key,
)
direct_register_custom_op(
op_name="aiter_fp8_attn_wrapper",
op_func=_rocm_aiter_fp8_attn_impl,
mutates_args=[],
fake_impl=_rocm_aiter_fp8_attn_fake,
dispatch_key=current_platform.dispatch_key,
)
direct_register_custom_op(
op_name="rocm_aiter_gemm_a8wfp4",
op_func=_rocm_aiter_gemm_a8wfp4_impl,
@@ -2830,6 +2897,34 @@ class rocm_aiter_ops:
sink_ptr=sink_ptr,
)
@staticmethod
def fp8_attn_wrapper(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
q_descale: torch.Tensor,
k_descale: torch.Tensor,
v_descale: torch.Tensor,
batch_size: int,
output_dtype: torch.dtype,
scale: float | None = None,
cu_seqlens: torch.Tensor | None = None,
max_seqlen: torch.Tensor | None = None,
) -> torch.Tensor:
return torch.ops.vllm.aiter_fp8_attn_wrapper(
q,
k,
v,
q_descale,
k_descale,
v_descale,
batch_size,
output_dtype,
scale,
cu_seqlens,
max_seqlen,
)
@staticmethod
def pa_fwd_asm(
Q: torch.Tensor,
@@ -24,6 +24,7 @@ from vllm.model_executor.models.vision import (
get_multimodal_config,
get_vit_attn_backend,
)
from vllm.platforms import current_platform
from vllm.utils.flashinfer import (
is_flashinfer_cudnn_fp8_prefill_attn_supported,
)
@@ -32,6 +33,7 @@ from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.attention.ops.vit_attn_wrappers import (
vit_aiter_fp8_attn_wrapper,
vit_flash_attn_wrapper,
vit_flashinfer_wrapper,
vit_torch_sdpa_wrapper,
@@ -380,8 +382,8 @@ class MMEncoderAttention(CustomOp):
No-op if FP8 is not requested. Raises ``ValueError`` if FP8 is
requested but the platform does not support it.
"""
# Populate defaults so ``_forward_flashinfer`` can
# check ``self.fp8_enabled`` and others without AttributeError.
# Populate defaults so backend forward methods can check
# ``self.fp8_enabled`` and related state without AttributeError.
self.fp8_enabled = False
self._fp8_dynamic_scale = False
self.fp8_quant: QuantFP8 | None = None
@@ -393,13 +395,38 @@ class MMEncoderAttention(CustomOp):
if mm_cfg is None or mm_cfg.mm_encoder_attn_dtype != "fp8":
return
# FP8 path
if not is_flashinfer_cudnn_fp8_prefill_attn_supported():
if self.attn_backend == AttentionBackendEnum.ROCM_AITER_FA:
if not current_platform.is_rocm():
raise ValueError("AITER FP8 ViT attention requires ROCm.")
from vllm.platforms.rocm import on_mi3xx
if not on_mi3xx():
raise ValueError(
"AITER FP8 ViT attention requires an MI300-series or "
"MI350-series GPU (gfx942 or gfx950)."
)
try:
from aiter import flash_attn_varlen_fp8_pertensor_func # noqa: F401
except ImportError as exc:
raise ValueError(
"mm_encoder_attn_dtype='fp8' with ROCM_AITER_FA requires "
"an AITER build that provides "
"flash_attn_varlen_fp8_pertensor_func."
) from exc
elif self.attn_backend == AttentionBackendEnum.FLASHINFER:
if not is_flashinfer_cudnn_fp8_prefill_attn_supported():
raise ValueError(
"mm_encoder_attn_dtype='fp8' requires the FlashInfer "
"cuDNN backend with cuDNN >= 9.17.1 on Blackwell (SM 100) "
"or newer. cuDNN's FP8 SDPA path with bf16/fp16 output is "
"not available on Hopper (H100/H200) or earlier."
)
else:
raise ValueError(
"mm_encoder_attn_dtype='fp8' requires the FlashInfer "
"cuDNN backend with cuDNN >= 9.17.1 on Blackwell (SM 100) "
"or newer. cuDNN's FP8 SDPA path with bf16/fp16 output is "
"not available on Hopper (H100/H200) or earlier."
"mm_encoder_attn_dtype='fp8' requires either the "
"ROCM_AITER_FA backend on ROCm or the FlashInfer cuDNN "
f"backend on CUDA, got {self.attn_backend}."
)
self.fp8_enabled = True
@@ -642,6 +669,37 @@ class MMEncoderAttention(CustomOp):
buffer_wrapped,
)
def _quantize_qkv_fp8(
self,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
assert self.fp8_quant is not None
if self._fp8_dynamic_scale:
self._record_amax_and_update_scales(query, key, value)
query = quantize_fp8_maybe_pad_head_dim(
query,
self._fp8_q_scale,
skip_scale=self.skip_scale_q,
fp8_quant=self.fp8_quant,
)
key = quantize_fp8_maybe_pad_head_dim(
key,
self._fp8_k_scale,
skip_scale=self.skip_scale_k,
fp8_quant=self.fp8_quant,
)
value = quantize_fp8_maybe_pad_head_dim(
value,
self._fp8_v_scale,
skip_scale=self.skip_scale_v,
fp8_quant=self.fp8_quant,
)
return query, key, value
def _forward_flashinfer(
self,
query: torch.Tensor,
@@ -653,29 +711,7 @@ class MMEncoderAttention(CustomOp):
| None = None, # Only used for FlashInfer CuDNN backend
) -> torch.Tensor:
if self.fp8_enabled:
assert self.fp8_quant is not None
if self._fp8_dynamic_scale:
self._record_amax_and_update_scales(query, key, value)
query = quantize_fp8_maybe_pad_head_dim(
query,
self._fp8_q_scale,
skip_scale=self.skip_scale_q,
fp8_quant=self.fp8_quant,
)
key = quantize_fp8_maybe_pad_head_dim(
key,
self._fp8_k_scale,
skip_scale=self.skip_scale_k,
fp8_quant=self.fp8_quant,
)
value = quantize_fp8_maybe_pad_head_dim(
value,
self._fp8_v_scale,
skip_scale=self.skip_scale_v,
fp8_quant=self.fp8_quant,
)
query, key, value = self._quantize_qkv_fp8(query, key, value)
output = vit_flashinfer_wrapper(
q=query,
@@ -697,6 +733,44 @@ class MMEncoderAttention(CustomOp):
return output
def _forward_aiter_fp8(
self,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
cu_seqlens: torch.Tensor | None = None,
max_seqlen: torch.Tensor | None = None,
) -> torch.Tensor:
assert (cu_seqlens is not None and max_seqlen is not None) or (
cu_seqlens is None and max_seqlen is None
), "cu_seqlens and max_seqlen should be both set or both None."
bsz, q_len = query.size()[:2]
kv_len = key.size(1)
is_reshaped = query.dim() != 4
query, key, value = self.view_qkv_to_4d(query, key, value, bsz, q_len, kv_len)
query, key, value = self._quantize_qkv_fp8(query, key, value)
output = vit_aiter_fp8_attn_wrapper(
q=query,
k=key,
v=value,
q_descale=self._fp8_q_scale,
k_descale=self._fp8_k_scale,
v_descale=self._fp8_v_scale,
batch_size=bsz,
output_dtype=self.dtype,
scale=self.scale,
cu_seqlens=cu_seqlens,
max_seqlen=max_seqlen,
)
if output.shape[-1] != self.head_size:
output = output[..., : self.head_size].contiguous()
if is_reshaped:
output = output.reshape(bsz, q_len, -1)
return output
def forward_native(
self,
query: torch.Tensor,
@@ -719,7 +793,9 @@ class MMEncoderAttention(CustomOp):
sequence_lengths: torch.Tensor
| None = None, # Only used for FlashInfer CuDNN backend
) -> torch.Tensor:
if self.is_flash_attn_backend:
if self.fp8_enabled and self.attn_backend == AttentionBackendEnum.ROCM_AITER_FA:
return self._forward_aiter_fp8(query, key, value, cu_seqlens, max_seqlen)
elif self.is_flash_attn_backend:
return self._forward_fa(query, key, value, cu_seqlens, max_seqlen)
elif self.attn_backend == AttentionBackendEnum.TRITON_ATTN:
return self._forward_triton(query, key, value, cu_seqlens, max_seqlen)
+1 -1
View File
@@ -657,7 +657,7 @@ class RocmPlatform(Platform):
from vllm._aiter_ops import rocm_aiter_ops
if rocm_aiter_ops.is_enabled() and on_gfx9():
if rocm_aiter_ops.is_mha_enabled() and on_gfx9():
logger.info_once("Using AITER Flash Attention backend for ViT model.")
return AttentionBackendEnum.ROCM_AITER_FA
@@ -18,6 +18,7 @@ import einops
import torch
import torch.nn.functional as F
from vllm._aiter_ops import rocm_aiter_ops
from vllm.platforms import current_platform
from vllm.utils.torch_utils import direct_register_custom_op
@@ -114,6 +115,34 @@ def vit_flash_attn_wrapper(
)
def vit_aiter_fp8_attn_wrapper(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
q_descale: torch.Tensor,
k_descale: torch.Tensor,
v_descale: torch.Tensor,
batch_size: int,
output_dtype: torch.dtype,
scale: float | None = None,
cu_seqlens: torch.Tensor | None = None,
max_seqlen: torch.Tensor | None = None,
) -> torch.Tensor:
return rocm_aiter_ops.fp8_attn_wrapper(
q,
k,
v,
q_descale,
k_descale,
v_descale,
batch_size,
output_dtype,
scale,
cu_seqlens,
max_seqlen,
)
def triton_attn_wrapper(
q: torch.Tensor,
k: torch.Tensor,