[ROCm] Add AITER FP8 ViT encoder attention (#49937)
Signed-off-by: Liuyinfeng01 <yinfeliu@amd.com>
This commit is contained in:
@@ -0,0 +1,124 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Benchmark complete AITER BF16 and FP8 ViT attention calls on ROCm.
|
||||
|
||||
The FP8 timing includes Q/K/V quantization and attention. Dynamic scales are
|
||||
calibrated once per input before timing so the measured path matches serving
|
||||
with a static scale file.
|
||||
|
||||
Example:
|
||||
python benchmarks/kernels/benchmark_vit_aiter_fp8_attn.py \
|
||||
--seq-lens 2304 4096 8192 16384 --head-dim 72
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.config.multimodal import MultiModalConfig
|
||||
from vllm.model_executor.layers.attention.mm_encoder_attention import (
|
||||
MMEncoderAttention,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import triton
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
|
||||
def make_attention(num_heads: int, head_dim: int, fp8: bool) -> MMEncoderAttention:
|
||||
mm_config = MultiModalConfig(
|
||||
mm_encoder_attn_backend=AttentionBackendEnum.ROCM_AITER_FA,
|
||||
mm_encoder_attn_dtype="fp8" if fp8 else None,
|
||||
)
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.model_config = SimpleNamespace(multimodal_config=mm_config)
|
||||
with set_current_vllm_config(vllm_config):
|
||||
return MMEncoderAttention(num_heads, head_dim).to("cuda")
|
||||
|
||||
|
||||
def bench(
|
||||
seq_lens: list[int],
|
||||
num_heads: int,
|
||||
head_dim: int,
|
||||
warmup_ms: int,
|
||||
repeat_ms: int,
|
||||
) -> None:
|
||||
if not current_platform.is_rocm():
|
||||
raise RuntimeError("This benchmark requires ROCm and AITER.")
|
||||
|
||||
old_dtype = torch.get_default_dtype()
|
||||
torch.set_default_dtype(torch.bfloat16)
|
||||
try:
|
||||
fp8_attention = make_attention(num_heads, head_dim, fp8=True)
|
||||
bf16_attention = make_attention(num_heads, head_dim, fp8=False)
|
||||
finally:
|
||||
torch.set_default_dtype(old_dtype)
|
||||
|
||||
print(
|
||||
f"{'seq_len':>8} {'BF16 ms':>12} {'FP8 ms':>12} "
|
||||
f"{'speedup':>10} {'FP8/BF16':>12}"
|
||||
)
|
||||
print("-" * 60)
|
||||
for seq_len in seq_lens:
|
||||
torch.manual_seed(0)
|
||||
qkv = torch.randn(
|
||||
1,
|
||||
seq_len,
|
||||
3,
|
||||
num_heads,
|
||||
head_dim,
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
q, k, v = qkv.unbind(dim=2)
|
||||
cu_seqlens = torch.tensor([0, seq_len], device="cuda", dtype=torch.int32)
|
||||
max_seqlen = torch.tensor(seq_len, device="cuda", dtype=torch.int32)
|
||||
|
||||
# Calibrate per-tensor scales once, then benchmark the static-scale path.
|
||||
fp8_attention._fp8_dynamic_scale = True
|
||||
fp8_attention._forward_aiter_fp8(q, k, v, cu_seqlens, max_seqlen)
|
||||
fp8_attention._fp8_dynamic_scale = False
|
||||
|
||||
bf16_ms = triton.testing.do_bench(
|
||||
lambda q=q, k=k, v=v, cu=cu_seqlens, ms=max_seqlen: (
|
||||
bf16_attention._forward_fa(q, k, v, cu, ms)
|
||||
),
|
||||
warmup=warmup_ms,
|
||||
rep=repeat_ms,
|
||||
)
|
||||
fp8_ms = triton.testing.do_bench(
|
||||
lambda q=q, k=k, v=v, cu=cu_seqlens, ms=max_seqlen: (
|
||||
fp8_attention._forward_aiter_fp8(q, k, v, cu, ms)
|
||||
),
|
||||
warmup=warmup_ms,
|
||||
rep=repeat_ms,
|
||||
)
|
||||
speedup = bf16_ms / fp8_ms
|
||||
ratio = fp8_ms / bf16_ms
|
||||
print(
|
||||
f"{seq_len:>8} {bf16_ms:>12.3f} {fp8_ms:>12.3f} "
|
||||
f"{speedup:>9.2f}x {ratio:>12.3f}"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = FlexibleArgumentParser(
|
||||
description="Benchmark AITER BF16 vs FP8 ViT attention."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--seq-lens", type=int, nargs="+", default=[2304, 4096, 8192, 16384]
|
||||
)
|
||||
parser.add_argument("--num-heads", type=int, default=16)
|
||||
parser.add_argument("--head-dim", type=int, default=72)
|
||||
parser.add_argument("--warmup-ms", type=int, default=100)
|
||||
parser.add_argument("--repeat-ms", type=int, default=500)
|
||||
args = parser.parse_args()
|
||||
bench(
|
||||
args.seq_lens,
|
||||
args.num_heads,
|
||||
args.head_dim,
|
||||
args.warmup_ms,
|
||||
args.repeat_ms,
|
||||
)
|
||||
@@ -3,9 +3,9 @@
|
||||
For visual understanding workloads with large images (e.g. QHD, 4K) and relatively
|
||||
short text prompts/generation, the ViT encoder attention can become a significant
|
||||
bottleneck, especially when the text model is quantized (e.g. NVFP4). vLLM
|
||||
supports optional FP8 quantization for the ViT encoder attention via the
|
||||
FlashInfer cuDNN backend. Q/K/V are quantized on-the-fly to FP8 before the
|
||||
cuDNN attention call.
|
||||
supports optional FP8 quantization for ViT encoder attention via FlashInfer
|
||||
cuDNN on NVIDIA GPUs and AITER on AMD GPUs. Q/K/V are quantized on-the-fly to
|
||||
FP8 before the attention call.
|
||||
|
||||
!!! note
|
||||
- Currently supports Qwen3-VL family models only (`qwen3_vl`, `qwen3_vl_moe`,
|
||||
@@ -15,20 +15,29 @@ cuDNN attention call.
|
||||
requests. Smaller images may see no speedup due to quantization overhead
|
||||
(3 quantization kernel launches + un-padding).
|
||||
- FP8 tensor-core speedup is more pronounced on GB300 than GB200.
|
||||
- On ROCm, packed variable-length image and video batches are supported by
|
||||
AITER's native varlen FP8 attention.
|
||||
|
||||
## Requirements
|
||||
|
||||
- FlashInfer cuDNN backend with cuDNN >= 9.17.1.
|
||||
- NVIDIA: FlashInfer cuDNN backend with cuDNN >= 9.17.1.
|
||||
- AMD: AITER with `flash_attn_varlen_fp8_pertensor_func` support on gfx942
|
||||
(MI300 series) or gfx950 (MI350 series).
|
||||
|
||||
## Usage
|
||||
|
||||
Enable FP8 ViT attention by passing `--mm-encoder-attn-dtype fp8` together
|
||||
with `--mm-encoder-attn-backend FLASHINFER`:
|
||||
Enable FP8 ViT attention by passing `--mm-encoder-attn-dtype fp8` and selecting
|
||||
the backend for the current platform:
|
||||
|
||||
```bash
|
||||
vllm serve $MODEL \
|
||||
--mm-encoder-attn-backend FLASHINFER \
|
||||
--mm-encoder-attn-dtype fp8
|
||||
|
||||
# AMD ROCm
|
||||
vllm serve $MODEL \
|
||||
--mm-encoder-attn-backend ROCM_AITER_FA \
|
||||
--mm-encoder-attn-dtype fp8
|
||||
```
|
||||
|
||||
By default (no scale file), **dynamic scaling** is used: a 16-entry circular
|
||||
@@ -45,7 +54,7 @@ reuse them to avoid the dynamic overhead:
|
||||
# Step 1: calibrate and save scales (runs dynamic scaling for 16 passes,
|
||||
# then dumps the learned scales to JSON).
|
||||
vllm bench mm-processor \
|
||||
--model $MODEL --mm-encoder-attn-backend FLASHINFER \
|
||||
--model $MODEL --mm-encoder-attn-backend $MM_ATTN_BACKEND \
|
||||
--mm-encoder-attn-dtype fp8 \
|
||||
--mm-encoder-fp8-scale-save-path /path/to/scales.json \
|
||||
--dataset-name hf --dataset-path lmarena-ai/VisionArena-Chat \
|
||||
@@ -53,7 +62,7 @@ vllm bench mm-processor \
|
||||
|
||||
# Step 2: serve with static scales (no dynamic overhead).
|
||||
vllm serve $MODEL \
|
||||
--mm-encoder-attn-backend FLASHINFER \
|
||||
--mm-encoder-attn-backend $MM_ATTN_BACKEND \
|
||||
--mm-encoder-attn-dtype fp8 \
|
||||
--mm-encoder-fp8-scale-path /path/to/scales.json
|
||||
```
|
||||
@@ -94,6 +103,16 @@ Keys `q_scale` / `k_scale` / `v_scale` are accepted as aliases.
|
||||
|
||||
Crossover is around FullHD with 3 images/request. At QHD and above, FP8 wins.
|
||||
|
||||
**Complete AITER attention call on MI300X** (BF16 input, 16 heads,
|
||||
head_dim=72; FP8 includes Q/K/V quantization):
|
||||
|
||||
| Sequence length | AITER BF16 | AITER FP8 | Speedup |
|
||||
| --------------- | ---------- | --------- | ------- |
|
||||
| 2304 | 0.467 ms | 0.337 ms | **1.38x** |
|
||||
| 4096 | 0.812 ms | 0.764 ms | **1.06x** |
|
||||
| 8192 | 2.555 ms | 2.364 ms | **1.08x** |
|
||||
| 16384 | 9.769 ms | 8.655 ms | **1.13x** |
|
||||
|
||||
## Accuracy
|
||||
|
||||
ChartQA, Qwen3-VL-8B-Instruct, 500 samples. FP8 static uses scales calibrated
|
||||
|
||||
@@ -20,7 +20,7 @@ from vllm.platforms import current_platform
|
||||
from vllm.platforms.cpu import CpuPlatform
|
||||
from vllm.platforms.cuda import CudaPlatform
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.platforms.rocm import RocmPlatform
|
||||
from vllm.platforms.rocm import RocmPlatform, on_mi3xx
|
||||
from vllm.utils.torch_utils import set_default_torch_dtype, set_random_seed
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.attention.selector import _cached_get_attn_backend
|
||||
@@ -346,3 +346,137 @@ def test_mha_attn_varlen_forward_flashinfer(
|
||||
torch.testing.assert_close(output, ref_output, atol=1e-2, rtol=1e-2)
|
||||
finally:
|
||||
vllm_config.model_config = old_model_config
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm() or not on_mi3xx(),
|
||||
reason="AITER FP8 attention requires MI300/MI350",
|
||||
)
|
||||
@pytest.mark.parametrize("var_seq_len", [[128, 193], [256, 384, 511]])
|
||||
@pytest.mark.parametrize("head_size", [64, 72, 80])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half])
|
||||
def test_mha_attn_varlen_forward_aiter_fp8(
|
||||
default_vllm_config,
|
||||
var_seq_len: list[int],
|
||||
head_size: int,
|
||||
dtype: torch.dtype,
|
||||
):
|
||||
"""Compare packed AITER FP8 ViT attention with a BF16/FP16 reference."""
|
||||
aiter = pytest.importorskip("aiter")
|
||||
if not hasattr(aiter, "flash_attn_varlen_fp8_pertensor_func"):
|
||||
pytest.skip("Installed AITER does not provide varlen FP8 attention")
|
||||
|
||||
num_heads = 4
|
||||
set_random_seed(0)
|
||||
torch.set_default_device("cuda")
|
||||
torch.set_default_dtype(dtype)
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
old_model_config = getattr(vllm_config, "model_config", None)
|
||||
minimal_model_config = type(
|
||||
"MinimalModelConfig",
|
||||
(),
|
||||
{
|
||||
"multimodal_config": MultiModalConfig(
|
||||
mm_encoder_attn_backend=AttentionBackendEnum.ROCM_AITER_FA,
|
||||
mm_encoder_attn_dtype="fp8",
|
||||
),
|
||||
},
|
||||
)()
|
||||
vllm_config.model_config = minimal_model_config
|
||||
try:
|
||||
total_len = sum(var_seq_len)
|
||||
# Keep Q/K/V as non-contiguous views, matching interleaved ViT QKV.
|
||||
qkv = torch.randn(1, total_len, 3, num_heads, head_size)
|
||||
q, k, v = qkv.unbind(dim=2)
|
||||
cu_seqlens = torch.tensor(
|
||||
[0] + list(itertools.accumulate(var_seq_len)), dtype=torch.int32
|
||||
)
|
||||
max_seqlen = torch.tensor(max(var_seq_len), dtype=torch.int32)
|
||||
scale = 1.0 / head_size**0.5
|
||||
|
||||
attn = MMEncoderAttention(num_heads, head_size, scale=scale)
|
||||
assert attn.attn_backend == AttentionBackendEnum.ROCM_AITER_FA
|
||||
assert attn.fp8_enabled
|
||||
output = attn(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
cu_seqlens=cu_seqlens,
|
||||
max_seqlen=max_seqlen,
|
||||
)
|
||||
|
||||
ref_output = torch.cat(
|
||||
[
|
||||
ref_attention(q_i, k_i, v_i, scale=scale)
|
||||
for q_i, k_i, v_i in zip(
|
||||
torch.split(q, var_seq_len, dim=1),
|
||||
torch.split(k, var_seq_len, dim=1),
|
||||
torch.split(v, var_seq_len, dim=1),
|
||||
)
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
diff = (output.float() - ref_output.float()).abs()
|
||||
cosine = torch.nn.functional.cosine_similarity(
|
||||
output.float().flatten(),
|
||||
ref_output.float().flatten(),
|
||||
dim=0,
|
||||
)
|
||||
|
||||
assert output.shape == ref_output.shape
|
||||
assert output.dtype == dtype
|
||||
assert cosine.item() >= 0.99
|
||||
assert diff.mean().item() <= 0.03
|
||||
assert diff.max().item() <= 0.30
|
||||
finally:
|
||||
vllm_config.model_config = old_model_config
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="AITER FP8 attention requires ROCm"
|
||||
)
|
||||
def test_mha_attn_aiter_fp8_rejects_unsupported_arch(default_vllm_config):
|
||||
vllm_config = get_current_vllm_config()
|
||||
old_model_config = getattr(vllm_config, "model_config", None)
|
||||
vllm_config.model_config = type(
|
||||
"MinimalModelConfig",
|
||||
(),
|
||||
{
|
||||
"multimodal_config": MultiModalConfig(
|
||||
mm_encoder_attn_backend=AttentionBackendEnum.ROCM_AITER_FA,
|
||||
mm_encoder_attn_dtype="fp8",
|
||||
),
|
||||
},
|
||||
)()
|
||||
try:
|
||||
with (
|
||||
patch("vllm.platforms.rocm.on_mi3xx", return_value=False),
|
||||
pytest.raises(ValueError, match="gfx942 or gfx950"),
|
||||
):
|
||||
MMEncoderAttention(4, 64)
|
||||
finally:
|
||||
vllm_config.model_config = old_model_config
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="AITER FP8 attention requires ROCm"
|
||||
)
|
||||
def test_mha_attn_fp8_rejects_wrong_backend(default_vllm_config):
|
||||
vllm_config = get_current_vllm_config()
|
||||
old_model_config = getattr(vllm_config, "model_config", None)
|
||||
vllm_config.model_config = type(
|
||||
"MinimalModelConfig",
|
||||
(),
|
||||
{
|
||||
"multimodal_config": MultiModalConfig(
|
||||
mm_encoder_attn_backend=AttentionBackendEnum.FLASH_ATTN,
|
||||
mm_encoder_attn_dtype="fp8",
|
||||
),
|
||||
},
|
||||
)()
|
||||
try:
|
||||
with pytest.raises(ValueError, match="requires either"):
|
||||
MMEncoderAttention(4, 64)
|
||||
finally:
|
||||
vllm_config.model_config = old_model_config
|
||||
|
||||
@@ -14,6 +14,7 @@ from vllm.model_executor.layers.attention.mm_encoder_attention import (
|
||||
_FP8_AMAX_HISTORY_LEN,
|
||||
_FP8_MAX,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import (
|
||||
is_flashinfer_cudnn_fp8_prefill_attn_supported,
|
||||
)
|
||||
@@ -29,8 +30,17 @@ NUM_HEADS = 16
|
||||
HEAD_DIM = 72
|
||||
|
||||
|
||||
def _is_mi3xx() -> bool:
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
|
||||
from vllm.platforms.rocm import on_mi3xx
|
||||
|
||||
return on_mi3xx()
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _build_attention(mm_config):
|
||||
def _build_attention(mm_config, attn_backend=None):
|
||||
"""Yield an MMEncoderAttention with the given multimodal config.
|
||||
|
||||
The VllmConfig context stays active while the test runs so that
|
||||
@@ -44,7 +54,13 @@ def _build_attention(mm_config):
|
||||
)
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
if not is_flashinfer_cudnn_fp8_prefill_attn_supported():
|
||||
if attn_backend is None:
|
||||
attn_backend = AttentionBackendEnum.FLASHINFER
|
||||
|
||||
if (
|
||||
attn_backend == AttentionBackendEnum.FLASHINFER
|
||||
and not is_flashinfer_cudnn_fp8_prefill_attn_supported()
|
||||
):
|
||||
yield None
|
||||
return
|
||||
|
||||
@@ -56,7 +72,7 @@ def _build_attention(mm_config):
|
||||
patch(
|
||||
"vllm.model_executor.layers.attention.mm_encoder_attention"
|
||||
".get_vit_attn_backend",
|
||||
return_value=AttentionBackendEnum.FLASHINFER,
|
||||
return_value=attn_backend,
|
||||
),
|
||||
):
|
||||
attn = MMEncoderAttention(
|
||||
@@ -169,6 +185,31 @@ def test_static_scales_loaded(_make_static_attention) -> None:
|
||||
assert not hasattr(attn, "_fp8_q_amax")
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not _is_mi3xx(),
|
||||
reason="AITER FP8 attention requires MI300/MI350",
|
||||
)
|
||||
def test_aiter_static_scales_loaded(tmp_path) -> None:
|
||||
"""Verify AITER reuses the existing static FP8 scale loading path."""
|
||||
from vllm.config.multimodal import MultiModalConfig
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
scale_file = tmp_path / "aiter_scales.json"
|
||||
scale_file.write_text(json.dumps({LAYER_0: {"q": 224.0, "k": 198.0, "v": 210.0}}))
|
||||
mm_config = MultiModalConfig(
|
||||
mm_encoder_attn_dtype="fp8",
|
||||
mm_encoder_fp8_scale_path=str(scale_file),
|
||||
)
|
||||
|
||||
with _build_attention(mm_config, AttentionBackendEnum.ROCM_AITER_FA) as attn:
|
||||
assert attn is not None
|
||||
assert attn.fp8_enabled
|
||||
assert not attn._fp8_dynamic_scale
|
||||
assert attn._fp8_q_scale.item() == 224.0
|
||||
assert attn._fp8_k_scale.item() == 198.0
|
||||
assert attn._fp8_v_scale.item() == 210.0
|
||||
|
||||
|
||||
def test_static_scales_missing_layer(tmp_path) -> None:
|
||||
"""Verify error when requested layer is not in the scale file."""
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
|
||||
@@ -1404,6 +1404,65 @@ def _triton_rotary_embedding_fake(
|
||||
return
|
||||
|
||||
|
||||
def _rocm_aiter_fp8_attn_impl(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
q_descale: torch.Tensor,
|
||||
k_descale: torch.Tensor,
|
||||
v_descale: torch.Tensor,
|
||||
batch_size: int,
|
||||
output_dtype: torch.dtype,
|
||||
scale: float | None = None,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
max_seqlen: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Run AITER FP8 attention for fixed or packed inputs."""
|
||||
from aiter import flash_attn_varlen_fp8_pertensor_func
|
||||
|
||||
q_len = q.size(1)
|
||||
if cu_seqlens is None:
|
||||
cu_seqlens = torch.arange(
|
||||
0, (batch_size + 1) * q_len, step=q_len, dtype=torch.int32, device=q.device
|
||||
)
|
||||
max_seqlen_value = q_len if max_seqlen is None else max_seqlen.item()
|
||||
|
||||
q, k, v = (x.flatten(0, 1) for x in (q, k, v))
|
||||
output = flash_attn_varlen_fp8_pertensor_func(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
q_descale=q_descale,
|
||||
k_descale=k_descale,
|
||||
v_descale=v_descale,
|
||||
cu_seqlens_q=cu_seqlens,
|
||||
cu_seqlens_k=cu_seqlens,
|
||||
max_seqlen_q=max_seqlen_value,
|
||||
max_seqlen_k=max_seqlen_value,
|
||||
causal=False,
|
||||
softmax_scale=scale,
|
||||
)
|
||||
return output.to(output_dtype).reshape(batch_size, q_len, *output.shape[1:])
|
||||
|
||||
|
||||
def _rocm_aiter_fp8_attn_fake(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
q_descale: torch.Tensor,
|
||||
k_descale: torch.Tensor,
|
||||
v_descale: torch.Tensor,
|
||||
batch_size: int,
|
||||
output_dtype: torch.dtype,
|
||||
scale: float | None = None,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
max_seqlen: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty(
|
||||
(*q.shape[:-1], v.shape[-1]), device=q.device, dtype=output_dtype
|
||||
)
|
||||
|
||||
|
||||
# Global flag to ensure ops are registered only once
|
||||
_OPS_REGISTERED = False
|
||||
|
||||
@@ -1959,6 +2018,14 @@ class rocm_aiter_ops:
|
||||
dispatch_key=current_platform.dispatch_key,
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="aiter_fp8_attn_wrapper",
|
||||
op_func=_rocm_aiter_fp8_attn_impl,
|
||||
mutates_args=[],
|
||||
fake_impl=_rocm_aiter_fp8_attn_fake,
|
||||
dispatch_key=current_platform.dispatch_key,
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="rocm_aiter_gemm_a8wfp4",
|
||||
op_func=_rocm_aiter_gemm_a8wfp4_impl,
|
||||
@@ -2830,6 +2897,34 @@ class rocm_aiter_ops:
|
||||
sink_ptr=sink_ptr,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def fp8_attn_wrapper(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
q_descale: torch.Tensor,
|
||||
k_descale: torch.Tensor,
|
||||
v_descale: torch.Tensor,
|
||||
batch_size: int,
|
||||
output_dtype: torch.dtype,
|
||||
scale: float | None = None,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
max_seqlen: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return torch.ops.vllm.aiter_fp8_attn_wrapper(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
q_descale,
|
||||
k_descale,
|
||||
v_descale,
|
||||
batch_size,
|
||||
output_dtype,
|
||||
scale,
|
||||
cu_seqlens,
|
||||
max_seqlen,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def pa_fwd_asm(
|
||||
Q: torch.Tensor,
|
||||
|
||||
@@ -24,6 +24,7 @@ from vllm.model_executor.models.vision import (
|
||||
get_multimodal_config,
|
||||
get_vit_attn_backend,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import (
|
||||
is_flashinfer_cudnn_fp8_prefill_attn_supported,
|
||||
)
|
||||
@@ -32,6 +33,7 @@ from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.attention.ops.vit_attn_wrappers import (
|
||||
vit_aiter_fp8_attn_wrapper,
|
||||
vit_flash_attn_wrapper,
|
||||
vit_flashinfer_wrapper,
|
||||
vit_torch_sdpa_wrapper,
|
||||
@@ -380,8 +382,8 @@ class MMEncoderAttention(CustomOp):
|
||||
No-op if FP8 is not requested. Raises ``ValueError`` if FP8 is
|
||||
requested but the platform does not support it.
|
||||
"""
|
||||
# Populate defaults so ``_forward_flashinfer`` can
|
||||
# check ``self.fp8_enabled`` and others without AttributeError.
|
||||
# Populate defaults so backend forward methods can check
|
||||
# ``self.fp8_enabled`` and related state without AttributeError.
|
||||
self.fp8_enabled = False
|
||||
self._fp8_dynamic_scale = False
|
||||
self.fp8_quant: QuantFP8 | None = None
|
||||
@@ -393,13 +395,38 @@ class MMEncoderAttention(CustomOp):
|
||||
if mm_cfg is None or mm_cfg.mm_encoder_attn_dtype != "fp8":
|
||||
return
|
||||
|
||||
# FP8 path
|
||||
if not is_flashinfer_cudnn_fp8_prefill_attn_supported():
|
||||
if self.attn_backend == AttentionBackendEnum.ROCM_AITER_FA:
|
||||
if not current_platform.is_rocm():
|
||||
raise ValueError("AITER FP8 ViT attention requires ROCm.")
|
||||
|
||||
from vllm.platforms.rocm import on_mi3xx
|
||||
|
||||
if not on_mi3xx():
|
||||
raise ValueError(
|
||||
"AITER FP8 ViT attention requires an MI300-series or "
|
||||
"MI350-series GPU (gfx942 or gfx950)."
|
||||
)
|
||||
try:
|
||||
from aiter import flash_attn_varlen_fp8_pertensor_func # noqa: F401
|
||||
except ImportError as exc:
|
||||
raise ValueError(
|
||||
"mm_encoder_attn_dtype='fp8' with ROCM_AITER_FA requires "
|
||||
"an AITER build that provides "
|
||||
"flash_attn_varlen_fp8_pertensor_func."
|
||||
) from exc
|
||||
elif self.attn_backend == AttentionBackendEnum.FLASHINFER:
|
||||
if not is_flashinfer_cudnn_fp8_prefill_attn_supported():
|
||||
raise ValueError(
|
||||
"mm_encoder_attn_dtype='fp8' requires the FlashInfer "
|
||||
"cuDNN backend with cuDNN >= 9.17.1 on Blackwell (SM 100) "
|
||||
"or newer. cuDNN's FP8 SDPA path with bf16/fp16 output is "
|
||||
"not available on Hopper (H100/H200) or earlier."
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
"mm_encoder_attn_dtype='fp8' requires the FlashInfer "
|
||||
"cuDNN backend with cuDNN >= 9.17.1 on Blackwell (SM 100) "
|
||||
"or newer. cuDNN's FP8 SDPA path with bf16/fp16 output is "
|
||||
"not available on Hopper (H100/H200) or earlier."
|
||||
"mm_encoder_attn_dtype='fp8' requires either the "
|
||||
"ROCM_AITER_FA backend on ROCm or the FlashInfer cuDNN "
|
||||
f"backend on CUDA, got {self.attn_backend}."
|
||||
)
|
||||
|
||||
self.fp8_enabled = True
|
||||
@@ -642,6 +669,37 @@ class MMEncoderAttention(CustomOp):
|
||||
buffer_wrapped,
|
||||
)
|
||||
|
||||
def _quantize_qkv_fp8(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
assert self.fp8_quant is not None
|
||||
|
||||
if self._fp8_dynamic_scale:
|
||||
self._record_amax_and_update_scales(query, key, value)
|
||||
|
||||
query = quantize_fp8_maybe_pad_head_dim(
|
||||
query,
|
||||
self._fp8_q_scale,
|
||||
skip_scale=self.skip_scale_q,
|
||||
fp8_quant=self.fp8_quant,
|
||||
)
|
||||
key = quantize_fp8_maybe_pad_head_dim(
|
||||
key,
|
||||
self._fp8_k_scale,
|
||||
skip_scale=self.skip_scale_k,
|
||||
fp8_quant=self.fp8_quant,
|
||||
)
|
||||
value = quantize_fp8_maybe_pad_head_dim(
|
||||
value,
|
||||
self._fp8_v_scale,
|
||||
skip_scale=self.skip_scale_v,
|
||||
fp8_quant=self.fp8_quant,
|
||||
)
|
||||
return query, key, value
|
||||
|
||||
def _forward_flashinfer(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
@@ -653,29 +711,7 @@ class MMEncoderAttention(CustomOp):
|
||||
| None = None, # Only used for FlashInfer CuDNN backend
|
||||
) -> torch.Tensor:
|
||||
if self.fp8_enabled:
|
||||
assert self.fp8_quant is not None
|
||||
|
||||
if self._fp8_dynamic_scale:
|
||||
self._record_amax_and_update_scales(query, key, value)
|
||||
|
||||
query = quantize_fp8_maybe_pad_head_dim(
|
||||
query,
|
||||
self._fp8_q_scale,
|
||||
skip_scale=self.skip_scale_q,
|
||||
fp8_quant=self.fp8_quant,
|
||||
)
|
||||
key = quantize_fp8_maybe_pad_head_dim(
|
||||
key,
|
||||
self._fp8_k_scale,
|
||||
skip_scale=self.skip_scale_k,
|
||||
fp8_quant=self.fp8_quant,
|
||||
)
|
||||
value = quantize_fp8_maybe_pad_head_dim(
|
||||
value,
|
||||
self._fp8_v_scale,
|
||||
skip_scale=self.skip_scale_v,
|
||||
fp8_quant=self.fp8_quant,
|
||||
)
|
||||
query, key, value = self._quantize_qkv_fp8(query, key, value)
|
||||
|
||||
output = vit_flashinfer_wrapper(
|
||||
q=query,
|
||||
@@ -697,6 +733,44 @@ class MMEncoderAttention(CustomOp):
|
||||
|
||||
return output
|
||||
|
||||
def _forward_aiter_fp8(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
max_seqlen: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert (cu_seqlens is not None and max_seqlen is not None) or (
|
||||
cu_seqlens is None and max_seqlen is None
|
||||
), "cu_seqlens and max_seqlen should be both set or both None."
|
||||
|
||||
bsz, q_len = query.size()[:2]
|
||||
kv_len = key.size(1)
|
||||
is_reshaped = query.dim() != 4
|
||||
query, key, value = self.view_qkv_to_4d(query, key, value, bsz, q_len, kv_len)
|
||||
|
||||
query, key, value = self._quantize_qkv_fp8(query, key, value)
|
||||
|
||||
output = vit_aiter_fp8_attn_wrapper(
|
||||
q=query,
|
||||
k=key,
|
||||
v=value,
|
||||
q_descale=self._fp8_q_scale,
|
||||
k_descale=self._fp8_k_scale,
|
||||
v_descale=self._fp8_v_scale,
|
||||
batch_size=bsz,
|
||||
output_dtype=self.dtype,
|
||||
scale=self.scale,
|
||||
cu_seqlens=cu_seqlens,
|
||||
max_seqlen=max_seqlen,
|
||||
)
|
||||
if output.shape[-1] != self.head_size:
|
||||
output = output[..., : self.head_size].contiguous()
|
||||
if is_reshaped:
|
||||
output = output.reshape(bsz, q_len, -1)
|
||||
return output
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
@@ -719,7 +793,9 @@ class MMEncoderAttention(CustomOp):
|
||||
sequence_lengths: torch.Tensor
|
||||
| None = None, # Only used for FlashInfer CuDNN backend
|
||||
) -> torch.Tensor:
|
||||
if self.is_flash_attn_backend:
|
||||
if self.fp8_enabled and self.attn_backend == AttentionBackendEnum.ROCM_AITER_FA:
|
||||
return self._forward_aiter_fp8(query, key, value, cu_seqlens, max_seqlen)
|
||||
elif self.is_flash_attn_backend:
|
||||
return self._forward_fa(query, key, value, cu_seqlens, max_seqlen)
|
||||
elif self.attn_backend == AttentionBackendEnum.TRITON_ATTN:
|
||||
return self._forward_triton(query, key, value, cu_seqlens, max_seqlen)
|
||||
|
||||
@@ -657,7 +657,7 @@ class RocmPlatform(Platform):
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
if rocm_aiter_ops.is_enabled() and on_gfx9():
|
||||
if rocm_aiter_ops.is_mha_enabled() and on_gfx9():
|
||||
logger.info_once("Using AITER Flash Attention backend for ViT model.")
|
||||
return AttentionBackendEnum.ROCM_AITER_FA
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ import einops
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
@@ -114,6 +115,34 @@ def vit_flash_attn_wrapper(
|
||||
)
|
||||
|
||||
|
||||
def vit_aiter_fp8_attn_wrapper(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
q_descale: torch.Tensor,
|
||||
k_descale: torch.Tensor,
|
||||
v_descale: torch.Tensor,
|
||||
batch_size: int,
|
||||
output_dtype: torch.dtype,
|
||||
scale: float | None = None,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
max_seqlen: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return rocm_aiter_ops.fp8_attn_wrapper(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
q_descale,
|
||||
k_descale,
|
||||
v_descale,
|
||||
batch_size,
|
||||
output_dtype,
|
||||
scale,
|
||||
cu_seqlens,
|
||||
max_seqlen,
|
||||
)
|
||||
|
||||
|
||||
def triton_attn_wrapper(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user