[ROCm] Add AITER FP8 ViT encoder attention (#49937)
Signed-off-by: Liuyinfeng01 <yinfeliu@amd.com>
This commit is contained in:
@@ -1404,6 +1404,65 @@ def _triton_rotary_embedding_fake(
|
||||
return
|
||||
|
||||
|
||||
def _rocm_aiter_fp8_attn_impl(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
q_descale: torch.Tensor,
|
||||
k_descale: torch.Tensor,
|
||||
v_descale: torch.Tensor,
|
||||
batch_size: int,
|
||||
output_dtype: torch.dtype,
|
||||
scale: float | None = None,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
max_seqlen: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Run AITER FP8 attention for fixed or packed inputs."""
|
||||
from aiter import flash_attn_varlen_fp8_pertensor_func
|
||||
|
||||
q_len = q.size(1)
|
||||
if cu_seqlens is None:
|
||||
cu_seqlens = torch.arange(
|
||||
0, (batch_size + 1) * q_len, step=q_len, dtype=torch.int32, device=q.device
|
||||
)
|
||||
max_seqlen_value = q_len if max_seqlen is None else max_seqlen.item()
|
||||
|
||||
q, k, v = (x.flatten(0, 1) for x in (q, k, v))
|
||||
output = flash_attn_varlen_fp8_pertensor_func(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
q_descale=q_descale,
|
||||
k_descale=k_descale,
|
||||
v_descale=v_descale,
|
||||
cu_seqlens_q=cu_seqlens,
|
||||
cu_seqlens_k=cu_seqlens,
|
||||
max_seqlen_q=max_seqlen_value,
|
||||
max_seqlen_k=max_seqlen_value,
|
||||
causal=False,
|
||||
softmax_scale=scale,
|
||||
)
|
||||
return output.to(output_dtype).reshape(batch_size, q_len, *output.shape[1:])
|
||||
|
||||
|
||||
def _rocm_aiter_fp8_attn_fake(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
q_descale: torch.Tensor,
|
||||
k_descale: torch.Tensor,
|
||||
v_descale: torch.Tensor,
|
||||
batch_size: int,
|
||||
output_dtype: torch.dtype,
|
||||
scale: float | None = None,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
max_seqlen: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty(
|
||||
(*q.shape[:-1], v.shape[-1]), device=q.device, dtype=output_dtype
|
||||
)
|
||||
|
||||
|
||||
# Global flag to ensure ops are registered only once
|
||||
_OPS_REGISTERED = False
|
||||
|
||||
@@ -1959,6 +2018,14 @@ class rocm_aiter_ops:
|
||||
dispatch_key=current_platform.dispatch_key,
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="aiter_fp8_attn_wrapper",
|
||||
op_func=_rocm_aiter_fp8_attn_impl,
|
||||
mutates_args=[],
|
||||
fake_impl=_rocm_aiter_fp8_attn_fake,
|
||||
dispatch_key=current_platform.dispatch_key,
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="rocm_aiter_gemm_a8wfp4",
|
||||
op_func=_rocm_aiter_gemm_a8wfp4_impl,
|
||||
@@ -2830,6 +2897,34 @@ class rocm_aiter_ops:
|
||||
sink_ptr=sink_ptr,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def fp8_attn_wrapper(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
q_descale: torch.Tensor,
|
||||
k_descale: torch.Tensor,
|
||||
v_descale: torch.Tensor,
|
||||
batch_size: int,
|
||||
output_dtype: torch.dtype,
|
||||
scale: float | None = None,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
max_seqlen: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return torch.ops.vllm.aiter_fp8_attn_wrapper(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
q_descale,
|
||||
k_descale,
|
||||
v_descale,
|
||||
batch_size,
|
||||
output_dtype,
|
||||
scale,
|
||||
cu_seqlens,
|
||||
max_seqlen,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def pa_fwd_asm(
|
||||
Q: torch.Tensor,
|
||||
|
||||
@@ -24,6 +24,7 @@ from vllm.model_executor.models.vision import (
|
||||
get_multimodal_config,
|
||||
get_vit_attn_backend,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import (
|
||||
is_flashinfer_cudnn_fp8_prefill_attn_supported,
|
||||
)
|
||||
@@ -32,6 +33,7 @@ from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.attention.ops.vit_attn_wrappers import (
|
||||
vit_aiter_fp8_attn_wrapper,
|
||||
vit_flash_attn_wrapper,
|
||||
vit_flashinfer_wrapper,
|
||||
vit_torch_sdpa_wrapper,
|
||||
@@ -380,8 +382,8 @@ class MMEncoderAttention(CustomOp):
|
||||
No-op if FP8 is not requested. Raises ``ValueError`` if FP8 is
|
||||
requested but the platform does not support it.
|
||||
"""
|
||||
# Populate defaults so ``_forward_flashinfer`` can
|
||||
# check ``self.fp8_enabled`` and others without AttributeError.
|
||||
# Populate defaults so backend forward methods can check
|
||||
# ``self.fp8_enabled`` and related state without AttributeError.
|
||||
self.fp8_enabled = False
|
||||
self._fp8_dynamic_scale = False
|
||||
self.fp8_quant: QuantFP8 | None = None
|
||||
@@ -393,13 +395,38 @@ class MMEncoderAttention(CustomOp):
|
||||
if mm_cfg is None or mm_cfg.mm_encoder_attn_dtype != "fp8":
|
||||
return
|
||||
|
||||
# FP8 path
|
||||
if not is_flashinfer_cudnn_fp8_prefill_attn_supported():
|
||||
if self.attn_backend == AttentionBackendEnum.ROCM_AITER_FA:
|
||||
if not current_platform.is_rocm():
|
||||
raise ValueError("AITER FP8 ViT attention requires ROCm.")
|
||||
|
||||
from vllm.platforms.rocm import on_mi3xx
|
||||
|
||||
if not on_mi3xx():
|
||||
raise ValueError(
|
||||
"AITER FP8 ViT attention requires an MI300-series or "
|
||||
"MI350-series GPU (gfx942 or gfx950)."
|
||||
)
|
||||
try:
|
||||
from aiter import flash_attn_varlen_fp8_pertensor_func # noqa: F401
|
||||
except ImportError as exc:
|
||||
raise ValueError(
|
||||
"mm_encoder_attn_dtype='fp8' with ROCM_AITER_FA requires "
|
||||
"an AITER build that provides "
|
||||
"flash_attn_varlen_fp8_pertensor_func."
|
||||
) from exc
|
||||
elif self.attn_backend == AttentionBackendEnum.FLASHINFER:
|
||||
if not is_flashinfer_cudnn_fp8_prefill_attn_supported():
|
||||
raise ValueError(
|
||||
"mm_encoder_attn_dtype='fp8' requires the FlashInfer "
|
||||
"cuDNN backend with cuDNN >= 9.17.1 on Blackwell (SM 100) "
|
||||
"or newer. cuDNN's FP8 SDPA path with bf16/fp16 output is "
|
||||
"not available on Hopper (H100/H200) or earlier."
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
"mm_encoder_attn_dtype='fp8' requires the FlashInfer "
|
||||
"cuDNN backend with cuDNN >= 9.17.1 on Blackwell (SM 100) "
|
||||
"or newer. cuDNN's FP8 SDPA path with bf16/fp16 output is "
|
||||
"not available on Hopper (H100/H200) or earlier."
|
||||
"mm_encoder_attn_dtype='fp8' requires either the "
|
||||
"ROCM_AITER_FA backend on ROCm or the FlashInfer cuDNN "
|
||||
f"backend on CUDA, got {self.attn_backend}."
|
||||
)
|
||||
|
||||
self.fp8_enabled = True
|
||||
@@ -642,6 +669,37 @@ class MMEncoderAttention(CustomOp):
|
||||
buffer_wrapped,
|
||||
)
|
||||
|
||||
def _quantize_qkv_fp8(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
assert self.fp8_quant is not None
|
||||
|
||||
if self._fp8_dynamic_scale:
|
||||
self._record_amax_and_update_scales(query, key, value)
|
||||
|
||||
query = quantize_fp8_maybe_pad_head_dim(
|
||||
query,
|
||||
self._fp8_q_scale,
|
||||
skip_scale=self.skip_scale_q,
|
||||
fp8_quant=self.fp8_quant,
|
||||
)
|
||||
key = quantize_fp8_maybe_pad_head_dim(
|
||||
key,
|
||||
self._fp8_k_scale,
|
||||
skip_scale=self.skip_scale_k,
|
||||
fp8_quant=self.fp8_quant,
|
||||
)
|
||||
value = quantize_fp8_maybe_pad_head_dim(
|
||||
value,
|
||||
self._fp8_v_scale,
|
||||
skip_scale=self.skip_scale_v,
|
||||
fp8_quant=self.fp8_quant,
|
||||
)
|
||||
return query, key, value
|
||||
|
||||
def _forward_flashinfer(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
@@ -653,29 +711,7 @@ class MMEncoderAttention(CustomOp):
|
||||
| None = None, # Only used for FlashInfer CuDNN backend
|
||||
) -> torch.Tensor:
|
||||
if self.fp8_enabled:
|
||||
assert self.fp8_quant is not None
|
||||
|
||||
if self._fp8_dynamic_scale:
|
||||
self._record_amax_and_update_scales(query, key, value)
|
||||
|
||||
query = quantize_fp8_maybe_pad_head_dim(
|
||||
query,
|
||||
self._fp8_q_scale,
|
||||
skip_scale=self.skip_scale_q,
|
||||
fp8_quant=self.fp8_quant,
|
||||
)
|
||||
key = quantize_fp8_maybe_pad_head_dim(
|
||||
key,
|
||||
self._fp8_k_scale,
|
||||
skip_scale=self.skip_scale_k,
|
||||
fp8_quant=self.fp8_quant,
|
||||
)
|
||||
value = quantize_fp8_maybe_pad_head_dim(
|
||||
value,
|
||||
self._fp8_v_scale,
|
||||
skip_scale=self.skip_scale_v,
|
||||
fp8_quant=self.fp8_quant,
|
||||
)
|
||||
query, key, value = self._quantize_qkv_fp8(query, key, value)
|
||||
|
||||
output = vit_flashinfer_wrapper(
|
||||
q=query,
|
||||
@@ -697,6 +733,44 @@ class MMEncoderAttention(CustomOp):
|
||||
|
||||
return output
|
||||
|
||||
def _forward_aiter_fp8(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
max_seqlen: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert (cu_seqlens is not None and max_seqlen is not None) or (
|
||||
cu_seqlens is None and max_seqlen is None
|
||||
), "cu_seqlens and max_seqlen should be both set or both None."
|
||||
|
||||
bsz, q_len = query.size()[:2]
|
||||
kv_len = key.size(1)
|
||||
is_reshaped = query.dim() != 4
|
||||
query, key, value = self.view_qkv_to_4d(query, key, value, bsz, q_len, kv_len)
|
||||
|
||||
query, key, value = self._quantize_qkv_fp8(query, key, value)
|
||||
|
||||
output = vit_aiter_fp8_attn_wrapper(
|
||||
q=query,
|
||||
k=key,
|
||||
v=value,
|
||||
q_descale=self._fp8_q_scale,
|
||||
k_descale=self._fp8_k_scale,
|
||||
v_descale=self._fp8_v_scale,
|
||||
batch_size=bsz,
|
||||
output_dtype=self.dtype,
|
||||
scale=self.scale,
|
||||
cu_seqlens=cu_seqlens,
|
||||
max_seqlen=max_seqlen,
|
||||
)
|
||||
if output.shape[-1] != self.head_size:
|
||||
output = output[..., : self.head_size].contiguous()
|
||||
if is_reshaped:
|
||||
output = output.reshape(bsz, q_len, -1)
|
||||
return output
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
@@ -719,7 +793,9 @@ class MMEncoderAttention(CustomOp):
|
||||
sequence_lengths: torch.Tensor
|
||||
| None = None, # Only used for FlashInfer CuDNN backend
|
||||
) -> torch.Tensor:
|
||||
if self.is_flash_attn_backend:
|
||||
if self.fp8_enabled and self.attn_backend == AttentionBackendEnum.ROCM_AITER_FA:
|
||||
return self._forward_aiter_fp8(query, key, value, cu_seqlens, max_seqlen)
|
||||
elif self.is_flash_attn_backend:
|
||||
return self._forward_fa(query, key, value, cu_seqlens, max_seqlen)
|
||||
elif self.attn_backend == AttentionBackendEnum.TRITON_ATTN:
|
||||
return self._forward_triton(query, key, value, cu_seqlens, max_seqlen)
|
||||
|
||||
@@ -657,7 +657,7 @@ class RocmPlatform(Platform):
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
if rocm_aiter_ops.is_enabled() and on_gfx9():
|
||||
if rocm_aiter_ops.is_mha_enabled() and on_gfx9():
|
||||
logger.info_once("Using AITER Flash Attention backend for ViT model.")
|
||||
return AttentionBackendEnum.ROCM_AITER_FA
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ import einops
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
@@ -114,6 +115,34 @@ def vit_flash_attn_wrapper(
|
||||
)
|
||||
|
||||
|
||||
def vit_aiter_fp8_attn_wrapper(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
q_descale: torch.Tensor,
|
||||
k_descale: torch.Tensor,
|
||||
v_descale: torch.Tensor,
|
||||
batch_size: int,
|
||||
output_dtype: torch.dtype,
|
||||
scale: float | None = None,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
max_seqlen: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return rocm_aiter_ops.fp8_attn_wrapper(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
q_descale,
|
||||
k_descale,
|
||||
v_descale,
|
||||
batch_size,
|
||||
output_dtype,
|
||||
scale,
|
||||
cu_seqlens,
|
||||
max_seqlen,
|
||||
)
|
||||
|
||||
|
||||
def triton_attn_wrapper(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user