[ROCm] Add AITER FP8 ViT encoder attention (#49937)

Signed-off-by: Liuyinfeng01 <yinfeliu@amd.com>
This commit is contained in:
yinfengLiu
2026-07-30 06:22:10 +00:00
committed by GitHub
parent 0028fc8d8d
commit f1e8fd27ae
8 changed files with 563 additions and 45 deletions
+44 -3
View File
@@ -14,6 +14,7 @@ from vllm.model_executor.layers.attention.mm_encoder_attention import (
_FP8_AMAX_HISTORY_LEN,
_FP8_MAX,
)
from vllm.platforms import current_platform
from vllm.utils.flashinfer import (
is_flashinfer_cudnn_fp8_prefill_attn_supported,
)
@@ -29,8 +30,17 @@ NUM_HEADS = 16
HEAD_DIM = 72
def _is_mi3xx() -> bool:
if not current_platform.is_rocm():
return False
from vllm.platforms.rocm import on_mi3xx
return on_mi3xx()
@contextlib.contextmanager
def _build_attention(mm_config):
def _build_attention(mm_config, attn_backend=None):
"""Yield an MMEncoderAttention with the given multimodal config.
The VllmConfig context stays active while the test runs so that
@@ -44,7 +54,13 @@ def _build_attention(mm_config):
)
from vllm.v1.attention.backends.registry import AttentionBackendEnum
if not is_flashinfer_cudnn_fp8_prefill_attn_supported():
if attn_backend is None:
attn_backend = AttentionBackendEnum.FLASHINFER
if (
attn_backend == AttentionBackendEnum.FLASHINFER
and not is_flashinfer_cudnn_fp8_prefill_attn_supported()
):
yield None
return
@@ -56,7 +72,7 @@ def _build_attention(mm_config):
patch(
"vllm.model_executor.layers.attention.mm_encoder_attention"
".get_vit_attn_backend",
return_value=AttentionBackendEnum.FLASHINFER,
return_value=attn_backend,
),
):
attn = MMEncoderAttention(
@@ -169,6 +185,31 @@ def test_static_scales_loaded(_make_static_attention) -> None:
assert not hasattr(attn, "_fp8_q_amax")
@pytest.mark.skipif(
not _is_mi3xx(),
reason="AITER FP8 attention requires MI300/MI350",
)
def test_aiter_static_scales_loaded(tmp_path) -> None:
"""Verify AITER reuses the existing static FP8 scale loading path."""
from vllm.config.multimodal import MultiModalConfig
from vllm.v1.attention.backends.registry import AttentionBackendEnum
scale_file = tmp_path / "aiter_scales.json"
scale_file.write_text(json.dumps({LAYER_0: {"q": 224.0, "k": 198.0, "v": 210.0}}))
mm_config = MultiModalConfig(
mm_encoder_attn_dtype="fp8",
mm_encoder_fp8_scale_path=str(scale_file),
)
with _build_attention(mm_config, AttentionBackendEnum.ROCM_AITER_FA) as attn:
assert attn is not None
assert attn.fp8_enabled
assert not attn._fp8_dynamic_scale
assert attn._fp8_q_scale.item() == 224.0
assert attn._fp8_k_scale.item() == 198.0
assert attn._fp8_v_scale.item() == 210.0
def test_static_scales_missing_layer(tmp_path) -> None:
"""Verify error when requested layer is not in the scale file."""
from vllm.config import VllmConfig, set_current_vllm_config