[Perf] FP8 FlashInfer Attn for ViT (#38065)

Signed-off-by: Zhanda Zhu <zhandazhu@gmail.com>
Co-authored-by: Yubo Gao <ybgao-nvidia@users.noreply.github.com>
This commit is contained in:
Zhanda Zhu
2026-04-27 13:44:15 +08:00
committed by GitHub
co-authored by Yubo Gao
parent 592ae6805c
commit 5d5c776444
18 changed files with 1829 additions and 49 deletions
+279
View File
@@ -0,0 +1,279 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the full FP8 ViT attention path (quantize -> cuDNN -> un-pad)."""
import contextlib
import pytest
import torch
from vllm.triton_utils import HAS_TRITON
from vllm.utils.flashinfer import (
is_flashinfer_cudnn_fp8_prefill_attn_supported,
)
from vllm.v1.attention.backends.registry import AttentionBackendEnum
def _has_flashinfer_cudnn() -> bool:
"""Check if FlashInfer cuDNN backend is available."""
try:
from flashinfer.prefill import (
cudnn_batch_prefill_with_kv_cache, # noqa: F401
)
return True
except ImportError:
return False
HEAD_DIMS = [72, 80]
SEQ_LENS = [256]
NUM_HEADS = [16]
@pytest.fixture
def _fp8_attention():
"""Create FP8-enabled MMEncoderAttention via config."""
from types import SimpleNamespace
from unittest.mock import patch
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.config.multimodal import MultiModalConfig
if not is_flashinfer_cudnn_fp8_prefill_attn_supported():
pytest.skip("FlashInfer cuDNN FP8 prefill attention not supported")
mm_config = MultiModalConfig(mm_encoder_attn_dtype="fp8")
vllm_config = VllmConfig()
vllm_config.model_config = SimpleNamespace(multimodal_config=mm_config)
# MMEncoderAttention reads torch.get_default_dtype() during init
# to determine the output dtype. In real model loading this is bf16.
old_dtype = torch.get_default_dtype()
torch.set_default_dtype(torch.bfloat16)
with (
set_current_vllm_config(vllm_config),
patch(
"vllm.model_executor.layers.attention.mm_encoder_attention"
".get_vit_attn_backend",
return_value=AttentionBackendEnum.FLASHINFER,
),
):
yield
torch.set_default_dtype(old_dtype)
def _build_cu_seqlens_and_meta(
seq_len: int,
num_heads: int,
head_dim: int,
fp8_padded_hidden_size: int | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Build cu_seqlens, max_seqlen, sequence_lengths for a single sequence."""
import numpy as np
from vllm.model_executor.layers.attention.mm_encoder_attention import (
MMEncoderAttention,
)
cu_seqlens_np = np.array([0, seq_len], dtype=np.int32)
sequence_lengths = MMEncoderAttention.maybe_compute_seq_lens(
AttentionBackendEnum.FLASHINFER,
cu_seqlens_np,
torch.device("cuda"),
)
max_seqlen = torch.tensor(
MMEncoderAttention.compute_max_seqlen(
AttentionBackendEnum.FLASHINFER, cu_seqlens_np
),
dtype=torch.int32,
)
cu_seqlens = MMEncoderAttention.maybe_recompute_cu_seqlens(
AttentionBackendEnum.FLASHINFER,
cu_seqlens_np,
num_heads * head_dim,
1, # tp_size
torch.device("cuda"),
fp8_padded_hidden_size=fp8_padded_hidden_size,
)
return cu_seqlens, max_seqlen, sequence_lengths
@pytest.mark.skipif(
not (HAS_TRITON and _has_flashinfer_cudnn()),
reason="Triton and FlashInfer cuDNN required",
)
@pytest.mark.parametrize("head_dim", HEAD_DIMS)
@pytest.mark.parametrize("seq_len", SEQ_LENS)
@pytest.mark.parametrize("num_heads", NUM_HEADS)
def test_fp8_attn_output_shape(
head_dim: int,
seq_len: int,
num_heads: int,
_fp8_attention,
) -> None:
"""Verify FP8 attention produces correct output shape after un-padding."""
from vllm.model_executor.layers.attention.mm_encoder_attention import (
MMEncoderAttention,
)
from vllm.utils.math_utils import round_up
attn = None
with contextlib.suppress(ValueError, ImportError):
attn = MMEncoderAttention(
num_heads=num_heads,
head_size=head_dim,
prefix="visual.blocks.0.attn",
).to("cuda")
if attn is None or not attn.fp8_enabled:
pytest.skip("FP8 MMEncoderAttention not available")
assert attn is not None # mypy narrowing
# FP8 always needs fp8_padded_hidden_size for correct cu_seqlens
fp8_padded_hidden_size = num_heads * round_up(head_dim, 16)
cu_seqlens, max_seqlen, sequence_lengths = _build_cu_seqlens_and_meta(
seq_len, num_heads, head_dim, fp8_padded_hidden_size=fp8_padded_hidden_size
)
q = torch.randn(
seq_len,
num_heads,
head_dim,
device="cuda",
dtype=torch.bfloat16,
)
k = torch.randn_like(q)
v = torch.randn_like(q)
output = attn._forward_flashinfer(q, k, v, cu_seqlens, max_seqlen, sequence_lengths)
# Output should have original head_dim (un-padded)
assert output.shape[-1] == head_dim
assert output.dtype == torch.bfloat16
@pytest.mark.skipif(
not (HAS_TRITON and _has_flashinfer_cudnn()),
reason="Triton and FlashInfer cuDNN required",
)
@pytest.mark.parametrize("head_dim", HEAD_DIMS)
@pytest.mark.parametrize("seq_len", SEQ_LENS)
@pytest.mark.parametrize("num_heads", NUM_HEADS)
def test_fp8_vs_bf16_close(
head_dim: int, seq_len: int, num_heads: int, _fp8_attention
) -> None:
"""FP8 attention output should be reasonably close to BF16 baseline."""
from vllm.model_executor.layers.attention.mm_encoder_attention import (
MMEncoderAttention,
)
from vllm.utils.math_utils import round_up
torch.manual_seed(42)
q = torch.randn(
1,
seq_len,
num_heads,
head_dim,
device="cuda",
dtype=torch.bfloat16,
)
k = torch.randn_like(q)
v = torch.randn_like(q)
# FP8 path
attn_fp8 = None
with contextlib.suppress(ValueError, ImportError):
attn_fp8 = MMEncoderAttention(
num_heads=num_heads,
head_size=head_dim,
prefix="visual.blocks.0.attn",
).to("cuda")
if attn_fp8 is None or not attn_fp8.fp8_enabled:
pytest.skip("FP8 MMEncoderAttention not available")
assert attn_fp8 is not None # mypy narrowing
fp8_padded_hidden_size = num_heads * round_up(head_dim, 16)
cu_seqlens, max_seqlen, seq_lengths = _build_cu_seqlens_and_meta(
seq_len,
num_heads,
head_dim,
fp8_padded_hidden_size=fp8_padded_hidden_size,
)
out_fp8 = attn_fp8._forward_flashinfer(
q.clone(),
k.clone(),
v.clone(),
cu_seqlens,
max_seqlen,
seq_lengths,
)
# BF16 baseline (create non-FP8 attention by using scale=attn_fp8.scale
# and calling the wrapper directly without FP8 quantization)
from vllm.model_executor.layers.attention.mm_encoder_attention import (
_get_flashinfer_workspace_buffer,
)
from vllm.v1.attention.ops.vit_attn_wrappers import (
vit_flashinfer_wrapper,
)
out_bf16 = vit_flashinfer_wrapper(
q=q.clone(),
k=k.clone(),
v=v.clone(),
scale=attn_fp8.scale,
workspace_buffer=_get_flashinfer_workspace_buffer(),
cu_seqlens=cu_seqlens,
max_seqlen=max_seqlen,
sequence_lengths=seq_lengths,
)
out_fp8_f = out_fp8.float()
out_bf16_f = out_bf16.float()
abs_diff = (out_fp8_f - out_bf16_f).abs()
abs_diff_flat = abs_diff.flatten()
# Relative diff (avoid division by zero)
denom = out_bf16_f.abs().clamp(min=1e-6)
rel_diff_flat = (abs_diff / denom).flatten()
cosine_sim = torch.nn.functional.cosine_similarity(
out_fp8_f.flatten().unsqueeze(0),
out_bf16_f.flatten().unsqueeze(0),
).item()
pcts = [50, 90, 95, 99, 99.9]
abs_pct = {p: torch.quantile(abs_diff_flat, p / 100).item() for p in pcts}
rel_pct = {p: torch.quantile(rel_diff_flat, p / 100).item() for p in pcts}
print(f"\nFP8 vs BF16 (head_dim={head_dim}, seq_len={seq_len}):")
print(f" cosine_sim={cosine_sim:.6f}")
print(
f" abs_diff: max={abs_diff_flat.max().item():.6f}, "
f"mean={abs_diff_flat.mean().item():.6f}, "
+ ", ".join(f"p{p}={abs_pct[p]:.6f}" for p in pcts)
)
print(
f" rel_diff: max={rel_diff_flat.max().item():.6f}, "
f"mean={rel_diff_flat.mean().item():.6f}, "
+ ", ".join(f"p{p}={rel_pct[p]:.6f}" for p in pcts)
)
assert abs_diff_flat.max().item() < 0.3, (
f"FP8 vs BF16 max abs diff too large: {abs_diff_flat.max().item()}"
)
assert abs_diff_flat.mean().item() < 0.03, (
f"FP8 vs BF16 mean abs diff too large: {abs_diff_flat.mean().item()}"
)
assert cosine_sim > 0.99, f"Cosine similarity too low: {cosine_sim:.6f}"
+124
View File
@@ -0,0 +1,124 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the stride-aware FP8 quantization kernel with head_dim padding."""
import pytest
import torch
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
if HAS_TRITON:
from vllm.kernels.triton.qkv_padded_fp8_quant import (
quantize_fp8_pad_head_dim_triton,
)
HEAD_DIMS = [72, 80, 128]
SEQ_LENS = [64, 256]
NUM_HEADS = [16]
SCALES = [0.01, 0.1, 1.0]
def _naive_fp8_quantize(
tensor: torch.Tensor, scale: torch.Tensor, skip_scale: bool
) -> torch.Tensor:
"""Reference FP8 quantization in PyTorch."""
fp8_dtype = current_platform.fp8_dtype()
fp8_max = torch.finfo(fp8_dtype).max
fp8_min = -fp8_max
x = tensor.float()
if not skip_scale:
x = x / scale.item()
x = x.clamp(fp8_min, fp8_max)
return x.to(fp8_dtype)
@pytest.mark.skipif(not HAS_TRITON, reason="Triton not available")
@pytest.mark.parametrize("head_dim", HEAD_DIMS)
@pytest.mark.parametrize("seq_len", SEQ_LENS)
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@pytest.mark.parametrize("scale_val", SCALES)
def test_quantize_contiguous(
head_dim: int, seq_len: int, num_heads: int, scale_val: float
) -> None:
"""Test quantization of contiguous 3D tensors."""
torch.manual_seed(42)
tensor = torch.randn(
seq_len, num_heads, head_dim, device="cuda", dtype=torch.bfloat16
)
scale = torch.tensor([scale_val], dtype=torch.float32, device="cuda").view(
1, 1, 1, 1
)
result = quantize_fp8_pad_head_dim_triton(tensor, scale)
padded_dim = (head_dim + 15) // 16 * 16
assert result.shape == (seq_len, num_heads, padded_dim)
assert result.is_contiguous()
assert result.dtype == current_platform.fp8_dtype()
# Compare unpadded portion against reference
ref = _naive_fp8_quantize(tensor, scale, skip_scale=False)
torch.testing.assert_close(result[:, :, :head_dim].float(), ref.float())
# Padded region should be zero
if padded_dim > head_dim:
assert (result[:, :, head_dim:].float() == 0).all()
@pytest.mark.skipif(not HAS_TRITON, reason="Triton not available")
@pytest.mark.parametrize("head_dim", [72, 80])
def test_quantize_non_contiguous(head_dim: int) -> None:
"""Test quantization from non-contiguous QKV views (interleaved buffer)."""
seq_len, num_heads = 64, 16
# Simulate interleaved QKV buffer: shape (seq_len, 3 * num_heads, head_dim)
qkv = torch.randn(
seq_len, 3 * num_heads, head_dim, device="cuda", dtype=torch.bfloat16
)
# Q is every 3rd head slice - non-contiguous view
q = qkv[:, 0::3, :]
assert not q.is_contiguous()
scale = torch.tensor([0.1], dtype=torch.float32, device="cuda").view(1, 1, 1, 1)
result = quantize_fp8_pad_head_dim_triton(q, scale)
padded_dim = (head_dim + 15) // 16 * 16
assert result.shape == (seq_len, num_heads, padded_dim)
assert result.is_contiguous()
# Compare against contiguous reference
ref = _naive_fp8_quantize(q.contiguous(), scale, skip_scale=False)
torch.testing.assert_close(result[:, :, :head_dim].float(), ref.float())
@pytest.mark.skipif(not HAS_TRITON, reason="Triton not available")
def test_skip_scale() -> None:
"""Test skip_scale=True produces cast-only output (no division)."""
seq_len, num_heads, head_dim = 32, 8, 80
tensor = torch.randn(
seq_len, num_heads, head_dim, device="cuda", dtype=torch.bfloat16
)
scale = torch.tensor([0.5], dtype=torch.float32, device="cuda").view(1, 1, 1, 1)
result_skip = quantize_fp8_pad_head_dim_triton(tensor, scale, skip_scale=True)
result_noskip = quantize_fp8_pad_head_dim_triton(tensor, scale, skip_scale=False)
# skip_scale should just cast, not divide
ref_cast = _naive_fp8_quantize(tensor, scale, skip_scale=True)
torch.testing.assert_close(result_skip[:, :, :head_dim].float(), ref_cast.float())
# With scale != 1.0, skip and no-skip should differ
assert not torch.equal(result_skip.float(), result_noskip.float())
@pytest.mark.skipif(not HAS_TRITON, reason="Triton not available")
def test_4d_input() -> None:
"""Test that 4D input (B, S, H, D) is handled correctly."""
B, S, H, D = 2, 32, 8, 72
tensor = torch.randn(B, S, H, D, device="cuda", dtype=torch.bfloat16)
scale = torch.tensor([0.1], dtype=torch.float32, device="cuda").view(1, 1, 1, 1)
result = quantize_fp8_pad_head_dim_triton(tensor, scale)
padded_dim = (D + 15) // 16 * 16
assert result.shape == (B, S, H, padded_dim)
+251
View File
@@ -0,0 +1,251 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for FP8 scaling (dynamic and static) in MMEncoderAttention."""
import contextlib
import json
from types import SimpleNamespace
from unittest.mock import patch
import pytest
import torch
from vllm.model_executor.layers.attention.mm_encoder_attention import (
_FP8_AMAX_HISTORY_LEN,
_FP8_MAX,
)
from vllm.utils.flashinfer import (
is_flashinfer_cudnn_fp8_prefill_attn_supported,
)
LAYER_0 = "visual.blocks.0.attn.attn"
LAYER_1 = "visual.blocks.1.attn.attn"
NUM_HEADS = 16
HEAD_DIM = 72
@contextlib.contextmanager
def _build_attention(mm_config):
"""Yield an MMEncoderAttention with the given multimodal config.
The VllmConfig context stays active while the test runs so that
``get_multimodal_config()`` calls during the forward path resolve. Also
invokes ``process_weights_after_loading`` to simulate the model loader's
auto-scan. Yields ``None`` if FlashInfer cuDNN is not available.
"""
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.attention.mm_encoder_attention import (
MMEncoderAttention,
)
from vllm.v1.attention.backends.registry import AttentionBackendEnum
if not is_flashinfer_cudnn_fp8_prefill_attn_supported():
yield None
return
vllm_config = VllmConfig()
vllm_config.model_config = SimpleNamespace(multimodal_config=mm_config)
with (
set_current_vllm_config(vllm_config),
patch(
"vllm.model_executor.layers.attention.mm_encoder_attention"
".get_vit_attn_backend",
return_value=AttentionBackendEnum.FLASHINFER,
),
):
attn = MMEncoderAttention(
num_heads=NUM_HEADS,
head_size=HEAD_DIM,
prefix=LAYER_0,
)
attn.process_weights_after_loading(torch.bfloat16)
yield attn
@pytest.fixture
def _make_attention():
"""Create an MMEncoderAttention with dynamic FP8 scaling."""
from vllm.config.multimodal import MultiModalConfig
with _build_attention(MultiModalConfig(mm_encoder_attn_dtype="fp8")) as attn:
yield attn
@pytest.fixture
def _make_static_attention(tmp_path):
"""Create an MMEncoderAttention with static FP8 scales from a file."""
from vllm.config.multimodal import MultiModalConfig
scale_file = tmp_path / "scales.json"
scale_file.write_text(
json.dumps(
{
LAYER_0: {"q": 224.0, "k": 198.0, "v": 210.0},
LAYER_1: {"q": 100.0, "k": 110.0, "v": 120.0},
}
)
)
with _build_attention(
MultiModalConfig(
mm_encoder_attn_dtype="fp8",
mm_encoder_fp8_scale_path=str(scale_file),
)
) as attn:
yield attn
def test_dynamic_scaling_updates_scales(_make_attention) -> None:
"""Verify that _record_amax_and_update_scales updates scale buffers."""
attn = _make_attention
if attn is None or not attn.fp8_enabled:
pytest.skip("FP8 attention not available (FlashInfer backend required)")
attn = attn.to("cuda")
S, H, D = 32, NUM_HEADS, HEAD_DIM
q = torch.full((S, H, D), 2.0, device="cuda", dtype=torch.bfloat16)
k = torch.full((S, H, D), 3.0, device="cuda", dtype=torch.bfloat16)
v = torch.full((S, H, D), 4.0, device="cuda", dtype=torch.bfloat16)
attn._record_amax_and_update_scales(q, k, v)
expected_q_scale = 2.0 / _FP8_MAX
expected_k_scale = 3.0 / _FP8_MAX
expected_v_scale = 4.0 / _FP8_MAX
torch.testing.assert_close(attn._fp8_q_scale.item(), expected_q_scale)
torch.testing.assert_close(attn._fp8_k_scale.item(), expected_k_scale)
torch.testing.assert_close(attn._fp8_v_scale.item(), expected_v_scale)
def test_circular_buffer_wraps(_make_attention) -> None:
"""Verify the amax circular buffer wraps at HISTORY_LEN."""
attn = _make_attention
if attn is None or not attn.fp8_enabled:
pytest.skip("FP8 attention not available (FlashInfer backend required)")
attn = attn.to("cuda")
S, H, D = 16, NUM_HEADS, HEAD_DIM
for i in range(_FP8_AMAX_HISTORY_LEN + 2):
mag = float(i + 1)
q = torch.full((S, H, D), mag, device="cuda", dtype=torch.bfloat16)
k = torch.full((S, H, D), mag, device="cuda", dtype=torch.bfloat16)
v = torch.full((S, H, D), mag, device="cuda", dtype=torch.bfloat16)
attn._record_amax_and_update_scales(q, k, v)
assert attn._fp8_amax_pos == 2
expected_max = float(_FP8_AMAX_HISTORY_LEN + 2)
expected_scale = expected_max / _FP8_MAX
torch.testing.assert_close(attn._fp8_q_scale.item(), expected_scale)
def test_static_scales_loaded(_make_static_attention) -> None:
"""Verify static scales are loaded from the JSON file."""
attn = _make_static_attention
if attn is None or not attn.fp8_enabled:
pytest.skip("FP8 attention not available (FlashInfer backend required)")
assert attn.fp8_enabled
assert not attn._fp8_dynamic_scale
# Layer 0 scales (the layer this attention was created with).
assert attn._fp8_q_scale.item() == 224.0
assert attn._fp8_k_scale.item() == 198.0
assert attn._fp8_v_scale.item() == 210.0
assert not attn.skip_scale_q
assert not attn.skip_scale_k
assert not attn.skip_scale_v
# No amax history buffers for static scaling.
assert not hasattr(attn, "_fp8_q_amax")
def test_static_scales_missing_layer(tmp_path) -> None:
"""Verify error when requested layer is not in the scale file."""
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.config.multimodal import MultiModalConfig
from vllm.v1.attention.backends.registry import AttentionBackendEnum
if not is_flashinfer_cudnn_fp8_prefill_attn_supported():
pytest.skip("FlashInfer cuDNN not available")
scale_file = tmp_path / "wrong_layer.json"
scale_file.write_text(
json.dumps({"visual.blocks.99.attn": {"q": 1.0, "k": 1.0, "v": 1.0}})
)
mm_config = MultiModalConfig(
mm_encoder_attn_dtype="fp8",
mm_encoder_fp8_scale_path=str(scale_file),
)
vllm_config = VllmConfig()
vllm_config.model_config = SimpleNamespace(multimodal_config=mm_config)
from vllm.model_executor.layers.attention.mm_encoder_attention import (
MMEncoderAttention,
)
with (
set_current_vllm_config(vllm_config),
patch(
"vllm.model_executor.layers.attention.mm_encoder_attention"
".get_vit_attn_backend",
return_value=AttentionBackendEnum.FLASHINFER,
),
):
attn = MMEncoderAttention(
num_heads=NUM_HEADS,
head_size=HEAD_DIM,
prefix=LAYER_0,
)
with pytest.raises(ValueError, match="scales not found for layer"):
attn.process_weights_after_loading(torch.bfloat16)
def test_dynamic_scales_auto_save(tmp_path) -> None:
"""Verify scales are saved to disk after the amax buffer fills."""
import vllm.model_executor.layers.attention.mm_encoder_attention as _mod
from vllm.config.multimodal import MultiModalConfig
if not is_flashinfer_cudnn_fp8_prefill_attn_supported():
pytest.skip("FlashInfer cuDNN not available")
# Reset module-level state between runs (other tests may have left
# state behind after triggering a save).
_mod._fp8_scale_save_path = None
_mod._fp8_saved_scale_refs.clear()
save_file = tmp_path / "auto_scales.json"
with _build_attention(
MultiModalConfig(
mm_encoder_attn_dtype="fp8",
mm_encoder_fp8_scale_save_path=str(save_file),
)
) as attn:
if attn is None or not attn.fp8_enabled:
pytest.skip("FP8 attention not available")
attn = attn.to("cuda")
S, H, D = 16, NUM_HEADS, HEAD_DIM
# Run exactly _FP8_AMAX_HISTORY_LEN forward passes.
for i in range(_FP8_AMAX_HISTORY_LEN):
mag = float(i + 1)
q = torch.full((S, H, D), mag, device="cuda", dtype=torch.bfloat16)
k = torch.full((S, H, D), mag * 0.5, device="cuda", dtype=torch.bfloat16)
v = torch.full((S, H, D), mag * 0.3, device="cuda", dtype=torch.bfloat16)
attn._record_amax_and_update_scales(q, k, v)
# File should have been written on the 16th call (buffer wrap).
assert save_file.is_file(), "Scale file was not saved"
scales = json.loads(save_file.read_text())
assert LAYER_0 in scales
assert set(scales[LAYER_0].keys()) == {"q", "k", "v"}
for val in scales[LAYER_0].values():
assert isinstance(val, float) and val > 0
# Path is cleared after the one-shot save fires.
assert _mod._fp8_scale_save_path is None