[Perf] FP8 FlashInfer Attn for ViT (#38065)

Signed-off-by: Zhanda Zhu <zhandazhu@gmail.com>
Co-authored-by: Yubo Gao <ybgao-nvidia@users.noreply.github.com>
This commit is contained in:
Zhanda Zhu
2026-04-27 13:44:15 +08:00
committed by GitHub
co-authored by Yubo Gao
parent 592ae6805c
commit 5d5c776444
18 changed files with 1829 additions and 49 deletions
+9
View File
@@ -136,6 +136,7 @@ from .utils import (
maybe_prefix,
)
from .vision import (
get_fp8_padded_hidden_size,
get_vit_attn_backend,
is_vit_use_data_parallel,
run_dp_sharded_mrope_vision_model,
@@ -562,6 +563,13 @@ class Qwen3_VisionTransformer(nn.Module):
norm_layer = partial(nn.LayerNorm, eps=norm_eps)
head_dim = self.hidden_size // self.num_heads
# FP8 attention: Q/K/V become independent contiguous tensors
# after quantization, so cu_seqlens uses uniform stride (no 3x V).
self.fp8_padded_hidden_size = get_fp8_padded_hidden_size(
self.num_heads, head_dim
)
self.rotary_pos_emb = get_rope(
head_size=head_dim,
max_position=8192,
@@ -776,6 +784,7 @@ class Qwen3_VisionTransformer(nn.Module):
self.hidden_size,
self.tp_size,
device,
fp8_padded_hidden_size=self.fp8_padded_hidden_size,
)
return metadata
+32 -28
View File
@@ -10,7 +10,7 @@ from typing import Final, Generic, Literal, Protocol, TypeAlias, TypeVar
import torch
from transformers import PretrainedConfig
from vllm.config import MultiModalConfig, VllmConfig, get_current_vllm_config
from vllm.config import MultiModalConfig, get_current_vllm_config_or_none
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
@@ -18,6 +18,7 @@ from vllm.distributed import (
)
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.math_utils import round_up
from vllm.v1.attention.backends.registry import AttentionBackendEnum
logger = init_logger(__name__)
@@ -102,45 +103,48 @@ def get_vit_attn_backend(
"""
Get the attention backend for Vision Transformer.
"""
try:
vllm_config: VllmConfig = get_current_vllm_config()
model_config = vllm_config.model_config
multimodal_config: MultiModalConfig | None = (
model_config.multimodal_config if model_config is not None else None
)
except (AssertionError, AttributeError):
multimodal_config = None
mm_cfg = get_multimodal_config()
attn_backend_override = (
multimodal_config.mm_encoder_attn_backend
if multimodal_config is not None
else None
mm_cfg.mm_encoder_attn_backend if mm_cfg is not None else None
)
attn_backend = _get_vit_attn_backend(
return _get_vit_attn_backend(
head_size,
dtype,
attn_backend_override=attn_backend_override,
)
return attn_backend
def get_multimodal_config() -> MultiModalConfig | None:
"""Return the current ``MultiModalConfig``, or ``None`` when no engine
config context is active (e.g., during unit tests) or when the current
``model_config`` does not carry a ``multimodal_config`` (e.g., minimal
stubs used in tests)."""
vllm_config = get_current_vllm_config_or_none()
if vllm_config is None or vllm_config.model_config is None:
return None
return getattr(vllm_config.model_config, "multimodal_config", None)
def get_fp8_padded_hidden_size(num_heads: int, head_dim: int) -> int | None:
"""Return the padded hidden size for FP8 ViT encoder attention, or
``None`` when FP8 is not enabled.
cuDNN FP8 prefill attention requires ``head_dim`` to be a multiple of
16. For non-aligned ``head_dim`` (e.g. 72), Q/K/V are padded to the
nearest multiple of 16.
"""
mm_cfg = get_multimodal_config()
if mm_cfg is None or mm_cfg.mm_encoder_attn_dtype != "fp8":
return None
return num_heads * round_up(head_dim, 16)
def is_vit_use_data_parallel():
"""
Get the tensor parallel type for Vision Transformer.
"""
try:
vllm_config: VllmConfig = get_current_vllm_config()
model_config = vllm_config.model_config
multimodal_config: MultiModalConfig | None = (
model_config.multimodal_config if model_config is not None else None
)
except (AssertionError, AttributeError):
multimodal_config = None
mm_encoder_tp_mode = (
multimodal_config.mm_encoder_tp_mode if multimodal_config is not None else None
)
return mm_encoder_tp_mode == "data"
mm_cfg = get_multimodal_config()
return mm_cfg is not None and mm_cfg.mm_encoder_tp_mode == "data"
VisionFeatureSelectStrategyStr = Literal["class", "default", "full"]