[Perf] FP8 FlashInfer Attn for ViT (#38065)
Signed-off-by: Zhanda Zhu <zhandazhu@gmail.com> Co-authored-by: Yubo Gao <ybgao-nvidia@users.noreply.github.com>
This commit is contained in:
@@ -136,6 +136,7 @@ from .utils import (
|
||||
maybe_prefix,
|
||||
)
|
||||
from .vision import (
|
||||
get_fp8_padded_hidden_size,
|
||||
get_vit_attn_backend,
|
||||
is_vit_use_data_parallel,
|
||||
run_dp_sharded_mrope_vision_model,
|
||||
@@ -562,6 +563,13 @@ class Qwen3_VisionTransformer(nn.Module):
|
||||
|
||||
norm_layer = partial(nn.LayerNorm, eps=norm_eps)
|
||||
head_dim = self.hidden_size // self.num_heads
|
||||
|
||||
# FP8 attention: Q/K/V become independent contiguous tensors
|
||||
# after quantization, so cu_seqlens uses uniform stride (no 3x V).
|
||||
self.fp8_padded_hidden_size = get_fp8_padded_hidden_size(
|
||||
self.num_heads, head_dim
|
||||
)
|
||||
|
||||
self.rotary_pos_emb = get_rope(
|
||||
head_size=head_dim,
|
||||
max_position=8192,
|
||||
@@ -776,6 +784,7 @@ class Qwen3_VisionTransformer(nn.Module):
|
||||
self.hidden_size,
|
||||
self.tp_size,
|
||||
device,
|
||||
fp8_padded_hidden_size=self.fp8_padded_hidden_size,
|
||||
)
|
||||
|
||||
return metadata
|
||||
|
||||
@@ -10,7 +10,7 @@ from typing import Final, Generic, Literal, Protocol, TypeAlias, TypeVar
|
||||
import torch
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.config import MultiModalConfig, VllmConfig, get_current_vllm_config
|
||||
from vllm.config import MultiModalConfig, get_current_vllm_config_or_none
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
@@ -18,6 +18,7 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import round_up
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -102,45 +103,48 @@ def get_vit_attn_backend(
|
||||
"""
|
||||
Get the attention backend for Vision Transformer.
|
||||
"""
|
||||
try:
|
||||
vllm_config: VllmConfig = get_current_vllm_config()
|
||||
model_config = vllm_config.model_config
|
||||
multimodal_config: MultiModalConfig | None = (
|
||||
model_config.multimodal_config if model_config is not None else None
|
||||
)
|
||||
except (AssertionError, AttributeError):
|
||||
multimodal_config = None
|
||||
|
||||
mm_cfg = get_multimodal_config()
|
||||
attn_backend_override = (
|
||||
multimodal_config.mm_encoder_attn_backend
|
||||
if multimodal_config is not None
|
||||
else None
|
||||
mm_cfg.mm_encoder_attn_backend if mm_cfg is not None else None
|
||||
)
|
||||
attn_backend = _get_vit_attn_backend(
|
||||
return _get_vit_attn_backend(
|
||||
head_size,
|
||||
dtype,
|
||||
attn_backend_override=attn_backend_override,
|
||||
)
|
||||
return attn_backend
|
||||
|
||||
|
||||
def get_multimodal_config() -> MultiModalConfig | None:
|
||||
"""Return the current ``MultiModalConfig``, or ``None`` when no engine
|
||||
config context is active (e.g., during unit tests) or when the current
|
||||
``model_config`` does not carry a ``multimodal_config`` (e.g., minimal
|
||||
stubs used in tests)."""
|
||||
vllm_config = get_current_vllm_config_or_none()
|
||||
if vllm_config is None or vllm_config.model_config is None:
|
||||
return None
|
||||
return getattr(vllm_config.model_config, "multimodal_config", None)
|
||||
|
||||
|
||||
def get_fp8_padded_hidden_size(num_heads: int, head_dim: int) -> int | None:
|
||||
"""Return the padded hidden size for FP8 ViT encoder attention, or
|
||||
``None`` when FP8 is not enabled.
|
||||
|
||||
cuDNN FP8 prefill attention requires ``head_dim`` to be a multiple of
|
||||
16. For non-aligned ``head_dim`` (e.g. 72), Q/K/V are padded to the
|
||||
nearest multiple of 16.
|
||||
"""
|
||||
mm_cfg = get_multimodal_config()
|
||||
if mm_cfg is None or mm_cfg.mm_encoder_attn_dtype != "fp8":
|
||||
return None
|
||||
return num_heads * round_up(head_dim, 16)
|
||||
|
||||
|
||||
def is_vit_use_data_parallel():
|
||||
"""
|
||||
Get the tensor parallel type for Vision Transformer.
|
||||
"""
|
||||
try:
|
||||
vllm_config: VllmConfig = get_current_vllm_config()
|
||||
model_config = vllm_config.model_config
|
||||
multimodal_config: MultiModalConfig | None = (
|
||||
model_config.multimodal_config if model_config is not None else None
|
||||
)
|
||||
except (AssertionError, AttributeError):
|
||||
multimodal_config = None
|
||||
|
||||
mm_encoder_tp_mode = (
|
||||
multimodal_config.mm_encoder_tp_mode if multimodal_config is not None else None
|
||||
)
|
||||
return mm_encoder_tp_mode == "data"
|
||||
mm_cfg = get_multimodal_config()
|
||||
return mm_cfg is not None and mm_cfg.mm_encoder_tp_mode == "data"
|
||||
|
||||
|
||||
VisionFeatureSelectStrategyStr = Literal["class", "default", "full"]
|
||||
|
||||
Reference in New Issue
Block a user