[Perf] FP8 FlashInfer Attn for ViT (#38065)

Signed-off-by: Zhanda Zhu <zhandazhu@gmail.com>
Co-authored-by: Yubo Gao <ybgao-nvidia@users.noreply.github.com>
This commit is contained in:
Zhanda Zhu
2026-04-27 13:44:15 +08:00
committed by GitHub
co-authored by Yubo Gao
parent 592ae6805c
commit 5d5c776444
18 changed files with 1829 additions and 49 deletions
+12
View File
@@ -326,6 +326,10 @@ class ModelConfig:
mm_encoder_only: InitVar[bool | None] = None
mm_encoder_tp_mode: InitVar[MMEncoderTPMode | None] = None
mm_encoder_attn_backend: InitVar[AttentionBackendEnum | str | None] = None
mm_encoder_attn_dtype: InitVar[str | None] = None
mm_encoder_fp8_scale_path: InitVar[str | None] = None
mm_encoder_fp8_scale_save_path: InitVar[str | None] = None
mm_encoder_fp8_scale_save_margin: InitVar[float | None] = None
interleave_mm_strings: InitVar[bool | None] = None
skip_mm_profiling: InitVar[bool | None] = None
video_pruning_rate: InitVar[float | None] = None
@@ -447,6 +451,10 @@ class ModelConfig:
mm_encoder_only: bool | None,
mm_encoder_tp_mode: MMEncoderTPMode | None,
mm_encoder_attn_backend: AttentionBackendEnum | str | None,
mm_encoder_attn_dtype: str | None,
mm_encoder_fp8_scale_path: str | None,
mm_encoder_fp8_scale_save_path: str | None,
mm_encoder_fp8_scale_save_margin: float | None,
interleave_mm_strings: bool | None,
skip_mm_profiling: bool | None,
video_pruning_rate: float | None,
@@ -643,6 +651,10 @@ class ModelConfig:
mm_encoder_only=mm_encoder_only,
mm_encoder_tp_mode=mm_encoder_tp_mode,
mm_encoder_attn_backend=mm_encoder_attn_backend,
mm_encoder_attn_dtype=mm_encoder_attn_dtype,
mm_encoder_fp8_scale_path=mm_encoder_fp8_scale_path,
mm_encoder_fp8_scale_save_path=mm_encoder_fp8_scale_save_path,
mm_encoder_fp8_scale_save_margin=mm_encoder_fp8_scale_save_margin,
interleave_mm_strings=interleave_mm_strings,
skip_mm_profiling=skip_mm_profiling,
video_pruning_rate=video_pruning_rate,
+51
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Mapping
from pathlib import Path
from typing import Any, Literal, TypeAlias, TypedDict, final
from pydantic import ConfigDict, Field, field_validator, model_validator
@@ -158,6 +159,24 @@ class MultiModalConfig:
"""Optional override for the multi-modal encoder attention backend when
using vision transformers. Accepts any value from
`vllm.v1.attention.backends.registry.AttentionBackendEnum` (e.g. `FLASH_ATTN`)."""
mm_encoder_attn_dtype: Literal["fp8"] | None = None
"""Optional dtype override for ViT encoder attention. Set to `"fp8"` to
enable FP8 quantization via the FlashInfer cuDNN backend. When set to
`"fp8"` without a scale file, dynamic scaling is used automatically.
See docs/features/quantization/fp8_vit_attn.md for details."""
mm_encoder_fp8_scale_path: str | None = None
"""Path to a JSON file containing per-layer FP8 Q/K/V scales for ViT
encoder attention. When provided (with `mm_encoder_attn_dtype="fp8"`),
static scaling is used. When omitted, dynamic scaling is used."""
mm_encoder_fp8_scale_save_path: str | None = None
"""When set with dynamic FP8 scaling (`mm_encoder_attn_dtype="fp8"`
and no `mm_encoder_fp8_scale_path`), saves the calibrated scales to
this file after the amax history buffer is full. The saved file can
then be used as `mm_encoder_fp8_scale_path` in subsequent runs."""
mm_encoder_fp8_scale_save_margin: float = Field(default=1.5, gt=0.0)
"""Safety margin multiplied onto scales when auto-saving. A value > 1
leaves headroom so that inputs with larger activations than the
calibration set do not overflow FP8 range. Default 1.5."""
interleave_mm_strings: bool = False
"""Enable fully interleaved support for multimodal prompts, while using
--chat-template-content-format=string."""
@@ -233,6 +252,36 @@ class MultiModalConfig:
"'mm_shm_cache_max_object_size_mb' should only be set when "
"'mm_processor_cache_type' is 'shm'."
)
# Validate FP8 scale path combinations.
if self.mm_encoder_attn_dtype != "fp8" and (
self.mm_encoder_fp8_scale_path is not None
or self.mm_encoder_fp8_scale_save_path is not None
):
raise ValueError(
"'mm_encoder_fp8_scale_path' and "
"'mm_encoder_fp8_scale_save_path' require "
"'mm_encoder_attn_dtype' to be 'fp8'."
)
if (
self.mm_encoder_fp8_scale_path is not None
and self.mm_encoder_fp8_scale_save_path is not None
):
raise ValueError(
"'mm_encoder_fp8_scale_save_path' cannot be used with "
"'mm_encoder_fp8_scale_path' (saving requires dynamic scaling)."
)
# Validate file paths exist.
if self.mm_encoder_fp8_scale_path is not None:
scale_path = Path(self.mm_encoder_fp8_scale_path)
if not scale_path.is_file():
raise FileNotFoundError(f"FP8 scale file not found: {scale_path}")
if self.mm_encoder_fp8_scale_save_path is not None:
save_parent = Path(self.mm_encoder_fp8_scale_save_path).parent
if not save_parent.is_dir():
raise FileNotFoundError(
f"Parent directory for FP8 scale save path not found: {save_parent}"
)
return self
def compute_hash(self) -> str:
@@ -252,6 +301,8 @@ class MultiModalConfig:
if self.mm_encoder_attn_backend is not None
else None,
self.mm_encoder_tp_mode,
self.mm_encoder_attn_dtype,
self.mm_encoder_fp8_scale_path,
]
hash_str = safe_hash(str(factors).encode(), usedforsecurity=False).hexdigest()
return hash_str
+28
View File
@@ -542,6 +542,14 @@ class EngineArgs:
mm_encoder_attn_backend: AttentionBackendEnum | str | None = (
MultiModalConfig.mm_encoder_attn_backend
)
mm_encoder_attn_dtype: str | None = MultiModalConfig.mm_encoder_attn_dtype
mm_encoder_fp8_scale_path: str | None = MultiModalConfig.mm_encoder_fp8_scale_path
mm_encoder_fp8_scale_save_path: str | None = (
MultiModalConfig.mm_encoder_fp8_scale_save_path
)
mm_encoder_fp8_scale_save_margin: float = (
MultiModalConfig.mm_encoder_fp8_scale_save_margin
)
io_processor_plugin: str | None = None
renderer_num_workers: int = 1
skip_mm_profiling: bool = MultiModalConfig.skip_mm_profiling
@@ -1179,6 +1187,22 @@ class EngineArgs:
"--mm-encoder-attn-backend",
**multimodal_kwargs["mm_encoder_attn_backend"],
)
multimodal_group.add_argument(
"--mm-encoder-attn-dtype",
**multimodal_kwargs["mm_encoder_attn_dtype"],
)
multimodal_group.add_argument(
"--mm-encoder-fp8-scale-path",
**multimodal_kwargs["mm_encoder_fp8_scale_path"],
)
multimodal_group.add_argument(
"--mm-encoder-fp8-scale-save-path",
**multimodal_kwargs["mm_encoder_fp8_scale_save_path"],
)
multimodal_group.add_argument(
"--mm-encoder-fp8-scale-save-margin",
**multimodal_kwargs["mm_encoder_fp8_scale_save_margin"],
)
multimodal_group.add_argument(
"--interleave-mm-strings", **multimodal_kwargs["interleave_mm_strings"]
)
@@ -1517,6 +1541,10 @@ class EngineArgs:
mm_encoder_only=self.mm_encoder_only,
mm_encoder_tp_mode=self.mm_encoder_tp_mode,
mm_encoder_attn_backend=self.mm_encoder_attn_backend,
mm_encoder_attn_dtype=self.mm_encoder_attn_dtype,
mm_encoder_fp8_scale_path=self.mm_encoder_fp8_scale_path,
mm_encoder_fp8_scale_save_path=self.mm_encoder_fp8_scale_save_path,
mm_encoder_fp8_scale_save_margin=self.mm_encoder_fp8_scale_save_margin,
pooler_config=self.pooler_config,
generation_config=self.generation_config,
override_generation_config=self.override_generation_config,
+3
View File
@@ -0,0 +1,3 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Triton kernel implementations."""
+180
View File
@@ -0,0 +1,180 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Stride-aware FP8 quantization with head_dim padding for ViT attention.
Reads directly from non-contiguous QKV views using 3D strides and pads
head_dim to a multiple of 16 for cuDNN compatibility.
"""
import torch
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
from vllm.model_executor.layers.quantization.utils.quant_utils import (
get_fp8_min_max,
)
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON, tl, triton
from vllm.utils.math_utils import round_up
_FP8_MIN, _FP8_MAX = get_fp8_min_max()
@triton.jit
def _quantize_pad_fp8_kernel(
x_ptr,
y_ptr,
scale_ptr,
stride_xs,
stride_xh,
stride_xd,
stride_ys,
stride_yh,
stride_yd,
num_heads,
n_rows,
n_cols,
n_cols_padded,
fp8_min,
fp8_max,
SKIP_SCALE: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
):
pid_m = tl.program_id(0)
pid_n = tl.program_id(1)
offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
mask_m = offs_m < n_rows
mask_out = mask_m[:, None] & (offs_n[None, :] < n_cols_padded)
mask_in = mask_m[:, None] & (offs_n[None, :] < n_cols)
# Decompose flattened row into (token, head) for 3D stride indexing.
s = offs_m // num_heads
h = offs_m % num_heads
x_ptrs = (
x_ptr
+ s[:, None] * stride_xs
+ h[:, None] * stride_xh
+ offs_n[None, :] * stride_xd
)
x = tl.load(x_ptrs, mask=mask_in, other=0.0).to(tl.float32)
if SKIP_SCALE:
x_q = x
else:
scale = tl.load(scale_ptr)
x_q = x / scale
x_q = tl.clamp(x_q, fp8_min, fp8_max).to(y_ptr.dtype.element_ty)
y_ptrs = (
y_ptr
+ s[:, None] * stride_ys
+ h[:, None] * stride_yh
+ offs_n[None, :] * stride_yd
)
tl.store(y_ptrs, x_q, mask=mask_out)
def _get_fp8_pad_quant_config(padded_head_dim: int) -> tuple[int, int, int]:
block_n = triton.next_power_of_2(padded_head_dim)
block_n = max(16, min(block_n, 128))
block_m = 16
num_warps = 4
return block_m, block_n, num_warps
def quantize_fp8_pad_head_dim_triton(
tensor: torch.Tensor,
scale: torch.Tensor,
skip_scale: bool = False,
block_m: int | None = None,
block_n: int | None = None,
num_warps: int | None = None,
) -> torch.Tensor:
"""Quantize a 3D/4D tensor to FP8, padding head_dim to a multiple of 16.
Reads directly from the input using its 3D strides, so non-contiguous
views (e.g. Q/K/V slices from an interleaved QKV buffer) are handled
without an extra copy. Output is always a fresh contiguous tensor
with shape (S, H, padded_D).
"""
if not HAS_TRITON:
raise RuntimeError("Triton is required to quantize with head_dim padding.")
original_shape = tensor.shape
if tensor.dim() == 4:
tensor = tensor.view(-1, tensor.shape[-2], tensor.shape[-1])
assert tensor.dim() == 3, f"Expected 3D input (S, H, D), got {tensor.dim()}D"
S, H, D = tensor.shape
padded_head_dim = round_up(D, 16)
out_dtype = current_platform.fp8_dtype()
output = torch.empty(
(S, H, padded_head_dim),
device=tensor.device,
dtype=out_dtype,
)
scale_1d = scale.reshape(-1)
n_rows = S * H
if block_m is None or block_n is None or num_warps is None:
block_m, block_n, num_warps = _get_fp8_pad_quant_config(padded_head_dim)
grid = (
triton.cdiv(n_rows, block_m),
triton.cdiv(padded_head_dim, block_n),
)
_quantize_pad_fp8_kernel[grid](
tensor,
output,
scale_1d,
tensor.stride(0),
tensor.stride(1),
tensor.stride(2),
output.stride(0),
output.stride(1),
output.stride(2),
H,
n_rows,
D,
padded_head_dim,
_FP8_MIN,
_FP8_MAX,
SKIP_SCALE=skip_scale,
BLOCK_M=block_m,
BLOCK_N=block_n,
num_warps=num_warps,
)
return output.view((*original_shape[:-1], padded_head_dim))
def quantize_fp8_maybe_pad_head_dim(
tensor: torch.Tensor,
scale: torch.Tensor,
fp8_quant: QuantFP8,
skip_scale: bool = False,
) -> torch.Tensor:
"""Quantize a 3D/4D tensor to FP8, padding head_dim to a multiple of 16
only when needed.
Accepts (S, H, D) or (B, S, H, D) input. Uses ``fp8_quant`` (a
:class:`QuantFP8` CustomOp) when head_dim is already aligned to 16
(no padding); otherwise falls back to a stride-aware Triton kernel
that pads head_dim to a multiple of 16.
"""
head_dim = tensor.shape[-1]
if head_dim % 16 != 0:
return quantize_fp8_pad_head_dim_triton(tensor, scale, skip_scale=skip_scale)
if skip_scale:
return tensor.to(current_platform.fp8_dtype())
# QuantFP8 expects 2D: flatten all dims except (H, D).
orig_shape = tensor.shape
total_tokens = tensor.numel() // (orig_shape[-1] * orig_shape[-2])
tensor_2d = tensor.reshape(total_tokens, -1)
fp8_tensor, _ = fp8_quant(tensor_2d, scale=scale)
return fp8_tensor.reshape(orig_shape)
@@ -1,13 +1,32 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import functools
import json
import numpy as np
import torch
from vllm.config import MultiModalConfig
from vllm.kernels.triton.qkv_padded_fp8_quant import (
quantize_fp8_maybe_pad_head_dim,
)
from vllm.logger import init_logger
from vllm.model_executor.custom_op import CustomOp, maybe_get_oot_by_class
from vllm.model_executor.models.vision import get_vit_attn_backend
from vllm.model_executor.layers.quantization.input_quant_fp8 import (
QuantFP8,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
get_fp8_min_max,
)
from vllm.model_executor.models.vision import (
get_multimodal_config,
get_vit_attn_backend,
)
from vllm.utils.flashinfer import (
is_flashinfer_cudnn_fp8_prefill_attn_supported,
)
from vllm.utils.math_utils import round_up
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
from vllm.v1.attention.backends.registry import AttentionBackendEnum
@@ -20,6 +39,108 @@ from vllm.v1.attention.ops.vit_attn_wrappers import (
logger = init_logger(__name__)
_, _FP8_MAX = get_fp8_min_max()
_FP8_AMAX_HISTORY_LEN = 16
# Module-level state for auto-saving dynamic scales. The save is a one-shot
# triggered by the first layer whose amax buffer wraps. Path and margin are
# captured during layer init (set_current_vllm_config context only lives
# across model init, not forward passes).
_fp8_scale_save_path: str | None = None
_fp8_scale_save_margin: float = MultiModalConfig.mm_encoder_fp8_scale_save_margin
_fp8_saved_scale_refs: dict[str, tuple[torch.Tensor, torch.Tensor, torch.Tensor]] = {}
@functools.cache
def _load_fp8_scales_file(path: str | None) -> dict[str, dict[str, float]]:
"""Load per-layer FP8 Q/K/V scales from a JSON file. Results are cached.
Expected format (keys ``q_scale`` / ``k_scale`` / ``v_scale`` also accepted)::
{
"visual.blocks.0.attn.attn": {"q": 224.0, "k": 198.0, "v": 210.0},
"visual.blocks.1.attn.attn": {"q": 218.0, "k": 195.0, "v": 207.0},
}
To produce such a file, run with ``mm_encoder_fp8_scale_save_path`` set.
"""
if path is None:
return {}
with open(path, encoding="utf-8") as f:
data = json.load(f)
# Handle nested "layers" format
if "layers" in data and isinstance(data["layers"], dict):
data = data["layers"]
scales: dict[str, dict[str, float]] = {}
for layer_name, layer_scales in data.items():
if not isinstance(layer_scales, dict):
continue
q = layer_scales.get("q", layer_scales.get("q_scale"))
k = layer_scales.get("k", layer_scales.get("k_scale"))
v = layer_scales.get("v", layer_scales.get("v_scale"))
if q is not None and k is not None and v is not None:
q_f, k_f, v_f = float(q), float(k), float(v)
if q_f <= 0 or k_f <= 0 or v_f <= 0:
raise ValueError(
f"FP8 scales must be positive, got q={q_f}, "
f"k={k_f}, v={v_f} for layer '{layer_name}'"
)
scales[layer_name] = {"q": q_f, "k": k_f, "v": v_f}
logger.info_once(
"Loaded FP8 attention scales from %s (%d layers)", path, len(scales)
)
return scales
def _maybe_save_fp8_scales(
layer_name: str,
q_scale: torch.Tensor,
k_scale: torch.Tensor,
v_scale: torch.Tensor,
buffer_wrapped: bool,
) -> None:
"""Accumulate a layer's scale tensors; on the first amax buffer wrap,
dump all accumulated scales to ``mm_encoder_fp8_scale_save_path``.
No-op unless auto-save is configured. Tensor references are stored on
every call (no GPU->CPU sync); ``.item()`` is only called at the single
save point to avoid stalling the forward path.
"""
global _fp8_scale_save_path
# Fast path: auto-save either disabled or already finished. Path is
# captured at layer init and cleared once the save fires.
if _fp8_scale_save_path is None:
return
# Stash scale tensor refs (no GPU->CPU sync yet); wait until the amax
# history has seen a full cycle before committing scales to disk.
_fp8_saved_scale_refs[layer_name] = (q_scale, k_scale, v_scale)
if not buffer_wrapped:
return
# Buffer just wrapped for the first time: materialize scales (with
# safety margin) and dump to disk. Clearing _fp8_scale_save_path
# makes this a one-shot across all layers.
path, margin = _fp8_scale_save_path, _fp8_scale_save_margin
scales = {
name: {
"q": q.item() * margin,
"k": k.item() * margin,
"v": v.item() * margin,
}
for name, (q, k, v) in _fp8_saved_scale_refs.items()
}
_fp8_scale_save_path = None
_fp8_saved_scale_refs.clear()
with open(path, "w", encoding="utf-8") as f:
json.dump(scales, f, indent=2)
logger.info("Saved FP8 scales (%d layers) to %s", len(scales), path)
# Batch buckets for cuDNN graph caching.
# Graphs use batch size and max sequence length as cache key.
# This avoids creating a new graph for each unique set of
@@ -148,27 +269,47 @@ class MMEncoderAttention(CustomOp):
hidden_size: int,
tp_size: int,
device: torch.device,
fp8_padded_hidden_size: int | None = None,
) -> torch.Tensor:
if (oot_class := maybe_get_oot_by_class(cls)) is not cls:
return oot_class.maybe_recompute_cu_seqlens( # type: ignore[attr-defined]
attn_backend, cu_seqlens, hidden_size, tp_size, device
attn_backend,
cu_seqlens,
hidden_size,
tp_size,
device,
fp8_padded_hidden_size=fp8_padded_hidden_size,
)
if attn_backend == AttentionBackendEnum.FLASHINFER:
batch_size = len(cu_seqlens) - 1
scale = hidden_size // tp_size
cu_seqlens = cu_seqlens * scale
cu_seqlens_qko = cu_seqlens
cu_seqlens_v = cu_seqlens * 3
if fp8_padded_hidden_size is not None:
# FP8 path: after quantization Q/K/V are each independent
# contiguous tensors with stride H * padded_D per token.
# All sections use the same element stride.
scale = fp8_padded_hidden_size // tp_size
cu_seqlens = cu_seqlens * scale
cu_seqlens_padded = add_padding_to_seqlens(
cu_seqlens, batch_size, cu_seqlens[-1]
)
cu_seqlens = np.concatenate([cu_seqlens_padded, cu_seqlens_padded])
else:
# BF16 path: Q/K/V are non-contiguous views into shared
# buffers. V section has 3x stride from interleaved QKV.
scale = hidden_size // tp_size
cu_seqlens = cu_seqlens * scale
cu_seqlens_qko = add_padding_to_seqlens(
cu_seqlens_qko, batch_size, cu_seqlens_qko[-1]
)
cu_seqlens_v = add_padding_to_seqlens(
cu_seqlens_v, batch_size, cu_seqlens_v[-1]
)
cu_seqlens = np.concatenate([cu_seqlens_qko, cu_seqlens_v])
cu_seqlens_qko = cu_seqlens
cu_seqlens_v = cu_seqlens * 3
cu_seqlens_qko = add_padding_to_seqlens(
cu_seqlens_qko, batch_size, cu_seqlens_qko[-1]
)
cu_seqlens_v = add_padding_to_seqlens(
cu_seqlens_v, batch_size, cu_seqlens_v[-1]
)
cu_seqlens = np.concatenate([cu_seqlens_qko, cu_seqlens_v])
cu_seqlens = torch.from_numpy(cu_seqlens).to(device, non_blocking=True)
return cu_seqlens
@@ -206,6 +347,7 @@ class MMEncoderAttention(CustomOp):
# During model initialization, the default dtype is set as the model
# weight and activation dtype.
dtype = torch.get_default_dtype()
self.dtype = dtype
# Get device-specific vision attention backend.
self.attn_backend = get_vit_attn_backend(
@@ -229,6 +371,113 @@ class MMEncoderAttention(CustomOp):
logger.info_once(f"Using {self.attn_backend} for MMEncoderAttention.")
self._init_fp8_state()
def _init_fp8_state(self) -> None:
"""Initialize FP8 attention state from multimodal config.
No-op if FP8 is not requested. Raises ``ValueError`` if FP8 is
requested but the platform does not support it.
"""
# Populate defaults so ``_forward_flashinfer`` can
# check ``self.fp8_enabled`` and others without AttributeError.
self.fp8_enabled = False
self._fp8_dynamic_scale = False
self.fp8_quant: QuantFP8 | None = None
self.skip_scale_q = False
self.skip_scale_k = False
self.skip_scale_v = False
mm_cfg = get_multimodal_config()
if mm_cfg is None or mm_cfg.mm_encoder_attn_dtype != "fp8":
return
# FP8 path
if not is_flashinfer_cudnn_fp8_prefill_attn_supported():
raise ValueError(
"mm_encoder_attn_dtype='fp8' requires the FlashInfer "
"cuDNN backend with cuDNN >= 9.17.1 on a GPU with native "
"FP8 support."
)
self.fp8_enabled = True
self._fp8_dynamic_scale = mm_cfg.mm_encoder_fp8_scale_path is None
self.fp8_quant = QuantFP8(static=True, group_shape=GroupShape.PER_TENSOR)
# Register buffers pre-device-move; values populated in
# process_weights_after_loading. Shape (1, 1, 1, 1) is required by cuDNN.
for attr in ("_fp8_q_scale", "_fp8_k_scale", "_fp8_v_scale"):
self.register_buffer(
attr, torch.ones(1, dtype=torch.float32).view(1, 1, 1, 1)
)
if self._fp8_dynamic_scale:
for attr in ("_fp8_q_amax", "_fp8_k_amax", "_fp8_v_amax"):
self.register_buffer(
attr,
torch.zeros(_FP8_AMAX_HISTORY_LEN, dtype=torch.float32),
persistent=False,
)
self._fp8_amax_pos = 0
# Capture auto-save config now: the VllmConfig context only lives
# across model init, not forward passes, so ``_maybe_save_fp8_scales``
# reads these globals instead of re-querying ``get_multimodal_config``.
if (
mm_cfg.mm_encoder_fp8_scale_save_path is not None
and self._fp8_dynamic_scale
):
global _fp8_scale_save_path, _fp8_scale_save_margin
_fp8_scale_save_path = mm_cfg.mm_encoder_fp8_scale_save_path
_fp8_scale_save_margin = mm_cfg.mm_encoder_fp8_scale_save_margin
def process_weights_after_loading(self, act_dtype: torch.dtype) -> None:
"""Populate FP8 scale buffers after weights are loaded.
``act_dtype`` matches the signature used by :class:`Attention` and
:class:`MLAAttention` for the loader auto-scan but is unused:
FP8 scales are always float32.
"""
if not self.fp8_enabled:
return
mm_cfg = get_multimodal_config()
scale_path = mm_cfg.mm_encoder_fp8_scale_path if mm_cfg is not None else None
if scale_path is None:
logger.info_once(
"FP8 attention enabled with dynamic scaling "
"(no scale file provided). Scales will adapt from "
"observed Q/K/V amax values (history_len=%d).",
_FP8_AMAX_HISTORY_LEN,
)
return
all_scales = _load_fp8_scales_file(scale_path)
layer_scales = all_scales.get(self.layer_name)
if layer_scales is None:
raise ValueError(
"FP8 attention enabled but scales not found for layer "
f"'{self.layer_name}' in {scale_path}. "
f"Available layers: {list(all_scales.keys())}"
)
for attr, key in (
("_fp8_q_scale", "q"),
("_fp8_k_scale", "k"),
("_fp8_v_scale", "v"),
):
getattr(self, attr).fill_(layer_scales[key])
self.skip_scale_q = layer_scales["q"] == 1.0
self.skip_scale_k = layer_scales["k"] == 1.0
self.skip_scale_v = layer_scales["v"] == 1.0
logger.debug(
"FP8 attention enabled for %s: q=%.4f, k=%.4f, v=%.4f",
self.layer_name if self.layer_name else "MMEncoderAttention",
layer_scales["q"],
layer_scales["k"],
layer_scales["v"],
)
@classmethod
def enabled(cls) -> bool:
return True
@@ -353,6 +602,44 @@ class MMEncoderAttention(CustomOp):
output = output.reshape(bsz, q_len, -1)
return output
@torch.no_grad()
def _record_amax_and_update_scales(
self,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
) -> None:
"""Record Q/K/V amax into circular history and recompute scales.
All work stays on GPU with no device-to-host sync. The Python-side
history position counter is mutated, so this method must NOT be
called inside CUDA graph capture/replay. When CUDA graphs are
used for the encoder, dynamic scaling should be disabled by
providing a static scale file via --mm-encoder-fp8-scale-path.
"""
pos = self._fp8_amax_pos
self._fp8_amax_pos = (pos + 1) % _FP8_AMAX_HISTORY_LEN
for tensor, amax_buf, scale_buf in (
(query, self._fp8_q_amax, self._fp8_q_scale),
(key, self._fp8_k_amax, self._fp8_k_scale),
(value, self._fp8_v_amax, self._fp8_v_scale),
):
amax_buf[pos] = tensor.amax()
max_amax = amax_buf.max()
scale_buf.fill_(
torch.clamp(max_amax, min=torch.finfo(torch.float32).tiny) / _FP8_MAX
)
buffer_wrapped = self._fp8_amax_pos == 0 and pos == _FP8_AMAX_HISTORY_LEN - 1
_maybe_save_fp8_scales(
self.layer_name,
self._fp8_q_scale,
self._fp8_k_scale,
self._fp8_v_scale,
buffer_wrapped,
)
def _forward_flashinfer(
self,
query: torch.Tensor,
@@ -363,7 +650,32 @@ class MMEncoderAttention(CustomOp):
sequence_lengths: torch.Tensor
| None = None, # Only used for FlashInfer CuDNN backend
) -> torch.Tensor:
return vit_flashinfer_wrapper(
if self.fp8_enabled:
assert self.fp8_quant is not None
if self._fp8_dynamic_scale:
self._record_amax_and_update_scales(query, key, value)
query = quantize_fp8_maybe_pad_head_dim(
query,
self._fp8_q_scale,
skip_scale=self.skip_scale_q,
fp8_quant=self.fp8_quant,
)
key = quantize_fp8_maybe_pad_head_dim(
key,
self._fp8_k_scale,
skip_scale=self.skip_scale_k,
fp8_quant=self.fp8_quant,
)
value = quantize_fp8_maybe_pad_head_dim(
value,
self._fp8_v_scale,
skip_scale=self.skip_scale_v,
fp8_quant=self.fp8_quant,
)
output = vit_flashinfer_wrapper(
q=query,
k=key,
v=value,
@@ -372,8 +684,17 @@ class MMEncoderAttention(CustomOp):
cu_seqlens=cu_seqlens,
max_seqlen=max_seqlen,
sequence_lengths=sequence_lengths,
q_scale=self._fp8_q_scale if self.fp8_enabled else None,
k_scale=self._fp8_k_scale if self.fp8_enabled else None,
v_scale=self._fp8_v_scale if self.fp8_enabled else None,
o_data_type=self.dtype if self.fp8_enabled else None,
)
if self.fp8_enabled and output.shape[-1] != self.head_size:
output = output[..., : self.head_size].contiguous()
return output
def forward_native(
self,
query: torch.Tensor,
+9 -5
View File
@@ -15,7 +15,11 @@ from typing_extensions import assert_never
import vllm.envs as envs
from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention, MLAAttention
from vllm.model_executor.layers.attention import (
Attention,
MLAAttention,
MMEncoderAttention,
)
from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig,
QuantizeMethodBase,
@@ -106,12 +110,12 @@ def process_weights_after_loading(
with device_loading_context(module, target_device):
quant_method.process_weights_after_loading(module)
# Initialize post-load attention weights for both Attention and MLA.
# Initialize post-load attention weights for Attention, MLA, and MM encoder.
# NOTE: Happens after other modules so we can easily decompress weights.
for _, module in model.named_modules():
if isinstance(module, (Attention, MLAAttention)) and hasattr(
module, "process_weights_after_loading"
):
if isinstance(
module, (Attention, MLAAttention, MMEncoderAttention)
) and hasattr(module, "process_weights_after_loading"):
# TODO(lucas): see if there is a way to unify the signatures
# of process_weights_after_loading
with device_loading_context(module, target_device):
+9
View File
@@ -136,6 +136,7 @@ from .utils import (
maybe_prefix,
)
from .vision import (
get_fp8_padded_hidden_size,
get_vit_attn_backend,
is_vit_use_data_parallel,
run_dp_sharded_mrope_vision_model,
@@ -562,6 +563,13 @@ class Qwen3_VisionTransformer(nn.Module):
norm_layer = partial(nn.LayerNorm, eps=norm_eps)
head_dim = self.hidden_size // self.num_heads
# FP8 attention: Q/K/V become independent contiguous tensors
# after quantization, so cu_seqlens uses uniform stride (no 3x V).
self.fp8_padded_hidden_size = get_fp8_padded_hidden_size(
self.num_heads, head_dim
)
self.rotary_pos_emb = get_rope(
head_size=head_dim,
max_position=8192,
@@ -776,6 +784,7 @@ class Qwen3_VisionTransformer(nn.Module):
self.hidden_size,
self.tp_size,
device,
fp8_padded_hidden_size=self.fp8_padded_hidden_size,
)
return metadata
+32 -28
View File
@@ -10,7 +10,7 @@ from typing import Final, Generic, Literal, Protocol, TypeAlias, TypeVar
import torch
from transformers import PretrainedConfig
from vllm.config import MultiModalConfig, VllmConfig, get_current_vllm_config
from vllm.config import MultiModalConfig, get_current_vllm_config_or_none
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
@@ -18,6 +18,7 @@ from vllm.distributed import (
)
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.math_utils import round_up
from vllm.v1.attention.backends.registry import AttentionBackendEnum
logger = init_logger(__name__)
@@ -102,45 +103,48 @@ def get_vit_attn_backend(
"""
Get the attention backend for Vision Transformer.
"""
try:
vllm_config: VllmConfig = get_current_vllm_config()
model_config = vllm_config.model_config
multimodal_config: MultiModalConfig | None = (
model_config.multimodal_config if model_config is not None else None
)
except (AssertionError, AttributeError):
multimodal_config = None
mm_cfg = get_multimodal_config()
attn_backend_override = (
multimodal_config.mm_encoder_attn_backend
if multimodal_config is not None
else None
mm_cfg.mm_encoder_attn_backend if mm_cfg is not None else None
)
attn_backend = _get_vit_attn_backend(
return _get_vit_attn_backend(
head_size,
dtype,
attn_backend_override=attn_backend_override,
)
return attn_backend
def get_multimodal_config() -> MultiModalConfig | None:
"""Return the current ``MultiModalConfig``, or ``None`` when no engine
config context is active (e.g., during unit tests) or when the current
``model_config`` does not carry a ``multimodal_config`` (e.g., minimal
stubs used in tests)."""
vllm_config = get_current_vllm_config_or_none()
if vllm_config is None or vllm_config.model_config is None:
return None
return getattr(vllm_config.model_config, "multimodal_config", None)
def get_fp8_padded_hidden_size(num_heads: int, head_dim: int) -> int | None:
"""Return the padded hidden size for FP8 ViT encoder attention, or
``None`` when FP8 is not enabled.
cuDNN FP8 prefill attention requires ``head_dim`` to be a multiple of
16. For non-aligned ``head_dim`` (e.g. 72), Q/K/V are padded to the
nearest multiple of 16.
"""
mm_cfg = get_multimodal_config()
if mm_cfg is None or mm_cfg.mm_encoder_attn_dtype != "fp8":
return None
return num_heads * round_up(head_dim, 16)
def is_vit_use_data_parallel():
"""
Get the tensor parallel type for Vision Transformer.
"""
try:
vllm_config: VllmConfig = get_current_vllm_config()
model_config = vllm_config.model_config
multimodal_config: MultiModalConfig | None = (
model_config.multimodal_config if model_config is not None else None
)
except (AssertionError, AttributeError):
multimodal_config = None
mm_encoder_tp_mode = (
multimodal_config.mm_encoder_tp_mode if multimodal_config is not None else None
)
return mm_encoder_tp_mode == "data"
mm_cfg = get_multimodal_config()
return mm_cfg is not None and mm_cfg.mm_encoder_tp_mode == "data"
VisionFeatureSelectStrategyStr = Literal["class", "default", "full"]
+35
View File
@@ -772,6 +772,40 @@ def should_use_flashinfer_for_blockscale_fp8_gemm(
return should_use_flashinfer
_MIN_CUDNN_FP8 = 91701 # cuDNN >= 9.17.1 required for FP8 attention
@functools.cache
def is_flashinfer_cudnn_fp8_prefill_attn_supported() -> bool:
"""Check if FP8 ViT attention is supported on this platform.
Requires native FP8 hardware support, the FlashInfer cuDNN backend,
and cuDNN >= 9.17.1.
"""
from vllm.v1.attention.backends.registry import AttentionBackendEnum
# cuDNN SDPA FP8 requires Hopper (SM 90) or newer.
if not current_platform.has_device_capability(90):
return False
try:
supported = current_platform.get_supported_vit_attn_backends()
if AttentionBackendEnum.FLASHINFER not in supported:
return False
except (ImportError, AttributeError):
return False
try:
import torch.backends.cudnn as cudnn
if cudnn.is_available() and cudnn.version() < _MIN_CUDNN_FP8:
return False
except (ImportError, AttributeError):
pass
return True
__all__ = [
"has_flashinfer",
"flashinfer_trtllm_fp8_block_scale_moe",
@@ -803,4 +837,5 @@ __all__ = [
"flashinfer_fp8_blockscale_gemm",
"should_use_flashinfer_for_blockscale_fp8_gemm",
"is_flashinfer_fp8_blockscale_gemm_supported",
"is_flashinfer_cudnn_fp8_prefill_attn_supported",
]
+29 -2
View File
@@ -279,6 +279,10 @@ def flashinfer_wrapper(
cu_seqlens: torch.Tensor | None = None,
max_seqlen: torch.Tensor | None = None,
sequence_lengths: torch.Tensor | None = None,
q_scale: torch.Tensor | None = None,
k_scale: torch.Tensor | None = None,
v_scale: torch.Tensor | None = None,
o_data_type: torch.dtype | None = None,
) -> torch.Tensor:
from flashinfer.prefill import cudnn_batch_prefill_with_kv_cache
@@ -318,6 +322,10 @@ def flashinfer_wrapper(
batch_offsets_k=batch_offsets_qko,
batch_offsets_v=batch_offsets_v,
batch_offsets_o=batch_offsets_qko,
q_scale=q_scale,
k_scale=k_scale,
v_scale=v_scale,
o_data_type=o_data_type,
)
if is_reshaped:
@@ -335,8 +343,12 @@ def vit_flashinfer_wrapper_fake(
cu_seqlens: torch.Tensor | None = None,
max_seqlen: torch.Tensor | None = None,
sequence_lengths: torch.Tensor | None = None,
q_scale: torch.Tensor | None = None,
k_scale: torch.Tensor | None = None,
v_scale: torch.Tensor | None = None,
o_data_type: torch.dtype | None = None,
) -> torch.Tensor:
return torch.empty_like(q)
return torch.empty_like(q, dtype=o_data_type or q.dtype)
direct_register_custom_op(
@@ -355,7 +367,22 @@ def vit_flashinfer_wrapper(
cu_seqlens: torch.Tensor | None = None,
max_seqlen: torch.Tensor | None = None,
sequence_lengths: torch.Tensor | None = None,
q_scale: torch.Tensor | None = None,
k_scale: torch.Tensor | None = None,
v_scale: torch.Tensor | None = None,
o_data_type: torch.dtype | None = None,
) -> torch.Tensor:
return torch.ops.vllm.flashinfer_wrapper(
q, k, v, scale, workspace_buffer, cu_seqlens, max_seqlen, sequence_lengths
q,
k,
v,
scale,
workspace_buffer,
cu_seqlens,
max_seqlen,
sequence_lengths,
q_scale,
k_scale,
v_scale,
o_data_type,
)