[Perf] FP8 FlashInfer Attn for ViT (#38065)
Signed-off-by: Zhanda Zhu <zhandazhu@gmail.com> Co-authored-by: Yubo Gao <ybgao-nvidia@users.noreply.github.com>
This commit is contained in:
@@ -1,13 +1,32 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import functools
|
||||
import json
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.config import MultiModalConfig
|
||||
from vllm.kernels.triton.qkv_padded_fp8_quant import (
|
||||
quantize_fp8_maybe_pad_head_dim,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import CustomOp, maybe_get_oot_by_class
|
||||
from vllm.model_executor.models.vision import get_vit_attn_backend
|
||||
from vllm.model_executor.layers.quantization.input_quant_fp8 import (
|
||||
QuantFP8,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
get_fp8_min_max,
|
||||
)
|
||||
from vllm.model_executor.models.vision import (
|
||||
get_multimodal_config,
|
||||
get_vit_attn_backend,
|
||||
)
|
||||
from vllm.utils.flashinfer import (
|
||||
is_flashinfer_cudnn_fp8_prefill_attn_supported,
|
||||
)
|
||||
from vllm.utils.math_utils import round_up
|
||||
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
@@ -20,6 +39,108 @@ from vllm.v1.attention.ops.vit_attn_wrappers import (
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_, _FP8_MAX = get_fp8_min_max()
|
||||
_FP8_AMAX_HISTORY_LEN = 16
|
||||
|
||||
# Module-level state for auto-saving dynamic scales. The save is a one-shot
|
||||
# triggered by the first layer whose amax buffer wraps. Path and margin are
|
||||
# captured during layer init (set_current_vllm_config context only lives
|
||||
# across model init, not forward passes).
|
||||
_fp8_scale_save_path: str | None = None
|
||||
_fp8_scale_save_margin: float = MultiModalConfig.mm_encoder_fp8_scale_save_margin
|
||||
_fp8_saved_scale_refs: dict[str, tuple[torch.Tensor, torch.Tensor, torch.Tensor]] = {}
|
||||
|
||||
|
||||
@functools.cache
|
||||
def _load_fp8_scales_file(path: str | None) -> dict[str, dict[str, float]]:
|
||||
"""Load per-layer FP8 Q/K/V scales from a JSON file. Results are cached.
|
||||
|
||||
Expected format (keys ``q_scale`` / ``k_scale`` / ``v_scale`` also accepted)::
|
||||
|
||||
{
|
||||
"visual.blocks.0.attn.attn": {"q": 224.0, "k": 198.0, "v": 210.0},
|
||||
"visual.blocks.1.attn.attn": {"q": 218.0, "k": 195.0, "v": 207.0},
|
||||
}
|
||||
|
||||
To produce such a file, run with ``mm_encoder_fp8_scale_save_path`` set.
|
||||
"""
|
||||
if path is None:
|
||||
return {}
|
||||
|
||||
with open(path, encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
|
||||
# Handle nested "layers" format
|
||||
if "layers" in data and isinstance(data["layers"], dict):
|
||||
data = data["layers"]
|
||||
|
||||
scales: dict[str, dict[str, float]] = {}
|
||||
for layer_name, layer_scales in data.items():
|
||||
if not isinstance(layer_scales, dict):
|
||||
continue
|
||||
q = layer_scales.get("q", layer_scales.get("q_scale"))
|
||||
k = layer_scales.get("k", layer_scales.get("k_scale"))
|
||||
v = layer_scales.get("v", layer_scales.get("v_scale"))
|
||||
if q is not None and k is not None and v is not None:
|
||||
q_f, k_f, v_f = float(q), float(k), float(v)
|
||||
if q_f <= 0 or k_f <= 0 or v_f <= 0:
|
||||
raise ValueError(
|
||||
f"FP8 scales must be positive, got q={q_f}, "
|
||||
f"k={k_f}, v={v_f} for layer '{layer_name}'"
|
||||
)
|
||||
scales[layer_name] = {"q": q_f, "k": k_f, "v": v_f}
|
||||
|
||||
logger.info_once(
|
||||
"Loaded FP8 attention scales from %s (%d layers)", path, len(scales)
|
||||
)
|
||||
return scales
|
||||
|
||||
|
||||
def _maybe_save_fp8_scales(
|
||||
layer_name: str,
|
||||
q_scale: torch.Tensor,
|
||||
k_scale: torch.Tensor,
|
||||
v_scale: torch.Tensor,
|
||||
buffer_wrapped: bool,
|
||||
) -> None:
|
||||
"""Accumulate a layer's scale tensors; on the first amax buffer wrap,
|
||||
dump all accumulated scales to ``mm_encoder_fp8_scale_save_path``.
|
||||
|
||||
No-op unless auto-save is configured. Tensor references are stored on
|
||||
every call (no GPU->CPU sync); ``.item()`` is only called at the single
|
||||
save point to avoid stalling the forward path.
|
||||
"""
|
||||
global _fp8_scale_save_path
|
||||
# Fast path: auto-save either disabled or already finished. Path is
|
||||
# captured at layer init and cleared once the save fires.
|
||||
if _fp8_scale_save_path is None:
|
||||
return
|
||||
|
||||
# Stash scale tensor refs (no GPU->CPU sync yet); wait until the amax
|
||||
# history has seen a full cycle before committing scales to disk.
|
||||
_fp8_saved_scale_refs[layer_name] = (q_scale, k_scale, v_scale)
|
||||
if not buffer_wrapped:
|
||||
return
|
||||
|
||||
# Buffer just wrapped for the first time: materialize scales (with
|
||||
# safety margin) and dump to disk. Clearing _fp8_scale_save_path
|
||||
# makes this a one-shot across all layers.
|
||||
path, margin = _fp8_scale_save_path, _fp8_scale_save_margin
|
||||
scales = {
|
||||
name: {
|
||||
"q": q.item() * margin,
|
||||
"k": k.item() * margin,
|
||||
"v": v.item() * margin,
|
||||
}
|
||||
for name, (q, k, v) in _fp8_saved_scale_refs.items()
|
||||
}
|
||||
_fp8_scale_save_path = None
|
||||
_fp8_saved_scale_refs.clear()
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
json.dump(scales, f, indent=2)
|
||||
logger.info("Saved FP8 scales (%d layers) to %s", len(scales), path)
|
||||
|
||||
|
||||
# Batch buckets for cuDNN graph caching.
|
||||
# Graphs use batch size and max sequence length as cache key.
|
||||
# This avoids creating a new graph for each unique set of
|
||||
@@ -148,27 +269,47 @@ class MMEncoderAttention(CustomOp):
|
||||
hidden_size: int,
|
||||
tp_size: int,
|
||||
device: torch.device,
|
||||
fp8_padded_hidden_size: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
if (oot_class := maybe_get_oot_by_class(cls)) is not cls:
|
||||
return oot_class.maybe_recompute_cu_seqlens( # type: ignore[attr-defined]
|
||||
attn_backend, cu_seqlens, hidden_size, tp_size, device
|
||||
attn_backend,
|
||||
cu_seqlens,
|
||||
hidden_size,
|
||||
tp_size,
|
||||
device,
|
||||
fp8_padded_hidden_size=fp8_padded_hidden_size,
|
||||
)
|
||||
|
||||
if attn_backend == AttentionBackendEnum.FLASHINFER:
|
||||
batch_size = len(cu_seqlens) - 1
|
||||
scale = hidden_size // tp_size
|
||||
cu_seqlens = cu_seqlens * scale
|
||||
|
||||
cu_seqlens_qko = cu_seqlens
|
||||
cu_seqlens_v = cu_seqlens * 3
|
||||
if fp8_padded_hidden_size is not None:
|
||||
# FP8 path: after quantization Q/K/V are each independent
|
||||
# contiguous tensors with stride H * padded_D per token.
|
||||
# All sections use the same element stride.
|
||||
scale = fp8_padded_hidden_size // tp_size
|
||||
cu_seqlens = cu_seqlens * scale
|
||||
cu_seqlens_padded = add_padding_to_seqlens(
|
||||
cu_seqlens, batch_size, cu_seqlens[-1]
|
||||
)
|
||||
cu_seqlens = np.concatenate([cu_seqlens_padded, cu_seqlens_padded])
|
||||
else:
|
||||
# BF16 path: Q/K/V are non-contiguous views into shared
|
||||
# buffers. V section has 3x stride from interleaved QKV.
|
||||
scale = hidden_size // tp_size
|
||||
cu_seqlens = cu_seqlens * scale
|
||||
|
||||
cu_seqlens_qko = add_padding_to_seqlens(
|
||||
cu_seqlens_qko, batch_size, cu_seqlens_qko[-1]
|
||||
)
|
||||
cu_seqlens_v = add_padding_to_seqlens(
|
||||
cu_seqlens_v, batch_size, cu_seqlens_v[-1]
|
||||
)
|
||||
cu_seqlens = np.concatenate([cu_seqlens_qko, cu_seqlens_v])
|
||||
cu_seqlens_qko = cu_seqlens
|
||||
cu_seqlens_v = cu_seqlens * 3
|
||||
|
||||
cu_seqlens_qko = add_padding_to_seqlens(
|
||||
cu_seqlens_qko, batch_size, cu_seqlens_qko[-1]
|
||||
)
|
||||
cu_seqlens_v = add_padding_to_seqlens(
|
||||
cu_seqlens_v, batch_size, cu_seqlens_v[-1]
|
||||
)
|
||||
cu_seqlens = np.concatenate([cu_seqlens_qko, cu_seqlens_v])
|
||||
|
||||
cu_seqlens = torch.from_numpy(cu_seqlens).to(device, non_blocking=True)
|
||||
return cu_seqlens
|
||||
@@ -206,6 +347,7 @@ class MMEncoderAttention(CustomOp):
|
||||
# During model initialization, the default dtype is set as the model
|
||||
# weight and activation dtype.
|
||||
dtype = torch.get_default_dtype()
|
||||
self.dtype = dtype
|
||||
|
||||
# Get device-specific vision attention backend.
|
||||
self.attn_backend = get_vit_attn_backend(
|
||||
@@ -229,6 +371,113 @@ class MMEncoderAttention(CustomOp):
|
||||
|
||||
logger.info_once(f"Using {self.attn_backend} for MMEncoderAttention.")
|
||||
|
||||
self._init_fp8_state()
|
||||
|
||||
def _init_fp8_state(self) -> None:
|
||||
"""Initialize FP8 attention state from multimodal config.
|
||||
|
||||
No-op if FP8 is not requested. Raises ``ValueError`` if FP8 is
|
||||
requested but the platform does not support it.
|
||||
"""
|
||||
# Populate defaults so ``_forward_flashinfer`` can
|
||||
# check ``self.fp8_enabled`` and others without AttributeError.
|
||||
self.fp8_enabled = False
|
||||
self._fp8_dynamic_scale = False
|
||||
self.fp8_quant: QuantFP8 | None = None
|
||||
self.skip_scale_q = False
|
||||
self.skip_scale_k = False
|
||||
self.skip_scale_v = False
|
||||
|
||||
mm_cfg = get_multimodal_config()
|
||||
if mm_cfg is None or mm_cfg.mm_encoder_attn_dtype != "fp8":
|
||||
return
|
||||
|
||||
# FP8 path
|
||||
if not is_flashinfer_cudnn_fp8_prefill_attn_supported():
|
||||
raise ValueError(
|
||||
"mm_encoder_attn_dtype='fp8' requires the FlashInfer "
|
||||
"cuDNN backend with cuDNN >= 9.17.1 on a GPU with native "
|
||||
"FP8 support."
|
||||
)
|
||||
|
||||
self.fp8_enabled = True
|
||||
self._fp8_dynamic_scale = mm_cfg.mm_encoder_fp8_scale_path is None
|
||||
self.fp8_quant = QuantFP8(static=True, group_shape=GroupShape.PER_TENSOR)
|
||||
|
||||
# Register buffers pre-device-move; values populated in
|
||||
# process_weights_after_loading. Shape (1, 1, 1, 1) is required by cuDNN.
|
||||
for attr in ("_fp8_q_scale", "_fp8_k_scale", "_fp8_v_scale"):
|
||||
self.register_buffer(
|
||||
attr, torch.ones(1, dtype=torch.float32).view(1, 1, 1, 1)
|
||||
)
|
||||
if self._fp8_dynamic_scale:
|
||||
for attr in ("_fp8_q_amax", "_fp8_k_amax", "_fp8_v_amax"):
|
||||
self.register_buffer(
|
||||
attr,
|
||||
torch.zeros(_FP8_AMAX_HISTORY_LEN, dtype=torch.float32),
|
||||
persistent=False,
|
||||
)
|
||||
self._fp8_amax_pos = 0
|
||||
|
||||
# Capture auto-save config now: the VllmConfig context only lives
|
||||
# across model init, not forward passes, so ``_maybe_save_fp8_scales``
|
||||
# reads these globals instead of re-querying ``get_multimodal_config``.
|
||||
if (
|
||||
mm_cfg.mm_encoder_fp8_scale_save_path is not None
|
||||
and self._fp8_dynamic_scale
|
||||
):
|
||||
global _fp8_scale_save_path, _fp8_scale_save_margin
|
||||
_fp8_scale_save_path = mm_cfg.mm_encoder_fp8_scale_save_path
|
||||
_fp8_scale_save_margin = mm_cfg.mm_encoder_fp8_scale_save_margin
|
||||
|
||||
def process_weights_after_loading(self, act_dtype: torch.dtype) -> None:
|
||||
"""Populate FP8 scale buffers after weights are loaded.
|
||||
|
||||
``act_dtype`` matches the signature used by :class:`Attention` and
|
||||
:class:`MLAAttention` for the loader auto-scan but is unused:
|
||||
FP8 scales are always float32.
|
||||
"""
|
||||
if not self.fp8_enabled:
|
||||
return
|
||||
|
||||
mm_cfg = get_multimodal_config()
|
||||
scale_path = mm_cfg.mm_encoder_fp8_scale_path if mm_cfg is not None else None
|
||||
if scale_path is None:
|
||||
logger.info_once(
|
||||
"FP8 attention enabled with dynamic scaling "
|
||||
"(no scale file provided). Scales will adapt from "
|
||||
"observed Q/K/V amax values (history_len=%d).",
|
||||
_FP8_AMAX_HISTORY_LEN,
|
||||
)
|
||||
return
|
||||
|
||||
all_scales = _load_fp8_scales_file(scale_path)
|
||||
layer_scales = all_scales.get(self.layer_name)
|
||||
if layer_scales is None:
|
||||
raise ValueError(
|
||||
"FP8 attention enabled but scales not found for layer "
|
||||
f"'{self.layer_name}' in {scale_path}. "
|
||||
f"Available layers: {list(all_scales.keys())}"
|
||||
)
|
||||
|
||||
for attr, key in (
|
||||
("_fp8_q_scale", "q"),
|
||||
("_fp8_k_scale", "k"),
|
||||
("_fp8_v_scale", "v"),
|
||||
):
|
||||
getattr(self, attr).fill_(layer_scales[key])
|
||||
self.skip_scale_q = layer_scales["q"] == 1.0
|
||||
self.skip_scale_k = layer_scales["k"] == 1.0
|
||||
self.skip_scale_v = layer_scales["v"] == 1.0
|
||||
|
||||
logger.debug(
|
||||
"FP8 attention enabled for %s: q=%.4f, k=%.4f, v=%.4f",
|
||||
self.layer_name if self.layer_name else "MMEncoderAttention",
|
||||
layer_scales["q"],
|
||||
layer_scales["k"],
|
||||
layer_scales["v"],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def enabled(cls) -> bool:
|
||||
return True
|
||||
@@ -353,6 +602,44 @@ class MMEncoderAttention(CustomOp):
|
||||
output = output.reshape(bsz, q_len, -1)
|
||||
return output
|
||||
|
||||
@torch.no_grad()
|
||||
def _record_amax_and_update_scales(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
) -> None:
|
||||
"""Record Q/K/V amax into circular history and recompute scales.
|
||||
|
||||
All work stays on GPU with no device-to-host sync. The Python-side
|
||||
history position counter is mutated, so this method must NOT be
|
||||
called inside CUDA graph capture/replay. When CUDA graphs are
|
||||
used for the encoder, dynamic scaling should be disabled by
|
||||
providing a static scale file via --mm-encoder-fp8-scale-path.
|
||||
"""
|
||||
pos = self._fp8_amax_pos
|
||||
self._fp8_amax_pos = (pos + 1) % _FP8_AMAX_HISTORY_LEN
|
||||
|
||||
for tensor, amax_buf, scale_buf in (
|
||||
(query, self._fp8_q_amax, self._fp8_q_scale),
|
||||
(key, self._fp8_k_amax, self._fp8_k_scale),
|
||||
(value, self._fp8_v_amax, self._fp8_v_scale),
|
||||
):
|
||||
amax_buf[pos] = tensor.amax()
|
||||
max_amax = amax_buf.max()
|
||||
scale_buf.fill_(
|
||||
torch.clamp(max_amax, min=torch.finfo(torch.float32).tiny) / _FP8_MAX
|
||||
)
|
||||
|
||||
buffer_wrapped = self._fp8_amax_pos == 0 and pos == _FP8_AMAX_HISTORY_LEN - 1
|
||||
_maybe_save_fp8_scales(
|
||||
self.layer_name,
|
||||
self._fp8_q_scale,
|
||||
self._fp8_k_scale,
|
||||
self._fp8_v_scale,
|
||||
buffer_wrapped,
|
||||
)
|
||||
|
||||
def _forward_flashinfer(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
@@ -363,7 +650,32 @@ class MMEncoderAttention(CustomOp):
|
||||
sequence_lengths: torch.Tensor
|
||||
| None = None, # Only used for FlashInfer CuDNN backend
|
||||
) -> torch.Tensor:
|
||||
return vit_flashinfer_wrapper(
|
||||
if self.fp8_enabled:
|
||||
assert self.fp8_quant is not None
|
||||
|
||||
if self._fp8_dynamic_scale:
|
||||
self._record_amax_and_update_scales(query, key, value)
|
||||
|
||||
query = quantize_fp8_maybe_pad_head_dim(
|
||||
query,
|
||||
self._fp8_q_scale,
|
||||
skip_scale=self.skip_scale_q,
|
||||
fp8_quant=self.fp8_quant,
|
||||
)
|
||||
key = quantize_fp8_maybe_pad_head_dim(
|
||||
key,
|
||||
self._fp8_k_scale,
|
||||
skip_scale=self.skip_scale_k,
|
||||
fp8_quant=self.fp8_quant,
|
||||
)
|
||||
value = quantize_fp8_maybe_pad_head_dim(
|
||||
value,
|
||||
self._fp8_v_scale,
|
||||
skip_scale=self.skip_scale_v,
|
||||
fp8_quant=self.fp8_quant,
|
||||
)
|
||||
|
||||
output = vit_flashinfer_wrapper(
|
||||
q=query,
|
||||
k=key,
|
||||
v=value,
|
||||
@@ -372,8 +684,17 @@ class MMEncoderAttention(CustomOp):
|
||||
cu_seqlens=cu_seqlens,
|
||||
max_seqlen=max_seqlen,
|
||||
sequence_lengths=sequence_lengths,
|
||||
q_scale=self._fp8_q_scale if self.fp8_enabled else None,
|
||||
k_scale=self._fp8_k_scale if self.fp8_enabled else None,
|
||||
v_scale=self._fp8_v_scale if self.fp8_enabled else None,
|
||||
o_data_type=self.dtype if self.fp8_enabled else None,
|
||||
)
|
||||
|
||||
if self.fp8_enabled and output.shape[-1] != self.head_size:
|
||||
output = output[..., : self.head_size].contiguous()
|
||||
|
||||
return output
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user