[Perf] FP8 FlashInfer Attn for ViT (#38065)
Signed-off-by: Zhanda Zhu <zhandazhu@gmail.com> Co-authored-by: Yubo Gao <ybgao-nvidia@users.noreply.github.com>
This commit is contained in:
@@ -326,6 +326,10 @@ class ModelConfig:
|
||||
mm_encoder_only: InitVar[bool | None] = None
|
||||
mm_encoder_tp_mode: InitVar[MMEncoderTPMode | None] = None
|
||||
mm_encoder_attn_backend: InitVar[AttentionBackendEnum | str | None] = None
|
||||
mm_encoder_attn_dtype: InitVar[str | None] = None
|
||||
mm_encoder_fp8_scale_path: InitVar[str | None] = None
|
||||
mm_encoder_fp8_scale_save_path: InitVar[str | None] = None
|
||||
mm_encoder_fp8_scale_save_margin: InitVar[float | None] = None
|
||||
interleave_mm_strings: InitVar[bool | None] = None
|
||||
skip_mm_profiling: InitVar[bool | None] = None
|
||||
video_pruning_rate: InitVar[float | None] = None
|
||||
@@ -447,6 +451,10 @@ class ModelConfig:
|
||||
mm_encoder_only: bool | None,
|
||||
mm_encoder_tp_mode: MMEncoderTPMode | None,
|
||||
mm_encoder_attn_backend: AttentionBackendEnum | str | None,
|
||||
mm_encoder_attn_dtype: str | None,
|
||||
mm_encoder_fp8_scale_path: str | None,
|
||||
mm_encoder_fp8_scale_save_path: str | None,
|
||||
mm_encoder_fp8_scale_save_margin: float | None,
|
||||
interleave_mm_strings: bool | None,
|
||||
skip_mm_profiling: bool | None,
|
||||
video_pruning_rate: float | None,
|
||||
@@ -643,6 +651,10 @@ class ModelConfig:
|
||||
mm_encoder_only=mm_encoder_only,
|
||||
mm_encoder_tp_mode=mm_encoder_tp_mode,
|
||||
mm_encoder_attn_backend=mm_encoder_attn_backend,
|
||||
mm_encoder_attn_dtype=mm_encoder_attn_dtype,
|
||||
mm_encoder_fp8_scale_path=mm_encoder_fp8_scale_path,
|
||||
mm_encoder_fp8_scale_save_path=mm_encoder_fp8_scale_save_path,
|
||||
mm_encoder_fp8_scale_save_margin=mm_encoder_fp8_scale_save_margin,
|
||||
interleave_mm_strings=interleave_mm_strings,
|
||||
skip_mm_profiling=skip_mm_profiling,
|
||||
video_pruning_rate=video_pruning_rate,
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Mapping
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal, TypeAlias, TypedDict, final
|
||||
|
||||
from pydantic import ConfigDict, Field, field_validator, model_validator
|
||||
@@ -158,6 +159,24 @@ class MultiModalConfig:
|
||||
"""Optional override for the multi-modal encoder attention backend when
|
||||
using vision transformers. Accepts any value from
|
||||
`vllm.v1.attention.backends.registry.AttentionBackendEnum` (e.g. `FLASH_ATTN`)."""
|
||||
mm_encoder_attn_dtype: Literal["fp8"] | None = None
|
||||
"""Optional dtype override for ViT encoder attention. Set to `"fp8"` to
|
||||
enable FP8 quantization via the FlashInfer cuDNN backend. When set to
|
||||
`"fp8"` without a scale file, dynamic scaling is used automatically.
|
||||
See docs/features/quantization/fp8_vit_attn.md for details."""
|
||||
mm_encoder_fp8_scale_path: str | None = None
|
||||
"""Path to a JSON file containing per-layer FP8 Q/K/V scales for ViT
|
||||
encoder attention. When provided (with `mm_encoder_attn_dtype="fp8"`),
|
||||
static scaling is used. When omitted, dynamic scaling is used."""
|
||||
mm_encoder_fp8_scale_save_path: str | None = None
|
||||
"""When set with dynamic FP8 scaling (`mm_encoder_attn_dtype="fp8"`
|
||||
and no `mm_encoder_fp8_scale_path`), saves the calibrated scales to
|
||||
this file after the amax history buffer is full. The saved file can
|
||||
then be used as `mm_encoder_fp8_scale_path` in subsequent runs."""
|
||||
mm_encoder_fp8_scale_save_margin: float = Field(default=1.5, gt=0.0)
|
||||
"""Safety margin multiplied onto scales when auto-saving. A value > 1
|
||||
leaves headroom so that inputs with larger activations than the
|
||||
calibration set do not overflow FP8 range. Default 1.5."""
|
||||
interleave_mm_strings: bool = False
|
||||
"""Enable fully interleaved support for multimodal prompts, while using
|
||||
--chat-template-content-format=string."""
|
||||
@@ -233,6 +252,36 @@ class MultiModalConfig:
|
||||
"'mm_shm_cache_max_object_size_mb' should only be set when "
|
||||
"'mm_processor_cache_type' is 'shm'."
|
||||
)
|
||||
# Validate FP8 scale path combinations.
|
||||
if self.mm_encoder_attn_dtype != "fp8" and (
|
||||
self.mm_encoder_fp8_scale_path is not None
|
||||
or self.mm_encoder_fp8_scale_save_path is not None
|
||||
):
|
||||
raise ValueError(
|
||||
"'mm_encoder_fp8_scale_path' and "
|
||||
"'mm_encoder_fp8_scale_save_path' require "
|
||||
"'mm_encoder_attn_dtype' to be 'fp8'."
|
||||
)
|
||||
if (
|
||||
self.mm_encoder_fp8_scale_path is not None
|
||||
and self.mm_encoder_fp8_scale_save_path is not None
|
||||
):
|
||||
raise ValueError(
|
||||
"'mm_encoder_fp8_scale_save_path' cannot be used with "
|
||||
"'mm_encoder_fp8_scale_path' (saving requires dynamic scaling)."
|
||||
)
|
||||
|
||||
# Validate file paths exist.
|
||||
if self.mm_encoder_fp8_scale_path is not None:
|
||||
scale_path = Path(self.mm_encoder_fp8_scale_path)
|
||||
if not scale_path.is_file():
|
||||
raise FileNotFoundError(f"FP8 scale file not found: {scale_path}")
|
||||
if self.mm_encoder_fp8_scale_save_path is not None:
|
||||
save_parent = Path(self.mm_encoder_fp8_scale_save_path).parent
|
||||
if not save_parent.is_dir():
|
||||
raise FileNotFoundError(
|
||||
f"Parent directory for FP8 scale save path not found: {save_parent}"
|
||||
)
|
||||
return self
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
@@ -252,6 +301,8 @@ class MultiModalConfig:
|
||||
if self.mm_encoder_attn_backend is not None
|
||||
else None,
|
||||
self.mm_encoder_tp_mode,
|
||||
self.mm_encoder_attn_dtype,
|
||||
self.mm_encoder_fp8_scale_path,
|
||||
]
|
||||
hash_str = safe_hash(str(factors).encode(), usedforsecurity=False).hexdigest()
|
||||
return hash_str
|
||||
|
||||
@@ -542,6 +542,14 @@ class EngineArgs:
|
||||
mm_encoder_attn_backend: AttentionBackendEnum | str | None = (
|
||||
MultiModalConfig.mm_encoder_attn_backend
|
||||
)
|
||||
mm_encoder_attn_dtype: str | None = MultiModalConfig.mm_encoder_attn_dtype
|
||||
mm_encoder_fp8_scale_path: str | None = MultiModalConfig.mm_encoder_fp8_scale_path
|
||||
mm_encoder_fp8_scale_save_path: str | None = (
|
||||
MultiModalConfig.mm_encoder_fp8_scale_save_path
|
||||
)
|
||||
mm_encoder_fp8_scale_save_margin: float = (
|
||||
MultiModalConfig.mm_encoder_fp8_scale_save_margin
|
||||
)
|
||||
io_processor_plugin: str | None = None
|
||||
renderer_num_workers: int = 1
|
||||
skip_mm_profiling: bool = MultiModalConfig.skip_mm_profiling
|
||||
@@ -1179,6 +1187,22 @@ class EngineArgs:
|
||||
"--mm-encoder-attn-backend",
|
||||
**multimodal_kwargs["mm_encoder_attn_backend"],
|
||||
)
|
||||
multimodal_group.add_argument(
|
||||
"--mm-encoder-attn-dtype",
|
||||
**multimodal_kwargs["mm_encoder_attn_dtype"],
|
||||
)
|
||||
multimodal_group.add_argument(
|
||||
"--mm-encoder-fp8-scale-path",
|
||||
**multimodal_kwargs["mm_encoder_fp8_scale_path"],
|
||||
)
|
||||
multimodal_group.add_argument(
|
||||
"--mm-encoder-fp8-scale-save-path",
|
||||
**multimodal_kwargs["mm_encoder_fp8_scale_save_path"],
|
||||
)
|
||||
multimodal_group.add_argument(
|
||||
"--mm-encoder-fp8-scale-save-margin",
|
||||
**multimodal_kwargs["mm_encoder_fp8_scale_save_margin"],
|
||||
)
|
||||
multimodal_group.add_argument(
|
||||
"--interleave-mm-strings", **multimodal_kwargs["interleave_mm_strings"]
|
||||
)
|
||||
@@ -1517,6 +1541,10 @@ class EngineArgs:
|
||||
mm_encoder_only=self.mm_encoder_only,
|
||||
mm_encoder_tp_mode=self.mm_encoder_tp_mode,
|
||||
mm_encoder_attn_backend=self.mm_encoder_attn_backend,
|
||||
mm_encoder_attn_dtype=self.mm_encoder_attn_dtype,
|
||||
mm_encoder_fp8_scale_path=self.mm_encoder_fp8_scale_path,
|
||||
mm_encoder_fp8_scale_save_path=self.mm_encoder_fp8_scale_save_path,
|
||||
mm_encoder_fp8_scale_save_margin=self.mm_encoder_fp8_scale_save_margin,
|
||||
pooler_config=self.pooler_config,
|
||||
generation_config=self.generation_config,
|
||||
override_generation_config=self.override_generation_config,
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Triton kernel implementations."""
|
||||
@@ -0,0 +1,180 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Stride-aware FP8 quantization with head_dim padding for ViT attention.
|
||||
|
||||
Reads directly from non-contiguous QKV views using 3D strides and pads
|
||||
head_dim to a multiple of 16 for cuDNN compatibility.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
get_fp8_min_max,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON, tl, triton
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
_FP8_MIN, _FP8_MAX = get_fp8_min_max()
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _quantize_pad_fp8_kernel(
|
||||
x_ptr,
|
||||
y_ptr,
|
||||
scale_ptr,
|
||||
stride_xs,
|
||||
stride_xh,
|
||||
stride_xd,
|
||||
stride_ys,
|
||||
stride_yh,
|
||||
stride_yd,
|
||||
num_heads,
|
||||
n_rows,
|
||||
n_cols,
|
||||
n_cols_padded,
|
||||
fp8_min,
|
||||
fp8_max,
|
||||
SKIP_SCALE: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
):
|
||||
pid_m = tl.program_id(0)
|
||||
pid_n = tl.program_id(1)
|
||||
|
||||
offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
|
||||
mask_m = offs_m < n_rows
|
||||
mask_out = mask_m[:, None] & (offs_n[None, :] < n_cols_padded)
|
||||
mask_in = mask_m[:, None] & (offs_n[None, :] < n_cols)
|
||||
|
||||
# Decompose flattened row into (token, head) for 3D stride indexing.
|
||||
s = offs_m // num_heads
|
||||
h = offs_m % num_heads
|
||||
|
||||
x_ptrs = (
|
||||
x_ptr
|
||||
+ s[:, None] * stride_xs
|
||||
+ h[:, None] * stride_xh
|
||||
+ offs_n[None, :] * stride_xd
|
||||
)
|
||||
x = tl.load(x_ptrs, mask=mask_in, other=0.0).to(tl.float32)
|
||||
if SKIP_SCALE:
|
||||
x_q = x
|
||||
else:
|
||||
scale = tl.load(scale_ptr)
|
||||
x_q = x / scale
|
||||
x_q = tl.clamp(x_q, fp8_min, fp8_max).to(y_ptr.dtype.element_ty)
|
||||
|
||||
y_ptrs = (
|
||||
y_ptr
|
||||
+ s[:, None] * stride_ys
|
||||
+ h[:, None] * stride_yh
|
||||
+ offs_n[None, :] * stride_yd
|
||||
)
|
||||
tl.store(y_ptrs, x_q, mask=mask_out)
|
||||
|
||||
|
||||
def _get_fp8_pad_quant_config(padded_head_dim: int) -> tuple[int, int, int]:
|
||||
block_n = triton.next_power_of_2(padded_head_dim)
|
||||
block_n = max(16, min(block_n, 128))
|
||||
block_m = 16
|
||||
num_warps = 4
|
||||
return block_m, block_n, num_warps
|
||||
|
||||
|
||||
def quantize_fp8_pad_head_dim_triton(
|
||||
tensor: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
skip_scale: bool = False,
|
||||
block_m: int | None = None,
|
||||
block_n: int | None = None,
|
||||
num_warps: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Quantize a 3D/4D tensor to FP8, padding head_dim to a multiple of 16.
|
||||
|
||||
Reads directly from the input using its 3D strides, so non-contiguous
|
||||
views (e.g. Q/K/V slices from an interleaved QKV buffer) are handled
|
||||
without an extra copy. Output is always a fresh contiguous tensor
|
||||
with shape (S, H, padded_D).
|
||||
"""
|
||||
if not HAS_TRITON:
|
||||
raise RuntimeError("Triton is required to quantize with head_dim padding.")
|
||||
|
||||
original_shape = tensor.shape
|
||||
if tensor.dim() == 4:
|
||||
tensor = tensor.view(-1, tensor.shape[-2], tensor.shape[-1])
|
||||
assert tensor.dim() == 3, f"Expected 3D input (S, H, D), got {tensor.dim()}D"
|
||||
S, H, D = tensor.shape
|
||||
padded_head_dim = round_up(D, 16)
|
||||
out_dtype = current_platform.fp8_dtype()
|
||||
output = torch.empty(
|
||||
(S, H, padded_head_dim),
|
||||
device=tensor.device,
|
||||
dtype=out_dtype,
|
||||
)
|
||||
|
||||
scale_1d = scale.reshape(-1)
|
||||
n_rows = S * H
|
||||
|
||||
if block_m is None or block_n is None or num_warps is None:
|
||||
block_m, block_n, num_warps = _get_fp8_pad_quant_config(padded_head_dim)
|
||||
|
||||
grid = (
|
||||
triton.cdiv(n_rows, block_m),
|
||||
triton.cdiv(padded_head_dim, block_n),
|
||||
)
|
||||
|
||||
_quantize_pad_fp8_kernel[grid](
|
||||
tensor,
|
||||
output,
|
||||
scale_1d,
|
||||
tensor.stride(0),
|
||||
tensor.stride(1),
|
||||
tensor.stride(2),
|
||||
output.stride(0),
|
||||
output.stride(1),
|
||||
output.stride(2),
|
||||
H,
|
||||
n_rows,
|
||||
D,
|
||||
padded_head_dim,
|
||||
_FP8_MIN,
|
||||
_FP8_MAX,
|
||||
SKIP_SCALE=skip_scale,
|
||||
BLOCK_M=block_m,
|
||||
BLOCK_N=block_n,
|
||||
num_warps=num_warps,
|
||||
)
|
||||
|
||||
return output.view((*original_shape[:-1], padded_head_dim))
|
||||
|
||||
|
||||
def quantize_fp8_maybe_pad_head_dim(
|
||||
tensor: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
fp8_quant: QuantFP8,
|
||||
skip_scale: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""Quantize a 3D/4D tensor to FP8, padding head_dim to a multiple of 16
|
||||
only when needed.
|
||||
|
||||
Accepts (S, H, D) or (B, S, H, D) input. Uses ``fp8_quant`` (a
|
||||
:class:`QuantFP8` CustomOp) when head_dim is already aligned to 16
|
||||
(no padding); otherwise falls back to a stride-aware Triton kernel
|
||||
that pads head_dim to a multiple of 16.
|
||||
"""
|
||||
head_dim = tensor.shape[-1]
|
||||
if head_dim % 16 != 0:
|
||||
return quantize_fp8_pad_head_dim_triton(tensor, scale, skip_scale=skip_scale)
|
||||
|
||||
if skip_scale:
|
||||
return tensor.to(current_platform.fp8_dtype())
|
||||
|
||||
# QuantFP8 expects 2D: flatten all dims except (H, D).
|
||||
orig_shape = tensor.shape
|
||||
total_tokens = tensor.numel() // (orig_shape[-1] * orig_shape[-2])
|
||||
tensor_2d = tensor.reshape(total_tokens, -1)
|
||||
fp8_tensor, _ = fp8_quant(tensor_2d, scale=scale)
|
||||
return fp8_tensor.reshape(orig_shape)
|
||||
@@ -1,13 +1,32 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import functools
|
||||
import json
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.config import MultiModalConfig
|
||||
from vllm.kernels.triton.qkv_padded_fp8_quant import (
|
||||
quantize_fp8_maybe_pad_head_dim,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import CustomOp, maybe_get_oot_by_class
|
||||
from vllm.model_executor.models.vision import get_vit_attn_backend
|
||||
from vllm.model_executor.layers.quantization.input_quant_fp8 import (
|
||||
QuantFP8,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
get_fp8_min_max,
|
||||
)
|
||||
from vllm.model_executor.models.vision import (
|
||||
get_multimodal_config,
|
||||
get_vit_attn_backend,
|
||||
)
|
||||
from vllm.utils.flashinfer import (
|
||||
is_flashinfer_cudnn_fp8_prefill_attn_supported,
|
||||
)
|
||||
from vllm.utils.math_utils import round_up
|
||||
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
@@ -20,6 +39,108 @@ from vllm.v1.attention.ops.vit_attn_wrappers import (
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_, _FP8_MAX = get_fp8_min_max()
|
||||
_FP8_AMAX_HISTORY_LEN = 16
|
||||
|
||||
# Module-level state for auto-saving dynamic scales. The save is a one-shot
|
||||
# triggered by the first layer whose amax buffer wraps. Path and margin are
|
||||
# captured during layer init (set_current_vllm_config context only lives
|
||||
# across model init, not forward passes).
|
||||
_fp8_scale_save_path: str | None = None
|
||||
_fp8_scale_save_margin: float = MultiModalConfig.mm_encoder_fp8_scale_save_margin
|
||||
_fp8_saved_scale_refs: dict[str, tuple[torch.Tensor, torch.Tensor, torch.Tensor]] = {}
|
||||
|
||||
|
||||
@functools.cache
|
||||
def _load_fp8_scales_file(path: str | None) -> dict[str, dict[str, float]]:
|
||||
"""Load per-layer FP8 Q/K/V scales from a JSON file. Results are cached.
|
||||
|
||||
Expected format (keys ``q_scale`` / ``k_scale`` / ``v_scale`` also accepted)::
|
||||
|
||||
{
|
||||
"visual.blocks.0.attn.attn": {"q": 224.0, "k": 198.0, "v": 210.0},
|
||||
"visual.blocks.1.attn.attn": {"q": 218.0, "k": 195.0, "v": 207.0},
|
||||
}
|
||||
|
||||
To produce such a file, run with ``mm_encoder_fp8_scale_save_path`` set.
|
||||
"""
|
||||
if path is None:
|
||||
return {}
|
||||
|
||||
with open(path, encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
|
||||
# Handle nested "layers" format
|
||||
if "layers" in data and isinstance(data["layers"], dict):
|
||||
data = data["layers"]
|
||||
|
||||
scales: dict[str, dict[str, float]] = {}
|
||||
for layer_name, layer_scales in data.items():
|
||||
if not isinstance(layer_scales, dict):
|
||||
continue
|
||||
q = layer_scales.get("q", layer_scales.get("q_scale"))
|
||||
k = layer_scales.get("k", layer_scales.get("k_scale"))
|
||||
v = layer_scales.get("v", layer_scales.get("v_scale"))
|
||||
if q is not None and k is not None and v is not None:
|
||||
q_f, k_f, v_f = float(q), float(k), float(v)
|
||||
if q_f <= 0 or k_f <= 0 or v_f <= 0:
|
||||
raise ValueError(
|
||||
f"FP8 scales must be positive, got q={q_f}, "
|
||||
f"k={k_f}, v={v_f} for layer '{layer_name}'"
|
||||
)
|
||||
scales[layer_name] = {"q": q_f, "k": k_f, "v": v_f}
|
||||
|
||||
logger.info_once(
|
||||
"Loaded FP8 attention scales from %s (%d layers)", path, len(scales)
|
||||
)
|
||||
return scales
|
||||
|
||||
|
||||
def _maybe_save_fp8_scales(
|
||||
layer_name: str,
|
||||
q_scale: torch.Tensor,
|
||||
k_scale: torch.Tensor,
|
||||
v_scale: torch.Tensor,
|
||||
buffer_wrapped: bool,
|
||||
) -> None:
|
||||
"""Accumulate a layer's scale tensors; on the first amax buffer wrap,
|
||||
dump all accumulated scales to ``mm_encoder_fp8_scale_save_path``.
|
||||
|
||||
No-op unless auto-save is configured. Tensor references are stored on
|
||||
every call (no GPU->CPU sync); ``.item()`` is only called at the single
|
||||
save point to avoid stalling the forward path.
|
||||
"""
|
||||
global _fp8_scale_save_path
|
||||
# Fast path: auto-save either disabled or already finished. Path is
|
||||
# captured at layer init and cleared once the save fires.
|
||||
if _fp8_scale_save_path is None:
|
||||
return
|
||||
|
||||
# Stash scale tensor refs (no GPU->CPU sync yet); wait until the amax
|
||||
# history has seen a full cycle before committing scales to disk.
|
||||
_fp8_saved_scale_refs[layer_name] = (q_scale, k_scale, v_scale)
|
||||
if not buffer_wrapped:
|
||||
return
|
||||
|
||||
# Buffer just wrapped for the first time: materialize scales (with
|
||||
# safety margin) and dump to disk. Clearing _fp8_scale_save_path
|
||||
# makes this a one-shot across all layers.
|
||||
path, margin = _fp8_scale_save_path, _fp8_scale_save_margin
|
||||
scales = {
|
||||
name: {
|
||||
"q": q.item() * margin,
|
||||
"k": k.item() * margin,
|
||||
"v": v.item() * margin,
|
||||
}
|
||||
for name, (q, k, v) in _fp8_saved_scale_refs.items()
|
||||
}
|
||||
_fp8_scale_save_path = None
|
||||
_fp8_saved_scale_refs.clear()
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
json.dump(scales, f, indent=2)
|
||||
logger.info("Saved FP8 scales (%d layers) to %s", len(scales), path)
|
||||
|
||||
|
||||
# Batch buckets for cuDNN graph caching.
|
||||
# Graphs use batch size and max sequence length as cache key.
|
||||
# This avoids creating a new graph for each unique set of
|
||||
@@ -148,27 +269,47 @@ class MMEncoderAttention(CustomOp):
|
||||
hidden_size: int,
|
||||
tp_size: int,
|
||||
device: torch.device,
|
||||
fp8_padded_hidden_size: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
if (oot_class := maybe_get_oot_by_class(cls)) is not cls:
|
||||
return oot_class.maybe_recompute_cu_seqlens( # type: ignore[attr-defined]
|
||||
attn_backend, cu_seqlens, hidden_size, tp_size, device
|
||||
attn_backend,
|
||||
cu_seqlens,
|
||||
hidden_size,
|
||||
tp_size,
|
||||
device,
|
||||
fp8_padded_hidden_size=fp8_padded_hidden_size,
|
||||
)
|
||||
|
||||
if attn_backend == AttentionBackendEnum.FLASHINFER:
|
||||
batch_size = len(cu_seqlens) - 1
|
||||
scale = hidden_size // tp_size
|
||||
cu_seqlens = cu_seqlens * scale
|
||||
|
||||
cu_seqlens_qko = cu_seqlens
|
||||
cu_seqlens_v = cu_seqlens * 3
|
||||
if fp8_padded_hidden_size is not None:
|
||||
# FP8 path: after quantization Q/K/V are each independent
|
||||
# contiguous tensors with stride H * padded_D per token.
|
||||
# All sections use the same element stride.
|
||||
scale = fp8_padded_hidden_size // tp_size
|
||||
cu_seqlens = cu_seqlens * scale
|
||||
cu_seqlens_padded = add_padding_to_seqlens(
|
||||
cu_seqlens, batch_size, cu_seqlens[-1]
|
||||
)
|
||||
cu_seqlens = np.concatenate([cu_seqlens_padded, cu_seqlens_padded])
|
||||
else:
|
||||
# BF16 path: Q/K/V are non-contiguous views into shared
|
||||
# buffers. V section has 3x stride from interleaved QKV.
|
||||
scale = hidden_size // tp_size
|
||||
cu_seqlens = cu_seqlens * scale
|
||||
|
||||
cu_seqlens_qko = add_padding_to_seqlens(
|
||||
cu_seqlens_qko, batch_size, cu_seqlens_qko[-1]
|
||||
)
|
||||
cu_seqlens_v = add_padding_to_seqlens(
|
||||
cu_seqlens_v, batch_size, cu_seqlens_v[-1]
|
||||
)
|
||||
cu_seqlens = np.concatenate([cu_seqlens_qko, cu_seqlens_v])
|
||||
cu_seqlens_qko = cu_seqlens
|
||||
cu_seqlens_v = cu_seqlens * 3
|
||||
|
||||
cu_seqlens_qko = add_padding_to_seqlens(
|
||||
cu_seqlens_qko, batch_size, cu_seqlens_qko[-1]
|
||||
)
|
||||
cu_seqlens_v = add_padding_to_seqlens(
|
||||
cu_seqlens_v, batch_size, cu_seqlens_v[-1]
|
||||
)
|
||||
cu_seqlens = np.concatenate([cu_seqlens_qko, cu_seqlens_v])
|
||||
|
||||
cu_seqlens = torch.from_numpy(cu_seqlens).to(device, non_blocking=True)
|
||||
return cu_seqlens
|
||||
@@ -206,6 +347,7 @@ class MMEncoderAttention(CustomOp):
|
||||
# During model initialization, the default dtype is set as the model
|
||||
# weight and activation dtype.
|
||||
dtype = torch.get_default_dtype()
|
||||
self.dtype = dtype
|
||||
|
||||
# Get device-specific vision attention backend.
|
||||
self.attn_backend = get_vit_attn_backend(
|
||||
@@ -229,6 +371,113 @@ class MMEncoderAttention(CustomOp):
|
||||
|
||||
logger.info_once(f"Using {self.attn_backend} for MMEncoderAttention.")
|
||||
|
||||
self._init_fp8_state()
|
||||
|
||||
def _init_fp8_state(self) -> None:
|
||||
"""Initialize FP8 attention state from multimodal config.
|
||||
|
||||
No-op if FP8 is not requested. Raises ``ValueError`` if FP8 is
|
||||
requested but the platform does not support it.
|
||||
"""
|
||||
# Populate defaults so ``_forward_flashinfer`` can
|
||||
# check ``self.fp8_enabled`` and others without AttributeError.
|
||||
self.fp8_enabled = False
|
||||
self._fp8_dynamic_scale = False
|
||||
self.fp8_quant: QuantFP8 | None = None
|
||||
self.skip_scale_q = False
|
||||
self.skip_scale_k = False
|
||||
self.skip_scale_v = False
|
||||
|
||||
mm_cfg = get_multimodal_config()
|
||||
if mm_cfg is None or mm_cfg.mm_encoder_attn_dtype != "fp8":
|
||||
return
|
||||
|
||||
# FP8 path
|
||||
if not is_flashinfer_cudnn_fp8_prefill_attn_supported():
|
||||
raise ValueError(
|
||||
"mm_encoder_attn_dtype='fp8' requires the FlashInfer "
|
||||
"cuDNN backend with cuDNN >= 9.17.1 on a GPU with native "
|
||||
"FP8 support."
|
||||
)
|
||||
|
||||
self.fp8_enabled = True
|
||||
self._fp8_dynamic_scale = mm_cfg.mm_encoder_fp8_scale_path is None
|
||||
self.fp8_quant = QuantFP8(static=True, group_shape=GroupShape.PER_TENSOR)
|
||||
|
||||
# Register buffers pre-device-move; values populated in
|
||||
# process_weights_after_loading. Shape (1, 1, 1, 1) is required by cuDNN.
|
||||
for attr in ("_fp8_q_scale", "_fp8_k_scale", "_fp8_v_scale"):
|
||||
self.register_buffer(
|
||||
attr, torch.ones(1, dtype=torch.float32).view(1, 1, 1, 1)
|
||||
)
|
||||
if self._fp8_dynamic_scale:
|
||||
for attr in ("_fp8_q_amax", "_fp8_k_amax", "_fp8_v_amax"):
|
||||
self.register_buffer(
|
||||
attr,
|
||||
torch.zeros(_FP8_AMAX_HISTORY_LEN, dtype=torch.float32),
|
||||
persistent=False,
|
||||
)
|
||||
self._fp8_amax_pos = 0
|
||||
|
||||
# Capture auto-save config now: the VllmConfig context only lives
|
||||
# across model init, not forward passes, so ``_maybe_save_fp8_scales``
|
||||
# reads these globals instead of re-querying ``get_multimodal_config``.
|
||||
if (
|
||||
mm_cfg.mm_encoder_fp8_scale_save_path is not None
|
||||
and self._fp8_dynamic_scale
|
||||
):
|
||||
global _fp8_scale_save_path, _fp8_scale_save_margin
|
||||
_fp8_scale_save_path = mm_cfg.mm_encoder_fp8_scale_save_path
|
||||
_fp8_scale_save_margin = mm_cfg.mm_encoder_fp8_scale_save_margin
|
||||
|
||||
def process_weights_after_loading(self, act_dtype: torch.dtype) -> None:
|
||||
"""Populate FP8 scale buffers after weights are loaded.
|
||||
|
||||
``act_dtype`` matches the signature used by :class:`Attention` and
|
||||
:class:`MLAAttention` for the loader auto-scan but is unused:
|
||||
FP8 scales are always float32.
|
||||
"""
|
||||
if not self.fp8_enabled:
|
||||
return
|
||||
|
||||
mm_cfg = get_multimodal_config()
|
||||
scale_path = mm_cfg.mm_encoder_fp8_scale_path if mm_cfg is not None else None
|
||||
if scale_path is None:
|
||||
logger.info_once(
|
||||
"FP8 attention enabled with dynamic scaling "
|
||||
"(no scale file provided). Scales will adapt from "
|
||||
"observed Q/K/V amax values (history_len=%d).",
|
||||
_FP8_AMAX_HISTORY_LEN,
|
||||
)
|
||||
return
|
||||
|
||||
all_scales = _load_fp8_scales_file(scale_path)
|
||||
layer_scales = all_scales.get(self.layer_name)
|
||||
if layer_scales is None:
|
||||
raise ValueError(
|
||||
"FP8 attention enabled but scales not found for layer "
|
||||
f"'{self.layer_name}' in {scale_path}. "
|
||||
f"Available layers: {list(all_scales.keys())}"
|
||||
)
|
||||
|
||||
for attr, key in (
|
||||
("_fp8_q_scale", "q"),
|
||||
("_fp8_k_scale", "k"),
|
||||
("_fp8_v_scale", "v"),
|
||||
):
|
||||
getattr(self, attr).fill_(layer_scales[key])
|
||||
self.skip_scale_q = layer_scales["q"] == 1.0
|
||||
self.skip_scale_k = layer_scales["k"] == 1.0
|
||||
self.skip_scale_v = layer_scales["v"] == 1.0
|
||||
|
||||
logger.debug(
|
||||
"FP8 attention enabled for %s: q=%.4f, k=%.4f, v=%.4f",
|
||||
self.layer_name if self.layer_name else "MMEncoderAttention",
|
||||
layer_scales["q"],
|
||||
layer_scales["k"],
|
||||
layer_scales["v"],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def enabled(cls) -> bool:
|
||||
return True
|
||||
@@ -353,6 +602,44 @@ class MMEncoderAttention(CustomOp):
|
||||
output = output.reshape(bsz, q_len, -1)
|
||||
return output
|
||||
|
||||
@torch.no_grad()
|
||||
def _record_amax_and_update_scales(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
) -> None:
|
||||
"""Record Q/K/V amax into circular history and recompute scales.
|
||||
|
||||
All work stays on GPU with no device-to-host sync. The Python-side
|
||||
history position counter is mutated, so this method must NOT be
|
||||
called inside CUDA graph capture/replay. When CUDA graphs are
|
||||
used for the encoder, dynamic scaling should be disabled by
|
||||
providing a static scale file via --mm-encoder-fp8-scale-path.
|
||||
"""
|
||||
pos = self._fp8_amax_pos
|
||||
self._fp8_amax_pos = (pos + 1) % _FP8_AMAX_HISTORY_LEN
|
||||
|
||||
for tensor, amax_buf, scale_buf in (
|
||||
(query, self._fp8_q_amax, self._fp8_q_scale),
|
||||
(key, self._fp8_k_amax, self._fp8_k_scale),
|
||||
(value, self._fp8_v_amax, self._fp8_v_scale),
|
||||
):
|
||||
amax_buf[pos] = tensor.amax()
|
||||
max_amax = amax_buf.max()
|
||||
scale_buf.fill_(
|
||||
torch.clamp(max_amax, min=torch.finfo(torch.float32).tiny) / _FP8_MAX
|
||||
)
|
||||
|
||||
buffer_wrapped = self._fp8_amax_pos == 0 and pos == _FP8_AMAX_HISTORY_LEN - 1
|
||||
_maybe_save_fp8_scales(
|
||||
self.layer_name,
|
||||
self._fp8_q_scale,
|
||||
self._fp8_k_scale,
|
||||
self._fp8_v_scale,
|
||||
buffer_wrapped,
|
||||
)
|
||||
|
||||
def _forward_flashinfer(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
@@ -363,7 +650,32 @@ class MMEncoderAttention(CustomOp):
|
||||
sequence_lengths: torch.Tensor
|
||||
| None = None, # Only used for FlashInfer CuDNN backend
|
||||
) -> torch.Tensor:
|
||||
return vit_flashinfer_wrapper(
|
||||
if self.fp8_enabled:
|
||||
assert self.fp8_quant is not None
|
||||
|
||||
if self._fp8_dynamic_scale:
|
||||
self._record_amax_and_update_scales(query, key, value)
|
||||
|
||||
query = quantize_fp8_maybe_pad_head_dim(
|
||||
query,
|
||||
self._fp8_q_scale,
|
||||
skip_scale=self.skip_scale_q,
|
||||
fp8_quant=self.fp8_quant,
|
||||
)
|
||||
key = quantize_fp8_maybe_pad_head_dim(
|
||||
key,
|
||||
self._fp8_k_scale,
|
||||
skip_scale=self.skip_scale_k,
|
||||
fp8_quant=self.fp8_quant,
|
||||
)
|
||||
value = quantize_fp8_maybe_pad_head_dim(
|
||||
value,
|
||||
self._fp8_v_scale,
|
||||
skip_scale=self.skip_scale_v,
|
||||
fp8_quant=self.fp8_quant,
|
||||
)
|
||||
|
||||
output = vit_flashinfer_wrapper(
|
||||
q=query,
|
||||
k=key,
|
||||
v=value,
|
||||
@@ -372,8 +684,17 @@ class MMEncoderAttention(CustomOp):
|
||||
cu_seqlens=cu_seqlens,
|
||||
max_seqlen=max_seqlen,
|
||||
sequence_lengths=sequence_lengths,
|
||||
q_scale=self._fp8_q_scale if self.fp8_enabled else None,
|
||||
k_scale=self._fp8_k_scale if self.fp8_enabled else None,
|
||||
v_scale=self._fp8_v_scale if self.fp8_enabled else None,
|
||||
o_data_type=self.dtype if self.fp8_enabled else None,
|
||||
)
|
||||
|
||||
if self.fp8_enabled and output.shape[-1] != self.head_size:
|
||||
output = output[..., : self.head_size].contiguous()
|
||||
|
||||
return output
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
|
||||
@@ -15,7 +15,11 @@ from typing_extensions import assert_never
|
||||
import vllm.envs as envs
|
||||
from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention, MLAAttention
|
||||
from vllm.model_executor.layers.attention import (
|
||||
Attention,
|
||||
MLAAttention,
|
||||
MMEncoderAttention,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
QuantizeMethodBase,
|
||||
@@ -106,12 +110,12 @@ def process_weights_after_loading(
|
||||
with device_loading_context(module, target_device):
|
||||
quant_method.process_weights_after_loading(module)
|
||||
|
||||
# Initialize post-load attention weights for both Attention and MLA.
|
||||
# Initialize post-load attention weights for Attention, MLA, and MM encoder.
|
||||
# NOTE: Happens after other modules so we can easily decompress weights.
|
||||
for _, module in model.named_modules():
|
||||
if isinstance(module, (Attention, MLAAttention)) and hasattr(
|
||||
module, "process_weights_after_loading"
|
||||
):
|
||||
if isinstance(
|
||||
module, (Attention, MLAAttention, MMEncoderAttention)
|
||||
) and hasattr(module, "process_weights_after_loading"):
|
||||
# TODO(lucas): see if there is a way to unify the signatures
|
||||
# of process_weights_after_loading
|
||||
with device_loading_context(module, target_device):
|
||||
|
||||
@@ -136,6 +136,7 @@ from .utils import (
|
||||
maybe_prefix,
|
||||
)
|
||||
from .vision import (
|
||||
get_fp8_padded_hidden_size,
|
||||
get_vit_attn_backend,
|
||||
is_vit_use_data_parallel,
|
||||
run_dp_sharded_mrope_vision_model,
|
||||
@@ -562,6 +563,13 @@ class Qwen3_VisionTransformer(nn.Module):
|
||||
|
||||
norm_layer = partial(nn.LayerNorm, eps=norm_eps)
|
||||
head_dim = self.hidden_size // self.num_heads
|
||||
|
||||
# FP8 attention: Q/K/V become independent contiguous tensors
|
||||
# after quantization, so cu_seqlens uses uniform stride (no 3x V).
|
||||
self.fp8_padded_hidden_size = get_fp8_padded_hidden_size(
|
||||
self.num_heads, head_dim
|
||||
)
|
||||
|
||||
self.rotary_pos_emb = get_rope(
|
||||
head_size=head_dim,
|
||||
max_position=8192,
|
||||
@@ -776,6 +784,7 @@ class Qwen3_VisionTransformer(nn.Module):
|
||||
self.hidden_size,
|
||||
self.tp_size,
|
||||
device,
|
||||
fp8_padded_hidden_size=self.fp8_padded_hidden_size,
|
||||
)
|
||||
|
||||
return metadata
|
||||
|
||||
@@ -10,7 +10,7 @@ from typing import Final, Generic, Literal, Protocol, TypeAlias, TypeVar
|
||||
import torch
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.config import MultiModalConfig, VllmConfig, get_current_vllm_config
|
||||
from vllm.config import MultiModalConfig, get_current_vllm_config_or_none
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
@@ -18,6 +18,7 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import round_up
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -102,45 +103,48 @@ def get_vit_attn_backend(
|
||||
"""
|
||||
Get the attention backend for Vision Transformer.
|
||||
"""
|
||||
try:
|
||||
vllm_config: VllmConfig = get_current_vllm_config()
|
||||
model_config = vllm_config.model_config
|
||||
multimodal_config: MultiModalConfig | None = (
|
||||
model_config.multimodal_config if model_config is not None else None
|
||||
)
|
||||
except (AssertionError, AttributeError):
|
||||
multimodal_config = None
|
||||
|
||||
mm_cfg = get_multimodal_config()
|
||||
attn_backend_override = (
|
||||
multimodal_config.mm_encoder_attn_backend
|
||||
if multimodal_config is not None
|
||||
else None
|
||||
mm_cfg.mm_encoder_attn_backend if mm_cfg is not None else None
|
||||
)
|
||||
attn_backend = _get_vit_attn_backend(
|
||||
return _get_vit_attn_backend(
|
||||
head_size,
|
||||
dtype,
|
||||
attn_backend_override=attn_backend_override,
|
||||
)
|
||||
return attn_backend
|
||||
|
||||
|
||||
def get_multimodal_config() -> MultiModalConfig | None:
|
||||
"""Return the current ``MultiModalConfig``, or ``None`` when no engine
|
||||
config context is active (e.g., during unit tests) or when the current
|
||||
``model_config`` does not carry a ``multimodal_config`` (e.g., minimal
|
||||
stubs used in tests)."""
|
||||
vllm_config = get_current_vllm_config_or_none()
|
||||
if vllm_config is None or vllm_config.model_config is None:
|
||||
return None
|
||||
return getattr(vllm_config.model_config, "multimodal_config", None)
|
||||
|
||||
|
||||
def get_fp8_padded_hidden_size(num_heads: int, head_dim: int) -> int | None:
|
||||
"""Return the padded hidden size for FP8 ViT encoder attention, or
|
||||
``None`` when FP8 is not enabled.
|
||||
|
||||
cuDNN FP8 prefill attention requires ``head_dim`` to be a multiple of
|
||||
16. For non-aligned ``head_dim`` (e.g. 72), Q/K/V are padded to the
|
||||
nearest multiple of 16.
|
||||
"""
|
||||
mm_cfg = get_multimodal_config()
|
||||
if mm_cfg is None or mm_cfg.mm_encoder_attn_dtype != "fp8":
|
||||
return None
|
||||
return num_heads * round_up(head_dim, 16)
|
||||
|
||||
|
||||
def is_vit_use_data_parallel():
|
||||
"""
|
||||
Get the tensor parallel type for Vision Transformer.
|
||||
"""
|
||||
try:
|
||||
vllm_config: VllmConfig = get_current_vllm_config()
|
||||
model_config = vllm_config.model_config
|
||||
multimodal_config: MultiModalConfig | None = (
|
||||
model_config.multimodal_config if model_config is not None else None
|
||||
)
|
||||
except (AssertionError, AttributeError):
|
||||
multimodal_config = None
|
||||
|
||||
mm_encoder_tp_mode = (
|
||||
multimodal_config.mm_encoder_tp_mode if multimodal_config is not None else None
|
||||
)
|
||||
return mm_encoder_tp_mode == "data"
|
||||
mm_cfg = get_multimodal_config()
|
||||
return mm_cfg is not None and mm_cfg.mm_encoder_tp_mode == "data"
|
||||
|
||||
|
||||
VisionFeatureSelectStrategyStr = Literal["class", "default", "full"]
|
||||
|
||||
@@ -772,6 +772,40 @@ def should_use_flashinfer_for_blockscale_fp8_gemm(
|
||||
return should_use_flashinfer
|
||||
|
||||
|
||||
_MIN_CUDNN_FP8 = 91701 # cuDNN >= 9.17.1 required for FP8 attention
|
||||
|
||||
|
||||
@functools.cache
|
||||
def is_flashinfer_cudnn_fp8_prefill_attn_supported() -> bool:
|
||||
"""Check if FP8 ViT attention is supported on this platform.
|
||||
|
||||
Requires native FP8 hardware support, the FlashInfer cuDNN backend,
|
||||
and cuDNN >= 9.17.1.
|
||||
"""
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
# cuDNN SDPA FP8 requires Hopper (SM 90) or newer.
|
||||
if not current_platform.has_device_capability(90):
|
||||
return False
|
||||
|
||||
try:
|
||||
supported = current_platform.get_supported_vit_attn_backends()
|
||||
if AttentionBackendEnum.FLASHINFER not in supported:
|
||||
return False
|
||||
except (ImportError, AttributeError):
|
||||
return False
|
||||
|
||||
try:
|
||||
import torch.backends.cudnn as cudnn
|
||||
|
||||
if cudnn.is_available() and cudnn.version() < _MIN_CUDNN_FP8:
|
||||
return False
|
||||
except (ImportError, AttributeError):
|
||||
pass
|
||||
|
||||
return True
|
||||
|
||||
|
||||
__all__ = [
|
||||
"has_flashinfer",
|
||||
"flashinfer_trtllm_fp8_block_scale_moe",
|
||||
@@ -803,4 +837,5 @@ __all__ = [
|
||||
"flashinfer_fp8_blockscale_gemm",
|
||||
"should_use_flashinfer_for_blockscale_fp8_gemm",
|
||||
"is_flashinfer_fp8_blockscale_gemm_supported",
|
||||
"is_flashinfer_cudnn_fp8_prefill_attn_supported",
|
||||
]
|
||||
|
||||
@@ -279,6 +279,10 @@ def flashinfer_wrapper(
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
max_seqlen: torch.Tensor | None = None,
|
||||
sequence_lengths: torch.Tensor | None = None,
|
||||
q_scale: torch.Tensor | None = None,
|
||||
k_scale: torch.Tensor | None = None,
|
||||
v_scale: torch.Tensor | None = None,
|
||||
o_data_type: torch.dtype | None = None,
|
||||
) -> torch.Tensor:
|
||||
from flashinfer.prefill import cudnn_batch_prefill_with_kv_cache
|
||||
|
||||
@@ -318,6 +322,10 @@ def flashinfer_wrapper(
|
||||
batch_offsets_k=batch_offsets_qko,
|
||||
batch_offsets_v=batch_offsets_v,
|
||||
batch_offsets_o=batch_offsets_qko,
|
||||
q_scale=q_scale,
|
||||
k_scale=k_scale,
|
||||
v_scale=v_scale,
|
||||
o_data_type=o_data_type,
|
||||
)
|
||||
|
||||
if is_reshaped:
|
||||
@@ -335,8 +343,12 @@ def vit_flashinfer_wrapper_fake(
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
max_seqlen: torch.Tensor | None = None,
|
||||
sequence_lengths: torch.Tensor | None = None,
|
||||
q_scale: torch.Tensor | None = None,
|
||||
k_scale: torch.Tensor | None = None,
|
||||
v_scale: torch.Tensor | None = None,
|
||||
o_data_type: torch.dtype | None = None,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty_like(q)
|
||||
return torch.empty_like(q, dtype=o_data_type or q.dtype)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
@@ -355,7 +367,22 @@ def vit_flashinfer_wrapper(
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
max_seqlen: torch.Tensor | None = None,
|
||||
sequence_lengths: torch.Tensor | None = None,
|
||||
q_scale: torch.Tensor | None = None,
|
||||
k_scale: torch.Tensor | None = None,
|
||||
v_scale: torch.Tensor | None = None,
|
||||
o_data_type: torch.dtype | None = None,
|
||||
) -> torch.Tensor:
|
||||
return torch.ops.vllm.flashinfer_wrapper(
|
||||
q, k, v, scale, workspace_buffer, cu_seqlens, max_seqlen, sequence_lengths
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
scale,
|
||||
workspace_buffer,
|
||||
cu_seqlens,
|
||||
max_seqlen,
|
||||
sequence_lengths,
|
||||
q_scale,
|
||||
k_scale,
|
||||
v_scale,
|
||||
o_data_type,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user