[Perf] FP8 FlashInfer Attn for ViT (#38065)
Signed-off-by: Zhanda Zhu <zhandazhu@gmail.com> Co-authored-by: Yubo Gao <ybgao-nvidia@users.noreply.github.com>
This commit is contained in:
@@ -772,6 +772,40 @@ def should_use_flashinfer_for_blockscale_fp8_gemm(
|
||||
return should_use_flashinfer
|
||||
|
||||
|
||||
_MIN_CUDNN_FP8 = 91701 # cuDNN >= 9.17.1 required for FP8 attention
|
||||
|
||||
|
||||
@functools.cache
|
||||
def is_flashinfer_cudnn_fp8_prefill_attn_supported() -> bool:
|
||||
"""Check if FP8 ViT attention is supported on this platform.
|
||||
|
||||
Requires native FP8 hardware support, the FlashInfer cuDNN backend,
|
||||
and cuDNN >= 9.17.1.
|
||||
"""
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
# cuDNN SDPA FP8 requires Hopper (SM 90) or newer.
|
||||
if not current_platform.has_device_capability(90):
|
||||
return False
|
||||
|
||||
try:
|
||||
supported = current_platform.get_supported_vit_attn_backends()
|
||||
if AttentionBackendEnum.FLASHINFER not in supported:
|
||||
return False
|
||||
except (ImportError, AttributeError):
|
||||
return False
|
||||
|
||||
try:
|
||||
import torch.backends.cudnn as cudnn
|
||||
|
||||
if cudnn.is_available() and cudnn.version() < _MIN_CUDNN_FP8:
|
||||
return False
|
||||
except (ImportError, AttributeError):
|
||||
pass
|
||||
|
||||
return True
|
||||
|
||||
|
||||
__all__ = [
|
||||
"has_flashinfer",
|
||||
"flashinfer_trtllm_fp8_block_scale_moe",
|
||||
@@ -803,4 +837,5 @@ __all__ = [
|
||||
"flashinfer_fp8_blockscale_gemm",
|
||||
"should_use_flashinfer_for_blockscale_fp8_gemm",
|
||||
"is_flashinfer_fp8_blockscale_gemm_supported",
|
||||
"is_flashinfer_cudnn_fp8_prefill_attn_supported",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user