[Misc] Use vllm-flash-attn instead of flash-attn (#4686)

This commit is contained in:
Woosuk Kwon
2024-05-08 13:15:34 -07:00
committed by GitHub
parent 230c4b38c1
commit 89579a201f
6 changed files with 16 additions and 31 deletions
+1 -1
View File
@@ -8,7 +8,7 @@ from dataclasses import dataclass
from typing import List, Optional, Tuple, Type
import torch
from flash_attn import flash_attn_varlen_func
from vllm_flash_attn import flash_attn_varlen_func
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
AttentionMetadata,
+1 -1
View File
@@ -3,8 +3,8 @@ from typing import Any, Dict, List, Optional, Set, Tuple, Type
import flashinfer
import torch
from flash_attn import flash_attn_varlen_func
from flashinfer import BatchDecodeWithPagedKVCacheWrapper
from vllm_flash_attn import flash_attn_varlen_func
from vllm import _custom_ops as ops
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
+4 -3
View File
@@ -76,11 +76,12 @@ def _which_attn_to_use(dtype: torch.dtype) -> _Backend:
return _Backend.XFORMERS
try:
import flash_attn # noqa: F401
import vllm_flash_attn # noqa: F401
except ImportError:
logger.info(
"Cannot use FlashAttention-2 backend because the flash_attn "
"package is not found. Please install it for better performance.")
"Cannot use FlashAttention-2 backend because the vllm_flash_attn "
"package is not found. `pip install vllm-flash-attn` for better "
"performance.")
return _Backend.XFORMERS
backend_by_env_var = envs.VLLM_ATTENTION_BACKEND