[Kernel] Use flash-attn for decoding (#3648)

Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Co-authored-by: LiuXiaoxuanPKU <lilyliupku@gmail.com>
This commit is contained in:
Stephen Krider
2024-05-13 15:50:33 -07:00
committed by GitHub
co-authored by Woosuk Kwon LiuXiaoxuanPKU
parent ce532ff45c
commit 1356df53bd
6 changed files with 313 additions and 65 deletions
+1 -1
View File
@@ -12,7 +12,7 @@ MODELS = [
# "Deci/DeciLM-7b", # Broken
# "tiiuae/falcon-7b", # Broken
"EleutherAI/gpt-j-6b",
"mosaicml/mpt-7b",
# "mosaicml/mpt-7b", # Broken
# "Qwen/Qwen1.5-0.5B" # Broken,
]