Support FP8-E5M2 KV Cache (#2279)

Co-authored-by: zhaoyang <zhao.yang16@zte.com.cn>
Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>
This commit is contained in:
zhaoyang-star
2024-01-28 16:43:54 -08:00
committed by GitHub
co-authored by zhaoyang Zhuohan Li
parent 7d648418b8
commit 9090bf02e7
26 changed files with 912 additions and 196 deletions
+3
View File
@@ -98,6 +98,7 @@ class PagedAttention(nn.Module):
key_cache,
value_cache,
input_metadata.slot_mapping.flatten(),
input_metadata.kv_cache_dtype,
)
if input_metadata.is_prompt:
@@ -265,6 +266,7 @@ def _paged_attention(
block_size,
input_metadata.max_context_len,
alibi_slopes,
input_metadata.kv_cache_dtype,
)
else:
# Run PagedAttention V2.
@@ -295,5 +297,6 @@ def _paged_attention(
block_size,
input_metadata.max_context_len,
alibi_slopes,
input_metadata.kv_cache_dtype,
)
return output