Support FP8-E5M2 KV Cache (#2279)

Co-authored-by: zhaoyang <zhao.yang16@zte.com.cn>
Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>
This commit is contained in:
zhaoyang-star
2024-01-28 16:43:54 -08:00
committed by GitHub
co-authored by zhaoyang Zhuohan Li
parent 7d648418b8
commit 9090bf02e7
26 changed files with 912 additions and 196 deletions
+5 -1
View File
@@ -12,6 +12,7 @@ class InputMetadata:
max_context_len: The maximum context length.
context_lens: the length of attention context for each sequence.
block_tables: The block tables. (Seq id -> list of physical block)
kv_cache_dtype: Data type to store kv cache.
"""
def __init__(
@@ -25,6 +26,7 @@ class InputMetadata:
context_lens: Optional[torch.Tensor],
block_tables: Optional[torch.Tensor],
use_cuda_graph: bool,
kv_cache_dtype: str,
) -> None:
self.is_prompt = is_prompt
self.prompt_lens = prompt_lens
@@ -35,6 +37,7 @@ class InputMetadata:
self.context_lens = context_lens
self.block_tables = block_tables
self.use_cuda_graph = use_cuda_graph
self.kv_cache_dtype = kv_cache_dtype
# Set during the execution of the first attention op.
# FIXME(woosuk): This is a hack.
@@ -47,4 +50,5 @@ class InputMetadata:
f"slot_mapping={self.slot_mapping}, "
f"context_lens={self.context_lens}, "
f"block_tables={self.block_tables}, "
f"use_cuda_graph={self.use_cuda_graph})")
f"use_cuda_graph={self.use_cuda_graph}, "
f"kv_cache_dtype={self.kv_cache_dtype})")
+3
View File
@@ -98,6 +98,7 @@ class PagedAttention(nn.Module):
key_cache,
value_cache,
input_metadata.slot_mapping.flatten(),
input_metadata.kv_cache_dtype,
)
if input_metadata.is_prompt:
@@ -265,6 +266,7 @@ def _paged_attention(
block_size,
input_metadata.max_context_len,
alibi_slopes,
input_metadata.kv_cache_dtype,
)
else:
# Run PagedAttention V2.
@@ -295,5 +297,6 @@ def _paged_attention(
block_size,
input_metadata.max_context_len,
alibi_slopes,
input_metadata.kv_cache_dtype,
)
return output