[Experimental] Prefix Caching Support (#1669)
Co-authored-by: DouHappy <2278958187@qq.com> Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>
This commit is contained in:
co-authored by
DouHappy
Zhuohan Li
parent
14cc317ba4
commit
d10f8e1d43
@@ -18,12 +18,18 @@ class InputMetadata:
|
||||
self,
|
||||
is_prompt: bool,
|
||||
slot_mapping: torch.Tensor,
|
||||
prompt_lens: Optional[torch.Tensor],
|
||||
max_seq_len: Optional[int],
|
||||
start_loc: Optional[torch.Tensor],
|
||||
max_context_len: Optional[int],
|
||||
context_lens: Optional[torch.Tensor],
|
||||
block_tables: Optional[torch.Tensor],
|
||||
use_cuda_graph: bool,
|
||||
) -> None:
|
||||
self.is_prompt = is_prompt
|
||||
self.prompt_lens = prompt_lens
|
||||
self.max_seq_len = max_seq_len
|
||||
self.start_loc = start_loc
|
||||
self.max_context_len = max_context_len
|
||||
self.slot_mapping = slot_mapping
|
||||
self.context_lens = context_lens
|
||||
|
||||
Reference in New Issue
Block a user