[Core] Add generic typing to LRUCache (#3511)

This commit is contained in:
Nick Hill
2024-03-20 00:36:09 -07:00
committed by GitHub
parent 9474e89ba4
commit 4ad521d8b5
4 changed files with 29 additions and 22 deletions
@@ -21,10 +21,8 @@ class TokenizerGroup(BaseTokenizerGroup):
self.enable_lora = enable_lora
self.max_input_length = max_input_length
self.tokenizer = get_tokenizer(self.tokenizer_id, **tokenizer_config)
if enable_lora:
self.lora_tokenizers = LRUCache(capacity=max_num_seqs)
else:
self.lora_tokenizers = None
self.lora_tokenizers = LRUCache[PreTrainedTokenizer](
capacity=max_num_seqs) if enable_lora else None
def ping(self) -> bool:
"""Check if the tokenizer group is alive."""