Use slow tokenizer for LLaMA (#84)

This commit is contained in:
Woosuk Kwon
2023-05-09 16:03:44 -07:00
committed by GitHub
parent add055e151
commit 85eb631839
3 changed files with 26 additions and 5 deletions
+2 -3
View File
@@ -1,8 +1,7 @@
import time
from typing import List, Optional, Tuple
from transformers import AutoTokenizer
from cacheflow.frontend.utils import get_tokenizer
from cacheflow.logger import init_logger
from cacheflow.sampling_params import SamplingParams
from cacheflow.sequence import Sequence, SequenceGroup
@@ -21,7 +20,7 @@ class SimpleFrontend:
) -> None:
self.block_size = block_size
self.tokenizer = AutoTokenizer.from_pretrained(model_name)
self.tokenizer = get_tokenizer(model_name)
self.seq_group_counter = Counter()
self.seq_counter = Counter()
self.inputs: List[Tuple[SequenceGroup, SamplingParams]] = []