[Tokenizer] Add an option to specify tokenizer (#284)

This commit is contained in:
Woosuk Kwon
2023-06-28 09:46:58 -07:00
committed by GitHub
parent bdd6b4c8bc
commit 4338cc4750
10 changed files with 61 additions and 60 deletions
+2 -9
View File
@@ -24,20 +24,13 @@ from typing import AsyncGenerator, List, Tuple
import aiohttp
import numpy as np
from transformers import AutoConfig, AutoTokenizer, PreTrainedTokenizerBase
from transformers import PreTrainedTokenizerBase
from vllm.transformers_utils.tokenizer import get_tokenizer
# (prompt len, output len, latency)
REQUEST_LATENCY: List[Tuple[int, int, float]] = []
def get_tokenizer(model_name: str) -> PreTrainedTokenizerBase:
config = AutoConfig.from_pretrained(model_name)
if config.model_type == "llama":
# A workaround for potential protobuf errors.
model_name = "hf-internal-testing/llama-tokenizer"
return AutoTokenizer.from_pretrained(model_name)
def sample_requests(
dataset_path: str,
num_requests: int,