[Tokenizer] Add tokenizer mode (#298)

This commit is contained in:
Woosuk Kwon
2023-06-28 14:19:22 -07:00
committed by GitHub
parent 425040d4c1
commit 998d9d1509
6 changed files with 38 additions and 5 deletions
+4
View File
@@ -26,6 +26,8 @@ class LLM:
Args:
model: The name or path of a HuggingFace Transformers model.
tokenizer: The name or path of a HuggingFace Transformers tokenizer.
tokenizer_mode: The tokenizer mode. "auto" will use the fast tokenizer
if available, and "slow" will always use the slow tokenizer.
tensor_parallel_size: The number of GPUs to use for distributed
execution with tensor parallelism.
dtype: The data type for the model weights and activations. Currently,
@@ -40,6 +42,7 @@ class LLM:
self,
model: str,
tokenizer: Optional[str] = None,
tokenizer_mode: str = "auto",
tensor_parallel_size: int = 1,
dtype: str = "auto",
seed: int = 0,
@@ -50,6 +53,7 @@ class LLM:
engine_args = EngineArgs(
model=model,
tokenizer=tokenizer,
tokenizer_mode=tokenizer_mode,
tensor_parallel_size=tensor_parallel_size,
dtype=dtype,
seed=seed,
+1 -1
View File
@@ -313,7 +313,7 @@ if __name__ == "__main__":
engine = AsyncLLMEngine.from_engine_args(engine_args)
# A separate tokenizer to map token IDs to strings.
tokenizer = get_tokenizer(args.model)
tokenizer = get_tokenizer(engine_args.tokenizer, engine_args.tokenizer_mode)
uvicorn.run(app, host=args.host, port=args.port, log_level="info",
timeout_keep_alive=TIMEOUT_KEEP_ALIVE)