Add Model Revision Support (#1014)

Co-authored-by: Jasmond Loh <Jasmond.Loh@hotmail.com>
Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>
This commit is contained in:
Jasmond L
2023-09-13 15:20:02 -07:00
committed by GitHub
co-authored by Jasmond Loh Zhuohan Li
parent 9841d48a10
commit ab019eea75
20 changed files with 75 additions and 35 deletions
+10 -1
View File
@@ -28,6 +28,7 @@ class EngineArgs:
max_num_batched_tokens: int = 2560
max_num_seqs: int = 256
disable_log_stats: bool = False
revision: Optional[str] = None
def __post_init__(self):
if self.tokenizer is None:
@@ -49,6 +50,13 @@ class EngineArgs:
type=str,
default=EngineArgs.tokenizer,
help='name or path of the huggingface tokenizer to use')
parser.add_argument(
'--revision',
type=str,
default=None,
help='the specific model version to use. It can be a branch '
'name, a tag name, or a commit id. If unspecified, will use '
'the default version.')
parser.add_argument('--tokenizer-mode',
type=str,
default=EngineArgs.tokenizer_mode,
@@ -159,7 +167,8 @@ class EngineArgs:
model_config = ModelConfig(self.model, self.tokenizer,
self.tokenizer_mode, self.trust_remote_code,
self.download_dir, self.load_format,
self.dtype, self.seed, self.max_model_len)
self.dtype, self.seed, self.revision,
self.max_model_len)
cache_config = CacheConfig(self.block_size,
self.gpu_memory_utilization,
self.swap_space)
+3 -1
View File
@@ -74,6 +74,7 @@ class LLMEngine:
f"model={model_config.model!r}, "
f"tokenizer={model_config.tokenizer!r}, "
f"tokenizer_mode={model_config.tokenizer_mode}, "
f"revision={model_config.revision}, "
f"trust_remote_code={model_config.trust_remote_code}, "
f"dtype={model_config.dtype}, "
f"download_dir={model_config.download_dir!r}, "
@@ -92,7 +93,8 @@ class LLMEngine:
self.tokenizer = get_tokenizer(
model_config.tokenizer,
tokenizer_mode=model_config.tokenizer_mode,
trust_remote_code=model_config.trust_remote_code)
trust_remote_code=model_config.trust_remote_code,
revision=model_config.revision)
self.seq_counter = Counter()
# Create the parallel GPU workers.