[Bugfix] Support eos_token_id from config.json (#5954)

This commit is contained in:
Cyrus Leung
2024-06-29 11:19:02 +00:00
committed by GitHub
parent 329df38f1a
commit 51e971d39e
3 changed files with 67 additions and 11 deletions
+23 -1
View File
@@ -1,7 +1,7 @@
import contextlib
from typing import Dict, Optional, Type
from transformers import PretrainedConfig
from transformers import GenerationConfig, PretrainedConfig
from vllm.envs import VLLM_USE_MODELSCOPE
from vllm.logger import init_logger
@@ -80,3 +80,25 @@ def get_hf_text_config(config: PretrainedConfig):
return config.text_config
else:
return config
def try_get_generation_config(
model: str,
trust_remote_code: bool,
revision: Optional[str] = None,
) -> Optional[GenerationConfig]:
try:
return GenerationConfig.from_pretrained(
model,
revision=revision,
)
except OSError: # Not found
try:
config = get_config(
model,
trust_remote_code=trust_remote_code,
revision=revision,
)
return GenerationConfig.from_model_config(config)
except OSError: # Not found
return None