[Bugfix] Honor code_revision in config-based tokenizer loading (#61611)
Signed-off-by: Yuan Li <dskwelmcy@163.com>
This commit is contained in:
@@ -174,6 +174,10 @@ def resolve_tokenizer_args(
|
||||
token=kwargs.get("token"),
|
||||
)
|
||||
|
||||
if tokenizer_mode == "mistral":
|
||||
# MistralCommonBackend rejects HF-only keyword arguments, even None.
|
||||
kwargs.pop("code_revision", None)
|
||||
|
||||
return tokenizer_mode, tokenizer_name, args, kwargs
|
||||
|
||||
|
||||
@@ -181,6 +185,10 @@ cached_resolve_tokenizer_args = lru_cache(resolve_tokenizer_args)
|
||||
|
||||
|
||||
def tokenizer_args_from_config(config: "ModelConfig", **kwargs):
|
||||
if config.tokenizer == config.model:
|
||||
code_revision = getattr(config, "code_revision", None)
|
||||
if code_revision is not None:
|
||||
kwargs.setdefault("code_revision", code_revision)
|
||||
return cached_resolve_tokenizer_args(
|
||||
config.tokenizer,
|
||||
runner_type=config.runner_type,
|
||||
@@ -241,6 +249,7 @@ def get_tokenizer(
|
||||
tokenizer_name,
|
||||
trust_remote_code=trust_remote_code,
|
||||
revision=revision,
|
||||
code_revision=kwargs.get("code_revision"),
|
||||
config_format=config_format,
|
||||
token=kwargs.get("token"),
|
||||
)
|
||||
@@ -288,6 +297,10 @@ def cached_tokenizer_from_config(model_config: "ModelConfig", **kwargs):
|
||||
return None
|
||||
|
||||
_maybe_register_hf_config(getattr(model_config, "hf_config", None))
|
||||
if model_config.tokenizer == model_config.model:
|
||||
code_revision = getattr(model_config, "code_revision", None)
|
||||
if code_revision is not None:
|
||||
kwargs.setdefault("code_revision", code_revision)
|
||||
|
||||
return cached_get_tokenizer(
|
||||
model_config.tokenizer,
|
||||
|
||||
Reference in New Issue
Block a user