[Bugfix] Honor code_revision in config-based tokenizer loading (#61611)

Signed-off-by: Yuan Li <dskwelmcy@163.com>
This commit is contained in:
dskwe
2026-10-16 05:21:12 +00:00
committed by GitHub
parent 4ac1f8d2ac
commit 49cb2bbf71
3 changed files with 277 additions and 3 deletions
+13
View File
@@ -174,6 +174,10 @@ def resolve_tokenizer_args(
token=kwargs.get("token"),
)
if tokenizer_mode == "mistral":
# MistralCommonBackend rejects HF-only keyword arguments, even None.
kwargs.pop("code_revision", None)
return tokenizer_mode, tokenizer_name, args, kwargs
@@ -181,6 +185,10 @@ cached_resolve_tokenizer_args = lru_cache(resolve_tokenizer_args)
def tokenizer_args_from_config(config: "ModelConfig", **kwargs):
if config.tokenizer == config.model:
code_revision = getattr(config, "code_revision", None)
if code_revision is not None:
kwargs.setdefault("code_revision", code_revision)
return cached_resolve_tokenizer_args(
config.tokenizer,
runner_type=config.runner_type,
@@ -241,6 +249,7 @@ def get_tokenizer(
tokenizer_name,
trust_remote_code=trust_remote_code,
revision=revision,
code_revision=kwargs.get("code_revision"),
config_format=config_format,
token=kwargs.get("token"),
)
@@ -288,6 +297,10 @@ def cached_tokenizer_from_config(model_config: "ModelConfig", **kwargs):
return None
_maybe_register_hf_config(getattr(model_config, "hf_config", None))
if model_config.tokenizer == model_config.model:
code_revision = getattr(model_config, "code_revision", None)
if code_revision is not None:
kwargs.setdefault("code_revision", code_revision)
return cached_get_tokenizer(
model_config.tokenizer,