[Bugfix] Preserve independent tokenizer code_revision in snapshot resolution (#61272)

Signed-off-by: Yuan Li <dskwelmcy@163.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
dskwe
2026-10-13 04:23:45 +00:00
committed by GitHub
co-authored by Cyrus Leung
parent 657a2e9417
commit 28d679e762
2 changed files with 92 additions and 6 deletions
+7 -6
View File
@@ -165,12 +165,13 @@ def resolve_tokenizer_args(
"mistral_format=True is not supported with tokenizer_mode='hf'"
)
kwargs["mistral_format"] = False
tokenizer_name = get_non_weight_snapshot_path(
str(tokenizer_name),
revision=revision,
cache_dir=download_dir,
token=kwargs.get("token"),
)
if kwargs.get("code_revision") in (None, revision):
tokenizer_name = get_non_weight_snapshot_path(
str(tokenizer_name),
revision=revision,
cache_dir=download_dir,
token=kwargs.get("token"),
)
return tokenizer_mode, tokenizer_name, args, kwargs