[Misc] Update TokenizerLike interface and move get_cached_tokenizer (#29730)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2025-11-30 14:59:47 +08:00
committed by GitHub
parent 9381b5cde0
commit 2afcec4dec
15 changed files with 260 additions and 174 deletions
+3 -3
View File
@@ -356,8 +356,8 @@ class TestMistralTokenizer:
)
attn_mask = [1 for _ in range(len(token_ids))]
# Test 1: default
assert mistral_tokenizer("Hello world !") == {
# Test 1: no special tokens
assert mistral_tokenizer("Hello world !", add_special_tokens=False) == {
"attention_mask": attn_mask[1:],
"input_ids": token_ids[1:],
}
@@ -381,7 +381,7 @@ class TestMistralTokenizer:
"input_ids": token_ids,
}
# Test 5: empty string
assert mistral_tokenizer("") == {
assert mistral_tokenizer("", add_special_tokens=False) == {
"attention_mask": [],
"input_ids": [],
}