[Tokenizer] Drop dead Mistral tokenizer shims for transformers#41962 (#56325)

Signed-off-by: Aditya Ganesh Kumar <adty910@gmail.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Aditya Ganesh Kumar
2026-09-18 12:46:41 +01:00
committed by GitHub
co-authored by Claude Opus 5 Harry Mellor
parent c58532c86b
commit e562003adb
+5 -17
View File
@@ -354,22 +354,13 @@ class MistralTokenizer(TokenizerLike):
"`text_pair` is not supported by `MistralTokenizer.__call__`."
)
encoded = self.transformers_tokenizer(
return self.transformers_tokenizer(
text=text,
text_pair=text_pair,
add_special_tokens=add_special_tokens,
truncation=truncation,
max_length=max_length,
)
# TODO(juliendenize): once https://github.com/huggingface/transformers/pull/41962
# is in, revert to only call self.transformers_tokenizer(...).
# Hack to fix wrongly added eos token, when fix will be supported the condition
# below will be False even before the revert is done.
if encoded["input_ids"] and encoded["input_ids"][-1] == self.eos_token_id:
encoded["input_ids"].pop(-1)
if attention_mask := encoded.get("attention_mask"):
attention_mask.pop(-1)
return encoded
@property
def vocab(self) -> list[str]:
@@ -389,8 +380,10 @@ class MistralTokenizer(TokenizerLike):
max_length: int | None = None,
add_special_tokens: bool = True,
) -> list[int]:
# TODO(juliendenize): once https://github.com/huggingface/transformers/pull/41962
# is in, directly call self.transformers_tokenizer.encode(...).
# NOTE: transformers guards truncation with `if max_length and ...`, so
# `max_length=0` returns the full sequence instead of an empty one. vLLM
# treats `truncate_prompt_tokens=0` as an empty prompt, so keep slicing
# here until that is fixed upstream.
encoded = self.tokenizer.encode(text, bos=add_special_tokens, eos=False)
if truncation is not False and max_length is not None:
@@ -437,11 +430,6 @@ class MistralTokenizer(TokenizerLike):
def decode(
self, ids: Sequence[int] | int, skip_special_tokens: bool = False
) -> str:
# TODO(juliendenize): once https://github.com/huggingface/transformers/pull/41962
# is in, directly call self.transformers_tokenizer.decode(...).
if isinstance(ids, int):
ids = [ids]
return self.transformers_tokenizer.decode(
ids, skip_special_tokens=skip_special_tokens
)