From e562003adbb9c4eb177194d1318eb2e6862a8e15 Mon Sep 17 00:00:00 2001 From: Aditya Ganesh Kumar Date: Fri, 18 Sep 2026 19:46:41 +0800 Subject: [PATCH] [Tokenizer] Drop dead Mistral tokenizer shims for transformers#41962 (#56325) Signed-off-by: Aditya Ganesh Kumar Co-authored-by: Claude Opus 5 Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> --- vllm/tokenizers/mistral.py | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/vllm/tokenizers/mistral.py b/vllm/tokenizers/mistral.py index 66b69089f0..4ffefe0db6 100644 --- a/vllm/tokenizers/mistral.py +++ b/vllm/tokenizers/mistral.py @@ -354,22 +354,13 @@ class MistralTokenizer(TokenizerLike): "`text_pair` is not supported by `MistralTokenizer.__call__`." ) - encoded = self.transformers_tokenizer( + return self.transformers_tokenizer( text=text, text_pair=text_pair, add_special_tokens=add_special_tokens, truncation=truncation, max_length=max_length, ) - # TODO(juliendenize): once https://github.com/huggingface/transformers/pull/41962 - # is in, revert to only call self.transformers_tokenizer(...). - # Hack to fix wrongly added eos token, when fix will be supported the condition - # below will be False even before the revert is done. - if encoded["input_ids"] and encoded["input_ids"][-1] == self.eos_token_id: - encoded["input_ids"].pop(-1) - if attention_mask := encoded.get("attention_mask"): - attention_mask.pop(-1) - return encoded @property def vocab(self) -> list[str]: @@ -389,8 +380,10 @@ class MistralTokenizer(TokenizerLike): max_length: int | None = None, add_special_tokens: bool = True, ) -> list[int]: - # TODO(juliendenize): once https://github.com/huggingface/transformers/pull/41962 - # is in, directly call self.transformers_tokenizer.encode(...). + # NOTE: transformers guards truncation with `if max_length and ...`, so + # `max_length=0` returns the full sequence instead of an empty one. vLLM + # treats `truncate_prompt_tokens=0` as an empty prompt, so keep slicing + # here until that is fixed upstream. encoded = self.tokenizer.encode(text, bos=add_special_tokens, eos=False) if truncation is not False and max_length is not None: @@ -437,11 +430,6 @@ class MistralTokenizer(TokenizerLike): def decode( self, ids: Sequence[int] | int, skip_special_tokens: bool = False ) -> str: - # TODO(juliendenize): once https://github.com/huggingface/transformers/pull/41962 - # is in, directly call self.transformers_tokenizer.decode(...). - if isinstance(ids, int): - ids = [ids] - return self.transformers_tokenizer.decode( ids, skip_special_tokens=skip_special_tokens )