[Tokenizer] Drop dead Mistral tokenizer shims for transformers#41962 (#56325)
Signed-off-by: Aditya Ganesh Kumar <adty910@gmail.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5
Harry Mellor
parent
c58532c86b
commit
e562003adb
@@ -354,22 +354,13 @@ class MistralTokenizer(TokenizerLike):
|
||||
"`text_pair` is not supported by `MistralTokenizer.__call__`."
|
||||
)
|
||||
|
||||
encoded = self.transformers_tokenizer(
|
||||
return self.transformers_tokenizer(
|
||||
text=text,
|
||||
text_pair=text_pair,
|
||||
add_special_tokens=add_special_tokens,
|
||||
truncation=truncation,
|
||||
max_length=max_length,
|
||||
)
|
||||
# TODO(juliendenize): once https://github.com/huggingface/transformers/pull/41962
|
||||
# is in, revert to only call self.transformers_tokenizer(...).
|
||||
# Hack to fix wrongly added eos token, when fix will be supported the condition
|
||||
# below will be False even before the revert is done.
|
||||
if encoded["input_ids"] and encoded["input_ids"][-1] == self.eos_token_id:
|
||||
encoded["input_ids"].pop(-1)
|
||||
if attention_mask := encoded.get("attention_mask"):
|
||||
attention_mask.pop(-1)
|
||||
return encoded
|
||||
|
||||
@property
|
||||
def vocab(self) -> list[str]:
|
||||
@@ -389,8 +380,10 @@ class MistralTokenizer(TokenizerLike):
|
||||
max_length: int | None = None,
|
||||
add_special_tokens: bool = True,
|
||||
) -> list[int]:
|
||||
# TODO(juliendenize): once https://github.com/huggingface/transformers/pull/41962
|
||||
# is in, directly call self.transformers_tokenizer.encode(...).
|
||||
# NOTE: transformers guards truncation with `if max_length and ...`, so
|
||||
# `max_length=0` returns the full sequence instead of an empty one. vLLM
|
||||
# treats `truncate_prompt_tokens=0` as an empty prompt, so keep slicing
|
||||
# here until that is fixed upstream.
|
||||
encoded = self.tokenizer.encode(text, bos=add_special_tokens, eos=False)
|
||||
|
||||
if truncation is not False and max_length is not None:
|
||||
@@ -437,11 +430,6 @@ class MistralTokenizer(TokenizerLike):
|
||||
def decode(
|
||||
self, ids: Sequence[int] | int, skip_special_tokens: bool = False
|
||||
) -> str:
|
||||
# TODO(juliendenize): once https://github.com/huggingface/transformers/pull/41962
|
||||
# is in, directly call self.transformers_tokenizer.decode(...).
|
||||
if isinstance(ids, int):
|
||||
ids = [ids]
|
||||
|
||||
return self.transformers_tokenizer.decode(
|
||||
ids, skip_special_tokens=skip_special_tokens
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user