[Frontend] Remove the slow tokenizer mode (#58545)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-24 08:48:28 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 344fcc252a
commit d636456b1f
12 changed files with 11 additions and 39 deletions
+1 -1
View File
@@ -20,7 +20,7 @@ def fast_tokenizer():
"""gpt2 ships a Fast tokenizer; use it to test the offsets happy path."""
from transformers import AutoTokenizer
return AutoTokenizer.from_pretrained("openai-community/gpt2", use_fast=True)
return AutoTokenizer.from_pretrained("openai-community/gpt2")
def _make_base_renderer_with(tokenizer):
+1 -1
View File
@@ -23,7 +23,7 @@ def _assert_tokenizer_like(tokenizer: object):
def test_tokenizer_like_protocol():
tokenizer = get_tokenizer("openai-community/gpt2", use_fast=True)
tokenizer = get_tokenizer("openai-community/gpt2")
assert isinstance(tokenizer, TokenizersBackend)
_assert_tokenizer_like(tokenizer)
+5 -16
View File
@@ -69,21 +69,11 @@ def test_resolve_tokenizer_args_idempotent(runner_type):
)
@pytest.mark.parametrize(
("tokenizer_mode", "input_kwargs"),
[
("hf", {}),
("hf", {"mistral_format": False}),
("slow", {}),
("slow", {"mistral_format": False}),
],
)
def test_resolve_tokenizer_args_forces_hf_mistral_format_false(
tokenizer_mode, input_kwargs
):
@pytest.mark.parametrize("input_kwargs", [{}, {"mistral_format": False}])
def test_resolve_tokenizer_args_forces_hf_mistral_format_false(input_kwargs):
resolved_mode, _, _, kwargs = resolve_tokenizer_args(
"mistralai/Mistral-Nemo-Instruct-2407",
tokenizer_mode=tokenizer_mode,
tokenizer_mode="hf",
**input_kwargs,
)
@@ -91,15 +81,14 @@ def test_resolve_tokenizer_args_forces_hf_mistral_format_false(
assert kwargs["mistral_format"] is False
@pytest.mark.parametrize("tokenizer_mode", ["hf", "slow"])
def test_resolve_tokenizer_args_rejects_hf_mistral_format_true(tokenizer_mode):
def test_resolve_tokenizer_args_rejects_hf_mistral_format_true():
with pytest.raises(
ValueError,
match="mistral_format=True is not supported with tokenizer_mode='hf'",
):
resolve_tokenizer_args(
"mistralai/Mistral-Nemo-Instruct-2407",
tokenizer_mode=tokenizer_mode,
tokenizer_mode="hf",
mistral_format=True,
)