[Frontend] Remove the slow tokenizer mode (#58545)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
344fcc252a
commit
d636456b1f
@@ -20,7 +20,7 @@ def fast_tokenizer():
|
||||
"""gpt2 ships a Fast tokenizer; use it to test the offsets happy path."""
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
return AutoTokenizer.from_pretrained("openai-community/gpt2", use_fast=True)
|
||||
return AutoTokenizer.from_pretrained("openai-community/gpt2")
|
||||
|
||||
|
||||
def _make_base_renderer_with(tokenizer):
|
||||
|
||||
@@ -23,7 +23,7 @@ def _assert_tokenizer_like(tokenizer: object):
|
||||
|
||||
|
||||
def test_tokenizer_like_protocol():
|
||||
tokenizer = get_tokenizer("openai-community/gpt2", use_fast=True)
|
||||
tokenizer = get_tokenizer("openai-community/gpt2")
|
||||
assert isinstance(tokenizer, TokenizersBackend)
|
||||
_assert_tokenizer_like(tokenizer)
|
||||
|
||||
|
||||
@@ -69,21 +69,11 @@ def test_resolve_tokenizer_args_idempotent(runner_type):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tokenizer_mode", "input_kwargs"),
|
||||
[
|
||||
("hf", {}),
|
||||
("hf", {"mistral_format": False}),
|
||||
("slow", {}),
|
||||
("slow", {"mistral_format": False}),
|
||||
],
|
||||
)
|
||||
def test_resolve_tokenizer_args_forces_hf_mistral_format_false(
|
||||
tokenizer_mode, input_kwargs
|
||||
):
|
||||
@pytest.mark.parametrize("input_kwargs", [{}, {"mistral_format": False}])
|
||||
def test_resolve_tokenizer_args_forces_hf_mistral_format_false(input_kwargs):
|
||||
resolved_mode, _, _, kwargs = resolve_tokenizer_args(
|
||||
"mistralai/Mistral-Nemo-Instruct-2407",
|
||||
tokenizer_mode=tokenizer_mode,
|
||||
tokenizer_mode="hf",
|
||||
**input_kwargs,
|
||||
)
|
||||
|
||||
@@ -91,15 +81,14 @@ def test_resolve_tokenizer_args_forces_hf_mistral_format_false(
|
||||
assert kwargs["mistral_format"] is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tokenizer_mode", ["hf", "slow"])
|
||||
def test_resolve_tokenizer_args_rejects_hf_mistral_format_true(tokenizer_mode):
|
||||
def test_resolve_tokenizer_args_rejects_hf_mistral_format_true():
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="mistral_format=True is not supported with tokenizer_mode='hf'",
|
||||
):
|
||||
resolve_tokenizer_args(
|
||||
"mistralai/Mistral-Nemo-Instruct-2407",
|
||||
tokenizer_mode=tokenizer_mode,
|
||||
tokenizer_mode="hf",
|
||||
mistral_format=True,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user