[Frontend] Remove the slow tokenizer mode (#58545)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

benchmarks/backend_request_func.py

@@ -630,10 +630,6 @@ def get_tokenizer(
         pretrained_model_name_or_path
     ):
         pretrained_model_name_or_path = get_model(pretrained_model_name_or_path)
-    if tokenizer_mode == "slow":
-        if kwargs.get("use_fast", False):
-            raise ValueError("Cannot use the fast tokenizer in slow tokenizer mode.")
-        kwargs["use_fast"] = False
     if tokenizer_mode == "mistral":
         try:
             from vllm.tokenizers.mistral import MistralTokenizer

examples/generate/multimodal/vision_language_multi_image_offline.py

@@ -55,7 +55,6 @@ def load_aria(question: str, image_urls: list[str]) -> ModelRequestData:
     model_name = "rhymes-ai/Aria"
     engine_args = EngineArgs(
         model=model_name,
-        tokenizer_mode="slow",
         trust_remote_code=True,
         dtype="bfloat16",
         limit_mm_per_prompt={"image": len(image_urls)},

rust/src/bench/src/cli.rs

@@ -176,7 +176,7 @@ pub struct BenchServeArgs {
     #[arg(long)]
     pub tokenizer: Option<String>,
 
-    /// Tokenizer mode (auto, hf, slow, mistral). Accepted for Python CLI
+    /// Tokenizer mode (auto, hf, mistral). Accepted for Python CLI
     /// compatibility; non-auto values are ignored with a warning.
     #[arg(long, default_value = "auto")]
     pub tokenizer_mode: String,

tests/renderers/test_token_offsets.py

@@ -20,7 +20,7 @@ def fast_tokenizer():
     """gpt2 ships a Fast tokenizer; use it to test the offsets happy path."""
     from transformers import AutoTokenizer
 
-    return AutoTokenizer.from_pretrained("openai-community/gpt2", use_fast=True)
+    return AutoTokenizer.from_pretrained("openai-community/gpt2")
 
 
 def _make_base_renderer_with(tokenizer):

tests/tokenizers_/test_basic.py

@@ -23,7 +23,7 @@ def _assert_tokenizer_like(tokenizer: object):
 
 
 def test_tokenizer_like_protocol():
-    tokenizer = get_tokenizer("openai-community/gpt2", use_fast=True)
+    tokenizer = get_tokenizer("openai-community/gpt2")
     assert isinstance(tokenizer, TokenizersBackend)
     _assert_tokenizer_like(tokenizer)
 

tests/tokenizers_/test_registry.py

@@ -69,21 +69,11 @@ def test_resolve_tokenizer_args_idempotent(runner_type):
     )
 
 
-@pytest.mark.parametrize(
-    ("tokenizer_mode", "input_kwargs"),
-    [
-        ("hf", {}),
-        ("hf", {"mistral_format": False}),
-        ("slow", {}),
-        ("slow", {"mistral_format": False}),
-    ],
-)
-def test_resolve_tokenizer_args_forces_hf_mistral_format_false(
-    tokenizer_mode, input_kwargs
-):
+@pytest.mark.parametrize("input_kwargs", [{}, {"mistral_format": False}])
+def test_resolve_tokenizer_args_forces_hf_mistral_format_false(input_kwargs):
     resolved_mode, _, _, kwargs = resolve_tokenizer_args(
         "mistralai/Mistral-Nemo-Instruct-2407",
-        tokenizer_mode=tokenizer_mode,
+        tokenizer_mode="hf",
         **input_kwargs,
     )
 
@@ -91,15 +81,14 @@ def test_resolve_tokenizer_args_forces_hf_mistral_format_false(
     assert kwargs["mistral_format"] is False
 
 
-@pytest.mark.parametrize("tokenizer_mode", ["hf", "slow"])
-def test_resolve_tokenizer_args_rejects_hf_mistral_format_true(tokenizer_mode):
+def test_resolve_tokenizer_args_rejects_hf_mistral_format_true():
     with pytest.raises(
         ValueError,
         match="mistral_format=True is not supported with tokenizer_mode='hf'",
     ):
         resolve_tokenizer_args(
             "mistralai/Mistral-Nemo-Instruct-2407",
-            tokenizer_mode=tokenizer_mode,
+            tokenizer_mode="hf",
             mistral_format=True,
         )
 

vllm/benchmarks/serve.py

@@ -1695,7 +1695,6 @@ def add_cli_args(parser: FlexibleArgumentParser):
         - "auto" will use the tokenizer from `mistral_common` for Mistral models
         if available, otherwise it will use the "hf" tokenizer.\n
         - "hf" will use the fast tokenizer if available.\n
-        - "slow" will always use the slow tokenizer.\n
         - "mistral" will always use the tokenizer from `mistral_common`.\n
         - "deepseek_v32" will always use the tokenizer from `deepseek_v32`.\n
         - Other custom values can be supported via plugins.""",

vllm/config/model.py

@@ -89,7 +89,6 @@ ConvertOption = Literal["auto", ConvertType]
 TokenizerMode = Literal[
     "auto",
     "hf",
-    "slow",
     "mistral",
     "deepseek_v32",
     "deepseek_v4",
@@ -148,7 +147,6 @@ class ModelConfig:
     - "auto" will use the tokenizer from `mistral_common` for Mistral models
       if available, otherwise it will use the "hf" tokenizer.
     - "hf" will use the fast tokenizer if available.
-    - "slow" will always use the slow tokenizer.
     - "mistral" will always use the tokenizer from `mistral_common`.
     - "deepseek_v32" will always use the tokenizer from `deepseek_v32`.
     - "deepseek_v4" will always use the tokenizer from `deepseek_v4`.

vllm/entrypoints/llm.py

@@ -76,8 +76,8 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
     Args:
         model: The name or path of a HuggingFace Transformers model.
         tokenizer: The name or path of a HuggingFace Transformers tokenizer.
-        tokenizer_mode: The tokenizer mode. "auto" will use the fast tokenizer
-            if available, and "slow" will always use the slow tokenizer.
+        tokenizer_mode: The tokenizer mode. See
+            [ModelConfig.tokenizer_mode][vllm.config.ModelConfig.tokenizer_mode].
         skip_tokenizer_init: If true, skip initialization of tokenizer and
             detokenizer. Expect valid prompt_token_ids and None for prompt
             from the input.

vllm/renderers/hf.py

@@ -966,8 +966,7 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
             )
 
     def _can_produce_offsets(self) -> bool:
-        # HF tokenizers may be slow (use_fast=False); only fast tokenizers
-        # expose offset_mapping.
+        # Only fast tokenizers expose offset_mapping.
         return self.tokenizer is not None and self.tokenizer.is_fast
 
     def render_messages(

vllm/tokenizers/registry.py

@@ -145,13 +145,6 @@ def resolve_tokenizer_args(
         else:
             assert_never(runner_type)
 
-    if tokenizer_mode == "slow":
-        if kwargs.get("use_fast", False):
-            raise ValueError("Cannot use the fast tokenizer in slow tokenizer mode.")
-
-        tokenizer_mode = "hf"
-        kwargs["use_fast"] = False
-
     # Try to use official Mistral tokenizer if possible
     if (
         tokenizer_mode == "auto"

vllm/transformers_utils/processors/moondream3.py

@@ -209,7 +209,6 @@ class Moondream3Processor(ProcessorMixin):
             "revision",
             "subfolder",
             "token",
-            "use_fast",
         ):
             if key in kwargs:
                 tokenizer_kwargs[key] = kwargs[key]