[Frontend] Remove the slow tokenizer mode (#58545)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-24 08:48:28 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 344fcc252a
commit d636456b1f
12 changed files with 11 additions and 39 deletions
-1
View File
@@ -1695,7 +1695,6 @@ def add_cli_args(parser: FlexibleArgumentParser):
- "auto" will use the tokenizer from `mistral_common` for Mistral models
if available, otherwise it will use the "hf" tokenizer.\n
- "hf" will use the fast tokenizer if available.\n
- "slow" will always use the slow tokenizer.\n
- "mistral" will always use the tokenizer from `mistral_common`.\n
- "deepseek_v32" will always use the tokenizer from `deepseek_v32`.\n
- Other custom values can be supported via plugins.""",
-2
View File
@@ -89,7 +89,6 @@ ConvertOption = Literal["auto", ConvertType]
TokenizerMode = Literal[
"auto",
"hf",
"slow",
"mistral",
"deepseek_v32",
"deepseek_v4",
@@ -148,7 +147,6 @@ class ModelConfig:
- "auto" will use the tokenizer from `mistral_common` for Mistral models
if available, otherwise it will use the "hf" tokenizer.
- "hf" will use the fast tokenizer if available.
- "slow" will always use the slow tokenizer.
- "mistral" will always use the tokenizer from `mistral_common`.
- "deepseek_v32" will always use the tokenizer from `deepseek_v32`.
- "deepseek_v4" will always use the tokenizer from `deepseek_v4`.
+2 -2
View File
@@ -76,8 +76,8 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
Args:
model: The name or path of a HuggingFace Transformers model.
tokenizer: The name or path of a HuggingFace Transformers tokenizer.
tokenizer_mode: The tokenizer mode. "auto" will use the fast tokenizer
if available, and "slow" will always use the slow tokenizer.
tokenizer_mode: The tokenizer mode. See
[ModelConfig.tokenizer_mode][vllm.config.ModelConfig.tokenizer_mode].
skip_tokenizer_init: If true, skip initialization of tokenizer and
detokenizer. Expect valid prompt_token_ids and None for prompt
from the input.
+1 -2
View File
@@ -966,8 +966,7 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
)
def _can_produce_offsets(self) -> bool:
# HF tokenizers may be slow (use_fast=False); only fast tokenizers
# expose offset_mapping.
# Only fast tokenizers expose offset_mapping.
return self.tokenizer is not None and self.tokenizer.is_fast
def render_messages(
-7
View File
@@ -145,13 +145,6 @@ def resolve_tokenizer_args(
else:
assert_never(runner_type)
if tokenizer_mode == "slow":
if kwargs.get("use_fast", False):
raise ValueError("Cannot use the fast tokenizer in slow tokenizer mode.")
tokenizer_mode = "hf"
kwargs["use_fast"] = False
# Try to use official Mistral tokenizer if possible
if (
tokenizer_mode == "auto"
@@ -209,7 +209,6 @@ class Moondream3Processor(ProcessorMixin):
"revision",
"subfolder",
"token",
"use_fast",
):
if key in kwargs:
tokenizer_kwargs[key] = kwargs[key]