[Frontend] Remove the slow tokenizer mode (#58545)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
344fcc252a
commit
d636456b1f
@@ -1695,7 +1695,6 @@ def add_cli_args(parser: FlexibleArgumentParser):
|
||||
- "auto" will use the tokenizer from `mistral_common` for Mistral models
|
||||
if available, otherwise it will use the "hf" tokenizer.\n
|
||||
- "hf" will use the fast tokenizer if available.\n
|
||||
- "slow" will always use the slow tokenizer.\n
|
||||
- "mistral" will always use the tokenizer from `mistral_common`.\n
|
||||
- "deepseek_v32" will always use the tokenizer from `deepseek_v32`.\n
|
||||
- Other custom values can be supported via plugins.""",
|
||||
|
||||
@@ -89,7 +89,6 @@ ConvertOption = Literal["auto", ConvertType]
|
||||
TokenizerMode = Literal[
|
||||
"auto",
|
||||
"hf",
|
||||
"slow",
|
||||
"mistral",
|
||||
"deepseek_v32",
|
||||
"deepseek_v4",
|
||||
@@ -148,7 +147,6 @@ class ModelConfig:
|
||||
- "auto" will use the tokenizer from `mistral_common` for Mistral models
|
||||
if available, otherwise it will use the "hf" tokenizer.
|
||||
- "hf" will use the fast tokenizer if available.
|
||||
- "slow" will always use the slow tokenizer.
|
||||
- "mistral" will always use the tokenizer from `mistral_common`.
|
||||
- "deepseek_v32" will always use the tokenizer from `deepseek_v32`.
|
||||
- "deepseek_v4" will always use the tokenizer from `deepseek_v4`.
|
||||
|
||||
@@ -76,8 +76,8 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
|
||||
Args:
|
||||
model: The name or path of a HuggingFace Transformers model.
|
||||
tokenizer: The name or path of a HuggingFace Transformers tokenizer.
|
||||
tokenizer_mode: The tokenizer mode. "auto" will use the fast tokenizer
|
||||
if available, and "slow" will always use the slow tokenizer.
|
||||
tokenizer_mode: The tokenizer mode. See
|
||||
[ModelConfig.tokenizer_mode][vllm.config.ModelConfig.tokenizer_mode].
|
||||
skip_tokenizer_init: If true, skip initialization of tokenizer and
|
||||
detokenizer. Expect valid prompt_token_ids and None for prompt
|
||||
from the input.
|
||||
|
||||
@@ -966,8 +966,7 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
)
|
||||
|
||||
def _can_produce_offsets(self) -> bool:
|
||||
# HF tokenizers may be slow (use_fast=False); only fast tokenizers
|
||||
# expose offset_mapping.
|
||||
# Only fast tokenizers expose offset_mapping.
|
||||
return self.tokenizer is not None and self.tokenizer.is_fast
|
||||
|
||||
def render_messages(
|
||||
|
||||
@@ -145,13 +145,6 @@ def resolve_tokenizer_args(
|
||||
else:
|
||||
assert_never(runner_type)
|
||||
|
||||
if tokenizer_mode == "slow":
|
||||
if kwargs.get("use_fast", False):
|
||||
raise ValueError("Cannot use the fast tokenizer in slow tokenizer mode.")
|
||||
|
||||
tokenizer_mode = "hf"
|
||||
kwargs["use_fast"] = False
|
||||
|
||||
# Try to use official Mistral tokenizer if possible
|
||||
if (
|
||||
tokenizer_mode == "auto"
|
||||
|
||||
@@ -209,7 +209,6 @@ class Moondream3Processor(ProcessorMixin):
|
||||
"revision",
|
||||
"subfolder",
|
||||
"token",
|
||||
"use_fast",
|
||||
):
|
||||
if key in kwargs:
|
||||
tokenizer_kwargs[key] = kwargs[key]
|
||||
|
||||
Reference in New Issue
Block a user