tokenizer: Add fastokens support (#41741)

Signed-off-by: AlonKejzman <alonkeizman@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
AlonKejzman
2026-05-07 22:49:42 +08:00
committed by GitHub
co-authored by Roger Wang
parent 003159d98b
commit 2a16ece2d3
6 changed files with 93 additions and 5 deletions
+6 -1
View File
@@ -83,7 +83,9 @@ logger = init_logger(__name__)
RunnerOption = Literal["auto", RunnerType]
ConvertType = Literal["none", "embed", "classify"]
ConvertOption = Literal["auto", ConvertType]
TokenizerMode = Literal["auto", "hf", "slow", "mistral", "deepseek_v32", "deepseek_v4"]
TokenizerMode = Literal[
"auto", "hf", "slow", "mistral", "deepseek_v32", "deepseek_v4", "fastokens"
]
ModelDType = Literal["auto", "half", "float16", "bfloat16", "float", "float32"]
LogprobsMode = Literal[
"raw_logits", "raw_logprobs", "processed_logits", "processed_logprobs"
@@ -136,6 +138,9 @@ class ModelConfig:
- "deepseek_v32" will always use the tokenizer from `deepseek_v32`.
- "deepseek_v4" will always use the tokenizer from `deepseek_v4`.
- "qwen_vl" will always use the tokenizer from `qwen_vl`.
- "fastokens" loads a Hugging Face fast tokenizer powered by the
[fastokens](https://github.com/crusoecloud/fastokens) Rust BPE backend
(requires the `fastokens` package to be installed).
- Other custom values can be supported via plugins."""
trust_remote_code: bool = False
"""Trust remote code (e.g., from HuggingFace) when downloading the model
+47
View File
@@ -0,0 +1,47 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""``fastokens`` tokenizer mode.
Loads a Hugging Face fast tokenizer whose internal Rust tokenizer is replaced
by the fastokens shim. fastokens also rebinds
``tokenizers.decoders.DecodeStream`` so the streaming detokenizer accepts the
shim. Both patches are installed for the lifetime of the process —
``patch_transformers()`` is idempotent.
"""
from pathlib import Path
from .hf import CachedHfTokenizer, HfTokenizer
from .protocol import TokenizerLike
def _apply_fastokens_patch() -> None:
try:
import fastokens
except ImportError as e:
raise ImportError(
"The 'fastokens' package is required for tokenizer_mode='fastokens'."
) from e
fastokens.patch_transformers()
class FastokensTokenizer(TokenizerLike):
@classmethod
def from_pretrained(
cls,
path_or_repo_id: str | Path,
*args,
trust_remote_code: bool = False,
revision: str | None = None,
download_dir: str | None = None,
**kwargs,
) -> HfTokenizer:
_apply_fastokens_patch()
return CachedHfTokenizer.from_pretrained(
path_or_repo_id,
*args,
trust_remote_code=trust_remote_code,
revision=revision,
download_dir=download_dir,
**kwargs,
)
+1
View File
@@ -43,6 +43,7 @@ _MODEL_TYPES_WITH_INCORRECT_TOKENIZER_CLASS: set[str] = {"step3_vl"}
_VLLM_TOKENIZERS = {
"deepseek_v32": ("deepseek_v32", "DeepseekV32Tokenizer"),
"deepseek_v4": ("deepseek_v4", "DeepseekV4Tokenizer"),
"fastokens": ("fastokens", "FastokensTokenizer"),
"grok2": ("grok2", "Grok2Tokenizer"),
"hf": ("hf", "CachedHfTokenizer"),
"kimi_audio": ("kimi_audio", "KimiAudioTokenizer"),
+8 -3
View File
@@ -3,9 +3,9 @@
from abc import ABC, abstractmethod
import tokenizers
import tokenizers.decoders
from packaging import version
from tokenizers import Tokenizer
from tokenizers.decoders import DecodeStream
from transformers import PreTrainedTokenizerFast
from vllm.logger import init_logger
@@ -177,7 +177,10 @@ class FastIncrementalDetokenizer(BaseIncrementalDetokenizer):
self.tokenizer: Tokenizer = tokenizer._tokenizer
# Use native prefill to prime the decode stream with prompt tokens.
self.stream = DecodeStream(
# Look up DecodeStream on the module so backend patches (e.g. the
# fastokens shim that replaces ``tokenizers.decoders.DecodeStream``)
# are honored regardless of import order.
self.stream = tokenizers.decoders.DecodeStream(
ids=request.prompt_token_ids,
skip_special_tokens=self.skip_special_tokens,
)
@@ -237,7 +240,9 @@ class FastIncrementalDetokenizer(BaseIncrementalDetokenizer):
" for request %s, resetting decode stream.",
self.request_id,
)
self.stream = DecodeStream(skip_special_tokens=self.skip_special_tokens)
self.stream = tokenizers.decoders.DecodeStream(
skip_special_tokens=self.skip_special_tokens
)
token = self.stream.step(self.tokenizer, next_token_id)
return token