tokenizer: Add fastokens support (#41741)
Signed-off-by: AlonKejzman <alonkeizman@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
co-authored by
Roger Wang
parent
003159d98b
commit
2a16ece2d3
@@ -83,7 +83,9 @@ logger = init_logger(__name__)
|
||||
RunnerOption = Literal["auto", RunnerType]
|
||||
ConvertType = Literal["none", "embed", "classify"]
|
||||
ConvertOption = Literal["auto", ConvertType]
|
||||
TokenizerMode = Literal["auto", "hf", "slow", "mistral", "deepseek_v32", "deepseek_v4"]
|
||||
TokenizerMode = Literal[
|
||||
"auto", "hf", "slow", "mistral", "deepseek_v32", "deepseek_v4", "fastokens"
|
||||
]
|
||||
ModelDType = Literal["auto", "half", "float16", "bfloat16", "float", "float32"]
|
||||
LogprobsMode = Literal[
|
||||
"raw_logits", "raw_logprobs", "processed_logits", "processed_logprobs"
|
||||
@@ -136,6 +138,9 @@ class ModelConfig:
|
||||
- "deepseek_v32" will always use the tokenizer from `deepseek_v32`.
|
||||
- "deepseek_v4" will always use the tokenizer from `deepseek_v4`.
|
||||
- "qwen_vl" will always use the tokenizer from `qwen_vl`.
|
||||
- "fastokens" loads a Hugging Face fast tokenizer powered by the
|
||||
[fastokens](https://github.com/crusoecloud/fastokens) Rust BPE backend
|
||||
(requires the `fastokens` package to be installed).
|
||||
- Other custom values can be supported via plugins."""
|
||||
trust_remote_code: bool = False
|
||||
"""Trust remote code (e.g., from HuggingFace) when downloading the model
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""``fastokens`` tokenizer mode.
|
||||
|
||||
Loads a Hugging Face fast tokenizer whose internal Rust tokenizer is replaced
|
||||
by the fastokens shim. fastokens also rebinds
|
||||
``tokenizers.decoders.DecodeStream`` so the streaming detokenizer accepts the
|
||||
shim. Both patches are installed for the lifetime of the process —
|
||||
``patch_transformers()`` is idempotent.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from .hf import CachedHfTokenizer, HfTokenizer
|
||||
from .protocol import TokenizerLike
|
||||
|
||||
|
||||
def _apply_fastokens_patch() -> None:
|
||||
try:
|
||||
import fastokens
|
||||
except ImportError as e:
|
||||
raise ImportError(
|
||||
"The 'fastokens' package is required for tokenizer_mode='fastokens'."
|
||||
) from e
|
||||
fastokens.patch_transformers()
|
||||
|
||||
|
||||
class FastokensTokenizer(TokenizerLike):
|
||||
@classmethod
|
||||
def from_pretrained(
|
||||
cls,
|
||||
path_or_repo_id: str | Path,
|
||||
*args,
|
||||
trust_remote_code: bool = False,
|
||||
revision: str | None = None,
|
||||
download_dir: str | None = None,
|
||||
**kwargs,
|
||||
) -> HfTokenizer:
|
||||
_apply_fastokens_patch()
|
||||
return CachedHfTokenizer.from_pretrained(
|
||||
path_or_repo_id,
|
||||
*args,
|
||||
trust_remote_code=trust_remote_code,
|
||||
revision=revision,
|
||||
download_dir=download_dir,
|
||||
**kwargs,
|
||||
)
|
||||
@@ -43,6 +43,7 @@ _MODEL_TYPES_WITH_INCORRECT_TOKENIZER_CLASS: set[str] = {"step3_vl"}
|
||||
_VLLM_TOKENIZERS = {
|
||||
"deepseek_v32": ("deepseek_v32", "DeepseekV32Tokenizer"),
|
||||
"deepseek_v4": ("deepseek_v4", "DeepseekV4Tokenizer"),
|
||||
"fastokens": ("fastokens", "FastokensTokenizer"),
|
||||
"grok2": ("grok2", "Grok2Tokenizer"),
|
||||
"hf": ("hf", "CachedHfTokenizer"),
|
||||
"kimi_audio": ("kimi_audio", "KimiAudioTokenizer"),
|
||||
|
||||
@@ -3,9 +3,9 @@
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
import tokenizers
|
||||
import tokenizers.decoders
|
||||
from packaging import version
|
||||
from tokenizers import Tokenizer
|
||||
from tokenizers.decoders import DecodeStream
|
||||
from transformers import PreTrainedTokenizerFast
|
||||
|
||||
from vllm.logger import init_logger
|
||||
@@ -177,7 +177,10 @@ class FastIncrementalDetokenizer(BaseIncrementalDetokenizer):
|
||||
self.tokenizer: Tokenizer = tokenizer._tokenizer
|
||||
|
||||
# Use native prefill to prime the decode stream with prompt tokens.
|
||||
self.stream = DecodeStream(
|
||||
# Look up DecodeStream on the module so backend patches (e.g. the
|
||||
# fastokens shim that replaces ``tokenizers.decoders.DecodeStream``)
|
||||
# are honored regardless of import order.
|
||||
self.stream = tokenizers.decoders.DecodeStream(
|
||||
ids=request.prompt_token_ids,
|
||||
skip_special_tokens=self.skip_special_tokens,
|
||||
)
|
||||
@@ -237,7 +240,9 @@ class FastIncrementalDetokenizer(BaseIncrementalDetokenizer):
|
||||
" for request %s, resetting decode stream.",
|
||||
self.request_id,
|
||||
)
|
||||
self.stream = DecodeStream(skip_special_tokens=self.skip_special_tokens)
|
||||
self.stream = tokenizers.decoders.DecodeStream(
|
||||
skip_special_tokens=self.skip_special_tokens
|
||||
)
|
||||
token = self.stream.step(self.tokenizer, next_token_id)
|
||||
return token
|
||||
|
||||
|
||||
Reference in New Issue
Block a user