[Bugfix] Fix RuntimeError: Already borrowed by adding thread-safe Hugging Face fast-tokenizer wrappers (#41181)

Signed-off-by: Yifan Zong <yzong@redhat.com>
Co-authored-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
yzong-rh
2026-05-05 14:04:01 +00:00
committed by GitHub
co-authored by wang.yuqi
parent 6fca518157
commit 20dcd984f9
5 changed files with 107 additions and 35 deletions
+2
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .hf import maybe_make_thread_pool
from .protocol import TokenizerLike
from .registry import (
TokenizerRegistry,
@@ -15,4 +16,5 @@ __all__ = [
"cached_get_tokenizer",
"get_tokenizer",
"cached_tokenizer_from_config",
"maybe_make_thread_pool",
]
+88 -1
View File
@@ -2,8 +2,9 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import contextlib
import copy
import queue
from pathlib import Path
from typing import TypeAlias
from typing import TypeAlias, TypeVar
from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast
@@ -12,6 +13,92 @@ from vllm.transformers_utils.config import get_sentence_transformer_tokenizer_co
from .protocol import TokenizerLike
HfTokenizer: TypeAlias = PreTrainedTokenizer | PreTrainedTokenizerFast
_T = TypeVar("_T", bound=TokenizerLike)
class ThreadSafeHFTokenizerMixin:
"""Mixin class for thread-safe HF fast tokenizers."""
pass
def maybe_make_thread_pool(tokenizer: _T, copies: int = 1):
"""
If `tokenizer` is a `PreTrainedTokenizerFast`, modify the tokenizer
in-place to make the public interface thread-safe by routing calls
through a deep-copied tokenizer pool.
Note that:
- Only ``TokenizerLike``'s public interface is thread-safe.
This doesn't include ``_tokenizer`` property nor any mutation
methods like ``add_special_tokens`` or ``add_tokens``.
- Adjacent method calls could happen on different deep copies.
"""
if not isinstance(tokenizer, PreTrainedTokenizerFast) or isinstance(
tokenizer, ThreadSafeHFTokenizerMixin
):
return tokenizer
og_tokenizer = copy.copy(tokenizer)
tokenizer_pool: queue.Queue[PreTrainedTokenizerFast] = queue.Queue()
for _ in range(copies):
tokenizer_pool.put(copy.deepcopy(og_tokenizer))
@contextlib.contextmanager
def _borrow_from_pool():
try:
tok = tokenizer_pool.get_nowait()
yield tok
except queue.Empty:
tok = copy.deepcopy(og_tokenizer)
yield tok
finally:
tokenizer_pool.put(tok)
class TokenizerPool(tokenizer.__class__, ThreadSafeHFTokenizerMixin): # type: ignore
def apply_chat_template(self, *args, **kwargs):
with _borrow_from_pool() as tok:
return tok.apply_chat_template(*args, **kwargs)
def batch_decode(self, *args, **kwargs):
with _borrow_from_pool() as tok:
return tok.batch_decode(*args, **kwargs)
def batch_encode(self, *args, **kwargs):
with _borrow_from_pool() as tok:
return tok.batch_encode(*args, **kwargs)
def convert_tokens_to_ids(self, *args, **kwargs):
with _borrow_from_pool() as tok:
return tok.convert_tokens_to_ids(*args, **kwargs)
def convert_ids_to_tokens(self, *args, **kwargs):
with _borrow_from_pool() as tok:
return tok.convert_ids_to_tokens(*args, **kwargs)
def convert_tokens_to_string(self, *args, **kwargs):
with _borrow_from_pool() as tok:
return tok.convert_tokens_to_string(*args, **kwargs)
def decode(self, *args, **kwargs):
with _borrow_from_pool() as tok:
return tok.decode(*args, **kwargs)
def encode(self, *args, **kwargs):
with _borrow_from_pool() as tok:
return tok.encode(*args, **kwargs)
def __call__(self, *args, **kwargs):
with _borrow_from_pool() as tok:
return tok(*args, **kwargs)
def __reduce__(self):
return maybe_make_thread_pool, (og_tokenizer, copies)
TokenizerPool.__name__ = f"TokenizerPool{og_tokenizer.__class__.__name__}"
tokenizer.__class__ = TokenizerPool
def get_cached_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer: