[Bugfix] Fix RuntimeError: Already borrowed by adding thread-safe Hugging Face fast-tokenizer wrappers (#41181)
Signed-off-by: Yifan Zong <yzong@redhat.com> Co-authored-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from .hf import maybe_make_thread_pool
|
||||
from .protocol import TokenizerLike
|
||||
from .registry import (
|
||||
TokenizerRegistry,
|
||||
@@ -15,4 +16,5 @@ __all__ = [
|
||||
"cached_get_tokenizer",
|
||||
"get_tokenizer",
|
||||
"cached_tokenizer_from_config",
|
||||
"maybe_make_thread_pool",
|
||||
]
|
||||
|
||||
+88
-1
@@ -2,8 +2,9 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import contextlib
|
||||
import copy
|
||||
import queue
|
||||
from pathlib import Path
|
||||
from typing import TypeAlias
|
||||
from typing import TypeAlias, TypeVar
|
||||
|
||||
from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast
|
||||
|
||||
@@ -12,6 +13,92 @@ from vllm.transformers_utils.config import get_sentence_transformer_tokenizer_co
|
||||
from .protocol import TokenizerLike
|
||||
|
||||
HfTokenizer: TypeAlias = PreTrainedTokenizer | PreTrainedTokenizerFast
|
||||
_T = TypeVar("_T", bound=TokenizerLike)
|
||||
|
||||
|
||||
class ThreadSafeHFTokenizerMixin:
|
||||
"""Mixin class for thread-safe HF fast tokenizers."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
def maybe_make_thread_pool(tokenizer: _T, copies: int = 1):
|
||||
"""
|
||||
If `tokenizer` is a `PreTrainedTokenizerFast`, modify the tokenizer
|
||||
in-place to make the public interface thread-safe by routing calls
|
||||
through a deep-copied tokenizer pool.
|
||||
|
||||
Note that:
|
||||
- Only ``TokenizerLike``'s public interface is thread-safe.
|
||||
This doesn't include ``_tokenizer`` property nor any mutation
|
||||
methods like ``add_special_tokens`` or ``add_tokens``.
|
||||
- Adjacent method calls could happen on different deep copies.
|
||||
"""
|
||||
if not isinstance(tokenizer, PreTrainedTokenizerFast) or isinstance(
|
||||
tokenizer, ThreadSafeHFTokenizerMixin
|
||||
):
|
||||
return tokenizer
|
||||
|
||||
og_tokenizer = copy.copy(tokenizer)
|
||||
|
||||
tokenizer_pool: queue.Queue[PreTrainedTokenizerFast] = queue.Queue()
|
||||
for _ in range(copies):
|
||||
tokenizer_pool.put(copy.deepcopy(og_tokenizer))
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _borrow_from_pool():
|
||||
try:
|
||||
tok = tokenizer_pool.get_nowait()
|
||||
yield tok
|
||||
except queue.Empty:
|
||||
tok = copy.deepcopy(og_tokenizer)
|
||||
yield tok
|
||||
finally:
|
||||
tokenizer_pool.put(tok)
|
||||
|
||||
class TokenizerPool(tokenizer.__class__, ThreadSafeHFTokenizerMixin): # type: ignore
|
||||
def apply_chat_template(self, *args, **kwargs):
|
||||
with _borrow_from_pool() as tok:
|
||||
return tok.apply_chat_template(*args, **kwargs)
|
||||
|
||||
def batch_decode(self, *args, **kwargs):
|
||||
with _borrow_from_pool() as tok:
|
||||
return tok.batch_decode(*args, **kwargs)
|
||||
|
||||
def batch_encode(self, *args, **kwargs):
|
||||
with _borrow_from_pool() as tok:
|
||||
return tok.batch_encode(*args, **kwargs)
|
||||
|
||||
def convert_tokens_to_ids(self, *args, **kwargs):
|
||||
with _borrow_from_pool() as tok:
|
||||
return tok.convert_tokens_to_ids(*args, **kwargs)
|
||||
|
||||
def convert_ids_to_tokens(self, *args, **kwargs):
|
||||
with _borrow_from_pool() as tok:
|
||||
return tok.convert_ids_to_tokens(*args, **kwargs)
|
||||
|
||||
def convert_tokens_to_string(self, *args, **kwargs):
|
||||
with _borrow_from_pool() as tok:
|
||||
return tok.convert_tokens_to_string(*args, **kwargs)
|
||||
|
||||
def decode(self, *args, **kwargs):
|
||||
with _borrow_from_pool() as tok:
|
||||
return tok.decode(*args, **kwargs)
|
||||
|
||||
def encode(self, *args, **kwargs):
|
||||
with _borrow_from_pool() as tok:
|
||||
return tok.encode(*args, **kwargs)
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
with _borrow_from_pool() as tok:
|
||||
return tok(*args, **kwargs)
|
||||
|
||||
def __reduce__(self):
|
||||
return maybe_make_thread_pool, (og_tokenizer, copies)
|
||||
|
||||
TokenizerPool.__name__ = f"TokenizerPool{og_tokenizer.__class__.__name__}"
|
||||
|
||||
tokenizer.__class__ = TokenizerPool
|
||||
|
||||
|
||||
def get_cached_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
|
||||
|
||||
Reference in New Issue
Block a user