[Bugfix] Return the tokenizer from maybe_make_thread_pool so it survives pickling (#45460)
Signed-off-by: Wayne Chiu <waynehacking8@gmail.com>
This commit is contained in:
@@ -7,7 +7,11 @@ import pytest
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tokenizers.hf import get_cached_tokenizer
|
||||
from vllm.tokenizers.hf import (
|
||||
ThreadSafeHFTokenizerMixin,
|
||||
get_cached_tokenizer,
|
||||
maybe_make_thread_pool,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["gpt2", "zai-org/chatglm3-6b"])
|
||||
@@ -41,3 +45,23 @@ def _check_consistency(target: TokenizerLike, expected: TokenizerLike):
|
||||
)
|
||||
|
||||
assert target.encode("prompt") == expected.encode("prompt")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["gpt2"])
|
||||
def test_thread_pool_tokenizer_pickle(model_id: str):
|
||||
"""Regression test for issue #45433: the thread-pool tokenizer wrapper
|
||||
reconstructs through maybe_make_thread_pool on unpickling, which used to
|
||||
fall off the end and return None."""
|
||||
reference_tokenizer = AutoTokenizer.from_pretrained(model_id)
|
||||
|
||||
pooled_tokenizer = maybe_make_thread_pool(deepcopy(reference_tokenizer))
|
||||
assert pooled_tokenizer is not None
|
||||
assert isinstance(pooled_tokenizer, ThreadSafeHFTokenizerMixin)
|
||||
|
||||
unpickled_tokenizer = pickle.loads(pickle.dumps(pooled_tokenizer))
|
||||
assert unpickled_tokenizer is not None
|
||||
assert isinstance(unpickled_tokenizer, ThreadSafeHFTokenizerMixin)
|
||||
assert unpickled_tokenizer.encode("prompt") == reference_tokenizer.encode("prompt")
|
||||
|
||||
# Idempotence: wrapping an already-pooled tokenizer returns it unchanged.
|
||||
assert maybe_make_thread_pool(pooled_tokenizer) is pooled_tokenizer
|
||||
|
||||
@@ -99,6 +99,9 @@ def maybe_make_thread_pool(tokenizer: _T, copies: int = 1):
|
||||
TokenizerPool.__name__ = f"TokenizerPool{og_tokenizer.__class__.__name__}"
|
||||
|
||||
tokenizer.__class__ = TokenizerPool
|
||||
# Return the tokenizer: TokenizerPool.__reduce__ reconstructs through this
|
||||
# function, so falling off the end would unpickle to None (issue #45433).
|
||||
return tokenizer
|
||||
|
||||
|
||||
def get_cached_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
|
||||
|
||||
Reference in New Issue
Block a user