[Bugfix] Return the tokenizer from maybe_make_thread_pool so it survives pickling (#45460)

Signed-off-by: Wayne Chiu <waynehacking8@gmail.com>
This commit is contained in:
WEI CHENG CHIU
2026-06-13 06:01:35 +00:00
committed by GitHub
parent 43f0e024bc
commit 5b2943f5a6
2 changed files with 28 additions and 1 deletions
+25 -1
View File
@@ -7,7 +7,11 @@ import pytest
from transformers import AutoTokenizer
from vllm.tokenizers import TokenizerLike
from vllm.tokenizers.hf import get_cached_tokenizer
from vllm.tokenizers.hf import (
ThreadSafeHFTokenizerMixin,
get_cached_tokenizer,
maybe_make_thread_pool,
)
@pytest.mark.parametrize("model_id", ["gpt2", "zai-org/chatglm3-6b"])
@@ -41,3 +45,23 @@ def _check_consistency(target: TokenizerLike, expected: TokenizerLike):
)
assert target.encode("prompt") == expected.encode("prompt")
@pytest.mark.parametrize("model_id", ["gpt2"])
def test_thread_pool_tokenizer_pickle(model_id: str):
"""Regression test for issue #45433: the thread-pool tokenizer wrapper
reconstructs through maybe_make_thread_pool on unpickling, which used to
fall off the end and return None."""
reference_tokenizer = AutoTokenizer.from_pretrained(model_id)
pooled_tokenizer = maybe_make_thread_pool(deepcopy(reference_tokenizer))
assert pooled_tokenizer is not None
assert isinstance(pooled_tokenizer, ThreadSafeHFTokenizerMixin)
unpickled_tokenizer = pickle.loads(pickle.dumps(pooled_tokenizer))
assert unpickled_tokenizer is not None
assert isinstance(unpickled_tokenizer, ThreadSafeHFTokenizerMixin)
assert unpickled_tokenizer.encode("prompt") == reference_tokenizer.encode("prompt")
# Idempotence: wrapping an already-pooled tokenizer returns it unchanged.
assert maybe_make_thread_pool(pooled_tokenizer) is pooled_tokenizer
+3
View File
@@ -99,6 +99,9 @@ def maybe_make_thread_pool(tokenizer: _T, copies: int = 1):
TokenizerPool.__name__ = f"TokenizerPool{og_tokenizer.__class__.__name__}"
tokenizer.__class__ = TokenizerPool
# Return the tokenizer: TokenizerPool.__reduce__ reconstructs through this
# function, so falling off the end would unpickle to None (issue #45433).
return tokenizer
def get_cached_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer: