From 5b2943f5a6c5fb267d1b2029c666f9d6b0e4ebd6 Mon Sep 17 00:00:00 2001 From: WEI CHENG CHIU Date: Sat, 13 Jun 2026 14:01:35 +0800 Subject: [PATCH] [Bugfix] Return the tokenizer from maybe_make_thread_pool so it survives pickling (#45460) Signed-off-by: Wayne Chiu --- tests/tokenizers_/test_hf.py | 26 +++++++++++++++++++++++++- vllm/tokenizers/hf.py | 3 +++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/tests/tokenizers_/test_hf.py b/tests/tokenizers_/test_hf.py index c1238900ce..3ccbbd73e7 100644 --- a/tests/tokenizers_/test_hf.py +++ b/tests/tokenizers_/test_hf.py @@ -7,7 +7,11 @@ import pytest from transformers import AutoTokenizer from vllm.tokenizers import TokenizerLike -from vllm.tokenizers.hf import get_cached_tokenizer +from vllm.tokenizers.hf import ( + ThreadSafeHFTokenizerMixin, + get_cached_tokenizer, + maybe_make_thread_pool, +) @pytest.mark.parametrize("model_id", ["gpt2", "zai-org/chatglm3-6b"]) @@ -41,3 +45,23 @@ def _check_consistency(target: TokenizerLike, expected: TokenizerLike): ) assert target.encode("prompt") == expected.encode("prompt") + + +@pytest.mark.parametrize("model_id", ["gpt2"]) +def test_thread_pool_tokenizer_pickle(model_id: str): + """Regression test for issue #45433: the thread-pool tokenizer wrapper + reconstructs through maybe_make_thread_pool on unpickling, which used to + fall off the end and return None.""" + reference_tokenizer = AutoTokenizer.from_pretrained(model_id) + + pooled_tokenizer = maybe_make_thread_pool(deepcopy(reference_tokenizer)) + assert pooled_tokenizer is not None + assert isinstance(pooled_tokenizer, ThreadSafeHFTokenizerMixin) + + unpickled_tokenizer = pickle.loads(pickle.dumps(pooled_tokenizer)) + assert unpickled_tokenizer is not None + assert isinstance(unpickled_tokenizer, ThreadSafeHFTokenizerMixin) + assert unpickled_tokenizer.encode("prompt") == reference_tokenizer.encode("prompt") + + # Idempotence: wrapping an already-pooled tokenizer returns it unchanged. + assert maybe_make_thread_pool(pooled_tokenizer) is pooled_tokenizer diff --git a/vllm/tokenizers/hf.py b/vllm/tokenizers/hf.py index b4248e229a..45370bbb39 100644 --- a/vllm/tokenizers/hf.py +++ b/vllm/tokenizers/hf.py @@ -99,6 +99,9 @@ def maybe_make_thread_pool(tokenizer: _T, copies: int = 1): TokenizerPool.__name__ = f"TokenizerPool{og_tokenizer.__class__.__name__}" tokenizer.__class__ = TokenizerPool + # Return the tokenizer: TokenizerPool.__reduce__ reconstructs through this + # function, so falling off the end would unpickle to None (issue #45433). + return tokenizer def get_cached_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer: