[CI] Bump huggingface-hub from v1.10.2 to v1.22.0 (#47551)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -14,7 +14,7 @@ from vllm.tokenizers.hf import (
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["gpt2", "zai-org/chatglm3-6b"])
|
||||
@pytest.mark.parametrize("model_id", ["openai-community/gpt2", "zai-org/chatglm3-6b"])
|
||||
def test_cached_tokenizer(model_id: str):
|
||||
reference_tokenizer = AutoTokenizer.from_pretrained(
|
||||
model_id, trust_remote_code=True
|
||||
@@ -47,7 +47,7 @@ def _check_consistency(target: TokenizerLike, expected: TokenizerLike):
|
||||
assert target.encode("prompt") == expected.encode("prompt")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["gpt2"])
|
||||
@pytest.mark.parametrize("model_id", ["openai-community/gpt2"])
|
||||
def test_thread_pool_tokenizer_pickle(model_id: str):
|
||||
"""Regression test for issue #45433: the thread-pool tokenizer wrapper
|
||||
reconstructs through maybe_make_thread_pool on unpickling, which used to
|
||||
|
||||
Reference in New Issue
Block a user