[CI] Bump huggingface-hub from v1.10.2 to v1.22.0 (#47551)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -23,7 +23,7 @@ def _assert_tokenizer_like(tokenizer: object):
|
||||
|
||||
|
||||
def test_tokenizer_like_protocol():
|
||||
tokenizer = get_tokenizer("gpt2", use_fast=True)
|
||||
tokenizer = get_tokenizer("openai-community/gpt2", use_fast=True)
|
||||
assert isinstance(tokenizer, PreTrainedTokenizerFast)
|
||||
_assert_tokenizer_like(tokenizer)
|
||||
|
||||
@@ -43,7 +43,9 @@ def test_tokenizer_like_protocol():
|
||||
_assert_tokenizer_like(tokenizer)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tokenizer_name", ["facebook/opt-125m", "gpt2"])
|
||||
@pytest.mark.parametrize(
|
||||
"tokenizer_name", ["facebook/opt-125m", "openai-community/gpt2"]
|
||||
)
|
||||
def test_tokenizer_revision(tokenizer_name: str):
|
||||
# Assume that "main" branch always exists
|
||||
tokenizer = get_tokenizer(tokenizer_name, revision="main")
|
||||
|
||||
@@ -33,7 +33,7 @@ TRUTH = [
|
||||
|
||||
TOKENIZERS = [
|
||||
"facebook/opt-125m",
|
||||
"gpt2",
|
||||
"openai-community/gpt2",
|
||||
"bigcode/tiny_starcoder_py",
|
||||
"EleutherAI/gpt-j-6b",
|
||||
"EleutherAI/pythia-70m",
|
||||
|
||||
@@ -14,7 +14,7 @@ from vllm.tokenizers.hf import (
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["gpt2", "zai-org/chatglm3-6b"])
|
||||
@pytest.mark.parametrize("model_id", ["openai-community/gpt2", "zai-org/chatglm3-6b"])
|
||||
def test_cached_tokenizer(model_id: str):
|
||||
reference_tokenizer = AutoTokenizer.from_pretrained(
|
||||
model_id, trust_remote_code=True
|
||||
@@ -47,7 +47,7 @@ def _check_consistency(target: TokenizerLike, expected: TokenizerLike):
|
||||
assert target.encode("prompt") == expected.encode("prompt")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["gpt2"])
|
||||
@pytest.mark.parametrize("model_id", ["openai-community/gpt2"])
|
||||
def test_thread_pool_tokenizer_pickle(model_id: str):
|
||||
"""Regression test for issue #45433: the thread-pool tokenizer wrapper
|
||||
reconstructs through maybe_make_thread_pool on unpickling, which used to
|
||||
|
||||
Reference in New Issue
Block a user