[CI] Bump huggingface-hub from v1.10.2 to v1.22.0 (#47551)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -21,7 +21,7 @@ def _patch_hf_api(side_effect):
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def hf_tokenizer() -> PreTrainedTokenizerBase:
|
||||
return AutoTokenizer.from_pretrained("gpt2")
|
||||
return AutoTokenizer.from_pretrained("openai-community/gpt2")
|
||||
|
||||
|
||||
_FAKE_ROWS = {
|
||||
|
||||
@@ -12,7 +12,7 @@ from vllm.benchmarks.datasets import get_samples
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def hf_tokenizer() -> PreTrainedTokenizerBase:
|
||||
return AutoTokenizer.from_pretrained("gpt2")
|
||||
return AutoTokenizer.from_pretrained("openai-community/gpt2")
|
||||
|
||||
|
||||
def _write_jsonl(path: Path, n_rows: int) -> None:
|
||||
|
||||
@@ -17,7 +17,7 @@ from vllm.benchmarks.datasets import (
|
||||
@pytest.fixture(scope="session")
|
||||
def hf_tokenizer() -> PreTrainedTokenizerBase:
|
||||
# Use a small, commonly available tokenizer
|
||||
return AutoTokenizer.from_pretrained("gpt2")
|
||||
return AutoTokenizer.from_pretrained("openai-community/gpt2")
|
||||
|
||||
|
||||
class Params(NamedTuple):
|
||||
|
||||
@@ -16,7 +16,7 @@ from vllm.benchmarks.datasets import RandomMultiModalDataset, SampleRequest
|
||||
@pytest.fixture(scope="session")
|
||||
def hf_tokenizer() -> PreTrainedTokenizerBase:
|
||||
"""Use a small, commonly available tokenizer."""
|
||||
return AutoTokenizer.from_pretrained("gpt2")
|
||||
return AutoTokenizer.from_pretrained("openai-community/gpt2")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
||||
@@ -13,7 +13,7 @@ from vllm.benchmarks.datasets.create_txt_slices_dataset import create_txt_slices
|
||||
@pytest.fixture(scope="session")
|
||||
def hf_tokenizer() -> PreTrainedTokenizerBase:
|
||||
# Use a small, commonly available tokenizer
|
||||
return AutoTokenizer.from_pretrained("gpt2")
|
||||
return AutoTokenizer.from_pretrained("openai-community/gpt2")
|
||||
|
||||
|
||||
text_content = """
|
||||
@@ -39,7 +39,7 @@ def test_create_txt_slices_jsonl(
|
||||
create_txt_slices_jsonl(
|
||||
input_path=str(txt_path),
|
||||
output_path=str(jsonl_path),
|
||||
tokenizer_name="gpt2",
|
||||
tokenizer_name="openai-community/gpt2",
|
||||
num_prompts=10,
|
||||
input_len=10,
|
||||
output_len=10,
|
||||
|
||||
@@ -502,7 +502,7 @@ def test_gpt2_cache_hit(monkeypatch: pytest.MonkeyPatch):
|
||||
m.setenv("VLLM_USE_AOT_COMPILE", "1")
|
||||
# First compilation - initialize model and generate
|
||||
llm_model = LLM(
|
||||
model="gpt2",
|
||||
model="openai-community/gpt2",
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
),
|
||||
@@ -519,7 +519,7 @@ def test_gpt2_cache_hit(monkeypatch: pytest.MonkeyPatch):
|
||||
# Second compilation - should hit cache
|
||||
m.setenv("VLLM_FORCE_AOT_LOAD", "1")
|
||||
llm_model = LLM(
|
||||
model="gpt2",
|
||||
model="openai-community/gpt2",
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
),
|
||||
|
||||
@@ -24,7 +24,7 @@ from vllm.utils.torch_utils import is_torch_equal_or_newer
|
||||
def get_test_models():
|
||||
"""Get list of models to test based on PyTorch version"""
|
||||
models = [
|
||||
"gpt2",
|
||||
"openai-community/gpt2",
|
||||
"Qwen/Qwen2-7B-Instruct",
|
||||
"meta-llama/Llama-3.1-8B",
|
||||
]
|
||||
|
||||
@@ -114,7 +114,7 @@ TEXT_GENERATION_MODELS = {
|
||||
"tiiuae/falcon-7b": PPTestSettings.fast(),
|
||||
"google/gemma-1.1-2b-it": PPTestSettings.fast(),
|
||||
"google/gemma-2-9b": PPTestSettings.fast(),
|
||||
"gpt2": PPTestSettings.fast(),
|
||||
"openai-community/gpt2": PPTestSettings.fast(),
|
||||
"EleutherAI/gpt-j-6b": PPTestSettings.fast(),
|
||||
"EleutherAI/pythia-1.4b": PPTestSettings.fast(),
|
||||
"ibm/PowerLM-3b": PPTestSettings.fast(),
|
||||
|
||||
@@ -18,7 +18,7 @@ from vllm.renderers.embed_utils import safe_load_prompt_embeds
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_prompt():
|
||||
model_name = "gpt2"
|
||||
model_name = "openai-community/gpt2"
|
||||
server_args = ["--enforce-eager"]
|
||||
with RemoteOpenAIServer(model_name, server_args) as remote_server:
|
||||
client = remote_server.get_async_client()
|
||||
@@ -38,7 +38,7 @@ async def test_empty_prompt():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_out_of_vocab_token_ids():
|
||||
model_name = "gpt2"
|
||||
model_name = "openai-community/gpt2"
|
||||
server_args = ["--enforce-eager"]
|
||||
with RemoteOpenAIServer(model_name, server_args) as remote_server:
|
||||
client = remote_server.get_async_client()
|
||||
|
||||
@@ -115,6 +115,7 @@ def test_chatglm3_lora_tp4_fully_sharded_loras(chatglm3_lora_files):
|
||||
enable_lora=True,
|
||||
max_loras=2,
|
||||
max_lora_rank=64,
|
||||
max_num_seqs=16,
|
||||
tensor_parallel_size=4,
|
||||
trust_remote_code=True,
|
||||
fully_sharded_loras=True,
|
||||
|
||||
@@ -296,7 +296,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"GlmMoeDsaForCausalLM": _HfExamplesInfo(
|
||||
"zai-org/GLM-5", min_transformers_version="5.0.1", is_available_online=False
|
||||
),
|
||||
"GPT2LMHeadModel": _HfExamplesInfo("openai-community/gpt2", {"alias": "gpt2"}),
|
||||
"GPT2LMHeadModel": _HfExamplesInfo("openai-community/gpt2"),
|
||||
"GPTBigCodeForCausalLM": _HfExamplesInfo(
|
||||
"bigcode/starcoder",
|
||||
extras={
|
||||
|
||||
@@ -40,7 +40,7 @@ from vllm.renderers.hf import (
|
||||
# Qwen2TokenizerFast (SentencePiece BPE variant)
|
||||
# BertTokenizerFast (WordPiece)
|
||||
TOKENIZER_IDS: Final[list[str]] = [
|
||||
"gpt2",
|
||||
"openai-community/gpt2",
|
||||
"Qwen/Qwen2.5-1.5B-Instruct",
|
||||
"bert-base-uncased",
|
||||
]
|
||||
|
||||
@@ -23,7 +23,7 @@ def _assert_tokenizer_like(tokenizer: object):
|
||||
|
||||
|
||||
def test_tokenizer_like_protocol():
|
||||
tokenizer = get_tokenizer("gpt2", use_fast=True)
|
||||
tokenizer = get_tokenizer("openai-community/gpt2", use_fast=True)
|
||||
assert isinstance(tokenizer, PreTrainedTokenizerFast)
|
||||
_assert_tokenizer_like(tokenizer)
|
||||
|
||||
@@ -43,7 +43,9 @@ def test_tokenizer_like_protocol():
|
||||
_assert_tokenizer_like(tokenizer)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tokenizer_name", ["facebook/opt-125m", "gpt2"])
|
||||
@pytest.mark.parametrize(
|
||||
"tokenizer_name", ["facebook/opt-125m", "openai-community/gpt2"]
|
||||
)
|
||||
def test_tokenizer_revision(tokenizer_name: str):
|
||||
# Assume that "main" branch always exists
|
||||
tokenizer = get_tokenizer(tokenizer_name, revision="main")
|
||||
|
||||
@@ -33,7 +33,7 @@ TRUTH = [
|
||||
|
||||
TOKENIZERS = [
|
||||
"facebook/opt-125m",
|
||||
"gpt2",
|
||||
"openai-community/gpt2",
|
||||
"bigcode/tiny_starcoder_py",
|
||||
"EleutherAI/gpt-j-6b",
|
||||
"EleutherAI/pythia-70m",
|
||||
|
||||
@@ -14,7 +14,7 @@ from vllm.tokenizers.hf import (
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["gpt2", "zai-org/chatglm3-6b"])
|
||||
@pytest.mark.parametrize("model_id", ["openai-community/gpt2", "zai-org/chatglm3-6b"])
|
||||
def test_cached_tokenizer(model_id: str):
|
||||
reference_tokenizer = AutoTokenizer.from_pretrained(
|
||||
model_id, trust_remote_code=True
|
||||
@@ -47,7 +47,7 @@ def _check_consistency(target: TokenizerLike, expected: TokenizerLike):
|
||||
assert target.encode("prompt") == expected.encode("prompt")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["gpt2"])
|
||||
@pytest.mark.parametrize("model_id", ["openai-community/gpt2"])
|
||||
def test_thread_pool_tokenizer_pickle(model_id: str):
|
||||
"""Regression test for issue #45433: the thread-pool tokenizer wrapper
|
||||
reconstructs through maybe_make_thread_pool on unpickling, which used to
|
||||
|
||||
@@ -9,4 +9,4 @@ from vllm.tokenizers import TokenizerLike
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def default_tokenizer() -> TokenizerLike:
|
||||
return AutoTokenizer.from_pretrained("gpt2")
|
||||
return AutoTokenizer.from_pretrained("openai-community/gpt2")
|
||||
|
||||
@@ -19,7 +19,7 @@ from vllm.tool_parsers import ToolParser, ToolParserManager
|
||||
def default_tokenizer() -> TokenizerLike:
|
||||
"""Override module-scoped default_tokenizer because gigachat tests
|
||||
mutate the tokenizer via ``add_tokens``."""
|
||||
return AutoTokenizer.from_pretrained("gpt2")
|
||||
return AutoTokenizer.from_pretrained("openai-community/gpt2")
|
||||
|
||||
|
||||
MSG_SEP_TOKEN = "<|message_sep|>\n\n"
|
||||
|
||||
@@ -17,7 +17,7 @@ from vllm.v1.structured_output import StructuredOutputManager
|
||||
from vllm.v1.structured_output.backend_guidance import GuidanceBackend
|
||||
from vllm.v1.structured_output.backend_types import StructuredOutputOptions
|
||||
|
||||
TOKENIZER = "gpt2"
|
||||
TOKENIZER = "openai-community/gpt2"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
|
||||
Reference in New Issue
Block a user