[CI] Don't try and download files that we already know don't exist (#46854)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-06-26 23:56:39 +00:00
committed by GitHub
parent 1d41009e81
commit 3f67477497
3 changed files with 55 additions and 9 deletions
@@ -7,9 +7,11 @@ from pathlib import Path
from unittest.mock import MagicMock, call, patch
import pytest
from huggingface_hub import _CACHED_NO_EXIST
from vllm.transformers_utils.repo_utils import (
any_pattern_in_repo_files,
get_hf_file_to_dict,
is_mistral_model_repo,
list_filtered_repo_files,
)
@@ -115,6 +117,33 @@ def test_one_filtered_repo_files(allow_patterns: list[str], expected_bool: bool)
)
@pytest.mark.parametrize(
("cache_result", "should_download"),
[
# HF Hub recorded a prior 404: don't re-probe the Hub.
(_CACHED_NO_EXIST, False),
# File not in cache and existence unknown: preserve download behavior.
(None, True),
],
)
def test_get_hf_file_to_dict_honors_no_exist_marker(
cache_result: object, should_download: bool
):
with (
patch(
"vllm.transformers_utils.repo_utils.try_to_load_from_cache",
MagicMock(return_value=cache_result),
),
patch(
"vllm.transformers_utils.repo_utils._try_download_from_hf_hub",
MagicMock(return_value=None),
) as mock_download,
):
result = get_hf_file_to_dict("processor_config.json", "some/repo")
assert result is None
assert mock_download.call_count == int(should_download)
@pytest.mark.parametrize(
("files", "expected_bool"),
[
+3 -3
View File
@@ -896,9 +896,9 @@ def get_sentence_transformer_tokenizer_config(
encoder_dict = None
for config_file in sentence_transformer_config_files:
if (
try_get_local_file(model=model, file_name=config_file, revision=revision)
is not None
if isinstance(
try_get_local_file(model=model, file_name=config_file, revision=revision),
Path,
):
encoder_dict = get_hf_file_to_dict(config_file, model, revision)
if encoder_dict:
+23 -6
View File
@@ -9,7 +9,7 @@ import time
from collections.abc import Callable
from functools import cache
from pathlib import Path
from typing import TypeVar
from typing import Any, TypeVar
import huggingface_hub
from huggingface_hub import HfApi, try_to_load_from_cache
@@ -218,8 +218,11 @@ def file_or_path_exists(
# NB: file_exists will only check for the existence of the config file on
# hf_hub. This will fail in offline mode.
# Call HF to check if the file exists
return file_exists(str(model), config_name, revision=revision)
if cached_filepath is None:
# The config file is not cached - check if it exists on hf_hub
return file_exists(str(model), config_name, revision=revision)
# The config file is known to not exist in cache - we can return False
return False
def get_model_path(model: str | Path, revision: str | None = None):
@@ -288,7 +291,7 @@ def get_hf_file_bytes(
if file_path is None:
file_path = _try_download_from_hf_hub(model, file_name, revision)
if file_path is not None and file_path.is_file():
if isinstance(file_path, Path) and file_path.is_file():
with open(file_path, "rb") as file:
return file.read()
@@ -297,7 +300,20 @@ def get_hf_file_bytes(
def try_get_local_file(
model: str | Path, file_name: str, revision: str | None = "main"
) -> Path | None:
) -> Path | Any | None:
"""
Try to get a local file from the HuggingFace repository.
The possible return values are:
- A `Path` object if the local file is found
- The `huggingface_hub._CACHED_NO_EXIST` sentinel if the file is known to not exist
- `None` if the file is not found and we cannot determine if it exists or not
Callers of this method should handle the `_CACHED_NO_EXIST` sentinel appropriately.
Checking if the return value `is not None` is not sufficient because it does not
distinguish between the file not existing and the file not being found.
"""
file_path = Path(model) / file_name
if file_path.is_file():
return file_path
@@ -308,6 +324,7 @@ def try_get_local_file(
)
if isinstance(cached_filepath, str):
return Path(cached_filepath)
return cached_filepath
except ValueError:
...
return None
@@ -335,7 +352,7 @@ def get_hf_file_to_dict(
if file_path is None:
file_path = _try_download_from_hf_hub(model, file_name, revision)
if file_path is not None and file_path.is_file():
if isinstance(file_path, Path) and file_path.is_file():
with open(file_path) as file:
return json.load(file)