[CI] Don't try and download files that we already know don't exist (#46854)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -896,9 +896,9 @@ def get_sentence_transformer_tokenizer_config(
|
||||
encoder_dict = None
|
||||
|
||||
for config_file in sentence_transformer_config_files:
|
||||
if (
|
||||
try_get_local_file(model=model, file_name=config_file, revision=revision)
|
||||
is not None
|
||||
if isinstance(
|
||||
try_get_local_file(model=model, file_name=config_file, revision=revision),
|
||||
Path,
|
||||
):
|
||||
encoder_dict = get_hf_file_to_dict(config_file, model, revision)
|
||||
if encoder_dict:
|
||||
|
||||
@@ -9,7 +9,7 @@ import time
|
||||
from collections.abc import Callable
|
||||
from functools import cache
|
||||
from pathlib import Path
|
||||
from typing import TypeVar
|
||||
from typing import Any, TypeVar
|
||||
|
||||
import huggingface_hub
|
||||
from huggingface_hub import HfApi, try_to_load_from_cache
|
||||
@@ -218,8 +218,11 @@ def file_or_path_exists(
|
||||
# NB: file_exists will only check for the existence of the config file on
|
||||
# hf_hub. This will fail in offline mode.
|
||||
|
||||
# Call HF to check if the file exists
|
||||
return file_exists(str(model), config_name, revision=revision)
|
||||
if cached_filepath is None:
|
||||
# The config file is not cached - check if it exists on hf_hub
|
||||
return file_exists(str(model), config_name, revision=revision)
|
||||
# The config file is known to not exist in cache - we can return False
|
||||
return False
|
||||
|
||||
|
||||
def get_model_path(model: str | Path, revision: str | None = None):
|
||||
@@ -288,7 +291,7 @@ def get_hf_file_bytes(
|
||||
if file_path is None:
|
||||
file_path = _try_download_from_hf_hub(model, file_name, revision)
|
||||
|
||||
if file_path is not None and file_path.is_file():
|
||||
if isinstance(file_path, Path) and file_path.is_file():
|
||||
with open(file_path, "rb") as file:
|
||||
return file.read()
|
||||
|
||||
@@ -297,7 +300,20 @@ def get_hf_file_bytes(
|
||||
|
||||
def try_get_local_file(
|
||||
model: str | Path, file_name: str, revision: str | None = "main"
|
||||
) -> Path | None:
|
||||
) -> Path | Any | None:
|
||||
"""
|
||||
Try to get a local file from the HuggingFace repository.
|
||||
|
||||
The possible return values are:
|
||||
|
||||
- A `Path` object if the local file is found
|
||||
- The `huggingface_hub._CACHED_NO_EXIST` sentinel if the file is known to not exist
|
||||
- `None` if the file is not found and we cannot determine if it exists or not
|
||||
|
||||
Callers of this method should handle the `_CACHED_NO_EXIST` sentinel appropriately.
|
||||
Checking if the return value `is not None` is not sufficient because it does not
|
||||
distinguish between the file not existing and the file not being found.
|
||||
"""
|
||||
file_path = Path(model) / file_name
|
||||
if file_path.is_file():
|
||||
return file_path
|
||||
@@ -308,6 +324,7 @@ def try_get_local_file(
|
||||
)
|
||||
if isinstance(cached_filepath, str):
|
||||
return Path(cached_filepath)
|
||||
return cached_filepath
|
||||
except ValueError:
|
||||
...
|
||||
return None
|
||||
@@ -335,7 +352,7 @@ def get_hf_file_to_dict(
|
||||
if file_path is None:
|
||||
file_path = _try_download_from_hf_hub(model, file_name, revision)
|
||||
|
||||
if file_path is not None and file_path.is_file():
|
||||
if isinstance(file_path, Path) and file_path.is_file():
|
||||
with open(file_path) as file:
|
||||
return json.load(file)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user