[Core] Enable hf_transfer by default if available (#3817)

This commit is contained in:
Michael Feil
2024-04-04 04:02:43 +00:00
committed by GitHub
parent 294f8f6665
commit 537ee25f43
2 changed files with 42 additions and 0 deletions
+16
View File
@@ -8,6 +8,7 @@ from collections import defaultdict
from typing import Any, Iterable, Iterator, List, Optional, Tuple
import filelock
import huggingface_hub.constants
import numpy as np
import torch
from huggingface_hub import HfFileSystem, snapshot_download
@@ -30,6 +31,21 @@ temp_dir = os.environ.get('TMPDIR') or os.environ.get(
'TEMP') or os.environ.get('TMP') or "/tmp/"
def enable_hf_transfer():
"""automatically activates hf_transfer
"""
if "HF_HUB_ENABLE_HF_TRANSFER" not in os.environ:
try:
# enable hf hub transfer if available
import hf_transfer # type: ignore # noqa
huggingface_hub.constants.HF_HUB_ENABLE_HF_TRANSFER = True
except ImportError:
pass
enable_hf_transfer()
class Disabledtqdm(tqdm):
def __init__(self, *args, **kwargs):