[1/n][CI] Load models in CI from S3 instead of HF (#13205)

Signed-off-by: <>
Co-authored-by: EC2 Default User <ec2-user@ip-172-31-20-117.us-west-2.compute.internal>
This commit is contained in:
Kevin H. Luu
2025-02-19 07:34:59 +00:00
committed by GitHub
co-authored by EC2 Default User
parent fd84857f64
commit d5d214ac7f
43 changed files with 225 additions and 76 deletions
+9 -2
View File
@@ -46,6 +46,8 @@ def glob(s3=None,
"""
if s3 is None:
s3 = boto3.client("s3")
if not path.endswith("/"):
path = path + "/"
bucket_name, _, paths = list_files(s3,
path=path,
allow_pattern=allow_pattern)
@@ -109,6 +111,7 @@ class S3Model:
for sig in (signal.SIGINT, signal.SIGTERM):
existing_handler = signal.getsignal(sig)
signal.signal(sig, self._close_by_signal(existing_handler))
self.dir = tempfile.mkdtemp()
def __del__(self):
@@ -140,6 +143,9 @@ class S3Model:
ignore_pattern: A list of patterns of which files not to pull.
"""
if not s3_model_path.endswith("/"):
s3_model_path = s3_model_path + "/"
bucket_name, base_dir, files = list_files(self.s3, s3_model_path,
allow_pattern,
ignore_pattern)
@@ -147,8 +153,9 @@ class S3Model:
return
for file in files:
destination_file = os.path.join(self.dir,
file.removeprefix(base_dir))
destination_file = os.path.join(
self.dir,
file.removeprefix(base_dir).lstrip("/"))
local_dir = Path(destination_file).parent
os.makedirs(local_dir, exist_ok=True)
self.s3.download_file(bucket_name, file, destination_file)