[1/n][CI] Load models in CI from S3 instead of HF (#13205)
Signed-off-by: <> Co-authored-by: EC2 Default User <ec2-user@ip-172-31-20-117.us-west-2.compute.internal>
This commit is contained in:
co-authored by
EC2 Default User
parent
fd84857f64
commit
d5d214ac7f
@@ -46,6 +46,8 @@ def glob(s3=None,
|
||||
"""
|
||||
if s3 is None:
|
||||
s3 = boto3.client("s3")
|
||||
if not path.endswith("/"):
|
||||
path = path + "/"
|
||||
bucket_name, _, paths = list_files(s3,
|
||||
path=path,
|
||||
allow_pattern=allow_pattern)
|
||||
@@ -109,6 +111,7 @@ class S3Model:
|
||||
for sig in (signal.SIGINT, signal.SIGTERM):
|
||||
existing_handler = signal.getsignal(sig)
|
||||
signal.signal(sig, self._close_by_signal(existing_handler))
|
||||
|
||||
self.dir = tempfile.mkdtemp()
|
||||
|
||||
def __del__(self):
|
||||
@@ -140,6 +143,9 @@ class S3Model:
|
||||
ignore_pattern: A list of patterns of which files not to pull.
|
||||
|
||||
"""
|
||||
if not s3_model_path.endswith("/"):
|
||||
s3_model_path = s3_model_path + "/"
|
||||
|
||||
bucket_name, base_dir, files = list_files(self.s3, s3_model_path,
|
||||
allow_pattern,
|
||||
ignore_pattern)
|
||||
@@ -147,8 +153,9 @@ class S3Model:
|
||||
return
|
||||
|
||||
for file in files:
|
||||
destination_file = os.path.join(self.dir,
|
||||
file.removeprefix(base_dir))
|
||||
destination_file = os.path.join(
|
||||
self.dir,
|
||||
file.removeprefix(base_dir).lstrip("/"))
|
||||
local_dir = Path(destination_file).parent
|
||||
os.makedirs(local_dir, exist_ok=True)
|
||||
self.s3.download_file(bucket_name, file, destination_file)
|
||||
|
||||
Reference in New Issue
Block a user