fix: correct load_weights track logic and enable weight integrity for… (#41811)
Signed-off-by: Yipeng Hu <i26268@metax-tech.com> Signed-off-by: HuYiPeng <144002351+MynameFelix@users.noreply.github.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Yipeng Hu <i26268@metax-tech.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
co-authored by
Yipeng Hu
Isotr0py
parent
b12cca6a23
commit
85c09e9885
@@ -595,6 +595,14 @@ def filter_duplicate_safetensors_files(
|
||||
weight_files_in_index = set()
|
||||
for weight_name in weight_map:
|
||||
weight_files_in_index.add(os.path.join(hf_folder, weight_map[weight_name]))
|
||||
# Check if files referenced in model.safetensors.index.json actually exist.
|
||||
# Raise error if any file is missing.
|
||||
hf_weights_files_set = set(hf_weights_files)
|
||||
missing_files = weight_files_in_index - hf_weights_files_set
|
||||
if missing_files:
|
||||
raise FileNotFoundError(
|
||||
f"Weight files referenced in index but missing: {missing_files}"
|
||||
)
|
||||
# Filter out any fields that are not found in the index file.
|
||||
hf_weights_files = [f for f in hf_weights_files if f in weight_files_in_index]
|
||||
return hf_weights_files
|
||||
|
||||
Reference in New Issue
Block a user