fix: correct load_weights track logic and enable weight integrity for… (#41811)

Signed-off-by: Yipeng Hu <i26268@metax-tech.com>
Signed-off-by: HuYiPeng <144002351+MynameFelix@users.noreply.github.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Yipeng Hu <i26268@metax-tech.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
HuYiPeng
2026-07-10 14:08:20 +00:00
committed by GitHub
co-authored by Yipeng Hu Isotr0py
parent b12cca6a23
commit 85c09e9885
2 changed files with 87 additions and 0 deletions
@@ -595,6 +595,14 @@ def filter_duplicate_safetensors_files(
weight_files_in_index = set()
for weight_name in weight_map:
weight_files_in_index.add(os.path.join(hf_folder, weight_map[weight_name]))
# Check if files referenced in model.safetensors.index.json actually exist.
# Raise error if any file is missing.
hf_weights_files_set = set(hf_weights_files)
missing_files = weight_files_in_index - hf_weights_files_set
if missing_files:
raise FileNotFoundError(
f"Weight files referenced in index but missing: {missing_files}"
)
# Filter out any fields that are not found in the index file.
hf_weights_files = [f for f in hf_weights_files if f in weight_files_in_index]
return hf_weights_files