Remove KV cache scale boilerplate from model weight loading methods (#43167)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-06-05 05:19:04 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 7fe7800fa4
commit 62215e72c6
56 changed files with 87 additions and 730 deletions
+15
View File
@@ -52,6 +52,7 @@ class WeightsMapper:
def __or__(self, other: "WeightsMapper") -> "WeightsMapper":
"""Combine two `WeightsMapper`s by merging their mappings."""
return WeightsMapper(
orig_to_new_regex={**self.orig_to_new_regex, **other.orig_to_new_regex},
orig_to_new_substr={**self.orig_to_new_substr, **other.orig_to_new_substr},
orig_to_new_prefix={**self.orig_to_new_prefix, **other.orig_to_new_prefix},
orig_to_new_suffix={**self.orig_to_new_suffix, **other.orig_to_new_suffix},
@@ -343,6 +344,20 @@ class AutoWeightsLoader:
*,
mapper: WeightsMapper | None = None,
) -> set[str]:
# Many models store quant_config in the base model instead of the causal model.
# We look at the causal model's direct children for this reason.
modules = (self.module, *self.module.children())
iterator = (m.quant_config for m in modules if hasattr(m, "quant_config"))
quant_config = next(iterator, None)
cache_scale_mapper = (
quant_config.get_cache_scale_mapper() if quant_config is not None else None
)
if cache_scale_mapper is not None:
mapper = (
mapper | cache_scale_mapper
if mapper is not None
else cache_scale_mapper
)
if mapper is not None:
weights = mapper.apply(weights)
# filter out weights with first-prefix/substr to skip in name