Remove KV cache scale boilerplate from model weight loading methods (#43167)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
7fe7800fa4
commit
62215e72c6
@@ -52,6 +52,7 @@ class WeightsMapper:
|
||||
def __or__(self, other: "WeightsMapper") -> "WeightsMapper":
|
||||
"""Combine two `WeightsMapper`s by merging their mappings."""
|
||||
return WeightsMapper(
|
||||
orig_to_new_regex={**self.orig_to_new_regex, **other.orig_to_new_regex},
|
||||
orig_to_new_substr={**self.orig_to_new_substr, **other.orig_to_new_substr},
|
||||
orig_to_new_prefix={**self.orig_to_new_prefix, **other.orig_to_new_prefix},
|
||||
orig_to_new_suffix={**self.orig_to_new_suffix, **other.orig_to_new_suffix},
|
||||
@@ -343,6 +344,20 @@ class AutoWeightsLoader:
|
||||
*,
|
||||
mapper: WeightsMapper | None = None,
|
||||
) -> set[str]:
|
||||
# Many models store quant_config in the base model instead of the causal model.
|
||||
# We look at the causal model's direct children for this reason.
|
||||
modules = (self.module, *self.module.children())
|
||||
iterator = (m.quant_config for m in modules if hasattr(m, "quant_config"))
|
||||
quant_config = next(iterator, None)
|
||||
cache_scale_mapper = (
|
||||
quant_config.get_cache_scale_mapper() if quant_config is not None else None
|
||||
)
|
||||
if cache_scale_mapper is not None:
|
||||
mapper = (
|
||||
mapper | cache_scale_mapper
|
||||
if mapper is not None
|
||||
else cache_scale_mapper
|
||||
)
|
||||
if mapper is not None:
|
||||
weights = mapper.apply(weights)
|
||||
# filter out weights with first-prefix/substr to skip in name
|
||||
|
||||
Reference in New Issue
Block a user