[Bugfix] when loading weights skip empty expert bias if model does not support them (#50937)
Signed-off-by: walterbm <walter.beller.morales@gmail.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Harry Mellor
parent
ac70ce96e0
commit
70456e5e6f
@@ -936,7 +936,14 @@ class RoutedExperts(PluggableLayer):
|
||||
matched = True
|
||||
weight_name = qual_name.replace(weight_name, param_name)
|
||||
param_name = weight_name.removeprefix(f"{self.layer_name}.")
|
||||
param = getattr(self, param_name)
|
||||
param = getattr(self, param_name, None)
|
||||
if param is None:
|
||||
if param_name.endswith(("w13_bias", "w2_bias")):
|
||||
continue
|
||||
raise AttributeError(
|
||||
f"Layer {self.layer_name} has no parameter {param_name!r} "
|
||||
f"for checkpoint weight {qual_name!r}"
|
||||
)
|
||||
if is_fused:
|
||||
# w1 and w3 share one fused tensor; use a local copy so the
|
||||
# transpose below doesn't mutate loaded_weight across
|
||||
|
||||
Reference in New Issue
Block a user