[Bugfix] LoRA: extend expert base_layer loading to Qwen3.5 and Step3.x (#37114)
Signed-off-by: Hollow Man <hollowman@opensuse.org>
This commit is contained in:
@@ -626,12 +626,15 @@ class Step3p5Model(nn.Module):
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
base_layer = (
|
||||
"base_layer." if any(".base_layer." in name for name in params_dict) else ""
|
||||
)
|
||||
|
||||
# Old packed 3D format: .moe.gate_proj.weight [num_experts, out, in]
|
||||
expert_params_mapping = [
|
||||
(".moe.experts.w13_weight", ".moe.gate_proj.weight", "w1"),
|
||||
(".moe.experts.w13_weight", ".moe.up_proj.weight", "w3"),
|
||||
(".moe.experts.w2_weight", ".moe.down_proj.weight", "w2"),
|
||||
(f".moe.experts.{base_layer}w13_weight", ".moe.gate_proj.weight", "w1"),
|
||||
(f".moe.experts.{base_layer}w13_weight", ".moe.up_proj.weight", "w3"),
|
||||
(f".moe.experts.{base_layer}w2_weight", ".moe.down_proj.weight", "w2"),
|
||||
]
|
||||
|
||||
# New per-expert format: .moe.experts.E.gate_proj.weight_packed [out, in]
|
||||
|
||||
Reference in New Issue
Block a user