Signed-off-by: Bill Nell <bnell@redhat.com>
This commit is contained in:
@@ -635,36 +635,48 @@ class Step3p5Model(nn.Module):
|
||||
|
||||
# Old packed 3D format: .moe.gate_proj.weight [num_experts, out, in]
|
||||
expert_params_mapping = [
|
||||
(f".moe.experts.{base_layer}w13_weight", ".moe.gate_proj.weight", "w1"),
|
||||
(f".moe.experts.{base_layer}w13_weight", ".moe.up_proj.weight", "w3"),
|
||||
(f".moe.experts.{base_layer}w2_weight", ".moe.down_proj.weight", "w2"),
|
||||
(
|
||||
f".moe.experts.{base_layer}w13_weight_scale_2",
|
||||
f".moe.experts.routed_experts.{base_layer}w13_weight",
|
||||
".moe.gate_proj.weight",
|
||||
"w1",
|
||||
),
|
||||
(
|
||||
f".moe.experts.routed_experts.{base_layer}w13_weight",
|
||||
".moe.up_proj.weight",
|
||||
"w3",
|
||||
),
|
||||
(
|
||||
f".moe.experts.routed_experts.{base_layer}w2_weight",
|
||||
".moe.down_proj.weight",
|
||||
"w2",
|
||||
),
|
||||
(
|
||||
f".moe.experts.routed_experts.{base_layer}w13_weight_scale_2",
|
||||
".moe.gate_proj.weight_scale_2",
|
||||
"w1",
|
||||
),
|
||||
(
|
||||
f".moe.experts.{base_layer}w13_weight_scale_2",
|
||||
f".moe.experts.routed_experts.{base_layer}w13_weight_scale_2",
|
||||
".moe.up_proj.weight_scale_2",
|
||||
"w3",
|
||||
),
|
||||
(
|
||||
f".moe.experts.{base_layer}w2_weight_scale_2",
|
||||
f".moe.experts.routed_experts.{base_layer}w2_weight_scale_2",
|
||||
".moe.down_proj.weight_scale_2",
|
||||
"w2",
|
||||
),
|
||||
(
|
||||
f".moe.experts.{base_layer}w13_weight_scale",
|
||||
f".moe.experts.routed_experts.{base_layer}w13_weight_scale",
|
||||
".moe.gate_proj.weight_scale",
|
||||
"w1",
|
||||
),
|
||||
(
|
||||
f".moe.experts.{base_layer}w13_weight_scale",
|
||||
f".moe.experts.routed_experts.{base_layer}w13_weight_scale",
|
||||
".moe.up_proj.weight_scale",
|
||||
"w3",
|
||||
),
|
||||
(
|
||||
f".moe.experts.{base_layer}w2_weight_scale",
|
||||
f".moe.experts.routed_experts.{base_layer}w2_weight_scale",
|
||||
".moe.down_proj.weight_scale",
|
||||
"w2",
|
||||
),
|
||||
@@ -672,17 +684,17 @@ class Step3p5Model(nn.Module):
|
||||
# input scales are stored as moe.{gate,up,down}_proj.input_scale
|
||||
# rather than the standard per-expert format handled generically.
|
||||
(
|
||||
f".moe.experts.{base_layer}w13_input_scale",
|
||||
f".moe.experts.routed_experts.{base_layer}w13_input_scale",
|
||||
".moe.gate_proj.input_scale",
|
||||
"w1",
|
||||
),
|
||||
(
|
||||
f".moe.experts.{base_layer}w13_input_scale",
|
||||
f".moe.experts.routed_experts.{base_layer}w13_input_scale",
|
||||
".moe.up_proj.input_scale",
|
||||
"w3",
|
||||
),
|
||||
(
|
||||
f".moe.experts.{base_layer}w2_input_scale",
|
||||
f".moe.experts.routed_experts.{base_layer}w2_input_scale",
|
||||
".moe.down_proj.input_scale",
|
||||
"w2",
|
||||
),
|
||||
|
||||
Reference in New Issue
Block a user