[ Misc ] Apply MoE Refactor to Deepseekv2 To Support Fp8 (#6417)

This commit is contained in:
Robert Shaw
2024-07-13 20:03:58 -07:00
committed by GitHub
parent eeceadaecc
commit fb6af8bc08
9 changed files with 222 additions and 136 deletions
+7 -25
View File
@@ -372,31 +372,13 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
("qkv_proj", "v_proj", "v"),
]
expert_params_mapping = [
# These are the weight scales for the experts
# (param_name, weight_name, expert_id, shard_id)
("experts.w13_scale"
if weight_name in ["w1", "w3"] else "experts.w2_scale",
f"experts.{expert_id}.{weight_name}.weight_scale", expert_id,
shard_id) for expert_id in range(self.config.num_local_experts)
for shard_id, weight_name in enumerate(["w1", "w2", "w3"])
] + [
# These are the weights for the experts
# (param_name, weight_name, expert_id)
("experts.w13_weight"
if weight_name in ["w1", "w3"] else "experts.w2_weight",
f"experts.{expert_id}.{weight_name}.weight", expert_id, shard_id)
for expert_id in range(self.config.num_local_experts)
for shard_id, weight_name in enumerate(["w1", "w2", "w3"])
] + [
# These are the activation scales for the experts
# (param_name, weight_name, expert_id)
("experts.a13_scale"
if weight_name in ["w1", "w3"] else "experts.a2_scale",
f"experts.{expert_id}.{weight_name}.input_scale", expert_id,
shard_id) for expert_id in range(self.config.num_local_experts)
for shard_id, weight_name in enumerate(["w1", "w2", "w3"])
]
# Params for weights, fp8 weight scales, fp8 activation scales
# (param_name, weight_name, expert_id, shard_id)
expert_params_mapping = FusedMoE.make_expert_params_mapping(
ckpt_gate_proj_name="w1",
ckpt_down_proj_name="w2",
ckpt_up_proj_name="w3",
num_experts=self.config.num_local_experts)
params_dict = dict(self.named_parameters())
for name, loaded_weight in weights: