[ Misc ] Apply MoE Refactor to Deepseekv2 To Support Fp8 (#6417)
This commit is contained in:
@@ -372,31 +372,13 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
]
|
||||
|
||||
expert_params_mapping = [
|
||||
# These are the weight scales for the experts
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
("experts.w13_scale"
|
||||
if weight_name in ["w1", "w3"] else "experts.w2_scale",
|
||||
f"experts.{expert_id}.{weight_name}.weight_scale", expert_id,
|
||||
shard_id) for expert_id in range(self.config.num_local_experts)
|
||||
for shard_id, weight_name in enumerate(["w1", "w2", "w3"])
|
||||
] + [
|
||||
# These are the weights for the experts
|
||||
# (param_name, weight_name, expert_id)
|
||||
("experts.w13_weight"
|
||||
if weight_name in ["w1", "w3"] else "experts.w2_weight",
|
||||
f"experts.{expert_id}.{weight_name}.weight", expert_id, shard_id)
|
||||
for expert_id in range(self.config.num_local_experts)
|
||||
for shard_id, weight_name in enumerate(["w1", "w2", "w3"])
|
||||
] + [
|
||||
# These are the activation scales for the experts
|
||||
# (param_name, weight_name, expert_id)
|
||||
("experts.a13_scale"
|
||||
if weight_name in ["w1", "w3"] else "experts.a2_scale",
|
||||
f"experts.{expert_id}.{weight_name}.input_scale", expert_id,
|
||||
shard_id) for expert_id in range(self.config.num_local_experts)
|
||||
for shard_id, weight_name in enumerate(["w1", "w2", "w3"])
|
||||
]
|
||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
expert_params_mapping = FusedMoE.make_expert_params_mapping(
|
||||
ckpt_gate_proj_name="w1",
|
||||
ckpt_down_proj_name="w2",
|
||||
ckpt_up_proj_name="w3",
|
||||
num_experts=self.config.num_local_experts)
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
for name, loaded_weight in weights:
|
||||
|
||||
Reference in New Issue
Block a user