[Bugfix] Allocate HY V3 expert_bias in float32 to prevent silent downcasting (#47797)
Signed-off-by: 辰言 <oncwnuIWp30GguOyJ615Fqj8H-yc@git.weixin.qq.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: 辰言 <oncwnuIWp30GguOyJ615Fqj8H-yc@git.weixin.qq.com> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
co-authored by
辰言
Jee Jee Li
Jee Jee Li
parent
e7b3853bac
commit
a1ab51afb6
@@ -177,7 +177,9 @@ class HYV3MoEFused(nn.Module):
|
||||
else:
|
||||
self.shared_mlp = None
|
||||
|
||||
self.expert_bias = nn.Parameter(torch.empty(config.num_experts))
|
||||
self.expert_bias = nn.Parameter(
|
||||
torch.empty(config.num_experts, dtype=torch.float32)
|
||||
)
|
||||
scoring_func = "sigmoid"
|
||||
e_score_correction_bias = self.expert_bias
|
||||
|
||||
|
||||
Reference in New Issue
Block a user