Remove router weight upcast for DSv2-related models (#47970)
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
@@ -307,9 +307,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
self.gate = GateLinear(
|
||||
config.hidden_size,
|
||||
config.n_routed_experts,
|
||||
params_dtype=self.router_dtype,
|
||||
out_dtype=self.router_dtype,
|
||||
force_fp32_compute=self.router_dtype == torch.float32,
|
||||
prefix=f"{prefix}.gate",
|
||||
)
|
||||
if getattr(config, "topk_method", None) == "noaux_tc":
|
||||
|
||||
Reference in New Issue
Block a user