[Bugfix][RL] Load attention sinks through weight_loader for layerwise reload (#57797)
Signed-off-by: Ronald1995 <ronaldautomobile@163.com> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
@@ -377,14 +377,6 @@ class MiMoV2MTP(nn.Module):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
# attention_sink_bias is head-parallel; slice by tp
|
||||
if "attention_sink_bias" in name:
|
||||
total_heads = loaded_weight.shape[0]
|
||||
heads_per_rank = total_heads // tp_size
|
||||
loaded_weight = loaded_weight.narrow(
|
||||
0, tp_rank * heads_per_rank, heads_per_rank
|
||||
)
|
||||
|
||||
direct_weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user