[Bugfix] Fix DeepSeek V4 mHC broadcast buffer for weight sync (#52626)

Signed-off-by: Hollow Man <hollowman@opensuse.org>
This commit is contained in:
ℍ𝕠𝕝𝕝𝕠𝕨 𝕄𝕒𝕟
2026-08-18 10:55:37 +08:00
committed by GitHub
parent f4b161d7fc
commit d5f5de7a7d
2 changed files with 66 additions and 1 deletions
+5 -1
View File
@@ -1372,11 +1372,15 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
return
layer = self.layers[self.start_layer]
if isinstance(layer, DeepseekV4DecoderLayer):
layer.hc_attn_fn_broadcast = (
broadcast = (
layer.hc_attn_fn.detach()
.view(-1, layer.hc_mult, layer.hidden_size)
.sum(dim=1)
)
if layer.hc_attn_fn_broadcast is None:
layer.hc_attn_fn_broadcast = broadcast
else:
layer.hc_attn_fn_broadcast.copy_(broadcast)
def _make_deepseek_v4_weights_mapper(expert_dtype: str) -> WeightsMapper: