[Bugfix] Fix DeepSeek V4 mHC broadcast buffer for weight sync (#52626)
Signed-off-by: Hollow Man <hollowman@opensuse.org>
This commit is contained in:
@@ -1372,11 +1372,15 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
return
|
||||
layer = self.layers[self.start_layer]
|
||||
if isinstance(layer, DeepseekV4DecoderLayer):
|
||||
layer.hc_attn_fn_broadcast = (
|
||||
broadcast = (
|
||||
layer.hc_attn_fn.detach()
|
||||
.view(-1, layer.hc_mult, layer.hidden_size)
|
||||
.sum(dim=1)
|
||||
)
|
||||
if layer.hc_attn_fn_broadcast is None:
|
||||
layer.hc_attn_fn_broadcast = broadcast
|
||||
else:
|
||||
layer.hc_attn_fn_broadcast.copy_(broadcast)
|
||||
|
||||
|
||||
def _make_deepseek_v4_weights_mapper(expert_dtype: str) -> WeightsMapper:
|
||||
|
||||
Reference in New Issue
Block a user