[Bugfix] fix wrong partial_rotary_factor calculation for bailing_moe model. (#43770)

Signed-off-by: zzt <zengzetang.zzt@antgroup.com>
Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com>
This commit is contained in:
zzt
2026-06-01 02:42:49 -07:00
committed by GitHub
co-authored by Jiangyun Zhu
parent de21863419
commit 8796838910
+6 -1
View File
@@ -130,7 +130,12 @@ class BailingAttention(nn.Module):
prefix=f"{prefix}.dense",
)
rotary_dim = getattr(config, "rotary_dim", self.head_dim)
rotary_dim = getattr(config, "rotary_dim", None)
if rotary_dim is None:
partial_rotary_factor = getattr(config, "partial_rotary_factor", 1.0)
rotary_dim = int(self.head_dim * partial_rotary_factor)
if rotary_dim is None:
rotary_dim = self.head_dim
config.rope_parameters["partial_rotary_factor"] = rotary_dim / self.head_dim
self.rotary_emb = get_rope(