[MM Encoder]: Make MMEncoderAttention's scale takes effect properly (#31950)

Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
Isotr0py
2026-01-08 02:33:48 -08:00
committed by GitHub
parent 5576227bc1
commit 2972a05473
11 changed files with 32 additions and 8 deletions
+1
View File
@@ -390,6 +390,7 @@ class MoonVitEncoderLayer(nn.Module):
self.attn = MMEncoderAttention(
num_heads=self.num_attention_heads_per_partition,
head_size=self.hidden_size_per_attention_head,
scale=self.hidden_size_per_attention_head**-0.5,
multimodal_config=multimodal_config,
prefix=f"{prefix}.attn",
)