[Bugfix] Fix stale HPC QK-norm weights after weight refit (#56706)
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
This commit is contained in:
@@ -300,8 +300,11 @@ class HYV3Attention(nn.Module):
|
||||
prefix=f"{prefix}.attn",
|
||||
)
|
||||
if self.use_qk_norm:
|
||||
self.q_norm = RMSNorm(self.head_dim, rms_norm_eps)
|
||||
self.k_norm = RMSNorm(self.head_dim, rms_norm_eps)
|
||||
# The HPC fused kernel reads norm weights directly in float32;
|
||||
# the fallback path keeps the model default dtype.
|
||||
norm_dtype = torch.float32 if rope_support else None
|
||||
self.q_norm = RMSNorm(self.head_dim, rms_norm_eps, dtype=norm_dtype)
|
||||
self.k_norm = RMSNorm(self.head_dim, rms_norm_eps, dtype=norm_dtype)
|
||||
|
||||
# HPC fused RoPE + QK-Norm + KV-Cache-Write (+ optional FP8 Q quant).
|
||||
# HunYuan V3 applies QK-Norm *before* RoPE, so NORM_THEN_ROPE.
|
||||
|
||||
Reference in New Issue
Block a user