[BugFix] Don't apply weight in batch-invariant RMSNorm when has_weight=False (#48741)
Signed-off-by: Josephasafg <ajgard7@gmail.com> Co-authored-by: Michael Gokhman <michael.gokhman@yahoo.com>
This commit is contained in:
co-authored by
Michael Gokhman
parent
2db39c7049
commit
df8a0900df
@@ -102,9 +102,12 @@ class RMSNorm(CustomOp):
|
||||
assert self.variance_size_override is None, (
|
||||
"Batch invariance is not supported for variance_size_override"
|
||||
)
|
||||
pass_weight = (
|
||||
self.pass_weight_add if residual is not None else self.pass_weight
|
||||
)
|
||||
return rms_norm_batch_invariant(
|
||||
x,
|
||||
self.weight.data,
|
||||
self.weight.data if pass_weight else None,
|
||||
self.variance_epsilon,
|
||||
residual=residual,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user