[Qwen][Bugfix] Fixes sigmoid activation in torch impl of RMSNormGated. (#40245)
Signed-off-by: Tao He <linzhu.ht@alibaba-inc.com>
This commit is contained in:
@@ -478,9 +478,12 @@ class RMSNormGated(CustomOp):
|
||||
weight = self.weight.float()
|
||||
z = z.float() if z is not None else None
|
||||
|
||||
assert self.activation in ["silu", "sigmoid", "swish"]
|
||||
act_fn = F.sigmoid if self.activation == "sigmoid" else F.silu
|
||||
|
||||
# Apply gating before normalization if needed
|
||||
if z is not None and not self.norm_before_gate:
|
||||
x = x * F.silu(z)
|
||||
x = x * act_fn(z)
|
||||
|
||||
# RMS Normalization
|
||||
if self.group_size is None:
|
||||
@@ -499,7 +502,7 @@ class RMSNormGated(CustomOp):
|
||||
|
||||
# Apply gating after normalization if needed
|
||||
if z is not None and self.norm_before_gate:
|
||||
out = out * F.silu(z)
|
||||
out = out * act_fn(z)
|
||||
|
||||
return out.to(orig_dtype)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user