Add custom kernel for RMS normalization (#16)

This commit is contained in:
Woosuk Kwon
2023-04-01 00:51:22 +08:00
committed by GitHub
parent c45f3c3ab6
commit 09e9245478
9 changed files with 243 additions and 58 deletions
+26
View File
@@ -0,0 +1,26 @@
import torch
import torch.nn as nn
from cacheflow import layernorm_ops
class RMSNorm(nn.Module):
def __init__(
self,
hidden_size: int,
eps: float = 1e-6,
) -> None:
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden_size))
self.variance_epsilon = eps
def forward(self, x: torch.Tensor) -> torch.Tensor:
out = torch.empty_like(x)
layernorm_ops.rms_norm(
out,
x,
self.weight.data,
self.variance_epsilon,
)
return out