GPTBigCode (StarCoder, SantaCoder Support) (#209)

This commit is contained in:
Michael Feil
2023-06-23 01:49:27 +08:00
committed by GitHub
parent 83658c8ace
commit 298695b766
4 changed files with 298 additions and 2 deletions
+1
View File
@@ -8,6 +8,7 @@ _ACTIVATION_REGISTRY = {
"gelu": nn.GELU(),
"gelu_new": nn.GELU(approximate="tanh"), # NOTE: This may introduce small rounding errors.
"gelu_fast": nn.GELU(approximate="tanh"), # NOTE: This may introduce small rounding errors.
"gelu_pytorch_tanh": nn.GELU(approximate="tanh"), # NOTE: This may introduce small rounding errors.
"relu": nn.ReLU(),
}