Change scheduler & input tensor shape (#1381)

This commit is contained in:
Woosuk Kwon
2023-10-16 17:48:42 -07:00
committed by GitHub
parent 651c614aa4
commit c1376e0f82
13 changed files with 180 additions and 178 deletions
+8 -12
View File
@@ -8,17 +8,17 @@ from vllm import activation_ops
class SiluAndMul(nn.Module):
"""An activation function for SwiGLU.
The function computes x -> silu(x[:d]) * x[d:] where d = x.shape[1] // 2.
The function computes x -> silu(x[:d]) * x[d:] where d = x.shape[-1] // 2.
Shapes:
x: (num_tokens, 2 * d)
return: (num_tokens, d)
x: (batch_size, seq_len, 2 * d) or (num_tokens, 2 * d)
return: (batch_size, seq_len, d) or (num_tokens, d)
"""
def forward(self, x: torch.Tensor) -> torch.Tensor:
num_tokens = x.shape[0]
d = x.shape[1] // 2
out = torch.empty(num_tokens, d, dtype=x.dtype, device=x.device)
d = x.shape[-1] // 2
output_shape = (x.shape[:-1] + (d, ))
out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
activation_ops.silu_and_mul(out, x)
return out
@@ -26,9 +26,7 @@ class SiluAndMul(nn.Module):
class NewGELU(nn.Module):
def forward(self, x: torch.Tensor) -> torch.Tensor:
num_tokens = x.shape[0]
d = x.shape[1]
out = torch.empty(num_tokens, d, dtype=x.dtype, device=x.device)
out = torch.empty_like(x)
activation_ops.gelu_new(out, x)
return out
@@ -36,9 +34,7 @@ class NewGELU(nn.Module):
class FastGELU(nn.Module):
def forward(self, x: torch.Tensor) -> torch.Tensor:
num_tokens = x.shape[0]
d = x.shape[1]
out = torch.empty(num_tokens, d, dtype=x.dtype, device=x.device)
out = torch.empty_like(x)
activation_ops.gelu_fast(out, x)
return out