Add batched RoPE kernel (#3095)

This commit is contained in:
Terry
2024-03-13 13:45:26 -07:00
committed by GitHub
parent ae0ccb4017
commit 7e9bd08f60
6 changed files with 421 additions and 41 deletions
+10
View File
@@ -53,6 +53,16 @@ void rotary_embedding(
torch::Tensor& cos_sin_cache,
bool is_neox);
void batched_rotary_embedding(
torch::Tensor& positions,
torch::Tensor& query,
torch::Tensor& key,
int head_size,
torch::Tensor& cos_sin_cache,
bool is_neox,
int rot_dim,
torch::Tensor& cos_sin_cache_offsets);
void silu_and_mul(
torch::Tensor& out,
torch::Tensor& input);