[Kernel] Have rotary embeddings support tensors (#18046)
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
This commit is contained in:
+3
-16
@@ -254,14 +254,8 @@ def rotary_embedding(
|
||||
cos_sin_cache: torch.Tensor,
|
||||
is_neox: bool,
|
||||
) -> None:
|
||||
# TODO: Remove this contiguous call when the kernel is updated to support tensor slices
|
||||
query_contiguous = query.contiguous()
|
||||
key_contiguous = key.contiguous() if key is not None else None
|
||||
torch.ops._C.rotary_embedding(positions, query_contiguous, key_contiguous,
|
||||
head_size, cos_sin_cache, is_neox)
|
||||
query.copy_(query_contiguous)
|
||||
if key is not None:
|
||||
key.copy_(key_contiguous)
|
||||
torch.ops._C.rotary_embedding(positions, query, key, head_size,
|
||||
cos_sin_cache, is_neox)
|
||||
|
||||
|
||||
def batched_rotary_embedding(positions: torch.Tensor, query: torch.Tensor,
|
||||
@@ -269,16 +263,9 @@ def batched_rotary_embedding(positions: torch.Tensor, query: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor, is_neox: bool,
|
||||
rot_dim: int,
|
||||
cos_sin_cache_offsets: torch.Tensor) -> None:
|
||||
# TODO: Remove this contiguous call when the kernel is updated to support tensor slices
|
||||
query_contiguous = query.contiguous()
|
||||
key_contiguous = key.contiguous() if key is not None else None
|
||||
torch.ops._C.batched_rotary_embedding(positions, query_contiguous,
|
||||
key_contiguous, head_size,
|
||||
torch.ops._C.batched_rotary_embedding(positions, query, key, head_size,
|
||||
cos_sin_cache, is_neox, rot_dim,
|
||||
cos_sin_cache_offsets)
|
||||
query.copy_(query_contiguous)
|
||||
if key is not None:
|
||||
key.copy_(key_contiguous)
|
||||
|
||||
|
||||
# layer norm ops
|
||||
|
||||
Reference in New Issue
Block a user